+++ /dev/null
-/* Copyright (c) 2011, Vsevolod Stakhov
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "config.h"
-#include "kvstorage.h"
-#include "main.h"
-#include "radix.h"
-#ifdef WITH_JUDY
-#include <Judy.h>
-#endif
-
-#define MAX_EXPIRE_STEPS 10
-
-/** Create new kv storage */
-struct rspamd_kv_storage *
-rspamd_kv_storage_new (gint id,
- const gchar *name,
- struct rspamd_kv_cache *cache,
- struct rspamd_kv_backend *backend,
- struct rspamd_kv_expire *expire,
- gsize max_elts,
- gsize max_memory,
- gboolean no_overwrite)
-{
- struct rspamd_kv_storage *new;
-
- new = g_slice_alloc (sizeof (struct rspamd_kv_storage));
- new->elts = 0;
- new->memory = 0;
-
- new->cache = cache;
- new->backend = backend;
- new->expire = expire;
-
- new->max_elts = max_elts;
- new->max_memory = max_memory;
-
- new->id = id;
-
- new->no_overwrite = no_overwrite;
-
- if (name != NULL) {
- new->name = g_strdup (name);
- }
- else {
- /* Name is absent, use ID as name */
- new->name = g_malloc (sizeof ("18446744073709551616"));
- rspamd_snprintf (new->name, sizeof ("18446744073709551616"), "%d", id);
- }
-#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 30))
- g_rw_lock_init (&new->rwlock);
-#else
- g_static_rw_lock_init (&new->rwlock);
-#endif
-
-
- /* Init structures */
- if (new->cache->init_func) {
- new->cache->init_func (new->cache);
- }
- if (new->backend && new->backend->init_func) {
- new->backend->init_func (new->backend);
- }
- if (new->expire && new->expire->init_func) {
- new->expire->init_func (new->expire);
- }
-
- return new;
-}
-
-/** Internal insertion to the kv storage from backend */
-gboolean
-rspamd_kv_storage_insert_cache (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- gpointer data,
- gsize len,
- gint flags,
- guint expire,
- struct rspamd_kv_element **pelt)
-{
- gint steps = 0;
- struct rspamd_kv_element *elt;
-
- RW_W_LOCK (&storage->rwlock);
- /* Hard limit */
- if (storage->max_memory > 0) {
- if (len > storage->max_memory) {
- msg_info (
- "<%s>: trying to insert value of length %z while limit is %z",
- storage->name,
- len,
- storage->max_memory);
- RW_W_UNLOCK (&storage->rwlock);
- return FALSE;
- }
-
- /* Now check limits */
- while (storage->memory + len > storage->max_memory) {
- if (storage->expire) {
- storage->expire->step_func (storage->expire, storage, time (
- NULL), steps);
- }
- else {
- msg_warn (
- "<%s>: storage is full and no expire function is defined",
- storage->name);
- }
- if (++steps > MAX_EXPIRE_STEPS) {
- RW_W_UNLOCK (&storage->rwlock);
- msg_warn ("<%s>: cannot expire enough keys in storage",
- storage->name);
- return FALSE;
- }
- }
- }
-
- /* Insert elt to the cache */
-
- elt = storage->cache->insert_func (storage->cache, key, keylen, data, len);
-
-
- /* Copy data */
- elt->flags = flags;
- elt->expire = expire;
-
- if (pelt != NULL) {
- *pelt = elt;
- }
-
- /* Insert to the expire */
- if (storage->expire) {
- storage->expire->insert_func (storage->expire, elt);
- }
-
- storage->elts++;
- storage->memory += ELT_SIZE (elt);
- RW_W_UNLOCK (&storage->rwlock);
-
- return TRUE;
-}
-
-/** Insert new element to the kv storage */
-gboolean
-rspamd_kv_storage_insert (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- gpointer data,
- gsize len,
- gint flags,
- guint expire)
-{
- gint steps = 0;
- struct rspamd_kv_element *elt;
- gboolean res = TRUE;
- glong longval;
-
- /* Hard limit */
- RW_W_LOCK (&storage->rwlock);
- if (storage->max_memory > 0) {
- if (len + sizeof (struct rspamd_kv_element) + keylen >=
- storage->max_memory) {
- msg_warn (
- "<%s>: trying to insert value of length %z while limit is %z",
- storage->name,
- len,
- storage->max_memory);
- RW_W_UNLOCK (&storage->rwlock);
- return FALSE;
- }
-
- /* Now check limits */
- while (storage->memory + len + keylen > storage->max_memory) {
- if (storage->expire) {
- storage->expire->step_func (storage->expire, storage, time (
- NULL), steps);
- }
- else {
- msg_warn (
- "<%s>: storage is full and no expire function is defined",
- storage->name);
- }
- if (++steps > MAX_EXPIRE_STEPS) {
- RW_W_UNLOCK (&storage->rwlock);
- msg_warn ("<%s>: cannot expire enough keys in storage",
- storage->name);
- return FALSE;
- }
- }
- }
- if (storage->max_elts > 0 && storage->elts > storage->max_elts) {
- /* More expire */
- steps = 0;
- while (storage->elts > storage->max_elts) {
- if (storage->expire) {
- storage->expire->step_func (storage->expire, storage, time (
- NULL), steps);
- }
- else {
- msg_warn (
- "<%s>: storage is full and no expire function is defined",
- storage->name);
- }
- if (++steps > MAX_EXPIRE_STEPS) {
- RW_W_UNLOCK (&storage->rwlock);
- msg_warn ("<%s>: cannot expire enough keys in storage",
- storage->name);
- return FALSE;
- }
- }
- }
-
- /* First try to search it in cache */
-
- elt = storage->cache->lookup_func (storage->cache, key, keylen);
- if (elt) {
- if (!storage->no_overwrite) {
- /* Remove old elt */
- if (storage->expire) {
- storage->expire->delete_func (storage->expire, elt);
- }
- storage->memory -= ELT_SIZE (elt);
- storage->cache->steal_func (storage->cache, elt);
- if (elt->flags & KV_ELT_DIRTY) {
- /* Element is in backend storage queue */
- elt->flags |= KV_ELT_NEED_FREE;
- }
- else {
- g_slice_free1 (ELT_SIZE (elt), elt);
- }
- }
- else {
- /* Just do incref and nothing more */
- if (storage->backend && storage->backend->incref_func) {
- if (storage->backend->incref_func (storage->backend, key,
- keylen)) {
- RW_W_UNLOCK (&storage->rwlock);
- return TRUE;
- }
- else {
- RW_W_UNLOCK (&storage->rwlock);
- return FALSE;
- }
- }
- }
- }
-
- /* Insert elt to the cache */
-
- /* First of all check element for integer */
- if (rspamd_strtol (data, len, &longval)) {
- elt = storage->cache->insert_func (storage->cache,
- key,
- keylen,
- &longval,
- sizeof (glong));
- if (elt == NULL) {
- return FALSE;
- }
- else {
- elt->flags |= KV_ELT_INTEGER;
- }
- }
- else {
- elt = storage->cache->insert_func (storage->cache,
- key,
- keylen,
- data,
- len);
- if (elt == NULL) {
- RW_W_UNLOCK (&storage->rwlock);
- return FALSE;
- }
- }
-
- elt->flags |= flags;
- elt->expire = expire;
- if (expire == 0) {
- elt->flags |= KV_ELT_PERSISTENT;
- }
-
- /* Place to the backend */
- if (storage->backend) {
- res =
- storage->backend->insert_func (storage->backend, key, keylen, elt);
- }
-
- /* Insert to the expire */
- if (storage->expire) {
- storage->expire->insert_func (storage->expire, elt);
- }
-
- storage->elts++;
- storage->memory += ELT_SIZE (elt);
- RW_W_UNLOCK (&storage->rwlock);
-
- return res;
-}
-
-/** Replace an element in the kv storage */
-gboolean
-rspamd_kv_storage_replace (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- struct rspamd_kv_element *elt)
-{
- gboolean res = TRUE;
- gint steps = 0;
-
- /* Hard limit */
- if (storage->max_memory > 0) {
- if (elt->size > storage->max_memory) {
- msg_info (
- "<%s>: trying to replace value of length %z while limit is %z",
- storage->name,
- elt->size,
- storage->max_memory);
- return FALSE;
- }
-
- /* Now check limits */
- while (storage->memory + ELT_SIZE (elt) > storage->max_memory) {
- if (storage->expire) {
- RW_W_LOCK (&storage->rwlock);
- storage->expire->step_func (storage->expire, storage, time (
- NULL), steps);
- RW_W_UNLOCK (&storage->rwlock);
- }
- else {
- msg_warn (
- "<%s>: storage is full and no expire function is defined",
- storage->name);
- }
- if (++steps > MAX_EXPIRE_STEPS) {
- msg_warn ("<%s>: cannot expire enough keys in storage",
- storage->name);
- return FALSE;
- }
- }
- }
-
- RW_W_LOCK (&storage->rwlock);
- /* Insert elt to the cache */
- res = storage->cache->replace_func (storage->cache, key, keylen, elt);
-
- /* Place to the backend */
- if (res && storage->backend) {
- res =
- storage->backend->replace_func (storage->backend, key, keylen, elt);
- }
- RW_W_UNLOCK (&storage->rwlock);
-
- return res;
-}
-
-/** Increment value in kvstorage */
-gboolean
-rspamd_kv_storage_increment (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- glong *value)
-{
- struct rspamd_kv_element *elt = NULL, *belt;
- glong *lp;
-
- /* First try to look at cache */
- RW_W_LOCK (&storage->rwlock);
- elt = storage->cache->lookup_func (storage->cache, key, keylen);
-
- if (elt == NULL && storage->backend) {
- belt = storage->backend->lookup_func (storage->backend, key, keylen);
- if (belt) {
- /* Put this element into cache */
- if ((belt->flags & KV_ELT_INTEGER) != 0) {
- RW_W_UNLOCK (&storage->rwlock);
- rspamd_kv_storage_insert_cache (storage, ELT_KEY (
- belt), keylen, ELT_DATA (belt),
- belt->size, belt->flags,
- belt->expire, &elt);
- RW_W_LOCK (&storage->rwlock);
- }
- if ((belt->flags & KV_ELT_DIRTY) == 0) {
- g_free (belt);
- }
- }
- }
- if (elt && (elt->flags & KV_ELT_INTEGER) != 0) {
- lp = &ELT_LONG (elt);
- /* Handle need expire here */
- if (elt->flags & KV_ELT_NEED_EXPIRE) {
- *lp = *value;
- }
- else {
- *lp += *value;
- *value = *lp;
- }
- elt->age = time (NULL);
- if (storage->backend) {
- if (storage->backend->replace_func (storage->backend, key, keylen,
- elt)) {
- RW_W_UNLOCK (&storage->rwlock);
- return TRUE;
- }
- else {
- RW_W_UNLOCK (&storage->rwlock);
- return FALSE;
- }
- }
- else {
- RW_W_UNLOCK (&storage->rwlock);
- return TRUE;
- }
- }
-
- RW_W_UNLOCK (&storage->rwlock);
-
- return FALSE;
-}
-
-/** Lookup an element inside kv storage */
-struct rspamd_kv_element *
-rspamd_kv_storage_lookup (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- time_t now)
-{
- struct rspamd_kv_element *elt = NULL, *belt;
-
- /* First try to look at cache */
- RW_R_LOCK (&storage->rwlock);
- elt = storage->cache->lookup_func (storage->cache, key, keylen);
-
- /* Next look at the backend */
- if (elt == NULL && storage->backend) {
- belt = storage->backend->lookup_func (storage->backend, key, keylen);
-
- if (belt) {
- /* Put this element into cache */
- if ((belt->flags & KV_ELT_DIRTY) == 0) {
- belt->flags |= KV_ELT_NEED_INSERT;
- return belt;
- }
- else {
- elt = belt;
- }
- }
- }
-
- if (elt && (elt->flags & KV_ELT_PERSISTENT) == 0 && elt->expire > 0) {
- /* Check expiration */
- if (now - elt->age > (gint)elt->expire) {
- /* Set need expire as we have no write lock here */
- elt->flags |= KV_ELT_NEED_EXPIRE;
- elt = NULL;
- }
- }
-
- /* RWlock is still locked */
- return elt;
-}
-
-/** Expire an element from kv storage */
-struct rspamd_kv_element *
-rspamd_kv_storage_delete (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen)
-{
- struct rspamd_kv_element *elt;
-
- /* First delete key from cache */
- RW_W_LOCK (&storage->rwlock);
- elt = storage->cache->delete_func (storage->cache, key, keylen);
-
- /* Now delete from backend */
- if (storage->backend) {
- storage->backend->delete_func (storage->backend, key, keylen);
- }
- /* Notify expire */
- if (elt) {
- if (storage->expire) {
- storage->expire->delete_func (storage->expire, elt);
- }
- storage->elts--;
- storage->memory -= elt->size;
- if ((elt->flags & KV_ELT_DIRTY) != 0) {
- elt->flags |= KV_ELT_NEED_FREE;
- }
- else {
- g_slice_free1 (ELT_SIZE (elt), elt);
- }
- }
-
- RW_W_UNLOCK (&storage->rwlock);
-
- return elt;
-}
-
-/** Destroy kv storage */
-void
-rspamd_kv_storage_destroy (struct rspamd_kv_storage *storage)
-{
- RW_W_LOCK (&storage->rwlock);
- if (storage->backend && storage->backend->destroy_func) {
- storage->backend->destroy_func (storage->backend);
- }
- if (storage->expire && storage->expire->destroy_func) {
- storage->expire->destroy_func (storage->expire);
- }
- if (storage->cache && storage->cache->destroy_func) {
- storage->cache->destroy_func (storage->cache);
- }
-
- g_free (storage->name);
-
- RW_W_UNLOCK (&storage->rwlock);
- g_slice_free1 (sizeof (struct rspamd_kv_storage), storage);
-}
-
-/** Insert array */
-gboolean
-rspamd_kv_storage_insert_array (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- guint elt_size,
- gpointer data,
- gsize len,
- gint flags,
- guint expire)
-{
- struct rspamd_kv_element *elt;
- guint *es;
- gpointer arr_data;
-
- /* Make temporary copy */
- arr_data = g_slice_alloc (len + sizeof (guint));
- es = arr_data;
- *es = elt_size;
- memcpy (arr_data, (gchar *)data + sizeof (guint), len);
- if (!rspamd_kv_storage_insert_cache (storage, key, keylen, arr_data, len +
- sizeof (guint),
- flags, expire, &elt)) {
- g_slice_free1 (len + sizeof (guint), arr_data);
- return FALSE;
- }
- /* Now set special data of element */
- elt->flags |= KV_ELT_ARRAY;
- g_slice_free1 (len + sizeof (guint), arr_data);
- /* Place to the backend */
-
- if (storage->backend) {
- return storage->backend->insert_func (storage->backend, key, keylen,
- elt);
- }
-
- return TRUE;
-}
-
-/** Set element inside array */
-gboolean
-rspamd_kv_storage_set_array (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- guint elt_num,
- gpointer data,
- gsize len,
- time_t now)
-{
- struct rspamd_kv_element *elt;
- guint *es;
- gpointer target;
-
- elt = rspamd_kv_storage_lookup (storage, key, keylen, now);
- if (elt == NULL) {
- return FALSE;
- }
-
- if ((elt->flags & KV_ELT_ARRAY) == 0) {
- return FALSE;
- }
- /* Get element size */
- es = (guint *)ELT_DATA (elt);
- if (elt_num > (elt->size - sizeof (guint)) / (*es)) {
- /* Invalid index */
- return FALSE;
- }
- target = (gchar *)ELT_DATA (elt) + sizeof (guint) + (*es) * elt_num;
- if (len != *es) {
- /* Invalid size */
- return FALSE;
- }
- memcpy (target, data, len);
- /* Place to the backend */
- if (storage->backend) {
- return storage->backend->replace_func (storage->backend,
- key,
- keylen,
- elt);
- }
-
- return TRUE;
-}
-
-/** Get element inside array */
-gboolean
-rspamd_kv_storage_get_array (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- guint elt_num,
- gpointer *data,
- gsize *len,
- time_t now)
-{
- struct rspamd_kv_element *elt;
- guint *es;
- gpointer target;
-
- elt = rspamd_kv_storage_lookup (storage, key, keylen, now);
- if (elt == NULL) {
- return FALSE;
- }
-
- if ((elt->flags & KV_ELT_ARRAY) == 0) {
- return FALSE;
- }
- /* Get element size */
- es = (guint *)ELT_DATA (elt);
- if (elt_num > (elt->size - sizeof (guint)) / (*es)) {
- /* Invalid index */
- return FALSE;
- }
- target = ELT_DATA (elt) + sizeof (guint) + (*es) * elt_num;
-
- *len = *es;
- *data = target;
-
- return TRUE;
-}
-
-/**
- * LRU expire functions
- */
-
-struct rspamd_kv_lru_expire {
- expire_init init_func; /*< this callback is called on kv storage initialization */
- expire_insert insert_func; /*< this callback is called when element is inserted */
- expire_step step_func; /*< this callback is used when cache is full */
- expire_delete delete_func; /*< this callback is called when an element is deleted */
- expire_destroy destroy_func; /*< this callback is used for destroying all elements inside expire */
-
- TAILQ_HEAD (eltq, rspamd_kv_element) head;
-};
-
-/**
- * Insert an element into expire queue
- */
-static void
-rspamd_lru_insert (struct rspamd_kv_expire *e, struct rspamd_kv_element *elt)
-{
- struct rspamd_kv_lru_expire *expire = (struct rspamd_kv_lru_expire *)e;
-
- /* Get a proper queue */
- TAILQ_INSERT_TAIL (&expire->head, elt, entry);
-}
-/**
- * Delete an element from expire queue
- */
-static void
-rspamd_lru_delete (struct rspamd_kv_expire *e, struct rspamd_kv_element *elt)
-{
- struct rspamd_kv_lru_expire *expire = (struct rspamd_kv_lru_expire *)e;
-
- /* Unlink element */
- TAILQ_REMOVE (&expire->head, elt, entry);
-}
-
-/**
- * Expire elements
- */
-static gboolean
-rspamd_lru_expire_step (struct rspamd_kv_expire *e,
- struct rspamd_kv_storage *storage,
- time_t now,
- gboolean forced)
-{
- struct rspamd_kv_lru_expire *expire = (struct rspamd_kv_lru_expire *)e;
- struct rspamd_kv_element *elt, *oldest_elt = NULL, *temp;
- time_t diff;
- gboolean res = FALSE;
-
- elt = TAILQ_FIRST (&expire->head);
- if (elt &&
- (forced || (elt->flags & (KV_ELT_PERSISTENT | KV_ELT_DIRTY)) == 0)) {
- diff = elt->expire - (now - elt->age);
- if (diff > 0 || (forced && elt->expire == 0)) {
- oldest_elt = elt;
- }
- else {
- /* This element is already expired */
- storage->cache->steal_func (storage->cache, elt);
- storage->memory -= ELT_SIZE (elt);
- storage->elts--;
- TAILQ_REMOVE (&expire->head, elt, entry);
- /* Free memory */
- if ((elt->flags & (KV_ELT_DIRTY | KV_ELT_NEED_INSERT)) != 0) {
- elt->flags |= KV_ELT_NEED_FREE;
- }
- else {
- g_slice_free1 (ELT_SIZE (elt), elt);
- }
- res = TRUE;
- /* Check other elements in this queue */
- TAILQ_FOREACH_SAFE (elt, &expire->head, entry, temp)
- {
- if ((!forced &&
- (elt->flags & (KV_ELT_PERSISTENT | KV_ELT_DIRTY)) != 0) ||
- (gint)elt->expire < (now - elt->age)) {
- break;
- }
- storage->memory -= ELT_SIZE (elt);
- storage->elts--;
- storage->cache->steal_func (storage->cache, elt);
- TAILQ_REMOVE (&expire->head, elt, entry);
- /* Free memory */
- if ((elt->flags & (KV_ELT_DIRTY | KV_ELT_NEED_INSERT)) != 0) {
- elt->flags |= KV_ELT_NEED_FREE;
- }
- else {
- g_slice_free1 (ELT_SIZE (elt), elt);
- }
-
- }
- }
- }
-
- if (!res && oldest_elt != NULL) {
- storage->memory -= ELT_SIZE (oldest_elt);
- storage->elts--;
- storage->cache->steal_func (storage->cache, oldest_elt);
- TAILQ_REMOVE (&expire->head, oldest_elt, entry);
- /* Free memory */
- if ((oldest_elt->flags & (KV_ELT_DIRTY | KV_ELT_NEED_INSERT)) != 0) {
- oldest_elt->flags |= KV_ELT_NEED_FREE;
- }
- else {
- g_slice_free1 (ELT_SIZE (oldest_elt), oldest_elt);
- }
- }
-
- return TRUE;
-}
-
-/**
- * Destroy LRU expire memory
- */
-static void
-rspamd_lru_destroy (struct rspamd_kv_expire *e)
-{
- struct rspamd_kv_lru_expire *expire = (struct rspamd_kv_lru_expire *)e;
-
- g_slice_free1 (sizeof (struct rspamd_kv_lru_expire), expire);
-}
-
-/**
- * Create new LRU cache
- */
-struct rspamd_kv_expire *
-rspamd_lru_expire_new (void)
-{
- struct rspamd_kv_lru_expire *new;
-
- new = g_slice_alloc (sizeof (struct rspamd_kv_lru_expire));
- TAILQ_INIT (&new->head);
-
- /* Set callbacks */
- new->init_func = NULL;
- new->insert_func = rspamd_lru_insert;
- new->delete_func = rspamd_lru_delete;
- new->step_func = rspamd_lru_expire_step;
- new->destroy_func = rspamd_lru_destroy;
-
- return (struct rspamd_kv_expire *)new;
-}
-
-/*
- * KV cache hash table
- */
-struct rspamd_kv_hash_cache {
- cache_init init_func; /*< this callback is called on kv storage initialization */
- cache_insert insert_func; /*< this callback is called when element is inserted */
- cache_replace replace_func; /*< this callback is called when element is replace */
- cache_lookup lookup_func; /*< this callback is used for lookup of element */
- cache_delete delete_func; /*< this callback is called when an element is deleted */
- cache_steal steal_func; /*< this callback is used to replace duplicates in cache */
- cache_destroy destroy_func; /*< this callback is used for destroying all elements inside cache */
- GHashTable *hash;
-};
-
-/**
- * Insert an element inside cache
- */
-static struct rspamd_kv_element *
-rspamd_kv_hash_insert (struct rspamd_kv_cache *c,
- gpointer key,
- guint keylen,
- gpointer value,
- gsize len)
-{
- struct rspamd_kv_element *elt;
- struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c;
- struct rspamd_kv_element search_elt;
-
- search_elt.keylen = keylen;
- search_elt.p = key;
-
- if ((elt = g_hash_table_lookup (cache->hash, &search_elt)) == NULL) {
- elt = g_slice_alloc (
- sizeof (struct rspamd_kv_element) + len + keylen + 1);
- elt->age = time (NULL);
- elt->keylen = keylen;
- elt->size = len;
- elt->flags = 0;
- memcpy (ELT_KEY (elt), key, keylen + 1);
- memcpy (ELT_DATA (elt), value, len);
- elt->p = &elt->data;
- g_hash_table_insert (cache->hash, elt, elt);
- }
- else {
- g_hash_table_steal (cache->hash, elt);
- if ((elt->flags & KV_ELT_DIRTY) != 0) {
- elt->flags |= KV_ELT_NEED_FREE;
- }
- else {
- /* Free it by self */
- g_slice_free1 (ELT_SIZE (elt), elt);
- }
- elt = g_slice_alloc (
- sizeof (struct rspamd_kv_element) + len + keylen + 1);
- elt->age = time (NULL);
- elt->keylen = keylen;
- elt->size = len;
- elt->flags = 0;
- memcpy (ELT_KEY (elt), key, keylen + 1);
- memcpy (ELT_DATA (elt), value, len);
- elt->p = &elt->data;
- g_hash_table_insert (cache->hash, elt, elt);
- }
-
- return elt;
-}
-
-/**
- * Lookup an item inside hash
- */
-static struct rspamd_kv_element *
-rspamd_kv_hash_lookup (struct rspamd_kv_cache *c, gpointer key, guint keylen)
-{
- struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c;
- struct rspamd_kv_element search_elt;
-
- search_elt.keylen = keylen;
- search_elt.p = key;
-
- return g_hash_table_lookup (cache->hash, &search_elt);
-}
-
-/**
- * Replace an element inside cache
- */
-static gboolean
-rspamd_kv_hash_replace (struct rspamd_kv_cache *c,
- gpointer key,
- guint keylen,
- struct rspamd_kv_element *elt)
-{
- struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c;
- struct rspamd_kv_element *oldelt, search_elt;
-
- search_elt.keylen = keylen;
- search_elt.p = key;
-
- if ((oldelt = g_hash_table_lookup (cache->hash, &search_elt)) != NULL) {
- g_hash_table_steal (cache->hash, oldelt);
-
- if ((oldelt->flags & KV_ELT_DIRTY) != 0) {
- oldelt->flags |= KV_ELT_NEED_FREE;
- }
- else {
- /* Free it by self */
- g_slice_free1 (ELT_SIZE (oldelt), oldelt);
- }
- g_hash_table_insert (cache->hash, elt, elt);
- return TRUE;
- }
-
- return FALSE;
-}
-
-/**
- * Delete an element from cache
- */
-static struct rspamd_kv_element *
-rspamd_kv_hash_delete (struct rspamd_kv_cache *c, gpointer key, guint keylen)
-{
- struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c;
- struct rspamd_kv_element *elt;
- struct rspamd_kv_element search_elt;
-
- search_elt.keylen = keylen;
- search_elt.p = key;
-
- elt = g_hash_table_lookup (cache->hash, &search_elt);
- if (elt) {
- g_hash_table_steal (cache->hash, &search_elt);
- }
- return elt;
-}
-
-/**
- * Steal an element from cache
- */
-static void
-rspamd_kv_hash_steal (struct rspamd_kv_cache *c, struct rspamd_kv_element *elt)
-{
- struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c;
-
- g_hash_table_steal (cache->hash, elt);
-}
-
-/**
- * Destroy the whole cache
- */
-
-static void
-rspamd_kv_hash_destroy_cb (gpointer key, gpointer value, gpointer unused)
-{
- struct rspamd_kv_element *elt = value;
-
- g_slice_free1 (ELT_SIZE (elt), elt);
-}
-
-static void
-rspamd_kv_hash_destroy (struct rspamd_kv_cache *c)
-{
- struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c;
-
- g_hash_table_foreach (cache->hash, rspamd_kv_hash_destroy_cb, NULL);
- g_hash_table_destroy (cache->hash);
- g_slice_free1 (sizeof (struct rspamd_kv_hash_cache), cache);
-}
-
-/**
- * Make hash for element
- */
-#define rot(x,k) (((x) << (k)) ^ ((x) >> (32 - (k))))
-#define mix(a,b,c) \
- { \
- a -= c; a ^= rot (c, 4); c += b; \
- b -= a; b ^= rot (a, 6); a += c; \
- c -= b; c ^= rot (b, 8); b += a; \
- a -= c; a ^= rot (c,16); c += b; \
- b -= a; b ^= rot (a,19); a += c; \
- c -= b; c ^= rot (b, 4); b += a; \
- }
-#define final(a,b,c) \
- { \
- c ^= b; c -= rot (b,14); \
- a ^= c; a -= rot (c,11); \
- b ^= a; b -= rot (a,25); \
- c ^= b; c -= rot (b,16); \
- a ^= c; a -= rot (c,4); \
- b ^= a; b -= rot (a,14); \
- c ^= b; c -= rot (b,24); \
- }
-/*
- * The hash function used here is by Bob Jenkins, 1996:
- * <http://burtleburtle.net/bob/hash/doobs.html>
- * "By Bob Jenkins, 1996. bob_jenkins@burtleburtle.net.
- * You may use this code any way you wish, private, educational,
- * or commercial. It's free."
- *
- */
-guint
-kv_elt_hash_func (gconstpointer e)
-{
- struct rspamd_kv_element *elt = (struct rspamd_kv_element *)e;
- guint32 a, b, c;
- union { const void *ptr; size_t i; } u;
- guint length;
-
- /* Set up the internal state */
- length = elt->keylen;
- a = b = c = 0xdeadbeef + length;
-
- u.ptr = elt->p;
- if (((u.i & 0x3) == 0)) {
- const guint32 *k = (const guint32 *)elt->p; /* read 32-bit chunks */
-
- /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */
- while (length > 12) {
- a += k[0];
- b += k[1];
- c += k[2];
- mix (a,b,c);
- length -= 12;
- k += 3;
- }
-
- /*----------------------------- handle the last (probably partial) block */
- /*
- * "k[2]&0xffffff" actually reads beyond the end of the string, but
- * then masks off the part it's not allowed to read. Because the
- * string is aligned, the masked-off tail is in the same word as the
- * rest of the string. Every machine with memory protection I've seen
- * does it on word boundaries, so is OK with this. But VALGRIND will
- * still catch it and complain. The masking trick does make the hash
- * noticably faster for short strings (like English words).
- */
- switch (length)
- {
- case 12: c += k[2]; b += k[1]; a += k[0]; break;
- case 11: c += k[2] & 0xffffff; b += k[1]; a += k[0]; break;
- case 10: c += k[2] & 0xffff; b += k[1]; a += k[0]; break;
- case 9: c += k[2] & 0xff; b += k[1]; a += k[0]; break;
- case 8: b += k[1]; a += k[0]; break;
- case 7: b += k[1] & 0xffffff; a += k[0]; break;
- case 6: b += k[1] & 0xffff; a += k[0]; break;
- case 5: b += k[1] & 0xff; a += k[0]; break;
- case 4: a += k[0]; break;
- case 3: a += k[0] & 0xffffff; break;
- case 2: a += k[0] & 0xffff; break;
- case 1: a += k[0] & 0xff; break;
- case 0: return c; /* zero length strings require no mixing */
- }
-
- } else if (((u.i & 0x1) == 0)) {
- const guint16 *k = (const guint16 *)elt->p; /* read 16-bit chunks */
- const guint8 *k8;
-
- /*--------------- all but last block: aligned reads and different mixing */
- while (length > 12) {
- a += k[0] + (((guint32)k[1]) << 16);
- b += k[2] + (((guint32)k[3]) << 16);
- c += k[4] + (((guint32)k[5]) << 16);
- mix (a,b,c);
- length -= 12;
- k += 6;
- }
-
- /*----------------------------- handle the last (probably partial) block */
- k8 = (const guint8 *)k;
- switch (length)
- {
- case 12: c += k[4] + (((guint32)k[5]) << 16);
- b += k[2] + (((guint32)k[3]) << 16);
- a += k[0] + (((guint32)k[1]) << 16);
- break;
- case 11: c += ((guint32)k8[10]) << 16; /* @fallthrough */
- case 10: c += k[4]; /* @fallthrough@ */
- b += k[2] + (((guint32)k[3]) << 16);
- a += k[0] + (((guint32)k[1]) << 16);
- break;
- case 9: c += k8[8]; /* @fallthrough */
- case 8: b += k[2] + (((guint32)k[3]) << 16);
- a += k[0] + (((guint32)k[1]) << 16);
- break;
- case 7: b += ((guint32)k8[6]) << 16; /* @fallthrough */
- case 6: b += k[2];
- a += k[0] + (((guint32)k[1]) << 16);
- break;
- case 5: b += k8[4]; /* @fallthrough */
- case 4: a += k[0] + (((guint32)k[1]) << 16);
- break;
- case 3: a += ((guint32)k8[2]) << 16; /* @fallthrough */
- case 2: a += k[0];
- break;
- case 1: a += k8[0];
- break;
- case 0: return c; /* zero length strings require no mixing */
- }
-
- } else { /* need to read the key one byte at a time */
- const guint8 *k = elt->p;
-
- /*--------------- all but the last block: affect some 32 bits of (a,b,c) */
- while (length > 12)
- {
- a += k[0];
- a += ((guint32)k[1]) << 8;
- a += ((guint32)k[2]) << 16;
- a += ((guint32)k[3]) << 24;
- b += k[4];
- b += ((guint32)k[5]) << 8;
- b += ((guint32)k[6]) << 16;
- b += ((guint32)k[7]) << 24;
- c += k[8];
- c += ((guint32)k[9]) << 8;
- c += ((guint32)k[10]) << 16;
- c += ((guint32)k[11]) << 24;
- mix (a,b,c);
- length -= 12;
- k += 12;
- }
-
- /*-------------------------------- last block: affect all 32 bits of (c) */
- switch (length) /* all the case statements fall through */
- {
- case 12: c += ((guint32)k[11]) << 24;
- case 11: c += ((guint32)k[10]) << 16;
- case 10: c += ((guint32)k[9]) << 8;
- case 9: c += k[8];
- case 8: b += ((guint32)k[7]) << 24;
- case 7: b += ((guint32)k[6]) << 16;
- case 6: b += ((guint32)k[5]) << 8;
- case 5: b += k[4];
- case 4: a += ((guint32)k[3]) << 24;
- case 3: a += ((guint32)k[2]) << 16;
- case 2: a += ((guint32)k[1]) << 8;
- case 1: a += k[0];
- break;
- case 0: return c; /* zero length strings require no mixing */
- }
- }
-
- final (a,b,c);
- return c; /* zero length strings require no mixing */
-}
-
-gboolean
-kv_elt_compare_func (gconstpointer e1, gconstpointer e2)
-{
- struct rspamd_kv_element *elt1 = (struct rspamd_kv_element *) e1,
- *elt2 = (struct rspamd_kv_element *) e2;
-
- if (elt1->keylen == elt2->keylen) {
- return memcmp (elt1->p, elt2->p, elt1->keylen) == 0;
- }
-
- return FALSE;
-}
-
-/**
- * Create new hash kv cache
- */
-struct rspamd_kv_cache *
-rspamd_kv_hash_new (void)
-{
- struct rspamd_kv_hash_cache *new;
-
- new = g_slice_alloc (sizeof (struct rspamd_kv_hash_cache));
- new->hash = g_hash_table_new_full (kv_elt_hash_func,
- kv_elt_compare_func,
- NULL,
- NULL);
- new->init_func = NULL;
- new->insert_func = rspamd_kv_hash_insert;
- new->lookup_func = rspamd_kv_hash_lookup;
- new->replace_func = rspamd_kv_hash_replace;
- new->delete_func = rspamd_kv_hash_delete;
- new->steal_func = rspamd_kv_hash_steal;
- new->destroy_func = rspamd_kv_hash_destroy;
-
- return (struct rspamd_kv_cache *)new;
-}
-
-/*
- * Radix cache hash table
- */
-struct rspamd_kv_radix_cache {
- cache_init init_func; /*< this callback is called on kv storage initialization */
- cache_insert insert_func; /*< this callback is called when element is inserted */
- cache_replace replace_func; /*< this callback is called when element is replace */
- cache_lookup lookup_func; /*< this callback is used for lookup of element */
- cache_delete delete_func; /*< this callback is called when an element is deleted */
- cache_steal steal_func; /*< this callback is used to replace duplicates in cache */
- cache_destroy destroy_func; /*< this callback is used for destroying all elements inside cache */
- radix_tree_t *tree;
-};
-
-/**
- * Validate a key for radix
- */
-static guint32
-rspamd_kv_radix_validate (gpointer key, guint keylen)
-{
- struct in_addr addr;
-
- if (inet_aton (key, &addr) == 0) {
- return 0;
- }
-
- return addr.s_addr;
-}
-
-/**
- * Insert an element inside cache
- */
-static struct rspamd_kv_element *
-rspamd_kv_radix_insert (struct rspamd_kv_cache *c,
- gpointer key,
- guint keylen,
- gpointer value,
- gsize len)
-{
- struct rspamd_kv_element *elt;
- struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c;
- guint32 rkey = rspamd_kv_radix_validate (key, keylen);
-
- if (rkey == 0) {
- return NULL;
- }
-
- elt = (struct rspamd_kv_element *)radix32tree_find (cache->tree, rkey);
- if ((uintptr_t)elt == RADIX_NO_VALUE) {
- elt = g_slice_alloc (
- sizeof (struct rspamd_kv_element) + len + keylen + 1);
- elt->age = time (NULL);
- elt->keylen = keylen;
- elt->size = len;
- elt->flags = 0;
- memcpy (ELT_KEY (elt), key, keylen + 1);
- memcpy (ELT_DATA (elt), value, len);
- elt->p = &elt->data;
- radix32tree_insert (cache->tree, rkey, 0xffffffff, (uintptr_t)elt);
- }
- else {
- radix32tree_delete (cache->tree, rkey, 0xffffffff);
- if ((elt->flags & KV_ELT_DIRTY) != 0) {
- elt->flags |= KV_ELT_NEED_FREE;
- }
- else {
- /* Free it by self */
- g_slice_free1 (ELT_SIZE (elt), elt);
- }
- elt = g_slice_alloc (
- sizeof (struct rspamd_kv_element) + len + keylen + 1);
- elt->age = time (NULL);
- elt->keylen = keylen;
- elt->size = len;
- elt->flags = 0;
- memcpy (ELT_KEY (elt), key, keylen + 1);
- memcpy (ELT_DATA (elt), value, len);
- elt->p = &elt->data;
- radix32tree_insert (cache->tree, rkey, 0xffffffff, (uintptr_t)elt);
- }
-
- return elt;
-}
-
-/**
- * Lookup an item inside radix
- */
-static struct rspamd_kv_element *
-rspamd_kv_radix_lookup (struct rspamd_kv_cache *c, gpointer key, guint keylen)
-{
- struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c;
- guint32 rkey = rspamd_kv_radix_validate (key, keylen);
- struct rspamd_kv_element *elt;
-
- elt = (struct rspamd_kv_element *)radix32tree_find (cache->tree, rkey);
- if ((uintptr_t)elt == RADIX_NO_VALUE) {
- return NULL;
- }
-
- return elt;
-}
-
-/**
- * Replace an element inside cache
- */
-static gboolean
-rspamd_kv_radix_replace (struct rspamd_kv_cache *c,
- gpointer key,
- guint keylen,
- struct rspamd_kv_element *elt)
-{
- struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c;
- guint32 rkey = rspamd_kv_radix_validate (key, keylen);
- struct rspamd_kv_element *oldelt;
-
- oldelt = (struct rspamd_kv_element *)radix32tree_find (cache->tree, rkey);
- if ((uintptr_t)oldelt != RADIX_NO_VALUE) {
- radix32tree_delete (cache->tree, rkey, 0xffffffff);
-
- if ((oldelt->flags & KV_ELT_DIRTY) != 0) {
- oldelt->flags |= KV_ELT_NEED_FREE;
- }
- else {
- /* Free it by self */
- g_slice_free1 (ELT_SIZE (oldelt), oldelt);
- }
- radix32tree_insert (cache->tree, rkey, 0xffffffff, (uintptr_t)elt);
- return TRUE;
- }
-
- return FALSE;
-}
-
-/**
- * Delete an element from cache
- */
-static struct rspamd_kv_element *
-rspamd_kv_radix_delete (struct rspamd_kv_cache *c, gpointer key, guint keylen)
-{
- struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c;
- struct rspamd_kv_element *elt;
- guint32 rkey = rspamd_kv_radix_validate (key, keylen);
-
- elt = (struct rspamd_kv_element *)radix32tree_find (cache->tree, rkey);
- if ((uintptr_t)elt != RADIX_NO_VALUE) {
- radix32tree_delete (cache->tree, rkey, 0xffffffff);
- }
- else {
- return NULL;
- }
- return elt;
-}
-
-/**
- * Delete an element from cache
- */
-static void
-rspamd_kv_radix_steal (struct rspamd_kv_cache *c, struct rspamd_kv_element *elt)
-{
- struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c;
- guint32 rkey = rspamd_kv_radix_validate (ELT_KEY (elt), elt->keylen);
-
-
- radix32tree_delete (cache->tree, rkey, 0xffffffff);
-}
-
-/**
- * Destroy the whole cache
- */
-static void
-rspamd_kv_radix_destroy (struct rspamd_kv_cache *c)
-{
- struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c;
-
- radix_tree_free (cache->tree);
- g_slice_free1 (sizeof (struct rspamd_kv_radix_cache), cache);
-}
-
-/**
- * Create new radix kv cache
- */
-struct rspamd_kv_cache *
-rspamd_kv_radix_new (void)
-{
- struct rspamd_kv_radix_cache *new;
-
- new = g_slice_alloc (sizeof (struct rspamd_kv_radix_cache));
- new->tree = radix_tree_create ();
- new->init_func = NULL;
- new->insert_func = rspamd_kv_radix_insert;
- new->lookup_func = rspamd_kv_radix_lookup;
- new->replace_func = rspamd_kv_radix_replace;
- new->delete_func = rspamd_kv_radix_delete;
- new->steal_func = rspamd_kv_radix_steal;
- new->destroy_func = rspamd_kv_radix_destroy;
-
- return (struct rspamd_kv_cache *)new;
-}
-
-
-#ifdef WITH_JUDY
-/*
- * KV cache hash table
- */
-struct rspamd_kv_judy_cache {
- cache_init init_func; /*< this callback is called on kv storage initialization */
- cache_insert insert_func; /*< this callback is called when element is inserted */
- cache_replace replace_func; /*< this callback is called when element is replace */
- cache_lookup lookup_func; /*< this callback is used for lookup of element */
- cache_delete delete_func; /*< this callback is called when an element is deleted */
- cache_steal steal_func; /*< this callback is used to replace duplicates in cache */
- cache_destroy destroy_func; /*< this callback is used for destroying all elements inside cache */
- Pvoid_t judy;
-};
-
-
-/**
- * Lookup an item inside judy
- */
-static struct rspamd_kv_element *
-rspamd_kv_judy_lookup (struct rspamd_kv_cache *c, gpointer key, guint keylen)
-{
- struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c;
- struct rspamd_kv_element *elt = NULL, **pelt;
-
- JHSG (pelt, cache->judy, key, keylen);
- if (pelt != NULL) {
- elt = *pelt;
- }
- return elt;
-}
-
-/**
- * Delete an element from cache
- */
-static struct rspamd_kv_element *
-rspamd_kv_judy_delete (struct rspamd_kv_cache *c, gpointer key, guint keylen)
-{
- struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c;
- struct rspamd_kv_element *elt;
- gint rc;
-
- elt = rspamd_kv_judy_lookup (c, key, keylen);
- if (elt) {
- JHSD (rc, cache->judy, ELT_KEY (elt), elt->keylen);
- }
- return elt;
-}
-
-/**
- * Steal an element from cache
- */
-static void
-rspamd_kv_judy_steal (struct rspamd_kv_cache *c, struct rspamd_kv_element *elt)
-{
- struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c;
- gint rc;
-
- JHSD (rc, cache->judy, ELT_KEY (elt), elt->keylen);
-}
-
-/**
- * Insert an element inside cache
- */
-static struct rspamd_kv_element *
-rspamd_kv_judy_insert (struct rspamd_kv_cache *c,
- gpointer key,
- guint keylen,
- gpointer value,
- gsize len)
-{
- struct rspamd_kv_element *elt, **pelt;
- struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c;
-
- if ((elt = rspamd_kv_judy_lookup (c, key, keylen)) == NULL) {
- elt = g_slice_alloc (
- sizeof (struct rspamd_kv_element) + len + keylen + 1);
- elt->age = time (NULL);
- elt->keylen = keylen;
- elt->size = len;
- elt->flags = 0;
- memcpy (ELT_KEY (elt), key, keylen);
- memcpy (ELT_DATA (elt), value, len);
- JHSI (pelt, cache->judy, ELT_KEY (elt), elt->keylen);
- elt->p = &elt->data;
- *pelt = elt;
- }
- else {
- rspamd_kv_judy_steal (c, elt);
- if ((elt->flags & KV_ELT_DIRTY) != 0) {
- elt->flags |= KV_ELT_NEED_FREE;
- }
- else {
- /* Free it by self */
- g_slice_free1 (ELT_SIZE (elt), elt);
- }
- elt = g_slice_alloc0 (
- sizeof (struct rspamd_kv_element) + len + keylen + 1);
- elt->age = time (NULL);
- elt->keylen = keylen;
- elt->size = len;
- elt->flags = 0;
- memcpy (ELT_KEY (elt), key, keylen);
- memcpy (ELT_DATA (elt), value, len);
- elt->p = &elt->data;
- JHSI (pelt, cache->judy, ELT_KEY (elt), elt->keylen);
- *pelt = elt;
- }
-
- return elt;
-}
-
-/**
- * Replace an element inside cache
- */
-static gboolean
-rspamd_kv_judy_replace (struct rspamd_kv_cache *c,
- gpointer key,
- guint keylen,
- struct rspamd_kv_element *elt)
-{
- struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c;
- struct rspamd_kv_element *oldelt, **pelt;
-
- if ((oldelt = rspamd_kv_judy_lookup (c, key, keylen)) != NULL) {
- rspamd_kv_judy_steal (c, elt);
-
- if ((oldelt->flags & KV_ELT_DIRTY) != 0) {
- oldelt->flags |= KV_ELT_NEED_FREE;
- }
- else {
- /* Free it by self */
- g_slice_free1 (ELT_SIZE (oldelt), oldelt);
- }
- JHSI (pelt, cache->judy, ELT_KEY (elt), elt->keylen);
- *pelt = elt;
- return TRUE;
- }
-
- return FALSE;
-}
-
-/**
- * Destroy the whole cache
- */
-static void
-rspamd_kv_judy_destroy (struct rspamd_kv_cache *c)
-{
- struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c;
- glong bytes;
-
- JHSFA (bytes, cache->judy);
- g_slice_free1 (sizeof (struct rspamd_kv_judy_cache), cache);
-}
-
-/**
- * Judy tree
- */
-struct rspamd_kv_cache *
-rspamd_kv_judy_new (void)
-{
- struct rspamd_kv_judy_cache *new;
-
- new = g_slice_alloc (sizeof (struct rspamd_kv_judy_cache));
- new->judy = NULL;
- new->init_func = NULL;
- new->insert_func = rspamd_kv_judy_insert;
- new->lookup_func = rspamd_kv_judy_lookup;
- new->replace_func = rspamd_kv_judy_replace;
- new->delete_func = rspamd_kv_judy_delete;
- new->steal_func = rspamd_kv_judy_steal;
- new->destroy_func = rspamd_kv_judy_destroy;
-
- return (struct rspamd_kv_cache *)new;
-}
-#endif
+++ /dev/null
-/* Copyright (c) 2011, Vsevolod Stakhov
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef KVSTORAGE_H_
-#define KVSTORAGE_H_
-
-#include "config.h"
-
-struct rspamd_kv_cache;
-struct rspamd_kv_backend;
-struct rspamd_kv_storage;
-struct rspamd_kv_expire;
-struct rspamd_kv_element;
-
-/* Locking definitions */
-#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 30))
-#define RW_R_LOCK g_rw_lock_reader_lock
-#define RW_R_UNLOCK g_rw_lock_reader_unlock
-#define RW_W_LOCK g_rw_lock_writer_lock
-#define RW_W_UNLOCK g_rw_lock_writer_unlock
-#else
-#define RW_R_LOCK g_static_rw_lock_reader_lock
-#define RW_R_UNLOCK g_static_rw_lock_reader_unlock
-#define RW_W_LOCK g_static_rw_lock_writer_lock
-#define RW_W_UNLOCK g_static_rw_lock_writer_unlock
-#endif
-
-/* Callbacks for cache */
-typedef void (*cache_init)(struct rspamd_kv_cache *cache);
-typedef struct rspamd_kv_element * (*cache_insert)(struct rspamd_kv_cache *cache,
- gpointer key, guint keylen, gpointer value, gsize len);
-typedef gboolean (*cache_replace)(struct rspamd_kv_cache *cache, gpointer key,
- guint keylen,
- struct rspamd_kv_element *elt);
-typedef struct rspamd_kv_element * (*cache_lookup)(struct rspamd_kv_cache *cache,
- gpointer key, guint keylen);
-typedef struct rspamd_kv_element * (*cache_delete)(struct rspamd_kv_cache *cache,
- gpointer key, guint keylen);
-typedef void (*cache_steal)(struct rspamd_kv_cache *cache,
- struct rspamd_kv_element * elt);
-typedef void (*cache_destroy)(struct rspamd_kv_cache *cache);
-
-/* Callbacks for backend */
-typedef void (*backend_init)(struct rspamd_kv_backend *backend);
-typedef gboolean (*backend_insert)(struct rspamd_kv_backend *backend,
- gpointer key, guint keylen,
- struct rspamd_kv_element *elt);
-typedef gboolean (*backend_replace)(struct rspamd_kv_backend *backend,
- gpointer key, guint keylen,
- struct rspamd_kv_element *elt);
-typedef struct rspamd_kv_element * (*backend_lookup)(struct rspamd_kv_backend *
- backend, gpointer key,
- guint keylen);
-typedef void (*backend_delete)(struct rspamd_kv_backend *backend, gpointer key,
- guint keylen);
-typedef gboolean (*backend_sync)(struct rspamd_kv_backend *backend);
-typedef gboolean (*backend_incref)(struct rspamd_kv_backend *backend,
- gpointer key, guint keylen);
-typedef void (*backend_destroy)(struct rspamd_kv_backend *backend);
-
-/* Callbacks for expire */
-typedef void (*expire_init)(struct rspamd_kv_expire *expire);
-typedef void (*expire_insert)(struct rspamd_kv_expire *expire,
- struct rspamd_kv_element *elt);
-typedef void (*expire_delete)(struct rspamd_kv_expire *expire,
- struct rspamd_kv_element *elt);
-typedef gboolean (*expire_step)(struct rspamd_kv_expire *expire,
- struct rspamd_kv_storage *storage,
- time_t now, gboolean forced);
-typedef void (*expire_destroy)(struct rspamd_kv_expire *expire);
-
-
-/* Flags of element */
-enum rspamd_kv_flags {
- KV_ELT_ARRAY = 1 << 0,
- KV_ELT_PERSISTENT = 1 << 1,
- KV_ELT_DIRTY = 1 << 2,
- KV_ELT_OUSTED = 1 << 3,
- KV_ELT_NEED_FREE = 1 << 4,
- KV_ELT_INTEGER = 1 << 5,
- KV_ELT_NEED_INSERT = 1 << 6,
- KV_ELT_NEED_EXPIRE = 1 << 7
-};
-
-#define ELT_DATA(elt) (gchar *)(elt)->data + (elt)->keylen + 1
-#define ELT_LONG(elt) *((glong *)((elt)->data + (elt)->keylen + 1))
-#define ELT_KEY(elt) (gchar *)(elt)->data
-#define ELT_SIZE(elt) elt->size + sizeof(struct rspamd_kv_element) + \
- elt->keylen + 1
-
-/* Common structures description */
-
-struct rspamd_kv_element {
- time_t age; /*< age of element */
- guint32 expire; /*< expire of element */
- gint flags; /*< element flags */
- gsize size; /*< size of element */
- TAILQ_ENTRY (rspamd_kv_element) entry; /*< list entry */
- guint keylen; /*< length of key */
-
- gpointer p; /*< pointer to data */
- gchar data[1]; /*< expandable data */
-};
-
-struct rspamd_kv_cache {
- cache_init init_func; /*< this callback is called on kv storage initialization */
- cache_insert insert_func; /*< this callback is called when element is inserted */
- cache_replace replace_func; /*< this callback is called when element is replace */
- cache_lookup lookup_func; /*< this callback is used for lookup of element */
- cache_delete delete_func; /*< this callback is called when an element is deleted */
- cache_steal steal_func; /*< this callback is used to replace duplicates in cache */
- cache_destroy destroy_func; /*< this callback is used for destroying all elements inside cache */
-};
-struct rspamd_kv_backend {
- backend_init init_func; /*< this callback is called on kv storage initialization */
- backend_insert insert_func; /*< this callback is called when element is inserted */
- backend_replace replace_func; /*< this callback is called when element is replaced */
- backend_lookup lookup_func; /*< this callback is used for lookup of element */
- backend_delete delete_func; /*< this callback is called when an element is deleted */
- backend_sync sync_func; /*< this callback is called when backend need to be synced */
- backend_incref incref_func; /*< this callback is called when element must be ref'd */
- backend_destroy destroy_func; /*< this callback is used for destroying all elements inside backend */
-};
-struct rspamd_kv_expire {
- expire_init init_func; /*< this callback is called on kv storage initialization */
- expire_insert insert_func; /*< this callback is called when element is inserted */
- expire_step step_func; /*< this callback is used when cache is full */
- expire_delete delete_func; /*< this callback is called when an element is deleted */
- expire_destroy destroy_func; /*< this callback is used for destroying all elements inside expire */
-};
-
-/* Main kv storage structure */
-
-struct rspamd_kv_storage {
- struct rspamd_kv_cache *cache;
- struct rspamd_kv_backend *backend;
- struct rspamd_kv_expire *expire;
-
- gsize elts; /*< current elements count in a storage */
- gsize max_elts; /*< maximum number of elements in a storage */
-
- gsize memory; /*< memory eaten */
- gsize max_memory; /*< memory limit */
-
- gint id; /* char ID */
- gchar *name; /* numeric ID */
-
- gboolean no_overwrite; /* do not overwrite data with the same keys */
-#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 30))
- GRWLock rwlock; /* rwlock in new glib */
-#else
- GStaticRWLock rwlock; /* rwlock for threaded access */
-#endif
-};
-
-/** Create new kv storage */
-struct rspamd_kv_storage * rspamd_kv_storage_new (gint id, const gchar *name,
- struct rspamd_kv_cache *cache, struct rspamd_kv_backend *backend,
- struct rspamd_kv_expire *expire,
- gsize max_elts, gsize max_memory, gboolean no_overwrite);
-
-/** Insert new element to the kv storage */
-gboolean rspamd_kv_storage_insert (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- gpointer data,
- gsize len,
- gint flags,
- guint expire);
-
-/** Insert element only in cache */
-gboolean rspamd_kv_storage_insert_cache (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- gpointer data,
- gsize len,
- gint flags,
- guint expire,
- struct rspamd_kv_element **pelt);
-
-/** Replace an element in the kv storage */
-gboolean rspamd_kv_storage_replace (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- struct rspamd_kv_element *elt);
-
-/** Increment value in kvstorage */
-gboolean rspamd_kv_storage_increment (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- glong *value);
-
-/** Lookup an element inside kv storage */
-struct rspamd_kv_element * rspamd_kv_storage_lookup (
- struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- time_t now);
-
-/** Expire an element from kv storage */
-struct rspamd_kv_element * rspamd_kv_storage_delete (
- struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen);
-
-/** Destroy kv storage */
-void rspamd_kv_storage_destroy (struct rspamd_kv_storage *storage);
-
-/** Insert array */
-gboolean rspamd_kv_storage_insert_array (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- guint elt_size,
- gpointer data,
- gsize len,
- gint flags,
- guint expire);
-
-/** Set element inside array */
-gboolean rspamd_kv_storage_set_array (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- guint elt_num,
- gpointer data,
- gsize len,
- time_t now);
-
-/** Get element inside array */
-gboolean rspamd_kv_storage_get_array (struct rspamd_kv_storage *storage,
- gpointer key,
- guint keylen,
- guint elt_num,
- gpointer *data,
- gsize *len,
- time_t now);
-
-/* Hash table functions */
-guint kv_elt_hash_func (gconstpointer e);
-gboolean kv_elt_compare_func (gconstpointer e1, gconstpointer e2);
-
-/**
- * LRU expire
- */
-struct rspamd_kv_expire * rspamd_lru_expire_new (void);
-
-/**
- * Ordinary hash
- */
-struct rspamd_kv_cache * rspamd_kv_hash_new (void);
-
-/**
- * Radix tree
- */
-struct rspamd_kv_cache * rspamd_kv_radix_new (void);
-
-#ifdef WITH_JUDY
-/**
- * Judy tree
- */
-struct rspamd_kv_cache * rspamd_kv_judy_new (void);
-#endif
-
-#endif /* KVSTORAGE_H_ */
+++ /dev/null
-/* Copyright (c) 2010, Vsevolod Stakhov
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#include "config.h"
-#include "kvstorage.h"
-#include "kvstorage_bdb.h"
-#include "util.h"
-#include "main.h"
-#include <db.h>
-
-struct bdb_op {
- struct rspamd_kv_element *elt;
- enum {
- BDB_OP_INSERT,
- BDB_OP_DELETE,
- BDB_OP_REPLACE
- } op;
-};
-
-/* Main bdb structure */
-struct rspamd_bdb_backend {
- backend_init init_func; /*< this callback is called on kv storage initialization */
- backend_insert insert_func; /*< this callback is called when element is inserted */
- backend_replace replace_func; /*< this callback is called when element is replaced */
- backend_lookup lookup_func; /*< this callback is used for lookup of element */
- backend_delete delete_func; /*< this callback is called when an element is deleted */
- backend_sync sync_func; /*< this callback is called when backend need to be synced */
- backend_destroy destroy_func; /*< this callback is used for destroying all elements inside backend */
- DB_ENV *envp; /*< db environment */
- DB *dbp; /*< db pointer */
- gchar *filename;
- gchar *dirname;
- guint sync_ops;
- GQueue *ops_queue;
- GHashTable *ops_hash;
- gboolean initialized;
-};
-
-/* Process single bdb operation */
-static gboolean
-bdb_process_single_op (struct rspamd_bdb_backend *db,
- DB_TXN *txn,
- struct bdb_op *op)
-{
- DBT db_key, db_data;
-
- memset (&db_key, 0, sizeof(DBT));
- memset (&db_data, 0, sizeof(DBT));
-
- db_key.size = op->elt->keylen;
- db_key.data = ELT_KEY (op->elt);
- db_data.size = op->elt->size + sizeof (struct rspamd_kv_element) +
- op->elt->keylen + 1;
- db_data.data = op->elt;
-
- switch (op->op) {
- case BDB_OP_INSERT:
- case BDB_OP_REPLACE:
- db_data.flags = DB_DBT_USERMEM;
- if (db->dbp->put (db->dbp, NULL, &db_key, &db_data, 0) != 0) {
- return FALSE;
- }
- break;
- case BDB_OP_DELETE:
- db_data.flags = DB_DBT_USERMEM;
- /* Set cursor */
- if (db->dbp->del (db->dbp, NULL, &db_key, 0) != 0) {
- return FALSE;
- }
- break;
- }
-
- op->elt->flags &= ~KV_ELT_DIRTY;
- return TRUE;
-}
-
-/* Process operations queue */
-static gboolean
-bdb_process_queue (struct rspamd_kv_backend *backend)
-{
- struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend;
- struct bdb_op *op;
- GList *cur;
-
- cur = db->ops_queue->head;
- while (cur) {
- op = cur->data;
- if (!bdb_process_single_op (db, NULL, op)) {
- return FALSE;
- }
- cur = g_list_next (cur);
- }
-
- /* Clean the queue */
- cur = db->ops_queue->head;
- while (cur) {
- op = cur->data;
- if (op->op == BDB_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) !=
- 0) {
- /* Also clean memory */
- g_slice_free1 (ELT_SIZE (op->elt), op->elt);
- }
- g_slice_free1 (sizeof (struct bdb_op), op);
- cur = g_list_next (cur);
- }
-
- g_hash_table_remove_all (db->ops_hash);
- g_queue_clear (db->ops_queue);
-
- return TRUE;
-
-}
-
-/* Backend callbacks */
-static void
-rspamd_bdb_init (struct rspamd_kv_backend *backend)
-{
- struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend;
- guint32 flags;
- gint ret;
-
- if ((ret = db_env_create (&db->envp, 0)) != 0) {
- /* Cannot create environment */
- goto err;
- }
-
- flags = DB_INIT_MPOOL |
- DB_CREATE | /* Create the environment if it does not already exist. */
- DB_INIT_LOCK | /* Initialize locking. */
- DB_THREAD; /* Use threads */
-
- if ((ret = db->envp->open (db->envp, db->dirname, flags, 0)) != 0) {
- /* Cannot open environment */
- goto err;
- }
- /*
- * Configure db to perform deadlock detection internally, and to
- * choose the transaction that has performed the least amount of
- * writing to break the deadlock in the event that one is detected.
- */
- db->envp->set_lk_detect (db->envp, DB_LOCK_DEFAULT);
-
- /*
- * Avoid explicit sync on committing
- */
- db->envp->set_flags (db->envp, DB_TXN_NOSYNC, 1);
-
- flags = DB_CREATE | DB_THREAD;
- /* Create and open db pointer */
- if ((ret = db_create (&db->dbp, db->envp, 0)) != 0) {
- goto err;
- }
-
- if ((ret =
- db->dbp->open (db->dbp, NULL, db->filename, NULL, DB_HASH, flags,
- 0)) != 0) {
- goto err;
- }
-
- db->initialized = TRUE;
-
- return;
-err:
- if (db->dbp != NULL) {
- msg_err ("error opening bdb database: %s", db_strerror (ret));
- db->dbp->close (db->dbp, 0);
- }
- if (db->envp != NULL) {
- msg_err ("error opening bdb environment: %s", db_strerror (ret));
- db->envp->close (db->envp, 0);
- }
-}
-
-static gboolean
-rspamd_bdb_insert (struct rspamd_kv_backend *backend,
- gpointer key,
- guint keylen,
- struct rspamd_kv_element *elt)
-{
- struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend;
- struct bdb_op *op;
-
- if (!db->initialized) {
- return FALSE;
- }
-
- op = g_slice_alloc (sizeof (struct bdb_op));
- op->op = BDB_OP_INSERT;
- op->elt = elt;
- elt->flags |= KV_ELT_DIRTY;
-
- g_queue_push_head (db->ops_queue, op);
- g_hash_table_insert (db->ops_hash, elt, op);
-
- if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >=
- db->sync_ops) {
- return bdb_process_queue (backend);
- }
-
- return TRUE;
-}
-
-static gboolean
-rspamd_bdb_replace (struct rspamd_kv_backend *backend,
- gpointer key,
- guint keylen,
- struct rspamd_kv_element *elt)
-{
- struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend;
- struct bdb_op *op;
-
- if (!db->initialized) {
- return FALSE;
- }
-
- op = g_slice_alloc (sizeof (struct bdb_op));
- op->op = BDB_OP_REPLACE;
- op->elt = elt;
- elt->flags |= KV_ELT_DIRTY;
-
- g_queue_push_head (db->ops_queue, op);
- g_hash_table_insert (db->ops_hash, elt, op);
-
- if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >=
- db->sync_ops) {
- return bdb_process_queue (backend);
- }
-
- return TRUE;
-}
-
-static struct rspamd_kv_element *
-rspamd_bdb_lookup (struct rspamd_kv_backend *backend, gpointer key,
- guint keylen)
-{
- struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend;
- struct bdb_op *op;
- DBT db_key, db_data;
- struct rspamd_kv_element *elt = NULL;
- struct rspamd_kv_element search_elt;
-
- search_elt.keylen = keylen;
- search_elt.p = key;
-
- if (!db->initialized) {
- return NULL;
- }
- /* First search in ops queue */
- if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) {
- if (op->op == BDB_OP_DELETE) {
- /* To delete, so assume it as not found */
- return NULL;
- }
- return op->elt;
- }
-
- memset (&db_key, 0, sizeof(DBT));
- memset (&db_data, 0, sizeof(DBT));
- db_key.size = keylen;
- db_key.data = key;
- db_data.flags = DB_DBT_MALLOC;
-
- if (db->dbp->get (db->dbp, NULL, &db_key, &db_data, 0) == 0) {
- elt = db_data.data;
- elt->flags &= ~KV_ELT_DIRTY;
- }
-
- return elt;
-}
-
-static void
-rspamd_bdb_delete (struct rspamd_kv_backend *backend, gpointer key,
- guint keylen)
-{
- struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend;
- struct bdb_op *op;
- struct rspamd_kv_element *elt;
- struct rspamd_kv_element search_elt;
-
- search_elt.keylen = keylen;
- search_elt.p = key;
-
- if (!db->initialized) {
- return;
- }
-
- if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) {
- op->op = BDB_OP_DELETE;
- return;
- }
-
- elt = rspamd_bdb_lookup (backend, key, keylen);
- if (elt == NULL) {
- return;
- }
- op = g_slice_alloc (sizeof (struct bdb_op));
- op->op = BDB_OP_DELETE;
- op->elt = elt;
- elt->flags |= KV_ELT_DIRTY;
-
- g_queue_push_head (db->ops_queue, op);
- g_hash_table_insert (db->ops_hash, elt, op);
-
- if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >=
- db->sync_ops) {
- bdb_process_queue (backend);
- }
-
- return;
-}
-
-static void
-rspamd_bdb_destroy (struct rspamd_kv_backend *backend)
-{
- struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend;
-
- if (db->initialized) {
- bdb_process_queue (backend);
- if (db->dbp != NULL) {
- db->dbp->close (db->dbp, 0);
- }
- if (db->envp != NULL) {
- db->envp->close (db->envp, 0);
- }
- g_free (db->filename);
- g_free (db->dirname);
- g_queue_free (db->ops_queue);
- g_hash_table_unref (db->ops_hash);
- g_slice_free1 (sizeof (struct rspamd_bdb_backend), db);
- }
-}
-
-/* Create new bdb backend */
-struct rspamd_kv_backend *
-rspamd_kv_bdb_new (const gchar *filename, guint sync_ops)
-{
- struct rspamd_bdb_backend *new;
- struct stat st;
- gchar *dirname;
-
- if (filename == NULL) {
- return NULL;
- }
- dirname = g_path_get_dirname (filename);
- if (dirname == NULL || stat (dirname, &st) == -1 || !S_ISDIR (st.st_mode)) {
- /* Inaccessible path */
- if (dirname != NULL) {
- g_free (dirname);
- }
- msg_err ("invalid file: %s", filename);
- return NULL;
- }
-
- new = g_slice_alloc0 (sizeof (struct rspamd_bdb_backend));
- new->dirname = dirname;
- new->filename = g_strdup (filename);
- new->sync_ops = sync_ops;
- new->ops_queue = g_queue_new ();
- new->ops_hash = g_hash_table_new (kv_elt_hash_func, kv_elt_compare_func);
-
- /* Init callbacks */
- new->init_func = rspamd_bdb_init;
- new->insert_func = rspamd_bdb_insert;
- new->lookup_func = rspamd_bdb_lookup;
- new->delete_func = rspamd_bdb_delete;
- new->replace_func = rspamd_bdb_replace;
- new->sync_func = bdb_process_queue;
- new->destroy_func = rspamd_bdb_destroy;
-
- return (struct rspamd_kv_backend *)new;
-}
+++ /dev/null
-/* Copyright (c) 2010, Vsevolod Stakhov
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#ifndef KVSTORAGE_BDB_H_
-#define KVSTORAGE_BDB_H_
-
-#include "config.h"
-#include "kvstorage.h"
-
-#ifdef WITH_DB
-
-/* Create new bdb backend */
-struct rspamd_kv_backend * rspamd_kv_bdb_new (const gchar *filename,
- guint sync_ops);
-
-#endif
-
-#endif /* KVSTORAGE_BDB_H_ */
+++ /dev/null
-/* Copyright (c) 2010, Vsevolod Stakhov
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "kvstorage_config.h"
-#include "main.h"
-#include "cfg_xml.h"
-#ifdef WITH_DB
-#include "kvstorage_bdb.h"
-#endif
-#ifdef WITH_SQLITE
-#include "kvstorage_sqlite.h"
-#endif
-#include "kvstorage_file.h"
-
-#define FILE_STORAGE_LEVELS 3
-
-
-/* Global hash of storages indexed by id */
-GHashTable *storages = NULL;
-/* Last used id for explicit numbering */
-gint last_id = 0;
-
-struct kvstorage_config_parser {
- enum {
- KVSTORAGE_STATE_INIT,
- KVSTORAGE_STATE_PARAM,
- KVSTORAGE_STATE_BACKEND,
- KVSTORAGE_STATE_EXPIRE,
- KVSTORAGE_STATE_ID,
- KVSTORAGE_STATE_NAME,
- KVSTORAGE_STATE_CACHE_TYPE,
- KVSTORAGE_STATE_CACHE_MAX_ELTS,
- KVSTORAGE_STATE_CACHE_MAX_MEM,
- KVSTORAGE_STATE_CACHE_NO_OVERWRITE,
- KVSTORAGE_STATE_BACKEND_TYPE,
- KVSTORAGE_STATE_BACKEND_FILENAME,
- KVSTORAGE_STATE_BACKEND_SYNC_OPS,
- KVSTORAGE_STATE_BACKEND_DO_FSYNC,
- KVSTORAGE_STATE_BACKEND_DO_REF,
- KVSTORAGE_STATE_EXPIRE_TYPE,
- KVSTORAGE_STATE_ERROR
- } state;
- struct kvstorage_config *current_storage;
- rspamd_mempool_t *pool;
- gchar *cur_elt;
-};
-
-static void
-kvstorage_config_destroy (gpointer k)
-{
- struct kvstorage_config *kconf = k;
-
- if (kconf->name) {
- g_free (kconf->name);
- }
-
- if (kconf->storage) {
- rspamd_kv_storage_destroy (kconf->storage);
- }
-
- g_free (kconf);
-}
-
-/* Init kvstorage */
-static void
-kvstorage_init_callback (const gpointer key,
- const gpointer value,
- gpointer unused)
-{
- struct kvstorage_config *kconf = value;
- struct rspamd_kv_cache *cache;
- struct rspamd_kv_backend *backend = NULL;
- struct rspamd_kv_expire *expire = NULL;
-
- switch (kconf->cache.type) {
- case KVSTORAGE_TYPE_CACHE_HASH:
- cache = rspamd_kv_hash_new ();
- break;
- case KVSTORAGE_TYPE_CACHE_RADIX:
- cache = rspamd_kv_radix_new ();
- break;
-#ifdef WITH_JUDY
- case KVSTORAGE_TYPE_CACHE_JUDY:
- cache = rspamd_kv_judy_new ();
- break;
-#endif
- default:
- msg_err ("unknown cache type, internal error");
- return;
- }
-
- switch (kconf->backend.type) {
- case KVSTORAGE_TYPE_BACKEND_NULL:
- case KVSTORAGE_TYPE_BACKEND_MAX:
- backend = NULL;
- break;
- case KVSTORAGE_TYPE_BACKEND_FILE:
- backend = rspamd_kv_file_new (kconf->backend.filename,
- kconf->backend.sync_ops,
- FILE_STORAGE_LEVELS,
- kconf->backend.do_fsync,
- kconf->backend.do_ref);
- break;
-#ifdef WITH_DB
- case KVSTORAGE_TYPE_BACKEND_BDB:
- backend = rspamd_kv_bdb_new (kconf->backend.filename,
- kconf->backend.sync_ops);
- break;
-#endif
-#ifdef WITH_SQLITE
- case KVSTORAGE_TYPE_BACKEND_SQLITE:
- backend = rspamd_kv_sqlite_new (kconf->backend.filename,
- kconf->backend.sync_ops);
- break;
-#endif
- }
-
- switch (kconf->expire.type) {
- case KVSTORAGE_TYPE_EXPIRE_LRU:
- expire = rspamd_lru_expire_new ();
- break;
- }
-
- kconf->storage = rspamd_kv_storage_new (kconf->id,
- kconf->name,
- cache,
- backend,
- expire,
- kconf->cache.max_elements,
- kconf->cache.max_memory,
- kconf->cache.no_overwrite);
-}
-
-/* XML parse callbacks */
-/* Called for open tags <foo bar="baz"> */
-void
-kvstorage_xml_start_element (GMarkupParseContext *context,
- const gchar *element_name,
- const gchar **attribute_names,
- const gchar **attribute_values,
- gpointer user_data,
- GError **error)
-{
- struct kvstorage_config_parser *kv_parser = user_data;
-
- switch (kv_parser->state) {
- case KVSTORAGE_STATE_INIT:
- /* XXX: never get this state */
- break;
- case KVSTORAGE_STATE_PARAM:
- if (kv_parser->current_storage == NULL) {
- /* Make temporary pool */
- if (kv_parser->pool != NULL) {
- rspamd_mempool_delete (kv_parser->pool);
- }
- kv_parser->pool =
- rspamd_mempool_new (rspamd_mempool_suggest_size ());
-
- /* Create new kvstorage_config */
- kv_parser->current_storage =
- g_malloc0 (sizeof (struct kvstorage_config));
- kv_parser->current_storage->id = last_id++;
- }
- if (g_ascii_strcasecmp (element_name, "type") == 0) {
- kv_parser->state = KVSTORAGE_STATE_CACHE_TYPE;
- kv_parser->cur_elt = "type";
- }
- else if (g_ascii_strcasecmp (element_name, "max_elements") == 0) {
- kv_parser->state = KVSTORAGE_STATE_CACHE_MAX_ELTS;
- kv_parser->cur_elt = "max_elements";
- }
- else if (g_ascii_strcasecmp (element_name, "max_memory") == 0) {
- kv_parser->state = KVSTORAGE_STATE_CACHE_MAX_MEM;
- kv_parser->cur_elt = "max_memory";
- }
- else if (g_ascii_strcasecmp (element_name, "no_overwrite") == 0) {
- kv_parser->state = KVSTORAGE_STATE_CACHE_NO_OVERWRITE;
- kv_parser->cur_elt = "no_overwrite";
- }
- else if (g_ascii_strcasecmp (element_name, "id") == 0) {
- kv_parser->state = KVSTORAGE_STATE_ID;
- kv_parser->cur_elt = "id";
- }
- else if (g_ascii_strcasecmp (element_name, "name") == 0) {
- kv_parser->state = KVSTORAGE_STATE_NAME;
- kv_parser->cur_elt = "name";
- }
- else if (g_ascii_strcasecmp (element_name, "backend") == 0) {
- kv_parser->state = KVSTORAGE_STATE_BACKEND;
- kv_parser->cur_elt = "backend";
- }
- else if (g_ascii_strcasecmp (element_name, "expire") == 0) {
- kv_parser->state = KVSTORAGE_STATE_EXPIRE;
- kv_parser->cur_elt = "expire";
- }
- else {
- if (*error == NULL) {
- *error = g_error_new (
- xml_error_quark (), XML_EXTRA_ELEMENT, "element %s is unexpected",
- element_name);
- }
- kv_parser->state = KVSTORAGE_STATE_ERROR;
- }
- break;
- case KVSTORAGE_STATE_BACKEND:
- if (g_ascii_strcasecmp (element_name, "type") == 0) {
- kv_parser->state = KVSTORAGE_STATE_BACKEND_TYPE;
- kv_parser->cur_elt = "type";
- }
- else if (g_ascii_strcasecmp (element_name, "filename") == 0) {
- kv_parser->state = KVSTORAGE_STATE_BACKEND_FILENAME;
- kv_parser->cur_elt = "filename";
- }
- else if (g_ascii_strcasecmp (element_name, "sync_ops") == 0) {
- kv_parser->state = KVSTORAGE_STATE_BACKEND_SYNC_OPS;
- kv_parser->cur_elt = "sync_ops";
- }
- else if (g_ascii_strcasecmp (element_name, "fsync") == 0) {
- kv_parser->state = KVSTORAGE_STATE_BACKEND_DO_FSYNC;
- kv_parser->cur_elt = "fsync";
- }
- else if (g_ascii_strcasecmp (element_name, "ref") == 0) {
- kv_parser->state = KVSTORAGE_STATE_BACKEND_DO_REF;
- kv_parser->cur_elt = "ref";
- }
- else {
- if (*error == NULL) {
- *error = g_error_new (
- xml_error_quark (), XML_EXTRA_ELEMENT, "element %s is unexpected in backend definition",
- element_name);
- }
- kv_parser->state = KVSTORAGE_STATE_ERROR;
- }
- break;
- case KVSTORAGE_STATE_EXPIRE:
- if (g_ascii_strcasecmp (element_name, "type") == 0) {
- kv_parser->state = KVSTORAGE_STATE_EXPIRE_TYPE;
- kv_parser->cur_elt = "type";
- }
- else {
- if (*error == NULL) {
- *error = g_error_new (
- xml_error_quark (), XML_EXTRA_ELEMENT, "element %s is unexpected in expire definition",
- element_name);
- }
- kv_parser->state = KVSTORAGE_STATE_ERROR;
- }
- break;
- default:
- /* Do nothing at other states */
- break;
- }
-
-}
-
-#define CHECK_TAG(s) \
- do { \
- if (g_ascii_strcasecmp (element_name, kv_parser->cur_elt) == 0) { \
- kv_parser->state = (s); \
- } \
- else { \
- if (*error == NULL) *error = g_error_new ( \
- xml_error_quark (), XML_UNMATCHED_TAG, "element %s is unexpected in this state, expected %s", element_name, \
- kv_parser->cur_elt); \
- kv_parser->state = KVSTORAGE_STATE_ERROR; \
- } \
- } while (0)
-
-/* Called for close tags </foo> */
-void
-kvstorage_xml_end_element (GMarkupParseContext *context,
- const gchar *element_name,
- gpointer user_data,
- GError **error)
-{
- struct kvstorage_config_parser *kv_parser = user_data;
-
- switch (kv_parser->state) {
- case KVSTORAGE_STATE_INIT:
- case KVSTORAGE_STATE_PARAM:
- if (g_ascii_strcasecmp (element_name, "keystorage") == 0) {
- /* XXX: Init actual storage */
- return;
- }
- if (*error == NULL) {
- *error = g_error_new (
- xml_error_quark (), XML_EXTRA_ELEMENT, "end element %s is unexpected, expected start element",
- element_name);
- }
- kv_parser->state = KVSTORAGE_STATE_ERROR;
- break;
- case KVSTORAGE_STATE_ID:
- case KVSTORAGE_STATE_NAME:
- case KVSTORAGE_STATE_CACHE_TYPE:
- case KVSTORAGE_STATE_CACHE_MAX_ELTS:
- case KVSTORAGE_STATE_CACHE_MAX_MEM:
- case KVSTORAGE_STATE_CACHE_NO_OVERWRITE:
- CHECK_TAG (KVSTORAGE_STATE_PARAM);
- break;
- case KVSTORAGE_STATE_BACKEND_TYPE:
- case KVSTORAGE_STATE_BACKEND_FILENAME:
- case KVSTORAGE_STATE_BACKEND_SYNC_OPS:
- case KVSTORAGE_STATE_BACKEND_DO_FSYNC:
- case KVSTORAGE_STATE_BACKEND_DO_REF:
- CHECK_TAG (KVSTORAGE_STATE_BACKEND);
- break;
- case KVSTORAGE_STATE_EXPIRE_TYPE:
- CHECK_TAG (KVSTORAGE_STATE_EXPIRE);
- break;
- case KVSTORAGE_STATE_BACKEND:
- if (g_ascii_strcasecmp (element_name, "backend") == 0) {
- kv_parser->state = KVSTORAGE_STATE_PARAM;
- }
- else {
- if (*error == NULL) {
- *error = g_error_new (
- xml_error_quark (), XML_EXTRA_ELEMENT, "element %s is unexpected",
- element_name);
- }
- kv_parser->state = KVSTORAGE_STATE_ERROR;
- }
- break;
- case KVSTORAGE_STATE_EXPIRE:
- if (g_ascii_strcasecmp (element_name, "expire") == 0) {
- kv_parser->state = KVSTORAGE_STATE_PARAM;
- }
- else {
- if (*error == NULL) {
- *error = g_error_new (
- xml_error_quark (), XML_EXTRA_ELEMENT, "element %s is unexpected",
- element_name);
- }
- kv_parser->state = KVSTORAGE_STATE_ERROR;
- }
- break;
- default:
- /* Do nothing at other states */
- break;
- }
-}
-#undef CHECK_TAG
-
-/* text is not nul-terminated */
-void
-kvstorage_xml_text (GMarkupParseContext *context,
- const gchar *text,
- gsize text_len,
- gpointer user_data,
- GError **error)
-{
- struct kvstorage_config_parser *kv_parser = user_data;
- gchar *err_str;
-
- /* Strip space symbols */
- while (*text && g_ascii_isspace (*text)) {
- text++;
- }
- if (*text == '\0') {
- /* Skip empty text */
- return;
- }
-
- switch (kv_parser->state) {
- case KVSTORAGE_STATE_INIT:
- case KVSTORAGE_STATE_PARAM:
- if (*error == NULL) {
- *error = g_error_new (
- xml_error_quark (), XML_EXTRA_ELEMENT,
- "text is unexpected, expected start element");
- }
- kv_parser->state = KVSTORAGE_STATE_ERROR;
- break;
- case KVSTORAGE_STATE_ID:
- kv_parser->current_storage->id = strtoul (text, &err_str, 10);
- if ((gsize)(err_str - text) != text_len) {
- if (*error == NULL) {
- *error = g_error_new (
- xml_error_quark (), XML_EXTRA_ELEMENT, "invalid number: %*s",
- (int)text_len, text);
- }
- kv_parser->state = KVSTORAGE_STATE_ERROR;
- }
- else {
- last_id++;
- last_id = MAX (kv_parser->current_storage->id, last_id);
- }
- break;
- case KVSTORAGE_STATE_NAME:
- kv_parser->current_storage->name = g_malloc (text_len + 1);
- rspamd_strlcpy (kv_parser->current_storage->name, text, text_len + 1);
- break;
- case KVSTORAGE_STATE_CACHE_MAX_ELTS:
- kv_parser->current_storage->cache.max_elements =
- rspamd_config_parse_limit (text, text_len);
- break;
- case KVSTORAGE_STATE_CACHE_MAX_MEM:
- kv_parser->current_storage->cache.max_memory =
- rspamd_config_parse_limit (text, text_len);
- break;
- case KVSTORAGE_STATE_CACHE_NO_OVERWRITE:
- kv_parser->current_storage->cache.no_overwrite =
- rspamd_config_parse_flag (text);
- break;
- case KVSTORAGE_STATE_CACHE_TYPE:
- if (g_ascii_strncasecmp (text, "hash",
- MIN (text_len, sizeof ("hash") - 1)) == 0) {
- kv_parser->current_storage->cache.type = KVSTORAGE_TYPE_CACHE_HASH;
- }
- else if (g_ascii_strncasecmp (text, "radix",
- MIN (text_len, sizeof ("radix") - 1)) == 0) {
- kv_parser->current_storage->cache.type = KVSTORAGE_TYPE_CACHE_RADIX;
- }
-#ifdef WITH_JUDY
- else if (g_ascii_strncasecmp (text, "judy",
- MIN (text_len, sizeof ("judy") - 1)) == 0) {
- kv_parser->current_storage->cache.type = KVSTORAGE_TYPE_CACHE_JUDY;
- }
-#endif
- else {
- if (*error == NULL) {
- *error = g_error_new (
- xml_error_quark (), XML_EXTRA_ELEMENT, "invalid cache type: %*s",
- (int)text_len, text);
- }
- kv_parser->state = KVSTORAGE_STATE_ERROR;
- }
- break;
- case KVSTORAGE_STATE_BACKEND_TYPE:
- if (g_ascii_strncasecmp (text, "null",
- MIN (text_len, sizeof ("null") - 1)) == 0) {
- kv_parser->current_storage->backend.type =
- KVSTORAGE_TYPE_BACKEND_NULL;
- }
- else if (g_ascii_strncasecmp (text, "file",
- MIN (text_len, sizeof ("file") - 1)) == 0) {
- kv_parser->current_storage->backend.type =
- KVSTORAGE_TYPE_BACKEND_FILE;
- }
-#ifdef WITH_DB
- else if (g_ascii_strncasecmp (text, "bdb",
- MIN (text_len, sizeof ("bdb") - 1)) == 0) {
- kv_parser->current_storage->backend.type =
- KVSTORAGE_TYPE_BACKEND_BDB;
- }
-#endif
-#ifdef WITH_SQLITE
- else if (g_ascii_strncasecmp (text, "sqlite",
- MIN (text_len, sizeof ("sqlite") - 1)) == 0) {
- kv_parser->current_storage->backend.type =
- KVSTORAGE_TYPE_BACKEND_SQLITE;
- }
-#endif
- else {
- if (*error == NULL) {
- *error = g_error_new (
- xml_error_quark (), XML_EXTRA_ELEMENT, "invalid backend type: %*s",
- (int)text_len, text);
- }
- kv_parser->state = KVSTORAGE_STATE_ERROR;
- }
- break;
- case KVSTORAGE_STATE_BACKEND_FILENAME:
- kv_parser->current_storage->backend.filename = g_malloc (text_len + 1);
- rspamd_strlcpy (kv_parser->current_storage->backend.filename,
- text,
- text_len + 1);
- break;
- case KVSTORAGE_STATE_BACKEND_SYNC_OPS:
- kv_parser->current_storage->backend.sync_ops =
- rspamd_config_parse_limit (text, text_len);
- break;
- case KVSTORAGE_STATE_BACKEND_DO_FSYNC:
- kv_parser->current_storage->backend.do_fsync =
- rspamd_config_parse_flag (text);
- break;
- case KVSTORAGE_STATE_BACKEND_DO_REF:
- kv_parser->current_storage->backend.do_ref = rspamd_config_parse_flag (
- text);
- break;
- case KVSTORAGE_STATE_EXPIRE_TYPE:
- if (g_ascii_strncasecmp (text, "lru",
- MIN (text_len, sizeof ("lru") - 1)) == 0) {
- kv_parser->current_storage->expire.type = KVSTORAGE_TYPE_EXPIRE_LRU;
- }
- else {
- if (*error == NULL) {
- *error = g_error_new (
- xml_error_quark (), XML_EXTRA_ELEMENT, "invalid expire type: %*s",
- (int)text_len, text);
- }
- kv_parser->state = KVSTORAGE_STATE_ERROR;
- }
- break;
- default:
- /* Do nothing at other states */
- break;
- }
-
-}
-
-/* Called on error, including one set by other
- * methods in the vtable. The GError should not be freed.
- */
-void
-kvstorage_xml_error (GMarkupParseContext *context,
- GError *error,
- gpointer user_data)
-{
- msg_err ("kvstorage xml parser error: %s", error->message);
-}
-
-/*
- * Cleanup kvstorage after end tag was read
- */
-static void
-kvstorage_cleanup (gpointer ud)
-{
- struct kvstorage_config_parser *kv_parser = ud;
-
- g_hash_table_insert (storages,
- &kv_parser->current_storage->id,
- kv_parser->current_storage);
- kv_parser->state = KVSTORAGE_STATE_INIT;
- g_hash_table_foreach (storages, kvstorage_init_callback, NULL);
- kv_parser->current_storage = NULL;
-}
-
-/** Public API */
-
-/* Init subparser of kvstorage config */
-void
-init_kvstorage_config (void)
-{
- GMarkupParser *parser;
- struct kvstorage_config_parser *kv_parser;
-
- if (storages == NULL) {
- storages = g_hash_table_new_full (g_int_hash,
- g_int_equal,
- NULL,
- kvstorage_config_destroy);
- }
- else {
- /* Create new global table */
- g_hash_table_destroy (storages);
- storages = g_hash_table_new_full (g_int_hash,
- g_int_equal,
- NULL,
- kvstorage_config_destroy);
- }
-
- /* Create and register subparser */
- parser = g_malloc0 (sizeof (GMarkupParser));
- parser->start_element = kvstorage_xml_start_element;
- parser->end_element = kvstorage_xml_end_element;
- parser->error = kvstorage_xml_error;
- parser->text = kvstorage_xml_text;
-
- kv_parser = g_malloc0 (sizeof (struct kvstorage_config_parser));
- kv_parser->state = KVSTORAGE_STATE_PARAM;
- kv_parser->pool = rspamd_mempool_new (rspamd_mempool_suggest_size ());
-
- register_subparser ("keystorage", 0, parser, kvstorage_cleanup, kv_parser);
-}
-
-/* Get configuration for kvstorage with specified ID */
-struct kvstorage_config *
-get_kvstorage_config (gint id)
-{
- if (storages == NULL) {
- return NULL;
- }
- return g_hash_table_lookup (storages, &id);
-}
-
-void
-destroy_kvstorage_config (void)
-{
- if (storages != NULL) {
- g_hash_table_destroy (storages);
- storages = NULL;
- }
-}
+++ /dev/null
-/* Copyright (c) 2010, Vsevolod Stakhov
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#ifndef KVSTORAGE_CONFIG_H_
-#define KVSTORAGE_CONFIG_H_
-
-#include "config.h"
-#include "kvstorage.h"
-
-/* Type of kvstorage cache */
-enum kvstorage_cache_type {
- KVSTORAGE_TYPE_CACHE_HASH,
- KVSTORAGE_TYPE_CACHE_RADIX,
-#ifdef WITH_JUDY
- KVSTORAGE_TYPE_CACHE_JUDY,
-#endif
- KVSTORAGE_TYPE_MAX = 255
-};
-
-/* Type of kvstorage backend */
-enum kvstorage_backend_type {
- KVSTORAGE_TYPE_BACKEND_NULL = 0,
- KVSTORAGE_TYPE_BACKEND_FILE,
-#ifdef WITH_DB
- KVSTORAGE_TYPE_BACKEND_BDB,
-#endif
-#ifdef WITH_SQLITE
- KVSTORAGE_TYPE_BACKEND_SQLITE,
-#endif
- KVSTORAGE_TYPE_BACKEND_MAX = 255
-};
-
-/* Type of kvstorage expire */
-enum kvstorage_expire_type {
- KVSTORAGE_TYPE_EXPIRE_LRU
-};
-
-/* Cache config */
-struct kvstorage_cache_config {
- gsize max_elements;
- gsize max_memory;
- gboolean no_overwrite;
- enum kvstorage_cache_type type;
-};
-
-/* Backend config */
-struct kvstorage_backend_config {
- enum kvstorage_backend_type type;
- gchar *filename;
- guint sync_ops;
- gboolean do_fsync;
- gboolean do_ref;
-};
-
-
-/* Expire config */
-struct kvstorage_expire_config {
- enum kvstorage_expire_type type;
-};
-
-/* The main keystorage config */
-struct kvstorage_config {
- gint id;
- gchar *name;
- struct kvstorage_cache_config cache;
- struct kvstorage_backend_config backend;
- struct kvstorage_expire_config expire;
- struct rspamd_kv_storage *storage;
-};
-
-/* Init subparser of kvstorage config */
-void init_kvstorage_config (void);
-
-/* Get configuration for kvstorage with specified ID */
-struct kvstorage_config * get_kvstorage_config (gint id);
-
-void destroy_kvstorage_config (void);
-
-#endif /* KVSTORAGE_CONFIG_H_ */
+++ /dev/null
-/* Copyright (c) 2010, Vsevolod Stakhov
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#include "config.h"
-#include "kvstorage.h"
-#include "kvstorage_file.h"
-#include "util.h"
-#include "main.h"
-
-struct file_op {
- struct rspamd_kv_element *elt;
- enum {
- FILE_OP_INSERT,
- FILE_OP_DELETE,
- FILE_OP_REPLACE
- } op;
- guint32 ref;
-};
-
-/* Main file structure */
-struct rspamd_file_backend {
- backend_init init_func; /*< this callback is called on kv storage initialization */
- backend_insert insert_func; /*< this callback is called when element is inserted */
- backend_replace replace_func; /*< this callback is called when element is replaced */
- backend_lookup lookup_func; /*< this callback is used for lookup of element */
- backend_delete delete_func; /*< this callback is called when an element is deleted */
- backend_sync sync_func; /*< this callback is called when backend need to be synced */
- backend_incref incref_func; /*< this callback is called when element must be ref'd */
- backend_destroy destroy_func; /*< this callback is used for destroying all elements inside backend */
- gchar *filename;
- gchar *dirname;
- guint dirlen;
- guint sync_ops;
- guint levels;
- GQueue *ops_queue;
- GHashTable *ops_hash;
- gboolean do_fsync;
- gboolean do_ref;
- gboolean initialized;
-};
-
-static const gchar hexdigits[] =
-{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e',
- 'f'};
-
-/* Generate file name for operation */
-static gboolean
-get_file_name (struct rspamd_file_backend *db,
- gchar *key,
- guint keylen,
- gchar *filebuf,
- guint buflen)
-{
- gchar *p = filebuf, *end = filebuf + buflen,
- *k = key, t;
- guint i;
-
- /* First copy backend dirname to file buf */
- if (buflen <= db->dirlen) {
- return FALSE;
- }
- memcpy (p, db->dirname, db->dirlen);
- p += db->dirlen;
- *p++ = G_DIR_SEPARATOR;
- for (i = 0; i < MIN (keylen, db->levels); i++) {
- if (p == end) {
- /* Filebuf is not large enough */
- return FALSE;
- }
- t = *k;
- *p++ = hexdigits[(t & 0xf) ^ ((t & 0xf0) >> 4)];
- *p++ = G_DIR_SEPARATOR;
- k++;
- }
- /* Now we have directory, append base64 encoded filename */
- k = key;
- if (end - p < (gint)keylen * 2 + 1) {
- /* Filebuf is not large enough */
- return FALSE;
- }
-
- i = 0;
- while (k < key + keylen) {
- t = *k;
- *p++ = hexdigits[(t >> 4) & 0xf];
- *p++ = hexdigits[t & 0xf];
- k++;
- }
- *p = '\0';
-
- return TRUE;
-}
-
-/* Read reference from specified file */
-static guint32
-file_get_ref (gint fd)
-{
- guint32 target;
-
- if (read (fd, &target, sizeof (guint32)) != sizeof (guint32)) {
- return 0;
- }
-
- return target;
-}
-
-/* Set reference to specified file */
-static gboolean
-file_set_ref (gint fd, guint32 ref)
-{
- if (write (fd, &ref, sizeof (guint32)) != sizeof (guint32)) {
- return FALSE;
- }
-
- return TRUE;
-}
-
-/*
- * Open file, set posix_fadvise and all necessary flags
- */
-static gint
-file_open_fd (const gchar *path, gsize *len, gint flags)
-{
- gint fd;
- struct stat st;
-
- if ((flags & O_CREAT) != 0) {
- /* Open file */
- if ((fd = open (path, flags, S_IRUSR | S_IWUSR | S_IRGRP)) != -1) {
- rspamd_fallocate (fd, 0, *len);
-#ifdef HAVE_FADVISE
- posix_fadvise (fd, 0, *len, POSIX_FADV_SEQUENTIAL);
-#endif
- }
- }
- else {
- /* Open file */
- if ((fd = open (path, flags)) == -1) {
- return -1;
- }
-
- if (fstat (fd, &st) == -1) {
- close (fd);
- return -1;
- }
-
-#ifdef HAVE_FADVISE
- posix_fadvise (fd, 0, st.st_size, POSIX_FADV_SEQUENTIAL);
-#endif
- *len = st.st_size;
- }
-
- return fd;
-}
-
-/* Process single file operation */
-static gboolean
-file_process_single_op (struct rspamd_file_backend *db,
- struct file_op *op,
- gint *pfd)
-{
- gchar filebuf[PATH_MAX];
- gint fd;
- gsize len;
- struct iovec iov[2];
- guint32 ref;
-
- /* Get filename */
- if (!get_file_name (db, ELT_KEY (op->elt), op->elt->keylen, filebuf,
- sizeof (filebuf))) {
- return FALSE;
- }
-
- if (db->do_ref) {
- len = ELT_SIZE (op->elt) + sizeof (guint32);
- }
- else {
- len = ELT_SIZE (op->elt);
- }
-
- if (op->op == FILE_OP_DELETE) {
- if (db->do_ref) {
- if ((fd = file_open_fd (filebuf, &len, O_RDWR)) == -1) {
- *pfd = -1;
- return FALSE;
- }
- if ((ref = file_get_ref (fd)) <= 1) {
- /* Refcount is not enough, remove file */
- close (fd);
- *pfd = -1;
- return unlink (filebuf) != -1;
- }
- else {
- /* Decrease ref */
- lseek (fd, 0, SEEK_SET);
- if (!file_set_ref (fd, --ref)) {
- *pfd = fd;
- return FALSE;
- }
- }
- }
- else {
- *pfd = -1;
- return unlink (filebuf) != -1;
- }
- }
- else {
- if ((fd =
- file_open_fd (filebuf, &len, O_CREAT | O_WRONLY | O_TRUNC)) == -1) {
- *pfd = -1;
- return FALSE;
- }
- if (db->do_ref) {
- iov[0].iov_base = &op->ref;
- iov[0].iov_len = sizeof (guint32);
- iov[1].iov_base = op->elt;
- iov[1].iov_len = ELT_SIZE (op->elt);
- if (writev (fd, iov, G_N_ELEMENTS (iov)) == -1) {
- msg_info ("%d: %s", errno, strerror (errno));
- *pfd = fd;
- return FALSE;
- }
- }
- else {
- if (write (fd, op->elt, ELT_SIZE (op->elt)) == -1) {
- msg_info ("%d: %s", errno, strerror (errno));
- *pfd = fd;
- return FALSE;
- }
- }
- }
-
- *pfd = fd;
- return TRUE;
-}
-
-/* Sync vector of descriptors */
-static void
-file_sync_fds (gint *fds, gint len, gboolean do_fsync)
-{
- gint i, fd;
-
- for (i = 0; i < len; i++) {
- fd = fds[i];
- if (fd != -1) {
- if (do_fsync) {
-#ifdef HAVE_FDATASYNC
- fdatasync (fd);
-#else
- fsync (fd);
-#endif
- }
- close (fd);
- }
- }
-}
-
-/* Process operations queue */
-static gboolean
-file_process_queue (struct rspamd_kv_backend *backend)
-{
- struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend;
- struct file_op *op;
- GList *cur;
- gint *fds, i = 0, len;
-
- len = g_queue_get_length (db->ops_queue);
- if (len == 0) {
- /* Nothing to process */
- return TRUE;
- }
-
- fds = g_slice_alloc (len * sizeof (gint));
- cur = db->ops_queue->head;
- while (cur) {
- op = cur->data;
- if (!file_process_single_op (db, op, &fds[i])) {
- file_sync_fds (fds, i, db->do_fsync);
- g_slice_free1 (len * sizeof (gint), fds);
- return FALSE;
- }
- i++;
- cur = g_list_next (cur);
- }
-
- file_sync_fds (fds, i, db->do_fsync);
- g_slice_free1 (len * sizeof (gint), fds);
-
- /* Clean the queue */
- g_hash_table_remove_all (db->ops_hash);
- cur = db->ops_queue->head;
- while (cur) {
- op = cur->data;
- if (op->op == FILE_OP_DELETE ||
- ((op->elt->flags & KV_ELT_NEED_FREE) != 0 &&
- (op->elt->flags & KV_ELT_NEED_INSERT) == 0)) {
- /* Also clean memory */
- g_slice_free1 (ELT_SIZE (op->elt), op->elt);
- }
- else {
- /* Unset dirty flag */
- op->elt->flags &= ~KV_ELT_DIRTY;
- }
- g_slice_free1 (sizeof (struct file_op), op);
- cur = g_list_next (cur);
- }
-
- g_queue_clear (db->ops_queue);
-
- return TRUE;
-
-}
-
-
-/* Make 16 directories for each level */
-static gboolean
-rspamd_recursive_mkdir (guint levels)
-{
- guint i;
- gchar nbuf[5];
-
- /* Create directories for backend */
- if (levels > 0) {
- /* Create 16 directories */
- for (i = 0; i < 16; i++) {
- rspamd_snprintf (nbuf, sizeof (nbuf), "./%c", hexdigits[i]);
- if (mkdir (nbuf, 0755) != 0 && errno != EEXIST) {
- msg_info ("cannot create directory %s: %s", nbuf,
- strerror (errno));
- return FALSE;
- }
- else if (levels > 1) {
- if (chdir (nbuf) == -1) {
- msg_err ("chdir to %s failed: %s", nbuf, strerror (errno));
- return FALSE;
- }
- if (!rspamd_recursive_mkdir (levels - 1)) {
- return FALSE;
- }
- if (chdir ("../") == -1) {
- msg_err ("chdir to ../ failed: %s", strerror (errno));
- return FALSE;
- }
- }
- }
- }
- return TRUE;
-
-}
-
-/* Backend callbacks */
-static void
-rspamd_file_init (struct rspamd_kv_backend *backend)
-{
- struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend;
- gchar pathbuf[PATH_MAX];
-
- /* Save current directory */
- if (getcwd (pathbuf, sizeof (pathbuf) - 1) == NULL) {
- pathbuf[0] = '\0';
- msg_err ("getcwd failed: %s", strerror (errno));
- goto err;
- }
-
- /* Chdir to the working dir */
- if (chdir (db->dirname) == -1) {
- msg_err ("chdir failed: %s", strerror (errno));
- goto err;
- }
-
- /* Create directories for backend */
- if (!rspamd_recursive_mkdir (db->levels)) {
- goto err;
- }
-
- db->initialized = TRUE;
-
- if (chdir (pathbuf) == -1) {
- msg_err ("chdir to %s failed: %s", pathbuf, strerror (errno));
- }
- return;
-err:
- if (pathbuf[0] != '\0') {
- if (chdir (pathbuf) == -1) {
- msg_err ("chdir to %s failed: %s", pathbuf, strerror (errno));
- }
- }
-}
-
-static gboolean
-rspamd_file_insert (struct rspamd_kv_backend *backend,
- gpointer key,
- guint keylen,
- struct rspamd_kv_element *elt)
-{
- struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend;
- struct file_op *op;
- struct rspamd_kv_element search_elt;
-
- search_elt.keylen = keylen;
- search_elt.p = key;
-
- if (!db->initialized) {
- return FALSE;
- }
-
- if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) {
- /* We found another op with such key in this queue */
- if (op->op == FILE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) !=
- 0) {
- /* Also clean memory */
- g_hash_table_steal (db->ops_hash, &search_elt);
- g_slice_free1 (ELT_SIZE (op->elt), op->elt);
- }
- op->op = FILE_OP_INSERT;
- op->ref++;
- op->elt = elt;
- elt->flags |= KV_ELT_DIRTY;
- g_hash_table_insert (db->ops_hash, elt, op);
- }
- else {
- op = g_slice_alloc (sizeof (struct file_op));
- op->op = FILE_OP_INSERT;
- op->elt = elt;
- op->ref = 1;
- elt->flags |= KV_ELT_DIRTY;
-
- g_queue_push_head (db->ops_queue, op);
- g_hash_table_insert (db->ops_hash, elt, op);
- }
-
- if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >=
- db->sync_ops) {
- return file_process_queue (backend);
- }
-
- return TRUE;
-}
-
-static gboolean
-rspamd_file_replace (struct rspamd_kv_backend *backend,
- gpointer key,
- guint keylen,
- struct rspamd_kv_element *elt)
-{
- struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend;
- struct file_op *op;
- struct rspamd_kv_element search_elt;
-
- search_elt.keylen = keylen;
- search_elt.p = key;
-
- if (!db->initialized) {
- return FALSE;
- }
- if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) {
- /* We found another op with such key in this queue */
- if (op->op == FILE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) !=
- 0) {
- /* Also clean memory */
- g_hash_table_steal (db->ops_hash, &search_elt);
- g_slice_free1 (ELT_SIZE (op->elt), op->elt);
- }
- op->op = FILE_OP_REPLACE;
- op->elt = elt;
- elt->flags |= KV_ELT_DIRTY;
- g_hash_table_insert (db->ops_hash, elt, op);
- }
- else {
- op = g_slice_alloc (sizeof (struct file_op));
- op->op = FILE_OP_REPLACE;
- op->elt = elt;
- op->ref = 1;
- elt->flags |= KV_ELT_DIRTY;
-
- g_queue_push_head (db->ops_queue, op);
- g_hash_table_insert (db->ops_hash, elt, op);
- }
-
- if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >=
- db->sync_ops) {
- return file_process_queue (backend);
- }
-
- return TRUE;
-}
-
-static struct rspamd_kv_element *
-rspamd_file_lookup (struct rspamd_kv_backend *backend,
- gpointer key,
- guint keylen)
-{
- struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend;
- struct file_op *op;
- struct rspamd_kv_element *elt = NULL;
- gchar filebuf[PATH_MAX];
- gint fd;
- struct rspamd_kv_element search_elt;
- gsize len;
-
- search_elt.keylen = keylen;
- search_elt.p = key;
-
- if (!db->initialized) {
- return NULL;
- }
- /* First search in ops queue */
- if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) {
- if (op->op == FILE_OP_DELETE) {
- /* To delete, so assume it as not found */
- return NULL;
- }
- return op->elt;
- }
-
- /* Get filename */
- if (!get_file_name (db, key, keylen, filebuf, sizeof (filebuf))) {
- return NULL;
- }
-
- if ((fd = file_open_fd (filebuf, &len, O_RDONLY)) == -1) {
- return NULL;
- }
-
- /* Read element */
- if (db->do_ref) {
- lseek (fd, sizeof (guint32), SEEK_CUR);
- len -= sizeof (guint32);
- }
- elt = g_malloc (len);
- if (read (fd, elt, len) == -1) {
- g_free (elt);
- close (fd);
- return NULL;
- }
-
- close (fd);
-
- elt->flags &= ~(KV_ELT_DIRTY | KV_ELT_NEED_FREE);
-
- return elt;
-}
-
-static void
-rspamd_file_delete (struct rspamd_kv_backend *backend,
- gpointer key,
- guint keylen)
-{
- struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend;
- gchar filebuf[PATH_MAX];
- struct rspamd_kv_element search_elt;
- struct file_op *op;
- gsize len;
- gint fd;
- guint32 ref;
-
- if (!db->initialized) {
- return;
- }
-
- search_elt.keylen = keylen;
- search_elt.p = key;
- /* First search in ops queue */
- if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) {
- op->op = FILE_OP_DELETE;
- if (op->ref > 0) {
- op->ref--;
- }
- return;
- }
- /* Get filename */
- if (!get_file_name (db, key, keylen, filebuf, sizeof (filebuf))) {
- return;
- }
-
- if (db->do_ref) {
- if ((fd = file_open_fd (filebuf, &len, O_RDWR)) == -1) {
- return;
- }
- if ((ref = file_get_ref (fd)) <= 1) {
- /* Refcount is not enough, remove file */
- close (fd);
- unlink (filebuf);
- }
- else {
- /* Decrease ref */
- lseek (fd, 0, SEEK_SET);
- file_set_ref (fd, --ref);
- }
- return;
- }
-
- unlink (filebuf);
-}
-
-static gboolean
-rspamd_file_incref (struct rspamd_kv_backend *backend,
- gpointer key,
- guint keylen)
-{
- struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend;
- gchar filebuf[PATH_MAX];
- struct rspamd_kv_element search_elt;
- struct file_op *op;
- gsize len;
- gint fd;
- guint32 ref;
-
- if (!db->initialized) {
- return FALSE;
- }
- if (!db->do_ref) {
- return TRUE;
- }
-
- search_elt.keylen = keylen;
- search_elt.p = key;
- /* First search in ops queue */
- if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) {
- op->ref++;
- if (op->op == FILE_OP_DELETE) {
- op->op = FILE_OP_INSERT;
- }
- return TRUE;
- }
-
- /* Get filename */
- if (!get_file_name (db, key, keylen, filebuf, sizeof (filebuf))) {
- return FALSE;
- }
-
- if ((fd = file_open_fd (filebuf, &len, O_RDWR)) == -1) {
- return FALSE;
- }
-
- ref = file_get_ref (fd);
-
- /* Decrease ref */
- lseek (fd, 0, SEEK_SET);
-
- if (file_set_ref (fd, ++ref)) {
- close (fd);
- return TRUE;
- }
- else {
- close (fd);
- return FALSE;
- }
-}
-
-static void
-rspamd_file_destroy (struct rspamd_kv_backend *backend)
-{
- struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend;
-
- if (db->initialized) {
- file_process_queue (backend);
- g_free (db->filename);
- g_free (db->dirname);
- g_queue_free (db->ops_queue);
- g_hash_table_unref (db->ops_hash);
- g_slice_free1 (sizeof (struct rspamd_file_backend), db);
-
- /* Sync again */
- sync ();
- }
-}
-
-/* Create new file backend */
-struct rspamd_kv_backend *
-rspamd_kv_file_new (const gchar *filename,
- guint sync_ops,
- guint levels,
- gboolean do_fsync,
- gboolean do_ref)
-{
- struct rspamd_file_backend *new;
- struct stat st;
- gchar *dirname;
-
- if (filename == NULL) {
- return NULL;
- }
-
- dirname = g_path_get_dirname (filename);
- if (dirname == NULL || stat (dirname, &st) == -1 || !S_ISDIR (st.st_mode)) {
- /* Inaccessible path */
- if (dirname != NULL) {
- g_free (dirname);
- }
- msg_err ("invalid file: %s", filename);
- return NULL;
- }
-
- new = g_slice_alloc0 (sizeof (struct rspamd_file_backend));
- new->dirname = dirname;
- new->dirlen = strlen (dirname);
- new->filename = g_strdup (filename);
- new->sync_ops = sync_ops;
- new->levels = levels;
- new->do_fsync = do_fsync;
- new->do_ref = do_ref;
- new->ops_queue = g_queue_new ();
- new->ops_hash = g_hash_table_new (kv_elt_hash_func, kv_elt_compare_func);
-
- /* Init callbacks */
- new->init_func = rspamd_file_init;
- new->insert_func = rspamd_file_insert;
- new->lookup_func = rspamd_file_lookup;
- new->delete_func = rspamd_file_delete;
- new->replace_func = rspamd_file_replace;
- new->sync_func = file_process_queue;
- new->incref_func = rspamd_file_incref;
- new->destroy_func = rspamd_file_destroy;
-
- return (struct rspamd_kv_backend *)new;
-}
-
+++ /dev/null
-/* Copyright (c) 2010, Vsevolod Stakhov
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Rambler BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#ifndef KVSTORAGE_FILE_H_
-#define KVSTORAGE_FILE_H_
-
-#include "config.h"
-#include "kvstorage.h"
-
-/* Create new file backend */
-struct rspamd_kv_backend * rspamd_kv_file_new (const gchar *filename,
- guint sync_ops,
- guint levels,
- gboolean do_fsync,
- gboolean do_ref);
-
-
-#endif /* KVSTORAGE_FILE_H_ */
+++ /dev/null
-/* Copyright (c) 2010, Vsevolod Stakhov
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#include "config.h"
-#include "kvstorage.h"
-#include "kvstorage_config.h"
-#include "kvstorage_server.h"
-#include "cfg_file.h"
-#include "cfg_xml.h"
-#include "main.h"
-
-#define ERROR_COMMON "ERROR" CRLF
-#define ERROR_UNKNOWN_COMMAND "CLIENT_ERROR unknown command" CRLF
-#define ERROR_NOT_STORED "NOT_STORED" CRLF
-#define ERROR_EXISTS "EXISTS" CRLF
-#define ERROR_NOT_FOUND "NOT_FOUND" CRLF
-#define ERROR_INVALID_KEYSTORAGE "CLIENT_ERROR storage does not exists" CRLF
-
-#define ERROR_REDIS_OK "+OK" CRLF
-
-
-static sig_atomic_t wanna_die = 0;
-static sig_atomic_t do_reopen_log = 0;
-static sig_atomic_t soft_wanna_die = 0;
-
-/* Logging functions */
-#define thr_err(...) do { \
- g_mutex_lock (thr->log_mtx); \
- rspamd_common_log_function (rspamd_main->logger, \
- G_LOG_LEVEL_CRITICAL, \
- __FUNCTION__, \
- __VA_ARGS__); \
- g_mutex_unlock (thr->log_mtx); \
-} while (0)
-
-#define thr_warn(...) do { \
- g_mutex_lock (thr->log_mtx); \
- rspamd_common_log_function (rspamd_main->logger, \
- G_LOG_LEVEL_WARNING, \
- __FUNCTION__, \
- __VA_ARGS__); \
- g_mutex_unlock (thr->log_mtx); \
-} while (0)
-
-#define thr_info(...) do { \
- g_mutex_lock (thr->log_mtx); \
- rspamd_common_log_function (rspamd_main->logger, \
- G_LOG_LEVEL_INFO, \
- __FUNCTION__, \
- __VA_ARGS__); \
- g_mutex_unlock (thr->log_mtx); \
-} while (0)
-
-/* Init functions */
-gpointer init_keystorage (void);
-void start_keystorage (struct rspamd_worker *worker);
-
-worker_t keystorage_worker = {
- "keystorage", /* Name */
- init_keystorage, /* Init function */
- start_keystorage, /* Start function */
- TRUE, /* Has socket */
- FALSE, /* Non unique */
- TRUE, /* Non threaded */
- FALSE, /* Non killable */
- SOCK_STREAM /* TCP socket */
-};
-
-#ifndef HAVE_SA_SIGINFO
-static void
-sig_handler (gint signo)
-#else
-static void
-sig_handler (gint signo, siginfo_t *info, void *unused)
-#endif
-{
- switch (signo) {
- case SIGUSR1:
- do_reopen_log = 1;
- break;
- case SIGINT:
- case SIGTERM:
- wanna_die = 1;
- break;
- case SIGUSR2:
- soft_wanna_die = 1;
- break;
- }
-}
-
-gpointer
-init_keystorage (void)
-{
- struct kvstorage_worker_ctx *ctx;
- GQuark type;
-
- type = g_quark_try_string ("keystorage");
- ctx = g_malloc0 (sizeof (struct kvstorage_worker_ctx));
- ctx->pool = rspamd_mempool_new (rspamd_mempool_suggest_size ());
-
- /* Set default values */
- ctx->timeout_raw = 300000;
-
- register_worker_opt (type, "timeout", xml_handle_seconds, ctx,
- G_STRUCT_OFFSET (struct kvstorage_worker_ctx, timeout_raw));
- register_worker_opt (type, "redis", xml_handle_boolean, ctx,
- G_STRUCT_OFFSET (struct kvstorage_worker_ctx, is_redis));
- return ctx;
-}
-
-/* Make post-init configuration */
-static gboolean
-config_kvstorage_worker (struct rspamd_worker *worker)
-{
- struct kvstorage_worker_ctx *ctx = worker->ctx;
-
- /* Init timeval */
- msec_to_tv (ctx->timeout_raw, &ctx->io_timeout);
-
- return TRUE;
-}
-
-/*
- * Free kvstorage session
- */
-static void
-free_kvstorage_session (struct kvstorage_session *session)
-{
- rspamd_remove_dispatcher (session->dispather);
- rspamd_mempool_delete (session->pool);
- close (session->sock);
- g_slice_free1 (sizeof (struct kvstorage_session), session);
-}
-
-/*
- * Parse kvstorage command
- */
-static gboolean
-parse_kvstorage_command (struct kvstorage_session *session, gchar *c, guint len)
-{
- if (len == 3) {
- /* Set or get command */
- if ((c[0] == 'g' || c[0] == 'G') &&
- (c[1] == 'e' || c[1] == 'E') &&
- (c[2] == 't' || c[2] == 'T')) {
- session->command = KVSTORAGE_CMD_GET;
- }
- else if ((c[0] == 's' || c[0] == 'S') &&
- (c[1] == 'e' || c[1] == 'E') &&
- (c[2] == 't' || c[2] == 'T')) {
- session->command = KVSTORAGE_CMD_SET;
- }
- else if ((c[0] == 'd' || c[0] == 'D') &&
- (c[1] == 'e' || c[1] == 'E') &&
- (c[2] == 'l' || c[2] == 'L')) {
- session->command = KVSTORAGE_CMD_DELETE;
- }
- else {
- /* Error */
- return FALSE;
- }
- }
- else if (len == 4) {
- if ((c[0] == 'i' || c[0] == 'I') &&
- (c[1] == 'n' || c[1] == 'N') &&
- (c[2] == 'c' || c[2] == 'C') &&
- (c[3] == 'r' || c[3] == 'R')) {
- session->command = KVSTORAGE_CMD_INCR;
- session->arg_data.value = 1;
- }
- else if ((c[0] == 'd' || c[0] == 'D') &&
- (c[1] == 'e' || c[1] == 'E') &&
- (c[2] == 'c' || c[2] == 'C') &&
- (c[3] == 'r' || c[3] == 'R')) {
- session->command = KVSTORAGE_CMD_DECR;
- session->arg_data.value = -1;
- }
- else if (g_ascii_strncasecmp (c, "quit", 4) == 0) {
- session->command = KVSTORAGE_CMD_QUIT;
- }
- else if (g_ascii_strncasecmp (c, "sync",
- 4) == 0 || g_ascii_strncasecmp (c, "save", 4) == 0) {
- session->command = KVSTORAGE_CMD_SYNC;
- }
- }
- else if (len == 6) {
- if ((c[0] == 'i' || c[0] == 'I') &&
- (c[1] == 'n' || c[1] == 'N') &&
- (c[2] == 'c' || c[2] == 'C') &&
- (c[3] == 'r' || c[3] == 'R') &&
- (c[4] == 'b' || c[4] == 'B') &&
- (c[5] == 'y' || c[5] == 'Y')) {
- session->command = KVSTORAGE_CMD_INCR;
- session->arg_data.value = 1;
- }
- else if ((c[0] == 'd' || c[0] == 'D') &&
- (c[1] == 'e' || c[1] == 'E') &&
- (c[2] == 'c' || c[2] == 'C') &&
- (c[3] == 'r' || c[3] == 'R') &&
- (c[4] == 'b' || c[4] == 'B') &&
- (c[5] == 'y' || c[5] == 'Y')) {
- session->command = KVSTORAGE_CMD_DECR;
- session->arg_data.value = -1;
- }
- else if ((c[0] == 'd' || c[0] == 'D') &&
- (c[1] == 'e' || c[1] == 'E') &&
- (c[2] == 'l' || c[2] == 'L') &&
- (c[3] == 'e' || c[3] == 'E') &&
- (c[4] == 't' || c[4] == 'T') &&
- (c[5] == 'e' || c[5] == 'E')) {
- session->command = KVSTORAGE_CMD_DELETE;
- }
- else if (g_ascii_strncasecmp (c, "select", 6) == 0) {
- session->command = KVSTORAGE_CMD_SELECT;
- }
- else {
- return FALSE;
- }
- }
-
- return TRUE;
-}
-
-/**
- * Parse kvstorage line
- */
-static gboolean
-parse_kvstorage_line (struct kvstorage_session *session, rspamd_fstring_t *in)
-{
- gchar *p, *c, *end;
- gint state = 0, next_state = 0;
- gboolean is_redis;
-
- p = in->begin;
- end = in->begin + in->len;
- c = p;
- is_redis = session->thr->ctx->is_redis;
-
- /* State machine for parsing */
- while (p <= end) {
- switch (state) {
- case 0:
- /* At this state we try to read identifier of storage */
- if (g_ascii_isdigit (*p)) {
- p++;
- }
- else {
- if (g_ascii_isspace (*p) && p != c) {
- /* We have some digits, so parse id */
- session->id = strtoul (c, NULL, 10);
- state = 99;
- next_state = 1;
- }
- else if (c == p) {
- if (*p != '*') {
- /* We have some character, so assume id as 0 and parse command */
- session->id = 0;
- state = 1;
- }
- else {
- /* In fact it is redis number of commands */
- c = ++p;
- state = 7;
- session->id = 0;
- }
- }
- else {
- /* We have something wrong here (like some digits and then come non-digits) */
- return FALSE;
- }
- }
- break;
- case 1:
- /* At this state we parse command */
- if (g_ascii_isalpha (*p) && p != end) {
- p++;
- }
- else {
- if (parse_kvstorage_command (session, c, p - c)) {
- switch (session->command) {
-
- case KVSTORAGE_CMD_QUIT:
- case KVSTORAGE_CMD_SYNC:
- /* Single argument command */
- state = 100;
- break;
- case KVSTORAGE_CMD_SELECT:
- /* Select command, read id next */
- state = 99;
- next_state = 6;
- break;
- default:
- /* Normal command, read key */
- state = 99;
- next_state = 2;
- }
- }
- else {
- /* Some error */
- return FALSE;
- }
- }
- break;
- case 2:
- /* Read and store key */
- if (!g_ascii_isspace (*p) && end != p) {
- p++;
- }
- else {
- if (p == c) {
- return FALSE;
- }
- else {
- session->key = rspamd_mempool_alloc (session->pool,
- p - c + 1);
- rspamd_strlcpy (session->key, c, p - c + 1);
- session->keylen = p - c;
- /* Now we must select next state based on command */
- if (session->command == KVSTORAGE_CMD_SET ||
- session->command == KVSTORAGE_CMD_INCR ||
- session->command == KVSTORAGE_CMD_DECR) {
- /* Read flags */
- state = 99;
- if (is_redis) {
- next_state = 5;
- session->flags = 0;
- session->expire = 0;
- }
- else {
- if (session->command == KVSTORAGE_CMD_SET) {
- next_state = 3;
- }
- else {
- next_state = 5;
- }
- }
- }
- else {
- /* Nothing to read for other commands */
- state = 100;
- }
- }
- }
- break;
- case 3:
- /* Read flags */
- if (g_ascii_isdigit (*p)) {
- p++;
- }
- else {
- if (g_ascii_isspace (*p)) {
- session->flags = strtoul (c, NULL, 10);
- state = 99;
- if (session->command == KVSTORAGE_CMD_SET) {
- next_state = 4;
- }
- else {
- /* INCR and DECR */
- next_state = 5;
- }
- }
- else {
- return FALSE;
- }
- }
- break;
- case 4:
- /* Read exptime */
- if (g_ascii_isdigit (*p)) {
- p++;
- }
- else {
- if (g_ascii_isspace (*p)) {
- session->expire = strtoul (c, NULL, 10);
- state = 99;
- next_state = 5;
- }
- else {
- return FALSE;
- }
- }
- break;
- case 5:
- /* Read size or incr/decr values */
- if (g_ascii_isdigit (*p)) {
- p++;
- }
- else {
- if (g_ascii_isspace (*p) || p >= end - 1) {
- if (session->command == KVSTORAGE_CMD_SET) {
- session->arg_data.length = strtoul (c, NULL, 10);
- }
- else {
- if (p != c) {
- session->arg_data.value = strtoul (c, NULL, 10);
- if (session->command == KVSTORAGE_CMD_DECR) {
- session->arg_data.value =
- -session->arg_data.value;
- }
- }
- else if (session->command == KVSTORAGE_CMD_INCR) {
- session->arg_data.value = 1;
- }
- else {
- session->arg_data.value = -1;
- }
- }
- state = 100;
- }
- else {
- return FALSE;
- }
- }
- break;
- case 6:
- /* Read index of storage */
- if (g_ascii_isdigit (*p)) {
- p++;
- }
- else {
- if (g_ascii_isspace (*p) || end == p) {
- session->id = strtoul (c, NULL, 10);
- state = 100;
- }
- else {
- return FALSE;
- }
- }
- break;
- case 7:
- /* Read arguments count */
- if (g_ascii_isdigit (*p)) {
- p++;
- }
- else {
- if (g_ascii_isspace (*p) || end == p) {
- session->argc = strtoul (c, NULL, 10);
- session->argnum = 0;
- state = 100;
- /* Switch to arglen state */
- session->state = KVSTORAGE_STATE_READ_ARGLEN;
- }
- else {
- return FALSE;
- }
- }
- break;
- case 99:
- /* Skip spaces state */
- if (g_ascii_isspace (*p)) {
- p++;
- }
- else {
- c = p;
- state = next_state;
- }
- break;
- case 100:
- /* Successful state */
- return TRUE;
- break;
- }
- }
-
- return state == 100;
-}
-
-/* Process normal kvstorage command */
-static gboolean
-kvstorage_process_command (struct kvstorage_session *session, gboolean is_redis)
-{
- gint r;
- gchar outbuf[BUFSIZ], intbuf[sizeof ("9223372036854775807")];
- gboolean res;
- struct rspamd_kv_element *elt;
- guint eltlen;
- glong longval;
-
- if (session->command == KVSTORAGE_CMD_SET) {
- session->state = KVSTORAGE_STATE_READ_DATA;
- rspamd_set_dispatcher_policy (session->dispather,
- BUFFER_CHARACTER,
- session->arg_data.length);
- }
- else if (session->command == KVSTORAGE_CMD_GET) {
- elt = rspamd_kv_storage_lookup (session->cf->storage,
- session->key,
- session->keylen,
- session->now);
- if (elt == NULL) {
- RW_R_UNLOCK (&session->cf->storage->rwlock);
- if (!is_redis) {
- return rspamd_dispatcher_write (session->dispather,
- ERROR_NOT_FOUND,
- sizeof (ERROR_NOT_FOUND) - 1,
- FALSE,
- TRUE);
- }
- else {
- return rspamd_dispatcher_write (session->dispather, "$-1" CRLF,
- sizeof ("$-1" CRLF) - 1, FALSE, TRUE);
- }
- }
- else {
- if (elt->flags & KV_ELT_INTEGER) {
- eltlen = rspamd_snprintf (intbuf,
- sizeof (intbuf),
- "%l",
- ELT_LONG (elt));
-
- }
- else {
- eltlen = elt->size;
- }
-
- if (!is_redis) {
- r = rspamd_snprintf (outbuf,
- sizeof (outbuf),
- "VALUE %s %ud %ud" CRLF,
- ELT_KEY (elt),
- elt->flags,
- eltlen);
- }
- else {
- r = rspamd_snprintf (outbuf, sizeof (outbuf), "$%ud" CRLF,
- eltlen);
- }
- if (!rspamd_dispatcher_write (session->dispather, outbuf,
- r, TRUE, FALSE)) {
- RW_R_UNLOCK (&session->cf->storage->rwlock);
- return FALSE;
- }
- if (elt->flags & KV_ELT_INTEGER) {
- if (!rspamd_dispatcher_write (session->dispather, intbuf,
- eltlen, TRUE, TRUE)) {
- RW_R_UNLOCK (&session->cf->storage->rwlock);
- return FALSE;
- }
- }
- else {
- if (!rspamd_dispatcher_write (session->dispather,
- ELT_DATA (elt), eltlen, TRUE, TRUE)) {
- RW_R_UNLOCK (&session->cf->storage->rwlock);
- return FALSE;
- }
- }
- session->elt = elt;
- if (!is_redis) {
- res = rspamd_dispatcher_write (session->dispather,
- CRLF "END" CRLF,
- sizeof (CRLF "END" CRLF) - 1,
- FALSE,
- TRUE);
- }
- else {
- res = rspamd_dispatcher_write (session->dispather, CRLF,
- sizeof (CRLF) - 1, FALSE, TRUE);
- }
- if (!res) {
- RW_R_UNLOCK (&session->cf->storage->rwlock);
- }
-
- return res;
- }
- }
- else if (session->command == KVSTORAGE_CMD_DELETE) {
- elt = rspamd_kv_storage_delete (session->cf->storage,
- session->key,
- session->keylen);
- if (elt != NULL) {
- if ((elt->flags & KV_ELT_DIRTY) == 0) {
- /* Free memory if backend has deleted this element */
- g_slice_free1 (ELT_SIZE (elt), elt);
- }
- if (!is_redis) {
- return rspamd_dispatcher_write (session->dispather,
- "DELETED" CRLF,
- sizeof ("DELETED" CRLF) - 1,
- FALSE,
- TRUE);
- }
- else {
- return rspamd_dispatcher_write (session->dispather, ":1" CRLF,
- sizeof (":1" CRLF) - 1, FALSE, TRUE);
- }
- }
- else {
- if (!is_redis) {
- return rspamd_dispatcher_write (session->dispather,
- ERROR_NOT_FOUND,
- sizeof (ERROR_NOT_FOUND) - 1,
- FALSE,
- TRUE);
- }
- else {
- return rspamd_dispatcher_write (session->dispather, ":0" CRLF,
- sizeof (":0" CRLF) - 1, FALSE, TRUE);
- }
- }
- }
- else if (session->command == KVSTORAGE_CMD_INCR || session->command ==
- KVSTORAGE_CMD_DECR) {
- longval = session->arg_data.value;
- if (!rspamd_kv_storage_increment (session->cf->storage, session->key,
- session->keylen, &longval)) {
- if (!is_redis) {
- return rspamd_dispatcher_write (session->dispather,
- ERROR_NOT_FOUND,
- sizeof (ERROR_NOT_FOUND) - 1,
- FALSE,
- TRUE);
- }
- else {
- return rspamd_dispatcher_write (session->dispather,
- "-ERR not found" CRLF,
- sizeof ("-ERR not found" CRLF) - 1,
- FALSE,
- TRUE);
- }
- }
- else {
- if (!is_redis) {
- r = rspamd_snprintf (outbuf, sizeof (outbuf), "%l" CRLF,
- longval);
- }
- else {
- r = rspamd_snprintf (outbuf, sizeof (outbuf), ":%l" CRLF,
- longval);
- }
- if (!rspamd_dispatcher_write (session->dispather, outbuf,
- r, FALSE, FALSE)) {
- return FALSE;
- }
- }
- }
- else if (session->command == KVSTORAGE_CMD_SYNC) {
- if (session->cf->storage->backend == NULL ||
- session->cf->storage->backend->sync_func == NULL) {
- if (!is_redis) {
- return rspamd_dispatcher_write (session->dispather,
- ERROR_COMMON,
- sizeof (ERROR_COMMON) - 1,
- FALSE,
- TRUE);
- }
- else {
- return rspamd_dispatcher_write (session->dispather,
- "-ERR unsupported" CRLF,
- sizeof ("-ERR unsupported" CRLF) - 1,
- FALSE,
- TRUE);
- }
- }
- else {
- if (session->cf->storage->backend->sync_func (session->cf->storage->
- backend)) {
- if (!is_redis) {
- return rspamd_dispatcher_write (session->dispather,
- "SYNCED" CRLF,
- sizeof ("SYNCED" CRLF) - 1,
- FALSE,
- TRUE);
- }
- else {
- return rspamd_dispatcher_write (session->dispather,
- "+OK" CRLF,
- sizeof ("+OK" CRLF) - 1,
- FALSE,
- TRUE);
- }
- }
- else {
- if (!is_redis) {
- return rspamd_dispatcher_write (session->dispather,
- "NOT_SYNCED" CRLF,
- sizeof ("NOT_SYNCED" CRLF) - 1,
- FALSE,
- TRUE);
- }
- else {
- return rspamd_dispatcher_write (session->dispather,
- "-ERR not synced" CRLF,
- sizeof ("-ERR not synced" CRLF) - 1,
- FALSE,
- TRUE);
- }
- }
- }
- }
- else if (session->command == KVSTORAGE_CMD_SELECT) {
- if (!is_redis) {
- return rspamd_dispatcher_write (session->dispather, "SELECTED" CRLF,
- sizeof ("SELECTED" CRLF) - 1, FALSE, TRUE);
- }
- else {
- return rspamd_dispatcher_write (session->dispather, "+OK" CRLF,
- sizeof ("+OK" CRLF) - 1, FALSE, TRUE);
- }
- }
- else if (session->command == KVSTORAGE_CMD_QUIT) {
- /* Quit session */
- free_kvstorage_session (session);
- return FALSE;
- }
-
- return TRUE;
-}
-
-static gboolean
-kvstorage_read_arglen (rspamd_fstring_t *in, guint *len)
-{
- gchar *p = in->begin, *end = in->begin + in->len, *c;
- gint state = 0;
-
- c = p;
- while (p < end) {
- switch (state) {
- case 0:
- if (*p != '$') {
- return FALSE;
- }
- else {
- p++;
- c = p;
- state = 1;
- }
- break;
- case 1:
- if (g_ascii_isdigit (*p) && p != end - 1) {
- p++;
- }
- else {
- if (p != end - 1) {
- return FALSE;
- }
- else {
- *len = strtoul (c, NULL, 10);
- return TRUE;
- }
- }
- break;
- }
- }
-
- return TRUE;
-}
-
-/*
- * Check number of arguments for a command
- */
-static gboolean
-kvstorage_check_argnum (struct kvstorage_session *session)
-{
- switch (session->command) {
- case KVSTORAGE_CMD_QUIT:
- case KVSTORAGE_CMD_SYNC:
- return session->argc == 1;
- case KVSTORAGE_CMD_SET:
- return session->argc == 3 || session->argc == 4;
- case KVSTORAGE_CMD_INCR:
- case KVSTORAGE_CMD_DECR:
- return session->argc == 2 || session->argc == 3;
- default:
- return session->argc == 2;
- }
-
- /* Unreachable */
- return FALSE;
-}
-
-/**
- * Dispatcher callbacks
- */
-
-/*
- * Callback that is called when there is data to read in buffer
- */
-static gboolean
-kvstorage_read_socket (rspamd_fstring_t * in, void *arg)
-{
- struct kvstorage_session *session = (struct kvstorage_session *) arg;
- struct kvstorage_worker_thread *thr;
- gint r;
- guint arglen = 0;
- gchar outbuf[BUFSIZ];
- gboolean is_redis;
-
- if (in->len == 0) {
- /* Skip empty commands */
- return TRUE;
- }
- thr = session->thr;
- is_redis = thr->ctx->is_redis;
-
- switch (session->state) {
- case KVSTORAGE_STATE_READ_CMD:
- /* Update timestamp */
- session->now = time (NULL);
- if (!parse_kvstorage_line (session, in)) {
- thr_info ("%ud: unknown command: %V", thr->id, in);
- if (!is_redis) {
- return rspamd_dispatcher_write (session->dispather,
- ERROR_UNKNOWN_COMMAND,
- sizeof (ERROR_UNKNOWN_COMMAND) - 1,
- FALSE,
- TRUE);
- }
- else {
- r = rspamd_snprintf (outbuf,
- sizeof (outbuf),
- "-ERR unknown command '%V'" CRLF,
- in);
- return rspamd_dispatcher_write (session->dispather, outbuf,
- r, FALSE, TRUE);
- }
- }
- else {
- session->cf = get_kvstorage_config (session->id);
- if (session->cf == NULL) {
- thr_info ("%ud: bad keystorage: %ud", thr->id, session->id);
- if (!is_redis) {
- return rspamd_dispatcher_write (session->dispather,
- ERROR_INVALID_KEYSTORAGE,
- sizeof (ERROR_INVALID_KEYSTORAGE) - 1,
- FALSE,
- TRUE);
- }
- else {
- return rspamd_dispatcher_write (session->dispather,
- "-ERR unknown keystorage" CRLF,
- sizeof ("-ERR unknown keystorage" CRLF) - 1,
- FALSE,
- TRUE);
- }
- }
- if (session->state != KVSTORAGE_STATE_READ_ARGLEN) {
- return kvstorage_process_command (session, is_redis);
- }
- }
- break;
- case KVSTORAGE_STATE_READ_ARGLEN:
- if (!kvstorage_read_arglen (in, &arglen)) {
- session->state = KVSTORAGE_STATE_READ_CMD;
- r = rspamd_snprintf (outbuf,
- sizeof (outbuf),
- "-ERR unknown arglen '%V'" CRLF,
- in);
- return rspamd_dispatcher_write (session->dispather, outbuf,
- r, FALSE, TRUE);
- }
- else {
- session->state = KVSTORAGE_STATE_READ_ARG;
- rspamd_set_dispatcher_policy (session->dispather,
- BUFFER_CHARACTER,
- arglen);
- }
- break;
- case KVSTORAGE_STATE_READ_ARG:
- if (session->argnum == 0) {
- /* Read command */
- if (!parse_kvstorage_command (session, in->begin, in->len)) {
- session->state = KVSTORAGE_STATE_READ_CMD;
- r = rspamd_snprintf (outbuf,
- sizeof (outbuf),
- "-ERR unknown command '%V'" CRLF,
- in);
- return rspamd_dispatcher_write (session->dispather, outbuf,
- r, FALSE, TRUE);
- }
- else {
- if (!kvstorage_check_argnum (session)) {
- session->state = KVSTORAGE_STATE_READ_CMD;
- r = rspamd_snprintf (outbuf,
- sizeof (outbuf),
- "-ERR invalid argnum for command '%V': %ud" CRLF,
- in,
- session->argc);
- return rspamd_dispatcher_write (session->dispather, outbuf,
- r, FALSE, TRUE);
- }
- else {
- if (session->argnum == session->argc - 1) {
- session->state = KVSTORAGE_STATE_READ_CMD;
- rspamd_set_dispatcher_policy (session->dispather,
- BUFFER_LINE,
- -1);
- return kvstorage_process_command (session, TRUE);
- }
- else {
- session->argnum++;
- session->state = KVSTORAGE_STATE_READ_ARGLEN;
- rspamd_set_dispatcher_policy (session->dispather,
- BUFFER_LINE,
- -1);
- }
- }
- }
- }
- else if (session->argnum == 1) {
- if (session->command != KVSTORAGE_CMD_SELECT) {
- /* This argument is a key for normal command */
- session->key = rspamd_mempool_fstrdup (session->pool, in);
- session->keylen = in->len;
- if (session->argnum == session->argc - 1) {
- session->state = KVSTORAGE_STATE_READ_CMD;
- rspamd_set_dispatcher_policy (session->dispather,
- BUFFER_LINE,
- -1);
- return kvstorage_process_command (session, TRUE);
- }
- else {
- session->argnum++;
- session->state = KVSTORAGE_STATE_READ_ARGLEN;
- rspamd_set_dispatcher_policy (session->dispather,
- BUFFER_LINE,
- -1);
- }
- }
- else {
- /* Special case for select command */
- session->state = KVSTORAGE_STATE_READ_CMD;
- rspamd_strlcpy (outbuf, in->begin, MIN (sizeof (outbuf),
- in->len));
- session->id = strtoul (outbuf, NULL, 10);
- rspamd_set_dispatcher_policy (session->dispather,
- BUFFER_LINE,
- -1);
- return kvstorage_process_command (session, TRUE);
- }
- }
- else if (session->argnum == 2) {
- /* We get datablock for set command */
- if (session->command == KVSTORAGE_CMD_SET && session->argc == 3) {
- session->state = KVSTORAGE_STATE_READ_CMD;
- rspamd_set_dispatcher_policy (session->dispather,
- BUFFER_LINE,
- -1);
- if (rspamd_kv_storage_insert (session->cf->storage,
- session->key, session->keylen,
- in->begin, in->len,
- session->flags, session->expire)) {
- return rspamd_dispatcher_write (session->dispather,
- "+OK" CRLF,
- sizeof ("+OK" CRLF) - 1,
- FALSE,
- TRUE);
- }
- else {
- return rspamd_dispatcher_write (session->dispather,
- "-ERR not stored" CRLF,
- sizeof ("-ERR not stored" CRLF) - 1,
- FALSE,
- TRUE);
- }
- }
- else if (session->command == KVSTORAGE_CMD_SET && session->argc ==
- 4) {
- /* It is expire argument */
- session->state = KVSTORAGE_STATE_READ_CMD;
- rspamd_strtol (in->begin, in->len, (glong *)&session->expire);
- session->argnum++;
- session->state = KVSTORAGE_STATE_READ_ARGLEN;
- rspamd_set_dispatcher_policy (session->dispather,
- BUFFER_LINE,
- -1);
- }
- else {
- session->state = KVSTORAGE_STATE_READ_CMD;
- rspamd_strtol (in->begin, in->len, &session->arg_data.value);
- if (session->command == KVSTORAGE_CMD_DECR) {
- session->arg_data.value = -session->arg_data.value;
- }
- rspamd_set_dispatcher_policy (session->dispather,
- BUFFER_LINE,
- -1);
- return kvstorage_process_command (session, TRUE);
- }
- }
- else if (session->argnum == 3) {
- /* We get datablock for set command */
- if (session->command == KVSTORAGE_CMD_SET && session->argc == 4) {
- session->state = KVSTORAGE_STATE_READ_CMD;
- rspamd_set_dispatcher_policy (session->dispather,
- BUFFER_LINE,
- -1);
- if (rspamd_kv_storage_insert (session->cf->storage,
- session->key, session->keylen,
- in->begin, in->len,
- session->flags, session->expire)) {
- return rspamd_dispatcher_write (session->dispather,
- "+OK" CRLF,
- sizeof ("+OK" CRLF) - 1,
- FALSE,
- TRUE);
- }
- else {
- return rspamd_dispatcher_write (session->dispather,
- "-ERR not stored" CRLF,
- sizeof ("-ERR not stored" CRLF) - 1,
- FALSE,
- TRUE);
- }
- }
- }
- break;
- case KVSTORAGE_STATE_READ_DATA:
- session->state = KVSTORAGE_STATE_READ_CMD;
- rspamd_set_dispatcher_policy (session->dispather, BUFFER_LINE, -1);
- if (rspamd_kv_storage_insert (session->cf->storage, session->key,
- session->keylen,
- in->begin, in->len,
- session->flags, session->expire)) {
- if (!is_redis) {
- return rspamd_dispatcher_write (session->dispather,
- "STORED" CRLF,
- sizeof ("STORED" CRLF) - 1,
- FALSE,
- TRUE);
- }
- else {
- return rspamd_dispatcher_write (session->dispather, "+OK" CRLF,
- sizeof ("+OK" CRLF) - 1, FALSE, TRUE);
- }
- }
- else {
- if (!is_redis) {
- return rspamd_dispatcher_write (session->dispather,
- ERROR_NOT_STORED,
- sizeof (ERROR_NOT_STORED) - 1,
- FALSE,
- TRUE);
- }
- else {
- return rspamd_dispatcher_write (session->dispather,
- "-ERR not stored" CRLF,
- sizeof ("-ERR not stored" CRLF) - 1,
- FALSE,
- TRUE);
- }
- }
-
- break;
- }
-
- return TRUE;
-}
-
-/*
- * Called if buffers were written
- */
-static gboolean
-kvstorage_write_socket (void *arg)
-{
- struct kvstorage_session *session = (struct kvstorage_session *) arg;
-
- if (session->elt) {
- if ((session->elt->flags & KV_ELT_NEED_INSERT) != 0) {
- /* Insert to cache and free element */
- session->elt->flags &= ~KV_ELT_NEED_INSERT;
- RW_R_UNLOCK (&session->cf->storage->rwlock);
- rspamd_kv_storage_insert_cache (session->cf->storage,
- ELT_KEY (session->elt),
- session->elt->keylen, ELT_DATA (session->elt),
- session->elt->size, session->elt->flags,
- session->elt->expire, NULL);
- g_free (session->elt);
- session->elt = NULL;
- return TRUE;
- }
- RW_R_UNLOCK (&session->cf->storage->rwlock);
- session->elt = NULL;
-
- }
-
- return TRUE;
-}
-
-/*
- * Called if something goes wrong
- */
-static void
-kvstorage_err_socket (GError * err, void *arg)
-{
- struct kvstorage_session *session = (struct kvstorage_session *) arg;
- struct kvstorage_worker_thread *thr;
-
- thr = session->thr;
- if (err->code != -1) {
- thr_info ("%ud: abnormally closing connection from: %s, error: %s",
- thr->id, inet_ntoa (session->client_addr), err->message);
- }
-
- if (session->elt) {
- RW_R_UNLOCK (&session->cf->storage->rwlock);
- session->elt = NULL;
- }
-
- g_error_free (err);
- free_kvstorage_session (session);
-}
-
-/**
- * Accept function
- */
-static void
-thr_accept_socket (gint fd, short what, void *arg)
-{
- struct kvstorage_worker_thread *thr = (struct kvstorage_worker_thread *)arg;
- union sa_union su;
- socklen_t addrlen = sizeof (su.ss);
- gint nfd;
- struct kvstorage_session *session;
-
- g_mutex_lock (thr->accept_mtx);
- if ((nfd =
- accept_from_socket (fd, (struct sockaddr *)&su.ss, &addrlen)) == -1) {
- thr_warn ("%ud: accept failed: %s", thr->id, strerror (errno));
- g_mutex_unlock (thr->accept_mtx);
- return;
- }
-
- /* Check for EAGAIN */
- if (nfd == 0) {
- g_mutex_unlock (thr->accept_mtx);
- return;
- }
-
- session = g_slice_alloc0 (sizeof (struct kvstorage_session));
- session->pool = rspamd_mempool_new (rspamd_mempool_suggest_size ());
- session->state = KVSTORAGE_STATE_READ_CMD;
- session->thr = thr;
- session->sock = nfd;
- session->dispather = rspamd_create_dispatcher (thr->ev_base,
- nfd,
- BUFFER_LINE,
- kvstorage_read_socket,
- kvstorage_write_socket,
- kvstorage_err_socket,
- thr->tv,
- session);
-
- g_mutex_unlock (thr->accept_mtx);
- session->elt = NULL;
-
- if (su.ss.ss_family == AF_UNIX) {
- session->client_addr.s_addr = INADDR_NONE;
- }
- else if (su.ss.ss_family == AF_INET) {
- memcpy (&session->client_addr, &su.s4.sin_addr,
- sizeof (struct in_addr));
- }
-}
-
-/**
- * Handle termination
- */
-static void
-thr_term_socket (gint fd, short what, void *arg)
-{
- struct kvstorage_worker_thread *thr = (struct kvstorage_worker_thread *)arg;
- struct timeval tv;
-
- if (read (fd, &tv, sizeof (struct timeval)) != sizeof (struct timeval)) {
- thr_err ("cannot read data from socket: %s", strerror (errno));
- tv.tv_sec = 0;
- tv.tv_usec = 0;
- }
-
- event_base_loopexit (thr->ev_base, &tv);
- event_del (&thr->bind_ev);
-}
-
-/**
- * Thread main worker function
- */
-static gpointer
-kvstorage_thread (gpointer ud)
-{
- struct kvstorage_worker_thread *thr = ud;
-
- /* Block signals as it is dispatcher deity */
- sigprocmask (SIG_BLOCK, thr->signals, NULL);
- /* Init thread specific events */
- thr->ev_base = event_init ();
-
- event_set (&thr->bind_ev,
- thr->worker->cf->listen_sock,
- EV_READ | EV_PERSIST,
- thr_accept_socket,
- (void *)thr);
- event_base_set (thr->ev_base, &thr->bind_ev);
- event_add (&thr->bind_ev, NULL);
-
- event_set (&thr->term_ev,
- thr->term_sock[0],
- EV_READ | EV_PERSIST,
- thr_term_socket,
- (void *)thr);
- event_base_set (thr->ev_base, &thr->term_ev);
- event_add (&thr->term_ev, NULL);
-
- event_base_loop (thr->ev_base, 0);
-
- return NULL;
-}
-
-/**
- * Create new thread, set it detached
- */
-static struct kvstorage_worker_thread *
-create_kvstorage_thread (struct rspamd_worker *worker,
- struct kvstorage_worker_ctx *ctx,
- guint id,
- sigset_t *signals)
-{
- struct kvstorage_worker_thread *new;
- GError *err = NULL;
-
- new =
- rspamd_mempool_alloc (ctx->pool,
- sizeof (struct kvstorage_worker_thread));
- new->ctx = ctx;
- new->worker = worker;
- new->tv = &ctx->io_timeout;
- new->log_mtx = ctx->log_mtx;
- new->accept_mtx = ctx->accept_mtx;
- new->id = id;
-
- /* Create and setup terminating socket */
- if (rspamd_socketpair (new->term_sock) == -1) {
- msg_err ("socket failed: %s", strerror (errno));
- return NULL;
- }
- rspamd_socket_nonblocking (new->term_sock[0]);
-
-#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
- new->thr = g_thread_create (kvstorage_thread, new, FALSE, &err);
-#else
- gchar *name;
-
- name = rspamd_mempool_alloc (ctx->pool,
- sizeof ("kvstorage_thread") + sizeof ("4294967296") - 1);
- rspamd_snprintf (name,
- sizeof ("kvstorage_thread") + sizeof ("4294967296") - 1,
- "kvstorage_thread%d",
- id);
-
- new->thr = g_thread_new (name, kvstorage_thread, new);
-#endif
- new->ev_base = NULL;
- new->signals = signals;
-
- if (new->thr == NULL) {
- msg_err ("cannot create thread: %s", err->message);
- }
-
- return new;
-}
-
-/*
- * Start worker process
- */
-void
-start_keystorage (struct rspamd_worker *worker)
-{
- struct sigaction signals;
- struct kvstorage_worker_ctx *ctx = worker->ctx;
- guint i;
- struct kvstorage_worker_thread *thr;
- struct timeval tv;
- GList *cur;
-
- gperf_profiler_init (worker->srv->cfg, "kvstorage");
-
- if (!g_thread_supported ()) {
- msg_err (
- "threads support is not supported on your system so kvstorage is not functionable");
- exit (EXIT_SUCCESS);
- }
- /* Create socketpair */
- if (rspamd_socketpair (ctx->s_pair) == -1) {
- msg_err ("cannot create socketpair, exiting");
- exit (EXIT_SUCCESS);
- }
- worker->srv->pid = getpid ();
- ctx->threads = NULL;
-
-#if _EVENT_NUMERIC_VERSION > 0x02000000
- if (evthread_use_pthreads () == -1) {
- msg_err (
- "threads support is not supported in your libevent so kvstorage is not functionable");
- exit (EXIT_SUCCESS);
- }
-#endif
-
- /* Set kvstorage options */
- if ( !config_kvstorage_worker (worker)) {
- msg_err ("cannot configure kvstorage worker, exiting");
- exit (EXIT_SUCCESS);
- }
-
- rspamd_signals_init (&signals, sig_handler);
-
- /* Set umask */
- umask (S_IWGRP | S_IWOTH | S_IROTH | S_IRGRP);
-
- /* Init mutexes */
-#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30))
- ctx->log_mtx = g_mutex_new ();
- ctx->accept_mtx = g_mutex_new ();
-#else
- ctx->log_mtx = rspamd_mempool_alloc (ctx->pool, sizeof (GMutex));
- ctx->accept_mtx = rspamd_mempool_alloc (ctx->pool, sizeof (GMutex));
- g_mutex_init (ctx->log_mtx);
- g_mutex_init (ctx->accept_mtx);
-#endif
-
- /* Start workers threads */
- for (i = 0; i < worker->cf->count; i++) {
- thr = create_kvstorage_thread (worker, ctx, i, &signals.sa_mask);
- if (thr != NULL) {
- ctx->threads = g_list_prepend (ctx->threads, thr);
- }
- }
-
- sigprocmask (SIG_BLOCK, &signals.sa_mask, NULL);
- /* Signal processing cycle */
- for (;; ) {
- msg_debug ("calling sigsuspend");
- sigemptyset (&signals.sa_mask);
- sigsuspend (&signals.sa_mask);
- if (wanna_die == 1) {
- wanna_die = 0;
- tv.tv_sec = 0;
- tv.tv_usec = 0;
- msg_info ("worker's immediately shutdown is requested");
- cur = ctx->threads;
- while (cur) {
- thr = cur->data;
- while (write (thr->term_sock[1], &tv,
- sizeof (struct timeval)) == -1) {
- if (errno != EAGAIN) {
- msg_err ("write to term socket failed: %s",
- strerror (errno));
- abort ();
- }
- }
- cur = g_list_next (cur);
- }
- break;
- }
- else if (soft_wanna_die == 1) {
- soft_wanna_die = 0;
- tv.tv_sec = SOFT_SHUTDOWN_TIME;
- tv.tv_usec = 0;
- msg_info ("worker's shutdown is pending in %d sec",
- SOFT_SHUTDOWN_TIME);
- cur = ctx->threads;
- while (cur) {
- thr = cur->data;
- while (write (thr->term_sock[1], &tv,
- sizeof (struct timeval)) == -1) {
- if (errno != EAGAIN) {
- msg_err ("write to term socket failed: %s",
- strerror (errno));
- abort ();
- }
- }
- cur = g_list_next (cur);
- }
- break;
- }
- else if (do_reopen_log == 1) {
- do_reopen_log = 0;
- rspamd_log_reopen (rspamd_main->logger);
- }
- }
-
- msg_info ("syncing storages");
- /* Wait for threads in the recent glib */
-#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 30))
- cur = ctx->threads;
- while (cur) {
- thr = cur->data;
- (void)g_thread_join (thr->thr);
- cur = g_list_next (cur);
- }
-#endif
- destroy_kvstorage_config ();
- rspamd_log_close (rspamd_main->logger);
- exit (EXIT_SUCCESS);
-}
+++ /dev/null
-/* Copyright (c) 2010, Vsevolod Stakhov
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#ifndef KVSTORAGE_SERVER_H_
-#define KVSTORAGE_SERVER_H_
-
-#include "config.h"
-#include "mem_pool.h"
-#include "buffer.h"
-
-/* Configuration context for kvstorage worker */
-struct kvstorage_worker_ctx {
- struct timeval io_timeout;
- guint32 timeout_raw;
- GList *threads;
- gint s_pair[2];
- gboolean is_redis;
- rspamd_mempool_t *pool;
- struct event_base *ev_base;
- GMutex *log_mtx;
- GMutex *accept_mtx;
-};
-
-struct kvstorage_worker_thread {
- struct event bind_ev;
- struct event term_ev;
- struct timeval *tv;
- struct kvstorage_worker_ctx *ctx;
- struct rspamd_worker *worker;
- GThread *thr;
- struct event_base *ev_base;
- GMutex *log_mtx;
- GMutex *accept_mtx;
- guint id;
- sigset_t *signals;
- gint term_sock[2];
-};
-
-struct kvstorage_session {
- rspamd_io_dispatcher_t *dispather;
- enum {
- KVSTORAGE_STATE_READ_CMD,
- KVSTORAGE_STATE_READ_ARGLEN,
- KVSTORAGE_STATE_READ_ARG,
- KVSTORAGE_STATE_READ_DATA
- } state;
- enum {
- KVSTORAGE_CMD_SET,
- KVSTORAGE_CMD_GET,
- KVSTORAGE_CMD_DELETE,
- KVSTORAGE_CMD_SYNC,
- KVSTORAGE_CMD_SELECT,
- KVSTORAGE_CMD_INCR,
- KVSTORAGE_CMD_DECR,
- KVSTORAGE_CMD_QUIT
- } command;
- guint id;
- guint argc;
- guint argnum;
- rspamd_mempool_t *pool;
- gchar *key;
- guint keylen;
- struct kvstorage_config *cf;
- struct kvstorage_worker_thread *thr;
- struct rspamd_kv_element *elt;
- struct in_addr client_addr;
- gint sock;
- guint flags;
- guint expire;
- union {
- glong value;
- guint length;
- } arg_data;
- time_t now;
-};
-
-#endif /* KVSTORAGE_SERVER_H_ */
+++ /dev/null
-/* Copyright (c) 2010, Vsevolod Stakhov
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#include "config.h"
-#include "kvstorage.h"
-#include "kvstorage_sqlite.h"
-#include "util.h"
-#include "main.h"
-#include <sqlite3.h>
-
-#define TABLE_NAME "kvstorage"
-#define CREATE_TABLE_SQL "CREATE TABLE " TABLE_NAME \
- " (key TEXT CONSTRAINT _key PRIMARY KEY, data BLOB)"
-#define SET_SQL "INSERT OR REPLACE INTO " TABLE_NAME \
- " (key, data) VALUES (?1, ?2)"
-#define GET_SQL "SELECT data FROM " TABLE_NAME " WHERE key = ?1"
-#define DELETE_SQL "DELETE FROM " TABLE_NAME " WHERE key = ?1"
-
-struct sqlite_op {
- struct rspamd_kv_element *elt;
- enum {
- SQLITE_OP_INSERT,
- SQLITE_OP_DELETE,
- SQLITE_OP_REPLACE
- } op;
-};
-
-/* Main sqlite structure */
-struct rspamd_sqlite_backend {
- backend_init init_func; /*< this callback is called on kv storage initialization */
- backend_insert insert_func; /*< this callback is called when element is inserted */
- backend_replace replace_func; /*< this callback is called when element is replaced */
- backend_lookup lookup_func; /*< this callback is used for lookup of element */
- backend_delete delete_func; /*< this callback is called when an element is deleted */
- backend_sync sync_func; /*< this callback is called when backend need to be synced */
- backend_destroy destroy_func; /*< this callback is used for destroying all elements inside backend */
- sqlite3 *dbp;
- gchar *filename;
- gchar *dirname;
- guint sync_ops;
- GQueue *ops_queue;
- GHashTable *ops_hash;
- gboolean initialized;
- sqlite3_stmt *get_stmt;
- sqlite3_stmt *set_stmt;
- sqlite3_stmt *delete_stmt;
-};
-
-/* Process single sqlite operation */
-static gboolean
-sqlite_process_single_op (struct rspamd_sqlite_backend *db,
- struct sqlite_op *op)
-{
- gboolean res = FALSE;
-
- op->elt->flags &= ~KV_ELT_DIRTY;
- switch (op->op) {
- case SQLITE_OP_INSERT:
- case SQLITE_OP_REPLACE:
- if (sqlite3_bind_text (db->set_stmt, 1, ELT_KEY (op->elt),
- op->elt->keylen, SQLITE_STATIC) == SQLITE_OK &&
- sqlite3_bind_blob (db->set_stmt, 2, op->elt, ELT_SIZE (op->elt),
- SQLITE_STATIC) == SQLITE_OK) {
- if (sqlite3_step (db->set_stmt) == SQLITE_DONE) {
- res = TRUE;
- }
- }
- sqlite3_reset (db->set_stmt);
- break;
- case SQLITE_OP_DELETE:
- if (sqlite3_bind_text (db->delete_stmt, 1, ELT_KEY (op->elt),
- op->elt->keylen, SQLITE_STATIC) == SQLITE_OK) {
- if (sqlite3_step (db->delete_stmt) == SQLITE_DONE) {
- res = TRUE;
- }
- }
- sqlite3_reset (db->delete_stmt);
- break;
- }
-
- if (!res) {
- op->elt->flags |= KV_ELT_DIRTY;
- }
- return res;
-}
-
-/* Process operations queue */
-static gboolean
-sqlite_process_queue (struct rspamd_kv_backend *backend)
-{
- struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend;
- struct sqlite_op *op;
- GList *cur;
-
- cur = db->ops_queue->head;
- while (cur) {
- op = cur->data;
- if (!sqlite_process_single_op (db, op)) {
- return FALSE;
- }
- cur = g_list_next (cur);
- }
-
- /* Clean the queue */
- cur = db->ops_queue->head;
- while (cur) {
- op = cur->data;
- if (op->op == SQLITE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) !=
- 0) {
- /* Also clean memory */
- g_slice_free1 (ELT_SIZE (op->elt), op->elt);
- }
- g_slice_free1 (sizeof (struct sqlite_op), op);
- cur = g_list_next (cur);
- }
-
- g_hash_table_remove_all (db->ops_hash);
- g_queue_clear (db->ops_queue);
-
- return TRUE;
-
-}
-
-/* Create table for kvstorage */
-static gboolean
-rspamd_sqlite_create_table (struct rspamd_sqlite_backend *db)
-{
- gint ret;
- sqlite3_stmt *stmt = NULL;
-
- ret =
- sqlite3_prepare_v2 (db->dbp,
- CREATE_TABLE_SQL,
- sizeof (CREATE_TABLE_SQL) - 1,
- &stmt,
- NULL);
- if (ret != SQLITE_OK) {
- if (stmt != NULL) {
- sqlite3_finalize (stmt);
- }
- return FALSE;
- }
-
- ret = sqlite3_step (stmt);
- if (ret != SQLITE_DONE) {
- sqlite3_finalize (stmt);
- return FALSE;
- }
-
- sqlite3_finalize (stmt);
- return TRUE;
-}
-
-/* Backend callbacks */
-static void
-rspamd_sqlite_init (struct rspamd_kv_backend *backend)
-{
- struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend;
- guint32 flags;
- gint ret, r;
- gchar sqlbuf[BUFSIZ];
- sqlite3_stmt *stmt = NULL;
-
- /* Set multi-threaded mode */
- if (sqlite3_config (SQLITE_CONFIG_MULTITHREAD) != SQLITE_OK) {
- goto err;
- }
-
- flags = SQLITE_OPEN_READWRITE |
- SQLITE_OPEN_CREATE |
- SQLITE_OPEN_NOMUTEX;
-
- ret = sqlite3_open_v2 (db->filename, &db->dbp, flags, NULL);
-
- if (ret != 0) {
- goto err;
- }
- /* Now check if we have table */
- r = rspamd_snprintf (sqlbuf,
- sizeof (sqlbuf),
- "SELECT * FROM " TABLE_NAME " LIMIT 1");
- ret = sqlite3_prepare_v2 (db->dbp, sqlbuf, r, &stmt, NULL);
-
- if (ret == SQLITE_ERROR) {
- /* Try to create table */
- if (!rspamd_sqlite_create_table (db)) {
- goto err;
- }
- }
- else if (ret != SQLITE_OK) {
- goto err;
- }
- /* We have table here, perform vacuum */
- sqlite3_finalize (stmt);
- r = rspamd_snprintf (sqlbuf, sizeof (sqlbuf), "VACUUM");
- ret = sqlite3_prepare_v2 (db->dbp, sqlbuf, r, &stmt, NULL);
- if (ret != SQLITE_OK) {
- goto err;
- }
- /* Perform VACUUM */
- sqlite3_step (stmt);
- sqlite3_finalize (stmt);
-
- /* Prepare required statements */
- ret = sqlite3_prepare_v2 (db->dbp,
- GET_SQL,
- sizeof (GET_SQL) - 1,
- &db->get_stmt,
- NULL);
- if (ret != SQLITE_OK) {
- goto err;
- }
- ret = sqlite3_prepare_v2 (db->dbp,
- SET_SQL,
- sizeof (SET_SQL) - 1,
- &db->set_stmt,
- NULL);
- if (ret != SQLITE_OK) {
- goto err;
- }
- ret = sqlite3_prepare_v2 (db->dbp,
- DELETE_SQL,
- sizeof (DELETE_SQL) - 1,
- &db->delete_stmt,
- NULL);
- if (ret != SQLITE_OK) {
- goto err;
- }
-
- db->initialized = TRUE;
-
- return;
-err:
- if (db->dbp != NULL) {
- msg_err ("error opening sqlite database: %d", ret);
- }
- if (stmt != NULL) {
- msg_err ("error executing statement: %d", ret);
- sqlite3_finalize (stmt);
- }
- if (db->get_stmt != NULL) {
- sqlite3_finalize (db->get_stmt);
- }
- if (db->set_stmt != NULL) {
- sqlite3_finalize (db->set_stmt);
- }
- if (db->delete_stmt != NULL) {
- sqlite3_finalize (db->delete_stmt);
- }
-}
-
-static gboolean
-rspamd_sqlite_insert (struct rspamd_kv_backend *backend,
- gpointer key,
- guint keylen,
- struct rspamd_kv_element *elt)
-{
- struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend;
- struct sqlite_op *op;
- struct rspamd_kv_element search_elt;
-
- search_elt.keylen = keylen;
- search_elt.p = key;
-
- if (!db->initialized) {
- return FALSE;
- }
-
- if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) {
- /* We found another op with such key in this queue */
- if (op->op == SQLITE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) !=
- 0) {
- /* Also clean memory */
- g_slice_free1 (ELT_SIZE (op->elt), op->elt);
- }
- op->op = SQLITE_OP_INSERT;
- op->elt = elt;
- }
- else {
- op = g_slice_alloc (sizeof (struct sqlite_op));
- op->op = SQLITE_OP_INSERT;
- op->elt = elt;
- elt->flags |= KV_ELT_DIRTY;
-
- g_queue_push_head (db->ops_queue, op);
- g_hash_table_insert (db->ops_hash, elt, op);
- }
-
- if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >=
- db->sync_ops) {
- return sqlite_process_queue (backend);
- }
-
- return TRUE;
-}
-
-static gboolean
-rspamd_sqlite_replace (struct rspamd_kv_backend *backend,
- gpointer key,
- guint keylen,
- struct rspamd_kv_element *elt)
-{
- struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend;
- struct sqlite_op *op;
-
- if (!db->initialized) {
- return FALSE;
- }
- if ((op = g_hash_table_lookup (db->ops_hash, elt)) != NULL) {
- /* We found another op with such key in this queue */
- if (op->op == SQLITE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) !=
- 0) {
- /* Also clean memory */
- g_slice_free1 (ELT_SIZE (op->elt), op->elt);
- }
- op->op = SQLITE_OP_REPLACE;
- op->elt = elt;
- }
- else {
- op = g_slice_alloc (sizeof (struct sqlite_op));
- op->op = SQLITE_OP_REPLACE;
- op->elt = elt;
- elt->flags |= KV_ELT_DIRTY;
-
- g_queue_push_head (db->ops_queue, op);
- g_hash_table_insert (db->ops_hash, elt, op);
- }
-
- if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >=
- db->sync_ops) {
- return sqlite_process_queue (backend);
- }
-
- return TRUE;
-}
-
-static struct rspamd_kv_element *
-rspamd_sqlite_lookup (struct rspamd_kv_backend *backend,
- gpointer key,
- guint keylen)
-{
- struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend;
- struct sqlite_op *op;
- struct rspamd_kv_element *elt = NULL;
- gint l;
- gconstpointer d;
- struct rspamd_kv_element search_elt;
-
- search_elt.keylen = keylen;
- search_elt.p = key;
-
- if (!db->initialized) {
- return NULL;
- }
- /* First search in ops queue */
- if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) {
- if (op->op == SQLITE_OP_DELETE) {
- /* To delete, so assume it as not found */
- return NULL;
- }
- return op->elt;
- }
-
- if (sqlite3_bind_text (db->get_stmt, 1, key, keylen,
- SQLITE_STATIC) == SQLITE_OK) {
- if (sqlite3_step (db->get_stmt) == SQLITE_ROW) {
- l = sqlite3_column_bytes (db->get_stmt, 0);
- elt = g_malloc (l);
- d = sqlite3_column_blob (db->get_stmt, 0);
- /* Make temporary copy */
- memcpy (elt, d, l);
- }
- }
-
- sqlite3_reset (db->get_stmt);
- return elt;
-}
-
-static void
-rspamd_sqlite_delete (struct rspamd_kv_backend *backend,
- gpointer key,
- guint keylen)
-{
- struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend;
- struct sqlite_op *op;
- struct rspamd_kv_element *elt;
- struct rspamd_kv_element search_elt;
-
- search_elt.keylen = keylen;
- search_elt.p = key;
-
- if (!db->initialized) {
- return;
- }
-
- if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) {
- op->op = SQLITE_OP_DELETE;
- return;
- }
-
- elt = rspamd_sqlite_lookup (backend, key, keylen);
- if (elt == NULL) {
- return;
- }
- op = g_slice_alloc (sizeof (struct sqlite_op));
- op->op = SQLITE_OP_DELETE;
- op->elt = elt;
- elt->flags |= KV_ELT_DIRTY;
-
- g_queue_push_head (db->ops_queue, op);
- g_hash_table_insert (db->ops_hash, elt, op);
-
- if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >=
- db->sync_ops) {
- sqlite_process_queue (backend);
- }
-
- return;
-}
-
-static void
-rspamd_sqlite_destroy (struct rspamd_kv_backend *backend)
-{
- struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend;
-
- if (db->initialized) {
- sqlite_process_queue (backend);
- if (db->get_stmt != NULL) {
- sqlite3_finalize (db->get_stmt);
- }
- if (db->set_stmt != NULL) {
- sqlite3_finalize (db->set_stmt);
- }
- if (db->delete_stmt != NULL) {
- sqlite3_finalize (db->delete_stmt);
- }
- sqlite3_close (db->dbp);
- g_free (db->filename);
- g_free (db->dirname);
- g_queue_free (db->ops_queue);
- g_hash_table_unref (db->ops_hash);
- g_slice_free1 (sizeof (struct rspamd_sqlite_backend), db);
- }
-}
-
-/* Create new sqlite backend */
-struct rspamd_kv_backend *
-rspamd_kv_sqlite_new (const gchar *filename, guint sync_ops)
-{
- struct rspamd_sqlite_backend *new;
- struct stat st;
- gchar *dirname;
-
- if (filename == NULL) {
- return NULL;
- }
-
- dirname = g_path_get_dirname (filename);
- if (dirname == NULL || stat (dirname, &st) == -1 || !S_ISDIR (st.st_mode)) {
- /* Inaccessible path */
- if (dirname != NULL) {
- g_free (dirname);
- }
- msg_err ("invalid file: %s", filename);
- return NULL;
- }
-
- new = g_slice_alloc0 (sizeof (struct rspamd_sqlite_backend));
- new->dirname = dirname;
- new->filename = g_strdup (filename);
- new->sync_ops = sync_ops;
- new->ops_queue = g_queue_new ();
- new->ops_hash = g_hash_table_new (kv_elt_hash_func, kv_elt_compare_func);
-
- /* Init callbacks */
- new->init_func = rspamd_sqlite_init;
- new->insert_func = rspamd_sqlite_insert;
- new->lookup_func = rspamd_sqlite_lookup;
- new->delete_func = rspamd_sqlite_delete;
- new->replace_func = rspamd_sqlite_replace;
- new->sync_func = sqlite_process_queue;
- new->destroy_func = rspamd_sqlite_destroy;
-
- return (struct rspamd_kv_backend *)new;
-}
-
+++ /dev/null
-/* Copyright (c) 2010, Vsevolod Stakhov
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Rambler BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#ifndef KVSTORAGE_SQLITE_H_
-#define KVSTORAGE_SQLITE_H_
-
-#include "config.h"
-#include "kvstorage.h"
-
-#ifdef WITH_SQLITE
-
-/* Create new sqlite backend */
-struct rspamd_kv_backend * rspamd_kv_sqlite_new (const gchar *filename,
- guint sync_ops);
-
-#endif
-
-#endif /* KVSTORAGE_SQLITE_H_ */
#include "uthash_strcase.h"
#include "filter.h"
#include "lua/lua_common.h"
-#include "kvstorage_config.h"
#include "map.h"
#include "dynamic_cfg.h"
#include "utlist.h"
#include "smtp.h"
#include "libutil/map.h"
#include "fuzzy_storage.h"
-#include "kvstorage_server.h"
#include "libserver/symbols_cache.h"
#include "lua/lua_common.h"
#include "libserver/worker_util.h"
projects / thirdparty / rspamd.git / commitdiff
