]> git.ipfire.org Git - thirdparty/rspamd.git/commitdiff
[Project] CDB maps: Start making cdb a first class citizen
authorVsevolod Stakhov <vsevolod@highsecure.ru>
Fri, 14 Feb 2020 16:25:41 +0000 (16:25 +0000)
committerVsevolod Stakhov <vsevolod@highsecure.ru>
Fri, 14 Feb 2020 16:25:41 +0000 (16:25 +0000)
14 files changed:
src/fuzzy_storage.c
src/libserver/cfg_utils.c
src/libserver/dynamic_cfg.c
src/libserver/maps/map.c
src/libserver/maps/map.h
src/libserver/maps/map_helpers.c
src/libserver/maps/map_helpers.h
src/libserver/maps/map_private.h
src/lua/lua_cdb.c
src/lua/lua_common.h
src/lua/lua_map.c
src/lua/lua_task.c
src/plugins/dkim_check.c
src/plugins/fuzzy_check.c

index f91fb61ac8eead9db240e128a9053a839462028e..b7835080008da9e3e41bc3520355e7dcf130aaa4 100644 (file)
@@ -936,7 +936,8 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session)
                                        hexbuf, sizeof (hexbuf) - 1);
                                hexbuf[sizeof (hexbuf) - 1] = '\0';
 
-                               if (rspamd_match_hash_map (session->ctx->skip_hashes, hexbuf)) {
+                               if (rspamd_match_hash_map (session->ctx->skip_hashes,
+                                               hexbuf, sizeof (hexbuf) - 1)) {
                                        result.v1.value = 401;
                                        result.v1.prob = 0.0f;
 
@@ -2010,7 +2011,7 @@ start_fuzzy (struct rspamd_worker *worker)
                                rspamd_kv_list_fin,
                                rspamd_kv_list_dtor,
                                (void **)&ctx->skip_hashes,
-                               worker)) == NULL) {
+                               worker, RSPAMD_MAP_DEFAULT)) == NULL) {
                        msg_warn_config ("cannot load hashes list from %s",
                                        ucl_object_tostring (ctx->skip_map));
                }
index 0d4953d5127be77eaabb6497f6b0d138179a9b16..be1d948a7282970cd777803ea56208986986124b 100644 (file)
@@ -1180,7 +1180,7 @@ rspamd_include_map_handler (const guchar *data, gsize len,
                           rspamd_ucl_fin_cb,
                           rspamd_ucl_dtor_cb,
                           (void **)pcbdata,
-                          NULL) != NULL;
+                          NULL, RSPAMD_MAP_DEFAULT) != NULL;
 }
 
 /*
@@ -2240,7 +2240,7 @@ rspamd_config_radix_from_ucl (struct rspamd_config *cfg,
                                                rspamd_radix_fin,
                                                rspamd_radix_dtor,
                                                (void **)target,
-                                               worker) == NULL) {
+                                               worker, RSPAMD_MAP_DEFAULT) == NULL) {
                                        g_set_error (err,
                                                        g_quark_from_static_string ("rspamd-config"),
                                                        EINVAL, "bad map definition %s for %s", str,
@@ -2267,7 +2267,7 @@ rspamd_config_radix_from_ucl (struct rspamd_config *cfg,
                                        rspamd_radix_fin,
                                        rspamd_radix_dtor,
                                        (void **)target,
-                                       worker) == NULL) {
+                                       worker, RSPAMD_MAP_DEFAULT) == NULL) {
                                g_set_error (err,
                                                g_quark_from_static_string ("rspamd-config"),
                                                EINVAL, "bad map object for %s", ucl_object_key (obj));
index 8b1f464f11e7e5fa93512b06e1cc0cff212af9d0..e3ec391071bfb8ba7047e139c0277d97712f911d 100644 (file)
@@ -283,7 +283,7 @@ init_dynamic_config (struct rspamd_config *cfg)
                        json_config_read_cb,
                        json_config_fin_cb,
                        json_config_dtor_cb,
-                       (void **)pjb, NULL)) {
+                       (void **)pjb, NULL, RSPAMD_MAP_DEFAULT)) {
                msg_err ("cannot add map for configuration %s", cfg->dynamic_conf);
        }
 }
index ff3a38f90f2524673560e885713e0b014b20a757..a837b11acdd3d934f209c80e91f79d7cd72af808 100644 (file)
@@ -788,72 +788,79 @@ read_map_file (struct rspamd_map *map, struct file_map_data *data,
        }
 
        if (len > 0) {
-               if (bk->is_compressed) {
-                       bytes = rspamd_file_xmap (data->filename, PROT_READ, &len, TRUE);
-
-                       if (bytes == NULL) {
-                               msg_err_map ("can't open map %s: %s", data->filename, strerror (errno));
-                               return FALSE;
-                       }
-
-                       ZSTD_DStream *zstream;
-                       ZSTD_inBuffer zin;
-                       ZSTD_outBuffer zout;
-                       guchar *out;
-                       gsize outlen, r;
+               if (map->no_file_read) {
+                       /* We just call read callback with backend name */
+                       map->read_callback (data->filename, strlen (data->filename),
+                                       &periodic->cbdata, TRUE);
+               }
+               else {
+                       if (bk->is_compressed) {
+                               bytes = rspamd_file_xmap (data->filename, PROT_READ, &len, TRUE);
 
-                       zstream = ZSTD_createDStream ();
-                       ZSTD_initDStream (zstream);
+                               if (bytes == NULL) {
+                                       msg_err_map ("can't open map %s: %s", data->filename, strerror (errno));
+                                       return FALSE;
+                               }
 
-                       zin.pos = 0;
-                       zin.src = bytes;
-                       zin.size = len;
+                               ZSTD_DStream *zstream;
+                               ZSTD_inBuffer zin;
+                               ZSTD_outBuffer zout;
+                               guchar *out;
+                               gsize outlen, r;
 
-                       if ((outlen = ZSTD_getDecompressedSize (zin.src, zin.size)) == 0) {
-                               outlen = ZSTD_DStreamOutSize ();
-                       }
+                               zstream = ZSTD_createDStream ();
+                               ZSTD_initDStream (zstream);
 
-                       out = g_malloc (outlen);
+                               zin.pos = 0;
+                               zin.src = bytes;
+                               zin.size = len;
 
-                       zout.dst = out;
-                       zout.pos = 0;
-                       zout.size = outlen;
+                               if ((outlen = ZSTD_getDecompressedSize (zin.src, zin.size)) == 0) {
+                                       outlen = ZSTD_DStreamOutSize ();
+                               }
 
-                       while (zin.pos < zin.size) {
-                               r = ZSTD_decompressStream (zstream, &zout, &zin);
+                               out = g_malloc (outlen);
 
-                               if (ZSTD_isError (r)) {
-                                       msg_err_map ("%s: cannot decompress data: %s",
-                                                       data->filename,
-                                                       ZSTD_getErrorName (r));
-                                       ZSTD_freeDStream (zstream);
-                                       g_free (out);
-                                       munmap (bytes, len);
-                                       return FALSE;
-                               }
+                               zout.dst = out;
+                               zout.pos = 0;
+                               zout.size = outlen;
+
+                               while (zin.pos < zin.size) {
+                                       r = ZSTD_decompressStream (zstream, &zout, &zin);
+
+                                       if (ZSTD_isError (r)) {
+                                               msg_err_map ("%s: cannot decompress data: %s",
+                                                               data->filename,
+                                                               ZSTD_getErrorName (r));
+                                               ZSTD_freeDStream (zstream);
+                                               g_free (out);
+                                               munmap (bytes, len);
+                                               return FALSE;
+                                       }
 
-                               if (zout.pos == zout.size) {
-                                       /* We need to extend output buffer */
-                                       zout.size = zout.size * 2 + 1;
-                                       out = g_realloc (zout.dst, zout.size);
-                                       zout.dst = out;
+                                       if (zout.pos == zout.size) {
+                                               /* We need to extend output buffer */
+                                               zout.size = zout.size * 2 + 1;
+                                               out = g_realloc (zout.dst, zout.size);
+                                               zout.dst = out;
+                                       }
                                }
-                       }
 
-                       ZSTD_freeDStream (zstream);
-                       msg_info_map ("%s: read map data, %z bytes compressed, "
-                                       "%z uncompressed)", data->filename,
-                                       len, zout.pos);
-                       map->read_callback (out, zout.pos, &periodic->cbdata, TRUE);
-                       g_free (out);
+                               ZSTD_freeDStream (zstream);
+                               msg_info_map ("%s: read map data, %z bytes compressed, "
+                                                         "%z uncompressed)", data->filename,
+                                               len, zout.pos);
+                               map->read_callback (out, zout.pos, &periodic->cbdata, TRUE);
+                               g_free (out);
 
-                       munmap (bytes, len);
-               }
-               else {
-                       /* Perform buffered read: fail-safe */
-                       if (!read_map_file_chunks (map, &periodic->cbdata, data->filename,
-                                       len, 0)) {
-                               return FALSE;
+                               munmap (bytes, len);
+                       }
+                       else {
+                               /* Perform buffered read: fail-safe */
+                               if (!read_map_file_chunks (map, &periodic->cbdata, data->filename,
+                                               len, 0)) {
+                                       return FALSE;
+                               }
                        }
                }
        }
@@ -2610,7 +2617,8 @@ rspamd_map_add (struct rspamd_config *cfg,
                                map_fin_cb_t fin_callback,
                                map_dtor_t dtor,
                                void **user_data,
-                               struct rspamd_worker *worker)
+                               struct rspamd_worker *worker,
+                               int flags)
 {
        struct rspamd_map *map;
        struct rspamd_map_backend *bk;
@@ -2642,6 +2650,7 @@ rspamd_map_add (struct rspamd_config *cfg,
                        map->backends);
        g_ptr_array_add (map->backends, bk);
        map->name = rspamd_mempool_strdup (cfg->cfg_pool, map_line);
+       map->no_file_read = (flags & RSPAMD_MAP_FILE_NO_READ);
 
        if (bk->protocol == MAP_PROTO_FILE) {
                map->poll_timeout = (cfg->map_timeout * cfg->map_file_watch_multiplier);
@@ -2685,7 +2694,8 @@ rspamd_map_add_from_ucl (struct rspamd_config *cfg,
                                                 map_fin_cb_t fin_callback,
                                                 map_dtor_t dtor,
                                                 void **user_data,
-                                                struct rspamd_worker *worker)
+                                                struct rspamd_worker *worker,
+                                                gint flags)
 {
        ucl_object_iter_t it = NULL;
        const ucl_object_t *cur, *elt;
@@ -2698,7 +2708,7 @@ rspamd_map_add_from_ucl (struct rspamd_config *cfg,
        if (ucl_object_type (obj) == UCL_STRING) {
                /* Just a plain string */
                return rspamd_map_add (cfg, ucl_object_tostring (obj), description,
-                               read_callback, fin_callback, dtor, user_data, worker);
+                               read_callback, fin_callback, dtor, user_data, worker, flags);
        }
 
        map = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_map));
@@ -2712,6 +2722,7 @@ rspamd_map_add_from_ucl (struct rspamd_config *cfg,
                        rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint));
        map->backends = g_ptr_array_new ();
        map->wrk = worker;
+       map->no_file_read = (flags & RSPAMD_MAP_FILE_NO_READ);
        rspamd_mempool_add_destructor (cfg->cfg_pool, rspamd_ptr_array_free_hard,
                        map->backends);
        map->poll_timeout = cfg->map_timeout;
index ce49bacbbd430ef60eea8d4d17caf69751bee902..2d3883e118e151b57064c808abaf2676a869f954 100644 (file)
@@ -61,6 +61,12 @@ struct map_cb_data {
  */
 gboolean rspamd_map_is_map (const gchar *map_line);
 
+enum rspamd_map_flags {
+       RSPAMD_MAP_DEFAULT = 0,
+       RSPAMD_MAP_FILE_ONLY = 1u << 0u,
+       RSPAMD_MAP_FILE_NO_READ = 1u << 1u,
+};
+
 /**
  * Add map from line
  */
@@ -71,7 +77,8 @@ struct rspamd_map *rspamd_map_add (struct rspamd_config *cfg,
                                                                   map_fin_cb_t fin_callback,
                                                                   map_dtor_t dtor,
                                                                   void **user_data,
-                                                                  struct rspamd_worker *worker);
+                                                                  struct rspamd_worker *worker,
+                                                                  int flags);
 
 /**
  * Add map from ucl
@@ -83,7 +90,8 @@ struct rspamd_map *rspamd_map_add_from_ucl (struct rspamd_config *cfg,
                                                                                        map_fin_cb_t fin_callback,
                                                                                        map_dtor_t dtor,
                                                                                        void **user_data,
-                                                                                       struct rspamd_worker *worker);
+                                                                                       struct rspamd_worker *worker,
+                                                                                       int flags);
 
 enum rspamd_map_watch_type {
        RSPAMD_MAP_WATCH_MIN = 9,
index d179d44f5c4f8ab43a949552e54dbf98540bfaf0..02dae0bea5180394d407979a96bfbe01bafd6e40 100644 (file)
@@ -21,6 +21,7 @@
 #include "rspamd.h"
 #include "cryptobox.h"
 #include "contrib/fastutf8/fastutf8.h"
+#include "contrib/cdb/cdb.h"
 
 #ifdef WITH_HYPERSCAN
 #include "hs.h"
@@ -58,6 +59,12 @@ struct rspamd_hash_map_helper {
        rspamd_cryptobox_fast_hash_state_t hst;
 };
 
+struct rspamd_cdb_map_helper {
+       GQueue cdbs;
+       rspamd_cryptobox_fast_hash_state_t hst;
+       gsize total_size;
+};
+
 struct rspamd_regexp_map_helper {
        rspamd_mempool_t *pool;
        struct rspamd_map *map;
@@ -1332,7 +1339,8 @@ rspamd_match_regexp_map_all (struct rspamd_regexp_map_helper *map,
 }
 
 gconstpointer
-rspamd_match_hash_map (struct rspamd_hash_map_helper *map, const gchar *in)
+rspamd_match_hash_map (struct rspamd_hash_map_helper *map, const gchar *in,
+               gsize len)
 {
        khiter_t k;
        struct rspamd_map_helper_value *val;
@@ -1393,5 +1401,175 @@ rspamd_match_radix_map_addr (struct rspamd_radix_map_helper *map,
                return val->value;
        }
 
+       return NULL;
+}
+
+
+/*
+ * CBD stuff
+ */
+
+struct rspamd_cdb_map_helper *
+rspamd_map_helper_new_cdb (struct rspamd_map *map)
+{
+       struct rspamd_cdb_map_helper *n;
+
+       n = g_malloc0 (sizeof (*n));
+       n->cdbs = (GQueue)G_QUEUE_INIT;
+
+       rspamd_cryptobox_fast_hash_init (&n->hst, map_hash_seed);
+
+       return n;
+}
+
+void
+rspamd_map_helper_destroy_cdb (struct rspamd_cdb_map_helper *c)
+{
+       if (c == NULL) {
+               return;
+       }
+
+       GList *cur = c->cdbs.head;
+
+       while (cur) {
+               struct cdb *cdb = (struct cdb *)cur->data;
+
+               cdb_free (cdb);
+               g_free (cdb->filename);
+               close (cdb->cdb_fd);
+               g_free (cdb);
+
+               cur = g_list_next (cur);
+       }
+
+       g_queue_clear (&c->cdbs);
+
+       g_free (c);
+}
+
+gchar *
+rspamd_cdb_list_read (gchar *chunk,
+                                         gint len,
+                                         struct map_cb_data *data,
+                                         gboolean final)
+{
+       struct rspamd_cdb_map_helper *cdb_data;
+       struct cdb *found = NULL;
+       struct rspamd_map *map = data->map;
+
+       g_assert (map->no_file_read);
+
+       if (data->cur_data == NULL) {
+               cdb_data = rspamd_map_helper_new_cdb (data->map);
+               data->cur_data = cdb_data;
+       }
+       else {
+               cdb_data = (struct rspamd_cdb_map_helper *)data->cur_data;
+       }
+
+       GList *cur = cdb_data->cdbs.head;
+
+       while (cur) {
+               struct cdb *elt = (struct cdb *)cur->data;
+
+               if (strcmp (elt->filename, chunk) == 0) {
+                       found = elt;
+                       break;
+               }
+
+               cur = g_list_next (cur);
+       }
+
+       if (found == NULL) {
+               /* New cdb */
+               gint fd;
+               struct cdb *cdb;
+
+               fd = rspamd_file_xopen (chunk, O_RDONLY, 0, TRUE);
+
+               if (fd == -1) {
+                       msg_err_map ("cannot open cdb map from %s: %s", chunk, strerror (errno));
+
+                       return NULL;
+               }
+
+               cdb = g_malloc0 (sizeof (struct cdb));
+
+               if (cdb_init (cdb, fd) == -1) {
+                       msg_err_map ("cannot init cdb map from %s: %s", chunk, strerror (errno));
+
+                       return NULL;
+               }
+
+               cdb->filename = g_strdup (chunk);
+               g_queue_push_tail (&cdb_data->cdbs, cdb);
+               cdb_data->total_size += cdb->cdb_fsize;
+               rspamd_cryptobox_fast_hash_update (&cdb_data->hst, chunk, len);
+       }
+
+       return chunk + len;
+}
+
+void
+rspamd_cdb_list_fin (struct map_cb_data *data, void **target)
+{
+       struct rspamd_map *map = data->map;
+       struct rspamd_cdb_map_helper *cdb_data;
+
+       if (data->cur_data) {
+               cdb_data = (struct rspamd_cdb_map_helper *)data->cur_data;
+               msg_info_map ("read cdb of %Hz size", cdb_data->total_size);
+               data->map->traverse_function = NULL;
+               data->map->nelts = 0;
+               data->map->digest = rspamd_cryptobox_fast_hash_final (&cdb_data->hst);
+       }
+
+       if (target) {
+               *target = data->cur_data;
+       }
+
+       if (data->prev_data) {
+               cdb_data = (struct rspamd_cdb_map_helper *)data->prev_data;
+               rspamd_map_helper_destroy_cdb (cdb_data);
+       }
+}
+void
+rspamd_cdb_list_dtor (struct map_cb_data *data)
+{
+       if (data->cur_data) {
+               rspamd_map_helper_destroy_cdb (data->cur_data);
+       }
+}
+
+gconstpointer
+rspamd_match_cdb_map (struct rspamd_cdb_map_helper *map,
+                                         const gchar *in, gsize inlen)
+{
+       if (map == NULL || map->cdbs.head == NULL) {
+               return NULL;
+       }
+
+       GList *cur = map->cdbs.head;
+       static rspamd_ftok_t found;
+
+       while (cur) {
+               struct cdb *cdb = (struct cdb *)cur->data;
+
+               if (cdb_find (cdb, in, inlen) > 0) {
+                       /* Extract and push value to lua as string */
+                       unsigned vlen;
+                       gconstpointer vpos;
+
+                       vpos = cdb->cdb_mem + cdb_datapos (cdb);
+                       vlen = cdb_datalen (cdb);
+                       found.len = vlen;
+                       found.begin = vpos;
+
+                       return &found; /* Do not reuse! */
+               }
+
+               cur = g_list_next (cur);
+       }
+
        return NULL;
 }
\ No newline at end of file
index 4f7b5b804e25688e472604e80ccd2354157fe2e7..b5d8cf774218825e9f92654b6ffe3fe42f64d3cb 100644 (file)
@@ -38,6 +38,7 @@ extern "C" {
 struct rspamd_radix_map_helper;
 struct rspamd_hash_map_helper;
 struct rspamd_regexp_map_helper;
+struct rspamd_cdb_map_helper;
 struct rspamd_map_helper_value;
 
 enum rspamd_regexp_map_flags {
@@ -75,6 +76,18 @@ void rspamd_kv_list_fin (struct map_cb_data *data, void **target);
 
 void rspamd_kv_list_dtor (struct map_cb_data *data);
 
+/**
+ * Cdb is a cdb mapped file with shared data
+ * chunk must be filename!
+ */
+gchar *rspamd_cdb_list_read (
+               gchar *chunk,
+               gint len,
+               struct map_cb_data *data,
+               gboolean final);
+void rspamd_cdb_list_fin (struct map_cb_data *data, void **target);
+void rspamd_cdb_list_dtor (struct map_cb_data *data);
+
 /**
  * Regexp list is a list of regular expressions
  */
@@ -149,7 +162,17 @@ GPtrArray *rspamd_match_regexp_map_all (struct rspamd_regexp_map_helper *map,
  * @return
  */
 gconstpointer rspamd_match_hash_map (struct rspamd_hash_map_helper *map,
-                                                                        const gchar *in);
+                                                                        const gchar *in, gsize len);
+
+/**
+ * Find value matching specific key in a cdb map
+ * @param map
+ * @param in
+ * @param len
+ * @return rspamd_ftok_t pointer (allocated in a static buffer!)
+ */
+gconstpointer rspamd_match_cdb_map (struct rspamd_cdb_map_helper *map,
+                                                                        const gchar *in, gsize len);
 
 /**
  * Find value matching specific key in a hash map
index 347f635383e70df7f4072a4823428ab19e0b33e9..37def8c18083bb8a51ff8a35c5de3767d1b54392 100644 (file)
@@ -155,10 +155,11 @@ struct rspamd_map {
        ev_tstamp timeout;
        gdouble poll_timeout;
        time_t next_check;
-       gboolean active_http;
-       gboolean non_trivial; /* E.g. has http backends in active mode */
-       gboolean file_only; /* No HTTP backends found */
-       gboolean static_only; /* No need to check */
+       bool active_http;
+       bool non_trivial; /* E.g. has http backends in active mode */
+       bool file_only; /* No HTTP backends found */
+       bool static_only; /* No need to check */
+       bool no_file_read; /* Do not read files */
        /* Shared lock for temporary disabling of map reading (e.g. when this map is written by UI) */
        gint *locked;
        gchar tag[MEMPOOL_UID_LEN];
index 1f9b48260d23e36d0b672aeb76957686056a8618..0762c8d359305ccfae37542383707c5595e3ae36 100644 (file)
@@ -66,6 +66,8 @@ lua_cdb_create (lua_State *L)
                cdb = g_malloc (sizeof (struct cdb));
                cdb->filename = g_strdup (filename);
                if (cdb_init (cdb, fd) == -1) {
+                       g_free (cdb->filename);
+                       g_free (cdb);
                        msg_warn ("cannot open cdb: %s, %s", filename, strerror (errno));
                        lua_pushnil (L);
                }
index 935a7c7d76baa6c7c4dbc0c1a2af1529edbb5495..6ded08eb95acf1421124dc141ca0271e38c98a52 100644 (file)
@@ -124,6 +124,7 @@ enum rspamd_lua_map_type {
        RSPAMD_LUA_MAP_REGEXP,
        RSPAMD_LUA_MAP_REGEXP_MULTIPLE,
        RSPAMD_LUA_MAP_CALLBACK,
+       RSPAMD_LUA_MAP_CDB,
        RSPAMD_LUA_MAP_UNKNOWN,
 };
 
@@ -136,6 +137,7 @@ struct rspamd_lua_map {
                struct rspamd_radix_map_helper *radix;
                struct rspamd_hash_map_helper *hash;
                struct rspamd_regexp_map_helper *re_map;
+               struct rspamd_cdb_map_helper *cdb_map;
                struct lua_map_callback_data *cbdata;
        } data;
 };
index ed37625d585e7fe81fb79b047aca92105281aff3..087478ccc5c8fde937ea06ed5d9e9a99f4254074 100644 (file)
@@ -162,7 +162,7 @@ lua_config_add_radix_map (lua_State *L)
                                rspamd_radix_fin,
                                rspamd_radix_dtor,
                                (void **)&map->data.radix,
-                               NULL)) == NULL) {
+                               NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
                        msg_warn_config ("invalid radix map %s", map_line);
                        lua_pushnil (L);
 
@@ -220,7 +220,7 @@ lua_config_radix_from_config (lua_State *L)
                                        rspamd_radix_fin,
                                        rspamd_radix_dtor,
                                        (void **)&map->data.radix,
-                                       NULL)) == NULL) {
+                                       NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
                                msg_err_config ("invalid radix map static");
                                lua_pushnil (L);
                                ucl_object_unref (fake_obj);
@@ -282,7 +282,7 @@ lua_config_radix_from_ucl (lua_State *L)
                                rspamd_radix_fin,
                                rspamd_radix_dtor,
                                (void **)&map->data.radix,
-                               NULL)) == NULL) {
+                               NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
                        msg_err_config ("invalid radix map static");
                        lua_pushnil (L);
                        ucl_object_unref (fake_obj);
@@ -328,7 +328,7 @@ lua_config_add_hash_map (lua_State *L)
                                rspamd_kv_list_fin,
                                rspamd_kv_list_dtor,
                                (void **)&map->data.hash,
-                               NULL)) == NULL) {
+                               NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
                        msg_warn_config ("invalid set map %s", map_line);
                        lua_pushnil (L);
                        return 1;
@@ -369,7 +369,7 @@ lua_config_add_kv_map (lua_State *L)
                                rspamd_kv_list_fin,
                                rspamd_kv_list_dtor,
                                (void **)&map->data.hash,
-                               NULL)) == NULL) {
+                               NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
                        msg_warn_config ("invalid hash map %s", map_line);
                        lua_pushnil (L);
 
@@ -535,7 +535,7 @@ lua_config_add_map (lua_State *L)
                                        lua_map_fin,
                                        lua_map_dtor,
                                        (void **)&map->data.cbdata,
-                                       NULL)) == NULL) {
+                                       NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
 
                                if (cbidx != -1) {
                                        luaL_unref (L, LUA_REGISTRYINDEX, cbidx);
@@ -561,7 +561,7 @@ lua_config_add_map (lua_State *L)
                                        rspamd_kv_list_fin,
                                        rspamd_kv_list_dtor,
                                        (void **)&map->data.hash,
-                                       NULL)) == NULL) {
+                                       NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
                                lua_pushnil (L);
                                ucl_object_unref (map_obj);
 
@@ -579,7 +579,7 @@ lua_config_add_map (lua_State *L)
                                        rspamd_kv_list_fin,
                                        rspamd_kv_list_dtor,
                                        (void **)&map->data.hash,
-                                       NULL)) == NULL) {
+                                       NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
                                lua_pushnil (L);
                                ucl_object_unref (map_obj);
 
@@ -597,7 +597,7 @@ lua_config_add_map (lua_State *L)
                                        rspamd_radix_fin,
                                        rspamd_radix_dtor,
                                        (void **)&map->data.radix,
-                                       NULL)) == NULL) {
+                                       NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
                                lua_pushnil (L);
                                ucl_object_unref (map_obj);
 
@@ -615,7 +615,7 @@ lua_config_add_map (lua_State *L)
                                        rspamd_regexp_list_fin,
                                        rspamd_regexp_list_dtor,
                                        (void **) &map->data.re_map,
-                                       NULL)) == NULL) {
+                                       NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
                                lua_pushnil (L);
                                ucl_object_unref (map_obj);
 
@@ -633,7 +633,7 @@ lua_config_add_map (lua_State *L)
                                        rspamd_regexp_list_fin,
                                        rspamd_regexp_list_dtor,
                                        (void **) &map->data.re_map,
-                                       NULL)) == NULL) {
+                                       NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
                                lua_pushnil (L);
                                ucl_object_unref (map_obj);
 
@@ -651,7 +651,7 @@ lua_config_add_map (lua_State *L)
                                        rspamd_regexp_list_fin,
                                        rspamd_regexp_list_dtor,
                                        (void **) &map->data.re_map,
-                                       NULL)) == NULL) {
+                                       NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
                                lua_pushnil (L);
                                ucl_object_unref (map_obj);
 
@@ -669,7 +669,25 @@ lua_config_add_map (lua_State *L)
                                        rspamd_regexp_list_fin,
                                        rspamd_regexp_list_dtor,
                                        (void **) &map->data.re_map,
-                                       NULL)) == NULL) {
+                                       NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+                               lua_pushnil (L);
+                               ucl_object_unref (map_obj);
+
+                               return 1;
+                       }
+                       m->lua_map = map;
+               }
+               else if (strcmp (type, "cdb") == 0) {
+                       map = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (*map));
+                       map->data.cdb_map = NULL;
+                       map->type = RSPAMD_LUA_MAP_CDB;
+
+                       if ((m = rspamd_map_add_from_ucl (cfg, map_obj, description,
+                                       rspamd_cdb_list_read,
+                                       rspamd_cdb_list_fin,
+                                       rspamd_cdb_list_dtor,
+                                       (void **) &map->data.cdb_map,
+                                       NULL, RSPAMD_MAP_FILE_ONLY|RSPAMD_MAP_FILE_NO_READ)) == NULL) {
                                lua_pushnil (L);
                                ucl_object_unref (map_obj);
 
@@ -857,7 +875,7 @@ lua_map_get_key (lua_State * L)
                        key = lua_map_process_string_key (L, 2, &len);
 
                        if (key && map->data.hash) {
-                               ret = rspamd_match_hash_map (map->data.hash, key) != NULL;
+                               ret = rspamd_match_hash_map (map->data.hash, key, len) != NULL;
                        }
                }
                else if (map->type == RSPAMD_LUA_MAP_REGEXP) {
@@ -903,7 +921,7 @@ lua_map_get_key (lua_State * L)
                        key = lua_map_process_string_key (L, 2, &len);
 
                        if (key && map->data.hash) {
-                               value = rspamd_match_hash_map (map->data.hash, key);
+                               value = rspamd_match_hash_map (map->data.hash, key, len);
                        }
 
                        if (value) {
@@ -911,6 +929,21 @@ lua_map_get_key (lua_State * L)
                                return 1;
                        }
                }
+               else if (map->type == RSPAMD_LUA_MAP_CDB) {
+                       /* cdb map */
+                       const rspamd_ftok_t *tok = NULL;
+
+                       key = lua_map_process_string_key (L, 2, &len);
+
+                       if (key && map->data.cdb_map) {
+                               tok = rspamd_match_cdb_map (map->data.cdb_map, key, len);
+                       }
+
+                       if (tok) {
+                               lua_pushlstring (L, tok->begin, tok->len);
+                               return 1;
+                       }
+               }
                else {
                        /* callback map or unknown type map */
                        lua_pushnil (L);
index 5bb61644d853630e472770202bb31f62b8725671..7e14b05debf4f7bceb46ab397bb79a869ce15554 100644 (file)
@@ -6382,7 +6382,7 @@ lua_lookup_words_array (lua_State *L,
                case RSPAMD_LUA_MAP_SET:
                case RSPAMD_LUA_MAP_HASH:
                        /* We know that tok->normalized is zero terminated in fact */
-                       if (rspamd_match_hash_map (map->data.hash, key)) {
+                       if (rspamd_match_hash_map (map->data.hash, key, keylen)) {
                                matched = TRUE;
                        }
                        break;
index da7e092f77474ee9bfdc9a6b5513c9a06dbe1221..aa4f4119a953cfa6c98f5b66d8638aeb959123d4 100644 (file)
@@ -446,7 +446,7 @@ dkim_module_config (struct rspamd_config *cfg)
                                rspamd_kv_list_fin,
                                rspamd_kv_list_dtor,
                                (void **)&dkim_module_ctx->dkim_domains,
-                               NULL)) {
+                               NULL, RSPAMD_MAP_DEFAULT)) {
                        msg_warn_config ("cannot load dkim domains list from %s",
                                ucl_object_tostring (value));
                }
@@ -463,7 +463,7 @@ dkim_module_config (struct rspamd_config *cfg)
                                rspamd_kv_list_fin,
                                rspamd_kv_list_dtor,
                                (void **)&dkim_module_ctx->dkim_domains,
-                               NULL)) {
+                               NULL, RSPAMD_MAP_DEFAULT)) {
                        msg_warn_config ("cannot load dkim domains list from %s",
                                        ucl_object_tostring (value));
                }
@@ -950,9 +950,12 @@ dkim_module_check (struct dkim_check_result *res)
 
                        if (dkim_module_ctx->dkim_domains != NULL) {
                                /* Perform strict check */
+                               const gchar *domain = rspamd_dkim_get_domain (cur->ctx);
+
                                if ((strict_value =
                                                rspamd_match_hash_map (dkim_module_ctx->dkim_domains,
-                                                               rspamd_dkim_get_domain (cur->ctx))) != NULL) {
+                                                               domain,
+                                                               strlen (domain))) != NULL) {
                                        if (!dkim_module_parse_strict (strict_value, &cur->mult_allow,
                                                        &cur->mult_deny)) {
                                                cur->mult_allow = dkim_module_ctx->strict_multiplier;
@@ -1212,11 +1215,12 @@ dkim_symbol_callback (struct rspamd_task *task,
                        else {
                                /* Get key */
                                cur->ctx = ctx;
+                               const gchar *domain = rspamd_dkim_get_domain (cur->ctx);
 
                                if (dkim_module_ctx->trusted_only &&
                                                (dkim_module_ctx->dkim_domains == NULL ||
                                                                rspamd_match_hash_map (dkim_module_ctx->dkim_domains,
-                                                                               rspamd_dkim_get_domain (ctx)) == NULL)) {
+                                                                               domain, strlen (domain)) == NULL)) {
                                        msg_debug_task ("skip dkim check for %s domain",
                                                        rspamd_dkim_get_domain (ctx));
 
index aaab34ccbf42393b61b7657ad1c66d89daf760e9..4df88e2a809b274f954cb0d951fdb2d4f81b76b8 100644 (file)
@@ -361,7 +361,7 @@ fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj,
                                rspamd_kv_list_fin,
                                rspamd_kv_list_dtor,
                                (void **)&rule->skip_map,
-                               NULL);
+                               NULL, RSPAMD_MAP_DEFAULT);
        }
 
        if ((value = ucl_object_lookup (obj, "headers")) != NULL) {
@@ -1987,7 +1987,8 @@ fuzzy_insert_result (struct fuzzy_client_session *session,
                        rspamd_encode_hex_buf (cmd->digest, sizeof (cmd->digest),
                                hexbuf, sizeof (hexbuf) - 1);
                        hexbuf[sizeof (hexbuf) - 1] = '\0';
-                       if (rspamd_match_hash_map (session->rule->skip_map, hexbuf)) {
+                       if (rspamd_match_hash_map (session->rule->skip_map, hexbuf,
+                                       sizeof (hexbuf) - 1)) {
                                return;
                        }
                }