From: Victor Julien <victor@inliniac.net>
Date: Wed, 14 Mar 2012 06:56:11 +0000 (+0100)
Subject: Introduce host table, make tag use it
X-Git-Tag: suricata-1.3beta1~114
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a05df345de3560c720d73e06da8567b630e40b91;p=thirdparty%2Fsuricata.git

Introduce host table, make tag use it

Add a host table similar to the flow table. A hash using fine grained
locking. Flow manager for now takes care of book keeping / garbage
collecting.

Tag subsystem now uses this for host based tagging instead of the
global tag hash table. Because the latter used a global lock and the
new code uses very fine grained locking this patch should improve
scalability.
---

diff --git a/src/Makefile.am b/src/Makefile.am
index 6050d1a082..84f882c072 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -55,6 +55,8 @@ flow-bit.c flow-bit.h \
 flow-alert-sid.c flow-alert-sid.h \
 pkt-var.c pkt-var.h \
 host.c host.h \
+host-queue.c host-queue.h \
+host-timeout.c host-timeout.h \
 reputation.c reputation.h \
 detect.c detect.h \
 detect-engine-sigorder.c detect-engine-sigorder.h \
diff --git a/src/detect-engine-tag.c b/src/detect-engine-tag.c
index 5478d237f8..c13a2d01f7 100644
--- a/src/detect-engine-tag.c
+++ b/src/detect-engine-tag.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2010 Open Information Security Foundation
+/* Copyright (C) 2007-2012 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -18,6 +18,7 @@
 /**
  * \file detect-engine-tag.c
  *
+ * \author Victor Julien <victor@inliniac.net>
  * \author Pablo Rincon Crespo <pablo.rincon.crespo@gmail.com>
  *
  * Implements a global context to store data related to hosts flagged
@@ -31,93 +32,13 @@
 #include "util-hashlist.h"
 #include "detect-engine-tag.h"
 #include "detect-tag.h"
-
-static void TagTimeoutRemove(DetectTagHostCtx *tag_ctx, struct timeval *tv);
+#include "host.h"
 
 SC_ATOMIC_DECLARE(unsigned int, num_tags);  /**< Atomic counter, to know if we
                                                  have tagged hosts/sessions,
                                                  to avoid locking */
 
-/* Global Ctx for tagging hosts */
-DetectTagHostCtx *tag_ctx = NULL;
-
-void TagFreeFunc(void *data)
-{
-    DetectTagDataListFree(data);
-    return;
-}
-
-/**
- * \brief Compare elements into the hash table
- *
- * \param data1 First element to compare
- * \param len1 length of first element
- * \param data2 Second element to compare
- * \param len2 length of second element
- *
- * \retval 1 Match or 0 No Match
- */
-char TagCompareFunc(void *data1, uint16_t len1, void *data2,uint16_t len2)
-{
-    SCEnter();
-
-    DetectTagDataEntryList *a = (DetectTagDataEntryList *)data1;
-    DetectTagDataEntryList *b = (DetectTagDataEntryList *)data2;
-
-    if (CMP_ADDR(&a->addr,&b->addr)) {
-        SCReturnInt(1);
-    }
-
-    SCReturnInt(0);
-}
-
-/**
- * \brief Create the hash for tag tables
- *
- * \param ht Hash Table
- * \param data DataEntry that will be used to create the hash
- * \param datalen DataEntry length
- *
- * \retval hash the hash
- */
-uint32_t TagHashFunc(HashListTable *ht, void *data, uint16_t datalen)
-{
-    SCEnter();
-
-    if (data == NULL) return 0;
-    DetectTagDataEntryList *dt = (DetectTagDataEntryList *)data;
-    uint32_t hash = 0;
-
-    if (dt->ipv == 4)
-        hash = (dt->addr.addr_data32[0]);
-    else if (dt->ipv == 6)
-        hash = (dt->addr.addr_data32[0] +
-                dt->addr.addr_data32[1] +
-                dt->addr.addr_data32[2] +
-                dt->addr.addr_data32[3]);
-    else {
-        SCLogDebug("no dt->ipv");
-    }
-
-    SCReturnInt(hash % TAG_HASH_SIZE);
-}
-
 void TagInitCtx(void) {
-    tag_ctx = SCMalloc(sizeof(DetectTagHostCtx));
-    if (tag_ctx == NULL) {
-        exit(EXIT_FAILURE);
-    }
-    memset(tag_ctx, 0, sizeof(DetectTagHostCtx));
-
-    TimeGet(&tag_ctx->last_ts);
-
-    if (SCMutexInit(&tag_ctx->lock, NULL) != 0) {
-        SCLogError(SC_ERR_MEM_ALLOC,
-                "Tag: Failed to initialize hash table mutex.");
-        exit(EXIT_FAILURE);
-    }
-
-    TagHashInit(tag_ctx);
     SC_ATOMIC_INIT(num_tags);
 }
 
@@ -129,16 +50,6 @@ void TagInitCtx(void) {
  */
 void TagDestroyCtx(void)
 {
-    HashListTableFree(tag_ctx->tag_hash_table_ipv4);
-    tag_ctx->tag_hash_table_ipv4 = NULL;
-
-    HashListTableFree(tag_ctx->tag_hash_table_ipv6);
-    tag_ctx->tag_hash_table_ipv6 = NULL;
-
-    SCMutexDestroy(&tag_ctx->lock);
-
-    SCFree(tag_ctx);
-    tag_ctx = NULL;
 #ifdef DEBUG
     BUG_ON(SC_ATOMIC_GET(num_tags) != 0);
 #endif
@@ -152,81 +63,85 @@ void TagRestartCtx() {
     TagInitCtx();
 }
 
-/**
- * \brief Init tag context hash tables
- *
- * \param det_ctx Dectection Thread Context
- *
- */
-void TagHashInit(DetectTagHostCtx *tag_ctx)
-{
-    tag_ctx->tag_hash_table_ipv4 = HashListTableInit(TAG_HASH_SIZE, TagHashFunc, TagCompareFunc, TagFreeFunc);
-    if(tag_ctx->tag_hash_table_ipv4 == NULL)    {
-        SCLogError(SC_ERR_MEM_ALLOC,
-                "Tag: Failed to initialize ipv4 dst hash table.");
-        exit(EXIT_FAILURE);
+static DetectTagDataEntry *DetectTagDataCopy(DetectTagDataEntry *dtd) {
+    DetectTagDataEntry *tde = SCMalloc(sizeof(DetectTagDataEntry));
+    if (tde == NULL) {
+        return NULL;
     }
+    memset(tde, 0, sizeof(DetectTagDataEntry));
 
-    tag_ctx->tag_hash_table_ipv6 = HashListTableInit(TAG_HASH_SIZE, TagHashFunc, TagCompareFunc, TagFreeFunc);
-    if(tag_ctx->tag_hash_table_ipv6 == NULL)    {
-        SCLogError(SC_ERR_MEM_ALLOC,
-                "Tag: Failed to initialize ipv4 src hash table.");
-        exit(EXIT_FAILURE);
-    }
+    tde->sid = dtd->sid;
+    tde->gid = dtd->gid;
+    tde->flags = dtd->flags;
+    tde->metric = dtd->metric;
+    tde->count = dtd->count;
+
+    tde->first_ts = dtd->first_ts;
+    tde->last_ts = dtd->last_ts;
+    return tde;
 }
 
 /**
- * \brief Search for a tag data into tag hash table
+ * \brief This function is used to add a tag to a session (type session)
+ *        or update it if it's already installed. The number of times to
+ *        allow an update is limited by DETECT_TAG_MATCH_LIMIT. This way
+ *        repetitive matches to the same rule are limited of setting tags,
+ *        to avoid DOS attacks
  *
- * \param de_ctx Dectection Context
- * \param dtde Tag element
- * \param p Packet structure
+ * \param p pointer to the current packet
+ * \param tde pointer to the new DetectTagDataEntry
  *
- * \retval lookup_tde Return the tag element
+ * \retval 0 if the tde was added succesfuly
+ * \retval 1 if an entry of this sid/gid already exist and was updated
  */
-DetectTagDataEntryList *TagHashSearch(DetectTagHostCtx *tag_ctx, DetectTagDataEntryList *dtde,
-                                  Packet *p)
-{
-    SCEnter();
+int TagFlowAdd(Packet *p, DetectTagDataEntry *tde) {
+    uint8_t updated = 0;
+    uint16_t num_tags = 0;
+    DetectTagDataEntry *iter = NULL;
 
-    DetectTagDataEntryList *lookup_tde = NULL;
+    if (p->flow == NULL)
+        return 1;
 
-    if (PKT_IS_IPV4(p)) {
-        SCLogDebug("ipv4 search");
-        lookup_tde = HashListTableLookup(tag_ctx->tag_hash_table_ipv4, dtde, sizeof(DetectTagDataEntryList));
-    } else if (PKT_IS_IPV6(p)) {
-        SCLogDebug("ipv6 search");
-        lookup_tde = HashListTableLookup(tag_ctx->tag_hash_table_ipv6, dtde, sizeof(DetectTagDataEntryList));
-    }
+    SCMutexLock(&p->flow->m);
 
-    SCReturnPtr(lookup_tde, "DetectTagDataEntryList");
-}
+    if (p->flow->tag_list != NULL) {
+        iter = p->flow->tag_list;
 
-/**
- * \brief Add tag element into hash table
- *
- * \param de_ctx Dectection Context
- * \param dtde Tag element
- * \param p Packet structure
- *
- */
-int TagHashAdd(DetectTagHostCtx *tag_ctx, DetectTagDataEntryList *dtde, Packet *p)
-{
-    SCEnter();
+        /* First iterate installed entries searching a duplicated sid/gid */
+        for (; iter != NULL; iter = iter->next) {
+            num_tags++;
+
+            if (iter->sid == tde->sid && iter->gid == tde->gid) {
+                iter->cnt_match++;
 
-    int ret = 0;
+                /* If so, update data, unless the maximum MATCH limit is
+                 * reached. This prevents possible DOS attacks */
+                if (iter->cnt_match < DETECT_TAG_MATCH_LIMIT) {
+                    /* Reset time and counters */
+                    iter->first_ts = iter->last_ts = tde->first_ts;
+                    iter->packets = 0;
+                    iter->bytes = 0;
+                }
+                updated = 1;
+                break;
+            }
+        }
+    }
 
-    if (PKT_IS_IPV4(p)) {
-        dtde->ipv = 4;
-        ret = HashListTableAdd(tag_ctx->tag_hash_table_ipv4,
-                               dtde, sizeof(DetectTagDataEntry));
-    } else if (PKT_IS_IPV6(p)) {
-        dtde->ipv = 6;
-        ret = HashListTableAdd(tag_ctx->tag_hash_table_ipv6,
-                               dtde, sizeof(DetectTagDataEntry));
+    /* If there was no entry of this rule, prepend the new tde */
+    if (updated == 0 && num_tags < DETECT_TAG_MAX_TAGS) {
+        DetectTagDataEntry *new_tde = DetectTagDataCopy(tde);
+        if (new_tde != NULL) {
+            new_tde->next = p->flow->tag_list;
+            p->flow->tag_list = new_tde;
+            SC_ATOMIC_ADD(num_tags, 1);
+        }
+    } else if (num_tags == DETECT_TAG_MAX_TAGS) {
+        SCLogDebug("Max tags for sessions reached (%"PRIu16")", num_tags);
     }
 
-    SCReturnInt((ret == 0));
+    SCMutexUnlock(&p->flow->m);
+    return updated;
 }
 
 /**
@@ -238,60 +153,32 @@ int TagHashAdd(DetectTagHostCtx *tag_ctx, DetectTagDataEntryList *dtde, Packet *
  *
  * \retval 0 if it was added, 1 if it was updated
  */
-int TagHashAddTag(DetectTagHostCtx *tag_ctx, DetectTagDataEntry *tde, Packet *p)
+int TagHashAddTag(DetectTagDataEntry *tde, Packet *p)
 {
     SCEnter();
 
-    DetectTagDataEntryList *entry = NULL;
     uint8_t updated = 0;
     uint16_t num_tags = 0;
-    /* local, just for searching */
-    DetectTagDataEntryList tdl;
-    tdl.header_entry = tde;
-
-    SCMutexLock(&tag_ctx->lock);
-
-    /* first search if we already have an entry of this host */
-    if (PKT_IS_IPV4(p)) {
-        tdl.ipv = 4;
-        if (tde->td->direction == DETECT_TAG_DIR_SRC) {
-            SET_IPV4_SRC_ADDR(p, &tdl.addr);
-        } else if (tde->td->direction == DETECT_TAG_DIR_DST) {
-            SET_IPV4_DST_ADDR(p, &tdl.addr);
-        }
-    } else if (PKT_IS_IPV6(p)) {
-        tdl.ipv = 6;
-        if (tde->td->direction == DETECT_TAG_DIR_SRC) {
-            SET_IPV6_SRC_ADDR(p, &tdl.addr);
-        } else if (tde->td->direction == DETECT_TAG_DIR_DST) {
-            SET_IPV6_DST_ADDR(p, &tdl.addr);
-        }
+    Host *host = NULL;
+
+    /* Lookup host in the hash. If it doesn't exist yet it's
+     * created. */
+    if (tde->flags & TAG_ENTRY_FLAG_DIR_SRC) {
+        host = HostGetHostFromHash(&p->src);
+    } else if (tde->flags & TAG_ENTRY_FLAG_DIR_DST) {
+        host = HostGetHostFromHash(&p->dst);
+    }
+    /* no host for us */
+    if (host == NULL) {
+        return -1;
     }
 
-    entry = TagHashSearch(tag_ctx, &tdl, p);
-    if (entry == NULL) {
-        DetectTagDataEntryList *new = SCMalloc(sizeof(DetectTagDataEntryList));
-        if (new != NULL) {
-            memcpy(new, &tdl, sizeof(DetectTagDataEntryList));
-
-            /* get a new tde as the one we have is on the stack */
-            DetectTagDataEntry *new_tde = DetectTagDataCopy(tde);
-            if (new_tde == NULL) {
-                SCFree(new);
-            } else {
-                new->header_entry = new_tde;
-
-                /* increment num_tags before adding to prevent a minor race,
-                 * on setting and checking the first tag */
-                SC_ATOMIC_ADD(num_tags, 1);
-                if (!(TagHashAdd(tag_ctx, new, p))) {
-                    SC_ATOMIC_SUB(num_tags, 1);
-                    SCFree(new_tde);
-                    SCFree(new);
-                }
-            }
-        } else {
-            SCLogDebug("Failed to allocate a new session");
+    if (host->tag == NULL) {
+        /* get a new tde as the one we have is on the stack */
+        DetectTagDataEntry *new_tde = DetectTagDataCopy(tde);
+        if (new_tde != NULL) {
+            host->tag = new_tde;
+            SC_ATOMIC_ADD(num_tags, 1);
         }
     } else {
         /* Append the tag to the list of this host */
@@ -299,7 +186,7 @@ int TagHashAddTag(DetectTagHostCtx *tag_ctx, DetectTagDataEntry *tde, Packet *p)
         /* First iterate installed entries searching a duplicated sid/gid */
         DetectTagDataEntry *iter = NULL;
 
-        for (iter = entry->header_entry; iter != NULL; iter = iter->next) {
+        for (iter = host->tag; iter != NULL; iter = iter->next) {
             num_tags++;
             if (iter->sid == tde->sid && iter->gid == tde->gid) {
                 iter->cnt_match++;
@@ -307,7 +194,7 @@ int TagHashAddTag(DetectTagHostCtx *tag_ctx, DetectTagDataEntry *tde, Packet *p)
                  * reached. This prevents possible DOS attacks */
                 if (iter->cnt_match < DETECT_TAG_MATCH_LIMIT) {
                     /* Reset time and counters */
-                    iter->first_ts.tv_sec = iter->last_ts.tv_sec = tde->first_ts.tv_sec;
+                    iter->first_ts = iter->last_ts = tde->first_ts;
                     iter->packets = 0;
                     iter->bytes = 0;
                 }
@@ -322,386 +209,283 @@ int TagHashAddTag(DetectTagHostCtx *tag_ctx, DetectTagDataEntry *tde, Packet *p)
             DetectTagDataEntry *new_tde = DetectTagDataCopy(tde);
             if (new_tde != NULL) {
                 SC_ATOMIC_ADD(num_tags, 1);
-                new_tde->next = entry->header_entry;
-                entry->header_entry = new_tde;
+
+                new_tde->next = host->tag;
+                host->tag = new_tde;
             }
         } else if (num_tags == DETECT_TAG_MAX_TAGS) {
             SCLogDebug("Max tags for sessions reached (%"PRIu16")", num_tags);
         }
     }
 
-    SCMutexUnlock(&tag_ctx->lock);
+    HostRelease(host);
     SCReturnInt(updated);
 }
 
-/**
- * \brief Search tags for src and dst. Update entries of the tag, remove if necessary
- *
- * \param de_ctx Detect context
- * \param det_ctx Detect thread context
- * \param p packet
- *
- */
-void TagHandlePacket(DetectEngineCtx *de_ctx,
-        DetectEngineThreadCtx *det_ctx, Packet *p)
-{
+static void TagHandlePacketFlow(Flow *f, Packet *p) {
+    if (f->tag_list == NULL)
+        return;
 
     DetectTagDataEntry *tde = NULL;
     DetectTagDataEntry *prev = NULL;
-    DetectTagDataEntry *iter = NULL;
-    DetectTagDataEntryList tdl;
-    DetectTagDataEntryList *tde_src = NULL;
-    DetectTagDataEntryList *tde_dst = NULL;
-
-    /* If there's no tag, get out of here */
-    unsigned int current_tags = SC_ATOMIC_GET(num_tags);
-    if (current_tags == 0)
-        return;
-
+    DetectTagDataEntry *iter = f->tag_list;
     uint8_t flag_added = 0;
 
-    /* First update and get session tags */
-    if (p->flow != NULL) {
-        SCMutexLock(&p->flow->m);
-        if (p->flow->tag_list != NULL) {
-            iter = p->flow->tag_list->header_entry;
-            prev = NULL;
-            while (iter != NULL) {
-                /* update counters */
-                iter->last_ts.tv_sec = p->ts.tv_sec;
+    while (iter != NULL) {
+        /* update counters */
+        iter->last_ts = p->ts.tv_sec;
+        switch (iter->metric) {
+            case DETECT_TAG_METRIC_PACKET:
                 iter->packets++;
+                break;
+            case DETECT_TAG_METRIC_BYTES:
                 iter->bytes += GET_PKT_LEN(p);
+                break;
+        }
 
-                /* If this packet triggered the rule with tag, we dont need
-                 * to log it (the alert will log it) */
-                if (iter->skipped_first == 0) {
-                    iter->skipped_first = 1;
-                } else if (iter->td != NULL) {
-                    /* Update metrics; remove if tag expired; and set alerts */
-                    switch (iter->td->metric) {
-                        case DETECT_TAG_METRIC_PACKET:
-                            if (iter->packets > iter->td->count) {
-                                /* tag expired */
-                                if (prev != NULL) {
-                                    tde = iter;
-                                    prev->next = iter->next;
-                                    iter = iter->next;
-                                    SCFree(tde);
-                                    SC_ATOMIC_SUB(num_tags, 1);
-                                    continue;
-                                } else {
-                                    p->flow->tag_list->header_entry = iter->next;
-                                    tde = iter;
-                                    iter = iter->next;
-                                    SCFree(tde);
-                                    SC_ATOMIC_SUB(num_tags, 1);
-                                    continue;
-                                }
-                            } else if (flag_added == 0) {
-                                /* It's matching the tag. Add it to be logged and
-                                 * update "flag_added" to add the packet once. */
-                                p->flags |= PKT_HAS_TAG;
-                                flag_added++;
-                            }
-                            break;
-                        case DETECT_TAG_METRIC_BYTES:
-                            if (iter->bytes > iter->td->count) {
-                                /* tag expired */
-                                if (prev != NULL) {
-                                    tde = iter;
-                                    prev->next = iter->next;
-                                    iter = iter->next;
-                                    SCFree(tde);
-                                    SC_ATOMIC_SUB(num_tags, 1);
-                                    continue;
-                                } else {
-                                    p->flow->tag_list->header_entry = iter->next;
-                                    tde = iter;
-                                    iter = iter->next;
-                                    SCFree(tde);
-                                    SC_ATOMIC_SUB(num_tags, 1);
-                                    continue;
-                                }
-                            } else if (flag_added == 0) {
-                                /* It's matching the tag. Add it to be logged and
-                                 * update "flag_added" to add the packet once. */
-                                p->flags |= PKT_HAS_TAG;
-                                flag_added++;
-                            }
-                            break;
-                        case DETECT_TAG_METRIC_SECONDS:
-                            /* last_ts handles this metric, but also a generic time based
-                             * expiration to prevent dead sessions/hosts */
-                            if (iter->last_ts.tv_sec - iter->first_ts.tv_sec > (int)iter->td->count) {
-                                /* tag expired */
-                                if (prev != NULL) {
-                                    tde = iter;
-                                    prev->next = iter->next;
-                                    iter = iter->next;
-                                    SCFree(tde);
-                                    SC_ATOMIC_SUB(num_tags, 1);
-                                    continue;
-                                } else {
-                                    p->flow->tag_list->header_entry = iter->next;
-                                    tde = iter;
-                                    iter = iter->next;
-                                    SCFree(tde);
-                                    SC_ATOMIC_SUB(num_tags, 1);
-                                    continue;
-                                }
-                            } else if (flag_added == 0) {
-                                /* It's matching the tag. Add it to be logged and
-                                 * update "flag_added" to add the packet once. */
-                                p->flags |= PKT_HAS_TAG;
-                                flag_added++;
-                            }
-                            break;
+        /* If this packet triggered the rule with tag, we dont need
+         * to log it (the alert will log it) */
+        if (!(iter->flags & TAG_ENTRY_FLAG_SKIPPED_FIRST)) {
+            iter->flags |= TAG_ENTRY_FLAG_SKIPPED_FIRST;
+        } else {
+            /* Update metrics; remove if tag expired; and set alerts */
+            switch (iter->metric) {
+                case DETECT_TAG_METRIC_PACKET:
+                    if (iter->packets > iter->count) {
+                        /* tag expired */
+                        if (prev != NULL) {
+                            tde = iter;
+                            prev->next = iter->next;
+                            iter = iter->next;
+                            SCFree(tde);
+                            SC_ATOMIC_SUB(num_tags, 1);
+                            continue;
+                        } else {
+                            p->flow->tag_list = iter->next;
+                            tde = iter;
+                            iter = iter->next;
+                            SCFree(tde);
+                            SC_ATOMIC_SUB(num_tags, 1);
+                            continue;
+                        }
+                    } else if (flag_added == 0) {
+                        /* It's matching the tag. Add it to be logged and
+                         * update "flag_added" to add the packet once. */
+                        p->flags |= PKT_HAS_TAG;
+                        flag_added++;
                     }
-
-                }
-
-                prev = iter;
-                iter = iter->next;
+                    break;
+                case DETECT_TAG_METRIC_BYTES:
+                    if (iter->bytes > iter->count) {
+                        /* tag expired */
+                        if (prev != NULL) {
+                            tde = iter;
+                            prev->next = iter->next;
+                            iter = iter->next;
+                            SCFree(tde);
+                            SC_ATOMIC_SUB(num_tags, 1);
+                            continue;
+                        } else {
+                            p->flow->tag_list = iter->next;
+                            tde = iter;
+                            iter = iter->next;
+                            SCFree(tde);
+                            SC_ATOMIC_SUB(num_tags, 1);
+                            continue;
+                        }
+                    } else if (flag_added == 0) {
+                        /* It's matching the tag. Add it to be logged and
+                         * update "flag_added" to add the packet once. */
+                        p->flags |= PKT_HAS_TAG;
+                        flag_added++;
+                    }
+                    break;
+                case DETECT_TAG_METRIC_SECONDS:
+                    /* last_ts handles this metric, but also a generic time based
+                     * expiration to prevent dead sessions/hosts */
+                    if (iter->last_ts - iter->first_ts > iter->count) {
+                        /* tag expired */
+                        if (prev != NULL) {
+                            tde = iter;
+                            prev->next = iter->next;
+                            iter = iter->next;
+                            SCFree(tde);
+                            SC_ATOMIC_SUB(num_tags, 1);
+                            continue;
+                        } else {
+                            p->flow->tag_list = iter->next;
+                            tde = iter;
+                            iter = iter->next;
+                            SCFree(tde);
+                            SC_ATOMIC_SUB(num_tags, 1);
+                            continue;
+                        }
+                    } else if (flag_added == 0) {
+                        /* It's matching the tag. Add it to be logged and
+                         * update "flag_added" to add the packet once. */
+                        p->flags |= PKT_HAS_TAG;
+                        flag_added++;
+                    }
+                    break;
             }
 
-            iter = NULL;
         }
-        SCMutexUnlock(&p->flow->m);
-    }
-
-    /* Then search the src and dst hosts at the ctx */
-    SCMutexLock(&tag_ctx->lock);
 
-    /* Check for timeout tags if we reached the interval for checking it */
-    if (p->ts.tv_sec - tag_ctx->last_ts.tv_sec > TAG_TIMEOUT_CHECK_INTERVAL) {
-        TagTimeoutRemove(tag_ctx, &p->ts);
-        tag_ctx->last_ts.tv_sec = p->ts.tv_sec;
+        prev = iter;
+        iter = iter->next;
     }
+}
 
-    if (PKT_IS_IPV4(p)) {
-        tdl.ipv = 4;
-        /* search tags for source */
-        SET_IPV4_SRC_ADDR(p, &tdl.addr);
-        tde_src = TagHashSearch(tag_ctx, &tdl, p);
-
-        /* search tags for dest */
-        SET_IPV4_DST_ADDR(p, &tdl.addr);
-        tde_dst = TagHashSearch(tag_ctx, &tdl, p);
-    } else if (PKT_IS_IPV6(p)) {
-        tdl.ipv = 6;
-        /* search tags for source */
-        SET_IPV6_SRC_ADDR(p, &tdl.addr);
-        tde_src = TagHashSearch(tag_ctx, &tdl, p);
-
-        /* search tags for dest */
-        SET_IPV6_DST_ADDR(p, &tdl.addr);
-        tde_dst = TagHashSearch(tag_ctx, &tdl, p);
-    }
+void TagHandlePacketHost(Host *host, Packet *p) {
+    DetectTagDataEntry *tde = NULL;
+    DetectTagDataEntry *prev = NULL;
+    DetectTagDataEntry *iter;
+    uint8_t flag_added = 0;
+
+    iter = host->tag;
+    prev = NULL;
+    while (iter != NULL) {
+        /* update counters */
+        iter->last_ts = p->ts.tv_sec;
+        switch (iter->metric) {
+            case DETECT_TAG_METRIC_PACKET:
+                iter->packets++;
+                break;
+            case DETECT_TAG_METRIC_BYTES:
+                iter->bytes += GET_PKT_LEN(p);
+                break;
+        }
 
-    if (tde_src != NULL) {
-        iter = tde_src->header_entry;
-        prev = NULL;
-        while (iter != NULL) {
-            /* update counters */
-            iter->last_ts.tv_sec = p->ts.tv_sec;
-            iter->packets++;
-            iter->bytes += GET_PKT_LEN(p);
-
-            /* If this packet triggered the rule with tag, we dont need
-             * to log it (the alert will log it) */
-            if (iter->skipped_first == 0) {
-                iter->skipped_first = 1;
-            } else if (iter->td != NULL) {
-                /* Update metrics; remove if tag expired; and set alerts */
-                switch (iter->td->metric) {
-                    case DETECT_TAG_METRIC_PACKET:
-                        if (iter->packets > iter->td->count) {
-                            /* tag expired */
-                            if (prev != NULL) {
-                                tde = iter;
-                                prev->next = iter->next;
-                                iter = iter->next;
-                                SCFree(tde);
-                                SC_ATOMIC_SUB(num_tags, 1);
-                                continue;
-                            } else {
-                                tde = iter;
-                                iter = iter->next;
-                                SCFree(tde);
-                                SC_ATOMIC_SUB(num_tags, 1);
-                                tde_src->header_entry = iter;
-                                continue;
-                            }
-                        } else if (flag_added == 0) {
-                            /* It's matching the tag. Add it to be logged and
-                             * update "flag_added" to add the packet once. */
-                            p->flags |= PKT_HAS_TAG;
-                            flag_added++;
+        /* If this packet triggered the rule with tag, we dont need
+         * to log it (the alert will log it) */
+        if (!(iter->flags & TAG_ENTRY_FLAG_SKIPPED_FIRST)) {
+            iter->flags |= TAG_ENTRY_FLAG_SKIPPED_FIRST;
+        } else {
+            /* Update metrics; remove if tag expired; and set alerts */
+            switch (iter->metric) {
+                case DETECT_TAG_METRIC_PACKET:
+                    if (iter->packets > iter->count) {
+                        /* tag expired */
+                        if (prev != NULL) {
+                            tde = iter;
+                            prev->next = iter->next;
+                            iter = iter->next;
+                            SCFree(tde);
+                            SC_ATOMIC_SUB(num_tags, 1);
+                            continue;
+                        } else {
+                            tde = iter;
+                            iter = iter->next;
+                            SCFree(tde);
+                            SC_ATOMIC_SUB(num_tags, 1);
+                            host->tag = iter;
+                            continue;
                         }
-                        break;
-                    case DETECT_TAG_METRIC_BYTES:
-                        if (iter->bytes > iter->td->count) {
-                            /* tag expired */
-                            if (prev != NULL) {
-                                tde = iter;
-                                prev->next = iter->next;
-                                iter = iter->next;
-                                SCFree(tde);
-                                SC_ATOMIC_SUB(num_tags, 1);
-                                continue;
-                            } else {
-                                tde = iter;
-                                iter = iter->next;
-                                SCFree(tde);
-                                SC_ATOMIC_SUB(num_tags, 1);
-                                tde_src->header_entry = iter;
-                                continue;
-                            }
-                        } else if (flag_added == 0) {
-                            /* It's matching the tag. Add it to be logged and
-                             * update "flag_added" to add the packet once. */
-                            p->flags |= PKT_HAS_TAG;
-                            flag_added++;
+                    } else if (flag_added == 0) {
+                        /* It's matching the tag. Add it to be logged and
+                         * update "flag_added" to add the packet once. */
+                        p->flags |= PKT_HAS_TAG;
+                        flag_added++;
+                    }
+                    break;
+                case DETECT_TAG_METRIC_BYTES:
+                    if (iter->bytes > iter->count) {
+                        /* tag expired */
+                        if (prev != NULL) {
+                            tde = iter;
+                            prev->next = iter->next;
+                            iter = iter->next;
+                            SCFree(tde);
+                            SC_ATOMIC_SUB(num_tags, 1);
+                            continue;
+                        } else {
+                            tde = iter;
+                            iter = iter->next;
+                            SCFree(tde);
+                            SC_ATOMIC_SUB(num_tags, 1);
+                            host->tag = iter;
+                            continue;
                         }
-                        break;
-                    case DETECT_TAG_METRIC_SECONDS:
-                        /* last_ts handles this metric, but also a generic time based
-                         * expiration to prevent dead sessions/hosts */
-                        if (iter->last_ts.tv_sec - iter->first_ts.tv_sec > (int)iter->td->count) {
-                            /* tag expired */
-                            if (prev != NULL) {
-                                tde = iter;
-                                prev->next = iter->next;
-                                iter = iter->next;
-                                SCFree(tde);
-                                SC_ATOMIC_SUB(num_tags, 1);
-                                continue;
-                            } else {
-                                tde = iter;
-                                iter = iter->next;
-                                SCFree(tde);
-                                SC_ATOMIC_SUB(num_tags, 1);
-                                tde_src->header_entry = iter;
-                                continue;
-                            }
-                        } else if (flag_added == 0) {
-                            /* It's matching the tag. Add it to be logged and
-                             * update "flag_added" to add the packet once. */
-                            p->flags |= PKT_HAS_TAG;
-                            flag_added++;
+                    } else if (flag_added == 0) {
+                        /* It's matching the tag. Add it to be logged and
+                         * update "flag_added" to add the packet once. */
+                        p->flags |= PKT_HAS_TAG;
+                        flag_added++;
+                    }
+                    break;
+                case DETECT_TAG_METRIC_SECONDS:
+                    /* last_ts handles this metric, but also a generic time based
+                     * expiration to prevent dead sessions/hosts */
+                    if (iter->last_ts - iter->first_ts > iter->count) {
+                        /* tag expired */
+                        if (prev != NULL) {
+                            tde = iter;
+                            prev->next = iter->next;
+                            iter = iter->next;
+                            SCFree(tde);
+                            SC_ATOMIC_SUB(num_tags, 1);
+                            continue;
+                        } else {
+                            tde = iter;
+                            iter = iter->next;
+                            SCFree(tde);
+                            SC_ATOMIC_SUB(num_tags, 1);
+                            host->tag = iter;
+                            continue;
                         }
-                        break;
-                }
-
+                    } else if (flag_added == 0) {
+                        /* It's matching the tag. Add it to be logged and
+                         * update "flag_added" to add the packet once. */
+                        p->flags |= PKT_HAS_TAG;
+                        flag_added++;
+                    }
+                    break;
             }
 
-            prev = iter;
-            iter = iter->next;
         }
+
+        prev = iter;
+        iter = iter->next;
     }
+}
 
-    if (tde_dst != NULL) {
-        iter = tde_dst->header_entry;
-        prev = NULL;
-        while (iter != NULL) {
-            /* update counters */
-            iter->last_ts.tv_sec = p->ts.tv_sec;
-            iter->packets++;
-            iter->bytes += GET_PKT_LEN(p);
-
-            /* If this packet triggered the rule with tag, we dont need
-             * to log it (the alert will log it) */
-            if (iter->skipped_first == 0) {
-                iter->skipped_first = 1;
-            } else if (iter->td != NULL) {
-                /* Update metrics; remove if tag expired; and set alerts */
-                switch (iter->td->metric) {
-                    case DETECT_TAG_METRIC_PACKET:
-                        if (iter->packets > iter->td->count) {
-                            /* tag expired */
-                            if (prev != NULL) {
-                                tde = iter;
-                                prev->next = iter->next;
-                                iter = iter->next;
-                                SCFree(tde);
-                                SC_ATOMIC_SUB(num_tags, 1);
-                                continue;
-                            } else {
-                                tde = iter;
-                                iter = iter->next;
-                                SCFree(tde);
-                                SC_ATOMIC_SUB(num_tags, 1);
-                                tde_dst->header_entry = iter;
-                                continue;
-                            }
-                        } else if (flag_added == 0) {
-                            /* It's matching the tag. Add it to be logged and
-                             * update "flag_added" to add the packet once. */
-                            p->flags |= PKT_HAS_TAG;
-                            flag_added++;
-                        }
-                        break;
-                    case DETECT_TAG_METRIC_BYTES:
-                        if (iter->bytes > iter->td->count) {
-                            /* tag expired */
-                            if (prev != NULL) {
-                                tde = iter;
-                                prev->next = iter->next;
-                                iter = iter->next;
-                                SCFree(tde);
-                                SC_ATOMIC_SUB(num_tags, 1);
-                                continue;
-                            } else {
-                                tde = iter;
-                                iter = iter->next;
-                                SCFree(tde);
-                                SC_ATOMIC_SUB(num_tags, 1);
-                                tde_dst->header_entry = iter;
-                                continue;
-                            }
-                        }  else if (flag_added == 0) {
-                            /* It's matching the tag. Add it to be logged and
-                             * update "flag_added" to add the packet once. */
-                            p->flags |= PKT_HAS_TAG;
-                            flag_added++;
-                        }
-                        break;
-                    case DETECT_TAG_METRIC_SECONDS:
-                        /* last_ts handles this metric, but also a generic time based
-                         * expiration to prevent dead sessions/hosts */
-                        if (iter->last_ts.tv_sec - iter->first_ts.tv_sec > (int)iter->td->count) {
-                            /* tag expired */
-                            if (prev != NULL) {
-                                tde = iter;
-                                prev->next = iter->next;
-                                iter = iter->next;
-                                SCFree(tde);
-                                SC_ATOMIC_SUB(num_tags, 1);
-                                continue;
-                            } else {
-                                tde = iter;
-                                iter = iter->next;
-                                SCFree(tde);
-                                SC_ATOMIC_SUB(num_tags, 1);
-                                tde_dst->header_entry = iter;
-                                continue;
-                            }
-                        } else if (flag_added == 0) {
-                            /* It's matching the tag. Add it to be logged and
-                             * update "flag_added" to add the packet once. */
-                            p->flags |= PKT_HAS_TAG;
-                            flag_added++;
-                        }
-                        break;
-                }
-            }
+/**
+ * \brief Search tags for src and dst. Update entries of the tag, remove if necessary
+ *
+ * \param de_ctx Detect context
+ * \param det_ctx Detect thread context
+ * \param p packet
+ *
+ */
+void TagHandlePacket(DetectEngineCtx *de_ctx,
+        DetectEngineThreadCtx *det_ctx, Packet *p)
+{
+    /* If there's no tag, get out of here */
+    unsigned int current_tags = SC_ATOMIC_GET(num_tags);
+    if (current_tags == 0)
+        return;
 
-            prev = iter;
-            iter = iter->next;
-        }
+    /* First update and get session tags */
+    if (p->flow != NULL) {
+        SCMutexLock(&p->flow->m);
+        TagHandlePacketFlow(p->flow, p);
+        SCMutexUnlock(&p->flow->m);
     }
 
-    SCMutexUnlock(&tag_ctx->lock);
+    Host *src = HostLookupHostFromHash(&p->src);
+    if (src) {
+        if (src->tag != NULL) {
+            TagHandlePacketHost(src,p);
+        }
+        HostRelease(src);
+    }
+    Host *dst = HostLookupHostFromHash(&p->dst);
+    if (dst) {
+        if (dst->tag != NULL) {
+            TagHandlePacketHost(dst,p);
+        }
+        HostRelease(dst);
+    }
 }
 
 /**
@@ -710,100 +494,50 @@ void TagHandlePacket(DetectEngineCtx *de_ctx,
  * \param tag_ctx Tag context
  * \param ts the current time
  *
+ * \retval 1 no tags or tags removed -- host is free to go (from tag perspective)
+ * \retval 0 still active tags
  */
-static void TagTimeoutRemove(DetectTagHostCtx *tag_ctx, struct timeval *tv)
+int TagTimeoutCheck(Host *host, struct timeval *tv)
 {
-    HashListTableBucket *next = NULL;
-    HashListTableBucket *buck = NULL;
-
     DetectTagDataEntry *tde = NULL;
     DetectTagDataEntry *tmp = NULL;
     DetectTagDataEntry *prev = NULL;
+    int retval = 1;
 
-    DetectTagDataEntryList *tdl = NULL;
-
-    buck = HashListTableGetListHead(tag_ctx->tag_hash_table_ipv4);
-
-    while (buck != NULL) {
-        /* get the next before we free "buck" */
-        next = HashListTableGetListNext(buck);
-        tdl = HashListTableGetListData(buck);
-
-        if (tdl != NULL && tdl->header_entry != NULL) {
-            tmp = tdl->header_entry;
-
-            prev = NULL;
-            while (tmp != NULL) {
-
-                if ((tv->tv_sec - tmp->last_ts.tv_sec) <= TAG_MAX_LAST_TIME_SEEN) {
-                    prev = tmp;
-                    tmp = tmp->next;
-                    continue;
-                }
-
-                if (prev != NULL) {
-                    prev->next = tmp->next;
-
-                    tde = tmp;
-                    tmp = tmp->next;
-
-                    SCFree(tde);
-                    SC_ATOMIC_SUB(num_tags, 1);
-                } else {
-                    tdl->header_entry = tmp->next;
+    if (host->tag == NULL)
+        return 1;
 
-                    tde = tmp;
-                    tmp = tmp->next;
+    tmp = host->tag;
 
-                    SCFree(tde);
-                    SC_ATOMIC_SUB(num_tags, 1);
-                }
-            }
+    prev = NULL;
+    while (tmp != NULL) {
+        if ((tv->tv_sec - tmp->last_ts) <= TAG_MAX_LAST_TIME_SEEN) {
+            prev = tmp;
+            tmp = tmp->next;
+            retval = 0;
+            continue;
         }
-        buck = next;
-    }
-
-    buck = HashListTableGetListHead(tag_ctx->tag_hash_table_ipv6);
-
-    while (buck != NULL) {
-        /* get the next before we free "buck" */
-        next = HashListTableGetListNext(buck);
-        tdl = HashListTableGetListData(buck);
-
-        if (tdl != NULL && tdl->header_entry != NULL) {
-            tmp = tdl->header_entry;
 
-            prev = NULL;
-            while (tmp != NULL) {
+        /* timed out */
 
-                if ((tv->tv_sec - tmp->last_ts.tv_sec) <= TAG_MAX_LAST_TIME_SEEN) {
-                    prev = tmp;
-                    tmp = tmp->next;
-                    continue;
-                }
-
-                if (prev != NULL) {
-                    prev->next = tmp->next;
+        if (prev != NULL) {
+            prev->next = tmp->next;
 
-                    tde = tmp;
-                    tmp = tmp->next;
+            tde = tmp;
+            tmp = tde->next;
 
-                    SCFree(tde);
-                    SC_ATOMIC_SUB(num_tags, 1);
-                } else {
-                    tdl->header_entry = tmp->next;
+            SCFree(tde);
+            SC_ATOMIC_SUB(num_tags, 1);
+        } else {
+            host->tag = tmp->next;
 
-                    tde = tmp;
-                    tmp = tmp->next;
+            tde = tmp;
+            tmp = tde->next;
 
-                    SCFree(tde);
-                    SC_ATOMIC_SUB(num_tags, 1);
-                }
-            }
+            SCFree(tde);
+            SC_ATOMIC_SUB(num_tags, 1);
         }
-        buck = next;
     }
-
-    return;
+    return retval;
 }
 
diff --git a/src/detect-engine-tag.h b/src/detect-engine-tag.h
index 58044245a8..aada77a211 100644
--- a/src/detect-engine-tag.h
+++ b/src/detect-engine-tag.h
@@ -27,10 +27,9 @@
 #ifndef __DETECT_ENGINE_TAG_H__
 #define __DETECT_ENGINE_TAG_H__
 
+#include "host.h"
 #include "detect.h"
 
-#define TAG_HASH_SIZE 0xffff
-
 /* This limit should be overwriten/predefined at the config file
  * to limit the options to prevent possible DOS situations. We should also
  * create a limit for bytes and a limit for number of packets */
@@ -43,16 +42,18 @@
 #define TAG_SIG_GEN           2
 #define TAG_SIG_ID            1
 
-void TagHashInit(DetectTagHostCtx *);
-int TagHashAddTag(DetectTagHostCtx *, DetectTagDataEntry *, Packet *);
-void TagContextDestroy(DetectTagHostCtx *);
-void TagHandlePacket(DetectEngineCtx *, DetectEngineThreadCtx *,
-                     Packet *);
+int TagHashAddTag(DetectTagDataEntry *, Packet *);
+int TagFlowAdd(Packet *, DetectTagDataEntry *);
+
+void TagContextDestroy(void);
+void TagHandlePacket(DetectEngineCtx *, DetectEngineThreadCtx *, Packet *);
 
 void TagInitCtx(void);
 void TagDestroyCtx(void);
 void TagRestartCtx(void);
 
+int TagTimeoutCheck(Host *, struct timeval *);
+
 #endif /* __DETECT_ENGINE_TAG_H__ */
 
 
diff --git a/src/detect-tag.c b/src/detect-tag.c
index 05c1ac2b7e..0a6048b752 100644
--- a/src/detect-tag.c
+++ b/src/detect-tag.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2011 Open Information Security Foundation
+/* Copyright (C) 2007-2012 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -50,7 +50,6 @@
 #include "threads.h"
 
 SC_ATOMIC_EXTERN(unsigned int, num_tags);
-extern DetectTagHostCtx *tag_ctx;
 
 /* format: tag: <type>, <count>, <metric>, [direction]; */
 #define PARSE_REGEX  "^\\s*(host|session)\\s*(,\\s*(\\d+)\\s*,\\s*(packets|bytes|seconds)\\s*(,\\s*(src|dst))?\\s*)?$"
@@ -97,97 +96,6 @@ error:
     return;
 }
 
-DetectTagDataEntry *DetectTagDataCopy(DetectTagDataEntry *dtd) {
-    DetectTagDataEntry *tde = SCMalloc(sizeof(DetectTagDataEntry));
-    if (tde == NULL) {
-        return NULL;
-    }
-    memset(tde, 0, sizeof(DetectTagDataEntry));
-
-    tde->sid = dtd->sid;
-    tde->gid = dtd->gid;
-
-    tde->td = dtd->td;
-    tde->first_ts.tv_sec = dtd->first_ts.tv_sec;
-    tde->first_ts.tv_usec = dtd->first_ts.tv_usec;
-    tde->last_ts.tv_sec = dtd->last_ts.tv_sec;
-    tde->last_ts.tv_usec = dtd->last_ts.tv_usec;
-    return tde;
-}
-
-/**
- * \brief This function is used to add a tag to a session (type session)
- *        or update it if it's already installed. The number of times to
- *        allow an update is limited by DETECT_TAG_MATCH_LIMIT. This way
- *        repetitive matches to the same rule are limited of setting tags,
- *        to avoid DOS attacks
- *
- * \param p pointer to the current packet
- * \param tde pointer to the new DetectTagDataEntry
- *
- * \retval 0 if the tde was added succesfuly
- * \retval 1 if an entry of this sid/gid already exist and was updated
- */
-int DetectTagFlowAdd(Packet *p, DetectTagDataEntry *tde) {
-    uint8_t updated = 0;
-    uint16_t num_tags = 0;
-    DetectTagDataEntry *iter = NULL;
-
-    if (p->flow == NULL)
-        return 1;
-
-    SCMutexLock(&p->flow->m);
-
-    if (p->flow->tag_list == NULL) {
-        p->flow->tag_list = SCMalloc(sizeof(DetectTagDataEntryList));
-        if (p->flow->tag_list == NULL) {
-            goto error;
-        }
-        memset(p->flow->tag_list, 0, sizeof(DetectTagDataEntryList));
-    } else {
-        iter = p->flow->tag_list->header_entry;
-
-        /* First iterate installed entries searching a duplicated sid/gid */
-        for (; iter != NULL; iter = iter->next) {
-            num_tags++;
-
-            if (iter->sid == tde->sid && iter->gid == tde->gid) {
-                iter->cnt_match++;
-
-                /* If so, update data, unless the maximum MATCH limit is
-                 * reached. This prevents possible DOS attacks */
-                if (iter->cnt_match < DETECT_TAG_MATCH_LIMIT) {
-                    /* Reset time and counters */
-                    iter->first_ts.tv_sec = iter->last_ts.tv_sec = tde->first_ts.tv_sec;
-                    iter->packets = 0;
-                    iter->bytes = 0;
-                }
-                updated = 1;
-                break;
-            }
-        }
-    }
-
-    /* If there was no entry of this rule, prepend the new tde */
-    if (updated == 0 && num_tags < DETECT_TAG_MAX_TAGS) {
-        DetectTagDataEntry *new_tde = DetectTagDataCopy(tde);
-        if (new_tde != NULL) {
-            new_tde->next = p->flow->tag_list->header_entry;
-            p->flow->tag_list->header_entry = new_tde;
-            SC_ATOMIC_ADD(num_tags, 1);
-        }
-    } else if (num_tags == DETECT_TAG_MAX_TAGS) {
-        SCLogDebug("Max tags for sessions reached (%"PRIu16")", num_tags);
-    }
-
-    SCMutexUnlock(&p->flow->m);
-    return updated;
-
-error:
-    SCMutexUnlock(&p->flow->m);
-    return 1;
-}
-
 /**
  * \brief This function is used to setup a tag for session/host
  *
@@ -202,25 +110,40 @@ error:
 int DetectTagMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Packet *p, Signature *s, SigMatch *m)
 {
     DetectTagData *td = (DetectTagData *) m->ctx;
-    DetectTagDataEntry tde;
-    memset(&tde, 0, sizeof(DetectTagDataEntry));
-    tde.sid = s->id;
-    tde.gid = s->gid;
-    tde.td = td;
-    tde.last_ts.tv_sec = tde.first_ts.tv_sec = p->ts.tv_usec;
 
     switch (td->type) {
         case DETECT_TAG_TYPE_HOST:
 #ifdef DEBUG
             BUG_ON(!(td->direction == DETECT_TAG_DIR_SRC || td->direction == DETECT_TAG_DIR_DST));
 #endif
+
+            DetectTagDataEntry tde;
+            memset(&tde, 0, sizeof(DetectTagDataEntry));
+            tde.sid = s->id;
+            tde.gid = s->gid;
+            tde.last_ts = tde.first_ts = p->ts.tv_sec;
+            tde.metric = td->metric;
+            tde.count = td->count;
+            if (td->direction == DETECT_TAG_DIR_SRC)
+                tde.flags |= TAG_ENTRY_FLAG_DIR_SRC;
+            else if (td->direction == DETECT_TAG_DIR_DST)
+                tde.flags |= TAG_ENTRY_FLAG_DIR_DST;
+
             SCLogDebug("Tagging Host with sid %"PRIu32":%"PRIu32"", s->id, s->gid);
-            TagHashAddTag(tag_ctx, &tde, p);
+            TagHashAddTag(&tde, p);
             break;
         case DETECT_TAG_TYPE_SESSION:
             if (p->flow != NULL) {
                 /* If it already exists it will be updated */
-                DetectTagFlowAdd(p, &tde);
+                DetectTagDataEntry tde;
+                memset(&tde, 0, sizeof(DetectTagDataEntry));
+                tde.sid = s->id;
+                tde.gid = s->gid;
+                tde.last_ts = tde.first_ts = p->ts.tv_usec;
+                tde.metric = td->metric;
+                tde.count = td->count;
+
+                TagFlowAdd(p, &tde);
             } else {
                 SCLogDebug("No flow to append the session tag");
             }
@@ -388,6 +311,19 @@ error:
 
 }
 
+/** \internal
+ *  \brief this function will free memory associated with
+ *        DetectTagDataEntry
+ *
+ *  \param td pointer to DetectTagDataEntry
+ */
+static void DetectTagDataEntryFree(void *ptr) {
+    if (ptr != NULL) {
+        DetectTagDataEntry *dte = (DetectTagDataEntry *)ptr;
+        SCFree(dte);
+    }
+}
+
 
 /**
  * \brief this function will free all the entries of a list
@@ -397,8 +333,7 @@ error:
  */
 void DetectTagDataListFree(void *ptr) {
     if (ptr != NULL) {
-        DetectTagDataEntryList *list = (DetectTagDataEntryList *)ptr;
-        DetectTagDataEntry *entry = list->header_entry;
+        DetectTagDataEntry *entry = ptr;
 
         while (entry != NULL) {
             DetectTagDataEntry *next_entry = entry->next;
@@ -406,19 +341,6 @@ void DetectTagDataListFree(void *ptr) {
             SC_ATOMIC_SUB(num_tags, 1);
             entry = next_entry;
         }
-        SCFree(list);
-    }
-}
-/**
- * \brief this function will free memory associated with
- *        DetectTagDataEntry
- *
- * \param td pointer to DetectTagDataEntry
- */
-void DetectTagDataEntryFree(void *ptr) {
-    if (ptr != NULL) {
-        DetectTagDataEntry *dte = (DetectTagDataEntry *)ptr;
-        SCFree(dte);
     }
 }
 
diff --git a/src/detect-tag.h b/src/detect-tag.h
index 25138b0d6f..af901fcb52 100644
--- a/src/detect-tag.h
+++ b/src/detect-tag.h
@@ -71,33 +71,35 @@ typedef struct DetectTagData_ {
 
 /** This is the installed data at the session/global or host table */
 typedef struct DetectTagDataEntry_ {
-    DetectTagData *td;                  /**< Pointer referencing the tag parameters */
+    uint8_t flags:3;
+    uint8_t metric:5;
+    uint8_t pad0;
+    uint16_t cnt_match;                 /**< number of times this tag was reset/updated */
+
+    uint32_t count;                     /**< count setting from rule */
     uint32_t sid;                       /**< sid originating the tag */
     uint32_t gid;                       /**< gid originating the tag */
-    uint32_t packets;                   /**< number of packets */
-    uint32_t bytes;                     /**< number of bytes */
-    struct timeval first_ts;            /**< First time seen (for metric = seconds) */
-    struct timeval last_ts;             /**< Last time seen (to prune old sessions) */
+    union {
+        uint32_t packets;               /**< number of packets (metric packets) */
+        uint32_t bytes;                 /**< number of bytes (metric bytes) */
+    };
+    uint32_t first_ts;                  /**< First time seen (for metric = seconds) */
+    uint32_t last_ts;                   /**< Last time seen (to prune old sessions) */
+#if __WORDSIZE == 64
+    uint32_t pad1;
+#endif
     struct DetectTagDataEntry_ *next;   /**< Pointer to the next tag of this
-                                         * session/src_host/dst_host (if any from other rule) */
-    uint16_t cnt_match;                 /**< number of times this tag was reset/updated */
-    uint8_t skipped_first;              /**< Used for output. The first packet write the
-                                             header with the data of the sig. The next packets use
-                                             gid/sid/rev of the tagging engine */
+                                         *   session/src_host/dst_host (if any from other rule) */
 } DetectTagDataEntry;
 
-typedef struct DetectTagDataEntryList_ {
-    DetectTagDataEntry *header_entry;
-    Address addr;                       /**< Var used to store dst or src addr */
-    uint8_t ipv;                        /**< IP Version */
-} DetectTagDataEntryList;
+#define TAG_ENTRY_FLAG_DIR_SRC          0x01
+#define TAG_ENTRY_FLAG_DIR_DST          0x02
+#define TAG_ENTRY_FLAG_SKIPPED_FIRST    0x04
 
 /* prototypes */
 void DetectTagRegister (void);
 void DetectTagDataFree(void *ptr);
-void DetectTagDataEntryFree(void *ptr);
 void DetectTagDataListFree(void *ptr);
-DetectTagDataEntry *DetectTagDataCopy(DetectTagDataEntry *dtd);
 
 #endif /* __DETECT_TAG_H__ */
 
diff --git a/src/detect.h b/src/detect.h
index b7603b9b97..62f34aad2d 100644
--- a/src/detect.h
+++ b/src/detect.h
@@ -525,14 +525,6 @@ typedef struct ThresholdCtx_    {
     uint32_t th_size;
 } ThresholdCtx;
 
-/** \brief tag ctx */
-typedef struct DetectTagHostCtx_ {
-    HashListTable *tag_hash_table_ipv4;   /**< Ipv4 hash table      */
-    HashListTable *tag_hash_table_ipv6;   /**< Ipv6 hash table      */
-    SCMutex lock;                         /**< Mutex for the ctx    */
-    struct timeval last_ts;               /**< Last time the ctx was pruned */
-} DetectTagHostCtx;
-
 /** \brief main detection engine ctx */
 typedef struct DetectEngineCtx_ {
     uint8_t flags;
diff --git a/src/flow-manager.c b/src/flow-manager.c
index 7a970755b7..c13bd67acd 100644
--- a/src/flow-manager.c
+++ b/src/flow-manager.c
@@ -60,6 +60,8 @@
 
 #include "app-layer-parser.h"
 
+#include "host-timeout.h"
+
 /* Run mode selected at suricata.c */
 extern int run_mode;
 
@@ -378,7 +380,18 @@ void *FlowManagerThread(void *td)
     struct timespec cond_time;
     int flow_update_delay_sec = FLOW_NORMAL_MODE_UPDATE_DELAY_SEC;
     int flow_update_delay_nsec = FLOW_NORMAL_MODE_UPDATE_DELAY_NSEC;
-
+/* VJ leaving disabled for now, as hosts are only used by tags and the numbers
+ * are really low. Might confuse ppl
+    uint16_t flow_mgr_host_prune = SCPerfTVRegisterCounter("hosts.pruned", th_v,
+            SC_PERF_TYPE_UINT64,
+            "NULL");
+    uint16_t flow_mgr_host_active = SCPerfTVRegisterCounter("hosts.active", th_v,
+            SC_PERF_TYPE_Q_NORMAL,
+            "NULL");
+    uint16_t flow_mgr_host_spare = SCPerfTVRegisterCounter("hosts.spare", th_v,
+            SC_PERF_TYPE_Q_NORMAL,
+            "NULL");
+*/
     uint16_t flow_mgr_cnt_clo = SCPerfTVRegisterCounter("flow_mgr.closed_pruned", th_v,
             SC_PERF_TYPE_UINT64,
             "NULL");
@@ -452,6 +465,16 @@ void *FlowManagerThread(void *td)
         FlowTimeoutCounters counters = { 0, 0, 0, };
         FlowTimeoutHash(&ts, 0 /* check all */, &counters);
 
+
+        //uint32_t hosts_pruned =
+        HostTimeoutHash(&ts);
+/*
+        SCPerfCounterAddUI64(flow_mgr_host_prune, th_v->sc_perf_pca, (uint64_t)hosts_pruned);
+        uint32_t hosts_active = HostGetActiveCount();
+        SCPerfCounterSetUI64(flow_mgr_host_active, th_v->sc_perf_pca, (uint64_t)hosts_active);
+        uint32_t hosts_spare = HostGetSpareCount();
+        SCPerfCounterSetUI64(flow_mgr_host_spare, th_v->sc_perf_pca, (uint64_t)hosts_spare);
+*/
         SCPerfCounterAddUI64(flow_mgr_cnt_clo, th_v->sc_perf_pca, (uint64_t)counters.clo);
         SCPerfCounterAddUI64(flow_mgr_cnt_new, th_v->sc_perf_pca, (uint64_t)counters.new);
         SCPerfCounterAddUI64(flow_mgr_cnt_est, th_v->sc_perf_pca, (uint64_t)counters.est);
diff --git a/src/flow.h b/src/flow.h
index e5d62312c3..5557509a26 100644
--- a/src/flow.h
+++ b/src/flow.h
@@ -290,7 +290,7 @@ typedef struct Flow_
     struct SigGroupHead_ *sgh_toserver;
 
     /** List of tags of this flow (from "tag" keyword of type "session") */
-    DetectTagDataEntryList *tag_list;
+    void *tag_list;
 
     /* pointer to the var list */
     GenericVar *flowvar;
diff --git a/src/host-queue.c b/src/host-queue.c
new file mode 100644
index 0000000000..0e44b01e17
--- /dev/null
+++ b/src/host-queue.c
@@ -0,0 +1,138 @@
+/* Copyright (C) 2007-2012 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ *
+ * Host queue handler functions
+ */
+
+#include "suricata-common.h"
+#include "threads.h"
+#include "debug.h"
+#include "host-queue.h"
+#include "util-error.h"
+#include "util-debug.h"
+#include "util-print.h"
+
+HostQueue *HostQueueInit (HostQueue *q) {
+    if (q != NULL) {
+        memset(q, 0, sizeof(HostQueue));
+        HQLOCK_INIT(q);
+    }
+    return q;
+}
+
+HostQueue *HostQueueNew() {
+    HostQueue *q = (HostQueue *)SCMalloc(sizeof(HostQueue));
+    if (q == NULL) {
+        SCLogError(SC_ERR_FATAL, "Fatal error encountered in HostQueueNew. Exiting...");
+        exit(EXIT_SUCCESS);
+    }
+    q = HostQueueInit(q);
+    return q;
+}
+
+/**
+ *  \brief Destroy a host queue
+ *
+ *  \param q the host queue to destroy
+ */
+void HostQueueDestroy (HostQueue *q) {
+    HQLOCK_DESTROY(q);
+}
+
+/**
+ *  \brief add a host to a queue
+ *
+ *  \param q queue
+ *  \param h host
+ */
+void HostEnqueue (HostQueue *q, Host *h) {
+#ifdef DEBUG
+    BUG_ON(q == NULL || h == NULL);
+#endif
+
+    HQLOCK_LOCK(q);
+
+    /* more hosts in queue */
+    if (q->top != NULL) {
+        h->lnext = q->top;
+        q->top->lprev = h;
+        q->top = h;
+    /* only host */
+    } else {
+        q->top = h;
+        q->bot = h;
+    }
+    q->len++;
+#ifdef DBG_PERF
+    if (q->len > q->dbg_maxlen)
+        q->dbg_maxlen = q->len;
+#endif /* DBG_PERF */
+    HQLOCK_UNLOCK(q);
+}
+
+/**
+ *  \brief remove a host from the queue
+ *
+ *  \param q queue
+ *
+ *  \retval h host or NULL if empty list.
+ */
+Host *HostDequeue (HostQueue *q) {
+    HQLOCK_LOCK(q);
+
+    Host *h = q->bot;
+    if (h == NULL) {
+        HQLOCK_UNLOCK(q);
+        return NULL;
+    }
+
+    /* more packets in queue */
+    if (q->bot->lprev != NULL) {
+        q->bot = q->bot->lprev;
+        q->bot->lnext = NULL;
+    /* just the one we remove, so now empty */
+    } else {
+        q->top = NULL;
+        q->bot = NULL;
+    }
+
+#ifdef DEBUG
+    BUG_ON(q->len == 0);
+#endif
+    if (q->len > 0)
+        q->len--;
+
+    h->lnext = NULL;
+    h->lprev = NULL;
+
+    HQLOCK_UNLOCK(q);
+    return h;
+}
+
+uint32_t HostQueueLen(HostQueue *q) {
+    uint32_t len;
+    HQLOCK_LOCK(q);
+    len = q->len;
+    HQLOCK_UNLOCK(q);
+    return len;
+}
+
diff --git a/src/host-queue.h b/src/host-queue.h
new file mode 100644
index 0000000000..386d0f6e7e
--- /dev/null
+++ b/src/host-queue.h
@@ -0,0 +1,84 @@
+/* Copyright (C) 2007-2012 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ */
+
+#ifndef __HOST_QUEUE_H__
+#define __HOST_QUEUE_H__
+
+#include "suricata-common.h"
+#include "host.h"
+
+/** Spinlocks or Mutex for the host queues. */
+//#define HQLOCK_SPIN
+#define HQLOCK_MUTEX
+
+#ifdef HQLOCK_SPIN
+    #ifdef HQLOCK_MUTEX
+        #error Cannot enable both HQLOCK_SPIN and HQLOCK_MUTEX
+    #endif
+#endif
+
+/* Define a queue for storing hosts */
+typedef struct HostQueue_
+{
+    Host *top;
+    Host *bot;
+    uint32_t len;
+#ifdef DBG_PERF
+    uint32_t dbg_maxlen;
+#endif /* DBG_PERF */
+#ifdef HQLOCK_MUTEX
+    SCMutex m;
+#elif defined HQLOCK_SPIN
+    SCSpinlock s;
+#else
+    #error Enable HQLOCK_SPIN or HQLOCK_MUTEX
+#endif
+} HostQueue;
+
+#ifdef HQLOCK_SPIN
+    #define HQLOCK_INIT(q) SCSpinInit(&(q)->s, 0)
+    #define HQLOCK_DESTROY(q) SCSpinDestroy(&(q)->s)
+    #define HQLOCK_LOCK(q) SCSpinLock(&(q)->s)
+    #define HQLOCK_TRYLOCK(q) SCSpinTrylock(&(q)->s)
+    #define HQLOCK_UNLOCK(q) SCSpinUnlock(&(q)->s)
+#elif defined HQLOCK_MUTEX
+    #define HQLOCK_INIT(q) SCMutexInit(&(q)->m, NULL)
+    #define HQLOCK_DESTROY(q) SCMutexDestroy(&(q)->m)
+    #define HQLOCK_LOCK(q) SCMutexLock(&(q)->m)
+    #define HQLOCK_TRYLOCK(q) SCMutexTrylock(&(q)->m)
+    #define HQLOCK_UNLOCK(q) SCMutexUnlock(&(q)->m)
+#else
+    #error Enable HQLOCK_SPIN or HQLOCK_MUTEX
+#endif
+
+/* prototypes */
+HostQueue *HostQueueNew();
+HostQueue *HostQueueInit(HostQueue *);
+void HostQueueDestroy (HostQueue *);
+
+void HostEnqueue (HostQueue *, Host *);
+Host *HostDequeue (HostQueue *);
+uint32_t HostQueueLen(HostQueue *);
+
+#endif /* __HOST_QUEUE_H__ */
+
diff --git a/src/host-timeout.c b/src/host-timeout.c
new file mode 100644
index 0000000000..991d90f004
--- /dev/null
+++ b/src/host-timeout.c
@@ -0,0 +1,159 @@
+/* Copyright (C) 2007-2012 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ */
+
+#include "suricata-common.h"
+#include "host.h"
+#include "detect-engine-tag.h"
+
+uint32_t HostGetSpareCount(void) {
+    return HostSpareQueueGetSize();
+}
+
+uint32_t HostGetActiveCount(void) {
+    return SC_ATOMIC_GET(host_counter);
+}
+
+/** \internal
+ *  \brief See if we can really discard this host. Check use_cnt reference.
+ *
+ *  \param h host
+ *  \param ts timestamp
+ *
+ *  \retval 0 not timed out just yet
+ *  \retval 1 fully timed out, lets kill it
+ */
+static int HostHostTimedOut(Host *h, struct timeval *ts) {
+    int tags = 0;
+    int thresholds = 0;
+
+    /** never prune a host that is used by a packet
+     *  we are currently processing in one of the threads */
+    if (h->use_cnt > 0) {
+        return 0;
+    }
+
+    if (h->tag && TagTimeoutCheck(h, ts) == 0) {
+        tags = 1;
+    }
+    if (h->threshold) {
+        // run threshold cleanup
+    }
+
+    if (tags || thresholds)
+        return 0;
+
+    return 1;
+}
+
+/**
+ *  \internal
+ *
+ *  \brief check all hosts in a hash row for timing out
+ *
+ *  \param hb host hash row *LOCKED*
+ *  \param h last host in the hash row
+ *  \param ts timestamp
+ *
+ *  \retval cnt timed out hosts
+ */
+static uint32_t HostHashRowTimeout(HostHashRow *hb, Host *h, struct timeval *ts)
+{
+    uint32_t cnt = 0;
+
+    do {
+        if (SCMutexTrylock(&h->m) != 0) {
+            h = h->hprev;
+            continue;
+        }
+
+        Host *next_host = h->hprev;
+
+        /* check if the host is fully timed out and
+         * ready to be discarded. */
+        if (HostHostTimedOut(h, ts) == 1) {
+            /* remove from the hash */
+            if (h->hprev != NULL)
+                h->hprev->hnext = h->hnext;
+            if (h->hnext != NULL)
+                h->hnext->hprev = h->hprev;
+            if (hb->head == h)
+                hb->head = h->hnext;
+            if (hb->tail == h)
+                hb->tail = h->hprev;
+
+            h->hnext = NULL;
+            h->hprev = NULL;
+
+            HostClearMemory (h);
+
+            /* no one is referring to this host, use_cnt 0, removed from hash
+             * so we can unlock it and move it back to the spare queue. */
+            SCMutexUnlock(&h->m);
+
+            /* move to spare list */
+            HostMoveToSpare(h);
+
+            cnt++;
+        } else {
+            SCMutexUnlock(&h->m);
+        }
+
+        h = next_host;
+    } while (h != NULL);
+
+    return cnt;
+}
+
+/**
+ *  \brief time out hosts from the hash
+ *
+ *  \param ts timestamp
+ *
+ *  \retval cnt number of timed out host
+ */
+uint32_t HostTimeoutHash(struct timeval *ts) {
+    uint32_t idx = 0;
+    uint32_t cnt = 0;
+
+    for (idx = 0; idx < host_config.hash_size; idx++) {
+        HostHashRow *hb = &host_hash[idx];
+        if (hb == NULL)
+            continue;
+        if (HRLOCK_TRYLOCK(hb) != 0)
+            continue;
+
+        /* host hash bucket is now locked */
+
+        if (hb->tail == NULL) {
+            HRLOCK_UNLOCK(hb);
+            continue;
+        }
+
+        /* we have a host, or more than one */
+        cnt += HostHashRowTimeout(hb, hb->tail, ts);
+        HRLOCK_UNLOCK(hb);
+    }
+
+    return cnt;
+}
+
diff --git a/src/host-timeout.h b/src/host-timeout.h
new file mode 100644
index 0000000000..6ea1e894e8
--- /dev/null
+++ b/src/host-timeout.h
@@ -0,0 +1,33 @@
+/* Copyright (C) 2007-2012 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ */
+
+#ifndef __HOST_TIMEOUT_H__
+#define __HOST_TIMEOUT_H__
+
+uint32_t HostTimeoutHash(struct timeval *ts);
+
+uint32_t HostGetSpareCount(void);
+uint32_t HostGetActiveCount(void);
+
+#endif
+
diff --git a/src/host.c b/src/host.c
index e7a87fdb2c..007619b679 100644
--- a/src/host.c
+++ b/src/host.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2010 Open Information Security Foundation
+/* Copyright (C) 2007-2012 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -20,18 +20,49 @@
  *
  * \author Victor Julien <victor@inliniac.net>
  *
- * Information about hosts for ip reputation.
+ * Information about hosts.
  */
 
 #include "suricata-common.h"
+#include "conf.h"
+
 #include "util-debug.h"
 #include "host.h"
 
+#include "util-random.h"
+#include "util-misc.h"
+#include "util-byte.h"
+
+#include "host-queue.h"
+
+#include "detect-tag.h"
+
+static Host *HostGetUsedHost(void);
+
+/** queue with spare hosts */
+static HostQueue host_spare_q;
+
+uint32_t HostSpareQueueGetSize(void) {
+    return HostQueueLen(&host_spare_q);
+}
+
+void HostMoveToSpare(Host *h) {
+    HostEnqueue(&host_spare_q, h);
+    SC_ATOMIC_SUB(host_counter, 1);
+}
+
 Host *HostAlloc(void) {
+    if ((SC_ATOMIC_GET(host_memuse) + sizeof(Host)) > host_config.memcap) {
+        return NULL;
+    }
+
+    SC_ATOMIC_ADD(host_memuse, sizeof(Host));
+
     Host *h = SCMalloc(sizeof(Host));
     if (h == NULL)
         goto error;
 
+    SCMutexInit(&h->m, NULL);
     return h;
 
 error:
@@ -39,7 +70,13 @@ error:
 }
 
 void HostFree(Host *h) {
-    SCFree(h);
+    if (h != NULL) {
+        HostClearMemory(h);
+
+        SCMutexDestroy(&h->m);
+        SCFree(h);
+        SC_ATOMIC_SUB(host_memuse, sizeof(Host));
+    }
 }
 
 Host *HostNew(Address *a) {
@@ -48,10 +85,7 @@ Host *HostNew(Address *a) {
         goto error;
 
     /* copy address */
-
-    /* set os and reputation to 0 */
-    h->os = HOST_OS_UNKNOWN;
-    h->reputation = HOST_REPU_UNKNOWN;
+    COPY_ADDRESS(a, &h->a);
 
     return h;
 
@@ -59,3 +93,503 @@ error:
     return NULL;
 }
 
+void HostClearMemory(Host *h) {
+    if (h->tag != NULL) {
+        DetectTagDataListFree(h->tag);
+        h->tag = NULL;
+    }
+}
+
+#define HOST_DEFAULT_HASHSIZE 4096
+#define HOST_DEFAULT_MEMCAP 16777216
+#define HOST_DEFAULT_PREALLOC 10000
+
+/** \brief initialize the configuration
+ *  \warning Not thread safe */
+void HostInitConfig(char quiet)
+{
+    SCLogDebug("initializing host engine...");
+
+    memset(&host_config,  0, sizeof(host_config));
+    //SC_ATOMIC_INIT(flow_flags);
+    SC_ATOMIC_INIT(host_memuse);
+    SC_ATOMIC_INIT(host_prune_idx);
+    HostQueueInit(&host_spare_q);
+
+    unsigned int seed = RandomTimePreseed();
+    /* set defaults */
+    host_config.hash_rand   = (int)( HOST_DEFAULT_HASHSIZE * (rand_r(&seed) / RAND_MAX + 1.0));
+
+    host_config.hash_size   = HOST_DEFAULT_HASHSIZE;
+    host_config.memcap      = HOST_DEFAULT_MEMCAP;
+    host_config.prealloc    = HOST_DEFAULT_PREALLOC;
+
+    /* Check if we have memcap and hash_size defined at config */
+    char *conf_val;
+    uint32_t configval = 0;
+
+    /** set config values for memcap, prealloc and hash_size */
+    if ((ConfGet("host.memcap", &conf_val)) == 1)
+    {
+        if (ParseSizeStringU64(conf_val, &host_config.memcap) < 0) {
+            SCLogError(SC_ERR_SIZE_PARSE, "Error parsing host.memcap "
+                       "from conf file - %s.  Killing engine",
+                       conf_val);
+            exit(EXIT_FAILURE);
+        }
+    }
+    if ((ConfGet("host.hash-size", &conf_val)) == 1)
+    {
+        if (ByteExtractStringUint32(&configval, 10, strlen(conf_val),
+                                    conf_val) > 0) {
+            host_config.hash_size = configval;
+        }
+    }
+
+    if ((ConfGet("host.prealloc", &conf_val)) == 1)
+    {
+        if (ByteExtractStringUint32(&configval, 10, strlen(conf_val),
+                                    conf_val) > 0) {
+            host_config.prealloc = configval;
+        }
+    }
+    SCLogDebug("Host config from suricata.yaml: memcap: %"PRIu64", hash-size: "
+               "%"PRIu32", prealloc: %"PRIu32, host_config.memcap,
+               host_config.hash_size, host_config.prealloc);
+
+    /* alloc hash memory */
+    host_hash = SCCalloc(host_config.hash_size, sizeof(HostHashRow));
+    if (host_hash == NULL) {
+        SCLogError(SC_ERR_FATAL, "Fatal error encountered in HostInitConfig. Exiting...");
+        exit(EXIT_FAILURE);
+    }
+    memset(host_hash, 0, host_config.hash_size * sizeof(HostHashRow));
+
+    uint32_t i = 0;
+    for (i = 0; i < host_config.hash_size; i++) {
+        HRLOCK_INIT(&host_hash[i]);
+    }
+    SC_ATOMIC_ADD(host_memuse, (host_config.hash_size * sizeof(HostHashRow)));
+
+    if (quiet == FALSE) {
+        SCLogInfo("allocated %llu bytes of memory for the host hash... "
+                  "%" PRIu32 " buckets of size %" PRIuMAX "",
+                  SC_ATOMIC_GET(host_memuse), host_config.hash_size,
+                  (uintmax_t)sizeof(HostHashRow));
+    }
+
+    /* pre allocate hosts */
+    for (i = 0; i < host_config.prealloc; i++) {
+        if ((SC_ATOMIC_GET(host_memuse) + sizeof(Host)) > host_config.memcap) {
+            printf("ERROR: HostAlloc failed (max host memcap reached): %s\n", strerror(errno));
+            exit(1);
+        }
+
+        Host *h = HostAlloc();
+        if (h == NULL) {
+            printf("ERROR: HostAlloc failed: %s\n", strerror(errno));
+            exit(1);
+        }
+        HostEnqueue(&host_spare_q,h);
+    }
+
+    if (quiet == FALSE) {
+        SCLogInfo("preallocated %" PRIu32 " hosts of size %" PRIuMAX "",
+                host_spare_q.len, (uintmax_t)sizeof(Host));
+        SCLogInfo("host memory usage: %llu bytes, maximum: %"PRIu64,
+                SC_ATOMIC_GET(host_memuse), host_config.memcap);
+    }
+
+    return;
+}
+
+/** \brief print some host stats
+ *  \warning Not thread safe */
+static void HostPrintStats (void)
+{
+#ifdef HOSTBITS_STATS
+    SCLogInfo("hostbits added: %" PRIu32 ", removed: %" PRIu32 ", max memory usage: %" PRIu32 "",
+        hostbits_added, hostbits_removed, hostbits_memuse_max);
+#endif /* HOSTBITS_STATS */
+    return;
+}
+
+/** \brief shutdown the flow engine
+ *  \warning Not thread safe */
+void HostShutdown(void)
+{
+    Host *h;
+    uint32_t u;
+
+    HostPrintStats();
+
+    /* free spare queue */
+    while((h = HostDequeue(&host_spare_q))) {
+        BUG_ON(h->use_cnt > 0);
+        HostFree(h);
+    }
+
+    /* clear and free the hash */
+    if (host_hash != NULL) {
+        for (u = 0; u < host_config.hash_size; u++) {
+            Host *h = host_hash[u].head;
+            while (h) {
+                Host *n = h->hnext;
+                HostClearMemory(h);
+                HostFree(h);
+                h = n;
+            }
+
+            HRLOCK_DESTROY(&host_hash[u]);
+        }
+        SCFree(host_hash);
+        host_hash = NULL;
+    }
+    SC_ATOMIC_SUB(host_memuse, host_config.hash_size * sizeof(HostHashRow));
+    HostQueueDestroy(&host_spare_q);
+
+    SC_ATOMIC_DESTROY(flow_prune_idx);
+    SC_ATOMIC_DESTROY(flow_memuse);
+    //SC_ATOMIC_DESTROY(flow_flags);
+    return;
+}
+
+/* calculate the hash key for this packet
+ *
+ * we're using:
+ *  hash_rand -- set at init time
+ *  source address
+ */
+uint32_t HostGetKey(Address *a) {
+    uint32_t key;
+
+    if (a->family == AF_INET) {
+        key = (host_config.hash_rand + a->addr_data32[0]) % host_config.hash_size;
+    } else if (a->family == AF_INET6) {
+        key = (host_config.hash_rand + a->addr_data32[0] + \
+               a->addr_data32[1] + a->addr_data32[2] + \
+               a->addr_data32[3]) % host_config.hash_size;
+    } else
+        key = 0;
+
+    return key;
+}
+
+/* Since two or more hosts can have the same hash key, we need to compare
+ * the flow with the current flow key. */
+#define CMP_HOST(h,a) \
+    (CMP_ADDR(&(h)->a, (a)))
+
+static inline int HostCompare(Host *h, Address *a) {
+    return CMP_HOST(h, a);
+}
+
+/**
+ *  \brief Get a new host
+ *
+ *  Get a new host. We're checking memcap first and will try to make room
+ *  if the memcap is reached.
+ *
+ *  \retval h *LOCKED* host on succes, NULL on error.
+ */
+static Host *HostGetNew(Address *a) {
+    Host *h = NULL;
+
+    /* get a host from the spare queue */
+    h = HostDequeue(&host_spare_q);
+    if (h == NULL) {
+        /* If we reached the max memcap, we get a used host */
+        if ((SC_ATOMIC_GET(host_memuse) + sizeof(Host)) > host_config.memcap) {
+            /* declare state of emergency */
+            //if (!(SC_ATOMIC_GET(host_flags) & HOST_EMERGENCY)) {
+            //    SC_ATOMIC_OR(host_flags, HOST_EMERGENCY);
+
+                /* under high load, waking up the flow mgr each time leads
+                 * to high cpu usage. Flows are not timed out much faster if
+                 * we check a 1000 times a second. */
+            //    FlowWakeupFlowManagerThread();
+            //}
+
+            h = HostGetUsedHost();
+            if (h == NULL) {
+                return NULL;
+            }
+
+            /* freed a host, but it's unlocked */
+        } else {
+            /* now see if we can alloc a new host */
+            h = HostNew(a);
+            if (h == NULL) {
+                return NULL;
+            }
+
+            /* host is initialized but *unlocked* */
+        }
+    } else {
+        /* host has been recycled before it went into the spare queue */
+
+        /* host is initialized (recylced) but *unlocked* */
+    }
+
+    SC_ATOMIC_ADD(host_counter, 1);
+    SCMutexLock(&h->m);
+    return h;
+}
+
+#define HostIncrUsecnt(h) \
+    (h)->use_cnt++
+#define HostDecrUsecnt(h) \
+    (h)->use_cnt--
+
+void HostInit(Host *h, Address *a) {
+//    SCMutexLock(&h->m);
+    COPY_ADDRESS(a, &h->a);
+    HostIncrUsecnt(h);
+}
+
+void HostRelease(Host *h) {
+    HostDecrUsecnt(h);
+    SCMutexUnlock(&h->m);
+}
+
+/* HostGetHostFromHash
+ *
+ * Hash retrieval function for hosts. Looks up the hash bucket containing the
+ * host pointer. Then compares the packet with the found host to see if it is
+ * the host we need. If it isn't, walk the list until the right host is found.
+ *
+ * returns a *LOCKED* host or NULL
+ */
+Host *HostGetHostFromHash (Address *a)
+{
+    Host *h = NULL;
+
+    /* get the key to our bucket */
+    uint32_t key = HostGetKey(a);
+    /* get our hash bucket and lock it */
+    HostHashRow *hb = &host_hash[key];
+    HRLOCK_LOCK(hb);
+
+    /* see if the bucket already has a host */
+    if (hb->head == NULL) {
+        h = HostGetNew(a);
+        if (h == NULL) {
+            HRLOCK_UNLOCK(hb);
+            return NULL;
+        }
+
+        /* host is locked */
+        hb->head = h;
+        hb->tail = h;
+
+        /* got one, now lock, initialize and return */
+        HostInit(h,a);
+
+        HRLOCK_UNLOCK(hb);
+        return h;
+    }
+
+    /* ok, we have a host in the bucket. Let's find out if it is our host */
+    h = hb->head;
+
+    /* see if this is the host we are looking for */
+    if (HostCompare(h, a) == 0) {
+        Host *ph = NULL; /* previous host */
+
+        while (h) {
+            ph = h;
+            h = h->hnext;
+
+            if (h == NULL) {
+                h = ph->hnext = HostGetNew(a);
+                if (h == NULL) {
+                    HRLOCK_UNLOCK(hb);
+                    return NULL;
+                }
+                hb->tail = h;
+
+                /* host is locked */
+
+                h->hprev = ph;
+
+                /* initialize and return */
+                HostInit(h,a);
+
+                HRLOCK_UNLOCK(hb);
+                return h;
+            }
+
+            if (HostCompare(h, a) != 0) {
+                /* we found our host, lets put it on top of the
+                 * hash list -- this rewards active hosts */
+                if (h->hnext) {
+                    h->hnext->hprev = h->hprev;
+                }
+                if (h->hprev) {
+                    h->hprev->hnext = h->hnext;
+                }
+                if (h == hb->tail) {
+                    hb->tail = h->hprev;
+                }
+
+                h->hnext = hb->head;
+                h->hprev = NULL;
+                hb->head->hprev = h;
+                hb->head = h;
+
+                /* found our host, lock & return */
+                SCMutexLock(&h->m);
+                HostIncrUsecnt(h);
+                HRLOCK_UNLOCK(hb);
+                return h;
+            }
+        }
+    }
+
+    /* lock & return */
+    SCMutexLock(&h->m);
+    HostIncrUsecnt(h);
+    HRLOCK_UNLOCK(hb);
+    return h;
+}
+
+/** \brief look up a host in the hash
+ *
+ *  \param a address to look up
+ *
+ *  \retval h *LOCKED* host or NULL
+ */
+Host *HostLookupHostFromHash (Address *a)
+{
+    Host *h = NULL;
+
+    /* get the key to our bucket */
+    uint32_t key = HostGetKey(a);
+    /* get our hash bucket and lock it */
+    HostHashRow *hb = &host_hash[key];
+    HRLOCK_LOCK(hb);
+
+    /* see if the bucket already has a host */
+    if (hb->head == NULL) {
+        HRLOCK_UNLOCK(hb);
+        return h;
+    }
+
+    /* ok, we have a host in the bucket. Let's find out if it is our host */
+    h = hb->head;
+
+    /* see if this is the host we are looking for */
+    if (HostCompare(h, a) == 0) {
+        while (h) {
+            h = h->hnext;
+
+            if (h == NULL) {
+                HRLOCK_UNLOCK(hb);
+                return h;
+            }
+
+            if (HostCompare(h, a) != 0) {
+                /* we found our host, lets put it on top of the
+                 * hash list -- this rewards active hosts */
+                if (h->hnext) {
+                    h->hnext->hprev = h->hprev;
+                }
+                if (h->hprev) {
+                    h->hprev->hnext = h->hnext;
+                }
+                if (h == hb->tail) {
+                    hb->tail = h->hprev;
+                }
+
+                h->hnext = hb->head;
+                h->hprev = NULL;
+                hb->head->hprev = h;
+                hb->head = h;
+
+                /* found our host, lock & return */
+                SCMutexLock(&h->m);
+                HostIncrUsecnt(h);
+                HRLOCK_UNLOCK(hb);
+                return h;
+            }
+        }
+    }
+
+    /* lock & return */
+    SCMutexLock(&h->m);
+    HostIncrUsecnt(h);
+    HRLOCK_UNLOCK(hb);
+    return h;
+}
+
+/** \internal
+ *  \brief Get a host from the hash directly.
+ *
+ *  Called in conditions where the spare queue is empty and memcap is reached.
+ *
+ *  Walks the hash until a host can be freed. "host_prune_idx" atomic int makes
+ *  sure we don't start at the top each time since that would clear the top of
+ *  the hash leading to longer and longer search times under high pressure (observed).
+ *
+ *  \retval h host or NULL
+ */
+static Host *HostGetUsedHost(void) {
+    uint32_t idx = SC_ATOMIC_GET(host_prune_idx) % host_config.hash_size;
+    uint32_t cnt = host_config.hash_size;
+
+    while (cnt--) {
+        if (idx++ >= host_config.hash_size)
+            idx = 0;
+
+        HostHashRow *hb = &host_hash[idx];
+        if (hb == NULL)
+            continue;
+
+        if (HRLOCK_TRYLOCK(hb) != 0)
+            continue;
+
+        Host *h = hb->tail;
+        if (h == NULL) {
+            HRLOCK_UNLOCK(hb);
+            continue;
+        }
+
+        if (SCMutexTrylock(&h->m) != 0) {
+            HRLOCK_UNLOCK(hb);
+            continue;
+        }
+
+        /** never prune a host that is used by a packets
+         *  we are currently processing in one of the threads */
+        if (h->use_cnt > 0) {
+            HRLOCK_UNLOCK(hb);
+            SCMutexUnlock(&h->m);
+            continue;
+        }
+
+        /* remove from the hash */
+        if (h->hprev != NULL)
+            h->hprev->hnext = h->hnext;
+        if (h->hnext != NULL)
+            h->hnext->hprev = h->hprev;
+        if (hb->head == h)
+            hb->head = h->hnext;
+        if (hb->tail == h)
+            hb->tail = h->hprev;
+
+        h->hnext = NULL;
+        h->hprev = NULL;
+        HRLOCK_UNLOCK(hb);
+
+        HostClearMemory (h);
+
+        SCMutexUnlock(&h->m);
+
+        SC_ATOMIC_ADD(host_prune_idx, (host_config.hash_size - cnt));
+        return h;
+    }
+
+    return NULL;
+}
+
+
diff --git a/src/host.h b/src/host.h
index 7ca1f11cd7..a4603e5b2d 100644
--- a/src/host.h
+++ b/src/host.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2010 Open Information Security Foundation
+/* Copyright (C) 2007-2012 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -25,35 +25,91 @@
 #define __HOST_H__
 
 #include "decode.h"
-#include "util-hash.h"
-#include "util-bloomfilter-counting.h"
 
-typedef struct HostTable_ {
-    SCMutex m;
+/** Spinlocks or Mutex for the flow buckets. */
+//#define HRLOCK_SPIN
+#define HRLOCK_MUTEX
 
-    /* storage & lookup */
-    HashTable *hash;
-    BloomFilterCounting *bf;
+#ifdef HRLOCK_SPIN
+    #ifdef HRLOCK_MUTEX
+        #error Cannot enable both HRLOCK_SPIN and HRLOCK_MUTEX
+    #endif
+#endif
 
-    uint32_t cnt;
-} HostTable;
+#ifdef HRLOCK_SPIN
+    #define HRLOCK_TYPE SCSpinlock
+    #define HRLOCK_INIT(fb) SCSpinInit(&(fb)->lock, 0)
+    #define HRLOCK_DESTROY(fb) SCSpinDestroy(&(fb)->lock)
+    #define HRLOCK_LOCK(fb) SCSpinLock(&(fb)->lock)
+    #define HRLOCK_TRYLOCK(fb) SCSpinTrylock(&(fb)->lock)
+    #define HRLOCK_UNLOCK(fb) SCSpinUnlock(&(fb)->lock)
+#elif defined HRLOCK_MUTEX
+    #define HRLOCK_TYPE SCMutex
+    #define HRLOCK_INIT(fb) SCMutexInit(&(fb)->lock, NULL)
+    #define HRLOCK_DESTROY(fb) SCMutexDestroy(&(fb)->lock)
+    #define HRLOCK_LOCK(fb) SCMutexLock(&(fb)->lock)
+    #define HRLOCK_TRYLOCK(fb) SCMutexTrylock(&(fb)->lock)
+    #define HRLOCK_UNLOCK(fb) SCMutexUnlock(&(fb)->lock)
+#else
+    #error Enable HRLOCK_SPIN or HRLOCK_MUTEX
+#endif
 
 typedef struct Host_ {
+    /** host mutex */
     SCMutex m;
 
-    Address addr;
-    uint8_t os;
-    uint8_t reputation;
+    /** host address -- ipv4 or ipv6 */
+    Address a;
+
+    /** use cnt, reference counter */
+    uint16_t use_cnt;
+
+    /** pointers to tag and threshold storage */
+    void *tag;
+    void *threshold;
 
-    uint64_t bytes;
-    uint32_t pkts;
+    /** hash pointers, protected by hash row mutex/spin */
+    struct Host_ *hnext;
+    struct Host_ *hprev;
+
+    /** list pointers, protected by host-queue mutex/spin */
+    struct Host_ *lnext;
+    struct Host_ *lprev;
 } Host;
 
-#define HOST_OS_UNKNOWN 0
-/* XXX define more */
+typedef struct HostHashRow_ {
+    HRLOCK_TYPE lock;
+    Host *head;
+    Host *tail;
+} HostHashRow;
+
+/** host hash table */
+HostHashRow *host_hash;
+
+#define HOST_VERBOSE    0
+#define HOST_QUIET      1
+
+typedef struct HostConfig_ {
+    uint64_t memcap;
+    uint32_t hash_rand;
+    uint32_t hash_size;
+    uint32_t prealloc;
+} HostConfig;
+
+HostConfig host_config;
+SC_ATOMIC_DECLARE(unsigned long long int,host_memuse);
+SC_ATOMIC_DECLARE(unsigned int,host_counter);
+SC_ATOMIC_DECLARE(unsigned int,host_prune_idx);
+
+void HostInitConfig(char quiet);
+void HostShutdown(void);
 
-#define HOST_REPU_UNKNOWN 0
-/* XXX see how we deal with this */
+Host *HostLookupHostFromHash (Address *);
+Host *HostGetHostFromHash (Address *);
+void HostRelease(Host *);
+void HostClearMemory(Host *);
+void HostMoveToSpare(Host *);
+uint32_t HostSpareQueueGetSize(void);
 
 #endif /* __HOST_H__ */
 
diff --git a/src/suricata.c b/src/suricata.c
index 7e3c934c17..b1db347886 100644
--- a/src/suricata.c
+++ b/src/suricata.c
@@ -127,6 +127,8 @@
 #include "flow-alert-sid.h"
 #include "pkt-var.h"
 
+#include "host.h"
+
 #include "app-layer-detect-proto.h"
 #include "app-layer-parser.h"
 #include "app-layer-smb.h"
@@ -1578,6 +1580,7 @@ int main(int argc, char **argv)
     SCLogInfo("preallocated %"PRIiMAX" packets. Total memory %"PRIuMAX"",
         max_pending_packets, (uintmax_t)(max_pending_packets*SIZE_OF_PACKET));
 
+    HostInitConfig(HOST_VERBOSE);
     FlowInitConfig(FLOW_VERBOSE);
 
     DetectEngineCtx *de_ctx = DetectEngineCtxInit();
@@ -1794,6 +1797,7 @@ int main(int argc, char **argv)
     TmThreadKillThreads();
     SCPerfReleaseResources();
     FlowShutdown();
+    HostShutdown();
     StreamTcpFreeConfig(STREAM_VERBOSE);
     HTPFreeConfig();
     HTPAtExitPrintStats();