From: Willy Tarreau Date: Mon, 26 Oct 2009 18:48:54 +0000 (+0100) Subject: [IMPORT] import ebtree v5.0 into directory ebtree/ X-Git-Tag: v1.4-dev5~86 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=c218602b1dd0182ede86d7a660562714e3907660;p=thirdparty%2Fhaproxy.git [IMPORT] import ebtree v5.0 into directory ebtree/ We needed to upgrade ebtree to v5.0 to support string indexing, and it was getting very painful to have it split across 2 dirs and to have to patch it. Now we just have to copy the .c and .h files to the right place. --- diff --git a/ebtree/eb32tree.c b/ebtree/eb32tree.c new file mode 100644 index 0000000000..84a47e871c --- /dev/null +++ b/ebtree/eb32tree.c @@ -0,0 +1,217 @@ +/* + * Elastic Binary Trees - exported functions for operations on 32bit nodes. + * (C) 2002-2009 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* Consult eb32tree.h for more details about those functions */ + +#include "eb32tree.h" + +REGPRM2 struct eb32_node *eb32_insert(struct eb_root *root, struct eb32_node *new) +{ + return __eb32_insert(root, new); +} + +REGPRM2 struct eb32_node *eb32i_insert(struct eb_root *root, struct eb32_node *new) +{ + return __eb32i_insert(root, new); +} + +REGPRM2 struct eb32_node *eb32_lookup(struct eb_root *root, u32 x) +{ + return __eb32_lookup(root, x); +} + +REGPRM2 struct eb32_node *eb32i_lookup(struct eb_root *root, s32 x) +{ + return __eb32i_lookup(root, x); +} + +/* + * Find the last occurrence of the highest key in the tree , which is + * equal to or less than . NULL is returned is no key matches. + */ +REGPRM2 struct eb32_node *eb32_lookup_le(struct eb_root *root, u32 x) +{ + struct eb32_node *node; + eb_troot_t *troot; + + troot = root->b[EB_LEFT]; + if (unlikely(troot == NULL)) + return NULL; + + while (1) { + if ((eb_gettag(troot) == EB_LEAF)) { + /* We reached a leaf, which means that the whole upper + * parts were common. We will return either the current + * node or its next one if the former is too small. + */ + node = container_of(eb_untag(troot, EB_LEAF), + struct eb32_node, node.branches); + if (node->key <= x) + return node; + /* return prev */ + troot = node->node.leaf_p; + break; + } + node = container_of(eb_untag(troot, EB_NODE), + struct eb32_node, node.branches); + + if (node->node.bit < 0) { + /* We're at the top of a dup tree. Either we got a + * matching value and we return the rightmost node, or + * we don't and we skip the whole subtree to return the + * prev node before the subtree. Note that since we're + * at the top of the dup tree, we can simply return the + * prev node without first trying to escape from the + * tree. + */ + if (node->key <= x) { + troot = node->node.branches.b[EB_RGHT]; + while (eb_gettag(troot) != EB_LEAF) + troot = (eb_untag(troot, EB_NODE))->b[EB_RGHT]; + return container_of(eb_untag(troot, EB_LEAF), + struct eb32_node, node.branches); + } + /* return prev */ + troot = node->node.node_p; + break; + } + + if (((x ^ node->key) >> node->node.bit) >= EB_NODE_BRANCHES) { + /* No more common bits at all. Either this node is too + * small and we need to get its highest value, or it is + * too large, and we need to get the prev value. + */ + if ((node->key >> node->node.bit) > (x >> node->node.bit)) { + troot = node->node.branches.b[EB_RGHT]; + return eb32_entry(eb_walk_down(troot, EB_RGHT), struct eb32_node, node); + } + + /* Further values will be too high here, so return the prev + * unique node (if it exists). + */ + troot = node->node.node_p; + break; + } + troot = node->node.branches.b[(x >> node->node.bit) & EB_NODE_BRANCH_MASK]; + } + + /* If we get here, it means we want to report previous node before the + * current one which is not above. is already initialised to + * the parent's branches. + */ + while (eb_gettag(troot) == EB_LEFT) { + /* Walking up from left branch. We must ensure that we never + * walk beyond root. + */ + if (unlikely(eb_clrtag((eb_untag(troot, EB_LEFT))->b[EB_RGHT]) == NULL)) + return NULL; + troot = (eb_root_to_node(eb_untag(troot, EB_LEFT)))->node_p; + } + /* Note that cannot be NULL at this stage */ + troot = (eb_untag(troot, EB_RGHT))->b[EB_LEFT]; + node = eb32_entry(eb_walk_down(troot, EB_RGHT), struct eb32_node, node); + return node; +} + +/* + * Find the first occurrence of the lowest key in the tree , which is + * equal to or greater than . NULL is returned is no key matches. + */ +REGPRM2 struct eb32_node *eb32_lookup_ge(struct eb_root *root, u32 x) +{ + struct eb32_node *node; + eb_troot_t *troot; + + troot = root->b[EB_LEFT]; + if (unlikely(troot == NULL)) + return NULL; + + while (1) { + if ((eb_gettag(troot) == EB_LEAF)) { + /* We reached a leaf, which means that the whole upper + * parts were common. We will return either the current + * node or its next one if the former is too small. + */ + node = container_of(eb_untag(troot, EB_LEAF), + struct eb32_node, node.branches); + if (node->key >= x) + return node; + /* return next */ + troot = node->node.leaf_p; + break; + } + node = container_of(eb_untag(troot, EB_NODE), + struct eb32_node, node.branches); + + if (node->node.bit < 0) { + /* We're at the top of a dup tree. Either we got a + * matching value and we return the leftmost node, or + * we don't and we skip the whole subtree to return the + * next node after the subtree. Note that since we're + * at the top of the dup tree, we can simply return the + * next node without first trying to escape from the + * tree. + */ + if (node->key >= x) { + troot = node->node.branches.b[EB_LEFT]; + while (eb_gettag(troot) != EB_LEAF) + troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT]; + return container_of(eb_untag(troot, EB_LEAF), + struct eb32_node, node.branches); + } + /* return next */ + troot = node->node.node_p; + break; + } + + if (((x ^ node->key) >> node->node.bit) >= EB_NODE_BRANCHES) { + /* No more common bits at all. Either this node is too + * large and we need to get its lowest value, or it is too + * small, and we need to get the next value. + */ + if ((node->key >> node->node.bit) > (x >> node->node.bit)) { + troot = node->node.branches.b[EB_LEFT]; + return eb32_entry(eb_walk_down(troot, EB_LEFT), struct eb32_node, node); + } + + /* Further values will be too low here, so return the next + * unique node (if it exists). + */ + troot = node->node.node_p; + break; + } + troot = node->node.branches.b[(x >> node->node.bit) & EB_NODE_BRANCH_MASK]; + } + + /* If we get here, it means we want to report next node after the + * current one which is not below. is already initialised + * to the parent's branches. + */ + while (eb_gettag(troot) != EB_LEFT) + /* Walking up from right branch, so we cannot be below root */ + troot = (eb_root_to_node(eb_untag(troot, EB_RGHT)))->node_p; + + /* Note that cannot be NULL at this stage */ + troot = (eb_untag(troot, EB_LEFT))->b[EB_RGHT]; + if (eb_clrtag(troot) == NULL) + return NULL; + + node = eb32_entry(eb_walk_down(troot, EB_LEFT), struct eb32_node, node); + return node; +} diff --git a/ebtree/eb32tree.h b/ebtree/eb32tree.h new file mode 100644 index 0000000000..037c458e58 --- /dev/null +++ b/ebtree/eb32tree.h @@ -0,0 +1,548 @@ +/* + * Elastic Binary Trees - macros and structures for operations on 32bit nodes. + * Version 5.0 + * (C) 2002-2009 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef _EB32TREE_H +#define _EB32TREE_H + +#include "ebtree.h" + + +/* Return the structure of type whose member points to */ +#define eb32_entry(ptr, type, member) container_of(ptr, type, member) + +#define EB32_ROOT EB_ROOT +#define EB32_TREE_HEAD EB_TREE_HEAD + +/* These types may sometimes already be defined */ +typedef unsigned int u32; +typedef signed int s32; + +/* This structure carries a node, a leaf, and a key. It must start with the + * eb_node so that it can be cast into an eb_node. We could also have put some + * sort of transparent union here to reduce the indirection level, but the fact + * is, the end user is not meant to manipulate internals, so this is pointless. + */ +struct eb32_node { + struct eb_node node; /* the tree node, must be at the beginning */ + u32 key; +}; + +/* + * Exported functions and macros. + * Many of them are always inlined because they are extremely small, and + * are generally called at most once or twice in a program. + */ + +/* Return leftmost node in the tree, or NULL if none */ +static inline struct eb32_node *eb32_first(struct eb_root *root) +{ + return eb32_entry(eb_first(root), struct eb32_node, node); +} + +/* Return rightmost node in the tree, or NULL if none */ +static inline struct eb32_node *eb32_last(struct eb_root *root) +{ + return eb32_entry(eb_last(root), struct eb32_node, node); +} + +/* Return next node in the tree, or NULL if none */ +static inline struct eb32_node *eb32_next(struct eb32_node *eb32) +{ + return eb32_entry(eb_next(&eb32->node), struct eb32_node, node); +} + +/* Return previous node in the tree, or NULL if none */ +static inline struct eb32_node *eb32_prev(struct eb32_node *eb32) +{ + return eb32_entry(eb_prev(&eb32->node), struct eb32_node, node); +} + +/* Return next node in the tree, skipping duplicates, or NULL if none */ +static inline struct eb32_node *eb32_next_unique(struct eb32_node *eb32) +{ + return eb32_entry(eb_next_unique(&eb32->node), struct eb32_node, node); +} + +/* Return previous node in the tree, skipping duplicates, or NULL if none */ +static inline struct eb32_node *eb32_prev_unique(struct eb32_node *eb32) +{ + return eb32_entry(eb_prev_unique(&eb32->node), struct eb32_node, node); +} + +/* Delete node from the tree if it was linked in. Mark the node unused. Note + * that this function relies on a non-inlined generic function: eb_delete. + */ +static inline void eb32_delete(struct eb32_node *eb32) +{ + eb_delete(&eb32->node); +} + +/* + * The following functions are not inlined by default. They are declared + * in eb32tree.c, which simply relies on their inline version. + */ +REGPRM2 struct eb32_node *eb32_lookup(struct eb_root *root, u32 x); +REGPRM2 struct eb32_node *eb32i_lookup(struct eb_root *root, s32 x); +REGPRM2 struct eb32_node *eb32_lookup_le(struct eb_root *root, u32 x); +REGPRM2 struct eb32_node *eb32_lookup_ge(struct eb_root *root, u32 x); +REGPRM2 struct eb32_node *eb32_insert(struct eb_root *root, struct eb32_node *new); +REGPRM2 struct eb32_node *eb32i_insert(struct eb_root *root, struct eb32_node *new); + +/* + * The following functions are less likely to be used directly, because their + * code is larger. The non-inlined version is preferred. + */ + +/* Delete node from the tree if it was linked in. Mark the node unused. */ +static forceinline void __eb32_delete(struct eb32_node *eb32) +{ + __eb_delete(&eb32->node); +} + +/* + * Find the first occurence of a key in the tree . If none can be + * found, return NULL. + */ +static forceinline struct eb32_node *__eb32_lookup(struct eb_root *root, u32 x) +{ + struct eb32_node *node; + eb_troot_t *troot; + u32 y; + + troot = root->b[EB_LEFT]; + if (unlikely(troot == NULL)) + return NULL; + + while (1) { + if ((eb_gettag(troot) == EB_LEAF)) { + node = container_of(eb_untag(troot, EB_LEAF), + struct eb32_node, node.branches); + if (node->key == x) + return node; + else + return NULL; + } + node = container_of(eb_untag(troot, EB_NODE), + struct eb32_node, node.branches); + + y = node->key ^ x; + if (!y) { + /* Either we found the node which holds the key, or + * we have a dup tree. In the later case, we have to + * walk it down left to get the first entry. + */ + if (node->node.bit < 0) { + troot = node->node.branches.b[EB_LEFT]; + while (eb_gettag(troot) != EB_LEAF) + troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT]; + node = container_of(eb_untag(troot, EB_LEAF), + struct eb32_node, node.branches); + } + return node; + } + + if ((y >> node->node.bit) >= EB_NODE_BRANCHES) + return NULL; /* no more common bits */ + + troot = node->node.branches.b[(x >> node->node.bit) & EB_NODE_BRANCH_MASK]; + } +} + +/* + * Find the first occurence of a signed key in the tree . If none can + * be found, return NULL. + */ +static forceinline struct eb32_node *__eb32i_lookup(struct eb_root *root, s32 x) +{ + struct eb32_node *node; + eb_troot_t *troot; + u32 key = x ^ 0x80000000; + u32 y; + + troot = root->b[EB_LEFT]; + if (unlikely(troot == NULL)) + return NULL; + + while (1) { + if ((eb_gettag(troot) == EB_LEAF)) { + node = container_of(eb_untag(troot, EB_LEAF), + struct eb32_node, node.branches); + if (node->key == x) + return node; + else + return NULL; + } + node = container_of(eb_untag(troot, EB_NODE), + struct eb32_node, node.branches); + + y = node->key ^ x; + if (!y) { + /* Either we found the node which holds the key, or + * we have a dup tree. In the later case, we have to + * walk it down left to get the first entry. + */ + if (node->node.bit < 0) { + troot = node->node.branches.b[EB_LEFT]; + while (eb_gettag(troot) != EB_LEAF) + troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT]; + node = container_of(eb_untag(troot, EB_LEAF), + struct eb32_node, node.branches); + } + return node; + } + + if ((y >> node->node.bit) >= EB_NODE_BRANCHES) + return NULL; /* no more common bits */ + + troot = node->node.branches.b[(key >> node->node.bit) & EB_NODE_BRANCH_MASK]; + } +} + +/* Insert eb32_node into subtree starting at node root . + * Only new->key needs be set with the key. The eb32_node is returned. + * If root->b[EB_RGHT]==1, the tree may only contain unique keys. + */ +static forceinline struct eb32_node * +__eb32_insert(struct eb_root *root, struct eb32_node *new) { + struct eb32_node *old; + unsigned int side; + eb_troot_t *troot; + u32 newkey; /* caching the key saves approximately one cycle */ + eb_troot_t *root_right = root; + + side = EB_LEFT; + troot = root->b[EB_LEFT]; + root_right = root->b[EB_RGHT]; + if (unlikely(troot == NULL)) { + /* Tree is empty, insert the leaf part below the left branch */ + root->b[EB_LEFT] = eb_dotag(&new->node.branches, EB_LEAF); + new->node.leaf_p = eb_dotag(root, EB_LEFT); + new->node.node_p = NULL; /* node part unused */ + return new; + } + + /* The tree descent is fairly easy : + * - first, check if we have reached a leaf node + * - second, check if we have gone too far + * - third, reiterate + * Everywhere, we use for the node node we are inserting, + * for the node we attach it to, and for the node we are + * displacing below . will always point to the future node + * (tagged with its type). carries the side the node is + * attached to below its parent, which is also where previous node + * was attached. carries the key being inserted. + */ + newkey = new->key; + + while (1) { + if (unlikely(eb_gettag(troot) == EB_LEAF)) { + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_leaf; + + old = container_of(eb_untag(troot, EB_LEAF), + struct eb32_node, node.branches); + + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_leaf = eb_dotag(&old->node.branches, EB_LEAF); + + new->node.node_p = old->node.leaf_p; + + /* Right here, we have 3 possibilities : + - the tree does not contain the key, and we have + new->key < old->key. We insert new above old, on + the left ; + + - the tree does not contain the key, and we have + new->key > old->key. We insert new above old, on + the right ; + + - the tree does contain the key, which implies it + is alone. We add the new key next to it as a + first duplicate. + + The last two cases can easily be partially merged. + */ + + if (new->key < old->key) { + new->node.leaf_p = new_left; + old->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_leaf; + } else { + /* we may refuse to duplicate this key if the tree is + * tagged as containing only unique keys. + */ + if ((new->key == old->key) && eb_gettag(root_right)) + return old; + + /* new->key >= old->key, new goes the right */ + old->node.leaf_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_leaf; + new->node.branches.b[EB_RGHT] = new_leaf; + + if (new->key == old->key) { + new->node.bit = -1; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + return new; + } + } + break; + } + + /* OK we're walking down this link */ + old = container_of(eb_untag(troot, EB_NODE), + struct eb32_node, node.branches); + + /* Stop going down when we don't have common bits anymore. We + * also stop in front of a duplicates tree because it means we + * have to insert above. + */ + + if ((old->node.bit < 0) || /* we're above a duplicate tree, stop here */ + (((new->key ^ old->key) >> old->node.bit) >= EB_NODE_BRANCHES)) { + /* The tree did not contain the key, so we insert before the node + * , and set ->bit to designate the lowest bit position in + * which applies to ->branches.b[]. + */ + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_node; + + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_node = eb_dotag(&old->node.branches, EB_NODE); + + new->node.node_p = old->node.node_p; + + if (new->key < old->key) { + new->node.leaf_p = new_left; + old->node.node_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_node; + } + else if (new->key > old->key) { + old->node.node_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_node; + new->node.branches.b[EB_RGHT] = new_leaf; + } + else { + struct eb_node *ret; + ret = eb_insert_dup(&old->node, &new->node); + return container_of(ret, struct eb32_node, node); + } + break; + } + + /* walk down */ + root = &old->node.branches; + side = (newkey >> old->node.bit) & EB_NODE_BRANCH_MASK; + troot = root->b[side]; + } + + /* Ok, now we are inserting between and . 's + * parent is already set to , and the 's branch is still in + * . Update the root's leaf till we have it. Note that we can also + * find the side by checking the side of new->node.node_p. + */ + + /* We need the common higher bits between new->key and old->key. + * What differences are there between new->key and the node here ? + * NOTE that bit(new) is always < bit(root) because highest + * bit of new->key and old->key are identical here (otherwise they + * would sit on different branches). + */ + // note that if EB_NODE_BITS > 1, we should check that it's still >= 0 + new->node.bit = flsnz(new->key ^ old->key) - EB_NODE_BITS; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + + return new; +} + +/* Insert eb32_node into subtree starting at node root , using + * signed keys. Only new->key needs be set with the key. The eb32_node + * is returned. If root->b[EB_RGHT]==1, the tree may only contain unique keys. + */ +static forceinline struct eb32_node * +__eb32i_insert(struct eb_root *root, struct eb32_node *new) { + struct eb32_node *old; + unsigned int side; + eb_troot_t *troot; + int newkey; /* caching the key saves approximately one cycle */ + eb_troot_t *root_right = root; + + side = EB_LEFT; + troot = root->b[EB_LEFT]; + root_right = root->b[EB_RGHT]; + if (unlikely(troot == NULL)) { + /* Tree is empty, insert the leaf part below the left branch */ + root->b[EB_LEFT] = eb_dotag(&new->node.branches, EB_LEAF); + new->node.leaf_p = eb_dotag(root, EB_LEFT); + new->node.node_p = NULL; /* node part unused */ + return new; + } + + /* The tree descent is fairly easy : + * - first, check if we have reached a leaf node + * - second, check if we have gone too far + * - third, reiterate + * Everywhere, we use for the node node we are inserting, + * for the node we attach it to, and for the node we are + * displacing below . will always point to the future node + * (tagged with its type). carries the side the node is + * attached to below its parent, which is also where previous node + * was attached. carries a high bit shift of the key being + * inserted in order to have negative keys stored before positive + * ones. + */ + newkey = new->key + 0x80000000; + + while (1) { + if (unlikely(eb_gettag(troot) == EB_LEAF)) { + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_leaf; + + old = container_of(eb_untag(troot, EB_LEAF), + struct eb32_node, node.branches); + + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_leaf = eb_dotag(&old->node.branches, EB_LEAF); + + new->node.node_p = old->node.leaf_p; + + /* Right here, we have 3 possibilities : + - the tree does not contain the key, and we have + new->key < old->key. We insert new above old, on + the left ; + + - the tree does not contain the key, and we have + new->key > old->key. We insert new above old, on + the right ; + + - the tree does contain the key, which implies it + is alone. We add the new key next to it as a + first duplicate. + + The last two cases can easily be partially merged. + */ + + if ((s32)new->key < (s32)old->key) { + new->node.leaf_p = new_left; + old->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_leaf; + } else { + /* we may refuse to duplicate this key if the tree is + * tagged as containing only unique keys. + */ + if ((new->key == old->key) && eb_gettag(root_right)) + return old; + + /* new->key >= old->key, new goes the right */ + old->node.leaf_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_leaf; + new->node.branches.b[EB_RGHT] = new_leaf; + + if (new->key == old->key) { + new->node.bit = -1; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + return new; + } + } + break; + } + + /* OK we're walking down this link */ + old = container_of(eb_untag(troot, EB_NODE), + struct eb32_node, node.branches); + + /* Stop going down when we don't have common bits anymore. We + * also stop in front of a duplicates tree because it means we + * have to insert above. + */ + + if ((old->node.bit < 0) || /* we're above a duplicate tree, stop here */ + (((new->key ^ old->key) >> old->node.bit) >= EB_NODE_BRANCHES)) { + /* The tree did not contain the key, so we insert before the node + * , and set ->bit to designate the lowest bit position in + * which applies to ->branches.b[]. + */ + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_node; + + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_node = eb_dotag(&old->node.branches, EB_NODE); + + new->node.node_p = old->node.node_p; + + if ((s32)new->key < (s32)old->key) { + new->node.leaf_p = new_left; + old->node.node_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_node; + } + else if ((s32)new->key > (s32)old->key) { + old->node.node_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_node; + new->node.branches.b[EB_RGHT] = new_leaf; + } + else { + struct eb_node *ret; + ret = eb_insert_dup(&old->node, &new->node); + return container_of(ret, struct eb32_node, node); + } + break; + } + + /* walk down */ + root = &old->node.branches; + side = (newkey >> old->node.bit) & EB_NODE_BRANCH_MASK; + troot = root->b[side]; + } + + /* Ok, now we are inserting between and . 's + * parent is already set to , and the 's branch is still in + * . Update the root's leaf till we have it. Note that we can also + * find the side by checking the side of new->node.node_p. + */ + + /* We need the common higher bits between new->key and old->key. + * What differences are there between new->key and the node here ? + * NOTE that bit(new) is always < bit(root) because highest + * bit of new->key and old->key are identical here (otherwise they + * would sit on different branches). + */ + // note that if EB_NODE_BITS > 1, we should check that it's still >= 0 + new->node.bit = flsnz(new->key ^ old->key) - EB_NODE_BITS; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + + return new; +} + +#endif /* _EB32_TREE_H */ diff --git a/ebtree/eb64tree.c b/ebtree/eb64tree.c new file mode 100644 index 0000000000..1abb4c8b86 --- /dev/null +++ b/ebtree/eb64tree.c @@ -0,0 +1,217 @@ +/* + * Elastic Binary Trees - exported functions for operations on 64bit nodes. + * (C) 2002-2007 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* Consult eb64tree.h for more details about those functions */ + +#include "eb64tree.h" + +REGPRM2 struct eb64_node *eb64_insert(struct eb_root *root, struct eb64_node *new) +{ + return __eb64_insert(root, new); +} + +REGPRM2 struct eb64_node *eb64i_insert(struct eb_root *root, struct eb64_node *new) +{ + return __eb64i_insert(root, new); +} + +REGPRM2 struct eb64_node *eb64_lookup(struct eb_root *root, u64 x) +{ + return __eb64_lookup(root, x); +} + +REGPRM2 struct eb64_node *eb64i_lookup(struct eb_root *root, s64 x) +{ + return __eb64i_lookup(root, x); +} + +/* + * Find the last occurrence of the highest key in the tree , which is + * equal to or less than . NULL is returned is no key matches. + */ +REGPRM2 struct eb64_node *eb64_lookup_le(struct eb_root *root, u64 x) +{ + struct eb64_node *node; + eb_troot_t *troot; + + troot = root->b[EB_LEFT]; + if (unlikely(troot == NULL)) + return NULL; + + while (1) { + if ((eb_gettag(troot) == EB_LEAF)) { + /* We reached a leaf, which means that the whole upper + * parts were common. We will return either the current + * node or its next one if the former is too small. + */ + node = container_of(eb_untag(troot, EB_LEAF), + struct eb64_node, node.branches); + if (node->key <= x) + return node; + /* return prev */ + troot = node->node.leaf_p; + break; + } + node = container_of(eb_untag(troot, EB_NODE), + struct eb64_node, node.branches); + + if (node->node.bit < 0) { + /* We're at the top of a dup tree. Either we got a + * matching value and we return the rightmost node, or + * we don't and we skip the whole subtree to return the + * prev node before the subtree. Note that since we're + * at the top of the dup tree, we can simply return the + * prev node without first trying to escape from the + * tree. + */ + if (node->key <= x) { + troot = node->node.branches.b[EB_RGHT]; + while (eb_gettag(troot) != EB_LEAF) + troot = (eb_untag(troot, EB_NODE))->b[EB_RGHT]; + return container_of(eb_untag(troot, EB_LEAF), + struct eb64_node, node.branches); + } + /* return prev */ + troot = node->node.node_p; + break; + } + + if (((x ^ node->key) >> node->node.bit) >= EB_NODE_BRANCHES) { + /* No more common bits at all. Either this node is too + * small and we need to get its highest value, or it is + * too large, and we need to get the prev value. + */ + if ((node->key >> node->node.bit) > (x >> node->node.bit)) { + troot = node->node.branches.b[EB_RGHT]; + return eb64_entry(eb_walk_down(troot, EB_RGHT), struct eb64_node, node); + } + + /* Further values will be too high here, so return the prev + * unique node (if it exists). + */ + troot = node->node.node_p; + break; + } + troot = node->node.branches.b[(x >> node->node.bit) & EB_NODE_BRANCH_MASK]; + } + + /* If we get here, it means we want to report previous node before the + * current one which is not above. is already initialised to + * the parent's branches. + */ + while (eb_gettag(troot) == EB_LEFT) { + /* Walking up from left branch. We must ensure that we never + * walk beyond root. + */ + if (unlikely(eb_clrtag((eb_untag(troot, EB_LEFT))->b[EB_RGHT]) == NULL)) + return NULL; + troot = (eb_root_to_node(eb_untag(troot, EB_LEFT)))->node_p; + } + /* Note that cannot be NULL at this stage */ + troot = (eb_untag(troot, EB_RGHT))->b[EB_LEFT]; + node = eb64_entry(eb_walk_down(troot, EB_RGHT), struct eb64_node, node); + return node; +} + +/* + * Find the first occurrence of the lowest key in the tree , which is + * equal to or greater than . NULL is returned is no key matches. + */ +REGPRM2 struct eb64_node *eb64_lookup_ge(struct eb_root *root, u64 x) +{ + struct eb64_node *node; + eb_troot_t *troot; + + troot = root->b[EB_LEFT]; + if (unlikely(troot == NULL)) + return NULL; + + while (1) { + if ((eb_gettag(troot) == EB_LEAF)) { + /* We reached a leaf, which means that the whole upper + * parts were common. We will return either the current + * node or its next one if the former is too small. + */ + node = container_of(eb_untag(troot, EB_LEAF), + struct eb64_node, node.branches); + if (node->key >= x) + return node; + /* return next */ + troot = node->node.leaf_p; + break; + } + node = container_of(eb_untag(troot, EB_NODE), + struct eb64_node, node.branches); + + if (node->node.bit < 0) { + /* We're at the top of a dup tree. Either we got a + * matching value and we return the leftmost node, or + * we don't and we skip the whole subtree to return the + * next node after the subtree. Note that since we're + * at the top of the dup tree, we can simply return the + * next node without first trying to escape from the + * tree. + */ + if (node->key >= x) { + troot = node->node.branches.b[EB_LEFT]; + while (eb_gettag(troot) != EB_LEAF) + troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT]; + return container_of(eb_untag(troot, EB_LEAF), + struct eb64_node, node.branches); + } + /* return next */ + troot = node->node.node_p; + break; + } + + if (((x ^ node->key) >> node->node.bit) >= EB_NODE_BRANCHES) { + /* No more common bits at all. Either this node is too + * large and we need to get its lowest value, or it is too + * small, and we need to get the next value. + */ + if ((node->key >> node->node.bit) > (x >> node->node.bit)) { + troot = node->node.branches.b[EB_LEFT]; + return eb64_entry(eb_walk_down(troot, EB_LEFT), struct eb64_node, node); + } + + /* Further values will be too low here, so return the next + * unique node (if it exists). + */ + troot = node->node.node_p; + break; + } + troot = node->node.branches.b[(x >> node->node.bit) & EB_NODE_BRANCH_MASK]; + } + + /* If we get here, it means we want to report next node after the + * current one which is not below. is already initialised + * to the parent's branches. + */ + while (eb_gettag(troot) != EB_LEFT) + /* Walking up from right branch, so we cannot be below root */ + troot = (eb_root_to_node(eb_untag(troot, EB_RGHT)))->node_p; + + /* Note that cannot be NULL at this stage */ + troot = (eb_untag(troot, EB_LEFT))->b[EB_RGHT]; + if (eb_clrtag(troot) == NULL) + return NULL; + + node = eb64_entry(eb_walk_down(troot, EB_LEFT), struct eb64_node, node); + return node; +} diff --git a/ebtree/eb64tree.h b/ebtree/eb64tree.h new file mode 100644 index 0000000000..6ffa58650a --- /dev/null +++ b/ebtree/eb64tree.h @@ -0,0 +1,568 @@ +/* + * Elastic Binary Trees - macros and structures for operations on 64bit nodes. + * Version 5.0 + * (C) 2002-2009 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef _EB64TREE_H +#define _EB64TREE_H + +#include "ebtree.h" + + +/* Return the structure of type whose member points to */ +#define eb64_entry(ptr, type, member) container_of(ptr, type, member) + +#define EB64_ROOT EB_ROOT +#define EB64_TREE_HEAD EB_TREE_HEAD + +/* These types may sometimes already be defined */ +typedef unsigned long long u64; +typedef signed long long s64; + +/* This structure carries a node, a leaf, and a key. It must start with the + * eb_node so that it can be cast into an eb_node. We could also have put some + * sort of transparent union here to reduce the indirection level, but the fact + * is, the end user is not meant to manipulate internals, so this is pointless. + */ +struct eb64_node { + struct eb_node node; /* the tree node, must be at the beginning */ + u64 key; +}; + +/* + * Exported functions and macros. + * Many of them are always inlined because they are extremely small, and + * are generally called at most once or twice in a program. + */ + +/* Return leftmost node in the tree, or NULL if none */ +static inline struct eb64_node *eb64_first(struct eb_root *root) +{ + return eb64_entry(eb_first(root), struct eb64_node, node); +} + +/* Return rightmost node in the tree, or NULL if none */ +static inline struct eb64_node *eb64_last(struct eb_root *root) +{ + return eb64_entry(eb_last(root), struct eb64_node, node); +} + +/* Return next node in the tree, or NULL if none */ +static inline struct eb64_node *eb64_next(struct eb64_node *eb64) +{ + return eb64_entry(eb_next(&eb64->node), struct eb64_node, node); +} + +/* Return previous node in the tree, or NULL if none */ +static inline struct eb64_node *eb64_prev(struct eb64_node *eb64) +{ + return eb64_entry(eb_prev(&eb64->node), struct eb64_node, node); +} + +/* Return next node in the tree, skipping duplicates, or NULL if none */ +static inline struct eb64_node *eb64_next_unique(struct eb64_node *eb64) +{ + return eb64_entry(eb_next_unique(&eb64->node), struct eb64_node, node); +} + +/* Return previous node in the tree, skipping duplicates, or NULL if none */ +static inline struct eb64_node *eb64_prev_unique(struct eb64_node *eb64) +{ + return eb64_entry(eb_prev_unique(&eb64->node), struct eb64_node, node); +} + +/* Delete node from the tree if it was linked in. Mark the node unused. Note + * that this function relies on a non-inlined generic function: eb_delete. + */ +static inline void eb64_delete(struct eb64_node *eb64) +{ + eb_delete(&eb64->node); +} + +/* + * The following functions are not inlined by default. They are declared + * in eb64tree.c, which simply relies on their inline version. + */ +REGPRM2 struct eb64_node *eb64_lookup(struct eb_root *root, u64 x); +REGPRM2 struct eb64_node *eb64i_lookup(struct eb_root *root, s64 x); +REGPRM2 struct eb64_node *eb64_lookup_le(struct eb_root *root, u64 x); +REGPRM2 struct eb64_node *eb64_lookup_ge(struct eb_root *root, u64 x); +REGPRM2 struct eb64_node *eb64_insert(struct eb_root *root, struct eb64_node *new); +REGPRM2 struct eb64_node *eb64i_insert(struct eb_root *root, struct eb64_node *new); + +/* + * The following functions are less likely to be used directly, because their + * code is larger. The non-inlined version is preferred. + */ + +/* Delete node from the tree if it was linked in. Mark the node unused. */ +static forceinline void __eb64_delete(struct eb64_node *eb64) +{ + __eb_delete(&eb64->node); +} + +/* + * Find the first occurence of a key in the tree . If none can be + * found, return NULL. + */ +static forceinline struct eb64_node *__eb64_lookup(struct eb_root *root, u64 x) +{ + struct eb64_node *node; + eb_troot_t *troot; + u64 y; + + troot = root->b[EB_LEFT]; + if (unlikely(troot == NULL)) + return NULL; + + while (1) { + if ((eb_gettag(troot) == EB_LEAF)) { + node = container_of(eb_untag(troot, EB_LEAF), + struct eb64_node, node.branches); + if (node->key == x) + return node; + else + return NULL; + } + node = container_of(eb_untag(troot, EB_NODE), + struct eb64_node, node.branches); + + y = node->key ^ x; + if (!y) { + /* Either we found the node which holds the key, or + * we have a dup tree. In the later case, we have to + * walk it down left to get the first entry. + */ + if (node->node.bit < 0) { + troot = node->node.branches.b[EB_LEFT]; + while (eb_gettag(troot) != EB_LEAF) + troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT]; + node = container_of(eb_untag(troot, EB_LEAF), + struct eb64_node, node.branches); + } + return node; + } + + if ((y >> node->node.bit) >= EB_NODE_BRANCHES) + return NULL; /* no more common bits */ + + troot = node->node.branches.b[(x >> node->node.bit) & EB_NODE_BRANCH_MASK]; + } +} + +/* + * Find the first occurence of a signed key in the tree . If none can + * be found, return NULL. + */ +static forceinline struct eb64_node *__eb64i_lookup(struct eb_root *root, s64 x) +{ + struct eb64_node *node; + eb_troot_t *troot; + u64 key = x ^ (1ULL << 63); + u64 y; + + troot = root->b[EB_LEFT]; + if (unlikely(troot == NULL)) + return NULL; + + while (1) { + if ((eb_gettag(troot) == EB_LEAF)) { + node = container_of(eb_untag(troot, EB_LEAF), + struct eb64_node, node.branches); + if (node->key == x) + return node; + else + return NULL; + } + node = container_of(eb_untag(troot, EB_NODE), + struct eb64_node, node.branches); + + y = node->key ^ x; + if (!y) { + /* Either we found the node which holds the key, or + * we have a dup tree. In the later case, we have to + * walk it down left to get the first entry. + */ + if (node->node.bit < 0) { + troot = node->node.branches.b[EB_LEFT]; + while (eb_gettag(troot) != EB_LEAF) + troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT]; + node = container_of(eb_untag(troot, EB_LEAF), + struct eb64_node, node.branches); + } + return node; + } + + if ((y >> node->node.bit) >= EB_NODE_BRANCHES) + return NULL; /* no more common bits */ + + troot = node->node.branches.b[(key >> node->node.bit) & EB_NODE_BRANCH_MASK]; + } +} + +/* Insert eb64_node into subtree starting at node root . + * Only new->key needs be set with the key. The eb64_node is returned. + * If root->b[EB_RGHT]==1, the tree may only contain unique keys. + */ +static forceinline struct eb64_node * +__eb64_insert(struct eb_root *root, struct eb64_node *new) { + struct eb64_node *old; + unsigned int side; + eb_troot_t *troot; + u64 newkey; /* caching the key saves approximately one cycle */ + eb_troot_t *root_right = root; + + side = EB_LEFT; + troot = root->b[EB_LEFT]; + root_right = root->b[EB_RGHT]; + if (unlikely(troot == NULL)) { + /* Tree is empty, insert the leaf part below the left branch */ + root->b[EB_LEFT] = eb_dotag(&new->node.branches, EB_LEAF); + new->node.leaf_p = eb_dotag(root, EB_LEFT); + new->node.node_p = NULL; /* node part unused */ + return new; + } + + /* The tree descent is fairly easy : + * - first, check if we have reached a leaf node + * - second, check if we have gone too far + * - third, reiterate + * Everywhere, we use for the node node we are inserting, + * for the node we attach it to, and for the node we are + * displacing below . will always point to the future node + * (tagged with its type). carries the side the node is + * attached to below its parent, which is also where previous node + * was attached. carries the key being inserted. + */ + newkey = new->key; + + while (1) { + if (unlikely(eb_gettag(troot) == EB_LEAF)) { + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_leaf; + + old = container_of(eb_untag(troot, EB_LEAF), + struct eb64_node, node.branches); + + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_leaf = eb_dotag(&old->node.branches, EB_LEAF); + + new->node.node_p = old->node.leaf_p; + + /* Right here, we have 3 possibilities : + - the tree does not contain the key, and we have + new->key < old->key. We insert new above old, on + the left ; + + - the tree does not contain the key, and we have + new->key > old->key. We insert new above old, on + the right ; + + - the tree does contain the key, which implies it + is alone. We add the new key next to it as a + first duplicate. + + The last two cases can easily be partially merged. + */ + + if (new->key < old->key) { + new->node.leaf_p = new_left; + old->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_leaf; + } else { + /* we may refuse to duplicate this key if the tree is + * tagged as containing only unique keys. + */ + if ((new->key == old->key) && eb_gettag(root_right)) + return old; + + /* new->key >= old->key, new goes the right */ + old->node.leaf_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_leaf; + new->node.branches.b[EB_RGHT] = new_leaf; + + if (new->key == old->key) { + new->node.bit = -1; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + return new; + } + } + break; + } + + /* OK we're walking down this link */ + old = container_of(eb_untag(troot, EB_NODE), + struct eb64_node, node.branches); + + /* Stop going down when we don't have common bits anymore. We + * also stop in front of a duplicates tree because it means we + * have to insert above. + */ + + if ((old->node.bit < 0) || /* we're above a duplicate tree, stop here */ + (((new->key ^ old->key) >> old->node.bit) >= EB_NODE_BRANCHES)) { + /* The tree did not contain the key, so we insert before the node + * , and set ->bit to designate the lowest bit position in + * which applies to ->branches.b[]. + */ + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_node; + + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_node = eb_dotag(&old->node.branches, EB_NODE); + + new->node.node_p = old->node.node_p; + + if (new->key < old->key) { + new->node.leaf_p = new_left; + old->node.node_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_node; + } + else if (new->key > old->key) { + old->node.node_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_node; + new->node.branches.b[EB_RGHT] = new_leaf; + } + else { + struct eb_node *ret; + ret = eb_insert_dup(&old->node, &new->node); + return container_of(ret, struct eb64_node, node); + } + break; + } + + /* walk down */ + root = &old->node.branches; +#if BITS_PER_LONG >= 64 + side = (newkey >> old->node.bit) & EB_NODE_BRANCH_MASK; +#else + side = newkey; + side >>= old->node.bit; + if (old->node.bit >= 32) { + side = newkey >> 32; + side >>= old->node.bit & 0x1F; + } + side &= EB_NODE_BRANCH_MASK; +#endif + troot = root->b[side]; + } + + /* Ok, now we are inserting between and . 's + * parent is already set to , and the 's branch is still in + * . Update the root's leaf till we have it. Note that we can also + * find the side by checking the side of new->node.node_p. + */ + + /* We need the common higher bits between new->key and old->key. + * What differences are there between new->key and the node here ? + * NOTE that bit(new) is always < bit(root) because highest + * bit of new->key and old->key are identical here (otherwise they + * would sit on different branches). + */ + // note that if EB_NODE_BITS > 1, we should check that it's still >= 0 + new->node.bit = fls64(new->key ^ old->key) - EB_NODE_BITS; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + + return new; +} + +/* Insert eb64_node into subtree starting at node root , using + * signed keys. Only new->key needs be set with the key. The eb64_node + * is returned. If root->b[EB_RGHT]==1, the tree may only contain unique keys. + */ +static forceinline struct eb64_node * +__eb64i_insert(struct eb_root *root, struct eb64_node *new) { + struct eb64_node *old; + unsigned int side; + eb_troot_t *troot; + u64 newkey; /* caching the key saves approximately one cycle */ + eb_troot_t *root_right = root; + + side = EB_LEFT; + troot = root->b[EB_LEFT]; + root_right = root->b[EB_RGHT]; + if (unlikely(troot == NULL)) { + /* Tree is empty, insert the leaf part below the left branch */ + root->b[EB_LEFT] = eb_dotag(&new->node.branches, EB_LEAF); + new->node.leaf_p = eb_dotag(root, EB_LEFT); + new->node.node_p = NULL; /* node part unused */ + return new; + } + + /* The tree descent is fairly easy : + * - first, check if we have reached a leaf node + * - second, check if we have gone too far + * - third, reiterate + * Everywhere, we use for the node node we are inserting, + * for the node we attach it to, and for the node we are + * displacing below . will always point to the future node + * (tagged with its type). carries the side the node is + * attached to below its parent, which is also where previous node + * was attached. carries a high bit shift of the key being + * inserted in order to have negative keys stored before positive + * ones. + */ + newkey = new->key ^ (1ULL << 63); + + while (1) { + if (unlikely(eb_gettag(troot) == EB_LEAF)) { + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_leaf; + + old = container_of(eb_untag(troot, EB_LEAF), + struct eb64_node, node.branches); + + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_leaf = eb_dotag(&old->node.branches, EB_LEAF); + + new->node.node_p = old->node.leaf_p; + + /* Right here, we have 3 possibilities : + - the tree does not contain the key, and we have + new->key < old->key. We insert new above old, on + the left ; + + - the tree does not contain the key, and we have + new->key > old->key. We insert new above old, on + the right ; + + - the tree does contain the key, which implies it + is alone. We add the new key next to it as a + first duplicate. + + The last two cases can easily be partially merged. + */ + + if ((s64)new->key < (s64)old->key) { + new->node.leaf_p = new_left; + old->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_leaf; + } else { + /* we may refuse to duplicate this key if the tree is + * tagged as containing only unique keys. + */ + if ((new->key == old->key) && eb_gettag(root_right)) + return old; + + /* new->key >= old->key, new goes the right */ + old->node.leaf_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_leaf; + new->node.branches.b[EB_RGHT] = new_leaf; + + if (new->key == old->key) { + new->node.bit = -1; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + return new; + } + } + break; + } + + /* OK we're walking down this link */ + old = container_of(eb_untag(troot, EB_NODE), + struct eb64_node, node.branches); + + /* Stop going down when we don't have common bits anymore. We + * also stop in front of a duplicates tree because it means we + * have to insert above. + */ + + if ((old->node.bit < 0) || /* we're above a duplicate tree, stop here */ + (((new->key ^ old->key) >> old->node.bit) >= EB_NODE_BRANCHES)) { + /* The tree did not contain the key, so we insert before the node + * , and set ->bit to designate the lowest bit position in + * which applies to ->branches.b[]. + */ + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_node; + + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_node = eb_dotag(&old->node.branches, EB_NODE); + + new->node.node_p = old->node.node_p; + + if ((s64)new->key < (s64)old->key) { + new->node.leaf_p = new_left; + old->node.node_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_node; + } + else if ((s64)new->key > (s64)old->key) { + old->node.node_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_node; + new->node.branches.b[EB_RGHT] = new_leaf; + } + else { + struct eb_node *ret; + ret = eb_insert_dup(&old->node, &new->node); + return container_of(ret, struct eb64_node, node); + } + break; + } + + /* walk down */ + root = &old->node.branches; +#if BITS_PER_LONG >= 64 + side = (newkey >> old->node.bit) & EB_NODE_BRANCH_MASK; +#else + side = newkey; + side >>= old->node.bit; + if (old->node.bit >= 32) { + side = newkey >> 32; + side >>= old->node.bit & 0x1F; + } + side &= EB_NODE_BRANCH_MASK; +#endif + troot = root->b[side]; + } + + /* Ok, now we are inserting between and . 's + * parent is already set to , and the 's branch is still in + * . Update the root's leaf till we have it. Note that we can also + * find the side by checking the side of new->node.node_p. + */ + + /* We need the common higher bits between new->key and old->key. + * What differences are there between new->key and the node here ? + * NOTE that bit(new) is always < bit(root) because highest + * bit of new->key and old->key are identical here (otherwise they + * would sit on different branches). + */ + // note that if EB_NODE_BITS > 1, we should check that it's still >= 0 + new->node.bit = fls64(new->key ^ old->key) - EB_NODE_BITS; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + + return new; +} + +#endif /* _EB64_TREE_H */ diff --git a/ebtree/ebimtree.c b/ebtree/ebimtree.c new file mode 100644 index 0000000000..5e63937196 --- /dev/null +++ b/ebtree/ebimtree.c @@ -0,0 +1,44 @@ +/* + * Elastic Binary Trees - exported functinos for Indirect Multi-Byte data nodes. + * Version 5.0 + * (C) 2002-2009 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* Consult ebimtree.h for more details about those functions */ + +#include "ebpttree.h" +#include "ebimtree.h" + +/* Find the first occurence of a key of bytes in the tree . + * If none can be found, return NULL. + */ +REGPRM3 struct ebpt_node * +ebim_lookup(struct eb_root *root, const void *x, unsigned int len) +{ + return __ebim_lookup(root, x, len); +} + +/* Insert ebpt_node into subtree starting at node root . + * Only new->key needs be set with the key. The ebpt_node is returned. + * If root->b[EB_RGHT]==1, the tree may only contain unique keys. The + * len is specified in bytes. + */ +REGPRM3 struct ebpt_node * +ebim_insert(struct eb_root *root, struct ebpt_node *new, unsigned int len) +{ + return __ebim_insert(root, new, len); +} diff --git a/ebtree/ebimtree.h b/ebtree/ebimtree.h new file mode 100644 index 0000000000..f53a26a8b7 --- /dev/null +++ b/ebtree/ebimtree.h @@ -0,0 +1,263 @@ +/* + * Elastic Binary Trees - macros for Indirect Multi-Byte data nodes. + * Version 5.0 + * (C) 2002-2009 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +#include "ebtree.h" +#include "ebpttree.h" + +/* These functions and macros rely on Pointer nodes and use the entry as + * a pointer to an indirect key. Most operations are performed using ebpt_*. + */ + +/* The following functions are not inlined by default. They are declared + * in ebimtree.c, which simply relies on their inline version. + */ +REGPRM3 struct ebpt_node *ebim_lookup(struct eb_root *root, const void *x, unsigned int len); +REGPRM3 struct ebpt_node *ebim_insert(struct eb_root *root, struct ebpt_node *new, unsigned int len); + +/* Find the first occurence of a key of bytes in the tree . + * If none can be found, return NULL. + */ +static forceinline struct ebpt_node * +__ebim_lookup(struct eb_root *root, const void *x, unsigned int len) +{ + struct ebpt_node *node; + eb_troot_t *troot; + unsigned int bit; + + troot = root->b[EB_LEFT]; + if (unlikely(troot == NULL)) + return NULL; + + bit = 0; + while (1) { + if ((eb_gettag(troot) == EB_LEAF)) { + node = container_of(eb_untag(troot, EB_LEAF), + struct ebpt_node, node.branches); + if (memcmp(node->key, x, len) == 0) + return node; + else + return NULL; + } + node = container_of(eb_untag(troot, EB_NODE), + struct ebpt_node, node.branches); + + if (node->node.bit < 0) { + /* We have a dup tree now. Either it's for the same + * value, and we walk down left, or it's a different + * one and we don't have our key. + */ + if (memcmp(node->key, x, len) != 0) + return NULL; + + troot = node->node.branches.b[EB_LEFT]; + while (eb_gettag(troot) != EB_LEAF) + troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT]; + node = container_of(eb_untag(troot, EB_LEAF), + struct ebpt_node, node.branches); + return node; + } + + /* OK, normal data node, let's walk down */ + bit = equal_bits(x, node->key, bit, node->node.bit); + if (bit < node->node.bit) + return NULL; /* no more common bits */ + + troot = node->node.branches.b[(((unsigned char*)x)[node->node.bit >> 3] >> + (~node->node.bit & 7)) & 1]; + } +} + +/* Insert ebpt_node into subtree starting at node root . + * Only new->key needs be set with the key. The ebpt_node is returned. + * If root->b[EB_RGHT]==1, the tree may only contain unique keys. The + * len is specified in bytes. + */ +static forceinline struct ebpt_node * +__ebim_insert(struct eb_root *root, struct ebpt_node *new, unsigned int len) +{ + struct ebpt_node *old; + unsigned int side; + eb_troot_t *troot; + eb_troot_t *root_right = root; + int diff; + int bit; + + side = EB_LEFT; + troot = root->b[EB_LEFT]; + root_right = root->b[EB_RGHT]; + if (unlikely(troot == NULL)) { + /* Tree is empty, insert the leaf part below the left branch */ + root->b[EB_LEFT] = eb_dotag(&new->node.branches, EB_LEAF); + new->node.leaf_p = eb_dotag(root, EB_LEFT); + new->node.node_p = NULL; /* node part unused */ + return new; + } + + len <<= 3; + + /* The tree descent is fairly easy : + * - first, check if we have reached a leaf node + * - second, check if we have gone too far + * - third, reiterate + * Everywhere, we use for the node node we are inserting, + * for the node we attach it to, and for the node we are + * displacing below . will always point to the future node + * (tagged with its type). carries the side the node is + * attached to below its parent, which is also where previous node + * was attached. + */ + + bit = 0; + while (1) { + if (unlikely(eb_gettag(troot) == EB_LEAF)) { + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_leaf; + + old = container_of(eb_untag(troot, EB_LEAF), + struct ebpt_node, node.branches); + + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_leaf = eb_dotag(&old->node.branches, EB_LEAF); + + new->node.node_p = old->node.leaf_p; + + /* Right here, we have 3 possibilities : + * - the tree does not contain the key, and we have + * new->key < old->key. We insert new above old, on + * the left ; + * + * - the tree does not contain the key, and we have + * new->key > old->key. We insert new above old, on + * the right ; + * + * - the tree does contain the key, which implies it + * is alone. We add the new key next to it as a + * first duplicate. + * + * The last two cases can easily be partially merged. + */ + bit = equal_bits(new->key, old->key, bit, len); + diff = cmp_bits(new->key, old->key, bit); + + if (diff < 0) { + new->node.leaf_p = new_left; + old->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_leaf; + } else { + /* we may refuse to duplicate this key if the tree is + * tagged as containing only unique keys. + */ + if (diff == 0 && eb_gettag(root_right)) + return old; + + /* new->key >= old->key, new goes the right */ + old->node.leaf_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_leaf; + new->node.branches.b[EB_RGHT] = new_leaf; + + if (diff == 0) { + new->node.bit = -1; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + return new; + } + } + break; + } + + /* OK we're walking down this link */ + old = container_of(eb_untag(troot, EB_NODE), + struct ebpt_node, node.branches); + + /* Stop going down when we don't have common bits anymore. We + * also stop in front of a duplicates tree because it means we + * have to insert above. Note: we can compare more bits than + * the current node's because as long as they are identical, we + * know we descend along the correct side. + */ + if (old->node.bit < 0) { + /* we're above a duplicate tree, we must compare till the end */ + bit = equal_bits(new->key, old->key, bit, len); + goto dup_tree; + } + else if (bit < old->node.bit) { + bit = equal_bits(new->key, old->key, bit, old->node.bit); + } + + if (bit < old->node.bit) { /* we don't have all bits in common */ + /* The tree did not contain the key, so we insert before the node + * , and set ->bit to designate the lowest bit position in + * which applies to ->branches.b[]. + */ + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_node; + + dup_tree: + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_node = eb_dotag(&old->node.branches, EB_NODE); + + new->node.node_p = old->node.node_p; + + diff = cmp_bits(new->key, old->key, bit); + if (diff < 0) { + new->node.leaf_p = new_left; + old->node.node_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_node; + } + else if (diff > 0) { + old->node.node_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_node; + new->node.branches.b[EB_RGHT] = new_leaf; + } + else { + struct eb_node *ret; + ret = eb_insert_dup(&old->node, &new->node); + return container_of(ret, struct ebpt_node, node); + } + break; + } + + /* walk down */ + root = &old->node.branches; + side = (((unsigned char *)new->key)[old->node.bit >> 3] >> (~old->node.bit & 7)) & 1; + troot = root->b[side]; + } + + /* Ok, now we are inserting between and . 's + * parent is already set to , and the 's branch is still in + * . Update the root's leaf till we have it. Note that we can also + * find the side by checking the side of new->node.node_p. + */ + + /* We need the common higher bits between new->key and old->key. + * This number of bits is already in . + */ + new->node.bit = bit; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + return new; +} diff --git a/ebtree/ebistree.c b/ebtree/ebistree.c new file mode 100644 index 0000000000..acaf3fa524 --- /dev/null +++ b/ebtree/ebistree.c @@ -0,0 +1,42 @@ +/* + * Elastic Binary Trees - exported functions for Indirect String data nodes. + * Version 5.0 + * (C) 2002-2009 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* Consult ebistree.h for more details about those functions */ + +#include "ebistree.h" + +/* Find the first occurence of a zero-terminated string in the tree . + * It's the caller's reponsibility to use this function only on trees which + * only contain zero-terminated strings. If none can be found, return NULL. + */ +REGPRM2 struct ebpt_node *ebis_lookup(struct eb_root *root, const char *x) +{ + return __ebis_lookup(root, x); +} + +/* Insert ebpt_node into subtree starting at node root . Only + * new->key needs be set with the zero-terminated string key. The ebpt_node is + * returned. If root->b[EB_RGHT]==1, the tree may only contain unique keys. The + * caller is responsible for properly terminating the key with a zero. + */ +REGPRM2 struct ebpt_node *ebis_insert(struct eb_root *root, struct ebpt_node *new) +{ + return __ebis_insert(root, new); +} diff --git a/ebtree/ebistree.h b/ebtree/ebistree.h new file mode 100644 index 0000000000..984df2e647 --- /dev/null +++ b/ebtree/ebistree.h @@ -0,0 +1,263 @@ +/* + * Elastic Binary Trees - macros to manipulate Indirect String data nodes. + * Version 5.0 + * (C) 2002-2009 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* These functions and macros rely on Multi-Byte nodes */ + +#include +#include "ebtree.h" +#include "ebpttree.h" + +/* These functions and macros rely on Pointer nodes and use the entry as + * a pointer to an indirect key. Most operations are performed using ebpt_*. + */ + +/* The following functions are not inlined by default. They are declared + * in ebistree.c, which simply relies on their inline version. + */ +REGPRM2 struct ebpt_node *ebis_lookup(struct eb_root *root, const char *x); +REGPRM2 struct ebpt_node *ebis_insert(struct eb_root *root, struct ebpt_node *new); + +/* Find the first occurence of a zero-terminated string in the tree . + * It's the caller's reponsibility to use this function only on trees which + * only contain zero-terminated strings. If none can be found, return NULL. + */ +static forceinline struct ebpt_node *__ebis_lookup(struct eb_root *root, const void *x) +{ + struct ebpt_node *node; + eb_troot_t *troot; + unsigned int bit; + + troot = root->b[EB_LEFT]; + if (unlikely(troot == NULL)) + return NULL; + + bit = 0; + while (1) { + if ((eb_gettag(troot) == EB_LEAF)) { + node = container_of(eb_untag(troot, EB_LEAF), + struct ebpt_node, node.branches); + if (strcmp(node->key, x) == 0) + return node; + else + return NULL; + } + node = container_of(eb_untag(troot, EB_NODE), + struct ebpt_node, node.branches); + + if (node->node.bit < 0) { + /* We have a dup tree now. Either it's for the same + * value, and we walk down left, or it's a different + * one and we don't have our key. + */ + if (strcmp(node->key, x) != 0) + return NULL; + + troot = node->node.branches.b[EB_LEFT]; + while (eb_gettag(troot) != EB_LEAF) + troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT]; + node = container_of(eb_untag(troot, EB_LEAF), + struct ebpt_node, node.branches); + return node; + } + + /* OK, normal data node, let's walk down */ + bit = string_equal_bits(x, node->key, bit); + if (bit < node->node.bit) + return NULL; /* no more common bits */ + + troot = node->node.branches.b[(((unsigned char*)x)[node->node.bit >> 3] >> + (~node->node.bit & 7)) & 1]; + } +} + +/* Insert ebpt_node into subtree starting at node root . Only + * new->key needs be set with the zero-terminated string key. The ebpt_node is + * returned. If root->b[EB_RGHT]==1, the tree may only contain unique keys. The + * caller is responsible for properly terminating the key with a zero. + */ +static forceinline struct ebpt_node * +__ebis_insert(struct eb_root *root, struct ebpt_node *new) +{ + struct ebpt_node *old; + unsigned int side; + eb_troot_t *troot; + eb_troot_t *root_right = root; + int diff; + int bit; + + side = EB_LEFT; + troot = root->b[EB_LEFT]; + root_right = root->b[EB_RGHT]; + if (unlikely(troot == NULL)) { + /* Tree is empty, insert the leaf part below the left branch */ + root->b[EB_LEFT] = eb_dotag(&new->node.branches, EB_LEAF); + new->node.leaf_p = eb_dotag(root, EB_LEFT); + new->node.node_p = NULL; /* node part unused */ + return new; + } + + /* The tree descent is fairly easy : + * - first, check if we have reached a leaf node + * - second, check if we have gone too far + * - third, reiterate + * Everywhere, we use for the node node we are inserting, + * for the node we attach it to, and for the node we are + * displacing below . will always point to the future node + * (tagged with its type). carries the side the node is + * attached to below its parent, which is also where previous node + * was attached. + */ + + bit = 0; + while (1) { + if (unlikely(eb_gettag(troot) == EB_LEAF)) { + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_leaf; + + old = container_of(eb_untag(troot, EB_LEAF), + struct ebpt_node, node.branches); + + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_leaf = eb_dotag(&old->node.branches, EB_LEAF); + + new->node.node_p = old->node.leaf_p; + + /* Right here, we have 3 possibilities : + * - the tree does not contain the key, and we have + * new->key < old->key. We insert new above old, on + * the left ; + * + * - the tree does not contain the key, and we have + * new->key > old->key. We insert new above old, on + * the right ; + * + * - the tree does contain the key, which implies it + * is alone. We add the new key next to it as a + * first duplicate. + * + * The last two cases can easily be partially merged. + */ + bit = string_equal_bits(new->key, old->key, bit); + diff = cmp_bits(new->key, old->key, bit); + + if (diff < 0) { + new->node.leaf_p = new_left; + old->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_leaf; + } else { + /* we may refuse to duplicate this key if the tree is + * tagged as containing only unique keys. + */ + if (diff == 0 && eb_gettag(root_right)) + return old; + + /* new->key >= old->key, new goes the right */ + old->node.leaf_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_leaf; + new->node.branches.b[EB_RGHT] = new_leaf; + + if (diff == 0) { + new->node.bit = -1; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + return new; + } + } + break; + } + + /* OK we're walking down this link */ + old = container_of(eb_untag(troot, EB_NODE), + struct ebpt_node, node.branches); + + /* Stop going down when we don't have common bits anymore. We + * also stop in front of a duplicates tree because it means we + * have to insert above. Note: we can compare more bits than + * the current node's because as long as they are identical, we + * know we descend along the correct side. + */ + if (old->node.bit < 0) { + /* we're above a duplicate tree, we must compare till the end */ + bit = string_equal_bits(new->key, old->key, bit); + goto dup_tree; + } + else if (bit < old->node.bit) { + bit = string_equal_bits(new->key, old->key, bit); + } + + if (bit < old->node.bit) { /* we don't have all bits in common */ + /* The tree did not contain the key, so we insert before the node + * , and set ->bit to designate the lowest bit position in + * which applies to ->branches.b[]. + */ + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_node; + dup_tree: + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_node = eb_dotag(&old->node.branches, EB_NODE); + + new->node.node_p = old->node.node_p; + + diff = cmp_bits(new->key, old->key, bit); + if (diff < 0) { + new->node.leaf_p = new_left; + old->node.node_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_node; + } + else if (diff > 0) { + old->node.node_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_node; + new->node.branches.b[EB_RGHT] = new_leaf; + } + else { + struct eb_node *ret; + ret = eb_insert_dup(&old->node, &new->node); + return container_of(ret, struct ebpt_node, node); + } + break; + } + + /* walk down */ + root = &old->node.branches; + side = (((unsigned char *)new->key)[old->node.bit >> 3] >> (~old->node.bit & 7)) & 1; + troot = root->b[side]; + } + + /* Ok, now we are inserting between and . 's + * parent is already set to , and the 's branch is still in + * . Update the root's leaf till we have it. Note that we can also + * find the side by checking the side of new->node.node_p. + */ + + /* We need the common higher bits between new->key and old->key. + * This number of bits is already in . + */ + new->node.bit = bit; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + return new; +} + diff --git a/ebtree/ebmbtree.c b/ebtree/ebmbtree.c new file mode 100644 index 0000000000..a4a0354022 --- /dev/null +++ b/ebtree/ebmbtree.c @@ -0,0 +1,43 @@ +/* + * Elastic Binary Trees - exported functinos for Multi-Byte data nodes. + * Version 5.0 + * (C) 2002-2009 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* Consult ebmbtree.h for more details about those functions */ + +#include "ebmbtree.h" + +/* Find the first occurence of a key of bytes in the tree . + * If none can be found, return NULL. + */ +REGPRM3 struct ebmb_node * +ebmb_lookup(struct eb_root *root, const void *x, unsigned int len) +{ + return __ebmb_lookup(root, x, len); +} + +/* Insert ebmb_node into subtree starting at node root . + * Only new->key needs be set with the key. The ebmb_node is returned. + * If root->b[EB_RGHT]==1, the tree may only contain unique keys. The + * len is specified in bytes. + */ +REGPRM3 struct ebmb_node * +ebmb_insert(struct eb_root *root, struct ebmb_node *new, unsigned int len) +{ + return __ebmb_insert(root, new, len); +} diff --git a/ebtree/ebmbtree.h b/ebtree/ebmbtree.h new file mode 100644 index 0000000000..947dd5e8ba --- /dev/null +++ b/ebtree/ebmbtree.h @@ -0,0 +1,336 @@ +/* + * Elastic Binary Trees - macros and structures for Multi-Byte data nodes. + * Version 5.0 + * (C) 2002-2009 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +#include "ebtree.h" + +/* Return the structure of type whose member points to */ +#define ebmb_entry(ptr, type, member) container_of(ptr, type, member) + +#define EBMB_ROOT EB_ROOT +#define EBMB_TREE_HEAD EB_TREE_HEAD + +/* This structure carries a node, a leaf, and a key. It must start with the + * eb_node so that it can be cast into an eb_node. We could also have put some + * sort of transparent union here to reduce the indirection level, but the fact + * is, the end user is not meant to manipulate internals, so this is pointless. + * The 'node.bit' value here works differently from scalar types, as it contains + * the number of identical bits between the two branches. + */ +struct ebmb_node { + struct eb_node node; /* the tree node, must be at the beginning */ + unsigned char key[0]; /* the key, its size depends on the application */ +}; + +/* + * Exported functions and macros. + * Many of them are always inlined because they are extremely small, and + * are generally called at most once or twice in a program. + */ + +/* Return leftmost node in the tree, or NULL if none */ +static forceinline struct ebmb_node *ebmb_first(struct eb_root *root) +{ + return ebmb_entry(eb_first(root), struct ebmb_node, node); +} + +/* Return rightmost node in the tree, or NULL if none */ +static forceinline struct ebmb_node *ebmb_last(struct eb_root *root) +{ + return ebmb_entry(eb_last(root), struct ebmb_node, node); +} + +/* Return next node in the tree, or NULL if none */ +static forceinline struct ebmb_node *ebmb_next(struct ebmb_node *ebmb) +{ + return ebmb_entry(eb_next(&ebmb->node), struct ebmb_node, node); +} + +/* Return previous node in the tree, or NULL if none */ +static forceinline struct ebmb_node *ebmb_prev(struct ebmb_node *ebmb) +{ + return ebmb_entry(eb_prev(&ebmb->node), struct ebmb_node, node); +} + +/* Return next node in the tree, skipping duplicates, or NULL if none */ +static forceinline struct ebmb_node *ebmb_next_unique(struct ebmb_node *ebmb) +{ + return ebmb_entry(eb_next_unique(&ebmb->node), struct ebmb_node, node); +} + +/* Return previous node in the tree, skipping duplicates, or NULL if none */ +static forceinline struct ebmb_node *ebmb_prev_unique(struct ebmb_node *ebmb) +{ + return ebmb_entry(eb_prev_unique(&ebmb->node), struct ebmb_node, node); +} + +/* Delete node from the tree if it was linked in. Mark the node unused. Note + * that this function relies on a non-inlined generic function: eb_delete. + */ +static forceinline void ebmb_delete(struct ebmb_node *ebmb) +{ + eb_delete(&ebmb->node); +} + +/* The following functions are not inlined by default. They are declared + * in ebmbtree.c, which simply relies on their inline version. + */ +REGPRM3 struct ebmb_node *ebmb_lookup(struct eb_root *root, const void *x, unsigned int len); +REGPRM3 struct ebmb_node *ebmb_insert(struct eb_root *root, struct ebmb_node *new, unsigned int len); + +/* The following functions are less likely to be used directly, because their + * code is larger. The non-inlined version is preferred. + */ + +/* Delete node from the tree if it was linked in. Mark the node unused. */ +static forceinline void __ebmb_delete(struct ebmb_node *ebmb) +{ + __eb_delete(&ebmb->node); +} + +/* Find the first occurence of a key of bytes in the tree . + * If none can be found, return NULL. + */ +static forceinline struct ebmb_node *__ebmb_lookup(struct eb_root *root, const void *x, unsigned int len) +{ + struct ebmb_node *node; + eb_troot_t *troot; + unsigned int bit; + + troot = root->b[EB_LEFT]; + if (unlikely(troot == NULL)) + return NULL; + + bit = 0; + while (1) { + if ((eb_gettag(troot) == EB_LEAF)) { + node = container_of(eb_untag(troot, EB_LEAF), + struct ebmb_node, node.branches); + if (memcmp(node->key, x, len) == 0) + return node; + else + return NULL; + } + node = container_of(eb_untag(troot, EB_NODE), + struct ebmb_node, node.branches); + + if (node->node.bit < 0) { + /* We have a dup tree now. Either it's for the same + * value, and we walk down left, or it's a different + * one and we don't have our key. + */ + if (memcmp(node->key, x, len) != 0) + return NULL; + + troot = node->node.branches.b[EB_LEFT]; + while (eb_gettag(troot) != EB_LEAF) + troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT]; + node = container_of(eb_untag(troot, EB_LEAF), + struct ebmb_node, node.branches); + return node; + } + + /* OK, normal data node, let's walk down */ + bit = equal_bits(x, node->key, bit, node->node.bit); + if (bit < node->node.bit) + return NULL; /* no more common bits */ + + troot = node->node.branches.b[(((unsigned char*)x)[node->node.bit >> 3] >> + (~node->node.bit & 7)) & 1]; + } +} + +/* Insert ebmb_node into subtree starting at node root . + * Only new->key needs be set with the key. The ebmb_node is returned. + * If root->b[EB_RGHT]==1, the tree may only contain unique keys. The + * len is specified in bytes. + */ +static forceinline struct ebmb_node * +__ebmb_insert(struct eb_root *root, struct ebmb_node *new, unsigned int len) +{ + struct ebmb_node *old; + unsigned int side; + eb_troot_t *troot; + eb_troot_t *root_right = root; + int diff; + int bit; + + side = EB_LEFT; + troot = root->b[EB_LEFT]; + root_right = root->b[EB_RGHT]; + if (unlikely(troot == NULL)) { + /* Tree is empty, insert the leaf part below the left branch */ + root->b[EB_LEFT] = eb_dotag(&new->node.branches, EB_LEAF); + new->node.leaf_p = eb_dotag(root, EB_LEFT); + new->node.node_p = NULL; /* node part unused */ + return new; + } + + len <<= 3; + + /* The tree descent is fairly easy : + * - first, check if we have reached a leaf node + * - second, check if we have gone too far + * - third, reiterate + * Everywhere, we use for the node node we are inserting, + * for the node we attach it to, and for the node we are + * displacing below . will always point to the future node + * (tagged with its type). carries the side the node is + * attached to below its parent, which is also where previous node + * was attached. + */ + + bit = 0; + while (1) { + if (unlikely(eb_gettag(troot) == EB_LEAF)) { + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_leaf; + + old = container_of(eb_untag(troot, EB_LEAF), + struct ebmb_node, node.branches); + + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_leaf = eb_dotag(&old->node.branches, EB_LEAF); + + new->node.node_p = old->node.leaf_p; + + /* Right here, we have 3 possibilities : + * - the tree does not contain the key, and we have + * new->key < old->key. We insert new above old, on + * the left ; + * + * - the tree does not contain the key, and we have + * new->key > old->key. We insert new above old, on + * the right ; + * + * - the tree does contain the key, which implies it + * is alone. We add the new key next to it as a + * first duplicate. + * + * The last two cases can easily be partially merged. + */ + bit = equal_bits(new->key, old->key, bit, len); + diff = cmp_bits(new->key, old->key, bit); + + if (diff < 0) { + new->node.leaf_p = new_left; + old->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_leaf; + } else { + /* we may refuse to duplicate this key if the tree is + * tagged as containing only unique keys. + */ + if (diff == 0 && eb_gettag(root_right)) + return old; + + /* new->key >= old->key, new goes the right */ + old->node.leaf_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_leaf; + new->node.branches.b[EB_RGHT] = new_leaf; + + if (diff == 0) { + new->node.bit = -1; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + return new; + } + } + break; + } + + /* OK we're walking down this link */ + old = container_of(eb_untag(troot, EB_NODE), + struct ebmb_node, node.branches); + + /* Stop going down when we don't have common bits anymore. We + * also stop in front of a duplicates tree because it means we + * have to insert above. Note: we can compare more bits than + * the current node's because as long as they are identical, we + * know we descend along the correct side. + */ + if (old->node.bit < 0) { + /* we're above a duplicate tree, we must compare till the end */ + bit = equal_bits(new->key, old->key, bit, len); + goto dup_tree; + } + else if (bit < old->node.bit) { + bit = equal_bits(new->key, old->key, bit, old->node.bit); + } + + if (bit < old->node.bit) { /* we don't have all bits in common */ + /* The tree did not contain the key, so we insert before the node + * , and set ->bit to designate the lowest bit position in + * which applies to ->branches.b[]. + */ + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_node; + + dup_tree: + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_node = eb_dotag(&old->node.branches, EB_NODE); + + new->node.node_p = old->node.node_p; + + diff = cmp_bits(new->key, old->key, bit); + if (diff < 0) { + new->node.leaf_p = new_left; + old->node.node_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_node; + } + else if (diff > 0) { + old->node.node_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_node; + new->node.branches.b[EB_RGHT] = new_leaf; + } + else { + struct eb_node *ret; + ret = eb_insert_dup(&old->node, &new->node); + return container_of(ret, struct ebmb_node, node); + } + break; + } + + /* walk down */ + root = &old->node.branches; + side = (new->key[old->node.bit >> 3] >> (~old->node.bit & 7)) & 1; + troot = root->b[side]; + } + + /* Ok, now we are inserting between and . 's + * parent is already set to , and the 's branch is still in + * . Update the root's leaf till we have it. Note that we can also + * find the side by checking the side of new->node.node_p. + */ + + /* We need the common higher bits between new->key and old->key. + * This number of bits is already in . + */ + new->node.bit = bit; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + return new; +} + diff --git a/ebtree/ebpttree.c b/ebtree/ebpttree.c new file mode 100644 index 0000000000..76e6db8f97 --- /dev/null +++ b/ebtree/ebpttree.c @@ -0,0 +1,207 @@ +/* + * Elastic Binary Trees - exported functions for operations on pointer nodes. + * (C) 2002-2007 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* Consult ebpttree.h for more details about those functions */ + +#include "ebpttree.h" + +REGPRM2 struct ebpt_node *ebpt_insert(struct eb_root *root, struct ebpt_node *new) +{ + return __ebpt_insert(root, new); +} + +REGPRM2 struct ebpt_node *ebpt_lookup(struct eb_root *root, void *x) +{ + return __ebpt_lookup(root, x); +} + +/* + * Find the last occurrence of the highest key in the tree , which is + * equal to or less than . NULL is returned is no key matches. + */ +REGPRM2 struct ebpt_node *ebpt_lookup_le(struct eb_root *root, void *x) +{ + struct ebpt_node *node; + eb_troot_t *troot; + + troot = root->b[EB_LEFT]; + if (unlikely(troot == NULL)) + return NULL; + + while (1) { + if ((eb_gettag(troot) == EB_LEAF)) { + /* We reached a leaf, which means that the whole upper + * parts were common. We will return either the current + * node or its next one if the former is too small. + */ + node = container_of(eb_untag(troot, EB_LEAF), + struct ebpt_node, node.branches); + if (node->key <= x) + return node; + /* return prev */ + troot = node->node.leaf_p; + break; + } + node = container_of(eb_untag(troot, EB_NODE), + struct ebpt_node, node.branches); + + if (node->node.bit < 0) { + /* We're at the top of a dup tree. Either we got a + * matching value and we return the rightmost node, or + * we don't and we skip the whole subtree to return the + * prev node before the subtree. Note that since we're + * at the top of the dup tree, we can simply return the + * prev node without first trying to escape from the + * tree. + */ + if (node->key <= x) { + troot = node->node.branches.b[EB_RGHT]; + while (eb_gettag(troot) != EB_LEAF) + troot = (eb_untag(troot, EB_NODE))->b[EB_RGHT]; + return container_of(eb_untag(troot, EB_LEAF), + struct ebpt_node, node.branches); + } + /* return prev */ + troot = node->node.node_p; + break; + } + + if ((((ptr_t)x ^ (ptr_t)node->key) >> node->node.bit) >= EB_NODE_BRANCHES) { + /* No more common bits at all. Either this node is too + * small and we need to get its highest value, or it is + * too large, and we need to get the prev value. + */ + if (((ptr_t)node->key >> node->node.bit) > ((ptr_t)x >> node->node.bit)) { + troot = node->node.branches.b[EB_RGHT]; + return ebpt_entry(eb_walk_down(troot, EB_RGHT), struct ebpt_node, node); + } + + /* Further values will be too high here, so return the prev + * unique node (if it exists). + */ + troot = node->node.node_p; + break; + } + troot = node->node.branches.b[((ptr_t)x >> node->node.bit) & EB_NODE_BRANCH_MASK]; + } + + /* If we get here, it means we want to report previous node before the + * current one which is not above. is already initialised to + * the parent's branches. + */ + while (eb_gettag(troot) == EB_LEFT) { + /* Walking up from left branch. We must ensure that we never + * walk beyond root. + */ + if (unlikely(eb_clrtag((eb_untag(troot, EB_LEFT))->b[EB_RGHT]) == NULL)) + return NULL; + troot = (eb_root_to_node(eb_untag(troot, EB_LEFT)))->node_p; + } + /* Note that cannot be NULL at this stage */ + troot = (eb_untag(troot, EB_RGHT))->b[EB_LEFT]; + node = ebpt_entry(eb_walk_down(troot, EB_RGHT), struct ebpt_node, node); + return node; +} + +/* + * Find the first occurrence of the lowest key in the tree , which is + * equal to or greater than . NULL is returned is no key matches. + */ +REGPRM2 struct ebpt_node *ebpt_lookup_ge(struct eb_root *root, void *x) +{ + struct ebpt_node *node; + eb_troot_t *troot; + + troot = root->b[EB_LEFT]; + if (unlikely(troot == NULL)) + return NULL; + + while (1) { + if ((eb_gettag(troot) == EB_LEAF)) { + /* We reached a leaf, which means that the whole upper + * parts were common. We will return either the current + * node or its next one if the former is too small. + */ + node = container_of(eb_untag(troot, EB_LEAF), + struct ebpt_node, node.branches); + if (node->key >= x) + return node; + /* return next */ + troot = node->node.leaf_p; + break; + } + node = container_of(eb_untag(troot, EB_NODE), + struct ebpt_node, node.branches); + + if (node->node.bit < 0) { + /* We're at the top of a dup tree. Either we got a + * matching value and we return the leftmost node, or + * we don't and we skip the whole subtree to return the + * next node after the subtree. Note that since we're + * at the top of the dup tree, we can simply return the + * next node without first trying to escape from the + * tree. + */ + if (node->key >= x) { + troot = node->node.branches.b[EB_LEFT]; + while (eb_gettag(troot) != EB_LEAF) + troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT]; + return container_of(eb_untag(troot, EB_LEAF), + struct ebpt_node, node.branches); + } + /* return next */ + troot = node->node.node_p; + break; + } + + if ((((ptr_t)x ^ (ptr_t)node->key) >> node->node.bit) >= EB_NODE_BRANCHES) { + /* No more common bits at all. Either this node is too + * large and we need to get its lowest value, or it is too + * small, and we need to get the next value. + */ + if (((ptr_t)node->key >> node->node.bit) > ((ptr_t)x >> node->node.bit)) { + troot = node->node.branches.b[EB_LEFT]; + return ebpt_entry(eb_walk_down(troot, EB_LEFT), struct ebpt_node, node); + } + + /* Further values will be too low here, so return the next + * unique node (if it exists). + */ + troot = node->node.node_p; + break; + } + troot = node->node.branches.b[((ptr_t)x >> node->node.bit) & EB_NODE_BRANCH_MASK]; + } + + /* If we get here, it means we want to report next node after the + * current one which is not below. is already initialised + * to the parent's branches. + */ + while (eb_gettag(troot) != EB_LEFT) + /* Walking up from right branch, so we cannot be below root */ + troot = (eb_root_to_node(eb_untag(troot, EB_RGHT)))->node_p; + + /* Note that cannot be NULL at this stage */ + troot = (eb_untag(troot, EB_LEFT))->b[EB_RGHT]; + if (eb_clrtag(troot) == NULL) + return NULL; + + node = ebpt_entry(eb_walk_down(troot, EB_LEFT), struct ebpt_node, node); + return node; +} diff --git a/ebtree/ebpttree.h b/ebtree/ebpttree.h new file mode 100644 index 0000000000..f6e521986e --- /dev/null +++ b/ebtree/ebpttree.h @@ -0,0 +1,165 @@ +/* + * Elastic Binary Trees - macros and structures for operations on pointer nodes. + * Version 5.0 + * (C) 2002-2009 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef _EBPTTREE_H +#define _EBPTTREE_H + +#include "ebtree.h" +#include "eb32tree.h" +#include "eb64tree.h" + + +/* Return the structure of type whose member points to */ +#define ebpt_entry(ptr, type, member) container_of(ptr, type, member) + +#define EBPT_ROOT EB_ROOT +#define EBPT_TREE_HEAD EB_TREE_HEAD + +/* on *almost* all platforms, a pointer can be cast into a size_t which is unsigned */ +#ifndef PTR_INT_TYPE +#define PTR_INT_TYPE size_t +#endif + +typedef PTR_INT_TYPE ptr_t; + +/* This structure carries a node, a leaf, and a key. It must start with the + * eb_node so that it can be cast into an eb_node. We could also have put some + * sort of transparent union here to reduce the indirection level, but the fact + * is, the end user is not meant to manipulate internals, so this is pointless. + * Internally, it is automatically cast as an eb32_node or eb64_node. + */ +struct ebpt_node { + struct eb_node node; /* the tree node, must be at the beginning */ + void *key; +}; + +/* + * Exported functions and macros. + * Many of them are always inlined because they are extremely small, and + * are generally called at most once or twice in a program. + */ + +/* Return leftmost node in the tree, or NULL if none */ +static forceinline struct ebpt_node *ebpt_first(struct eb_root *root) +{ + return ebpt_entry(eb_first(root), struct ebpt_node, node); +} + +/* Return rightmost node in the tree, or NULL if none */ +static forceinline struct ebpt_node *ebpt_last(struct eb_root *root) +{ + return ebpt_entry(eb_last(root), struct ebpt_node, node); +} + +/* Return next node in the tree, or NULL if none */ +static forceinline struct ebpt_node *ebpt_next(struct ebpt_node *ebpt) +{ + return ebpt_entry(eb_next(&ebpt->node), struct ebpt_node, node); +} + +/* Return previous node in the tree, or NULL if none */ +static forceinline struct ebpt_node *ebpt_prev(struct ebpt_node *ebpt) +{ + return ebpt_entry(eb_prev(&ebpt->node), struct ebpt_node, node); +} + +/* Return next node in the tree, skipping duplicates, or NULL if none */ +static forceinline struct ebpt_node *ebpt_next_unique(struct ebpt_node *ebpt) +{ + return ebpt_entry(eb_next_unique(&ebpt->node), struct ebpt_node, node); +} + +/* Return previous node in the tree, skipping duplicates, or NULL if none */ +static forceinline struct ebpt_node *ebpt_prev_unique(struct ebpt_node *ebpt) +{ + return ebpt_entry(eb_prev_unique(&ebpt->node), struct ebpt_node, node); +} + +/* Delete node from the tree if it was linked in. Mark the node unused. Note + * that this function relies on a non-inlined generic function: eb_delete. + */ +static forceinline void ebpt_delete(struct ebpt_node *ebpt) +{ + eb_delete(&ebpt->node); +} + +/* + * The following functions are inlined but derived from the integer versions. + */ +static forceinline struct ebpt_node *ebpt_lookup(struct eb_root *root, void *x) +{ + if (sizeof(void *) == 4) + return (struct ebpt_node *)eb32_lookup(root, (u32)(PTR_INT_TYPE)x); + else + return (struct ebpt_node *)eb64_lookup(root, (u64)(PTR_INT_TYPE)x); +} + +static forceinline struct ebpt_node *ebpt_lookup_le(struct eb_root *root, void *x) +{ + if (sizeof(void *) == 4) + return (struct ebpt_node *)eb32_lookup_le(root, (u32)(PTR_INT_TYPE)x); + else + return (struct ebpt_node *)eb64_lookup_le(root, (u64)(PTR_INT_TYPE)x); +} + +static forceinline struct ebpt_node *ebpt_lookup_ge(struct eb_root *root, void *x) +{ + if (sizeof(void *) == 4) + return (struct ebpt_node *)eb32_lookup_ge(root, (u32)(PTR_INT_TYPE)x); + else + return (struct ebpt_node *)eb64_lookup_ge(root, (u64)(PTR_INT_TYPE)x); +} + +static forceinline struct ebpt_node *ebpt_insert(struct eb_root *root, struct ebpt_node *new) +{ + if (sizeof(void *) == 4) + return (struct ebpt_node *)eb32_insert(root, (struct eb32_node *)new); + else + return (struct ebpt_node *)eb64_insert(root, (struct eb64_node *)new); +} + +/* + * The following functions are less likely to be used directly, because + * their code is larger. The non-inlined version is preferred. + */ + +/* Delete node from the tree if it was linked in. Mark the node unused. */ +static forceinline void __ebpt_delete(struct ebpt_node *ebpt) +{ + __eb_delete(&ebpt->node); +} + +static forceinline struct ebpt_node *__ebpt_lookup(struct eb_root *root, void *x) +{ + if (sizeof(void *) == 4) + return (struct ebpt_node *)__eb32_lookup(root, (u32)(PTR_INT_TYPE)x); + else + return (struct ebpt_node *)__eb64_lookup(root, (u64)(PTR_INT_TYPE)x); +} + +static forceinline struct ebpt_node *__ebpt_insert(struct eb_root *root, struct ebpt_node *new) +{ + if (sizeof(void *) == 4) + return (struct ebpt_node *)__eb32_insert(root, (struct eb32_node *)new); + else + return (struct ebpt_node *)__eb64_insert(root, (struct eb64_node *)new); +} + +#endif /* _EBPT_TREE_H */ diff --git a/ebtree/ebsttree.c b/ebtree/ebsttree.c new file mode 100644 index 0000000000..8b666bc55a --- /dev/null +++ b/ebtree/ebsttree.c @@ -0,0 +1,42 @@ +/* + * Elastic Binary Trees - exported functions for String data nodes. + * Version 5.0 + * (C) 2002-2009 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* Consult ebsttree.h for more details about those functions */ + +#include "ebsttree.h" + +/* Find the first occurence of a zero-terminated string in the tree . + * It's the caller's reponsibility to use this function only on trees which + * only contain zero-terminated strings. If none can be found, return NULL. + */ +REGPRM2 struct ebmb_node *ebst_lookup(struct eb_root *root, const char *x) +{ + return __ebst_lookup(root, x); +} + +/* Insert ebmb_node into subtree starting at node root . Only + * new->key needs be set with the zero-terminated string key. The ebmb_node is + * returned. If root->b[EB_RGHT]==1, the tree may only contain unique keys. The + * caller is responsible for properly terminating the key with a zero. + */ +REGPRM2 struct ebmb_node *ebst_insert(struct eb_root *root, struct ebmb_node *new) +{ + return __ebst_insert(root, new); +} diff --git a/ebtree/ebsttree.h b/ebtree/ebsttree.h new file mode 100644 index 0000000000..fb34ea9a41 --- /dev/null +++ b/ebtree/ebsttree.h @@ -0,0 +1,258 @@ +/* + * Elastic Binary Trees - macros to manipulate String data nodes. + * Version 5.0 + * (C) 2002-2009 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* These functions and macros rely on Multi-Byte nodes */ + +#include "ebtree.h" +#include "ebmbtree.h" + +/* The following functions are not inlined by default. They are declared + * in ebsttree.c, which simply relies on their inline version. + */ +REGPRM2 struct ebmb_node *ebst_lookup(struct eb_root *root, const char *x); +REGPRM2 struct ebmb_node *ebst_insert(struct eb_root *root, struct ebmb_node *new); + +/* Find the first occurence of a zero-terminated string in the tree . + * It's the caller's reponsibility to use this function only on trees which + * only contain zero-terminated strings. If none can be found, return NULL. + */ +static forceinline struct ebmb_node *__ebst_lookup(struct eb_root *root, const void *x) +{ + struct ebmb_node *node; + eb_troot_t *troot; + unsigned int bit; + + troot = root->b[EB_LEFT]; + if (unlikely(troot == NULL)) + return NULL; + + bit = 0; + while (1) { + if ((eb_gettag(troot) == EB_LEAF)) { + node = container_of(eb_untag(troot, EB_LEAF), + struct ebmb_node, node.branches); + if (strcmp(node->key, x) == 0) + return node; + else + return NULL; + } + node = container_of(eb_untag(troot, EB_NODE), + struct ebmb_node, node.branches); + + if (node->node.bit < 0) { + /* We have a dup tree now. Either it's for the same + * value, and we walk down left, or it's a different + * one and we don't have our key. + */ + if (strcmp(node->key, x) != 0) + return NULL; + + troot = node->node.branches.b[EB_LEFT]; + while (eb_gettag(troot) != EB_LEAF) + troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT]; + node = container_of(eb_untag(troot, EB_LEAF), + struct ebmb_node, node.branches); + return node; + } + + /* OK, normal data node, let's walk down */ + bit = string_equal_bits(x, node->key, bit); + if (bit < node->node.bit) + return NULL; /* no more common bits */ + + troot = node->node.branches.b[(((unsigned char*)x)[node->node.bit >> 3] >> + (~node->node.bit & 7)) & 1]; + } +} + +/* Insert ebmb_node into subtree starting at node root . Only + * new->key needs be set with the zero-terminated string key. The ebmb_node is + * returned. If root->b[EB_RGHT]==1, the tree may only contain unique keys. The + * caller is responsible for properly terminating the key with a zero. + */ +static forceinline struct ebmb_node * +__ebst_insert(struct eb_root *root, struct ebmb_node *new) +{ + struct ebmb_node *old; + unsigned int side; + eb_troot_t *troot; + eb_troot_t *root_right = root; + int diff; + int bit; + + side = EB_LEFT; + troot = root->b[EB_LEFT]; + root_right = root->b[EB_RGHT]; + if (unlikely(troot == NULL)) { + /* Tree is empty, insert the leaf part below the left branch */ + root->b[EB_LEFT] = eb_dotag(&new->node.branches, EB_LEAF); + new->node.leaf_p = eb_dotag(root, EB_LEFT); + new->node.node_p = NULL; /* node part unused */ + return new; + } + + /* The tree descent is fairly easy : + * - first, check if we have reached a leaf node + * - second, check if we have gone too far + * - third, reiterate + * Everywhere, we use for the node node we are inserting, + * for the node we attach it to, and for the node we are + * displacing below . will always point to the future node + * (tagged with its type). carries the side the node is + * attached to below its parent, which is also where previous node + * was attached. + */ + + bit = 0; + while (1) { + if (unlikely(eb_gettag(troot) == EB_LEAF)) { + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_leaf; + + old = container_of(eb_untag(troot, EB_LEAF), + struct ebmb_node, node.branches); + + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_leaf = eb_dotag(&old->node.branches, EB_LEAF); + + new->node.node_p = old->node.leaf_p; + + /* Right here, we have 3 possibilities : + * - the tree does not contain the key, and we have + * new->key < old->key. We insert new above old, on + * the left ; + * + * - the tree does not contain the key, and we have + * new->key > old->key. We insert new above old, on + * the right ; + * + * - the tree does contain the key, which implies it + * is alone. We add the new key next to it as a + * first duplicate. + * + * The last two cases can easily be partially merged. + */ + bit = string_equal_bits(new->key, old->key, bit); + diff = cmp_bits(new->key, old->key, bit); + + if (diff < 0) { + new->node.leaf_p = new_left; + old->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_leaf; + } else { + /* we may refuse to duplicate this key if the tree is + * tagged as containing only unique keys. + */ + if (diff == 0 && eb_gettag(root_right)) + return old; + + /* new->key >= old->key, new goes the right */ + old->node.leaf_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_leaf; + new->node.branches.b[EB_RGHT] = new_leaf; + + if (diff == 0) { + new->node.bit = -1; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + return new; + } + } + break; + } + + /* OK we're walking down this link */ + old = container_of(eb_untag(troot, EB_NODE), + struct ebmb_node, node.branches); + + /* Stop going down when we don't have common bits anymore. We + * also stop in front of a duplicates tree because it means we + * have to insert above. Note: we can compare more bits than + * the current node's because as long as they are identical, we + * know we descend along the correct side. + */ + if (old->node.bit < 0) { + /* we're above a duplicate tree, we must compare till the end */ + bit = string_equal_bits(new->key, old->key, bit); + goto dup_tree; + } + else if (bit < old->node.bit) { + bit = string_equal_bits(new->key, old->key, bit); + } + + if (bit < old->node.bit) { /* we don't have all bits in common */ + /* The tree did not contain the key, so we insert before the node + * , and set ->bit to designate the lowest bit position in + * which applies to ->branches.b[]. + */ + eb_troot_t *new_left, *new_rght; + eb_troot_t *new_leaf, *old_node; + dup_tree: + new_left = eb_dotag(&new->node.branches, EB_LEFT); + new_rght = eb_dotag(&new->node.branches, EB_RGHT); + new_leaf = eb_dotag(&new->node.branches, EB_LEAF); + old_node = eb_dotag(&old->node.branches, EB_NODE); + + new->node.node_p = old->node.node_p; + + diff = cmp_bits(new->key, old->key, bit); + if (diff < 0) { + new->node.leaf_p = new_left; + old->node.node_p = new_rght; + new->node.branches.b[EB_LEFT] = new_leaf; + new->node.branches.b[EB_RGHT] = old_node; + } + else if (diff > 0) { + old->node.node_p = new_left; + new->node.leaf_p = new_rght; + new->node.branches.b[EB_LEFT] = old_node; + new->node.branches.b[EB_RGHT] = new_leaf; + } + else { + struct eb_node *ret; + ret = eb_insert_dup(&old->node, &new->node); + return container_of(ret, struct ebmb_node, node); + } + break; + } + + /* walk down */ + root = &old->node.branches; + side = (new->key[old->node.bit >> 3] >> (~old->node.bit & 7)) & 1; + troot = root->b[side]; + } + + /* Ok, now we are inserting between and . 's + * parent is already set to , and the 's branch is still in + * . Update the root's leaf till we have it. Note that we can also + * find the side by checking the side of new->node.node_p. + */ + + /* We need the common higher bits between new->key and old->key. + * This number of bits is already in . + */ + new->node.bit = bit; + root->b[side] = eb_dotag(&new->node.branches, EB_NODE); + return new; +} + diff --git a/ebtree/ebtree.c b/ebtree/ebtree.c new file mode 100644 index 0000000000..8f45da2d0d --- /dev/null +++ b/ebtree/ebtree.c @@ -0,0 +1,31 @@ +/* + * Elastic Binary Trees - exported generic functions + * (C) 2002-2007 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "ebtree.h" + +void eb_delete(struct eb_node *node) +{ + __eb_delete(node); +} + +/* used by insertion primitives */ +REGPRM1 struct eb_node *eb_insert_dup(struct eb_node *sub, struct eb_node *new) +{ + return __eb_insert_dup(sub, new); +} diff --git a/ebtree/ebtree.h b/ebtree/ebtree.h new file mode 100644 index 0000000000..77813c1e82 --- /dev/null +++ b/ebtree/ebtree.h @@ -0,0 +1,885 @@ +/* + * Elastic Binary Trees - generic macros and structures. + * Version 5.0 + * (C) 2002-2009 - Willy Tarreau + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * + * Short history : + * + * 2007/09/28: full support for the duplicates tree => v3 + * 2007/07/08: merge back cleanups from kernel version. + * 2007/07/01: merge into Linux Kernel (try 1). + * 2007/05/27: version 2: compact everything into one single struct + * 2007/05/18: adapted the structure to support embedded nodes + * 2007/05/13: adapted to mempools v2. + */ + + + +/* + General idea: + ------------- + In a radix binary tree, we may have up to 2N-1 nodes for N keys if all of + them are leaves. If we find a way to differentiate intermediate nodes (later + called "nodes") and final nodes (later called "leaves"), and we associate + them by two, it is possible to build sort of a self-contained radix tree with + intermediate nodes always present. It will not be as cheap as the ultree for + optimal cases as shown below, but the optimal case almost never happens : + + Eg, to store 8, 10, 12, 13, 14 : + + ultree this theorical tree + + 8 8 + / \ / \ + 10 12 10 12 + / \ / \ + 13 14 12 14 + / \ + 12 13 + + Note that on real-world tests (with a scheduler), is was verified that the + case with data on an intermediate node never happens. This is because the + data spectrum is too large for such coincidences to happen. It would require + for instance that a task has its expiration time at an exact second, with + other tasks sharing that second. This is too rare to try to optimize for it. + + What is interesting is that the node will only be added above the leaf when + necessary, which implies that it will always remain somewhere above it. So + both the leaf and the node can share the exact value of the leaf, because + when going down the node, the bit mask will be applied to comparisons. So we + are tempted to have one single key shared between the node and the leaf. + + The bit only serves the nodes, and the dups only serve the leaves. So we can + put a lot of information in common. This results in one single entity with + two branch pointers and two parent pointers, one for the node part, and one + for the leaf part : + + node's leaf's + parent parent + | | + [node] [leaf] + / \ + left right + branch branch + + The node may very well refer to its leaf counterpart in one of its branches, + indicating that its own leaf is just below it : + + node's + parent + | + [node] + / \ + left [leaf] + branch + + Adding keys in such a tree simply consists in inserting nodes between + other nodes and/or leaves : + + [root] + | + [node2] + / \ + [leaf1] [node3] + / \ + [leaf2] [leaf3] + + On this diagram, we notice that [node2] and [leaf2] have been pulled away + from each other due to the insertion of [node3], just as if there would be + an elastic between both parts. This elastic-like behaviour gave its name to + the tree : "Elastic Binary Tree", or "EBtree". The entity which associates a + node part and a leaf part will be called an "EB node". + + We also notice on the diagram that there is a root entity required to attach + the tree. It only contains two branches and there is nothing above it. This + is an "EB root". Some will note that [leaf1] has no [node1]. One property of + the EBtree is that all nodes have their branches filled, and that if a node + has only one branch, it does not need to exist. Here, [leaf1] was added + below [root] and did not need any node. + + An EB node contains : + - a pointer to the node's parent (node_p) + - a pointer to the leaf's parent (leaf_p) + - two branches pointing to lower nodes or leaves (branches) + - a bit position (bit) + - an optional key. + + The key here is optional because it's used only during insertion, in order + to classify the nodes. Nothing else in the tree structure requires knowledge + of the key. This makes it possible to write type-agnostic primitives for + everything, and type-specific insertion primitives. This has led to consider + two types of EB nodes. The type-agnostic ones will serve as a header for the + other ones, and will simply be called "struct eb_node". The other ones will + have their type indicated in the structure name. Eg: "struct eb32_node" for + nodes carrying 32 bit keys. + + We will also node that the two branches in a node serve exactly the same + purpose as an EB root. For this reason, a "struct eb_root" will be used as + well inside the struct eb_node. In order to ease pointer manipulation and + ROOT detection when walking upwards, all the pointers inside an eb_node will + point to the eb_root part of the referenced EB nodes, relying on the same + principle as the linked lists in Linux. + + Another important point to note, is that when walking inside a tree, it is + very convenient to know where a node is attached in its parent, and what + type of branch it has below it (leaf or node). In order to simplify the + operations and to speed up the processing, it was decided in this specific + implementation to use the lowest bit from the pointer to designate the side + of the upper pointers (left/right) and the type of a branch (leaf/node). + This practise is not mandatory by design, but an implementation-specific + optimisation permitted on all platforms on which data must be aligned. All + known 32 bit platforms align their integers and pointers to 32 bits, leaving + the two lower bits unused. So, we say that the pointers are "tagged". And + since they designate pointers to root parts, we simply call them + "tagged root pointers", or "eb_troot" in the code. + + Duplicate keys are stored in a special manner. When inserting a key, if + the same one is found, then an incremental binary tree is built at this + place from these keys. This ensures that no special case has to be written + to handle duplicates when walking through the tree or when deleting entries. + It also guarantees that duplicates will be walked in the exact same order + they were inserted. This is very important when trying to achieve fair + processing distribution for instance. + + Algorithmic complexity can be derived from 3 variables : + - the number of possible different keys in the tree : P + - the number of entries in the tree : N + - the number of duplicates for one key : D + + Note that this tree is deliberately NOT balanced. For this reason, the worst + case may happen with a small tree (eg: 32 distinct keys of one bit). BUT, + the operations required to manage such data are so much cheap that they make + it worth using it even under such conditions. For instance, a balanced tree + may require only 6 levels to store those 32 keys when this tree will + require 32. But if per-level operations are 5 times cheaper, it wins. + + Minimal, Maximal and Average times are specified in number of operations. + Minimal is given for best condition, Maximal for worst condition, and the + average is reported for a tree containing random keys. An operation + generally consists in jumping from one node to the other. + + Complexity : + - lookup : min=1, max=log(P), avg=log(N) + - insertion from root : min=1, max=log(P), avg=log(N) + - insertion of dups : min=1, max=log(D), avg=log(D)/2 after lookup + - deletion : min=1, max=1, avg=1 + - prev/next : min=1, max=log(P), avg=2 : + N/2 nodes need 1 hop => 1*N/2 + N/4 nodes need 2 hops => 2*N/4 + N/8 nodes need 3 hops => 3*N/8 + ... + N/x nodes need log(x) hops => log2(x)*N/x + Total cost for all N nodes : sum[i=1..N](log2(i)*N/i) = N*sum[i=1..N](log2(i)/i) + Average cost across N nodes = total / N = sum[i=1..N](log2(i)/i) = 2 + + This design is currently limited to only two branches per node. Most of the + tree descent algorithm would be compatible with more branches (eg: 4, to cut + the height in half), but this would probably require more complex operations + and the deletion algorithm would be problematic. + + Useful properties : + - a node is always added above the leaf it is tied to, and never can get + below nor in another branch. This implies that leaves directly attached + to the root do not use their node part, which is indicated by a NULL + value in node_p. This also enhances the cache efficiency when walking + down the tree, because when the leaf is reached, its node part will + already have been visited (unless it's the first leaf in the tree). + + - pointers to lower nodes or leaves are stored in "branch" pointers. Only + the root node may have a NULL in either branch, it is not possible for + other branches. Since the nodes are attached to the left branch of the + root, it is not possible to see a NULL left branch when walking up a + tree. Thus, an empty tree is immediately identified by a NULL left + branch at the root. Conversely, the one and only way to identify the + root node is to check that it right branch is NULL. Note that the + NULL pointer may have a few low-order bits set. + + - a node connected to its own leaf will have branch[0|1] pointing to + itself, and leaf_p pointing to itself. + + - a node can never have node_p pointing to itself. + + - a node is linked in a tree if and only if it has a non-null leaf_p. + + - a node can never have both branches equal, except for the root which can + have them both NULL. + + - deletion only applies to leaves. When a leaf is deleted, its parent must + be released too (unless it's the root), and its sibling must attach to + the grand-parent, replacing the parent. Also, when a leaf is deleted, + the node tied to this leaf will be removed and must be released too. If + this node is different from the leaf's parent, the freshly released + leaf's parent will be used to replace the node which must go. A released + node will never be used anymore, so there's no point in tracking it. + + - the bit index in a node indicates the bit position in the key which is + represented by the branches. That means that a node with (bit == 0) is + just above two leaves. Negative bit values are used to build a duplicate + tree. The first node above two identical leaves gets (bit == -1). This + value logarithmically decreases as the duplicate tree grows. During + duplicate insertion, a node is inserted above the highest bit value (the + lowest absolute value) in the tree during the right-sided walk. If bit + -1 is not encountered (highest < -1), we insert above last leaf. + Otherwise, we insert above the node with the highest value which was not + equal to the one of its parent + 1. + + - the "eb_next" primitive walks from left to right, which means from lower + to higher keys. It returns duplicates in the order they were inserted. + The "eb_first" primitive returns the left-most entry. + + - the "eb_prev" primitive walks from right to left, which means from + higher to lower keys. It returns duplicates in the opposite order they + were inserted. The "eb_last" primitive returns the right-most entry. + + - a tree which has 1 in the lower bit of its root's right branch is a + tree with unique nodes. This means that when a node is inserted with + a key which already exists will not be inserted, and the previous + entry will be returned. + + */ + +#ifndef _EBTREE_H +#define _EBTREE_H + +#include + +/* Note: we never need to run fls on null keys, so we can optimize the fls + * function by removing a conditional jump. + */ +#if defined(__i386__) +static inline int flsnz(int x) +{ + int r; + __asm__("bsrl %1,%0\n" + : "=r" (r) : "rm" (x)); + return r+1; +} +#else +// returns 1 to 32 for 1<<0 to 1<<31. Undefined for 0. +#define flsnz(___a) ({ \ + register int ___x, ___bits = 0; \ + ___x = (___a); \ + if (___x & 0xffff0000) { ___x &= 0xffff0000; ___bits += 16;} \ + if (___x & 0xff00ff00) { ___x &= 0xff00ff00; ___bits += 8;} \ + if (___x & 0xf0f0f0f0) { ___x &= 0xf0f0f0f0; ___bits += 4;} \ + if (___x & 0xcccccccc) { ___x &= 0xcccccccc; ___bits += 2;} \ + if (___x & 0xaaaaaaaa) { ___x &= 0xaaaaaaaa; ___bits += 1;} \ + ___bits + 1; \ + }) +#endif + +static inline int fls64(unsigned long long x) +{ + unsigned int h; + unsigned int bits = 32; + + h = x >> 32; + if (!h) { + h = x; + bits = 0; + } + return flsnz(h) + bits; +} + +#define fls_auto(x) ((sizeof(x) > 4) ? fls64(x) : flsnz(x)) + +/* Linux-like "container_of". It returns a pointer to the structure of type + * which has its member stored at address . + */ +#ifndef container_of +#define container_of(ptr, type, name) ((type *)(((void *)(ptr)) - ((long)&((type *)0)->name))) +#endif + +/* + * Gcc >= 3 provides the ability for the program to give hints to the compiler + * about what branch of an if is most likely to be taken. This helps the + * compiler produce the most compact critical paths, which is generally better + * for the cache and to reduce the number of jumps. Be very careful not to use + * this in inline functions, because the code reordering it causes very often + * has a negative impact on the calling functions. + */ +#if !defined(likely) +#if __GNUC__ < 3 +#define __builtin_expect(x,y) (x) +#define likely(x) (x) +#define unlikely(x) (x) +#elif __GNUC__ < 4 +/* gcc 3.x does the best job at this */ +#define likely(x) (__builtin_expect((x) != 0, 1)) +#define unlikely(x) (__builtin_expect((x) != 0, 0)) +#else +/* GCC 4.x is stupid, it performs the comparison then compares it to 1, + * so we cheat in a dirty way to prevent it from doing this. This will + * only work with ints and booleans though. + */ +#define likely(x) (x) +#define unlikely(x) (__builtin_expect((unsigned long)(x), 0)) +#endif +#endif + +/* By default, gcc does not inline large chunks of code, but we want it to + * respect our choices. + */ +#if !defined(forceinline) +#if __GNUC__ < 3 +#define forceinline inline +#else +#define forceinline inline __attribute__((always_inline)) +#endif +#endif + +/* Support passing function parameters in registers. For this, the + * CONFIG_EBTREE_REGPARM macro has to be set to the maximal number of registers + * allowed. Some functions have intentionally received a regparm lower than + * their parameter count, it is in order to avoid register clobbering where + * they are called. + */ +#ifndef REGPRM1 +#if CONFIG_EBTREE_REGPARM >= 1 +#define REGPRM1 __attribute__((regparm(1))) +#else +#define REGPRM1 +#endif +#endif + +#ifndef REGPRM2 +#if CONFIG_EBTREE_REGPARM >= 2 +#define REGPRM2 __attribute__((regparm(2))) +#else +#define REGPRM2 REGPRM1 +#endif +#endif + +#ifndef REGPRM3 +#if CONFIG_EBTREE_REGPARM >= 3 +#define REGPRM3 __attribute__((regparm(3))) +#else +#define REGPRM3 REGPRM2 +#endif +#endif + +/* Number of bits per node, and number of leaves per node */ +#define EB_NODE_BITS 1 +#define EB_NODE_BRANCHES (1 << EB_NODE_BITS) +#define EB_NODE_BRANCH_MASK (EB_NODE_BRANCHES - 1) + +/* Be careful not to tweak those values. The walking code is optimized for NULL + * detection on the assumption that the following values are intact. + */ +#define EB_LEFT 0 +#define EB_RGHT 1 +#define EB_LEAF 0 +#define EB_NODE 1 + +/* Tags to set in root->b[EB_RGHT] : + * - EB_NORMAL is a normal tree which stores duplicate keys. + * - EB_UNIQUE is a tree which stores unique keys. + */ +#define EB_NORMAL 0 +#define EB_UNIQUE 1 + +/* This is the same as an eb_node pointer, except that the lower bit embeds + * a tag. See eb_dotag()/eb_untag()/eb_gettag(). This tag has two meanings : + * - 0=left, 1=right to designate the parent's branch for leaf_p/node_p + * - 0=link, 1=leaf to designate the branch's type for branch[] + */ +typedef void eb_troot_t; + +/* The eb_root connects the node which contains it, to two nodes below it, one + * of which may be the same node. At the top of the tree, we use an eb_root + * too, which always has its right branch NULL (+/1 low-order bits). + */ +struct eb_root { + eb_troot_t *b[EB_NODE_BRANCHES]; /* left and right branches */ +}; + +/* The eb_node contains the two parts, one for the leaf, which always exists, + * and one for the node, which remains unused in the very first node inserted + * into the tree. This structure is 20 bytes per node on 32-bit machines. Do + * not change the order, benchmarks have shown that it's optimal this way. + */ +struct eb_node { + struct eb_root branches; /* branches, must be at the beginning */ + eb_troot_t *node_p; /* link node's parent */ + eb_troot_t *leaf_p; /* leaf node's parent */ + int bit; /* link's bit position. */ +}; + +/* Return the structure of type whose member points to */ +#define eb_entry(ptr, type, member) container_of(ptr, type, member) + +/* The root of a tree is an eb_root initialized with both pointers NULL. + * During its life, only the left pointer will change. The right one will + * always remain NULL, which is the way we detect it. + */ +#define EB_ROOT \ + (struct eb_root) { \ + .b = {[0] = NULL, [1] = NULL }, \ + } + +#define EB_ROOT_UNIQUE \ + (struct eb_root) { \ + .b = {[0] = NULL, [1] = (void *)1 }, \ + } + +#define EB_TREE_HEAD(name) \ + struct eb_root name = EB_ROOT + + +/***************************************\ + * Private functions. Not for end-user * +\***************************************/ + +/* Converts a root pointer to its equivalent eb_troot_t pointer, + * ready to be stored in ->branch[], leaf_p or node_p. NULL is not + * conserved. To be used with EB_LEAF, EB_NODE, EB_LEFT or EB_RGHT in . + */ +static inline eb_troot_t *eb_dotag(const struct eb_root *root, const int tag) +{ + return (eb_troot_t *)((void *)root + tag); +} + +/* Converts an eb_troot_t pointer pointer to its equivalent eb_root pointer, + * for use with pointers from ->branch[], leaf_p or node_p. NULL is conserved + * as long as the tree is not corrupted. To be used with EB_LEAF, EB_NODE, + * EB_LEFT or EB_RGHT in . + */ +static inline struct eb_root *eb_untag(const eb_troot_t *troot, const int tag) +{ + return (struct eb_root *)((void *)troot - tag); +} + +/* returns the tag associated with an eb_troot_t pointer */ +static inline int eb_gettag(eb_troot_t *troot) +{ + return (unsigned long)troot & 1; +} + +/* Converts a root pointer to its equivalent eb_troot_t pointer and clears the + * tag, no matter what its value was. + */ +static inline struct eb_root *eb_clrtag(const eb_troot_t *troot) +{ + return (struct eb_root *)((unsigned long)troot & ~1UL); +} + +/* Returns a pointer to the eb_node holding */ +static inline struct eb_node *eb_root_to_node(struct eb_root *root) +{ + return container_of(root, struct eb_node, branches); +} + +/* Walks down starting at root pointer , and always walking on side + * . It either returns the node hosting the first leaf on that side, + * or NULL if no leaf is found. may either be NULL or a branch pointer. + * The pointer to the leaf (or NULL) is returned. + */ +static inline struct eb_node *eb_walk_down(eb_troot_t *start, unsigned int side) +{ + /* A NULL pointer on an empty tree root will be returned as-is */ + while (eb_gettag(start) == EB_NODE) + start = (eb_untag(start, EB_NODE))->b[side]; + /* NULL is left untouched (root==eb_node, EB_LEAF==0) */ + return eb_root_to_node(eb_untag(start, EB_LEAF)); +} + +/* This function is used to build a tree of duplicates by adding a new node to + * a subtree of at least 2 entries. It will probably never be needed inlined, + * and it is not for end-user. + */ +static forceinline struct eb_node * +__eb_insert_dup(struct eb_node *sub, struct eb_node *new) +{ + struct eb_node *head = sub; + + struct eb_troot *new_left = eb_dotag(&new->branches, EB_LEFT); + struct eb_troot *new_rght = eb_dotag(&new->branches, EB_RGHT); + struct eb_troot *new_leaf = eb_dotag(&new->branches, EB_LEAF); + + /* first, identify the deepest hole on the right branch */ + while (eb_gettag(head->branches.b[EB_RGHT]) != EB_LEAF) { + struct eb_node *last = head; + head = container_of(eb_untag(head->branches.b[EB_RGHT], EB_NODE), + struct eb_node, branches); + if (head->bit > last->bit + 1) + sub = head; /* there's a hole here */ + } + + /* Here we have a leaf attached to (head)->b[EB_RGHT] */ + if (head->bit < -1) { + /* A hole exists just before the leaf, we insert there */ + new->bit = -1; + sub = container_of(eb_untag(head->branches.b[EB_RGHT], EB_LEAF), + struct eb_node, branches); + head->branches.b[EB_RGHT] = eb_dotag(&new->branches, EB_NODE); + + new->node_p = sub->leaf_p; + new->leaf_p = new_rght; + sub->leaf_p = new_left; + new->branches.b[EB_LEFT] = eb_dotag(&sub->branches, EB_LEAF); + new->branches.b[EB_RGHT] = new_leaf; + return new; + } else { + int side; + /* No hole was found before a leaf. We have to insert above + * . Note that we cannot be certain that is attached + * to the right of its parent, as this is only true if + * is inside the dup tree, not at the head. + */ + new->bit = sub->bit - 1; /* install at the lowest level */ + side = eb_gettag(sub->node_p); + head = container_of(eb_untag(sub->node_p, side), struct eb_node, branches); + head->branches.b[side] = eb_dotag(&new->branches, EB_NODE); + + new->node_p = sub->node_p; + new->leaf_p = new_rght; + sub->node_p = new_left; + new->branches.b[EB_LEFT] = eb_dotag(&sub->branches, EB_NODE); + new->branches.b[EB_RGHT] = new_leaf; + return new; + } +} + + +/**************************************\ + * Public functions, for the end-user * +\**************************************/ + +/* Return the first leaf in the tree starting at , or NULL if none */ +static inline struct eb_node *eb_first(struct eb_root *root) +{ + return eb_walk_down(root->b[0], EB_LEFT); +} + +/* Return the last leaf in the tree starting at , or NULL if none */ +static inline struct eb_node *eb_last(struct eb_root *root) +{ + return eb_walk_down(root->b[0], EB_RGHT); +} + +/* Return previous leaf node before an existing leaf node, or NULL if none. */ +static inline struct eb_node *eb_prev(struct eb_node *node) +{ + eb_troot_t *t = node->leaf_p; + + while (eb_gettag(t) == EB_LEFT) { + /* Walking up from left branch. We must ensure that we never + * walk beyond root. + */ + if (unlikely(eb_clrtag((eb_untag(t, EB_LEFT))->b[EB_RGHT]) == NULL)) + return NULL; + t = (eb_root_to_node(eb_untag(t, EB_LEFT)))->node_p; + } + /* Note that cannot be NULL at this stage */ + t = (eb_untag(t, EB_RGHT))->b[EB_LEFT]; + return eb_walk_down(t, EB_RGHT); +} + +/* Return next leaf node after an existing leaf node, or NULL if none. */ +static inline struct eb_node *eb_next(struct eb_node *node) +{ + eb_troot_t *t = node->leaf_p; + + while (eb_gettag(t) != EB_LEFT) + /* Walking up from right branch, so we cannot be below root */ + t = (eb_root_to_node(eb_untag(t, EB_RGHT)))->node_p; + + /* Note that cannot be NULL at this stage */ + t = (eb_untag(t, EB_LEFT))->b[EB_RGHT]; + if (eb_clrtag(t) == NULL) + return NULL; + return eb_walk_down(t, EB_LEFT); +} + +/* Return previous leaf node before an existing leaf node, skipping duplicates, + * or NULL if none. */ +static inline struct eb_node *eb_prev_unique(struct eb_node *node) +{ + eb_troot_t *t = node->leaf_p; + + while (1) { + if (eb_gettag(t) != EB_LEFT) { + node = eb_root_to_node(eb_untag(t, EB_RGHT)); + /* if we're right and not in duplicates, stop here */ + if (node->bit >= 0) + break; + t = node->node_p; + } + else { + /* Walking up from left branch. We must ensure that we never + * walk beyond root. + */ + if (unlikely(eb_clrtag((eb_untag(t, EB_LEFT))->b[EB_RGHT]) == NULL)) + return NULL; + t = (eb_root_to_node(eb_untag(t, EB_LEFT)))->node_p; + } + } + /* Note that cannot be NULL at this stage */ + t = (eb_untag(t, EB_RGHT))->b[EB_LEFT]; + return eb_walk_down(t, EB_RGHT); +} + +/* Return next leaf node after an existing leaf node, skipping duplicates, or + * NULL if none. + */ +static inline struct eb_node *eb_next_unique(struct eb_node *node) +{ + eb_troot_t *t = node->leaf_p; + + while (1) { + if (eb_gettag(t) == EB_LEFT) { + if (unlikely(eb_clrtag((eb_untag(t, EB_LEFT))->b[EB_RGHT]) == NULL)) + return NULL; /* we reached root */ + node = eb_root_to_node(eb_untag(t, EB_LEFT)); + /* if we're left and not in duplicates, stop here */ + if (node->bit >= 0) + break; + t = node->node_p; + } + else { + /* Walking up from right branch, so we cannot be below root */ + t = (eb_root_to_node(eb_untag(t, EB_RGHT)))->node_p; + } + } + + /* Note that cannot be NULL at this stage */ + t = (eb_untag(t, EB_LEFT))->b[EB_RGHT]; + if (eb_clrtag(t) == NULL) + return NULL; + return eb_walk_down(t, EB_LEFT); +} + + +/* Removes a leaf node from the tree if it was still in it. Marks the node + * as unlinked. + */ +static forceinline void __eb_delete(struct eb_node *node) +{ + __label__ delete_unlink; + unsigned int pside, gpside, sibtype; + struct eb_node *parent; + struct eb_root *gparent; + + if (!node->leaf_p) + return; + + /* we need the parent, our side, and the grand parent */ + pside = eb_gettag(node->leaf_p); + parent = eb_root_to_node(eb_untag(node->leaf_p, pside)); + + /* We likely have to release the parent link, unless it's the root, + * in which case we only set our branch to NULL. Note that we can + * only be attached to the root by its left branch. + */ + + if (eb_clrtag(parent->branches.b[EB_RGHT]) == NULL) { + /* we're just below the root, it's trivial. */ + parent->branches.b[EB_LEFT] = NULL; + goto delete_unlink; + } + + /* To release our parent, we have to identify our sibling, and reparent + * it directly to/from the grand parent. Note that the sibling can + * either be a link or a leaf. + */ + + gpside = eb_gettag(parent->node_p); + gparent = eb_untag(parent->node_p, gpside); + + gparent->b[gpside] = parent->branches.b[!pside]; + sibtype = eb_gettag(gparent->b[gpside]); + + if (sibtype == EB_LEAF) { + eb_root_to_node(eb_untag(gparent->b[gpside], EB_LEAF))->leaf_p = + eb_dotag(gparent, gpside); + } else { + eb_root_to_node(eb_untag(gparent->b[gpside], EB_NODE))->node_p = + eb_dotag(gparent, gpside); + } + /* Mark the parent unused. Note that we do not check if the parent is + * our own node, but that's not a problem because if it is, it will be + * marked unused at the same time, which we'll use below to know we can + * safely remove it. + */ + parent->node_p = NULL; + + /* The parent node has been detached, and is currently unused. It may + * belong to another node, so we cannot remove it that way. Also, our + * own node part might still be used. so we can use this spare node + * to replace ours if needed. + */ + + /* If our link part is unused, we can safely exit now */ + if (!node->node_p) + goto delete_unlink; + + /* From now on, and are necessarily different, and the + * 's node part is in use. By definition, is at least + * below , so keeping its key for the bit string is OK. + */ + + parent->node_p = node->node_p; + parent->branches = node->branches; + parent->bit = node->bit; + + /* We must now update the new node's parent... */ + gpside = eb_gettag(parent->node_p); + gparent = eb_untag(parent->node_p, gpside); + gparent->b[gpside] = eb_dotag(&parent->branches, EB_NODE); + + /* ... and its branches */ + for (pside = 0; pside <= 1; pside++) { + if (eb_gettag(parent->branches.b[pside]) == EB_NODE) { + eb_root_to_node(eb_untag(parent->branches.b[pside], EB_NODE))->node_p = + eb_dotag(&parent->branches, pside); + } else { + eb_root_to_node(eb_untag(parent->branches.b[pside], EB_LEAF))->leaf_p = + eb_dotag(&parent->branches, pside); + } + } + delete_unlink: + /* Now the node has been completely unlinked */ + node->leaf_p = NULL; + return; /* tree is not empty yet */ +} + +/* Compare blocks and byte-to-byte, from bit to bit . + * Return the number of equal bits between strings, assuming that the first + * bits are already identical. It is possible to return slightly more + * than bits if does not stop on a byte boundary and we find exact + * bytes. Note that parts or all of bits may be rechecked. It is only + * passed here as a hint to speed up the check. + */ +static forceinline unsigned int equal_bits(const unsigned char *a, + const unsigned char *b, + unsigned int ignore, unsigned int len) +{ + unsigned int beg; + unsigned int end; + unsigned int ret; + unsigned char c; + + beg = ignore >> 3; + end = (len + 7) >> 3; + ret = end << 3; + + do { + if (beg >= end) + goto out; + beg++; + c = a[beg-1] ^ b[beg-1]; + } while (!c); + + /* OK now we know that a and b differ at byte and that holds + * the bit differences. We have to find what bit is differing and report + * it as the number of identical bits. Note that low bit numbers are + * assigned to high positions in the byte, as we compare them as strings. + */ + ret = beg << 3; + if (c & 0xf0) { c >>= 4; ret -= 4; } + if (c & 0x0c) { c >>= 2; ret -= 2; } + ret -= (c >> 1); + ret--; + out: + return ret; +} + +/* Compare strings and byte-to-byte, from bit to the last 0. + * Return the number of equal bits between strings, assuming that the first + * bits are already identical. Note that parts or all of bits + * may be rechecked. It is only passed here as a hint to speed up the check. + * The caller is responsible for not passing an value larger than any + * of the two strings. However, referencing any bit from the trailing zero is + * permitted. + */ +static forceinline unsigned int string_equal_bits(const unsigned char *a, + const unsigned char *b, + unsigned int ignore) +{ + unsigned int beg; + unsigned char c; + + beg = ignore >> 3; + + /* skip known and identical bits. We stop at the first different byte + * or at the first zero we encounter on either side. + */ + while (1) { + unsigned char d; + + c = a[beg]; + d = b[beg]; + beg++; + + c ^= d; + if (c) + break; + if (!d) + break; + } + + /* OK now we know that a and b differ at byte , or that both are zero. + * We have to find what bit is differing and report it as the number of + * identical bits. Note that low bit numbers are assigned to high positions + * in the byte, as we compare them as strings. + */ + beg <<= 3; + if (c & 0xf0) { c >>= 4; beg -= 4; } + if (c & 0x0c) { c >>= 2; beg -= 2; } + beg -= (c >> 1); + if (c) + beg--; + + return beg; +} + +static forceinline int cmp_bits(const unsigned char *a, const unsigned char *b, unsigned int pos) +{ + unsigned int ofs; + unsigned char bit_a, bit_b; + + ofs = pos >> 3; + pos = ~pos & 7; + + bit_a = (a[ofs] >> pos) & 1; + bit_b = (b[ofs] >> pos) & 1; + + return bit_a - bit_b; /* -1: ab */ +} + +static forceinline int get_bit(const unsigned char *a, unsigned int pos) +{ + unsigned int ofs; + + ofs = pos >> 3; + pos = ~pos & 7; + return (a[ofs] >> pos) & 1; +} + +/* These functions are declared in ebtree.c */ +void eb_delete(struct eb_node *node); +REGPRM1 struct eb_node *eb_insert_dup(struct eb_node *sub, struct eb_node *new); + +#endif /* _EB_TREE_H */ + +/* + * Local variables: + * c-indent-level: 8 + * c-basic-offset: 8 + * End: + */