IMPORT: eb32/64: optimize insert for modern CPUs

Similar to previous patches, let's improve the insert() descent loop to
avoid discovering mandatory data too late. The change here is even
simpler than previous ones, a prefetch was installed and troot is
calculated before last instruction in a speculative way. This was enough
to gain +50% insertion rate on random data.

This is ebtree commit e893f8cc4d44b10f406b9d1d78bd4a9bd9183ccf.

diff --git a/include/import/eb32tree.h b/include/import/eb32tree.h
index 4c7a3a6494ba6464a5ee39801ff382f4a0e54e48..292a10e3940d4b5650ab49ba3e6eef18e6c591d8 100644 (file)
--- a/include/import/eb32tree.h
+++ b/include/import/eb32tree.h
@@ -270,6 +270,10 @@ __eb32_insert(struct eb_root *root, struct eb32_node *new) {
                /* OK we're walking down this link */
                old = container_of(eb_untag(troot, EB_NODE),
                                    struct eb32_node, node.branches);
+
+               __builtin_prefetch(old->node.branches.b[0], 0);
+               __builtin_prefetch(old->node.branches.b[1], 0);
+
                old_node_bit = old->node.bit;
 
                /* Stop going down when we don't have common bits anymore. We
@@ -289,9 +293,9 @@ __eb32_insert(struct eb_root *root, struct eb32_node *new) {
                }
 
                /* walk down */
-               root = &old->node.branches;
                side = (newkey >> old_node_bit) & EB_NODE_BRANCH_MASK;
-               troot = root->b[side];
+               troot = side ? old->node.branches.b[1] : old->node.branches.b[0];
+               root = &old->node.branches;
        }
 
        new_left = eb_dotag(&new->node.branches, EB_LEFT);
@@ -403,6 +407,10 @@ __eb32i_insert(struct eb_root *root, struct eb32_node *new) {
                /* OK we're walking down this link */
                old = container_of(eb_untag(troot, EB_NODE),
                                    struct eb32_node, node.branches);
+
+               __builtin_prefetch(old->node.branches.b[0], 0);
+               __builtin_prefetch(old->node.branches.b[1], 0);
+
                old_node_bit = old->node.bit;
 
                /* Stop going down when we don't have common bits anymore. We
@@ -422,9 +430,9 @@ __eb32i_insert(struct eb_root *root, struct eb32_node *new) {
                }
 
                /* walk down */
-               root = &old->node.branches;
                side = (newkey >> old_node_bit) & EB_NODE_BRANCH_MASK;
-               troot = root->b[side];
+               troot = side ? old->node.branches.b[1] : old->node.branches.b[0];
+               root = &old->node.branches;
        }
 
        new_left = eb_dotag(&new->node.branches, EB_LEFT);

diff --git a/include/import/eb64tree.h b/include/import/eb64tree.h
index 7e3a0f0f30639ac13ab7d530a929dc2416cb58f0..5de45dd6c2d6de703edff49318be3ee639e226d5 100644 (file)
--- a/include/import/eb64tree.h
+++ b/include/import/eb64tree.h
@@ -316,6 +316,10 @@ __eb64_insert(struct eb_root *root, struct eb64_node *new) {
                /* OK we're walking down this link */
                old = container_of(eb_untag(troot, EB_NODE),
                                    struct eb64_node, node.branches);
+
+               __builtin_prefetch(old->node.branches.b[0], 0);
+               __builtin_prefetch(old->node.branches.b[1], 0);
+
                old_node_bit = old->node.bit;
 
                /* Stop going down when we don't have common bits anymore. We
@@ -360,7 +364,6 @@ __eb64_insert(struct eb_root *root, struct eb64_node *new) {
                }
 
                /* walk down */
-               root = &old->node.branches;
 
                if (sizeof(long) >= 8) {
                        side = newkey >> old_node_bit;
@@ -376,7 +379,8 @@ __eb64_insert(struct eb_root *root, struct eb64_node *new) {
                        }
                }
                side &= EB_NODE_BRANCH_MASK;
-               troot = root->b[side];
+               troot = side ? old->node.branches.b[1] : old->node.branches.b[0];
+               root = &old->node.branches;
        }
 
        /* Ok, now we are inserting <new> between <root> and <old>. <old>'s
@@ -498,6 +502,10 @@ __eb64i_insert(struct eb_root *root, struct eb64_node *new) {
                /* OK we're walking down this link */
                old = container_of(eb_untag(troot, EB_NODE),
                                    struct eb64_node, node.branches);
+
+               __builtin_prefetch(old->node.branches.b[0], 0);
+               __builtin_prefetch(old->node.branches.b[1], 0);
+
                old_node_bit = old->node.bit;
 
                /* Stop going down when we don't have common bits anymore. We
@@ -542,7 +550,6 @@ __eb64i_insert(struct eb_root *root, struct eb64_node *new) {
                }
 
                /* walk down */
-               root = &old->node.branches;
 
                if (sizeof(long) >= 8) {
                        side = newkey >> old_node_bit;
@@ -558,7 +565,8 @@ __eb64i_insert(struct eb_root *root, struct eb64_node *new) {
                        }
                }
                side &= EB_NODE_BRANCH_MASK;
-               troot = root->b[side];
+               troot = side ? old->node.branches.b[1] : old->node.branches.b[0];
+               root = &old->node.branches;
        }
 
        /* Ok, now we are inserting <new> between <root> and <old>. <old>'s