From: Maria Matejka Date: Tue, 9 Apr 2019 20:51:40 +0000 (+0200) Subject: Trie Index: Growing works X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=29314142e9c9a196bd79aedc2030f0b9a8f5e9bd;p=thirdparty%2Fbird.git Trie Index: Growing works --- diff --git a/lib/idm.c b/lib/idm.c index 94c7f11c4..90d7f712e 100644 --- a/lib/idm.c +++ b/lib/idm.c @@ -40,7 +40,7 @@ idm_alloc(struct idm *m) goto found; /* If we are at least 7/8 full, expand (if we are allowed to) */ - if ((m->used * 32 < m->max) && (m->used > m->size * 28)) + if ((m->used < m->max) && (m->used > m->size * 28)) { m->size *= 2; m->data = mb_realloc(m->data, m->size * sizeof(u32)); diff --git a/lib/tindex.c b/lib/tindex.c index fedbe95ae..f65ad210c 100644 --- a/lib/tindex.c +++ b/lib/tindex.c @@ -17,6 +17,9 @@ #define TI_MIN_UNIT_SIZE 4 #define TI_MIN_ADDRESS_SIZE 6 +#define TDB 32 +#define uTDB u32 + union tindex_data { u32 data4[0]; u16 data6[0]; @@ -29,6 +32,7 @@ struct tindex { u64 *exists; pool *p; struct idm idm; + u16 depth; u8 unit_size; u8 address_size; }; @@ -43,33 +47,34 @@ tindex_new(pool *p) ti->index_data = mb_allocz(p, ti->unit_size * (1 << ti->address_size)); ti->exists = mb_allocz(p, (1 << (ti->address_size - 3))); idm_init(&(ti->idm), p, (1 << (ti->address_size - 5)), (1 << ti->address_size)); - u32 rootnode = idm_alloc(&(ti->idm)); + u64 rootnode = idm_alloc(&(ti->idm)); ASSERT(rootnode == 1); return ti; } static inline u64 -tindex_data(const struct tindex *ti, u64 asize, u64 usize, u64 dsize, u64 dshift, u64 idx, uint *len) +tindex_data(const union tindex_data *id, u64 usize, u64 asize, u64 dsize, u64 dshift, u64 idx, uint *len) { + ASSERT(dsize <= TDB); u64 data; switch (usize) { case 4: - data = ti->index_data->data4[idx] >> dshift; + data = id->data4[idx] >> dshift; break; case 6: data = - ((u64)(ti->index_data->data6[idx * 3] >> asize) << (dshift * 2)) | - ((u64)(ti->index_data->data6[idx * 3 + 1] >> asize) << (dshift)) | - (u64)(ti->index_data->data6[idx * 3 + 2] >> asize); + ((u64)(id->data6[idx * 3] >> asize) << (dshift * 2)) | + ((u64)(id->data6[idx * 3 + 1] >> asize) << (dshift)) | + (u64)(id->data6[idx * 3 + 2] >> asize); break; case 8: - data = ti->index_data->data8[idx] >> dshift; + data = id->data8[idx] >> dshift; break; case 12: data = - ((u64)(ti->index_data->data12[idx * 3] >> asize) << (dshift * 2)) | - ((u64)(ti->index_data->data12[idx * 3 + 1] >> asize) << (dshift)) | - (u64)(ti->index_data->data12[idx * 3 + 2] >> asize); + ((u64)(id->data12[idx * 3] >> asize) << (dshift * 2)) | + ((u64)(id->data12[idx * 3 + 1] >> asize) << (dshift)) | + (u64)(id->data12[idx * 3 + 2] >> asize); break; default: bug("This shall never happen"); @@ -86,128 +91,151 @@ tindex_data(const struct tindex *ti, u64 asize, u64 usize, u64 dsize, u64 dshift } static inline u64 -tindex_left(const struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 addrmask) +tindex_left(const union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 addrmask) { switch (usize) { - case 4: return (ti->index_data->data4[idx] >> (asize * 2)) & addrmask; - case 6: return ti->index_data->data6[idx * 3] & addrmask; - case 8: return (ti->index_data->data8[idx] >> (asize * 2)) & addrmask; - case 12: return ti->index_data->data12[idx * 3] & addrmask; + case 4: return (id->data4[idx] >> (asize * 2)) & addrmask; + case 6: return id->data6[idx * 3] & addrmask; + case 8: return (id->data8[idx] >> (asize * 2)) & addrmask; + case 12: return id->data12[idx * 3] & addrmask; default: bug("This shall never happen"); } } static inline u64 -tindex_right(const struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 addrmask) +tindex_right(const union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 addrmask) { switch (usize) { - case 4: return (ti->index_data->data4[idx] >> (asize)) & addrmask; - case 6: return ti->index_data->data6[idx * 3 + 1] & addrmask; - case 8: return (ti->index_data->data8[idx] >> (asize)) & addrmask; - case 12: return ti->index_data->data12[idx * 3 + 1] & addrmask; + case 4: return (id->data4[idx] >> (asize)) & addrmask; + case 6: return id->data6[idx * 3 + 1] & addrmask; + case 8: return (id->data8[idx] >> (asize)) & addrmask; + case 12: return id->data12[idx * 3 + 1] & addrmask; default: bug("This shall never happen"); } } static inline u64 -tindex_up(const struct tindex *ti, u64 idx, u64 usize, u64 addrmask) +tindex_up(const union tindex_data *id, u64 idx, u64 usize, u64 addrmask) { switch (usize) { - case 4: return ti->index_data->data4[idx] & addrmask; - case 6: return ti->index_data->data6[idx * 3 + 2] & addrmask; - case 8: return ti->index_data->data8[idx] & addrmask; - case 12: return ti->index_data->data12[idx * 3 + 2] & addrmask; + case 4: return id->data4[idx] & addrmask; + case 6: return id->data6[idx * 3 + 2] & addrmask; + case 8: return id->data8[idx] & addrmask; + case 12: return id->data12[idx * 3 + 2] & addrmask; default: bug("This shall never happen"); } } static inline void -tindex_put(struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 dsize, u64 dshift, u64 data, uint dlen, u64 left, u64 right, u64 up) +tindex_put(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 dsize, u64 dshift, u64 data, uint dlen, u64 left, u64 right, u64 up) { const u64 dsmask = (1LL << dshift) - 1; data = u64_var_encode(data, dsize - dlen); switch (usize) { case 4: - ti->index_data->data4[idx] = (data << dshift) | (left << (asize * 2)) | (right << asize) | up; + id->data4[idx] = (data << dshift) | (left << (asize * 2)) | (right << asize) | up; return; case 6: - ti->index_data->data6[idx * 3 ] = left | ((data >> (2 * dshift)) << asize); - ti->index_data->data6[idx * 3 + 1] = right | (((data >> dshift) & dsmask) << asize); - ti->index_data->data6[idx * 3 + 2] = up | ((data & dsmask) << asize); + id->data6[idx * 3 ] = left | ((data >> (2 * dshift)) << asize); + id->data6[idx * 3 + 1] = right | (((data >> dshift) & dsmask) << asize); + id->data6[idx * 3 + 2] = up | ((data & dsmask) << asize); return; case 8: - ti->index_data->data8[idx] = (data << dshift) | (left << (asize * 2)) | (right << asize) | up; + id->data8[idx] = (data << dshift) | (left << (asize * 2)) | (right << asize) | up; return; case 12: - ti->index_data->data12[idx * 3 ] = left | ((data >> (2 * dshift)) << asize); - ti->index_data->data12[idx * 3 + 1] = right | (((data >> dshift) & dsmask) << asize); - ti->index_data->data12[idx * 3 + 2] = up | ((data & dsmask) << asize); + id->data12[idx * 3 ] = left | ((data >> (2 * dshift)) << asize); + id->data12[idx * 3 + 1] = right | (((data >> dshift) & dsmask) << asize); + id->data12[idx * 3 + 2] = up | ((data & dsmask) << asize); return; default: bug("This shall never happen"); } } static inline void -tindex_left_clear(struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 addrmask) +tindex_left_clear(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 addrmask) +{ + switch (usize) { + case 4: id->data4[idx] &= ~(addrmask << (asize * 2)); break; + case 6: id->data6[idx * 3] &= ~addrmask; break; + case 8: id->data8[idx] &= ~(addrmask << (asize * 2)); break; + case 12: id->data12[idx * 3] &= ~addrmask; break; + } +} + +static inline void +tindex_right_clear(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 addrmask) { switch (usize) { - case 4: ti->index_data->data4[idx] &= ~(addrmask << (asize * 2)); break; - case 6: ti->index_data->data6[idx * 3] &= ~addrmask; break; - case 8: ti->index_data->data8[idx] &= ~(addrmask << (asize * 2)); break; - case 12: ti->index_data->data6[idx * 3] &= ~addrmask; break; + case 4: id->data4[idx] &= ~(addrmask << asize); break; + case 6: id->data6[idx * 3 + 1] &= ~addrmask; break; + case 8: id->data8[idx] &= ~(addrmask << asize); break; + case 12: id->data12[idx * 3 + 1] &= ~addrmask; break; } } static inline void -tindex_right_clear(struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 addrmask) +tindex_up_clear(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 addrmask) { switch (usize) { - case 4: ti->index_data->data4[idx] &= ~(addrmask << asize); break; - case 6: ti->index_data->data6[idx * 3 + 1] &= ~addrmask; break; - case 8: ti->index_data->data8[idx] &= ~(addrmask << asize); break; - case 12: ti->index_data->data6[idx * 3 + 1] &= ~addrmask; break; + case 4: id->data4[idx] &= ~addrmask; break; + case 6: id->data6[idx * 3 + 2] &= ~addrmask; break; + case 8: id->data8[idx] &= ~addrmask; break; + case 12: id->data12[idx * 3 + 2] &= ~addrmask; break; } } static inline void -tindex_left_set(struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 nidx) +tindex_left_set(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 nidx) { /* The left child must have been zero before */ switch (usize) { - case 4: ti->index_data->data4[idx] |= nidx << (asize * 2); break; - case 6: ti->index_data->data6[idx * 3] |= nidx; break; - case 8: ti->index_data->data8[idx] |= nidx << (asize * 2); break; - case 12: ti->index_data->data6[idx * 3] |= nidx; break; + case 4: id->data4[idx] |= nidx << (asize * 2); break; + case 6: id->data6[idx * 3] |= nidx; break; + case 8: id->data8[idx] |= nidx << (asize * 2); break; + case 12: id->data12[idx * 3] |= nidx; break; } } static inline void -tindex_right_set(struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 nidx) +tindex_right_set(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 nidx) { /* The right child must have been zero before */ switch (usize) { - case 4: ti->index_data->data4[idx] |= nidx << asize; break; - case 6: ti->index_data->data6[idx * 3 + 1] |= nidx; break; - case 8: ti->index_data->data8[idx] |= nidx << asize; break; - case 12: ti->index_data->data6[idx * 3 + 1] |= nidx; break; + case 4: id->data4[idx] |= nidx << asize; break; + case 6: id->data6[idx * 3 + 1] |= nidx; break; + case 8: id->data8[idx] |= nidx << asize; break; + case 12: id->data12[idx * 3 + 1] |= nidx; break; + } +} + +static inline void +tindex_up_set(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 nidx) +{ + /* The parent must have been zero before */ + switch (usize) { + case 4: id->data4[idx] |= nidx; break; + case 6: id->data6[idx * 3 + 2] |= nidx; break; + case 8: id->data8[idx] |= nidx; break; + case 12: id->data12[idx * 3 + 2] |= nidx; break; } } static inline void -tindex_child_update(struct tindex *ti, u64 idx, u64 usize, u64 asize, u64 addrmask, u64 oidx, u64 nidx) +tindex_child_update(union tindex_data *id, u64 idx, u64 usize, u64 asize, u64 addrmask, u64 oidx, u64 nidx) { - if (oidx == tindex_left(ti, idx, usize, asize, addrmask)) { - tindex_left_clear(ti, idx, usize, asize, addrmask); - tindex_left_set(ti, idx, usize, asize, nidx); + if (oidx == tindex_left(id, idx, usize, asize, addrmask)) { + tindex_left_clear(id, idx, usize, asize, addrmask); + tindex_left_set(id, idx, usize, asize, nidx); } else { - ASSERT(oidx == tindex_right(ti, idx, usize, asize, addrmask)); - tindex_right_clear(ti, idx, usize, asize, addrmask); - tindex_right_set(ti, idx, usize, asize, nidx); + ASSERT(oidx == tindex_right(id, idx, usize, asize, addrmask)); + tindex_right_clear(id, idx, usize, asize, addrmask); + tindex_right_set(id, idx, usize, asize, nidx); } } -static inline uint tindex_input_bits(const u64 *bits_in, const uint blen, uint *bpos, const uint dlen, u64 *bits) { +static inline uint tindex_input_bits(const uTDB *bits_in, const uint blen, uint *bpos, const uint dlen, u64 *bits) { uint bmax = blen - *bpos; /* How much remains in the input */ uint ilen = MIN(bmax, dlen); /* How much we really take */ @@ -216,23 +244,23 @@ static inline uint tindex_input_bits(const u64 *bits_in, const uint blen, uint * return 0; } - ASSERT(ilen <= 64); /* The limit of output bit count is 64 */ + ASSERT(ilen <= TDB); /* The limit of output bit count is TDB */ uint bend = *bpos + ilen - 1; /* The last bit, inclusive (!) */ /* Crop the bits at the end */ - *bits = (bits_in[bend / 64] >> (63 - (bend % 64))); + *bits = (bits_in[bend / TDB] >> (TDB - 1 - (bend % TDB))); /* Prepend bits from the previous item if the range goes over */ - if (bend / 64 > *bpos / 64) - *bits |= bits_in[*bpos / 64] << (1 + bend % 64); + if (bend / TDB > *bpos / TDB) + *bits |= bits_in[*bpos / TDB] << (1 + bend % TDB); else - ASSERT(bend / 64 == *bpos / 64); + ASSERT(bend / TDB == *bpos / TDB); /* Advance the bit pointer */ *bpos += ilen; /* Return the wanted bits */ - *bits &= ((1 << ilen) - 1); + *bits &= ((1ULL << ilen) - 1); return ilen; } @@ -264,6 +292,8 @@ _tindex_dump(const struct tindex *ti, u64 idx, uint depth, uint bit) const uint usize = ti->unit_size; const uint dsize = usize * 8 - asize * 3; + union tindex_data *idata = ti->index_data; + const uint dshift = (usize % 3) ? (asize * 3) : (dsize / 3); const u64 addrmask = (1ULL << ti->address_size) - 1; @@ -277,18 +307,18 @@ _tindex_dump(const struct tindex *ti, u64 idx, uint depth, uint bit) } uint dlen; - u64 data = tindex_data(ti, asize, usize, dsize, dshift, idx, &dlen); + u64 data = tindex_data(idata, usize, asize, dsize, dshift, idx, &dlen); if (depth && bit) data |= 1ULL << dlen; if (depth) dlen++; debug("%s0x%x/%u (%lu %c)\n", INDENT, data, dlen, idx, tindex_exists(ti, idx) ? '*' : ' '); - u64 left = tindex_left(ti, idx, usize, asize, addrmask); + u64 left = tindex_left(idata, idx, usize, asize, addrmask); if (left) _tindex_dump(ti, left, depth+1, 0); - u64 right = tindex_right(ti, idx, usize, asize, addrmask); + u64 right = tindex_right(idata, idx, usize, asize, addrmask); if (right) _tindex_dump(ti, right, depth+1, 1); } @@ -296,16 +326,206 @@ _tindex_dump(const struct tindex *ti, u64 idx, uint depth, uint bit) void tindex_dump(const struct tindex *ti) { + debug("Trie index; usize = %u, asize = %u, dsize = %u, depth = %u\n", + ti->unit_size, ti->address_size, ti->unit_size * 8 - ti->address_size * 3, + ti->depth); _tindex_dump(ti, 1, 0, 0); } +void +tindex_do_grow(struct tindex *ti, const uint nasize, const uint nusize) +{ + const uint asize = ti->address_size; + const uint usize = ti->unit_size; + const uint dsize = usize * 8 - asize * 3; + + const uint dshift = (usize % 3) ? (asize * 3) : (dsize / 3); + const u64 addrmask = (1ULL << ti->address_size) - 1; + + ti->unit_size = nusize; + ti->address_size = nasize; + + union tindex_data *odata = ti->index_data; + ti->index_data = mb_allocz(ti->p, nusize * (1 << nasize)); + + u64 *oexists = ti->exists; + ti->exists = mb_allocz(ti->p, (1 << (nasize - 3))); + memcpy(ti->exists, oexists, 1 << (asize - 3)); + mb_free(oexists); + + ti->idm.max = 1 << nasize; + + uint bpos = 0; + uTDB *bits = alloca(((ti->depth / TDB) + 1)*sizeof(uTDB)); + memset(bits, 0, ((ti->depth / TDB) + 1)*sizeof(uTDB)); + + void migrate(u64 idx) { + uint dlen; + u64 data = tindex_data(odata, usize, asize, dsize, dshift, idx, &dlen); + u64 mask = (1 << dlen) - 1; + uint sbpos = bpos; + if (dlen) { + uint bend = bpos + dlen - 1; + + if (bend / TDB > bpos / TDB) { + bits[bpos / TDB] &= ~(mask >> (1 + bend % TDB)); + bits[bpos / TDB] |= data >> (1 + bend % TDB); + } + + bits[bend / TDB] &= ~(mask << (TDB - 1 - (bend % TDB))); + bits[bend / TDB] |= data << (TDB - 1 - (bend % TDB)); + + bpos = bend + 1; + } + + /* Migration of non-root nodes */ + if (idx > 1) + if (tindex_exists(ti, idx)) + tindex_find(ti, bits, bpos, idx); + else + idm_free(&(ti->idm), idx); + + u64 left = tindex_left(odata, idx, usize, asize, addrmask); + if (left) { + bits[bpos / TDB] &= ~(1ULL << (TDB - 1 - (bpos % TDB))); + bpos++; + migrate(left); + bpos--; + } + + u64 right = tindex_right(odata, idx, usize, asize, addrmask); + if (right) { + bits[bpos / TDB] |= 1ULL << (TDB - 1 - (bpos % TDB)); + bpos++; + migrate(right); + bpos--; + } + + bpos = sbpos; + } + + migrate(1); + mb_free(odata); +} + +void +tindex_grow(struct tindex *ti) +{ + /* We want bigger index space so we have to change parameters + * of the tindex and completely rebuild it. Then we'll free the + * old index_data. + * + * Assigned indices are kept, internal nodes may be rearranged + * and renumbered. + */ + + const uint asize = ti->address_size; + const uint usize = ti->unit_size; + const uint dsize = usize * 8 - asize * 3; + const union tindex_data *idata = ti->index_data; + + const uint dshift = (usize % 3) ? (asize * 3) : (dsize / 3); + const u64 addrmask = (1ULL << ti->address_size) - 1; + + if (dsize > 3) { + /* First we'll try to estimate whether it is feasible to shorten + * the data part while getting more space for the indices */ + + u64 needsplit = 0; + u64 total = 0; + + void lencnt(u64 idx) { + uint dlen; + tindex_data(idata, usize, asize, dsize, dshift, idx, &dlen); + ASSERT(dlen < dsize); + if (dlen >= dsize - 3) + needsplit++; + total++; + + u64 left = tindex_left(idata, idx, usize, asize, addrmask); + if (left) + lencnt(left); + + u64 right = tindex_right(idata, idx, usize, asize, addrmask); + if (right) + lencnt(right); + } + + lencnt(1); + + /* After shortening the data part, needsplit/total nodes will duplicate (or triplicate!). + * If the overall index usage goes up by at most 20% by doing this change, + * we consider it feasible. By math: + * + * ((float)(needsplit / total)) * ((int)(dsize / (dsize - 3)) + 1) < 0.2 + * needsplit * ((dsize / (dsize - 3)) + 1) < 0.2 * total + * 5 * needsplit * ((dsize / (dsize - 3)) + 1) < total + */ + + if (5 * needsplit * ((dsize / (dsize - 3)) + 1) < total) + return tindex_do_grow(ti, asize + 1, usize); + } + + switch (usize) { +#define UP_ASIZE(usize) (1+MAX((((usize-4)*8)/3),asize)) + case 4: return tindex_do_grow(ti, UP_ASIZE(6), 6); + case 6: return tindex_do_grow(ti, UP_ASIZE(8), 8); + case 8: return tindex_do_grow(ti, UP_ASIZE(12), 12); + case 12: bug("Not implemented yet."); + default: bug("This shall not happen."); + } +} + +static inline void +tindex_renumber(union tindex_data *idata, u64 usize, u64 asize, u64 dsize, u64 dshift, u64 addrmask, u64 oidx, u64 nidx) +{ + u64 up = tindex_up(idata, oidx, usize, addrmask); + u64 left = tindex_left(idata, oidx, usize, asize, addrmask); + u64 right = tindex_right(idata, oidx, usize, asize, addrmask); + + if (up) + tindex_child_update(idata, up, usize, asize, addrmask, oidx, nidx); + + if (left) { + tindex_up_clear(idata, left, usize, asize, addrmask); + tindex_up_set(idata, left, usize, asize, nidx); + } + + if (right) { + tindex_up_clear(idata, right, usize, asize, addrmask); + tindex_up_set(idata, right, usize, asize, nidx); + } + + switch (usize) { + case 4: idata->data4[nidx] = idata->data4[oidx]; + break; + case 6: memcpy(&(idata->data6[nidx * 3]), &(idata->data6[oidx * 3]), 3*sizeof(idata->data6[0])); + break; + case 8: idata->data8[nidx] = idata->data8[oidx]; + break; + case 12: memcpy(&(idata->data12[nidx * 3]), &(idata->data12[oidx * 3]), 3*sizeof(idata->data12[0])); + break; + default: bug("This shall never happen"); + } +} + +#define TINDEX_ALLOC_IDX ({ u64 out = idm_alloc(&(ti->idm)); if (!out) goto noidx; out; }) + u64 -tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int create) +tindex_find(struct tindex *ti, const uTDB *bits_in, const uint blen, const u64 create) { + if (blen > ti->depth) + if (create) + ti->depth = blen; + else + return 0; + const uint asize = ti->address_size; const uint usize = ti->unit_size; const uint dsize = usize * 8 - asize * 3; + union tindex_data *idata = ti->index_data; + const uint dshift = (usize % 3) ? (asize * 3) : (dsize / 3); const u64 addrmask = (1ULL << ti->address_size) - 1; @@ -326,7 +546,7 @@ tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int cr while (1) { /* Get data from trie */ uint dlen; - u64 data = tindex_data(ti, asize, usize, dsize, dshift, idx, &dlen); + u64 data = tindex_data(idata, usize, asize, dsize, dshift, idx, &dlen); /* Get data from input */ u64 bits; @@ -346,13 +566,20 @@ tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int cr /* No more bits, we're done */ if (!ilen) { - /* Existence bits fiddling */ - if (create) + if (create == TINDEX_CREATE) { + /* Creating at any index -> do it */ tindex_exists_set(ti, idx); - else if (!tindex_exists(ti, idx)) + return idx; + } else if (create) { + /* Migration from old version -> renumber */ + tindex_renumber(idata, usize, asize, dsize, dshift, addrmask, idx, create); + idm_free(&(ti->idm), idx); + return create; + } else if (tindex_exists(ti, idx)) + /* Shan't create but it already exists */ + return idx; + else return 0; - - return idx; } /* Just one bit, to be sure */ @@ -360,7 +587,7 @@ tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int cr ASSERT(ilen == 1); /* Go left or right? */ - u64 nidx = bits ? tindex_right(ti, idx, usize, asize, addrmask) : tindex_left(ti, idx, usize, asize, addrmask); + u64 nidx = bits ? tindex_right(idata, idx, usize, asize, addrmask) : tindex_left(idata, idx, usize, asize, addrmask); /* There is a path, we'll follow it. */ if (nidx) { @@ -374,13 +601,13 @@ tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int cr return 0; /* So there will be a new node on path. */ - nidx = idm_alloc(&(ti->idm)); + nidx = TINDEX_ALLOC_IDX; /* Left or right? */ if (bits) - tindex_right_set(ti, idx, usize, asize, nidx); + tindex_right_set(idata, idx, usize, asize, nidx); else - tindex_left_set(ti, idx, usize, asize, nidx); + tindex_left_set(idata, idx, usize, asize, nidx); /* Go there. */ uidx = idx; @@ -416,31 +643,41 @@ tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int cr data &= (1 << dlen) - 1; /* Allocate the splitting index */ - u64 midx = idm_alloc(&(ti->idm)); + u64 midx = TINDEX_ALLOC_IDX; /* Allocate the new node if it shall exist */ - u64 nidx = split ? idm_alloc(&(ti->idm)) : 0; + u64 nidx = split ? TINDEX_ALLOC_IDX : 0; /* Relink idx -> midx in the parent node */ if (uidx) - tindex_child_update(ti, uidx, usize, asize, addrmask, idx, midx); + tindex_child_update(idata, uidx, usize, asize, addrmask, idx, midx); /* Setup the splitting index (midx) */ - tindex_put(ti, midx, usize, asize, dsize, dshift, common, comlen, dataright ? nidx : idx, dataright ? idx : nidx, uidx); + tindex_put(idata, midx, usize, asize, dsize, dshift, common, comlen, dataright ? nidx : idx, dataright ? idx : nidx, uidx); /* Update the existing index (idx) */ - tindex_put(ti, idx, usize, asize, dsize, dshift, data, dlen, tindex_left(ti, idx, usize, asize, addrmask), tindex_right(ti, idx, usize, asize, addrmask), midx); + tindex_put(idata, idx, usize, asize, dsize, dshift, data, dlen, tindex_left(idata, idx, usize, asize, addrmask), tindex_right(idata, idx, usize, asize, addrmask), midx); - /* Go down to the child */ - uidx = idx; - idx = nidx; + if (split) { + /* The new parent is the splitting node */ + uidx = midx; - /* Grow there a branch if it has to be grown, otherwise return */ - if (split) + /* The current node is the newly allocated */ + idx = nidx; + + /* Grow there a branch */ break; - else { + + } else if (create == TINDEX_CREATE) { + /* This internal node exists */ tindex_exists_set(ti, midx); return midx; + + } else { + /* This internal node must be renumbered to the right one */ + tindex_renumber(idata, usize, asize, dsize, dshift, addrmask, midx, create); + idm_free(&(ti->idm), midx); + return create; } } @@ -455,22 +692,43 @@ tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int cr /* End of input data */ if ((ilen < dsize - 1) || !tindex_input_bits(bits_in, blen, &bpos, 1, &dataright)) { - tindex_put(ti, idx, usize, asize, dsize, dshift, data, ilen, 0, 0, uidx); - tindex_exists_set(ti, idx); - return idx; + tindex_put(idata, idx, usize, asize, dsize, dshift, data, ilen, 0, 0, uidx); + if (create == TINDEX_CREATE) { + tindex_exists_set(ti, idx); + return idx; + } else { + tindex_renumber(idata, usize, asize, dsize, dshift, addrmask, idx, create); + return create; + } } /* Just one bit. */ ASSERT(dataright < 2); /* Create a new node */ - uint nidx = idm_alloc(&(ti->idm)); + uint nidx = TINDEX_ALLOC_IDX; /* Link it into the trie */ - tindex_put(ti, idx, usize, asize, dsize, dshift, data, ilen, dataright ? 0 : nidx, dataright ? nidx : 0, uidx); + tindex_put(idata, idx, usize, asize, dsize, dshift, data, ilen, dataright ? 0 : nidx, dataright ? nidx : 0, uidx); /* And continue there */ uidx = idx; idx = nidx; } + + /* This statement should be unreachable */ + ASSERT(0); + + /* No index available for alloc */ +noidx: + /* This may happen only directly while adding. + * It should never hapṕen when growing. + * */ + ASSERT(create == TINDEX_CREATE); + + /* Grow the tindex */ + tindex_grow(ti); + + /* And retry */ + return tindex_find(ti, bits_in, blen, create); } diff --git a/lib/tindex.h b/lib/tindex.h index bdc31ba52..cb35e6720 100644 --- a/lib/tindex.h +++ b/lib/tindex.h @@ -17,15 +17,30 @@ */ struct tindex* tindex_new(pool *p); +#define TINDEX_CREATE (~(0ULL)) +#define TINDEX_FIND 0 + /** - * Find an index by the auxiliary funcction @tib. - * @t: the index to look into - * @tib: the auxiliary function; see before - * @create: 0 to find only existing records, 1 to create new - * Return value: 0 for not found (create == 0) or retry (create == 1); nonzero = the index + * Find an index + * @ti: the tindex to look into + * @bits_in: data + * @blen: number of bits to extract from bits_in. + * If @blen is not multiple of 64, the LSB's of the last u64 are ignored. + * @create: TINDEX_FIND to find existing, TINDEX_CREATE to create new records, + * every other value is for internal use + * + * Return value: 0 for not found; nonzero = the index */ -u64 tindex_find(struct tindex *ti, const u64 *bits_in, const uint blen, const int create); +u64 tindex_find(struct tindex *ti, const u32 *bits_in, const uint blen, const u64 create); + +/** + * Delete an index. + * @ti: the tindex to use + * @idx: the index to delete + **/ + +u64 tindex_delete(struct tindex *ti, const u64 idx); /** * Dump the index. Useful for debugging. diff --git a/lib/tindex_test.c b/lib/tindex_test.c index 4698b8a37..c916480d2 100644 --- a/lib/tindex_test.c +++ b/lib/tindex_test.c @@ -18,11 +18,12 @@ struct test_trie { }; static inline void test_trie_add(struct test_trie *tt, u64 data) { - u64 idx = tindex_find(tt->ti, &data, 64, 1); + u32 dtb[2] = { data >> 32, data }; + u64 idx = tindex_find(tt->ti, dtb, 64, TINDEX_CREATE); bt_assert(idx > 0); u64 nlen = tt->len; - while (idx > nlen) + while (idx >= nlen) nlen *= 2; if (nlen > tt->len) { @@ -34,15 +35,29 @@ static inline void test_trie_add(struct test_trie *tt, u64 data) { tt->data[idx]++; } -static inline u64 test_trie_get(struct test_trie *tt, u64 data) { - u64 idx = tindex_find(tt->ti, &data, 64, 0); - if (!idx) return 0; - return tt->data[idx]; +static inline void test_trie_get(struct test_trie *tt, u64 data, u64 cnt) { + u64 out = 0; + u32 dtb[2] = { data >> 32, data }; + u64 idx = tindex_find(tt->ti, dtb, 64, TINDEX_FIND); + if (idx) out = tt->data[idx]; + bt_assert_msg(out == cnt, "Index %lu shall be in trie %lu times, is %lu times.", data, cnt, out); } +/* +static inline void test_trie_remove(struct test_trie *tt, u64 data) { + u64 idx = tindex_find(tt->ti, &data, 64, TINDEX_FIND); + ASSERT(idx); + ASSERT(tt->data[idx]); + if (!--tt->data[idx]) + tindex_delete(tt->ti, idx); +} +*/ + static int t_simple(void) { + const u64 mul = 0xf906f046b1fd4863ULL; + const u64 add = 0xb3a35ec46d09489bULL; pool *p = rp_new(&root_pool, "tindex test"); struct test_trie tt = { .ti = tindex_new(p), @@ -51,14 +66,23 @@ t_simple(void) }; bt_assert(tt.ti); - for (u64 i = 0; i < 20; i++) { + for (u64 i = 0; i < (1<<16); i++) { bt_debug("Trie add: %lu\n", i); test_trie_add(&tt, i); - tindex_dump(tt.ti); + test_trie_add(&tt, i * mul + add); } + for (u64 i = 0; i < (1<<16); i++) { + test_trie_get(&tt, i, 1); + test_trie_get(&tt, i * mul + add, 1); + } + + /* for (u64 i = 0; i < 20; i++) - bt_assert_msg(test_trie_get(&tt, i) == 1, "Index %lu shall be in trie", i); + test_trie_remove(&tt, i); + */ + + tindex_dump(tt.ti); return 1; }