lib: bits - new fractional log-like helper

For stats gathering, where the data can have a wide range of values, you
don't necessarily need the same granularity along the full range of values.
For example, 1ms and 11ms latencies are very different, but 1.001s and
1.011s latencies are not worth distinguishing. Something logarithmic seems
more apt. Simply looking at power-of-2 sized bands (e.g. doing log2(n)),
however, is too granular, so these new helpers let you specify how fine
to (linearly) subdivide each of those bands. 1 fractional bit splits
each power of 2 band into 2 halves. 2 fractional bits splits each power
of 2 band into 4 quarters, and so on. 0 fractional bits is just log2().

Exact identification of percentiles is impossible, but it was anyway, as you
simply cannot store all the data required to calculate them. However, a mere
896 buckets will permit you to have 32 bands per power of 2, 5 fracional bits.
The above example would have buckets such as 2.432s-2.496s, and 55.3s-56.3s.
Assuming smooth distribution lets you calculate percentiles more accurately,
just assume within each bucket is a trapezial distribution. This holds even
if the distribution is multi-modal, which it will be. However, maths required.

Signed-off-by: Phil Carmody <phil@dovecot.fi>

diff --git a/src/lib/bits.h b/src/lib/bits.h
index 0ff6dd3915031989835a2aa678f7f0f235bec173..cdd39a4058cedd415859118e20b1006d77cdeaba 100644 (file)
--- a/src/lib/bits.h
+++ b/src/lib/bits.h
@@ -29,4 +29,66 @@ unsigned int bits_required64(uint64_t num)
                : 32 + bits_required32(num >> 32);
 }
 
+/* These functions look too big to be inline, but in almost all expected
+   uses, 'fracbits' will be a compile-time constant, and most of the
+   expressions will simplify greatly.
+*/
+
+/* Perform a piecewise-linear approximation to a log2, with fracbits "fractional" bits.
+   Best explained with examples:
+   With 2 fractional bits splitting each power of 2 into 4 bands:
+     00,   01,   10,   11 ->   00,   01,   10,   11 (small corner cases)
+    100,  101,  110,  111 ->  100,  101,  110,  111 ([4-8) split into 4 bands)
+   1000, 1001, 1010, 1011 -> 1000, 1000, 1001, 1001 ([8-15) split ...
+   1100, 1101, 1110, 1111 -> 1010, 1010, 1011, 1011  ... into 4 bands)
+   [16..31) -> 11bb
+   [32..63) -> 100bb
+   [64..127) -> 101bb
+   [128..255) -> 110bb
+   e.g. 236 = 11101100 -> ((8-2)<<2 == 11000) + (111.....>> 5 == 111) - 100 == 11011
+ */
+static inline unsigned int ATTR_CONST
+bits_fraclog(unsigned int val, unsigned int fracbits)
+{
+       unsigned bits = bits_required32(val);
+       if (bits <= fracbits + 1)
+               return val;
+
+       unsigned int bandnum = bits - fracbits;
+       unsigned int bandstart = bandnum << fracbits;
+       unsigned int fracoffsbad = val >> (bandnum - 1); /* has leading 1 still */
+       unsigned int bucket = bandstart + fracoffsbad - BIT(fracbits);
+       return bucket;
+}
+static inline unsigned int ATTR_CONST
+bits_fraclog_bucket_start(unsigned int bucket, unsigned int fracbits)
+{
+       unsigned int bandnum = bucket >> fracbits;
+       if (bandnum <= 1)
+               return bucket;
+       if (fracbits == 0)
+               return BIT(bucket - 1);
+       unsigned int fracoffs = bucket & (BIT(fracbits)-1);
+       unsigned int fracoffs1 = BIT(fracbits) + fracoffs;
+       unsigned int bandstart = fracoffs1 << (bandnum - 1);
+       return bandstart;
+}
+static inline unsigned int ATTR_CONST
+bits_fraclog_bucket_end(unsigned int bucket, unsigned int fracbits)
+{
+       unsigned int bandnum = bucket >> fracbits;
+       if (bandnum <= 1)
+               return bucket;
+       if (fracbits == 0)
+               return BIT(bucket - 1) * 2 - 1;
+       unsigned int fracoffs = bucket & (BIT(fracbits)-1);
+       unsigned int nextfracoffs1 = 1 + BIT(fracbits) + fracoffs;
+       unsigned int nextbandstart = nextfracoffs1 << (bandnum - 1);
+       return nextbandstart - 1;
+}
+/* UNSAFE: multiple use of parameter (but expecting a constant in reality).
+   But a macro as it's most likely to be used to declare an array size.
+*/
+#define BITS_FRACLOG_BUCKETS(bits) ((33u - (bits)) << (bits))
+
 #endif