Display Memoize planner estimates in EXPLAIN

There've been a few complaints that it can be overly difficult to figure
out why the planner picked a Memoize plan.  To help address that, here we
adjust the EXPLAIN output to display the following additional details:

1) The estimated number of cache entries that can be stored at once
2) The estimated number of unique lookup keys that we expect to see
3) The number of lookups we expect
4) The estimated hit ratio

Technically #4 can be calculated using #1, #2 and #3, but it's not a
particularly obvious calculation, so we opt to display it explicitly.
The original patch by Lukas Fittl only displayed the hit ratio, but
there was a fear that might lead to more questions about how that was
calculated.  The idea with displaying all 4 is to be transparent which
may allow queries to be tuned more easily.  For example, if #2 isn't
correct then maybe extended statistics or a manual n_distinct estimate can
be used to help fix poor plan choices.

Author: Ilia Evdokimov <ilya.evdokimov@tantorlabs.com>
Author: Lukas Fittl <lukas@fittl.com>
Reviewed-by: David Rowley <dgrowleyml@gmail.com>
Reviewed-by: Andrei Lepikhov <lepihov@gmail.com>
Reviewed-by: Robert Haas <robertmhaas@gmail.com>
Discussion: https://postgr.es/m/CAP53Pky29GWAVVk3oBgKBDqhND0BRBN6yTPeguV_qSivFL5N_g%40mail.gmail.com

diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 7e2792ead715b8c1b7a54bb17101127e9ec5b884..8345bc0264b23264f011420a50f4c34e0a0905ce 100644 (file)
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -3582,6 +3582,7 @@ static void
 show_memoize_info(MemoizeState *mstate, List *ancestors, ExplainState *es)
 {
        Plan       *plan = ((PlanState *) mstate)->plan;
+       Memoize    *mplan = (Memoize *) plan;
        ListCell   *lc;
        List       *context;
        StringInfoData keystr;
@@ -3602,7 +3603,7 @@ show_memoize_info(MemoizeState *mstate, List *ancestors, ExplainState *es)
                                                                           plan,
                                                                           ancestors);
 
-       foreach(lc, ((Memoize *) plan)->param_exprs)
+       foreach(lc, mplan->param_exprs)
        {
                Node       *expr = (Node *) lfirst(lc);
 
@@ -3618,6 +3619,24 @@ show_memoize_info(MemoizeState *mstate, List *ancestors, ExplainState *es)
 
        pfree(keystr.data);
 
+       if (es->costs)
+       {
+               if (es->format == EXPLAIN_FORMAT_TEXT)
+               {
+                       ExplainIndentText(es);
+                       appendStringInfo(es->str, "Estimates: capacity=%u distinct keys=%.0f lookups=%.0f hit percent=%.2f%%\n",
+                                                        mplan->est_entries, mplan->est_unique_keys,
+                                                        mplan->est_calls, mplan->est_hit_ratio * 100.0);
+               }
+               else
+               {
+                       ExplainPropertyUInteger("Estimated Capacity", NULL, mplan->est_entries, es);
+                       ExplainPropertyFloat("Estimated Distinct Lookup Keys", NULL, mplan->est_unique_keys, 0, es);
+                       ExplainPropertyFloat("Estimated Lookups", NULL, mplan->est_calls, 0, es);
+                       ExplainPropertyFloat("Estimated Hit Percent", NULL, mplan->est_hit_ratio * 100.0, 2, es);
+               }
+       }
+
        if (!es->analyze)
                return;
 

diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 1f04a2c182ca993d78e3175d6ab9e112318bf5a3..344a3188317b165c8acd5724d0f51cc89a89b545 100644 (file)
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -2572,13 +2572,13 @@ cost_memoize_rescan(PlannerInfo *root, MemoizePath *mpath,
        Cost            input_startup_cost = mpath->subpath->startup_cost;
        Cost            input_total_cost = mpath->subpath->total_cost;
        double          tuples = mpath->subpath->rows;
-       double          calls = mpath->calls;
+       Cardinality est_calls = mpath->est_calls;
        int                     width = mpath->subpath->pathtarget->width;
 
        double          hash_mem_bytes;
        double          est_entry_bytes;
-       double          est_cache_entries;
-       double          ndistinct;
+       Cardinality est_cache_entries;
+       Cardinality ndistinct;
        double          evict_ratio;
        double          hit_ratio;
        Cost            startup_cost;
@@ -2604,7 +2604,7 @@ cost_memoize_rescan(PlannerInfo *root, MemoizePath *mpath,
        est_cache_entries = floor(hash_mem_bytes / est_entry_bytes);
 
        /* estimate on the distinct number of parameter values */
-       ndistinct = estimate_num_groups(root, mpath->param_exprs, calls, NULL,
+       ndistinct = estimate_num_groups(root, mpath->param_exprs, est_calls, NULL,
                                                                        &estinfo);
 
        /*
@@ -2616,7 +2616,10 @@ cost_memoize_rescan(PlannerInfo *root, MemoizePath *mpath,
         * certainly mean a MemoizePath will never survive add_path().
         */
        if ((estinfo.flags & SELFLAG_USED_DEFAULT) != 0)
-               ndistinct = calls;
+               ndistinct = est_calls;
+
+       /* Remember the ndistinct estimate for EXPLAIN */
+       mpath->est_unique_keys = ndistinct;
 
        /*
         * Since we've already estimated the maximum number of entries we can
@@ -2644,9 +2647,12 @@ cost_memoize_rescan(PlannerInfo *root, MemoizePath *mpath,
         * must look at how many scans are estimated in total for this node and
         * how many of those scans we expect to get a cache hit.
         */
-       hit_ratio = ((calls - ndistinct) / calls) *
+       hit_ratio = ((est_calls - ndistinct) / est_calls) *
                (est_cache_entries / Max(ndistinct, est_cache_entries));
 
+       /* Remember the hit ratio estimate for EXPLAIN */
+       mpath->est_hit_ratio = hit_ratio;
+
        Assert(hit_ratio >= 0 && hit_ratio <= 1.0);
 
        /*

diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c
index 8a9f1d7a943a8bd0c34080f9e5708c6026eec006..bfefc7dbea106003cc1711544db5262a722471bd 100644 (file)
--- a/src/backend/optimizer/plan/createplan.c
+++ b/src/backend/optimizer/plan/createplan.c
@@ -284,7 +284,10 @@ static Material *make_material(Plan *lefttree);
 static Memoize *make_memoize(Plan *lefttree, Oid *hashoperators,
                                                         Oid *collations, List *param_exprs,
                                                         bool singlerow, bool binary_mode,
-                                                        uint32 est_entries, Bitmapset *keyparamids);
+                                                        uint32 est_entries, Bitmapset *keyparamids,
+                                                        Cardinality est_calls,
+                                                        Cardinality est_unique_keys,
+                                                        double est_hit_ratio);
 static WindowAgg *make_windowagg(List *tlist, WindowClause *wc,
                                                                 int partNumCols, AttrNumber *partColIdx, Oid *partOperators, Oid *partCollations,
                                                                 int ordNumCols, AttrNumber *ordColIdx, Oid *ordOperators, Oid *ordCollations,
@@ -1753,7 +1756,8 @@ create_memoize_plan(PlannerInfo *root, MemoizePath *best_path, int flags)
 
        plan = make_memoize(subplan, operators, collations, param_exprs,
                                                best_path->singlerow, best_path->binary_mode,
-                                               best_path->est_entries, keyparamids);
+                                               best_path->est_entries, keyparamids, best_path->est_calls,
+                                               best_path->est_unique_keys, best_path->est_hit_ratio);
 
        copy_generic_path_info(&plan->plan, (Path *) best_path);
 
@@ -6749,7 +6753,9 @@ materialize_finished_plan(Plan *subplan)
 static Memoize *
 make_memoize(Plan *lefttree, Oid *hashoperators, Oid *collations,
                         List *param_exprs, bool singlerow, bool binary_mode,
-                        uint32 est_entries, Bitmapset *keyparamids)
+                        uint32 est_entries, Bitmapset *keyparamids,
+                        Cardinality est_calls, Cardinality est_unique_keys,
+                        double est_hit_ratio)
 {
        Memoize    *node = makeNode(Memoize);
        Plan       *plan = &node->plan;
@@ -6767,6 +6773,9 @@ make_memoize(Plan *lefttree, Oid *hashoperators, Oid *collations,
        node->binary_mode = binary_mode;
        node->est_entries = est_entries;
        node->keyparamids = keyparamids;
+       node->est_calls = est_calls;
+       node->est_unique_keys = est_unique_keys;
+       node->est_hit_ratio = est_hit_ratio;
 
        return node;
 }

diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c
index 9cc602788eaae54e6c8d20276a32b04f892a6fb1..a4c5867cdcb8477b1f33f044a2349f3a578a098f 100644 (file)
--- a/src/backend/optimizer/util/pathnode.c
+++ b/src/backend/optimizer/util/pathnode.c
@@ -1689,7 +1689,7 @@ create_material_path(RelOptInfo *rel, Path *subpath)
 MemoizePath *
 create_memoize_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath,
                                        List *param_exprs, List *hash_operators,
-                                       bool singlerow, bool binary_mode, double calls)
+                                       bool singlerow, bool binary_mode, Cardinality est_calls)
 {
        MemoizePath *pathnode = makeNode(MemoizePath);
 
@@ -1710,7 +1710,6 @@ create_memoize_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath,
        pathnode->param_exprs = param_exprs;
        pathnode->singlerow = singlerow;
        pathnode->binary_mode = binary_mode;
-       pathnode->calls = clamp_row_est(calls);
 
        /*
         * For now we set est_entries to 0.  cost_memoize_rescan() does all the
@@ -1720,6 +1719,12 @@ create_memoize_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath,
         */
        pathnode->est_entries = 0;
 
+       pathnode->est_calls = clamp_row_est(est_calls);
+
+       /* These will also be set later in cost_memoize_rescan() */
+       pathnode->est_unique_keys = 0.0;
+       pathnode->est_hit_ratio = 0.0;
+
        /* we should not generate this path type when enable_memoize=false */
        Assert(enable_memoize);
        pathnode->path.disabled_nodes = subpath->disabled_nodes;
@@ -4259,7 +4264,7 @@ reparameterize_path(PlannerInfo *root, Path *path,
                                                                                                        mpath->hash_operators,
                                                                                                        mpath->singlerow,
                                                                                                        mpath->binary_mode,
-                                                                                                       mpath->calls);
+                                                                                                       mpath->est_calls);
                        }
                default:
                        break;

diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h
index e5dd15098f63523e7761b95e9b16ce53adf9b0ad..ad2726f026f7dff5a17c16f19a9eaf12d99a0ecb 100644 (file)
--- a/src/include/nodes/pathnodes.h
+++ b/src/include/nodes/pathnodes.h
@@ -2133,10 +2133,12 @@ typedef struct MemoizePath
                                                                 * complete after caching the first record. */
        bool            binary_mode;    /* true when cache key should be compared bit
                                                                 * by bit, false when using hash equality ops */
-       Cardinality calls;                      /* expected number of rescans */
        uint32          est_entries;    /* The maximum number of entries that the
                                                                 * planner expects will fit in the cache, or 0
                                                                 * if unknown */
+       Cardinality est_calls;          /* expected number of rescans */
+       Cardinality est_unique_keys;    /* estimated unique keys, for EXPLAIN */
+       double          est_hit_ratio;  /* estimated cache hit ratio, for EXPLAIN */
 } MemoizePath;
 
 /*

diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h
index 46e2e09ea35be190cc0a8d41af9e419f4131b016..6d8e1e99db3bd4ddf3b52ca5e7c515ea722dfb6a 100644 (file)
--- a/src/include/nodes/plannodes.h
+++ b/src/include/nodes/plannodes.h
@@ -1073,6 +1073,16 @@ typedef struct Memoize
 
        /* paramids from param_exprs */
        Bitmapset  *keyparamids;
+
+       /* Estimated number of rescans, for EXPLAIN */
+       Cardinality est_calls;
+
+       /* Estimated number of distinct lookup keys, for EXPLAIN */
+       Cardinality est_unique_keys;
+
+       /* Estimated cache hit ratio, for EXPLAIN */
+       double          est_hit_ratio;
+
 } Memoize;
 
 /* ----------------

diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h
index 60dcdb77e41be59a26e6ed979c297f86cde18b5d..58936e963cb6bd9250018e6d701fca1f6b041841 100644 (file)
--- a/src/include/optimizer/pathnode.h
+++ b/src/include/optimizer/pathnode.h
@@ -90,7 +90,7 @@ extern MemoizePath *create_memoize_path(PlannerInfo *root,
                                                                                List *hash_operators,
                                                                                bool singlerow,
                                                                                bool binary_mode,
-                                                                               double calls);
+                                                                               Cardinality est_calls);
 extern UniquePath *create_unique_path(PlannerInfo *root, RelOptInfo *rel,
                                                                          Path *subpath, SpecialJoinInfo *sjinfo);
 extern GatherPath *create_gather_path(PlannerInfo *root,