improve rrd_graph performance for large (>1000) number of *DEF variables by using...

diff --git a/src/rrd_graph.c b/src/rrd_graph.c
index 9a4b217862d1de1338795e7aeccb329e7fc5b3b5..5547aa6d0f19b5d9e331b7ba9cb83887bcc9ff4c 100644 (file)
--- a/src/rrd_graph.c
+++ b/src/rrd_graph.c
@@ -335,6 +335,13 @@ int im_free(
     if (im->daemon_addr != NULL)
       free(im->daemon_addr);
 
+    if (im->gdef_map){
+        g_hash_table_destroy(im->gdef_map);        
+    }
+    if (im->rrd_map){
+        g_hash_table_destroy(im->rrd_map);        
+    }
+
     for (i = 0; i < (unsigned) im->gdes_c; i++) {
         if (im->gdes[i].data_first) {
             /* careful here, because a single pointer can occur several times */
@@ -362,26 +369,27 @@ int im_free(
     if (im->font_options)
         cairo_font_options_destroy(im->font_options);
 
+    if (im->surface)
+        cairo_surface_destroy(im->surface);
+
     if (im->cr) {
         status = cairo_status(im->cr);
         cairo_destroy(im->cr);
     }
 
+    if (status)
+        fprintf(stderr, "OOPS: Cairo has issues it can't even die: %s\n",
+                cairo_status_to_string(status));
+
 
     if (im->rendered_image) {
         free(im->rendered_image);
     }
 
     if (im->layout) {
-        g_object_unref (im->layout);
+        g_object_unref(im->layout);
     }
 
-    if (im->surface)
-        cairo_surface_destroy(im->surface);
-
-    if (status)
-        fprintf(stderr, "OOPS: Cairo has issues it can't even die: %s\n",
-                cairo_status_to_string(status));
 
     return 0;
 }
@@ -840,40 +848,27 @@ int data_fetch(
 {
     int       i, ii;
     int       skip;
-
     /* pull the data from the rrd files ... */
     for (i = 0; i < (int) im->gdes_c; i++) {
         /* only GF_DEF elements fetch data */
         if (im->gdes[i].gf != GF_DEF)
             continue;
 
-        skip = 0;
         /* do we have it already ? */
-        for (ii = 0; ii < i; ii++) {
-            if (im->gdes[ii].gf != GF_DEF)
-                continue;
-            if ((strcmp(im->gdes[i].rrd, im->gdes[ii].rrd) == 0)
-                && (im->gdes[i].cf == im->gdes[ii].cf)
-                && (im->gdes[i].cf_reduce == im->gdes[ii].cf_reduce)
-                && (im->gdes[i].start_orig == im->gdes[ii].start_orig)
-                && (im->gdes[i].end_orig == im->gdes[ii].end_orig)
-                && (im->gdes[i].step_orig == im->gdes[ii].step_orig)) {
-                /* OK, the data is already there.
-                 ** Just copy the header portion
-                 */
-                im->gdes[i].start = im->gdes[ii].start;
-                im->gdes[i].end = im->gdes[ii].end;
-                im->gdes[i].step = im->gdes[ii].step;
-                im->gdes[i].ds_cnt = im->gdes[ii].ds_cnt;
-                im->gdes[i].ds_namv = im->gdes[ii].ds_namv;
-                im->gdes[i].data = im->gdes[ii].data;
-                im->gdes[i].data_first = 0;
-                skip = 1;
-            }
-            if (skip)
-                break;
-        }
-        if (!skip) {
+        gpointer value;
+        char *key = gdes_fetch_key(im->gdes[i]);
+        gboolean ok = g_hash_table_lookup_extended(im->rrd_map,key,NULL,&value);
+        free(key);
+        if (ok){
+            ii = GPOINTER_TO_INT(value);
+            im->gdes[i].start = im->gdes[ii].start;
+            im->gdes[i].end = im->gdes[ii].end;
+            im->gdes[i].step = im->gdes[ii].step;
+            im->gdes[i].ds_cnt = im->gdes[ii].ds_cnt;
+            im->gdes[i].ds_namv = im->gdes[ii].ds_namv;
+            im->gdes[i].data = im->gdes[ii].data;
+            im->gdes[i].data_first = 0;
+        } else {
             unsigned long ft_step = im->gdes[i].step;   /* ft_step will record what we got from fetch */
             const char *rrd_daemon;
             int status;
@@ -958,29 +953,6 @@ int data_fetch(
  * CDEF stuff
  *************************************************************/
 
-long find_var_wrapper(
-    void *arg1,
-    char *key)
-{
-    return find_var((image_desc_t *) arg1, key);
-}
-
-/* find gdes containing var*/
-long find_var(
-    image_desc_t *im,
-    char *key)
-{
-    long      ii;
-
-    for (ii = 0; ii < im->gdes_c - 1; ii++) {
-        if ((im->gdes[ii].gf == GF_DEF
-             || im->gdes[ii].gf == GF_VDEF || im->gdes[ii].gf == GF_CDEF)
-            && (strcmp(im->gdes[ii].vname, key) == 0)) {
-            return ii;
-        }
-    }
-    return -1;
-}
 
 /* find the greatest common divisor for all the numbers
    in the 0 terminated num array */
@@ -4228,6 +4200,8 @@ void rrd_graph_init(
 #ifdef HAVE_TZSET
     tzset();
 #endif
+    im->gdef_map = g_hash_table_new_full(g_str_hash, g_str_equal,g_free,NULL);
+    im->rrd_map = g_hash_table_new_full(g_str_hash, g_str_equal,g_free,NULL);
     im->graph_type = GTYPE_TIME;
     im->base = 1000;
     im->daemon_addr = NULL;

diff --git a/src/rrd_graph.h b/src/rrd_graph.h
index ca039abfffa10d94899f3f3722481f454705d56a..e96f957d09a10a825b82c2daed9bb1c3ab601756 100644 (file)
--- a/src/rrd_graph.h
+++ b/src/rrd_graph.h
@@ -26,6 +26,9 @@
 #include "rrd_tool.h"
 #include "rrd_rpncalc.h"
 
+#include <glib.h>
+
+
 #ifdef WIN32
 #  include <windows.h>
 #  define MAXPATH MAX_PATH
@@ -46,6 +49,8 @@
 #define FULL_SIZE_MODE     0x200    /* -width and -height indicate the total size of the image */
 #define NO_RRDTOOL_TAG 0x400  /* disable the rrdtool tag */
 
+#define gdes_fetch_key(x)  sprintf_alloc("%s:%d:%d:%d:%d",x.rrd,x.cf,x.cf_reduce,x.start_orig,x.end_orig,x.step_orig)
+
 enum tmt_en { TMT_SECOND = 0, TMT_MINUTE, TMT_HOUR, TMT_DAY,
     TMT_WEEK, TMT_MONTH, TMT_YEAR
 };
@@ -331,6 +336,8 @@ typedef struct image_desc_t {
     PangoLayout *layout; /* the pango layout we use for writing fonts */
     rrd_info_t *grinfo; /* root pointer to extra graph info */
     rrd_info_t *grinfo_current; /* pointing to current entry */
+    GHashTable* gdef_map;  /* a map of all *def gdef entries for quick access */
+    GHashTable* rrd_map;  /* a map of all rrd files in use for gdef entries */
 } image_desc_t;
 
 /* Prototypes */
@@ -375,12 +382,6 @@ void      reduce_data(
     rrd_value_t **);
 int       data_fetch(
     image_desc_t *);
-long      find_var(
-    image_desc_t *,
-    char *);
-long      find_var_wrapper(
-    void *arg1,
-    char *key);
 long      lcd(
     long *);
 int       data_calc(

diff --git a/src/rrd_graph_helper.c b/src/rrd_graph_helper.c
index 8232ce84ba35caebf6425fe242a705c33b6ffdd9..2a9091a540d79cf71ab4d4f6c7b813a36ad49a1e 100644 (file)
--- a/src/rrd_graph_helper.c
+++ b/src/rrd_graph_helper.c
@@ -353,6 +353,45 @@ int parse_color( const char *const string, struct gfx_color_t *c)
 #define PARSE_VNAMERPN         (PARSE_POSITIONAL|PARSE_VNAMEDEF|PARSE_RPN)
 #define PARSE_VNAMEREFPOS      (PARSE_POSITIONAL|PARSE_VNAMEREF)
 
+GHashTable* gdef_map;
+
+/* find gdes containing var*/
+static long find_var(
+    image_desc_t *im,
+    char *key)
+{
+    /* this makes only sense for a sufficient number of items */
+    long match = -1;
+    if ( im->gdes_c > 12 ) {
+        gpointer value;
+        gboolean ok = g_hash_table_lookup_extended(im->gdef_map,key,NULL,&value);
+        if (ok){
+            match = GPOINTER_TO_INT(value);
+        }
+    }
+    else {
+        long      ii;
+        for (ii = 0; ii < im->gdes_c - 1; ii++) {
+            if ((im->gdes[ii].gf == GF_DEF
+                || im->gdes[ii].gf == GF_VDEF || im->gdes[ii].gf == GF_CDEF)
+                && (strcmp(im->gdes[ii].vname, key) == 0)) {
+                match = ii;
+                break;
+            }
+        }
+    }
+    return match;
+
+}
+
+static long find_var_wrapper(
+    void *arg1,
+    char *key)
+{
+    return find_var((image_desc_t *) arg1, key);
+}
+
+
 graph_desc_t* newGraphDescription(image_desc_t *const,enum gf_en,parsedargs_t*,unsigned long);
 graph_desc_t* newGraphDescription(image_desc_t *const im,enum gf_en gf,parsedargs_t* pa,unsigned long bits) {
   /* check that none of the othe bitfield marker is set */
@@ -693,7 +732,7 @@ graph_desc_t* newGraphDescription(image_desc_t *const im,enum gf_en gf,parsedarg
   }
   
   /* and assign it */
-  if (vname) { strncpy(gdp->vname,vname,MAX_VNAME_LEN + 1); }
+  if (vname) { strncpy(gdp->vname,vname,MAX_VNAME_LEN + 1);}
   if (rrd) { strncpy(gdp->rrd,rrd,1024); }
   if (ds) { strncpy(gdp->ds_nam,ds,DS_NAM_SIZE); }
   if (cf) { 
@@ -733,6 +772,14 @@ graph_desc_t* newGraphDescription(image_desc_t *const im,enum gf_en gf,parsedarg
       gdp->yrule=val;
     }
   }
+  /* remember the index for faster varfind */
+  char *key = gdes_fetch_key((*gdp));
+  if (gdp->gf == GF_DEF && !g_hash_table_lookup_extended(im->rrd_map,key,NULL,NULL)){
+      g_hash_table_insert(im->gdef_map,g_strdup(key),GINT_TO_POINTER(im->gdes_c-1));
+  } 
+  free(key);
+  g_hash_table_insert(im->gdef_map,g_strdup(gdp->vname),GINT_TO_POINTER(im->gdes_c-1));
+// g_hash_table_insert(im->gdef_map,gdp->vname,GINT_TO_POINTER(im->gdes_c-1));
   /* and return it */
   return gdp;
 }

diff --git a/src/rrd_rpncalc.c b/src/rrd_rpncalc.c
index 1e76ee5d803f88e2cca688712e24529252b3fe0c..491a8215514f8aad1a26844f100b318d7bd438b2 100644 (file)
--- a/src/rrd_rpncalc.c
+++ b/src/rrd_rpncalc.c
@@ -422,7 +422,7 @@ void rpnstack_init(
 {
     rpnstack->s = NULL;
     rpnstack->dc_stacksize = 0;
-    rpnstack->dc_stackblock = 100;
+    rpnstack->dc_stackblock = 1000;
 }
 
 void rpnstack_free(
@@ -465,7 +465,7 @@ short rpn_calc(
     int output_idx)
 {
     int       rpi;
-    long      stptr = -1;
+    long      stptr = -1;    
 
     /* process each op from the rpn in turn */
     for (rpi = 0; rpnp[rpi].op != OP_END; rpi++) {