From: Tobias Oetiker Date: Thu, 28 Feb 2013 14:08:52 +0000 (+0100) Subject: improve rrd_graph performance for large (>1000) number of *DEF variables by using... X-Git-Tag: v1.5.0-rc1~194 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=37092fc25543d0d020fd59448c3e876d7e4f8e64;p=thirdparty%2Frrdtool-1.x.git improve rrd_graph performance for large (>1000) number of *DEF variables by using hash_table lookups instead of linear search --- diff --git a/src/rrd_graph.c b/src/rrd_graph.c index 9a4b2178..5547aa6d 100644 --- a/src/rrd_graph.c +++ b/src/rrd_graph.c @@ -335,6 +335,13 @@ int im_free( if (im->daemon_addr != NULL) free(im->daemon_addr); + if (im->gdef_map){ + g_hash_table_destroy(im->gdef_map); + } + if (im->rrd_map){ + g_hash_table_destroy(im->rrd_map); + } + for (i = 0; i < (unsigned) im->gdes_c; i++) { if (im->gdes[i].data_first) { /* careful here, because a single pointer can occur several times */ @@ -362,26 +369,27 @@ int im_free( if (im->font_options) cairo_font_options_destroy(im->font_options); + if (im->surface) + cairo_surface_destroy(im->surface); + if (im->cr) { status = cairo_status(im->cr); cairo_destroy(im->cr); } + if (status) + fprintf(stderr, "OOPS: Cairo has issues it can't even die: %s\n", + cairo_status_to_string(status)); + if (im->rendered_image) { free(im->rendered_image); } if (im->layout) { - g_object_unref (im->layout); + g_object_unref(im->layout); } - if (im->surface) - cairo_surface_destroy(im->surface); - - if (status) - fprintf(stderr, "OOPS: Cairo has issues it can't even die: %s\n", - cairo_status_to_string(status)); return 0; } @@ -840,40 +848,27 @@ int data_fetch( { int i, ii; int skip; - /* pull the data from the rrd files ... */ for (i = 0; i < (int) im->gdes_c; i++) { /* only GF_DEF elements fetch data */ if (im->gdes[i].gf != GF_DEF) continue; - skip = 0; /* do we have it already ? */ - for (ii = 0; ii < i; ii++) { - if (im->gdes[ii].gf != GF_DEF) - continue; - if ((strcmp(im->gdes[i].rrd, im->gdes[ii].rrd) == 0) - && (im->gdes[i].cf == im->gdes[ii].cf) - && (im->gdes[i].cf_reduce == im->gdes[ii].cf_reduce) - && (im->gdes[i].start_orig == im->gdes[ii].start_orig) - && (im->gdes[i].end_orig == im->gdes[ii].end_orig) - && (im->gdes[i].step_orig == im->gdes[ii].step_orig)) { - /* OK, the data is already there. - ** Just copy the header portion - */ - im->gdes[i].start = im->gdes[ii].start; - im->gdes[i].end = im->gdes[ii].end; - im->gdes[i].step = im->gdes[ii].step; - im->gdes[i].ds_cnt = im->gdes[ii].ds_cnt; - im->gdes[i].ds_namv = im->gdes[ii].ds_namv; - im->gdes[i].data = im->gdes[ii].data; - im->gdes[i].data_first = 0; - skip = 1; - } - if (skip) - break; - } - if (!skip) { + gpointer value; + char *key = gdes_fetch_key(im->gdes[i]); + gboolean ok = g_hash_table_lookup_extended(im->rrd_map,key,NULL,&value); + free(key); + if (ok){ + ii = GPOINTER_TO_INT(value); + im->gdes[i].start = im->gdes[ii].start; + im->gdes[i].end = im->gdes[ii].end; + im->gdes[i].step = im->gdes[ii].step; + im->gdes[i].ds_cnt = im->gdes[ii].ds_cnt; + im->gdes[i].ds_namv = im->gdes[ii].ds_namv; + im->gdes[i].data = im->gdes[ii].data; + im->gdes[i].data_first = 0; + } else { unsigned long ft_step = im->gdes[i].step; /* ft_step will record what we got from fetch */ const char *rrd_daemon; int status; @@ -958,29 +953,6 @@ int data_fetch( * CDEF stuff *************************************************************/ -long find_var_wrapper( - void *arg1, - char *key) -{ - return find_var((image_desc_t *) arg1, key); -} - -/* find gdes containing var*/ -long find_var( - image_desc_t *im, - char *key) -{ - long ii; - - for (ii = 0; ii < im->gdes_c - 1; ii++) { - if ((im->gdes[ii].gf == GF_DEF - || im->gdes[ii].gf == GF_VDEF || im->gdes[ii].gf == GF_CDEF) - && (strcmp(im->gdes[ii].vname, key) == 0)) { - return ii; - } - } - return -1; -} /* find the greatest common divisor for all the numbers in the 0 terminated num array */ @@ -4228,6 +4200,8 @@ void rrd_graph_init( #ifdef HAVE_TZSET tzset(); #endif + im->gdef_map = g_hash_table_new_full(g_str_hash, g_str_equal,g_free,NULL); + im->rrd_map = g_hash_table_new_full(g_str_hash, g_str_equal,g_free,NULL); im->graph_type = GTYPE_TIME; im->base = 1000; im->daemon_addr = NULL; diff --git a/src/rrd_graph.h b/src/rrd_graph.h index ca039abf..e96f957d 100644 --- a/src/rrd_graph.h +++ b/src/rrd_graph.h @@ -26,6 +26,9 @@ #include "rrd_tool.h" #include "rrd_rpncalc.h" +#include + + #ifdef WIN32 # include # define MAXPATH MAX_PATH @@ -46,6 +49,8 @@ #define FULL_SIZE_MODE 0x200 /* -width and -height indicate the total size of the image */ #define NO_RRDTOOL_TAG 0x400 /* disable the rrdtool tag */ +#define gdes_fetch_key(x) sprintf_alloc("%s:%d:%d:%d:%d",x.rrd,x.cf,x.cf_reduce,x.start_orig,x.end_orig,x.step_orig) + enum tmt_en { TMT_SECOND = 0, TMT_MINUTE, TMT_HOUR, TMT_DAY, TMT_WEEK, TMT_MONTH, TMT_YEAR }; @@ -331,6 +336,8 @@ typedef struct image_desc_t { PangoLayout *layout; /* the pango layout we use for writing fonts */ rrd_info_t *grinfo; /* root pointer to extra graph info */ rrd_info_t *grinfo_current; /* pointing to current entry */ + GHashTable* gdef_map; /* a map of all *def gdef entries for quick access */ + GHashTable* rrd_map; /* a map of all rrd files in use for gdef entries */ } image_desc_t; /* Prototypes */ @@ -375,12 +382,6 @@ void reduce_data( rrd_value_t **); int data_fetch( image_desc_t *); -long find_var( - image_desc_t *, - char *); -long find_var_wrapper( - void *arg1, - char *key); long lcd( long *); int data_calc( diff --git a/src/rrd_graph_helper.c b/src/rrd_graph_helper.c index 8232ce84..2a9091a5 100644 --- a/src/rrd_graph_helper.c +++ b/src/rrd_graph_helper.c @@ -353,6 +353,45 @@ int parse_color( const char *const string, struct gfx_color_t *c) #define PARSE_VNAMERPN (PARSE_POSITIONAL|PARSE_VNAMEDEF|PARSE_RPN) #define PARSE_VNAMEREFPOS (PARSE_POSITIONAL|PARSE_VNAMEREF) +GHashTable* gdef_map; + +/* find gdes containing var*/ +static long find_var( + image_desc_t *im, + char *key) +{ + /* this makes only sense for a sufficient number of items */ + long match = -1; + if ( im->gdes_c > 12 ) { + gpointer value; + gboolean ok = g_hash_table_lookup_extended(im->gdef_map,key,NULL,&value); + if (ok){ + match = GPOINTER_TO_INT(value); + } + } + else { + long ii; + for (ii = 0; ii < im->gdes_c - 1; ii++) { + if ((im->gdes[ii].gf == GF_DEF + || im->gdes[ii].gf == GF_VDEF || im->gdes[ii].gf == GF_CDEF) + && (strcmp(im->gdes[ii].vname, key) == 0)) { + match = ii; + break; + } + } + } + return match; + +} + +static long find_var_wrapper( + void *arg1, + char *key) +{ + return find_var((image_desc_t *) arg1, key); +} + + graph_desc_t* newGraphDescription(image_desc_t *const,enum gf_en,parsedargs_t*,unsigned long); graph_desc_t* newGraphDescription(image_desc_t *const im,enum gf_en gf,parsedargs_t* pa,unsigned long bits) { /* check that none of the othe bitfield marker is set */ @@ -693,7 +732,7 @@ graph_desc_t* newGraphDescription(image_desc_t *const im,enum gf_en gf,parsedarg } /* and assign it */ - if (vname) { strncpy(gdp->vname,vname,MAX_VNAME_LEN + 1); } + if (vname) { strncpy(gdp->vname,vname,MAX_VNAME_LEN + 1);} if (rrd) { strncpy(gdp->rrd,rrd,1024); } if (ds) { strncpy(gdp->ds_nam,ds,DS_NAM_SIZE); } if (cf) { @@ -733,6 +772,14 @@ graph_desc_t* newGraphDescription(image_desc_t *const im,enum gf_en gf,parsedarg gdp->yrule=val; } } + /* remember the index for faster varfind */ + char *key = gdes_fetch_key((*gdp)); + if (gdp->gf == GF_DEF && !g_hash_table_lookup_extended(im->rrd_map,key,NULL,NULL)){ + g_hash_table_insert(im->gdef_map,g_strdup(key),GINT_TO_POINTER(im->gdes_c-1)); + } + free(key); + g_hash_table_insert(im->gdef_map,g_strdup(gdp->vname),GINT_TO_POINTER(im->gdes_c-1)); +// g_hash_table_insert(im->gdef_map,gdp->vname,GINT_TO_POINTER(im->gdes_c-1)); /* and return it */ return gdp; } diff --git a/src/rrd_rpncalc.c b/src/rrd_rpncalc.c index 1e76ee5d..491a8215 100644 --- a/src/rrd_rpncalc.c +++ b/src/rrd_rpncalc.c @@ -422,7 +422,7 @@ void rpnstack_init( { rpnstack->s = NULL; rpnstack->dc_stacksize = 0; - rpnstack->dc_stackblock = 100; + rpnstack->dc_stackblock = 1000; } void rpnstack_free( @@ -465,7 +465,7 @@ short rpn_calc( int output_idx) { int rpi; - long stptr = -1; + long stptr = -1; /* process each op from the rpn in turn */ for (rpi = 0; rpnp[rpi].op != OP_END; rpi++) {