prefill last DS values

diff --git a/src/rrd_create.c b/src/rrd_create.c
index 0627cf1ebaa7af068d275f1cc82c4c8c3eb72a02..ce1a06b55cdc87964423a6bc5efe70ed8d97386f 100644 (file)
--- a/src/rrd_create.c
+++ b/src/rrd_create.c
@@ -31,6 +31,7 @@
 # include <process.h>
 #endif
 
+static void reset_pdp_prep(rrd_t *rrd);
 static int rrd_init_data(rrd_t *rrd);
 static int rrd_prefill_data(rrd_t *rrd, const GList *sources_rrd_files);
 static int positive_mod(int a, int b);
@@ -752,6 +753,7 @@ int rrd_create_r2(
     
         if (last_up == -1) {
             rrd.live_head->last_up = sources_latest_last_up;
+            reset_pdp_prep(&rrd);
         }
         rrd_prefill_data(&rrd, sources_rrd_files);
     }
@@ -943,6 +945,21 @@ void init_cdp(const rrd_t *rrd, const rra_def_t *rra_def, const pdp_prep_t *pdp_
     }
 }
 
+
+static void reset_pdp_prep(rrd_t *rrd) {
+    unsigned long ds_index;
+    
+    for (ds_index = 0 ; ds_index < rrd->stat_head->ds_cnt ; ds_index++) {
+        strcpy(rrd->pdp_prep[ds_index].last_ds, "U");
+        /* interestingly, "U" is associated with a value of 0.0, traditionally.
+         * I do not know why. Ask Tobi.
+         */
+        rrd->pdp_prep[ds_index].scratch[PDP_val].u_val = 0;
+        rrd->pdp_prep[ds_index].scratch[PDP_unkn_sec_cnt].u_cnt =
+            rrd->live_head->last_up % rrd->stat_head->pdp_step;
+    }
+}
+
 /* create and empty rrd file according to the specs given */
 
 static int rrd_init_data(rrd_t *rrd)
@@ -977,12 +994,7 @@ static int rrd_init_data(rrd_t *rrd)
             rrd_set_error("cannot allocate memory");
             goto done;
         }
-        for (i = 0 ; i < rrd->stat_head->ds_cnt ; i++) {
-            strcpy(rrd->pdp_prep[i].last_ds, "U");
-            rrd->pdp_prep[i].scratch[PDP_val].u_val = 0.0;
-            rrd->pdp_prep[i].scratch[PDP_unkn_sec_cnt].u_cnt =
-                rrd->live_head->last_up % rrd->stat_head->pdp_step;
-        }
+        reset_pdp_prep(rrd);
     }
 
     if (rrd->cdp_prep == NULL) {
@@ -1647,6 +1659,155 @@ done:
     if (coverage) free(coverage);
 }
 
+
+/* 
+ * prefill last value for the target RRA if we haven't done so yet (That is, 
+ * if the last value is still unknown). We can only *really* do this, if the 
+ * last update of the target RRD is compatible with one of the source RRDs,
+ * because there is generally no way to deduce the input data from an RRA bin.
+ */
+static void prefill_pdp_prep(candidate_t *target, const candidate_t *candidates, int candidate_cnt) {
+    rrd_t *rrd = target->rrd;
+    int ds_index = target->extra.l;
+    
+    if (strncmp(rrd->pdp_prep[ds_index].last_ds, "U", LAST_DS_LEN) != 0) {
+        /* PDP prep for this DS already set. */
+        return;
+    }
+    
+    time_t current_pdp_begin_time = 
+            (rrd->live_head->last_up / rrd->stat_head->pdp_step) *
+             rrd->stat_head->pdp_step + 1;
+    time_t current_pdp_end_time = current_pdp_begin_time + rrd->stat_head->pdp_step - 1;
+
+    /* walk all source RRAs to find a matching last update value. 
+     * We use the candidate RRA list for this, to make sure that we
+     * won't erroneously use an RRD we have no candidate for... */
+
+    const rrd_t *srrd = NULL;
+    for (int c = 0 ; c < candidate_cnt ; c++) {
+        const candidate_t *candidate = candidates + c;
+        /* only look at each source RRD once. We use the fact that all 
+         * candidates are block-sorted by the source RRDs */
+        if (srrd == candidate->rrd) continue;
+        srrd = candidate->rrd;
+
+        if (strncmp(srrd->pdp_prep[candidate->extra.l].last_ds, "U", LAST_DS_LEN) == 0) {
+            /* this source RRD does not have a usable last DS value - skip */
+            continue;
+        }
+
+        time_t source_pdp_begin_time = 
+            (srrd->live_head->last_up / srrd->stat_head->pdp_step) *
+            srrd->stat_head->pdp_step + 1;
+        time_t source_pdp_end_time = source_pdp_begin_time + srrd->stat_head->pdp_step - 1;
+
+        if (source_pdp_end_time <= current_pdp_end_time && source_pdp_end_time >= current_pdp_begin_time) {
+            /* only really copy the data if the DS Type matches... */
+            if (strcmp(rrd->ds_def[ds_index].dst, srrd->ds_def[candidate->extra.l].dst) == 0) {
+                memcpy(rrd->pdp_prep + ds_index, srrd->pdp_prep + candidate->extra.l,
+                        sizeof(pdp_prep_t));
+
+                /* in case the step sizes of target and source do not match, 
+                 * we take an additional look at the unknown seconds, because that
+                 * value should not be larger than the step size.
+                 * 
+                 * We also assume, that it is unusual to explicitly add unknown data to an
+                 * RRD, so we use the current value in any case, and assume the unknown data to
+                 * be at the beginning of the current pdp_prep time range.
+                 * 
+                 * I really hope that I got the semantics right here - PSt.
+                 */
+
+                if (rrd->pdp_prep[ds_index].scratch[PDP_unkn_sec_cnt].u_val > rrd->stat_head->pdp_step) {
+                    long source_known = srrd->stat_head->pdp_step - rrd->pdp_prep[ds_index].scratch[PDP_unkn_sec_cnt].u_val;
+                    long max_target_known = rrd->live_head->last_up - current_pdp_begin_time;
+
+                    rrd->pdp_prep[ds_index].scratch[PDP_unkn_sec_cnt].u_cnt = max(0, max_target_known - source_known);
+                }
+
+                /* we now have set a known last DS value in the pdp_prep, break 
+                 * out of the loop, because we are done */
+                break;
+            }
+        }
+    }
+}
+
+
+static unsigned long find_ds_match(const ds_def_t *ds_def, const rrd_t *src_rrd) {
+    unsigned long source_ds_index;
+    
+    for (source_ds_index = 0 ; source_ds_index < src_rrd->stat_head->ds_cnt ; source_ds_index++) {
+        if (strcmp(ds_def->ds_nam, src_rrd->ds_def[source_ds_index].ds_nam) == 0) {
+            return source_ds_index;
+        }
+    }
+    return src_rrd->stat_head->ds_cnt;
+}
+
+
+/* Find a set of RRAs that can be used to prefill RRA bins for the target RRA from
+ * among the source RRDs. */
+static candidate_t *find_prefill_candidates(const candidate_t *target, 
+        const GList *sources, int *candidate_cnt) {
+    ds_def_t *ds_def = target->rrd->ds_def + target->extra.l;
+
+    const GList *src;
+    candidate_t *candidates = NULL;
+    
+    int cnt = 0;
+    
+    for (src = sources ; src ;  src = g_list_next(src)) {
+        // first: find matching DS
+
+        const rrd_file_t *rrd_file = src->data;
+        if (rrd_file == NULL) continue;
+
+        const rrd_t *src_rrd = rrd_file->rrd;
+        if (src_rrd == NULL) continue; 
+
+        unsigned long source_ds_index = find_ds_match(ds_def, src_rrd);
+        if (source_ds_index < src_rrd->stat_head->ds_cnt) {
+            // match found
+
+            candidate_extra_t extra = { .l = source_ds_index };
+            // candidates = g_list_append(candidates, (gpointer) src);
+            int candidate_cnt_for_source = 0;
+            candidate_t *candidates_for_source = 
+                    find_candidate_rras(src_rrd, target->rra, &candidate_cnt_for_source, 
+                                        extra,
+                                        select_create_candidates);
+
+            if (candidates_for_source && candidate_cnt_for_source > 0) {
+                quick_sort(candidates_for_source, sizeof(candidate_t),
+                        candidate_cnt_for_source, (compar_ex_t*)order_candidates, (void*) target);
+
+                candidates = realloc(candidates, 
+                                     sizeof(candidate_t) * (cnt + candidate_cnt_for_source));
+                if (candidates == NULL) {
+                    rrd_set_error("Cannot realloc memory");
+                    free(candidates_for_source);
+                    goto done;
+                }
+                memcpy(candidates + cnt, 
+                       candidates_for_source,
+                       sizeof(candidate_t) * candidate_cnt_for_source);
+
+                cnt += candidate_cnt_for_source;
+            }
+            if (candidates_for_source) {
+                free(candidates_for_source);
+            }
+        }
+    }
+done:
+    *candidate_cnt = cnt;
+    return candidates;
+}
+
+
+
 static int rrd_prefill_data(rrd_t *rrd, const GList *sources) {
     int rc = -1;
     
@@ -1656,16 +1817,16 @@ static int rrd_prefill_data(rrd_t *rrd, const GList *sources) {
         goto done;
     }
 
-    unsigned long i, j, sj;
+    unsigned long rra_index, ds_index;
     unsigned long total_rows = 0;
     
     debug_dump_rra(((rrd_file_t *) sources->data)->rrd, 0, 0);
 
     // process one RRA after the other
-    for (i = 0 ; i < rrd->stat_head->rra_cnt ; i++) {
-        prefill_debug("PREFILL RRA %ld\n", i);
+    for (rra_index = 0 ; rra_index < rrd->stat_head->rra_cnt ; rra_index++) {
+        prefill_debug("PREFILL RRA %ld\n", rra_index);
 
-        rra_def_t *rra_def = rrd->rra_def + i;
+        rra_def_t *rra_def = rrd->rra_def + rra_index;
         
         /*
          * Re-use candidate_t as a container for all information, because the data structure contains
@@ -1677,84 +1838,25 @@ static int rrd_prefill_data(rrd_t *rrd, const GList *sources) {
             .rrd = rrd,
             .rra = rra_def,
             .rra_cf = cf_conv(rra_def->cf_nam),
-            .rra_index = i,
+            .rra_index = rra_index,
             .values = rrd->rrd_value + rrd->stat_head->ds_cnt * total_rows,
         };
         
 
-        for (j = 0 ; j < rrd->stat_head->ds_cnt ; j++) {
-            target.extra.l = j;
+        for (ds_index = 0 ; ds_index < rrd->stat_head->ds_cnt ; ds_index++) {
+            target.extra.l = ds_index;
             /* for each DS in each RRA within rrd find a list of candidate DS/RRAs from 
              * the sources list that match by name... */
     
-            ds_def_t  *ds_def  = rrd->ds_def + j;
             
-            const GList *src;
-            candidate_t *candidates = NULL;
             int candidate_cnt = 0;
+            candidate_t *candidates = find_prefill_candidates(&target, sources, &candidate_cnt);
+            
             
-            for (src = sources ; src ;  src = g_list_next(src)) {
-                // first: find matching DS
-                
-                const rrd_file_t *rrd_file = src->data;
-                if (rrd_file == NULL) continue;
-                
-                const rrd_t *src_rrd = rrd_file->rrd;
-                if (src_rrd == NULL) continue; 
-
-                prefill_debug("cur_rows: %ld %ld\n", 
-                        rrd->rra_ptr[0].cur_row, src_rrd->rra_ptr[0].cur_row);
-                
-                prefill_debug("src rrd last_up %ld\n", src_rrd->live_head->last_up);
-                prefill_debug("dst rrd last_up %ld\n", rrd->live_head->last_up);
-                
-                for (sj = 0 ; sj < src_rrd->stat_head->ds_cnt ; sj++) {
-                    if (strcmp(ds_def->ds_nam, src_rrd->ds_def[sj].ds_nam) == 0) {
-                        // name match!!!
-                        
-                        candidate_extra_t extra = { .l = sj };
-                        // candidates = g_list_append(candidates, (gpointer) src);
-                        int candidate_cnt_for_source = 0;
-                        candidate_t *candidates_for_source = 
-                                find_candidate_rras(src_rrd, rra_def, &candidate_cnt_for_source, 
-                                                    extra,
-                                                    select_create_candidates);
-                        
-                        if (candidates_for_source && candidate_cnt_for_source > 0) {
-                            quick_sort(candidates_for_source, sizeof(candidate_t),
-                                    candidate_cnt_for_source, (compar_ex_t*)order_candidates, &target);
-                            
-                            candidates = realloc(candidates, 
-                                                 sizeof(candidate_t) * (candidate_cnt + candidate_cnt_for_source));
-                            if (candidates == NULL) {
-                                rrd_set_error("Cannot realloc memory");
-                                goto done;
-                            }
-                            memcpy(candidates + candidate_cnt, 
-                                   candidates_for_source,
-                                   sizeof(candidate_t) * candidate_cnt_for_source);
-                            
-                            candidate_cnt += candidate_cnt_for_source;
-                        }
-                        if (candidates_for_source) {
-                            free(candidates_for_source);
-                        }
-#ifdef DEBUG_PREFILL                        
-                        for (unsigned int tt = 0 ; tt < src_rrd->stat_head->rra_cnt ; tt++) {
-                            fprintf(stderr, "SRC RRA %d row_cnt=%ld\n", tt, src_rrd->rra_def[tt].row_cnt);
-                        }                        
-                        for (int tt = 0 ; tt < candidate_cnt ; tt++) {
-                            fprintf(stderr, "CAND SRC RRA %d row_cnt=%ld\n", candidates[tt].rra_index, candidates[tt].rra->row_cnt);
-                        }            
-#endif
-                    }
-                }
-            }
-
 #ifdef DEBUG_PREFILL
             // report candidates
             fprintf(stderr, "ds=%s candidates for %s %d rows=%d\n", 
-                    ds_def->ds_nam,
+                    rrd->ds_def[ds_index].ds_nam,
                     rra_def->cf_nam, rra_def->pdp_cnt, 
                             rra_def->row_cnt);
             for (int rr = 0 ; rr < candidate_cnt ; rr++) {
@@ -1765,6 +1867,7 @@ static int rrd_prefill_data(rrd_t *rrd, const GList *sources) {
 #endif            
             /* if we have no candidates for an RRA we just skip to the next... */
             if (candidates == NULL) {
+                if (rrd_test_error()) goto done;
                 continue;
             }
             
@@ -1777,10 +1880,9 @@ static int rrd_prefill_data(rrd_t *rrd, const GList *sources) {
                 prefill_bin(&target, cnt, candidates, candidate_cnt);
             }
 
-            if (candidates) {
-                free(candidates);
-                candidates = NULL;
-            }
+            prefill_pdp_prep(&target, candidates, candidate_cnt);
+            
+            free(candidates);
         }
         total_rows += rra_def->row_cnt;
     }

diff --git a/src/rrd_modify.h b/src/rrd_modify.h
index 4ed94c172e0605b8f94e4f7f2fea06db9382af4d..967295fda2ef65becb25bda9f7e2b8b2a8f658b4 100644 (file)
--- a/src/rrd_modify.h
+++ b/src/rrd_modify.h
@@ -38,7 +38,7 @@ typedef union {
 } candidate_extra_t;
 
 typedef struct {
-    const rrd_t *rrd;
+    rrd_t *rrd;
     int rra_index;
     rrd_value_t *values;
     rra_def_t *rra;

diff --git a/tests/create-with-source-1 b/tests/create-with-source-1
index 58fe8e4f677aa40c7c1a3720522e391d0f458eb9..e2c4f9147900b26e5c0c199ff988d565b3cf9206 100755 (executable)
--- a/tests/create-with-source-1
+++ b/tests/create-with-source-1
@@ -18,7 +18,7 @@ function xmlfilter {
 #+               <unknown_sec> 40 </unknown_sec>
 
 
-       perl -n -e '$a=join("",<>); $a=~s,<(cdp_prep|last_ds|value|unknown_sec).*?</\1>,,msg ; print $a'
+       perl -n -e '$a=join("",<>); $a=~s,<(cdp_prep).*?</\1>,,msg ; print $a'
 }
 
 # create 2 RRDs only differing in the way that the second contains an additional RRA

diff --git a/tests/create-with-source-2 b/tests/create-with-source-2
index 6441bc93b3296e23cbb1c94ab37ed52151eb6606..c4c1224c00161510184a21ab90f519d69995a656 100755 (executable)
--- a/tests/create-with-source-2
+++ b/tests/create-with-source-2
@@ -9,16 +9,7 @@ PREFIX=$BUILDDIR/$(basename $0)-test
 # comparison of RRD dumps, we just filter out those parts we do not
 # expect to match anyway...
 function xmlfilter {
-
-#-               <last_ds>1010</last_ds>
-#-               <value>4.0400000000e+04</value>
-#-               <unknown_sec> 0 </unknown_sec>
-#+               <last_ds>U</last_ds>
-#+               <value>0.0000000000e+00</value>
-#+               <unknown_sec> 40 </unknown_sec>
-
-
-       perl -n -e '$a=join("",<>); $a=~s,<(cdp_prep|last_ds|value|unknown_sec).*?</\1>,,msg ; print $a'
+       perl -n -e '$a=join("",<>); $a=~s,<(cdp_prep).*?</\1>,,msg ; print $a'
 }
 
 

diff --git a/tests/create-with-source-3 b/tests/create-with-source-3
index 661838e0e3f068aaf3bb4c1bd1f8023fa407b5f1..cdebeb97b225830e813a3b7f85a405bb45d1818f 100755 (executable)
--- a/tests/create-with-source-3
+++ b/tests/create-with-source-3
@@ -9,16 +9,11 @@ PREFIX=$BUILDDIR/$(basename $0)-test
 # comparison of RRD dumps, we just filter out those parts we do not
 # expect to match anyway...
 function xmlfilter {
+       perl -n -e '$a=join("",<>); $a=~s,<(cdp_prep).*?</\1>,,msg ; print $a'
+}
 
-#-               <last_ds>1010</last_ds>
-#-               <value>4.0400000000e+04</value>
-#-               <unknown_sec> 0 </unknown_sec>
-#+               <last_ds>U</last_ds>
-#+               <value>0.0000000000e+00</value>
-#+               <unknown_sec> 40 </unknown_sec>
-
-
-       perl -n -e '$a=join("",<>); $a=~s,<(cdp_prep|last_ds|value|unknown_sec).*?</\1>,,msg ; print $a'
+function pdp_prep_filter {
+       perl -n -e '$a=join("",<>); $a=~s,<(last_ds|value|unknown_sec).*?</\1>,,msg ; print $a'
 }
 
 
@@ -99,21 +94,21 @@ report create-ab-with-four-sources
 
 
 # now re-extract single value RRDs from the combined one....
-
+# NOTE: We cannot re-create the pdp_prep information here, so filter it out....
 
 $RRDTOOL create ${PREFIX}a2_x.rrd --start $ST_X --step 60 --source ${PREFIX}ab2_xy.rrd DS:a:GAUGE:120:0:U RRA:AVERAGE:0.5:1:100 RRA:AVERAGE:0.5:5:2 RRA:MIN:0.5:5:2 RRA:MAX:0.5:5:2 RRA:LAST:0.5:5:2
 report recreate-original-a1_x-from-combined-source
 
-$RRDTOOL dump ${PREFIX}a1_x.rrd | xmlfilter > ${PREFIX}a1_x.xml
-$RRDTOOL dump ${PREFIX}a2_x.rrd | xmlfilter > ${PREFIX}a2_x.xml
+$RRDTOOL dump ${PREFIX}a1_x.rrd | xmlfilter | pdp_prep_filter > ${PREFIX}a1_x.xml
+$RRDTOOL dump ${PREFIX}a2_x.rrd | xmlfilter | pdp_prep_filter > ${PREFIX}a2_x.xml
 $DIFF ${PREFIX}a1_x.xml ${PREFIX}a2_x.xml
 report data-match-a1_x
 
 $RRDTOOL create ${PREFIX}b2_x.rrd --start $ST_X --step 60 --source ${PREFIX}ab2_xy.rrd DS:b:GAUGE:120:0:U RRA:AVERAGE:0.5:1:100 RRA:AVERAGE:0.5:5:2 RRA:MIN:0.5:5:2 RRA:MAX:0.5:5:2 RRA:LAST:0.5:5:2
 report recreate-original-b1_x-from-combined-source
 
-$RRDTOOL dump ${PREFIX}b1_x.rrd | xmlfilter > ${PREFIX}b1_x.xml
-$RRDTOOL dump ${PREFIX}b2_x.rrd | xmlfilter > ${PREFIX}b2_x.xml
+$RRDTOOL dump ${PREFIX}b1_x.rrd | xmlfilter | pdp_prep_filter > ${PREFIX}b1_x.xml
+$RRDTOOL dump ${PREFIX}b2_x.rrd | xmlfilter | pdp_prep_filter > ${PREFIX}b2_x.xml
 $DIFF ${PREFIX}b1_x.xml ${PREFIX}b2_x.xml
 report data-match-b1_x
 

diff --git a/tests/create-with-source-4 b/tests/create-with-source-4
index a141dd70070b1fd97f711089e2518565c5e94a49..0f0fda0a3bf39af5a855ee82484e1dc6034a99f3 100755 (executable)
--- a/tests/create-with-source-4
+++ b/tests/create-with-source-4
@@ -18,7 +18,7 @@ function xmlfilter {
 #+               <unknown_sec> 40 </unknown_sec>
 
 
-       perl -n -e '$a=join("",<>); $a=~s,<(cdp_prep|last_ds|value|unknown_sec).*?</\1>,,msg ; print $a'
+       perl -n -e '$a=join("",<>); $a=~s,<(cdp_prep).*?</\1>,,msg ; print $a'
 }
 
 function numberfilter {