#include "rrd_is_thread_safe.h"
#include "rrd_modify.h"
+#include "unused.h"
+
#ifdef WIN32
# include <process.h>
#endif
return (time >= start && time <= end);
}
-static void debug_dump_rra(const rrd_t *rrd, int rra_index, int ds_index) {
+inline static void debug_dump_rra(const rrd_t *rrd, int rra_index, int ds_index) {
+#ifdef DEBUG_PREFILL
long total_cnt = 0;
for (int zz = 0 ; zz < rra_index ; zz++) {
total_cnt += rrd->rra_def[rra_index].row_cnt;
rrd->rra_ptr[rra_index].cur_row, zt);
fprintf(stderr, "%d %ld %g %d\n", zz, zt, v, zz_calc);
}
+#endif
}
typedef struct {
c->end = end;
}
+#ifdef DEBUG_PREFILL
static void dump_coverage_array(const coverage_t *current_coverage, const int *coverage_array_size) {
for (int i = 0 ; i < *coverage_array_size ; i++) {
fprintf(stderr, "%d covered=%d start=%ld end=%ld\n", i, current_coverage[i].covered,
current_coverage[i].start, current_coverage[i].end);
}
}
+#endif
+
+// #define add_coverage_debug(...) fprintf(stderr, __VA_ARGS__)
+#define add_coverage_debug(...) do {} while(0)
static coverage_t *add_coverage(coverage_t *current_coverage, int *coverage_array_size,
time_t start, time_t end,
int *newly_covered_interval)
{
int i;
- fprintf(stderr, "ADDING %ld %ld\n", start, end);
+ add_coverage_debug("ADDING %ld %ld\n", start, end);
if (coverage_array_size == NULL) return NULL;
if (current_coverage == NULL) return NULL;
}
*newly_covered_interval = 0;
+
+ /* out of boundaries check */
+ if (end < current_coverage->start ||
+ start > (current_coverage + (*coverage_array_size - 1))->end) {
+ return current_coverage;
+ }
for (i = 0 ; i < *coverage_array_size ; i++) {
coverage_t *cc = current_coverage + i;
time_t org_start = cc->start;
time_t org_end = cc->end;
- fprintf(stderr, "check %ld %ld against %d/%d (%ld %ld)\n", start, end, i, *coverage_array_size, org_start, org_end);
+ add_coverage_debug("check %ld %ld against %d/%d (%ld %ld)\n", start, end, i, *coverage_array_size, org_start, org_end);
if (is_interval_within_interval(start, end, org_start, org_end)) {
/*
* Case (A): newly added interval is fully contained within the current one.
*/
- fprintf(stderr, "(A)\n");
- fprintf(stderr, "OVERLAP %ld %ld %ld %ld\n", start, end, org_start, org_end);
+ add_coverage_debug("(A)\n");
+ add_coverage_debug("OVERLAP %ld %ld %ld %ld\n", start, end, org_start, org_end);
if (cc->covered) {
// no new data .. already fully covered, just return
break;
* Note that if this case happens, case (A) above will NEVER happen...
*/
if (is_interval_within_interval(org_start, org_end, start, end)) {
- fprintf(stderr, "(B)\n");
+ add_coverage_debug("(B)\n");
if (! cc->covered) {
/* just turn the current interval into a covered one. Report
* the range as newly covered */
*/
if (is_time_within_interval(start, org_start, org_end)) {
- fprintf(stderr, "(C)\n");
+ add_coverage_debug("(C)\n");
/* If the current interval is a covered one, we do nothing but
* to adjust the start interval for the next iteration.
*/
if (cc->covered) {
- fprintf(stderr, "(C1)\n");
+ add_coverage_debug("(C1)\n");
start = org_end + 1;
continue;
}
/* if the current interval is not covered... */
if (cc->start == start) {
- fprintf(stderr, "(C2)\n");
+ add_coverage_debug("(C2)\n");
/* ... and the new interval starts with the current one, we just turn it into a
* covered one and adjust the start... */
cc->covered = 1;
start = org_end + 1;
} else {
- fprintf(stderr, "(C3) %d\n", *coverage_array_size + 1);
+ add_coverage_debug("(C3) %d\n", *coverage_array_size + 1);
/* ... and the new interval does NOT start with the current one, we have to split the interval .. */
current_coverage = realloc(current_coverage, sizeof(coverage_t) * (*coverage_array_size + 1));
coverage_t *next = cc + 1;
if (cc->covered == next->covered) {
- fprintf(stderr, "Collapse %ld %ld %ld %ld\n", cc->start, cc->end, next->start, next->end);
+ add_coverage_debug("Collapse %ld %ld %ld %ld\n", cc->start, cc->end, next->start, next->end);
cc->end = next->end;
memmove(next, next + 1, sizeof(coverage_t) * (*coverage_array_size - i - 1));
(*coverage_array_size)--;
- fprintf(stderr, "%d intervals left\n", *coverage_array_size);
+ add_coverage_debug("%d intervals left\n", *coverage_array_size);
// re-iterate with i unchanged !!
continue;
return total;
}
+#ifdef DEBUG_PREFILL
+# define prefill_debug(...) fprintf(stderr, __VA_ARGS__)
+#else
+# define prefill_debug(...) do {} while(0)
+#endif
+
+static rrd_value_t prefill_consolidate(rra_def_t UNUSED(*rra_def), enum cf_en current_cf,
+ rrd_value_t current_estimate,
+ rrd_value_t added_value,
+ int target_bin_size, int newly_covered_size) {
+ switch (current_cf) {
+ case CF_MINIMUM:
+ if (isnan(current_estimate)) return added_value;
+ return current_estimate < added_value ? current_estimate : added_value;
+ case CF_MAXIMUM:
+ if (isnan(current_estimate)) return added_value;
+ return current_estimate > added_value ? current_estimate : added_value;
+ case CF_LAST:
+ if (isnan(current_estimate)) return added_value;
+ /* FIXME: There are better ways to do this, but it requires more information
+ * from the caller.
+ */
+ return added_value;
+ case CF_AVERAGE:
+ default:
+ if (isnan(current_estimate)) current_estimate = 0.0;
+
+ return current_estimate + added_value / target_bin_size * newly_covered_size;
+ }
+}
+
+static rrd_value_t prefill_finish(rra_def_t UNUSED(*rra_def), enum cf_en current_cf,
+ rrd_value_t current_estimate,
+ int target_bin_size, int total_covered_size) {
+ if (total_covered_size == 0) return DNAN;
+
+ switch (current_cf) {
+ case CF_AVERAGE:
+ return current_estimate / total_covered_size * target_bin_size;
+ case CF_MINIMUM:
+ case CF_MAXIMUM:
+ case CF_LAST:
+ default:
+ return current_estimate;
+ }
+}
+
static int rrd_prefill_data(rrd_t *rrd, const GList *sources) {
int rc = -1;
// process one RRA after the other
for (i = 0 ; i < rrd->stat_head->rra_cnt ; i++) {
- fprintf(stderr, "PREFILL RRA %ld\n", i);
+ prefill_debug("PREFILL RRA %ld\n", i);
rra_def_t *rra_def = rrd->rra_def + i;
- unsigned long cur_row = rrd->rra_ptr[i].cur_row;
+ enum cf_en current_cf = cf_conv(rra_def->cf_nam);
for (j = 0 ; j < rrd->stat_head->ds_cnt ; j++) {
/* for each DS in each RRA within rrd find a list of candidate DS/RRAs from
const rrd_t *src_rrd = rrd_file->rrd;
if (src_rrd == NULL) continue;
- fprintf(stderr, "cur_rows: %ld %ld\n",
+ prefill_debug("cur_rows: %ld %ld\n",
rrd->rra_ptr[0].cur_row, src_rrd->rra_ptr[0].cur_row);
- fprintf(stderr, "src rrd last_up %ld\n", src_rrd->live_head->last_up);
- fprintf(stderr, "dst rrd last_up %ld\n", rrd->live_head->last_up);
+ prefill_debug("src rrd last_up %ld\n", src_rrd->live_head->last_up);
+ prefill_debug("dst rrd last_up %ld\n", rrd->live_head->last_up);
for (sj = 0 ; sj < src_rrd->stat_head->ds_cnt ; sj++) {
if (strcmp(ds_def->ds_nam, src_rrd->ds_def[sj].ds_nam) == 0) {
candidate_extra_t extra = { .l = sj };
// candidates = g_list_append(candidates, (gpointer) src);
- candidates = find_candidate_rras(src_rrd, rra_def, &candidate_cnt, extra);
+ int candidate_cnt_for_source = 0;
+ candidate_t *candidates_for_source = find_candidate_rras(src_rrd, rra_def, &candidate_cnt_for_source, extra);
-for (unsigned int tt = 0 ; tt < src_rrd->stat_head->rra_cnt ; tt++) {
- fprintf(stderr, "SRC RRA %d row_cnt=%ld\n", tt, src_rrd->rra_def[tt].row_cnt);
-}
-for (int tt = 0 ; tt < candidate_cnt ; tt++) {
- fprintf(stderr, "CAND SRC RRA %d row_cnt=%ld\n", candidates[tt].rra_index, candidates[tt].rra->row_cnt);
-}
+ if (candidates_for_source && candidate_cnt_for_source > 0) {
+ candidates = realloc(candidates,
+ sizeof(candidate_t) * (candidate_cnt + candidate_cnt_for_source));
+ if (candidates == NULL) {
+ rrd_set_error("Cannot realloc memory");
+ goto done;
+ }
+ memcpy(candidates + candidate_cnt,
+ candidates_for_source,
+ sizeof(candidate_t) * candidate_cnt_for_source);
+
+ candidate_cnt += candidate_cnt_for_source;
+ }
+ if (candidates_for_source) {
+ free(candidates_for_source);
+ }
+#ifdef DEBUG_PREFILL
+ for (unsigned int tt = 0 ; tt < src_rrd->stat_head->rra_cnt ; tt++) {
+ fprintf(stderr, "SRC RRA %d row_cnt=%ld\n", tt, src_rrd->rra_def[tt].row_cnt);
+ }
+ for (int tt = 0 ; tt < candidate_cnt ; tt++) {
+ fprintf(stderr, "CAND SRC RRA %d row_cnt=%ld\n", candidates[tt].rra_index, candidates[tt].rra->row_cnt);
+ }
+#endif
}
}
}
time_t bin_end_time = end_time_for_row_simple(rrd, i, cnt);
time_t bin_start_time = bin_end_time - bin_size + 1;
- fprintf(stderr, "Bin %ld from %ld to %ld\n", cnt, bin_start_time, bin_end_time);
+ prefill_debug("Bin %ld from %ld to %ld\n", cnt, bin_start_time, bin_end_time);
/* find corresponding range of bins in all candidates... */
+
+ coverage_t *coverage = malloc(sizeof(coverage_t));
+ int coverage_size = 1;
+ if (coverage == NULL) {
+ rrd_set_error("Cannot allocate memory");
+ goto done;
+ }
+ set_interval(coverage, 0, bin_start_time, bin_end_time);
+
+ long total_covered = 0, covering_bins = 0;
+ rrd_value_t value = DNAN; // init_prefill_consolidate(rra_def, current_cf);
- rrd_value_t best_value = 0, best_covered = 0, best_covering_bins = 0;
- for (k = 0 ; k < (unsigned long) candidate_cnt ; k++) {
+ for (k = 0 ; k < (unsigned long) candidate_cnt && total_covered < bin_size ; k++) {
candidate_t *candidate = candidates + k;
rra_def_t * candidate_rra_def = candidate->rra;
unsigned long start_bin = row_for_time(candidate->rrd, candidate->rra,
candidate->rrd->rra_ptr[candidate->rra_index].cur_row,
bin_start_time);
- fprintf(stderr, " candidate #%ld (index=%d) from %ld to %ld (row_cnt=%ld)\n", k,
+ prefill_debug(" candidate #%ld (index=%d) from %ld to %ld (row_cnt=%ld)\n", k,
candidate->rra_index,
start_bin, end_bin,
candidate_rra_def->row_cnt);
- if (start_bin < candidate_rra_def->row_cnt && end_bin < candidate_rra_def->row_cnt) {
+ if (start_bin < candidate_rra_def->row_cnt
+ && end_bin < candidate_rra_def->row_cnt) {
int bin_count = positive_mod(end_bin - start_bin + 1, candidate_rra_def->row_cnt);
- fprintf(stderr, " bin_count %d\n", bin_count);
-
- long total_covered = 0, covering_bins = 0;
- rrd_value_t value = 0;
+ prefill_debug(" bin_count %d\n", bin_count);
- for (int ci = start_bin ; bin_count > 0 ; ci++, bin_count-- ) {
+ for (unsigned int ci = start_bin ; bin_count > 0 && total_covered < bin_size ; ci++, bin_count-- ) {
+ if (ci == candidate->rra->row_cnt) ci = 0;
+
// find overlap range....
long cand_bin_size = candidate_rra_def->pdp_cnt * candidate->rrd->stat_head->pdp_step;
time_t cand_bin_end_time = end_time_for_row_simple(candidate->rrd, candidate->rra_index, ci);
long covered = overlap(bin_start_time, bin_end_time,
cand_bin_start_time, cand_bin_end_time) +1;
rrd_value_t v = candidate->values[ci * candidate->rrd->stat_head->ds_cnt + candidate->extra.l];
- if (covered > 0 && v != NAN) {
- total_covered += covered;
- covering_bins++;
+ if (covered > 0 && ! isnan(v)) {
+ int newly_covered = 0;
+ coverage = add_coverage(coverage, &coverage_size, cand_bin_start_time, cand_bin_end_time, &newly_covered);
+ if (coverage == NULL) {
+ rrd_set_error("Memory allocation failed");
+ goto done;
+ }
- value += v / cand_bin_size * covered;
+ if (newly_covered > 0) {
+ total_covered += newly_covered;
+ covering_bins++;
+
+ value = prefill_consolidate(rra_def, current_cf,
+ value, v,
+ bin_size, newly_covered);
+
+ prefill_debug(" newly covered %d/%ld added value=%g (ds #%ld) consolidated=%g\n",
+ newly_covered, bin_size, v, candidate->extra.l, value);
+
+ }
}
- fprintf(stderr, " covers from %ld to %ld overlap is %g value=%g (ds #%d)\n",
- cand_bin_start_time, cand_bin_end_time,
- (float) covered / bin_size, v, candidate->extra.l);
- }
- fprintf(stderr, "total coverage=%ld/%ld from %ld bins\n", total_covered, bin_size, covering_bins);
-
- if (total_covered > best_covered || (total_covered == best_covered && covering_bins < best_covering_bins)) {
- fprintf(stderr, "Choosing as current best value %g\n", value);
- best_value = value;
- best_covered = total_covered;
- best_covering_bins = covering_bins;
}
+ prefill_debug("total coverage=%ld/%ld from %ld bins value=%g\n",
+ total_covered, bin_size, covering_bins, value);
}
}
//row_for_time();
- if (best_covered > 0) {
-
- *(rrd->rrd_value + rrd->stat_head->ds_cnt * (total_rows + cnt) + j) = best_value;
+ if (total_covered > 0) {
+ value = prefill_finish(rra_def, current_cf, value, bin_size, total_covered);
+ *(rrd->rrd_value + rrd->stat_head->ds_cnt * (total_rows + cnt) + j) = value;
}
+
+ free(coverage);
+
}
if (candidates) {
--- /dev/null
+#!/bin/bash
+
+. $(dirname $0)/functions
+
+BASE=$BASEDIR/$(basename $0)-test
+BUILD=$BUILDDIR/$(basename $0)-test
+
+# currently, we do not properly copy cdp and pdp information, so for
+# comparison of RRD dumps, we just filter out those parts we do not
+# expect to match anyway...
+function xmlfilter {
+
+#- <last_ds>1010</last_ds>
+#- <value>4.0400000000e+04</value>
+#- <unknown_sec> 0 </unknown_sec>
+#+ <last_ds>U</last_ds>
+#+ <value>0.0000000000e+00</value>
+#+ <unknown_sec> 40 </unknown_sec>
+
+
+ perl -n -e '$a=join("",<>); $a=~s,<(cdp_prep|last_ds|value|unknown_sec).*?</\1>,,msg ; print $a'
+}
+
+
+ST=1300000000
+
+rm -f ${BUILD}*.rrd ${BUILD}*.xml
+$RRDTOOL create ${BUILD}a1.rrd --start $(($ST-1)) --step 60 DS:a:GAUGE:120:0:U RRA:AVERAGE:0.5:1:100 RRA:AVERAGE:0.5:5:2 RRA:MIN:0.5:5:2 RRA:MAX:0.5:5:2 RRA:LAST:0.5:5:2
+report createa1
+
+$RRDTOOL create ${BUILD}b1.rrd --start $(($ST-1)) --step 60 DS:b:GAUGE:120:0:U RRA:AVERAGE:0.5:1:100 RRA:AVERAGE:0.5:5:2 RRA:MIN:0.5:5:2 RRA:MAX:0.5:5:2 RRA:LAST:0.5:5:2
+report createb1
+
+$RRDTOOL create ${BUILD}ab1.rrd --start $(($ST-1)) --step 60 DS:a:GAUGE:120:0:U DS:b:GAUGE:120:0:U RRA:AVERAGE:0.5:1:100 RRA:AVERAGE:0.5:5:2 RRA:MIN:0.5:5:2 RRA:MAX:0.5:5:2 RRA:LAST:0.5:5:2
+report createab1
+
+
+UPDATEA=
+UPDATEB=
+UPDATEAB=
+V=10
+for A in $(seq $ST 60 $(($ST + 3000)) ) ; do
+ UPDATEA="$UPDATEA $A:$V"
+ UPDATEB="$UPDATEB $A:$(($V * 2))"
+ UPDATEAB="$UPDATEAB $A:$V:$(($V * 2))"
+ V=$(($V + 20))
+ ST=$A
+done
+$RRDTOOL update ${BUILD}a1.rrd $UPDATEA
+$RRDTOOL update ${BUILD}b1.rrd $UPDATEB
+$RRDTOOL update ${BUILD}ab1.rrd $UPDATEAB
+
+
+$RRDTOOL create ${BUILD}ab2.rrd --start $ST --step 60 --source ${BUILD}a1.rrd --source ${BUILD}b1.rrd DS:a:GAUGE:120:0:U DS:b:GAUGE:120:0:U RRA:AVERAGE:0.5:1:100 RRA:AVERAGE:0.5:5:2 RRA:MIN:0.5:5:2 RRA:MAX:0.5:5:2 RRA:LAST:0.5:5:2
+report create-with-two-sources
+
+
+
+
+$RRDTOOL dump ${BUILD}ab1.rrd | xmlfilter > ${BUILD}ab1.xml
+$RRDTOOL dump ${BUILD}ab2.rrd | xmlfilter > ${BUILD}ab2.xml
+$DIFF ${BUILD}ab1.xml ${BUILD}ab2.xml
+report data-match
+
+AB1=$($RRDTOOL graph ${BUILD}ab1.png --end "$ST" --start end-1h DEF:a=${BUILD}ab1.rrd:a:AVERAGE DEF:b=${BUILD}ab1.rrd:b:AVERAGE CDEF:c=b,a,/ VDEF:v=c,TOTAL PRINT:v:%lg)
+AB2=$($RRDTOOL graph ${BUILD}ab2.png --end "$ST" --start end-1h DEF:a=${BUILD}ab2.rrd:a:AVERAGE DEF:b=${BUILD}ab2.rrd:b:AVERAGE CDEF:c=b,a,/ VDEF:v=c,TOTAL PRINT:v:%lg)
+
+[ "$AB1" = "$AB2" ]
+report totalled-ratio
+exit 0
+
+
+$RRDTOOL dump ${BUILD}a1.rrd | xmlfilter > ${BUILD}a1.xml
+$RRDTOOL dump ${BUILD}a2.rrd | xmlfilter > ${BUILD}a2.xml
+$DIFF ${BUILD}a1.xml ${BUILD}a2.xml
+report data-match
+
+$RRDTOOL create ${BUILD}a3.rrd --start $ST --step 60 --source ${BUILD}a2.rrd DS:a:GAUGE:120:0:U RRA:AVERAGE:0.5:1:100 RRA:AVERAGE:0.5:5:2 RRA:MIN:0.5:5:2 RRA:MAX:0.5:5:2 RRA:LAST:0.5:5:2
+$RRDTOOL dump ${BUILD}a3.rrd | xmlfilter > ${BUILD}a3.xml
+$DIFF ${BUILD}a1.xml ${BUILD}a3.xml
+report data-match-againg
+
+$RRDTOOL create ${BUILD}a4.rrd --start $ST --step 60 --source ${BUILD}a2.rrd DS:b:GAUGE:120:0:U DS:a:GAUGE:120:0:U RRA:AVERAGE:0.5:1:100 RRA:AVERAGE:0.5:5:2 RRA:MIN:0.5:5:2 RRA:MAX:0.5:5:2 RRA:LAST:0.5:5:2
+report create-with-source-effectively-adding-DS
+
+UPDATE=
+ST=$(($ST + 60))
+for A in $(seq $ST 60 $(($ST + 3000)) ) ; do
+ UPDATE="$UPDATE $A:$V:$((2 * $V))"
+ V=$(($V + 20))
+ ST=$A
+done
+
+$RRDTOOL update ${BUILD}a4.rrd --template a:b $UPDATE
+report update-with-two-data-sources
+
+# now swap the two data sources
+$RRDTOOL create ${BUILD}a5.rrd --start $ST --step 60 --source ${BUILD}a4.rrd DS:a:GAUGE:120:0:U DS:b:GAUGE:120:0:U RRA:AVERAGE:0.5:1:100 RRA:AVERAGE:0.5:5:2 RRA:MIN:0.5:5:2 RRA:MAX:0.5:5:2 RRA:LAST:0.5:5:2
+
+# and swap the two data sources back, so we can then compare the outputs....
+$RRDTOOL create ${BUILD}a6.rrd --start $ST --step 60 --source ${BUILD}a5.rrd DS:b:GAUGE:120:0:U DS:a:GAUGE:120:0:U RRA:AVERAGE:0.5:1:100 RRA:AVERAGE:0.5:5:2 RRA:MIN:0.5:5:2 RRA:MAX:0.5:5:2 RRA:LAST:0.5:5:2
+# now a4 and a6 must match....
+
+$RRDTOOL dump ${BUILD}a4.rrd | xmlfilter > ${BUILD}a4.xml
+$RRDTOOL dump ${BUILD}a6.rrd | xmlfilter > ${BUILD}a6.xml
+$DIFF ${BUILD}a4.xml ${BUILD}a6.xml
+report data-match-after-swap
projects / thirdparty / rrdtool-1.x.git / commitdiff
