src/rrd_fetch.c

   1 /*****************************************************************************
   2  * RRDtool 1.7.2 Copyright by Tobi Oetiker, 1997-2019
   3  *****************************************************************************
   4  * rrd_fetch.c  read date from an rrd to use for further processing
   5  *****************************************************************************
   6  * $Id$
   7  * $Log$
   8  * Revision 1.8  2004/05/18 18:53:03  oetiker
   9  * big spell checking patch -- slif@bellsouth.net
  10  *
  11  * Revision 1.7  2003/11/11 19:46:21  oetiker
  12  * replaced time_value with rrd_time_value as MacOS X introduced a struct of that name in their standard headers
  13  *
  14  * Revision 1.6  2003/01/16 23:27:54  oetiker
  15  * fix border condition in rra selection of rrd_fetch
  16  * -- Stanislav Sinyagin <ssinyagin@yahoo.com>
  17  *
  18  * Revision 1.5  2002/06/23 22:29:40  alex
  19  * Added "step=1800" and such to "DEF"
  20  * Cleaned some of the signed vs. unsigned problems
  21  *
  22  * Revision 1.4  2002/02/01 20:34:49  oetiker
  23  * fixed version number and date/time
  24  *
  25  * Revision 1.3  2001/12/24 06:51:49  alex
  26  * A patch of size 44Kbytes... in short:
  27  *
  28  * Found and repaired the off-by-one error in rrd_fetch_fn().
  29  * As a result I had to remove the hacks in rrd_fetch_fn(),
  30  * rrd_tool.c, vdef_calc(), data_calc(), data_proc() and
  31  * reduce_data().  There may be other places which I didn't
  32  * find so be careful.
  33  *
  34  * Enhanced debugging in rrd_fetch_fn(), it shows the RRA selection
  35  * process.
  36  *
  37  * Added the ability to print VDEF timestamps.  At the moment it
  38  * is a hack, I needed it now to fix the off-by-one error.
  39  * If the format string is "%c" (and nothing else!), the time
  40  * will be printed by both ctime() and as a long int.
  41  *
  42  * Moved some code around (slightly altering it) from rrd_graph()
  43  *   initializing     now in rrd_graph_init()
  44  *   options parsing  now in rrd_graph_options()
  45  *   script parsing   now in rrd_graph_script()
  46  *
  47  * Revision 1.2  2001/12/17 12:48:43  oetiker
  48  * fix overflow error ...
  49  *
  50  * Revision 1.1.1.1  2001/02/25 22:25:05  oetiker
  51  * checkin
  52  *
  53  *****************************************************************************/
  54
  55 #include "rrd_tool.h"
  56 #include "rrd_client.h"
  57
  58 #include "rrd_is_thread_safe.h"
  59
  60 /* #define DEBUG  */
  61
  62 int rrd_fetch(
  63     int argc,
  64     char **argv,
  65     time_t *start,
  66     time_t *end,        /* which time frame do you want ?
  67                          * will be changed to represent reality */
  68     unsigned long *step,    /* which stepsize do you want?
  69                              * will be changed to represent reality */
  70     unsigned long *ds_cnt,  /* number of data sources in file */
  71     char ***ds_namv,    /* names of data sources */
  72     rrd_value_t **data)
  73 {                       /* two dimensional array containing the data */
  74     unsigned long step_tmp = 1;
  75     time_t    start_tmp = 0, end_tmp = 0;
  76     const char *cf;
  77     char *opt_daemon = NULL;
  78     int align_start = 0;
  79     int status;
  80
  81     rrd_time_value_t start_tv, end_tv;
  82     const char *parsetime_error = NULL;
  83     struct optparse_long longopts[] = {
  84         {"resolution", 'r', OPTPARSE_REQUIRED},
  85         {"start", 's', OPTPARSE_REQUIRED},
  86         {"end", 'e', OPTPARSE_REQUIRED},
  87         {"align-start", 'a', OPTPARSE_NONE},
  88         {"daemon", 'd', OPTPARSE_REQUIRED},
  89         {0},
  90     };
  91     struct optparse options;
  92     int opt;
  93
  94     /* init start and end time */
  95     rrd_parsetime("end-24h", &start_tv);
  96     rrd_parsetime("now", &end_tv);
  97
  98     optparse_init(&options, argc, argv);
  99     while ((opt = optparse_long(&options, longopts, NULL)) != -1) {
 100         switch (opt) {
 101         case 's':
 102             if ((parsetime_error = rrd_parsetime(options.optarg, &start_tv))) {
 103                 rrd_set_error("start time: %s", parsetime_error);
 104                 if (opt_daemon != NULL) {
 105                     free(opt_daemon);
 106                 }
 107                 return -1;
 108             }
 109             break;
 110         case 'e':
 111             if ((parsetime_error = rrd_parsetime(options.optarg, &end_tv))) {
 112                 rrd_set_error("end time: %s", parsetime_error);
 113                 if (opt_daemon != NULL) {
 114                     free(opt_daemon);
 115                 }
 116                 return -1;
 117             }
 118             break;
 119         case 'a':
 120             align_start = 1;
 121             break;
 122         case 'r':
 123             if ((parsetime_error = rrd_scaled_duration(options.optarg, 1, &step_tmp))) {
 124                 rrd_set_error("resolution: %s", parsetime_error);
 125                 if (opt_daemon != NULL) {
 126                     free(opt_daemon);
 127                 }
 128                 return -1;
 129             }
 130             break;
 131
 132         case 'd':
 133             if (opt_daemon != NULL) {
 134                 free (opt_daemon);
 135             }
 136             opt_daemon = strdup(options.optarg);
 137             if (opt_daemon == NULL)
 138             {
 139                 rrd_set_error ("strdup failed.");
 140                 return -1;
 141             }
 142             break;
 143
 144         case '?':
 145             rrd_set_error("%s", options.errmsg);
 146             if (opt_daemon != NULL) {
 147                 free(opt_daemon);
 148             }
 149             return -1;
 150         }
 151     }
 152
 153
 154     if (rrd_proc_start_end(&start_tv, &end_tv, &start_tmp, &end_tmp) == -1) {
 155         if (opt_daemon != NULL) {
 156             free(opt_daemon);
 157         }
 158         return -1;
 159     }
 160
 161     if (start_tmp < 3600 * 24 * 365 * 10) {
 162         rrd_set_error("the first entry to fetch should be after 1980");
 163         if (opt_daemon != NULL) {
 164             free(opt_daemon);
 165         }
 166         return (-1);
 167     }
 168
 169     if (align_start) {
 170         time_t delta = (start_tmp % step_tmp);
 171         start_tmp -= delta;
 172         end_tmp -= delta;
 173     }
 174
 175     if (end_tmp < start_tmp) {
 176         rrd_set_error("start (%ld) should be less than end (%ld)", start_tmp,
 177                       end_tmp);
 178         if (opt_daemon != NULL) {
 179             free(opt_daemon);
 180         }
 181         return (-1);
 182     }
 183
 184     *start = start_tmp;
 185     *end = end_tmp;
 186     *step = step_tmp;
 187
 188     if (options.optind + 1 >= options.argc) {
 189         rrd_set_error("Usage: rrdtool %s <file> <CF> [options]", options.argv[0]);
 190         if (opt_daemon != NULL) {
 191             free(opt_daemon);
 192         }
 193         return -1;
 194     }
 195
 196     cf = options.argv[options.optind + 1];
 197
 198     rrdc_connect (opt_daemon);
 199     if (rrdc_is_connected (opt_daemon))
 200             status = rrdc_fetch (options.argv[options.optind],
 201                             cf, start, end, step, ds_cnt, ds_namv, data);
 202
 203     else
 204             status = rrd_fetch_r(options.argv[options.optind],
 205                             cf, start, end, step, ds_cnt, ds_namv, data);
 206
 207     if (opt_daemon != NULL) {
 208         free(opt_daemon);
 209     }
 210     if (status != 0)
 211         return (-1);
 212     return (0);
 213 }
 214
 215 int rrd_fetch_r(
 216     const char *filename,   /* name of the rrd */
 217     const char *cf,     /* which consolidation function ? */
 218     time_t *start,
 219     time_t *end,        /* which time frame do you want ?
 220                          * will be changed to represent reality */
 221     unsigned long *step,    /* which stepsize do you want?
 222                              * will be changed to represent reality */
 223     unsigned long *ds_cnt,  /* number of data sources in file */
 224     char ***ds_namv,    /* names of data_sources */
 225     rrd_value_t **data)
 226 {                       /* two dimensional array containing the data */
 227     enum cf_en cf_idx;
 228
 229     if ((int) (cf_idx = rrd_cf_conv(cf)) == -1) {
 230         return -1;
 231     }
 232
 233     return (rrd_fetch_fn
 234             (filename, cf_idx, start, end, step, ds_cnt, ds_namv, data));
 235 } /* int rrd_fetch_r */
 236
 237 int rrd_fetch_empty(
 238     time_t *start,
 239     time_t *end,        /* which time frame do you want ? */
 240     unsigned long *step,    /* which stepsize do you want? */
 241     unsigned long *ds_cnt,  /* number of data sources in file */
 242     char *ds_nam,           /* wanted data source */
 243     char ***ds_namv,    /* names of data_sources */
 244     rrd_value_t **data)
 245 {
 246     unsigned long rows;
 247
 248     if (((*ds_namv) =
 249          (char **) malloc(sizeof(char *))) == NULL) {
 250         rrd_set_error("malloc fetch ds_namv array");
 251         return (-1);
 252     }
 253     if ((((*ds_namv)[0]) = (char*)strdup(ds_nam)) == NULL) {
 254         rrd_set_error("malloc fetch ds_namv entry");
 255         free(*ds_namv);
 256         return (-1);
 257     }
 258
 259     *ds_cnt = 1;
 260     if (*step == 0) *step = (*end - *start) / 100;
 261     *start -= (*start % *step);
 262     *end += (*step - *end % *step);
 263     rows = (*end - *start) / *step + 1;
 264
 265     if (((*data) = (rrd_value_t*)malloc(rows * sizeof(rrd_value_t))) == NULL) {
 266         rrd_set_error("malloc fetch data area");
 267         free((*ds_namv)[0]);
 268         free(*ds_namv);
 269         return (-1);
 270     }
 271
 272     while (--rows)
 273         (*data)[rows-1] = DNAN;
 274     return (0);
 275 }
 276
 277 int rrd_fetch_fn(
 278     const char *filename,   /* name of the rrd */
 279     enum cf_en cf_idx,  /* which consolidation function ? */
 280     time_t *start,
 281     time_t *end,        /* which time frame do you want ?
 282                          * will be changed to represent reality */
 283     unsigned long *step,    /* which stepsize do you want?
 284                              * will be changed to represent reality */
 285     unsigned long *ds_cnt,  /* number of data sources in file */
 286     char ***ds_namv,    /* names of data_sources */
 287     rrd_value_t **data)
 288 {                       /* two dimensional array containing the data */
 289     long      i, ii;
 290     time_t    cal_start, cal_end, rra_start_time, rra_end_time;
 291     long      best_full_rra = 0, best_part_rra = 0, chosen_rra =
 292         0, rra_pointer = 0;
 293     long      best_full_step_diff = 0, best_part_step_diff =
 294         0, tmp_step_diff = 0, tmp_match = 0, best_match = 0;
 295     long      full_match, rra_base;
 296     off_t     start_offset, end_offset;
 297     int       first_full = 1;
 298     int       first_part = 1;
 299     rrd_t     rrd;
 300     rrd_file_t *rrd_file;
 301     rrd_value_t *data_ptr;
 302     unsigned long rows;
 303
 304 #ifdef DEBUG
 305     fprintf(stderr, "Entered rrd_fetch_fn() searching for the best match\n");
 306     fprintf(stderr, "Looking for: start %10lu end %10lu step %5lu\n",
 307             *start, *end, *step);
 308 #endif
 309
 310 #ifdef HAVE_LIBDBI
 311     /* handle libdbi datasources */
 312     if (strncmp("sql//",filename,5)==0 || strncmp("sql||",filename,5)==0) {
 313         return rrd_fetch_fn_libdbi(filename,cf_idx,start,end,step,ds_cnt,ds_namv,data);
 314     }
 315 #endif
 316     if (strncmp("cb//",filename,4)==0) {
 317         return rrd_fetch_fn_cb(filename,cf_idx,start,end,step,ds_cnt,ds_namv,data);
 318     }
 319
 320     rrd_init(&rrd);
 321     rrd_file = rrd_open(filename, &rrd, RRD_READONLY | RRD_LOCK);
 322     if (rrd_file == NULL)
 323         goto err_free;
 324
 325     /* when was the really last update of this file ? */
 326
 327     if (((*ds_namv) =
 328          (char **) malloc(rrd.stat_head->ds_cnt * sizeof(char *))) == NULL) {
 329         rrd_set_error("malloc fetch ds_namv array");
 330         goto err_close;
 331     }
 332
 333     for (i = 0; (unsigned long) i < rrd.stat_head->ds_cnt; i++) {
 334         if ((((*ds_namv)[i]) = (char*)malloc(sizeof(char) * DS_NAM_SIZE)) == NULL) {
 335             rrd_set_error("malloc fetch ds_namv entry");
 336             goto err_free_ds_namv;
 337         }
 338         strncpy((*ds_namv)[i], rrd.ds_def[i].ds_nam, DS_NAM_SIZE);
 339         (*ds_namv)[i][DS_NAM_SIZE - 1] = '\0';
 340
 341     }
 342
 343     /* find the rra which best matches the requirements */
 344     for (i = 0; (unsigned) i < rrd.stat_head->rra_cnt; i++) {
 345       enum cf_en rratype=rrd_cf_conv(rrd.rra_def[i].cf_nam);
 346       /* handle this RRA */
 347       if (
 348           /* if we found a direct match */
 349           (rratype == cf_idx)
 350           ||
 351           /*if we found a DS with interval 1
 352             and CF (requested,available) are MIN,MAX,AVERAGE,LAST
 353           */
 354           (
 355               /* only if we are on interval 1 */
 356               (rrd.rra_def[i].pdp_cnt==1)
 357               && (
 358                   /* and requested CF is MIN,MAX,AVERAGE,LAST */
 359                   (cf_idx == CF_MINIMUM)
 360                   ||(cf_idx == CF_MAXIMUM)
 361                   ||(cf_idx == CF_AVERAGE)
 362                   ||(cf_idx == CF_LAST)
 363                   )
 364               && (
 365                   /* and found CF is MIN,MAX,AVERAGE,LAST */
 366                   (rratype == CF_MINIMUM)
 367                   ||(rratype == CF_MAXIMUM)
 368                   ||(rratype == CF_AVERAGE)
 369                   ||(rratype == CF_LAST)
 370                   )
 371               )
 372           ){
 373
 374             cal_end = (rrd.live_head->last_up - (rrd.live_head->last_up
 375                                                  % (rrd.rra_def[i].pdp_cnt
 376                                                     *
 377                                                     rrd.stat_head->
 378                                                     pdp_step)));
 379             cal_start =
 380                 (cal_end -
 381                  (rrd.rra_def[i].pdp_cnt * rrd.rra_def[i].row_cnt *
 382                   rrd.stat_head->pdp_step));
 383
 384             full_match = *end - *start;
 385 #ifdef DEBUG
 386             fprintf(stderr, "Considering: start %10lu end %10lu step %5lu ",
 387                     cal_start, cal_end,
 388                     rrd.stat_head->pdp_step * rrd.rra_def[i].pdp_cnt);
 389 #endif
 390             /* we need step difference in either full or partial case */
 391             tmp_step_diff =
 392                 labs((long) *step -
 393                      ((long) rrd.stat_head->pdp_step *
 394                       (long) rrd.rra_def[i].pdp_cnt));
 395             /* best full match */
 396             if (cal_start <= *start) {
 397                 if (first_full || (tmp_step_diff < best_full_step_diff)) {
 398                     first_full = 0;
 399                     best_full_step_diff = tmp_step_diff;
 400                     best_full_rra = i;
 401 #ifdef DEBUG
 402                     fprintf(stderr, "best full match so far\n");
 403                 } else {
 404                     fprintf(stderr, "full match, not best\n");
 405 #endif
 406                 }
 407
 408             } else {
 409                 /* best partial match */
 410                 tmp_match = full_match;
 411                 if (cal_start > *start)
 412                     tmp_match -= (cal_start - *start);
 413                 if (first_part ||
 414                     (best_match < tmp_match) ||
 415                     (best_match == tmp_match &&
 416                      tmp_step_diff < best_part_step_diff)) {
 417 #ifdef DEBUG
 418                     fprintf(stderr, "best partial so far\n");
 419 #endif
 420                     first_part = 0;
 421                     best_match = tmp_match;
 422                     best_part_step_diff = tmp_step_diff;
 423                     best_part_rra = i;
 424                 } else {
 425 #ifdef DEBUG
 426                     fprintf(stderr, "partial match, not best\n");
 427 #endif
 428                 }
 429             }
 430         }
 431     }
 432
 433     /* lets see how the matching went. */
 434     if (first_full == 0)
 435         chosen_rra = best_full_rra;
 436     else if (first_part == 0)
 437         chosen_rra = best_part_rra;
 438     else {
 439         rrd_set_error
 440             ("the RRD does not contain an RRA matching the chosen CF");
 441         goto err_free_all_ds_namv;
 442     }
 443
 444     /* set the wish parameters to their real values */
 445     *step = rrd.stat_head->pdp_step * rrd.rra_def[chosen_rra].pdp_cnt;
 446     *start -= (*start % *step);
 447     *end += (*step - *end % *step);
 448     rows = (*end - *start) / *step + 1;
 449
 450 #ifdef DEBUG
 451     fprintf(stderr,
 452             "We found:    start %10lu end %10lu step %5lu rows  %lu\n",
 453             *start, *end, *step, rows);
 454 #endif
 455
 456 /* Start and end are now multiples of the step size.  The amount of
 457 ** steps we want is (end-start)/step and *not* an extra one.
 458 ** Reasoning:  if step is s and we want to graph from t to t+s,
 459 ** we need exactly ((t+s)-t)/s rows.  The row to collect from the
 460 ** database is the one with time stamp (t+s) which means t to t+s.
 461 */
 462     *ds_cnt = rrd.stat_head->ds_cnt;
 463     if (((*data) = (rrd_value_t*)malloc(*ds_cnt * rows * sizeof(rrd_value_t))) == NULL) {
 464         rrd_set_error("malloc fetch data area");
 465         goto err_free_all_ds_namv;
 466     }
 467
 468     data_ptr = (*data);
 469
 470     /* find base address of rra */
 471     rra_base = rrd_file->header_len;
 472     for (i = 0; i < chosen_rra; i++)
 473         rra_base += (*ds_cnt * rrd.rra_def[i].row_cnt * sizeof(rrd_value_t));
 474
 475     /* find start and end offset */
 476     rra_end_time = (rrd.live_head->last_up
 477                     - (rrd.live_head->last_up % *step));
 478     rra_start_time = (rra_end_time
 479                       - (*step * (rrd.rra_def[chosen_rra].row_cnt - 1)));
 480     /* here's an error by one if we don't be careful */
 481     start_offset = ((long long)*start + (long long)*step - (long long)rra_start_time) / (long long) *step;
 482     end_offset = ((long long)rra_end_time - (long long)*end) / (long long) *step;
 483 #ifdef DEBUG
 484     fprintf(stderr,
 485             "start %10lu step %10lu rra_start %lld, rra_end %lld, start_off %lld, end_off %lld\n",
 486             *start, *step,(long long)rra_start_time, (long long)rra_end_time, (long long)start_offset, (long long)end_offset);
 487 #endif
 488     /* only seek if the start time is before the end time */
 489     if (*start <= rra_end_time && *end >= rra_start_time - (off_t)*step ){
 490         if (start_offset <= 0)
 491             rra_pointer = rrd.rra_ptr[chosen_rra].cur_row + 1;
 492         else
 493             rra_pointer = rrd.rra_ptr[chosen_rra].cur_row + 1 + start_offset;
 494
 495         rra_pointer = rra_pointer % (signed) rrd.rra_def[chosen_rra].row_cnt;
 496
 497         if (rrd_seek(rrd_file, (rra_base + (rra_pointer * (*ds_cnt)
 498                                         * sizeof(rrd_value_t))),
 499                  SEEK_SET) != 0) {
 500             rrd_set_error("seek error in RRA");
 501             goto err_free_data;
 502         }
 503 #ifdef DEBUG
 504         fprintf(stderr, "First Seek: rra_base %lu rra_pointer %lu\n",
 505                 rra_base, rra_pointer);
 506 #endif
 507     }
 508
 509     /* step trough the array */
 510
 511     for (i = start_offset;
 512          i < (signed) rrd.rra_def[chosen_rra].row_cnt - end_offset; i++) {
 513         /* no valid data yet */
 514         if (i < 0) {
 515 #ifdef DEBUG
 516             fprintf(stderr, "pre fetch %li -- ", i);
 517 #endif
 518             for (ii = 0; (unsigned) ii < *ds_cnt; ii++) {
 519                 *(data_ptr++) = DNAN;
 520 #ifdef DEBUG
 521                 fprintf(stderr, "%10.2f ", *(data_ptr - 1));
 522 #endif
 523             }
 524         }
 525         /* past the valid data area */
 526         else if (i >= (signed) rrd.rra_def[chosen_rra].row_cnt) {
 527 #ifdef DEBUG
 528             fprintf(stderr, "past fetch %li -- ", i);
 529 #endif
 530             for (ii = 0; (unsigned) ii < *ds_cnt; ii++) {
 531                 *(data_ptr++) = DNAN;
 532 #ifdef DEBUG
 533                 fprintf(stderr, "%10.2f ", *(data_ptr - 1));
 534 #endif
 535             }
 536         } else {
 537             /* OK we are inside the valid area but the pointer has to
 538              * be wrapped*/
 539             if (rra_pointer >= (signed) rrd.rra_def[chosen_rra].row_cnt) {
 540                 rra_pointer -= rrd.rra_def[chosen_rra].row_cnt;
 541                 if (rrd_seek(rrd_file, (rra_base + rra_pointer * (*ds_cnt)
 542                                         * sizeof(rrd_value_t)),
 543                              SEEK_SET) != 0) {
 544                     rrd_set_error("wrap seek in RRA did fail");
 545                     goto err_free_data;
 546                 }
 547 #ifdef DEBUG
 548                 fprintf(stderr, "wrap seek ...\n");
 549 #endif
 550             }
 551
 552             if (rrd_read(rrd_file, data_ptr, sizeof(rrd_value_t) * (*ds_cnt))
 553                 != (ssize_t) (sizeof(rrd_value_t) * (*ds_cnt))) {
 554                 rrd_set_error("fetching cdp from rra");
 555                 goto err_free_data;
 556             }
 557 #ifdef DEBUG
 558             fprintf(stderr, "post fetch %li -- ", i);
 559             for (ii = 0; ii < *ds_cnt; ii++)
 560                 fprintf(stderr, "%10.2f ", *(data_ptr + ii));
 561 #endif
 562             data_ptr += *ds_cnt;
 563             rra_pointer++;
 564         }
 565 #ifdef DEBUG
 566         fprintf(stderr, "\n");
 567 #endif
 568
 569     }
 570
 571     rrd_close(rrd_file);
 572     rrd_free(&rrd);
 573     return (0);
 574   err_free_data:
 575     free(*data);
 576     *data = NULL;
 577   err_free_all_ds_namv:
 578     for (i = 0; (unsigned long) i < rrd.stat_head->ds_cnt; ++i)
 579         free((*ds_namv)[i]);
 580   err_free_ds_namv:
 581     free(*ds_namv);
 582   err_close:
 583     rrd_close(rrd_file);
 584   err_free:
 585     rrd_free(&rrd);
 586     return (-1);
 587 }