1 /*****************************************************************************
2 * RRDtool 1.8.0 Copyright by Tobi Oetiker, 1997-2022
3 *****************************************************************************
4 * rrd_open.c Open an RRD File
5 *****************************************************************************
7 *****************************************************************************/
11 #include <winsdkver.h> /* Defines _WIN32_MAXVER */
12 #if _WIN32_MAXVER >= 0x0602 /* _WIN32_WINNT_WIN8 */
24 #include "compat-cloexec.h"
27 #ifdef HAVE_BROKEN_MS_ASYNC
28 #include <sys/types.h>
33 #include "rrd_rados.h"
39 #define _LK_UNLCK 0 /* Unlock */
40 #define _LK_LOCK 1 /* Lock */
41 #define _LK_NBLCK 2 /* Non-blocking lock */
42 #define _LK_RLCK 3 /* "Same as _LK_NBLCK" */
43 #define _LK_NBRLCK 4 /* "Same as _LK_LOCK" */
46 #define LK_UNLCK _LK_UNLCK
47 #define LK_LOCK _LK_LOCK
48 #define LK_NBLCK _LK_NBLCK
49 #define LK_RLCK _LK_RLCK
50 #define LK_NBRLCK _LK_NBRLCK
52 /* Variables for CreateFileA(). Names of variables are according to
53 * https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-createfilea */
54 DWORD dwDesiredAccess
= 0;
55 DWORD dwCreationDisposition
= 0;
58 /* DEBUG 2 prints information obtained via mincore(2) */
60 /* do not calculate exact madvise hints but assume 1 page for headers and
61 * set DONTNEED for the rest, which is assumed to be data */
62 /* Avoid calling madvise on areas that were already hinted. May be beneficial if
63 * your syscalls are very slow */
66 /* the cast to void* is there to avoid this warning seen on ia64 with certain
67 versions of gcc: 'cast increases required alignment of target type'
69 #define __rrd_read_mmap(dst, dst_t, cnt) { \
70 size_t wanted = sizeof(dst_t)*(cnt); \
71 if (offset + wanted > rrd_file->file_len) { \
72 rrd_set_error("reached EOF while loading header " #dst); \
75 (dst) = (dst_t*)(void*) (data + offset); \
79 #define __rrd_read_seq(dst, dst_t, cnt) { \
80 size_t wanted = sizeof(dst_t)*(cnt); \
82 if ((dst = (dst_t*)malloc(wanted)) == NULL) { \
83 rrd_set_error(#dst " malloc"); \
86 got = read (rrd_simple_file->fd, dst, wanted); \
87 if (got != wanted) { \
88 rrd_set_error("short read while reading header " #dst); \
96 #define __rrd_read_rados(dst, dst_t, cnt) { \
97 size_t wanted = sizeof(dst_t)*(cnt); \
99 if ((dst = (dst_t*)malloc(wanted)) == NULL) { \
100 rrd_set_error(#dst " malloc"); \
103 got = rrd_rados_read(rrd_file->rados, dst, wanted, offset); \
104 if (got != wanted) { \
105 rrd_set_error("short read while reading header " #dst); \
112 #if defined(HAVE_LIBRADOS) && defined(HAVE_MMAP)
113 #define __rrd_read(dst, dst_t, cnt) { \
114 if (rrd_file->rados) \
115 __rrd_read_rados(dst, dst_t, cnt) \
117 __rrd_read_mmap(dst, dst_t, cnt) \
119 #elif defined(HAVE_LIBRADOS) && !defined(HAVE_MMAP)
120 #define __rrd_read(dst, dst_t, cnt) { \
121 if (rrd_file->rados) \
122 __rrd_read_rados(dst, dst_t, cnt) \
124 __rrd_read_seq(dst, dst_t, cnt) \
126 #elif defined(HAVE_MMAP)
127 #define __rrd_read(dst, dst_t, cnt) \
128 __rrd_read_mmap(dst, dst_t, cnt)
130 #define __rrd_read(dst, dst_t, cnt) \
131 __rrd_read_seq(dst, dst_t, cnt)
134 /* get the address of the start of this page */
135 #if defined USE_MADVISE || defined HAVE_POSIX_FADVISE
137 #define PAGE_START(addr) ((addr)&(~(_page_size-1)))
141 static int rrd_rwlock(
142 rrd_file_t
*rrd_file
,
145 static int close_and_unlock(
148 /* Open a database file, return its header and an open filehandle,
149 * positioned to the first cdp in the first rra.
150 * In the error path of rrd_open, only rrd_free(&rrd) has to be called
151 * before returning an error. Do not call rrd_close upon failure of rrd_open.
152 * If creating a new file, the parameter rrd must be initialized with
153 * details of the file content.
154 * If opening an existing file, then use rrd must be initialized by
155 * rrd_init(rrd) prior to invoking rrd_open
158 rrd_file_t
*rrd_open(
159 const char *const file_name
,
168 char *data
= MAP_FAILED
;
172 rrd_file_t
*rrd_file
= NULL
;
173 rrd_simple_file_t
*rrd_simple_file
= NULL
;
174 size_t newfile_size
= 0;
176 if ((rdwr
& RRD_LOCK_MASK
) == RRD_LOCK_DEFAULT
) {
177 rdwr
&= ~RRD_LOCK_MASK
;
178 rdwr
|= _rrd_lock_flags(_rrd_lock_default());
181 /* Are we creating a new file? */
182 if (rdwr
& RRD_CREAT
) {
183 size_t header_len
, value_cnt
, data_len
;
185 header_len
= rrd_get_header_size(rrd
);
188 for (ui
= 0; ui
< rrd
->stat_head
->rra_cnt
; ui
++)
189 value_cnt
+= rrd
->stat_head
->ds_cnt
* rrd
->rra_def
[ui
].row_cnt
;
191 data_len
= sizeof(rrd_value_t
) * value_cnt
;
193 newfile_size
= header_len
+ data_len
;
196 rrd_file
= (rrd_file_t
*) malloc(sizeof(rrd_file_t
));
197 if (rrd_file
== NULL
) {
198 rrd_set_error("allocating rrd_file descriptor for '%s'", file_name
);
201 memset(rrd_file
, 0, sizeof(rrd_file_t
));
204 rrd_file
->pvt
= malloc(sizeof(rrd_simple_file_t
));
205 if (rrd_file
->pvt
== NULL
) {
206 rrd_set_error("allocating rrd_simple_file for '%s'", file_name
);
210 memset(rrd_file
->pvt
, 0, sizeof(rrd_simple_file_t
));
211 rrd_simple_file
= (rrd_simple_file_t
*) rrd_file
->pvt
;
212 rrd_simple_file
->fd
= -1;
215 if ((rdwr
& (RRD_READONLY
| RRD_READWRITE
)) ==
216 (RRD_READONLY
| RRD_READWRITE
)) {
217 /* Both READONLY and READWRITE were given, which is invalid. */
218 rrd_set_error("in read/write request mask");
225 if (strncmp("ceph//", file_name
, 6) == 0) {
226 rrd_file
->rados
= rrd_rados_open(file_name
+ 6);
227 if (rrd_file
->rados
== NULL
)
230 if (rdwr
& RRD_LOCK
) {
231 /* Note: rados read lock is not implemented. See rrd_lock(). */
232 if (rrd_rwlock(rrd_file
, rdwr
& RRD_READWRITE
) != 0) {
233 rrd_set_error("could not lock RRD");
238 if (rdwr
& RRD_CREAT
)
246 rrd_simple_file
->mm_prot
= PROT_READ
;
247 rrd_simple_file
->mm_flags
= 0;
250 if (rdwr
& RRD_READONLY
) {
253 dwDesiredAccess
= GENERIC_READ
;
254 dwCreationDisposition
= OPEN_EXISTING
;
258 rrd_simple_file
->mm_flags
= MAP_PRIVATE
;
260 # ifdef MAP_NORESERVE
261 rrd_simple_file
->mm_flags
|= MAP_NORESERVE
; /* readonly, so no swap backing needed */
265 if (rdwr
& RRD_READWRITE
) {
268 dwDesiredAccess
= GENERIC_READ
| GENERIC_WRITE
;
269 dwCreationDisposition
= OPEN_EXISTING
;
272 rrd_simple_file
->mm_flags
= MAP_SHARED
;
273 rrd_simple_file
->mm_prot
|= PROT_WRITE
;
276 if (rdwr
& RRD_CREAT
) {
277 flags
|= (O_CREAT
| O_TRUNC
);
279 dwDesiredAccess
= GENERIC_READ
| GENERIC_WRITE
;
280 dwCreationDisposition
= CREATE_ALWAYS
;
283 if (rdwr
& RRD_EXCL
) {
286 dwDesiredAccess
= GENERIC_READ
| GENERIC_WRITE
;
287 dwCreationDisposition
= CREATE_NEW
;
291 if (rdwr
& RRD_READAHEAD
) {
293 rrd_simple_file
->mm_flags
|= MAP_POPULATE
; /* populate ptes and data */
295 #if defined MAP_NONBLOCK
296 rrd_simple_file
->mm_flags
|= MAP_NONBLOCK
; /* just populate ptes */
299 #if defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)
304 /* In Windows we need FILE_SHARE_DELETE, so that the file can be
305 * renamed/replaced later on in rrd_create.c
306 * This is only possible using CreateFileA() first and not using open() alone */
310 CreateFileA(file_name
, dwDesiredAccess
,
311 FILE_SHARE_READ
| FILE_SHARE_WRITE
| FILE_SHARE_DELETE
,
312 NULL
, dwCreationDisposition
, FILE_ATTRIBUTE_NORMAL
, NULL
);
313 if (handle
== INVALID_HANDLE_VALUE
) {
314 LPVOID lpMsgBuf
= NULL
;
316 FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER
|
317 FORMAT_MESSAGE_FROM_SYSTEM
|
318 FORMAT_MESSAGE_IGNORE_INSERTS
, NULL
, GetLastError(), 0,
319 (LPTSTR
) & lpMsgBuf
, 0, NULL
);
320 rrd_set_error("opening '%s': %s", file_name
, (LPTSTR
) lpMsgBuf
);
324 if ((rrd_simple_file
->fd
= _open_osfhandle((intptr_t) handle
, flags
)) < 0) {
325 rrd_set_error("opening '%s': %s", file_name
, rrd_strerror(errno
));
329 if ((rrd_simple_file
->fd
= open(file_name
, flags
| O_CLOEXEC
, 0666)) < 0) {
330 rrd_set_error("opening '%s': %s", file_name
, rrd_strerror(errno
));
336 #ifdef HAVE_BROKEN_MS_ASYNC
337 if (rdwr
& RRD_READWRITE
) {
338 /* some unices, the files mtime does not get updated
339 on memory mapped files, in order to help them,
340 we update the timestamp at this point.
341 The thing happens pretty 'close' to the open
342 call so the chances of a race should be minimal.
344 Maybe ask your vendor to fix your OS ... */
345 utime(file_name
, NULL
);
350 if (rrd_rwlock(rrd_file
, rdwr
& RRD_READWRITE
, rdwr
& RRD_LOCK_MASK
) != 0) {
351 rrd_set_error("could not lock RRD");
355 /* Better try to avoid seeks as much as possible. stat may be heavy but
356 * many concurrent seeks are even worse. */
357 if (newfile_size
== 0 && ((fstat(rrd_simple_file
->fd
, &statb
)) < 0)) {
358 rrd_set_error("fstat '%s': %s", file_name
, rrd_strerror(errno
));
361 if (newfile_size
== 0) {
362 rrd_file
->file_len
= statb
.st_size
;
364 rrd_file
->file_len
= newfile_size
;
365 #ifdef HAVE_POSIX_FALLOCATE
366 /* man: posix_fallocate() returns zero on success,
367 * or an error number on failure. Note that errno is not set.
370 posix_fallocate(rrd_simple_file
->fd
, 0, newfile_size
);
371 /* ZFS (on FreeBSD) does not support posix_fallocate(), always returning
372 * EINVAL. Ignore this error and continue anyway.
373 * Without this, resize isn't possible on ZFS filesystems.
375 if (fret
== EINVAL
) {
378 rrd_set_error("posix_fallocate '%s': %s", file_name
,
382 goto no_lseek_necessary
;
385 if (lseek(rrd_simple_file
->fd
, newfile_size
- 1, SEEK_SET
) == -1) {
386 rrd_set_error("lseek '%s': %s", file_name
, rrd_strerror(errno
));
389 if (write(rrd_simple_file
->fd
, "\0", 1) == -1) { /* poke */
390 rrd_set_error("write '%s': %s", file_name
, rrd_strerror(errno
));
393 if (lseek(rrd_simple_file
->fd
, 0, SEEK_SET
) == -1) {
394 rrd_set_error("lseek '%s': %s", file_name
, rrd_strerror(errno
));
398 #ifdef HAVE_POSIX_FALLOCATE
403 #ifndef HAVE_POSIX_FALLOCATE
404 /* force allocating the file on the underlying filesystem to prevent any
405 * future bus error when the filesystem is full and attempting to write
406 * trough the file mapping. Filling the file using memset on the file
407 * mapping can also lead some bus error, so we use the old fashioned
410 if (rdwr
& RRD_CREAT
) {
414 memset(buf
, DNAN
, sizeof buf
);
415 lseek(rrd_simple_file
->fd
, offset
, SEEK_SET
);
417 for (i
= 0; i
< (newfile_size
- 1) / sizeof buf
; ++i
) {
418 if (write(rrd_simple_file
->fd
, buf
, sizeof buf
) == -1) {
419 rrd_set_error("write '%s': %s", file_name
,
420 rrd_strerror(errno
));
425 if (write(rrd_simple_file
->fd
, buf
,
426 (newfile_size
- 1) % sizeof buf
) == -1) {
427 rrd_set_error("write '%s': %s", file_name
, rrd_strerror(errno
));
431 lseek(rrd_simple_file
->fd
, 0, SEEK_SET
);
435 data
= mmap(0, rrd_file
->file_len
,
436 rrd_simple_file
->mm_prot
, rrd_simple_file
->mm_flags
,
437 rrd_simple_file
->fd
, offset
);
439 /* lets see if the first read worked */
440 if (data
== MAP_FAILED
) {
441 rrd_set_error("mmaping file '%s': %s", file_name
,
442 rrd_strerror(errno
));
445 rrd
->__mmap_start
= data
;
446 rrd
->__mmap_size
= rrd_file
->file_len
;
448 rrd_simple_file
->file_start
= data
;
450 if (rdwr
& RRD_CREAT
)
453 if (rdwr
& RRD_READAHEAD
) {
454 /* If perfect READAHEAD is not achieved for whatever reason, caller
455 will not thank us for advising the kernel of RANDOM access below. */
458 /* In general we need no read-ahead when dealing with rrd_files.
459 When we stop reading, it is highly unlikely that we start up again.
460 In this manner we actually save time and disk access (and buffer cache).
461 Thanks to Dave Plonka for the Idea of using POSIX_FADV_RANDOM here. */
463 if (rdwr
& RRD_COPY
) {
464 /* We will read everything in a moment (copying) */
465 madvise(data
, rrd_file
->file_len
, MADV_SEQUENTIAL
);
467 /* We do not need to read anything in for the moment */
468 madvise(data
, rrd_file
->file_len
, MADV_RANDOM
);
471 #if !defined(HAVE_MMAP) && defined(HAVE_POSIX_FADVISE)
472 if (rdwr
& RRD_COPY
) {
473 posix_fadvise(rrd_simple_file
->fd
, 0, 0, POSIX_FADV_SEQUENTIAL
);
475 posix_fadvise(rrd_simple_file
->fd
, 0, 0, POSIX_FADV_RANDOM
);
483 __rrd_read(rrd
->stat_head
, stat_head_t
,
486 /* lets do some test if we are on track ... */
487 if (memcmp(rrd
->stat_head
->cookie
, RRD_COOKIE
, sizeof(RRD_COOKIE
)) != 0) {
488 rrd_set_error("'%s' is not an RRD file", file_name
);
492 if (rrd
->stat_head
->float_cookie
!= FLOAT_COOKIE
) {
493 rrd_set_error("This RRD was created on another architecture");
497 version
= atoi(rrd
->stat_head
->version
);
499 if (version
> atoi(RRD_VERSION5
)) {
500 rrd_set_error("can't handle RRD file version %s",
501 rrd
->stat_head
->version
);
504 __rrd_read(rrd
->ds_def
, ds_def_t
,
505 rrd
->stat_head
->ds_cnt
);
507 __rrd_read(rrd
->rra_def
, rra_def_t
,
508 rrd
->stat_head
->rra_cnt
);
510 /* handle different format for the live_head */
512 rrd
->live_head
= (live_head_t
*) malloc(sizeof(live_head_t
));
513 if (rrd
->live_head
== NULL
) {
514 rrd_set_error("live_head_t malloc");
517 __rrd_read(rrd
->legacy_last_up
, time_t,
520 rrd
->live_head
->last_up
= *rrd
->legacy_last_up
;
521 rrd
->live_head
->last_up_usec
= 0;
523 __rrd_read(rrd
->live_head
, live_head_t
,
526 __rrd_read(rrd
->pdp_prep
, pdp_prep_t
,
527 rrd
->stat_head
->ds_cnt
);
528 __rrd_read(rrd
->cdp_prep
, cdp_prep_t
,
529 rrd
->stat_head
->rra_cnt
* rrd
->stat_head
->ds_cnt
);
530 __rrd_read(rrd
->rra_ptr
, rra_ptr_t
,
531 rrd
->stat_head
->rra_cnt
);
533 rrd_file
->header_len
= offset
;
534 rrd_file
->pos
= offset
;
536 #if defined(HAVE_MMAP) && defined(USE_MADVISE)
537 if (data
!= MAP_FAILED
) {
538 /* MADV_SEQUENTIAL mentions drop-behind. Override it for the header
539 * now we've read it, in case anyone implemented drop-behind.
541 * Do *not* fall back to fadvise() for !HAVE_MMAP. In that case,
542 * we've copied the header and will not read it again. Doing e.g.
543 * FADV_NORMAL on Linux (4.12) on *any* region would negate the
544 * effect of previous FADV_SEQUENTIAL.
546 madvise(data
, sysconf(_SC_PAGESIZE
), MADV_NORMAL
);
547 madvise(data
, sysconf(_SC_PAGESIZE
), MADV_WILLNEED
);
552 unsigned long row_cnt
= 0;
554 for (ui
= 0; ui
< rrd
->stat_head
->rra_cnt
; ui
++)
555 row_cnt
+= rrd
->rra_def
[ui
].row_cnt
;
557 size_t correct_len
= rrd_file
->header_len
+
558 sizeof(rrd_value_t
) * row_cnt
* rrd
->stat_head
->ds_cnt
;
561 /* skip length checking for rados file */
562 if (rrd_file
->rados
) {
563 rrd_file
->file_len
= correct_len
;
567 if (correct_len
> rrd_file
->file_len
) {
568 rrd_set_error("'%s' is too small (should be %ld bytes)",
569 file_name
, (long long) correct_len
);
572 if (rdwr
& RRD_READVALUES
) {
573 __rrd_read(rrd
->rrd_value
, rrd_value_t
,
574 row_cnt
* rrd
->stat_head
->ds_cnt
);
576 if (rrd_seek(rrd_file
, rrd_file
->header_len
, SEEK_SET
) != 0)
586 if (data
!= MAP_FAILED
)
587 munmap(data
, rrd_file
->file_len
);
591 rrd_rados_close(rrd_file
->rados
);
593 if (rrd_simple_file
->fd
>= 0) {
594 /* keep the original error */
595 char *e
= strdup(rrd_get_error());
597 close_and_unlock(rrd_simple_file
->fd
);
603 rrd_set_error("error message was lost (out of memory)");
612 #if defined DEBUG && DEBUG > 1
613 /* Print list of in-core pages of a the current rrd_file. */
616 rrd_file_t
*rrd_file
,
619 rrd_simple_file_t
*rrd_simple_file
;
621 rrd_simple_file
= (rrd_simple_file_t
*) rrd_file
->pvt
;
623 /* pretty print blocks in core */
626 ssize_t _page_size
= sysconf(_SC_PAGESIZE
);
628 off
= rrd_file
->file_len
+
629 ((rrd_file
->file_len
+ _page_size
- 1) / _page_size
);
633 if (mincore(rrd_simple_file
->file_start
, rrd_file
->file_len
, vec
) ==
636 unsigned is_in
= 0, was_in
= 0;
638 for (off
= 0, prev
= 0; off
< rrd_file
->file_len
; ++off
) {
639 is_in
= vec
[off
] & 1; /* if lsb set then is core resident */
642 if (was_in
!= is_in
) {
643 fprintf(stderr
, "%s: %sin core: %p len %ld\n", mark
,
644 was_in
? "" : "not ", vec
+ prev
, off
- prev
);
650 "%s: %sin core: %p len %ld\n", mark
,
651 was_in
? "" : "not ", vec
+ prev
, off
- prev
);
653 fprintf(stderr
, "mincore: %s", rrd_strerror(errno
));
656 fprintf(stderr
, "sorry mincore only works with mmap");
659 #endif /* defined DEBUG && DEBUG > 1 */
662 * get exclusive lock to whole file.
663 * lock gets removed when we close the file
665 * returns 0 on success
668 rrd_file_t
*rrd_file
)
670 return rrd_rwlock(rrd_file
, 1, RRD_LOCK_DEFAULT
);
673 #if defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)
674 #define USE_WINDOWS_LOCK 1
677 #ifdef USE_WINDOWS_LOCK
679 int rrd_windows_lock(
686 * _locking() is relative to fd position.
687 * We need to consistently lock bytes starting from 0,
688 * so we can successfully unlock on close.
690 * Note rrd_lock() API doesn't set a specific error message.
691 * Knowing that rrd_lock() (or even rrd_open()) failed should
692 * be specific enough, if someone manages to invoke rrdtool
693 * on something silly like a named pipe or COM1.
699 if (lseek(fd
, 0, SEEK_SET
) < 0)
703 ret
= _locking(fd
, _LK_NBLCK
, LONG_MAX
);
708 /* EACCES: someone else has the lock. */
711 * Wait 0.01 seconds before trying again. _locking()
712 * with _LK_LOCK would work similarly but waits 1 second
713 * between tries, which seems less desirable.
718 /* restore saved fd position */
719 if (lseek(fd
, pos
, SEEK_SET
) < 0)
727 int close_and_unlock(
732 #ifdef USE_WINDOWS_LOCK
734 * "If a process closes a file that has outstanding locks, the locks are
735 * unlocked by the operating system. However, the time it takes for the
736 * operating system to unlock these locks depends upon available system
737 * resources. Therefore, it is recommended that your process explicitly
738 * unlock all files it has locked when it terminates." (?!)
741 if (lseek(fd
, 0, SEEK_SET
) < 0) {
742 rrd_set_error("lseek: %s", rrd_strerror(errno
));
747 ret
= _locking(fd
, LK_UNLCK
, LONG_MAX
);
748 if (ret
!= 0 && errno
== EACCES
)
749 /* fd was not locked - this is entirely possible, ignore the error */
753 rrd_set_error("unlock file: %s", rrd_strerror(errno
));
757 if (close(fd
) != 0) {
759 rrd_set_error("closing file: %s", rrd_strerror(errno
));
767 rrd_file_t
*rrd_file
,
771 if (lock_mode
== RRD_LOCK_NONE
)
779 if (rrd_file
->rados
) {
781 * No read lock on rados. It would be complicated by the
782 * use of a short finite lock duration in rrd_rados_lock().
783 * Also rados does not provide blocking locks.
785 * Rados users may use snapshots if they need to
786 * e.g. obtain a consistent backup.
789 return rrd_rados_lock(rrd_file
->rados
);
795 rrd_simple_file_t
*rrd_simple_file
;
797 rrd_simple_file
= (rrd_simple_file_t
*) rrd_file
->pvt
;
798 #ifdef USE_WINDOWS_LOCK
799 /* _locking() does not support read locks; we always take a write lock */
800 rcstat
= rrd_windows_lock(rrd_simple_file
->fd
);
801 /* Silence unused parameter compiler warning */
806 int op
= lock_mode
== RRD_LOCK_TRY
? F_SETLK
: F_SETLKW
;
808 lock
.l_type
= writelock
? F_WRLCK
: /* exclusive write lock or */
809 F_RDLCK
; /* shared read lock */
810 lock
.l_len
= 0; /* whole file */
811 lock
.l_start
= 0; /* start of file */
812 lock
.l_whence
= SEEK_SET
; /* end of file */
814 rcstat
= fcntl(rrd_simple_file
->fd
, op
, &lock
);
823 /* drop cache except for the header and the active pages */
825 rrd_file_t
*rrd_file
,
828 rrd_simple_file_t
*rrd_simple_file
;
830 #if defined USE_MADVISE || defined HAVE_POSIX_FADVISE
831 size_t dontneed_start
;
835 ssize_t _page_size
= sysconf(_SC_PAGESIZE
);
837 if (rrd_file
== NULL
) {
838 #if defined DEBUG && DEBUG
839 fprintf(stderr
, "rrd_dontneed: Argument 'rrd_file' is NULL.\n");
843 rrd_simple_file
= (rrd_simple_file_t
*) rrd_file
->pvt
;
845 #if defined DEBUG && DEBUG > 1
846 mincore_print(rrd_file
, "before");
849 /* ignoring errors from RRDs that are smaller then the file_len+rounding */
850 rra_start
= rrd_file
->header_len
;
851 dontneed_start
= PAGE_START(rra_start
) + _page_size
;
852 for (i
= 0; i
< rrd
->stat_head
->rra_cnt
; ++i
) {
855 + rrd
->rra_ptr
[i
].cur_row
856 * rrd
->stat_head
->ds_cnt
* sizeof(rrd_value_t
));
857 if (active_block
> dontneed_start
) {
859 madvise(rrd_simple_file
->file_start
+ dontneed_start
,
860 active_block
- dontneed_start
- 1, MADV_DONTNEED
);
862 #ifdef HAVE_POSIX_FADVISE
863 /* in linux at least only fadvise DONTNEED seems to purge pages from cache */
864 posix_fadvise(rrd_simple_file
->fd
, dontneed_start
,
865 active_block
- dontneed_start
- 1,
866 POSIX_FADV_DONTNEED
);
870 dontneed_start
= active_block
;
871 /* do not release 'hot' block if update for this RAA will occur
872 * within 10 minutes */
873 if (rrd
->stat_head
->pdp_step
* rrd
->rra_def
[i
].pdp_cnt
-
874 rrd
->live_head
->last_up
% (rrd
->stat_head
->pdp_step
*
875 rrd
->rra_def
[i
].pdp_cnt
) < 10 * 60) {
876 dontneed_start
+= _page_size
;
879 rrd
->rra_def
[i
].row_cnt
* rrd
->stat_head
->ds_cnt
*
883 if (dontneed_start
< rrd_file
->file_len
) {
885 madvise(rrd_simple_file
->file_start
+ dontneed_start
,
886 rrd_file
->file_len
- dontneed_start
, MADV_DONTNEED
);
888 #ifdef HAVE_POSIX_FADVISE
889 posix_fadvise(rrd_simple_file
->fd
, dontneed_start
,
890 rrd_file
->file_len
- dontneed_start
,
891 POSIX_FADV_DONTNEED
);
896 #if defined DEBUG && DEBUG > 1
897 mincore_print(rrd_file
, "after");
899 #else /* #if defined USE_MADVISE || defined HAVE_POSIX_FADVISE */
900 /* Silence compiler warnings about unused variables and parameters */
901 (void) rrd_simple_file
;
904 #endif /* without madvise and posix_fadvise it does not make much sense todo anything */
912 rrd_file_t
*rrd_file
)
914 rrd_simple_file_t
*rrd_simple_file
;
916 rrd_simple_file
= (rrd_simple_file_t
*) rrd_file
->pvt
;
920 if (rrd_file
->rados
) {
921 if (rrd_rados_close(rrd_file
->rados
) != 0)
926 if (rrd_simple_file
->file_start
!= NULL
) {
927 if (munmap(rrd_simple_file
->file_start
, rrd_file
->file_len
) != 0) {
929 rrd_set_error("munmap rrd_file: %s", rrd_strerror(errno
));
933 if (rrd_simple_file
->fd
>= 0) {
934 if (close_and_unlock(rrd_simple_file
->fd
) != 0)
943 /* Set position of rrd_file. */
946 rrd_file_t
*rrd_file
,
951 /* no seek for rados */
952 if (rrd_file
->rados
) {
961 rrd_simple_file_t
*rrd_simple_file
;
963 rrd_simple_file
= (rrd_simple_file_t
*) rrd_file
->pvt
;
967 if (whence
== SEEK_SET
)
969 else if (whence
== SEEK_CUR
)
970 rrd_file
->pos
+= off
;
971 else if (whence
== SEEK_END
)
972 rrd_file
->pos
= rrd_file
->file_len
+ off
;
974 ret
= lseek(rrd_simple_file
->fd
, off
, whence
);
976 rrd_set_error("lseek: %s", rrd_strerror(errno
));
979 /* mimic fseek, which returns 0 upon success */
980 return ret
< 0; /*XXX: or just ret to mimic lseek */
984 /* Get current position in rrd_file. */
987 rrd_file_t
*rrd_file
)
989 return rrd_file
->pos
;
993 /* Read count bytes into buffer buf, starting at rrd_file->pos.
994 * Returns the number of bytes read or <0 on error. */
997 rrd_file_t
*rrd_file
,
1001 #ifdef HAVE_LIBRADOS
1002 if (rrd_file
->rados
) {
1004 rrd_rados_read(rrd_file
->rados
, buf
, count
, rrd_file
->pos
);
1006 rrd_file
->pos
+= ret
;
1010 rrd_simple_file_t
*rrd_simple_file
= (rrd_simple_file_t
*) rrd_file
->pvt
;
1013 size_t _cnt
= count
;
1016 if (rrd_file
->pos
> rrd_file
->file_len
|| _cnt
== 0) /* EOF */
1019 return -1; /* EINVAL */
1020 _surplus
= rrd_file
->pos
+ _cnt
- rrd_file
->file_len
;
1021 if (_surplus
> 0) { /* short read */
1026 buf
= memcpy(buf
, rrd_simple_file
->file_start
+ rrd_file
->pos
, _cnt
);
1028 rrd_file
->pos
+= _cnt
; /* mimic read() semantics */
1033 ret
= read(rrd_simple_file
->fd
, buf
, count
);
1035 rrd_file
->pos
+= ret
; /* mimic read() semantics */
1041 /* Write count bytes from buffer buf to the current position
1042 * rrd_file->pos of rrd_simple_file->fd.
1043 * Returns the number of bytes written or <0 on error. */
1046 rrd_file_t
*rrd_file
,
1050 #ifdef HAVE_LIBRADOS
1051 if (rrd_file
->rados
) {
1053 rrd_rados_write(rrd_file
->rados
, buf
, count
, rrd_file
->pos
);
1055 rrd_file
->pos
+= count
;
1059 rrd_simple_file_t
*rrd_simple_file
= (rrd_simple_file_t
*) rrd_file
->pvt
;
1062 size_t old_size
= rrd_file
->file_len
;
1067 return -1; /* EINVAL */
1069 if ((rrd_file
->pos
+ count
) > old_size
) {
1071 ("attempting to write beyond end of file (%ld + %ld > %ld)",
1072 rrd_file
->pos
, count
, old_size
);
1075 /* can't use memcpy since the areas overlap when tuning */
1076 memmove(rrd_simple_file
->file_start
+ rrd_file
->pos
, buf
, count
);
1077 rrd_file
->pos
+= count
;
1078 return count
; /* mimic write() semantics */
1080 ssize_t _sz
= write(rrd_simple_file
->fd
, buf
, count
);
1083 rrd_file
->pos
+= _sz
;
1089 /* this is a leftover from the old days, it serves no purpose
1090 and is therefore turned into a no-op */
1092 rrd_file_t
UNUSED(*rrd_file
))
1096 /* Initialize RRD header. */
1101 rrd
->stat_head
= NULL
;
1103 rrd
->rra_def
= NULL
;
1104 rrd
->live_head
= NULL
;
1105 rrd
->legacy_last_up
= NULL
;
1106 rrd
->rra_ptr
= NULL
;
1107 rrd
->pdp_prep
= NULL
;
1108 rrd
->cdp_prep
= NULL
;
1109 rrd
->rrd_value
= NULL
;
1110 rrd
->__mmap_start
= NULL
;
1111 rrd
->__mmap_size
= 0;
1115 /* free RRD data, act correctly, regardless of mmap'ped or malloc'd memory. */
1116 static void free_rrd_ptr_if_not_mmapped(
1123 if (rrd
== NULL
|| rrd
->__mmap_start
== NULL
) {
1128 /* is this ALWAYS correct on all supported platforms ??? */
1129 long ofs
= (char *) m
- (char *) rrd
->__mmap_start
;
1131 if (ofs
< rrd
->__mmap_size
) {
1132 // DO NOT FREE, this memory is mmapped!!
1145 free_rrd_ptr_if_not_mmapped(rrd
->live_head
, rrd
);
1146 rrd
->live_head
= NULL
;
1147 free_rrd_ptr_if_not_mmapped(rrd
->stat_head
, rrd
);
1148 rrd
->stat_head
= NULL
;
1149 free_rrd_ptr_if_not_mmapped(rrd
->ds_def
, rrd
);
1151 free_rrd_ptr_if_not_mmapped(rrd
->rra_def
, rrd
);
1152 rrd
->rra_def
= NULL
;
1153 free_rrd_ptr_if_not_mmapped(rrd
->rra_ptr
, rrd
);
1154 rrd
->rra_ptr
= NULL
;
1155 free_rrd_ptr_if_not_mmapped(rrd
->pdp_prep
, rrd
);
1156 rrd
->pdp_prep
= NULL
;
1157 free_rrd_ptr_if_not_mmapped(rrd
->cdp_prep
, rrd
);
1158 rrd
->cdp_prep
= NULL
;
1159 free_rrd_ptr_if_not_mmapped(rrd
->rrd_value
, rrd
);
1160 rrd
->rrd_value
= NULL
;
1163 /* routine used by external libraries to free memory allocated by
1173 * rra_update informs us about the RRAs being updated
1174 * The low level storage API may use this information for
1175 * aligning RRAs within stripes, or other performance enhancements
1177 void rrd_notify_row(
1178 rrd_file_t
UNUSED(*rrd_file
),
1179 int UNUSED(rra_idx
),
1180 unsigned long UNUSED(rra_row
),
1181 time_t UNUSED(rra_time
))
1186 * This function is called when creating a new RRD
1187 * The storage implementation can use this opportunity to select
1188 * a sensible starting row within the file.
1189 * The default implementation is random, to ensure that all RRAs
1190 * don't change to a new disk block at the same time
1192 unsigned long rrd_select_initial_row(
1193 rrd_file_t
UNUSED(*rrd_file
),
1194 int UNUSED(rra_idx
),
1197 return rrd_random() % rra
->row_cnt
;
1201 * Translates a string in a RRD_FLAGS_LOCKING_xxx constant.
1203 * Empty or non-existing strings are valid and will be mapped to a default
1206 * Functions returns -1 on unsupported values but does not emit diagnostics.
1208 static int _rrd_lock_parse(const char *opt
)
1210 /* non-existing and empty values */
1211 if (!opt
|| !opt
[0])
1212 /* the default locking mode */
1213 return RRD_FLAGS_LOCKING_MODE_TRY
;
1214 else if (strcmp(opt
, "try") == 0)
1215 return RRD_FLAGS_LOCKING_MODE_TRY
;
1216 else if (strcmp(opt
, "block") == 0)
1217 return RRD_FLAGS_LOCKING_MODE_BLOCK
;
1218 else if (strcmp(opt
, "none") == 0)
1219 return RRD_FLAGS_LOCKING_MODE_NONE
;
1225 * Returns the default locking method.
1227 * It reads the $RRD_LOCKING environment.
1229 * Function always succeeds; unsupported values will emit a
1230 * diagnostic and function returns a default value in this case.
1232 int _rrd_lock_default(void)
1234 const char *opt
= getenv("RRD_LOCKING");
1235 int flags
= _rrd_lock_parse(opt
);
1239 "unsupported locking mode '%s' in $RRD_LOCKING; assuming 'try'\n",
1241 return RRD_FLAGS_LOCKING_MODE_TRY
;
1248 * Translates a string to a RRD_FLAGS_LOCKING_xxx constant and updates flags.
1250 * Function will fail on unsupported values and return -1. It sets rrd_set_error()
1253 * Else, the RRD_FLAGS_LOCKING_xxx related bits in 'out_flags' will be cleared
1254 * and updated. Function returns 0 then.
1256 int _rrd_lock_from_opt(int *out_flags
, const char *opt
)
1258 int flags
= _rrd_lock_parse(opt
);
1261 rrd_set_error("unsupported locking mode '%s'\n", opt
);
1265 *out_flags
&= ~RRD_FLAGS_LOCKING_MODE_MASK
;
1266 *out_flags
|= flags
;
1272 * Translates RRD_FLAGS_LOCKING_MODE_xxx to RRD_LOCK_xxx
1274 * Function removes unrelated bits from 'extra_flags' and maps it to the
1275 * RRD_LOCK_xxx constants.
1277 int _rrd_lock_flags(int extra_flags
)
1279 /* Due to legacy reasons, we have to map this manually.
1281 * E.g. RRD_LOCK_DEFAULT (which might be used by deprecated direct calls
1282 * to rrd_open()) must be non-zero. But RRD_FLAGS_LOCKING_MODE_DEFAULT
1283 * must be 0 because not all users of the updatex api might have been
1286 switch (extra_flags
& RRD_FLAGS_LOCKING_MODE_MASK
) {
1287 case RRD_FLAGS_LOCKING_MODE_NONE
:
1288 return RRD_LOCK_NONE
;
1289 case RRD_FLAGS_LOCKING_MODE_TRY
:
1290 return RRD_LOCK_TRY
;
1291 case RRD_FLAGS_LOCKING_MODE_BLOCK
:
1292 return RRD_LOCK_BLOCK
;
1293 case RRD_FLAGS_LOCKING_MODE_DEFAULT
:
1294 return RRD_LOCK_DEFAULT
;