From: Marek77 Date: Fri, 17 Feb 2017 15:33:16 +0000 (+0100) Subject: rrd_list: adds --recursive option (daemon: LIST [RECURSIVE] /) (#767) X-Git-Tag: v1.7.0~9 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c5bbebab6098c3329daf0599f64331ca2e111fa1;p=thirdparty%2Frrdtool-1.x.git rrd_list: adds --recursive option (daemon: LIST [RECURSIVE] /) (#767) * rrd_list: adds --recursive option (daemon: LIST [RECURSIVE] /) * tests/list1: updates LIST tests --- diff --git a/bindings/python/rrdtoolmodule.c b/bindings/python/rrdtoolmodule.c index 4a762abd..4eb7f394 100644 --- a/bindings/python/rrdtoolmodule.c +++ b/bindings/python/rrdtoolmodule.c @@ -770,7 +770,8 @@ static char _rrdtool_list__doc__[] = "List RRDs in storage.\n\n" \ "Usage: list(args..)\n\ Arguments:\n\n\ dirname\n\ - [--daemon HOST]"; + [-r|--recursive]\n\ + [-d|--daemon address]"; static PyObject * _rrdtool_list(PyObject *Py_UNUSED(self), PyObject *args) diff --git a/doc/rrdcached.pod b/doc/rrdcached.pod index 582a85de..bf022a39 100644 --- a/doc/rrdcached.pod +++ b/doc/rrdcached.pod @@ -744,10 +744,10 @@ message itself. The first user command after B is command number one. server: 1 message for command 1 server: 12 message for command 12 -=item B I +=item B [RECURSIVE] I/ This command allows to list directories and rrd databases as seen by the daemon. -The root "directory" is the base_dir (see '-b dir'). +The root "directory" is the base_dir (see '-b dir'). When invoked with 'LIST RECURSIVE /' it will behave similarly to 'ls -R' but limited to rrd files (listing all the rrd bases in the subtree of , skipping empty directories). =item B diff --git a/src/rrd.h b/src/rrd.h index 92b2d1fc..c42f915d 100644 --- a/src/rrd.h +++ b/src/rrd.h @@ -166,7 +166,7 @@ struct rrd_t; void rrd_info_free( rrd_info_t *); char *rrd_list(int, char **); - char *rrd_list_r(char *dirname); + char *rrd_list_r(int, char *dirname); int rrd_update( int, char **); diff --git a/src/rrd_client.c b/src/rrd_client.c index b219a374..d7137d88 100644 --- a/src/rrd_client.c +++ b/src/rrd_client.c @@ -1297,7 +1297,7 @@ rrd_info_t * rrdc_info (const char *filename) /* {{{ */ return info; } /* }}} int rrdc_info */ -char *rrd_client_list(rrd_client_t *client, const char *dirname) /* {{{ */ +char *rrd_client_list(rrd_client_t *client, int recursive, const char *dirname) /* {{{ */ { char buffer[RRD_CMD_MAX]; char *buffer_ptr; @@ -1327,6 +1327,15 @@ char *rrd_client_list(rrd_client_t *client, const char *dirname) /* {{{ */ return NULL; } + if (recursive) { + status = buffer_add_string ("RECURSIVE", &buffer_ptr, &buffer_free); + if (status != 0) + { + rrd_set_error ("rrdc_list: out of memory"); + return (NULL); + } + } + status = buffer_add_string (dirname, &buffer_ptr, &buffer_free); if (status != 0) { @@ -1396,11 +1405,11 @@ out_free_res: return list; } /* }}} char *rrd_client_list */ -char *rrdc_list(const char *dirname) /* {{{ */ +char *rrdc_list(int recursive, const char *dirname) /* {{{ */ { char *files; mutex_lock(&lock); - files = rrd_client_list(&default_client, dirname); + files = rrd_client_list(&default_client, recursive, dirname); mutex_unlock(&lock); return files; } /* }}} char *rrdc_list */ diff --git a/src/rrd_client.h b/src/rrd_client.h index b8c6a27e..c97463b5 100644 --- a/src/rrd_client.h +++ b/src/rrd_client.h @@ -74,7 +74,7 @@ int rrd_client_update(rrd_client_t *client, const char *filename, int values_num const char * const *values); rrd_info_t * rrd_client_info(rrd_client_t *client, const char *filename); -char *rrd_client_list(rrd_client_t *client, const char *dirname); +char *rrd_client_list(rrd_client_t *client, int recursive, const char *dirname); time_t rrd_client_last(rrd_client_t *client, const char *filename); time_t rrd_client_first(rrd_client_t *client, const char *filename, int rraindex); int rrd_client_create(rrd_client_t *client, const char *filename, @@ -121,7 +121,7 @@ int rrdc_update (const char *filename, int values_num, const char * const *values); rrd_info_t * rrdc_info (const char *filename); -char *rrdc_list(const char *dirname); +char *rrdc_list(int recursive, const char *dirname); time_t rrdc_last (const char *filename); time_t rrdc_first (const char *filename, int rraindex); int rrdc_create (const char *filename, diff --git a/src/rrd_daemon.c b/src/rrd_daemon.c index 3a8fe11f..23faeba0 100644 --- a/src/rrd_daemon.c +++ b/src/rrd_daemon.c @@ -2425,7 +2425,9 @@ done: static int handle_request_list (HANDLER_PROTO) /* {{{ */ { - char *filename; + char *filename = NULL; + char *rec = NULL; + int recursive = 0; char *list, *start_ptr, *end_ptr, *ptr; char fullpath[PATH_MAX], current[PATH_MAX], absolute[PATH_MAX]; char bwc[PATH_MAX], bwd[PATH_MAX]; @@ -2438,11 +2440,29 @@ static int handle_request_list (HANDLER_PROTO) /* {{{ */ return send_response(sock, RESP_ERR, "No base directory defined\n"); } - /* Get pathname */ + /* get 'RECURSIVE' option */ + status = buffer_get_field(&buffer, &buffer_size, &rec); + if (status == 0) { + /* as 'RECURSIVE' is optional, the first argument may be the filename */ + if (rec[0] != '/' && strcmp(rec, "RECURSIVE") != 0) { + return syntax_error(sock, cmd); + } + + if (rec[0] == '/') { + filename = rec; + + } else if (strcmp(rec, "RECURSIVE") == 0) { + recursive = 1; + } + } + + /* Get pathname if not done already */ + if (!filename) { status = buffer_get_field(&buffer, &buffer_size, &filename); if (status != 0) return syntax_error(sock,cmd); + } /* get full pathname */ snprintf(fullpath, PATH_MAX, "%s%s%s", @@ -2468,7 +2488,7 @@ static int handle_request_list (HANDLER_PROTO) /* {{{ */ base = &bwd[0]; } - list = rrd_list_r(fullpath); + list = rrd_list_r(recursive, fullpath); if (list == NULL) { /* Empty directory listing */ @@ -2772,11 +2792,14 @@ static command_t list_of_commands[] = { /* {{{ */ "LIST", handle_request_list, CMD_CONTEXT_CLIENT, - "LIST\n", - "This command lists the RRD files in the storage base directory.\n" + "LIST [RECURSIVE] /[]\n", + "This command lists the RRD files in the storage base directory (/).\n" "Note that this is the list of RRD files on storage as of the last update.\n" "There may be pending updates in the queue, so a FLUSH may have to be run\n" "beforehand.\n" + "When invoked with 'LIST RECURSIVE /' it will behave similarly to\n" + "'ls -R' but limited to rrd files (listing all the rrd bases in the subtree\n" + " of , skipping empty directories).\n" }, { "QUIT", diff --git a/src/rrd_list.c b/src/rrd_list.c index 8a1c7f64..84b96df5 100644 --- a/src/rrd_list.c +++ b/src/rrd_list.c @@ -8,34 +8,140 @@ #include "rrd_tool.h" #include "rrd_client.h" -char *rrd_list_r(char *dirname) +static char *move_past_prefix(const char *prefix, const char *string) { -#define SANE_ASPRINTF(_dest_str, _format, ...) \ + int index = 0; + + if (strlen(prefix) > strlen(string)) { + return (char *)string; + } + + while (prefix[index] != '\0') { + if (prefix[index] != string[index]) { + break; + } + index++; + } + + return (char *)&(string[index]); +} + +static char *rrd_list_rec(int recursive, char *root, char *dirname) +{ +#define SANE_ASPRINTF2(_dest_str, _format, ...) \ if (asprintf(&_dest_str, _format, __VA_ARGS__) == -1) { \ if (out != NULL) { \ free(out); \ } \ + closedir(dir); \ errno = ENOMEM; \ return NULL; \ } -#define SANE_ASPRINTF2(_dest_str, _format, ...) \ + + struct stat st; + struct dirent *entry; + DIR *dir; + char *out = NULL, *out_rec, *out_short, *tmp, *ptr; + char current[PATH_MAX], fullpath[PATH_MAX]; + + dir = opendir(dirname); + + if (dir == NULL) { + /* opendir sets errno */ + return NULL; + } + + while ((entry = readdir(dir)) != NULL) { + + if ((strcmp(entry->d_name, ".") == 0) || + (strcmp(entry->d_name, "..") == 0)) { + continue; + } + + if (strlen(dirname) + strlen(entry->d_name) + 1 >= PATH_MAX) { + continue; + } + + snprintf(¤t[0], PATH_MAX, "%s/%s", dirname, entry->d_name); + + /* NOTE: stat(2) follows symlinks and gives info on target. */ + if (stat(current, &st) != 0) { + continue; + } + + if (S_ISDIR(st.st_mode) && recursive) { + snprintf(&fullpath[0], PATH_MAX, "%s/%s", + dirname, entry->d_name); + out_rec = rrd_list_rec(recursive, root, fullpath); + + if (out_rec == NULL) { + continue; + } + + if (out == NULL) { + SANE_ASPRINTF2(out, "%s", out_rec); + + } else { + tmp = out; + SANE_ASPRINTF2(out, "%s%s", out, out_rec); + free(tmp); + } + free(out_rec); + + } else { + + if (S_ISREG(st.st_mode)) { + + ptr = strstr(entry->d_name, ".rrd"); + + /* strlen() used to make sure it's matching + * the end of string. Non-rrd-suffixed regular + * files are skipped. + */ + if (ptr == NULL || strlen(ptr) != 4) { + continue; + } + } + snprintf(&fullpath[0], PATH_MAX, "%s/%s", + dirname, entry->d_name); + out_short = move_past_prefix(root, fullpath); + + /* don't start output with a '/' */ + if (out_short[0] == '/') { + out_short++; + } + + if (out == NULL) { + SANE_ASPRINTF2(out, "%s\n", out_short); + + } else { + tmp = out; + SANE_ASPRINTF2(out, "%s%s\n", out, out_short); + free(tmp); + } + } + } + closedir(dir); + + return out; +} + +char *rrd_list_r(int recursive, char *dirname) +{ +#define SANE_ASPRINTF(_dest_str, _format, ...) \ if (asprintf(&_dest_str, _format, __VA_ARGS__) == -1) { \ if (out != NULL) { \ free(out); \ } \ - closedir(dir); \ errno = ENOMEM; \ return NULL; \ } char *out = NULL, *tmp; - char current[PATH_MAX]; glob_t buf; char *ptr; unsigned int i; struct stat st; - DIR *dir; - struct dirent *entry; /* Prevent moving up the directory tree */ if (strstr(dirname, "..")) { @@ -46,6 +152,12 @@ char *rrd_list_r(char *dirname) /* if filename contains wildcards, then use glob() */ if (strchr(dirname, '*') || strchr(dirname, '?')) { + /* recursive list + globbing forbidden */ + if (recursive) { + errno = EINVAL; + return NULL; + } + if (glob(dirname, 0, NULL, &buf)) { globfree(&buf); errno = ENOENT; @@ -94,61 +206,14 @@ char *rrd_list_r(char *dirname) } /* Process directory */ - dir = opendir(dirname); - - if (dir == NULL) { - /* opendir sets errno */ + if (stat(dirname, &st) != 0) { return NULL; } - while ((entry = readdir(dir)) != NULL) { - - if ((strcmp(entry->d_name, ".") == 0) || - (strcmp(entry->d_name, "..") == 0)) { - continue; - } - - if (strlen(dirname) + strlen(entry->d_name) + 1 >= PATH_MAX) { - continue; - } - snprintf(¤t[0], PATH_MAX, "%s/%s", dirname, entry->d_name); - - /* Only return directories and rrd files. - * NOTE: stat(2) follows symlinks and gives info on target. */ - if (stat(current, &st) != 0) { - continue; - } - - if (!S_ISDIR(st.st_mode)) { - - if (S_ISREG(st.st_mode)) { - - ptr = strstr(entry->d_name, ".rrd"); - - /* strlen() used to make sure it's matching - * the end of string. - */ - if (ptr == NULL || strlen(ptr) != 4) { - continue; - } - - } else { - continue; - } - } - - if (out == NULL) { - SANE_ASPRINTF2(out, "%s\n", entry->d_name); - - } else { - tmp = out; - SANE_ASPRINTF2(out, "%s%s\n", out, entry->d_name); - free(tmp); - } + if (!S_ISDIR(st.st_mode)) { + return NULL; } - closedir(dir); - - return out; + return rrd_list_rec(recursive, dirname, dirname); } char *rrd_list(int argc, char **argv) @@ -156,11 +221,13 @@ char *rrd_list(int argc, char **argv) char *opt_daemon = NULL; int status; int flushfirst = 1; + int recursive = 0; char *list; static struct optparse_long long_options[] = { {"daemon", 'd', OPTPARSE_REQUIRED}, {"noflush", 'F', OPTPARSE_NONE}, + {"recursive", 'r', OPTPARSE_NONE}, {0}, }; struct optparse options; @@ -188,6 +255,9 @@ char *rrd_list(int argc, char **argv) flushfirst = 0; break; + case 'r': + recursive=1; + break; case '?': if (opt_daemon) @@ -210,7 +280,7 @@ char *rrd_list(int argc, char **argv) } if ((argc - options.optind) != 1) { - rrd_set_error ("Usage: rrdtool %s [--daemon [--noflush]] ", + rrd_set_error ("Usage: rrdtool %s [--daemon [--noflush]] [--recursive] ", argv[0]); if (opt_daemon != NULL) { @@ -235,7 +305,7 @@ char *rrd_list(int argc, char **argv) rrdc_connect (opt_daemon); if (rrdc_is_connected (opt_daemon)) { - list = rrdc_list(argv[options.optind]); + list = rrdc_list(recursive, argv[options.optind]); rrdc_disconnect(); } else { @@ -250,7 +320,7 @@ char *rrd_list(int argc, char **argv) free(opt_daemon); return NULL; } - list = rrd_list_r(argv[options.optind]); + list = rrd_list_r(recursive, argv[options.optind]); if (list == NULL) { fprintf(stderr, "%s", strerror(errno)); diff --git a/tests/list1 b/tests/list1 index d4dc4f55..040d33d8 100755 --- a/tests/list1 +++ b/tests/list1 @@ -6,7 +6,7 @@ BASE=$BASEDIR/`basename $0` BUILD=$BUILDDIR/`basename $0` LIST_DIR=$BUILDDIR/`basename $0`_dir -# This is used both for 'direct' tests and for tests via rrdcached +# This is used both for 'direct' tests and for tests via rrdcached # (when RRDCACHED_ADDRESS is exported). In that case the 'root' directory # is BASEDIR (see '-b' in functions::run_cached) and the paths in tests # must be changed accordingly (see $LIST_TEST_DIR, $rrd) @@ -28,19 +28,29 @@ function do_list_tests() cp ${BUILD}.rrd "$LIST_TEST_DIR"/ cp ${BUILD}.rrd "$LIST_TEST_DIR"/second.rrd cp ${BUILD}.rrd "$LIST_TEST_DIR"/third.rrd - list_count=`$RRDTOOL list "$LIST_TEST_DIR" | wc -l` + + list_count=`$RRDTOOL list "/$1" | wc -l` test $list_count -eq 3 report "directory with several RRDs" touch "$LIST_TEST_DIR"/not_an_rrd - list_count=`$RRDTOOL list "$LIST_TEST_DIR" | wc -l` + list_count=`$RRDTOOL list "/$1" | wc -l` test $list_count -eq 3 report "only lists files with .rrd suffix" mkdir -p "$LIST_TEST_DIR"/new_dir - list_count=`$RRDTOOL list "$LIST_TEST_DIR" | wc -l` + list_count=`$RRDTOOL list "/$1" | wc -l` test $list_count -eq 4 report "only lists RRDs and directories" + + mkdir -p "$LIST_TEST_DIR"/new_dir2 + mkdir -p "$LIST_TEST_DIR"/new_dir3 + mkdir -p "$LIST_TEST_DIR"/new_dir4 + cp ${BUILD}.rrd "$LIST_TEST_DIR"/new_dir2/fourth.rrd + cp ${BUILD}.rrd "$LIST_TEST_DIR"/new_dir2/fifth.rrd + list_count=`$RRDTOOL list --recursive "/$1" | wc -l` + test $list_count -eq 5 + report "recursive list only lists rrd files" } ################################################################################ @@ -71,7 +81,7 @@ else fi if is_cached; then - mkdir -p "$LIST_DIR" + mkdir -p "$LIST_DIR" # This relies on '-b' setting in functions::run_cached() CACHED_DIR=`echo "$LIST_DIR" | sed "s|^$BASEDIR/||"` do_list_tests "$CACHED_DIR"