libsmartcols: (filter) Add on-demand data filler

When dealing with really large data sets, we need to reduce overhead
by filtering out unnecessary lines. The traditional approach, using:

 lstool | grep <expr>

requires the tool to read all data from the system and then filter out
(and throw away) a lot of data.

The filter-filler now allows us to use an empty line. The filter will
request data through a callback, and when a line passes the filter,
the application can fill in the rest of the columns.

For example, in a query like "FOO > 10 && BAR < 10," libsmartcols will
never ask for "BAR" if "FOO" is smaller than 10. This means the
application doesn't have to gather additional columns.

Signed-off-by: Karel Zak <kzak@redhat.com>

diff --git a/libsmartcols/src/cell.c b/libsmartcols/src/cell.c
index f6ceff21d77f2936aeb5a5527b87f2b02791f2c5..0ffbfd12e4839aa841c683ade0c106a68857da7f 100644 (file)
--- a/libsmartcols/src/cell.c
+++ b/libsmartcols/src/cell.c
@@ -67,6 +67,7 @@ int scols_cell_set_data(struct libscols_cell *ce, const char *data)
        if (!ce)
                return -EINVAL;
 
+       ce->is_filled = 1;
        rc = strdup_to_struct_member(ce, data, data);
        ce->datasiz = ce->data ? strlen(ce->data) + 1: 0;
        return rc;
@@ -91,6 +92,7 @@ int scols_cell_refer_data(struct libscols_cell *ce, char *data)
        free(ce->data);
        ce->data = data;
        ce->datasiz = ce->data ? strlen(ce->data) + 1: 0;
+       ce->is_filled = 1;
        return 0;
 }
 

diff --git a/libsmartcols/src/filter-param.c b/libsmartcols/src/filter-param.c
index 295695e0a6b32b81865ed552120b78270cc45de3..78b6e97c8881cb3a87bb6dbd672f08c2a4039c1c 100644 (file)
--- a/libsmartcols/src/filter-param.c
+++ b/libsmartcols/src/filter-param.c
@@ -263,6 +263,12 @@ static int fetch_holder_data(struct libscols_filter *fltr __attribute__((__unuse
        }
        DBG(FPARAM, ul_debugobj(n, "fetching %s data", n->holder_name));
 
+       if (fltr->filler_cb && !scols_line_is_filled(ln, n->col->seqnum)) {
+               rc = fltr->filler_cb(fltr, ln, n->col->seqnum, fltr->filler_data);
+               if (rc)
+                       return rc;
+       }
+
        /* read column data, use it as string */
        data = scols_line_get_column_data(ln, n->col);
        rc = param_set_data(n, F_DATA_STRING, data);

diff --git a/libsmartcols/src/filter.c b/libsmartcols/src/filter.c
index 8d9e8a4a110f68fa924ca354af6d2cd1897e44a9..e8d88d79983a020f2a23164b9218b504e01aa371 100644 (file)
--- a/libsmartcols/src/filter.c
+++ b/libsmartcols/src/filter.c
@@ -188,3 +188,16 @@ int scols_line_apply_filter(struct libscols_line *ln,
        DBG(FLTR, ul_debugobj(fltr, "filter done [rc=%d, status=%d]", rc, *status));
        return rc;
 }
+
+int scols_filter_set_filler_cb(struct libscols_filter *fltr,
+                               int (*cb)(struct libscols_filter *,
+                                         struct libscols_line *, size_t, void *),
+                               void *userdata)
+{
+       if (!fltr)
+               return -EINVAL;
+       fltr->filler_cb = cb;
+       fltr->filler_data = userdata;
+
+       return 0;
+}

diff --git a/libsmartcols/src/libsmartcols.h.in b/libsmartcols/src/libsmartcols.h.in
index 90a3978a584794c6be930e1207e0e41ca37ed7bd..3305a037bccbed68616185198c18fe121d8c61dc 100644 (file)
--- a/libsmartcols/src/libsmartcols.h.in
+++ b/libsmartcols/src/libsmartcols.h.in
@@ -253,6 +253,7 @@ extern struct libscols_cell *scols_line_get_column_cell(
                                        struct libscols_column *cl);
 extern int scols_line_set_data(struct libscols_line *ln, size_t n, const char *data);
 extern int scols_line_refer_data(struct libscols_line *ln, size_t n, char *data);
+extern int scols_line_is_filled(struct libscols_line *ln, size_t n);
 extern int scols_line_set_column_data(struct libscols_line *ln, struct libscols_column *cl, const char *data);
 extern const char *scols_line_get_column_data(struct libscols_line *ln, struct libscols_column *cl);
 extern int scols_line_refer_column_data(struct libscols_line *ln, struct libscols_column *cl, char *data);
@@ -383,6 +384,10 @@ extern int scols_filter_next_holder(struct libscols_filter *fltr,
 extern int scols_filter_assign_column(struct libscols_filter *fltr,
                        struct libscols_iter *itr,
                         const char *name, struct libscols_column *col);
+extern int scols_filter_set_filler_cb(struct libscols_filter *fltr,
+                                int (*cb)(struct libscols_filter *,
+                                          struct libscols_line *, size_t, void *),
+                                void *userdata);
 
 #ifdef __cplusplus
 }

diff --git a/libsmartcols/src/libsmartcols.sym b/libsmartcols/src/libsmartcols.sym
index 2b170906937c4c72b0003d01e79bdbe02ec49a95..1cd7ec62645f3b679354e147c0fad2e67e804032 100644 (file)
--- a/libsmartcols/src/libsmartcols.sym
+++ b/libsmartcols/src/libsmartcols.sym
@@ -229,4 +229,6 @@ SMARTCOLS_2.40 {
        scols_filter_next_holder;
        scols_filter_assign_column;
        scols_line_apply_filter;
+       scols_filter_set_filler_cb;
+       scols_line_is_filled;
 } SMARTCOLS_2.39;

diff --git a/libsmartcols/src/line.c b/libsmartcols/src/line.c
index 813e6179a027fc6f5c900733fff957b3e9db83d7..2289db0f9ec4918c21923750ac575495ff570241 100644 (file)
--- a/libsmartcols/src/line.c
+++ b/libsmartcols/src/line.c
@@ -504,6 +504,20 @@ int scols_line_refer_data(struct libscols_line *ln, size_t n, char *data)
        return scols_cell_refer_data(ce, data);
 }
 
+/**
+ * scols_line_is_filled:
+ * @ln: a pointer to a struct libscols_line instance
+ * @n: number of the cell
+ *
+ * Returns: 0 or 1 if cell was already filled (note that NULL is also valid filler)
+ */
+int scols_line_is_filled(struct libscols_line *ln, size_t n)
+{
+       struct libscols_cell *ce = scols_line_get_cell(ln, n);
+
+       return ce ? ce->is_filled : 0;
+}
+
 /**
  * scols_line_refer_column_data:
  * @ln: a pointer to a struct libscols_line instance

diff --git a/libsmartcols/src/smartcolsP.h b/libsmartcols/src/smartcolsP.h
index f892855c7c9addad25e4364f995962227e525c45..45e281cb3d50d40f2bd970052114485709e73765 100644 (file)
--- a/libsmartcols/src/smartcolsP.h
+++ b/libsmartcols/src/smartcolsP.h
@@ -89,6 +89,8 @@ struct libscols_cell {
        void    *userdata;
        int     flags;
        size_t  width;
+
+       unsigned int is_filled : 1;
 };
 
 extern int scols_line_move_cells(struct libscols_line *ln, size_t newn, size_t oldn);
@@ -536,6 +538,9 @@ struct libscols_filter {
        struct filter_node *root;
        FILE *src;
 
+       int (*filler_cb)(struct libscols_filter *, struct libscols_line *, size_t, void *);
+       void *filler_data;
+
        struct list_head params;
 };
 
@@ -546,7 +551,6 @@ void filter_unref_node(struct filter_node *n);
 void filter_dump_node(struct ul_jsonwrt *json, struct filter_node *n);
 int filter_eval_node(struct libscols_filter *fltr, struct libscols_line *ln,
                        struct filter_node *n, int *status);
-
 /* param */
 int filter_compile_param(struct libscols_filter *fltr, struct filter_param *n);
 void filter_dump_param(struct ul_jsonwrt *json, struct filter_param *n);