lib-sql: Add API support for asynchronously iterating over rows.

sql_query() can already do an async lookup, but the full result needs
to be available immediately. This can be inefficient for large results.
Add a new SQL_RESULT_NEXT_MORE return value and sql_result_more() for
asynchronously requesting more results.

This changes the API a bit, but isn't done by default by any drivers yet.
Also callers that can't handle this are hopefully checking for "ret < 0",
which allows them to handle such an async-more request as an error
instead.

sql_result_next_row() will be changed to return enum in a separate commit to
keep backwards compatibility in v2.2.x.

diff --git a/src/lib-sql/driver-cassandra.c b/src/lib-sql/driver-cassandra.c
index 87f75322f93ab75a690610f8ec8ca29fe372bfe5..a2213dfedee1f3bc871a843de7660d9fc164649a 100644 (file)
--- a/src/lib-sql/driver-cassandra.c
+++ b/src/lib-sql/driver-cassandra.c
@@ -1515,7 +1515,8 @@ const struct sql_result driver_cassandra_result = {
                driver_cassandra_result_get_field_value_binary,
                driver_cassandra_result_find_field_value,
                driver_cassandra_result_get_values,
-               driver_cassandra_result_get_error
+               driver_cassandra_result_get_error,
+               NULL,
        }
 };
 

diff --git a/src/lib-sql/driver-mysql.c b/src/lib-sql/driver-mysql.c
index 6ad4928d689a032a30656b14e537559b36c4c609..56e70bfa4d5c8635bd2f7794ab937a44bfd126a8 100644 (file)
--- a/src/lib-sql/driver-mysql.c
+++ b/src/lib-sql/driver-mysql.c
@@ -694,7 +694,8 @@ const struct sql_result driver_mysql_result = {
                driver_mysql_result_get_field_value_binary,
                driver_mysql_result_find_field_value,
                driver_mysql_result_get_values,
-               driver_mysql_result_get_error
+               driver_mysql_result_get_error,
+               NULL,
        }
 };
 
@@ -709,7 +710,8 @@ const struct sql_result driver_mysql_error_result = {
                driver_mysql_result_free,
                driver_mysql_result_error_next_row,
                NULL, NULL, NULL, NULL, NULL, NULL, NULL,
-               driver_mysql_result_get_error
+               driver_mysql_result_get_error,
+               NULL,
        },
        .failed_try_retry = TRUE
 };

diff --git a/src/lib-sql/driver-pgsql.c b/src/lib-sql/driver-pgsql.c
index 54cee202a7cbfd9566064b5e2b48eaafe9a70489..1e626dbf72223d909fc211e98199c49d76df9be1 100644 (file)
--- a/src/lib-sql/driver-pgsql.c
+++ b/src/lib-sql/driver-pgsql.c
@@ -1206,7 +1206,8 @@ const struct sql_result driver_pgsql_result = {
                driver_pgsql_result_get_field_value_binary,
                driver_pgsql_result_find_field_value,
                driver_pgsql_result_get_values,
-               driver_pgsql_result_get_error
+               driver_pgsql_result_get_error,
+               NULL,
        }
 };
 

diff --git a/src/lib-sql/driver-sqlite.c b/src/lib-sql/driver-sqlite.c
index 627748dba025cb208c0bf6731c03440f5fcc2890..1508a90ef098067a6a4d91ffe48cdc7d6072d81d 100644 (file)
--- a/src/lib-sql/driver-sqlite.c
+++ b/src/lib-sql/driver-sqlite.c
@@ -441,7 +441,8 @@ const struct sql_result driver_sqlite_result = {
                driver_sqlite_result_get_field_value_binary,
                driver_sqlite_result_find_field_value,
                driver_sqlite_result_get_values,
-               driver_sqlite_result_get_error
+               driver_sqlite_result_get_error,
+               NULL,
        }
 };
 
@@ -456,7 +457,8 @@ const struct sql_result driver_sqlite_error_result = {
                driver_sqlite_result_free,
                driver_sqlite_result_error_next_row,
                NULL, NULL, NULL, NULL, NULL, NULL, NULL,
-               driver_sqlite_result_get_error
+               driver_sqlite_result_get_error,
+               NULL,
        }
 };
 

diff --git a/src/lib-sql/driver-test.c b/src/lib-sql/driver-test.c
index fd648786dd42d2f8dc07505a70f405cac594336c..f21d73a4ae9317a51f2f9e3fa3494b544e5fd28b 100644 (file)
--- a/src/lib-sql/driver-test.c
+++ b/src/lib-sql/driver-test.c
@@ -164,7 +164,8 @@ const struct sql_result driver_test_result = {
                driver_test_result_get_field_value_binary,
                driver_test_result_find_field_value,
                driver_test_result_get_values,
-               driver_test_result_get_error
+               driver_test_result_get_error,
+               NULL,
        }
 };
 

diff --git a/src/lib-sql/sql-api-private.h b/src/lib-sql/sql-api-private.h
index 2a9f4709447abbd91ee85b6deaff7b2819962a43..ca62a44770eae95c140800491a9962ebec3cf314 100644 (file)
--- a/src/lib-sql/sql-api-private.h
+++ b/src/lib-sql/sql-api-private.h
@@ -123,6 +123,8 @@ struct sql_result_vfuncs {
        const char *const *(*get_values)(struct sql_result *result);
 
        const char *(*get_error)(struct sql_result *result);
+       void (*more)(struct sql_result **result, bool async,
+                    sql_query_callback_t *callback, void *context);
 };
 
 struct sql_field_map {

diff --git a/src/lib-sql/sql-api.c b/src/lib-sql/sql-api.c
index 519c7a7ad415ec6790fa5b324d2a5464efbb1ad9..6f9dadf7d18aab34fb928d68f9338c04b852a944 100644 (file)
--- a/src/lib-sql/sql-api.c
+++ b/src/lib-sql/sql-api.c
@@ -278,6 +278,30 @@ int sql_result_next_row(struct sql_result *result)
        return 1;
 }
 
+#undef sql_result_more
+void sql_result_more(struct sql_result **result,
+                    sql_query_callback_t *callback, void *context)
+{
+       i_assert((*result)->v.more != NULL);
+
+       (*result)->v.more(result, TRUE, callback, context);
+}
+
+static void
+sql_result_more_sync_callback(struct sql_result *result, void *context)
+{
+       struct sql_result **dest_result = context;
+
+       *dest_result = result;
+}
+
+void sql_result_more_s(struct sql_result **result)
+{
+       i_assert((*result)->v.more != NULL);
+
+       (*result)->v.more(result, FALSE, sql_result_more_sync_callback, result);
+}
+
 unsigned int sql_result_get_fields_count(struct sql_result *result)
 {
        return result->v.get_fields_count(result);
@@ -424,7 +448,8 @@ struct sql_result sql_not_connected_result = {
                sql_result_not_connected_free,
                sql_result_not_connected_next_row,
                NULL, NULL, NULL, NULL, NULL, NULL, NULL,
-               sql_result_not_connected_get_error
+               sql_result_not_connected_get_error,
+               NULL,
        },
        .failed_try_retry = TRUE
 };

diff --git a/src/lib-sql/sql-api.h b/src/lib-sql/sql-api.h
index 0ba87b83b2e7940997533be28524fcef43e93c5e..256b77995de80420498cc8dfa7297c5f43ed6574 100644 (file)
--- a/src/lib-sql/sql-api.h
+++ b/src/lib-sql/sql-api.h
@@ -31,6 +31,17 @@ enum sql_result_error_type {
        SQL_RESULT_ERROR_TYPE_WRITE_UNCERTAIN
 };
 
+enum sql_result_next {
+       /* Row was returned */
+       SQL_RESULT_NEXT_OK = 1,
+       /* There are no more rows */
+       SQL_RESULT_NEXT_LAST = 0,
+       /* Error occurred - see sql_result_get_error*() */
+       SQL_RESULT_NEXT_ERROR = -1,
+       /* There are more results - call sql_result_more() */
+       SQL_RESULT_NEXT_MORE = -99
+};
+
 #define SQL_DEF_STRUCT(name, struct_name, type, c_type) \
        { (type) + COMPILE_ERROR_IF_TYPES_NOT_COMPATIBLE( \
                ((struct struct_name *)0)->name, c_type), \
@@ -105,10 +116,23 @@ void sql_result_setup_fetch(struct sql_result *result,
                            const struct sql_field_def *fields,
                            void *dest, size_t dest_size);
 
-/* Go to next row, returns 1 if ok, 0 if this was the last row or -1 if error
-   occurred. This needs to be the first call for result. */
+/* Go to next row. See enum sql_result_next. */
 int sql_result_next_row(struct sql_result *result);
 
+/* If sql_result_next_row() returned SQL_RESULT_NEXT_MORE, this can be called
+   to continue returning more results. The result is freed with this call, so
+   it must not be accesed anymore until the callback is finished. */
+void sql_result_more(struct sql_result **result,
+                    sql_query_callback_t *callback, void *context);
+#define sql_result_more(result, callback, context) \
+       sql_result_more(result + \
+               CALLBACK_TYPECHECK(callback, void (*)( \
+                       struct sql_result *, typeof(context))), \
+               (sql_query_callback_t *)callback, context)
+/* Synchronous version of sql_result_more(). The result will be replaced with
+   the new result. */
+void sql_result_more_s(struct sql_result **result);
+
 void sql_result_ref(struct sql_result *result);
 /* Needs to be called only with sql_query_s() or when result has been
    explicitly referenced. */