lib-sql: Add API support for asynchronously iterating over rows.

sql_query() can already do an async lookup, but the full result needs
to be available immediately. This can be inefficient for large results.
Add a new SQL_RESULT_NEXT_MORE return value and sql_result_more() for
asynchronously requesting more results.

This changes the API a bit, but isn't done by default by any drivers yet.
Also callers that can't handle this are hopefully checking for "ret < 0",
which allows them to handle such an async-more request as an error
instead.

sql_result_next_row() will be changed to return enum in a separate commit to
keep backwards compatibility in v2.2.x.

diff --git a/src/lib-sql/driver-cassandra.c b/src/lib-sql/driver-cassandra.c
index 7608fb566311594422fc4dc75c73d2ad5804aa76..040e605a3f47d14ca460f7a397b3d4bf018adfcb 100644 (file)
--- a/src/lib-sql/driver-cassandra.c
+++ b/src/lib-sql/driver-cassandra.c
@@ -1511,7 +1511,8 @@ const struct sql_result driver_cassandra_result = {
                driver_cassandra_result_get_field_value_binary,
                driver_cassandra_result_find_field_value,
                driver_cassandra_result_get_values,
-               driver_cassandra_result_get_error
+               driver_cassandra_result_get_error,
+               NULL,
        }
 };
 

diff --git a/src/lib-sql/driver-mysql.c b/src/lib-sql/driver-mysql.c
index b9a51e718306a998f466f8d12e1b8702d1dd8dcc..e10713b8366a0851ab9941862f4ccbdc0ee2ccab 100644 (file)
--- a/src/lib-sql/driver-mysql.c
+++ b/src/lib-sql/driver-mysql.c
@@ -694,7 +694,8 @@ const struct sql_result driver_mysql_result = {
                driver_mysql_result_get_field_value_binary,
                driver_mysql_result_find_field_value,
                driver_mysql_result_get_values,
-               driver_mysql_result_get_error
+               driver_mysql_result_get_error,
+               NULL,
        }
 };
 
@@ -709,7 +710,8 @@ const struct sql_result driver_mysql_error_result = {
                driver_mysql_result_free,
                driver_mysql_result_error_next_row,
                NULL, NULL, NULL, NULL, NULL, NULL, NULL,
-               driver_mysql_result_get_error
+               driver_mysql_result_get_error,
+               NULL,
        },
        .failed_try_retry = TRUE
 };

diff --git a/src/lib-sql/driver-pgsql.c b/src/lib-sql/driver-pgsql.c
index cbe0183980022fa1a2e33bf0321b35f672166c0d..01b8cc1b8ebd37005da86705c22ca3b872495f5a 100644 (file)
--- a/src/lib-sql/driver-pgsql.c
+++ b/src/lib-sql/driver-pgsql.c
@@ -1181,7 +1181,8 @@ const struct sql_result driver_pgsql_result = {
                driver_pgsql_result_get_field_value_binary,
                driver_pgsql_result_find_field_value,
                driver_pgsql_result_get_values,
-               driver_pgsql_result_get_error
+               driver_pgsql_result_get_error,
+               NULL,
        }
 };
 

diff --git a/src/lib-sql/driver-sqlite.c b/src/lib-sql/driver-sqlite.c
index 668e0753d9b37c1db4130ea720ad53e32f26d797..3e4289157c3a9f05d5fe8519d2b2c041be8a511b 100644 (file)
--- a/src/lib-sql/driver-sqlite.c
+++ b/src/lib-sql/driver-sqlite.c
@@ -441,7 +441,8 @@ const struct sql_result driver_sqlite_result = {
                driver_sqlite_result_get_field_value_binary,
                driver_sqlite_result_find_field_value,
                driver_sqlite_result_get_values,
-               driver_sqlite_result_get_error
+               driver_sqlite_result_get_error,
+               NULL,
        }
 };
 
@@ -456,7 +457,8 @@ const struct sql_result driver_sqlite_error_result = {
                driver_sqlite_result_free,
                driver_sqlite_result_error_next_row,
                NULL, NULL, NULL, NULL, NULL, NULL, NULL,
-               driver_sqlite_result_get_error
+               driver_sqlite_result_get_error,
+               NULL,
        }
 };
 

diff --git a/src/lib-sql/sql-api-private.h b/src/lib-sql/sql-api-private.h
index ff75946104ff3b01b8fbe1e84a0cccee4ec7fc26..264c615cbc97fe12a9cb3da770fb287342b7babe 100644 (file)
--- a/src/lib-sql/sql-api-private.h
+++ b/src/lib-sql/sql-api-private.h
@@ -126,6 +126,8 @@ struct sql_result_vfuncs {
        const char *const *(*get_values)(struct sql_result *result);
 
        const char *(*get_error)(struct sql_result *result);
+       void (*more)(struct sql_result **result, bool async,
+                    sql_query_callback_t *callback, void *context);
 };
 
 struct sql_field_map {

diff --git a/src/lib-sql/sql-api.c b/src/lib-sql/sql-api.c
index 16567a40dd57436a13d3e2f1d62febb6af2d99f2..705e5667640c95606e7dc19e12dc073ca1646756 100644 (file)
--- a/src/lib-sql/sql-api.c
+++ b/src/lib-sql/sql-api.c
@@ -278,6 +278,30 @@ int sql_result_next_row(struct sql_result *result)
        return 1;
 }
 
+#undef sql_result_more
+void sql_result_more(struct sql_result **result,
+                    sql_query_callback_t *callback, void *context)
+{
+       i_assert((*result)->v.more != NULL);
+
+       (*result)->v.more(result, TRUE, callback, context);
+}
+
+static void
+sql_result_more_sync_callback(struct sql_result *result, void *context)
+{
+       struct sql_result **dest_result = context;
+
+       *dest_result = result;
+}
+
+void sql_result_more_s(struct sql_result **result)
+{
+       i_assert((*result)->v.more != NULL);
+
+       (*result)->v.more(result, FALSE, sql_result_more_sync_callback, result);
+}
+
 unsigned int sql_result_get_fields_count(struct sql_result *result)
 {
        return result->v.get_fields_count(result);
@@ -480,7 +504,8 @@ struct sql_result sql_not_connected_result = {
                sql_result_not_connected_free,
                sql_result_not_connected_next_row,
                NULL, NULL, NULL, NULL, NULL, NULL, NULL,
-               sql_result_not_connected_get_error
+               sql_result_not_connected_get_error,
+               NULL,
        },
        .failed_try_retry = TRUE
 };

diff --git a/src/lib-sql/sql-api.h b/src/lib-sql/sql-api.h
index 065b5b842fa5cb2a26740b782388baf9b1b70051..91f82f23143c39ad747b3efadaefb9643311ef63 100644 (file)
--- a/src/lib-sql/sql-api.h
+++ b/src/lib-sql/sql-api.h
@@ -31,6 +31,17 @@ enum sql_result_error_type {
        SQL_RESULT_ERROR_TYPE_WRITE_UNCERTAIN
 };
 
+enum sql_result_next {
+       /* Row was returned */
+       SQL_RESULT_NEXT_OK = 1,
+       /* There are no more rows */
+       SQL_RESULT_NEXT_LAST = 0,
+       /* Error occurred - see sql_result_get_error*() */
+       SQL_RESULT_NEXT_ERROR = -1,
+       /* There are more results - call sql_result_more() */
+       SQL_RESULT_NEXT_MORE = -99
+};
+
 #define SQL_DEF_STRUCT(name, struct_name, type, c_type) \
        { (type) + COMPILE_ERROR_IF_TYPES_NOT_COMPATIBLE( \
                ((struct struct_name *)0)->name, c_type), \
@@ -106,10 +117,23 @@ void sql_result_setup_fetch(struct sql_result *result,
                            const struct sql_field_def *fields,
                            void *dest, size_t dest_size);
 
-/* Go to next row, returns 1 if ok, 0 if this was the last row or -1 if error
-   occurred. This needs to be the first call for result. */
+/* Go to next row. See enum sql_result_next. */
 int sql_result_next_row(struct sql_result *result);
 
+/* If sql_result_next_row() returned SQL_RESULT_NEXT_MORE, this can be called
+   to continue returning more results. The result is freed with this call, so
+   it must not be accesed anymore until the callback is finished. */
+void sql_result_more(struct sql_result **result,
+                    sql_query_callback_t *callback, void *context);
+#define sql_result_more(result, callback, context) \
+       sql_result_more(result + \
+               CALLBACK_TYPECHECK(callback, void (*)( \
+                       struct sql_result *, typeof(context))), \
+               (sql_query_callback_t *)callback, context)
+/* Synchronous version of sql_result_more(). The result will be replaced with
+   the new result. */
+void sql_result_more_s(struct sql_result **result);
+
 void sql_result_ref(struct sql_result *result);
 /* Needs to be called only with sql_query_s() or when result has been
    explicitly referenced. */