From: Timo Sirainen Date: Mon, 17 Jul 2017 11:22:35 +0000 (+0300) Subject: lib-sql: Add API support for asynchronously iterating over rows. X-Git-Tag: 2.3.0.rc1~1256 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=402f9bcf48cbccc17fdb5f3ea411a7967aed0fd0;p=thirdparty%2Fdovecot%2Fcore.git lib-sql: Add API support for asynchronously iterating over rows. sql_query() can already do an async lookup, but the full result needs to be available immediately. This can be inefficient for large results. Add a new SQL_RESULT_NEXT_MORE return value and sql_result_more() for asynchronously requesting more results. This changes the API a bit, but isn't done by default by any drivers yet. Also callers that can't handle this are hopefully checking for "ret < 0", which allows them to handle such an async-more request as an error instead. sql_result_next_row() will be changed to return enum in a separate commit to keep backwards compatibility in v2.2.x. --- diff --git a/src/lib-sql/driver-cassandra.c b/src/lib-sql/driver-cassandra.c index 87f75322f9..a2213dfede 100644 --- a/src/lib-sql/driver-cassandra.c +++ b/src/lib-sql/driver-cassandra.c @@ -1515,7 +1515,8 @@ const struct sql_result driver_cassandra_result = { driver_cassandra_result_get_field_value_binary, driver_cassandra_result_find_field_value, driver_cassandra_result_get_values, - driver_cassandra_result_get_error + driver_cassandra_result_get_error, + NULL, } }; diff --git a/src/lib-sql/driver-mysql.c b/src/lib-sql/driver-mysql.c index 6ad4928d68..56e70bfa4d 100644 --- a/src/lib-sql/driver-mysql.c +++ b/src/lib-sql/driver-mysql.c @@ -694,7 +694,8 @@ const struct sql_result driver_mysql_result = { driver_mysql_result_get_field_value_binary, driver_mysql_result_find_field_value, driver_mysql_result_get_values, - driver_mysql_result_get_error + driver_mysql_result_get_error, + NULL, } }; @@ -709,7 +710,8 @@ const struct sql_result driver_mysql_error_result = { driver_mysql_result_free, driver_mysql_result_error_next_row, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - driver_mysql_result_get_error + driver_mysql_result_get_error, + NULL, }, .failed_try_retry = TRUE }; diff --git a/src/lib-sql/driver-pgsql.c b/src/lib-sql/driver-pgsql.c index 54cee202a7..1e626dbf72 100644 --- a/src/lib-sql/driver-pgsql.c +++ b/src/lib-sql/driver-pgsql.c @@ -1206,7 +1206,8 @@ const struct sql_result driver_pgsql_result = { driver_pgsql_result_get_field_value_binary, driver_pgsql_result_find_field_value, driver_pgsql_result_get_values, - driver_pgsql_result_get_error + driver_pgsql_result_get_error, + NULL, } }; diff --git a/src/lib-sql/driver-sqlite.c b/src/lib-sql/driver-sqlite.c index 627748dba0..1508a90ef0 100644 --- a/src/lib-sql/driver-sqlite.c +++ b/src/lib-sql/driver-sqlite.c @@ -441,7 +441,8 @@ const struct sql_result driver_sqlite_result = { driver_sqlite_result_get_field_value_binary, driver_sqlite_result_find_field_value, driver_sqlite_result_get_values, - driver_sqlite_result_get_error + driver_sqlite_result_get_error, + NULL, } }; @@ -456,7 +457,8 @@ const struct sql_result driver_sqlite_error_result = { driver_sqlite_result_free, driver_sqlite_result_error_next_row, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - driver_sqlite_result_get_error + driver_sqlite_result_get_error, + NULL, } }; diff --git a/src/lib-sql/driver-test.c b/src/lib-sql/driver-test.c index fd648786dd..f21d73a4ae 100644 --- a/src/lib-sql/driver-test.c +++ b/src/lib-sql/driver-test.c @@ -164,7 +164,8 @@ const struct sql_result driver_test_result = { driver_test_result_get_field_value_binary, driver_test_result_find_field_value, driver_test_result_get_values, - driver_test_result_get_error + driver_test_result_get_error, + NULL, } }; diff --git a/src/lib-sql/sql-api-private.h b/src/lib-sql/sql-api-private.h index 2a9f470944..ca62a44770 100644 --- a/src/lib-sql/sql-api-private.h +++ b/src/lib-sql/sql-api-private.h @@ -123,6 +123,8 @@ struct sql_result_vfuncs { const char *const *(*get_values)(struct sql_result *result); const char *(*get_error)(struct sql_result *result); + void (*more)(struct sql_result **result, bool async, + sql_query_callback_t *callback, void *context); }; struct sql_field_map { diff --git a/src/lib-sql/sql-api.c b/src/lib-sql/sql-api.c index 519c7a7ad4..6f9dadf7d1 100644 --- a/src/lib-sql/sql-api.c +++ b/src/lib-sql/sql-api.c @@ -278,6 +278,30 @@ int sql_result_next_row(struct sql_result *result) return 1; } +#undef sql_result_more +void sql_result_more(struct sql_result **result, + sql_query_callback_t *callback, void *context) +{ + i_assert((*result)->v.more != NULL); + + (*result)->v.more(result, TRUE, callback, context); +} + +static void +sql_result_more_sync_callback(struct sql_result *result, void *context) +{ + struct sql_result **dest_result = context; + + *dest_result = result; +} + +void sql_result_more_s(struct sql_result **result) +{ + i_assert((*result)->v.more != NULL); + + (*result)->v.more(result, FALSE, sql_result_more_sync_callback, result); +} + unsigned int sql_result_get_fields_count(struct sql_result *result) { return result->v.get_fields_count(result); @@ -424,7 +448,8 @@ struct sql_result sql_not_connected_result = { sql_result_not_connected_free, sql_result_not_connected_next_row, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - sql_result_not_connected_get_error + sql_result_not_connected_get_error, + NULL, }, .failed_try_retry = TRUE }; diff --git a/src/lib-sql/sql-api.h b/src/lib-sql/sql-api.h index 0ba87b83b2..256b77995d 100644 --- a/src/lib-sql/sql-api.h +++ b/src/lib-sql/sql-api.h @@ -31,6 +31,17 @@ enum sql_result_error_type { SQL_RESULT_ERROR_TYPE_WRITE_UNCERTAIN }; +enum sql_result_next { + /* Row was returned */ + SQL_RESULT_NEXT_OK = 1, + /* There are no more rows */ + SQL_RESULT_NEXT_LAST = 0, + /* Error occurred - see sql_result_get_error*() */ + SQL_RESULT_NEXT_ERROR = -1, + /* There are more results - call sql_result_more() */ + SQL_RESULT_NEXT_MORE = -99 +}; + #define SQL_DEF_STRUCT(name, struct_name, type, c_type) \ { (type) + COMPILE_ERROR_IF_TYPES_NOT_COMPATIBLE( \ ((struct struct_name *)0)->name, c_type), \ @@ -105,10 +116,23 @@ void sql_result_setup_fetch(struct sql_result *result, const struct sql_field_def *fields, void *dest, size_t dest_size); -/* Go to next row, returns 1 if ok, 0 if this was the last row or -1 if error - occurred. This needs to be the first call for result. */ +/* Go to next row. See enum sql_result_next. */ int sql_result_next_row(struct sql_result *result); +/* If sql_result_next_row() returned SQL_RESULT_NEXT_MORE, this can be called + to continue returning more results. The result is freed with this call, so + it must not be accesed anymore until the callback is finished. */ +void sql_result_more(struct sql_result **result, + sql_query_callback_t *callback, void *context); +#define sql_result_more(result, callback, context) \ + sql_result_more(result + \ + CALLBACK_TYPECHECK(callback, void (*)( \ + struct sql_result *, typeof(context))), \ + (sql_query_callback_t *)callback, context) +/* Synchronous version of sql_result_more(). The result will be replaced with + the new result. */ +void sql_result_more_s(struct sql_result **result); + void sql_result_ref(struct sql_result *result); /* Needs to be called only with sql_query_s() or when result has been explicitly referenced. */