From: Michael Tremer Date: Mon, 12 Aug 2024 14:23:31 +0000 (+0000) Subject: httpclient: Remove queueing our own transfers X-Git-Tag: 0.9.30~1215 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=3a89e0bb4f96b56c756adffcd1bb2ef656704cd4;p=pakfire.git httpclient: Remove queueing our own transfers We now throw everything into cURL and let it handle all of that... Signed-off-by: Michael Tremer --- diff --git a/src/libpakfire/httpclient.c b/src/libpakfire/httpclient.c index 2af55f02c..557bc789c 100644 --- a/src/libpakfire/httpclient.c +++ b/src/libpakfire/httpclient.c @@ -20,7 +20,6 @@ #include #include -#include #include @@ -34,12 +33,6 @@ // The number of concurrent downloads #define DEFAULT_MAX_PARALLEL 4 -struct pakfire_xfer_element { - TAILQ_ENTRY(pakfire_xfer_element) nodes; - - struct pakfire_xfer* xfer; -}; - struct pakfire_httpclient { struct pakfire_ctx* ctx; int nrefs; @@ -63,84 +56,14 @@ struct pakfire_httpclient { // cURL multi handle CURLM* curl; - // Transfers - TAILQ_HEAD(xfers_queued, pakfire_xfer_element) xfers_queued; - TAILQ_HEAD(xfers_running, pakfire_xfer_element) xfers_running; -}; - -static struct pakfire_xfer_element* pakfire_httpclient_xfer_create(struct pakfire_xfer* xfer) { - struct pakfire_xfer_element* x = NULL; - - // Allocate a new element - x = calloc(1, sizeof(*x)); - if (!x) - return NULL; + // How many xfers do we have queued? + unsigned int total_xfers; - // Store a reference to the xfer - x->xfer = pakfire_xfer_ref(xfer); - - return x; -} - -static void pakfire_httpclient_xfer_free(struct pakfire_xfer_element* x) { - if (x->xfer) - pakfire_xfer_unref(x->xfer); - - free(x); -} - -static int pakfire_httpclient_start_transfers( - struct pakfire_httpclient* client, struct pakfire_progress* progress) { - struct pakfire_xfer_element* x = NULL; - int r; - - // Keep running until we have reached our ceiling - while (client->still_running < client->max_parallel) { - // We are done if there are no more transfers in the queue - if (TAILQ_EMPTY(&client->xfers_queued)) - break; - - // Fetch the next transfer - x = TAILQ_LAST(&client->xfers_queued, xfers_queued); - TAILQ_REMOVE(&client->xfers_queued, x, nodes); - - // Prepare the xfer - r = pakfire_xfer_prepare(x->xfer, progress, 0); - if (r) - goto ERROR; - - // Add the handle to cURL - r = curl_multi_add_handle(client->curl, pakfire_xfer_handle(x->xfer)); - if (r) { - CTX_ERROR(client->ctx, "Adding handle failed: %s\n", curl_multi_strerror(r)); - goto ERROR; - } - - TAILQ_INSERT_TAIL(&client->xfers_running, x, nodes); - } - - return 0; - -ERROR: - pakfire_httpclient_xfer_free(x); - - return r; -} - -static struct pakfire_xfer_element* pakfire_httpclient_xfer_find_running( - struct pakfire_httpclient* client, struct pakfire_xfer* xfer) { - struct pakfire_xfer_element* x = NULL; - - TAILQ_FOREACH(x, &client->xfers_running, nodes) { - if (x->xfer == xfer) - return x; - } - - return NULL; -} + // The total downloadsize + unsigned int total_downloadsize; +}; static int pakfire_httpclient_check(struct pakfire_httpclient* client) { - struct pakfire_xfer_element* x = NULL; struct pakfire_xfer* xfer = NULL; int r; @@ -161,28 +84,28 @@ static int pakfire_httpclient_check(struct pakfire_httpclient* client) { // Remove the handle curl_multi_remove_handle(client->curl, pakfire_xfer_handle(xfer)); - // Find the matching xfer element - x = pakfire_httpclient_xfer_find_running(client, xfer); + // Decrement the xfers counter + client->total_xfers--; - // Remove the transfer from the running list - if (x) - TAILQ_REMOVE(&client->xfers_running, x, nodes); + // Reduce the total download size + client->total_downloadsize -= pakfire_xfer_get_size(xfer); // Call the done callback r = pakfire_xfer_done(xfer, msg->data.result); switch (-r) { // If we are asked to try again we will re-queue the transfer case EAGAIN: - if (x) - TAILQ_INSERT_TAIL(&client->xfers_queued, x, nodes); + r = pakfire_httpclient_enqueue_xfer(client, xfer); + if (r) + return r; + break; // Otherwise this transfer has finished default: - if (x) - pakfire_httpclient_xfer_free(x); if (r) return r; + break; } @@ -471,24 +394,6 @@ static int pakfire_httpclient_setup_progress(struct pakfire_httpclient* client) } static void pakfire_httpclient_free(struct pakfire_httpclient* client) { - struct pakfire_xfer_element* x = NULL; - - // Free any queued transfers - while (!TAILQ_EMPTY(&client->xfers_queued)) { - x = TAILQ_LAST(&client->xfers_queued, xfers_queued); - TAILQ_REMOVE(&client->xfers_queued, x, nodes); - - pakfire_httpclient_xfer_free(x); - } - - // Free any running transfers - while (!TAILQ_EMPTY(&client->xfers_running)) { - x = TAILQ_LAST(&client->xfers_running, xfers_running); - TAILQ_REMOVE(&client->xfers_running, x, nodes); - - pakfire_httpclient_xfer_free(x); - } - if (client->progress) pakfire_progress_unref(client->progress); if (client->share) @@ -528,10 +433,6 @@ int pakfire_httpclient_create(struct pakfire_httpclient** client, struct pakfire // Set parallelism c->max_parallel = DEFAULT_MAX_PARALLEL; - // Init transfer queues - TAILQ_INIT(&c->xfers_queued); - TAILQ_INIT(&c->xfers_running); - // Setup event loop r = pakfire_httpclient_setup_loop(c); if (r) @@ -577,79 +478,48 @@ CURLSH* pakfire_httpclient_share(struct pakfire_httpclient* client) { } int pakfire_httpclient_create_xfer(struct pakfire_xfer** xfer, - struct pakfire_httpclient* client, const char* url) { + struct pakfire_httpclient* client, const char* url) { return pakfire_xfer_create(xfer, client->ctx, client, url); } int pakfire_httpclient_enqueue_xfer(struct pakfire_httpclient* client, struct pakfire_xfer* xfer) { - struct pakfire_xfer_element* x = NULL; - - // Create a new queueable object - x = pakfire_httpclient_xfer_create(xfer); - if (!x) - return -errno; - - // Push this transfer onto the queue - TAILQ_INSERT_HEAD(&client->xfers_queued, x, nodes); - - return 0; -} + int r; -static size_t pakfire_httpclient_total_downloadsize(struct pakfire_httpclient* client) { - struct pakfire_xfer_element* x = NULL; - size_t size; + // Prepare the xfer + r = pakfire_xfer_prepare(xfer, client->progress, 0); + if (r) + return r; - size_t total_size = 0; + // Increment the xfer counter + client->total_xfers++; - TAILQ_FOREACH(x, &client->xfers_queued, nodes) { - size = pakfire_xfer_get_size(x->xfer); + // Update the total download size + client->total_downloadsize += pakfire_xfer_get_size(xfer); - // Return zero if the size isn't known - if (!size) - return 0; + // Add the handle to cURL + r = curl_multi_add_handle(client->curl, pakfire_xfer_handle(xfer)); + if (r) { + CTX_ERROR(client->ctx, "Adding handle failed: %s\n", curl_multi_strerror(r)); - total_size += size; + return r; } - return total_size; -} - -static unsigned int pakfire_httpclient_total_queued_xfers(struct pakfire_httpclient* client) { - struct pakfire_xfer_element* x = NULL; - unsigned int counter = 0; - - TAILQ_FOREACH(x, &client->xfers_queued, nodes) - counter++; - - return counter; + return 0; } int pakfire_httpclient_run(struct pakfire_httpclient* client, const char* title) { int r = 0; - struct pakfire_xfer_element* x = NULL; - - // Fetch the total downloadsize - const size_t downloadsize = pakfire_httpclient_total_downloadsize(client); - - // Fetch the total number of queued transfers - const unsigned int num_queued_xfers = pakfire_httpclient_total_queued_xfers(client); - // Set the title r = pakfire_progress_set_title(client->progress, title); if (r) - goto ERROR; + return r; // Start the progress - r = pakfire_progress_start(client->progress, (downloadsize) ? downloadsize : num_queued_xfers); + r = pakfire_progress_start(client->progress, client->total_downloadsize); if (r) - goto ERROR; - - // Make sure that we have up to parallel transfers active - r = pakfire_httpclient_start_transfers(client, client->progress); - if (r) - goto ERROR; + return r; // Run the event loop do { @@ -657,24 +527,14 @@ int pakfire_httpclient_run(struct pakfire_httpclient* client, const char* title) if (r < 0) { CTX_ERROR(client->ctx, "Event loop failed: %s\n", strerror(-r)); - goto ERROR; + return r; } } while (client->still_running > 0); // We are finished! r = pakfire_progress_finish(client->progress); if (r) - goto ERROR; - -ERROR: - // If the client was aborted, we need to fail all remaining running transfers - while (!TAILQ_EMPTY(&client->xfers_running)) { - x = TAILQ_LAST(&client->xfers_running, xfers_running); - TAILQ_REMOVE(&client->xfers_running, x, nodes); - - // Fail the transfer - pakfire_xfer_fail(x->xfer); - } + return r; - return r; + return 0; }