src/libpakfire/httpclient.c

   1 /*#############################################################################
   2 #                                                                             #
   3 # Pakfire - The IPFire package management system                              #
   4 # Copyright (C) 2021 Pakfire development team                                 #
   5 #                                                                             #
   6 # This program is free software: you can redistribute it and/or modify        #
   7 # it under the terms of the GNU General Public License as published by        #
   8 # the Free Software Foundation, either version 3 of the License, or           #
   9 # (at your option) any later version.                                         #
  10 #                                                                             #
  11 # This program is distributed in the hope that it will be useful,             #
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  14 # GNU General Public License for more details.                                #
  15 #                                                                             #
  16 # You should have received a copy of the GNU General Public License           #
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  18 #                                                                             #
  19 #############################################################################*/
  20
  21 #include <errno.h>
  22 #include <stdlib.h>
  23 #include <sys/queue.h>
  24
  25 #include <curl/curl.h>
  26
  27 #include <pakfire/httpclient.h>
  28 #include <pakfire/logging.h>
  29 #include <pakfire/progress.h>
  30 #include <pakfire/xfer.h>
  31
  32 // The number of concurrent downloads
  33 #define MAX_PARALLEL                    4
  34
  35 struct pakfire_xfer_element {
  36         TAILQ_ENTRY(pakfire_xfer_element) nodes;
  37
  38         struct pakfire_xfer* xfer;
  39 };
  40
  41 struct pakfire_httpclient {
  42         struct pakfire_ctx* ctx;
  43         int nrefs;
  44
  45         unsigned int parallel;
  46
  47         // cURL share handle
  48         CURLSH* share;
  49
  50         // cURL multi handle
  51         CURLM* curl;
  52
  53         // Transfers
  54         TAILQ_HEAD(xfers_queued, pakfire_xfer_element) xfers_queued;
  55         TAILQ_HEAD(xfers_running, pakfire_xfer_element) xfers_running;
  56 };
  57
  58 static int pakfire_httpclient_setup_curl(struct pakfire_httpclient* client) {
  59         int r;
  60
  61         // Initialize cURL
  62         r = curl_global_init(CURL_GLOBAL_DEFAULT);
  63         if (r) {
  64                 CTX_ERROR(client->ctx, "Could not initialize cURL: %d\n", r);
  65                 return r;
  66         }
  67
  68         // Create a new share handle
  69         client->share = curl_share_init();
  70         if (!client->share) {
  71                 CTX_ERROR(client->ctx, "Could not setup cURL share handle\n");
  72                 return 1;
  73         }
  74
  75         // Create a new multi handle
  76         client->curl = curl_multi_init();
  77         if (!client->curl) {
  78                 CTX_ERROR(client->ctx, "Could not create cURL multi handle\n");
  79                 return 1;
  80         }
  81
  82         // Share connections between handles
  83         r = curl_share_setopt(client->share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT);
  84         if (r) {
  85                 CTX_ERROR(client->ctx, "Could not configure the share handle: %s\n",
  86                         curl_share_strerror(r));
  87                 return r;
  88         }
  89
  90         return 0;
  91 }
  92
  93 static struct pakfire_xfer_element* pakfire_httpclient_xfer_create(struct pakfire_xfer* xfer) {
  94         struct pakfire_xfer_element* x = NULL;
  95
  96         // Allocate a new element
  97         x = calloc(1, sizeof(*x));
  98         if (!x)
  99                 return NULL;
 100
 101         // Store a reference to the xfer
 102         x->xfer = pakfire_xfer_ref(xfer);
 103
 104         return x;
 105 }
 106
 107 static struct pakfire_xfer_element* pakfire_httpclient_xfer_find_running(
 108                 struct pakfire_httpclient* client, struct pakfire_xfer* xfer) {
 109         struct pakfire_xfer_element* x = NULL;
 110
 111         TAILQ_FOREACH(x, &client->xfers_running, nodes) {
 112                 if (x->xfer == xfer)
 113                         return x;
 114         }
 115
 116         return NULL;
 117 }
 118
 119 static void pakfire_httpclient_xfer_free(struct pakfire_xfer_element* x) {
 120         if (x->xfer)
 121                 pakfire_xfer_unref(x->xfer);
 122
 123         free(x);
 124 }
 125
 126 static void pakfire_httpclient_free(struct pakfire_httpclient* client) {
 127         struct pakfire_xfer_element* x = NULL;
 128
 129         // Free any queued transfers
 130         while (!TAILQ_EMPTY(&client->xfers_queued)) {
 131                 x = TAILQ_LAST(&client->xfers_queued, xfers_queued);
 132                 TAILQ_REMOVE(&client->xfers_queued, x, nodes);
 133
 134                 pakfire_httpclient_xfer_free(x);
 135         }
 136
 137         // Free any running transfers
 138         while (!TAILQ_EMPTY(&client->xfers_running)) {
 139                 x = TAILQ_LAST(&client->xfers_running, xfers_running);
 140                 TAILQ_REMOVE(&client->xfers_running, x, nodes);
 141
 142                 pakfire_httpclient_xfer_free(x);
 143         }
 144
 145         if (client->share)
 146                 curl_share_cleanup(client->share);
 147         if (client->curl)
 148                 curl_multi_cleanup(client->curl);
 149         if (client->ctx)
 150                 pakfire_ctx_unref(client->ctx);
 151         free(client);
 152 }
 153
 154 int pakfire_httpclient_create(struct pakfire_httpclient** client, struct pakfire_ctx* ctx) {
 155         struct pakfire_httpclient* c = NULL;
 156         int r;
 157
 158         // Fail if the context is flagged as offline
 159         if (pakfire_ctx_has_flag(ctx, PAKFIRE_CTX_OFFLINE)) {
 160                 CTX_ERROR(ctx, "Cannot initialize downloader in offline mode\n");
 161                 return -EPERM;
 162         }
 163
 164         // Allocate a new object
 165         c = calloc(1, sizeof(*c));
 166         if (!c)
 167                 return -ENOMEM;
 168
 169         // Store reference to the context
 170         c->ctx = pakfire_ctx_ref(ctx);
 171
 172         // Initialize reference counting
 173         c->nrefs = 1;
 174
 175         // Set parallelism
 176         c->parallel = MAX_PARALLEL;
 177
 178         // Init transfer queues
 179         TAILQ_INIT(&c->xfers_queued);
 180         TAILQ_INIT(&c->xfers_running);
 181
 182         // Setup cURL
 183         r = pakfire_httpclient_setup_curl(c);
 184         if (r)
 185                 goto ERROR;
 186
 187         // Success
 188         *client = c;
 189
 190         return 0;
 191
 192 ERROR:
 193         pakfire_httpclient_free(c);
 194
 195         return r;
 196 }
 197
 198 struct pakfire_httpclient* pakfire_httpclient_ref(struct pakfire_httpclient* client) {
 199         ++client->nrefs;
 200
 201         return client;
 202 }
 203
 204 struct pakfire_httpclient* pakfire_httpclient_unref(struct pakfire_httpclient* client) {
 205         if (--client->nrefs > 0)
 206                 return client;
 207
 208         pakfire_httpclient_free(client);
 209         return NULL;
 210 }
 211
 212 CURLSH* pakfire_httpclient_share(struct pakfire_httpclient* client) {
 213         return client->share;
 214 }
 215
 216 int pakfire_httpclient_create_xfer(struct pakfire_xfer** xfer,
 217                 struct pakfire_httpclient* client, const char* url) {
 218         return pakfire_xfer_create(xfer, client->ctx, client, url);
 219 }
 220
 221 int pakfire_httpclient_enqueue_xfer(struct pakfire_httpclient* client,
 222                 struct pakfire_xfer* xfer) {
 223         struct pakfire_xfer_element* x = NULL;
 224
 225         // Create a new queueable object
 226         x = pakfire_httpclient_xfer_create(xfer);
 227         if (!x)
 228                 return -errno;
 229
 230         // Push this transfer onto the queue
 231         TAILQ_INSERT_HEAD(&client->xfers_queued, x, nodes);
 232
 233         return 0;
 234 }
 235
 236 static size_t pakfire_httpclient_total_downloadsize(struct pakfire_httpclient* client) {
 237         struct pakfire_xfer_element* x = NULL;
 238         size_t size;
 239
 240         size_t total_size = 0;
 241
 242         TAILQ_FOREACH(x, &client->xfers_queued, nodes) {
 243                 size = pakfire_xfer_get_size(x->xfer);
 244
 245                 // Return zero if the size isn't known
 246                 if (!size)
 247                         return 0;
 248
 249                 total_size += size;
 250         }
 251
 252         return total_size;
 253 }
 254
 255 static unsigned int pakfire_httpclient_total_queued_xfers(struct pakfire_httpclient* client) {
 256         struct pakfire_xfer_element* x = NULL;
 257         unsigned int counter = 0;
 258
 259         TAILQ_FOREACH(x, &client->xfers_queued, nodes)
 260                 counter++;
 261
 262         return counter;
 263 }
 264
 265 static int pakfire_httpclient_start_transfers(struct pakfire_httpclient* client,
 266                 struct pakfire_progress* progress, unsigned int* running_transfers) {
 267         struct pakfire_xfer_element* x = NULL;
 268         int r;
 269
 270         // Keep running until we have reached our ceiling
 271         while (*running_transfers < client->parallel) {
 272                 // We are done if there are no more transfers in the queue
 273                 if (TAILQ_EMPTY(&client->xfers_queued))
 274                         break;
 275
 276                 // Fetch the next transfer
 277                 x = TAILQ_LAST(&client->xfers_queued, xfers_queued);
 278                 TAILQ_REMOVE(&client->xfers_queued, x, nodes);
 279
 280                 // Prepare the xfer
 281                 r = pakfire_xfer_prepare(x->xfer, progress, 0);
 282                 if (r)
 283                         goto ERROR;
 284
 285                 // Add the handle to cURL
 286                 r = curl_multi_add_handle(client->curl, pakfire_xfer_handle(x->xfer));
 287                 if (r) {
 288                         CTX_ERROR(client->ctx, "Adding handle failed: %s\n", curl_multi_strerror(r));
 289                         goto ERROR;
 290                 }
 291
 292                 TAILQ_INSERT_TAIL(&client->xfers_running, x, nodes);
 293                 (*running_transfers)++;
 294         }
 295
 296         return 0;
 297
 298 ERROR:
 299         pakfire_httpclient_xfer_free(x);
 300
 301         return r;
 302 }
 303
 304 int pakfire_httpclient_run(struct pakfire_httpclient* client, const char* title) {
 305         struct pakfire_progress* progress = NULL;
 306         struct pakfire_xfer* xfer = NULL;
 307         unsigned int running_xfers = 0;
 308         int progress_flags =
 309                 PAKFIRE_PROGRESS_SHOW_PERCENTAGE |
 310                 PAKFIRE_PROGRESS_SHOW_ETA;
 311         int r = 0;
 312
 313         struct pakfire_xfer_element* x = NULL;
 314
 315         CURLMsg* msg = NULL;
 316         int still_running;
 317         int msgs_left = -1;
 318
 319         // Fetch the total downloadsize
 320         const size_t downloadsize = pakfire_httpclient_total_downloadsize(client);
 321
 322         // If we know the downloadsize, we can show more details
 323         if (downloadsize) {
 324                 progress_flags |=
 325                         PAKFIRE_PROGRESS_SHOW_BYTES_TRANSFERRED |
 326                         PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED;
 327         } else {
 328                 progress_flags |=
 329                         PAKFIRE_PROGRESS_SHOW_COUNTER;
 330         }
 331
 332         // Fetch the total number of queued transfers
 333         const unsigned int num_queued_xfers = pakfire_httpclient_total_queued_xfers(client);
 334
 335         // Create a new progress indicator
 336         r = pakfire_progress_create(&progress, client->ctx, progress_flags, NULL);
 337         if (r)
 338                 goto ERROR;
 339
 340         // Set the title
 341         if (title) {
 342                 r = pakfire_progress_set_title(progress, title);
 343                 if (r)
 344                         goto ERROR;
 345         }
 346
 347         // Start the progress
 348         r = pakfire_progress_start(progress, (downloadsize) ? downloadsize : num_queued_xfers);
 349         if (r)
 350                 goto ERROR;
 351
 352         do {
 353                 // Make sure that we have up to parallel transfers active
 354                 if (!r) {
 355                         r = pakfire_httpclient_start_transfers(client, progress, &running_xfers);
 356                         if (r)
 357                                 goto ERROR;
 358                 }
 359
 360                 // Run cURL
 361                 r = curl_multi_perform(client->curl, &still_running);
 362                 if (r) {
 363                         CTX_ERROR(client->ctx, "cURL error: %s\n", curl_easy_strerror(r));
 364                         goto ERROR;
 365                 }
 366
 367                 for (;;) {
 368                         // Read the next message
 369                         msg = curl_multi_info_read(client->curl, &msgs_left);
 370                         if (!msg)
 371                                 break;
 372
 373                         switch (msg->msg) {
 374                                 case CURLMSG_DONE:
 375                                         // Update reference to transfer
 376                                         curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &xfer);
 377
 378                                         // Remove the handle
 379                                         curl_multi_remove_handle(client->curl, pakfire_xfer_handle(xfer));
 380
 381                                         // Find the matching xfer element
 382                                         x = pakfire_httpclient_xfer_find_running(client, xfer);
 383
 384                                         // Remove the transfer from the running list
 385                                         if (x)
 386                                                 TAILQ_REMOVE(&client->xfers_running, x, nodes);
 387                                         running_xfers--;
 388
 389                                         // Call the done callback
 390                                         r = pakfire_xfer_done(xfer, msg->data.result);
 391                                         switch (-r) {
 392                                                 // If we are asked to try again we will re-queue the transfer
 393                                                 case EAGAIN:
 394                                                         if (x)
 395                                                                 TAILQ_INSERT_TAIL(&client->xfers_queued, x, nodes);
 396                                                         break;
 397
 398                                                 // Otherwise this transfer has finished
 399                                                 default:
 400                                                         if (x)
 401                                                                 pakfire_httpclient_xfer_free(x);
 402                                                         if (r)
 403                                                                 goto ERROR;
 404                                                         break;
 405                                         }
 406
 407                                         // Reset transfer
 408                                         xfer = NULL;
 409                                         break;
 410
 411                                 default:
 412                                         CTX_ERROR(client->ctx, "Received unhandled cURL message %u\n", msg->msg);
 413                                         break;
 414                         }
 415                 }
 416
 417                 // Wait a little before going through the loop again
 418                 if (still_running)
 419                         curl_multi_wait(client->curl, NULL, 0, 100, NULL);
 420         } while (still_running || !TAILQ_EMPTY(&client->xfers_queued));
 421
 422         // We are finished!
 423         r = pakfire_progress_finish(progress);
 424         if (r)
 425                 goto ERROR;
 426
 427 ERROR:
 428         // If the client was aborted, we need to fail all remaining running transfers
 429         while (!TAILQ_EMPTY(&client->xfers_running)) {
 430                 x = TAILQ_LAST(&client->xfers_running, xfers_running);
 431                 TAILQ_REMOVE(&client->xfers_running, x, nodes);
 432
 433                 // Fail the transfer
 434                 pakfire_xfer_fail(x->xfer);
 435         }
 436
 437         if (progress)
 438                 pakfire_progress_unref(progress);
 439
 440         return r;
 441 }