src/libpakfire/filelist.c

   1 /*#############################################################################
   2 #                                                                             #
   3 # Pakfire - The IPFire package management system                              #
   4 # Copyright (C) 2014 Pakfire development team                                 #
   5 #                                                                             #
   6 # This program is free software: you can redistribute it and/or modify        #
   7 # it under the terms of the GNU General Public License as published by        #
   8 # the Free Software Foundation, either version 3 of the License, or           #
   9 # (at your option) any later version.                                         #
  10 #                                                                             #
  11 # This program is distributed in the hope that it will be useful,             #
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  14 # GNU General Public License for more details.                                #
  15 #                                                                             #
  16 # You should have received a copy of the GNU General Public License           #
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  18 #                                                                             #
  19 #############################################################################*/
  20
  21 #include <errno.h>
  22 #include <fnmatch.h>
  23 #include <fts.h>
  24 #include <stdlib.h>
  25 #include <string.h>
  26 #include <sys/queue.h>
  27
  28 #include <archive.h>
  29
  30 #include <pakfire/file.h>
  31 #include <pakfire/filelist.h>
  32 #include <pakfire/i18n.h>
  33 #include <pakfire/logging.h>
  34 #include <pakfire/pakfire.h>
  35 #include <pakfire/private.h>
  36 #include <pakfire/progressbar.h>
  37 #include <pakfire/string.h>
  38 #include <pakfire/util.h>
  39
  40 struct pakfire_filelist_element {
  41         TAILQ_ENTRY(pakfire_filelist_element) nodes;
  42
  43         struct pakfire_file* file;
  44 };
  45
  46 struct pakfire_filelist {
  47         struct pakfire* pakfire;
  48         int nrefs;
  49
  50         TAILQ_HEAD(entries, pakfire_filelist_element) files;
  51 };
  52
  53 PAKFIRE_EXPORT int pakfire_filelist_create(struct pakfire_filelist** list, struct pakfire* pakfire) {
  54         struct pakfire_filelist* l = calloc(1, sizeof(*l));
  55         if (!l)
  56                 return -ENOMEM;
  57
  58         l->pakfire = pakfire_ref(pakfire);
  59         l->nrefs = 1;
  60
  61         // Initialise files
  62         TAILQ_INIT(&l->files);
  63
  64         *list = l;
  65         return 0;
  66 }
  67
  68 static void pakfire_filelist_free(struct pakfire_filelist* list) {
  69         pakfire_filelist_clear(list);
  70         pakfire_unref(list->pakfire);
  71         free(list);
  72 }
  73
  74 PAKFIRE_EXPORT struct pakfire_filelist* pakfire_filelist_ref(struct pakfire_filelist* list) {
  75         list->nrefs++;
  76
  77         return list;
  78 }
  79
  80 PAKFIRE_EXPORT struct pakfire_filelist* pakfire_filelist_unref(struct pakfire_filelist* list) {
  81         if (--list->nrefs > 0)
  82                 return list;
  83
  84         pakfire_filelist_free(list);
  85         return NULL;
  86 }
  87
  88 PAKFIRE_EXPORT size_t pakfire_filelist_length(struct pakfire_filelist* list) {
  89         struct pakfire_filelist_element* element = NULL;
  90         size_t size = 0;
  91
  92         TAILQ_FOREACH(element, &list->files, nodes)
  93                 size++;
  94
  95         return size;
  96 }
  97
  98 size_t pakfire_filelist_total_size(struct pakfire_filelist* list) {
  99         struct pakfire_filelist_element* element = NULL;
 100         size_t size = 0;
 101
 102         TAILQ_FOREACH(element, &list->files, nodes)
 103                 size += pakfire_file_get_size(element->file);
 104
 105         return size;
 106 }
 107
 108 PAKFIRE_EXPORT int pakfire_filelist_is_empty(struct pakfire_filelist* list) {
 109         return TAILQ_EMPTY(&list->files);
 110 }
 111
 112 PAKFIRE_EXPORT void pakfire_filelist_clear(struct pakfire_filelist* list) {
 113         struct pakfire_filelist_element* element = NULL;
 114
 115         while (!TAILQ_EMPTY(&list->files)) {
 116                 // Fetch the first element
 117                 element = TAILQ_FIRST(&list->files);
 118
 119                 // Remove it from the list
 120                 TAILQ_REMOVE(&list->files, element, nodes);
 121
 122                 // Dereference the file
 123                 pakfire_file_unref(element->file);
 124
 125                 // Free it all
 126                 free(element);
 127         }
 128 }
 129
 130 PAKFIRE_EXPORT struct pakfire_file* pakfire_filelist_get(struct pakfire_filelist* list, size_t index) {
 131         struct pakfire_filelist_element* element = NULL;
 132
 133         // Fetch the first element
 134         element = TAILQ_FIRST(&list->files);
 135
 136         while (element && index--)
 137                 element = TAILQ_NEXT(element, nodes);
 138
 139         return pakfire_file_ref(element->file);
 140 }
 141
 142 PAKFIRE_EXPORT int pakfire_filelist_add(struct pakfire_filelist* list, struct pakfire_file* file) {
 143         struct pakfire_filelist_element* element = NULL;
 144         struct pakfire_filelist_element* e = NULL;
 145
 146         // Allocate a new element
 147         element = calloc(1, sizeof *element);
 148         if (!element) {
 149                 ERROR(list->pakfire, "Could not allocate a new filelist element: %m\n");
 150                 return 1;
 151         }
 152
 153         // Reference the file
 154         element->file = pakfire_file_ref(file);
 155
 156         // Fetch the last element
 157         e = TAILQ_LAST(&list->files, entries);
 158
 159         // Skip all elements that are "greater than" the element we want to add
 160         while (e) {
 161                 if (pakfire_file_cmp(e->file, file) <= 0)
 162                         break;
 163
 164                 e = TAILQ_PREV(e, entries, nodes);
 165         }
 166
 167         // If we found an element on the list, we will append after it,
 168         // otherwise we add right at the start.
 169         if (e)
 170                 TAILQ_INSERT_AFTER(&list->files, e, element, nodes);
 171         else
 172                 TAILQ_INSERT_HEAD(&list->files, element, nodes);
 173
 174         return 0;
 175 }
 176
 177 static int pakfire_filelist_remove(struct pakfire_filelist* list, struct pakfire_file* file) {
 178         struct pakfire_filelist_element* element = NULL;
 179
 180         TAILQ_FOREACH(element, &list->files, nodes) {
 181                 if (element->file == file) {
 182                         // Remove the element from the list
 183                         TAILQ_REMOVE(&list->files, element, nodes);
 184
 185                         // Free the element
 186                         pakfire_file_unref(element->file);
 187                         free(element);
 188
 189                         return 0;
 190                 }
 191         }
 192
 193         return 0;
 194 }
 195
 196 static int __pakfire_filelist_remove_one(
 197                 struct pakfire* pakfire, struct pakfire_file* file, void* data) {
 198         struct pakfire_filelist* list = (struct pakfire_filelist*)data;
 199
 200         // Remove the file from the given filelist
 201         return pakfire_filelist_remove(list, file);
 202 }
 203
 204 int pakfire_filelist_remove_all(
 205                 struct pakfire_filelist* list, struct pakfire_filelist* removees) {
 206         return pakfire_filelist_walk(removees, __pakfire_filelist_remove_one, list, 0);
 207 }
 208
 209 // Returns true if s contains globbing characters
 210 static int is_glob(const char* s) {
 211         if (strchr(s, '*'))
 212                 return 1;
 213
 214         if (strchr(s, '?'))
 215                 return 1;
 216
 217         return 0;
 218 }
 219
 220 static int pakfire_filelist_match_patterns(const char* path, const char** patterns) {
 221         int flags = 0;
 222         int r;
 223
 224         for (const char** pattern = patterns; *pattern; pattern++) {
 225                 // Match any subdirectories
 226                 if (pakfire_string_startswith(path, *pattern))
 227                         return 1;
 228
 229                 // Skip fnmatch if the pattern doesn't have any globbing characters
 230                 if (!is_glob(*pattern))
 231                         continue;
 232
 233                 // Reset flags
 234                 flags = 0;
 235
 236                 /*
 237                         fnmatch is way too eager for patterns line /usr/lib/lib*.so which will also match
 238                         things like /usr/lib/python3.x/blah/libblubb.so.
 239                         To prevent this for absolute file paths, we set the FNM_FILE_NAME flag so that
 240                         asterisk (*) won't match any slashes (/).
 241                 */
 242                 if (**pattern == '/')
 243                         flags |= FNM_FILE_NAME;
 244
 245                 // Perform matching
 246                 r = fnmatch(*pattern, path, flags);
 247
 248                 // Found a match
 249                 if (r == 0)
 250                         return 1;
 251
 252                 // No match found
 253                 else if (r == FNM_NOMATCH)
 254                         continue;
 255
 256                 // Any other error
 257                 else
 258                         return r;
 259         }
 260
 261         // No match
 262         return 0;
 263 }
 264
 265 int pakfire_filelist_scan(struct pakfire_filelist* list, const char* root,
 266                 const char** includes, const char** excludes) {
 267         struct pakfire_file* file = NULL;
 268         struct archive_entry* entry = NULL;
 269         int r = 1;
 270
 271         // Root must be absolute
 272         if (!pakfire_path_is_absolute(root)) {
 273                 errno = EINVAL;
 274                 return 1;
 275         }
 276
 277         DEBUG(list->pakfire, "Scanning %s...\n", root);
 278
 279         if (includes) {
 280                 DEBUG(list->pakfire, "  Includes:\n");
 281
 282                 for (const char** include = includes; *include; include++)
 283                         DEBUG(list->pakfire, "    %s\n", *include);
 284         }
 285
 286         if (excludes) {
 287                 DEBUG(list->pakfire, "  Excludes:\n");
 288
 289                 for (const char** exclude = excludes; *exclude; exclude++)
 290                         DEBUG(list->pakfire, "    %s\n", *exclude);
 291         }
 292
 293         struct archive* reader = pakfire_make_archive_disk_reader(list->pakfire, 1);
 294         if (!reader)
 295                 return 1;
 296
 297         // Allocate a new file entry
 298         entry = archive_entry_new();
 299         if (!entry)
 300                 goto ERROR;
 301
 302         char* paths[] = {
 303                 (char*)root, NULL,
 304         };
 305
 306         // Walk through the whole file system tree and find all matching files
 307         FTS* tree = fts_open(paths, FTS_NOCHDIR, 0);
 308         if (!tree)
 309                 goto ERROR;
 310
 311         FTSENT* node = NULL;
 312         const char* path = NULL;
 313
 314         while ((node = fts_read(tree))) {
 315                 // Ignore any directories in post order
 316                 if (node->fts_info == FTS_DP)
 317                         continue;
 318
 319                 // Compute the relative path
 320                 path = pakfire_path_relpath(root, node->fts_path);
 321                 if (!path || !*path)
 322                         continue;
 323
 324                 // Skip excludes
 325                 if (excludes && pakfire_filelist_match_patterns(path, excludes)) {
 326                         DEBUG(list->pakfire, "Skipping %s...\n", path);
 327
 328                         r = fts_set(tree, node, FTS_SKIP);
 329                         if (r)
 330                                 goto ERROR;
 331
 332                         continue;
 333                 }
 334
 335                 // Skip what is not included
 336                 if (includes && !pakfire_filelist_match_patterns(path, includes)) {
 337                         DEBUG(list->pakfire, "Skipping %s...\n", path);
 338
 339                         // We do not mark the whole tree as to skip because some matches might
 340                         // look for file extensions, etc.
 341                         continue;
 342                 }
 343
 344                 DEBUG(list->pakfire, "Processing %s...\n", path);
 345
 346                 // Reset the file entry
 347                 entry = archive_entry_clear(entry);
 348
 349                 // Set path
 350                 archive_entry_set_pathname(entry, path);
 351
 352                 // Set source path
 353                 archive_entry_copy_sourcepath(entry, node->fts_path);
 354
 355                 // Read all file attributes from disk
 356                 r = archive_read_disk_entry_from_file(reader, entry, -1, node->fts_statp);
 357                 if (r) {
 358                         ERROR(list->pakfire, "Could not read from %s: %m\n", node->fts_path);
 359                         goto ERROR;
 360                 }
 361
 362                 // Create file
 363                 r = pakfire_file_create_from_archive_entry(&file, list->pakfire, entry);
 364                 if (r)
 365                         goto ERROR;
 366
 367                 // Append it to the list
 368                 r = pakfire_filelist_add(list, file);
 369                 if (r) {
 370                         pakfire_file_unref(file);
 371                         goto ERROR;
 372                 }
 373
 374                 pakfire_file_unref(file);
 375         }
 376
 377         // Success
 378         r = 0;
 379
 380 ERROR:
 381         if (entry)
 382                 archive_entry_free(entry);
 383         archive_read_free(reader);
 384
 385         return r;
 386 }
 387
 388 int pakfire_filelist_contains(struct pakfire_filelist* list, const char* pattern) {
 389         struct pakfire_filelist_element* element = NULL;
 390
 391         if (!pattern) {
 392                 errno = EINVAL;
 393                 return -1;
 394         }
 395
 396         TAILQ_FOREACH(element, &list->files, nodes) {
 397                 const char* path = pakfire_file_get_path(element->file);
 398                 if (!path)
 399                         return -1;
 400
 401                 int r = fnmatch(pattern, path, 0);
 402                 if (r == 0)
 403                         return 1;
 404         }
 405
 406         return 0;
 407 }
 408
 409 int pakfire_filelist_walk(struct pakfire_filelist* list,
 410                 pakfire_filelist_walk_callback callback, void* data, int flags) {
 411         struct pakfire_filelist_element* element = NULL;
 412         int r = 0;
 413
 414         // Call the callback once for every element on the list
 415         TAILQ_FOREACH(element, &list->files, nodes) {
 416                 // Call the callback
 417                 r = callback(list->pakfire, element->file, data);
 418                 if (r)
 419                         break;
 420         }
 421
 422         return r;
 423 }
 424
 425 static int __pakfire_filelist_dump(
 426                 struct pakfire* pakfire, struct pakfire_file* file, void* data) {
 427         int* flags = (int*)data;
 428
 429         char* s = pakfire_file_dump(file, *flags);
 430         if (s) {
 431                 INFO(pakfire, "%s\n", s);
 432                 free(s);
 433         }
 434
 435         return 0;
 436 }
 437
 438 int pakfire_filelist_dump(struct pakfire_filelist* list, int flags) {
 439         return pakfire_filelist_walk(list, __pakfire_filelist_dump, &flags, 0);
 440 }
 441
 442 /*
 443         Verifies all files on the filelist
 444 */
 445 int pakfire_filelist_verify(struct pakfire_filelist* list, struct pakfire_filelist* errors) {
 446         struct pakfire_filelist_element* element = NULL;
 447         struct pakfire_progressbar* progressbar = NULL;
 448         int status;
 449         int r;
 450
 451         const size_t length = pakfire_filelist_length(list);
 452
 453         DEBUG(list->pakfire, "Verifying filelist (%zu file(s))...\n", length);
 454
 455         // Setup progressbar
 456         r = pakfire_progressbar_create(&progressbar, NULL);
 457         if (r)
 458                 goto ERROR;
 459
 460         // Add status
 461         r = pakfire_progressbar_add_string(progressbar, _("Verifying Files..."));
 462         if (r)
 463                 goto ERROR;
 464
 465         // Add bar
 466         r = pakfire_progressbar_add_bar(progressbar);
 467         if (r)
 468                 goto ERROR;
 469
 470         // Add percentage
 471         r = pakfire_progressbar_add_percentage(progressbar);
 472         if (r)
 473                 goto ERROR;
 474
 475         // Add ETA
 476         r = pakfire_progressbar_add_eta(progressbar);
 477         if (r)
 478                 goto ERROR;
 479
 480         // Start the progressbar
 481         r = pakfire_progressbar_start(progressbar, length);
 482         if (r)
 483                 goto ERROR;
 484
 485         // Iterate over the entire list
 486         TAILQ_FOREACH(element, &list->files, nodes) {
 487                 // Verify the file
 488                 r = pakfire_file_verify(element->file, &status);
 489                 if (r)
 490                         goto ERROR;
 491
 492                 // If the verification failed, we append it to the errors list
 493                 if (status) {
 494                         // Append the file to the error list
 495                         r = pakfire_filelist_add(errors, element->file);
 496                         if (r)
 497                                 goto ERROR;
 498                 }
 499
 500                 // Increment progressbar
 501                 r = pakfire_progressbar_increment(progressbar, 1);
 502                 if (r)
 503                         goto ERROR;
 504         }
 505
 506         // Finish
 507         r = pakfire_progressbar_finish(progressbar);
 508         if (r)
 509                 goto ERROR;
 510
 511 ERROR:
 512         if (progressbar)
 513                 pakfire_progressbar_unref(progressbar);
 514
 515         return r;
 516 }
 517
 518 int pakfire_filelist_cleanup(struct pakfire_filelist* list) {
 519         struct pakfire_filelist_element* element = NULL;
 520         int r;
 521
 522         // Nothing to do if the filelist is empty
 523         if (pakfire_filelist_is_empty(list))
 524                 return 0;
 525
 526         // Walk through the list backwards
 527         TAILQ_FOREACH_REVERSE(element, &list->files, entries, nodes) {
 528                 r = pakfire_file_cleanup(element->file);
 529                 if (r)
 530                         return r;
 531         }
 532
 533         return 0;
 534 }
 535
 536 static int __pakfire_filelist_matches_class(struct pakfire* pakfire,
 537                 struct pakfire_file* file, void* p) {
 538         int* class = (int*)p;
 539
 540         return pakfire_file_matches_class(file, *class);
 541 }
 542
 543 /*
 544         Returns true if any file on the list matches class
 545 */
 546 int pakfire_filelist_matches_class(struct pakfire_filelist* list, int class) {
 547         return pakfire_filelist_walk(list, __pakfire_filelist_matches_class, &class, 0);
 548 }