src/libpakfire/packager.c

   1 /*#############################################################################
   2 #                                                                             #
   3 # Pakfire - The IPFire package management system                              #
   4 # Copyright (C) 2021 Pakfire development team                                 #
   5 #                                                                             #
   6 # This program is free software: you can redistribute it and/or modify        #
   7 # it under the terms of the GNU General Public License as published by        #
   8 # the Free Software Foundation, either version 3 of the License, or           #
   9 # (at your option) any later version.                                         #
  10 #                                                                             #
  11 # This program is distributed in the hope that it will be useful,             #
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  14 # GNU General Public License for more details.                                #
  15 #                                                                             #
  16 # You should have received a copy of the GNU General Public License           #
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  18 #                                                                             #
  19 #############################################################################*/
  20
  21 #include <errno.h>
  22 #include <fcntl.h>
  23 #include <linux/limits.h>
  24 #include <stdlib.h>
  25 #include <sys/stat.h>
  26 #include <sys/types.h>
  27 #include <time.h>
  28 #include <unistd.h>
  29
  30 #include <archive.h>
  31 #include <archive_entry.h>
  32
  33 #include <json.h>
  34
  35 // Enable legacy logging
  36 #define PAKFIRE_LEGACY_LOGGING
  37
  38 #include <pakfire/archive.h>
  39 #include <pakfire/compress.h>
  40 #include <pakfire/constants.h>
  41 #include <pakfire/logging.h>
  42 #include <pakfire/package.h>
  43 #include <pakfire/packager.h>
  44 #include <pakfire/pakfire.h>
  45 #include <pakfire/pwd.h>
  46 #include <pakfire/string.h>
  47 #include <pakfire/util.h>
  48
  49 struct pakfire_packager {
  50         struct pakfire* pakfire;
  51         int nrefs;
  52         time_t time_created;
  53
  54         struct pakfire_package* pkg;
  55         char filename[PATH_MAX];
  56
  57         // Reader
  58         struct archive* reader;
  59
  60         // Payload
  61         struct pakfire_filelist* filelist;
  62
  63         // Scriptlets
  64         struct pakfire_scriptlet** scriptlets;
  65         unsigned int num_scriptlets;
  66
  67         // Digests
  68         unsigned int digests;
  69 };
  70
  71 static void pakfire_packager_free(struct pakfire_packager* packager) {
  72         // Scriptlets
  73         if (packager->scriptlets) {
  74                 for (unsigned int i = 0; i < packager->num_scriptlets; i++)
  75                         pakfire_scriptlet_unref(packager->scriptlets[i]);
  76                 free(packager->scriptlets);
  77         }
  78
  79         if (packager->reader)
  80                 archive_read_free(packager->reader);
  81
  82         if (packager->filelist)
  83                 pakfire_filelist_unref(packager->filelist);
  84         pakfire_package_unref(packager->pkg);
  85         pakfire_unref(packager->pakfire);
  86         free(packager);
  87 }
  88
  89 int pakfire_packager_create(struct pakfire_packager** packager,
  90                 struct pakfire* pakfire, struct pakfire_package* pkg) {
  91         struct pakfire_packager* p = NULL;
  92         char hostname[HOST_NAME_MAX];
  93         int r = 1;
  94
  95         // Allocate the packager object
  96         p = calloc(1, sizeof(*p));
  97         if (!p)
  98                 return -errno;
  99
 100         // Save creation time
 101         p->time_created = time(NULL);
 102
 103         // Initialize reference counting
 104         p->nrefs = 1;
 105
 106         // Store a reference to Pakfire
 107         p->pakfire = pakfire_ref(pakfire);
 108
 109         // Store a reference to the package
 110         p->pkg = pakfire_package_ref(pkg);
 111
 112         // Use the default digests
 113         p->digests = PAKFIRE_PACKAGER_DIGESTS;
 114
 115         // Set distribution
 116         const char* tag = pakfire_get_distro_tag(p->pakfire);
 117         if (!tag) {
 118                 ERROR(p->pakfire, "Distribution tag is not configured: %m\n");
 119                 goto ERROR;
 120         }
 121
 122         pakfire_package_set_string(pkg, PAKFIRE_PKG_DISTRO, tag);
 123
 124         // Fetch the hostname
 125         r = gethostname(hostname, sizeof(hostname));
 126         if (r) {
 127                 ERROR(p->pakfire, "Could not determine the hostname: %m\n");
 128                 goto ERROR;
 129         }
 130
 131         // Set build host
 132         r = pakfire_package_set_string(pkg, PAKFIRE_PKG_BUILD_HOST, hostname);
 133         if (r) {
 134                 ERROR_ERRNO(p->pakfire, r, "Could not set the hostname: %m\n");
 135                 goto ERROR;
 136         }
 137
 138         // Set build time
 139         pakfire_package_set_num(pkg, PAKFIRE_PKG_BUILD_TIME, p->time_created);
 140
 141         // Create reader
 142         p->reader = pakfire_make_archive_disk_reader(p->pakfire, 1);
 143         if (!p->reader)
 144                 goto ERROR;
 145
 146         // Create filelist
 147         r = pakfire_filelist_create(&p->filelist, p->pakfire);
 148         if (r)
 149                 goto ERROR;
 150
 151         // Add a requirement for the cryptographic algorithms we are using
 152         if (p->digests & PAKFIRE_DIGEST_SHA3_512) {
 153                 r = pakfire_package_add_dep(p->pkg,
 154                         PAKFIRE_PKG_REQUIRES, "pakfire(Digest-SHA3-512)");
 155                 if (r)
 156                         goto ERROR;
 157         }
 158         if (p->digests & PAKFIRE_DIGEST_SHA3_256) {
 159                 r = pakfire_package_add_dep(p->pkg,
 160                         PAKFIRE_PKG_REQUIRES, "pakfire(Digest-SHA3-256)");
 161                 if (r)
 162                         goto ERROR;
 163         }
 164         if (p->digests & PAKFIRE_DIGEST_BLAKE2B512) {
 165                 r = pakfire_package_add_dep(p->pkg,
 166                         PAKFIRE_PKG_REQUIRES, "pakfire(Digest-BLAKE2b512)");
 167                 if (r)
 168                         goto ERROR;
 169         }
 170         if (p->digests & PAKFIRE_DIGEST_BLAKE2S256) {
 171                 r = pakfire_package_add_dep(p->pkg,
 172                         PAKFIRE_PKG_REQUIRES, "pakfire(Digest-BLAKE2s256)");
 173                 if (r)
 174                         goto ERROR;
 175         }
 176         if (p->digests & PAKFIRE_DIGEST_SHA2_512) {
 177                 r = pakfire_package_add_dep(p->pkg,
 178                         PAKFIRE_PKG_REQUIRES, "pakfire(Digest-SHA2-512)");
 179                 if (r)
 180                         goto ERROR;
 181         }
 182         if (p->digests & PAKFIRE_DIGEST_SHA2_256) {
 183                 r = pakfire_package_add_dep(p->pkg,
 184                         PAKFIRE_PKG_REQUIRES, "pakfire(Digest-SHA2-256)");
 185                 if (r)
 186                         goto ERROR;
 187         }
 188
 189         // Return a reference
 190         *packager = pakfire_packager_ref(p);
 191
 192 ERROR:
 193         if (p)
 194                 pakfire_packager_unref(p);
 195
 196         return r;
 197 }
 198
 199 struct pakfire_packager* pakfire_packager_ref(
 200                 struct pakfire_packager* packager) {
 201         ++packager->nrefs;
 202
 203         return packager;
 204 }
 205
 206 struct pakfire_packager* pakfire_packager_unref(
 207                 struct pakfire_packager* packager) {
 208         if (--packager->nrefs > 0)
 209                 return packager;
 210
 211         pakfire_packager_free(packager);
 212         return NULL;
 213 }
 214
 215 const char* pakfire_packager_filename(struct pakfire_packager* packager) {
 216         if (!*packager->filename) {
 217                 const char* filename = pakfire_package_get_string(packager->pkg, PAKFIRE_PKG_FILENAME);
 218
 219                 // Add arch
 220                 if (pakfire_package_is_source(packager->pkg))
 221                         pakfire_string_set(packager->filename, filename);
 222                 else {
 223                         const char* arch = pakfire_package_get_string(packager->pkg, PAKFIRE_PKG_ARCH);
 224
 225                         pakfire_string_format(packager->filename, "%s/%s", arch, filename);
 226                 }
 227         }
 228
 229         return packager->filename;
 230 }
 231
 232 static struct archive_entry* pakfire_packager_create_file(
 233                 struct pakfire_packager* packager, const char* filename, size_t size, mode_t mode) {
 234         // Create a new file entry
 235         struct archive_entry* entry = archive_entry_new();
 236         if (!entry)
 237                 return NULL;
 238
 239         // Set filename
 240         archive_entry_set_pathname(entry, filename);
 241
 242         // This is a regular file
 243     archive_entry_set_filetype(entry, AE_IFREG);
 244     archive_entry_set_perm(entry, mode);
 245
 246         // Set size
 247         archive_entry_set_size(entry, size);
 248
 249         // Set ownership
 250         archive_entry_set_uname(entry, "root");
 251         archive_entry_set_uid(entry, 0);
 252         archive_entry_set_gname(entry, "root");
 253         archive_entry_set_gid(entry, 0);
 254
 255         // Set times
 256         archive_entry_set_birthtime(entry, packager->time_created, 0);
 257         archive_entry_set_ctime(entry, packager->time_created, 0);
 258         archive_entry_set_mtime(entry, packager->time_created, 0);
 259         archive_entry_set_atime(entry, packager->time_created, 0);
 260
 261         return entry;
 262 }
 263
 264 static int pakfire_packager_write_file_from_buffer(struct pakfire_packager* packager,
 265                 struct archive* a, const char* filename, mode_t mode, const char* buffer) {
 266         size_t size = strlen(buffer);
 267
 268         // Create a new file
 269         struct archive_entry* entry = pakfire_packager_create_file(packager, filename, size, mode);
 270         if (!entry) {
 271                 ERROR(packager->pakfire, "Could not create file '%s'\n", filename);
 272                 return 1;
 273         }
 274
 275         // This is the end of the header
 276         int r = archive_write_header(a, entry);
 277         if (r) {
 278                 ERROR(packager->pakfire, "Error writing header: %s\n", archive_error_string(a));
 279                 goto ERROR;
 280         }
 281
 282         // Write content
 283         r = archive_write_data(a, buffer, strlen(buffer));
 284         if (r < 0) {
 285                 ERROR(packager->pakfire, "Error writing data: %s\n", archive_error_string(a));
 286                 goto ERROR;
 287         }
 288
 289         // Success
 290         r = 0;
 291
 292 ERROR:
 293         archive_entry_free(entry);
 294
 295         return r;
 296 }
 297
 298 static int pakfire_packager_write_format(struct pakfire_packager* packager,
 299                 struct archive* a) {
 300         const char buffer[] = TO_STRING(PACKAGE_FORMAT) "\n";
 301
 302         DEBUG(packager->pakfire, "Writing package format\n");
 303
 304         int r = pakfire_packager_write_file_from_buffer(packager, a,
 305                 "pakfire-format", 0444, buffer);
 306         if (r)
 307                 return r;
 308
 309         // Add package format marker
 310         r = pakfire_package_add_dep(packager->pkg, PAKFIRE_PKG_REQUIRES,
 311                 "pakfire(PackageFormat-" TO_STRING(PACKAGE_FORMAT) ")");
 312         if (r)
 313                 return r;
 314
 315         return 0;
 316 }
 317
 318 static char* pakfire_packager_make_metadata(struct pakfire_packager* packager) {
 319         char* result = NULL;
 320
 321         // Convert all package metadata to JSON
 322         struct json_object* md = pakfire_package_to_json(packager->pkg);
 323         if (!md)
 324                 goto ERROR;
 325
 326         // Serialize JSON to file
 327         const char* s = json_object_to_json_string_ext(md, 0);
 328         if (!s)
 329                 goto ERROR;
 330
 331         // Copy result onto heap
 332         result = strdup(s);
 333         if (!result)
 334                 goto ERROR;
 335
 336 ERROR:
 337         // Free metadata
 338         if (md)
 339                 json_object_put(md);
 340
 341         return result;
 342 }
 343
 344 static int pakfire_packager_write_metadata(struct pakfire_packager* packager,
 345                 struct archive* a) {
 346         // Make metadata
 347         char* buffer = pakfire_packager_make_metadata(packager);
 348         if (!buffer)
 349                 return 1;
 350
 351         DEBUG(packager->pakfire, "Generated package metadata:\n%s\n", buffer);
 352
 353         // Write buffer
 354         int r = pakfire_packager_write_file_from_buffer(packager, a,
 355                 ".PKGINFO", 0444, buffer);
 356
 357         free(buffer);
 358
 359         return r;
 360 }
 361
 362 static int pakfire_packager_write_scriptlet(struct pakfire_packager* packager,
 363                 struct archive* a, struct pakfire_scriptlet* scriptlet) {
 364         char filename[PATH_MAX];
 365         size_t size;
 366         int r;
 367
 368         // Fetch type
 369         const char* type = pakfire_scriptlet_get_type(scriptlet);
 370
 371         DEBUG(packager->pakfire, "Writing scriptlet '%s' to package\n", type);
 372
 373         // Make filename
 374         r = pakfire_string_format(filename, ".scriptlets/%s", type);
 375         if (r)
 376                 return r;
 377
 378         // Fetch scriptlet
 379         const char* data = pakfire_scriptlet_get_data(scriptlet, &size);
 380
 381         // Write file
 382         return pakfire_packager_write_file_from_buffer(packager, a, filename, 0544, data);
 383 }
 384
 385 /*
 386         This function is being called at the end when all data has been added to the package.
 387
 388         It will create a new archive and write the package to the given file descriptor.
 389 */
 390 int pakfire_packager_finish(struct pakfire_packager* packager, FILE* f) {
 391         struct archive* a = NULL;
 392         unsigned int level;
 393         int r = 1;
 394
 395         const char* nevra = pakfire_package_get_string(packager->pkg, PAKFIRE_PKG_NEVRA);
 396
 397         // Add requires feature markers
 398         if (pakfire_package_has_rich_deps(packager->pkg)) {
 399                 r = pakfire_package_add_dep(packager->pkg,
 400                         PAKFIRE_PKG_REQUIRES, "pakfire(RichDependencies)");
 401                 if (r)
 402                         goto ERROR;
 403         }
 404
 405         // Add filelist
 406         r = pakfire_package_set_filelist(packager->pkg, packager->filelist);
 407         if (r)
 408                 goto ERROR;
 409
 410         const size_t installsize = pakfire_filelist_total_size(packager->filelist);
 411
 412         // Store total install size
 413         r = pakfire_package_set_num(packager->pkg, PAKFIRE_PKG_INSTALLSIZE, installsize);
 414         if (r)
 415                 goto ERROR;
 416
 417         // Select compression level
 418         if (pakfire_package_is_source(packager->pkg))
 419                 level = 1;
 420         else
 421                 level = 22;
 422
 423         // Create a new archive that is being compressed as fast as possible
 424         r = pakfire_compress_create_archive(packager->pakfire, &a, f,
 425                 PAKFIRE_COMPRESS_ZSTD, level);
 426         if (r)
 427                 goto ERROR;
 428
 429         // Add feature marker
 430         pakfire_package_add_dep(packager->pkg,
 431                 PAKFIRE_PKG_REQUIRES, "pakfire(Compress-Zstandard)");
 432
 433         // Start with the format file
 434         r = pakfire_packager_write_format(packager, a);
 435         if (r) {
 436                 ERROR(packager->pakfire, "Could not add format file to archive: %s\n",
 437                         archive_error_string(a));
 438                 goto ERROR;
 439         }
 440
 441         // Write the metadata
 442         r = pakfire_packager_write_metadata(packager, a);
 443         if (r) {
 444                 ERROR(packager->pakfire, "Could not add metadata file to archive: %s\n",
 445                         archive_error_string(a));
 446                 goto ERROR;
 447         }
 448
 449         // Write scriptlets
 450         for (unsigned int i = 0; i < packager->num_scriptlets; i++) {
 451                 r = pakfire_packager_write_scriptlet(packager, a, packager->scriptlets[i]);
 452                 if (r) {
 453                         ERROR(packager->pakfire, "Could not add scriptlet to the archive: %m\n");
 454                         goto ERROR;
 455                 }
 456         }
 457
 458         // Write the payload
 459         r = pakfire_compress(packager->pakfire, a, packager->filelist, nevra,
 460                 PAKFIRE_COMPRESS_SHOW_THROUGHPUT, PAKFIRE_PACKAGER_DIGESTS);
 461         if (r)
 462                 goto ERROR;
 463
 464         // Flush all buffers to disk
 465         fflush(f);
 466
 467         // Success
 468         r = 0;
 469
 470 ERROR:
 471         if (a)
 472                 archive_write_free(a);
 473
 474         return r;
 475 }
 476
 477 int pakfire_packager_finish_to_directory(struct pakfire_packager* packager,
 478                 const char* target, char** result) {
 479         char path[PATH_MAX];
 480         char tmppath[PATH_MAX];
 481         FILE* f = NULL;
 482         int r = 1;
 483
 484         // target cannot be empty
 485         if (!target) {
 486                 errno = EINVAL;
 487                 return 1;
 488         }
 489
 490         // Get the filename of the package
 491         const char* filename = pakfire_packager_filename(packager);
 492         if (!filename) {
 493                 ERROR(packager->pakfire, "Could not generate filename for package: %m\n");
 494                 r = 1;
 495                 goto ERROR;
 496         }
 497
 498         // Make the package path
 499         r = pakfire_string_format(path, "%s/%s", target, filename);
 500         if (r)
 501                 goto ERROR;
 502
 503         // Create the parent directory
 504         r = pakfire_mkparentdir(path, 0755);
 505         if (r)
 506                 goto ERROR;
 507
 508         // Create a temporary file in the target directory
 509         r = pakfire_string_format(tmppath, "%s.XXXXXX", path);
 510         if (r)
 511                 goto ERROR;
 512
 513         // Create a temporary result file
 514         f = pakfire_mktemp(tmppath, 0);
 515         if (!f)
 516                 goto ERROR;
 517
 518         // Write the finished package
 519         r = pakfire_packager_finish(packager, f);
 520         if (r) {
 521                 ERROR(packager->pakfire, "pakfire_packager_finish() failed: %m\n");
 522                 goto ERROR;
 523         }
 524
 525         // Move the temporary file to destination
 526         r = rename(tmppath, path);
 527         if (r) {
 528                 ERROR(packager->pakfire, "Could not move %s to %s: %m\n", tmppath, path);
 529                 goto ERROR;
 530         }
 531
 532         DEBUG(packager->pakfire, "Package written to %s\n", path);
 533
 534         // Store result path if requested
 535         if (result) {
 536                 *result = strdup(path);
 537                 if (!*result) {
 538                         r = 1;
 539                         goto ERROR;
 540                 }
 541         }
 542
 543         // Success
 544         r = 0;
 545
 546 ERROR:
 547         if (f)
 548                 fclose(f);
 549
 550         // Remove temporary file
 551         if (r && *tmppath)
 552                 unlink(tmppath);
 553
 554         return r;
 555 }
 556
 557 int pakfire_packager_add_file(struct pakfire_packager* packager, struct pakfire_file* file) {
 558         int r;
 559
 560         // Check input
 561         if (!file) {
 562                 errno = EINVAL;
 563                 return 1;
 564         }
 565
 566         // Fetch path
 567         const char* path = pakfire_file_get_path(file);
 568
 569         // Files cannot have an empty path
 570         if (!path || !*path) {
 571                 ERROR(packager->pakfire, "Cannot add a file with an empty path\n");
 572                 errno = EPERM;
 573                 return 1;
 574
 575         // Hidden files cannot be added
 576         } else if (*path == '.') {
 577                 ERROR(packager->pakfire, "Hidden files cannot be added to a package: %s\n", path);
 578                 errno = EPERM;
 579                 return 1;
 580         }
 581
 582         DEBUG(packager->pakfire, "Adding file to payload: %s\n", path);
 583
 584         // Detect the MIME type
 585         r = pakfire_file_detect_mimetype(file);
 586         if (r)
 587                 return r;
 588
 589         // Overwrite a couple of things for source archives
 590         if (pakfire_package_is_source(packager->pkg)) {
 591                 // Reset permissions
 592                 pakfire_file_set_perms(file, 0644);
 593
 594                 // Reset file ownership
 595                 pakfire_file_set_uname(file, "root");
 596                 pakfire_file_set_gname(file, "root");
 597         }
 598
 599         // Handle systemd sysusers
 600         if (pakfire_file_matches(file, "/usr/lib/sysusers.d/*.conf")) {
 601                 pakfire_package_add_dep(packager->pkg,
 602                         PAKFIRE_PKG_REQUIRES, "pakfire(systemd-sysusers)");
 603
 604                 // Ask to pre-install /usr/bin/systemd-sysusers
 605                 pakfire_package_add_dep(packager->pkg,
 606                         PAKFIRE_PKG_PREREQUIRES, "/usr/bin/systemd-sysusers");
 607         }
 608
 609         // Handle systemd tmpfiles
 610         if (pakfire_file_matches(file, "/usr/lib/tmpfiles.d/*.conf")) {
 611                 pakfire_package_add_dep(packager->pkg,
 612                         PAKFIRE_PKG_REQUIRES, "pakfire(systemd-tmpfiles)");
 613
 614                 // Ask to pre-install systemd
 615                 pakfire_package_add_dep(packager->pkg,
 616                         PAKFIRE_PKG_PREREQUIRES, "systemd");
 617         }
 618
 619         // Append the file to the filelist
 620         return pakfire_filelist_add(packager->filelist, file);
 621 }
 622
 623 int pakfire_packager_add(struct pakfire_packager* packager,
 624                 const char* sourcepath, const char* path) {
 625         struct pakfire_file* file = NULL;
 626         int r;
 627
 628         // Create a new file object
 629         r = pakfire_file_create(&file, packager->pakfire);
 630         if (r)
 631                 goto ERROR;
 632
 633         // Read the meta information
 634         r = pakfire_file_read(file, packager->reader, sourcepath);
 635         if (r)
 636                 goto ERROR;
 637
 638         if (!path)
 639                 path = sourcepath;
 640
 641         // Assign a new path for inside the archive
 642         r = pakfire_file_set_path(file, path);
 643         if (r)
 644                 goto ERROR;
 645
 646         // Call the main function
 647         r = pakfire_packager_add_file(packager, file);
 648
 649 ERROR:
 650         if (file)
 651                 pakfire_file_unref(file);
 652
 653         return r;
 654 }
 655
 656 static int __pakfire_packager_add_files(struct pakfire* pakfire,
 657                 struct pakfire_file* file, void* p) {
 658         struct pakfire_packager* packager = (struct pakfire_packager*)p;
 659
 660         return pakfire_packager_add_file(packager, file);
 661 }
 662
 663 int pakfire_packager_add_files(
 664                 struct pakfire_packager* packager, struct pakfire_filelist* filelist) {
 665         // Add all files on the filelist
 666         return pakfire_filelist_walk(filelist, __pakfire_packager_add_files, packager, 0);
 667 }
 668
 669 int pakfire_packager_add_scriptlet(struct pakfire_packager* packager,
 670                 struct pakfire_scriptlet* scriptlet) {
 671         if (!scriptlet) {
 672                 errno = EINVAL;
 673                 return 1;
 674         }
 675
 676         // Extend array
 677         packager->scriptlets = reallocarray(packager->scriptlets,
 678                 packager->num_scriptlets + 1, sizeof(*packager->scriptlets));
 679         if (!packager->scriptlets)
 680                 return 1;
 681
 682         // Append scriptlet
 683         packager->scriptlets[packager->num_scriptlets++] = pakfire_scriptlet_ref(scriptlet);
 684
 685         return 0;
 686 }
 687
 688 /*
 689         Removes all files on the filelist
 690 */
 691 int pakfire_packager_cleanup(struct pakfire_packager* packager) {
 692         // Delete all files on the filelist
 693         return pakfire_filelist_cleanup(packager->filelist, PAKFIRE_FILE_CLEANUP_TIDY);
 694 }