python/pakfire/packages/packager.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import collections
  23 import fnmatch
  24 import glob
  25 import hashlib
  26 import os
  27 import progressbar
  28 import re
  29 import shutil
  30 import sys
  31 import tarfile
  32 import tempfile
  33 import time
  34 import uuid
  35 import zlib
  36
  37 import logging
  38 log = logging.getLogger("pakfire")
  39
  40 import pakfire.lzma as lzma
  41 import pakfire.util as util
  42
  43 from pakfire.constants import *
  44 from pakfire.i18n import _
  45
  46 from file import BinaryPackage, InnerTarFile, InnerTarFileXz, SourcePackage
  47
  48 class Packager(object):
  49         payload_compression = None
  50
  51         def __init__(self, pakfire, pkg):
  52                 self.pakfire = pakfire
  53                 self.pkg = pkg
  54
  55                 self.files = []
  56                 self.tmpfiles = []
  57
  58         def __del__(self):
  59                 for file in self.tmpfiles:
  60                         if not os.path.exists(file):
  61                                 continue
  62
  63                         log.debug("Removing tmpfile: %s" % file)
  64
  65                         if os.path.isdir(file):
  66                                 util.rm(file)
  67                         else:
  68                                 os.remove(file)
  69
  70         def mktemp(self, directory=False):
  71                 if directory:
  72                         filename = os.path.join("/", LOCAL_TMP_PATH, util.random_string())
  73                         os.makedirs(filename)
  74                 else:
  75                         f = tempfile.NamedTemporaryFile(mode="w", delete=False)
  76                         f.close()
  77
  78                         filename = f.name
  79
  80                 self.tmpfiles.append(filename)
  81
  82                 return filename
  83
  84         def save(self, filename):
  85                 # Create a new tar archive.
  86                 tar = tarfile.TarFile(filename, mode="w", format=tarfile.PAX_FORMAT)
  87
  88                 # Add package formation information.
  89                 # Must always be the first file in the archive.
  90                 formatfile = self.create_package_format()
  91                 tar.add(formatfile, arcname="pakfire-format")
  92
  93                 # XXX make sure all files belong to the root user
  94
  95                 # Create checksum file.
  96                 chksumsfile = self.mktemp()
  97                 chksums = open(chksumsfile, "w")
  98
  99                 # Add all files to tar file.
 100                 for arcname, filename in self.files:
 101                         tar.add(filename, arcname=arcname)
 102
 103                         # Calculating the hash sum of the added file
 104                         # and store it in the chksums file.
 105                         f = open(filename)
 106                         h = hashlib.sha512()
 107                         while True:
 108                                 buf = f.read(BUFFER_SIZE)
 109                                 if not buf:
 110                                         break
 111
 112                                 h.update(buf)
 113                         f.close()
 114
 115                         chksums.write("%-10s %s\n" % (arcname, h.hexdigest()))
 116
 117                 # Close checksum file and attach it to the end.
 118                 chksums.close()
 119                 tar.add(chksumsfile, "chksums")
 120
 121                 # Close the tar file.
 122                 tar.close()
 123
 124         def add(self, filename, arcname=None):
 125                 if not arcname:
 126                         arcname = os.path.basename(filename)
 127
 128                 log.debug("Adding %s (as %s) to tarball." % (filename, arcname))
 129                 self.files.append((arcname, filename))
 130
 131         def create_package_format(self):
 132                 filename = self.mktemp()
 133
 134                 f = open(filename, "w")
 135                 f.write("%s\n" % PACKAGE_FORMAT)
 136                 f.close()
 137
 138                 return filename
 139
 140         def create_filelist(self, datafile):
 141                 filelist = self.mktemp()
 142
 143                 f = open(filelist, "w")
 144
 145                 if self.payload_compression == "xz":
 146                         datafile = InnerTarFileXz.open(datafile)
 147                 else:
 148                         datafile = InnerTarFile.open(datafile)
 149
 150                 while True:
 151                         m = datafile.next()
 152                         if not m:
 153                                 break
 154
 155                         log.debug("  %s %-8s %-8s %s %6s %s" % \
 156                                 (tarfile.filemode(m.mode), m.uname, m.gname,
 157                                 "%d-%02d-%02d %02d:%02d:%02d" % time.localtime(m.mtime)[:6],
 158                                 util.format_size(m.size), m.name))
 159
 160                         f.write("%(name)-40s %(type)1s %(size)-10d %(uname)-10s %(gname)-10s %(mode)-6d %(mtime)-12d" \
 161                                 % m.get_info(tarfile.ENCODING, "strict"))
 162
 163                         # Calculate SHA512 hash of regular files.
 164                         if m.isreg():
 165                                 mobj = datafile.extractfile(m)
 166                                 h = hashlib.sha512()
 167
 168                                 while True:
 169                                         buf = mobj.read(BUFFER_SIZE)
 170                                         if not buf:
 171                                                 break
 172                                         h.update(buf)
 173
 174                                 mobj.close()
 175                                 f.write(" %s" % h.hexdigest())
 176                         else:
 177                                 f.write(" -")
 178
 179                         caps = m.pax_headers.get("PAKFIRE.capabilities", None)
 180                         if caps:
 181                                 f.write(" %s" % caps)
 182                         else:
 183                                 f.write(" -")
 184
 185                         f.write("\n")
 186
 187                 log.info("")
 188
 189                 datafile.close()
 190                 f.close()
 191
 192                 return filelist
 193
 194         def run(self):
 195                 raise NotImplementedError
 196
 197         def getsize(self, filename):
 198                 if tarfile.is_tarfile(filename):
 199                         return os.path.getsize(filename)
 200
 201                 size = 0
 202                 f = lzma.LZMAFile(filename)
 203
 204                 while True:
 205                         buf = f.read(BUFFER_SIZE)
 206                         if not buf:
 207                                 break
 208
 209                         size += len(buf)
 210                 f.close()
 211
 212                 return size
 213
 214
 215 class BinaryPackager(Packager):
 216         payload_compression = "xz"
 217
 218         def __init__(self, pakfire, pkg, builder, buildroot):
 219                 Packager.__init__(self, pakfire, pkg)
 220
 221                 self.builder = builder
 222                 self.buildroot = buildroot
 223
 224         def create_metafile(self, datafile):
 225                 info = collections.defaultdict(lambda: "")
 226
 227                 # Extract datafile in temporary directory and scan for dependencies.
 228                 tmpdir = self.mktemp(directory=True)
 229
 230                 if self.payload_compression == "xz":
 231                         tarfile = InnerTarFileXz.open(datafile)
 232                 else:
 233                         tarfile = InnerTarFile.open(datafile)
 234
 235                 tarfile.extractall(path=tmpdir)
 236                 tarfile.close()
 237
 238                 # Run the dependency tracker.
 239                 self.pkg.track_dependencies(self.builder, tmpdir)
 240
 241                 # Generic package information including Pakfire information.
 242                 info.update({
 243                         "pakfire_version" : PAKFIRE_VERSION,
 244                         "uuid"            : self.pkg.uuid,
 245                         "type"            : "binary",
 246                 })
 247
 248                 # Include distribution information.
 249                 info.update(self.pakfire.distro.info)
 250                 info.update(self.pkg.info)
 251
 252                 # Update package information for string formatting.
 253                 info.update({
 254                         "groups"      : " ".join(self.pkg.groups),
 255                         "prerequires" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
 256                                 for d in self.pkg.prerequires]),
 257                         "requires"    : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
 258                                 for d in self.pkg.requires]),
 259                         "provides"    : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
 260                                 for d in self.pkg.provides]),
 261                         "conflicts"   : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
 262                                 for d in self.pkg.conflicts]),
 263                         "obsoletes"   : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
 264                                 for d in self.pkg.obsoletes]),
 265                 })
 266
 267                 # Format description.
 268                 description = [PACKAGE_INFO_DESCRIPTION_LINE % l \
 269                         for l in util.text_wrap(self.pkg.description, length=80)]
 270                 info["description"] = "\n".join(description)
 271
 272                 # Build information.
 273                 info.update({
 274                         # Package it built right now.
 275                         "build_time" : int(time.time()),
 276                         "build_id"   : uuid.uuid4(),
 277                 })
 278
 279                 # Installed size (equals size of the uncompressed tarball).
 280                 info.update({
 281                         "inst_size" : self.getsize(datafile),
 282                 })
 283
 284                 metafile = self.mktemp()
 285
 286                 f = open(metafile, "w")
 287                 f.write(PACKAGE_INFO % info)
 288                 f.close()
 289
 290                 return metafile
 291
 292         def create_datafile(self):
 293                 includes = []
 294                 excludes = []
 295
 296                 # List of all patterns, which grows.
 297                 patterns = self.pkg.files
 298
 299                 for pattern in patterns:
 300                         # Check if we are running in include or exclude mode.
 301                         if pattern.startswith("!"):
 302                                 files = excludes
 303
 304                                 # Strip the ! character.
 305                                 pattern = pattern[1:]
 306                         else:
 307                                 files = includes
 308
 309                         # Expand file to point to chroot.
 310                         if pattern.startswith("/"):
 311                                 pattern = pattern[1:]
 312                         pattern = os.path.join(self.buildroot, pattern)
 313
 314                         # Recognize the type of the pattern. Patterns could be a glob
 315                         # pattern that is expanded here or just a directory which will
 316                         # be included recursively.
 317                         if "*" in pattern or "?" in pattern or ("[" in pattern and "]" in pattern):
 318                                 _patterns = glob.glob(pattern)
 319                         else:
 320                                 _patterns = [pattern,]
 321
 322                         for pattern in _patterns:
 323                                 # Try to stat the pattern. If that is not successful, we cannot go on.
 324                                 try:
 325                                         os.lstat(pattern)
 326                                 except OSError:
 327                                         continue
 328
 329                                 # Add directories recursively but skip those symlinks
 330                                 # that point to a directory.
 331                                 if os.path.isdir(pattern) and not os.path.islink(pattern):
 332                                         # Add directory itself.
 333                                         files.append(pattern)
 334
 335                                         for dir, subdirs, _files in os.walk(pattern):
 336                                                 for subdir in subdirs:
 337                                                         if subdir in ORPHAN_DIRECTORIES:
 338                                                                 continue
 339
 340                                                         subdir = os.path.join(dir, subdir)
 341                                                         files.append(subdir)
 342
 343                                                 for file in _files:
 344                                                         file = os.path.join(dir, file)
 345                                                         files.append(file)
 346
 347                                 # All other files are just added.
 348                                 else:
 349                                         files.append(pattern)
 350
 351                 # ...
 352                 orphan_directories = [os.path.join(self.buildroot, d) for d in ORPHAN_DIRECTORIES]
 353
 354                 files = []
 355                 for file in includes:
 356                         # Skip if file is already in the file set or
 357                         # marked to be excluded from this archive.
 358                         if file in excludes or file in files:
 359                                 continue
 360
 361                         # Skip orphan directories.
 362                         if file in orphan_directories and not os.listdir(file):
 363                                 log.debug("Found an orphaned directory: %s" % file)
 364                                 continue
 365
 366                         files.append(file)
 367
 368                         while True:
 369                                 file = os.path.dirname(file)
 370
 371                                 if file == self.buildroot:
 372                                         break
 373
 374                                 if not file in files:
 375                                         files.append(file)
 376
 377                 files.sort()
 378
 379                 # Load progressbar.
 380                 message = "%-10s : %s" % (_("Packaging"), self.pkg.friendly_name)
 381                 pb = util.make_progress(message, len(files), eta=False)
 382
 383                 datafile = self.mktemp()
 384                 if self.payload_compression == "xz":
 385                         tar = InnerTarFileXz.open(datafile, mode="w")
 386                 else:
 387                         tar = InnerTarFile.open(datafile, mode="w")
 388
 389                 # All files in the tarball are relative to this directory.
 390                 basedir = self.buildroot
 391
 392                 i = 0
 393                 for file in files:
 394                         if pb:
 395                                 i += 1
 396                                 pb.update(i)
 397
 398                         # Never package /.
 399                         if os.path.normpath(file) == os.path.normpath(basedir):
 400                                 continue
 401
 402                         # Name of the file in the archive.
 403                         arcname = "/%s" % os.path.relpath(file, basedir)
 404
 405                         # Add file to tarball.
 406                         tar.add(file, arcname=arcname, recursive=False)
 407
 408                 # Remove all packaged files.
 409                 for file in reversed(files):
 410                         # It's okay if we cannot remove directories,
 411                         # when they are not empty.
 412                         if os.path.isdir(file):
 413                                 try:
 414                                         os.rmdir(file)
 415                                 except OSError:
 416                                         continue
 417                         else:
 418                                 try:
 419                                         os.unlink(file)
 420                                 except OSError:
 421                                         pass
 422
 423                         while True:
 424                                 file = os.path.dirname(file)
 425
 426                                 if not file.startswith(basedir):
 427                                         break
 428
 429                                 try:
 430                                         os.rmdir(file)
 431                                 except OSError:
 432                                         break
 433
 434                 # Close the tarfile.
 435                 tar.close()
 436
 437                 # Finish progressbar.
 438                 if pb:
 439                         pb.finish()
 440
 441                 return datafile
 442
 443         def create_scriptlets(self):
 444                 scriptlets = []
 445
 446                 # Collect all prerequires for the scriptlets.
 447                 prerequires = []
 448
 449                 for scriptlet_name in SCRIPTS:
 450                         scriptlet = self.pkg.get_scriptlet(scriptlet_name)
 451
 452                         if not scriptlet:
 453                                 continue
 454
 455                         # Write script to a file.
 456                         scriptlet_file = self.mktemp()
 457
 458                         if scriptlet["lang"] == "bin":
 459                                 path = lang["path"]
 460                                 try:
 461                                         f = open(path, "b")
 462                                 except OSError:
 463                                         raise Exception, "Cannot open script file: %s" % lang["path"]
 464
 465                                 s = open(scriptlet_file, "wb")
 466
 467                                 while True:
 468                                         buf = f.read(BUFFER_SIZE)
 469                                         if not buf:
 470                                                 break
 471
 472                                         s.write(buf)
 473
 474                                 f.close()
 475                                 s.close()
 476
 477                         elif scriptlet["lang"] == "shell":
 478                                 s = open(scriptlet_file, "w")
 479
 480                                 # Write shell script to file.
 481                                 s.write("#!/bin/sh -e\n\n")
 482                                 s.write(scriptlet["scriptlet"])
 483                                 s.write("\n\nexit 0\n")
 484
 485                                 s.close()
 486
 487                                 if scriptlet_name in SCRIPTS_PREREQUIRES:
 488                                         # Shell scripts require a shell to be executed.
 489                                         prerequires.append("/bin/sh")
 490
 491                                         prerequires += self.builder.find_prerequires(scriptlet_file)
 492
 493                         else:
 494                                 raise Exception, "Unknown scriptlet language: %s" % scriptlet["lang"]
 495
 496                         scriptlets.append((scriptlet_name, scriptlet_file))
 497
 498                 # Cleanup prerequires.
 499                 self.pkg.update_prerequires(prerequires)
 500
 501                 return scriptlets
 502
 503         def create_configs(self, datafile):
 504                 if self.payload_compression == "xz":
 505                         datafile = InnerTarFileXz.open(datafile)
 506                 else:
 507                         datafile = InnerTarFile.open(datafile)
 508
 509                 members = datafile.getmembers()
 510
 511                 configfiles = []
 512                 configdirs  = []
 513
 514                 # Find all directories in the config file list.
 515                 for file in self.pkg.configfiles:
 516                         if file.startswith("/"):
 517                                 file = file[1:]
 518
 519                         for member in members:
 520                                 if member.name == file and member.isdir():
 521                                         configdirs.append(file)
 522
 523                 for configdir in configdirs:
 524                         for member in members:
 525                                 if not member.isdir() and member.name.startswith(configdir):
 526                                         configfiles.append(member.name)
 527
 528                 for pattern in self.pkg.configfiles:
 529                         if pattern.startswith("/"):
 530                                 pattern = pattern[1:]
 531
 532                         for member in members:
 533                                 if not fnmatch.fnmatch(member.name, pattern):
 534                                         continue
 535
 536                                 if member.name in configfiles:
 537                                         continue
 538
 539                                 configfiles.append(member.name)
 540
 541                 # Sort list alphabetically.
 542                 configfiles.sort()
 543
 544                 configsfile = self.mktemp()
 545
 546                 f = open(configsfile, "w")
 547                 for file in configfiles:
 548                         f.write("%s\n" % file)
 549                 f.close()
 550
 551                 return configsfile
 552
 553         def run(self, resultdir):
 554                 # Add all files to this package.
 555                 datafile = self.create_datafile()
 556
 557                 # Get filelist from datafile.
 558                 filelist = self.create_filelist(datafile)
 559                 configs  = self.create_configs(datafile)
 560
 561                 # Create script files.
 562                 scriptlets = self.create_scriptlets()
 563
 564                 metafile = self.create_metafile(datafile)
 565
 566                 # Add files to the tar archive in correct order.
 567                 self.add(metafile, "info")
 568                 self.add(filelist, "filelist")
 569                 self.add(configs,  "configs")
 570                 self.add(datafile, "data.img")
 571
 572                 for scriptlet_name, scriptlet_file in scriptlets:
 573                         self.add(scriptlet_file, "scriptlets/%s" % scriptlet_name)
 574
 575                 # Build the final package.
 576                 tempfile = self.mktemp()
 577                 self.save(tempfile)
 578
 579                 # Add architecture information to path.
 580                 resultdir = "%s/%s" % (resultdir, self.pkg.arch)
 581
 582                 if not os.path.exists(resultdir):
 583                         os.makedirs(resultdir)
 584
 585                 resultfile = os.path.join(resultdir, self.pkg.package_filename)
 586                 log.info("Saving package to %s" % resultfile)
 587                 try:
 588                         os.link(tempfile, resultfile)
 589                 except OSError:
 590                         shutil.copy2(tempfile, resultfile)
 591
 592                 return BinaryPackage(self.pakfire, self.pakfire.repos.dummy, resultfile)
 593
 594
 595 class SourcePackager(Packager):
 596         payload_compression = None
 597
 598         def create_metafile(self, datafile):
 599                 info = collections.defaultdict(lambda: "")
 600
 601                 # Generic package information including Pakfire information.
 602                 info.update({
 603                         "pakfire_version" : PAKFIRE_VERSION,
 604                         "type"            : "source",
 605                 })
 606
 607                 # Include distribution information.
 608                 info.update(self.pakfire.distro.info)
 609                 info.update(self.pkg.info)
 610
 611                 # Size is the size of the (uncompressed) datafile.
 612                 info["inst_size"] = self.getsize(datafile)
 613
 614                 # Update package information for string formatting.
 615                 requires = [PACKAGE_INFO_DEPENDENCY_LINE % r for r in self.pkg.requires]
 616                 info.update({
 617                         "groups"   : " ".join(self.pkg.groups),
 618                         "requires" : "\n".join(requires),
 619                 })
 620
 621                 # Format description.
 622                 description = [PACKAGE_INFO_DESCRIPTION_LINE % l \
 623                         for l in util.text_wrap(self.pkg.description, length=80)]
 624                 info["description"] = "\n".join(description)
 625
 626                 # Build information.
 627                 info.update({
 628                         # Package it built right now.
 629                         "build_time" : int(time.time()),
 630                         "build_id"   : uuid.uuid4(),
 631                 })
 632
 633                 # Arches equals supported arches.
 634                 info["arch"] = self.pkg.supported_arches
 635
 636                 # Set UUID
 637                 # XXX replace this by the payload hash
 638                 info.update({
 639                         "uuid"       : uuid.uuid4(),
 640                 })
 641
 642                 metafile = self.mktemp()
 643
 644                 f = open(metafile, "w")
 645                 f.write(PACKAGE_INFO % info)
 646                 f.close()
 647
 648                 return metafile
 649
 650         def create_datafile(self):
 651                 # Create a list of all files that have to be put into the
 652                 # package.
 653                 files = []
 654
 655                 # Download all files that go into the package.
 656                 for file in self.pkg.download():
 657                         assert os.path.getsize(file), "Don't package empty files"
 658                         files.append(("files/%s" % os.path.basename(file), file))
 659
 660                 # Add all files in the package directory.
 661                 for file in self.pkg.files:
 662                         files.append((os.path.relpath(file, self.pkg.path), file))
 663
 664                 # Add files in alphabetical order.
 665                 files.sort()
 666
 667                 # Load progressbar.
 668                 message = "%-10s : %s" % (_("Packaging"), self.pkg.friendly_name)
 669                 pb = util.make_progress(message, len(files), eta=False)
 670
 671                 filename = self.mktemp()
 672                 if self.payload_compression == "xz":
 673                         datafile = InnerTarFileXz.open(filename, mode="w")
 674                 else:
 675                         datafile = InnerTarFile.open(filename, mode="w")
 676
 677                 i = 0
 678                 for arcname, file in files:
 679                         if pb:
 680                                 i += 1
 681                                 pb.update(i)
 682
 683                         datafile.add(file, arcname)
 684                 datafile.close()
 685
 686                 if pb:
 687                         pb.finish()
 688
 689                 return filename
 690
 691         def run(self, resultdir):
 692                 # Create resultdir if it does not exist yet.
 693                 if not os.path.exists(resultdir):
 694                         os.makedirs(resultdir)
 695
 696                 log.info(_("Building source package %s:") % self.pkg.package_filename)
 697
 698                 # The filename where this source package is saved at.
 699                 target_filename = os.path.join(resultdir, self.pkg.package_filename)
 700
 701                 # Add datafile to package.
 702                 datafile = self.create_datafile()
 703
 704                 # Create filelist out of data.
 705                 filelist = self.create_filelist(datafile)
 706
 707                 # Create metadata.
 708                 metafile = self.create_metafile(datafile)
 709
 710                 # Add files to the tar archive in correct order.
 711                 self.add(metafile, "info")
 712                 self.add(filelist, "filelist")
 713                 self.add(datafile, "data.img")
 714
 715                 # Build the final tarball.
 716                 try:
 717                         self.save(target_filename)
 718                 except:
 719                         # Remove the target file when anything went wrong.
 720                         os.unlink(target_filename)
 721                         raise
 722
 723                 return target_filename