python/pakfire/packages/packager.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import collections
  23 import fnmatch
  24 import glob
  25 import hashlib
  26 import logging
  27 import lzma
  28 import os
  29 import progressbar
  30 import re
  31 import shutil
  32 import sys
  33 import tarfile
  34 import tempfile
  35 import time
  36 import uuid
  37 import xattr
  38 import zlib
  39
  40 import pakfire.compress
  41 import pakfire.util as util
  42
  43 from pakfire.constants import *
  44 from pakfire.i18n import _
  45
  46 from file import BinaryPackage, InnerTarFile, SourcePackage
  47
  48 class Packager(object):
  49         def __init__(self, pakfire, pkg):
  50                 self.pakfire = pakfire
  51                 self.pkg = pkg
  52
  53                 self.files = []
  54                 self.tmpfiles = []
  55
  56         def __del__(self):
  57                 for file in self.tmpfiles:
  58                         if not os.path.exists(file):
  59                                 continue
  60
  61                         logging.debug("Removing tmpfile: %s" % file)
  62
  63                         if os.path.isdir(file):
  64                                 util.rm(file)
  65                         else:
  66                                 os.remove(file)
  67
  68         def mktemp(self, directory=False):
  69                 # XXX use real mk(s)temp here
  70                 filename = os.path.join("/", LOCAL_TMP_PATH, util.random_string())
  71
  72                 if directory:
  73                         os.makedirs(filename)
  74
  75                 self.tmpfiles.append(filename)
  76
  77                 return filename
  78
  79         def save(self, filename):
  80                 # Create a new tar archive.
  81                 tar = tarfile.TarFile(filename, mode="w", format=tarfile.PAX_FORMAT)
  82
  83                 # Add package formation information.
  84                 # Must always be the first file in the archive.
  85                 formatfile = self.create_package_format()
  86                 tar.add(formatfile, arcname="pakfire-format")
  87
  88                 # XXX make sure all files belong to the root user
  89
  90                 # Create checksum file.
  91                 chksumsfile = self.mktemp()
  92                 chksums = open(chksumsfile, "w")
  93
  94                 # Add all files to tar file.
  95                 for arcname, filename in self.files:
  96                         tar.add(filename, arcname=arcname)
  97
  98                         # Calculating the hash sum of the added file
  99                         # and store it in the chksums file.
 100                         f = open(filename)
 101                         h = hashlib.sha512()
 102                         while True:
 103                                 buf = f.read(BUFFER_SIZE)
 104                                 if not buf:
 105                                         break
 106
 107                                 h.update(buf)
 108                         f.close()
 109
 110                         chksums.write("%-10s %s\n" % (arcname, h.hexdigest()))
 111
 112                 # Close checksum file and attach it to the end.
 113                 chksums.close()
 114                 tar.add(chksumsfile, "chksums")
 115
 116                 # Close the tar file.
 117                 tar.close()
 118
 119         def add(self, filename, arcname=None):
 120                 if not arcname:
 121                         arcname = os.path.basename(filename)
 122
 123                 logging.debug("Adding %s (as %s) to tarball." % (filename, arcname))
 124                 self.files.append((arcname, filename))
 125
 126         def create_package_format(self):
 127                 filename = self.mktemp()
 128
 129                 f = open(filename, "w")
 130                 f.write("%s\n" % PACKAGE_FORMAT)
 131                 f.close()
 132
 133                 return filename
 134
 135         def create_filelist(self, datafile):
 136                 filelist = self.mktemp()
 137
 138                 f = open(filelist, "w")
 139                 datafile = InnerTarFile(datafile)
 140
 141                 for m in datafile.getmembers():
 142                         logging.debug("  %s %-8s %-8s %s %6s %s" % \
 143                                 (tarfile.filemode(m.mode), m.uname, m.gname,
 144                                 "%d-%02d-%02d %02d:%02d:%02d" % time.localtime(m.mtime)[:6],
 145                                 util.format_size(m.size), m.name))
 146
 147                         f.write("%(name)-40s %(type)1s %(size)-10d %(uname)-10s %(gname)-10s %(mode)-6d %(mtime)-12d" \
 148                                 % m.get_info(tarfile.ENCODING, "strict"))
 149
 150                         # Calculate SHA512 hash of regular files.
 151                         if m.isreg():
 152                                 mobj = datafile.extractfile(m)
 153                                 h = hashlib.sha512()
 154
 155                                 while True:
 156                                         buf = mobj.read(BUFFER_SIZE)
 157                                         if not buf:
 158                                                 break
 159                                         h.update(buf)
 160
 161                                 mobj.close()
 162                                 f.write(" %s\n" % h.hexdigest())
 163
 164                         # For other files, just finish the line.
 165                         else:
 166                                 f.write(" -\n")
 167
 168                 logging.info("")
 169
 170                 datafile.close()
 171                 f.close()
 172
 173                 return filelist
 174
 175         def run(self):
 176                 raise NotImplementedError
 177
 178
 179 class BinaryPackager(Packager):
 180         def __init__(self, pakfire, pkg, builder, buildroot):
 181                 Packager.__init__(self, pakfire, pkg)
 182
 183                 self.builder = builder
 184                 self.buildroot = buildroot
 185
 186         def create_metafile(self, datafile):
 187                 info = collections.defaultdict(lambda: "")
 188
 189                 # Extract datafile in temporary directory and scan for dependencies.
 190                 tmpdir = self.mktemp(directory=True)
 191
 192                 tarfile = InnerTarFile(datafile)
 193                 tarfile.extractall(path=tmpdir)
 194                 tarfile.close()
 195
 196                 # Run the dependency tracker.
 197                 self.pkg.track_dependencies(self.builder, tmpdir)
 198
 199                 # Generic package information including Pakfire information.
 200                 info.update({
 201                         "pakfire_version" : PAKFIRE_VERSION,
 202                         "uuid"            : uuid.uuid4(),
 203                 })
 204
 205                 # Include distribution information.
 206                 info.update(self.pakfire.distro.info)
 207                 info.update(self.pkg.info)
 208
 209                 # Update package information for string formatting.
 210                 info.update({
 211                         "groups"      : " ".join(self.pkg.groups),
 212                         "prerequires" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
 213                                 for d in self.pkg.prerequires]),
 214                         "requires"    : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
 215                                 for d in self.pkg.requires]),
 216                         "provides"    : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
 217                                 for d in self.pkg.provides]),
 218                         "conflicts"   : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
 219                                 for d in self.pkg.conflicts]),
 220                         "obsoletes"   : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
 221                                 for d in self.pkg.obsoletes]),
 222                 })
 223
 224                 # Format description.
 225                 description = [PACKAGE_INFO_DESCRIPTION_LINE % l \
 226                         for l in util.text_wrap(self.pkg.description, length=80)]
 227                 info["description"] = "\n".join(description)
 228
 229                 # Build information.
 230                 info.update({
 231                         # Package it built right now.
 232                         "build_time" : int(time.time()),
 233                         "build_id"   : uuid.uuid4(),
 234                 })
 235
 236                 # Installed size (equals size of the uncompressed tarball).
 237                 info.update({
 238                         "inst_size" : os.path.getsize(datafile),
 239                 })
 240
 241                 metafile = self.mktemp()
 242
 243                 f = open(metafile, "w")
 244                 f.write(PACKAGE_INFO % info)
 245                 f.close()
 246
 247                 return metafile
 248
 249         def create_datafile(self):
 250                 includes = []
 251                 excludes = []
 252
 253                 # List of all patterns, which grows.
 254                 patterns = self.pkg.files
 255
 256                 for pattern in patterns:
 257                         # Check if we are running in include or exclude mode.
 258                         if pattern.startswith("!"):
 259                                 files = excludes
 260
 261                                 # Strip the ! character.
 262                                 pattern = pattern[1:]
 263                         else:
 264                                 files = includes
 265
 266                         # Expand file to point to chroot.
 267                         if pattern.startswith("/"):
 268                                 pattern = pattern[1:]
 269                         pattern = os.path.join(self.buildroot, pattern)
 270
 271                         # Recognize the type of the pattern. Patterns could be a glob
 272                         # pattern that is expanded here or just a directory which will
 273                         # be included recursively.
 274                         if "*" in pattern or "?" in pattern or ("[" in pattern and "]" in pattern):
 275                                 _patterns = glob.glob(pattern)
 276                         else:
 277                                 _patterns = [pattern,]
 278
 279                         for pattern in _patterns:
 280                                 if not os.path.exists(pattern):
 281                                         continue
 282
 283                                 # Add directories recursively...
 284                                 if os.path.isdir(pattern):
 285                                         # Add directory itself.
 286                                         files.append(pattern)
 287
 288                                         for dir, subdirs, _files in os.walk(pattern):
 289                                                 for subdir in subdirs:
 290                                                         if subdir in ORPHAN_DIRECTORIES:
 291                                                                 continue
 292
 293                                                         subdir = os.path.join(dir, subdir)
 294                                                         files.append(subdir)
 295
 296                                                 for file in _files:
 297                                                         file = os.path.join(dir, file)
 298                                                         files.append(file)
 299
 300                                 # all other files are just added.
 301                                 else:
 302                                         files.append(pattern)
 303
 304                 files = []
 305                 for file in includes:
 306                         # Skip if file is already in the file set or
 307                         # marked to be excluded from this archive.
 308                         if file in excludes or file in files:
 309                                 continue
 310
 311                         files.append(file)
 312                 files.sort()
 313
 314                 # Load progressbar.
 315                 message = "%-10s : %s" % (_("Packaging"), self.pkg.friendly_name)
 316                 pb = util.make_progress(message, len(files), eta=False)
 317
 318                 datafile = self.mktemp()
 319                 tar = InnerTarFile(datafile, mode="w")
 320
 321                 # All files in the tarball are relative to this directory.
 322                 basedir = self.buildroot
 323
 324                 i = 0
 325                 for file in files:
 326                         if pb:
 327                                 i += 1
 328                                 pb.update(i)
 329
 330                         # Never package /.
 331                         if os.path.normpath(file) == os.path.normpath(basedir):
 332                                 continue
 333
 334                         arcname = "/%s" % os.path.relpath(file, basedir)
 335
 336                         # Special handling for directories.
 337                         if os.path.isdir(file):
 338                                 # Empty directories that are in the list of ORPHAN_DIRECTORIES
 339                                 # can be skipped and removed.
 340                                 if arcname in ORPHAN_DIRECTORIES and not os.listdir(file):
 341                                         logging.debug("Found an orphaned directory: %s" % arcname)
 342                                         try:
 343                                                 os.unlink(file)
 344                                         except OSError:
 345                                                 pass
 346
 347                                         continue
 348
 349                         # Add file to tarball.
 350                         tar.add(file, arcname=arcname, recursive=False)
 351
 352                 # Remove all packaged files.
 353                 for file in reversed(files):
 354                         if not os.path.exists(file):
 355                                 continue
 356
 357                         # It's okay if we cannot remove directories,
 358                         # when they are not empty.
 359                         if os.path.isdir(file):
 360                                 try:
 361                                         os.rmdir(file)
 362                                 except OSError:
 363                                         continue
 364                         else:
 365                                 os.unlink(file)
 366
 367                         while True:
 368                                 file = os.path.dirname(file)
 369
 370                                 if not file.startswith(basedir):
 371                                         break
 372
 373                                 try:
 374                                         os.rmdir(file)
 375                                 except OSError:
 376                                         break
 377
 378                 # Close the tarfile.
 379                 tar.close()
 380
 381                 # Finish progressbar.
 382                 if pb:
 383                         pb.finish()
 384
 385                 return datafile
 386
 387         def create_scriptlets(self):
 388                 scriptlets = []
 389
 390                 for scriptlet_name in SCRIPTS:
 391                         scriptlet = self.pkg.get_scriptlet(scriptlet_name)
 392
 393                         if not scriptlet:
 394                                 continue
 395
 396                         # Write script to a file.
 397                         scriptlet_file = self.mktemp()
 398
 399                         if scriptlet["lang"] == "bin":
 400                                 path = lang["path"]
 401                                 try:
 402                                         f = open(path, "b")
 403                                 except OSError:
 404                                         raise Exception, "Cannot open script file: %s" % lang["path"]
 405
 406                                 s = open(scriptlet_file, "wb")
 407
 408                                 while True:
 409                                         buf = f.read(BUFFER_SIZE)
 410                                         if not buf:
 411                                                 break
 412
 413                                         s.write(buf)
 414
 415                                 f.close()
 416                                 s.close()
 417
 418                         elif scriptlet["lang"] == "shell":
 419                                 s = open(scriptlet_file, "w")
 420
 421                                 # Write shell script to file.
 422                                 s.write("#!/bin/sh -e\n\n")
 423                                 s.write(scriptlet["scriptlet"])
 424                                 s.write("\n\nexit 0\n")
 425
 426                                 s.close()
 427
 428                         else:
 429                                 raise Exception, "Unknown scriptlet language: %s" % scriptlet["lang"]
 430
 431                         scriptlets.append((scriptlet_name, scriptlet_file))
 432
 433                 # XXX scan for script dependencies
 434
 435                 return scriptlets
 436
 437         def create_configs(self, datafile):
 438                 datafile = InnerTarFile(datafile)
 439
 440                 members = datafile.getmembers()
 441
 442                 configfiles = []
 443                 configdirs  = []
 444
 445                 # Find all directories in the config file list.
 446                 for file in self.pkg.configfiles:
 447                         if file.startswith("/"):
 448                                 file = file[1:]
 449
 450                         for member in members:
 451                                 if member.name == file and member.isdir():
 452                                         configdirs.append(file)
 453
 454                 for configdir in configdirs:
 455                         for member in members:
 456                                 if not member.isdir() and member.name.startswith(configdir):
 457                                         configfiles.append(member.name)
 458
 459                 for pattern in self.pkg.configfiles:
 460                         if pattern.startswith("/"):
 461                                 pattern = pattern[1:]
 462
 463                         for member in members:
 464                                 if not fnmatch.fnmatch(member.name, pattern):
 465                                         continue
 466
 467                                 if member.name in configfiles:
 468                                         continue
 469
 470                                 configfiles.append(member.name)
 471
 472                 # Sort list alphabetically.
 473                 configfiles.sort()
 474
 475                 configsfile = self.mktemp()
 476
 477                 f = open(configsfile, "w")
 478                 for file in configfiles:
 479                         f.write("%s\n" % file)
 480                 f.close()
 481
 482                 return configsfile
 483
 484         def compress_datafile(self, datafile, algo="xz"):
 485                 outputfile = self.mktemp()
 486
 487                 # Compress the datafile with the choosen algorithm.
 488                 pakfire.compress.compress_file(datafile, outputfile, algo=algo,
 489                         progress=True, message=_("Compressing %s") % self.pkg.friendly_name)
 490
 491                 # We do not need the uncompressed output anymore.
 492                 os.unlink(datafile)
 493
 494                 # The outputfile becomes out new datafile.
 495                 return outputfile
 496
 497         def run(self, resultdir):
 498                 # Add all files to this package.
 499                 datafile = self.create_datafile()
 500
 501                 # Get filelist from datafile.
 502                 filelist = self.create_filelist(datafile)
 503                 configs  = self.create_configs(datafile)
 504
 505                 # Create script files.
 506                 scriptlets = self.create_scriptlets()
 507
 508                 metafile = self.create_metafile(datafile)
 509
 510                 # XXX make xz in variable
 511                 datafile = self.compress_datafile(datafile, algo="xz")
 512
 513                 # Add files to the tar archive in correct order.
 514                 self.add(metafile, "info")
 515                 self.add(filelist, "filelist")
 516                 self.add(configs,  "configs")
 517                 self.add(datafile, "data.img")
 518
 519                 for scriptlet_name, scriptlet_file in scriptlets:
 520                         self.add(scriptlet_file, "scriptlets/%s" % scriptlet_name)
 521
 522                 # Build the final package.
 523                 tempfile = self.mktemp()
 524                 self.save(tempfile)
 525
 526                 # Add architecture information to path.
 527                 resultdir = "%s/%s" % (resultdir, self.pkg.arch)
 528
 529                 if not os.path.exists(resultdir):
 530                         os.makedirs(resultdir)
 531
 532                 resultfile = os.path.join(resultdir, self.pkg.package_filename)
 533                 logging.info("Saving package to %s" % resultfile)
 534                 try:
 535                         os.link(tempfile, resultfile)
 536                 except OSError:
 537                         shutil.copy2(tempfile, resultfile)
 538
 539                 return BinaryPackage(self.pakfire, self.pakfire.repos.dummy, resultfile)
 540
 541
 542 class SourcePackager(Packager):
 543         def create_metafile(self, datafile):
 544                 info = collections.defaultdict(lambda: "")
 545
 546                 # Generic package information including Pakfire information.
 547                 info.update({
 548                         "pakfire_version" : PAKFIRE_VERSION,
 549                 })
 550
 551                 # Include distribution information.
 552                 info.update(self.pakfire.distro.info)
 553                 info.update(self.pkg.info)
 554
 555                 # Update package information for string formatting.
 556                 requires = [PACKAGE_INFO_DEPENDENCY_LINE % r for r in self.pkg.requires]
 557                 info.update({
 558                         "groups"   : " ".join(self.pkg.groups),
 559                         "requires" : "\n".join(requires),
 560                 })
 561
 562                 # Format description.
 563                 description = [PACKAGE_INFO_DESCRIPTION_LINE % l \
 564                         for l in util.text_wrap(self.pkg.description, length=80)]
 565                 info["description"] = "\n".join(description)
 566
 567                 # Build information.
 568                 info.update({
 569                         # Package it built right now.
 570                         "build_time" : int(time.time()),
 571                         "build_id"   : uuid.uuid4(),
 572                 })
 573
 574                 # Set UUID
 575                 # XXX replace this by the payload hash
 576                 info.update({
 577                         "uuid"       : uuid.uuid4(),
 578                 })
 579
 580                 metafile = self.mktemp()
 581
 582                 f = open(metafile, "w")
 583                 f.write(PACKAGE_INFO % info)
 584                 f.close()
 585
 586                 return metafile
 587
 588         def create_datafile(self):
 589                 filename = self.mktemp()
 590                 datafile = InnerTarFile(filename, mode="w")
 591
 592                 # Add all downloaded files to the package.
 593                 for file in self.pkg.download():
 594                         datafile.add(file, "files/%s" % os.path.basename(file))
 595
 596                 # Add all files in the package directory.
 597                 for file in sorted(self.pkg.files):
 598                         arcname = os.path.relpath(file, self.pkg.path)
 599                         datafile.add(file, arcname)
 600
 601                 datafile.close()
 602
 603                 return filename
 604
 605         def run(self, resultdirs=[]):
 606                 assert resultdirs
 607
 608                 logging.info(_("Building source package %s:") % self.pkg.package_filename)
 609
 610                 # Add datafile to package.
 611                 datafile = self.create_datafile()
 612
 613                 # Create filelist out of data.
 614                 filelist = self.create_filelist(datafile)
 615
 616                 # Create metadata.
 617                 metafile = self.create_metafile(datafile)
 618
 619                 # Add files to the tar archive in correct order.
 620                 self.add(metafile, "info")
 621                 self.add(filelist, "filelist")
 622                 self.add(datafile, "data.img")
 623
 624                 # Build the final tarball.
 625                 tempfile = self.mktemp()
 626                 self.save(tempfile)
 627
 628                 for resultdir in resultdirs:
 629                         # XXX sometimes, there has been a None in resultdirs
 630                         if not resultdir:
 631                                 continue
 632
 633                         resultdir = "%s/%s" % (resultdir, self.pkg.arch)
 634
 635                         if not os.path.exists(resultdir):
 636                                 os.makedirs(resultdir)
 637
 638                         resultfile = os.path.join(resultdir, self.pkg.package_filename)
 639                         logging.info("Saving package to %s" % resultfile)
 640                         try:
 641                                 os.link(tempfile, resultfile)
 642                         except OSError:
 643                                 shutil.copy2(tempfile, resultfile)
 644
 645                 # Dump package information.
 646                 pkg = SourcePackage(self.pakfire, self.pakfire.repos.dummy, tempfile)
 647                 for line in pkg.dump(long=True).splitlines():
 648                         logging.info(line)
 649                 logging.info("")