python/pakfire/packages/packager.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import collections
  23 import fnmatch
  24 import glob
  25 import hashlib
  26 import logging
  27 import lzma
  28 import os
  29 import progressbar
  30 import re
  31 import shutil
  32 import sys
  33 import tarfile
  34 import tempfile
  35 import time
  36 import uuid
  37 import zlib
  38
  39 import pakfire.compress
  40 import pakfire.util as util
  41
  42 from pakfire.constants import *
  43 from pakfire.i18n import _
  44
  45 from file import BinaryPackage, InnerTarFile, SourcePackage
  46
  47 class Packager(object):
  48         def __init__(self, pakfire, pkg):
  49                 self.pakfire = pakfire
  50                 self.pkg = pkg
  51
  52                 self.files = []
  53                 self.tmpfiles = []
  54
  55         def __del__(self):
  56                 for file in self.tmpfiles:
  57                         if not os.path.exists(file):
  58                                 continue
  59
  60                         logging.debug("Removing tmpfile: %s" % file)
  61
  62                         if os.path.isdir(file):
  63                                 util.rm(file)
  64                         else:
  65                                 os.remove(file)
  66
  67         def mktemp(self, directory=False):
  68                 # XXX use real mk(s)temp here
  69                 filename = os.path.join("/", LOCAL_TMP_PATH, util.random_string())
  70
  71                 if directory:
  72                         os.makedirs(filename)
  73
  74                 self.tmpfiles.append(filename)
  75
  76                 return filename
  77
  78         def save(self, filename):
  79                 # Create a new tar archive.
  80                 tar = tarfile.TarFile(filename, mode="w", format=tarfile.PAX_FORMAT)
  81
  82                 # Add package formation information.
  83                 # Must always be the first file in the archive.
  84                 formatfile = self.create_package_format()
  85                 tar.add(formatfile, arcname="pakfire-format")
  86
  87                 # XXX make sure all files belong to the root user
  88
  89                 # Create checksum file.
  90                 chksumsfile = self.mktemp()
  91                 chksums = open(chksumsfile, "w")
  92
  93                 # Add all files to tar file.
  94                 for arcname, filename in self.files:
  95                         tar.add(filename, arcname=arcname)
  96
  97                         # Calculating the hash sum of the added file
  98                         # and store it in the chksums file.
  99                         f = open(filename)
 100                         h = hashlib.sha512()
 101                         while True:
 102                                 buf = f.read(BUFFER_SIZE)
 103                                 if not buf:
 104                                         break
 105
 106                                 h.update(buf)
 107                         f.close()
 108
 109                         chksums.write("%-10s %s\n" % (arcname, h.hexdigest()))
 110
 111                 # Close checksum file and attach it to the end.
 112                 chksums.close()
 113                 tar.add(chksumsfile, "chksums")
 114
 115                 # Close the tar file.
 116                 tar.close()
 117
 118         def add(self, filename, arcname=None):
 119                 if not arcname:
 120                         arcname = os.path.basename(filename)
 121
 122                 logging.debug("Adding %s (as %s) to tarball." % (filename, arcname))
 123                 self.files.append((arcname, filename))
 124
 125         def create_package_format(self):
 126                 filename = self.mktemp()
 127
 128                 f = open(filename, "w")
 129                 f.write("%s\n" % PACKAGE_FORMAT)
 130                 f.close()
 131
 132                 return filename
 133
 134         def create_filelist(self, datafile):
 135                 filelist = self.mktemp()
 136
 137                 f = open(filelist, "w")
 138                 datafile = InnerTarFile(datafile)
 139
 140                 for m in datafile.getmembers():
 141                         logging.debug("  %s %-8s %-8s %s %6s %s" % \
 142                                 (tarfile.filemode(m.mode), m.uname, m.gname,
 143                                 "%d-%02d-%02d %02d:%02d:%02d" % time.localtime(m.mtime)[:6],
 144                                 util.format_size(m.size), m.name))
 145
 146                         f.write("%(name)-40s %(type)1s %(size)-10d %(uname)-10s %(gname)-10s %(mode)-6d %(mtime)-12d" \
 147                                 % m.get_info(tarfile.ENCODING, "strict"))
 148
 149                         # Calculate SHA512 hash of regular files.
 150                         if m.isreg():
 151                                 mobj = datafile.extractfile(m)
 152                                 h = hashlib.sha512()
 153
 154                                 while True:
 155                                         buf = mobj.read(BUFFER_SIZE)
 156                                         if not buf:
 157                                                 break
 158                                         h.update(buf)
 159
 160                                 mobj.close()
 161                                 f.write(" %s\n" % h.hexdigest())
 162
 163                         # For other files, just finish the line.
 164                         else:
 165                                 f.write(" -\n")
 166
 167                 logging.info("")
 168
 169                 datafile.close()
 170                 f.close()
 171
 172                 return filelist
 173
 174         def run(self):
 175                 raise NotImplementedError
 176
 177
 178 class BinaryPackager(Packager):
 179         def __init__(self, pakfire, pkg, builder, buildroot):
 180                 Packager.__init__(self, pakfire, pkg)
 181
 182                 self.builder = builder
 183                 self.buildroot = buildroot
 184
 185         def create_metafile(self, datafile):
 186                 info = collections.defaultdict(lambda: "")
 187
 188                 # Extract datafile in temporary directory and scan for dependencies.
 189                 tmpdir = self.mktemp(directory=True)
 190
 191                 tarfile = InnerTarFile(datafile)
 192                 tarfile.extractall(path=tmpdir)
 193                 tarfile.close()
 194
 195                 # Run the dependency tracker.
 196                 self.pkg.track_dependencies(self.builder, tmpdir)
 197
 198                 # Generic package information including Pakfire information.
 199                 info.update({
 200                         "pakfire_version" : PAKFIRE_VERSION,
 201                         "uuid"            : uuid.uuid4(),
 202                         "type"            : "binary",
 203                 })
 204
 205                 # Include distribution information.
 206                 info.update(self.pakfire.distro.info)
 207                 info.update(self.pkg.info)
 208
 209                 # Update package information for string formatting.
 210                 info.update({
 211                         "groups"      : " ".join(self.pkg.groups),
 212                         "prerequires" : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
 213                                 for d in self.pkg.prerequires]),
 214                         "requires"    : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
 215                                 for d in self.pkg.requires]),
 216                         "provides"    : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
 217                                 for d in self.pkg.provides]),
 218                         "conflicts"   : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
 219                                 for d in self.pkg.conflicts]),
 220                         "obsoletes"   : "\n".join([PACKAGE_INFO_DEPENDENCY_LINE % d \
 221                                 for d in self.pkg.obsoletes]),
 222                 })
 223
 224                 # Format description.
 225                 description = [PACKAGE_INFO_DESCRIPTION_LINE % l \
 226                         for l in util.text_wrap(self.pkg.description, length=80)]
 227                 info["description"] = "\n".join(description)
 228
 229                 # Build information.
 230                 info.update({
 231                         # Package it built right now.
 232                         "build_time" : int(time.time()),
 233                         "build_id"   : uuid.uuid4(),
 234                 })
 235
 236                 # Installed size (equals size of the uncompressed tarball).
 237                 info.update({
 238                         "inst_size" : os.path.getsize(datafile),
 239                 })
 240
 241                 metafile = self.mktemp()
 242
 243                 f = open(metafile, "w")
 244                 f.write(PACKAGE_INFO % info)
 245                 f.close()
 246
 247                 return metafile
 248
 249         def create_datafile(self):
 250                 includes = []
 251                 excludes = []
 252
 253                 # List of all patterns, which grows.
 254                 patterns = self.pkg.files
 255
 256                 for pattern in patterns:
 257                         # Check if we are running in include or exclude mode.
 258                         if pattern.startswith("!"):
 259                                 files = excludes
 260
 261                                 # Strip the ! character.
 262                                 pattern = pattern[1:]
 263                         else:
 264                                 files = includes
 265
 266                         # Expand file to point to chroot.
 267                         if pattern.startswith("/"):
 268                                 pattern = pattern[1:]
 269                         pattern = os.path.join(self.buildroot, pattern)
 270
 271                         # Recognize the type of the pattern. Patterns could be a glob
 272                         # pattern that is expanded here or just a directory which will
 273                         # be included recursively.
 274                         if "*" in pattern or "?" in pattern or ("[" in pattern and "]" in pattern):
 275                                 _patterns = glob.glob(pattern)
 276                         else:
 277                                 _patterns = [pattern,]
 278
 279                         for pattern in _patterns:
 280                                 if not os.path.exists(pattern):
 281                                         continue
 282
 283                                 # Add directories recursively...
 284                                 if os.path.isdir(pattern):
 285                                         # Add directory itself.
 286                                         files.append(pattern)
 287
 288                                         for dir, subdirs, _files in os.walk(pattern):
 289                                                 for subdir in subdirs:
 290                                                         if subdir in ORPHAN_DIRECTORIES:
 291                                                                 continue
 292
 293                                                         subdir = os.path.join(dir, subdir)
 294                                                         files.append(subdir)
 295
 296                                                 for file in _files:
 297                                                         file = os.path.join(dir, file)
 298                                                         files.append(file)
 299
 300                                 # all other files are just added.
 301                                 else:
 302                                         files.append(pattern)
 303
 304                 # ...
 305                 orphan_directories = [os.path.join(self.buildroot, d) for d in ORPHAN_DIRECTORIES]
 306
 307                 files = []
 308                 for file in includes:
 309                         # Skip if file is already in the file set or
 310                         # marked to be excluded from this archive.
 311                         if file in excludes or file in files:
 312                                 continue
 313
 314                         # Skip orphan directories.
 315                         if file in orphan_directories and not os.listdir(file):
 316                                 logging.debug("Found an orphaned directory: %s" % file)
 317                                 continue
 318
 319                         files.append(file)
 320
 321                         while True:
 322                                 file = os.path.dirname(file)
 323
 324                                 if file == self.buildroot:
 325                                         break
 326
 327                                 if not file in files:
 328                                         files.append(file)
 329
 330                 files.sort()
 331
 332                 # Load progressbar.
 333                 message = "%-10s : %s" % (_("Packaging"), self.pkg.friendly_name)
 334                 pb = util.make_progress(message, len(files), eta=False)
 335
 336                 datafile = self.mktemp()
 337                 tar = InnerTarFile(datafile, mode="w")
 338
 339                 # All files in the tarball are relative to this directory.
 340                 basedir = self.buildroot
 341
 342                 i = 0
 343                 for file in files:
 344                         if pb:
 345                                 i += 1
 346                                 pb.update(i)
 347
 348                         # Never package /.
 349                         if os.path.normpath(file) == os.path.normpath(basedir):
 350                                 continue
 351
 352                         # Name of the file in the archive.
 353                         arcname = "/%s" % os.path.relpath(file, basedir)
 354
 355                         # Add file to tarball.
 356                         tar.add(file, arcname=arcname, recursive=False)
 357
 358                 # Remove all packaged files.
 359                 for file in reversed(files):
 360                         # It's okay if we cannot remove directories,
 361                         # when they are not empty.
 362                         if os.path.isdir(file):
 363                                 try:
 364                                         os.rmdir(file)
 365                                 except OSError:
 366                                         continue
 367                         else:
 368                                 try:
 369                                         os.unlink(file)
 370                                 except OSError:
 371                                         pass
 372
 373                         while True:
 374                                 file = os.path.dirname(file)
 375
 376                                 if not file.startswith(basedir):
 377                                         break
 378
 379                                 try:
 380                                         os.rmdir(file)
 381                                 except OSError:
 382                                         break
 383
 384                 # Close the tarfile.
 385                 tar.close()
 386
 387                 # Finish progressbar.
 388                 if pb:
 389                         pb.finish()
 390
 391                 return datafile
 392
 393         def create_scriptlets(self):
 394                 scriptlets = []
 395
 396                 for scriptlet_name in SCRIPTS:
 397                         scriptlet = self.pkg.get_scriptlet(scriptlet_name)
 398
 399                         if not scriptlet:
 400                                 continue
 401
 402                         # Write script to a file.
 403                         scriptlet_file = self.mktemp()
 404
 405                         if scriptlet["lang"] == "bin":
 406                                 path = lang["path"]
 407                                 try:
 408                                         f = open(path, "b")
 409                                 except OSError:
 410                                         raise Exception, "Cannot open script file: %s" % lang["path"]
 411
 412                                 s = open(scriptlet_file, "wb")
 413
 414                                 while True:
 415                                         buf = f.read(BUFFER_SIZE)
 416                                         if not buf:
 417                                                 break
 418
 419                                         s.write(buf)
 420
 421                                 f.close()
 422                                 s.close()
 423
 424                         elif scriptlet["lang"] == "shell":
 425                                 s = open(scriptlet_file, "w")
 426
 427                                 # Write shell script to file.
 428                                 s.write("#!/bin/sh -e\n\n")
 429                                 s.write(scriptlet["scriptlet"])
 430                                 s.write("\n\nexit 0\n")
 431
 432                                 s.close()
 433
 434                         else:
 435                                 raise Exception, "Unknown scriptlet language: %s" % scriptlet["lang"]
 436
 437                         scriptlets.append((scriptlet_name, scriptlet_file))
 438
 439                 # XXX scan for script dependencies
 440
 441                 return scriptlets
 442
 443         def create_configs(self, datafile):
 444                 datafile = InnerTarFile(datafile)
 445
 446                 members = datafile.getmembers()
 447
 448                 configfiles = []
 449                 configdirs  = []
 450
 451                 # Find all directories in the config file list.
 452                 for file in self.pkg.configfiles:
 453                         if file.startswith("/"):
 454                                 file = file[1:]
 455
 456                         for member in members:
 457                                 if member.name == file and member.isdir():
 458                                         configdirs.append(file)
 459
 460                 for configdir in configdirs:
 461                         for member in members:
 462                                 if not member.isdir() and member.name.startswith(configdir):
 463                                         configfiles.append(member.name)
 464
 465                 for pattern in self.pkg.configfiles:
 466                         if pattern.startswith("/"):
 467                                 pattern = pattern[1:]
 468
 469                         for member in members:
 470                                 if not fnmatch.fnmatch(member.name, pattern):
 471                                         continue
 472
 473                                 if member.name in configfiles:
 474                                         continue
 475
 476                                 configfiles.append(member.name)
 477
 478                 # Sort list alphabetically.
 479                 configfiles.sort()
 480
 481                 configsfile = self.mktemp()
 482
 483                 f = open(configsfile, "w")
 484                 for file in configfiles:
 485                         f.write("%s\n" % file)
 486                 f.close()
 487
 488                 return configsfile
 489
 490         def compress_datafile(self, datafile, algo="xz"):
 491                 outputfile = self.mktemp()
 492
 493                 # Compress the datafile with the choosen algorithm.
 494                 pakfire.compress.compress_file(datafile, outputfile, algo=algo,
 495                         progress=True, message=_("Compressing %s") % self.pkg.friendly_name)
 496
 497                 # We do not need the uncompressed output anymore.
 498                 os.unlink(datafile)
 499
 500                 # The outputfile becomes out new datafile.
 501                 return outputfile
 502
 503         def run(self, resultdir):
 504                 # Add all files to this package.
 505                 datafile = self.create_datafile()
 506
 507                 # Get filelist from datafile.
 508                 filelist = self.create_filelist(datafile)
 509                 configs  = self.create_configs(datafile)
 510
 511                 # Create script files.
 512                 scriptlets = self.create_scriptlets()
 513
 514                 metafile = self.create_metafile(datafile)
 515
 516                 # XXX make xz in variable
 517                 datafile = self.compress_datafile(datafile, algo="xz")
 518
 519                 # Add files to the tar archive in correct order.
 520                 self.add(metafile, "info")
 521                 self.add(filelist, "filelist")
 522                 self.add(configs,  "configs")
 523                 self.add(datafile, "data.img")
 524
 525                 for scriptlet_name, scriptlet_file in scriptlets:
 526                         self.add(scriptlet_file, "scriptlets/%s" % scriptlet_name)
 527
 528                 # Build the final package.
 529                 tempfile = self.mktemp()
 530                 self.save(tempfile)
 531
 532                 # Add architecture information to path.
 533                 resultdir = "%s/%s" % (resultdir, self.pkg.arch)
 534
 535                 if not os.path.exists(resultdir):
 536                         os.makedirs(resultdir)
 537
 538                 resultfile = os.path.join(resultdir, self.pkg.package_filename)
 539                 logging.info("Saving package to %s" % resultfile)
 540                 try:
 541                         os.link(tempfile, resultfile)
 542                 except OSError:
 543                         shutil.copy2(tempfile, resultfile)
 544
 545                 return BinaryPackage(self.pakfire, self.pakfire.repos.dummy, resultfile)
 546
 547
 548 class SourcePackager(Packager):
 549         def create_metafile(self, datafile):
 550                 info = collections.defaultdict(lambda: "")
 551
 552                 # Generic package information including Pakfire information.
 553                 info.update({
 554                         "pakfire_version" : PAKFIRE_VERSION,
 555                         "type"            : "source",
 556                 })
 557
 558                 # Include distribution information.
 559                 info.update(self.pakfire.distro.info)
 560                 info.update(self.pkg.info)
 561
 562                 # Size is the size of the (uncompressed) datafile.
 563                 info["inst_size"] = os.path.getsize(datafile)
 564
 565                 # Update package information for string formatting.
 566                 requires = [PACKAGE_INFO_DEPENDENCY_LINE % r for r in self.pkg.requires]
 567                 info.update({
 568                         "groups"   : " ".join(self.pkg.groups),
 569                         "requires" : "\n".join(requires),
 570                 })
 571
 572                 # Format description.
 573                 description = [PACKAGE_INFO_DESCRIPTION_LINE % l \
 574                         for l in util.text_wrap(self.pkg.description, length=80)]
 575                 info["description"] = "\n".join(description)
 576
 577                 # Build information.
 578                 info.update({
 579                         # Package it built right now.
 580                         "build_time" : int(time.time()),
 581                         "build_id"   : uuid.uuid4(),
 582                 })
 583
 584                 # Arches equals supported arches.
 585                 info["arch"] = self.pkg.supported_arches
 586
 587                 # Set UUID
 588                 # XXX replace this by the payload hash
 589                 info.update({
 590                         "uuid"       : uuid.uuid4(),
 591                 })
 592
 593                 metafile = self.mktemp()
 594
 595                 f = open(metafile, "w")
 596                 f.write(PACKAGE_INFO % info)
 597                 f.close()
 598
 599                 return metafile
 600
 601         def create_datafile(self):
 602                 filename = self.mktemp()
 603                 datafile = InnerTarFile(filename, mode="w")
 604
 605                 # Add all downloaded files to the package.
 606                 for file in self.pkg.download():
 607                         datafile.add(file, "files/%s" % os.path.basename(file))
 608
 609                 # Add all files in the package directory.
 610                 for file in sorted(self.pkg.files):
 611                         arcname = os.path.relpath(file, self.pkg.path)
 612                         datafile.add(file, arcname)
 613
 614                 datafile.close()
 615
 616                 return filename
 617
 618         def run(self, resultdirs=[]):
 619                 assert resultdirs
 620
 621                 logging.info(_("Building source package %s:") % self.pkg.package_filename)
 622
 623                 # Add datafile to package.
 624                 datafile = self.create_datafile()
 625
 626                 # Create filelist out of data.
 627                 filelist = self.create_filelist(datafile)
 628
 629                 # Create metadata.
 630                 metafile = self.create_metafile(datafile)
 631
 632                 # Add files to the tar archive in correct order.
 633                 self.add(metafile, "info")
 634                 self.add(filelist, "filelist")
 635                 self.add(datafile, "data.img")
 636
 637                 # Build the final tarball.
 638                 tempfile = self.mktemp()
 639                 self.save(tempfile)
 640
 641                 for resultdir in resultdirs:
 642                         # XXX sometimes, there has been a None in resultdirs
 643                         if not resultdir:
 644                                 continue
 645
 646                         resultdir = "%s/%s" % (resultdir, self.pkg.arch)
 647
 648                         if not os.path.exists(resultdir):
 649                                 os.makedirs(resultdir)
 650
 651                         resultfile = os.path.join(resultdir, self.pkg.package_filename)
 652                         logging.info("Saving package to %s" % resultfile)
 653                         try:
 654                                 os.link(tempfile, resultfile)
 655                         except OSError:
 656                                 shutil.copy2(tempfile, resultfile)
 657
 658                 # Dump package information.
 659                 pkg = SourcePackage(self.pakfire, self.pakfire.repos.dummy, tempfile)
 660                 for line in pkg.dump(long=True).splitlines():
 661                         logging.info(line)
 662                 logging.info("")