pakfire/packages/packager.py

   1 #!/usr/bin/python
   2
   3 import glob
   4 import logging
   5 import lzma
   6 import os
   7 import progressbar
   8 import shutil
   9 import sys
  10 import tarfile
  11 import tempfile
  12 import uuid
  13 import xattr
  14 import zlib
  15
  16 from pakfire.constants import *
  17 from pakfire.i18n import _
  18
  19 class Extractor(object):
  20         def __init__(self, pakfire, pkg):
  21                 self.pakfire = pakfire
  22                 self.pkg = pkg
  23
  24                 self.data = pkg.get_file("data.img")
  25
  26                 self.archive = None
  27                 self._tempfile = None
  28
  29                 if self.pkg.payload_compression:
  30                         self.uncompress_payload()
  31                 else:
  32                         self.archive = tarfile.open(fileobj=self.data)
  33
  34         def cleanup(self):
  35                 # XXX not called by anything
  36                 if self._tempfile:
  37                         os.unlink(self._tempfile)
  38
  39         def uncompress_payload(self):
  40                 # XXX this function uncompresses the data.img file
  41                 # and saves the bare tarball to /tmp which takes a lot
  42                 # of space.
  43
  44                 # Create a temporary file to save the content in
  45                 f, self._tempfile = tempfile.mkstemp()
  46                 f = open(self._tempfile, "w")
  47
  48                 if self.pkg.payload_compression == "xz":
  49                         decompressor = lzma.LZMADecompressor()
  50
  51                 elif self.pkg.payload_compression == "zlib":
  52                         decompressor = zlib.decompressobj()
  53
  54                 buf = self.data.read(BUFFER_SIZE)
  55                 while buf:
  56                         f.write(decompressor.decompress(buf))
  57
  58                         buf = self.data.read(BUFFER_SIZE)
  59
  60                 f.write(decompressor.flush())
  61                 f.close()
  62
  63                 self.archive = tarfile.open(self._tempfile)
  64
  65         @property
  66         def files(self):
  67                 return self.archive.getnames()
  68
  69         def extractall(self, path="/", callback=None):
  70                 pbar = self._make_progressbar()
  71
  72                 if pbar:
  73                         pbar.start()
  74                 else:
  75                         print "  %s %-20s" % (_("Extracting"), self.pkg.name)
  76
  77                 i = 0
  78                 for name in self.files:
  79                         i += 1
  80                         self.extract(name, path, callback=callback)
  81
  82                         if pbar:
  83                                 pbar.update(i)
  84
  85                 if pbar:
  86                         pbar.finish()
  87                         #sys.stdout.write("\n")
  88
  89         def extract(self, filename, path="/", callback=None):
  90                 member = self.archive.getmember(filename)
  91                 target = os.path.join(path, filename)
  92
  93                 # If the member is a directory and if it already exists, we
  94                 # don't need to create it again.
  95                 if member.isdir() and os.path.exists(target):
  96                         return
  97
  98                 #if self.pakfire.config.get("debug"):
  99                 #       msg = "Creating file (%s:%03d:%03d) " % \
 100                 #               (tarfile.filemode(member.mode), member.uid, member.gid)
 101                 #       if member.issym():
 102                 #               msg += "/%s -> %s" % (member.name, member.linkname)
 103                 #       elif member.islnk():
 104                 #               msg += "/%s link to /%s" % (member.name, member.linkname)
 105                 #       else:
 106                 #               msg += "/%s" % member.name
 107                 #       logging.debug(msg)
 108
 109                 # Remove file if it has been existant
 110                 if not member.isdir() and os.path.exists(target):
 111                         os.unlink(target)
 112
 113                 self.archive.extract(member, path=path)
 114
 115                 # XXX implement setting of xattrs/acls here
 116
 117                 if callback and not member.isdir():
 118                         callback(member.name, hash1="XXX", size=member.size)
 119
 120         def _make_progressbar(self):
 121                 # Don't display a progressbar if we are running in debug mode.
 122                 if self.pakfire.config.get("debug"):
 123                         return
 124
 125                 if not sys.stdout.isatty():
 126                         return
 127
 128                 widgets = [
 129                         "  ",
 130                         "%s %-20s" % (_("Extracting:"), self.pkg.name),
 131                         " ",
 132                         progressbar.Bar(left="[", right="]"),
 133                         "  ",
 134 #                       progressbar.Percentage(),
 135 #                       "  ",
 136                         progressbar.ETA(),
 137                         "  ",
 138                 ]
 139
 140                 # maxval must be > 0 and so we assume that
 141                 # empty packages have at least one file.
 142                 maxval = len(self.files) or 1
 143
 144                 return progressbar.ProgressBar(
 145                         widgets=widgets,
 146                         maxval=maxval,
 147                         term_width=80,
 148                 )
 149
 150
 151 class InnerTarFile(tarfile.TarFile):
 152         def __init__(self, *args, **kwargs):
 153                 # Force the pax format
 154                 kwargs["format"] = tarfile.PAX_FORMAT
 155
 156                 if kwargs.has_key("env"):
 157                         self.env = kwargs.pop("env")
 158
 159                 tarfile.TarFile.__init__(self, *args, **kwargs)
 160
 161         def __filter_xattrs(self, tarinfo):
 162                 logging.debug("Adding file: %s" % tarinfo.name)
 163
 164                 filename = self.env.chrootPath(self.env.buildroot, tarinfo.name)
 165
 166                 # xattrs do only exists for regular files. If we don't have one,
 167                 # simply skip.
 168                 if os.path.isfile(filename):
 169                         for attr, value in xattr.get_all(filename):
 170                                 tarinfo.pax_headers[attr] = value
 171
 172                                 logging.debug("  xattr: %s=%s" % (attr, value))
 173
 174                 return tarinfo
 175
 176         def add(self, *args, **kwargs):
 177                 # Add filter for xattrs if no other filter is set.
 178                 if not kwargs.has_key("filter") and len(args) < 5:
 179                         kwargs["filter"] = self.__filter_xattrs
 180
 181                 tarfile.TarFile.add(self, *args, **kwargs)
 182
 183
 184 # XXX this is totally ugly and needs to be done right!
 185
 186 class Packager(object):
 187         ARCHIVE_FILES = ("info", "filelist", "signature", "data.img")
 188
 189         def __init__(self, pakfire, pkg, env):
 190                 self.pakfire = pakfire
 191                 self.pkg = pkg
 192                 self.env = env
 193
 194                 self.tarball = None
 195
 196                 # Store meta information
 197                 self.info = {
 198                         "package_format" : PACKAGE_FORMAT,
 199                         "package_uuid" : uuid.uuid4(),
 200                         "payload_comp" : None,
 201                 }
 202                 self.info.update(self.pkg.info)
 203                 self.info.update(self.pakfire.distro.info)
 204                 self.info.update(self.env.info)
 205
 206                 ### Create temporary files
 207                 # Create temp directory to where we extract all files again and
 208                 # gather some information about them like requirements and provides.
 209                 self.tempdir = self.env.chrootPath("tmp", "%s_data" % self.pkg.friendly_name)
 210                 if not os.path.exists(self.tempdir):
 211                         os.makedirs(self.tempdir)
 212
 213                 # Create files that have the archive data
 214                 self.archive_files = {}
 215                 for i in self.ARCHIVE_FILES:
 216                         self.archive_files[i] = \
 217                                 self.env.chrootPath("tmp", "%s_%s" % (self.pkg.friendly_name, i))
 218
 219         def __call__(self):
 220                 logging.debug("Packaging %s" % self.pkg.friendly_name)
 221
 222                 # Create the tarball and add all data to it.
 223                 self.create_tarball()
 224
 225                 chroot_tempdir = self.tempdir[len(self.env.chrootPath()):]
 226                 self.info.update({
 227                         "requires" : self.env.do("/usr/lib/buildsystem-tools/dependency-tracker requires %s" % chroot_tempdir,
 228                                 returnOutput=True, env=self.pkg.env).strip(),
 229                         "provides" : self.env.do("/usr/lib/buildsystem-tools/dependency-tracker provides %s" % chroot_tempdir,
 230                                 returnOutput=True, env=self.pkg.env).strip(),
 231                 })
 232
 233                 self.create_info()
 234                 self.create_signature()
 235
 236                 # Create the outer tarball.
 237                 resultdir = os.path.join(self.env.chrootPath("result", self.pkg.arch))
 238                 if not os.path.exists(resultdir):
 239                         os.makedirs(resultdir)
 240
 241                 filename = os.path.join(resultdir, self.pkg.filename)
 242
 243                 tar = tarfile.TarFile(filename, mode="w", format=tarfile.PAX_FORMAT)
 244
 245                 for i in self.ARCHIVE_FILES:
 246                         tar.add(self.archive_files[i], arcname=i)
 247
 248                 tar.close()
 249
 250         def create_tarball(self, compress="xz"):
 251                 tar = InnerTarFile(self.archive_files["data.img"], mode="w", env=self.env)
 252
 253                 includes = []
 254                 excludes = []
 255
 256                 for pattern in self.pkg.file_patterns:
 257                         # Check if we are running in include or exclude mode.
 258                         if pattern.startswith("!"):
 259                                 files = excludes
 260
 261                                 # Strip the ! charater
 262                                 pattern = pattern[1:]
 263
 264                         else:
 265                                 files = includes
 266
 267                         if pattern.startswith("/"):
 268                                 pattern = pattern[1:]
 269                         pattern = self.env.chrootPath(self.env.buildroot, pattern)
 270
 271                         # Recognize the type of the pattern. Patterns could be a glob
 272                         # pattern that is expanded here or just a directory which will
 273                         # be included recursively.
 274                         if "*" in pattern or "?" in pattern:
 275                                 files += glob.glob(pattern)
 276
 277                         elif os.path.exists(pattern):
 278                                 # Add directories recursively...
 279                                 if os.path.isdir(pattern):
 280                                         for dir, subdirs, _files in os.walk(pattern):
 281                                                 for file in _files:
 282                                                         file = os.path.join(dir, file)
 283                                                         files.append(file)
 284
 285                                 # all other files are just added.
 286                                 else:
 287                                         files.append(pattern)
 288
 289                         else:
 290                                 logging.warning("Unrecognized pattern type: %s" % pattern)
 291
 292                 files = []
 293                 for file in includes:
 294                         # Skip if file is already in the file set or
 295                         # marked to be excluded from this archive.
 296                         if file in excludes or file in files:
 297                                 continue
 298
 299                         files.append(file)
 300
 301                 files.sort()
 302
 303                 filelist = open(self.archive_files["filelist"], mode="w")
 304
 305                 for file_real in files:
 306                         file_tar = file_real[len(self.env.chrootPath(self.env.buildroot)) + 1:]
 307                         file_tmp = os.path.join(self.tempdir, file_tar)
 308
 309                         tar.add(file_real, arcname=file_tar, recursive=False)
 310
 311                         # Record the packaged file to the filelist.
 312                         filelist.write("/%s\n" % file_tar)
 313
 314                         # "Copy" the file to the tmp path for later investigation.
 315                         if os.path.isdir(file_real):
 316                                 file_dir = file_tmp
 317                         else:
 318                                 file_dir = os.path.dirname(file_tmp)
 319
 320                         if not os.path.exists(file_dir):
 321                                 os.makedirs(file_dir)
 322
 323                         if os.path.isfile(file_real):
 324                                 os.link(file_real, file_tmp)
 325
 326                         elif os.path.islink(file_real):
 327                                 # Dead symlinks cannot be copied by shutil.
 328                                 os.symlink(os.readlink(file_real), file_tmp)
 329
 330                         elif os.path.isdir(file_real):
 331                                 if not os.path.exists(file_tmp):
 332                                         os.makedirs(file_tmp)
 333
 334                         else:
 335                                 shutil.copy2(file_real, file_tmp)
 336
 337                         # Unlink the file and remove empty directories.
 338                         if not os.path.isdir(file_real):
 339                                 os.unlink(file_real)
 340
 341                         elif os.path.isdir(file_real) and not os.listdir(file_real):
 342                                 os.rmdir(file_real)
 343
 344                 # Dump all files that are in the archive.
 345                 tar.list()
 346
 347                 # Write all data to disk.
 348                 tar.close()
 349                 filelist.close()
 350
 351                 # compress the tarball here
 352                 if compress:
 353                         # Save algorithm to metadata.
 354                         self.info["payload_comp"] = compress
 355
 356                         logging.debug("Compressing package with %s algorithm." % compress or "no")
 357
 358                         filename = self.archive_files["data.img"]
 359                         i = open(filename)
 360                         os.unlink(filename)
 361
 362                         o = open(filename, "w")
 363
 364                         if compress == "xz":
 365                                 comp = lzma.LZMACompressor()
 366
 367                         elif compress == "zlib":
 368                                 comp = zlib.compressobj(9)
 369
 370                         buf = i.read(BUFFER_SIZE)
 371                         while buf:
 372                                 o.write(comp.compress(buf))
 373
 374                                 buf = i.read(BUFFER_SIZE)
 375
 376                         o.write(comp.flush())
 377
 378                         i.close()
 379                         o.close()
 380
 381         def create_info(self):
 382                 f = open(self.archive_files["info"], "w")
 383                 f.write(BINARY_PACKAGE_META % self.info)
 384                 f.close()
 385
 386         def create_signature(self):
 387                 # Create an empty signature.
 388                 f = open(self.archive_files["signature"], "w")
 389                 f.close()