python/pakfire/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import math
  25 import os
  26 import re
  27 import shutil
  28 import socket
  29 import time
  30 import uuid
  31
  32 import base
  33 import chroot
  34 import logger
  35 import packages
  36 import packages.file
  37 import packages.packager
  38 import repository
  39 import util
  40 import _pakfire
  41
  42 import logging
  43 log = logging.getLogger("pakfire")
  44
  45 from system import system
  46 from constants import *
  47 from i18n import _
  48 from errors import BuildError, BuildRootLocked, Error
  49
  50
  51 BUILD_LOG_HEADER = """
  52  ____       _     __ _            _           _ _     _
  53 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  54 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  55 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  56 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  57
  58         Version : %(version)s
  59         Host    : %(hostname)s (%(host_arch)s)
  60         Time    : %(time)s
  61
  62 """
  63
  64 class BuildEnviron(object):
  65         # The version of the kernel this machine is running.
  66         kernel_version = os.uname()[2]
  67
  68         def __init__(self, filename=None, distro_config=None, build_id=None, logfile=None,
  69                         builder_mode="release", use_cache=None, **pakfire_args):
  70                 # Set mode.
  71                 assert builder_mode in ("development", "release",)
  72                 self.mode = builder_mode
  73
  74                 # Disable the build repository in release mode.
  75                 if self.mode == "release":
  76                         if pakfire_args.has_key("disable_repos") and pakfire_args["disable_repos"]:
  77                                 pakfire_args["disable_repos"] += ["build",]
  78                         else:
  79                                 pakfire_args["disable_repos"] = ["build",]
  80
  81                 # Save the build id and generate one if no build id was provided.
  82                 if not build_id:
  83                         build_id = "%s" % uuid.uuid4()
  84
  85                 self.build_id = build_id
  86
  87                 # Setup the logging.
  88                 if logfile:
  89                         self.log = log.getChild(self.build_id)
  90                         # Propage everything to the root logger that we will see something
  91                         # on the terminal.
  92                         self.log.propagate = 1
  93                         self.log.setLevel(logging.INFO)
  94
  95                         # Add the given logfile to the logger.
  96                         h = logging.FileHandler(logfile)
  97                         self.log.addHandler(h)
  98
  99                         # Format the log output for the file.
 100                         f = logger.BuildFormatter()
 101                         h.setFormatter(f)
 102                 else:
 103                         # If no logile was given, we use the root logger.
 104                         self.log = logging.getLogger("pakfire")
 105
 106                 # Log information about pakfire and some more information, when we
 107                 # are running in release mode.
 108                 if self.mode == "release":
 109                         logdata = {
 110                                 "host_arch"  : system.arch,
 111                                 "hostname"   : system.hostname,
 112                                 "time"       : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
 113                                 "version"    : "Pakfire %s" % PAKFIRE_VERSION,
 114                         }
 115
 116                         for line in BUILD_LOG_HEADER.splitlines():
 117                                 self.log.info(line % logdata)
 118
 119                 # Create pakfire instance.
 120                 if pakfire_args.has_key("mode"):
 121                         del pakfire_args["mode"]
 122                 self.pakfire = base.Pakfire(mode="builder", distro_config=distro_config, **pakfire_args)
 123                 self.distro = self.pakfire.distro
 124                 self.path = self.pakfire.path
 125
 126                 # Check if this host can build the requested architecture.
 127                 if not self.arch in self.pakfire.config.supported_arches:
 128                         raise BuildError, _("Cannot build for %s on this host.") % self.arch
 129
 130                 # Where do we put the result?
 131                 self.resultdir = os.path.join(self.path, "result")
 132
 133                 # Check weather to use or not use the cache.
 134                 if use_cache is None:
 135                         # If use_cache is None, the user did not provide anything and
 136                         # so we guess.
 137                         if self.mode == "development":
 138                                 use_cache = True
 139                         else:
 140                                 use_cache = False
 141
 142                 self.use_cache = use_cache
 143
 144                 # Open package.
 145                 # If we have a plain makefile, we first build a source package and go with that.
 146                 if filename:
 147                         if filename.endswith(".%s" % MAKEFILE_EXTENSION):
 148                                 pkg = packages.Makefile(self.pakfire, filename)
 149                                 pkg.dist([self.resultdir,])
 150
 151                                 filename = os.path.join(self.resultdir, "src", pkg.package_filename)
 152                                 assert os.path.exists(filename), filename
 153
 154                         # Open source package.
 155                         self.pkg = packages.SourcePackage(self.pakfire, None, filename)
 156                         assert self.pkg, filename
 157
 158                         # Log the package information.
 159                         self.log.info(_("Package information:"))
 160                         for line in self.pkg.dump(long=True).splitlines():
 161                                 self.log.info("  %s" % line)
 162                         self.log.info("")
 163
 164                         # Path where we extract the package and put all the source files.
 165                         self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name)
 166                 else:
 167                         # No package :(
 168                         self.pkg = None
 169
 170                 # XXX need to make this configureable
 171                 self.settings = {
 172                         "enable_loop_devices" : True,
 173                         "enable_ccache"   : True,
 174                         "enable_icecream" : False,
 175                 }
 176                 #self.settings.update(settings)
 177
 178                 # Lock the buildroot
 179                 self._lock = None
 180                 self.lock()
 181
 182                 # Save the build time.
 183                 self.build_time = int(time.time())
 184
 185         def start(self):
 186                 # Mount the directories.
 187                 self._mountall()
 188
 189                 # Populate /dev.
 190                 self.populate_dev()
 191
 192                 # Setup domain name resolution in chroot.
 193                 self.setup_dns()
 194
 195                 # Extract all needed packages.
 196                 self.extract()
 197
 198         def stop(self):
 199                 # Kill all still running processes.
 200                 util.orphans_kill(self.path)
 201
 202                 # Close pakfire instance.
 203                 del self.pakfire
 204
 205                 # Umount the build environment.
 206                 self._umountall()
 207
 208                 # Remove all files.
 209                 self.destroy()
 210
 211         @property
 212         def arch(self):
 213                 """
 214                         Inherit architecture from distribution configuration.
 215                 """
 216                 return self.distro.arch
 217
 218         @property
 219         def info(self):
 220                 return {
 221                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 222                         "build_host" : socket.gethostname(),
 223                         "build_id"   : self.build_id,
 224                         "build_time" : self.build_time,
 225                 }
 226
 227         def lock(self):
 228                 filename = os.path.join(self.path, ".lock")
 229
 230                 try:
 231                         self._lock = open(filename, "a+")
 232                 except IOError, e:
 233                         return 0
 234
 235                 try:
 236                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 237                 except IOError, e:
 238                         raise BuildRootLocked, "Buildroot is locked"
 239
 240                 return 1
 241
 242         def unlock(self):
 243                 if self._lock:
 244                         self._lock.close()
 245                         self._lock = None
 246
 247         def copyin(self, file_out, file_in):
 248                 if file_in.startswith("/"):
 249                         file_in = file_in[1:]
 250
 251                 file_in = self.chrootPath(file_in)
 252
 253                 #if not os.path.exists(file_out):
 254                 #       return
 255
 256                 dir_in = os.path.dirname(file_in)
 257                 if not os.path.exists(dir_in):
 258                         os.makedirs(dir_in)
 259
 260                 self.log.debug("%s --> %s" % (file_out, file_in))
 261
 262                 shutil.copy2(file_out, file_in)
 263
 264         def copyout(self, file_in, file_out):
 265                 if file_in.startswith("/"):
 266                         file_in = file_in[1:]
 267
 268                 file_in = self.chrootPath(file_in)
 269
 270                 #if not os.path.exists(file_in):
 271                 #       return
 272
 273                 dir_out = os.path.dirname(file_out)
 274                 if not os.path.exists(dir_out):
 275                         os.makedirs(dir_out)
 276
 277                 self.log.debug("%s --> %s" % (file_in, file_out))
 278
 279                 shutil.copy2(file_in, file_out)
 280
 281         def copy_result(self, resultdir):
 282                 dir_in = self.chrootPath("result")
 283
 284                 for dir, subdirs, files in os.walk(dir_in):
 285                         basename = os.path.basename(dir)
 286                         dir = dir[len(self.chrootPath()):]
 287                         for file in files:
 288                                 file_in = os.path.join(dir, file)
 289
 290                                 file_out = os.path.join(
 291                                         resultdir,
 292                                         basename,
 293                                         file,
 294                                 )
 295
 296                                 self.copyout(file_in, file_out)
 297
 298         def extract(self, requires=None, build_deps=True):
 299                 """
 300                         Gets a dependency set and extracts all packages
 301                         to the environment.
 302                 """
 303                 if not requires:
 304                         requires = []
 305
 306                 if self.use_cache and os.path.exists(self.cache_file):
 307                         # If we are told to use the cache, we just import the
 308                         # file.
 309                         self.cache_extract()
 310                 else:
 311                         # Add neccessary build dependencies.
 312                         requires += BUILD_PACKAGES
 313
 314                 # If we have ccache enabled, we need to extract it
 315                 # to the build chroot.
 316                 if self.settings.get("enable_ccache"):
 317                         requires.append("ccache")
 318
 319                 # If we have icecream enabled, we need to extract it
 320                 # to the build chroot.
 321                 if self.settings.get("enable_icecream"):
 322                         requires.append("icecream")
 323
 324                 # Get build dependencies from source package.
 325                 if self.pkg:
 326                         for req in self.pkg.requires:
 327                                 requires.append(req)
 328
 329                 # Install all packages.
 330                 self.log.info(_("Install packages needed for build..."))
 331                 self.install(requires)
 332
 333                 # Copy the makefile and load source tarballs.
 334                 if self.pkg:
 335                         self.pkg.extract(_("Extracting"), prefix=self.build_dir)
 336
 337         def install(self, requires):
 338                 """
 339                         Install everything that is required in requires.
 340                 """
 341                 # If we got nothing to do, we quit immediately.
 342                 if not requires:
 343                         return
 344
 345                 try:
 346                         self.pakfire.install(requires, interactive=False,
 347                                 allow_downgrade=True, logger=self.log)
 348
 349                 # Catch dependency errors and log it.
 350                 except DependencyError, e:
 351                         raise
 352
 353         def install_test(self):
 354                 pkgs = []
 355                 for dir, subdirs, files in os.walk(self.chrootPath("result")):
 356                         for file in files:
 357                                 pkgs.append(os.path.join(dir, file))
 358
 359                 self.pakfire.localinstall(pkgs, yes=True, allow_uninstall=True)
 360
 361         def chrootPath(self, *args):
 362                 # Remove all leading slashes
 363                 _args = []
 364                 for arg in args:
 365                         if arg.startswith("/"):
 366                                 arg = arg[1:]
 367                         _args.append(arg)
 368                 args = _args
 369
 370                 ret = os.path.join(self.path, *args)
 371                 ret = ret.replace("//", "/")
 372
 373                 assert ret.startswith(self.path)
 374
 375                 return ret
 376
 377         def populate_dev(self):
 378                 nodes = [
 379                         "/dev/null",
 380                         "/dev/zero",
 381                         "/dev/full",
 382                         "/dev/random",
 383                         "/dev/urandom",
 384                         "/dev/tty",
 385                         "/dev/ptmx",
 386                         "/dev/kmsg",
 387                         "/dev/rtc0",
 388                         "/dev/console",
 389                 ]
 390
 391                 # If we need loop devices (which are optional) we create them here.
 392                 if self.settings["enable_loop_devices"]:
 393                         for i in range(0, 7):
 394                                 nodes.append("/dev/loop%d" % i)
 395
 396                 for node in nodes:
 397                         # Stat the original node of the host system and copy it to
 398                         # the build chroot.
 399                         node_stat = os.stat(node)
 400
 401                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 402
 403                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 404                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 405                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 406                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 407
 408         def setup_dns(self):
 409                 """
 410                         Add DNS resolution facility to chroot environment by copying
 411                         /etc/resolv.conf and /etc/hosts.
 412                 """
 413                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 414                         self.copyin(i, i)
 415
 416         def _create_node(self, filename, mode, device):
 417                 self.log.debug("Create node: %s (%s)" % (filename, mode))
 418
 419                 filename = self.chrootPath(filename)
 420
 421                 # Create parent directory if it is missing.
 422                 dirname = os.path.dirname(filename)
 423                 if not os.path.exists(dirname):
 424                         os.makedirs(dirname)
 425
 426                 os.mknod(filename, mode, device)
 427
 428         def destroy(self):
 429                 self.log.debug("Destroying environment %s" % self.path)
 430
 431                 if os.path.exists(self.path):
 432                         util.rm(self.path)
 433
 434         def cleanup(self):
 435                 self.log.debug("Cleaning environemnt.")
 436
 437                 # Remove the build directory and buildroot.
 438                 dirs = (self.build_dir, self.chrootPath("result"),)
 439
 440                 for d in dirs:
 441                         if not os.path.exists(d):
 442                                 continue
 443
 444                         util.rm(d)
 445                         os.makedirs(d)
 446
 447         def _mountall(self):
 448                 self.log.debug("Mounting environment")
 449                 for src, dest, fs, options in self.mountpoints:
 450                         mountpoint = self.chrootPath(dest)
 451                         if options:
 452                                 options = "-o %s" % options
 453
 454                         # Eventually create mountpoint directory
 455                         if not os.path.exists(mountpoint):
 456                                 os.makedirs(mountpoint)
 457
 458                         cmd = "mount -n -t %s %s %s %s" % \
 459                                 (fs, options, src, mountpoint)
 460                         chroot.do(cmd, shell=True)
 461
 462         def _umountall(self):
 463                 self.log.debug("Umounting environment")
 464
 465                 mountpoints = []
 466                 for src, dest, fs, options in reversed(self.mountpoints):
 467                         if not dest in mountpoints:
 468                                 mountpoints.append(dest)
 469
 470                 for dest in mountpoints:
 471                         mountpoint = self.chrootPath(dest)
 472
 473                         chroot.do("umount -n %s" % mountpoint, raiseExc=0, shell=True)
 474
 475         @property
 476         def mountpoints(self):
 477                 mountpoints = []
 478
 479                 # Make root as a tmpfs.
 480                 #mountpoints += [
 481                 #       ("pakfire_root", "/", "tmpfs", "defaults"),
 482                 #]
 483
 484                 mountpoints += [
 485                         # src, dest, fs, options
 486                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 487                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 488                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 489                         ("/sys",          "/sys",      "bind",  "bind"),
 490                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 491                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 492                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 493                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 494                 ]
 495
 496                 # If selinux is enabled.
 497                 if os.path.exists("/sys/fs/selinux"):
 498                         mountpoints += [
 499                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 500                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 501                         ]
 502
 503                 # If ccache support is requested, we bind mount the cache.
 504                 if self.settings.get("enable_ccache"):
 505                         # Create ccache cache directory if it does not exist.
 506                         if not os.path.exists(CCACHE_CACHE_DIR):
 507                                 os.makedirs(CCACHE_CACHE_DIR)
 508
 509                         mountpoints += [
 510                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 511                         ]
 512
 513                 return mountpoints
 514
 515         @property
 516         def environ(self):
 517                 env = {
 518                         # Add HOME manually, because it is occasionally not set
 519                         # and some builds get in trouble then.
 520                         "HOME" : "/root",
 521                         "TERM" : os.environ.get("TERM", "dumb"),
 522                         "PS1"  : "\u:\w\$ ",
 523
 524                         # Set the container that we can detect, if we are inside a
 525                         # chroot.
 526                         "container" : "pakfire-builder",
 527                 }
 528
 529                 # Inherit environment from distro
 530                 env.update(self.pakfire.distro.environ)
 531
 532                 # Icecream environment settings
 533                 if self.settings.get("enable_icecream", False):
 534                         # Set the toolchain path
 535                         if self.settings.get("icecream_toolchain", None):
 536                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 537
 538                         # Set preferred host if configured.
 539                         if self.settings.get("icecream_preferred_host", None):
 540                                 env["ICECC_PREFERRED_HOST"] = \
 541                                         self.settings.get("icecream_preferred_host")
 542
 543                 # Fake UTS_MACHINE, when we cannot use the personality syscall and
 544                 # if the host architecture is not equal to the target architecture.
 545                 if not self.pakfire.distro.personality and \
 546                                 not self.pakfire.config.host_arch == self.pakfire.distro.arch:
 547                         env.update({
 548                                 "LD_PRELOAD"  : "/usr/lib/libpakfire_preload.so",
 549                                 "UTS_MACHINE" : self.pakfire.distro.arch,
 550                         })
 551
 552                 return env
 553
 554         @property
 555         def installed_packages(self):
 556                 """
 557                         Returns an iterator over all installed packages in this build environment.
 558                 """
 559                 # Get the repository of all installed packages.
 560                 repo = self.pakfire.repos.get_repo("@system")
 561
 562                 # Return an iterator over the packages.
 563                 return iter(repo)
 564
 565         def do(self, command, shell=True, personality=None, logger=None, *args, **kwargs):
 566                 ret = None
 567
 568                 # Environment variables
 569                 env = self.environ
 570
 571                 if kwargs.has_key("env"):
 572                         env.update(kwargs.pop("env"))
 573
 574                 self.log.debug("Environment:")
 575                 for k, v in sorted(env.items()):
 576                         self.log.debug("  %s=%s" % (k, v))
 577
 578                 # Update personality it none was set
 579                 if not personality:
 580                         personality = self.distro.personality
 581
 582                 # Make every shell to a login shell because we set a lot of
 583                 # environment things there.
 584                 if shell:
 585                         command = ["bash", "--login", "-c", command]
 586
 587                 if not kwargs.has_key("chrootPath"):
 588                         kwargs["chrootPath"] = self.chrootPath()
 589
 590                 ret = chroot.do(
 591                         command,
 592                         personality=personality,
 593                         shell=False,
 594                         env=env,
 595                         logger=logger,
 596                         *args,
 597                         **kwargs
 598                 )
 599
 600                 return ret
 601
 602         def build(self, install_test=True):
 603                 if not self.pkg:
 604                         raise BuildError, _("You cannot run a build when no package was given.")
 605
 606                 # Search for the package file in build_dir and raise BuildError if it is not present.
 607                 pkgfile = os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION))
 608                 if not os.path.exists(pkgfile):
 609                         raise BuildError, _("Could not find makefile in build root: %s") % pkgfile
 610                 pkgfile = "/%s" % os.path.relpath(pkgfile, self.chrootPath())
 611
 612                 resultdir = self.chrootPath("/result")
 613
 614                 # Create the build command, that is executed in the chroot.
 615                 build_command = ["/usr/lib/pakfire/builder", "--offline",
 616                         "build", pkgfile, "--arch", self.arch, "--nodeps",
 617                         "--resultdir=/result",]
 618
 619                 try:
 620                         self.do(" ".join(build_command), logger=self.log)
 621
 622                 except Error:
 623                         raise BuildError, _("The build command failed. See logfile for details.")
 624
 625                 # Perform install test.
 626                 if install_test:
 627                         self.install_test()
 628
 629                 # Copy the final packages and stuff.
 630                 # XXX TODO resultdir
 631
 632         def shell(self, args=[]):
 633                 if not util.cli_is_interactive():
 634                         self.log.warning("Cannot run shell on non-interactive console.")
 635                         return
 636
 637                 # Install all packages that are needed to run a shell.
 638                 self.install(SHELL_PACKAGES)
 639
 640                 # XXX need to set CFLAGS here
 641                 command = "/usr/sbin/chroot %s %s %s" % \
 642                         (self.chrootPath(), SHELL_SCRIPT, " ".join(args))
 643
 644                 # Add personality if we require one
 645                 if self.pakfire.distro.personality:
 646                         command = "%s %s" % (self.pakfire.distro.personality, command)
 647
 648                 for key, val in self.environ.items():
 649                         command = "%s=\"%s\" " % (key, val) + command
 650
 651                 # Empty the environment
 652                 command = "env -i - %s" % command
 653
 654                 self.log.debug("Shell command: %s" % command)
 655
 656                 shell = os.system(command)
 657                 return os.WEXITSTATUS(shell)
 658
 659         @property
 660         def cache_file(self):
 661                 comps = [
 662                         self.pakfire.distro.sname,      # name of the distribution
 663                         self.pakfire.distro.release,    # release version
 664                         self.pakfire.distro.arch,       # architecture
 665                 ]
 666
 667                 return os.path.join(CACHE_ENVIRON_DIR, "%s.cache" %"-".join(comps))
 668
 669         def cache_export(self, filename):
 670                 # Sync all disk caches.
 671                 _pakfire.sync()
 672
 673                 # A list to store all mountpoints, so we don't package them.
 674                 mountpoints = []
 675
 676                 # A list containing all files we want to package.
 677                 filelist = []
 678
 679                 # Walk through the whole tree and collect all files
 680                 # that are on the same disk (not crossing mountpoints).
 681                 log.info(_("Creating filelist..."))
 682                 root = self.chrootPath()
 683                 for dir, subdirs, files in os.walk(root):
 684                         # Search for mountpoints and skip them.
 685                         if not dir == root and os.path.ismount(dir):
 686                                 mountpoints.append(dir)
 687                                 continue
 688
 689                         # Skip all directories under mountpoints.
 690                         if any([dir.startswith(m) for m in mountpoints]):
 691                                 continue
 692
 693                         # Add all other files.
 694                         filelist.append(dir)
 695                         for file in files:
 696                                 file = os.path.join(dir, file)
 697                                 filelist.append(file)
 698
 699                 # Create a nice progressbar.
 700                 p = util.make_progress(_("Compressing files..."), len(filelist))
 701                 i = 0
 702
 703                 # Create tar file and add all files to it.
 704                 f = packages.file.InnerTarFile.open(filename, "w:gz")
 705                 for file in filelist:
 706                         i += 1
 707                         if p:
 708                                 p.update(i)
 709
 710                         f.add(file, os.path.relpath(file, root), recursive=False)
 711                 f.close()
 712
 713                 # Finish progressbar.
 714                 if p:
 715                         p.finish()
 716
 717                 filesize = os.path.getsize(filename)
 718
 719                 log.info(_("Cache file was successfully created at %s.") % filename)
 720                 log.info(_("  Containing %(files)s files, it has a size of %(size)s.") % \
 721                         { "files" : len(filelist), "size" : util.format_size(filesize), })
 722
 723         def cache_extract(self):
 724                 root = self.chrootPath()
 725                 filename = self.cache_file
 726
 727                 f = packages.file.InnerTarFile.open(filename, "r:gz")
 728                 members = f.getmembers()
 729
 730                 # Make a nice progress bar as always.
 731                 p = util.make_progress(_("Extracting files..."), len(members))
 732
 733                 # Extract all files from the cache.
 734                 i = 0
 735                 for member in members:
 736                         if p:
 737                                 i += 1
 738                                 p.update(i)
 739
 740                         f.extract(member, path=root)
 741                 f.close()
 742
 743                 # Finish progressbar.
 744                 if p:
 745                         p.finish()
 746
 747                 # Re-read local repository.
 748                 self.pakfire.repos.local.update(force=True)
 749
 750                 # Update all packages.
 751                 self.log.info(_("Updating packages from cache..."))
 752                 self.pakfire.update(interactive=False, logger=self.log,
 753                         allow_archchange=True, allow_vendorchange=True, allow_downgrade=True)
 754
 755
 756 class Builder(object):
 757         def __init__(self, pakfire, filename, resultdir, **kwargs):
 758                 self.pakfire = pakfire
 759
 760                 self.filename = filename
 761
 762                 self.resultdir = resultdir
 763
 764                 # Open package file.
 765                 self.pkg = packages.Makefile(self.pakfire, self.filename)
 766
 767                 self._environ = {
 768                         "LANG"             : "C",
 769                 }
 770
 771         def mktemp(self):
 772                 """
 773                         Create a temporary file in the build environment.
 774                 """
 775                 file = "/tmp/pakfire_%s" % util.random_string()
 776
 777                 # Touch the file.
 778                 f = open(file, "w")
 779                 f.close()
 780
 781                 return file
 782
 783         @property
 784         def buildroot(self):
 785                 return self.pkg.buildroot
 786
 787         @property
 788         def distro(self):
 789                 return self.pakfire.distro
 790
 791         @property
 792         def environ(self):
 793                 environ = os.environ
 794
 795                 # Get all definitions from the package.
 796                 environ.update(self.pkg.exports)
 797
 798                 # Overwrite some definitions by default values.
 799                 environ.update(self._environ)
 800
 801                 return environ
 802
 803         def do(self, command, shell=True, personality=None, cwd=None, *args, **kwargs):
 804                 # Environment variables
 805                 log.debug("Environment:")
 806                 for k, v in sorted(self.environ.items()):
 807                         log.debug("  %s=%s" % (k, v))
 808
 809                 # Update personality it none was set
 810                 if not personality:
 811                         personality = self.distro.personality
 812
 813                 if not cwd:
 814                         cwd = "/%s" % LOCAL_TMP_PATH
 815
 816                 # Make every shell to a login shell because we set a lot of
 817                 # environment things there.
 818                 if shell:
 819                         command = ["bash", "--login", "-c", command]
 820
 821                 return chroot.do(
 822                         command,
 823                         personality=personality,
 824                         shell=False,
 825                         env=self.environ,
 826                         logger=logging.getLogger("pakfire"),
 827                         cwd=cwd,
 828                         *args,
 829                         **kwargs
 830                 )
 831
 832         def create_icecream_toolchain(self):
 833                 try:
 834                         out = self.do("icecc --build-native 2>/dev/null", returnOutput=True, cwd="/tmp")
 835                 except Error:
 836                         return
 837
 838                 for line in out.splitlines():
 839                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
 840                         if m:
 841                                 self._environ["ICECC_VERSION"] = "/tmp/%s" % m.group(1)
 842
 843         def create_buildscript(self, stage):
 844                 file = "/tmp/build_%s" % util.random_string()
 845
 846                 # Get buildscript from the package.
 847                 script = self.pkg.get_buildscript(stage)
 848
 849                 # Write script to an empty file.
 850                 f = open(file, "w")
 851                 f.write("#!/bin/sh\n\n")
 852                 f.write("set -e\n")
 853                 f.write("set -x\n")
 854                 f.write("\n%s\n" % script)
 855                 f.write("exit 0\n")
 856                 f.close()
 857                 os.chmod(file, 700)
 858
 859                 return file
 860
 861         def build(self):
 862                 # Create buildroot and remove all content if it was existant.
 863                 util.rm(self.buildroot)
 864                 os.makedirs(self.buildroot)
 865
 866                 # Build icecream toolchain if icecream is installed.
 867                 self.create_icecream_toolchain()
 868
 869                 for stage in ("prepare", "build", "test", "install"):
 870                         self.build_stage(stage)
 871
 872                 # Run post-build stuff.
 873                 self.post_compress_man_pages()
 874                 self.post_remove_static_libs()
 875                 self.post_extract_debuginfo()
 876
 877                 # Package the result.
 878                 # Make all these little package from the build environment.
 879                 log.info(_("Creating packages:"))
 880                 pkgs = []
 881                 for pkg in reversed(self.pkg.packages):
 882                         packager = packages.packager.BinaryPackager(self.pakfire, pkg,
 883                                 self, self.buildroot)
 884                         pkg = packager.run(self.resultdir)
 885                         pkgs.append(pkg)
 886                 log.info("")
 887
 888                 for pkg in sorted(pkgs):
 889                         for line in pkg.dump(long=True).splitlines():
 890                                 log.info(line)
 891                         log.info("")
 892                 log.info("")
 893
 894         def build_stage(self, stage):
 895                 # Get the buildscript for this stage.
 896                 buildscript = self.create_buildscript(stage)
 897
 898                 # Execute the buildscript of this stage.
 899                 log.info(_("Running stage %s:") % stage)
 900
 901                 try:
 902                         self.do(buildscript, shell=False)
 903
 904                 finally:
 905                         # Remove the buildscript.
 906                         if os.path.exists(buildscript):
 907                                 os.unlink(buildscript)
 908
 909         def post_remove_static_libs(self):
 910                 keep_libs = self.pkg.lexer.build.get_var("keep_libraries")
 911                 keep_libs = keep_libs.split()
 912
 913                 try:
 914                         self.do("%s/remove-static-libs %s %s" % \
 915                                 (SCRIPT_DIR, self.buildroot, " ".join(keep_libs)))
 916                 except Error, e:
 917                         log.warning(_("Could not remove static libraries: %s") % e)
 918
 919         def post_compress_man_pages(self):
 920                 try:
 921                         self.do("%s/compress-man-pages %s" % (SCRIPT_DIR, self.buildroot))
 922                 except Error, e:
 923                         log.warning(_("Compressing man pages did not complete successfully."))
 924
 925         def post_extract_debuginfo(self):
 926                 args = []
 927
 928                 # Check if we need to run with strict build-id.
 929                 strict_id = self.pkg.lexer.build.get_var("debuginfo_strict_build_id", "true")
 930                 if strict_id in ("true", "yes", "1"):
 931                         args.append("--strict-build-id")
 932
 933                 args.append("--buildroot=%s" % self.pkg.buildroot)
 934                 args.append("--sourcedir=%s" % self.pkg.sourcedir)
 935
 936                 # Get additional options to pass to script.
 937                 options = self.pkg.lexer.build.get_var("debuginfo_options", "")
 938                 args += options.split()
 939
 940                 try:
 941                         self.do("%s/extract-debuginfo %s %s" % (SCRIPT_DIR, " ".join(args), self.pkg.buildroot))
 942                 except Error, e:
 943                         log.error(_("Extracting debuginfo did not complete with success. Aborting build."))
 944                         raise
 945
 946         def cleanup(self):
 947                 if os.path.exists(self.buildroot):
 948                         util.rm(self.buildroot)