python/pakfire/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import math
  25 import os
  26 import re
  27 import shutil
  28 import socket
  29 import time
  30 import uuid
  31
  32 import base
  33 import chroot
  34 import logger
  35 import packages
  36 import packages.file
  37 import packages.packager
  38 import repository
  39 import util
  40 import _pakfire
  41
  42 import logging
  43 log = logging.getLogger("pakfire")
  44
  45 from config import ConfigBuilder
  46 from system import system
  47 from constants import *
  48 from i18n import _
  49 from errors import BuildError, BuildRootLocked, Error
  50
  51
  52 BUILD_LOG_HEADER = """
  53  ____       _     __ _            _           _ _     _
  54 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  55 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  56 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  57 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  58
  59         Version : %(version)s
  60         Host    : %(hostname)s (%(host_arch)s)
  61         Time    : %(time)s
  62
  63 """
  64
  65 class BuildEnviron(object):
  66         # The version of the kernel this machine is running.
  67         kernel_version = os.uname()[2]
  68
  69         def __init__(self, filename=None, distro_name=None, config=None, configs=None, arch=None,
  70                         build_id=None, logfile=None, builder_mode="release", use_cache=None, **pakfire_args):
  71                 # Set mode.
  72                 assert builder_mode in ("development", "release",)
  73                 self.mode = builder_mode
  74
  75                 # Disable the build repository in release mode.
  76                 if self.mode == "release":
  77                         if pakfire_args.has_key("disable_repos") and pakfire_args["disable_repos"]:
  78                                 pakfire_args["disable_repos"] += ["build",]
  79                         else:
  80                                 pakfire_args["disable_repos"] = ["build",]
  81
  82                 # Save the build id and generate one if no build id was provided.
  83                 if not build_id:
  84                         build_id = "%s" % uuid.uuid4()
  85
  86                 self.build_id = build_id
  87
  88                 # Setup the logging.
  89                 if logfile:
  90                         self.log = log.getChild(self.build_id)
  91                         # Propage everything to the root logger that we will see something
  92                         # on the terminal.
  93                         self.log.propagate = 1
  94                         self.log.setLevel(logging.INFO)
  95
  96                         # Add the given logfile to the logger.
  97                         h = logging.FileHandler(logfile)
  98                         self.log.addHandler(h)
  99
 100                         # Format the log output for the file.
 101                         f = logger.BuildFormatter()
 102                         h.setFormatter(f)
 103                 else:
 104                         # If no logile was given, we use the root logger.
 105                         self.log = logging.getLogger("pakfire")
 106
 107                 # Log information about pakfire and some more information, when we
 108                 # are running in release mode.
 109                 if self.mode == "release":
 110                         logdata = {
 111                                 "host_arch"  : system.arch,
 112                                 "hostname"   : system.hostname,
 113                                 "time"       : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
 114                                 "version"    : "Pakfire %s" % PAKFIRE_VERSION,
 115                         }
 116
 117                         for line in BUILD_LOG_HEADER.splitlines():
 118                                 self.log.info(line % logdata)
 119
 120                 # Create pakfire instance.
 121                 if pakfire_args.has_key("mode"):
 122                         del pakfire_args["mode"]
 123
 124                 if config is None:
 125                         config = ConfigBuilder(files=configs)
 126
 127                         if not configs:
 128                                 if distro_name is None:
 129                                         distro_name = config.get("builder", "distro", None)
 130                                 config.load_distro_config(distro_name)
 131
 132                 if not config.has_distro():
 133                         log.error(_("You have not set the distribution for which you want to build."))
 134                         log.error(_("Please do so in builder.conf or on the CLI."))
 135                         raise ConfigError, _("Distribution configuration is missing.")
 136
 137                 self.pakfire = base.Pakfire(
 138                         mode="builder",
 139                         config=config,
 140                         arch=arch,
 141                         **pakfire_args
 142                 )
 143
 144                 self.distro = self.pakfire.distro
 145                 self.path = self.pakfire.path
 146
 147                 # Check if this host can build the requested architecture.
 148                 if not system.host_supports_arch(self.arch):
 149                         raise BuildError, _("Cannot build for %s on this host.") % self.arch
 150
 151                 # Where do we put the result?
 152                 self.resultdir = os.path.join(self.path, "result")
 153
 154                 # Check weather to use or not use the cache.
 155                 if use_cache is None:
 156                         # If use_cache is None, the user did not provide anything and
 157                         # so we guess.
 158                         if self.mode == "development":
 159                                 use_cache = True
 160                         else:
 161                                 use_cache = False
 162
 163                 self.use_cache = use_cache
 164
 165                 # Open package.
 166                 # If we have a plain makefile, we first build a source package and go with that.
 167                 if filename:
 168                         if filename.endswith(".%s" % MAKEFILE_EXTENSION):
 169                                 pkg = packages.Makefile(self.pakfire, filename)
 170                                 filename = pkg.dist(os.path.join(self.resultdir, "src"))
 171
 172                                 assert os.path.exists(filename), filename
 173
 174                         # Open source package.
 175                         self.pkg = packages.SourcePackage(self.pakfire, None, filename)
 176                         assert self.pkg, filename
 177
 178                         # Log the package information.
 179                         self.log.info(_("Package information:"))
 180                         for line in self.pkg.dump(long=True).splitlines():
 181                                 self.log.info("  %s" % line)
 182                         self.log.info("")
 183
 184                         # Path where we extract the package and put all the source files.
 185                         self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name)
 186                 else:
 187                         # No package :(
 188                         self.pkg = None
 189
 190                 # XXX need to make this configureable
 191                 self.settings = {
 192                         "enable_loop_devices" : True,
 193                         "enable_ccache"   : True,
 194                         "enable_icecream" : False,
 195                         "sign_packages"   : True,
 196                 }
 197                 #self.settings.update(settings)
 198
 199                 # Lock the buildroot
 200                 self._lock = None
 201                 self.lock()
 202
 203                 # Save the build time.
 204                 self.build_time = int(time.time())
 205
 206         def start(self):
 207                 # Mount the directories.
 208                 self._mountall()
 209
 210                 # Populate /dev.
 211                 self.populate_dev()
 212
 213                 # Setup domain name resolution in chroot.
 214                 self.setup_dns()
 215
 216                 # Extract all needed packages.
 217                 self.extract()
 218
 219         def stop(self):
 220                 # Kill all still running processes.
 221                 util.orphans_kill(self.path)
 222
 223                 # Close pakfire instance.
 224                 del self.pakfire
 225
 226                 # Umount the build environment.
 227                 self._umountall()
 228
 229                 # Remove all files.
 230                 self.destroy()
 231
 232         @property
 233         def arch(self):
 234                 """
 235                         Inherit architecture from distribution configuration.
 236                 """
 237                 return self.distro.arch
 238
 239         @property
 240         def info(self):
 241                 return {
 242                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 243                         "build_host" : socket.gethostname(),
 244                         "build_id"   : self.build_id,
 245                         "build_time" : self.build_time,
 246                 }
 247
 248         @property
 249         def keyring(self):
 250                 """
 251                         Shortcut to access the pakfire keyring.
 252
 253                         (Makes also sure that it is properly initialized.)
 254                 """
 255                 assert self.pakfire
 256
 257                 if not self.pakfire.keyring.initialized:
 258                         self.pakfire.keyring.init()
 259
 260                 return self.pakfire.keyring
 261
 262         def lock(self):
 263                 filename = os.path.join(self.path, ".lock")
 264
 265                 try:
 266                         self._lock = open(filename, "a+")
 267                 except IOError, e:
 268                         return 0
 269
 270                 try:
 271                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 272                 except IOError, e:
 273                         raise BuildRootLocked, "Buildroot is locked"
 274
 275                 return 1
 276
 277         def unlock(self):
 278                 if self._lock:
 279                         self._lock.close()
 280                         self._lock = None
 281
 282         def copyin(self, file_out, file_in):
 283                 if file_in.startswith("/"):
 284                         file_in = file_in[1:]
 285
 286                 file_in = self.chrootPath(file_in)
 287
 288                 #if not os.path.exists(file_out):
 289                 #       return
 290
 291                 dir_in = os.path.dirname(file_in)
 292                 if not os.path.exists(dir_in):
 293                         os.makedirs(dir_in)
 294
 295                 self.log.debug("%s --> %s" % (file_out, file_in))
 296
 297                 shutil.copy2(file_out, file_in)
 298
 299         def copyout(self, file_in, file_out):
 300                 if file_in.startswith("/"):
 301                         file_in = file_in[1:]
 302
 303                 file_in = self.chrootPath(file_in)
 304
 305                 #if not os.path.exists(file_in):
 306                 #       return
 307
 308                 dir_out = os.path.dirname(file_out)
 309                 if not os.path.exists(dir_out):
 310                         os.makedirs(dir_out)
 311
 312                 self.log.debug("%s --> %s" % (file_in, file_out))
 313
 314                 shutil.copy2(file_in, file_out)
 315
 316         def copy_result(self, resultdir):
 317                 # XXX should use find_result_packages
 318
 319                 dir_in = self.chrootPath("result")
 320
 321                 for dir, subdirs, files in os.walk(dir_in):
 322                         basename = os.path.basename(dir)
 323                         dir = dir[len(self.chrootPath()):]
 324                         for file in files:
 325                                 file_in = os.path.join(dir, file)
 326
 327                                 file_out = os.path.join(
 328                                         resultdir,
 329                                         basename,
 330                                         file,
 331                                 )
 332
 333                                 self.copyout(file_in, file_out)
 334
 335         def find_result_packages(self):
 336                 ret = []
 337
 338                 for dir, subdirs, files in os.walk(self.resultdir):
 339                         for file in files:
 340                                 if not file.endswith(".%s" % PACKAGE_EXTENSION):
 341                                         continue
 342
 343                                 file = os.path.join(dir, file)
 344                                 ret.append(file)
 345
 346                 return ret
 347
 348         def extract(self, requires=None, build_deps=True):
 349                 """
 350                         Gets a dependency set and extracts all packages
 351                         to the environment.
 352                 """
 353                 if not requires:
 354                         requires = []
 355
 356                 if self.use_cache and os.path.exists(self.cache_file):
 357                         # If we are told to use the cache, we just import the
 358                         # file.
 359                         self.cache_extract()
 360                 else:
 361                         # Add neccessary build dependencies.
 362                         requires += BUILD_PACKAGES
 363
 364                 # If we have ccache enabled, we need to extract it
 365                 # to the build chroot.
 366                 if self.settings.get("enable_ccache"):
 367                         requires.append("ccache")
 368
 369                 # If we have icecream enabled, we need to extract it
 370                 # to the build chroot.
 371                 if self.settings.get("enable_icecream"):
 372                         requires.append("icecream")
 373
 374                 # Get build dependencies from source package.
 375                 if self.pkg:
 376                         for req in self.pkg.requires:
 377                                 requires.append(req)
 378
 379                 # Install all packages.
 380                 self.log.info(_("Install packages needed for build..."))
 381                 self.install(requires)
 382
 383                 # Copy the makefile and load source tarballs.
 384                 if self.pkg:
 385                         self.pkg.extract(_("Extracting"), prefix=self.build_dir)
 386
 387         def install(self, requires):
 388                 """
 389                         Install everything that is required in requires.
 390                 """
 391                 # If we got nothing to do, we quit immediately.
 392                 if not requires:
 393                         return
 394
 395                 try:
 396                         self.pakfire.install(requires, interactive=False,
 397                                 allow_downgrade=True, logger=self.log)
 398
 399                 # Catch dependency errors and log it.
 400                 except DependencyError, e:
 401                         raise
 402
 403         def install_test(self):
 404                 try:
 405                         self.pakfire.localinstall(self.find_result_packages(), yes=True, allow_uninstall=True, logger=self.log)
 406
 407                 # Dependency errors when trying to install the result packages are build errors.
 408                 except DependencyError, e:
 409                         # Dump all packages (for debugging).
 410                         self.dump()
 411
 412                         raise BuildError, e
 413
 414         def chrootPath(self, *args):
 415                 # Remove all leading slashes
 416                 _args = []
 417                 for arg in args:
 418                         if arg.startswith("/"):
 419                                 arg = arg[1:]
 420                         _args.append(arg)
 421                 args = _args
 422
 423                 ret = os.path.join(self.path, *args)
 424                 ret = ret.replace("//", "/")
 425
 426                 assert ret.startswith(self.path)
 427
 428                 return ret
 429
 430         def populate_dev(self):
 431                 nodes = [
 432                         "/dev/null",
 433                         "/dev/zero",
 434                         "/dev/full",
 435                         "/dev/random",
 436                         "/dev/urandom",
 437                         "/dev/tty",
 438                         "/dev/ptmx",
 439                         "/dev/kmsg",
 440                         "/dev/rtc0",
 441                         "/dev/console",
 442                 ]
 443
 444                 # If we need loop devices (which are optional) we create them here.
 445                 if self.settings["enable_loop_devices"]:
 446                         for i in range(0, 7):
 447                                 nodes.append("/dev/loop%d" % i)
 448
 449                 for node in nodes:
 450                         # Stat the original node of the host system and copy it to
 451                         # the build chroot.
 452                         node_stat = os.stat(node)
 453
 454                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 455
 456                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 457                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 458                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 459                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 460
 461         def setup_dns(self):
 462                 """
 463                         Add DNS resolution facility to chroot environment by copying
 464                         /etc/resolv.conf and /etc/hosts.
 465                 """
 466                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 467                         self.copyin(i, i)
 468
 469         def _create_node(self, filename, mode, device):
 470                 self.log.debug("Create node: %s (%s)" % (filename, mode))
 471
 472                 filename = self.chrootPath(filename)
 473
 474                 # Create parent directory if it is missing.
 475                 dirname = os.path.dirname(filename)
 476                 if not os.path.exists(dirname):
 477                         os.makedirs(dirname)
 478
 479                 os.mknod(filename, mode, device)
 480
 481         def destroy(self):
 482                 self.log.debug("Destroying environment %s" % self.path)
 483
 484                 if os.path.exists(self.path):
 485                         util.rm(self.path)
 486
 487         def cleanup(self):
 488                 self.log.debug("Cleaning environemnt.")
 489
 490                 # Remove the build directory and buildroot.
 491                 dirs = (self.build_dir, self.chrootPath("result"),)
 492
 493                 for d in dirs:
 494                         if not os.path.exists(d):
 495                                 continue
 496
 497                         util.rm(d)
 498                         os.makedirs(d)
 499
 500         def _mountall(self):
 501                 self.log.debug("Mounting environment")
 502                 for src, dest, fs, options in self.mountpoints:
 503                         mountpoint = self.chrootPath(dest)
 504                         if options:
 505                                 options = "-o %s" % options
 506
 507                         # Eventually create mountpoint directory
 508                         if not os.path.exists(mountpoint):
 509                                 os.makedirs(mountpoint)
 510
 511                         cmd = "mount -n -t %s %s %s %s" % \
 512                                 (fs, options, src, mountpoint)
 513                         chroot.do(cmd, shell=True)
 514
 515         def _umountall(self):
 516                 self.log.debug("Umounting environment")
 517
 518                 mountpoints = []
 519                 for src, dest, fs, options in reversed(self.mountpoints):
 520                         if not dest in mountpoints:
 521                                 mountpoints.append(dest)
 522
 523                 for dest in mountpoints:
 524                         mountpoint = self.chrootPath(dest)
 525
 526                         chroot.do("umount -n %s" % mountpoint, raiseExc=0, shell=True)
 527
 528         @property
 529         def mountpoints(self):
 530                 mountpoints = []
 531
 532                 # Make root as a tmpfs.
 533                 #mountpoints += [
 534                 #       ("pakfire_root", "/", "tmpfs", "defaults"),
 535                 #]
 536
 537                 mountpoints += [
 538                         # src, dest, fs, options
 539                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 540                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 541                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 542                         ("/sys",          "/sys",      "bind",  "bind"),
 543                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 544                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 545                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 546                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 547                 ]
 548
 549                 # If selinux is enabled.
 550                 if os.path.exists("/sys/fs/selinux"):
 551                         mountpoints += [
 552                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 553                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 554                         ]
 555
 556                 # If ccache support is requested, we bind mount the cache.
 557                 if self.settings.get("enable_ccache"):
 558                         # Create ccache cache directory if it does not exist.
 559                         if not os.path.exists(CCACHE_CACHE_DIR):
 560                                 os.makedirs(CCACHE_CACHE_DIR)
 561
 562                         mountpoints += [
 563                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 564                         ]
 565
 566                 return mountpoints
 567
 568         @property
 569         def environ(self):
 570                 env = {
 571                         # Add HOME manually, because it is occasionally not set
 572                         # and some builds get in trouble then.
 573                         "HOME" : "/root",
 574                         "TERM" : os.environ.get("TERM", "dumb"),
 575                         "PS1"  : "\u:\w\$ ",
 576
 577                         # Sanitize language.
 578                         "LANG" : os.environ.setdefault("LANG", "en_US.UTF-8"),
 579
 580                         # Set the container that we can detect, if we are inside a
 581                         # chroot.
 582                         "container" : "pakfire-builder",
 583                 }
 584
 585                 # Inherit environment from distro
 586                 env.update(self.pakfire.distro.environ)
 587
 588                 # Icecream environment settings
 589                 if self.settings.get("enable_icecream", False):
 590                         # Set the toolchain path
 591                         if self.settings.get("icecream_toolchain", None):
 592                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 593
 594                         # Set preferred host if configured.
 595                         if self.settings.get("icecream_preferred_host", None):
 596                                 env["ICECC_PREFERRED_HOST"] = \
 597                                         self.settings.get("icecream_preferred_host")
 598
 599                 # Fake UTS_MACHINE, when we cannot use the personality syscall and
 600                 # if the host architecture is not equal to the target architecture.
 601                 if not self.pakfire.distro.personality and \
 602                                 not system.native_arch == self.pakfire.distro.arch:
 603                         env.update({
 604                                 "LD_PRELOAD"  : "/usr/lib/libpakfire_preload.so",
 605                                 "UTS_MACHINE" : self.pakfire.distro.arch,
 606                         })
 607
 608                 return env
 609
 610         @property
 611         def installed_packages(self):
 612                 """
 613                         Returns an iterator over all installed packages in this build environment.
 614                 """
 615                 # Get the repository of all installed packages.
 616                 repo = self.pakfire.repos.get_repo("@system")
 617
 618                 # Return an iterator over the packages.
 619                 return iter(repo)
 620
 621         def write_config(self):
 622                 # Cleanup everything in /etc/pakfire.
 623                 util.rm(self.chrootPath(CONFIG_DIR))
 624
 625                 for i in (CONFIG_DIR, CONFIG_REPOS_DIR):
 626                         i = self.chrootPath(i)
 627                         if not os.path.exists(i):
 628                                 os.makedirs(i)
 629
 630                 # Write general.conf.
 631                 f = open(self.chrootPath(CONFIG_DIR, "general.conf"), "w")
 632                 f.close()
 633
 634                 # Write builder.conf.
 635                 f = open(self.chrootPath(CONFIG_DIR, "builder.conf"), "w")
 636                 f.write(self.distro.get_config())
 637                 f.close()
 638
 639                 # Create pakfire configuration files.
 640                 for repo in self.pakfire.repos:
 641                         conf = repo.get_config()
 642
 643                         if not conf:
 644                                 continue
 645
 646                         filename = self.chrootPath(CONFIG_REPOS_DIR, "%s.repo" % repo.name)
 647                         f = open(filename, "w")
 648                         f.write("\n".join(conf))
 649                         f.close()
 650
 651         def do(self, command, shell=True, personality=None, logger=None, *args, **kwargs):
 652                 ret = None
 653
 654                 # Environment variables
 655                 env = self.environ
 656
 657                 if kwargs.has_key("env"):
 658                         env.update(kwargs.pop("env"))
 659
 660                 self.log.debug("Environment:")
 661                 for k, v in sorted(env.items()):
 662                         self.log.debug("  %s=%s" % (k, v))
 663
 664                 # Update personality it none was set
 665                 if not personality:
 666                         personality = self.distro.personality
 667
 668                 # Make every shell to a login shell because we set a lot of
 669                 # environment things there.
 670                 if shell:
 671                         command = ["bash", "--login", "-c", command]
 672
 673                 if not kwargs.has_key("chrootPath"):
 674                         kwargs["chrootPath"] = self.chrootPath()
 675
 676                 ret = chroot.do(
 677                         command,
 678                         personality=personality,
 679                         shell=False,
 680                         env=env,
 681                         logger=logger,
 682                         *args,
 683                         **kwargs
 684                 )
 685
 686                 return ret
 687
 688         def build(self, install_test=True):
 689                 if not self.pkg:
 690                         raise BuildError, _("You cannot run a build when no package was given.")
 691
 692                 # Search for the package file in build_dir and raise BuildError if it is not present.
 693                 pkgfile = os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION))
 694                 if not os.path.exists(pkgfile):
 695                         raise BuildError, _("Could not find makefile in build root: %s") % pkgfile
 696                 pkgfile = "/%s" % os.path.relpath(pkgfile, self.chrootPath())
 697
 698                 # Write pakfire configuration into the chroot.
 699                 self.write_config()
 700
 701                 # Create the build command, that is executed in the chroot.
 702                 build_command = [
 703                         "/usr/lib/pakfire/builder",
 704                         "--offline",
 705                         "build",
 706                         pkgfile,
 707                         "--arch", self.arch,
 708                         "--nodeps",
 709                         "--resultdir=/result",
 710                 ]
 711
 712                 try:
 713                         self.do(" ".join(build_command), logger=self.log)
 714
 715                 except Error:
 716                         self.log.error(_("Build failed."), exc_info=True)
 717
 718                         raise BuildError, _("The build command failed. See logfile for details.")
 719
 720                 # Sign all built packages with the host key (if available).
 721                 if self.settings.get("sign_packages"):
 722                         host_key = self.keyring.get_host_key()
 723                         assert host_key
 724
 725                         # Do the signing...
 726                         self.sign(host_key)
 727
 728                 # Perform install test.
 729                 if install_test:
 730                         self.install_test()
 731
 732                 # Dump package information.
 733                 self.dump()
 734
 735         def shell(self, args=[]):
 736                 if not util.cli_is_interactive():
 737                         self.log.warning("Cannot run shell on non-interactive console.")
 738                         return
 739
 740                 # Install all packages that are needed to run a shell.
 741                 self.install(SHELL_PACKAGES)
 742
 743                 # XXX need to set CFLAGS here
 744                 command = "/usr/sbin/chroot %s %s %s" % \
 745                         (self.chrootPath(), SHELL_SCRIPT, " ".join(args))
 746
 747                 # Add personality if we require one
 748                 if self.pakfire.distro.personality:
 749                         command = "%s %s" % (self.pakfire.distro.personality, command)
 750
 751                 for key, val in self.environ.items():
 752                         command = "%s=\"%s\" " % (key, val) + command
 753
 754                 # Empty the environment
 755                 command = "env -i - %s" % command
 756
 757                 self.log.debug("Shell command: %s" % command)
 758
 759                 shell = os.system(command)
 760                 return os.WEXITSTATUS(shell)
 761
 762         def sign(self, keyfp):
 763                 assert self.keyring.get_key(keyfp), "Key for signing does not exist"
 764
 765                 # Find all files to process.
 766                 files = self.find_result_packages()
 767
 768                 # Create a progressbar.
 769                 p = util.make_progress(_("Signing files (%s)") % keyfp, len(files))
 770                 i = 0
 771
 772                 for file in files:
 773                         # Update progressbar.
 774                         if p:
 775                                 i += 1
 776                                 p.update(i)
 777
 778                         # Open package file.
 779                         pkg = packages.open(self.pakfire, None, file)
 780
 781                         # Sign it.
 782                         pkg.sign(keyfp)
 783
 784                 # Close progressbar.
 785                 if p:
 786                         p.finish()
 787
 788         def dump(self):
 789                 pkgs = []
 790
 791                 for file in self.find_result_packages():
 792                         pkg = packages.open(self.pakfire, None, file)
 793                         pkgs.append(pkg)
 794
 795                 # If there are no packages, there is nothing to do.
 796                 if not pkgs:
 797                         return
 798
 799                 pkgs.sort()
 800
 801                 self.log.info(_("Dumping package information:"))
 802                 for pkg in pkgs:
 803                         dump = pkg.dump(long=True)
 804
 805                         for line in dump.splitlines():
 806                                 self.log.info("  %s" % line)
 807                         self.log.info("") # Empty line.
 808
 809         @property
 810         def cache_file(self):
 811                 comps = [
 812                         self.pakfire.distro.sname,      # name of the distribution
 813                         self.pakfire.distro.release,    # release version
 814                         self.pakfire.distro.arch,       # architecture
 815                 ]
 816
 817                 return os.path.join(CACHE_ENVIRON_DIR, "%s.cache" %"-".join(comps))
 818
 819         def cache_export(self, filename):
 820                 # Sync all disk caches.
 821                 _pakfire.sync()
 822
 823                 # A list to store all mountpoints, so we don't package them.
 824                 mountpoints = []
 825
 826                 # A list containing all files we want to package.
 827                 filelist = []
 828
 829                 # Walk through the whole tree and collect all files
 830                 # that are on the same disk (not crossing mountpoints).
 831                 log.info(_("Creating filelist..."))
 832                 root = self.chrootPath()
 833                 for dir, subdirs, files in os.walk(root):
 834                         # Search for mountpoints and skip them.
 835                         if not dir == root and os.path.ismount(dir):
 836                                 mountpoints.append(dir)
 837                                 continue
 838
 839                         # Skip all directories under mountpoints.
 840                         if any([dir.startswith(m) for m in mountpoints]):
 841                                 continue
 842
 843                         # Add all other files.
 844                         filelist.append(dir)
 845                         for file in files:
 846                                 file = os.path.join(dir, file)
 847                                 filelist.append(file)
 848
 849                 # Create a nice progressbar.
 850                 p = util.make_progress(_("Compressing files..."), len(filelist))
 851                 i = 0
 852
 853                 # Create tar file and add all files to it.
 854                 f = packages.file.InnerTarFile.open(filename, "w:gz")
 855                 for file in filelist:
 856                         i += 1
 857                         if p:
 858                                 p.update(i)
 859
 860                         f.add(file, os.path.relpath(file, root), recursive=False)
 861                 f.close()
 862
 863                 # Finish progressbar.
 864                 if p:
 865                         p.finish()
 866
 867                 filesize = os.path.getsize(filename)
 868
 869                 log.info(_("Cache file was successfully created at %s.") % filename)
 870                 log.info(_("  Containing %(files)s files, it has a size of %(size)s.") % \
 871                         { "files" : len(filelist), "size" : util.format_size(filesize), })
 872
 873         def cache_extract(self):
 874                 root = self.chrootPath()
 875                 filename = self.cache_file
 876
 877                 f = packages.file.InnerTarFile.open(filename, "r:gz")
 878                 members = f.getmembers()
 879
 880                 # Make a nice progress bar as always.
 881                 p = util.make_progress(_("Extracting files..."), len(members))
 882
 883                 # Extract all files from the cache.
 884                 i = 0
 885                 for member in members:
 886                         if p:
 887                                 i += 1
 888                                 p.update(i)
 889
 890                         f.extract(member, path=root)
 891                 f.close()
 892
 893                 # Finish progressbar.
 894                 if p:
 895                         p.finish()
 896
 897                 # Re-read local repository.
 898                 self.pakfire.repos.local.update(force=True)
 899
 900                 # Update all packages.
 901                 self.log.info(_("Updating packages from cache..."))
 902                 self.pakfire.update(interactive=False, logger=self.log,
 903                         allow_archchange=True, allow_vendorchange=True, allow_downgrade=True)
 904
 905
 906 class Builder(object):
 907         def __init__(self, pakfire, filename, resultdir, **kwargs):
 908                 self.pakfire = pakfire
 909
 910                 self.filename = filename
 911
 912                 self.resultdir = resultdir
 913
 914                 # Open package file.
 915                 self.pkg = packages.Makefile(self.pakfire, self.filename)
 916
 917                 self._environ = {
 918                         "LANG"             : "C",
 919                 }
 920
 921         def mktemp(self):
 922                 """
 923                         Create a temporary file in the build environment.
 924                 """
 925                 file = "/tmp/pakfire_%s" % util.random_string()
 926
 927                 # Touch the file.
 928                 f = open(file, "w")
 929                 f.close()
 930
 931                 return file
 932
 933         @property
 934         def buildroot(self):
 935                 return self.pkg.buildroot
 936
 937         @property
 938         def distro(self):
 939                 return self.pakfire.distro
 940
 941         @property
 942         def environ(self):
 943                 environ = os.environ
 944
 945                 # Get all definitions from the package.
 946                 environ.update(self.pkg.exports)
 947
 948                 # Overwrite some definitions by default values.
 949                 environ.update(self._environ)
 950
 951                 return environ
 952
 953         def do(self, command, shell=True, personality=None, cwd=None, *args, **kwargs):
 954                 # Environment variables
 955                 log.debug("Environment:")
 956                 for k, v in sorted(self.environ.items()):
 957                         log.debug("  %s=%s" % (k, v))
 958
 959                 # Update personality it none was set
 960                 if not personality:
 961                         personality = self.distro.personality
 962
 963                 if not cwd:
 964                         cwd = "/%s" % LOCAL_TMP_PATH
 965
 966                 # Make every shell to a login shell because we set a lot of
 967                 # environment things there.
 968                 if shell:
 969                         command = ["bash", "--login", "-c", command]
 970
 971                 return chroot.do(
 972                         command,
 973                         personality=personality,
 974                         shell=False,
 975                         env=self.environ,
 976                         logger=logging.getLogger("pakfire"),
 977                         cwd=cwd,
 978                         *args,
 979                         **kwargs
 980                 )
 981
 982         def create_icecream_toolchain(self):
 983                 try:
 984                         out = self.do("icecc --build-native 2>/dev/null", returnOutput=True, cwd="/tmp")
 985                 except Error:
 986                         return
 987
 988                 for line in out.splitlines():
 989                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
 990                         if m:
 991                                 self._environ["ICECC_VERSION"] = "/tmp/%s" % m.group(1)
 992
 993         def create_buildscript(self, stage):
 994                 file = "/tmp/build_%s" % util.random_string()
 995
 996                 # Get buildscript from the package.
 997                 script = self.pkg.get_buildscript(stage)
 998
 999                 # Write script to an empty file.
1000                 f = open(file, "w")
1001                 f.write("#!/bin/sh\n\n")
1002                 f.write("set -e\n")
1003                 f.write("set -x\n")
1004                 f.write("\n%s\n" % script)
1005                 f.write("exit 0\n")
1006                 f.close()
1007                 os.chmod(file, 700)
1008
1009                 return file
1010
1011         def build(self):
1012                 # Create buildroot and remove all content if it was existant.
1013                 util.rm(self.buildroot)
1014                 os.makedirs(self.buildroot)
1015
1016                 # Build icecream toolchain if icecream is installed.
1017                 self.create_icecream_toolchain()
1018
1019                 for stage in ("prepare", "build", "test", "install"):
1020                         self.build_stage(stage)
1021
1022                 # Run post-build stuff.
1023                 self.post_compress_man_pages()
1024                 self.post_remove_static_libs()
1025                 self.post_extract_debuginfo()
1026
1027                 # Package the result.
1028                 # Make all these little package from the build environment.
1029                 log.info(_("Creating packages:"))
1030                 pkgs = []
1031                 for pkg in reversed(self.pkg.packages):
1032                         packager = packages.packager.BinaryPackager(self.pakfire, pkg,
1033                                 self, self.buildroot)
1034                         pkg = packager.run(self.resultdir)
1035                         pkgs.append(pkg)
1036                 log.info("")
1037
1038         def build_stage(self, stage):
1039                 # Get the buildscript for this stage.
1040                 buildscript = self.create_buildscript(stage)
1041
1042                 # Execute the buildscript of this stage.
1043                 log.info(_("Running stage %s:") % stage)
1044
1045                 try:
1046                         self.do(buildscript, shell=False)
1047
1048                 finally:
1049                         # Remove the buildscript.
1050                         if os.path.exists(buildscript):
1051                                 os.unlink(buildscript)
1052
1053         def post_remove_static_libs(self):
1054                 keep_libs = self.pkg.lexer.build.get_var("keep_libraries")
1055                 keep_libs = keep_libs.split()
1056
1057                 try:
1058                         self.do("%s/remove-static-libs %s %s" % \
1059                                 (SCRIPT_DIR, self.buildroot, " ".join(keep_libs)))
1060                 except Error, e:
1061                         log.warning(_("Could not remove static libraries: %s") % e)
1062
1063         def post_compress_man_pages(self):
1064                 try:
1065                         self.do("%s/compress-man-pages %s" % (SCRIPT_DIR, self.buildroot))
1066                 except Error, e:
1067                         log.warning(_("Compressing man pages did not complete successfully."))
1068
1069         def post_extract_debuginfo(self):
1070                 args = []
1071
1072                 # Check if we need to run with strict build-id.
1073                 strict_id = self.pkg.lexer.build.get_var("debuginfo_strict_build_id", "true")
1074                 if strict_id in ("true", "yes", "1"):
1075                         args.append("--strict-build-id")
1076
1077                 args.append("--buildroot=%s" % self.pkg.buildroot)
1078                 args.append("--sourcedir=%s" % self.pkg.sourcedir)
1079
1080                 # Get additional options to pass to script.
1081                 options = self.pkg.lexer.build.get_var("debuginfo_options", "")
1082                 args += options.split()
1083
1084                 try:
1085                         self.do("%s/extract-debuginfo %s %s" % (SCRIPT_DIR, " ".join(args), self.pkg.buildroot))
1086                 except Error, e:
1087                         log.error(_("Extracting debuginfo did not complete with success. Aborting build."))
1088                         raise
1089
1090         def cleanup(self):
1091                 if os.path.exists(self.buildroot):
1092                         util.rm(self.buildroot)