python/pakfire/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import math
  25 import os
  26 import re
  27 import shutil
  28 import signal
  29 import socket
  30 import tempfile
  31 import time
  32 import uuid
  33
  34 import base
  35 import cgroup
  36 import logger
  37 import packages
  38 import packages.file
  39 import packages.packager
  40 import repository
  41 import shell
  42 import util
  43 import _pakfire
  44
  45 import logging
  46 log = logging.getLogger("pakfire")
  47
  48 from config import ConfigBuilder
  49 from system import system
  50 from constants import *
  51 from i18n import _
  52 from errors import BuildError, BuildRootLocked, Error
  53
  54
  55 BUILD_LOG_HEADER = """
  56  ____       _     __ _            _           _ _     _
  57 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  58 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  59 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  60 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  61
  62         Version : %(version)s
  63         Host    : %(hostname)s (%(host_arch)s)
  64         Time    : %(time)s
  65
  66 """
  67
  68 class BuildEnviron(object):
  69         # The version of the kernel this machine is running.
  70         kernel_version = os.uname()[2]
  71
  72         def __init__(self, pakfire, filename=None, distro_name=None, build_id=None, logfile=None, release_build=True, **kwargs):
  73                 self.pakfire = pakfire
  74
  75                 # Check if the given pakfire instance is of the correct type.
  76                 assert isinstance(self.pakfire, base.PakfireBuilder)
  77
  78                 # Check if this host can build the requested architecture.
  79                 if not system.host_supports_arch(self.arch):
  80                         raise BuildError, _("Cannot build for %s on this host.") % self.arch
  81
  82                 # Save the build id and generate one if no build id was provided.
  83                 if not build_id:
  84                         build_id = "%s" % uuid.uuid4()
  85
  86                 self.build_id = build_id
  87
  88                 # Setup the logging.
  89                 self.init_logging(logfile)
  90
  91                 # Initialize a cgroup (if supported).
  92                 self.init_cgroup()
  93
  94                 # This build is a release build?
  95                 self.release_build = release_build
  96
  97                 if self.release_build:
  98                         # Disable the local build repository in release mode.
  99                         self.pakfire.repos.disable_repo("build")
 100
 101                         # Log information about pakfire and some more information, when we
 102                         # are running in release mode.
 103                         logdata = {
 104                                 "host_arch"  : system.arch,
 105                                 "hostname"   : system.hostname,
 106                                 "time"       : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
 107                                 "version"    : "Pakfire %s" % PAKFIRE_VERSION,
 108                         }
 109
 110                         for line in BUILD_LOG_HEADER.splitlines():
 111                                 self.log.info(line % logdata)
 112
 113                 # Settings array.
 114                 self.settings = {
 115                         "enable_loop_devices" : self.config.get_bool("builder", "use_loop_devices", True),
 116                         "enable_ccache"       : self.config.get_bool("builder", "use_ccache", True),
 117                         "enable_icecream"     : self.config.get_bool("builder", "use_icecream", False),
 118                         "sign_packages"       : False,
 119                         "buildroot_tmpfs"     : self.config.get_bool("builder", "use_tmpfs", False),
 120                         "private_network"     : self.config.get_bool("builder", "private_network", False),
 121                 }
 122
 123                 # Get ccache settings.
 124                 if self.settings.get("enable_ccache", False):
 125                         self.settings.update({
 126                                 "ccache_compress" : self.config.get_bool("ccache", "compress", True),
 127                         })
 128
 129                 # Try to get the configured host key. If it is available,
 130                 # we will automatically sign all packages with it.
 131                 if self.keyring.get_host_key(secret=True):
 132                         self.settings["sign_packages"] = True
 133
 134                 # Add settings from keyword arguments.
 135                 self.settings.update(kwargs)
 136
 137                 # Where do we put the result?
 138                 self.resultdir = os.path.join(self.pakfire.path, "result")
 139
 140                 # Open package.
 141                 # If we have a plain makefile, we first build a source package and go with that.
 142                 if filename:
 143                         # Open source package.
 144                         self.pkg = packages.SourcePackage(self.pakfire, None, filename)
 145                         assert self.pkg, filename
 146
 147                         # Log the package information.
 148                         self.log.info(_("Package information:"))
 149                         for line in self.pkg.dump(long=True).splitlines():
 150                                 self.log.info("  %s" % line)
 151                         self.log.info("")
 152
 153                         # Path where we extract the package and put all the source files.
 154                         self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name)
 155                 else:
 156                         # No package :(
 157                         self.pkg = None
 158
 159                 # Lock the buildroot
 160                 self._lock = None
 161
 162                 # Save the build time.
 163                 self.build_time = time.time()
 164
 165         def setup_signal_handlers(self):
 166                 pass
 167
 168         def start(self):
 169                 assert not self.pakfire.initialized, "Pakfire has already been initialized"
 170
 171                 # Unshare namepsace.
 172                 # If this fails because the kernel has no support for CLONE_NEWIPC or CLONE_NEWUTS,
 173                 # we try to fall back to just set CLONE_NEWNS.
 174                 try:
 175                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNS|_pakfire.SCHED_CLONE_NEWIPC|_pakfire.SCHED_CLONE_NEWUTS)
 176                 except RuntimeError, e:
 177                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNS)
 178
 179                 # Mount the directories.
 180                 self._mountall()
 181
 182                 # Lock the build environment.
 183                 self.lock()
 184
 185                 # Initialize pakfire instance.
 186                 self.pakfire.initialize()
 187
 188                 # Optionally enable private networking.
 189                 if self.settings.get("private_network", None):
 190                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNET)
 191
 192                 # Populate /dev.
 193                 self.populate_dev()
 194
 195                 # Setup domain name resolution in chroot.
 196                 self.setup_dns()
 197
 198                 # Extract all needed packages.
 199                 self.extract()
 200
 201         def stop(self):
 202                 if self.cgroup:
 203                         # Kill all still running processes in the cgroup.
 204                         self.cgroup.kill_and_wait()
 205
 206                         # Remove cgroup and all parent cgroups if they are empty.
 207                         self.cgroup.migrate_task(self.cgroup.root, os.getpid())
 208                         self.cgroup.destroy()
 209
 210                         parent = self.cgroup.parent
 211                         while parent:
 212                                 if not parent.is_empty(recursive=True):
 213                                         break
 214
 215                                 parent.destroy()
 216                                 parent = parent.parent
 217
 218                 else:
 219                         util.orphans_kill(self.path)
 220
 221                 # Shut down pakfire instance.
 222                 self.pakfire.destroy()
 223
 224                 # Unlock build environment.
 225                 self.unlock()
 226
 227                 # Umount the build environment.
 228                 self._umountall()
 229
 230                 # Remove all files.
 231                 self.destroy()
 232
 233         @property
 234         def config(self):
 235                 """
 236                         Proxy method for easy access to the configuration.
 237                 """
 238                 return self.pakfire.config
 239
 240         @property
 241         def distro(self):
 242                 """
 243                         Proxy method for easy access to the distribution.
 244                 """
 245                 return self.pakfire.distro
 246
 247         @property
 248         def path(self):
 249                 """
 250                         Proxy method for easy access to the path.
 251                 """
 252                 return self.pakfire.path
 253
 254         @property
 255         def arch(self):
 256                 """
 257                         Inherit architecture from distribution configuration.
 258                 """
 259                 return self.pakfire.distro.arch
 260
 261         @property
 262         def personality(self):
 263                 """
 264                         Gets the personality from the distribution configuration.
 265                 """
 266                 return self.pakfire.distro.personality
 267
 268         @property
 269         def info(self):
 270                 return {
 271                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 272                         "build_host" : socket.gethostname(),
 273                         "build_id"   : self.build_id,
 274                         "build_time" : self.build_time,
 275                 }
 276
 277         @property
 278         def keyring(self):
 279                 """
 280                         Shortcut to access the pakfire keyring.
 281                 """
 282                 return self.pakfire.keyring
 283
 284         def lock(self):
 285                 filename = os.path.join(self.path, ".lock")
 286
 287                 try:
 288                         self._lock = open(filename, "a+")
 289                 except IOError, e:
 290                         return 0
 291
 292                 try:
 293                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 294                 except IOError, e:
 295                         raise BuildRootLocked, "Buildroot is locked"
 296
 297                 return 1
 298
 299         def unlock(self):
 300                 if self._lock:
 301                         self._lock.close()
 302                         self._lock = None
 303
 304         def init_cgroup(self):
 305                 """
 306                         Initialize cgroup (if the system supports it).
 307                 """
 308                 if not cgroup.supported():
 309                         self.cgroup = None
 310                         return
 311
 312                 self.cgroup = cgroup.CGroup("pakfire/builder/%s" % self.build_id)
 313
 314                 # Attach the pakfire-builder process to the parent group.
 315                 self.cgroup.parent.attach()
 316
 317         def init_logging(self, logfile):
 318                 if logfile:
 319                         self.log = log.getChild(self.build_id)
 320                         # Propage everything to the root logger that we will see something
 321                         # on the terminal.
 322                         self.log.propagate = 1
 323                         self.log.setLevel(logging.INFO)
 324
 325                         # Add the given logfile to the logger.
 326                         h = logging.FileHandler(logfile)
 327                         self.log.addHandler(h)
 328
 329                         # Format the log output for the file.
 330                         f = logger.BuildFormatter()
 331                         h.setFormatter(f)
 332                 else:
 333                         # If no logile was given, we use the root logger.
 334                         self.log = logging.getLogger("pakfire")
 335
 336         def copyin(self, file_out, file_in):
 337                 if file_in.startswith("/"):
 338                         file_in = file_in[1:]
 339
 340                 file_in = self.chrootPath(file_in)
 341
 342                 #if not os.path.exists(file_out):
 343                 #       return
 344
 345                 dir_in = os.path.dirname(file_in)
 346                 if not os.path.exists(dir_in):
 347                         os.makedirs(dir_in)
 348
 349                 self.log.debug("%s --> %s" % (file_out, file_in))
 350
 351                 shutil.copy2(file_out, file_in)
 352
 353         def copyout(self, file_in, file_out):
 354                 if file_in.startswith("/"):
 355                         file_in = file_in[1:]
 356
 357                 file_in = self.chrootPath(file_in)
 358
 359                 #if not os.path.exists(file_in):
 360                 #       return
 361
 362                 dir_out = os.path.dirname(file_out)
 363                 if not os.path.exists(dir_out):
 364                         os.makedirs(dir_out)
 365
 366                 self.log.debug("%s --> %s" % (file_in, file_out))
 367
 368                 shutil.copy2(file_in, file_out)
 369
 370         def copy_result(self, resultdir):
 371                 # XXX should use find_result_packages
 372
 373                 dir_in = self.chrootPath("result")
 374
 375                 for dir, subdirs, files in os.walk(dir_in):
 376                         basename = os.path.basename(dir)
 377                         dir = dir[len(self.chrootPath()):]
 378                         for file in files:
 379                                 file_in = os.path.join(dir, file)
 380
 381                                 file_out = os.path.join(
 382                                         resultdir,
 383                                         basename,
 384                                         file,
 385                                 )
 386
 387                                 self.copyout(file_in, file_out)
 388
 389         def find_result_packages(self):
 390                 ret = []
 391
 392                 for dir, subdirs, files in os.walk(self.resultdir):
 393                         for file in files:
 394                                 if not file.endswith(".%s" % PACKAGE_EXTENSION):
 395                                         continue
 396
 397                                 file = os.path.join(dir, file)
 398                                 ret.append(file)
 399
 400                 return ret
 401
 402         def extract(self, requires=None):
 403                 """
 404                         Gets a dependency set and extracts all packages
 405                         to the environment.
 406                 """
 407                 if not requires:
 408                         requires = []
 409
 410                 # Add neccessary build dependencies.
 411                 requires += BUILD_PACKAGES
 412
 413                 # If we have ccache enabled, we need to extract it
 414                 # to the build chroot.
 415                 if self.settings.get("enable_ccache"):
 416                         requires.append("ccache")
 417
 418                 # If we have icecream enabled, we need to extract it
 419                 # to the build chroot.
 420                 if self.settings.get("enable_icecream"):
 421                         requires.append("icecream")
 422
 423                 # Get build dependencies from source package.
 424                 if self.pkg:
 425                         for req in self.pkg.requires:
 426                                 requires.append(req)
 427
 428                 # Install all packages.
 429                 self.log.info(_("Install packages needed for build..."))
 430                 self.install(requires)
 431
 432                 # Copy the makefile and load source tarballs.
 433                 if self.pkg:
 434                         self.pkg.extract(_("Extracting"), prefix=self.build_dir)
 435
 436                         # Add an empty line at the end.
 437                         self.log.info("")
 438
 439         def install(self, requires, **kwargs):
 440                 """
 441                         Install everything that is required in requires.
 442                 """
 443                 # If we got nothing to do, we quit immediately.
 444                 if not requires:
 445                         return
 446
 447                 kwargs.update({
 448                         "interactive" : False,
 449                         "logger" : self.log,
 450                 })
 451
 452                 if not kwargs.has_key("allow_downgrade"):
 453                         kwargs["allow_downgrade"] = True
 454
 455                 # Install everything.
 456                 self.pakfire.install(requires, **kwargs)
 457
 458         def chrootPath(self, *args):
 459                 # Remove all leading slashes
 460                 _args = []
 461                 for arg in args:
 462                         if arg.startswith("/"):
 463                                 arg = arg[1:]
 464                         _args.append(arg)
 465                 args = _args
 466
 467                 ret = os.path.join(self.path, *args)
 468                 ret = ret.replace("//", "/")
 469
 470                 assert ret.startswith(self.path)
 471
 472                 return ret
 473
 474         def populate_dev(self):
 475                 nodes = [
 476                         "/dev/null",
 477                         "/dev/zero",
 478                         "/dev/full",
 479                         "/dev/random",
 480                         "/dev/urandom",
 481                         "/dev/tty",
 482                         "/dev/ptmx",
 483                         "/dev/kmsg",
 484                         "/dev/rtc0",
 485                         "/dev/console",
 486                 ]
 487
 488                 # If we need loop devices (which are optional) we create them here.
 489                 if self.settings["enable_loop_devices"]:
 490                         for i in range(0, 7):
 491                                 nodes.append("/dev/loop%d" % i)
 492
 493                 for node in nodes:
 494                         # Stat the original node of the host system and copy it to
 495                         # the build chroot.
 496                         node_stat = os.stat(node)
 497
 498                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 499
 500                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 501                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 502                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 503                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 504
 505         def setup_dns(self):
 506                 """
 507                         Add DNS resolution facility to chroot environment by copying
 508                         /etc/resolv.conf and /etc/hosts.
 509                 """
 510                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 511                         self.copyin(i, i)
 512
 513         def _create_node(self, filename, mode, device):
 514                 self.log.debug("Create node: %s (%s)" % (filename, mode))
 515
 516                 filename = self.chrootPath(filename)
 517
 518                 # Create parent directory if it is missing.
 519                 dirname = os.path.dirname(filename)
 520                 if not os.path.exists(dirname):
 521                         os.makedirs(dirname)
 522
 523                 os.mknod(filename, mode, device)
 524
 525         def destroy(self):
 526                 self.log.debug("Destroying environment %s" % self.path)
 527
 528                 if os.path.exists(self.path):
 529                         util.rm(self.path)
 530
 531         def cleanup(self):
 532                 self.log.debug("Cleaning environemnt.")
 533
 534                 # Remove the build directory and buildroot.
 535                 dirs = (self.build_dir, self.chrootPath("result"),)
 536
 537                 for d in dirs:
 538                         if not os.path.exists(d):
 539                                 continue
 540
 541                         util.rm(d)
 542                         os.makedirs(d)
 543
 544         def _mountall(self):
 545                 self.log.debug("Mounting environment")
 546                 for src, dest, fs, options in self.mountpoints:
 547                         mountpoint = self.chrootPath(dest)
 548                         if options:
 549                                 options = "-o %s" % options
 550
 551                         # Eventually create mountpoint directory
 552                         if not os.path.exists(mountpoint):
 553                                 os.makedirs(mountpoint)
 554
 555                         self.execute_root("mount -n -t %s %s %s %s" % (fs, options, src, mountpoint), shell=True)
 556
 557         def _umountall(self):
 558                 self.log.debug("Umounting environment")
 559
 560                 mountpoints = []
 561                 for src, dest, fs, options in reversed(self.mountpoints):
 562                         dest = self.chrootPath(dest)
 563
 564                         if not dest in mountpoints:
 565                                 mountpoints.append(dest)
 566
 567                 while mountpoints:
 568                         for mp in mountpoints:
 569                                 try:
 570                                         self.execute_root("umount -n %s" % mp, shell=True)
 571                                 except ShellEnvironmentError:
 572                                         pass
 573
 574                                 if not os.path.ismount(mp):
 575                                         mountpoints.remove(mp)
 576
 577         @property
 578         def mountpoints(self):
 579                 mountpoints = []
 580
 581                 # Make root as a tmpfs if enabled.
 582                 if self.settings.get("buildroot_tmpfs"):
 583                         mountpoints += [
 584                                 ("pakfire_root", "/", "tmpfs", "defaults"),
 585                         ]
 586
 587                 mountpoints += [
 588                         # src, dest, fs, options
 589                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 590                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 591                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 592                         ("/sys",          "/sys",      "bind",  "bind"),
 593                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 594                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 595                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 596                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 597                         ("pakfire_tmpfs", "/tmp",      "tmpfs", "mode=755,nosuid,nodev"),
 598                 ]
 599
 600                 # If selinux is enabled.
 601                 if os.path.exists("/sys/fs/selinux"):
 602                         mountpoints += [
 603                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 604                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 605                         ]
 606
 607                 # If ccache support is requested, we bind mount the cache.
 608                 if self.settings.get("enable_ccache"):
 609                         # Create ccache cache directory if it does not exist.
 610                         if not os.path.exists(CCACHE_CACHE_DIR):
 611                                 os.makedirs(CCACHE_CACHE_DIR)
 612
 613                         mountpoints += [
 614                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 615                         ]
 616
 617                 return mountpoints
 618
 619         @property
 620         def environ(self):
 621                 env = {
 622                         # Add HOME manually, because it is occasionally not set
 623                         # and some builds get in trouble then.
 624                         "PATH" : "/usr/bin:/bin:/usr/sbin:/sbin",
 625                         "HOME" : "/root",
 626                         "TERM" : os.environ.get("TERM", "vt100"),
 627                         "PS1"  : "\u:\w\$ ",
 628
 629                         # Sanitize language.
 630                         "LANG" : os.environ.setdefault("LANG", "en_US.UTF-8"),
 631
 632                         # Set the container that we can detect, if we are inside a
 633                         # chroot.
 634                         "container" : "pakfire-builder",
 635                 }
 636
 637                 # Inherit environment from distro
 638                 env.update(self.pakfire.distro.environ)
 639
 640                 # ccache environment settings
 641                 if self.settings.get("enable_ccache", False):
 642                         compress = self.settings.get("ccache_compress", False)
 643                         if compress:
 644                                 env["CCACHE_COMPRESS"] = "1"
 645
 646                         # Let ccache create its temporary files in /tmp.
 647                         env["CCACHE_TEMPDIR"] = "/tmp"
 648
 649                 # Icecream environment settings
 650                 if self.settings.get("enable_icecream", False):
 651                         # Set the toolchain path
 652                         if self.settings.get("icecream_toolchain", None):
 653                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 654
 655                         # Set preferred host if configured.
 656                         if self.settings.get("icecream_preferred_host", None):
 657                                 env["ICECC_PREFERRED_HOST"] = \
 658                                         self.settings.get("icecream_preferred_host")
 659
 660                 # Fake UTS_MACHINE, when we cannot use the personality syscall and
 661                 # if the host architecture is not equal to the target architecture.
 662                 if not self.pakfire.distro.personality and \
 663                                 not system.native_arch == self.pakfire.distro.arch:
 664                         env.update({
 665                                 "LD_PRELOAD"  : "/usr/lib/libpakfire_preload.so",
 666                                 "UTS_MACHINE" : self.pakfire.distro.arch,
 667                         })
 668
 669                 return env
 670
 671         @property
 672         def installed_packages(self):
 673                 """
 674                         Returns an iterator over all installed packages in this build environment.
 675                 """
 676                 # Get the repository of all installed packages.
 677                 repo = self.pakfire.repos.get_repo("@system")
 678
 679                 # Return an iterator over the packages.
 680                 return iter(repo)
 681
 682         def write_config(self):
 683                 # Cleanup everything in /etc/pakfire.
 684                 util.rm(self.chrootPath(CONFIG_DIR))
 685
 686                 for i in (CONFIG_DIR, CONFIG_REPOS_DIR):
 687                         i = self.chrootPath(i)
 688                         if not os.path.exists(i):
 689                                 os.makedirs(i)
 690
 691                 # Write general.conf.
 692                 f = open(self.chrootPath(CONFIG_DIR, "general.conf"), "w")
 693                 f.close()
 694
 695                 # Write builder.conf.
 696                 f = open(self.chrootPath(CONFIG_DIR, "builder.conf"), "w")
 697                 f.write(self.distro.get_config())
 698                 f.close()
 699
 700                 # Create pakfire configuration files.
 701                 for repo in self.pakfire.repos:
 702                         conf = repo.get_config()
 703
 704                         if not conf:
 705                                 continue
 706
 707                         filename = self.chrootPath(CONFIG_REPOS_DIR, "%s.repo" % repo.name)
 708                         f = open(filename, "w")
 709                         f.write("\n".join(conf))
 710                         f.close()
 711
 712         @property
 713         def pkg_makefile(self):
 714                 return os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION))
 715
 716         def execute(self, command, logger=None, **kwargs):
 717                 """
 718                         Executes the given command in the build chroot.
 719                 """
 720                 # Environment variables
 721                 env = self.environ
 722
 723                 if kwargs.has_key("env"):
 724                         env.update(kwargs.pop("env"))
 725
 726                 self.log.debug("Environment:")
 727                 for k, v in sorted(env.items()):
 728                         self.log.debug("  %s=%s" % (k, v))
 729
 730                 # Make every shell to a login shell because we set a lot of
 731                 # environment things there.
 732                 command = ["bash", "--login", "-c", command]
 733
 734                 args = {
 735                         "chroot_path" : self.chrootPath(),
 736                         "cgroup"      : self.cgroup,
 737                         "env"         : env,
 738                         "logger"      : logger,
 739                         "personality" : self.personality,
 740                         "shell"       : False,
 741                 }
 742                 args.update(kwargs)
 743
 744                 # Run the shit.
 745                 shellenv = shell.ShellExecuteEnvironment(command, **args)
 746                 shellenv.execute()
 747
 748                 return shellenv
 749
 750         def execute_root(self, command, **kwargs):
 751                 """
 752                         Executes the given command outside the build chroot.
 753                 """
 754                 shellenv = shell.ShellExecuteEnvironment(command, **kwargs)
 755                 shellenv.execute()
 756
 757                 return shellenv
 758
 759         def build(self, install_test=True, prepare=False):
 760                 if not self.pkg:
 761                         raise BuildError, _("You cannot run a build when no package was given.")
 762
 763                 # Search for the package file in build_dir and raise BuildError if it is not present.
 764                 if not os.path.exists(self.pkg_makefile):
 765                         raise BuildError, _("Could not find makefile in build root: %s") % self.pkg_makefile
 766
 767                 # Write pakfire configuration into the chroot.
 768                 self.write_config()
 769
 770                 # Create the build command, that is executed in the chroot.
 771                 build_command = [
 772                         "/usr/lib/pakfire/builder",
 773                         "--offline",
 774                         "build",
 775                         "/%s" % os.path.relpath(self.pkg_makefile, self.chrootPath()),
 776                         "--arch", self.arch,
 777                         "--nodeps",
 778                         "--resultdir=/result",
 779                 ]
 780
 781                 # Check if only the preparation stage should be run.
 782                 if prepare:
 783                         build_command.append("--prepare")
 784
 785                 build_command = " ".join(build_command)
 786
 787                 try:
 788                         self.execute(build_command, logger=self.log)
 789
 790                         # Perform the install test after the actual build.
 791                         if install_test and not prepare:
 792                                 self.install_test()
 793
 794                 except ShellEnvironmentError:
 795                         self.log.error(_("Build failed"))
 796
 797                 except KeyboardInterrupt:
 798                         self.log.error(_("Build interrupted"))
 799
 800                         raise
 801
 802                 # Catch all other errors.
 803                 except:
 804                         self.log.error(_("Build failed."), exc_info=True)
 805
 806                 else:
 807                         # Don't sign packages in prepare mode.
 808                         if prepare:
 809                                 return
 810
 811                         # Sign all built packages with the host key (if available).
 812                         self.sign_packages()
 813
 814                         # Dump package information.
 815                         self.dump()
 816
 817                         return
 818
 819                 # End here in case of an error.
 820                 raise BuildError, _("The build command failed. See logfile for details.")
 821
 822         def install_test(self):
 823                 self.log.info(_("Running installation test..."))
 824
 825                 # Install all packages that were built.
 826                 self.install(self.find_result_packages(), allow_vendorchange=True,
 827                         allow_uninstall=True, signatures_mode="disabled")
 828
 829                 self.log.info(_("Installation test succeeded."))
 830                 self.log.info("")
 831
 832         def shell(self, args=[]):
 833                 if not util.cli_is_interactive():
 834                         self.log.warning("Cannot run shell on non-interactive console.")
 835                         return
 836
 837                 # Install all packages that are needed to run a shell.
 838                 self.install(SHELL_PACKAGES)
 839
 840                 # XXX need to set CFLAGS here
 841                 command = "/usr/sbin/chroot %s %s %s" % \
 842                         (self.chrootPath(), SHELL_SCRIPT, " ".join(args))
 843
 844                 # Add personality if we require one
 845                 if self.pakfire.distro.personality:
 846                         command = "%s %s" % (self.pakfire.distro.personality, command)
 847
 848                 for key, val in self.environ.items():
 849                         command = "%s=\"%s\" " % (key, val) + command
 850
 851                 # Empty the environment
 852                 command = "env -i - %s" % command
 853
 854                 self.log.debug("Shell command: %s" % command)
 855
 856                 shell = os.system(command)
 857                 return os.WEXITSTATUS(shell)
 858
 859         def sign_packages(self, keyfp=None):
 860                 # Do nothing if signing is not requested.
 861                 if not self.settings.get("sign_packages"):
 862                         return
 863
 864                 # Get key, that should be used for signing.
 865                 if not keyfp:
 866                         keyfp = self.keyring.get_host_key_id()
 867
 868                 # Find all files to process.
 869                 files = self.find_result_packages()
 870
 871                 # Create a progressbar.
 872                 print _("Signing packages...")
 873                 p = util.make_progress(keyfp, len(files))
 874                 i = 0
 875
 876                 for file in files:
 877                         # Update progressbar.
 878                         if p:
 879                                 i += 1
 880                                 p.update(i)
 881
 882                         # Open package file.
 883                         pkg = packages.open(self.pakfire, None, file)
 884
 885                         # Sign it.
 886                         pkg.sign(keyfp)
 887
 888                 # Close progressbar.
 889                 if p:
 890                         p.finish()
 891                         print "" # Print an empty line.
 892
 893         def dump(self):
 894                 pkgs = []
 895
 896                 for file in self.find_result_packages():
 897                         pkg = packages.open(self.pakfire, None, file)
 898                         pkgs.append(pkg)
 899
 900                 # If there are no packages, there is nothing to do.
 901                 if not pkgs:
 902                         return
 903
 904                 pkgs.sort()
 905
 906                 self.log.info(_("Dumping package information:"))
 907                 for pkg in pkgs:
 908                         dump = pkg.dump(long=True)
 909
 910                         for line in dump.splitlines():
 911                                 self.log.info("  %s" % line)
 912                         self.log.info("") # Empty line.
 913
 914
 915 class Builder(object):
 916         def __init__(self, pakfire, filename, resultdir, **kwargs):
 917                 self.pakfire = pakfire
 918
 919                 self.filename = filename
 920
 921                 self.resultdir = resultdir
 922
 923                 # Open package file.
 924                 self.pkg = packages.Makefile(self.pakfire, self.filename)
 925
 926                 self._environ = {
 927                         "LANG"             : "C",
 928                 }
 929
 930         @property
 931         def buildroot(self):
 932                 return self.pkg.buildroot
 933
 934         @property
 935         def distro(self):
 936                 return self.pakfire.distro
 937
 938         @property
 939         def environ(self):
 940                 environ = os.environ
 941
 942                 # Get all definitions from the package.
 943                 environ.update(self.pkg.exports)
 944
 945                 # Overwrite some definitions by default values.
 946                 environ.update(self._environ)
 947
 948                 return environ
 949
 950         def execute(self, command, logger=None, **kwargs):
 951                 if logger is None:
 952                         logger = logging.getLogger("pakfire")
 953
 954                 # Make every shell to a login shell because we set a lot of
 955                 # environment things there.
 956                 command = ["bash", "--login", "-c", command]
 957
 958                 args = {
 959                         "cwd"         : "/%s" % LOCAL_TMP_PATH,
 960                         "env"         : self.environ,
 961                         "logger"      : logger,
 962                         "personality" : self.distro.personality,
 963                         "shell"       : False,
 964                 }
 965                 args.update(kwargs)
 966
 967                 try:
 968                         shellenv = shell.ShellExecuteEnvironment(command, **args)
 969                         shellenv.execute()
 970
 971                 except ShellEnvironmentError:
 972                         logger.error("Command exited with an error: %s" % command)
 973                         raise
 974
 975                 return shellenv
 976
 977         def run_script(self, script, *args):
 978                 if not script.startswith("/"):
 979                         script = os.path.join(SCRIPT_DIR, script)
 980
 981                 assert os.path.exists(script), "Script we should run does not exist: %s" % script
 982
 983                 cmd = [script,]
 984                 for arg in args:
 985                         cmd.append(arg)
 986                 cmd = " ".join(cmd)
 987
 988                 # Returns the output of the command, but the output won't get
 989                 # logged.
 990                 exe = self.execute(cmd, record_output=True, log_output=False)
 991
 992                 # Return the output of the command.
 993                 if exe.exitcode == 0:
 994                         return exe.output
 995
 996         def create_icecream_toolchain(self):
 997                 try:
 998                         exe = self.execute(
 999                                 "icecc --build-native 2>/dev/null",
1000                                 record_output=True, record_stderr=False,
1001                                 log_output=False, log_errors=False,
1002                                 cwd="/tmp",
1003                         )
1004                 except ShellEnvironmentError:
1005                         return
1006
1007                 for line in exe.output.splitlines():
1008                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
1009                         if m:
1010                                 self._environ["ICECC_VERSION"] = "/tmp/%s" % m.group(1)
1011
1012         def create_buildscript(self, stage):
1013                 # Get buildscript from the package.
1014                 script = self.pkg.get_buildscript(stage)
1015
1016                 # Write script to an empty file.
1017                 f = tempfile.NamedTemporaryFile(mode="w", delete=False)
1018                 f.write("#!/bin/sh\n\n")
1019                 f.write("set -e\n")
1020                 f.write("set -x\n")
1021                 f.write("\n%s\n" % script)
1022                 f.write("exit 0\n")
1023                 f.close()
1024
1025                 # Make the script executable.
1026                 os.chmod(f.name, 700)
1027
1028                 return f.name
1029
1030         def build(self, stages=None):
1031                 # Create buildroot and remove all content if it was existant.
1032                 util.rm(self.buildroot)
1033                 os.makedirs(self.buildroot)
1034
1035                 # Build icecream toolchain if icecream is installed.
1036                 self.create_icecream_toolchain()
1037
1038                 # Process stages in order.
1039                 for stage in ("prepare", "build", "test", "install"):
1040                         # Skip unwanted stages.
1041                         if stages and not stage in stages:
1042                                 continue
1043
1044                         # Run stage.
1045                         self.build_stage(stage)
1046
1047                 # Stop if install stage has not been processed.
1048                 if stages and not "install" in stages:
1049                         return
1050
1051                 # Run post-build stuff.
1052                 self.post_compress_man_pages()
1053                 self.post_remove_static_libs()
1054                 self.post_extract_debuginfo()
1055
1056                 # Package the result.
1057                 # Make all these little package from the build environment.
1058                 log.info(_("Creating packages:"))
1059                 pkgs = []
1060                 for pkg in reversed(self.pkg.packages):
1061                         packager = packages.packager.BinaryPackager(self.pakfire, pkg,
1062                                 self, self.buildroot)
1063                         pkg = packager.run(self.resultdir)
1064                         pkgs.append(pkg)
1065                 log.info("")
1066
1067         def build_stage(self, stage):
1068                 # Get the buildscript for this stage.
1069                 buildscript = self.create_buildscript(stage)
1070
1071                 # Execute the buildscript of this stage.
1072                 log.info(_("Running stage %s:") % stage)
1073
1074                 try:
1075                         self.execute(buildscript)
1076
1077                 finally:
1078                         # Remove the buildscript.
1079                         if os.path.exists(buildscript):
1080                                 os.unlink(buildscript)
1081
1082         def post_remove_static_libs(self):
1083                 keep_libs = self.pkg.lexer.build.get_var("keep_libraries")
1084                 keep_libs = keep_libs.split()
1085
1086                 try:
1087                         self.execute("%s/remove-static-libs %s %s" % \
1088                                 (SCRIPT_DIR, self.buildroot, " ".join(keep_libs)))
1089                 except ShellEnvironmentError, e:
1090                         log.warning(_("Could not remove static libraries: %s") % e)
1091
1092         def post_compress_man_pages(self):
1093                 try:
1094                         self.execute("%s/compress-man-pages %s" % (SCRIPT_DIR, self.buildroot))
1095                 except ShellEnvironmentError, e:
1096                         log.warning(_("Compressing man pages did not complete successfully."))
1097
1098         def post_extract_debuginfo(self):
1099                 args = []
1100
1101                 # Check if we need to run with strict build-id.
1102                 strict_id = self.pkg.lexer.build.get_var("debuginfo_strict_build_id", "true")
1103                 if strict_id in ("true", "yes", "1"):
1104                         args.append("--strict-build-id")
1105
1106                 args.append("--buildroot=%s" % self.pkg.buildroot)
1107                 args.append("--sourcedir=%s" % self.pkg.sourcedir)
1108
1109                 # Get additional options to pass to script.
1110                 options = self.pkg.lexer.build.get_var("debuginfo_options", "")
1111                 args += options.split()
1112
1113                 try:
1114                         self.execute("%s/extract-debuginfo %s %s" % (SCRIPT_DIR, " ".join(args), self.pkg.buildroot))
1115                 except ShellEnvironmentError, e:
1116                         log.error(_("Extracting debuginfo did not complete with success. Aborting build."))
1117                         raise
1118
1119         def find_prerequires(self, scriptlet_file):
1120                 assert os.path.exists(scriptlet_file), "Scriptlet file does not exist: %s" % scriptlet_file
1121
1122                 res = self.run_script("find-prerequires", scriptlet_file)
1123                 prerequires = set(res.splitlines())
1124
1125                 return prerequires
1126
1127         def cleanup(self):
1128                 if os.path.exists(self.buildroot):
1129                         util.rm(self.buildroot)