src/pakfire/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import math
  25 import os
  26 import re
  27 import shutil
  28 import signal
  29 import socket
  30 import tempfile
  31 import time
  32 import uuid
  33
  34 import base
  35 import cgroup
  36 import logger
  37 import packages
  38 import packages.file
  39 import packages.packager
  40 import repository
  41 import shell
  42 import util
  43 import _pakfire
  44
  45 import logging
  46 log = logging.getLogger("pakfire")
  47
  48 from config import ConfigBuilder
  49 from system import system
  50 from constants import *
  51 from i18n import _
  52 from errors import BuildError, BuildRootLocked, Error
  53
  54
  55 BUILD_LOG_HEADER = """
  56  ____       _     __ _            _           _ _     _
  57 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  58 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  59 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  60 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  61
  62         Version : %(version)s
  63         Host    : %(hostname)s (%(host_arch)s)
  64         Time    : %(time)s
  65
  66 """
  67
  68 class BuildEnviron(object):
  69         # The version of the kernel this machine is running.
  70         kernel_version = os.uname()[2]
  71
  72         def __init__(self, pakfire, filename=None, distro_name=None, build_id=None, logfile=None, release_build=True, **kwargs):
  73                 self.pakfire = pakfire
  74
  75                 # Check if the given pakfire instance is of the correct type.
  76                 assert isinstance(self.pakfire, base.PakfireBuilder)
  77
  78                 # Check if this host can build the requested architecture.
  79                 if not system.host_supports_arch(self.arch):
  80                         raise BuildError, _("Cannot build for %s on this host.") % self.arch
  81
  82                 # Save the build id and generate one if no build id was provided.
  83                 if not build_id:
  84                         build_id = "%s" % uuid.uuid4()
  85
  86                 self.build_id = build_id
  87
  88                 # Setup the logging.
  89                 self.init_logging(logfile)
  90
  91                 # Initialize a cgroup (if supported).
  92                 self.init_cgroup()
  93
  94                 # This build is a release build?
  95                 self.release_build = release_build
  96
  97                 if self.release_build:
  98                         # Disable the local build repository in release mode.
  99                         self.pakfire.repos.disable_repo("build")
 100
 101                         # Log information about pakfire and some more information, when we
 102                         # are running in release mode.
 103                         logdata = {
 104                                 "host_arch"  : system.arch,
 105                                 "hostname"   : system.hostname,
 106                                 "time"       : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
 107                                 "version"    : "Pakfire %s" % PAKFIRE_VERSION,
 108                         }
 109
 110                         for line in BUILD_LOG_HEADER.splitlines():
 111                                 self.log.info(line % logdata)
 112
 113                 # Settings array.
 114                 self.settings = {
 115                         "enable_loop_devices" : self.config.get_bool("builder", "use_loop_devices", True),
 116                         "enable_ccache"       : self.config.get_bool("builder", "use_ccache", True),
 117                         "enable_icecream"     : self.config.get_bool("builder", "use_icecream", False),
 118                         "sign_packages"       : False,
 119                         "buildroot_tmpfs"     : self.config.get_bool("builder", "use_tmpfs", False),
 120                         "private_network"     : self.config.get_bool("builder", "private_network", False),
 121                 }
 122
 123                 # Get ccache settings.
 124                 if self.settings.get("enable_ccache", False):
 125                         self.settings.update({
 126                                 "ccache_compress" : self.config.get_bool("ccache", "compress", True),
 127                         })
 128
 129                 # Try to get the configured host key. If it is available,
 130                 # we will automatically sign all packages with it.
 131                 if self.keyring.get_host_key(secret=True):
 132                         self.settings["sign_packages"] = True
 133
 134                 # Add settings from keyword arguments.
 135                 self.settings.update(kwargs)
 136
 137                 # Where do we put the result?
 138                 self.resultdir = os.path.join(self.pakfire.path, "result")
 139
 140                 # Open package.
 141                 # If we have a plain makefile, we first build a source package and go with that.
 142                 if filename:
 143                         # Open source package.
 144                         self.pkg = packages.SourcePackage(self.pakfire, None, filename)
 145                         assert self.pkg, filename
 146
 147                         # Log the package information.
 148                         self.log.info(_("Package information:"))
 149                         for line in self.pkg.dump(long=True).splitlines():
 150                                 self.log.info("  %s" % line)
 151                         self.log.info("")
 152
 153                         # Path where we extract the package and put all the source files.
 154                         self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name)
 155                 else:
 156                         # No package :(
 157                         self.pkg = None
 158
 159                 # Lock the buildroot
 160                 self._lock = None
 161
 162                 # Save the build time.
 163                 self.build_time = time.time()
 164
 165         def setup_signal_handlers(self):
 166                 pass
 167
 168         def start(self):
 169                 assert not self.pakfire.initialized, "Pakfire has already been initialized"
 170
 171                 # Unshare namepsace.
 172                 # If this fails because the kernel has no support for CLONE_NEWIPC or CLONE_NEWUTS,
 173                 # we try to fall back to just set CLONE_NEWNS.
 174                 try:
 175                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNS|_pakfire.SCHED_CLONE_NEWIPC|_pakfire.SCHED_CLONE_NEWUTS)
 176                 except RuntimeError, e:
 177                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNS)
 178
 179                 # Mount the directories.
 180                 try:
 181                         self._mountall()
 182                 except OSError, e:
 183                         if e.errno == 30: # Read-only FS
 184                                 raise BuildError, "Buildroot is read-only: %s" % self.pakfire.path
 185
 186                         # Raise all other errors.
 187                         raise
 188
 189                 # Lock the build environment.
 190                 self.lock()
 191
 192                 # Initialize pakfire instance.
 193                 self.pakfire.initialize()
 194
 195                 # Optionally enable private networking.
 196                 if self.settings.get("private_network", None):
 197                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNET)
 198
 199                 # Populate /dev.
 200                 self.populate_dev()
 201
 202                 # Setup domain name resolution in chroot.
 203                 self.setup_dns()
 204
 205                 # Extract all needed packages.
 206                 self.extract()
 207
 208         def stop(self):
 209                 if self.cgroup:
 210                         # Move the builder process out of the cgroup.
 211                         self.cgroup.migrate_task(self.cgroup.parent, os.getpid())
 212
 213                         # Kill all still running processes in the cgroup.
 214                         self.cgroup.kill_and_wait()
 215
 216                         # Remove cgroup and all parent cgroups if they are empty.
 217                         self.cgroup.destroy()
 218
 219                         parent = self.cgroup.parent
 220                         while parent:
 221                                 if not parent.is_empty(recursive=True):
 222                                         break
 223
 224                                 parent.destroy()
 225                                 parent = parent.parent
 226
 227                 else:
 228                         util.orphans_kill(self.path)
 229
 230                 # Shut down pakfire instance.
 231                 self.pakfire.destroy()
 232
 233                 # Unlock build environment.
 234                 self.unlock()
 235
 236                 # Umount the build environment.
 237                 self._umountall()
 238
 239                 # Remove all files.
 240                 self.destroy()
 241
 242         @property
 243         def config(self):
 244                 """
 245                         Proxy method for easy access to the configuration.
 246                 """
 247                 return self.pakfire.config
 248
 249         @property
 250         def distro(self):
 251                 """
 252                         Proxy method for easy access to the distribution.
 253                 """
 254                 return self.pakfire.distro
 255
 256         @property
 257         def path(self):
 258                 """
 259                         Proxy method for easy access to the path.
 260                 """
 261                 return self.pakfire.path
 262
 263         @property
 264         def arch(self):
 265                 """
 266                         Inherit architecture from distribution configuration.
 267                 """
 268                 return self.pakfire.distro.arch
 269
 270         @property
 271         def personality(self):
 272                 """
 273                         Gets the personality from the distribution configuration.
 274                 """
 275                 return self.pakfire.distro.personality
 276
 277         @property
 278         def info(self):
 279                 return {
 280                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 281                         "build_host" : socket.gethostname(),
 282                         "build_id"   : self.build_id,
 283                         "build_time" : self.build_time,
 284                 }
 285
 286         @property
 287         def keyring(self):
 288                 """
 289                         Shortcut to access the pakfire keyring.
 290                 """
 291                 return self.pakfire.keyring
 292
 293         def lock(self):
 294                 filename = os.path.join(self.path, ".lock")
 295
 296                 try:
 297                         self._lock = open(filename, "a+")
 298                 except IOError, e:
 299                         return 0
 300
 301                 try:
 302                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 303                 except IOError, e:
 304                         raise BuildRootLocked, "Buildroot is locked"
 305
 306                 return 1
 307
 308         def unlock(self):
 309                 if self._lock:
 310                         self._lock.close()
 311                         self._lock = None
 312
 313         def init_cgroup(self):
 314                 """
 315                         Initialize cgroup (if the system supports it).
 316                 """
 317                 if not cgroup.supported():
 318                         self.cgroup = None
 319                         return
 320
 321                 # Search for the cgroup this process is currently running in.
 322                 parent_cgroup = cgroup.find_by_pid(os.getpid())
 323                 if not parent_cgroup:
 324                         return
 325
 326                 # Create our own cgroup inside the parent cgroup.
 327                 self.cgroup = parent_cgroup.create_child_cgroup("pakfire/builder/%s" % self.build_id)
 328
 329                 # Attach the pakfire-builder process to the group.
 330                 self.cgroup.attach()
 331
 332         def init_logging(self, logfile):
 333                 if logfile:
 334                         self.log = log.getChild(self.build_id)
 335                         # Propage everything to the root logger that we will see something
 336                         # on the terminal.
 337                         self.log.propagate = 1
 338                         self.log.setLevel(logging.INFO)
 339
 340                         # Add the given logfile to the logger.
 341                         h = logging.FileHandler(logfile)
 342                         self.log.addHandler(h)
 343
 344                         # Format the log output for the file.
 345                         f = logger.BuildFormatter()
 346                         h.setFormatter(f)
 347                 else:
 348                         # If no logile was given, we use the root logger.
 349                         self.log = logging.getLogger("pakfire")
 350
 351         def copyin(self, file_out, file_in):
 352                 if file_in.startswith("/"):
 353                         file_in = file_in[1:]
 354
 355                 file_in = self.chrootPath(file_in)
 356
 357                 #if not os.path.exists(file_out):
 358                 #       return
 359
 360                 dir_in = os.path.dirname(file_in)
 361                 if not os.path.exists(dir_in):
 362                         os.makedirs(dir_in)
 363
 364                 self.log.debug("%s --> %s" % (file_out, file_in))
 365
 366                 shutil.copy2(file_out, file_in)
 367
 368         def copyout(self, file_in, file_out):
 369                 if file_in.startswith("/"):
 370                         file_in = file_in[1:]
 371
 372                 file_in = self.chrootPath(file_in)
 373
 374                 #if not os.path.exists(file_in):
 375                 #       return
 376
 377                 dir_out = os.path.dirname(file_out)
 378                 if not os.path.exists(dir_out):
 379                         os.makedirs(dir_out)
 380
 381                 self.log.debug("%s --> %s" % (file_in, file_out))
 382
 383                 shutil.copy2(file_in, file_out)
 384
 385         def copy_result(self, resultdir):
 386                 # XXX should use find_result_packages
 387
 388                 dir_in = self.chrootPath("result")
 389
 390                 for dir, subdirs, files in os.walk(dir_in):
 391                         basename = os.path.basename(dir)
 392                         dir = dir[len(self.chrootPath()):]
 393                         for file in files:
 394                                 file_in = os.path.join(dir, file)
 395
 396                                 file_out = os.path.join(
 397                                         resultdir,
 398                                         basename,
 399                                         file,
 400                                 )
 401
 402                                 self.copyout(file_in, file_out)
 403
 404         def find_result_packages(self):
 405                 ret = []
 406
 407                 for dir, subdirs, files in os.walk(self.resultdir):
 408                         for file in files:
 409                                 if not file.endswith(".%s" % PACKAGE_EXTENSION):
 410                                         continue
 411
 412                                 file = os.path.join(dir, file)
 413                                 ret.append(file)
 414
 415                 return ret
 416
 417         def extract(self, requires=None):
 418                 """
 419                         Gets a dependency set and extracts all packages
 420                         to the environment.
 421                 """
 422                 if not requires:
 423                         requires = []
 424
 425                 # Add neccessary build dependencies.
 426                 requires += BUILD_PACKAGES
 427
 428                 # If we have ccache enabled, we need to extract it
 429                 # to the build chroot.
 430                 if self.settings.get("enable_ccache"):
 431                         requires.append("ccache")
 432
 433                 # If we have icecream enabled, we need to extract it
 434                 # to the build chroot.
 435                 if self.settings.get("enable_icecream"):
 436                         requires.append("icecream")
 437
 438                 # Get build dependencies from source package.
 439                 if self.pkg:
 440                         for req in self.pkg.requires:
 441                                 requires.append(req)
 442
 443                 # Install all packages.
 444                 self.log.info(_("Install packages needed for build..."))
 445                 self.install(requires)
 446
 447                 # Copy the makefile and load source tarballs.
 448                 if self.pkg:
 449                         self.pkg.extract(_("Extracting"), prefix=self.build_dir)
 450
 451                         # Add an empty line at the end.
 452                         self.log.info("")
 453
 454         def install(self, requires, **kwargs):
 455                 """
 456                         Install everything that is required in requires.
 457                 """
 458                 # If we got nothing to do, we quit immediately.
 459                 if not requires:
 460                         return
 461
 462                 kwargs.update({
 463                         "interactive" : False,
 464                         "logger" : self.log,
 465                 })
 466
 467                 if not kwargs.has_key("allow_downgrade"):
 468                         kwargs["allow_downgrade"] = True
 469
 470                 # Install everything.
 471                 self.pakfire.install(requires, **kwargs)
 472
 473         def chrootPath(self, *args):
 474                 # Remove all leading slashes
 475                 _args = []
 476                 for arg in args:
 477                         if arg.startswith("/"):
 478                                 arg = arg[1:]
 479                         _args.append(arg)
 480                 args = _args
 481
 482                 ret = os.path.join(self.path, *args)
 483                 ret = ret.replace("//", "/")
 484
 485                 assert ret.startswith(self.path)
 486
 487                 return ret
 488
 489         def populate_dev(self):
 490                 nodes = [
 491                         "/dev/null",
 492                         "/dev/zero",
 493                         "/dev/full",
 494                         "/dev/random",
 495                         "/dev/urandom",
 496                         "/dev/tty",
 497                         "/dev/ptmx",
 498                         "/dev/kmsg",
 499                         "/dev/rtc0",
 500                         "/dev/console",
 501                 ]
 502
 503                 # If we need loop devices (which are optional) we create them here.
 504                 if self.settings["enable_loop_devices"]:
 505                         for i in range(0, 7):
 506                                 nodes.append("/dev/loop%d" % i)
 507
 508                 for node in nodes:
 509                         # Stat the original node of the host system and copy it to
 510                         # the build chroot.
 511                         try:
 512                                 node_stat = os.stat(node)
 513
 514                         # If it cannot be found, just go on.
 515                         except OSError:
 516                                 continue
 517
 518                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 519
 520                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 521                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 522                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 523                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 524
 525         def setup_dns(self):
 526                 """
 527                         Add DNS resolution facility to chroot environment by copying
 528                         /etc/resolv.conf and /etc/hosts.
 529                 """
 530                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 531                         self.copyin(i, i)
 532
 533         def _create_node(self, filename, mode, device):
 534                 self.log.debug("Create node: %s (%s)" % (filename, mode))
 535
 536                 filename = self.chrootPath(filename)
 537
 538                 # Create parent directory if it is missing.
 539                 dirname = os.path.dirname(filename)
 540                 if not os.path.exists(dirname):
 541                         os.makedirs(dirname)
 542
 543                 os.mknod(filename, mode, device)
 544
 545         def destroy(self):
 546                 self.log.debug("Destroying environment %s" % self.path)
 547
 548                 if os.path.exists(self.path):
 549                         util.rm(self.path)
 550
 551         def cleanup(self):
 552                 self.log.debug("Cleaning environemnt.")
 553
 554                 # Remove the build directory and buildroot.
 555                 dirs = (self.build_dir, self.chrootPath("result"),)
 556
 557                 for d in dirs:
 558                         if not os.path.exists(d):
 559                                 continue
 560
 561                         util.rm(d)
 562                         os.makedirs(d)
 563
 564         def _mountall(self):
 565                 self.log.debug("Mounting environment")
 566                 for src, dest, fs, options in self.mountpoints:
 567                         mountpoint = self.chrootPath(dest)
 568                         if options:
 569                                 options = "-o %s" % options
 570
 571                         # Eventually create mountpoint directory
 572                         if not os.path.exists(mountpoint):
 573                                 os.makedirs(mountpoint)
 574
 575                         self.execute_root("mount -n -t %s %s %s %s" % (fs, options, src, mountpoint), shell=True)
 576
 577         def _umountall(self):
 578                 self.log.debug("Umounting environment")
 579
 580                 mountpoints = []
 581                 for src, dest, fs, options in reversed(self.mountpoints):
 582                         dest = self.chrootPath(dest)
 583
 584                         if not dest in mountpoints:
 585                                 mountpoints.append(dest)
 586
 587                 while mountpoints:
 588                         for mp in mountpoints:
 589                                 try:
 590                                         self.execute_root("umount -n %s" % mp, shell=True)
 591                                 except ShellEnvironmentError:
 592                                         pass
 593
 594                                 if not os.path.ismount(mp):
 595                                         mountpoints.remove(mp)
 596
 597         @property
 598         def mountpoints(self):
 599                 mountpoints = []
 600
 601                 # Make root as a tmpfs if enabled.
 602                 if self.settings.get("buildroot_tmpfs"):
 603                         mountpoints += [
 604                                 ("pakfire_root", "/", "tmpfs", "defaults"),
 605                         ]
 606
 607                 mountpoints += [
 608                         # src, dest, fs, options
 609                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 610                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 611                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 612                         ("/sys",          "/sys",      "bind",  "bind"),
 613                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 614                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 615                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 616                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 617                         ("pakfire_tmpfs", "/tmp",      "tmpfs", "mode=755,nosuid,nodev"),
 618                 ]
 619
 620                 # If selinux is enabled.
 621                 if os.path.exists("/sys/fs/selinux"):
 622                         mountpoints += [
 623                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 624                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 625                         ]
 626
 627                 # If ccache support is requested, we bind mount the cache.
 628                 if self.settings.get("enable_ccache"):
 629                         # Create ccache cache directory if it does not exist.
 630                         if not os.path.exists(CCACHE_CACHE_DIR):
 631                                 os.makedirs(CCACHE_CACHE_DIR)
 632
 633                         mountpoints += [
 634                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 635                         ]
 636
 637                 return mountpoints
 638
 639         @property
 640         def environ(self):
 641                 env = MINIMAL_ENVIRONMENT.copy()
 642                 env.update({
 643                         # Add HOME manually, because it is occasionally not set
 644                         # and some builds get in trouble then.
 645                         "TERM" : os.environ.get("TERM", "vt100"),
 646
 647                         # Sanitize language.
 648                         "LANG" : os.environ.setdefault("LANG", "en_US.UTF-8"),
 649
 650                         # Set the container that we can detect, if we are inside a
 651                         # chroot.
 652                         "container" : "pakfire-builder",
 653                 })
 654
 655                 # Inherit environment from distro
 656                 env.update(self.pakfire.distro.environ)
 657
 658                 # ccache environment settings
 659                 if self.settings.get("enable_ccache", False):
 660                         compress = self.settings.get("ccache_compress", False)
 661                         if compress:
 662                                 env["CCACHE_COMPRESS"] = "1"
 663
 664                         # Let ccache create its temporary files in /tmp.
 665                         env["CCACHE_TEMPDIR"] = "/tmp"
 666
 667                 # Icecream environment settings
 668                 if self.settings.get("enable_icecream", False):
 669                         # Set the toolchain path
 670                         if self.settings.get("icecream_toolchain", None):
 671                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 672
 673                         # Set preferred host if configured.
 674                         if self.settings.get("icecream_preferred_host", None):
 675                                 env["ICECC_PREFERRED_HOST"] = \
 676                                         self.settings.get("icecream_preferred_host")
 677
 678                 # Fake UTS_MACHINE, when we cannot use the personality syscall and
 679                 # if the host architecture is not equal to the target architecture.
 680                 if not self.pakfire.distro.personality and \
 681                                 not system.native_arch == self.pakfire.distro.arch:
 682                         env.update({
 683                                 "LD_PRELOAD"  : "/usr/lib/libpakfire_preload.so",
 684                                 "UTS_MACHINE" : self.pakfire.distro.arch,
 685                         })
 686
 687                 return env
 688
 689         @property
 690         def installed_packages(self):
 691                 """
 692                         Returns an iterator over all installed packages in this build environment.
 693                 """
 694                 # Get the repository of all installed packages.
 695                 repo = self.pakfire.repos.get_repo("@system")
 696
 697                 # Return an iterator over the packages.
 698                 return iter(repo)
 699
 700         def write_config(self):
 701                 # Cleanup everything in /etc/pakfire.
 702                 util.rm(self.chrootPath(CONFIG_DIR))
 703
 704                 for i in (CONFIG_DIR, CONFIG_REPOS_DIR):
 705                         i = self.chrootPath(i)
 706                         if not os.path.exists(i):
 707                                 os.makedirs(i)
 708
 709                 # Write general.conf.
 710                 f = open(self.chrootPath(CONFIG_DIR, "general.conf"), "w")
 711                 f.close()
 712
 713                 # Write builder.conf.
 714                 f = open(self.chrootPath(CONFIG_DIR, "builder.conf"), "w")
 715                 f.write(self.distro.get_config())
 716                 f.close()
 717
 718                 # Create pakfire configuration files.
 719                 for repo in self.pakfire.repos:
 720                         conf = repo.get_config()
 721
 722                         if not conf:
 723                                 continue
 724
 725                         filename = self.chrootPath(CONFIG_REPOS_DIR, "%s.repo" % repo.name)
 726                         f = open(filename, "w")
 727                         f.write("\n".join(conf))
 728                         f.close()
 729
 730         @property
 731         def pkg_makefile(self):
 732                 return os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION))
 733
 734         def execute(self, command, logger=None, **kwargs):
 735                 """
 736                         Executes the given command in the build chroot.
 737                 """
 738                 # Environment variables
 739                 env = self.environ
 740
 741                 if kwargs.has_key("env"):
 742                         env.update(kwargs.pop("env"))
 743
 744                 self.log.debug("Environment:")
 745                 for k, v in sorted(env.items()):
 746                         self.log.debug("  %s=%s" % (k, v))
 747
 748                 # Make every shell to a login shell because we set a lot of
 749                 # environment things there.
 750                 command = ["bash", "--login", "-c", command]
 751
 752                 args = {
 753                         "chroot_path" : self.chrootPath(),
 754                         "cgroup"      : self.cgroup,
 755                         "env"         : env,
 756                         "logger"      : logger,
 757                         "personality" : self.personality,
 758                         "shell"       : False,
 759                 }
 760                 args.update(kwargs)
 761
 762                 # Run the shit.
 763                 shellenv = shell.ShellExecuteEnvironment(command, **args)
 764                 shellenv.execute()
 765
 766                 return shellenv
 767
 768         def execute_root(self, command, **kwargs):
 769                 """
 770                         Executes the given command outside the build chroot.
 771                 """
 772                 shellenv = shell.ShellExecuteEnvironment(command, **kwargs)
 773                 shellenv.execute()
 774
 775                 return shellenv
 776
 777         def build(self, install_test=True, prepare=False):
 778                 if not self.pkg:
 779                         raise BuildError, _("You cannot run a build when no package was given.")
 780
 781                 # Search for the package file in build_dir and raise BuildError if it is not present.
 782                 if not os.path.exists(self.pkg_makefile):
 783                         raise BuildError, _("Could not find makefile in build root: %s") % self.pkg_makefile
 784
 785                 # Write pakfire configuration into the chroot.
 786                 self.write_config()
 787
 788                 # Create the build command, that is executed in the chroot.
 789                 build_command = [
 790                         "/usr/lib/pakfire/builder",
 791                         "--offline",
 792                         "build",
 793                         "/%s" % os.path.relpath(self.pkg_makefile, self.chrootPath()),
 794                         "--arch", self.arch,
 795                         "--nodeps",
 796                         "--resultdir=/result",
 797                 ]
 798
 799                 # Check if only the preparation stage should be run.
 800                 if prepare:
 801                         build_command.append("--prepare")
 802
 803                 build_command = " ".join(build_command)
 804
 805                 try:
 806                         self.execute(build_command, logger=self.log)
 807
 808                         # Perform the install test after the actual build.
 809                         if install_test and not prepare:
 810                                 self.install_test()
 811
 812                 except ShellEnvironmentError:
 813                         self.log.error(_("Build failed"))
 814
 815                 except KeyboardInterrupt:
 816                         self.log.error(_("Build interrupted"))
 817
 818                         raise
 819
 820                 # Catch all other errors.
 821                 except:
 822                         self.log.error(_("Build failed."), exc_info=True)
 823
 824                 else:
 825                         # Don't sign packages in prepare mode.
 826                         if prepare:
 827                                 return
 828
 829                         # Sign all built packages with the host key (if available).
 830                         self.sign_packages()
 831
 832                         # Dump package information.
 833                         self.dump()
 834
 835                         return
 836
 837                 # End here in case of an error.
 838                 raise BuildError, _("The build command failed. See logfile for details.")
 839
 840         def install_test(self):
 841                 self.log.info(_("Running installation test..."))
 842
 843                 # Install all packages that were built.
 844                 self.install(self.find_result_packages(), allow_vendorchange=True,
 845                         allow_uninstall=True, signatures_mode="disabled")
 846
 847                 self.log.info(_("Installation test succeeded."))
 848                 self.log.info("")
 849
 850         def shell(self, args=[]):
 851                 if not util.cli_is_interactive():
 852                         self.log.warning("Cannot run shell on non-interactive console.")
 853                         return
 854
 855                 # Install all packages that are needed to run a shell.
 856                 self.install(SHELL_PACKAGES)
 857
 858                 # XXX need to set CFLAGS here
 859                 command = "/usr/sbin/chroot %s %s %s" % \
 860                         (self.chrootPath(), SHELL_SCRIPT, " ".join(args))
 861
 862                 # Add personality if we require one
 863                 if self.pakfire.distro.personality:
 864                         command = "%s %s" % (self.pakfire.distro.personality, command)
 865
 866                 for key, val in self.environ.items():
 867                         command = "%s=\"%s\" " % (key, val) + command
 868
 869                 # Empty the environment
 870                 command = "env -i - %s" % command
 871
 872                 self.log.debug("Shell command: %s" % command)
 873
 874                 shell = os.system(command)
 875                 return os.WEXITSTATUS(shell)
 876
 877         def sign_packages(self, keyfp=None):
 878                 # Do nothing if signing is not requested.
 879                 if not self.settings.get("sign_packages"):
 880                         return
 881
 882                 # Get key, that should be used for signing.
 883                 if not keyfp:
 884                         keyfp = self.keyring.get_host_key_id()
 885
 886                 # Find all files to process.
 887                 files = self.find_result_packages()
 888
 889                 # Create a progressbar.
 890                 print _("Signing packages...")
 891                 p = util.make_progress(keyfp, len(files))
 892                 i = 0
 893
 894                 for file in files:
 895                         # Update progressbar.
 896                         if p:
 897                                 i += 1
 898                                 p.update(i)
 899
 900                         # Open package file.
 901                         pkg = packages.open(self.pakfire, None, file)
 902
 903                         # Sign it.
 904                         pkg.sign(keyfp)
 905
 906                 # Close progressbar.
 907                 if p:
 908                         p.finish()
 909                         print "" # Print an empty line.
 910
 911         def dump(self):
 912                 pkgs = []
 913
 914                 for file in self.find_result_packages():
 915                         pkg = packages.open(self.pakfire, None, file)
 916                         pkgs.append(pkg)
 917
 918                 # If there are no packages, there is nothing to do.
 919                 if not pkgs:
 920                         return
 921
 922                 pkgs.sort()
 923
 924                 self.log.info(_("Dumping package information:"))
 925                 for pkg in pkgs:
 926                         dump = pkg.dump(long=True)
 927
 928                         for line in dump.splitlines():
 929                                 self.log.info("  %s" % line)
 930                         self.log.info("") # Empty line.
 931
 932
 933 class Builder(object):
 934         def __init__(self, pakfire, filename, resultdir, **kwargs):
 935                 self.pakfire = pakfire
 936
 937                 self.filename = filename
 938
 939                 self.resultdir = resultdir
 940
 941                 # Open package file.
 942                 self.pkg = packages.Makefile(self.pakfire, self.filename)
 943
 944                 self._environ = {
 945                         "LANG"             : "C",
 946                 }
 947
 948         @property
 949         def buildroot(self):
 950                 return self.pkg.buildroot
 951
 952         @property
 953         def distro(self):
 954                 return self.pakfire.distro
 955
 956         @property
 957         def environ(self):
 958                 environ = os.environ
 959
 960                 # Get all definitions from the package.
 961                 environ.update(self.pkg.exports)
 962
 963                 # Overwrite some definitions by default values.
 964                 environ.update(self._environ)
 965
 966                 return environ
 967
 968         def execute(self, command, logger=None, **kwargs):
 969                 if logger is None:
 970                         logger = logging.getLogger("pakfire")
 971
 972                 # Make every shell to a login shell because we set a lot of
 973                 # environment things there.
 974                 command = ["bash", "--login", "-c", command]
 975
 976                 args = {
 977                         "cwd"         : "/%s" % LOCAL_TMP_PATH,
 978                         "env"         : self.environ,
 979                         "logger"      : logger,
 980                         "personality" : self.distro.personality,
 981                         "shell"       : False,
 982                 }
 983                 args.update(kwargs)
 984
 985                 try:
 986                         shellenv = shell.ShellExecuteEnvironment(command, **args)
 987                         shellenv.execute()
 988
 989                 except ShellEnvironmentError:
 990                         logger.error("Command exited with an error: %s" % command)
 991                         raise
 992
 993                 return shellenv
 994
 995         def run_script(self, script, *args):
 996                 if not script.startswith("/"):
 997                         script = os.path.join(SCRIPT_DIR, script)
 998
 999                 assert os.path.exists(script), "Script we should run does not exist: %s" % script
1000
1001                 cmd = [script,]
1002                 for arg in args:
1003                         cmd.append(arg)
1004                 cmd = " ".join(cmd)
1005
1006                 # Returns the output of the command, but the output won't get
1007                 # logged.
1008                 exe = self.execute(cmd, record_output=True, log_output=False)
1009
1010                 # Return the output of the command.
1011                 if exe.exitcode == 0:
1012                         return exe.output
1013
1014         def create_icecream_toolchain(self):
1015                 try:
1016                         exe = self.execute(
1017                                 "icecc --build-native 2>/dev/null",
1018                                 record_output=True, record_stderr=False,
1019                                 log_output=False, log_errors=False,
1020                                 cwd="/tmp",
1021                         )
1022                 except ShellEnvironmentError:
1023                         return
1024
1025                 for line in exe.output.splitlines():
1026                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
1027                         if m:
1028                                 self._environ["ICECC_VERSION"] = "/tmp/%s" % m.group(1)
1029
1030         def create_buildscript(self, stage):
1031                 # Get buildscript from the package.
1032                 script = self.pkg.get_buildscript(stage)
1033
1034                 # Write script to an empty file.
1035                 f = tempfile.NamedTemporaryFile(mode="w", delete=False)
1036                 f.write("#!/bin/sh\n\n")
1037                 f.write("set -e\n")
1038                 f.write("set -x\n")
1039                 f.write("\n%s\n" % script)
1040                 f.write("exit 0\n")
1041                 f.close()
1042
1043                 # Make the script executable.
1044                 os.chmod(f.name, 700)
1045
1046                 return f.name
1047
1048         def build(self, stages=None):
1049                 # Create buildroot and remove all content if it was existant.
1050                 util.rm(self.buildroot)
1051                 os.makedirs(self.buildroot)
1052
1053                 # Build icecream toolchain if icecream is installed.
1054                 self.create_icecream_toolchain()
1055
1056                 # Process stages in order.
1057                 for stage in ("prepare", "build", "test", "install"):
1058                         # Skip unwanted stages.
1059                         if stages and not stage in stages:
1060                                 continue
1061
1062                         # Run stage.
1063                         self.build_stage(stage)
1064
1065                 # Stop if install stage has not been processed.
1066                 if stages and not "install" in stages:
1067                         return
1068
1069                 # Run post-build stuff.
1070                 self.post_compress_man_pages()
1071                 self.post_remove_static_libs()
1072                 self.post_extract_debuginfo()
1073
1074                 # Package the result.
1075                 # Make all these little package from the build environment.
1076                 log.info(_("Creating packages:"))
1077                 pkgs = []
1078                 for pkg in reversed(self.pkg.packages):
1079                         packager = packages.packager.BinaryPackager(self.pakfire, pkg,
1080                                 self, self.buildroot)
1081                         pkg = packager.run(self.resultdir)
1082                         pkgs.append(pkg)
1083                 log.info("")
1084
1085         def build_stage(self, stage):
1086                 # Get the buildscript for this stage.
1087                 buildscript = self.create_buildscript(stage)
1088
1089                 # Execute the buildscript of this stage.
1090                 log.info(_("Running stage %s:") % stage)
1091
1092                 try:
1093                         self.execute(buildscript)
1094
1095                 finally:
1096                         # Remove the buildscript.
1097                         if os.path.exists(buildscript):
1098                                 os.unlink(buildscript)
1099
1100         def post_remove_static_libs(self):
1101                 keep_libs = self.pkg.lexer.build.get_var("keep_libraries")
1102                 keep_libs = keep_libs.split()
1103
1104                 try:
1105                         self.execute("%s/remove-static-libs %s %s" % \
1106                                 (SCRIPT_DIR, self.buildroot, " ".join(keep_libs)))
1107                 except ShellEnvironmentError, e:
1108                         log.warning(_("Could not remove static libraries: %s") % e)
1109
1110         def post_compress_man_pages(self):
1111                 try:
1112                         self.execute("%s/compress-man-pages %s" % (SCRIPT_DIR, self.buildroot))
1113                 except ShellEnvironmentError, e:
1114                         log.warning(_("Compressing man pages did not complete successfully."))
1115
1116         def post_extract_debuginfo(self):
1117                 args = []
1118
1119                 # Check if we need to run with strict build-id.
1120                 strict_id = self.pkg.lexer.build.get_var("debuginfo_strict_build_id", "true")
1121                 if strict_id in ("true", "yes", "1"):
1122                         args.append("--strict-build-id")
1123
1124                 args.append("--buildroot=%s" % self.pkg.buildroot)
1125                 args.append("--sourcedir=%s" % self.pkg.sourcedir)
1126
1127                 # Get additional options to pass to script.
1128                 options = self.pkg.lexer.build.get_var("debuginfo_options", "")
1129                 args += options.split()
1130
1131                 try:
1132                         self.execute("%s/extract-debuginfo %s %s" % (SCRIPT_DIR, " ".join(args), self.pkg.buildroot))
1133                 except ShellEnvironmentError, e:
1134                         log.error(_("Extracting debuginfo did not complete with success. Aborting build."))
1135                         raise
1136
1137         def find_prerequires(self, scriptlet_file):
1138                 assert os.path.exists(scriptlet_file), "Scriptlet file does not exist: %s" % scriptlet_file
1139
1140                 res = self.run_script("find-prerequires", scriptlet_file)
1141                 prerequires = set(res.splitlines())
1142
1143                 return prerequires
1144
1145         def cleanup(self):
1146                 if os.path.exists(self.buildroot):
1147                         util.rm(self.buildroot)