python/pakfire/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import math
  25 import os
  26 import re
  27 import shutil
  28 import signal
  29 import socket
  30 import tempfile
  31 import time
  32 import uuid
  33
  34 import base
  35 import cgroup
  36 import logger
  37 import packages
  38 import packages.file
  39 import packages.packager
  40 import repository
  41 import shell
  42 import util
  43 import _pakfire
  44
  45 import logging
  46 log = logging.getLogger("pakfire")
  47
  48 from config import ConfigBuilder
  49 from system import system
  50 from constants import *
  51 from i18n import _
  52 from errors import BuildError, BuildRootLocked, Error
  53
  54
  55 BUILD_LOG_HEADER = """
  56  ____       _     __ _            _           _ _     _
  57 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  58 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  59 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  60 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  61
  62         Version : %(version)s
  63         Host    : %(hostname)s (%(host_arch)s)
  64         Time    : %(time)s
  65
  66 """
  67
  68 class BuildEnviron(object):
  69         # The version of the kernel this machine is running.
  70         kernel_version = os.uname()[2]
  71
  72         def __init__(self, pakfire, filename=None, distro_name=None, build_id=None, logfile=None, release_build=True, **kwargs):
  73                 self.pakfire = pakfire
  74
  75                 # Check if the given pakfire instance is of the correct type.
  76                 assert isinstance(self.pakfire, base.PakfireBuilder)
  77
  78                 # Check if this host can build the requested architecture.
  79                 if not system.host_supports_arch(self.arch):
  80                         raise BuildError, _("Cannot build for %s on this host.") % self.arch
  81
  82                 # Save the build id and generate one if no build id was provided.
  83                 if not build_id:
  84                         build_id = "%s" % uuid.uuid4()
  85
  86                 self.build_id = build_id
  87
  88                 # Setup the logging.
  89                 self.init_logging(logfile)
  90
  91                 # Initialize a cgroup (if supported).
  92                 self.init_cgroup()
  93
  94                 # This build is a release build?
  95                 self.release_build = release_build
  96
  97                 if self.release_build:
  98                         # Disable the local build repository in release mode.
  99                         self.pakfire.repos.disable_repo("build")
 100
 101                         # Log information about pakfire and some more information, when we
 102                         # are running in release mode.
 103                         logdata = {
 104                                 "host_arch"  : system.arch,
 105                                 "hostname"   : system.hostname,
 106                                 "time"       : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
 107                                 "version"    : "Pakfire %s" % PAKFIRE_VERSION,
 108                         }
 109
 110                         for line in BUILD_LOG_HEADER.splitlines():
 111                                 self.log.info(line % logdata)
 112
 113                 # Settings array.
 114                 self.settings = {
 115                         "enable_loop_devices" : self.config.get_bool("builder", "use_loop_devices", True),
 116                         "enable_ccache"       : self.config.get_bool("builder", "use_ccache", True),
 117                         "enable_icecream"     : self.config.get_bool("builder", "use_icecream", False),
 118                         "sign_packages"       : False,
 119                         "buildroot_tmpfs"     : self.config.get_bool("builder", "use_tmpfs", False),
 120                         "private_network"     : self.config.get_bool("builder", "private_network", False),
 121                 }
 122
 123                 # Get ccache settings.
 124                 if self.settings.get("enable_ccache", False):
 125                         self.settings.update({
 126                                 "ccache_compress" : self.config.get_bool("ccache", "compress", True),
 127                         })
 128
 129                 # Try to get the configured host key. If it is available,
 130                 # we will automatically sign all packages with it.
 131                 if self.keyring.get_host_key(secret=True):
 132                         self.settings["sign_packages"] = True
 133
 134                 # Add settings from keyword arguments.
 135                 self.settings.update(kwargs)
 136
 137                 # Where do we put the result?
 138                 self.resultdir = os.path.join(self.pakfire.path, "result")
 139
 140                 # Open package.
 141                 # If we have a plain makefile, we first build a source package and go with that.
 142                 if filename:
 143                         # Open source package.
 144                         self.pkg = packages.SourcePackage(self.pakfire, None, filename)
 145                         assert self.pkg, filename
 146
 147                         # Log the package information.
 148                         self.log.info(_("Package information:"))
 149                         for line in self.pkg.dump(long=True).splitlines():
 150                                 self.log.info("  %s" % line)
 151                         self.log.info("")
 152
 153                         # Path where we extract the package and put all the source files.
 154                         self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name)
 155                 else:
 156                         # No package :(
 157                         self.pkg = None
 158
 159                 # Lock the buildroot
 160                 self._lock = None
 161
 162                 # Save the build time.
 163                 self.build_time = time.time()
 164
 165         def setup_signal_handlers(self):
 166                 pass
 167
 168         def start(self):
 169                 assert not self.pakfire.initialized, "Pakfire has already been initialized"
 170
 171                 # Unshare namepsace.
 172                 # If this fails because the kernel has no support for CLONE_NEWIPC or CLONE_NEWUTS,
 173                 # we try to fall back to just set CLONE_NEWNS.
 174                 try:
 175                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNS|_pakfire.SCHED_CLONE_NEWIPC|_pakfire.SCHED_CLONE_NEWUTS)
 176                 except RuntimeError, e:
 177                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNS)
 178
 179                 # Mount the directories.
 180                 try:
 181                         self._mountall()
 182                 except OSError, e:
 183                         if e.errno == 30: # Read-only FS
 184                                 raise BuildError, "Buildroot is read-only: %s" % self.pakfire.path
 185
 186                         # Raise all other errors.
 187                         raise
 188
 189                 # Lock the build environment.
 190                 self.lock()
 191
 192                 # Initialize pakfire instance.
 193                 self.pakfire.initialize()
 194
 195                 # Optionally enable private networking.
 196                 if self.settings.get("private_network", None):
 197                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNET)
 198
 199                 # Populate /dev.
 200                 self.populate_dev()
 201
 202                 # Setup domain name resolution in chroot.
 203                 self.setup_dns()
 204
 205                 # Extract all needed packages.
 206                 self.extract()
 207
 208         def stop(self):
 209                 if self.cgroup:
 210                         # Move the builder process out of the cgroup.
 211                         self.cgroup.migrate_task(self.cgroup.parent, os.getpid())
 212
 213                         # Kill all still running processes in the cgroup.
 214                         self.cgroup.kill_and_wait()
 215
 216                         # Remove cgroup and all parent cgroups if they are empty.
 217                         self.cgroup.destroy()
 218
 219                         parent = self.cgroup.parent
 220                         while parent:
 221                                 if not parent.is_empty(recursive=True):
 222                                         break
 223
 224                                 parent.destroy()
 225                                 parent = parent.parent
 226
 227                 else:
 228                         util.orphans_kill(self.path)
 229
 230                 # Shut down pakfire instance.
 231                 self.pakfire.destroy()
 232
 233                 # Unlock build environment.
 234                 self.unlock()
 235
 236                 # Umount the build environment.
 237                 self._umountall()
 238
 239                 # Remove all files.
 240                 self.destroy()
 241
 242         @property
 243         def config(self):
 244                 """
 245                         Proxy method for easy access to the configuration.
 246                 """
 247                 return self.pakfire.config
 248
 249         @property
 250         def distro(self):
 251                 """
 252                         Proxy method for easy access to the distribution.
 253                 """
 254                 return self.pakfire.distro
 255
 256         @property
 257         def path(self):
 258                 """
 259                         Proxy method for easy access to the path.
 260                 """
 261                 return self.pakfire.path
 262
 263         @property
 264         def arch(self):
 265                 """
 266                         Inherit architecture from distribution configuration.
 267                 """
 268                 return self.pakfire.distro.arch
 269
 270         @property
 271         def personality(self):
 272                 """
 273                         Gets the personality from the distribution configuration.
 274                 """
 275                 return self.pakfire.distro.personality
 276
 277         @property
 278         def info(self):
 279                 return {
 280                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 281                         "build_host" : socket.gethostname(),
 282                         "build_id"   : self.build_id,
 283                         "build_time" : self.build_time,
 284                 }
 285
 286         @property
 287         def keyring(self):
 288                 """
 289                         Shortcut to access the pakfire keyring.
 290                 """
 291                 return self.pakfire.keyring
 292
 293         def lock(self):
 294                 filename = os.path.join(self.path, ".lock")
 295
 296                 try:
 297                         self._lock = open(filename, "a+")
 298                 except IOError, e:
 299                         return 0
 300
 301                 try:
 302                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 303                 except IOError, e:
 304                         raise BuildRootLocked, "Buildroot is locked"
 305
 306                 return 1
 307
 308         def unlock(self):
 309                 if self._lock:
 310                         self._lock.close()
 311                         self._lock = None
 312
 313         def init_cgroup(self):
 314                 """
 315                         Initialize cgroup (if the system supports it).
 316                 """
 317                 if not cgroup.supported():
 318                         self.cgroup = None
 319                         return
 320
 321                 # Search for the cgroup this process is currently running in.
 322                 parent_cgroup = cgroup.find_by_pid(os.getpid())
 323                 if not parent_cgroup:
 324                         return
 325
 326                 # Create our own cgroup inside the parent cgroup.
 327                 self.cgroup = parent_cgroup.create_child_cgroup("pakfire/builder/%s" % self.build_id)
 328
 329                 # Attach the pakfire-builder process to the group.
 330                 self.cgroup.attach()
 331
 332         def init_logging(self, logfile):
 333                 if logfile:
 334                         self.log = log.getChild(self.build_id)
 335                         # Propage everything to the root logger that we will see something
 336                         # on the terminal.
 337                         self.log.propagate = 1
 338                         self.log.setLevel(logging.INFO)
 339
 340                         # Add the given logfile to the logger.
 341                         h = logging.FileHandler(logfile)
 342                         self.log.addHandler(h)
 343
 344                         # Format the log output for the file.
 345                         f = logger.BuildFormatter()
 346                         h.setFormatter(f)
 347                 else:
 348                         # If no logile was given, we use the root logger.
 349                         self.log = logging.getLogger("pakfire")
 350
 351         def copyin(self, file_out, file_in):
 352                 if file_in.startswith("/"):
 353                         file_in = file_in[1:]
 354
 355                 file_in = self.chrootPath(file_in)
 356
 357                 #if not os.path.exists(file_out):
 358                 #       return
 359
 360                 dir_in = os.path.dirname(file_in)
 361                 if not os.path.exists(dir_in):
 362                         os.makedirs(dir_in)
 363
 364                 self.log.debug("%s --> %s" % (file_out, file_in))
 365
 366                 shutil.copy2(file_out, file_in)
 367
 368         def copyout(self, file_in, file_out):
 369                 if file_in.startswith("/"):
 370                         file_in = file_in[1:]
 371
 372                 file_in = self.chrootPath(file_in)
 373
 374                 #if not os.path.exists(file_in):
 375                 #       return
 376
 377                 dir_out = os.path.dirname(file_out)
 378                 if not os.path.exists(dir_out):
 379                         os.makedirs(dir_out)
 380
 381                 self.log.debug("%s --> %s" % (file_in, file_out))
 382
 383                 shutil.copy2(file_in, file_out)
 384
 385         def copy_result(self, resultdir):
 386                 # XXX should use find_result_packages
 387
 388                 dir_in = self.chrootPath("result")
 389
 390                 for dir, subdirs, files in os.walk(dir_in):
 391                         basename = os.path.basename(dir)
 392                         dir = dir[len(self.chrootPath()):]
 393                         for file in files:
 394                                 file_in = os.path.join(dir, file)
 395
 396                                 file_out = os.path.join(
 397                                         resultdir,
 398                                         basename,
 399                                         file,
 400                                 )
 401
 402                                 self.copyout(file_in, file_out)
 403
 404         def find_result_packages(self):
 405                 ret = []
 406
 407                 for dir, subdirs, files in os.walk(self.resultdir):
 408                         for file in files:
 409                                 if not file.endswith(".%s" % PACKAGE_EXTENSION):
 410                                         continue
 411
 412                                 file = os.path.join(dir, file)
 413                                 ret.append(file)
 414
 415                 return ret
 416
 417         def extract(self, requires=None):
 418                 """
 419                         Gets a dependency set and extracts all packages
 420                         to the environment.
 421                 """
 422                 if not requires:
 423                         requires = []
 424
 425                 # Add neccessary build dependencies.
 426                 requires += BUILD_PACKAGES
 427
 428                 # If we have ccache enabled, we need to extract it
 429                 # to the build chroot.
 430                 if self.settings.get("enable_ccache"):
 431                         requires.append("ccache")
 432
 433                 # If we have icecream enabled, we need to extract it
 434                 # to the build chroot.
 435                 if self.settings.get("enable_icecream"):
 436                         requires.append("icecream")
 437
 438                 # Get build dependencies from source package.
 439                 if self.pkg:
 440                         for req in self.pkg.requires:
 441                                 requires.append(req)
 442
 443                 # Install all packages.
 444                 self.log.info(_("Install packages needed for build..."))
 445                 self.install(requires)
 446
 447                 # Copy the makefile and load source tarballs.
 448                 if self.pkg:
 449                         self.pkg.extract(_("Extracting"), prefix=self.build_dir)
 450
 451                         # Add an empty line at the end.
 452                         self.log.info("")
 453
 454         def install(self, requires, **kwargs):
 455                 """
 456                         Install everything that is required in requires.
 457                 """
 458                 # If we got nothing to do, we quit immediately.
 459                 if not requires:
 460                         return
 461
 462                 kwargs.update({
 463                         "interactive" : False,
 464                         "logger" : self.log,
 465                 })
 466
 467                 if not kwargs.has_key("allow_downgrade"):
 468                         kwargs["allow_downgrade"] = True
 469
 470                 # Install everything.
 471                 self.pakfire.install(requires, **kwargs)
 472
 473         def chrootPath(self, *args):
 474                 # Remove all leading slashes
 475                 _args = []
 476                 for arg in args:
 477                         if arg.startswith("/"):
 478                                 arg = arg[1:]
 479                         _args.append(arg)
 480                 args = _args
 481
 482                 ret = os.path.join(self.path, *args)
 483                 ret = ret.replace("//", "/")
 484
 485                 assert ret.startswith(self.path)
 486
 487                 return ret
 488
 489         def populate_dev(self):
 490                 nodes = [
 491                         "/dev/null",
 492                         "/dev/zero",
 493                         "/dev/full",
 494                         "/dev/random",
 495                         "/dev/urandom",
 496                         "/dev/tty",
 497                         "/dev/ptmx",
 498                         "/dev/kmsg",
 499                         "/dev/rtc0",
 500                         "/dev/console",
 501                 ]
 502
 503                 # If we need loop devices (which are optional) we create them here.
 504                 if self.settings["enable_loop_devices"]:
 505                         for i in range(0, 7):
 506                                 nodes.append("/dev/loop%d" % i)
 507
 508                 for node in nodes:
 509                         # Stat the original node of the host system and copy it to
 510                         # the build chroot.
 511                         try:
 512                                 node_stat = os.stat(node)
 513
 514                         # If it cannot be found, just go on.
 515                         except OSError:
 516                                 continue
 517
 518                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 519
 520                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 521                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 522                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 523                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 524
 525         def setup_dns(self):
 526                 """
 527                         Add DNS resolution facility to chroot environment by copying
 528                         /etc/resolv.conf and /etc/hosts.
 529                 """
 530                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 531                         self.copyin(i, i)
 532
 533         def _create_node(self, filename, mode, device):
 534                 self.log.debug("Create node: %s (%s)" % (filename, mode))
 535
 536                 filename = self.chrootPath(filename)
 537
 538                 # Create parent directory if it is missing.
 539                 dirname = os.path.dirname(filename)
 540                 if not os.path.exists(dirname):
 541                         os.makedirs(dirname)
 542
 543                 os.mknod(filename, mode, device)
 544
 545         def destroy(self):
 546                 self.log.debug("Destroying environment %s" % self.path)
 547
 548                 if os.path.exists(self.path):
 549                         util.rm(self.path)
 550
 551         def cleanup(self):
 552                 self.log.debug("Cleaning environemnt.")
 553
 554                 # Remove the build directory and buildroot.
 555                 dirs = (self.build_dir, self.chrootPath("result"),)
 556
 557                 for d in dirs:
 558                         if not os.path.exists(d):
 559                                 continue
 560
 561                         util.rm(d)
 562                         os.makedirs(d)
 563
 564         def _mountall(self):
 565                 self.log.debug("Mounting environment")
 566                 for src, dest, fs, options in self.mountpoints:
 567                         mountpoint = self.chrootPath(dest)
 568                         if options:
 569                                 options = "-o %s" % options
 570
 571                         # Eventually create mountpoint directory
 572                         if not os.path.exists(mountpoint):
 573                                 os.makedirs(mountpoint)
 574
 575                         self.execute_root("mount -n -t %s %s %s %s" % (fs, options, src, mountpoint), shell=True)
 576
 577         def _umountall(self):
 578                 self.log.debug("Umounting environment")
 579
 580                 mountpoints = []
 581                 for src, dest, fs, options in reversed(self.mountpoints):
 582                         dest = self.chrootPath(dest)
 583
 584                         if not dest in mountpoints:
 585                                 mountpoints.append(dest)
 586
 587                 while mountpoints:
 588                         for mp in mountpoints:
 589                                 try:
 590                                         self.execute_root("umount -n %s" % mp, shell=True)
 591                                 except ShellEnvironmentError:
 592                                         pass
 593
 594                                 if not os.path.ismount(mp):
 595                                         mountpoints.remove(mp)
 596
 597         @property
 598         def mountpoints(self):
 599                 mountpoints = []
 600
 601                 # Make root as a tmpfs if enabled.
 602                 if self.settings.get("buildroot_tmpfs"):
 603                         mountpoints += [
 604                                 ("pakfire_root", "/", "tmpfs", "defaults"),
 605                         ]
 606
 607                 mountpoints += [
 608                         # src, dest, fs, options
 609                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 610                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 611                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 612                         ("/sys",          "/sys",      "bind",  "bind"),
 613                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 614                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 615                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 616                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 617                         ("pakfire_tmpfs", "/tmp",      "tmpfs", "mode=755,nosuid,nodev"),
 618                 ]
 619
 620                 # If selinux is enabled.
 621                 if os.path.exists("/sys/fs/selinux"):
 622                         mountpoints += [
 623                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 624                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 625                         ]
 626
 627                 # If ccache support is requested, we bind mount the cache.
 628                 if self.settings.get("enable_ccache"):
 629                         # Create ccache cache directory if it does not exist.
 630                         if not os.path.exists(CCACHE_CACHE_DIR):
 631                                 os.makedirs(CCACHE_CACHE_DIR)
 632
 633                         mountpoints += [
 634                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 635                         ]
 636
 637                 return mountpoints
 638
 639         @property
 640         def environ(self):
 641                 env = {
 642                         # Add HOME manually, because it is occasionally not set
 643                         # and some builds get in trouble then.
 644                         "PATH" : "/usr/bin:/bin:/usr/sbin:/sbin",
 645                         "HOME" : "/root",
 646                         "TERM" : os.environ.get("TERM", "vt100"),
 647                         "PS1"  : "\u:\w\$ ",
 648
 649                         # Sanitize language.
 650                         "LANG" : os.environ.setdefault("LANG", "en_US.UTF-8"),
 651
 652                         # Set the container that we can detect, if we are inside a
 653                         # chroot.
 654                         "container" : "pakfire-builder",
 655                 }
 656
 657                 # Inherit environment from distro
 658                 env.update(self.pakfire.distro.environ)
 659
 660                 # ccache environment settings
 661                 if self.settings.get("enable_ccache", False):
 662                         compress = self.settings.get("ccache_compress", False)
 663                         if compress:
 664                                 env["CCACHE_COMPRESS"] = "1"
 665
 666                         # Let ccache create its temporary files in /tmp.
 667                         env["CCACHE_TEMPDIR"] = "/tmp"
 668
 669                 # Icecream environment settings
 670                 if self.settings.get("enable_icecream", False):
 671                         # Set the toolchain path
 672                         if self.settings.get("icecream_toolchain", None):
 673                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 674
 675                         # Set preferred host if configured.
 676                         if self.settings.get("icecream_preferred_host", None):
 677                                 env["ICECC_PREFERRED_HOST"] = \
 678                                         self.settings.get("icecream_preferred_host")
 679
 680                 # Fake UTS_MACHINE, when we cannot use the personality syscall and
 681                 # if the host architecture is not equal to the target architecture.
 682                 if not self.pakfire.distro.personality and \
 683                                 not system.native_arch == self.pakfire.distro.arch:
 684                         env.update({
 685                                 "LD_PRELOAD"  : "/usr/lib/libpakfire_preload.so",
 686                                 "UTS_MACHINE" : self.pakfire.distro.arch,
 687                         })
 688
 689                 return env
 690
 691         @property
 692         def installed_packages(self):
 693                 """
 694                         Returns an iterator over all installed packages in this build environment.
 695                 """
 696                 # Get the repository of all installed packages.
 697                 repo = self.pakfire.repos.get_repo("@system")
 698
 699                 # Return an iterator over the packages.
 700                 return iter(repo)
 701
 702         def write_config(self):
 703                 # Cleanup everything in /etc/pakfire.
 704                 util.rm(self.chrootPath(CONFIG_DIR))
 705
 706                 for i in (CONFIG_DIR, CONFIG_REPOS_DIR):
 707                         i = self.chrootPath(i)
 708                         if not os.path.exists(i):
 709                                 os.makedirs(i)
 710
 711                 # Write general.conf.
 712                 f = open(self.chrootPath(CONFIG_DIR, "general.conf"), "w")
 713                 f.close()
 714
 715                 # Write builder.conf.
 716                 f = open(self.chrootPath(CONFIG_DIR, "builder.conf"), "w")
 717                 f.write(self.distro.get_config())
 718                 f.close()
 719
 720                 # Create pakfire configuration files.
 721                 for repo in self.pakfire.repos:
 722                         conf = repo.get_config()
 723
 724                         if not conf:
 725                                 continue
 726
 727                         filename = self.chrootPath(CONFIG_REPOS_DIR, "%s.repo" % repo.name)
 728                         f = open(filename, "w")
 729                         f.write("\n".join(conf))
 730                         f.close()
 731
 732         @property
 733         def pkg_makefile(self):
 734                 return os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION))
 735
 736         def execute(self, command, logger=None, **kwargs):
 737                 """
 738                         Executes the given command in the build chroot.
 739                 """
 740                 # Environment variables
 741                 env = self.environ
 742
 743                 if kwargs.has_key("env"):
 744                         env.update(kwargs.pop("env"))
 745
 746                 self.log.debug("Environment:")
 747                 for k, v in sorted(env.items()):
 748                         self.log.debug("  %s=%s" % (k, v))
 749
 750                 # Make every shell to a login shell because we set a lot of
 751                 # environment things there.
 752                 command = ["bash", "--login", "-c", command]
 753
 754                 args = {
 755                         "chroot_path" : self.chrootPath(),
 756                         "cgroup"      : self.cgroup,
 757                         "env"         : env,
 758                         "logger"      : logger,
 759                         "personality" : self.personality,
 760                         "shell"       : False,
 761                 }
 762                 args.update(kwargs)
 763
 764                 # Run the shit.
 765                 shellenv = shell.ShellExecuteEnvironment(command, **args)
 766                 shellenv.execute()
 767
 768                 return shellenv
 769
 770         def execute_root(self, command, **kwargs):
 771                 """
 772                         Executes the given command outside the build chroot.
 773                 """
 774                 shellenv = shell.ShellExecuteEnvironment(command, **kwargs)
 775                 shellenv.execute()
 776
 777                 return shellenv
 778
 779         def build(self, install_test=True, prepare=False):
 780                 if not self.pkg:
 781                         raise BuildError, _("You cannot run a build when no package was given.")
 782
 783                 # Search for the package file in build_dir and raise BuildError if it is not present.
 784                 if not os.path.exists(self.pkg_makefile):
 785                         raise BuildError, _("Could not find makefile in build root: %s") % self.pkg_makefile
 786
 787                 # Write pakfire configuration into the chroot.
 788                 self.write_config()
 789
 790                 # Create the build command, that is executed in the chroot.
 791                 build_command = [
 792                         "/usr/lib/pakfire/builder",
 793                         "--offline",
 794                         "build",
 795                         "/%s" % os.path.relpath(self.pkg_makefile, self.chrootPath()),
 796                         "--arch", self.arch,
 797                         "--nodeps",
 798                         "--resultdir=/result",
 799                 ]
 800
 801                 # Check if only the preparation stage should be run.
 802                 if prepare:
 803                         build_command.append("--prepare")
 804
 805                 build_command = " ".join(build_command)
 806
 807                 try:
 808                         self.execute(build_command, logger=self.log)
 809
 810                         # Perform the install test after the actual build.
 811                         if install_test and not prepare:
 812                                 self.install_test()
 813
 814                 except ShellEnvironmentError:
 815                         self.log.error(_("Build failed"))
 816
 817                 except KeyboardInterrupt:
 818                         self.log.error(_("Build interrupted"))
 819
 820                         raise
 821
 822                 # Catch all other errors.
 823                 except:
 824                         self.log.error(_("Build failed."), exc_info=True)
 825
 826                 else:
 827                         # Don't sign packages in prepare mode.
 828                         if prepare:
 829                                 return
 830
 831                         # Sign all built packages with the host key (if available).
 832                         self.sign_packages()
 833
 834                         # Dump package information.
 835                         self.dump()
 836
 837                         return
 838
 839                 # End here in case of an error.
 840                 raise BuildError, _("The build command failed. See logfile for details.")
 841
 842         def install_test(self):
 843                 self.log.info(_("Running installation test..."))
 844
 845                 # Install all packages that were built.
 846                 self.install(self.find_result_packages(), allow_vendorchange=True,
 847                         allow_uninstall=True, signatures_mode="disabled")
 848
 849                 self.log.info(_("Installation test succeeded."))
 850                 self.log.info("")
 851
 852         def shell(self, args=[]):
 853                 if not util.cli_is_interactive():
 854                         self.log.warning("Cannot run shell on non-interactive console.")
 855                         return
 856
 857                 # Install all packages that are needed to run a shell.
 858                 self.install(SHELL_PACKAGES)
 859
 860                 # XXX need to set CFLAGS here
 861                 command = "/usr/sbin/chroot %s %s %s" % \
 862                         (self.chrootPath(), SHELL_SCRIPT, " ".join(args))
 863
 864                 # Add personality if we require one
 865                 if self.pakfire.distro.personality:
 866                         command = "%s %s" % (self.pakfire.distro.personality, command)
 867
 868                 for key, val in self.environ.items():
 869                         command = "%s=\"%s\" " % (key, val) + command
 870
 871                 # Empty the environment
 872                 command = "env -i - %s" % command
 873
 874                 self.log.debug("Shell command: %s" % command)
 875
 876                 shell = os.system(command)
 877                 return os.WEXITSTATUS(shell)
 878
 879         def sign_packages(self, keyfp=None):
 880                 # Do nothing if signing is not requested.
 881                 if not self.settings.get("sign_packages"):
 882                         return
 883
 884                 # Get key, that should be used for signing.
 885                 if not keyfp:
 886                         keyfp = self.keyring.get_host_key_id()
 887
 888                 # Find all files to process.
 889                 files = self.find_result_packages()
 890
 891                 # Create a progressbar.
 892                 print _("Signing packages...")
 893                 p = util.make_progress(keyfp, len(files))
 894                 i = 0
 895
 896                 for file in files:
 897                         # Update progressbar.
 898                         if p:
 899                                 i += 1
 900                                 p.update(i)
 901
 902                         # Open package file.
 903                         pkg = packages.open(self.pakfire, None, file)
 904
 905                         # Sign it.
 906                         pkg.sign(keyfp)
 907
 908                 # Close progressbar.
 909                 if p:
 910                         p.finish()
 911                         print "" # Print an empty line.
 912
 913         def dump(self):
 914                 pkgs = []
 915
 916                 for file in self.find_result_packages():
 917                         pkg = packages.open(self.pakfire, None, file)
 918                         pkgs.append(pkg)
 919
 920                 # If there are no packages, there is nothing to do.
 921                 if not pkgs:
 922                         return
 923
 924                 pkgs.sort()
 925
 926                 self.log.info(_("Dumping package information:"))
 927                 for pkg in pkgs:
 928                         dump = pkg.dump(long=True)
 929
 930                         for line in dump.splitlines():
 931                                 self.log.info("  %s" % line)
 932                         self.log.info("") # Empty line.
 933
 934
 935 class Builder(object):
 936         def __init__(self, pakfire, filename, resultdir, **kwargs):
 937                 self.pakfire = pakfire
 938
 939                 self.filename = filename
 940
 941                 self.resultdir = resultdir
 942
 943                 # Open package file.
 944                 self.pkg = packages.Makefile(self.pakfire, self.filename)
 945
 946                 self._environ = {
 947                         "LANG"             : "C",
 948                 }
 949
 950         @property
 951         def buildroot(self):
 952                 return self.pkg.buildroot
 953
 954         @property
 955         def distro(self):
 956                 return self.pakfire.distro
 957
 958         @property
 959         def environ(self):
 960                 environ = os.environ
 961
 962                 # Get all definitions from the package.
 963                 environ.update(self.pkg.exports)
 964
 965                 # Overwrite some definitions by default values.
 966                 environ.update(self._environ)
 967
 968                 return environ
 969
 970         def execute(self, command, logger=None, **kwargs):
 971                 if logger is None:
 972                         logger = logging.getLogger("pakfire")
 973
 974                 # Make every shell to a login shell because we set a lot of
 975                 # environment things there.
 976                 command = ["bash", "--login", "-c", command]
 977
 978                 args = {
 979                         "cwd"         : "/%s" % LOCAL_TMP_PATH,
 980                         "env"         : self.environ,
 981                         "logger"      : logger,
 982                         "personality" : self.distro.personality,
 983                         "shell"       : False,
 984                 }
 985                 args.update(kwargs)
 986
 987                 try:
 988                         shellenv = shell.ShellExecuteEnvironment(command, **args)
 989                         shellenv.execute()
 990
 991                 except ShellEnvironmentError:
 992                         logger.error("Command exited with an error: %s" % command)
 993                         raise
 994
 995                 return shellenv
 996
 997         def run_script(self, script, *args):
 998                 if not script.startswith("/"):
 999                         script = os.path.join(SCRIPT_DIR, script)
1000
1001                 assert os.path.exists(script), "Script we should run does not exist: %s" % script
1002
1003                 cmd = [script,]
1004                 for arg in args:
1005                         cmd.append(arg)
1006                 cmd = " ".join(cmd)
1007
1008                 # Returns the output of the command, but the output won't get
1009                 # logged.
1010                 exe = self.execute(cmd, record_output=True, log_output=False)
1011
1012                 # Return the output of the command.
1013                 if exe.exitcode == 0:
1014                         return exe.output
1015
1016         def create_icecream_toolchain(self):
1017                 try:
1018                         exe = self.execute(
1019                                 "icecc --build-native 2>/dev/null",
1020                                 record_output=True, record_stderr=False,
1021                                 log_output=False, log_errors=False,
1022                                 cwd="/tmp",
1023                         )
1024                 except ShellEnvironmentError:
1025                         return
1026
1027                 for line in exe.output.splitlines():
1028                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
1029                         if m:
1030                                 self._environ["ICECC_VERSION"] = "/tmp/%s" % m.group(1)
1031
1032         def create_buildscript(self, stage):
1033                 # Get buildscript from the package.
1034                 script = self.pkg.get_buildscript(stage)
1035
1036                 # Write script to an empty file.
1037                 f = tempfile.NamedTemporaryFile(mode="w", delete=False)
1038                 f.write("#!/bin/sh\n\n")
1039                 f.write("set -e\n")
1040                 f.write("set -x\n")
1041                 f.write("\n%s\n" % script)
1042                 f.write("exit 0\n")
1043                 f.close()
1044
1045                 # Make the script executable.
1046                 os.chmod(f.name, 700)
1047
1048                 return f.name
1049
1050         def build(self, stages=None):
1051                 # Create buildroot and remove all content if it was existant.
1052                 util.rm(self.buildroot)
1053                 os.makedirs(self.buildroot)
1054
1055                 # Build icecream toolchain if icecream is installed.
1056                 self.create_icecream_toolchain()
1057
1058                 # Process stages in order.
1059                 for stage in ("prepare", "build", "test", "install"):
1060                         # Skip unwanted stages.
1061                         if stages and not stage in stages:
1062                                 continue
1063
1064                         # Run stage.
1065                         self.build_stage(stage)
1066
1067                 # Stop if install stage has not been processed.
1068                 if stages and not "install" in stages:
1069                         return
1070
1071                 # Run post-build stuff.
1072                 self.post_compress_man_pages()
1073                 self.post_remove_static_libs()
1074                 self.post_extract_debuginfo()
1075
1076                 # Package the result.
1077                 # Make all these little package from the build environment.
1078                 log.info(_("Creating packages:"))
1079                 pkgs = []
1080                 for pkg in reversed(self.pkg.packages):
1081                         packager = packages.packager.BinaryPackager(self.pakfire, pkg,
1082                                 self, self.buildroot)
1083                         pkg = packager.run(self.resultdir)
1084                         pkgs.append(pkg)
1085                 log.info("")
1086
1087         def build_stage(self, stage):
1088                 # Get the buildscript for this stage.
1089                 buildscript = self.create_buildscript(stage)
1090
1091                 # Execute the buildscript of this stage.
1092                 log.info(_("Running stage %s:") % stage)
1093
1094                 try:
1095                         self.execute(buildscript)
1096
1097                 finally:
1098                         # Remove the buildscript.
1099                         if os.path.exists(buildscript):
1100                                 os.unlink(buildscript)
1101
1102         def post_remove_static_libs(self):
1103                 keep_libs = self.pkg.lexer.build.get_var("keep_libraries")
1104                 keep_libs = keep_libs.split()
1105
1106                 try:
1107                         self.execute("%s/remove-static-libs %s %s" % \
1108                                 (SCRIPT_DIR, self.buildroot, " ".join(keep_libs)))
1109                 except ShellEnvironmentError, e:
1110                         log.warning(_("Could not remove static libraries: %s") % e)
1111
1112         def post_compress_man_pages(self):
1113                 try:
1114                         self.execute("%s/compress-man-pages %s" % (SCRIPT_DIR, self.buildroot))
1115                 except ShellEnvironmentError, e:
1116                         log.warning(_("Compressing man pages did not complete successfully."))
1117
1118         def post_extract_debuginfo(self):
1119                 args = []
1120
1121                 # Check if we need to run with strict build-id.
1122                 strict_id = self.pkg.lexer.build.get_var("debuginfo_strict_build_id", "true")
1123                 if strict_id in ("true", "yes", "1"):
1124                         args.append("--strict-build-id")
1125
1126                 args.append("--buildroot=%s" % self.pkg.buildroot)
1127                 args.append("--sourcedir=%s" % self.pkg.sourcedir)
1128
1129                 # Get additional options to pass to script.
1130                 options = self.pkg.lexer.build.get_var("debuginfo_options", "")
1131                 args += options.split()
1132
1133                 try:
1134                         self.execute("%s/extract-debuginfo %s %s" % (SCRIPT_DIR, " ".join(args), self.pkg.buildroot))
1135                 except ShellEnvironmentError, e:
1136                         log.error(_("Extracting debuginfo did not complete with success. Aborting build."))
1137                         raise
1138
1139         def find_prerequires(self, scriptlet_file):
1140                 assert os.path.exists(scriptlet_file), "Scriptlet file does not exist: %s" % scriptlet_file
1141
1142                 res = self.run_script("find-prerequires", scriptlet_file)
1143                 prerequires = set(res.splitlines())
1144
1145                 return prerequires
1146
1147         def cleanup(self):
1148                 if os.path.exists(self.buildroot):
1149                         util.rm(self.buildroot)