python/pakfire/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import math
  25 import os
  26 import re
  27 import shutil
  28 import signal
  29 import socket
  30 import tempfile
  31 import time
  32 import uuid
  33
  34 import base
  35 import cgroup
  36 import logger
  37 import packages
  38 import packages.file
  39 import packages.packager
  40 import repository
  41 import shell
  42 import util
  43 import _pakfire
  44
  45 import logging
  46 log = logging.getLogger("pakfire")
  47
  48 from config import ConfigBuilder
  49 from system import system
  50 from constants import *
  51 from i18n import _
  52 from errors import BuildError, BuildRootLocked, Error
  53
  54
  55 BUILD_LOG_HEADER = """
  56  ____       _     __ _            _           _ _     _
  57 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  58 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  59 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  60 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  61
  62         Version : %(version)s
  63         Host    : %(hostname)s (%(host_arch)s)
  64         Time    : %(time)s
  65
  66 """
  67
  68 class BuildEnviron(object):
  69         # The version of the kernel this machine is running.
  70         kernel_version = os.uname()[2]
  71
  72         def __init__(self, pakfire, filename=None, distro_name=None, build_id=None, logfile=None, release_build=True):
  73                 self.pakfire = pakfire
  74
  75                 # Check if the given pakfire instance is of the correct type.
  76                 assert isinstance(self.pakfire, base.PakfireBuilder)
  77
  78                 # Check if this host can build the requested architecture.
  79                 if not system.host_supports_arch(self.arch):
  80                         raise BuildError, _("Cannot build for %s on this host.") % self.arch
  81
  82                 # Save the build id and generate one if no build id was provided.
  83                 if not build_id:
  84                         build_id = "%s" % uuid.uuid4()
  85
  86                 self.build_id = build_id
  87
  88                 # Setup the logging.
  89                 self.init_logging(logfile)
  90
  91                 # Initialize a cgroup (if supported).
  92                 self.init_cgroup()
  93
  94                 # This build is a release build?
  95                 self.release_build = release_build
  96
  97                 if self.release_build:
  98                         # Disable the local build repository in release mode.
  99                         self.pakfire.repos.disable_repo("build")
 100
 101                         # Log information about pakfire and some more information, when we
 102                         # are running in release mode.
 103                         logdata = {
 104                                 "host_arch"  : system.arch,
 105                                 "hostname"   : system.hostname,
 106                                 "time"       : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
 107                                 "version"    : "Pakfire %s" % PAKFIRE_VERSION,
 108                         }
 109
 110                         for line in BUILD_LOG_HEADER.splitlines():
 111                                 self.log.info(line % logdata)
 112
 113                 # Settings array.
 114                 self.settings = {
 115                         "enable_loop_devices" : self.config.get_bool("builder", "use_loop_devices", True),
 116                         "enable_ccache"       : self.config.get_bool("builder", "use_ccache", True),
 117                         "enable_icecream"     : self.config.get_bool("builder", "use_icecream", False),
 118                         "sign_packages"       : False,
 119                         "buildroot_tmpfs"     : self.config.get_bool("builder", "use_tmpfs", False),
 120                 }
 121
 122                 # Try to get the configured host key. If it is available,
 123                 # we will automatically sign all packages with it.
 124                 if self.keyring.get_host_key(secret=True):
 125                         self.settings["sign_packages"] = True
 126
 127                 # Where do we put the result?
 128                 self.resultdir = os.path.join(self.pakfire.path, "result")
 129
 130                 # Open package.
 131                 # If we have a plain makefile, we first build a source package and go with that.
 132                 if filename:
 133                         # Open source package.
 134                         self.pkg = packages.SourcePackage(self.pakfire, None, filename)
 135                         assert self.pkg, filename
 136
 137                         # Log the package information.
 138                         self.log.info(_("Package information:"))
 139                         for line in self.pkg.dump(long=True).splitlines():
 140                                 self.log.info("  %s" % line)
 141                         self.log.info("")
 142
 143                         # Path where we extract the package and put all the source files.
 144                         self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name)
 145                 else:
 146                         # No package :(
 147                         self.pkg = None
 148
 149                 # Lock the buildroot
 150                 self._lock = None
 151
 152                 # Save the build time.
 153                 self.build_time = time.time()
 154
 155         def setup_signal_handlers(self):
 156                 pass
 157
 158         def start(self):
 159                 assert not self.pakfire.initialized, "Pakfire has already been initialized"
 160
 161                 # Mount the directories.
 162                 self._mountall()
 163
 164                 # Lock the build environment.
 165                 self.lock()
 166
 167                 # Initialize pakfire instance.
 168                 self.pakfire.initialize()
 169
 170                 # Populate /dev.
 171                 self.populate_dev()
 172
 173                 # Setup domain name resolution in chroot.
 174                 self.setup_dns()
 175
 176                 # Extract all needed packages.
 177                 self.extract()
 178
 179         def stop(self):
 180                 if self.cgroup:
 181                         # Kill all still running processes in the cgroup.
 182                         self.cgroup.kill_and_wait()
 183
 184                         # Remove cgroup and all parent cgroups if they are empty.
 185                         self.cgroup.migrate_task(self.cgroup.root, os.getpid())
 186                         self.cgroup.destroy()
 187
 188                         parent = self.cgroup.parent
 189                         while parent:
 190                                 if not parent.is_empty(recursive=True):
 191                                         break
 192
 193                                 parent.destroy()
 194                                 parent = parent.parent
 195
 196                 else:
 197                         util.orphans_kill(self.path)
 198
 199                 # Shut down pakfire instance.
 200                 self.pakfire.destroy()
 201
 202                 # Unlock build environment.
 203                 self.unlock()
 204
 205                 # Umount the build environment.
 206                 self._umountall()
 207
 208                 # Remove all files.
 209                 self.destroy()
 210
 211         @property
 212         def config(self):
 213                 """
 214                         Proxy method for easy access to the configuration.
 215                 """
 216                 return self.pakfire.config
 217
 218         @property
 219         def distro(self):
 220                 """
 221                         Proxy method for easy access to the distribution.
 222                 """
 223                 return self.pakfire.distro
 224
 225         @property
 226         def path(self):
 227                 """
 228                         Proxy method for easy access to the path.
 229                 """
 230                 return self.pakfire.path
 231
 232         @property
 233         def arch(self):
 234                 """
 235                         Inherit architecture from distribution configuration.
 236                 """
 237                 return self.pakfire.distro.arch
 238
 239         @property
 240         def personality(self):
 241                 """
 242                         Gets the personality from the distribution configuration.
 243                 """
 244                 return self.pakfire.distro.personality
 245
 246         @property
 247         def info(self):
 248                 return {
 249                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 250                         "build_host" : socket.gethostname(),
 251                         "build_id"   : self.build_id,
 252                         "build_time" : self.build_time,
 253                 }
 254
 255         @property
 256         def keyring(self):
 257                 """
 258                         Shortcut to access the pakfire keyring.
 259                 """
 260                 return self.pakfire.keyring
 261
 262         def lock(self):
 263                 filename = os.path.join(self.path, ".lock")
 264
 265                 try:
 266                         self._lock = open(filename, "a+")
 267                 except IOError, e:
 268                         return 0
 269
 270                 try:
 271                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 272                 except IOError, e:
 273                         raise BuildRootLocked, "Buildroot is locked"
 274
 275                 return 1
 276
 277         def unlock(self):
 278                 if self._lock:
 279                         self._lock.close()
 280                         self._lock = None
 281
 282         def init_cgroup(self):
 283                 """
 284                         Initialize cgroup (if the system supports it).
 285                 """
 286                 if not cgroup.supported():
 287                         self.cgroup = None
 288                         return
 289
 290                 self.cgroup = cgroup.CGroup("pakfire/builder/%s" % self.build_id)
 291
 292                 # Attach the pakfire-builder process to the parent group.
 293                 self.cgroup.parent.attach()
 294
 295         def init_logging(self, logfile):
 296                 if logfile:
 297                         self.log = log.getChild(self.build_id)
 298                         # Propage everything to the root logger that we will see something
 299                         # on the terminal.
 300                         self.log.propagate = 1
 301                         self.log.setLevel(logging.INFO)
 302
 303                         # Add the given logfile to the logger.
 304                         h = logging.FileHandler(logfile)
 305                         self.log.addHandler(h)
 306
 307                         # Format the log output for the file.
 308                         f = logger.BuildFormatter()
 309                         h.setFormatter(f)
 310                 else:
 311                         # If no logile was given, we use the root logger.
 312                         self.log = logging.getLogger("pakfire")
 313
 314         def copyin(self, file_out, file_in):
 315                 if file_in.startswith("/"):
 316                         file_in = file_in[1:]
 317
 318                 file_in = self.chrootPath(file_in)
 319
 320                 #if not os.path.exists(file_out):
 321                 #       return
 322
 323                 dir_in = os.path.dirname(file_in)
 324                 if not os.path.exists(dir_in):
 325                         os.makedirs(dir_in)
 326
 327                 self.log.debug("%s --> %s" % (file_out, file_in))
 328
 329                 shutil.copy2(file_out, file_in)
 330
 331         def copyout(self, file_in, file_out):
 332                 if file_in.startswith("/"):
 333                         file_in = file_in[1:]
 334
 335                 file_in = self.chrootPath(file_in)
 336
 337                 #if not os.path.exists(file_in):
 338                 #       return
 339
 340                 dir_out = os.path.dirname(file_out)
 341                 if not os.path.exists(dir_out):
 342                         os.makedirs(dir_out)
 343
 344                 self.log.debug("%s --> %s" % (file_in, file_out))
 345
 346                 shutil.copy2(file_in, file_out)
 347
 348         def copy_result(self, resultdir):
 349                 # XXX should use find_result_packages
 350
 351                 dir_in = self.chrootPath("result")
 352
 353                 for dir, subdirs, files in os.walk(dir_in):
 354                         basename = os.path.basename(dir)
 355                         dir = dir[len(self.chrootPath()):]
 356                         for file in files:
 357                                 file_in = os.path.join(dir, file)
 358
 359                                 file_out = os.path.join(
 360                                         resultdir,
 361                                         basename,
 362                                         file,
 363                                 )
 364
 365                                 self.copyout(file_in, file_out)
 366
 367         def find_result_packages(self):
 368                 ret = []
 369
 370                 for dir, subdirs, files in os.walk(self.resultdir):
 371                         for file in files:
 372                                 if not file.endswith(".%s" % PACKAGE_EXTENSION):
 373                                         continue
 374
 375                                 file = os.path.join(dir, file)
 376                                 ret.append(file)
 377
 378                 return ret
 379
 380         def extract(self, requires=None):
 381                 """
 382                         Gets a dependency set and extracts all packages
 383                         to the environment.
 384                 """
 385                 if not requires:
 386                         requires = []
 387
 388                 # Add neccessary build dependencies.
 389                 requires += BUILD_PACKAGES
 390
 391                 # If we have ccache enabled, we need to extract it
 392                 # to the build chroot.
 393                 if self.settings.get("enable_ccache"):
 394                         requires.append("ccache")
 395
 396                 # If we have icecream enabled, we need to extract it
 397                 # to the build chroot.
 398                 if self.settings.get("enable_icecream"):
 399                         requires.append("icecream")
 400
 401                 # Get build dependencies from source package.
 402                 if self.pkg:
 403                         for req in self.pkg.requires:
 404                                 requires.append(req)
 405
 406                 # Install all packages.
 407                 self.log.info(_("Install packages needed for build..."))
 408                 self.install(requires)
 409
 410                 # Copy the makefile and load source tarballs.
 411                 if self.pkg:
 412                         self.pkg.extract(_("Extracting"), prefix=self.build_dir)
 413
 414                         # Add an empty line at the end.
 415                         self.log.info("")
 416
 417         def install(self, requires, **kwargs):
 418                 """
 419                         Install everything that is required in requires.
 420                 """
 421                 # If we got nothing to do, we quit immediately.
 422                 if not requires:
 423                         return
 424
 425                 kwargs.update({
 426                         "interactive" : False,
 427                         "logger" : self.log,
 428                 })
 429
 430                 if not kwargs.has_key("allow_downgrade"):
 431                         kwargs["allow_downgrade"] = True
 432
 433                 # Install everything.
 434                 self.pakfire.install(requires, **kwargs)
 435
 436         def chrootPath(self, *args):
 437                 # Remove all leading slashes
 438                 _args = []
 439                 for arg in args:
 440                         if arg.startswith("/"):
 441                                 arg = arg[1:]
 442                         _args.append(arg)
 443                 args = _args
 444
 445                 ret = os.path.join(self.path, *args)
 446                 ret = ret.replace("//", "/")
 447
 448                 assert ret.startswith(self.path)
 449
 450                 return ret
 451
 452         def populate_dev(self):
 453                 nodes = [
 454                         "/dev/null",
 455                         "/dev/zero",
 456                         "/dev/full",
 457                         "/dev/random",
 458                         "/dev/urandom",
 459                         "/dev/tty",
 460                         "/dev/ptmx",
 461                         "/dev/kmsg",
 462                         "/dev/rtc0",
 463                         "/dev/console",
 464                 ]
 465
 466                 # If we need loop devices (which are optional) we create them here.
 467                 if self.settings["enable_loop_devices"]:
 468                         for i in range(0, 7):
 469                                 nodes.append("/dev/loop%d" % i)
 470
 471                 for node in nodes:
 472                         # Stat the original node of the host system and copy it to
 473                         # the build chroot.
 474                         node_stat = os.stat(node)
 475
 476                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 477
 478                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 479                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 480                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 481                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 482
 483         def setup_dns(self):
 484                 """
 485                         Add DNS resolution facility to chroot environment by copying
 486                         /etc/resolv.conf and /etc/hosts.
 487                 """
 488                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 489                         self.copyin(i, i)
 490
 491         def _create_node(self, filename, mode, device):
 492                 self.log.debug("Create node: %s (%s)" % (filename, mode))
 493
 494                 filename = self.chrootPath(filename)
 495
 496                 # Create parent directory if it is missing.
 497                 dirname = os.path.dirname(filename)
 498                 if not os.path.exists(dirname):
 499                         os.makedirs(dirname)
 500
 501                 os.mknod(filename, mode, device)
 502
 503         def destroy(self):
 504                 self.log.debug("Destroying environment %s" % self.path)
 505
 506                 if os.path.exists(self.path):
 507                         util.rm(self.path)
 508
 509         def cleanup(self):
 510                 self.log.debug("Cleaning environemnt.")
 511
 512                 # Remove the build directory and buildroot.
 513                 dirs = (self.build_dir, self.chrootPath("result"),)
 514
 515                 for d in dirs:
 516                         if not os.path.exists(d):
 517                                 continue
 518
 519                         util.rm(d)
 520                         os.makedirs(d)
 521
 522         def _mountall(self):
 523                 self.log.debug("Mounting environment")
 524                 for src, dest, fs, options in self.mountpoints:
 525                         mountpoint = self.chrootPath(dest)
 526                         if options:
 527                                 options = "-o %s" % options
 528
 529                         # Eventually create mountpoint directory
 530                         if not os.path.exists(mountpoint):
 531                                 os.makedirs(mountpoint)
 532
 533                         self.execute_root("mount -n -t %s %s %s %s" % (fs, options, src, mountpoint), shell=True)
 534
 535         def _umountall(self):
 536                 self.log.debug("Umounting environment")
 537
 538                 mountpoints = []
 539                 for src, dest, fs, options in reversed(self.mountpoints):
 540                         dest = self.chrootPath(dest)
 541
 542                         if not dest in mountpoints:
 543                                 mountpoints.append(dest)
 544
 545                 while mountpoints:
 546                         for mp in mountpoints:
 547                                 try:
 548                                         self.execute_root("umount -n %s" % mp, shell=True)
 549                                 except ShellEnvironmentError:
 550                                         pass
 551
 552                                 if not os.path.ismount(mp):
 553                                         mountpoints.remove(mp)
 554
 555         @property
 556         def mountpoints(self):
 557                 mountpoints = []
 558
 559                 # Make root as a tmpfs if enabled.
 560                 if self.settings.get("buildroot_tmpfs"):
 561                         mountpoints += [
 562                                 ("pakfire_root", "/", "tmpfs", "defaults"),
 563                         ]
 564
 565                 mountpoints += [
 566                         # src, dest, fs, options
 567                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 568                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 569                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 570                         ("/sys",          "/sys",      "bind",  "bind"),
 571                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 572                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 573                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 574                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 575                 ]
 576
 577                 # If selinux is enabled.
 578                 if os.path.exists("/sys/fs/selinux"):
 579                         mountpoints += [
 580                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 581                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 582                         ]
 583
 584                 # If ccache support is requested, we bind mount the cache.
 585                 if self.settings.get("enable_ccache"):
 586                         # Create ccache cache directory if it does not exist.
 587                         if not os.path.exists(CCACHE_CACHE_DIR):
 588                                 os.makedirs(CCACHE_CACHE_DIR)
 589
 590                         mountpoints += [
 591                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 592                         ]
 593
 594                 return mountpoints
 595
 596         @property
 597         def environ(self):
 598                 env = {
 599                         # Add HOME manually, because it is occasionally not set
 600                         # and some builds get in trouble then.
 601                         "PATH" : "/usr/bin:/bin:/usr/sbin:/sbin",
 602                         "HOME" : "/root",
 603                         "TERM" : os.environ.get("TERM", "vt100"),
 604                         "PS1"  : "\u:\w\$ ",
 605
 606                         # Sanitize language.
 607                         "LANG" : os.environ.setdefault("LANG", "en_US.UTF-8"),
 608
 609                         # Set the container that we can detect, if we are inside a
 610                         # chroot.
 611                         "container" : "pakfire-builder",
 612                 }
 613
 614                 # Inherit environment from distro
 615                 env.update(self.pakfire.distro.environ)
 616
 617                 # Icecream environment settings
 618                 if self.settings.get("enable_icecream", False):
 619                         # Set the toolchain path
 620                         if self.settings.get("icecream_toolchain", None):
 621                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 622
 623                         # Set preferred host if configured.
 624                         if self.settings.get("icecream_preferred_host", None):
 625                                 env["ICECC_PREFERRED_HOST"] = \
 626                                         self.settings.get("icecream_preferred_host")
 627
 628                 # Fake UTS_MACHINE, when we cannot use the personality syscall and
 629                 # if the host architecture is not equal to the target architecture.
 630                 if not self.pakfire.distro.personality and \
 631                                 not system.native_arch == self.pakfire.distro.arch:
 632                         env.update({
 633                                 "LD_PRELOAD"  : "/usr/lib/libpakfire_preload.so",
 634                                 "UTS_MACHINE" : self.pakfire.distro.arch,
 635                         })
 636
 637                 return env
 638
 639         @property
 640         def installed_packages(self):
 641                 """
 642                         Returns an iterator over all installed packages in this build environment.
 643                 """
 644                 # Get the repository of all installed packages.
 645                 repo = self.pakfire.repos.get_repo("@system")
 646
 647                 # Return an iterator over the packages.
 648                 return iter(repo)
 649
 650         def write_config(self):
 651                 # Cleanup everything in /etc/pakfire.
 652                 util.rm(self.chrootPath(CONFIG_DIR))
 653
 654                 for i in (CONFIG_DIR, CONFIG_REPOS_DIR):
 655                         i = self.chrootPath(i)
 656                         if not os.path.exists(i):
 657                                 os.makedirs(i)
 658
 659                 # Write general.conf.
 660                 f = open(self.chrootPath(CONFIG_DIR, "general.conf"), "w")
 661                 f.close()
 662
 663                 # Write builder.conf.
 664                 f = open(self.chrootPath(CONFIG_DIR, "builder.conf"), "w")
 665                 f.write(self.distro.get_config())
 666                 f.close()
 667
 668                 # Create pakfire configuration files.
 669                 for repo in self.pakfire.repos:
 670                         conf = repo.get_config()
 671
 672                         if not conf:
 673                                 continue
 674
 675                         filename = self.chrootPath(CONFIG_REPOS_DIR, "%s.repo" % repo.name)
 676                         f = open(filename, "w")
 677                         f.write("\n".join(conf))
 678                         f.close()
 679
 680         @property
 681         def pkg_makefile(self):
 682                 return os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION))
 683
 684         def execute(self, command, logger=None, **kwargs):
 685                 """
 686                         Executes the given command in the build chroot.
 687                 """
 688                 # Environment variables
 689                 env = self.environ
 690
 691                 if kwargs.has_key("env"):
 692                         env.update(kwargs.pop("env"))
 693
 694                 self.log.debug("Environment:")
 695                 for k, v in sorted(env.items()):
 696                         self.log.debug("  %s=%s" % (k, v))
 697
 698                 # Make every shell to a login shell because we set a lot of
 699                 # environment things there.
 700                 command = ["bash", "--login", "-c", command]
 701
 702                 args = {
 703                         "chroot_path" : self.chrootPath(),
 704                         "cgroup"      : self.cgroup,
 705                         "env"         : env,
 706                         "logger"      : logger,
 707                         "personality" : self.personality,
 708                         "shell"       : False,
 709                 }
 710                 args.update(kwargs)
 711
 712                 # Run the shit.
 713                 shellenv = shell.ShellExecuteEnvironment(command, **args)
 714                 shellenv.execute()
 715
 716                 return shellenv
 717
 718         def execute_root(self, command, **kwargs):
 719                 """
 720                         Executes the given command outside the build chroot.
 721                 """
 722                 shellenv = shell.ShellExecuteEnvironment(command, **kwargs)
 723                 shellenv.execute()
 724
 725                 return shellenv
 726
 727         def build(self, install_test=True, prepare=False):
 728                 if not self.pkg:
 729                         raise BuildError, _("You cannot run a build when no package was given.")
 730
 731                 # Search for the package file in build_dir and raise BuildError if it is not present.
 732                 if not os.path.exists(self.pkg_makefile):
 733                         raise BuildError, _("Could not find makefile in build root: %s") % self.pkg_makefile
 734
 735                 # Write pakfire configuration into the chroot.
 736                 self.write_config()
 737
 738                 # Create the build command, that is executed in the chroot.
 739                 build_command = [
 740                         "/usr/lib/pakfire/builder",
 741                         "--offline",
 742                         "build",
 743                         "/%s" % os.path.relpath(self.pkg_makefile, self.chrootPath()),
 744                         "--arch", self.arch,
 745                         "--nodeps",
 746                         "--resultdir=/result",
 747                 ]
 748
 749                 # Check if only the preparation stage should be run.
 750                 if prepare:
 751                         build_command.append("--prepare")
 752
 753                 build_command = " ".join(build_command)
 754
 755                 error = False
 756                 try:
 757                         self.execute(build_command, logger=self.log)
 758
 759                         # Perform the install test after the actual build.
 760                         if install_test and not prepare:
 761                                 self.install_test()
 762
 763                 except ShellEnvironmentError:
 764                         error = True
 765                         self.log.error(_("Build failed"))
 766
 767                 except KeyboardInterrupt:
 768                         error = True
 769                         self.log.error(_("Build interrupted"))
 770
 771                         raise
 772
 773                 # Catch all other errors.
 774                 except:
 775                         error = True
 776                         self.log.error(_("Build failed."), exc_info=True)
 777
 778                 else:
 779                         # Don't sign packages in prepare mode.
 780                         if prepare:
 781                                 return
 782
 783                         # Sign all built packages with the host key (if available).
 784                         self.sign_packages()
 785
 786                         # Dump package information.
 787                         self.dump()
 788
 789                         return
 790
 791                 # End here in case of an error.
 792                 raise BuildError, _("The build command failed. See logfile for details.")
 793
 794         def install_test(self):
 795                 self.log.info(_("Running installation test..."))
 796
 797                 # Install all packages that were built.
 798                 self.install(self.find_result_packages(), allow_vendorchange=True,
 799                         allow_uninstall=True, signatures_mode="disabled")
 800
 801                 self.log.info(_("Installation test succeeded."))
 802                 self.log.info("")
 803
 804         def shell(self, args=[]):
 805                 if not util.cli_is_interactive():
 806                         self.log.warning("Cannot run shell on non-interactive console.")
 807                         return
 808
 809                 # Install all packages that are needed to run a shell.
 810                 self.install(SHELL_PACKAGES)
 811
 812                 # XXX need to set CFLAGS here
 813                 command = "/usr/sbin/chroot %s %s %s" % \
 814                         (self.chrootPath(), SHELL_SCRIPT, " ".join(args))
 815
 816                 # Add personality if we require one
 817                 if self.pakfire.distro.personality:
 818                         command = "%s %s" % (self.pakfire.distro.personality, command)
 819
 820                 for key, val in self.environ.items():
 821                         command = "%s=\"%s\" " % (key, val) + command
 822
 823                 # Empty the environment
 824                 command = "env -i - %s" % command
 825
 826                 self.log.debug("Shell command: %s" % command)
 827
 828                 shell = os.system(command)
 829                 return os.WEXITSTATUS(shell)
 830
 831         def sign_packages(self, keyfp=None):
 832                 # Do nothing if signing is not requested.
 833                 if not self.settings.get("sign_packages"):
 834                         return
 835
 836                 # Get key, that should be used for signing.
 837                 if not keyfp:
 838                         keyfp = self.keyring.get_host_key_id()
 839
 840                 # Find all files to process.
 841                 files = self.find_result_packages()
 842
 843                 # Create a progressbar.
 844                 print _("Signing packages...")
 845                 p = util.make_progress(keyfp, len(files))
 846                 i = 0
 847
 848                 for file in files:
 849                         # Update progressbar.
 850                         if p:
 851                                 i += 1
 852                                 p.update(i)
 853
 854                         # Open package file.
 855                         pkg = packages.open(self.pakfire, None, file)
 856
 857                         # Sign it.
 858                         pkg.sign(keyfp)
 859
 860                 # Close progressbar.
 861                 if p:
 862                         p.finish()
 863                         print "" # Print an empty line.
 864
 865         def dump(self):
 866                 pkgs = []
 867
 868                 for file in self.find_result_packages():
 869                         pkg = packages.open(self.pakfire, None, file)
 870                         pkgs.append(pkg)
 871
 872                 # If there are no packages, there is nothing to do.
 873                 if not pkgs:
 874                         return
 875
 876                 pkgs.sort()
 877
 878                 self.log.info(_("Dumping package information:"))
 879                 for pkg in pkgs:
 880                         dump = pkg.dump(long=True)
 881
 882                         for line in dump.splitlines():
 883                                 self.log.info("  %s" % line)
 884                         self.log.info("") # Empty line.
 885
 886
 887 class Builder(object):
 888         def __init__(self, pakfire, filename, resultdir, **kwargs):
 889                 self.pakfire = pakfire
 890
 891                 self.filename = filename
 892
 893                 self.resultdir = resultdir
 894
 895                 # Open package file.
 896                 self.pkg = packages.Makefile(self.pakfire, self.filename)
 897
 898                 self._environ = {
 899                         "LANG"             : "C",
 900                 }
 901
 902         @property
 903         def buildroot(self):
 904                 return self.pkg.buildroot
 905
 906         @property
 907         def distro(self):
 908                 return self.pakfire.distro
 909
 910         @property
 911         def environ(self):
 912                 environ = os.environ
 913
 914                 # Get all definitions from the package.
 915                 environ.update(self.pkg.exports)
 916
 917                 # Overwrite some definitions by default values.
 918                 environ.update(self._environ)
 919
 920                 return environ
 921
 922         def execute(self, command, logger=None, **kwargs):
 923                 if logger is None:
 924                         logger = logging.getLogger("pakfire")
 925
 926                 # Make every shell to a login shell because we set a lot of
 927                 # environment things there.
 928                 command = ["bash", "--login", "-c", command]
 929
 930                 args = {
 931                         "cwd"         : "/%s" % LOCAL_TMP_PATH,
 932                         "env"         : self.environ,
 933                         "logger"      : logger,
 934                         "personality" : self.distro.personality,
 935                         "shell"       : False,
 936                 }
 937                 args.update(kwargs)
 938
 939                 try:
 940                         shellenv = shell.ShellExecuteEnvironment(command, **args)
 941                         shellenv.execute()
 942
 943                 except ShellEnvironmentError:
 944                         logger.error("Command exited with an error: %s" % command)
 945                         raise
 946
 947                 return shellenv
 948
 949         def run_script(self, script, *args):
 950                 if not script.startswith("/"):
 951                         script = os.path.join(SCRIPT_DIR, script)
 952
 953                 assert os.path.exists(script), "Script we should run does not exist: %s" % script
 954
 955                 cmd = [script,]
 956                 for arg in args:
 957                         cmd.append(arg)
 958                 cmd = " ".join(cmd)
 959
 960                 # Returns the output of the command, but the output won't get
 961                 # logged.
 962                 exe = self.execute(cmd, record_output=True, log_output=False)
 963
 964                 # Return the output of the command.
 965                 if exe.exitcode == 0:
 966                         return exe.output
 967
 968         def create_icecream_toolchain(self):
 969                 try:
 970                         exe = self.execute(
 971                                 "icecc --build-native 2>/dev/null",
 972                                 record_output=True, record_stderr=False,
 973                                 log_output=False, log_errors=False,
 974                                 cwd="/tmp",
 975                         )
 976                 except ShellEnvironmentError:
 977                         return
 978
 979                 for line in exe.output.splitlines():
 980                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
 981                         if m:
 982                                 self._environ["ICECC_VERSION"] = "/tmp/%s" % m.group(1)
 983
 984         def create_buildscript(self, stage):
 985                 # Get buildscript from the package.
 986                 script = self.pkg.get_buildscript(stage)
 987
 988                 # Write script to an empty file.
 989                 f = tempfile.NamedTemporaryFile(mode="w", delete=False)
 990                 f.write("#!/bin/sh\n\n")
 991                 f.write("set -e\n")
 992                 f.write("set -x\n")
 993                 f.write("\n%s\n" % script)
 994                 f.write("exit 0\n")
 995                 f.close()
 996
 997                 # Make the script executable.
 998                 os.chmod(f.name, 700)
 999
1000                 return f.name
1001
1002         def build(self, stages=None):
1003                 # Create buildroot and remove all content if it was existant.
1004                 util.rm(self.buildroot)
1005                 os.makedirs(self.buildroot)
1006
1007                 # Build icecream toolchain if icecream is installed.
1008                 self.create_icecream_toolchain()
1009
1010                 # Process stages in order.
1011                 for stage in ("prepare", "build", "test", "install"):
1012                         # Skip unwanted stages.
1013                         if stages and not stage in stages:
1014                                 continue
1015
1016                         # Run stage.
1017                         self.build_stage(stage)
1018
1019                 # Stop if install stage has not been processed.
1020                 if stages and not "install" in stages:
1021                         return
1022
1023                 # Run post-build stuff.
1024                 self.post_compress_man_pages()
1025                 self.post_remove_static_libs()
1026                 self.post_extract_debuginfo()
1027
1028                 # Package the result.
1029                 # Make all these little package from the build environment.
1030                 log.info(_("Creating packages:"))
1031                 pkgs = []
1032                 for pkg in reversed(self.pkg.packages):
1033                         packager = packages.packager.BinaryPackager(self.pakfire, pkg,
1034                                 self, self.buildroot)
1035                         pkg = packager.run(self.resultdir)
1036                         pkgs.append(pkg)
1037                 log.info("")
1038
1039         def build_stage(self, stage):
1040                 # Get the buildscript for this stage.
1041                 buildscript = self.create_buildscript(stage)
1042
1043                 # Execute the buildscript of this stage.
1044                 log.info(_("Running stage %s:") % stage)
1045
1046                 try:
1047                         self.execute(buildscript)
1048
1049                 finally:
1050                         # Remove the buildscript.
1051                         if os.path.exists(buildscript):
1052                                 os.unlink(buildscript)
1053
1054         def post_remove_static_libs(self):
1055                 keep_libs = self.pkg.lexer.build.get_var("keep_libraries")
1056                 keep_libs = keep_libs.split()
1057
1058                 try:
1059                         self.execute("%s/remove-static-libs %s %s" % \
1060                                 (SCRIPT_DIR, self.buildroot, " ".join(keep_libs)))
1061                 except ShellEnvironmentError, e:
1062                         log.warning(_("Could not remove static libraries: %s") % e)
1063
1064         def post_compress_man_pages(self):
1065                 try:
1066                         self.execute("%s/compress-man-pages %s" % (SCRIPT_DIR, self.buildroot))
1067                 except ShellEnvironmentError, e:
1068                         log.warning(_("Compressing man pages did not complete successfully."))
1069
1070         def post_extract_debuginfo(self):
1071                 args = []
1072
1073                 # Check if we need to run with strict build-id.
1074                 strict_id = self.pkg.lexer.build.get_var("debuginfo_strict_build_id", "true")
1075                 if strict_id in ("true", "yes", "1"):
1076                         args.append("--strict-build-id")
1077
1078                 args.append("--buildroot=%s" % self.pkg.buildroot)
1079                 args.append("--sourcedir=%s" % self.pkg.sourcedir)
1080
1081                 # Get additional options to pass to script.
1082                 options = self.pkg.lexer.build.get_var("debuginfo_options", "")
1083                 args += options.split()
1084
1085                 try:
1086                         self.execute("%s/extract-debuginfo %s %s" % (SCRIPT_DIR, " ".join(args), self.pkg.buildroot))
1087                 except ShellEnvironmentError, e:
1088                         log.error(_("Extracting debuginfo did not complete with success. Aborting build."))
1089                         raise
1090
1091         def find_prerequires(self, scriptlet_file):
1092                 assert os.path.exists(scriptlet_file), "Scriptlet file does not exist: %s" % scriptlet_file
1093
1094                 res = self.run_script("find-prerequires", scriptlet_file)
1095                 prerequires = set(res.splitlines())
1096
1097                 return prerequires
1098
1099         def cleanup(self):
1100                 if os.path.exists(self.buildroot):
1101                         util.rm(self.buildroot)