python/pakfire/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import math
  25 import os
  26 import re
  27 import shutil
  28 import signal
  29 import socket
  30 import tempfile
  31 import time
  32 import uuid
  33
  34 import base
  35 import cgroup
  36 import logger
  37 import packages
  38 import packages.file
  39 import packages.packager
  40 import repository
  41 import shell
  42 import util
  43 import _pakfire
  44
  45 import logging
  46 log = logging.getLogger("pakfire")
  47
  48 from config import ConfigBuilder
  49 from system import system
  50 from constants import *
  51 from i18n import _
  52 from errors import BuildError, BuildRootLocked, Error
  53
  54
  55 BUILD_LOG_HEADER = """
  56  ____       _     __ _            _           _ _     _
  57 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  58 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  59 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  60 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  61
  62         Version : %(version)s
  63         Host    : %(hostname)s (%(host_arch)s)
  64         Time    : %(time)s
  65
  66 """
  67
  68 class BuildEnviron(object):
  69         # The version of the kernel this machine is running.
  70         kernel_version = os.uname()[2]
  71
  72         def __init__(self, pakfire, filename=None, distro_name=None, build_id=None, logfile=None, release_build=True):
  73                 self.pakfire = pakfire
  74
  75                 # Check if the given pakfire instance is of the correct type.
  76                 assert isinstance(self.pakfire, base.PakfireBuilder)
  77
  78                 # Check if this host can build the requested architecture.
  79                 if not system.host_supports_arch(self.arch):
  80                         raise BuildError, _("Cannot build for %s on this host.") % self.arch
  81
  82                 # Save the build id and generate one if no build id was provided.
  83                 if not build_id:
  84                         build_id = "%s" % uuid.uuid4()
  85
  86                 self.build_id = build_id
  87
  88                 # Setup the logging.
  89                 self.init_logging(logfile)
  90
  91                 # Initialize a cgroup (if supported).
  92                 self.init_cgroup()
  93
  94                 # This build is a release build?
  95                 self.release_build = release_build
  96
  97                 if self.release_build:
  98                         # Disable the local build repository in release mode.
  99                         self.pakfire.repos.disable_repo("build")
 100
 101                         # Log information about pakfire and some more information, when we
 102                         # are running in release mode.
 103                         logdata = {
 104                                 "host_arch"  : system.arch,
 105                                 "hostname"   : system.hostname,
 106                                 "time"       : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
 107                                 "version"    : "Pakfire %s" % PAKFIRE_VERSION,
 108                         }
 109
 110                         for line in BUILD_LOG_HEADER.splitlines():
 111                                 self.log.info(line % logdata)
 112
 113                 # XXX need to make this configureable
 114                 self.settings = {
 115                         "enable_loop_devices" : True,
 116                         "enable_ccache"   : True,
 117                         "enable_icecream" : False,
 118                         "sign_packages"   : False,
 119                         "buildroot_tmpfs" : False,
 120                 }
 121                 #self.settings.update(settings)
 122
 123                 # Try to get the configured host key. If it is available,
 124                 # we will automatically sign all packages with it.
 125                 if self.keyring.get_host_key(secret=True):
 126                         self.settings["sign_packages"] = True
 127
 128                 # Where do we put the result?
 129                 self.resultdir = os.path.join(self.pakfire.path, "result")
 130
 131                 # Open package.
 132                 # If we have a plain makefile, we first build a source package and go with that.
 133                 if filename:
 134                         # Open source package.
 135                         self.pkg = packages.SourcePackage(self.pakfire, None, filename)
 136                         assert self.pkg, filename
 137
 138                         # Log the package information.
 139                         self.log.info(_("Package information:"))
 140                         for line in self.pkg.dump(long=True).splitlines():
 141                                 self.log.info("  %s" % line)
 142                         self.log.info("")
 143
 144                         # Path where we extract the package and put all the source files.
 145                         self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name)
 146                 else:
 147                         # No package :(
 148                         self.pkg = None
 149
 150                 # Lock the buildroot
 151                 self._lock = None
 152
 153                 # Save the build time.
 154                 self.build_time = time.time()
 155
 156         def setup_signal_handlers(self):
 157                 pass
 158
 159         def start(self):
 160                 assert not self.pakfire.initialized, "Pakfire has already been initialized"
 161
 162                 # Mount the directories.
 163                 self._mountall()
 164
 165                 # Lock the build environment.
 166                 self.lock()
 167
 168                 # Initialize pakfire instance.
 169                 self.pakfire.initialize()
 170
 171                 # Populate /dev.
 172                 self.populate_dev()
 173
 174                 # Setup domain name resolution in chroot.
 175                 self.setup_dns()
 176
 177                 # Extract all needed packages.
 178                 self.extract()
 179
 180         def stop(self):
 181                 if self.cgroup:
 182                         # Kill all still running processes in the cgroup.
 183                         self.cgroup.kill_and_wait()
 184
 185                         # Remove cgroup and all parent cgroups if they are empty.
 186                         self.cgroup.migrate_task(self.cgroup.root, os.getpid())
 187                         self.cgroup.destroy()
 188
 189                         parent = self.cgroup.parent
 190                         while parent:
 191                                 if not parent.is_empty(recursive=True):
 192                                         break
 193
 194                                 parent.destroy()
 195                                 parent = parent.parent
 196
 197                 else:
 198                         util.orphans_kill(self.path)
 199
 200                 # Shut down pakfire instance.
 201                 self.pakfire.destroy()
 202
 203                 # Unlock build environment.
 204                 self.unlock()
 205
 206                 # Umount the build environment.
 207                 self._umountall()
 208
 209                 # Remove all files.
 210                 self.destroy()
 211
 212         @property
 213         def distro(self):
 214                 """
 215                         Proxy method for easy access to the distribution.
 216                 """
 217                 return self.pakfire.distro
 218
 219         @property
 220         def path(self):
 221                 """
 222                         Proxy method for easy access to the path.
 223                 """
 224                 return self.pakfire.path
 225
 226         @property
 227         def arch(self):
 228                 """
 229                         Inherit architecture from distribution configuration.
 230                 """
 231                 return self.pakfire.distro.arch
 232
 233         @property
 234         def personality(self):
 235                 """
 236                         Gets the personality from the distribution configuration.
 237                 """
 238                 return self.pakfire.distro.personality
 239
 240         @property
 241         def info(self):
 242                 return {
 243                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 244                         "build_host" : socket.gethostname(),
 245                         "build_id"   : self.build_id,
 246                         "build_time" : self.build_time,
 247                 }
 248
 249         @property
 250         def keyring(self):
 251                 """
 252                         Shortcut to access the pakfire keyring.
 253                 """
 254                 return self.pakfire.keyring
 255
 256         def lock(self):
 257                 filename = os.path.join(self.path, ".lock")
 258
 259                 try:
 260                         self._lock = open(filename, "a+")
 261                 except IOError, e:
 262                         return 0
 263
 264                 try:
 265                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 266                 except IOError, e:
 267                         raise BuildRootLocked, "Buildroot is locked"
 268
 269                 return 1
 270
 271         def unlock(self):
 272                 if self._lock:
 273                         self._lock.close()
 274                         self._lock = None
 275
 276         def init_cgroup(self):
 277                 """
 278                         Initialize cgroup (if the system supports it).
 279                 """
 280                 if not cgroup.supported():
 281                         self.cgroup = None
 282                         return
 283
 284                 self.cgroup = cgroup.CGroup("pakfire/builder/%s" % self.build_id)
 285
 286                 # Attach the pakfire-builder process to the parent group.
 287                 self.cgroup.parent.attach()
 288
 289         def init_logging(self, logfile):
 290                 if logfile:
 291                         self.log = log.getChild(self.build_id)
 292                         # Propage everything to the root logger that we will see something
 293                         # on the terminal.
 294                         self.log.propagate = 1
 295                         self.log.setLevel(logging.INFO)
 296
 297                         # Add the given logfile to the logger.
 298                         h = logging.FileHandler(logfile)
 299                         self.log.addHandler(h)
 300
 301                         # Format the log output for the file.
 302                         f = logger.BuildFormatter()
 303                         h.setFormatter(f)
 304                 else:
 305                         # If no logile was given, we use the root logger.
 306                         self.log = logging.getLogger("pakfire")
 307
 308         def copyin(self, file_out, file_in):
 309                 if file_in.startswith("/"):
 310                         file_in = file_in[1:]
 311
 312                 file_in = self.chrootPath(file_in)
 313
 314                 #if not os.path.exists(file_out):
 315                 #       return
 316
 317                 dir_in = os.path.dirname(file_in)
 318                 if not os.path.exists(dir_in):
 319                         os.makedirs(dir_in)
 320
 321                 self.log.debug("%s --> %s" % (file_out, file_in))
 322
 323                 shutil.copy2(file_out, file_in)
 324
 325         def copyout(self, file_in, file_out):
 326                 if file_in.startswith("/"):
 327                         file_in = file_in[1:]
 328
 329                 file_in = self.chrootPath(file_in)
 330
 331                 #if not os.path.exists(file_in):
 332                 #       return
 333
 334                 dir_out = os.path.dirname(file_out)
 335                 if not os.path.exists(dir_out):
 336                         os.makedirs(dir_out)
 337
 338                 self.log.debug("%s --> %s" % (file_in, file_out))
 339
 340                 shutil.copy2(file_in, file_out)
 341
 342         def copy_result(self, resultdir):
 343                 # XXX should use find_result_packages
 344
 345                 dir_in = self.chrootPath("result")
 346
 347                 for dir, subdirs, files in os.walk(dir_in):
 348                         basename = os.path.basename(dir)
 349                         dir = dir[len(self.chrootPath()):]
 350                         for file in files:
 351                                 file_in = os.path.join(dir, file)
 352
 353                                 file_out = os.path.join(
 354                                         resultdir,
 355                                         basename,
 356                                         file,
 357                                 )
 358
 359                                 self.copyout(file_in, file_out)
 360
 361         def find_result_packages(self):
 362                 ret = []
 363
 364                 for dir, subdirs, files in os.walk(self.resultdir):
 365                         for file in files:
 366                                 if not file.endswith(".%s" % PACKAGE_EXTENSION):
 367                                         continue
 368
 369                                 file = os.path.join(dir, file)
 370                                 ret.append(file)
 371
 372                 return ret
 373
 374         def extract(self, requires=None):
 375                 """
 376                         Gets a dependency set and extracts all packages
 377                         to the environment.
 378                 """
 379                 if not requires:
 380                         requires = []
 381
 382                 # Add neccessary build dependencies.
 383                 requires += BUILD_PACKAGES
 384
 385                 # If we have ccache enabled, we need to extract it
 386                 # to the build chroot.
 387                 if self.settings.get("enable_ccache"):
 388                         requires.append("ccache")
 389
 390                 # If we have icecream enabled, we need to extract it
 391                 # to the build chroot.
 392                 if self.settings.get("enable_icecream"):
 393                         requires.append("icecream")
 394
 395                 # Get build dependencies from source package.
 396                 if self.pkg:
 397                         for req in self.pkg.requires:
 398                                 requires.append(req)
 399
 400                 # Install all packages.
 401                 self.log.info(_("Install packages needed for build..."))
 402                 self.install(requires)
 403
 404                 # Copy the makefile and load source tarballs.
 405                 if self.pkg:
 406                         self.pkg.extract(_("Extracting"), prefix=self.build_dir)
 407
 408         def install(self, requires, **kwargs):
 409                 """
 410                         Install everything that is required in requires.
 411                 """
 412                 # If we got nothing to do, we quit immediately.
 413                 if not requires:
 414                         return
 415
 416                 kwargs.update({
 417                         "interactive" : False,
 418                         "logger" : self.log,
 419                 })
 420
 421                 if not kwargs.has_key("allow_downgrade"):
 422                         kwargs["allow_downgrade"] = True
 423
 424                 # Install everything.
 425                 self.pakfire.install(requires, **kwargs)
 426
 427         def chrootPath(self, *args):
 428                 # Remove all leading slashes
 429                 _args = []
 430                 for arg in args:
 431                         if arg.startswith("/"):
 432                                 arg = arg[1:]
 433                         _args.append(arg)
 434                 args = _args
 435
 436                 ret = os.path.join(self.path, *args)
 437                 ret = ret.replace("//", "/")
 438
 439                 assert ret.startswith(self.path)
 440
 441                 return ret
 442
 443         def populate_dev(self):
 444                 nodes = [
 445                         "/dev/null",
 446                         "/dev/zero",
 447                         "/dev/full",
 448                         "/dev/random",
 449                         "/dev/urandom",
 450                         "/dev/tty",
 451                         "/dev/ptmx",
 452                         "/dev/kmsg",
 453                         "/dev/rtc0",
 454                         "/dev/console",
 455                 ]
 456
 457                 # If we need loop devices (which are optional) we create them here.
 458                 if self.settings["enable_loop_devices"]:
 459                         for i in range(0, 7):
 460                                 nodes.append("/dev/loop%d" % i)
 461
 462                 for node in nodes:
 463                         # Stat the original node of the host system and copy it to
 464                         # the build chroot.
 465                         node_stat = os.stat(node)
 466
 467                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 468
 469                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 470                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 471                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 472                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 473
 474         def setup_dns(self):
 475                 """
 476                         Add DNS resolution facility to chroot environment by copying
 477                         /etc/resolv.conf and /etc/hosts.
 478                 """
 479                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 480                         self.copyin(i, i)
 481
 482         def _create_node(self, filename, mode, device):
 483                 self.log.debug("Create node: %s (%s)" % (filename, mode))
 484
 485                 filename = self.chrootPath(filename)
 486
 487                 # Create parent directory if it is missing.
 488                 dirname = os.path.dirname(filename)
 489                 if not os.path.exists(dirname):
 490                         os.makedirs(dirname)
 491
 492                 os.mknod(filename, mode, device)
 493
 494         def destroy(self):
 495                 self.log.debug("Destroying environment %s" % self.path)
 496
 497                 if os.path.exists(self.path):
 498                         util.rm(self.path)
 499
 500         def cleanup(self):
 501                 self.log.debug("Cleaning environemnt.")
 502
 503                 # Remove the build directory and buildroot.
 504                 dirs = (self.build_dir, self.chrootPath("result"),)
 505
 506                 for d in dirs:
 507                         if not os.path.exists(d):
 508                                 continue
 509
 510                         util.rm(d)
 511                         os.makedirs(d)
 512
 513         def _mountall(self):
 514                 self.log.debug("Mounting environment")
 515                 for src, dest, fs, options in self.mountpoints:
 516                         mountpoint = self.chrootPath(dest)
 517                         if options:
 518                                 options = "-o %s" % options
 519
 520                         # Eventually create mountpoint directory
 521                         if not os.path.exists(mountpoint):
 522                                 os.makedirs(mountpoint)
 523
 524                         self.execute_root("mount -n -t %s %s %s %s" % (fs, options, src, mountpoint), shell=True)
 525
 526         def _umountall(self):
 527                 self.log.debug("Umounting environment")
 528
 529                 mountpoints = []
 530                 for src, dest, fs, options in reversed(self.mountpoints):
 531                         dest = self.chrootPath(dest)
 532
 533                         if not dest in mountpoints:
 534                                 mountpoints.append(dest)
 535
 536                 while mountpoints:
 537                         for mp in mountpoints:
 538                                 try:
 539                                         self.execute_root("umount -n %s" % mp, shell=True)
 540                                 except ShellEnvironmentError:
 541                                         pass
 542
 543                                 if not os.path.ismount(mp):
 544                                         mountpoints.remove(mp)
 545
 546         @property
 547         def mountpoints(self):
 548                 mountpoints = []
 549
 550                 # Make root as a tmpfs if enabled.
 551                 if self.settings.get("buildroot_tmpfs"):
 552                         mountpoints += [
 553                                 ("pakfire_root", "/", "tmpfs", "defaults"),
 554                         ]
 555
 556                 mountpoints += [
 557                         # src, dest, fs, options
 558                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 559                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 560                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 561                         ("/sys",          "/sys",      "bind",  "bind"),
 562                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 563                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 564                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 565                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 566                 ]
 567
 568                 # If selinux is enabled.
 569                 if os.path.exists("/sys/fs/selinux"):
 570                         mountpoints += [
 571                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 572                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 573                         ]
 574
 575                 # If ccache support is requested, we bind mount the cache.
 576                 if self.settings.get("enable_ccache"):
 577                         # Create ccache cache directory if it does not exist.
 578                         if not os.path.exists(CCACHE_CACHE_DIR):
 579                                 os.makedirs(CCACHE_CACHE_DIR)
 580
 581                         mountpoints += [
 582                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 583                         ]
 584
 585                 return mountpoints
 586
 587         @property
 588         def environ(self):
 589                 env = {
 590                         # Add HOME manually, because it is occasionally not set
 591                         # and some builds get in trouble then.
 592                         "PATH" : "/usr/bin:/bin:/usr/sbin:/sbin",
 593                         "HOME" : "/root",
 594                         "TERM" : os.environ.get("TERM", "vt100"),
 595                         "PS1"  : "\u:\w\$ ",
 596
 597                         # Sanitize language.
 598                         "LANG" : os.environ.setdefault("LANG", "en_US.UTF-8"),
 599
 600                         # Set the container that we can detect, if we are inside a
 601                         # chroot.
 602                         "container" : "pakfire-builder",
 603                 }
 604
 605                 # Inherit environment from distro
 606                 env.update(self.pakfire.distro.environ)
 607
 608                 # Icecream environment settings
 609                 if self.settings.get("enable_icecream", False):
 610                         # Set the toolchain path
 611                         if self.settings.get("icecream_toolchain", None):
 612                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 613
 614                         # Set preferred host if configured.
 615                         if self.settings.get("icecream_preferred_host", None):
 616                                 env["ICECC_PREFERRED_HOST"] = \
 617                                         self.settings.get("icecream_preferred_host")
 618
 619                 # Fake UTS_MACHINE, when we cannot use the personality syscall and
 620                 # if the host architecture is not equal to the target architecture.
 621                 if not self.pakfire.distro.personality and \
 622                                 not system.native_arch == self.pakfire.distro.arch:
 623                         env.update({
 624                                 "LD_PRELOAD"  : "/usr/lib/libpakfire_preload.so",
 625                                 "UTS_MACHINE" : self.pakfire.distro.arch,
 626                         })
 627
 628                 return env
 629
 630         @property
 631         def installed_packages(self):
 632                 """
 633                         Returns an iterator over all installed packages in this build environment.
 634                 """
 635                 # Get the repository of all installed packages.
 636                 repo = self.pakfire.repos.get_repo("@system")
 637
 638                 # Return an iterator over the packages.
 639                 return iter(repo)
 640
 641         def write_config(self):
 642                 # Cleanup everything in /etc/pakfire.
 643                 util.rm(self.chrootPath(CONFIG_DIR))
 644
 645                 for i in (CONFIG_DIR, CONFIG_REPOS_DIR):
 646                         i = self.chrootPath(i)
 647                         if not os.path.exists(i):
 648                                 os.makedirs(i)
 649
 650                 # Write general.conf.
 651                 f = open(self.chrootPath(CONFIG_DIR, "general.conf"), "w")
 652                 f.close()
 653
 654                 # Write builder.conf.
 655                 f = open(self.chrootPath(CONFIG_DIR, "builder.conf"), "w")
 656                 f.write(self.distro.get_config())
 657                 f.close()
 658
 659                 # Create pakfire configuration files.
 660                 for repo in self.pakfire.repos:
 661                         conf = repo.get_config()
 662
 663                         if not conf:
 664                                 continue
 665
 666                         filename = self.chrootPath(CONFIG_REPOS_DIR, "%s.repo" % repo.name)
 667                         f = open(filename, "w")
 668                         f.write("\n".join(conf))
 669                         f.close()
 670
 671         @property
 672         def pkg_makefile(self):
 673                 return os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION))
 674
 675         def execute(self, command, logger=None, **kwargs):
 676                 """
 677                         Executes the given command in the build chroot.
 678                 """
 679                 # Environment variables
 680                 env = self.environ
 681
 682                 if kwargs.has_key("env"):
 683                         env.update(kwargs.pop("env"))
 684
 685                 self.log.debug("Environment:")
 686                 for k, v in sorted(env.items()):
 687                         self.log.debug("  %s=%s" % (k, v))
 688
 689                 # Make every shell to a login shell because we set a lot of
 690                 # environment things there.
 691                 command = ["bash", "--login", "-c", command]
 692
 693                 args = {
 694                         "chroot_path" : self.chrootPath(),
 695                         "cgroup"      : self.cgroup,
 696                         "env"         : env,
 697                         "logger"      : logger,
 698                         "personality" : self.personality,
 699                         "shell"       : False,
 700                 }
 701                 args.update(kwargs)
 702
 703                 # Run the shit.
 704                 shellenv = shell.ShellExecuteEnvironment(command, **args)
 705                 shellenv.execute()
 706
 707                 return shellenv
 708
 709         def execute_root(self, command, **kwargs):
 710                 """
 711                         Executes the given command outside the build chroot.
 712                 """
 713                 shellenv = shell.ShellExecuteEnvironment(command, **kwargs)
 714                 shellenv.execute()
 715
 716                 return shellenv
 717
 718         def build(self, install_test=True, prepare=False):
 719                 if not self.pkg:
 720                         raise BuildError, _("You cannot run a build when no package was given.")
 721
 722                 # Search for the package file in build_dir and raise BuildError if it is not present.
 723                 if not os.path.exists(self.pkg_makefile):
 724                         raise BuildError, _("Could not find makefile in build root: %s") % self.pkg_makefile
 725
 726                 # Write pakfire configuration into the chroot.
 727                 self.write_config()
 728
 729                 # Create the build command, that is executed in the chroot.
 730                 build_command = [
 731                         "/usr/lib/pakfire/builder",
 732                         "--offline",
 733                         "build",
 734                         "/%s" % os.path.relpath(self.pkg_makefile, self.chrootPath()),
 735                         "--arch", self.arch,
 736                         "--nodeps",
 737                         "--resultdir=/result",
 738                 ]
 739
 740                 # Check if only the preparation stage should be run.
 741                 if prepare:
 742                         build_command.append("--prepare")
 743
 744                 build_command = " ".join(build_command)
 745
 746                 error = False
 747                 try:
 748                         self.execute(build_command, logger=self.log)
 749
 750                         # Perform the install test after the actual build.
 751                         if install_test and not prepare:
 752                                 self.install_test()
 753
 754                 except ShellEnvironmentError:
 755                         error = True
 756                         self.log.error(_("Build failed"))
 757
 758                 # Catch all other errors.
 759                 except:
 760                         error = True
 761                         self.log.error(_("Build failed."), exc_info=True)
 762
 763                 else:
 764                         # Don't sign packages in prepare mode.
 765                         if prepare:
 766                                 return
 767
 768                         # Sign all built packages with the host key (if available).
 769                         self.sign_packages()
 770
 771                         # Dump package information.
 772                         self.dump()
 773
 774                         return
 775
 776                 # End here in case of an error.
 777                 raise BuildError, _("The build command failed. See logfile for details.")
 778
 779         def install_test(self):
 780                 self.log.info(_("Running installation test..."))
 781
 782                 # Install all packages that were built.
 783                 self.install(self.find_result_packages(),
 784                         allow_vendorchange=True, allow_archchange=True,
 785                         allow_uninstall=True, signatures_mode="disabled")
 786
 787                 self.log.info(_("Installation test succeeded."))
 788                 self.log.info("")
 789
 790         def shell(self, args=[]):
 791                 if not util.cli_is_interactive():
 792                         self.log.warning("Cannot run shell on non-interactive console.")
 793                         return
 794
 795                 # Install all packages that are needed to run a shell.
 796                 self.install(SHELL_PACKAGES)
 797
 798                 # XXX need to set CFLAGS here
 799                 command = "/usr/sbin/chroot %s %s %s" % \
 800                         (self.chrootPath(), SHELL_SCRIPT, " ".join(args))
 801
 802                 # Add personality if we require one
 803                 if self.pakfire.distro.personality:
 804                         command = "%s %s" % (self.pakfire.distro.personality, command)
 805
 806                 for key, val in self.environ.items():
 807                         command = "%s=\"%s\" " % (key, val) + command
 808
 809                 # Empty the environment
 810                 command = "env -i - %s" % command
 811
 812                 self.log.debug("Shell command: %s" % command)
 813
 814                 shell = os.system(command)
 815                 return os.WEXITSTATUS(shell)
 816
 817         def sign_packages(self, keyfp=None):
 818                 # Do nothing if signing is not requested.
 819                 if not self.settings.get("sign_packages"):
 820                         return
 821
 822                 # Get key, that should be used for signing.
 823                 if not keyfp:
 824                         keyfp = self.keyring.get_host_key_id()
 825
 826                 # Find all files to process.
 827                 files = self.find_result_packages()
 828
 829                 # Create a progressbar.
 830                 print _("Signing packages...")
 831                 p = util.make_progress(keyfp, len(files))
 832                 i = 0
 833
 834                 for file in files:
 835                         # Update progressbar.
 836                         if p:
 837                                 i += 1
 838                                 p.update(i)
 839
 840                         # Open package file.
 841                         pkg = packages.open(self.pakfire, None, file)
 842
 843                         # Sign it.
 844                         pkg.sign(keyfp)
 845
 846                 # Close progressbar.
 847                 if p:
 848                         p.finish()
 849                         print "" # Print an empty line.
 850
 851         def dump(self):
 852                 pkgs = []
 853
 854                 for file in self.find_result_packages():
 855                         pkg = packages.open(self.pakfire, None, file)
 856                         pkgs.append(pkg)
 857
 858                 # If there are no packages, there is nothing to do.
 859                 if not pkgs:
 860                         return
 861
 862                 pkgs.sort()
 863
 864                 self.log.info(_("Dumping package information:"))
 865                 for pkg in pkgs:
 866                         dump = pkg.dump(long=True)
 867
 868                         for line in dump.splitlines():
 869                                 self.log.info("  %s" % line)
 870                         self.log.info("") # Empty line.
 871
 872
 873 class Builder(object):
 874         def __init__(self, pakfire, filename, resultdir, **kwargs):
 875                 self.pakfire = pakfire
 876
 877                 self.filename = filename
 878
 879                 self.resultdir = resultdir
 880
 881                 # Open package file.
 882                 self.pkg = packages.Makefile(self.pakfire, self.filename)
 883
 884                 self._environ = {
 885                         "LANG"             : "C",
 886                 }
 887
 888         @property
 889         def buildroot(self):
 890                 return self.pkg.buildroot
 891
 892         @property
 893         def distro(self):
 894                 return self.pakfire.distro
 895
 896         @property
 897         def environ(self):
 898                 environ = os.environ
 899
 900                 # Get all definitions from the package.
 901                 environ.update(self.pkg.exports)
 902
 903                 # Overwrite some definitions by default values.
 904                 environ.update(self._environ)
 905
 906                 return environ
 907
 908         def execute(self, command, logger=None, **kwargs):
 909                 if logger is None:
 910                         logger = logging.getLogger("pakfire")
 911
 912                 # Make every shell to a login shell because we set a lot of
 913                 # environment things there.
 914                 command = ["bash", "--login", "-c", command]
 915
 916                 args = {
 917                         "cwd"         : "/%s" % LOCAL_TMP_PATH,
 918                         "env"         : self.environ,
 919                         "logger"      : logger,
 920                         "personality" : self.distro.personality,
 921                         "shell"       : False,
 922                 }
 923                 args.update(kwargs)
 924
 925                 try:
 926                         shellenv = shell.ShellExecuteEnvironment(command, **args)
 927                         shellenv.execute()
 928
 929                 except ShellEnvironmentError:
 930                         logger.error("Command exited with an error: %s" % command)
 931                         raise
 932
 933                 return shellenv
 934
 935         def run_script(self, script, *args):
 936                 if not script.startswith("/"):
 937                         script = os.path.join(SCRIPT_DIR, script)
 938
 939                 assert os.path.exists(script), "Script we should run does not exist: %s" % script
 940
 941                 cmd = [script,]
 942                 for arg in args:
 943                         cmd.append(arg)
 944                 cmd = " ".join(cmd)
 945
 946                 # Returns the output of the command, but the output won't get
 947                 # logged.
 948                 exe = self.execute(cmd, record_output=True, log_output=False)
 949
 950                 # Return the output of the command.
 951                 if exe.exitcode == 0:
 952                         return exe.output
 953
 954         def create_icecream_toolchain(self):
 955                 try:
 956                         exe = self.execute(
 957                                 "icecc --build-native 2>/dev/null",
 958                                 record_output=True, record_stderr=False,
 959                                 log_output=False, log_errors=False,
 960                                 cwd="/tmp",
 961                         )
 962                 except ShellEnvironmentError:
 963                         return
 964
 965                 for line in exe.output.splitlines():
 966                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
 967                         if m:
 968                                 self._environ["ICECC_VERSION"] = "/tmp/%s" % m.group(1)
 969
 970         def create_buildscript(self, stage):
 971                 # Get buildscript from the package.
 972                 script = self.pkg.get_buildscript(stage)
 973
 974                 # Write script to an empty file.
 975                 f = tempfile.NamedTemporaryFile(mode="w", delete=False)
 976                 f.write("#!/bin/sh\n\n")
 977                 f.write("set -e\n")
 978                 f.write("set -x\n")
 979                 f.write("\n%s\n" % script)
 980                 f.write("exit 0\n")
 981                 f.close()
 982
 983                 # Make the script executable.
 984                 os.chmod(f.name, 700)
 985
 986                 return f.name
 987
 988         def build(self, stages=None):
 989                 # Create buildroot and remove all content if it was existant.
 990                 util.rm(self.buildroot)
 991                 os.makedirs(self.buildroot)
 992
 993                 # Build icecream toolchain if icecream is installed.
 994                 self.create_icecream_toolchain()
 995
 996                 # Process stages in order.
 997                 for stage in ("prepare", "build", "test", "install"):
 998                         # Skip unwanted stages.
 999                         if stages and not stage in stages:
1000                                 continue
1001
1002                         # Run stage.
1003                         self.build_stage(stage)
1004
1005                 # Stop if install stage has not been processed.
1006                 if stages and not "install" in stages:
1007                         return
1008
1009                 # Run post-build stuff.
1010                 self.post_compress_man_pages()
1011                 self.post_remove_static_libs()
1012                 self.post_extract_debuginfo()
1013
1014                 # Package the result.
1015                 # Make all these little package from the build environment.
1016                 log.info(_("Creating packages:"))
1017                 pkgs = []
1018                 for pkg in reversed(self.pkg.packages):
1019                         packager = packages.packager.BinaryPackager(self.pakfire, pkg,
1020                                 self, self.buildroot)
1021                         pkg = packager.run(self.resultdir)
1022                         pkgs.append(pkg)
1023                 log.info("")
1024
1025         def build_stage(self, stage):
1026                 # Get the buildscript for this stage.
1027                 buildscript = self.create_buildscript(stage)
1028
1029                 # Execute the buildscript of this stage.
1030                 log.info(_("Running stage %s:") % stage)
1031
1032                 try:
1033                         self.execute(buildscript)
1034
1035                 finally:
1036                         # Remove the buildscript.
1037                         if os.path.exists(buildscript):
1038                                 os.unlink(buildscript)
1039
1040         def post_remove_static_libs(self):
1041                 keep_libs = self.pkg.lexer.build.get_var("keep_libraries")
1042                 keep_libs = keep_libs.split()
1043
1044                 try:
1045                         self.execute("%s/remove-static-libs %s %s" % \
1046                                 (SCRIPT_DIR, self.buildroot, " ".join(keep_libs)))
1047                 except ShellEnvironmentError, e:
1048                         log.warning(_("Could not remove static libraries: %s") % e)
1049
1050         def post_compress_man_pages(self):
1051                 try:
1052                         self.execute("%s/compress-man-pages %s" % (SCRIPT_DIR, self.buildroot))
1053                 except ShellEnvironmentError, e:
1054                         log.warning(_("Compressing man pages did not complete successfully."))
1055
1056         def post_extract_debuginfo(self):
1057                 args = []
1058
1059                 # Check if we need to run with strict build-id.
1060                 strict_id = self.pkg.lexer.build.get_var("debuginfo_strict_build_id", "true")
1061                 if strict_id in ("true", "yes", "1"):
1062                         args.append("--strict-build-id")
1063
1064                 args.append("--buildroot=%s" % self.pkg.buildroot)
1065                 args.append("--sourcedir=%s" % self.pkg.sourcedir)
1066
1067                 # Get additional options to pass to script.
1068                 options = self.pkg.lexer.build.get_var("debuginfo_options", "")
1069                 args += options.split()
1070
1071                 try:
1072                         self.execute("%s/extract-debuginfo %s %s" % (SCRIPT_DIR, " ".join(args), self.pkg.buildroot))
1073                 except ShellEnvironmentError, e:
1074                         log.error(_("Extracting debuginfo did not complete with success. Aborting build."))
1075                         raise
1076
1077         def find_prerequires(self, scriptlet_file):
1078                 assert os.path.exists(scriptlet_file), "Scriptlet file does not exist: %s" % scriptlet_file
1079
1080                 res = self.run_script("find-prerequires", scriptlet_file)
1081                 prerequires = set(res.splitlines())
1082
1083                 return prerequires
1084
1085         def cleanup(self):
1086                 if os.path.exists(self.buildroot):
1087                         util.rm(self.buildroot)