python/pakfire/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import math
  25 import os
  26 import re
  27 import shutil
  28 import signal
  29 import socket
  30 import tempfile
  31 import time
  32 import uuid
  33
  34 import base
  35 import cgroup
  36 import logger
  37 import packages
  38 import packages.file
  39 import packages.packager
  40 import repository
  41 import shell
  42 import util
  43 import _pakfire
  44
  45 import logging
  46 log = logging.getLogger("pakfire")
  47
  48 from config import ConfigBuilder
  49 from system import system
  50 from constants import *
  51 from i18n import _
  52 from errors import BuildError, BuildRootLocked, Error
  53
  54
  55 BUILD_LOG_HEADER = """
  56  ____       _     __ _            _           _ _     _
  57 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  58 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  59 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  60 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  61
  62         Version : %(version)s
  63         Host    : %(hostname)s (%(host_arch)s)
  64         Time    : %(time)s
  65
  66 """
  67
  68 class BuildEnviron(object):
  69         # The version of the kernel this machine is running.
  70         kernel_version = os.uname()[2]
  71
  72         def __init__(self, pakfire, filename=None, distro_name=None, build_id=None, logfile=None, release_build=True, **kwargs):
  73                 self.pakfire = pakfire
  74
  75                 # Check if the given pakfire instance is of the correct type.
  76                 assert isinstance(self.pakfire, base.PakfireBuilder)
  77
  78                 # Check if this host can build the requested architecture.
  79                 if not system.host_supports_arch(self.arch):
  80                         raise BuildError, _("Cannot build for %s on this host.") % self.arch
  81
  82                 # Save the build id and generate one if no build id was provided.
  83                 if not build_id:
  84                         build_id = "%s" % uuid.uuid4()
  85
  86                 self.build_id = build_id
  87
  88                 # Setup the logging.
  89                 self.init_logging(logfile)
  90
  91                 # Initialize a cgroup (if supported).
  92                 self.init_cgroup()
  93
  94                 # This build is a release build?
  95                 self.release_build = release_build
  96
  97                 if self.release_build:
  98                         # Disable the local build repository in release mode.
  99                         self.pakfire.repos.disable_repo("build")
 100
 101                         # Log information about pakfire and some more information, when we
 102                         # are running in release mode.
 103                         logdata = {
 104                                 "host_arch"  : system.arch,
 105                                 "hostname"   : system.hostname,
 106                                 "time"       : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
 107                                 "version"    : "Pakfire %s" % PAKFIRE_VERSION,
 108                         }
 109
 110                         for line in BUILD_LOG_HEADER.splitlines():
 111                                 self.log.info(line % logdata)
 112
 113                 # Settings array.
 114                 self.settings = {
 115                         "enable_loop_devices" : self.config.get_bool("builder", "use_loop_devices", True),
 116                         "enable_ccache"       : self.config.get_bool("builder", "use_ccache", True),
 117                         "enable_icecream"     : self.config.get_bool("builder", "use_icecream", False),
 118                         "sign_packages"       : False,
 119                         "buildroot_tmpfs"     : self.config.get_bool("builder", "use_tmpfs", False),
 120                         "private_network"     : self.config.get_bool("builder", "private_network", False),
 121                 }
 122
 123                 # Get ccache settings.
 124                 if self.settings.get("enable_ccache", False):
 125                         self.settings.update({
 126                                 "ccache_compress" : self.config.get_bool("ccache", "compress", True),
 127                         })
 128
 129                 # Try to get the configured host key. If it is available,
 130                 # we will automatically sign all packages with it.
 131                 if self.keyring.get_host_key(secret=True):
 132                         self.settings["sign_packages"] = True
 133
 134                 # Add settings from keyword arguments.
 135                 self.settings.update(kwargs)
 136
 137                 # Where do we put the result?
 138                 self.resultdir = os.path.join(self.pakfire.path, "result")
 139
 140                 # Open package.
 141                 # If we have a plain makefile, we first build a source package and go with that.
 142                 if filename:
 143                         # Open source package.
 144                         self.pkg = packages.SourcePackage(self.pakfire, None, filename)
 145                         assert self.pkg, filename
 146
 147                         # Log the package information.
 148                         self.log.info(_("Package information:"))
 149                         for line in self.pkg.dump(long=True).splitlines():
 150                                 self.log.info("  %s" % line)
 151                         self.log.info("")
 152
 153                         # Path where we extract the package and put all the source files.
 154                         self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name)
 155                 else:
 156                         # No package :(
 157                         self.pkg = None
 158
 159                 # Lock the buildroot
 160                 self._lock = None
 161
 162                 # Save the build time.
 163                 self.build_time = time.time()
 164
 165         def setup_signal_handlers(self):
 166                 pass
 167
 168         def start(self):
 169                 assert not self.pakfire.initialized, "Pakfire has already been initialized"
 170
 171                 # Unshare namepsace.
 172                 # If this fails because the kernel has no support for CLONE_NEWIPC or CLONE_NEWUTS,
 173                 # we try to fall back to just set CLONE_NEWNS.
 174                 try:
 175                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNS|_pakfire.SCHED_CLONE_NEWIPC|_pakfire.SCHED_CLONE_NEWUTS)
 176                 except RuntimeError, e:
 177                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNS)
 178
 179                 # Mount the directories.
 180                 self._mountall()
 181
 182                 # Lock the build environment.
 183                 self.lock()
 184
 185                 # Initialize pakfire instance.
 186                 self.pakfire.initialize()
 187
 188                 # Optionally enable private networking.
 189                 if self.settings.get("private_network", None):
 190                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNET)
 191
 192                 # Populate /dev.
 193                 self.populate_dev()
 194
 195                 # Setup domain name resolution in chroot.
 196                 self.setup_dns()
 197
 198                 # Extract all needed packages.
 199                 self.extract()
 200
 201         def stop(self):
 202                 if self.cgroup:
 203                         # Kill all still running processes in the cgroup.
 204                         self.cgroup.kill_and_wait()
 205
 206                         # Remove cgroup and all parent cgroups if they are empty.
 207                         self.cgroup.migrate_task(self.cgroup.root, os.getpid())
 208                         self.cgroup.destroy()
 209
 210                         parent = self.cgroup.parent
 211                         while parent:
 212                                 if not parent.is_empty(recursive=True):
 213                                         break
 214
 215                                 parent.destroy()
 216                                 parent = parent.parent
 217
 218                 else:
 219                         util.orphans_kill(self.path)
 220
 221                 # Shut down pakfire instance.
 222                 self.pakfire.destroy()
 223
 224                 # Unlock build environment.
 225                 self.unlock()
 226
 227                 # Umount the build environment.
 228                 self._umountall()
 229
 230                 # Remove all files.
 231                 self.destroy()
 232
 233         @property
 234         def config(self):
 235                 """
 236                         Proxy method for easy access to the configuration.
 237                 """
 238                 return self.pakfire.config
 239
 240         @property
 241         def distro(self):
 242                 """
 243                         Proxy method for easy access to the distribution.
 244                 """
 245                 return self.pakfire.distro
 246
 247         @property
 248         def path(self):
 249                 """
 250                         Proxy method for easy access to the path.
 251                 """
 252                 return self.pakfire.path
 253
 254         @property
 255         def arch(self):
 256                 """
 257                         Inherit architecture from distribution configuration.
 258                 """
 259                 return self.pakfire.distro.arch
 260
 261         @property
 262         def personality(self):
 263                 """
 264                         Gets the personality from the distribution configuration.
 265                 """
 266                 return self.pakfire.distro.personality
 267
 268         @property
 269         def info(self):
 270                 return {
 271                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 272                         "build_host" : socket.gethostname(),
 273                         "build_id"   : self.build_id,
 274                         "build_time" : self.build_time,
 275                 }
 276
 277         @property
 278         def keyring(self):
 279                 """
 280                         Shortcut to access the pakfire keyring.
 281                 """
 282                 return self.pakfire.keyring
 283
 284         def lock(self):
 285                 filename = os.path.join(self.path, ".lock")
 286
 287                 try:
 288                         self._lock = open(filename, "a+")
 289                 except IOError, e:
 290                         return 0
 291
 292                 try:
 293                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 294                 except IOError, e:
 295                         raise BuildRootLocked, "Buildroot is locked"
 296
 297                 return 1
 298
 299         def unlock(self):
 300                 if self._lock:
 301                         self._lock.close()
 302                         self._lock = None
 303
 304         def init_cgroup(self):
 305                 """
 306                         Initialize cgroup (if the system supports it).
 307                 """
 308                 if not cgroup.supported():
 309                         self.cgroup = None
 310                         return
 311
 312                 # Search for the cgroup this process is currently running in.
 313                 parent_cgroup = cgroup.find_by_pid(os.getpid())
 314                 if not parent_cgroup:
 315                         return
 316
 317                 # Create our own cgroup inside the parent cgroup.
 318                 self.cgroup = parent_cgroup.create_child_cgroup("pakfire/builder/%s" % self.build_id)
 319
 320                 # Attach the pakfire-builder process to the group.
 321                 self.cgroup.attach()
 322
 323         def init_logging(self, logfile):
 324                 if logfile:
 325                         self.log = log.getChild(self.build_id)
 326                         # Propage everything to the root logger that we will see something
 327                         # on the terminal.
 328                         self.log.propagate = 1
 329                         self.log.setLevel(logging.INFO)
 330
 331                         # Add the given logfile to the logger.
 332                         h = logging.FileHandler(logfile)
 333                         self.log.addHandler(h)
 334
 335                         # Format the log output for the file.
 336                         f = logger.BuildFormatter()
 337                         h.setFormatter(f)
 338                 else:
 339                         # If no logile was given, we use the root logger.
 340                         self.log = logging.getLogger("pakfire")
 341
 342         def copyin(self, file_out, file_in):
 343                 if file_in.startswith("/"):
 344                         file_in = file_in[1:]
 345
 346                 file_in = self.chrootPath(file_in)
 347
 348                 #if not os.path.exists(file_out):
 349                 #       return
 350
 351                 dir_in = os.path.dirname(file_in)
 352                 if not os.path.exists(dir_in):
 353                         os.makedirs(dir_in)
 354
 355                 self.log.debug("%s --> %s" % (file_out, file_in))
 356
 357                 shutil.copy2(file_out, file_in)
 358
 359         def copyout(self, file_in, file_out):
 360                 if file_in.startswith("/"):
 361                         file_in = file_in[1:]
 362
 363                 file_in = self.chrootPath(file_in)
 364
 365                 #if not os.path.exists(file_in):
 366                 #       return
 367
 368                 dir_out = os.path.dirname(file_out)
 369                 if not os.path.exists(dir_out):
 370                         os.makedirs(dir_out)
 371
 372                 self.log.debug("%s --> %s" % (file_in, file_out))
 373
 374                 shutil.copy2(file_in, file_out)
 375
 376         def copy_result(self, resultdir):
 377                 # XXX should use find_result_packages
 378
 379                 dir_in = self.chrootPath("result")
 380
 381                 for dir, subdirs, files in os.walk(dir_in):
 382                         basename = os.path.basename(dir)
 383                         dir = dir[len(self.chrootPath()):]
 384                         for file in files:
 385                                 file_in = os.path.join(dir, file)
 386
 387                                 file_out = os.path.join(
 388                                         resultdir,
 389                                         basename,
 390                                         file,
 391                                 )
 392
 393                                 self.copyout(file_in, file_out)
 394
 395         def find_result_packages(self):
 396                 ret = []
 397
 398                 for dir, subdirs, files in os.walk(self.resultdir):
 399                         for file in files:
 400                                 if not file.endswith(".%s" % PACKAGE_EXTENSION):
 401                                         continue
 402
 403                                 file = os.path.join(dir, file)
 404                                 ret.append(file)
 405
 406                 return ret
 407
 408         def extract(self, requires=None):
 409                 """
 410                         Gets a dependency set and extracts all packages
 411                         to the environment.
 412                 """
 413                 if not requires:
 414                         requires = []
 415
 416                 # Add neccessary build dependencies.
 417                 requires += BUILD_PACKAGES
 418
 419                 # If we have ccache enabled, we need to extract it
 420                 # to the build chroot.
 421                 if self.settings.get("enable_ccache"):
 422                         requires.append("ccache")
 423
 424                 # If we have icecream enabled, we need to extract it
 425                 # to the build chroot.
 426                 if self.settings.get("enable_icecream"):
 427                         requires.append("icecream")
 428
 429                 # Get build dependencies from source package.
 430                 if self.pkg:
 431                         for req in self.pkg.requires:
 432                                 requires.append(req)
 433
 434                 # Install all packages.
 435                 self.log.info(_("Install packages needed for build..."))
 436                 self.install(requires)
 437
 438                 # Copy the makefile and load source tarballs.
 439                 if self.pkg:
 440                         self.pkg.extract(_("Extracting"), prefix=self.build_dir)
 441
 442                         # Add an empty line at the end.
 443                         self.log.info("")
 444
 445         def install(self, requires, **kwargs):
 446                 """
 447                         Install everything that is required in requires.
 448                 """
 449                 # If we got nothing to do, we quit immediately.
 450                 if not requires:
 451                         return
 452
 453                 kwargs.update({
 454                         "interactive" : False,
 455                         "logger" : self.log,
 456                 })
 457
 458                 if not kwargs.has_key("allow_downgrade"):
 459                         kwargs["allow_downgrade"] = True
 460
 461                 # Install everything.
 462                 self.pakfire.install(requires, **kwargs)
 463
 464         def chrootPath(self, *args):
 465                 # Remove all leading slashes
 466                 _args = []
 467                 for arg in args:
 468                         if arg.startswith("/"):
 469                                 arg = arg[1:]
 470                         _args.append(arg)
 471                 args = _args
 472
 473                 ret = os.path.join(self.path, *args)
 474                 ret = ret.replace("//", "/")
 475
 476                 assert ret.startswith(self.path)
 477
 478                 return ret
 479
 480         def populate_dev(self):
 481                 nodes = [
 482                         "/dev/null",
 483                         "/dev/zero",
 484                         "/dev/full",
 485                         "/dev/random",
 486                         "/dev/urandom",
 487                         "/dev/tty",
 488                         "/dev/ptmx",
 489                         "/dev/kmsg",
 490                         "/dev/rtc0",
 491                         "/dev/console",
 492                 ]
 493
 494                 # If we need loop devices (which are optional) we create them here.
 495                 if self.settings["enable_loop_devices"]:
 496                         for i in range(0, 7):
 497                                 nodes.append("/dev/loop%d" % i)
 498
 499                 for node in nodes:
 500                         # Stat the original node of the host system and copy it to
 501                         # the build chroot.
 502                         node_stat = os.stat(node)
 503
 504                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 505
 506                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 507                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 508                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 509                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 510
 511         def setup_dns(self):
 512                 """
 513                         Add DNS resolution facility to chroot environment by copying
 514                         /etc/resolv.conf and /etc/hosts.
 515                 """
 516                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 517                         self.copyin(i, i)
 518
 519         def _create_node(self, filename, mode, device):
 520                 self.log.debug("Create node: %s (%s)" % (filename, mode))
 521
 522                 filename = self.chrootPath(filename)
 523
 524                 # Create parent directory if it is missing.
 525                 dirname = os.path.dirname(filename)
 526                 if not os.path.exists(dirname):
 527                         os.makedirs(dirname)
 528
 529                 os.mknod(filename, mode, device)
 530
 531         def destroy(self):
 532                 self.log.debug("Destroying environment %s" % self.path)
 533
 534                 if os.path.exists(self.path):
 535                         util.rm(self.path)
 536
 537         def cleanup(self):
 538                 self.log.debug("Cleaning environemnt.")
 539
 540                 # Remove the build directory and buildroot.
 541                 dirs = (self.build_dir, self.chrootPath("result"),)
 542
 543                 for d in dirs:
 544                         if not os.path.exists(d):
 545                                 continue
 546
 547                         util.rm(d)
 548                         os.makedirs(d)
 549
 550         def _mountall(self):
 551                 self.log.debug("Mounting environment")
 552                 for src, dest, fs, options in self.mountpoints:
 553                         mountpoint = self.chrootPath(dest)
 554                         if options:
 555                                 options = "-o %s" % options
 556
 557                         # Eventually create mountpoint directory
 558                         if not os.path.exists(mountpoint):
 559                                 os.makedirs(mountpoint)
 560
 561                         self.execute_root("mount -n -t %s %s %s %s" % (fs, options, src, mountpoint), shell=True)
 562
 563         def _umountall(self):
 564                 self.log.debug("Umounting environment")
 565
 566                 mountpoints = []
 567                 for src, dest, fs, options in reversed(self.mountpoints):
 568                         dest = self.chrootPath(dest)
 569
 570                         if not dest in mountpoints:
 571                                 mountpoints.append(dest)
 572
 573                 while mountpoints:
 574                         for mp in mountpoints:
 575                                 try:
 576                                         self.execute_root("umount -n %s" % mp, shell=True)
 577                                 except ShellEnvironmentError:
 578                                         pass
 579
 580                                 if not os.path.ismount(mp):
 581                                         mountpoints.remove(mp)
 582
 583         @property
 584         def mountpoints(self):
 585                 mountpoints = []
 586
 587                 # Make root as a tmpfs if enabled.
 588                 if self.settings.get("buildroot_tmpfs"):
 589                         mountpoints += [
 590                                 ("pakfire_root", "/", "tmpfs", "defaults"),
 591                         ]
 592
 593                 mountpoints += [
 594                         # src, dest, fs, options
 595                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 596                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 597                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 598                         ("/sys",          "/sys",      "bind",  "bind"),
 599                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 600                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 601                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 602                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 603                         ("pakfire_tmpfs", "/tmp",      "tmpfs", "mode=755,nosuid,nodev"),
 604                 ]
 605
 606                 # If selinux is enabled.
 607                 if os.path.exists("/sys/fs/selinux"):
 608                         mountpoints += [
 609                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 610                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 611                         ]
 612
 613                 # If ccache support is requested, we bind mount the cache.
 614                 if self.settings.get("enable_ccache"):
 615                         # Create ccache cache directory if it does not exist.
 616                         if not os.path.exists(CCACHE_CACHE_DIR):
 617                                 os.makedirs(CCACHE_CACHE_DIR)
 618
 619                         mountpoints += [
 620                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 621                         ]
 622
 623                 return mountpoints
 624
 625         @property
 626         def environ(self):
 627                 env = {
 628                         # Add HOME manually, because it is occasionally not set
 629                         # and some builds get in trouble then.
 630                         "PATH" : "/usr/bin:/bin:/usr/sbin:/sbin",
 631                         "HOME" : "/root",
 632                         "TERM" : os.environ.get("TERM", "vt100"),
 633                         "PS1"  : "\u:\w\$ ",
 634
 635                         # Sanitize language.
 636                         "LANG" : os.environ.setdefault("LANG", "en_US.UTF-8"),
 637
 638                         # Set the container that we can detect, if we are inside a
 639                         # chroot.
 640                         "container" : "pakfire-builder",
 641                 }
 642
 643                 # Inherit environment from distro
 644                 env.update(self.pakfire.distro.environ)
 645
 646                 # ccache environment settings
 647                 if self.settings.get("enable_ccache", False):
 648                         compress = self.settings.get("ccache_compress", False)
 649                         if compress:
 650                                 env["CCACHE_COMPRESS"] = "1"
 651
 652                         # Let ccache create its temporary files in /tmp.
 653                         env["CCACHE_TEMPDIR"] = "/tmp"
 654
 655                 # Icecream environment settings
 656                 if self.settings.get("enable_icecream", False):
 657                         # Set the toolchain path
 658                         if self.settings.get("icecream_toolchain", None):
 659                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 660
 661                         # Set preferred host if configured.
 662                         if self.settings.get("icecream_preferred_host", None):
 663                                 env["ICECC_PREFERRED_HOST"] = \
 664                                         self.settings.get("icecream_preferred_host")
 665
 666                 # Fake UTS_MACHINE, when we cannot use the personality syscall and
 667                 # if the host architecture is not equal to the target architecture.
 668                 if not self.pakfire.distro.personality and \
 669                                 not system.native_arch == self.pakfire.distro.arch:
 670                         env.update({
 671                                 "LD_PRELOAD"  : "/usr/lib/libpakfire_preload.so",
 672                                 "UTS_MACHINE" : self.pakfire.distro.arch,
 673                         })
 674
 675                 return env
 676
 677         @property
 678         def installed_packages(self):
 679                 """
 680                         Returns an iterator over all installed packages in this build environment.
 681                 """
 682                 # Get the repository of all installed packages.
 683                 repo = self.pakfire.repos.get_repo("@system")
 684
 685                 # Return an iterator over the packages.
 686                 return iter(repo)
 687
 688         def write_config(self):
 689                 # Cleanup everything in /etc/pakfire.
 690                 util.rm(self.chrootPath(CONFIG_DIR))
 691
 692                 for i in (CONFIG_DIR, CONFIG_REPOS_DIR):
 693                         i = self.chrootPath(i)
 694                         if not os.path.exists(i):
 695                                 os.makedirs(i)
 696
 697                 # Write general.conf.
 698                 f = open(self.chrootPath(CONFIG_DIR, "general.conf"), "w")
 699                 f.close()
 700
 701                 # Write builder.conf.
 702                 f = open(self.chrootPath(CONFIG_DIR, "builder.conf"), "w")
 703                 f.write(self.distro.get_config())
 704                 f.close()
 705
 706                 # Create pakfire configuration files.
 707                 for repo in self.pakfire.repos:
 708                         conf = repo.get_config()
 709
 710                         if not conf:
 711                                 continue
 712
 713                         filename = self.chrootPath(CONFIG_REPOS_DIR, "%s.repo" % repo.name)
 714                         f = open(filename, "w")
 715                         f.write("\n".join(conf))
 716                         f.close()
 717
 718         @property
 719         def pkg_makefile(self):
 720                 return os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION))
 721
 722         def execute(self, command, logger=None, **kwargs):
 723                 """
 724                         Executes the given command in the build chroot.
 725                 """
 726                 # Environment variables
 727                 env = self.environ
 728
 729                 if kwargs.has_key("env"):
 730                         env.update(kwargs.pop("env"))
 731
 732                 self.log.debug("Environment:")
 733                 for k, v in sorted(env.items()):
 734                         self.log.debug("  %s=%s" % (k, v))
 735
 736                 # Make every shell to a login shell because we set a lot of
 737                 # environment things there.
 738                 command = ["bash", "--login", "-c", command]
 739
 740                 args = {
 741                         "chroot_path" : self.chrootPath(),
 742                         "cgroup"      : self.cgroup,
 743                         "env"         : env,
 744                         "logger"      : logger,
 745                         "personality" : self.personality,
 746                         "shell"       : False,
 747                 }
 748                 args.update(kwargs)
 749
 750                 # Run the shit.
 751                 shellenv = shell.ShellExecuteEnvironment(command, **args)
 752                 shellenv.execute()
 753
 754                 return shellenv
 755
 756         def execute_root(self, command, **kwargs):
 757                 """
 758                         Executes the given command outside the build chroot.
 759                 """
 760                 shellenv = shell.ShellExecuteEnvironment(command, **kwargs)
 761                 shellenv.execute()
 762
 763                 return shellenv
 764
 765         def build(self, install_test=True, prepare=False):
 766                 if not self.pkg:
 767                         raise BuildError, _("You cannot run a build when no package was given.")
 768
 769                 # Search for the package file in build_dir and raise BuildError if it is not present.
 770                 if not os.path.exists(self.pkg_makefile):
 771                         raise BuildError, _("Could not find makefile in build root: %s") % self.pkg_makefile
 772
 773                 # Write pakfire configuration into the chroot.
 774                 self.write_config()
 775
 776                 # Create the build command, that is executed in the chroot.
 777                 build_command = [
 778                         "/usr/lib/pakfire/builder",
 779                         "--offline",
 780                         "build",
 781                         "/%s" % os.path.relpath(self.pkg_makefile, self.chrootPath()),
 782                         "--arch", self.arch,
 783                         "--nodeps",
 784                         "--resultdir=/result",
 785                 ]
 786
 787                 # Check if only the preparation stage should be run.
 788                 if prepare:
 789                         build_command.append("--prepare")
 790
 791                 build_command = " ".join(build_command)
 792
 793                 try:
 794                         self.execute(build_command, logger=self.log)
 795
 796                         # Perform the install test after the actual build.
 797                         if install_test and not prepare:
 798                                 self.install_test()
 799
 800                 except ShellEnvironmentError:
 801                         self.log.error(_("Build failed"))
 802
 803                 except KeyboardInterrupt:
 804                         self.log.error(_("Build interrupted"))
 805
 806                         raise
 807
 808                 # Catch all other errors.
 809                 except:
 810                         self.log.error(_("Build failed."), exc_info=True)
 811
 812                 else:
 813                         # Don't sign packages in prepare mode.
 814                         if prepare:
 815                                 return
 816
 817                         # Sign all built packages with the host key (if available).
 818                         self.sign_packages()
 819
 820                         # Dump package information.
 821                         self.dump()
 822
 823                         return
 824
 825                 # End here in case of an error.
 826                 raise BuildError, _("The build command failed. See logfile for details.")
 827
 828         def install_test(self):
 829                 self.log.info(_("Running installation test..."))
 830
 831                 # Install all packages that were built.
 832                 self.install(self.find_result_packages(), allow_vendorchange=True,
 833                         allow_uninstall=True, signatures_mode="disabled")
 834
 835                 self.log.info(_("Installation test succeeded."))
 836                 self.log.info("")
 837
 838         def shell(self, args=[]):
 839                 if not util.cli_is_interactive():
 840                         self.log.warning("Cannot run shell on non-interactive console.")
 841                         return
 842
 843                 # Install all packages that are needed to run a shell.
 844                 self.install(SHELL_PACKAGES)
 845
 846                 # XXX need to set CFLAGS here
 847                 command = "/usr/sbin/chroot %s %s %s" % \
 848                         (self.chrootPath(), SHELL_SCRIPT, " ".join(args))
 849
 850                 # Add personality if we require one
 851                 if self.pakfire.distro.personality:
 852                         command = "%s %s" % (self.pakfire.distro.personality, command)
 853
 854                 for key, val in self.environ.items():
 855                         command = "%s=\"%s\" " % (key, val) + command
 856
 857                 # Empty the environment
 858                 command = "env -i - %s" % command
 859
 860                 self.log.debug("Shell command: %s" % command)
 861
 862                 shell = os.system(command)
 863                 return os.WEXITSTATUS(shell)
 864
 865         def sign_packages(self, keyfp=None):
 866                 # Do nothing if signing is not requested.
 867                 if not self.settings.get("sign_packages"):
 868                         return
 869
 870                 # Get key, that should be used for signing.
 871                 if not keyfp:
 872                         keyfp = self.keyring.get_host_key_id()
 873
 874                 # Find all files to process.
 875                 files = self.find_result_packages()
 876
 877                 # Create a progressbar.
 878                 print _("Signing packages...")
 879                 p = util.make_progress(keyfp, len(files))
 880                 i = 0
 881
 882                 for file in files:
 883                         # Update progressbar.
 884                         if p:
 885                                 i += 1
 886                                 p.update(i)
 887
 888                         # Open package file.
 889                         pkg = packages.open(self.pakfire, None, file)
 890
 891                         # Sign it.
 892                         pkg.sign(keyfp)
 893
 894                 # Close progressbar.
 895                 if p:
 896                         p.finish()
 897                         print "" # Print an empty line.
 898
 899         def dump(self):
 900                 pkgs = []
 901
 902                 for file in self.find_result_packages():
 903                         pkg = packages.open(self.pakfire, None, file)
 904                         pkgs.append(pkg)
 905
 906                 # If there are no packages, there is nothing to do.
 907                 if not pkgs:
 908                         return
 909
 910                 pkgs.sort()
 911
 912                 self.log.info(_("Dumping package information:"))
 913                 for pkg in pkgs:
 914                         dump = pkg.dump(long=True)
 915
 916                         for line in dump.splitlines():
 917                                 self.log.info("  %s" % line)
 918                         self.log.info("") # Empty line.
 919
 920
 921 class Builder(object):
 922         def __init__(self, pakfire, filename, resultdir, **kwargs):
 923                 self.pakfire = pakfire
 924
 925                 self.filename = filename
 926
 927                 self.resultdir = resultdir
 928
 929                 # Open package file.
 930                 self.pkg = packages.Makefile(self.pakfire, self.filename)
 931
 932                 self._environ = {
 933                         "LANG"             : "C",
 934                 }
 935
 936         @property
 937         def buildroot(self):
 938                 return self.pkg.buildroot
 939
 940         @property
 941         def distro(self):
 942                 return self.pakfire.distro
 943
 944         @property
 945         def environ(self):
 946                 environ = os.environ
 947
 948                 # Get all definitions from the package.
 949                 environ.update(self.pkg.exports)
 950
 951                 # Overwrite some definitions by default values.
 952                 environ.update(self._environ)
 953
 954                 return environ
 955
 956         def execute(self, command, logger=None, **kwargs):
 957                 if logger is None:
 958                         logger = logging.getLogger("pakfire")
 959
 960                 # Make every shell to a login shell because we set a lot of
 961                 # environment things there.
 962                 command = ["bash", "--login", "-c", command]
 963
 964                 args = {
 965                         "cwd"         : "/%s" % LOCAL_TMP_PATH,
 966                         "env"         : self.environ,
 967                         "logger"      : logger,
 968                         "personality" : self.distro.personality,
 969                         "shell"       : False,
 970                 }
 971                 args.update(kwargs)
 972
 973                 try:
 974                         shellenv = shell.ShellExecuteEnvironment(command, **args)
 975                         shellenv.execute()
 976
 977                 except ShellEnvironmentError:
 978                         logger.error("Command exited with an error: %s" % command)
 979                         raise
 980
 981                 return shellenv
 982
 983         def run_script(self, script, *args):
 984                 if not script.startswith("/"):
 985                         script = os.path.join(SCRIPT_DIR, script)
 986
 987                 assert os.path.exists(script), "Script we should run does not exist: %s" % script
 988
 989                 cmd = [script,]
 990                 for arg in args:
 991                         cmd.append(arg)
 992                 cmd = " ".join(cmd)
 993
 994                 # Returns the output of the command, but the output won't get
 995                 # logged.
 996                 exe = self.execute(cmd, record_output=True, log_output=False)
 997
 998                 # Return the output of the command.
 999                 if exe.exitcode == 0:
1000                         return exe.output
1001
1002         def create_icecream_toolchain(self):
1003                 try:
1004                         exe = self.execute(
1005                                 "icecc --build-native 2>/dev/null",
1006                                 record_output=True, record_stderr=False,
1007                                 log_output=False, log_errors=False,
1008                                 cwd="/tmp",
1009                         )
1010                 except ShellEnvironmentError:
1011                         return
1012
1013                 for line in exe.output.splitlines():
1014                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
1015                         if m:
1016                                 self._environ["ICECC_VERSION"] = "/tmp/%s" % m.group(1)
1017
1018         def create_buildscript(self, stage):
1019                 # Get buildscript from the package.
1020                 script = self.pkg.get_buildscript(stage)
1021
1022                 # Write script to an empty file.
1023                 f = tempfile.NamedTemporaryFile(mode="w", delete=False)
1024                 f.write("#!/bin/sh\n\n")
1025                 f.write("set -e\n")
1026                 f.write("set -x\n")
1027                 f.write("\n%s\n" % script)
1028                 f.write("exit 0\n")
1029                 f.close()
1030
1031                 # Make the script executable.
1032                 os.chmod(f.name, 700)
1033
1034                 return f.name
1035
1036         def build(self, stages=None):
1037                 # Create buildroot and remove all content if it was existant.
1038                 util.rm(self.buildroot)
1039                 os.makedirs(self.buildroot)
1040
1041                 # Build icecream toolchain if icecream is installed.
1042                 self.create_icecream_toolchain()
1043
1044                 # Process stages in order.
1045                 for stage in ("prepare", "build", "test", "install"):
1046                         # Skip unwanted stages.
1047                         if stages and not stage in stages:
1048                                 continue
1049
1050                         # Run stage.
1051                         self.build_stage(stage)
1052
1053                 # Stop if install stage has not been processed.
1054                 if stages and not "install" in stages:
1055                         return
1056
1057                 # Run post-build stuff.
1058                 self.post_compress_man_pages()
1059                 self.post_remove_static_libs()
1060                 self.post_extract_debuginfo()
1061
1062                 # Package the result.
1063                 # Make all these little package from the build environment.
1064                 log.info(_("Creating packages:"))
1065                 pkgs = []
1066                 for pkg in reversed(self.pkg.packages):
1067                         packager = packages.packager.BinaryPackager(self.pakfire, pkg,
1068                                 self, self.buildroot)
1069                         pkg = packager.run(self.resultdir)
1070                         pkgs.append(pkg)
1071                 log.info("")
1072
1073         def build_stage(self, stage):
1074                 # Get the buildscript for this stage.
1075                 buildscript = self.create_buildscript(stage)
1076
1077                 # Execute the buildscript of this stage.
1078                 log.info(_("Running stage %s:") % stage)
1079
1080                 try:
1081                         self.execute(buildscript)
1082
1083                 finally:
1084                         # Remove the buildscript.
1085                         if os.path.exists(buildscript):
1086                                 os.unlink(buildscript)
1087
1088         def post_remove_static_libs(self):
1089                 keep_libs = self.pkg.lexer.build.get_var("keep_libraries")
1090                 keep_libs = keep_libs.split()
1091
1092                 try:
1093                         self.execute("%s/remove-static-libs %s %s" % \
1094                                 (SCRIPT_DIR, self.buildroot, " ".join(keep_libs)))
1095                 except ShellEnvironmentError, e:
1096                         log.warning(_("Could not remove static libraries: %s") % e)
1097
1098         def post_compress_man_pages(self):
1099                 try:
1100                         self.execute("%s/compress-man-pages %s" % (SCRIPT_DIR, self.buildroot))
1101                 except ShellEnvironmentError, e:
1102                         log.warning(_("Compressing man pages did not complete successfully."))
1103
1104         def post_extract_debuginfo(self):
1105                 args = []
1106
1107                 # Check if we need to run with strict build-id.
1108                 strict_id = self.pkg.lexer.build.get_var("debuginfo_strict_build_id", "true")
1109                 if strict_id in ("true", "yes", "1"):
1110                         args.append("--strict-build-id")
1111
1112                 args.append("--buildroot=%s" % self.pkg.buildroot)
1113                 args.append("--sourcedir=%s" % self.pkg.sourcedir)
1114
1115                 # Get additional options to pass to script.
1116                 options = self.pkg.lexer.build.get_var("debuginfo_options", "")
1117                 args += options.split()
1118
1119                 try:
1120                         self.execute("%s/extract-debuginfo %s %s" % (SCRIPT_DIR, " ".join(args), self.pkg.buildroot))
1121                 except ShellEnvironmentError, e:
1122                         log.error(_("Extracting debuginfo did not complete with success. Aborting build."))
1123                         raise
1124
1125         def find_prerequires(self, scriptlet_file):
1126                 assert os.path.exists(scriptlet_file), "Scriptlet file does not exist: %s" % scriptlet_file
1127
1128                 res = self.run_script("find-prerequires", scriptlet_file)
1129                 prerequires = set(res.splitlines())
1130
1131                 return prerequires
1132
1133         def cleanup(self):
1134                 if os.path.exists(self.buildroot):
1135                         util.rm(self.buildroot)