python/pakfire/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import math
  25 import os
  26 import re
  27 import shutil
  28 import signal
  29 import socket
  30 import tempfile
  31 import time
  32 import uuid
  33
  34 import base
  35 import cgroup
  36 import logger
  37 import packages
  38 import packages.file
  39 import packages.packager
  40 import repository
  41 import shell
  42 import util
  43 import _pakfire
  44
  45 import logging
  46 log = logging.getLogger("pakfire")
  47
  48 from config import ConfigBuilder
  49 from system import system
  50 from constants import *
  51 from i18n import _
  52 from errors import BuildError, BuildRootLocked, Error
  53
  54
  55 BUILD_LOG_HEADER = """
  56  ____       _     __ _            _           _ _     _
  57 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  58 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  59 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  60 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  61
  62         Version : %(version)s
  63         Host    : %(hostname)s (%(host_arch)s)
  64         Time    : %(time)s
  65
  66 """
  67
  68 class BuildEnviron(object):
  69         # The version of the kernel this machine is running.
  70         kernel_version = os.uname()[2]
  71
  72         def __init__(self, pakfire, filename=None, distro_name=None, build_id=None, logfile=None, release_build=True):
  73                 self.pakfire = pakfire
  74
  75                 # Check if the given pakfire instance is of the correct type.
  76                 assert isinstance(self.pakfire, base.PakfireBuilder)
  77
  78                 # Check if this host can build the requested architecture.
  79                 if not system.host_supports_arch(self.arch):
  80                         raise BuildError, _("Cannot build for %s on this host.") % self.arch
  81
  82                 # Save the build id and generate one if no build id was provided.
  83                 if not build_id:
  84                         build_id = "%s" % uuid.uuid4()
  85
  86                 self.build_id = build_id
  87
  88                 # Setup the logging.
  89                 self.init_logging(logfile)
  90
  91                 # Initialize a cgroup (if supported).
  92                 self.init_cgroup()
  93
  94                 # This build is a release build?
  95                 self.release_build = release_build
  96
  97                 if self.release_build:
  98                         # Disable the local build repository in release mode.
  99                         self.pakfire.repos.disable_repo("build")
 100
 101                         # Log information about pakfire and some more information, when we
 102                         # are running in release mode.
 103                         logdata = {
 104                                 "host_arch"  : system.arch,
 105                                 "hostname"   : system.hostname,
 106                                 "time"       : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
 107                                 "version"    : "Pakfire %s" % PAKFIRE_VERSION,
 108                         }
 109
 110                         for line in BUILD_LOG_HEADER.splitlines():
 111                                 self.log.info(line % logdata)
 112
 113                 # Settings array.
 114                 self.settings = {
 115                         "enable_loop_devices" : self.config.get_bool("builder", "use_loop_devices", True),
 116                         "enable_ccache"       : self.config.get_bool("builder", "use_ccache", True),
 117                         "enable_icecream"     : self.config.get_bool("builder", "use_icecream", False),
 118                         "sign_packages"       : False,
 119                         "buildroot_tmpfs"     : self.config.get_bool("builder", "use_tmpfs", False),
 120                 }
 121
 122                 # Try to get the configured host key. If it is available,
 123                 # we will automatically sign all packages with it.
 124                 if self.keyring.get_host_key(secret=True):
 125                         self.settings["sign_packages"] = True
 126
 127                 # Where do we put the result?
 128                 self.resultdir = os.path.join(self.pakfire.path, "result")
 129
 130                 # Open package.
 131                 # If we have a plain makefile, we first build a source package and go with that.
 132                 if filename:
 133                         # Open source package.
 134                         self.pkg = packages.SourcePackage(self.pakfire, None, filename)
 135                         assert self.pkg, filename
 136
 137                         # Log the package information.
 138                         self.log.info(_("Package information:"))
 139                         for line in self.pkg.dump(long=True).splitlines():
 140                                 self.log.info("  %s" % line)
 141                         self.log.info("")
 142
 143                         # Path where we extract the package and put all the source files.
 144                         self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name)
 145                 else:
 146                         # No package :(
 147                         self.pkg = None
 148
 149                 # Lock the buildroot
 150                 self._lock = None
 151
 152                 # Save the build time.
 153                 self.build_time = time.time()
 154
 155         def setup_signal_handlers(self):
 156                 pass
 157
 158         def start(self):
 159                 assert not self.pakfire.initialized, "Pakfire has already been initialized"
 160
 161                 # Mount the directories.
 162                 self._mountall()
 163
 164                 # Lock the build environment.
 165                 self.lock()
 166
 167                 # Initialize pakfire instance.
 168                 self.pakfire.initialize()
 169
 170                 # Populate /dev.
 171                 self.populate_dev()
 172
 173                 # Setup domain name resolution in chroot.
 174                 self.setup_dns()
 175
 176                 # Extract all needed packages.
 177                 self.extract()
 178
 179         def stop(self):
 180                 if self.cgroup:
 181                         # Kill all still running processes in the cgroup.
 182                         self.cgroup.kill_and_wait()
 183
 184                         # Remove cgroup and all parent cgroups if they are empty.
 185                         self.cgroup.migrate_task(self.cgroup.root, os.getpid())
 186                         self.cgroup.destroy()
 187
 188                         parent = self.cgroup.parent
 189                         while parent:
 190                                 if not parent.is_empty(recursive=True):
 191                                         break
 192
 193                                 parent.destroy()
 194                                 parent = parent.parent
 195
 196                 else:
 197                         util.orphans_kill(self.path)
 198
 199                 # Shut down pakfire instance.
 200                 self.pakfire.destroy()
 201
 202                 # Unlock build environment.
 203                 self.unlock()
 204
 205                 # Umount the build environment.
 206                 self._umountall()
 207
 208                 # Remove all files.
 209                 self.destroy()
 210
 211         @property
 212         def config(self):
 213                 """
 214                         Proxy method for easy access to the configuration.
 215                 """
 216                 return self.pakfire.config
 217
 218         @property
 219         def distro(self):
 220                 """
 221                         Proxy method for easy access to the distribution.
 222                 """
 223                 return self.pakfire.distro
 224
 225         @property
 226         def path(self):
 227                 """
 228                         Proxy method for easy access to the path.
 229                 """
 230                 return self.pakfire.path
 231
 232         @property
 233         def arch(self):
 234                 """
 235                         Inherit architecture from distribution configuration.
 236                 """
 237                 return self.pakfire.distro.arch
 238
 239         @property
 240         def personality(self):
 241                 """
 242                         Gets the personality from the distribution configuration.
 243                 """
 244                 return self.pakfire.distro.personality
 245
 246         @property
 247         def info(self):
 248                 return {
 249                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 250                         "build_host" : socket.gethostname(),
 251                         "build_id"   : self.build_id,
 252                         "build_time" : self.build_time,
 253                 }
 254
 255         @property
 256         def keyring(self):
 257                 """
 258                         Shortcut to access the pakfire keyring.
 259                 """
 260                 return self.pakfire.keyring
 261
 262         def lock(self):
 263                 filename = os.path.join(self.path, ".lock")
 264
 265                 try:
 266                         self._lock = open(filename, "a+")
 267                 except IOError, e:
 268                         return 0
 269
 270                 try:
 271                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 272                 except IOError, e:
 273                         raise BuildRootLocked, "Buildroot is locked"
 274
 275                 return 1
 276
 277         def unlock(self):
 278                 if self._lock:
 279                         self._lock.close()
 280                         self._lock = None
 281
 282         def init_cgroup(self):
 283                 """
 284                         Initialize cgroup (if the system supports it).
 285                 """
 286                 if not cgroup.supported():
 287                         self.cgroup = None
 288                         return
 289
 290                 self.cgroup = cgroup.CGroup("pakfire/builder/%s" % self.build_id)
 291
 292                 # Attach the pakfire-builder process to the parent group.
 293                 self.cgroup.parent.attach()
 294
 295         def init_logging(self, logfile):
 296                 if logfile:
 297                         self.log = log.getChild(self.build_id)
 298                         # Propage everything to the root logger that we will see something
 299                         # on the terminal.
 300                         self.log.propagate = 1
 301                         self.log.setLevel(logging.INFO)
 302
 303                         # Add the given logfile to the logger.
 304                         h = logging.FileHandler(logfile)
 305                         self.log.addHandler(h)
 306
 307                         # Format the log output for the file.
 308                         f = logger.BuildFormatter()
 309                         h.setFormatter(f)
 310                 else:
 311                         # If no logile was given, we use the root logger.
 312                         self.log = logging.getLogger("pakfire")
 313
 314         def copyin(self, file_out, file_in):
 315                 if file_in.startswith("/"):
 316                         file_in = file_in[1:]
 317
 318                 file_in = self.chrootPath(file_in)
 319
 320                 #if not os.path.exists(file_out):
 321                 #       return
 322
 323                 dir_in = os.path.dirname(file_in)
 324                 if not os.path.exists(dir_in):
 325                         os.makedirs(dir_in)
 326
 327                 self.log.debug("%s --> %s" % (file_out, file_in))
 328
 329                 shutil.copy2(file_out, file_in)
 330
 331         def copyout(self, file_in, file_out):
 332                 if file_in.startswith("/"):
 333                         file_in = file_in[1:]
 334
 335                 file_in = self.chrootPath(file_in)
 336
 337                 #if not os.path.exists(file_in):
 338                 #       return
 339
 340                 dir_out = os.path.dirname(file_out)
 341                 if not os.path.exists(dir_out):
 342                         os.makedirs(dir_out)
 343
 344                 self.log.debug("%s --> %s" % (file_in, file_out))
 345
 346                 shutil.copy2(file_in, file_out)
 347
 348         def copy_result(self, resultdir):
 349                 # XXX should use find_result_packages
 350
 351                 dir_in = self.chrootPath("result")
 352
 353                 for dir, subdirs, files in os.walk(dir_in):
 354                         basename = os.path.basename(dir)
 355                         dir = dir[len(self.chrootPath()):]
 356                         for file in files:
 357                                 file_in = os.path.join(dir, file)
 358
 359                                 file_out = os.path.join(
 360                                         resultdir,
 361                                         basename,
 362                                         file,
 363                                 )
 364
 365                                 self.copyout(file_in, file_out)
 366
 367         def find_result_packages(self):
 368                 ret = []
 369
 370                 for dir, subdirs, files in os.walk(self.resultdir):
 371                         for file in files:
 372                                 if not file.endswith(".%s" % PACKAGE_EXTENSION):
 373                                         continue
 374
 375                                 file = os.path.join(dir, file)
 376                                 ret.append(file)
 377
 378                 return ret
 379
 380         def extract(self, requires=None):
 381                 """
 382                         Gets a dependency set and extracts all packages
 383                         to the environment.
 384                 """
 385                 if not requires:
 386                         requires = []
 387
 388                 # Add neccessary build dependencies.
 389                 requires += BUILD_PACKAGES
 390
 391                 # If we have ccache enabled, we need to extract it
 392                 # to the build chroot.
 393                 if self.settings.get("enable_ccache"):
 394                         requires.append("ccache")
 395
 396                 # If we have icecream enabled, we need to extract it
 397                 # to the build chroot.
 398                 if self.settings.get("enable_icecream"):
 399                         requires.append("icecream")
 400
 401                 # Get build dependencies from source package.
 402                 if self.pkg:
 403                         for req in self.pkg.requires:
 404                                 requires.append(req)
 405
 406                 # Install all packages.
 407                 self.log.info(_("Install packages needed for build..."))
 408                 self.install(requires)
 409
 410                 # Copy the makefile and load source tarballs.
 411                 if self.pkg:
 412                         self.pkg.extract(_("Extracting"), prefix=self.build_dir)
 413
 414                         # Add an empty line at the end.
 415                         self.log.info("")
 416
 417         def install(self, requires, **kwargs):
 418                 """
 419                         Install everything that is required in requires.
 420                 """
 421                 # If we got nothing to do, we quit immediately.
 422                 if not requires:
 423                         return
 424
 425                 kwargs.update({
 426                         "interactive" : False,
 427                         "logger" : self.log,
 428                 })
 429
 430                 if not kwargs.has_key("allow_downgrade"):
 431                         kwargs["allow_downgrade"] = True
 432
 433                 # Install everything.
 434                 self.pakfire.install(requires, **kwargs)
 435
 436         def chrootPath(self, *args):
 437                 # Remove all leading slashes
 438                 _args = []
 439                 for arg in args:
 440                         if arg.startswith("/"):
 441                                 arg = arg[1:]
 442                         _args.append(arg)
 443                 args = _args
 444
 445                 ret = os.path.join(self.path, *args)
 446                 ret = ret.replace("//", "/")
 447
 448                 assert ret.startswith(self.path)
 449
 450                 return ret
 451
 452         def populate_dev(self):
 453                 nodes = [
 454                         "/dev/null",
 455                         "/dev/zero",
 456                         "/dev/full",
 457                         "/dev/random",
 458                         "/dev/urandom",
 459                         "/dev/tty",
 460                         "/dev/ptmx",
 461                         "/dev/kmsg",
 462                         "/dev/rtc0",
 463                         "/dev/console",
 464                 ]
 465
 466                 # If we need loop devices (which are optional) we create them here.
 467                 if self.settings["enable_loop_devices"]:
 468                         for i in range(0, 7):
 469                                 nodes.append("/dev/loop%d" % i)
 470
 471                 for node in nodes:
 472                         # Stat the original node of the host system and copy it to
 473                         # the build chroot.
 474                         node_stat = os.stat(node)
 475
 476                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 477
 478                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 479                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 480                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 481                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 482
 483         def setup_dns(self):
 484                 """
 485                         Add DNS resolution facility to chroot environment by copying
 486                         /etc/resolv.conf and /etc/hosts.
 487                 """
 488                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 489                         self.copyin(i, i)
 490
 491         def _create_node(self, filename, mode, device):
 492                 self.log.debug("Create node: %s (%s)" % (filename, mode))
 493
 494                 filename = self.chrootPath(filename)
 495
 496                 # Create parent directory if it is missing.
 497                 dirname = os.path.dirname(filename)
 498                 if not os.path.exists(dirname):
 499                         os.makedirs(dirname)
 500
 501                 os.mknod(filename, mode, device)
 502
 503         def destroy(self):
 504                 self.log.debug("Destroying environment %s" % self.path)
 505
 506                 if os.path.exists(self.path):
 507                         util.rm(self.path)
 508
 509         def cleanup(self):
 510                 self.log.debug("Cleaning environemnt.")
 511
 512                 # Remove the build directory and buildroot.
 513                 dirs = (self.build_dir, self.chrootPath("result"),)
 514
 515                 for d in dirs:
 516                         if not os.path.exists(d):
 517                                 continue
 518
 519                         util.rm(d)
 520                         os.makedirs(d)
 521
 522         def _mountall(self):
 523                 self.log.debug("Mounting environment")
 524                 for src, dest, fs, options in self.mountpoints:
 525                         mountpoint = self.chrootPath(dest)
 526                         if options:
 527                                 options = "-o %s" % options
 528
 529                         # Eventually create mountpoint directory
 530                         if not os.path.exists(mountpoint):
 531                                 os.makedirs(mountpoint)
 532
 533                         self.execute_root("mount -n -t %s %s %s %s" % (fs, options, src, mountpoint), shell=True)
 534
 535         def _umountall(self):
 536                 self.log.debug("Umounting environment")
 537
 538                 mountpoints = []
 539                 for src, dest, fs, options in reversed(self.mountpoints):
 540                         dest = self.chrootPath(dest)
 541
 542                         if not dest in mountpoints:
 543                                 mountpoints.append(dest)
 544
 545                 while mountpoints:
 546                         for mp in mountpoints:
 547                                 try:
 548                                         self.execute_root("umount -n %s" % mp, shell=True)
 549                                 except ShellEnvironmentError:
 550                                         pass
 551
 552                                 if not os.path.ismount(mp):
 553                                         mountpoints.remove(mp)
 554
 555         @property
 556         def mountpoints(self):
 557                 mountpoints = []
 558
 559                 # Make root as a tmpfs if enabled.
 560                 if self.settings.get("buildroot_tmpfs"):
 561                         mountpoints += [
 562                                 ("pakfire_root", "/", "tmpfs", "defaults"),
 563                         ]
 564
 565                 mountpoints += [
 566                         # src, dest, fs, options
 567                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 568                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 569                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 570                         ("/sys",          "/sys",      "bind",  "bind"),
 571                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 572                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 573                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 574                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 575                         ("pakfire_tmpfs", "/tmp",      "tmpfs", "mode=755,nosuid,nodev"),
 576                 ]
 577
 578                 # If selinux is enabled.
 579                 if os.path.exists("/sys/fs/selinux"):
 580                         mountpoints += [
 581                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 582                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 583                         ]
 584
 585                 # If ccache support is requested, we bind mount the cache.
 586                 if self.settings.get("enable_ccache"):
 587                         # Create ccache cache directory if it does not exist.
 588                         if not os.path.exists(CCACHE_CACHE_DIR):
 589                                 os.makedirs(CCACHE_CACHE_DIR)
 590
 591                         mountpoints += [
 592                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 593                         ]
 594
 595                 return mountpoints
 596
 597         @property
 598         def environ(self):
 599                 env = {
 600                         # Add HOME manually, because it is occasionally not set
 601                         # and some builds get in trouble then.
 602                         "PATH" : "/usr/bin:/bin:/usr/sbin:/sbin",
 603                         "HOME" : "/root",
 604                         "TERM" : os.environ.get("TERM", "vt100"),
 605                         "PS1"  : "\u:\w\$ ",
 606
 607                         # Sanitize language.
 608                         "LANG" : os.environ.setdefault("LANG", "en_US.UTF-8"),
 609
 610                         # Set the container that we can detect, if we are inside a
 611                         # chroot.
 612                         "container" : "pakfire-builder",
 613                 }
 614
 615                 # Inherit environment from distro
 616                 env.update(self.pakfire.distro.environ)
 617
 618                 # Icecream environment settings
 619                 if self.settings.get("enable_icecream", False):
 620                         # Set the toolchain path
 621                         if self.settings.get("icecream_toolchain", None):
 622                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 623
 624                         # Set preferred host if configured.
 625                         if self.settings.get("icecream_preferred_host", None):
 626                                 env["ICECC_PREFERRED_HOST"] = \
 627                                         self.settings.get("icecream_preferred_host")
 628
 629                 # Fake UTS_MACHINE, when we cannot use the personality syscall and
 630                 # if the host architecture is not equal to the target architecture.
 631                 if not self.pakfire.distro.personality and \
 632                                 not system.native_arch == self.pakfire.distro.arch:
 633                         env.update({
 634                                 "LD_PRELOAD"  : "/usr/lib/libpakfire_preload.so",
 635                                 "UTS_MACHINE" : self.pakfire.distro.arch,
 636                         })
 637
 638                 return env
 639
 640         @property
 641         def installed_packages(self):
 642                 """
 643                         Returns an iterator over all installed packages in this build environment.
 644                 """
 645                 # Get the repository of all installed packages.
 646                 repo = self.pakfire.repos.get_repo("@system")
 647
 648                 # Return an iterator over the packages.
 649                 return iter(repo)
 650
 651         def write_config(self):
 652                 # Cleanup everything in /etc/pakfire.
 653                 util.rm(self.chrootPath(CONFIG_DIR))
 654
 655                 for i in (CONFIG_DIR, CONFIG_REPOS_DIR):
 656                         i = self.chrootPath(i)
 657                         if not os.path.exists(i):
 658                                 os.makedirs(i)
 659
 660                 # Write general.conf.
 661                 f = open(self.chrootPath(CONFIG_DIR, "general.conf"), "w")
 662                 f.close()
 663
 664                 # Write builder.conf.
 665                 f = open(self.chrootPath(CONFIG_DIR, "builder.conf"), "w")
 666                 f.write(self.distro.get_config())
 667                 f.close()
 668
 669                 # Create pakfire configuration files.
 670                 for repo in self.pakfire.repos:
 671                         conf = repo.get_config()
 672
 673                         if not conf:
 674                                 continue
 675
 676                         filename = self.chrootPath(CONFIG_REPOS_DIR, "%s.repo" % repo.name)
 677                         f = open(filename, "w")
 678                         f.write("\n".join(conf))
 679                         f.close()
 680
 681         @property
 682         def pkg_makefile(self):
 683                 return os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION))
 684
 685         def execute(self, command, logger=None, **kwargs):
 686                 """
 687                         Executes the given command in the build chroot.
 688                 """
 689                 # Environment variables
 690                 env = self.environ
 691
 692                 if kwargs.has_key("env"):
 693                         env.update(kwargs.pop("env"))
 694
 695                 self.log.debug("Environment:")
 696                 for k, v in sorted(env.items()):
 697                         self.log.debug("  %s=%s" % (k, v))
 698
 699                 # Make every shell to a login shell because we set a lot of
 700                 # environment things there.
 701                 command = ["bash", "--login", "-c", command]
 702
 703                 args = {
 704                         "chroot_path" : self.chrootPath(),
 705                         "cgroup"      : self.cgroup,
 706                         "env"         : env,
 707                         "logger"      : logger,
 708                         "personality" : self.personality,
 709                         "shell"       : False,
 710                 }
 711                 args.update(kwargs)
 712
 713                 # Run the shit.
 714                 shellenv = shell.ShellExecuteEnvironment(command, **args)
 715                 shellenv.execute()
 716
 717                 return shellenv
 718
 719         def execute_root(self, command, **kwargs):
 720                 """
 721                         Executes the given command outside the build chroot.
 722                 """
 723                 shellenv = shell.ShellExecuteEnvironment(command, **kwargs)
 724                 shellenv.execute()
 725
 726                 return shellenv
 727
 728         def build(self, install_test=True, prepare=False):
 729                 if not self.pkg:
 730                         raise BuildError, _("You cannot run a build when no package was given.")
 731
 732                 # Search for the package file in build_dir and raise BuildError if it is not present.
 733                 if not os.path.exists(self.pkg_makefile):
 734                         raise BuildError, _("Could not find makefile in build root: %s") % self.pkg_makefile
 735
 736                 # Write pakfire configuration into the chroot.
 737                 self.write_config()
 738
 739                 # Create the build command, that is executed in the chroot.
 740                 build_command = [
 741                         "/usr/lib/pakfire/builder",
 742                         "--offline",
 743                         "build",
 744                         "/%s" % os.path.relpath(self.pkg_makefile, self.chrootPath()),
 745                         "--arch", self.arch,
 746                         "--nodeps",
 747                         "--resultdir=/result",
 748                 ]
 749
 750                 # Check if only the preparation stage should be run.
 751                 if prepare:
 752                         build_command.append("--prepare")
 753
 754                 build_command = " ".join(build_command)
 755
 756                 try:
 757                         self.execute(build_command, logger=self.log)
 758
 759                         # Perform the install test after the actual build.
 760                         if install_test and not prepare:
 761                                 self.install_test()
 762
 763                 except ShellEnvironmentError:
 764                         self.log.error(_("Build failed"))
 765
 766                 except KeyboardInterrupt:
 767                         self.log.error(_("Build interrupted"))
 768
 769                         raise
 770
 771                 # Catch all other errors.
 772                 except:
 773                         self.log.error(_("Build failed."), exc_info=True)
 774
 775                 else:
 776                         # Don't sign packages in prepare mode.
 777                         if prepare:
 778                                 return
 779
 780                         # Sign all built packages with the host key (if available).
 781                         self.sign_packages()
 782
 783                         # Dump package information.
 784                         self.dump()
 785
 786                         return
 787
 788                 # End here in case of an error.
 789                 raise BuildError, _("The build command failed. See logfile for details.")
 790
 791         def install_test(self):
 792                 self.log.info(_("Running installation test..."))
 793
 794                 # Install all packages that were built.
 795                 self.install(self.find_result_packages(), allow_vendorchange=True,
 796                         allow_uninstall=True, signatures_mode="disabled")
 797
 798                 self.log.info(_("Installation test succeeded."))
 799                 self.log.info("")
 800
 801         def shell(self, args=[]):
 802                 if not util.cli_is_interactive():
 803                         self.log.warning("Cannot run shell on non-interactive console.")
 804                         return
 805
 806                 # Install all packages that are needed to run a shell.
 807                 self.install(SHELL_PACKAGES)
 808
 809                 # XXX need to set CFLAGS here
 810                 command = "/usr/sbin/chroot %s %s %s" % \
 811                         (self.chrootPath(), SHELL_SCRIPT, " ".join(args))
 812
 813                 # Add personality if we require one
 814                 if self.pakfire.distro.personality:
 815                         command = "%s %s" % (self.pakfire.distro.personality, command)
 816
 817                 for key, val in self.environ.items():
 818                         command = "%s=\"%s\" " % (key, val) + command
 819
 820                 # Empty the environment
 821                 command = "env -i - %s" % command
 822
 823                 self.log.debug("Shell command: %s" % command)
 824
 825                 shell = os.system(command)
 826                 return os.WEXITSTATUS(shell)
 827
 828         def sign_packages(self, keyfp=None):
 829                 # Do nothing if signing is not requested.
 830                 if not self.settings.get("sign_packages"):
 831                         return
 832
 833                 # Get key, that should be used for signing.
 834                 if not keyfp:
 835                         keyfp = self.keyring.get_host_key_id()
 836
 837                 # Find all files to process.
 838                 files = self.find_result_packages()
 839
 840                 # Create a progressbar.
 841                 print _("Signing packages...")
 842                 p = util.make_progress(keyfp, len(files))
 843                 i = 0
 844
 845                 for file in files:
 846                         # Update progressbar.
 847                         if p:
 848                                 i += 1
 849                                 p.update(i)
 850
 851                         # Open package file.
 852                         pkg = packages.open(self.pakfire, None, file)
 853
 854                         # Sign it.
 855                         pkg.sign(keyfp)
 856
 857                 # Close progressbar.
 858                 if p:
 859                         p.finish()
 860                         print "" # Print an empty line.
 861
 862         def dump(self):
 863                 pkgs = []
 864
 865                 for file in self.find_result_packages():
 866                         pkg = packages.open(self.pakfire, None, file)
 867                         pkgs.append(pkg)
 868
 869                 # If there are no packages, there is nothing to do.
 870                 if not pkgs:
 871                         return
 872
 873                 pkgs.sort()
 874
 875                 self.log.info(_("Dumping package information:"))
 876                 for pkg in pkgs:
 877                         dump = pkg.dump(long=True)
 878
 879                         for line in dump.splitlines():
 880                                 self.log.info("  %s" % line)
 881                         self.log.info("") # Empty line.
 882
 883
 884 class Builder(object):
 885         def __init__(self, pakfire, filename, resultdir, **kwargs):
 886                 self.pakfire = pakfire
 887
 888                 self.filename = filename
 889
 890                 self.resultdir = resultdir
 891
 892                 # Open package file.
 893                 self.pkg = packages.Makefile(self.pakfire, self.filename)
 894
 895                 self._environ = {
 896                         "LANG"             : "C",
 897                 }
 898
 899         @property
 900         def buildroot(self):
 901                 return self.pkg.buildroot
 902
 903         @property
 904         def distro(self):
 905                 return self.pakfire.distro
 906
 907         @property
 908         def environ(self):
 909                 environ = os.environ
 910
 911                 # Get all definitions from the package.
 912                 environ.update(self.pkg.exports)
 913
 914                 # Overwrite some definitions by default values.
 915                 environ.update(self._environ)
 916
 917                 return environ
 918
 919         def execute(self, command, logger=None, **kwargs):
 920                 if logger is None:
 921                         logger = logging.getLogger("pakfire")
 922
 923                 # Make every shell to a login shell because we set a lot of
 924                 # environment things there.
 925                 command = ["bash", "--login", "-c", command]
 926
 927                 args = {
 928                         "cwd"         : "/%s" % LOCAL_TMP_PATH,
 929                         "env"         : self.environ,
 930                         "logger"      : logger,
 931                         "personality" : self.distro.personality,
 932                         "shell"       : False,
 933                 }
 934                 args.update(kwargs)
 935
 936                 try:
 937                         shellenv = shell.ShellExecuteEnvironment(command, **args)
 938                         shellenv.execute()
 939
 940                 except ShellEnvironmentError:
 941                         logger.error("Command exited with an error: %s" % command)
 942                         raise
 943
 944                 return shellenv
 945
 946         def run_script(self, script, *args):
 947                 if not script.startswith("/"):
 948                         script = os.path.join(SCRIPT_DIR, script)
 949
 950                 assert os.path.exists(script), "Script we should run does not exist: %s" % script
 951
 952                 cmd = [script,]
 953                 for arg in args:
 954                         cmd.append(arg)
 955                 cmd = " ".join(cmd)
 956
 957                 # Returns the output of the command, but the output won't get
 958                 # logged.
 959                 exe = self.execute(cmd, record_output=True, log_output=False)
 960
 961                 # Return the output of the command.
 962                 if exe.exitcode == 0:
 963                         return exe.output
 964
 965         def create_icecream_toolchain(self):
 966                 try:
 967                         exe = self.execute(
 968                                 "icecc --build-native 2>/dev/null",
 969                                 record_output=True, record_stderr=False,
 970                                 log_output=False, log_errors=False,
 971                                 cwd="/tmp",
 972                         )
 973                 except ShellEnvironmentError:
 974                         return
 975
 976                 for line in exe.output.splitlines():
 977                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
 978                         if m:
 979                                 self._environ["ICECC_VERSION"] = "/tmp/%s" % m.group(1)
 980
 981         def create_buildscript(self, stage):
 982                 # Get buildscript from the package.
 983                 script = self.pkg.get_buildscript(stage)
 984
 985                 # Write script to an empty file.
 986                 f = tempfile.NamedTemporaryFile(mode="w", delete=False)
 987                 f.write("#!/bin/sh\n\n")
 988                 f.write("set -e\n")
 989                 f.write("set -x\n")
 990                 f.write("\n%s\n" % script)
 991                 f.write("exit 0\n")
 992                 f.close()
 993
 994                 # Make the script executable.
 995                 os.chmod(f.name, 700)
 996
 997                 return f.name
 998
 999         def build(self, stages=None):
1000                 # Create buildroot and remove all content if it was existant.
1001                 util.rm(self.buildroot)
1002                 os.makedirs(self.buildroot)
1003
1004                 # Build icecream toolchain if icecream is installed.
1005                 self.create_icecream_toolchain()
1006
1007                 # Process stages in order.
1008                 for stage in ("prepare", "build", "test", "install"):
1009                         # Skip unwanted stages.
1010                         if stages and not stage in stages:
1011                                 continue
1012
1013                         # Run stage.
1014                         self.build_stage(stage)
1015
1016                 # Stop if install stage has not been processed.
1017                 if stages and not "install" in stages:
1018                         return
1019
1020                 # Run post-build stuff.
1021                 self.post_compress_man_pages()
1022                 self.post_remove_static_libs()
1023                 self.post_extract_debuginfo()
1024
1025                 # Package the result.
1026                 # Make all these little package from the build environment.
1027                 log.info(_("Creating packages:"))
1028                 pkgs = []
1029                 for pkg in reversed(self.pkg.packages):
1030                         packager = packages.packager.BinaryPackager(self.pakfire, pkg,
1031                                 self, self.buildroot)
1032                         pkg = packager.run(self.resultdir)
1033                         pkgs.append(pkg)
1034                 log.info("")
1035
1036         def build_stage(self, stage):
1037                 # Get the buildscript for this stage.
1038                 buildscript = self.create_buildscript(stage)
1039
1040                 # Execute the buildscript of this stage.
1041                 log.info(_("Running stage %s:") % stage)
1042
1043                 try:
1044                         self.execute(buildscript)
1045
1046                 finally:
1047                         # Remove the buildscript.
1048                         if os.path.exists(buildscript):
1049                                 os.unlink(buildscript)
1050
1051         def post_remove_static_libs(self):
1052                 keep_libs = self.pkg.lexer.build.get_var("keep_libraries")
1053                 keep_libs = keep_libs.split()
1054
1055                 try:
1056                         self.execute("%s/remove-static-libs %s %s" % \
1057                                 (SCRIPT_DIR, self.buildroot, " ".join(keep_libs)))
1058                 except ShellEnvironmentError, e:
1059                         log.warning(_("Could not remove static libraries: %s") % e)
1060
1061         def post_compress_man_pages(self):
1062                 try:
1063                         self.execute("%s/compress-man-pages %s" % (SCRIPT_DIR, self.buildroot))
1064                 except ShellEnvironmentError, e:
1065                         log.warning(_("Compressing man pages did not complete successfully."))
1066
1067         def post_extract_debuginfo(self):
1068                 args = []
1069
1070                 # Check if we need to run with strict build-id.
1071                 strict_id = self.pkg.lexer.build.get_var("debuginfo_strict_build_id", "true")
1072                 if strict_id in ("true", "yes", "1"):
1073                         args.append("--strict-build-id")
1074
1075                 args.append("--buildroot=%s" % self.pkg.buildroot)
1076                 args.append("--sourcedir=%s" % self.pkg.sourcedir)
1077
1078                 # Get additional options to pass to script.
1079                 options = self.pkg.lexer.build.get_var("debuginfo_options", "")
1080                 args += options.split()
1081
1082                 try:
1083                         self.execute("%s/extract-debuginfo %s %s" % (SCRIPT_DIR, " ".join(args), self.pkg.buildroot))
1084                 except ShellEnvironmentError, e:
1085                         log.error(_("Extracting debuginfo did not complete with success. Aborting build."))
1086                         raise
1087
1088         def find_prerequires(self, scriptlet_file):
1089                 assert os.path.exists(scriptlet_file), "Scriptlet file does not exist: %s" % scriptlet_file
1090
1091                 res = self.run_script("find-prerequires", scriptlet_file)
1092                 prerequires = set(res.splitlines())
1093
1094                 return prerequires
1095
1096         def cleanup(self):
1097                 if os.path.exists(self.buildroot):
1098                         util.rm(self.buildroot)