src/pakfire/builder.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import math
  25 import os
  26 import re
  27 import shutil
  28 import socket
  29 import tempfile
  30 import time
  31 import uuid
  32
  33 from . import _pakfire
  34 from . import arch
  35 from . import base
  36 from . import cgroup
  37 from . import config
  38 from . import downloaders
  39 from . import logger
  40 from . import packages
  41 from . import repository
  42 from . import shell
  43 from . import util
  44
  45 import logging
  46 log = logging.getLogger("pakfire.builder")
  47
  48 from .system import system
  49 from .constants import *
  50 from .i18n import _
  51 from .errors import BuildError, BuildRootLocked, Error
  52
  53
  54 BUILD_LOG_HEADER = """
  55  ____       _     __ _            _           _ _     _
  56 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  57 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  58 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  59 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  60
  61         Version : %(version)s
  62         Host    : %(hostname)s (%(host_arch)s)
  63         Time    : %(time)s
  64
  65 """
  66
  67 class Builder(object):
  68         def __init__(self, package=None, arch=None, build_id=None, logfile=None, **kwargs):
  69                 self.config = config.Config("general.conf", "builder.conf")
  70
  71                 distro_name = self.config.get("builder", "distro", None)
  72                 if distro_name:
  73                         self.config.read("distros/%s.conf" % distro_name)
  74
  75                 # Settings array.
  76                 self.settings = {
  77                         "enable_loop_devices" : self.config.get_bool("builder", "use_loop_devices", True),
  78                         "enable_ccache"       : self.config.get_bool("builder", "use_ccache", True),
  79                         "buildroot_tmpfs"     : self.config.get_bool("builder", "use_tmpfs", False),
  80                         "private_network"     : self.config.get_bool("builder", "private_network", False),
  81                 }
  82
  83                 # Get ccache settings.
  84                 if self.settings.get("enable_ccache", False):
  85                         self.settings.update({
  86                                 "ccache_compress" : self.config.get_bool("ccache", "compress", True),
  87                         })
  88
  89                 # Add settings from keyword arguments
  90                 self.settings.update(kwargs)
  91
  92                 # Setup logging
  93                 self.setup_logging(logfile)
  94
  95                 # Generate a build ID
  96                 self.build_id = build_id or "%s" % uuid.uuid4()
  97
  98                 # Path
  99                 self.path = os.path.join(BUILD_ROOT, self.build_id)
 100                 self._lock = None
 101
 102                 # Architecture to build for
 103                 self.arch = arch or system.arch
 104
 105                 # Check if this host can build the requested architecture.
 106                 if not system.host_supports_arch(self.arch):
 107                         raise BuildError(_("Cannot build for %s on this host") % self.arch)
 108
 109                 # Initialize a cgroup (if supported)
 110                 self.cgroup = self.make_cgroup()
 111
 112                 # Unshare namepsace.
 113                 # If this fails because the kernel has no support for CLONE_NEWIPC or CLONE_NEWUTS,
 114                 # we try to fall back to just set CLONE_NEWNS.
 115                 try:
 116                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNS|_pakfire.SCHED_CLONE_NEWIPC|_pakfire.SCHED_CLONE_NEWUTS)
 117                 except RuntimeError as e:
 118                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNS)
 119
 120                 # Optionally enable private networking.
 121                 if self.settings.get("private_network", None):
 122                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNET)
 123
 124                 # Create Pakfire instance
 125                 self.pakfire = base.Pakfire(path=self.path, config=self.config, distro=self.config.distro, arch=arch)
 126
 127         def __del__(self):
 128                 """
 129                         Releases build environment and clean up
 130                 """
 131                 # Umount the build environment
 132                 self._umountall()
 133
 134                 # Destroy the pakfire instance
 135                 del self.pakfire
 136
 137                 # Unlock build environment
 138                 self.unlock()
 139
 140                 # Delete everything
 141                 self._destroy()
 142
 143         def __enter__(self):
 144                 self.log.debug("Entering %s" % self.path)
 145
 146                 # Mount the directories
 147                 try:
 148                         self._mountall()
 149                 except OSError as e:
 150                         if e.errno == 30: # Read-only FS
 151                                 raise BuildError("Buildroot is read-only: %s" % self.pakfire.path)
 152
 153                         # Raise all other errors
 154                         raise
 155
 156                 # Lock the build environment
 157                 self.lock()
 158
 159                 # Populate /dev
 160                 self.populate_dev()
 161
 162                 # Setup domain name resolution in chroot
 163                 self.setup_dns()
 164
 165                 return BuilderContext(self)
 166
 167         def __exit__(self, type, value, traceback):
 168                 self.log.debug("Leaving %s" % self.path)
 169
 170                 # Kill all remaining processes in the build environment
 171                 if self.cgroup:
 172                         # Move the builder process out of the cgroup.
 173                         self.cgroup.migrate_task(self.cgroup.parent, os.getpid())
 174
 175                         # Kill all still running processes in the cgroup.
 176                         self.cgroup.kill_and_wait()
 177
 178                         # Remove cgroup and all parent cgroups if they are empty.
 179                         self.cgroup.destroy()
 180
 181                         parent = self.cgroup.parent
 182                         while parent:
 183                                 if not parent.is_empty(recursive=True):
 184                                         break
 185
 186                                 parent.destroy()
 187                                 parent = parent.parent
 188
 189                 else:
 190                         util.orphans_kill(self.path)
 191
 192         def setup_logging(self, logfile):
 193                 if logfile:
 194                         self.log = log.getChild(self.build_id)
 195                         # Propage everything to the root logger that we will see something
 196                         # on the terminal.
 197                         self.log.propagate = 1
 198                         self.log.setLevel(logging.INFO)
 199
 200                         # Add the given logfile to the logger.
 201                         h = logging.FileHandler(logfile)
 202                         self.log.addHandler(h)
 203
 204                         # Format the log output for the file.
 205                         f = logger.BuildFormatter()
 206                         h.setFormatter(f)
 207                 else:
 208                         # If no logile was given, we use the root logger.
 209                         self.log = logging.getLogger("pakfire")
 210
 211         def make_cgroup(self):
 212                 """
 213                         Initialize cgroup (if the system supports it).
 214                 """
 215                 if not cgroup.supported():
 216                         return
 217
 218                 # Search for the cgroup this process is currently running in.
 219                 parent_cgroup = cgroup.find_by_pid(os.getpid())
 220                 if not parent_cgroup:
 221                         return
 222
 223                 # Create our own cgroup inside the parent cgroup.
 224                 c = parent_cgroup.create_child_cgroup("pakfire/builder/%s" % self.build_id)
 225
 226                 # Attach the pakfire-builder process to the group.
 227                 c.attach()
 228
 229                 return c
 230
 231         def lock(self):
 232                 filename = os.path.join(self.path, ".lock")
 233
 234                 try:
 235                         self._lock = open(filename, "a+")
 236                 except IOError as e:
 237                         return 0
 238
 239                 try:
 240                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 241                 except IOError as e:
 242                         raise BuildRootLocked("Buildroot is locked")
 243
 244                 return 1
 245
 246         def unlock(self):
 247                 if self._lock:
 248                         self._lock.close()
 249                         self._lock = None
 250
 251         def _destroy(self):
 252                 self.log.debug("Destroying environment %s" % self.path)
 253
 254                 if os.path.exists(self.path):
 255                         util.rm(self.path)
 256
 257         @property
 258         def mountpoints(self):
 259                 mountpoints = []
 260
 261                 # Make root as a tmpfs if enabled.
 262                 if self.settings.get("buildroot_tmpfs"):
 263                         mountpoints += [
 264                                 ("pakfire_root", "/", "tmpfs", "defaults"),
 265                         ]
 266
 267                 mountpoints += [
 268                         # src, dest, fs, options
 269                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 270                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 271                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 272                         ("/sys",          "/sys",      "bind",  "bind"),
 273                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 274                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 275                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 276                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 277                         ("pakfire_tmpfs", "/tmp",      "tmpfs", "mode=755,nosuid,nodev"),
 278                 ]
 279
 280                 # If selinux is enabled.
 281                 if os.path.exists("/sys/fs/selinux"):
 282                         mountpoints += [
 283                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 284                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 285                         ]
 286
 287                 # If ccache support is requested, we bind mount the cache.
 288                 if self.settings.get("enable_ccache"):
 289                         # Create ccache cache directory if it does not exist.
 290                         if not os.path.exists(CCACHE_CACHE_DIR):
 291                                 os.makedirs(CCACHE_CACHE_DIR)
 292
 293                         mountpoints += [
 294                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 295                         ]
 296
 297                 return mountpoints
 298
 299         def _mountall(self):
 300                 self.log.debug("Mounting environment")
 301
 302                 for src, dest, fs, options in self.mountpoints:
 303                         mountpoint = self.chrootPath(dest)
 304                         if options:
 305                                 options = "-o %s" % options
 306
 307                         # Eventually create mountpoint directory
 308                         if not os.path.exists(mountpoint):
 309                                 os.makedirs(mountpoint)
 310
 311                         self.execute_root("mount -n -t %s %s %s %s" % (fs, options, src, mountpoint), shell=True)
 312
 313         def _umountall(self):
 314                 self.log.debug("Umounting environment")
 315
 316                 mountpoints = []
 317                 for src, dest, fs, options in reversed(self.mountpoints):
 318                         dest = self.chrootPath(dest)
 319
 320                         if not dest in mountpoints:
 321                                 mountpoints.append(dest)
 322
 323                 while mountpoints:
 324                         for mp in mountpoints:
 325                                 try:
 326                                         self.execute_root("umount -n %s" % mp, shell=True)
 327                                 except ShellEnvironmentError:
 328                                         pass
 329
 330                                 if not os.path.ismount(mp):
 331                                         mountpoints.remove(mp)
 332
 333         def copyin(self, file_out, file_in):
 334                 if file_in.startswith("/"):
 335                         file_in = file_in[1:]
 336
 337                 file_in = self.chrootPath(file_in)
 338
 339                 #if not os.path.exists(file_out):
 340                 #       return
 341
 342                 dir_in = os.path.dirname(file_in)
 343                 if not os.path.exists(dir_in):
 344                         os.makedirs(dir_in)
 345
 346                 self.log.debug("%s --> %s" % (file_out, file_in))
 347
 348                 shutil.copy2(file_out, file_in)
 349
 350         def copyout(self, file_in, file_out):
 351                 if file_in.startswith("/"):
 352                         file_in = file_in[1:]
 353
 354                 file_in = self.chrootPath(file_in)
 355
 356                 #if not os.path.exists(file_in):
 357                 #       return
 358
 359                 dir_out = os.path.dirname(file_out)
 360                 if not os.path.exists(dir_out):
 361                         os.makedirs(dir_out)
 362
 363                 self.log.debug("%s --> %s" % (file_in, file_out))
 364
 365                 shutil.copy2(file_in, file_out)
 366
 367         def populate_dev(self):
 368                 nodes = [
 369                         "/dev/null",
 370                         "/dev/zero",
 371                         "/dev/full",
 372                         "/dev/random",
 373                         "/dev/urandom",
 374                         "/dev/tty",
 375                         "/dev/ptmx",
 376                         "/dev/kmsg",
 377                         "/dev/rtc0",
 378                         "/dev/console",
 379                 ]
 380
 381                 # If we need loop devices (which are optional) we create them here.
 382                 if self.settings["enable_loop_devices"]:
 383                         for i in range(0, 7):
 384                                 nodes.append("/dev/loop%d" % i)
 385
 386                 for node in nodes:
 387                         # Stat the original node of the host system and copy it to
 388                         # the build chroot.
 389                         try:
 390                                 node_stat = os.stat(node)
 391
 392                         # If it cannot be found, just go on.
 393                         except OSError:
 394                                 continue
 395
 396                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 397
 398                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 399                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 400                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 401                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 402
 403         def chrootPath(self, *args):
 404                 # Remove all leading slashes
 405                 _args = []
 406                 for arg in args:
 407                         if arg.startswith("/"):
 408                                 arg = arg[1:]
 409                         _args.append(arg)
 410                 args = _args
 411
 412                 ret = os.path.join(self.path, *args)
 413                 ret = ret.replace("//", "/")
 414
 415                 assert ret.startswith(self.path)
 416
 417                 return ret
 418
 419         def setup_dns(self):
 420                 """
 421                         Add DNS resolution facility to chroot environment by copying
 422                         /etc/resolv.conf and /etc/hosts.
 423                 """
 424                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 425                         self.copyin(i, i)
 426
 427         def _create_node(self, filename, mode, device):
 428                 self.log.debug("Create node: %s (%s)" % (filename, mode))
 429
 430                 filename = self.chrootPath(filename)
 431
 432                 # Create parent directory if it is missing.
 433                 dirname = os.path.dirname(filename)
 434                 if not os.path.exists(dirname):
 435                         os.makedirs(dirname)
 436
 437                 os.mknod(filename, mode, device)
 438
 439         def execute_root(self, command, **kwargs):
 440                 """
 441                         Executes the given command outside the build chroot.
 442                 """
 443                 shellenv = shell.ShellExecuteEnvironment(command, logger=self.log, **kwargs)
 444                 shellenv.execute()
 445
 446                 return shellenv
 447
 448
 449 class BuilderContext(object):
 450         def __init__(self, builder):
 451                 self.builder = builder
 452
 453                 # Get a reference to Pakfire
 454                 self.pakfire = self.builder.pakfire
 455
 456                 # Get a reference to the logger
 457                 self.log = self.builder.log
 458
 459         @property
 460         def arch(self):
 461                 return self.pakfire.arch
 462
 463         @property
 464         def environ(self):
 465                 env = MINIMAL_ENVIRONMENT.copy()
 466                 env.update({
 467                         # Add HOME manually, because it is occasionally not set
 468                         # and some builds get in trouble then.
 469                         "TERM" : os.environ.get("TERM", "vt100"),
 470
 471                         # Sanitize language.
 472                         "LANG" : os.environ.setdefault("LANG", "en_US.UTF-8"),
 473
 474                         # Set the container that we can detect, if we are inside a
 475                         # chroot.
 476                         "container" : "pakfire-builder",
 477                 })
 478
 479                 # Inherit environment from distro
 480                 env.update(self.pakfire.distro.environ)
 481
 482                 # ccache environment settings
 483                 if self.builder.settings.get("enable_ccache", False):
 484                         compress = self.builder.settings.get("ccache_compress", False)
 485                         if compress:
 486                                 env["CCACHE_COMPRESS"] = "1"
 487
 488                         # Let ccache create its temporary files in /tmp.
 489                         env["CCACHE_TEMPDIR"] = "/tmp"
 490
 491                 # Fake UTS_MACHINE, when we cannot use the personality syscall and
 492                 # if the host architecture is not equal to the target architecture.
 493                 if not self.arch.personality and \
 494                                 not system.native_arch == self.arch.name:
 495                         env.update({
 496                                 "LD_PRELOAD"  : "/usr/lib/libpakfire_preload.so",
 497                                 "UTS_MACHINE" : self.arch.name,
 498                         })
 499
 500                 return env
 501
 502         def setup(self, install=None):
 503                 self.log.info(_("Install packages needed for build..."))
 504
 505                 packages = [
 506                         "@Build",
 507                         "pakfire-build >= %s" % self.pakfire.__version__,
 508                 ]
 509
 510                 # If we have ccache enabled, we need to extract it
 511                 # to the build chroot
 512                 if self.builder.settings.get("enable_ccache"):
 513                         packages.append("ccache")
 514
 515                 # Install additional packages
 516                 if install:
 517                         packages += install
 518
 519                 # Logging
 520                 self.log.debug(_("Installing build requirements: %s") % ", ".join(packages))
 521
 522                 # Initialise Pakfire
 523                 with self.pakfire as p:
 524                         # Install all required packages
 525                         transaction = p.install(packages)
 526
 527                         # Dump transaction to log
 528                         t = transaction.dump()
 529                         self.log.info(t)
 530
 531                         # Download transaction
 532                         d = downloaders.TransactionDownloader(self.pakfire, transaction)
 533                         d.download()
 534
 535                         # Run the transaction
 536                         transaction.run()
 537
 538         def build(self, package, private_network=True, shell=True):
 539                 archive = _pakfire.Archive(self.pakfire, package)
 540
 541                 requires = archive.get("dependencies.requires")
 542
 543                 # Setup the environment including any build dependencies
 544                 self.setup(install=requires.splitlines())
 545
 546         def shell(self, install=[]):
 547                 if not util.cli_is_interactive():
 548                         self.log.warning("Cannot run shell on non-interactive console.")
 549                         return
 550
 551                 # Install our standard shell packages
 552                 install += SHELL_PACKAGES
 553
 554                 self.setup(install=install)
 555
 556                 command = "/usr/sbin/chroot %s %s %s" % (self.chrootPath(), SHELL_SCRIPT)
 557
 558                 # Add personality if we require one
 559                 if self.pakfire.distro.personality:
 560                         command = "%s %s" % (self.pakfire.distro.personality, command)
 561
 562                 for key, val in list(self.environ.items()):
 563                         command = "%s=\"%s\" " % (key, val) + command
 564
 565                 # Empty the environment
 566                 command = "env -i - %s" % command
 567
 568                 self.log.debug("Shell command: %s" % command)
 569
 570                 shell = os.system(command)
 571                 return os.WEXITSTATUS(shell)
 572
 573
 574 class BuildEnviron(object):
 575         def __init__(self, pakfire, filename=None, distro_name=None, build_id=None, logfile=None, release_build=True, **kwargs):
 576                 self.pakfire = pakfire
 577
 578                 # This build is a release build?
 579                 self.release_build = release_build
 580
 581                 if self.release_build:
 582                         # Disable the local build repository in release mode.
 583                         self.pakfire.repos.disable_repo("build")
 584
 585                         # Log information about pakfire and some more information, when we
 586                         # are running in release mode.
 587                         logdata = {
 588                                 "host_arch"  : system.arch,
 589                                 "hostname"   : system.hostname,
 590                                 "time"       : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
 591                                 "version"    : "Pakfire %s" % PAKFIRE_VERSION,
 592                         }
 593
 594                         for line in BUILD_LOG_HEADER.splitlines():
 595                                 self.log.info(line % logdata)
 596
 597                 # Where do we put the result?
 598                 self.resultdir = os.path.join(self.pakfire.path, "result")
 599
 600                 # Open package.
 601                 # If we have a plain makefile, we first build a source package and go with that.
 602                 if filename:
 603                         # Open source package.
 604                         self.pkg = packages.SourcePackage(self.pakfire, None, filename)
 605                         assert self.pkg, filename
 606
 607                         # Log the package information.
 608                         self.log.info(_("Package information:"))
 609                         for line in self.pkg.dump(int=True).splitlines():
 610                                 self.log.info("  %s" % line)
 611                         self.log.info("")
 612
 613                         # Path where we extract the package and put all the source files.
 614                         self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name)
 615                 else:
 616                         # No package :(
 617                         self.pkg = None
 618
 619                 # Lock the buildroot
 620                 self._lock = None
 621
 622                 # Save the build time.
 623                 self.build_time = time.time()
 624
 625         def start(self):
 626                 # Extract all needed packages.
 627                 self.extract()
 628
 629         def stop(self):
 630                 # Shut down pakfire instance.
 631                 self.pakfire.destroy()
 632
 633         @property
 634         def config(self):
 635                 """
 636                         Proxy method for easy access to the configuration.
 637                 """
 638                 return self.pakfire.config
 639
 640         @property
 641         def distro(self):
 642                 """
 643                         Proxy method for easy access to the distribution.
 644                 """
 645                 return self.pakfire.distro
 646
 647         @property
 648         def path(self):
 649                 """
 650                         Proxy method for easy access to the path.
 651                 """
 652                 return self.pakfire.path
 653
 654         @property
 655         def arch(self):
 656                 """
 657                         Inherit architecture from distribution configuration.
 658                 """
 659                 return self.pakfire.distro.arch
 660
 661         @property
 662         def personality(self):
 663                 """
 664                         Gets the personality from the distribution configuration.
 665                 """
 666                 return self.pakfire.distro.personality
 667
 668         @property
 669         def info(self):
 670                 return {
 671                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 672                         "build_host" : socket.gethostname(),
 673                         "build_id"   : self.build_id,
 674                         "build_time" : self.build_time,
 675                 }
 676
 677         def copy_result(self, resultdir):
 678                 # XXX should use find_result_packages
 679
 680                 dir_in = self.chrootPath("result")
 681
 682                 for dir, subdirs, files in os.walk(dir_in):
 683                         basename = os.path.basename(dir)
 684                         dir = dir[len(self.chrootPath()):]
 685                         for file in files:
 686                                 file_in = os.path.join(dir, file)
 687
 688                                 file_out = os.path.join(
 689                                         resultdir,
 690                                         basename,
 691                                         file,
 692                                 )
 693
 694                                 self.copyout(file_in, file_out)
 695
 696         def find_result_packages(self):
 697                 ret = []
 698
 699                 for dir, subdirs, files in os.walk(self.resultdir):
 700                         for file in files:
 701                                 if not file.endswith(".%s" % PACKAGE_EXTENSION):
 702                                         continue
 703
 704                                 file = os.path.join(dir, file)
 705                                 ret.append(file)
 706
 707                 return ret
 708
 709         def extract(self, requires=None):
 710                 """
 711                         Gets a dependency set and extracts all packages
 712                         to the environment.
 713                 """
 714                 if not requires:
 715                         requires = []
 716
 717                 # Add neccessary build dependencies.
 718                 requires += BUILD_PACKAGES
 719
 720                 # If we have ccache enabled, we need to extract it
 721                 # to the build chroot.
 722                 if self.settings.get("enable_ccache"):
 723                         requires.append("ccache")
 724
 725                 # Get build dependencies from source package.
 726                 if self.pkg:
 727                         for req in self.pkg.requires:
 728                                 requires.append(req)
 729
 730                 # Install all packages.
 731                 self.log.info(_("Install packages needed for build..."))
 732                 self.install(requires)
 733
 734                 # Copy the makefile and load source tarballs.
 735                 if self.pkg:
 736                         self.pkg.extract(_("Extracting"), prefix=self.build_dir)
 737
 738                         # Add an empty line at the end.
 739                         self.log.info("")
 740
 741         def install(self, requires, **kwargs):
 742                 """
 743                         Install everything that is required in requires.
 744                 """
 745                 # If we got nothing to do, we quit immediately.
 746                 if not requires:
 747                         return
 748
 749                 kwargs.update({
 750                         "interactive" : False,
 751                         "logger" : self.log,
 752                 })
 753
 754                 if "allow_downgrade" not in kwargs:
 755                         kwargs["allow_downgrade"] = True
 756
 757                 # Install everything.
 758                 self.pakfire.install(requires, **kwargs)
 759
 760         def cleanup(self):
 761                 self.log.debug("Cleaning environemnt.")
 762
 763                 # Remove the build directory and buildroot.
 764                 dirs = (self.build_dir, self.chrootPath("result"),)
 765
 766                 for d in dirs:
 767                         if not os.path.exists(d):
 768                                 continue
 769
 770                         util.rm(d)
 771                         os.makedirs(d)
 772
 773         @property
 774         def installed_packages(self):
 775                 """
 776                         Returns an iterator over all installed packages in this build environment.
 777                 """
 778                 # Get the repository of all installed packages.
 779                 repo = self.pakfire.repos.get_repo("@system")
 780
 781                 # Return an iterator over the packages.
 782                 return iter(repo)
 783
 784         def write_config(self):
 785                 # Cleanup everything in /etc/pakfire.
 786                 util.rm(self.chrootPath(CONFIG_DIR))
 787
 788                 for i in (CONFIG_DIR, CONFIG_REPOS_DIR):
 789                         i = self.chrootPath(i)
 790                         if not os.path.exists(i):
 791                                 os.makedirs(i)
 792
 793                 # Write general.conf.
 794                 f = open(self.chrootPath(CONFIG_DIR, "general.conf"), "w")
 795                 f.close()
 796
 797                 # Write builder.conf.
 798                 f = open(self.chrootPath(CONFIG_DIR, "builder.conf"), "w")
 799                 f.write(self.distro.get_config())
 800                 f.close()
 801
 802                 # Create pakfire configuration files.
 803                 for repo in self.pakfire.repos:
 804                         conf = repo.get_config()
 805
 806                         if not conf:
 807                                 continue
 808
 809                         filename = self.chrootPath(CONFIG_REPOS_DIR, "%s.repo" % repo.name)
 810                         f = open(filename, "w")
 811                         f.write("\n".join(conf))
 812                         f.close()
 813
 814         @property
 815         def pkg_makefile(self):
 816                 return os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION))
 817
 818         def execute(self, command, logger=None, **kwargs):
 819                 """
 820                         Executes the given command in the build chroot.
 821                 """
 822                 # Environment variables
 823                 env = self.environ
 824
 825                 if "env" in kwargs:
 826                         env.update(kwargs.pop("env"))
 827
 828                 self.log.debug("Environment:")
 829                 for k, v in sorted(env.items()):
 830                         self.log.debug("  %s=%s" % (k, v))
 831
 832                 # Make every shell to a login shell because we set a lot of
 833                 # environment things there.
 834                 command = ["bash", "--login", "-c", command]
 835
 836                 args = {
 837                         "chroot_path" : self.chrootPath(),
 838                         "cgroup"      : self.cgroup,
 839                         "env"         : env,
 840                         "logger"      : logger,
 841                         "personality" : self.personality,
 842                         "shell"       : False,
 843                 }
 844                 args.update(kwargs)
 845
 846                 # Run the shit.
 847                 shellenv = shell.ShellExecuteEnvironment(command, **args)
 848                 shellenv.execute()
 849
 850                 return shellenv
 851
 852         def build(self, install_test=True, prepare=False):
 853                 if not self.pkg:
 854                         raise BuildError(_("You cannot run a build when no package was given."))
 855
 856                 # Search for the package file in build_dir and raise BuildError if it is not present.
 857                 if not os.path.exists(self.pkg_makefile):
 858                         raise BuildError(_("Could not find makefile in build root: %s") % self.pkg_makefile)
 859
 860                 # Write pakfire configuration into the chroot.
 861                 self.write_config()
 862
 863                 # Create the build command, that is executed in the chroot.
 864                 build_command = [
 865                         "/usr/lib/pakfire/builder",
 866                         "--offline",
 867                         "build",
 868                         "/%s" % os.path.relpath(self.pkg_makefile, self.chrootPath()),
 869                         "--arch", self.arch,
 870                         "--nodeps",
 871                         "--resultdir=/result",
 872                 ]
 873
 874                 # Check if only the preparation stage should be run.
 875                 if prepare:
 876                         build_command.append("--prepare")
 877
 878                 build_command = " ".join(build_command)
 879
 880                 try:
 881                         self.execute(build_command, logger=self.log)
 882
 883                         # Perform the install test after the actual build.
 884                         if install_test and not prepare:
 885                                 self.install_test()
 886
 887                 except ShellEnvironmentError:
 888                         self.log.error(_("Build failed"))
 889
 890                 except KeyboardInterrupt:
 891                         self.log.error(_("Build interrupted"))
 892
 893                         raise
 894
 895                 # Catch all other errors.
 896                 except:
 897                         self.log.error(_("Build failed."), exc_info=True)
 898
 899                 else:
 900                         # Don't sign packages in prepare mode.
 901                         if prepare:
 902                                 return
 903
 904                         # Sign all built packages with the host key (if available).
 905                         self.sign_packages()
 906
 907                         # Dump package information.
 908                         self.dump()
 909
 910                         return
 911
 912                 # End here in case of an error.
 913                 raise BuildError(_("The build command failed. See logfile for details."))
 914
 915         def install_test(self):
 916                 self.log.info(_("Running installation test..."))
 917
 918                 # Install all packages that were built.
 919                 self.install(self.find_result_packages(), allow_vendorchange=True,
 920                         allow_uninstall=True, signatures_mode="disabled")
 921
 922                 self.log.info(_("Installation test succeeded."))
 923                 self.log.info("")
 924
 925
 926         def sign_packages(self, keyfp=None):
 927                 # TODO needs to use new key code from libpakfire
 928
 929                 # Do nothing if signing is not requested.
 930                 if not self.settings.get("sign_packages"):
 931                         return
 932
 933                 # Get key, that should be used for signing.
 934                 if not keyfp:
 935                         keyfp = self.keyring.get_host_key_id()
 936
 937                 # Find all files to process.
 938                 files = self.find_result_packages()
 939
 940                 # Create a progressbar.
 941                 print(_("Signing packages..."))
 942                 p = util.make_progress(keyfp, len(files))
 943                 i = 0
 944
 945                 for file in files:
 946                         # Update progressbar.
 947                         if p:
 948                                 i += 1
 949                                 p.update(i)
 950
 951                         # Open package file.
 952                         pkg = packages.open(self.pakfire, None, file)
 953
 954                         # Sign it.
 955                         pkg.sign(keyfp)
 956
 957                 # Close progressbar.
 958                 if p:
 959                         p.finish()
 960                         print("") # Print an empty line.
 961
 962         def dump(self):
 963                 pkgs = []
 964
 965                 for file in self.find_result_packages():
 966                         pkg = packages.open(self.pakfire, None, file)
 967                         pkgs.append(pkg)
 968
 969                 # If there are no packages, there is nothing to do.
 970                 if not pkgs:
 971                         return
 972
 973                 pkgs.sort()
 974
 975                 self.log.info(_("Dumping package information:"))
 976                 for pkg in pkgs:
 977                         dump = pkg.dump(int=True)
 978
 979                         for line in dump.splitlines():
 980                                 self.log.info("  %s" % line)
 981                         self.log.info("") # Empty line.
 982
 983
 984 class BuilderInternal(object):
 985         def __init__(self, pakfire, filename, resultdir, **kwargs):
 986                 self.pakfire = pakfire
 987
 988                 self.filename = filename
 989
 990                 self.resultdir = resultdir
 991
 992                 # Open package file.
 993                 self.pkg = packages.Makefile(self.pakfire, self.filename)
 994
 995                 self._environ = {
 996                         "LANG"             : "C",
 997                 }
 998
 999         @property
1000         def buildroot(self):
1001                 return self.pkg.buildroot
1002
1003         @property
1004         def distro(self):
1005                 return self.pakfire.distro
1006
1007         @property
1008         def environ(self):
1009                 environ = os.environ
1010
1011                 # Get all definitions from the package.
1012                 environ.update(self.pkg.exports)
1013
1014                 # Overwrite some definitions by default values.
1015                 environ.update(self._environ)
1016
1017                 return environ
1018
1019         def execute(self, command, logger=None, **kwargs):
1020                 if logger is None:
1021                         logger = logging.getLogger("pakfire")
1022
1023                 # Make every shell to a login shell because we set a lot of
1024                 # environment things there.
1025                 command = ["bash", "--login", "-c", command]
1026
1027                 args = {
1028                         "cwd"         : "/%s" % LOCAL_TMP_PATH,
1029                         "env"         : self.environ,
1030                         "logger"      : logger,
1031                         "personality" : self.distro.personality,
1032                         "shell"       : False,
1033                 }
1034                 args.update(kwargs)
1035
1036                 try:
1037                         shellenv = shell.ShellExecuteEnvironment(command, **args)
1038                         shellenv.execute()
1039
1040                 except ShellEnvironmentError:
1041                         logger.error("Command exited with an error: %s" % command)
1042                         raise
1043
1044                 return shellenv
1045
1046         def run_script(self, script, *args):
1047                 if not script.startswith("/"):
1048                         script = os.path.join(SCRIPT_DIR, script)
1049
1050                 assert os.path.exists(script), "Script we should run does not exist: %s" % script
1051
1052                 cmd = [script,]
1053                 for arg in args:
1054                         cmd.append(arg)
1055                 cmd = " ".join(cmd)
1056
1057                 # Returns the output of the command, but the output won't get
1058                 # logged.
1059                 exe = self.execute(cmd, record_output=True, log_output=False)
1060
1061                 # Return the output of the command.
1062                 if exe.exitcode == 0:
1063                         return exe.output
1064
1065         def create_buildscript(self, stage):
1066                 # Get buildscript from the package.
1067                 script = self.pkg.get_buildscript(stage)
1068
1069                 # Write script to an empty file.
1070                 f = tempfile.NamedTemporaryFile(mode="w", delete=False)
1071                 f.write("#!/bin/sh\n\n")
1072                 f.write("set -e\n")
1073                 f.write("set -x\n")
1074                 f.write("\n%s\n" % script)
1075                 f.write("exit 0\n")
1076                 f.close()
1077
1078                 # Make the script executable.
1079                 os.chmod(f.name, 700)
1080
1081                 return f.name
1082
1083         def build(self, stages=None):
1084                 # Create buildroot and remove all content if it was existant.
1085                 util.rm(self.buildroot)
1086                 os.makedirs(self.buildroot)
1087
1088                 # Process stages in order.
1089                 for stage in ("prepare", "build", "test", "install"):
1090                         # Skip unwanted stages.
1091                         if stages and not stage in stages:
1092                                 continue
1093
1094                         # Run stage.
1095                         self.build_stage(stage)
1096
1097                 # Stop if install stage has not been processed.
1098                 if stages and not "install" in stages:
1099                         return
1100
1101                 # Run post-build stuff.
1102                 self.post_compress_man_pages()
1103                 self.post_remove_static_libs()
1104                 self.post_extract_debuginfo()
1105
1106                 # Package the result.
1107                 # Make all these little package from the build environment.
1108                 log.info(_("Creating packages:"))
1109                 pkgs = []
1110                 for pkg in reversed(self.pkg.packages):
1111                         packager = packages.packager.BinaryPackager(self.pakfire, pkg,
1112                                 self, self.buildroot)
1113                         pkg = packager.run(self.resultdir)
1114                         pkgs.append(pkg)
1115                 log.info("")
1116
1117         def build_stage(self, stage):
1118                 # Get the buildscript for this stage.
1119                 buildscript = self.create_buildscript(stage)
1120
1121                 # Execute the buildscript of this stage.
1122                 log.info(_("Running stage %s:") % stage)
1123
1124                 try:
1125                         self.execute(buildscript)
1126
1127                 finally:
1128                         # Remove the buildscript.
1129                         if os.path.exists(buildscript):
1130                                 os.unlink(buildscript)
1131
1132         def post_remove_static_libs(self):
1133                 keep_libs = self.pkg.lexer.build.get_var("keep_libraries")
1134                 keep_libs = keep_libs.split()
1135
1136                 try:
1137                         self.execute("%s/remove-static-libs %s %s" % \
1138                                 (SCRIPT_DIR, self.buildroot, " ".join(keep_libs)))
1139                 except ShellEnvironmentError as e:
1140                         log.warning(_("Could not remove static libraries: %s") % e)
1141
1142         def post_compress_man_pages(self):
1143                 try:
1144                         self.execute("%s/compress-man-pages %s" % (SCRIPT_DIR, self.buildroot))
1145                 except ShellEnvironmentError as e:
1146                         log.warning(_("Compressing man pages did not complete successfully."))
1147
1148         def post_extract_debuginfo(self):
1149                 args = []
1150
1151                 # Check if we need to run with strict build-id.
1152                 strict_id = self.pkg.lexer.build.get_var("debuginfo_strict_build_id", "true")
1153                 if strict_id in ("true", "yes", "1"):
1154                         args.append("--strict-build-id")
1155
1156                 args.append("--buildroot=%s" % self.pkg.buildroot)
1157                 args.append("--sourcedir=%s" % self.pkg.sourcedir)
1158
1159                 # Get additional options to pass to script.
1160                 options = self.pkg.lexer.build.get_var("debuginfo_options", "")
1161                 args += options.split()
1162
1163                 try:
1164                         self.execute("%s/extract-debuginfo %s %s" % (SCRIPT_DIR, " ".join(args), self.pkg.buildroot))
1165                 except ShellEnvironmentError as e:
1166                         log.error(_("Extracting debuginfo did not complete with success. Aborting build."))
1167                         raise
1168
1169         def find_prerequires(self, scriptlet_file):
1170                 assert os.path.exists(scriptlet_file), "Scriptlet file does not exist: %s" % scriptlet_file
1171
1172                 res = self.run_script("find-prerequires", scriptlet_file)
1173                 prerequires = set(res.splitlines())
1174
1175                 return prerequires
1176
1177         def cleanup(self):
1178                 if os.path.exists(self.buildroot):
1179                         util.rm(self.buildroot)