python/pakfire/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import math
  25 import os
  26 import re
  27 import shutil
  28 import signal
  29 import socket
  30 import tempfile
  31 import time
  32 import uuid
  33
  34 import base
  35 import cgroup
  36 import chroot
  37 import logger
  38 import packages
  39 import packages.file
  40 import packages.packager
  41 import repository
  42 import util
  43 import _pakfire
  44
  45 import logging
  46 log = logging.getLogger("pakfire")
  47
  48 from config import ConfigBuilder
  49 from system import system
  50 from constants import *
  51 from i18n import _
  52 from errors import BuildError, BuildRootLocked, Error
  53
  54
  55 BUILD_LOG_HEADER = """
  56  ____       _     __ _            _           _ _     _
  57 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  58 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  59 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  60 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  61
  62         Version : %(version)s
  63         Host    : %(hostname)s (%(host_arch)s)
  64         Time    : %(time)s
  65
  66 """
  67
  68 class BuildEnviron(object):
  69         # The version of the kernel this machine is running.
  70         kernel_version = os.uname()[2]
  71
  72         def __init__(self, filename=None, distro_name=None, config=None, configs=None, arch=None,
  73                         build_id=None, logfile=None, builder_mode="release", use_cache=None, **pakfire_args):
  74                 # Set mode.
  75                 assert builder_mode in ("development", "release",)
  76                 self.mode = builder_mode
  77
  78                 # Disable the build repository in release mode.
  79                 if self.mode == "release":
  80                         if pakfire_args.has_key("disable_repos") and pakfire_args["disable_repos"]:
  81                                 pakfire_args["disable_repos"] += ["build",]
  82                         else:
  83                                 pakfire_args["disable_repos"] = ["build",]
  84
  85                 # Save the build id and generate one if no build id was provided.
  86                 if not build_id:
  87                         build_id = "%s" % uuid.uuid4()
  88
  89                 self.build_id = build_id
  90
  91                 # Setup the logging.
  92                 if logfile:
  93                         self.log = log.getChild(self.build_id)
  94                         # Propage everything to the root logger that we will see something
  95                         # on the terminal.
  96                         self.log.propagate = 1
  97                         self.log.setLevel(logging.INFO)
  98
  99                         # Add the given logfile to the logger.
 100                         h = logging.FileHandler(logfile)
 101                         self.log.addHandler(h)
 102
 103                         # Format the log output for the file.
 104                         f = logger.BuildFormatter()
 105                         h.setFormatter(f)
 106                 else:
 107                         # If no logile was given, we use the root logger.
 108                         self.log = logging.getLogger("pakfire")
 109
 110                 # Initialize a cgroup (if supported).
 111                 self.cgroup = None
 112                 if cgroup.supported():
 113                         self.cgroup = cgroup.CGroup("pakfire/builder/%s" % self.build_id)
 114
 115                         # Attach the pakfire-builder process to the parent group.
 116                         self.cgroup.parent.attach()
 117
 118                 # Log information about pakfire and some more information, when we
 119                 # are running in release mode.
 120                 if self.mode == "release":
 121                         logdata = {
 122                                 "host_arch"  : system.arch,
 123                                 "hostname"   : system.hostname,
 124                                 "time"       : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
 125                                 "version"    : "Pakfire %s" % PAKFIRE_VERSION,
 126                         }
 127
 128                         for line in BUILD_LOG_HEADER.splitlines():
 129                                 self.log.info(line % logdata)
 130
 131                 # Create pakfire instance.
 132                 if pakfire_args.has_key("mode"):
 133                         del pakfire_args["mode"]
 134
 135                 if config is None:
 136                         config = ConfigBuilder(files=configs)
 137
 138                         if not configs:
 139                                 if distro_name is None:
 140                                         distro_name = config.get("builder", "distro", None)
 141                                 config.load_distro_config(distro_name)
 142
 143                 if not config.has_distro():
 144                         log.error(_("You have not set the distribution for which you want to build."))
 145                         log.error(_("Please do so in builder.conf or on the CLI."))
 146                         raise ConfigError, _("Distribution configuration is missing.")
 147
 148                 self.pakfire = base.Pakfire(
 149                         mode="builder",
 150                         config=config,
 151                         arch=arch,
 152                         **pakfire_args
 153                 )
 154
 155                 self.distro = self.pakfire.distro
 156                 self.path = self.pakfire.path
 157
 158                 # Check if this host can build the requested architecture.
 159                 if not system.host_supports_arch(self.arch):
 160                         raise BuildError, _("Cannot build for %s on this host.") % self.arch
 161
 162                 # Where do we put the result?
 163                 self.resultdir = os.path.join(self.path, "result")
 164
 165                 # Check weather to use or not use the cache.
 166                 if use_cache is None:
 167                         # If use_cache is None, the user did not provide anything and
 168                         # so we guess.
 169                         if self.mode == "development":
 170                                 use_cache = True
 171                         else:
 172                                 use_cache = False
 173
 174                 self.use_cache = use_cache
 175
 176                 # Open package.
 177                 # If we have a plain makefile, we first build a source package and go with that.
 178                 if filename:
 179                         if filename.endswith(".%s" % MAKEFILE_EXTENSION):
 180                                 pkg = packages.Makefile(self.pakfire, filename)
 181                                 filename = pkg.dist(os.path.join(self.resultdir, "src"))
 182
 183                                 assert os.path.exists(filename), filename
 184
 185                         # Open source package.
 186                         self.pkg = packages.SourcePackage(self.pakfire, None, filename)
 187                         assert self.pkg, filename
 188
 189                         # Log the package information.
 190                         self.log.info(_("Package information:"))
 191                         for line in self.pkg.dump(long=True).splitlines():
 192                                 self.log.info("  %s" % line)
 193                         self.log.info("")
 194
 195                         # Path where we extract the package and put all the source files.
 196                         self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name)
 197                 else:
 198                         # No package :(
 199                         self.pkg = None
 200
 201                 # XXX need to make this configureable
 202                 self.settings = {
 203                         "enable_loop_devices" : True,
 204                         "enable_ccache"   : True,
 205                         "enable_icecream" : False,
 206                         "sign_packages"   : False,
 207                 }
 208                 #self.settings.update(settings)
 209
 210                 # Try to get the configured host key. If it is available,
 211                 # we will automatically sign all packages with it.
 212                 if self.keyring.get_host_key(secret=True):
 213                         self.settings["sign_packages"] = True
 214
 215                 # Lock the buildroot
 216                 self._lock = None
 217                 self.lock()
 218
 219                 # Save the build time.
 220                 self.build_time = int(time.time())
 221
 222         def setup_signal_handlers(self):
 223                 pass
 224
 225         def start(self):
 226                 # Mount the directories.
 227                 self._mountall()
 228
 229                 # Populate /dev.
 230                 self.populate_dev()
 231
 232                 # Setup domain name resolution in chroot.
 233                 self.setup_dns()
 234
 235                 # Extract all needed packages.
 236                 self.extract()
 237
 238         def stop(self):
 239                 if self.cgroup:
 240                         # Kill all still running processes in the cgroup.
 241                         self.cgroup.kill_and_wait()
 242
 243                         # Remove cgroup and all parent cgroups if they are empty.
 244                         self.cgroup.migrate_task(self.cgroup.root, os.getpid())
 245                         self.cgroup.destroy()
 246
 247                         parent = self.cgroup.parent
 248                         while parent:
 249                                 if not parent.is_empty(recursive=True):
 250                                         break
 251
 252                                 parent.destroy()
 253                                 parent = parent.parent
 254
 255                 else:
 256                         util.orphans_kill(self.path)
 257
 258                 # Close pakfire instance.
 259                 del self.pakfire
 260
 261                 # Umount the build environment.
 262                 self._umountall()
 263
 264                 # Remove all files.
 265                 self.destroy()
 266
 267         @property
 268         def arch(self):
 269                 """
 270                         Inherit architecture from distribution configuration.
 271                 """
 272                 return self.distro.arch
 273
 274         @property
 275         def info(self):
 276                 return {
 277                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 278                         "build_host" : socket.gethostname(),
 279                         "build_id"   : self.build_id,
 280                         "build_time" : self.build_time,
 281                 }
 282
 283         @property
 284         def keyring(self):
 285                 """
 286                         Shortcut to access the pakfire keyring.
 287                 """
 288                 return self.pakfire.keyring
 289
 290         def lock(self):
 291                 filename = os.path.join(self.path, ".lock")
 292
 293                 try:
 294                         self._lock = open(filename, "a+")
 295                 except IOError, e:
 296                         return 0
 297
 298                 try:
 299                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 300                 except IOError, e:
 301                         raise BuildRootLocked, "Buildroot is locked"
 302
 303                 return 1
 304
 305         def unlock(self):
 306                 if self._lock:
 307                         self._lock.close()
 308                         self._lock = None
 309
 310         def copyin(self, file_out, file_in):
 311                 if file_in.startswith("/"):
 312                         file_in = file_in[1:]
 313
 314                 file_in = self.chrootPath(file_in)
 315
 316                 #if not os.path.exists(file_out):
 317                 #       return
 318
 319                 dir_in = os.path.dirname(file_in)
 320                 if not os.path.exists(dir_in):
 321                         os.makedirs(dir_in)
 322
 323                 self.log.debug("%s --> %s" % (file_out, file_in))
 324
 325                 shutil.copy2(file_out, file_in)
 326
 327         def copyout(self, file_in, file_out):
 328                 if file_in.startswith("/"):
 329                         file_in = file_in[1:]
 330
 331                 file_in = self.chrootPath(file_in)
 332
 333                 #if not os.path.exists(file_in):
 334                 #       return
 335
 336                 dir_out = os.path.dirname(file_out)
 337                 if not os.path.exists(dir_out):
 338                         os.makedirs(dir_out)
 339
 340                 self.log.debug("%s --> %s" % (file_in, file_out))
 341
 342                 shutil.copy2(file_in, file_out)
 343
 344         def copy_result(self, resultdir):
 345                 # XXX should use find_result_packages
 346
 347                 dir_in = self.chrootPath("result")
 348
 349                 for dir, subdirs, files in os.walk(dir_in):
 350                         basename = os.path.basename(dir)
 351                         dir = dir[len(self.chrootPath()):]
 352                         for file in files:
 353                                 file_in = os.path.join(dir, file)
 354
 355                                 file_out = os.path.join(
 356                                         resultdir,
 357                                         basename,
 358                                         file,
 359                                 )
 360
 361                                 self.copyout(file_in, file_out)
 362
 363         def find_result_packages(self):
 364                 ret = []
 365
 366                 for dir, subdirs, files in os.walk(self.resultdir):
 367                         for file in files:
 368                                 if not file.endswith(".%s" % PACKAGE_EXTENSION):
 369                                         continue
 370
 371                                 file = os.path.join(dir, file)
 372                                 ret.append(file)
 373
 374                 return ret
 375
 376         def extract(self, requires=None, build_deps=True):
 377                 """
 378                         Gets a dependency set and extracts all packages
 379                         to the environment.
 380                 """
 381                 if not requires:
 382                         requires = []
 383
 384                 if self.use_cache and os.path.exists(self.cache_file):
 385                         # If we are told to use the cache, we just import the
 386                         # file.
 387                         self.cache_extract()
 388                 else:
 389                         # Add neccessary build dependencies.
 390                         requires += BUILD_PACKAGES
 391
 392                 # If we have ccache enabled, we need to extract it
 393                 # to the build chroot.
 394                 if self.settings.get("enable_ccache"):
 395                         requires.append("ccache")
 396
 397                 # If we have icecream enabled, we need to extract it
 398                 # to the build chroot.
 399                 if self.settings.get("enable_icecream"):
 400                         requires.append("icecream")
 401
 402                 # Get build dependencies from source package.
 403                 if self.pkg:
 404                         for req in self.pkg.requires:
 405                                 requires.append(req)
 406
 407                 # Install all packages.
 408                 self.log.info(_("Install packages needed for build..."))
 409                 self.install(requires)
 410
 411                 # Copy the makefile and load source tarballs.
 412                 if self.pkg:
 413                         self.pkg.extract(_("Extracting"), prefix=self.build_dir)
 414
 415         def install(self, requires, **kwargs):
 416                 """
 417                         Install everything that is required in requires.
 418                 """
 419                 # If we got nothing to do, we quit immediately.
 420                 if not requires:
 421                         return
 422
 423                 kwargs.update({
 424                         "interactive" : False,
 425                         "logger" : self.log,
 426                 })
 427
 428                 if not kwargs.has_key("allow_downgrade"):
 429                         kwargs["allow_downgrade"] = True
 430
 431                 # Install everything.
 432                 self.pakfire.install(requires, **kwargs)
 433
 434         def chrootPath(self, *args):
 435                 # Remove all leading slashes
 436                 _args = []
 437                 for arg in args:
 438                         if arg.startswith("/"):
 439                                 arg = arg[1:]
 440                         _args.append(arg)
 441                 args = _args
 442
 443                 ret = os.path.join(self.path, *args)
 444                 ret = ret.replace("//", "/")
 445
 446                 assert ret.startswith(self.path)
 447
 448                 return ret
 449
 450         def populate_dev(self):
 451                 nodes = [
 452                         "/dev/null",
 453                         "/dev/zero",
 454                         "/dev/full",
 455                         "/dev/random",
 456                         "/dev/urandom",
 457                         "/dev/tty",
 458                         "/dev/ptmx",
 459                         "/dev/kmsg",
 460                         "/dev/rtc0",
 461                         "/dev/console",
 462                 ]
 463
 464                 # If we need loop devices (which are optional) we create them here.
 465                 if self.settings["enable_loop_devices"]:
 466                         for i in range(0, 7):
 467                                 nodes.append("/dev/loop%d" % i)
 468
 469                 for node in nodes:
 470                         # Stat the original node of the host system and copy it to
 471                         # the build chroot.
 472                         node_stat = os.stat(node)
 473
 474                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 475
 476                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 477                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 478                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 479                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 480
 481         def setup_dns(self):
 482                 """
 483                         Add DNS resolution facility to chroot environment by copying
 484                         /etc/resolv.conf and /etc/hosts.
 485                 """
 486                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 487                         self.copyin(i, i)
 488
 489         def _create_node(self, filename, mode, device):
 490                 self.log.debug("Create node: %s (%s)" % (filename, mode))
 491
 492                 filename = self.chrootPath(filename)
 493
 494                 # Create parent directory if it is missing.
 495                 dirname = os.path.dirname(filename)
 496                 if not os.path.exists(dirname):
 497                         os.makedirs(dirname)
 498
 499                 os.mknod(filename, mode, device)
 500
 501         def destroy(self):
 502                 self.log.debug("Destroying environment %s" % self.path)
 503
 504                 if os.path.exists(self.path):
 505                         util.rm(self.path)
 506
 507         def cleanup(self):
 508                 self.log.debug("Cleaning environemnt.")
 509
 510                 # Remove the build directory and buildroot.
 511                 dirs = (self.build_dir, self.chrootPath("result"),)
 512
 513                 for d in dirs:
 514                         if not os.path.exists(d):
 515                                 continue
 516
 517                         util.rm(d)
 518                         os.makedirs(d)
 519
 520         def _mountall(self):
 521                 self.log.debug("Mounting environment")
 522                 for src, dest, fs, options in self.mountpoints:
 523                         mountpoint = self.chrootPath(dest)
 524                         if options:
 525                                 options = "-o %s" % options
 526
 527                         # Eventually create mountpoint directory
 528                         if not os.path.exists(mountpoint):
 529                                 os.makedirs(mountpoint)
 530
 531                         cmd = "mount -n -t %s %s %s %s" % \
 532                                 (fs, options, src, mountpoint)
 533                         chroot.do(cmd, shell=True)
 534
 535         def _umountall(self):
 536                 self.log.debug("Umounting environment")
 537
 538                 mountpoints = []
 539                 for src, dest, fs, options in reversed(self.mountpoints):
 540                         if not dest in mountpoints:
 541                                 mountpoints.append(dest)
 542
 543                 for dest in mountpoints:
 544                         mountpoint = self.chrootPath(dest)
 545
 546                         chroot.do("umount -n %s" % mountpoint, raiseExc=0, shell=True)
 547
 548         @property
 549         def mountpoints(self):
 550                 mountpoints = []
 551
 552                 # Make root as a tmpfs.
 553                 #mountpoints += [
 554                 #       ("pakfire_root", "/", "tmpfs", "defaults"),
 555                 #]
 556
 557                 mountpoints += [
 558                         # src, dest, fs, options
 559                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 560                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 561                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 562                         ("/sys",          "/sys",      "bind",  "bind"),
 563                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 564                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 565                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 566                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 567                 ]
 568
 569                 # If selinux is enabled.
 570                 if os.path.exists("/sys/fs/selinux"):
 571                         mountpoints += [
 572                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 573                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 574                         ]
 575
 576                 # If ccache support is requested, we bind mount the cache.
 577                 if self.settings.get("enable_ccache"):
 578                         # Create ccache cache directory if it does not exist.
 579                         if not os.path.exists(CCACHE_CACHE_DIR):
 580                                 os.makedirs(CCACHE_CACHE_DIR)
 581
 582                         mountpoints += [
 583                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 584                         ]
 585
 586                 return mountpoints
 587
 588         @property
 589         def environ(self):
 590                 env = {
 591                         # Add HOME manually, because it is occasionally not set
 592                         # and some builds get in trouble then.
 593                         "PATH" : "/usr/bin:/bin:/usr/sbin:/sbin",
 594                         "HOME" : "/root",
 595                         "TERM" : os.environ.get("TERM", "vt100"),
 596                         "PS1"  : "\u:\w\$ ",
 597
 598                         # Sanitize language.
 599                         "LANG" : os.environ.setdefault("LANG", "en_US.UTF-8"),
 600
 601                         # Set the container that we can detect, if we are inside a
 602                         # chroot.
 603                         "container" : "pakfire-builder",
 604                 }
 605
 606                 # Inherit environment from distro
 607                 env.update(self.pakfire.distro.environ)
 608
 609                 # Icecream environment settings
 610                 if self.settings.get("enable_icecream", False):
 611                         # Set the toolchain path
 612                         if self.settings.get("icecream_toolchain", None):
 613                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 614
 615                         # Set preferred host if configured.
 616                         if self.settings.get("icecream_preferred_host", None):
 617                                 env["ICECC_PREFERRED_HOST"] = \
 618                                         self.settings.get("icecream_preferred_host")
 619
 620                 # Fake UTS_MACHINE, when we cannot use the personality syscall and
 621                 # if the host architecture is not equal to the target architecture.
 622                 if not self.pakfire.distro.personality and \
 623                                 not system.native_arch == self.pakfire.distro.arch:
 624                         env.update({
 625                                 "LD_PRELOAD"  : "/usr/lib/libpakfire_preload.so",
 626                                 "UTS_MACHINE" : self.pakfire.distro.arch,
 627                         })
 628
 629                 return env
 630
 631         @property
 632         def installed_packages(self):
 633                 """
 634                         Returns an iterator over all installed packages in this build environment.
 635                 """
 636                 # Get the repository of all installed packages.
 637                 repo = self.pakfire.repos.get_repo("@system")
 638
 639                 # Return an iterator over the packages.
 640                 return iter(repo)
 641
 642         def write_config(self):
 643                 # Cleanup everything in /etc/pakfire.
 644                 util.rm(self.chrootPath(CONFIG_DIR))
 645
 646                 for i in (CONFIG_DIR, CONFIG_REPOS_DIR):
 647                         i = self.chrootPath(i)
 648                         if not os.path.exists(i):
 649                                 os.makedirs(i)
 650
 651                 # Write general.conf.
 652                 f = open(self.chrootPath(CONFIG_DIR, "general.conf"), "w")
 653                 f.close()
 654
 655                 # Write builder.conf.
 656                 f = open(self.chrootPath(CONFIG_DIR, "builder.conf"), "w")
 657                 f.write(self.distro.get_config())
 658                 f.close()
 659
 660                 # Create pakfire configuration files.
 661                 for repo in self.pakfire.repos:
 662                         conf = repo.get_config()
 663
 664                         if not conf:
 665                                 continue
 666
 667                         filename = self.chrootPath(CONFIG_REPOS_DIR, "%s.repo" % repo.name)
 668                         f = open(filename, "w")
 669                         f.write("\n".join(conf))
 670                         f.close()
 671
 672         def do(self, command, shell=True, personality=None, logger=None, *args, **kwargs):
 673                 ret = None
 674
 675                 # Environment variables
 676                 env = self.environ
 677
 678                 if kwargs.has_key("env"):
 679                         env.update(kwargs.pop("env"))
 680
 681                 self.log.debug("Environment:")
 682                 for k, v in sorted(env.items()):
 683                         self.log.debug("  %s=%s" % (k, v))
 684
 685                 # Update personality it none was set
 686                 if not personality:
 687                         personality = self.distro.personality
 688
 689                 # Make every shell to a login shell because we set a lot of
 690                 # environment things there.
 691                 if shell:
 692                         command = ["bash", "--login", "-c", command]
 693
 694                 if not kwargs.has_key("chrootPath"):
 695                         kwargs["chrootPath"] = self.chrootPath()
 696
 697                 if not kwargs.has_key("cgroup"):
 698                         kwargs["cgroup"] = self.cgroup
 699
 700                 ret = chroot.do(
 701                         command,
 702                         personality=personality,
 703                         shell=False,
 704                         env=env,
 705                         logger=logger,
 706                         *args,
 707                         **kwargs
 708                 )
 709
 710                 return ret
 711
 712         def build(self, install_test=True):
 713                 if not self.pkg:
 714                         raise BuildError, _("You cannot run a build when no package was given.")
 715
 716                 # Search for the package file in build_dir and raise BuildError if it is not present.
 717                 pkgfile = os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION))
 718                 if not os.path.exists(pkgfile):
 719                         raise BuildError, _("Could not find makefile in build root: %s") % pkgfile
 720                 pkgfile = "/%s" % os.path.relpath(pkgfile, self.chrootPath())
 721
 722                 # Write pakfire configuration into the chroot.
 723                 self.write_config()
 724
 725                 # Create the build command, that is executed in the chroot.
 726                 build_command = [
 727                         "/usr/lib/pakfire/builder",
 728                         "--offline",
 729                         "build",
 730                         pkgfile,
 731                         "--arch", self.arch,
 732                         "--nodeps",
 733                         "--resultdir=/result",
 734                 ]
 735
 736                 try:
 737                         self.do(" ".join(build_command), logger=self.log)
 738
 739                         # Perform the install test after the actual build.
 740                         if install_test:
 741                                 self.install_test()
 742
 743                 except Error:
 744                         self.log.error(_("Build failed."), exc_info=True)
 745
 746                         raise BuildError, _("The build command failed. See logfile for details.")
 747
 748                 # Sign all built packages with the host key (if available).
 749                 if self.settings.get("sign_packages"):
 750                         host_key = self.keyring.get_host_key_id()
 751                         assert host_key
 752
 753                         # Do the signing...
 754                         self.sign(host_key)
 755
 756                 # Dump package information.
 757                 self.dump()
 758
 759         def install_test(self):
 760                 self.log.info(_("Running installation test..."))
 761
 762                 # Install all packages that were built.
 763                 self.install(self.find_result_packages(),
 764                         allow_vendorchange=True, allow_archchange=True,
 765                         allow_uninstall=True, signatures_mode="disabled")
 766
 767                 self.log.info(_("Installation test succeeded."))
 768                 self.log.info("")
 769
 770         def shell(self, args=[]):
 771                 if not util.cli_is_interactive():
 772                         self.log.warning("Cannot run shell on non-interactive console.")
 773                         return
 774
 775                 # Install all packages that are needed to run a shell.
 776                 self.install(SHELL_PACKAGES)
 777
 778                 # XXX need to set CFLAGS here
 779                 command = "/usr/sbin/chroot %s %s %s" % \
 780                         (self.chrootPath(), SHELL_SCRIPT, " ".join(args))
 781
 782                 # Add personality if we require one
 783                 if self.pakfire.distro.personality:
 784                         command = "%s %s" % (self.pakfire.distro.personality, command)
 785
 786                 for key, val in self.environ.items():
 787                         command = "%s=\"%s\" " % (key, val) + command
 788
 789                 # Empty the environment
 790                 command = "env -i - %s" % command
 791
 792                 self.log.debug("Shell command: %s" % command)
 793
 794                 shell = os.system(command)
 795                 return os.WEXITSTATUS(shell)
 796
 797         def sign(self, keyfp):
 798                 assert self.keyring.get_key(keyfp), "Key for signing does not exist"
 799
 800                 # Find all files to process.
 801                 files = self.find_result_packages()
 802
 803                 # Create a progressbar.
 804                 print _("Signing packages...")
 805                 p = util.make_progress(keyfp, len(files))
 806                 i = 0
 807
 808                 for file in files:
 809                         # Update progressbar.
 810                         if p:
 811                                 i += 1
 812                                 p.update(i)
 813
 814                         # Open package file.
 815                         pkg = packages.open(self.pakfire, None, file)
 816
 817                         # Sign it.
 818                         pkg.sign(keyfp)
 819
 820                 # Close progressbar.
 821                 if p:
 822                         p.finish()
 823                         print "" # Print an empty line.
 824
 825         def dump(self):
 826                 pkgs = []
 827
 828                 for file in self.find_result_packages():
 829                         pkg = packages.open(self.pakfire, None, file)
 830                         pkgs.append(pkg)
 831
 832                 # If there are no packages, there is nothing to do.
 833                 if not pkgs:
 834                         return
 835
 836                 pkgs.sort()
 837
 838                 self.log.info(_("Dumping package information:"))
 839                 for pkg in pkgs:
 840                         dump = pkg.dump(long=True)
 841
 842                         for line in dump.splitlines():
 843                                 self.log.info("  %s" % line)
 844                         self.log.info("") # Empty line.
 845
 846         @property
 847         def cache_file(self):
 848                 comps = [
 849                         self.pakfire.distro.sname,      # name of the distribution
 850                         self.pakfire.distro.release,    # release version
 851                         self.pakfire.distro.arch,       # architecture
 852                 ]
 853
 854                 return os.path.join(CACHE_ENVIRON_DIR, "%s.cache" %"-".join(comps))
 855
 856         def cache_export(self, filename):
 857                 # Sync all disk caches.
 858                 _pakfire.sync()
 859
 860                 # A list to store all mountpoints, so we don't package them.
 861                 mountpoints = []
 862
 863                 # A list containing all files we want to package.
 864                 filelist = []
 865
 866                 # Walk through the whole tree and collect all files
 867                 # that are on the same disk (not crossing mountpoints).
 868                 log.info(_("Creating filelist..."))
 869                 root = self.chrootPath()
 870                 for dir, subdirs, files in os.walk(root):
 871                         # Search for mountpoints and skip them.
 872                         if not dir == root and os.path.ismount(dir):
 873                                 mountpoints.append(dir)
 874                                 continue
 875
 876                         # Skip all directories under mountpoints.
 877                         if any([dir.startswith(m) for m in mountpoints]):
 878                                 continue
 879
 880                         # Add all other files.
 881                         filelist.append(dir)
 882                         for file in files:
 883                                 file = os.path.join(dir, file)
 884                                 filelist.append(file)
 885
 886                 # Create a nice progressbar.
 887                 p = util.make_progress(_("Compressing files..."), len(filelist))
 888                 i = 0
 889
 890                 # Create tar file and add all files to it.
 891                 f = packages.file.InnerTarFile.open(filename, "w:gz")
 892                 for file in filelist:
 893                         i += 1
 894                         if p:
 895                                 p.update(i)
 896
 897                         f.add(file, os.path.relpath(file, root), recursive=False)
 898                 f.close()
 899
 900                 # Finish progressbar.
 901                 if p:
 902                         p.finish()
 903
 904                 filesize = os.path.getsize(filename)
 905
 906                 log.info(_("Cache file was successfully created at %s.") % filename)
 907                 log.info(_("  Containing %(files)s files, it has a size of %(size)s.") % \
 908                         { "files" : len(filelist), "size" : util.format_size(filesize), })
 909
 910         def cache_extract(self):
 911                 root = self.chrootPath()
 912                 filename = self.cache_file
 913
 914                 f = packages.file.InnerTarFile.open(filename, "r:gz")
 915                 members = f.getmembers()
 916
 917                 # Make a nice progress bar as always.
 918                 p = util.make_progress(_("Extracting files..."), len(members))
 919
 920                 # Extract all files from the cache.
 921                 i = 0
 922                 for member in members:
 923                         if p:
 924                                 i += 1
 925                                 p.update(i)
 926
 927                         f.extract(member, path=root)
 928                 f.close()
 929
 930                 # Finish progressbar.
 931                 if p:
 932                         p.finish()
 933
 934                 # Re-read local repository.
 935                 self.pakfire.repos.local.update(force=True)
 936
 937                 # Update all packages.
 938                 self.log.info(_("Updating packages from cache..."))
 939                 self.pakfire.update(interactive=False, logger=self.log,
 940                         allow_archchange=True, allow_vendorchange=True, allow_downgrade=True)
 941
 942
 943 class Builder(object):
 944         def __init__(self, pakfire, filename, resultdir, **kwargs):
 945                 self.pakfire = pakfire
 946
 947                 self.filename = filename
 948
 949                 self.resultdir = resultdir
 950
 951                 # Open package file.
 952                 self.pkg = packages.Makefile(self.pakfire, self.filename)
 953
 954                 self._environ = {
 955                         "LANG"             : "C",
 956                 }
 957
 958         @property
 959         def buildroot(self):
 960                 return self.pkg.buildroot
 961
 962         @property
 963         def distro(self):
 964                 return self.pakfire.distro
 965
 966         @property
 967         def environ(self):
 968                 environ = os.environ
 969
 970                 # Get all definitions from the package.
 971                 environ.update(self.pkg.exports)
 972
 973                 # Overwrite some definitions by default values.
 974                 environ.update(self._environ)
 975
 976                 return environ
 977
 978         def do(self, command, shell=True, *args, **kwargs):
 979                 try:
 980                         logger = kwargs["logger"]
 981                 except KeyError:
 982                         logger = logging.getLogger("pakfire")
 983                         kwargs["logger"] = logger
 984
 985                 # Environment variables
 986                 log.debug("Environment:")
 987                 for k, v in sorted(self.environ.items()):
 988                         log.debug("  %s=%s" % (k, v))
 989
 990                 # Update personality it none was set
 991                 if not kwargs.has_key("personality"):
 992                         kwargs["personality"] = self.distro.personality
 993
 994                 if not kwargs.has_key("cwd"):
 995                         kwargs["cwd"] = "/%s" % LOCAL_TMP_PATH
 996
 997                 # Make every shell to a login shell because we set a lot of
 998                 # environment things there.
 999                 if shell:
1000                         command = ["bash", "--login", "-c", command]
1001                         kwargs["shell"] = False
1002
1003                 kwargs["env"] = self.environ
1004
1005                 try:
1006                         return chroot.do(command, *args, **kwargs)
1007                 except Error:
1008                         if not logger:
1009                                 logger = logging.getLogger("pakfire")
1010
1011                         logger.error("Command exited with an error: %s" % command)
1012                         raise
1013
1014         def run_script(self, script, *args):
1015                 if not script.startswith("/"):
1016                         script = os.path.join(SCRIPT_DIR, script)
1017
1018                 assert os.path.exists(script), "Script we should run does not exist: %s" % script
1019
1020                 cmd = [script,]
1021                 for arg in args:
1022                         cmd.append(arg)
1023                 cmd = " ".join(cmd)
1024
1025                 # Returns the output of the command, but the output won't get
1026                 # logged.
1027                 return self.do(cmd, returnOutput=True, logger=None)
1028
1029         def create_icecream_toolchain(self):
1030                 try:
1031                         out = self.do("icecc --build-native 2>/dev/null", returnOutput=True, cwd="/tmp")
1032                 except Error:
1033                         return
1034
1035                 for line in out.splitlines():
1036                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
1037                         if m:
1038                                 self._environ["ICECC_VERSION"] = "/tmp/%s" % m.group(1)
1039
1040         def create_buildscript(self, stage):
1041                 # Get buildscript from the package.
1042                 script = self.pkg.get_buildscript(stage)
1043
1044                 # Write script to an empty file.
1045                 f = tempfile.NamedTemporaryFile(mode="w", delete=False)
1046                 f.write("#!/bin/sh\n\n")
1047                 f.write("set -e\n")
1048                 f.write("set -x\n")
1049                 f.write("\n%s\n" % script)
1050                 f.write("exit 0\n")
1051                 f.close()
1052
1053                 # Make the script executable.
1054                 os.chmod(f.name, 700)
1055
1056                 return f.name
1057
1058         def build(self):
1059                 # Create buildroot and remove all content if it was existant.
1060                 util.rm(self.buildroot)
1061                 os.makedirs(self.buildroot)
1062
1063                 # Build icecream toolchain if icecream is installed.
1064                 self.create_icecream_toolchain()
1065
1066                 for stage in ("prepare", "build", "test", "install"):
1067                         self.build_stage(stage)
1068
1069                 # Run post-build stuff.
1070                 self.post_compress_man_pages()
1071                 self.post_remove_static_libs()
1072                 self.post_extract_debuginfo()
1073
1074                 # Package the result.
1075                 # Make all these little package from the build environment.
1076                 log.info(_("Creating packages:"))
1077                 pkgs = []
1078                 for pkg in reversed(self.pkg.packages):
1079                         packager = packages.packager.BinaryPackager(self.pakfire, pkg,
1080                                 self, self.buildroot)
1081                         pkg = packager.run(self.resultdir)
1082                         pkgs.append(pkg)
1083                 log.info("")
1084
1085         def build_stage(self, stage):
1086                 # Get the buildscript for this stage.
1087                 buildscript = self.create_buildscript(stage)
1088
1089                 # Execute the buildscript of this stage.
1090                 log.info(_("Running stage %s:") % stage)
1091
1092                 try:
1093                         self.do(buildscript, shell=False)
1094
1095                 finally:
1096                         # Remove the buildscript.
1097                         if os.path.exists(buildscript):
1098                                 os.unlink(buildscript)
1099
1100         def post_remove_static_libs(self):
1101                 keep_libs = self.pkg.lexer.build.get_var("keep_libraries")
1102                 keep_libs = keep_libs.split()
1103
1104                 try:
1105                         self.do("%s/remove-static-libs %s %s" % \
1106                                 (SCRIPT_DIR, self.buildroot, " ".join(keep_libs)))
1107                 except Error, e:
1108                         log.warning(_("Could not remove static libraries: %s") % e)
1109
1110         def post_compress_man_pages(self):
1111                 try:
1112                         self.do("%s/compress-man-pages %s" % (SCRIPT_DIR, self.buildroot))
1113                 except Error, e:
1114                         log.warning(_("Compressing man pages did not complete successfully."))
1115
1116         def post_extract_debuginfo(self):
1117                 args = []
1118
1119                 # Check if we need to run with strict build-id.
1120                 strict_id = self.pkg.lexer.build.get_var("debuginfo_strict_build_id", "true")
1121                 if strict_id in ("true", "yes", "1"):
1122                         args.append("--strict-build-id")
1123
1124                 args.append("--buildroot=%s" % self.pkg.buildroot)
1125                 args.append("--sourcedir=%s" % self.pkg.sourcedir)
1126
1127                 # Get additional options to pass to script.
1128                 options = self.pkg.lexer.build.get_var("debuginfo_options", "")
1129                 args += options.split()
1130
1131                 try:
1132                         self.do("%s/extract-debuginfo %s %s" % (SCRIPT_DIR, " ".join(args), self.pkg.buildroot))
1133                 except Error, e:
1134                         log.error(_("Extracting debuginfo did not complete with success. Aborting build."))
1135                         raise
1136
1137         def find_prerequires(self, scriptlet_file):
1138                 assert os.path.exists(scriptlet_file), "Scriptlet file does not exist: %s" % scriptlet_file
1139
1140                 res = self.run_script("find-prerequires", scriptlet_file)
1141                 prerequires = set(res.splitlines())
1142
1143                 return prerequires
1144
1145         def cleanup(self):
1146                 if os.path.exists(self.buildroot):
1147                         util.rm(self.buildroot)