python/pakfire/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import math
  25 import os
  26 import re
  27 import shutil
  28 import socket
  29 import time
  30 import uuid
  31
  32 import base
  33 import chroot
  34 import logger
  35 import packages
  36 import packages.file
  37 import packages.packager
  38 import repository
  39 import util
  40 import _pakfire
  41
  42 import logging
  43 log = logging.getLogger("pakfire")
  44
  45 from config import ConfigBuilder
  46 from system import system
  47 from constants import *
  48 from i18n import _
  49 from errors import BuildError, BuildRootLocked, Error
  50
  51
  52 BUILD_LOG_HEADER = """
  53  ____       _     __ _            _           _ _     _
  54 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  55 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  56 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  57 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  58
  59         Version : %(version)s
  60         Host    : %(hostname)s (%(host_arch)s)
  61         Time    : %(time)s
  62
  63 """
  64
  65 class BuildEnviron(object):
  66         # The version of the kernel this machine is running.
  67         kernel_version = os.uname()[2]
  68
  69         def __init__(self, filename=None, distro_name=None, config=None, configs=None, arch=None,
  70                         build_id=None, logfile=None, builder_mode="release", use_cache=None, **pakfire_args):
  71                 # Set mode.
  72                 assert builder_mode in ("development", "release",)
  73                 self.mode = builder_mode
  74
  75                 # Disable the build repository in release mode.
  76                 if self.mode == "release":
  77                         if pakfire_args.has_key("disable_repos") and pakfire_args["disable_repos"]:
  78                                 pakfire_args["disable_repos"] += ["build",]
  79                         else:
  80                                 pakfire_args["disable_repos"] = ["build",]
  81
  82                 # Save the build id and generate one if no build id was provided.
  83                 if not build_id:
  84                         build_id = "%s" % uuid.uuid4()
  85
  86                 self.build_id = build_id
  87
  88                 # Setup the logging.
  89                 if logfile:
  90                         self.log = log.getChild(self.build_id)
  91                         # Propage everything to the root logger that we will see something
  92                         # on the terminal.
  93                         self.log.propagate = 1
  94                         self.log.setLevel(logging.INFO)
  95
  96                         # Add the given logfile to the logger.
  97                         h = logging.FileHandler(logfile)
  98                         self.log.addHandler(h)
  99
 100                         # Format the log output for the file.
 101                         f = logger.BuildFormatter()
 102                         h.setFormatter(f)
 103                 else:
 104                         # If no logile was given, we use the root logger.
 105                         self.log = logging.getLogger("pakfire")
 106
 107                 # Log information about pakfire and some more information, when we
 108                 # are running in release mode.
 109                 if self.mode == "release":
 110                         logdata = {
 111                                 "host_arch"  : system.arch,
 112                                 "hostname"   : system.hostname,
 113                                 "time"       : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
 114                                 "version"    : "Pakfire %s" % PAKFIRE_VERSION,
 115                         }
 116
 117                         for line in BUILD_LOG_HEADER.splitlines():
 118                                 self.log.info(line % logdata)
 119
 120                 # Create pakfire instance.
 121                 if pakfire_args.has_key("mode"):
 122                         del pakfire_args["mode"]
 123
 124                 if config is None:
 125                         config = ConfigBuilder(files=configs)
 126
 127                         if not configs:
 128                                 if distro_name is None:
 129                                         distro_name = config.get("builder", "distro", None)
 130                                 config.load_distro_config(distro_name)
 131
 132                 if not config.has_distro():
 133                         log.error(_("You have not set the distribution for which you want to build."))
 134                         log.error(_("Please do so in builder.conf or on the CLI."))
 135                         raise ConfigError, _("Distribution configuration is missing.")
 136
 137                 self.pakfire = base.Pakfire(
 138                         mode="builder",
 139                         config=config,
 140                         arch=arch,
 141                         **pakfire_args
 142                 )
 143
 144                 self.distro = self.pakfire.distro
 145                 self.path = self.pakfire.path
 146
 147                 # Check if this host can build the requested architecture.
 148                 if not system.host_supports_arch(self.arch):
 149                         raise BuildError, _("Cannot build for %s on this host.") % self.arch
 150
 151                 # Where do we put the result?
 152                 self.resultdir = os.path.join(self.path, "result")
 153
 154                 # Check weather to use or not use the cache.
 155                 if use_cache is None:
 156                         # If use_cache is None, the user did not provide anything and
 157                         # so we guess.
 158                         if self.mode == "development":
 159                                 use_cache = True
 160                         else:
 161                                 use_cache = False
 162
 163                 self.use_cache = use_cache
 164
 165                 # Open package.
 166                 # If we have a plain makefile, we first build a source package and go with that.
 167                 if filename:
 168                         if filename.endswith(".%s" % MAKEFILE_EXTENSION):
 169                                 pkg = packages.Makefile(self.pakfire, filename)
 170                                 filename = pkg.dist(os.path.join(self.resultdir, "src"))
 171
 172                                 assert os.path.exists(filename), filename
 173
 174                         # Open source package.
 175                         self.pkg = packages.SourcePackage(self.pakfire, None, filename)
 176                         assert self.pkg, filename
 177
 178                         # Log the package information.
 179                         self.log.info(_("Package information:"))
 180                         for line in self.pkg.dump(long=True).splitlines():
 181                                 self.log.info("  %s" % line)
 182                         self.log.info("")
 183
 184                         # Path where we extract the package and put all the source files.
 185                         self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name)
 186                 else:
 187                         # No package :(
 188                         self.pkg = None
 189
 190                 # XXX need to make this configureable
 191                 self.settings = {
 192                         "enable_loop_devices" : True,
 193                         "enable_ccache"   : True,
 194                         "enable_icecream" : False,
 195                 }
 196                 #self.settings.update(settings)
 197
 198                 # Lock the buildroot
 199                 self._lock = None
 200                 self.lock()
 201
 202                 # Save the build time.
 203                 self.build_time = int(time.time())
 204
 205         def start(self):
 206                 # Mount the directories.
 207                 self._mountall()
 208
 209                 # Populate /dev.
 210                 self.populate_dev()
 211
 212                 # Setup domain name resolution in chroot.
 213                 self.setup_dns()
 214
 215                 # Extract all needed packages.
 216                 self.extract()
 217
 218         def stop(self):
 219                 # Kill all still running processes.
 220                 util.orphans_kill(self.path)
 221
 222                 # Close pakfire instance.
 223                 del self.pakfire
 224
 225                 # Umount the build environment.
 226                 self._umountall()
 227
 228                 # Remove all files.
 229                 self.destroy()
 230
 231         @property
 232         def arch(self):
 233                 """
 234                         Inherit architecture from distribution configuration.
 235                 """
 236                 return self.distro.arch
 237
 238         @property
 239         def info(self):
 240                 return {
 241                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 242                         "build_host" : socket.gethostname(),
 243                         "build_id"   : self.build_id,
 244                         "build_time" : self.build_time,
 245                 }
 246
 247         def lock(self):
 248                 filename = os.path.join(self.path, ".lock")
 249
 250                 try:
 251                         self._lock = open(filename, "a+")
 252                 except IOError, e:
 253                         return 0
 254
 255                 try:
 256                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 257                 except IOError, e:
 258                         raise BuildRootLocked, "Buildroot is locked"
 259
 260                 return 1
 261
 262         def unlock(self):
 263                 if self._lock:
 264                         self._lock.close()
 265                         self._lock = None
 266
 267         def copyin(self, file_out, file_in):
 268                 if file_in.startswith("/"):
 269                         file_in = file_in[1:]
 270
 271                 file_in = self.chrootPath(file_in)
 272
 273                 #if not os.path.exists(file_out):
 274                 #       return
 275
 276                 dir_in = os.path.dirname(file_in)
 277                 if not os.path.exists(dir_in):
 278                         os.makedirs(dir_in)
 279
 280                 self.log.debug("%s --> %s" % (file_out, file_in))
 281
 282                 shutil.copy2(file_out, file_in)
 283
 284         def copyout(self, file_in, file_out):
 285                 if file_in.startswith("/"):
 286                         file_in = file_in[1:]
 287
 288                 file_in = self.chrootPath(file_in)
 289
 290                 #if not os.path.exists(file_in):
 291                 #       return
 292
 293                 dir_out = os.path.dirname(file_out)
 294                 if not os.path.exists(dir_out):
 295                         os.makedirs(dir_out)
 296
 297                 self.log.debug("%s --> %s" % (file_in, file_out))
 298
 299                 shutil.copy2(file_in, file_out)
 300
 301         def copy_result(self, resultdir):
 302                 # XXX should use find_result_packages
 303
 304                 dir_in = self.chrootPath("result")
 305
 306                 for dir, subdirs, files in os.walk(dir_in):
 307                         basename = os.path.basename(dir)
 308                         dir = dir[len(self.chrootPath()):]
 309                         for file in files:
 310                                 file_in = os.path.join(dir, file)
 311
 312                                 file_out = os.path.join(
 313                                         resultdir,
 314                                         basename,
 315                                         file,
 316                                 )
 317
 318                                 self.copyout(file_in, file_out)
 319
 320         def find_result_packages(self):
 321                 ret = []
 322
 323                 for dir, subdirs, files in os.walk(self.resultdir):
 324                         for file in files:
 325                                 if not file.endswith(".%s" % PACKAGE_EXTENSION):
 326                                         continue
 327
 328                                 file = os.path.join(dir, file)
 329                                 ret.append(file)
 330
 331                 return ret
 332
 333         def extract(self, requires=None, build_deps=True):
 334                 """
 335                         Gets a dependency set and extracts all packages
 336                         to the environment.
 337                 """
 338                 if not requires:
 339                         requires = []
 340
 341                 if self.use_cache and os.path.exists(self.cache_file):
 342                         # If we are told to use the cache, we just import the
 343                         # file.
 344                         self.cache_extract()
 345                 else:
 346                         # Add neccessary build dependencies.
 347                         requires += BUILD_PACKAGES
 348
 349                 # If we have ccache enabled, we need to extract it
 350                 # to the build chroot.
 351                 if self.settings.get("enable_ccache"):
 352                         requires.append("ccache")
 353
 354                 # If we have icecream enabled, we need to extract it
 355                 # to the build chroot.
 356                 if self.settings.get("enable_icecream"):
 357                         requires.append("icecream")
 358
 359                 # Get build dependencies from source package.
 360                 if self.pkg:
 361                         for req in self.pkg.requires:
 362                                 requires.append(req)
 363
 364                 # Install all packages.
 365                 self.log.info(_("Install packages needed for build..."))
 366                 self.install(requires)
 367
 368                 # Copy the makefile and load source tarballs.
 369                 if self.pkg:
 370                         self.pkg.extract(_("Extracting"), prefix=self.build_dir)
 371
 372         def install(self, requires):
 373                 """
 374                         Install everything that is required in requires.
 375                 """
 376                 # If we got nothing to do, we quit immediately.
 377                 if not requires:
 378                         return
 379
 380                 try:
 381                         self.pakfire.install(requires, interactive=False,
 382                                 allow_downgrade=True, logger=self.log)
 383
 384                 # Catch dependency errors and log it.
 385                 except DependencyError, e:
 386                         raise
 387
 388         def install_test(self):
 389                 try:
 390                         self.pakfire.localinstall(self.find_result_packages(), yes=True, allow_uninstall=True, logger=self.log)
 391
 392                 # Dependency errors when trying to install the result packages are build errors.
 393                 except DependencyError, e:
 394                         raise BuildError, e
 395
 396         def chrootPath(self, *args):
 397                 # Remove all leading slashes
 398                 _args = []
 399                 for arg in args:
 400                         if arg.startswith("/"):
 401                                 arg = arg[1:]
 402                         _args.append(arg)
 403                 args = _args
 404
 405                 ret = os.path.join(self.path, *args)
 406                 ret = ret.replace("//", "/")
 407
 408                 assert ret.startswith(self.path)
 409
 410                 return ret
 411
 412         def populate_dev(self):
 413                 nodes = [
 414                         "/dev/null",
 415                         "/dev/zero",
 416                         "/dev/full",
 417                         "/dev/random",
 418                         "/dev/urandom",
 419                         "/dev/tty",
 420                         "/dev/ptmx",
 421                         "/dev/kmsg",
 422                         "/dev/rtc0",
 423                         "/dev/console",
 424                 ]
 425
 426                 # If we need loop devices (which are optional) we create them here.
 427                 if self.settings["enable_loop_devices"]:
 428                         for i in range(0, 7):
 429                                 nodes.append("/dev/loop%d" % i)
 430
 431                 for node in nodes:
 432                         # Stat the original node of the host system and copy it to
 433                         # the build chroot.
 434                         node_stat = os.stat(node)
 435
 436                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 437
 438                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 439                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 440                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 441                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 442
 443         def setup_dns(self):
 444                 """
 445                         Add DNS resolution facility to chroot environment by copying
 446                         /etc/resolv.conf and /etc/hosts.
 447                 """
 448                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 449                         self.copyin(i, i)
 450
 451         def _create_node(self, filename, mode, device):
 452                 self.log.debug("Create node: %s (%s)" % (filename, mode))
 453
 454                 filename = self.chrootPath(filename)
 455
 456                 # Create parent directory if it is missing.
 457                 dirname = os.path.dirname(filename)
 458                 if not os.path.exists(dirname):
 459                         os.makedirs(dirname)
 460
 461                 os.mknod(filename, mode, device)
 462
 463         def destroy(self):
 464                 self.log.debug("Destroying environment %s" % self.path)
 465
 466                 if os.path.exists(self.path):
 467                         util.rm(self.path)
 468
 469         def cleanup(self):
 470                 self.log.debug("Cleaning environemnt.")
 471
 472                 # Remove the build directory and buildroot.
 473                 dirs = (self.build_dir, self.chrootPath("result"),)
 474
 475                 for d in dirs:
 476                         if not os.path.exists(d):
 477                                 continue
 478
 479                         util.rm(d)
 480                         os.makedirs(d)
 481
 482         def _mountall(self):
 483                 self.log.debug("Mounting environment")
 484                 for src, dest, fs, options in self.mountpoints:
 485                         mountpoint = self.chrootPath(dest)
 486                         if options:
 487                                 options = "-o %s" % options
 488
 489                         # Eventually create mountpoint directory
 490                         if not os.path.exists(mountpoint):
 491                                 os.makedirs(mountpoint)
 492
 493                         cmd = "mount -n -t %s %s %s %s" % \
 494                                 (fs, options, src, mountpoint)
 495                         chroot.do(cmd, shell=True)
 496
 497         def _umountall(self):
 498                 self.log.debug("Umounting environment")
 499
 500                 mountpoints = []
 501                 for src, dest, fs, options in reversed(self.mountpoints):
 502                         if not dest in mountpoints:
 503                                 mountpoints.append(dest)
 504
 505                 for dest in mountpoints:
 506                         mountpoint = self.chrootPath(dest)
 507
 508                         chroot.do("umount -n %s" % mountpoint, raiseExc=0, shell=True)
 509
 510         @property
 511         def mountpoints(self):
 512                 mountpoints = []
 513
 514                 # Make root as a tmpfs.
 515                 #mountpoints += [
 516                 #       ("pakfire_root", "/", "tmpfs", "defaults"),
 517                 #]
 518
 519                 mountpoints += [
 520                         # src, dest, fs, options
 521                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 522                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 523                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 524                         ("/sys",          "/sys",      "bind",  "bind"),
 525                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 526                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 527                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 528                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 529                 ]
 530
 531                 # If selinux is enabled.
 532                 if os.path.exists("/sys/fs/selinux"):
 533                         mountpoints += [
 534                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 535                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 536                         ]
 537
 538                 # If ccache support is requested, we bind mount the cache.
 539                 if self.settings.get("enable_ccache"):
 540                         # Create ccache cache directory if it does not exist.
 541                         if not os.path.exists(CCACHE_CACHE_DIR):
 542                                 os.makedirs(CCACHE_CACHE_DIR)
 543
 544                         mountpoints += [
 545                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 546                         ]
 547
 548                 return mountpoints
 549
 550         @property
 551         def environ(self):
 552                 env = {
 553                         # Add HOME manually, because it is occasionally not set
 554                         # and some builds get in trouble then.
 555                         "HOME" : "/root",
 556                         "TERM" : os.environ.get("TERM", "dumb"),
 557                         "PS1"  : "\u:\w\$ ",
 558
 559                         # Sanitize language.
 560                         "LANG" : os.environ.setdefault("LANG", "en_US.UTF-8"),
 561
 562                         # Set the container that we can detect, if we are inside a
 563                         # chroot.
 564                         "container" : "pakfire-builder",
 565                 }
 566
 567                 # Inherit environment from distro
 568                 env.update(self.pakfire.distro.environ)
 569
 570                 # Icecream environment settings
 571                 if self.settings.get("enable_icecream", False):
 572                         # Set the toolchain path
 573                         if self.settings.get("icecream_toolchain", None):
 574                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 575
 576                         # Set preferred host if configured.
 577                         if self.settings.get("icecream_preferred_host", None):
 578                                 env["ICECC_PREFERRED_HOST"] = \
 579                                         self.settings.get("icecream_preferred_host")
 580
 581                 # Fake UTS_MACHINE, when we cannot use the personality syscall and
 582                 # if the host architecture is not equal to the target architecture.
 583                 if not self.pakfire.distro.personality and \
 584                                 not system.native_arch == self.pakfire.distro.arch:
 585                         env.update({
 586                                 "LD_PRELOAD"  : "/usr/lib/libpakfire_preload.so",
 587                                 "UTS_MACHINE" : self.pakfire.distro.arch,
 588                         })
 589
 590                 return env
 591
 592         @property
 593         def installed_packages(self):
 594                 """
 595                         Returns an iterator over all installed packages in this build environment.
 596                 """
 597                 # Get the repository of all installed packages.
 598                 repo = self.pakfire.repos.get_repo("@system")
 599
 600                 # Return an iterator over the packages.
 601                 return iter(repo)
 602
 603         def write_config(self):
 604                 # Cleanup everything in /etc/pakfire.
 605                 util.rm(self.chrootPath(CONFIG_DIR))
 606
 607                 for i in (CONFIG_DIR, CONFIG_REPOS_DIR):
 608                         i = self.chrootPath(i)
 609                         if not os.path.exists(i):
 610                                 os.makedirs(i)
 611
 612                 # Write general.conf.
 613                 f = open(self.chrootPath(CONFIG_DIR, "general.conf"), "w")
 614                 f.close()
 615
 616                 # Write builder.conf.
 617                 f = open(self.chrootPath(CONFIG_DIR, "builder.conf"), "w")
 618                 f.write(self.distro.get_config())
 619                 f.close()
 620
 621                 # Create pakfire configuration files.
 622                 for repo in self.pakfire.repos:
 623                         conf = repo.get_config()
 624
 625                         if not conf:
 626                                 continue
 627
 628                         filename = self.chrootPath(CONFIG_REPOS_DIR, "%s.repo" % repo.name)
 629                         f = open(filename, "w")
 630                         f.write("\n".join(conf))
 631                         f.close()
 632
 633         def do(self, command, shell=True, personality=None, logger=None, *args, **kwargs):
 634                 ret = None
 635
 636                 # Environment variables
 637                 env = self.environ
 638
 639                 if kwargs.has_key("env"):
 640                         env.update(kwargs.pop("env"))
 641
 642                 self.log.debug("Environment:")
 643                 for k, v in sorted(env.items()):
 644                         self.log.debug("  %s=%s" % (k, v))
 645
 646                 # Update personality it none was set
 647                 if not personality:
 648                         personality = self.distro.personality
 649
 650                 # Make every shell to a login shell because we set a lot of
 651                 # environment things there.
 652                 if shell:
 653                         command = ["bash", "--login", "-c", command]
 654
 655                 if not kwargs.has_key("chrootPath"):
 656                         kwargs["chrootPath"] = self.chrootPath()
 657
 658                 ret = chroot.do(
 659                         command,
 660                         personality=personality,
 661                         shell=False,
 662                         env=env,
 663                         logger=logger,
 664                         *args,
 665                         **kwargs
 666                 )
 667
 668                 return ret
 669
 670         def build(self, install_test=True):
 671                 if not self.pkg:
 672                         raise BuildError, _("You cannot run a build when no package was given.")
 673
 674                 # Search for the package file in build_dir and raise BuildError if it is not present.
 675                 pkgfile = os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION))
 676                 if not os.path.exists(pkgfile):
 677                         raise BuildError, _("Could not find makefile in build root: %s") % pkgfile
 678                 pkgfile = "/%s" % os.path.relpath(pkgfile, self.chrootPath())
 679
 680                 # Write pakfire configuration into the chroot.
 681                 self.write_config()
 682
 683                 # Create the build command, that is executed in the chroot.
 684                 build_command = [
 685                         "/usr/lib/pakfire/builder",
 686                         "--offline",
 687                         "build",
 688                         pkgfile,
 689                         "--arch", self.arch,
 690                         "--nodeps",
 691                         "--resultdir=/result",
 692                 ]
 693
 694                 try:
 695                         self.do(" ".join(build_command), logger=self.log)
 696
 697                 except Error:
 698                         self.log.error(_("Build failed."), exc_info=True)
 699
 700                         raise BuildError, _("The build command failed. See logfile for details.")
 701
 702                 # Perform install test.
 703                 if install_test:
 704                         self.install_test()
 705
 706         def shell(self, args=[]):
 707                 if not util.cli_is_interactive():
 708                         self.log.warning("Cannot run shell on non-interactive console.")
 709                         return
 710
 711                 # Install all packages that are needed to run a shell.
 712                 self.install(SHELL_PACKAGES)
 713
 714                 # XXX need to set CFLAGS here
 715                 command = "/usr/sbin/chroot %s %s %s" % \
 716                         (self.chrootPath(), SHELL_SCRIPT, " ".join(args))
 717
 718                 # Add personality if we require one
 719                 if self.pakfire.distro.personality:
 720                         command = "%s %s" % (self.pakfire.distro.personality, command)
 721
 722                 for key, val in self.environ.items():
 723                         command = "%s=\"%s\" " % (key, val) + command
 724
 725                 # Empty the environment
 726                 command = "env -i - %s" % command
 727
 728                 self.log.debug("Shell command: %s" % command)
 729
 730                 shell = os.system(command)
 731                 return os.WEXITSTATUS(shell)
 732
 733         @property
 734         def cache_file(self):
 735                 comps = [
 736                         self.pakfire.distro.sname,      # name of the distribution
 737                         self.pakfire.distro.release,    # release version
 738                         self.pakfire.distro.arch,       # architecture
 739                 ]
 740
 741                 return os.path.join(CACHE_ENVIRON_DIR, "%s.cache" %"-".join(comps))
 742
 743         def cache_export(self, filename):
 744                 # Sync all disk caches.
 745                 _pakfire.sync()
 746
 747                 # A list to store all mountpoints, so we don't package them.
 748                 mountpoints = []
 749
 750                 # A list containing all files we want to package.
 751                 filelist = []
 752
 753                 # Walk through the whole tree and collect all files
 754                 # that are on the same disk (not crossing mountpoints).
 755                 log.info(_("Creating filelist..."))
 756                 root = self.chrootPath()
 757                 for dir, subdirs, files in os.walk(root):
 758                         # Search for mountpoints and skip them.
 759                         if not dir == root and os.path.ismount(dir):
 760                                 mountpoints.append(dir)
 761                                 continue
 762
 763                         # Skip all directories under mountpoints.
 764                         if any([dir.startswith(m) for m in mountpoints]):
 765                                 continue
 766
 767                         # Add all other files.
 768                         filelist.append(dir)
 769                         for file in files:
 770                                 file = os.path.join(dir, file)
 771                                 filelist.append(file)
 772
 773                 # Create a nice progressbar.
 774                 p = util.make_progress(_("Compressing files..."), len(filelist))
 775                 i = 0
 776
 777                 # Create tar file and add all files to it.
 778                 f = packages.file.InnerTarFile.open(filename, "w:gz")
 779                 for file in filelist:
 780                         i += 1
 781                         if p:
 782                                 p.update(i)
 783
 784                         f.add(file, os.path.relpath(file, root), recursive=False)
 785                 f.close()
 786
 787                 # Finish progressbar.
 788                 if p:
 789                         p.finish()
 790
 791                 filesize = os.path.getsize(filename)
 792
 793                 log.info(_("Cache file was successfully created at %s.") % filename)
 794                 log.info(_("  Containing %(files)s files, it has a size of %(size)s.") % \
 795                         { "files" : len(filelist), "size" : util.format_size(filesize), })
 796
 797         def cache_extract(self):
 798                 root = self.chrootPath()
 799                 filename = self.cache_file
 800
 801                 f = packages.file.InnerTarFile.open(filename, "r:gz")
 802                 members = f.getmembers()
 803
 804                 # Make a nice progress bar as always.
 805                 p = util.make_progress(_("Extracting files..."), len(members))
 806
 807                 # Extract all files from the cache.
 808                 i = 0
 809                 for member in members:
 810                         if p:
 811                                 i += 1
 812                                 p.update(i)
 813
 814                         f.extract(member, path=root)
 815                 f.close()
 816
 817                 # Finish progressbar.
 818                 if p:
 819                         p.finish()
 820
 821                 # Re-read local repository.
 822                 self.pakfire.repos.local.update(force=True)
 823
 824                 # Update all packages.
 825                 self.log.info(_("Updating packages from cache..."))
 826                 self.pakfire.update(interactive=False, logger=self.log,
 827                         allow_archchange=True, allow_vendorchange=True, allow_downgrade=True)
 828
 829
 830 class Builder(object):
 831         def __init__(self, pakfire, filename, resultdir, **kwargs):
 832                 self.pakfire = pakfire
 833
 834                 self.filename = filename
 835
 836                 self.resultdir = resultdir
 837
 838                 # Open package file.
 839                 self.pkg = packages.Makefile(self.pakfire, self.filename)
 840
 841                 self._environ = {
 842                         "LANG"             : "C",
 843                 }
 844
 845         def mktemp(self):
 846                 """
 847                         Create a temporary file in the build environment.
 848                 """
 849                 file = "/tmp/pakfire_%s" % util.random_string()
 850
 851                 # Touch the file.
 852                 f = open(file, "w")
 853                 f.close()
 854
 855                 return file
 856
 857         @property
 858         def buildroot(self):
 859                 return self.pkg.buildroot
 860
 861         @property
 862         def distro(self):
 863                 return self.pakfire.distro
 864
 865         @property
 866         def environ(self):
 867                 environ = os.environ
 868
 869                 # Get all definitions from the package.
 870                 environ.update(self.pkg.exports)
 871
 872                 # Overwrite some definitions by default values.
 873                 environ.update(self._environ)
 874
 875                 return environ
 876
 877         def do(self, command, shell=True, personality=None, cwd=None, *args, **kwargs):
 878                 # Environment variables
 879                 log.debug("Environment:")
 880                 for k, v in sorted(self.environ.items()):
 881                         log.debug("  %s=%s" % (k, v))
 882
 883                 # Update personality it none was set
 884                 if not personality:
 885                         personality = self.distro.personality
 886
 887                 if not cwd:
 888                         cwd = "/%s" % LOCAL_TMP_PATH
 889
 890                 # Make every shell to a login shell because we set a lot of
 891                 # environment things there.
 892                 if shell:
 893                         command = ["bash", "--login", "-c", command]
 894
 895                 return chroot.do(
 896                         command,
 897                         personality=personality,
 898                         shell=False,
 899                         env=self.environ,
 900                         logger=logging.getLogger("pakfire"),
 901                         cwd=cwd,
 902                         *args,
 903                         **kwargs
 904                 )
 905
 906         def create_icecream_toolchain(self):
 907                 try:
 908                         out = self.do("icecc --build-native 2>/dev/null", returnOutput=True, cwd="/tmp")
 909                 except Error:
 910                         return
 911
 912                 for line in out.splitlines():
 913                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
 914                         if m:
 915                                 self._environ["ICECC_VERSION"] = "/tmp/%s" % m.group(1)
 916
 917         def create_buildscript(self, stage):
 918                 file = "/tmp/build_%s" % util.random_string()
 919
 920                 # Get buildscript from the package.
 921                 script = self.pkg.get_buildscript(stage)
 922
 923                 # Write script to an empty file.
 924                 f = open(file, "w")
 925                 f.write("#!/bin/sh\n\n")
 926                 f.write("set -e\n")
 927                 f.write("set -x\n")
 928                 f.write("\n%s\n" % script)
 929                 f.write("exit 0\n")
 930                 f.close()
 931                 os.chmod(file, 700)
 932
 933                 return file
 934
 935         def build(self):
 936                 # Create buildroot and remove all content if it was existant.
 937                 util.rm(self.buildroot)
 938                 os.makedirs(self.buildroot)
 939
 940                 # Build icecream toolchain if icecream is installed.
 941                 self.create_icecream_toolchain()
 942
 943                 for stage in ("prepare", "build", "test", "install"):
 944                         self.build_stage(stage)
 945
 946                 # Run post-build stuff.
 947                 self.post_compress_man_pages()
 948                 self.post_remove_static_libs()
 949                 self.post_extract_debuginfo()
 950
 951                 # Package the result.
 952                 # Make all these little package from the build environment.
 953                 log.info(_("Creating packages:"))
 954                 pkgs = []
 955                 for pkg in reversed(self.pkg.packages):
 956                         packager = packages.packager.BinaryPackager(self.pakfire, pkg,
 957                                 self, self.buildroot)
 958                         pkg = packager.run(self.resultdir)
 959                         pkgs.append(pkg)
 960                 log.info("")
 961
 962                 for pkg in sorted(pkgs):
 963                         for line in pkg.dump(long=True).splitlines():
 964                                 log.info(line)
 965                         log.info("")
 966                 log.info("")
 967
 968         def build_stage(self, stage):
 969                 # Get the buildscript for this stage.
 970                 buildscript = self.create_buildscript(stage)
 971
 972                 # Execute the buildscript of this stage.
 973                 log.info(_("Running stage %s:") % stage)
 974
 975                 try:
 976                         self.do(buildscript, shell=False)
 977
 978                 finally:
 979                         # Remove the buildscript.
 980                         if os.path.exists(buildscript):
 981                                 os.unlink(buildscript)
 982
 983         def post_remove_static_libs(self):
 984                 keep_libs = self.pkg.lexer.build.get_var("keep_libraries")
 985                 keep_libs = keep_libs.split()
 986
 987                 try:
 988                         self.do("%s/remove-static-libs %s %s" % \
 989                                 (SCRIPT_DIR, self.buildroot, " ".join(keep_libs)))
 990                 except Error, e:
 991                         log.warning(_("Could not remove static libraries: %s") % e)
 992
 993         def post_compress_man_pages(self):
 994                 try:
 995                         self.do("%s/compress-man-pages %s" % (SCRIPT_DIR, self.buildroot))
 996                 except Error, e:
 997                         log.warning(_("Compressing man pages did not complete successfully."))
 998
 999         def post_extract_debuginfo(self):
1000                 args = []
1001
1002                 # Check if we need to run with strict build-id.
1003                 strict_id = self.pkg.lexer.build.get_var("debuginfo_strict_build_id", "true")
1004                 if strict_id in ("true", "yes", "1"):
1005                         args.append("--strict-build-id")
1006
1007                 args.append("--buildroot=%s" % self.pkg.buildroot)
1008                 args.append("--sourcedir=%s" % self.pkg.sourcedir)
1009
1010                 # Get additional options to pass to script.
1011                 options = self.pkg.lexer.build.get_var("debuginfo_options", "")
1012                 args += options.split()
1013
1014                 try:
1015                         self.do("%s/extract-debuginfo %s %s" % (SCRIPT_DIR, " ".join(args), self.pkg.buildroot))
1016                 except Error, e:
1017                         log.error(_("Extracting debuginfo did not complete with success. Aborting build."))
1018                         raise
1019
1020         def cleanup(self):
1021                 if os.path.exists(self.buildroot):
1022                         util.rm(self.buildroot)