python/pakfire/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import math
  25 import os
  26 import re
  27 import shutil
  28 import socket
  29 import time
  30 import uuid
  31
  32 import base
  33 import chroot
  34 import logger
  35 import packages
  36 import packages.file
  37 import packages.packager
  38 import repository
  39 import util
  40 import _pakfire
  41
  42 import logging
  43 log = logging.getLogger("pakfire")
  44
  45 from constants import *
  46 from i18n import _
  47 from errors import BuildError, BuildRootLocked, Error
  48
  49
  50 BUILD_LOG_HEADER = """
  51  ____       _     __ _            _           _ _     _
  52 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  53 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  54 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  55 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  56
  57         Time    : %(time)s
  58         Host    : %(host)s
  59         Version : %(version)s
  60
  61 """
  62
  63 class BuildEnviron(object):
  64         # The version of the kernel this machine is running.
  65         kernel_version = os.uname()[2]
  66
  67         def __init__(self, filename=None, distro_config=None, build_id=None, logfile=None,
  68                         builder_mode="release", use_cache=None, **pakfire_args):
  69                 # Set mode.
  70                 assert builder_mode in ("development", "release",)
  71                 self.mode = builder_mode
  72
  73                 # Disable the build repository in release mode.
  74                 if self.mode == "release":
  75                         if pakfire_args.has_key("disable_repos") and pakfire_args["disable_repos"]:
  76                                 pakfire_args["disable_repos"] += ["build",]
  77                         else:
  78                                 pakfire_args["disable_repos"] = ["build",]
  79
  80                 # Save the build id and generate one if no build id was provided.
  81                 if not build_id:
  82                         build_id = "%s" % uuid.uuid4()
  83
  84                 self.build_id = build_id
  85
  86                 # Setup the logging.
  87                 if logfile:
  88                         self.log = logging.getLogger(self.build_id)
  89                         # Propage everything to the root logger that we will see something
  90                         # on the terminal.
  91                         self.log.propagate = 1
  92                         self.log.setLevel(logging.INFO)
  93
  94                         # Add the given logfile to the logger.
  95                         h = logging.FileHandler(logfile)
  96                         self.log.addHandler(h)
  97
  98                         # Format the log output for the file.
  99                         f = logger.BuildFormatter()
 100                         h.setFormatter(f)
 101                 else:
 102                         # If no logile was given, we use the root logger.
 103                         self.log = logging.getLogger("pakfire")
 104
 105                 # Log information about pakfire and some more information, when we
 106                 # are running in release mode.
 107                 if self.mode == "release":
 108                         logdata = {
 109                                 "host"    : socket.gethostname(),
 110                                 "time"    : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
 111                                 "version" : "Pakfire %s" % PAKFIRE_VERSION,
 112                         }
 113
 114                         for line in BUILD_LOG_HEADER.splitlines():
 115                                 self.log.info(line % logdata)
 116
 117                 # Create pakfire instance.
 118                 if pakfire_args.has_key("mode"):
 119                         del pakfire_args["mode"]
 120                 self.pakfire = base.Pakfire(mode="builder", distro_config=distro_config, **pakfire_args)
 121                 self.distro = self.pakfire.distro
 122                 self.path = self.pakfire.path
 123
 124                 # Check if this host can build the requested architecture.
 125                 if not self.arch in self.pakfire.config.supported_arches:
 126                         raise BuildError, _("Cannot build for %s on this host.") % self.arch
 127
 128                 # Where do we put the result?
 129                 self.resultdir = os.path.join(self.path, "result")
 130
 131                 # Check weather to use or not use the cache.
 132                 if use_cache is None:
 133                         # If use_cache is None, the user did not provide anything and
 134                         # so we guess.
 135                         if self.mode == "development":
 136                                 use_cache = True
 137                         else:
 138                                 use_cache = False
 139
 140                 self.use_cache = use_cache
 141
 142                 # Open package.
 143                 # If we have a plain makefile, we first build a source package and go with that.
 144                 if filename:
 145                         if filename.endswith(".%s" % MAKEFILE_EXTENSION):
 146                                 pkg = packages.Makefile(self.pakfire, filename)
 147                                 pkg.dist([self.resultdir,])
 148
 149                                 filename = os.path.join(self.resultdir, "src", pkg.package_filename)
 150                                 assert os.path.exists(filename), filename
 151
 152                         # Open source package.
 153                         self.pkg = packages.SourcePackage(self.pakfire, None, filename)
 154                         assert self.pkg, filename
 155
 156                         # Log the package information.
 157                         self.log.info(_("Package information:"))
 158                         for line in self.pkg.dump(long=True).splitlines():
 159                                 self.log.info("  %s" % line)
 160                         self.log.info("")
 161
 162                         # Path where we extract the package and put all the source files.
 163                         self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name)
 164                 else:
 165                         # No package :(
 166                         self.pkg = None
 167
 168                 # XXX need to make this configureable
 169                 self.settings = {
 170                         "enable_loop_devices" : True,
 171                         "enable_ccache"   : True,
 172                         "enable_icecream" : False,
 173                 }
 174                 #self.settings.update(settings)
 175
 176                 # Lock the buildroot
 177                 self._lock = None
 178                 self.lock()
 179
 180                 # Save the build time.
 181                 self.build_time = int(time.time())
 182
 183         def start(self):
 184                 # Mount the directories.
 185                 self._mountall()
 186
 187                 # Populate /dev.
 188                 self.populate_dev()
 189
 190                 # Setup domain name resolution in chroot.
 191                 self.setup_dns()
 192
 193                 # Extract all needed packages.
 194                 self.extract()
 195
 196         def stop(self):
 197                 # Kill all still running processes.
 198                 util.orphans_kill(self.path)
 199
 200                 # Close pakfire instance.
 201                 del self.pakfire
 202
 203                 # Umount the build environment.
 204                 self._umountall()
 205
 206                 # Remove all files.
 207                 self.destroy()
 208
 209         @property
 210         def arch(self):
 211                 """
 212                         Inherit architecture from distribution configuration.
 213                 """
 214                 return self.distro.arch
 215
 216         @property
 217         def info(self):
 218                 return {
 219                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 220                         "build_host" : socket.gethostname(),
 221                         "build_id"   : self.build_id,
 222                         "build_time" : self.build_time,
 223                 }
 224
 225         def lock(self):
 226                 filename = os.path.join(self.path, ".lock")
 227
 228                 try:
 229                         self._lock = open(filename, "a+")
 230                 except IOError, e:
 231                         return 0
 232
 233                 try:
 234                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 235                 except IOError, e:
 236                         raise BuildRootLocked, "Buildroot is locked"
 237
 238                 return 1
 239
 240         def unlock(self):
 241                 if self._lock:
 242                         self._lock.close()
 243                         self._lock = None
 244
 245         def copyin(self, file_out, file_in):
 246                 if file_in.startswith("/"):
 247                         file_in = file_in[1:]
 248
 249                 file_in = self.chrootPath(file_in)
 250
 251                 #if not os.path.exists(file_out):
 252                 #       return
 253
 254                 dir_in = os.path.dirname(file_in)
 255                 if not os.path.exists(dir_in):
 256                         os.makedirs(dir_in)
 257
 258                 self.log.debug("%s --> %s" % (file_out, file_in))
 259
 260                 shutil.copy2(file_out, file_in)
 261
 262         def copyout(self, file_in, file_out):
 263                 if file_in.startswith("/"):
 264                         file_in = file_in[1:]
 265
 266                 file_in = self.chrootPath(file_in)
 267
 268                 #if not os.path.exists(file_in):
 269                 #       return
 270
 271                 dir_out = os.path.dirname(file_out)
 272                 if not os.path.exists(dir_out):
 273                         os.makedirs(dir_out)
 274
 275                 self.log.debug("%s --> %s" % (file_in, file_out))
 276
 277                 shutil.copy2(file_in, file_out)
 278
 279         def copy_result(self, resultdir):
 280                 dir_in = self.chrootPath("result")
 281
 282                 for dir, subdirs, files in os.walk(dir_in):
 283                         basename = os.path.basename(dir)
 284                         dir = dir[len(self.chrootPath()):]
 285                         for file in files:
 286                                 file_in = os.path.join(dir, file)
 287
 288                                 file_out = os.path.join(
 289                                         resultdir,
 290                                         basename,
 291                                         file,
 292                                 )
 293
 294                                 self.copyout(file_in, file_out)
 295
 296         def extract(self, requires=None, build_deps=True):
 297                 """
 298                         Gets a dependency set and extracts all packages
 299                         to the environment.
 300                 """
 301                 if not requires:
 302                         requires = []
 303
 304                 if self.use_cache and os.path.exists(self.cache_file):
 305                         # If we are told to use the cache, we just import the
 306                         # file.
 307                         self.cache_extract()
 308                 else:
 309                         # Add neccessary build dependencies.
 310                         requires += BUILD_PACKAGES
 311
 312                 # If we have ccache enabled, we need to extract it
 313                 # to the build chroot.
 314                 if self.settings.get("enable_ccache"):
 315                         requires.append("ccache")
 316
 317                 # If we have icecream enabled, we need to extract it
 318                 # to the build chroot.
 319                 if self.settings.get("enable_icecream"):
 320                         requires.append("icecream")
 321
 322                 # Get build dependencies from source package.
 323                 if self.pkg:
 324                         for req in self.pkg.requires:
 325                                 requires.append(req)
 326
 327                 # Install all packages.
 328                 self.log.info(_("Install packages needed for build..."))
 329                 self.install(requires)
 330
 331                 # Copy the makefile and load source tarballs.
 332                 if self.pkg:
 333                         self.pkg.extract(_("Extracting"), prefix=self.build_dir)
 334
 335         def install(self, requires):
 336                 """
 337                         Install everything that is required in requires.
 338                 """
 339                 # If we got nothing to do, we quit immediately.
 340                 if not requires:
 341                         return
 342
 343                 self.pakfire.install(requires, interactive=False,
 344                         allow_downgrade=True, logger=self.log)
 345
 346         def install_test(self):
 347                 pkgs = []
 348                 for dir, subdirs, files in os.walk(self.chrootPath("result")):
 349                         for file in files:
 350                                 pkgs.append(os.path.join(dir, file))
 351
 352                 self.pakfire.localinstall(pkgs, yes=True, allow_uninstall=True)
 353
 354         def chrootPath(self, *args):
 355                 # Remove all leading slashes
 356                 _args = []
 357                 for arg in args:
 358                         if arg.startswith("/"):
 359                                 arg = arg[1:]
 360                         _args.append(arg)
 361                 args = _args
 362
 363                 ret = os.path.join(self.path, *args)
 364                 ret = ret.replace("//", "/")
 365
 366                 assert ret.startswith(self.path)
 367
 368                 return ret
 369
 370         def populate_dev(self):
 371                 nodes = [
 372                         "/dev/null",
 373                         "/dev/zero",
 374                         "/dev/full",
 375                         "/dev/random",
 376                         "/dev/urandom",
 377                         "/dev/tty",
 378                         "/dev/ptmx",
 379                         "/dev/kmsg",
 380                         "/dev/rtc0",
 381                         "/dev/console",
 382                 ]
 383
 384                 # If we need loop devices (which are optional) we create them here.
 385                 if self.settings["enable_loop_devices"]:
 386                         for i in range(0, 7):
 387                                 nodes.append("/dev/loop%d" % i)
 388
 389                 for node in nodes:
 390                         # Stat the original node of the host system and copy it to
 391                         # the build chroot.
 392                         node_stat = os.stat(node)
 393
 394                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 395
 396                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 397                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 398                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 399                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 400
 401         def setup_dns(self):
 402                 """
 403                         Add DNS resolution facility to chroot environment by copying
 404                         /etc/resolv.conf and /etc/hosts.
 405                 """
 406                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 407                         self.copyin(i, i)
 408
 409         def _create_node(self, filename, mode, device):
 410                 self.log.debug("Create node: %s (%s)" % (filename, mode))
 411
 412                 filename = self.chrootPath(filename)
 413
 414                 # Create parent directory if it is missing.
 415                 dirname = os.path.dirname(filename)
 416                 if not os.path.exists(dirname):
 417                         os.makedirs(dirname)
 418
 419                 os.mknod(filename, mode, device)
 420
 421         def destroy(self):
 422                 self.log.debug("Destroying environment %s" % self.path)
 423
 424                 if os.path.exists(self.path):
 425                         util.rm(self.path)
 426
 427         def cleanup(self):
 428                 self.log.debug("Cleaning environemnt.")
 429
 430                 # Remove the build directory and buildroot.
 431                 dirs = (self.build_dir, self.chrootPath("result"),)
 432
 433                 for d in dirs:
 434                         if not os.path.exists(d):
 435                                 continue
 436
 437                         util.rm(d)
 438                         os.makedirs(d)
 439
 440         def _mountall(self):
 441                 self.log.debug("Mounting environment")
 442                 for src, dest, fs, options in self.mountpoints:
 443                         mountpoint = self.chrootPath(dest)
 444                         if options:
 445                                 options = "-o %s" % options
 446
 447                         # Eventually create mountpoint directory
 448                         if not os.path.exists(mountpoint):
 449                                 os.makedirs(mountpoint)
 450
 451                         cmd = "mount -n -t %s %s %s %s" % \
 452                                 (fs, options, src, mountpoint)
 453                         chroot.do(cmd, shell=True)
 454
 455         def _umountall(self):
 456                 self.log.debug("Umounting environment")
 457
 458                 mountpoints = []
 459                 for src, dest, fs, options in reversed(self.mountpoints):
 460                         if not dest in mountpoints:
 461                                 mountpoints.append(dest)
 462
 463                 for dest in mountpoints:
 464                         mountpoint = self.chrootPath(dest)
 465
 466                         chroot.do("umount -n %s" % mountpoint, raiseExc=0, shell=True)
 467
 468         @property
 469         def mountpoints(self):
 470                 mountpoints = []
 471
 472                 # Make root as a tmpfs.
 473                 #mountpoints += [
 474                 #       ("pakfire_root", "/", "tmpfs", "defaults"),
 475                 #]
 476
 477                 mountpoints += [
 478                         # src, dest, fs, options
 479                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 480                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 481                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 482                         ("/sys",          "/sys",      "bind",  "bind"),
 483                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 484                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 485                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 486                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 487                 ]
 488
 489                 # If selinux is enabled.
 490                 if os.path.exists("/sys/fs/selinux"):
 491                         mountpoints += [
 492                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 493                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 494                         ]
 495
 496                 # If ccache support is requested, we bind mount the cache.
 497                 if self.settings.get("enable_ccache"):
 498                         # Create ccache cache directory if it does not exist.
 499                         if not os.path.exists(CCACHE_CACHE_DIR):
 500                                 os.makedirs(CCACHE_CACHE_DIR)
 501
 502                         mountpoints += [
 503                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 504                         ]
 505
 506                 return mountpoints
 507
 508         @property
 509         def environ(self):
 510                 env = {
 511                         # Add HOME manually, because it is occasionally not set
 512                         # and some builds get in trouble then.
 513                         "HOME" : "/root",
 514                         "TERM" : os.environ.get("TERM", "dumb"),
 515                         "PS1"  : "\u:\w\$ ",
 516
 517                         # Set the container that we can detect, if we are inside a
 518                         # chroot.
 519                         "container" : "pakfire-builder",
 520                 }
 521
 522                 # Inherit environment from distro
 523                 env.update(self.pakfire.distro.environ)
 524
 525                 # Icecream environment settings
 526                 if self.settings.get("enable_icecream", False):
 527                         # Set the toolchain path
 528                         if self.settings.get("icecream_toolchain", None):
 529                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 530
 531                         # Set preferred host if configured.
 532                         if self.settings.get("icecream_preferred_host", None):
 533                                 env["ICECC_PREFERRED_HOST"] = \
 534                                         self.settings.get("icecream_preferred_host")
 535
 536                 # Fake UTS_MACHINE, when we cannot use the personality syscall and
 537                 # if the host architecture is not equal to the target architecture.
 538                 if not self.pakfire.distro.personality and \
 539                                 not self.pakfire.config.host_arch == self.pakfire.distro.arch:
 540                         env.update({
 541                                 "LD_PRELOAD"  : "/usr/lib/libpakfire_preload.so",
 542                                 "UTS_MACHINE" : self.pakfire.distro.arch,
 543                         })
 544
 545                 return env
 546
 547         def do(self, command, shell=True, personality=None, logger=None, *args, **kwargs):
 548                 ret = None
 549
 550                 # Environment variables
 551                 env = self.environ
 552
 553                 if kwargs.has_key("env"):
 554                         env.update(kwargs.pop("env"))
 555
 556                 self.log.debug("Environment:")
 557                 for k, v in sorted(env.items()):
 558                         self.log.debug("  %s=%s" % (k, v))
 559
 560                 # Update personality it none was set
 561                 if not personality:
 562                         personality = self.distro.personality
 563
 564                 # Make every shell to a login shell because we set a lot of
 565                 # environment things there.
 566                 if shell:
 567                         command = ["bash", "--login", "-c", command]
 568
 569                 if not kwargs.has_key("chrootPath"):
 570                         kwargs["chrootPath"] = self.chrootPath()
 571
 572                 ret = chroot.do(
 573                         command,
 574                         personality=personality,
 575                         shell=False,
 576                         env=env,
 577                         logger=logger,
 578                         *args,
 579                         **kwargs
 580                 )
 581
 582                 return ret
 583
 584         def build(self, install_test=True):
 585                 if not self.pkg:
 586                         raise BuildError, _("You cannot run a build when no package was given.")
 587
 588                 # Search for the package file in build_dir and raise BuildError if it is not present.
 589                 pkgfile = os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION))
 590                 if not os.path.exists(pkgfile):
 591                         raise BuildError, _("Could not find makefile in build root: %s") % pkgfile
 592                 pkgfile = "/%s" % os.path.relpath(pkgfile, self.chrootPath())
 593
 594                 resultdir = self.chrootPath("/result")
 595
 596                 # Create the build command, that is executed in the chroot.
 597                 build_command = ["/usr/lib/pakfire/builder", "--offline",
 598                         "build", pkgfile, "--arch", self.arch, "--nodeps",
 599                         "--resultdir=/result",]
 600
 601                 try:
 602                         self.do(" ".join(build_command), logger=self.log)
 603
 604                 except Error:
 605                         raise BuildError, _("The build command failed. See logfile for details.")
 606
 607                 # Perform install test.
 608                 if install_test:
 609                         self.install_test()
 610
 611                 # Copy the final packages and stuff.
 612                 # XXX TODO resultdir
 613
 614         def shell(self, args=[]):
 615                 if not util.cli_is_interactive():
 616                         self.log.warning("Cannot run shell on non-interactive console.")
 617                         return
 618
 619                 # Install all packages that are needed to run a shell.
 620                 self.install(SHELL_PACKAGES)
 621
 622                 # XXX need to set CFLAGS here
 623                 command = "/usr/sbin/chroot %s %s %s" % \
 624                         (self.chrootPath(), SHELL_SCRIPT, " ".join(args))
 625
 626                 # Add personality if we require one
 627                 if self.pakfire.distro.personality:
 628                         command = "%s %s" % (self.pakfire.distro.personality, command)
 629
 630                 for key, val in self.environ.items():
 631                         command = "%s=\"%s\" " % (key, val) + command
 632
 633                 # Empty the environment
 634                 command = "env -i - %s" % command
 635
 636                 self.log.debug("Shell command: %s" % command)
 637
 638                 shell = os.system(command)
 639                 return os.WEXITSTATUS(shell)
 640
 641         @property
 642         def cache_file(self):
 643                 comps = [
 644                         self.pakfire.distro.sname,      # name of the distribution
 645                         self.pakfire.distro.release,    # release version
 646                         self.pakfire.distro.arch,       # architecture
 647                 ]
 648
 649                 return os.path.join(CACHE_ENVIRON_DIR, "%s.cache" %"-".join(comps))
 650
 651         def cache_export(self, filename):
 652                 # Sync all disk caches.
 653                 _pakfire.sync()
 654
 655                 # A list to store all mountpoints, so we don't package them.
 656                 mountpoints = []
 657
 658                 # A list containing all files we want to package.
 659                 filelist = []
 660
 661                 # Walk through the whole tree and collect all files
 662                 # that are on the same disk (not crossing mountpoints).
 663                 log.info(_("Creating filelist..."))
 664                 root = self.chrootPath()
 665                 for dir, subdirs, files in os.walk(root):
 666                         # Search for mountpoints and skip them.
 667                         if not dir == root and os.path.ismount(dir):
 668                                 mountpoints.append(dir)
 669                                 continue
 670
 671                         # Skip all directories under mountpoints.
 672                         if any([dir.startswith(m) for m in mountpoints]):
 673                                 continue
 674
 675                         # Add all other files.
 676                         filelist.append(dir)
 677                         for file in files:
 678                                 file = os.path.join(dir, file)
 679                                 filelist.append(file)
 680
 681                 # Create a nice progressbar.
 682                 p = util.make_progress(_("Compressing files..."), len(filelist))
 683                 i = 0
 684
 685                 # Create tar file and add all files to it.
 686                 f = packages.file.InnerTarFile.open(filename, "w:gz")
 687                 for file in filelist:
 688                         i += 1
 689                         if p:
 690                                 p.update(i)
 691
 692                         f.add(file, os.path.relpath(file, root), recursive=False)
 693                 f.close()
 694
 695                 # Finish progressbar.
 696                 if p:
 697                         p.finish()
 698
 699                 filesize = os.path.getsize(filename)
 700
 701                 log.info(_("Cache file was successfully created at %s.") % filename)
 702                 log.info(_("  Containing %(files)s files, it has a size of %(size)s.") % \
 703                         { "files" : len(filelist), "size" : util.format_size(filesize), })
 704
 705         def cache_extract(self):
 706                 root = self.chrootPath()
 707                 filename = self.cache_file
 708
 709                 f = packages.file.InnerTarFile.open(filename, "r:gz")
 710                 members = f.getmembers()
 711
 712                 # Make a nice progress bar as always.
 713                 p = util.make_progress(_("Extracting files..."), len(members))
 714
 715                 # Extract all files from the cache.
 716                 i = 0
 717                 for member in members:
 718                         if p:
 719                                 i += 1
 720                                 p.update(i)
 721
 722                         f.extract(member, path=root)
 723                 f.close()
 724
 725                 # Finish progressbar.
 726                 if p:
 727                         p.finish()
 728
 729                 # Re-read local repository.
 730                 self.pakfire.repos.local.update(force=True)
 731
 732                 # Update all packages.
 733                 self.log.info(_("Updating packages from cache..."))
 734                 self.pakfire.update(interactive=False, logger=self.log,
 735                         allow_archchange=True, allow_vendorchange=True, allow_downgrade=True)
 736
 737
 738 class Builder(object):
 739         def __init__(self, pakfire, filename, resultdir, **kwargs):
 740                 self.pakfire = pakfire
 741
 742                 self.filename = filename
 743
 744                 self.resultdir = resultdir
 745
 746                 # Open package file.
 747                 self.pkg = packages.Makefile(self.pakfire, self.filename)
 748
 749                 self._environ = {
 750                         "LANG"             : "C",
 751                 }
 752
 753         def mktemp(self):
 754                 """
 755                         Create a temporary file in the build environment.
 756                 """
 757                 file = "/tmp/pakfire_%s" % util.random_string()
 758
 759                 # Touch the file.
 760                 f = open(file, "w")
 761                 f.close()
 762
 763                 return file
 764
 765         @property
 766         def buildroot(self):
 767                 return self.pkg.buildroot
 768
 769         @property
 770         def distro(self):
 771                 return self.pakfire.distro
 772
 773         @property
 774         def environ(self):
 775                 environ = os.environ
 776
 777                 # Get all definitions from the package.
 778                 environ.update(self.pkg.exports)
 779
 780                 # Overwrite some definitions by default values.
 781                 environ.update(self._environ)
 782
 783                 return environ
 784
 785         def do(self, command, shell=True, personality=None, cwd=None, *args, **kwargs):
 786                 # Environment variables
 787                 log.debug("Environment:")
 788                 for k, v in sorted(self.environ.items()):
 789                         log.debug("  %s=%s" % (k, v))
 790
 791                 # Update personality it none was set
 792                 if not personality:
 793                         personality = self.distro.personality
 794
 795                 if not cwd:
 796                         cwd = "/%s" % LOCAL_TMP_PATH
 797
 798                 # Make every shell to a login shell because we set a lot of
 799                 # environment things there.
 800                 if shell:
 801                         command = ["bash", "--login", "-c", command]
 802
 803                 return chroot.do(
 804                         command,
 805                         personality=personality,
 806                         shell=False,
 807                         env=self.environ,
 808                         logger=logging.getLogger("pakfire"),
 809                         cwd=cwd,
 810                         *args,
 811                         **kwargs
 812                 )
 813
 814         def create_icecream_toolchain(self):
 815                 try:
 816                         out = self.do("icecc --build-native 2>/dev/null", returnOutput=True, cwd="/tmp")
 817                 except Error:
 818                         return
 819
 820                 for line in out.splitlines():
 821                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
 822                         if m:
 823                                 self._environ["ICECC_VERSION"] = "/tmp/%s" % m.group(1)
 824
 825         def create_buildscript(self, stage):
 826                 file = "/tmp/build_%s" % util.random_string()
 827
 828                 # Get buildscript from the package.
 829                 script = self.pkg.get_buildscript(stage)
 830
 831                 # Write script to an empty file.
 832                 f = open(file, "w")
 833                 f.write("#!/bin/sh\n\n")
 834                 f.write("set -e\n")
 835                 f.write("set -x\n")
 836                 f.write("\n%s\n" % script)
 837                 f.write("exit 0\n")
 838                 f.close()
 839                 os.chmod(file, 700)
 840
 841                 return file
 842
 843         def build(self):
 844                 # Create buildroot and remove all content if it was existant.
 845                 util.rm(self.buildroot)
 846                 os.makedirs(self.buildroot)
 847
 848                 # Build icecream toolchain if icecream is installed.
 849                 self.create_icecream_toolchain()
 850
 851                 for stage in ("prepare", "build", "test", "install"):
 852                         self.build_stage(stage)
 853
 854                 # Run post-build stuff.
 855                 self.post_compress_man_pages()
 856                 self.post_remove_static_libs()
 857                 self.post_extract_debuginfo()
 858
 859                 # Package the result.
 860                 # Make all these little package from the build environment.
 861                 log.info(_("Creating packages:"))
 862                 pkgs = []
 863                 for pkg in reversed(self.pkg.packages):
 864                         packager = packages.packager.BinaryPackager(self.pakfire, pkg,
 865                                 self, self.buildroot)
 866                         pkg = packager.run(self.resultdir)
 867                         pkgs.append(pkg)
 868                 log.info("")
 869
 870                 for pkg in sorted(pkgs):
 871                         for line in pkg.dump(long=True).splitlines():
 872                                 log.info(line)
 873                         log.info("")
 874                 log.info("")
 875
 876         def build_stage(self, stage):
 877                 # Get the buildscript for this stage.
 878                 buildscript = self.create_buildscript(stage)
 879
 880                 # Execute the buildscript of this stage.
 881                 log.info(_("Running stage %s:") % stage)
 882
 883                 try:
 884                         self.do(buildscript, shell=False)
 885
 886                 finally:
 887                         # Remove the buildscript.
 888                         if os.path.exists(buildscript):
 889                                 os.unlink(buildscript)
 890
 891         def post_remove_static_libs(self):
 892                 keep_libs = self.pkg.lexer.build.get_var("keep_libraries")
 893                 keep_libs = keep_libs.split()
 894
 895                 try:
 896                         self.do("%s/remove-static-libs %s %s" % \
 897                                 (SCRIPT_DIR, self.buildroot, " ".join(keep_libs)))
 898                 except Error, e:
 899                         log.warning(_("Could not remove static libraries: %s") % e)
 900
 901         def post_compress_man_pages(self):
 902                 try:
 903                         self.do("%s/compress-man-pages %s" % (SCRIPT_DIR, self.buildroot))
 904                 except Error, e:
 905                         log.warning(_("Compressing man pages did not complete successfully."))
 906
 907         def post_extract_debuginfo(self):
 908                 args = []
 909
 910                 # Check if we need to run with strict build-id.
 911                 strict_id = self.pkg.lexer.build.get_var("debuginfo_strict_build_id", "true")
 912                 if strict_id in ("true", "yes", "1"):
 913                         args.append("--strict-build-id")
 914
 915                 args.append("--buildroot=%s" % self.pkg.buildroot)
 916                 args.append("--sourcedir=%s" % self.pkg.sourcedir)
 917
 918                 # Get additional options to pass to script.
 919                 options = self.pkg.lexer.build.get_var("debuginfo_options", "")
 920                 args += options.split()
 921
 922                 try:
 923                         self.do("%s/extract-debuginfo %s %s" % (SCRIPT_DIR, " ".join(args), self.pkg.buildroot))
 924                 except Error, e:
 925                         log.error(_("Extracting debuginfo did not complete with success. Aborting build."))
 926                         raise
 927
 928         def cleanup(self):
 929                 if os.path.exists(self.buildroot):
 930                         util.rm(self.buildroot)