python/pakfire/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import math
  25 import os
  26 import re
  27 import shutil
  28 import socket
  29 import time
  30 import uuid
  31
  32 import base
  33 import chroot
  34 import logger
  35 import packages
  36 import packages.file
  37 import packages.packager
  38 import repository
  39 import util
  40 import _pakfire
  41
  42 import logging
  43 log = logging.getLogger("pakfire")
  44
  45 from system import system
  46 from constants import *
  47 from i18n import _
  48 from errors import BuildError, BuildRootLocked, Error
  49
  50
  51 BUILD_LOG_HEADER = """
  52  ____       _     __ _            _           _ _     _
  53 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  54 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  55 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  56 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  57
  58         Version : %(version)s
  59         Host    : %(hostname)s (%(host_arch)s)
  60         Time    : %(time)s
  61
  62 """
  63
  64 class BuildEnviron(object):
  65         # The version of the kernel this machine is running.
  66         kernel_version = os.uname()[2]
  67
  68         def __init__(self, filename=None, distro_config=None, build_id=None, logfile=None,
  69                         builder_mode="release", use_cache=None, **pakfire_args):
  70                 # Set mode.
  71                 assert builder_mode in ("development", "release",)
  72                 self.mode = builder_mode
  73
  74                 # Disable the build repository in release mode.
  75                 if self.mode == "release":
  76                         if pakfire_args.has_key("disable_repos") and pakfire_args["disable_repos"]:
  77                                 pakfire_args["disable_repos"] += ["build",]
  78                         else:
  79                                 pakfire_args["disable_repos"] = ["build",]
  80
  81                 # Save the build id and generate one if no build id was provided.
  82                 if not build_id:
  83                         build_id = "%s" % uuid.uuid4()
  84
  85                 self.build_id = build_id
  86
  87                 # Setup the logging.
  88                 if logfile:
  89                         self.log = log.getChild(self.build_id)
  90                         # Propage everything to the root logger that we will see something
  91                         # on the terminal.
  92                         self.log.propagate = 1
  93                         self.log.setLevel(logging.INFO)
  94
  95                         # Add the given logfile to the logger.
  96                         h = logging.FileHandler(logfile)
  97                         self.log.addHandler(h)
  98
  99                         # Format the log output for the file.
 100                         f = logger.BuildFormatter()
 101                         h.setFormatter(f)
 102                 else:
 103                         # If no logile was given, we use the root logger.
 104                         self.log = logging.getLogger("pakfire")
 105
 106                 # Log information about pakfire and some more information, when we
 107                 # are running in release mode.
 108                 if self.mode == "release":
 109                         logdata = {
 110                                 "host_arch"  : system.arch,
 111                                 "hostname"   : system.hostname,
 112                                 "time"       : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
 113                                 "version"    : "Pakfire %s" % PAKFIRE_VERSION,
 114                         }
 115
 116                         for line in BUILD_LOG_HEADER.splitlines():
 117                                 self.log.info(line % logdata)
 118
 119                 # Create pakfire instance.
 120                 if pakfire_args.has_key("mode"):
 121                         del pakfire_args["mode"]
 122                 self.pakfire = base.Pakfire(mode="builder", distro_config=distro_config, **pakfire_args)
 123                 self.distro = self.pakfire.distro
 124                 self.path = self.pakfire.path
 125
 126                 # Check if this host can build the requested architecture.
 127                 if not self.arch in self.pakfire.config.supported_arches:
 128                         raise BuildError, _("Cannot build for %s on this host.") % self.arch
 129
 130                 # Where do we put the result?
 131                 self.resultdir = os.path.join(self.path, "result")
 132
 133                 # Check weather to use or not use the cache.
 134                 if use_cache is None:
 135                         # If use_cache is None, the user did not provide anything and
 136                         # so we guess.
 137                         if self.mode == "development":
 138                                 use_cache = True
 139                         else:
 140                                 use_cache = False
 141
 142                 self.use_cache = use_cache
 143
 144                 # Open package.
 145                 # If we have a plain makefile, we first build a source package and go with that.
 146                 if filename:
 147                         if filename.endswith(".%s" % MAKEFILE_EXTENSION):
 148                                 pkg = packages.Makefile(self.pakfire, filename)
 149                                 filename = pkg.dist(os.path.join(self.resultdir, "src"))
 150
 151                                 assert os.path.exists(filename), filename
 152
 153                         # Open source package.
 154                         self.pkg = packages.SourcePackage(self.pakfire, None, filename)
 155                         assert self.pkg, filename
 156
 157                         # Log the package information.
 158                         self.log.info(_("Package information:"))
 159                         for line in self.pkg.dump(long=True).splitlines():
 160                                 self.log.info("  %s" % line)
 161                         self.log.info("")
 162
 163                         # Path where we extract the package and put all the source files.
 164                         self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name)
 165                 else:
 166                         # No package :(
 167                         self.pkg = None
 168
 169                 # XXX need to make this configureable
 170                 self.settings = {
 171                         "enable_loop_devices" : True,
 172                         "enable_ccache"   : True,
 173                         "enable_icecream" : False,
 174                 }
 175                 #self.settings.update(settings)
 176
 177                 # Lock the buildroot
 178                 self._lock = None
 179                 self.lock()
 180
 181                 # Save the build time.
 182                 self.build_time = int(time.time())
 183
 184         def start(self):
 185                 # Mount the directories.
 186                 self._mountall()
 187
 188                 # Populate /dev.
 189                 self.populate_dev()
 190
 191                 # Setup domain name resolution in chroot.
 192                 self.setup_dns()
 193
 194                 # Extract all needed packages.
 195                 self.extract()
 196
 197         def stop(self):
 198                 # Kill all still running processes.
 199                 util.orphans_kill(self.path)
 200
 201                 # Close pakfire instance.
 202                 del self.pakfire
 203
 204                 # Umount the build environment.
 205                 self._umountall()
 206
 207                 # Remove all files.
 208                 self.destroy()
 209
 210         @property
 211         def arch(self):
 212                 """
 213                         Inherit architecture from distribution configuration.
 214                 """
 215                 return self.distro.arch
 216
 217         @property
 218         def info(self):
 219                 return {
 220                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 221                         "build_host" : socket.gethostname(),
 222                         "build_id"   : self.build_id,
 223                         "build_time" : self.build_time,
 224                 }
 225
 226         def lock(self):
 227                 filename = os.path.join(self.path, ".lock")
 228
 229                 try:
 230                         self._lock = open(filename, "a+")
 231                 except IOError, e:
 232                         return 0
 233
 234                 try:
 235                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 236                 except IOError, e:
 237                         raise BuildRootLocked, "Buildroot is locked"
 238
 239                 return 1
 240
 241         def unlock(self):
 242                 if self._lock:
 243                         self._lock.close()
 244                         self._lock = None
 245
 246         def copyin(self, file_out, file_in):
 247                 if file_in.startswith("/"):
 248                         file_in = file_in[1:]
 249
 250                 file_in = self.chrootPath(file_in)
 251
 252                 #if not os.path.exists(file_out):
 253                 #       return
 254
 255                 dir_in = os.path.dirname(file_in)
 256                 if not os.path.exists(dir_in):
 257                         os.makedirs(dir_in)
 258
 259                 self.log.debug("%s --> %s" % (file_out, file_in))
 260
 261                 shutil.copy2(file_out, file_in)
 262
 263         def copyout(self, file_in, file_out):
 264                 if file_in.startswith("/"):
 265                         file_in = file_in[1:]
 266
 267                 file_in = self.chrootPath(file_in)
 268
 269                 #if not os.path.exists(file_in):
 270                 #       return
 271
 272                 dir_out = os.path.dirname(file_out)
 273                 if not os.path.exists(dir_out):
 274                         os.makedirs(dir_out)
 275
 276                 self.log.debug("%s --> %s" % (file_in, file_out))
 277
 278                 shutil.copy2(file_in, file_out)
 279
 280         def copy_result(self, resultdir):
 281                 dir_in = self.chrootPath("result")
 282
 283                 for dir, subdirs, files in os.walk(dir_in):
 284                         basename = os.path.basename(dir)
 285                         dir = dir[len(self.chrootPath()):]
 286                         for file in files:
 287                                 file_in = os.path.join(dir, file)
 288
 289                                 file_out = os.path.join(
 290                                         resultdir,
 291                                         basename,
 292                                         file,
 293                                 )
 294
 295                                 self.copyout(file_in, file_out)
 296
 297         def extract(self, requires=None, build_deps=True):
 298                 """
 299                         Gets a dependency set and extracts all packages
 300                         to the environment.
 301                 """
 302                 if not requires:
 303                         requires = []
 304
 305                 if self.use_cache and os.path.exists(self.cache_file):
 306                         # If we are told to use the cache, we just import the
 307                         # file.
 308                         self.cache_extract()
 309                 else:
 310                         # Add neccessary build dependencies.
 311                         requires += BUILD_PACKAGES
 312
 313                 # If we have ccache enabled, we need to extract it
 314                 # to the build chroot.
 315                 if self.settings.get("enable_ccache"):
 316                         requires.append("ccache")
 317
 318                 # If we have icecream enabled, we need to extract it
 319                 # to the build chroot.
 320                 if self.settings.get("enable_icecream"):
 321                         requires.append("icecream")
 322
 323                 # Get build dependencies from source package.
 324                 if self.pkg:
 325                         for req in self.pkg.requires:
 326                                 requires.append(req)
 327
 328                 # Install all packages.
 329                 self.log.info(_("Install packages needed for build..."))
 330                 self.install(requires)
 331
 332                 # Copy the makefile and load source tarballs.
 333                 if self.pkg:
 334                         self.pkg.extract(_("Extracting"), prefix=self.build_dir)
 335
 336         def install(self, requires):
 337                 """
 338                         Install everything that is required in requires.
 339                 """
 340                 # If we got nothing to do, we quit immediately.
 341                 if not requires:
 342                         return
 343
 344                 try:
 345                         self.pakfire.install(requires, interactive=False,
 346                                 allow_downgrade=True, logger=self.log)
 347
 348                 # Catch dependency errors and log it.
 349                 except DependencyError, e:
 350                         raise
 351
 352         def install_test(self):
 353                 pkgs = []
 354                 for dir, subdirs, files in os.walk(self.chrootPath("result")):
 355                         for file in files:
 356                                 pkgs.append(os.path.join(dir, file))
 357
 358                 self.pakfire.localinstall(pkgs, yes=True, allow_uninstall=True)
 359
 360         def chrootPath(self, *args):
 361                 # Remove all leading slashes
 362                 _args = []
 363                 for arg in args:
 364                         if arg.startswith("/"):
 365                                 arg = arg[1:]
 366                         _args.append(arg)
 367                 args = _args
 368
 369                 ret = os.path.join(self.path, *args)
 370                 ret = ret.replace("//", "/")
 371
 372                 assert ret.startswith(self.path)
 373
 374                 return ret
 375
 376         def populate_dev(self):
 377                 nodes = [
 378                         "/dev/null",
 379                         "/dev/zero",
 380                         "/dev/full",
 381                         "/dev/random",
 382                         "/dev/urandom",
 383                         "/dev/tty",
 384                         "/dev/ptmx",
 385                         "/dev/kmsg",
 386                         "/dev/rtc0",
 387                         "/dev/console",
 388                 ]
 389
 390                 # If we need loop devices (which are optional) we create them here.
 391                 if self.settings["enable_loop_devices"]:
 392                         for i in range(0, 7):
 393                                 nodes.append("/dev/loop%d" % i)
 394
 395                 for node in nodes:
 396                         # Stat the original node of the host system and copy it to
 397                         # the build chroot.
 398                         node_stat = os.stat(node)
 399
 400                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 401
 402                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 403                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 404                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 405                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 406
 407         def setup_dns(self):
 408                 """
 409                         Add DNS resolution facility to chroot environment by copying
 410                         /etc/resolv.conf and /etc/hosts.
 411                 """
 412                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 413                         self.copyin(i, i)
 414
 415         def _create_node(self, filename, mode, device):
 416                 self.log.debug("Create node: %s (%s)" % (filename, mode))
 417
 418                 filename = self.chrootPath(filename)
 419
 420                 # Create parent directory if it is missing.
 421                 dirname = os.path.dirname(filename)
 422                 if not os.path.exists(dirname):
 423                         os.makedirs(dirname)
 424
 425                 os.mknod(filename, mode, device)
 426
 427         def destroy(self):
 428                 self.log.debug("Destroying environment %s" % self.path)
 429
 430                 if os.path.exists(self.path):
 431                         util.rm(self.path)
 432
 433         def cleanup(self):
 434                 self.log.debug("Cleaning environemnt.")
 435
 436                 # Remove the build directory and buildroot.
 437                 dirs = (self.build_dir, self.chrootPath("result"),)
 438
 439                 for d in dirs:
 440                         if not os.path.exists(d):
 441                                 continue
 442
 443                         util.rm(d)
 444                         os.makedirs(d)
 445
 446         def _mountall(self):
 447                 self.log.debug("Mounting environment")
 448                 for src, dest, fs, options in self.mountpoints:
 449                         mountpoint = self.chrootPath(dest)
 450                         if options:
 451                                 options = "-o %s" % options
 452
 453                         # Eventually create mountpoint directory
 454                         if not os.path.exists(mountpoint):
 455                                 os.makedirs(mountpoint)
 456
 457                         cmd = "mount -n -t %s %s %s %s" % \
 458                                 (fs, options, src, mountpoint)
 459                         chroot.do(cmd, shell=True)
 460
 461         def _umountall(self):
 462                 self.log.debug("Umounting environment")
 463
 464                 mountpoints = []
 465                 for src, dest, fs, options in reversed(self.mountpoints):
 466                         if not dest in mountpoints:
 467                                 mountpoints.append(dest)
 468
 469                 for dest in mountpoints:
 470                         mountpoint = self.chrootPath(dest)
 471
 472                         chroot.do("umount -n %s" % mountpoint, raiseExc=0, shell=True)
 473
 474         @property
 475         def mountpoints(self):
 476                 mountpoints = []
 477
 478                 # Make root as a tmpfs.
 479                 #mountpoints += [
 480                 #       ("pakfire_root", "/", "tmpfs", "defaults"),
 481                 #]
 482
 483                 mountpoints += [
 484                         # src, dest, fs, options
 485                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 486                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 487                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 488                         ("/sys",          "/sys",      "bind",  "bind"),
 489                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 490                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 491                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 492                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 493                 ]
 494
 495                 # If selinux is enabled.
 496                 if os.path.exists("/sys/fs/selinux"):
 497                         mountpoints += [
 498                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 499                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 500                         ]
 501
 502                 # If ccache support is requested, we bind mount the cache.
 503                 if self.settings.get("enable_ccache"):
 504                         # Create ccache cache directory if it does not exist.
 505                         if not os.path.exists(CCACHE_CACHE_DIR):
 506                                 os.makedirs(CCACHE_CACHE_DIR)
 507
 508                         mountpoints += [
 509                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 510                         ]
 511
 512                 return mountpoints
 513
 514         @property
 515         def environ(self):
 516                 env = {
 517                         # Add HOME manually, because it is occasionally not set
 518                         # and some builds get in trouble then.
 519                         "HOME" : "/root",
 520                         "TERM" : os.environ.get("TERM", "dumb"),
 521                         "PS1"  : "\u:\w\$ ",
 522
 523                         # Set the container that we can detect, if we are inside a
 524                         # chroot.
 525                         "container" : "pakfire-builder",
 526                 }
 527
 528                 # Inherit environment from distro
 529                 env.update(self.pakfire.distro.environ)
 530
 531                 # Icecream environment settings
 532                 if self.settings.get("enable_icecream", False):
 533                         # Set the toolchain path
 534                         if self.settings.get("icecream_toolchain", None):
 535                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 536
 537                         # Set preferred host if configured.
 538                         if self.settings.get("icecream_preferred_host", None):
 539                                 env["ICECC_PREFERRED_HOST"] = \
 540                                         self.settings.get("icecream_preferred_host")
 541
 542                 # Fake UTS_MACHINE, when we cannot use the personality syscall and
 543                 # if the host architecture is not equal to the target architecture.
 544                 if not self.pakfire.distro.personality and \
 545                                 not self.pakfire.config.host_arch == self.pakfire.distro.arch:
 546                         env.update({
 547                                 "LD_PRELOAD"  : "/usr/lib/libpakfire_preload.so",
 548                                 "UTS_MACHINE" : self.pakfire.distro.arch,
 549                         })
 550
 551                 return env
 552
 553         @property
 554         def installed_packages(self):
 555                 """
 556                         Returns an iterator over all installed packages in this build environment.
 557                 """
 558                 # Get the repository of all installed packages.
 559                 repo = self.pakfire.repos.get_repo("@system")
 560
 561                 # Return an iterator over the packages.
 562                 return iter(repo)
 563
 564         def do(self, command, shell=True, personality=None, logger=None, *args, **kwargs):
 565                 ret = None
 566
 567                 # Environment variables
 568                 env = self.environ
 569
 570                 if kwargs.has_key("env"):
 571                         env.update(kwargs.pop("env"))
 572
 573                 self.log.debug("Environment:")
 574                 for k, v in sorted(env.items()):
 575                         self.log.debug("  %s=%s" % (k, v))
 576
 577                 # Update personality it none was set
 578                 if not personality:
 579                         personality = self.distro.personality
 580
 581                 # Make every shell to a login shell because we set a lot of
 582                 # environment things there.
 583                 if shell:
 584                         command = ["bash", "--login", "-c", command]
 585
 586                 if not kwargs.has_key("chrootPath"):
 587                         kwargs["chrootPath"] = self.chrootPath()
 588
 589                 ret = chroot.do(
 590                         command,
 591                         personality=personality,
 592                         shell=False,
 593                         env=env,
 594                         logger=logger,
 595                         *args,
 596                         **kwargs
 597                 )
 598
 599                 return ret
 600
 601         def build(self, install_test=True):
 602                 if not self.pkg:
 603                         raise BuildError, _("You cannot run a build when no package was given.")
 604
 605                 # Search for the package file in build_dir and raise BuildError if it is not present.
 606                 pkgfile = os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION))
 607                 if not os.path.exists(pkgfile):
 608                         raise BuildError, _("Could not find makefile in build root: %s") % pkgfile
 609                 pkgfile = "/%s" % os.path.relpath(pkgfile, self.chrootPath())
 610
 611                 resultdir = self.chrootPath("/result")
 612
 613                 # Create the build command, that is executed in the chroot.
 614                 build_command = ["/usr/lib/pakfire/builder", "--offline",
 615                         "build", pkgfile, "--arch", self.arch, "--nodeps",
 616                         "--resultdir=/result",]
 617
 618                 try:
 619                         self.do(" ".join(build_command), logger=self.log)
 620
 621                 except Error:
 622                         raise BuildError, _("The build command failed. See logfile for details.")
 623
 624                 # Perform install test.
 625                 if install_test:
 626                         self.install_test()
 627
 628                 # Copy the final packages and stuff.
 629                 # XXX TODO resultdir
 630
 631         def shell(self, args=[]):
 632                 if not util.cli_is_interactive():
 633                         self.log.warning("Cannot run shell on non-interactive console.")
 634                         return
 635
 636                 # Install all packages that are needed to run a shell.
 637                 self.install(SHELL_PACKAGES)
 638
 639                 # XXX need to set CFLAGS here
 640                 command = "/usr/sbin/chroot %s %s %s" % \
 641                         (self.chrootPath(), SHELL_SCRIPT, " ".join(args))
 642
 643                 # Add personality if we require one
 644                 if self.pakfire.distro.personality:
 645                         command = "%s %s" % (self.pakfire.distro.personality, command)
 646
 647                 for key, val in self.environ.items():
 648                         command = "%s=\"%s\" " % (key, val) + command
 649
 650                 # Empty the environment
 651                 command = "env -i - %s" % command
 652
 653                 self.log.debug("Shell command: %s" % command)
 654
 655                 shell = os.system(command)
 656                 return os.WEXITSTATUS(shell)
 657
 658         @property
 659         def cache_file(self):
 660                 comps = [
 661                         self.pakfire.distro.sname,      # name of the distribution
 662                         self.pakfire.distro.release,    # release version
 663                         self.pakfire.distro.arch,       # architecture
 664                 ]
 665
 666                 return os.path.join(CACHE_ENVIRON_DIR, "%s.cache" %"-".join(comps))
 667
 668         def cache_export(self, filename):
 669                 # Sync all disk caches.
 670                 _pakfire.sync()
 671
 672                 # A list to store all mountpoints, so we don't package them.
 673                 mountpoints = []
 674
 675                 # A list containing all files we want to package.
 676                 filelist = []
 677
 678                 # Walk through the whole tree and collect all files
 679                 # that are on the same disk (not crossing mountpoints).
 680                 log.info(_("Creating filelist..."))
 681                 root = self.chrootPath()
 682                 for dir, subdirs, files in os.walk(root):
 683                         # Search for mountpoints and skip them.
 684                         if not dir == root and os.path.ismount(dir):
 685                                 mountpoints.append(dir)
 686                                 continue
 687
 688                         # Skip all directories under mountpoints.
 689                         if any([dir.startswith(m) for m in mountpoints]):
 690                                 continue
 691
 692                         # Add all other files.
 693                         filelist.append(dir)
 694                         for file in files:
 695                                 file = os.path.join(dir, file)
 696                                 filelist.append(file)
 697
 698                 # Create a nice progressbar.
 699                 p = util.make_progress(_("Compressing files..."), len(filelist))
 700                 i = 0
 701
 702                 # Create tar file and add all files to it.
 703                 f = packages.file.InnerTarFile.open(filename, "w:gz")
 704                 for file in filelist:
 705                         i += 1
 706                         if p:
 707                                 p.update(i)
 708
 709                         f.add(file, os.path.relpath(file, root), recursive=False)
 710                 f.close()
 711
 712                 # Finish progressbar.
 713                 if p:
 714                         p.finish()
 715
 716                 filesize = os.path.getsize(filename)
 717
 718                 log.info(_("Cache file was successfully created at %s.") % filename)
 719                 log.info(_("  Containing %(files)s files, it has a size of %(size)s.") % \
 720                         { "files" : len(filelist), "size" : util.format_size(filesize), })
 721
 722         def cache_extract(self):
 723                 root = self.chrootPath()
 724                 filename = self.cache_file
 725
 726                 f = packages.file.InnerTarFile.open(filename, "r:gz")
 727                 members = f.getmembers()
 728
 729                 # Make a nice progress bar as always.
 730                 p = util.make_progress(_("Extracting files..."), len(members))
 731
 732                 # Extract all files from the cache.
 733                 i = 0
 734                 for member in members:
 735                         if p:
 736                                 i += 1
 737                                 p.update(i)
 738
 739                         f.extract(member, path=root)
 740                 f.close()
 741
 742                 # Finish progressbar.
 743                 if p:
 744                         p.finish()
 745
 746                 # Re-read local repository.
 747                 self.pakfire.repos.local.update(force=True)
 748
 749                 # Update all packages.
 750                 self.log.info(_("Updating packages from cache..."))
 751                 self.pakfire.update(interactive=False, logger=self.log,
 752                         allow_archchange=True, allow_vendorchange=True, allow_downgrade=True)
 753
 754
 755 class Builder(object):
 756         def __init__(self, pakfire, filename, resultdir, **kwargs):
 757                 self.pakfire = pakfire
 758
 759                 self.filename = filename
 760
 761                 self.resultdir = resultdir
 762
 763                 # Open package file.
 764                 self.pkg = packages.Makefile(self.pakfire, self.filename)
 765
 766                 self._environ = {
 767                         "LANG"             : "C",
 768                 }
 769
 770         def mktemp(self):
 771                 """
 772                         Create a temporary file in the build environment.
 773                 """
 774                 file = "/tmp/pakfire_%s" % util.random_string()
 775
 776                 # Touch the file.
 777                 f = open(file, "w")
 778                 f.close()
 779
 780                 return file
 781
 782         @property
 783         def buildroot(self):
 784                 return self.pkg.buildroot
 785
 786         @property
 787         def distro(self):
 788                 return self.pakfire.distro
 789
 790         @property
 791         def environ(self):
 792                 environ = os.environ
 793
 794                 # Get all definitions from the package.
 795                 environ.update(self.pkg.exports)
 796
 797                 # Overwrite some definitions by default values.
 798                 environ.update(self._environ)
 799
 800                 return environ
 801
 802         def do(self, command, shell=True, personality=None, cwd=None, *args, **kwargs):
 803                 # Environment variables
 804                 log.debug("Environment:")
 805                 for k, v in sorted(self.environ.items()):
 806                         log.debug("  %s=%s" % (k, v))
 807
 808                 # Update personality it none was set
 809                 if not personality:
 810                         personality = self.distro.personality
 811
 812                 if not cwd:
 813                         cwd = "/%s" % LOCAL_TMP_PATH
 814
 815                 # Make every shell to a login shell because we set a lot of
 816                 # environment things there.
 817                 if shell:
 818                         command = ["bash", "--login", "-c", command]
 819
 820                 return chroot.do(
 821                         command,
 822                         personality=personality,
 823                         shell=False,
 824                         env=self.environ,
 825                         logger=logging.getLogger("pakfire"),
 826                         cwd=cwd,
 827                         *args,
 828                         **kwargs
 829                 )
 830
 831         def create_icecream_toolchain(self):
 832                 try:
 833                         out = self.do("icecc --build-native 2>/dev/null", returnOutput=True, cwd="/tmp")
 834                 except Error:
 835                         return
 836
 837                 for line in out.splitlines():
 838                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
 839                         if m:
 840                                 self._environ["ICECC_VERSION"] = "/tmp/%s" % m.group(1)
 841
 842         def create_buildscript(self, stage):
 843                 file = "/tmp/build_%s" % util.random_string()
 844
 845                 # Get buildscript from the package.
 846                 script = self.pkg.get_buildscript(stage)
 847
 848                 # Write script to an empty file.
 849                 f = open(file, "w")
 850                 f.write("#!/bin/sh\n\n")
 851                 f.write("set -e\n")
 852                 f.write("set -x\n")
 853                 f.write("\n%s\n" % script)
 854                 f.write("exit 0\n")
 855                 f.close()
 856                 os.chmod(file, 700)
 857
 858                 return file
 859
 860         def build(self):
 861                 # Create buildroot and remove all content if it was existant.
 862                 util.rm(self.buildroot)
 863                 os.makedirs(self.buildroot)
 864
 865                 # Build icecream toolchain if icecream is installed.
 866                 self.create_icecream_toolchain()
 867
 868                 for stage in ("prepare", "build", "test", "install"):
 869                         self.build_stage(stage)
 870
 871                 # Run post-build stuff.
 872                 self.post_compress_man_pages()
 873                 self.post_remove_static_libs()
 874                 self.post_extract_debuginfo()
 875
 876                 # Package the result.
 877                 # Make all these little package from the build environment.
 878                 log.info(_("Creating packages:"))
 879                 pkgs = []
 880                 for pkg in reversed(self.pkg.packages):
 881                         packager = packages.packager.BinaryPackager(self.pakfire, pkg,
 882                                 self, self.buildroot)
 883                         pkg = packager.run(self.resultdir)
 884                         pkgs.append(pkg)
 885                 log.info("")
 886
 887                 for pkg in sorted(pkgs):
 888                         for line in pkg.dump(long=True).splitlines():
 889                                 log.info(line)
 890                         log.info("")
 891                 log.info("")
 892
 893         def build_stage(self, stage):
 894                 # Get the buildscript for this stage.
 895                 buildscript = self.create_buildscript(stage)
 896
 897                 # Execute the buildscript of this stage.
 898                 log.info(_("Running stage %s:") % stage)
 899
 900                 try:
 901                         self.do(buildscript, shell=False)
 902
 903                 finally:
 904                         # Remove the buildscript.
 905                         if os.path.exists(buildscript):
 906                                 os.unlink(buildscript)
 907
 908         def post_remove_static_libs(self):
 909                 keep_libs = self.pkg.lexer.build.get_var("keep_libraries")
 910                 keep_libs = keep_libs.split()
 911
 912                 try:
 913                         self.do("%s/remove-static-libs %s %s" % \
 914                                 (SCRIPT_DIR, self.buildroot, " ".join(keep_libs)))
 915                 except Error, e:
 916                         log.warning(_("Could not remove static libraries: %s") % e)
 917
 918         def post_compress_man_pages(self):
 919                 try:
 920                         self.do("%s/compress-man-pages %s" % (SCRIPT_DIR, self.buildroot))
 921                 except Error, e:
 922                         log.warning(_("Compressing man pages did not complete successfully."))
 923
 924         def post_extract_debuginfo(self):
 925                 args = []
 926
 927                 # Check if we need to run with strict build-id.
 928                 strict_id = self.pkg.lexer.build.get_var("debuginfo_strict_build_id", "true")
 929                 if strict_id in ("true", "yes", "1"):
 930                         args.append("--strict-build-id")
 931
 932                 args.append("--buildroot=%s" % self.pkg.buildroot)
 933                 args.append("--sourcedir=%s" % self.pkg.sourcedir)
 934
 935                 # Get additional options to pass to script.
 936                 options = self.pkg.lexer.build.get_var("debuginfo_options", "")
 937                 args += options.split()
 938
 939                 try:
 940                         self.do("%s/extract-debuginfo %s %s" % (SCRIPT_DIR, " ".join(args), self.pkg.buildroot))
 941                 except Error, e:
 942                         log.error(_("Extracting debuginfo did not complete with success. Aborting build."))
 943                         raise
 944
 945         def cleanup(self):
 946                 if os.path.exists(self.buildroot):
 947                         util.rm(self.buildroot)