python/pakfire/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import logging
  25 import math
  26 import os
  27 import re
  28 import shutil
  29 import socket
  30 import time
  31 import uuid
  32
  33 import base
  34 import chroot
  35 import logger
  36 import packages
  37 import packages.packager
  38 import repository
  39 import util
  40
  41 from constants import *
  42 from i18n import _
  43 from errors import BuildError, BuildRootLocked, Error
  44
  45
  46 BUILD_LOG_HEADER = """
  47  ____       _     __ _            _           _ _     _
  48 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  49 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  50 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  51 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  52
  53         Time    : %(time)s
  54         Host    : %(host)s
  55         Version : %(version)s
  56
  57 """
  58
  59 class BuildEnviron(object):
  60         # The version of the kernel this machine is running.
  61         kernel_version = os.uname()[2]
  62
  63         def __init__(self, pkg=None, distro_config=None, build_id=None, logfile=None,
  64                         builder_mode="release", **pakfire_args):
  65                 # Set mode.
  66                 assert builder_mode in ("development", "release",)
  67                 self.mode = builder_mode
  68
  69                 # Disable the build repository in release mode.
  70                 if self.mode == "release":
  71                         if pakfire_args.has_key("disable_repos") and pakfire_args["disable_repos"]:
  72                                 pakfire_args["disable_repos"] += ["build",]
  73                         else:
  74                                 pakfire_args["disable_repos"] = ["build",]
  75
  76                 # Save the build id and generate one if no build id was provided.
  77                 if not build_id:
  78                         build_id = "%s" % uuid.uuid4()
  79
  80                 self.build_id = build_id
  81
  82                 # Setup the logging.
  83                 if logfile:
  84                         self.log = logging.getLogger(self.build_id)
  85                         # Propage everything to the root logger that we will see something
  86                         # on the terminal.
  87                         self.log.propagate = 1
  88                         self.log.setLevel(logging.INFO)
  89
  90                         # Add the given logfile to the logger.
  91                         h = logging.FileHandler(logfile)
  92                         self.log.addHandler(h)
  93
  94                         # Format the log output for the file.
  95                         f = logger.BuildFormatter()
  96                         h.setFormatter(f)
  97                 else:
  98                         # If no logile was given, we use the root logger.
  99                         self.log = logging.getLogger()
 100
 101                 # Log information about pakfire and some more information, when we
 102                 # are running in release mode.
 103                 if self.mode == "release":
 104                         logdata = {
 105                                 "host"    : socket.gethostname(),
 106                                 "time"    : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
 107                                 "version" : "Pakfire %s" % PAKFIRE_VERSION,
 108                         }
 109
 110                         for line in BUILD_LOG_HEADER.splitlines():
 111                                 self.log.info(line % logdata)
 112
 113                 # Create pakfire instance.
 114                 if pakfire_args.has_key("mode"):
 115                         del pakfire_args["mode"]
 116                 self.pakfire = base.Pakfire(mode="builder", distro_config=distro_config, **pakfire_args)
 117                 self.distro = self.pakfire.distro
 118                 self.path = self.pakfire.path
 119
 120                 # Log the package information.
 121                 self.pkg = packages.Makefile(self.pakfire, pkg)
 122                 self.log.info(_("Package information:"))
 123                 for line in self.pkg.dump(long=True).splitlines():
 124                         self.log.info("  %s" % line)
 125                 self.log.info("")
 126
 127                 # Download all package files.
 128                 self.pkg.download()
 129
 130                 # XXX need to make this configureable
 131                 self.settings = {
 132                         "enable_loop_devices" : True,
 133                         "enable_ccache"   : True,
 134                         "enable_icecream" : False,
 135                 }
 136                 #self.settings.update(settings)
 137
 138                 # Lock the buildroot
 139                 self._lock = None
 140                 self.lock()
 141
 142                 # Save the build time.
 143                 self.build_time = int(time.time())
 144
 145         def start(self):
 146                 # Mount the directories.
 147                 self._mountall()
 148
 149                 # Populate /dev.
 150                 self.populate_dev()
 151
 152                 # Setup domain name resolution in chroot.
 153                 self.setup_dns()
 154
 155                 # Extract all needed packages.
 156                 self.extract()
 157
 158         def stop(self):
 159                 # Kill all still running processes.
 160                 util.orphans_kill(self.path)
 161
 162                 # Close pakfire instance.
 163                 del self.pakfire
 164
 165                 # Umount the build environment.
 166                 self._umountall()
 167
 168                 # Remove all files.
 169                 self.destroy()
 170
 171         @property
 172         def arch(self):
 173                 """
 174                         Inherit architecture from distribution configuration.
 175                 """
 176                 return self.distro.arch
 177
 178         @property
 179         def info(self):
 180                 return {
 181                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 182                         "build_host" : socket.gethostname(),
 183                         "build_id"   : self.build_id,
 184                         "build_time" : self.build_time,
 185                 }
 186
 187         def lock(self):
 188                 filename = os.path.join(self.path, ".lock")
 189
 190                 try:
 191                         self._lock = open(filename, "a+")
 192                 except IOError, e:
 193                         return 0
 194
 195                 try:
 196                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 197                 except IOError, e:
 198                         raise BuildRootLocked, "Buildroot is locked"
 199
 200                 return 1
 201
 202         def unlock(self):
 203                 if self._lock:
 204                         self._lock.close()
 205                         self._lock = None
 206
 207         def copyin(self, file_out, file_in):
 208                 if file_in.startswith("/"):
 209                         file_in = file_in[1:]
 210
 211                 file_in = self.chrootPath(file_in)
 212
 213                 #if not os.path.exists(file_out):
 214                 #       return
 215
 216                 dir_in = os.path.dirname(file_in)
 217                 if not os.path.exists(dir_in):
 218                         os.makedirs(dir_in)
 219
 220                 logging.debug("%s --> %s" % (file_out, file_in))
 221
 222                 shutil.copy2(file_out, file_in)
 223
 224         def copyout(self, file_in, file_out):
 225                 if file_in.startswith("/"):
 226                         file_in = file_in[1:]
 227
 228                 file_in = self.chrootPath(file_in)
 229
 230                 #if not os.path.exists(file_in):
 231                 #       return
 232
 233                 dir_out = os.path.dirname(file_out)
 234                 if not os.path.exists(dir_out):
 235                         os.makedirs(dir_out)
 236
 237                 logging.debug("%s --> %s" % (file_in, file_out))
 238
 239                 shutil.copy2(file_in, file_out)
 240
 241         def copy_result(self, resultdir):
 242                 dir_in = self.chrootPath("result")
 243
 244                 for dir, subdirs, files in os.walk(dir_in):
 245                         basename = os.path.basename(dir)
 246                         dir = dir[len(self.chrootPath()):]
 247                         for file in files:
 248                                 file_in = os.path.join(dir, file)
 249
 250                                 file_out = os.path.join(
 251                                         resultdir,
 252                                         basename,
 253                                         file,
 254                                 )
 255
 256                                 self.copyout(file_in, file_out)
 257
 258         def extract(self, requires=None, build_deps=True):
 259                 """
 260                         Gets a dependency set and extracts all packages
 261                         to the environment.
 262                 """
 263                 if not requires:
 264                         requires = []
 265
 266                 # Add neccessary build dependencies.
 267                 requires += BUILD_PACKAGES
 268
 269                 # If we have ccache enabled, we need to extract it
 270                 # to the build chroot.
 271                 if self.settings.get("enable_ccache"):
 272                         requires.append("ccache")
 273
 274                 # If we have icecream enabled, we need to extract it
 275                 # to the build chroot.
 276                 if self.settings.get("enable_icecream"):
 277                         requires.append("icecream")
 278
 279                 # Get build dependencies from source package.
 280                 for req in self.pkg.requires:
 281                         requires.append(req)
 282
 283                 # Install all packages.
 284                 self.install(requires)
 285
 286                 # Copy the makefile and load source tarballs.
 287                 self.pkg.extract(_("Extracting"),
 288                         prefix=os.path.join(self.path, "build"))
 289
 290         def install(self, requires):
 291                 """
 292                         Install everything that is required in requires.
 293                 """
 294                 # If we got nothing to do, we quit immediately.
 295                 if not requires:
 296                         return
 297
 298                 self.pakfire.install(requires, interactive=False,
 299                         allow_downgrade=True, logger=self.log)
 300
 301         def install_test(self):
 302                 pkgs = []
 303                 for dir, subdirs, files in os.walk(self.chrootPath("result")):
 304                         for file in files:
 305                                 pkgs.append(os.path.join(dir, file))
 306
 307                 self.pakfire.localinstall(pkgs, yes=True, allow_uninstall=True)
 308
 309         def chrootPath(self, *args):
 310                 # Remove all leading slashes
 311                 _args = []
 312                 for arg in args:
 313                         if arg.startswith("/"):
 314                                 arg = arg[1:]
 315                         _args.append(arg)
 316                 args = _args
 317
 318                 ret = os.path.join(self.path, *args)
 319                 ret = ret.replace("//", "/")
 320
 321                 assert ret.startswith(self.path)
 322
 323                 return ret
 324
 325         def populate_dev(self):
 326                 nodes = [
 327                         "/dev/null",
 328                         "/dev/zero",
 329                         "/dev/full",
 330                         "/dev/random",
 331                         "/dev/urandom",
 332                         "/dev/tty",
 333                         "/dev/ptmx",
 334                         "/dev/kmsg",
 335                         "/dev/rtc0",
 336                         "/dev/console",
 337                 ]
 338
 339                 # If we need loop devices (which are optional) we create them here.
 340                 if self.settings["enable_loop_devices"]:
 341                         for i in range(0, 7):
 342                                 nodes.append("/dev/loop%d" % i)
 343
 344                 for node in nodes:
 345                         # Stat the original node of the host system and copy it to
 346                         # the build chroot.
 347                         node_stat = os.stat(node)
 348
 349                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 350
 351                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 352                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 353                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 354                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 355
 356         def setup_dns(self):
 357                 """
 358                         Add DNS resolution facility to chroot environment by copying
 359                         /etc/resolv.conf and /etc/hosts.
 360                 """
 361                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 362                         self.copyin(i, i)
 363
 364         def _create_node(self, filename, mode, device):
 365                 logging.debug("Create node: %s (%s)" % (filename, mode))
 366
 367                 filename = self.chrootPath(filename)
 368
 369                 # Create parent directory if it is missing.
 370                 dirname = os.path.dirname(filename)
 371                 if not os.path.exists(dirname):
 372                         os.makedirs(dirname)
 373
 374                 os.mknod(filename, mode, device)
 375
 376         def destroy(self):
 377                 logging.debug("Destroying environment %s" % self.path)
 378
 379                 if os.path.exists(self.path):
 380                         util.rm(self.path)
 381
 382         def cleanup(self):
 383                 logging.debug("Cleaning environemnt.")
 384
 385                 # Remove the build directory and buildroot.
 386                 dirs = ("build", "result")
 387
 388                 for d in dirs:
 389                         d = self.chrootPath(d)
 390                         if not os.path.exists(d):
 391                                 continue
 392
 393                         util.rm(d)
 394                         os.makedirs(d)
 395
 396         def _mountall(self):
 397                 self.log.debug("Mounting environment")
 398                 for src, dest, fs, options in self.mountpoints:
 399                         mountpoint = self.chrootPath(dest)
 400                         if options:
 401                                 options = "-o %s" % options
 402
 403                         # Eventually create mountpoint directory
 404                         if not os.path.exists(mountpoint):
 405                                 os.makedirs(mountpoint)
 406
 407                         cmd = "mount -n -t %s %s %s %s" % \
 408                                 (fs, options, src, mountpoint)
 409                         chroot.do(cmd, shell=True)
 410
 411         def _umountall(self):
 412                 self.log.debug("Umounting environment")
 413
 414                 mountpoints = []
 415                 for src, dest, fs, options in reversed(self.mountpoints):
 416                         if not dest in mountpoints:
 417                                 mountpoints.append(dest)
 418
 419                 for dest in mountpoints:
 420                         mountpoint = self.chrootPath(dest)
 421
 422                         chroot.do("umount -n %s" % mountpoint, raiseExc=0, shell=True)
 423
 424         @property
 425         def mountpoints(self):
 426                 mountpoints = []
 427
 428                 # Make root as a tmpfs.
 429                 #mountpoints += [
 430                 #       ("pakfire_root", "/", "tmpfs", "defaults"),
 431                 #]
 432
 433                 mountpoints += [
 434                         # src, dest, fs, options
 435                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 436                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 437                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 438                         ("/sys",          "/sys",      "bind",  "bind"),
 439                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 440                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 441                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 442                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 443                 ]
 444
 445                 # If selinux is enabled.
 446                 if os.path.exists("/sys/fs/selinux"):
 447                         mountpoints += [
 448                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 449                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 450                         ]
 451
 452                 # If ccache support is requested, we bind mount the cache.
 453                 if self.settings.get("enable_ccache"):
 454                         # Create ccache cache directory if it does not exist.
 455                         if not os.path.exists(CCACHE_CACHE_DIR):
 456                                 os.makedirs(CCACHE_CACHE_DIR)
 457
 458                         mountpoints += [
 459                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 460                         ]
 461
 462                 return mountpoints
 463
 464         @property
 465         def environ(self):
 466                 env = {
 467                         # Add HOME manually, because it is occasionally not set
 468                         # and some builds get in trouble then.
 469                         "HOME" : "/root",
 470                         "TERM" : os.environ.get("TERM", "dumb"),
 471                         "PS1"  : "\u:\w\$ ",
 472
 473                         # Set the container that we can detect, if we are inside a
 474                         # chroot.
 475                         "container" : "pakfire-builder",
 476                 }
 477
 478                 # Inherit environment from distro
 479                 env.update(self.pakfire.distro.environ)
 480
 481                 # Icecream environment settings
 482                 if self.settings.get("enable_icecream", False):
 483                         # Set the toolchain path
 484                         if self.settings.get("icecream_toolchain", None):
 485                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 486
 487                         # Set preferred host if configured.
 488                         if self.settings.get("icecream_preferred_host", None):
 489                                 env["ICECC_PREFERRED_HOST"] = \
 490                                         self.settings.get("icecream_preferred_host")
 491
 492                 # XXX what do we need else?
 493
 494                 return env
 495
 496         def do(self, command, shell=True, personality=None, logger=None, *args, **kwargs):
 497                 ret = None
 498
 499                 # Environment variables
 500                 env = self.environ
 501
 502                 if kwargs.has_key("env"):
 503                         env.update(kwargs.pop("env"))
 504
 505                 logging.debug("Environment:")
 506                 for k, v in sorted(env.items()):
 507                         logging.debug("  %s=%s" % (k, v))
 508
 509                 # Update personality it none was set
 510                 if not personality:
 511                         personality = self.distro.personality
 512
 513                 # Make every shell to a login shell because we set a lot of
 514                 # environment things there.
 515                 if shell:
 516                         command = ["bash", "--login", "-c", command]
 517
 518                 if not kwargs.has_key("chrootPath"):
 519                         kwargs["chrootPath"] = self.chrootPath()
 520
 521                 ret = chroot.do(
 522                         command,
 523                         personality=personality,
 524                         shell=False,
 525                         env=env,
 526                         logger=logger,
 527                         *args,
 528                         **kwargs
 529                 )
 530
 531                 return ret
 532
 533         def build(self, install_test=True):
 534                 assert self.pkg
 535
 536                 pkgfile = os.path.join("/build", os.path.basename(self.pkg.filename))
 537                 resultdir = self.chrootPath("/result")
 538
 539                 # Create the build command, that is executed in the chroot.
 540                 build_command = ["/usr/lib/pakfire/builder", "--offline", "build", pkgfile,
 541                         "--nodeps", "--resultdir=/result",]
 542
 543                 try:
 544                         self.do(" ".join(build_command), logger=self.log)
 545
 546                 except Error:
 547                         raise BuildError, _("The build command failed. See logfile for details.")
 548
 549                 # Perform install test.
 550                 if install_test:
 551                         self.install_test()
 552
 553                 # Copy the final packages and stuff.
 554                 # XXX TODO resultdir
 555
 556         def shell(self, args=[]):
 557                 if not util.cli_is_interactive():
 558                         logging.warning("Cannot run shell on non-interactive console.")
 559                         return
 560
 561                 # Install all packages that are needed to run a shell.
 562                 self.install(SHELL_PACKAGES)
 563
 564                 # XXX need to set CFLAGS here
 565                 command = "/usr/sbin/chroot %s %s %s" % \
 566                         (self.chrootPath(), SHELL_SCRIPT, " ".join(args))
 567
 568                 # Add personality if we require one
 569                 if self.pakfire.distro.personality:
 570                         command = "%s %s" % (self.pakfire.distro.personality, command)
 571
 572                 for key, val in self.environ.items():
 573                         command = "%s=\"%s\" " % (key, val) + command
 574
 575                 # Empty the environment
 576                 command = "env -i - %s" % command
 577
 578                 logging.debug("Shell command: %s" % command)
 579
 580                 shell = os.system(command)
 581                 return os.WEXITSTATUS(shell)
 582
 583
 584 class Builder(object):
 585         def __init__(self, pakfire, filename, resultdir, **kwargs):
 586                 self.pakfire = pakfire
 587
 588                 self.filename = filename
 589
 590                 self.resultdir = resultdir
 591
 592                 # Open package file.
 593                 self.pkg = packages.Makefile(self.pakfire, self.filename)
 594
 595                 self._environ = {
 596                         "LANG"             : "C",
 597                 }
 598
 599         @property
 600         def buildroot(self):
 601                 return self.pkg.buildroot
 602
 603         @property
 604         def distro(self):
 605                 return self.pakfire.distro
 606
 607         @property
 608         def environ(self):
 609                 environ = os.environ
 610
 611                 # Get all definitions from the package.
 612                 environ.update(self.pkg.exports)
 613
 614                 # Overwrite some definitions by default values.
 615                 environ.update(self._environ)
 616
 617                 return environ
 618
 619         def do(self, command, shell=True, personality=None, cwd=None, *args, **kwargs):
 620                 # Environment variables
 621                 logging.debug("Environment:")
 622                 for k, v in sorted(self.environ.items()):
 623                         logging.debug("  %s=%s" % (k, v))
 624
 625                 # Update personality it none was set
 626                 if not personality:
 627                         personality = self.distro.personality
 628
 629                 if not cwd:
 630                         cwd = "/%s" % LOCAL_TMP_PATH
 631
 632                 # Make every shell to a login shell because we set a lot of
 633                 # environment things there.
 634                 if shell:
 635                         command = ["bash", "--login", "-c", command]
 636
 637                 return chroot.do(
 638                         command,
 639                         personality=personality,
 640                         shell=False,
 641                         env=self.environ,
 642                         logger=logging.getLogger(),
 643                         cwd=cwd,
 644                         *args,
 645                         **kwargs
 646                 )
 647
 648         def create_icecream_toolchain(self):
 649                 try:
 650                         out = self.do("icecc --build-native 2>/dev/null", returnOutput=True)
 651                 except Error:
 652                         return
 653
 654                 for line in out.splitlines():
 655                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
 656                         if m:
 657                                 self._environ["icecream_toolchain"] = "/%s" % m.group(1)
 658
 659         def create_buildscript(self, stage):
 660                 file = "/tmp/build_%s" % util.random_string()
 661
 662                 # Get buildscript from the package.
 663                 script = self.pkg.get_buildscript(stage)
 664
 665                 # Write script to an empty file.
 666                 f = open(file, "w")
 667                 f.write("#!/bin/sh\n\n")
 668                 f.write("set -e\n")
 669                 f.write("set -x\n")
 670                 f.write("\n%s\n" % script)
 671                 f.write("exit 0\n")
 672                 f.close()
 673                 os.chmod(file, 700)
 674
 675                 return file
 676
 677         def build(self):
 678                 # Create buildroot.
 679                 if not os.path.exists(self.buildroot):
 680                         os.makedirs(self.buildroot)
 681
 682                 # Build icecream toolchain if icecream is installed.
 683                 self.create_icecream_toolchain()
 684
 685                 for stage in ("prepare", "build", "test", "install"):
 686                         self.build_stage(stage)
 687
 688                 # Package the result.
 689                 # Make all these little package from the build environment.
 690                 logging.info(_("Creating packages:"))
 691                 pkgs = []
 692                 for pkg in reversed(self.pkg.packages):
 693                         packager = packages.packager.BinaryPackager(self.pakfire, pkg,
 694                                 self, self.buildroot)
 695                         pkg = packager.run(self.resultdir)
 696                         pkgs.append(pkg)
 697                 logging.info("")
 698
 699                 for pkg in sorted(pkgs):
 700                         for line in pkg.dump(long=True).splitlines():
 701                                 logging.info(line)
 702                         logging.info("")
 703                 logging.info("")
 704
 705         def build_stage(self, stage):
 706                 # Get the buildscript for this stage.
 707                 buildscript = self.create_buildscript(stage)
 708
 709                 # Execute the buildscript of this stage.
 710                 logging.info(_("Running stage %s:") % stage)
 711
 712                 try:
 713                         self.do(buildscript, shell=False)
 714
 715                 finally:
 716                         # Remove the buildscript.
 717                         if os.path.exists(buildscript):
 718                                 os.unlink(buildscript)
 719
 720         def cleanup(self):
 721                 if os.path.exists(self.buildroot):
 722                         util.rm(self.buildroot)