src/pakfire/builder.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import fcntl
  23 import grp
  24 import math
  25 import os
  26 import re
  27 import shutil
  28 import socket
  29 import tempfile
  30 import time
  31 import uuid
  32
  33 from . import _pakfire
  34 from . import base
  35 from . import cgroups
  36 from . import config
  37 from . import downloaders
  38 from . import logger
  39 from . import packages
  40 from . import repository
  41 from . import shell
  42 from . import util
  43
  44 import logging
  45 log = logging.getLogger("pakfire.builder")
  46
  47 from .system import system
  48 from .constants import *
  49 from .i18n import _
  50 from .errors import BuildError, BuildRootLocked, Error
  51
  52
  53 BUILD_LOG_HEADER = """
  54  ____       _     __ _            _           _ _     _
  55 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  56 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  57 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  58 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  59
  60         Version : %(version)s
  61         Host    : %(hostname)s (%(host_arch)s)
  62         Time    : %(time)s
  63
  64 """
  65
  66 class Builder(object):
  67         def __init__(self, arch=None, build_id=None, logfile=None, **kwargs):
  68                 self.config = config.Config("general.conf", "builder.conf")
  69
  70                 distro_name = self.config.get("builder", "distro", None)
  71                 if distro_name:
  72                         self.config.read("distros/%s.conf" % distro_name)
  73
  74                 # Settings array.
  75                 self.settings = {
  76                         "enable_loop_devices" : self.config.get_bool("builder", "use_loop_devices", True),
  77                         "enable_ccache"       : self.config.get_bool("builder", "use_ccache", True),
  78                         "buildroot_tmpfs"     : self.config.get_bool("builder", "use_tmpfs", False),
  79                         "private_network"     : self.config.get_bool("builder", "private_network", False),
  80                 }
  81
  82                 # Get ccache settings.
  83                 if self.settings.get("enable_ccache", False):
  84                         self.settings.update({
  85                                 "ccache_compress" : self.config.get_bool("ccache", "compress", True),
  86                         })
  87
  88                 # Add settings from keyword arguments
  89                 self.settings.update(kwargs)
  90
  91                 # Setup logging
  92                 self.setup_logging(logfile)
  93
  94                 # Generate a build ID
  95                 self.build_id = build_id or "%s" % uuid.uuid4()
  96
  97                 # Path
  98                 self.path = os.path.join(BUILD_ROOT, self.build_id)
  99                 self._lock = None
 100
 101                 # Architecture to build for
 102                 self.arch = arch or _pakfire.native_arch()
 103
 104                 # Check if this host can build the requested architecture.
 105                 if not _pakfire.arch_supported_by_host(self.arch):
 106                         raise BuildError(_("Cannot build for %s on this host") % self.arch)
 107
 108                 # Initialize cgroups
 109                 self.cgroup = self._make_cgroup()
 110
 111                 # Unshare namepsace.
 112                 # If this fails because the kernel has no support for CLONE_NEWIPC or CLONE_NEWUTS,
 113                 # we try to fall back to just set CLONE_NEWNS.
 114                 try:
 115                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNS|_pakfire.SCHED_CLONE_NEWIPC|_pakfire.SCHED_CLONE_NEWUTS)
 116                 except RuntimeError as e:
 117                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNS)
 118
 119                 # Optionally enable private networking.
 120                 if self.settings.get("private_network", None):
 121                         _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNET)
 122
 123         def __enter__(self):
 124                 self.log.debug("Entering %s" % self.path)
 125
 126                 # Mount the directories
 127                 try:
 128                         self._mountall()
 129                 except OSError as e:
 130                         if e.errno == 30: # Read-only FS
 131                                 raise BuildError("Buildroot is read-only: %s" % self.path)
 132
 133                         # Raise all other errors
 134                         raise
 135
 136                 # Lock the build environment
 137                 self.lock()
 138
 139                 # Populate /dev
 140                 self.populate_dev()
 141
 142                 # Setup domain name resolution in chroot
 143                 self.setup_dns()
 144
 145                 return BuilderContext(self)
 146
 147         def __exit__(self, type, value, traceback):
 148                 self.log.debug("Leaving %s" % self.path)
 149
 150                 # Kill all remaining processes in the build environment
 151                 self.cgroup.killall()
 152
 153                 # Destroy the cgroup
 154                 self.cgroup.destroy()
 155                 self.cgroup = None
 156
 157                 # Umount the build environment
 158                 self._umountall()
 159
 160                 # Unlock build environment
 161                 self.unlock()
 162
 163                 # Delete everything
 164                 self._destroy()
 165
 166         def setup_logging(self, logfile):
 167                 if logfile:
 168                         self.log = log.getChild(self.build_id)
 169                         # Propage everything to the root logger that we will see something
 170                         # on the terminal.
 171                         self.log.propagate = 1
 172                         self.log.setLevel(logging.INFO)
 173
 174                         # Add the given logfile to the logger.
 175                         h = logging.FileHandler(logfile)
 176                         self.log.addHandler(h)
 177
 178                         # Format the log output for the file.
 179                         f = logger.BuildFormatter()
 180                         h.setFormatter(f)
 181                 else:
 182                         # If no logile was given, we use the root logger.
 183                         self.log = logging.getLogger("pakfire")
 184
 185         def _make_cgroup(self):
 186                 """
 187                         Initialises a cgroup so that we can enforce resource limits
 188                         and can identify processes belonging to this build environment.
 189                 """
 190                 # Find our current group
 191                 parent = cgroups.get_own_group()
 192
 193                 # Create a sub-group
 194                 cgroup = parent.create_subgroup("pakfire-%s" % self.build_id)
 195
 196                 # Make this process join the new group
 197                 cgroup.attach_self()
 198
 199                 return cgroup
 200
 201         def lock(self):
 202                 filename = os.path.join(self.path, ".lock")
 203
 204                 try:
 205                         self._lock = open(filename, "a+")
 206                 except IOError as e:
 207                         return 0
 208
 209                 try:
 210                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 211                 except IOError as e:
 212                         raise BuildRootLocked("Buildroot is locked")
 213
 214                 return 1
 215
 216         def unlock(self):
 217                 if self._lock:
 218                         self._lock.close()
 219                         self._lock = None
 220
 221         def _destroy(self):
 222                 self.log.debug("Destroying environment %s" % self.path)
 223
 224                 if os.path.exists(self.path):
 225                         util.rm(self.path)
 226
 227         @property
 228         def mountpoints(self):
 229                 mountpoints = []
 230
 231                 # Make root as a tmpfs if enabled.
 232                 if self.settings.get("buildroot_tmpfs"):
 233                         mountpoints += [
 234                                 ("pakfire_root", "/", "tmpfs", "defaults"),
 235                         ]
 236
 237                 mountpoints += [
 238                         # src, dest, fs, options
 239                         ("pakfire_proc",  "/proc",     "proc",  "nosuid,noexec,nodev"),
 240                         ("/proc/sys",     "/proc/sys", "bind",  "bind"),
 241                         ("/proc/sys",     "/proc/sys", "bind",  "bind,ro,remount"),
 242                         ("/sys",          "/sys",      "bind",  "bind"),
 243                         ("/sys",          "/sys",      "bind",  "bind,ro,remount"),
 244                         ("pakfire_tmpfs", "/dev",      "tmpfs", "mode=755,nosuid"),
 245                         ("/dev/pts",      "/dev/pts",  "bind",  "bind"),
 246                         ("pakfire_tmpfs", "/run",      "tmpfs", "mode=755,nosuid,nodev"),
 247                         ("pakfire_tmpfs", "/tmp",      "tmpfs", "mode=755,nosuid,nodev"),
 248                 ]
 249
 250                 # If selinux is enabled.
 251                 if os.path.exists("/sys/fs/selinux"):
 252                         mountpoints += [
 253                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
 254                                 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
 255                         ]
 256
 257                 # If ccache support is requested, we bind mount the cache.
 258                 if self.settings.get("enable_ccache"):
 259                         # Create ccache cache directory if it does not exist.
 260                         if not os.path.exists(CCACHE_CACHE_DIR):
 261                                 os.makedirs(CCACHE_CACHE_DIR)
 262
 263                         mountpoints += [
 264                                 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
 265                         ]
 266
 267                 return mountpoints
 268
 269         def _mountall(self):
 270                 self.log.debug("Mounting environment")
 271
 272                 for src, dest, fs, options in self.mountpoints:
 273                         mountpoint = self.chrootPath(dest)
 274                         if options:
 275                                 options = "-o %s" % options
 276
 277                         # Eventually create mountpoint directory
 278                         if not os.path.exists(mountpoint):
 279                                 os.makedirs(mountpoint)
 280
 281                         self.execute_root("mount -n -t %s %s %s %s" % (fs, options, src, mountpoint), shell=True)
 282
 283         def _umountall(self):
 284                 self.log.debug("Umounting environment")
 285
 286                 mountpoints = []
 287                 for src, dest, fs, options in reversed(self.mountpoints):
 288                         dest = self.chrootPath(dest)
 289
 290                         if not dest in mountpoints:
 291                                 mountpoints.append(dest)
 292
 293                 while mountpoints:
 294                         for mp in mountpoints:
 295                                 try:
 296                                         self.execute_root("umount -n %s" % mp, shell=True)
 297                                 except ShellEnvironmentError:
 298                                         pass
 299
 300                                 if not os.path.ismount(mp):
 301                                         mountpoints.remove(mp)
 302
 303         def copyin(self, file_out, file_in):
 304                 if file_in.startswith("/"):
 305                         file_in = file_in[1:]
 306
 307                 file_in = self.chrootPath(file_in)
 308
 309                 #if not os.path.exists(file_out):
 310                 #       return
 311
 312                 dir_in = os.path.dirname(file_in)
 313                 if not os.path.exists(dir_in):
 314                         os.makedirs(dir_in)
 315
 316                 self.log.debug("%s --> %s" % (file_out, file_in))
 317
 318                 shutil.copy2(file_out, file_in)
 319
 320         def copyout(self, file_in, file_out):
 321                 if file_in.startswith("/"):
 322                         file_in = file_in[1:]
 323
 324                 file_in = self.chrootPath(file_in)
 325
 326                 #if not os.path.exists(file_in):
 327                 #       return
 328
 329                 dir_out = os.path.dirname(file_out)
 330                 if not os.path.exists(dir_out):
 331                         os.makedirs(dir_out)
 332
 333                 self.log.debug("%s --> %s" % (file_in, file_out))
 334
 335                 shutil.copy2(file_in, file_out)
 336
 337         def populate_dev(self):
 338                 nodes = [
 339                         "/dev/null",
 340                         "/dev/zero",
 341                         "/dev/full",
 342                         "/dev/random",
 343                         "/dev/urandom",
 344                         "/dev/tty",
 345                         "/dev/ptmx",
 346                         "/dev/kmsg",
 347                         "/dev/rtc0",
 348                         "/dev/console",
 349                 ]
 350
 351                 # If we need loop devices (which are optional) we create them here.
 352                 if self.settings["enable_loop_devices"]:
 353                         for i in range(0, 7):
 354                                 nodes.append("/dev/loop%d" % i)
 355
 356                 for node in nodes:
 357                         # Stat the original node of the host system and copy it to
 358                         # the build chroot.
 359                         try:
 360                                 node_stat = os.stat(node)
 361
 362                         # If it cannot be found, just go on.
 363                         except OSError:
 364                                 continue
 365
 366                         self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
 367
 368                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 369                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 370                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 371                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 372
 373         def chrootPath(self, *args):
 374                 # Remove all leading slashes
 375                 _args = []
 376                 for arg in args:
 377                         if arg.startswith("/"):
 378                                 arg = arg[1:]
 379                         _args.append(arg)
 380                 args = _args
 381
 382                 ret = os.path.join(self.path, *args)
 383                 ret = ret.replace("//", "/")
 384
 385                 assert ret.startswith(self.path)
 386
 387                 return ret
 388
 389         def setup_dns(self):
 390                 """
 391                         Add DNS resolution facility to chroot environment by copying
 392                         /etc/resolv.conf and /etc/hosts.
 393                 """
 394                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 395                         self.copyin(i, i)
 396
 397         def _create_node(self, filename, mode, device):
 398                 self.log.debug("Create node: %s (%s)" % (filename, mode))
 399
 400                 filename = self.chrootPath(filename)
 401
 402                 # Create parent directory if it is missing.
 403                 dirname = os.path.dirname(filename)
 404                 if not os.path.exists(dirname):
 405                         os.makedirs(dirname)
 406
 407                 os.mknod(filename, mode, device)
 408
 409         def execute_root(self, command, **kwargs):
 410                 """
 411                         Executes the given command outside the build chroot.
 412                 """
 413                 shellenv = shell.ShellExecuteEnvironment(command, logger=self.log, **kwargs)
 414                 shellenv.execute()
 415
 416                 return shellenv
 417
 418
 419 class BuilderContext(object):
 420         def __init__(self, builder):
 421                 self.builder = builder
 422
 423                 # Get a reference to the logger
 424                 self.log = self.builder.log
 425
 426                 # Initialise Pakfire instance
 427                 self.pakfire = base.Pakfire(
 428                         path=self.builder.path,
 429                         config=self.builder.config,
 430                         distro=self.builder.config.distro,
 431                         arch=self.builder.arch,
 432                 )
 433
 434         @property
 435         def environ(self):
 436                 # Build a minimal environment for executing, but try to inherit TERM and LANG
 437                 env = {
 438                         "HOME" : "/root",
 439                         "PATH" : "/usr/bin:/bin:/usr/sbin:/sbin",
 440                         "PS1"  : "pakfire-chroot \w> ",
 441                         "TERM" : os.environ.get("TERM", "vt100"),
 442                         "LANG" : os.environ.get("LANG", "en_US.UTF-8"),
 443                 }
 444
 445                 # Inherit environment from distro
 446                 env.update(self.pakfire.distro.environ)
 447
 448                 # ccache environment settings
 449                 if self.builder.settings.get("enable_ccache", False):
 450                         compress = self.builder.settings.get("ccache_compress", False)
 451                         if compress:
 452                                 env["CCACHE_COMPRESS"] = "1"
 453
 454                         # Let ccache create its temporary files in /tmp.
 455                         env["CCACHE_TEMPDIR"] = "/tmp"
 456
 457                 # Fake UTS_MACHINE, when we cannot use the personality syscall and
 458                 # if the host architecture is not equal to the target architecture.
 459                 if not _pakfire.native_arch() == self.pakfire.arch:
 460                         env.update({
 461                                 "LD_PRELOAD"  : "/usr/lib/libpakfire_preload.so",
 462                                 "UTS_MACHINE" : self.pakfire.arch,
 463                         })
 464
 465                 return env
 466
 467         def _install(self, packages):
 468                 self.log.debug(_("Installing packages in build environment:"))
 469                 for package in packages:
 470                         self.log.debug("        %s" % package)
 471
 472                 # Initialise Pakfire
 473                 with self.pakfire as p:
 474                         # Install all required packages
 475                         transaction = p.install(packages)
 476
 477                         # Dump transaction to log
 478                         t = transaction.dump()
 479                         self.log.info(t)
 480
 481                         # Download transaction
 482                         d = downloaders.TransactionDownloader(self.pakfire, transaction)
 483                         d.download()
 484
 485                         # Run the transaction
 486                         transaction.run()
 487
 488         def build(self, package, private_network=True, shell=True):
 489                 # Install build environment
 490                 packages = [
 491                         "@Build",
 492                 ]
 493
 494                 # If we have ccache enabled, we need to install it, too
 495                 if self.builder.settings.get("enable_ccache"):
 496                         packages.append("ccache")
 497
 498                 # Open the package archive
 499                 archive = _pakfire.Archive(self.pakfire, package)
 500
 501                 requires = archive.get("dependencies.requires")
 502                 packages += requires.splitlines()
 503
 504                 # Setup the environment including any build dependencies
 505                 self._install(packages)
 506
 507                 # XXX perform build
 508
 509         def shell(self, install=None):
 510                 if not util.cli_is_interactive():
 511                         self.log.warning("Cannot run shell on non-interactive console.")
 512                         return
 513
 514                 # Collect packages to install
 515                 packages = []
 516
 517                 # Install our standard shell packages
 518                 packages += SHELL_PACKAGES
 519
 520                 # Install any packages the user requested
 521                 if install:
 522                         packages += install
 523
 524                 # Install all required packages
 525                 self._install(packages)
 526
 527                 # Enter the shell
 528                 self.pakfire.execute(["/usr/bin/bash", "--login"],
 529                         environ=self.environ, enable_network=True)