From: Michael Tremer Date: Sat, 20 May 2017 10:11:02 +0000 (+0200) Subject: Big rewrite of the builder class X-Git-Tag: 0.9.28~1285^2~1342 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2274aab38c8fe7ac0b0f3fdc126f4210820734ba;p=pakfire.git Big rewrite of the builder class This is WIP and mainly a reformatting and rewrite to use the new interfaces of the Pakfire class. Signed-off-by: Michael Tremer --- diff --git a/src/pakfire/base.py b/src/pakfire/base.py index 33bf00940..c4aa41101 100644 --- a/src/pakfire/base.py +++ b/src/pakfire/base.py @@ -25,8 +25,6 @@ import string from . import _pakfire from . import actions -from . import builder -from . import config from . import distro from . import filelist from . import keyring diff --git a/src/pakfire/builder.py b/src/pakfire/builder.py index 158d2d276..8b99600f2 100644 --- a/src/pakfire/builder.py +++ b/src/pakfire/builder.py @@ -25,14 +25,15 @@ import math import os import re import shutil -import signal import socket import tempfile import time import uuid from . import _pakfire +from . import base from . import cgroup +from . import config from . import logger from . import packages from . import repository @@ -40,9 +41,8 @@ from . import shell from . import util import logging -log = logging.getLogger("pakfire") +log = logging.getLogger("pakfire.builder") -from .config import ConfigBuilder from .system import system from .constants import * from .i18n import _ @@ -62,50 +62,18 @@ BUILD_LOG_HEADER = """ """ -class BuildEnviron(object): - def __init__(self, pakfire, filename=None, distro_name=None, build_id=None, logfile=None, release_build=True, **kwargs): - self.pakfire = pakfire - - # Check if this host can build the requested architecture. - if not system.host_supports_arch(self.arch): - raise BuildError(_("Cannot build for %s on this host.") % self.arch) - - # Save the build id and generate one if no build id was provided. - if not build_id: - build_id = "%s" % uuid.uuid4() - - self.build_id = build_id - - # Setup the logging. - self.init_logging(logfile) - - # Initialize a cgroup (if supported). - self.init_cgroup() - - # This build is a release build? - self.release_build = release_build - - if self.release_build: - # Disable the local build repository in release mode. - self.pakfire.repos.disable_repo("build") - - # Log information about pakfire and some more information, when we - # are running in release mode. - logdata = { - "host_arch" : system.arch, - "hostname" : system.hostname, - "time" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()), - "version" : "Pakfire %s" % PAKFIRE_VERSION, - } +class Builder(object): + def __init__(self, package=None, arch=None, build_id=None, logfile=None, **kwargs): + self.config = config.Config("general.conf", "builder.conf") - for line in BUILD_LOG_HEADER.splitlines(): - self.log.info(line % logdata) + distro_name = self.config.get("builder", "distro", None) + if distro_name: + self.config.read("distros/%s.conf" % distro_name) # Settings array. self.settings = { "enable_loop_devices" : self.config.get_bool("builder", "use_loop_devices", True), "enable_ccache" : self.config.get_bool("builder", "use_ccache", True), - "sign_packages" : False, "buildroot_tmpfs" : self.config.get_bool("builder", "use_tmpfs", False), "private_network" : self.config.get_bool("builder", "private_network", False), } @@ -116,46 +84,29 @@ class BuildEnviron(object): "ccache_compress" : self.config.get_bool("ccache", "compress", True), }) - # Try to get the configured host key. If it is available, - # we will automatically sign all packages with it. - if self.keyring.get_host_key(secret=True): - self.settings["sign_packages"] = True - - # Add settings from keyword arguments. + # Add settings from keyword arguments self.settings.update(kwargs) - # Where do we put the result? - self.resultdir = os.path.join(self.pakfire.path, "result") - - # Open package. - # If we have a plain makefile, we first build a source package and go with that. - if filename: - # Open source package. - self.pkg = packages.SourcePackage(self.pakfire, None, filename) - assert self.pkg, filename + # Setup logging + self.setup_logging(logfile) - # Log the package information. - self.log.info(_("Package information:")) - for line in self.pkg.dump(int=True).splitlines(): - self.log.info(" %s" % line) - self.log.info("") + # Generate a build ID + self.build_id = build_id or "%s" % uuid.uuid4() - # Path where we extract the package and put all the source files. - self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name) - else: - # No package :( - self.pkg = None - - # Lock the buildroot + # Path + self.path = os.path.join(BUILD_ROOT, self.build_id) self._lock = None - # Save the build time. - self.build_time = time.time() + # Architecture to build for + self.arch = arch or system.arch - def setup_signal_handlers(self): - pass + # Check if this host can build the requested architecture. + if not system.host_supports_arch(self.arch): + raise BuildError(_("Cannot build for %s on this host") % self.arch) + + # Initialize a cgroup (if supported) + self.cgroup = self.make_cgroup() - def start(self): # Unshare namepsace. # If this fails because the kernel has no support for CLONE_NEWIPC or CLONE_NEWUTS, # we try to fall back to just set CLONE_NEWNS. @@ -164,33 +115,57 @@ class BuildEnviron(object): except RuntimeError as e: _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNS) - # Mount the directories. + # Optionally enable private networking. + if self.settings.get("private_network", None): + _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNET) + + # Create Pakfire instance + self.pakfire = base.Pakfire(path=self.path, config=self.config, distro=self.config.distro, arch=arch) + + def __del__(self): + """ + Releases build environment and clean up + """ + # Umount the build environment + self._umountall() + + # Destroy the pakfire instance + del self.pakfire + + # Unlock build environment + self.unlock() + + # Delete everything + self._destroy() + + def __enter__(self): + self.log.debug("Entering %s" % self.path) + + # Mount the directories try: self._mountall() except OSError as e: if e.errno == 30: # Read-only FS raise BuildError("Buildroot is read-only: %s" % self.pakfire.path) - # Raise all other errors. + # Raise all other errors raise - # Lock the build environment. + # Lock the build environment self.lock() - # Optionally enable private networking. - if self.settings.get("private_network", None): - _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNET) - - # Populate /dev. + # Populate /dev self.populate_dev() - # Setup domain name resolution in chroot. + # Setup domain name resolution in chroot self.setup_dns() - # Extract all needed packages. - self.extract() + return BuilderContext(self) - def stop(self): + def __exit__(self, type, value, traceback): + self.log.debug("Leaving %s" % self.path) + + # Kill all remaining processes in the build environment if self.cgroup: # Move the builder process out of the cgroup. self.cgroup.migrate_task(self.cgroup.parent, os.getpid()) @@ -212,68 +187,44 @@ class BuildEnviron(object): else: util.orphans_kill(self.path) - # Shut down pakfire instance. - self.pakfire.destroy() - - # Unlock build environment. - self.unlock() - - # Umount the build environment. - self._umountall() - - # Remove all files. - self.destroy() + def setup_logging(self, logfile): + if logfile: + self.log = log.getChild(self.build_id) + # Propage everything to the root logger that we will see something + # on the terminal. + self.log.propagate = 1 + self.log.setLevel(logging.INFO) - @property - def config(self): - """ - Proxy method for easy access to the configuration. - """ - return self.pakfire.config + # Add the given logfile to the logger. + h = logging.FileHandler(logfile) + self.log.addHandler(h) - @property - def distro(self): - """ - Proxy method for easy access to the distribution. - """ - return self.pakfire.distro + # Format the log output for the file. + f = logger.BuildFormatter() + h.setFormatter(f) + else: + # If no logile was given, we use the root logger. + self.log = logging.getLogger("pakfire") - @property - def path(self): + def make_cgroup(self): """ - Proxy method for easy access to the path. + Initialize cgroup (if the system supports it). """ - return self.pakfire.path + if not cgroup.supported(): + return - @property - def arch(self): - """ - Inherit architecture from distribution configuration. - """ - return self.pakfire.distro.arch + # Search for the cgroup this process is currently running in. + parent_cgroup = cgroup.find_by_pid(os.getpid()) + if not parent_cgroup: + return - @property - def personality(self): - """ - Gets the personality from the distribution configuration. - """ - return self.pakfire.distro.personality + # Create our own cgroup inside the parent cgroup. + c = parent_cgroup.create_child_cgroup("pakfire/builder/%s" % self.build_id) - @property - def info(self): - return { - "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)), - "build_host" : socket.gethostname(), - "build_id" : self.build_id, - "build_time" : self.build_time, - } + # Attach the pakfire-builder process to the group. + c.attach() - @property - def keyring(self): - """ - Shortcut to access the pakfire keyring. - """ - return self.pakfire.keyring + return c def lock(self): filename = os.path.join(self.path, ".lock") @@ -295,43 +246,87 @@ class BuildEnviron(object): self._lock.close() self._lock = None - def init_cgroup(self): - """ - Initialize cgroup (if the system supports it). - """ - if not cgroup.supported(): - self.cgroup = None - return + def _destroy(self): + self.log.debug("Destroying environment %s" % self.path) - # Search for the cgroup this process is currently running in. - parent_cgroup = cgroup.find_by_pid(os.getpid()) - if not parent_cgroup: - return + if os.path.exists(self.path): + util.rm(self.path) - # Create our own cgroup inside the parent cgroup. - self.cgroup = parent_cgroup.create_child_cgroup("pakfire/builder/%s" % self.build_id) + @property + def mountpoints(self): + mountpoints = [] - # Attach the pakfire-builder process to the group. - self.cgroup.attach() + # Make root as a tmpfs if enabled. + if self.settings.get("buildroot_tmpfs"): + mountpoints += [ + ("pakfire_root", "/", "tmpfs", "defaults"), + ] - def init_logging(self, logfile): - if logfile: - self.log = log.getChild(self.build_id) - # Propage everything to the root logger that we will see something - # on the terminal. - self.log.propagate = 1 - self.log.setLevel(logging.INFO) + mountpoints += [ + # src, dest, fs, options + ("pakfire_proc", "/proc", "proc", "nosuid,noexec,nodev"), + ("/proc/sys", "/proc/sys", "bind", "bind"), + ("/proc/sys", "/proc/sys", "bind", "bind,ro,remount"), + ("/sys", "/sys", "bind", "bind"), + ("/sys", "/sys", "bind", "bind,ro,remount"), + ("pakfire_tmpfs", "/dev", "tmpfs", "mode=755,nosuid"), + ("/dev/pts", "/dev/pts", "bind", "bind"), + ("pakfire_tmpfs", "/run", "tmpfs", "mode=755,nosuid,nodev"), + ("pakfire_tmpfs", "/tmp", "tmpfs", "mode=755,nosuid,nodev"), + ] - # Add the given logfile to the logger. - h = logging.FileHandler(logfile) - self.log.addHandler(h) + # If selinux is enabled. + if os.path.exists("/sys/fs/selinux"): + mountpoints += [ + ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"), + ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"), + ] - # Format the log output for the file. - f = logger.BuildFormatter() - h.setFormatter(f) - else: - # If no logile was given, we use the root logger. - self.log = logging.getLogger("pakfire") + # If ccache support is requested, we bind mount the cache. + if self.settings.get("enable_ccache"): + # Create ccache cache directory if it does not exist. + if not os.path.exists(CCACHE_CACHE_DIR): + os.makedirs(CCACHE_CACHE_DIR) + + mountpoints += [ + (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"), + ] + + return mountpoints + + def _mountall(self): + self.log.debug("Mounting environment") + + for src, dest, fs, options in self.mountpoints: + mountpoint = self.chrootPath(dest) + if options: + options = "-o %s" % options + + # Eventually create mountpoint directory + if not os.path.exists(mountpoint): + os.makedirs(mountpoint) + + self.execute_root("mount -n -t %s %s %s %s" % (fs, options, src, mountpoint), shell=True) + + def _umountall(self): + self.log.debug("Umounting environment") + + mountpoints = [] + for src, dest, fs, options in reversed(self.mountpoints): + dest = self.chrootPath(dest) + + if not dest in mountpoints: + mountpoints.append(dest) + + while mountpoints: + for mp in mountpoints: + try: + self.execute_root("umount -n %s" % mp, shell=True) + except ShellEnvironmentError: + pass + + if not os.path.ismount(mp): + mountpoints.remove(mp) def copyin(self, file_out, file_in): if file_in.startswith("/"): @@ -354,18 +349,326 @@ class BuildEnviron(object): if file_in.startswith("/"): file_in = file_in[1:] - file_in = self.chrootPath(file_in) + file_in = self.chrootPath(file_in) + + #if not os.path.exists(file_in): + # return + + dir_out = os.path.dirname(file_out) + if not os.path.exists(dir_out): + os.makedirs(dir_out) + + self.log.debug("%s --> %s" % (file_in, file_out)) + + shutil.copy2(file_in, file_out) + + def populate_dev(self): + nodes = [ + "/dev/null", + "/dev/zero", + "/dev/full", + "/dev/random", + "/dev/urandom", + "/dev/tty", + "/dev/ptmx", + "/dev/kmsg", + "/dev/rtc0", + "/dev/console", + ] + + # If we need loop devices (which are optional) we create them here. + if self.settings["enable_loop_devices"]: + for i in range(0, 7): + nodes.append("/dev/loop%d" % i) + + for node in nodes: + # Stat the original node of the host system and copy it to + # the build chroot. + try: + node_stat = os.stat(node) + + # If it cannot be found, just go on. + except OSError: + continue + + self._create_node(node, node_stat.st_mode, node_stat.st_rdev) + + os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin")) + os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout")) + os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr")) + os.symlink("/proc/self/fd", self.chrootPath("dev", "fd")) + + def chrootPath(self, *args): + # Remove all leading slashes + _args = [] + for arg in args: + if arg.startswith("/"): + arg = arg[1:] + _args.append(arg) + args = _args + + ret = os.path.join(self.path, *args) + ret = ret.replace("//", "/") + + assert ret.startswith(self.path) + + return ret + + def setup_dns(self): + """ + Add DNS resolution facility to chroot environment by copying + /etc/resolv.conf and /etc/hosts. + """ + for i in ("/etc/resolv.conf", "/etc/hosts"): + self.copyin(i, i) + + def _create_node(self, filename, mode, device): + self.log.debug("Create node: %s (%s)" % (filename, mode)) + + filename = self.chrootPath(filename) + + # Create parent directory if it is missing. + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname) + + os.mknod(filename, mode, device) + + def execute_root(self, command, **kwargs): + """ + Executes the given command outside the build chroot. + """ + shellenv = shell.ShellExecuteEnvironment(command, logger=self.log, **kwargs) + shellenv.execute() + + return shellenv + + +class BuilderContext(object): + def __init__(self, builder): + self.builder = builder + + # Get a reference to Pakfire + self.pakfire = self.builder.pakfire + + # Get a reference to the logger + self.log = self.builder.log + + @property + def environ(self): + env = MINIMAL_ENVIRONMENT.copy() + env.update({ + # Add HOME manually, because it is occasionally not set + # and some builds get in trouble then. + "TERM" : os.environ.get("TERM", "vt100"), + + # Sanitize language. + "LANG" : os.environ.setdefault("LANG", "en_US.UTF-8"), + + # Set the container that we can detect, if we are inside a + # chroot. + "container" : "pakfire-builder", + }) + + # Inherit environment from distro + env.update(self.pakfire.distro.environ) + + # ccache environment settings + if self.builder.settings.get("enable_ccache", False): + compress = self.builder.settings.get("ccache_compress", False) + if compress: + env["CCACHE_COMPRESS"] = "1" + + # Let ccache create its temporary files in /tmp. + env["CCACHE_TEMPDIR"] = "/tmp" + + # Fake UTS_MACHINE, when we cannot use the personality syscall and + # if the host architecture is not equal to the target architecture. + if not self.pakfire.arch.personality and \ + not system.native_arch == self.pakfire.arch.name: + env.update({ + "LD_PRELOAD" : "/usr/lib/libpakfire_preload.so", + "UTS_MACHINE" : self.pakfire.arch.name, + }) + + return env + + def setup(self, install=None): + self.log.info(_("Install packages needed for build...")) + + packages = [ + "@Build", + #"pakfire-build >= %s" % self.pakfire.__version__, + ] + + # If we have ccache enabled, we need to extract it + # to the build chroot + if self.builder.settings.get("enable_ccache"): + packages.append("ccache") + + # Install additional packages + if install: + packages += install + + # Logging + self.log.debug(_("Installing build requirements: %s") % ", ".join(packages)) + + # Initialise Pakfire and install all required packages + with self.pakfire as p: + p.install(packages) + + def build(self, package, private_network=True, shell=True): + package = self._prepare_package(package) + assert package + + # Setup the environment including any build dependencies + self.setup(install=package.requires) + + def _prepare_package(self, package): + # Check if the file exists + if not os.path.exists(package): + raise FileNotFoundError(package) + + # Try opening the package + return packages.open(self.pakfire, None, package) + + def shell(self, install=[]): + if not util.cli_is_interactive(): + self.log.warning("Cannot run shell on non-interactive console.") + return + + # Install our standard shell packages + install += SHELL_PACKAGES + + self.setup(install=install) + + command = "/usr/sbin/chroot %s %s %s" % (self.chrootPath(), SHELL_SCRIPT) + + # Add personality if we require one + if self.pakfire.distro.personality: + command = "%s %s" % (self.pakfire.distro.personality, command) + + for key, val in list(self.environ.items()): + command = "%s=\"%s\" " % (key, val) + command + + # Empty the environment + command = "env -i - %s" % command + + self.log.debug("Shell command: %s" % command) + + shell = os.system(command) + return os.WEXITSTATUS(shell) + + +class BuildEnviron(object): + def __init__(self, pakfire, filename=None, distro_name=None, build_id=None, logfile=None, release_build=True, **kwargs): + self.pakfire = pakfire + + # This build is a release build? + self.release_build = release_build + + if self.release_build: + # Disable the local build repository in release mode. + self.pakfire.repos.disable_repo("build") + + # Log information about pakfire and some more information, when we + # are running in release mode. + logdata = { + "host_arch" : system.arch, + "hostname" : system.hostname, + "time" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()), + "version" : "Pakfire %s" % PAKFIRE_VERSION, + } + + for line in BUILD_LOG_HEADER.splitlines(): + self.log.info(line % logdata) + + # Where do we put the result? + self.resultdir = os.path.join(self.pakfire.path, "result") + + # Open package. + # If we have a plain makefile, we first build a source package and go with that. + if filename: + # Open source package. + self.pkg = packages.SourcePackage(self.pakfire, None, filename) + assert self.pkg, filename + + # Log the package information. + self.log.info(_("Package information:")) + for line in self.pkg.dump(int=True).splitlines(): + self.log.info(" %s" % line) + self.log.info("") + + # Path where we extract the package and put all the source files. + self.build_dir = os.path.join(self.path, "usr/src/packages", self.pkg.friendly_name) + else: + # No package :( + self.pkg = None + + # Lock the buildroot + self._lock = None + + # Save the build time. + self.build_time = time.time() + + def start(self): + # Extract all needed packages. + self.extract() + + def stop(self): + # Shut down pakfire instance. + self.pakfire.destroy() + + @property + def config(self): + """ + Proxy method for easy access to the configuration. + """ + return self.pakfire.config + + @property + def distro(self): + """ + Proxy method for easy access to the distribution. + """ + return self.pakfire.distro + + @property + def path(self): + """ + Proxy method for easy access to the path. + """ + return self.pakfire.path - #if not os.path.exists(file_in): - # return + @property + def arch(self): + """ + Inherit architecture from distribution configuration. + """ + return self.pakfire.distro.arch - dir_out = os.path.dirname(file_out) - if not os.path.exists(dir_out): - os.makedirs(dir_out) + @property + def personality(self): + """ + Gets the personality from the distribution configuration. + """ + return self.pakfire.distro.personality - self.log.debug("%s --> %s" % (file_in, file_out)) + @property + def info(self): + return { + "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)), + "build_host" : socket.gethostname(), + "build_id" : self.build_id, + "build_time" : self.build_time, + } - shutil.copy2(file_in, file_out) + @property + def keyring(self): + """ + Shortcut to access the pakfire keyring. + """ + return self.pakfire.keyring def copy_result(self, resultdir): # XXX should use find_result_packages @@ -450,84 +753,6 @@ class BuildEnviron(object): # Install everything. self.pakfire.install(requires, **kwargs) - def chrootPath(self, *args): - # Remove all leading slashes - _args = [] - for arg in args: - if arg.startswith("/"): - arg = arg[1:] - _args.append(arg) - args = _args - - ret = os.path.join(self.path, *args) - ret = ret.replace("//", "/") - - assert ret.startswith(self.path) - - return ret - - def populate_dev(self): - nodes = [ - "/dev/null", - "/dev/zero", - "/dev/full", - "/dev/random", - "/dev/urandom", - "/dev/tty", - "/dev/ptmx", - "/dev/kmsg", - "/dev/rtc0", - "/dev/console", - ] - - # If we need loop devices (which are optional) we create them here. - if self.settings["enable_loop_devices"]: - for i in range(0, 7): - nodes.append("/dev/loop%d" % i) - - for node in nodes: - # Stat the original node of the host system and copy it to - # the build chroot. - try: - node_stat = os.stat(node) - - # If it cannot be found, just go on. - except OSError: - continue - - self._create_node(node, node_stat.st_mode, node_stat.st_rdev) - - os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin")) - os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout")) - os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr")) - os.symlink("/proc/self/fd", self.chrootPath("dev", "fd")) - - def setup_dns(self): - """ - Add DNS resolution facility to chroot environment by copying - /etc/resolv.conf and /etc/hosts. - """ - for i in ("/etc/resolv.conf", "/etc/hosts"): - self.copyin(i, i) - - def _create_node(self, filename, mode, device): - self.log.debug("Create node: %s (%s)" % (filename, mode)) - - filename = self.chrootPath(filename) - - # Create parent directory if it is missing. - dirname = os.path.dirname(filename) - if not os.path.exists(dirname): - os.makedirs(dirname) - - os.mknod(filename, mode, device) - - def destroy(self): - self.log.debug("Destroying environment %s" % self.path) - - if os.path.exists(self.path): - util.rm(self.path) - def cleanup(self): self.log.debug("Cleaning environemnt.") @@ -541,120 +766,6 @@ class BuildEnviron(object): util.rm(d) os.makedirs(d) - def _mountall(self): - self.log.debug("Mounting environment") - for src, dest, fs, options in self.mountpoints: - mountpoint = self.chrootPath(dest) - if options: - options = "-o %s" % options - - # Eventually create mountpoint directory - if not os.path.exists(mountpoint): - os.makedirs(mountpoint) - - self.execute_root("mount -n -t %s %s %s %s" % (fs, options, src, mountpoint), shell=True) - - def _umountall(self): - self.log.debug("Umounting environment") - - mountpoints = [] - for src, dest, fs, options in reversed(self.mountpoints): - dest = self.chrootPath(dest) - - if not dest in mountpoints: - mountpoints.append(dest) - - while mountpoints: - for mp in mountpoints: - try: - self.execute_root("umount -n %s" % mp, shell=True) - except ShellEnvironmentError: - pass - - if not os.path.ismount(mp): - mountpoints.remove(mp) - - @property - def mountpoints(self): - mountpoints = [] - - # Make root as a tmpfs if enabled. - if self.settings.get("buildroot_tmpfs"): - mountpoints += [ - ("pakfire_root", "/", "tmpfs", "defaults"), - ] - - mountpoints += [ - # src, dest, fs, options - ("pakfire_proc", "/proc", "proc", "nosuid,noexec,nodev"), - ("/proc/sys", "/proc/sys", "bind", "bind"), - ("/proc/sys", "/proc/sys", "bind", "bind,ro,remount"), - ("/sys", "/sys", "bind", "bind"), - ("/sys", "/sys", "bind", "bind,ro,remount"), - ("pakfire_tmpfs", "/dev", "tmpfs", "mode=755,nosuid"), - ("/dev/pts", "/dev/pts", "bind", "bind"), - ("pakfire_tmpfs", "/run", "tmpfs", "mode=755,nosuid,nodev"), - ("pakfire_tmpfs", "/tmp", "tmpfs", "mode=755,nosuid,nodev"), - ] - - # If selinux is enabled. - if os.path.exists("/sys/fs/selinux"): - mountpoints += [ - ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"), - ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"), - ] - - # If ccache support is requested, we bind mount the cache. - if self.settings.get("enable_ccache"): - # Create ccache cache directory if it does not exist. - if not os.path.exists(CCACHE_CACHE_DIR): - os.makedirs(CCACHE_CACHE_DIR) - - mountpoints += [ - (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"), - ] - - return mountpoints - - @property - def environ(self): - env = MINIMAL_ENVIRONMENT.copy() - env.update({ - # Add HOME manually, because it is occasionally not set - # and some builds get in trouble then. - "TERM" : os.environ.get("TERM", "vt100"), - - # Sanitize language. - "LANG" : os.environ.setdefault("LANG", "en_US.UTF-8"), - - # Set the container that we can detect, if we are inside a - # chroot. - "container" : "pakfire-builder", - }) - - # Inherit environment from distro - env.update(self.pakfire.distro.environ) - - # ccache environment settings - if self.settings.get("enable_ccache", False): - compress = self.settings.get("ccache_compress", False) - if compress: - env["CCACHE_COMPRESS"] = "1" - - # Let ccache create its temporary files in /tmp. - env["CCACHE_TEMPDIR"] = "/tmp" - - # Fake UTS_MACHINE, when we cannot use the personality syscall and - # if the host architecture is not equal to the target architecture. - if not self.pakfire.distro.personality and \ - not system.native_arch == self.pakfire.distro.arch: - env.update({ - "LD_PRELOAD" : "/usr/lib/libpakfire_preload.so", - "UTS_MACHINE" : self.pakfire.distro.arch, - }) - - return env - @property def installed_packages(self): """ @@ -734,15 +845,6 @@ class BuildEnviron(object): return shellenv - def execute_root(self, command, **kwargs): - """ - Executes the given command outside the build chroot. - """ - shellenv = shell.ShellExecuteEnvironment(command, **kwargs) - shellenv.execute() - - return shellenv - def build(self, install_test=True, prepare=False): if not self.pkg: raise BuildError(_("You cannot run a build when no package was given.")) @@ -816,32 +918,6 @@ class BuildEnviron(object): self.log.info(_("Installation test succeeded.")) self.log.info("") - def shell(self, args=[]): - if not util.cli_is_interactive(): - self.log.warning("Cannot run shell on non-interactive console.") - return - - # Install all packages that are needed to run a shell. - self.install(SHELL_PACKAGES) - - # XXX need to set CFLAGS here - command = "/usr/sbin/chroot %s %s %s" % \ - (self.chrootPath(), SHELL_SCRIPT, " ".join(args)) - - # Add personality if we require one - if self.pakfire.distro.personality: - command = "%s %s" % (self.pakfire.distro.personality, command) - - for key, val in list(self.environ.items()): - command = "%s=\"%s\" " % (key, val) + command - - # Empty the environment - command = "env -i - %s" % command - - self.log.debug("Shell command: %s" % command) - - shell = os.system(command) - return os.WEXITSTATUS(shell) def sign_packages(self, keyfp=None): # Do nothing if signing is not requested. @@ -899,7 +975,7 @@ class BuildEnviron(object): self.log.info("") # Empty line. -class Builder(object): +class BuilderInternal(object): def __init__(self, pakfire, filename, resultdir, **kwargs): self.pakfire = pakfire diff --git a/src/pakfire/cli.py b/src/pakfire/cli.py index 5ecdd0cb8..b2108a0a6 100644 --- a/src/pakfire/cli.py +++ b/src/pakfire/cli.py @@ -27,7 +27,9 @@ import sys import tempfile import time +from . import arch from . import base +from . import builder from . import client from . import config from . import daemon @@ -434,67 +436,23 @@ class CliBuilder(Cli): return parser.parse_args() - def handle_build(self): - # Get the package descriptor from the command line options - pkg = self.args.package[0] - - # Check, if we got a regular file - if os.path.exists(pkg): - pkg = os.path.abspath(pkg) - - else: - raise FileNotFoundError(pkg) - - # Build argument list. - kwargs = { - "after_shell" : self.args.after_shell, - # Check whether to enable the install test. - "install_test" : not self.args.no_install_test, - "result_dir" : [self.args.resultdir,], - "shell" : True, - } + def builder(self, ns): + a = arch.Arch(ns.arch or system.native_arch) - if self.args.mode == "release": - kwargs["release_build"] = True - else: - kwargs["release_build"] = False - - if self.args.private_network: - kwargs["private_network"] = True - - p = self.create_pakfire() - p.build(pkg, **kwargs) + b = builder.Builder(arch=a) - def handle_shell(self): - pkg = None + return b - # Get the package descriptor from the command line options - if self.args.package: - pkg = self.args.package - - # Check, if we got a regular file - if os.path.exists(pkg): - pkg = os.path.abspath(pkg) - - else: - raise FileNotFoundError(pkg) - - if self.args.mode == "release": - release_build = True - else: - release_build = False - - p = self.create_pakfire() - - kwargs = { - "release_build" : release_build, - } + def handle_build(self, ns): + package, = ns.package - # Private network - if self.args.private_network: - kwargs["private_network"] = True + # Initialise a builder instance and build this package + with self.builder(ns) as b: + b.build(package) - p.shell(pkg, **kwargs) + def handle_shell(self, ns): + with self.builder(ns) as b: + b.shell() def handle_dist(self, ns): # Get the packages from the command line options diff --git a/src/pakfire/constants.py b/src/pakfire/constants.py index e7d90ad29..c9d22ce66 100644 --- a/src/pakfire/constants.py +++ b/src/pakfire/constants.py @@ -80,11 +80,6 @@ DATABASE_FORMATS_SUPPORTED = [0, 1, 2, 3, 4, 5, 6, 7] PACKAGE_FILENAME_FMT = "%(name)s-%(version)s-%(release)s.%(arch)s.%(ext)s" -BUILD_PACKAGES = [ - "@Build", - "pakfire-build>=%s" % PAKFIRE_LEAST_COMPATIBLE_VERSION, -] - # A script that is called, when a user is dropped to a chroot shell. SHELL_SCRIPT = "/usr/lib/pakfire/chroot-shell" SHELL_PACKAGES = ["elinks", "less", "vim", SHELL_SCRIPT,]