#!/usr/bin/python
+###############################################################################
+# #
+# Pakfire - The IPFire package management system #
+# Copyright (C) 2011 Pakfire development team #
+# #
+# This program is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# This program is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with this program. If not, see <http://www.gnu.org/licenses/>. #
+# #
+###############################################################################
import fcntl
import grp
import logging
+import math
import os
import re
import shutil
+import socket
import stat
import time
+import uuid
-import depsolve
+import base
+import chroot
+import logger
import packages
-import transaction
+import packages.packager
+import repository
import util
from constants import *
-from errors import BuildRootLocked
+from i18n import _
+from errors import BuildError, BuildRootLocked, Error
-class Builder(object):
+BUILD_LOG_HEADER = """
+ ____ _ __ _ _ _ _ _
+| _ \ __ _| | __/ _(_)_ __ ___ | |__ _ _(_) | __| | ___ _ __
+| |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
+| __/ (_| | <| _| | | | __/ | |_) | |_| | | | (_| | __/ |
+|_| \__,_|_|\_\_| |_|_| \___| |_.__/ \__,_|_|_|\__,_|\___|_|
+
+ Time : %(time)s
+ Host : %(host)s
+ Version : %(version)s
+
+"""
+
+class BuildEnviron(object):
# The version of the kernel this machine is running.
kernel_version = os.uname()[2]
- def __init__(self, pakfire, pkg):
- self.pakfire = pakfire
- self.pkg = pkg
-
+ def __init__(self, pkg=None, distro_config=None, build_id=None, logfile=None,
+ builder_mode="release", **pakfire_args):
+ # Set mode.
+ assert builder_mode in ("development", "release",)
+ self.mode = builder_mode
+
+ # Disable the build repository in release mode.
+ if self.mode == "release":
+ if pakfire_args.has_key("disable_repos") and pakfire_args["disable_repos"]:
+ pakfire_args["disable_repos"] += ["build",]
+ else:
+ pakfire_args["disable_repos"] = ["build",]
+
+ # Save the build id and generate one if no build id was provided.
+ if not build_id:
+ build_id = "%s" % uuid.uuid4()
+
+ self.build_id = build_id
+
+ # Setup the logging.
+ if logfile:
+ self.log = logging.getLogger(self.build_id)
+ # Propage everything to the root logger that we will see something
+ # on the terminal.
+ self.log.propagate = 1
+ self.log.setLevel(logging.INFO)
+
+ # Add the given logfile to the logger.
+ h = logging.FileHandler(logfile)
+ self.log.addHandler(h)
+
+ # Format the log output for the file.
+ f = logger.BuildFormatter()
+ h.setFormatter(f)
+ else:
+ # If no logile was given, we use the root logger.
+ self.log = logging.getLogger()
+
+ # Log information about pakfire and some more information, when we
+ # are running in release mode.
+ if self.mode == "release":
+ logdata = {
+ "host" : socket.gethostname(),
+ "time" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
+ "version" : "Pakfire %s" % PAKFIRE_VERSION,
+ }
+
+ for line in BUILD_LOG_HEADER.splitlines():
+ self.log.info(line % logdata)
+
+ # Create pakfire instance.
+ if pakfire_args.has_key("mode"):
+ del pakfire_args["mode"]
+ self.pakfire = base.Pakfire(mode="builder", distro_config=distro_config, **pakfire_args)
+ self.distro = self.pakfire.distro
+ self.path = self.pakfire.path
+
+ # Log the package information.
+ self.pkg = packages.Makefile(self.pakfire, pkg)
+ self.log.info(_("Package information:"))
+ for line in self.pkg.dump(long=True).splitlines():
+ self.log.info(" %s" % line)
+ self.log.info("")
+
+ # XXX need to make this configureable
self.settings = {
"enable_loop_devices" : True,
+ "enable_ccache" : True,
+ "enable_icecream" : False,
}
+ #self.settings.update(settings)
self.buildroot = "/buildroot"
self._lock = None
self.lock()
- # Initialize the environment
- self.prepare()
+ # Save the build time.
+ self.build_time = int(time.time())
@property
- def path(self):
- return self.pakfire.path
+ def arch(self):
+ """
+ Inherit architecture from distribution configuration.
+ """
+ return self.distro.arch
+
+ @property
+ def info(self):
+ return {
+ "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
+ "build_host" : socket.gethostname(),
+ "build_id" : self.build_id,
+ "build_time" : self.build_time,
+ }
def lock(self):
filename = os.path.join(self.path, ".lock")
self.copyout(file_in, file_out)
- def extract(self, requires=[], build_deps=True):
+ def get_pwuid(self, uid):
+ users = {}
+
+ f = open(self.chrootPath("/etc/passwd"))
+ for line in f.readlines():
+ m = re.match(r"^([a-z][a-z0-9_\-]{,30}):x:(\d+):(\d+):(.*):(.*)$", line)
+ if not m:
+ continue
+
+ item = {
+ "name" : m.group(1),
+ "uid" : int(m.group(2)),
+ "gid" : int(m.group(3)),
+ "home" : m.group(4),
+ "shell" : m.group(5),
+ }
+
+ assert not users.has_key(item["uid"])
+ users[item["uid"]] = item
+
+ f.close()
+
+ return users.get(uid, None)
+
+ def get_grgid(self, gid):
+ groups = {}
+
+ f = open(self.chrootPath("/etc/group"))
+ for line in f.readlines():
+ m = re.match(r"^([a-z][a-z0-9_\-]{,30}):x:(\d+):(.*)$", line)
+ if not m:
+ continue
+
+ item = {
+ "name" : m.group(1),
+ "gid" : int(m.group(2)),
+ }
+
+ # XXX re-enable later
+ #assert not groups.has_key(item["gid"])
+ groups[item["gid"]] = item
+
+ f.close()
+
+ return groups.get(gid, None)
+
+ def extract(self, requires=None, build_deps=True):
"""
Gets a dependency set and extracts all packages
to the environment.
"""
- ds = depsolve.DependencySet(self.pakfire)
- for p in BUILD_PACKAGES + requires:
- ds.add_requires(p)
- ds.resolve()
+ if not requires:
+ requires = []
+
+ # Add neccessary build dependencies.
+ requires += BUILD_PACKAGES
+
+ # If we have ccache enabled, we need to extract it
+ # to the build chroot.
+ if self.settings.get("enable_ccache"):
+ requires.append("ccache")
+
+ # If we have icecream enabled, we need to extract it
+ # to the build chroot.
+ if self.settings.get("enable_icecream"):
+ requires.append("icecream")
# Get build dependencies from source package.
- if isinstance(self.pkg, packages.SourcePackage):
- for req in self.pkg.requires:
- ds.add_requires(req)
+ for req in self.pkg.requires:
+ requires.append(req)
- ts = transaction.TransactionSet(self.pakfire, ds)
- ts.dump()
- ts.run()
+ # Install all packages.
+ self.install(requires)
# Copy the makefile and load source tarballs.
- if isinstance(self.pkg, packages.Makefile):
- self.pkg.extract(self)
+ self.pkg.extract(_("Extracting"),
+ prefix=os.path.join(self.path, "build"))
- # If we have a makefile, we can only get the build dependencies
- # after we have extracted all the rest.
- if build_deps and isinstance(self.pkg, packages.Makefile):
- requires = self.make_requires()
- if not requires:
- return
+ def install(self, requires):
+ """
+ Install everything that is required in requires.
+ """
+ # If we got nothing to do, we quit immediately.
+ if not requires:
+ return
- ds = depsolve.DependencySet(self.pakfire)
- for r in requires:
- ds.add_requires(r)
- ds.resolve()
+ self.pakfire.install(requires, interactive=False,
+ allow_downgrade=True, logger=self.log)
- ts = transaction.TransactionSet(self.pakfire, ds)
- ts.dump()
- ts.run()
+ def install_test(self):
+ pkgs = []
+ for dir, subdirs, files in os.walk(self.chrootPath("result")):
+ for file in files:
+ pkgs.append(os.path.join(dir, file))
- @property
- def log(self):
- # XXX for now, return the root logger
- return logging.getLogger()
+ self.pakfire.localinstall(pkgs, yes=True)
def chrootPath(self, *args):
# Remove all leading slashes
return ret
def prepare(self):
+ prepared_tag = ".prepared"
+
+ if os.path.exists(self.chrootPath(prepared_tag)):
+ return
+
# Create directory.
if not os.path.exists(self.path):
os.makedirs(self.path)
"tmp",
"usr/src",
]
+
+ # Create cache dir if ccache is enabled.
+ if self.settings.get("enable_ccache"):
+ dirs.append("var/cache/ccache")
+
+ if not os.path.exists(CCACHE_CACHE_DIR):
+ os.makedirs(CCACHE_CACHE_DIR)
+
for dir in dirs:
dir = self.chrootPath(dir)
if not os.path.exists(dir):
os.makedirs(dir)
+ # Create neccessary files like /etc/fstab and /etc/mtab.
+ files = (
+ "etc/fstab",
+ "etc/mtab",
+ prepared_tag,
+ )
+
+ for file in files:
+ file = self.chrootPath(file)
+ dir = os.path.dirname(file)
+ if not os.path.exists(dir):
+ os.makedirs(dir)
+ f = open(file, "w")
+ f.close()
+
self._prepare_dev()
- self._prepare_users()
self._prepare_dns()
def _prepare_dev(self):
os.umask(prevMask)
- def _prepare_users(self):
- f = open(self.chrootPath("etc", "passwd"), "w")
- f.write("root:x:0:0:root:/root:/bin/bash\n")
- f.write("nobody:x:99:99:Nobody:/:/sbin/nologin\n")
- f.close()
-
- f = open(self.chrootPath("etc", "group"), "w")
- f.write("root:x:0:root\n")
- f.write("nobody:x:99:\n")
- f.close()
-
def _prepare_dns(self):
"""
Add DNS resolution facility to chroot environment by copying
os.mknod(filename, mode, device)
- def cleanup(self):
- logging.debug("Cleanup environment %s" % self.path)
+ def destroy(self):
+ util.orphans_kill(self.path)
+
+ logging.debug("Destroying environment %s" % self.path)
if os.path.exists(self.path):
util.rm(self.path)
+ def cleanup(self):
+ logging.debug("Cleaning environemnt.")
+
+ # Remove the build directory and buildroot.
+ dirs = ("build", self.buildroot, "result")
+
+ for d in dirs:
+ d = self.chrootPath(d)
+ if not os.path.exists(d):
+ continue
+
+ util.rm(d)
+ os.makedirs(d)
+
def _mountall(self):
self.log.debug("Mounting environment")
for cmd, mountpoint in self.mountpoints:
cmd = "%s %s" % (cmd, self.chrootPath(mountpoint))
- util.do(cmd, shell=True)
+ chroot.do(cmd, shell=True)
def _umountall(self):
self.log.debug("Umounting environment")
for cmd, mountpoint in self.mountpoints:
cmd = "umount -n %s" % self.chrootPath(mountpoint)
- util.do(cmd, raiseExc=0, shell=True)
+ chroot.do(cmd, raiseExc=0, shell=True)
@property
def mountpoints(self):
("mount -n -t tmpfs pakfire_chroot_shmfs", "dev/shm"),
])
+ if self.settings.get("enable_ccache"):
+ ret.append(("mount -n --bind %s" % CCACHE_CACHE_DIR, "var/cache/ccache"))
+
return ret
@property
def environ(self):
env = {
+ # Add HOME manually, because it is occasionally not set
+ # and some builds get in trouble then.
+ "HOME" : "/root",
+ "TERM" : os.environ.get("TERM", "dumb"),
+ "PS1" : "\u:\w\$ ",
+
"BUILDROOT" : self.buildroot,
+ "PARALLELISMFLAGS" : "-j%s" % util.calc_parallelism(),
}
# Inherit environment from distro
env.update(self.pakfire.distro.environ)
+ # Icecream environment settings
+ if self.settings.get("enable_icecream", False):
+ # Set the toolchain path
+ if self.settings.get("icecream_toolchain", None):
+ env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
+
+ # Set preferred host if configured.
+ if self.settings.get("icecream_preferred_host", None):
+ env["ICECC_PREFERRED_HOST"] = \
+ self.settings.get("icecream_preferred_host")
+
# XXX what do we need else?
return env
- def do(self, command, shell=True, personality=None, *args, **kwargs):
+ def do(self, command, shell=True, personality=None, logger=None, *args, **kwargs):
ret = None
try:
# Environment variables
if kwargs.has_key("env"):
env.update(kwargs.pop("env"))
+ logging.debug("Environment:")
+ for k, v in sorted(env.items()):
+ logging.debug(" %s=%s" % (k, v))
+
# Update personality it none was set
if not personality:
- personality = self.pakfire.distro.personality
+ personality = self.distro.personality
+
+ # Make every shell to a login shell because we set a lot of
+ # environment things there.
+ if shell:
+ command = ["bash", "--login", "-c", command]
self._mountall()
if not kwargs.has_key("chrootPath"):
kwargs["chrootPath"] = self.chrootPath()
- ret = util.do(
+ ret = chroot.do(
command,
personality=personality,
- shell=shell,
+ shell=False,
env=env,
- logger=self.log,
+ logger=logger,
*args,
**kwargs
)
return ret
- def make(self, *args, **kwargs):
- command = ["bash", "--login", "-c",]
- command.append("make -f /build/%s %s" % \
- (os.path.basename(self.pkg.filename), " ".join(args)))
+ def build(self, install_test=True):
+ assert self.pkg
- return self.do(command, shell=False, **kwargs)
+ pkgfile = os.path.join("/build", os.path.basename(self.pkg.filename))
+ resultdir = self.chrootPath("/result")
- @property
- def make_info(self):
- if not hasattr(self, "_make_info"):
- info = {}
+ # Create the build command, that is executed in the chroot.
+ build_command = ["pakfire-build2", "--offline", "build", pkgfile,
+ "--nodeps", "--resultdir=/result",]
- output = self.make("buildinfo", returnOutput=True)
+ try:
+ self.do(" ".join(build_command), logger=self.log)
- for line in output.splitlines():
- # XXX temporarily
- if not line:
- break
+ except Error:
+ raise BuildError, _("The build command failed. See logfile for details.")
- m = re.match(r"^(\w+)=(.*)$", line)
- if not m:
- continue
+ # Perform install test.
+ if install_test:
+ self.install_test()
- info[m.group(1)] = m.group(2).strip("\"")
+ # Copy the final packages and stuff.
+ # XXX TODO resultdir
- self._make_info = info
+ def shell(self, args=[]):
+ if not util.cli_is_interactive():
+ logging.warning("Cannot run shell on non-interactive console.")
+ return
- return self._make_info
+ # Install all packages that are needed to run a shell.
+ self.install(SHELL_PACKAGES)
- @property
- def packages(self):
- if hasattr(self, "_packages"):
- return self._packages
+ # XXX need to set CFLAGS here
+ command = "/usr/sbin/chroot %s /usr/bin/chroot-shell %s" % \
+ (self.chrootPath(), " ".join(args))
- pkgs = []
- output = self.make("packageinfo", returnOutput=True)
+ # Add personality if we require one
+ if self.pakfire.distro.personality:
+ command = "%s %s" % (self.pakfire.distro.personality, command)
- pkg = {}
- for line in output.splitlines():
- if not line:
- pkgs.append(pkg)
- pkg = {}
+ for key, val in self.environ.items():
+ command = "%s=\"%s\" " % (key, val) + command
- m = re.match(r"^(\w+)=(.*)$", line)
- if not m:
- continue
+ # Empty the environment
+ command = "env -i - %s" % command
- k, v = m.groups()
- pkg[k] = v.strip("\"")
+ logging.debug("Shell command: %s" % command)
- self._packages = []
- for pkg in pkgs:
- pkg = packages.VirtualPackage(pkg)
- self._packages.append(pkg)
+ try:
+ self._mountall()
- return self._packages
+ shell = os.system(command)
+ return os.WEXITSTATUS(shell)
- def make_requires(self):
- return self.make_info.get("PKG_BUILD_DEPS", "").split()
+ finally:
+ self._umountall()
- def make_sources(self):
- return self.make_info.get("PKG_FILES", "").split()
+# XXX backwards compatibilty
+Builder = BuildEnviron
- def build(self):
- self.make("build")
+class Builder2(object):
+ def __init__(self, pakfire, filename, resultdir, **kwargs):
+ self.pakfire = pakfire
- for pkg in reversed(self.packages):
- packager = packages.Packager(self.pakfire, pkg, self)
- packager()
+ self.filename = filename
- def dist(self):
- self.pkg.dist(self)
+ self.resultdir = resultdir
- def shell(self, args=[]):
- # XXX need to add linux32 or linux64 to the command line
- # XXX need to set CFLAGS here
- command = "chroot %s /usr/bin/chroot-shell %s" % \
- (self.chrootPath(), " ".join(args))
+ # Open package file.
+ self.pkg = packages.Makefile(self.pakfire, self.filename)
- for key, val in self.environ.items():
- command = "%s=\"%s\" " % (key, val) + command
-
- # Add personality if we require one
- if self.pakfire.distro.personality:
- command = "%s %s" % (self.pakfire.disto.personality, command)
+ #self.buildroot = "/tmp/pakfire_buildroot/%s" % util.random_string(20)
+ self.buildroot = "/buildroot"
- # Empty the environment
- #command = "env -i - %s" % command
+ self._environ = {
+ "BUILDROOT" : self.buildroot,
+ "LANG" : "C",
+ }
- logging.debug("Shell command: %s" % command)
+ @property
+ def distro(self):
+ return self.pakfire.distro
+ @property
+ def environ(self):
+ environ = os.environ
+ environ.update(self._environ)
+
+ return environ
+
+ def do(self, command, shell=True, personality=None, cwd=None, *args, **kwargs):
+ # Environment variables
+ logging.debug("Environment:")
+ for k, v in sorted(self.environ.items()):
+ logging.debug(" %s=%s" % (k, v))
+
+ # Update personality it none was set
+ if not personality:
+ personality = self.distro.personality
+
+ if not cwd:
+ cwd = "/%s" % LOCAL_TMP_PATH
+
+ # Make every shell to a login shell because we set a lot of
+ # environment things there.
+ if shell:
+ command = ["bash", "--login", "-c", command]
+
+ return chroot.do(
+ command,
+ personality=personality,
+ shell=False,
+ env=self.environ,
+ logger=logging.getLogger(),
+ cwd=cwd,
+ *args,
+ **kwargs
+ )
+
+ def create_icecream_toolchain(self):
try:
- self._mountall()
+ out = self.do("icecc --build-native", returnOutput=True)
+ except Error:
+ return
+
+ for line in out.splitlines():
+ m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
+ if m:
+ self._environ["icecream_toolchain"] = "/%s" % m.group(1)
+
+ def create_buildscript(self, stage):
+ file = "/tmp/build_%s" % util.random_string()
+
+ # Get buildscript from the package.
+ script = self.pkg.get_buildscript(stage)
+
+ # Write script to an empty file.
+ f = open(file, "w")
+ f.write("#!/bin/sh\n\n")
+ f.write("set -e\n")
+ f.write("set -x\n")
+ f.write("\n%s\n" % script)
+ f.write("exit 0\n")
+ f.close()
+ os.chmod(file, 700)
- shell = os.system(command)
- return os.WEXITSTATUS(shell)
+ return file
+
+ def build(self):
+ # Create buildroot.
+ if not os.path.exists(self.buildroot):
+ os.makedirs(self.buildroot)
+
+ # Build icecream toolchain if icecream is installed.
+ self.create_icecream_toolchain()
+
+ for stage in ("prepare", "build", "test", "install"):
+ self.build_stage(stage)
+
+ # Package the result.
+ # Make all these little package from the build environment.
+ logging.info(_("Creating packages:"))
+ for pkg in reversed(self.pkg.packages):
+ packager = packages.packager.BinaryPackager(self.pakfire, pkg, self.buildroot)
+ packager.run(self.resultdir)
+ logging.info("")
+
+ def build_stage(self, stage):
+ # Get the buildscript for this stage.
+ buildscript = self.create_buildscript(stage)
+
+ # Execute the buildscript of this stage.
+ logging.info(_("Running stage %s:") % stage)
+
+ try:
+ self.do(buildscript, shell=False)
finally:
- self._umountall()
+ # Remove the buildscript.
+ if os.path.exists(buildscript):
+ os.unlink(buildscript)
+
+ def cleanup(self):
+ if os.path.exists(self.buildroot):
+ util.rm(self.buildroot)
projects / pakfire.git / blobdiff