[people/stevee/pakfire.git] / pakfire / builder.py

#!/usr/bin/python

import fcntl
import grp
import logging
import math
import os
import re
import shutil
import socket
import stat
import time
import uuid

import base
import chroot
import depsolve
import packages
import repository
import transaction
import util

from constants import *
from i18n import _
from errors import BuildError, BuildRootLocked, Error


class Builder(object):
	# The version of the kernel this machine is running.
	kernel_version = os.uname()[2]

	def __init__(self, pkg=None, distro_config=None, build_id=None, **pakfire_args):
		pakfire_args.update({
			"builder" : True,
		})

		# Create pakfire instance.
		self.pakfire = base.Pakfire(distro_config=distro_config, **pakfire_args)
		self.distro = self.pakfire.distro
		self.path = self.pakfire.path

		# Open the package.
		self.pkg = pkg

		# XXX need to make this configureable
		self.settings = {
			"enable_loop_devices" : True,
			"enable_ccache"   : True,
			"enable_icecream" : False,
		}
		#self.settings.update(settings)

		self.buildroot = "/buildroot"

		# Lock the buildroot
		self._lock = None
		self.lock()

		# Save the build time.
		self.build_time = int(time.time())

		# Save the build id and generate one if no build id was provided.
		if not build_id:
			build_id = uuid.uuid4()

		self.build_id = build_id

	def get_pkg(self):
		return getattr(self, "_pkg", None)

	def set_pkg(self, pkg):
		if pkg is None:
			self.__pkg = None
			return

		self._pkg = packages.open(self.pakfire, None, pkg)

		# Log the package information.
		if not isinstance(self._pkg, packages.Makefile):
			dump = self._pkg.dump(long=True)
			self.log.info(dump)

		assert self.pkg

	pkg = property(get_pkg, set_pkg)

	@property
	def arch(self):
		"""
			Inherit architecture from distribution configuration.
		"""
		return self.distro.arch

	@property
	def info(self):
		return {
			"build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
			"build_host" : socket.gethostname(),
			"build_id"   : self.build_id,
			"build_time" : self.build_time,
		}

	def lock(self):
		filename = os.path.join(self.path, ".lock")

		try:
			self._lock = open(filename, "a+")
		except IOError, e:
			return 0

		try:
			fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
		except IOError, e:
			raise BuildRootLocked, "Buildroot is locked"

		return 1

	def unlock(self):
		if self._lock:
			self._lock.close()
			self._lock = None

	def copyin(self, file_out, file_in):
		if file_in.startswith("/"):
			file_in = file_in[1:]

		file_in = self.chrootPath(file_in)

		#if not os.path.exists(file_out):
		#	return

		dir_in = os.path.dirname(file_in)
		if not os.path.exists(dir_in):
			os.makedirs(dir_in)

		logging.debug("%s --> %s" % (file_out, file_in))

		shutil.copy2(file_out, file_in)

	def copyout(self, file_in, file_out):
		if file_in.startswith("/"):
			file_in = file_in[1:]

		file_in = self.chrootPath(file_in)

		#if not os.path.exists(file_in):
		#	return

		dir_out = os.path.dirname(file_out)
		if not os.path.exists(dir_out):
			os.makedirs(dir_out)

		logging.debug("%s --> %s" % (file_in, file_out))

		shutil.copy2(file_in, file_out)

	def copy_result(self, resultdir):
		dir_in = self.chrootPath("result")

		for dir, subdirs, files in os.walk(dir_in):
			basename = os.path.basename(dir)
			dir = dir[len(self.chrootPath()):]
			for file in files:
				file_in = os.path.join(dir, file)

				file_out = os.path.join(
					resultdir,
					basename,
					file,
				)

				self.copyout(file_in, file_out)

	def extract(self, requires=None, build_deps=True):
		"""
			Gets a dependency set and extracts all packages
			to the environment.
		"""
		if not requires:
			requires = []

		# Add neccessary build dependencies.
		requires += BUILD_PACKAGES

		# If we have ccache enabled, we need to extract it
		# to the build chroot.
		if self.settings.get("enable_ccache"):
			requires.append("ccache")

		# If we have icecream enabled, we need to extract it
		# to the build chroot.
		if self.settings.get("enable_icecream"):
			requires.append("icecream")

		# Get build dependencies from source package.
		if isinstance(self.pkg, packages.SourcePackage):
			for req in self.pkg.requires:
				requires.append(req)

		# Install all packages.
		self.install(requires)

		# Copy the makefile and load source tarballs.
		if isinstance(self.pkg, packages.Makefile):
			self.pkg.extract(self)

		elif isinstance(self.pkg, packages.SourcePackage):
			self.pkg.extract(_("Extracting: %s (source)") % self.pkg.name,
				prefix=os.path.join(self.path, "build"))

		# If we have a makefile, we can only get the build dependencies
		# after we have extracted all the rest.
		if build_deps and isinstance(self.pkg, packages.Makefile):
			requires = self.make_requires()
			if not requires:
				return

			self.install(requires)

	def install(self, requires):
		"""
			Install everything that is required in requires.
		"""
		# If we got nothing to do, we quit immediately.
		if not requires:
			return

		ds = depsolve.DependencySet(self.pakfire)
		for r in requires:
			if isinstance(r, packages.BinaryPackage):
				ds.add_package(r)
			else:
				ds.add_requires(r)
		ds.resolve()
		ds.dump()

		ts = transaction.Transaction(self.pakfire, ds)
		ts.run()

	def install_test(self):
		pkgs = []

		# Connect packages to the FS repository.
		r = repository.FileSystemRepository(self.pakfire)

		for dir, subdirs, files in os.walk(self.chrootPath("result")):
			for file in files:
				file = os.path.join(dir, file)

				if not file.endswith(PACKAGE_EXTENSION):
					continue

				p = packages.BinaryPackage(self.pakfire, r, file)
				pkgs.append(p)

		self.install(pkgs)

	@property
	def log(self):
		# XXX for now, return the root logger
		return logging.getLogger()

	def chrootPath(self, *args):
		# Remove all leading slashes
		_args = []
		for arg in args:
			if arg.startswith("/"):
				arg = arg[1:]
			_args.append(arg)
		args = _args

		ret = os.path.join(self.path, *args)
		ret = ret.replace("//", "/")

		assert ret.startswith(self.path)

		return ret

	def prepare(self):
		prepared_tag = ".prepared"

		if os.path.exists(self.chrootPath(prepared_tag)):
			return

		# Create directory.
		if not os.path.exists(self.path):
			os.makedirs(self.path)

		# Create important directories.
		dirs = [
			"build",
			self.buildroot,
			"dev",
			"dev/pts",
			"dev/shm",
			"etc",
			"proc",
			"result",
			"sys",
			"tmp",
			"usr/src",
		]

		# Create cache dir if ccache is enabled.
		if self.settings.get("enable_ccache"):
			dirs.append("var/cache/ccache")

			if not os.path.exists(CCACHE_CACHE_DIR):
				os.makedirs(CCACHE_CACHE_DIR)

		for dir in dirs:
			dir = self.chrootPath(dir)
			if not os.path.exists(dir):
				os.makedirs(dir)

		# Create neccessary files like /etc/fstab and /etc/mtab.
		files = (
			"etc/fstab",
			"etc/mtab",
			prepared_tag,
		)

		for file in files:
			file = self.chrootPath(file)
			dir = os.path.dirname(file)
			if not os.path.exists(dir):
				os.makedirs(dir)
			f = open(file, "w")
			f.close()

		self._prepare_dev()
		self._prepare_dns()

	def _prepare_dev(self):
		prevMask = os.umask(0000)

		nodes = [
			("dev/null",	stat.S_IFCHR | 0666, os.makedev(1, 3)),
			("dev/full",	stat.S_IFCHR | 0666, os.makedev(1, 7)),
			("dev/zero",	stat.S_IFCHR | 0666, os.makedev(1, 5)),
			("dev/random",	stat.S_IFCHR | 0666, os.makedev(1, 8)),
			("dev/urandom",	stat.S_IFCHR | 0444, os.makedev(1, 9)),
			("dev/tty",		stat.S_IFCHR | 0666, os.makedev(5, 0)),
			("dev/console",	stat.S_IFCHR | 0600, os.makedev(5, 1)),
		]

		# If we need loop devices (which are optional) we create them here.
		if self.settings["enable_loop_devices"]:
			for i in range(0, 7):
				nodes.append(("dev/loop%d" % i, stat.S_IFBLK | 0660, os.makedev(7, i)))

		# Create all the nodes.
		for node in nodes:
			self._create_node(*node)

		os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
		os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
		os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
		os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))

		# make device node for el4 and el5
		if self.kernel_version < "2.6.19":
			self._make_node("dev/ptmx", stat.S_IFCHR | 0666, os.makedev(5, 2))
		else:
			os.symlink("/dev/pts/ptmx", self.chrootPath("dev", "ptmx"))

		os.umask(prevMask)

	def _prepare_dns(self):
		"""
			Add DNS resolution facility to chroot environment by copying
			/etc/resolv.conf and /etc/hosts.
		"""
		for i in ("/etc/resolv.conf", "/etc/hosts"):
			self.copyin(i, i)

	def _create_node(self, filename, mode, device):
		logging.debug("Create node: %s (%s)" % (filename, mode))

		filename = self.chrootPath(filename)

		# Create parent directory if it is missing.
		dirname = os.path.dirname(filename)
		if not os.path.exists(dirname):
			os.makedirs(dirname)

		os.mknod(filename, mode, device)

	def destroy(self):
		logging.debug("Destroying environment %s" % self.path)

		if os.path.exists(self.path):
			util.rm(self.path)

	def cleanup(self):
		logging.debug("Cleaning environemnt.")

		# Run make clean and let it cleanup its stuff.
		self.make("clean")

		# Remove the build directory and buildroot.
		dirs = ("build", self.buildroot, "result")

		for d in dirs:
			d = self.chrootPath(d)
			if not os.path.exists(d):
				continue

			util.rm(d)
			os.makedirs(d)

		# Clear make_info cache.
		if hasattr(self, "_make_info"):
			del self._make_info

	def _mountall(self):
		self.log.debug("Mounting environment")
		for cmd, mountpoint in self.mountpoints:
			cmd = "%s %s" % (cmd, self.chrootPath(mountpoint))
			chroot.do(cmd, shell=True)

	def _umountall(self):
		self.log.debug("Umounting environment")
		for cmd, mountpoint in self.mountpoints:
			cmd = "umount -n %s" % self.chrootPath(mountpoint)
			chroot.do(cmd, raiseExc=0, shell=True)

	@property
	def mountpoints(self):
		ret = [
			("mount -n -t proc pakfire_chroot_proc", "proc"),
			("mount -n -t sysfs pakfire_chroot_sysfs", "sys"),
		]

		mountopt = "gid=%d,mode=0620,ptmxmode=0666" % grp.getgrnam("tty").gr_gid
		if self.kernel_version >= "2.6.29":
			mountopt += ",newinstance"

		ret.extend([
			("mount -n -t devpts -o %s pakfire_chroot_devpts" % mountopt, "dev/pts"),
			("mount -n -t tmpfs pakfire_chroot_shmfs", "dev/shm"),
		])

		if self.settings.get("enable_ccache"):
			ret.append(("mount -n --bind %s" % CCACHE_CACHE_DIR, "var/cache/ccache"))

		return ret

	@staticmethod
	def calc_parallelism():
		"""
			Calculate how many processes to run
			at the same time.

			We take the log10(number of processors) * factor
		"""
		num = os.sysconf("SC_NPROCESSORS_CONF")
		if num == 1:
			return 2
		else:
			return int(round(math.log10(num) * 26))

	@property
	def environ(self):
		env = {
			# Add HOME manually, because it is occasionally not set
			# and some builds get in trouble then.
			"HOME" : "/root",
			"TERM" : os.environ.get("TERM", "dumb"),
			"PS1"  : "\u:\w\$ ",

			"BUILDROOT" : self.buildroot,
			"PARALLELISMFLAGS" : "-j%s" % self.calc_parallelism(),
		}

		# Inherit environment from distro
		env.update(self.pakfire.distro.environ)

		# Icecream environment settings
		if self.settings.get("enable_icecream", None):
			# Set the toolchain path
			if self.settings.get("icecream_toolchain", None):
				env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")

			# Set preferred host if configured.
			if self.settings.get("icecream_preferred_host", None):
				env["ICECC_PREFERRED_HOST"] = \
					self.settings.get("icecream_preferred_host")

		# XXX what do we need else?

		return env

	def do(self, command, shell=True, personality=None, *args, **kwargs):
		ret = None
		try:
			# Environment variables
			env = self.environ

			if kwargs.has_key("env"):
				env.update(kwargs.pop("env"))

			logging.debug("Environment:")
			for k, v in sorted(env.items()):
				logging.debug("  %s=%s" % (k, v))

			# Update personality it none was set
			if not personality:
				personality = self.distro.personality

			# Make every shell to a login shell because we set a lot of
			# environment things there.
			if shell:
				command = ["bash", "--login", "-c", command]

			self._mountall()

			if not kwargs.has_key("chrootPath"):
				kwargs["chrootPath"] = self.chrootPath()

			ret = chroot.do(
				command,
				personality=personality,
				shell=False,
				env=env,
				logger=self.log,
				*args,
				**kwargs
			)

		finally:
			self._umountall()

		return ret

	def make(self, *args, **kwargs):
		if isinstance(self.pkg, packages.Makefile):
			filename = os.path.basename(self.pkg.filename)
		elif isinstance(self.pkg, packages.SourcePackage):
			filename = "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION)

		return self.do("make -f /build/%s %s" % (filename, " ".join(args)),
			**kwargs)

	@property
	def make_info(self):
		if not hasattr(self, "_make_info"):
			info = {}

			output = self.make("buildinfo", returnOutput=True)

			for line in output.splitlines():
				# XXX temporarily
				if not line:
					break

				m = re.match(r"^(\w+)=(.*)$", line)
				if not m:
					continue

				info[m.group(1)] = m.group(2).strip("\"")

			self._make_info = info

		return self._make_info

	@property
	def packages(self):
		if hasattr(self, "_packages"):
			return self._packages

		pkgs = []
		output = self.make("packageinfo", returnOutput=True)

		pkg = {}
		for line in output.splitlines():
			if not line:
				pkgs.append(pkg)
				pkg = {}

			m = re.match(r"^(\w+)=(.*)$", line)
			if not m:
				continue

			k, v = m.groups()
			pkg[k] = v.strip("\"")

		# Create a dummy repository to link the virtual packages to
		repo = repository.DummyRepository(self.pakfire)

		self._packages = []
		for pkg in pkgs:
			pkg = packages.VirtualPackage(self.pakfire, pkg) # XXX had to remove repo here?!
			self._packages.append(pkg)

		return self._packages

	def make_requires(self):
		return self.make_info.get("PKG_BUILD_DEPS", "").split()

	def make_sources(self):
		return self.make_info.get("PKG_FILES", "").split()

	def create_icecream_toolchain(self):
		if not self.settings.get("enable_icecream", None):
			return

		out = self.do("icecc --build-native", returnOutput=True)

		for line in out.splitlines():
			m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
			if m:
				self.settings["icecream_toolchain"] = "/%s" % m.group(1)

	def build(self):
		self.create_icecream_toolchain()

		try:
			self.make("build")

		except Error:
			raise BuildError, "The build command failed."

		for pkg in reversed(self.packages):
			packager = packages.BinaryPackager(self.pakfire, pkg, self)
			packager()

	def dist(self):
		self.pkg.dist(self)

	def shell(self, args=[]):
		if not util.cli_is_interactive():
			logging.warning("Cannot run shell on non-interactive console.")
			return

		# Install all packages that are needed to run a shell.
		self.install(SHELL_PACKAGES)

		# XXX need to set CFLAGS here
		command = "/usr/sbin/chroot %s /usr/bin/chroot-shell %s" % \
			(self.chrootPath(), " ".join(args))

		# Add personality if we require one
		if self.pakfire.distro.personality:
			command = "%s %s" % (self.pakfire.distro.personality, command)

		for key, val in self.environ.items():
			command = "%s=\"%s\" " % (key, val) + command

		# Empty the environment
		command = "env -i - %s" % command

		logging.debug("Shell command: %s" % command)

		try:
			self._mountall()

			shell = os.system(command)
			return os.WEXITSTATUS(shell)

		finally:
			self._umountall()
Commit	Line	Data
	1	#!/usr/bin/python
	2
	3	import fcntl
	4	import grp
	5	import logging
	6	import math
	7	import os
	8	import re
	9	import shutil
	10	import socket
	11	import stat
	12	import time
	13	import uuid
	14
	15	import base
	16	import chroot
	17	import depsolve
	18	import packages
	19	import repository
	20	import transaction
	21	import util
	22
	23	from constants import *
	24	from i18n import _
	25	from errors import BuildError, BuildRootLocked, Error
	26
	27
	28	class Builder(object):
	29	# The version of the kernel this machine is running.
	30	kernel_version = os.uname()[2]
	31
	32	def __init__(self, pkg=None, distro_config=None, build_id=None, **pakfire_args):
	33	pakfire_args.update({
	34	"builder" : True,
	35	})
	36
	37	# Create pakfire instance.
	38	self.pakfire = base.Pakfire(distro_config=distro_config, **pakfire_args)
	39	self.distro = self.pakfire.distro
	40	self.path = self.pakfire.path
	41
	42	# Open the package.
	43	self.pkg = pkg
	44
	45	# XXX need to make this configureable
	46	self.settings = {
	47	"enable_loop_devices" : True,
	48	"enable_ccache" : True,
	49	"enable_icecream" : False,
	50	}
	51	#self.settings.update(settings)
	52
	53	self.buildroot = "/buildroot"
	54
	55	# Lock the buildroot
	56	self._lock = None
	57	self.lock()
	58
	59	# Save the build time.
	60	self.build_time = int(time.time())
	61
	62	# Save the build id and generate one if no build id was provided.
	63	if not build_id:
	64	build_id = uuid.uuid4()
	65
	66	self.build_id = build_id
	67
	68	def get_pkg(self):
	69	return getattr(self, "_pkg", None)
	70
	71	def set_pkg(self, pkg):
	72	if pkg is None:
	73	self.__pkg = None
	74	return
	75
	76	self._pkg = packages.open(self.pakfire, None, pkg)
	77
	78	# Log the package information.
	79	if not isinstance(self._pkg, packages.Makefile):
	80	dump = self._pkg.dump(long=True)
	81	self.log.info(dump)
	82
	83	assert self.pkg
	84
	85	pkg = property(get_pkg, set_pkg)
	86
	87	@property
	88	def arch(self):
	89	"""
	90	Inherit architecture from distribution configuration.
	91	"""
	92	return self.distro.arch
	93
	94	@property
	95	def info(self):
	96	return {
	97	"build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
	98	"build_host" : socket.gethostname(),
	99	"build_id" : self.build_id,
	100	"build_time" : self.build_time,
	101	}
	102
	103	def lock(self):
	104	filename = os.path.join(self.path, ".lock")
	105
	106	try:
	107	self._lock = open(filename, "a+")
	108	except IOError, e:
	109	return 0
	110
	111	try:
	112	fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX \| fcntl.LOCK_NB)
	113	except IOError, e:
	114	raise BuildRootLocked, "Buildroot is locked"
	115
	116	return 1
	117
	118	def unlock(self):
	119	if self._lock:
	120	self._lock.close()
	121	self._lock = None
	122
	123	def copyin(self, file_out, file_in):
	124	if file_in.startswith("/"):
	125	file_in = file_in[1:]
	126
	127	file_in = self.chrootPath(file_in)
	128
	129	#if not os.path.exists(file_out):
	130	# return
	131
	132	dir_in = os.path.dirname(file_in)
	133	if not os.path.exists(dir_in):
	134	os.makedirs(dir_in)
	135
	136	logging.debug("%s --> %s" % (file_out, file_in))
	137
	138	shutil.copy2(file_out, file_in)
	139
	140	def copyout(self, file_in, file_out):
	141	if file_in.startswith("/"):
	142	file_in = file_in[1:]
	143
	144	file_in = self.chrootPath(file_in)
	145
	146	#if not os.path.exists(file_in):
	147	# return
	148
	149	dir_out = os.path.dirname(file_out)
	150	if not os.path.exists(dir_out):
	151	os.makedirs(dir_out)
	152
	153	logging.debug("%s --> %s" % (file_in, file_out))
	154
	155	shutil.copy2(file_in, file_out)
	156
	157	def copy_result(self, resultdir):
	158	dir_in = self.chrootPath("result")
	159
	160	for dir, subdirs, files in os.walk(dir_in):
	161	basename = os.path.basename(dir)
	162	dir = dir[len(self.chrootPath()):]
	163	for file in files:
	164	file_in = os.path.join(dir, file)
	165
	166	file_out = os.path.join(
	167	resultdir,
	168	basename,
	169	file,
	170	)
	171
	172	self.copyout(file_in, file_out)
	173
	174	def extract(self, requires=None, build_deps=True):
	175	"""
	176	Gets a dependency set and extracts all packages
	177	to the environment.
	178	"""
	179	if not requires:
	180	requires = []
	181
	182	# Add neccessary build dependencies.
	183	requires += BUILD_PACKAGES
	184
	185	# If we have ccache enabled, we need to extract it
	186	# to the build chroot.
	187	if self.settings.get("enable_ccache"):
	188	requires.append("ccache")
	189
	190	# If we have icecream enabled, we need to extract it
	191	# to the build chroot.
	192	if self.settings.get("enable_icecream"):
	193	requires.append("icecream")
	194
	195	# Get build dependencies from source package.
	196	if isinstance(self.pkg, packages.SourcePackage):
	197	for req in self.pkg.requires:
	198	requires.append(req)
	199
	200	# Install all packages.
	201	self.install(requires)
	202
	203	# Copy the makefile and load source tarballs.
	204	if isinstance(self.pkg, packages.Makefile):
	205	self.pkg.extract(self)
	206
	207	elif isinstance(self.pkg, packages.SourcePackage):
	208	self.pkg.extract(_("Extracting: %s (source)") % self.pkg.name,
	209	prefix=os.path.join(self.path, "build"))
	210
	211	# If we have a makefile, we can only get the build dependencies
	212	# after we have extracted all the rest.
	213	if build_deps and isinstance(self.pkg, packages.Makefile):
	214	requires = self.make_requires()
	215	if not requires:
	216	return
	217
	218	self.install(requires)
	219
	220	def install(self, requires):
	221	"""
	222	Install everything that is required in requires.
	223	"""
	224	# If we got nothing to do, we quit immediately.
	225	if not requires:
	226	return
	227
	228	ds = depsolve.DependencySet(self.pakfire)
	229	for r in requires:
	230	if isinstance(r, packages.BinaryPackage):
	231	ds.add_package(r)
	232	else:
	233	ds.add_requires(r)
	234	ds.resolve()
	235	ds.dump()
	236
	237	ts = transaction.Transaction(self.pakfire, ds)
	238	ts.run()
	239
	240	def install_test(self):
	241	pkgs = []
	242
	243	# Connect packages to the FS repository.
	244	r = repository.FileSystemRepository(self.pakfire)
	245
	246	for dir, subdirs, files in os.walk(self.chrootPath("result")):
	247	for file in files:
	248	file = os.path.join(dir, file)
	249
	250	if not file.endswith(PACKAGE_EXTENSION):
	251	continue
	252
	253	p = packages.BinaryPackage(self.pakfire, r, file)
	254	pkgs.append(p)
	255
	256	self.install(pkgs)
	257
	258	@property
	259	def log(self):
	260	# XXX for now, return the root logger
	261	return logging.getLogger()
	262
	263	def chrootPath(self, *args):
	264	# Remove all leading slashes
	265	_args = []
	266	for arg in args:
	267	if arg.startswith("/"):
	268	arg = arg[1:]
	269	_args.append(arg)
	270	args = _args
	271
	272	ret = os.path.join(self.path, *args)
	273	ret = ret.replace("//", "/")
	274
	275	assert ret.startswith(self.path)
	276
	277	return ret
	278
	279	def prepare(self):
	280	prepared_tag = ".prepared"
	281
	282	if os.path.exists(self.chrootPath(prepared_tag)):
	283	return
	284
	285	# Create directory.
	286	if not os.path.exists(self.path):
	287	os.makedirs(self.path)
	288
	289	# Create important directories.
	290	dirs = [
	291	"build",
	292	self.buildroot,
	293	"dev",
	294	"dev/pts",
	295	"dev/shm",
	296	"etc",
	297	"proc",
	298	"result",
	299	"sys",
	300	"tmp",
	301	"usr/src",
	302	]
	303
	304	# Create cache dir if ccache is enabled.
	305	if self.settings.get("enable_ccache"):
	306	dirs.append("var/cache/ccache")
	307
	308	if not os.path.exists(CCACHE_CACHE_DIR):
	309	os.makedirs(CCACHE_CACHE_DIR)
	310
	311	for dir in dirs:
	312	dir = self.chrootPath(dir)
	313	if not os.path.exists(dir):
	314	os.makedirs(dir)
	315
	316	# Create neccessary files like /etc/fstab and /etc/mtab.
	317	files = (
	318	"etc/fstab",
	319	"etc/mtab",
	320	prepared_tag,
	321	)
	322
	323	for file in files:
	324	file = self.chrootPath(file)
	325	dir = os.path.dirname(file)
	326	if not os.path.exists(dir):
	327	os.makedirs(dir)
	328	f = open(file, "w")
	329	f.close()
	330
	331	self._prepare_dev()
	332	self._prepare_dns()
	333
	334	def _prepare_dev(self):
	335	prevMask = os.umask(0000)
	336
	337	nodes = [
	338	("dev/null", stat.S_IFCHR \| 0666, os.makedev(1, 3)),
	339	("dev/full", stat.S_IFCHR \| 0666, os.makedev(1, 7)),
	340	("dev/zero", stat.S_IFCHR \| 0666, os.makedev(1, 5)),
	341	("dev/random", stat.S_IFCHR \| 0666, os.makedev(1, 8)),
	342	("dev/urandom", stat.S_IFCHR \| 0444, os.makedev(1, 9)),
	343	("dev/tty", stat.S_IFCHR \| 0666, os.makedev(5, 0)),
	344	("dev/console", stat.S_IFCHR \| 0600, os.makedev(5, 1)),
	345	]
	346
	347	# If we need loop devices (which are optional) we create them here.
	348	if self.settings["enable_loop_devices"]:
	349	for i in range(0, 7):
	350	nodes.append(("dev/loop%d" % i, stat.S_IFBLK \| 0660, os.makedev(7, i)))
	351
	352	# Create all the nodes.
	353	for node in nodes:
	354	self._create_node(*node)
	355
	356	os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
	357	os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
	358	os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
	359	os.symlink("/proc/self/fd", self.chrootPath("dev", "fd"))
	360
	361	# make device node for el4 and el5
	362	if self.kernel_version < "2.6.19":
	363	self._make_node("dev/ptmx", stat.S_IFCHR \| 0666, os.makedev(5, 2))
	364	else:
	365	os.symlink("/dev/pts/ptmx", self.chrootPath("dev", "ptmx"))
	366
	367	os.umask(prevMask)
	368
	369	def _prepare_dns(self):
	370	"""
	371	Add DNS resolution facility to chroot environment by copying
	372	/etc/resolv.conf and /etc/hosts.
	373	"""
	374	for i in ("/etc/resolv.conf", "/etc/hosts"):
	375	self.copyin(i, i)
	376
	377	def _create_node(self, filename, mode, device):
	378	logging.debug("Create node: %s (%s)" % (filename, mode))
	379
	380	filename = self.chrootPath(filename)
	381
	382	# Create parent directory if it is missing.
	383	dirname = os.path.dirname(filename)
	384	if not os.path.exists(dirname):
	385	os.makedirs(dirname)
	386
	387	os.mknod(filename, mode, device)
	388
	389	def destroy(self):
	390	logging.debug("Destroying environment %s" % self.path)
	391
	392	if os.path.exists(self.path):
	393	util.rm(self.path)
	394
	395	def cleanup(self):
	396	logging.debug("Cleaning environemnt.")
	397
	398	# Run make clean and let it cleanup its stuff.
	399	self.make("clean")
	400
	401	# Remove the build directory and buildroot.
	402	dirs = ("build", self.buildroot, "result")
	403
	404	for d in dirs:
	405	d = self.chrootPath(d)
	406	if not os.path.exists(d):
	407	continue
	408
	409	util.rm(d)
	410	os.makedirs(d)
	411
	412	# Clear make_info cache.
	413	if hasattr(self, "_make_info"):
	414	del self._make_info
	415
	416	def _mountall(self):
	417	self.log.debug("Mounting environment")
	418	for cmd, mountpoint in self.mountpoints:
	419	cmd = "%s %s" % (cmd, self.chrootPath(mountpoint))
	420	chroot.do(cmd, shell=True)
	421
	422	def _umountall(self):
	423	self.log.debug("Umounting environment")
	424	for cmd, mountpoint in self.mountpoints:
	425	cmd = "umount -n %s" % self.chrootPath(mountpoint)
	426	chroot.do(cmd, raiseExc=0, shell=True)
	427
	428	@property
	429	def mountpoints(self):
	430	ret = [
	431	("mount -n -t proc pakfire_chroot_proc", "proc"),
	432	("mount -n -t sysfs pakfire_chroot_sysfs", "sys"),
	433	]
	434
	435	mountopt = "gid=%d,mode=0620,ptmxmode=0666" % grp.getgrnam("tty").gr_gid
	436	if self.kernel_version >= "2.6.29":
	437	mountopt += ",newinstance"
	438
	439	ret.extend([
	440	("mount -n -t devpts -o %s pakfire_chroot_devpts" % mountopt, "dev/pts"),
	441	("mount -n -t tmpfs pakfire_chroot_shmfs", "dev/shm"),
	442	])
	443
	444	if self.settings.get("enable_ccache"):
	445	ret.append(("mount -n --bind %s" % CCACHE_CACHE_DIR, "var/cache/ccache"))
	446
	447	return ret
	448
	449	@staticmethod
	450	def calc_parallelism():
	451	"""
	452	Calculate how many processes to run
	453	at the same time.
	454
	455	We take the log10(number of processors) * factor
	456	"""
	457	num = os.sysconf("SC_NPROCESSORS_CONF")
	458	if num == 1:
	459	return 2
	460	else:
	461	return int(round(math.log10(num) * 26))
	462
	463	@property
	464	def environ(self):
	465	env = {
	466	# Add HOME manually, because it is occasionally not set
	467	# and some builds get in trouble then.
	468	"HOME" : "/root",
	469	"TERM" : os.environ.get("TERM", "dumb"),
	470	"PS1" : "\u:\w\$ ",
	471
	472	"BUILDROOT" : self.buildroot,
	473	"PARALLELISMFLAGS" : "-j%s" % self.calc_parallelism(),
	474	}
	475
	476	# Inherit environment from distro
	477	env.update(self.pakfire.distro.environ)
	478
	479	# Icecream environment settings
	480	if self.settings.get("enable_icecream", None):
	481	# Set the toolchain path
	482	if self.settings.get("icecream_toolchain", None):
	483	env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
	484
	485	# Set preferred host if configured.
	486	if self.settings.get("icecream_preferred_host", None):
	487	env["ICECC_PREFERRED_HOST"] = \
	488	self.settings.get("icecream_preferred_host")
	489
	490	# XXX what do we need else?
	491
	492	return env
	493
	494	def do(self, command, shell=True, personality=None, args, *kwargs):
	495	ret = None
	496	try:
	497	# Environment variables
	498	env = self.environ
	499
	500	if kwargs.has_key("env"):
	501	env.update(kwargs.pop("env"))
	502
	503	logging.debug("Environment:")
	504	for k, v in sorted(env.items()):
	505	logging.debug(" %s=%s" % (k, v))
	506
	507	# Update personality it none was set
	508	if not personality:
	509	personality = self.distro.personality
	510
	511	# Make every shell to a login shell because we set a lot of
	512	# environment things there.
	513	if shell:
	514	command = ["bash", "--login", "-c", command]
	515
	516	self._mountall()
	517
	518	if not kwargs.has_key("chrootPath"):
	519	kwargs["chrootPath"] = self.chrootPath()
	520
	521	ret = chroot.do(
	522	command,
	523	personality=personality,
	524	shell=False,
	525	env=env,
	526	logger=self.log,
	527	*args,
	528	**kwargs
	529	)
	530
	531	finally:
	532	self._umountall()
	533
	534	return ret
	535
	536	def make(self, args, *kwargs):
	537	if isinstance(self.pkg, packages.Makefile):
	538	filename = os.path.basename(self.pkg.filename)
	539	elif isinstance(self.pkg, packages.SourcePackage):
	540	filename = "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION)
	541
	542	return self.do("make -f /build/%s %s" % (filename, " ".join(args)),
	543	**kwargs)
	544
	545	@property
	546	def make_info(self):
	547	if not hasattr(self, "_make_info"):
	548	info = {}
	549
	550	output = self.make("buildinfo", returnOutput=True)
	551
	552	for line in output.splitlines():
	553	# XXX temporarily
	554	if not line:
	555	break
	556
	557	m = re.match(r"^(\w+)=(.*)$", line)
	558	if not m:
	559	continue
	560
	561	info[m.group(1)] = m.group(2).strip("\"")
	562
	563	self._make_info = info
	564
	565	return self._make_info
	566
	567	@property
	568	def packages(self):
	569	if hasattr(self, "_packages"):
	570	return self._packages
	571
	572	pkgs = []
	573	output = self.make("packageinfo", returnOutput=True)
	574
	575	pkg = {}
	576	for line in output.splitlines():
	577	if not line:
	578	pkgs.append(pkg)
	579	pkg = {}
	580
	581	m = re.match(r"^(\w+)=(.*)$", line)
	582	if not m:
	583	continue
	584
	585	k, v = m.groups()
	586	pkg[k] = v.strip("\"")
	587
	588	# Create a dummy repository to link the virtual packages to
	589	repo = repository.DummyRepository(self.pakfire)
	590
	591	self._packages = []
	592	for pkg in pkgs:
	593	pkg = packages.VirtualPackage(self.pakfire, pkg) # XXX had to remove repo here?!
	594	self._packages.append(pkg)
	595
	596	return self._packages
	597
	598	def make_requires(self):
	599	return self.make_info.get("PKG_BUILD_DEPS", "").split()
	600
	601	def make_sources(self):
	602	return self.make_info.get("PKG_FILES", "").split()
	603
	604	def create_icecream_toolchain(self):
	605	if not self.settings.get("enable_icecream", None):
	606	return
	607
	608	out = self.do("icecc --build-native", returnOutput=True)
	609
	610	for line in out.splitlines():
	611	m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
	612	if m:
	613	self.settings["icecream_toolchain"] = "/%s" % m.group(1)
	614
	615	def build(self):
	616	self.create_icecream_toolchain()
	617
	618	try:
	619	self.make("build")
	620
	621	except Error:
	622	raise BuildError, "The build command failed."
	623
	624	for pkg in reversed(self.packages):
	625	packager = packages.BinaryPackager(self.pakfire, pkg, self)
	626	packager()
	627
	628	def dist(self):
	629	self.pkg.dist(self)
	630
	631	def shell(self, args=[]):
	632	if not util.cli_is_interactive():
	633	logging.warning("Cannot run shell on non-interactive console.")
	634	return
	635
	636	# Install all packages that are needed to run a shell.
	637	self.install(SHELL_PACKAGES)
	638
	639	# XXX need to set CFLAGS here
	640	command = "/usr/sbin/chroot %s /usr/bin/chroot-shell %s" % \
	641	(self.chrootPath(), " ".join(args))
	642
	643	# Add personality if we require one
	644	if self.pakfire.distro.personality:
	645	command = "%s %s" % (self.pakfire.distro.personality, command)
	646
	647	for key, val in self.environ.items():
	648	command = "%s=\"%s\" " % (key, val) + command
	649
	650	# Empty the environment
	651	command = "env -i - %s" % command
	652
	653	logging.debug("Shell command: %s" % command)
	654
	655	try:
	656	self._mountall()
	657
	658	shell = os.system(command)
	659	return os.WEXITSTATUS(shell)
	660
	661	finally:
	662	self._umountall()