[people/stevee/pakfire.git] / pakfire / packages / packager.py

#!/usr/bin/python

import glob
import logging
import lzma
import os
import progressbar
import sys
import tarfile
import tempfile
import xattr

from pakfire.constants import *
from pakfire.i18n import _

class Extractor(object):
	def __init__(self, pakfire, pkg):
		self.pakfire = pakfire
		self.pkg = pkg

		self.data = pkg.get_file("data.img")

		self.archive = None
		self._tempfile = None

		if pkg.payload_compression == "XXX":
			self.archive = tarfile.open(fileobj=self.data)
		else:
			self._uncompress_data()

	def cleanup(self):
		# XXX not called by anything
		if self._tempfile:
			os.unlink(self._tempfile)

	def _uncompress_data(self):
		# XXX this function uncompresses the data.img file
		# and saves the bare tarball to /tmp which takes a lot
		# of space.

		self.data.seek(0)

		# Create a temporary file to save the content in
		f, self._tempfile = tempfile.mkstemp()
		f = open(self._tempfile, "w")

		decompressor = lzma.LZMADecompressor()

		buf = self.data.read(BUFFER_SIZE)
		while buf:
			f.write(decompressor.decompress(buf))

			buf = self.data.read(BUFFER_SIZE)

		f.write(decompressor.flush())
		f.close()

		self.archive = tarfile.open(self._tempfile)

	@property
	def files(self):
		return self.archive.getnames()

	def extractall(self, path="/", callback=None):
		pbar = self._make_progressbar()

		if pbar:
			pbar.start()
		else:
			print "  %s %-20s" % (_("Extracting"), self.pkg.name)

		i = 0
		for name in self.files:
			i += 1
			self.extract(name, path, callback=callback)

			if pbar:
				pbar.update(i)

		if pbar:
			pbar.finish()
			#sys.stdout.write("\n")

	def extract(self, filename, path="/", callback=None):
		member = self.archive.getmember(filename)
		target = os.path.join(path, filename)

		# If the member is a directory and if it already exists, we
		# don't need to create it again.
		if member.isdir() and os.path.exists(target):
			return

		#if self.pakfire.config.get("debug"):
		#	msg = "Creating file (%s:%03d:%03d) " % \
		#		(tarfile.filemode(member.mode), member.uid, member.gid)
		#	if member.issym():
		#		msg += "/%s -> %s" % (member.name, member.linkname)
		#	elif member.islnk():
		#		msg += "/%s link to /%s" % (member.name, member.linkname)
		#	else:
		#		msg += "/%s" % member.name
		#	logging.debug(msg)

		# Remove file if it has been existant
		if not member.isdir() and os.path.exists(target):
			os.unlink(target)

		self.archive.extract(member, path=path)

		# XXX implement setting of xattrs/acls here

		if callback and not member.isdir():
			callback(member.name, hash1="XXX", size=member.size)

	def _make_progressbar(self):
		# Don't display a progressbar if we are running in debug mode.
		if self.pakfire.config.get("debug"):
			return

		if not sys.stdout.isatty():
			return

		widgets = [
			"  ",
			"%s %-20s" % (_("Extracting:"), self.pkg.name),
			" ",
			progressbar.Bar(left="[", right="]"),
			"  ",
#			progressbar.Percentage(),
#			"  ",
			progressbar.ETA(),
			"  ",
		]

		# maxval must be > 0 and so we assume that
		# empty packages have at least one file.
		maxval = len(self.files) or 1

		return progressbar.ProgressBar(
			widgets=widgets,
			maxval=maxval,
			term_width=80,
		)


class InnerTarFile(tarfile.TarFile):
	def __init__(self, *args, **kwargs):
		# Force the pax format
		kwargs["format"] = tarfile.PAX_FORMAT

		if kwargs.has_key("env"):
			self.env = kwargs.pop("env")

		tarfile.TarFile.__init__(self, *args, **kwargs)

	def __filter_xattrs(self, tarinfo):
		logging.debug("Adding file: %s" % tarinfo.name)

		filename = self.env.chrootPath(self.env.buildroot, tarinfo.name)

		# xattrs do only exists for regular files. If we don't have one,
		# simply skip.
		if os.path.isfile(filename):
			for attr, value in xattr.get_all(filename):
				tarinfo.pax_headers[attr] = value

				logging.debug("  xattr: %s=%s" % (attr, value))

		return tarinfo

	def add(self, *args, **kwargs):
		# Add filter for xattrs if no other filter is set.
		if not kwargs.has_key("filter") and len(args) < 5:
			kwargs["filter"] = self.__filter_xattrs

		tarfile.TarFile.add(self, *args, **kwargs)


# XXX this is totally ugly and needs to be done right!

class Packager(object):
	ARCHIVE_FILES = ("info", "filelist", "signature", "data.img")

	def __init__(self, pakfire, pkg, env):
		self.pakfire = pakfire
		self.pkg = pkg
		self.env = env

		self.tarball = None

		# Store meta information
		self.info = {
			"package_format" : PACKAGE_FORMAT,
		}
		self.info.update(self.pkg.info)
		self.info.update(self.pakfire.distro.info)

		### Create temporary files
		# Create temp directory to where we extract all files again and
		# gather some information about them like requirements and provides.
		self.tempdir = self.env.chrootPath("tmp", "%s_data" % self.pkg.friendly_name)
		if not os.path.exists(self.tempdir):
			os.makedirs(self.tempdir)

		# Create files that have the archive data
		self.archive_files = {}
		for i in self.ARCHIVE_FILES:
			self.archive_files[i] = \
				self.env.chrootPath("tmp", "%s_%s" % (self.pkg.friendly_name, i))

	def __call__(self):
		logging.debug("Packaging %s" % self.pkg.friendly_name)

		# Create the tarball and add all data to it.
		self.create_tarball()

		chroot_tempdir = self.tempdir[len(self.env.chrootPath()):]
		self.info.update({
			"requires" : self.env.do("/usr/lib/buildsystem-tools/dependency-tracker requires %s" % chroot_tempdir,
				returnOutput=True, env=self.pkg.env).strip(),
			"provides" : self.env.do("/usr/lib/buildsystem-tools/dependency-tracker provides %s" % chroot_tempdir,
				returnOutput=True, env=self.pkg.env).strip(),
		})

		self.create_info()
		self.create_signature()

		# Create the outer tarball.
		resultdir = os.path.join(self.env.chrootPath("result", self.pkg.arch))
		if not os.path.exists(resultdir):
			os.makedirs(resultdir)

		filename = os.path.join(resultdir, self.pkg.filename)

		tar = tarfile.TarFile(filename, mode="w", format=tarfile.PAX_FORMAT)

		for i in self.ARCHIVE_FILES:
			tar.add(self.archive_files[i], arcname=i)

		tar.close()

	def create_tarball(self):
		tar = InnerTarFile(self.archive_files["data.img"], mode="w", env=self.env)

		includes = []
		excludes = []

		for pattern in self.pkg.file_patterns:
			# Check if we are running in include or exclude mode.
			if pattern.startswith("!"):
				files = excludes

				# Strip the ! charater
				pattern = pattern[1:]

			else:
				files = includes

			if pattern.startswith("/"):
				pattern = pattern[1:]
			pattern = self.env.chrootPath(self.env.buildroot, pattern)

			# Recognize the type of the pattern. Patterns could be a glob
			# pattern that is expanded here or just a directory which will
			# be included recursively.
			if "*" in pattern or "?" in pattern:
				files += glob.glob(pattern)

			elif os.path.exists(pattern):
				# Add directories recursively...
				if os.path.isdir(pattern):
					for dir, subdirs, _files in os.walk(pattern):
						for file in _files:
							file = os.path.join(dir, file)
							files.append(file)

				# all other files are just added.
				else:
					files.append(pattern)

			else:
				logging.warning("Unrecognized pattern type: %s" % pattern)

		files = []
		for file in includes:
			# Skip if file is already in the file set or
			# marked to be excluded from this archive.
			if file in excludes or file in files:
				continue

			files.append(file)

		files.sort()

		for file_real in files:
			file_tar = file_real[len(self.env.chrootPath(self.env.buildroot)) + 1:]

			tar.add(file_real, arcname=file_tar, recursive=False)
			os.unlink(file_real)

		# Dump all files that are in the archive.
		tar.list()

		# Write all data to disk.
		tar.close()

		# Reopen the tarfile in read mode and extract all content to tempdir
		tar = InnerTarFile(self.archive_files["data.img"])
		tar.extractall(path=self.tempdir)

		# Write filelist
		f = open(self.archive_files["filelist"], mode="w")
		for filename in tar.getnames():
			if not filename.startswith("/"):
				filename = "/%s" % filename

			f.write("%s\n" % filename)
		f.close()

		tar.close()

		# XXX compress the tarball here

	def create_info(self):
		f = open(self.archive_files["info"], "w")
		f.write(BINARY_PACKAGE_META % self.info)
		f.close()

	def create_signature(self):
		# Create an empty signature.
		f = open(self.archive_files["signature"], "w")
		f.close()
Commit	Line	Data
47a4cb89 MT	1	#!/usr/bin/python
	2
	3	import glob
	4	import logging
	5	import lzma
	6	import os
	7	import progressbar
	8	import sys
	9	import tarfile
	10	import tempfile
	11	import xattr
	12
	13	from pakfire.constants import *
	14	from pakfire.i18n import _
	15
	16	class Extractor(object):
	17	def __init__(self, pakfire, pkg):
	18	self.pakfire = pakfire
	19	self.pkg = pkg
	20
	21	self.data = pkg.get_file("data.img")
	22
	23	self.archive = None
	24	self._tempfile = None
	25
fa6d335b MT	26	if pkg.payload_compression == "XXX":
	27	self.archive = tarfile.open(fileobj=self.data)
	28	else:
	29	self._uncompress_data()
47a4cb89 MT	30
	31	def cleanup(self):
	32	# XXX not called by anything
	33	if self._tempfile:
	34	os.unlink(self._tempfile)
	35
	36	def _uncompress_data(self):
	37	# XXX this function uncompresses the data.img file
	38	# and saves the bare tarball to /tmp which takes a lot
	39	# of space.
	40
	41	self.data.seek(0)
	42
	43	# Create a temporary file to save the content in
	44	f, self._tempfile = tempfile.mkstemp()
	45	f = open(self._tempfile, "w")
	46
	47	decompressor = lzma.LZMADecompressor()
	48
	49	buf = self.data.read(BUFFER_SIZE)
	50	while buf:
	51	f.write(decompressor.decompress(buf))
	52
	53	buf = self.data.read(BUFFER_SIZE)
	54
	55	f.write(decompressor.flush())
	56	f.close()
	57
	58	self.archive = tarfile.open(self._tempfile)
	59
	60	@property
	61	def files(self):
	62	return self.archive.getnames()
	63
	64	def extractall(self, path="/", callback=None):
	65	pbar = self._make_progressbar()
	66
	67	if pbar:
	68	pbar.start()
	69	else:
	70	print " %s %-20s" % (_("Extracting"), self.pkg.name)
	71
	72	i = 0
	73	for name in self.files:
	74	i += 1
	75	self.extract(name, path, callback=callback)
	76
	77	if pbar:
	78	pbar.update(i)
	79
	80	if pbar:
	81	pbar.finish()
	82	#sys.stdout.write("\n")
	83
	84	def extract(self, filename, path="/", callback=None):
	85	member = self.archive.getmember(filename)
	86	target = os.path.join(path, filename)
	87
	88	# If the member is a directory and if it already exists, we
	89	# don't need to create it again.
	90	if member.isdir() and os.path.exists(target):
	91	return
	92
fa6d335b MT	93	#if self.pakfire.config.get("debug"):
	94	# msg = "Creating file (%s:%03d:%03d) " % \
	95	# (tarfile.filemode(member.mode), member.uid, member.gid)
	96	# if member.issym():
	97	# msg += "/%s -> %s" % (member.name, member.linkname)
	98	# elif member.islnk():
	99	# msg += "/%s link to /%s" % (member.name, member.linkname)
	100	# else:
	101	# msg += "/%s" % member.name
	102	# logging.debug(msg)
47a4cb89 MT	103
	104	# Remove file if it has been existant
	105	if not member.isdir() and os.path.exists(target):
	106	os.unlink(target)
	107
	108	self.archive.extract(member, path=path)
	109
	110	# XXX implement setting of xattrs/acls here
	111
	112	if callback and not member.isdir():
	113	callback(member.name, hash1="XXX", size=member.size)
	114
	115	def _make_progressbar(self):
	116	# Don't display a progressbar if we are running in debug mode.
	117	if self.pakfire.config.get("debug"):
	118	return
	119
	120	if not sys.stdout.isatty():
	121	return
	122
	123	widgets = [
	124	" ",
	125	"%s %-20s" % (_("Extracting:"), self.pkg.name),
	126	" ",
	127	progressbar.Bar(left="[", right="]"),
	128	" ",
	129	# progressbar.Percentage(),
	130	# " ",
	131	progressbar.ETA(),
	132	" ",
	133	]
	134
64302ca2 MT	135	# maxval must be > 0 and so we assume that
	136	# empty packages have at least one file.
	137	maxval = len(self.files) or 1
	138
47a4cb89 MT	139	return progressbar.ProgressBar(
47a4cb89 MT	140	widgets=widgets,
64302ca2	141	maxval=maxval,
47a4cb89 MT	142	term_width=80,
	143	)
	144
	145
	146	class InnerTarFile(tarfile.TarFile):
	147	def __init__(self, args, *kwargs):
	148	# Force the pax format
	149	kwargs["format"] = tarfile.PAX_FORMAT
	150
	151	if kwargs.has_key("env"):
	152	self.env = kwargs.pop("env")
	153
	154	tarfile.TarFile.__init__(self, args, *kwargs)
	155
	156	def __filter_xattrs(self, tarinfo):
	157	logging.debug("Adding file: %s" % tarinfo.name)
	158
	159	filename = self.env.chrootPath(self.env.buildroot, tarinfo.name)
47a4cb89	160
9f2980a9 MT	161	# xattrs do only exists for regular files. If we don't have one,
	162	# simply skip.
	163	if os.path.isfile(filename):
	164	for attr, value in xattr.get_all(filename):
	165	tarinfo.pax_headers[attr] = value
	166
	167	logging.debug(" xattr: %s=%s" % (attr, value))
47a4cb89 MT	168
	169	return tarinfo
	170
	171	def add(self, args, *kwargs):
	172	# Add filter for xattrs if no other filter is set.
	173	if not kwargs.has_key("filter") and len(args) < 5:
	174	kwargs["filter"] = self.__filter_xattrs
	175
	176	tarfile.TarFile.add(self, args, *kwargs)
	177
	178
	179	# XXX this is totally ugly and needs to be done right!
	180
	181	class Packager(object):
2d2f5762	182	ARCHIVE_FILES = ("info", "filelist", "signature", "data.img")
47a4cb89 MT	183
	184	def __init__(self, pakfire, pkg, env):
	185	self.pakfire = pakfire
	186	self.pkg = pkg
	187	self.env = env
	188
	189	self.tarball = None
	190
	191	# Store meta information
	192	self.info = {
	193	"package_format" : PACKAGE_FORMAT,
	194	}
	195	self.info.update(self.pkg.info)
	196	self.info.update(self.pakfire.distro.info)
	197
	198	### Create temporary files
	199	# Create temp directory to where we extract all files again and
	200	# gather some information about them like requirements and provides.
	201	self.tempdir = self.env.chrootPath("tmp", "%s_data" % self.pkg.friendly_name)
	202	if not os.path.exists(self.tempdir):
	203	os.makedirs(self.tempdir)
	204
	205	# Create files that have the archive data
	206	self.archive_files = {}
	207	for i in self.ARCHIVE_FILES:
	208	self.archive_files[i] = \
	209	self.env.chrootPath("tmp", "%s_%s" % (self.pkg.friendly_name, i))
	210
	211	def __call__(self):
	212	logging.debug("Packaging %s" % self.pkg.friendly_name)
	213
	214	# Create the tarball and add all data to it.
	215	self.create_tarball()
	216
	217	chroot_tempdir = self.tempdir[len(self.env.chrootPath()):]
	218	self.info.update({
	219	"requires" : self.env.do("/usr/lib/buildsystem-tools/dependency-tracker requires %s" % chroot_tempdir,
a0577731	220	returnOutput=True, env=self.pkg.env).strip(),
47a4cb89	221	"provides" : self.env.do("/usr/lib/buildsystem-tools/dependency-tracker provides %s" % chroot_tempdir,
a0577731	222	returnOutput=True, env=self.pkg.env).strip(),
47a4cb89 MT	223	})
	224
	225	self.create_info()
	226	self.create_signature()
	227
	228	# Create the outer tarball.
	229	resultdir = os.path.join(self.env.chrootPath("result", self.pkg.arch))
	230	if not os.path.exists(resultdir):
	231	os.makedirs(resultdir)
	232
	233	filename = os.path.join(resultdir, self.pkg.filename)
	234
	235	tar = tarfile.TarFile(filename, mode="w", format=tarfile.PAX_FORMAT)
	236
	237	for i in self.ARCHIVE_FILES:
	238	tar.add(self.archive_files[i], arcname=i)
	239
	240	tar.close()
	241
	242	def create_tarball(self):
	243	tar = InnerTarFile(self.archive_files["data.img"], mode="w", env=self.env)
	244
715a78e9 MT	245	includes = []
	246	excludes = []
	247
47a4cb89	248	for pattern in self.pkg.file_patterns:
715a78e9 MT	249	# Check if we are running in include or exclude mode.
	250	if pattern.startswith("!"):
	251	files = excludes
	252
	253	# Strip the ! charater
	254	pattern = pattern[1:]
	255
	256	else:
	257	files = includes
	258
47a4cb89 MT	259	if pattern.startswith("/"):
	260	pattern = pattern[1:]
	261	pattern = self.env.chrootPath(self.env.buildroot, pattern)
	262
	263	# Recognize the type of the pattern. Patterns could be a glob
	264	# pattern that is expanded here or just a directory which will
	265	# be included recursively.
	266	if "*" in pattern or "?" in pattern:
	267	files += glob.glob(pattern)
	268
	269	elif os.path.exists(pattern):
	270	# Add directories recursively...
	271	if os.path.isdir(pattern):
	272	for dir, subdirs, _files in os.walk(pattern):
	273	for file in _files:
	274	file = os.path.join(dir, file)
	275	files.append(file)
	276
	277	# all other files are just added.
	278	else:
	279	files.append(pattern)
	280
	281	else:
	282	logging.warning("Unrecognized pattern type: %s" % pattern)
	283
715a78e9 MT	284	files = []
715a78e9 MT	285	for file in includes:
58d47ee6 MT	286	# Skip if file is already in the file set or
	287	# marked to be excluded from this archive.
	288	if file in excludes or file in files:
715a78e9 MT	289	continue
	290
	291	files.append(file)
	292
47a4cb89 MT	293	files.sort()
	294
	295	for file_real in files:
	296	file_tar = file_real[len(self.env.chrootPath(self.env.buildroot)) + 1:]
	297
715a78e9	298	tar.add(file_real, arcname=file_tar, recursive=False)
58d47ee6	299	os.unlink(file_real)
47a4cb89 MT	300
	301	# Dump all files that are in the archive.
	302	tar.list()
	303
	304	# Write all data to disk.
	305	tar.close()
	306
	307	# Reopen the tarfile in read mode and extract all content to tempdir
	308	tar = InnerTarFile(self.archive_files["data.img"])
	309	tar.extractall(path=self.tempdir)
2d2f5762 MT	310
	311	# Write filelist
	312	f = open(self.archive_files["filelist"], mode="w")
	313	for filename in tar.getnames():
	314	if not filename.startswith("/"):
	315	filename = "/%s" % filename
	316
	317	f.write("%s\n" % filename)
	318	f.close()
	319
47a4cb89 MT	320	tar.close()
	321
	322	# XXX compress the tarball here
	323
	324	def create_info(self):
	325	f = open(self.archive_files["info"], "w")
	326	f.write(BINARY_PACKAGE_META % self.info)
	327	f.close()
	328
	329	def create_signature(self):
	330	# Create an empty signature.
	331	f = open(self.archive_files["signature"], "w")
	332	f.close()