[pakfire.git] / python / pakfire / packages / make.py

#!/usr/bin/python
###############################################################################
#                                                                             #
# Pakfire - The IPFire package management system                              #
# Copyright (C) 2011 Pakfire development team                                 #
#                                                                             #
# This program is free software: you can redistribute it and/or modify        #
# it under the terms of the GNU General Public License as published by        #
# the Free Software Foundation, either version 3 of the License, or           #
# (at your option) any later version.                                         #
#                                                                             #
# This program is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
# GNU General Public License for more details.                                #
#                                                                             #
# You should have received a copy of the GNU General Public License           #
# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
#                                                                             #
###############################################################################

import logging
import os
import re
import shutil
import socket
import tarfile

from urlgrabber.grabber import URLGrabber, URLGrabError
from urlgrabber.progress import TextMeter

import lexer
import packager

import pakfire.chroot as chroot
import pakfire.downloader as downloader
import pakfire.util as util

from base import Package
from file import SourcePackage

from pakfire.constants import *
from pakfire.i18n import _

class MakefileBase(Package):
	def __init__(self, pakfire, filename):
		Package.__init__(self, pakfire)

		# Save the filename of the makefile.
		self.filename = os.path.abspath(filename)

		# Update environment.
		environ = self.pakfire.environ
		environ.update({
			"PARALLELISMFLAGS" : "-j%s" % util.calc_parallelism(),
		})

		# Open and parse the makefile.
		self.lexer = lexer.RootLexer.open(self.filename, environ=environ)

	@property
	def package_filename(self):
		return PACKAGE_FILENAME_FMT % {
			"arch"    : self.arch,
			"ext"     : PACKAGE_EXTENSION,
			"name"    : self.name,
			"release" : self.release,
			"version" : self.version,
		}

	def lint(self):
		errors = []

		if not self.name:
			errors.append(_("Package name is undefined."))

		if not self.version:
			errors.append(_("Package version is undefined."))

		# XXX to do...

		return errors

	@property
	def name(self):
		return self.lexer.get_var("name")

	@property
	def epoch(self):
		epoch = self.lexer.get_var("epoch")
		if not epoch:
			return 0

		return int(epoch)

	@property
	def version(self):
		return self.lexer.get_var("version")

	@property
	def release(self):
		release = self.lexer.get_var("release")
		assert release

		tag = self.lexer.get_var("DISTRO_DISTTAG")
		assert tag

		return ".".join((release, tag))

	@property
	def summary(self):
		return self.lexer.get_var("summary")

	@property
	def description(self):
		description = self.lexer.get_var("description")

		# Replace all backslashes at the end of a line.
		return description.replace("\\\n", "\n")

	@property
	def groups(self):
		groups = self.lexer.get_var("groups").split()

		return sorted(groups)

	@property
	def url(self):
		return self.lexer.get_var("url")

	@property
	def license(self):
		return self.lexer.get_var("license")

	@property
	def maintainer(self):
		maintainer = self.lexer.get_var("maintainer")

		if not maintainer:
			maintainer = self.lexer.get_var("DISTRO_MAINTAINER")

		return maintainer

	@property
	def vendor(self):
		return self.lexer.get_var("DISTRO_VENDOR")

	@property
	def buildroot(self):
		return self.lexer.get_var("BUILDROOT")

	@property
	def build_host(self):
		return socket.gethostname()

	# XXX build_id and build_time are used to create a source package

	@property
	def build_id(self):
		# XXX todo
		# Not existant for Makefiles
		return None

	@property
	def build_time(self):
		# XXX todo
		# Not existant for Makefiles
		return None


class Makefile(MakefileBase):
	@property
	def uuid(self):
		hash1 = util.calc_hash1(self.filename)

		# Return UUID version 5 (SHA1 hash)
		return "%8s-%4s-5%3s-%4s-%11s" % \
			(hash1[0:8], hash1[9:13], hash1[14:17], hash1[18:22], hash1[23:34])

	@property
	def path(self):
		return os.path.dirname(self.filename)

	@property
	def arch(self):
		return "src"

	@property
	def supported_arches(self):
		"""
			These are the supported arches. Which means, packages of these
			architectures can be built out of this source package.
		"""
		return self.lexer.get_var("sup_arches", "all")

	@property
	def packages(self):
		pkgs = []

		for lexer in self.lexer.packages:
			name = lexer.get_var("_name")

			pkg = MakefilePackage(self.pakfire, name, lexer)
			pkgs.append(pkg)

		return pkgs

	@property
	def source_dl(self):
		dls = []

		if self.pakfire.distro.source_dl:
			dls.append(self.pakfire.distro.source_dl)

		dl = self.lexer.get_var("source_dl")
		if dl:
			dls.append(dl)

		return dls

	def download(self):
		"""
			Download all external sources and return a list with the local
			copies.
		"""
		# Download source files.
		grabber = downloader.SourceDownloader(self.pakfire,
			mirrors=self.source_dl)

		return grabber.download(self.sources)

	def dist(self, resultdirs):
		"""
			Create a source package.
		"""
		# Download all files we need for this package.
		self.download()

		p = packager.SourcePackager(self.pakfire, self)
		p.run(resultdirs)

	def dump(self, *args, **kwargs):
		dump = MakefileBase.dump(self, *args, **kwargs)
		dump = dump.splitlines()

		#dump += ["", _("Containing the following binary packages:"),]
		#
		#for pkg in self.packages:
		#	_dump = pkg.dump(*args, **kwargs)
		#
		#	for line in _dump.splitlines():
		#		dump.append("  %s" % line)
		#	dump.append("")

		return "\n".join(dump)

	def get_buildscript(self, stage):
		return self.lexer.build.get_var("_%s" % stage)

	@property
	def prerequires(self):
		return []

	@property
	def requires(self):
		return self.lexer.build.get_var("requires", "").split()

	@property
	def provides(self):
		return []

	@property
	def obsoletes(self):
		return []

	@property
	def conflicts(self):
		return []

	@property
	def files(self):
		files = []
		basedir = os.path.dirname(self.filename)

		for dirs, subdirs, _files in os.walk(basedir):
			for f in _files:
				files.append(os.path.join(dirs, f))

		return files

	@property
	def sources(self):
		return self.lexer.get_var("sources").split()

	@property
	def exports(self):
		exports = {}

		# Include quality agent exports.
		exports.update(self.lexer.quality_agent.exports)

		for export in self.lexer.build.exports:
			exports[export] = self.lexer.build.get_var(export)

		return exports

	def extract(self, message=None, prefix=None):
		# XXX neeed to make this waaaaaaaaaay better.

		files = self.files

		# Load progressbar.
		pb = None
		if message:
			message = "%-10s : %s" % (message, self.friendly_name)
			pb = util.make_progress(message, len(files), eta=False)

		dir_len = len(os.path.dirname(self.filename))

		# Copy all files that belong to the package
		i = 0
		for f in files:
			if pb:
				i += 1
				pb.update(i)

			_f = f[dir_len:]
			logging.debug("%s/%s" % (prefix, _f))
	
			path = "%s/%s" % (prefix, _f)

			path_dir = os.path.dirname(path)
			if not os.path.exists(path_dir):
				os.makedirs(path_dir)

			shutil.copy2(f, path)

		if pb:
			pb.finish()

		# Download source files.
		for _filename in self.download():
			filename = "%s/files/%s" % (prefix, os.path.basename(_filename))
			dirname = os.path.dirname(filename)

			if not os.path.exists(dirname):
				os.makedirs(dirname)

			shutil.copy2(_filename, filename)


class MakefilePackage(MakefileBase):
	def __init__(self, pakfire, name, lexer):
		Package.__init__(self, pakfire)

		self._name = name
		self.lexer = lexer

		# Store additional dependencies in here.
		self._dependencies = {}

	@property
	def name(self):
		return self._name

	@property
	def arch(self):
		return self.lexer.get_var("arch", "%{DISTRO_ARCH}")

	@property
	def configfiles(self):
		return self.lexer.get_var("configfiles").split()

	@property
	def files(self):
		return self.lexer.get_var("files").split()

	@property
	def uuid(self):
		return None

	def track_dependencies(self, builder, path):
		# Dependency types.
		dep_types = ("prerequires", "requires", "provides", "conflicts", "obsoletes",)

		result = builder.do("/usr/lib/pakfire/dependency-tracker %s" \
			% path, returnOutput=True)

		for line in result.splitlines():
			m = re.match(r"^(\w+)=(.*)$", line)
			if m is None:
				continue

			key, val = m.groups()

			if not key in dep_types:
				continue

			val = val.strip("\"")
			val = val.split()

			self._dependencies[key] = sorted(val)

		# Filter dependencies.
		for key in dep_types:
			self._dependencies[key] = self.filter_deps(
				self._dependencies.get(key, []),
				self.lexer.get_var("filter_%s" % key)
			)

	@staticmethod
	def filter_deps(deps, filters):
		if not filters:
			return deps

		filters = filters.splitlines()
		filtered_deps = []

		for dep in deps:
			filtered = False
			for filter in filters:
				m = re.search(filter, dep)
				if not m:
					continue

				# Yes, we found a match.
				filtered = True

			if not filtered:
				filtered_deps.append(dep)

		return filtered_deps

	def get_deps(self, key):
		# Collect all dependencies that were set in the makefile by the user.
		deps = self.lexer.get_var(key).split()

		# Collect all dependencies that were discovered by the tracker.
		deps += self._dependencies.get(key, [])

		# Remove duplicates.
		deps = set(deps)
		deps = list(deps)

		return sorted(deps)

	@property
	def prerequires(self):
		return self.get_deps("prerequires")

	@property
	def requires(self):
		return self.get_deps("requires")

	@property
	def provides(self):
		return self.get_deps("provides")

	@property
	def obsoletes(self):
		return self.get_deps("obsoletes")

	@property
	def conflicts(self):
		return self.get_deps("conflicts")

	def get_scriptlet(self, type):
		return self.lexer.get_scriptlet(type)

	@property
	def inst_size(self):
		# The size of this is unknown.
		return 0
Commit	Line	Data
47a4cb89	1	#!/usr/bin/python
b792d887 MT	2	###############################################################################
	3	# #
	4	# Pakfire - The IPFire package management system #
	5	# Copyright (C) 2011 Pakfire development team #
	6	# #
	7	# This program is free software: you can redistribute it and/or modify #
	8	# it under the terms of the GNU General Public License as published by #
	9	# the Free Software Foundation, either version 3 of the License, or #
	10	# (at your option) any later version. #
	11	# #
	12	# This program is distributed in the hope that it will be useful, #
	13	# but WITHOUT ANY WARRANTY; without even the implied warranty of #
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
	15	# GNU General Public License for more details. #
	16	# #
	17	# You should have received a copy of the GNU General Public License #
	18	# along with this program. If not, see <http://www.gnu.org/licenses/>. #
	19	# #
	20	###############################################################################
47a4cb89	21
c07a3ca7	22	import logging
47a4cb89	23	import os
5dda54e4	24	import re
c07a3ca7 MT	25	import shutil
c07a3ca7 MT	26	import socket
47a4cb89 MT	27	import tarfile
	28
	29	from urlgrabber.grabber import URLGrabber, URLGrabError
	30	from urlgrabber.progress import TextMeter
	31
c07a3ca7	32	import lexer
47a4cb89 MT	33	import packager
47a4cb89 MT	34
5dda54e4	35	import pakfire.chroot as chroot
062699ee	36	import pakfire.downloader as downloader
c07a3ca7 MT	37	import pakfire.util as util
c07a3ca7 MT	38
47a4cb89	39	from base import Package
e0636b31 MT	40	from file import SourcePackage
e0636b31 MT	41
47a4cb89	42	from pakfire.constants import *
c07a3ca7	43	from pakfire.i18n import _
47a4cb89	44
c07a3ca7	45	class MakefileBase(Package):
3723913b	46	def __init__(self, pakfire, filename):
a2d1644c	47	Package.__init__(self, pakfire)
47a4cb89	48
c07a3ca7 MT	49	# Save the filename of the makefile.
c07a3ca7 MT	50	self.filename = os.path.abspath(filename)
47a4cb89	51
440f3cb8 MT	52	# Update environment.
	53	environ = self.pakfire.environ
	54	environ.update({
	55	"PARALLELISMFLAGS" : "-j%s" % util.calc_parallelism(),
	56	})
	57
c07a3ca7	58	# Open and parse the makefile.
440f3cb8	59	self.lexer = lexer.RootLexer.open(self.filename, environ=environ)
47a4cb89 MT	60
	61	@property
	62	def package_filename(self):
	63	return PACKAGE_FILENAME_FMT % {
	64	"arch" : self.arch,
	65	"ext" : PACKAGE_EXTENSION,
	66	"name" : self.name,
	67	"release" : self.release,
	68	"version" : self.version,
	69	}
	70
c07a3ca7 MT	71	def lint(self):
	72	errors = []
	73
	74	if not self.name:
	75	errors.append(_("Package name is undefined."))
	76
	77	if not self.version:
	78	errors.append(_("Package version is undefined."))
	79
	80	# XXX to do...
	81
	82	return errors
	83
	84	@property
	85	def name(self):
	86	return self.lexer.get_var("name")
	87
	88	@property
	89	def epoch(self):
	90	epoch = self.lexer.get_var("epoch")
	91	if not epoch:
	92	return 0
	93
	94	return int(epoch)
	95
	96	@property
	97	def version(self):
	98	return self.lexer.get_var("version")
	99
	100	@property
	101	def release(self):
	102	release = self.lexer.get_var("release")
	103	assert release
	104
	105	tag = self.lexer.get_var("DISTRO_DISTTAG")
	106	assert tag
	107
	108	return ".".join((release, tag))
	109
	110	@property
	111	def summary(self):
	112	return self.lexer.get_var("summary")
	113
	114	@property
	115	def description(self):
c4c528f7 MT	116	description = self.lexer.get_var("description")
	117
	118	# Replace all backslashes at the end of a line.
	119	return description.replace("\\\n", "\n")
c07a3ca7 MT	120
	121	@property
	122	def groups(self):
	123	groups = self.lexer.get_var("groups").split()
	124
	125	return sorted(groups)
	126
	127	@property
	128	def url(self):
	129	return self.lexer.get_var("url")
	130
	131	@property
	132	def license(self):
	133	return self.lexer.get_var("license")
	134
	135	@property
	136	def maintainer(self):
	137	maintainer = self.lexer.get_var("maintainer")
	138
	139	if not maintainer:
	140	maintainer = self.lexer.get_var("DISTRO_MAINTAINER")
	141
	142	return maintainer
	143
	144	@property
	145	def vendor(self):
	146	return self.lexer.get_var("DISTRO_VENDOR")
	147
0d96815e MT	148	@property
	149	def buildroot(self):
	150	return self.lexer.get_var("BUILDROOT")
	151
c07a3ca7 MT	152	@property
	153	def build_host(self):
	154	return socket.gethostname()
	155
	156	# XXX build_id and build_time are used to create a source package
	157
	158	@property
	159	def build_id(self):
	160	# XXX todo
	161	# Not existant for Makefiles
	162	return None
	163
	164	@property
	165	def build_time(self):
	166	# XXX todo
	167	# Not existant for Makefiles
	168	return None
	169
	170
	171	class Makefile(MakefileBase):
	172	@property
	173	def uuid(self):
	174	hash1 = util.calc_hash1(self.filename)
	175
	176	# Return UUID version 5 (SHA1 hash)
	177	return "%8s-%4s-5%3s-%4s-%11s" % \
	178	(hash1[0:8], hash1[9:13], hash1[14:17], hash1[18:22], hash1[23:34])
	179
	180	@property
	181	def path(self):
	182	return os.path.dirname(self.filename)
	183
47a4cb89 MT	184	@property
47a4cb89 MT	185	def arch(self):
1b59091e MT	186	return "src"
	187
	188	@property
	189	def supported_arches(self):
47a4cb89	190	"""
1b59091e MT	191	These are the supported arches. Which means, packages of these
1b59091e MT	192	architectures can be built out of this source package.
47a4cb89	193	"""
1b59091e	194	return self.lexer.get_var("sup_arches", "all")
47a4cb89	195
c07a3ca7 MT	196	@property
	197	def packages(self):
	198	pkgs = []
	199
	200	for lexer in self.lexer.packages:
f0618326	201	name = lexer.get_var("_name")
c07a3ca7 MT	202
	203	pkg = MakefilePackage(self.pakfire, name, lexer)
	204	pkgs.append(pkg)
	205
afe4e55c	206	return pkgs
c07a3ca7 MT	207
	208	@property
	209	def source_dl(self):
	210	dls = []
	211
	212	if self.pakfire.distro.source_dl:
	213	dls.append(self.pakfire.distro.source_dl)
	214
	215	dl = self.lexer.get_var("source_dl")
	216	if dl:
	217	dls.append(dl)
	218
	219	return dls
	220
	221	def download(self):
	222	"""
	223	Download all external sources and return a list with the local
	224	copies.
47a4cb89	225	"""
c07a3ca7	226	# Download source files.
062699ee MT	227	grabber = downloader.SourceDownloader(self.pakfire,
062699ee MT	228	mirrors=self.source_dl)
47a4cb89	229
062699ee	230	return grabber.download(self.sources)
c07a3ca7 MT	231
c07a3ca7 MT	232	def dist(self, resultdirs):
47a4cb89	233	"""
c07a3ca7	234	Create a source package.
c07a3ca7	235	"""
b33bcb02 MT	236	# Download all files we need for this package.
b33bcb02 MT	237	self.download()
c07a3ca7 MT	238
	239	p = packager.SourcePackager(self.pakfire, self)
	240	p.run(resultdirs)
	241
	242	def dump(self, args, *kwargs):
	243	dump = MakefileBase.dump(self, args, *kwargs)
	244	dump = dump.splitlines()
	245
	246	#dump += ["", _("Containing the following binary packages:"),]
	247	#
	248	#for pkg in self.packages:
	249	# _dump = pkg.dump(args, *kwargs)
	250	#
	251	# for line in _dump.splitlines():
	252	# dump.append(" %s" % line)
	253	# dump.append("")
	254
	255	return "\n".join(dump)
	256
	257	def get_buildscript(self, stage):
	258	return self.lexer.build.get_var("_%s" % stage)
	259
	260	@property
	261	def prerequires(self):
	262	return []
	263
	264	@property
	265	def requires(self):
f0618326	266	return self.lexer.build.get_var("requires", "").split()
c07a3ca7 MT	267
	268	@property
	269	def provides(self):
	270	return []
	271
	272	@property
	273	def obsoletes(self):
	274	return []
	275
	276	@property
	277	def conflicts(self):
	278	return []
	279
	280	@property
	281	def files(self):
	282	files = []
	283	basedir = os.path.dirname(self.filename)
	284
	285	for dirs, subdirs, _files in os.walk(basedir):
	286	for f in _files:
	287	files.append(os.path.join(dirs, f))
	288
	289	return files
	290
	291	@property
	292	def sources(self):
	293	return self.lexer.get_var("sources").split()
	294
3c45a6af MT	295	@property
	296	def exports(self):
	297	exports = {}
	298
4d5abec7 MT	299	# Include quality agent exports.
	300	exports.update(self.lexer.quality_agent.exports)
	301
3c45a6af MT	302	for export in self.lexer.build.exports:
	303	exports[export] = self.lexer.build.get_var(export)
	304
	305	return exports
	306
c07a3ca7 MT	307	def extract(self, message=None, prefix=None):
	308	# XXX neeed to make this waaaaaaaaaay better.
	309
	310	files = self.files
	311
	312	# Load progressbar.
	313	pb = None
	314	if message:
	315	message = "%-10s : %s" % (message, self.friendly_name)
	316	pb = util.make_progress(message, len(files), eta=False)
	317
	318	dir_len = len(os.path.dirname(self.filename))
	319
	320	# Copy all files that belong to the package
	321	i = 0
	322	for f in files:
	323	if pb:
	324	i += 1
	325	pb.update(i)
47a4cb89	326
c07a3ca7 MT	327	_f = f[dir_len:]
	328	logging.debug("%s/%s" % (prefix, _f))
	329
	330	path = "%s/%s" % (prefix, _f)
	331
	332	path_dir = os.path.dirname(path)
	333	if not os.path.exists(path_dir):
	334	os.makedirs(path_dir)
	335
	336	shutil.copy2(f, path)
	337
	338	if pb:
	339	pb.finish()
	340
	341	# Download source files.
062699ee	342	for _filename in self.download():
c07a3ca7 MT	343	filename = "%s/files/%s" % (prefix, os.path.basename(_filename))
	344	dirname = os.path.dirname(filename)
	345
	346	if not os.path.exists(dirname):
	347	os.makedirs(dirname)
062699ee	348
c07a3ca7 MT	349	shutil.copy2(_filename, filename)
	350
	351
	352	class MakefilePackage(MakefileBase):
	353	def __init__(self, pakfire, name, lexer):
	354	Package.__init__(self, pakfire)
	355
	356	self._name = name
	357	self.lexer = lexer
	358
5dda54e4 MT	359	# Store additional dependencies in here.
	360	self._dependencies = {}
	361
c07a3ca7 MT	362	@property
	363	def name(self):
	364	return self._name
	365
	366	@property
	367	def arch(self):
	368	return self.lexer.get_var("arch", "%{DISTRO_ARCH}")
	369
	370	@property
	371	def configfiles(self):
	372	return self.lexer.get_var("configfiles").split()
	373
	374	@property
	375	def files(self):
	376	return self.lexer.get_var("files").split()
	377
	378	@property
	379	def uuid(self):
	380	return None
	381
5dda54e4	382	def track_dependencies(self, builder, path):
d9211764 MT	383	# Dependency types.
	384	dep_types = ("prerequires", "requires", "provides", "conflicts", "obsoletes",)
	385
a1388014	386	result = builder.do("/usr/lib/pakfire/dependency-tracker %s" \
5dda54e4 MT	387	% path, returnOutput=True)
	388
	389	for line in result.splitlines():
	390	m = re.match(r"^(\w+)=(.*)$", line)
	391	if m is None:
	392	continue
	393
	394	key, val = m.groups()
	395
d9211764	396	if not key in dep_types:
5dda54e4 MT	397	continue
	398
	399	val = val.strip("\"")
	400	val = val.split()
	401
	402	self._dependencies[key] = sorted(val)
	403
d9211764 MT	404	# Filter dependencies.
	405	for key in dep_types:
	406	self._dependencies[key] = self.filter_deps(
	407	self._dependencies.get(key, []),
	408	self.lexer.get_var("filter_%s" % key)
	409	)
	410
	411	@staticmethod
	412	def filter_deps(deps, filters):
	413	if not filters:
	414	return deps
	415
	416	filters = filters.splitlines()
	417	filtered_deps = []
	418
	419	for dep in deps:
	420	filtered = False
	421	for filter in filters:
	422	m = re.search(filter, dep)
	423	if not m:
	424	continue
	425
	426	# Yes, we found a match.
	427	filtered = True
	428
	429	if not filtered:
	430	filtered_deps.append(dep)
	431
	432	return filtered_deps
	433
5dda54e4 MT	434	def get_deps(self, key):
	435	# Collect all dependencies that were set in the makefile by the user.
	436	deps = self.lexer.get_var(key).split()
	437
	438	# Collect all dependencies that were discovered by the tracker.
	439	deps += self._dependencies.get(key, [])
	440
	441	# Remove duplicates.
	442	deps = set(deps)
	443	deps = list(deps)
	444
	445	return sorted(deps)
c07a3ca7 MT	446
	447	@property
	448	def prerequires(self):
5dda54e4	449	return self.get_deps("prerequires")
c07a3ca7 MT	450
	451	@property
	452	def requires(self):
5dda54e4	453	return self.get_deps("requires")
c07a3ca7 MT	454
	455	@property
	456	def provides(self):
5dda54e4	457	return self.get_deps("provides")
c07a3ca7 MT	458
	459	@property
	460	def obsoletes(self):
5dda54e4	461	return self.get_deps("obsoletes")
c07a3ca7 MT	462
	463	@property
	464	def conflicts(self):
5dda54e4	465	return self.get_deps("conflicts")
47a4cb89	466
c07a3ca7	467	def get_scriptlet(self, type):
afb769e5	468	return self.lexer.get_scriptlet(type)
0304200a MT	469
	470	@property
	471	def inst_size(self):
	472	# The size of this is unknown.
	473	return 0