[pakfire.git] / python / pakfire / packages / make.py

#!/usr/bin/python
###############################################################################
#                                                                             #
# Pakfire - The IPFire package management system                              #
# Copyright (C) 2011 Pakfire development team                                 #
#                                                                             #
# This program is free software: you can redistribute it and/or modify        #
# it under the terms of the GNU General Public License as published by        #
# the Free Software Foundation, either version 3 of the License, or           #
# (at your option) any later version.                                         #
#                                                                             #
# This program is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
# GNU General Public License for more details.                                #
#                                                                             #
# You should have received a copy of the GNU General Public License           #
# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
#                                                                             #
###############################################################################

import logging
import os
import re
import shutil
import socket
import tarfile

from urlgrabber.grabber import URLGrabber, URLGrabError
from urlgrabber.progress import TextMeter

import lexer
import packager

import pakfire.chroot as chroot
import pakfire.downloader as downloader
import pakfire.util as util

from base import Package
from file import SourcePackage

from pakfire.constants import *
from pakfire.i18n import _

class MakefileBase(Package):
	def __init__(self, pakfire, filename):
		Package.__init__(self, pakfire)

		# Save the filename of the makefile.
		self.filename = os.path.abspath(filename)

		# Update environment.
		environ = self.pakfire.environ
		environ.update({
			"PARALLELISMFLAGS" : "-j%s" % util.calc_parallelism(),
		})

		# Open and parse the makefile.
		self.lexer = lexer.RootLexer.open(self.filename, environ=environ)

	@property
	def package_filename(self):
		return PACKAGE_FILENAME_FMT % {
			"arch"    : self.arch,
			"ext"     : PACKAGE_EXTENSION,
			"name"    : self.name,
			"release" : self.release,
			"version" : self.version,
		}

	def lint(self):
		errors = []

		if not self.name:
			errors.append(_("Package name is undefined."))

		if not self.version:
			errors.append(_("Package version is undefined."))

		# XXX to do...

		return errors

	@property
	def name(self):
		return self.lexer.get_var("name")

	@property
	def epoch(self):
		epoch = self.lexer.get_var("epoch")
		if not epoch:
			return 0

		return int(epoch)

	@property
	def version(self):
		return self.lexer.get_var("version")

	@property
	def release(self):
		release = self.lexer.get_var("release")
		assert release

		tag = self.lexer.get_var("DISTRO_DISTTAG")
		assert tag

		return ".".join((release, tag))

	@property
	def summary(self):
		return self.lexer.get_var("summary")

	@property
	def description(self):
		description = self.lexer.get_var("description")

		# Replace all backslashes at the end of a line.
		return description.replace("\\\n", "\n")

	@property
	def groups(self):
		groups = self.lexer.get_var("groups").split()

		return sorted(groups)

	@property
	def url(self):
		return self.lexer.get_var("url")

	@property
	def license(self):
		return self.lexer.get_var("license")

	@property
	def maintainer(self):
		maintainer = self.lexer.get_var("maintainer")

		if not maintainer:
			maintainer = self.lexer.get_var("DISTRO_MAINTAINER")

		return maintainer

	@property
	def vendor(self):
		return self.lexer.get_var("DISTRO_VENDOR")

	@property
	def buildroot(self):
		return self.lexer.get_var("BUILDROOT")

	@property
	def build_host(self):
		return socket.gethostname()

	# XXX build_id and build_time are used to create a source package

	@property
	def build_id(self):
		# XXX todo
		# Not existant for Makefiles
		return None

	@property
	def build_time(self):
		# XXX todo
		# Not existant for Makefiles
		return None


class Makefile(MakefileBase):
	@property
	def uuid(self):
		hash1 = util.calc_hash1(self.filename)

		# Return UUID version 5 (SHA1 hash)
		return "%8s-%4s-5%3s-%4s-%11s" % \
			(hash1[0:8], hash1[9:13], hash1[14:17], hash1[18:22], hash1[23:34])

	@property
	def path(self):
		return os.path.dirname(self.filename)

	@property
	def arch(self):
		"""
			This is only used to create the name of the source package.
		"""
		return "src"

	@property
	def packages(self):
		pkgs = []

		for lexer in self.lexer.packages:
			name = lexer.get_var("_name")

			pkg = MakefilePackage(self.pakfire, name, lexer)
			pkgs.append(pkg)

		return pkgs

	@property
	def source_dl(self):
		dls = []

		if self.pakfire.distro.source_dl:
			dls.append(self.pakfire.distro.source_dl)

		dl = self.lexer.get_var("source_dl")
		if dl:
			dls.append(dl)

		return dls

	def download(self):
		"""
			Download all external sources and return a list with the local
			copies.
		"""
		# Download source files.
		grabber = downloader.SourceDownloader(self.pakfire,
			mirrors=self.source_dl)

		return grabber.download(self.sources)

	def dist(self, resultdirs):
		"""
			Create a source package.
		"""
		# Download all files we need for this package.
		self.download()

		p = packager.SourcePackager(self.pakfire, self)
		p.run(resultdirs)

	def dump(self, *args, **kwargs):
		dump = MakefileBase.dump(self, *args, **kwargs)
		dump = dump.splitlines()

		#dump += ["", _("Containing the following binary packages:"),]
		#
		#for pkg in self.packages:
		#	_dump = pkg.dump(*args, **kwargs)
		#
		#	for line in _dump.splitlines():
		#		dump.append("  %s" % line)
		#	dump.append("")

		return "\n".join(dump)

	def get_buildscript(self, stage):
		return self.lexer.build.get_var("_%s" % stage)

	@property
	def prerequires(self):
		return []

	@property
	def requires(self):
		return self.lexer.build.get_var("requires", "").split()

	@property
	def provides(self):
		return []

	@property
	def obsoletes(self):
		return []

	@property
	def conflicts(self):
		return []

	@property
	def files(self):
		files = []
		basedir = os.path.dirname(self.filename)

		for dirs, subdirs, _files in os.walk(basedir):
			for f in _files:
				files.append(os.path.join(dirs, f))

		return files

	@property
	def sources(self):
		return self.lexer.get_var("sources").split()

	@property
	def exports(self):
		exports = {}

		# Include quality agent exports.
		exports.update(self.lexer.quality_agent.exports)

		for export in self.lexer.build.exports:
			exports[export] = self.lexer.build.get_var(export)

		return exports

	def extract(self, message=None, prefix=None):
		# XXX neeed to make this waaaaaaaaaay better.

		files = self.files

		# Load progressbar.
		pb = None
		if message:
			message = "%-10s : %s" % (message, self.friendly_name)
			pb = util.make_progress(message, len(files), eta=False)

		dir_len = len(os.path.dirname(self.filename))

		# Copy all files that belong to the package
		i = 0
		for f in files:
			if pb:
				i += 1
				pb.update(i)

			_f = f[dir_len:]
			logging.debug("%s/%s" % (prefix, _f))
	
			path = "%s/%s" % (prefix, _f)

			path_dir = os.path.dirname(path)
			if not os.path.exists(path_dir):
				os.makedirs(path_dir)

			shutil.copy2(f, path)

		if pb:
			pb.finish()

		# Download source files.
		for _filename in self.download():
			filename = "%s/files/%s" % (prefix, os.path.basename(_filename))
			dirname = os.path.dirname(filename)

			if not os.path.exists(dirname):
				os.makedirs(dirname)

			shutil.copy2(_filename, filename)


class MakefilePackage(MakefileBase):
	def __init__(self, pakfire, name, lexer):
		Package.__init__(self, pakfire)

		self._name = name
		self.lexer = lexer

		# Store additional dependencies in here.
		self._dependencies = {}

	@property
	def name(self):
		return self._name

	@property
	def arch(self):
		return self.lexer.get_var("arch", "%{DISTRO_ARCH}")

	@property
	def configfiles(self):
		return self.lexer.get_var("configfiles").split()

	@property
	def files(self):
		return self.lexer.get_var("files").split()

	@property
	def uuid(self):
		return None

	def track_dependencies(self, builder, path):
		result = builder.do("/usr/lib/pakfire/dependency-tracker %s" \
			% path, returnOutput=True)

		for line in result.splitlines():
			m = re.match(r"^(\w+)=(.*)$", line)
			if m is None:
				continue

			key, val = m.groups()

			if not key in ("prerequires", "requires", "provides", "conflicts", "obsoletes",):
				continue

			val = val.strip("\"")
			val = val.split()

			self._dependencies[key] = sorted(val)

	def get_deps(self, key):
		# Collect all dependencies that were set in the makefile by the user.
		deps = self.lexer.get_var(key).split()

		# Collect all dependencies that were discovered by the tracker.
		deps += self._dependencies.get(key, [])

		# Remove duplicates.
		deps = set(deps)
		deps = list(deps)

		return sorted(deps)

	@property
	def prerequires(self):
		return self.get_deps("prerequires")

	@property
	def requires(self):
		return self.get_deps("requires")

	@property
	def provides(self):
		return self.get_deps("provides")

	@property
	def obsoletes(self):
		return self.get_deps("obsoletes")

	@property
	def conflicts(self):
		return self.get_deps("conflicts")

	def get_scriptlet(self, type):
		scriptlet = self.lexer.scriptlets.get(type, None)

		if scriptlet is None and self.lexer.template:
			scriptlet = self.lexer.template.scriptlets.get(type, None)

		return scriptlet

	@property
	def inst_size(self):
		# The size of this is unknown.
		return 0
Commit	Line	Data
47a4cb89	1	#!/usr/bin/python
b792d887 MT	2	###############################################################################
	3	# #
	4	# Pakfire - The IPFire package management system #
	5	# Copyright (C) 2011 Pakfire development team #
	6	# #
	7	# This program is free software: you can redistribute it and/or modify #
	8	# it under the terms of the GNU General Public License as published by #
	9	# the Free Software Foundation, either version 3 of the License, or #
	10	# (at your option) any later version. #
	11	# #
	12	# This program is distributed in the hope that it will be useful, #
	13	# but WITHOUT ANY WARRANTY; without even the implied warranty of #
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
	15	# GNU General Public License for more details. #
	16	# #
	17	# You should have received a copy of the GNU General Public License #
	18	# along with this program. If not, see <http://www.gnu.org/licenses/>. #
	19	# #
	20	###############################################################################
47a4cb89	21
c07a3ca7	22	import logging
47a4cb89	23	import os
5dda54e4	24	import re
c07a3ca7 MT	25	import shutil
c07a3ca7 MT	26	import socket
47a4cb89 MT	27	import tarfile
	28
	29	from urlgrabber.grabber import URLGrabber, URLGrabError
	30	from urlgrabber.progress import TextMeter
	31
c07a3ca7	32	import lexer
47a4cb89 MT	33	import packager
47a4cb89 MT	34
5dda54e4	35	import pakfire.chroot as chroot
062699ee	36	import pakfire.downloader as downloader
c07a3ca7 MT	37	import pakfire.util as util
c07a3ca7 MT	38
47a4cb89	39	from base import Package
e0636b31 MT	40	from file import SourcePackage
e0636b31 MT	41
47a4cb89	42	from pakfire.constants import *
c07a3ca7	43	from pakfire.i18n import _
47a4cb89	44
c07a3ca7	45	class MakefileBase(Package):
3723913b	46	def __init__(self, pakfire, filename):
a2d1644c	47	Package.__init__(self, pakfire)
47a4cb89	48
c07a3ca7 MT	49	# Save the filename of the makefile.
c07a3ca7 MT	50	self.filename = os.path.abspath(filename)
47a4cb89	51
440f3cb8 MT	52	# Update environment.
	53	environ = self.pakfire.environ
	54	environ.update({
	55	"PARALLELISMFLAGS" : "-j%s" % util.calc_parallelism(),
	56	})
	57
c07a3ca7	58	# Open and parse the makefile.
440f3cb8	59	self.lexer = lexer.RootLexer.open(self.filename, environ=environ)
47a4cb89 MT	60
	61	@property
	62	def package_filename(self):
	63	return PACKAGE_FILENAME_FMT % {
	64	"arch" : self.arch,
	65	"ext" : PACKAGE_EXTENSION,
	66	"name" : self.name,
	67	"release" : self.release,
	68	"version" : self.version,
	69	}
	70
c07a3ca7 MT	71	def lint(self):
	72	errors = []
	73
	74	if not self.name:
	75	errors.append(_("Package name is undefined."))
	76
	77	if not self.version:
	78	errors.append(_("Package version is undefined."))
	79
	80	# XXX to do...
	81
	82	return errors
	83
	84	@property
	85	def name(self):
	86	return self.lexer.get_var("name")
	87
	88	@property
	89	def epoch(self):
	90	epoch = self.lexer.get_var("epoch")
	91	if not epoch:
	92	return 0
	93
	94	return int(epoch)
	95
	96	@property
	97	def version(self):
	98	return self.lexer.get_var("version")
	99
	100	@property
	101	def release(self):
	102	release = self.lexer.get_var("release")
	103	assert release
	104
	105	tag = self.lexer.get_var("DISTRO_DISTTAG")
	106	assert tag
	107
	108	return ".".join((release, tag))
	109
	110	@property
	111	def summary(self):
	112	return self.lexer.get_var("summary")
	113
	114	@property
	115	def description(self):
c4c528f7 MT	116	description = self.lexer.get_var("description")
	117
	118	# Replace all backslashes at the end of a line.
	119	return description.replace("\\\n", "\n")
c07a3ca7 MT	120
	121	@property
	122	def groups(self):
	123	groups = self.lexer.get_var("groups").split()
	124
	125	return sorted(groups)
	126
	127	@property
	128	def url(self):
	129	return self.lexer.get_var("url")
	130
	131	@property
	132	def license(self):
	133	return self.lexer.get_var("license")
	134
	135	@property
	136	def maintainer(self):
	137	maintainer = self.lexer.get_var("maintainer")
	138
	139	if not maintainer:
	140	maintainer = self.lexer.get_var("DISTRO_MAINTAINER")
	141
	142	return maintainer
	143
	144	@property
	145	def vendor(self):
	146	return self.lexer.get_var("DISTRO_VENDOR")
	147
0d96815e MT	148	@property
	149	def buildroot(self):
	150	return self.lexer.get_var("BUILDROOT")
	151
c07a3ca7 MT	152	@property
	153	def build_host(self):
	154	return socket.gethostname()
	155
	156	# XXX build_id and build_time are used to create a source package
	157
	158	@property
	159	def build_id(self):
	160	# XXX todo
	161	# Not existant for Makefiles
	162	return None
	163
	164	@property
	165	def build_time(self):
	166	# XXX todo
	167	# Not existant for Makefiles
	168	return None
	169
	170
	171	class Makefile(MakefileBase):
	172	@property
	173	def uuid(self):
	174	hash1 = util.calc_hash1(self.filename)
	175
	176	# Return UUID version 5 (SHA1 hash)
	177	return "%8s-%4s-5%3s-%4s-%11s" % \
	178	(hash1[0:8], hash1[9:13], hash1[14:17], hash1[18:22], hash1[23:34])
	179
	180	@property
	181	def path(self):
	182	return os.path.dirname(self.filename)
	183
47a4cb89 MT	184	@property
	185	def arch(self):
	186	"""
	187	This is only used to create the name of the source package.
	188	"""
	189	return "src"
	190
c07a3ca7 MT	191	@property
	192	def packages(self):
	193	pkgs = []
	194
	195	for lexer in self.lexer.packages:
f0618326	196	name = lexer.get_var("_name")
c07a3ca7 MT	197
	198	pkg = MakefilePackage(self.pakfire, name, lexer)
	199	pkgs.append(pkg)
	200
afe4e55c	201	return pkgs
c07a3ca7 MT	202
	203	@property
	204	def source_dl(self):
	205	dls = []
	206
	207	if self.pakfire.distro.source_dl:
	208	dls.append(self.pakfire.distro.source_dl)
	209
	210	dl = self.lexer.get_var("source_dl")
	211	if dl:
	212	dls.append(dl)
	213
	214	return dls
	215
	216	def download(self):
	217	"""
	218	Download all external sources and return a list with the local
	219	copies.
47a4cb89	220	"""
c07a3ca7	221	# Download source files.
062699ee MT	222	grabber = downloader.SourceDownloader(self.pakfire,
062699ee MT	223	mirrors=self.source_dl)
47a4cb89	224
062699ee	225	return grabber.download(self.sources)
c07a3ca7 MT	226
c07a3ca7 MT	227	def dist(self, resultdirs):
47a4cb89	228	"""
c07a3ca7	229	Create a source package.
c07a3ca7	230	"""
b33bcb02 MT	231	# Download all files we need for this package.
b33bcb02 MT	232	self.download()
c07a3ca7 MT	233
	234	p = packager.SourcePackager(self.pakfire, self)
	235	p.run(resultdirs)
	236
	237	def dump(self, args, *kwargs):
	238	dump = MakefileBase.dump(self, args, *kwargs)
	239	dump = dump.splitlines()
	240
	241	#dump += ["", _("Containing the following binary packages:"),]
	242	#
	243	#for pkg in self.packages:
	244	# _dump = pkg.dump(args, *kwargs)
	245	#
	246	# for line in _dump.splitlines():
	247	# dump.append(" %s" % line)
	248	# dump.append("")
	249
	250	return "\n".join(dump)
	251
	252	def get_buildscript(self, stage):
	253	return self.lexer.build.get_var("_%s" % stage)
	254
	255	@property
	256	def prerequires(self):
	257	return []
	258
	259	@property
	260	def requires(self):
f0618326	261	return self.lexer.build.get_var("requires", "").split()
c07a3ca7 MT	262
	263	@property
	264	def provides(self):
	265	return []
	266
	267	@property
	268	def obsoletes(self):
	269	return []
	270
	271	@property
	272	def conflicts(self):
	273	return []
	274
	275	@property
	276	def files(self):
	277	files = []
	278	basedir = os.path.dirname(self.filename)
	279
	280	for dirs, subdirs, _files in os.walk(basedir):
	281	for f in _files:
	282	files.append(os.path.join(dirs, f))
	283
	284	return files
	285
	286	@property
	287	def sources(self):
	288	return self.lexer.get_var("sources").split()
	289
3c45a6af MT	290	@property
	291	def exports(self):
	292	exports = {}
	293
4d5abec7 MT	294	# Include quality agent exports.
	295	exports.update(self.lexer.quality_agent.exports)
	296
3c45a6af MT	297	for export in self.lexer.build.exports:
	298	exports[export] = self.lexer.build.get_var(export)
	299
	300	return exports
	301
c07a3ca7 MT	302	def extract(self, message=None, prefix=None):
	303	# XXX neeed to make this waaaaaaaaaay better.
	304
	305	files = self.files
	306
	307	# Load progressbar.
	308	pb = None
	309	if message:
	310	message = "%-10s : %s" % (message, self.friendly_name)
	311	pb = util.make_progress(message, len(files), eta=False)
	312
	313	dir_len = len(os.path.dirname(self.filename))
	314
	315	# Copy all files that belong to the package
	316	i = 0
	317	for f in files:
	318	if pb:
	319	i += 1
	320	pb.update(i)
47a4cb89	321
c07a3ca7 MT	322	_f = f[dir_len:]
	323	logging.debug("%s/%s" % (prefix, _f))
	324
	325	path = "%s/%s" % (prefix, _f)
	326
	327	path_dir = os.path.dirname(path)
	328	if not os.path.exists(path_dir):
	329	os.makedirs(path_dir)
	330
	331	shutil.copy2(f, path)
	332
	333	if pb:
	334	pb.finish()
	335
	336	# Download source files.
062699ee	337	for _filename in self.download():
c07a3ca7 MT	338	filename = "%s/files/%s" % (prefix, os.path.basename(_filename))
	339	dirname = os.path.dirname(filename)
	340
	341	if not os.path.exists(dirname):
	342	os.makedirs(dirname)
062699ee	343
c07a3ca7 MT	344	shutil.copy2(_filename, filename)
	345
	346
	347	class MakefilePackage(MakefileBase):
	348	def __init__(self, pakfire, name, lexer):
	349	Package.__init__(self, pakfire)
	350
	351	self._name = name
	352	self.lexer = lexer
	353
5dda54e4 MT	354	# Store additional dependencies in here.
	355	self._dependencies = {}
	356
c07a3ca7 MT	357	@property
	358	def name(self):
	359	return self._name
	360
	361	@property
	362	def arch(self):
	363	return self.lexer.get_var("arch", "%{DISTRO_ARCH}")
	364
	365	@property
	366	def configfiles(self):
	367	return self.lexer.get_var("configfiles").split()
	368
	369	@property
	370	def files(self):
	371	return self.lexer.get_var("files").split()
	372
	373	@property
	374	def uuid(self):
	375	return None
	376
5dda54e4	377	def track_dependencies(self, builder, path):
a1388014	378	result = builder.do("/usr/lib/pakfire/dependency-tracker %s" \
5dda54e4 MT	379	% path, returnOutput=True)
	380
	381	for line in result.splitlines():
	382	m = re.match(r"^(\w+)=(.*)$", line)
	383	if m is None:
	384	continue
	385
	386	key, val = m.groups()
	387
	388	if not key in ("prerequires", "requires", "provides", "conflicts", "obsoletes",):
	389	continue
	390
	391	val = val.strip("\"")
	392	val = val.split()
	393
	394	self._dependencies[key] = sorted(val)
	395
	396	def get_deps(self, key):
	397	# Collect all dependencies that were set in the makefile by the user.
	398	deps = self.lexer.get_var(key).split()
	399
	400	# Collect all dependencies that were discovered by the tracker.
	401	deps += self._dependencies.get(key, [])
	402
	403	# Remove duplicates.
	404	deps = set(deps)
	405	deps = list(deps)
	406
	407	return sorted(deps)
c07a3ca7 MT	408
	409	@property
	410	def prerequires(self):
5dda54e4	411	return self.get_deps("prerequires")
c07a3ca7 MT	412
	413	@property
	414	def requires(self):
5dda54e4	415	return self.get_deps("requires")
c07a3ca7 MT	416
	417	@property
	418	def provides(self):
5dda54e4	419	return self.get_deps("provides")
c07a3ca7 MT	420
	421	@property
	422	def obsoletes(self):
5dda54e4	423	return self.get_deps("obsoletes")
c07a3ca7 MT	424
	425	@property
	426	def conflicts(self):
5dda54e4	427	return self.get_deps("conflicts")
47a4cb89	428
c07a3ca7	429	def get_scriptlet(self, type):
e360b2c8 MT	430	scriptlet = self.lexer.scriptlets.get(type, None)
	431
	432	if scriptlet is None and self.lexer.template:
	433	scriptlet = self.lexer.template.scriptlets.get(type, None)
	434
	435	return scriptlet
0304200a MT	436
	437	@property
	438	def inst_size(self):
	439	# The size of this is unknown.
	440	return 0