[pakfire.git] / pakfire / packages / make.py

#!/usr/bin/python
###############################################################################
#                                                                             #
# Pakfire - The IPFire package management system                              #
# Copyright (C) 2011 Pakfire development team                                 #
#                                                                             #
# This program is free software: you can redistribute it and/or modify        #
# it under the terms of the GNU General Public License as published by        #
# the Free Software Foundation, either version 3 of the License, or           #
# (at your option) any later version.                                         #
#                                                                             #
# This program is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
# GNU General Public License for more details.                                #
#                                                                             #
# You should have received a copy of the GNU General Public License           #
# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
#                                                                             #
###############################################################################

import logging
import os
import re
import shutil
import socket
import tarfile

from urlgrabber.grabber import URLGrabber, URLGrabError
from urlgrabber.progress import TextMeter

import lexer
import packager

import pakfire.chroot as chroot
import pakfire.util as util

from base import Package
from file import SourcePackage

from pakfire.constants import *
from pakfire.i18n import _

# XXX to be moved to pakfire.downloader
class SourceDownloader(object):
	def __init__(self, pakfire, mirrors=None):
		self.pakfire = pakfire
		self.mirrors = mirrors

		# XXX need to use downloader.py
		self.grabber = URLGrabber(
			prefix = self.pakfire.config.get("source_download_url"),
			progress_obj = TextMeter(),
			quote = 0,
		)

	def download(self, filename):
		filename = os.path.join(SOURCE_CACHE_DIR, filename)

		if os.path.exists(filename):
			return filename

		dirname = os.path.dirname(filename)
		if not os.path.exists(dirname):
			os.makedirs(dirname)

		try:
			self.grabber.urlgrab(os.path.basename(filename), filename=filename)
		except URLGrabError, e:
			raise DownloadError, "%s %s" % (os.path.basename(filename), e)

		return filename


class MakefileBase(Package):
	def __init__(self, pakfire, filename):
		Package.__init__(self, pakfire)

		# Save the filename of the makefile.
		self.filename = os.path.abspath(filename)

		# Open and parse the makefile.
		# XXX pass environment to lexer
		self.lexer = lexer.RootLexer.open(self.filename,
			environ=self.pakfire.environ)

	@property
	def package_filename(self):
		return PACKAGE_FILENAME_FMT % {
			"arch"    : self.arch,
			"ext"     : PACKAGE_EXTENSION,
			"name"    : self.name,
			"release" : self.release,
			"version" : self.version,
		}

	def lint(self):
		errors = []

		if not self.name:
			errors.append(_("Package name is undefined."))

		if not self.version:
			errors.append(_("Package version is undefined."))

		# XXX to do...

		return errors

	@property
	def name(self):
		return self.lexer.get_var("name")

	@property
	def epoch(self):
		epoch = self.lexer.get_var("epoch")
		if not epoch:
			return 0

		return int(epoch)

	@property
	def version(self):
		return self.lexer.get_var("version")

	@property
	def release(self):
		release = self.lexer.get_var("release")
		assert release

		tag = self.lexer.get_var("DISTRO_DISTTAG")
		assert tag

		return ".".join((release, tag))

	@property
	def summary(self):
		return self.lexer.get_var("summary")

	@property
	def description(self):
		return self.lexer.get_var("description")

	@property
	def groups(self):
		groups = self.lexer.get_var("groups").split()

		return sorted(groups)

	@property
	def url(self):
		return self.lexer.get_var("url")

	@property
	def license(self):
		return self.lexer.get_var("license")

	@property
	def maintainer(self):
		maintainer = self.lexer.get_var("maintainer")

		if not maintainer:
			maintainer = self.lexer.get_var("DISTRO_MAINTAINER")

		return maintainer

	@property
	def vendor(self):
		return self.lexer.get_var("DISTRO_VENDOR")

	@property
	def build_host(self):
		return socket.gethostname()

	# XXX build_id and build_time are used to create a source package

	@property
	def build_id(self):
		# XXX todo
		# Not existant for Makefiles
		return None

	@property
	def build_time(self):
		# XXX todo
		# Not existant for Makefiles
		return None


class Makefile(MakefileBase):
	@property
	def uuid(self):
		hash1 = util.calc_hash1(self.filename)

		# Return UUID version 5 (SHA1 hash)
		return "%8s-%4s-5%3s-%4s-%11s" % \
			(hash1[0:8], hash1[9:13], hash1[14:17], hash1[18:22], hash1[23:34])

	@property
	def path(self):
		return os.path.dirname(self.filename)

	@property
	def arch(self):
		"""
			This is only used to create the name of the source package.
		"""
		return "src"

	@property
	def packages(self):
		pkgs = []

		for lexer in self.lexer.packages:
			name = lexer.get_var("_name")

			pkg = MakefilePackage(self.pakfire, name, lexer)
			pkgs.append(pkg)

		return pkgs

	@property
	def source_dl(self):
		dls = []

		if self.pakfire.distro.source_dl:
			dls.append(self.pakfire.distro.source_dl)

		dl = self.lexer.get_var("source_dl")
		if dl:
			dls.append(dl)

		return dls

	def download(self):
		"""
			Download all external sources and return a list with the local
			copies.
		"""
		# Download source files.
		# XXX need to implement mirrors
		downloader = SourceDownloader(self.pakfire, mirrors=self.source_dl)

		files = []
		for filename in self.sources:
			filename = downloader.download(filename)
			files.append(filename)

		return files

	def dist(self, resultdirs):
		"""
			Create a source package.

			We assume that all required files are in /build.
		"""
		#dump = self.dump()
		#for line in dump.splitlines():
		#	logging.info(line)

		p = packager.SourcePackager(self.pakfire, self)
		p.run(resultdirs)

	def dump(self, *args, **kwargs):
		dump = MakefileBase.dump(self, *args, **kwargs)
		dump = dump.splitlines()

		#dump += ["", _("Containing the following binary packages:"),]
		#
		#for pkg in self.packages:
		#	_dump = pkg.dump(*args, **kwargs)
		#
		#	for line in _dump.splitlines():
		#		dump.append("  %s" % line)
		#	dump.append("")

		return "\n".join(dump)

	def get_buildscript(self, stage):
		return self.lexer.build.get_var("_%s" % stage)

	@property
	def prerequires(self):
		return []

	@property
	def requires(self):
		return self.lexer.build.get_var("requires", "").split()

	@property
	def provides(self):
		return []

	@property
	def obsoletes(self):
		return []

	@property
	def conflicts(self):
		return []

	@property
	def files(self):
		files = []
		basedir = os.path.dirname(self.filename)

		for dirs, subdirs, _files in os.walk(basedir):
			for f in _files:
				files.append(os.path.join(dirs, f))

		return files

	@property
	def sources(self):
		return self.lexer.get_var("sources").split()

	def extract(self, message=None, prefix=None):
		# XXX neeed to make this waaaaaaaaaay better.

		files = self.files

		# Load progressbar.
		pb = None
		if message:
			message = "%-10s : %s" % (message, self.friendly_name)
			pb = util.make_progress(message, len(files), eta=False)

		dir_len = len(os.path.dirname(self.filename))

		# Copy all files that belong to the package
		i = 0
		for f in files:
			if pb:
				i += 1
				pb.update(i)

			_f = f[dir_len:]
			logging.debug("%s/%s" % (prefix, _f))
	
			path = "%s/%s" % (prefix, _f)

			path_dir = os.path.dirname(path)
			if not os.path.exists(path_dir):
				os.makedirs(path_dir)

			shutil.copy2(f, path)

		if pb:
			pb.finish()

		# Download source files.
		downloader = SourceDownloader(self.pakfire, mirrors=self.source_dl)
		for filename in self.sources:
			_filename = downloader.download(filename)
			assert _filename

			filename = "%s/files/%s" % (prefix, os.path.basename(_filename))
			dirname = os.path.dirname(filename)

			if not os.path.exists(dirname):
				os.makedirs(dirname)
				
			shutil.copy2(_filename, filename)


class MakefilePackage(MakefileBase):
	def __init__(self, pakfire, name, lexer):
		Package.__init__(self, pakfire)

		self._name = name
		self.lexer = lexer

		# Store additional dependencies in here.
		self._dependencies = {}

	@property
	def name(self):
		return self._name

	@property
	def arch(self):
		return self.lexer.get_var("arch", "%{DISTRO_ARCH}")

	@property
	def configfiles(self):
		return self.lexer.get_var("configfiles").split()

	@property
	def files(self):
		return self.lexer.get_var("files").split()

	@property
	def uuid(self):
		return None

	def track_dependencies(self, builder, path):
		result = builder.do("/usr/lib/buildsystem-tools/dependency-tracker %s" \
			% path, returnOutput=True)

		for line in result.splitlines():
			m = re.match(r"^(\w+)=(.*)$", line)
			if m is None:
				continue

			key, val = m.groups()

			if not key in ("prerequires", "requires", "provides", "conflicts", "obsoletes",):
				continue

			val = val.strip("\"")
			val = val.split()

			self._dependencies[key] = sorted(val)

	def get_deps(self, key):
		# Collect all dependencies that were set in the makefile by the user.
		deps = self.lexer.get_var(key).split()

		# Collect all dependencies that were discovered by the tracker.
		deps += self._dependencies.get(key, [])

		# Remove duplicates.
		deps = set(deps)
		deps = list(deps)

		return sorted(deps)

	@property
	def prerequires(self):
		return self.get_deps("prerequires")

	@property
	def requires(self):
		return self.get_deps("requires")

	@property
	def provides(self):
		return self.get_deps("provides")

	@property
	def obsoletes(self):
		return self.get_deps("obsoletes")

	@property
	def conflicts(self):
		return self.get_deps("conflicts")

	def get_scriptlet(self, type):
		return self.lexer.scriptlets.get(type, None)

	@property
	def inst_size(self):
		# The size of this is unknown.
		return 0
Commit	Line	Data
47a4cb89	1	#!/usr/bin/python
b792d887 MT	2	###############################################################################
	3	# #
	4	# Pakfire - The IPFire package management system #
	5	# Copyright (C) 2011 Pakfire development team #
	6	# #
	7	# This program is free software: you can redistribute it and/or modify #
	8	# it under the terms of the GNU General Public License as published by #
	9	# the Free Software Foundation, either version 3 of the License, or #
	10	# (at your option) any later version. #
	11	# #
	12	# This program is distributed in the hope that it will be useful, #
	13	# but WITHOUT ANY WARRANTY; without even the implied warranty of #
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
	15	# GNU General Public License for more details. #
	16	# #
	17	# You should have received a copy of the GNU General Public License #
	18	# along with this program. If not, see <http://www.gnu.org/licenses/>. #
	19	# #
	20	###############################################################################
47a4cb89	21
c07a3ca7	22	import logging
47a4cb89	23	import os
5dda54e4	24	import re
c07a3ca7 MT	25	import shutil
c07a3ca7 MT	26	import socket
47a4cb89 MT	27	import tarfile
	28
	29	from urlgrabber.grabber import URLGrabber, URLGrabError
	30	from urlgrabber.progress import TextMeter
	31
c07a3ca7	32	import lexer
47a4cb89 MT	33	import packager
47a4cb89 MT	34
5dda54e4	35	import pakfire.chroot as chroot
c07a3ca7 MT	36	import pakfire.util as util
c07a3ca7 MT	37
47a4cb89	38	from base import Package
e0636b31 MT	39	from file import SourcePackage
e0636b31 MT	40
47a4cb89	41	from pakfire.constants import *
c07a3ca7	42	from pakfire.i18n import _
47a4cb89	43
c07a3ca7	44	# XXX to be moved to pakfire.downloader
47a4cb89	45	class SourceDownloader(object):
c07a3ca7	46	def __init__(self, pakfire, mirrors=None):
47a4cb89	47	self.pakfire = pakfire
c07a3ca7	48	self.mirrors = mirrors
47a4cb89	49
80104a80	50	# XXX need to use downloader.py
47a4cb89 MT	51	self.grabber = URLGrabber(
	52	prefix = self.pakfire.config.get("source_download_url"),
	53	progress_obj = TextMeter(),
	54	quote = 0,
	55	)
	56
	57	def download(self, filename):
	58	filename = os.path.join(SOURCE_CACHE_DIR, filename)
	59
	60	if os.path.exists(filename):
	61	return filename
	62
	63	dirname = os.path.dirname(filename)
	64	if not os.path.exists(dirname):
	65	os.makedirs(dirname)
	66
	67	try:
	68	self.grabber.urlgrab(os.path.basename(filename), filename=filename)
	69	except URLGrabError, e:
	70	raise DownloadError, "%s %s" % (os.path.basename(filename), e)
	71
	72	return filename
	73
	74
c07a3ca7	75	class MakefileBase(Package):
3723913b	76	def __init__(self, pakfire, filename):
a2d1644c	77	Package.__init__(self, pakfire)
47a4cb89	78
c07a3ca7 MT	79	# Save the filename of the makefile.
c07a3ca7 MT	80	self.filename = os.path.abspath(filename)
47a4cb89	81
c07a3ca7 MT	82	# Open and parse the makefile.
	83	# XXX pass environment to lexer
	84	self.lexer = lexer.RootLexer.open(self.filename,
	85	environ=self.pakfire.environ)
47a4cb89 MT	86
	87	@property
	88	def package_filename(self):
	89	return PACKAGE_FILENAME_FMT % {
	90	"arch" : self.arch,
	91	"ext" : PACKAGE_EXTENSION,
	92	"name" : self.name,
	93	"release" : self.release,
	94	"version" : self.version,
	95	}
	96
c07a3ca7 MT	97	def lint(self):
	98	errors = []
	99
	100	if not self.name:
	101	errors.append(_("Package name is undefined."))
	102
	103	if not self.version:
	104	errors.append(_("Package version is undefined."))
	105
	106	# XXX to do...
	107
	108	return errors
	109
	110	@property
	111	def name(self):
	112	return self.lexer.get_var("name")
	113
	114	@property
	115	def epoch(self):
	116	epoch = self.lexer.get_var("epoch")
	117	if not epoch:
	118	return 0
	119
	120	return int(epoch)
	121
	122	@property
	123	def version(self):
	124	return self.lexer.get_var("version")
	125
	126	@property
	127	def release(self):
	128	release = self.lexer.get_var("release")
	129	assert release
	130
	131	tag = self.lexer.get_var("DISTRO_DISTTAG")
	132	assert tag
	133
	134	return ".".join((release, tag))
	135
	136	@property
	137	def summary(self):
	138	return self.lexer.get_var("summary")
	139
	140	@property
	141	def description(self):
	142	return self.lexer.get_var("description")
	143
	144	@property
	145	def groups(self):
	146	groups = self.lexer.get_var("groups").split()
	147
	148	return sorted(groups)
	149
	150	@property
	151	def url(self):
	152	return self.lexer.get_var("url")
	153
	154	@property
	155	def license(self):
	156	return self.lexer.get_var("license")
	157
	158	@property
	159	def maintainer(self):
	160	maintainer = self.lexer.get_var("maintainer")
161
162	if not maintainer:
163	maintainer = self.lexer.get_var("DISTRO_MAINTAINER")
164
165	return maintainer
166
167	@property
168	def vendor(self):
169	return self.lexer.get_var("DISTRO_VENDOR")
170
171	@property
172	def build_host(self):
173	return socket.gethostname()
174
175	# XXX build_id and build_time are used to create a source package
176
177	@property
178	def build_id(self):
179	# XXX todo
180	# Not existant for Makefiles
181	return None
182
183	@property
184	def build_time(self):
185	# XXX todo
186	# Not existant for Makefiles
187	return None
188
189
190	class Makefile(MakefileBase):
191	@property
192	def uuid(self):
193	hash1 = util.calc_hash1(self.filename)
194
195	# Return UUID version 5 (SHA1 hash)
196	return "%8s-%4s-5%3s-%4s-%11s" % \
197	(hash1[0:8], hash1[9:13], hash1[14:17], hash1[18:22], hash1[23:34])
198
199	@property
200	def path(self):
201	return os.path.dirname(self.filename)
202
47a4cb89 MT	203	@property
	204	def arch(self):
	205	"""
	206	This is only used to create the name of the source package.
	207	"""
	208	return "src"
	209
c07a3ca7 MT	210	@property
	211	def packages(self):
	212	pkgs = []
	213
	214	for lexer in self.lexer.packages:
f0618326	215	name = lexer.get_var("_name")
c07a3ca7 MT	216
	217	pkg = MakefilePackage(self.pakfire, name, lexer)
	218	pkgs.append(pkg)
	219
afe4e55c	220	return pkgs
c07a3ca7 MT	221
	222	@property
	223	def source_dl(self):
	224	dls = []
	225
	226	if self.pakfire.distro.source_dl:
	227	dls.append(self.pakfire.distro.source_dl)
	228
	229	dl = self.lexer.get_var("source_dl")
	230	if dl:
	231	dls.append(dl)
	232
	233	return dls
	234
	235	def download(self):
	236	"""
	237	Download all external sources and return a list with the local
	238	copies.
47a4cb89	239	"""
c07a3ca7 MT	240	# Download source files.
	241	# XXX need to implement mirrors
	242	downloader = SourceDownloader(self.pakfire, mirrors=self.source_dl)
	243
	244	files = []
	245	for filename in self.sources:
	246	filename = downloader.download(filename)
	247	files.append(filename)
47a4cb89	248
c07a3ca7 MT	249	return files
	250
	251	def dist(self, resultdirs):
47a4cb89	252	"""
c07a3ca7 MT	253	Create a source package.
	254
	255	We assume that all required files are in /build.
	256	"""
	257	#dump = self.dump()
	258	#for line in dump.splitlines():
	259	# logging.info(line)
	260
	261	p = packager.SourcePackager(self.pakfire, self)
	262	p.run(resultdirs)
	263
	264	def dump(self, args, *kwargs):
	265	dump = MakefileBase.dump(self, args, *kwargs)
	266	dump = dump.splitlines()
	267
	268	#dump += ["", _("Containing the following binary packages:"),]
	269	#
	270	#for pkg in self.packages:
	271	# _dump = pkg.dump(args, *kwargs)
	272	#
	273	# for line in _dump.splitlines():
	274	# dump.append(" %s" % line)
	275	# dump.append("")
	276
	277	return "\n".join(dump)
	278
	279	def get_buildscript(self, stage):
	280	return self.lexer.build.get_var("_%s" % stage)
	281
	282	@property
	283	def prerequires(self):
	284	return []
	285
	286	@property
	287	def requires(self):
f0618326	288	return self.lexer.build.get_var("requires", "").split()
c07a3ca7 MT	289
	290	@property
	291	def provides(self):
	292	return []
	293
	294	@property
	295	def obsoletes(self):
	296	return []
	297
	298	@property
	299	def conflicts(self):
	300	return []
	301
	302	@property
	303	def files(self):
	304	files = []
	305	basedir = os.path.dirname(self.filename)
	306
	307	for dirs, subdirs, _files in os.walk(basedir):
	308	for f in _files:
	309	files.append(os.path.join(dirs, f))
	310
	311	return files
	312
	313	@property
	314	def sources(self):
	315	return self.lexer.get_var("sources").split()
	316
	317	def extract(self, message=None, prefix=None):
	318	# XXX neeed to make this waaaaaaaaaay better.
	319
	320	files = self.files
	321
	322	# Load progressbar.
	323	pb = None
	324	if message:
	325	message = "%-10s : %s" % (message, self.friendly_name)
	326	pb = util.make_progress(message, len(files), eta=False)
	327
	328	dir_len = len(os.path.dirname(self.filename))
	329
	330	# Copy all files that belong to the package
	331	i = 0
	332	for f in files:
	333	if pb:
	334	i += 1
	335	pb.update(i)
47a4cb89	336
c07a3ca7 MT	337	_f = f[dir_len:]
	338	logging.debug("%s/%s" % (prefix, _f))
	339
	340	path = "%s/%s" % (prefix, _f)
	341
	342	path_dir = os.path.dirname(path)
	343	if not os.path.exists(path_dir):
	344	os.makedirs(path_dir)
	345
	346	shutil.copy2(f, path)
	347
	348	if pb:
	349	pb.finish()
	350
	351	# Download source files.
	352	downloader = SourceDownloader(self.pakfire, mirrors=self.source_dl)
	353	for filename in self.sources:
	354	_filename = downloader.download(filename)
	355	assert _filename
	356
	357	filename = "%s/files/%s" % (prefix, os.path.basename(_filename))
	358	dirname = os.path.dirname(filename)
	359
	360	if not os.path.exists(dirname):
	361	os.makedirs(dirname)
	362
	363	shutil.copy2(_filename, filename)
	364
	365
	366	class MakefilePackage(MakefileBase):
	367	def __init__(self, pakfire, name, lexer):
	368	Package.__init__(self, pakfire)
	369
	370	self._name = name
	371	self.lexer = lexer
	372
5dda54e4 MT	373	# Store additional dependencies in here.
	374	self._dependencies = {}
	375
c07a3ca7 MT	376	@property
	377	def name(self):
	378	return self._name
	379
	380	@property
	381	def arch(self):
	382	return self.lexer.get_var("arch", "%{DISTRO_ARCH}")
	383
	384	@property
	385	def configfiles(self):
	386	return self.lexer.get_var("configfiles").split()
	387
	388	@property
	389	def files(self):
	390	return self.lexer.get_var("files").split()
	391
	392	@property
	393	def uuid(self):
	394	return None
	395
5dda54e4 MT	396	def track_dependencies(self, builder, path):
	397	result = builder.do("/usr/lib/buildsystem-tools/dependency-tracker %s" \
	398	% path, returnOutput=True)
	399
	400	for line in result.splitlines():
	401	m = re.match(r"^(\w+)=(.*)$", line)
	402	if m is None:
	403	continue
	404
	405	key, val = m.groups()
	406
	407	if not key in ("prerequires", "requires", "provides", "conflicts", "obsoletes",):
	408	continue
	409
	410	val = val.strip("\"")
	411	val = val.split()
	412
	413	self._dependencies[key] = sorted(val)
	414
	415	def get_deps(self, key):
	416	# Collect all dependencies that were set in the makefile by the user.
	417	deps = self.lexer.get_var(key).split()
	418
	419	# Collect all dependencies that were discovered by the tracker.
	420	deps += self._dependencies.get(key, [])
	421
	422	# Remove duplicates.
	423	deps = set(deps)
	424	deps = list(deps)
	425
	426	return sorted(deps)
c07a3ca7 MT	427
	428	@property
	429	def prerequires(self):
5dda54e4	430	return self.get_deps("prerequires")
c07a3ca7 MT	431
	432	@property
	433	def requires(self):
5dda54e4	434	return self.get_deps("requires")
c07a3ca7 MT	435
	436	@property
	437	def provides(self):
5dda54e4	438	return self.get_deps("provides")
c07a3ca7 MT	439
	440	@property
	441	def obsoletes(self):
5dda54e4	442	return self.get_deps("obsoletes")
c07a3ca7 MT	443
	444	@property
	445	def conflicts(self):
5dda54e4	446	return self.get_deps("conflicts")
47a4cb89	447
c07a3ca7 MT	448	def get_scriptlet(self, type):
c07a3ca7 MT	449	return self.lexer.scriptlets.get(type, None)
0304200a MT	450
	451	@property
	452	def inst_size(self):
	453	# The size of this is unknown.
	454	return 0