From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 30 Sep 2011 13:18:17 +0000 (+0200)
Subject: Improve source download code and clean things up.
X-Git-Tag: 0.9.10~13
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=062699eeacc87abc52598b0faf3b30022248c521;p=pakfire.git

Improve source download code and clean things up.
---

diff --git a/python/pakfire/builder.py b/python/pakfire/builder.py
index 2953e7372..75e81fa05 100644
--- a/python/pakfire/builder.py
+++ b/python/pakfire/builder.py
@@ -124,6 +124,9 @@ class BuildEnviron(object):
 			self.log.info("  %s" % line)
 		self.log.info("")
 
+		# Download all package files.
+		self.pkg.download()
+
 		# XXX need to make this configureable
 		self.settings = {
 			"enable_loop_devices" : True,
diff --git a/python/pakfire/downloader.py b/python/pakfire/downloader.py
index 84043c93d..24aac0bd2 100644
--- a/python/pakfire/downloader.py
+++ b/python/pakfire/downloader.py
@@ -21,6 +21,7 @@
 
 import json
 import logging
+import os
 import random
 
 from config import Config
@@ -30,6 +31,7 @@ from urlgrabber.mirror import MirrorGroup
 from urlgrabber.progress import TextMeter
 
 from pakfire.constants import *
+from pakfire.i18n import _
 
 class PakfireGrabber(URLGrabber):
 	"""
@@ -95,6 +97,49 @@ class DatabaseDownloader(PackageDownloader):
 		PackageDownloader.__init__(self, pakfire, *args, **kwargs)
 
 
+class SourceDownloader(object):
+	def __init__(self, pakfire, mirrors=None):
+		self.pakfire = pakfire
+
+		self.grabber = PakfireGrabber(
+			self.pakfire,
+			progress_obj = TextMeter(),
+		)
+
+		if mirrors:
+			self.grabber = MirrorGroup(self.grabber,
+				[{ "mirror" : m } for m in mirrors])
+
+	def download(self, files):
+		existant_files = []
+		download_files = []
+
+		for file in files:
+			filename = os.path.join(SOURCE_CACHE_DIR, file)
+
+			if os.path.exists(filename):
+				existant_files.append(filename)
+			else:
+				download_files.append(filename)
+
+		if download_files:
+			logging.info(_("Downloading source files:"))
+
+			# Create source download directory.
+			if not os.path.exists(SOURCE_CACHE_DIR):
+				os.makedirs(SOURCE_CACHE_DIR)
+
+			for filename in download_files:
+				try:
+					self.grabber.urlgrab(os.path.basename(filename), filename=filename)
+				except URLGrabError, e:
+					raise DownloadError, "%s %s" % (os.path.basename(filename), e)
+
+			logging.info("")
+
+		return existant_files + download_files
+
+
 class Mirror(object):
 	def __init__(self, url, location=None, preferred=False):
 		# Save URL of the mirror in full format
diff --git a/python/pakfire/packages/make.py b/python/pakfire/packages/make.py
index 90a43d8d1..80bb12aa7 100644
--- a/python/pakfire/packages/make.py
+++ b/python/pakfire/packages/make.py
@@ -33,6 +33,7 @@ import lexer
 import packager
 
 import pakfire.chroot as chroot
+import pakfire.downloader as downloader
 import pakfire.util as util
 
 from base import Package
@@ -41,37 +42,6 @@ from file import SourcePackage
 from pakfire.constants import *
 from pakfire.i18n import _
 
-# XXX to be moved to pakfire.downloader
-class SourceDownloader(object):
-	def __init__(self, pakfire, mirrors=None):
-		self.pakfire = pakfire
-		self.mirrors = mirrors
-
-		# XXX need to use downloader.py
-		self.grabber = URLGrabber(
-			prefix = self.pakfire.config.get("source_download_url"),
-			progress_obj = TextMeter(),
-			quote = 0,
-		)
-
-	def download(self, filename):
-		filename = os.path.join(SOURCE_CACHE_DIR, filename)
-
-		if os.path.exists(filename):
-			return filename
-
-		dirname = os.path.dirname(filename)
-		if not os.path.exists(dirname):
-			os.makedirs(dirname)
-
-		try:
-			self.grabber.urlgrab(os.path.basename(filename), filename=filename)
-		except URLGrabError, e:
-			raise DownloadError, "%s %s" % (os.path.basename(filename), e)
-
-		return filename
-
-
 class MakefileBase(Package):
 	def __init__(self, pakfire, filename):
 		Package.__init__(self, pakfire)
@@ -244,15 +214,10 @@ class Makefile(MakefileBase):
 			copies.
 		"""
 		# Download source files.
-		# XXX need to implement mirrors
-		downloader = SourceDownloader(self.pakfire, mirrors=self.source_dl)
-
-		files = []
-		for filename in self.sources:
-			filename = downloader.download(filename)
-			files.append(filename)
+		grabber = downloader.SourceDownloader(self.pakfire,
+			mirrors=self.source_dl)
 
-		return files
+		return grabber.download(self.sources)
 
 	def dist(self, resultdirs):
 		"""
@@ -367,17 +332,13 @@ class Makefile(MakefileBase):
 			pb.finish()
 
 		# Download source files.
-		downloader = SourceDownloader(self.pakfire, mirrors=self.source_dl)
-		for filename in self.sources:
-			_filename = downloader.download(filename)
-			assert _filename
-
+		for _filename in self.download():
 			filename = "%s/files/%s" % (prefix, os.path.basename(_filename))
 			dirname = os.path.dirname(filename)
 
 			if not os.path.exists(dirname):
 				os.makedirs(dirname)
-				
+
 			shutil.copy2(_filename, filename)
 
 	@property