From: Michael Tremer Date: Tue, 22 Feb 2011 20:53:36 +0000 (+0100) Subject: Improve downloader. X-Git-Tag: 0.9.3~135 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=14ea3228c8128c69b3105aaebce21b25a5550fb3;p=pakfire.git Improve downloader. We now do have different downloaders for different type of data: i.e metadata, packages, etc. that do come with different properties. --- diff --git a/pakfire/constants.py b/pakfire/constants.py index 1756c2af2..2e3fbdc76 100644 --- a/pakfire/constants.py +++ b/pakfire/constants.py @@ -2,6 +2,8 @@ import os.path +PAKFIRE_VERSION = "testing" + SYSCONFDIR = os.path.join(os.path.dirname(__file__), "..", "examples") if not os.path.exists(SYSCONFDIR): SYSCONFDIR = "/etc" diff --git a/pakfire/downloader.py b/pakfire/downloader.py index 31a72652b..b2d9991c2 100644 --- a/pakfire/downloader.py +++ b/pakfire/downloader.py @@ -6,6 +6,7 @@ import random from urlgrabber.grabber import URLGrabber, URLGrabError from urlgrabber.mirror import MirrorGroup +from urlgrabber.progress import TextMeter from constants import * @@ -15,9 +16,42 @@ class PakfireGrabber(URLGrabber): """ Class to make some modifications on the urlgrabber configuration. """ - # XXX add proxy, user_agent, keep-alive, throttle things here - pass + # XXX add proxy, throttle things here + def __init__(self, *args, **kwargs): + kwargs.update({ + "quote" : 0, + "user_agent" : "pakfire/%s" % PAKFIRE_VERSION, + }) + + URLGrabber.__init__(self, *args, **kwargs) + + +class PackageDownloader(PakfireGrabber): + def __init__(self, *args, **kwargs): + kwargs.update({ + "progress_obj" : TextMeter(), + }) + + PakfireGrabber.__init__(self, *args, **kwargs) + + +class MetadataDownloader(PakfireGrabber): + def __init__(self, *args, **kwargs): + kwargs.update({ + "http_headers" : (('Pragma', 'no-cache'),), + }) + + PakfireGrabber.__init__(self, *args, **kwargs) + + +class DatabaseDownloader(PackageDownloader): + def __init__(self, *args, **kwargs): + kwargs.update({ + "http_headers" : (('Pragma', 'no-cache'),), + }) + + PackageDownloader.__init__(self, *args, **kwargs) class Mirror(object): @@ -73,7 +107,7 @@ class MirrorList(object): force = True if force: - g = PakfireGrabber() + g = MetadataDownloader() try: mirrordata = g.urlread(self.mirrorlist, limit=MIRRORLIST_MAXSIZE) diff --git a/pakfire/index.py b/pakfire/index.py index 841511e8c..2e9de7c3c 100644 --- a/pakfire/index.py +++ b/pakfire/index.py @@ -6,11 +6,13 @@ import random import shutil import database +import downloader import packages import repository import util from constants import * +from i18n import _ class Index(object): def __init__(self, pakfire, repo): @@ -156,8 +158,12 @@ class DatabaseIndex(Index): download = False if download: + # Initialize a grabber for download. + grabber = downloader.MetadataDownloader() + grabber = self.repo.mirrors.group(grabber) + # XXX do we need limit here for security reasons? - metadata = self.repo.grabber.urlread("repodata/repomd.json") + metadata = grabber.urlread("repodata/repomd.json") with cache.open(cache_filename, "w") as o: o.write(metadata) @@ -169,9 +175,15 @@ class DatabaseIndex(Index): cache_filename = "metadata/packages.db" # XXX just for now if not cache.exists(cache_filename): + # Initialize a grabber for download. + grabber = downloader.DatabaseDownloader( + text = _("%s: package database") % self.repo.name, + ) + grabber = self.repo.mirrors.group(grabber) + + i = grabber.urlopen("repodata/packages.db") # XXX just for now o = cache.open(cache_filename, "w") - i = self.repo.grabber.urlopen("repodata/packages.db") # XXX just for now - + buf = i.read(BUFFER_SIZE) while buf: o.write(buf) diff --git a/pakfire/packages/installed.py b/pakfire/packages/installed.py index f7e674309..a41bf40eb 100644 --- a/pakfire/packages/installed.py +++ b/pakfire/packages/installed.py @@ -2,6 +2,8 @@ import os +import pakfire.downloader + from base import Package from binary import BinaryPackage @@ -148,7 +150,7 @@ class DatabasePackage(Package): c.close() - def download(self): + def download(self, text=""): """ Downloads the package from repository and returns a new instance of BinaryPackage. @@ -172,12 +174,19 @@ class DatabasePackage(Package): cache.remove(cache_filename) if download: - # Open input and output files and download the file. - o = cache.open(cache_filename, "w") # Make sure filename is of type string (and not unicode) filename = str(self.filename) - i = self.repo.grabber.urlopen(filename) + # Get a package grabber and add mirror download capabilities to it. + grabber = pakfire.downloader.PackageDownloader( + text=text + os.path.basename(filename), + ) + grabber = self.repo.mirrors.group(grabber) + + i = grabber.urlopen(filename) + + # Open input and output files and download the file. + o = cache.open(cache_filename, "w") buf = i.read(BUFFER_SIZE) while buf: diff --git a/pakfire/repository.py b/pakfire/repository.py index f92b28b30..dc780c605 100644 --- a/pakfire/repository.py +++ b/pakfire/repository.py @@ -7,7 +7,6 @@ import stat import time from ConfigParser import ConfigParser -from urlgrabber.progress import TextMeter, TextMultiFileMeter import base import database @@ -488,18 +487,6 @@ class RemoteRepository(RepositoryFactory): return priority - @property - def grabber(self): - if not self.__grabber: - grabber = downloader.PakfireGrabber( -# progress_obj = TextMultiFileMeter(), # XXX broken? - progress_obj = TextMeter(), - ) - - self.__grabber = self.mirrors.group(grabber) - - return self.__grabber - def update_index(self, force=False): if self.index: self.index.update(force=force) diff --git a/pakfire/transaction.py b/pakfire/transaction.py index 546f267ab..45634e999 100644 --- a/pakfire/transaction.py +++ b/pakfire/transaction.py @@ -151,8 +151,19 @@ class TransactionSet(object): """ Convert all packages to BinaryPackage. """ - for download in self.downloads: - pkg = download.download() + pkgs = [] + for pkg in self.downloads: + pkgs.append(pkg) + + # If there are no packages to download skip the rest. + if not pkgs: + return + + logging.info("Downloading packages:") + i = 0 + for download in pkgs: + i += 1 + pkg = download.download(text="(%2d/%02d): " % (i, len(pkgs))) for download_list in self.download_lists: if download in download_list: @@ -160,6 +171,9 @@ class TransactionSet(object): download_list.append(pkg) break + # Just an empty line to seperate the downloads from the extractions. + logging.info("") + class Transaction(object): def __init__(self, pakfire, ds): diff --git a/po/pakfire.pot b/po/pakfire.pot index 61be70016..ed6c65c1c 100644 --- a/po/pakfire.pot +++ b/po/pakfire.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2011-02-22 17:27+0100\n" +"POT-Creation-Date: 2011-02-22 21:49+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -215,11 +215,16 @@ msgstr "" msgid "Remove" msgstr "" -#: ../pakfire/depsolve.py:240 +#: ../pakfire/depsolve.py:242 #, python-format msgid "Total download size: %s" msgstr "" +#: ../pakfire/index.py:180 +#, python-format +msgid "%s: package database" +msgstr "" + #: ../pakfire/__init__.py:172 msgid "Is this okay?" msgstr ""