python/pakfire/downloader.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # Pakfire - The IPFire package management system                              #
   5 # Copyright (C) 2011 Pakfire development team                                 #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import json
  23 import os
  24 import random
  25
  26 import logging
  27 log = logging.getLogger("pakfire")
  28
  29 from config import Config
  30
  31 from urlgrabber.grabber import URLGrabber, URLGrabError
  32 from urlgrabber.mirror import MirrorGroup
  33 from urlgrabber.progress import TextMeter
  34
  35 from pakfire.constants import *
  36 from pakfire.i18n import _
  37
  38 class PakfireGrabber(URLGrabber):
  39         """
  40                 Class to make some modifications on the urlgrabber configuration.
  41         """
  42         def __init__(self, pakfire, *args, **kwargs):
  43                 kwargs.update({
  44                         "quote" : 0,
  45                         "user_agent" : "pakfire/%s" % PAKFIRE_VERSION,
  46                 })
  47
  48                 if isinstance(pakfire, Config):
  49                         config = pakfire
  50                 else:
  51                         config = pakfire.config
  52
  53                 if config.get("offline"):
  54                         raise OfflineModeError, "Cannot use %s in offline mode." % self.__class__.__name__
  55
  56                 # Set throttle setting.
  57                 bandwidth_throttle = config.get("bandwidth_throttle")
  58                 if bandwidth_throttle:
  59                         try:
  60                                 bandwidth_throttle = int(bandwidth_throttle)
  61                         except ValueError:
  62                                 log.error("Configuration value for bandwidth_throttle is invalid.")
  63                                 bandwidth_throttle = 0
  64
  65                         kwargs.update({ "throttle" : bandwidth_throttle })
  66
  67                 # Configure HTTP proxy.
  68                 http_proxy = config.get("http_proxy")
  69                 if http_proxy:
  70                         kwargs.update({ "proxies" : { "http" : http_proxy }})
  71
  72                 URLGrabber.__init__(self, *args, **kwargs)
  73
  74         def urlread(self, filename, *args, **kwargs):
  75                 # This is for older versions of urlgrabber which are packaged in Debian
  76                 # and Ubuntu and cannot handle filenames as a normal Python string but need
  77                 # a unicode string.
  78                 return URLGrabber.urlread(self, filename.encode("utf-8"), *args, **kwargs)
  79
  80
  81 class PackageDownloader(PakfireGrabber):
  82         def __init__(self, pakfire, *args, **kwargs):
  83                 kwargs.update({
  84                                 "progress_obj" : TextMeter(),
  85                 })
  86
  87                 PakfireGrabber.__init__(self, pakfire, *args, **kwargs)
  88
  89
  90 class MetadataDownloader(PakfireGrabber):
  91         def __init__(self, pakfire, *args, **kwargs):
  92                 kwargs.update({
  93                         "http_headers" : (('Pragma', 'no-cache'),),
  94                 })
  95
  96                 PakfireGrabber.__init__(self, pakfire, *args, **kwargs)
  97
  98
  99 class DatabaseDownloader(PackageDownloader):
 100         def __init__(self, pakfire, *args, **kwargs):
 101                 kwargs.update({
 102                         "http_headers" : (('Pragma', 'no-cache'),),
 103                 })
 104
 105                 PackageDownloader.__init__(self, pakfire, *args, **kwargs)
 106
 107
 108 class SourceDownloader(object):
 109         def __init__(self, pakfire, mirrors=None):
 110                 self.pakfire = pakfire
 111
 112                 self.grabber = PakfireGrabber(
 113                         self.pakfire,
 114                         progress_obj = TextMeter(),
 115                 )
 116
 117                 if mirrors:
 118                         self.grabber = MirrorGroup(self.grabber,
 119                                 [{ "mirror" : m.encode("utf-8") } for m in mirrors])
 120
 121         def download(self, files):
 122                 existant_files = []
 123                 download_files = []
 124
 125                 for file in files:
 126                         filename = os.path.join(SOURCE_CACHE_DIR, file)
 127
 128                         if os.path.exists(filename):
 129                                 existant_files.append(filename)
 130                         else:
 131                                 download_files.append(filename)
 132
 133                 if download_files:
 134                         log.info(_("Downloading source files:"))
 135
 136                         # Create source download directory.
 137                         if not os.path.exists(SOURCE_CACHE_DIR):
 138                                 os.makedirs(SOURCE_CACHE_DIR)
 139
 140                         for filename in download_files:
 141                                 try:
 142                                         self.grabber.urlgrab(os.path.basename(filename), filename=filename)
 143                                 except URLGrabError, e:
 144                                         raise DownloadError, "%s %s" % (os.path.basename(filename), e)
 145
 146                         log.info("")
 147
 148                 return existant_files + download_files
 149
 150
 151 class Mirror(object):
 152         def __init__(self, url, location=None, preferred=False):
 153                 # Save URL of the mirror in full format
 154                 self.url = url
 155
 156                 # Save the location (if given)
 157                 self.location = location
 158
 159                 # Save preference
 160                 self.preferred = False
 161
 162
 163 class MirrorList(object):
 164         def __init__(self, pakfire, repo):
 165                 self.pakfire = pakfire
 166                 self.repo = repo
 167
 168                 self.__mirrors = []
 169
 170                 # Save URL to more mirrors.
 171                 self.mirrorlist = repo.mirrorlist
 172
 173                 self.update(force=False)
 174
 175         @property
 176         def cache(self):
 177                 """
 178                         Shortcut to cache from repository.
 179                 """
 180                 return self.repo.cache
 181
 182         def update(self, force=False):
 183                 # XXX should this be allowed?
 184                 if not self.mirrorlist:
 185                         return
 186
 187                 # If the system is not online, we cannot download anything.
 188                 if self.pakfire.offline:
 189                         return
 190
 191                 log.debug("Updating mirrorlist for repository '%s' (force=%s)" % (self.repo.name, force))
 192
 193                 cache_filename = "mirrors/mirrorlist"
 194
 195                 # Force the update if no mirrorlist is available.
 196                 if not self.cache.exists(cache_filename):
 197                         force = True
 198
 199                 if not force and self.cache.exists(cache_filename):
 200                         age = self.cache.age(cache_filename)
 201
 202                         # If the age could be determined and is higher than 24h,
 203                         # we force an update.
 204                         if age and age > TIME_24H:
 205                                 force = True
 206
 207                 if force:
 208                         g = MetadataDownloader(self.pakfire)
 209
 210                         try:
 211                                 mirrordata = g.urlread(self.mirrorlist, limit=MIRRORLIST_MAXSIZE)
 212                         except URLGrabError, e:
 213                                 log.warning("Could not update the mirrorlist for repo '%s': %s" % (self.repo.name, e))
 214                                 return
 215
 216                         # XXX check for empty files or damaged output
 217
 218                         # Save new mirror data to cache.
 219                         f = self.cache.open(cache_filename, "w")
 220                         f.write(mirrordata)
 221                         f.close()
 222
 223                 # Read mirrorlist from cache and parse it.
 224                 with self.cache.open(cache_filename) as f:
 225                         self.parse_mirrordata(f.read())
 226
 227         def parse_mirrordata(self, data):
 228                 data = json.loads(data)
 229
 230                 for mirror in data["mirrors"]:
 231                         self.add_mirror(**mirror)
 232
 233         def add_mirror(self, *args, **kwargs):
 234                 mirror = Mirror(*args, **kwargs)
 235
 236                 self.__mirrors.append(mirror)
 237
 238         @property
 239         def preferred(self):
 240                 """
 241                         Return a generator for all mirrors that are preferred.
 242                 """
 243                 for mirror in self.__mirrors:
 244                         if mirror.preferred:
 245                                 yield mirror
 246
 247         @property
 248         def non_preferred(self):
 249                 """
 250                         Return a generator for all mirrors that are not preferred.
 251                 """
 252                 for mirror in self.__mirrors:
 253                         if not mirror.preferred:
 254                                 yield mirror
 255
 256         @property
 257         def all(self):
 258                 """
 259                         Return a generator for all mirrors.
 260                 """
 261                 for mirror in self.__mirrors:
 262                         yield mirror
 263
 264         def group(self, grabber):
 265                 """
 266                         Return a MirrorGroup object for the given grabber.
 267                 """
 268                 # A list of mirrors that is passed to MirrorGroup.
 269                 mirrors = []
 270
 271                 # Add all preferred mirrors at the first place and shuffle them
 272                 # that we will start at a random place.
 273                 for mirror in self.preferred:
 274                         mirrors.append(mirror.url.encode("utf-8"))
 275                 random.shuffle(mirrors)
 276
 277                 # All other mirrors are added as well and will only be used if all
 278                 # preferred mirrors did not work.
 279                 for mirror in self.all:
 280                         if mirror.url in mirrors:
 281                                 continue
 282
 283                         mirrors.append({ "mirror" : mirror.url.encode("utf-8") })
 284
 285                 return MirrorGroup(grabber, mirrors)
 286
 287
 288
 289 class Downloader(object):
 290         def __init__(self, mirrors, files):
 291                 self.grabber = PakfireGrabber()
 292
 293                 self.mirrorgroup = mirrors.group(self.grabber)
 294
 295