From: Michael Tremer Date: Mon, 22 Jan 2018 18:41:21 +0000 (+0100) Subject: python: Move refreshing remote repositories into own module X-Git-Tag: 0.9.28~1285^2~1124 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=3265ba7386826cf164150ee8c7cbbb04467d8e0d;p=pakfire.git python: Move refreshing remote repositories into own module Signed-off-by: Michael Tremer --- diff --git a/Makefile.am b/Makefile.am index 44127b4c3..516981dc8 100644 --- a/Makefile.am +++ b/Makefile.am @@ -124,6 +124,7 @@ pakfire_PYTHON = \ src/pakfire/constants.py \ src/pakfire/daemon.py \ src/pakfire/distro.py \ + src/pakfire/downloaders.py \ src/pakfire/errors.py \ src/pakfire/filelist.py \ src/pakfire/http.py \ diff --git a/src/pakfire/base.py b/src/pakfire/base.py index daeb099ab..f0b6b824e 100644 --- a/src/pakfire/base.py +++ b/src/pakfire/base.py @@ -25,6 +25,7 @@ import string from . import _pakfire from . import distro +from . import downloaders from . import filelist from . import packages from . import repository @@ -88,7 +89,7 @@ class Pakfire(_pakfire.Pakfire): self.config.dump() # Refresh repositories - self.repos.refresh() + self.refresh_repositories() return PakfireContext(self) @@ -102,6 +103,17 @@ class Pakfire(_pakfire.Pakfire): """ return self.config.get("downloader", "offline", False) + def refresh_repositories(self, force=False): + for repo in self.repos: + if not repo.enabled: + continue + + if repo == self.installed_repo: + continue + + d = downloaders.RepositoryDownloader(self, repo) + d.refresh(force=force) + def check_root_user(self): if not os.getuid() == 0 or not os.getgid() == 0: raise Exception("You must run pakfire as the root user.") diff --git a/src/pakfire/downloaders.py b/src/pakfire/downloaders.py new file mode 100644 index 000000000..5408af2f6 --- /dev/null +++ b/src/pakfire/downloaders.py @@ -0,0 +1,174 @@ +#!/usr/bin/python3 +############################################################################### +# # +# Pakfire - The IPFire package management system # +# Copyright (C) 2018 Pakfire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +import json +import logging + +from . import http +from .repository import metadata +from .i18n import _ + +log = logging.getLogger("pakfire.downloader") +log.propagate = 1 + + +class Downloader(object): + pass + + +class RepositoryDownloader(Downloader): + def __init__(self, pakfire, repo): + self.pakfire = pakfire + self.repo = repo + + def make_downloader(self): + """ + Creates a downloader that can be used to download + metadata, databases or packages from this repository. + """ + downloader = http.Client(baseurl=self.repo.baseurl) + + # Add any mirrors that we know of + for mirror in self.mirrorlist: + downloader.add_mirror(mirror.get("url")) + + return downloader + + def refresh(self, force=False): + # Don't do anything if running in offline mode + if self.pakfire.offline: + log.debug(_("Skipping refreshing %s since we are running in offline mode") % self) + return + + # Refresh the mirror list + self._refresh_mirror_list(force=force) + + # Refresh metadata + self._refresh_metadata(force=force) + + # Refresh database + self._refresh_database() + + # Read database + if self.database: + self.repo.read_solv(self.database) + + @property + def mirrorlist(self): + """ + Opens a cached mirror list + """ + try: + with self.repo.cache_open("mirrors", "r") as f: + mirrors = json.load(f) + + return mirrors.get("mirrors") + + # If there is no mirror list in the cache, + # we won't be able to open it + except IOError: + pass + + return [] + + def _refresh_mirror_list(self, force=False): + # Check age of the mirror list first + age = self.repo.cache_age("mirrors") + + # Don't refresh anything if the mirror list + # has been refreshed in the last 24 hours + if not force and age and age <= 24 * 3600: + return + + # (Re-)download the mirror list + if not self.repo.mirrorlist: + return + + # Use a generic downloader + downloader = http.Client() + + # Download a new mirror list + mirrorlist = downloader.get(url, decode="json") + + # Write new list to disk + with self.repo.cache_open("mirrors", "w") as f: + s = json.dumps(mirrorlist) + f.write(s) + + @property + def metadata(self): + if not self.repo.cache_exists("repomd.json"): + return + + with self.repo.cache_open("repomd.json", "r") as f: + return metadata.Metadata(self.pakfire, metadata=f.read()) + + def _refresh_metadata(self, force=False): + # Check age of the metadata first + age = self.repo.cache_age("repomd.json") + + # Don't refresh anything if the metadata + # has been refresh within the last 10 minutes + if not force and age and age <= 600: + return + + # Get a downloader + downloader = self.make_downloader() + + while True: + data = downloader.get("%s/repodata/repomd.json" % self.pakfire.arch, decode="ascii") + + # Parse new metadata for comparison + md = metadata.Metadata(self.pakfire, metadata=data) + + if self.metadata and md < self.metadata: + log.warning(_("The downloaded metadata was less recent than the current one.")) + downloader.skip_current_mirror() + continue + + # If the download went well, we write the downloaded data to disk + # and break the loop. + with self.repo.cache_open("repomd.json", "w") as f: + md.save(f) + + break + + @property + def database(self): + if self.metadata and self.metadata.database and self.repo.cache_exists(self.metadata.database): + return self.repo.cache_path(self.metadata.database) + + def _refresh_database(self): + assert self.metadata, "Metadata does not exist" + + # Exit if the file already exists in the cache + if self.repo.cache_exists(self.metadata.database): + return + + # Make the downloader + downloader = self.make_downloader() + + # This is where the file will be saved after download + path = self.repo.cache_path(self.metadata.database) + + # XXX compare checksum here + downloader.retrieve("repodata/%s" % self.metadata.database, filename=path, + message=_("%s: package database") % self.repo.name) diff --git a/src/pakfire/repository/__init__.py b/src/pakfire/repository/__init__.py index 8e3f606db..1f662fc85 100644 --- a/src/pakfire/repository/__init__.py +++ b/src/pakfire/repository/__init__.py @@ -71,14 +71,6 @@ class Repositories(object): """ return len([r for r in self if r.enabled]) - def refresh(self): - """ - Refreshes all repositories - """ - for repo in self: - if repo.enabled: - repo.refresh() - @property def distro(self): return self.pakfire.distro diff --git a/src/pakfire/repository/base.py b/src/pakfire/repository/base.py index bef7aec81..ded31c5a2 100644 --- a/src/pakfire/repository/base.py +++ b/src/pakfire/repository/base.py @@ -49,15 +49,6 @@ class RepositoryFactory(_pakfire.Repo): def init(self, **kwargs): pass # To be overwritten by inheriting classes - def refresh(self): - """ - Called to refresh the repository metadata. - - This is probably only hand for remote repositories - that need to re-download data. - """ - pass - @property def local(self): """ diff --git a/src/pakfire/repository/remote.py b/src/pakfire/repository/remote.py index 289c1c2ce..beec40f61 100644 --- a/src/pakfire/repository/remote.py +++ b/src/pakfire/repository/remote.py @@ -50,37 +50,7 @@ class RepositoryRemote(base.RepositoryFactory): enabled = self.settings.get("enabled", True) self.enabled = util.is_enabled(enabled) - def make_downloader(self): - """ - Creates a downloader that can be used to download - metadata, databases or packages from this repository. - """ - downloader = http.Client(baseurl=self.baseurl) - - # Add any mirrors that we know of - for mirror in self.mirrorlist: - downloader.add_mirror(mirror.get("url")) - - return downloader - - def refresh(self, force=False): - # Don't do anything if running in offline mode - if self.pakfire.offline: - log.debug(_("Skipping refreshing %s since we are running in offline mode") % self) - return - - # Refresh the mirror list - self._refresh_mirror_list(force=force) - - # Refresh metadata - self._refresh_metadata(force=force) - - # Refresh database - self._refresh_database() - # Read database - if self.database: - self.read_solv(self.database) @property def mirrorlist(self):