From: Michael Tremer Date: Sat, 13 Mar 2021 17:55:52 +0000 (+0000) Subject: Drop old downloader and metadata parser X-Git-Tag: 0.9.28~1285^2~529 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=976063f612a09b0c7ac453afdf3ab34fec35f9b2;p=pakfire.git Drop old downloader and metadata parser Signed-off-by: Michael Tremer --- diff --git a/Makefile.am b/Makefile.am index 685255141..735989ff1 100644 --- a/Makefile.am +++ b/Makefile.am @@ -123,7 +123,6 @@ pakfire_PYTHON = \ src/pakfire/constants.py \ src/pakfire/daemon.py \ src/pakfire/distro.py \ - src/pakfire/downloaders.py \ src/pakfire/errors.py \ src/pakfire/http.py \ src/pakfire/hub.py \ @@ -146,8 +145,7 @@ pakfire_packagesdir = $(pythondir)/pakfire/packages # ------------------------------------------------------------------------------ pakfire_repository_PYTHON = \ - src/pakfire/repository/__init__.py \ - src/pakfire/repository/metadata.py + src/pakfire/repository/__init__.py pakfire_repositorydir = $(pythondir)/pakfire/repository diff --git a/src/pakfire/base.py b/src/pakfire/base.py index 1852f3f2a..22032db78 100644 --- a/src/pakfire/base.py +++ b/src/pakfire/base.py @@ -26,7 +26,6 @@ import string from . import _pakfire from . import distro -from . import downloaders from . import logger from . import packages from . import repository diff --git a/src/pakfire/cli.py b/src/pakfire/cli.py index e9d75c929..fa469bd72 100644 --- a/src/pakfire/cli.py +++ b/src/pakfire/cli.py @@ -34,7 +34,6 @@ from . import builder from . import client from . import config from . import daemon -from . import downloaders from . import packages from . import repository from . import ui @@ -254,15 +253,6 @@ class Cli(object): for line in t.splitlines(): self.ui.message(line) - def _download_transaction(self, transaction): - """ - Downloads all packages required for this transaction - """ - d = downloaders.TransactionDownloader(self.pakfire, transaction) - - # Run the download - d.download() - def _execute_transaction(self, transaction): # Don't do anything if the transaction is empty if len(transaction) == 0: @@ -278,7 +268,7 @@ class Cli(object): return # Download transaction - self._download_transaction(transaction) + transaction.download() # Run the transaction transaction.run() diff --git a/src/pakfire/downloaders.py b/src/pakfire/downloaders.py deleted file mode 100644 index 1b8e9d2f0..000000000 --- a/src/pakfire/downloaders.py +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/python3 -############################################################################### -# # -# Pakfire - The IPFire package management system # -# Copyright (C) 2018 Pakfire development team # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -import json -import logging - -from . import http -from .repository import metadata -from .i18n import _ - -log = logging.getLogger("pakfire.downloader") -log.propagate = 1 - -class Downloader(object): - pass - - -class RepositoryDownloader(Downloader): - def __init__(self, pakfire, repo): - self.pakfire = pakfire - self.repo = repo - - def make_downloader(self): - """ - Creates a downloader that can be used to download - metadata, databases or packages from this repository. - """ - downloader = http.Client(baseurl=self.repo.baseurl) - - # Add any mirrors that we know of - for mirror in self.mirrorlist: - downloader.add_mirror(mirror.get("url")) - - return downloader - - def refresh(self, force=False): - # Don't do anything if running in offline mode - if self.pakfire.offline: - log.debug(_("Skipping refreshing %s since we are running in offline mode") % self) - return - - # Refresh the mirror list - self._refresh_mirror_list(force=force) - - # Refresh metadata - self._refresh_metadata(force=force) - - # Refresh database - self._refresh_database() - - # Read database - if self.database: - self.repo.read_solv(self.database) - - @property - def mirrorlist(self): - """ - Opens a cached mirror list - """ - try: - with self.repo.cache_open("mirrors", "r") as f: - mirrors = json.load(f) - - return mirrors.get("mirrors") - - # If there is no mirror list in the cache, - # we won't be able to open it - except IOError: - pass - - return [] - - def _refresh_mirror_list(self, force=False): - # Check age of the mirror list first - age = self.repo.cache_age("mirrors") - - # Don't refresh anything if the mirror list - # has been refreshed in the last 24 hours - if not force and age and age <= 24 * 3600: - return - - # (Re-)download the mirror list - if not self.repo.mirrorlist: - return - - # Use a generic downloader - downloader = http.Client() - - # Download a new mirror list - try: - mirrorlist = downloader.get(self.repo.mirrorlist, decode="json") - except http.DownloadError as e: - log.warning("Could not download mirrorlist for %s: %s" % (self.repo, e)) - return - - # Write new list to disk - with self.repo.cache_open("mirrors", "w") as f: - s = json.dumps(mirrorlist) - f.write(s) - - @property - def metadata(self): - if not self.repo.cache_exists("repomd.json"): - return - - with self.repo.cache_open("repomd.json", "r") as f: - return metadata.Metadata(self.pakfire, metadata=f.read()) - - def _refresh_metadata(self, force=False): - # Check age of the metadata first - age = self.repo.cache_age("repomd.json") - - # Don't refresh anything if the metadata - # has been refresh within the last 10 minutes - if not force and age and age <= 600: - return - - # Get a downloader - downloader = self.make_downloader() - - while True: - data = downloader.get("%s/repodata/repomd.json" % self.pakfire.arch, decode="ascii") - - # Parse new metadata for comparison - md = metadata.Metadata(self.pakfire, metadata=data) - - if self.metadata and md < self.metadata: - log.warning(_("The downloaded metadata was less recent than the current one.")) - downloader.skip_current_mirror() - continue - - # If the download went well, we write the downloaded data to disk - # and break the loop. - with self.repo.cache_open("repomd.json", "w") as f: - md.save(f) - - break - - @property - def database(self): - if self.metadata and self.metadata.database and self.repo.cache_exists(self.metadata.database): - return self.repo.cache_path(self.metadata.database) - - def _refresh_database(self): - assert self.metadata, "Metadata does not exist" - - # Exit if the file already exists in the cache - if self.repo.cache_exists(self.metadata.database): - return - - # Make the downloader - downloader = self.make_downloader() - - # This is where the file will be saved after download - path = self.repo.cache_path(self.metadata.database) - - # XXX compare checksum here - downloader.retrieve("repodata/%s" % self.metadata.database, filename=path, - message=_("%s: package database") % self.repo.name) - - def download_package(self, pkg): - """ - Downloads a package to it's cache path - """ - downloader = self.make_downloader() - - downloader.retrieve(pkg.filename, filename=pkg.cache_path, - message="%s" % pkg, checksum_algo="sha1", checksum=pkg.checksum) - - -class TransactionDownloader(Downloader): - def __init__(self, pakfire, transaction): - self.pakfire = pakfire - - # The transaction that we process - self.transaction = transaction - - # Cache repository downloaders - self._repo_downloaders = {} - - def _get_repo_downloader(self, repo): - try: - return self._repo_downloaders[repo] - except KeyError: - d = RepositoryDownloader(self.pakfire, repo) - self._repo_downloaders[repo] = d - - return d - - def download(self): - for step in self.transaction: - # Skip any steps that do not need a download - if not step.needs_download: - continue - - # Get the downloader - downloader = self._get_repo_downloader(step.package.repo) - - # Download the package - downloader.download_package(step.package) diff --git a/src/pakfire/repository/metadata.py b/src/pakfire/repository/metadata.py deleted file mode 100644 index d1b7a3d90..000000000 --- a/src/pakfire/repository/metadata.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/python -############################################################################### -# # -# Pakfire - The IPFire package management system # -# Copyright (C) 2011 Pakfire development team # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -import json -import os -import time - -from pakfire.constants import * - -class Metadata(object): - def __init__(self, pakfire, metafile=None, metadata=None): - self.pakfire = pakfire - self.filename = metafile - - # Place where we save the data. - self._data = { - "database" : None, - "revision" : int(time.time()), - "version" : METADATA_FORMAT, - } - - # If a file was passed, we open it. - if self.filename: - self.open() - - # ... or parse the one that was passed. - elif metadata: - self.parse(metadata) - - def __lt__(self, other): - """ - Compare two sets of metadata by their revision. - """ - return self.revision < other.revision - - def parse(self, metadata): - try: - self._data = json.loads(metadata) - except: - raise # XXX catch json exceptions here - - def open(self, filename=None): - """ - Open a given file or use the default one and parse the - data in it. - """ - if not filename: - filename = self.filename - - assert os.path.exists(filename), "Metadata file does not exist." - - with open(filename) as f: - self.parse(f.read()) - - def save(self, fp): - """ - Save all data to a file that could be exported to a - remote repository. - """ - json.dump(self._data, fp, indent=2) - - @property - def version(self): - """ - Returns the version of the metadata. - """ - return self._data.get("version") - - @property - def revision(self): - """ - Returns the revision of the metadata. - """ - return self._data.get("revision") - - def get_database(self): - return self._data.get("database") - - def set_database(self, val): - self._data["database"] = val - - database = property(get_database, set_database) - - def get_database_hash1(self): - return self._data.get("database_hash1", None) - - def set_database_hash1(self, val): - self._data["database_hash1"] = val - - database_hash1 = property(get_database_hash1, set_database_hash1) - - def get_database_compression(self): - return self._data.get("database_compression", None) - - def set_database_compression(self, val): - self._data["database_compression"] = val - - database_compression = property(get_database_compression, - set_database_compression)