From b2059099520d2405cab1fa86c5a3effee0abf096 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Wed, 6 Feb 2019 15:42:04 +0000 Subject: [PATCH] Fix mirror check This didn't run at all Signed-off-by: Michael Tremer --- src/backend/base.py | 1 + src/backend/mirrors.py | 67 +++++++++++++++++++++++------------------- src/crontab/ipfire.org | 3 ++ 3 files changed, 41 insertions(+), 30 deletions(-) diff --git a/src/backend/base.py b/src/backend/base.py index c1088282..86e34611 100644 --- a/src/backend/base.py +++ b/src/backend/base.py @@ -88,6 +88,7 @@ class Backend(object): @tornado.gen.coroutine def run_task(self, task, *args, **kwargs): tasks = { + "check-mirrors" : self.mirrors.check_all, "cleanup-messages" : self.messages.queue.cleanup, "scan-files" : self.releases.scan_files, "send-all-messages" : self.messages.queue.send_all, diff --git a/src/backend/mirrors.py b/src/backend/mirrors.py index 646979bc..5a1f3f2b 100644 --- a/src/backend/mirrors.py +++ b/src/backend/mirrors.py @@ -7,6 +7,7 @@ import os.path import random import socket import time +import tornado.gen import tornado.httpclient import tornado.netutil import urllib.parse @@ -34,9 +35,11 @@ class Mirrors(Object): return iter(mirrors) + @tornado.gen.coroutine def check_all(self): for mirror in self: - mirror.check() + with self.db.transaction(): + yield mirror.check() def get_for_download(self, filename, country_code=None): # Try to find a good mirror for this country first @@ -192,16 +195,19 @@ class Mirror(Object): def disabled(self): return not self.enabled + @tornado.gen.coroutine def check(self): logging.info("Running check for mirror %s" % self.hostname) self.db.execute("UPDATE mirrors SET address = %s WHERE id = %s", self.address, self.id) - self.check_timestamp() - self.check_filelist() + success = yield self.check_timestamp() - def check_state(self): + if success: + yield self.check_filelist() + + def check_state(self, last_update): logging.debug("Checking state of mirror %s" % self.id) if not self.enabled: @@ -210,7 +216,7 @@ class Mirror(Object): now = datetime.datetime.utcnow() - time_delta = now - self.last_update + time_delta = now - last_update time_diff = time_delta.total_seconds() time_down = self.settings.get_int("mirrors_time_down", 3*24*60*60) @@ -225,14 +231,18 @@ class Mirror(Object): self.set_state("UP") + @tornado.gen.coroutine def check_timestamp(self): http = tornado.httpclient.AsyncHTTPClient() - http.fetch(self.url + ".timestamp", - headers={ "Pragma" : "no-cache" }, - callback=self.__check_timestamp_response) + try: + response = yield http.fetch(self.url + ".timestamp", + headers={ "Pragma" : "no-cache" }) + except tornado.httpclient.HTTPError as e: + logging.error("Error getting timestamp from %s: %s" % (self.hostname, e)) + self.set_state("DOWN") + return False - def __check_timestamp_response(self, response): if response.error: logging.debug("Error getting timestamp from %s" % self.hostname) self.set_state("DOWN") @@ -248,13 +258,13 @@ class Mirror(Object): self.db.execute("UPDATE mirrors SET last_update = %s WHERE id = %s", timestamp, self.id) - # Reload changed settings - self.data["timestamp"] = timestamp - - self.check_state() + # Update state + self.check_state(timestamp) logging.info("Successfully updated timestamp from %s" % self.hostname) + return True + @tornado.gen.coroutine def check_filelist(self): # XXX need to remove data from disabled mirrors if not self.enabled: @@ -262,30 +272,27 @@ class Mirror(Object): http = tornado.httpclient.AsyncHTTPClient() - http.fetch(self.url + ".filelist", - headers={ "Pragma" : "no-cache" }, - callback=self.__check_filelist_response) + try: + response = yield http.fetch(self.url + ".filelist", + headers={ "Pragma" : "no-cache" }) + except tornado.httpclient.HTTPError as e: + logging.error("Error getting filelist from %s: %s" % (self.hostname, e)) + self.set_state("DOWN") + return - def __check_filelist_response(self, response): if response.error: - logging.debug("Error getting timestamp from %s" % self.hostname) + logging.debug("Error getting filelist from %s" % self.hostname) return - files = self.filelist + # Drop the old filelist + self.db.execute("DELETE FROM mirror_files WHERE mirror = %s", self.id) + # Add them all again for file in response.body.splitlines(): - file = os.path.join(self.prefix, file) - - if file in files: - files.remove(file) - continue - - self.db.execute("INSERT INTO mirror_files(mirror, filename) VALUES(%s, %s)", - self.id, file) + file = os.path.join(self.prefix, file.decode()) - for file in files: - self.db.execute("DELETE FROM mirror_files WHERE mirror=%s AND filename=%s", - self.id, file) + self.db.execute("INSERT INTO mirror_files(mirror, filename) \ + VALUES(%s, %s)", self.id, file) logging.info("Successfully updated mirror filelist from %s" % self.hostname) diff --git a/src/crontab/ipfire.org b/src/crontab/ipfire.org index 2aed7c94..31ce551d 100644 --- a/src/crontab/ipfire.org +++ b/src/crontab/ipfire.org @@ -9,3 +9,6 @@ # Cleanup once a an hour 30 * * * * nobody ipfire.org cleanup-messages + +# Check mirrors once every 30 min +*/30 * * * nobody ipfire.org check-mirrors -- 2.47.2