Fix mirror check

This didn't run at all

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/backend/base.py b/src/backend/base.py
index c1088282e6e6a9025686b0a02f14ebb5f574201d..86e34611ea62fc9eee6dae97befdb51a0b1cef05 100644 (file)
--- a/src/backend/base.py
+++ b/src/backend/base.py
@@ -88,6 +88,7 @@ class Backend(object):
        @tornado.gen.coroutine
        def run_task(self, task, *args, **kwargs):
                tasks = {
+                       "check-mirrors"     : self.mirrors.check_all,
                        "cleanup-messages"  : self.messages.queue.cleanup,
                        "scan-files"        : self.releases.scan_files,
                        "send-all-messages" : self.messages.queue.send_all,

diff --git a/src/backend/mirrors.py b/src/backend/mirrors.py
index 646979bc3b78aecf7a0ee8b9274153dee1ad4677..5a1f3f2ba8a38e56a43056818835533ddca269ba 100644 (file)
--- a/src/backend/mirrors.py
+++ b/src/backend/mirrors.py
@@ -7,6 +7,7 @@ import os.path
 import random
 import socket
 import time
+import tornado.gen
 import tornado.httpclient
 import tornado.netutil
 import urllib.parse
@@ -34,9 +35,11 @@ class Mirrors(Object):
 
                return iter(mirrors)
 
+       @tornado.gen.coroutine
        def check_all(self):
                for mirror in self:
-                       mirror.check()
+                       with self.db.transaction():
+                               yield mirror.check()
 
        def get_for_download(self, filename, country_code=None):
                # Try to find a good mirror for this country first
@@ -192,16 +195,19 @@ class Mirror(Object):
        def disabled(self):
                return not self.enabled
 
+       @tornado.gen.coroutine
        def check(self):
                logging.info("Running check for mirror %s" % self.hostname)
 
                self.db.execute("UPDATE mirrors SET address = %s WHERE id = %s",
                        self.address, self.id)
 
-               self.check_timestamp()
-               self.check_filelist()
+               success = yield self.check_timestamp()
 
-       def check_state(self):
+               if success:
+                       yield self.check_filelist()
+
+       def check_state(self, last_update):
                logging.debug("Checking state of mirror %s" % self.id)
 
                if not self.enabled:
@@ -210,7 +216,7 @@ class Mirror(Object):
 
                now = datetime.datetime.utcnow()
 
-               time_delta = now - self.last_update
+               time_delta = now - last_update
                time_diff = time_delta.total_seconds()
 
                time_down = self.settings.get_int("mirrors_time_down", 3*24*60*60)
@@ -225,14 +231,18 @@ class Mirror(Object):
 
                self.set_state("UP")
 
+       @tornado.gen.coroutine
        def check_timestamp(self):
                http = tornado.httpclient.AsyncHTTPClient()
 
-               http.fetch(self.url + ".timestamp",
-                       headers={ "Pragma" : "no-cache" },
-                       callback=self.__check_timestamp_response)
+               try:
+                       response = yield http.fetch(self.url + ".timestamp",
+                               headers={ "Pragma" : "no-cache" })
+               except tornado.httpclient.HTTPError as e:
+                       logging.error("Error getting timestamp from %s: %s" % (self.hostname, e))
+                       self.set_state("DOWN")
+                       return False
 
-       def __check_timestamp_response(self, response):
                if response.error:
                        logging.debug("Error getting timestamp from %s" % self.hostname)
                        self.set_state("DOWN")
@@ -248,13 +258,13 @@ class Mirror(Object):
                self.db.execute("UPDATE mirrors SET last_update = %s WHERE id = %s",
                        timestamp, self.id)
 
-               # Reload changed settings
-               self.data["timestamp"] = timestamp
-
-               self.check_state()
+               # Update state
+               self.check_state(timestamp)
 
                logging.info("Successfully updated timestamp from %s" % self.hostname)
+               return True
 
+       @tornado.gen.coroutine
        def check_filelist(self):
                # XXX need to remove data from disabled mirrors
                if not self.enabled:
@@ -262,30 +272,27 @@ class Mirror(Object):
 
                http = tornado.httpclient.AsyncHTTPClient()
 
-               http.fetch(self.url + ".filelist",
-                       headers={ "Pragma" : "no-cache" },
-                       callback=self.__check_filelist_response)
+               try:
+                       response = yield http.fetch(self.url + ".filelist",
+                               headers={ "Pragma" : "no-cache" })
+               except tornado.httpclient.HTTPError as e:
+                       logging.error("Error getting filelist from %s: %s" % (self.hostname, e))
+                       self.set_state("DOWN")
+                       return
 
-       def __check_filelist_response(self, response):
                if response.error:
-                       logging.debug("Error getting timestamp from %s" % self.hostname)
+                       logging.debug("Error getting filelist from %s" % self.hostname)
                        return
 
-               files = self.filelist
+               # Drop the old filelist
+               self.db.execute("DELETE FROM mirror_files WHERE mirror = %s", self.id)
 
+               # Add them all again
                for file in response.body.splitlines():
-                       file = os.path.join(self.prefix, file)
-
-                       if file in files:
-                               files.remove(file)
-                               continue
-
-                       self.db.execute("INSERT INTO mirror_files(mirror, filename) VALUES(%s, %s)",
-                               self.id, file)
+                       file = os.path.join(self.prefix, file.decode())
 
-               for file in files:
-                       self.db.execute("DELETE FROM mirror_files WHERE mirror=%s AND filename=%s",
-                               self.id, file)
+                       self.db.execute("INSERT INTO mirror_files(mirror, filename) \
+                               VALUES(%s, %s)", self.id, file)
 
                logging.info("Successfully updated mirror filelist from %s" % self.hostname)
 

diff --git a/src/crontab/ipfire.org b/src/crontab/ipfire.org
index 2aed7c94375ef5883a3b8e7b36e4949fb8eec7dd..31ce551db615fff5740c26273f86485c582a2056 100644 (file)
--- a/src/crontab/ipfire.org
+++ b/src/crontab/ipfire.org
@@ -9,3 +9,6 @@
 
 # Cleanup once a an hour
 30 * * * *     nobody  ipfire.org cleanup-messages
+
+# Check mirrors once every 30 min
+*/30 * * *     nobody  ipfire.org check-mirrors