nopaste: Store blobs in a separate table

[ipfire.org.git] / src / backend / mirrors.py

diff --git a/src/backend/mirrors.py b/src/backend/mirrors.py
index 50ceb34d585611889c4b0911051c1dc2d0caabec..62f077656d550f6b49d6558e45bdfd5c4f6461b2 100644 (file)
--- a/src/backend/mirrors.py
+++ b/src/backend/mirrors.py
@@ -2,6 +2,7 @@
 
 import datetime
 import logging
+import iso3166
 import math
 import os.path
 import random
@@ -14,6 +15,7 @@ import tornado.netutil
 import urllib.parse
 
 from . import countries
+from . import util
 from .misc import Object
 from .decorators import *
 
@@ -68,6 +70,17 @@ class Mirrors(Object):
                return self._get_mirror("SELECT * FROM mirrors \
                        WHERE hostname = %s", hostname)
 
+       def get_by_countries(self):
+               mirrors = {}
+
+               for m in self:
+                       try:
+                               mirrors[m.country].append(m)
+                       except KeyError:
+                               mirrors[m.country] = [m]
+
+               return mirrors
+
 
 class Mirror(Object):
        def init(self, id, data=None):
@@ -88,6 +101,9 @@ class Mirror(Object):
                if isinstance(other, self.__class__):
                        return self.hostname < other.hostname
 
+       def __hash__(self):
+               return self.id
+
        @lazy_property
        def url(self):
                url = "%s://%s" % ("https" if self.supports_https else "http", self.hostname)
@@ -106,6 +122,14 @@ class Mirror(Object):
        def hostname(self):
                return self.data.hostname
 
+       @lazy_property
+       def address(self):
+               """
+                       Returns the stored address
+               """
+               if self.data.address:
+                       return util.Address(self.backend, self.data.address)
+
        @property
        def path(self):
                return self.data.path
@@ -114,47 +138,30 @@ class Mirror(Object):
        def supports_https(self):
                return self.data.supports_https
 
-       @property
-       def address(self):
-               for addr in self.addresses4:
-                       return addr
-
-               for addr in self.addresses6:
-                       return addr
-
        @property
        def owner(self):
                return self.data.owner
 
-       @lazy_property
-       def location(self):
-               return self.geoip.get_location(self.address)
-
-       @property
-       def latitude(self):
-               if self.location:
-                       return self.location.latitude
-
        @property
-       def longitude(self):
-               if self.location:
-                       return self.location.longitude
+       def country(self):
+               return iso3166.countries.get(self.country_code)
 
        @property
        def country_code(self):
-               return self.data.country_code
+               if self.data.country_code:
+                       return self.data.country_code
 
-       @property
-       def country_name(self):
-               return self.geoip.get_country_name(self.country_code)
+               if self.address:
+                       return self.address.country_code
 
        @property
        def zone(self):
                return countries.get_zone(self.country_name)
 
-       @lazy_property
+       @property
        def asn(self):
-               return self.geoip.get_asn(self.address)
+               if self.address:
+                       return self.address.asn
 
        @property
        def filelist(self):
@@ -199,7 +206,7 @@ class Mirror(Object):
                logging.debug("Running check for mirror %s" % self.hostname)
 
                self.db.execute("UPDATE mirrors SET address = %s WHERE id = %s",
-                       self.address, self.id)
+                       await self.resolve(), self.id)
 
                success = await self.check_timestamp()
                if success:
@@ -236,27 +243,27 @@ class Mirror(Object):
                        response = await http.fetch(self.url + ".timestamp",
                                headers={ "Pragma" : "no-cache" })
                except tornado.httpclient.HTTPError as e:
-                       logging.error("Error getting timestamp from %s: %s" % (self.hostname, e))
+                       logging.warning("Error getting timestamp from %s: %s" % (self.hostname, e))
                        self.set_state("DOWN")
                        return False
 
                except ssl.SSLError as e:
-                       logging.error("SSL error when getting timestamp from %s: %s" % (self.hostname, e))
+                       logging.warning("SSL error when getting timestamp from %s: %s" % (self.hostname, e))
                        self.set_state("DOWN")
                        return False
 
                except tornado.iostream.StreamClosedError as e:
-                       logging.error("Connection closed unexpectedly for %s: %s" % (self.hostname, e))
+                       logging.warning("Connection closed unexpectedly for %s: %s" % (self.hostname, e))
                        self.set_state("DOWN")
                        return False
 
                except OSError as e:
-                       logging.error("Could not connect to %s: %s" % (self.hostname, e))
+                       logging.warning("Could not connect to %s: %s" % (self.hostname, e))
                        self.set_state("DOWN")
                        return False
 
                if response.error:
-                       logging.debug("Error getting timestamp from %s" % self.hostname)
+                       logging.warning("Error getting timestamp from %s" % self.hostname)
                        self.set_state("DOWN")
                        return
 
@@ -288,7 +295,7 @@ class Mirror(Object):
                        response = await http.fetch(self.url + ".filelist",
                                headers={ "Pragma" : "no-cache" })
                except tornado.httpclient.HTTPError as e:
-                       logging.error("Error getting filelist from %s: %s" % (self.hostname, e))
+                       logging.warning("Error getting filelist from %s: %s" % (self.hostname, e))
                        self.set_state("DOWN")
                        return
 
@@ -316,27 +323,14 @@ class Mirror(Object):
        def mirrorlist(self):
                return self.data.get("mirrorlist", False)
 
-       @lazy_property
-       def addresses(self):
-               try:
-                       addrinfo = socket.getaddrinfo(self.hostname, 0, socket.AF_UNSPEC, socket.SOCK_STREAM)
-               except:
-                       raise Exception("Could not resolve %s" % self.hostname)
-
-               ret = []
-               for family, socktype, proto, canonname, address in addrinfo:
-                       if family == socket.AF_INET:
-                               address, port = address
-                       elif family == socket.AF_INET6:
-                               address, port, flowid, scopeid = address
-                       ret.append((family, address))
+       async def resolve(self):
+               """
+                       Returns a single IP address of this mirror
+               """
+               addresses = await self.backend.resolver.resolve(self.hostname, 0)
 
-               return ret
+               # Return the first address
+               for family, address in addresses:
+                       host, port = address
 
-       @property
-       def addresses6(self):
-               return [address for family, address in self.addresses if family == socket.AF_INET6]
-
-       @property
-       def addresses4(self):
-               return [address for family, address in self.addresses if family == socket.AF_INET]
+                       return host