src/backend/mirrors.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import logging
   5 import math
   6 import os.path
   7 import random
   8 import socket
   9 import time
  10 import ssl
  11 import tornado.gen
  12 import tornado.httpclient
  13 import tornado.netutil
  14 import urllib.parse
  15
  16 from . import countries
  17 from .misc import Object
  18 from .decorators import *
  19
  20 class Mirrors(Object):
  21         def _get_mirrors(self, query, *args):
  22                 res = self.db.query(query, *args)
  23
  24                 for row in res:
  25                         yield Mirror(self.backend, row.id, data=row)
  26
  27         def _get_mirror(self, query, *args):
  28                 res = self.db.get(query, *args)
  29
  30                 if res:
  31                         return Mirror(self.backend, res.id, data=res)
  32
  33         def __iter__(self):
  34                 mirrors = self._get_mirrors("SELECT * FROM mirrors \
  35                         WHERE enabled IS TRUE ORDER BY hostname")
  36
  37                 return iter(mirrors)
  38
  39         @tornado.gen.coroutine
  40         def check_all(self):
  41                 for mirror in self:
  42                         with self.db.transaction():
  43                                 yield mirror.check()
  44
  45         def get_for_download(self, filename, country_code=None):
  46                 # Try to find a good mirror for this country first
  47                 if country_code:
  48                         zone = countries.get_zone(country_code)
  49
  50                         mirror = self._get_mirror("SELECT mirrors.* FROM mirror_files files \
  51                                 LEFT JOIN mirrors ON files.mirror = mirrors.id \
  52                                 WHERE files.filename = %s \
  53                                         AND mirrors.enabled IS TRUE AND mirrors.state = %s \
  54                                         AND mirrors.country_code = ANY(%s) \
  55                                         ORDER BY RANDOM() LIMIT 1", filename, "UP",
  56                                         countries.get_in_zone(zone))
  57
  58                         if mirror:
  59                                 return mirror
  60
  61                 # Get a random mirror that serves the file
  62                 return self._get_mirror("SELECT mirrors.* FROM mirror_files files \
  63                         LEFT JOIN mirrors ON files.mirror = mirrors.id \
  64                         WHERE files.filename = %s \
  65                                 AND mirrors.enabled IS TRUE AND mirrors.state = %s \
  66                                 ORDER BY RANDOM() LIMIT 1", filename, "UP")
  67
  68         def get_by_hostname(self, hostname):
  69                 return self._get_mirror("SELECT * FROM mirrors \
  70                         WHERE hostname = %s", hostname)
  71
  72
  73 class Mirror(Object):
  74         def init(self, id, data=None):
  75                 self.id   = id
  76                 self.data = data
  77
  78         def __str__(self):
  79                 return self.hostname
  80
  81         def __repr__(self):
  82                 return "<%s %s>" % (self.__class__.__name__, self.url)
  83
  84         def __eq__(self, other):
  85                 if isinstance(other, self.__class__):
  86                         return self.id == other.id
  87
  88         def __lt__(self, other):
  89                 if isinstance(other, self.__class__):
  90                         return self.hostname < other.hostname
  91
  92         @lazy_property
  93         def url(self):
  94                 url = "%s://%s" % ("https" if self.supports_https else "http", self.hostname)
  95
  96                 if not self.path.startswith("/"):
  97                         url += "/"
  98
  99                 url += "%s" % self.path
 100
 101                 if not self.path.endswith("/"):
 102                         url += "/"
 103
 104                 return url
 105
 106         @property
 107         def hostname(self):
 108                 return self.data.hostname
 109
 110         @property
 111         def path(self):
 112                 return self.data.path
 113
 114         @property
 115         def supports_https(self):
 116                 return self.data.supports_https
 117
 118         @property
 119         def address(self):
 120                 for addr in self.addresses4:
 121                         return addr
 122
 123                 for addr in self.addresses6:
 124                         return addr
 125
 126         @property
 127         def owner(self):
 128                 return self.data.owner
 129
 130         @lazy_property
 131         def location(self):
 132                 return self.geoip.get_location(self.address)
 133
 134         @property
 135         def latitude(self):
 136                 if self.location:
 137                         return self.location.latitude
 138
 139         @property
 140         def longitude(self):
 141                 if self.location:
 142                         return self.location.longitude
 143
 144         @property
 145         def country_code(self):
 146                 return self.data.country_code
 147
 148         @property
 149         def country_name(self):
 150                 return self.geoip.get_country_name(self.country_code)
 151
 152         @property
 153         def zone(self):
 154                 return countries.get_zone(self.country_name)
 155
 156         @lazy_property
 157         def asn(self):
 158                 return self.geoip.get_asn(self.address)
 159
 160         @property
 161         def filelist(self):
 162                 filelist = self.db.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self.id)
 163                 return [f.filename for f in filelist]
 164
 165         @property
 166         def prefix(self):
 167                 return ""
 168
 169         def build_url(self, filename):
 170                 return urllib.parse.urljoin(self.url, filename)
 171
 172         @property
 173         def last_update(self):
 174                 return self.data.last_update
 175
 176         @property
 177         def state(self):
 178                 return self.data.state
 179
 180         def set_state(self, state):
 181                 logging.debug("Setting state of %s to %s" % (self.hostname, state))
 182
 183                 if self.state == state:
 184                         return
 185
 186                 self.db.execute("UPDATE mirrors SET state = %s WHERE id = %s", state, self.id)
 187
 188                 # Reload changed settings
 189                 self.data["state"] = state
 190
 191         @property
 192         def enabled(self):
 193                 return self.data.enabled
 194
 195         @property
 196         def disabled(self):
 197                 return not self.enabled
 198
 199         @tornado.gen.coroutine
 200         def check(self):
 201                 logging.debug("Running check for mirror %s" % self.hostname)
 202
 203                 self.db.execute("UPDATE mirrors SET address = %s WHERE id = %s",
 204                         self.address, self.id)
 205
 206                 success = yield self.check_timestamp()
 207
 208                 if success:
 209                         yield self.check_filelist()
 210
 211         def check_state(self, last_update):
 212                 logging.debug("Checking state of mirror %s" % self.id)
 213
 214                 if not self.enabled:
 215                         self.set_state("DOWN")
 216                         return
 217
 218                 now = datetime.datetime.utcnow()
 219
 220                 time_delta = now - last_update
 221                 time_diff = time_delta.total_seconds()
 222
 223                 time_down = self.settings.get_int("mirrors_time_down", 3*24*60*60)
 224                 if time_diff >= time_down:
 225                         self.set_state("DOWN")
 226                         return
 227
 228                 time_outofsync = self.settings.get_int("mirrors_time_outofsync", 6*60*60)
 229                 if time_diff >= time_outofsync:
 230                         self.set_state("OUTOFSYNC")
 231                         return
 232
 233                 self.set_state("UP")
 234
 235         @tornado.gen.coroutine
 236         def check_timestamp(self):
 237                 http = tornado.httpclient.AsyncHTTPClient()
 238
 239                 try:
 240                         response = yield http.fetch(self.url + ".timestamp",
 241                                 headers={ "Pragma" : "no-cache" })
 242                 except tornado.httpclient.HTTPError as e:
 243                         logging.error("Error getting timestamp from %s: %s" % (self.hostname, e))
 244                         self.set_state("DOWN")
 245                         return False
 246
 247                 except ssl.SSLError as e:
 248                         logging.error("SSL error when getting timestamp from %s: %s" % (self.hostname, e))
 249                         self.set_state("DOWN")
 250                         return False
 251
 252                 if response.error:
 253                         logging.debug("Error getting timestamp from %s" % self.hostname)
 254                         self.set_state("DOWN")
 255                         return
 256
 257                 try:
 258                         timestamp = int(response.body.strip())
 259                 except ValueError:
 260                         timestamp = 0
 261
 262                 timestamp = datetime.datetime.utcfromtimestamp(timestamp)
 263
 264                 self.db.execute("UPDATE mirrors SET last_update = %s WHERE id = %s",
 265                         timestamp, self.id)
 266
 267                 # Update state
 268                 self.check_state(timestamp)
 269
 270                 logging.debug("Successfully updated timestamp from %s" % self.hostname)
 271
 272                 return True
 273
 274         @tornado.gen.coroutine
 275         def check_filelist(self):
 276                 # XXX need to remove data from disabled mirrors
 277                 if not self.enabled:
 278                         return
 279
 280                 http = tornado.httpclient.AsyncHTTPClient()
 281
 282                 try:
 283                         response = yield http.fetch(self.url + ".filelist",
 284                                 headers={ "Pragma" : "no-cache" })
 285                 except tornado.httpclient.HTTPError as e:
 286                         logging.error("Error getting filelist from %s: %s" % (self.hostname, e))
 287                         self.set_state("DOWN")
 288                         return
 289
 290                 if response.error:
 291                         logging.debug("Error getting filelist from %s" % self.hostname)
 292                         return
 293
 294                 # Drop the old filelist
 295                 self.db.execute("DELETE FROM mirror_files WHERE mirror = %s", self.id)
 296
 297                 # Add them all again
 298                 for file in response.body.splitlines():
 299                         file = os.path.join(self.prefix, file.decode())
 300
 301                         self.db.execute("INSERT INTO mirror_files(mirror, filename) \
 302                                 VALUES(%s, %s)", self.id, file)
 303
 304                 logging.debug("Successfully updated mirror filelist from %s" % self.hostname)
 305
 306         @property
 307         def development(self):
 308                 return self.data.get("mirrorlist_devel", False)
 309
 310         @property
 311         def mirrorlist(self):
 312                 return self.data.get("mirrorlist", False)
 313
 314         @lazy_property
 315         def addresses(self):
 316                 try:
 317                         addrinfo = socket.getaddrinfo(self.hostname, 0, socket.AF_UNSPEC, socket.SOCK_STREAM)
 318                 except:
 319                         raise Exception("Could not resolve %s" % self.hostname)
 320
 321                 ret = []
 322                 for family, socktype, proto, canonname, address in addrinfo:
 323                         if family == socket.AF_INET:
 324                                 address, port = address
 325                         elif family == socket.AF_INET6:
 326                                 address, port, flowid, scopeid = address
 327                         ret.append((family, address))
 328
 329                 return ret
 330
 331         @property
 332         def addresses6(self):
 333                 return [address for family, address in self.addresses if family == socket.AF_INET6]
 334
 335         @property
 336         def addresses4(self):
 337                 return [address for family, address in self.addresses if family == socket.AF_INET]