src/backend/mirrors.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import logging
   5 import math
   6 import os.path
   7 import random
   8 import socket
   9 import time
  10 import tornado.gen
  11 import tornado.httpclient
  12 import tornado.netutil
  13 import urllib.parse
  14
  15 from . import countries
  16 from .misc import Object
  17 from .decorators import *
  18
  19 class Mirrors(Object):
  20         def _get_mirrors(self, query, *args):
  21                 res = self.db.query(query, *args)
  22
  23                 for row in res:
  24                         yield Mirror(self.backend, row.id, data=row)
  25
  26         def _get_mirror(self, query, *args):
  27                 res = self.db.get(query, *args)
  28
  29                 if res:
  30                         return Mirror(self.backend, res.id, data=res)
  31
  32         def __iter__(self):
  33                 mirrors = self._get_mirrors("SELECT * FROM mirrors \
  34                         WHERE enabled IS TRUE ORDER BY hostname")
  35
  36                 return iter(mirrors)
  37
  38         @tornado.gen.coroutine
  39         def check_all(self):
  40                 for mirror in self:
  41                         with self.db.transaction():
  42                                 yield mirror.check()
  43
  44         def get_for_download(self, filename, country_code=None):
  45                 # Try to find a good mirror for this country first
  46                 if country_code:
  47                         zone = countries.get_zone(country_code)
  48
  49                         mirror = self._get_mirror("SELECT mirrors.* FROM mirror_files files \
  50                                 LEFT JOIN mirrors ON files.mirror = mirrors.id \
  51                                 WHERE files.filename = %s \
  52                                         AND mirrors.enabled IS TRUE AND mirrors.state = %s \
  53                                         AND mirrors.country_code = ANY(%s) \
  54                                         ORDER BY RANDOM() LIMIT 1", filename, "UP",
  55                                         countries.get_in_zone(zone))
  56
  57                         if mirror:
  58                                 return mirror
  59
  60                 # Get a random mirror that serves the file
  61                 return self._get_mirror("SELECT mirrors.* FROM mirror_files files \
  62                         LEFT JOIN mirrors ON files.mirror = mirrors.id \
  63                         WHERE files.filename = %s \
  64                                 AND mirrors.enabled IS TRUE AND mirrors.state = %s \
  65                                 ORDER BY RANDOM() LIMIT 1", filename, "UP")
  66
  67         def get_by_hostname(self, hostname):
  68                 return self._get_mirror("SELECT * FROM mirrors \
  69                         WHERE hostname = %s", hostname)
  70
  71
  72 class Mirror(Object):
  73         def init(self, id, data=None):
  74                 self.id   = id
  75                 self.data = data
  76
  77         def __str__(self):
  78                 return self.hostname
  79
  80         def __repr__(self):
  81                 return "<%s %s>" % (self.__class__.__name__, self.url)
  82
  83         def __eq__(self, other):
  84                 if isinstance(other, self.__class__):
  85                         return self.id == other.id
  86
  87         def __lt__(self, other):
  88                 if isinstance(other, self.__class__):
  89                         return self.hostname < other.hostname
  90
  91         @lazy_property
  92         def url(self):
  93                 url = "%s://%s" % ("https" if self.supports_https else "http", self.hostname)
  94
  95                 if not self.path.startswith("/"):
  96                         url += "/"
  97
  98                 url += "%s" % self.path
  99
 100                 if not self.path.endswith("/"):
 101                         url += "/"
 102
 103                 return url
 104
 105         @property
 106         def hostname(self):
 107                 return self.data.hostname
 108
 109         @property
 110         def path(self):
 111                 return self.data.path
 112
 113         @property
 114         def supports_https(self):
 115                 return self.data.supports_https
 116
 117         @property
 118         def address(self):
 119                 for addr in self.addresses4:
 120                         return addr
 121
 122                 for addr in self.addresses6:
 123                         return addr
 124
 125         @property
 126         def owner(self):
 127                 return self.data.owner
 128
 129         @lazy_property
 130         def location(self):
 131                 return self.geoip.get_location(self.address)
 132
 133         @property
 134         def latitude(self):
 135                 if self.location:
 136                         return self.location.latitude
 137
 138         @property
 139         def longitude(self):
 140                 if self.location:
 141                         return self.location.longitude
 142
 143         @property
 144         def country_code(self):
 145                 return self.data.country_code
 146
 147         @property
 148         def country_name(self):
 149                 return self.geoip.get_country_name(self.country_code)
 150
 151         @property
 152         def zone(self):
 153                 return countries.get_zone(self.country_name)
 154
 155         @lazy_property
 156         def asn(self):
 157                 return self.geoip.get_asn(self.address)
 158
 159         @property
 160         def filelist(self):
 161                 filelist = self.db.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self.id)
 162                 return [f.filename for f in filelist]
 163
 164         @property
 165         def prefix(self):
 166                 return ""
 167
 168         def build_url(self, filename):
 169                 return urllib.parse.urljoin(self.url, filename)
 170
 171         @property
 172         def last_update(self):
 173                 return self.data.last_update
 174
 175         @property
 176         def state(self):
 177                 return self.data.state
 178
 179         def set_state(self, state):
 180                 logging.info("Setting state of %s to %s" % (self.hostname, state))
 181
 182                 if self.state == state:
 183                         return
 184
 185                 self.db.execute("UPDATE mirrors SET state = %s WHERE id = %s", state, self.id)
 186
 187                 # Reload changed settings
 188                 self.data["state"] = state
 189
 190         @property
 191         def enabled(self):
 192                 return self.data.enabled
 193
 194         @property
 195         def disabled(self):
 196                 return not self.enabled
 197
 198         @tornado.gen.coroutine
 199         def check(self):
 200                 logging.info("Running check for mirror %s" % self.hostname)
 201
 202                 self.db.execute("UPDATE mirrors SET address = %s WHERE id = %s",
 203                         self.address, self.id)
 204
 205                 success = yield self.check_timestamp()
 206
 207                 if success:
 208                         yield self.check_filelist()
 209
 210         def check_state(self, last_update):
 211                 logging.debug("Checking state of mirror %s" % self.id)
 212
 213                 if not self.enabled:
 214                         self.set_state("DOWN")
 215                         return
 216
 217                 now = datetime.datetime.utcnow()
 218
 219                 time_delta = now - last_update
 220                 time_diff = time_delta.total_seconds()
 221
 222                 time_down = self.settings.get_int("mirrors_time_down", 3*24*60*60)
 223                 if time_diff >= time_down:
 224                         self.set_state("DOWN")
 225                         return
 226
 227                 time_outofsync = self.settings.get_int("mirrors_time_outofsync", 6*60*60)
 228                 if time_diff >= time_outofsync:
 229                         self.set_state("OUTOFSYNC")
 230                         return
 231
 232                 self.set_state("UP")
 233
 234         @tornado.gen.coroutine
 235         def check_timestamp(self):
 236                 http = tornado.httpclient.AsyncHTTPClient()
 237
 238                 try:
 239                         response = yield http.fetch(self.url + ".timestamp",
 240                                 headers={ "Pragma" : "no-cache" })
 241                 except tornado.httpclient.HTTPError as e:
 242                         logging.error("Error getting timestamp from %s: %s" % (self.hostname, e))
 243                         self.set_state("DOWN")
 244                         return False
 245
 246                 if response.error:
 247                         logging.debug("Error getting timestamp from %s" % self.hostname)
 248                         self.set_state("DOWN")
 249                         return
 250
 251                 try:
 252                         timestamp = int(response.body.strip())
 253                 except ValueError:
 254                         timestamp = 0
 255
 256                 timestamp = datetime.datetime.utcfromtimestamp(timestamp)
 257
 258                 self.db.execute("UPDATE mirrors SET last_update = %s WHERE id = %s",
 259                         timestamp, self.id)
 260
 261                 # Update state
 262                 self.check_state(timestamp)
 263
 264                 logging.info("Successfully updated timestamp from %s" % self.hostname)
 265                 return True
 266
 267         @tornado.gen.coroutine
 268         def check_filelist(self):
 269                 # XXX need to remove data from disabled mirrors
 270                 if not self.enabled:
 271                         return
 272
 273                 http = tornado.httpclient.AsyncHTTPClient()
 274
 275                 try:
 276                         response = yield http.fetch(self.url + ".filelist",
 277                                 headers={ "Pragma" : "no-cache" })
 278                 except tornado.httpclient.HTTPError as e:
 279                         logging.error("Error getting filelist from %s: %s" % (self.hostname, e))
 280                         self.set_state("DOWN")
 281                         return
 282
 283                 if response.error:
 284                         logging.debug("Error getting filelist from %s" % self.hostname)
 285                         return
 286
 287                 # Drop the old filelist
 288                 self.db.execute("DELETE FROM mirror_files WHERE mirror = %s", self.id)
 289
 290                 # Add them all again
 291                 for file in response.body.splitlines():
 292                         file = os.path.join(self.prefix, file.decode())
 293
 294                         self.db.execute("INSERT INTO mirror_files(mirror, filename) \
 295                                 VALUES(%s, %s)", self.id, file)
 296
 297                 logging.info("Successfully updated mirror filelist from %s" % self.hostname)
 298
 299         @property
 300         def development(self):
 301                 return self.data.get("mirrorlist_devel", False)
 302
 303         @property
 304         def mirrorlist(self):
 305                 return self.data.get("mirrorlist", False)
 306
 307         @lazy_property
 308         def addresses(self):
 309                 try:
 310                         addrinfo = socket.getaddrinfo(self.hostname, 0, socket.AF_UNSPEC, socket.SOCK_STREAM)
 311                 except:
 312                         raise Exception("Could not resolve %s" % self.hostname)
 313
 314                 ret = []
 315                 for family, socktype, proto, canonname, address in addrinfo:
 316                         if family == socket.AF_INET:
 317                                 address, port = address
 318                         elif family == socket.AF_INET6:
 319                                 address, port, flowid, scopeid = address
 320                         ret.append((family, address))
 321
 322                 return ret
 323
 324         @property
 325         def addresses6(self):
 326                 return [address for family, address in self.addresses if family == socket.AF_INET6]
 327
 328         @property
 329         def addresses4(self):
 330                 return [address for family, address in self.addresses if family == socket.AF_INET]