src/backend/mirrors.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import logging
   5 import math
   6 import os.path
   7 import random
   8 import socket
   9 import time
  10 import tornado.httpclient
  11 import tornado.netutil
  12 import urllib.parse
  13
  14 from .misc import Object
  15 from .decorators import *
  16
  17 class Downloads(Object):
  18         @property
  19         def total(self):
  20                 ret = self.db.get("SELECT COUNT(*) AS total FROM log_download")
  21
  22                 return ret.total
  23
  24         @property
  25         def today(self):
  26                 ret = self.db.get("SELECT COUNT(*) AS today FROM log_download WHERE date::date = NOW()::date")
  27
  28                 return ret.today
  29
  30         @property
  31         def yesterday(self):
  32                 ret = self.db.get("SELECT COUNT(*) AS yesterday FROM log_download WHERE date::date = (NOW() - INTERVAL '1 day')::date")
  33
  34                 return ret.yesterday
  35
  36         @property
  37         def daily_map(self):
  38                 ret = self.db.query("WITH downloads AS (SELECT * FROM log_download \
  39                         WHERE DATE(date) BETWEEN (NOW()::date - INTERVAL '30 days') AND DATE(NOW())) \
  40                         SELECT DATE(date) AS date, COUNT(*) AS count FROM downloads \
  41                         GROUP BY DATE(date) ORDER BY date")
  42
  43                 return ret
  44
  45         def get_countries(self, duration="all"):
  46                 query = "SELECT country_code, count(country_code) AS count FROM log_download"
  47
  48                 if duration == "today":
  49                         query += " WHERE date::date = NOW()::date"
  50
  51                 query += " GROUP BY country_code ORDER BY count DESC"
  52
  53                 results = self.db.query(query)
  54                 ret = []
  55
  56                 count = sum([o.count for o in results])
  57                 if count:
  58                         for res in results:
  59                                 ret.append((res.country_code, res.count / count))
  60
  61                 return ret
  62
  63         def get_mirror_load(self, duration="all"):
  64                 query = "SELECT mirror, COUNT(mirror) AS count FROM log_download"
  65
  66                 if duration == "today":
  67                         query += " WHERE date::date = NOW()::date"
  68
  69                 query += " GROUP BY mirror ORDER BY count DESC"
  70
  71                 results = self.db.query(query)
  72                 ret = {}
  73
  74                 count = sum([o.count for o in results])
  75                 if count:
  76                         for res in results:
  77                                 mirror = self.mirrors.get(res.mirror)
  78                                 ret[mirror.hostname] = res.count / count
  79
  80                 return ret
  81
  82
  83 class Mirrors(Object):
  84         def _get_mirrors(self, query, *args):
  85                 res = self.db.query(query, *args)
  86
  87                 for row in res:
  88                         yield Mirror(self.backend, row.id, data=row)
  89
  90         def __iter__(self):
  91                 mirrors = self._get_mirrors("SELECT * FROM mirrors \
  92                         WHERE enabled IS TRUE ORDER BY hostname")
  93
  94                 return iter(mirrors)
  95
  96         def check_all(self):
  97                 for mirror in self:
  98                         mirror.check()
  99
 100         def get(self, id):
 101                 return Mirror(self.backend, id)
 102
 103         def get_by_hostname(self, hostname):
 104                 ret = self.db.get("SELECT * FROM mirrors WHERE hostname = %s", hostname)
 105
 106                 if ret:
 107                         return Mirror(self.backend, ret.id, ret)
 108
 109         def get_with_file(self, filename, country=None):
 110                 # XXX quick and dirty solution - needs a performance boost
 111                 mirror_ids = [m.mirror for m in self.db.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename)]
 112
 113                 #if country:
 114                 #       # Sort out all mirrors that are not preferred to the given country
 115                 #       for mirror in self.get_for_country(country):
 116                 #               if not mirror.id in mirror_ids:
 117                 #                       mirror_ids.remove(mirror.id)
 118
 119                 mirrors = []
 120                 for mirror_id in mirror_ids:
 121                         mirror = self.get(mirror_id)
 122                         if not mirror.state == "UP":
 123                                 continue
 124                         mirrors.append(mirror)
 125
 126                 return mirrors
 127
 128         def get_for_location(self, location, max_distance=4000, filename=None):
 129                 if not location:
 130                         return []
 131
 132                 if filename:
 133                         res = self.db.query("\
 134                                 WITH client AS (SELECT point(%s, %s) AS location) \
 135                                 SELECT * FROM mirrors WHERE mirrors.state = %s \
 136                                         AND mirrors.id IN ( \
 137                                                 SELECT mirror FROM mirror_files WHERE filename = %s \
 138                                         ) AND mirrors.id IN ( \
 139                                         SELECT id FROM mirrors_locations, client \
 140                                                 WHERE geodistance(mirrors_locations.location, client.location) <= %s \
 141                                 )",
 142                                 location.latitude, location.longitude, "UP", filename, max_distance)
 143                 else:
 144                         res = self.db.query("\
 145                                 WITH client AS (SELECT point(%s, %s) AS location) \
 146                                 SELECT * FROM mirrors WHERE mirrors.state = %s AND mirrors.id IN ( \
 147                                         SELECT id FROM mirrors_locations, client \
 148                                                 WHERE geodistance(mirrors_locations.location, client.location) <= %s \
 149                                 )",
 150                                 location.latitude, location.longitude, "UP", max_distance)
 151
 152                 mirrors = []
 153                 for row in res:
 154                         mirror = Mirror(self.backend, row.id, row)
 155                         mirrors.append(mirror)
 156
 157                 return sorted(mirrors, reverse=True)
 158
 159         def get_all_files(self):
 160                 files = []
 161
 162                 for mirror in self:
 163                         if not mirror.state == "UP":
 164                                 continue
 165
 166                         for file in mirror.filelist:
 167                                 if not file in files:
 168                                         files.append(file)
 169
 170                 return files
 171
 172         def get_random(self, filename=None):
 173                 if filename:
 174                         ret = self.db.get("SELECT * FROM mirrors WHERE state = %s \
 175                                 AND mirrors.id IN (SELECT mirror FROM mirror_files \
 176                                         WHERE filename = %s) ORDER BY RANDOM() LIMIT 1", "UP", filename)
 177                 else:
 178                         ret = self.db.get("SELECT * FROM mirrors WHERE state = %s \
 179                                 ORDER BY RANDOM() LIMIT 1", "UP")
 180
 181                 if ret:
 182                         return Mirror(self.backend, ret.id, ret)
 183
 184         def file_exists(self, filename):
 185                 ret = self.db.get("SELECT 1 FROM mirror_files \
 186                         WHERE filename = %s LIMIT 1", filename)
 187
 188                 if ret:
 189                         return True
 190
 191                 return False
 192
 193
 194 class MirrorSet(Object):
 195         def __init__(self, backend, mirrors):
 196                 Object.__init__(self, backend)
 197
 198                 self._mirrors = mirrors
 199
 200         def __add__(self, other):
 201                 mirrors = []
 202
 203                 for mirror in self._mirrors + other._mirrors:
 204                         if mirror in mirrors:
 205                                 continue
 206
 207                         mirrors.append(mirror)
 208
 209                 return MirrorSet(self.backend, mirrors)
 210
 211         def __sub__(self, other):
 212                 mirrors = self._mirrors[:]
 213
 214                 for mirror in other._mirrors:
 215                         if mirror in mirrors:
 216                                 mirrors.remove(mirror)
 217
 218                 return MirrorSet(self.backend, mirrors)
 219
 220         def __iter__(self):
 221                 return iter(self._mirrors)
 222
 223         def __len__(self):
 224                 return len(self._mirrors)
 225
 226         def __str__(self):
 227                 return "<MirrorSet %s>" % ", ".join([m.hostname for m in self._mirrors])
 228
 229         def get_with_file(self, filename):
 230                 with_file = [m.mirror for m in self.db.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename)]
 231
 232                 mirrors = []
 233                 for mirror in self._mirrors:
 234                         if mirror.id in with_file:
 235                                 mirrors.append(mirror)
 236
 237                 return MirrorSet(self.backend, mirrors)
 238
 239         def get_random(self):
 240                 mirrors = []
 241                 for mirror in self._mirrors:
 242                         for i in range(0, mirror.priority):
 243                                 mirrors.append(mirror)
 244
 245                 return random.choice(mirrors)
 246
 247         def get_for_location(self, location):
 248                 distance = 2500
 249                 mirrors = []
 250
 251                 if location:
 252                         while len(mirrors) <= 3 and distance <= 8000:
 253                                 for mirror in self._mirrors:
 254                                         if mirror in mirrors:
 255                                                 continue
 256
 257                                         mirror_distance = mirror.distance_to(location)
 258                                         if mirror_distance is None:
 259                                                 continue
 260
 261                                         if mirror_distance <= distance:
 262                                                 mirrors.append(mirror)
 263
 264                                 distance *= 1.2
 265
 266                 return MirrorSet(self.backend, mirrors)
 267
 268         def get_with_state(self, state):
 269                 mirrors = []
 270
 271                 for mirror in self._mirrors:
 272                         if mirror.state == state:
 273                                 mirrors.append(mirror)
 274
 275                 return MirrorSet(self.backend, mirrors)
 276
 277
 278 class Mirror(Object):
 279         def init(self, id, data=None):
 280                 self.id   = id
 281                 self.data = data
 282
 283         def __str__(self):
 284                 return self.hostname
 285
 286         def __repr__(self):
 287                 return "<%s %s>" % (self.__class__.__name__, self.url)
 288
 289         def __eq__(self, other):
 290                 if isinstance(other, self.__class__):
 291                         return self.id == other.id
 292
 293         def __lt__(self, other):
 294                 if isinstance(other, self.__class__):
 295                         return self.hostname < other.hostname
 296
 297         @lazy_property
 298         def url(self):
 299                 url = "%s://%s" % ("https" if self.supports_https else "http", self.hostname)
 300
 301                 if not self.path.startswith("/"):
 302                         url += "/"
 303
 304                 url += "%s" % self.path
 305
 306                 if not self.path.endswith("/"):
 307                         url += "/"
 308
 309                 return url
 310
 311         @property
 312         def hostname(self):
 313                 return self.data.hostname
 314
 315         @property
 316         def path(self):
 317                 return self.data.path
 318
 319         @property
 320         def supports_https(self):
 321                 return self.data.supports_https
 322
 323         @property
 324         def address(self):
 325                 for addr in self.addresses4:
 326                         return addr
 327
 328                 for addr in self.addresses6:
 329                         return addr
 330
 331         @property
 332         def owner(self):
 333                 return self.data.owner
 334
 335         @lazy_property
 336         def location(self):
 337                 return self.geoip.get_location(self.address)
 338
 339         @property
 340         def latitude(self):
 341                 if self.location:
 342                         return self.location.latitude
 343
 344         @property
 345         def longitude(self):
 346                 if self.location:
 347                         return self.location.longitude
 348
 349         @property
 350         def country_code(self):
 351                 if self.location:
 352                         return self.location.country
 353
 354         @property
 355         def country_name(self):
 356                 return self.geoip.get_country_name(self.country_code)
 357
 358         @lazy_property
 359         def asn(self):
 360                 return self.geoip.get_asn(self.address)
 361
 362         @property
 363         def filelist(self):
 364                 filelist = self.db.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self.id)
 365                 return [f.filename for f in filelist]
 366
 367         @property
 368         def prefix(self):
 369                 return ""
 370
 371         def build_url(self, filename):
 372                 return urllib.parse.urljoin(self.url, filename)
 373
 374         @property
 375         def last_update(self):
 376                 return self.data.last_update
 377
 378         @property
 379         def state(self):
 380                 return self.data.state
 381
 382         def set_state(self, state):
 383                 logging.info("Setting state of %s to %s" % (self.hostname, state))
 384
 385                 if self.state == state:
 386                         return
 387
 388                 self.db.execute("UPDATE mirrors SET state = %s WHERE id = %s", state, self.id)
 389
 390                 # Reload changed settings
 391                 self.data["state"] = state
 392
 393         @property
 394         def enabled(self):
 395                 return self.data.enabled
 396
 397         @property
 398         def disabled(self):
 399                 return not self.enabled
 400
 401         def check(self):
 402                 logging.info("Running check for mirror %s" % self.hostname)
 403
 404                 self.db.execute("UPDATE mirrors SET address = %s WHERE id = %s",
 405                         self.address, self.id)
 406
 407                 self.check_timestamp()
 408                 self.check_filelist()
 409
 410         def check_state(self):
 411                 logging.debug("Checking state of mirror %s" % self.id)
 412
 413                 if not self.enabled:
 414                         self.set_state("DOWN")
 415                         return
 416
 417                 now = datetime.datetime.utcnow()
 418
 419                 time_delta = now - self.last_update
 420                 time_diff = time_delta.total_seconds()
 421
 422                 time_down = self.settings.get_int("mirrors_time_down", 3*24*60*60)
 423                 if time_diff >= time_down:
 424                         self.set_state("DOWN")
 425                         return
 426
 427                 time_outofsync = self.settings.get_int("mirrors_time_outofsync", 6*60*60)
 428                 if time_diff >= time_outofsync:
 429                         self.set_state("OUTOFSYNC")
 430                         return
 431
 432                 self.set_state("UP")
 433
 434         def check_timestamp(self):
 435                 http = tornado.httpclient.AsyncHTTPClient()
 436
 437                 http.fetch(self.url + ".timestamp",
 438                         headers={ "Pragma" : "no-cache" },
 439                         callback=self.__check_timestamp_response)
 440
 441         def __check_timestamp_response(self, response):
 442                 if response.error:
 443                         logging.debug("Error getting timestamp from %s" % self.hostname)
 444                         self.set_state("DOWN")
 445                         return
 446
 447                 try:
 448                         timestamp = int(response.body.strip())
 449                 except ValueError:
 450                         timestamp = 0
 451
 452                 timestamp = datetime.datetime.utcfromtimestamp(timestamp)
 453
 454                 self.db.execute("UPDATE mirrors SET last_update = %s WHERE id = %s",
 455                         timestamp, self.id)
 456
 457                 # Reload changed settings
 458                 self.data["timestamp"] = timestamp
 459
 460                 self.check_state()
 461
 462                 logging.info("Successfully updated timestamp from %s" % self.hostname)
 463
 464         def check_filelist(self):
 465                 # XXX need to remove data from disabled mirrors
 466                 if not self.enabled:
 467                         return
 468
 469                 http = tornado.httpclient.AsyncHTTPClient()
 470
 471                 http.fetch(self.url + ".filelist",
 472                         headers={ "Pragma" : "no-cache" },
 473                         callback=self.__check_filelist_response)
 474
 475         def __check_filelist_response(self, response):
 476                 if response.error:
 477                         logging.debug("Error getting timestamp from %s" % self.hostname)
 478                         return
 479
 480                 files = self.filelist
 481
 482                 for file in response.body.splitlines():
 483                         file = os.path.join(self.prefix, file)
 484
 485                         if file in files:
 486                                 files.remove(file)
 487                                 continue
 488
 489                         self.db.execute("INSERT INTO mirror_files(mirror, filename) VALUES(%s, %s)",
 490                                 self.id, file)
 491
 492                 for file in files:
 493                         self.db.execute("DELETE FROM mirror_files WHERE mirror=%s AND filename=%s",
 494                                 self.id, file)
 495
 496                 logging.info("Successfully updated mirror filelist from %s" % self.hostname)
 497
 498         @property
 499         def prefer_for_countries(self):
 500                 countries = self.data.get("prefer_for_countries", "")
 501                 if countries:
 502                         return sorted(countries.split(", "))
 503
 504                 return []
 505
 506         @property
 507         def prefer_for_countries_names(self):
 508                 countries = [self.geoip.get_country_name(c.upper()) for c in self.prefer_for_countries]
 509
 510                 return sorted(countries)
 511
 512         def distance_to(self, location, ignore_preference=False):
 513                 if not location:
 514                         return None
 515
 516                 country_code = None
 517                 if location.country:
 518                         country_code = location.country.lower()
 519
 520                 if not ignore_preference and country_code in self.prefer_for_countries:
 521                         return 0
 522
 523                 # http://www.movable-type.co.uk/scripts/latlong.html
 524
 525                 if self.latitude is None:
 526                         return None
 527
 528                 if self.longitude is None:
 529                         return None
 530
 531                 earth = 6371 # km
 532                 delta_lat = math.radians(self.latitude - location.latitude)
 533                 delta_lon = math.radians(self.longitude - location.longitude)
 534
 535                 lat1 = math.radians(self.latitude)
 536                 lat2 = math.radians(location.latitude)
 537
 538                 a = math.sin(delta_lat / 2) ** 2
 539                 a += math.cos(lat1) * math.cos(lat2) * (math.sin(delta_lon / 2) ** 2)
 540
 541                 b1 = math.sqrt(a)
 542                 b2 = math.sqrt(1 - a)
 543
 544                 c = 2 * math.atan2(b1, b2)
 545
 546                 return c * earth
 547
 548         @property
 549         def priority(self):
 550                 return self.data.get("priority", 10)
 551
 552         @property
 553         def development(self):
 554                 return self.data.get("mirrorlist_devel", False)
 555
 556         @property
 557         def mirrorlist(self):
 558                 return self.data.get("mirrorlist", False)
 559
 560         @property
 561         def addresses(self):
 562                 if not hasattr(self, "__addresses"):
 563                         try:
 564                                 addrinfo = socket.getaddrinfo(self.hostname, 0, socket.AF_UNSPEC, socket.SOCK_STREAM)
 565                         except:
 566                                 raise Exception("Could not resolve %s" % self.hostname)
 567
 568                         ret = []
 569                         for family, socktype, proto, canonname, address in addrinfo:
 570                                 if family == socket.AF_INET:
 571                                         address, port = address
 572                                 elif family == socket.AF_INET6:
 573                                         address, port, flowid, scopeid = address
 574                                 ret.append((family, address))
 575
 576                         self.__addresses = ret
 577
 578                 return self.__addresses
 579
 580         @property
 581         def addresses6(self):
 582                 return [address for family, address in self.addresses if family == socket.AF_INET6]
 583
 584         @property
 585         def addresses4(self):
 586                 return [address for family, address in self.addresses if family == socket.AF_INET]