src/buildservice/mirrors.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import logging
   5 import math
   6 import socket
   7 import time
   8 import tornado.httpclient
   9 import urlparse
  10
  11 from . import base
  12 from . import logs
  13
  14 log = logging.getLogger("mirrors")
  15 log.propagate = 1
  16
  17 from .decorators import lazy_property
  18
  19 class Mirrors(base.Object):
  20         def __iter__(self):
  21                 res = self.db.query("SELECT * FROM mirrors \
  22                         WHERE deleted IS FALSE ORDER BY hostname")
  23
  24                 mirrors = []
  25                 for row in res:
  26                         mirror = Mirror(self.backend, row.id, data=row)
  27                         mirrors.append(mirror)
  28
  29                 return iter(mirrors)
  30
  31         def _get_mirror(self, query, *args):
  32                 res = self.db.get(query, *args)
  33
  34                 if res:
  35                         return Mirror(self.backend, res.id, data=res)
  36
  37         def create(self, hostname, path="", owner=None, contact=None, user=None):
  38                 mirror = self._get_mirror("INSERT INTO mirrors(hostname, path, owner, contact) \
  39                         VALUES(%s, %s, %s, %s) RETURNING *", hostname, path, owner, contact)
  40
  41                 # Log creation
  42                 mirror.log("created", user=user)
  43
  44                 return mirror
  45
  46         def get_random(self, limit=None):
  47                 query = "SELECT id FROM mirrors WHERE status = 'enabled' ORDER BY RAND()"
  48                 args  = []
  49
  50                 if limit:
  51                         query += " LIMIT %s"
  52                         args.append(limit)
  53
  54                 mirrors = []
  55                 for mirror in self.db.query(query, *args):
  56                         mirror = Mirror(self.pakfire, mirror.id)
  57                         mirrors.append(mirror)
  58
  59                 return mirrors
  60
  61         def get_by_id(self, id):
  62                 return self._get_mirror("SELECT * FROM mirrors WHERE id = %s", id)
  63
  64         def get_by_hostname(self, hostname):
  65                 return self._get_mirror("SELECT * FROM mirrors \
  66                         WHERE hostname = %s AND deleted IS FALSE", hostname)
  67
  68         def get_for_location(self, address):
  69                 country_code = self.backend.geoip.guess_from_address(address)
  70
  71                 # Cannot return any good mirrors if location is unknown
  72                 if not country_code:
  73                         return []
  74
  75                 mirrors = []
  76
  77                 # Walk through all mirrors
  78                 for mirror in self.get_all():
  79                         if not mirror.enabled:
  80                                 continue
  81
  82                         if mirror.country_code == country_code:
  83                                 mirrors.append(mirror)
  84
  85                         # XXX needs to search for nearby countries
  86
  87                 return mirrors
  88
  89         def get_history(self, limit=None, offset=None, mirror=None, user=None):
  90                 query = "SELECT * FROM mirrors_history"
  91                 args  = []
  92
  93                 conditions = []
  94
  95                 if mirror:
  96                         conditions.append("mirror_id = %s")
  97                         args.append(mirror.id)
  98
  99                 if user:
 100                         conditions.append("user_id = %s")
 101                         args.append(user.id)
 102
 103                 if conditions:
 104                         query += " WHERE %s" % " AND ".join(conditions)
 105
 106                 query += " ORDER BY time DESC"
 107
 108                 if limit:
 109                         if offset:
 110                                 query += " LIMIT %s,%s"
 111                                 args  += [offset, limit,]
 112                         else:
 113                                 query += " LIMIT %s"
 114                                 args  += [limit,]
 115
 116                 entries = []
 117                 for entry in self.db.query(query, *args):
 118                         entry = logs.MirrorLogEntry(self.pakfire, entry)
 119                         entries.append(entry)
 120
 121                 return entries
 122
 123         def check(self, **kwargs):
 124                 """
 125                         Runs the mirror check for all mirrors
 126                 """
 127                 for mirror in self:
 128                         with self.db.transaction():
 129                                 mirror.check(**kwargs)
 130
 131
 132 class Mirror(base.DataObject):
 133         table = "mirrors"
 134
 135         def __eq__(self, other):
 136                 if isinstance(other, self.__class__):
 137                         return self.id == other.id
 138
 139         def log(self, action, user=None):
 140                 user_id = None
 141                 if user:
 142                         user_id = user.id
 143
 144                 self.db.execute("INSERT INTO mirrors_history(mirror_id, action, user_id, time) \
 145                         VALUES(%s, %s, %s, NOW())", self.id, action, user_id)
 146
 147         def set_hostname(self, hostname):
 148                 self._set_attribute("hostname", hostname)
 149
 150         hostname = property(lambda self: self.data.hostname, set_hostname)
 151
 152         @property
 153         def path(self):
 154                 return self.data.path
 155
 156         def set_path(self, path):
 157                 self._set_attribute("path", path)
 158
 159         path = property(lambda self: self.data.path, set_path)
 160
 161         @property
 162         def url(self):
 163                 return self.make_url()
 164
 165         def make_url(self, path=""):
 166                 url = "http://%s%s" % (self.hostname, self.path)
 167
 168                 if path.startswith("/"):
 169                         path = path[1:]
 170
 171                 return urlparse.urljoin(url, path)
 172
 173         def set_owner(self, owner):
 174                 self._set_attribute("owner", owner)
 175
 176         owner = property(lambda self: self.data.owner or "", set_owner)
 177
 178         def set_contact(self, contact):
 179                 self._set_attribute("contact", contact)
 180
 181         contact = property(lambda self: self.data.contact or "", set_contact)
 182
 183         def check(self, connect_timeout=10, request_timeout=10):
 184                 log.info("Running mirror check for %s" % self.hostname)
 185
 186                 client = tornado.httpclient.HTTPClient()
 187
 188                 # Get URL for .timestamp
 189                 url = self.make_url(".timestamp")
 190                 log.debug("  Fetching %s..." % url)
 191
 192                 # Record start time
 193                 time_start = time.time()
 194
 195                 http_status = None
 196                 last_sync_at = None
 197                 status = "OK"
 198
 199                 # XXX needs to catch connection resets, DNS errors, etc.
 200
 201                 try:
 202                         response = client.fetch(url,
 203                                 connect_timeout=connect_timeout,
 204                                 request_timeout=request_timeout)
 205
 206                         # We expect the response to be an integer
 207                         # which holds the timestamp of the last sync
 208                         # in seconds since epoch UTC
 209                         try:
 210                                 timestamp = int(response.body)
 211                         except ValueError:
 212                                 raise
 213
 214                         # Convert to datetime
 215                         last_sync_at = datetime.datetime.utcfromtimestamp(timestamp)
 216
 217                         # Must have synced within 24 hours
 218                         now = datetime.datetime.utcnow()
 219                         if now - last_sync_at >= datetime.timedelta(hours=24):
 220                                 status = "OUTOFSYNC"
 221
 222                 except tornado.httpclient.HTTPError as e:
 223                         http_status = e.code
 224                         status = "ERROR"
 225
 226                 finally:
 227                         response_time = time.time() - time_start
 228
 229                 # Log check
 230                 self.db.execute("INSERT INTO mirrors_checks(mirror_id, response_time, \
 231                         http_status, last_sync_at, status) VALUES(%s, %s, %s, %s, %s)",
 232                         self.id, response_time, http_status, last_sync_at, status)
 233
 234         @lazy_property
 235         def last_check(self):
 236                 res = self.db.get("SELECT * FROM mirrors_checks \
 237                         WHERE mirror_id = %s ORDER BY timestamp DESC LIMIT 1", self.id)
 238
 239                 return res
 240
 241         @property
 242         def status(self):
 243                 if self.last_check:
 244                         return self.last_check.status
 245
 246         @property
 247         def average_response_time(self):
 248                 res = self.db.get("SELECT AVG(response_time) AS response_time \
 249                         FROM mirrors_checks WHERE mirror_id = %s \
 250                                 AND timestamp >= NOW() - '24 hours'::interval", self.id)
 251
 252                 return res.response_time
 253
 254         @property
 255         def address(self):
 256                 return socket.gethostbyname(self.hostname)
 257
 258         @lazy_property
 259         def country_code(self):
 260                 return self.backend.geoip.guess_from_address(self.address) or "UNKNOWN"
 261
 262         def get_history(self, *args, **kwargs):
 263                 kwargs["mirror"] = self
 264
 265                 return self.pakfire.mirrors.get_history(*args, **kwargs)