src/buildservice/mirrors.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import logging
   5 import math
   6 import socket
   7 import time
   8 import tornado.httpclient
   9 import urlparse
  10
  11 from . import base
  12 from . import logs
  13
  14 log = logging.getLogger("mirrors")
  15 log.propagate = 1
  16
  17 from .decorators import lazy_property
  18
  19 class Mirrors(base.Object):
  20         def _get_mirror(self, query, *args):
  21                 res = self.db.get(query, *args)
  22
  23                 if res:
  24                         return Mirror(self.backend, res.id, data=res)
  25
  26         def _get_mirrors(self, query, *args):
  27                 res = self.db.query(query, *args)
  28
  29                 for row in res:
  30                         yield Mirror(self.backend, row.id, data=row)
  31
  32         def __iter__(self):
  33                 mirrors = self._get_mirrors("SELECT * FROM mirrors \
  34                         WHERE deleted IS FALSE ORDER BY hostname")
  35
  36                 return iter(mirrors)
  37
  38         def create(self, hostname, path="", owner=None, contact=None, user=None):
  39                 mirror = self._get_mirror("INSERT INTO mirrors(hostname, path, owner, contact) \
  40                         VALUES(%s, %s, %s, %s) RETURNING *", hostname, path, owner, contact)
  41
  42                 # Log creation
  43                 mirror.log("created", user=user)
  44
  45                 return mirror
  46
  47         def get_by_id(self, id):
  48                 return self._get_mirror("SELECT * FROM mirrors WHERE id = %s", id)
  49
  50         def get_by_hostname(self, hostname):
  51                 return self._get_mirror("SELECT * FROM mirrors \
  52                         WHERE hostname = %s AND deleted IS FALSE", hostname)
  53
  54         def get_for_location(self, address):
  55                 country_code = self.backend.geoip.guess_from_address(address)
  56
  57                 # Cannot return any good mirrors if location is unknown
  58                 if not country_code:
  59                         return []
  60
  61                 mirrors = []
  62
  63                 # Walk through all mirrors
  64                 for mirror in self:
  65                         if mirror.country_code == country_code:
  66                                 mirrors.append(mirror)
  67
  68                         # XXX needs to search for nearby countries
  69
  70                 return mirrors
  71
  72         def get_history(self, limit=None, offset=None, mirror=None, user=None):
  73                 query = "SELECT * FROM mirrors_history"
  74                 args  = []
  75
  76                 conditions = []
  77
  78                 if mirror:
  79                         conditions.append("mirror_id = %s")
  80                         args.append(mirror.id)
  81
  82                 if user:
  83                         conditions.append("user_id = %s")
  84                         args.append(user.id)
  85
  86                 if conditions:
  87                         query += " WHERE %s" % " AND ".join(conditions)
  88
  89                 query += " ORDER BY time DESC"
  90
  91                 if limit:
  92                         if offset:
  93                                 query += " LIMIT %s,%s"
  94                                 args  += [offset, limit,]
  95                         else:
  96                                 query += " LIMIT %s"
  97                                 args  += [limit,]
  98
  99                 entries = []
 100                 for entry in self.db.query(query, *args):
 101                         entry = logs.MirrorLogEntry(self.pakfire, entry)
 102                         entries.append(entry)
 103
 104                 return entries
 105
 106         def check(self, **kwargs):
 107                 """
 108                         Runs the mirror check for all mirrors
 109                 """
 110                 for mirror in self:
 111                         with self.db.transaction():
 112                                 mirror.check(**kwargs)
 113
 114
 115 class Mirror(base.DataObject):
 116         table = "mirrors"
 117
 118         def __eq__(self, other):
 119                 if isinstance(other, self.__class__):
 120                         return self.id == other.id
 121
 122         def log(self, action, user=None):
 123                 user_id = None
 124                 if user:
 125                         user_id = user.id
 126
 127                 self.db.execute("INSERT INTO mirrors_history(mirror_id, action, user_id, time) \
 128                         VALUES(%s, %s, %s, NOW())", self.id, action, user_id)
 129
 130         def set_hostname(self, hostname):
 131                 self._set_attribute("hostname", hostname)
 132
 133         hostname = property(lambda self: self.data.hostname, set_hostname)
 134
 135         def set_deleted(self, deleted):
 136                 self._set_attribute("deleted", deleted)
 137
 138         deleted = property(lambda s: s.data.deleted, set_deleted)
 139
 140         @property
 141         def path(self):
 142                 return self.data.path
 143
 144         def set_path(self, path):
 145                 self._set_attribute("path", path)
 146
 147         path = property(lambda self: self.data.path, set_path)
 148
 149         @property
 150         def url(self):
 151                 return self.make_url()
 152
 153         def make_url(self, path=""):
 154                 url = "%s://%s%s" % (
 155                         "https" if self.supports_https else "http",
 156                         self.hostname,
 157                         self.path
 158                 )
 159
 160                 if path.startswith("/"):
 161                         path = path[1:]
 162
 163                 return urlparse.urljoin(url, path)
 164
 165         def set_supports_https(self, supports_https):
 166                 self._set_attribute("supports_https", supports_https)
 167
 168         supports_https = property(lambda s: s.data.supports_https, set_supports_https)
 169
 170         def set_owner(self, owner):
 171                 self._set_attribute("owner", owner)
 172
 173         owner = property(lambda self: self.data.owner or "", set_owner)
 174
 175         def set_contact(self, contact):
 176                 self._set_attribute("contact", contact)
 177
 178         contact = property(lambda self: self.data.contact or "", set_contact)
 179
 180         def check(self, connect_timeout=10, request_timeout=10):
 181                 log.info("Running mirror check for %s" % self.hostname)
 182
 183                 client = tornado.httpclient.HTTPClient()
 184
 185                 # Get URL for .timestamp
 186                 url = self.make_url(".timestamp")
 187                 log.debug("  Fetching %s..." % url)
 188
 189                 # Record start time
 190                 time_start = time.time()
 191
 192                 http_status = None
 193                 last_sync_at = None
 194                 status = "OK"
 195
 196                 # XXX needs to catch connection resets, DNS errors, etc.
 197
 198                 try:
 199                         response = client.fetch(url,
 200                                 connect_timeout=connect_timeout,
 201                                 request_timeout=request_timeout)
 202
 203                         # We expect the response to be an integer
 204                         # which holds the timestamp of the last sync
 205                         # in seconds since epoch UTC
 206                         try:
 207                                 timestamp = int(response.body)
 208                         except ValueError:
 209                                 raise
 210
 211                         # Convert to datetime
 212                         last_sync_at = datetime.datetime.utcfromtimestamp(timestamp)
 213
 214                         # Must have synced within 24 hours
 215                         now = datetime.datetime.utcnow()
 216                         if now - last_sync_at >= datetime.timedelta(hours=24):
 217                                 status = "OUTOFSYNC"
 218
 219                 except tornado.httpclient.HTTPError as e:
 220                         http_status = e.code
 221                         status = "ERROR"
 222
 223                 finally:
 224                         response_time = time.time() - time_start
 225
 226                 # Log check
 227                 self.db.execute("INSERT INTO mirrors_checks(mirror_id, response_time, \
 228                         http_status, last_sync_at, status) VALUES(%s, %s, %s, %s, %s)",
 229                         self.id, response_time, http_status, last_sync_at, status)
 230
 231         @lazy_property
 232         def last_check(self):
 233                 res = self.db.get("SELECT * FROM mirrors_checks \
 234                         WHERE mirror_id = %s ORDER BY timestamp DESC LIMIT 1", self.id)
 235
 236                 return res
 237
 238         @property
 239         def status(self):
 240                 if self.last_check:
 241                         return self.last_check.status
 242
 243         @property
 244         def average_response_time(self):
 245                 res = self.db.get("SELECT AVG(response_time) AS response_time \
 246                         FROM mirrors_checks WHERE mirror_id = %s \
 247                                 AND timestamp >= NOW() - '24 hours'::interval", self.id)
 248
 249                 return res.response_time
 250
 251         @property
 252         def address(self):
 253                 return socket.gethostbyname(self.hostname)
 254
 255         @lazy_property
 256         def country_code(self):
 257                 return self.backend.geoip.guess_from_address(self.address) or "UNKNOWN"
 258
 259         def get_history(self, *args, **kwargs):
 260                 kwargs["mirror"] = self
 261
 262                 return self.pakfire.mirrors.get_history(*args, **kwargs)