www/webapp/mirrors.py

   1 #!/usr/bin/python
   2
   3 import tornado.httpclient
   4
   5 import random
   6 import simplejson
   7 import threading
   8 import time
   9
  10 from helpers import Item, _stringify, ping
  11
  12 class Mirrors(threading.Thread):
  13         def __init__(self, filename):
  14                 threading.Thread.__init__(self, name="Mirror Monitor")
  15
  16                 self.items = []
  17                 self.load(filename)
  18
  19                 self.__running = True
  20
  21                 self.start()
  22
  23         def load(self, filename):
  24                 f = open(filename)
  25                 data = f.read()
  26                 f.close()
  27
  28                 for item in simplejson.loads(data):
  29                         self.items.append(MirrorItem(**_stringify(item)))
  30
  31         @property
  32         def all(self):
  33                 return sorted(self.items)
  34
  35         @property
  36         def random(self):
  37                 # Doesnt work :(
  38                 #return random.shuffle(self.items)
  39                 ret = []
  40                 items = self.items[:]
  41                 while items:
  42                         rnd = random.randint(0, len(items)-1)
  43                         ret.append(items.pop(rnd))
  44                 return ret
  45
  46         @property
  47         def reachable(self):
  48                 ret = []
  49                 for mirror in self.items:
  50                         if not mirror.reachable:
  51                                 continue
  52                         ret.append(mirror)
  53                 return ret
  54
  55         @property
  56         def unreachable(self):
  57                 ret = []
  58                 for mirror in self.all:
  59                         if mirror in self.reachable:
  60                                 continue
  61                         ret.append(mirror)
  62                 return ret
  63
  64         def pickone(self, reachable=False):
  65                 mirrors = self.items
  66                 if reachable:
  67                         mirrors = self.reachable
  68                 if not mirrors:
  69                         return None
  70                 return random.choice(mirrors)
  71
  72         def with_file(self, path):
  73                 ret = []
  74                 for mirror in self.random:
  75                         if not mirror["serves"]["isos"]:
  76                                 continue
  77                         if path in mirror.files:
  78                                 ret.append(mirror)
  79                 return ret
  80
  81         def shutdown(self):
  82                 self.__running = False
  83
  84         def run(self):
  85                 for mirror in self.random:
  86                         if not self.__running:
  87                                 return
  88                         mirror.update()
  89
  90                 count = 0
  91                 while self.__running:
  92                         if not count:
  93                                 count = 300 # 30 secs
  94                                 mirror = self.pickone()
  95                                 if mirror:
  96                                         mirror.update()
  97
  98                         time.sleep(0.1)
  99                         count -= 1
 100
 101
 102 class MirrorItem(Item):
 103         def __init__(self, *args, **kwargs):
 104                 Item.__init__(self, *args, **kwargs)
 105
 106                 self.filelist = MirrorFilelist(self)
 107                 self.latency = "N/A"
 108
 109         def __cmp__(self, other):
 110                 return cmp(self.name, other.name)
 111
 112         def update(self):
 113                 self.latency = ping(self.hostname) or "N/A"
 114                 if self.filelist.outdated:
 115                         self.filelist.update()
 116
 117         @property
 118         def reachable(self):
 119                 return not self.latency == "N/A"
 120
 121         @property
 122         def url(self):
 123                 ret = "http://" + self.hostname
 124                 if not self.path.startswith("/"):
 125                         ret += "/"
 126                 ret += self.path
 127                 if not ret.endswith("/"):
 128                         ret += "/"
 129                 return ret
 130
 131         @property
 132         def files(self):
 133                 return self.filelist.files
 134
 135         def has_file(self, path):
 136                 return path in self.files
 137
 138
 139 class MirrorFilelist(object):
 140         def __init__(self, mirror):
 141                 self.mirror = mirror
 142
 143                 self.__files = []
 144                 self.__time = 0
 145
 146                 #self.update(now=True)
 147
 148         def update(self, now=False):
 149                 args = {}
 150
 151                 if now:
 152                         while not self.mirror.reachable:
 153                                 time.sleep(10)
 154
 155                 http = tornado.httpclient.HTTPClient()
 156
 157                 if not now:
 158                         http = tornado.httpclient.AsyncHTTPClient()
 159                         args["callback"] = self.on_response
 160
 161                 try:
 162                         reponse = http.fetch(self.mirror.url + ".filelist", **args)
 163                 except tornado.httpclient.HTTPError:
 164                         self.__time = time.time()
 165                         return
 166
 167                 if now:
 168                         self.on_response(reponse)
 169
 170         def on_response(self, response):
 171                 self.__files = []
 172                 self.__time = time.time()
 173
 174                 if not response.code == 200:
 175                         return
 176
 177                 # If invalid html content...
 178                 if response.body.startswith("<!"):
 179                         return
 180
 181                 for line in response.body.split("\n"):
 182                         if not line:
 183                                 continue
 184                         self.__files.append(line)
 185
 186         @property
 187         def outdated(self):
 188                 return (time.time() - self.__time) > 60*60
 189
 190         @property
 191         def files(self):
 192                 #if self.outdated:
 193                 #       self.update()
 194                 return self.__files
 195
 196
 197 mirrors = Mirrors("mirrors.json")