www/webapp/backend/releases.py

   1 #!/usr/bin/python
   2
   3 import hashlib
   4 import logging
   5 import os
   6 import re
   7 import urllib
   8 import urlparse
   9
  10 import tracker
  11
  12 from databases import Databases
  13 from misc import Singleton
  14 from settings import Settings
  15
  16 class File(object):
  17         def __init__(self, release, id):
  18                 self.id = id
  19                 self._release = release
  20
  21                 # get all data from database
  22                 self.__data = None
  23
  24         @property
  25         def db(self):
  26                 return Databases().webapp
  27
  28         @property
  29         def tracker(self):
  30                 return self.release.tracker
  31
  32         @property
  33         def data(self):
  34                 if self.__data is None:
  35                         self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
  36                         assert self.__data
  37
  38                 return self.__data
  39
  40         @property
  41         def release(self):
  42                 if not self._release:
  43                         release_id = self.data.get("releases")
  44                         self._release = Release(release_id)
  45
  46                 return self._release
  47
  48         @property
  49         def type(self):
  50                 filename = self.filename
  51
  52                 if filename.endswith(".iso"):
  53                         return "iso"
  54
  55                 elif filename.endswith(".torrent"):
  56                         return "torrent"
  57
  58                 elif "xen" in filename:
  59                         return "xen"
  60
  61                 elif "sources" in filename:
  62                         return "source"
  63
  64                 elif "usb-fdd" in filename:
  65                         return "usbfdd"
  66
  67                 elif "usb-hdd" in filename:
  68                         return "usbhdd"
  69
  70                 elif "armv5tel" in filename:
  71                         return "armv5tel"
  72
  73                 elif "scon" in filename:
  74                         return "alix"
  75
  76                 elif filename.endswith(".img.gz"):
  77                         return "flash"
  78
  79                 else:
  80                         return "unknown"
  81
  82         @property
  83         def url(self):
  84                 baseurl = Settings().get("download_url")
  85
  86                 return urlparse.urljoin(baseurl, self.filename)
  87
  88         @property
  89         def desc(self):
  90                 _ = lambda x: x
  91
  92                 descriptions = {
  93                         "armv5tel"      : _("Image for the armv5tel architecture"),
  94                         "iso"           : _("Installable CD image"),
  95                         "torrent"       : _("Torrent file"),
  96                         "flash"         : _("Flash image"),
  97                         "alix"          : _("Alix image"),
  98                         "usbfdd"        : _("USB FDD Image"),
  99                         "usbhdd"        : _("USB HDD Image"),
 100                         "xen"           : _("Pregenerated Xen image"),
 101                 }
 102
 103                 try:
 104                         return descriptions[self.type]
 105                 except KeyError:
 106                         return _("Unknown image type")
 107
 108         @property
 109         def prio(self):
 110                 priorities = {
 111                         "iso"           : 10,
 112                         "torrent"       : 20,
 113                         "flash"         : 40,
 114                         "alix"          : 41,
 115                         "usbfdd"        : 31,
 116                         "usbhdd"        : 30,
 117                         "armv5tel"  : 40,
 118                         "xen"           : 50,
 119                 }
 120
 121                 try:
 122                         return priorities[self.type]
 123                 except KeyError:
 124                         return 999
 125
 126         @property
 127         def rem(self):
 128                 _ = lambda x: x
 129
 130                 remarks = {
 131                         "armv5tel"      : _("This image runs on many ARM-based boards"),
 132                         "iso"           : _("Use this image to burn a CD and install IPFire from it."),
 133                         "torrent"       : _("Download the CD image from the torrent network."),
 134                         "flash"         : _("An image that is meant to run on embedded devices."),
 135                         "alix"          : _("Flash image where a serial console is enabled by default."),
 136                         "usbfdd"        : _("Install IPFire from a floppy-formated USB key."),
 137                         "usbhdd"        : _("If the floppy image doesn't work, use this image instead."),
 138                         "xen"           : _("A ready-to-run image for Xen."),
 139                 }
 140
 141                 try:
 142                         return remarks[self.type]
 143                 except KeyError:
 144                         return _("Unknown image type")
 145
 146         @property
 147         def sha1(self):
 148                 return self.data.get("sha1")
 149
 150         @property
 151         def filename(self):
 152                 return self.data.get("filename")
 153
 154         @property
 155         def basename(self):
 156                 return os.path.basename(self.filename)
 157
 158         @property
 159         def size(self):
 160                 return self.data.get("filesize")
 161
 162         @property
 163         def arch(self):
 164                 known_arches = ("i586", "arm")
 165
 166                 for arch in known_arches:
 167                         if arch in self.basename:
 168                                 return arch
 169
 170                 return "N/A"
 171
 172         @property
 173         def torrent_hash(self):
 174                 return self.data.get("torrent_hash", None)
 175
 176         @property
 177         def magnet_link(self):
 178                 # Don't return anything if we have no torrent hash.
 179                 if self.torrent_hash is None:
 180                         return
 181
 182                 s = "magnet:?xt=urn:btih:%s" % self.torrent_hash
 183
 184                 #s += "&xl=%d" % self.size
 185                 s += "&dn=%s" % urllib.quote(self.basename)
 186
 187                 # Add our tracker.
 188                 s += "&tr=http://tracker.ipfire.org:6969/announce"
 189
 190                 return s
 191
 192         @property
 193         def seeders(self):
 194                 if not self.torrent_hash:
 195                         return
 196
 197                 return self.tracker.get_seeds(self.torrent_hash)
 198
 199         @property
 200         def peers(self):
 201                 if not self.torrent_hash:
 202                         return
 203
 204                 return self.tracker.get_peers(self.torrent_hash)
 205
 206         @property
 207         def completed(self):
 208                 if not self.torrent_hash:
 209                         return
 210
 211                 return self.tracker.complete(self.torrent_hash)
 212
 213
 214 class Release(object):
 215         @property
 216         def db(self):
 217                 return Releases().db
 218
 219         @property
 220         def tracker(self):
 221                 return tracker.Tracker()
 222
 223         def __init__(self, id):
 224                 self.id = id
 225
 226                 # get all data from database
 227                 self.__data = \
 228                         self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
 229                 assert self.__data
 230
 231                 self.__files = []
 232
 233         def __repr__(self):
 234                 return "<%s %s>" % (self.__class__.__name__, self.name)
 235
 236         @property
 237         def files(self):
 238                 if not self.__files:
 239                         files = self.db.query("SELECT id, filename FROM files WHERE releases = %s \
 240                                         AND loadable = 'Y' AND NOT filename LIKE '%%.torrent'", self.id)
 241
 242                         self.__files = [File(self, f.id) for f in files]
 243                         self.__files.sort(lambda a, b: cmp(a.prio, b.prio))
 244
 245                 return self.__files
 246
 247         @property
 248         def torrents(self):
 249                 torrents = []
 250
 251                 for file in self.files:
 252                         if not file.torrent_hash:
 253                                 continue
 254
 255                         torrents.append(file)
 256
 257                 return torrents
 258
 259         @property
 260         def name(self):
 261                 return self.__data.get("name")
 262
 263         @property
 264         def stable(self):
 265                 return self.__data.get("stable") == "Y"
 266
 267         @property
 268         def published(self):
 269                 return self.__data.get("published") == "Y"
 270
 271         @property
 272         def date(self):
 273                 return self.__data.get("date")
 274
 275         @property
 276         def path(self):
 277                 return self.__data.get("path")
 278
 279         def get_file(self, type):
 280                 for file in self.files:
 281                         if file.type == type:
 282                                 return file
 283
 284         def __file_hash(self, filename):
 285                 sha1 = hashlib.sha1()
 286
 287                 with open(filename) as f:
 288                         buf_size = 1024
 289                         buf = f.read(buf_size)
 290                         while buf:
 291                                 sha1.update(buf)
 292                                 buf = f.read(buf_size)
 293
 294                 return sha1.hexdigest()
 295
 296         def scan_files(self, basepath="/srv/mirror0"):
 297                 if not self.path:
 298                         return
 299
 300                 path = os.path.join(basepath, self.path)
 301                 if not os.path.exists(path):
 302                         return
 303
 304                 files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
 305                 files = [f.filename for f in files]
 306
 307                 # Make files that do not exists not loadable.
 308                 for filename in files:
 309                         _filename = os.path.join(basepath, filename)
 310                         if not os.path.exists(_filename):
 311                                 self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
 312
 313                 for filename in os.listdir(path):
 314                         filename = os.path.join(path, filename)
 315
 316                         if os.path.isdir(filename):
 317                                 continue
 318
 319                         _filename = re.match(".*(releases/.*)", filename).group(1)
 320                         if _filename in files:
 321                                 continue
 322
 323                         if filename.endswith(".md5"):
 324                                 continue
 325
 326                         filehash = self.__file_hash(filename)
 327                         filesize = os.path.getsize(filename)
 328
 329                         # Check if there is a torrent download available for this file:
 330                         torrent_hash = ""
 331                         torrent_file = "%s.torrent" % filename
 332                         if os.path.exists(torrent_file):
 333                                 torrent_hash = self.torrent_read_hash(torrent_file)
 334
 335                         self.db.execute("INSERT INTO files(releases, filename, filesize, \
 336                                 sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s)",
 337                                 self.id, _filename, filesize, filehash, torrent_hash)
 338
 339                 # Search for all files that miss a torrent hash.
 340                 files = self.db.query("SELECT id, filename FROM files \
 341                         WHERE releases = %s AND torrent_hash IS NULL", self.id)
 342
 343                 for file in files:
 344                         path = os.path.join(basepath, file.filename)
 345
 346                         torrent_file = "%s.torrent" % path
 347                         if os.path.exists(torrent_file):
 348                                 torrent_hash = self.torrent_read_hash(torrent_file)
 349
 350                                 self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
 351                                         torrent_hash, file.id)
 352
 353         def torrent_read_hash(self, filename):
 354                 f = None
 355                 try:
 356                         f = open(filename, "rb")
 357
 358                         metainfo = tracker.bdecode(f.read())
 359                         metainfo = tracker.bencode(metainfo["info"])
 360
 361                         hash = hashlib.sha1()
 362                         hash.update(metainfo)
 363
 364                         return hash.hexdigest()
 365
 366                 finally:
 367                         if f:
 368                                 f.close()
 369
 370
 371 class Releases(object):
 372         __metaclass__ = Singleton
 373
 374         @property
 375         def db(self):
 376                 return Databases().webapp
 377
 378         def list(self):
 379                 return [Release(r.id) for r in self.db.query("SELECT id FROM releases ORDER BY date DESC")]
 380
 381         def get_by_id(self, id):
 382                 id = int(id)
 383                 if id in [r.id for r in self.db.query("SELECT id FROM releases")]:
 384                         return Release(id)
 385
 386         def get_latest(self, stable=1):
 387                 query = "SELECT id FROM releases WHERE published='Y' AND"
 388                 if stable:
 389                         query += " stable='Y'"
 390                 else:
 391                         query += " stable='N'"
 392
 393                 query += " ORDER BY date DESC LIMIT 1"
 394
 395                 release = self.db.get(query)
 396                 if release:
 397                         return Release(release.id)
 398
 399         def get_stable(self):
 400                 releases = self.db.query("""SELECT id FROM releases
 401                         WHERE published='Y' AND stable='Y'
 402                         ORDER BY date DESC""")
 403
 404                 return [Release(r.id) for r in releases]
 405
 406         def get_unstable(self):
 407                 releases = self.db.query("""SELECT id FROM releases
 408                         WHERE published='Y' AND stable='N'
 409                         ORDER BY date DESC""")
 410
 411                 return [Release(r.id) for r in releases]
 412
 413         def get_all(self):
 414                 releases = self.db.query("""SELECT id FROM releases
 415                         WHERE published='Y' ORDER BY date DESC""")
 416
 417                 return [Release(r.id) for r in releases]
 418
 419         def get_file_for_torrent_hash(self, torrent_hash):
 420                 file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
 421                         torrent_hash)
 422
 423                 if not file:
 424                         return
 425
 426                 release = Release(file.releases)
 427                 file = File(release, file.id)
 428
 429                 return file
 430
 431
 432 if __name__ == "__main__":
 433         r = Releases()
 434
 435         for release in r.get_all():
 436                 print release.name
 437
 438         print r.get_latest()