webapp/backend/releases.py

   1 #!/usr/bin/python
   2
   3 import hashlib
   4 import logging
   5 import os
   6 import re
   7 import urllib
   8 import urlparse
   9
  10 import database
  11 import tracker
  12 from misc import Object
  13
  14 class File(Object):
  15         def __init__(self, backend, release, id, data=None):
  16                 Object.__init__(self, backend)
  17
  18                 self.id = id
  19                 self._release = release
  20
  21                 # get all data from database
  22                 self.__data = data
  23
  24         def __cmp__(self, other):
  25                 return cmp(self.prio, other.prio)
  26
  27         @property
  28         def data(self):
  29                 if self.__data is None:
  30                         self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
  31                         assert self.__data
  32
  33                 return self.__data
  34
  35         @property
  36         def release(self):
  37                 if not self._release:
  38                         release_id = self.data.get("releases")
  39                         self._release = Release(release_id)
  40
  41                 return self._release
  42
  43         @property
  44         def type(self):
  45                 filename = self.filename
  46
  47                 if filename.endswith(".iso"):
  48                         return "iso"
  49
  50                 elif filename.endswith(".torrent"):
  51                         return "torrent"
  52
  53                 elif "xen" in filename:
  54                         if "downloader" in filename:
  55                                 return "xen-downloader"
  56
  57                         return "xen"
  58
  59                 elif "sources" in filename:
  60                         return "source"
  61
  62                 elif "usb-fdd" in filename:
  63                         return "usbfdd"
  64
  65                 elif "usb-hdd" in filename:
  66                         return "usbhdd"
  67
  68                 elif "armv5tel" in filename and "scon" in filename:
  69                         return "armv5tel-scon"
  70
  71                 elif "armv5tel" in filename:
  72                         return "armv5tel"
  73
  74                 elif "scon" in filename:
  75                         return "alix"
  76
  77                 elif filename.endswith(".img.gz"):
  78                         return "flash"
  79
  80                 else:
  81                         return "unknown"
  82
  83         @property
  84         def url(self):
  85                 baseurl = self.settings.get("download_url", "http://downloads.ipfire.org")
  86
  87                 return urlparse.urljoin(baseurl, self.filename)
  88
  89         @property
  90         def desc(self):
  91                 _ = lambda x: x
  92
  93                 descriptions = {
  94                         "armv5tel"      : _("Image for the armv5tel architecture"),
  95                         "armv5tel-scon" : _("armv5tel image for boards with serial console"),
  96                         "iso"           : _("Installable CD image"),
  97                         "torrent"       : _("Torrent file"),
  98                         "flash"         : _("Flash image"),
  99                         "alix"          : _("Flash image for devices with serial console"),
 100                         "usbfdd"        : _("USB FDD Image"),
 101                         "usbhdd"        : _("USB HDD Image"),
 102                         "xen"           : _("Pregenerated Xen image"),
 103                         "xen-downloader": _("Xen-Image Generator"),
 104                 }
 105
 106                 try:
 107                         return descriptions[self.type]
 108                 except KeyError:
 109                         return _("Unknown image type")
 110
 111         @property
 112         def prio(self):
 113                 priorities = {
 114                         "iso"           : 10,
 115                         "torrent"       : 20,
 116                         "flash"         : 40,
 117                         "alix"          : 41,
 118                         "usbfdd"        : 31,
 119                         "usbhdd"        : 30,
 120                         "armv5tel"      : 40,
 121                         "armv5tel-scon" : 41,
 122                         "xen"           : 50,
 123                         "xen-downloader": 51,
 124                 }
 125
 126                 try:
 127                         return priorities[self.type]
 128                 except KeyError:
 129                         return 999
 130
 131         @property
 132         def rem(self):
 133                 _ = lambda x: x
 134
 135                 remarks = {
 136                         "armv5tel"      : _("This image runs on many ARM-based boards"),
 137                         "armv5tel-scon" : _("This image runs on ARM boards with a serial console"),
 138                         "iso"           : _("Use this image to burn a CD and install IPFire from it."),
 139                         "torrent"       : _("Download the CD image from the torrent network."),
 140                         "flash"         : _("An image that is meant to run on embedded devices."),
 141                         "alix"          : _("Flash image where a serial console is enabled by default."),
 142                         "usbfdd"        : _("Install IPFire from a floppy-formated USB key."),
 143                         "usbhdd"        : _("If the floppy image doesn't work, use this image instead."),
 144                         "xen"           : _("A ready-to-run image for Xen."),
 145                         "xen-downloader": _("Generator for creating a Xen image."),
 146                 }
 147
 148                 try:
 149                         return remarks[self.type]
 150                 except KeyError:
 151                         return _("Unknown image type")
 152
 153         @property
 154         def sha1(self):
 155                 return self.data.get("sha1")
 156
 157         @property
 158         def filename(self):
 159                 return self.data.get("filename")
 160
 161         @property
 162         def basename(self):
 163                 return os.path.basename(self.filename)
 164
 165         @property
 166         def size(self):
 167                 return self.data.get("filesize")
 168
 169         @property
 170         def arch(self):
 171                 known_arches = ("i586", "arm")
 172
 173                 for arch in known_arches:
 174                         if arch in self.basename:
 175                                 return arch
 176
 177                 return "N/A"
 178
 179         @property
 180         def torrent_hash(self):
 181                 return self.data.get("torrent_hash", None)
 182
 183         @property
 184         def magnet_link(self):
 185                 # Don't return anything if we have no torrent hash.
 186                 if self.torrent_hash is None:
 187                         return
 188
 189                 s = "magnet:?xt=urn:btih:%s" % self.torrent_hash
 190
 191                 #s += "&xl=%d" % self.size
 192                 s += "&dn=%s" % urllib.quote(self.basename)
 193
 194                 # Add our tracker.
 195                 s += "&tr=http://tracker.ipfire.org:6969/announce"
 196
 197                 return s
 198
 199         @property
 200         def seeders(self):
 201                 if not self.torrent_hash:
 202                         return
 203
 204                 return self.backend.tracker.complete(self.torrent_hash)
 205
 206         @property
 207         def peers(self):
 208                 if not self.torrent_hash:
 209                         return
 210
 211                 return self.backend.tracker.incomplete(self.torrent_hash)
 212
 213
 214 class Release(Object):
 215         def __init__(self, backend, id, data=None):
 216                 Object.__init__(self, backend)
 217                 self.id = id
 218
 219                 # get all data from database
 220                 self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
 221                 assert self.__data
 222
 223                 self.__files = []
 224
 225         def __repr__(self):
 226                 return "<%s %s>" % (self.__class__.__name__, self.name)
 227
 228         def __cmp__(self, other):
 229                 return cmp(self.id, other.id)
 230
 231         @property
 232         def files(self):
 233                 if not self.__files:
 234                         files = self.db.query("SELECT * FROM files WHERE releases = %s \
 235                                 AND NOT filename LIKE '%%.torrent'", self.id)
 236
 237                         self.__files = [File(self.backend, self, f.id, f) for f in files]
 238                         self.__files.sort()
 239
 240                 return self.__files
 241
 242         @property
 243         def torrents(self):
 244                 torrents = []
 245
 246                 for file in self.files:
 247                         if not file.torrent_hash:
 248                                 continue
 249
 250                         torrents.append(file)
 251
 252                 return torrents
 253
 254         @property
 255         def name(self):
 256                 return self.__data.name
 257
 258         @property
 259         def sname(self):
 260                 return self.__data.sname
 261
 262         @property
 263         def stable(self):
 264                 return self.__data.stable
 265
 266         @property
 267         def published(self):
 268                 return self.__data.published
 269
 270         date = published
 271
 272         @property
 273         def path(self):
 274                 return self.__data.path
 275
 276         def get_file(self, type):
 277                 for file in self.files:
 278                         if file.type == type:
 279                                 return file
 280
 281         def __file_hash(self, filename):
 282                 sha1 = hashlib.sha1()
 283
 284                 with open(filename) as f:
 285                         buf_size = 1024
 286                         buf = f.read(buf_size)
 287                         while buf:
 288                                 sha1.update(buf)
 289                                 buf = f.read(buf_size)
 290
 291                 return sha1.hexdigest()
 292
 293         def scan_files(self, basepath="/srv/mirror0"):
 294                 if not self.path:
 295                         return
 296
 297                 path = os.path.join(basepath, self.path)
 298                 if not os.path.exists(path):
 299                         return
 300
 301                 files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
 302                 files = [f.filename for f in files]
 303
 304                 # Make files that do not exists not loadable.
 305                 for filename in files:
 306                         _filename = os.path.join(basepath, filename)
 307                         if not os.path.exists(_filename):
 308                                 self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
 309
 310                 for filename in os.listdir(path):
 311                         filename = os.path.join(path, filename)
 312
 313                         if os.path.isdir(filename):
 314                                 continue
 315
 316                         _filename = re.match(".*(releases/.*)", filename).group(1)
 317                         if _filename in files:
 318                                 continue
 319
 320                         if filename.endswith(".md5"):
 321                                 continue
 322
 323                         logging.info("Hashing %s..." % filename)
 324                         filehash = self.__file_hash(filename)
 325                         filesize = os.path.getsize(filename)
 326
 327                         # Check if there is a torrent download available for this file:
 328                         torrent_hash = ""
 329                         torrent_file = "%s.torrent" % filename
 330                         if os.path.exists(torrent_file):
 331                                 torrent_hash = self.torrent_read_hash(torrent_file)
 332
 333                         self.db.execute("INSERT INTO files(releases, filename, filesize, \
 334                                 sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s)",
 335                                 self.id, _filename, filesize, filehash, torrent_hash)
 336
 337                 # Search for all files that miss a torrent hash.
 338                 files = self.db.query("SELECT id, filename FROM files \
 339                         WHERE releases = %s AND torrent_hash IS NULL", self.id)
 340
 341                 for file in files:
 342                         path = os.path.join(basepath, file.filename)
 343
 344                         torrent_file = "%s.torrent" % path
 345                         if os.path.exists(torrent_file):
 346                                 torrent_hash = self.torrent_read_hash(torrent_file)
 347
 348                                 self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
 349                                         torrent_hash, file.id)
 350
 351         def torrent_read_hash(self, filename):
 352                 f = None
 353                 try:
 354                         f = open(filename, "rb")
 355
 356                         metainfo = tracker.bdecode(f.read())
 357                         metainfo = tracker.bencode(metainfo["info"])
 358
 359                         hash = hashlib.sha1()
 360                         hash.update(metainfo)
 361
 362                         return hash.hexdigest()
 363
 364                 finally:
 365                         if f:
 366                                 f.close()
 367
 368         def is_netboot_capable(self):
 369                 return self.path and "ipfire-2.x" in self.path
 370
 371         @property
 372         def netboot_kernel(self):
 373                 return "http://downloads.ipfire.org/%s/images/vmlinuz" % self.path
 374
 375         @property
 376         def netboot_initrd(self):
 377                 return "http://downloads.ipfire.org/%s/images/instroot" % self.path
 378
 379         @property
 380         def netboot_append(self):
 381                 return "ro"
 382
 383
 384 class Releases(Object):
 385         def get_by_id(self, id):
 386                 ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)
 387
 388                 if ret:
 389                         return Release(self.backend, ret.id, data=ret)
 390
 391         def get_by_sname(self, sname):
 392                 ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)
 393
 394                 if ret:
 395                         return Release(self.backend, ret.id, data=ret)
 396
 397         def get_latest(self, stable=True):
 398                 ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
 399                         AND stable = %s ORDER BY published DESC LIMIT 1", stable)
 400
 401                 if ret:
 402                         return Release(self.backend, ret.id, data=ret)
 403
 404         def get_latest_unstable(self):
 405                 ret = self.db.get("SELECT * FROM releases r1 \
 406                         WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
 407                         AND stable = %s AND NOT EXISTS ( \
 408                                 SELECT * FROM releases r2 WHERE r2.stable = %s AND \
 409                                         r2.published IS NOT NULL AND r2.published >= r1.published \
 410                                 ) ORDER BY r1.published DESC LIMIT 1", False, True)
 411
 412                 if ret:
 413                         return Release(self.backend, ret.id, data=ret)
 414
 415         def get_stable(self):
 416                 query = self.db.query("SELECT * FROM releases \
 417                         WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
 418                         ORDER BY published DESC")
 419
 420                 releases = []
 421                 for row in query:
 422                         release = Release(self.backend, row.id, data=row)
 423                         releases.append(release)
 424
 425                 return releases
 426
 427         def get_unstable(self):
 428                 query = self.db.query("SELECT * FROM releases \
 429                         WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
 430                         ORDER BY published DESC")
 431
 432                 releases = []
 433                 for row in query:
 434                         release = Release(self.backend, row.id, data=row)
 435                         releases.append(release)
 436
 437                 return releases
 438
 439         def get_all(self):
 440                 query = self.db.query("SELECT * FROM releases \
 441                         WHERE published IS NOT NULL AND published <= NOW() \
 442                         ORDER BY published DESC")
 443
 444                 releases = []
 445                 for row in query:
 446                         release = Release(self.backend, row.id, data=row)
 447                         releases.append(release)
 448
 449                 return releases
 450
 451         def _get_all(self):
 452                 query = self.db.query("SELECT * FROM releases ORDER BY published DESC")
 453
 454                 releases = []
 455                 for row in query:
 456                         release = Release(self.backend, row.id, data=row)
 457                         releases.append(release)
 458
 459                 return releases
 460
 461         def get_file_for_torrent_hash(self, torrent_hash):
 462                 file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
 463                         torrent_hash)
 464
 465                 if not file:
 466                         return
 467
 468                 release = Release(self.backend, file.releases)
 469                 file = File(self.backend, release, file.id)
 470
 471                 return file