src/backend/releases.py

   1 #!/usr/bin/python
   2
   3 import hashlib
   4 import logging
   5 import os
   6 import re
   7 import urllib.request, urllib.parse, urllib.error
   8 import urllib.parse
   9
  10 from . import database
  11 from . import tracker
  12 from .misc import Object
  13
  14 TRACKERS = (
  15     "http://ipv4.tracker.ipfire.org:6969/announce",
  16     "udp://ipv4.tracker.ipfire.org:6969",
  17     "http://ipv6.tracker.ipfire.org:6969/announce",
  18     "udp://ipv6.tracker.ipfire.org:6969",
  19 )
  20
  21 class File(Object):
  22         def __init__(self, backend, release, id, data=None):
  23                 Object.__init__(self, backend)
  24
  25                 self.id = id
  26                 self._release = release
  27
  28                 # get all data from database
  29                 self.__data = data
  30
  31         def __eq__(self, other):
  32                 if isinstance(other, self.__class__):
  33                         return self.id == otherid
  34
  35         def __lt__(self, other):
  36                 if isinstance(other, self.__class__):
  37                         return self.prio < other.prio
  38
  39         @property
  40         def data(self):
  41                 if self.__data is None:
  42                         self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
  43                         assert self.__data
  44
  45                 return self.__data
  46
  47         @property
  48         def release(self):
  49                 if not self._release:
  50                         release_id = self.data.get("releases")
  51                         self._release = Release(release_id)
  52
  53                 return self._release
  54
  55         @property
  56         def type(self):
  57                 filename = self.filename
  58
  59                 if filename.endswith(".iso"):
  60                         return "iso"
  61
  62                 elif filename.endswith(".torrent"):
  63                         return "torrent"
  64
  65                 elif "xen" in filename:
  66                         if "downloader" in filename:
  67                                 return "xen-downloader"
  68
  69                         return "xen"
  70
  71                 elif "sources" in filename:
  72                         return "source"
  73
  74                 elif "usb-fdd" in filename:
  75                         return "usbfdd"
  76
  77                 elif "usb-hdd" in filename:
  78                         return "usbhdd"
  79
  80                 elif "armv5tel" in filename and "scon" in filename:
  81                         return "armv5tel-scon"
  82
  83                 elif "armv5tel" in filename:
  84                         return "armv5tel"
  85
  86                 elif "scon" in filename:
  87                         return "alix"
  88
  89                 elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
  90                         return "flash"
  91
  92                 else:
  93                         return "unknown"
  94
  95         @property
  96         def url(self):
  97                 return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)
  98
  99         @property
 100         def desc(self):
 101                 _ = lambda x: x
 102
 103                 descriptions = {
 104                         "armv5tel"      : _("Flash Image"),
 105                         "armv5tel-scon" : _("Flash Image with serial console"),
 106                         "iso"           : _("CD Image"),
 107                         "torrent"       : _("Torrent File"),
 108                         "flash"         : _("Flash Image"),
 109                         "alix"          : _("Flash Image with serial console"),
 110                         "usbfdd"        : _("USB FDD Image"),
 111                         "usbhdd"        : _("USB HDD Image"),
 112                         "xen"           : _("Pre-generated Xen Image"),
 113                         "xen-downloader": _("Xen-Image Generator"),
 114                 }
 115
 116                 try:
 117                         return descriptions[self.type]
 118                 except KeyError:
 119                         return _("Unknown image type")
 120
 121         @property
 122         def prio(self):
 123                 priorities = {
 124                         "iso"           : 10,
 125                         "torrent"       : 20,
 126                         "flash"         : 40,
 127                         "alix"          : 41,
 128                         "usbfdd"        : 31,
 129                         "usbhdd"        : 30,
 130                         "armv5tel"      : 40,
 131                         "armv5tel-scon" : 41,
 132                         "xen"           : 50,
 133                         "xen-downloader": 51,
 134                 }
 135
 136                 try:
 137                         return priorities[self.type]
 138                 except KeyError:
 139                         return 999
 140
 141         @property
 142         def rem(self):
 143                 _ = lambda x: x
 144
 145                 remarks = {
 146                         "armv5tel"      : _("This image runs on many ARM-based boards"),
 147                         "armv5tel-scon" : _("This image runs on ARM boards with a serial console"),
 148                         "iso"           : _("Use this image to burn a CD and install IPFire from it."),
 149                         "torrent"       : _("Download the CD image from the torrent network."),
 150                         "flash"         : _("An image that is meant to run on embedded devices."),
 151                         "alix"          : _("Flash image where a serial console is enabled by default."),
 152                         "usbfdd"        : _("Install IPFire from a floppy-formated USB key."),
 153                         "usbhdd"        : _("If the floppy image doesn't work, use this image instead."),
 154                         "xen"           : _("A ready-to-run image for Xen."),
 155                         "xen-downloader": _("Generator for creating a Xen image."),
 156                 }
 157
 158                 try:
 159                         return remarks[self.type]
 160                 except KeyError:
 161                         return _("Unknown image type")
 162
 163         @property
 164         def sha1(self):
 165                 return self.data.get("sha1")
 166
 167         @property
 168         def filename(self):
 169                 return self.data.get("filename")
 170
 171         @property
 172         def basename(self):
 173                 return os.path.basename(self.filename)
 174
 175         @property
 176         def size(self):
 177                 return self.data.get("filesize")
 178
 179         @property
 180         def arch(self):
 181                 known_arches = ("x86_64", "i586", "arm")
 182
 183                 for arch in known_arches:
 184                         if arch in self.basename:
 185                                 return arch
 186
 187                 return "N/A"
 188
 189         @property
 190         def torrent_hash(self):
 191                 return self.data.get("torrent_hash", None)
 192
 193         @property
 194         def torrent_url(self):
 195                 if self.torrent_hash:
 196                         return "%s.torrent" % self.url
 197
 198         @property
 199         def magnet_link(self):
 200                 # Don't return anything if we have no torrent hash.
 201                 if self.torrent_hash is None:
 202                         return
 203
 204                 s = "magnet:?xt=urn:btih:%s" % self.torrent_hash
 205
 206                 #s += "&xl=%d" % self.size
 207                 s += "&dn=%s" % urllib.parse.quote(self.basename)
 208
 209                 # Add our tracker.
 210                 for tracker in TRACKERS:
 211                         s += "&tr=%s" % tracker
 212
 213                 # Add web download URL
 214                 s += "&as=%s" % urllib.parse.quote(self.url)
 215
 216                 return s
 217
 218
 219 class Release(Object):
 220         def __init__(self, backend, id, data=None):
 221                 Object.__init__(self, backend)
 222                 self.id = id
 223
 224                 # get all data from database
 225                 self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
 226                 assert self.__data
 227
 228                 self.__files = []
 229
 230         def __str__(self):
 231                 return self.name
 232
 233         def __repr__(self):
 234                 return "<%s %s>" % (self.__class__.__name__, self.name)
 235
 236         def __cmp__(self, other):
 237                 return cmp(self.id, other.id)
 238
 239         @property
 240         def arches(self):
 241                 for arch in ("x86_64", "aarch64", "i586", "arm"):
 242                         if arch in (f.arch for f in self.files):
 243                                 yield arch
 244
 245         @property
 246         def files(self):
 247                 if not self.__files:
 248                         files = self.db.query("SELECT * FROM files WHERE releases = %s \
 249                                 AND NOT filename LIKE '%%.torrent'", self.id)
 250
 251                         self.__files = [File(self.backend, self, f.id, f) for f in files]
 252                         self.__files.sort()
 253
 254                 return self.__files
 255
 256         def get_files_by_arch(self, arch):
 257                 for f in self.files:
 258                         if f.arch == arch:
 259                                 yield f
 260
 261         @property
 262         def torrents(self):
 263                 torrents = []
 264
 265                 for file in self.files:
 266                         if not file.torrent_hash:
 267                                 continue
 268
 269                         torrents.append(file)
 270
 271                 return torrents
 272
 273         @property
 274         def name(self):
 275                 return self.__data.name
 276
 277         @property
 278         def slug(self):
 279                 return self.__data.sname
 280
 281         # XXX compat
 282         sname = slug
 283
 284         # XXX cache this
 285         @property
 286         def blog(self):
 287                 if self.__data.blog_id:
 288                         return self.backend.blog.get_by_id(self.__data.blog_id)
 289
 290         @property
 291         def fireinfo_id(self):
 292                 name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")
 293
 294                 res = self.db.get("SELECT id FROM fireinfo_releases \
 295                         WHERE name = %s", name)
 296
 297                 if res:
 298                         return res.id
 299
 300         @property
 301         def stable(self):
 302                 return self.__data.stable
 303
 304         @property
 305         def published(self):
 306                 return self.__data.published
 307
 308         date = published
 309
 310         @property
 311         def path(self):
 312                 return self.__data.path
 313
 314         def get_file(self, type):
 315                 for file in self.files:
 316                         if file.type == type:
 317                                 return file
 318
 319         def __file_hash(self, filename):
 320                 sha1 = hashlib.sha1()
 321
 322                 with open(filename) as f:
 323                         buf_size = 1024
 324                         buf = f.read(buf_size)
 325                         while buf:
 326                                 sha1.update(buf)
 327                                 buf = f.read(buf_size)
 328
 329                 return sha1.hexdigest()
 330
 331         def scan_files(self, basepath="/srv/mirror0"):
 332                 if not self.path:
 333                         return
 334
 335                 path = os.path.join(basepath, self.path)
 336                 if not os.path.exists(path):
 337                         return
 338
 339                 files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
 340                 files = [f.filename for f in files]
 341
 342                 # Make files that do not exists not loadable.
 343                 for filename in files:
 344                         _filename = os.path.join(basepath, filename)
 345                         if not os.path.exists(_filename):
 346                                 self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
 347
 348                 for filename in os.listdir(path):
 349                         filename = os.path.join(path, filename)
 350
 351                         if os.path.isdir(filename):
 352                                 continue
 353
 354                         _filename = re.match(".*(releases/.*)", filename).group(1)
 355                         if _filename in files:
 356                                 continue
 357
 358                         if filename.endswith(".md5"):
 359                                 continue
 360
 361                         logging.info("Hashing %s..." % filename)
 362                         filehash = self.__file_hash(filename)
 363                         filesize = os.path.getsize(filename)
 364
 365                         # Check if there is a torrent download available for this file:
 366                         torrent_hash = ""
 367                         torrent_file = "%s.torrent" % filename
 368                         if os.path.exists(torrent_file):
 369                                 torrent_hash = self.torrent_read_hash(torrent_file)
 370
 371                         self.db.execute("INSERT INTO files(releases, filename, filesize, \
 372                                 sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s)",
 373                                 self.id, _filename, filesize, filehash, torrent_hash)
 374
 375                 # Search for all files that miss a torrent hash.
 376                 files = self.db.query("SELECT id, filename FROM files \
 377                         WHERE releases = %s AND torrent_hash IS NULL", self.id)
 378
 379                 for file in files:
 380                         path = os.path.join(basepath, file.filename)
 381
 382                         torrent_file = "%s.torrent" % path
 383                         if os.path.exists(torrent_file):
 384                                 torrent_hash = self.torrent_read_hash(torrent_file)
 385
 386                                 self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
 387                                         torrent_hash, file.id)
 388
 389         def torrent_read_hash(self, filename):
 390                 f = None
 391                 try:
 392                         f = open(filename, "rb")
 393
 394                         metainfo = tracker.bdecode(f.read())
 395                         metainfo = tracker.bencode(metainfo["info"])
 396
 397                         hash = hashlib.sha1()
 398                         hash.update(metainfo)
 399
 400                         return hash.hexdigest()
 401
 402                 finally:
 403                         if f:
 404                                 f.close()
 405
 406         def supports_arch(self, arch):
 407                 return arch in ("x86_64", "i586")
 408
 409         def supports_platform(self, platform):
 410                 # Currently there is nothing else than pcbios supported
 411                 if platform == "pcbios":
 412                         return True
 413
 414                 return False
 415
 416         def is_netboot_capable(self):
 417                 return self.path and "ipfire-2.x" in self.path
 418
 419         def netboot_kernel_url(self, arch, platform):
 420                 assert self.supports_arch(arch)
 421                 assert self.supports_platform(platform)
 422
 423                 if self.sname >= "ipfire-2.19-core100":
 424                         return "https://downloads.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)
 425
 426                 return "https://downloads.ipfire.org/%s/images/vmlinuz" % self.path
 427
 428         def netboot_initrd_url(self, arch, platform):
 429                 assert self.supports_arch(arch)
 430                 assert self.supports_platform(platform)
 431
 432                 if self.sname >= "ipfire-2.19-core100":
 433                         return "https://downloads.ipfire.org/%s/images/%s/instroot" % (self.path, arch)
 434
 435                 return "https://downloads.ipfire.org/%s/images/instroot" % self.path
 436
 437         def netboot_args(self, arch, platform):
 438                 return ""
 439
 440         @property
 441         def post(self):
 442                 if self.__data.blog_id:
 443                         return self.backend.blog.get_by_id(self.__data.blog_id)
 444
 445         # Fireinfo Stuff
 446
 447         @property
 448         def penetration(self):
 449                 penetration = self.memcache.get("%s-penetration" % self.sname)
 450
 451                 # Cache HIT
 452                 if penetration:
 453                         return penetration
 454
 455                 # Get penetration from fireinfo
 456                 penetration = self.backend.fireinfo.get_release_penetration(self)
 457
 458                 # Cache for 1 hour
 459                 self.memcache.set("%s-penetration" % self.sname, penetration, 3600)
 460
 461                 return penetration
 462
 463
 464 class Releases(Object):
 465         def _get_release(self, query, *args):
 466                 res = self.db.get(query, *args)
 467
 468                 if res:
 469                         return Release(self.backend, res.id, data=res)
 470
 471         def _get_releases(self, query, *args):
 472                 res = self.db.query(query, *args)
 473
 474                 for row in res:
 475                         yield Release(self.backend, row.id, data=row)
 476
 477         def get_by_id(self, id):
 478                 ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)
 479
 480                 if ret:
 481                         return Release(self.backend, ret.id, data=ret)
 482
 483         def get_by_sname(self, sname):
 484                 ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)
 485
 486                 if ret:
 487                         return Release(self.backend, ret.id, data=ret)
 488
 489         def get_by_news_id(self, news_id):
 490                 ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)
 491
 492                 if ret:
 493                         return Release(self.backend, ret.id, data=ret)
 494
 495         def get_latest(self, stable=True):
 496                 ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
 497                         AND stable = %s ORDER BY published DESC LIMIT 1", stable)
 498
 499                 if ret:
 500                         return Release(self.backend, ret.id, data=ret)
 501
 502         def get_releases_older_than(self, release, limit=None):
 503                 return self._get_releases("SELECT * FROM releases \
 504                         WHERE published IS NOT NULL AND published < %s \
 505                                 ORDER BY published DESC LIMIT %s", release.published, limit)
 506
 507         def get_latest_unstable(self):
 508                 ret = self.db.get("SELECT * FROM releases r1 \
 509                         WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
 510                         AND stable = %s AND NOT EXISTS ( \
 511                                 SELECT * FROM releases r2 WHERE r2.stable = %s AND \
 512                                         r2.published IS NOT NULL AND r2.published >= r1.published \
 513                                 ) ORDER BY r1.published DESC LIMIT 1", False, True)
 514
 515                 if ret:
 516                         return Release(self.backend, ret.id, data=ret)
 517
 518         def get_stable(self):
 519                 query = self.db.query("SELECT * FROM releases \
 520                         WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
 521                         ORDER BY published DESC")
 522
 523                 releases = []
 524                 for row in query:
 525                         release = Release(self.backend, row.id, data=row)
 526                         releases.append(release)
 527
 528                 return releases
 529
 530         def get_unstable(self):
 531                 query = self.db.query("SELECT * FROM releases \
 532                         WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
 533                         ORDER BY published DESC")
 534
 535                 releases = []
 536                 for row in query:
 537                         release = Release(self.backend, row.id, data=row)
 538                         releases.append(release)
 539
 540                 return releases
 541
 542         def get_all(self):
 543                 query = self.db.query("SELECT * FROM releases \
 544                         WHERE published IS NOT NULL AND published <= NOW() \
 545                         ORDER BY published DESC")
 546
 547                 releases = []
 548                 for row in query:
 549                         release = Release(self.backend, row.id, data=row)
 550                         releases.append(release)
 551
 552                 return releases
 553
 554         def _get_all(self):
 555                 query = self.db.query("SELECT * FROM releases ORDER BY published DESC")
 556
 557                 releases = []
 558                 for row in query:
 559                         release = Release(self.backend, row.id, data=row)
 560                         releases.append(release)
 561
 562                 return releases
 563
 564         def get_file_for_torrent_hash(self, torrent_hash):
 565                 file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
 566                         torrent_hash)
 567
 568                 if not file:
 569                         return
 570
 571                 release = Release(self.backend, file.releases)
 572                 file = File(self.backend, release, file.id)
 573
 574                 return file