src/backend/releases.py

   1 #!/usr/bin/python
   2
   3 import hashlib
   4 import logging
   5 import os
   6 import re
   7 import urllib.request, urllib.parse, urllib.error
   8 import urllib.parse
   9
  10 from . import database
  11 from . import tracker
  12 from .misc import Object
  13
  14 TRACKERS = (
  15     "http://ipv4.tracker.ipfire.org:6969/announce",
  16     "udp://ipv4.tracker.ipfire.org:6969",
  17     "http://ipv6.tracker.ipfire.org:6969/announce",
  18     "udp://ipv6.tracker.ipfire.org:6969",
  19 )
  20
  21 class File(Object):
  22         def __init__(self, backend, release, id, data=None):
  23                 Object.__init__(self, backend)
  24
  25                 self.id = id
  26                 self._release = release
  27
  28                 # get all data from database
  29                 self.__data = data
  30
  31         def __cmp__(self, other):
  32                 return cmp(self.prio, other.prio)
  33
  34         @property
  35         def data(self):
  36                 if self.__data is None:
  37                         self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
  38                         assert self.__data
  39
  40                 return self.__data
  41
  42         @property
  43         def release(self):
  44                 if not self._release:
  45                         release_id = self.data.get("releases")
  46                         self._release = Release(release_id)
  47
  48                 return self._release
  49
  50         @property
  51         def type(self):
  52                 filename = self.filename
  53
  54                 if filename.endswith(".iso"):
  55                         return "iso"
  56
  57                 elif filename.endswith(".torrent"):
  58                         return "torrent"
  59
  60                 elif "xen" in filename:
  61                         if "downloader" in filename:
  62                                 return "xen-downloader"
  63
  64                         return "xen"
  65
  66                 elif "sources" in filename:
  67                         return "source"
  68
  69                 elif "usb-fdd" in filename:
  70                         return "usbfdd"
  71
  72                 elif "usb-hdd" in filename:
  73                         return "usbhdd"
  74
  75                 elif "armv5tel" in filename and "scon" in filename:
  76                         return "armv5tel-scon"
  77
  78                 elif "armv5tel" in filename:
  79                         return "armv5tel"
  80
  81                 elif "scon" in filename:
  82                         return "alix"
  83
  84                 elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
  85                         return "flash"
  86
  87                 else:
  88                         return "unknown"
  89
  90         @property
  91         def url(self):
  92                 return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)
  93
  94         @property
  95         def desc(self):
  96                 _ = lambda x: x
  97
  98                 descriptions = {
  99                         "armv5tel"      : _("Flash Image"),
 100                         "armv5tel-scon" : _("Flash Image with serial console"),
 101                         "iso"           : _("CD Image"),
 102                         "torrent"       : _("Torrent File"),
 103                         "flash"         : _("Flash Image"),
 104                         "alix"          : _("Flash Image with serial console"),
 105                         "usbfdd"        : _("USB FDD Image"),
 106                         "usbhdd"        : _("USB HDD Image"),
 107                         "xen"           : _("Pre-generated Xen Image"),
 108                         "xen-downloader": _("Xen-Image Generator"),
 109                 }
 110
 111                 try:
 112                         return descriptions[self.type]
 113                 except KeyError:
 114                         return _("Unknown image type")
 115
 116         @property
 117         def prio(self):
 118                 priorities = {
 119                         "iso"           : 10,
 120                         "torrent"       : 20,
 121                         "flash"         : 40,
 122                         "alix"          : 41,
 123                         "usbfdd"        : 31,
 124                         "usbhdd"        : 30,
 125                         "armv5tel"      : 40,
 126                         "armv5tel-scon" : 41,
 127                         "xen"           : 50,
 128                         "xen-downloader": 51,
 129                 }
 130
 131                 try:
 132                         return priorities[self.type]
 133                 except KeyError:
 134                         return 999
 135
 136         @property
 137         def rem(self):
 138                 _ = lambda x: x
 139
 140                 remarks = {
 141                         "armv5tel"      : _("This image runs on many ARM-based boards"),
 142                         "armv5tel-scon" : _("This image runs on ARM boards with a serial console"),
 143                         "iso"           : _("Use this image to burn a CD and install IPFire from it."),
 144                         "torrent"       : _("Download the CD image from the torrent network."),
 145                         "flash"         : _("An image that is meant to run on embedded devices."),
 146                         "alix"          : _("Flash image where a serial console is enabled by default."),
 147                         "usbfdd"        : _("Install IPFire from a floppy-formated USB key."),
 148                         "usbhdd"        : _("If the floppy image doesn't work, use this image instead."),
 149                         "xen"           : _("A ready-to-run image for Xen."),
 150                         "xen-downloader": _("Generator for creating a Xen image."),
 151                 }
 152
 153                 try:
 154                         return remarks[self.type]
 155                 except KeyError:
 156                         return _("Unknown image type")
 157
 158         @property
 159         def sha1(self):
 160                 return self.data.get("sha1")
 161
 162         @property
 163         def filename(self):
 164                 return self.data.get("filename")
 165
 166         @property
 167         def basename(self):
 168                 return os.path.basename(self.filename)
 169
 170         @property
 171         def size(self):
 172                 return self.data.get("filesize")
 173
 174         @property
 175         def arch(self):
 176                 known_arches = ("x86_64", "i586", "arm")
 177
 178                 for arch in known_arches:
 179                         if arch in self.basename:
 180                                 return arch
 181
 182                 return "N/A"
 183
 184         @property
 185         def torrent_hash(self):
 186                 return self.data.get("torrent_hash", None)
 187
 188         @property
 189         def torrent_url(self):
 190                 if self.torrent_hash:
 191                         return "%s.torrent" % self.url
 192
 193         @property
 194         def magnet_link(self):
 195                 # Don't return anything if we have no torrent hash.
 196                 if self.torrent_hash is None:
 197                         return
 198
 199                 s = "magnet:?xt=urn:btih:%s" % self.torrent_hash
 200
 201                 #s += "&xl=%d" % self.size
 202                 s += "&dn=%s" % urllib.parse.quote(self.basename)
 203
 204                 # Add our tracker.
 205                 for tracker in TRACKERS:
 206                         s += "&tr=%s" % tracker
 207
 208                 # Add web download URL
 209                 s += "&as=%s" % urllib.parse.quote(self.url)
 210
 211                 return s
 212
 213
 214 class Release(Object):
 215         def __init__(self, backend, id, data=None):
 216                 Object.__init__(self, backend)
 217                 self.id = id
 218
 219                 # get all data from database
 220                 self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
 221                 assert self.__data
 222
 223                 self.__files = []
 224
 225         def __str__(self):
 226                 return self.name
 227
 228         def __repr__(self):
 229                 return "<%s %s>" % (self.__class__.__name__, self.name)
 230
 231         def __cmp__(self, other):
 232                 return cmp(self.id, other.id)
 233
 234         @property
 235         def arches(self):
 236                 for arch in ("x86_64", "aarch64", "i586", "arm"):
 237                         if arch in (f.arch for f in self.files):
 238                                 yield arch
 239
 240         @property
 241         def files(self):
 242                 if not self.__files:
 243                         files = self.db.query("SELECT * FROM files WHERE releases = %s \
 244                                 AND NOT filename LIKE '%%.torrent'", self.id)
 245
 246                         self.__files = [File(self.backend, self, f.id, f) for f in files]
 247                         self.__files.sort()
 248
 249                 return self.__files
 250
 251         def get_files_by_arch(self, arch):
 252                 for f in self.files:
 253                         if f.arch == arch:
 254                                 yield f
 255
 256         @property
 257         def torrents(self):
 258                 torrents = []
 259
 260                 for file in self.files:
 261                         if not file.torrent_hash:
 262                                 continue
 263
 264                         torrents.append(file)
 265
 266                 return torrents
 267
 268         @property
 269         def name(self):
 270                 return self.__data.name
 271
 272         @property
 273         def slug(self):
 274                 return self.__data.sname
 275
 276         # XXX compat
 277         sname = slug
 278
 279         # XXX cache this
 280         @property
 281         def blog(self):
 282                 if self.__data.blog_id:
 283                         return self.backend.blog.get_by_id(self.__data.blog_id)
 284
 285         @property
 286         def fireinfo_id(self):
 287                 name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")
 288
 289                 res = self.db.get("SELECT id FROM fireinfo_releases \
 290                         WHERE name = %s", name)
 291
 292                 if res:
 293                         return res.id
 294
 295         @property
 296         def stable(self):
 297                 return self.__data.stable
 298
 299         @property
 300         def published(self):
 301                 return self.__data.published
 302
 303         date = published
 304
 305         @property
 306         def path(self):
 307                 return self.__data.path
 308
 309         def get_file(self, type):
 310                 for file in self.files:
 311                         if file.type == type:
 312                                 return file
 313
 314         def __file_hash(self, filename):
 315                 sha1 = hashlib.sha1()
 316
 317                 with open(filename) as f:
 318                         buf_size = 1024
 319                         buf = f.read(buf_size)
 320                         while buf:
 321                                 sha1.update(buf)
 322                                 buf = f.read(buf_size)
 323
 324                 return sha1.hexdigest()
 325
 326         def scan_files(self, basepath="/srv/mirror0"):
 327                 if not self.path:
 328                         return
 329
 330                 path = os.path.join(basepath, self.path)
 331                 if not os.path.exists(path):
 332                         return
 333
 334                 files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
 335                 files = [f.filename for f in files]
 336
 337                 # Make files that do not exists not loadable.
 338                 for filename in files:
 339                         _filename = os.path.join(basepath, filename)
 340                         if not os.path.exists(_filename):
 341                                 self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
 342
 343                 for filename in os.listdir(path):
 344                         filename = os.path.join(path, filename)
 345
 346                         if os.path.isdir(filename):
 347                                 continue
 348
 349                         _filename = re.match(".*(releases/.*)", filename).group(1)
 350                         if _filename in files:
 351                                 continue
 352
 353                         if filename.endswith(".md5"):
 354                                 continue
 355
 356                         logging.info("Hashing %s..." % filename)
 357                         filehash = self.__file_hash(filename)
 358                         filesize = os.path.getsize(filename)
 359
 360                         # Check if there is a torrent download available for this file:
 361                         torrent_hash = ""
 362                         torrent_file = "%s.torrent" % filename
 363                         if os.path.exists(torrent_file):
 364                                 torrent_hash = self.torrent_read_hash(torrent_file)
 365
 366                         self.db.execute("INSERT INTO files(releases, filename, filesize, \
 367                                 sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s)",
 368                                 self.id, _filename, filesize, filehash, torrent_hash)
 369
 370                 # Search for all files that miss a torrent hash.
 371                 files = self.db.query("SELECT id, filename FROM files \
 372                         WHERE releases = %s AND torrent_hash IS NULL", self.id)
 373
 374                 for file in files:
 375                         path = os.path.join(basepath, file.filename)
 376
 377                         torrent_file = "%s.torrent" % path
 378                         if os.path.exists(torrent_file):
 379                                 torrent_hash = self.torrent_read_hash(torrent_file)
 380
 381                                 self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
 382                                         torrent_hash, file.id)
 383
 384         def torrent_read_hash(self, filename):
 385                 f = None
 386                 try:
 387                         f = open(filename, "rb")
 388
 389                         metainfo = tracker.bdecode(f.read())
 390                         metainfo = tracker.bencode(metainfo["info"])
 391
 392                         hash = hashlib.sha1()
 393                         hash.update(metainfo)
 394
 395                         return hash.hexdigest()
 396
 397                 finally:
 398                         if f:
 399                                 f.close()
 400
 401         def supports_arch(self, arch):
 402                 return arch in ("x86_64", "i586")
 403
 404         def supports_platform(self, platform):
 405                 # Currently there is nothing else than pcbios supported
 406                 if platform == "pcbios":
 407                         return True
 408
 409                 return False
 410
 411         def is_netboot_capable(self):
 412                 return self.path and "ipfire-2.x" in self.path
 413
 414         def netboot_kernel_url(self, arch, platform):
 415                 assert self.supports_arch(arch)
 416                 assert self.supports_platform(platform)
 417
 418                 if self.sname >= "ipfire-2.19-core100":
 419                         return "https://downloads.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)
 420
 421                 return "https://downloads.ipfire.org/%s/images/vmlinuz" % self.path
 422
 423         def netboot_initrd_url(self, arch, platform):
 424                 assert self.supports_arch(arch)
 425                 assert self.supports_platform(platform)
 426
 427                 if self.sname >= "ipfire-2.19-core100":
 428                         return "https://downloads.ipfire.org/%s/images/%s/instroot" % (self.path, arch)
 429
 430                 return "https://downloads.ipfire.org/%s/images/instroot" % self.path
 431
 432         def netboot_args(self, arch, platform):
 433                 return ""
 434
 435         @property
 436         def post(self):
 437                 if self.__data.blog_id:
 438                         return self.backend.blog.get_by_id(self.__data.blog_id)
 439
 440         # Fireinfo Stuff
 441
 442         @property
 443         def penetration(self):
 444                 penetration = self.memcache.get("%s-penetration" % self.sname)
 445
 446                 # Cache HIT
 447                 if penetration:
 448                         return penetration
 449
 450                 # Get penetration from fireinfo
 451                 penetration = self.backend.fireinfo.get_release_penetration(self)
 452
 453                 # Cache for 1 hour
 454                 self.memcache.set("%s-penetration" % self.sname, penetration, 3600)
 455
 456                 return penetration
 457
 458
 459 class Releases(Object):
 460         def _get_release(self, query, *args):
 461                 res = self.db.get(query, *args)
 462
 463                 if res:
 464                         return Release(self.backend, res.id, data=res)
 465
 466         def _get_releases(self, query, *args):
 467                 res = self.db.query(query, *args)
 468
 469                 for row in res:
 470                         yield Release(self.backend, row.id, data=row)
 471
 472         def get_by_id(self, id):
 473                 ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)
 474
 475                 if ret:
 476                         return Release(self.backend, ret.id, data=ret)
 477
 478         def get_by_sname(self, sname):
 479                 ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)
 480
 481                 if ret:
 482                         return Release(self.backend, ret.id, data=ret)
 483
 484         def get_by_news_id(self, news_id):
 485                 ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)
 486
 487                 if ret:
 488                         return Release(self.backend, ret.id, data=ret)
 489
 490         def get_latest(self, stable=True):
 491                 ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
 492                         AND stable = %s ORDER BY published DESC LIMIT 1", stable)
 493
 494                 if ret:
 495                         return Release(self.backend, ret.id, data=ret)
 496
 497         def get_releases_older_than(self, release, limit=None):
 498                 return self._get_releases("SELECT * FROM releases \
 499                         WHERE published IS NOT NULL AND published < %s \
 500                                 ORDER BY published DESC LIMIT %s", release.published, limit)
 501
 502         def get_latest_unstable(self):
 503                 ret = self.db.get("SELECT * FROM releases r1 \
 504                         WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
 505                         AND stable = %s AND NOT EXISTS ( \
 506                                 SELECT * FROM releases r2 WHERE r2.stable = %s AND \
 507                                         r2.published IS NOT NULL AND r2.published >= r1.published \
 508                                 ) ORDER BY r1.published DESC LIMIT 1", False, True)
 509
 510                 if ret:
 511                         return Release(self.backend, ret.id, data=ret)
 512
 513         def get_stable(self):
 514                 query = self.db.query("SELECT * FROM releases \
 515                         WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
 516                         ORDER BY published DESC")
 517
 518                 releases = []
 519                 for row in query:
 520                         release = Release(self.backend, row.id, data=row)
 521                         releases.append(release)
 522
 523                 return releases
 524
 525         def get_unstable(self):
 526                 query = self.db.query("SELECT * FROM releases \
 527                         WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
 528                         ORDER BY published DESC")
 529
 530                 releases = []
 531                 for row in query:
 532                         release = Release(self.backend, row.id, data=row)
 533                         releases.append(release)
 534
 535                 return releases
 536
 537         def get_all(self):
 538                 query = self.db.query("SELECT * FROM releases \
 539                         WHERE published IS NOT NULL AND published <= NOW() \
 540                         ORDER BY published DESC")
 541
 542                 releases = []
 543                 for row in query:
 544                         release = Release(self.backend, row.id, data=row)
 545                         releases.append(release)
 546
 547                 return releases
 548
 549         def _get_all(self):
 550                 query = self.db.query("SELECT * FROM releases ORDER BY published DESC")
 551
 552                 releases = []
 553                 for row in query:
 554                         release = Release(self.backend, row.id, data=row)
 555                         releases.append(release)
 556
 557                 return releases
 558
 559         def get_file_for_torrent_hash(self, torrent_hash):
 560                 file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
 561                         torrent_hash)
 562
 563                 if not file:
 564                         return
 565
 566                 release = Release(self.backend, file.releases)
 567                 file = File(self.backend, release, file.id)
 568
 569                 return file