src/backend/releases.py

   1 #!/usr/bin/python
   2
   3 import hashlib
   4 import logging
   5 import os
   6 import re
   7 import urllib.parse
   8 import yabencode
   9
  10 from . import database
  11 from .misc import Object
  12 from .decorators import *
  13
  14 TRACKERS = (
  15     "http://ipv4.tracker.ipfire.org:6969/announce",
  16     "udp://ipv4.tracker.ipfire.org:6969",
  17     "http://ipv6.tracker.ipfire.org:6969/announce",
  18     "udp://ipv6.tracker.ipfire.org:6969",
  19 )
  20
  21 class File(Object):
  22         def __init__(self, backend, release, id, data=None):
  23                 Object.__init__(self, backend)
  24
  25                 self.id = id
  26                 self._release = release
  27
  28                 # get all data from database
  29                 self.__data = data
  30
  31         def __eq__(self, other):
  32                 if isinstance(other, self.__class__):
  33                         return self.id == otherid
  34
  35         def __lt__(self, other):
  36                 if isinstance(other, self.__class__):
  37                         return self.prio < other.prio
  38
  39         @property
  40         def data(self):
  41                 if self.__data is None:
  42                         self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
  43                         assert self.__data
  44
  45                 return self.__data
  46
  47         @property
  48         def release(self):
  49                 if not self._release:
  50                         release_id = self.data.get("releases")
  51                         self._release = Release(release_id)
  52
  53                 return self._release
  54
  55         @property
  56         def type(self):
  57                 filename = self.filename
  58
  59                 if filename.endswith(".iso"):
  60                         return "iso"
  61
  62                 elif filename.endswith(".torrent"):
  63                         return "torrent"
  64
  65                 elif "xen" in filename:
  66                         if "downloader" in filename:
  67                                 return "xen-downloader"
  68
  69                         return "xen"
  70
  71                 elif "sources" in filename:
  72                         return "source"
  73
  74                 elif "usb-fdd" in filename:
  75                         return "usbfdd"
  76
  77                 elif "usb-hdd" in filename:
  78                         return "usbhdd"
  79
  80                 elif "armv5tel" in filename and "scon" in filename:
  81                         return "armv5tel-scon"
  82
  83                 elif "armv5tel" in filename:
  84                         return "armv5tel"
  85
  86                 elif "scon" in filename:
  87                         return "alix"
  88
  89                 elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
  90                         return "flash"
  91
  92                 else:
  93                         return "unknown"
  94
  95         @property
  96         def url(self):
  97                 return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)
  98
  99         @property
 100         def desc(self):
 101                 _ = lambda x: x
 102
 103                 descriptions = {
 104                         "armv5tel"      : _("Flash Image"),
 105                         "armv5tel-scon" : _("Flash Image with serial console"),
 106                         "iso"           : _("ISO Image"),
 107                         "torrent"       : _("Torrent File"),
 108                         "flash"         : _("Flash Image"),
 109                         "alix"          : _("Flash Image with serial console"),
 110                         "usbfdd"        : _("USB FDD Image"),
 111                         "usbhdd"        : _("USB HDD Image"),
 112                         "xen"           : _("Pre-generated Xen Image"),
 113                         "xen-downloader": _("Xen-Image Generator"),
 114                 }
 115
 116                 try:
 117                         return descriptions[self.type]
 118                 except KeyError:
 119                         return _("Unknown image type")
 120
 121         @property
 122         def prio(self):
 123                 priorities = {
 124                         "iso"           : 10,
 125                         "torrent"       : 20,
 126                         "flash"         : 40,
 127                         "alix"          : 41,
 128                         "usbfdd"        : 31,
 129                         "usbhdd"        : 30,
 130                         "armv5tel"      : 40,
 131                         "armv5tel-scon" : 41,
 132                         "xen"           : 50,
 133                         "xen-downloader": 51,
 134                 }
 135
 136                 try:
 137                         return priorities[self.type]
 138                 except KeyError:
 139                         return 999
 140
 141         @property
 142         def sha256(self):
 143                 return self.data.get("sha256")
 144
 145         @property
 146         def sha1(self):
 147                 return self.data.get("sha1")
 148
 149         @property
 150         def filename(self):
 151                 return self.data.get("filename")
 152
 153         @property
 154         def basename(self):
 155                 return os.path.basename(self.filename)
 156
 157         @property
 158         def size(self):
 159                 return self.data.get("filesize")
 160
 161         @property
 162         def arch(self):
 163                 known_arches = ("x86_64", "aarch64", "arm", "i586")
 164
 165                 for arch in known_arches:
 166                         if arch in self.basename:
 167                                 return arch
 168
 169                 return "N/A"
 170
 171         @property
 172         def torrent_hash(self):
 173                 return self.data.get("torrent_hash", None)
 174
 175         @property
 176         def torrent_url(self):
 177                 if self.torrent_hash:
 178                         return "%s.torrent" % self.url
 179
 180         @property
 181         def magnet_link(self):
 182                 # Don't return anything if we have no torrent hash.
 183                 if self.torrent_hash is None:
 184                         return
 185
 186                 s = "magnet:?xt=urn:btih:%s" % self.torrent_hash
 187
 188                 #s += "&xl=%d" % self.size
 189                 s += "&dn=%s" % urllib.parse.quote(self.basename)
 190
 191                 # Add our tracker.
 192                 for tracker in TRACKERS:
 193                         s += "&tr=%s" % tracker
 194
 195                 # Add web download URL
 196                 s += "&as=%s" % urllib.parse.quote(self.url)
 197
 198                 return s
 199
 200
 201 class Release(Object):
 202         def __init__(self, backend, id, data=None):
 203                 Object.__init__(self, backend)
 204                 self.id = id
 205
 206                 # get all data from database
 207                 self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
 208                 assert self.__data
 209
 210                 self.__files = []
 211
 212         def __str__(self):
 213                 return self.name
 214
 215         def __repr__(self):
 216                 return "<%s %s>" % (self.__class__.__name__, self.name)
 217
 218         def __cmp__(self, other):
 219                 return cmp(self.id, other.id)
 220
 221         @property
 222         def arches(self):
 223                 for arch in ("x86_64", "aarch64", "i586", "arm"):
 224                         if arch in (f.arch for f in self.files):
 225                                 yield arch
 226
 227         @property
 228         def primary_arches(self):
 229                 arches = []
 230
 231                 # Add x86_64 when available, otherwise add i586
 232                 for arch in ("x86_64", "i586"):
 233                         if arch in self.arches:
 234                                 arches.append(arch)
 235                                 break
 236
 237                 # Add ARM if available
 238                 if "arm" in self.arches:
 239                         arches.append("arm")
 240
 241                 return arches
 242
 243         @property
 244         def secondary_arches(self):
 245                 arches = []
 246
 247                 for arch in self.arches:
 248                         if arch in self.primary_arches:
 249                                 continue
 250
 251                         arches.append(arch)
 252
 253                 return arches
 254
 255         @property
 256         def experimental_arches(self):
 257                 return ("aarch64",)
 258
 259         @property
 260         def files(self):
 261                 if not self.__files:
 262                         files = self.db.query("SELECT * FROM files WHERE releases = %s \
 263                                 AND NOT filename LIKE '%%.torrent'", self.id)
 264
 265                         self.__files = [File(self.backend, self, f.id, f) for f in files]
 266                         self.__files.sort()
 267
 268                 return self.__files
 269
 270         def get_files_by_arch(self, arch):
 271                 for f in self.files:
 272                         if f.arch == arch:
 273                                 yield f
 274
 275         @property
 276         def torrents(self):
 277                 torrents = []
 278
 279                 for file in self.files:
 280                         if not file.torrent_hash:
 281                                 continue
 282
 283                         torrents.append(file)
 284
 285                 return torrents
 286
 287         @property
 288         def name(self):
 289                 return self.__data.name
 290
 291         @property
 292         def slug(self):
 293                 return self.__data.sname
 294
 295         # XXX compat
 296         sname = slug
 297
 298         @lazy_property
 299         def blog(self):
 300                 if self.__data.blog_id:
 301                         return self.backend.blog.get_by_id(self.__data.blog_id)
 302
 303         @property
 304         def fireinfo_id(self):
 305                 name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")
 306
 307                 res = self.db.get("SELECT id FROM fireinfo_releases \
 308                         WHERE name = %s", name)
 309
 310                 if res:
 311                         return res.id
 312
 313         @property
 314         def stable(self):
 315                 return self.__data.stable
 316
 317         @property
 318         def published(self):
 319                 return self.__data.published
 320
 321         date = published
 322
 323         @property
 324         def path(self):
 325                 return self.__data.path
 326
 327         def get_file(self, type):
 328                 for file in self.files:
 329                         if file.type == type:
 330                                 return file
 331
 332         def __file_hash(self, filename, algo="sha256"):
 333                 h = hashlib.new(algo)
 334
 335                 with open(filename, "rb") as f:
 336                         buf_size = 1024
 337                         buf = f.read(buf_size)
 338                         while buf:
 339                                 h.update(buf)
 340                                 buf = f.read(buf_size)
 341
 342                 return h.hexdigest()
 343
 344         def scan_files(self, basepath="/pub/mirror"):
 345                 if not self.path:
 346                         return
 347
 348                 path = os.path.join(basepath, self.path)
 349                 if not os.path.exists(path):
 350                         return
 351
 352                 files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
 353                 files = [f.filename for f in files]
 354
 355                 # Make files that do not exists not loadable.
 356                 for filename in files:
 357                         _filename = os.path.join(basepath, filename)
 358                         if not os.path.exists(_filename):
 359                                 self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
 360
 361                 for filename in os.listdir(path):
 362                         filename = os.path.join(path, filename)
 363
 364                         if os.path.isdir(filename):
 365                                 continue
 366
 367                         _filename = re.match(".*(releases/.*)", filename).group(1)
 368                         if _filename in files:
 369                                 continue
 370
 371                         if filename.endswith(".md5"):
 372                                 continue
 373
 374                         logging.info("Hashing %s..." % filename)
 375                         hash_sha256 = self.__file_hash(filename, "sha256")
 376                         hash_sha1   = self.__file_hash(filename, "sha1")
 377                         filesize = os.path.getsize(filename)
 378
 379                         # Check if there is a torrent download available for this file:
 380                         torrent_hash = ""
 381                         torrent_file = "%s.torrent" % filename
 382                         if os.path.exists(torrent_file):
 383                                 torrent_hash = self.torrent_read_hash(torrent_file)
 384
 385                         self.db.execute("INSERT INTO files(releases, filename, filesize, \
 386                                 sha256, sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s, %s)",
 387                                 self.id, _filename, filesize, hash_sha256, hash_sha1, torrent_hash)
 388
 389                 # Search for all files that miss a torrent hash.
 390                 files = self.db.query("SELECT id, filename FROM files \
 391                         WHERE releases = %s AND torrent_hash IS NULL", self.id)
 392
 393                 for file in files:
 394                         path = os.path.join(basepath, file.filename)
 395
 396                         torrent_file = "%s.torrent" % path
 397                         if os.path.exists(torrent_file):
 398                                 torrent_hash = self.torrent_read_hash(torrent_file)
 399
 400                                 self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
 401                                         torrent_hash, file.id)
 402
 403         def torrent_read_hash(self, filename):
 404                 with open(filename, "rb") as f:
 405                         metainfo = yabencode.decode(f)
 406                         metainfo = yabencode.encode(metainfo["info"])
 407
 408                         h = hashlib.new("sha1")
 409                         h.update(metainfo)
 410
 411                         return h.hexdigest()
 412
 413         def supports_arch(self, arch):
 414                 return arch in ("x86_64", "i586")
 415
 416         def supports_platform(self, platform):
 417                 # Currently there is nothing else than pcbios supported
 418                 if platform == "pcbios":
 419                         return True
 420
 421                 return False
 422
 423         def is_netboot_capable(self):
 424                 return self.path and "ipfire-2.x" in self.path
 425
 426         def netboot_kernel_url(self, arch, platform):
 427                 assert self.supports_arch(arch)
 428                 assert self.supports_platform(platform)
 429
 430                 if self.sname >= "ipfire-2.19-core100":
 431                         return "http://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)
 432
 433                 return "http://boot.ipfire.org/%s/images/vmlinuz" % self.path
 434
 435         def netboot_initrd_url(self, arch, platform):
 436                 assert self.supports_arch(arch)
 437                 assert self.supports_platform(platform)
 438
 439                 if self.sname >= "ipfire-2.19-core100":
 440                         return "http://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch)
 441
 442                 return "http://boot.ipfire.org/%s/images/instroot" % self.path
 443
 444         def netboot_args(self, arch, platform):
 445                 return ""
 446
 447         @property
 448         def post(self):
 449                 if self.__data.blog_id:
 450                         return self.backend.blog.get_by_id(self.__data.blog_id)
 451
 452         # Fireinfo Stuff
 453
 454         @property
 455         def penetration(self):
 456                 # Get penetration from fireinfo
 457                 return self.backend.fireinfo.get_release_penetration(self)
 458
 459
 460 class Releases(Object):
 461         def _get_release(self, query, *args):
 462                 res = self.db.get(query, *args)
 463
 464                 if res:
 465                         return Release(self.backend, res.id, data=res)
 466
 467         def _get_releases(self, query, *args):
 468                 res = self.db.query(query, *args)
 469
 470                 for row in res:
 471                         yield Release(self.backend, row.id, data=row)
 472
 473         def __iter__(self):
 474                 releases = self._get_releases("SELECT * FROM releases \
 475                         ORDER BY published DESC NULLS FIRST")
 476
 477                 return iter(releases)
 478
 479         def get_by_id(self, id):
 480                 ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)
 481
 482                 if ret:
 483                         return Release(self.backend, ret.id, data=ret)
 484
 485         def get_by_sname(self, sname):
 486                 ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)
 487
 488                 if ret:
 489                         return Release(self.backend, ret.id, data=ret)
 490
 491         def get_by_news_id(self, news_id):
 492                 ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)
 493
 494                 if ret:
 495                         return Release(self.backend, ret.id, data=ret)
 496
 497         def get_latest(self, stable=True):
 498                 ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
 499                         AND stable = %s ORDER BY published DESC LIMIT 1", stable)
 500
 501                 if ret:
 502                         return Release(self.backend, ret.id, data=ret)
 503
 504         def get_releases_older_than(self, release, limit=None):
 505                 return self._get_releases("SELECT * FROM releases \
 506                         WHERE published IS NOT NULL AND published < %s \
 507                                 ORDER BY published DESC LIMIT %s", release.published, limit)
 508
 509         def get_latest_unstable(self):
 510                 ret = self.db.get("SELECT * FROM releases r1 \
 511                         WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
 512                         AND stable = %s AND NOT EXISTS ( \
 513                                 SELECT * FROM releases r2 WHERE r2.stable = %s AND \
 514                                         r2.published IS NOT NULL AND r2.published >= r1.published \
 515                                 ) ORDER BY r1.published DESC LIMIT 1", False, True)
 516
 517                 if ret:
 518                         return Release(self.backend, ret.id, data=ret)
 519
 520         def get_stable(self):
 521                 query = self.db.query("SELECT * FROM releases \
 522                         WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
 523                         ORDER BY published DESC")
 524
 525                 releases = []
 526                 for row in query:
 527                         release = Release(self.backend, row.id, data=row)
 528                         releases.append(release)
 529
 530                 return releases
 531
 532         def get_unstable(self):
 533                 query = self.db.query("SELECT * FROM releases \
 534                         WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
 535                         ORDER BY published DESC")
 536
 537                 releases = []
 538                 for row in query:
 539                         release = Release(self.backend, row.id, data=row)
 540                         releases.append(release)
 541
 542                 return releases
 543
 544         def get_all(self):
 545                 query = self.db.query("SELECT * FROM releases \
 546                         WHERE published IS NOT NULL AND published <= NOW() \
 547                         ORDER BY published DESC")
 548
 549                 releases = []
 550                 for row in query:
 551                         release = Release(self.backend, row.id, data=row)
 552                         releases.append(release)
 553
 554                 return releases
 555
 556         def _get_all(self):
 557                 query = self.db.query("SELECT * FROM releases ORDER BY published DESC")
 558
 559                 releases = []
 560                 for row in query:
 561                         release = Release(self.backend, row.id, data=row)
 562                         releases.append(release)
 563
 564                 return releases
 565
 566         def get_file_for_torrent_hash(self, torrent_hash):
 567                 file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
 568                         torrent_hash)
 569
 570                 if not file:
 571                         return
 572
 573                 release = Release(self.backend, file.releases)
 574                 file = File(self.backend, release, file.id)
 575
 576                 return file
 577
 578         async def scan_files(self, basepath="/pub/mirror"):
 579                 for release in self:
 580                         logging.debug("Scanning %s..." % release)
 581
 582                         with self.db.transaction():
 583                                 release.scan_files(basepath=basepath)