src/backend/releases.py

   1 #!/usr/bin/python
   2
   3 import hashlib
   4 import logging
   5 import os
   6 import re
   7 import tornado.gen
   8 import urllib.parse
   9 import yabencode
  10
  11 from . import database
  12 from .misc import Object
  13 from .decorators import *
  14
  15 TRACKERS = (
  16     "http://ipv4.tracker.ipfire.org:6969/announce",
  17     "udp://ipv4.tracker.ipfire.org:6969",
  18     "http://ipv6.tracker.ipfire.org:6969/announce",
  19     "udp://ipv6.tracker.ipfire.org:6969",
  20 )
  21
  22 class File(Object):
  23         def __init__(self, backend, release, id, data=None):
  24                 Object.__init__(self, backend)
  25
  26                 self.id = id
  27                 self._release = release
  28
  29                 # get all data from database
  30                 self.__data = data
  31
  32         def __eq__(self, other):
  33                 if isinstance(other, self.__class__):
  34                         return self.id == otherid
  35
  36         def __lt__(self, other):
  37                 if isinstance(other, self.__class__):
  38                         return self.prio < other.prio
  39
  40         @property
  41         def data(self):
  42                 if self.__data is None:
  43                         self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
  44                         assert self.__data
  45
  46                 return self.__data
  47
  48         @property
  49         def release(self):
  50                 if not self._release:
  51                         release_id = self.data.get("releases")
  52                         self._release = Release(release_id)
  53
  54                 return self._release
  55
  56         @property
  57         def type(self):
  58                 filename = self.filename
  59
  60                 if filename.endswith(".iso"):
  61                         return "iso"
  62
  63                 elif filename.endswith(".torrent"):
  64                         return "torrent"
  65
  66                 elif "xen" in filename:
  67                         if "downloader" in filename:
  68                                 return "xen-downloader"
  69
  70                         return "xen"
  71
  72                 elif "sources" in filename:
  73                         return "source"
  74
  75                 elif "usb-fdd" in filename:
  76                         return "usbfdd"
  77
  78                 elif "usb-hdd" in filename:
  79                         return "usbhdd"
  80
  81                 elif "armv5tel" in filename and "scon" in filename:
  82                         return "armv5tel-scon"
  83
  84                 elif "armv5tel" in filename:
  85                         return "armv5tel"
  86
  87                 elif "scon" in filename:
  88                         return "alix"
  89
  90                 elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
  91                         return "flash"
  92
  93                 else:
  94                         return "unknown"
  95
  96         @property
  97         def url(self):
  98                 return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)
  99
 100         @property
 101         def desc(self):
 102                 _ = lambda x: x
 103
 104                 descriptions = {
 105                         "armv5tel"      : _("Flash Image"),
 106                         "armv5tel-scon" : _("Flash Image with serial console"),
 107                         "iso"           : _("ISO Image"),
 108                         "torrent"       : _("Torrent File"),
 109                         "flash"         : _("Flash Image"),
 110                         "alix"          : _("Flash Image with serial console"),
 111                         "usbfdd"        : _("USB FDD Image"),
 112                         "usbhdd"        : _("USB HDD Image"),
 113                         "xen"           : _("Pre-generated Xen Image"),
 114                         "xen-downloader": _("Xen-Image Generator"),
 115                 }
 116
 117                 try:
 118                         return descriptions[self.type]
 119                 except KeyError:
 120                         return _("Unknown image type")
 121
 122         @property
 123         def prio(self):
 124                 priorities = {
 125                         "iso"           : 10,
 126                         "torrent"       : 20,
 127                         "flash"         : 40,
 128                         "alix"          : 41,
 129                         "usbfdd"        : 31,
 130                         "usbhdd"        : 30,
 131                         "armv5tel"      : 40,
 132                         "armv5tel-scon" : 41,
 133                         "xen"           : 50,
 134                         "xen-downloader": 51,
 135                 }
 136
 137                 try:
 138                         return priorities[self.type]
 139                 except KeyError:
 140                         return 999
 141
 142         @property
 143         def sha256(self):
 144                 return self.data.get("sha256")
 145
 146         @property
 147         def sha1(self):
 148                 return self.data.get("sha1")
 149
 150         @property
 151         def filename(self):
 152                 return self.data.get("filename")
 153
 154         @property
 155         def basename(self):
 156                 return os.path.basename(self.filename)
 157
 158         @property
 159         def size(self):
 160                 return self.data.get("filesize")
 161
 162         @property
 163         def arch(self):
 164                 known_arches = ("x86_64", "i586", "arm")
 165
 166                 for arch in known_arches:
 167                         if arch in self.basename:
 168                                 return arch
 169
 170                 return "N/A"
 171
 172         @property
 173         def torrent_hash(self):
 174                 return self.data.get("torrent_hash", None)
 175
 176         @property
 177         def torrent_url(self):
 178                 if self.torrent_hash:
 179                         return "%s.torrent" % self.url
 180
 181         @property
 182         def magnet_link(self):
 183                 # Don't return anything if we have no torrent hash.
 184                 if self.torrent_hash is None:
 185                         return
 186
 187                 s = "magnet:?xt=urn:btih:%s" % self.torrent_hash
 188
 189                 #s += "&xl=%d" % self.size
 190                 s += "&dn=%s" % urllib.parse.quote(self.basename)
 191
 192                 # Add our tracker.
 193                 for tracker in TRACKERS:
 194                         s += "&tr=%s" % tracker
 195
 196                 # Add web download URL
 197                 s += "&as=%s" % urllib.parse.quote(self.url)
 198
 199                 return s
 200
 201
 202 class Release(Object):
 203         def __init__(self, backend, id, data=None):
 204                 Object.__init__(self, backend)
 205                 self.id = id
 206
 207                 # get all data from database
 208                 self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
 209                 assert self.__data
 210
 211                 self.__files = []
 212
 213         def __str__(self):
 214                 return self.name
 215
 216         def __repr__(self):
 217                 return "<%s %s>" % (self.__class__.__name__, self.name)
 218
 219         def __cmp__(self, other):
 220                 return cmp(self.id, other.id)
 221
 222         @property
 223         def arches(self):
 224                 for arch in ("x86_64", "aarch64", "i586", "arm"):
 225                         if arch in (f.arch for f in self.files):
 226                                 yield arch
 227
 228         @property
 229         def files(self):
 230                 if not self.__files:
 231                         files = self.db.query("SELECT * FROM files WHERE releases = %s \
 232                                 AND NOT filename LIKE '%%.torrent'", self.id)
 233
 234                         self.__files = [File(self.backend, self, f.id, f) for f in files]
 235                         self.__files.sort()
 236
 237                 return self.__files
 238
 239         def get_files_by_arch(self, arch):
 240                 for f in self.files:
 241                         if f.arch == arch:
 242                                 yield f
 243
 244         @property
 245         def torrents(self):
 246                 torrents = []
 247
 248                 for file in self.files:
 249                         if not file.torrent_hash:
 250                                 continue
 251
 252                         torrents.append(file)
 253
 254                 return torrents
 255
 256         @property
 257         def name(self):
 258                 return self.__data.name
 259
 260         @property
 261         def slug(self):
 262                 return self.__data.sname
 263
 264         # XXX compat
 265         sname = slug
 266
 267         @lazy_property
 268         def blog(self):
 269                 if self.__data.blog_id:
 270                         return self.backend.blog.get_by_id(self.__data.blog_id)
 271
 272         @property
 273         def fireinfo_id(self):
 274                 name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")
 275
 276                 res = self.db.get("SELECT id FROM fireinfo_releases \
 277                         WHERE name = %s", name)
 278
 279                 if res:
 280                         return res.id
 281
 282         @property
 283         def stable(self):
 284                 return self.__data.stable
 285
 286         @property
 287         def published(self):
 288                 return self.__data.published
 289
 290         date = published
 291
 292         @property
 293         def path(self):
 294                 return self.__data.path
 295
 296         def get_file(self, type):
 297                 for file in self.files:
 298                         if file.type == type:
 299                                 return file
 300
 301         def __file_hash(self, filename, algo="sha256"):
 302                 h = hashlib.new(algo)
 303
 304                 with open(filename, "rb") as f:
 305                         buf_size = 1024
 306                         buf = f.read(buf_size)
 307                         while buf:
 308                                 h.update(buf)
 309                                 buf = f.read(buf_size)
 310
 311                 return h.hexdigest()
 312
 313         def scan_files(self, basepath="/pub/mirror"):
 314                 if not self.path:
 315                         return
 316
 317                 path = os.path.join(basepath, self.path)
 318                 if not os.path.exists(path):
 319                         return
 320
 321                 files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
 322                 files = [f.filename for f in files]
 323
 324                 # Make files that do not exists not loadable.
 325                 for filename in files:
 326                         _filename = os.path.join(basepath, filename)
 327                         if not os.path.exists(_filename):
 328                                 self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
 329
 330                 for filename in os.listdir(path):
 331                         filename = os.path.join(path, filename)
 332
 333                         if os.path.isdir(filename):
 334                                 continue
 335
 336                         _filename = re.match(".*(releases/.*)", filename).group(1)
 337                         if _filename in files:
 338                                 continue
 339
 340                         if filename.endswith(".md5"):
 341                                 continue
 342
 343                         logging.info("Hashing %s..." % filename)
 344                         hash_sha256 = self.__file_hash(filename, "sha256")
 345                         hash_sha1   = self.__file_hash(filename, "sha1")
 346                         filesize = os.path.getsize(filename)
 347
 348                         # Check if there is a torrent download available for this file:
 349                         torrent_hash = ""
 350                         torrent_file = "%s.torrent" % filename
 351                         if os.path.exists(torrent_file):
 352                                 torrent_hash = self.torrent_read_hash(torrent_file)
 353
 354                         self.db.execute("INSERT INTO files(releases, filename, filesize, \
 355                                 sha256, sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s, %s)",
 356                                 self.id, _filename, filesize, hash_sha256, hash_sha1, torrent_hash)
 357
 358                 # Search for all files that miss a torrent hash.
 359                 files = self.db.query("SELECT id, filename FROM files \
 360                         WHERE releases = %s AND torrent_hash IS NULL", self.id)
 361
 362                 for file in files:
 363                         path = os.path.join(basepath, file.filename)
 364
 365                         torrent_file = "%s.torrent" % path
 366                         if os.path.exists(torrent_file):
 367                                 torrent_hash = self.torrent_read_hash(torrent_file)
 368
 369                                 self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
 370                                         torrent_hash, file.id)
 371
 372         def torrent_read_hash(self, filename):
 373                 with open(filename, "rb") as f:
 374                         metainfo = yabencode.decode(f)
 375                         metainfo = yabencode.encode(metainfo["info"])
 376
 377                         h = hashlib.new("sha1")
 378                         h.update(metainfo)
 379
 380                         return h.hexdigest()
 381
 382         def supports_arch(self, arch):
 383                 return arch in ("x86_64", "i586")
 384
 385         def supports_platform(self, platform):
 386                 # Currently there is nothing else than pcbios supported
 387                 if platform == "pcbios":
 388                         return True
 389
 390                 return False
 391
 392         def is_netboot_capable(self):
 393                 return self.path and "ipfire-2.x" in self.path
 394
 395         def netboot_kernel_url(self, arch, platform):
 396                 assert self.supports_arch(arch)
 397                 assert self.supports_platform(platform)
 398
 399                 if self.sname >= "ipfire-2.19-core100":
 400                         return "https://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)
 401
 402                 return "https://boot.ipfire.org/%s/images/vmlinuz" % self.path
 403
 404         def netboot_initrd_url(self, arch, platform):
 405                 assert self.supports_arch(arch)
 406                 assert self.supports_platform(platform)
 407
 408                 if self.sname >= "ipfire-2.19-core100":
 409                         return "https://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch)
 410
 411                 return "https://boot.ipfire.org/%s/images/instroot" % self.path
 412
 413         def netboot_args(self, arch, platform):
 414                 return ""
 415
 416         @property
 417         def post(self):
 418                 if self.__data.blog_id:
 419                         return self.backend.blog.get_by_id(self.__data.blog_id)
 420
 421         # Fireinfo Stuff
 422
 423         @property
 424         def penetration(self):
 425                 # Get penetration from fireinfo
 426                 return self.backend.fireinfo.get_release_penetration(self)
 427
 428
 429 class Releases(Object):
 430         def _get_release(self, query, *args):
 431                 res = self.db.get(query, *args)
 432
 433                 if res:
 434                         return Release(self.backend, res.id, data=res)
 435
 436         def _get_releases(self, query, *args):
 437                 res = self.db.query(query, *args)
 438
 439                 for row in res:
 440                         yield Release(self.backend, row.id, data=row)
 441
 442         def __iter__(self):
 443                 releases = self._get_releases("SELECT * FROM releases \
 444                         ORDER BY published DESC NULLS FIRST")
 445
 446                 return iter(releases)
 447
 448         def get_by_id(self, id):
 449                 ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)
 450
 451                 if ret:
 452                         return Release(self.backend, ret.id, data=ret)
 453
 454         def get_by_sname(self, sname):
 455                 ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)
 456
 457                 if ret:
 458                         return Release(self.backend, ret.id, data=ret)
 459
 460         def get_by_news_id(self, news_id):
 461                 ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)
 462
 463                 if ret:
 464                         return Release(self.backend, ret.id, data=ret)
 465
 466         def get_latest(self, stable=True):
 467                 ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
 468                         AND stable = %s ORDER BY published DESC LIMIT 1", stable)
 469
 470                 if ret:
 471                         return Release(self.backend, ret.id, data=ret)
 472
 473         def get_releases_older_than(self, release, limit=None):
 474                 return self._get_releases("SELECT * FROM releases \
 475                         WHERE published IS NOT NULL AND published < %s \
 476                                 ORDER BY published DESC LIMIT %s", release.published, limit)
 477
 478         def get_latest_unstable(self):
 479                 ret = self.db.get("SELECT * FROM releases r1 \
 480                         WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
 481                         AND stable = %s AND NOT EXISTS ( \
 482                                 SELECT * FROM releases r2 WHERE r2.stable = %s AND \
 483                                         r2.published IS NOT NULL AND r2.published >= r1.published \
 484                                 ) ORDER BY r1.published DESC LIMIT 1", False, True)
 485
 486                 if ret:
 487                         return Release(self.backend, ret.id, data=ret)
 488
 489         def get_stable(self):
 490                 query = self.db.query("SELECT * FROM releases \
 491                         WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
 492                         ORDER BY published DESC")
 493
 494                 releases = []
 495                 for row in query:
 496                         release = Release(self.backend, row.id, data=row)
 497                         releases.append(release)
 498
 499                 return releases
 500
 501         def get_unstable(self):
 502                 query = self.db.query("SELECT * FROM releases \
 503                         WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
 504                         ORDER BY published DESC")
 505
 506                 releases = []
 507                 for row in query:
 508                         release = Release(self.backend, row.id, data=row)
 509                         releases.append(release)
 510
 511                 return releases
 512
 513         def get_all(self):
 514                 query = self.db.query("SELECT * FROM releases \
 515                         WHERE published IS NOT NULL AND published <= NOW() \
 516                         ORDER BY published DESC")
 517
 518                 releases = []
 519                 for row in query:
 520                         release = Release(self.backend, row.id, data=row)
 521                         releases.append(release)
 522
 523                 return releases
 524
 525         def _get_all(self):
 526                 query = self.db.query("SELECT * FROM releases ORDER BY published DESC")
 527
 528                 releases = []
 529                 for row in query:
 530                         release = Release(self.backend, row.id, data=row)
 531                         releases.append(release)
 532
 533                 return releases
 534
 535         def get_file_for_torrent_hash(self, torrent_hash):
 536                 file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
 537                         torrent_hash)
 538
 539                 if not file:
 540                         return
 541
 542                 release = Release(self.backend, file.releases)
 543                 file = File(self.backend, release, file.id)
 544
 545                 return file
 546
 547         @tornado.gen.coroutine
 548         def scan_files(self, basepath="/pub/mirror"):
 549                 for release in self:
 550                         logging.info("Scanning %s..." % release)
 551
 552                         with self.db.transaction():
 553                                 release.scan_files(basepath=basepath)