src/backend/releases.py

   1 #!/usr/bin/python
   2
   3 import hashlib
   4 import logging
   5 import os
   6 import re
   7 import tornado.gen
   8 import urllib.parse
   9
  10 from . import database
  11 from . import tracker
  12 from .misc import Object
  13
  14 TRACKERS = (
  15     "http://ipv4.tracker.ipfire.org:6969/announce",
  16     "udp://ipv4.tracker.ipfire.org:6969",
  17     "http://ipv6.tracker.ipfire.org:6969/announce",
  18     "udp://ipv6.tracker.ipfire.org:6969",
  19 )
  20
  21 class File(Object):
  22         def __init__(self, backend, release, id, data=None):
  23                 Object.__init__(self, backend)
  24
  25                 self.id = id
  26                 self._release = release
  27
  28                 # get all data from database
  29                 self.__data = data
  30
  31         def __eq__(self, other):
  32                 if isinstance(other, self.__class__):
  33                         return self.id == otherid
  34
  35         def __lt__(self, other):
  36                 if isinstance(other, self.__class__):
  37                         return self.prio < other.prio
  38
  39         @property
  40         def data(self):
  41                 if self.__data is None:
  42                         self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
  43                         assert self.__data
  44
  45                 return self.__data
  46
  47         @property
  48         def release(self):
  49                 if not self._release:
  50                         release_id = self.data.get("releases")
  51                         self._release = Release(release_id)
  52
  53                 return self._release
  54
  55         @property
  56         def type(self):
  57                 filename = self.filename
  58
  59                 if filename.endswith(".iso"):
  60                         return "iso"
  61
  62                 elif filename.endswith(".torrent"):
  63                         return "torrent"
  64
  65                 elif "xen" in filename:
  66                         if "downloader" in filename:
  67                                 return "xen-downloader"
  68
  69                         return "xen"
  70
  71                 elif "sources" in filename:
  72                         return "source"
  73
  74                 elif "usb-fdd" in filename:
  75                         return "usbfdd"
  76
  77                 elif "usb-hdd" in filename:
  78                         return "usbhdd"
  79
  80                 elif "armv5tel" in filename and "scon" in filename:
  81                         return "armv5tel-scon"
  82
  83                 elif "armv5tel" in filename:
  84                         return "armv5tel"
  85
  86                 elif "scon" in filename:
  87                         return "alix"
  88
  89                 elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
  90                         return "flash"
  91
  92                 else:
  93                         return "unknown"
  94
  95         @property
  96         def url(self):
  97                 return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)
  98
  99         @property
 100         def desc(self):
 101                 _ = lambda x: x
 102
 103                 descriptions = {
 104                         "armv5tel"      : _("Flash Image"),
 105                         "armv5tel-scon" : _("Flash Image with serial console"),
 106                         "iso"           : _("ISO Image"),
 107                         "torrent"       : _("Torrent File"),
 108                         "flash"         : _("Flash Image"),
 109                         "alix"          : _("Flash Image with serial console"),
 110                         "usbfdd"        : _("USB FDD Image"),
 111                         "usbhdd"        : _("USB HDD Image"),
 112                         "xen"           : _("Pre-generated Xen Image"),
 113                         "xen-downloader": _("Xen-Image Generator"),
 114                 }
 115
 116                 try:
 117                         return descriptions[self.type]
 118                 except KeyError:
 119                         return _("Unknown image type")
 120
 121         @property
 122         def prio(self):
 123                 priorities = {
 124                         "iso"           : 10,
 125                         "torrent"       : 20,
 126                         "flash"         : 40,
 127                         "alix"          : 41,
 128                         "usbfdd"        : 31,
 129                         "usbhdd"        : 30,
 130                         "armv5tel"      : 40,
 131                         "armv5tel-scon" : 41,
 132                         "xen"           : 50,
 133                         "xen-downloader": 51,
 134                 }
 135
 136                 try:
 137                         return priorities[self.type]
 138                 except KeyError:
 139                         return 999
 140
 141         @property
 142         def sha1(self):
 143                 return self.data.get("sha1")
 144
 145         @property
 146         def filename(self):
 147                 return self.data.get("filename")
 148
 149         @property
 150         def basename(self):
 151                 return os.path.basename(self.filename)
 152
 153         @property
 154         def size(self):
 155                 return self.data.get("filesize")
 156
 157         @property
 158         def arch(self):
 159                 known_arches = ("x86_64", "i586", "arm")
 160
 161                 for arch in known_arches:
 162                         if arch in self.basename:
 163                                 return arch
 164
 165                 return "N/A"
 166
 167         @property
 168         def torrent_hash(self):
 169                 return self.data.get("torrent_hash", None)
 170
 171         @property
 172         def torrent_url(self):
 173                 if self.torrent_hash:
 174                         return "%s.torrent" % self.url
 175
 176         @property
 177         def magnet_link(self):
 178                 # Don't return anything if we have no torrent hash.
 179                 if self.torrent_hash is None:
 180                         return
 181
 182                 s = "magnet:?xt=urn:btih:%s" % self.torrent_hash
 183
 184                 #s += "&xl=%d" % self.size
 185                 s += "&dn=%s" % urllib.parse.quote(self.basename)
 186
 187                 # Add our tracker.
 188                 for tracker in TRACKERS:
 189                         s += "&tr=%s" % tracker
 190
 191                 # Add web download URL
 192                 s += "&as=%s" % urllib.parse.quote(self.url)
 193
 194                 return s
 195
 196
 197 class Release(Object):
 198         def __init__(self, backend, id, data=None):
 199                 Object.__init__(self, backend)
 200                 self.id = id
 201
 202                 # get all data from database
 203                 self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
 204                 assert self.__data
 205
 206                 self.__files = []
 207
 208         def __str__(self):
 209                 return self.name
 210
 211         def __repr__(self):
 212                 return "<%s %s>" % (self.__class__.__name__, self.name)
 213
 214         def __cmp__(self, other):
 215                 return cmp(self.id, other.id)
 216
 217         @property
 218         def arches(self):
 219                 for arch in ("x86_64", "aarch64", "i586", "arm"):
 220                         if arch in (f.arch for f in self.files):
 221                                 yield arch
 222
 223         @property
 224         def files(self):
 225                 if not self.__files:
 226                         files = self.db.query("SELECT * FROM files WHERE releases = %s \
 227                                 AND NOT filename LIKE '%%.torrent'", self.id)
 228
 229                         self.__files = [File(self.backend, self, f.id, f) for f in files]
 230                         self.__files.sort()
 231
 232                 return self.__files
 233
 234         def get_files_by_arch(self, arch):
 235                 for f in self.files:
 236                         if f.arch == arch:
 237                                 yield f
 238
 239         @property
 240         def torrents(self):
 241                 torrents = []
 242
 243                 for file in self.files:
 244                         if not file.torrent_hash:
 245                                 continue
 246
 247                         torrents.append(file)
 248
 249                 return torrents
 250
 251         @property
 252         def name(self):
 253                 return self.__data.name
 254
 255         @property
 256         def slug(self):
 257                 return self.__data.sname
 258
 259         # XXX compat
 260         sname = slug
 261
 262         # XXX cache this
 263         @property
 264         def blog(self):
 265                 if self.__data.blog_id:
 266                         return self.backend.blog.get_by_id(self.__data.blog_id)
 267
 268         @property
 269         def fireinfo_id(self):
 270                 name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")
 271
 272                 res = self.db.get("SELECT id FROM fireinfo_releases \
 273                         WHERE name = %s", name)
 274
 275                 if res:
 276                         return res.id
 277
 278         @property
 279         def stable(self):
 280                 return self.__data.stable
 281
 282         @property
 283         def published(self):
 284                 return self.__data.published
 285
 286         date = published
 287
 288         @property
 289         def path(self):
 290                 return self.__data.path
 291
 292         def get_file(self, type):
 293                 for file in self.files:
 294                         if file.type == type:
 295                                 return file
 296
 297         def __file_hash(self, filename, algo="sha256"):
 298                 h = hashlib.new(algo)
 299
 300                 with open(filename, "rb") as f:
 301                         buf_size = 1024
 302                         buf = f.read(buf_size)
 303                         while buf:
 304                                 h.update(buf)
 305                                 buf = f.read(buf_size)
 306
 307                 return h.hexdigest()
 308
 309         def scan_files(self, basepath="/pub/mirror"):
 310                 if not self.path:
 311                         return
 312
 313                 path = os.path.join(basepath, self.path)
 314                 if not os.path.exists(path):
 315                         return
 316
 317                 files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
 318                 files = [f.filename for f in files]
 319
 320                 # Make files that do not exists not loadable.
 321                 for filename in files:
 322                         _filename = os.path.join(basepath, filename)
 323                         if not os.path.exists(_filename):
 324                                 self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
 325
 326                 for filename in os.listdir(path):
 327                         filename = os.path.join(path, filename)
 328
 329                         if os.path.isdir(filename):
 330                                 continue
 331
 332                         _filename = re.match(".*(releases/.*)", filename).group(1)
 333                         if _filename in files:
 334                                 continue
 335
 336                         if filename.endswith(".md5"):
 337                                 continue
 338
 339                         logging.info("Hashing %s..." % filename)
 340                         hash_sha256 = self.__file_hash(filename, "sha256")
 341                         hash_sha1   = self.__file_hash(filename, "sha1")
 342                         filesize = os.path.getsize(filename)
 343
 344                         # Check if there is a torrent download available for this file:
 345                         torrent_hash = ""
 346                         torrent_file = "%s.torrent" % filename
 347                         if os.path.exists(torrent_file):
 348                                 torrent_hash = self.torrent_read_hash(torrent_file)
 349
 350                         self.db.execute("INSERT INTO files(releases, filename, filesize, \
 351                                 sha256, sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s, %s)",
 352                                 self.id, _filename, filesize, hash_sha256, hash_sha1, torrent_hash)
 353
 354                 # Search for all files that miss a torrent hash.
 355                 files = self.db.query("SELECT id, filename FROM files \
 356                         WHERE releases = %s AND torrent_hash IS NULL", self.id)
 357
 358                 for file in files:
 359                         path = os.path.join(basepath, file.filename)
 360
 361                         torrent_file = "%s.torrent" % path
 362                         if os.path.exists(torrent_file):
 363                                 torrent_hash = self.torrent_read_hash(torrent_file)
 364
 365                                 self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
 366                                         torrent_hash, file.id)
 367
 368         def torrent_read_hash(self, filename):
 369                 f = None
 370                 try:
 371                         f = open(filename, "rb")
 372
 373                         metainfo = tracker.bdecode(f.read())
 374                         metainfo = tracker.bencode(metainfo["info"])
 375
 376                         hash = hashlib.sha1()
 377                         hash.update(metainfo)
 378
 379                         return hash.hexdigest()
 380
 381                 finally:
 382                         if f:
 383                                 f.close()
 384
 385         def supports_arch(self, arch):
 386                 return arch in ("x86_64", "i586")
 387
 388         def supports_platform(self, platform):
 389                 # Currently there is nothing else than pcbios supported
 390                 if platform == "pcbios":
 391                         return True
 392
 393                 return False
 394
 395         def is_netboot_capable(self):
 396                 return self.path and "ipfire-2.x" in self.path
 397
 398         def netboot_kernel_url(self, arch, platform):
 399                 assert self.supports_arch(arch)
 400                 assert self.supports_platform(platform)
 401
 402                 if self.sname >= "ipfire-2.19-core100":
 403                         return "https://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)
 404
 405                 return "https://boot.ipfire.org/%s/images/vmlinuz" % self.path
 406
 407         def netboot_initrd_url(self, arch, platform):
 408                 assert self.supports_arch(arch)
 409                 assert self.supports_platform(platform)
 410
 411                 if self.sname >= "ipfire-2.19-core100":
 412                         return "https://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch)
 413
 414                 return "https://boot.ipfire.org/%s/images/instroot" % self.path
 415
 416         def netboot_args(self, arch, platform):
 417                 return ""
 418
 419         @property
 420         def post(self):
 421                 if self.__data.blog_id:
 422                         return self.backend.blog.get_by_id(self.__data.blog_id)
 423
 424         # Fireinfo Stuff
 425
 426         @property
 427         def penetration(self):
 428                 # Get penetration from fireinfo
 429                 return self.backend.fireinfo.get_release_penetration(self)
 430
 431
 432 class Releases(Object):
 433         def _get_release(self, query, *args):
 434                 res = self.db.get(query, *args)
 435
 436                 if res:
 437                         return Release(self.backend, res.id, data=res)
 438
 439         def _get_releases(self, query, *args):
 440                 res = self.db.query(query, *args)
 441
 442                 for row in res:
 443                         yield Release(self.backend, row.id, data=row)
 444
 445         def __iter__(self):
 446                 releases = self._get_releases("SELECT * FROM releases \
 447                         ORDER BY published DESC NULLS FIRST")
 448
 449                 return iter(releases)
 450
 451         def get_by_id(self, id):
 452                 ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)
 453
 454                 if ret:
 455                         return Release(self.backend, ret.id, data=ret)
 456
 457         def get_by_sname(self, sname):
 458                 ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)
 459
 460                 if ret:
 461                         return Release(self.backend, ret.id, data=ret)
 462
 463         def get_by_news_id(self, news_id):
 464                 ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)
 465
 466                 if ret:
 467                         return Release(self.backend, ret.id, data=ret)
 468
 469         def get_latest(self, stable=True):
 470                 ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
 471                         AND stable = %s ORDER BY published DESC LIMIT 1", stable)
 472
 473                 if ret:
 474                         return Release(self.backend, ret.id, data=ret)
 475
 476         def get_releases_older_than(self, release, limit=None):
 477                 return self._get_releases("SELECT * FROM releases \
 478                         WHERE published IS NOT NULL AND published < %s \
 479                                 ORDER BY published DESC LIMIT %s", release.published, limit)
 480
 481         def get_latest_unstable(self):
 482                 ret = self.db.get("SELECT * FROM releases r1 \
 483                         WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
 484                         AND stable = %s AND NOT EXISTS ( \
 485                                 SELECT * FROM releases r2 WHERE r2.stable = %s AND \
 486                                         r2.published IS NOT NULL AND r2.published >= r1.published \
 487                                 ) ORDER BY r1.published DESC LIMIT 1", False, True)
 488
 489                 if ret:
 490                         return Release(self.backend, ret.id, data=ret)
 491
 492         def get_stable(self):
 493                 query = self.db.query("SELECT * FROM releases \
 494                         WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
 495                         ORDER BY published DESC")
 496
 497                 releases = []
 498                 for row in query:
 499                         release = Release(self.backend, row.id, data=row)
 500                         releases.append(release)
 501
 502                 return releases
 503
 504         def get_unstable(self):
 505                 query = self.db.query("SELECT * FROM releases \
 506                         WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
 507                         ORDER BY published DESC")
 508
 509                 releases = []
 510                 for row in query:
 511                         release = Release(self.backend, row.id, data=row)
 512                         releases.append(release)
 513
 514                 return releases
 515
 516         def get_all(self):
 517                 query = self.db.query("SELECT * FROM releases \
 518                         WHERE published IS NOT NULL AND published <= NOW() \
 519                         ORDER BY published DESC")
 520
 521                 releases = []
 522                 for row in query:
 523                         release = Release(self.backend, row.id, data=row)
 524                         releases.append(release)
 525
 526                 return releases
 527
 528         def _get_all(self):
 529                 query = self.db.query("SELECT * FROM releases ORDER BY published DESC")
 530
 531                 releases = []
 532                 for row in query:
 533                         release = Release(self.backend, row.id, data=row)
 534                         releases.append(release)
 535
 536                 return releases
 537
 538         def get_file_for_torrent_hash(self, torrent_hash):
 539                 file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
 540                         torrent_hash)
 541
 542                 if not file:
 543                         return
 544
 545                 release = Release(self.backend, file.releases)
 546                 file = File(self.backend, release, file.id)
 547
 548                 return file
 549
 550         @tornado.gen.coroutine
 551         def scan_files(self, basepath="/pub/mirror"):
 552                 for release in self:
 553                         logging.info("Scanning %s..." % release)
 554
 555                         with self.db.transaction():
 556                                 release.scan_files(basepath=basepath)