src/backend/releases.py

   1 #!/usr/bin/python
   2
   3 import hashlib
   4 import logging
   5 import os
   6 import re
   7 import urllib.parse
   8
   9 from . import database
  10 from . import tracker
  11 from .misc import Object
  12
  13 TRACKERS = (
  14     "http://ipv4.tracker.ipfire.org:6969/announce",
  15     "udp://ipv4.tracker.ipfire.org:6969",
  16     "http://ipv6.tracker.ipfire.org:6969/announce",
  17     "udp://ipv6.tracker.ipfire.org:6969",
  18 )
  19
  20 class File(Object):
  21         def __init__(self, backend, release, id, data=None):
  22                 Object.__init__(self, backend)
  23
  24                 self.id = id
  25                 self._release = release
  26
  27                 # get all data from database
  28                 self.__data = data
  29
  30         def __eq__(self, other):
  31                 if isinstance(other, self.__class__):
  32                         return self.id == otherid
  33
  34         def __lt__(self, other):
  35                 if isinstance(other, self.__class__):
  36                         return self.prio < other.prio
  37
  38         @property
  39         def data(self):
  40                 if self.__data is None:
  41                         self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
  42                         assert self.__data
  43
  44                 return self.__data
  45
  46         @property
  47         def release(self):
  48                 if not self._release:
  49                         release_id = self.data.get("releases")
  50                         self._release = Release(release_id)
  51
  52                 return self._release
  53
  54         @property
  55         def type(self):
  56                 filename = self.filename
  57
  58                 if filename.endswith(".iso"):
  59                         return "iso"
  60
  61                 elif filename.endswith(".torrent"):
  62                         return "torrent"
  63
  64                 elif "xen" in filename:
  65                         if "downloader" in filename:
  66                                 return "xen-downloader"
  67
  68                         return "xen"
  69
  70                 elif "sources" in filename:
  71                         return "source"
  72
  73                 elif "usb-fdd" in filename:
  74                         return "usbfdd"
  75
  76                 elif "usb-hdd" in filename:
  77                         return "usbhdd"
  78
  79                 elif "armv5tel" in filename and "scon" in filename:
  80                         return "armv5tel-scon"
  81
  82                 elif "armv5tel" in filename:
  83                         return "armv5tel"
  84
  85                 elif "scon" in filename:
  86                         return "alix"
  87
  88                 elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
  89                         return "flash"
  90
  91                 else:
  92                         return "unknown"
  93
  94         @property
  95         def url(self):
  96                 return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)
  97
  98         @property
  99         def desc(self):
 100                 _ = lambda x: x
 101
 102                 descriptions = {
 103                         "armv5tel"      : _("Flash Image"),
 104                         "armv5tel-scon" : _("Flash Image with serial console"),
 105                         "iso"           : _("ISO Image"),
 106                         "torrent"       : _("Torrent File"),
 107                         "flash"         : _("Flash Image"),
 108                         "alix"          : _("Flash Image with serial console"),
 109                         "usbfdd"        : _("USB FDD Image"),
 110                         "usbhdd"        : _("USB HDD Image"),
 111                         "xen"           : _("Pre-generated Xen Image"),
 112                         "xen-downloader": _("Xen-Image Generator"),
 113                 }
 114
 115                 try:
 116                         return descriptions[self.type]
 117                 except KeyError:
 118                         return _("Unknown image type")
 119
 120         @property
 121         def prio(self):
 122                 priorities = {
 123                         "iso"           : 10,
 124                         "torrent"       : 20,
 125                         "flash"         : 40,
 126                         "alix"          : 41,
 127                         "usbfdd"        : 31,
 128                         "usbhdd"        : 30,
 129                         "armv5tel"      : 40,
 130                         "armv5tel-scon" : 41,
 131                         "xen"           : 50,
 132                         "xen-downloader": 51,
 133                 }
 134
 135                 try:
 136                         return priorities[self.type]
 137                 except KeyError:
 138                         return 999
 139
 140         @property
 141         def sha1(self):
 142                 return self.data.get("sha1")
 143
 144         @property
 145         def filename(self):
 146                 return self.data.get("filename")
 147
 148         @property
 149         def basename(self):
 150                 return os.path.basename(self.filename)
 151
 152         @property
 153         def size(self):
 154                 return self.data.get("filesize")
 155
 156         @property
 157         def arch(self):
 158                 known_arches = ("x86_64", "i586", "arm")
 159
 160                 for arch in known_arches:
 161                         if arch in self.basename:
 162                                 return arch
 163
 164                 return "N/A"
 165
 166         @property
 167         def torrent_hash(self):
 168                 return self.data.get("torrent_hash", None)
 169
 170         @property
 171         def torrent_url(self):
 172                 if self.torrent_hash:
 173                         return "%s.torrent" % self.url
 174
 175         @property
 176         def magnet_link(self):
 177                 # Don't return anything if we have no torrent hash.
 178                 if self.torrent_hash is None:
 179                         return
 180
 181                 s = "magnet:?xt=urn:btih:%s" % self.torrent_hash
 182
 183                 #s += "&xl=%d" % self.size
 184                 s += "&dn=%s" % urllib.parse.quote(self.basename)
 185
 186                 # Add our tracker.
 187                 for tracker in TRACKERS:
 188                         s += "&tr=%s" % tracker
 189
 190                 # Add web download URL
 191                 s += "&as=%s" % urllib.parse.quote(self.url)
 192
 193                 return s
 194
 195
 196 class Release(Object):
 197         def __init__(self, backend, id, data=None):
 198                 Object.__init__(self, backend)
 199                 self.id = id
 200
 201                 # get all data from database
 202                 self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
 203                 assert self.__data
 204
 205                 self.__files = []
 206
 207         def __str__(self):
 208                 return self.name
 209
 210         def __repr__(self):
 211                 return "<%s %s>" % (self.__class__.__name__, self.name)
 212
 213         def __cmp__(self, other):
 214                 return cmp(self.id, other.id)
 215
 216         @property
 217         def arches(self):
 218                 for arch in ("x86_64", "aarch64", "i586", "arm"):
 219                         if arch in (f.arch for f in self.files):
 220                                 yield arch
 221
 222         @property
 223         def files(self):
 224                 if not self.__files:
 225                         files = self.db.query("SELECT * FROM files WHERE releases = %s \
 226                                 AND NOT filename LIKE '%%.torrent'", self.id)
 227
 228                         self.__files = [File(self.backend, self, f.id, f) for f in files]
 229                         self.__files.sort()
 230
 231                 return self.__files
 232
 233         def get_files_by_arch(self, arch):
 234                 for f in self.files:
 235                         if f.arch == arch:
 236                                 yield f
 237
 238         @property
 239         def torrents(self):
 240                 torrents = []
 241
 242                 for file in self.files:
 243                         if not file.torrent_hash:
 244                                 continue
 245
 246                         torrents.append(file)
 247
 248                 return torrents
 249
 250         @property
 251         def name(self):
 252                 return self.__data.name
 253
 254         @property
 255         def slug(self):
 256                 return self.__data.sname
 257
 258         # XXX compat
 259         sname = slug
 260
 261         # XXX cache this
 262         @property
 263         def blog(self):
 264                 if self.__data.blog_id:
 265                         return self.backend.blog.get_by_id(self.__data.blog_id)
 266
 267         @property
 268         def fireinfo_id(self):
 269                 name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")
 270
 271                 res = self.db.get("SELECT id FROM fireinfo_releases \
 272                         WHERE name = %s", name)
 273
 274                 if res:
 275                         return res.id
 276
 277         @property
 278         def stable(self):
 279                 return self.__data.stable
 280
 281         @property
 282         def published(self):
 283                 return self.__data.published
 284
 285         date = published
 286
 287         @property
 288         def path(self):
 289                 return self.__data.path
 290
 291         def get_file(self, type):
 292                 for file in self.files:
 293                         if file.type == type:
 294                                 return file
 295
 296         def __file_hash(self, filename):
 297                 sha1 = hashlib.sha1()
 298
 299                 with open(filename) as f:
 300                         buf_size = 1024
 301                         buf = f.read(buf_size)
 302                         while buf:
 303                                 sha1.update(buf)
 304                                 buf = f.read(buf_size)
 305
 306                 return sha1.hexdigest()
 307
 308         def scan_files(self, basepath="/srv/mirror0"):
 309                 if not self.path:
 310                         return
 311
 312                 path = os.path.join(basepath, self.path)
 313                 if not os.path.exists(path):
 314                         return
 315
 316                 files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
 317                 files = [f.filename for f in files]
 318
 319                 # Make files that do not exists not loadable.
 320                 for filename in files:
 321                         _filename = os.path.join(basepath, filename)
 322                         if not os.path.exists(_filename):
 323                                 self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
 324
 325                 for filename in os.listdir(path):
 326                         filename = os.path.join(path, filename)
 327
 328                         if os.path.isdir(filename):
 329                                 continue
 330
 331                         _filename = re.match(".*(releases/.*)", filename).group(1)
 332                         if _filename in files:
 333                                 continue
 334
 335                         if filename.endswith(".md5"):
 336                                 continue
 337
 338                         logging.info("Hashing %s..." % filename)
 339                         filehash = self.__file_hash(filename)
 340                         filesize = os.path.getsize(filename)
 341
 342                         # Check if there is a torrent download available for this file:
 343                         torrent_hash = ""
 344                         torrent_file = "%s.torrent" % filename
 345                         if os.path.exists(torrent_file):
 346                                 torrent_hash = self.torrent_read_hash(torrent_file)
 347
 348                         self.db.execute("INSERT INTO files(releases, filename, filesize, \
 349                                 sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s)",
 350                                 self.id, _filename, filesize, filehash, torrent_hash)
 351
 352                 # Search for all files that miss a torrent hash.
 353                 files = self.db.query("SELECT id, filename FROM files \
 354                         WHERE releases = %s AND torrent_hash IS NULL", self.id)
 355
 356                 for file in files:
 357                         path = os.path.join(basepath, file.filename)
 358
 359                         torrent_file = "%s.torrent" % path
 360                         if os.path.exists(torrent_file):
 361                                 torrent_hash = self.torrent_read_hash(torrent_file)
 362
 363                                 self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
 364                                         torrent_hash, file.id)
 365
 366         def torrent_read_hash(self, filename):
 367                 f = None
 368                 try:
 369                         f = open(filename, "rb")
 370
 371                         metainfo = tracker.bdecode(f.read())
 372                         metainfo = tracker.bencode(metainfo["info"])
 373
 374                         hash = hashlib.sha1()
 375                         hash.update(metainfo)
 376
 377                         return hash.hexdigest()
 378
 379                 finally:
 380                         if f:
 381                                 f.close()
 382
 383         def supports_arch(self, arch):
 384                 return arch in ("x86_64", "i586")
 385
 386         def supports_platform(self, platform):
 387                 # Currently there is nothing else than pcbios supported
 388                 if platform == "pcbios":
 389                         return True
 390
 391                 return False
 392
 393         def is_netboot_capable(self):
 394                 return self.path and "ipfire-2.x" in self.path
 395
 396         def netboot_kernel_url(self, arch, platform):
 397                 assert self.supports_arch(arch)
 398                 assert self.supports_platform(platform)
 399
 400                 if self.sname >= "ipfire-2.19-core100":
 401                         return "https://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)
 402
 403                 return "https://boot.ipfire.org/%s/images/vmlinuz" % self.path
 404
 405         def netboot_initrd_url(self, arch, platform):
 406                 assert self.supports_arch(arch)
 407                 assert self.supports_platform(platform)
 408
 409                 if self.sname >= "ipfire-2.19-core100":
 410                         return "https://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch)
 411
 412                 return "https://boot.ipfire.org/%s/images/instroot" % self.path
 413
 414         def netboot_args(self, arch, platform):
 415                 return ""
 416
 417         @property
 418         def post(self):
 419                 if self.__data.blog_id:
 420                         return self.backend.blog.get_by_id(self.__data.blog_id)
 421
 422         # Fireinfo Stuff
 423
 424         @property
 425         def penetration(self):
 426                 # Get penetration from fireinfo
 427                 return self.backend.fireinfo.get_release_penetration(self)
 428
 429
 430 class Releases(Object):
 431         def _get_release(self, query, *args):
 432                 res = self.db.get(query, *args)
 433
 434                 if res:
 435                         return Release(self.backend, res.id, data=res)
 436
 437         def _get_releases(self, query, *args):
 438                 res = self.db.query(query, *args)
 439
 440                 for row in res:
 441                         yield Release(self.backend, row.id, data=row)
 442
 443         def get_by_id(self, id):
 444                 ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)
 445
 446                 if ret:
 447                         return Release(self.backend, ret.id, data=ret)
 448
 449         def get_by_sname(self, sname):
 450                 ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)
 451
 452                 if ret:
 453                         return Release(self.backend, ret.id, data=ret)
 454
 455         def get_by_news_id(self, news_id):
 456                 ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)
 457
 458                 if ret:
 459                         return Release(self.backend, ret.id, data=ret)
 460
 461         def get_latest(self, stable=True):
 462                 ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
 463                         AND stable = %s ORDER BY published DESC LIMIT 1", stable)
 464
 465                 if ret:
 466                         return Release(self.backend, ret.id, data=ret)
 467
 468         def get_releases_older_than(self, release, limit=None):
 469                 return self._get_releases("SELECT * FROM releases \
 470                         WHERE published IS NOT NULL AND published < %s \
 471                                 ORDER BY published DESC LIMIT %s", release.published, limit)
 472
 473         def get_latest_unstable(self):
 474                 ret = self.db.get("SELECT * FROM releases r1 \
 475                         WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
 476                         AND stable = %s AND NOT EXISTS ( \
 477                                 SELECT * FROM releases r2 WHERE r2.stable = %s AND \
 478                                         r2.published IS NOT NULL AND r2.published >= r1.published \
 479                                 ) ORDER BY r1.published DESC LIMIT 1", False, True)
 480
 481                 if ret:
 482                         return Release(self.backend, ret.id, data=ret)
 483
 484         def get_stable(self):
 485                 query = self.db.query("SELECT * FROM releases \
 486                         WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
 487                         ORDER BY published DESC")
 488
 489                 releases = []
 490                 for row in query:
 491                         release = Release(self.backend, row.id, data=row)
 492                         releases.append(release)
 493
 494                 return releases
 495
 496         def get_unstable(self):
 497                 query = self.db.query("SELECT * FROM releases \
 498                         WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
 499                         ORDER BY published DESC")
 500
 501                 releases = []
 502                 for row in query:
 503                         release = Release(self.backend, row.id, data=row)
 504                         releases.append(release)
 505
 506                 return releases
 507
 508         def get_all(self):
 509                 query = self.db.query("SELECT * FROM releases \
 510                         WHERE published IS NOT NULL AND published <= NOW() \
 511                         ORDER BY published DESC")
 512
 513                 releases = []
 514                 for row in query:
 515                         release = Release(self.backend, row.id, data=row)
 516                         releases.append(release)
 517
 518                 return releases
 519
 520         def _get_all(self):
 521                 query = self.db.query("SELECT * FROM releases ORDER BY published DESC")
 522
 523                 releases = []
 524                 for row in query:
 525                         release = Release(self.backend, row.id, data=row)
 526                         releases.append(release)
 527
 528                 return releases
 529
 530         def get_file_for_torrent_hash(self, torrent_hash):
 531                 file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
 532                         torrent_hash)
 533
 534                 if not file:
 535                         return
 536
 537                 release = Release(self.backend, file.releases)
 538                 file = File(self.backend, release, file.id)
 539
 540                 return file