src/backend/releases.py

   1 #!/usr/bin/python
   2
   3 import hashlib
   4 import logging
   5 import os
   6 import re
   7 import urllib.parse
   8 import yabencode
   9
  10 from . import database
  11 from .misc import Object
  12 from .decorators import *
  13
  14 TRACKERS = (
  15     "http://ipv4.tracker.ipfire.org:6969/announce",
  16     "udp://ipv4.tracker.ipfire.org:6969",
  17     "http://ipv6.tracker.ipfire.org:6969/announce",
  18     "udp://ipv6.tracker.ipfire.org:6969",
  19 )
  20
  21 class File(Object):
  22         def __init__(self, backend, release, id, data=None):
  23                 Object.__init__(self, backend)
  24
  25                 self.id = id
  26                 self._release = release
  27
  28                 # get all data from database
  29                 self.__data = data
  30
  31         def __eq__(self, other):
  32                 if isinstance(other, self.__class__):
  33                         return self.id == otherid
  34
  35         def __lt__(self, other):
  36                 if isinstance(other, self.__class__):
  37                         return self.prio < other.prio
  38
  39         @property
  40         def data(self):
  41                 if self.__data is None:
  42                         self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
  43                         assert self.__data
  44
  45                 return self.__data
  46
  47         @property
  48         def release(self):
  49                 if not self._release:
  50                         release_id = self.data.get("releases")
  51                         self._release = Release(release_id)
  52
  53                 return self._release
  54
  55         @property
  56         def type(self):
  57                 filename = self.filename
  58
  59                 if filename.endswith(".iso"):
  60                         return "iso"
  61
  62                 elif filename.endswith(".torrent"):
  63                         return "torrent"
  64
  65                 elif "xen" in filename:
  66                         if "downloader" in filename:
  67                                 return "xen-downloader"
  68
  69                         return "xen"
  70
  71                 elif "sources" in filename:
  72                         return "source"
  73
  74                 elif "usb-fdd" in filename:
  75                         return "usbfdd"
  76
  77                 elif "usb-hdd" in filename:
  78                         return "usbhdd"
  79
  80                 elif "armv5tel" in filename and "scon" in filename:
  81                         return "armv5tel-scon"
  82
  83                 elif "armv5tel" in filename:
  84                         return "armv5tel"
  85
  86                 elif "scon" in filename:
  87                         return "alix"
  88
  89                 elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
  90                         return "flash"
  91
  92                 else:
  93                         return "unknown"
  94
  95         @property
  96         def url(self):
  97                 return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)
  98
  99         @property
 100         def desc(self):
 101                 _ = lambda x: x
 102
 103                 descriptions = {
 104                         "armv5tel"      : _("Flash Image"),
 105                         "armv5tel-scon" : _("Flash Image with serial console"),
 106                         "iso"           : _("ISO Image"),
 107                         "torrent"       : _("Torrent File"),
 108                         "flash"         : _("Flash Image"),
 109                         "alix"          : _("Flash Image with serial console"),
 110                         "usbfdd"        : _("USB FDD Image"),
 111                         "usbhdd"        : _("USB HDD Image"),
 112                         "xen"           : _("Pre-generated Xen Image"),
 113                         "xen-downloader": _("Xen-Image Generator"),
 114                 }
 115
 116                 try:
 117                         return descriptions[self.type]
 118                 except KeyError:
 119                         return _("Unknown image type")
 120
 121         @property
 122         def prio(self):
 123                 priorities = {
 124                         "iso"           : 10,
 125                         "torrent"       : 20,
 126                         "flash"         : 40,
 127                         "alix"          : 41,
 128                         "usbfdd"        : 31,
 129                         "usbhdd"        : 30,
 130                         "armv5tel"      : 40,
 131                         "armv5tel-scon" : 41,
 132                         "xen"           : 50,
 133                         "xen-downloader": 51,
 134                 }
 135
 136                 try:
 137                         return priorities[self.type]
 138                 except KeyError:
 139                         return 999
 140
 141         @property
 142         def sha256(self):
 143                 return self.data.get("sha256")
 144
 145         @property
 146         def sha1(self):
 147                 return self.data.get("sha1")
 148
 149         @property
 150         def filename(self):
 151                 return self.data.get("filename")
 152
 153         @property
 154         def basename(self):
 155                 return os.path.basename(self.filename)
 156
 157         @property
 158         def size(self):
 159                 return self.data.get("filesize")
 160
 161         @property
 162         def arch(self):
 163                 known_arches = ("x86_64", "i586", "arm")
 164
 165                 for arch in known_arches:
 166                         if arch in self.basename:
 167                                 return arch
 168
 169                 return "N/A"
 170
 171         @property
 172         def torrent_hash(self):
 173                 return self.data.get("torrent_hash", None)
 174
 175         @property
 176         def torrent_url(self):
 177                 if self.torrent_hash:
 178                         return "%s.torrent" % self.url
 179
 180         @property
 181         def magnet_link(self):
 182                 # Don't return anything if we have no torrent hash.
 183                 if self.torrent_hash is None:
 184                         return
 185
 186                 s = "magnet:?xt=urn:btih:%s" % self.torrent_hash
 187
 188                 #s += "&xl=%d" % self.size
 189                 s += "&dn=%s" % urllib.parse.quote(self.basename)
 190
 191                 # Add our tracker.
 192                 for tracker in TRACKERS:
 193                         s += "&tr=%s" % tracker
 194
 195                 # Add web download URL
 196                 s += "&as=%s" % urllib.parse.quote(self.url)
 197
 198                 return s
 199
 200
 201 class Release(Object):
 202         def __init__(self, backend, id, data=None):
 203                 Object.__init__(self, backend)
 204                 self.id = id
 205
 206                 # get all data from database
 207                 self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
 208                 assert self.__data
 209
 210                 self.__files = []
 211
 212         def __str__(self):
 213                 return self.name
 214
 215         def __repr__(self):
 216                 return "<%s %s>" % (self.__class__.__name__, self.name)
 217
 218         def __cmp__(self, other):
 219                 return cmp(self.id, other.id)
 220
 221         @property
 222         def arches(self):
 223                 for arch in ("x86_64", "aarch64", "i586", "arm"):
 224                         if arch in (f.arch for f in self.files):
 225                                 yield arch
 226
 227         @property
 228         def files(self):
 229                 if not self.__files:
 230                         files = self.db.query("SELECT * FROM files WHERE releases = %s \
 231                                 AND NOT filename LIKE '%%.torrent'", self.id)
 232
 233                         self.__files = [File(self.backend, self, f.id, f) for f in files]
 234                         self.__files.sort()
 235
 236                 return self.__files
 237
 238         def get_files_by_arch(self, arch):
 239                 for f in self.files:
 240                         if f.arch == arch:
 241                                 yield f
 242
 243         @property
 244         def torrents(self):
 245                 torrents = []
 246
 247                 for file in self.files:
 248                         if not file.torrent_hash:
 249                                 continue
 250
 251                         torrents.append(file)
 252
 253                 return torrents
 254
 255         @property
 256         def name(self):
 257                 return self.__data.name
 258
 259         @property
 260         def slug(self):
 261                 return self.__data.sname
 262
 263         # XXX compat
 264         sname = slug
 265
 266         @lazy_property
 267         def blog(self):
 268                 if self.__data.blog_id:
 269                         return self.backend.blog.get_by_id(self.__data.blog_id)
 270
 271         @property
 272         def fireinfo_id(self):
 273                 name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")
 274
 275                 res = self.db.get("SELECT id FROM fireinfo_releases \
 276                         WHERE name = %s", name)
 277
 278                 if res:
 279                         return res.id
 280
 281         @property
 282         def stable(self):
 283                 return self.__data.stable
 284
 285         @property
 286         def published(self):
 287                 return self.__data.published
 288
 289         date = published
 290
 291         @property
 292         def path(self):
 293                 return self.__data.path
 294
 295         def get_file(self, type):
 296                 for file in self.files:
 297                         if file.type == type:
 298                                 return file
 299
 300         def __file_hash(self, filename, algo="sha256"):
 301                 h = hashlib.new(algo)
 302
 303                 with open(filename, "rb") as f:
 304                         buf_size = 1024
 305                         buf = f.read(buf_size)
 306                         while buf:
 307                                 h.update(buf)
 308                                 buf = f.read(buf_size)
 309
 310                 return h.hexdigest()
 311
 312         def scan_files(self, basepath="/pub/mirror"):
 313                 if not self.path:
 314                         return
 315
 316                 path = os.path.join(basepath, self.path)
 317                 if not os.path.exists(path):
 318                         return
 319
 320                 files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
 321                 files = [f.filename for f in files]
 322
 323                 # Make files that do not exists not loadable.
 324                 for filename in files:
 325                         _filename = os.path.join(basepath, filename)
 326                         if not os.path.exists(_filename):
 327                                 self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
 328
 329                 for filename in os.listdir(path):
 330                         filename = os.path.join(path, filename)
 331
 332                         if os.path.isdir(filename):
 333                                 continue
 334
 335                         _filename = re.match(".*(releases/.*)", filename).group(1)
 336                         if _filename in files:
 337                                 continue
 338
 339                         if filename.endswith(".md5"):
 340                                 continue
 341
 342                         logging.info("Hashing %s..." % filename)
 343                         hash_sha256 = self.__file_hash(filename, "sha256")
 344                         hash_sha1   = self.__file_hash(filename, "sha1")
 345                         filesize = os.path.getsize(filename)
 346
 347                         # Check if there is a torrent download available for this file:
 348                         torrent_hash = ""
 349                         torrent_file = "%s.torrent" % filename
 350                         if os.path.exists(torrent_file):
 351                                 torrent_hash = self.torrent_read_hash(torrent_file)
 352
 353                         self.db.execute("INSERT INTO files(releases, filename, filesize, \
 354                                 sha256, sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s, %s)",
 355                                 self.id, _filename, filesize, hash_sha256, hash_sha1, torrent_hash)
 356
 357                 # Search for all files that miss a torrent hash.
 358                 files = self.db.query("SELECT id, filename FROM files \
 359                         WHERE releases = %s AND torrent_hash IS NULL", self.id)
 360
 361                 for file in files:
 362                         path = os.path.join(basepath, file.filename)
 363
 364                         torrent_file = "%s.torrent" % path
 365                         if os.path.exists(torrent_file):
 366                                 torrent_hash = self.torrent_read_hash(torrent_file)
 367
 368                                 self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
 369                                         torrent_hash, file.id)
 370
 371         def torrent_read_hash(self, filename):
 372                 with open(filename, "rb") as f:
 373                         metainfo = yabencode.decode(f)
 374                         metainfo = yabencode.encode(metainfo["info"])
 375
 376                         h = hashlib.new("sha1")
 377                         h.update(metainfo)
 378
 379                         return h.hexdigest()
 380
 381         def supports_arch(self, arch):
 382                 return arch in ("x86_64", "i586")
 383
 384         def supports_platform(self, platform):
 385                 # Currently there is nothing else than pcbios supported
 386                 if platform == "pcbios":
 387                         return True
 388
 389                 return False
 390
 391         def is_netboot_capable(self):
 392                 return self.path and "ipfire-2.x" in self.path
 393
 394         def netboot_kernel_url(self, arch, platform):
 395                 assert self.supports_arch(arch)
 396                 assert self.supports_platform(platform)
 397
 398                 if self.sname >= "ipfire-2.19-core100":
 399                         return "http://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)
 400
 401                 return "http://boot.ipfire.org/%s/images/vmlinuz" % self.path
 402
 403         def netboot_initrd_url(self, arch, platform):
 404                 assert self.supports_arch(arch)
 405                 assert self.supports_platform(platform)
 406
 407                 if self.sname >= "ipfire-2.19-core100":
 408                         return "http://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch)
 409
 410                 return "http://boot.ipfire.org/%s/images/instroot" % self.path
 411
 412         def netboot_args(self, arch, platform):
 413                 return ""
 414
 415         @property
 416         def post(self):
 417                 if self.__data.blog_id:
 418                         return self.backend.blog.get_by_id(self.__data.blog_id)
 419
 420         # Fireinfo Stuff
 421
 422         @property
 423         def penetration(self):
 424                 # Get penetration from fireinfo
 425                 return self.backend.fireinfo.get_release_penetration(self)
 426
 427
 428 class Releases(Object):
 429         def _get_release(self, query, *args):
 430                 res = self.db.get(query, *args)
 431
 432                 if res:
 433                         return Release(self.backend, res.id, data=res)
 434
 435         def _get_releases(self, query, *args):
 436                 res = self.db.query(query, *args)
 437
 438                 for row in res:
 439                         yield Release(self.backend, row.id, data=row)
 440
 441         def __iter__(self):
 442                 releases = self._get_releases("SELECT * FROM releases \
 443                         ORDER BY published DESC NULLS FIRST")
 444
 445                 return iter(releases)
 446
 447         def get_by_id(self, id):
 448                 ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)
 449
 450                 if ret:
 451                         return Release(self.backend, ret.id, data=ret)
 452
 453         def get_by_sname(self, sname):
 454                 ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)
 455
 456                 if ret:
 457                         return Release(self.backend, ret.id, data=ret)
 458
 459         def get_by_news_id(self, news_id):
 460                 ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)
 461
 462                 if ret:
 463                         return Release(self.backend, ret.id, data=ret)
 464
 465         def get_latest(self, stable=True):
 466                 ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
 467                         AND stable = %s ORDER BY published DESC LIMIT 1", stable)
 468
 469                 if ret:
 470                         return Release(self.backend, ret.id, data=ret)
 471
 472         def get_releases_older_than(self, release, limit=None):
 473                 return self._get_releases("SELECT * FROM releases \
 474                         WHERE published IS NOT NULL AND published < %s \
 475                                 ORDER BY published DESC LIMIT %s", release.published, limit)
 476
 477         def get_latest_unstable(self):
 478                 ret = self.db.get("SELECT * FROM releases r1 \
 479                         WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
 480                         AND stable = %s AND NOT EXISTS ( \
 481                                 SELECT * FROM releases r2 WHERE r2.stable = %s AND \
 482                                         r2.published IS NOT NULL AND r2.published >= r1.published \
 483                                 ) ORDER BY r1.published DESC LIMIT 1", False, True)
 484
 485                 if ret:
 486                         return Release(self.backend, ret.id, data=ret)
 487
 488         def get_stable(self):
 489                 query = self.db.query("SELECT * FROM releases \
 490                         WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
 491                         ORDER BY published DESC")
 492
 493                 releases = []
 494                 for row in query:
 495                         release = Release(self.backend, row.id, data=row)
 496                         releases.append(release)
 497
 498                 return releases
 499
 500         def get_unstable(self):
 501                 query = self.db.query("SELECT * FROM releases \
 502                         WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
 503                         ORDER BY published DESC")
 504
 505                 releases = []
 506                 for row in query:
 507                         release = Release(self.backend, row.id, data=row)
 508                         releases.append(release)
 509
 510                 return releases
 511
 512         def get_all(self):
 513                 query = self.db.query("SELECT * FROM releases \
 514                         WHERE published IS NOT NULL AND published <= NOW() \
 515                         ORDER BY published DESC")
 516
 517                 releases = []
 518                 for row in query:
 519                         release = Release(self.backend, row.id, data=row)
 520                         releases.append(release)
 521
 522                 return releases
 523
 524         def _get_all(self):
 525                 query = self.db.query("SELECT * FROM releases ORDER BY published DESC")
 526
 527                 releases = []
 528                 for row in query:
 529                         release = Release(self.backend, row.id, data=row)
 530                         releases.append(release)
 531
 532                 return releases
 533
 534         def get_file_for_torrent_hash(self, torrent_hash):
 535                 file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
 536                         torrent_hash)
 537
 538                 if not file:
 539                         return
 540
 541                 release = Release(self.backend, file.releases)
 542                 file = File(self.backend, release, file.id)
 543
 544                 return file
 545
 546         async def scan_files(self, basepath="/pub/mirror"):
 547                 for release in self:
 548                         logging.debug("Scanning %s..." % release)
 549
 550                         with self.db.transaction():
 551                                 release.scan_files(basepath=basepath)