src/backend/releases.py

   1 #!/usr/bin/python
   2
   3 import hashlib
   4 import logging
   5 import os
   6 import re
   7 import urllib.parse
   8
   9 from . import database
  10 from .misc import Object
  11 from .decorators import *
  12
  13 class File(Object):
  14         def __init__(self, backend, release, id, data=None):
  15                 Object.__init__(self, backend)
  16
  17                 self.id = id
  18                 self._release = release
  19
  20                 # get all data from database
  21                 self.__data = data
  22
  23         def __eq__(self, other):
  24                 if isinstance(other, self.__class__):
  25                         return self.id == otherid
  26
  27         def __lt__(self, other):
  28                 if isinstance(other, self.__class__):
  29                         return self.prio < other.prio
  30
  31         @property
  32         def data(self):
  33                 if self.__data is None:
  34                         self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
  35                         assert self.__data
  36
  37                 return self.__data
  38
  39         @property
  40         def release(self):
  41                 if not self._release:
  42                         release_id = self.data.get("releases")
  43                         self._release = Release(release_id)
  44
  45                 return self._release
  46
  47         @property
  48         def type(self):
  49                 filename = self.filename
  50
  51                 if filename.endswith(".iso"):
  52                         return "iso"
  53
  54                 elif "xen" in filename:
  55                         if "downloader" in filename:
  56                                 return "xen-downloader"
  57
  58                         return "xen"
  59
  60                 elif "sources" in filename:
  61                         return "source"
  62
  63                 elif "usb-fdd" in filename:
  64                         return "usbfdd"
  65
  66                 elif "usb-hdd" in filename:
  67                         return "usbhdd"
  68
  69                 elif "armv5tel" in filename and "scon" in filename:
  70                         return "armv5tel-scon"
  71
  72                 elif "armv5tel" in filename:
  73                         return "armv5tel"
  74
  75                 elif "scon" in filename:
  76                         return "alix"
  77
  78                 elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
  79                         return "flash"
  80
  81                 else:
  82                         return "unknown"
  83
  84         @property
  85         def url(self):
  86                 return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)
  87
  88         @property
  89         def desc(self):
  90                 _ = lambda x: x
  91
  92                 descriptions = {
  93                         "armv5tel"      : _("Flash Image"),
  94                         "armv5tel-scon" : _("Flash Image with serial console"),
  95                         "iso"           : _("ISO Image"),
  96                         "flash"         : _("Flash Image"),
  97                         "alix"          : _("Flash Image with serial console"),
  98                         "usbfdd"        : _("USB FDD Image"),
  99                         "usbhdd"        : _("USB HDD Image"),
 100                         "xen"           : _("Pre-generated Xen Image"),
 101                         "xen-downloader": _("Xen-Image Generator"),
 102                 }
 103
 104                 try:
 105                         return descriptions[self.type]
 106                 except KeyError:
 107                         return _("Unknown image type")
 108
 109         @property
 110         def prio(self):
 111                 priorities = {
 112                         "iso"           : 10,
 113                         "flash"         : 40,
 114                         "alix"          : 41,
 115                         "usbfdd"        : 31,
 116                         "usbhdd"        : 30,
 117                         "armv5tel"      : 40,
 118                         "armv5tel-scon" : 41,
 119                         "xen"           : 50,
 120                         "xen-downloader": 51,
 121                 }
 122
 123                 try:
 124                         return priorities[self.type]
 125                 except KeyError:
 126                         return 999
 127
 128         @property
 129         def sha256(self):
 130                 return self.data.get("sha256")
 131
 132         @property
 133         def sha1(self):
 134                 return self.data.get("sha1")
 135
 136         @property
 137         def filename(self):
 138                 return self.data.get("filename")
 139
 140         @property
 141         def basename(self):
 142                 return os.path.basename(self.filename)
 143
 144         @property
 145         def size(self):
 146                 return self.data.get("filesize")
 147
 148         @property
 149         def arch(self):
 150                 known_arches = ("x86_64", "aarch64", "arm", "i586")
 151
 152                 for arch in known_arches:
 153                         if arch in self.basename:
 154                                 return arch
 155
 156                 return "N/A"
 157
 158
 159 class Release(Object):
 160         def __init__(self, backend, id, data=None):
 161                 Object.__init__(self, backend)
 162                 self.id = id
 163
 164                 # get all data from database
 165                 self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
 166                 assert self.__data
 167
 168                 self.__files = []
 169
 170         def __str__(self):
 171                 return self.name
 172
 173         def __repr__(self):
 174                 return "<%s %s>" % (self.__class__.__name__, self.name)
 175
 176         def __cmp__(self, other):
 177                 return cmp(self.id, other.id)
 178
 179         @property
 180         def arches(self):
 181                 for arch in ("x86_64", "aarch64", "i586", "arm"):
 182                         if arch in (f.arch for f in self.files):
 183                                 yield arch
 184
 185         @property
 186         def primary_arches(self):
 187                 arches = []
 188
 189                 # Add x86_64 when available, otherwise add i586
 190                 for arch in ("x86_64", "i586"):
 191                         if arch in self.arches:
 192                                 arches.append(arch)
 193                                 break
 194
 195                 # Add aarch64 if available
 196                 if "aarch64" in self.arches:
 197                         arches.append("aarch64")
 198
 199                 # Add ARM before 2.27 if available
 200                 if "arm" in self.arches and self.sname < "ipfire-2.27-core159":
 201                         arches.append("arm")
 202
 203                 return arches
 204
 205         @property
 206         def secondary_arches(self):
 207                 arches = []
 208
 209                 for arch in self.arches:
 210                         if arch in self.primary_arches:
 211                                 continue
 212
 213                         arches.append(arch)
 214
 215                 return arches
 216
 217         @property
 218         def experimental_arches(self):
 219                 return []
 220
 221         @property
 222         def files(self):
 223                 if not self.__files:
 224                         files = self.db.query("SELECT * FROM files WHERE releases = %s \
 225                                 AND NOT filename LIKE '%%.torrent'", self.id)
 226
 227                         self.__files = [File(self.backend, self, f.id, f) for f in files]
 228                         self.__files.sort()
 229
 230                 return self.__files
 231
 232         def get_files_by_arch(self, arch):
 233                 for f in self.files:
 234                         if f.arch == arch:
 235                                 yield f
 236
 237         @property
 238         def name(self):
 239                 return self.__data.name
 240
 241         @property
 242         def slug(self):
 243                 return self.__data.sname
 244
 245         # XXX compat
 246         sname = slug
 247
 248         @lazy_property
 249         def blog(self):
 250                 if self.__data.blog_id:
 251                         return self.backend.blog.get_by_id(self.__data.blog_id)
 252
 253         @property
 254         def stable(self):
 255                 return self.__data.stable
 256
 257         @property
 258         def published(self):
 259                 return self.__data.published
 260
 261         date = published
 262
 263         @property
 264         def path(self):
 265                 return self.__data.path
 266
 267         def get_file(self, type, arch=None):
 268                 for file in self.files:
 269                         # Skip anything where the type does not match
 270                         if not file.type == type:
 271                                 continue
 272
 273                         # Optionally skip the architecture
 274                         if arch and not file.arch == arch:
 275                                 continue
 276
 277                         return file
 278
 279         def __file_hash(self, filename, algo="sha256"):
 280                 h = hashlib.new(algo)
 281
 282                 with open(filename, "rb") as f:
 283                         buf_size = 1024
 284                         buf = f.read(buf_size)
 285                         while buf:
 286                                 h.update(buf)
 287                                 buf = f.read(buf_size)
 288
 289                 return h.hexdigest()
 290
 291         def scan_files(self, basepath="/pub/mirror"):
 292                 if not self.path:
 293                         return
 294
 295                 path = os.path.join(basepath, self.path)
 296                 if not os.path.exists(path):
 297                         return
 298
 299                 files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
 300                 files = [f.filename for f in files]
 301
 302                 # Make files that do not exists not loadable.
 303                 for filename in files:
 304                         _filename = os.path.join(basepath, filename)
 305                         if not os.path.exists(_filename):
 306                                 self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
 307
 308                 for filename in os.listdir(path):
 309                         filename = os.path.join(path, filename)
 310
 311                         if os.path.isdir(filename):
 312                                 continue
 313
 314                         _filename = re.match(".*(releases/.*)", filename).group(1)
 315                         if _filename in files:
 316                                 continue
 317
 318                         if filename.endswith(".b2") or filename.endswith(".md5"):
 319                                 continue
 320
 321                         logging.info("Hashing %s..." % filename)
 322                         hash_sha256 = self.__file_hash(filename, "sha256")
 323                         hash_sha1   = self.__file_hash(filename, "sha1")
 324                         filesize = os.path.getsize(filename)
 325
 326                         self.db.execute("INSERT INTO files(releases, filename, filesize, \
 327                                 sha256, sha1) VALUES(%s, %s, %s, %s, %s)",
 328                                 self.id, _filename, filesize, hash_sha256, hash_sha1)
 329
 330         def supports_arch(self, arch):
 331                 return arch in ("x86_64", "i586")
 332
 333         def supports_platform(self, platform):
 334                 # Currently there is nothing else than pcbios supported
 335                 if platform == "pcbios":
 336                         return True
 337
 338                 return False
 339
 340         def is_netboot_capable(self):
 341                 return self.path and "ipfire-2.x" in self.path
 342
 343         def netboot_kernel_url(self, arch, platform):
 344                 assert self.supports_arch(arch)
 345                 assert self.supports_platform(platform)
 346
 347                 if self.sname >= "ipfire-2.19-core100":
 348                         return "http://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)
 349
 350                 return "http://boot.ipfire.org/%s/images/vmlinuz" % self.path
 351
 352         def netboot_initrd_url(self, arch, platform):
 353                 assert self.supports_arch(arch)
 354                 assert self.supports_platform(platform)
 355
 356                 if self.sname >= "ipfire-2.19-core100":
 357                         return "http://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch)
 358
 359                 return "http://boot.ipfire.org/%s/images/instroot" % self.path
 360
 361         def netboot_args(self, arch, platform):
 362                 return ""
 363
 364         # OS List (For Raspberry Pi Imager)
 365
 366         def make_os_list(self, arch):
 367                 """
 368                         Returns an object according to this spec:
 369
 370                         https://github.com/raspberrypi/rpi-imager/blob/qml/doc/json-schema/os-list-schema.json
 371                         https://github.com/raspberrypi/rpi-imager/blob/qml/doc/os-sublist-example.json
 372                 """
 373                 # Fetch the flash image
 374                 file = self.get_file(type="flash", arch=arch)
 375
 376                 # Return an empty object if we could not find the image file
 377                 if not file:
 378                         return {}
 379
 380                 # Make the document
 381                 os_list = {
 382                         "name"         : self.name,
 383                         "description"  : "The Open Source Linux-based Firewall Operating System"
 384                                 " with a Comprehensive Feature Set",
 385                         "url"          : file.url,
 386                         "icon"         : "https://www.ipfire.org/static/img/ipfire-tux-40x40.png",
 387                         "website"      : "https://www.ipfire.org/",
 388                         "release_date" : self.published.strftime("%Y-%m-%d"),
 389
 390                         # Image Metadata
 391                         "image_download_size"   : file.size,
 392                         "image_download_sha256" : file.sha256,
 393                 }
 394
 395                 return {
 396                         "os_list" : [
 397                                 os_list,
 398                         ],
 399                 }
 400
 401         @property
 402         def post(self):
 403                 if self.__data.blog_id:
 404                         return self.backend.blog.get_by_id(self.__data.blog_id)
 405
 406         # Fireinfo Stuff
 407
 408         def get_usage(self, when=None):
 409                 name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")
 410
 411                 # Get penetration from fireinfo
 412                 releases = self.backend.fireinfo.get_releases_map(when=when)
 413
 414                 return releases.get(name, 0)
 415
 416
 417 class Releases(Object):
 418         def _get_release(self, query, *args):
 419                 res = self.db.get(query, *args)
 420
 421                 if res:
 422                         return Release(self.backend, res.id, data=res)
 423
 424         def _get_releases(self, query, *args):
 425                 res = self.db.query(query, *args)
 426
 427                 for row in res:
 428                         yield Release(self.backend, row.id, data=row)
 429
 430         def __iter__(self):
 431                 releases = self._get_releases("SELECT * FROM releases \
 432                         ORDER BY published DESC NULLS FIRST")
 433
 434                 return iter(releases)
 435
 436         def get_by_id(self, id):
 437                 ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)
 438
 439                 if ret:
 440                         return Release(self.backend, ret.id, data=ret)
 441
 442         def get_by_sname(self, sname):
 443                 ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)
 444
 445                 if ret:
 446                         return Release(self.backend, ret.id, data=ret)
 447
 448         def get_by_news_id(self, news_id):
 449                 ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)
 450
 451                 if ret:
 452                         return Release(self.backend, ret.id, data=ret)
 453
 454         def get_latest(self, stable=True):
 455                 ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
 456                         AND stable = %s ORDER BY published DESC LIMIT 1", stable)
 457
 458                 if ret:
 459                         return Release(self.backend, ret.id, data=ret)
 460
 461         def get_releases_older_than(self, release, limit=None):
 462                 return self._get_releases("SELECT * FROM releases \
 463                         WHERE published IS NOT NULL AND published < %s \
 464                                 ORDER BY published DESC LIMIT %s", release.published, limit)
 465
 466         def get_latest_unstable(self):
 467                 ret = self.db.get("SELECT * FROM releases r1 \
 468                         WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
 469                         AND stable = %s AND NOT EXISTS ( \
 470                                 SELECT * FROM releases r2 WHERE r2.stable = %s AND \
 471                                         r2.published IS NOT NULL AND r2.published >= r1.published \
 472                                 ) ORDER BY r1.published DESC LIMIT 1", False, True)
 473
 474                 if ret:
 475                         return Release(self.backend, ret.id, data=ret)
 476
 477         def get_stable(self):
 478                 query = self.db.query("SELECT * FROM releases \
 479                         WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
 480                         ORDER BY published DESC")
 481
 482                 releases = []
 483                 for row in query:
 484                         release = Release(self.backend, row.id, data=row)
 485                         releases.append(release)
 486
 487                 return releases
 488
 489         def get_unstable(self):
 490                 query = self.db.query("SELECT * FROM releases \
 491                         WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
 492                         ORDER BY published DESC")
 493
 494                 releases = []
 495                 for row in query:
 496                         release = Release(self.backend, row.id, data=row)
 497                         releases.append(release)
 498
 499                 return releases
 500
 501         def get_all(self):
 502                 query = self.db.query("SELECT * FROM releases \
 503                         WHERE published IS NOT NULL AND published <= NOW() \
 504                         ORDER BY published DESC")
 505
 506                 releases = []
 507                 for row in query:
 508                         release = Release(self.backend, row.id, data=row)
 509                         releases.append(release)
 510
 511                 return releases
 512
 513         def _get_all(self):
 514                 query = self.db.query("SELECT * FROM releases ORDER BY published DESC")
 515
 516                 releases = []
 517                 for row in query:
 518                         release = Release(self.backend, row.id, data=row)
 519                         releases.append(release)
 520
 521                 return releases
 522
 523         async def scan_files(self, basepath="/pub/mirror"):
 524                 for release in self:
 525                         logging.debug("Scanning %s..." % release)
 526
 527                         with self.db.transaction():
 528                                 release.scan_files(basepath=basepath)
 529
 530         def get_file_by_filename(self, filename):
 531                 ret = self.db.get("SELECT * FROM files WHERE filename = %s", filename)
 532
 533                 if ret:
 534                         return File(self.backend, None, ret.id, data=ret)