src/backend/releases.py

   1 #!/usr/bin/python
   2
   3 import hashlib
   4 import logging
   5 import os
   6 import re
   7 import urllib.parse
   8
   9 from . import database
  10 from .misc import Object
  11 from .decorators import *
  12
  13 class File(Object):
  14         def __init__(self, backend, release, id, data=None):
  15                 Object.__init__(self, backend)
  16
  17                 self.id = id
  18                 self._release = release
  19
  20                 # get all data from database
  21                 self.__data = data
  22
  23         def __eq__(self, other):
  24                 if isinstance(other, self.__class__):
  25                         return self.id == otherid
  26
  27         def __lt__(self, other):
  28                 if isinstance(other, self.__class__):
  29                         return self.prio < other.prio
  30
  31         @property
  32         def data(self):
  33                 if self.__data is None:
  34                         self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
  35                         assert self.__data
  36
  37                 return self.__data
  38
  39         @property
  40         def release(self):
  41                 if not self._release:
  42                         release_id = self.data.get("releases")
  43                         self._release = Release(release_id)
  44
  45                 return self._release
  46
  47         @property
  48         def type(self):
  49                 filename = self.filename
  50
  51                 if filename.endswith(".iso"):
  52                         return "iso"
  53
  54                 elif "xen" in filename:
  55                         if "downloader" in filename:
  56                                 return "xen-downloader"
  57
  58                         return "xen"
  59
  60                 elif "sources" in filename:
  61                         return "source"
  62
  63                 elif "usb-fdd" in filename:
  64                         return "usbfdd"
  65
  66                 elif "usb-hdd" in filename:
  67                         return "usbhdd"
  68
  69                 elif "armv5tel" in filename and "scon" in filename:
  70                         return "armv5tel-scon"
  71
  72                 elif "armv5tel" in filename:
  73                         return "armv5tel"
  74
  75                 elif "scon" in filename:
  76                         return "alix"
  77
  78                 elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
  79                         return "flash"
  80
  81                 else:
  82                         return "unknown"
  83
  84         @property
  85         def url(self):
  86                 return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)
  87
  88         @property
  89         def desc(self):
  90                 _ = lambda x: x
  91
  92                 descriptions = {
  93                         "armv5tel"      : _("Flash Image"),
  94                         "armv5tel-scon" : _("Flash Image with serial console"),
  95                         "iso"           : _("ISO Image"),
  96                         "flash"         : _("Flash Image"),
  97                         "alix"          : _("Flash Image with serial console"),
  98                         "usbfdd"        : _("USB FDD Image"),
  99                         "usbhdd"        : _("USB HDD Image"),
 100                         "xen"           : _("Pre-generated Xen Image"),
 101                         "xen-downloader": _("Xen-Image Generator"),
 102                 }
 103
 104                 try:
 105                         return descriptions[self.type]
 106                 except KeyError:
 107                         return _("Unknown image type")
 108
 109         @property
 110         def prio(self):
 111                 priorities = {
 112                         "iso"           : 10,
 113                         "flash"         : 40,
 114                         "alix"          : 41,
 115                         "usbfdd"        : 31,
 116                         "usbhdd"        : 30,
 117                         "armv5tel"      : 40,
 118                         "armv5tel-scon" : 41,
 119                         "xen"           : 50,
 120                         "xen-downloader": 51,
 121                 }
 122
 123                 try:
 124                         return priorities[self.type]
 125                 except KeyError:
 126                         return 999
 127
 128         @property
 129         def sha256(self):
 130                 return self.data.get("sha256")
 131
 132         @property
 133         def sha1(self):
 134                 return self.data.get("sha1")
 135
 136         @property
 137         def filename(self):
 138                 return self.data.get("filename")
 139
 140         @property
 141         def basename(self):
 142                 return os.path.basename(self.filename)
 143
 144         @property
 145         def size(self):
 146                 return self.data.get("filesize")
 147
 148         @property
 149         def arch(self):
 150                 known_arches = ("x86_64", "aarch64", "arm", "i586")
 151
 152                 for arch in known_arches:
 153                         if arch in self.basename:
 154                                 return arch
 155
 156                 return "N/A"
 157
 158
 159 class Release(Object):
 160         def __init__(self, backend, id, data=None):
 161                 Object.__init__(self, backend)
 162                 self.id = id
 163
 164                 # get all data from database
 165                 self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
 166                 assert self.__data
 167
 168                 self.__files = []
 169
 170         def __str__(self):
 171                 return self.name
 172
 173         def __repr__(self):
 174                 return "<%s %s>" % (self.__class__.__name__, self.name)
 175
 176         def __cmp__(self, other):
 177                 return cmp(self.id, other.id)
 178
 179         @property
 180         def arches(self):
 181                 for arch in ("x86_64", "aarch64", "i586", "arm"):
 182                         if arch in (f.arch for f in self.files):
 183                                 yield arch
 184
 185         @property
 186         def primary_arches(self):
 187                 arches = []
 188
 189                 # Add x86_64 when available, otherwise add i586
 190                 for arch in ("x86_64", "i586"):
 191                         if arch in self.arches:
 192                                 arches.append(arch)
 193                                 break
 194
 195                 # Add aarch64 if available
 196                 if "aarch64" in self.arches:
 197                         arches.append("aarch64")
 198
 199                 # Add ARM before 2.27 if available
 200                 if "arm" in self.arches and self.sname < "ipfire-2.27-core159":
 201                         arches.append("arm")
 202
 203                 return arches
 204
 205         @property
 206         def secondary_arches(self):
 207                 arches = []
 208
 209                 for arch in self.arches:
 210                         if arch in self.primary_arches:
 211                                 continue
 212
 213                         arches.append(arch)
 214
 215                 return arches
 216
 217         @property
 218         def experimental_arches(self):
 219                 return []
 220
 221         @property
 222         def files(self):
 223                 if not self.__files:
 224                         files = self.db.query("SELECT * FROM files WHERE releases = %s \
 225                                 AND NOT filename LIKE '%%.torrent'", self.id)
 226
 227                         self.__files = [File(self.backend, self, f.id, f) for f in files]
 228                         self.__files.sort()
 229
 230                 return self.__files
 231
 232         def get_files_by_arch(self, arch):
 233                 for f in self.files:
 234                         if f.arch == arch:
 235                                 yield f
 236
 237         @property
 238         def name(self):
 239                 return self.__data.name
 240
 241         @property
 242         def slug(self):
 243                 return self.__data.sname
 244
 245         # XXX compat
 246         sname = slug
 247
 248         @lazy_property
 249         def blog(self):
 250                 if self.__data.blog_id:
 251                         return self.backend.blog.get_by_id(self.__data.blog_id)
 252
 253         @property
 254         def stable(self):
 255                 return self.__data.stable
 256
 257         @property
 258         def published(self):
 259                 return self.__data.published
 260
 261         date = published
 262
 263         @property
 264         def path(self):
 265                 return self.__data.path
 266
 267         def get_file(self, type):
 268                 for file in self.files:
 269                         if file.type == type:
 270                                 return file
 271
 272         def __file_hash(self, filename, algo="sha256"):
 273                 h = hashlib.new(algo)
 274
 275                 with open(filename, "rb") as f:
 276                         buf_size = 1024
 277                         buf = f.read(buf_size)
 278                         while buf:
 279                                 h.update(buf)
 280                                 buf = f.read(buf_size)
 281
 282                 return h.hexdigest()
 283
 284         def scan_files(self, basepath="/pub/mirror"):
 285                 if not self.path:
 286                         return
 287
 288                 path = os.path.join(basepath, self.path)
 289                 if not os.path.exists(path):
 290                         return
 291
 292                 files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
 293                 files = [f.filename for f in files]
 294
 295                 # Make files that do not exists not loadable.
 296                 for filename in files:
 297                         _filename = os.path.join(basepath, filename)
 298                         if not os.path.exists(_filename):
 299                                 self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
 300
 301                 for filename in os.listdir(path):
 302                         filename = os.path.join(path, filename)
 303
 304                         if os.path.isdir(filename):
 305                                 continue
 306
 307                         _filename = re.match(".*(releases/.*)", filename).group(1)
 308                         if _filename in files:
 309                                 continue
 310
 311                         if filename.endswith(".b2") or filename.endswith(".md5"):
 312                                 continue
 313
 314                         logging.info("Hashing %s..." % filename)
 315                         hash_sha256 = self.__file_hash(filename, "sha256")
 316                         hash_sha1   = self.__file_hash(filename, "sha1")
 317                         filesize = os.path.getsize(filename)
 318
 319                         self.db.execute("INSERT INTO files(releases, filename, filesize, \
 320                                 sha256, sha1) VALUES(%s, %s, %s, %s, %s)",
 321                                 self.id, _filename, filesize, hash_sha256, hash_sha1)
 322
 323         def supports_arch(self, arch):
 324                 return arch in ("x86_64", "i586")
 325
 326         def supports_platform(self, platform):
 327                 # Currently there is nothing else than pcbios supported
 328                 if platform == "pcbios":
 329                         return True
 330
 331                 return False
 332
 333         def is_netboot_capable(self):
 334                 return self.path and "ipfire-2.x" in self.path
 335
 336         def netboot_kernel_url(self, arch, platform):
 337                 assert self.supports_arch(arch)
 338                 assert self.supports_platform(platform)
 339
 340                 if self.sname >= "ipfire-2.19-core100":
 341                         return "http://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)
 342
 343                 return "http://boot.ipfire.org/%s/images/vmlinuz" % self.path
 344
 345         def netboot_initrd_url(self, arch, platform):
 346                 assert self.supports_arch(arch)
 347                 assert self.supports_platform(platform)
 348
 349                 if self.sname >= "ipfire-2.19-core100":
 350                         return "http://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch)
 351
 352                 return "http://boot.ipfire.org/%s/images/instroot" % self.path
 353
 354         def netboot_args(self, arch, platform):
 355                 return ""
 356
 357         @property
 358         def post(self):
 359                 if self.__data.blog_id:
 360                         return self.backend.blog.get_by_id(self.__data.blog_id)
 361
 362         # Fireinfo Stuff
 363
 364         def get_usage(self, when=None):
 365                 name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")
 366
 367                 # Get penetration from fireinfo
 368                 releases = self.backend.fireinfo.get_releases_map(when=when)
 369
 370                 return releases.get(name, 0)
 371
 372
 373 class Releases(Object):
 374         def _get_release(self, query, *args):
 375                 res = self.db.get(query, *args)
 376
 377                 if res:
 378                         return Release(self.backend, res.id, data=res)
 379
 380         def _get_releases(self, query, *args):
 381                 res = self.db.query(query, *args)
 382
 383                 for row in res:
 384                         yield Release(self.backend, row.id, data=row)
 385
 386         def __iter__(self):
 387                 releases = self._get_releases("SELECT * FROM releases \
 388                         ORDER BY published DESC NULLS FIRST")
 389
 390                 return iter(releases)
 391
 392         def get_by_id(self, id):
 393                 ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)
 394
 395                 if ret:
 396                         return Release(self.backend, ret.id, data=ret)
 397
 398         def get_by_sname(self, sname):
 399                 ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)
 400
 401                 if ret:
 402                         return Release(self.backend, ret.id, data=ret)
 403
 404         def get_by_news_id(self, news_id):
 405                 ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)
 406
 407                 if ret:
 408                         return Release(self.backend, ret.id, data=ret)
 409
 410         def get_latest(self, stable=True):
 411                 ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
 412                         AND stable = %s ORDER BY published DESC LIMIT 1", stable)
 413
 414                 if ret:
 415                         return Release(self.backend, ret.id, data=ret)
 416
 417         def get_releases_older_than(self, release, limit=None):
 418                 return self._get_releases("SELECT * FROM releases \
 419                         WHERE published IS NOT NULL AND published < %s \
 420                                 ORDER BY published DESC LIMIT %s", release.published, limit)
 421
 422         def get_latest_unstable(self):
 423                 ret = self.db.get("SELECT * FROM releases r1 \
 424                         WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
 425                         AND stable = %s AND NOT EXISTS ( \
 426                                 SELECT * FROM releases r2 WHERE r2.stable = %s AND \
 427                                         r2.published IS NOT NULL AND r2.published >= r1.published \
 428                                 ) ORDER BY r1.published DESC LIMIT 1", False, True)
 429
 430                 if ret:
 431                         return Release(self.backend, ret.id, data=ret)
 432
 433         def get_stable(self):
 434                 query = self.db.query("SELECT * FROM releases \
 435                         WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
 436                         ORDER BY published DESC")
 437
 438                 releases = []
 439                 for row in query:
 440                         release = Release(self.backend, row.id, data=row)
 441                         releases.append(release)
 442
 443                 return releases
 444
 445         def get_unstable(self):
 446                 query = self.db.query("SELECT * FROM releases \
 447                         WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
 448                         ORDER BY published DESC")
 449
 450                 releases = []
 451                 for row in query:
 452                         release = Release(self.backend, row.id, data=row)
 453                         releases.append(release)
 454
 455                 return releases
 456
 457         def get_all(self):
 458                 query = self.db.query("SELECT * FROM releases \
 459                         WHERE published IS NOT NULL AND published <= NOW() \
 460                         ORDER BY published DESC")
 461
 462                 releases = []
 463                 for row in query:
 464                         release = Release(self.backend, row.id, data=row)
 465                         releases.append(release)
 466
 467                 return releases
 468
 469         def _get_all(self):
 470                 query = self.db.query("SELECT * FROM releases ORDER BY published DESC")
 471
 472                 releases = []
 473                 for row in query:
 474                         release = Release(self.backend, row.id, data=row)
 475                         releases.append(release)
 476
 477                 return releases
 478
 479         async def scan_files(self, basepath="/pub/mirror"):
 480                 for release in self:
 481                         logging.debug("Scanning %s..." % release)
 482
 483                         with self.db.transaction():
 484                                 release.scan_files(basepath=basepath)