releases: Promote aarch64 and demote armv6l
[ipfire.org.git] / src / backend / releases.py
1 #!/usr/bin/python
2
3 import hashlib
4 import logging
5 import os
6 import re
7 import urllib.parse
8 import yabencode
9
10 from . import database
11 from .misc import Object
12 from .decorators import *
13
14 TRACKERS = (
15 "http://ipv4.tracker.ipfire.org:6969/announce",
16 "udp://ipv4.tracker.ipfire.org:6969",
17 "http://ipv6.tracker.ipfire.org:6969/announce",
18 "udp://ipv6.tracker.ipfire.org:6969",
19 )
20
21 class File(Object):
22 def __init__(self, backend, release, id, data=None):
23 Object.__init__(self, backend)
24
25 self.id = id
26 self._release = release
27
28 # get all data from database
29 self.__data = data
30
31 def __eq__(self, other):
32 if isinstance(other, self.__class__):
33 return self.id == otherid
34
35 def __lt__(self, other):
36 if isinstance(other, self.__class__):
37 return self.prio < other.prio
38
39 @property
40 def data(self):
41 if self.__data is None:
42 self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
43 assert self.__data
44
45 return self.__data
46
47 @property
48 def release(self):
49 if not self._release:
50 release_id = self.data.get("releases")
51 self._release = Release(release_id)
52
53 return self._release
54
55 @property
56 def type(self):
57 filename = self.filename
58
59 if filename.endswith(".iso"):
60 return "iso"
61
62 elif filename.endswith(".torrent"):
63 return "torrent"
64
65 elif "xen" in filename:
66 if "downloader" in filename:
67 return "xen-downloader"
68
69 return "xen"
70
71 elif "sources" in filename:
72 return "source"
73
74 elif "usb-fdd" in filename:
75 return "usbfdd"
76
77 elif "usb-hdd" in filename:
78 return "usbhdd"
79
80 elif "armv5tel" in filename and "scon" in filename:
81 return "armv5tel-scon"
82
83 elif "armv5tel" in filename:
84 return "armv5tel"
85
86 elif "scon" in filename:
87 return "alix"
88
89 elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
90 return "flash"
91
92 else:
93 return "unknown"
94
95 @property
96 def url(self):
97 return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)
98
99 @property
100 def desc(self):
101 _ = lambda x: x
102
103 descriptions = {
104 "armv5tel" : _("Flash Image"),
105 "armv5tel-scon" : _("Flash Image with serial console"),
106 "iso" : _("ISO Image"),
107 "torrent" : _("Torrent File"),
108 "flash" : _("Flash Image"),
109 "alix" : _("Flash Image with serial console"),
110 "usbfdd" : _("USB FDD Image"),
111 "usbhdd" : _("USB HDD Image"),
112 "xen" : _("Pre-generated Xen Image"),
113 "xen-downloader": _("Xen-Image Generator"),
114 }
115
116 try:
117 return descriptions[self.type]
118 except KeyError:
119 return _("Unknown image type")
120
121 @property
122 def prio(self):
123 priorities = {
124 "iso" : 10,
125 "torrent" : 20,
126 "flash" : 40,
127 "alix" : 41,
128 "usbfdd" : 31,
129 "usbhdd" : 30,
130 "armv5tel" : 40,
131 "armv5tel-scon" : 41,
132 "xen" : 50,
133 "xen-downloader": 51,
134 }
135
136 try:
137 return priorities[self.type]
138 except KeyError:
139 return 999
140
141 @property
142 def sha256(self):
143 return self.data.get("sha256")
144
145 @property
146 def sha1(self):
147 return self.data.get("sha1")
148
149 @property
150 def filename(self):
151 return self.data.get("filename")
152
153 @property
154 def basename(self):
155 return os.path.basename(self.filename)
156
157 @property
158 def size(self):
159 return self.data.get("filesize")
160
161 @property
162 def arch(self):
163 known_arches = ("x86_64", "aarch64", "arm", "i586")
164
165 for arch in known_arches:
166 if arch in self.basename:
167 return arch
168
169 return "N/A"
170
171 @property
172 def torrent_hash(self):
173 return self.data.get("torrent_hash", None)
174
175 @property
176 def torrent_url(self):
177 if self.torrent_hash:
178 return "%s.torrent" % self.url
179
180 @property
181 def magnet_link(self):
182 # Don't return anything if we have no torrent hash.
183 if self.torrent_hash is None:
184 return
185
186 s = "magnet:?xt=urn:btih:%s" % self.torrent_hash
187
188 #s += "&xl=%d" % self.size
189 s += "&dn=%s" % urllib.parse.quote(self.basename)
190
191 # Add our tracker.
192 for tracker in TRACKERS:
193 s += "&tr=%s" % tracker
194
195 # Add web download URL
196 s += "&as=%s" % urllib.parse.quote(self.url)
197
198 return s
199
200
201 class Release(Object):
202 def __init__(self, backend, id, data=None):
203 Object.__init__(self, backend)
204 self.id = id
205
206 # get all data from database
207 self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
208 assert self.__data
209
210 self.__files = []
211
212 def __str__(self):
213 return self.name
214
215 def __repr__(self):
216 return "<%s %s>" % (self.__class__.__name__, self.name)
217
218 def __cmp__(self, other):
219 return cmp(self.id, other.id)
220
221 @property
222 def arches(self):
223 for arch in ("x86_64", "aarch64", "i586", "arm"):
224 if arch in (f.arch for f in self.files):
225 yield arch
226
227 @property
228 def primary_arches(self):
229 arches = []
230
231 # Add x86_64 when available, otherwise add i586
232 for arch in ("x86_64", "i586"):
233 if arch in self.arches:
234 arches.append(arch)
235 break
236
237 # Add aarch64 if available
238 if "aarch64" in self.arches:
239 arches.append("aarch64")
240
241 # Add ARM before 2.27 if available
242 if "arm" in self.arches and self.sname < "ipfire-2.27-core159":
243 arches.append("arm")
244
245 return arches
246
247 @property
248 def secondary_arches(self):
249 arches = []
250
251 for arch in self.arches:
252 if arch in self.primary_arches:
253 continue
254
255 arches.append(arch)
256
257 return arches
258
259 @property
260 def experimental_arches(self):
261 return []
262
263 @property
264 def files(self):
265 if not self.__files:
266 files = self.db.query("SELECT * FROM files WHERE releases = %s \
267 AND NOT filename LIKE '%%.torrent'", self.id)
268
269 self.__files = [File(self.backend, self, f.id, f) for f in files]
270 self.__files.sort()
271
272 return self.__files
273
274 def get_files_by_arch(self, arch):
275 for f in self.files:
276 if f.arch == arch:
277 yield f
278
279 @property
280 def torrents(self):
281 torrents = []
282
283 for file in self.files:
284 if not file.torrent_hash:
285 continue
286
287 torrents.append(file)
288
289 return torrents
290
291 @property
292 def name(self):
293 return self.__data.name
294
295 @property
296 def slug(self):
297 return self.__data.sname
298
299 # XXX compat
300 sname = slug
301
302 @lazy_property
303 def blog(self):
304 if self.__data.blog_id:
305 return self.backend.blog.get_by_id(self.__data.blog_id)
306
307 @property
308 def fireinfo_id(self):
309 name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")
310
311 res = self.db.get("SELECT id FROM fireinfo_releases \
312 WHERE name = %s", name)
313
314 if res:
315 return res.id
316
317 @property
318 def stable(self):
319 return self.__data.stable
320
321 @property
322 def published(self):
323 return self.__data.published
324
325 date = published
326
327 @property
328 def path(self):
329 return self.__data.path
330
331 def get_file(self, type):
332 for file in self.files:
333 if file.type == type:
334 return file
335
336 def __file_hash(self, filename, algo="sha256"):
337 h = hashlib.new(algo)
338
339 with open(filename, "rb") as f:
340 buf_size = 1024
341 buf = f.read(buf_size)
342 while buf:
343 h.update(buf)
344 buf = f.read(buf_size)
345
346 return h.hexdigest()
347
348 def scan_files(self, basepath="/pub/mirror"):
349 if not self.path:
350 return
351
352 path = os.path.join(basepath, self.path)
353 if not os.path.exists(path):
354 return
355
356 files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
357 files = [f.filename for f in files]
358
359 # Make files that do not exists not loadable.
360 for filename in files:
361 _filename = os.path.join(basepath, filename)
362 if not os.path.exists(_filename):
363 self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
364
365 for filename in os.listdir(path):
366 filename = os.path.join(path, filename)
367
368 if os.path.isdir(filename):
369 continue
370
371 _filename = re.match(".*(releases/.*)", filename).group(1)
372 if _filename in files:
373 continue
374
375 if filename.endswith(".md5"):
376 continue
377
378 logging.info("Hashing %s..." % filename)
379 hash_sha256 = self.__file_hash(filename, "sha256")
380 hash_sha1 = self.__file_hash(filename, "sha1")
381 filesize = os.path.getsize(filename)
382
383 # Check if there is a torrent download available for this file:
384 torrent_hash = ""
385 torrent_file = "%s.torrent" % filename
386 if os.path.exists(torrent_file):
387 torrent_hash = self.torrent_read_hash(torrent_file)
388
389 self.db.execute("INSERT INTO files(releases, filename, filesize, \
390 sha256, sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s, %s)",
391 self.id, _filename, filesize, hash_sha256, hash_sha1, torrent_hash)
392
393 # Search for all files that miss a torrent hash.
394 files = self.db.query("SELECT id, filename FROM files \
395 WHERE releases = %s AND torrent_hash IS NULL", self.id)
396
397 for file in files:
398 path = os.path.join(basepath, file.filename)
399
400 torrent_file = "%s.torrent" % path
401 if os.path.exists(torrent_file):
402 torrent_hash = self.torrent_read_hash(torrent_file)
403
404 self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
405 torrent_hash, file.id)
406
407 def torrent_read_hash(self, filename):
408 with open(filename, "rb") as f:
409 metainfo = yabencode.decode(f)
410 metainfo = yabencode.encode(metainfo["info"])
411
412 h = hashlib.new("sha1")
413 h.update(metainfo)
414
415 return h.hexdigest()
416
417 def supports_arch(self, arch):
418 return arch in ("x86_64", "i586")
419
420 def supports_platform(self, platform):
421 # Currently there is nothing else than pcbios supported
422 if platform == "pcbios":
423 return True
424
425 return False
426
427 def is_netboot_capable(self):
428 return self.path and "ipfire-2.x" in self.path
429
430 def netboot_kernel_url(self, arch, platform):
431 assert self.supports_arch(arch)
432 assert self.supports_platform(platform)
433
434 if self.sname >= "ipfire-2.19-core100":
435 return "http://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)
436
437 return "http://boot.ipfire.org/%s/images/vmlinuz" % self.path
438
439 def netboot_initrd_url(self, arch, platform):
440 assert self.supports_arch(arch)
441 assert self.supports_platform(platform)
442
443 if self.sname >= "ipfire-2.19-core100":
444 return "http://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch)
445
446 return "http://boot.ipfire.org/%s/images/instroot" % self.path
447
448 def netboot_args(self, arch, platform):
449 return ""
450
451 @property
452 def post(self):
453 if self.__data.blog_id:
454 return self.backend.blog.get_by_id(self.__data.blog_id)
455
456 # Fireinfo Stuff
457
458 @property
459 def penetration(self):
460 # Get penetration from fireinfo
461 return self.backend.fireinfo.get_release_penetration(self)
462
463
464 class Releases(Object):
465 def _get_release(self, query, *args):
466 res = self.db.get(query, *args)
467
468 if res:
469 return Release(self.backend, res.id, data=res)
470
471 def _get_releases(self, query, *args):
472 res = self.db.query(query, *args)
473
474 for row in res:
475 yield Release(self.backend, row.id, data=row)
476
477 def __iter__(self):
478 releases = self._get_releases("SELECT * FROM releases \
479 ORDER BY published DESC NULLS FIRST")
480
481 return iter(releases)
482
483 def get_by_id(self, id):
484 ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)
485
486 if ret:
487 return Release(self.backend, ret.id, data=ret)
488
489 def get_by_sname(self, sname):
490 ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)
491
492 if ret:
493 return Release(self.backend, ret.id, data=ret)
494
495 def get_by_news_id(self, news_id):
496 ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)
497
498 if ret:
499 return Release(self.backend, ret.id, data=ret)
500
501 def get_latest(self, stable=True):
502 ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
503 AND stable = %s ORDER BY published DESC LIMIT 1", stable)
504
505 if ret:
506 return Release(self.backend, ret.id, data=ret)
507
508 def get_releases_older_than(self, release, limit=None):
509 return self._get_releases("SELECT * FROM releases \
510 WHERE published IS NOT NULL AND published < %s \
511 ORDER BY published DESC LIMIT %s", release.published, limit)
512
513 def get_latest_unstable(self):
514 ret = self.db.get("SELECT * FROM releases r1 \
515 WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
516 AND stable = %s AND NOT EXISTS ( \
517 SELECT * FROM releases r2 WHERE r2.stable = %s AND \
518 r2.published IS NOT NULL AND r2.published >= r1.published \
519 ) ORDER BY r1.published DESC LIMIT 1", False, True)
520
521 if ret:
522 return Release(self.backend, ret.id, data=ret)
523
524 def get_stable(self):
525 query = self.db.query("SELECT * FROM releases \
526 WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
527 ORDER BY published DESC")
528
529 releases = []
530 for row in query:
531 release = Release(self.backend, row.id, data=row)
532 releases.append(release)
533
534 return releases
535
536 def get_unstable(self):
537 query = self.db.query("SELECT * FROM releases \
538 WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
539 ORDER BY published DESC")
540
541 releases = []
542 for row in query:
543 release = Release(self.backend, row.id, data=row)
544 releases.append(release)
545
546 return releases
547
548 def get_all(self):
549 query = self.db.query("SELECT * FROM releases \
550 WHERE published IS NOT NULL AND published <= NOW() \
551 ORDER BY published DESC")
552
553 releases = []
554 for row in query:
555 release = Release(self.backend, row.id, data=row)
556 releases.append(release)
557
558 return releases
559
560 def _get_all(self):
561 query = self.db.query("SELECT * FROM releases ORDER BY published DESC")
562
563 releases = []
564 for row in query:
565 release = Release(self.backend, row.id, data=row)
566 releases.append(release)
567
568 return releases
569
570 def get_file_for_torrent_hash(self, torrent_hash):
571 file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
572 torrent_hash)
573
574 if not file:
575 return
576
577 release = Release(self.backend, file.releases)
578 file = File(self.backend, release, file.id)
579
580 return file
581
582 async def scan_files(self, basepath="/pub/mirror"):
583 for release in self:
584 logging.debug("Scanning %s..." % release)
585
586 with self.db.transaction():
587 release.scan_files(basepath=basepath)