a0a8cf93154adf8dc7446a14d0081d4485bdab2f
[ipfire.org.git] / src / backend / releases.py
1 #!/usr/bin/python
2
3 import hashlib
4 import logging
5 import os
6 import re
7 import urllib.parse
8 import yabencode
9
10 from . import database
11 from .misc import Object
12 from .decorators import *
13
14 TRACKERS = (
15 "http://ipv4.tracker.ipfire.org:6969/announce",
16 "udp://ipv4.tracker.ipfire.org:6969",
17 "http://ipv6.tracker.ipfire.org:6969/announce",
18 "udp://ipv6.tracker.ipfire.org:6969",
19 )
20
21 class File(Object):
22 def __init__(self, backend, release, id, data=None):
23 Object.__init__(self, backend)
24
25 self.id = id
26 self._release = release
27
28 # get all data from database
29 self.__data = data
30
31 def __eq__(self, other):
32 if isinstance(other, self.__class__):
33 return self.id == otherid
34
35 def __lt__(self, other):
36 if isinstance(other, self.__class__):
37 return self.prio < other.prio
38
39 @property
40 def data(self):
41 if self.__data is None:
42 self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id)
43 assert self.__data
44
45 return self.__data
46
47 @property
48 def release(self):
49 if not self._release:
50 release_id = self.data.get("releases")
51 self._release = Release(release_id)
52
53 return self._release
54
55 @property
56 def type(self):
57 filename = self.filename
58
59 if filename.endswith(".iso"):
60 return "iso"
61
62 elif filename.endswith(".torrent"):
63 return "torrent"
64
65 elif "xen" in filename:
66 if "downloader" in filename:
67 return "xen-downloader"
68
69 return "xen"
70
71 elif "sources" in filename:
72 return "source"
73
74 elif "usb-fdd" in filename:
75 return "usbfdd"
76
77 elif "usb-hdd" in filename:
78 return "usbhdd"
79
80 elif "armv5tel" in filename and "scon" in filename:
81 return "armv5tel-scon"
82
83 elif "armv5tel" in filename:
84 return "armv5tel"
85
86 elif "scon" in filename:
87 return "alix"
88
89 elif filename.endswith(".img.gz") or filename.endswith(".img.xz"):
90 return "flash"
91
92 else:
93 return "unknown"
94
95 @property
96 def url(self):
97 return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename)
98
99 @property
100 def desc(self):
101 _ = lambda x: x
102
103 descriptions = {
104 "armv5tel" : _("Flash Image"),
105 "armv5tel-scon" : _("Flash Image with serial console"),
106 "iso" : _("ISO Image"),
107 "torrent" : _("Torrent File"),
108 "flash" : _("Flash Image"),
109 "alix" : _("Flash Image with serial console"),
110 "usbfdd" : _("USB FDD Image"),
111 "usbhdd" : _("USB HDD Image"),
112 "xen" : _("Pre-generated Xen Image"),
113 "xen-downloader": _("Xen-Image Generator"),
114 }
115
116 try:
117 return descriptions[self.type]
118 except KeyError:
119 return _("Unknown image type")
120
121 @property
122 def prio(self):
123 priorities = {
124 "iso" : 10,
125 "torrent" : 20,
126 "flash" : 40,
127 "alix" : 41,
128 "usbfdd" : 31,
129 "usbhdd" : 30,
130 "armv5tel" : 40,
131 "armv5tel-scon" : 41,
132 "xen" : 50,
133 "xen-downloader": 51,
134 }
135
136 try:
137 return priorities[self.type]
138 except KeyError:
139 return 999
140
141 @property
142 def sha256(self):
143 return self.data.get("sha256")
144
145 @property
146 def sha1(self):
147 return self.data.get("sha1")
148
149 @property
150 def filename(self):
151 return self.data.get("filename")
152
153 @property
154 def basename(self):
155 return os.path.basename(self.filename)
156
157 @property
158 def size(self):
159 return self.data.get("filesize")
160
161 @property
162 def arch(self):
163 known_arches = ("x86_64", "aarch64", "arm", "i586")
164
165 for arch in known_arches:
166 if arch in self.basename:
167 return arch
168
169 return "N/A"
170
171 @property
172 def torrent_hash(self):
173 return self.data.get("torrent_hash", None)
174
175 @property
176 def torrent_url(self):
177 if self.torrent_hash:
178 return "%s.torrent" % self.url
179
180 @property
181 def magnet_link(self):
182 # Don't return anything if we have no torrent hash.
183 if self.torrent_hash is None:
184 return
185
186 s = "magnet:?xt=urn:btih:%s" % self.torrent_hash
187
188 #s += "&xl=%d" % self.size
189 s += "&dn=%s" % urllib.parse.quote(self.basename)
190
191 # Add our tracker.
192 for tracker in TRACKERS:
193 s += "&tr=%s" % tracker
194
195 # Add web download URL
196 s += "&as=%s" % urllib.parse.quote(self.url)
197
198 return s
199
200
201 class Release(Object):
202 def __init__(self, backend, id, data=None):
203 Object.__init__(self, backend)
204 self.id = id
205
206 # get all data from database
207 self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id)
208 assert self.__data
209
210 self.__files = []
211
212 def __str__(self):
213 return self.name
214
215 def __repr__(self):
216 return "<%s %s>" % (self.__class__.__name__, self.name)
217
218 def __cmp__(self, other):
219 return cmp(self.id, other.id)
220
221 @property
222 def arches(self):
223 for arch in ("x86_64", "aarch64", "i586", "arm"):
224 if arch in (f.arch for f in self.files):
225 yield arch
226
227 @property
228 def primary_arches(self):
229 arches = []
230
231 # Add x86_64 when available, otherwise add i586
232 for arch in ("x86_64", "i586"):
233 if arch in self.arches:
234 arches.append(arch)
235 break
236
237 # Add ARM if available
238 if "arm" in self.arches:
239 arches.append("arm")
240
241 return arches
242
243 @property
244 def secondary_arches(self):
245 arches = []
246
247 for arch in self.arches:
248 if arch in self.primary_arches:
249 continue
250
251 arches.append(arch)
252
253 return arches
254
255 @property
256 def experimental_arches(self):
257 return ("aarch64",)
258
259 @property
260 def files(self):
261 if not self.__files:
262 files = self.db.query("SELECT * FROM files WHERE releases = %s \
263 AND NOT filename LIKE '%%.torrent'", self.id)
264
265 self.__files = [File(self.backend, self, f.id, f) for f in files]
266 self.__files.sort()
267
268 return self.__files
269
270 def get_files_by_arch(self, arch):
271 for f in self.files:
272 if f.arch == arch:
273 yield f
274
275 @property
276 def torrents(self):
277 torrents = []
278
279 for file in self.files:
280 if not file.torrent_hash:
281 continue
282
283 torrents.append(file)
284
285 return torrents
286
287 @property
288 def name(self):
289 return self.__data.name
290
291 @property
292 def slug(self):
293 return self.__data.sname
294
295 # XXX compat
296 sname = slug
297
298 @lazy_property
299 def blog(self):
300 if self.__data.blog_id:
301 return self.backend.blog.get_by_id(self.__data.blog_id)
302
303 @property
304 def fireinfo_id(self):
305 name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ")
306
307 res = self.db.get("SELECT id FROM fireinfo_releases \
308 WHERE name = %s", name)
309
310 if res:
311 return res.id
312
313 @property
314 def stable(self):
315 return self.__data.stable
316
317 @property
318 def published(self):
319 return self.__data.published
320
321 date = published
322
323 @property
324 def path(self):
325 return self.__data.path
326
327 def get_file(self, type):
328 for file in self.files:
329 if file.type == type:
330 return file
331
332 def __file_hash(self, filename, algo="sha256"):
333 h = hashlib.new(algo)
334
335 with open(filename, "rb") as f:
336 buf_size = 1024
337 buf = f.read(buf_size)
338 while buf:
339 h.update(buf)
340 buf = f.read(buf_size)
341
342 return h.hexdigest()
343
344 def scan_files(self, basepath="/pub/mirror"):
345 if not self.path:
346 return
347
348 path = os.path.join(basepath, self.path)
349 if not os.path.exists(path):
350 return
351
352 files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id)
353 files = [f.filename for f in files]
354
355 # Make files that do not exists not loadable.
356 for filename in files:
357 _filename = os.path.join(basepath, filename)
358 if not os.path.exists(_filename):
359 self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename)
360
361 for filename in os.listdir(path):
362 filename = os.path.join(path, filename)
363
364 if os.path.isdir(filename):
365 continue
366
367 _filename = re.match(".*(releases/.*)", filename).group(1)
368 if _filename in files:
369 continue
370
371 if filename.endswith(".md5"):
372 continue
373
374 logging.info("Hashing %s..." % filename)
375 hash_sha256 = self.__file_hash(filename, "sha256")
376 hash_sha1 = self.__file_hash(filename, "sha1")
377 filesize = os.path.getsize(filename)
378
379 # Check if there is a torrent download available for this file:
380 torrent_hash = ""
381 torrent_file = "%s.torrent" % filename
382 if os.path.exists(torrent_file):
383 torrent_hash = self.torrent_read_hash(torrent_file)
384
385 self.db.execute("INSERT INTO files(releases, filename, filesize, \
386 sha256, sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s, %s)",
387 self.id, _filename, filesize, hash_sha256, hash_sha1, torrent_hash)
388
389 # Search for all files that miss a torrent hash.
390 files = self.db.query("SELECT id, filename FROM files \
391 WHERE releases = %s AND torrent_hash IS NULL", self.id)
392
393 for file in files:
394 path = os.path.join(basepath, file.filename)
395
396 torrent_file = "%s.torrent" % path
397 if os.path.exists(torrent_file):
398 torrent_hash = self.torrent_read_hash(torrent_file)
399
400 self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s",
401 torrent_hash, file.id)
402
403 def torrent_read_hash(self, filename):
404 with open(filename, "rb") as f:
405 metainfo = yabencode.decode(f)
406 metainfo = yabencode.encode(metainfo["info"])
407
408 h = hashlib.new("sha1")
409 h.update(metainfo)
410
411 return h.hexdigest()
412
413 def supports_arch(self, arch):
414 return arch in ("x86_64", "i586")
415
416 def supports_platform(self, platform):
417 # Currently there is nothing else than pcbios supported
418 if platform == "pcbios":
419 return True
420
421 return False
422
423 def is_netboot_capable(self):
424 return self.path and "ipfire-2.x" in self.path
425
426 def netboot_kernel_url(self, arch, platform):
427 assert self.supports_arch(arch)
428 assert self.supports_platform(platform)
429
430 if self.sname >= "ipfire-2.19-core100":
431 return "http://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch)
432
433 return "http://boot.ipfire.org/%s/images/vmlinuz" % self.path
434
435 def netboot_initrd_url(self, arch, platform):
436 assert self.supports_arch(arch)
437 assert self.supports_platform(platform)
438
439 if self.sname >= "ipfire-2.19-core100":
440 return "http://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch)
441
442 return "http://boot.ipfire.org/%s/images/instroot" % self.path
443
444 def netboot_args(self, arch, platform):
445 return ""
446
447 @property
448 def post(self):
449 if self.__data.blog_id:
450 return self.backend.blog.get_by_id(self.__data.blog_id)
451
452 # Fireinfo Stuff
453
454 @property
455 def penetration(self):
456 # Get penetration from fireinfo
457 return self.backend.fireinfo.get_release_penetration(self)
458
459
460 class Releases(Object):
461 def _get_release(self, query, *args):
462 res = self.db.get(query, *args)
463
464 if res:
465 return Release(self.backend, res.id, data=res)
466
467 def _get_releases(self, query, *args):
468 res = self.db.query(query, *args)
469
470 for row in res:
471 yield Release(self.backend, row.id, data=row)
472
473 def __iter__(self):
474 releases = self._get_releases("SELECT * FROM releases \
475 ORDER BY published DESC NULLS FIRST")
476
477 return iter(releases)
478
479 def get_by_id(self, id):
480 ret = self.db.get("SELECT * FROM releases WHERE id = %s", id)
481
482 if ret:
483 return Release(self.backend, ret.id, data=ret)
484
485 def get_by_sname(self, sname):
486 ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname)
487
488 if ret:
489 return Release(self.backend, ret.id, data=ret)
490
491 def get_by_news_id(self, news_id):
492 ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id)
493
494 if ret:
495 return Release(self.backend, ret.id, data=ret)
496
497 def get_latest(self, stable=True):
498 ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \
499 AND stable = %s ORDER BY published DESC LIMIT 1", stable)
500
501 if ret:
502 return Release(self.backend, ret.id, data=ret)
503
504 def get_releases_older_than(self, release, limit=None):
505 return self._get_releases("SELECT * FROM releases \
506 WHERE published IS NOT NULL AND published < %s \
507 ORDER BY published DESC LIMIT %s", release.published, limit)
508
509 def get_latest_unstable(self):
510 ret = self.db.get("SELECT * FROM releases r1 \
511 WHERE r1.published IS NOT NULL AND r1.published <= NOW() \
512 AND stable = %s AND NOT EXISTS ( \
513 SELECT * FROM releases r2 WHERE r2.stable = %s AND \
514 r2.published IS NOT NULL AND r2.published >= r1.published \
515 ) ORDER BY r1.published DESC LIMIT 1", False, True)
516
517 if ret:
518 return Release(self.backend, ret.id, data=ret)
519
520 def get_stable(self):
521 query = self.db.query("SELECT * FROM releases \
522 WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \
523 ORDER BY published DESC")
524
525 releases = []
526 for row in query:
527 release = Release(self.backend, row.id, data=row)
528 releases.append(release)
529
530 return releases
531
532 def get_unstable(self):
533 query = self.db.query("SELECT * FROM releases \
534 WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \
535 ORDER BY published DESC")
536
537 releases = []
538 for row in query:
539 release = Release(self.backend, row.id, data=row)
540 releases.append(release)
541
542 return releases
543
544 def get_all(self):
545 query = self.db.query("SELECT * FROM releases \
546 WHERE published IS NOT NULL AND published <= NOW() \
547 ORDER BY published DESC")
548
549 releases = []
550 for row in query:
551 release = Release(self.backend, row.id, data=row)
552 releases.append(release)
553
554 return releases
555
556 def _get_all(self):
557 query = self.db.query("SELECT * FROM releases ORDER BY published DESC")
558
559 releases = []
560 for row in query:
561 release = Release(self.backend, row.id, data=row)
562 releases.append(release)
563
564 return releases
565
566 def get_file_for_torrent_hash(self, torrent_hash):
567 file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1",
568 torrent_hash)
569
570 if not file:
571 return
572
573 release = Release(self.backend, file.releases)
574 file = File(self.backend, release, file.id)
575
576 return file
577
578 async def scan_files(self, basepath="/pub/mirror"):
579 for release in self:
580 logging.debug("Scanning %s..." % release)
581
582 with self.db.transaction():
583 release.scan_files(basepath=basepath)