]>
Commit | Line | Data |
---|---|---|
1 | #!/usr/bin/python | |
2 | ||
3 | import hashlib | |
4 | import logging | |
5 | import os | |
6 | import re | |
7 | import urllib.parse | |
8 | import yabencode | |
9 | ||
10 | from . import database | |
11 | from .misc import Object | |
12 | from .decorators import * | |
13 | ||
14 | TRACKERS = ( | |
15 | "http://ipv4.tracker.ipfire.org:6969/announce", | |
16 | "udp://ipv4.tracker.ipfire.org:6969", | |
17 | "http://ipv6.tracker.ipfire.org:6969/announce", | |
18 | "udp://ipv6.tracker.ipfire.org:6969", | |
19 | ) | |
20 | ||
21 | class File(Object): | |
22 | def __init__(self, backend, release, id, data=None): | |
23 | Object.__init__(self, backend) | |
24 | ||
25 | self.id = id | |
26 | self._release = release | |
27 | ||
28 | # get all data from database | |
29 | self.__data = data | |
30 | ||
31 | def __eq__(self, other): | |
32 | if isinstance(other, self.__class__): | |
33 | return self.id == otherid | |
34 | ||
35 | def __lt__(self, other): | |
36 | if isinstance(other, self.__class__): | |
37 | return self.prio < other.prio | |
38 | ||
39 | @property | |
40 | def data(self): | |
41 | if self.__data is None: | |
42 | self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id) | |
43 | assert self.__data | |
44 | ||
45 | return self.__data | |
46 | ||
47 | @property | |
48 | def release(self): | |
49 | if not self._release: | |
50 | release_id = self.data.get("releases") | |
51 | self._release = Release(release_id) | |
52 | ||
53 | return self._release | |
54 | ||
55 | @property | |
56 | def type(self): | |
57 | filename = self.filename | |
58 | ||
59 | if filename.endswith(".iso"): | |
60 | return "iso" | |
61 | ||
62 | elif filename.endswith(".torrent"): | |
63 | return "torrent" | |
64 | ||
65 | elif "xen" in filename: | |
66 | if "downloader" in filename: | |
67 | return "xen-downloader" | |
68 | ||
69 | return "xen" | |
70 | ||
71 | elif "sources" in filename: | |
72 | return "source" | |
73 | ||
74 | elif "usb-fdd" in filename: | |
75 | return "usbfdd" | |
76 | ||
77 | elif "usb-hdd" in filename: | |
78 | return "usbhdd" | |
79 | ||
80 | elif "armv5tel" in filename and "scon" in filename: | |
81 | return "armv5tel-scon" | |
82 | ||
83 | elif "armv5tel" in filename: | |
84 | return "armv5tel" | |
85 | ||
86 | elif "scon" in filename: | |
87 | return "alix" | |
88 | ||
89 | elif filename.endswith(".img.gz") or filename.endswith(".img.xz"): | |
90 | return "flash" | |
91 | ||
92 | else: | |
93 | return "unknown" | |
94 | ||
95 | @property | |
96 | def url(self): | |
97 | return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename) | |
98 | ||
99 | @property | |
100 | def desc(self): | |
101 | _ = lambda x: x | |
102 | ||
103 | descriptions = { | |
104 | "armv5tel" : _("Flash Image"), | |
105 | "armv5tel-scon" : _("Flash Image with serial console"), | |
106 | "iso" : _("ISO Image"), | |
107 | "torrent" : _("Torrent File"), | |
108 | "flash" : _("Flash Image"), | |
109 | "alix" : _("Flash Image with serial console"), | |
110 | "usbfdd" : _("USB FDD Image"), | |
111 | "usbhdd" : _("USB HDD Image"), | |
112 | "xen" : _("Pre-generated Xen Image"), | |
113 | "xen-downloader": _("Xen-Image Generator"), | |
114 | } | |
115 | ||
116 | try: | |
117 | return descriptions[self.type] | |
118 | except KeyError: | |
119 | return _("Unknown image type") | |
120 | ||
121 | @property | |
122 | def prio(self): | |
123 | priorities = { | |
124 | "iso" : 10, | |
125 | "torrent" : 20, | |
126 | "flash" : 40, | |
127 | "alix" : 41, | |
128 | "usbfdd" : 31, | |
129 | "usbhdd" : 30, | |
130 | "armv5tel" : 40, | |
131 | "armv5tel-scon" : 41, | |
132 | "xen" : 50, | |
133 | "xen-downloader": 51, | |
134 | } | |
135 | ||
136 | try: | |
137 | return priorities[self.type] | |
138 | except KeyError: | |
139 | return 999 | |
140 | ||
141 | @property | |
142 | def sha256(self): | |
143 | return self.data.get("sha256") | |
144 | ||
145 | @property | |
146 | def sha1(self): | |
147 | return self.data.get("sha1") | |
148 | ||
149 | @property | |
150 | def filename(self): | |
151 | return self.data.get("filename") | |
152 | ||
153 | @property | |
154 | def basename(self): | |
155 | return os.path.basename(self.filename) | |
156 | ||
157 | @property | |
158 | def size(self): | |
159 | return self.data.get("filesize") | |
160 | ||
161 | @property | |
162 | def arch(self): | |
163 | known_arches = ("x86_64", "aarch64", "arm", "i586") | |
164 | ||
165 | for arch in known_arches: | |
166 | if arch in self.basename: | |
167 | return arch | |
168 | ||
169 | return "N/A" | |
170 | ||
171 | @property | |
172 | def torrent_hash(self): | |
173 | return self.data.get("torrent_hash", None) | |
174 | ||
175 | @property | |
176 | def torrent_url(self): | |
177 | if self.torrent_hash: | |
178 | return "%s.torrent" % self.url | |
179 | ||
180 | @property | |
181 | def magnet_link(self): | |
182 | # Don't return anything if we have no torrent hash. | |
183 | if self.torrent_hash is None: | |
184 | return | |
185 | ||
186 | s = "magnet:?xt=urn:btih:%s" % self.torrent_hash | |
187 | ||
188 | #s += "&xl=%d" % self.size | |
189 | s += "&dn=%s" % urllib.parse.quote(self.basename) | |
190 | ||
191 | # Add our tracker. | |
192 | for tracker in TRACKERS: | |
193 | s += "&tr=%s" % tracker | |
194 | ||
195 | # Add web download URL | |
196 | s += "&as=%s" % urllib.parse.quote(self.url) | |
197 | ||
198 | return s | |
199 | ||
200 | ||
201 | class Release(Object): | |
202 | def __init__(self, backend, id, data=None): | |
203 | Object.__init__(self, backend) | |
204 | self.id = id | |
205 | ||
206 | # get all data from database | |
207 | self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id) | |
208 | assert self.__data | |
209 | ||
210 | self.__files = [] | |
211 | ||
212 | def __str__(self): | |
213 | return self.name | |
214 | ||
215 | def __repr__(self): | |
216 | return "<%s %s>" % (self.__class__.__name__, self.name) | |
217 | ||
218 | def __cmp__(self, other): | |
219 | return cmp(self.id, other.id) | |
220 | ||
221 | @property | |
222 | def arches(self): | |
223 | for arch in ("x86_64", "aarch64", "i586", "arm"): | |
224 | if arch in (f.arch for f in self.files): | |
225 | yield arch | |
226 | ||
227 | @property | |
228 | def primary_arches(self): | |
229 | arches = [] | |
230 | ||
231 | # Add x86_64 when available, otherwise add i586 | |
232 | for arch in ("x86_64", "i586"): | |
233 | if arch in self.arches: | |
234 | arches.append(arch) | |
235 | break | |
236 | ||
237 | # Add aarch64 if available | |
238 | if "aarch64" in self.arches: | |
239 | arches.append("aarch64") | |
240 | ||
241 | # Add ARM before 2.27 if available | |
242 | if "arm" in self.arches and self.sname < "ipfire-2.27-core159": | |
243 | arches.append("arm") | |
244 | ||
245 | return arches | |
246 | ||
247 | @property | |
248 | def secondary_arches(self): | |
249 | arches = [] | |
250 | ||
251 | for arch in self.arches: | |
252 | if arch in self.primary_arches: | |
253 | continue | |
254 | ||
255 | arches.append(arch) | |
256 | ||
257 | return arches | |
258 | ||
259 | @property | |
260 | def experimental_arches(self): | |
261 | return [] | |
262 | ||
263 | @property | |
264 | def files(self): | |
265 | if not self.__files: | |
266 | files = self.db.query("SELECT * FROM files WHERE releases = %s \ | |
267 | AND NOT filename LIKE '%%.torrent'", self.id) | |
268 | ||
269 | self.__files = [File(self.backend, self, f.id, f) for f in files] | |
270 | self.__files.sort() | |
271 | ||
272 | return self.__files | |
273 | ||
274 | def get_files_by_arch(self, arch): | |
275 | for f in self.files: | |
276 | if f.arch == arch: | |
277 | yield f | |
278 | ||
279 | @property | |
280 | def torrents(self): | |
281 | torrents = [] | |
282 | ||
283 | for file in self.files: | |
284 | if not file.torrent_hash: | |
285 | continue | |
286 | ||
287 | torrents.append(file) | |
288 | ||
289 | return torrents | |
290 | ||
291 | @property | |
292 | def name(self): | |
293 | return self.__data.name | |
294 | ||
295 | @property | |
296 | def slug(self): | |
297 | return self.__data.sname | |
298 | ||
299 | # XXX compat | |
300 | sname = slug | |
301 | ||
302 | @lazy_property | |
303 | def blog(self): | |
304 | if self.__data.blog_id: | |
305 | return self.backend.blog.get_by_id(self.__data.blog_id) | |
306 | ||
307 | @property | |
308 | def fireinfo_id(self): | |
309 | name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ") | |
310 | ||
311 | res = self.db.get("SELECT id FROM fireinfo_releases \ | |
312 | WHERE name = %s", name) | |
313 | ||
314 | if res: | |
315 | return res.id | |
316 | ||
317 | @property | |
318 | def stable(self): | |
319 | return self.__data.stable | |
320 | ||
321 | @property | |
322 | def published(self): | |
323 | return self.__data.published | |
324 | ||
325 | date = published | |
326 | ||
327 | @property | |
328 | def path(self): | |
329 | return self.__data.path | |
330 | ||
331 | def get_file(self, type): | |
332 | for file in self.files: | |
333 | if file.type == type: | |
334 | return file | |
335 | ||
336 | def __file_hash(self, filename, algo="sha256"): | |
337 | h = hashlib.new(algo) | |
338 | ||
339 | with open(filename, "rb") as f: | |
340 | buf_size = 1024 | |
341 | buf = f.read(buf_size) | |
342 | while buf: | |
343 | h.update(buf) | |
344 | buf = f.read(buf_size) | |
345 | ||
346 | return h.hexdigest() | |
347 | ||
348 | def scan_files(self, basepath="/pub/mirror"): | |
349 | if not self.path: | |
350 | return | |
351 | ||
352 | path = os.path.join(basepath, self.path) | |
353 | if not os.path.exists(path): | |
354 | return | |
355 | ||
356 | files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id) | |
357 | files = [f.filename for f in files] | |
358 | ||
359 | # Make files that do not exists not loadable. | |
360 | for filename in files: | |
361 | _filename = os.path.join(basepath, filename) | |
362 | if not os.path.exists(_filename): | |
363 | self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename) | |
364 | ||
365 | for filename in os.listdir(path): | |
366 | filename = os.path.join(path, filename) | |
367 | ||
368 | if os.path.isdir(filename): | |
369 | continue | |
370 | ||
371 | _filename = re.match(".*(releases/.*)", filename).group(1) | |
372 | if _filename in files: | |
373 | continue | |
374 | ||
375 | if filename.endswith(".md5"): | |
376 | continue | |
377 | ||
378 | logging.info("Hashing %s..." % filename) | |
379 | hash_sha256 = self.__file_hash(filename, "sha256") | |
380 | hash_sha1 = self.__file_hash(filename, "sha1") | |
381 | filesize = os.path.getsize(filename) | |
382 | ||
383 | # Check if there is a torrent download available for this file: | |
384 | torrent_hash = "" | |
385 | torrent_file = "%s.torrent" % filename | |
386 | if os.path.exists(torrent_file): | |
387 | torrent_hash = self.torrent_read_hash(torrent_file) | |
388 | ||
389 | self.db.execute("INSERT INTO files(releases, filename, filesize, \ | |
390 | sha256, sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s, %s)", | |
391 | self.id, _filename, filesize, hash_sha256, hash_sha1, torrent_hash) | |
392 | ||
393 | # Search for all files that miss a torrent hash. | |
394 | files = self.db.query("SELECT id, filename FROM files \ | |
395 | WHERE releases = %s AND torrent_hash IS NULL", self.id) | |
396 | ||
397 | for file in files: | |
398 | path = os.path.join(basepath, file.filename) | |
399 | ||
400 | torrent_file = "%s.torrent" % path | |
401 | if os.path.exists(torrent_file): | |
402 | torrent_hash = self.torrent_read_hash(torrent_file) | |
403 | ||
404 | self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s", | |
405 | torrent_hash, file.id) | |
406 | ||
407 | def torrent_read_hash(self, filename): | |
408 | with open(filename, "rb") as f: | |
409 | metainfo = yabencode.decode(f) | |
410 | metainfo = yabencode.encode(metainfo["info"]) | |
411 | ||
412 | h = hashlib.new("sha1") | |
413 | h.update(metainfo) | |
414 | ||
415 | return h.hexdigest() | |
416 | ||
417 | def supports_arch(self, arch): | |
418 | return arch in ("x86_64", "i586") | |
419 | ||
420 | def supports_platform(self, platform): | |
421 | # Currently there is nothing else than pcbios supported | |
422 | if platform == "pcbios": | |
423 | return True | |
424 | ||
425 | return False | |
426 | ||
427 | def is_netboot_capable(self): | |
428 | return self.path and "ipfire-2.x" in self.path | |
429 | ||
430 | def netboot_kernel_url(self, arch, platform): | |
431 | assert self.supports_arch(arch) | |
432 | assert self.supports_platform(platform) | |
433 | ||
434 | if self.sname >= "ipfire-2.19-core100": | |
435 | return "http://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch) | |
436 | ||
437 | return "http://boot.ipfire.org/%s/images/vmlinuz" % self.path | |
438 | ||
439 | def netboot_initrd_url(self, arch, platform): | |
440 | assert self.supports_arch(arch) | |
441 | assert self.supports_platform(platform) | |
442 | ||
443 | if self.sname >= "ipfire-2.19-core100": | |
444 | return "http://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch) | |
445 | ||
446 | return "http://boot.ipfire.org/%s/images/instroot" % self.path | |
447 | ||
448 | def netboot_args(self, arch, platform): | |
449 | return "" | |
450 | ||
451 | @property | |
452 | def post(self): | |
453 | if self.__data.blog_id: | |
454 | return self.backend.blog.get_by_id(self.__data.blog_id) | |
455 | ||
456 | # Fireinfo Stuff | |
457 | ||
458 | @property | |
459 | def penetration(self): | |
460 | # Get penetration from fireinfo | |
461 | return self.backend.fireinfo.get_release_penetration(self) | |
462 | ||
463 | ||
464 | class Releases(Object): | |
465 | def _get_release(self, query, *args): | |
466 | res = self.db.get(query, *args) | |
467 | ||
468 | if res: | |
469 | return Release(self.backend, res.id, data=res) | |
470 | ||
471 | def _get_releases(self, query, *args): | |
472 | res = self.db.query(query, *args) | |
473 | ||
474 | for row in res: | |
475 | yield Release(self.backend, row.id, data=row) | |
476 | ||
477 | def __iter__(self): | |
478 | releases = self._get_releases("SELECT * FROM releases \ | |
479 | ORDER BY published DESC NULLS FIRST") | |
480 | ||
481 | return iter(releases) | |
482 | ||
483 | def get_by_id(self, id): | |
484 | ret = self.db.get("SELECT * FROM releases WHERE id = %s", id) | |
485 | ||
486 | if ret: | |
487 | return Release(self.backend, ret.id, data=ret) | |
488 | ||
489 | def get_by_sname(self, sname): | |
490 | ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname) | |
491 | ||
492 | if ret: | |
493 | return Release(self.backend, ret.id, data=ret) | |
494 | ||
495 | def get_by_news_id(self, news_id): | |
496 | ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id) | |
497 | ||
498 | if ret: | |
499 | return Release(self.backend, ret.id, data=ret) | |
500 | ||
501 | def get_latest(self, stable=True): | |
502 | ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \ | |
503 | AND stable = %s ORDER BY published DESC LIMIT 1", stable) | |
504 | ||
505 | if ret: | |
506 | return Release(self.backend, ret.id, data=ret) | |
507 | ||
508 | def get_releases_older_than(self, release, limit=None): | |
509 | return self._get_releases("SELECT * FROM releases \ | |
510 | WHERE published IS NOT NULL AND published < %s \ | |
511 | ORDER BY published DESC LIMIT %s", release.published, limit) | |
512 | ||
513 | def get_latest_unstable(self): | |
514 | ret = self.db.get("SELECT * FROM releases r1 \ | |
515 | WHERE r1.published IS NOT NULL AND r1.published <= NOW() \ | |
516 | AND stable = %s AND NOT EXISTS ( \ | |
517 | SELECT * FROM releases r2 WHERE r2.stable = %s AND \ | |
518 | r2.published IS NOT NULL AND r2.published >= r1.published \ | |
519 | ) ORDER BY r1.published DESC LIMIT 1", False, True) | |
520 | ||
521 | if ret: | |
522 | return Release(self.backend, ret.id, data=ret) | |
523 | ||
524 | def get_stable(self): | |
525 | query = self.db.query("SELECT * FROM releases \ | |
526 | WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \ | |
527 | ORDER BY published DESC") | |
528 | ||
529 | releases = [] | |
530 | for row in query: | |
531 | release = Release(self.backend, row.id, data=row) | |
532 | releases.append(release) | |
533 | ||
534 | return releases | |
535 | ||
536 | def get_unstable(self): | |
537 | query = self.db.query("SELECT * FROM releases \ | |
538 | WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \ | |
539 | ORDER BY published DESC") | |
540 | ||
541 | releases = [] | |
542 | for row in query: | |
543 | release = Release(self.backend, row.id, data=row) | |
544 | releases.append(release) | |
545 | ||
546 | return releases | |
547 | ||
548 | def get_all(self): | |
549 | query = self.db.query("SELECT * FROM releases \ | |
550 | WHERE published IS NOT NULL AND published <= NOW() \ | |
551 | ORDER BY published DESC") | |
552 | ||
553 | releases = [] | |
554 | for row in query: | |
555 | release = Release(self.backend, row.id, data=row) | |
556 | releases.append(release) | |
557 | ||
558 | return releases | |
559 | ||
560 | def _get_all(self): | |
561 | query = self.db.query("SELECT * FROM releases ORDER BY published DESC") | |
562 | ||
563 | releases = [] | |
564 | for row in query: | |
565 | release = Release(self.backend, row.id, data=row) | |
566 | releases.append(release) | |
567 | ||
568 | return releases | |
569 | ||
570 | def get_file_for_torrent_hash(self, torrent_hash): | |
571 | file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1", | |
572 | torrent_hash) | |
573 | ||
574 | if not file: | |
575 | return | |
576 | ||
577 | release = Release(self.backend, file.releases) | |
578 | file = File(self.backend, release, file.id) | |
579 | ||
580 | return file | |
581 | ||
582 | async def scan_files(self, basepath="/pub/mirror"): | |
583 | for release in self: | |
584 | logging.debug("Scanning %s..." % release) | |
585 | ||
586 | with self.db.transaction(): | |
587 | release.scan_files(basepath=basepath) |