]>
Commit | Line | Data |
---|---|---|
940227cb MT |
1 | #!/usr/bin/python |
2 | ||
3c4f2edc | 3 | import hashlib |
940227cb | 4 | import logging |
3c4f2edc MT |
5 | import os |
6 | import re | |
856748b0 | 7 | import tornado.gen |
11347e46 | 8 | import urllib.parse |
3ec6114d | 9 | import yabencode |
940227cb | 10 | |
11347e46 | 11 | from . import database |
11347e46 | 12 | from .misc import Object |
45c7049d | 13 | from .decorators import * |
fadcfd00 | 14 | |
f7635119 MT |
15 | TRACKERS = ( |
16 | "http://ipv4.tracker.ipfire.org:6969/announce", | |
17 | "udp://ipv4.tracker.ipfire.org:6969", | |
18 | "http://ipv6.tracker.ipfire.org:6969/announce", | |
19 | "udp://ipv6.tracker.ipfire.org:6969", | |
20 | ) | |
21 | ||
9068dba1 MT |
22 | class File(Object): |
23 | def __init__(self, backend, release, id, data=None): | |
24 | Object.__init__(self, backend) | |
940227cb | 25 | |
938d083d MT |
26 | self.id = id |
27 | self._release = release | |
940227cb MT |
28 | |
29 | # get all data from database | |
9068dba1 | 30 | self.__data = data |
940227cb | 31 | |
5ecd3be7 MT |
32 | def __eq__(self, other): |
33 | if isinstance(other, self.__class__): | |
34 | return self.id == otherid | |
35 | ||
36 | def __lt__(self, other): | |
37 | if isinstance(other, self.__class__): | |
38 | return self.prio < other.prio | |
938d083d MT |
39 | |
40 | @property | |
41 | def data(self): | |
42 | if self.__data is None: | |
43 | self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id) | |
44 | assert self.__data | |
45 | ||
46 | return self.__data | |
47 | ||
48 | @property | |
49 | def release(self): | |
50 | if not self._release: | |
51 | release_id = self.data.get("releases") | |
52 | self._release = Release(release_id) | |
53 | ||
54 | return self._release | |
940227cb MT |
55 | |
56 | @property | |
57 | def type(self): | |
fadcfd00 MT |
58 | filename = self.filename |
59 | ||
60 | if filename.endswith(".iso"): | |
61 | return "iso" | |
62 | ||
63 | elif filename.endswith(".torrent"): | |
64 | return "torrent" | |
65 | ||
66 | elif "xen" in filename: | |
80594ae3 MT |
67 | if "downloader" in filename: |
68 | return "xen-downloader" | |
69 | ||
fadcfd00 MT |
70 | return "xen" |
71 | ||
72 | elif "sources" in filename: | |
73 | return "source" | |
74 | ||
75 | elif "usb-fdd" in filename: | |
76 | return "usbfdd" | |
77 | ||
78 | elif "usb-hdd" in filename: | |
79 | return "usbhdd" | |
80 | ||
3230a47f MT |
81 | elif "armv5tel" in filename and "scon" in filename: |
82 | return "armv5tel-scon" | |
83 | ||
fadcfd00 MT |
84 | elif "armv5tel" in filename: |
85 | return "armv5tel" | |
86 | ||
87 | elif "scon" in filename: | |
88 | return "alix" | |
89 | ||
116312c5 | 90 | elif filename.endswith(".img.gz") or filename.endswith(".img.xz"): |
fadcfd00 MT |
91 | return "flash" |
92 | ||
93 | else: | |
94 | return "unknown" | |
940227cb MT |
95 | |
96 | @property | |
97 | def url(self): | |
eea71144 | 98 | return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename) |
940227cb MT |
99 | |
100 | @property | |
101 | def desc(self): | |
102 | _ = lambda x: x | |
103 | ||
104 | descriptions = { | |
e77cd04c MT |
105 | "armv5tel" : _("Flash Image"), |
106 | "armv5tel-scon" : _("Flash Image with serial console"), | |
a47f3c78 | 107 | "iso" : _("ISO Image"), |
e77cd04c MT |
108 | "torrent" : _("Torrent File"), |
109 | "flash" : _("Flash Image"), | |
110 | "alix" : _("Flash Image with serial console"), | |
940227cb MT |
111 | "usbfdd" : _("USB FDD Image"), |
112 | "usbhdd" : _("USB HDD Image"), | |
e77cd04c | 113 | "xen" : _("Pre-generated Xen Image"), |
80594ae3 | 114 | "xen-downloader": _("Xen-Image Generator"), |
940227cb MT |
115 | } |
116 | ||
117 | try: | |
118 | return descriptions[self.type] | |
119 | except KeyError: | |
120 | return _("Unknown image type") | |
121 | ||
122 | @property | |
123 | def prio(self): | |
124 | priorities = { | |
125 | "iso" : 10, | |
126 | "torrent" : 20, | |
127 | "flash" : 40, | |
128 | "alix" : 41, | |
129 | "usbfdd" : 31, | |
130 | "usbhdd" : 30, | |
3230a47f MT |
131 | "armv5tel" : 40, |
132 | "armv5tel-scon" : 41, | |
940227cb | 133 | "xen" : 50, |
80594ae3 | 134 | "xen-downloader": 51, |
940227cb MT |
135 | } |
136 | ||
137 | try: | |
138 | return priorities[self.type] | |
139 | except KeyError: | |
140 | return 999 | |
141 | ||
752c8888 MT |
142 | @property |
143 | def sha256(self): | |
144 | return self.data.get("sha256") | |
145 | ||
940227cb MT |
146 | @property |
147 | def sha1(self): | |
938d083d | 148 | return self.data.get("sha1") |
940227cb MT |
149 | |
150 | @property | |
151 | def filename(self): | |
938d083d | 152 | return self.data.get("filename") |
940227cb | 153 | |
c9698a0b MT |
154 | @property |
155 | def basename(self): | |
156 | return os.path.basename(self.filename) | |
157 | ||
60024cc8 MT |
158 | @property |
159 | def size(self): | |
938d083d | 160 | return self.data.get("filesize") |
60024cc8 MT |
161 | |
162 | @property | |
163 | def arch(self): | |
6371bfda | 164 | known_arches = ("x86_64", "i586", "arm") |
60024cc8 MT |
165 | |
166 | for arch in known_arches: | |
167 | if arch in self.basename: | |
168 | return arch | |
169 | ||
fadcfd00 MT |
170 | return "N/A" |
171 | ||
172 | @property | |
173 | def torrent_hash(self): | |
938d083d | 174 | return self.data.get("torrent_hash", None) |
fadcfd00 | 175 | |
a69e87a1 MT |
176 | @property |
177 | def torrent_url(self): | |
178 | if self.torrent_hash: | |
179 | return "%s.torrent" % self.url | |
5276cd41 | 180 | |
fadcfd00 MT |
181 | @property |
182 | def magnet_link(self): | |
183 | # Don't return anything if we have no torrent hash. | |
184 | if self.torrent_hash is None: | |
185 | return | |
186 | ||
187 | s = "magnet:?xt=urn:btih:%s" % self.torrent_hash | |
188 | ||
189 | #s += "&xl=%d" % self.size | |
11347e46 | 190 | s += "&dn=%s" % urllib.parse.quote(self.basename) |
fadcfd00 MT |
191 | |
192 | # Add our tracker. | |
a69e87a1 MT |
193 | for tracker in TRACKERS: |
194 | s += "&tr=%s" % tracker | |
fadcfd00 | 195 | |
a69e87a1 MT |
196 | # Add web download URL |
197 | s += "&as=%s" % urllib.parse.quote(self.url) | |
dc7b8879 | 198 | |
fadcfd00 | 199 | return s |
60024cc8 | 200 | |
940227cb | 201 | |
9068dba1 MT |
202 | class Release(Object): |
203 | def __init__(self, backend, id, data=None): | |
204 | Object.__init__(self, backend) | |
940227cb MT |
205 | self.id = id |
206 | ||
207 | # get all data from database | |
9068dba1 | 208 | self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id) |
940227cb MT |
209 | assert self.__data |
210 | ||
211 | self.__files = [] | |
212 | ||
e77cd04c MT |
213 | def __str__(self): |
214 | return self.name | |
215 | ||
940227cb MT |
216 | def __repr__(self): |
217 | return "<%s %s>" % (self.__class__.__name__, self.name) | |
218 | ||
9068dba1 MT |
219 | def __cmp__(self, other): |
220 | return cmp(self.id, other.id) | |
221 | ||
e77cd04c MT |
222 | @property |
223 | def arches(self): | |
224 | for arch in ("x86_64", "aarch64", "i586", "arm"): | |
225 | if arch in (f.arch for f in self.files): | |
226 | yield arch | |
227 | ||
940227cb MT |
228 | @property |
229 | def files(self): | |
230 | if not self.__files: | |
9068dba1 MT |
231 | files = self.db.query("SELECT * FROM files WHERE releases = %s \ |
232 | AND NOT filename LIKE '%%.torrent'", self.id) | |
940227cb | 233 | |
9068dba1 MT |
234 | self.__files = [File(self.backend, self, f.id, f) for f in files] |
235 | self.__files.sort() | |
940227cb MT |
236 | |
237 | return self.__files | |
238 | ||
e77cd04c MT |
239 | def get_files_by_arch(self, arch): |
240 | for f in self.files: | |
241 | if f.arch == arch: | |
242 | yield f | |
243 | ||
938d083d MT |
244 | @property |
245 | def torrents(self): | |
246 | torrents = [] | |
247 | ||
248 | for file in self.files: | |
249 | if not file.torrent_hash: | |
250 | continue | |
251 | ||
252 | torrents.append(file) | |
253 | ||
254 | return torrents | |
255 | ||
940227cb MT |
256 | @property |
257 | def name(self): | |
9068dba1 MT |
258 | return self.__data.name |
259 | ||
260 | @property | |
e77cd04c | 261 | def slug(self): |
9068dba1 | 262 | return self.__data.sname |
940227cb | 263 | |
e77cd04c MT |
264 | # XXX compat |
265 | sname = slug | |
266 | ||
45c7049d | 267 | @lazy_property |
e77cd04c MT |
268 | def blog(self): |
269 | if self.__data.blog_id: | |
270 | return self.backend.blog.get_by_id(self.__data.blog_id) | |
271 | ||
dd3a5446 MT |
272 | @property |
273 | def fireinfo_id(self): | |
274 | name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ") | |
275 | ||
276 | res = self.db.get("SELECT id FROM fireinfo_releases \ | |
277 | WHERE name = %s", name) | |
278 | ||
279 | if res: | |
280 | return res.id | |
281 | ||
940227cb MT |
282 | @property |
283 | def stable(self): | |
9068dba1 | 284 | return self.__data.stable |
940227cb MT |
285 | |
286 | @property | |
287 | def published(self): | |
9068dba1 | 288 | return self.__data.published |
940227cb | 289 | |
9068dba1 | 290 | date = published |
940227cb | 291 | |
3c4f2edc MT |
292 | @property |
293 | def path(self): | |
9068dba1 | 294 | return self.__data.path |
3c4f2edc | 295 | |
940227cb MT |
296 | def get_file(self, type): |
297 | for file in self.files: | |
298 | if file.type == type: | |
299 | return file | |
300 | ||
8b99a680 MT |
301 | def __file_hash(self, filename, algo="sha256"): |
302 | h = hashlib.new(algo) | |
3c4f2edc | 303 | |
3ca4157e | 304 | with open(filename, "rb") as f: |
b5f4eef0 MT |
305 | buf_size = 1024 |
306 | buf = f.read(buf_size) | |
307 | while buf: | |
8b99a680 | 308 | h.update(buf) |
b5f4eef0 | 309 | buf = f.read(buf_size) |
3c4f2edc | 310 | |
8b99a680 | 311 | return h.hexdigest() |
3c4f2edc | 312 | |
e136d583 | 313 | def scan_files(self, basepath="/pub/mirror"): |
3c4f2edc MT |
314 | if not self.path: |
315 | return | |
316 | ||
317 | path = os.path.join(basepath, self.path) | |
921d98cc MT |
318 | if not os.path.exists(path): |
319 | return | |
320 | ||
fadcfd00 MT |
321 | files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id) |
322 | files = [f.filename for f in files] | |
3c4f2edc MT |
323 | |
324 | # Make files that do not exists not loadable. | |
325 | for filename in files: | |
326 | _filename = os.path.join(basepath, filename) | |
327 | if not os.path.exists(_filename): | |
328 | self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename) | |
329 | ||
330 | for filename in os.listdir(path): | |
331 | filename = os.path.join(path, filename) | |
332 | ||
333 | if os.path.isdir(filename): | |
334 | continue | |
335 | ||
336 | _filename = re.match(".*(releases/.*)", filename).group(1) | |
337 | if _filename in files: | |
338 | continue | |
339 | ||
340 | if filename.endswith(".md5"): | |
341 | continue | |
342 | ||
9068dba1 | 343 | logging.info("Hashing %s..." % filename) |
8b99a680 MT |
344 | hash_sha256 = self.__file_hash(filename, "sha256") |
345 | hash_sha1 = self.__file_hash(filename, "sha1") | |
3c4f2edc | 346 | filesize = os.path.getsize(filename) |
3c4f2edc | 347 | |
fadcfd00 MT |
348 | # Check if there is a torrent download available for this file: |
349 | torrent_hash = "" | |
350 | torrent_file = "%s.torrent" % filename | |
351 | if os.path.exists(torrent_file): | |
352 | torrent_hash = self.torrent_read_hash(torrent_file) | |
353 | ||
354 | self.db.execute("INSERT INTO files(releases, filename, filesize, \ | |
8b99a680 MT |
355 | sha256, sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s, %s)", |
356 | self.id, _filename, filesize, hash_sha256, hash_sha1, torrent_hash) | |
fadcfd00 MT |
357 | |
358 | # Search for all files that miss a torrent hash. | |
359 | files = self.db.query("SELECT id, filename FROM files \ | |
360 | WHERE releases = %s AND torrent_hash IS NULL", self.id) | |
361 | ||
362 | for file in files: | |
363 | path = os.path.join(basepath, file.filename) | |
364 | ||
365 | torrent_file = "%s.torrent" % path | |
366 | if os.path.exists(torrent_file): | |
367 | torrent_hash = self.torrent_read_hash(torrent_file) | |
368 | ||
369 | self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s", | |
370 | torrent_hash, file.id) | |
371 | ||
372 | def torrent_read_hash(self, filename): | |
3ec6114d MT |
373 | with open(filename, "rb") as f: |
374 | metainfo = yabencode.decode(f) | |
375 | metainfo = yabencode.encode(metainfo["info"]) | |
fadcfd00 | 376 | |
3ec6114d MT |
377 | h = hashlib.new("sha1") |
378 | h.update(metainfo) | |
fadcfd00 | 379 | |
3ec6114d | 380 | return h.hexdigest() |
3c4f2edc | 381 | |
37b5c0cf | 382 | def supports_arch(self, arch): |
6371bfda | 383 | return arch in ("x86_64", "i586") |
37b5c0cf MT |
384 | |
385 | def supports_platform(self, platform): | |
386 | # Currently there is nothing else than pcbios supported | |
387 | if platform == "pcbios": | |
388 | return True | |
389 | ||
390 | return False | |
391 | ||
9068dba1 MT |
392 | def is_netboot_capable(self): |
393 | return self.path and "ipfire-2.x" in self.path | |
394 | ||
37b5c0cf MT |
395 | def netboot_kernel_url(self, arch, platform): |
396 | assert self.supports_arch(arch) | |
397 | assert self.supports_platform(platform) | |
398 | ||
6371bfda | 399 | if self.sname >= "ipfire-2.19-core100": |
453c817c | 400 | return "http://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch) |
6371bfda | 401 | |
453c817c | 402 | return "http://boot.ipfire.org/%s/images/vmlinuz" % self.path |
940227cb | 403 | |
37b5c0cf MT |
404 | def netboot_initrd_url(self, arch, platform): |
405 | assert self.supports_arch(arch) | |
406 | assert self.supports_platform(platform) | |
407 | ||
6371bfda | 408 | if self.sname >= "ipfire-2.19-core100": |
453c817c | 409 | return "http://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch) |
6371bfda | 410 | |
453c817c | 411 | return "http://boot.ipfire.org/%s/images/instroot" % self.path |
940227cb | 412 | |
37b5c0cf MT |
413 | def netboot_args(self, arch, platform): |
414 | return "" | |
415 | ||
2fed2438 | 416 | @property |
5baa3aef | 417 | def post(self): |
487417ad MT |
418 | if self.__data.blog_id: |
419 | return self.backend.blog.get_by_id(self.__data.blog_id) | |
2fed2438 | 420 | |
dd3a5446 MT |
421 | # Fireinfo Stuff |
422 | ||
423 | @property | |
424 | def penetration(self): | |
dd3a5446 | 425 | # Get penetration from fireinfo |
b8a4acfb | 426 | return self.backend.fireinfo.get_release_penetration(self) |
dd3a5446 | 427 | |
940227cb | 428 | |
9068dba1 | 429 | class Releases(Object): |
984e4e7b MT |
430 | def _get_release(self, query, *args): |
431 | res = self.db.get(query, *args) | |
432 | ||
433 | if res: | |
434 | return Release(self.backend, res.id, data=res) | |
435 | ||
e77cd04c MT |
436 | def _get_releases(self, query, *args): |
437 | res = self.db.query(query, *args) | |
438 | ||
439 | for row in res: | |
440 | yield Release(self.backend, row.id, data=row) | |
441 | ||
856748b0 MT |
442 | def __iter__(self): |
443 | releases = self._get_releases("SELECT * FROM releases \ | |
444 | ORDER BY published DESC NULLS FIRST") | |
445 | ||
446 | return iter(releases) | |
447 | ||
940227cb | 448 | def get_by_id(self, id): |
9068dba1 MT |
449 | ret = self.db.get("SELECT * FROM releases WHERE id = %s", id) |
450 | ||
451 | if ret: | |
452 | return Release(self.backend, ret.id, data=ret) | |
453 | ||
454 | def get_by_sname(self, sname): | |
455 | ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname) | |
456 | ||
457 | if ret: | |
458 | return Release(self.backend, ret.id, data=ret) | |
940227cb | 459 | |
9de13943 MT |
460 | def get_by_news_id(self, news_id): |
461 | ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id) | |
462 | ||
463 | if ret: | |
464 | return Release(self.backend, ret.id, data=ret) | |
465 | ||
9068dba1 MT |
466 | def get_latest(self, stable=True): |
467 | ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \ | |
468 | AND stable = %s ORDER BY published DESC LIMIT 1", stable) | |
940227cb | 469 | |
9068dba1 MT |
470 | if ret: |
471 | return Release(self.backend, ret.id, data=ret) | |
940227cb | 472 | |
e77cd04c MT |
473 | def get_releases_older_than(self, release, limit=None): |
474 | return self._get_releases("SELECT * FROM releases \ | |
475 | WHERE published IS NOT NULL AND published < %s \ | |
476 | ORDER BY published DESC LIMIT %s", release.published, limit) | |
477 | ||
0aab96e0 MT |
478 | def get_latest_unstable(self): |
479 | ret = self.db.get("SELECT * FROM releases r1 \ | |
480 | WHERE r1.published IS NOT NULL AND r1.published <= NOW() \ | |
481 | AND stable = %s AND NOT EXISTS ( \ | |
482 | SELECT * FROM releases r2 WHERE r2.stable = %s AND \ | |
483 | r2.published IS NOT NULL AND r2.published >= r1.published \ | |
484 | ) ORDER BY r1.published DESC LIMIT 1", False, True) | |
485 | ||
486 | if ret: | |
487 | return Release(self.backend, ret.id, data=ret) | |
488 | ||
940227cb | 489 | def get_stable(self): |
9068dba1 MT |
490 | query = self.db.query("SELECT * FROM releases \ |
491 | WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \ | |
492 | ORDER BY published DESC") | |
940227cb | 493 | |
9068dba1 MT |
494 | releases = [] |
495 | for row in query: | |
496 | release = Release(self.backend, row.id, data=row) | |
497 | releases.append(release) | |
498 | ||
499 | return releases | |
940227cb MT |
500 | |
501 | def get_unstable(self): | |
9068dba1 MT |
502 | query = self.db.query("SELECT * FROM releases \ |
503 | WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \ | |
504 | ORDER BY published DESC") | |
505 | ||
506 | releases = [] | |
507 | for row in query: | |
508 | release = Release(self.backend, row.id, data=row) | |
509 | releases.append(release) | |
940227cb | 510 | |
9068dba1 | 511 | return releases |
940227cb MT |
512 | |
513 | def get_all(self): | |
9068dba1 MT |
514 | query = self.db.query("SELECT * FROM releases \ |
515 | WHERE published IS NOT NULL AND published <= NOW() \ | |
516 | ORDER BY published DESC") | |
940227cb | 517 | |
9068dba1 MT |
518 | releases = [] |
519 | for row in query: | |
520 | release = Release(self.backend, row.id, data=row) | |
521 | releases.append(release) | |
522 | ||
523 | return releases | |
940227cb | 524 | |
d0eee7f8 MT |
525 | def _get_all(self): |
526 | query = self.db.query("SELECT * FROM releases ORDER BY published DESC") | |
527 | ||
528 | releases = [] | |
529 | for row in query: | |
530 | release = Release(self.backend, row.id, data=row) | |
531 | releases.append(release) | |
532 | ||
533 | return releases | |
534 | ||
938d083d MT |
535 | def get_file_for_torrent_hash(self, torrent_hash): |
536 | file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1", | |
fadcfd00 MT |
537 | torrent_hash) |
538 | ||
539 | if not file: | |
540 | return | |
541 | ||
ea324f48 MT |
542 | release = Release(self.backend, file.releases) |
543 | file = File(self.backend, release, file.id) | |
938d083d MT |
544 | |
545 | return file | |
856748b0 MT |
546 | |
547 | @tornado.gen.coroutine | |
548 | def scan_files(self, basepath="/pub/mirror"): | |
549 | for release in self: | |
550 | logging.info("Scanning %s..." % release) | |
c49610bb MT |
551 | |
552 | with self.db.transaction(): | |
553 | release.scan_files(basepath=basepath) |