]>
Commit | Line | Data |
---|---|---|
940227cb MT |
1 | #!/usr/bin/python |
2 | ||
3c4f2edc | 3 | import hashlib |
940227cb | 4 | import logging |
3c4f2edc MT |
5 | import os |
6 | import re | |
fadcfd00 | 7 | import urllib |
940227cb MT |
8 | import urlparse |
9 | ||
9068dba1 | 10 | import database |
fadcfd00 | 11 | import tracker |
9068dba1 | 12 | from misc import Object |
fadcfd00 | 13 | |
f7635119 MT |
14 | TRACKERS = ( |
15 | "http://ipv4.tracker.ipfire.org:6969/announce", | |
16 | "udp://ipv4.tracker.ipfire.org:6969", | |
17 | "http://ipv6.tracker.ipfire.org:6969/announce", | |
18 | "udp://ipv6.tracker.ipfire.org:6969", | |
19 | ) | |
20 | ||
9068dba1 MT |
21 | class File(Object): |
22 | def __init__(self, backend, release, id, data=None): | |
23 | Object.__init__(self, backend) | |
940227cb | 24 | |
938d083d MT |
25 | self.id = id |
26 | self._release = release | |
940227cb MT |
27 | |
28 | # get all data from database | |
9068dba1 | 29 | self.__data = data |
940227cb | 30 | |
9068dba1 MT |
31 | def __cmp__(self, other): |
32 | return cmp(self.prio, other.prio) | |
938d083d MT |
33 | |
34 | @property | |
35 | def data(self): | |
36 | if self.__data is None: | |
37 | self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id) | |
38 | assert self.__data | |
39 | ||
40 | return self.__data | |
41 | ||
42 | @property | |
43 | def release(self): | |
44 | if not self._release: | |
45 | release_id = self.data.get("releases") | |
46 | self._release = Release(release_id) | |
47 | ||
48 | return self._release | |
940227cb MT |
49 | |
50 | @property | |
51 | def type(self): | |
fadcfd00 MT |
52 | filename = self.filename |
53 | ||
54 | if filename.endswith(".iso"): | |
55 | return "iso" | |
56 | ||
57 | elif filename.endswith(".torrent"): | |
58 | return "torrent" | |
59 | ||
60 | elif "xen" in filename: | |
80594ae3 MT |
61 | if "downloader" in filename: |
62 | return "xen-downloader" | |
63 | ||
fadcfd00 MT |
64 | return "xen" |
65 | ||
66 | elif "sources" in filename: | |
67 | return "source" | |
68 | ||
69 | elif "usb-fdd" in filename: | |
70 | return "usbfdd" | |
71 | ||
72 | elif "usb-hdd" in filename: | |
73 | return "usbhdd" | |
74 | ||
3230a47f MT |
75 | elif "armv5tel" in filename and "scon" in filename: |
76 | return "armv5tel-scon" | |
77 | ||
fadcfd00 MT |
78 | elif "armv5tel" in filename: |
79 | return "armv5tel" | |
80 | ||
81 | elif "scon" in filename: | |
82 | return "alix" | |
83 | ||
116312c5 | 84 | elif filename.endswith(".img.gz") or filename.endswith(".img.xz"): |
fadcfd00 MT |
85 | return "flash" |
86 | ||
87 | else: | |
88 | return "unknown" | |
940227cb MT |
89 | |
90 | @property | |
91 | def url(self): | |
73493ea6 | 92 | return urlparse.urljoin("https://downloads.ipfire.org", self.filename) |
940227cb MT |
93 | |
94 | @property | |
95 | def desc(self): | |
96 | _ = lambda x: x | |
97 | ||
98 | descriptions = { | |
e77cd04c MT |
99 | "armv5tel" : _("Flash Image"), |
100 | "armv5tel-scon" : _("Flash Image with serial console"), | |
101 | "iso" : _("CD Image"), | |
102 | "torrent" : _("Torrent File"), | |
103 | "flash" : _("Flash Image"), | |
104 | "alix" : _("Flash Image with serial console"), | |
940227cb MT |
105 | "usbfdd" : _("USB FDD Image"), |
106 | "usbhdd" : _("USB HDD Image"), | |
e77cd04c | 107 | "xen" : _("Pre-generated Xen Image"), |
80594ae3 | 108 | "xen-downloader": _("Xen-Image Generator"), |
940227cb MT |
109 | } |
110 | ||
111 | try: | |
112 | return descriptions[self.type] | |
113 | except KeyError: | |
114 | return _("Unknown image type") | |
115 | ||
116 | @property | |
117 | def prio(self): | |
118 | priorities = { | |
119 | "iso" : 10, | |
120 | "torrent" : 20, | |
121 | "flash" : 40, | |
122 | "alix" : 41, | |
123 | "usbfdd" : 31, | |
124 | "usbhdd" : 30, | |
3230a47f MT |
125 | "armv5tel" : 40, |
126 | "armv5tel-scon" : 41, | |
940227cb | 127 | "xen" : 50, |
80594ae3 | 128 | "xen-downloader": 51, |
940227cb MT |
129 | } |
130 | ||
131 | try: | |
132 | return priorities[self.type] | |
133 | except KeyError: | |
134 | return 999 | |
135 | ||
136 | @property | |
137 | def rem(self): | |
138 | _ = lambda x: x | |
139 | ||
140 | remarks = { | |
fadcfd00 | 141 | "armv5tel" : _("This image runs on many ARM-based boards"), |
3230a47f | 142 | "armv5tel-scon" : _("This image runs on ARM boards with a serial console"), |
940227cb MT |
143 | "iso" : _("Use this image to burn a CD and install IPFire from it."), |
144 | "torrent" : _("Download the CD image from the torrent network."), | |
145 | "flash" : _("An image that is meant to run on embedded devices."), | |
146 | "alix" : _("Flash image where a serial console is enabled by default."), | |
147 | "usbfdd" : _("Install IPFire from a floppy-formated USB key."), | |
148 | "usbhdd" : _("If the floppy image doesn't work, use this image instead."), | |
149 | "xen" : _("A ready-to-run image for Xen."), | |
80594ae3 | 150 | "xen-downloader": _("Generator for creating a Xen image."), |
940227cb MT |
151 | } |
152 | ||
153 | try: | |
154 | return remarks[self.type] | |
155 | except KeyError: | |
156 | return _("Unknown image type") | |
157 | ||
158 | @property | |
159 | def sha1(self): | |
938d083d | 160 | return self.data.get("sha1") |
940227cb MT |
161 | |
162 | @property | |
163 | def filename(self): | |
938d083d | 164 | return self.data.get("filename") |
940227cb | 165 | |
c9698a0b MT |
166 | @property |
167 | def basename(self): | |
168 | return os.path.basename(self.filename) | |
169 | ||
60024cc8 MT |
170 | @property |
171 | def size(self): | |
938d083d | 172 | return self.data.get("filesize") |
60024cc8 MT |
173 | |
174 | @property | |
175 | def arch(self): | |
6371bfda | 176 | known_arches = ("x86_64", "i586", "arm") |
60024cc8 MT |
177 | |
178 | for arch in known_arches: | |
179 | if arch in self.basename: | |
180 | return arch | |
181 | ||
fadcfd00 MT |
182 | return "N/A" |
183 | ||
184 | @property | |
185 | def torrent_hash(self): | |
938d083d | 186 | return self.data.get("torrent_hash", None) |
fadcfd00 | 187 | |
5276cd41 MT |
188 | @property |
189 | def torrent_url(self): | |
190 | if self.torrent_hash: | |
191 | return "%s.torrent" % self.url | |
192 | ||
fadcfd00 MT |
193 | @property |
194 | def magnet_link(self): | |
195 | # Don't return anything if we have no torrent hash. | |
196 | if self.torrent_hash is None: | |
197 | return | |
198 | ||
199 | s = "magnet:?xt=urn:btih:%s" % self.torrent_hash | |
200 | ||
201 | #s += "&xl=%d" % self.size | |
202 | s += "&dn=%s" % urllib.quote(self.basename) | |
203 | ||
204 | # Add our tracker. | |
f7635119 MT |
205 | for tracker in TRACKERS: |
206 | s += "&tr=%s" % tracker | |
fadcfd00 | 207 | |
dc7b8879 MT |
208 | # Add web download URL |
209 | s += "&as=%s" % urllib.quote(self.url) | |
210 | ||
fadcfd00 | 211 | return s |
60024cc8 | 212 | |
940227cb | 213 | |
9068dba1 MT |
214 | class Release(Object): |
215 | def __init__(self, backend, id, data=None): | |
216 | Object.__init__(self, backend) | |
940227cb MT |
217 | self.id = id |
218 | ||
219 | # get all data from database | |
9068dba1 | 220 | self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id) |
940227cb MT |
221 | assert self.__data |
222 | ||
223 | self.__files = [] | |
224 | ||
e77cd04c MT |
225 | def __str__(self): |
226 | return self.name | |
227 | ||
940227cb MT |
228 | def __repr__(self): |
229 | return "<%s %s>" % (self.__class__.__name__, self.name) | |
230 | ||
9068dba1 MT |
231 | def __cmp__(self, other): |
232 | return cmp(self.id, other.id) | |
233 | ||
e77cd04c MT |
234 | @property |
235 | def arches(self): | |
236 | for arch in ("x86_64", "aarch64", "i586", "arm"): | |
237 | if arch in (f.arch for f in self.files): | |
238 | yield arch | |
239 | ||
940227cb MT |
240 | @property |
241 | def files(self): | |
242 | if not self.__files: | |
9068dba1 MT |
243 | files = self.db.query("SELECT * FROM files WHERE releases = %s \ |
244 | AND NOT filename LIKE '%%.torrent'", self.id) | |
940227cb | 245 | |
9068dba1 MT |
246 | self.__files = [File(self.backend, self, f.id, f) for f in files] |
247 | self.__files.sort() | |
940227cb MT |
248 | |
249 | return self.__files | |
250 | ||
e77cd04c MT |
251 | def get_files_by_arch(self, arch): |
252 | for f in self.files: | |
253 | if f.arch == arch: | |
254 | yield f | |
255 | ||
938d083d MT |
256 | @property |
257 | def torrents(self): | |
258 | torrents = [] | |
259 | ||
260 | for file in self.files: | |
261 | if not file.torrent_hash: | |
262 | continue | |
263 | ||
264 | torrents.append(file) | |
265 | ||
266 | return torrents | |
267 | ||
940227cb MT |
268 | @property |
269 | def name(self): | |
9068dba1 MT |
270 | return self.__data.name |
271 | ||
272 | @property | |
e77cd04c | 273 | def slug(self): |
9068dba1 | 274 | return self.__data.sname |
940227cb | 275 | |
e77cd04c MT |
276 | # XXX compat |
277 | sname = slug | |
278 | ||
279 | # XXX cache this | |
280 | @property | |
281 | def blog(self): | |
282 | if self.__data.blog_id: | |
283 | return self.backend.blog.get_by_id(self.__data.blog_id) | |
284 | ||
dd3a5446 MT |
285 | @property |
286 | def fireinfo_id(self): | |
287 | name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ") | |
288 | ||
289 | res = self.db.get("SELECT id FROM fireinfo_releases \ | |
290 | WHERE name = %s", name) | |
291 | ||
292 | if res: | |
293 | return res.id | |
294 | ||
940227cb MT |
295 | @property |
296 | def stable(self): | |
9068dba1 | 297 | return self.__data.stable |
940227cb MT |
298 | |
299 | @property | |
300 | def published(self): | |
9068dba1 | 301 | return self.__data.published |
940227cb | 302 | |
9068dba1 | 303 | date = published |
940227cb | 304 | |
3c4f2edc MT |
305 | @property |
306 | def path(self): | |
9068dba1 | 307 | return self.__data.path |
3c4f2edc | 308 | |
940227cb MT |
309 | def get_file(self, type): |
310 | for file in self.files: | |
311 | if file.type == type: | |
312 | return file | |
313 | ||
3c4f2edc MT |
314 | def __file_hash(self, filename): |
315 | sha1 = hashlib.sha1() | |
316 | ||
317 | with open(filename) as f: | |
b5f4eef0 MT |
318 | buf_size = 1024 |
319 | buf = f.read(buf_size) | |
320 | while buf: | |
321 | sha1.update(buf) | |
322 | buf = f.read(buf_size) | |
3c4f2edc MT |
323 | |
324 | return sha1.hexdigest() | |
325 | ||
3c4f2edc MT |
326 | def scan_files(self, basepath="/srv/mirror0"): |
327 | if not self.path: | |
328 | return | |
329 | ||
330 | path = os.path.join(basepath, self.path) | |
921d98cc MT |
331 | if not os.path.exists(path): |
332 | return | |
333 | ||
fadcfd00 MT |
334 | files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id) |
335 | files = [f.filename for f in files] | |
3c4f2edc MT |
336 | |
337 | # Make files that do not exists not loadable. | |
338 | for filename in files: | |
339 | _filename = os.path.join(basepath, filename) | |
340 | if not os.path.exists(_filename): | |
341 | self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename) | |
342 | ||
343 | for filename in os.listdir(path): | |
344 | filename = os.path.join(path, filename) | |
345 | ||
346 | if os.path.isdir(filename): | |
347 | continue | |
348 | ||
349 | _filename = re.match(".*(releases/.*)", filename).group(1) | |
350 | if _filename in files: | |
351 | continue | |
352 | ||
353 | if filename.endswith(".md5"): | |
354 | continue | |
355 | ||
9068dba1 | 356 | logging.info("Hashing %s..." % filename) |
3c4f2edc MT |
357 | filehash = self.__file_hash(filename) |
358 | filesize = os.path.getsize(filename) | |
3c4f2edc | 359 | |
fadcfd00 MT |
360 | # Check if there is a torrent download available for this file: |
361 | torrent_hash = "" | |
362 | torrent_file = "%s.torrent" % filename | |
363 | if os.path.exists(torrent_file): | |
364 | torrent_hash = self.torrent_read_hash(torrent_file) | |
365 | ||
366 | self.db.execute("INSERT INTO files(releases, filename, filesize, \ | |
367 | sha1, torrent_hash) VALUES(%s, %s, %s, %s, %s)", | |
368 | self.id, _filename, filesize, filehash, torrent_hash) | |
369 | ||
370 | # Search for all files that miss a torrent hash. | |
371 | files = self.db.query("SELECT id, filename FROM files \ | |
372 | WHERE releases = %s AND torrent_hash IS NULL", self.id) | |
373 | ||
374 | for file in files: | |
375 | path = os.path.join(basepath, file.filename) | |
376 | ||
377 | torrent_file = "%s.torrent" % path | |
378 | if os.path.exists(torrent_file): | |
379 | torrent_hash = self.torrent_read_hash(torrent_file) | |
380 | ||
381 | self.db.execute("UPDATE files SET torrent_hash = %s WHERE id = %s", | |
382 | torrent_hash, file.id) | |
383 | ||
384 | def torrent_read_hash(self, filename): | |
385 | f = None | |
386 | try: | |
387 | f = open(filename, "rb") | |
388 | ||
389 | metainfo = tracker.bdecode(f.read()) | |
390 | metainfo = tracker.bencode(metainfo["info"]) | |
391 | ||
392 | hash = hashlib.sha1() | |
393 | hash.update(metainfo) | |
394 | ||
395 | return hash.hexdigest() | |
396 | ||
397 | finally: | |
398 | if f: | |
399 | f.close() | |
3c4f2edc | 400 | |
37b5c0cf | 401 | def supports_arch(self, arch): |
6371bfda | 402 | return arch in ("x86_64", "i586") |
37b5c0cf MT |
403 | |
404 | def supports_platform(self, platform): | |
405 | # Currently there is nothing else than pcbios supported | |
406 | if platform == "pcbios": | |
407 | return True | |
408 | ||
409 | return False | |
410 | ||
9068dba1 MT |
411 | def is_netboot_capable(self): |
412 | return self.path and "ipfire-2.x" in self.path | |
413 | ||
37b5c0cf MT |
414 | def netboot_kernel_url(self, arch, platform): |
415 | assert self.supports_arch(arch) | |
416 | assert self.supports_platform(platform) | |
417 | ||
6371bfda | 418 | if self.sname >= "ipfire-2.19-core100": |
73493ea6 | 419 | return "https://downloads.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch) |
6371bfda | 420 | |
73493ea6 | 421 | return "https://downloads.ipfire.org/%s/images/vmlinuz" % self.path |
940227cb | 422 | |
37b5c0cf MT |
423 | def netboot_initrd_url(self, arch, platform): |
424 | assert self.supports_arch(arch) | |
425 | assert self.supports_platform(platform) | |
426 | ||
6371bfda | 427 | if self.sname >= "ipfire-2.19-core100": |
73493ea6 | 428 | return "https://downloads.ipfire.org/%s/images/%s/instroot" % (self.path, arch) |
6371bfda | 429 | |
73493ea6 | 430 | return "https://downloads.ipfire.org/%s/images/instroot" % self.path |
940227cb | 431 | |
37b5c0cf MT |
432 | def netboot_args(self, arch, platform): |
433 | return "" | |
434 | ||
2fed2438 | 435 | @property |
5baa3aef | 436 | def post(self): |
487417ad MT |
437 | if self.__data.blog_id: |
438 | return self.backend.blog.get_by_id(self.__data.blog_id) | |
2fed2438 | 439 | |
dd3a5446 MT |
440 | # Fireinfo Stuff |
441 | ||
442 | @property | |
443 | def penetration(self): | |
444 | penetration = self.memcache.get("%s-penetration" % self.sname) | |
445 | ||
446 | # Cache HIT | |
447 | if penetration: | |
448 | return penetration | |
449 | ||
450 | # Get penetration from fireinfo | |
451 | penetration = self.backend.fireinfo.get_release_penetration(self) | |
452 | ||
453 | # Cache for 1 hour | |
454 | self.memcache.set("%s-penetration" % self.sname, penetration, 3600) | |
455 | ||
456 | return penetration | |
457 | ||
940227cb | 458 | |
9068dba1 | 459 | class Releases(Object): |
e77cd04c MT |
460 | def _get_releases(self, query, *args): |
461 | res = self.db.query(query, *args) | |
462 | ||
463 | for row in res: | |
464 | yield Release(self.backend, row.id, data=row) | |
465 | ||
940227cb | 466 | def get_by_id(self, id): |
9068dba1 MT |
467 | ret = self.db.get("SELECT * FROM releases WHERE id = %s", id) |
468 | ||
469 | if ret: | |
470 | return Release(self.backend, ret.id, data=ret) | |
471 | ||
472 | def get_by_sname(self, sname): | |
473 | ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname) | |
474 | ||
475 | if ret: | |
476 | return Release(self.backend, ret.id, data=ret) | |
940227cb | 477 | |
9de13943 MT |
478 | def get_by_news_id(self, news_id): |
479 | ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id) | |
480 | ||
481 | if ret: | |
482 | return Release(self.backend, ret.id, data=ret) | |
483 | ||
9068dba1 MT |
484 | def get_latest(self, stable=True): |
485 | ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \ | |
486 | AND stable = %s ORDER BY published DESC LIMIT 1", stable) | |
940227cb | 487 | |
9068dba1 MT |
488 | if ret: |
489 | return Release(self.backend, ret.id, data=ret) | |
940227cb | 490 | |
e77cd04c MT |
491 | def get_releases_older_than(self, release, limit=None): |
492 | return self._get_releases("SELECT * FROM releases \ | |
493 | WHERE published IS NOT NULL AND published < %s \ | |
494 | ORDER BY published DESC LIMIT %s", release.published, limit) | |
495 | ||
0aab96e0 MT |
496 | def get_latest_unstable(self): |
497 | ret = self.db.get("SELECT * FROM releases r1 \ | |
498 | WHERE r1.published IS NOT NULL AND r1.published <= NOW() \ | |
499 | AND stable = %s AND NOT EXISTS ( \ | |
500 | SELECT * FROM releases r2 WHERE r2.stable = %s AND \ | |
501 | r2.published IS NOT NULL AND r2.published >= r1.published \ | |
502 | ) ORDER BY r1.published DESC LIMIT 1", False, True) | |
503 | ||
504 | if ret: | |
505 | return Release(self.backend, ret.id, data=ret) | |
506 | ||
940227cb | 507 | def get_stable(self): |
9068dba1 MT |
508 | query = self.db.query("SELECT * FROM releases \ |
509 | WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \ | |
510 | ORDER BY published DESC") | |
940227cb | 511 | |
9068dba1 MT |
512 | releases = [] |
513 | for row in query: | |
514 | release = Release(self.backend, row.id, data=row) | |
515 | releases.append(release) | |
516 | ||
517 | return releases | |
940227cb MT |
518 | |
519 | def get_unstable(self): | |
9068dba1 MT |
520 | query = self.db.query("SELECT * FROM releases \ |
521 | WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \ | |
522 | ORDER BY published DESC") | |
523 | ||
524 | releases = [] | |
525 | for row in query: | |
526 | release = Release(self.backend, row.id, data=row) | |
527 | releases.append(release) | |
940227cb | 528 | |
9068dba1 | 529 | return releases |
940227cb MT |
530 | |
531 | def get_all(self): | |
9068dba1 MT |
532 | query = self.db.query("SELECT * FROM releases \ |
533 | WHERE published IS NOT NULL AND published <= NOW() \ | |
534 | ORDER BY published DESC") | |
940227cb | 535 | |
9068dba1 MT |
536 | releases = [] |
537 | for row in query: | |
538 | release = Release(self.backend, row.id, data=row) | |
539 | releases.append(release) | |
540 | ||
541 | return releases | |
940227cb | 542 | |
d0eee7f8 MT |
543 | def _get_all(self): |
544 | query = self.db.query("SELECT * FROM releases ORDER BY published DESC") | |
545 | ||
546 | releases = [] | |
547 | for row in query: | |
548 | release = Release(self.backend, row.id, data=row) | |
549 | releases.append(release) | |
550 | ||
551 | return releases | |
552 | ||
938d083d MT |
553 | def get_file_for_torrent_hash(self, torrent_hash): |
554 | file = self.db.get("SELECT id, releases FROM files WHERE torrent_hash = %s LIMIT 1", | |
fadcfd00 MT |
555 | torrent_hash) |
556 | ||
557 | if not file: | |
558 | return | |
559 | ||
ea324f48 MT |
560 | release = Release(self.backend, file.releases) |
561 | file = File(self.backend, release, file.id) | |
938d083d MT |
562 | |
563 | return file |