]>
Commit | Line | Data |
---|---|---|
940227cb MT |
1 | #!/usr/bin/python |
2 | ||
3c4f2edc | 3 | import hashlib |
940227cb | 4 | import logging |
3c4f2edc MT |
5 | import os |
6 | import re | |
11347e46 | 7 | import urllib.parse |
940227cb | 8 | |
11347e46 | 9 | from . import database |
11347e46 | 10 | from .misc import Object |
45c7049d | 11 | from .decorators import * |
fadcfd00 | 12 | |
9068dba1 MT |
13 | class File(Object): |
14 | def __init__(self, backend, release, id, data=None): | |
15 | Object.__init__(self, backend) | |
940227cb | 16 | |
938d083d MT |
17 | self.id = id |
18 | self._release = release | |
940227cb MT |
19 | |
20 | # get all data from database | |
9068dba1 | 21 | self.__data = data |
940227cb | 22 | |
5ecd3be7 MT |
23 | def __eq__(self, other): |
24 | if isinstance(other, self.__class__): | |
25 | return self.id == otherid | |
26 | ||
27 | def __lt__(self, other): | |
28 | if isinstance(other, self.__class__): | |
29 | return self.prio < other.prio | |
938d083d MT |
30 | |
31 | @property | |
32 | def data(self): | |
33 | if self.__data is None: | |
34 | self.__data = self.db.get("SELECT * FROM files WHERE id = %s", self.id) | |
35 | assert self.__data | |
36 | ||
37 | return self.__data | |
38 | ||
39 | @property | |
40 | def release(self): | |
41 | if not self._release: | |
42 | release_id = self.data.get("releases") | |
43 | self._release = Release(release_id) | |
44 | ||
45 | return self._release | |
940227cb MT |
46 | |
47 | @property | |
48 | def type(self): | |
fadcfd00 MT |
49 | filename = self.filename |
50 | ||
51 | if filename.endswith(".iso"): | |
52 | return "iso" | |
53 | ||
fadcfd00 | 54 | elif "xen" in filename: |
80594ae3 MT |
55 | if "downloader" in filename: |
56 | return "xen-downloader" | |
57 | ||
fadcfd00 MT |
58 | return "xen" |
59 | ||
60 | elif "sources" in filename: | |
61 | return "source" | |
62 | ||
63 | elif "usb-fdd" in filename: | |
64 | return "usbfdd" | |
65 | ||
66 | elif "usb-hdd" in filename: | |
67 | return "usbhdd" | |
68 | ||
3230a47f MT |
69 | elif "armv5tel" in filename and "scon" in filename: |
70 | return "armv5tel-scon" | |
71 | ||
fadcfd00 MT |
72 | elif "armv5tel" in filename: |
73 | return "armv5tel" | |
74 | ||
75 | elif "scon" in filename: | |
76 | return "alix" | |
77 | ||
116312c5 | 78 | elif filename.endswith(".img.gz") or filename.endswith(".img.xz"): |
fadcfd00 MT |
79 | return "flash" |
80 | ||
81 | else: | |
82 | return "unknown" | |
940227cb MT |
83 | |
84 | @property | |
85 | def url(self): | |
eea71144 | 86 | return urllib.parse.urljoin("https://downloads.ipfire.org", self.filename) |
940227cb MT |
87 | |
88 | @property | |
89 | def desc(self): | |
90 | _ = lambda x: x | |
91 | ||
92 | descriptions = { | |
e77cd04c MT |
93 | "armv5tel" : _("Flash Image"), |
94 | "armv5tel-scon" : _("Flash Image with serial console"), | |
a47f3c78 | 95 | "iso" : _("ISO Image"), |
e77cd04c MT |
96 | "flash" : _("Flash Image"), |
97 | "alix" : _("Flash Image with serial console"), | |
940227cb MT |
98 | "usbfdd" : _("USB FDD Image"), |
99 | "usbhdd" : _("USB HDD Image"), | |
e77cd04c | 100 | "xen" : _("Pre-generated Xen Image"), |
80594ae3 | 101 | "xen-downloader": _("Xen-Image Generator"), |
940227cb MT |
102 | } |
103 | ||
104 | try: | |
105 | return descriptions[self.type] | |
106 | except KeyError: | |
107 | return _("Unknown image type") | |
108 | ||
109 | @property | |
110 | def prio(self): | |
111 | priorities = { | |
112 | "iso" : 10, | |
940227cb MT |
113 | "flash" : 40, |
114 | "alix" : 41, | |
115 | "usbfdd" : 31, | |
116 | "usbhdd" : 30, | |
3230a47f MT |
117 | "armv5tel" : 40, |
118 | "armv5tel-scon" : 41, | |
940227cb | 119 | "xen" : 50, |
80594ae3 | 120 | "xen-downloader": 51, |
940227cb | 121 | } |
e40f251b | 122 | |
940227cb MT |
123 | try: |
124 | return priorities[self.type] | |
125 | except KeyError: | |
126 | return 999 | |
127 | ||
752c8888 MT |
128 | @property |
129 | def sha256(self): | |
130 | return self.data.get("sha256") | |
131 | ||
940227cb MT |
132 | @property |
133 | def sha1(self): | |
938d083d | 134 | return self.data.get("sha1") |
940227cb MT |
135 | |
136 | @property | |
137 | def filename(self): | |
938d083d | 138 | return self.data.get("filename") |
940227cb | 139 | |
c9698a0b MT |
140 | @property |
141 | def basename(self): | |
142 | return os.path.basename(self.filename) | |
143 | ||
60024cc8 MT |
144 | @property |
145 | def size(self): | |
938d083d | 146 | return self.data.get("filesize") |
60024cc8 MT |
147 | |
148 | @property | |
149 | def arch(self): | |
6c62d21d | 150 | known_arches = ("x86_64", "aarch64", "arm", "i586") |
60024cc8 MT |
151 | |
152 | for arch in known_arches: | |
153 | if arch in self.basename: | |
154 | return arch | |
155 | ||
fadcfd00 MT |
156 | return "N/A" |
157 | ||
940227cb | 158 | |
9068dba1 MT |
159 | class Release(Object): |
160 | def __init__(self, backend, id, data=None): | |
161 | Object.__init__(self, backend) | |
940227cb MT |
162 | self.id = id |
163 | ||
164 | # get all data from database | |
9068dba1 | 165 | self.__data = data or self.db.get("SELECT * FROM releases WHERE id = %s", self.id) |
940227cb MT |
166 | assert self.__data |
167 | ||
168 | self.__files = [] | |
169 | ||
e77cd04c MT |
170 | def __str__(self): |
171 | return self.name | |
172 | ||
940227cb MT |
173 | def __repr__(self): |
174 | return "<%s %s>" % (self.__class__.__name__, self.name) | |
175 | ||
9068dba1 MT |
176 | def __cmp__(self, other): |
177 | return cmp(self.id, other.id) | |
178 | ||
e77cd04c MT |
179 | @property |
180 | def arches(self): | |
181 | for arch in ("x86_64", "aarch64", "i586", "arm"): | |
182 | if arch in (f.arch for f in self.files): | |
183 | yield arch | |
184 | ||
6c62d21d MT |
185 | @property |
186 | def primary_arches(self): | |
187 | arches = [] | |
188 | ||
189 | # Add x86_64 when available, otherwise add i586 | |
190 | for arch in ("x86_64", "i586"): | |
191 | if arch in self.arches: | |
192 | arches.append(arch) | |
193 | break | |
194 | ||
4bf117d0 MT |
195 | # Add aarch64 if available |
196 | if "aarch64" in self.arches: | |
197 | arches.append("aarch64") | |
198 | ||
199 | # Add ARM before 2.27 if available | |
200 | if "arm" in self.arches and self.sname < "ipfire-2.27-core159": | |
6c62d21d MT |
201 | arches.append("arm") |
202 | ||
203 | return arches | |
204 | ||
205 | @property | |
206 | def secondary_arches(self): | |
207 | arches = [] | |
208 | ||
209 | for arch in self.arches: | |
210 | if arch in self.primary_arches: | |
211 | continue | |
212 | ||
213 | arches.append(arch) | |
214 | ||
215 | return arches | |
216 | ||
217 | @property | |
218 | def experimental_arches(self): | |
4bf117d0 | 219 | return [] |
6c62d21d | 220 | |
940227cb MT |
221 | @property |
222 | def files(self): | |
223 | if not self.__files: | |
9068dba1 MT |
224 | files = self.db.query("SELECT * FROM files WHERE releases = %s \ |
225 | AND NOT filename LIKE '%%.torrent'", self.id) | |
940227cb | 226 | |
9068dba1 MT |
227 | self.__files = [File(self.backend, self, f.id, f) for f in files] |
228 | self.__files.sort() | |
940227cb MT |
229 | |
230 | return self.__files | |
231 | ||
e77cd04c MT |
232 | def get_files_by_arch(self, arch): |
233 | for f in self.files: | |
234 | if f.arch == arch: | |
235 | yield f | |
236 | ||
940227cb MT |
237 | @property |
238 | def name(self): | |
9068dba1 MT |
239 | return self.__data.name |
240 | ||
241 | @property | |
e77cd04c | 242 | def slug(self): |
9068dba1 | 243 | return self.__data.sname |
940227cb | 244 | |
e77cd04c MT |
245 | # XXX compat |
246 | sname = slug | |
247 | ||
45c7049d | 248 | @lazy_property |
e77cd04c MT |
249 | def blog(self): |
250 | if self.__data.blog_id: | |
251 | return self.backend.blog.get_by_id(self.__data.blog_id) | |
252 | ||
940227cb MT |
253 | @property |
254 | def stable(self): | |
9068dba1 | 255 | return self.__data.stable |
940227cb MT |
256 | |
257 | @property | |
258 | def published(self): | |
9068dba1 | 259 | return self.__data.published |
940227cb | 260 | |
9068dba1 | 261 | date = published |
940227cb | 262 | |
3c4f2edc MT |
263 | @property |
264 | def path(self): | |
9068dba1 | 265 | return self.__data.path |
3c4f2edc | 266 | |
940227cb MT |
267 | def get_file(self, type): |
268 | for file in self.files: | |
269 | if file.type == type: | |
270 | return file | |
271 | ||
8b99a680 MT |
272 | def __file_hash(self, filename, algo="sha256"): |
273 | h = hashlib.new(algo) | |
3c4f2edc | 274 | |
3ca4157e | 275 | with open(filename, "rb") as f: |
b5f4eef0 MT |
276 | buf_size = 1024 |
277 | buf = f.read(buf_size) | |
278 | while buf: | |
8b99a680 | 279 | h.update(buf) |
b5f4eef0 | 280 | buf = f.read(buf_size) |
3c4f2edc | 281 | |
8b99a680 | 282 | return h.hexdigest() |
3c4f2edc | 283 | |
e136d583 | 284 | def scan_files(self, basepath="/pub/mirror"): |
3c4f2edc MT |
285 | if not self.path: |
286 | return | |
287 | ||
288 | path = os.path.join(basepath, self.path) | |
921d98cc MT |
289 | if not os.path.exists(path): |
290 | return | |
291 | ||
fadcfd00 MT |
292 | files = self.db.query("SELECT filename FROM files WHERE releases = %s", self.id) |
293 | files = [f.filename for f in files] | |
3c4f2edc MT |
294 | |
295 | # Make files that do not exists not loadable. | |
296 | for filename in files: | |
297 | _filename = os.path.join(basepath, filename) | |
298 | if not os.path.exists(_filename): | |
299 | self.db.execute("UPDATE files SET loadable='N' WHERE filename = %s", filename) | |
300 | ||
301 | for filename in os.listdir(path): | |
302 | filename = os.path.join(path, filename) | |
303 | ||
304 | if os.path.isdir(filename): | |
305 | continue | |
306 | ||
307 | _filename = re.match(".*(releases/.*)", filename).group(1) | |
308 | if _filename in files: | |
309 | continue | |
310 | ||
f2066c37 | 311 | if filename.endswith(".b2") or filename.endswith(".md5"): |
3c4f2edc MT |
312 | continue |
313 | ||
9068dba1 | 314 | logging.info("Hashing %s..." % filename) |
8b99a680 MT |
315 | hash_sha256 = self.__file_hash(filename, "sha256") |
316 | hash_sha1 = self.__file_hash(filename, "sha1") | |
3c4f2edc | 317 | filesize = os.path.getsize(filename) |
3c4f2edc | 318 | |
fadcfd00 | 319 | self.db.execute("INSERT INTO files(releases, filename, filesize, \ |
e40f251b MT |
320 | sha256, sha1) VALUES(%s, %s, %s, %s, %s)", |
321 | self.id, _filename, filesize, hash_sha256, hash_sha1) | |
3c4f2edc | 322 | |
37b5c0cf | 323 | def supports_arch(self, arch): |
6371bfda | 324 | return arch in ("x86_64", "i586") |
37b5c0cf MT |
325 | |
326 | def supports_platform(self, platform): | |
327 | # Currently there is nothing else than pcbios supported | |
328 | if platform == "pcbios": | |
329 | return True | |
330 | ||
331 | return False | |
332 | ||
9068dba1 MT |
333 | def is_netboot_capable(self): |
334 | return self.path and "ipfire-2.x" in self.path | |
335 | ||
37b5c0cf MT |
336 | def netboot_kernel_url(self, arch, platform): |
337 | assert self.supports_arch(arch) | |
338 | assert self.supports_platform(platform) | |
339 | ||
6371bfda | 340 | if self.sname >= "ipfire-2.19-core100": |
453c817c | 341 | return "http://boot.ipfire.org/%s/images/%s/vmlinuz" % (self.path, arch) |
6371bfda | 342 | |
453c817c | 343 | return "http://boot.ipfire.org/%s/images/vmlinuz" % self.path |
940227cb | 344 | |
37b5c0cf MT |
345 | def netboot_initrd_url(self, arch, platform): |
346 | assert self.supports_arch(arch) | |
347 | assert self.supports_platform(platform) | |
348 | ||
6371bfda | 349 | if self.sname >= "ipfire-2.19-core100": |
453c817c | 350 | return "http://boot.ipfire.org/%s/images/%s/instroot" % (self.path, arch) |
6371bfda | 351 | |
453c817c | 352 | return "http://boot.ipfire.org/%s/images/instroot" % self.path |
940227cb | 353 | |
37b5c0cf MT |
354 | def netboot_args(self, arch, platform): |
355 | return "" | |
356 | ||
2fed2438 | 357 | @property |
5baa3aef | 358 | def post(self): |
487417ad MT |
359 | if self.__data.blog_id: |
360 | return self.backend.blog.get_by_id(self.__data.blog_id) | |
2fed2438 | 361 | |
dd3a5446 MT |
362 | # Fireinfo Stuff |
363 | ||
278a2971 MT |
364 | def get_usage(self, when=None): |
365 | name = self.sname.replace("ipfire-", "IPFire ").replace("-", " - ") | |
366 | ||
dd3a5446 | 367 | # Get penetration from fireinfo |
278a2971 MT |
368 | releases = self.backend.fireinfo.get_releases_map(when=when) |
369 | ||
370 | return releases.get(name, 0) | |
dd3a5446 | 371 | |
940227cb | 372 | |
9068dba1 | 373 | class Releases(Object): |
984e4e7b MT |
374 | def _get_release(self, query, *args): |
375 | res = self.db.get(query, *args) | |
376 | ||
377 | if res: | |
378 | return Release(self.backend, res.id, data=res) | |
379 | ||
e77cd04c MT |
380 | def _get_releases(self, query, *args): |
381 | res = self.db.query(query, *args) | |
382 | ||
383 | for row in res: | |
384 | yield Release(self.backend, row.id, data=row) | |
385 | ||
856748b0 MT |
386 | def __iter__(self): |
387 | releases = self._get_releases("SELECT * FROM releases \ | |
388 | ORDER BY published DESC NULLS FIRST") | |
389 | ||
390 | return iter(releases) | |
391 | ||
940227cb | 392 | def get_by_id(self, id): |
9068dba1 MT |
393 | ret = self.db.get("SELECT * FROM releases WHERE id = %s", id) |
394 | ||
395 | if ret: | |
396 | return Release(self.backend, ret.id, data=ret) | |
397 | ||
398 | def get_by_sname(self, sname): | |
399 | ret = self.db.get("SELECT * FROM releases WHERE sname = %s", sname) | |
400 | ||
401 | if ret: | |
402 | return Release(self.backend, ret.id, data=ret) | |
940227cb | 403 | |
9de13943 MT |
404 | def get_by_news_id(self, news_id): |
405 | ret = self.db.get("SELECT * FROM releases WHERE news_id = %s", news_id) | |
406 | ||
407 | if ret: | |
408 | return Release(self.backend, ret.id, data=ret) | |
409 | ||
9068dba1 MT |
410 | def get_latest(self, stable=True): |
411 | ret = self.db.get("SELECT * FROM releases WHERE published IS NOT NULL AND published <= NOW() \ | |
412 | AND stable = %s ORDER BY published DESC LIMIT 1", stable) | |
940227cb | 413 | |
9068dba1 MT |
414 | if ret: |
415 | return Release(self.backend, ret.id, data=ret) | |
940227cb | 416 | |
e77cd04c MT |
417 | def get_releases_older_than(self, release, limit=None): |
418 | return self._get_releases("SELECT * FROM releases \ | |
419 | WHERE published IS NOT NULL AND published < %s \ | |
420 | ORDER BY published DESC LIMIT %s", release.published, limit) | |
421 | ||
0aab96e0 MT |
422 | def get_latest_unstable(self): |
423 | ret = self.db.get("SELECT * FROM releases r1 \ | |
424 | WHERE r1.published IS NOT NULL AND r1.published <= NOW() \ | |
425 | AND stable = %s AND NOT EXISTS ( \ | |
426 | SELECT * FROM releases r2 WHERE r2.stable = %s AND \ | |
427 | r2.published IS NOT NULL AND r2.published >= r1.published \ | |
428 | ) ORDER BY r1.published DESC LIMIT 1", False, True) | |
429 | ||
430 | if ret: | |
431 | return Release(self.backend, ret.id, data=ret) | |
432 | ||
940227cb | 433 | def get_stable(self): |
9068dba1 MT |
434 | query = self.db.query("SELECT * FROM releases \ |
435 | WHERE published IS NOT NULL AND published <= NOW() AND stable = TRUE \ | |
436 | ORDER BY published DESC") | |
940227cb | 437 | |
9068dba1 MT |
438 | releases = [] |
439 | for row in query: | |
440 | release = Release(self.backend, row.id, data=row) | |
441 | releases.append(release) | |
442 | ||
443 | return releases | |
940227cb MT |
444 | |
445 | def get_unstable(self): | |
9068dba1 MT |
446 | query = self.db.query("SELECT * FROM releases \ |
447 | WHERE published IS NOT NULL AND published <= NOW() AND stable = FALSE \ | |
448 | ORDER BY published DESC") | |
449 | ||
450 | releases = [] | |
451 | for row in query: | |
452 | release = Release(self.backend, row.id, data=row) | |
453 | releases.append(release) | |
940227cb | 454 | |
9068dba1 | 455 | return releases |
940227cb MT |
456 | |
457 | def get_all(self): | |
9068dba1 MT |
458 | query = self.db.query("SELECT * FROM releases \ |
459 | WHERE published IS NOT NULL AND published <= NOW() \ | |
460 | ORDER BY published DESC") | |
940227cb | 461 | |
9068dba1 MT |
462 | releases = [] |
463 | for row in query: | |
464 | release = Release(self.backend, row.id, data=row) | |
465 | releases.append(release) | |
466 | ||
467 | return releases | |
940227cb | 468 | |
d0eee7f8 MT |
469 | def _get_all(self): |
470 | query = self.db.query("SELECT * FROM releases ORDER BY published DESC") | |
471 | ||
472 | releases = [] | |
473 | for row in query: | |
474 | release = Release(self.backend, row.id, data=row) | |
475 | releases.append(release) | |
476 | ||
477 | return releases | |
478 | ||
9fdf4fb7 | 479 | async def scan_files(self, basepath="/pub/mirror"): |
856748b0 | 480 | for release in self: |
fb51c9c7 | 481 | logging.debug("Scanning %s..." % release) |
c49610bb MT |
482 | |
483 | with self.db.transaction(): | |
484 | release.scan_files(basepath=basepath) | |
4cfc0589 RH |
485 | |
486 | def get_file_by_filename(self, filename): | |
487 | ret = self.db.get("SELECT * FROM files WHERE filename = %s", filename) | |
488 | ||
489 | if ret: | |
490 | return File(self.backend, None, ret.id, data=ret) |