]>
Commit | Line | Data |
---|---|---|
9137135a MT |
1 | #!/usr/bin/python |
2 | ||
2c909128 | 3 | from __future__ import absolute_import, division |
f96eb5ed | 4 | |
9137135a MT |
5 | import datetime |
6 | import hashlib | |
7 | import logging | |
8 | import random | |
9 | import string | |
10 | import time | |
11 | ||
2c909128 MT |
12 | from . import base |
13 | from . import logs | |
f6e6ff79 | 14 | |
2c909128 | 15 | from .users import generate_password_hash, check_password_hash, generate_random_string |
9137135a MT |
16 | |
17 | class Builders(base.Object): | |
f6e6ff79 MT |
18 | def auth(self, name, passphrase): |
19 | # If either name or passphrase is None, we don't check at all. | |
20 | if None in (name, passphrase): | |
21 | return | |
22 | ||
23 | # Search for the hostname in the database. | |
24 | # The builder must not be deleted. | |
25 | builder = self.db.get("SELECT id FROM builders WHERE name = %s AND \ | |
26 | NOT status = 'deleted'", name) | |
27 | ||
28 | if not builder: | |
29 | return | |
30 | ||
31 | # Get the whole Builder object from the database. | |
32 | builder = self.get_by_id(builder.id) | |
33 | ||
34 | # If the builder was not found or the passphrase does not match, | |
35 | # you have bad luck. | |
36 | if not builder or not builder.validate_passphrase(passphrase): | |
37 | return | |
38 | ||
39 | # Otherwise we return the Builder object. | |
40 | return builder | |
41 | ||
9137135a | 42 | def get_all(self): |
9fa1787c | 43 | builders = self.db.query("SELECT * FROM builders WHERE NOT status = 'deleted' ORDER BY name") |
9137135a | 44 | |
9fa1787c | 45 | return [Builder(self.pakfire, b.id, b) for b in builders] |
9137135a MT |
46 | |
47 | def get_by_id(self, id): | |
48 | if not id: | |
49 | return | |
50 | ||
51 | return Builder(self.pakfire, id) | |
52 | ||
53 | def get_by_name(self, name): | |
9fa1787c | 54 | builder = self.db.get("SELECT * FROM builders WHERE name = %s LIMIT 1", name) |
9137135a MT |
55 | |
56 | if builder: | |
9fa1787c | 57 | return Builder(self.pakfire, builder.id, builder) |
9137135a MT |
58 | |
59 | def get_all_arches(self): | |
60 | arches = set() | |
61 | ||
62 | for result in self.db.query("SELECT DISTINCT arches FROM builders"): | |
63 | if not result.arches: | |
64 | continue | |
65 | ||
66 | _arches = result.arches.split() | |
67 | ||
68 | for arch in _arches: | |
69 | arches.add(arch) | |
70 | ||
71 | return sorted(arches) | |
72 | ||
f6e6ff79 | 73 | def get_load(self): |
f96eb5ed MT |
74 | res1 = self.db.get("SELECT SUM(max_jobs) AS max_jobs FROM builders \ |
75 | WHERE status = 'enabled'") | |
f6e6ff79 | 76 | |
f96eb5ed MT |
77 | res2 = self.db.get("SELECT COUNT(*) AS count FROM jobs \ |
78 | WHERE state = 'dispatching' OR state = 'running' OR state = 'uploading'") | |
f6e6ff79 | 79 | |
33857fed MT |
80 | try: |
81 | return (res2.count * 100 / res1.max_jobs) | |
82 | except: | |
83 | return 0 | |
f6e6ff79 MT |
84 | |
85 | def get_history(self, limit=None, offset=None, builder=None, user=None): | |
86 | query = "SELECT * FROM builders_history" | |
87 | args = [] | |
88 | ||
89 | conditions = [] | |
90 | ||
91 | if builder: | |
92 | conditions.append("builder_id = %s") | |
93 | args.append(builder.id) | |
94 | ||
95 | if user: | |
96 | conditions.append("user_id = %s") | |
97 | args.append(user.id) | |
98 | ||
99 | if conditions: | |
100 | query += " WHERE %s" % " AND ".join(conditions) | |
101 | ||
102 | query += " ORDER BY time DESC" | |
103 | ||
104 | if limit: | |
105 | if offset: | |
106 | query += " LIMIT %s,%s" | |
107 | args += [offset, limit,] | |
108 | else: | |
109 | query += " LIMIT %s" | |
110 | args += [limit,] | |
111 | ||
112 | entries = [] | |
113 | for entry in self.db.query(query, *args): | |
114 | entry = logs.BuilderLogEntry(self.pakfire, entry) | |
115 | entries.append(entry) | |
116 | ||
117 | return entries | |
118 | ||
9137135a MT |
119 | |
120 | class Builder(base.Object): | |
9fa1787c | 121 | def __init__(self, pakfire, id, data=None): |
9137135a MT |
122 | base.Object.__init__(self, pakfire) |
123 | ||
124 | self.id = id | |
125 | ||
f6e6ff79 | 126 | # Cache. |
9fa1787c | 127 | self._data = data |
f6e6ff79 | 128 | self._active_jobs = None |
f6e6ff79 | 129 | self._disabled_arches = None |
9137135a MT |
130 | |
131 | def __cmp__(self, other): | |
318a6dfd MT |
132 | if other is None: |
133 | return -1 | |
134 | ||
9137135a MT |
135 | return cmp(self.id, other.id) |
136 | ||
f6e6ff79 MT |
137 | @property |
138 | def data(self): | |
139 | if self._data is None: | |
f96eb5ed MT |
140 | self._data = self.db.get("SELECT * FROM builders WHERE id = %s", self.id) |
141 | assert self._data | |
f6e6ff79 MT |
142 | |
143 | return self._data | |
144 | ||
9137135a | 145 | @classmethod |
f6e6ff79 MT |
146 | def create(cls, pakfire, name, user=None, log=True): |
147 | """ | |
148 | Creates a new builder. | |
149 | """ | |
150 | builder_id = pakfire.db.execute("INSERT INTO builders(name, time_created) \ | |
151 | VALUES(%s, NOW())", name) | |
9137135a | 152 | |
f6e6ff79 MT |
153 | # Create Builder object. |
154 | builder = cls(pakfire, builder_id) | |
9137135a | 155 | |
f6e6ff79 MT |
156 | # Generate a new passphrase. |
157 | passphrase = builder.regenerate_passphrase() | |
158 | ||
159 | # Log what we have done. | |
160 | if log: | |
161 | builder.log("created", user=user) | |
162 | ||
163 | # The Builder object and the passphrase are returned. | |
164 | return builder, passphrase | |
165 | ||
166 | def log(self, action, user=None): | |
167 | user_id = None | |
168 | if user: | |
169 | user_id = user.id | |
170 | ||
171 | self.db.execute("INSERT INTO builders_history(builder_id, action, user_id, time) \ | |
172 | VALUES(%s, %s, %s, NOW())", self.id, action, user_id) | |
9137135a MT |
173 | |
174 | def set(self, key, value): | |
175 | self.db.execute("UPDATE builders SET %s = %%s WHERE id = %%s LIMIT 1" % key, | |
176 | value, self.id) | |
177 | self.data[key] = value | |
178 | ||
9137135a | 179 | def regenerate_passphrase(self): |
f6e6ff79 MT |
180 | """ |
181 | Generates a new random passphrase and stores it as a salted hash | |
182 | to the database. | |
183 | ||
184 | The new passphrase is returned to be sent to the user (once). | |
185 | """ | |
186 | # Generate a random string with 20 chars. | |
187 | passphrase = generate_random_string(length=20) | |
188 | ||
189 | # Create salted hash. | |
190 | passphrase_hash = generate_password_hash(passphrase) | |
191 | ||
192 | # Store the hash in the database. | |
193 | self.db.execute("UPDATE builders SET passphrase = %s WHERE id = %s", | |
194 | passphrase_hash, self.id) | |
9137135a | 195 | |
f6e6ff79 MT |
196 | # Return the clear-text passphrase. |
197 | return passphrase | |
9137135a MT |
198 | |
199 | def validate_passphrase(self, passphrase): | |
f6e6ff79 MT |
200 | """ |
201 | Compare the given passphrase with the one stored in the database. | |
202 | """ | |
203 | return check_password_hash(passphrase, self.data.passphrase) | |
204 | ||
205 | @property | |
206 | def description(self): | |
207 | return self.data.description or "" | |
208 | ||
209 | @property | |
210 | def status(self): | |
211 | return self.data.status | |
212 | ||
213 | def update_description(self, description): | |
214 | self.db.execute("UPDATE builders SET description = %s, time_updated = NOW() \ | |
215 | WHERE id = %s", description, self.id) | |
216 | ||
217 | if self._data: | |
218 | self._data["description"] = description | |
219 | ||
220 | @property | |
221 | def keepalive(self): | |
222 | """ | |
223 | Returns time of last keepalive message from this host. | |
224 | """ | |
225 | return self.data.time_keepalive | |
226 | ||
c2902b29 MT |
227 | def update_keepalive(self, loadavg1=None, loadavg5=None, loadavg15=None, |
228 | mem_total=None, mem_free=None, swap_total=None, swap_free=None, | |
229 | space_free=None): | |
f6e6ff79 MT |
230 | """ |
231 | Update the keepalive timestamp of this machine. | |
232 | """ | |
c2902b29 MT |
233 | self.db.execute("UPDATE builders SET time_keepalive = NOW(), \ |
234 | loadavg1 = %s, loadavg5 = %s, loadavg15 = %s, space_free = %s, \ | |
235 | mem_total = %s, mem_free = %s, swap_total = %s, swap_free = %s \ | |
236 | WHERE id = %s", loadavg1, loadavg5, loadavg15, space_free, | |
237 | mem_total, mem_free, swap_total, swap_free, self.id) | |
238 | ||
239 | def update_info(self, cpu_model=None, cpu_count=None, cpu_arch=None, cpu_bogomips=None, | |
240 | pakfire_version=None, host_key=None, os_name=None): | |
f6e6ff79 MT |
241 | # Update all the rest. |
242 | self.db.execute("UPDATE builders SET time_updated = NOW(), \ | |
c2902b29 MT |
243 | pakfire_version = %s, cpu_model = %s, cpu_count = %s, cpu_arch = %s, \ |
244 | cpu_bogomips = %s, host_key_id = %s, os_name = %s WHERE id = %s", | |
245 | pakfire_version, cpu_model, cpu_count, cpu_arch, cpu_bogomips, | |
246 | host_key, os_name, self.id) | |
f6e6ff79 MT |
247 | |
248 | def update_arches(self, arches): | |
249 | # Get all arches this builder does currently support. | |
250 | supported_arches = [a.name for a in self.get_arches()] | |
251 | ||
252 | # Noarch is always supported. | |
253 | if not "noarch" in arches: | |
254 | arches.append("noarch") | |
255 | ||
256 | arches_add = [] | |
257 | for arch in arches: | |
258 | if arch in supported_arches: | |
259 | supported_arches.remove(arch) | |
260 | continue | |
261 | ||
262 | arches_add.append(arch) | |
263 | arches_rem = supported_arches | |
264 | ||
265 | for arch_name in arches_add: | |
266 | arch = self.pakfire.arches.get_by_name(arch_name) | |
267 | if not arch: | |
268 | logging.info("Client sent unknown architecture: %s" % arch_name) | |
269 | continue | |
270 | ||
271 | self.db.execute("INSERT INTO builders_arches(builder_id, arch_id) \ | |
272 | VALUES(%s, %s)", self.id, arch.id) | |
273 | ||
274 | for arch_name in arches_rem: | |
275 | arch = self.pakfire.arches.get_by_name(arch_name) | |
276 | assert arch | |
277 | ||
278 | self.db.execute("DELETE FROM builders_arches WHERE builder_id = %s \ | |
279 | AND arch_id = %s", self.id, arch.id) | |
9137135a | 280 | |
9137135a | 281 | def get_enabled(self): |
f6e6ff79 | 282 | return self.status == "enabled" |
9137135a MT |
283 | |
284 | def set_enabled(self, value): | |
f6e6ff79 MT |
285 | # XXX deprecated |
286 | ||
9137135a | 287 | if value: |
f6e6ff79 | 288 | value = "enabled" |
9137135a | 289 | else: |
f6e6ff79 | 290 | value = "disabled" |
9137135a | 291 | |
f6e6ff79 | 292 | self.set_status(value) |
9137135a MT |
293 | |
294 | enabled = property(get_enabled, set_enabled) | |
295 | ||
296 | @property | |
297 | def disabled(self): | |
f6e6ff79 MT |
298 | return not self.enabled |
299 | ||
300 | def set_status(self, status, user=None, log=True): | |
301 | assert status in ("created", "enabled", "disabled", "deleted") | |
302 | ||
303 | if self.status == status: | |
304 | return | |
305 | ||
306 | self.db.execute("UPDATE builders SET status = %s WHERE id = %s", | |
307 | status, self.id) | |
308 | ||
309 | if self._data: | |
310 | self._data["status"] = status | |
311 | ||
312 | if log: | |
313 | self.log(status, user=user) | |
314 | ||
9137135a MT |
315 | @property |
316 | def arches(self): | |
c2902b29 | 317 | if not hasattr(self, "_arches"): |
32ba6ec7 | 318 | self._arches = [] |
9137135a | 319 | |
c2902b29 MT |
320 | if self.cpu_arch: |
321 | res = self.db.query("SELECT build_arch FROM arches_compat \ | |
322 | WHERE host_arch = %s", self.cpu_arch) | |
f6e6ff79 | 323 | |
c2902b29 MT |
324 | self._arches += [r.build_arch for r in res] |
325 | if not self.cpu_arch in self._arches: | |
326 | self._arches.append(self.cpu_arch) | |
f6e6ff79 | 327 | |
c2902b29 | 328 | return self._arches |
9137135a | 329 | |
f6e6ff79 MT |
330 | def get_build_release(self): |
331 | return self.data.build_release == "Y" | |
332 | ||
333 | def set_build_release(self, value): | |
9137135a MT |
334 | if value: |
335 | value = "Y" | |
336 | else: | |
337 | value = "N" | |
338 | ||
f6e6ff79 MT |
339 | self.db.execute("UPDATE builders SET build_release = %s WHERE id = %s", |
340 | value, self.id) | |
341 | ||
342 | # Update the cache. | |
343 | if self._data: | |
344 | self._data["build_release"] = value | |
9137135a | 345 | |
f6e6ff79 | 346 | build_release = property(get_build_release, set_build_release) |
9137135a | 347 | |
f6e6ff79 MT |
348 | def get_build_scratch(self): |
349 | return self.data.build_scratch == "Y" | |
9137135a | 350 | |
f6e6ff79 | 351 | def set_build_scratch(self, value): |
9137135a MT |
352 | if value: |
353 | value = "Y" | |
354 | else: | |
355 | value = "N" | |
356 | ||
f6e6ff79 MT |
357 | self.db.execute("UPDATE builders SET build_scratch = %s WHERE id = %s", |
358 | value, self.id) | |
359 | ||
360 | # Update the cache. | |
361 | if self._data: | |
362 | self._data["build_scratch"] = value | |
9137135a | 363 | |
f6e6ff79 | 364 | build_scratch = property(get_build_scratch, set_build_scratch) |
9137135a MT |
365 | |
366 | def get_build_test(self): | |
367 | return self.data.build_test == "Y" | |
368 | ||
369 | def set_build_test(self, value): | |
370 | if value: | |
371 | value = "Y" | |
372 | else: | |
373 | value = "N" | |
374 | ||
f6e6ff79 MT |
375 | self.db.execute("UPDATE builders SET build_test = %s WHERE id = %s", |
376 | value, self.id) | |
377 | ||
378 | # Update the cache. | |
379 | if self._data: | |
380 | self._data["build_test"] = value | |
9137135a MT |
381 | |
382 | build_test = property(get_build_test, set_build_test) | |
383 | ||
f6e6ff79 MT |
384 | @property |
385 | def build_types(self): | |
386 | ret = [] | |
387 | ||
388 | if self.build_release: | |
389 | ret.append("release") | |
390 | ||
391 | if self.build_scratch: | |
392 | ret.append("scratch") | |
393 | ||
394 | if self.build_test: | |
395 | ret.append("test") | |
396 | ||
397 | return ret | |
398 | ||
9137135a MT |
399 | def get_max_jobs(self): |
400 | return self.data.max_jobs | |
401 | ||
402 | def set_max_jobs(self, value): | |
403 | self.set("max_jobs", value) | |
404 | ||
405 | max_jobs = property(get_max_jobs, set_max_jobs) | |
406 | ||
407 | @property | |
408 | def name(self): | |
409 | return self.data.name | |
410 | ||
411 | @property | |
412 | def hostname(self): | |
413 | return self.name | |
414 | ||
415 | @property | |
416 | def passphrase(self): | |
417 | return self.data.passphrase | |
418 | ||
c2902b29 MT |
419 | # Load average |
420 | ||
9137135a MT |
421 | @property |
422 | def loadavg(self): | |
c2902b29 | 423 | return ", ".join(["%.2f" % l for l in (self.loadavg1, self.loadavg5, self.loadavg15)]) |
9137135a | 424 | |
f6e6ff79 | 425 | @property |
c2902b29 MT |
426 | def loadavg1(self): |
427 | return self.data.loadavg1 or 0.0 | |
428 | ||
429 | @property | |
430 | def loadavg5(self): | |
431 | return self.data.loadavg5 or 0.0 | |
f6e6ff79 | 432 | |
c2902b29 MT |
433 | @property |
434 | def loadavg15(self): | |
435 | return self.data.loadavg15 or 0.0 | |
f6e6ff79 MT |
436 | |
437 | @property | |
438 | def pakfire_version(self): | |
439 | return self.data.pakfire_version or "" | |
9137135a | 440 | |
c2902b29 MT |
441 | @property |
442 | def os_name(self): | |
443 | return self.data.os_name or "" | |
444 | ||
9137135a MT |
445 | @property |
446 | def cpu_model(self): | |
f6e6ff79 MT |
447 | return self.data.cpu_model or "" |
448 | ||
449 | @property | |
450 | def cpu_count(self): | |
451 | return self.data.cpu_count | |
9137135a MT |
452 | |
453 | @property | |
c2902b29 MT |
454 | def cpu_arch(self): |
455 | return self.data.cpu_arch | |
456 | ||
457 | @property | |
458 | def cpu_bogomips(self): | |
459 | return self.data.cpu_bogomips or 0.0 | |
460 | ||
461 | @property | |
462 | def mem_percentage(self): | |
463 | if not self.mem_total: | |
464 | return None | |
465 | ||
466 | return self.mem_used * 100 / self.mem_total | |
9137135a MT |
467 | |
468 | @property | |
c2902b29 MT |
469 | def mem_total(self): |
470 | return self.data.mem_total | |
471 | ||
472 | @property | |
473 | def mem_used(self): | |
474 | if self.mem_total and self.mem_free: | |
475 | return self.mem_total - self.mem_free | |
476 | ||
477 | @property | |
478 | def mem_free(self): | |
479 | return self.data.mem_free | |
480 | ||
481 | @property | |
482 | def swap_percentage(self): | |
483 | if not self.swap_total: | |
484 | return None | |
485 | ||
486 | return self.swap_used * 100 / self.swap_total | |
487 | ||
488 | @property | |
489 | def swap_total(self): | |
490 | return self.data.swap_total | |
491 | ||
492 | @property | |
493 | def swap_used(self): | |
494 | if self.swap_total and self.swap_free: | |
495 | return self.swap_total - self.swap_free | |
496 | ||
497 | @property | |
498 | def swap_free(self): | |
499 | return self.data.swap_free | |
500 | ||
501 | @property | |
502 | def space_free(self): | |
503 | return self.data.space_free | |
f6e6ff79 MT |
504 | |
505 | @property | |
506 | def overload(self): | |
c2902b29 MT |
507 | if not self.cpu_count or not self.loadavg1: |
508 | return None | |
509 | ||
510 | return self.loadavg1 >= self.cpu_count | |
f6e6ff79 MT |
511 | |
512 | @property | |
513 | def host_key_id(self): | |
514 | return self.data.host_key_id | |
515 | ||
516 | @property | |
517 | def state(self): | |
9137135a | 518 | if self.disabled: |
f6e6ff79 | 519 | return "disabled" |
9137135a | 520 | |
f6e6ff79 MT |
521 | if self.data.time_keepalive is None: |
522 | return "offline" | |
9137135a | 523 | |
f96eb5ed MT |
524 | #if self.data.updated >= 5*60: |
525 | # return "offline" | |
9137135a | 526 | |
f6e6ff79 | 527 | return "online" |
9137135a | 528 | |
163d9d8b MT |
529 | def get_active_jobs(self, *args, **kwargs): |
530 | if self._active_jobs is None: | |
531 | self._active_jobs = self.pakfire.jobs.get_active(builder=self, *args, **kwargs) | |
532 | ||
533 | return self._active_jobs | |
534 | ||
535 | def count_active_jobs(self): | |
536 | return len(self.get_active_jobs()) | |
537 | ||
538 | @property | |
539 | def too_many_jobs(self): | |
540 | """ | |
541 | Tell if this host is already running enough or too many jobs. | |
542 | """ | |
543 | return self.count_active_jobs() >= self.max_jobs | |
9e8a20d7 | 544 | |
c2902b29 MT |
545 | def get_next_jobs(self, limit=None): |
546 | """ | |
547 | Returns a list of jobs that can be built on this host. | |
548 | """ | |
549 | return self.pakfire.jobs.get_next(arches=self.arches, limit=limit) | |
9e8a20d7 | 550 | |
c2902b29 MT |
551 | def get_next_job(self): |
552 | """ | |
553 | Returns the next job in line for this builder. | |
554 | """ | |
555 | # Get the first item of all jobs in the list. | |
556 | jobs = self.pakfire.jobs.get_next(builder=self, state="pending", limit=1) | |
9137135a | 557 | |
163d9d8b MT |
558 | if jobs: |
559 | return jobs[0] | |
f6e6ff79 MT |
560 | |
561 | def get_history(self, *args, **kwargs): | |
562 | kwargs["builder"] = self | |
563 | ||
564 | return self.pakfire.builders.get_history(*args, **kwargs) |