]>
Commit | Line | Data |
---|---|---|
9137135a MT |
1 | #!/usr/bin/python |
2 | ||
2c909128 | 3 | from __future__ import absolute_import, division |
f96eb5ed | 4 | |
9137135a MT |
5 | import datetime |
6 | import hashlib | |
7 | import logging | |
8 | import random | |
9 | import string | |
10 | import time | |
11 | ||
2c909128 MT |
12 | from . import base |
13 | from . import logs | |
f6e6ff79 | 14 | |
3e990438 MT |
15 | from .decorators import * |
16 | ||
2c909128 | 17 | from .users import generate_password_hash, check_password_hash, generate_random_string |
9137135a MT |
18 | |
19 | class Builders(base.Object): | |
f6e6ff79 MT |
20 | def auth(self, name, passphrase): |
21 | # If either name or passphrase is None, we don't check at all. | |
22 | if None in (name, passphrase): | |
23 | return | |
24 | ||
25 | # Search for the hostname in the database. | |
26 | # The builder must not be deleted. | |
27 | builder = self.db.get("SELECT id FROM builders WHERE name = %s AND \ | |
28 | NOT status = 'deleted'", name) | |
29 | ||
30 | if not builder: | |
31 | return | |
32 | ||
33 | # Get the whole Builder object from the database. | |
34 | builder = self.get_by_id(builder.id) | |
35 | ||
36 | # If the builder was not found or the passphrase does not match, | |
37 | # you have bad luck. | |
38 | if not builder or not builder.validate_passphrase(passphrase): | |
39 | return | |
40 | ||
41 | # Otherwise we return the Builder object. | |
42 | return builder | |
43 | ||
9137135a | 44 | def get_all(self): |
9fa1787c | 45 | builders = self.db.query("SELECT * FROM builders WHERE NOT status = 'deleted' ORDER BY name") |
9137135a | 46 | |
9fa1787c | 47 | return [Builder(self.pakfire, b.id, b) for b in builders] |
9137135a MT |
48 | |
49 | def get_by_id(self, id): | |
50 | if not id: | |
51 | return | |
52 | ||
53 | return Builder(self.pakfire, id) | |
54 | ||
55 | def get_by_name(self, name): | |
9fa1787c | 56 | builder = self.db.get("SELECT * FROM builders WHERE name = %s LIMIT 1", name) |
9137135a MT |
57 | |
58 | if builder: | |
9fa1787c | 59 | return Builder(self.pakfire, builder.id, builder) |
9137135a | 60 | |
f6e6ff79 | 61 | def get_load(self): |
f96eb5ed MT |
62 | res1 = self.db.get("SELECT SUM(max_jobs) AS max_jobs FROM builders \ |
63 | WHERE status = 'enabled'") | |
f6e6ff79 | 64 | |
f96eb5ed MT |
65 | res2 = self.db.get("SELECT COUNT(*) AS count FROM jobs \ |
66 | WHERE state = 'dispatching' OR state = 'running' OR state = 'uploading'") | |
f6e6ff79 | 67 | |
33857fed MT |
68 | try: |
69 | return (res2.count * 100 / res1.max_jobs) | |
70 | except: | |
71 | return 0 | |
f6e6ff79 MT |
72 | |
73 | def get_history(self, limit=None, offset=None, builder=None, user=None): | |
74 | query = "SELECT * FROM builders_history" | |
75 | args = [] | |
76 | ||
77 | conditions = [] | |
78 | ||
79 | if builder: | |
80 | conditions.append("builder_id = %s") | |
81 | args.append(builder.id) | |
82 | ||
83 | if user: | |
84 | conditions.append("user_id = %s") | |
85 | args.append(user.id) | |
86 | ||
87 | if conditions: | |
88 | query += " WHERE %s" % " AND ".join(conditions) | |
89 | ||
90 | query += " ORDER BY time DESC" | |
91 | ||
92 | if limit: | |
93 | if offset: | |
94 | query += " LIMIT %s,%s" | |
95 | args += [offset, limit,] | |
96 | else: | |
97 | query += " LIMIT %s" | |
98 | args += [limit,] | |
99 | ||
100 | entries = [] | |
101 | for entry in self.db.query(query, *args): | |
102 | entry = logs.BuilderLogEntry(self.pakfire, entry) | |
103 | entries.append(entry) | |
104 | ||
105 | return entries | |
106 | ||
9137135a | 107 | |
3e990438 | 108 | class Builder(base.DataObject): |
92431da4 MT |
109 | table = "builders" |
110 | ||
9137135a | 111 | def __cmp__(self, other): |
318a6dfd MT |
112 | if other is None: |
113 | return -1 | |
114 | ||
9137135a MT |
115 | return cmp(self.id, other.id) |
116 | ||
117 | @classmethod | |
f6e6ff79 MT |
118 | def create(cls, pakfire, name, user=None, log=True): |
119 | """ | |
120 | Creates a new builder. | |
121 | """ | |
122 | builder_id = pakfire.db.execute("INSERT INTO builders(name, time_created) \ | |
123 | VALUES(%s, NOW())", name) | |
9137135a | 124 | |
f6e6ff79 MT |
125 | # Create Builder object. |
126 | builder = cls(pakfire, builder_id) | |
9137135a | 127 | |
f6e6ff79 MT |
128 | # Generate a new passphrase. |
129 | passphrase = builder.regenerate_passphrase() | |
130 | ||
131 | # Log what we have done. | |
132 | if log: | |
133 | builder.log("created", user=user) | |
134 | ||
135 | # The Builder object and the passphrase are returned. | |
136 | return builder, passphrase | |
137 | ||
138 | def log(self, action, user=None): | |
139 | user_id = None | |
140 | if user: | |
141 | user_id = user.id | |
142 | ||
143 | self.db.execute("INSERT INTO builders_history(builder_id, action, user_id, time) \ | |
144 | VALUES(%s, %s, %s, NOW())", self.id, action, user_id) | |
9137135a | 145 | |
9137135a | 146 | def regenerate_passphrase(self): |
f6e6ff79 MT |
147 | """ |
148 | Generates a new random passphrase and stores it as a salted hash | |
149 | to the database. | |
150 | ||
151 | The new passphrase is returned to be sent to the user (once). | |
152 | """ | |
153 | # Generate a random string with 20 chars. | |
154 | passphrase = generate_random_string(length=20) | |
155 | ||
156 | # Create salted hash. | |
157 | passphrase_hash = generate_password_hash(passphrase) | |
158 | ||
159 | # Store the hash in the database. | |
3e990438 | 160 | self._set_attribute("passphrase", passphrase_hash) |
9137135a | 161 | |
f6e6ff79 MT |
162 | # Return the clear-text passphrase. |
163 | return passphrase | |
9137135a MT |
164 | |
165 | def validate_passphrase(self, passphrase): | |
f6e6ff79 MT |
166 | """ |
167 | Compare the given passphrase with the one stored in the database. | |
168 | """ | |
169 | return check_password_hash(passphrase, self.data.passphrase) | |
170 | ||
3e990438 MT |
171 | # Description |
172 | ||
173 | def set_description(self, description): | |
174 | self._set_attribute("description", description) | |
175 | ||
176 | description = property(lambda s: s.data.description or "", set_description) | |
f6e6ff79 MT |
177 | |
178 | @property | |
179 | def status(self): | |
180 | return self.data.status | |
181 | ||
f6e6ff79 MT |
182 | @property |
183 | def keepalive(self): | |
184 | """ | |
185 | Returns time of last keepalive message from this host. | |
186 | """ | |
187 | return self.data.time_keepalive | |
188 | ||
c2902b29 MT |
189 | def update_keepalive(self, loadavg1=None, loadavg5=None, loadavg15=None, |
190 | mem_total=None, mem_free=None, swap_total=None, swap_free=None, | |
191 | space_free=None): | |
f6e6ff79 MT |
192 | """ |
193 | Update the keepalive timestamp of this machine. | |
194 | """ | |
c2902b29 MT |
195 | self.db.execute("UPDATE builders SET time_keepalive = NOW(), \ |
196 | loadavg1 = %s, loadavg5 = %s, loadavg15 = %s, space_free = %s, \ | |
197 | mem_total = %s, mem_free = %s, swap_total = %s, swap_free = %s \ | |
198 | WHERE id = %s", loadavg1, loadavg5, loadavg15, space_free, | |
199 | mem_total, mem_free, swap_total, swap_free, self.id) | |
200 | ||
201 | def update_info(self, cpu_model=None, cpu_count=None, cpu_arch=None, cpu_bogomips=None, | |
202 | pakfire_version=None, host_key=None, os_name=None): | |
f6e6ff79 MT |
203 | # Update all the rest. |
204 | self.db.execute("UPDATE builders SET time_updated = NOW(), \ | |
c2902b29 MT |
205 | pakfire_version = %s, cpu_model = %s, cpu_count = %s, cpu_arch = %s, \ |
206 | cpu_bogomips = %s, host_key_id = %s, os_name = %s WHERE id = %s", | |
207 | pakfire_version, cpu_model, cpu_count, cpu_arch, cpu_bogomips, | |
208 | host_key, os_name, self.id) | |
f6e6ff79 | 209 | |
9137135a | 210 | def get_enabled(self): |
f6e6ff79 | 211 | return self.status == "enabled" |
9137135a MT |
212 | |
213 | def set_enabled(self, value): | |
f6e6ff79 MT |
214 | # XXX deprecated |
215 | ||
9137135a | 216 | if value: |
f6e6ff79 | 217 | value = "enabled" |
9137135a | 218 | else: |
f6e6ff79 | 219 | value = "disabled" |
9137135a | 220 | |
f6e6ff79 | 221 | self.set_status(value) |
9137135a MT |
222 | |
223 | enabled = property(get_enabled, set_enabled) | |
224 | ||
225 | @property | |
226 | def disabled(self): | |
f6e6ff79 MT |
227 | return not self.enabled |
228 | ||
229 | def set_status(self, status, user=None, log=True): | |
230 | assert status in ("created", "enabled", "disabled", "deleted") | |
231 | ||
232 | if self.status == status: | |
233 | return | |
234 | ||
3e990438 | 235 | self._set_attribute("status", status) |
f6e6ff79 MT |
236 | |
237 | if log: | |
238 | self.log(status, user=user) | |
239 | ||
3e990438 | 240 | @lazy_property |
9137135a | 241 | def arches(self): |
3e990438 MT |
242 | if self.cpu_arch: |
243 | res = self.db.query("SELECT build_arch FROM arches_compat \ | |
244 | WHERE host_arch = %s", self.cpu_arch) | |
9137135a | 245 | |
3e990438 MT |
246 | arches += [r.build_arch for r in res] |
247 | if not self.cpu_arch in arches: | |
248 | arches.append(self.cpu_arch) | |
f6e6ff79 | 249 | |
3e990438 | 250 | return arches |
f6e6ff79 | 251 | |
3e990438 | 252 | return [] |
9137135a | 253 | |
f6e6ff79 MT |
254 | def get_build_release(self): |
255 | return self.data.build_release == "Y" | |
256 | ||
257 | def set_build_release(self, value): | |
9137135a MT |
258 | if value: |
259 | value = "Y" | |
260 | else: | |
261 | value = "N" | |
262 | ||
f6e6ff79 MT |
263 | self.db.execute("UPDATE builders SET build_release = %s WHERE id = %s", |
264 | value, self.id) | |
265 | ||
266 | # Update the cache. | |
267 | if self._data: | |
268 | self._data["build_release"] = value | |
9137135a | 269 | |
f6e6ff79 | 270 | build_release = property(get_build_release, set_build_release) |
9137135a | 271 | |
f6e6ff79 MT |
272 | def get_build_scratch(self): |
273 | return self.data.build_scratch == "Y" | |
9137135a | 274 | |
f6e6ff79 | 275 | def set_build_scratch(self, value): |
9137135a MT |
276 | if value: |
277 | value = "Y" | |
278 | else: | |
279 | value = "N" | |
280 | ||
f6e6ff79 MT |
281 | self.db.execute("UPDATE builders SET build_scratch = %s WHERE id = %s", |
282 | value, self.id) | |
283 | ||
284 | # Update the cache. | |
285 | if self._data: | |
286 | self._data["build_scratch"] = value | |
9137135a | 287 | |
f6e6ff79 | 288 | build_scratch = property(get_build_scratch, set_build_scratch) |
9137135a MT |
289 | |
290 | def get_build_test(self): | |
291 | return self.data.build_test == "Y" | |
292 | ||
293 | def set_build_test(self, value): | |
294 | if value: | |
295 | value = "Y" | |
296 | else: | |
297 | value = "N" | |
298 | ||
f6e6ff79 MT |
299 | self.db.execute("UPDATE builders SET build_test = %s WHERE id = %s", |
300 | value, self.id) | |
301 | ||
302 | # Update the cache. | |
303 | if self._data: | |
304 | self._data["build_test"] = value | |
9137135a MT |
305 | |
306 | build_test = property(get_build_test, set_build_test) | |
307 | ||
f6e6ff79 MT |
308 | @property |
309 | def build_types(self): | |
310 | ret = [] | |
311 | ||
312 | if self.build_release: | |
313 | ret.append("release") | |
314 | ||
315 | if self.build_scratch: | |
316 | ret.append("scratch") | |
317 | ||
318 | if self.build_test: | |
319 | ret.append("test") | |
320 | ||
321 | return ret | |
322 | ||
9137135a | 323 | def set_max_jobs(self, value): |
3e990438 | 324 | self._set_attribute("max_jobs", value) |
9137135a | 325 | |
3e990438 | 326 | max_jobs = property(lambda s: s.data.max_jobs, set_max_jobs) |
9137135a MT |
327 | |
328 | @property | |
329 | def name(self): | |
330 | return self.data.name | |
331 | ||
332 | @property | |
333 | def hostname(self): | |
334 | return self.name | |
335 | ||
336 | @property | |
337 | def passphrase(self): | |
338 | return self.data.passphrase | |
339 | ||
c2902b29 MT |
340 | # Load average |
341 | ||
9137135a MT |
342 | @property |
343 | def loadavg(self): | |
c2902b29 | 344 | return ", ".join(["%.2f" % l for l in (self.loadavg1, self.loadavg5, self.loadavg15)]) |
9137135a | 345 | |
f6e6ff79 | 346 | @property |
c2902b29 MT |
347 | def loadavg1(self): |
348 | return self.data.loadavg1 or 0.0 | |
349 | ||
350 | @property | |
351 | def loadavg5(self): | |
352 | return self.data.loadavg5 or 0.0 | |
f6e6ff79 | 353 | |
c2902b29 MT |
354 | @property |
355 | def loadavg15(self): | |
356 | return self.data.loadavg15 or 0.0 | |
f6e6ff79 MT |
357 | |
358 | @property | |
359 | def pakfire_version(self): | |
360 | return self.data.pakfire_version or "" | |
9137135a | 361 | |
c2902b29 MT |
362 | @property |
363 | def os_name(self): | |
364 | return self.data.os_name or "" | |
365 | ||
9137135a MT |
366 | @property |
367 | def cpu_model(self): | |
f6e6ff79 MT |
368 | return self.data.cpu_model or "" |
369 | ||
370 | @property | |
371 | def cpu_count(self): | |
372 | return self.data.cpu_count | |
9137135a MT |
373 | |
374 | @property | |
c2902b29 MT |
375 | def cpu_arch(self): |
376 | return self.data.cpu_arch | |
377 | ||
378 | @property | |
379 | def cpu_bogomips(self): | |
380 | return self.data.cpu_bogomips or 0.0 | |
381 | ||
382 | @property | |
383 | def mem_percentage(self): | |
384 | if not self.mem_total: | |
385 | return None | |
386 | ||
387 | return self.mem_used * 100 / self.mem_total | |
9137135a MT |
388 | |
389 | @property | |
c2902b29 MT |
390 | def mem_total(self): |
391 | return self.data.mem_total | |
392 | ||
393 | @property | |
394 | def mem_used(self): | |
395 | if self.mem_total and self.mem_free: | |
396 | return self.mem_total - self.mem_free | |
397 | ||
398 | @property | |
399 | def mem_free(self): | |
400 | return self.data.mem_free | |
401 | ||
402 | @property | |
403 | def swap_percentage(self): | |
404 | if not self.swap_total: | |
405 | return None | |
406 | ||
407 | return self.swap_used * 100 / self.swap_total | |
408 | ||
409 | @property | |
410 | def swap_total(self): | |
411 | return self.data.swap_total | |
412 | ||
413 | @property | |
414 | def swap_used(self): | |
415 | if self.swap_total and self.swap_free: | |
416 | return self.swap_total - self.swap_free | |
417 | ||
418 | @property | |
419 | def swap_free(self): | |
420 | return self.data.swap_free | |
421 | ||
422 | @property | |
423 | def space_free(self): | |
424 | return self.data.space_free | |
f6e6ff79 MT |
425 | |
426 | @property | |
427 | def overload(self): | |
c2902b29 MT |
428 | if not self.cpu_count or not self.loadavg1: |
429 | return None | |
430 | ||
431 | return self.loadavg1 >= self.cpu_count | |
f6e6ff79 MT |
432 | |
433 | @property | |
434 | def host_key_id(self): | |
435 | return self.data.host_key_id | |
436 | ||
437 | @property | |
438 | def state(self): | |
9137135a | 439 | if self.disabled: |
f6e6ff79 | 440 | return "disabled" |
9137135a | 441 | |
f6e6ff79 MT |
442 | if self.data.time_keepalive is None: |
443 | return "offline" | |
9137135a | 444 | |
f96eb5ed MT |
445 | #if self.data.updated >= 5*60: |
446 | # return "offline" | |
9137135a | 447 | |
f6e6ff79 | 448 | return "online" |
9137135a | 449 | |
3e990438 MT |
450 | @lazy_property |
451 | def active_jobs(self, *args, **kwargs): | |
452 | return self.pakfire.jobs.get_active(builder=self, *args, **kwargs) | |
163d9d8b MT |
453 | |
454 | @property | |
455 | def too_many_jobs(self): | |
456 | """ | |
457 | Tell if this host is already running enough or too many jobs. | |
458 | """ | |
3e990438 | 459 | return len(self.active_jobs) >= self.max_jobs |
9e8a20d7 | 460 | |
c2902b29 MT |
461 | def get_next_jobs(self, limit=None): |
462 | """ | |
463 | Returns a list of jobs that can be built on this host. | |
464 | """ | |
465 | return self.pakfire.jobs.get_next(arches=self.arches, limit=limit) | |
9e8a20d7 | 466 | |
c2902b29 MT |
467 | def get_next_job(self): |
468 | """ | |
469 | Returns the next job in line for this builder. | |
470 | """ | |
471 | # Get the first item of all jobs in the list. | |
472 | jobs = self.pakfire.jobs.get_next(builder=self, state="pending", limit=1) | |
9137135a | 473 | |
163d9d8b MT |
474 | if jobs: |
475 | return jobs[0] | |
f6e6ff79 MT |
476 | |
477 | def get_history(self, *args, **kwargs): | |
478 | kwargs["builder"] = self | |
479 | ||
480 | return self.pakfire.builders.get_history(*args, **kwargs) |