]> git.ipfire.org Git - people/jschlag/pbs.git/blame - src/buildservice/builders.py
arches: Base Arch class on DataObject
[people/jschlag/pbs.git] / src / buildservice / builders.py
CommitLineData
9137135a
MT
1#!/usr/bin/python
2
2c909128 3from __future__ import absolute_import, division
f96eb5ed 4
9137135a
MT
5import datetime
6import hashlib
7import logging
8import random
9import string
10import time
11
2c909128
MT
12from . import base
13from . import logs
f6e6ff79 14
2c909128 15from .users import generate_password_hash, check_password_hash, generate_random_string
9137135a
MT
16
17class Builders(base.Object):
f6e6ff79
MT
18 def auth(self, name, passphrase):
19 # If either name or passphrase is None, we don't check at all.
20 if None in (name, passphrase):
21 return
22
23 # Search for the hostname in the database.
24 # The builder must not be deleted.
25 builder = self.db.get("SELECT id FROM builders WHERE name = %s AND \
26 NOT status = 'deleted'", name)
27
28 if not builder:
29 return
30
31 # Get the whole Builder object from the database.
32 builder = self.get_by_id(builder.id)
33
34 # If the builder was not found or the passphrase does not match,
35 # you have bad luck.
36 if not builder or not builder.validate_passphrase(passphrase):
37 return
38
39 # Otherwise we return the Builder object.
40 return builder
41
9137135a 42 def get_all(self):
9fa1787c 43 builders = self.db.query("SELECT * FROM builders WHERE NOT status = 'deleted' ORDER BY name")
9137135a 44
9fa1787c 45 return [Builder(self.pakfire, b.id, b) for b in builders]
9137135a
MT
46
47 def get_by_id(self, id):
48 if not id:
49 return
50
51 return Builder(self.pakfire, id)
52
53 def get_by_name(self, name):
9fa1787c 54 builder = self.db.get("SELECT * FROM builders WHERE name = %s LIMIT 1", name)
9137135a
MT
55
56 if builder:
9fa1787c 57 return Builder(self.pakfire, builder.id, builder)
9137135a
MT
58
59 def get_all_arches(self):
60 arches = set()
61
62 for result in self.db.query("SELECT DISTINCT arches FROM builders"):
63 if not result.arches:
64 continue
65
66 _arches = result.arches.split()
67
68 for arch in _arches:
69 arches.add(arch)
70
71 return sorted(arches)
72
f6e6ff79 73 def get_load(self):
f96eb5ed
MT
74 res1 = self.db.get("SELECT SUM(max_jobs) AS max_jobs FROM builders \
75 WHERE status = 'enabled'")
f6e6ff79 76
f96eb5ed
MT
77 res2 = self.db.get("SELECT COUNT(*) AS count FROM jobs \
78 WHERE state = 'dispatching' OR state = 'running' OR state = 'uploading'")
f6e6ff79 79
33857fed
MT
80 try:
81 return (res2.count * 100 / res1.max_jobs)
82 except:
83 return 0
f6e6ff79
MT
84
85 def get_history(self, limit=None, offset=None, builder=None, user=None):
86 query = "SELECT * FROM builders_history"
87 args = []
88
89 conditions = []
90
91 if builder:
92 conditions.append("builder_id = %s")
93 args.append(builder.id)
94
95 if user:
96 conditions.append("user_id = %s")
97 args.append(user.id)
98
99 if conditions:
100 query += " WHERE %s" % " AND ".join(conditions)
101
102 query += " ORDER BY time DESC"
103
104 if limit:
105 if offset:
106 query += " LIMIT %s,%s"
107 args += [offset, limit,]
108 else:
109 query += " LIMIT %s"
110 args += [limit,]
111
112 entries = []
113 for entry in self.db.query(query, *args):
114 entry = logs.BuilderLogEntry(self.pakfire, entry)
115 entries.append(entry)
116
117 return entries
118
9137135a
MT
119
120class Builder(base.Object):
9fa1787c 121 def __init__(self, pakfire, id, data=None):
9137135a
MT
122 base.Object.__init__(self, pakfire)
123
124 self.id = id
125
f6e6ff79 126 # Cache.
9fa1787c 127 self._data = data
f6e6ff79 128 self._active_jobs = None
f6e6ff79 129 self._disabled_arches = None
9137135a
MT
130
131 def __cmp__(self, other):
318a6dfd
MT
132 if other is None:
133 return -1
134
9137135a
MT
135 return cmp(self.id, other.id)
136
f6e6ff79
MT
137 @property
138 def data(self):
139 if self._data is None:
f96eb5ed
MT
140 self._data = self.db.get("SELECT * FROM builders WHERE id = %s", self.id)
141 assert self._data
f6e6ff79
MT
142
143 return self._data
144
9137135a 145 @classmethod
f6e6ff79
MT
146 def create(cls, pakfire, name, user=None, log=True):
147 """
148 Creates a new builder.
149 """
150 builder_id = pakfire.db.execute("INSERT INTO builders(name, time_created) \
151 VALUES(%s, NOW())", name)
9137135a 152
f6e6ff79
MT
153 # Create Builder object.
154 builder = cls(pakfire, builder_id)
9137135a 155
f6e6ff79
MT
156 # Generate a new passphrase.
157 passphrase = builder.regenerate_passphrase()
158
159 # Log what we have done.
160 if log:
161 builder.log("created", user=user)
162
163 # The Builder object and the passphrase are returned.
164 return builder, passphrase
165
166 def log(self, action, user=None):
167 user_id = None
168 if user:
169 user_id = user.id
170
171 self.db.execute("INSERT INTO builders_history(builder_id, action, user_id, time) \
172 VALUES(%s, %s, %s, NOW())", self.id, action, user_id)
9137135a
MT
173
174 def set(self, key, value):
175 self.db.execute("UPDATE builders SET %s = %%s WHERE id = %%s LIMIT 1" % key,
176 value, self.id)
177 self.data[key] = value
178
9137135a 179 def regenerate_passphrase(self):
f6e6ff79
MT
180 """
181 Generates a new random passphrase and stores it as a salted hash
182 to the database.
183
184 The new passphrase is returned to be sent to the user (once).
185 """
186 # Generate a random string with 20 chars.
187 passphrase = generate_random_string(length=20)
188
189 # Create salted hash.
190 passphrase_hash = generate_password_hash(passphrase)
191
192 # Store the hash in the database.
193 self.db.execute("UPDATE builders SET passphrase = %s WHERE id = %s",
194 passphrase_hash, self.id)
9137135a 195
f6e6ff79
MT
196 # Return the clear-text passphrase.
197 return passphrase
9137135a
MT
198
199 def validate_passphrase(self, passphrase):
f6e6ff79
MT
200 """
201 Compare the given passphrase with the one stored in the database.
202 """
203 return check_password_hash(passphrase, self.data.passphrase)
204
205 @property
206 def description(self):
207 return self.data.description or ""
208
209 @property
210 def status(self):
211 return self.data.status
212
213 def update_description(self, description):
214 self.db.execute("UPDATE builders SET description = %s, time_updated = NOW() \
215 WHERE id = %s", description, self.id)
216
217 if self._data:
218 self._data["description"] = description
219
220 @property
221 def keepalive(self):
222 """
223 Returns time of last keepalive message from this host.
224 """
225 return self.data.time_keepalive
226
c2902b29
MT
227 def update_keepalive(self, loadavg1=None, loadavg5=None, loadavg15=None,
228 mem_total=None, mem_free=None, swap_total=None, swap_free=None,
229 space_free=None):
f6e6ff79
MT
230 """
231 Update the keepalive timestamp of this machine.
232 """
c2902b29
MT
233 self.db.execute("UPDATE builders SET time_keepalive = NOW(), \
234 loadavg1 = %s, loadavg5 = %s, loadavg15 = %s, space_free = %s, \
235 mem_total = %s, mem_free = %s, swap_total = %s, swap_free = %s \
236 WHERE id = %s", loadavg1, loadavg5, loadavg15, space_free,
237 mem_total, mem_free, swap_total, swap_free, self.id)
238
239 def update_info(self, cpu_model=None, cpu_count=None, cpu_arch=None, cpu_bogomips=None,
240 pakfire_version=None, host_key=None, os_name=None):
f6e6ff79
MT
241 # Update all the rest.
242 self.db.execute("UPDATE builders SET time_updated = NOW(), \
c2902b29
MT
243 pakfire_version = %s, cpu_model = %s, cpu_count = %s, cpu_arch = %s, \
244 cpu_bogomips = %s, host_key_id = %s, os_name = %s WHERE id = %s",
245 pakfire_version, cpu_model, cpu_count, cpu_arch, cpu_bogomips,
246 host_key, os_name, self.id)
f6e6ff79
MT
247
248 def update_arches(self, arches):
249 # Get all arches this builder does currently support.
250 supported_arches = [a.name for a in self.get_arches()]
251
252 # Noarch is always supported.
253 if not "noarch" in arches:
254 arches.append("noarch")
255
256 arches_add = []
257 for arch in arches:
258 if arch in supported_arches:
259 supported_arches.remove(arch)
260 continue
261
262 arches_add.append(arch)
263 arches_rem = supported_arches
264
265 for arch_name in arches_add:
266 arch = self.pakfire.arches.get_by_name(arch_name)
267 if not arch:
268 logging.info("Client sent unknown architecture: %s" % arch_name)
269 continue
270
271 self.db.execute("INSERT INTO builders_arches(builder_id, arch_id) \
272 VALUES(%s, %s)", self.id, arch.id)
273
274 for arch_name in arches_rem:
275 arch = self.pakfire.arches.get_by_name(arch_name)
276 assert arch
277
278 self.db.execute("DELETE FROM builders_arches WHERE builder_id = %s \
279 AND arch_id = %s", self.id, arch.id)
9137135a 280
9137135a 281 def get_enabled(self):
f6e6ff79 282 return self.status == "enabled"
9137135a
MT
283
284 def set_enabled(self, value):
f6e6ff79
MT
285 # XXX deprecated
286
9137135a 287 if value:
f6e6ff79 288 value = "enabled"
9137135a 289 else:
f6e6ff79 290 value = "disabled"
9137135a 291
f6e6ff79 292 self.set_status(value)
9137135a
MT
293
294 enabled = property(get_enabled, set_enabled)
295
296 @property
297 def disabled(self):
f6e6ff79
MT
298 return not self.enabled
299
300 def set_status(self, status, user=None, log=True):
301 assert status in ("created", "enabled", "disabled", "deleted")
302
303 if self.status == status:
304 return
305
306 self.db.execute("UPDATE builders SET status = %s WHERE id = %s",
307 status, self.id)
308
309 if self._data:
310 self._data["status"] = status
311
312 if log:
313 self.log(status, user=user)
314
9137135a
MT
315 @property
316 def arches(self):
c2902b29 317 if not hasattr(self, "_arches"):
32ba6ec7 318 self._arches = []
9137135a 319
c2902b29
MT
320 if self.cpu_arch:
321 res = self.db.query("SELECT build_arch FROM arches_compat \
322 WHERE host_arch = %s", self.cpu_arch)
f6e6ff79 323
c2902b29
MT
324 self._arches += [r.build_arch for r in res]
325 if not self.cpu_arch in self._arches:
326 self._arches.append(self.cpu_arch)
f6e6ff79 327
c2902b29 328 return self._arches
9137135a 329
f6e6ff79
MT
330 def get_build_release(self):
331 return self.data.build_release == "Y"
332
333 def set_build_release(self, value):
9137135a
MT
334 if value:
335 value = "Y"
336 else:
337 value = "N"
338
f6e6ff79
MT
339 self.db.execute("UPDATE builders SET build_release = %s WHERE id = %s",
340 value, self.id)
341
342 # Update the cache.
343 if self._data:
344 self._data["build_release"] = value
9137135a 345
f6e6ff79 346 build_release = property(get_build_release, set_build_release)
9137135a 347
f6e6ff79
MT
348 def get_build_scratch(self):
349 return self.data.build_scratch == "Y"
9137135a 350
f6e6ff79 351 def set_build_scratch(self, value):
9137135a
MT
352 if value:
353 value = "Y"
354 else:
355 value = "N"
356
f6e6ff79
MT
357 self.db.execute("UPDATE builders SET build_scratch = %s WHERE id = %s",
358 value, self.id)
359
360 # Update the cache.
361 if self._data:
362 self._data["build_scratch"] = value
9137135a 363
f6e6ff79 364 build_scratch = property(get_build_scratch, set_build_scratch)
9137135a
MT
365
366 def get_build_test(self):
367 return self.data.build_test == "Y"
368
369 def set_build_test(self, value):
370 if value:
371 value = "Y"
372 else:
373 value = "N"
374
f6e6ff79
MT
375 self.db.execute("UPDATE builders SET build_test = %s WHERE id = %s",
376 value, self.id)
377
378 # Update the cache.
379 if self._data:
380 self._data["build_test"] = value
9137135a
MT
381
382 build_test = property(get_build_test, set_build_test)
383
f6e6ff79
MT
384 @property
385 def build_types(self):
386 ret = []
387
388 if self.build_release:
389 ret.append("release")
390
391 if self.build_scratch:
392 ret.append("scratch")
393
394 if self.build_test:
395 ret.append("test")
396
397 return ret
398
9137135a
MT
399 def get_max_jobs(self):
400 return self.data.max_jobs
401
402 def set_max_jobs(self, value):
403 self.set("max_jobs", value)
404
405 max_jobs = property(get_max_jobs, set_max_jobs)
406
407 @property
408 def name(self):
409 return self.data.name
410
411 @property
412 def hostname(self):
413 return self.name
414
415 @property
416 def passphrase(self):
417 return self.data.passphrase
418
c2902b29
MT
419 # Load average
420
9137135a
MT
421 @property
422 def loadavg(self):
c2902b29 423 return ", ".join(["%.2f" % l for l in (self.loadavg1, self.loadavg5, self.loadavg15)])
9137135a 424
f6e6ff79 425 @property
c2902b29
MT
426 def loadavg1(self):
427 return self.data.loadavg1 or 0.0
428
429 @property
430 def loadavg5(self):
431 return self.data.loadavg5 or 0.0
f6e6ff79 432
c2902b29
MT
433 @property
434 def loadavg15(self):
435 return self.data.loadavg15 or 0.0
f6e6ff79
MT
436
437 @property
438 def pakfire_version(self):
439 return self.data.pakfire_version or ""
9137135a 440
c2902b29
MT
441 @property
442 def os_name(self):
443 return self.data.os_name or ""
444
9137135a
MT
445 @property
446 def cpu_model(self):
f6e6ff79
MT
447 return self.data.cpu_model or ""
448
449 @property
450 def cpu_count(self):
451 return self.data.cpu_count
9137135a
MT
452
453 @property
c2902b29
MT
454 def cpu_arch(self):
455 return self.data.cpu_arch
456
457 @property
458 def cpu_bogomips(self):
459 return self.data.cpu_bogomips or 0.0
460
461 @property
462 def mem_percentage(self):
463 if not self.mem_total:
464 return None
465
466 return self.mem_used * 100 / self.mem_total
9137135a
MT
467
468 @property
c2902b29
MT
469 def mem_total(self):
470 return self.data.mem_total
471
472 @property
473 def mem_used(self):
474 if self.mem_total and self.mem_free:
475 return self.mem_total - self.mem_free
476
477 @property
478 def mem_free(self):
479 return self.data.mem_free
480
481 @property
482 def swap_percentage(self):
483 if not self.swap_total:
484 return None
485
486 return self.swap_used * 100 / self.swap_total
487
488 @property
489 def swap_total(self):
490 return self.data.swap_total
491
492 @property
493 def swap_used(self):
494 if self.swap_total and self.swap_free:
495 return self.swap_total - self.swap_free
496
497 @property
498 def swap_free(self):
499 return self.data.swap_free
500
501 @property
502 def space_free(self):
503 return self.data.space_free
f6e6ff79
MT
504
505 @property
506 def overload(self):
c2902b29
MT
507 if not self.cpu_count or not self.loadavg1:
508 return None
509
510 return self.loadavg1 >= self.cpu_count
f6e6ff79
MT
511
512 @property
513 def host_key_id(self):
514 return self.data.host_key_id
515
516 @property
517 def state(self):
9137135a 518 if self.disabled:
f6e6ff79 519 return "disabled"
9137135a 520
f6e6ff79
MT
521 if self.data.time_keepalive is None:
522 return "offline"
9137135a 523
f96eb5ed
MT
524 #if self.data.updated >= 5*60:
525 # return "offline"
9137135a 526
f6e6ff79 527 return "online"
9137135a 528
163d9d8b
MT
529 def get_active_jobs(self, *args, **kwargs):
530 if self._active_jobs is None:
531 self._active_jobs = self.pakfire.jobs.get_active(builder=self, *args, **kwargs)
532
533 return self._active_jobs
534
535 def count_active_jobs(self):
536 return len(self.get_active_jobs())
537
538 @property
539 def too_many_jobs(self):
540 """
541 Tell if this host is already running enough or too many jobs.
542 """
543 return self.count_active_jobs() >= self.max_jobs
9e8a20d7 544
c2902b29
MT
545 def get_next_jobs(self, limit=None):
546 """
547 Returns a list of jobs that can be built on this host.
548 """
549 return self.pakfire.jobs.get_next(arches=self.arches, limit=limit)
9e8a20d7 550
c2902b29
MT
551 def get_next_job(self):
552 """
553 Returns the next job in line for this builder.
554 """
555 # Get the first item of all jobs in the list.
556 jobs = self.pakfire.jobs.get_next(builder=self, state="pending", limit=1)
9137135a 557
163d9d8b
MT
558 if jobs:
559 return jobs[0]
f6e6ff79
MT
560
561 def get_history(self, *args, **kwargs):
562 kwargs["builder"] = self
563
564 return self.pakfire.builders.get_history(*args, **kwargs)