]> git.ipfire.org Git - people/jschlag/pbs.git/blame - src/buildservice/builders.py
Remove complicated handling of static files
[people/jschlag/pbs.git] / src / buildservice / builders.py
CommitLineData
9137135a
MT
1#!/usr/bin/python
2
2c909128 3from __future__ import absolute_import, division
f96eb5ed 4
9137135a
MT
5import datetime
6import hashlib
7import logging
8import random
9import string
10import time
11
2c909128
MT
12from . import base
13from . import logs
f6e6ff79 14
3e990438
MT
15from .decorators import *
16
2c909128 17from .users import generate_password_hash, check_password_hash, generate_random_string
9137135a
MT
18
19class Builders(base.Object):
f6e6ff79
MT
20 def auth(self, name, passphrase):
21 # If either name or passphrase is None, we don't check at all.
22 if None in (name, passphrase):
23 return
24
25 # Search for the hostname in the database.
26 # The builder must not be deleted.
27 builder = self.db.get("SELECT id FROM builders WHERE name = %s AND \
28 NOT status = 'deleted'", name)
29
30 if not builder:
31 return
32
33 # Get the whole Builder object from the database.
34 builder = self.get_by_id(builder.id)
35
36 # If the builder was not found or the passphrase does not match,
37 # you have bad luck.
38 if not builder or not builder.validate_passphrase(passphrase):
39 return
40
41 # Otherwise we return the Builder object.
42 return builder
43
9137135a 44 def get_all(self):
9fa1787c 45 builders = self.db.query("SELECT * FROM builders WHERE NOT status = 'deleted' ORDER BY name")
9137135a 46
9fa1787c 47 return [Builder(self.pakfire, b.id, b) for b in builders]
9137135a
MT
48
49 def get_by_id(self, id):
50 if not id:
51 return
52
53 return Builder(self.pakfire, id)
54
55 def get_by_name(self, name):
9fa1787c 56 builder = self.db.get("SELECT * FROM builders WHERE name = %s LIMIT 1", name)
9137135a
MT
57
58 if builder:
9fa1787c 59 return Builder(self.pakfire, builder.id, builder)
9137135a
MT
60
61 def get_all_arches(self):
62 arches = set()
63
64 for result in self.db.query("SELECT DISTINCT arches FROM builders"):
65 if not result.arches:
66 continue
67
68 _arches = result.arches.split()
69
70 for arch in _arches:
71 arches.add(arch)
72
73 return sorted(arches)
74
f6e6ff79 75 def get_load(self):
f96eb5ed
MT
76 res1 = self.db.get("SELECT SUM(max_jobs) AS max_jobs FROM builders \
77 WHERE status = 'enabled'")
f6e6ff79 78
f96eb5ed
MT
79 res2 = self.db.get("SELECT COUNT(*) AS count FROM jobs \
80 WHERE state = 'dispatching' OR state = 'running' OR state = 'uploading'")
f6e6ff79 81
33857fed
MT
82 try:
83 return (res2.count * 100 / res1.max_jobs)
84 except:
85 return 0
f6e6ff79
MT
86
87 def get_history(self, limit=None, offset=None, builder=None, user=None):
88 query = "SELECT * FROM builders_history"
89 args = []
90
91 conditions = []
92
93 if builder:
94 conditions.append("builder_id = %s")
95 args.append(builder.id)
96
97 if user:
98 conditions.append("user_id = %s")
99 args.append(user.id)
100
101 if conditions:
102 query += " WHERE %s" % " AND ".join(conditions)
103
104 query += " ORDER BY time DESC"
105
106 if limit:
107 if offset:
108 query += " LIMIT %s,%s"
109 args += [offset, limit,]
110 else:
111 query += " LIMIT %s"
112 args += [limit,]
113
114 entries = []
115 for entry in self.db.query(query, *args):
116 entry = logs.BuilderLogEntry(self.pakfire, entry)
117 entries.append(entry)
118
119 return entries
120
9137135a 121
3e990438 122class Builder(base.DataObject):
9137135a 123 def __cmp__(self, other):
318a6dfd
MT
124 if other is None:
125 return -1
126
9137135a
MT
127 return cmp(self.id, other.id)
128
129 @classmethod
f6e6ff79
MT
130 def create(cls, pakfire, name, user=None, log=True):
131 """
132 Creates a new builder.
133 """
134 builder_id = pakfire.db.execute("INSERT INTO builders(name, time_created) \
135 VALUES(%s, NOW())", name)
9137135a 136
f6e6ff79
MT
137 # Create Builder object.
138 builder = cls(pakfire, builder_id)
9137135a 139
f6e6ff79
MT
140 # Generate a new passphrase.
141 passphrase = builder.regenerate_passphrase()
142
143 # Log what we have done.
144 if log:
145 builder.log("created", user=user)
146
147 # The Builder object and the passphrase are returned.
148 return builder, passphrase
149
150 def log(self, action, user=None):
151 user_id = None
152 if user:
153 user_id = user.id
154
155 self.db.execute("INSERT INTO builders_history(builder_id, action, user_id, time) \
156 VALUES(%s, %s, %s, NOW())", self.id, action, user_id)
9137135a 157
9137135a 158 def regenerate_passphrase(self):
f6e6ff79
MT
159 """
160 Generates a new random passphrase and stores it as a salted hash
161 to the database.
162
163 The new passphrase is returned to be sent to the user (once).
164 """
165 # Generate a random string with 20 chars.
166 passphrase = generate_random_string(length=20)
167
168 # Create salted hash.
169 passphrase_hash = generate_password_hash(passphrase)
170
171 # Store the hash in the database.
3e990438 172 self._set_attribute("passphrase", passphrase_hash)
9137135a 173
f6e6ff79
MT
174 # Return the clear-text passphrase.
175 return passphrase
9137135a
MT
176
177 def validate_passphrase(self, passphrase):
f6e6ff79
MT
178 """
179 Compare the given passphrase with the one stored in the database.
180 """
181 return check_password_hash(passphrase, self.data.passphrase)
182
3e990438
MT
183 # Description
184
185 def set_description(self, description):
186 self._set_attribute("description", description)
187
188 description = property(lambda s: s.data.description or "", set_description)
f6e6ff79
MT
189
190 @property
191 def status(self):
192 return self.data.status
193
f6e6ff79
MT
194 @property
195 def keepalive(self):
196 """
197 Returns time of last keepalive message from this host.
198 """
199 return self.data.time_keepalive
200
c2902b29
MT
201 def update_keepalive(self, loadavg1=None, loadavg5=None, loadavg15=None,
202 mem_total=None, mem_free=None, swap_total=None, swap_free=None,
203 space_free=None):
f6e6ff79
MT
204 """
205 Update the keepalive timestamp of this machine.
206 """
c2902b29
MT
207 self.db.execute("UPDATE builders SET time_keepalive = NOW(), \
208 loadavg1 = %s, loadavg5 = %s, loadavg15 = %s, space_free = %s, \
209 mem_total = %s, mem_free = %s, swap_total = %s, swap_free = %s \
210 WHERE id = %s", loadavg1, loadavg5, loadavg15, space_free,
211 mem_total, mem_free, swap_total, swap_free, self.id)
212
213 def update_info(self, cpu_model=None, cpu_count=None, cpu_arch=None, cpu_bogomips=None,
214 pakfire_version=None, host_key=None, os_name=None):
f6e6ff79
MT
215 # Update all the rest.
216 self.db.execute("UPDATE builders SET time_updated = NOW(), \
c2902b29
MT
217 pakfire_version = %s, cpu_model = %s, cpu_count = %s, cpu_arch = %s, \
218 cpu_bogomips = %s, host_key_id = %s, os_name = %s WHERE id = %s",
219 pakfire_version, cpu_model, cpu_count, cpu_arch, cpu_bogomips,
220 host_key, os_name, self.id)
f6e6ff79
MT
221
222 def update_arches(self, arches):
223 # Get all arches this builder does currently support.
224 supported_arches = [a.name for a in self.get_arches()]
225
226 # Noarch is always supported.
227 if not "noarch" in arches:
228 arches.append("noarch")
229
230 arches_add = []
231 for arch in arches:
232 if arch in supported_arches:
233 supported_arches.remove(arch)
234 continue
235
236 arches_add.append(arch)
237 arches_rem = supported_arches
238
239 for arch_name in arches_add:
240 arch = self.pakfire.arches.get_by_name(arch_name)
241 if not arch:
242 logging.info("Client sent unknown architecture: %s" % arch_name)
243 continue
244
245 self.db.execute("INSERT INTO builders_arches(builder_id, arch_id) \
246 VALUES(%s, %s)", self.id, arch.id)
247
248 for arch_name in arches_rem:
249 arch = self.pakfire.arches.get_by_name(arch_name)
250 assert arch
251
252 self.db.execute("DELETE FROM builders_arches WHERE builder_id = %s \
253 AND arch_id = %s", self.id, arch.id)
9137135a 254
9137135a 255 def get_enabled(self):
f6e6ff79 256 return self.status == "enabled"
9137135a
MT
257
258 def set_enabled(self, value):
f6e6ff79
MT
259 # XXX deprecated
260
9137135a 261 if value:
f6e6ff79 262 value = "enabled"
9137135a 263 else:
f6e6ff79 264 value = "disabled"
9137135a 265
f6e6ff79 266 self.set_status(value)
9137135a
MT
267
268 enabled = property(get_enabled, set_enabled)
269
270 @property
271 def disabled(self):
f6e6ff79
MT
272 return not self.enabled
273
274 def set_status(self, status, user=None, log=True):
275 assert status in ("created", "enabled", "disabled", "deleted")
276
277 if self.status == status:
278 return
279
3e990438 280 self._set_attribute("status", status)
f6e6ff79
MT
281
282 if log:
283 self.log(status, user=user)
284
3e990438 285 @lazy_property
9137135a 286 def arches(self):
3e990438
MT
287 if self.cpu_arch:
288 res = self.db.query("SELECT build_arch FROM arches_compat \
289 WHERE host_arch = %s", self.cpu_arch)
9137135a 290
3e990438
MT
291 arches += [r.build_arch for r in res]
292 if not self.cpu_arch in arches:
293 arches.append(self.cpu_arch)
f6e6ff79 294
3e990438 295 return arches
f6e6ff79 296
3e990438 297 return []
9137135a 298
f6e6ff79
MT
299 def get_build_release(self):
300 return self.data.build_release == "Y"
301
302 def set_build_release(self, value):
9137135a
MT
303 if value:
304 value = "Y"
305 else:
306 value = "N"
307
f6e6ff79
MT
308 self.db.execute("UPDATE builders SET build_release = %s WHERE id = %s",
309 value, self.id)
310
311 # Update the cache.
312 if self._data:
313 self._data["build_release"] = value
9137135a 314
f6e6ff79 315 build_release = property(get_build_release, set_build_release)
9137135a 316
f6e6ff79
MT
317 def get_build_scratch(self):
318 return self.data.build_scratch == "Y"
9137135a 319
f6e6ff79 320 def set_build_scratch(self, value):
9137135a
MT
321 if value:
322 value = "Y"
323 else:
324 value = "N"
325
f6e6ff79
MT
326 self.db.execute("UPDATE builders SET build_scratch = %s WHERE id = %s",
327 value, self.id)
328
329 # Update the cache.
330 if self._data:
331 self._data["build_scratch"] = value
9137135a 332
f6e6ff79 333 build_scratch = property(get_build_scratch, set_build_scratch)
9137135a
MT
334
335 def get_build_test(self):
336 return self.data.build_test == "Y"
337
338 def set_build_test(self, value):
339 if value:
340 value = "Y"
341 else:
342 value = "N"
343
f6e6ff79
MT
344 self.db.execute("UPDATE builders SET build_test = %s WHERE id = %s",
345 value, self.id)
346
347 # Update the cache.
348 if self._data:
349 self._data["build_test"] = value
9137135a
MT
350
351 build_test = property(get_build_test, set_build_test)
352
f6e6ff79
MT
353 @property
354 def build_types(self):
355 ret = []
356
357 if self.build_release:
358 ret.append("release")
359
360 if self.build_scratch:
361 ret.append("scratch")
362
363 if self.build_test:
364 ret.append("test")
365
366 return ret
367
9137135a 368 def set_max_jobs(self, value):
3e990438 369 self._set_attribute("max_jobs", value)
9137135a 370
3e990438 371 max_jobs = property(lambda s: s.data.max_jobs, set_max_jobs)
9137135a
MT
372
373 @property
374 def name(self):
375 return self.data.name
376
377 @property
378 def hostname(self):
379 return self.name
380
381 @property
382 def passphrase(self):
383 return self.data.passphrase
384
c2902b29
MT
385 # Load average
386
9137135a
MT
387 @property
388 def loadavg(self):
c2902b29 389 return ", ".join(["%.2f" % l for l in (self.loadavg1, self.loadavg5, self.loadavg15)])
9137135a 390
f6e6ff79 391 @property
c2902b29
MT
392 def loadavg1(self):
393 return self.data.loadavg1 or 0.0
394
395 @property
396 def loadavg5(self):
397 return self.data.loadavg5 or 0.0
f6e6ff79 398
c2902b29
MT
399 @property
400 def loadavg15(self):
401 return self.data.loadavg15 or 0.0
f6e6ff79
MT
402
403 @property
404 def pakfire_version(self):
405 return self.data.pakfire_version or ""
9137135a 406
c2902b29
MT
407 @property
408 def os_name(self):
409 return self.data.os_name or ""
410
9137135a
MT
411 @property
412 def cpu_model(self):
f6e6ff79
MT
413 return self.data.cpu_model or ""
414
415 @property
416 def cpu_count(self):
417 return self.data.cpu_count
9137135a
MT
418
419 @property
c2902b29
MT
420 def cpu_arch(self):
421 return self.data.cpu_arch
422
423 @property
424 def cpu_bogomips(self):
425 return self.data.cpu_bogomips or 0.0
426
427 @property
428 def mem_percentage(self):
429 if not self.mem_total:
430 return None
431
432 return self.mem_used * 100 / self.mem_total
9137135a
MT
433
434 @property
c2902b29
MT
435 def mem_total(self):
436 return self.data.mem_total
437
438 @property
439 def mem_used(self):
440 if self.mem_total and self.mem_free:
441 return self.mem_total - self.mem_free
442
443 @property
444 def mem_free(self):
445 return self.data.mem_free
446
447 @property
448 def swap_percentage(self):
449 if not self.swap_total:
450 return None
451
452 return self.swap_used * 100 / self.swap_total
453
454 @property
455 def swap_total(self):
456 return self.data.swap_total
457
458 @property
459 def swap_used(self):
460 if self.swap_total and self.swap_free:
461 return self.swap_total - self.swap_free
462
463 @property
464 def swap_free(self):
465 return self.data.swap_free
466
467 @property
468 def space_free(self):
469 return self.data.space_free
f6e6ff79
MT
470
471 @property
472 def overload(self):
c2902b29
MT
473 if not self.cpu_count or not self.loadavg1:
474 return None
475
476 return self.loadavg1 >= self.cpu_count
f6e6ff79
MT
477
478 @property
479 def host_key_id(self):
480 return self.data.host_key_id
481
482 @property
483 def state(self):
9137135a 484 if self.disabled:
f6e6ff79 485 return "disabled"
9137135a 486
f6e6ff79
MT
487 if self.data.time_keepalive is None:
488 return "offline"
9137135a 489
f96eb5ed
MT
490 #if self.data.updated >= 5*60:
491 # return "offline"
9137135a 492
f6e6ff79 493 return "online"
9137135a 494
3e990438
MT
495 @lazy_property
496 def active_jobs(self, *args, **kwargs):
497 return self.pakfire.jobs.get_active(builder=self, *args, **kwargs)
163d9d8b
MT
498
499 @property
500 def too_many_jobs(self):
501 """
502 Tell if this host is already running enough or too many jobs.
503 """
3e990438 504 return len(self.active_jobs) >= self.max_jobs
9e8a20d7 505
c2902b29
MT
506 def get_next_jobs(self, limit=None):
507 """
508 Returns a list of jobs that can be built on this host.
509 """
510 return self.pakfire.jobs.get_next(arches=self.arches, limit=limit)
9e8a20d7 511
c2902b29
MT
512 def get_next_job(self):
513 """
514 Returns the next job in line for this builder.
515 """
516 # Get the first item of all jobs in the list.
517 jobs = self.pakfire.jobs.get_next(builder=self, state="pending", limit=1)
9137135a 518
163d9d8b
MT
519 if jobs:
520 return jobs[0]
f6e6ff79
MT
521
522 def get_history(self, *args, **kwargs):
523 kwargs["builder"] = self
524
525 return self.pakfire.builders.get_history(*args, **kwargs)