]> git.ipfire.org Git - people/jschlag/pbs.git/blame - backend/builders.py
builders: Fix counting active jobs.
[people/jschlag/pbs.git] / backend / builders.py
CommitLineData
9137135a
MT
1#!/usr/bin/python
2
f96eb5ed
MT
3from __future__ import division
4
9137135a
MT
5import datetime
6import hashlib
7import logging
8import random
9import string
10import time
11
12import base
f6e6ff79
MT
13import logs
14
15from users import generate_password_hash, check_password_hash, generate_random_string
9137135a
MT
16
17class Builders(base.Object):
f6e6ff79
MT
18 def auth(self, name, passphrase):
19 # If either name or passphrase is None, we don't check at all.
20 if None in (name, passphrase):
21 return
22
23 # Search for the hostname in the database.
24 # The builder must not be deleted.
25 builder = self.db.get("SELECT id FROM builders WHERE name = %s AND \
26 NOT status = 'deleted'", name)
27
28 if not builder:
29 return
30
31 # Get the whole Builder object from the database.
32 builder = self.get_by_id(builder.id)
33
34 # If the builder was not found or the passphrase does not match,
35 # you have bad luck.
36 if not builder or not builder.validate_passphrase(passphrase):
37 return
38
39 # Otherwise we return the Builder object.
40 return builder
41
9137135a 42 def get_all(self):
9fa1787c 43 builders = self.db.query("SELECT * FROM builders WHERE NOT status = 'deleted' ORDER BY name")
9137135a 44
9fa1787c 45 return [Builder(self.pakfire, b.id, b) for b in builders]
9137135a
MT
46
47 def get_by_id(self, id):
48 if not id:
49 return
50
51 return Builder(self.pakfire, id)
52
53 def get_by_name(self, name):
9fa1787c 54 builder = self.db.get("SELECT * FROM builders WHERE name = %s LIMIT 1", name)
9137135a
MT
55
56 if builder:
9fa1787c 57 return Builder(self.pakfire, builder.id, builder)
9137135a
MT
58
59 def get_all_arches(self):
60 arches = set()
61
62 for result in self.db.query("SELECT DISTINCT arches FROM builders"):
63 if not result.arches:
64 continue
65
66 _arches = result.arches.split()
67
68 for arch in _arches:
69 arches.add(arch)
70
71 return sorted(arches)
72
f6e6ff79 73 def get_load(self):
f96eb5ed
MT
74 res1 = self.db.get("SELECT SUM(max_jobs) AS max_jobs FROM builders \
75 WHERE status = 'enabled'")
76 if not res1:
77 return 0
f6e6ff79 78
f96eb5ed
MT
79 res2 = self.db.get("SELECT COUNT(*) AS count FROM jobs \
80 WHERE state = 'dispatching' OR state = 'running' OR state = 'uploading'")
81 if not res2:
82 return 0
f6e6ff79 83
f96eb5ed 84 return (res2.count * 100 / res1.max_jobs)
f6e6ff79
MT
85
86 def get_history(self, limit=None, offset=None, builder=None, user=None):
87 query = "SELECT * FROM builders_history"
88 args = []
89
90 conditions = []
91
92 if builder:
93 conditions.append("builder_id = %s")
94 args.append(builder.id)
95
96 if user:
97 conditions.append("user_id = %s")
98 args.append(user.id)
99
100 if conditions:
101 query += " WHERE %s" % " AND ".join(conditions)
102
103 query += " ORDER BY time DESC"
104
105 if limit:
106 if offset:
107 query += " LIMIT %s,%s"
108 args += [offset, limit,]
109 else:
110 query += " LIMIT %s"
111 args += [limit,]
112
113 entries = []
114 for entry in self.db.query(query, *args):
115 entry = logs.BuilderLogEntry(self.pakfire, entry)
116 entries.append(entry)
117
118 return entries
119
9137135a
MT
120
121class Builder(base.Object):
9fa1787c 122 def __init__(self, pakfire, id, data=None):
9137135a
MT
123 base.Object.__init__(self, pakfire)
124
125 self.id = id
126
f6e6ff79 127 # Cache.
9fa1787c 128 self._data = data
f6e6ff79
MT
129 self._active_jobs = None
130 self._arches = None
131 self._disabled_arches = None
9137135a
MT
132
133 def __cmp__(self, other):
134 return cmp(self.id, other.id)
135
f6e6ff79
MT
136 @property
137 def data(self):
138 if self._data is None:
f96eb5ed
MT
139 self._data = self.db.get("SELECT * FROM builders WHERE id = %s", self.id)
140 assert self._data
f6e6ff79
MT
141
142 return self._data
143
9137135a 144 @classmethod
f6e6ff79
MT
145 def create(cls, pakfire, name, user=None, log=True):
146 """
147 Creates a new builder.
148 """
149 builder_id = pakfire.db.execute("INSERT INTO builders(name, time_created) \
150 VALUES(%s, NOW())", name)
9137135a 151
f6e6ff79
MT
152 # Create Builder object.
153 builder = cls(pakfire, builder_id)
9137135a 154
f6e6ff79
MT
155 # Generate a new passphrase.
156 passphrase = builder.regenerate_passphrase()
157
158 # Log what we have done.
159 if log:
160 builder.log("created", user=user)
161
162 # The Builder object and the passphrase are returned.
163 return builder, passphrase
164
165 def log(self, action, user=None):
166 user_id = None
167 if user:
168 user_id = user.id
169
170 self.db.execute("INSERT INTO builders_history(builder_id, action, user_id, time) \
171 VALUES(%s, %s, %s, NOW())", self.id, action, user_id)
9137135a
MT
172
173 def set(self, key, value):
174 self.db.execute("UPDATE builders SET %s = %%s WHERE id = %%s LIMIT 1" % key,
175 value, self.id)
176 self.data[key] = value
177
9137135a 178 def regenerate_passphrase(self):
f6e6ff79
MT
179 """
180 Generates a new random passphrase and stores it as a salted hash
181 to the database.
182
183 The new passphrase is returned to be sent to the user (once).
184 """
185 # Generate a random string with 20 chars.
186 passphrase = generate_random_string(length=20)
187
188 # Create salted hash.
189 passphrase_hash = generate_password_hash(passphrase)
190
191 # Store the hash in the database.
192 self.db.execute("UPDATE builders SET passphrase = %s WHERE id = %s",
193 passphrase_hash, self.id)
9137135a 194
f6e6ff79
MT
195 # Return the clear-text passphrase.
196 return passphrase
9137135a
MT
197
198 def validate_passphrase(self, passphrase):
f6e6ff79
MT
199 """
200 Compare the given passphrase with the one stored in the database.
201 """
202 return check_password_hash(passphrase, self.data.passphrase)
203
204 @property
205 def description(self):
206 return self.data.description or ""
207
208 @property
209 def status(self):
210 return self.data.status
211
212 def update_description(self, description):
213 self.db.execute("UPDATE builders SET description = %s, time_updated = NOW() \
214 WHERE id = %s", description, self.id)
215
216 if self._data:
217 self._data["description"] = description
218
219 @property
220 def keepalive(self):
221 """
222 Returns time of last keepalive message from this host.
223 """
224 return self.data.time_keepalive
225
226 def update_keepalive(self, loadavg, free_space):
227 """
228 Update the keepalive timestamp of this machine.
229 """
230 if free_space is None:
231 free_space = 0
232
233 self.db.execute("UPDATE builders SET time_keepalive = NOW(), loadavg = %s, \
234 free_space = %s WHERE id = %s", loadavg, free_space, self.id)
235
236 logging.debug("Builder %s updated it keepalive status: %s" \
237 % (self.name, loadavg))
238
239 def needs_update(self):
240 query = self.db.get("SELECT time_updated, NOW() - time_updated \
241 AS seconds FROM builders WHERE id = %s", self.id)
242
243 # If there has been no update at all, we will need a new one.
244 if query.time_updated is None:
245 return True
246
247 # Require an update after the data is older than 24 hours.
248 return query.seconds >= 24*3600
249
250 def update_info(self, arches, cpu_model, cpu_count, memory, pakfire_version=None, host_key_id=None):
251 # Update architecture information.
252 self.update_arches(arches)
253
254 # Update all the rest.
255 self.db.execute("UPDATE builders SET time_updated = NOW(), \
256 pakfire_version = %s, cpu_model = %s, cpu_count = %s, memory = %s, \
257 host_key_id = %s \
258 WHERE id = %s", pakfire_version or "", cpu_model, cpu_count, memory,
259 host_key_id, self.id)
260
261 def update_arches(self, arches):
262 # Get all arches this builder does currently support.
263 supported_arches = [a.name for a in self.get_arches()]
264
265 # Noarch is always supported.
266 if not "noarch" in arches:
267 arches.append("noarch")
268
269 arches_add = []
270 for arch in arches:
271 if arch in supported_arches:
272 supported_arches.remove(arch)
273 continue
274
275 arches_add.append(arch)
276 arches_rem = supported_arches
277
278 for arch_name in arches_add:
279 arch = self.pakfire.arches.get_by_name(arch_name)
280 if not arch:
281 logging.info("Client sent unknown architecture: %s" % arch_name)
282 continue
283
284 self.db.execute("INSERT INTO builders_arches(builder_id, arch_id) \
285 VALUES(%s, %s)", self.id, arch.id)
286
287 for arch_name in arches_rem:
288 arch = self.pakfire.arches.get_by_name(arch_name)
289 assert arch
290
291 self.db.execute("DELETE FROM builders_arches WHERE builder_id = %s \
292 AND arch_id = %s", self.id, arch.id)
9137135a 293
f6e6ff79
MT
294 def update_overload(self, overload):
295 if overload:
296 overload = "Y"
297 else:
298 overload = "N"
299
300 self.db.execute("UPDATE builders SET overload = %s WHERE id = %s",
301 overload, self.id)
302 self._data["overload"] = overload
303
304 logging.debug("Builder %s updated it overload status to %s" % \
305 (self.name, self.overload))
9137135a
MT
306
307 def get_enabled(self):
f6e6ff79 308 return self.status == "enabled"
9137135a
MT
309
310 def set_enabled(self, value):
f6e6ff79
MT
311 # XXX deprecated
312
9137135a 313 if value:
f6e6ff79 314 value = "enabled"
9137135a 315 else:
f6e6ff79 316 value = "disabled"
9137135a 317
f6e6ff79 318 self.set_status(value)
9137135a
MT
319
320 enabled = property(get_enabled, set_enabled)
321
322 @property
323 def disabled(self):
f6e6ff79
MT
324 return not self.enabled
325
326 def set_status(self, status, user=None, log=True):
327 assert status in ("created", "enabled", "disabled", "deleted")
328
329 if self.status == status:
330 return
331
332 self.db.execute("UPDATE builders SET status = %s WHERE id = %s",
333 status, self.id)
334
335 if self._data:
336 self._data["status"] = status
337
338 if log:
339 self.log(status, user=user)
340
341 def get_arches(self, enabled=None):
342 """
343 A list of architectures that are supported by this builder.
344 """
345 if enabled is True:
346 enabled = "Y"
347 elif enabled is False:
348 enabled = "N"
349 else:
350 enabled = None
351
352 query = "SELECT arch_id AS id FROM builders_arches WHERE builder_id = %s"
353 args = [self.id,]
354
355 if enabled:
356 query += " AND enabled = %s"
357 args.append(enabled)
358
359 # Get all other arches from the database.
360 arches = []
361 for arch in self.db.query(query, *args):
362 arch = self.pakfire.arches.get_by_id(arch.id)
363 arches.append(arch)
364
365 # Save a sorted list of supported architectures.
366 arches.sort()
367
368 return arches
9137135a
MT
369
370 @property
371 def arches(self):
f6e6ff79
MT
372 if self._arches is None:
373 self._arches = self.get_arches(enabled=True)
9137135a 374
f6e6ff79 375 return self._arches
9137135a 376
f6e6ff79
MT
377 @property
378 def disabled_arches(self):
379 if self._disabled_arches is None:
380 self._disabled_arches = self.get_arches(enabled=False)
9137135a 381
f6e6ff79 382 return self._disabled_arches
9137135a 383
f6e6ff79
MT
384 def set_arch_status(self, arch, enabled):
385 if enabled:
386 enabled = "Y"
387 else:
388 enabled = "N"
389
390 self.db.execute("UPDATE builders_arches SET enabled = %s \
391 WHERE builder_id = %s AND arch_id = %s", enabled, self.id, arch.id)
392
393 # Reset the arch cache.
394 self._arches = None
9137135a 395
f6e6ff79
MT
396 def get_build_release(self):
397 return self.data.build_release == "Y"
398
399 def set_build_release(self, value):
9137135a
MT
400 if value:
401 value = "Y"
402 else:
403 value = "N"
404
f6e6ff79
MT
405 self.db.execute("UPDATE builders SET build_release = %s WHERE id = %s",
406 value, self.id)
407
408 # Update the cache.
409 if self._data:
410 self._data["build_release"] = value
9137135a 411
f6e6ff79 412 build_release = property(get_build_release, set_build_release)
9137135a 413
f6e6ff79
MT
414 def get_build_scratch(self):
415 return self.data.build_scratch == "Y"
9137135a 416
f6e6ff79 417 def set_build_scratch(self, value):
9137135a
MT
418 if value:
419 value = "Y"
420 else:
421 value = "N"
422
f6e6ff79
MT
423 self.db.execute("UPDATE builders SET build_scratch = %s WHERE id = %s",
424 value, self.id)
425
426 # Update the cache.
427 if self._data:
428 self._data["build_scratch"] = value
9137135a 429
f6e6ff79 430 build_scratch = property(get_build_scratch, set_build_scratch)
9137135a
MT
431
432 def get_build_test(self):
433 return self.data.build_test == "Y"
434
435 def set_build_test(self, value):
436 if value:
437 value = "Y"
438 else:
439 value = "N"
440
f6e6ff79
MT
441 self.db.execute("UPDATE builders SET build_test = %s WHERE id = %s",
442 value, self.id)
443
444 # Update the cache.
445 if self._data:
446 self._data["build_test"] = value
9137135a
MT
447
448 build_test = property(get_build_test, set_build_test)
449
f6e6ff79
MT
450 @property
451 def build_types(self):
452 ret = []
453
454 if self.build_release:
455 ret.append("release")
456
457 if self.build_scratch:
458 ret.append("scratch")
459
460 if self.build_test:
461 ret.append("test")
462
463 return ret
464
9137135a
MT
465 def get_max_jobs(self):
466 return self.data.max_jobs
467
468 def set_max_jobs(self, value):
469 self.set("max_jobs", value)
470
471 max_jobs = property(get_max_jobs, set_max_jobs)
472
473 @property
474 def name(self):
475 return self.data.name
476
477 @property
478 def hostname(self):
479 return self.name
480
481 @property
482 def passphrase(self):
483 return self.data.passphrase
484
485 @property
486 def loadavg(self):
f6e6ff79
MT
487 if self.state == "online":
488 return self.data.loadavg
9137135a 489
f6e6ff79
MT
490 @property
491 def load1(self):
492 try:
493 load1, load5, load15 = self.loadavg.split(", ")
494 except:
495 return None
496
497 return load1
498
499 @property
500 def pakfire_version(self):
501 return self.data.pakfire_version or ""
9137135a
MT
502
503 @property
504 def cpu_model(self):
f6e6ff79
MT
505 return self.data.cpu_model or ""
506
507 @property
508 def cpu_count(self):
509 return self.data.cpu_count
9137135a
MT
510
511 @property
512 def memory(self):
f6e6ff79 513 return self.data.memory
9137135a
MT
514
515 @property
f6e6ff79
MT
516 def free_space(self):
517 return self.data.free_space or 0
518
519 @property
520 def overload(self):
521 return self.data.overload == "Y"
522
523 @property
524 def host_key_id(self):
525 return self.data.host_key_id
526
527 @property
528 def state(self):
9137135a 529 if self.disabled:
f6e6ff79 530 return "disabled"
9137135a 531
f6e6ff79
MT
532 if self.data.time_keepalive is None:
533 return "offline"
9137135a 534
f96eb5ed
MT
535 #if self.data.updated >= 5*60:
536 # return "offline"
9137135a 537
f6e6ff79 538 return "online"
9137135a 539
9e8a20d7
MT
540 def get_active_jobs(self, count=False):
541 query = self.db.query("\
542 SELECT * FROM jobs \
543 WHERE \
544 jobs.builder_id = %s AND \
545 (jobs.state = 'dispatching' OR jobs.state = 'running' OR jobs.state = 'uploading') \
546 ORDER BY time_started ASC",
547 self.id)
548
549 if count:
550 return len(query)
551
552 jobs = []
553 for job in query:
554 job = self.pakfire.jobs.get_by_id(job.id, job)
555 jobs.append(job)
556
557 return jobs
558
559 def count_active_jobs(self):
f6e6ff79 560 if self._active_jobs is None:
9e8a20d7 561 self._active_jobs = self.get_active_jobs(count=True)
9137135a 562
f6e6ff79
MT
563 return self._active_jobs
564
565 def get_history(self, *args, **kwargs):
566 kwargs["builder"] = self
567
568 return self.pakfire.builders.get_history(*args, **kwargs)