]> git.ipfire.org Git - people/jschlag/pbs.git/blame - backend/builders.py
Fix calculating the build times of a repository.
[people/jschlag/pbs.git] / backend / builders.py
CommitLineData
9137135a
MT
1#!/usr/bin/python
2
3import datetime
4import hashlib
5import logging
6import random
7import string
8import time
9
10import base
f6e6ff79
MT
11import logs
12
13from users import generate_password_hash, check_password_hash, generate_random_string
9137135a
MT
14
15class Builders(base.Object):
f6e6ff79
MT
16 def auth(self, name, passphrase):
17 # If either name or passphrase is None, we don't check at all.
18 if None in (name, passphrase):
19 return
20
21 # Search for the hostname in the database.
22 # The builder must not be deleted.
23 builder = self.db.get("SELECT id FROM builders WHERE name = %s AND \
24 NOT status = 'deleted'", name)
25
26 if not builder:
27 return
28
29 # Get the whole Builder object from the database.
30 builder = self.get_by_id(builder.id)
31
32 # If the builder was not found or the passphrase does not match,
33 # you have bad luck.
34 if not builder or not builder.validate_passphrase(passphrase):
35 return
36
37 # Otherwise we return the Builder object.
38 return builder
39
9137135a 40 def get_all(self):
9fa1787c 41 builders = self.db.query("SELECT * FROM builders WHERE NOT status = 'deleted' ORDER BY name")
9137135a 42
9fa1787c 43 return [Builder(self.pakfire, b.id, b) for b in builders]
9137135a
MT
44
45 def get_by_id(self, id):
46 if not id:
47 return
48
49 return Builder(self.pakfire, id)
50
51 def get_by_name(self, name):
9fa1787c 52 builder = self.db.get("SELECT * FROM builders WHERE name = %s LIMIT 1", name)
9137135a
MT
53
54 if builder:
9fa1787c 55 return Builder(self.pakfire, builder.id, builder)
9137135a
MT
56
57 def get_all_arches(self):
58 arches = set()
59
60 for result in self.db.query("SELECT DISTINCT arches FROM builders"):
61 if not result.arches:
62 continue
63
64 _arches = result.arches.split()
65
66 for arch in _arches:
67 arches.add(arch)
68
69 return sorted(arches)
70
f6e6ff79
MT
71 def get_load(self):
72 slots = 1
73 running_jobs = 0
74
75 for builder in self.get_all():
76 if not builder.state == "online":
77 continue
78
79 slots += builder.max_jobs
80 running_jobs += len(builder.get_active_jobs(uploads=False))
81
82 return int(running_jobs * 100 / slots)
83
84 def get_history(self, limit=None, offset=None, builder=None, user=None):
85 query = "SELECT * FROM builders_history"
86 args = []
87
88 conditions = []
89
90 if builder:
91 conditions.append("builder_id = %s")
92 args.append(builder.id)
93
94 if user:
95 conditions.append("user_id = %s")
96 args.append(user.id)
97
98 if conditions:
99 query += " WHERE %s" % " AND ".join(conditions)
100
101 query += " ORDER BY time DESC"
102
103 if limit:
104 if offset:
105 query += " LIMIT %s,%s"
106 args += [offset, limit,]
107 else:
108 query += " LIMIT %s"
109 args += [limit,]
110
111 entries = []
112 for entry in self.db.query(query, *args):
113 entry = logs.BuilderLogEntry(self.pakfire, entry)
114 entries.append(entry)
115
116 return entries
117
9137135a
MT
118
119class Builder(base.Object):
9fa1787c 120 def __init__(self, pakfire, id, data=None):
9137135a
MT
121 base.Object.__init__(self, pakfire)
122
123 self.id = id
124
f6e6ff79 125 # Cache.
9fa1787c 126 self._data = data
f6e6ff79
MT
127 self._active_jobs = None
128 self._arches = None
129 self._disabled_arches = None
9137135a
MT
130
131 def __cmp__(self, other):
132 return cmp(self.id, other.id)
133
f6e6ff79
MT
134 @property
135 def data(self):
136 if self._data is None:
137 self._data = \
138 self.db.get("SELECT *, NOW() - time_keepalive AS updated \
139 FROM builders WHERE id = %s", self.id)
140
141 return self._data
142
9137135a 143 @classmethod
f6e6ff79
MT
144 def create(cls, pakfire, name, user=None, log=True):
145 """
146 Creates a new builder.
147 """
148 builder_id = pakfire.db.execute("INSERT INTO builders(name, time_created) \
149 VALUES(%s, NOW())", name)
9137135a 150
f6e6ff79
MT
151 # Create Builder object.
152 builder = cls(pakfire, builder_id)
9137135a 153
f6e6ff79
MT
154 # Generate a new passphrase.
155 passphrase = builder.regenerate_passphrase()
156
157 # Log what we have done.
158 if log:
159 builder.log("created", user=user)
160
161 # The Builder object and the passphrase are returned.
162 return builder, passphrase
163
164 def log(self, action, user=None):
165 user_id = None
166 if user:
167 user_id = user.id
168
169 self.db.execute("INSERT INTO builders_history(builder_id, action, user_id, time) \
170 VALUES(%s, %s, %s, NOW())", self.id, action, user_id)
9137135a
MT
171
172 def set(self, key, value):
173 self.db.execute("UPDATE builders SET %s = %%s WHERE id = %%s LIMIT 1" % key,
174 value, self.id)
175 self.data[key] = value
176
9137135a 177 def regenerate_passphrase(self):
f6e6ff79
MT
178 """
179 Generates a new random passphrase and stores it as a salted hash
180 to the database.
181
182 The new passphrase is returned to be sent to the user (once).
183 """
184 # Generate a random string with 20 chars.
185 passphrase = generate_random_string(length=20)
186
187 # Create salted hash.
188 passphrase_hash = generate_password_hash(passphrase)
189
190 # Store the hash in the database.
191 self.db.execute("UPDATE builders SET passphrase = %s WHERE id = %s",
192 passphrase_hash, self.id)
9137135a 193
f6e6ff79
MT
194 # Return the clear-text passphrase.
195 return passphrase
9137135a
MT
196
197 def validate_passphrase(self, passphrase):
f6e6ff79
MT
198 """
199 Compare the given passphrase with the one stored in the database.
200 """
201 return check_password_hash(passphrase, self.data.passphrase)
202
203 @property
204 def description(self):
205 return self.data.description or ""
206
207 @property
208 def status(self):
209 return self.data.status
210
211 def update_description(self, description):
212 self.db.execute("UPDATE builders SET description = %s, time_updated = NOW() \
213 WHERE id = %s", description, self.id)
214
215 if self._data:
216 self._data["description"] = description
217
218 @property
219 def keepalive(self):
220 """
221 Returns time of last keepalive message from this host.
222 """
223 return self.data.time_keepalive
224
225 def update_keepalive(self, loadavg, free_space):
226 """
227 Update the keepalive timestamp of this machine.
228 """
229 if free_space is None:
230 free_space = 0
231
232 self.db.execute("UPDATE builders SET time_keepalive = NOW(), loadavg = %s, \
233 free_space = %s WHERE id = %s", loadavg, free_space, self.id)
234
235 logging.debug("Builder %s updated it keepalive status: %s" \
236 % (self.name, loadavg))
237
238 def needs_update(self):
239 query = self.db.get("SELECT time_updated, NOW() - time_updated \
240 AS seconds FROM builders WHERE id = %s", self.id)
241
242 # If there has been no update at all, we will need a new one.
243 if query.time_updated is None:
244 return True
245
246 # Require an update after the data is older than 24 hours.
247 return query.seconds >= 24*3600
248
249 def update_info(self, arches, cpu_model, cpu_count, memory, pakfire_version=None, host_key_id=None):
250 # Update architecture information.
251 self.update_arches(arches)
252
253 # Update all the rest.
254 self.db.execute("UPDATE builders SET time_updated = NOW(), \
255 pakfire_version = %s, cpu_model = %s, cpu_count = %s, memory = %s, \
256 host_key_id = %s \
257 WHERE id = %s", pakfire_version or "", cpu_model, cpu_count, memory,
258 host_key_id, self.id)
259
260 def update_arches(self, arches):
261 # Get all arches this builder does currently support.
262 supported_arches = [a.name for a in self.get_arches()]
263
264 # Noarch is always supported.
265 if not "noarch" in arches:
266 arches.append("noarch")
267
268 arches_add = []
269 for arch in arches:
270 if arch in supported_arches:
271 supported_arches.remove(arch)
272 continue
273
274 arches_add.append(arch)
275 arches_rem = supported_arches
276
277 for arch_name in arches_add:
278 arch = self.pakfire.arches.get_by_name(arch_name)
279 if not arch:
280 logging.info("Client sent unknown architecture: %s" % arch_name)
281 continue
282
283 self.db.execute("INSERT INTO builders_arches(builder_id, arch_id) \
284 VALUES(%s, %s)", self.id, arch.id)
285
286 for arch_name in arches_rem:
287 arch = self.pakfire.arches.get_by_name(arch_name)
288 assert arch
289
290 self.db.execute("DELETE FROM builders_arches WHERE builder_id = %s \
291 AND arch_id = %s", self.id, arch.id)
9137135a 292
f6e6ff79
MT
293 def update_overload(self, overload):
294 if overload:
295 overload = "Y"
296 else:
297 overload = "N"
298
299 self.db.execute("UPDATE builders SET overload = %s WHERE id = %s",
300 overload, self.id)
301 self._data["overload"] = overload
302
303 logging.debug("Builder %s updated it overload status to %s" % \
304 (self.name, self.overload))
9137135a
MT
305
306 def get_enabled(self):
f6e6ff79 307 return self.status == "enabled"
9137135a
MT
308
309 def set_enabled(self, value):
f6e6ff79
MT
310 # XXX deprecated
311
9137135a 312 if value:
f6e6ff79 313 value = "enabled"
9137135a 314 else:
f6e6ff79 315 value = "disabled"
9137135a 316
f6e6ff79 317 self.set_status(value)
9137135a
MT
318
319 enabled = property(get_enabled, set_enabled)
320
321 @property
322 def disabled(self):
f6e6ff79
MT
323 return not self.enabled
324
325 def set_status(self, status, user=None, log=True):
326 assert status in ("created", "enabled", "disabled", "deleted")
327
328 if self.status == status:
329 return
330
331 self.db.execute("UPDATE builders SET status = %s WHERE id = %s",
332 status, self.id)
333
334 if self._data:
335 self._data["status"] = status
336
337 if log:
338 self.log(status, user=user)
339
340 def get_arches(self, enabled=None):
341 """
342 A list of architectures that are supported by this builder.
343 """
344 if enabled is True:
345 enabled = "Y"
346 elif enabled is False:
347 enabled = "N"
348 else:
349 enabled = None
350
351 query = "SELECT arch_id AS id FROM builders_arches WHERE builder_id = %s"
352 args = [self.id,]
353
354 if enabled:
355 query += " AND enabled = %s"
356 args.append(enabled)
357
358 # Get all other arches from the database.
359 arches = []
360 for arch in self.db.query(query, *args):
361 arch = self.pakfire.arches.get_by_id(arch.id)
362 arches.append(arch)
363
364 # Save a sorted list of supported architectures.
365 arches.sort()
366
367 return arches
9137135a
MT
368
369 @property
370 def arches(self):
f6e6ff79
MT
371 if self._arches is None:
372 self._arches = self.get_arches(enabled=True)
9137135a 373
f6e6ff79 374 return self._arches
9137135a 375
f6e6ff79
MT
376 @property
377 def disabled_arches(self):
378 if self._disabled_arches is None:
379 self._disabled_arches = self.get_arches(enabled=False)
9137135a 380
f6e6ff79 381 return self._disabled_arches
9137135a 382
f6e6ff79
MT
383 def set_arch_status(self, arch, enabled):
384 if enabled:
385 enabled = "Y"
386 else:
387 enabled = "N"
388
389 self.db.execute("UPDATE builders_arches SET enabled = %s \
390 WHERE builder_id = %s AND arch_id = %s", enabled, self.id, arch.id)
391
392 # Reset the arch cache.
393 self._arches = None
9137135a 394
f6e6ff79
MT
395 def get_build_release(self):
396 return self.data.build_release == "Y"
397
398 def set_build_release(self, value):
9137135a
MT
399 if value:
400 value = "Y"
401 else:
402 value = "N"
403
f6e6ff79
MT
404 self.db.execute("UPDATE builders SET build_release = %s WHERE id = %s",
405 value, self.id)
406
407 # Update the cache.
408 if self._data:
409 self._data["build_release"] = value
9137135a 410
f6e6ff79 411 build_release = property(get_build_release, set_build_release)
9137135a 412
f6e6ff79
MT
413 def get_build_scratch(self):
414 return self.data.build_scratch == "Y"
9137135a 415
f6e6ff79 416 def set_build_scratch(self, value):
9137135a
MT
417 if value:
418 value = "Y"
419 else:
420 value = "N"
421
f6e6ff79
MT
422 self.db.execute("UPDATE builders SET build_scratch = %s WHERE id = %s",
423 value, self.id)
424
425 # Update the cache.
426 if self._data:
427 self._data["build_scratch"] = value
9137135a 428
f6e6ff79 429 build_scratch = property(get_build_scratch, set_build_scratch)
9137135a
MT
430
431 def get_build_test(self):
432 return self.data.build_test == "Y"
433
434 def set_build_test(self, value):
435 if value:
436 value = "Y"
437 else:
438 value = "N"
439
f6e6ff79
MT
440 self.db.execute("UPDATE builders SET build_test = %s WHERE id = %s",
441 value, self.id)
442
443 # Update the cache.
444 if self._data:
445 self._data["build_test"] = value
9137135a
MT
446
447 build_test = property(get_build_test, set_build_test)
448
f6e6ff79
MT
449 @property
450 def build_types(self):
451 ret = []
452
453 if self.build_release:
454 ret.append("release")
455
456 if self.build_scratch:
457 ret.append("scratch")
458
459 if self.build_test:
460 ret.append("test")
461
462 return ret
463
9137135a
MT
464 def get_max_jobs(self):
465 return self.data.max_jobs
466
467 def set_max_jobs(self, value):
468 self.set("max_jobs", value)
469
470 max_jobs = property(get_max_jobs, set_max_jobs)
471
472 @property
473 def name(self):
474 return self.data.name
475
476 @property
477 def hostname(self):
478 return self.name
479
480 @property
481 def passphrase(self):
482 return self.data.passphrase
483
484 @property
485 def loadavg(self):
f6e6ff79
MT
486 if self.state == "online":
487 return self.data.loadavg
9137135a 488
f6e6ff79
MT
489 @property
490 def load1(self):
491 try:
492 load1, load5, load15 = self.loadavg.split(", ")
493 except:
494 return None
495
496 return load1
497
498 @property
499 def pakfire_version(self):
500 return self.data.pakfire_version or ""
9137135a
MT
501
502 @property
503 def cpu_model(self):
f6e6ff79
MT
504 return self.data.cpu_model or ""
505
506 @property
507 def cpu_count(self):
508 return self.data.cpu_count
9137135a
MT
509
510 @property
511 def memory(self):
f6e6ff79 512 return self.data.memory
9137135a
MT
513
514 @property
f6e6ff79
MT
515 def free_space(self):
516 return self.data.free_space or 0
517
518 @property
519 def overload(self):
520 return self.data.overload == "Y"
521
522 @property
523 def host_key_id(self):
524 return self.data.host_key_id
525
526 @property
527 def state(self):
9137135a 528 if self.disabled:
f6e6ff79 529 return "disabled"
9137135a 530
f6e6ff79
MT
531 if self.data.time_keepalive is None:
532 return "offline"
9137135a 533
f6e6ff79
MT
534 if self.data.updated >= 5*60:
535 return "offline"
9137135a 536
f6e6ff79 537 return "online"
9137135a 538
f6e6ff79
MT
539 def get_active_jobs(self, uploads=True):
540 if self._active_jobs is None:
541 self._active_jobs = \
542 self.pakfire.jobs.get_active(host_id=self.id, uploads=uploads)
9137135a 543
f6e6ff79
MT
544 return self._active_jobs
545
546 def get_history(self, *args, **kwargs):
547 kwargs["builder"] = self
548
549 return self.pakfire.builders.get_history(*args, **kwargs)