]> git.ipfire.org Git - people/jschlag/pbs.git/blame - src/buildservice/builders.py
Drop builders_arches table
[people/jschlag/pbs.git] / src / buildservice / builders.py
CommitLineData
9137135a
MT
1#!/usr/bin/python
2
2c909128 3from __future__ import absolute_import, division
f96eb5ed 4
9137135a
MT
5import datetime
6import hashlib
7import logging
8import random
9import string
10import time
11
2c909128
MT
12from . import base
13from . import logs
f6e6ff79 14
3e990438
MT
15from .decorators import *
16
2c909128 17from .users import generate_password_hash, check_password_hash, generate_random_string
9137135a
MT
18
19class Builders(base.Object):
f6e6ff79
MT
20 def auth(self, name, passphrase):
21 # If either name or passphrase is None, we don't check at all.
22 if None in (name, passphrase):
23 return
24
25 # Search for the hostname in the database.
26 # The builder must not be deleted.
27 builder = self.db.get("SELECT id FROM builders WHERE name = %s AND \
28 NOT status = 'deleted'", name)
29
30 if not builder:
31 return
32
33 # Get the whole Builder object from the database.
34 builder = self.get_by_id(builder.id)
35
36 # If the builder was not found or the passphrase does not match,
37 # you have bad luck.
38 if not builder or not builder.validate_passphrase(passphrase):
39 return
40
41 # Otherwise we return the Builder object.
42 return builder
43
9137135a 44 def get_all(self):
9fa1787c 45 builders = self.db.query("SELECT * FROM builders WHERE NOT status = 'deleted' ORDER BY name")
9137135a 46
9fa1787c 47 return [Builder(self.pakfire, b.id, b) for b in builders]
9137135a
MT
48
49 def get_by_id(self, id):
50 if not id:
51 return
52
53 return Builder(self.pakfire, id)
54
55 def get_by_name(self, name):
9fa1787c 56 builder = self.db.get("SELECT * FROM builders WHERE name = %s LIMIT 1", name)
9137135a
MT
57
58 if builder:
9fa1787c 59 return Builder(self.pakfire, builder.id, builder)
9137135a 60
f6e6ff79 61 def get_load(self):
f96eb5ed
MT
62 res1 = self.db.get("SELECT SUM(max_jobs) AS max_jobs FROM builders \
63 WHERE status = 'enabled'")
f6e6ff79 64
f96eb5ed
MT
65 res2 = self.db.get("SELECT COUNT(*) AS count FROM jobs \
66 WHERE state = 'dispatching' OR state = 'running' OR state = 'uploading'")
f6e6ff79 67
33857fed
MT
68 try:
69 return (res2.count * 100 / res1.max_jobs)
70 except:
71 return 0
f6e6ff79
MT
72
73 def get_history(self, limit=None, offset=None, builder=None, user=None):
74 query = "SELECT * FROM builders_history"
75 args = []
76
77 conditions = []
78
79 if builder:
80 conditions.append("builder_id = %s")
81 args.append(builder.id)
82
83 if user:
84 conditions.append("user_id = %s")
85 args.append(user.id)
86
87 if conditions:
88 query += " WHERE %s" % " AND ".join(conditions)
89
90 query += " ORDER BY time DESC"
91
92 if limit:
93 if offset:
94 query += " LIMIT %s,%s"
95 args += [offset, limit,]
96 else:
97 query += " LIMIT %s"
98 args += [limit,]
99
100 entries = []
101 for entry in self.db.query(query, *args):
102 entry = logs.BuilderLogEntry(self.pakfire, entry)
103 entries.append(entry)
104
105 return entries
106
9137135a 107
3e990438 108class Builder(base.DataObject):
92431da4
MT
109 table = "builders"
110
9137135a 111 def __cmp__(self, other):
318a6dfd
MT
112 if other is None:
113 return -1
114
9137135a
MT
115 return cmp(self.id, other.id)
116
117 @classmethod
f6e6ff79
MT
118 def create(cls, pakfire, name, user=None, log=True):
119 """
120 Creates a new builder.
121 """
122 builder_id = pakfire.db.execute("INSERT INTO builders(name, time_created) \
123 VALUES(%s, NOW())", name)
9137135a 124
f6e6ff79
MT
125 # Create Builder object.
126 builder = cls(pakfire, builder_id)
9137135a 127
f6e6ff79
MT
128 # Generate a new passphrase.
129 passphrase = builder.regenerate_passphrase()
130
131 # Log what we have done.
132 if log:
133 builder.log("created", user=user)
134
135 # The Builder object and the passphrase are returned.
136 return builder, passphrase
137
138 def log(self, action, user=None):
139 user_id = None
140 if user:
141 user_id = user.id
142
143 self.db.execute("INSERT INTO builders_history(builder_id, action, user_id, time) \
144 VALUES(%s, %s, %s, NOW())", self.id, action, user_id)
9137135a 145
9137135a 146 def regenerate_passphrase(self):
f6e6ff79
MT
147 """
148 Generates a new random passphrase and stores it as a salted hash
149 to the database.
150
151 The new passphrase is returned to be sent to the user (once).
152 """
153 # Generate a random string with 20 chars.
154 passphrase = generate_random_string(length=20)
155
156 # Create salted hash.
157 passphrase_hash = generate_password_hash(passphrase)
158
159 # Store the hash in the database.
3e990438 160 self._set_attribute("passphrase", passphrase_hash)
9137135a 161
f6e6ff79
MT
162 # Return the clear-text passphrase.
163 return passphrase
9137135a
MT
164
165 def validate_passphrase(self, passphrase):
f6e6ff79
MT
166 """
167 Compare the given passphrase with the one stored in the database.
168 """
169 return check_password_hash(passphrase, self.data.passphrase)
170
3e990438
MT
171 # Description
172
173 def set_description(self, description):
174 self._set_attribute("description", description)
175
176 description = property(lambda s: s.data.description or "", set_description)
f6e6ff79
MT
177
178 @property
179 def status(self):
180 return self.data.status
181
f6e6ff79
MT
182 @property
183 def keepalive(self):
184 """
185 Returns time of last keepalive message from this host.
186 """
187 return self.data.time_keepalive
188
c2902b29
MT
189 def update_keepalive(self, loadavg1=None, loadavg5=None, loadavg15=None,
190 mem_total=None, mem_free=None, swap_total=None, swap_free=None,
191 space_free=None):
f6e6ff79
MT
192 """
193 Update the keepalive timestamp of this machine.
194 """
c2902b29
MT
195 self.db.execute("UPDATE builders SET time_keepalive = NOW(), \
196 loadavg1 = %s, loadavg5 = %s, loadavg15 = %s, space_free = %s, \
197 mem_total = %s, mem_free = %s, swap_total = %s, swap_free = %s \
198 WHERE id = %s", loadavg1, loadavg5, loadavg15, space_free,
199 mem_total, mem_free, swap_total, swap_free, self.id)
200
201 def update_info(self, cpu_model=None, cpu_count=None, cpu_arch=None, cpu_bogomips=None,
202 pakfire_version=None, host_key=None, os_name=None):
f6e6ff79
MT
203 # Update all the rest.
204 self.db.execute("UPDATE builders SET time_updated = NOW(), \
c2902b29
MT
205 pakfire_version = %s, cpu_model = %s, cpu_count = %s, cpu_arch = %s, \
206 cpu_bogomips = %s, host_key_id = %s, os_name = %s WHERE id = %s",
207 pakfire_version, cpu_model, cpu_count, cpu_arch, cpu_bogomips,
208 host_key, os_name, self.id)
f6e6ff79 209
9137135a 210 def get_enabled(self):
f6e6ff79 211 return self.status == "enabled"
9137135a
MT
212
213 def set_enabled(self, value):
f6e6ff79
MT
214 # XXX deprecated
215
9137135a 216 if value:
f6e6ff79 217 value = "enabled"
9137135a 218 else:
f6e6ff79 219 value = "disabled"
9137135a 220
f6e6ff79 221 self.set_status(value)
9137135a
MT
222
223 enabled = property(get_enabled, set_enabled)
224
225 @property
226 def disabled(self):
f6e6ff79
MT
227 return not self.enabled
228
229 def set_status(self, status, user=None, log=True):
230 assert status in ("created", "enabled", "disabled", "deleted")
231
232 if self.status == status:
233 return
234
3e990438 235 self._set_attribute("status", status)
f6e6ff79
MT
236
237 if log:
238 self.log(status, user=user)
239
3e990438 240 @lazy_property
9137135a 241 def arches(self):
3e990438
MT
242 if self.cpu_arch:
243 res = self.db.query("SELECT build_arch FROM arches_compat \
244 WHERE host_arch = %s", self.cpu_arch)
9137135a 245
3e990438
MT
246 arches += [r.build_arch for r in res]
247 if not self.cpu_arch in arches:
248 arches.append(self.cpu_arch)
f6e6ff79 249
3e990438 250 return arches
f6e6ff79 251
3e990438 252 return []
9137135a 253
f6e6ff79
MT
254 def get_build_release(self):
255 return self.data.build_release == "Y"
256
257 def set_build_release(self, value):
9137135a
MT
258 if value:
259 value = "Y"
260 else:
261 value = "N"
262
f6e6ff79
MT
263 self.db.execute("UPDATE builders SET build_release = %s WHERE id = %s",
264 value, self.id)
265
266 # Update the cache.
267 if self._data:
268 self._data["build_release"] = value
9137135a 269
f6e6ff79 270 build_release = property(get_build_release, set_build_release)
9137135a 271
f6e6ff79
MT
272 def get_build_scratch(self):
273 return self.data.build_scratch == "Y"
9137135a 274
f6e6ff79 275 def set_build_scratch(self, value):
9137135a
MT
276 if value:
277 value = "Y"
278 else:
279 value = "N"
280
f6e6ff79
MT
281 self.db.execute("UPDATE builders SET build_scratch = %s WHERE id = %s",
282 value, self.id)
283
284 # Update the cache.
285 if self._data:
286 self._data["build_scratch"] = value
9137135a 287
f6e6ff79 288 build_scratch = property(get_build_scratch, set_build_scratch)
9137135a
MT
289
290 def get_build_test(self):
291 return self.data.build_test == "Y"
292
293 def set_build_test(self, value):
294 if value:
295 value = "Y"
296 else:
297 value = "N"
298
f6e6ff79
MT
299 self.db.execute("UPDATE builders SET build_test = %s WHERE id = %s",
300 value, self.id)
301
302 # Update the cache.
303 if self._data:
304 self._data["build_test"] = value
9137135a
MT
305
306 build_test = property(get_build_test, set_build_test)
307
f6e6ff79
MT
308 @property
309 def build_types(self):
310 ret = []
311
312 if self.build_release:
313 ret.append("release")
314
315 if self.build_scratch:
316 ret.append("scratch")
317
318 if self.build_test:
319 ret.append("test")
320
321 return ret
322
9137135a 323 def set_max_jobs(self, value):
3e990438 324 self._set_attribute("max_jobs", value)
9137135a 325
3e990438 326 max_jobs = property(lambda s: s.data.max_jobs, set_max_jobs)
9137135a
MT
327
328 @property
329 def name(self):
330 return self.data.name
331
332 @property
333 def hostname(self):
334 return self.name
335
336 @property
337 def passphrase(self):
338 return self.data.passphrase
339
c2902b29
MT
340 # Load average
341
9137135a
MT
342 @property
343 def loadavg(self):
c2902b29 344 return ", ".join(["%.2f" % l for l in (self.loadavg1, self.loadavg5, self.loadavg15)])
9137135a 345
f6e6ff79 346 @property
c2902b29
MT
347 def loadavg1(self):
348 return self.data.loadavg1 or 0.0
349
350 @property
351 def loadavg5(self):
352 return self.data.loadavg5 or 0.0
f6e6ff79 353
c2902b29
MT
354 @property
355 def loadavg15(self):
356 return self.data.loadavg15 or 0.0
f6e6ff79
MT
357
358 @property
359 def pakfire_version(self):
360 return self.data.pakfire_version or ""
9137135a 361
c2902b29
MT
362 @property
363 def os_name(self):
364 return self.data.os_name or ""
365
9137135a
MT
366 @property
367 def cpu_model(self):
f6e6ff79
MT
368 return self.data.cpu_model or ""
369
370 @property
371 def cpu_count(self):
372 return self.data.cpu_count
9137135a
MT
373
374 @property
c2902b29
MT
375 def cpu_arch(self):
376 return self.data.cpu_arch
377
378 @property
379 def cpu_bogomips(self):
380 return self.data.cpu_bogomips or 0.0
381
382 @property
383 def mem_percentage(self):
384 if not self.mem_total:
385 return None
386
387 return self.mem_used * 100 / self.mem_total
9137135a
MT
388
389 @property
c2902b29
MT
390 def mem_total(self):
391 return self.data.mem_total
392
393 @property
394 def mem_used(self):
395 if self.mem_total and self.mem_free:
396 return self.mem_total - self.mem_free
397
398 @property
399 def mem_free(self):
400 return self.data.mem_free
401
402 @property
403 def swap_percentage(self):
404 if not self.swap_total:
405 return None
406
407 return self.swap_used * 100 / self.swap_total
408
409 @property
410 def swap_total(self):
411 return self.data.swap_total
412
413 @property
414 def swap_used(self):
415 if self.swap_total and self.swap_free:
416 return self.swap_total - self.swap_free
417
418 @property
419 def swap_free(self):
420 return self.data.swap_free
421
422 @property
423 def space_free(self):
424 return self.data.space_free
f6e6ff79
MT
425
426 @property
427 def overload(self):
c2902b29
MT
428 if not self.cpu_count or not self.loadavg1:
429 return None
430
431 return self.loadavg1 >= self.cpu_count
f6e6ff79
MT
432
433 @property
434 def host_key_id(self):
435 return self.data.host_key_id
436
437 @property
438 def state(self):
9137135a 439 if self.disabled:
f6e6ff79 440 return "disabled"
9137135a 441
f6e6ff79
MT
442 if self.data.time_keepalive is None:
443 return "offline"
9137135a 444
f96eb5ed
MT
445 #if self.data.updated >= 5*60:
446 # return "offline"
9137135a 447
f6e6ff79 448 return "online"
9137135a 449
3e990438
MT
450 @lazy_property
451 def active_jobs(self, *args, **kwargs):
452 return self.pakfire.jobs.get_active(builder=self, *args, **kwargs)
163d9d8b
MT
453
454 @property
455 def too_many_jobs(self):
456 """
457 Tell if this host is already running enough or too many jobs.
458 """
3e990438 459 return len(self.active_jobs) >= self.max_jobs
9e8a20d7 460
c2902b29
MT
461 def get_next_jobs(self, limit=None):
462 """
463 Returns a list of jobs that can be built on this host.
464 """
465 return self.pakfire.jobs.get_next(arches=self.arches, limit=limit)
9e8a20d7 466
c2902b29
MT
467 def get_next_job(self):
468 """
469 Returns the next job in line for this builder.
470 """
471 # Get the first item of all jobs in the list.
472 jobs = self.pakfire.jobs.get_next(builder=self, state="pending", limit=1)
9137135a 473
163d9d8b
MT
474 if jobs:
475 return jobs[0]
f6e6ff79
MT
476
477 def get_history(self, *args, **kwargs):
478 kwargs["builder"] = self
479
480 return self.pakfire.builders.get_history(*args, **kwargs)