]>
Commit | Line | Data |
---|---|---|
9137135a MT |
1 | #!/usr/bin/python |
2 | ||
2c909128 | 3 | from __future__ import absolute_import, division |
f96eb5ed | 4 | |
9137135a MT |
5 | import datetime |
6 | import hashlib | |
7 | import logging | |
8 | import random | |
9 | import string | |
10 | import time | |
11 | ||
2c909128 MT |
12 | from . import base |
13 | from . import logs | |
f6e6ff79 | 14 | |
3e990438 MT |
15 | from .decorators import * |
16 | ||
2c909128 | 17 | from .users import generate_password_hash, check_password_hash, generate_random_string |
9137135a MT |
18 | |
19 | class Builders(base.Object): | |
e704b8e2 MT |
20 | def _get_builder(self, query, *args): |
21 | res = self.db.get(query, *args) | |
22 | ||
23 | if res: | |
24 | return Builder(self.backend, res.id, data=res) | |
25 | ||
26 | def _get_builders(self, query, *args): | |
27 | res = self.db.query(query, *args) | |
28 | ||
29 | for row in res: | |
30 | yield Builder(self.backend, row.id, data=row) | |
31 | ||
32 | def __iter__(self): | |
33 | builders = self._get_builders("SELECT * FROM builders \ | |
34 | WHERE deleted IS FALSE ORDER BY name") | |
35 | ||
36 | return iter(builders) | |
37 | ||
38 | def create(self, name, user=None, log=True): | |
39 | """ | |
40 | Creates a new builder. | |
41 | """ | |
42 | builder = self._get_builder("INSERT INTO builders(name) \ | |
43 | VALUES(%s) RETURNING *", name) | |
44 | ||
45 | # Generate a new passphrase. | |
46 | passphrase = builder.regenerate_passphrase() | |
47 | ||
48 | # Log what we have done. | |
49 | if log: | |
50 | builder.log("created", user=user) | |
51 | ||
52 | # The Builder object and the passphrase are returned. | |
53 | return builder, passphrase | |
54 | ||
f6e6ff79 MT |
55 | def auth(self, name, passphrase): |
56 | # If either name or passphrase is None, we don't check at all. | |
57 | if None in (name, passphrase): | |
58 | return | |
59 | ||
60 | # Search for the hostname in the database. | |
e704b8e2 MT |
61 | builder = self._get_builder("SELECT * FROM builders \ |
62 | WHERE name = %s AND deleted IS FALSE", name) | |
f6e6ff79 MT |
63 | |
64 | # If the builder was not found or the passphrase does not match, | |
65 | # you have bad luck. | |
66 | if not builder or not builder.validate_passphrase(passphrase): | |
67 | return | |
68 | ||
69 | # Otherwise we return the Builder object. | |
70 | return builder | |
71 | ||
e704b8e2 MT |
72 | def get_by_id(self, builder_id): |
73 | return self._get_builder("SELECT * FROM builders WHERE id = %s", builder_id) | |
9137135a MT |
74 | |
75 | def get_by_name(self, name): | |
e704b8e2 MT |
76 | return self._get_builder("SELECT * FROM builders \ |
77 | WHERE name = %s AND deleted IS FALSE", name) | |
9137135a | 78 | |
f6e6ff79 | 79 | def get_load(self): |
f96eb5ed | 80 | res1 = self.db.get("SELECT SUM(max_jobs) AS max_jobs FROM builders \ |
e704b8e2 | 81 | WHERE enabled IS TRUE and deleted IS FALSE") |
f6e6ff79 | 82 | |
f96eb5ed MT |
83 | res2 = self.db.get("SELECT COUNT(*) AS count FROM jobs \ |
84 | WHERE state = 'dispatching' OR state = 'running' OR state = 'uploading'") | |
f6e6ff79 | 85 | |
33857fed MT |
86 | try: |
87 | return (res2.count * 100 / res1.max_jobs) | |
88 | except: | |
89 | return 0 | |
f6e6ff79 MT |
90 | |
91 | def get_history(self, limit=None, offset=None, builder=None, user=None): | |
92 | query = "SELECT * FROM builders_history" | |
93 | args = [] | |
94 | ||
95 | conditions = [] | |
96 | ||
97 | if builder: | |
98 | conditions.append("builder_id = %s") | |
99 | args.append(builder.id) | |
100 | ||
101 | if user: | |
102 | conditions.append("user_id = %s") | |
103 | args.append(user.id) | |
104 | ||
105 | if conditions: | |
106 | query += " WHERE %s" % " AND ".join(conditions) | |
107 | ||
108 | query += " ORDER BY time DESC" | |
109 | ||
110 | if limit: | |
111 | if offset: | |
112 | query += " LIMIT %s,%s" | |
113 | args += [offset, limit,] | |
114 | else: | |
115 | query += " LIMIT %s" | |
116 | args += [limit,] | |
117 | ||
118 | entries = [] | |
119 | for entry in self.db.query(query, *args): | |
120 | entry = logs.BuilderLogEntry(self.pakfire, entry) | |
121 | entries.append(entry) | |
122 | ||
123 | return entries | |
124 | ||
9137135a | 125 | |
3e990438 | 126 | class Builder(base.DataObject): |
92431da4 MT |
127 | table = "builders" |
128 | ||
e704b8e2 MT |
129 | def __eq__(self, other): |
130 | if isinstance(other, self.__class__): | |
131 | return self.id == other.id | |
9137135a | 132 | |
e704b8e2 MT |
133 | def __lt__(self, other): |
134 | if isinstance(other, self.__class__): | |
135 | return self.name < other.name | |
f6e6ff79 MT |
136 | |
137 | def log(self, action, user=None): | |
138 | user_id = None | |
139 | if user: | |
140 | user_id = user.id | |
141 | ||
142 | self.db.execute("INSERT INTO builders_history(builder_id, action, user_id, time) \ | |
143 | VALUES(%s, %s, %s, NOW())", self.id, action, user_id) | |
9137135a | 144 | |
9137135a | 145 | def regenerate_passphrase(self): |
f6e6ff79 MT |
146 | """ |
147 | Generates a new random passphrase and stores it as a salted hash | |
148 | to the database. | |
149 | ||
150 | The new passphrase is returned to be sent to the user (once). | |
151 | """ | |
e704b8e2 MT |
152 | # Generate a random string with 40 chars. |
153 | passphrase = generate_random_string(length=40) | |
f6e6ff79 MT |
154 | |
155 | # Create salted hash. | |
156 | passphrase_hash = generate_password_hash(passphrase) | |
157 | ||
158 | # Store the hash in the database. | |
3e990438 | 159 | self._set_attribute("passphrase", passphrase_hash) |
9137135a | 160 | |
f6e6ff79 MT |
161 | # Return the clear-text passphrase. |
162 | return passphrase | |
9137135a MT |
163 | |
164 | def validate_passphrase(self, passphrase): | |
f6e6ff79 MT |
165 | """ |
166 | Compare the given passphrase with the one stored in the database. | |
167 | """ | |
168 | return check_password_hash(passphrase, self.data.passphrase) | |
169 | ||
3e990438 MT |
170 | # Description |
171 | ||
172 | def set_description(self, description): | |
173 | self._set_attribute("description", description) | |
174 | ||
175 | description = property(lambda s: s.data.description or "", set_description) | |
f6e6ff79 | 176 | |
f6e6ff79 MT |
177 | @property |
178 | def keepalive(self): | |
179 | """ | |
180 | Returns time of last keepalive message from this host. | |
181 | """ | |
182 | return self.data.time_keepalive | |
183 | ||
c2902b29 MT |
184 | def update_keepalive(self, loadavg1=None, loadavg5=None, loadavg15=None, |
185 | mem_total=None, mem_free=None, swap_total=None, swap_free=None, | |
186 | space_free=None): | |
f6e6ff79 MT |
187 | """ |
188 | Update the keepalive timestamp of this machine. | |
189 | """ | |
c2902b29 MT |
190 | self.db.execute("UPDATE builders SET time_keepalive = NOW(), \ |
191 | loadavg1 = %s, loadavg5 = %s, loadavg15 = %s, space_free = %s, \ | |
192 | mem_total = %s, mem_free = %s, swap_total = %s, swap_free = %s \ | |
193 | WHERE id = %s", loadavg1, loadavg5, loadavg15, space_free, | |
194 | mem_total, mem_free, swap_total, swap_free, self.id) | |
195 | ||
196 | def update_info(self, cpu_model=None, cpu_count=None, cpu_arch=None, cpu_bogomips=None, | |
197 | pakfire_version=None, host_key=None, os_name=None): | |
f6e6ff79 MT |
198 | # Update all the rest. |
199 | self.db.execute("UPDATE builders SET time_updated = NOW(), \ | |
c2902b29 MT |
200 | pakfire_version = %s, cpu_model = %s, cpu_count = %s, cpu_arch = %s, \ |
201 | cpu_bogomips = %s, host_key_id = %s, os_name = %s WHERE id = %s", | |
202 | pakfire_version, cpu_model, cpu_count, cpu_arch, cpu_bogomips, | |
203 | host_key, os_name, self.id) | |
f6e6ff79 | 204 | |
e704b8e2 MT |
205 | def set_enabled(self, enabled): |
206 | self._set_attribute("enabled", enabled) | |
9137135a | 207 | |
e704b8e2 | 208 | enabled = property(lambda s: s.data.enabled, set_enabled) |
9137135a MT |
209 | |
210 | @property | |
211 | def disabled(self): | |
f6e6ff79 MT |
212 | return not self.enabled |
213 | ||
e704b8e2 MT |
214 | @property |
215 | def native_arch(self): | |
216 | """ | |
217 | The native architecture of this builder | |
218 | """ | |
219 | return self.cpu_arch | |
f6e6ff79 | 220 | |
e704b8e2 MT |
221 | @lazy_property |
222 | def supported_arches(self): | |
223 | # Every builder supports noarch | |
224 | arches = ["noarch"] | |
f6e6ff79 | 225 | |
e704b8e2 MT |
226 | # We can always build our native architeture |
227 | if self.native_arch: | |
228 | arches.append(self.native_arch) | |
f6e6ff79 | 229 | |
e704b8e2 | 230 | # Get all compatible architectures |
3e990438 | 231 | res = self.db.query("SELECT build_arch FROM arches_compat \ |
e704b8e2 | 232 | WHERE native_arch = %s", self.native_arch) |
f6e6ff79 | 233 | |
e704b8e2 MT |
234 | for row in res: |
235 | if not row.build_arch in arches: | |
236 | arches.append(row.build_arch) | |
f6e6ff79 | 237 | |
e704b8e2 | 238 | return sorted(arches) |
9137135a | 239 | |
f6e6ff79 MT |
240 | def get_build_release(self): |
241 | return self.data.build_release == "Y" | |
242 | ||
243 | def set_build_release(self, value): | |
9137135a MT |
244 | if value: |
245 | value = "Y" | |
246 | else: | |
247 | value = "N" | |
248 | ||
f6e6ff79 MT |
249 | self.db.execute("UPDATE builders SET build_release = %s WHERE id = %s", |
250 | value, self.id) | |
251 | ||
252 | # Update the cache. | |
253 | if self._data: | |
254 | self._data["build_release"] = value | |
9137135a | 255 | |
f6e6ff79 | 256 | build_release = property(get_build_release, set_build_release) |
9137135a | 257 | |
f6e6ff79 MT |
258 | def get_build_scratch(self): |
259 | return self.data.build_scratch == "Y" | |
9137135a | 260 | |
f6e6ff79 | 261 | def set_build_scratch(self, value): |
9137135a MT |
262 | if value: |
263 | value = "Y" | |
264 | else: | |
265 | value = "N" | |
266 | ||
f6e6ff79 MT |
267 | self.db.execute("UPDATE builders SET build_scratch = %s WHERE id = %s", |
268 | value, self.id) | |
269 | ||
270 | # Update the cache. | |
271 | if self._data: | |
272 | self._data["build_scratch"] = value | |
9137135a | 273 | |
f6e6ff79 | 274 | build_scratch = property(get_build_scratch, set_build_scratch) |
9137135a MT |
275 | |
276 | def get_build_test(self): | |
277 | return self.data.build_test == "Y" | |
278 | ||
279 | def set_build_test(self, value): | |
280 | if value: | |
281 | value = "Y" | |
282 | else: | |
283 | value = "N" | |
284 | ||
f6e6ff79 MT |
285 | self.db.execute("UPDATE builders SET build_test = %s WHERE id = %s", |
286 | value, self.id) | |
287 | ||
288 | # Update the cache. | |
289 | if self._data: | |
290 | self._data["build_test"] = value | |
9137135a MT |
291 | |
292 | build_test = property(get_build_test, set_build_test) | |
293 | ||
f6e6ff79 MT |
294 | @property |
295 | def build_types(self): | |
296 | ret = [] | |
297 | ||
298 | if self.build_release: | |
299 | ret.append("release") | |
300 | ||
301 | if self.build_scratch: | |
302 | ret.append("scratch") | |
303 | ||
304 | if self.build_test: | |
305 | ret.append("test") | |
306 | ||
307 | return ret | |
308 | ||
9137135a | 309 | def set_max_jobs(self, value): |
3e990438 | 310 | self._set_attribute("max_jobs", value) |
9137135a | 311 | |
3e990438 | 312 | max_jobs = property(lambda s: s.data.max_jobs, set_max_jobs) |
9137135a MT |
313 | |
314 | @property | |
315 | def name(self): | |
316 | return self.data.name | |
317 | ||
318 | @property | |
319 | def hostname(self): | |
320 | return self.name | |
321 | ||
322 | @property | |
323 | def passphrase(self): | |
324 | return self.data.passphrase | |
325 | ||
c2902b29 MT |
326 | # Load average |
327 | ||
9137135a MT |
328 | @property |
329 | def loadavg(self): | |
c2902b29 | 330 | return ", ".join(["%.2f" % l for l in (self.loadavg1, self.loadavg5, self.loadavg15)]) |
9137135a | 331 | |
f6e6ff79 | 332 | @property |
c2902b29 MT |
333 | def loadavg1(self): |
334 | return self.data.loadavg1 or 0.0 | |
335 | ||
336 | @property | |
337 | def loadavg5(self): | |
338 | return self.data.loadavg5 or 0.0 | |
f6e6ff79 | 339 | |
c2902b29 MT |
340 | @property |
341 | def loadavg15(self): | |
342 | return self.data.loadavg15 or 0.0 | |
f6e6ff79 MT |
343 | |
344 | @property | |
345 | def pakfire_version(self): | |
346 | return self.data.pakfire_version or "" | |
9137135a | 347 | |
c2902b29 MT |
348 | @property |
349 | def os_name(self): | |
350 | return self.data.os_name or "" | |
351 | ||
9137135a MT |
352 | @property |
353 | def cpu_model(self): | |
f6e6ff79 MT |
354 | return self.data.cpu_model or "" |
355 | ||
356 | @property | |
357 | def cpu_count(self): | |
358 | return self.data.cpu_count | |
9137135a MT |
359 | |
360 | @property | |
c2902b29 MT |
361 | def cpu_arch(self): |
362 | return self.data.cpu_arch | |
363 | ||
364 | @property | |
365 | def cpu_bogomips(self): | |
366 | return self.data.cpu_bogomips or 0.0 | |
367 | ||
368 | @property | |
369 | def mem_percentage(self): | |
370 | if not self.mem_total: | |
371 | return None | |
372 | ||
373 | return self.mem_used * 100 / self.mem_total | |
9137135a MT |
374 | |
375 | @property | |
c2902b29 MT |
376 | def mem_total(self): |
377 | return self.data.mem_total | |
378 | ||
379 | @property | |
380 | def mem_used(self): | |
381 | if self.mem_total and self.mem_free: | |
382 | return self.mem_total - self.mem_free | |
383 | ||
384 | @property | |
385 | def mem_free(self): | |
386 | return self.data.mem_free | |
387 | ||
388 | @property | |
389 | def swap_percentage(self): | |
390 | if not self.swap_total: | |
391 | return None | |
392 | ||
393 | return self.swap_used * 100 / self.swap_total | |
394 | ||
395 | @property | |
396 | def swap_total(self): | |
397 | return self.data.swap_total | |
398 | ||
399 | @property | |
400 | def swap_used(self): | |
401 | if self.swap_total and self.swap_free: | |
402 | return self.swap_total - self.swap_free | |
403 | ||
404 | @property | |
405 | def swap_free(self): | |
406 | return self.data.swap_free | |
407 | ||
408 | @property | |
409 | def space_free(self): | |
410 | return self.data.space_free | |
f6e6ff79 MT |
411 | |
412 | @property | |
413 | def overload(self): | |
c2902b29 MT |
414 | if not self.cpu_count or not self.loadavg1: |
415 | return None | |
416 | ||
417 | return self.loadavg1 >= self.cpu_count | |
f6e6ff79 MT |
418 | |
419 | @property | |
420 | def host_key_id(self): | |
421 | return self.data.host_key_id | |
422 | ||
423 | @property | |
424 | def state(self): | |
9137135a | 425 | if self.disabled: |
f6e6ff79 | 426 | return "disabled" |
9137135a | 427 | |
f6e6ff79 MT |
428 | if self.data.time_keepalive is None: |
429 | return "offline" | |
9137135a | 430 | |
f96eb5ed MT |
431 | #if self.data.updated >= 5*60: |
432 | # return "offline" | |
9137135a | 433 | |
f6e6ff79 | 434 | return "online" |
9137135a | 435 | |
3e990438 MT |
436 | @lazy_property |
437 | def active_jobs(self, *args, **kwargs): | |
438 | return self.pakfire.jobs.get_active(builder=self, *args, **kwargs) | |
163d9d8b MT |
439 | |
440 | @property | |
441 | def too_many_jobs(self): | |
442 | """ | |
443 | Tell if this host is already running enough or too many jobs. | |
444 | """ | |
3e990438 | 445 | return len(self.active_jobs) >= self.max_jobs |
9e8a20d7 | 446 | |
c2902b29 MT |
447 | def get_next_jobs(self, limit=None): |
448 | """ | |
449 | Returns a list of jobs that can be built on this host. | |
450 | """ | |
e704b8e2 | 451 | return self.pakfire.jobs.get_next(arches=self.buildable_arches, limit=limit) |
9e8a20d7 | 452 | |
c2902b29 MT |
453 | def get_next_job(self): |
454 | """ | |
455 | Returns the next job in line for this builder. | |
456 | """ | |
457 | # Get the first item of all jobs in the list. | |
458 | jobs = self.pakfire.jobs.get_next(builder=self, state="pending", limit=1) | |
9137135a | 459 | |
163d9d8b MT |
460 | if jobs: |
461 | return jobs[0] | |
f6e6ff79 MT |
462 | |
463 | def get_history(self, *args, **kwargs): | |
464 | kwargs["builder"] = self | |
465 | ||
466 | return self.pakfire.builders.get_history(*args, **kwargs) |