]> git.ipfire.org Git - people/jschlag/pbs.git/blame - src/buildservice/builders.py
Allow jobs to be superseeded by each other
[people/jschlag/pbs.git] / src / buildservice / builders.py
CommitLineData
9137135a
MT
1#!/usr/bin/python
2
2c909128 3from __future__ import absolute_import, division
f96eb5ed 4
9137135a
MT
5import datetime
6import hashlib
7import logging
8import random
9import string
10import time
11
2c909128
MT
12from . import base
13from . import logs
f6e6ff79 14
3e990438
MT
15from .decorators import *
16
2c909128 17from .users import generate_password_hash, check_password_hash, generate_random_string
9137135a
MT
18
19class Builders(base.Object):
e704b8e2
MT
20 def _get_builder(self, query, *args):
21 res = self.db.get(query, *args)
22
23 if res:
24 return Builder(self.backend, res.id, data=res)
25
26 def _get_builders(self, query, *args):
27 res = self.db.query(query, *args)
28
29 for row in res:
30 yield Builder(self.backend, row.id, data=row)
31
32 def __iter__(self):
33 builders = self._get_builders("SELECT * FROM builders \
34 WHERE deleted IS FALSE ORDER BY name")
35
36 return iter(builders)
37
38 def create(self, name, user=None, log=True):
39 """
40 Creates a new builder.
41 """
42 builder = self._get_builder("INSERT INTO builders(name) \
43 VALUES(%s) RETURNING *", name)
44
45 # Generate a new passphrase.
46 passphrase = builder.regenerate_passphrase()
47
48 # Log what we have done.
49 if log:
50 builder.log("created", user=user)
51
52 # The Builder object and the passphrase are returned.
53 return builder, passphrase
54
f6e6ff79
MT
55 def auth(self, name, passphrase):
56 # If either name or passphrase is None, we don't check at all.
57 if None in (name, passphrase):
58 return
59
60 # Search for the hostname in the database.
e704b8e2
MT
61 builder = self._get_builder("SELECT * FROM builders \
62 WHERE name = %s AND deleted IS FALSE", name)
f6e6ff79
MT
63
64 # If the builder was not found or the passphrase does not match,
65 # you have bad luck.
66 if not builder or not builder.validate_passphrase(passphrase):
67 return
68
69 # Otherwise we return the Builder object.
70 return builder
71
e704b8e2
MT
72 def get_by_id(self, builder_id):
73 return self._get_builder("SELECT * FROM builders WHERE id = %s", builder_id)
9137135a
MT
74
75 def get_by_name(self, name):
e704b8e2
MT
76 return self._get_builder("SELECT * FROM builders \
77 WHERE name = %s AND deleted IS FALSE", name)
9137135a 78
f6e6ff79 79 def get_load(self):
f96eb5ed 80 res1 = self.db.get("SELECT SUM(max_jobs) AS max_jobs FROM builders \
e704b8e2 81 WHERE enabled IS TRUE and deleted IS FALSE")
f6e6ff79 82
f96eb5ed
MT
83 res2 = self.db.get("SELECT COUNT(*) AS count FROM jobs \
84 WHERE state = 'dispatching' OR state = 'running' OR state = 'uploading'")
f6e6ff79 85
33857fed
MT
86 try:
87 return (res2.count * 100 / res1.max_jobs)
88 except:
89 return 0
f6e6ff79
MT
90
91 def get_history(self, limit=None, offset=None, builder=None, user=None):
92 query = "SELECT * FROM builders_history"
93 args = []
94
95 conditions = []
96
97 if builder:
98 conditions.append("builder_id = %s")
99 args.append(builder.id)
100
101 if user:
102 conditions.append("user_id = %s")
103 args.append(user.id)
104
105 if conditions:
106 query += " WHERE %s" % " AND ".join(conditions)
107
108 query += " ORDER BY time DESC"
109
110 if limit:
111 if offset:
112 query += " LIMIT %s,%s"
113 args += [offset, limit,]
114 else:
115 query += " LIMIT %s"
116 args += [limit,]
117
118 entries = []
119 for entry in self.db.query(query, *args):
120 entry = logs.BuilderLogEntry(self.pakfire, entry)
121 entries.append(entry)
122
123 return entries
124
9137135a 125
3e990438 126class Builder(base.DataObject):
92431da4
MT
127 table = "builders"
128
e704b8e2
MT
129 def __eq__(self, other):
130 if isinstance(other, self.__class__):
131 return self.id == other.id
9137135a 132
e704b8e2
MT
133 def __lt__(self, other):
134 if isinstance(other, self.__class__):
135 return self.name < other.name
f6e6ff79
MT
136
137 def log(self, action, user=None):
138 user_id = None
139 if user:
140 user_id = user.id
141
142 self.db.execute("INSERT INTO builders_history(builder_id, action, user_id, time) \
143 VALUES(%s, %s, %s, NOW())", self.id, action, user_id)
9137135a 144
9137135a 145 def regenerate_passphrase(self):
f6e6ff79
MT
146 """
147 Generates a new random passphrase and stores it as a salted hash
148 to the database.
149
150 The new passphrase is returned to be sent to the user (once).
151 """
e704b8e2
MT
152 # Generate a random string with 40 chars.
153 passphrase = generate_random_string(length=40)
f6e6ff79
MT
154
155 # Create salted hash.
156 passphrase_hash = generate_password_hash(passphrase)
157
158 # Store the hash in the database.
3e990438 159 self._set_attribute("passphrase", passphrase_hash)
9137135a 160
f6e6ff79
MT
161 # Return the clear-text passphrase.
162 return passphrase
9137135a
MT
163
164 def validate_passphrase(self, passphrase):
f6e6ff79
MT
165 """
166 Compare the given passphrase with the one stored in the database.
167 """
168 return check_password_hash(passphrase, self.data.passphrase)
169
3e990438
MT
170 # Description
171
172 def set_description(self, description):
173 self._set_attribute("description", description)
174
175 description = property(lambda s: s.data.description or "", set_description)
f6e6ff79 176
f6e6ff79
MT
177 @property
178 def keepalive(self):
179 """
180 Returns time of last keepalive message from this host.
181 """
182 return self.data.time_keepalive
183
c2902b29
MT
184 def update_keepalive(self, loadavg1=None, loadavg5=None, loadavg15=None,
185 mem_total=None, mem_free=None, swap_total=None, swap_free=None,
186 space_free=None):
f6e6ff79
MT
187 """
188 Update the keepalive timestamp of this machine.
189 """
c2902b29
MT
190 self.db.execute("UPDATE builders SET time_keepalive = NOW(), \
191 loadavg1 = %s, loadavg5 = %s, loadavg15 = %s, space_free = %s, \
192 mem_total = %s, mem_free = %s, swap_total = %s, swap_free = %s \
193 WHERE id = %s", loadavg1, loadavg5, loadavg15, space_free,
194 mem_total, mem_free, swap_total, swap_free, self.id)
195
196 def update_info(self, cpu_model=None, cpu_count=None, cpu_arch=None, cpu_bogomips=None,
197 pakfire_version=None, host_key=None, os_name=None):
f6e6ff79
MT
198 # Update all the rest.
199 self.db.execute("UPDATE builders SET time_updated = NOW(), \
c2902b29
MT
200 pakfire_version = %s, cpu_model = %s, cpu_count = %s, cpu_arch = %s, \
201 cpu_bogomips = %s, host_key_id = %s, os_name = %s WHERE id = %s",
202 pakfire_version, cpu_model, cpu_count, cpu_arch, cpu_bogomips,
203 host_key, os_name, self.id)
f6e6ff79 204
e704b8e2
MT
205 def set_enabled(self, enabled):
206 self._set_attribute("enabled", enabled)
9137135a 207
e704b8e2 208 enabled = property(lambda s: s.data.enabled, set_enabled)
9137135a
MT
209
210 @property
211 def disabled(self):
f6e6ff79
MT
212 return not self.enabled
213
e704b8e2
MT
214 @property
215 def native_arch(self):
216 """
217 The native architecture of this builder
218 """
219 return self.cpu_arch
f6e6ff79 220
e704b8e2
MT
221 @lazy_property
222 def supported_arches(self):
223 # Every builder supports noarch
224 arches = ["noarch"]
f6e6ff79 225
e704b8e2
MT
226 # We can always build our native architeture
227 if self.native_arch:
228 arches.append(self.native_arch)
f6e6ff79 229
e704b8e2 230 # Get all compatible architectures
3e990438 231 res = self.db.query("SELECT build_arch FROM arches_compat \
e704b8e2 232 WHERE native_arch = %s", self.native_arch)
f6e6ff79 233
e704b8e2
MT
234 for row in res:
235 if not row.build_arch in arches:
236 arches.append(row.build_arch)
f6e6ff79 237
e704b8e2 238 return sorted(arches)
9137135a 239
fd43d5e1
MT
240 def set_testmode(self, testmode):
241 self._set_attribute("testmode", testmode)
f6e6ff79 242
fd43d5e1 243 testmode = property(lambda s: s.data.testmode, set_testmode)
f6e6ff79 244
9137135a 245 def set_max_jobs(self, value):
3e990438 246 self._set_attribute("max_jobs", value)
9137135a 247
3e990438 248 max_jobs = property(lambda s: s.data.max_jobs, set_max_jobs)
9137135a
MT
249
250 @property
251 def name(self):
252 return self.data.name
253
254 @property
255 def hostname(self):
256 return self.name
257
258 @property
259 def passphrase(self):
260 return self.data.passphrase
261
c2902b29
MT
262 # Load average
263
9137135a
MT
264 @property
265 def loadavg(self):
c2902b29 266 return ", ".join(["%.2f" % l for l in (self.loadavg1, self.loadavg5, self.loadavg15)])
9137135a 267
f6e6ff79 268 @property
c2902b29
MT
269 def loadavg1(self):
270 return self.data.loadavg1 or 0.0
271
272 @property
273 def loadavg5(self):
274 return self.data.loadavg5 or 0.0
f6e6ff79 275
c2902b29
MT
276 @property
277 def loadavg15(self):
278 return self.data.loadavg15 or 0.0
f6e6ff79
MT
279
280 @property
281 def pakfire_version(self):
282 return self.data.pakfire_version or ""
9137135a 283
c2902b29
MT
284 @property
285 def os_name(self):
286 return self.data.os_name or ""
287
9137135a
MT
288 @property
289 def cpu_model(self):
f6e6ff79
MT
290 return self.data.cpu_model or ""
291
292 @property
293 def cpu_count(self):
294 return self.data.cpu_count
9137135a
MT
295
296 @property
c2902b29
MT
297 def cpu_arch(self):
298 return self.data.cpu_arch
299
300 @property
301 def cpu_bogomips(self):
302 return self.data.cpu_bogomips or 0.0
303
304 @property
305 def mem_percentage(self):
306 if not self.mem_total:
307 return None
308
309 return self.mem_used * 100 / self.mem_total
9137135a
MT
310
311 @property
c2902b29
MT
312 def mem_total(self):
313 return self.data.mem_total
314
315 @property
316 def mem_used(self):
317 if self.mem_total and self.mem_free:
318 return self.mem_total - self.mem_free
319
320 @property
321 def mem_free(self):
322 return self.data.mem_free
323
324 @property
325 def swap_percentage(self):
326 if not self.swap_total:
327 return None
328
329 return self.swap_used * 100 / self.swap_total
330
331 @property
332 def swap_total(self):
333 return self.data.swap_total
334
335 @property
336 def swap_used(self):
337 if self.swap_total and self.swap_free:
338 return self.swap_total - self.swap_free
339
340 @property
341 def swap_free(self):
342 return self.data.swap_free
343
344 @property
345 def space_free(self):
346 return self.data.space_free
f6e6ff79
MT
347
348 @property
349 def overload(self):
c2902b29
MT
350 if not self.cpu_count or not self.loadavg1:
351 return None
352
353 return self.loadavg1 >= self.cpu_count
f6e6ff79
MT
354
355 @property
356 def host_key_id(self):
357 return self.data.host_key_id
358
359 @property
360 def state(self):
9137135a 361 if self.disabled:
f6e6ff79 362 return "disabled"
9137135a 363
f6e6ff79
MT
364 if self.data.time_keepalive is None:
365 return "offline"
9137135a 366
f96eb5ed
MT
367 #if self.data.updated >= 5*60:
368 # return "offline"
9137135a 369
f6e6ff79 370 return "online"
9137135a 371
3e990438
MT
372 @lazy_property
373 def active_jobs(self, *args, **kwargs):
374 return self.pakfire.jobs.get_active(builder=self, *args, **kwargs)
163d9d8b
MT
375
376 @property
377 def too_many_jobs(self):
378 """
379 Tell if this host is already running enough or too many jobs.
380 """
3e990438 381 return len(self.active_jobs) >= self.max_jobs
9e8a20d7 382
fd43d5e1
MT
383 @lazy_property
384 def jobqueue(self):
385 return self.backend.jobqueue.for_arches(self.supported_arches)
9e8a20d7 386
c2902b29
MT
387 def get_next_job(self):
388 """
389 Returns the next job in line for this builder.
390 """
fd43d5e1
MT
391 # Don't return anything if the builder has already too many jobs running
392 if self.too_many_jobs:
393 return
394
395 for job in self.jobqueue:
396 # Only allow building test jobs in test mode
397 if self.testmode and not job.type == "test":
398 continue
9137135a 399
fd43d5e1 400 return job
f6e6ff79
MT
401
402 def get_history(self, *args, **kwargs):
403 kwargs["builder"] = self
404
405 return self.pakfire.builders.get_history(*args, **kwargs)