[people/jschlag/pbs.git] / src / buildservice / builders.py

#!/usr/bin/python

from __future__ import absolute_import, division

import datetime
import hashlib
import logging
import random
import string
import time

from . import base
from . import logs

from .decorators import *

from .users import generate_password_hash, check_password_hash, generate_random_string

class Builders(base.Object):
	def _get_builder(self, query, *args):
		res = self.db.get(query, *args)

		if res:
			return Builder(self.backend, res.id, data=res)

	def _get_builders(self, query, *args):
		res = self.db.query(query, *args)

		for row in res:
			yield Builder(self.backend, row.id, data=row)

	def __iter__(self):
		builders = self._get_builders("SELECT * FROM builders \
			WHERE deleted IS FALSE ORDER BY name")

		return iter(builders)

	def create(self, name, user=None, log=True):
		"""
			Creates a new builder.
		"""
		builder = self._get_builder("INSERT INTO builders(name) \
			VALUES(%s) RETURNING *", name)

		# Generate a new passphrase.
		passphrase = builder.regenerate_passphrase()

		# Log what we have done.
		if log:
			builder.log("created", user=user)

		# The Builder object and the passphrase are returned.
		return builder, passphrase

	def auth(self, name, passphrase):
		# If either name or passphrase is None, we don't check at all.
		if None in (name, passphrase):
			return

		# Search for the hostname in the database.
		builder = self._get_builder("SELECT * FROM builders \
			WHERE name = %s AND deleted IS FALSE", name)

		# If the builder was not found or the passphrase does not match,
		# you have bad luck.
		if not builder or not builder.validate_passphrase(passphrase):
			return

		# Otherwise we return the Builder object.
		return builder

	def get_by_id(self, builder_id):
		return self._get_builder("SELECT * FROM builders WHERE id = %s", builder_id)

	def get_by_name(self, name):
		return self._get_builder("SELECT * FROM builders \
			WHERE name = %s AND deleted IS FALSE", name)

	def get_load(self):
		res1 = self.db.get("SELECT SUM(max_jobs) AS max_jobs FROM builders \
			WHERE enabled IS TRUE and deleted IS FALSE")

		res2 = self.db.get("SELECT COUNT(*) AS count FROM jobs \
			WHERE state = 'dispatching' OR state = 'running' OR state = 'uploading'")

		try:
			return (res2.count * 100 / res1.max_jobs)
		except:
			return 0

	def get_history(self, limit=None, offset=None, builder=None, user=None):
		query = "SELECT * FROM builders_history"
		args  = []

		conditions = []

		if builder:
			conditions.append("builder_id = %s")
			args.append(builder.id)

		if user:
			conditions.append("user_id = %s")
			args.append(user.id)

		if conditions:
			query += " WHERE %s" % " AND ".join(conditions)

		query += " ORDER BY time DESC"

		if limit:
			if offset:
				query += " LIMIT %s,%s"
				args  += [offset, limit,]
			else:
				query += " LIMIT %s"
				args  += [limit,]

		entries = []
		for entry in self.db.query(query, *args):
			entry = logs.BuilderLogEntry(self.pakfire, entry)
			entries.append(entry)

		return entries


class Builder(base.DataObject):
	table = "builders"

	def __eq__(self, other):
		if isinstance(other, self.__class__):
			return self.id == other.id

	def __lt__(self, other):
		if isinstance(other, self.__class__):
			return self.name < other.name

	def log(self, action, user=None):
		user_id = None
		if user:
			user_id = user.id

		self.db.execute("INSERT INTO builders_history(builder_id, action, user_id, time) \
			VALUES(%s, %s, %s, NOW())", self.id, action, user_id)

	def regenerate_passphrase(self):
		"""
			Generates a new random passphrase and stores it as a salted hash
			to the database.

			The new passphrase is returned to be sent to the user (once).
		"""
		# Generate a random string with 40 chars.
		passphrase = generate_random_string(length=40)

		# Create salted hash.
		passphrase_hash = generate_password_hash(passphrase)

		# Store the hash in the database.
		self._set_attribute("passphrase", passphrase_hash)

		# Return the clear-text passphrase.
		return passphrase

	def validate_passphrase(self, passphrase):
		"""
			Compare the given passphrase with the one stored in the database.
		"""
		return check_password_hash(passphrase, self.data.passphrase)

	# Description

	def set_description(self, description):
		self._set_attribute("description", description)

	description = property(lambda s: s.data.description or "", set_description)

	@property
	def keepalive(self):
		"""
			Returns time of last keepalive message from this host.
		"""
		return self.data.time_keepalive

	def update_keepalive(self, loadavg1=None, loadavg5=None, loadavg15=None,
			mem_total=None, mem_free=None, swap_total=None, swap_free=None,
			space_free=None):
		"""
			Update the keepalive timestamp of this machine.
		"""
		self.db.execute("UPDATE builders SET time_keepalive = NOW(), \
			loadavg1 = %s, loadavg5 = %s, loadavg15 = %s, space_free = %s, \
			mem_total = %s, mem_free = %s, swap_total = %s, swap_free = %s \
			WHERE id = %s", loadavg1, loadavg5, loadavg15, space_free,
			mem_total, mem_free, swap_total, swap_free, self.id)

	def update_info(self, cpu_model=None, cpu_count=None, cpu_arch=None, cpu_bogomips=None,
			pakfire_version=None, host_key=None, os_name=None):
		# Update all the rest.
		self.db.execute("UPDATE builders SET time_updated = NOW(), \
			pakfire_version = %s, cpu_model = %s, cpu_count = %s, cpu_arch = %s, \
			cpu_bogomips = %s, host_key_id = %s, os_name = %s WHERE id = %s",
			pakfire_version, cpu_model, cpu_count, cpu_arch, cpu_bogomips,
			host_key, os_name, self.id)

	def set_enabled(self, enabled):
		self._set_attribute("enabled", enabled)

	enabled = property(lambda s: s.data.enabled, set_enabled)

	@property
	def disabled(self):
		return not self.enabled

	@property
	def native_arch(self):
		"""
			The native architecture of this builder
		"""
		return self.cpu_arch

	@lazy_property
	def supported_arches(self):
		# Every builder supports noarch
		arches = ["noarch"]

		# We can always build our native architeture
		if self.native_arch:
			arches.append(self.native_arch)

			# Get all compatible architectures
			res = self.db.query("SELECT build_arch FROM arches_compat \
				WHERE native_arch = %s", self.native_arch)

			for row in res:
				if not row.build_arch in arches:
					arches.append(row.build_arch)

		return sorted(arches)

	def set_testmode(self, testmode):
		self._set_attribute("testmode", testmode)

	testmode = property(lambda s: s.data.testmode, set_testmode)

	def set_max_jobs(self, value):
		self._set_attribute("max_jobs", value)

	max_jobs = property(lambda s: s.data.max_jobs, set_max_jobs)

	@property
	def name(self):
		return self.data.name

	@property
	def hostname(self):
		return self.name

	@property
	def passphrase(self):
		return self.data.passphrase

	# Load average

	@property
	def loadavg(self):
		return ", ".join(["%.2f" % l for l in (self.loadavg1, self.loadavg5, self.loadavg15)])

	@property
	def loadavg1(self):
		return self.data.loadavg1 or 0.0

	@property
	def loadavg5(self):
		return self.data.loadavg5 or 0.0

	@property
	def loadavg15(self):
		return self.data.loadavg15 or 0.0

	@property
	def pakfire_version(self):
		return self.data.pakfire_version or ""

	@property
	def os_name(self):
		return self.data.os_name or ""

	@property
	def cpu_model(self):
		return self.data.cpu_model or ""

	@property
	def cpu_count(self):
		return self.data.cpu_count

	@property
	def cpu_arch(self):
		return self.data.cpu_arch

	@property
	def cpu_bogomips(self):
		return self.data.cpu_bogomips or 0.0

	@property
	def mem_percentage(self):
		if not self.mem_total:
			return None

		return self.mem_used * 100 / self.mem_total

	@property
	def mem_total(self):
		return self.data.mem_total

	@property
	def mem_used(self):
		if self.mem_total and self.mem_free:
			return self.mem_total - self.mem_free

	@property
	def mem_free(self):
		return self.data.mem_free

	@property
	def swap_percentage(self):
		if not self.swap_total:
			return None

		return self.swap_used * 100 / self.swap_total

	@property
	def swap_total(self):
		return self.data.swap_total

	@property
	def swap_used(self):
		if self.swap_total and self.swap_free:
			return self.swap_total - self.swap_free

	@property
	def swap_free(self):
		return self.data.swap_free

	@property
	def space_free(self):
		return self.data.space_free

	@property
	def overload(self):
		if not self.cpu_count or not self.loadavg1:
			return None

		return self.loadavg1 >= self.cpu_count

	@property
	def host_key_id(self):
		return self.data.host_key_id

	@property
	def state(self):
		if self.disabled:
			return "disabled"

		if self.data.time_keepalive is None:
			return "offline"

		#if self.data.updated >= 5*60:
		#	return "offline"

		return "online"

	@lazy_property
	def active_jobs(self, *args, **kwargs):
		return self.pakfire.jobs.get_active(builder=self, *args, **kwargs)

	@property
	def too_many_jobs(self):
		"""
			Tell if this host is already running enough or too many jobs.
		"""
		return len(self.active_jobs) >= self.max_jobs

	@lazy_property
	def jobqueue(self):
		return self.backend.jobqueue.for_arches(self.supported_arches)

	def get_next_job(self):
		"""
			Returns the next job in line for this builder.
		"""
		# Don't return anything if the builder has already too many jobs running
		if self.too_many_jobs:
			return

		for job in self.jobqueue:
			# Only allow building test jobs in test mode
			if self.testmode and not job.type == "test":
				continue

			return job

	def get_history(self, *args, **kwargs):
		kwargs["builder"] = self

		return self.pakfire.builders.get_history(*args, **kwargs)
Commit	Line	Data
9137135a MT	1	#!/usr/bin/python
9137135a MT	2
2c909128	3	from __future__ import absolute_import, division
f96eb5ed	4
9137135a MT	5	import datetime
	6	import hashlib
	7	import logging
	8	import random
	9	import string
	10	import time
	11
2c909128 MT	12	from . import base
2c909128 MT	13	from . import logs
f6e6ff79	14
3e990438 MT	15	from .decorators import *
3e990438 MT	16
2c909128	17	from .users import generate_password_hash, check_password_hash, generate_random_string
9137135a MT	18
9137135a MT	19	class Builders(base.Object):
e704b8e2 MT	20	def _get_builder(self, query, *args):
	21	res = self.db.get(query, *args)
	22
	23	if res:
	24	return Builder(self.backend, res.id, data=res)
	25
	26	def _get_builders(self, query, *args):
	27	res = self.db.query(query, *args)
	28
	29	for row in res:
	30	yield Builder(self.backend, row.id, data=row)
	31
	32	def __iter__(self):
	33	builders = self._get_builders("SELECT * FROM builders \
	34	WHERE deleted IS FALSE ORDER BY name")
	35
	36	return iter(builders)
	37
	38	def create(self, name, user=None, log=True):
	39	"""
	40	Creates a new builder.
	41	"""
	42	builder = self._get_builder("INSERT INTO builders(name) \
	43	VALUES(%s) RETURNING *", name)
	44
	45	# Generate a new passphrase.
	46	passphrase = builder.regenerate_passphrase()
	47
	48	# Log what we have done.
	49	if log:
	50	builder.log("created", user=user)
	51
	52	# The Builder object and the passphrase are returned.
	53	return builder, passphrase
	54
f6e6ff79 MT	55	def auth(self, name, passphrase):
	56	# If either name or passphrase is None, we don't check at all.
	57	if None in (name, passphrase):
	58	return
	59
	60	# Search for the hostname in the database.
e704b8e2 MT	61	builder = self._get_builder("SELECT * FROM builders \
e704b8e2 MT	62	WHERE name = %s AND deleted IS FALSE", name)
f6e6ff79 MT	63
	64	# If the builder was not found or the passphrase does not match,
	65	# you have bad luck.
	66	if not builder or not builder.validate_passphrase(passphrase):
	67	return
	68
	69	# Otherwise we return the Builder object.
	70	return builder
	71
e704b8e2 MT	72	def get_by_id(self, builder_id):
e704b8e2 MT	73	return self._get_builder("SELECT * FROM builders WHERE id = %s", builder_id)
9137135a MT	74
9137135a MT	75	def get_by_name(self, name):
e704b8e2 MT	76	return self._get_builder("SELECT * FROM builders \
e704b8e2 MT	77	WHERE name = %s AND deleted IS FALSE", name)
9137135a	78
f6e6ff79	79	def get_load(self):
f96eb5ed	80	res1 = self.db.get("SELECT SUM(max_jobs) AS max_jobs FROM builders \
e704b8e2	81	WHERE enabled IS TRUE and deleted IS FALSE")
f6e6ff79	82
f96eb5ed MT	83	res2 = self.db.get("SELECT COUNT(*) AS count FROM jobs \
f96eb5ed MT	84	WHERE state = 'dispatching' OR state = 'running' OR state = 'uploading'")
f6e6ff79	85
33857fed MT	86	try:
	87	return (res2.count * 100 / res1.max_jobs)
	88	except:
	89	return 0
f6e6ff79 MT	90
	91	def get_history(self, limit=None, offset=None, builder=None, user=None):
	92	query = "SELECT * FROM builders_history"
	93	args = []
	94
	95	conditions = []
	96
	97	if builder:
	98	conditions.append("builder_id = %s")
	99	args.append(builder.id)
	100
	101	if user:
	102	conditions.append("user_id = %s")
	103	args.append(user.id)
	104
	105	if conditions:
	106	query += " WHERE %s" % " AND ".join(conditions)
	107
	108	query += " ORDER BY time DESC"
	109
	110	if limit:
	111	if offset:
	112	query += " LIMIT %s,%s"
	113	args += [offset, limit,]
	114	else:
	115	query += " LIMIT %s"
	116	args += [limit,]
	117
	118	entries = []
	119	for entry in self.db.query(query, *args):
	120	entry = logs.BuilderLogEntry(self.pakfire, entry)
	121	entries.append(entry)
	122
	123	return entries
	124
9137135a	125
3e990438	126	class Builder(base.DataObject):
92431da4 MT	127	table = "builders"
92431da4 MT	128
e704b8e2 MT	129	def __eq__(self, other):
	130	if isinstance(other, self.__class__):
	131	return self.id == other.id
9137135a	132
e704b8e2 MT	133	def __lt__(self, other):
	134	if isinstance(other, self.__class__):
	135	return self.name < other.name
f6e6ff79 MT	136
	137	def log(self, action, user=None):
	138	user_id = None
	139	if user:
	140	user_id = user.id
	141
	142	self.db.execute("INSERT INTO builders_history(builder_id, action, user_id, time) \
	143	VALUES(%s, %s, %s, NOW())", self.id, action, user_id)
9137135a	144
9137135a	145	def regenerate_passphrase(self):
f6e6ff79 MT	146	"""
	147	Generates a new random passphrase and stores it as a salted hash
	148	to the database.
	149
	150	The new passphrase is returned to be sent to the user (once).
	151	"""
e704b8e2 MT	152	# Generate a random string with 40 chars.
e704b8e2 MT	153	passphrase = generate_random_string(length=40)
f6e6ff79 MT	154
	155	# Create salted hash.
	156	passphrase_hash = generate_password_hash(passphrase)
	157
	158	# Store the hash in the database.
3e990438	159	self._set_attribute("passphrase", passphrase_hash)
9137135a	160
f6e6ff79 MT	161	# Return the clear-text passphrase.
f6e6ff79 MT	162	return passphrase
9137135a MT	163
9137135a MT	164	def validate_passphrase(self, passphrase):
f6e6ff79 MT	165	"""
	166	Compare the given passphrase with the one stored in the database.
	167	"""
	168	return check_password_hash(passphrase, self.data.passphrase)
	169
3e990438 MT	170	# Description
	171
	172	def set_description(self, description):
	173	self._set_attribute("description", description)
	174
	175	description = property(lambda s: s.data.description or "", set_description)
f6e6ff79	176
f6e6ff79 MT	177	@property
	178	def keepalive(self):
	179	"""
	180	Returns time of last keepalive message from this host.
	181	"""
	182	return self.data.time_keepalive
	183
c2902b29 MT	184	def update_keepalive(self, loadavg1=None, loadavg5=None, loadavg15=None,
	185	mem_total=None, mem_free=None, swap_total=None, swap_free=None,
	186	space_free=None):
f6e6ff79 MT	187	"""
	188	Update the keepalive timestamp of this machine.
	189	"""
c2902b29 MT	190	self.db.execute("UPDATE builders SET time_keepalive = NOW(), \
	191	loadavg1 = %s, loadavg5 = %s, loadavg15 = %s, space_free = %s, \
	192	mem_total = %s, mem_free = %s, swap_total = %s, swap_free = %s \
	193	WHERE id = %s", loadavg1, loadavg5, loadavg15, space_free,
	194	mem_total, mem_free, swap_total, swap_free, self.id)
	195
	196	def update_info(self, cpu_model=None, cpu_count=None, cpu_arch=None, cpu_bogomips=None,
	197	pakfire_version=None, host_key=None, os_name=None):
f6e6ff79 MT	198	# Update all the rest.
f6e6ff79 MT	199	self.db.execute("UPDATE builders SET time_updated = NOW(), \
c2902b29 MT	200	pakfire_version = %s, cpu_model = %s, cpu_count = %s, cpu_arch = %s, \
	201	cpu_bogomips = %s, host_key_id = %s, os_name = %s WHERE id = %s",
	202	pakfire_version, cpu_model, cpu_count, cpu_arch, cpu_bogomips,
	203	host_key, os_name, self.id)
f6e6ff79	204
e704b8e2 MT	205	def set_enabled(self, enabled):
e704b8e2 MT	206	self._set_attribute("enabled", enabled)
9137135a	207
e704b8e2	208	enabled = property(lambda s: s.data.enabled, set_enabled)
9137135a MT	209
	210	@property
	211	def disabled(self):
f6e6ff79 MT	212	return not self.enabled
f6e6ff79 MT	213
e704b8e2 MT	214	@property
	215	def native_arch(self):
	216	"""
	217	The native architecture of this builder
	218	"""
	219	return self.cpu_arch
f6e6ff79	220
e704b8e2 MT	221	@lazy_property
	222	def supported_arches(self):
	223	# Every builder supports noarch
	224	arches = ["noarch"]
f6e6ff79	225
e704b8e2 MT	226	# We can always build our native architeture
	227	if self.native_arch:
	228	arches.append(self.native_arch)
f6e6ff79	229
e704b8e2	230	# Get all compatible architectures
3e990438	231	res = self.db.query("SELECT build_arch FROM arches_compat \
e704b8e2	232	WHERE native_arch = %s", self.native_arch)
f6e6ff79	233
e704b8e2 MT	234	for row in res:
	235	if not row.build_arch in arches:
	236	arches.append(row.build_arch)
f6e6ff79	237
e704b8e2	238	return sorted(arches)
9137135a	239
fd43d5e1 MT	240	def set_testmode(self, testmode):
fd43d5e1 MT	241	self._set_attribute("testmode", testmode)
f6e6ff79	242
fd43d5e1	243	testmode = property(lambda s: s.data.testmode, set_testmode)
f6e6ff79	244
9137135a	245	def set_max_jobs(self, value):
3e990438	246	self._set_attribute("max_jobs", value)
9137135a	247
3e990438	248	max_jobs = property(lambda s: s.data.max_jobs, set_max_jobs)
9137135a MT	249
	250	@property
	251	def name(self):
	252	return self.data.name
	253
	254	@property
	255	def hostname(self):
	256	return self.name
	257
	258	@property
	259	def passphrase(self):
	260	return self.data.passphrase
	261
c2902b29 MT	262	# Load average
c2902b29 MT	263
9137135a MT	264	@property
9137135a MT	265	def loadavg(self):
c2902b29	266	return ", ".join(["%.2f" % l for l in (self.loadavg1, self.loadavg5, self.loadavg15)])
9137135a	267
f6e6ff79	268	@property
c2902b29 MT	269	def loadavg1(self):
	270	return self.data.loadavg1 or 0.0
	271
	272	@property
	273	def loadavg5(self):
	274	return self.data.loadavg5 or 0.0
f6e6ff79	275
c2902b29 MT	276	@property
	277	def loadavg15(self):
	278	return self.data.loadavg15 or 0.0
f6e6ff79 MT	279
	280	@property
	281	def pakfire_version(self):
	282	return self.data.pakfire_version or ""
9137135a	283
c2902b29 MT	284	@property
	285	def os_name(self):
	286	return self.data.os_name or ""
	287
9137135a MT	288	@property
9137135a MT	289	def cpu_model(self):
f6e6ff79 MT	290	return self.data.cpu_model or ""
	291
	292	@property
	293	def cpu_count(self):
	294	return self.data.cpu_count
9137135a MT	295
9137135a MT	296	@property
c2902b29 MT	297	def cpu_arch(self):
	298	return self.data.cpu_arch
	299
	300	@property
	301	def cpu_bogomips(self):
	302	return self.data.cpu_bogomips or 0.0
	303
	304	@property
	305	def mem_percentage(self):
	306	if not self.mem_total:
	307	return None
	308
	309	return self.mem_used * 100 / self.mem_total
9137135a MT	310
9137135a MT	311	@property
c2902b29 MT	312	def mem_total(self):
	313	return self.data.mem_total
	314
	315	@property
	316	def mem_used(self):
	317	if self.mem_total and self.mem_free:
	318	return self.mem_total - self.mem_free
	319
	320	@property
	321	def mem_free(self):
	322	return self.data.mem_free
	323
	324	@property
	325	def swap_percentage(self):
	326	if not self.swap_total:
	327	return None
	328
	329	return self.swap_used * 100 / self.swap_total
	330
	331	@property
	332	def swap_total(self):
	333	return self.data.swap_total
	334
	335	@property
	336	def swap_used(self):
	337	if self.swap_total and self.swap_free:
	338	return self.swap_total - self.swap_free
	339
	340	@property
	341	def swap_free(self):
	342	return self.data.swap_free
	343
	344	@property
	345	def space_free(self):
	346	return self.data.space_free
f6e6ff79 MT	347
	348	@property
	349	def overload(self):
c2902b29 MT	350	if not self.cpu_count or not self.loadavg1:
	351	return None
	352
	353	return self.loadavg1 >= self.cpu_count
f6e6ff79 MT	354
	355	@property
	356	def host_key_id(self):
	357	return self.data.host_key_id
	358
	359	@property
	360	def state(self):
9137135a	361	if self.disabled:
f6e6ff79	362	return "disabled"
9137135a	363
f6e6ff79 MT	364	if self.data.time_keepalive is None:
f6e6ff79 MT	365	return "offline"
9137135a	366
f96eb5ed MT	367	#if self.data.updated >= 5*60:
f96eb5ed MT	368	# return "offline"
9137135a	369
f6e6ff79	370	return "online"
9137135a	371
3e990438 MT	372	@lazy_property
	373	def active_jobs(self, args, *kwargs):
	374	return self.pakfire.jobs.get_active(builder=self, args, *kwargs)
163d9d8b MT	375
	376	@property
	377	def too_many_jobs(self):
	378	"""
	379	Tell if this host is already running enough or too many jobs.
	380	"""
3e990438	381	return len(self.active_jobs) >= self.max_jobs
9e8a20d7	382
fd43d5e1 MT	383	@lazy_property
	384	def jobqueue(self):
	385	return self.backend.jobqueue.for_arches(self.supported_arches)
9e8a20d7	386
c2902b29 MT	387	def get_next_job(self):
	388	"""
	389	Returns the next job in line for this builder.
	390	"""
fd43d5e1 MT	391	# Don't return anything if the builder has already too many jobs running
	392	if self.too_many_jobs:
	393	return
	394
	395	for job in self.jobqueue:
	396	# Only allow building test jobs in test mode
	397	if self.testmode and not job.type == "test":
	398	continue
9137135a	399
fd43d5e1	400	return job
f6e6ff79 MT	401
	402	def get_history(self, args, *kwargs):
	403	kwargs["builder"] = self
	404
	405	return self.pakfire.builders.get_history(args, *kwargs)