[ipfire.org.git] / build / builder.py

#!/usr/bin/python
###############################################################################
#                                                                             #
# IPFire.org - A linux based firewall                                         #
# Copyright (C) 2008  Michael Tremer & Christian Schmidt                      #
#                                                                             #
# This program is free software: you can redistribute it and/or modify        #
# it under the terms of the GNU General Public License as published by        #
# the Free Software Foundation, either version 3 of the License, or           #
# (at your option) any later version.                                         #
#                                                                             #
# This program is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
# GNU General Public License for more details.                                #
#                                                                             #
# You should have received a copy of the GNU General Public License           #
# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
#                                                                             #
###############################################################################

import os
import sys
import time
import socket
import base64
import shutil
from pysqlite2 import dbapi2 as sqlite

sys.path.append(".")

from constants import config

class Database:
	def __init__(self, path):
		self.db = sqlite.connect(os.path.join(path, config["db_name"]))
		c = self.cursor()
		c.executescript("""
			create table if not exists config(key, value, date);
			create table if not exists durations(duration);
		""")
		c.close()
	
	def __call__(self):
		return self.cursor()
	
	def __del__(self):
		self.commit()
		self.db.close()

	def cursor(self):
		return self.db.cursor()
	
	def commit(self):
		self.db.commit()

class DatabaseConfig:
	def __init__(self, db, key, base64=0):
		self.db = db
		self.key = key
		self.data = None
		self.date = None
		self.base64 = base64

	def get(self):
		if not self.data:
			c = self.db.cursor()
			c.execute("SELECT value FROM %(table)s WHERE key = '%(key)s'" \
						% { "table" : "config",
							"key"   : self.key, })
			try:
				self.data = c.fetchone()[0]
			except TypeError:
				self.data = None
			c.close()
		return self.data
	
	__call__ = get
	
	def time(self):
		if not self.date:
			c = self.db.cursor()
			c.execute("SELECT date FROM %(table)s WHERE key = '%(key)s'" \
						% { "table" : "config",
							"key"   : self.key, })
			try:
				self.date = float("%s" % c.fetchone()[0])
			except TypeError:
				self.date = None
			c.close()
		return self.date or float(0)

	def set(self, value):
		if self.base64:
			value = base64.b64decode(value)
		#value = (value,)
		c = self.db.cursor()
		if not self.get():
			sql = "INSERT INTO %(table)s(key, value, date) VALUES('%(key)s', '%(value)s', '%(date)s')" \
						% { "table" : "config",
							"key"   : self.key,
							"value" : value,
							"date"  : time.time(), }
			
		else:
			sql = "UPDATE %(table)s SET value='%(value)s', date='%(date)s' WHERE key='%(key)s'" \
						% { "table" : "config",
							"key"   : self.key,
							"value" : value,
							"date"  : time.time(), }
		c.execute(sql)
		c.close()
		self.data = value
		self.db.commit()
		return """Set "%s" to "%s".""" % (self.key, self.data,)

class DurationsConfig:
	def __init__(self, db):
		self.db = db
	
	def get(self, sort=0):
		c = self.db.cursor()
		c.execute("SELECT duration FROM durations")
		ret = []
		for value in c.fetchall():
			value = int("%s" % value)
			if value < 900: # 15min
				continue
			ret.append(value)
		c.close()
		if sort: ret.sort()
		return ret
	
	def set(self, value):
		#value = (value,)
		c = self.db.cursor()
		c.execute("INSERT INTO %(table)s(duration) VALUES('%(value)s')" \
					% { "table" : "durations",
						"value" : value, })
		c.close()
		self.db.commit()
		return """Accepted build duration of %s seconds.""" % (value,)

	def get_avg(self):
		sum = 0
		durations = self.get()
		if not len(durations):
			return None
		for value in durations:
			sum += value
		avg = sum / len(durations)
		return avg
	
	def get_eta(self, timestamp):
		avg = self.get_avg()
		if not avg:
			return "N/A"
		eta = int(timestamp) + avg
		return time.ctime(eta)

class DistccConfig(DatabaseConfig):
	def __init__(self, db, key, hostname, jobs):
		DatabaseConfig.__init__(self, db, key)
		self.hostname = hostname
		self.jobs = jobs
	
	def __str__(self):
		if not self.ping() or self.get() == "0":
			return ""
		return "%s:%s/%s,lzo   \t# %s" % \
			(socket.gethostbyname(self.hostname), self.get(), self.jobs or "4", self.hostname)

	def ping(self):
		if not self.hostname:
			return False
		return not os.system("ping -c1 -w1 %s &>/dev/null" % self.hostname)

	def version(self):
		return os.popen("distcc --version").readlines()

class FileConfig:
	def __init__(self, path, filetype):
		self.filename = os.path.join(path, config["path"][filetype])

		# Create the file if not existant
		if not os.access(self.filename, os.R_OK):
			f = open(self.filename, "w")
			f.close()

	def get(self):
		ret = []
		try:
			f = open(self.filename)
			ret = f.readlines()
			f.close()
		except:
			pass
		return ret or ["Log is empty."]
	
	__call__ = get
	
	def set(self, lines):
		f = open(self.filename, "w")
		for line in base64.b64decode(lines).split("\n"):
			f.write("%s\n" % line.rstrip("\n"))
		f.close()
		return """Saved file content to %s.""" % (self.filename,)

class Builder:
	def __init__(self, config, uuid):
		self.uuid = uuid
		self.config = config
		self.path = os.path.join(self.config['path']['db'], self.uuid)
		
		if not os.access(self.path, os.R_OK):
			try:
				os.mkdir(self.path)
			except:
				pass

		self.db = Database(self.path)

		self.hostname = DatabaseConfig(self.db, "hostname")
		self.state    = DatabaseConfig(self.db, "state")
		self.package  = DatabaseConfig(self.db, "package")
		self.target   = DatabaseConfig(self.db, "target")

		self.duration = DurationsConfig(self.db)
		self.jobs     = DatabaseConfig(self.db, "jobs")
		self.distcc   = DistccConfig(self.db, "distcc", self.hostname(), self.jobs())
		self.cpu      = DatabaseConfig(self.db, "cpu", base64=1)
		self.machine  = DatabaseConfig(self.db, "machine")

		self.log      = FileConfig(self.path, "log")
		
		# If host was longer than 3 days in state compiling we set it as idle.
		if self.state() == "compiling" and \
				(time.time() - self.state.time()) > 3*24*60*60:
			self.state.set("idle")
		
		# If host is idle and distcc is not disabled we set it as distcc host.
		if self.state() == "idle" and self.distcc() != "0":
			self.state.set("distcc")

		# If host is longer than 24h in error state we set it as distcc host.
		if self.state() == "error" and \
				(time.time() - self.state.time()) > 24*60*60:
			self.state.set("distcc")
		
		# If host was longer than two weeks in distcc state we set it as unknown.
		if self.state() == "error" and \
				(time.time() - self.state.time()) > 2*7*24*60*60:
			self.state.set("unknown")

		# If host was longer than four weels in distcc state we delete it.
		if self.state() in ("distcc", "unknown",) and \
				(time.time() - self.state.time()) > 4*7*24*60*60:
			del self.db
			shutil.rmtree(self.path)

	def set(self, key, value):
		return eval("self.%s.set(\"%s\")" % (key, value,))

	def get(self, key):
		return eval("self.%s.get()" % (key,))

def getAllBuilders(age=0):
	builders = []
	for uuid in os.listdir(config["path"]["db"]):
		if uuid == "empty.txt":	continue
		builder = Builder(config, uuid)
		# If there was no activity since "age" days -> continue...
		if age and (time.time() - builder.state.time()) > age*24*60*60:
				continue
		builders.append(builder)
	return builders
Commit	Line	Data
2b60fce9 MT	1	#!/usr/bin/python
	2	###############################################################################
	3	# #
	4	# IPFire.org - A linux based firewall #
	5	# Copyright (C) 2008 Michael Tremer & Christian Schmidt #
	6	# #
	7	# This program is free software: you can redistribute it and/or modify #
	8	# it under the terms of the GNU General Public License as published by #
	9	# the Free Software Foundation, either version 3 of the License, or #
	10	# (at your option) any later version. #
	11	# #
	12	# This program is distributed in the hope that it will be useful, #
	13	# but WITHOUT ANY WARRANTY; without even the implied warranty of #
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
	15	# GNU General Public License for more details. #
	16	# #
	17	# You should have received a copy of the GNU General Public License #
	18	# along with this program. If not, see <http://www.gnu.org/licenses/>. #
	19	# #
	20	###############################################################################
	21
	22	import os
	23	import sys
	24	import time
	25	import socket
10da7ad4	26	import base64
f006517e	27	import shutil
2b60fce9 MT	28	from pysqlite2 import dbapi2 as sqlite
	29
	30	sys.path.append(".")
	31
	32	from constants import config
	33
	34	class Database:
	35	def __init__(self, path):
	36	self.db = sqlite.connect(os.path.join(path, config["db_name"]))
	37	c = self.cursor()
	38	c.executescript("""
	39	create table if not exists config(key, value, date);
	40	create table if not exists durations(duration);
	41	""")
	42	c.close()
	43
	44	def __call__(self):
	45	return self.cursor()
	46
	47	def __del__(self):
	48	self.commit()
	49	self.db.close()
	50
	51	def cursor(self):
	52	return self.db.cursor()
	53
	54	def commit(self):
	55	self.db.commit()
	56
	57	class DatabaseConfig:
0359f8fc	58	def __init__(self, db, key, base64=0):
2b60fce9 MT	59	self.db = db
	60	self.key = key
	61	self.data = None
	62	self.date = None
0359f8fc	63	self.base64 = base64
2b60fce9 MT	64
	65	def get(self):
	66	if not self.data:
	67	c = self.db.cursor()
	68	c.execute("SELECT value FROM %(table)s WHERE key = '%(key)s'" \
	69	% { "table" : "config",
	70	"key" : self.key, })
	71	try:
	72	self.data = c.fetchone()[0]
	73	except TypeError:
	74	self.data = None
	75	c.close()
	76	return self.data
	77
	78	__call__ = get
	79
	80	def time(self):
	81	if not self.date:
	82	c = self.db.cursor()
	83	c.execute("SELECT date FROM %(table)s WHERE key = '%(key)s'" \
	84	% { "table" : "config",
	85	"key" : self.key, })
	86	try:
	87	self.date = float("%s" % c.fetchone()[0])
	88	except TypeError:
	89	self.date = None
	90	c.close()
	91	return self.date or float(0)
	92
	93	def set(self, value):
0359f8fc MT	94	if self.base64:
0359f8fc MT	95	value = base64.b64decode(value)
2b60fce9 MT	96	#value = (value,)
	97	c = self.db.cursor()
	98	if not self.get():
	99	sql = "INSERT INTO %(table)s(key, value, date) VALUES('%(key)s', '%(value)s', '%(date)s')" \
	100	% { "table" : "config",
	101	"key" : self.key,
	102	"value" : value,
	103	"date" : time.time(), }
	104
	105	else:
	106	sql = "UPDATE %(table)s SET value='%(value)s', date='%(date)s' WHERE key='%(key)s'" \
	107	% { "table" : "config",
	108	"key" : self.key,
	109	"value" : value,
	110	"date" : time.time(), }
	111	c.execute(sql)
	112	c.close()
	113	self.data = value
	114	self.db.commit()
76bb443a	115	return """Set "%s" to "%s".""" % (self.key, self.data,)
2b60fce9 MT	116
	117	class DurationsConfig:
	118	def __init__(self, db):
	119	self.db = db
	120
	121	def get(self, sort=0):
	122	c = self.db.cursor()
	123	c.execute("SELECT duration FROM durations")
1e0061eb MT	124	ret = []
	125	for value in c.fetchall():
	126	value = int("%s" % value)
906c175d	127	if value < 900: # 15min
1e0061eb MT	128	continue
1e0061eb MT	129	ret.append(value)
2b60fce9 MT	130	c.close()
	131	if sort: ret.sort()
	132	return ret
	133
	134	def set(self, value):
	135	#value = (value,)
	136	c = self.db.cursor()
	137	c.execute("INSERT INTO %(table)s(duration) VALUES('%(value)s')" \
	138	% { "table" : "durations",
	139	"value" : value, })
	140	c.close()
	141	self.db.commit()
76bb443a	142	return """Accepted build duration of %s seconds.""" % (value,)
2b60fce9 MT	143
	144	def get_avg(self):
	145	sum = 0
	146	durations = self.get()
	147	if not len(durations):
	148	return None
	149	for value in durations:
1e0061eb	150	sum += value
2b60fce9 MT	151	avg = sum / len(durations)
	152	return avg
	153
	154	def get_eta(self, timestamp):
	155	avg = self.get_avg()
	156	if not avg:
	157	return "N/A"
	158	eta = int(timestamp) + avg
	159	return time.ctime(eta)
	160
	161	class DistccConfig(DatabaseConfig):
10da7ad4	162	def __init__(self, db, key, hostname, jobs):
2b60fce9 MT	163	DatabaseConfig.__init__(self, db, key)
2b60fce9 MT	164	self.hostname = hostname
10da7ad4	165	self.jobs = jobs
2b60fce9 MT	166
2b60fce9 MT	167	def __str__(self):
fda98c60	168	if not self.ping() or self.get() == "0":
2b60fce9	169	return ""
10da7ad4 MT	170	return "%s:%s/%s,lzo \t# %s" % \
10da7ad4 MT	171	(socket.gethostbyname(self.hostname), self.get(), self.jobs or "4", self.hostname)
2b60fce9 MT	172
	173	def ping(self):
	174	if not self.hostname:
	175	return False
	176	return not os.system("ping -c1 -w1 %s &>/dev/null" % self.hostname)
	177
	178	def version(self):
	179	return os.popen("distcc --version").readlines()
	180
10da7ad4 MT	181	class FileConfig:
	182	def __init__(self, path, filetype):
	183	self.filename = os.path.join(path, config["path"][filetype])
	184
	185	# Create the file if not existant
	186	if not os.access(self.filename, os.R_OK):
	187	f = open(self.filename, "w")
	188	f.close()
	189
	190	def get(self):
	191	ret = []
	192	try:
	193	f = open(self.filename)
	194	ret = f.readlines()
	195	f.close()
	196	except:
	197	pass
	198	return ret or ["Log is empty."]
	199
	200	__call__ = get
	201
	202	def set(self, lines):
	203	f = open(self.filename, "w")
	204	for line in base64.b64decode(lines).split("\n"):
	205	f.write("%s\n" % line.rstrip("\n"))
	206	f.close()
76bb443a	207	return """Saved file content to %s.""" % (self.filename,)
10da7ad4	208
2b60fce9 MT	209	class Builder:
	210	def __init__(self, config, uuid):
	211	self.uuid = uuid
	212	self.config = config
	213	self.path = os.path.join(self.config['path']['db'], self.uuid)
	214
	215	if not os.access(self.path, os.R_OK):
	216	try:
	217	os.mkdir(self.path)
	218	except:
	219	pass
	220
	221	self.db = Database(self.path)
10da7ad4	222
2b60fce9 MT	223	self.hostname = DatabaseConfig(self.db, "hostname")
	224	self.state = DatabaseConfig(self.db, "state")
	225	self.package = DatabaseConfig(self.db, "package")
0dc54d31	226	self.target = DatabaseConfig(self.db, "target")
10da7ad4	227
2b60fce9	228	self.duration = DurationsConfig(self.db)
10da7ad4 MT	229	self.jobs = DatabaseConfig(self.db, "jobs")
10da7ad4 MT	230	self.distcc = DistccConfig(self.db, "distcc", self.hostname(), self.jobs())
0359f8fc MT	231	self.cpu = DatabaseConfig(self.db, "cpu", base64=1)
0359f8fc MT	232	self.machine = DatabaseConfig(self.db, "machine")
10da7ad4 MT	233
10da7ad4 MT	234	self.log = FileConfig(self.path, "log")
f006517e MT	235
	236	# If host was longer than 3 days in state compiling we set it as idle.
	237	if self.state() == "compiling" and \
	238	(time.time() - self.state.time()) > 32460*60:
	239	self.state.set("idle")
	240
	241	# If host is idle and distcc is not disabled we set it as distcc host.
	242	if self.state() == "idle" and self.distcc() != "0":
	243	self.state.set("distcc")
	244
	245	# If host is longer than 24h in error state we set it as distcc host.
	246	if self.state() == "error" and \
	247	(time.time() - self.state.time()) > 246060:
	248	self.state.set("distcc")
	249
	250	# If host was longer than two weeks in distcc state we set it as unknown.
	251	if self.state() == "error" and \
	252	(time.time() - self.state.time()) > 27246060:
	253	self.state.set("unknown")
	254
	255	# If host was longer than four weels in distcc state we delete it.
76bb443a	256	if self.state() in ("distcc", "unknown",) and \
f006517e MT	257	(time.time() - self.state.time()) > 47246060:
	258	del self.db
	259	shutil.rmtree(self.path)
2b60fce9 MT	260
2b60fce9 MT	261	def set(self, key, value):
76bb443a	262	return eval("self.%s.set(\"%s\")" % (key, value,))
2b60fce9 MT	263
	264	def get(self, key):
	265	return eval("self.%s.get()" % (key,))
	266
205adcfe	267	def getAllBuilders(age=0):
2b60fce9 MT	268	builders = []
	269	for uuid in os.listdir(config["path"]["db"]):
	270	if uuid == "empty.txt": continue
	271	builder = Builder(config, uuid)
205adcfe MT	272	# If there was no activity since "age" days -> continue...
	273	if age and (time.time() - builder.state.time()) > age2460*60:
	274	continue
2b60fce9 MT	275	builders.append(builder)
2b60fce9 MT	276	return builders