IPFire Builder

This site is for monitoring the build machines...

Made: %s

" % self.uuid - - def __call__(self): - if self.state == "hidden": - return - - self.hostname = self._profile_get("HOSTNAME") - self.distcc = self._profile_get("DISTCC") - self.build_durations = self._build_durations() - self.last_update = self._last_update() - - print "" - - ## give hostname or uuid - print "\t" - print "\t\t" - print "\t" - - ## give state - print "\t" - print "\t\t" - print "\t\t" - print "\t" - - ## give sys info - for key in ["CPU_NAME", "CPU_MHZ", "CPU_CNT", "CPU_MIPS", "MEM_SIZE"]: - print "\t" - print "\t\t" - print "\t\t" - print "\t" - - ## give durations - (min, avg, max, all) = self.build_durations - print "\t" - print "\t\t" - print "\t\t" - print "\t" - - ## give distcc - print "\t" - print "\t\t" - print "\t\t" - print "\t" - - ## give log - if self.state == "error": - print "\t" - print "\t\t" - print "\t" - - ## give lastupdate - if self.last_update: - print "\t" - print "\t\t" - print "\t" - - print "

", - if self.hostname: - print self.hostname, - else: - print self.uuid, - print "
", - print stage2desc[self.state], - print "		", - print "" % (self.state, self.state,), - print "
", - print sys2desc[key] - print "	", - print self._profile_get(key) or "N/A" - print "
", - print "Build durations", - print "	", - if avg: - print "Average: %s h " % format_time(avg), - if min: - print "Minimum: %s h " % format_time(min), - if max: - print "Maximum: %s h " % format_time(max), - if all: - print "As a whole: %s h" % format_time(all), - - if not avg and not min and not max and not all: - print "N/A", - - print "
", - print "Distcc capable", - print "	", - if self.distcc == None or self.distcc == "0": - print "No", - else: - print "Yes (port: %s)" \ - % (self.distcc), - print "
", - for line in self._log(): - print "%s " % (line,) - print "
", - print "Last update:", - print time.strftime("%Y-%b-%d %I:%M %p", time.localtime(self.last_update)), - print " - ", - print format_time(int(time.time() - self.last_update)), - print "ago

" - - print "
" - -form = cgi.FieldStorage() -action = form.getfirst('action') - -if action in [ "compiling", "error", "idle", "set" ]: - builder = Builder(form.getfirst('uuid')) - if not action == "set": - builder._state(action) - if action == "set": - key = form.getfirst('key') - val = form.getfirst('val') - if key == "duration": - builder._build_durations(val) - else: - builder._profile_set((key, val)) - elif action == "error": - log = form.getfirst('log') - builder._log(log) -else: - builders = [] - for uuid in os.listdir(DB_DIR): - if not os.path.isdir(os.path.join(DB_DIR, uuid)): continue - builders.append(Builder(uuid)) - - site = Site() - site() diff --git a/build/builder.py b/build/builder.py new file mode 100644 index 00000000..4ed3feb1 --- /dev/null +++ b/build/builder.py @@ -0,0 +1,201 @@ +#!/usr/bin/python +############################################################################### +# # +# IPFire.org - A linux based firewall # +# Copyright (C) 2008 Michael Tremer & Christian Schmidt # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +import os +import sys +import time +import socket +from pysqlite2 import dbapi2 as sqlite + +sys.path.append(".") + +from constants import config + +class Database: + def __init__(self, path): + self.db = sqlite.connect(os.path.join(path, config["db_name"])) + c = self.cursor() + c.executescript(""" + create table if not exists config(key, value, date); + create table if not exists durations(duration); + """) + c.close() + + def __call__(self): + return self.cursor() + + def __del__(self): + self.commit() + self.db.close() + + def cursor(self): + return self.db.cursor() + + def commit(self): + self.db.commit() + +class DatabaseConfig: + def __init__(self, db, key): + self.db = db + self.key = key + self.data = None + self.date = None + + def get(self): + if not self.data: + c = self.db.cursor() + c.execute("SELECT value FROM %(table)s WHERE key = '%(key)s'" \ + % { "table" : "config", + "key" : self.key, }) + try: + self.data = c.fetchone()[0] + except TypeError: + self.data = None + c.close() + return self.data + + __call__ = get + + def time(self): + if not self.date: + c = self.db.cursor() + c.execute("SELECT date FROM %(table)s WHERE key = '%(key)s'" \ + % { "table" : "config", + "key" : self.key, }) + try: + self.date = float("%s" % c.fetchone()[0]) + except TypeError: + self.date = None + c.close() + return self.date or float(0) + + def set(self, value): + #value = (value,) + c = self.db.cursor() + if not self.get(): + sql = "INSERT INTO %(table)s(key, value, date) VALUES('%(key)s', '%(value)s', '%(date)s')" \ + % { "table" : "config", + "key" : self.key, + "value" : value, + "date" : time.time(), } + + else: + sql = "UPDATE %(table)s SET value='%(value)s', date='%(date)s' WHERE key='%(key)s'" \ + % { "table" : "config", + "key" : self.key, + "value" : value, + "date" : time.time(), } + c.execute(sql) + c.close() + self.data = value + self.db.commit() + +class DurationsConfig: + def __init__(self, db): + self.db = db + + def get(self, sort=0): + c = self.db.cursor() + c.execute("SELECT duration FROM durations") + ret = c.fetchall() + c.close() + if sort: ret.sort() + return ret + + def set(self, value): + #value = (value,) + c = self.db.cursor() + c.execute("INSERT INTO %(table)s(duration) VALUES('%(value)s')" \ + % { "table" : "durations", + "value" : value, }) + c.close() + self.db.commit() + + def get_avg(self): + sum = 0 + durations = self.get() + if not len(durations): + return None + for value in durations: + sum += int("%s" % value) + avg = sum / len(durations) + return avg + + def get_eta(self, timestamp): + avg = self.get_avg() + if not avg: + return "N/A" + eta = int(timestamp) + avg + return time.ctime(eta) + +class DistccConfig(DatabaseConfig): + def __init__(self, db, key, hostname): + DatabaseConfig.__init__(self, db, key) + self.hostname = hostname + + def __str__(self): + if not self.hostname: + return "" + return "%s:%s/4,lzo" % \ + (socket.gethostbyname(self.hostname), self.get(),) + + def ping(self): + if not self.hostname: + return False + return not os.system("ping -c1 -w1 %s &>/dev/null" % self.hostname) + + def version(self): + return os.popen("distcc --version").readlines() + +class Builder: + def __init__(self, config, uuid): + self.uuid = uuid + self.config = config + self.path = os.path.join(self.config['path']['db'], self.uuid) + + if not os.access(self.path, os.R_OK): + try: + os.mkdir(self.path) + except: + pass + + self.db = Database(self.path) + + self.hostname = DatabaseConfig(self.db, "hostname") + self.state = DatabaseConfig(self.db, "state") + self.package = DatabaseConfig(self.db, "package") + + self.duration = DurationsConfig(self.db) + self.distcc = DistccConfig(self.db, "distcc", self.hostname.get()) + + def set(self, key, value): + eval("self.%s.set(\"%s\")" % (key, value,)) + + def get(self, key): + return eval("self.%s.get()" % (key,)) + +def getAllBuilders(): + builders = [] + for uuid in os.listdir(config["path"]["db"]): + if uuid == "empty.txt": continue + builder = Builder(config, uuid) + builders.append(builder) + return builders diff --git a/build/config.py b/build/config.py new file mode 100644 index 00000000..74064ef3 --- /dev/null +++ b/build/config.py @@ -0,0 +1,13 @@ +#!/usr/bin/python + +import os +import time + +config = { + "title" : "IPFire - Builder", + "nightly_url" : "ftp://ftp.ipfire.org/pub/nightly-builds", + "path" : { "db" : "db", }, + "script" : os.environ['SCRIPT_NAME'], + "time" : time.ctime(), + "db_name" : "builder.db", +} diff --git a/build/constants.py b/build/constants.py new file mode 100644 index 00000000..49060a90 --- /dev/null +++ b/build/constants.py @@ -0,0 +1,51 @@ +#!/usr/bin/python +############################################################################### +# # +# IPFire.org - A linux based firewall # +# Copyright (C) 2008 Michael Tremer & Christian Schmidt # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +import os +import time + +config = { + "title" : "IPFire - Builder", + "nightly_url" : ("ftp://ftp.ipfire.org/pub/nightly-builds/", "http://www.rowie.at/ipfire/builds/",), + "path" : { "db" : "db", }, + "script" : os.environ['SCRIPT_NAME'], + "db_name" : "builder.db", +} + +statedesc = { + None : "", + "unknown" : "Dunno what the host is doing at the moment...", + "compiling" : "The host is really hard working at the moment...", + "error" : "Oops! The host had an error...", + "idle" : "The host is idle at the moment...", +} + +ping2class = { + True : "online", + False : "offline", +} + +state2style = { + None : "", + "compiling" : "background: #8C8; border: 1px solid #0e0;", + "error" : "background: #c33; border: 1px solid #e00;", + "idle" : "height: 60px; background: #ddd; border: 1px solid #eee;", +} diff --git a/build/distcc.py b/build/distcc.py deleted file mode 100644 index 5373e5ab..00000000 --- a/build/distcc.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/python - -import cgitb, os, time, cgi, re, random, socket, DNS -cgitb.enable() - -def print_http_header(): - print "Content-type: text/html" - print - -def print_header(): - print_http_header() - print ''' - - - IPFire - DistccWatch - - - - -

IPFire DistccWatch

Made: %s

''' % (os.environ['SCRIPT_NAME'], time.ctime()) - -def print_footer(): - print "\t\n" - -def print_dcc_info(): - print "

",
-	print os.popen("distcc --version").read(),
-	print "

" - -def read_hosts(): - f = open("hosts") - hosts = [] - while True: - line = f.readline() - if len(line) == 0: - hosts.sort() - return hosts - break #EOF - - if not line.startswith("#"): - hosts.append(line.rstrip("\n")) - -def process_hosts(hosts=""): - if not hosts: - print "You need to specify the hosts you want to check in the config file." - return - - print ''' - - - - - ''' - - for hostname in hosts: - - state = "DOWN" - time = " " - command = "HOME=tmp DISTCC_HOSTS=\"%s\" DISTCC_VERBOSE=1 distcc probe.c -S -o /dev/null 2>&1" % hostname - state_pattern = re.compile("on %s completed ok" % hostname) - time_pattern = re.compile("elapsed compilation time ") - - for line in os.popen(command).readlines(): - state_result = re.search(state_pattern, line) - time_result = re.search(time_pattern, line) - if not state_result == None: - state = "UP" - if not time_result == None: - if state == "UP": - time = time_result.string[-10:] - - print "" % (hostname, state, time) - - print "

Host	State	Probe
%s	%s	%s

" - -def mixup_hosts(hosts=""): - string = "" - while True: - if len(hosts) == 0: - break - - rand = random.randint(0, len(hosts)-1) - host, options = re.split(re.compile(':'),hosts[rand]) - string += socket.gethostbyname(host) + ":" + options + " " - hosts.pop(rand) - - print string - -form = cgi.FieldStorage() -action = form.getfirst('action') - -if action == "raw": - print_http_header() - - hosts = read_hosts() - - mixup_hosts(hosts) - -else: - - print_header() - print_dcc_info() - - hosts = read_hosts() - - process_hosts(hosts) - - print_footer() diff --git a/build/hosts b/build/hosts deleted file mode 100644 index 5f25c816..00000000 --- a/build/hosts +++ /dev/null @@ -1,5 +0,0 @@ -#mirror1.ipfire.org:3632,lzo -#mirror2.ipfire.org:3632,lzo -#mirror3.ipfire.org:3632,lzo -compiler02.ipfire.org:3630/10,lzo -compiler02.ipfire.org:3631/10,lzo diff --git a/build/images/compiling.png b/build/images/compiling.png deleted file mode 100644 index 903a27ed..00000000 Binary files a/build/images/compiling.png and /dev/null differ diff --git a/build/images/error.png b/build/images/error.png deleted file mode 100644 index 9d67d8a8..00000000 Binary files a/build/images/error.png and /dev/null differ diff --git a/build/images/idle.png b/build/images/idle.png deleted file mode 100644 index 8df47a5b..00000000 Binary files a/build/images/idle.png and /dev/null differ diff --git a/build/images/unknown.png b/build/images/unknown.png deleted file mode 100644 index 61400e27..00000000 Binary files a/build/images/unknown.png and /dev/null differ diff --git a/build/index.py b/build/index.py new file mode 100644 index 00000000..9ef8c435 --- /dev/null +++ b/build/index.py @@ -0,0 +1,233 @@ +#!/usr/bin/python +############################################################################### +# # +# IPFire.org - A linux based firewall # +# Copyright (C) 2008 Michael Tremer & Christian Schmidt # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +import os +import sys +import time + +sys.path.append(".") + +from builder import Builder, getAllBuilders +from constants import * + +def format_time(seconds, use_hours=1): + if seconds is None or seconds < 0: + if use_hours: return '--:--:--' + else: return '--:--' + else: + seconds = int(seconds) + minutes = seconds / 60 + seconds = seconds % 60 + if use_hours: + hours = minutes / 60 + minutes = minutes % 60 + return '%02i:%02i:%02i' % (hours, minutes, seconds) + else: + return '%02i:%02i' % (minutes, seconds) + +class Site: + def __init__(self, config): + self.builders = None + self.config = config + print "Content-type: text/html" + print + + def __call__(self, builders=None): + self.builders = builders + print """ + + + %(title)s + + + + + """ + + self.content() + + print "\t\t\n" + + def content(self): + if self.builders: + count = 0 + print """\ +

""" + for builder in self.builders: + builder(count) + count += 1 + print """\ +

""" + +class Box: + def __init__(self, builder): + self.builder = builder + + def __call__(self, count): + print """\ + +

"""\ + % (self.builder.hostname(), state2style[self.builder.state()],) + print """\ +

""" + self.distccinfo() + self.package() + self.time() + print """\ +

""" + self.header() + self.stateinfo() + self.durations() + print """\ +

""" + + def __str__(self): + return """%(hostname)s""" % { "hostname" : self.builder.hostname(), } + + def header(self): + print """\ +

+ %(hostname)s [%(uuid)s] +

+ """ \ + % { "hostname" : self.builder.hostname(), + "state" : self.builder.state(), + "uuid" : self.builder.uuid, } + + def package(self): + if self.builder.state() in [ "compiling", "error", ]: + print """\ +

"""\ + % self.builder.package() + + def time(self): + print """\ +

""" \ + % time.ctime(float(self.builder.state.time())) + + def stateinfo(self): + if self.builder.state() in [ "compiling", "error", "idle", ]: + print """\ +

""" \ + % statedesc[self.builder.state()] + + def durations(self): + print """\ +

Average Build Duration: %s

""" \ + % format_time(self.builder.duration.get_avg()) + if self.builder.state() == "compiling": + print """\ +

ETA: %s

""" \ + % self.builder.duration.get_eta(self.builder.state.time()) + + def distccinfo(self): + state = self.builder.distcc.ping() + print """\ +

Distcc: %s

""" \ + % (ping2class[state], self.builder.distcc.get(),) + +site = Site(config) + +boxes = [] +for builder in getAllBuilders(): + boxes.append(Box(builder)) + +site(boxes) diff --git a/build/probe.c b/build/probe.c deleted file mode 100644 index dbec9109..00000000 --- a/build/probe.c +++ /dev/null @@ -1,3 +0,0 @@ -int main () { - return 0; -} diff --git a/build/rpc.py b/build/rpc.py new file mode 100644 index 00000000..bfcf44d8 --- /dev/null +++ b/build/rpc.py @@ -0,0 +1,93 @@ +#!/usr/bin/python +############################################################################### +# # +# IPFire.org - A linux based firewall # +# Copyright (C) 2008 Michael Tremer & Christian Schmidt # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +import os +import sys +import cgi +import time +import random + +sys.path.append(".") + +from builder import Builder, getAllBuilders +from constants import config + +class Response: + def __init__(self, config): + self.config = config + + self.code = "200" + self.mesg = "OK" + + def __call__(self, exit=0): + print "Status: %s" % self.code + print "Content-type: text/plain" + print + print "%s" % self.mesg + if exit: + os._exit(0) + + def set_code(self, code): + self.code = code + + def set_mesg(self, mesg): + self.mesg = mesg + +response = Response(config) + +data = cgi.FieldStorage() + +action = data.getvalue('action') +if action == "set": + uuid = data.getfirst("uuid") + if not uuid: + response.set_code("406") + response.set_mesg("UUID is not valid!") + response(1) + + builder = Builder(config, uuid) + + key = None + for key in [ "distcc", "duration", "hostname", "state", "package", ]: + for value in data.getlist(key): + builder.set(key, value) +elif action == "get": + for key in [ "distcc", ]: + if key == "distcc": + for value in data.getlist(key): + if value == "raw": + builders = [] + for builder in getAllBuilders(): + builders.append("%s" % builder.distcc) + string = "" + while True: + if not builders: break + rand = random.randint(0, len(builders)-1) + string += "%s\n" % builders[rand] + builders.pop(rand) + response.set_mesg(string) + +else: + response.set_code("501") + response.set_mesg("Don't know what to do with command \"%s\"" % action) + response(1) + +response() diff --git a/build/tmp/empty.txt b/build/tmp/empty.txt deleted file mode 100644 index e69de29b..00000000