build/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # IPFire.org - A linux based firewall                                         #
   5 # Copyright (C) 2008  Michael Tremer & Christian Schmidt                      #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import os
  23 import sys
  24 import time
  25 import socket
  26 import base64
  27 import shutil
  28 from pysqlite2 import dbapi2 as sqlite
  29
  30 sys.path.append(".")
  31
  32 from constants import config
  33
  34 class Database:
  35         def __init__(self, path):
  36                 self.db = sqlite.connect(os.path.join(path, config["db_name"]))
  37                 c = self.cursor()
  38                 c.executescript("""
  39                         create table if not exists config(key, value, date);
  40                         create table if not exists durations(duration);
  41                 """)
  42                 c.close()
  43
  44         def __call__(self):
  45                 return self.cursor()
  46
  47         def __del__(self):
  48                 self.commit()
  49                 self.db.close()
  50
  51         def cursor(self):
  52                 return self.db.cursor()
  53
  54         def commit(self):
  55                 self.db.commit()
  56
  57 class DatabaseConfig:
  58         def __init__(self, db, key):
  59                 self.db = db
  60                 self.key = key
  61                 self.data = None
  62                 self.date = None
  63
  64         def get(self):
  65                 if not self.data:
  66                         c = self.db.cursor()
  67                         c.execute("SELECT value FROM %(table)s WHERE key = '%(key)s'" \
  68                                                 % { "table" : "config",
  69                                                         "key"   : self.key, })
  70                         try:
  71                                 self.data = c.fetchone()[0]
  72                         except TypeError:
  73                                 self.data = None
  74                         c.close()
  75                 return self.data
  76
  77         __call__ = get
  78
  79         def time(self):
  80                 if not self.date:
  81                         c = self.db.cursor()
  82                         c.execute("SELECT date FROM %(table)s WHERE key = '%(key)s'" \
  83                                                 % { "table" : "config",
  84                                                         "key"   : self.key, })
  85                         try:
  86                                 self.date = float("%s" % c.fetchone()[0])
  87                         except TypeError:
  88                                 self.date = None
  89                         c.close()
  90                 return self.date or float(0)
  91
  92         def set(self, value):
  93                 #value = (value,)
  94                 c = self.db.cursor()
  95                 if not self.get():
  96                         sql = "INSERT INTO %(table)s(key, value, date) VALUES('%(key)s', '%(value)s', '%(date)s')" \
  97                                                 % { "table" : "config",
  98                                                         "key"   : self.key,
  99                                                         "value" : value,
 100                                                         "date"  : time.time(), }
 101
 102                 else:
 103                         sql = "UPDATE %(table)s SET value='%(value)s', date='%(date)s' WHERE key='%(key)s'" \
 104                                                 % { "table" : "config",
 105                                                         "key"   : self.key,
 106                                                         "value" : value,
 107                                                         "date"  : time.time(), }
 108                 c.execute(sql)
 109                 c.close()
 110                 self.data = value
 111                 self.db.commit()
 112
 113 class DurationsConfig:
 114         def __init__(self, db):
 115                 self.db = db
 116
 117         def get(self, sort=0):
 118                 c = self.db.cursor()
 119                 c.execute("SELECT duration FROM durations")
 120                 ret = []
 121                 for value in c.fetchall():
 122                         value = int("%s" % value)
 123                         if value < 900: # 15min
 124                                 continue
 125                         ret.append(value)
 126                 c.close()
 127                 if sort: ret.sort()
 128                 return ret
 129
 130         def set(self, value):
 131                 #value = (value,)
 132                 c = self.db.cursor()
 133                 c.execute("INSERT INTO %(table)s(duration) VALUES('%(value)s')" \
 134                                         % { "table" : "durations",
 135                                                 "value" : value, })
 136                 c.close()
 137                 self.db.commit()
 138
 139         def get_avg(self):
 140                 sum = 0
 141                 durations = self.get()
 142                 if not len(durations):
 143                         return None
 144                 for value in durations:
 145                         sum += value
 146                 avg = sum / len(durations)
 147                 return avg
 148
 149         def get_eta(self, timestamp):
 150                 avg = self.get_avg()
 151                 if not avg:
 152                         return "N/A"
 153                 eta = int(timestamp) + avg
 154                 return time.ctime(eta)
 155
 156 class DistccConfig(DatabaseConfig):
 157         def __init__(self, db, key, hostname, jobs):
 158                 DatabaseConfig.__init__(self, db, key)
 159                 self.hostname = hostname
 160                 self.jobs = jobs
 161
 162         def __str__(self):
 163                 if not self.ping() or self.get() == "0":
 164                         return ""
 165                 return "%s:%s/%s,lzo   \t# %s" % \
 166                         (socket.gethostbyname(self.hostname), self.get(), self.jobs or "4", self.hostname)
 167
 168         def ping(self):
 169                 if not self.hostname:
 170                         return False
 171                 return not os.system("ping -c1 -w1 %s &>/dev/null" % self.hostname)
 172
 173         def version(self):
 174                 return os.popen("distcc --version").readlines()
 175
 176 class FileConfig:
 177         def __init__(self, path, filetype):
 178                 self.filename = os.path.join(path, config["path"][filetype])
 179
 180                 # Create the file if not existant
 181                 if not os.access(self.filename, os.R_OK):
 182                         f = open(self.filename, "w")
 183                         f.close()
 184
 185         def get(self):
 186                 ret = []
 187                 try:
 188                         f = open(self.filename)
 189                         ret = f.readlines()
 190                         f.close()
 191                 except:
 192                         pass
 193                 return ret or ["Log is empty."]
 194
 195         __call__ = get
 196
 197         def set(self, lines):
 198                 f = open(self.filename, "w")
 199                 for line in base64.b64decode(lines).split("\n"):
 200                         f.write("%s\n" % line.rstrip("\n"))
 201                 f.close()
 202
 203 class Builder:
 204         def __init__(self, config, uuid):
 205                 self.uuid = uuid
 206                 self.config = config
 207                 self.path = os.path.join(self.config['path']['db'], self.uuid)
 208
 209                 if not os.access(self.path, os.R_OK):
 210                         try:
 211                                 os.mkdir(self.path)
 212                         except:
 213                                 pass
 214
 215                 self.db = Database(self.path)
 216
 217                 self.hostname = DatabaseConfig(self.db, "hostname")
 218                 self.state    = DatabaseConfig(self.db, "state")
 219                 self.package  = DatabaseConfig(self.db, "package")
 220                 self.target   = DatabaseConfig(self.db, "target")
 221
 222                 self.duration = DurationsConfig(self.db)
 223                 self.jobs     = DatabaseConfig(self.db, "jobs")
 224                 self.distcc   = DistccConfig(self.db, "distcc", self.hostname(), self.jobs())
 225
 226                 self.log      = FileConfig(self.path, "log")
 227
 228                 # If host was longer than 3 days in state compiling we set it as idle.
 229                 if self.state() == "compiling" and \
 230                                 (time.time() - self.state.time()) > 3*24*60*60:
 231                         self.state.set("idle")
 232
 233                 # If host is idle and distcc is not disabled we set it as distcc host.
 234                 if self.state() == "idle" and self.distcc() != "0":
 235                         self.state.set("distcc")
 236
 237                 # If host is longer than 24h in error state we set it as distcc host.
 238                 if self.state() == "error" and \
 239                                 (time.time() - self.state.time()) > 24*60*60:
 240                         self.state.set("distcc")
 241
 242                 # If host was longer than two weeks in distcc state we set it as unknown.
 243                 if self.state() == "error" and \
 244                                 (time.time() - self.state.time()) > 2*7*24*60*60:
 245                         self.state.set("unknown")
 246
 247                 # If host was longer than four weels in distcc state we delete it.
 248                 if self.state() == "unknown" and \
 249                                 (time.time() - self.state.time()) > 4*7*24*60*60:
 250                         del self.db
 251                         shutil.rmtree(self.path)
 252
 253         def set(self, key, value):
 254                 eval("self.%s.set(\"%s\")" % (key, value,))
 255
 256         def get(self, key):
 257                 return eval("self.%s.get()" % (key,))
 258
 259 def getAllBuilders(age=0):
 260         builders = []
 261         for uuid in os.listdir(config["path"]["db"]):
 262                 if uuid == "empty.txt": continue
 263                 builder = Builder(config, uuid)
 264                 # If there was no activity since "age" days -> continue...
 265                 if age and (time.time() - builder.state.time()) > age*24*60*60:
 266                                 continue
 267                 builders.append(builder)
 268         return builders