build/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # IPFire.org - A linux based firewall                                         #
   5 # Copyright (C) 2008  Michael Tremer & Christian Schmidt                      #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import os
  23 import sys
  24 import time
  25 import socket
  26 import base64
  27 import shutil
  28 from pysqlite2 import dbapi2 as sqlite
  29
  30 sys.path.append(".")
  31
  32 from constants import config
  33
  34 class Database:
  35         def __init__(self, path):
  36                 self.db = sqlite.connect(os.path.join(path, config["db_name"]))
  37                 c = self.cursor()
  38                 c.executescript("""
  39                         create table if not exists config(key, value, date);
  40                         create table if not exists durations(duration);
  41                 """)
  42                 c.close()
  43
  44         def __call__(self):
  45                 return self.cursor()
  46
  47         def __del__(self):
  48                 self.commit()
  49                 self.db.close()
  50
  51         def cursor(self):
  52                 return self.db.cursor()
  53
  54         def commit(self):
  55                 self.db.commit()
  56
  57 class DatabaseConfig:
  58         def __init__(self, db, key, base64=0):
  59                 self.db = db
  60                 self.key = key
  61                 self.data = None
  62                 self.date = None
  63                 self.base64 = base64
  64
  65         def get(self):
  66                 if not self.data:
  67                         c = self.db.cursor()
  68                         c.execute("SELECT value FROM %(table)s WHERE key = '%(key)s'" \
  69                                                 % { "table" : "config",
  70                                                         "key"   : self.key, })
  71                         try:
  72                                 self.data = c.fetchone()[0]
  73                         except TypeError:
  74                                 self.data = None
  75                         c.close()
  76                 return self.data
  77
  78         __call__ = get
  79
  80         def time(self):
  81                 if not self.date:
  82                         c = self.db.cursor()
  83                         c.execute("SELECT date FROM %(table)s WHERE key = '%(key)s'" \
  84                                                 % { "table" : "config",
  85                                                         "key"   : self.key, })
  86                         try:
  87                                 self.date = float("%s" % c.fetchone()[0])
  88                         except TypeError:
  89                                 self.date = None
  90                         c.close()
  91                 return self.date or float(0)
  92
  93         def set(self, value):
  94                 if self.base64:
  95                         value = base64.b64decode(value)
  96                 #value = (value,)
  97                 c = self.db.cursor()
  98                 if not self.get():
  99                         sql = "INSERT INTO %(table)s(key, value, date) VALUES('%(key)s', '%(value)s', '%(date)s')" \
 100                                                 % { "table" : "config",
 101                                                         "key"   : self.key,
 102                                                         "value" : value,
 103                                                         "date"  : time.time(), }
 104
 105                 else:
 106                         sql = "UPDATE %(table)s SET value='%(value)s', date='%(date)s' WHERE key='%(key)s'" \
 107                                                 % { "table" : "config",
 108                                                         "key"   : self.key,
 109                                                         "value" : value,
 110                                                         "date"  : time.time(), }
 111                 c.execute(sql)
 112                 c.close()
 113                 self.data = value
 114                 self.db.commit()
 115
 116 class DurationsConfig:
 117         def __init__(self, db):
 118                 self.db = db
 119
 120         def get(self, sort=0):
 121                 c = self.db.cursor()
 122                 c.execute("SELECT duration FROM durations")
 123                 ret = []
 124                 for value in c.fetchall():
 125                         value = int("%s" % value)
 126                         if value < 900: # 15min
 127                                 continue
 128                         ret.append(value)
 129                 c.close()
 130                 if sort: ret.sort()
 131                 return ret
 132
 133         def set(self, value):
 134                 #value = (value,)
 135                 c = self.db.cursor()
 136                 c.execute("INSERT INTO %(table)s(duration) VALUES('%(value)s')" \
 137                                         % { "table" : "durations",
 138                                                 "value" : value, })
 139                 c.close()
 140                 self.db.commit()
 141
 142         def get_avg(self):
 143                 sum = 0
 144                 durations = self.get()
 145                 if not len(durations):
 146                         return None
 147                 for value in durations:
 148                         sum += value
 149                 avg = sum / len(durations)
 150                 return avg
 151
 152         def get_eta(self, timestamp):
 153                 avg = self.get_avg()
 154                 if not avg:
 155                         return "N/A"
 156                 eta = int(timestamp) + avg
 157                 return time.ctime(eta)
 158
 159 class DistccConfig(DatabaseConfig):
 160         def __init__(self, db, key, hostname, jobs):
 161                 DatabaseConfig.__init__(self, db, key)
 162                 self.hostname = hostname
 163                 self.jobs = jobs
 164
 165         def __str__(self):
 166                 if not self.ping() or self.get() == "0":
 167                         return ""
 168                 return "%s:%s/%s,lzo   \t# %s" % \
 169                         (socket.gethostbyname(self.hostname), self.get(), self.jobs or "4", self.hostname)
 170
 171         def ping(self):
 172                 if not self.hostname:
 173                         return False
 174                 return not os.system("ping -c1 -w1 %s &>/dev/null" % self.hostname)
 175
 176         def version(self):
 177                 return os.popen("distcc --version").readlines()
 178
 179 class FileConfig:
 180         def __init__(self, path, filetype):
 181                 self.filename = os.path.join(path, config["path"][filetype])
 182
 183                 # Create the file if not existant
 184                 if not os.access(self.filename, os.R_OK):
 185                         f = open(self.filename, "w")
 186                         f.close()
 187
 188         def get(self):
 189                 ret = []
 190                 try:
 191                         f = open(self.filename)
 192                         ret = f.readlines()
 193                         f.close()
 194                 except:
 195                         pass
 196                 return ret or ["Log is empty."]
 197
 198         __call__ = get
 199
 200         def set(self, lines):
 201                 f = open(self.filename, "w")
 202                 for line in base64.b64decode(lines).split("\n"):
 203                         f.write("%s\n" % line.rstrip("\n"))
 204                 f.close()
 205
 206 class Builder:
 207         def __init__(self, config, uuid):
 208                 self.uuid = uuid
 209                 self.config = config
 210                 self.path = os.path.join(self.config['path']['db'], self.uuid)
 211
 212                 if not os.access(self.path, os.R_OK):
 213                         try:
 214                                 os.mkdir(self.path)
 215                         except:
 216                                 pass
 217
 218                 self.db = Database(self.path)
 219
 220                 self.hostname = DatabaseConfig(self.db, "hostname")
 221                 self.state    = DatabaseConfig(self.db, "state")
 222                 self.package  = DatabaseConfig(self.db, "package")
 223                 self.target   = DatabaseConfig(self.db, "target")
 224
 225                 self.duration = DurationsConfig(self.db)
 226                 self.jobs     = DatabaseConfig(self.db, "jobs")
 227                 self.distcc   = DistccConfig(self.db, "distcc", self.hostname(), self.jobs())
 228                 self.cpu      = DatabaseConfig(self.db, "cpu", base64=1)
 229                 self.machine  = DatabaseConfig(self.db, "machine")
 230
 231                 self.log      = FileConfig(self.path, "log")
 232
 233                 # If host was longer than 3 days in state compiling we set it as idle.
 234                 if self.state() == "compiling" and \
 235                                 (time.time() - self.state.time()) > 3*24*60*60:
 236                         self.state.set("idle")
 237
 238                 # If host is idle and distcc is not disabled we set it as distcc host.
 239                 if self.state() == "idle" and self.distcc() != "0":
 240                         self.state.set("distcc")
 241
 242                 # If host is longer than 24h in error state we set it as distcc host.
 243                 if self.state() == "error" and \
 244                                 (time.time() - self.state.time()) > 24*60*60:
 245                         self.state.set("distcc")
 246
 247                 # If host was longer than two weeks in distcc state we set it as unknown.
 248                 if self.state() == "error" and \
 249                                 (time.time() - self.state.time()) > 2*7*24*60*60:
 250                         self.state.set("unknown")
 251
 252                 # If host was longer than four weels in distcc state we delete it.
 253                 if self.state() == "unknown" and \
 254                                 (time.time() - self.state.time()) > 4*7*24*60*60:
 255                         del self.db
 256                         shutil.rmtree(self.path)
 257
 258         def set(self, key, value):
 259                 eval("self.%s.set(\"%s\")" % (key, value,))
 260
 261         def get(self, key):
 262                 return eval("self.%s.get()" % (key,))
 263
 264 def getAllBuilders(age=0):
 265         builders = []
 266         for uuid in os.listdir(config["path"]["db"]):
 267                 if uuid == "empty.txt": continue
 268                 builder = Builder(config, uuid)
 269                 # If there was no activity since "age" days -> continue...
 270                 if age and (time.time() - builder.state.time()) > age*24*60*60:
 271                                 continue
 272                 builders.append(builder)
 273         return builders