build/builder.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # IPFire.org - A linux based firewall                                         #
   5 # Copyright (C) 2008  Michael Tremer & Christian Schmidt                      #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import os
  23 import sys
  24 import time
  25 import socket
  26 import base64
  27 import shutil
  28 from pysqlite2 import dbapi2 as sqlite
  29
  30 sys.path.append(".")
  31
  32 from constants import config
  33
  34 class Database:
  35         def __init__(self, path):
  36                 self.db = sqlite.connect(os.path.join(path, config["db_name"]))
  37                 c = self.cursor()
  38                 c.executescript("""
  39                         create table if not exists config(key, value, date);
  40                         create table if not exists durations(duration);
  41                 """)
  42                 c.close()
  43
  44         def __call__(self):
  45                 return self.cursor()
  46
  47         def __del__(self):
  48                 self.commit()
  49                 self.db.close()
  50
  51         def cursor(self):
  52                 return self.db.cursor()
  53
  54         def commit(self):
  55                 self.db.commit()
  56
  57 class DatabaseConfig:
  58         def __init__(self, db, key, base64=0):
  59                 self.db = db
  60                 self.key = key
  61                 self.data = None
  62                 self.date = None
  63                 self.base64 = base64
  64
  65         def get(self):
  66                 if not self.data:
  67                         c = self.db.cursor()
  68                         c.execute("SELECT value FROM %(table)s WHERE key = '%(key)s'" \
  69                                                 % { "table" : "config",
  70                                                         "key"   : self.key, })
  71                         try:
  72                                 self.data = c.fetchone()[0]
  73                         except TypeError:
  74                                 self.data = None
  75                         c.close()
  76                 return self.data
  77
  78         __call__ = get
  79
  80         def time(self):
  81                 if not self.date:
  82                         c = self.db.cursor()
  83                         c.execute("SELECT date FROM %(table)s WHERE key = '%(key)s'" \
  84                                                 % { "table" : "config",
  85                                                         "key"   : self.key, })
  86                         try:
  87                                 self.date = float("%s" % c.fetchone()[0])
  88                         except TypeError:
  89                                 self.date = None
  90                         c.close()
  91                 return self.date or float(0)
  92
  93         def set(self, value):
  94                 if self.base64:
  95                         value = base64.b64decode(value)
  96                 #value = (value,)
  97                 c = self.db.cursor()
  98                 if not self.get():
  99                         sql = "INSERT INTO %(table)s(key, value, date) VALUES('%(key)s', '%(value)s', '%(date)s')" \
 100                                                 % { "table" : "config",
 101                                                         "key"   : self.key,
 102                                                         "value" : value,
 103                                                         "date"  : time.time(), }
 104
 105                 else:
 106                         sql = "UPDATE %(table)s SET value='%(value)s', date='%(date)s' WHERE key='%(key)s'" \
 107                                                 % { "table" : "config",
 108                                                         "key"   : self.key,
 109                                                         "value" : value,
 110                                                         "date"  : time.time(), }
 111                 c.execute(sql)
 112                 c.close()
 113                 self.data = value
 114                 self.db.commit()
 115                 return """Set "%s" to "%s".""" % (self.key, self.data,)
 116
 117 class DurationsConfig:
 118         def __init__(self, db):
 119                 self.db = db
 120
 121         def get(self, sort=0):
 122                 c = self.db.cursor()
 123                 c.execute("SELECT duration FROM durations")
 124                 ret = []
 125                 for value in c.fetchall():
 126                         value = int("%s" % value)
 127                         if value < 900: # 15min
 128                                 continue
 129                         ret.append(value)
 130                 c.close()
 131                 if sort: ret.sort()
 132                 return ret
 133
 134         def set(self, value):
 135                 #value = (value,)
 136                 c = self.db.cursor()
 137                 c.execute("INSERT INTO %(table)s(duration) VALUES('%(value)s')" \
 138                                         % { "table" : "durations",
 139                                                 "value" : value, })
 140                 c.close()
 141                 self.db.commit()
 142                 return """Accepted build duration of %s seconds.""" % (value,)
 143
 144         def get_avg(self):
 145                 sum = 0
 146                 durations = self.get()
 147                 if not len(durations):
 148                         return None
 149                 for value in durations:
 150                         sum += value
 151                 avg = sum / len(durations)
 152                 return avg
 153
 154         def get_eta(self, timestamp):
 155                 avg = self.get_avg()
 156                 if not avg:
 157                         return "N/A"
 158                 eta = int(timestamp) + avg
 159                 return time.ctime(eta)
 160
 161 class DistccConfig(DatabaseConfig):
 162         def __init__(self, db, key, hostname, jobs):
 163                 DatabaseConfig.__init__(self, db, key)
 164                 self.hostname = hostname
 165                 self.jobs = jobs
 166
 167         def __str__(self):
 168                 if not self.ping() or self.get() == "0":
 169                         return ""
 170                 return "%s:%s/%s,lzo" % \
 171                         (self.hostname, self.get(), self.jobs or "4",)
 172
 173         def ping(self):
 174                 if not self.hostname:
 175                         return False
 176                 return not os.system("ping -c1 -w1 %s &>/dev/null" % self.hostname)
 177
 178         def version(self):
 179                 return os.popen("distcc --version").readlines()
 180
 181 class FileConfig:
 182         def __init__(self, path, filetype):
 183                 self.filename = os.path.join(path, config["path"][filetype])
 184
 185                 # Create the file if not existant
 186                 if not os.access(self.filename, os.R_OK):
 187                         f = open(self.filename, "w")
 188                         f.close()
 189
 190         def get(self):
 191                 ret = []
 192                 try:
 193                         f = open(self.filename)
 194                         ret = f.readlines()
 195                         f.close()
 196                 except:
 197                         pass
 198                 return ret or ["Log is empty."]
 199
 200         __call__ = get
 201
 202         def set(self, lines):
 203                 f = open(self.filename, "w")
 204                 for line in base64.b64decode(lines).split("\n"):
 205                         f.write("%s\n" % line.rstrip("\n"))
 206                 f.close()
 207                 return """Saved file content to %s.""" % (self.filename,)
 208
 209 class Builder:
 210         def __init__(self, config, uuid):
 211                 self.uuid = uuid
 212                 self.config = config
 213                 self.path = os.path.join(self.config['path']['db'], self.uuid)
 214
 215                 if not os.access(self.path, os.R_OK):
 216                         try:
 217                                 os.mkdir(self.path)
 218                         except:
 219                                 pass
 220
 221                 self.db = Database(self.path)
 222
 223                 self.hostname = DatabaseConfig(self.db, "hostname")
 224                 self.state    = DatabaseConfig(self.db, "state")
 225                 self.package  = DatabaseConfig(self.db, "package")
 226                 self.target   = DatabaseConfig(self.db, "target")
 227                 self.toolchain= DatabaseConfig(self.db, "toolchain")
 228
 229                 self.duration = DurationsConfig(self.db)
 230                 self.jobs     = DatabaseConfig(self.db, "jobs")
 231                 self.distcc   = DistccConfig(self.db, "distcc", self.hostname(), self.jobs())
 232                 self.cpu      = DatabaseConfig(self.db, "cpu", base64=1)
 233                 self.machine  = DatabaseConfig(self.db, "machine")
 234                 self.system   = DatabaseConfig(self.db, "system", base64=1)
 235
 236                 self.log      = FileConfig(self.path, "log")
 237
 238                 # If host was longer than 3 days in state compiling we set it as idle.
 239                 if self.state() == "compiling" and \
 240                                 (time.time() - self.state.time()) > 3*24*60*60:
 241                         self.state.set("idle")
 242
 243                 # If host is idle and distcc is not disabled we set it as distcc host.
 244                 if self.state() == "idle" and self.distcc() != "0":
 245                         self.state.set("distcc")
 246
 247                 # If host is longer than 24h in error state we set it as distcc host.
 248                 if self.state() == "error" and \
 249                                 (time.time() - self.state.time()) > 24*60*60:
 250                         self.state.set("distcc")
 251
 252                 # If host was longer than two weeks in distcc state we set it as unknown.
 253                 if self.state() == "error" and \
 254                                 (time.time() - self.state.time()) > 2*7*24*60*60:
 255                         self.state.set("unknown")
 256
 257                 # If host was longer than four weels in distcc state we delete it.
 258                 if self.state() in ("distcc", "unknown",) and \
 259                                 (time.time() - self.state.time()) > 4*7*24*60*60:
 260                         del self.db
 261                         shutil.rmtree(self.path)
 262
 263         def set(self, key, value):
 264                 return eval("self.%s.set(\"%s\")" % (key, value,))
 265
 266         def get(self, key):
 267                 return eval("self.%s.get()" % (key,))
 268
 269 def getAllBuilders(age=0):
 270         builders = []
 271         for uuid in os.listdir(config["path"]["db"]):
 272                 if uuid == "empty.txt": continue
 273                 builder = Builder(config, uuid)
 274                 # If there was no activity since "age" days -> continue...
 275                 if age and (time.time() - builder.state.time()) > age*24*60*60:
 276                                 continue
 277                 builders.append(builder)
 278         return builders