src/buildservice/builders.py

   1 #!/usr/bin/python
   2
   3 from __future__ import absolute_import, division
   4
   5 import datetime
   6 import hashlib
   7 import logging
   8 import random
   9 import string
  10 import time
  11
  12 from . import base
  13 from . import logs
  14
  15 from .decorators import *
  16
  17 from .users import generate_password_hash, check_password_hash, generate_random_string
  18
  19 class Builders(base.Object):
  20         def _get_builder(self, query, *args):
  21                 res = self.db.get(query, *args)
  22
  23                 if res:
  24                         return Builder(self.backend, res.id, data=res)
  25
  26         def _get_builders(self, query, *args):
  27                 res = self.db.query(query, *args)
  28
  29                 for row in res:
  30                         yield Builder(self.backend, row.id, data=row)
  31
  32         def __iter__(self):
  33                 builders = self._get_builders("SELECT * FROM builders \
  34                         WHERE deleted IS FALSE ORDER BY name")
  35
  36                 return iter(builders)
  37
  38         def create(self, name, user=None, log=True):
  39                 """
  40                         Creates a new builder.
  41                 """
  42                 builder = self._get_builder("INSERT INTO builders(name) \
  43                         VALUES(%s) RETURNING *", name)
  44
  45                 # Generate a new passphrase.
  46                 passphrase = builder.regenerate_passphrase()
  47
  48                 # Log what we have done.
  49                 if log:
  50                         builder.log("created", user=user)
  51
  52                 # The Builder object and the passphrase are returned.
  53                 return builder, passphrase
  54
  55         def auth(self, name, passphrase):
  56                 # If either name or passphrase is None, we don't check at all.
  57                 if None in (name, passphrase):
  58                         return
  59
  60                 # Search for the hostname in the database.
  61                 builder = self._get_builder("SELECT * FROM builders \
  62                         WHERE name = %s AND deleted IS FALSE", name)
  63
  64                 # If the builder was not found or the passphrase does not match,
  65                 # you have bad luck.
  66                 if not builder or not builder.validate_passphrase(passphrase):
  67                         return
  68
  69                 # Otherwise we return the Builder object.
  70                 return builder
  71
  72         def get_by_id(self, builder_id):
  73                 return self._get_builder("SELECT * FROM builders WHERE id = %s", builder_id)
  74
  75         def get_by_name(self, name):
  76                 return self._get_builder("SELECT * FROM builders \
  77                         WHERE name = %s AND deleted IS FALSE", name)
  78
  79         def get_load(self):
  80                 res1 = self.db.get("SELECT SUM(max_jobs) AS max_jobs FROM builders \
  81                         WHERE enabled IS TRUE and deleted IS FALSE")
  82
  83                 res2 = self.db.get("SELECT COUNT(*) AS count FROM jobs \
  84                         WHERE state = 'dispatching' OR state = 'running' OR state = 'uploading'")
  85
  86                 try:
  87                         return (res2.count * 100 / res1.max_jobs)
  88                 except:
  89                         return 0
  90
  91         def get_history(self, limit=None, offset=None, builder=None, user=None):
  92                 query = "SELECT * FROM builders_history"
  93                 args  = []
  94
  95                 conditions = []
  96
  97                 if builder:
  98                         conditions.append("builder_id = %s")
  99                         args.append(builder.id)
 100
 101                 if user:
 102                         conditions.append("user_id = %s")
 103                         args.append(user.id)
 104
 105                 if conditions:
 106                         query += " WHERE %s" % " AND ".join(conditions)
 107
 108                 query += " ORDER BY time DESC"
 109
 110                 if limit:
 111                         if offset:
 112                                 query += " LIMIT %s,%s"
 113                                 args  += [offset, limit,]
 114                         else:
 115                                 query += " LIMIT %s"
 116                                 args  += [limit,]
 117
 118                 entries = []
 119                 for entry in self.db.query(query, *args):
 120                         entry = logs.BuilderLogEntry(self.pakfire, entry)
 121                         entries.append(entry)
 122
 123                 return entries
 124
 125
 126 class Builder(base.DataObject):
 127         table = "builders"
 128
 129         def __eq__(self, other):
 130                 if isinstance(other, self.__class__):
 131                         return self.id == other.id
 132
 133         def __lt__(self, other):
 134                 if isinstance(other, self.__class__):
 135                         return self.name < other.name
 136
 137         def log(self, action, user=None):
 138                 user_id = None
 139                 if user:
 140                         user_id = user.id
 141
 142                 self.db.execute("INSERT INTO builders_history(builder_id, action, user_id, time) \
 143                         VALUES(%s, %s, %s, NOW())", self.id, action, user_id)
 144
 145         def regenerate_passphrase(self):
 146                 """
 147                         Generates a new random passphrase and stores it as a salted hash
 148                         to the database.
 149
 150                         The new passphrase is returned to be sent to the user (once).
 151                 """
 152                 # Generate a random string with 40 chars.
 153                 passphrase = generate_random_string(length=40)
 154
 155                 # Create salted hash.
 156                 passphrase_hash = generate_password_hash(passphrase)
 157
 158                 # Store the hash in the database.
 159                 self._set_attribute("passphrase", passphrase_hash)
 160
 161                 # Return the clear-text passphrase.
 162                 return passphrase
 163
 164         def validate_passphrase(self, passphrase):
 165                 """
 166                         Compare the given passphrase with the one stored in the database.
 167                 """
 168                 return check_password_hash(passphrase, self.data.passphrase)
 169
 170         # Description
 171
 172         def set_description(self, description):
 173                 self._set_attribute("description", description)
 174
 175         description = property(lambda s: s.data.description or "", set_description)
 176
 177         @property
 178         def keepalive(self):
 179                 """
 180                         Returns time of last keepalive message from this host.
 181                 """
 182                 return self.data.time_keepalive
 183
 184         def update_keepalive(self, loadavg1=None, loadavg5=None, loadavg15=None,
 185                         mem_total=None, mem_free=None, swap_total=None, swap_free=None,
 186                         space_free=None):
 187                 """
 188                         Update the keepalive timestamp of this machine.
 189                 """
 190                 self.db.execute("UPDATE builders SET time_keepalive = NOW(), \
 191                         loadavg1 = %s, loadavg5 = %s, loadavg15 = %s, space_free = %s, \
 192                         mem_total = %s, mem_free = %s, swap_total = %s, swap_free = %s \
 193                         WHERE id = %s", loadavg1, loadavg5, loadavg15, space_free,
 194                         mem_total, mem_free, swap_total, swap_free, self.id)
 195
 196         def update_info(self, cpu_model=None, cpu_count=None, cpu_arch=None, cpu_bogomips=None,
 197                         pakfire_version=None, host_key=None, os_name=None):
 198                 # Update all the rest.
 199                 self.db.execute("UPDATE builders SET time_updated = NOW(), \
 200                         pakfire_version = %s, cpu_model = %s, cpu_count = %s, cpu_arch = %s, \
 201                         cpu_bogomips = %s, host_key_id = %s, os_name = %s WHERE id = %s",
 202                         pakfire_version, cpu_model, cpu_count, cpu_arch, cpu_bogomips,
 203                         host_key, os_name, self.id)
 204
 205         def set_enabled(self, enabled):
 206                 self._set_attribute("enabled", enabled)
 207
 208         enabled = property(lambda s: s.data.enabled, set_enabled)
 209
 210         @property
 211         def disabled(self):
 212                 return not self.enabled
 213
 214         @property
 215         def native_arch(self):
 216                 """
 217                         The native architecture of this builder
 218                 """
 219                 return self.cpu_arch
 220
 221         @lazy_property
 222         def supported_arches(self):
 223                 # Every builder supports noarch
 224                 arches = ["noarch"]
 225
 226                 # We can always build our native architeture
 227                 if self.native_arch:
 228                         arches.append(self.native_arch)
 229
 230                         # Get all compatible architectures
 231                         res = self.db.query("SELECT build_arch FROM arches_compat \
 232                                 WHERE native_arch = %s", self.native_arch)
 233
 234                         for row in res:
 235                                 if not row.build_arch in arches:
 236                                         arches.append(row.build_arch)
 237
 238                 return sorted(arches)
 239
 240         def get_build_release(self):
 241                 return self.data.build_release == "Y"
 242
 243         def set_build_release(self, value):
 244                 if value:
 245                         value = "Y"
 246                 else:
 247                         value = "N"
 248
 249                 self.db.execute("UPDATE builders SET build_release = %s WHERE id = %s",
 250                         value, self.id)
 251
 252                 # Update the cache.
 253                 if self._data:
 254                         self._data["build_release"] = value
 255
 256         build_release = property(get_build_release, set_build_release)
 257
 258         def get_build_scratch(self):
 259                 return self.data.build_scratch == "Y"
 260
 261         def set_build_scratch(self, value):
 262                 if value:
 263                         value = "Y"
 264                 else:
 265                         value = "N"
 266
 267                 self.db.execute("UPDATE builders SET build_scratch = %s WHERE id = %s",
 268                         value, self.id)
 269
 270                 # Update the cache.
 271                 if self._data:
 272                         self._data["build_scratch"] = value
 273
 274         build_scratch = property(get_build_scratch, set_build_scratch)
 275
 276         def get_build_test(self):
 277                 return self.data.build_test == "Y"
 278
 279         def set_build_test(self, value):
 280                 if value:
 281                         value = "Y"
 282                 else:
 283                         value = "N"
 284
 285                 self.db.execute("UPDATE builders SET build_test = %s WHERE id = %s",
 286                         value, self.id)
 287
 288                 # Update the cache.
 289                 if self._data:
 290                         self._data["build_test"] = value
 291
 292         build_test = property(get_build_test, set_build_test)
 293
 294         @property
 295         def build_types(self):
 296                 ret = []
 297
 298                 if self.build_release:
 299                         ret.append("release")
 300
 301                 if self.build_scratch:
 302                         ret.append("scratch")
 303
 304                 if self.build_test:
 305                         ret.append("test")
 306
 307                 return ret
 308
 309         def set_max_jobs(self, value):
 310                 self._set_attribute("max_jobs", value)
 311
 312         max_jobs = property(lambda s: s.data.max_jobs, set_max_jobs)
 313
 314         @property
 315         def name(self):
 316                 return self.data.name
 317
 318         @property
 319         def hostname(self):
 320                 return self.name
 321
 322         @property
 323         def passphrase(self):
 324                 return self.data.passphrase
 325
 326         # Load average
 327
 328         @property
 329         def loadavg(self):
 330                 return ", ".join(["%.2f" % l for l in (self.loadavg1, self.loadavg5, self.loadavg15)])
 331
 332         @property
 333         def loadavg1(self):
 334                 return self.data.loadavg1 or 0.0
 335
 336         @property
 337         def loadavg5(self):
 338                 return self.data.loadavg5 or 0.0
 339
 340         @property
 341         def loadavg15(self):
 342                 return self.data.loadavg15 or 0.0
 343
 344         @property
 345         def pakfire_version(self):
 346                 return self.data.pakfire_version or ""
 347
 348         @property
 349         def os_name(self):
 350                 return self.data.os_name or ""
 351
 352         @property
 353         def cpu_model(self):
 354                 return self.data.cpu_model or ""
 355
 356         @property
 357         def cpu_count(self):
 358                 return self.data.cpu_count
 359
 360         @property
 361         def cpu_arch(self):
 362                 return self.data.cpu_arch
 363
 364         @property
 365         def cpu_bogomips(self):
 366                 return self.data.cpu_bogomips or 0.0
 367
 368         @property
 369         def mem_percentage(self):
 370                 if not self.mem_total:
 371                         return None
 372
 373                 return self.mem_used * 100 / self.mem_total
 374
 375         @property
 376         def mem_total(self):
 377                 return self.data.mem_total
 378
 379         @property
 380         def mem_used(self):
 381                 if self.mem_total and self.mem_free:
 382                         return self.mem_total - self.mem_free
 383
 384         @property
 385         def mem_free(self):
 386                 return self.data.mem_free
 387
 388         @property
 389         def swap_percentage(self):
 390                 if not self.swap_total:
 391                         return None
 392
 393                 return self.swap_used * 100 / self.swap_total
 394
 395         @property
 396         def swap_total(self):
 397                 return self.data.swap_total
 398
 399         @property
 400         def swap_used(self):
 401                 if self.swap_total and self.swap_free:
 402                         return self.swap_total - self.swap_free
 403
 404         @property
 405         def swap_free(self):
 406                 return self.data.swap_free
 407
 408         @property
 409         def space_free(self):
 410                 return self.data.space_free
 411
 412         @property
 413         def overload(self):
 414                 if not self.cpu_count or not self.loadavg1:
 415                         return None
 416
 417                 return self.loadavg1 >= self.cpu_count
 418
 419         @property
 420         def host_key_id(self):
 421                 return self.data.host_key_id
 422
 423         @property
 424         def state(self):
 425                 if self.disabled:
 426                         return "disabled"
 427
 428                 if self.data.time_keepalive is None:
 429                         return "offline"
 430
 431                 #if self.data.updated >= 5*60:
 432                 #       return "offline"
 433
 434                 return "online"
 435
 436         @lazy_property
 437         def active_jobs(self, *args, **kwargs):
 438                 return self.pakfire.jobs.get_active(builder=self, *args, **kwargs)
 439
 440         @property
 441         def too_many_jobs(self):
 442                 """
 443                         Tell if this host is already running enough or too many jobs.
 444                 """
 445                 return len(self.active_jobs) >= self.max_jobs
 446
 447         def get_next_jobs(self, limit=None):
 448                 """
 449                         Returns a list of jobs that can be built on this host.
 450                 """
 451                 return self.pakfire.jobs.get_next(arches=self.buildable_arches, limit=limit)
 452
 453         def get_next_job(self):
 454                 """
 455                         Returns the next job in line for this builder.
 456                 """
 457                 # Get the first item of all jobs in the list.
 458                 jobs = self.pakfire.jobs.get_next(builder=self, state="pending", limit=1)
 459
 460                 if jobs:
 461                         return jobs[0]
 462
 463         def get_history(self, *args, **kwargs):
 464                 kwargs["builder"] = self
 465
 466                 return self.pakfire.builders.get_history(*args, **kwargs)