www/webapp/backend/stasy.py

   1 #!/usr/bin/python
   2
   3 from __future__ import division
   4
   5 import hwdata
   6 import logging
   7 import pymongo
   8
   9 from misc import Singleton
  10
  11 DATABASE_HOST = ["irma.ipfire.org", "madeye.ipfire.org"]
  12 DATABASE_NAME = "stasy"
  13
  14 CPU_SPEED_CONSTRAINTS = (0, 500, 1000, 1500, 2000, 2500, 3000, 3500)
  15 MEMORY_CONSTRAINTS = (0, 64, 128, 256, 512, 1024, 2048, 4096, 8128, 16384)
  16
  17 class ProfileDict(object):
  18         def __init__(self, data):
  19                 self._data = data
  20
  21
  22 class ProfileCPU(ProfileDict):
  23         @property
  24         def arch(self):
  25                 return self._data.get("arch")
  26
  27         @property
  28         def vendor(self):
  29                 return self._data.get("vendor")
  30
  31         @property
  32         def speed(self):
  33                 return self._data.get("speed")
  34
  35         @property
  36         def bogomips(self):
  37                 return self._data.get("bogomips")
  38
  39         @property
  40         def model(self):
  41                 return self._data.get("model")
  42
  43         @property
  44         def model_string(self):
  45                 return self._data.get("model_string")
  46
  47         @property
  48         def flags(self):
  49                 return self._data.get("flags")
  50
  51         @property
  52         def count(self):
  53                 return self._data.get("count")
  54
  55         @property
  56         def capable_64bit(self):
  57                 return "lm" in self.flags
  58
  59         @property
  60         def capable_pae(self):
  61                 return "pae" in self.flags
  62
  63         @property
  64         def capable_virt(self):
  65                 return "vmx" in self.flags or "svm" in self.flags
  66
  67
  68 class ProfileHypervisor(ProfileDict):
  69         def __repr__(self):
  70                 return "<%s %s-%s>" % (self.__class__.__name__, self.vendor, self.type)
  71
  72         @property
  73         def vendor(self):
  74                 return self._data.get("vendor")
  75
  76         @property
  77         def type(self):
  78                 return self._data.get("type")
  79
  80
  81 class ProfileNetwork(ProfileDict):
  82         def has_zone(self, name):
  83                 return self._data.get(name)
  84
  85         @property
  86         def green(self):
  87                 return self._data.get("green")
  88
  89         @property
  90         def red(self):
  91                 return self._data.get("red")
  92
  93         @property
  94         def blue(self):
  95                 return self._data.get("blue")
  96
  97         @property
  98         def orange(self):
  99                 return self._data.get("orange")
 100
 101
 102 class ProfileDevice(ProfileDict):
 103         subsystem2class = {
 104                 "pci" : hwdata.PCI,
 105                 "usb" : hwdata.USB,
 106         }
 107
 108         classid2name = {
 109                 "pci" : {
 110                         "00" : "Unclassified",
 111                         "01" : "Mass storage",
 112                         "02" : "Network",
 113                         "03" : "Display",
 114                         "04" : "Multimedia",
 115                         "05" : "Memory controller",
 116                         "06" : "Bridge",
 117                         "07" : "Communication",
 118                         "08" : "Generic system peripheral",
 119                         "09" : "Input device",
 120                         "0a" : "Docking station",
 121                         "0b" : "Processor",
 122                         "0c" : "Serial bus",
 123                         "0d" : "Wireless",
 124                         "0e" : "Intelligent controller",
 125                         "0f" : "Satellite communications controller",
 126                         "10" : "Encryption",
 127                         "11" : "Signal processing controller",
 128                         "ff" : "Unassigned class",
 129                 },
 130
 131                 "usb" : {
 132                         "00" : "Unclassified",
 133                         "01" : "Multimedia",
 134                         "02" : "Communication",
 135                         "03" : "Input device",
 136                         "05" : "Generic system peripheral",
 137                         "06" : "Image",
 138                         "07" : "Printer",
 139                         "08" : "Mass storage",
 140                         "09" : "Hub",
 141                         "0a" : "Communication",
 142                         "0b" : "Smart card",
 143                         "0d" : "Encryption",
 144                         "0e" : "Display",
 145                         "0f" : "Personal Healthcare",
 146                         "dc" : "Diagnostic Device",
 147                         "e0" : "Wireless",
 148                         "ef" : "Unclassified",
 149                         "fe" : "Unclassified",
 150                         "ff" : "Unclassified",
 151                 }
 152         }
 153
 154         def __cmp__(self, other):
 155                 return cmp(self.vendor, other.vendor) or \
 156                         cmp(self.model, other.model) or \
 157                         cmp(self.driver, other.driver)
 158
 159         @property
 160         def model(self):
 161                 return self._data.get("model")
 162
 163         @property
 164         def model_string(self):
 165                 cls = self.subsystem2class[self.subsystem]
 166
 167                 return cls().get_device(self.vendor, self.model)
 168
 169         @property
 170         def subsystem(self):
 171                 return self._data.get("subsystem")
 172
 173         @property
 174         def vendor(self):
 175                 return self._data.get("vendor")
 176
 177         @property
 178         def vendor_string(self):
 179                 cls = self.subsystem2class[self.subsystem]
 180
 181                 return cls().get_vendor(self.vendor)
 182
 183         @property
 184         def driver(self):
 185                 return self._data.get("driver")
 186
 187         @property
 188         def cls(self):
 189                 classid = self._data.get("deviceclass")
 190
 191                 if self.subsystem == "pci":
 192                         classid = classid[:-4]
 193                         if len(classid) == 1:
 194                                 classid = "0%s" % classid
 195
 196                 elif self.subsystem == "usb" and classid:
 197                         classid = classid.split("/")[0]
 198                         classid = "%02x" % int(classid)
 199
 200                 try:
 201                         return self.classid2name[self.subsystem][classid]
 202                 except KeyError:
 203                         return "N/A"
 204
 205
 206 class Profile(ProfileDict):
 207         def __init__(self, profile_blob):
 208                 ProfileDict.__init__(self, profile_blob.get("profile"))
 209
 210                 self.public_id = profile_blob.get("public_id")
 211                 self.updated = profile_blob.get("updated")
 212                 self._geoip = profile_blob.get("geoip", None)
 213
 214         def __repr__(self):
 215                 return "<%s %s>" % (self.__class__.__name__, self.public_id)
 216
 217         @property
 218         def cpu(self):
 219                 return ProfileCPU(self._data.get("cpu"))
 220
 221         @property
 222         def devices(self):
 223                 devices = []
 224                 for d in self._data.get("devices"):
 225                         d = ProfileDevice(d)
 226
 227                         if d.driver in ("pcieport", "usb", "hub"):
 228                                 continue
 229
 230                         devices.append(d)
 231
 232                 return devices
 233
 234         @property
 235         def hypervisor(self):
 236                 if self.virtual:
 237                         return ProfileHypervisor(self._data.get("hypervisor"))
 238
 239         @property
 240         def virtual(self):
 241                 return self.system.get("virtual")
 242
 243         @property
 244         def system(self):
 245                 return self._data.get("system")
 246
 247         @property
 248         def release(self):
 249                 return self.system.get("release")
 250
 251         @property
 252         def kernel(self):
 253                 return self.system.get("kernel_release")
 254
 255         @property
 256         def memory(self):
 257                 return self.system.get("memory")
 258
 259         @property
 260         def root_size(self):
 261                 return self.system.get("root_size") or 0
 262
 263         @property
 264         def vendor(self):
 265                 return self.system.get("vendor")
 266
 267         @property
 268         def model(self):
 269                 return self.system.get("model")
 270
 271         @property
 272         def country_code(self):
 273                 if self._geoip:
 274                         return self._geoip["country_code"].lower()
 275
 276                 return "unknown"
 277
 278         @property
 279         def network(self):
 280                 network = self._data.get("network", None)
 281                 if network:
 282                         return ProfileNetwork(network)
 283
 284
 285 class Stasy(object):
 286         __metaclass__ = Singleton
 287
 288         def __init__(self):
 289                 # Initialize database connection
 290                 self._conn = pymongo.Connection(DATABASE_HOST)
 291                 self._db = self._conn[DATABASE_NAME]
 292
 293         def get_profile_count(self):
 294                 # XXX need to implement something to get profiles updated since
 295                 # a given date
 296
 297                 # All distinct profiles (based on public_id)
 298                 return self._db.profiles.distinct("public_id").count()
 299
 300         #def _get_profile_cursor(self, public_id):
 301         #       c = self._db.profiles.find({ "public_id" : public_id })
 302         #       c.sort("updated", pymongo.ASCENDING)
 303         #
 304         #       return c
 305
 306         def profile_exists(self, public_id):
 307                 return self.query({ "public_id" : public_id }).count() >= 1
 308
 309         def get_profile(self, public_id):
 310                 # XXX should only find one object in the end
 311                 for p in self.query({ "public_id" : public_id }):
 312                         p = Profile(p)
 313
 314                 return p
 315
 316         def get_profiles(self):
 317                 # XXX needs nicer database query
 318                 profiles = []
 319                 for p in self._db.profiles.find():
 320                         if not p.get("public_id") in profiles:
 321                                 profiles.append(p.get("public_id"))
 322
 323                 return profiles
 324
 325         def query(self, query, archives=False, no_virt=False, all=False):
 326                 db = self._db.profiles
 327
 328                 if archives:
 329                         db = self.db.archives
 330
 331                 if not all:
 332                         # XXX cannot use the index?
 333                         query.update({ "profile" : { "$exists" : True }})
 334
 335                 if no_virt:
 336                         query.update({ "profile.system.virtual" : False })
 337
 338                 logging.debug("Executing query: %s" % query)
 339
 340                 return db.find(query)
 341
 342         @property
 343         def secret_ids(self):
 344                 return self._db.profiles.distinct("secret_id")
 345
 346         @property
 347         def cpus(self):
 348                 return self.query({}, no_virt=True).distinct("profile.cpu")
 349
 350         @property
 351         def cpu_vendors(self):
 352                 return self.query({}, no_virt=True).distinct("profile.cpu.vendor")
 353
 354         @property
 355         def cpu_vendors_map(self):
 356                 cpus = {}
 357
 358                 for vendor in self.cpu_vendors:
 359                         cpus[vendor] = \
 360                                 self.query({
 361                                         "profile.cpu.vendor" : vendor
 362                                 }, no_virt=True).count()
 363
 364                 return cpus
 365
 366         @property
 367         def cpu_speed_average(self):
 368                 speed = 0
 369
 370                 all = self.query({}, no_virt=True)
 371
 372                 # XXX ugly. needs to be done by group()
 373                 for m in all:
 374                         if not m.has_key("profile"):
 375                                 continue
 376                         speed += m.get("profile").get("cpu").get("speed")
 377
 378                 return (speed / all.count())
 379
 380         @property
 381         def cpu_speed_map(self):
 382                 cpu_speeds = {}
 383
 384                 for i in range(len(CPU_SPEED_CONSTRAINTS) - 1):
 385                         min, max = CPU_SPEED_CONSTRAINTS[i:i+2]
 386
 387                         cpu_speeds[min, max] = \
 388                                 self.query({
 389                                         "profile.cpu.speed" : {
 390                                                 "$gte" : min, "$lt" : max
 391                                         }
 392                                 }, no_virt=True).count()
 393
 394                 return cpu_speeds
 395
 396         def get_memory_map(self):
 397                 memory = {}
 398
 399                 for i in range(len(MEMORY_CONSTRAINTS) - 1):
 400                         min, max = MEMORY_CONSTRAINTS[i:i+2]
 401
 402                         memory[min, max] = \
 403                                 self.query(
 404                                         { "profile.system.memory" : {
 405                                                 "$gte" : min * 1024, "$lt" : max * 1024
 406                                         }
 407                                 }, no_virt=True).count()
 408
 409                 return memory
 410
 411         @property
 412         def memory_average(self):
 413                 memory = 0
 414
 415                 all = self.query({}, no_virt=True)
 416
 417                 # XXX ugly. needs to be done by group()
 418                 for m in all:
 419                         if not m.has_key("profile"):
 420                                 continue
 421                         memory += int(m.get("profile").get("system").get("memory"))
 422
 423                 return (memory / all.count()) / 1024
 424
 425         @property
 426         def hypervisor_vendors(self):
 427                 return self.query({}).distinct("profile.hypervisor.vendor")
 428
 429         @property
 430         def hypervisor_map(self):
 431                 hypervisors = {}
 432
 433                 for hypervisor in self.hypervisor_vendors:
 434                         hypervisors[hypervisor] = \
 435                                 self.query({
 436                                         "profile.hypervisor.vendor" : hypervisor
 437                                 }).count()
 438
 439                 return hypervisors
 440
 441         @property
 442         def hypervisor_models(self):
 443                 return self.query({}).distinct("profile.hypervisor.model")
 444
 445         @property
 446         def virtual_map(self):
 447                 virtual = {
 448                         True: None,
 449                         False: None,
 450                 }
 451
 452                 for k in virtual.keys():
 453                         virtual[k] = \
 454                                 self.query({ "profile.system.virtual": k }).count()
 455
 456                 return virtual
 457
 458         @property
 459         def languages(self):
 460                 return self.query({}).distinct("profile.system.language")
 461
 462         def get_language_map(self):
 463                 languages = {}
 464
 465                 for language in self.languages:
 466                         languages[language] = \
 467                                 self.query({
 468                                         "profile.system.language" : language
 469                                 }).count()
 470
 471                 return languages
 472
 473         @property
 474         def vendors(self):
 475                 return self.query({}).distinct("profile.system.vendor")
 476
 477         @property
 478         def vendor_map(self):
 479                 vendors = {}
 480
 481                 for vendor in self.vendors:
 482                         vendors[vendor] = \
 483                                 self.query({
 484                                         "profile.system.vendor" : vendor
 485                                 }).count()
 486
 487                 return vendors
 488
 489         @property
 490         def models(self):
 491                 return self.query({}).distinct("profile.system.model")
 492
 493         @property
 494         def model_map(self):
 495                 models = {}
 496
 497                 for model in self.models:
 498                         models[model] = \
 499                                 self.query({
 500                                         "profile.system.model" : model
 501                                 }).count()
 502
 503                 return models
 504
 505         @property
 506         def arches(self):
 507                 return self.query({}).distinct("profile.cpu.arch")
 508
 509         @property
 510         def arch_map(self):
 511                 arches = {}
 512
 513                 for arch in self.arches:
 514                         arches[arch] = \
 515                                 self.query({
 516                                         "profile.cpu.arch" : arch
 517                                 }).count()
 518
 519                 return arches
 520
 521         @property
 522         def kernels(self):
 523                 return self.query({}).distinct("profile.system.kernel_release")
 524
 525         @property
 526         def kernel_map(self):
 527                 kernels = {}
 528
 529                 for kernel in self.kernels:
 530                         kernels[kernel] = \
 531                                 self.query({
 532                                         "profile.system.kernel_release" : kernel
 533                                 }).count()
 534
 535                 return kernels
 536
 537         @property
 538         def releases(self):
 539                 return self.query({}).distinct("profile.system.release")
 540
 541         @property
 542         def release_map(self):
 543                 releases = {}
 544
 545                 for release in self.releases:
 546                         releases[release] = \
 547                                 self.query({
 548                                         "profile.system.release" : release
 549                                 }).count()
 550
 551                 return releases
 552
 553         def get_device_percentage(self, bus, vendor_id, model_id):
 554                 profiles_with_device = self.query({
 555                         "profile.devices.subsystem" : bus,
 556                         "profile.devices.vendor" : vendor_id,
 557                         "profile.devices.model" : model_id,
 558                 })
 559
 560                 profiles_all = self.query({})
 561
 562                 if not profiles_all.count():
 563                         return 0
 564
 565                 return profiles_with_device.count() / profiles_all.count()
 566
 567         def get_cpu_flag_map(self, flags):
 568                 # XXX needs a cleanup
 569
 570                 _flags = { True : 0 }
 571
 572                 if type(flags) == type("a"):
 573                         flags = [flags]
 574
 575                 for flag in flags:
 576                         _flags[True] += \
 577                                 self.query({
 578                                         "profile.cpu.flags" : flag,
 579                                 }, no_virt=True).count()
 580
 581                 _flags[False] = self.query({}, no_virt=True).count() - _flags[True]
 582
 583                 return _flags
 584
 585         @property
 586         def geo_locations(self):
 587                 return [code.lower() for code in self.query({}).distinct("geoip.country_code")]
 588
 589         def get_geo_location_map(self):
 590                 geo_locations = {}
 591
 592                 count = 0
 593                 for geo_location in self.geo_locations:
 594                         geo_locations[geo_location] = \
 595                                 self.query({
 596                                         "geoip.country_code" : geo_location
 597                                 }).count()
 598
 599                         count += geo_locations[geo_location]
 600
 601                 # XXX must only be systems that have profile data
 602                 profiles_all = self.query({})
 603
 604                 unknown_count = profiles_all.count() - count
 605                 if unknown_count:
 606                         geo_locations["unknown"] = unknown_count
 607
 608                 return geo_locations
 609
 610         def get_models_by_vendor(self, subsystem, vendor_id):
 611                 devices = []
 612
 613                 # XXX must only be systems that have profile data
 614                 profiles_all = self.query({})
 615
 616                 for profile in profiles_all:
 617                         if not profile.has_key("profile"):
 618                                 continue
 619
 620                         profile = Profile(profile)
 621
 622                         for device in profile.devices:
 623                                 if not device.vendor == vendor_id:
 624                                         continue
 625
 626                                 if not device in devices:
 627                                         devices.append(device)
 628
 629                 return devices
 630
 631         def get_network_zones_map(self):
 632                 zones = { "green" : 0, "blue" : 0, "orange" : 0, "red" : 0 }
 633
 634                 for zone in zones.keys():
 635                         zones[zone] = self.query({
 636                                 "profile.network.%s" % zone : True,
 637                         }).count()
 638
 639                 return zones
 640
 641
 642 if __name__ == "__main__":
 643         s = Stasy()
 644
 645         print s.get_profile("0" * 40)
 646         print s.cpu_vendors
 647         for id in s.secret_ids:
 648                 print "\t", id
 649
 650         #for p in s._db.profiles.find():
 651         #       print p
 652
 653         print s.cpu_map
 654         print s.memory_map
 655         print s.memory_average
 656         print s.hypervisor_vendors
 657         print s.hypervisor_models
 658         print s.languages
 659         print s.language_maps
 660         print s.vendors
 661         print s.vendor_map
 662         print s.models
 663         print s.cpus
 664         print s.cpu_map
 665         print s.arches
 666         print s.arch_map
 667         print s.kernels
 668         print s.kernel_map
 669         print s.releases
 670         print s.release_map
 671
 672         p = s.get_profile("0b5f4fe2162fdfbfa29b632610e317078fa70d34")
 673         print p._data
 674 #       print p.hypervisor
 675