www/webapp/backend/stasy.py

   1 #!/usr/bin/python
   2
   3 from __future__ import division
   4
   5 import hwdata
   6 import logging
   7 import pymongo
   8
   9 from misc import Singleton
  10
  11 DATABASE_HOST = ["irma.ipfire.org", "madeye.ipfire.org"]
  12 DATABASE_NAME = "stasy"
  13
  14 CPU_SPEED_CONSTRAINTS = (0, 500, 1000, 1500, 2000, 2500, 3000, 3500)
  15 MEMORY_CONSTRAINTS = (0, 64, 128, 256, 512, 1024, 2048, 4096, 8128, 16384)
  16
  17 class ProfileDict(object):
  18         def __init__(self, data):
  19                 self._data = data
  20
  21
  22 class ProfileCPU(ProfileDict):
  23         @property
  24         def arch(self):
  25                 return self._data.get("arch")
  26
  27         @property
  28         def vendor(self):
  29                 return self._data.get("vendor")
  30
  31         @property
  32         def speed(self):
  33                 return self._data.get("speed")
  34
  35         @property
  36         def bogomips(self):
  37                 return self._data.get("bogomips")
  38
  39         @property
  40         def model(self):
  41                 return self._data.get("model")
  42
  43         @property
  44         def model_string(self):
  45                 return self._data.get("model_string")
  46
  47         @property
  48         def flags(self):
  49                 return self._data.get("flags")
  50
  51         @property
  52         def count(self):
  53                 return self._data.get("count")
  54
  55         @property
  56         def capable_64bit(self):
  57                 return "lm" in self.flags
  58
  59         @property
  60         def capable_pae(self):
  61                 return "pae" in self.flags
  62
  63         @property
  64         def capable_virt(self):
  65                 return "vmx" in self.flags or "svm" in self.flags
  66
  67
  68 class ProfileHypervisor(ProfileDict):
  69         def __repr__(self):
  70                 return "<%s %s-%s>" % (self.__class__.__name__, self.vendor, self.type)
  71
  72         @property
  73         def vendor(self):
  74                 return self._data.get("vendor")
  75
  76         @property
  77         def type(self):
  78                 return self._data.get("type")
  79
  80
  81 class ProfileNetwork(ProfileDict):
  82         def has_zone(self, name):
  83                 return self._data.get(name)
  84
  85         @property
  86         def green(self):
  87                 return self._data.get("green")
  88
  89         @property
  90         def red(self):
  91                 return self._data.get("red")
  92
  93         @property
  94         def blue(self):
  95                 return self._data.get("blue")
  96
  97         @property
  98         def orange(self):
  99                 return self._data.get("orange")
 100
 101
 102 class ProfileDevice(ProfileDict):
 103         subsystem2class = {
 104                 "pci" : hwdata.PCI,
 105                 "usb" : hwdata.USB,
 106         }
 107
 108         classid2name = {
 109                 "pci" : {
 110                         "00" : "Unclassified",
 111                         "01" : "Mass storage",
 112                         "02" : "Network",
 113                         "03" : "Display",
 114                         "04" : "Multimedia",
 115                         "05" : "Memory controller",
 116                         "06" : "Bridge",
 117                         "07" : "Communication",
 118                         "08" : "Generic system peripheral",
 119                         "09" : "Input device",
 120                         "0a" : "Docking station",
 121                         "0b" : "Processor",
 122                         "0c" : "Serial bus",
 123                         "0d" : "Wireless",
 124                         "0e" : "Intelligent controller",
 125                         "0f" : "Satellite communications controller",
 126                         "10" : "Encryption",
 127                         "11" : "Signal processing controller",
 128                         "ff" : "Unassigned class",
 129                 },
 130
 131                 "usb" : {
 132                         "00" : "Unclassified",
 133                         "01" : "Multimedia",
 134                         "02" : "Communication",
 135                         "03" : "Input device",
 136                         "05" : "Generic system peripheral",
 137                         "06" : "Image",
 138                         "07" : "Printer",
 139                         "08" : "Mass storage",
 140                         "09" : "Hub",
 141                         "0a" : "Communication",
 142                         "0b" : "Smart card",
 143                         "0d" : "Encryption",
 144                         "0e" : "Display",
 145                         "0f" : "Personal Healthcare",
 146                         "dc" : "Diagnostic Device",
 147                         "e0" : "Wireless",
 148                         "ef" : "Unclassified",
 149                         "fe" : "Unclassified",
 150                         "ff" : "Unclassified",
 151                 }
 152         }
 153
 154         def __cmp__(self, other):
 155                 return cmp(self.vendor, other.vendor) or \
 156                         cmp(self.model, other.model) or \
 157                         cmp(self.driver, other.driver)
 158
 159         @property
 160         def model(self):
 161                 return self._data.get("model")
 162
 163         @property
 164         def model_string(self):
 165                 cls = self.subsystem2class[self.subsystem]
 166
 167                 return cls().get_device(self.vendor, self.model)
 168
 169         @property
 170         def subsystem(self):
 171                 return self._data.get("subsystem")
 172
 173         @property
 174         def vendor(self):
 175                 return self._data.get("vendor")
 176
 177         @property
 178         def vendor_string(self):
 179                 cls = self.subsystem2class[self.subsystem]
 180
 181                 return cls().get_vendor(self.vendor)
 182
 183         @property
 184         def driver(self):
 185                 return self._data.get("driver")
 186
 187         @property
 188         def cls(self):
 189                 classid = self._data.get("deviceclass")
 190
 191                 if self.subsystem == "pci":
 192                         classid = classid[:-4]
 193                         if len(classid) == 1:
 194                                 classid = "0%s" % classid
 195
 196                 elif self.subsystem == "usb" and classid:
 197                         classid = classid.split("/")[0]
 198                         classid = "%02x" % int(classid)
 199
 200                 try:
 201                         return self.classid2name[self.subsystem][classid]
 202                 except KeyError:
 203                         return "N/A"
 204
 205
 206 class Profile(ProfileDict):
 207         def __init__(self, profile_blob):
 208                 ProfileDict.__init__(self, profile_blob.get("profile"))
 209
 210                 self.public_id = profile_blob.get("public_id")
 211                 self.updated = profile_blob.get("updated")
 212                 self._geoip = profile_blob.get("geoip", None)
 213
 214         def __repr__(self):
 215                 return "<%s %s>" % (self.__class__.__name__, self.public_id)
 216
 217         @property
 218         def cpu(self):
 219                 return ProfileCPU(self._data.get("cpu"))
 220
 221         @property
 222         def devices(self):
 223                 devices = []
 224                 for d in self._data.get("devices"):
 225                         d = ProfileDevice(d)
 226
 227                         if d.driver in ("pcieport", "usb", "hub"):
 228                                 continue
 229
 230                         devices.append(d)
 231
 232                 return devices
 233
 234         @property
 235         def hypervisor(self):
 236                 if self.virtual:
 237                         return ProfileHypervisor(self._data.get("hypervisor"))
 238
 239         @property
 240         def virtual(self):
 241                 return self.system.get("virtual")
 242
 243         @property
 244         def system(self):
 245                 return self._data.get("system")
 246
 247         @property
 248         def release(self):
 249                 return self.system.get("release")
 250
 251         @property
 252         def kernel(self):
 253                 return self.system.get("kernel_release")
 254
 255         @property
 256         def memory(self):
 257                 return self.system.get("memory")
 258
 259         @property
 260         def root_size(self):
 261                 return self.system.get("root_size") or 0
 262
 263         @property
 264         def vendor(self):
 265                 return self.system.get("vendor")
 266
 267         @property
 268         def model(self):
 269                 return self.system.get("model")
 270
 271         @property
 272         def country_code(self):
 273                 if self._geoip:
 274                         return self._geoip["country_code"].lower()
 275
 276                 return "unknown"
 277
 278         @property
 279         def network(self):
 280                 network = self._data.get("network", None)
 281                 if network:
 282                         return ProfileNetwork(network)
 283
 284
 285 class Stasy(object):
 286         __metaclass__ = Singleton
 287
 288         def __init__(self):
 289                 # Initialize database connection
 290                 self._conn = pymongo.Connection(DATABASE_HOST)
 291                 self._db = self._conn[DATABASE_NAME]
 292
 293         def get_profile_count(self):
 294                 # XXX need to implement something to get profiles updated since
 295                 # a given date
 296
 297                 # All distinct profiles (based on public_id)
 298                 return self._db.profiles.distinct("public_id").count()
 299
 300         #def _get_profile_cursor(self, public_id):
 301         #       c = self._db.profiles.find({ "public_id" : public_id })
 302         #       c.sort("updated", pymongo.ASCENDING)
 303         #
 304         #       return c
 305
 306         def profile_exists(self, public_id):
 307                 return self.query({ "public_id" : public_id }).count() >= 1
 308
 309         def get_profile(self, public_id):
 310                 p = None
 311                 # XXX should only find one object in the end
 312                 for p in self.query({ "public_id" : public_id }):
 313                         p = Profile(p)
 314
 315                 return p
 316
 317         def get_profiles(self):
 318                 # XXX needs nicer database query
 319                 profiles = []
 320                 for p in self._db.profiles.find():
 321                         if not p.get("public_id") in profiles:
 322                                 profiles.append(p.get("public_id"))
 323
 324                 return profiles
 325
 326         def query(self, query, archives=False, no_virt=False, all=False):
 327                 db = self._db.profiles
 328
 329                 if archives:
 330                         db = self.db.archives
 331
 332                 if not all:
 333                         # XXX cannot use the index?
 334                         query.update({ "profile" : { "$exists" : True }})
 335
 336                 if no_virt:
 337                         query.update({ "profile.system.virtual" : False })
 338
 339                 logging.debug("Executing query: %s" % query)
 340
 341                 return db.find(query)
 342
 343         @property
 344         def secret_ids(self):
 345                 return self._db.profiles.distinct("secret_id")
 346
 347         @property
 348         def cpus(self):
 349                 return self.query({}, no_virt=True).distinct("profile.cpu")
 350
 351         @property
 352         def cpu_vendors(self):
 353                 return self.query({}, no_virt=True).distinct("profile.cpu.vendor")
 354
 355         @property
 356         def cpu_vendors_map(self):
 357                 cpus = {}
 358
 359                 for vendor in self.cpu_vendors:
 360                         cpus[vendor] = \
 361                                 self.query({
 362                                         "profile.cpu.vendor" : vendor
 363                                 }, no_virt=True).count()
 364
 365                 return cpus
 366
 367         @property
 368         def cpu_speed_average(self):
 369                 speed = 0
 370
 371                 all = self.query({}, no_virt=True)
 372
 373                 # XXX ugly. needs to be done by group()
 374                 for m in all:
 375                         if not m.has_key("profile"):
 376                                 continue
 377                         speed += m.get("profile").get("cpu").get("speed")
 378
 379                 return (speed / all.count())
 380
 381         @property
 382         def cpu_speed_map(self):
 383                 cpu_speeds = {}
 384
 385                 for i in range(len(CPU_SPEED_CONSTRAINTS) - 1):
 386                         min, max = CPU_SPEED_CONSTRAINTS[i:i+2]
 387
 388                         cpu_speeds[min, max] = \
 389                                 self.query({
 390                                         "profile.cpu.speed" : {
 391                                                 "$gte" : min, "$lt" : max
 392                                         }
 393                                 }, no_virt=True).count()
 394
 395                 return cpu_speeds
 396
 397         def get_memory_map(self):
 398                 memory = {}
 399
 400                 for i in range(len(MEMORY_CONSTRAINTS) - 1):
 401                         min, max = MEMORY_CONSTRAINTS[i:i+2]
 402
 403                         memory[min, max] = \
 404                                 self.query(
 405                                         { "profile.system.memory" : {
 406                                                 "$gte" : min * 1024, "$lt" : max * 1024
 407                                         }
 408                                 }, no_virt=True).count()
 409
 410                 return memory
 411
 412         @property
 413         def memory_average(self):
 414                 memory = 0
 415
 416                 all = self.query({}, no_virt=True)
 417
 418                 # XXX ugly. needs to be done by group()
 419                 for m in all:
 420                         if not m.has_key("profile"):
 421                                 continue
 422                         memory += int(m.get("profile").get("system").get("memory"))
 423
 424                 return (memory / all.count()) / 1024
 425
 426         @property
 427         def hypervisor_vendors(self):
 428                 return self.query({}).distinct("profile.hypervisor.vendor")
 429
 430         @property
 431         def hypervisor_map(self):
 432                 hypervisors = {}
 433
 434                 for hypervisor in self.hypervisor_vendors:
 435                         hypervisors[hypervisor] = \
 436                                 self.query({
 437                                         "profile.hypervisor.vendor" : hypervisor
 438                                 }).count()
 439
 440                 return hypervisors
 441
 442         @property
 443         def hypervisor_models(self):
 444                 return self.query({}).distinct("profile.hypervisor.model")
 445
 446         @property
 447         def virtual_map(self):
 448                 virtual = {
 449                         True: None,
 450                         False: None,
 451                 }
 452
 453                 for k in virtual.keys():
 454                         virtual[k] = \
 455                                 self.query({ "profile.system.virtual": k }).count()
 456
 457                 return virtual
 458
 459         @property
 460         def languages(self):
 461                 return self.query({}).distinct("profile.system.language")
 462
 463         def get_language_map(self):
 464                 languages = {}
 465
 466                 for language in self.languages:
 467                         languages[language] = \
 468                                 self.query({
 469                                         "profile.system.language" : language
 470                                 }).count()
 471
 472                 return languages
 473
 474         @property
 475         def vendors(self):
 476                 return self.query({}).distinct("profile.system.vendor")
 477
 478         @property
 479         def vendor_map(self):
 480                 vendors = {}
 481
 482                 for vendor in self.vendors:
 483                         vendors[vendor] = \
 484                                 self.query({
 485                                         "profile.system.vendor" : vendor
 486                                 }).count()
 487
 488                 return vendors
 489
 490         @property
 491         def models(self):
 492                 return self.query({}).distinct("profile.system.model")
 493
 494         @property
 495         def model_map(self):
 496                 models = {}
 497
 498                 for model in self.models:
 499                         models[model] = \
 500                                 self.query({
 501                                         "profile.system.model" : model
 502                                 }).count()
 503
 504                 return models
 505
 506         @property
 507         def arches(self):
 508                 return self.query({}).distinct("profile.cpu.arch")
 509
 510         @property
 511         def arch_map(self):
 512                 arches = {}
 513
 514                 for arch in self.arches:
 515                         arches[arch] = \
 516                                 self.query({
 517                                         "profile.cpu.arch" : arch
 518                                 }).count()
 519
 520                 return arches
 521
 522         @property
 523         def kernels(self):
 524                 return self.query({}).distinct("profile.system.kernel_release")
 525
 526         @property
 527         def kernel_map(self):
 528                 kernels = {}
 529
 530                 for kernel in self.kernels:
 531                         kernels[kernel] = \
 532                                 self.query({
 533                                         "profile.system.kernel_release" : kernel
 534                                 }).count()
 535
 536                 return kernels
 537
 538         @property
 539         def releases(self):
 540                 return self.query({}).distinct("profile.system.release")
 541
 542         @property
 543         def release_map(self):
 544                 releases = {}
 545
 546                 for release in self.releases:
 547                         releases[release] = \
 548                                 self.query({
 549                                         "profile.system.release" : release
 550                                 }).count()
 551
 552                 return releases
 553
 554         def get_device_percentage(self, bus, vendor_id, model_id):
 555                 profiles_with_device = self.query({
 556                         "profile.devices.subsystem" : bus,
 557                         "profile.devices.vendor" : vendor_id,
 558                         "profile.devices.model" : model_id,
 559                 })
 560
 561                 profiles_all = self.query({})
 562
 563                 if not profiles_all.count():
 564                         return 0
 565
 566                 return profiles_with_device.count() / profiles_all.count()
 567
 568         def get_cpu_flag_map(self, flags):
 569                 # XXX needs a cleanup
 570
 571                 _flags = { True : 0 }
 572
 573                 if type(flags) == type("a"):
 574                         flags = [flags]
 575
 576                 for flag in flags:
 577                         _flags[True] += \
 578                                 self.query({
 579                                         "profile.cpu.flags" : flag,
 580                                 }, no_virt=True).count()
 581
 582                 _flags[False] = self.query({}, no_virt=True).count() - _flags[True]
 583
 584                 return _flags
 585
 586         @property
 587         def geo_locations(self):
 588                 return [code.lower() for code in self.query({}).distinct("geoip.country_code")]
 589
 590         def get_geo_location_map(self):
 591                 geo_locations = {}
 592
 593                 count = 0
 594                 for geo_location in self.geo_locations:
 595                         geo_locations[geo_location] = \
 596                                 self.query({
 597                                         "geoip.country_code" : geo_location.upper()
 598                                 }).count()
 599
 600                         count += geo_locations[geo_location]
 601
 602                 # XXX must only be systems that have profile data
 603                 profiles_all = self.query({})
 604
 605                 unknown_count = profiles_all.count() - count
 606                 if unknown_count:
 607                         geo_locations["unknown"] = unknown_count
 608
 609                 return geo_locations
 610
 611         def get_models_by_vendor(self, subsystem, vendor_id):
 612                 devices = []
 613
 614                 # XXX must only be systems that have profile data
 615                 profiles_all = self.query({})
 616
 617                 for profile in profiles_all:
 618                         if not profile.has_key("profile"):
 619                                 continue
 620
 621                         profile = Profile(profile)
 622
 623                         for device in profile.devices:
 624                                 if not device.vendor == vendor_id:
 625                                         continue
 626
 627                                 if not device in devices:
 628                                         devices.append(device)
 629
 630                 return devices
 631
 632         def get_network_zones_map(self):
 633                 zones = { "green" : 0, "blue" : 0, "orange" : 0, "red" : 0 }
 634
 635                 all = self.query({ "profile.network" : { "$exists" : True }})
 636
 637                 for zone in zones.keys():
 638                         zones[zone] = self.query({
 639                                 "profile.network.%s" % zone : True,
 640                         }).count() / all.count()
 641
 642                 return zones
 643
 644
 645 if __name__ == "__main__":
 646         s = Stasy()
 647
 648         print s.get_profile("0" * 40)
 649         print s.cpu_vendors
 650         for id in s.secret_ids:
 651                 print "\t", id
 652
 653         #for p in s._db.profiles.find():
 654         #       print p
 655
 656         print s.cpu_map
 657         print s.memory_map
 658         print s.memory_average
 659         print s.hypervisor_vendors
 660         print s.hypervisor_models
 661         print s.languages
 662         print s.language_maps
 663         print s.vendors
 664         print s.vendor_map
 665         print s.models
 666         print s.cpus
 667         print s.cpu_map
 668         print s.arches
 669         print s.arch_map
 670         print s.kernels
 671         print s.kernel_map
 672         print s.releases
 673         print s.release_map
 674
 675         p = s.get_profile("0b5f4fe2162fdfbfa29b632610e317078fa70d34")
 676         print p._data
 677 #       print p.hypervisor
 678