www/webapp/backend/stasy.py

   1 #!/usr/bin/python
   2
   3 from __future__ import division
   4
   5 import datetime
   6 import hwdata
   7 import logging
   8 import pymongo
   9 import re
  10
  11 from misc import Singleton
  12
  13 DATABASE_HOST = ["wilhelmina.ipfire.org"]
  14 DATABASE_NAME = "stasy"
  15
  16 CPU_SPEED_CONSTRAINTS = (0, 500, 1000, 1500, 2000, 2500, 3000, 3500)
  17 MEMORY_CONSTRAINTS = (0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384)
  18
  19 CPU_VENDORS = {
  20         "AMDisbetter!" : "AMD",
  21         "AuthenticAMD" : "AMD",
  22         "CentaurHauls" : "VIA",
  23         "CyrixInstead" : "Cyrix",
  24         "GenuineIntel" : "Intel",
  25         "TransmetaCPU" : "Transmeta",
  26         "GenuineTMx86" : "Transmeta",
  27         "Geode by NSC" : "NSC",
  28         "NexGenDriven" : "NexGen",
  29         "RiseRiseRise" : "Rise",
  30         "SiS SiS SiS " : "SiS",
  31         "UMC UMC UMC " : "UMC",
  32         "VIA VIA VIA " : "VIA",
  33 }
  34
  35 CPU_STRINGS = (
  36         # AMD
  37         (r"(AMD Athlon).*(XP).*", r"\1 \2"),
  38         (r"(AMD Phenom).* ([0-9]+) .*", r"\1 \2"),
  39         (r"(AMD Phenom).*", r"\1"),
  40         (r"(AMD Sempron).*", r"\1"),
  41         (r"AMD Athlon.* II X2 ([a-z0-9]+).*", r"AMD Athlon X2 \1"),
  42         (r"(Geode).*", r"\1"),
  43
  44         # Intel
  45         (r"Intel.*(Atom|Celeron).*CPU\s*([A-Z0-9]+) .*", r"Intel \1 \2"),
  46         (r"(Intel).*(Celeron).*", r"\1 \2"),
  47         (r"Intel.* Core.*2 Duo *CPU .* ([A-Z0-9]+) .*", r"Intel C2D \1"),
  48         (r"Intel.* Core.*2 CPU .* ([A-Z0-9]+) .*", r"Intel C2 \1"),
  49         (r"Intel.* Core.*2 Quad *CPU .* ([A-Z0-9]+) .*", r"Intel C2Q \1"),
  50         (r"Intel.* Xeon.* CPU .* ([A-Z0-9]+) .*", r"Intel Xeon \1"),
  51         (r"(Intel).*(Xeon).*", r"\1 \2"),
  52         (r"Intel.* Pentium.* (D|4) .*", r"Intel Pentium \1"),
  53         (r"Intel.* Pentium.* Dual .* ([A-Z0-9]+) .*", r"Intel Pentium Dual \1"),
  54         (r"Pentium.* Dual-Core .* ([A-Z0-9]+) .*", r"Intel Pentium Dual \1"),
  55         (r"(Pentium I{2,3}).*", r"Intel \1"),
  56         (r"(Celeron \(Coppermine\))", r"Intel Celeron"),
  57
  58         # VIA
  59         (r"(VIA \w*).*", r"\1"),
  60
  61         # Qemu
  62         (r"QEMU Virtual CPU version .*", r"QEMU CPU"),
  63
  64         # ARM
  65         (r"Feroceon .*", r"ARM Feroceon"),
  66 )
  67
  68 CPU_CORES = range(1, 9)
  69
  70 class ProfileDict(object):
  71         def __init__(self, data):
  72                 self._data = data
  73
  74
  75 class ProfileCPU(ProfileDict):
  76         @property
  77         def arch(self):
  78                 return self._data.get("arch")
  79
  80         @property
  81         def vendor(self):
  82                 vendor = self._data.get("vendor")
  83
  84                 return CPU_VENDORS.get(vendor, vendor)
  85
  86         @property
  87         def speed(self):
  88                 speed = self._data.get("speed")
  89
  90                 # ARM boxes do not report speed but
  91                 # bogomips equals speed
  92                 if not speed:
  93                         return self.bogomips
  94
  95                 return speed
  96
  97         @property
  98         def friendly_speed(self):
  99                 if self.speed < 1000:
 100                         return "%dMHz" % self.speed
 101
 102                 return "%.1fGHz" % round(self.speed / 1000, 1)
 103
 104         @property
 105         def bogomips(self):
 106                 return self._data.get("bogomips")
 107
 108         @property
 109         def model(self):
 110                 return self._data.get("model")
 111
 112         @property
 113         def model_string(self):
 114                 return self._data.get("model_string")
 115
 116         @property
 117         def flags(self):
 118                 return self._data.get("flags")
 119
 120         @property
 121         def count(self):
 122                 return self._data.get("count")
 123
 124         @property
 125         def capable_64bit(self):
 126                 return "lm" in self.flags
 127
 128         @property
 129         def capable_pae(self):
 130                 return "pae" in self.flags
 131
 132         @property
 133         def capable_virt(self):
 134                 return "vmx" in self.flags or "svm" in self.flags
 135
 136         @property
 137         def friendly_vendor(self):
 138                 s = self.model_string
 139                 for pattern, repl in CPU_STRINGS:
 140                         if re.match(pattern, s) is None:
 141                                 continue
 142                         return re.sub(pattern, repl, s)
 143
 144                 return s
 145
 146         @property
 147         def friendly_string(self):
 148                 s = self.friendly_vendor
 149
 150                 if self.speed:
 151                         s += " @ %s" % self.friendly_speed
 152
 153                 return s
 154
 155
 156 class ProfileHypervisor(ProfileDict):
 157         def __repr__(self):
 158                 return "<%s %s-%s>" % (self.__class__.__name__, self.vendor, self.type)
 159
 160         @property
 161         def vendor(self):
 162                 return self._data.get("vendor")
 163
 164         @property
 165         def type(self):
 166                 return self._data.get("type")
 167
 168
 169 class ProfileNetwork(ProfileDict):
 170         def has_zone(self, name):
 171                 return self._data.get(name)
 172
 173         @property
 174         def green(self):
 175                 return self._data.get("green")
 176
 177         @property
 178         def red(self):
 179                 return self._data.get("red")
 180
 181         @property
 182         def blue(self):
 183                 return self._data.get("blue")
 184
 185         @property
 186         def orange(self):
 187                 return self._data.get("orange")
 188
 189
 190 class ProfileDevice(ProfileDict):
 191         subsystem2class = {
 192                 "pci" : hwdata.PCI,
 193                 "usb" : hwdata.USB,
 194         }
 195
 196         classid2name = {
 197                 "pci" : {
 198                         "00" : "Unclassified",
 199                         "01" : "Mass storage",
 200                         "02" : "Network",
 201                         "03" : "Display",
 202                         "04" : "Multimedia",
 203                         "05" : "Memory controller",
 204                         "06" : "Bridge",
 205                         "07" : "Communication",
 206                         "08" : "Generic system peripheral",
 207                         "09" : "Input device",
 208                         "0a" : "Docking station",
 209                         "0b" : "Processor",
 210                         "0c" : "Serial bus",
 211                         "0d" : "Wireless",
 212                         "0e" : "Intelligent controller",
 213                         "0f" : "Satellite communications controller",
 214                         "10" : "Encryption",
 215                         "11" : "Signal processing controller",
 216                         "ff" : "Unassigned class",
 217                 },
 218
 219                 "usb" : {
 220                         "00" : "Unclassified",
 221                         "01" : "Multimedia",
 222                         "02" : "Communication",
 223                         "03" : "Input device",
 224                         "05" : "Generic system peripheral",
 225                         "06" : "Image",
 226                         "07" : "Printer",
 227                         "08" : "Mass storage",
 228                         "09" : "Hub",
 229                         "0a" : "Communication",
 230                         "0b" : "Smart card",
 231                         "0d" : "Encryption",
 232                         "0e" : "Display",
 233                         "0f" : "Personal Healthcare",
 234                         "dc" : "Diagnostic Device",
 235                         "e0" : "Wireless",
 236                         "ef" : "Unclassified",
 237                         "fe" : "Unclassified",
 238                         "ff" : "Unclassified",
 239                 }
 240         }
 241
 242         def __cmp__(self, other):
 243                 return cmp(self.vendor, other.vendor) or \
 244                         cmp(self.model, other.model) or \
 245                         cmp(self.driver, other.driver)
 246
 247         @property
 248         def model(self):
 249                 return self._data.get("model")
 250
 251         @property
 252         def model_string(self):
 253                 cls = self.subsystem2class[self.subsystem]
 254
 255                 return cls().get_device(self.vendor, self.model)
 256
 257         @property
 258         def subsystem(self):
 259                 return self._data.get("subsystem")
 260
 261         @property
 262         def vendor(self):
 263                 return self._data.get("vendor")
 264
 265         @property
 266         def vendor_string(self):
 267                 cls = self.subsystem2class[self.subsystem]
 268
 269                 return cls().get_vendor(self.vendor)
 270
 271         @property
 272         def driver(self):
 273                 return self._data.get("driver")
 274
 275         @property
 276         def cls(self):
 277                 classid = self._data.get("deviceclass")
 278
 279                 if self.subsystem == "pci":
 280                         classid = classid[:-4]
 281                         if len(classid) == 1:
 282                                 classid = "0%s" % classid
 283
 284                 elif self.subsystem == "usb" and classid:
 285                         classid = classid.split("/")[0]
 286                         classid = "%02x" % int(classid)
 287
 288                 try:
 289                         return self.classid2name[self.subsystem][classid]
 290                 except KeyError:
 291                         return "N/A"
 292
 293
 294 class Profile(ProfileDict):
 295         def __init__(self, profile_blob):
 296                 ProfileDict.__init__(self, profile_blob.get("profile"))
 297
 298                 self.public_id = profile_blob.get("public_id")
 299                 self.updated = profile_blob.get("updated")
 300                 self._geoip = profile_blob.get("geoip", None)
 301
 302         def __repr__(self):
 303                 return "<%s %s>" % (self.__class__.__name__, self.public_id)
 304
 305         @property
 306         def cpu(self):
 307                 return ProfileCPU(self._data.get("cpu"))
 308
 309         @property
 310         def devices(self):
 311                 devices = []
 312                 for d in self._data.get("devices"):
 313                         d = ProfileDevice(d)
 314
 315                         if d.driver in ("pcieport", "usb", "hub"):
 316                                 continue
 317
 318                         devices.append(d)
 319
 320                 return devices
 321
 322         @property
 323         def hypervisor(self):
 324                 if self.virtual:
 325                         return ProfileHypervisor(self._data.get("hypervisor"))
 326
 327         @property
 328         def virtual(self):
 329                 return self.system.get("virtual")
 330
 331         @property
 332         def system(self):
 333                 return self._data.get("system")
 334
 335         @property
 336         def release(self):
 337                 return self.system.get("release")
 338
 339         @property
 340         def kernel(self):
 341                 return self.system.get("kernel_release")
 342
 343         @property
 344         def memory(self):
 345                 return self.system.get("memory")
 346
 347         @property
 348         def friendly_memory(self):
 349                 units = ("k", "M", "G", "T")
 350
 351                 mem = self.memory
 352                 i = 0
 353                 while mem >= 1024:
 354                         mem /= 1024
 355                         i += 1
 356
 357                 return "%d%s" % (round(mem, 0), units[i])
 358
 359         @property
 360         def root_size(self):
 361                 return self.system.get("root_size") or 0
 362
 363         @property
 364         def friendly_root_size(self):
 365                 units = ("k", "M", "G", "T")
 366
 367                 size = self.root_size
 368                 if not size:
 369                         return
 370
 371                 i = 0
 372                 while size >= 1024:
 373                         size /= 1024
 374                         i += 1
 375
 376                 return "%d%s" % (round(size, 0), units[i])
 377
 378         @property
 379         def vendor(self):
 380                 return self.system.get("vendor")
 381
 382         @property
 383         def model(self):
 384                 return self.system.get("model")
 385
 386         @property
 387         def country_code(self):
 388                 if self._geoip:
 389                         return self._geoip["country_code"].lower()
 390
 391                 return "unknown"
 392
 393         @property
 394         def network(self):
 395                 network = self._data.get("network", None)
 396                 if network:
 397                         return ProfileNetwork(network)
 398
 399
 400 class Stasy(object):
 401         __metaclass__ = Singleton
 402
 403         def __init__(self):
 404                 # Initialize database connection
 405                 self._conn = pymongo.Connection(DATABASE_HOST)
 406                 self._db = self._conn[DATABASE_NAME]
 407
 408         def get_profile_count(self):
 409                 # XXX need to implement something to get profiles updated since
 410                 # a given date
 411
 412                 # All distinct profiles (based on public_id)
 413                 # XXX possibly bad performance
 414                 return len(self._db.profiles.distinct("public_id"))
 415
 416         def get_archives_count(self):
 417                 return self._db.archives.count()
 418
 419         #def _get_profile_cursor(self, public_id):
 420         #       c = self._db.profiles.find({ "public_id" : public_id })
 421         #       c.sort("updated", pymongo.ASCENDING)
 422         #
 423         #       return c
 424
 425         def profile_exists(self, public_id):
 426                 return self.query({ "public_id" : public_id }).count() >= 1
 427
 428         def get_profile(self, public_id):
 429                 p = None
 430                 # XXX should only find one object in the end
 431                 for p in self.query({ "public_id" : public_id }):
 432                         p = Profile(p)
 433
 434                 return p
 435
 436         def get_profiles(self):
 437                 # XXX needs nicer database query
 438                 profiles = []
 439                 for p in self._db.profiles.find():
 440                         if not p.get("public_id") in profiles:
 441                                 profiles.append(p.get("public_id"))
 442
 443                 return profiles
 444
 445         def query(self, query, archives=False, no_virt=False, all=False, fields=None):
 446                 db = self._db.profiles
 447
 448                 if archives:
 449                         db = self.db.archives
 450
 451                 if not all:
 452                         # XXX cannot use the index?
 453                         query.update({ "profile" : { "$exists" : True }})
 454
 455                 if no_virt:
 456                         query.update({ "profile.system.virtual" : False })
 457
 458                 logging.debug("Executing query: %s" % query)
 459
 460                 return db.find(query, fields=fields)
 461
 462         @property
 463         def secret_ids(self):
 464                 return self._db.profiles.distinct("secret_id")
 465
 466         @property
 467         def cpus(self):
 468                 return self.query({}, no_virt=True).distinct("profile.cpu")
 469
 470         @property
 471         def cpu_vendors(self):
 472                 return self.query({}, no_virt=True).distinct("profile.cpu.vendor")
 473
 474         @property
 475         def cpu_vendors_map(self):
 476                 cpus = {}
 477
 478                 for vendor in self.cpu_vendors:
 479                         friendly_vendor = CPU_VENDORS.get(vendor, vendor)
 480                         cpus[friendly_vendor] = \
 481                                 self.query({
 482                                         "profile.cpu.vendor" : vendor
 483                                 }, no_virt=True).count()
 484
 485                 return cpus
 486
 487         @property
 488         def cpu_speed_average(self):
 489                 speed = 0
 490
 491                 all = self.query({}, no_virt=True)
 492
 493                 # XXX ugly. needs to be done by group()
 494                 for m in all:
 495                         if not m.has_key("profile"):
 496                                 continue
 497                         speed += m.get("profile").get("cpu").get("speed")
 498
 499                 return (speed / all.count())
 500
 501         def get_cpu_bogomips_accumulated(self):
 502                 profiles = self.query({}, no_virt=True, fields=["profile.cpu.bogomips"])
 503
 504                 return sum([int(o["profile"]["cpu"]["bogomips"]) for o in profiles])
 505
 506         @property
 507         def cpu_speed_map(self):
 508                 cpu_speeds = {}
 509
 510                 for i in range(len(CPU_SPEED_CONSTRAINTS) - 1):
 511                         min, max = CPU_SPEED_CONSTRAINTS[i:i+2]
 512
 513                         cpu_speeds[min, max] = \
 514                                 self.query({
 515                                         "profile.cpu.speed" : {
 516                                                 "$gte" : min, "$lt" : max
 517                                         }
 518                                 }, no_virt=True).count()
 519
 520                 return cpu_speeds
 521
 522         def get_cpu_cores_map(self):
 523                 cores = {}
 524
 525                 for i in CPU_CORES:
 526                         cores[i] = \
 527                                 self.query({
 528                                         "profile.cpu.count" : i,
 529                                 }, no_virt=True).count()
 530
 531                 return cores
 532
 533         def get_memory_map(self):
 534                 memory = {}
 535
 536                 for i in range(len(MEMORY_CONSTRAINTS) - 1):
 537                         min, max = MEMORY_CONSTRAINTS[i:i+2]
 538
 539                         memory[min, max] = \
 540                                 self.query(
 541                                         { "profile.system.memory" : {
 542                                                 "$gte" : min * 1024, "$lt" : max * 1024
 543                                         }
 544                                 }, no_virt=True).count()
 545
 546                 return memory
 547
 548         @property
 549         def memory_average(self):
 550                 memory = 0
 551
 552                 all = self.query({}, no_virt=True)
 553
 554                 # XXX ugly. needs to be done by group()
 555                 for m in all:
 556                         if not m.has_key("profile"):
 557                                 continue
 558                         memory += int(m.get("profile").get("system").get("memory"))
 559
 560                 return (memory / all.count()) / 1024
 561
 562         @property
 563         def hypervisor_vendors(self):
 564                 return self.query({}).distinct("profile.hypervisor.vendor")
 565
 566         @property
 567         def hypervisor_map(self):
 568                 hypervisors = {}
 569
 570                 for hypervisor in self.hypervisor_vendors:
 571                         hypervisors[hypervisor] = \
 572                                 self.query({
 573                                         "profile.hypervisor.vendor" : hypervisor
 574                                 }).count()
 575
 576                 return hypervisors
 577
 578         @property
 579         def hypervisor_models(self):
 580                 return self.query({}).distinct("profile.hypervisor.model")
 581
 582         @property
 583         def virtual_map(self):
 584                 virtual = {
 585                         True: None,
 586                         False: None,
 587                 }
 588
 589                 for k in virtual.keys():
 590                         virtual[k] = \
 591                                 self.query({ "profile.system.virtual": k }).count()
 592
 593                 return virtual
 594
 595         @property
 596         def languages(self):
 597                 return self.query({}).distinct("profile.system.language")
 598
 599         def get_language_map(self):
 600                 languages = {}
 601
 602                 for language in self.languages:
 603                         languages[language] = \
 604                                 self.query({
 605                                         "profile.system.language" : language
 606                                 }).count()
 607
 608                 return languages
 609
 610         @property
 611         def vendors(self):
 612                 return self.query({}).distinct("profile.system.vendor")
 613
 614         @property
 615         def vendor_map(self):
 616                 vendors = {}
 617
 618                 for vendor in self.vendors:
 619                         vendors[vendor] = \
 620                                 self.query({
 621                                         "profile.system.vendor" : vendor
 622                                 }).count()
 623
 624                 return vendors
 625
 626         @property
 627         def models(self):
 628                 return self.query({}).distinct("profile.system.model")
 629
 630         @property
 631         def model_map(self):
 632                 models = {}
 633
 634                 for model in self.models:
 635                         models[model] = \
 636                                 self.query({
 637                                         "profile.system.model" : model
 638                                 }).count()
 639
 640                 return models
 641
 642         @property
 643         def arches(self):
 644                 return self.query({}).distinct("profile.cpu.arch")
 645
 646         @property
 647         def arch_map(self):
 648                 arches = {}
 649
 650                 for arch in self.arches:
 651                         arches[arch] = \
 652                                 self.query({
 653                                         "profile.cpu.arch" : arch
 654                                 }).count()
 655
 656                 return arches
 657
 658         @property
 659         def kernels(self):
 660                 return self.query({}).distinct("profile.system.kernel_release")
 661
 662         @property
 663         def kernel_map(self):
 664                 kernels = {}
 665
 666                 for kernel in self.kernels:
 667                         kernels[kernel] = \
 668                                 self.query({
 669                                         "profile.system.kernel_release" : kernel
 670                                 }).count()
 671
 672                 return kernels
 673
 674         @property
 675         def releases(self):
 676                 return self.query({}).distinct("profile.system.release")
 677
 678         @property
 679         def release_map(self):
 680                 releases = {}
 681
 682                 for release in self.releases:
 683                         releases[release] = \
 684                                 self.query({
 685                                         "profile.system.release" : release
 686                                 }).count()
 687
 688                 return releases
 689
 690         def get_device_percentage(self, bus, vendor_id, model_id):
 691                 profiles_with_device = self.query({
 692                         "profile.devices.subsystem" : bus,
 693                         "profile.devices.vendor" : vendor_id,
 694                         "profile.devices.model" : model_id,
 695                 })
 696
 697                 profiles_all = self.query({})
 698
 699                 if not profiles_all.count():
 700                         return 0
 701
 702                 return profiles_with_device.count() / profiles_all.count()
 703
 704         def get_cpu_flag_map(self, flags):
 705                 # XXX needs a cleanup
 706
 707                 _flags = { True : 0 }
 708
 709                 if type(flags) == type("a"):
 710                         flags = [flags]
 711
 712                 for flag in flags:
 713                         _flags[True] += \
 714                                 self.query({
 715                                         "profile.cpu.flags" : flag,
 716                                 }, no_virt=True).count()
 717
 718                 _flags[False] = self.query({}, no_virt=True).count() - _flags[True]
 719
 720                 return _flags
 721
 722         @property
 723         def geo_locations(self):
 724                 return [code.lower() for code in self.query({}, all=True).distinct("geoip.country_code")]
 725
 726         def get_geo_location_map(self):
 727                 geo_locations = {}
 728
 729                 count = 0
 730                 for geo_location in self.geo_locations:
 731                         geo_locations[geo_location] = \
 732                                 self.query({
 733                                         "geoip.country_code" : geo_location.upper()
 734                                 }, all=True).count()
 735
 736                         count += geo_locations[geo_location]
 737
 738                 for geo_location in geo_locations.keys():
 739                         geo_locations[geo_location] /= count
 740
 741                 return geo_locations
 742
 743         def get_models_by_vendor(self, subsystem, vendor_id):
 744                 devices = []
 745
 746                 profiles_all = self.query({})
 747
 748                 for profile in profiles_all:
 749                         if not profile.has_key("profile"):
 750                                 continue
 751
 752                         profile = Profile(profile)
 753
 754                         for device in profile.devices:
 755                                 if not device.vendor == vendor_id:
 756                                         continue
 757
 758                                 if not device in devices:
 759                                         devices.append(device)
 760
 761                 return devices
 762
 763         def get_network_zones_map(self):
 764                 zones = { "green" : 0, "blue" : 0, "orange" : 0, "red" : 0 }
 765
 766                 all = self.query({ "profile.network" : { "$exists" : True }})
 767
 768                 for zone in zones.keys():
 769                         zones[zone] = self.query({
 770                                 "profile.network.%s" % zone : True,
 771                         }).count() / all.count()
 772
 773                 return zones
 774
 775         def get_profile_ratio(self):
 776                 return (self.query({}).count(), self.get_profile_count())
 777
 778         def get_updated_since(self, since, _query={}):
 779                 since = datetime.datetime.utcnow() - datetime.timedelta(**since)
 780
 781                 query = { "updated" : { "$gte" : since }}
 782                 query.update(_query)
 783
 784                 return self.query(query)
 785
 786         def get_updates_by_release_since(self, since):
 787                 updates = {}
 788
 789                 for release in self.releases:
 790                         updates[release] = self.get_updated_since(since,
 791                                 { "profile.system.release" : release }).count()
 792
 793                 return updates
 794
 795
 796 if __name__ == "__main__":
 797         s = Stasy()
 798