webapp/backend/stasy.py

   1 #!/usr/bin/python
   2
   3 from __future__ import division
   4
   5 import datetime
   6 import hwdata
   7 import logging
   8 import pymongo
   9 import re
  10
  11 from misc import Object
  12
  13 DATABASE_HOST = ["wilhelmina.ipfire.org", "miranda.ipfire.org", "falco.ipfire.org",]
  14 DATABASE_NAME = "stasy"
  15
  16 CPU_SPEED_CONSTRAINTS = (0, 500, 1000, 1500, 2000, 2500, 3000, 3500)
  17 MEMORY_CONSTRAINTS = (0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384)
  18
  19 CPU_VENDORS = {
  20         "AMDisbetter!" : "AMD",
  21         "AuthenticAMD" : "AMD",
  22         "CentaurHauls" : "VIA",
  23         "CyrixInstead" : "Cyrix",
  24         "GenuineIntel" : "Intel",
  25         "TransmetaCPU" : "Transmeta",
  26         "GenuineTMx86" : "Transmeta",
  27         "Geode by NSC" : "NSC",
  28         "NexGenDriven" : "NexGen",
  29         "RiseRiseRise" : "Rise",
  30         "SiS SiS SiS"  : "SiS",
  31         "SiS SiS SiS " : "SiS",
  32         "UMC UMC UMC " : "UMC",
  33         "VIA VIA VIA " : "VIA",
  34         "Vortex86 SoC" : "Vortex86",
  35 }
  36
  37 CPU_STRINGS = (
  38         # AMD
  39         (r"(AMD Athlon).*(XP).*", r"\1 \2"),
  40         (r"(AMD Phenom).* ([0-9]+) .*", r"\1 \2"),
  41         (r"(AMD Phenom).*", r"\1"),
  42         (r"(AMD Sempron).*", r"\1"),
  43         (r"AMD Athlon.* II X2 ([a-z0-9]+).*", r"AMD Athlon X2 \1"),
  44         (r"(Geode).*", r"\1"),
  45
  46         # Intel
  47         (r"Intel.*(Atom|Celeron).*CPU\s*([A-Z0-9]+) .*", r"Intel \1 \2"),
  48         (r"(Intel).*(Celeron).*", r"\1 \2"),
  49         (r"Intel.* Core.*2 Duo *CPU .* ([A-Z0-9]+) .*", r"Intel C2D \1"),
  50         (r"Intel.* Core.*2 CPU .* ([A-Z0-9]+) .*", r"Intel C2 \1"),
  51         (r"Intel.* Core.*2 Quad *CPU .* ([A-Z0-9]+) .*", r"Intel C2Q \1"),
  52         (r"Intel.* Xeon.* CPU .* ([A-Z0-9]+) .*", r"Intel Xeon \1"),
  53         (r"(Intel).*(Xeon).*", r"\1 \2"),
  54         (r"Intel.* Pentium.* (D|4) .*", r"Intel Pentium \1"),
  55         (r"Intel.* Pentium.* Dual .* ([A-Z0-9]+) .*", r"Intel Pentium Dual \1"),
  56         (r"Pentium.* Dual-Core .* ([A-Z0-9]+) .*", r"Intel Pentium Dual \1"),
  57         (r"(Pentium I{2,3}).*", r"Intel \1"),
  58         (r"(Celeron \(Coppermine\))", r"Intel Celeron"),
  59
  60         # VIA
  61         (r"(VIA \w*).*", r"\1"),
  62
  63         # Qemu
  64         (r"QEMU Virtual CPU version .*", r"QEMU CPU"),
  65
  66         # ARM
  67         (r"Feroceon .*", r"ARM Feroceon"),
  68 )
  69
  70 CPU_CORES = range(1, 9)
  71
  72 class ProfileDict(object):
  73         def __init__(self, data):
  74                 self._data = data
  75
  76
  77 class ProfileCPU(ProfileDict):
  78         @property
  79         def arch(self):
  80                 return self._data.get("arch")
  81
  82         @property
  83         def vendor(self):
  84                 vendor = self._data.get("vendor")
  85
  86                 return CPU_VENDORS.get(vendor, vendor)
  87
  88         @property
  89         def speed(self):
  90                 speed = self._data.get("speed")
  91
  92                 # ARM boxes do not report speed but
  93                 # bogomips equals speed
  94                 if not speed:
  95                         return self.bogomips
  96
  97                 return speed
  98
  99         @property
 100         def friendly_speed(self):
 101                 if self.speed < 1000:
 102                         return "%dMHz" % self.speed
 103
 104                 return "%.1fGHz" % round(self.speed / 1000, 1)
 105
 106         @property
 107         def bogomips(self):
 108                 return self._data.get("bogomips")
 109
 110         @property
 111         def model(self):
 112                 return self._data.get("model")
 113
 114         @property
 115         def model_string(self):
 116                 return self._data.get("model_string")
 117
 118         @property
 119         def flags(self):
 120                 return self._data.get("flags")
 121
 122         @property
 123         def count(self):
 124                 return self._data.get("count")
 125
 126         @property
 127         def capable_64bit(self):
 128                 return "lm" in self.flags
 129
 130         @property
 131         def capable_pae(self):
 132                 return "pae" in self.flags
 133
 134         @property
 135         def capable_virt(self):
 136                 return "vmx" in self.flags or "svm" in self.flags
 137
 138         @property
 139         def friendly_vendor(self):
 140                 s = self.model_string
 141                 for pattern, repl in CPU_STRINGS:
 142                         if re.match(pattern, s) is None:
 143                                 continue
 144                         return re.sub(pattern, repl, s)
 145
 146                 return s
 147
 148         @property
 149         def friendly_string(self):
 150                 s = self.friendly_vendor
 151
 152                 if self.speed:
 153                         s += " @ %s" % self.friendly_speed
 154
 155                 return s
 156
 157
 158 class ProfileHypervisor(ProfileDict):
 159         def __repr__(self):
 160                 return "<%s %s-%s>" % (self.__class__.__name__, self.vendor, self.type)
 161
 162         @property
 163         def vendor(self):
 164                 return self._data.get("vendor")
 165
 166         @property
 167         def type(self):
 168                 return self._data.get("type")
 169
 170
 171 class ProfileNetwork(ProfileDict):
 172         def has_zone(self, name):
 173                 return self._data.get(name)
 174
 175         @property
 176         def green(self):
 177                 return self._data.get("green")
 178
 179         @property
 180         def red(self):
 181                 return self._data.get("red")
 182
 183         @property
 184         def blue(self):
 185                 return self._data.get("blue")
 186
 187         @property
 188         def orange(self):
 189                 return self._data.get("orange")
 190
 191
 192 class ProfileDevice(ProfileDict):
 193         subsystem2class = {
 194                 "pci" : hwdata.PCI,
 195                 "usb" : hwdata.USB,
 196         }
 197
 198         classid2name = {
 199                 "pci" : {
 200                         "00" : "Unclassified",
 201                         "01" : "Mass storage",
 202                         "02" : "Network",
 203                         "03" : "Display",
 204                         "04" : "Multimedia",
 205                         "05" : "Memory controller",
 206                         "06" : "Bridge",
 207                         "07" : "Communication",
 208                         "08" : "Generic system peripheral",
 209                         "09" : "Input device",
 210                         "0a" : "Docking station",
 211                         "0b" : "Processor",
 212                         "0c" : "Serial bus",
 213                         "0d" : "Wireless",
 214                         "0e" : "Intelligent controller",
 215                         "0f" : "Satellite communications controller",
 216                         "10" : "Encryption",
 217                         "11" : "Signal processing controller",
 218                         "ff" : "Unassigned class",
 219                 },
 220
 221                 "usb" : {
 222                         "00" : "Unclassified",
 223                         "01" : "Multimedia",
 224                         "02" : "Communication",
 225                         "03" : "Input device",
 226                         "05" : "Generic system peripheral",
 227                         "06" : "Image",
 228                         "07" : "Printer",
 229                         "08" : "Mass storage",
 230                         "09" : "Hub",
 231                         "0a" : "Communication",
 232                         "0b" : "Smart card",
 233                         "0d" : "Encryption",
 234                         "0e" : "Display",
 235                         "0f" : "Personal Healthcare",
 236                         "dc" : "Diagnostic Device",
 237                         "e0" : "Wireless",
 238                         "ef" : "Unclassified",
 239                         "fe" : "Unclassified",
 240                         "ff" : "Unclassified",
 241                 }
 242         }
 243
 244         def __cmp__(self, other):
 245                 return cmp(self.vendor, other.vendor) or \
 246                         cmp(self.model, other.model) or \
 247                         cmp(self.driver, other.driver)
 248
 249         @property
 250         def model(self):
 251                 return self._data.get("model")
 252
 253         @property
 254         def model_string(self):
 255                 cls = self.subsystem2class[self.subsystem]
 256
 257                 return cls().get_device(self.vendor, self.model)
 258
 259         @property
 260         def subsystem(self):
 261                 return self._data.get("subsystem")
 262
 263         @property
 264         def vendor(self):
 265                 return self._data.get("vendor")
 266
 267         @property
 268         def vendor_string(self):
 269                 cls = self.subsystem2class[self.subsystem]
 270
 271                 return cls().get_vendor(self.vendor)
 272
 273         @property
 274         def driver(self):
 275                 return self._data.get("driver")
 276
 277         @property
 278         def cls(self):
 279                 classid = self._data.get("deviceclass")
 280
 281                 if self.subsystem == "pci":
 282                         classid = classid[:-4]
 283                         if len(classid) == 1:
 284                                 classid = "0%s" % classid
 285
 286                 elif self.subsystem == "usb" and classid:
 287                         classid = classid.split("/")[0]
 288                         classid = "%02x" % int(classid)
 289
 290                 try:
 291                         return self.classid2name[self.subsystem][classid]
 292                 except KeyError:
 293                         return "N/A"
 294
 295
 296 class Profile(ProfileDict):
 297         def __init__(self, profile_blob):
 298                 ProfileDict.__init__(self, profile_blob.get("profile"))
 299
 300                 self.public_id = profile_blob.get("public_id")
 301                 self.updated = profile_blob.get("updated")
 302                 self._geoip = profile_blob.get("geoip", None)
 303
 304         def __repr__(self):
 305                 return "<%s %s>" % (self.__class__.__name__, self.public_id)
 306
 307         @property
 308         def cpu(self):
 309                 return ProfileCPU(self._data.get("cpu"))
 310
 311         @property
 312         def devices(self):
 313                 devices = []
 314                 for d in self._data.get("devices"):
 315                         d = ProfileDevice(d)
 316
 317                         if d.driver in ("pcieport", "usb", "hub"):
 318                                 continue
 319
 320                         devices.append(d)
 321
 322                 return devices
 323
 324         @property
 325         def hypervisor(self):
 326                 if self.virtual:
 327                         return ProfileHypervisor(self._data.get("hypervisor"))
 328
 329         @property
 330         def virtual(self):
 331                 return self.system.get("virtual")
 332
 333         @property
 334         def system(self):
 335                 return self._data.get("system")
 336
 337         @property
 338         def release(self):
 339                 return self.system.get("release")
 340
 341         @property
 342         def kernel(self):
 343                 return self.system.get("kernel_release")
 344
 345         @property
 346         def memory(self):
 347                 return self.system.get("memory")
 348
 349         @property
 350         def friendly_memory(self):
 351                 units = ("k", "M", "G", "T")
 352
 353                 mem = self.memory
 354                 i = 0
 355                 while mem >= 1024:
 356                         mem /= 1024
 357                         i += 1
 358
 359                 return "%d%s" % (round(mem, 0), units[i])
 360
 361         @property
 362         def root_size(self):
 363                 return self.system.get("root_size") or 0
 364
 365         @property
 366         def friendly_root_size(self):
 367                 units = ("k", "M", "G", "T")
 368
 369                 size = self.root_size
 370                 if not size:
 371                         return
 372
 373                 i = 0
 374                 while size >= 1024:
 375                         size /= 1024
 376                         i += 1
 377
 378                 return "%d%s" % (round(size, 0), units[i])
 379
 380         @property
 381         def vendor(self):
 382                 return self.system.get("vendor")
 383
 384         @property
 385         def model(self):
 386                 return self.system.get("model")
 387
 388         @property
 389         def country_code(self):
 390                 if self._geoip:
 391                         return self._geoip["country_code"].lower()
 392
 393                 return "unknown"
 394
 395         @property
 396         def network(self):
 397                 network = self._data.get("network", None)
 398                 if network:
 399                         return ProfileNetwork(network)
 400
 401
 402 class Stasy(Object):
 403         def init(self):
 404                 # Initialize database connection
 405                 self._conn = pymongo.Connection(DATABASE_HOST)
 406                 self._db = self._conn[DATABASE_NAME]
 407
 408         def get_profile_count(self):
 409                 # XXX need to implement something to get profiles updated since
 410                 # a given date
 411
 412                 # All distinct profiles (based on public_id)
 413                 # XXX possibly bad performance
 414                 return len(self._db.profiles.distinct("public_id"))
 415
 416         # Shortcuts to database collections.
 417
 418         @property
 419         def archives(self):
 420                 return self._db.archives
 421
 422         @property
 423         def profiles(self):
 424                 return self._db.profiles
 425
 426         def get_archives_count(self):
 427                 return self._db.archives.count()
 428
 429         #def _get_profile_cursor(self, public_id):
 430         #       c = self._db.profiles.find({ "public_id" : public_id })
 431         #       c.sort("updated", pymongo.ASCENDING)
 432         #
 433         #       return c
 434
 435         def profile_exists(self, public_id):
 436                 return self.query({ "public_id" : public_id }).count() >= 1
 437
 438         def get_profile(self, public_id):
 439                 p = None
 440                 # XXX should only find one object in the end
 441                 for p in self.query({ "public_id" : public_id }):
 442                         p = Profile(p)
 443
 444                 return p
 445
 446         def get_profiles(self):
 447                 # XXX needs nicer database query
 448                 profiles = []
 449                 for p in self._db.profiles.find():
 450                         if not p.get("public_id") in profiles:
 451                                 profiles.append(p.get("public_id"))
 452
 453                 return profiles
 454
 455         def move_profiles(self, find):
 456                 """
 457                         Move all profiles by the "find" criteria.
 458                 """
 459                 for p in self.profiles.find(find):
 460                         self.archives.save(p)
 461                 self.profiles.remove(find)
 462
 463         def query(self, query, archives=False, no_virt=False, all=False, fields=None):
 464                 db = self._db.profiles
 465
 466                 if archives:
 467                         db = self.db.archives
 468
 469                 if not all:
 470                         # XXX cannot use the index?
 471                         query.update({ "profile" : { "$exists" : True }})
 472
 473                 if no_virt:
 474                         query.update({ "profile.system.virtual" : False })
 475
 476                 logging.debug("Executing query: %s" % query)
 477
 478                 return db.find(query, fields=fields)
 479
 480         @property
 481         def secret_ids(self):
 482                 return self._db.profiles.distinct("secret_id")
 483
 484         @property
 485         def cpus(self):
 486                 return self.query({}, no_virt=True).distinct("profile.cpu")
 487
 488         @property
 489         def cpu_vendors(self):
 490                 return self.query({}, no_virt=True).distinct("profile.cpu.vendor")
 491
 492         @property
 493         def cpu_vendors_map(self):
 494                 cpus = {}
 495
 496                 for vendor in self.cpu_vendors:
 497                         friendly_vendor = CPU_VENDORS.get(vendor, vendor)
 498                         cpus[friendly_vendor] = \
 499                                 self.query({
 500                                         "profile.cpu.vendor" : vendor
 501                                 }, no_virt=True).count()
 502
 503                 return cpus
 504
 505         @property
 506         def cpu_speed_average(self):
 507                 speed = 0
 508
 509                 all = self.query({}, no_virt=True)
 510
 511                 # XXX ugly. needs to be done by group()
 512                 for m in all:
 513                         if not m.has_key("profile"):
 514                                 continue
 515                         speed += m.get("profile").get("cpu").get("speed")
 516
 517                 return (speed / all.count())
 518
 519         def get_cpu_bogomips_accumulated(self):
 520                 profiles = self.query({}, no_virt=True, fields=["profile.cpu.bogomips"])
 521
 522                 return sum([int(o["profile"]["cpu"]["bogomips"]) for o in profiles])
 523
 524         @property
 525         def cpu_speed_map(self):
 526                 cpu_speeds = {}
 527
 528                 for i in range(len(CPU_SPEED_CONSTRAINTS) - 1):
 529                         min, max = CPU_SPEED_CONSTRAINTS[i:i+2]
 530
 531                         cpu_speeds[min, max] = \
 532                                 self.query({
 533                                         "profile.cpu.speed" : {
 534                                                 "$gte" : min, "$lt" : max
 535                                         }
 536                                 }, no_virt=True).count()
 537
 538                 return cpu_speeds
 539
 540         def get_cpu_cores_map(self):
 541                 cores = {}
 542
 543                 for i in CPU_CORES:
 544                         cores[i] = \
 545                                 self.query({
 546                                         "profile.cpu.count" : i,
 547                                 }, no_virt=True).count()
 548
 549                 return cores
 550
 551         def get_memory_map(self):
 552                 memory = {}
 553
 554                 for i in range(len(MEMORY_CONSTRAINTS) - 1):
 555                         min, max = MEMORY_CONSTRAINTS[i:i+2]
 556
 557                         memory[min, max] = \
 558                                 self.query(
 559                                         { "profile.system.memory" : {
 560                                                 "$gte" : min * 1024, "$lt" : max * 1024
 561                                         }
 562                                 }, no_virt=True).count()
 563
 564                 return memory
 565
 566         @property
 567         def memory_average(self):
 568                 memory = 0
 569
 570                 all = self.query({}, no_virt=True)
 571
 572                 # XXX ugly. needs to be done by group()
 573                 for m in all:
 574                         if not m.has_key("profile"):
 575                                 continue
 576                         memory += int(m.get("profile").get("system").get("memory"))
 577
 578                 return (memory / all.count()) / 1024
 579
 580         @property
 581         def hypervisor_vendors(self):
 582                 return self.query({}).distinct("profile.hypervisor.vendor")
 583
 584         @property
 585         def hypervisor_map(self):
 586                 hypervisors = {}
 587
 588                 for hypervisor in self.hypervisor_vendors:
 589                         hypervisors[hypervisor] = \
 590                                 self.query({
 591                                         "profile.hypervisor.vendor" : hypervisor
 592                                 }).count()
 593
 594                 return hypervisors
 595
 596         @property
 597         def hypervisor_models(self):
 598                 return self.query({}).distinct("profile.hypervisor.model")
 599
 600         @property
 601         def virtual_map(self):
 602                 virtual = {
 603                         True: None,
 604                         False: None,
 605                 }
 606
 607                 for k in virtual.keys():
 608                         virtual[k] = \
 609                                 self.query({ "profile.system.virtual": k }).count()
 610
 611                 return virtual
 612
 613         @property
 614         def languages(self):
 615                 return self.query({}).distinct("profile.system.language")
 616
 617         def get_language_map(self):
 618                 languages = {}
 619
 620                 for language in self.languages:
 621                         languages[language] = \
 622                                 self.query({
 623                                         "profile.system.language" : language
 624                                 }).count()
 625
 626                 return languages
 627
 628         @property
 629         def vendors(self):
 630                 return self.query({}).distinct("profile.system.vendor")
 631
 632         @property
 633         def vendor_map(self):
 634                 vendors = {}
 635
 636                 for vendor in self.vendors:
 637                         vendors[vendor] = \
 638                                 self.query({
 639                                         "profile.system.vendor" : vendor
 640                                 }).count()
 641
 642                 return vendors
 643
 644         @property
 645         def models(self):
 646                 return self.query({}).distinct("profile.system.model")
 647
 648         @property
 649         def model_map(self):
 650                 models = {}
 651
 652                 for model in self.models:
 653                         models[model] = \
 654                                 self.query({
 655                                         "profile.system.model" : model
 656                                 }).count()
 657
 658                 return models
 659
 660         @property
 661         def arches(self):
 662                 return self.query({}).distinct("profile.cpu.arch")
 663
 664         @property
 665         def arch_map(self):
 666                 arches = {}
 667
 668                 for arch in self.arches:
 669                         arches[arch] = \
 670                                 self.query({
 671                                         "profile.cpu.arch" : arch
 672                                 }).count()
 673
 674                 return arches
 675
 676         @property
 677         def kernels(self):
 678                 return self.query({}).distinct("profile.system.kernel_release")
 679
 680         @property
 681         def kernel_map(self):
 682                 kernels = {}
 683
 684                 for kernel in self.kernels:
 685                         kernels[kernel] = \
 686                                 self.query({
 687                                         "profile.system.kernel_release" : kernel
 688                                 }).count()
 689
 690                 return kernels
 691
 692         @property
 693         def releases(self):
 694                 return self.query({}).distinct("profile.system.release")
 695
 696         @property
 697         def release_map(self):
 698                 releases = {}
 699
 700                 for release in self.releases:
 701                         releases[release] = \
 702                                 self.query({
 703                                         "profile.system.release" : release
 704                                 }).count()
 705
 706                 return releases
 707
 708         def get_device_percentage(self, bus, vendor_id, model_id):
 709                 profiles_with_device = self.query({
 710                         "profile.devices.subsystem" : bus,
 711                         "profile.devices.vendor" : vendor_id,
 712                         "profile.devices.model" : model_id,
 713                 })
 714
 715                 profiles_all = self.query({})
 716
 717                 if not profiles_all.count():
 718                         return 0
 719
 720                 return profiles_with_device.count() / profiles_all.count()
 721
 722         def get_cpu_flag_map(self, flags):
 723                 # XXX needs a cleanup
 724
 725                 _flags = { True : 0 }
 726
 727                 if type(flags) == type("a"):
 728                         flags = [flags]
 729
 730                 for flag in flags:
 731                         _flags[True] += \
 732                                 self.query({
 733                                         "profile.cpu.flags" : flag,
 734                                 }, no_virt=True).count()
 735
 736                 _flags[False] = self.query({}, no_virt=True).count() - _flags[True]
 737
 738                 return _flags
 739
 740         @property
 741         def geo_locations(self):
 742                 return [code.lower() for code in self.query({}, all=True).distinct("geoip.country_code")]
 743
 744         def get_geo_location_map(self):
 745                 geo_locations = {}
 746
 747                 count = 0
 748                 for geo_location in self.geo_locations:
 749                         geo_locations[geo_location] = \
 750                                 self.query({
 751                                         "geoip.country_code" : geo_location.upper()
 752                                 }, all=True).count()
 753
 754                         count += geo_locations[geo_location]
 755
 756                 for geo_location in geo_locations.keys():
 757                         geo_locations[geo_location] /= count
 758
 759                 return geo_locations
 760
 761         def get_models_by_vendor(self, subsystem, vendor_id):
 762                 devices = []
 763
 764                 profiles_all = self.query({})
 765
 766                 for profile in profiles_all:
 767                         if not profile.has_key("profile"):
 768                                 continue
 769
 770                         profile = Profile(profile)
 771
 772                         for device in profile.devices:
 773                                 if not device.vendor == vendor_id:
 774                                         continue
 775
 776                                 if not device in devices:
 777                                         devices.append(device)
 778
 779                 return devices
 780
 781         def get_network_zones_map(self):
 782                 zones = { "green" : 0, "blue" : 0, "orange" : 0, "red" : 0 }
 783
 784                 all = self.query({ "profile.network" : { "$exists" : True }})
 785
 786                 for zone in zones.keys():
 787                         zones[zone] = self.query({
 788                                 "profile.network.%s" % zone : True,
 789                         }).count() / all.count()
 790
 791                 return zones
 792
 793         def get_profile_ratio(self):
 794                 return (self.query({}).count(), self.get_profile_count())
 795
 796         def get_updated_since(self, since, _query={}):
 797                 since = datetime.datetime.utcnow() - datetime.timedelta(**since)
 798
 799                 query = { "updated" : { "$gte" : since }}
 800                 query.update(_query)
 801
 802                 return self.query(query)
 803
 804         def get_updates_by_release_since(self, since):
 805                 updates = {}
 806
 807                 for release in self.releases:
 808                         updates[release] = self.get_updated_since(since,
 809                                 { "profile.system.release" : release }).count()
 810
 811                 return updates