From 372efc192b0054e07e6e7107a3049166ea2c5b78 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Thu, 16 Dec 2010 17:35:07 +0100 Subject: [PATCH] Initial import of fireinfo display. --- www/templates/modules/stasy-table.html | 8 + www/templates/stasy-profile.html | 112 ++++++-- www/templates/stasy-stats-cpus.html | 7 + www/templates/stasy-stats-virtual.html | 9 + www/templates/stasy-stats.html | 2 + www/webapp/__init__.py | 3 + www/webapp/backend/__init__.py | 1 + www/webapp/backend/stasy.py | 337 +++++++++++++++++++++++++ www/webapp/handlers_stasy.py | 15 ++ www/webapp/stasy.py | 179 ------------- www/webapp/ui_modules.py | 18 ++ 11 files changed, 495 insertions(+), 196 deletions(-) create mode 100644 www/templates/modules/stasy-table.html create mode 100644 www/templates/stasy-stats-cpus.html create mode 100644 www/templates/stasy-stats-virtual.html create mode 100644 www/templates/stasy-stats.html create mode 100644 www/webapp/backend/stasy.py delete mode 100644 www/webapp/stasy.py diff --git a/www/templates/modules/stasy-table.html b/www/templates/modules/stasy-table.html new file mode 100644 index 0000000..267179a --- /dev/null +++ b/www/templates/modules/stasy-table.html @@ -0,0 +1,8 @@ + + {% for k, v in items %} + + + + + {% end %} +
{{ k }}{{ "%.2f" % v }}%
diff --git a/www/templates/stasy-profile.html b/www/templates/stasy-profile.html index 15ed6c5..a3b4133 100644 --- a/www/templates/stasy-profile.html +++ b/www/templates/stasy-profile.html @@ -24,7 +24,7 @@ {{ _("Architecture") }} - {{ profile.arch }} + {{ profile.cpu.arch }} @@ -36,6 +36,37 @@ + + {% if profile.hypervisor %} +

{{ _("Hypervisor") }}

+ + + + + + + + + + + + + + +
+ {{ _("Vendor") }} + + {{ profile.hypervisor.vendor }} +
+ {{ _("Model") }} + + {{ profile.hypervisor.model or _("Unknown") }} +
+ {{ _("Type") }} + + {{ profile.hypervisor.type }} +
+ {% end %}

{{ _("Hardware") }}

@@ -49,7 +80,7 @@ {{ _("Vendor") }} @@ -57,9 +88,10 @@ {{ _("Model") }} + + + + + + + + + + + + + + - {% for device in profile.devices %} - - - - - {% end %} + + +
- {{ profile.cpu.vendor }} + {{ profile.cpu.vendor }}
- - {{ profile.cpu._data.get("model_string", _("Not available")) }} + {{ profile.cpu.model_string or _("Not available") }}
+ {{ _("Supports x86_64") }} + + {% if profile.cpu.capable_64bit %} + YES + {% else %} + NO + {% end %} +
+ {{ _("Supports PAE") }} + + {% if profile.cpu.capable_pae %} + YES + {% else %} + NO + {% end %} +
+ {{ _("Supports VT-x/AMD-V") }} + + {% if profile.cpu.capable_virt %} + YES + {% else %} + NO + {% end %} +
{{ _("Memory") }} @@ -99,19 +168,28 @@ {{ _("Devices") }}
- {{ device.subsystem.upper() }} - - {{ device.vendor }}:{{ device.model }} • - {{ device.vendor_string }} - {{ device.model_string }} -
+ + {% for device in profile.devices %} + + + + + + + {% end %} +
+ {{ device.subsystem.upper() }} + + {{ device.vendor_string }} + + {{ device.model_string }} + + {{ device.driver or " " }} +
+

diff --git a/www/templates/stasy-stats-cpus.html b/www/templates/stasy-stats-cpus.html new file mode 100644 index 0000000..da8a795 --- /dev/null +++ b/www/templates/stasy-stats-cpus.html @@ -0,0 +1,7 @@ +{% extends "stasy-base-1.html" %} + +{% block title %}{% end block %} + +{% block content %} + {{ modules.StasyTable(cpu_vendors) }} +{% end block %} diff --git a/www/templates/stasy-stats-virtual.html b/www/templates/stasy-stats-virtual.html new file mode 100644 index 0000000..c4442dc --- /dev/null +++ b/www/templates/stasy-stats-virtual.html @@ -0,0 +1,9 @@ +{% extends "stasy-base-1.html" %} + +{% block title %}{% end block %} + +{% block content %} + {{ modules.StasyTable(is_virtual) }} + + {{ modules.StasyTable(hypervisor_vendors) }} +{% end block %} diff --git a/www/templates/stasy-stats.html b/www/templates/stasy-stats.html new file mode 100644 index 0000000..8d0580e --- /dev/null +++ b/www/templates/stasy-stats.html @@ -0,0 +1,2 @@ +{% extends "stasy-base-1.html" %} + diff --git a/www/webapp/__init__.py b/www/webapp/__init__.py index 550161a..20cc5d3 100644 --- a/www/webapp/__init__.py +++ b/www/webapp/__init__.py @@ -38,6 +38,7 @@ class Application(tornado.web.Application): "ReleaseItem" : ReleaseItemModule, "SidebarBanner" : SidebarBannerModule, "SidebarRelease" : SidebarReleaseModule, + "StasyTable" : StasyTableModule, "TrackerPeerList": TrackerPeerListModule, }, xsrf_cookies = True, @@ -118,6 +119,8 @@ class Application(tornado.web.Application): self.add_handlers(r"stasy\.ipfire\.org", [ (r"/", StasyIndexHandler), (r"/profile/([a-z0-9]{40})", StasyProfileHandler), + (r"/statistics/cpu", StasyStatsCPUHandler), + (r"/statistics/virtual", StasyStatsVirtualHandler), ] + static_handlers) # source.ipfire.org diff --git a/www/webapp/backend/__init__.py b/www/webapp/backend/__init__.py index c685c19..3fb3ebb 100644 --- a/www/webapp/backend/__init__.py +++ b/www/webapp/backend/__init__.py @@ -9,4 +9,5 @@ from news import News from planet import Planet from releases import Releases from settings import Settings as Config +from stasy import Stasy from tracker import Tracker diff --git a/www/webapp/backend/stasy.py b/www/webapp/backend/stasy.py new file mode 100644 index 0000000..6f13a98 --- /dev/null +++ b/www/webapp/backend/stasy.py @@ -0,0 +1,337 @@ +#!/usr/bin/python + +import hwdata +import logging +import pymongo + +from misc import Singleton + +DATABASE_HOST = ["irma.ipfire.org", "madeye.ipfire.org"] +DATABASE_NAME = "stasy" + +MEMORY_CONSTRAINTS = (0, 64, 128, 256, 512, 1024, 2048, 4096, 8128, 16384) + +class ProfileDict(object): + def __init__(self, data): + self._data = data + + +class ProfileCPU(ProfileDict): + @property + def arch(self): + return self._data.get("arch") + + @property + def vendor(self): + return self._data.get("vendor") + + @property + def model(self): + return self._data.get("model") + + @property + def model_string(self): + return self._data.get("model_string") + + @property + def flags(self): + return self._data.get("flags") + + @property + def capable_64bit(self): + return "lm" in self.flags + + @property + def capable_pae(self): + return "pae" in self.flags + + @property + def capable_virt(self): + return "vmx" in self.flags or "svm" in self.flags + + +class ProfileHypervisor(ProfileDict): + def __repr__(self): + return "<%s %s-%s>" % (self.__class__.__name__, self.vendor, self.type) + + @property + def vendor(self): + return self._data.get("vendor") + + @property + def model(self): + return self._data.get("model") + + @property + def type(self): + return self._data.get("type") + + +class ProfileDevice(ProfileDict): + subsystem2class = { + "pci" : hwdata.PCI, + "usb" : hwdata.USB, + } + + @property + def model(self): + return self._data.get("model") + + @property + def model_string(self): + cls = self.subsystem2class[self.subsystem] + + return cls().get_device(self.vendor, self.model) + + @property + def subsystem(self): + return self._data.get("subsystem") + + @property + def vendor(self): + return self._data.get("vendor") + + @property + def vendor_string(self): + cls = self.subsystem2class[self.subsystem] + + return cls().get_vendor(self.vendor) + + @property + def driver(self): + return self._data.get("driver") + + +class Profile(ProfileDict): + def __init__(self, public_id, updated, data): + ProfileDict.__init__(self, data) + + self.public_id = public_id + self.updated = updated + + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, self.public_id) + + @property + def cpu(self): + return ProfileCPU(self._data.get("cpu")) + + @property + def devices(self): + devices = [] + for d in self._data.get("devices"): + d = ProfileDevice(d) + + if d.driver in ("usb", "hub"): + continue + + devices.append(d) + + return devices + + @property + def hypervisor(self): + if self.virtual: + return ProfileHypervisor(self._data.get("hypervisor")) + + @property + def virtual(self): + return self.system.get("virtual") + + @property + def system(self): + return self._data.get("system") + + @property + def release(self): + return self.system.get("release") + + @property + def kernel(self): + return self.system.get("kernel_release") + + @property + def memory(self): + return self.system.get("memory") + + @property + def root_size(self): + return self.system.get("root_size") + + + +class Stasy(object): + __metaclass__ = Singleton + + def __init__(self): + # Initialize database connection + self._conn = pymongo.Connection(DATABASE_HOST) + self._db = self._conn[DATABASE_NAME] + + def get_profile_count(self): + # XXX need to implement something to get profiles updated since + # a given date + + # All distinct profiles (based on public_id) + return self._db.profiles.distinct("public_id").count() + + #def _get_profile_cursor(self, public_id): + # c = self._db.profiles.find({ "public_id" : public_id }) + # c.sort("updated", pymongo.ASCENDING) + # + # return c + + def get_profile(self, public_id): + # XXX should only find one object in the end + for p in self._db.profiles.find({ "public_id" : public_id }): + p = Profile(p.get("public_id"), p.get("updated"), p.get("profile")) + + return p + + def get_profiles(self): + # XXX needs nicer database query + profiles = [] + for p in self._db.profiles.find(): + if not p.get("public_id") in profiles: + profiles.append(p.get("public_id")) + + return profiles + + @property + def secret_ids(self): + return self._db.profiles.distinct("secret_id") + + @property + def cpus(self): + return self._db.profiles.distinct("profile.cpu") + + @property + def cpu_vendors(self): + return self._db.profiles.distinct("profile.cpu.vendor") + + @property + def cpu_map(self): + cpus = {} + + for vendor in self.cpu_vendors: + cpus[vendor] = \ + self._db.profiles.find({ + "profile.cpu.vendor" : vendor + }).count() + + return cpus + + @property + def memory_map(self): + memory = {} + + for i in range(len(MEMORY_CONSTRAINTS) - 1): + min, max = MEMORY_CONSTRAINTS[i:i+2] + + memory[min, max] = \ + self._db.profiles.find( + { "profile.system.memory" : { + "$gte" : min * 1024, "$lt" : max * 1024 + } + }).count() + + return memory + + @property + def memory_average(self): + memory = 0 + + all = self._db.profiles.find() + + # XXX ugly. needs to be done by group() + for m in all: + if not m.has_key("profile"): + continue + memory += int(m.get("profile").get("system").get("memory")) + + return (memory / all.count()) / 1024 + + @property + def hypervisor_vendors(self): + return self._db.profiles.distinct("profile.hypervisor.vendor") + + @property + def hypervisor_map(self): + hypervisors = {} + + for hypervisor in self.hypervisor_vendors: + hypervisors[hypervisor] = \ + self._db.profiles.find({ + "profile.hypervisor.vendor" : hypervisor + }).count() + + return hypervisors + + @property + def hypervisor_models(self): + return self._db.profiles.distinct("profile.hypervisor.model") + + @property + def virtual_map(self): + virtual = { + True: None, + False: None, + } + + for k in virtual.keys(): + virtual[k] = \ + self._db.profiles.find({ "profile.system.virtual": k }).count() + + return virtual + + @property + def languages(self): + return self._db.profiles.distinct("profile.system.language") + + @property + def vendors(self): + return self._db.profiles.distinct("profile.system.vendor") + + @property + def vendor_map(self): + vendors = {} + + for vendor in self.vendors: + vendors[vendor] = \ + self._db.profiles.find({ + "profile.system.vendor" : vendor + }).count() + + return vendors + + @property + def models(self): + return self._db.profiles.distinct("profile.system.model") + + + +if __name__ == "__main__": + s = Stasy() + + print s.get_profile("0" * 40) + print s.cpu_vendors + for id in s.secret_ids: + print "\t", id + + #for p in s._db.profiles.find(): + # print p + + print s.cpu_map + print s.memory_map + print s.memory_average + print s.hypervisor_vendors + print s.hypervisor_models + print s.languages + print s.vendors + print s.vendor_map + print s.models + print s.cpus + + p = s.get_profile("0b5f4fe2162fdfbfa29b632610e317078fa70d34") + print p + print p.hypervisor diff --git a/www/webapp/handlers_stasy.py b/www/webapp/handlers_stasy.py index 5dc0a30..7b8c506 100644 --- a/www/webapp/handlers_stasy.py +++ b/www/webapp/handlers_stasy.py @@ -2,6 +2,8 @@ import tornado.web +import backend + from handlers_base import * class StasyBaseHandler(BaseHandler): @@ -24,3 +26,16 @@ class StasyProfileHandler(StasyBaseHandler): raise tornado.web.HTTPError(404, "Profile not found: %s" % profile_id) self.render("stasy-profile.html", profile=profile) + + +class StasyStatsCPUHandler(StasyBaseHandler): + def get(self): + return self.render("stasy-stats-cpus.html", + cpu_vendors = self.stasy.cpu_map) + + +class StasyStatsVirtualHandler(StasyBaseHandler): + def get(self): + return self.render("stasy-stats-virtual.html", + hypervisor_vendors = self.stasy.hypervisor_map, + is_virtual = self.stasy.virtual_map) diff --git a/www/webapp/stasy.py b/www/webapp/stasy.py deleted file mode 100644 index a64c1d9..0000000 --- a/www/webapp/stasy.py +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/python - -import logging -import pymongo - -DATABASE_HOST = ["irma.ipfire.org", "madeye.ipfire.org"] -DATABASE_NAME = "stasy" - -class ProfileDict(object): - def __init__(self, data): - self._data = data - - logging.debug("New: %s" % self._data) - - def __repr__(self): - return self.__str__() - - def __str__(self): - return "<%s %s>" % (self.__class__.__name__, self.public_id) - - def __getattr__(self, key): - try: - return self._data[key] - except KeyError: - raise AttributeError, key - - def __setattr(self, key, val): - self._data[key] = val - - -class ProfileCPU(ProfileDict): - @property - def capable_64bit(self): - return "lm" in self.flags - - @property - def capable_pae(self): - return "pae" in self.flags - - -class ProfileHypervisor(ProfileDict): - pass - - -class ProfileDevice(ProfileDict): - @property - def model_string(self): - return "XXX" - - @property - def vendor_string(self): - return "XXX" - - -class Profile(ProfileDict): - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.public_id) - - @property - def cpu(self): - return ProfileCPU(self._data["cpu"]) - - @property - def hypervisor(self): - return ProfileHypervisor(self._data["hypervisor"]) - - @property - def devices(self): - return [ProfileDevice(d) for d in self._data["devices"]] - - -class StasyDatabase(object): - def __init__(self): - # Initialize database connection - self._conn = pymongo.Connection(DATABASE_HOST) - self._db = self._conn[DATABASE_NAME] - - def get_profile_count(self): - # XXX need to implement something to get profiles updated since - # a given date - - # All distinct profiles (based on public_id) - c = self._db.profiles.find().distinct("public_id") - - return c.count() - - def _get_profile_cursor(self, public_id): - c = self._db.profiles.find({ "public_id" : public_id }) - c.sort("updated", pymongo.ASCENDING) - - return c - - def get_latest_profile(self, public_id): - # XXX still finds first one - for p in self._get_profile_cursor(public_id).limit(1): - return Profile(p) - - def get_profiles(self): - # XXX needs nicer database query - profiles = [] - for p in self._db.profiles.find(): - p = Profile(p) - if not p.public_id in profiles: - profiles.append(p.public_id) - - return profiles - - @property - def cpus(self): - return self._db.profiles.distinct("profile.cpu") - - @property - def cpu_vendors(self): - return self._db.profiles.distinct("profile.cpu.vendor") - - @property - def cpu_map(self): - cpus = {} - - for vendor in self.cpu_vendors: - cpus[vendor] = \ - self._db.profiles.find({ "profile.cpu.vendor" : vendor }).count() - - return cpus - - @property - def hypervisor_vendors(self): - return self._db.profiles.distinct("profile.hypervisor.vendor") - - @property - def hypervisor_models(self): - return self._db.profiles.distinct("profile.hypervisor.model") - - @property - def secret_ids(self): - return self._db.profiles.distinct("secret_id") - - @property - def languages(self): - return self._db.profiles.distinct("profile.system.language") - - @property - def vendors(self): - return self._db.profiles.distinct("profile.system.vendor") - - @property - def models(self): - return self._db.profiles.distinct("profile.system.model") - - -class Stasy(object): - def __init__(self): - self.db = StasyDatabase() - - def get_profile(self, public_id): - return self.db.get_latest_profile(public_id) - - def get_profiles(self): - return self.db.get_profiles() - - -if __name__ == "__main__": - s = Stasy() - - print s.get_profile("0" * 40) - print s.db.cpu_vendors - for id in s.db.secret_ids: - print "\t", id - - for p in s.db._db.profiles.find(): - print p - - print s.db.cpu_map - print s.db.hypervisor_vendors - print s.db.hypervisor_models - print s.db.languages - print s.db.vendors - print s.db.models - print s.db.cpus diff --git a/www/webapp/ui_modules.py b/www/webapp/ui_modules.py index 6bff06d..0ed5ccf 100644 --- a/www/webapp/ui_modules.py +++ b/www/webapp/ui_modules.py @@ -122,3 +122,21 @@ class TrackerPeerListModule(UIModule): return self.render_string("modules/tracker-peerlist.html", peers=[Row(p) for p in peers], percentages=percentages) + + +class StasyTableModule(UIModule): + def render(self, items): + hundred_percent = 0 + for v in items.values(): + hundred_percent += v + + if hundred_percent: + _items = [] + for k in sorted(items.keys()): + v = float(items[k] * 100) / hundred_percent + _items.append((k, v)) + items = _items + + print items + + return self.render_string("modules/stasy-table.html", items=items) -- 2.39.2