[ipfire.org.git] / www / webapp / backend / stasy.py

#!/usr/bin/python

from __future__ import division

import datetime
import hwdata
import logging
import pymongo
import re

from misc import Singleton

DATABASE_HOST = ["irma.ipfire.org", "madeye.ipfire.org"]
DATABASE_NAME = "stasy"

CPU_SPEED_CONSTRAINTS = (0, 500, 1000, 1500, 2000, 2500, 3000, 3500)
MEMORY_CONSTRAINTS = (0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384)

CPU_VENDORS = {
	"AMDisbetter!" : "AMD",
	"AuthenticAMD" : "AMD",
	"CentaurHauls" : "VIA",
	"CyrixInstead" : "Cyrix",
	"GenuineIntel" : "Intel",
	"TransmetaCPU" : "Transmeta",
	"GenuineTMx86" : "Transmeta",
	"Geode by NSC" : "NSC",
	"NexGenDriven" : "NexGen",
	"RiseRiseRise" : "Rise",
	"SiS SiS SiS " : "SiS",
	"UMC UMC UMC " : "UMC",
	"VIA VIA VIA " : "VIA",
}

CPU_STRINGS = (
	# AMD
	(r"(AMD Athlon).*(XP).*", r"\1 \2"),
	(r"(AMD Phenom).* ([0-9]+) .*", r"\1 \2"),
	(r"(AMD Phenom).*", r"\1"),
	(r"(AMD Sempron).*", r"\1"),
	(r"AMD Athlon.* II X2 ([a-z0-9]+).*", r"AMD Athlon X2 \1"),
	(r"(Geode).*", r"\1"),

	# Intel
	(r"Intel.*(Atom|Celeron).*CPU\s*([A-Z0-9]+) .*", r"Intel \1 \2"),
	(r"(Intel).*(Celeron).*", r"\1 \2"),
	(r"Intel.* Core.*2 Duo *CPU .* ([A-Z0-9]+) .*", r"Intel C2D \1"),
	(r"Intel.* Core.*2 CPU .* ([A-Z0-9]+) .*", r"Intel C2 \1"),
	(r"Intel.* Core.*2 Quad *CPU .* ([A-Z0-9]+) .*", r"Intel C2Q \1"),
	(r"Intel.* Xeon.* CPU .* ([A-Z0-9]+) .*", r"Intel Xeon \1"),
	(r"(Intel).*(Xeon).*", r"\1 \2"),
	(r"Intel.* Pentium.* (D|4) .*", r"Intel Pentium \1"),
	(r"Intel.* Pentium.* Dual .* ([A-Z0-9]+) .*", r"Intel Pentium Dual \1"),
	(r"Pentium.* Dual-Core .* ([A-Z0-9]+) .*", r"Intel Pentium Dual \1"),
	(r"(Pentium I{2,3}).*", r"Intel \1"),
	(r"(Celeron \(Coppermine\))", r"Intel Celeron"),

	# VIA
	(r"(VIA \w*).*", r"\1"),
)

CPU_CORES = range(1, 9)

class ProfileDict(object):
	def __init__(self, data):
		self._data = data


class ProfileCPU(ProfileDict):
	@property
	def arch(self):
		return self._data.get("arch")

	@property
	def vendor(self):
		vendor = self._data.get("vendor")

		return CPU_VENDORS.get(vendor, vendor)

	@property
	def speed(self):
		return self._data.get("speed")

	@property
	def friendly_speed(self):
		if self.speed < 1000:
			return "%dMHz" % self.speed

		return "%.1fGHz" % round(self.speed / 1000, 1)

	@property
	def bogomips(self):
		return self._data.get("bogomips")

	@property
	def model(self):
		return self._data.get("model")

	@property
	def model_string(self):
		return self._data.get("model_string")

	@property
	def flags(self):
		return self._data.get("flags")

	@property
	def count(self):
		return self._data.get("count")

	@property
	def capable_64bit(self):
		return "lm" in self.flags

	@property
	def capable_pae(self):
		return "pae" in self.flags

	@property
	def capable_virt(self):
		return "vmx" in self.flags or "svm" in self.flags

	@property
	def friendly_vendor(self):
		s = self.model_string
		for pattern, repl in CPU_STRINGS:
			if re.match(pattern, s) is None:
				continue
			return re.sub(pattern, repl, s)

		return s

	@property
	def friendly_string(self):
		return "%s @ %s" % (self.friendly_vendor, self.friendly_speed)


class ProfileHypervisor(ProfileDict):
	def __repr__(self):
		return "<%s %s-%s>" % (self.__class__.__name__, self.vendor, self.type)

	@property
	def vendor(self):
		return self._data.get("vendor")

	@property
	def type(self):
		return self._data.get("type")


class ProfileNetwork(ProfileDict):
	def has_zone(self, name):
		return self._data.get(name)

	@property
	def green(self):
		return self._data.get("green")

	@property
	def red(self):
		return self._data.get("red")

	@property
	def blue(self):
		return self._data.get("blue")

	@property
	def orange(self):
		return self._data.get("orange")


class ProfileDevice(ProfileDict):
	subsystem2class = {
		"pci" : hwdata.PCI,
		"usb" : hwdata.USB,
	}

	classid2name = {
		"pci" : {
			"00" : "Unclassified",
			"01" : "Mass storage",
			"02" : "Network",
			"03" : "Display",
			"04" : "Multimedia",
			"05" : "Memory controller",
			"06" : "Bridge",
			"07" : "Communication",
			"08" : "Generic system peripheral",
			"09" : "Input device",
			"0a" : "Docking station",
			"0b" : "Processor",
			"0c" : "Serial bus",
			"0d" : "Wireless",
			"0e" : "Intelligent controller",
			"0f" : "Satellite communications controller",
			"10" : "Encryption",
			"11" : "Signal processing controller",
			"ff" : "Unassigned class",
		},
		
		"usb" : {
			"00" : "Unclassified",
			"01" : "Multimedia",
			"02" : "Communication",
			"03" : "Input device",
			"05" : "Generic system peripheral",
			"06" : "Image",
			"07" : "Printer",
			"08" : "Mass storage",
			"09" : "Hub",
			"0a" : "Communication",
			"0b" : "Smart card",
			"0d" : "Encryption",
			"0e" : "Display",
			"0f" : "Personal Healthcare",
			"dc" : "Diagnostic Device",
			"e0" : "Wireless",
			"ef" : "Unclassified",
			"fe" : "Unclassified",
			"ff" : "Unclassified",
		}
	}

	def __cmp__(self, other):
		return cmp(self.vendor, other.vendor) or \
			cmp(self.model, other.model) or \
			cmp(self.driver, other.driver)

	@property
	def model(self):
		return self._data.get("model")

	@property
	def model_string(self):
		cls = self.subsystem2class[self.subsystem]

		return cls().get_device(self.vendor, self.model)

	@property
	def subsystem(self):
		return self._data.get("subsystem")

	@property
	def vendor(self):
		return self._data.get("vendor")

	@property
	def vendor_string(self):
		cls = self.subsystem2class[self.subsystem]

		return cls().get_vendor(self.vendor)

	@property
	def driver(self):
		return self._data.get("driver")

	@property
	def cls(self):
		classid = self._data.get("deviceclass")

		if self.subsystem == "pci":
			classid = classid[:-4]
			if len(classid) == 1:
				classid = "0%s" % classid

		elif self.subsystem == "usb" and classid:
			classid = classid.split("/")[0]
			classid = "%02x" % int(classid)

		try:
			return self.classid2name[self.subsystem][classid]
		except KeyError:
			return "N/A"


class Profile(ProfileDict):
	def __init__(self, profile_blob):
		ProfileDict.__init__(self, profile_blob.get("profile"))

		self.public_id = profile_blob.get("public_id")
		self.updated = profile_blob.get("updated")
		self._geoip = profile_blob.get("geoip", None)

	def __repr__(self):
		return "<%s %s>" % (self.__class__.__name__, self.public_id)

	@property
	def cpu(self):
		return ProfileCPU(self._data.get("cpu"))

	@property
	def devices(self):
		devices = []
		for d in self._data.get("devices"):
			d = ProfileDevice(d)

			if d.driver in ("pcieport", "usb", "hub"):
				continue

			devices.append(d)

		return devices

	@property
	def hypervisor(self):
		if self.virtual:
			return ProfileHypervisor(self._data.get("hypervisor"))

	@property
	def virtual(self):
		return self.system.get("virtual")

	@property
	def system(self):
		return self._data.get("system")

	@property
	def release(self):
		return self.system.get("release")

	@property
	def kernel(self):
		return self.system.get("kernel_release")

	@property
	def memory(self):
		return self.system.get("memory")

	@property
	def friendly_memory(self):
		units = ("k", "M", "G", "T")

		mem = self.memory
		i = 0
		while mem >= 1024:
			mem /= 1024
			i += 1

		return "%d%s" % (round(mem, 0), units[i])

	@property
	def root_size(self):
		return self.system.get("root_size") or 0

	@property
	def friendly_root_size(self):
		units = ("k", "M", "G", "T")

		size = self.root_size
		if not size:
			return

		i = 0
		while size >= 1024:
			size /= 1024
			i += 1

		return "%d%s" % (round(size, 0), units[i])

	@property
	def vendor(self):
		return self.system.get("vendor")

	@property
	def model(self):
		return self.system.get("model")

	@property
	def country_code(self):
		if self._geoip:
			return self._geoip["country_code"].lower()

		return "unknown"

	@property
	def network(self):
		network = self._data.get("network", None)
		if network:
			return ProfileNetwork(network)


class Stasy(object):
	__metaclass__ = Singleton

	def __init__(self):
		# Initialize database connection
		self._conn = pymongo.Connection(DATABASE_HOST)
		self._db = self._conn[DATABASE_NAME]

	def get_profile_count(self):
		# XXX need to implement something to get profiles updated since
		# a given date

		# All distinct profiles (based on public_id)
		# XXX possibly bad performance
		return len(self._db.profiles.distinct("public_id"))

	def get_archives_count(self):
		return self._db.archives.count()

	#def _get_profile_cursor(self, public_id):
	#	c = self._db.profiles.find({ "public_id" : public_id })
	#	c.sort("updated", pymongo.ASCENDING)
	#
	#	return c

	def profile_exists(self, public_id):
		return self.query({ "public_id" : public_id }).count() >= 1

	def get_profile(self, public_id):
		p = None
		# XXX should only find one object in the end
		for p in self.query({ "public_id" : public_id }):
			p = Profile(p)

		return p

	def get_profiles(self):
		# XXX needs nicer database query
		profiles = []
		for p in self._db.profiles.find():
			if not p.get("public_id") in profiles:
				profiles.append(p.get("public_id"))

		return profiles

	def query(self, query, archives=False, no_virt=False, all=False, fields=None):
		db = self._db.profiles

		if archives:
			db = self.db.archives

		if not all:
			# XXX cannot use the index?
			query.update({ "profile" : { "$exists" : True }})

		if no_virt:
			query.update({ "profile.system.virtual" : False })

		logging.debug("Executing query: %s" % query)

		return db.find(query, fields=fields)

	@property
	def secret_ids(self):
		return self._db.profiles.distinct("secret_id")

	@property
	def cpus(self):
		return self.query({}, no_virt=True).distinct("profile.cpu")

	@property
	def cpu_vendors(self):
		return self.query({}, no_virt=True).distinct("profile.cpu.vendor")

	@property
	def cpu_vendors_map(self):
		cpus = {}

		for vendor in self.cpu_vendors:
			friendly_vendor = CPU_VENDORS.get(vendor, vendor)
			cpus[friendly_vendor] = \
				self.query({
					"profile.cpu.vendor" : vendor
				}, no_virt=True).count()

		return cpus

	@property
	def cpu_speed_average(self):
		speed = 0

		all = self.query({}, no_virt=True)

		# XXX ugly. needs to be done by group()
		for m in all:
			if not m.has_key("profile"):
				continue
			speed += m.get("profile").get("cpu").get("speed")

		return (speed / all.count())

	def get_cpu_bogomips_accumulated(self):
		profiles = self.query({}, no_virt=True, fields=["profile.cpu.bogomips"])

		return sum([int(o["profile"]["cpu"]["bogomips"]) for o in profiles])

	@property
	def cpu_speed_map(self):
		cpu_speeds = {}

		for i in range(len(CPU_SPEED_CONSTRAINTS) - 1):
			min, max = CPU_SPEED_CONSTRAINTS[i:i+2]

			cpu_speeds[min, max] = \
				self.query({
					"profile.cpu.speed" : {
						"$gte" : min, "$lt" : max
					}
				}, no_virt=True).count()

		return cpu_speeds

	def get_cpu_cores_map(self):
		cores = {}

		for i in CPU_CORES:
			cores[i] = \
				self.query({
					"profile.cpu.count" : i,
				}, no_virt=True).count()

		return cores

	def get_memory_map(self):
		memory = {}

		for i in range(len(MEMORY_CONSTRAINTS) - 1):
			min, max = MEMORY_CONSTRAINTS[i:i+2]

			memory[min, max] = \
				self.query(
					{ "profile.system.memory" : {
						"$gte" : min * 1024, "$lt" : max * 1024
					}
				}, no_virt=True).count()

		return memory

	@property
	def memory_average(self):
		memory = 0

		all = self.query({}, no_virt=True)

		# XXX ugly. needs to be done by group()
		for m in all:
			if not m.has_key("profile"):
				continue
			memory += int(m.get("profile").get("system").get("memory"))

		return (memory / all.count()) / 1024

	@property
	def hypervisor_vendors(self):
		return self.query({}).distinct("profile.hypervisor.vendor")

	@property
	def hypervisor_map(self):
		hypervisors = {}

		for hypervisor in self.hypervisor_vendors:
			hypervisors[hypervisor] = \
				self.query({
					"profile.hypervisor.vendor" : hypervisor
				}).count()

		return hypervisors

	@property
	def hypervisor_models(self):
		return self.query({}).distinct("profile.hypervisor.model")

	@property
	def virtual_map(self):
		virtual = {
			True: None,
			False: None,
		}

		for k in virtual.keys():
			virtual[k] = \
				self.query({ "profile.system.virtual": k }).count()

		return virtual

	@property
	def languages(self):
		return self.query({}).distinct("profile.system.language")

	def get_language_map(self):
		languages = {}

		for language in self.languages:
			languages[language] = \
				self.query({
					"profile.system.language" : language
				}).count()

		return languages

	@property
	def vendors(self):
		return self.query({}).distinct("profile.system.vendor")

	@property
	def vendor_map(self):
		vendors = {}

		for vendor in self.vendors:
			vendors[vendor] = \
				self.query({
					"profile.system.vendor" : vendor
				}).count()

		return vendors

	@property
	def models(self):
		return self.query({}).distinct("profile.system.model")

	@property
	def model_map(self):
		models = {}

		for model in self.models:
			models[model] = \
				self.query({
					"profile.system.model" : model
				}).count()

		return models

	@property
	def arches(self):
		return self.query({}).distinct("profile.cpu.arch")

	@property
	def arch_map(self):
		arches = {}

		for arch in self.arches:
			arches[arch] = \
				self.query({
					"profile.cpu.arch" : arch
				}).count()

		return arches

	@property
	def kernels(self):
		return self.query({}).distinct("profile.system.kernel_release")

	@property
	def kernel_map(self):
		kernels = {}

		for kernel in self.kernels:
			kernels[kernel] = \
				self.query({
					"profile.system.kernel_release" : kernel
				}).count()

		return kernels

	@property
	def releases(self):
		return self.query({}).distinct("profile.system.release")

	@property
	def release_map(self):
		releases = {}

		for release in self.releases:
			releases[release] = \
				self.query({
					"profile.system.release" : release
				}).count()

		return releases

	def get_device_percentage(self, bus, vendor_id, model_id):
		profiles_with_device = self.query({
			"profile.devices.subsystem" : bus,
			"profile.devices.vendor" : vendor_id,
			"profile.devices.model" : model_id,
		})

		profiles_all = self.query({})

		if not profiles_all.count():
			return 0

		return profiles_with_device.count() / profiles_all.count()

	def get_cpu_flag_map(self, flags):
		# XXX needs a cleanup

		_flags = { True : 0 }

		if type(flags) == type("a"):
			flags = [flags]

		for flag in flags:
			_flags[True] += \
				self.query({
					"profile.cpu.flags" : flag,
				}, no_virt=True).count()

		_flags[False] = self.query({}, no_virt=True).count() - _flags[True]

		return _flags

	@property
	def geo_locations(self):
		return [code.lower() for code in self.query({}, all=True).distinct("geoip.country_code")]

	def get_geo_location_map(self):
		geo_locations = {}

		count = 0
		for geo_location in self.geo_locations:
			geo_locations[geo_location] = \
				self.query({
					"geoip.country_code" : geo_location.upper()
				}, all=True).count()

			count += geo_locations[geo_location]

		for geo_location in geo_locations.keys():
			geo_locations[geo_location] /= count

		return geo_locations

	def get_models_by_vendor(self, subsystem, vendor_id):
		devices = []

		profiles_all = self.query({})

		for profile in profiles_all:
			if not profile.has_key("profile"):
				continue

			profile = Profile(profile)

			for device in profile.devices:
				if not device.vendor == vendor_id:
					continue

				if not device in devices:
					devices.append(device)

		return devices

	def get_network_zones_map(self):
		zones = { "green" : 0, "blue" : 0, "orange" : 0, "red" : 0 }

		all = self.query({ "profile.network" : { "$exists" : True }})

		for zone in zones.keys():
			zones[zone] = self.query({
				"profile.network.%s" % zone : True,
			}).count() / all.count()

		return zones

	def get_profile_ratio(self):
		return (self.query({}).count(), self.get_profile_count())

	def get_updated_since(self, since, _query={}):
		since = datetime.datetime.utcnow() - datetime.timedelta(**since)

		query = { "updated" : { "$gte" : since }}
		query.update(_query)

		return self.query(query)

	def get_updates_by_release_since(self, since):
		updates = {}

		for release in self.releases:
			updates[release] = self.get_updated_since(since,
				{ "profile.system.release" : release }).count()

		return updates


if __name__ == "__main__":
	s = Stasy()
Commit	Line	Data
372efc19 MT	1	#!/usr/bin/python
372efc19 MT	2
91a446f0 MT	3	from __future__ import division
91a446f0 MT	4
cbb9726a	5	import datetime
372efc19 MT	6	import hwdata
	7	import logging
	8	import pymongo
abc3bb31	9	import re
372efc19 MT	10
	11	from misc import Singleton
	12
	13	DATABASE_HOST = ["irma.ipfire.org", "madeye.ipfire.org"]
	14	DATABASE_NAME = "stasy"
	15
91a446f0	16	CPU_SPEED_CONSTRAINTS = (0, 500, 1000, 1500, 2000, 2500, 3000, 3500)
8e413e4f	17	MEMORY_CONSTRAINTS = (0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384)
372efc19	18
52da761b MT	19	CPU_VENDORS = {
	20	"AMDisbetter!" : "AMD",
	21	"AuthenticAMD" : "AMD",
	22	"CentaurHauls" : "VIA",
	23	"CyrixInstead" : "Cyrix",
	24	"GenuineIntel" : "Intel",
	25	"TransmetaCPU" : "Transmeta",
	26	"GenuineTMx86" : "Transmeta",
	27	"Geode by NSC" : "NSC",
	28	"NexGenDriven" : "NexGen",
	29	"RiseRiseRise" : "Rise",
	30	"SiS SiS SiS " : "SiS",
	31	"UMC UMC UMC " : "UMC",
	32	"VIA VIA VIA " : "VIA",
	33	}
	34
abc3bb31 MT	35	CPU_STRINGS = (
	36	# AMD
	37	(r"(AMD Athlon).(XP).", r"\1 \2"),
	38	(r"(AMD Phenom).* ([0-9]+) .*", r"\1 \2"),
	39	(r"(AMD Phenom).*", r"\1"),
	40	(r"(AMD Sempron).*", r"\1"),
	41	(r"AMD Athlon.* II X2 ([a-z0-9]+).*", r"AMD Athlon X2 \1"),
	42	(r"(Geode).*", r"\1"),
	43
	44	# Intel
	45	(r"Intel.(Atom\|Celeron).CPU\s([A-Z0-9]+) .", r"Intel \1 \2"),
	46	(r"(Intel).(Celeron).", r"\1 \2"),
ab9d063c	47	(r"Intel.* Core.2 Duo CPU .* ([A-Z0-9]+) .*", r"Intel C2D \1"),
abc3bb31	48	(r"Intel.* Core.2 CPU . ([A-Z0-9]+) .*", r"Intel C2 \1"),
ab9d063c	49	(r"Intel.* Core.2 Quad CPU .* ([A-Z0-9]+) .*", r"Intel C2Q \1"),
abc3bb31 MT	50	(r"Intel.* Xeon.* CPU .* ([A-Z0-9]+) .*", r"Intel Xeon \1"),
	51	(r"(Intel).(Xeon).", r"\1 \2"),
	52	(r"Intel.* Pentium.* (D\|4) .*", r"Intel Pentium \1"),
	53	(r"Intel.* Pentium.* Dual .* ([A-Z0-9]+) .*", r"Intel Pentium Dual \1"),
	54	(r"Pentium.* Dual-Core .* ([A-Z0-9]+) .*", r"Intel Pentium Dual \1"),
	55	(r"(Pentium I{2,3}).*", r"Intel \1"),
	56	(r"(Celeron \(Coppermine\))", r"Intel Celeron"),
	57
	58	# VIA
	59	(r"(VIA \w).", r"\1"),
	60	)
	61
62c3fa48 MT	62	CPU_CORES = range(1, 9)
62c3fa48 MT	63
372efc19 MT	64	class ProfileDict(object):
	65	def __init__(self, data):
	66	self._data = data
	67
	68
	69	class ProfileCPU(ProfileDict):
	70	@property
	71	def arch(self):
	72	return self._data.get("arch")
	73
	74	@property
	75	def vendor(self):
52da761b MT	76	vendor = self._data.get("vendor")
	77
	78	return CPU_VENDORS.get(vendor, vendor)
372efc19	79
91a446f0 MT	80	@property
	81	def speed(self):
	82	return self._data.get("speed")
	83
abc3bb31 MT	84	@property
	85	def friendly_speed(self):
	86	if self.speed < 1000:
	87	return "%dMHz" % self.speed
	88
	89	return "%.1fGHz" % round(self.speed / 1000, 1)
	90
91a446f0 MT	91	@property
	92	def bogomips(self):
	93	return self._data.get("bogomips")
	94
372efc19 MT	95	@property
	96	def model(self):
	97	return self._data.get("model")
	98
	99	@property
	100	def model_string(self):
	101	return self._data.get("model_string")
	102
	103	@property
	104	def flags(self):
	105	return self._data.get("flags")
	106
91a446f0 MT	107	@property
	108	def count(self):
	109	return self._data.get("count")
	110
372efc19 MT	111	@property
	112	def capable_64bit(self):
	113	return "lm" in self.flags
	114
	115	@property
	116	def capable_pae(self):
	117	return "pae" in self.flags
	118
	119	@property
	120	def capable_virt(self):
	121	return "vmx" in self.flags or "svm" in self.flags
	122
abc3bb31 MT	123	@property
	124	def friendly_vendor(self):
	125	s = self.model_string
	126	for pattern, repl in CPU_STRINGS:
	127	if re.match(pattern, s) is None:
	128	continue
	129	return re.sub(pattern, repl, s)
	130
	131	return s
	132
	133	@property
	134	def friendly_string(self):
	135	return "%s @ %s" % (self.friendly_vendor, self.friendly_speed)
	136
372efc19 MT	137
	138	class ProfileHypervisor(ProfileDict):
	139	def __repr__(self):
	140	return "<%s %s-%s>" % (self.__class__.__name__, self.vendor, self.type)
	141
	142	@property
	143	def vendor(self):
	144	return self._data.get("vendor")
	145
372efc19 MT	146	@property
	147	def type(self):
	148	return self._data.get("type")
	149
	150
07785a9c MT	151	class ProfileNetwork(ProfileDict):
	152	def has_zone(self, name):
	153	return self._data.get(name)
	154
	155	@property
	156	def green(self):
	157	return self._data.get("green")
	158
	159	@property
	160	def red(self):
	161	return self._data.get("red")
	162
	163	@property
	164	def blue(self):
	165	return self._data.get("blue")
	166
	167	@property
	168	def orange(self):
	169	return self._data.get("orange")
	170
	171
372efc19 MT	172	class ProfileDevice(ProfileDict):
	173	subsystem2class = {
	174	"pci" : hwdata.PCI,
	175	"usb" : hwdata.USB,
	176	}
	177
91a446f0 MT	178	classid2name = {
91a446f0 MT	179	"pci" : {
bc19cabd	180	"00" : "Unclassified",
91a446f0 MT	181	"01" : "Mass storage",
	182	"02" : "Network",
	183	"03" : "Display",
	184	"04" : "Multimedia",
	185	"05" : "Memory controller",
	186	"06" : "Bridge",
	187	"07" : "Communication",
	188	"08" : "Generic system peripheral",
	189	"09" : "Input device",
	190	"0a" : "Docking station",
	191	"0b" : "Processor",
	192	"0c" : "Serial bus",
	193	"0d" : "Wireless",
	194	"0e" : "Intelligent controller",
	195	"0f" : "Satellite communications controller",
bc19cabd	196	"10" : "Encryption",
91a446f0 MT	197	"11" : "Signal processing controller",
	198	"ff" : "Unassigned class",
	199	},
	200
	201	"usb" : {
bc19cabd	202	"00" : "Unclassified",
91a446f0 MT	203	"01" : "Multimedia",
	204	"02" : "Communication",
	205	"03" : "Input device",
bc19cabd MT	206	"05" : "Generic system peripheral",
bc19cabd MT	207	"06" : "Image",
91a446f0 MT	208	"07" : "Printer",
	209	"08" : "Mass storage",
	210	"09" : "Hub",
bc19cabd MT	211	"0a" : "Communication",
	212	"0b" : "Smart card",
	213	"0d" : "Encryption",
91a446f0	214	"0e" : "Display",
bc19cabd	215	"0f" : "Personal Healthcare",
91a446f0 MT	216	"dc" : "Diagnostic Device",
91a446f0 MT	217	"e0" : "Wireless",
bc19cabd MT	218	"ef" : "Unclassified",
	219	"fe" : "Unclassified",
	220	"ff" : "Unclassified",
91a446f0 MT	221	}
	222	}
	223
	224	def __cmp__(self, other):
	225	return cmp(self.vendor, other.vendor) or \
	226	cmp(self.model, other.model) or \
	227	cmp(self.driver, other.driver)
	228
372efc19 MT	229	@property
	230	def model(self):
	231	return self._data.get("model")
	232
	233	@property
	234	def model_string(self):
	235	cls = self.subsystem2class[self.subsystem]
	236
	237	return cls().get_device(self.vendor, self.model)
	238
	239	@property
	240	def subsystem(self):
	241	return self._data.get("subsystem")
	242
	243	@property
	244	def vendor(self):
	245	return self._data.get("vendor")
	246
	247	@property
	248	def vendor_string(self):
	249	cls = self.subsystem2class[self.subsystem]
	250
	251	return cls().get_vendor(self.vendor)
	252
	253	@property
	254	def driver(self):
	255	return self._data.get("driver")
	256
91a446f0 MT	257	@property
	258	def cls(self):
	259	classid = self._data.get("deviceclass")
	260
	261	if self.subsystem == "pci":
	262	classid = classid[:-4]
	263	if len(classid) == 1:
	264	classid = "0%s" % classid
	265
	266	elif self.subsystem == "usb" and classid:
	267	classid = classid.split("/")[0]
	268	classid = "%02x" % int(classid)
	269
	270	try:
	271	return self.classid2name[self.subsystem][classid]
	272	except KeyError:
	273	return "N/A"
	274
372efc19 MT	275
372efc19 MT	276	class Profile(ProfileDict):
91a446f0 MT	277	def __init__(self, profile_blob):
91a446f0 MT	278	ProfileDict.__init__(self, profile_blob.get("profile"))
372efc19	279
91a446f0 MT	280	self.public_id = profile_blob.get("public_id")
	281	self.updated = profile_blob.get("updated")
	282	self._geoip = profile_blob.get("geoip", None)
372efc19 MT	283
	284	def __repr__(self):
	285	return "<%s %s>" % (self.__class__.__name__, self.public_id)
	286
	287	@property
	288	def cpu(self):
	289	return ProfileCPU(self._data.get("cpu"))
	290
	291	@property
	292	def devices(self):
	293	devices = []
	294	for d in self._data.get("devices"):
	295	d = ProfileDevice(d)
	296
91a446f0	297	if d.driver in ("pcieport", "usb", "hub"):
372efc19 MT	298	continue
	299
	300	devices.append(d)
	301
	302	return devices
	303
	304	@property
	305	def hypervisor(self):
	306	if self.virtual:
	307	return ProfileHypervisor(self._data.get("hypervisor"))
	308
	309	@property
	310	def virtual(self):
	311	return self.system.get("virtual")
	312
	313	@property
	314	def system(self):
	315	return self._data.get("system")
	316
	317	@property
	318	def release(self):
	319	return self.system.get("release")
	320
	321	@property
	322	def kernel(self):
	323	return self.system.get("kernel_release")
	324
	325	@property
	326	def memory(self):
	327	return self.system.get("memory")
	328
abc3bb31 MT	329	@property
	330	def friendly_memory(self):
	331	units = ("k", "M", "G", "T")
	332
	333	mem = self.memory
	334	i = 0
	335	while mem >= 1024:
	336	mem /= 1024
	337	i += 1
	338
	339	return "%d%s" % (round(mem, 0), units[i])
	340
372efc19 MT	341	@property
372efc19 MT	342	def root_size(self):
91a446f0 MT	343	return self.system.get("root_size") or 0
91a446f0 MT	344
abc3bb31 MT	345	@property
	346	def friendly_root_size(self):
	347	units = ("k", "M", "G", "T")
	348
	349	size = self.root_size
	350	if not size:
	351	return
	352
	353	i = 0
	354	while size >= 1024:
	355	size /= 1024
	356	i += 1
	357
	358	return "%d%s" % (round(size, 0), units[i])
	359
91a446f0 MT	360	@property
	361	def vendor(self):
	362	return self.system.get("vendor")
	363
	364	@property
	365	def model(self):
	366	return self.system.get("model")
	367
	368	@property
	369	def country_code(self):
	370	if self._geoip:
	371	return self._geoip["country_code"].lower()
372efc19	372
91a446f0	373	return "unknown"
372efc19	374
07785a9c MT	375	@property
	376	def network(self):
	377	network = self._data.get("network", None)
	378	if network:
	379	return ProfileNetwork(network)
	380
372efc19 MT	381
	382	class Stasy(object):
	383	__metaclass__ = Singleton
	384
	385	def __init__(self):
	386	# Initialize database connection
	387	self._conn = pymongo.Connection(DATABASE_HOST)
	388	self._db = self._conn[DATABASE_NAME]
	389
	390	def get_profile_count(self):
	391	# XXX need to implement something to get profiles updated since
	392	# a given date
	393
	394	# All distinct profiles (based on public_id)
cbb9726a MT	395	# XXX possibly bad performance
	396	return len(self._db.profiles.distinct("public_id"))
	397
	398	def get_archives_count(self):
	399	return self._db.archives.count()
372efc19 MT	400
	401	#def _get_profile_cursor(self, public_id):
	402	# c = self._db.profiles.find({ "public_id" : public_id })
	403	# c.sort("updated", pymongo.ASCENDING)
	404	#
	405	# return c
	406
91a446f0	407	def profile_exists(self, public_id):
910be037	408	return self.query({ "public_id" : public_id }).count() >= 1
91a446f0	409
372efc19	410	def get_profile(self, public_id):
54af860e	411	p = None
372efc19	412	# XXX should only find one object in the end
910be037	413	for p in self.query({ "public_id" : public_id }):
91a446f0	414	p = Profile(p)
372efc19 MT	415
	416	return p
	417
	418	def get_profiles(self):
	419	# XXX needs nicer database query
	420	profiles = []
	421	for p in self._db.profiles.find():
	422	if not p.get("public_id") in profiles:
	423	profiles.append(p.get("public_id"))
	424
	425	return profiles
	426
62c3fa48	427	def query(self, query, archives=False, no_virt=False, all=False, fields=None):
910be037 MT	428	db = self._db.profiles
	429
	430	if archives:
	431	db = self.db.archives
	432
	433	if not all:
	434	# XXX cannot use the index?
	435	query.update({ "profile" : { "$exists" : True }})
	436
	437	if no_virt:
	438	query.update({ "profile.system.virtual" : False })
	439
	440	logging.debug("Executing query: %s" % query)
	441
62c3fa48	442	return db.find(query, fields=fields)
910be037	443
372efc19 MT	444	@property
	445	def secret_ids(self):
	446	return self._db.profiles.distinct("secret_id")
	447
	448	@property
	449	def cpus(self):
910be037	450	return self.query({}, no_virt=True).distinct("profile.cpu")
372efc19 MT	451
	452	@property
	453	def cpu_vendors(self):
910be037	454	return self.query({}, no_virt=True).distinct("profile.cpu.vendor")
372efc19 MT	455
372efc19 MT	456	@property
91a446f0	457	def cpu_vendors_map(self):
372efc19 MT	458	cpus = {}
	459
	460	for vendor in self.cpu_vendors:
52da761b MT	461	friendly_vendor = CPU_VENDORS.get(vendor, vendor)
52da761b MT	462	cpus[friendly_vendor] = \
910be037	463	self.query({
372efc19	464	"profile.cpu.vendor" : vendor
910be037	465	}, no_virt=True).count()
372efc19 MT	466
	467	return cpus
	468
	469	@property
91a446f0 MT	470	def cpu_speed_average(self):
	471	speed = 0
	472
910be037	473	all = self.query({}, no_virt=True)
91a446f0 MT	474
	475	# XXX ugly. needs to be done by group()
	476	for m in all:
	477	if not m.has_key("profile"):
	478	continue
	479	speed += m.get("profile").get("cpu").get("speed")
	480
	481	return (speed / all.count())
	482
62c3fa48 MT	483	def get_cpu_bogomips_accumulated(self):
	484	profiles = self.query({}, no_virt=True, fields=["profile.cpu.bogomips"])
	485
	486	return sum([int(o["profile"]["cpu"]["bogomips"]) for o in profiles])
	487
91a446f0 MT	488	@property
	489	def cpu_speed_map(self):
	490	cpu_speeds = {}
	491
	492	for i in range(len(CPU_SPEED_CONSTRAINTS) - 1):
	493	min, max = CPU_SPEED_CONSTRAINTS[i:i+2]
	494
	495	cpu_speeds[min, max] = \
910be037 MT	496	self.query({
910be037 MT	497	"profile.cpu.speed" : {
91a446f0 MT	498	"$gte" : min, "$lt" : max
91a446f0 MT	499	}
910be037	500	}, no_virt=True).count()
91a446f0 MT	501
	502	return cpu_speeds
	503
62c3fa48 MT	504	def get_cpu_cores_map(self):
	505	cores = {}
	506
	507	for i in CPU_CORES:
	508	cores[i] = \
	509	self.query({
	510	"profile.cpu.count" : i,
	511	}, no_virt=True).count()
	512
	513	return cores
	514
91a446f0	515	def get_memory_map(self):
372efc19 MT	516	memory = {}
	517
	518	for i in range(len(MEMORY_CONSTRAINTS) - 1):
	519	min, max = MEMORY_CONSTRAINTS[i:i+2]
	520
	521	memory[min, max] = \
910be037	522	self.query(
372efc19 MT	523	{ "profile.system.memory" : {
	524	"$gte" : min * 1024, "$lt" : max * 1024
	525	}
910be037	526	}, no_virt=True).count()
372efc19 MT	527
	528	return memory
	529
	530	@property
	531	def memory_average(self):
	532	memory = 0
	533
910be037	534	all = self.query({}, no_virt=True)
372efc19 MT	535
	536	# XXX ugly. needs to be done by group()
	537	for m in all:
	538	if not m.has_key("profile"):
	539	continue
	540	memory += int(m.get("profile").get("system").get("memory"))
	541
	542	return (memory / all.count()) / 1024
	543
	544	@property
	545	def hypervisor_vendors(self):
910be037	546	return self.query({}).distinct("profile.hypervisor.vendor")
372efc19 MT	547
	548	@property
	549	def hypervisor_map(self):
	550	hypervisors = {}
	551
	552	for hypervisor in self.hypervisor_vendors:
	553	hypervisors[hypervisor] = \
910be037	554	self.query({
372efc19 MT	555	"profile.hypervisor.vendor" : hypervisor
	556	}).count()
	557
	558	return hypervisors
	559
	560	@property
	561	def hypervisor_models(self):
910be037	562	return self.query({}).distinct("profile.hypervisor.model")
372efc19 MT	563
	564	@property
	565	def virtual_map(self):
	566	virtual = {
	567	True: None,
	568	False: None,
	569	}
	570
	571	for k in virtual.keys():
	572	virtual[k] = \
910be037	573	self.query({ "profile.system.virtual": k }).count()
372efc19 MT	574
	575	return virtual
	576
	577	@property
	578	def languages(self):
910be037	579	return self.query({}).distinct("profile.system.language")
372efc19	580
91a446f0 MT	581	def get_language_map(self):
	582	languages = {}
	583
	584	for language in self.languages:
	585	languages[language] = \
910be037	586	self.query({
91a446f0 MT	587	"profile.system.language" : language
	588	}).count()
	589
	590	return languages
	591
372efc19 MT	592	@property
372efc19 MT	593	def vendors(self):
910be037	594	return self.query({}).distinct("profile.system.vendor")
372efc19 MT	595
	596	@property
	597	def vendor_map(self):
	598	vendors = {}
	599
	600	for vendor in self.vendors:
	601	vendors[vendor] = \
910be037	602	self.query({
372efc19 MT	603	"profile.system.vendor" : vendor
	604	}).count()
	605
	606	return vendors
	607
	608	@property
	609	def models(self):
910be037	610	return self.query({}).distinct("profile.system.model")
372efc19	611
91a446f0 MT	612	@property
	613	def model_map(self):
	614	models = {}
	615
	616	for model in self.models:
	617	models[model] = \
910be037	618	self.query({
91a446f0 MT	619	"profile.system.model" : model
	620	}).count()
	621
	622	return models
	623
	624	@property
	625	def arches(self):
910be037	626	return self.query({}).distinct("profile.cpu.arch")
91a446f0 MT	627
	628	@property
	629	def arch_map(self):
	630	arches = {}
	631
	632	for arch in self.arches:
	633	arches[arch] = \
910be037	634	self.query({
91a446f0 MT	635	"profile.cpu.arch" : arch
	636	}).count()
	637
	638	return arches
	639
	640	@property
	641	def kernels(self):
910be037	642	return self.query({}).distinct("profile.system.kernel_release")
91a446f0 MT	643
	644	@property
	645	def kernel_map(self):
	646	kernels = {}
	647
	648	for kernel in self.kernels:
	649	kernels[kernel] = \
910be037	650	self.query({
91a446f0 MT	651	"profile.system.kernel_release" : kernel
	652	}).count()
	653
	654	return kernels
	655
	656	@property
	657	def releases(self):
910be037	658	return self.query({}).distinct("profile.system.release")
91a446f0 MT	659
	660	@property
	661	def release_map(self):
	662	releases = {}
	663
	664	for release in self.releases:
	665	releases[release] = \
910be037	666	self.query({
91a446f0 MT	667	"profile.system.release" : release
	668	}).count()
	669
	670	return releases
	671
	672	def get_device_percentage(self, bus, vendor_id, model_id):
910be037	673	profiles_with_device = self.query({
91a446f0 MT	674	"profile.devices.subsystem" : bus,
	675	"profile.devices.vendor" : vendor_id,
	676	"profile.devices.model" : model_id,
	677	})
	678
910be037	679	profiles_all = self.query({})
91a446f0 MT	680
	681	if not profiles_all.count():
	682	return 0
	683
	684	return profiles_with_device.count() / profiles_all.count()
	685
910be037 MT	686	def get_cpu_flag_map(self, flags):
910be037 MT	687	# XXX needs a cleanup
91a446f0	688
910be037	689	_flags = { True : 0 }
91a446f0	690
910be037 MT	691	if type(flags) == type("a"):
	692	flags = [flags]
	693
	694	for flag in flags:
	695	_flags[True] += \
	696	self.query({
	697	"profile.cpu.flags" : flag,
	698	}, no_virt=True).count()
91a446f0	699
910be037	700	_flags[False] = self.query({}, no_virt=True).count() - _flags[True]
91a446f0	701
910be037	702	return _flags
91a446f0 MT	703
	704	@property
	705	def geo_locations(self):
5914104c	706	return [code.lower() for code in self.query({}, all=True).distinct("geoip.country_code")]
91a446f0 MT	707
	708	def get_geo_location_map(self):
	709	geo_locations = {}
	710
	711	count = 0
	712	for geo_location in self.geo_locations:
	713	geo_locations[geo_location] = \
910be037	714	self.query({
bc1a5c5e	715	"geoip.country_code" : geo_location.upper()
5914104c	716	}, all=True).count()
91a446f0 MT	717
	718	count += geo_locations[geo_location]
	719
638e9782 MT	720	for geo_location in geo_locations.keys():
	721	geo_locations[geo_location] /= count
	722
91a446f0 MT	723	return geo_locations
	724
	725	def get_models_by_vendor(self, subsystem, vendor_id):
	726	devices = []
	727
910be037	728	profiles_all = self.query({})
91a446f0 MT	729
	730	for profile in profiles_all:
	731	if not profile.has_key("profile"):
	732	continue
	733
	734	profile = Profile(profile)
	735
	736	for device in profile.devices:
	737	if not device.vendor == vendor_id:
	738	continue
	739
	740	if not device in devices:
	741	devices.append(device)
	742
	743	return devices
372efc19	744
07785a9c MT	745	def get_network_zones_map(self):
	746	zones = { "green" : 0, "blue" : 0, "orange" : 0, "red" : 0 }
	747
e6cff1b1 MT	748	all = self.query({ "profile.network" : { "$exists" : True }})
e6cff1b1 MT	749
07785a9c MT	750	for zone in zones.keys():
	751	zones[zone] = self.query({
	752	"profile.network.%s" % zone : True,
e6cff1b1	753	}).count() / all.count()
07785a9c MT	754
	755	return zones
	756
cbb9726a MT	757	def get_profile_ratio(self):
	758	return (self.query({}).count(), self.get_profile_count())
	759
	760	def get_updated_since(self, since, _query={}):
	761	since = datetime.datetime.utcnow() - datetime.timedelta(**since)
	762
	763	query = { "updated" : { "$gte" : since }}
	764	query.update(_query)
	765
	766	return self.query(query)
	767
	768	def get_updates_by_release_since(self, since):
	769	updates = {}
	770
	771	for release in self.releases:
	772	updates[release] = self.get_updated_since(since,
	773	{ "profile.system.release" : release }).count()
	774
	775	return updates
	776
372efc19 MT	777
	778	if __name__ == "__main__":
	779	s = Stasy()
	780