[ipfire.org.git] / www / webapp / backend / stasy.py

#!/usr/bin/python

from __future__ import division

import datetime
import hwdata
import logging
import pymongo
import re

from misc import Singleton

DATABASE_HOST = ["irma.ipfire.org", "madeye.ipfire.org"]
DATABASE_NAME = "stasy"

CPU_SPEED_CONSTRAINTS = (0, 500, 1000, 1500, 2000, 2500, 3000, 3500)
MEMORY_CONSTRAINTS = (0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384)

CPU_VENDORS = {
	"AMDisbetter!" : "AMD",
	"AuthenticAMD" : "AMD",
	"CentaurHauls" : "VIA",
	"CyrixInstead" : "Cyrix",
	"GenuineIntel" : "Intel",
	"TransmetaCPU" : "Transmeta",
	"GenuineTMx86" : "Transmeta",
	"Geode by NSC" : "NSC",
	"NexGenDriven" : "NexGen",
	"RiseRiseRise" : "Rise",
	"SiS SiS SiS " : "SiS",
	"UMC UMC UMC " : "UMC",
	"VIA VIA VIA " : "VIA",
}

CPU_STRINGS = (
	# AMD
	(r"(AMD Athlon).*(XP).*", r"\1 \2"),
	(r"(AMD Phenom).* ([0-9]+) .*", r"\1 \2"),
	(r"(AMD Phenom).*", r"\1"),
	(r"(AMD Sempron).*", r"\1"),
	(r"AMD Athlon.* II X2 ([a-z0-9]+).*", r"AMD Athlon X2 \1"),
	(r"(Geode).*", r"\1"),

	# Intel
	(r"Intel.*(Atom|Celeron).*CPU\s*([A-Z0-9]+) .*", r"Intel \1 \2"),
	(r"(Intel).*(Celeron).*", r"\1 \2"),
	(r"Intel.* Core.*2 Duo *CPU .* ([A-Z0-9]+) .*", r"Intel C2D \1"),
	(r"Intel.* Core.*2 CPU .* ([A-Z0-9]+) .*", r"Intel C2 \1"),
	(r"Intel.* Core.*2 Quad *CPU .* ([A-Z0-9]+) .*", r"Intel C2Q \1"),
	(r"Intel.* Xeon.* CPU .* ([A-Z0-9]+) .*", r"Intel Xeon \1"),
	(r"(Intel).*(Xeon).*", r"\1 \2"),
	(r"Intel.* Pentium.* (D|4) .*", r"Intel Pentium \1"),
	(r"Intel.* Pentium.* Dual .* ([A-Z0-9]+) .*", r"Intel Pentium Dual \1"),
	(r"Pentium.* Dual-Core .* ([A-Z0-9]+) .*", r"Intel Pentium Dual \1"),
	(r"(Pentium I{2,3}).*", r"Intel \1"),
	(r"(Celeron \(Coppermine\))", r"Intel Celeron"),

	# VIA
	(r"(VIA \w*).*", r"\1"),

	# Qemu
	(r"QEMU Virtual CPU version .*", r"QEMU CPU"),
)

CPU_CORES = range(1, 9)

class ProfileDict(object):
	def __init__(self, data):
		self._data = data


class ProfileCPU(ProfileDict):
	@property
	def arch(self):
		return self._data.get("arch")

	@property
	def vendor(self):
		vendor = self._data.get("vendor")

		return CPU_VENDORS.get(vendor, vendor)

	@property
	def speed(self):
		return self._data.get("speed")

	@property
	def friendly_speed(self):
		if self.speed < 1000:
			return "%dMHz" % self.speed

		return "%.1fGHz" % round(self.speed / 1000, 1)

	@property
	def bogomips(self):
		return self._data.get("bogomips")

	@property
	def model(self):
		return self._data.get("model")

	@property
	def model_string(self):
		return self._data.get("model_string")

	@property
	def flags(self):
		return self._data.get("flags")

	@property
	def count(self):
		return self._data.get("count")

	@property
	def capable_64bit(self):
		return "lm" in self.flags

	@property
	def capable_pae(self):
		return "pae" in self.flags

	@property
	def capable_virt(self):
		return "vmx" in self.flags or "svm" in self.flags

	@property
	def friendly_vendor(self):
		s = self.model_string
		for pattern, repl in CPU_STRINGS:
			if re.match(pattern, s) is None:
				continue
			return re.sub(pattern, repl, s)

		return s

	@property
	def friendly_string(self):
		return "%s @ %s" % (self.friendly_vendor, self.friendly_speed)


class ProfileHypervisor(ProfileDict):
	def __repr__(self):
		return "<%s %s-%s>" % (self.__class__.__name__, self.vendor, self.type)

	@property
	def vendor(self):
		return self._data.get("vendor")

	@property
	def type(self):
		return self._data.get("type")


class ProfileNetwork(ProfileDict):
	def has_zone(self, name):
		return self._data.get(name)

	@property
	def green(self):
		return self._data.get("green")

	@property
	def red(self):
		return self._data.get("red")

	@property
	def blue(self):
		return self._data.get("blue")

	@property
	def orange(self):
		return self._data.get("orange")


class ProfileDevice(ProfileDict):
	subsystem2class = {
		"pci" : hwdata.PCI,
		"usb" : hwdata.USB,
	}

	classid2name = {
		"pci" : {
			"00" : "Unclassified",
			"01" : "Mass storage",
			"02" : "Network",
			"03" : "Display",
			"04" : "Multimedia",
			"05" : "Memory controller",
			"06" : "Bridge",
			"07" : "Communication",
			"08" : "Generic system peripheral",
			"09" : "Input device",
			"0a" : "Docking station",
			"0b" : "Processor",
			"0c" : "Serial bus",
			"0d" : "Wireless",
			"0e" : "Intelligent controller",
			"0f" : "Satellite communications controller",
			"10" : "Encryption",
			"11" : "Signal processing controller",
			"ff" : "Unassigned class",
		},
		
		"usb" : {
			"00" : "Unclassified",
			"01" : "Multimedia",
			"02" : "Communication",
			"03" : "Input device",
			"05" : "Generic system peripheral",
			"06" : "Image",
			"07" : "Printer",
			"08" : "Mass storage",
			"09" : "Hub",
			"0a" : "Communication",
			"0b" : "Smart card",
			"0d" : "Encryption",
			"0e" : "Display",
			"0f" : "Personal Healthcare",
			"dc" : "Diagnostic Device",
			"e0" : "Wireless",
			"ef" : "Unclassified",
			"fe" : "Unclassified",
			"ff" : "Unclassified",
		}
	}

	def __cmp__(self, other):
		return cmp(self.vendor, other.vendor) or \
			cmp(self.model, other.model) or \
			cmp(self.driver, other.driver)

	@property
	def model(self):
		return self._data.get("model")

	@property
	def model_string(self):
		cls = self.subsystem2class[self.subsystem]

		return cls().get_device(self.vendor, self.model)

	@property
	def subsystem(self):
		return self._data.get("subsystem")

	@property
	def vendor(self):
		return self._data.get("vendor")

	@property
	def vendor_string(self):
		cls = self.subsystem2class[self.subsystem]

		return cls().get_vendor(self.vendor)

	@property
	def driver(self):
		return self._data.get("driver")

	@property
	def cls(self):
		classid = self._data.get("deviceclass")

		if self.subsystem == "pci":
			classid = classid[:-4]
			if len(classid) == 1:
				classid = "0%s" % classid

		elif self.subsystem == "usb" and classid:
			classid = classid.split("/")[0]
			classid = "%02x" % int(classid)

		try:
			return self.classid2name[self.subsystem][classid]
		except KeyError:
			return "N/A"


class Profile(ProfileDict):
	def __init__(self, profile_blob):
		ProfileDict.__init__(self, profile_blob.get("profile"))

		self.public_id = profile_blob.get("public_id")
		self.updated = profile_blob.get("updated")
		self._geoip = profile_blob.get("geoip", None)

	def __repr__(self):
		return "<%s %s>" % (self.__class__.__name__, self.public_id)

	@property
	def cpu(self):
		return ProfileCPU(self._data.get("cpu"))

	@property
	def devices(self):
		devices = []
		for d in self._data.get("devices"):
			d = ProfileDevice(d)

			if d.driver in ("pcieport", "usb", "hub"):
				continue

			devices.append(d)

		return devices

	@property
	def hypervisor(self):
		if self.virtual:
			return ProfileHypervisor(self._data.get("hypervisor"))

	@property
	def virtual(self):
		return self.system.get("virtual")

	@property
	def system(self):
		return self._data.get("system")

	@property
	def release(self):
		return self.system.get("release")

	@property
	def kernel(self):
		return self.system.get("kernel_release")

	@property
	def memory(self):
		return self.system.get("memory")

	@property
	def friendly_memory(self):
		units = ("k", "M", "G", "T")

		mem = self.memory
		i = 0
		while mem >= 1024:
			mem /= 1024
			i += 1

		return "%d%s" % (round(mem, 0), units[i])

	@property
	def root_size(self):
		return self.system.get("root_size") or 0

	@property
	def friendly_root_size(self):
		units = ("k", "M", "G", "T")

		size = self.root_size
		if not size:
			return

		i = 0
		while size >= 1024:
			size /= 1024
			i += 1

		return "%d%s" % (round(size, 0), units[i])

	@property
	def vendor(self):
		return self.system.get("vendor")

	@property
	def model(self):
		return self.system.get("model")

	@property
	def country_code(self):
		if self._geoip:
			return self._geoip["country_code"].lower()

		return "unknown"

	@property
	def network(self):
		network = self._data.get("network", None)
		if network:
			return ProfileNetwork(network)


class Stasy(object):
	__metaclass__ = Singleton

	def __init__(self):
		# Initialize database connection
		self._conn = pymongo.Connection(DATABASE_HOST)
		self._db = self._conn[DATABASE_NAME]

	def get_profile_count(self):
		# XXX need to implement something to get profiles updated since
		# a given date

		# All distinct profiles (based on public_id)
		# XXX possibly bad performance
		return len(self._db.profiles.distinct("public_id"))

	def get_archives_count(self):
		return self._db.archives.count()

	#def _get_profile_cursor(self, public_id):
	#	c = self._db.profiles.find({ "public_id" : public_id })
	#	c.sort("updated", pymongo.ASCENDING)
	#
	#	return c

	def profile_exists(self, public_id):
		return self.query({ "public_id" : public_id }).count() >= 1

	def get_profile(self, public_id):
		p = None
		# XXX should only find one object in the end
		for p in self.query({ "public_id" : public_id }):
			p = Profile(p)

		return p

	def get_profiles(self):
		# XXX needs nicer database query
		profiles = []
		for p in self._db.profiles.find():
			if not p.get("public_id") in profiles:
				profiles.append(p.get("public_id"))

		return profiles

	def query(self, query, archives=False, no_virt=False, all=False, fields=None):
		db = self._db.profiles

		if archives:
			db = self.db.archives

		if not all:
			# XXX cannot use the index?
			query.update({ "profile" : { "$exists" : True }})

		if no_virt:
			query.update({ "profile.system.virtual" : False })

		logging.debug("Executing query: %s" % query)

		return db.find(query, fields=fields)

	@property
	def secret_ids(self):
		return self._db.profiles.distinct("secret_id")

	@property
	def cpus(self):
		return self.query({}, no_virt=True).distinct("profile.cpu")

	@property
	def cpu_vendors(self):
		return self.query({}, no_virt=True).distinct("profile.cpu.vendor")

	@property
	def cpu_vendors_map(self):
		cpus = {}

		for vendor in self.cpu_vendors:
			friendly_vendor = CPU_VENDORS.get(vendor, vendor)
			cpus[friendly_vendor] = \
				self.query({
					"profile.cpu.vendor" : vendor
				}, no_virt=True).count()

		return cpus

	@property
	def cpu_speed_average(self):
		speed = 0

		all = self.query({}, no_virt=True)

		# XXX ugly. needs to be done by group()
		for m in all:
			if not m.has_key("profile"):
				continue
			speed += m.get("profile").get("cpu").get("speed")

		return (speed / all.count())

	def get_cpu_bogomips_accumulated(self):
		profiles = self.query({}, no_virt=True, fields=["profile.cpu.bogomips"])

		return sum([int(o["profile"]["cpu"]["bogomips"]) for o in profiles])

	@property
	def cpu_speed_map(self):
		cpu_speeds = {}

		for i in range(len(CPU_SPEED_CONSTRAINTS) - 1):
			min, max = CPU_SPEED_CONSTRAINTS[i:i+2]

			cpu_speeds[min, max] = \
				self.query({
					"profile.cpu.speed" : {
						"$gte" : min, "$lt" : max
					}
				}, no_virt=True).count()

		return cpu_speeds

	def get_cpu_cores_map(self):
		cores = {}

		for i in CPU_CORES:
			cores[i] = \
				self.query({
					"profile.cpu.count" : i,
				}, no_virt=True).count()

		return cores

	def get_memory_map(self):
		memory = {}

		for i in range(len(MEMORY_CONSTRAINTS) - 1):
			min, max = MEMORY_CONSTRAINTS[i:i+2]

			memory[min, max] = \
				self.query(
					{ "profile.system.memory" : {
						"$gte" : min * 1024, "$lt" : max * 1024
					}
				}, no_virt=True).count()

		return memory

	@property
	def memory_average(self):
		memory = 0

		all = self.query({}, no_virt=True)

		# XXX ugly. needs to be done by group()
		for m in all:
			if not m.has_key("profile"):
				continue
			memory += int(m.get("profile").get("system").get("memory"))

		return (memory / all.count()) / 1024

	@property
	def hypervisor_vendors(self):
		return self.query({}).distinct("profile.hypervisor.vendor")

	@property
	def hypervisor_map(self):
		hypervisors = {}

		for hypervisor in self.hypervisor_vendors:
			hypervisors[hypervisor] = \
				self.query({
					"profile.hypervisor.vendor" : hypervisor
				}).count()

		return hypervisors

	@property
	def hypervisor_models(self):
		return self.query({}).distinct("profile.hypervisor.model")

	@property
	def virtual_map(self):
		virtual = {
			True: None,
			False: None,
		}

		for k in virtual.keys():
			virtual[k] = \
				self.query({ "profile.system.virtual": k }).count()

		return virtual

	@property
	def languages(self):
		return self.query({}).distinct("profile.system.language")

	def get_language_map(self):
		languages = {}

		for language in self.languages:
			languages[language] = \
				self.query({
					"profile.system.language" : language
				}).count()

		return languages

	@property
	def vendors(self):
		return self.query({}).distinct("profile.system.vendor")

	@property
	def vendor_map(self):
		vendors = {}

		for vendor in self.vendors:
			vendors[vendor] = \
				self.query({
					"profile.system.vendor" : vendor
				}).count()

		return vendors

	@property
	def models(self):
		return self.query({}).distinct("profile.system.model")

	@property
	def model_map(self):
		models = {}

		for model in self.models:
			models[model] = \
				self.query({
					"profile.system.model" : model
				}).count()

		return models

	@property
	def arches(self):
		return self.query({}).distinct("profile.cpu.arch")

	@property
	def arch_map(self):
		arches = {}

		for arch in self.arches:
			arches[arch] = \
				self.query({
					"profile.cpu.arch" : arch
				}).count()

		return arches

	@property
	def kernels(self):
		return self.query({}).distinct("profile.system.kernel_release")

	@property
	def kernel_map(self):
		kernels = {}

		for kernel in self.kernels:
			kernels[kernel] = \
				self.query({
					"profile.system.kernel_release" : kernel
				}).count()

		return kernels

	@property
	def releases(self):
		return self.query({}).distinct("profile.system.release")

	@property
	def release_map(self):
		releases = {}

		for release in self.releases:
			releases[release] = \
				self.query({
					"profile.system.release" : release
				}).count()

		return releases

	def get_device_percentage(self, bus, vendor_id, model_id):
		profiles_with_device = self.query({
			"profile.devices.subsystem" : bus,
			"profile.devices.vendor" : vendor_id,
			"profile.devices.model" : model_id,
		})

		profiles_all = self.query({})

		if not profiles_all.count():
			return 0

		return profiles_with_device.count() / profiles_all.count()

	def get_cpu_flag_map(self, flags):
		# XXX needs a cleanup

		_flags = { True : 0 }

		if type(flags) == type("a"):
			flags = [flags]

		for flag in flags:
			_flags[True] += \
				self.query({
					"profile.cpu.flags" : flag,
				}, no_virt=True).count()

		_flags[False] = self.query({}, no_virt=True).count() - _flags[True]

		return _flags

	@property
	def geo_locations(self):
		return [code.lower() for code in self.query({}, all=True).distinct("geoip.country_code")]

	def get_geo_location_map(self):
		geo_locations = {}

		count = 0
		for geo_location in self.geo_locations:
			geo_locations[geo_location] = \
				self.query({
					"geoip.country_code" : geo_location.upper()
				}, all=True).count()

			count += geo_locations[geo_location]

		for geo_location in geo_locations.keys():
			geo_locations[geo_location] /= count

		return geo_locations

	def get_models_by_vendor(self, subsystem, vendor_id):
		devices = []

		profiles_all = self.query({})

		for profile in profiles_all:
			if not profile.has_key("profile"):
				continue

			profile = Profile(profile)

			for device in profile.devices:
				if not device.vendor == vendor_id:
					continue

				if not device in devices:
					devices.append(device)

		return devices

	def get_network_zones_map(self):
		zones = { "green" : 0, "blue" : 0, "orange" : 0, "red" : 0 }

		all = self.query({ "profile.network" : { "$exists" : True }})

		for zone in zones.keys():
			zones[zone] = self.query({
				"profile.network.%s" % zone : True,
			}).count() / all.count()

		return zones

	def get_profile_ratio(self):
		return (self.query({}).count(), self.get_profile_count())

	def get_updated_since(self, since, _query={}):
		since = datetime.datetime.utcnow() - datetime.timedelta(**since)

		query = { "updated" : { "$gte" : since }}
		query.update(_query)

		return self.query(query)

	def get_updates_by_release_since(self, since):
		updates = {}

		for release in self.releases:
			updates[release] = self.get_updated_since(since,
				{ "profile.system.release" : release }).count()

		return updates


if __name__ == "__main__":
	s = Stasy()
Commit	Line	Data
372efc19 MT	1	#!/usr/bin/python
372efc19 MT	2
91a446f0 MT	3	from __future__ import division
91a446f0 MT	4
cbb9726a	5	import datetime
372efc19 MT	6	import hwdata
	7	import logging
	8	import pymongo
abc3bb31	9	import re
372efc19 MT	10
	11	from misc import Singleton
	12
	13	DATABASE_HOST = ["irma.ipfire.org", "madeye.ipfire.org"]
	14	DATABASE_NAME = "stasy"
	15
91a446f0	16	CPU_SPEED_CONSTRAINTS = (0, 500, 1000, 1500, 2000, 2500, 3000, 3500)
8e413e4f	17	MEMORY_CONSTRAINTS = (0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384)
372efc19	18
52da761b MT	19	CPU_VENDORS = {
	20	"AMDisbetter!" : "AMD",
	21	"AuthenticAMD" : "AMD",
	22	"CentaurHauls" : "VIA",
	23	"CyrixInstead" : "Cyrix",
	24	"GenuineIntel" : "Intel",
	25	"TransmetaCPU" : "Transmeta",
	26	"GenuineTMx86" : "Transmeta",
	27	"Geode by NSC" : "NSC",
	28	"NexGenDriven" : "NexGen",
	29	"RiseRiseRise" : "Rise",
	30	"SiS SiS SiS " : "SiS",
	31	"UMC UMC UMC " : "UMC",
	32	"VIA VIA VIA " : "VIA",
	33	}
	34
abc3bb31 MT	35	CPU_STRINGS = (
	36	# AMD
	37	(r"(AMD Athlon).(XP).", r"\1 \2"),
	38	(r"(AMD Phenom).* ([0-9]+) .*", r"\1 \2"),
	39	(r"(AMD Phenom).*", r"\1"),
	40	(r"(AMD Sempron).*", r"\1"),
	41	(r"AMD Athlon.* II X2 ([a-z0-9]+).*", r"AMD Athlon X2 \1"),
	42	(r"(Geode).*", r"\1"),
	43
	44	# Intel
	45	(r"Intel.(Atom\|Celeron).CPU\s([A-Z0-9]+) .", r"Intel \1 \2"),
	46	(r"(Intel).(Celeron).", r"\1 \2"),
ab9d063c	47	(r"Intel.* Core.2 Duo CPU .* ([A-Z0-9]+) .*", r"Intel C2D \1"),
abc3bb31	48	(r"Intel.* Core.2 CPU . ([A-Z0-9]+) .*", r"Intel C2 \1"),
ab9d063c	49	(r"Intel.* Core.2 Quad CPU .* ([A-Z0-9]+) .*", r"Intel C2Q \1"),
abc3bb31 MT	50	(r"Intel.* Xeon.* CPU .* ([A-Z0-9]+) .*", r"Intel Xeon \1"),
	51	(r"(Intel).(Xeon).", r"\1 \2"),
	52	(r"Intel.* Pentium.* (D\|4) .*", r"Intel Pentium \1"),
	53	(r"Intel.* Pentium.* Dual .* ([A-Z0-9]+) .*", r"Intel Pentium Dual \1"),
	54	(r"Pentium.* Dual-Core .* ([A-Z0-9]+) .*", r"Intel Pentium Dual \1"),
	55	(r"(Pentium I{2,3}).*", r"Intel \1"),
	56	(r"(Celeron \(Coppermine\))", r"Intel Celeron"),
	57
	58	# VIA
	59	(r"(VIA \w).", r"\1"),
fe74b5e5 MT	60
	61	# Qemu
	62	(r"QEMU Virtual CPU version .*", r"QEMU CPU"),
abc3bb31 MT	63	)
abc3bb31 MT	64
62c3fa48 MT	65	CPU_CORES = range(1, 9)
62c3fa48 MT	66
372efc19 MT	67	class ProfileDict(object):
	68	def __init__(self, data):
	69	self._data = data
	70
	71
	72	class ProfileCPU(ProfileDict):
	73	@property
	74	def arch(self):
	75	return self._data.get("arch")
	76
	77	@property
	78	def vendor(self):
52da761b MT	79	vendor = self._data.get("vendor")
	80
	81	return CPU_VENDORS.get(vendor, vendor)
372efc19	82
91a446f0 MT	83	@property
	84	def speed(self):
	85	return self._data.get("speed")
	86
abc3bb31 MT	87	@property
	88	def friendly_speed(self):
	89	if self.speed < 1000:
	90	return "%dMHz" % self.speed
	91
	92	return "%.1fGHz" % round(self.speed / 1000, 1)
	93
91a446f0 MT	94	@property
	95	def bogomips(self):
	96	return self._data.get("bogomips")
	97
372efc19 MT	98	@property
	99	def model(self):
	100	return self._data.get("model")
	101
	102	@property
	103	def model_string(self):
	104	return self._data.get("model_string")
	105
	106	@property
	107	def flags(self):
	108	return self._data.get("flags")
	109
91a446f0 MT	110	@property
	111	def count(self):
	112	return self._data.get("count")
	113
372efc19 MT	114	@property
	115	def capable_64bit(self):
	116	return "lm" in self.flags
	117
	118	@property
	119	def capable_pae(self):
	120	return "pae" in self.flags
	121
	122	@property
	123	def capable_virt(self):
	124	return "vmx" in self.flags or "svm" in self.flags
	125
abc3bb31 MT	126	@property
	127	def friendly_vendor(self):
	128	s = self.model_string
	129	for pattern, repl in CPU_STRINGS:
	130	if re.match(pattern, s) is None:
	131	continue
	132	return re.sub(pattern, repl, s)
	133
	134	return s
	135
	136	@property
	137	def friendly_string(self):
	138	return "%s @ %s" % (self.friendly_vendor, self.friendly_speed)
	139
372efc19 MT	140
	141	class ProfileHypervisor(ProfileDict):
	142	def __repr__(self):
	143	return "<%s %s-%s>" % (self.__class__.__name__, self.vendor, self.type)
	144
	145	@property
	146	def vendor(self):
	147	return self._data.get("vendor")
	148
372efc19 MT	149	@property
	150	def type(self):
	151	return self._data.get("type")
	152
	153
07785a9c MT	154	class ProfileNetwork(ProfileDict):
	155	def has_zone(self, name):
	156	return self._data.get(name)
	157
	158	@property
	159	def green(self):
	160	return self._data.get("green")
	161
	162	@property
	163	def red(self):
	164	return self._data.get("red")
	165
	166	@property
	167	def blue(self):
	168	return self._data.get("blue")
	169
	170	@property
	171	def orange(self):
	172	return self._data.get("orange")
	173
	174
372efc19 MT	175	class ProfileDevice(ProfileDict):
	176	subsystem2class = {
	177	"pci" : hwdata.PCI,
	178	"usb" : hwdata.USB,
	179	}
	180
91a446f0 MT	181	classid2name = {
91a446f0 MT	182	"pci" : {
bc19cabd	183	"00" : "Unclassified",
91a446f0 MT	184	"01" : "Mass storage",
	185	"02" : "Network",
	186	"03" : "Display",
	187	"04" : "Multimedia",
	188	"05" : "Memory controller",
	189	"06" : "Bridge",
	190	"07" : "Communication",
	191	"08" : "Generic system peripheral",
	192	"09" : "Input device",
	193	"0a" : "Docking station",
	194	"0b" : "Processor",
	195	"0c" : "Serial bus",
	196	"0d" : "Wireless",
	197	"0e" : "Intelligent controller",
	198	"0f" : "Satellite communications controller",
bc19cabd	199	"10" : "Encryption",
91a446f0 MT	200	"11" : "Signal processing controller",
	201	"ff" : "Unassigned class",
	202	},
	203
	204	"usb" : {
bc19cabd	205	"00" : "Unclassified",
91a446f0 MT	206	"01" : "Multimedia",
	207	"02" : "Communication",
	208	"03" : "Input device",
bc19cabd MT	209	"05" : "Generic system peripheral",
bc19cabd MT	210	"06" : "Image",
91a446f0 MT	211	"07" : "Printer",
	212	"08" : "Mass storage",
	213	"09" : "Hub",
bc19cabd MT	214	"0a" : "Communication",
	215	"0b" : "Smart card",
	216	"0d" : "Encryption",
91a446f0	217	"0e" : "Display",
bc19cabd	218	"0f" : "Personal Healthcare",
91a446f0 MT	219	"dc" : "Diagnostic Device",
91a446f0 MT	220	"e0" : "Wireless",
bc19cabd MT	221	"ef" : "Unclassified",
	222	"fe" : "Unclassified",
	223	"ff" : "Unclassified",
91a446f0 MT	224	}
	225	}
	226
	227	def __cmp__(self, other):
	228	return cmp(self.vendor, other.vendor) or \
	229	cmp(self.model, other.model) or \
	230	cmp(self.driver, other.driver)
	231
372efc19 MT	232	@property
	233	def model(self):
	234	return self._data.get("model")
	235
	236	@property
	237	def model_string(self):
	238	cls = self.subsystem2class[self.subsystem]
	239
	240	return cls().get_device(self.vendor, self.model)
	241
	242	@property
	243	def subsystem(self):
	244	return self._data.get("subsystem")
	245
	246	@property
	247	def vendor(self):
	248	return self._data.get("vendor")
	249
	250	@property
	251	def vendor_string(self):
	252	cls = self.subsystem2class[self.subsystem]
	253
	254	return cls().get_vendor(self.vendor)
	255
	256	@property
	257	def driver(self):
	258	return self._data.get("driver")
	259
91a446f0 MT	260	@property
	261	def cls(self):
	262	classid = self._data.get("deviceclass")
	263
	264	if self.subsystem == "pci":
	265	classid = classid[:-4]
	266	if len(classid) == 1:
	267	classid = "0%s" % classid
	268
	269	elif self.subsystem == "usb" and classid:
	270	classid = classid.split("/")[0]
	271	classid = "%02x" % int(classid)
	272
	273	try:
	274	return self.classid2name[self.subsystem][classid]
	275	except KeyError:
	276	return "N/A"
	277
372efc19 MT	278
372efc19 MT	279	class Profile(ProfileDict):
91a446f0 MT	280	def __init__(self, profile_blob):
91a446f0 MT	281	ProfileDict.__init__(self, profile_blob.get("profile"))
372efc19	282
91a446f0 MT	283	self.public_id = profile_blob.get("public_id")
	284	self.updated = profile_blob.get("updated")
	285	self._geoip = profile_blob.get("geoip", None)
372efc19 MT	286
	287	def __repr__(self):
	288	return "<%s %s>" % (self.__class__.__name__, self.public_id)
	289
	290	@property
	291	def cpu(self):
	292	return ProfileCPU(self._data.get("cpu"))
	293
	294	@property
	295	def devices(self):
	296	devices = []
	297	for d in self._data.get("devices"):
	298	d = ProfileDevice(d)
	299
91a446f0	300	if d.driver in ("pcieport", "usb", "hub"):
372efc19 MT	301	continue
	302
	303	devices.append(d)
	304
	305	return devices
	306
	307	@property
	308	def hypervisor(self):
	309	if self.virtual:
	310	return ProfileHypervisor(self._data.get("hypervisor"))
	311
	312	@property
	313	def virtual(self):
	314	return self.system.get("virtual")
	315
	316	@property
	317	def system(self):
	318	return self._data.get("system")
	319
	320	@property
	321	def release(self):
	322	return self.system.get("release")
	323
	324	@property
	325	def kernel(self):
	326	return self.system.get("kernel_release")
	327
	328	@property
	329	def memory(self):
	330	return self.system.get("memory")
	331
abc3bb31 MT	332	@property
	333	def friendly_memory(self):
	334	units = ("k", "M", "G", "T")
	335
	336	mem = self.memory
	337	i = 0
	338	while mem >= 1024:
	339	mem /= 1024
	340	i += 1
	341
	342	return "%d%s" % (round(mem, 0), units[i])
	343
372efc19 MT	344	@property
372efc19 MT	345	def root_size(self):
91a446f0 MT	346	return self.system.get("root_size") or 0
91a446f0 MT	347
abc3bb31 MT	348	@property
	349	def friendly_root_size(self):
	350	units = ("k", "M", "G", "T")
	351
	352	size = self.root_size
	353	if not size:
	354	return
	355
	356	i = 0
	357	while size >= 1024:
	358	size /= 1024
	359	i += 1
	360
	361	return "%d%s" % (round(size, 0), units[i])
	362
91a446f0 MT	363	@property
	364	def vendor(self):
	365	return self.system.get("vendor")
	366
	367	@property
	368	def model(self):
	369	return self.system.get("model")
	370
	371	@property
	372	def country_code(self):
	373	if self._geoip:
	374	return self._geoip["country_code"].lower()
372efc19	375
91a446f0	376	return "unknown"
372efc19	377
07785a9c MT	378	@property
	379	def network(self):
	380	network = self._data.get("network", None)
	381	if network:
	382	return ProfileNetwork(network)
	383
372efc19 MT	384
	385	class Stasy(object):
	386	__metaclass__ = Singleton
	387
	388	def __init__(self):
	389	# Initialize database connection
	390	self._conn = pymongo.Connection(DATABASE_HOST)
	391	self._db = self._conn[DATABASE_NAME]
	392
	393	def get_profile_count(self):
	394	# XXX need to implement something to get profiles updated since
	395	# a given date
	396
	397	# All distinct profiles (based on public_id)
cbb9726a MT	398	# XXX possibly bad performance
	399	return len(self._db.profiles.distinct("public_id"))
	400
	401	def get_archives_count(self):
	402	return self._db.archives.count()
372efc19 MT	403
	404	#def _get_profile_cursor(self, public_id):
	405	# c = self._db.profiles.find({ "public_id" : public_id })
	406	# c.sort("updated", pymongo.ASCENDING)
	407	#
	408	# return c
	409
91a446f0	410	def profile_exists(self, public_id):
910be037	411	return self.query({ "public_id" : public_id }).count() >= 1
91a446f0	412
372efc19	413	def get_profile(self, public_id):
54af860e	414	p = None
372efc19	415	# XXX should only find one object in the end
910be037	416	for p in self.query({ "public_id" : public_id }):
91a446f0	417	p = Profile(p)
372efc19 MT	418
	419	return p
	420
	421	def get_profiles(self):
	422	# XXX needs nicer database query
	423	profiles = []
	424	for p in self._db.profiles.find():
	425	if not p.get("public_id") in profiles:
	426	profiles.append(p.get("public_id"))
	427
	428	return profiles
	429
62c3fa48	430	def query(self, query, archives=False, no_virt=False, all=False, fields=None):
910be037 MT	431	db = self._db.profiles
	432
	433	if archives:
	434	db = self.db.archives
	435
	436	if not all:
	437	# XXX cannot use the index?
	438	query.update({ "profile" : { "$exists" : True }})
	439
	440	if no_virt:
	441	query.update({ "profile.system.virtual" : False })
	442
	443	logging.debug("Executing query: %s" % query)
	444
62c3fa48	445	return db.find(query, fields=fields)
910be037	446
372efc19 MT	447	@property
	448	def secret_ids(self):
	449	return self._db.profiles.distinct("secret_id")
	450
	451	@property
	452	def cpus(self):
910be037	453	return self.query({}, no_virt=True).distinct("profile.cpu")
372efc19 MT	454
	455	@property
	456	def cpu_vendors(self):
910be037	457	return self.query({}, no_virt=True).distinct("profile.cpu.vendor")
372efc19 MT	458
372efc19 MT	459	@property
91a446f0	460	def cpu_vendors_map(self):
372efc19 MT	461	cpus = {}
	462
	463	for vendor in self.cpu_vendors:
52da761b MT	464	friendly_vendor = CPU_VENDORS.get(vendor, vendor)
52da761b MT	465	cpus[friendly_vendor] = \
910be037	466	self.query({
372efc19	467	"profile.cpu.vendor" : vendor
910be037	468	}, no_virt=True).count()
372efc19 MT	469
	470	return cpus
	471
	472	@property
91a446f0 MT	473	def cpu_speed_average(self):
	474	speed = 0
	475
910be037	476	all = self.query({}, no_virt=True)
91a446f0 MT	477
	478	# XXX ugly. needs to be done by group()
	479	for m in all:
	480	if not m.has_key("profile"):
	481	continue
	482	speed += m.get("profile").get("cpu").get("speed")
	483
	484	return (speed / all.count())
	485
62c3fa48 MT	486	def get_cpu_bogomips_accumulated(self):
	487	profiles = self.query({}, no_virt=True, fields=["profile.cpu.bogomips"])
	488
	489	return sum([int(o["profile"]["cpu"]["bogomips"]) for o in profiles])
	490
91a446f0 MT	491	@property
	492	def cpu_speed_map(self):
	493	cpu_speeds = {}
	494
	495	for i in range(len(CPU_SPEED_CONSTRAINTS) - 1):
	496	min, max = CPU_SPEED_CONSTRAINTS[i:i+2]
	497
	498	cpu_speeds[min, max] = \
910be037 MT	499	self.query({
910be037 MT	500	"profile.cpu.speed" : {
91a446f0 MT	501	"$gte" : min, "$lt" : max
91a446f0 MT	502	}
910be037	503	}, no_virt=True).count()
91a446f0 MT	504
	505	return cpu_speeds
	506
62c3fa48 MT	507	def get_cpu_cores_map(self):
	508	cores = {}
	509
	510	for i in CPU_CORES:
	511	cores[i] = \
	512	self.query({
	513	"profile.cpu.count" : i,
	514	}, no_virt=True).count()
	515
	516	return cores
	517
91a446f0	518	def get_memory_map(self):
372efc19 MT	519	memory = {}
	520
	521	for i in range(len(MEMORY_CONSTRAINTS) - 1):
	522	min, max = MEMORY_CONSTRAINTS[i:i+2]
	523
	524	memory[min, max] = \
910be037	525	self.query(
372efc19 MT	526	{ "profile.system.memory" : {
	527	"$gte" : min * 1024, "$lt" : max * 1024
	528	}
910be037	529	}, no_virt=True).count()
372efc19 MT	530
	531	return memory
	532
	533	@property
	534	def memory_average(self):
	535	memory = 0
	536
910be037	537	all = self.query({}, no_virt=True)
372efc19 MT	538
	539	# XXX ugly. needs to be done by group()
	540	for m in all:
	541	if not m.has_key("profile"):
	542	continue
	543	memory += int(m.get("profile").get("system").get("memory"))
	544
	545	return (memory / all.count()) / 1024
	546
	547	@property
	548	def hypervisor_vendors(self):
910be037	549	return self.query({}).distinct("profile.hypervisor.vendor")
372efc19 MT	550
	551	@property
	552	def hypervisor_map(self):
	553	hypervisors = {}
	554
	555	for hypervisor in self.hypervisor_vendors:
	556	hypervisors[hypervisor] = \
910be037	557	self.query({
372efc19 MT	558	"profile.hypervisor.vendor" : hypervisor
	559	}).count()
	560
	561	return hypervisors
	562
	563	@property
	564	def hypervisor_models(self):
910be037	565	return self.query({}).distinct("profile.hypervisor.model")
372efc19 MT	566
	567	@property
	568	def virtual_map(self):
	569	virtual = {
	570	True: None,
	571	False: None,
	572	}
	573
	574	for k in virtual.keys():
	575	virtual[k] = \
910be037	576	self.query({ "profile.system.virtual": k }).count()
372efc19 MT	577
	578	return virtual
	579
	580	@property
	581	def languages(self):
910be037	582	return self.query({}).distinct("profile.system.language")
372efc19	583
91a446f0 MT	584	def get_language_map(self):
	585	languages = {}
	586
	587	for language in self.languages:
	588	languages[language] = \
910be037	589	self.query({
91a446f0 MT	590	"profile.system.language" : language
	591	}).count()
	592
	593	return languages
	594
372efc19 MT	595	@property
372efc19 MT	596	def vendors(self):
910be037	597	return self.query({}).distinct("profile.system.vendor")
372efc19 MT	598
	599	@property
	600	def vendor_map(self):
	601	vendors = {}
	602
	603	for vendor in self.vendors:
	604	vendors[vendor] = \
910be037	605	self.query({
372efc19 MT	606	"profile.system.vendor" : vendor
	607	}).count()
	608
	609	return vendors
	610
	611	@property
	612	def models(self):
910be037	613	return self.query({}).distinct("profile.system.model")
372efc19	614
91a446f0 MT	615	@property
	616	def model_map(self):
	617	models = {}
	618
	619	for model in self.models:
	620	models[model] = \
910be037	621	self.query({
91a446f0 MT	622	"profile.system.model" : model
	623	}).count()
	624
	625	return models
	626
	627	@property
	628	def arches(self):
910be037	629	return self.query({}).distinct("profile.cpu.arch")
91a446f0 MT	630
	631	@property
	632	def arch_map(self):
	633	arches = {}
	634
	635	for arch in self.arches:
	636	arches[arch] = \
910be037	637	self.query({
91a446f0 MT	638	"profile.cpu.arch" : arch
	639	}).count()
	640
	641	return arches
	642
	643	@property
	644	def kernels(self):
910be037	645	return self.query({}).distinct("profile.system.kernel_release")
91a446f0 MT	646
	647	@property
	648	def kernel_map(self):
	649	kernels = {}
	650
	651	for kernel in self.kernels:
	652	kernels[kernel] = \
910be037	653	self.query({
91a446f0 MT	654	"profile.system.kernel_release" : kernel
	655	}).count()
	656
	657	return kernels
	658
	659	@property
	660	def releases(self):
910be037	661	return self.query({}).distinct("profile.system.release")
91a446f0 MT	662
	663	@property
	664	def release_map(self):
	665	releases = {}
	666
	667	for release in self.releases:
	668	releases[release] = \
910be037	669	self.query({
91a446f0 MT	670	"profile.system.release" : release
	671	}).count()
	672
	673	return releases
	674
	675	def get_device_percentage(self, bus, vendor_id, model_id):
910be037	676	profiles_with_device = self.query({
91a446f0 MT	677	"profile.devices.subsystem" : bus,
	678	"profile.devices.vendor" : vendor_id,
	679	"profile.devices.model" : model_id,
	680	})
	681
910be037	682	profiles_all = self.query({})
91a446f0 MT	683
	684	if not profiles_all.count():
	685	return 0
	686
	687	return profiles_with_device.count() / profiles_all.count()
	688
910be037 MT	689	def get_cpu_flag_map(self, flags):
910be037 MT	690	# XXX needs a cleanup
91a446f0	691
910be037	692	_flags = { True : 0 }
91a446f0	693
910be037 MT	694	if type(flags) == type("a"):
	695	flags = [flags]
	696
	697	for flag in flags:
	698	_flags[True] += \
	699	self.query({
	700	"profile.cpu.flags" : flag,
	701	}, no_virt=True).count()
91a446f0	702
910be037	703	_flags[False] = self.query({}, no_virt=True).count() - _flags[True]
91a446f0	704
910be037	705	return _flags
91a446f0 MT	706
	707	@property
	708	def geo_locations(self):
5914104c	709	return [code.lower() for code in self.query({}, all=True).distinct("geoip.country_code")]
91a446f0 MT	710
	711	def get_geo_location_map(self):
	712	geo_locations = {}
	713
	714	count = 0
	715	for geo_location in self.geo_locations:
	716	geo_locations[geo_location] = \
910be037	717	self.query({
bc1a5c5e	718	"geoip.country_code" : geo_location.upper()
5914104c	719	}, all=True).count()
91a446f0 MT	720
	721	count += geo_locations[geo_location]
	722
638e9782 MT	723	for geo_location in geo_locations.keys():
	724	geo_locations[geo_location] /= count
	725
91a446f0 MT	726	return geo_locations
	727
	728	def get_models_by_vendor(self, subsystem, vendor_id):
	729	devices = []
	730
910be037	731	profiles_all = self.query({})
91a446f0 MT	732
	733	for profile in profiles_all:
	734	if not profile.has_key("profile"):
	735	continue
	736
	737	profile = Profile(profile)
	738
	739	for device in profile.devices:
	740	if not device.vendor == vendor_id:
	741	continue
	742
	743	if not device in devices:
	744	devices.append(device)
	745
	746	return devices
372efc19	747
07785a9c MT	748	def get_network_zones_map(self):
	749	zones = { "green" : 0, "blue" : 0, "orange" : 0, "red" : 0 }
	750
e6cff1b1 MT	751	all = self.query({ "profile.network" : { "$exists" : True }})
e6cff1b1 MT	752
07785a9c MT	753	for zone in zones.keys():
	754	zones[zone] = self.query({
	755	"profile.network.%s" % zone : True,
e6cff1b1	756	}).count() / all.count()
07785a9c MT	757
	758	return zones
	759
cbb9726a MT	760	def get_profile_ratio(self):
	761	return (self.query({}).count(), self.get_profile_count())
	762
	763	def get_updated_since(self, since, _query={}):
	764	since = datetime.datetime.utcnow() - datetime.timedelta(**since)
	765
	766	query = { "updated" : { "$gte" : since }}
	767	query.update(_query)
	768
	769	return self.query(query)
	770
	771	def get_updates_by_release_since(self, since):
	772	updates = {}
	773
	774	for release in self.releases:
	775	updates[release] = self.get_updated_since(since,
	776	{ "profile.system.release" : release }).count()
	777
	778	return updates
	779
372efc19 MT	780
	781	if __name__ == "__main__":
	782	s = Stasy()
	783