]>
git.ipfire.org Git - people/shoehn/ipfire.org.git/blob - webapp/backend/stasy.py
87fe2fc72cd0b53c5b0fb70150f56cee522468dd
3 from __future__
import division
11 from misc
import Object
13 DATABASE_HOST
= ["wilhelmina.ipfire.org", "miranda.ipfire.org", "falco.ipfire.org",]
14 DATABASE_NAME
= "stasy"
16 CPU_SPEED_CONSTRAINTS
= (0, 500, 1000, 1500, 2000, 2500, 3000, 3500)
17 MEMORY_CONSTRAINTS
= (0, 128, 256, 512, 1024, 2048, 4096, 8192, 16384)
20 "AMDisbetter!" : "AMD",
21 "AuthenticAMD" : "AMD",
22 "CentaurHauls" : "VIA",
23 "CyrixInstead" : "Cyrix",
24 "GenuineIntel" : "Intel",
25 "TransmetaCPU" : "Transmeta",
26 "GenuineTMx86" : "Transmeta",
27 "Geode by NSC" : "NSC",
28 "NexGenDriven" : "NexGen",
29 "RiseRiseRise" : "Rise",
30 "SiS SiS SiS" : "SiS",
31 "SiS SiS SiS " : "SiS",
32 "UMC UMC UMC " : "UMC",
33 "VIA VIA VIA " : "VIA",
34 "Vortex86 SoC" : "Vortex86",
39 (r
"(AMD Athlon).*(XP).*", r
"\1 \2"),
40 (r
"(AMD Phenom).* ([0-9]+) .*", r
"\1 \2"),
41 (r
"(AMD Phenom).*", r
"\1"),
42 (r
"(AMD Sempron).*", r
"\1"),
43 (r
"AMD Athlon.* II X2 ([a-z0-9]+).*", r
"AMD Athlon X2 \1"),
44 (r
"(Geode).*", r
"\1"),
47 (r
"Intel.*(Atom|Celeron).*CPU\s*([A-Z0-9]+) .*", r
"Intel \1 \2"),
48 (r
"(Intel).*(Celeron).*", r
"\1 \2"),
49 (r
"Intel.* Core.*2 Duo *CPU .* ([A-Z0-9]+) .*", r
"Intel C2D \1"),
50 (r
"Intel.* Core.*2 CPU .* ([A-Z0-9]+) .*", r
"Intel C2 \1"),
51 (r
"Intel.* Core.*2 Quad *CPU .* ([A-Z0-9]+) .*", r
"Intel C2Q \1"),
52 (r
"Intel.* Xeon.* CPU .* ([A-Z0-9]+) .*", r
"Intel Xeon \1"),
53 (r
"(Intel).*(Xeon).*", r
"\1 \2"),
54 (r
"Intel.* Pentium.* (D|4) .*", r
"Intel Pentium \1"),
55 (r
"Intel.* Pentium.* Dual .* ([A-Z0-9]+) .*", r
"Intel Pentium Dual \1"),
56 (r
"Pentium.* Dual-Core .* ([A-Z0-9]+) .*", r
"Intel Pentium Dual \1"),
57 (r
"(Pentium I{2,3}).*", r
"Intel \1"),
58 (r
"(Celeron \(Coppermine\))", r
"Intel Celeron"),
61 (r
"(VIA \w*).*", r
"\1"),
64 (r
"QEMU Virtual CPU version .*", r
"QEMU CPU"),
67 (r
"Feroceon .*", r
"ARM Feroceon"),
70 CPU_CORES
= range(1, 9)
72 class ProfileDict(object):
73 def __init__(self
, data
):
77 class ProfileCPU(ProfileDict
):
80 return self
._data
.get("arch")
84 vendor
= self
._data
.get("vendor")
86 return CPU_VENDORS
.get(vendor
, vendor
)
90 speed
= self
._data
.get("speed")
92 # ARM boxes do not report speed but
93 # bogomips equals speed
100 def friendly_speed(self
):
101 if self
.speed
< 1000:
102 return "%dMHz" % self
.speed
104 return "%.1fGHz" % round(self
.speed
/ 1000, 1)
108 return self
._data
.get("bogomips")
112 return self
._data
.get("model")
115 def model_string(self
):
116 return self
._data
.get("model_string")
120 return self
._data
.get("flags")
124 return self
._data
.get("count")
127 def capable_64bit(self
):
128 return "lm" in self
.flags
131 def capable_pae(self
):
132 return "pae" in self
.flags
135 def capable_virt(self
):
136 return "vmx" in self
.flags
or "svm" in self
.flags
139 def friendly_vendor(self
):
140 s
= self
.model_string
141 for pattern
, repl
in CPU_STRINGS
:
142 if re
.match(pattern
, s
) is None:
144 return re
.sub(pattern
, repl
, s
)
149 def friendly_string(self
):
150 s
= self
.friendly_vendor
153 s
+= " @ %s" % self
.friendly_speed
158 class ProfileHypervisor(ProfileDict
):
160 return "<%s %s-%s>" % (self
.__class
__.__name
__, self
.vendor
, self
.type)
164 return self
._data
.get("vendor")
168 return self
._data
.get("type")
171 class ProfileNetwork(ProfileDict
):
172 def has_zone(self
, name
):
173 return self
._data
.get(name
)
177 return self
._data
.get("green")
181 return self
._data
.get("red")
185 return self
._data
.get("blue")
189 return self
._data
.get("orange")
192 class ProfileDevice(ProfileDict
):
200 "00" : "Unclassified",
201 "01" : "Mass storage",
205 "05" : "Memory controller",
207 "07" : "Communication",
208 "08" : "Generic system peripheral",
209 "09" : "Input device",
210 "0a" : "Docking station",
214 "0e" : "Intelligent controller",
215 "0f" : "Satellite communications controller",
217 "11" : "Signal processing controller",
218 "ff" : "Unassigned class",
222 "00" : "Unclassified",
224 "02" : "Communication",
225 "03" : "Input device",
226 "05" : "Generic system peripheral",
229 "08" : "Mass storage",
231 "0a" : "Communication",
235 "0f" : "Personal Healthcare",
236 "dc" : "Diagnostic Device",
238 "ef" : "Unclassified",
239 "fe" : "Unclassified",
240 "ff" : "Unclassified",
244 def __cmp__(self
, other
):
245 return cmp(self
.vendor
, other
.vendor
) or \
246 cmp(self
.model
, other
.model
) or \
247 cmp(self
.driver
, other
.driver
)
251 return self
._data
.get("model")
254 def model_string(self
):
255 cls
= self
.subsystem2class
[self
.subsystem
]
257 return cls().get_device(self
.vendor
, self
.model
)
261 return self
._data
.get("subsystem")
265 return self
._data
.get("vendor")
268 def vendor_string(self
):
269 cls
= self
.subsystem2class
[self
.subsystem
]
271 return cls().get_vendor(self
.vendor
)
275 return self
._data
.get("driver")
279 classid
= self
._data
.get("deviceclass")
281 if self
.subsystem
== "pci":
282 classid
= classid
[:-4]
283 if len(classid
) == 1:
284 classid
= "0%s" % classid
286 elif self
.subsystem
== "usb" and classid
:
287 classid
= classid
.split("/")[0]
288 classid
= "%02x" % int(classid
)
291 return self
.classid2name
[self
.subsystem
][classid
]
296 class Profile(ProfileDict
):
297 def __init__(self
, profile_blob
):
298 ProfileDict
.__init
__(self
, profile_blob
.get("profile"))
300 self
.public_id
= profile_blob
.get("public_id")
301 self
.updated
= profile_blob
.get("updated")
302 self
._geoip
= profile_blob
.get("geoip", None)
305 return "<%s %s>" % (self
.__class
__.__name
__, self
.public_id
)
309 return ProfileCPU(self
._data
.get("cpu"))
314 for d
in self
._data
.get("devices"):
317 if d
.driver
in ("pcieport", "usb", "hub"):
325 def hypervisor(self
):
327 return ProfileHypervisor(self
._data
.get("hypervisor"))
331 return self
.system
.get("virtual")
335 return self
._data
.get("system")
339 return self
.system
.get("release")
343 return self
.system
.get("kernel_release")
347 return self
.system
.get("memory")
350 def friendly_memory(self
):
351 units
= ("k", "M", "G", "T")
359 return "%d%s" % (round(mem
, 0), units
[i
])
363 return self
.system
.get("root_size") or 0
366 def friendly_root_size(self
):
367 units
= ("k", "M", "G", "T")
369 size
= self
.root_size
378 return "%d%s" % (round(size
, 0), units
[i
])
382 return self
.system
.get("vendor")
386 return self
.system
.get("model")
389 def country_code(self
):
391 return self
._geoip
["country_code"].lower()
397 network
= self
._data
.get("network", None)
399 return ProfileNetwork(network
)
404 # Initialize database connection
405 self
._conn
= pymongo
.Connection(DATABASE_HOST
)
406 self
._db
= self
._conn
[DATABASE_NAME
]
408 def get_profile_count(self
):
409 # XXX need to implement something to get profiles updated since
412 # All distinct profiles (based on public_id)
413 # XXX possibly bad performance
414 return len(self
._db
.profiles
.distinct("public_id"))
416 # Shortcuts to database collections.
420 return self
._db
.archives
424 return self
._db
.profiles
426 def get_archives_count(self
):
427 return self
._db
.archives
.count()
429 #def _get_profile_cursor(self, public_id):
430 # c = self._db.profiles.find({ "public_id" : public_id })
431 # c.sort("updated", pymongo.ASCENDING)
435 def profile_exists(self
, public_id
):
436 return self
.query({ "public_id" : public_id
}).count() >= 1
438 def get_profile(self
, public_id
):
440 # XXX should only find one object in the end
441 for p
in self
.query({ "public_id" : public_id
}):
446 def get_profiles(self
):
447 # XXX needs nicer database query
449 for p
in self
._db
.profiles
.find():
450 if not p
.get("public_id") in profiles
:
451 profiles
.append(p
.get("public_id"))
455 def move_profiles(self
, find
):
457 Move all profiles by the "find" criteria.
459 for p
in self
.profiles
.find(find
):
460 self
.archives
.save(p
)
461 self
.profiles
.remove(find
)
463 def query(self
, query
, archives
=False, no_virt
=False, all
=False, fields
=None):
464 db
= self
._db
.profiles
467 db
= self
.db
.archives
470 # XXX cannot use the index?
471 query
.update({ "profile" : { "$exists" : True }})
474 query
.update({ "profile.system.virtual" : False })
476 logging
.debug("Executing query: %s" % query
)
478 return db
.find(query
, fields
=fields
)
481 def secret_ids(self
):
482 return self
._db
.profiles
.distinct("secret_id")
486 return self
.query({}, no_virt
=True).distinct("profile.cpu")
489 def cpu_vendors(self
):
490 return self
.query({}, no_virt
=True).distinct("profile.cpu.vendor")
493 def cpu_vendors_map(self
):
496 for vendor
in self
.cpu_vendors
:
497 friendly_vendor
= CPU_VENDORS
.get(vendor
, vendor
)
498 cpus
[friendly_vendor
] = \
500 "profile.cpu.vendor" : vendor
501 }, no_virt
=True).count()
506 def cpu_speed_average(self
):
509 all
= self
.query({}, no_virt
=True)
511 # XXX ugly. needs to be done by group()
513 if not m
.has_key("profile"):
515 speed
+= m
.get("profile").get("cpu").get("speed")
517 return (speed
/ all
.count())
519 def get_cpu_bogomips_accumulated(self
):
520 profiles
= self
.query({}, no_virt
=True, fields
=["profile.cpu.bogomips"])
522 return sum([int(o
["profile"]["cpu"]["bogomips"]) for o
in profiles
])
525 def cpu_speed_map(self
):
528 for i
in range(len(CPU_SPEED_CONSTRAINTS
) - 1):
529 min, max = CPU_SPEED_CONSTRAINTS
[i
:i
+2]
531 cpu_speeds
[min, max] = \
533 "profile.cpu.speed" : {
534 "$gte" : min, "$lt" : max
536 }, no_virt
=True).count()
540 def get_cpu_cores_map(self
):
546 "profile.cpu.count" : i
,
547 }, no_virt
=True).count()
551 def get_memory_map(self
):
554 for i
in range(len(MEMORY_CONSTRAINTS
) - 1):
555 min, max = MEMORY_CONSTRAINTS
[i
:i
+2]
559 { "profile.system.memory" : {
560 "$gte" : min * 1024, "$lt" : max * 1024
562 }, no_virt
=True).count()
567 def memory_average(self
):
570 all
= self
.query({}, no_virt
=True)
572 # XXX ugly. needs to be done by group()
574 if not m
.has_key("profile"):
576 memory
+= int(m
.get("profile").get("system").get("memory"))
578 return (memory
/ all
.count()) / 1024
581 def hypervisor_vendors(self
):
582 return self
.query({}).distinct("profile.hypervisor.vendor")
585 def hypervisor_map(self
):
588 for hypervisor
in self
.hypervisor_vendors
:
589 hypervisors
[hypervisor
] = \
591 "profile.hypervisor.vendor" : hypervisor
597 def hypervisor_models(self
):
598 return self
.query({}).distinct("profile.hypervisor.model")
601 def virtual_map(self
):
607 for k
in virtual
.keys():
609 self
.query({ "profile.system.virtual": k
}).count()
615 return self
.query({}).distinct("profile.system.language")
617 def get_language_map(self
):
620 for language
in self
.languages
:
621 languages
[language
] = \
623 "profile.system.language" : language
630 return self
.query({}).distinct("profile.system.vendor")
633 def vendor_map(self
):
636 for vendor
in self
.vendors
:
639 "profile.system.vendor" : vendor
646 return self
.query({}).distinct("profile.system.model")
652 for model
in self
.models
:
655 "profile.system.model" : model
662 return self
.query({}).distinct("profile.cpu.arch")
668 for arch
in self
.arches
:
671 "profile.cpu.arch" : arch
678 return self
.query({}).distinct("profile.system.kernel_release")
681 def kernel_map(self
):
684 for kernel
in self
.kernels
:
687 "profile.system.kernel_release" : kernel
694 return self
.query({}).distinct("profile.system.release")
697 def release_map(self
):
700 for release
in self
.releases
:
701 releases
[release
] = \
703 "profile.system.release" : release
708 def get_device_percentage(self
, bus
, vendor_id
, model_id
):
709 profiles_with_device
= self
.query({
710 "profile.devices.subsystem" : bus
,
711 "profile.devices.vendor" : vendor_id
,
712 "profile.devices.model" : model_id
,
715 profiles_all
= self
.query({})
717 if not profiles_all
.count():
720 return profiles_with_device
.count() / profiles_all
.count()
722 def get_cpu_flag_map(self
, flags
):
723 # XXX needs a cleanup
725 _flags
= { True : 0 }
727 if type(flags
) == type("a"):
733 "profile.cpu.flags" : flag
,
734 }, no_virt
=True).count()
736 _flags
[False] = self
.query({}, no_virt
=True).count() - _flags
[True]
741 def geo_locations(self
):
742 return [code
.lower() for code
in self
.query({}, all
=True).distinct("geoip.country_code")]
744 def get_geo_location_map(self
):
748 for geo_location
in self
.geo_locations
:
749 geo_locations
[geo_location
] = \
751 "geoip.country_code" : geo_location
.upper()
754 count
+= geo_locations
[geo_location
]
756 for geo_location
in geo_locations
.keys():
757 geo_locations
[geo_location
] /= count
761 def get_models_by_vendor(self
, subsystem
, vendor_id
):
764 profiles_all
= self
.query({})
766 for profile
in profiles_all
:
767 if not profile
.has_key("profile"):
770 profile
= Profile(profile
)
772 for device
in profile
.devices
:
773 if not device
.vendor
== vendor_id
:
776 if not device
in devices
:
777 devices
.append(device
)
781 def get_network_zones_map(self
):
782 zones
= { "green" : 0, "blue" : 0, "orange" : 0, "red" : 0 }
784 all
= self
.query({ "profile.network" : { "$exists" : True }})
786 for zone
in zones
.keys():
787 zones
[zone
] = self
.query({
788 "profile.network.%s" % zone
: True,
789 }).count() / all
.count()
793 def get_profile_ratio(self
):
794 return (self
.query({}).count(), self
.get_profile_count())
796 def get_updated_since(self
, since
, _query
={}):
797 since
= datetime
.datetime
.utcnow() - datetime
.timedelta(**since
)
799 query
= { "updated" : { "$gte" : since
}}
802 return self
.query(query
)
804 def get_updates_by_release_since(self
, since
):
807 for release
in self
.releases
:
808 updates
[release
] = self
.get_updated_since(since
,
809 { "profile.system.release" : release
}).count()