]>
Commit | Line | Data |
---|---|---|
372efc19 MT |
1 | #!/usr/bin/python |
2 | ||
91a446f0 MT |
3 | from __future__ import division |
4 | ||
372efc19 MT |
5 | import hwdata |
6 | import logging | |
7 | import pymongo | |
8 | ||
9 | from misc import Singleton | |
10 | ||
11 | DATABASE_HOST = ["irma.ipfire.org", "madeye.ipfire.org"] | |
12 | DATABASE_NAME = "stasy" | |
13 | ||
91a446f0 | 14 | CPU_SPEED_CONSTRAINTS = (0, 500, 1000, 1500, 2000, 2500, 3000, 3500) |
372efc19 MT |
15 | MEMORY_CONSTRAINTS = (0, 64, 128, 256, 512, 1024, 2048, 4096, 8128, 16384) |
16 | ||
17 | class ProfileDict(object): | |
18 | def __init__(self, data): | |
19 | self._data = data | |
20 | ||
21 | ||
22 | class ProfileCPU(ProfileDict): | |
23 | @property | |
24 | def arch(self): | |
25 | return self._data.get("arch") | |
26 | ||
27 | @property | |
28 | def vendor(self): | |
29 | return self._data.get("vendor") | |
30 | ||
91a446f0 MT |
31 | @property |
32 | def speed(self): | |
33 | return self._data.get("speed") | |
34 | ||
35 | @property | |
36 | def bogomips(self): | |
37 | return self._data.get("bogomips") | |
38 | ||
372efc19 MT |
39 | @property |
40 | def model(self): | |
41 | return self._data.get("model") | |
42 | ||
43 | @property | |
44 | def model_string(self): | |
45 | return self._data.get("model_string") | |
46 | ||
47 | @property | |
48 | def flags(self): | |
49 | return self._data.get("flags") | |
50 | ||
91a446f0 MT |
51 | @property |
52 | def count(self): | |
53 | return self._data.get("count") | |
54 | ||
372efc19 MT |
55 | @property |
56 | def capable_64bit(self): | |
57 | return "lm" in self.flags | |
58 | ||
59 | @property | |
60 | def capable_pae(self): | |
61 | return "pae" in self.flags | |
62 | ||
63 | @property | |
64 | def capable_virt(self): | |
65 | return "vmx" in self.flags or "svm" in self.flags | |
66 | ||
67 | ||
68 | class ProfileHypervisor(ProfileDict): | |
69 | def __repr__(self): | |
70 | return "<%s %s-%s>" % (self.__class__.__name__, self.vendor, self.type) | |
71 | ||
72 | @property | |
73 | def vendor(self): | |
74 | return self._data.get("vendor") | |
75 | ||
372efc19 MT |
76 | @property |
77 | def type(self): | |
78 | return self._data.get("type") | |
79 | ||
80 | ||
81 | class ProfileDevice(ProfileDict): | |
82 | subsystem2class = { | |
83 | "pci" : hwdata.PCI, | |
84 | "usb" : hwdata.USB, | |
85 | } | |
86 | ||
91a446f0 MT |
87 | classid2name = { |
88 | "pci" : { | |
89 | "00" : "unclassified", | |
90 | "01" : "Mass storage", | |
91 | "02" : "Network", | |
92 | "03" : "Display", | |
93 | "04" : "Multimedia", | |
94 | "05" : "Memory controller", | |
95 | "06" : "Bridge", | |
96 | "07" : "Communication", | |
97 | "08" : "Generic system peripheral", | |
98 | "09" : "Input device", | |
99 | "0a" : "Docking station", | |
100 | "0b" : "Processor", | |
101 | "0c" : "Serial bus", | |
102 | "0d" : "Wireless", | |
103 | "0e" : "Intelligent controller", | |
104 | "0f" : "Satellite communications controller", | |
105 | "10" : "Encryption controller", | |
106 | "11" : "Signal processing controller", | |
107 | "ff" : "Unassigned class", | |
108 | }, | |
109 | ||
110 | "usb" : { | |
111 | "00" : "???", | |
112 | "01" : "Multimedia", | |
113 | "02" : "Communication", | |
114 | "03" : "Input device", | |
115 | "05" : "Physical ???", | |
116 | "06" : "Image ???", | |
117 | "07" : "Printer", | |
118 | "08" : "Mass storage", | |
119 | "09" : "Hub", | |
120 | "0a" : "CDC-Data ???", | |
121 | "0b" : "Smart card ???", | |
122 | "0d" : "Content Security ???", | |
123 | "0e" : "Display", | |
124 | "0f" : "Personal healthcare ???", | |
125 | "dc" : "Diagnostic Device", | |
126 | "e0" : "Wireless", | |
127 | "ef" : "Misc", | |
128 | "fe" : "Application specific ???", | |
129 | "ff" : "Vendor specific ???", | |
130 | } | |
131 | } | |
132 | ||
133 | def __cmp__(self, other): | |
134 | return cmp(self.vendor, other.vendor) or \ | |
135 | cmp(self.model, other.model) or \ | |
136 | cmp(self.driver, other.driver) | |
137 | ||
372efc19 MT |
138 | @property |
139 | def model(self): | |
140 | return self._data.get("model") | |
141 | ||
142 | @property | |
143 | def model_string(self): | |
144 | cls = self.subsystem2class[self.subsystem] | |
145 | ||
146 | return cls().get_device(self.vendor, self.model) | |
147 | ||
148 | @property | |
149 | def subsystem(self): | |
150 | return self._data.get("subsystem") | |
151 | ||
152 | @property | |
153 | def vendor(self): | |
154 | return self._data.get("vendor") | |
155 | ||
156 | @property | |
157 | def vendor_string(self): | |
158 | cls = self.subsystem2class[self.subsystem] | |
159 | ||
160 | return cls().get_vendor(self.vendor) | |
161 | ||
162 | @property | |
163 | def driver(self): | |
164 | return self._data.get("driver") | |
165 | ||
91a446f0 MT |
166 | @property |
167 | def cls(self): | |
168 | classid = self._data.get("deviceclass") | |
169 | ||
170 | if self.subsystem == "pci": | |
171 | classid = classid[:-4] | |
172 | if len(classid) == 1: | |
173 | classid = "0%s" % classid | |
174 | ||
175 | elif self.subsystem == "usb" and classid: | |
176 | classid = classid.split("/")[0] | |
177 | classid = "%02x" % int(classid) | |
178 | ||
179 | try: | |
180 | return self.classid2name[self.subsystem][classid] | |
181 | except KeyError: | |
182 | return "N/A" | |
183 | ||
372efc19 MT |
184 | |
185 | class Profile(ProfileDict): | |
91a446f0 MT |
186 | def __init__(self, profile_blob): |
187 | ProfileDict.__init__(self, profile_blob.get("profile")) | |
372efc19 | 188 | |
91a446f0 MT |
189 | self.public_id = profile_blob.get("public_id") |
190 | self.updated = profile_blob.get("updated") | |
191 | self._geoip = profile_blob.get("geoip", None) | |
372efc19 MT |
192 | |
193 | def __repr__(self): | |
194 | return "<%s %s>" % (self.__class__.__name__, self.public_id) | |
195 | ||
196 | @property | |
197 | def cpu(self): | |
198 | return ProfileCPU(self._data.get("cpu")) | |
199 | ||
200 | @property | |
201 | def devices(self): | |
202 | devices = [] | |
203 | for d in self._data.get("devices"): | |
204 | d = ProfileDevice(d) | |
205 | ||
91a446f0 | 206 | if d.driver in ("pcieport", "usb", "hub"): |
372efc19 MT |
207 | continue |
208 | ||
209 | devices.append(d) | |
210 | ||
211 | return devices | |
212 | ||
213 | @property | |
214 | def hypervisor(self): | |
215 | if self.virtual: | |
216 | return ProfileHypervisor(self._data.get("hypervisor")) | |
217 | ||
218 | @property | |
219 | def virtual(self): | |
220 | return self.system.get("virtual") | |
221 | ||
222 | @property | |
223 | def system(self): | |
224 | return self._data.get("system") | |
225 | ||
226 | @property | |
227 | def release(self): | |
228 | return self.system.get("release") | |
229 | ||
230 | @property | |
231 | def kernel(self): | |
232 | return self.system.get("kernel_release") | |
233 | ||
234 | @property | |
235 | def memory(self): | |
236 | return self.system.get("memory") | |
237 | ||
238 | @property | |
239 | def root_size(self): | |
91a446f0 MT |
240 | return self.system.get("root_size") or 0 |
241 | ||
242 | @property | |
243 | def vendor(self): | |
244 | return self.system.get("vendor") | |
245 | ||
246 | @property | |
247 | def model(self): | |
248 | return self.system.get("model") | |
249 | ||
250 | @property | |
251 | def country_code(self): | |
252 | if self._geoip: | |
253 | return self._geoip["country_code"].lower() | |
372efc19 | 254 | |
91a446f0 | 255 | return "unknown" |
372efc19 MT |
256 | |
257 | ||
258 | class Stasy(object): | |
259 | __metaclass__ = Singleton | |
260 | ||
261 | def __init__(self): | |
262 | # Initialize database connection | |
263 | self._conn = pymongo.Connection(DATABASE_HOST) | |
264 | self._db = self._conn[DATABASE_NAME] | |
265 | ||
266 | def get_profile_count(self): | |
267 | # XXX need to implement something to get profiles updated since | |
268 | # a given date | |
269 | ||
270 | # All distinct profiles (based on public_id) | |
271 | return self._db.profiles.distinct("public_id").count() | |
272 | ||
273 | #def _get_profile_cursor(self, public_id): | |
274 | # c = self._db.profiles.find({ "public_id" : public_id }) | |
275 | # c.sort("updated", pymongo.ASCENDING) | |
276 | # | |
277 | # return c | |
278 | ||
91a446f0 MT |
279 | def profile_exists(self, public_id): |
280 | return self._db.profiles.find({ "public_id" : public_id }).count() >= 1 | |
281 | ||
372efc19 MT |
282 | def get_profile(self, public_id): |
283 | # XXX should only find one object in the end | |
284 | for p in self._db.profiles.find({ "public_id" : public_id }): | |
91a446f0 | 285 | p = Profile(p) |
372efc19 MT |
286 | |
287 | return p | |
288 | ||
289 | def get_profiles(self): | |
290 | # XXX needs nicer database query | |
291 | profiles = [] | |
292 | for p in self._db.profiles.find(): | |
293 | if not p.get("public_id") in profiles: | |
294 | profiles.append(p.get("public_id")) | |
295 | ||
296 | return profiles | |
297 | ||
298 | @property | |
299 | def secret_ids(self): | |
300 | return self._db.profiles.distinct("secret_id") | |
301 | ||
302 | @property | |
303 | def cpus(self): | |
304 | return self._db.profiles.distinct("profile.cpu") | |
305 | ||
306 | @property | |
307 | def cpu_vendors(self): | |
308 | return self._db.profiles.distinct("profile.cpu.vendor") | |
309 | ||
310 | @property | |
91a446f0 | 311 | def cpu_vendors_map(self): |
372efc19 MT |
312 | cpus = {} |
313 | ||
314 | for vendor in self.cpu_vendors: | |
315 | cpus[vendor] = \ | |
316 | self._db.profiles.find({ | |
317 | "profile.cpu.vendor" : vendor | |
318 | }).count() | |
319 | ||
320 | return cpus | |
321 | ||
322 | @property | |
91a446f0 MT |
323 | def cpu_speed_average(self): |
324 | speed = 0 | |
325 | ||
326 | all = self._db.profiles.find() | |
327 | ||
328 | # XXX ugly. needs to be done by group() | |
329 | for m in all: | |
330 | if not m.has_key("profile"): | |
331 | continue | |
332 | speed += m.get("profile").get("cpu").get("speed") | |
333 | ||
334 | return (speed / all.count()) | |
335 | ||
336 | @property | |
337 | def cpu_speed_map(self): | |
338 | cpu_speeds = {} | |
339 | ||
340 | for i in range(len(CPU_SPEED_CONSTRAINTS) - 1): | |
341 | min, max = CPU_SPEED_CONSTRAINTS[i:i+2] | |
342 | ||
343 | cpu_speeds[min, max] = \ | |
344 | self._db.profiles.find( | |
345 | { "profile.cpu.speed" : { | |
346 | "$gte" : min, "$lt" : max | |
347 | } | |
348 | }).count() | |
349 | ||
350 | return cpu_speeds | |
351 | ||
352 | def get_memory_map(self): | |
372efc19 MT |
353 | memory = {} |
354 | ||
355 | for i in range(len(MEMORY_CONSTRAINTS) - 1): | |
356 | min, max = MEMORY_CONSTRAINTS[i:i+2] | |
357 | ||
358 | memory[min, max] = \ | |
359 | self._db.profiles.find( | |
360 | { "profile.system.memory" : { | |
361 | "$gte" : min * 1024, "$lt" : max * 1024 | |
362 | } | |
363 | }).count() | |
364 | ||
365 | return memory | |
366 | ||
367 | @property | |
368 | def memory_average(self): | |
369 | memory = 0 | |
370 | ||
371 | all = self._db.profiles.find() | |
372 | ||
373 | # XXX ugly. needs to be done by group() | |
374 | for m in all: | |
375 | if not m.has_key("profile"): | |
376 | continue | |
377 | memory += int(m.get("profile").get("system").get("memory")) | |
378 | ||
379 | return (memory / all.count()) / 1024 | |
380 | ||
381 | @property | |
382 | def hypervisor_vendors(self): | |
383 | return self._db.profiles.distinct("profile.hypervisor.vendor") | |
384 | ||
385 | @property | |
386 | def hypervisor_map(self): | |
387 | hypervisors = {} | |
388 | ||
389 | for hypervisor in self.hypervisor_vendors: | |
390 | hypervisors[hypervisor] = \ | |
391 | self._db.profiles.find({ | |
392 | "profile.hypervisor.vendor" : hypervisor | |
393 | }).count() | |
394 | ||
395 | return hypervisors | |
396 | ||
397 | @property | |
398 | def hypervisor_models(self): | |
399 | return self._db.profiles.distinct("profile.hypervisor.model") | |
400 | ||
401 | @property | |
402 | def virtual_map(self): | |
403 | virtual = { | |
404 | True: None, | |
405 | False: None, | |
406 | } | |
407 | ||
408 | for k in virtual.keys(): | |
409 | virtual[k] = \ | |
410 | self._db.profiles.find({ "profile.system.virtual": k }).count() | |
411 | ||
412 | return virtual | |
413 | ||
414 | @property | |
415 | def languages(self): | |
416 | return self._db.profiles.distinct("profile.system.language") | |
417 | ||
91a446f0 MT |
418 | def get_language_map(self): |
419 | languages = {} | |
420 | ||
421 | for language in self.languages: | |
422 | languages[language] = \ | |
423 | self._db.profiles.find({ | |
424 | "profile.system.language" : language | |
425 | }).count() | |
426 | ||
427 | return languages | |
428 | ||
372efc19 MT |
429 | @property |
430 | def vendors(self): | |
431 | return self._db.profiles.distinct("profile.system.vendor") | |
432 | ||
433 | @property | |
434 | def vendor_map(self): | |
435 | vendors = {} | |
436 | ||
437 | for vendor in self.vendors: | |
438 | vendors[vendor] = \ | |
439 | self._db.profiles.find({ | |
440 | "profile.system.vendor" : vendor | |
441 | }).count() | |
442 | ||
443 | return vendors | |
444 | ||
445 | @property | |
446 | def models(self): | |
447 | return self._db.profiles.distinct("profile.system.model") | |
448 | ||
91a446f0 MT |
449 | @property |
450 | def model_map(self): | |
451 | models = {} | |
452 | ||
453 | for model in self.models: | |
454 | models[model] = \ | |
455 | self._db.profiles.find({ | |
456 | "profile.system.model" : model | |
457 | }).count() | |
458 | ||
459 | return models | |
460 | ||
461 | @property | |
462 | def arches(self): | |
463 | return self._db.profiles.distinct("profile.cpu.arch") | |
464 | ||
465 | @property | |
466 | def arch_map(self): | |
467 | arches = {} | |
468 | ||
469 | for arch in self.arches: | |
470 | arches[arch] = \ | |
471 | self._db.profiles.find({ | |
472 | "profile.cpu.arch" : arch | |
473 | }).count() | |
474 | ||
475 | return arches | |
476 | ||
477 | @property | |
478 | def kernels(self): | |
479 | return self._db.profiles.distinct("profile.system.kernel_release") | |
480 | ||
481 | @property | |
482 | def kernel_map(self): | |
483 | kernels = {} | |
484 | ||
485 | for kernel in self.kernels: | |
486 | kernels[kernel] = \ | |
487 | self._db.profiles.find({ | |
488 | "profile.system.kernel_release" : kernel | |
489 | }).count() | |
490 | ||
491 | return kernels | |
492 | ||
493 | @property | |
494 | def releases(self): | |
495 | return self._db.profiles.distinct("profile.system.release") | |
496 | ||
497 | @property | |
498 | def release_map(self): | |
499 | releases = {} | |
500 | ||
501 | for release in self.releases: | |
502 | releases[release] = \ | |
503 | self._db.profiles.find({ | |
504 | "profile.system.release" : release | |
505 | }).count() | |
506 | ||
507 | return releases | |
508 | ||
509 | def get_device_percentage(self, bus, vendor_id, model_id): | |
510 | profiles_with_device = self._db.profiles.find({ | |
511 | "profile.devices.subsystem" : bus, | |
512 | "profile.devices.vendor" : vendor_id, | |
513 | "profile.devices.model" : model_id, | |
514 | }) | |
515 | ||
516 | # XXX must only be systems that have profile data | |
517 | profiles_all = self._db.profiles.find() | |
518 | ||
519 | if not profiles_all.count(): | |
520 | return 0 | |
521 | ||
522 | return profiles_with_device.count() / profiles_all.count() | |
523 | ||
524 | def get_cpu_flag_map(self, flag): | |
525 | flags = {} | |
526 | ||
527 | flags[True] = \ | |
528 | self._db.profiles.find({ | |
529 | "profile.cpu.flags" : flag, | |
530 | }).count() | |
531 | ||
532 | # XXX must only be systems that have profile data | |
533 | profiles_all = self._db.profiles.find() | |
534 | ||
535 | flags[False] = profiles_all.count() - flags[True] | |
536 | ||
537 | return flags | |
538 | ||
539 | @property | |
540 | def geo_locations(self): | |
541 | return [code.lower() for code in self._db.profiles.distinct("geoip.country_code")] | |
542 | ||
543 | def get_geo_location_map(self): | |
544 | geo_locations = {} | |
545 | ||
546 | count = 0 | |
547 | for geo_location in self.geo_locations: | |
548 | geo_locations[geo_location] = \ | |
549 | self._db.profiles.find({ | |
550 | "geoip.country_code" : geo_location | |
551 | }).count() | |
552 | ||
553 | count += geo_locations[geo_location] | |
554 | ||
555 | # XXX must only be systems that have profile data | |
556 | profiles_all = self._db.profiles.find() | |
557 | ||
558 | unknown_count = profiles_all.count() - count | |
559 | if unknown_count: | |
560 | geo_locations["unknown"] = unknown_count | |
561 | ||
562 | return geo_locations | |
563 | ||
564 | def get_models_by_vendor(self, subsystem, vendor_id): | |
565 | devices = [] | |
566 | ||
567 | # XXX must only be systems that have profile data | |
568 | profiles_all = self._db.profiles.find() | |
569 | ||
570 | for profile in profiles_all: | |
571 | if not profile.has_key("profile"): | |
572 | continue | |
573 | ||
574 | profile = Profile(profile) | |
575 | ||
576 | for device in profile.devices: | |
577 | if not device.vendor == vendor_id: | |
578 | continue | |
579 | ||
580 | if not device in devices: | |
581 | devices.append(device) | |
582 | ||
583 | return devices | |
372efc19 MT |
584 | |
585 | ||
586 | if __name__ == "__main__": | |
587 | s = Stasy() | |
588 | ||
589 | print s.get_profile("0" * 40) | |
590 | print s.cpu_vendors | |
591 | for id in s.secret_ids: | |
592 | print "\t", id | |
593 | ||
594 | #for p in s._db.profiles.find(): | |
595 | # print p | |
596 | ||
597 | print s.cpu_map | |
598 | print s.memory_map | |
599 | print s.memory_average | |
600 | print s.hypervisor_vendors | |
601 | print s.hypervisor_models | |
602 | print s.languages | |
91a446f0 | 603 | print s.language_maps |
372efc19 MT |
604 | print s.vendors |
605 | print s.vendor_map | |
606 | print s.models | |
607 | print s.cpus | |
91a446f0 MT |
608 | print s.cpu_map |
609 | print s.arches | |
610 | print s.arch_map | |
611 | print s.kernels | |
612 | print s.kernel_map | |
613 | print s.releases | |
614 | print s.release_map | |
372efc19 MT |
615 | |
616 | p = s.get_profile("0b5f4fe2162fdfbfa29b632610e317078fa70d34") | |
91a446f0 MT |
617 | print p._data |
618 | # print p.hypervisor | |
619 |