]>
Commit | Line | Data |
---|---|---|
372efc19 MT |
1 | #!/usr/bin/python |
2 | ||
91a446f0 MT |
3 | from __future__ import division |
4 | ||
372efc19 MT |
5 | import hwdata |
6 | import logging | |
7 | import pymongo | |
8 | ||
9 | from misc import Singleton | |
10 | ||
11 | DATABASE_HOST = ["irma.ipfire.org", "madeye.ipfire.org"] | |
12 | DATABASE_NAME = "stasy" | |
13 | ||
91a446f0 | 14 | CPU_SPEED_CONSTRAINTS = (0, 500, 1000, 1500, 2000, 2500, 3000, 3500) |
372efc19 MT |
15 | MEMORY_CONSTRAINTS = (0, 64, 128, 256, 512, 1024, 2048, 4096, 8128, 16384) |
16 | ||
17 | class ProfileDict(object): | |
18 | def __init__(self, data): | |
19 | self._data = data | |
20 | ||
21 | ||
22 | class ProfileCPU(ProfileDict): | |
23 | @property | |
24 | def arch(self): | |
25 | return self._data.get("arch") | |
26 | ||
27 | @property | |
28 | def vendor(self): | |
29 | return self._data.get("vendor") | |
30 | ||
91a446f0 MT |
31 | @property |
32 | def speed(self): | |
33 | return self._data.get("speed") | |
34 | ||
35 | @property | |
36 | def bogomips(self): | |
37 | return self._data.get("bogomips") | |
38 | ||
372efc19 MT |
39 | @property |
40 | def model(self): | |
41 | return self._data.get("model") | |
42 | ||
43 | @property | |
44 | def model_string(self): | |
45 | return self._data.get("model_string") | |
46 | ||
47 | @property | |
48 | def flags(self): | |
49 | return self._data.get("flags") | |
50 | ||
91a446f0 MT |
51 | @property |
52 | def count(self): | |
53 | return self._data.get("count") | |
54 | ||
372efc19 MT |
55 | @property |
56 | def capable_64bit(self): | |
57 | return "lm" in self.flags | |
58 | ||
59 | @property | |
60 | def capable_pae(self): | |
61 | return "pae" in self.flags | |
62 | ||
63 | @property | |
64 | def capable_virt(self): | |
65 | return "vmx" in self.flags or "svm" in self.flags | |
66 | ||
67 | ||
68 | class ProfileHypervisor(ProfileDict): | |
69 | def __repr__(self): | |
70 | return "<%s %s-%s>" % (self.__class__.__name__, self.vendor, self.type) | |
71 | ||
72 | @property | |
73 | def vendor(self): | |
74 | return self._data.get("vendor") | |
75 | ||
372efc19 MT |
76 | @property |
77 | def type(self): | |
78 | return self._data.get("type") | |
79 | ||
80 | ||
07785a9c MT |
81 | class ProfileNetwork(ProfileDict): |
82 | def has_zone(self, name): | |
83 | return self._data.get(name) | |
84 | ||
85 | @property | |
86 | def green(self): | |
87 | return self._data.get("green") | |
88 | ||
89 | @property | |
90 | def red(self): | |
91 | return self._data.get("red") | |
92 | ||
93 | @property | |
94 | def blue(self): | |
95 | return self._data.get("blue") | |
96 | ||
97 | @property | |
98 | def orange(self): | |
99 | return self._data.get("orange") | |
100 | ||
101 | ||
372efc19 MT |
102 | class ProfileDevice(ProfileDict): |
103 | subsystem2class = { | |
104 | "pci" : hwdata.PCI, | |
105 | "usb" : hwdata.USB, | |
106 | } | |
107 | ||
91a446f0 MT |
108 | classid2name = { |
109 | "pci" : { | |
bc19cabd | 110 | "00" : "Unclassified", |
91a446f0 MT |
111 | "01" : "Mass storage", |
112 | "02" : "Network", | |
113 | "03" : "Display", | |
114 | "04" : "Multimedia", | |
115 | "05" : "Memory controller", | |
116 | "06" : "Bridge", | |
117 | "07" : "Communication", | |
118 | "08" : "Generic system peripheral", | |
119 | "09" : "Input device", | |
120 | "0a" : "Docking station", | |
121 | "0b" : "Processor", | |
122 | "0c" : "Serial bus", | |
123 | "0d" : "Wireless", | |
124 | "0e" : "Intelligent controller", | |
125 | "0f" : "Satellite communications controller", | |
bc19cabd | 126 | "10" : "Encryption", |
91a446f0 MT |
127 | "11" : "Signal processing controller", |
128 | "ff" : "Unassigned class", | |
129 | }, | |
130 | ||
131 | "usb" : { | |
bc19cabd | 132 | "00" : "Unclassified", |
91a446f0 MT |
133 | "01" : "Multimedia", |
134 | "02" : "Communication", | |
135 | "03" : "Input device", | |
bc19cabd MT |
136 | "05" : "Generic system peripheral", |
137 | "06" : "Image", | |
91a446f0 MT |
138 | "07" : "Printer", |
139 | "08" : "Mass storage", | |
140 | "09" : "Hub", | |
bc19cabd MT |
141 | "0a" : "Communication", |
142 | "0b" : "Smart card", | |
143 | "0d" : "Encryption", | |
91a446f0 | 144 | "0e" : "Display", |
bc19cabd | 145 | "0f" : "Personal Healthcare", |
91a446f0 MT |
146 | "dc" : "Diagnostic Device", |
147 | "e0" : "Wireless", | |
bc19cabd MT |
148 | "ef" : "Unclassified", |
149 | "fe" : "Unclassified", | |
150 | "ff" : "Unclassified", | |
91a446f0 MT |
151 | } |
152 | } | |
153 | ||
154 | def __cmp__(self, other): | |
155 | return cmp(self.vendor, other.vendor) or \ | |
156 | cmp(self.model, other.model) or \ | |
157 | cmp(self.driver, other.driver) | |
158 | ||
372efc19 MT |
159 | @property |
160 | def model(self): | |
161 | return self._data.get("model") | |
162 | ||
163 | @property | |
164 | def model_string(self): | |
165 | cls = self.subsystem2class[self.subsystem] | |
166 | ||
167 | return cls().get_device(self.vendor, self.model) | |
168 | ||
169 | @property | |
170 | def subsystem(self): | |
171 | return self._data.get("subsystem") | |
172 | ||
173 | @property | |
174 | def vendor(self): | |
175 | return self._data.get("vendor") | |
176 | ||
177 | @property | |
178 | def vendor_string(self): | |
179 | cls = self.subsystem2class[self.subsystem] | |
180 | ||
181 | return cls().get_vendor(self.vendor) | |
182 | ||
183 | @property | |
184 | def driver(self): | |
185 | return self._data.get("driver") | |
186 | ||
91a446f0 MT |
187 | @property |
188 | def cls(self): | |
189 | classid = self._data.get("deviceclass") | |
190 | ||
191 | if self.subsystem == "pci": | |
192 | classid = classid[:-4] | |
193 | if len(classid) == 1: | |
194 | classid = "0%s" % classid | |
195 | ||
196 | elif self.subsystem == "usb" and classid: | |
197 | classid = classid.split("/")[0] | |
198 | classid = "%02x" % int(classid) | |
199 | ||
200 | try: | |
201 | return self.classid2name[self.subsystem][classid] | |
202 | except KeyError: | |
203 | return "N/A" | |
204 | ||
372efc19 MT |
205 | |
206 | class Profile(ProfileDict): | |
91a446f0 MT |
207 | def __init__(self, profile_blob): |
208 | ProfileDict.__init__(self, profile_blob.get("profile")) | |
372efc19 | 209 | |
91a446f0 MT |
210 | self.public_id = profile_blob.get("public_id") |
211 | self.updated = profile_blob.get("updated") | |
212 | self._geoip = profile_blob.get("geoip", None) | |
372efc19 MT |
213 | |
214 | def __repr__(self): | |
215 | return "<%s %s>" % (self.__class__.__name__, self.public_id) | |
216 | ||
217 | @property | |
218 | def cpu(self): | |
219 | return ProfileCPU(self._data.get("cpu")) | |
220 | ||
221 | @property | |
222 | def devices(self): | |
223 | devices = [] | |
224 | for d in self._data.get("devices"): | |
225 | d = ProfileDevice(d) | |
226 | ||
91a446f0 | 227 | if d.driver in ("pcieport", "usb", "hub"): |
372efc19 MT |
228 | continue |
229 | ||
230 | devices.append(d) | |
231 | ||
232 | return devices | |
233 | ||
234 | @property | |
235 | def hypervisor(self): | |
236 | if self.virtual: | |
237 | return ProfileHypervisor(self._data.get("hypervisor")) | |
238 | ||
239 | @property | |
240 | def virtual(self): | |
241 | return self.system.get("virtual") | |
242 | ||
243 | @property | |
244 | def system(self): | |
245 | return self._data.get("system") | |
246 | ||
247 | @property | |
248 | def release(self): | |
249 | return self.system.get("release") | |
250 | ||
251 | @property | |
252 | def kernel(self): | |
253 | return self.system.get("kernel_release") | |
254 | ||
255 | @property | |
256 | def memory(self): | |
257 | return self.system.get("memory") | |
258 | ||
259 | @property | |
260 | def root_size(self): | |
91a446f0 MT |
261 | return self.system.get("root_size") or 0 |
262 | ||
263 | @property | |
264 | def vendor(self): | |
265 | return self.system.get("vendor") | |
266 | ||
267 | @property | |
268 | def model(self): | |
269 | return self.system.get("model") | |
270 | ||
271 | @property | |
272 | def country_code(self): | |
273 | if self._geoip: | |
274 | return self._geoip["country_code"].lower() | |
372efc19 | 275 | |
91a446f0 | 276 | return "unknown" |
372efc19 | 277 | |
07785a9c MT |
278 | @property |
279 | def network(self): | |
280 | network = self._data.get("network", None) | |
281 | if network: | |
282 | return ProfileNetwork(network) | |
283 | ||
372efc19 MT |
284 | |
285 | class Stasy(object): | |
286 | __metaclass__ = Singleton | |
287 | ||
288 | def __init__(self): | |
289 | # Initialize database connection | |
290 | self._conn = pymongo.Connection(DATABASE_HOST) | |
291 | self._db = self._conn[DATABASE_NAME] | |
292 | ||
293 | def get_profile_count(self): | |
294 | # XXX need to implement something to get profiles updated since | |
295 | # a given date | |
296 | ||
297 | # All distinct profiles (based on public_id) | |
298 | return self._db.profiles.distinct("public_id").count() | |
299 | ||
300 | #def _get_profile_cursor(self, public_id): | |
301 | # c = self._db.profiles.find({ "public_id" : public_id }) | |
302 | # c.sort("updated", pymongo.ASCENDING) | |
303 | # | |
304 | # return c | |
305 | ||
91a446f0 | 306 | def profile_exists(self, public_id): |
910be037 | 307 | return self.query({ "public_id" : public_id }).count() >= 1 |
91a446f0 | 308 | |
372efc19 | 309 | def get_profile(self, public_id): |
54af860e | 310 | p = None |
372efc19 | 311 | # XXX should only find one object in the end |
910be037 | 312 | for p in self.query({ "public_id" : public_id }): |
91a446f0 | 313 | p = Profile(p) |
372efc19 MT |
314 | |
315 | return p | |
316 | ||
317 | def get_profiles(self): | |
318 | # XXX needs nicer database query | |
319 | profiles = [] | |
320 | for p in self._db.profiles.find(): | |
321 | if not p.get("public_id") in profiles: | |
322 | profiles.append(p.get("public_id")) | |
323 | ||
324 | return profiles | |
325 | ||
910be037 MT |
326 | def query(self, query, archives=False, no_virt=False, all=False): |
327 | db = self._db.profiles | |
328 | ||
329 | if archives: | |
330 | db = self.db.archives | |
331 | ||
332 | if not all: | |
333 | # XXX cannot use the index? | |
334 | query.update({ "profile" : { "$exists" : True }}) | |
335 | ||
336 | if no_virt: | |
337 | query.update({ "profile.system.virtual" : False }) | |
338 | ||
339 | logging.debug("Executing query: %s" % query) | |
340 | ||
341 | return db.find(query) | |
342 | ||
372efc19 MT |
343 | @property |
344 | def secret_ids(self): | |
345 | return self._db.profiles.distinct("secret_id") | |
346 | ||
347 | @property | |
348 | def cpus(self): | |
910be037 | 349 | return self.query({}, no_virt=True).distinct("profile.cpu") |
372efc19 MT |
350 | |
351 | @property | |
352 | def cpu_vendors(self): | |
910be037 | 353 | return self.query({}, no_virt=True).distinct("profile.cpu.vendor") |
372efc19 MT |
354 | |
355 | @property | |
91a446f0 | 356 | def cpu_vendors_map(self): |
372efc19 MT |
357 | cpus = {} |
358 | ||
359 | for vendor in self.cpu_vendors: | |
360 | cpus[vendor] = \ | |
910be037 | 361 | self.query({ |
372efc19 | 362 | "profile.cpu.vendor" : vendor |
910be037 | 363 | }, no_virt=True).count() |
372efc19 MT |
364 | |
365 | return cpus | |
366 | ||
367 | @property | |
91a446f0 MT |
368 | def cpu_speed_average(self): |
369 | speed = 0 | |
370 | ||
910be037 | 371 | all = self.query({}, no_virt=True) |
91a446f0 MT |
372 | |
373 | # XXX ugly. needs to be done by group() | |
374 | for m in all: | |
375 | if not m.has_key("profile"): | |
376 | continue | |
377 | speed += m.get("profile").get("cpu").get("speed") | |
378 | ||
379 | return (speed / all.count()) | |
380 | ||
381 | @property | |
382 | def cpu_speed_map(self): | |
383 | cpu_speeds = {} | |
384 | ||
385 | for i in range(len(CPU_SPEED_CONSTRAINTS) - 1): | |
386 | min, max = CPU_SPEED_CONSTRAINTS[i:i+2] | |
387 | ||
388 | cpu_speeds[min, max] = \ | |
910be037 MT |
389 | self.query({ |
390 | "profile.cpu.speed" : { | |
91a446f0 MT |
391 | "$gte" : min, "$lt" : max |
392 | } | |
910be037 | 393 | }, no_virt=True).count() |
91a446f0 MT |
394 | |
395 | return cpu_speeds | |
396 | ||
397 | def get_memory_map(self): | |
372efc19 MT |
398 | memory = {} |
399 | ||
400 | for i in range(len(MEMORY_CONSTRAINTS) - 1): | |
401 | min, max = MEMORY_CONSTRAINTS[i:i+2] | |
402 | ||
403 | memory[min, max] = \ | |
910be037 | 404 | self.query( |
372efc19 MT |
405 | { "profile.system.memory" : { |
406 | "$gte" : min * 1024, "$lt" : max * 1024 | |
407 | } | |
910be037 | 408 | }, no_virt=True).count() |
372efc19 MT |
409 | |
410 | return memory | |
411 | ||
412 | @property | |
413 | def memory_average(self): | |
414 | memory = 0 | |
415 | ||
910be037 | 416 | all = self.query({}, no_virt=True) |
372efc19 MT |
417 | |
418 | # XXX ugly. needs to be done by group() | |
419 | for m in all: | |
420 | if not m.has_key("profile"): | |
421 | continue | |
422 | memory += int(m.get("profile").get("system").get("memory")) | |
423 | ||
424 | return (memory / all.count()) / 1024 | |
425 | ||
426 | @property | |
427 | def hypervisor_vendors(self): | |
910be037 | 428 | return self.query({}).distinct("profile.hypervisor.vendor") |
372efc19 MT |
429 | |
430 | @property | |
431 | def hypervisor_map(self): | |
432 | hypervisors = {} | |
433 | ||
434 | for hypervisor in self.hypervisor_vendors: | |
435 | hypervisors[hypervisor] = \ | |
910be037 | 436 | self.query({ |
372efc19 MT |
437 | "profile.hypervisor.vendor" : hypervisor |
438 | }).count() | |
439 | ||
440 | return hypervisors | |
441 | ||
442 | @property | |
443 | def hypervisor_models(self): | |
910be037 | 444 | return self.query({}).distinct("profile.hypervisor.model") |
372efc19 MT |
445 | |
446 | @property | |
447 | def virtual_map(self): | |
448 | virtual = { | |
449 | True: None, | |
450 | False: None, | |
451 | } | |
452 | ||
453 | for k in virtual.keys(): | |
454 | virtual[k] = \ | |
910be037 | 455 | self.query({ "profile.system.virtual": k }).count() |
372efc19 MT |
456 | |
457 | return virtual | |
458 | ||
459 | @property | |
460 | def languages(self): | |
910be037 | 461 | return self.query({}).distinct("profile.system.language") |
372efc19 | 462 | |
91a446f0 MT |
463 | def get_language_map(self): |
464 | languages = {} | |
465 | ||
466 | for language in self.languages: | |
467 | languages[language] = \ | |
910be037 | 468 | self.query({ |
91a446f0 MT |
469 | "profile.system.language" : language |
470 | }).count() | |
471 | ||
472 | return languages | |
473 | ||
372efc19 MT |
474 | @property |
475 | def vendors(self): | |
910be037 | 476 | return self.query({}).distinct("profile.system.vendor") |
372efc19 MT |
477 | |
478 | @property | |
479 | def vendor_map(self): | |
480 | vendors = {} | |
481 | ||
482 | for vendor in self.vendors: | |
483 | vendors[vendor] = \ | |
910be037 | 484 | self.query({ |
372efc19 MT |
485 | "profile.system.vendor" : vendor |
486 | }).count() | |
487 | ||
488 | return vendors | |
489 | ||
490 | @property | |
491 | def models(self): | |
910be037 | 492 | return self.query({}).distinct("profile.system.model") |
372efc19 | 493 | |
91a446f0 MT |
494 | @property |
495 | def model_map(self): | |
496 | models = {} | |
497 | ||
498 | for model in self.models: | |
499 | models[model] = \ | |
910be037 | 500 | self.query({ |
91a446f0 MT |
501 | "profile.system.model" : model |
502 | }).count() | |
503 | ||
504 | return models | |
505 | ||
506 | @property | |
507 | def arches(self): | |
910be037 | 508 | return self.query({}).distinct("profile.cpu.arch") |
91a446f0 MT |
509 | |
510 | @property | |
511 | def arch_map(self): | |
512 | arches = {} | |
513 | ||
514 | for arch in self.arches: | |
515 | arches[arch] = \ | |
910be037 | 516 | self.query({ |
91a446f0 MT |
517 | "profile.cpu.arch" : arch |
518 | }).count() | |
519 | ||
520 | return arches | |
521 | ||
522 | @property | |
523 | def kernels(self): | |
910be037 | 524 | return self.query({}).distinct("profile.system.kernel_release") |
91a446f0 MT |
525 | |
526 | @property | |
527 | def kernel_map(self): | |
528 | kernels = {} | |
529 | ||
530 | for kernel in self.kernels: | |
531 | kernels[kernel] = \ | |
910be037 | 532 | self.query({ |
91a446f0 MT |
533 | "profile.system.kernel_release" : kernel |
534 | }).count() | |
535 | ||
536 | return kernels | |
537 | ||
538 | @property | |
539 | def releases(self): | |
910be037 | 540 | return self.query({}).distinct("profile.system.release") |
91a446f0 MT |
541 | |
542 | @property | |
543 | def release_map(self): | |
544 | releases = {} | |
545 | ||
546 | for release in self.releases: | |
547 | releases[release] = \ | |
910be037 | 548 | self.query({ |
91a446f0 MT |
549 | "profile.system.release" : release |
550 | }).count() | |
551 | ||
552 | return releases | |
553 | ||
554 | def get_device_percentage(self, bus, vendor_id, model_id): | |
910be037 | 555 | profiles_with_device = self.query({ |
91a446f0 MT |
556 | "profile.devices.subsystem" : bus, |
557 | "profile.devices.vendor" : vendor_id, | |
558 | "profile.devices.model" : model_id, | |
559 | }) | |
560 | ||
910be037 | 561 | profiles_all = self.query({}) |
91a446f0 MT |
562 | |
563 | if not profiles_all.count(): | |
564 | return 0 | |
565 | ||
566 | return profiles_with_device.count() / profiles_all.count() | |
567 | ||
910be037 MT |
568 | def get_cpu_flag_map(self, flags): |
569 | # XXX needs a cleanup | |
91a446f0 | 570 | |
910be037 | 571 | _flags = { True : 0 } |
91a446f0 | 572 | |
910be037 MT |
573 | if type(flags) == type("a"): |
574 | flags = [flags] | |
575 | ||
576 | for flag in flags: | |
577 | _flags[True] += \ | |
578 | self.query({ | |
579 | "profile.cpu.flags" : flag, | |
580 | }, no_virt=True).count() | |
91a446f0 | 581 | |
910be037 | 582 | _flags[False] = self.query({}, no_virt=True).count() - _flags[True] |
91a446f0 | 583 | |
910be037 | 584 | return _flags |
91a446f0 MT |
585 | |
586 | @property | |
587 | def geo_locations(self): | |
910be037 | 588 | return [code.lower() for code in self.query({}).distinct("geoip.country_code")] |
91a446f0 MT |
589 | |
590 | def get_geo_location_map(self): | |
591 | geo_locations = {} | |
592 | ||
593 | count = 0 | |
594 | for geo_location in self.geo_locations: | |
595 | geo_locations[geo_location] = \ | |
910be037 | 596 | self.query({ |
91a446f0 MT |
597 | "geoip.country_code" : geo_location |
598 | }).count() | |
599 | ||
600 | count += geo_locations[geo_location] | |
601 | ||
602 | # XXX must only be systems that have profile data | |
910be037 | 603 | profiles_all = self.query({}) |
91a446f0 MT |
604 | |
605 | unknown_count = profiles_all.count() - count | |
606 | if unknown_count: | |
607 | geo_locations["unknown"] = unknown_count | |
608 | ||
609 | return geo_locations | |
610 | ||
611 | def get_models_by_vendor(self, subsystem, vendor_id): | |
612 | devices = [] | |
613 | ||
614 | # XXX must only be systems that have profile data | |
910be037 | 615 | profiles_all = self.query({}) |
91a446f0 MT |
616 | |
617 | for profile in profiles_all: | |
618 | if not profile.has_key("profile"): | |
619 | continue | |
620 | ||
621 | profile = Profile(profile) | |
622 | ||
623 | for device in profile.devices: | |
624 | if not device.vendor == vendor_id: | |
625 | continue | |
626 | ||
627 | if not device in devices: | |
628 | devices.append(device) | |
629 | ||
630 | return devices | |
372efc19 | 631 | |
07785a9c MT |
632 | def get_network_zones_map(self): |
633 | zones = { "green" : 0, "blue" : 0, "orange" : 0, "red" : 0 } | |
634 | ||
635 | for zone in zones.keys(): | |
636 | zones[zone] = self.query({ | |
637 | "profile.network.%s" % zone : True, | |
638 | }).count() | |
639 | ||
640 | return zones | |
641 | ||
372efc19 MT |
642 | |
643 | if __name__ == "__main__": | |
644 | s = Stasy() | |
645 | ||
646 | print s.get_profile("0" * 40) | |
647 | print s.cpu_vendors | |
648 | for id in s.secret_ids: | |
649 | print "\t", id | |
650 | ||
651 | #for p in s._db.profiles.find(): | |
652 | # print p | |
653 | ||
654 | print s.cpu_map | |
655 | print s.memory_map | |
656 | print s.memory_average | |
657 | print s.hypervisor_vendors | |
658 | print s.hypervisor_models | |
659 | print s.languages | |
91a446f0 | 660 | print s.language_maps |
372efc19 MT |
661 | print s.vendors |
662 | print s.vendor_map | |
663 | print s.models | |
664 | print s.cpus | |
91a446f0 MT |
665 | print s.cpu_map |
666 | print s.arches | |
667 | print s.arch_map | |
668 | print s.kernels | |
669 | print s.kernel_map | |
670 | print s.releases | |
671 | print s.release_map | |
372efc19 MT |
672 | |
673 | p = s.get_profile("0b5f4fe2162fdfbfa29b632610e317078fa70d34") | |
91a446f0 MT |
674 | print p._data |
675 | # print p.hypervisor | |
676 |