From: Naoki Kambe Date: Sat, 4 Aug 2012 22:53:29 +0000 (+0900) Subject: [2179] Supported differential statistics updates of Stats X-Git-Tag: trac2351_base~112^2~7 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=3e377dea92c7c685d7e9136e926bb7df4d7cb180;p=thirdparty%2Fkea.git [2179] Supported differential statistics updates of Stats Stats updates corresponding statistics data which each module returns. For that, the method "update_statistics_data()" was updated to merge recursively old value and new value each other. As for this merging, a new method "merge_oldnew" was introduced into stats.py. This method is intended to be internally used so far. --- diff --git a/src/bin/stats/stats.py.in b/src/bin/stats/stats.py.in index 66436e14c9..2dfbcb59c8 100755 --- a/src/bin/stats/stats.py.in +++ b/src/bin/stats/stats.py.in @@ -137,6 +137,33 @@ def _accum(a, b): # Nothing matches above, the first arg is returned return a +def merge_oldnew(old, new): + """ + Merges two arguments. If old data contains the corresponding name + against new data, the value of the name is replaced with the + corresponding value in new data. Otherwise, the new date are added + at same name or same id. Both old data and new data should be same + data type. This method returns the merged result. + """ + # If the first arg is dict or list type, two values + # would be merged + if type(old) is dict and type(new) is dict: + return dict([ (k, merge_oldnew(old[k], v)) \ + if k in old else (k, v) \ + for (k, v) in new.items() ] \ + + [ (k, v) \ + for (k, v) in old.items() \ + if k not in new ]) + elif type(old) is list and type(new) is list: + return [ merge_oldnew(old[i], new[i]) \ + if len(old) > i else new[i] \ + for i in range(len(new)) ] \ + + [ old[i] \ + for i in range(len(old)) \ + if len(new) <= i ] + else: + return new + class Callback(): """ A Callback handler class @@ -490,7 +517,11 @@ class Stats: if self.modules[owner].validate_statistics(False, data, errors): if owner in self.statistics_data_bymid: if mid in self.statistics_data_bymid[owner]: - self.statistics_data_bymid[owner][mid].update(data) + # merge recursively old value and new + # value each other + self.statistics_data_bymid[owner][mid] = \ + merge_oldnew(self.statistics_data_bymid[owner][mid], + data) else: self.statistics_data_bymid[owner][mid] = data else: diff --git a/src/bin/stats/tests/b10-stats_test.py b/src/bin/stats/tests/b10-stats_test.py index 5a7b6b1d0d..a115a93f28 100644 --- a/src/bin/stats/tests/b10-stats_test.py +++ b/src/bin/stats/tests/b10-stats_test.py @@ -138,6 +138,39 @@ class TestUtilties(unittest.TestCase): [ {}, {'four': 1, 'five': 2, 'six': 3} ]), [ {'one': 1, 'two': 2, 'three': 3}, {'four': 5, 'five': 7, 'six': 9} ]) + def test_merge_oldnre(self): + self.assertEqual(stats.merge_oldnew(1, 2), 2) + self.assertEqual(stats.merge_oldnew(0.5, 0.3), 0.3) + self.assertEqual(stats.merge_oldnew('aa','bb'), 'bb') + self.assertEqual(stats.merge_oldnew( + [1, 2, 3], [4, 5]), [4, 5, 3]) + self.assertEqual(stats.merge_oldnew( + [4, 5], [1, 2, 3]), [1, 2, 3]) + self.assertEqual(stats.merge_oldnew( + [1, 2, 3], [None, 5, 6]), [None, 5, 6]) + self.assertEqual(stats.merge_oldnew( + [None, 5, 6], [1, 2, 3]), [1, 2, 3]) + self.assertEqual(stats.merge_oldnew( + [1, 2, 3], [None, None, None, None]), [None, None, None, None]) + self.assertEqual(stats.merge_oldnew( + [[1,2],3],[[],5,6]), [[1,2],5,6]) + self.assertEqual(stats.merge_oldnew( + {'one': 1, 'two': 2, 'three': 3}, + {'one': 4, 'two': 5}), + {'one': 4, 'two': 5, 'three': 3}) + self.assertEqual(stats.merge_oldnew( + {'one': 1, 'two': 2, 'three': 3}, + {'four': 4, 'five': 5}), + {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5}) + self.assertEqual(stats.merge_oldnew( + {'one': [1, 2], 'two': [3, None, 5], 'three': [None, 3, None]}, + {'one': [2], 'two': [4, 5], 'three': [None, None, None], 'four': 'FOUR'}), + {'one':[2,2], 'two':[4,5,5], 'three':[None,None,None], 'four': 'FOUR'}) + self.assertEqual(stats.merge_oldnew( + [ {'one': 1, 'two': 2, 'three': 3}, {'four': 4, 'five': 5, 'six': 6} ], + [ {}, {'four': 1, 'five': 2, 'six': 3} ]), + [ {'one': 1, 'two': 2, 'three': 3}, {'four': 1, 'five': 2, 'six': 3} ]) + class TestCallback(unittest.TestCase): def setUp(self): self.dummy_func = lambda *x, **y : (x, y) @@ -385,8 +418,21 @@ class TestStats(unittest.TestCase): def test_update_statistics_data(self): self.stats = stats.Stats() - - # success + _test_exp1 = { + 'zonename': 'test1.example', + 'queries.tcp': 5, + 'queries.udp': 4 + } + _test_exp2 = { + 'zonename': 'test2.example', + 'queries.tcp': 3, + 'queries.udp': 2 + } + _test_exp3 ={} + _test_exp4 ={ + 'queries.udp': 4 + } + # Success cases self.assertEqual(self.stats.statistics_data['Stats']['lname'], self.stats.cc_session.lname) self.stats.update_statistics_data( @@ -394,13 +440,36 @@ class TestStats(unittest.TestCase): {'lname': 'foo@bar'}) self.assertEqual(self.stats.statistics_data['Stats']['lname'], 'foo@bar') - # error case + self.assertIsNone(self.stats.update_statistics_data( + 'Auth', 'foo1', {'queries.perzone': [_test_exp1]})) + self.assertEqual(self.stats.statistics_data_bymid['Auth']\ + ['foo1']['queries.perzone'],\ + [_test_exp1]) + self.assertIsNone(self.stats.update_statistics_data( + 'Auth', 'foo1', {'queries.perzone': [_test_exp2]})) + self.assertEqual(self.stats.statistics_data_bymid['Auth']\ + ['foo1']['queries.perzone'],\ + [_test_exp2]) + self.assertIsNone(self.stats.update_statistics_data( + 'Auth', 'foo1', {'queries.perzone': [_test_exp1,_test_exp2]})) + self.assertEqual(self.stats.statistics_data_bymid['Auth']\ + ['foo1']['queries.perzone'], + [_test_exp1,_test_exp2]) + # differential update + self.assertIsNone(self.stats.update_statistics_data( + 'Auth', 'foo1', {'queries.perzone': [_test_exp3,_test_exp4]})) + self.assertEqual(self.stats.statistics_data_bymid['Auth']\ + ['foo1']['queries.perzone'], \ + [_test_exp1,stats.merge_oldnew(_test_exp2,_test_exp4)]) + # Error cases self.assertEqual(self.stats.update_statistics_data('Stats', None, {'lname': 0.0}), ['0.0 should be a string']) self.assertEqual(self.stats.update_statistics_data('Dummy', None, {'foo': 'bar'}), ['unknown module name: Dummy']) + self.assertEqual(self.stats.update_statistics_data( + 'Auth', 'foo1', {'queries.perzone': [None]}), ['None should be a map']) def test_update_statistics_data_withmid(self): self.stats = stats.Stats()