Fix a statschannel system test zone loadtime issue

The check_loaded() function compares the zone's loadtime value and
an expected loadtime value, which is based on the zone file's mtime
extracted from the filesystem.

For the secondary zones there may be cases, when the zone file isn't
ready yet before the zone transfer is complete and the zone file is
dumped to the disk, so a so zero value mtime is retrieved.

In such cases wait one second and retry until timeout. Also modify
the affected check to allow a possible difference of the same amount
of seconds as the chosen timeout value.

(cherry picked from commit 4e94ff2541bc31dcb03c8b58a67455aa2b00fa4c)

diff --git a/bin/tests/system/statschannel/generic.py b/bin/tests/system/statschannel/generic.py
index 9e926537cc10ce932f0332cd1faa5a238c2d067f..1f2f7cd6e39522cc4ca658259b00ad5e12bb46d7 100644 (file)
--- a/bin/tests/system/statschannel/generic.py
+++ b/bin/tests/system/statschannel/generic.py
@@ -10,6 +10,7 @@
 # information regarding copyright ownership.
 
 from datetime import datetime, timedelta
+from time import sleep
 import os
 
 
@@ -21,6 +22,9 @@ max_refresh = timedelta(seconds=2419200)  # 4 weeks
 max_expires = timedelta(seconds=14515200)  # 24 weeks
 dayzero = datetime.utcfromtimestamp(0).replace(microsecond=0)
 
+# Wait for the secondary zone files to appear to extract their mtime
+max_secondary_zone_waittime_sec = 5
+
 
 # Generic helper functions
 def check_expires(expires, min_time, max_time):
@@ -35,7 +39,7 @@ def check_refresh(refresh, min_time, max_time):
 
 def check_loaded(loaded, expected, now):
     # Sanity check the zone timers values
-    assert loaded == expected
+    assert (loaded - expected).total_seconds() < max_secondary_zone_waittime_sec
     assert loaded <= now
 
 
@@ -86,12 +90,26 @@ def test_zone_timers_secondary(fetch_zones, load_timers, **kwargs):
     statsport = kwargs["statsport"]
     zonedir = kwargs["zonedir"]
 
-    zones = fetch_zones(statsip, statsport)
-
-    for zone in zones:
-        (name, loaded, expires, refresh) = load_timers(zone, False)
-        mtime = zone_mtime(zonedir, name)
-        check_zone_timers(loaded, expires, refresh, mtime)
+    # If any one of the zone files isn't ready, then retry until timeout.
+    tries = max_secondary_zone_waittime_sec
+    while tries >= 0:
+        zones = fetch_zones(statsip, statsport)
+        again = False
+        for zone in zones:
+            (name, loaded, expires, refresh) = load_timers(zone, False)
+            mtime = zone_mtime(zonedir, name)
+            if (mtime != dayzero) or (tries == 0):
+                # mtime was either retrieved successfully or no tries were
+                # left, run the check anyway.
+                check_zone_timers(loaded, expires, refresh, mtime)
+            else:
+                tries = tries - 1
+                again = True
+                break
+        if again:
+            sleep(1)
+        else:
+            break
 
 
 def test_zone_with_many_keys(fetch_zones, load_zone, **kwargs):