oeqa/ltp: rewrote LTP testcase and parser

The LTP test reporting appears to be a little fragile so I tried to make
it more reliable.

Primarily this is done by not passing -p to runltp, which results in
machine-readable logfiles instead of human-readable.  These are easier
to parse and have more context in, so we can also report correctly
skipped tests.

Signed-off-by: Ross Burton <ross.burton@arm.com>
Signed-off-by: Alexandre Belloni <alexandre.belloni@bootlin.com>

diff --git a/meta/lib/oeqa/runtime/cases/ltp.py b/meta/lib/oeqa/runtime/cases/ltp.py
index a66d5d13d75f91da3f5847bfa3f04cb7968774fe..29c26d7d324f860567b741cd3f406f4bd5adf392 100644 (file)
--- a/meta/lib/oeqa/runtime/cases/ltp.py
+++ b/meta/lib/oeqa/runtime/cases/ltp.py
@@ -65,29 +65,34 @@ class LtpTest(LtpTestBase):
     ltp_groups += ltp_fs
 
     def runltp(self, ltp_group):
-            cmd = '/opt/ltp/runltp -f %s -p -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group)
+            # LTP appends to log files, so ensure we start with a clean log
+            self.target.deleteFiles("/opt/ltp/results/", ltp_group)
+
+            cmd = '/opt/ltp/runltp -f %s -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group)
+
             starttime = time.time()
             (status, output) = self.target.run(cmd)
             endtime = time.time()
 
+            # Write the console log to disk for convenience
             with open(os.path.join(self.ltptest_log_dir, "%s-raw.log" % ltp_group), 'w') as f:
                 f.write(output)
 
+            # Also put the console log into the test result JSON
             self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output
 
-            # copy nice log from DUT
-            dst = os.path.join(self.ltptest_log_dir, "%s" %  ltp_group )
+            # Copy the machine-readable test results locally so we can parse it
+            dst = os.path.join(self.ltptest_log_dir, ltp_group)
             remote_src = "/opt/ltp/results/%s" % ltp_group 
             (status, output) = self.target.copyFrom(remote_src, dst, True)
-            msg = 'File could not be copied. Output: %s' % output
             if status:
+                msg = 'File could not be copied. Output: %s' % output
                 self.target.logger.warning(msg)
 
             parser = LtpParser()
             results, sections  = parser.parse(dst)
 
-            runtime = int(endtime-starttime)
-            sections['duration'] = runtime
+            sections['duration'] = int(endtime-starttime)
             self.sections[ltp_group] =  sections
 
             failed_tests = {}

diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py
index 8054acc853bd694668f35378f4ede5cc9de219e9..496d9e0c9031a6fb5ff58de78a3a80541a2e0e11 100644 (file)
--- a/meta/lib/oeqa/utils/logparser.py
+++ b/meta/lib/oeqa/utils/logparser.py
@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: MIT
 #
 
-import sys
+import enum
 import os
 import re
 
@@ -106,30 +106,48 @@ class PtestParser(object):
                     f.write(status + ": " + test_name + "\n")
 
 
-# ltp log parsing
-class LtpParser(object):
-    def __init__(self):
-        self.results = {}
-        self.section = {'duration': "", 'log': ""}
-
+class LtpParser:
+    """
+    Parse the machine-readable LTP log output into a ptest-friendly data structure.
+    """
     def parse(self, logfile):
-        test_regex = {}
-        test_regex['PASSED'] = re.compile(r"PASS")
-        test_regex['FAILED'] = re.compile(r"FAIL")
-        test_regex['SKIPPED'] = re.compile(r"SKIP")
-
-        with open(logfile, errors='replace') as f:
+        results = {}
+        # Aaccumulate the duration here but as the log rounds quick tests down
+        # to 0 seconds this is very much a lower bound. The caller can replace
+        # the value.
+        section = {"duration": 0, "log": ""}
+
+        class LtpExitCode(enum.IntEnum):
+            # Exit codes as defined in ltp/include/tst_res_flags.h
+            TPASS = 0  # Test passed flag
+            TFAIL = 1  # Test failed flag
+            TBROK = 2  # Test broken flag
+            TWARN = 4  # Test warning flag
+            TINFO = 16 # Test information flag
+            TCONF = 32 # Test not appropriate for configuration flag
+
+        with open(logfile, errors="replace") as f:
+            # Lines look like this:
+            # tag=cfs_bandwidth01 stime=1689762564 dur=0 exit=exited stat=32 core=no cu=0 cs=0
             for line in f:
-                for t in test_regex:
-                    result = test_regex[t].search(line)
-                    if result:
-                        self.results[line.split()[0].strip()] = t
-
-        for test in self.results:
-            result = self.results[test]
-            self.section['log'] = self.section['log'] + ("%s: %s\n" % (result.strip()[:-2], test.strip()))
+                if not line.startswith("tag="):
+                    continue
 
-        return self.results, self.section
+                values = dict(s.split("=") for s in line.strip().split())
+
+                section["duration"] += int(values["dur"])
+                exitcode = int(values["stat"])
+                if values["exit"] == "exited" and exitcode == LtpExitCode.TCONF:
+                    # Exited normally with the "invalid configuration" code
+                    results[values["tag"]] = "SKIPPED"
+                elif exitcode == LtpExitCode.TPASS:
+                    # Successful exit
+                    results[values["tag"]] = "PASSED"
+                else:
+                    # Other exit
+                    results[values["tag"]] = "FAILED"
+
+        return results, section
 
 
 # ltp Compliance log parsing