runltp has been removed from ltp and the official tool to invoke ltp
test is kirk.
Message when running runltp:
"INFO: runltp script is deprecated, try kirk
https://github.com/linux-test-project/kirk"
See also:
https://github.com/linux-test-project/ltp/commit/
6efd3605dc005c3ed135b463f182174e24bdce1b
The test coverage is the same since the logic there is not touched
but the output is different. The return value from oeqa are the same
as before.
Now it is a json file with the results, which is machine readable
without any extra parsing. Two files are created by test suite
<test>.json and <test>-raw.log
Output example of math.json:
{
"results": [
{
"test_fqn": "abs01",
"status": "pass",
"test": {
"command": "abs01",
"arguments": [],
"log": "abs01 1 TPASS : Test passed\nabs01 2 TPASS : Test passed\nabs01 3 TPASS : Test passed\n",
"retval": [
"0"
],
"duration": 0.
002702474594116211,
"failed": 0,
"passed": 3,
"broken": 0,
"skipped": 0,
"warnings": 0,
"result": "pass"
}
}, [...]
],
"stats": {
"runtime": 1.
4888691902160645,
"passed": 22,
"failed": 0,
"broken": 0,
"skipped": 0,
"warnings": 0
},
"environment": {
"distribution": "nodistro",
"distribution_version": "nodistro.0",
"kernel": "Linux 6.18.13-yocto-standard #1 SMP PREEMPT_DYNAMIC Tue Mar 3 16:48:55 UTC 2026",
"cmdline": "root=/dev/vda rw ip=192.168.7.2::192.168.7.1:255.255.255.0::eth0:off:8.8.8.8 net.ifnames=0 console=ttyS0 console=ttyS1 oprofile.timer=1 tsc=reliable no_timer_check rcupdate.rcu_expedited=1 swiotlb=0 printk.time=1",
"arch": "x86_64",
"cpu": "unknown",
"swap": "0 kB",
"RAM": "222368 kB"
}
}
Add in configuration file (local.conf)
IMAGE_CLASSES += "testimage"
CORE_IMAGE_EXTRA_INSTALL += "ltp openssh"
TEST_SUITES = "ping ssh ltp"
QB_MEM = <mem size>
QB_CPU_KVM = "-cpu host -smp <cpus>"
QB_SMP = "-smp <cpus>"
QB_KVM = "1"
IMAGE_ROOTFS_EXTRA_SPACE = "
2097152"
Tested with different VM sizes and architectures:
qemux86-64 MEM: 4096, CPU: 2 (all ok)
qemux86-64 MEM: 1024, CPU: 1 (all ok)
qemux86-64 MEM: 1024, CPU: 2 (all ok)
qemux86-64 MEM: 1024, CPU: 4 (all ok)
qemuarm64: MEM: 4096, CPU: 1 (all ok)
qemuriscv64: MEM 4096, CPU 4 (all ok)
Then:
bitbake core-image-minimal
bitbake core-image-minimal -c testimage
With these configs we didn't see any testimage failures with kirk and
run completes.
If we revert the ltp disable patches below, the testimage does not fail
but it gives warnings, since the ssh connection gets droped from the DUT
because of the OOM killer. Then in continues with the next testsuite.
* ltp: disable cve failing testcases
* ltp: disable pty testcase
* ltp: disable min_free_kbytes
Signed-off-by: Daniel Turull <daniel.turull@ericsson.com>
Signed-off-by: Pratik Farkase <pratik.farkase@est.tech>
Assisted-by: Claude, Anthropic
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
def runltp(self, ltp_group):
# LTP appends to log files, so ensure we start with a clean log
- self.target.deleteFiles("/opt/ltp/results/", ltp_group)
+ self.target.deleteFiles("/opt/ltp/results/", "%s.json" % ltp_group)
- cmd = '/opt/ltp/runltp -f %s -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group)
+ cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n -d /opt/ltp --exec-timeout 20m' % (ltp_group, ltp_group)
starttime = time.time()
(status, output) = self.target.run(cmd, timeout=1200)
self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output
# Copy the machine-readable test results locally so we can parse it
- dst = os.path.join(self.ltptest_log_dir, ltp_group)
- remote_src = "/opt/ltp/results/%s" % ltp_group
+ dst = os.path.join(self.ltptest_log_dir, "%s.json" % ltp_group)
+ remote_src = "/opt/ltp/results/%s.json" % ltp_group
(status, output) = self.target.copyFrom(remote_src, dst, True)
if status:
msg = 'File could not be copied. Output: %s' % output
# LTP runtime tests
@OETestDepends(['ssh.SSHTest.test_ssh'])
- @OEHasPackage(["ltp"])
+ @OEHasPackage(["ltp", "python3-kirk"])
def test_ltp_help(self):
- (status, output) = self.target.run('/opt/ltp/runltp --help')
+ (status, output) = self.target.run('kirk --help')
msg = 'Failed to get ltp help. Output: %s' % output
self.assertEqual(status, 0, msg=msg)
@OETestDepends(['ltp.LtpTest.test_ltp_help'])
def test_ltp_groups(self):
- for ltp_group in self.ltp_groups:
- self.runltp(ltp_group)
+ for ltp_group in self.ltp_groups:
+ try:
+ self.runltp(ltp_group)
+ except Exception as e:
+ self.extras['ltpresult.%s' % ltp_group] = {'status': 'FAILED'}
+ self.failmsg = self.failmsg + "Suite %s crashed: %s\n" % (ltp_group, e)
+ self.target.logger.warning("Suite %s crashed, continuing: %s" % (ltp_group, e))
@OETestDepends(['ltp.LtpTest.test_ltp_groups'])
def test_ltp_runltp_cve(self):
class LtpStressTest(LtpStressBase):
def runltp(self, stress_group):
- cmd = '/opt/ltp/runltp -f %s -p -q 2>@1 | tee /opt/ltp/results/%s' % (stress_group, stress_group)
+ cmd = 'kirk --run-suite %s --json-report /opt/ltp/results/%s.json -n -d /opt/ltp' % (stress_group, stress_group)
starttime = time.time()
(status, output) = self.target.run(cmd)
endtime = time.time()
self.extras['ltpstressresult.rawlogs']['log'] = self.extras['ltpstressresult.rawlogs']['log'] + output
+ # Copy kirk JSON report from target
+ dst = os.path.join(self.ltptest_log_dir, "%s.json" % stress_group)
+ remote_src = "/opt/ltp/results/%s.json" % stress_group
+ (status, output) = self.target.copyFrom(remote_src, dst, True)
+ if status:
+ msg = 'File could not be copied. Output: %s' % output
+ self.target.logger.warning(msg)
+
parser = LtpParser()
- results, sections = parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group))
+ results, sections = parser.parse(dst)
runtime = int(endtime-starttime)
sections['duration'] = runtime
# SPDX-License-Identifier: MIT
#
-import enum
+import json
import os
import re
class LtpParser:
"""
- Parse the machine-readable LTP log output into a ptest-friendly data structure.
+ Parse kirk JSON report into a ptest-friendly data structure.
"""
+
+ STATUS_MAP = {
+ "pass": "PASSED",
+ "fail": "FAILED",
+ "brok": "FAILED",
+ "conf": "SKIPPED",
+ "warn": "PASSED",
+ }
+
def parse(self, logfile):
+ with open(logfile, errors="replace") as f:
+ report = json.load(f)
+
results = {}
- # Aaccumulate the duration here but as the log rounds quick tests down
- # to 0 seconds this is very much a lower bound. The caller can replace
- # the value.
section = {"duration": 0, "log": ""}
- class LtpExitCode(enum.IntEnum):
- # Exit codes as defined in ltp/include/tst_res_flags.h
- TPASS = 0 # Test passed flag
- TFAIL = 1 # Test failed flag
- TBROK = 2 # Test broken flag
- TWARN = 4 # Test warning flag
- TINFO = 16 # Test information flag
- TCONF = 32 # Test not appropriate for configuration flag
-
- with open(logfile, errors="replace") as f:
- # Lines look like this:
- # tag=cfs_bandwidth01 stime=1689762564 dur=0 exit=exited stat=32 core=no cu=0 cs=0
- for line in f:
- if not line.startswith("tag="):
- continue
+ for entry in report.get("results", []):
+ results[entry["test_fqn"]] = self.STATUS_MAP.get(entry.get("status", ""), "FAILED")
+ section["log"] += entry.get("test", {}).get("log", "")
- values = dict(s.split("=") for s in line.strip().split())
-
- section["duration"] += int(values["dur"])
- exitcode = int(values["stat"])
- if values["exit"] == "exited" and exitcode == LtpExitCode.TCONF:
- # Exited normally with the "invalid configuration" code
- results[values["tag"]] = "SKIPPED"
- elif exitcode == LtpExitCode.TPASS:
- # Successful exit
- results[values["tag"]] = "PASSED"
- else:
- # Other exit
- results[values["tag"]] = "FAILED"
+ section["duration"] = int(report.get("stats", {}).get("runtime", 0))
return results, section
net-tools \
perl \
python3-core \
+ python3-kirk \
procps \
quota \
unzip \
projects / thirdparty / openembedded / openembedded-core-contrib.git / commitdiff
