test(fuzzing): adding variable coverage
Adding the capability to produce coverage of the arguments of the
SMC calls as generated by the fuzzer. The output from the FVP will
be routed to UART3 where a python flow will read the data to create
tables of each SMC call with its fields shown and values given. The
option is enabled by adding SMC_FUZZ_VARIABLE_COVERAGE=1 to the
corresponding TFTF config.
Change-Id: I2d4d310976aa2c0447efbd8ec0676bb9f8699828
Signed-off-by: Mark Dykes <mark.dykes@arm.com>
diff --git a/smc_fuzz/script/generate_var_coverage.py b/smc_fuzz/script/generate_var_coverage.py
new file mode 100755
index 0000000..2227f7a
--- /dev/null
+++ b/smc_fuzz/script/generate_var_coverage.py
@@ -0,0 +1,88 @@
+# !/usr/bin/env python
+#
+# Copyright (c) 2025 Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+
+#python3 generate_var_coverage.py -sd <smc definition file> -df <SMC data file>
+#This script generates variable coverage tables from SMC definition file and output of FVP SMC calls
+import re
+import copy
+import sys
+import argparse
+import readsmclist
+from tabulate import tabulate
+
+parser = argparse.ArgumentParser(
+ prog='generate_var_coverage.py',
+ description='Creates coverage data from SMC calls in FVP model',
+ epilog='two arguments input')
+
+parser.add_argument('-df', '--datafile',help="Data from UART output of the model .")
+parser.add_argument('-sd', '--smcdefinition',help="SMC definition file .")
+
+args = parser.parse_args()
+
+print("starting variable coverage")
+
+seq = 0
+
+readsmclist.readsmclist(args.smcdefinition,seq)
+
+arglst = readsmclist.arglst
+argnumfield = readsmclist.argnumfield
+argfieldname = readsmclist.argfieldname
+argstartbit = readsmclist.argstartbit
+argendbit = readsmclist.argendbit
+argdefval = readsmclist.argdefval
+smcname = readsmclist.smcname
+argnum = readsmclist.argnum
+argname = readsmclist.argname
+smcid = readsmclist.smcid
+
+varcovvalues = {}
+
+for sn in argfieldname:
+ varcovvalues[sn] = {}
+ for an in argfieldname[sn]:
+ for fn in argfieldname[sn][an]:
+ varcovvalues[sn][fn] = set()
+
+datafile = open(args.datafile, "r")
+data_lines = datafile.readlines()
+datafile.close()
+for dline in data_lines:
+ dl = dline.strip()
+ dinstr = re.search(r'^SMC FUZZER CALL fid:([a-fA-F0-9]+)\s+arg1:([a-fA-F0-9]+)\s+arg2:([a-fA-F0-9]+)\s+arg3:([a-fA-F0-9]+)\s+arg4:([a-fA-F0-9]+)\s+arg5:([a-fA-F0-9]+)\s+arg6:([a-fA-F0-9]+)\s+arg7:([a-fA-F0-9]+)$',dl)
+ if dinstr:
+ if dinstr.group(1) in smcid:
+ scall = smcid[dinstr.group(1)]
+ for an in argfieldname[scall]:
+ for fn in argfieldname[scall][an]:
+ if int(argnumfield[scall][an][fn]) < 8:
+ argval = dinstr.group(int(argnumfield[scall][an][fn])+1)
+ astbit = int(argstartbit[scall][an][fn])
+ aenbit = int(argendbit[scall][an][fn])
+ vval = hex((int(argval,16) >> astbit) & ((2 ** (aenbit - astbit + 1)) - 1))
+ varcovvalues[scall][fn].add(vval)
+
+for sn in varcovvalues:
+ print(sn)
+ largest = 0
+ hdrs = []
+ for fn in varcovvalues[sn]:
+ if len(varcovvalues[sn][fn]) > largest:
+ largest = len(varcovvalues[sn][fn])
+ te = [["-" for x in range(len(varcovvalues[sn]))] for x in range(largest)]
+ k = 0
+ for fn in varcovvalues[sn]:
+ i = 0
+ hdrs.append(fn)
+ for val in varcovvalues[sn][fn]:
+ te[i][k] = val
+ i = i + 1
+ k = k + 1
+ print(tabulate(te, headers=hdrs, tablefmt="grid"))
+ print()
diff --git a/smc_fuzz/script/readsmclist.py b/smc_fuzz/script/readsmclist.py
index 3bfcb6a..d351552 100755
--- a/smc_fuzz/script/readsmclist.py
+++ b/smc_fuzz/script/readsmclist.py
@@ -14,6 +14,9 @@
argstartbit = {}
argendbit = {}
argdefval = {}
+smcid = {}
+defval = {}
+sdef = {}
smcname = ""
argnum = ""
argname = ""
@@ -23,9 +26,54 @@
smclist_lines = smclistfile.readlines()
smclistfile.close()
for sline in smclist_lines:
+ svar = 0
lcon = 0
sl = sline.strip()
- sinstr = re.search(r'^smc:\s*([a-zA-Z0-9_]+)$',sl)
+ sinstr = re.search(r'^smc:\s*([a-zA-Z0-9_]+)\s*0x([a-fA-F0-9]+)\s*$',sl)
+ if sinstr:
+ smcname = sinstr.group(1)
+ arglst[sinstr.group(1)] = []
+ argnumfield[sinstr.group(1)] = {}
+ argfieldname[sinstr.group(1)] = {}
+ argstartbit[sinstr.group(1)] = {}
+ argendbit[sinstr.group(1)] = {}
+ argdefval[sinstr.group(1)] = {}
+ smcid[sinstr.group(2)] = sinstr.group(1)
+ svar = 1
+ lcon = 1
+ argoccupy = {}
+ if not seq:
+ seq = seq + 1
+ else:
+ if seq != 2:
+ print("Error: out of sequence for smc call",end=" ")
+ print(smcname)
+ sys.exit()
+ else:
+ seq = 1
+ sinstr = re.search(r'^smc:\s*([a-zA-Z0-9_]+)\s*([a-zA-Z0-9_]+)\s*$',sl)
+ if sinstr and (svar == 0):
+ smcname = sinstr.group(1)
+ arglst[sinstr.group(1)] = []
+ argnumfield[sinstr.group(1)] = {}
+ argfieldname[sinstr.group(1)] = {}
+ argstartbit[sinstr.group(1)] = {}
+ argendbit[sinstr.group(1)] = {}
+ argdefval[sinstr.group(1)] = {}
+ sdef[sinstr.group(1)] = sinstr.group(2)
+ smcid[sinstr.group(2)] = sinstr.group(1)
+ lcon = 1
+ argoccupy = {}
+ if not seq:
+ seq = seq + 1
+ else:
+ if seq != 2:
+ print("Error: out of sequence for smc call",end=" ")
+ print(smcname)
+ sys.exit()
+ else:
+ seq = 1
+ sinstr = re.search(r'^smc:\s*([a-zA-Z0-9_]+)\s*$',sl)
if sinstr:
smcname = sinstr.group(1)
arglst[sinstr.group(1)] = []
@@ -181,6 +229,10 @@
if seq != 2:
print("Error: out of sequence for field")
sys.exit()
+ sinstr = re.search(r'^define ([a-zA-Z0-9_]+)\s*=\s*0x([a-fA-F0-9]+|\d+)$',sl)
+ if sinstr:
+ defval[sinstr.group(1)] = sinstr.group(2)
+ lcon = 1
if not lcon:
cline = re.search(r'^#',sl)
if not cline:
@@ -191,3 +243,11 @@
if(seq != 2):
print("incorrect ending for smc specification")
sys.exit()
+
+ for smccal,dval in sdef.items():
+ if dval in defval:
+ smcid[defval[dval]] = smccal
+ else:
+ print("Error: cannot find define value",end=" ")
+ print(dval)
+ sys.exit()