test: extend analyze_outcomes.py in order to skip only some test in a suite
Signed-off-by: Valerio Setti <valerio.setti@nordicsemi.no>
diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py
index 7465217..6101a1c 100755
--- a/tests/scripts/analyze_outcomes.py
+++ b/tests/scripts/analyze_outcomes.py
@@ -61,24 +61,32 @@
# fixed this branch to have full coverage of test cases.
results.warning('Test case not executed: {}', key)
-def analyze_driver_vs_reference(outcomes, component_ref, component_driver, ignored_tests):
+def analyze_driver_vs_reference(outcomes, component_ref, component_driver,
+ ignored_suites, ignored_test=None):
"""Check that all tests executed in the reference component are also
executed in the corresponding driver component.
- Skip test suites provided in ignored_tests list.
+ Skip:
+ - full test suites provided in ignored_suites list
+ - only some specific test inside a test suite, for which the corresponding
+ output string is provided
"""
available = check_test_cases.collect_available_test_cases()
result = True
for key in available:
# Skip ignored test suites
- test_suite = key.split(';')[0] # retrieve test suit name
- test_suite = test_suite.split('.')[0] # retrieve main part of test suit name
- if test_suite in ignored_tests:
+ full_test_suite = key.split(';')[0] # retrieve full test suit name
+ test_string = key.split(';')[1] # retrieve the text string of this test
+ test_suite = full_test_suite.split('.')[0] # retrieve main part of test suit name
+ if test_suite in ignored_suites:
continue
# Continue if test was not executed by any component
hits = outcomes[key].hits() if key in outcomes else 0
if hits == 0:
continue
+ if ((full_test_suite in ignored_test) and
+ (test_string in ignored_test[full_test_suite])):
+ continue
# Search for tests that run in reference component and not in driver component
driver_test_passed = False
reference_test_passed = False
@@ -129,13 +137,14 @@
def do_analyze_driver_vs_reference(outcome_file, args):
"""Perform driver vs reference analyze."""
- ignored_tests = ['test_suite_' + x for x in args['ignored_suites']]
+ ignored_suites = ['test_suite_' + x for x in args['ignored_suites']]
outcomes = read_outcome_file(outcome_file)
print("\n*** Analyze driver {} vs reference {} ***\n".format(
args['component_driver'], args['component_ref']))
return analyze_driver_vs_reference(outcomes, args['component_ref'],
- args['component_driver'], ignored_tests)
+ args['component_driver'], ignored_suites,
+ args['ignored_tests'])
# List of tasks with a function that can handle this task and additional arguments if required
TASKS = {
@@ -154,7 +163,11 @@
'ignored_suites': [
'shax', 'mdx', # the software implementations that are being excluded
'md', # the legacy abstraction layer that's being excluded
- ]}},
+ ],
+ 'ignored_tests': {
+ }
+ }
+ },
'analyze_driver_vs_reference_ecdsa': {
'test_function': do_analyze_driver_vs_reference,
'args': {
@@ -170,7 +183,11 @@
'x509write', # #6858
'debug', # #6860
'ssl', # #6860
- ]}},
+ ],
+ 'ignored_tests': {
+ }
+ }
+ },
}
def main():