Add a flag for requiring full coverage in coverage tests
Introduce the --require-full-coverage in analyze_outcomes.py so that
when analyze_outcomes.py --require-full-coverage is called, those
tests that are not executed and are not in the allowed list issue an
error instead of a warning.
Note that it is useful to run analyze_outcomes.py on incomplete test
results, so this error mode needs to remain optional in the long
term.
Signed-off-by: Tomás González <tomasagustin.gonzalezorlando@arm.com>
diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py
index 390cbbc..14a9b86 100755
--- a/tests/scripts/analyze_outcomes.py
+++ b/tests/scripts/analyze_outcomes.py
@@ -51,23 +51,25 @@
"""
return len(self.successes) + len(self.failures)
-def analyze_coverage(results, outcomes, allow_list):
+def analyze_coverage(results, outcomes, allow_list, full_coverage):
"""Check that all available test cases are executed at least once."""
available = check_test_cases.collect_available_test_cases()
for key in available:
hits = outcomes[key].hits() if key in outcomes else 0
if hits == 0 and key not in allow_list:
- # Make this a warning, not an error, as long as we haven't
- # fixed this branch to have full coverage of test cases.
- results.warning('Test case not executed: {}', key)
+ if full_coverage:
+ results.error('Test case not executed: {}', key)
+ else:
+ results.warning('Test case not executed: {}', key)
elif hits != 0 and key in allow_list:
# Test Case should be removed from the allow list.
results.warning('Allow listed test case was executed: {}', key)
-def analyze_outcomes(outcomes, allow_list):
+def analyze_outcomes(outcomes, args):
"""Run all analyses on the given outcome collection."""
results = Results()
- analyze_coverage(results, outcomes, allow_list)
+ analyze_coverage(results, outcomes, args['allow_list'],
+ args['full_coverage'])
return results
def read_outcome_file(outcome_file):
@@ -95,7 +97,7 @@
"""Perform coverage analysis."""
outcomes = read_outcome_file(outcome_file)
Results.log("\n*** Analyze coverage ***\n")
- results = analyze_outcomes(outcomes, args['allow_list'])
+ results = analyze_outcomes(outcomes, args)
return results.error_count == 0
# List of tasks with a function that can handle this task and additional arguments if required
@@ -104,6 +106,7 @@
'test_function': do_analyze_coverage,
'args': {
'allow_list': [],
+ 'full_coverage': False,
}
},
}
@@ -120,6 +123,11 @@
'comma/space-separated list of tasks. ')
parser.add_argument('--list', action='store_true',
help='List all available tasks and exit.')
+ parser.add_argument('--require-full-coverage', action='store_true',
+ dest='full_coverage', help="Require all available "
+ "test cases to be executed and issue an error "
+ "otherwise. This flag is ignored if 'task' is "
+ "neither 'all' nor 'analyze_coverage'")
options = parser.parse_args()
if options.list:
@@ -139,6 +147,9 @@
Results.log('Error: invalid task: {}'.format(task))
sys.exit(1)
+ TASKS['analyze_coverage']['args']['full_coverage'] = \
+ options.full_coverage
+
for task in TASKS:
if task in tasks:
if not TASKS[task]['test_function'](options.outcomes, TASKS[task]['args']):