Check test case coverage
Check that every available test case in the test suites and ssl-opt.sh
has been executed at least once.
For the time being, only report a warning, because our coverage is
incomplete. Once we've updated all.sh to have full coverage, this
warning should become an error.
Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py
index 9d011db..96599bd 100755
--- a/tests/scripts/analyze_outcomes.py
+++ b/tests/scripts/analyze_outcomes.py
@@ -7,9 +7,12 @@
"""
import argparse
+import re
import sys
import traceback
+import check_test_cases
+
class Results:
"""Process analysis results."""
@@ -44,9 +47,40 @@
"""
return len(self.successes) + len(self.failures)
+class TestDescriptions(check_test_cases.TestDescriptionExplorer):
+ """Collect the available test cases."""
+
+ def __init__(self):
+ super().__init__()
+ self.descriptions = set()
+
+ def process_test_case(self, _per_file_state,
+ file_name, _line_number, description):
+ """Record an available test case."""
+ base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name))
+ key = ';'.join([base_name, description.decode('utf-8')])
+ self.descriptions.add(key)
+
+def collect_available_test_cases():
+ """Collect the available test cases."""
+ explorer = TestDescriptions()
+ explorer.walk_all()
+ return sorted(explorer.descriptions)
+
+def analyze_coverage(results, outcomes):
+ """Check that all available test cases are executed at least once."""
+ available = collect_available_test_cases()
+ for key in available:
+ hits = outcomes[key].hits() if key in outcomes else 0
+ if hits == 0:
+ # Make this a warning, not an error, as long as we haven't
+ # fixed this branch to have full coverage of test cases.
+ results.warning('Test case not executed: {}', key)
+
def analyze_outcomes(outcomes):
"""Run all analyses on the given outcome collection."""
results = Results()
+ analyze_coverage(results, outcomes)
return results
def read_outcome_file(outcome_file):