Split test case collection from checks
Move the test case collection code out of check_test_cases.py and into its
own module. This allows outcome analysis to depend only on the new module
and not on check_test_cases.py.
Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
diff --git a/tests/scripts/collect_test_cases.py b/tests/scripts/collect_test_cases.py
index 7e9031c..c326710 100644
--- a/tests/scripts/collect_test_cases.py
+++ b/tests/scripts/collect_test_cases.py
@@ -2,3 +2,166 @@
# Copyright The Mbed TLS Contributors
# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+
+import glob
+import os
+import re
+import subprocess
+import sys
+
+import scripts_path # pylint: disable=unused-import
+from mbedtls_framework import build_tree
+
+
+class ScriptOutputError(ValueError):
+ """A kind of ValueError that indicates we found
+ the script doesn't list test cases in an expected
+ pattern.
+ """
+
+ @property
+ def script_name(self):
+ return super().args[0]
+
+ @property
+ def idx(self):
+ return super().args[1]
+
+ @property
+ def line(self):
+ return super().args[2]
+
+class Results:
+ """Store file and line information about errors or warnings in test suites."""
+
+ def __init__(self, options):
+ self.errors = 0
+ self.warnings = 0
+ self.ignore_warnings = options.quiet
+
+ def error(self, file_name, line_number, fmt, *args):
+ sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n').
+ format(file_name, line_number, *args))
+ self.errors += 1
+
+ def warning(self, file_name, line_number, fmt, *args):
+ if not self.ignore_warnings:
+ sys.stderr.write(('{}:{}:Warning:' + fmt + '\n')
+ .format(file_name, line_number, *args))
+ self.warnings += 1
+
+class TestDescriptionExplorer:
+ """An iterator over test cases with descriptions.
+
+The test cases that have descriptions are:
+* Individual unit tests (entries in a .data file) in test suites.
+* Individual test cases in ssl-opt.sh.
+
+This is an abstract class. To use it, derive a class that implements
+the process_test_case method, and call walk_all().
+"""
+
+ def process_test_case(self, per_file_state,
+ file_name, line_number, description):
+ """Process a test case.
+
+per_file_state: an object created by new_per_file_state() at the beginning
+ of each file.
+file_name: a relative path to the file containing the test case.
+line_number: the line number in the given file.
+description: the test case description as a byte string.
+"""
+ raise NotImplementedError
+
+ def new_per_file_state(self):
+ """Return a new per-file state object.
+
+The default per-file state object is None. Child classes that require per-file
+state may override this method.
+"""
+ #pylint: disable=no-self-use
+ return None
+
+ def walk_test_suite(self, data_file_name):
+ """Iterate over the test cases in the given unit test data file."""
+ in_paragraph = False
+ descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
+ with open(data_file_name, 'rb') as data_file:
+ for line_number, line in enumerate(data_file, 1):
+ line = line.rstrip(b'\r\n')
+ if not line:
+ in_paragraph = False
+ continue
+ if line.startswith(b'#'):
+ continue
+ if not in_paragraph:
+ # This is a test case description line.
+ self.process_test_case(descriptions,
+ data_file_name, line_number, line)
+ in_paragraph = True
+
+ def collect_from_script(self, script_name):
+ """Collect the test cases in a script by calling its listing test cases
+option"""
+ descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
+ listed = subprocess.check_output(['sh', script_name, '--list-test-cases'])
+ # Assume test file is responsible for printing identical format of
+ # test case description between --list-test-cases and its OUTCOME.CSV
+ #
+ # idx indicates the number of test case since there is no line number
+ # in the script for each test case.
+ for idx, line in enumerate(listed.splitlines()):
+ # We are expecting the script to list the test cases in
+ # `<suite_name>;<description>` pattern.
+ script_outputs = line.split(b';', 1)
+ if len(script_outputs) == 2:
+ suite_name, description = script_outputs
+ else:
+ raise ScriptOutputError(script_name, idx, line.decode("utf-8"))
+
+ self.process_test_case(descriptions,
+ suite_name.decode('utf-8'),
+ idx,
+ description.rstrip())
+
+ @staticmethod
+ def collect_test_directories():
+ """Get the relative path for the TLS and Crypto test directories."""
+ mbedtls_root = build_tree.guess_mbedtls_root()
+ directories = [os.path.join(mbedtls_root, 'tests'),
+ os.path.join(mbedtls_root, 'tf-psa-crypto', 'tests')]
+ directories = [os.path.relpath(p) for p in directories]
+ return directories
+
+ def walk_all(self):
+ """Iterate over all named test cases."""
+ test_directories = self.collect_test_directories()
+ for directory in test_directories:
+ for data_file_name in glob.glob(os.path.join(directory, 'suites',
+ '*.data')):
+ self.walk_test_suite(data_file_name)
+
+ for sh_file in ['ssl-opt.sh', 'compat.sh']:
+ sh_file = os.path.join(directory, sh_file)
+ if os.path.isfile(sh_file):
+ self.collect_from_script(sh_file)
+
+class TestDescriptions(TestDescriptionExplorer):
+ """Collect the available test cases."""
+
+ def __init__(self):
+ super().__init__()
+ self.descriptions = set()
+
+ def process_test_case(self, _per_file_state,
+ file_name, _line_number, description):
+ """Record an available test case."""
+ base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name))
+ key = ';'.join([base_name, description.decode('utf-8')])
+ self.descriptions.add(key)
+
+def collect_available_test_cases():
+ """Collect the available test cases."""
+ explorer = TestDescriptions()
+ explorer.walk_all()
+ return sorted(explorer.descriptions)