Merge pull request #9594 from gilles-peskine-arm/analyze_outcomes-classes-development
analyze_outcomes.py refactoring: change stringly typed data to classes
diff --git a/.mypy.ini b/.mypy.ini
index 6b831dd..f727cc2 100644
--- a/.mypy.ini
+++ b/.mypy.ini
@@ -1,4 +1,4 @@
[mypy]
-mypy_path = scripts
+mypy_path = framework/scripts:scripts
namespace_packages = True
warn_unused_configs = True
diff --git a/.pylintrc b/.pylintrc
index f395fb9..f9c97d5 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -1,5 +1,5 @@
[MASTER]
-init-hook='import sys; sys.path.append("scripts")'
+init-hook='import sys; sys.path.append("scripts"); sys.path.append("framework/scripts")'
min-similarity-lines=10
[BASIC]
diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py
index e78e90c..188b68d 100755
--- a/tests/scripts/analyze_outcomes.py
+++ b/tests/scripts/analyze_outcomes.py
@@ -82,39 +82,9 @@
if ret_val != 0:
results.error("failed to run reference/driver components")
-def analyze_coverage(results: Results, outcomes: Outcomes,
- allow_list: typing.List[str], full_coverage: bool) -> None:
- """Check that all available test cases are executed at least once."""
- # Make sure that the generated data files are present (and up-to-date).
- # This allows analyze_outcomes.py to run correctly on a fresh Git
- # checkout.
- cp = subprocess.run(['make', 'generated_files'],
- cwd='tests',
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
- check=False)
- if cp.returncode != 0:
- sys.stderr.write(cp.stdout.decode('utf-8'))
- results.error("Failed \"make generated_files\" in tests. "
- "Coverage analysis may be incorrect.")
- available = check_test_cases.collect_available_test_cases()
- for suite_case in available:
- hit = any(suite_case in comp_outcomes.successes or
- suite_case in comp_outcomes.failures
- for comp_outcomes in outcomes.values())
+IgnoreEntry = typing.Union[str, typing.Pattern]
- if not hit and suite_case not in allow_list:
- if full_coverage:
- results.error('Test case not executed: {}', suite_case)
- else:
- results.warning('Test case not executed: {}', suite_case)
- elif hit and suite_case in allow_list:
- # Test Case should be removed from the allow list.
- if full_coverage:
- results.error('Allow listed test case was executed: {}', suite_case)
- else:
- results.warning('Allow listed test case was executed: {}', suite_case)
-
-def name_matches_pattern(name: str, str_or_re) -> bool:
+def name_matches_pattern(name: str, str_or_re: IgnoreEntry) -> bool:
"""Check if name matches a pattern, that may be a string or regex.
- If the pattern is a string, name must be equal to match.
- If the pattern is a regex, name must fully match.
@@ -126,55 +96,6 @@
else:
return str_or_re == name
-def analyze_driver_vs_reference(results: Results, outcomes: Outcomes,
- component_ref: str, component_driver: str,
- ignored_suites: typing.List[str], ignored_tests=None) -> None:
- """Check that all tests passing in the driver component are also
- passing in the corresponding reference component.
- Skip:
- - full test suites provided in ignored_suites list
- - only some specific test inside a test suite, for which the corresponding
- output string is provided
- """
- ref_outcomes = outcomes.get("component_" + component_ref)
- driver_outcomes = outcomes.get("component_" + component_driver)
-
- if ref_outcomes is None or driver_outcomes is None:
- results.error("required components are missing: bad outcome file?")
- return
-
- if not ref_outcomes.successes:
- results.error("no passing test in reference component: bad outcome file?")
- return
-
- for suite_case in ref_outcomes.successes:
- # suite_case is like "test_suite_foo.bar;Description of test case"
- (full_test_suite, test_string) = suite_case.split(';')
- test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name
-
- # Immediately skip fully-ignored test suites
- if test_suite in ignored_suites or full_test_suite in ignored_suites:
- continue
-
- # For ignored test cases inside test suites, just remember and:
- # don't issue an error if they're skipped with drivers,
- # but issue an error if they're not (means we have a bad entry).
- ignored = False
- for str_or_re in (ignored_tests.get(full_test_suite, []) +
- ignored_tests.get(test_suite, [])):
- if name_matches_pattern(test_string, str_or_re):
- ignored = True
-
- if not ignored and not suite_case in driver_outcomes.successes:
- results.error("SKIP/FAIL -> PASS: {}", suite_case)
- if ignored and suite_case in driver_outcomes.successes:
- results.error("uselessly ignored: {}", suite_case)
-
-def analyze_outcomes(results: Results, outcomes: Outcomes, args) -> None:
- """Run all analyses on the given outcome collection."""
- analyze_coverage(results, outcomes, args['allow_list'],
- args['full_coverage'])
-
def read_outcome_file(outcome_file: str) -> Outcomes:
"""Parse an outcome file and return an outcome collection.
"""
@@ -195,530 +116,659 @@
return outcomes
-def do_analyze_coverage(results: Results, outcomes: Outcomes, args) -> None:
- """Perform coverage analysis."""
- results.new_section("Analyze coverage")
- analyze_outcomes(results, outcomes, args)
-def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -> None:
- """Perform driver vs reference analyze."""
- results.new_section("Analyze driver {} vs reference {}",
- args['component_driver'], args['component_ref'])
+class Task:
+ """Base class for outcome analysis tasks."""
- ignored_suites = ['test_suite_' + x for x in args['ignored_suites']]
+ # Override the following in child classes.
+ # Map test suite names (with the test_suite_prefix) to a list of ignored
+ # test cases. Each element in the list can be either a string or a regex;
+ # see the `name_matches_pattern` function.
+ IGNORED_TESTS = {} #type: typing.Dict[str, typing.List[IgnoreEntry]]
- analyze_driver_vs_reference(results, outcomes,
- args['component_ref'], args['component_driver'],
- ignored_suites, args['ignored_tests'])
+ def __init__(self, options) -> None:
+ """Pass command line options to the tasks.
+
+ Each task decides which command line options it cares about.
+ """
+ pass
+
+ def section_name(self) -> str:
+ """The section name to use in results."""
+
+ def ignored_tests(self, test_suite: str) -> typing.Iterator[IgnoreEntry]:
+ """Generate the ignore list for the specified test suite."""
+ if test_suite in self.IGNORED_TESTS:
+ yield from self.IGNORED_TESTS[test_suite]
+ pos = test_suite.find('.')
+ if pos != -1:
+ base_test_suite = test_suite[:pos]
+ if base_test_suite in self.IGNORED_TESTS:
+ yield from self.IGNORED_TESTS[base_test_suite]
+
+ def is_test_case_ignored(self, test_suite: str, test_string: str) -> bool:
+ """Check if the specified test case is ignored."""
+ for str_or_re in self.ignored_tests(test_suite):
+ if name_matches_pattern(test_string, str_or_re):
+ return True
+ return False
+
+ def run(self, results: Results, outcomes: Outcomes):
+ """Run the analysis on the specified outcomes.
+
+ Signal errors via the results objects
+ """
+ raise NotImplementedError
+
+
+class CoverageTask(Task):
+ """Analyze test coverage."""
+
+ # Test cases whose suite and description are matched by an entry in
+ # IGNORED_TESTS are expected to be never executed.
+ # All other test cases are expected to be executed at least once.
+ IGNORED_TESTS = {
+ 'test_suite_psa_crypto_metadata': [
+ # Algorithm not supported yet
+ 'Asymmetric signature: pure EdDSA',
+ # Algorithm not supported yet
+ 'Cipher: XTS',
+ ],
+ }
+
+ def __init__(self, options) -> None:
+ super().__init__(options)
+ self.full_coverage = options.full_coverage #type: bool
+
+ @staticmethod
+ def section_name() -> str:
+ return "Analyze coverage"
+
+ def run(self, results: Results, outcomes: Outcomes) -> None:
+ """Check that all available test cases are executed at least once."""
+ # Make sure that the generated data files are present (and up-to-date).
+ # This allows analyze_outcomes.py to run correctly on a fresh Git
+ # checkout.
+ cp = subprocess.run(['make', 'generated_files'],
+ cwd='tests',
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ check=False)
+ if cp.returncode != 0:
+ sys.stderr.write(cp.stdout.decode('utf-8'))
+ results.error("Failed \"make generated_files\" in tests. "
+ "Coverage analysis may be incorrect.")
+ available = check_test_cases.collect_available_test_cases()
+ for suite_case in available:
+ hit = any(suite_case in comp_outcomes.successes or
+ suite_case in comp_outcomes.failures
+ for comp_outcomes in outcomes.values())
+ (test_suite, test_description) = suite_case.split(';')
+ ignored = self.is_test_case_ignored(test_suite, test_description)
+
+ if not hit and not ignored:
+ if self.full_coverage:
+ results.error('Test case not executed: {}', suite_case)
+ else:
+ results.warning('Test case not executed: {}', suite_case)
+ elif hit and ignored:
+ # If a test case is no longer always skipped, we should remove
+ # it from the ignore list.
+ if self.full_coverage:
+ results.error('Test case was executed but marked as ignored for coverage: {}',
+ suite_case)
+ else:
+ results.warning('Test case was executed but marked as ignored for coverage: {}',
+ suite_case)
+
+
+class DriverVSReference(Task):
+ """Compare outcomes from testing with and without a driver.
+
+ There are 2 options to use analyze_driver_vs_reference_xxx locally:
+ 1. Run tests and then analysis:
+ - tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver>
+ - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
+ 2. Let this script run both automatically:
+ - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
+ """
+
+ # Override the following in child classes.
+ # Configuration name (all.sh component) used as the reference.
+ REFERENCE = ''
+ # Configuration name (all.sh component) used as the driver.
+ DRIVER = ''
+ # Ignored test suites (without the test_suite_ prefix).
+ IGNORED_SUITES = [] #type: typing.List[str]
+
+ def __init__(self, options) -> None:
+ super().__init__(options)
+ self.ignored_suites = frozenset('test_suite_' + x
+ for x in self.IGNORED_SUITES)
+
+ def section_name(self) -> str:
+ return f"Analyze driver {self.DRIVER} vs reference {self.REFERENCE}"
+
+ def run(self, results: Results, outcomes: Outcomes) -> None:
+ """Check that all tests passing in the driver component are also
+ passing in the corresponding reference component.
+ Skip:
+ - full test suites provided in ignored_suites list
+ - only some specific test inside a test suite, for which the corresponding
+ output string is provided
+ """
+ ref_outcomes = outcomes.get("component_" + self.REFERENCE)
+ driver_outcomes = outcomes.get("component_" + self.DRIVER)
+
+ if ref_outcomes is None or driver_outcomes is None:
+ results.error("required components are missing: bad outcome file?")
+ return
+
+ if not ref_outcomes.successes:
+ results.error("no passing test in reference component: bad outcome file?")
+ return
+
+ for suite_case in ref_outcomes.successes:
+ # suite_case is like "test_suite_foo.bar;Description of test case"
+ (full_test_suite, test_string) = suite_case.split(';')
+ test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name
+
+ # Immediately skip fully-ignored test suites
+ if test_suite in self.ignored_suites or \
+ full_test_suite in self.ignored_suites:
+ continue
+
+ # For ignored test cases inside test suites, just remember and:
+ # don't issue an error if they're skipped with drivers,
+ # but issue an error if they're not (means we have a bad entry).
+ ignored = self.is_test_case_ignored(full_test_suite, test_string)
+
+ if not ignored and not suite_case in driver_outcomes.successes:
+ results.error("SKIP/FAIL -> PASS: {}", suite_case)
+ if ignored and suite_case in driver_outcomes.successes:
+ results.error("uselessly ignored: {}", suite_case)
+
+
+# The names that we give to classes derived from DriverVSReference do not
+# follow the usual naming convention, because it's more readable to use
+# underscores and parts of the configuration names. Also, these classes
+# are just there to specify some data, so they don't need repetitive
+# documentation.
+#pylint: disable=invalid-name,missing-class-docstring
+
+class DriverVSReference_hash(DriverVSReference):
+ REFERENCE = 'test_psa_crypto_config_reference_hash_use_psa'
+ DRIVER = 'test_psa_crypto_config_accel_hash_use_psa'
+ IGNORED_SUITES = [
+ 'shax', 'mdx', # the software implementations that are being excluded
+ 'md.psa', # purposefully depends on whether drivers are present
+ 'psa_crypto_low_hash.generated', # testing the builtins
+ ]
+ IGNORED_TESTS = {
+ 'test_suite_config': [
+ re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'),
+ ],
+ 'test_suite_platform': [
+ # Incompatible with sanitizers (e.g. ASan). If the driver
+ # component uses a sanitizer but the reference component
+ # doesn't, we have a PASS vs SKIP mismatch.
+ 'Check mbedtls_calloc overallocation',
+ ],
+ }
+
+class DriverVSReference_hmac(DriverVSReference):
+ REFERENCE = 'test_psa_crypto_config_reference_hmac'
+ DRIVER = 'test_psa_crypto_config_accel_hmac'
+ IGNORED_SUITES = [
+ # These suites require legacy hash support, which is disabled
+ # in the accelerated component.
+ 'shax', 'mdx',
+ # This suite tests builtins directly, but these are missing
+ # in the accelerated case.
+ 'psa_crypto_low_hash.generated',
+ ]
+ IGNORED_TESTS = {
+ 'test_suite_config': [
+ re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'),
+ re.compile(r'.*\bMBEDTLS_MD_C\b')
+ ],
+ 'test_suite_md': [
+ # Builtin HMAC is not supported in the accelerate component.
+ re.compile('.*HMAC.*'),
+ # Following tests make use of functions which are not available
+ # when MD_C is disabled, as it happens in the accelerated
+ # test component.
+ re.compile('generic .* Hash file .*'),
+ 'MD list',
+ ],
+ 'test_suite_md.psa': [
+ # "legacy only" tests require hash algorithms to be NOT
+ # accelerated, but this of course false for the accelerated
+ # test component.
+ re.compile('PSA dispatch .* legacy only'),
+ ],
+ 'test_suite_platform': [
+ # Incompatible with sanitizers (e.g. ASan). If the driver
+ # component uses a sanitizer but the reference component
+ # doesn't, we have a PASS vs SKIP mismatch.
+ 'Check mbedtls_calloc overallocation',
+ ],
+ }
+
+class DriverVSReference_cipher_aead_cmac(DriverVSReference):
+ REFERENCE = 'test_psa_crypto_config_reference_cipher_aead_cmac'
+ DRIVER = 'test_psa_crypto_config_accel_cipher_aead_cmac'
+ # Modules replaced by drivers.
+ IGNORED_SUITES = [
+ # low-level (block/stream) cipher modules
+ 'aes', 'aria', 'camellia', 'des', 'chacha20',
+ # AEAD modes and CMAC
+ 'ccm', 'chachapoly', 'cmac', 'gcm',
+ # The Cipher abstraction layer
+ 'cipher',
+ ]
+ IGNORED_TESTS = {
+ 'test_suite_config': [
+ re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA|CHACHA20|DES)_.*'),
+ re.compile(r'.*\bMBEDTLS_(CCM|CHACHAPOLY|CMAC|GCM)_.*'),
+ re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'),
+ re.compile(r'.*\bMBEDTLS_CIPHER_.*'),
+ ],
+ # PEM decryption is not supported so far.
+ # The rest of PEM (write, unencrypted read) works though.
+ 'test_suite_pem': [
+ re.compile(r'PEM read .*(AES|DES|\bencrypt).*'),
+ ],
+ 'test_suite_platform': [
+ # Incompatible with sanitizers (e.g. ASan). If the driver
+ # component uses a sanitizer but the reference component
+ # doesn't, we have a PASS vs SKIP mismatch.
+ 'Check mbedtls_calloc overallocation',
+ ],
+ # Following tests depend on AES_C/DES_C but are not about
+ # them really, just need to know some error code is there.
+ 'test_suite_error': [
+ 'Low and high error',
+ 'Single low error'
+ ],
+ # Similar to test_suite_error above.
+ 'test_suite_version': [
+ 'Check for MBEDTLS_AES_C when already present',
+ ],
+ # The en/decryption part of PKCS#12 is not supported so far.
+ # The rest of PKCS#12 (key derivation) works though.
+ 'test_suite_pkcs12': [
+ re.compile(r'PBE Encrypt, .*'),
+ re.compile(r'PBE Decrypt, .*'),
+ ],
+ # The en/decryption part of PKCS#5 is not supported so far.
+ # The rest of PKCS#5 (PBKDF2) works though.
+ 'test_suite_pkcs5': [
+ re.compile(r'PBES2 Encrypt, .*'),
+ re.compile(r'PBES2 Decrypt .*'),
+ ],
+ # Encrypted keys are not supported so far.
+ # pylint: disable=line-too-long
+ 'test_suite_pkparse': [
+ 'Key ASN1 (Encrypted key PKCS12, trailing garbage data)',
+ 'Key ASN1 (Encrypted key PKCS5, trailing garbage data)',
+ re.compile(r'Parse (RSA|EC) Key .*\(.* ([Ee]ncrypted|password).*\)'),
+ ],
+ # Encrypted keys are not supported so far.
+ 'ssl-opt': [
+ 'TLS: password protected server key',
+ 'TLS: password protected client key',
+ 'TLS: password protected server key, two certificates',
+ ],
+ }
+
+class DriverVSReference_ecp_light_only(DriverVSReference):
+ REFERENCE = 'test_psa_crypto_config_reference_ecc_ecp_light_only'
+ DRIVER = 'test_psa_crypto_config_accel_ecc_ecp_light_only'
+ IGNORED_SUITES = [
+ # Modules replaced by drivers
+ 'ecdsa', 'ecdh', 'ecjpake',
+ ]
+ IGNORED_TESTS = {
+ 'test_suite_config': [
+ re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
+ ],
+ 'test_suite_platform': [
+ # Incompatible with sanitizers (e.g. ASan). If the driver
+ # component uses a sanitizer but the reference component
+ # doesn't, we have a PASS vs SKIP mismatch.
+ 'Check mbedtls_calloc overallocation',
+ ],
+ # This test wants a legacy function that takes f_rng, p_rng
+ # arguments, and uses legacy ECDSA for that. The test is
+ # really about the wrapper around the PSA RNG, not ECDSA.
+ 'test_suite_random': [
+ 'PSA classic wrapper: ECDSA signature (SECP256R1)',
+ ],
+ # In the accelerated test ECP_C is not set (only ECP_LIGHT is)
+ # so we must ignore disparities in the tests for which ECP_C
+ # is required.
+ 'test_suite_ecp': [
+ re.compile(r'ECP check public-private .*'),
+ re.compile(r'ECP calculate public: .*'),
+ re.compile(r'ECP gen keypair .*'),
+ re.compile(r'ECP point muladd .*'),
+ re.compile(r'ECP point multiplication .*'),
+ re.compile(r'ECP test vectors .*'),
+ ],
+ 'test_suite_ssl': [
+ # This deprecated function is only present when ECP_C is On.
+ 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
+ ],
+ }
+
+class DriverVSReference_no_ecp_at_all(DriverVSReference):
+ REFERENCE = 'test_psa_crypto_config_reference_ecc_no_ecp_at_all'
+ DRIVER = 'test_psa_crypto_config_accel_ecc_no_ecp_at_all'
+ IGNORED_SUITES = [
+ # Modules replaced by drivers
+ 'ecp', 'ecdsa', 'ecdh', 'ecjpake',
+ ]
+ IGNORED_TESTS = {
+ 'test_suite_config': [
+ re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
+ re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
+ ],
+ 'test_suite_platform': [
+ # Incompatible with sanitizers (e.g. ASan). If the driver
+ # component uses a sanitizer but the reference component
+ # doesn't, we have a PASS vs SKIP mismatch.
+ 'Check mbedtls_calloc overallocation',
+ ],
+ # See ecp_light_only
+ 'test_suite_random': [
+ 'PSA classic wrapper: ECDSA signature (SECP256R1)',
+ ],
+ 'test_suite_pkparse': [
+ # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED
+ # is automatically enabled in build_info.h (backward compatibility)
+ # even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a
+ # consequence compressed points are supported in the reference
+ # component but not in the accelerated one, so they should be skipped
+ # while checking driver's coverage.
+ re.compile(r'Parse EC Key .*compressed\)'),
+ re.compile(r'Parse Public EC Key .*compressed\)'),
+ ],
+ # See ecp_light_only
+ 'test_suite_ssl': [
+ 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
+ ],
+ }
+
+class DriverVSReference_ecc_no_bignum(DriverVSReference):
+ REFERENCE = 'test_psa_crypto_config_reference_ecc_no_bignum'
+ DRIVER = 'test_psa_crypto_config_accel_ecc_no_bignum'
+ IGNORED_SUITES = [
+ # Modules replaced by drivers
+ 'ecp', 'ecdsa', 'ecdh', 'ecjpake',
+ 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
+ 'bignum.generated', 'bignum.misc',
+ ]
+ IGNORED_TESTS = {
+ 'test_suite_config': [
+ re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
+ re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
+ re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
+ ],
+ 'test_suite_platform': [
+ # Incompatible with sanitizers (e.g. ASan). If the driver
+ # component uses a sanitizer but the reference component
+ # doesn't, we have a PASS vs SKIP mismatch.
+ 'Check mbedtls_calloc overallocation',
+ ],
+ # See ecp_light_only
+ 'test_suite_random': [
+ 'PSA classic wrapper: ECDSA signature (SECP256R1)',
+ ],
+ # See no_ecp_at_all
+ 'test_suite_pkparse': [
+ re.compile(r'Parse EC Key .*compressed\)'),
+ re.compile(r'Parse Public EC Key .*compressed\)'),
+ ],
+ 'test_suite_asn1parse': [
+ 'INTEGER too large for mpi',
+ ],
+ 'test_suite_asn1write': [
+ re.compile(r'ASN.1 Write mpi.*'),
+ ],
+ 'test_suite_debug': [
+ re.compile(r'Debug print mbedtls_mpi.*'),
+ ],
+ # See ecp_light_only
+ 'test_suite_ssl': [
+ 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
+ ],
+ }
+
+class DriverVSReference_ecc_ffdh_no_bignum(DriverVSReference):
+ REFERENCE = 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum'
+ DRIVER = 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum'
+ IGNORED_SUITES = [
+ # Modules replaced by drivers
+ 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm',
+ 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
+ 'bignum.generated', 'bignum.misc',
+ ]
+ IGNORED_TESTS = {
+ 'ssl-opt': [
+ # DHE support in TLS 1.2 requires built-in MBEDTLS_DHM_C
+ # (because it needs custom groups, which PSA does not
+ # provide), even with MBEDTLS_USE_PSA_CRYPTO.
+ re.compile(r'PSK callback:.*\bdhe-psk\b.*'),
+ ],
+ 'test_suite_config': [
+ re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
+ re.compile(r'.*\bMBEDTLS_DHM_C\b.*'),
+ re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
+ re.compile(r'.*\bMBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED\b.*'),
+ re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
+ ],
+ 'test_suite_platform': [
+ # Incompatible with sanitizers (e.g. ASan). If the driver
+ # component uses a sanitizer but the reference component
+ # doesn't, we have a PASS vs SKIP mismatch.
+ 'Check mbedtls_calloc overallocation',
+ ],
+ # See ecp_light_only
+ 'test_suite_random': [
+ 'PSA classic wrapper: ECDSA signature (SECP256R1)',
+ ],
+ # See no_ecp_at_all
+ 'test_suite_pkparse': [
+ re.compile(r'Parse EC Key .*compressed\)'),
+ re.compile(r'Parse Public EC Key .*compressed\)'),
+ ],
+ 'test_suite_asn1parse': [
+ 'INTEGER too large for mpi',
+ ],
+ 'test_suite_asn1write': [
+ re.compile(r'ASN.1 Write mpi.*'),
+ ],
+ 'test_suite_debug': [
+ re.compile(r'Debug print mbedtls_mpi.*'),
+ ],
+ # See ecp_light_only
+ 'test_suite_ssl': [
+ 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
+ ],
+ }
+
+class DriverVSReference_ffdh_alg(DriverVSReference):
+ REFERENCE = 'test_psa_crypto_config_reference_ffdh'
+ DRIVER = 'test_psa_crypto_config_accel_ffdh'
+ IGNORED_SUITES = ['dhm']
+ IGNORED_TESTS = {
+ 'test_suite_config': [
+ re.compile(r'.*\bMBEDTLS_DHM_C\b.*'),
+ ],
+ 'test_suite_platform': [
+ # Incompatible with sanitizers (e.g. ASan). If the driver
+ # component uses a sanitizer but the reference component
+ # doesn't, we have a PASS vs SKIP mismatch.
+ 'Check mbedtls_calloc overallocation',
+ ],
+ }
+
+class DriverVSReference_tfm_config(DriverVSReference):
+ REFERENCE = 'test_tfm_config_no_p256m'
+ DRIVER = 'test_tfm_config_p256m_driver_accel_ec'
+ IGNORED_SUITES = [
+ # Modules replaced by drivers
+ 'asn1parse', 'asn1write',
+ 'ecp', 'ecdsa', 'ecdh', 'ecjpake',
+ 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
+ 'bignum.generated', 'bignum.misc',
+ ]
+ IGNORED_TESTS = {
+ 'test_suite_config': [
+ re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
+ re.compile(r'.*\bMBEDTLS_(ASN1\w+)_C\b.*'),
+ re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECP)_.*'),
+ re.compile(r'.*\bMBEDTLS_PSA_P256M_DRIVER_ENABLED\b.*')
+ ],
+ 'test_suite_config.crypto_combinations': [
+ 'Config: ECC: Weierstrass curves only',
+ ],
+ 'test_suite_platform': [
+ # Incompatible with sanitizers (e.g. ASan). If the driver
+ # component uses a sanitizer but the reference component
+ # doesn't, we have a PASS vs SKIP mismatch.
+ 'Check mbedtls_calloc overallocation',
+ ],
+ # See ecp_light_only
+ 'test_suite_random': [
+ 'PSA classic wrapper: ECDSA signature (SECP256R1)',
+ ],
+ }
+
+class DriverVSReference_rsa(DriverVSReference):
+ REFERENCE = 'test_psa_crypto_config_reference_rsa_crypto'
+ DRIVER = 'test_psa_crypto_config_accel_rsa_crypto'
+ IGNORED_SUITES = [
+ # Modules replaced by drivers.
+ 'rsa', 'pkcs1_v15', 'pkcs1_v21',
+ # We temporarily don't care about PK stuff.
+ 'pk', 'pkwrite', 'pkparse'
+ ]
+ IGNORED_TESTS = {
+ 'test_suite_config': [
+ re.compile(r'.*\bMBEDTLS_(PKCS1|RSA)_.*'),
+ re.compile(r'.*\bMBEDTLS_GENPRIME\b.*')
+ ],
+ 'test_suite_platform': [
+ # Incompatible with sanitizers (e.g. ASan). If the driver
+ # component uses a sanitizer but the reference component
+ # doesn't, we have a PASS vs SKIP mismatch.
+ 'Check mbedtls_calloc overallocation',
+ ],
+ # Following tests depend on RSA_C but are not about
+ # them really, just need to know some error code is there.
+ 'test_suite_error': [
+ 'Low and high error',
+ 'Single high error'
+ ],
+ # Constant time operations only used for PKCS1_V15
+ 'test_suite_constant_time': [
+ re.compile(r'mbedtls_ct_zeroize_if .*'),
+ re.compile(r'mbedtls_ct_memmove_left .*')
+ ],
+ 'test_suite_psa_crypto': [
+ # We don't support generate_key_custom entry points
+ # in drivers yet.
+ re.compile(r'PSA generate key custom: RSA, e=.*'),
+ re.compile(r'PSA generate key ext: RSA, e=.*'),
+ ],
+ }
+
+class DriverVSReference_block_cipher_dispatch(DriverVSReference):
+ REFERENCE = 'test_full_block_cipher_legacy_dispatch'
+ DRIVER = 'test_full_block_cipher_psa_dispatch'
+ IGNORED_SUITES = [
+ # Skipped in the accelerated component
+ 'aes', 'aria', 'camellia',
+ # These require AES_C, ARIA_C or CAMELLIA_C to be enabled in
+ # order for the cipher module (actually cipher_wrapper) to work
+ # properly. However these symbols are disabled in the accelerated
+ # component so we ignore them.
+ 'cipher.ccm', 'cipher.gcm', 'cipher.aes', 'cipher.aria',
+ 'cipher.camellia',
+ ]
+ IGNORED_TESTS = {
+ 'test_suite_config': [
+ re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA)_.*'),
+ re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'),
+ ],
+ 'test_suite_cmac': [
+ # Following tests require AES_C/ARIA_C/CAMELLIA_C to be enabled,
+ # but these are not available in the accelerated component.
+ 'CMAC null arguments',
+ re.compile('CMAC.* (AES|ARIA|Camellia).*'),
+ ],
+ 'test_suite_cipher.padding': [
+ # Following tests require AES_C/CAMELLIA_C to be enabled,
+ # but these are not available in the accelerated component.
+ re.compile('Set( non-existent)? padding with (AES|CAMELLIA).*'),
+ ],
+ 'test_suite_pkcs5': [
+ # The AES part of PKCS#5 PBES2 is not yet supported.
+ # The rest of PKCS#5 (PBKDF2) works, though.
+ re.compile(r'PBES2 .* AES-.*')
+ ],
+ 'test_suite_pkparse': [
+ # PEM (called by pkparse) requires AES_C in order to decrypt
+ # the key, but this is not available in the accelerated
+ # component.
+ re.compile('Parse RSA Key.*(password|AES-).*'),
+ ],
+ 'test_suite_pem': [
+ # Following tests require AES_C, but this is diabled in the
+ # accelerated component.
+ re.compile('PEM read .*AES.*'),
+ 'PEM read (unknown encryption algorithm)',
+ ],
+ 'test_suite_error': [
+ # Following tests depend on AES_C but are not about them
+ # really, just need to know some error code is there.
+ 'Single low error',
+ 'Low and high error',
+ ],
+ 'test_suite_version': [
+ # Similar to test_suite_error above.
+ 'Check for MBEDTLS_AES_C when already present',
+ ],
+ 'test_suite_platform': [
+ # Incompatible with sanitizers (e.g. ASan). If the driver
+ # component uses a sanitizer but the reference component
+ # doesn't, we have a PASS vs SKIP mismatch.
+ 'Check mbedtls_calloc overallocation',
+ ],
+ }
+
+#pylint: enable=invalid-name,missing-class-docstring
+
+
# List of tasks with a function that can handle this task and additional arguments if required
KNOWN_TASKS = {
- 'analyze_coverage': {
- 'test_function': do_analyze_coverage,
- 'args': {
- 'allow_list': [
- # Algorithm not supported yet
- 'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA',
- # Algorithm not supported yet
- 'test_suite_psa_crypto_metadata;Cipher: XTS',
- ],
- 'full_coverage': False,
- }
- },
- # There are 2 options to use analyze_driver_vs_reference_xxx locally:
- # 1. Run tests and then analysis:
- # - tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver>
- # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
- # 2. Let this script run both automatically:
- # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
- 'analyze_driver_vs_reference_hash': {
- 'test_function': do_analyze_driver_vs_reference,
- 'args': {
- 'component_ref': 'test_psa_crypto_config_reference_hash_use_psa',
- 'component_driver': 'test_psa_crypto_config_accel_hash_use_psa',
- 'ignored_suites': [
- 'shax', 'mdx', # the software implementations that are being excluded
- 'md.psa', # purposefully depends on whether drivers are present
- 'psa_crypto_low_hash.generated', # testing the builtins
- ],
- 'ignored_tests': {
- 'test_suite_config': [
- re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'),
- ],
- 'test_suite_platform': [
- # Incompatible with sanitizers (e.g. ASan). If the driver
- # component uses a sanitizer but the reference component
- # doesn't, we have a PASS vs SKIP mismatch.
- 'Check mbedtls_calloc overallocation',
- ],
- }
- }
- },
- 'analyze_driver_vs_reference_hmac': {
- 'test_function': do_analyze_driver_vs_reference,
- 'args': {
- 'component_ref': 'test_psa_crypto_config_reference_hmac',
- 'component_driver': 'test_psa_crypto_config_accel_hmac',
- 'ignored_suites': [
- # These suites require legacy hash support, which is disabled
- # in the accelerated component.
- 'shax', 'mdx',
- # This suite tests builtins directly, but these are missing
- # in the accelerated case.
- 'psa_crypto_low_hash.generated',
- ],
- 'ignored_tests': {
- 'test_suite_config': [
- re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'),
- re.compile(r'.*\bMBEDTLS_MD_C\b')
- ],
- 'test_suite_md': [
- # Builtin HMAC is not supported in the accelerate component.
- re.compile('.*HMAC.*'),
- # Following tests make use of functions which are not available
- # when MD_C is disabled, as it happens in the accelerated
- # test component.
- re.compile('generic .* Hash file .*'),
- 'MD list',
- ],
- 'test_suite_md.psa': [
- # "legacy only" tests require hash algorithms to be NOT
- # accelerated, but this of course false for the accelerated
- # test component.
- re.compile('PSA dispatch .* legacy only'),
- ],
- 'test_suite_platform': [
- # Incompatible with sanitizers (e.g. ASan). If the driver
- # component uses a sanitizer but the reference component
- # doesn't, we have a PASS vs SKIP mismatch.
- 'Check mbedtls_calloc overallocation',
- ],
- }
- }
- },
- 'analyze_driver_vs_reference_cipher_aead_cmac': {
- 'test_function': do_analyze_driver_vs_reference,
- 'args': {
- 'component_ref': 'test_psa_crypto_config_reference_cipher_aead_cmac',
- 'component_driver': 'test_psa_crypto_config_accel_cipher_aead_cmac',
- # Modules replaced by drivers.
- 'ignored_suites': [
- # low-level (block/stream) cipher modules
- 'aes', 'aria', 'camellia', 'des', 'chacha20',
- # AEAD modes and CMAC
- 'ccm', 'chachapoly', 'cmac', 'gcm',
- # The Cipher abstraction layer
- 'cipher',
- ],
- 'ignored_tests': {
- 'test_suite_config': [
- re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA|CHACHA20|DES)_.*'),
- re.compile(r'.*\bMBEDTLS_(CCM|CHACHAPOLY|CMAC|GCM)_.*'),
- re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'),
- re.compile(r'.*\bMBEDTLS_CIPHER_.*'),
- ],
- # PEM decryption is not supported so far.
- # The rest of PEM (write, unencrypted read) works though.
- 'test_suite_pem': [
- re.compile(r'PEM read .*(AES|DES|\bencrypt).*'),
- ],
- 'test_suite_platform': [
- # Incompatible with sanitizers (e.g. ASan). If the driver
- # component uses a sanitizer but the reference component
- # doesn't, we have a PASS vs SKIP mismatch.
- 'Check mbedtls_calloc overallocation',
- ],
- # Following tests depend on AES_C/DES_C but are not about
- # them really, just need to know some error code is there.
- 'test_suite_error': [
- 'Low and high error',
- 'Single low error'
- ],
- # Similar to test_suite_error above.
- 'test_suite_version': [
- 'Check for MBEDTLS_AES_C when already present',
- ],
- # The en/decryption part of PKCS#12 is not supported so far.
- # The rest of PKCS#12 (key derivation) works though.
- 'test_suite_pkcs12': [
- re.compile(r'PBE Encrypt, .*'),
- re.compile(r'PBE Decrypt, .*'),
- ],
- # The en/decryption part of PKCS#5 is not supported so far.
- # The rest of PKCS#5 (PBKDF2) works though.
- 'test_suite_pkcs5': [
- re.compile(r'PBES2 Encrypt, .*'),
- re.compile(r'PBES2 Decrypt .*'),
- ],
- # Encrypted keys are not supported so far.
- # pylint: disable=line-too-long
- 'test_suite_pkparse': [
- 'Key ASN1 (Encrypted key PKCS12, trailing garbage data)',
- 'Key ASN1 (Encrypted key PKCS5, trailing garbage data)',
- re.compile(r'Parse (RSA|EC) Key .*\(.* ([Ee]ncrypted|password).*\)'),
- ],
- # Encrypted keys are not supported so far.
- 'ssl-opt': [
- 'TLS: password protected server key',
- 'TLS: password protected client key',
- 'TLS: password protected server key, two certificates',
- ],
- }
- }
- },
- 'analyze_driver_vs_reference_ecp_light_only': {
- 'test_function': do_analyze_driver_vs_reference,
- 'args': {
- 'component_ref': 'test_psa_crypto_config_reference_ecc_ecp_light_only',
- 'component_driver': 'test_psa_crypto_config_accel_ecc_ecp_light_only',
- 'ignored_suites': [
- # Modules replaced by drivers
- 'ecdsa', 'ecdh', 'ecjpake',
- ],
- 'ignored_tests': {
- 'test_suite_config': [
- re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
- ],
- 'test_suite_platform': [
- # Incompatible with sanitizers (e.g. ASan). If the driver
- # component uses a sanitizer but the reference component
- # doesn't, we have a PASS vs SKIP mismatch.
- 'Check mbedtls_calloc overallocation',
- ],
- # This test wants a legacy function that takes f_rng, p_rng
- # arguments, and uses legacy ECDSA for that. The test is
- # really about the wrapper around the PSA RNG, not ECDSA.
- 'test_suite_random': [
- 'PSA classic wrapper: ECDSA signature (SECP256R1)',
- ],
- # In the accelerated test ECP_C is not set (only ECP_LIGHT is)
- # so we must ignore disparities in the tests for which ECP_C
- # is required.
- 'test_suite_ecp': [
- re.compile(r'ECP check public-private .*'),
- re.compile(r'ECP calculate public: .*'),
- re.compile(r'ECP gen keypair .*'),
- re.compile(r'ECP point muladd .*'),
- re.compile(r'ECP point multiplication .*'),
- re.compile(r'ECP test vectors .*'),
- ],
- 'test_suite_ssl': [
- # This deprecated function is only present when ECP_C is On.
- 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
- ],
- }
- }
- },
- 'analyze_driver_vs_reference_no_ecp_at_all': {
- 'test_function': do_analyze_driver_vs_reference,
- 'args': {
- 'component_ref': 'test_psa_crypto_config_reference_ecc_no_ecp_at_all',
- 'component_driver': 'test_psa_crypto_config_accel_ecc_no_ecp_at_all',
- 'ignored_suites': [
- # Modules replaced by drivers
- 'ecp', 'ecdsa', 'ecdh', 'ecjpake',
- ],
- 'ignored_tests': {
- 'test_suite_config': [
- re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
- re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
- ],
- 'test_suite_platform': [
- # Incompatible with sanitizers (e.g. ASan). If the driver
- # component uses a sanitizer but the reference component
- # doesn't, we have a PASS vs SKIP mismatch.
- 'Check mbedtls_calloc overallocation',
- ],
- # See ecp_light_only
- 'test_suite_random': [
- 'PSA classic wrapper: ECDSA signature (SECP256R1)',
- ],
- 'test_suite_pkparse': [
- # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED
- # is automatically enabled in build_info.h (backward compatibility)
- # even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a
- # consequence compressed points are supported in the reference
- # component but not in the accelerated one, so they should be skipped
- # while checking driver's coverage.
- re.compile(r'Parse EC Key .*compressed\)'),
- re.compile(r'Parse Public EC Key .*compressed\)'),
- ],
- # See ecp_light_only
- 'test_suite_ssl': [
- 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
- ],
- }
- }
- },
- 'analyze_driver_vs_reference_ecc_no_bignum': {
- 'test_function': do_analyze_driver_vs_reference,
- 'args': {
- 'component_ref': 'test_psa_crypto_config_reference_ecc_no_bignum',
- 'component_driver': 'test_psa_crypto_config_accel_ecc_no_bignum',
- 'ignored_suites': [
- # Modules replaced by drivers
- 'ecp', 'ecdsa', 'ecdh', 'ecjpake',
- 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
- 'bignum.generated', 'bignum.misc',
- ],
- 'ignored_tests': {
- 'test_suite_config': [
- re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
- re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
- re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
- ],
- 'test_suite_platform': [
- # Incompatible with sanitizers (e.g. ASan). If the driver
- # component uses a sanitizer but the reference component
- # doesn't, we have a PASS vs SKIP mismatch.
- 'Check mbedtls_calloc overallocation',
- ],
- # See ecp_light_only
- 'test_suite_random': [
- 'PSA classic wrapper: ECDSA signature (SECP256R1)',
- ],
- # See no_ecp_at_all
- 'test_suite_pkparse': [
- re.compile(r'Parse EC Key .*compressed\)'),
- re.compile(r'Parse Public EC Key .*compressed\)'),
- ],
- 'test_suite_asn1parse': [
- 'INTEGER too large for mpi',
- ],
- 'test_suite_asn1write': [
- re.compile(r'ASN.1 Write mpi.*'),
- ],
- 'test_suite_debug': [
- re.compile(r'Debug print mbedtls_mpi.*'),
- ],
- # See ecp_light_only
- 'test_suite_ssl': [
- 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
- ],
- }
- }
- },
- 'analyze_driver_vs_reference_ecc_ffdh_no_bignum': {
- 'test_function': do_analyze_driver_vs_reference,
- 'args': {
- 'component_ref': 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum',
- 'component_driver': 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum',
- 'ignored_suites': [
- # Modules replaced by drivers
- 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm',
- 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
- 'bignum.generated', 'bignum.misc',
- ],
- 'ignored_tests': {
- 'ssl-opt': [
- # DHE support in TLS 1.2 requires built-in MBEDTLS_DHM_C
- # (because it needs custom groups, which PSA does not
- # provide), even with MBEDTLS_USE_PSA_CRYPTO.
- re.compile(r'PSK callback:.*\bdhe-psk\b.*'),
- ],
- 'test_suite_config': [
- re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
- re.compile(r'.*\bMBEDTLS_DHM_C\b.*'),
- re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
- re.compile(r'.*\bMBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED\b.*'),
- re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
- ],
- 'test_suite_platform': [
- # Incompatible with sanitizers (e.g. ASan). If the driver
- # component uses a sanitizer but the reference component
- # doesn't, we have a PASS vs SKIP mismatch.
- 'Check mbedtls_calloc overallocation',
- ],
- # See ecp_light_only
- 'test_suite_random': [
- 'PSA classic wrapper: ECDSA signature (SECP256R1)',
- ],
- # See no_ecp_at_all
- 'test_suite_pkparse': [
- re.compile(r'Parse EC Key .*compressed\)'),
- re.compile(r'Parse Public EC Key .*compressed\)'),
- ],
- 'test_suite_asn1parse': [
- 'INTEGER too large for mpi',
- ],
- 'test_suite_asn1write': [
- re.compile(r'ASN.1 Write mpi.*'),
- ],
- 'test_suite_debug': [
- re.compile(r'Debug print mbedtls_mpi.*'),
- ],
- # See ecp_light_only
- 'test_suite_ssl': [
- 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
- ],
- }
- }
- },
- 'analyze_driver_vs_reference_ffdh_alg': {
- 'test_function': do_analyze_driver_vs_reference,
- 'args': {
- 'component_ref': 'test_psa_crypto_config_reference_ffdh',
- 'component_driver': 'test_psa_crypto_config_accel_ffdh',
- 'ignored_suites': ['dhm'],
- 'ignored_tests': {
- 'test_suite_config': [
- re.compile(r'.*\bMBEDTLS_DHM_C\b.*'),
- ],
- 'test_suite_platform': [
- # Incompatible with sanitizers (e.g. ASan). If the driver
- # component uses a sanitizer but the reference component
- # doesn't, we have a PASS vs SKIP mismatch.
- 'Check mbedtls_calloc overallocation',
- ],
- }
- }
- },
- 'analyze_driver_vs_reference_tfm_config': {
- 'test_function': do_analyze_driver_vs_reference,
- 'args': {
- 'component_ref': 'test_tfm_config_no_p256m',
- 'component_driver': 'test_tfm_config_p256m_driver_accel_ec',
- 'ignored_suites': [
- # Modules replaced by drivers
- 'asn1parse', 'asn1write',
- 'ecp', 'ecdsa', 'ecdh', 'ecjpake',
- 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
- 'bignum.generated', 'bignum.misc',
- ],
- 'ignored_tests': {
- 'test_suite_config': [
- re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
- re.compile(r'.*\bMBEDTLS_(ASN1\w+)_C\b.*'),
- re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECP)_.*'),
- re.compile(r'.*\bMBEDTLS_PSA_P256M_DRIVER_ENABLED\b.*')
- ],
- 'test_suite_config.crypto_combinations': [
- 'Config: ECC: Weierstrass curves only',
- ],
- 'test_suite_platform': [
- # Incompatible with sanitizers (e.g. ASan). If the driver
- # component uses a sanitizer but the reference component
- # doesn't, we have a PASS vs SKIP mismatch.
- 'Check mbedtls_calloc overallocation',
- ],
- # See ecp_light_only
- 'test_suite_random': [
- 'PSA classic wrapper: ECDSA signature (SECP256R1)',
- ],
- }
- }
- },
- 'analyze_driver_vs_reference_rsa': {
- 'test_function': do_analyze_driver_vs_reference,
- 'args': {
- 'component_ref': 'test_psa_crypto_config_reference_rsa_crypto',
- 'component_driver': 'test_psa_crypto_config_accel_rsa_crypto',
- 'ignored_suites': [
- # Modules replaced by drivers.
- 'rsa', 'pkcs1_v15', 'pkcs1_v21',
- # We temporarily don't care about PK stuff.
- 'pk', 'pkwrite', 'pkparse'
- ],
- 'ignored_tests': {
- 'test_suite_config': [
- re.compile(r'.*\bMBEDTLS_(PKCS1|RSA)_.*'),
- re.compile(r'.*\bMBEDTLS_GENPRIME\b.*')
- ],
- 'test_suite_platform': [
- # Incompatible with sanitizers (e.g. ASan). If the driver
- # component uses a sanitizer but the reference component
- # doesn't, we have a PASS vs SKIP mismatch.
- 'Check mbedtls_calloc overallocation',
- ],
- # Following tests depend on RSA_C but are not about
- # them really, just need to know some error code is there.
- 'test_suite_error': [
- 'Low and high error',
- 'Single high error'
- ],
- # Constant time operations only used for PKCS1_V15
- 'test_suite_constant_time': [
- re.compile(r'mbedtls_ct_zeroize_if .*'),
- re.compile(r'mbedtls_ct_memmove_left .*')
- ],
- 'test_suite_psa_crypto': [
- # We don't support generate_key_custom entry points
- # in drivers yet.
- re.compile(r'PSA generate key custom: RSA, e=.*'),
- re.compile(r'PSA generate key ext: RSA, e=.*'),
- ],
- }
- }
- },
- 'analyze_block_cipher_dispatch': {
- 'test_function': do_analyze_driver_vs_reference,
- 'args': {
- 'component_ref': 'test_full_block_cipher_legacy_dispatch',
- 'component_driver': 'test_full_block_cipher_psa_dispatch',
- 'ignored_suites': [
- # Skipped in the accelerated component
- 'aes', 'aria', 'camellia',
- # These require AES_C, ARIA_C or CAMELLIA_C to be enabled in
- # order for the cipher module (actually cipher_wrapper) to work
- # properly. However these symbols are disabled in the accelerated
- # component so we ignore them.
- 'cipher.ccm', 'cipher.gcm', 'cipher.aes', 'cipher.aria',
- 'cipher.camellia',
- ],
- 'ignored_tests': {
- 'test_suite_config': [
- re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA)_.*'),
- re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'),
- ],
- 'test_suite_cmac': [
- # Following tests require AES_C/ARIA_C/CAMELLIA_C to be enabled,
- # but these are not available in the accelerated component.
- 'CMAC null arguments',
- re.compile('CMAC.* (AES|ARIA|Camellia).*'),
- ],
- 'test_suite_cipher.padding': [
- # Following tests require AES_C/CAMELLIA_C to be enabled,
- # but these are not available in the accelerated component.
- re.compile('Set( non-existent)? padding with (AES|CAMELLIA).*'),
- ],
- 'test_suite_pkcs5': [
- # The AES part of PKCS#5 PBES2 is not yet supported.
- # The rest of PKCS#5 (PBKDF2) works, though.
- re.compile(r'PBES2 .* AES-.*')
- ],
- 'test_suite_pkparse': [
- # PEM (called by pkparse) requires AES_C in order to decrypt
- # the key, but this is not available in the accelerated
- # component.
- re.compile('Parse RSA Key.*(password|AES-).*'),
- ],
- 'test_suite_pem': [
- # Following tests require AES_C, but this is diabled in the
- # accelerated component.
- re.compile('PEM read .*AES.*'),
- 'PEM read (unknown encryption algorithm)',
- ],
- 'test_suite_error': [
- # Following tests depend on AES_C but are not about them
- # really, just need to know some error code is there.
- 'Single low error',
- 'Low and high error',
- ],
- 'test_suite_version': [
- # Similar to test_suite_error above.
- 'Check for MBEDTLS_AES_C when already present',
- ],
- 'test_suite_platform': [
- # Incompatible with sanitizers (e.g. ASan). If the driver
- # component uses a sanitizer but the reference component
- # doesn't, we have a PASS vs SKIP mismatch.
- 'Check mbedtls_calloc overallocation',
- ],
- }
- }
- }
+ 'analyze_coverage': CoverageTask,
+ 'analyze_driver_vs_reference_hash': DriverVSReference_hash,
+ 'analyze_driver_vs_reference_hmac': DriverVSReference_hmac,
+ 'analyze_driver_vs_reference_cipher_aead_cmac': DriverVSReference_cipher_aead_cmac,
+ 'analyze_driver_vs_reference_ecp_light_only': DriverVSReference_ecp_light_only,
+ 'analyze_driver_vs_reference_no_ecp_at_all': DriverVSReference_no_ecp_at_all,
+ 'analyze_driver_vs_reference_ecc_no_bignum': DriverVSReference_ecc_no_bignum,
+ 'analyze_driver_vs_reference_ecc_ffdh_no_bignum': DriverVSReference_ecc_ffdh_no_bignum,
+ 'analyze_driver_vs_reference_ffdh_alg': DriverVSReference_ffdh_alg,
+ 'analyze_driver_vs_reference_tfm_config': DriverVSReference_tfm_config,
+ 'analyze_driver_vs_reference_rsa': DriverVSReference_rsa,
+ 'analyze_block_cipher_dispatch': DriverVSReference_block_cipher_dispatch,
}
+
def main():
main_results = Results()
@@ -754,8 +804,6 @@
sys.stderr.write('invalid task: {}\n'.format(task))
sys.exit(2)
- KNOWN_TASKS['analyze_coverage']['args']['full_coverage'] = options.full_coverage
-
# If the outcome file exists, parse it once and share the result
# among tasks to improve performance.
# Otherwise, it will be generated by execute_reference_driver_tests.
@@ -766,21 +814,21 @@
task_name = tasks_list[0]
task = KNOWN_TASKS[task_name]
- if task['test_function'] != do_analyze_driver_vs_reference: # pylint: disable=comparison-with-callable
+ if not issubclass(task, DriverVSReference):
sys.stderr.write("please provide valid outcomes file for {}.\n".format(task_name))
sys.exit(2)
-
execute_reference_driver_tests(main_results,
- task['args']['component_ref'],
- task['args']['component_driver'],
+ task.REFERENCE,
+ task.DRIVER,
options.outcomes)
outcomes = read_outcome_file(options.outcomes)
- for task in tasks_list:
- test_function = KNOWN_TASKS[task]['test_function']
- test_args = KNOWN_TASKS[task]['args']
- test_function(main_results, outcomes, test_args)
+ for task_name in tasks_list:
+ task_constructor = KNOWN_TASKS[task_name]
+ task = task_constructor(options)
+ main_results.new_section(task.section_name())
+ task.run(main_results, outcomes)
main_results.info("Overall results: {} warnings and {} errors",
main_results.warning_count, main_results.error_count)