tests/scripts/analyze_outcomes.py

#!/usr/bin/env python3

"""Analyze the test outcomes from a full CI run.

This script can also run on outcomes from a partial run, but the results are
less likely to be useful.
"""

import argparse
import sys
import traceback
import re
import subprocess
import os
import typing

import check_test_cases


# `ComponentOutcomes` is a named tuple which is defined as:
# ComponentOutcomes(
#     successes = {
#         "<suite_case>",
#         ...
#     },
#     failures = {
#         "<suite_case>",
#         ...
#     }
# )
# suite_case = "<suite>;<case>"
ComponentOutcomes = typing.NamedTuple('ComponentOutcomes',
                                      [('successes', typing.Set[str]),
                                       ('failures', typing.Set[str])])

# `Outcomes` is a representation of the outcomes file,
# which defined as:
# Outcomes = {
#     "<component>": ComponentOutcomes,
#     ...
# }
Outcomes = typing.Dict[str, ComponentOutcomes]


class Results:
    """Process analysis results."""

    def __init__(self):
        self.error_count = 0
        self.warning_count = 0

    def new_section(self, fmt, *args, **kwargs):
        self._print_line('\n*** ' + fmt + ' ***\n', *args, **kwargs)

    def info(self, fmt, *args, **kwargs):
        self._print_line('Info: ' + fmt, *args, **kwargs)

    def error(self, fmt, *args, **kwargs):
        self.error_count += 1
        self._print_line('Error: ' + fmt, *args, **kwargs)

    def warning(self, fmt, *args, **kwargs):
        self.warning_count += 1
        self._print_line('Warning: ' + fmt, *args, **kwargs)

    @staticmethod
    def _print_line(fmt, *args, **kwargs):
        sys.stderr.write((fmt + '\n').format(*args, **kwargs))

def execute_reference_driver_tests(results: Results, ref_component: str, driver_component: str, \
                                   outcome_file: str) -> None:
    """Run the tests specified in ref_component and driver_component. Results
    are stored in the output_file and they will be used for the following
    coverage analysis"""
    results.new_section("Test {} and {}", ref_component, driver_component)

    shell_command = "tests/scripts/all.sh --outcome-file " + outcome_file + \
                    " " + ref_component + " " + driver_component
    results.info("Running: {}", shell_command)
    ret_val = subprocess.run(shell_command.split(), check=False).returncode

    if ret_val != 0:
        results.error("failed to run reference/driver components")

def analyze_coverage(results: Results, outcomes: Outcomes,
                     allow_list: typing.List[str], full_coverage: bool) -> None:
    """Check that all available test cases are executed at least once."""
    available = check_test_cases.collect_available_test_cases()
    for suite_case in available:
        hit = any(suite_case in comp_outcomes.successes or
                  suite_case in comp_outcomes.failures
                  for comp_outcomes in outcomes.values())

        if not hit and suite_case not in allow_list:
            if full_coverage:
                results.error('Test case not executed: {}', suite_case)
            else:
                results.warning('Test case not executed: {}', suite_case)
        elif hit and suite_case in allow_list:
            # Test Case should be removed from the allow list.
            if full_coverage:
                results.error('Allow listed test case was executed: {}', suite_case)
            else:
                results.warning('Allow listed test case was executed: {}', suite_case)

def name_matches_pattern(name: str, str_or_re) -> bool:
    """Check if name matches a pattern, that may be a string or regex.
    - If the pattern is a string, name must be equal to match.
    - If the pattern is a regex, name must fully match.
    """
    # The CI's python is too old for re.Pattern
    #if isinstance(str_or_re, re.Pattern):
    if not isinstance(str_or_re, str):
        return str_or_re.fullmatch(name) is not None
    else:
        return str_or_re == name

def analyze_driver_vs_reference(results: Results, outcomes: Outcomes,
                                component_ref: str, component_driver: str,
                                ignored_suites: typing.List[str], ignored_tests=None) -> None:
    """Check that all tests passing in the reference component are also
    passing in the corresponding driver component.
    Skip:
    - full test suites provided in ignored_suites list
    - only some specific test inside a test suite, for which the corresponding
      output string is provided
    """
    ref_outcomes = outcomes.get("component_" + component_ref)
    driver_outcomes = outcomes.get("component_" + component_driver)

    if ref_outcomes is None or driver_outcomes is None:
        results.error("required components are missing: bad outcome file?")
        return

    if not ref_outcomes.successes:
        results.error("no passing test in reference component: bad outcome file?")
        return

    for suite_case in ref_outcomes.successes:
        # suite_case is like "test_suite_foo.bar;Description of test case"
        (full_test_suite, test_string) = suite_case.split(';')
        test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name

        # Immediately skip fully-ignored test suites
        if test_suite in ignored_suites or full_test_suite in ignored_suites:
            continue

        # For ignored test cases inside test suites, just remember and:
        # don't issue an error if they're skipped with drivers,
        # but issue an error if they're not (means we have a bad entry).
        ignored = False
        if full_test_suite in ignored_tests:
            for str_or_re in ignored_tests[test_suite]:
                if name_matches_pattern(test_string, str_or_re):
                    ignored = True

        if not ignored and not suite_case in driver_outcomes.successes:
            results.error("PASS -> SKIP/FAIL: {}", suite_case)
        if ignored and suite_case in driver_outcomes.successes:
            results.error("uselessly ignored: {}", suite_case)

def analyze_outcomes(results: Results, outcomes: Outcomes, args) -> None:
    """Run all analyses on the given outcome collection."""
    analyze_coverage(results, outcomes, args['allow_list'],
                     args['full_coverage'])

def read_outcome_file(outcome_file: str) -> Outcomes:
    """Parse an outcome file and return an outcome collection.
    """
    outcomes = {}
    with open(outcome_file, 'r', encoding='utf-8') as input_file:
        for line in input_file:
            (_platform, component, suite, case, result, _cause) = line.split(';')
            # Note that `component` is not unique. If a test case passes on Linux
            # and fails on FreeBSD, it'll end up in both the successes set and
            # the failures set.
            suite_case = ';'.join([suite, case])
            if component not in outcomes:
                outcomes[component] = ComponentOutcomes(set(), set())
            if result == 'PASS':
                outcomes[component].successes.add(suite_case)
            elif result == 'FAIL':
                outcomes[component].failures.add(suite_case)

    return outcomes

def do_analyze_coverage(results: Results, outcomes: Outcomes, args) -> None:
    """Perform coverage analysis."""
    results.new_section("Analyze coverage")
    analyze_outcomes(results, outcomes, args)

def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -> None:
    """Perform driver vs reference analyze."""
    results.new_section("Analyze driver {} vs reference {}",
                        args['component_driver'], args['component_ref'])

    ignored_suites = ['test_suite_' + x for x in args['ignored_suites']]

    analyze_driver_vs_reference(results, outcomes,
                                args['component_ref'], args['component_driver'],
                                ignored_suites, args['ignored_tests'])

# List of tasks with a function that can handle this task and additional arguments if required
KNOWN_TASKS = {
    'analyze_coverage':                 {
        'test_function': do_analyze_coverage,
        'args': {
            'allow_list': [
                # Algorithm not supported yet
                'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA',
                # Algorithm not supported yet
                'test_suite_psa_crypto_metadata;Cipher: XTS',
            ],
            'full_coverage': False,
        }
    },
    # There are 2 options to use analyze_driver_vs_reference_xxx locally:
    # 1. Run tests and then analysis:
    #   - tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver>
    #   - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
    # 2. Let this script run both automatically:
    #   - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
    'analyze_driver_vs_reference_hash': {
        'test_function': do_analyze_driver_vs_reference,
        'args': {
            'component_ref': 'test_psa_crypto_config_reference_hash_use_psa',
            'component_driver': 'test_psa_crypto_config_accel_hash_use_psa',
            'ignored_suites': [
                'shax', 'mdx', # the software implementations that are being excluded
                'md.psa',  # purposefully depends on whether drivers are present
                'psa_crypto_low_hash.generated', # testing the builtins
            ],
            'ignored_tests': {
                'test_suite_platform': [
                    # Incompatible with sanitizers (e.g. ASan). If the driver
                    # component uses a sanitizer but the reference component
                    # doesn't, we have a PASS vs SKIP mismatch.
                    'Check mbedtls_calloc overallocation',
                ],
            }
        }
    },
    'analyze_driver_vs_reference_cipher_aead': {
        'test_function': do_analyze_driver_vs_reference,
        'args': {
            'component_ref': 'test_psa_crypto_config_reference_cipher_aead',
            'component_driver': 'test_psa_crypto_config_accel_cipher_aead',
            # Modules replaced by drivers.
            'ignored_suites': [
                # low-level (block/stream) cipher modules
                'aes', 'aria', 'camellia', 'des', 'chacha20',
                # AEAD modes
                'ccm', 'chachapoly', 'cmac', 'gcm',
                # The Cipher abstraction layer
                'cipher',
            ],
            'ignored_tests': {
                # PEM decryption is not supported so far.
                # The rest of PEM (write, unencrypted read) works though.
                'test_suite_pem': [
                    re.compile(r'PEM read .*(AES|DES|\bencrypt).*'),
                ],
                'test_suite_platform': [
                    # Incompatible with sanitizers (e.g. ASan). If the driver
                    # component uses a sanitizer but the reference component
                    # doesn't, we have a PASS vs SKIP mismatch.
                    'Check mbedtls_calloc overallocation',
                ],
                # Following tests depend on AES_C/DES_C but are not about
                # them really, just need to know some error code is there.
                'test_suite_error': [
                    'Low and high error',
                    'Single low error'
                ],
                # Similar to test_suite_error above.
                'test_suite_version': [
                    'Check for MBEDTLS_AES_C when already present',
                ],
                # The en/decryption part of PKCS#12 is not supported so far.
                # The rest of PKCS#12 (key derivation) works though.
                'test_suite_pkcs12': [
                    re.compile(r'PBE Encrypt, .*'),
                    re.compile(r'PBE Decrypt, .*'),
                ],
                # The en/decryption part of PKCS#5 is not supported so far.
                # The rest of PKCS#5 (PBKDF2) works though.
                'test_suite_pkcs5': [
                    re.compile(r'PBES2 Encrypt, .*'),
                    re.compile(r'PBES2 Decrypt .*'),
                ],
                # Encrypted keys are not supported so far.
                # pylint: disable=line-too-long
                'test_suite_pkparse': [
                    'Key ASN1 (Encrypted key PKCS12, trailing garbage data)',
                    'Key ASN1 (Encrypted key PKCS5, trailing garbage data)',
                    re.compile(r'Parse (RSA|EC) Key .*\(.* ([Ee]ncrypted|password).*\)'),
                ],
            }
        }
    },
    'analyze_driver_vs_reference_ecp_light_only': {
        'test_function': do_analyze_driver_vs_reference,
        'args': {
            'component_ref': 'test_psa_crypto_config_reference_ecc_ecp_light_only',
            'component_driver': 'test_psa_crypto_config_accel_ecc_ecp_light_only',
            'ignored_suites': [
                # Modules replaced by drivers
                'ecdsa', 'ecdh', 'ecjpake',
            ],
            'ignored_tests': {
                'test_suite_platform': [
                    # Incompatible with sanitizers (e.g. ASan). If the driver
                    # component uses a sanitizer but the reference component
                    # doesn't, we have a PASS vs SKIP mismatch.
                    'Check mbedtls_calloc overallocation',
                ],
                # This test wants a legacy function that takes f_rng, p_rng
                # arguments, and uses legacy ECDSA for that. The test is
                # really about the wrapper around the PSA RNG, not ECDSA.
                'test_suite_random': [
                    'PSA classic wrapper: ECDSA signature (SECP256R1)',
                ],
                # In the accelerated test ECP_C is not set (only ECP_LIGHT is)
                # so we must ignore disparities in the tests for which ECP_C
                # is required.
                'test_suite_ecp': [
                    re.compile(r'ECP check public-private .*'),
                    re.compile(r'ECP gen keypair .*'),
                    re.compile(r'ECP point muladd .*'),
                    re.compile(r'ECP point multiplication .*'),
                    re.compile(r'ECP test vectors .*'),
                ],
                'test_suite_ssl': [
                    # This deprecated function is only present when ECP_C is On.
                    'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
                ],
            }
        }
    },
    'analyze_driver_vs_reference_no_ecp_at_all': {
        'test_function': do_analyze_driver_vs_reference,
        'args': {
            'component_ref': 'test_psa_crypto_config_reference_ecc_no_ecp_at_all',
            'component_driver': 'test_psa_crypto_config_accel_ecc_no_ecp_at_all',
            'ignored_suites': [
                # Modules replaced by drivers
                'ecp', 'ecdsa', 'ecdh', 'ecjpake',
            ],
            'ignored_tests': {
                'test_suite_platform': [
                    # Incompatible with sanitizers (e.g. ASan). If the driver
                    # component uses a sanitizer but the reference component
                    # doesn't, we have a PASS vs SKIP mismatch.
                    'Check mbedtls_calloc overallocation',
                ],
                # See ecp_light_only
                'test_suite_random': [
                    'PSA classic wrapper: ECDSA signature (SECP256R1)',
                ],
                'test_suite_pkparse': [
                    # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED
                    # is automatically enabled in build_info.h (backward compatibility)
                    # even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a
                    # consequence compressed points are supported in the reference
                    # component but not in the accelerated one, so they should be skipped
                    # while checking driver's coverage.
                    re.compile(r'Parse EC Key .*compressed\)'),
                    re.compile(r'Parse Public EC Key .*compressed\)'),
                ],
                # See ecp_light_only
                'test_suite_ssl': [
                    'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
                ],
            }
        }
    },
    'analyze_driver_vs_reference_ecc_no_bignum': {
        'test_function': do_analyze_driver_vs_reference,
        'args': {
            'component_ref': 'test_psa_crypto_config_reference_ecc_no_bignum',
            'component_driver': 'test_psa_crypto_config_accel_ecc_no_bignum',
            'ignored_suites': [
                # Modules replaced by drivers
                'ecp', 'ecdsa', 'ecdh', 'ecjpake',
                'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
                'bignum.generated', 'bignum.misc',
            ],
            'ignored_tests': {
                'test_suite_platform': [
                    # Incompatible with sanitizers (e.g. ASan). If the driver
                    # component uses a sanitizer but the reference component
                    # doesn't, we have a PASS vs SKIP mismatch.
                    'Check mbedtls_calloc overallocation',
                ],
                # See ecp_light_only
                'test_suite_random': [
                    'PSA classic wrapper: ECDSA signature (SECP256R1)',
                ],
                # See no_ecp_at_all
                'test_suite_pkparse': [
                    re.compile(r'Parse EC Key .*compressed\)'),
                    re.compile(r'Parse Public EC Key .*compressed\)'),
                ],
                'test_suite_asn1parse': [
                    'INTEGER too large for mpi',
                ],
                'test_suite_asn1write': [
                    re.compile(r'ASN.1 Write mpi.*'),
                ],
                'test_suite_debug': [
                    re.compile(r'Debug print mbedtls_mpi.*'),
                ],
                # See ecp_light_only
                'test_suite_ssl': [
                    'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
                ],
            }
        }
    },
    'analyze_driver_vs_reference_ecc_ffdh_no_bignum': {
        'test_function': do_analyze_driver_vs_reference,
        'args': {
            'component_ref': 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum',
            'component_driver': 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum',
            'ignored_suites': [
                # Modules replaced by drivers
                'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm',
                'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
                'bignum.generated', 'bignum.misc',
            ],
            'ignored_tests': {
                'test_suite_platform': [
                    # Incompatible with sanitizers (e.g. ASan). If the driver
                    # component uses a sanitizer but the reference component
                    # doesn't, we have a PASS vs SKIP mismatch.
                    'Check mbedtls_calloc overallocation',
                ],
                # See ecp_light_only
                'test_suite_random': [
                    'PSA classic wrapper: ECDSA signature (SECP256R1)',
                ],
                # See no_ecp_at_all
                'test_suite_pkparse': [
                    re.compile(r'Parse EC Key .*compressed\)'),
                    re.compile(r'Parse Public EC Key .*compressed\)'),
                ],
                'test_suite_asn1parse': [
                    'INTEGER too large for mpi',
                ],
                'test_suite_asn1write': [
                    re.compile(r'ASN.1 Write mpi.*'),
                ],
                'test_suite_debug': [
                    re.compile(r'Debug print mbedtls_mpi.*'),
                ],
                # See ecp_light_only
                'test_suite_ssl': [
                    'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
                ],
            }
        }
    },
    'analyze_driver_vs_reference_ffdh_alg': {
        'test_function': do_analyze_driver_vs_reference,
        'args': {
            'component_ref': 'test_psa_crypto_config_reference_ffdh',
            'component_driver': 'test_psa_crypto_config_accel_ffdh',
            'ignored_suites': ['dhm'],
            'ignored_tests': {
                'test_suite_platform': [
                    # Incompatible with sanitizers (e.g. ASan). If the driver
                    # component uses a sanitizer but the reference component
                    # doesn't, we have a PASS vs SKIP mismatch.
                    'Check mbedtls_calloc overallocation',
                ],
            }
        }
    },
    'analyze_driver_vs_reference_tfm_config': {
        'test_function':  do_analyze_driver_vs_reference,
        'args': {
            'component_ref': 'test_tfm_config',
            'component_driver': 'test_tfm_config_p256m_driver_accel_ec',
            'ignored_suites': [
                # Modules replaced by drivers
                'asn1parse', 'asn1write',
                'ecp', 'ecdsa', 'ecdh', 'ecjpake',
                'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
                'bignum.generated', 'bignum.misc',
            ],
            'ignored_tests': {
                'test_suite_platform': [
                    # Incompatible with sanitizers (e.g. ASan). If the driver
                    # component uses a sanitizer but the reference component
                    # doesn't, we have a PASS vs SKIP mismatch.
                    'Check mbedtls_calloc overallocation',
                ],
                # See ecp_light_only
                'test_suite_random': [
                    'PSA classic wrapper: ECDSA signature (SECP256R1)',
                ],
            }
        }
    },
    'analyze_driver_vs_reference_rsa': {
        'test_function': do_analyze_driver_vs_reference,
        'args': {
            'component_ref': 'test_psa_crypto_config_reference_rsa_crypto',
            'component_driver': 'test_psa_crypto_config_accel_rsa_crypto',
            'ignored_suites': [
                # Modules replaced by drivers.
                'rsa', 'pkcs1_v15', 'pkcs1_v21',
                # We temporarily don't care about PK stuff.
                'pk', 'pkwrite', 'pkparse'
            ],
            'ignored_tests': {
                'test_suite_platform': [
                    # Incompatible with sanitizers (e.g. ASan). If the driver
                    # component uses a sanitizer but the reference component
                    # doesn't, we have a PASS vs SKIP mismatch.
                    'Check mbedtls_calloc overallocation',
                ],
                # Following tests depend on RSA_C but are not about
                # them really, just need to know some error code is there.
                'test_suite_error': [
                    'Low and high error',
                    'Single high error'
                ],
                # Constant time operations only used for PKCS1_V15
                'test_suite_constant_time': [
                    re.compile(r'mbedtls_ct_zeroize_if .*'),
                    re.compile(r'mbedtls_ct_memmove_left .*')
                ],
            }
        }
    },
    'analyze_block_cipher_dispatch': {
        'test_function': do_analyze_driver_vs_reference,
        'args': {
            'component_ref': 'test_full_common_reference',
            'component_driver': 'test_full_block_cipher_psa_dispatch',
            'ignored_suites': [
            ],
            'ignored_tests': {
                'test_suite_platform': [
                    # Incompatible with sanitizers (e.g. ASan). If the driver
                    # component uses a sanitizer but the reference component
                    # doesn't, we have a PASS vs SKIP mismatch.
                    'Check mbedtls_calloc overallocation',
                ],
            }
        }
    }
}

def main():
    main_results = Results()

    try:
        parser = argparse.ArgumentParser(description=__doc__)
        parser.add_argument('outcomes', metavar='OUTCOMES.CSV',
                            help='Outcome file to analyze')
        parser.add_argument('specified_tasks', default='all', nargs='?',
                            help='Analysis to be done. By default, run all tasks. '
                                 'With one or more TASK, run only those. '
                                 'TASK can be the name of a single task or '
                                 'comma/space-separated list of tasks. ')
        parser.add_argument('--list', action='store_true',
                            help='List all available tasks and exit.')
        parser.add_argument('--require-full-coverage', action='store_true',
                            dest='full_coverage', help="Require all available "
                            "test cases to be executed and issue an error "
                            "otherwise. This flag is ignored if 'task' is "
                            "neither 'all' nor 'analyze_coverage'")
        options = parser.parse_args()

        if options.list:
            for task in KNOWN_TASKS:
                print(task)
            sys.exit(0)

        if options.specified_tasks == 'all':
            tasks_list = KNOWN_TASKS.keys()
        else:
            tasks_list = re.split(r'[, ]+', options.specified_tasks)
            for task in tasks_list:
                if task not in KNOWN_TASKS:
                    sys.stderr.write('invalid task: {}\n'.format(task))
                    sys.exit(2)

        KNOWN_TASKS['analyze_coverage']['args']['full_coverage'] = options.full_coverage

        # If the outcome file exists, parse it once and share the result
        # among tasks to improve performance.
        # Otherwise, it will be generated by execute_reference_driver_tests.
        if not os.path.exists(options.outcomes):
            if len(tasks_list) > 1:
                sys.stderr.write("mutiple tasks found, please provide a valid outcomes file.\n")
                sys.exit(2)

            task_name = tasks_list[0]
            task = KNOWN_TASKS[task_name]
            if task['test_function'] != do_analyze_driver_vs_reference: # pylint: disable=comparison-with-callable
                sys.stderr.write("please provide valid outcomes file for {}.\n".format(task_name))
                sys.exit(2)

            execute_reference_driver_tests(main_results,
                                           task['args']['component_ref'],
                                           task['args']['component_driver'],
                                           options.outcomes)

        outcomes = read_outcome_file(options.outcomes)

        for task in tasks_list:
            test_function = KNOWN_TASKS[task]['test_function']
            test_args = KNOWN_TASKS[task]['args']
            test_function(main_results, outcomes, test_args)

        main_results.info("Overall results: {} warnings and {} errors",
                          main_results.warning_count, main_results.error_count)

        sys.exit(0 if (main_results.error_count == 0) else 1)

    except Exception: # pylint: disable=broad-except
        # Print the backtrace and exit explicitly with our chosen status.
        traceback.print_exc()
        sys.exit(120)

if __name__ == '__main__':
    main()