Move files out of Mbed TLS
The following files are moved to the framework repo (deleted here):
tests/scripts/test_psa_compliance.py
tests/scripts/test_psa_constant_names.py
Signed-off-by: Valerio Setti <valerio.setti@nordicsemi.no>
diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py
deleted file mode 100755
index 7c09afc..0000000
--- a/tests/scripts/test_psa_compliance.py
+++ /dev/null
@@ -1,172 +0,0 @@
-#!/usr/bin/env python3
-"""Run the PSA Crypto API compliance test suite.
-Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF,
-then compile and run the test suite. The clone is stored at <repository root>/psa-arch-tests.
-Known defects in either the test suite or mbedtls / TF-PSA-Crypto - identified by their test
-number - are ignored, while unexpected failures AND successes are reported as errors, to help
-keep the list of known defects as up to date as possible.
-"""
-
-# Copyright The Mbed TLS Contributors
-# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
-
-import argparse
-import os
-import re
-import shutil
-import subprocess
-import sys
-from typing import List
-
-#pylint: disable=unused-import
-import scripts_path
-from mbedtls_framework import build_tree
-
-# PSA Compliance tests we expect to fail due to known defects in Mbed TLS /
-# TF-PSA-Crypto (or the test suite).
-# The test numbers correspond to the numbers used by the console output of the test suite.
-# Test number 2xx corresponds to the files in the folder
-# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx
-EXPECTED_FAILURES = {} # type: dict
-
-PSA_ARCH_TESTS_REPO = 'https://github.com/ARM-software/psa-arch-tests.git'
-PSA_ARCH_TESTS_REF = 'v23.06_API1.5_ADAC_EAC'
-
-#pylint: disable=too-many-branches,too-many-statements,too-many-locals
-def main(library_build_dir: str):
- root_dir = os.getcwd()
-
- in_tf_psa_crypto_repo = build_tree.looks_like_tf_psa_crypto_root(root_dir)
-
- crypto_name = build_tree.crypto_library_filename(root_dir)
-
- # Temporary, while the crypto library is still located in the library
- # directory. This will not be the case anymore when it will be built by
- # the TF-PSA-Crypto build system.
- if in_tf_psa_crypto_repo:
- library_subdir = build_tree.crypto_core_directory(root_dir, relative=True)
- else:
- library_subdir = 'library'
-
- crypto_lib_filename = (library_build_dir + '/' +
- library_subdir + '/' +
- 'lib' + crypto_name + '.a')
-
- if not os.path.exists(crypto_lib_filename):
- #pylint: disable=bad-continuation
- subprocess.check_call([
- 'cmake', '.',
- '-GUnix Makefiles',
- '-B' + library_build_dir
- ])
- subprocess.check_call(['cmake', '--build', library_build_dir,
- '--target', crypto_name])
-
- psa_arch_tests_dir = 'psa-arch-tests'
- os.makedirs(psa_arch_tests_dir, exist_ok=True)
- try:
- os.chdir(psa_arch_tests_dir)
-
- # Reuse existing local clone
- subprocess.check_call(['git', 'init'])
- subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF])
- subprocess.check_call(['git', 'checkout', 'FETCH_HEAD'])
-
- build_dir = 'api-tests/build'
- try:
- shutil.rmtree(build_dir)
- except FileNotFoundError:
- pass
- os.mkdir(build_dir)
- os.chdir(build_dir)
-
- # Temporary while the PSA compliance test suite is still run as part
- # of Mbed TLS testing. When it is not the case anymore, the second case
- # can be removed.
- if in_tf_psa_crypto_repo:
- extra_includes = ';{}/drivers/builtin/include'.format(root_dir)
- elif os.path.isdir(os.path.join(root_dir, 'tf-psa-crypto')):
- extra_includes = ';{}/tf-psa-crypto/include'.format(root_dir) + \
- (';{}/tf-psa-crypto/drivers/builtin/include'.format(root_dir))
-
- #pylint: disable=bad-continuation
- subprocess.check_call([
- 'cmake', '..',
- '-GUnix Makefiles',
- '-DTARGET=tgt_dev_apis_stdc',
- '-DTOOLCHAIN=HOST_GCC',
- '-DSUITE=CRYPTO',
- '-DPSA_CRYPTO_LIB_FILENAME={}/{}'.format(root_dir,
- crypto_lib_filename),
- ('-DPSA_INCLUDE_PATHS={}/include' + extra_includes).format(root_dir)
- ])
- subprocess.check_call(['cmake', '--build', '.'])
-
- proc = subprocess.Popen(['./psa-arch-tests-crypto'],
- bufsize=1, stdout=subprocess.PIPE, universal_newlines=True)
-
- test_re = re.compile(
- '^TEST: (?P<test_num>[0-9]*)|'
- '^TEST RESULT: (?P<test_result>FAILED|PASSED)'
- )
- test = -1
- unexpected_successes = set(EXPECTED_FAILURES)
- expected_failures = [] # type: List[int]
- unexpected_failures = [] # type: List[int]
- if proc.stdout is None:
- return 1
-
- for line in proc.stdout:
- print(line, end='')
- match = test_re.match(line)
- if match is not None:
- groupdict = match.groupdict()
- test_num = groupdict['test_num']
- if test_num is not None:
- test = int(test_num)
- elif groupdict['test_result'] == 'FAILED':
- try:
- unexpected_successes.remove(test)
- expected_failures.append(test)
- print('Expected failure, ignoring')
- except KeyError:
- unexpected_failures.append(test)
- print('ERROR: Unexpected failure')
- elif test in unexpected_successes:
- print('ERROR: Unexpected success')
- proc.wait()
-
- print()
- print('***** test_psa_compliance.py report ******')
- print()
- print('Expected failures:', ', '.join(str(i) for i in expected_failures))
- print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures))
- print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes)))
- print()
- if unexpected_successes or unexpected_failures:
- if unexpected_successes:
- print('Unexpected successes encountered.')
- print('Please remove the corresponding tests from '
- 'EXPECTED_FAILURES in tests/scripts/compliance_test.py')
- print()
- print('FAILED')
- return 1
- else:
- print('SUCCESS')
- return 0
- finally:
- os.chdir(root_dir)
-
-if __name__ == '__main__':
- BUILD_DIR = 'out_of_source_build'
-
- # pylint: disable=invalid-name
- parser = argparse.ArgumentParser()
- parser.add_argument('--build-dir', nargs=1,
- help='path to Mbed TLS / TF-PSA-Crypto build directory')
- args = parser.parse_args()
-
- if args.build_dir is not None:
- BUILD_DIR = args.build_dir[0]
-
- sys.exit(main(BUILD_DIR))
diff --git a/tests/scripts/test_psa_constant_names.py b/tests/scripts/test_psa_constant_names.py
deleted file mode 100755
index e697d17..0000000
--- a/tests/scripts/test_psa_constant_names.py
+++ /dev/null
@@ -1,194 +0,0 @@
-#!/usr/bin/env python3
-"""Test the program psa_constant_names.
-Gather constant names from header files and test cases. Compile a C program
-to print out their numerical values, feed these numerical values to
-psa_constant_names, and check that the output is the original name.
-Return 0 if all test cases pass, 1 if the output was not always as expected,
-or 1 (with a Python backtrace) if there was an operational error.
-"""
-
-# Copyright The Mbed TLS Contributors
-# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
-
-import argparse
-from collections import namedtuple
-import os
-import re
-import subprocess
-import sys
-from typing import Iterable, List, Optional, Tuple
-
-import scripts_path # pylint: disable=unused-import
-from mbedtls_framework import c_build_helper
-from mbedtls_framework.macro_collector import InputsForTest, PSAMacroEnumerator
-from mbedtls_framework import typing_util
-
-def gather_inputs(headers: Iterable[str],
- test_suites: Iterable[str],
- inputs_class=InputsForTest) -> PSAMacroEnumerator:
- """Read the list of inputs to test psa_constant_names with."""
- inputs = inputs_class()
- for header in headers:
- inputs.parse_header(header)
- for test_cases in test_suites:
- inputs.parse_test_cases(test_cases)
- inputs.add_numerical_values()
- inputs.gather_arguments()
- return inputs
-
-def run_c(type_word: str,
- expressions: Iterable[str],
- include_path: Optional[str] = None,
- keep_c: bool = False) -> List[str]:
- """Generate and run a program to print out numerical values of C expressions."""
- if type_word == 'status':
- cast_to = 'long'
- printf_format = '%ld'
- else:
- cast_to = 'unsigned long'
- printf_format = '0x%08lx'
- return c_build_helper.get_c_expression_values(
- cast_to, printf_format,
- expressions,
- caller='test_psa_constant_names.py for {} values'.format(type_word),
- file_label=type_word,
- header='#include <psa/crypto.h>',
- include_path=include_path,
- keep_c=keep_c
- )
-
-NORMALIZE_STRIP_RE = re.compile(r'\s+')
-def normalize(expr: str) -> str:
- """Normalize the C expression so as not to care about trivial differences.
-
- Currently "trivial differences" means whitespace.
- """
- return re.sub(NORMALIZE_STRIP_RE, '', expr)
-
-ALG_TRUNCATED_TO_SELF_RE = \
- re.compile(r'PSA_ALG_AEAD_WITH_SHORTENED_TAG\('
- r'PSA_ALG_(?:CCM|CHACHA20_POLY1305|GCM)'
- r', *16\)\Z')
-
-def is_simplifiable(expr: str) -> bool:
- """Determine whether an expression is simplifiable.
-
- Simplifiable expressions can't be output in their input form, since
- the output will be the simple form. Therefore they must be excluded
- from testing.
- """
- if ALG_TRUNCATED_TO_SELF_RE.match(expr):
- return True
- return False
-
-def collect_values(inputs: InputsForTest,
- type_word: str,
- include_path: Optional[str] = None,
- keep_c: bool = False) -> Tuple[List[str], List[str]]:
- """Generate expressions using known macro names and calculate their values.
-
- Return a list of pairs of (expr, value) where expr is an expression and
- value is a string representation of its integer value.
- """
- names = inputs.get_names(type_word)
- expressions = sorted(expr
- for expr in inputs.generate_expressions(names)
- if not is_simplifiable(expr))
- values = run_c(type_word, expressions,
- include_path=include_path, keep_c=keep_c)
- return expressions, values
-
-class Tests:
- """An object representing tests and their results."""
-
- Error = namedtuple('Error',
- ['type', 'expression', 'value', 'output'])
-
- def __init__(self, options) -> None:
- self.options = options
- self.count = 0
- self.errors = [] #type: List[Tests.Error]
-
- def run_one(self, inputs: InputsForTest, type_word: str) -> None:
- """Test psa_constant_names for the specified type.
-
- Run the program on the names for this type.
- Use the inputs to figure out what arguments to pass to macros that
- take arguments.
- """
- expressions, values = collect_values(inputs, type_word,
- include_path=self.options.include,
- keep_c=self.options.keep_c)
- output_bytes = subprocess.check_output([self.options.program,
- type_word] + values)
- output = output_bytes.decode('ascii')
- outputs = output.strip().split('\n')
- self.count += len(expressions)
- for expr, value, output in zip(expressions, values, outputs):
- if self.options.show:
- sys.stdout.write('{} {}\t{}\n'.format(type_word, value, output))
- if normalize(expr) != normalize(output):
- self.errors.append(self.Error(type=type_word,
- expression=expr,
- value=value,
- output=output))
-
- def run_all(self, inputs: InputsForTest) -> None:
- """Run psa_constant_names on all the gathered inputs."""
- for type_word in ['status', 'algorithm', 'ecc_curve', 'dh_group',
- 'key_type', 'key_usage']:
- self.run_one(inputs, type_word)
-
- def report(self, out: typing_util.Writable) -> None:
- """Describe each case where the output is not as expected.
-
- Write the errors to ``out``.
- Also write a total.
- """
- for error in self.errors:
- out.write('For {} "{}", got "{}" (value: {})\n'
- .format(error.type, error.expression,
- error.output, error.value))
- out.write('{} test cases'.format(self.count))
- if self.errors:
- out.write(', {} FAIL\n'.format(len(self.errors)))
- else:
- out.write(' PASS\n')
-
-HEADERS = ['psa/crypto.h', 'psa/crypto_extra.h', 'psa/crypto_values.h']
-TEST_SUITES = ['tf-psa-crypto/tests/suites/test_suite_psa_crypto_metadata.data']
-
-def main():
- parser = argparse.ArgumentParser(description=globals()['__doc__'])
- parser.add_argument('--include', '-I',
- action='append', default=['tf-psa-crypto/include',
- 'tf-psa-crypto/drivers/builtin/include',
- 'tf-psa-crypto/drivers/everest/include',
- 'include'],
- help='Directory for header files')
- parser.add_argument('--keep-c',
- action='store_true', dest='keep_c', default=False,
- help='Keep the intermediate C file')
- parser.add_argument('--no-keep-c',
- action='store_false', dest='keep_c',
- help='Don\'t keep the intermediate C file (default)')
- parser.add_argument('--program',
- default='tf-psa-crypto/programs/psa/psa_constant_names',
- help='Program to test')
- parser.add_argument('--show',
- action='store_true',
- help='Show tested values on stdout')
- parser.add_argument('--no-show',
- action='store_false', dest='show',
- help='Don\'t show tested values (default)')
- options = parser.parse_args()
- headers = [os.path.join(options.include[0], h) for h in HEADERS]
- inputs = gather_inputs(headers, TEST_SUITES)
- tests = Tests(options)
- tests.run_all(inputs)
- tests.report(sys.stdout)
- if tests.errors:
- sys.exit(1)
-
-if __name__ == '__main__':
- main()