blob: 82cc1b1db87e9f12c318c34eacb880586623ccee [file] [log] [blame]
Bence Szépkúti80b31c52021-10-19 15:05:36 +02001#!/usr/bin/env python3
Shaun Case8b0ecbc2021-12-20 21:14:10 -08002"""Run the PSA Crypto API compliance test suite.
Bence Szépkúti449781f2021-11-02 13:41:14 +01003Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF,
David Horstmann3b8984a2023-08-29 10:32:26 +01004then compile and run the test suite. The clone is stored at <repository root>/psa-arch-tests.
Ronald Cron070e8652023-10-09 10:25:45 +02005Known defects in either the test suite or mbedtls / TF-PSA-Crypto - identified by their test
David Horstmann3b8984a2023-08-29 10:32:26 +01006number - are ignored, while unexpected failures AND successes are reported as errors, to help
7keep the list of known defects as up to date as possible.
Bence Szépkúti449781f2021-11-02 13:41:14 +01008"""
Bence Szépkúti67fb3142021-11-02 14:01:08 +01009
10# Copyright The Mbed TLS Contributors
Dave Rodgman16799db2023-11-02 19:47:20 +000011# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
Bence Szépkúti67fb3142021-11-02 14:01:08 +010012
David Horstmann4dcddcf2023-08-17 18:08:24 +010013import argparse
Bence Szépkúti80b31c52021-10-19 15:05:36 +020014import os
15import re
16import shutil
17import subprocess
18import sys
David Horstmann9cc6b2f2023-08-29 17:36:35 +010019from typing import List
Bence Szépkúti80b31c52021-10-19 15:05:36 +020020
David Horstmanne31014a2023-07-19 11:43:27 +010021#pylint: disable=unused-import
David Horstmann1d091842023-07-18 17:39:35 +010022import scripts_path
23from mbedtls_dev import build_tree
24
Ronald Cron070e8652023-10-09 10:25:45 +020025# PSA Compliance tests we expect to fail due to known defects in Mbed TLS /
26# TF-PSA-Crypto (or the test suite).
Bence Szépkúticb288712021-11-09 21:30:43 +010027# The test numbers correspond to the numbers used by the console output of the test suite.
28# Test number 2xx corresponds to the files in the folder
29# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx
Bence Szépkúti80b31c52021-10-19 15:05:36 +020030EXPECTED_FAILURES = {
Bence Szépkúticb288712021-11-09 21:30:43 +010031 # psa_hash_suspend() and psa_hash_resume() are not supported.
32 # - Tracked in issue #3274
33 262, 263
Bence Szépkúti80b31c52021-10-19 15:05:36 +020034}
Bence Szépkútie2855c32021-11-09 17:33:57 +010035
36# We currently use a fork of ARM-software/psa-arch-tests, with a couple of downstream patches
Thomas Daubney540324c2023-10-06 17:07:24 +010037# that allow it to build with Mbed TLS 3, and fixes a couple of issues in the compliance test suite.
Bence Szépkútie2855c32021-11-09 17:33:57 +010038# These fixes allow the tests numbered 216, 248 and 249 to complete successfully.
39#
40# Once all the fixes are upstreamed, this fork should be replaced with an upstream commit/tag.
Bence Szépkútib376eac2021-11-09 22:13:46 +010041# - Tracked in issue #5145
Bence Szépkútie2855c32021-11-09 17:33:57 +010042#
43# Web URL: https://github.com/bensze01/psa-arch-tests/tree/fixes-for-mbedtls-3
44PSA_ARCH_TESTS_REPO = 'https://github.com/bensze01/psa-arch-tests.git'
Gilles Peskine42ed9632022-05-17 17:23:09 +020045PSA_ARCH_TESTS_REF = 'fix-pr-5736'
Bence Szépkúti80b31c52021-10-19 15:05:36 +020046
David Horstmanne31014a2023-07-19 11:43:27 +010047#pylint: disable=too-many-branches,too-many-statements,too-many-locals
David Horstmann4dcddcf2023-08-17 18:08:24 +010048def main(library_build_dir: str):
David Horstmannf7570692023-08-29 10:27:13 +010049 root_dir = os.getcwd()
Bence Szépkúti80b31c52021-10-19 15:05:36 +020050
Ronald Cron070e8652023-10-09 10:25:45 +020051 in_tf_psa_crypto_repo = build_tree.looks_like_tf_psa_crypto_root(root_dir)
David Horstmann1d091842023-07-18 17:39:35 +010052
Thomas Daubney89324042023-11-23 10:14:12 +000053 crypto_name = build_tree.project_crypto_name()
Thomas Daubneyd0c30762023-11-23 09:59:57 +000054
55 library_subdir = build_tree.crypto_core_directory()
David Horstmannbeaee262023-08-29 13:56:17 +010056
57 crypto_lib_filename = (library_build_dir + '/' +
58 library_subdir + '/' +
59 'lib' + crypto_name + '.a')
David Horstmann98af1982023-08-29 10:25:26 +010060
61 if not os.path.exists(crypto_lib_filename):
David Horstmann2ba89be2023-08-29 10:37:29 +010062 #pylint: disable=bad-continuation
David Horstmann4dcddcf2023-08-17 18:08:24 +010063 subprocess.check_call([
64 'cmake', '.',
65 '-GUnix Makefiles',
David Horstmann41c316d2023-08-29 14:57:23 +010066 '-B' + library_build_dir
David Horstmann4dcddcf2023-08-17 18:08:24 +010067 ])
David Horstmannbeaee262023-08-29 13:56:17 +010068 subprocess.check_call(['cmake', '--build', library_build_dir,
David Horstmann8f3ec8e2023-08-30 09:46:20 +010069 '--target', crypto_name])
Bence Szépkúti80b31c52021-10-19 15:05:36 +020070
71 psa_arch_tests_dir = 'psa-arch-tests'
Bence Szépkútic63d1602021-11-02 14:06:40 +010072 os.makedirs(psa_arch_tests_dir, exist_ok=True)
Bence Szépkúti80b31c52021-10-19 15:05:36 +020073 try:
Bence Szépkúti34b5f562021-11-02 13:48:39 +010074 os.chdir(psa_arch_tests_dir)
Bence Szépkúti80b31c52021-10-19 15:05:36 +020075
Bence Szépkútib3818412021-11-03 11:32:51 +010076 # Reuse existing local clone
Bence Szépkúti34b5f562021-11-02 13:48:39 +010077 subprocess.check_call(['git', 'init'])
78 subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF])
79 subprocess.check_call(['git', 'checkout', 'FETCH_HEAD'])
Bence Szépkúti80b31c52021-10-19 15:05:36 +020080
Bence Szépkúti34b5f562021-11-02 13:48:39 +010081 build_dir = 'api-tests/build'
82 try:
83 shutil.rmtree(build_dir)
84 except FileNotFoundError:
85 pass
86 os.mkdir(build_dir)
87 os.chdir(build_dir)
Bence Szépkúti80b31c52021-10-19 15:05:36 +020088
David Horstmannf7570692023-08-29 10:27:13 +010089 extra_includes = (';{}/drivers/builtin/include'.format(root_dir)
Ronald Cron070e8652023-10-09 10:25:45 +020090 if in_tf_psa_crypto_repo else '')
David Horstmann1d091842023-07-18 17:39:35 +010091
Bence Szépkúti34b5f562021-11-02 13:48:39 +010092 #pylint: disable=bad-continuation
93 subprocess.check_call([
94 'cmake', '..',
95 '-GUnix Makefiles',
96 '-DTARGET=tgt_dev_apis_stdc',
97 '-DTOOLCHAIN=HOST_GCC',
98 '-DSUITE=CRYPTO',
David Horstmannf7570692023-08-29 10:27:13 +010099 '-DPSA_CRYPTO_LIB_FILENAME={}/{}'.format(root_dir,
David Horstmann7f93d222023-08-23 16:21:40 +0100100 crypto_lib_filename),
David Horstmannf7570692023-08-29 10:27:13 +0100101 ('-DPSA_INCLUDE_PATHS={}/include' + extra_includes).format(root_dir)
Bence Szépkúti34b5f562021-11-02 13:48:39 +0100102 ])
103 subprocess.check_call(['cmake', '--build', '.'])
Bence Szépkúti80b31c52021-10-19 15:05:36 +0200104
Bence Szépkúti34b5f562021-11-02 13:48:39 +0100105 proc = subprocess.Popen(['./psa-arch-tests-crypto'],
106 bufsize=1, stdout=subprocess.PIPE, universal_newlines=True)
107
108 test_re = re.compile(
109 '^TEST: (?P<test_num>[0-9]*)|'
110 '^TEST RESULT: (?P<test_result>FAILED|PASSED)'
111 )
112 test = -1
113 unexpected_successes = set(EXPECTED_FAILURES)
David Horstmannfd9264e2023-08-29 16:21:15 +0100114 expected_failures = [] # type: List[int]
115 unexpected_failures = [] # type: List[int]
116 if proc.stdout is None:
117 return 1
118
Bence Szépkúti34b5f562021-11-02 13:48:39 +0100119 for line in proc.stdout:
120 print(line, end='')
121 match = test_re.match(line)
122 if match is not None:
123 groupdict = match.groupdict()
124 test_num = groupdict['test_num']
125 if test_num is not None:
126 test = int(test_num)
127 elif groupdict['test_result'] == 'FAILED':
128 try:
129 unexpected_successes.remove(test)
130 expected_failures.append(test)
131 print('Expected failure, ignoring')
132 except KeyError:
133 unexpected_failures.append(test)
134 print('ERROR: Unexpected failure')
135 elif test in unexpected_successes:
136 print('ERROR: Unexpected success')
137 proc.wait()
138
139 print()
140 print('***** test_psa_compliance.py report ******')
141 print()
142 print('Expected failures:', ', '.join(str(i) for i in expected_failures))
143 print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures))
144 print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes)))
145 print()
146 if unexpected_successes or unexpected_failures:
147 if unexpected_successes:
148 print('Unexpected successes encountered.')
149 print('Please remove the corresponding tests from '
150 'EXPECTED_FAILURES in tests/scripts/compliance_test.py')
151 print()
152 print('FAILED')
153 return 1
154 else:
Bence Szépkúti34b5f562021-11-02 13:48:39 +0100155 print('SUCCESS')
156 return 0
157 finally:
David Horstmannf7570692023-08-29 10:27:13 +0100158 os.chdir(root_dir)
Bence Szépkúti80b31c52021-10-19 15:05:36 +0200159
160if __name__ == '__main__':
David Horstmannb48822c2023-08-29 14:12:53 +0100161 BUILD_DIR = 'out_of_source_build'
David Horstmann4dcddcf2023-08-17 18:08:24 +0100162
David Horstmann3ed18712023-08-29 18:20:01 +0100163 # pylint: disable=invalid-name
David Horstmann4dcddcf2023-08-17 18:08:24 +0100164 parser = argparse.ArgumentParser()
165 parser.add_argument('--build-dir', nargs=1,
Ronald Cron070e8652023-10-09 10:25:45 +0200166 help='path to Mbed TLS / TF-PSA-Crypto build directory')
David Horstmann4dcddcf2023-08-17 18:08:24 +0100167 args = parser.parse_args()
168
169 if args.build_dir is not None:
David Horstmannb48822c2023-08-29 14:12:53 +0100170 BUILD_DIR = args.build_dir[0]
David Horstmann4dcddcf2023-08-17 18:08:24 +0100171
David Horstmannb48822c2023-08-29 14:12:53 +0100172 sys.exit(main(BUILD_DIR))