blob: 90d28df1deaaaaba41955297c506821164c1543c [file] [log] [blame]
Gilles Peskine15c2cbf2020-06-25 18:36:28 +02001#!/usr/bin/env python3
2
3"""Analyze the test outcomes from a full CI run.
4
5This script can also run on outcomes from a partial run, but the results are
6less likely to be useful.
7"""
8
Przemek Stekiel85c54ea2022-11-17 11:50:23 +01009import re
Gilles Peskine2a71fac2024-09-17 15:07:22 +020010import typing
Gilles Peskine15c2cbf2020-06-25 18:36:28 +020011
Gilles Peskine31467722024-10-03 18:52:58 +020012import scripts_path # pylint: disable=unused-import
13from mbedtls_framework import outcome_analysis
Gilles Peskine8d3c70a2020-06-25 18:37:43 +020014
Pengyu Lvc2e8f3a2023-11-28 17:22:04 +080015
Gilles Peskine082eade2024-10-03 18:42:37 +020016class CoverageTask(outcome_analysis.CoverageTask):
Gilles Peskine095561c2024-10-04 16:24:26 +020017 """Justify test cases that are never executed."""
Gilles Peskine3f5022e2024-09-16 20:23:40 +020018
Gilles Peskine2a71fac2024-09-17 15:07:22 +020019 @staticmethod
Gilles Peskine5872c0d2024-09-17 17:15:29 +020020 def _has_word_re(words: typing.Iterable[str],
21 exclude: typing.Optional[str] = None) -> typing.Pattern:
Gilles Peskine2a71fac2024-09-17 15:07:22 +020022 """Construct a regex that matches if any of the words appears.
23
24 The occurrence must start and end at a word boundary.
Gilles Peskine5872c0d2024-09-17 17:15:29 +020025
26 If exclude is specified, strings containing a match for that
27 regular expression will not match the returned pattern.
Gilles Peskine2a71fac2024-09-17 15:07:22 +020028 """
Gilles Peskine5872c0d2024-09-17 17:15:29 +020029 exclude_clause = r''
30 if exclude:
31 exclude_clause = r'(?!.*' + exclude + ')'
32 return re.compile(exclude_clause +
33 r'.*\b(?:' + r'|'.join(words) + r')\b.*',
Gilles Peskine5e3ed3f2024-10-11 12:00:44 +020034 re.DOTALL)
Gilles Peskine2a71fac2024-09-17 15:07:22 +020035
36 # generate_psa_tests.py generates test cases involving cryptographic
37 # mechanisms (key types, families, algorithms) that are declared but
38 # not implemented. Until we improve the Python scripts, ignore those
39 # test cases in the analysis.
40 # https://github.com/Mbed-TLS/mbedtls/issues/9572
41 _PSA_MECHANISMS_NOT_IMPLEMENTED = [
42 r'CBC_MAC',
43 r'DETERMINISTIC_DSA',
44 r'DET_DSA',
45 r'DSA',
46 r'ECC_KEY_PAIR\(BRAINPOOL_P_R1\) (?:160|192|224|320)-bit',
47 r'ECC_KEY_PAIR\(SECP_K1\) 225-bit',
48 r'ECC_PAIR\(BP_R1\) (?:160|192|224|320)-bit',
49 r'ECC_PAIR\(SECP_K1\) 225-bit',
50 r'ECC_PUBLIC_KEY\(BRAINPOOL_P_R1\) (?:160|192|224|320)-bit',
51 r'ECC_PUBLIC_KEY\(SECP_K1\) 225-bit',
52 r'ECC_PUB\(BP_R1\) (?:160|192|224|320)-bit',
53 r'ECC_PUB\(SECP_K1\) 225-bit',
54 r'ED25519PH',
55 r'ED448PH',
56 r'PEPPER',
57 r'PURE_EDDSA',
58 r'SECP_R2',
59 r'SECT_K1',
60 r'SECT_R1',
61 r'SECT_R2',
62 r'SHAKE256_512',
63 r'SHA_512_224',
64 r'SHA_512_256',
65 r'TWISTED_EDWARDS',
66 r'XTS',
67 ]
68 PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE = \
69 _has_word_re(_PSA_MECHANISMS_NOT_IMPLEMENTED)
70
71 IGNORED_TESTS = {
Gilles Peskinede2316b2024-09-17 18:32:05 +020072 'ssl-opt': [
73 # We don't run ssl-opt.sh with Valgrind on the CI because
74 # it's extremely slow. We don't intend to change this.
75 'DTLS client reconnect from same port: reconnect, nbio, valgrind',
Gilles Peskinede2316b2024-09-17 18:32:05 +020076 # We don't have IPv6 in our CI environment.
77 # https://github.com/Mbed-TLS/mbedtls-test/issues/176
78 'DTLS cookie: enabled, IPv6',
79 # Disabled due to OpenSSL bug.
80 # https://github.com/openssl/openssl/issues/18887
81 'DTLS fragmenting: 3d, openssl client, DTLS 1.2',
82 # We don't run ssl-opt.sh with Valgrind on the CI because
83 # it's extremely slow. We don't intend to change this.
84 'DTLS fragmenting: proxy MTU: auto-reduction (with valgrind)',
Gilles Peskine24b03d82024-10-04 16:22:24 +020085 # TLS doesn't use restartable ECDH yet.
86 # https://github.com/Mbed-TLS/mbedtls/issues/7294
87 re.compile(r'EC restart:.*no USE_PSA.*'),
Gilles Peskinede2316b2024-09-17 18:32:05 +020088 ],
Gilles Peskine2fd25bb2024-09-17 19:46:18 +020089 'test_suite_config.mbedtls_boolean': [
Gilles Peskine2fd25bb2024-09-17 19:46:18 +020090 # Missing coverage of test configurations.
91 # https://github.com/Mbed-TLS/mbedtls/issues/9585
92 'Config: !MBEDTLS_SSL_DTLS_ANTI_REPLAY',
93 # Missing coverage of test configurations.
94 # https://github.com/Mbed-TLS/mbedtls/issues/9585
95 'Config: !MBEDTLS_SSL_DTLS_HELLO_VERIFY',
96 # We don't run test_suite_config when we test this.
97 # https://github.com/Mbed-TLS/mbedtls/issues/9586
98 'Config: !MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ENABLED',
Gilles Peskine2fd25bb2024-09-17 19:46:18 +020099 ],
100 'test_suite_config.psa_boolean': [
101 # We don't test with HMAC disabled.
102 # https://github.com/Mbed-TLS/mbedtls/issues/9591
103 'Config: !PSA_WANT_ALG_HMAC',
Gilles Peskine2fd25bb2024-09-17 19:46:18 +0200104 # The DERIVE key type is always enabled.
105 'Config: !PSA_WANT_KEY_TYPE_DERIVE',
106 # More granularity of key pair type enablement macros
107 # than we care to test.
108 # https://github.com/Mbed-TLS/mbedtls/issues/9590
109 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_EXPORT',
110 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_GENERATE',
111 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_IMPORT',
112 # More granularity of key pair type enablement macros
113 # than we care to test.
114 # https://github.com/Mbed-TLS/mbedtls/issues/9590
115 'Config: !PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT',
116 'Config: !PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT',
117 # We don't test with HMAC disabled.
118 # https://github.com/Mbed-TLS/mbedtls/issues/9591
119 'Config: !PSA_WANT_KEY_TYPE_HMAC',
120 # The PASSWORD key type is always enabled.
121 'Config: !PSA_WANT_KEY_TYPE_PASSWORD',
122 # The PASSWORD_HASH key type is always enabled.
123 'Config: !PSA_WANT_KEY_TYPE_PASSWORD_HASH',
124 # The RAW_DATA key type is always enabled.
125 'Config: !PSA_WANT_KEY_TYPE_RAW_DATA',
126 # More granularity of key pair type enablement macros
127 # than we care to test.
128 # https://github.com/Mbed-TLS/mbedtls/issues/9590
129 'Config: !PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_EXPORT',
130 'Config: !PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_IMPORT',
131 # Algorithm declared but not supported.
132 'Config: PSA_WANT_ALG_CBC_MAC',
133 # Algorithm declared but not supported.
134 'Config: PSA_WANT_ALG_XTS',
Gilles Peskine2fd25bb2024-09-17 19:46:18 +0200135 # More granularity of key pair type enablement macros
136 # than we care to test.
137 # https://github.com/Mbed-TLS/mbedtls/issues/9590
138 'Config: PSA_WANT_KEY_TYPE_DH_KEY_PAIR_DERIVE',
139 'Config: PSA_WANT_KEY_TYPE_ECC_KEY_PAIR',
140 'Config: PSA_WANT_KEY_TYPE_RSA_KEY_PAIR',
141 'Config: PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_DERIVE',
Minos Galanakis848333d2024-12-02 15:58:32 +0000142 # https://github.com/Mbed-TLS/mbedtls/issues/9583
143 'Config: !MBEDTLS_ECP_NIST_OPTIM',
144 # We never test without the PSA client code. Should we?
145 # https://github.com/Mbed-TLS/TF-PSA-Crypto/issues/112
146 'Config: !MBEDTLS_PSA_CRYPTO_CLIENT',
147 # We only test multithreading with pthreads.
148 # https://github.com/Mbed-TLS/mbedtls/issues/9584
149 'Config: !MBEDTLS_THREADING_PTHREAD',
150 # Built but not tested.
151 # https://github.com/Mbed-TLS/mbedtls/issues/9587
152 'Config: MBEDTLS_AES_USE_HARDWARE_ONLY',
153 # Untested platform-specific optimizations.
154 # https://github.com/Mbed-TLS/mbedtls/issues/9588
155 'Config: MBEDTLS_HAVE_SSE2',
156 # Obsolete configuration option, to be replaced by
157 # PSA entropy drivers.
158 # https://github.com/Mbed-TLS/mbedtls/issues/8150
159 'Config: MBEDTLS_NO_PLATFORM_ENTROPY',
160 # Untested aspect of the platform interface.
161 # https://github.com/Mbed-TLS/mbedtls/issues/9589
162 'Config: MBEDTLS_PLATFORM_NO_STD_FUNCTIONS',
163 # In a client-server build, test_suite_config runs in the
164 # client configuration, so it will never report
165 # MBEDTLS_PSA_CRYPTO_SPM as enabled. That's ok.
166 'Config: MBEDTLS_PSA_CRYPTO_SPM',
167 # We don't test on armv8 yet.
168 'Config: MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT',
169 'Config: MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY',
170 'Config: MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY',
171 'Config: MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY',
172 # We don't run test_suite_config when we test this.
173 # https://github.com/Mbed-TLS/mbedtls/issues/9586
174 'Config: MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND',
Gilles Peskine2fd25bb2024-09-17 19:46:18 +0200175 ],
176 'test_suite_config.psa_combinations': [
177 # We don't test this unusual, but sensible configuration.
178 # https://github.com/Mbed-TLS/mbedtls/issues/9592
179 'Config: PSA_WANT_ALG_DETERMINSTIC_ECDSA without PSA_WANT_ALG_ECDSA',
180 ],
Gilles Peskineb0ec85d2024-09-17 18:33:29 +0200181 'test_suite_pkcs12': [
Gilles Peskine2fd25bb2024-09-17 19:46:18 +0200182 # We never test with CBC/PKCS5/PKCS12 enabled but
183 # PKCS7 padding disabled.
Gilles Peskineb0ec85d2024-09-17 18:33:29 +0200184 # https://github.com/Mbed-TLS/mbedtls/issues/9580
185 'PBE Decrypt, (Invalid padding & PKCS7 padding disabled)',
186 'PBE Encrypt, pad = 8 (PKCS7 padding disabled)',
187 ],
188 'test_suite_pkcs5': [
Gilles Peskine2fd25bb2024-09-17 19:46:18 +0200189 # We never test with CBC/PKCS5/PKCS12 enabled but
190 # PKCS7 padding disabled.
Gilles Peskineb0ec85d2024-09-17 18:33:29 +0200191 # https://github.com/Mbed-TLS/mbedtls/issues/9580
192 'PBES2 Decrypt (Invalid padding & PKCS7 padding disabled)',
193 'PBES2 Encrypt, pad=6 (PKCS7 padding disabled)',
194 'PBES2 Encrypt, pad=8 (PKCS7 padding disabled)',
195 ],
Gilles Peskineeafc2752024-04-19 19:08:34 +0200196 'test_suite_psa_crypto': [
197 # We don't test this unusual, but sensible configuration.
198 # https://github.com/Mbed-TLS/mbedtls/issues/9592
199 re.compile(r'.*ECDSA.*only deterministic supported'),
200 ],
Gilles Peskine2a71fac2024-09-17 15:07:22 +0200201 'test_suite_psa_crypto_generate_key.generated': [
Gilles Peskine5872c0d2024-09-17 17:15:29 +0200202 # Ignore mechanisms that are not implemented, except
203 # for public keys for which we always test that
204 # psa_generate_key() returns PSA_ERROR_INVALID_ARGUMENT
205 # regardless of whether the specific key type is supported.
206 _has_word_re((mech
207 for mech in _PSA_MECHANISMS_NOT_IMPLEMENTED
208 if not mech.startswith('ECC_PUB')),
209 exclude=r'ECC_PUB'),
Gilles Peskine2a71fac2024-09-17 15:07:22 +0200210 ],
Gilles Peskineb0ec85d2024-09-17 18:33:29 +0200211 'test_suite_psa_crypto_metadata': [
212 # Algorithms declared but not supported.
213 # https://github.com/Mbed-TLS/mbedtls/issues/9579
214 'Asymmetric signature: Ed25519ph',
215 'Asymmetric signature: Ed448ph',
216 'Asymmetric signature: pure EdDSA',
217 'Cipher: XTS',
218 'MAC: CBC_MAC-3DES',
219 'MAC: CBC_MAC-AES-128',
220 'MAC: CBC_MAC-AES-192',
221 'MAC: CBC_MAC-AES-256',
222 ],
Gilles Peskine2a71fac2024-09-17 15:07:22 +0200223 'test_suite_psa_crypto_not_supported.generated': [
Gilles Peskineab5cc9b2024-09-17 17:57:11 +0200224 # It is a bug that not-supported test cases aren't getting
225 # run for never-implemented key types.
226 # https://github.com/Mbed-TLS/mbedtls/issues/7915
Gilles Peskine2a71fac2024-09-17 15:07:22 +0200227 PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE,
Gilles Peskine5e3ed3f2024-10-11 12:00:44 +0200228 # We never test with DH key support disabled but support
Gilles Peskineab5cc9b2024-09-17 17:57:11 +0200229 # for a DH group enabled. The dependencies of these test
230 # cases don't really make sense.
231 # https://github.com/Mbed-TLS/mbedtls/issues/9574
232 re.compile(r'PSA \w+ DH_.*type not supported'),
233 # We only test partial support for DH with the 2048-bit group
234 # enabled and the other groups disabled.
235 # https://github.com/Mbed-TLS/mbedtls/issues/9575
236 'PSA generate DH_KEY_PAIR(RFC7919) 2048-bit group not supported',
237 'PSA import DH_KEY_PAIR(RFC7919) 2048-bit group not supported',
238 'PSA import DH_PUBLIC_KEY(RFC7919) 2048-bit group not supported',
Gilles Peskine2a71fac2024-09-17 15:07:22 +0200239 ],
240 'test_suite_psa_crypto_op_fail.generated': [
Gilles Peskine5872c0d2024-09-17 17:15:29 +0200241 # Ignore mechanisms that are not implemented, except
242 # for test cases that assume the mechanism is not supported.
243 _has_word_re(_PSA_MECHANISMS_NOT_IMPLEMENTED,
244 exclude=(r'.*: !(?:' +
245 r'|'.join(_PSA_MECHANISMS_NOT_IMPLEMENTED) +
246 r')\b')),
Gilles Peskineab5cc9b2024-09-17 17:57:11 +0200247 # Incorrect dependency generation. To be fixed as part of the
248 # resolution of https://github.com/Mbed-TLS/mbedtls/issues/9167
249 # by forward-porting the commit
250 # "PSA test case generation: dependency inference class: operation fail"
251 # from https://github.com/Mbed-TLS/mbedtls/pull/9025 .
252 re.compile(r'.* with (?:DH|ECC)_(?:KEY_PAIR|PUBLIC_KEY)\(.*'),
Gilles Peskineeafc2752024-04-19 19:08:34 +0200253 # We don't test this unusual, but sensible configuration.
254 # https://github.com/Mbed-TLS/mbedtls/issues/9592
255 re.compile(r'.*: !ECDSA but DETERMINISTIC_ECDSA with ECC_.*'),
Gilles Peskineab5cc9b2024-09-17 17:57:11 +0200256 # We never test with the HMAC algorithm enabled but the HMAC
257 # key type disabled. Those dependencies don't really make sense.
258 # https://github.com/Mbed-TLS/mbedtls/issues/9573
259 re.compile(r'.* !HMAC with HMAC'),
Gilles Peskine2a71fac2024-09-17 15:07:22 +0200260 ],
Gilles Peskineeafc2752024-04-19 19:08:34 +0200261 'test_suite_psa_crypto_op_fail.misc': [
262 # We don't test this unusual, but sensible configuration.
263 # https://github.com/Mbed-TLS/mbedtls/issues/9592
264 'PSA sign DETERMINISTIC_ECDSA(SHA_256): !ECDSA but DETERMINISTIC_ECDSA with ECC_KEY_PAIR(SECP_R1)', #pylint: disable=line-too-long
265 ],
Gilles Peskine2a71fac2024-09-17 15:07:22 +0200266 'test_suite_psa_crypto_storage_format.current': [
267 PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE,
268 ],
269 'test_suite_psa_crypto_storage_format.v0': [
270 PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE,
271 ],
Gilles Peskinede2316b2024-09-17 18:32:05 +0200272 'tls13-misc': [
273 # Disabled due to OpenSSL bug.
274 # https://github.com/openssl/openssl/issues/10714
275 'TLS 1.3 O->m: resumption',
276 # Disabled due to OpenSSL command line limitation.
277 # https://github.com/Mbed-TLS/mbedtls/issues/9582
278 'TLS 1.3 m->O: resumption with early data',
279 ],
Gilles Peskine2a71fac2024-09-17 15:07:22 +0200280 }
281
Gilles Peskine82b16722024-09-16 19:57:10 +0200282
Gilles Peskine9df375b2024-09-16 20:14:26 +0200283# The names that we give to classes derived from DriverVSReference do not
284# follow the usual naming convention, because it's more readable to use
285# underscores and parts of the configuration names. Also, these classes
286# are just there to specify some data, so they don't need repetitive
287# documentation.
288#pylint: disable=invalid-name,missing-class-docstring
289
Gilles Peskine082eade2024-10-03 18:42:37 +0200290class DriverVSReference_hash(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200291 REFERENCE = 'test_psa_crypto_config_reference_hash_use_psa'
292 DRIVER = 'test_psa_crypto_config_accel_hash_use_psa'
293 IGNORED_SUITES = [
294 'shax', 'mdx', # the software implementations that are being excluded
295 'md.psa', # purposefully depends on whether drivers are present
296 'psa_crypto_low_hash.generated', # testing the builtins
297 ]
298 IGNORED_TESTS = {
299 'test_suite_config': [
300 re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'),
301 ],
302 'test_suite_platform': [
303 # Incompatible with sanitizers (e.g. ASan). If the driver
304 # component uses a sanitizer but the reference component
305 # doesn't, we have a PASS vs SKIP mismatch.
306 'Check mbedtls_calloc overallocation',
307 ],
308 }
309
Gilles Peskine082eade2024-10-03 18:42:37 +0200310class DriverVSReference_hmac(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200311 REFERENCE = 'test_psa_crypto_config_reference_hmac'
312 DRIVER = 'test_psa_crypto_config_accel_hmac'
313 IGNORED_SUITES = [
314 # These suites require legacy hash support, which is disabled
315 # in the accelerated component.
316 'shax', 'mdx',
317 # This suite tests builtins directly, but these are missing
318 # in the accelerated case.
319 'psa_crypto_low_hash.generated',
320 ]
321 IGNORED_TESTS = {
322 'test_suite_config': [
323 re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'),
324 re.compile(r'.*\bMBEDTLS_MD_C\b')
325 ],
326 'test_suite_md': [
327 # Builtin HMAC is not supported in the accelerate component.
328 re.compile('.*HMAC.*'),
329 # Following tests make use of functions which are not available
330 # when MD_C is disabled, as it happens in the accelerated
331 # test component.
332 re.compile('generic .* Hash file .*'),
333 'MD list',
334 ],
335 'test_suite_md.psa': [
336 # "legacy only" tests require hash algorithms to be NOT
337 # accelerated, but this of course false for the accelerated
338 # test component.
339 re.compile('PSA dispatch .* legacy only'),
340 ],
341 'test_suite_platform': [
342 # Incompatible with sanitizers (e.g. ASan). If the driver
343 # component uses a sanitizer but the reference component
344 # doesn't, we have a PASS vs SKIP mismatch.
345 'Check mbedtls_calloc overallocation',
346 ],
347 }
348
Gilles Peskine082eade2024-10-03 18:42:37 +0200349class DriverVSReference_cipher_aead_cmac(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200350 REFERENCE = 'test_psa_crypto_config_reference_cipher_aead_cmac'
351 DRIVER = 'test_psa_crypto_config_accel_cipher_aead_cmac'
352 # Modules replaced by drivers.
353 IGNORED_SUITES = [
354 # low-level (block/stream) cipher modules
355 'aes', 'aria', 'camellia', 'des', 'chacha20',
356 # AEAD modes and CMAC
357 'ccm', 'chachapoly', 'cmac', 'gcm',
358 # The Cipher abstraction layer
359 'cipher',
360 ]
361 IGNORED_TESTS = {
362 'test_suite_config': [
363 re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA|CHACHA20|DES)_.*'),
364 re.compile(r'.*\bMBEDTLS_(CCM|CHACHAPOLY|CMAC|GCM)_.*'),
365 re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'),
366 re.compile(r'.*\bMBEDTLS_CIPHER_.*'),
367 ],
368 # PEM decryption is not supported so far.
369 # The rest of PEM (write, unencrypted read) works though.
370 'test_suite_pem': [
371 re.compile(r'PEM read .*(AES|DES|\bencrypt).*'),
372 ],
373 'test_suite_platform': [
374 # Incompatible with sanitizers (e.g. ASan). If the driver
375 # component uses a sanitizer but the reference component
376 # doesn't, we have a PASS vs SKIP mismatch.
377 'Check mbedtls_calloc overallocation',
378 ],
379 # Following tests depend on AES_C/DES_C but are not about
380 # them really, just need to know some error code is there.
381 'test_suite_error': [
382 'Low and high error',
383 'Single low error'
384 ],
385 # Similar to test_suite_error above.
386 'test_suite_version': [
387 'Check for MBEDTLS_AES_C when already present',
388 ],
389 # The en/decryption part of PKCS#12 is not supported so far.
390 # The rest of PKCS#12 (key derivation) works though.
391 'test_suite_pkcs12': [
392 re.compile(r'PBE Encrypt, .*'),
393 re.compile(r'PBE Decrypt, .*'),
394 ],
395 # The en/decryption part of PKCS#5 is not supported so far.
396 # The rest of PKCS#5 (PBKDF2) works though.
397 'test_suite_pkcs5': [
398 re.compile(r'PBES2 Encrypt, .*'),
399 re.compile(r'PBES2 Decrypt .*'),
400 ],
401 # Encrypted keys are not supported so far.
402 # pylint: disable=line-too-long
403 'test_suite_pkparse': [
404 'Key ASN1 (Encrypted key PKCS12, trailing garbage data)',
405 'Key ASN1 (Encrypted key PKCS5, trailing garbage data)',
406 re.compile(r'Parse (RSA|EC) Key .*\(.* ([Ee]ncrypted|password).*\)'),
407 ],
408 # Encrypted keys are not supported so far.
409 'ssl-opt': [
410 'TLS: password protected server key',
411 'TLS: password protected client key',
412 'TLS: password protected server key, two certificates',
413 ],
414 }
415
Gilles Peskine082eade2024-10-03 18:42:37 +0200416class DriverVSReference_ecp_light_only(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200417 REFERENCE = 'test_psa_crypto_config_reference_ecc_ecp_light_only'
418 DRIVER = 'test_psa_crypto_config_accel_ecc_ecp_light_only'
419 IGNORED_SUITES = [
420 # Modules replaced by drivers
421 'ecdsa', 'ecdh', 'ecjpake',
Gilles Peskine77587ce2024-10-29 20:55:11 +0100422 # Unit tests for the built-in implementation
423 'psa_crypto_ecp',
Gilles Peskine9df375b2024-09-16 20:14:26 +0200424 ]
425 IGNORED_TESTS = {
426 'test_suite_config': [
427 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
428 ],
429 'test_suite_platform': [
430 # Incompatible with sanitizers (e.g. ASan). If the driver
431 # component uses a sanitizer but the reference component
432 # doesn't, we have a PASS vs SKIP mismatch.
433 'Check mbedtls_calloc overallocation',
434 ],
435 # This test wants a legacy function that takes f_rng, p_rng
436 # arguments, and uses legacy ECDSA for that. The test is
437 # really about the wrapper around the PSA RNG, not ECDSA.
438 'test_suite_random': [
439 'PSA classic wrapper: ECDSA signature (SECP256R1)',
440 ],
441 # In the accelerated test ECP_C is not set (only ECP_LIGHT is)
442 # so we must ignore disparities in the tests for which ECP_C
443 # is required.
444 'test_suite_ecp': [
445 re.compile(r'ECP check public-private .*'),
446 re.compile(r'ECP calculate public: .*'),
447 re.compile(r'ECP gen keypair .*'),
448 re.compile(r'ECP point muladd .*'),
449 re.compile(r'ECP point multiplication .*'),
450 re.compile(r'ECP test vectors .*'),
451 ],
452 'test_suite_ssl': [
453 # This deprecated function is only present when ECP_C is On.
454 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
455 ],
456 }
457
Gilles Peskine082eade2024-10-03 18:42:37 +0200458class DriverVSReference_no_ecp_at_all(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200459 REFERENCE = 'test_psa_crypto_config_reference_ecc_no_ecp_at_all'
460 DRIVER = 'test_psa_crypto_config_accel_ecc_no_ecp_at_all'
461 IGNORED_SUITES = [
462 # Modules replaced by drivers
463 'ecp', 'ecdsa', 'ecdh', 'ecjpake',
Gilles Peskine77587ce2024-10-29 20:55:11 +0100464 # Unit tests for the built-in implementation
465 'psa_crypto_ecp',
Gilles Peskine9df375b2024-09-16 20:14:26 +0200466 ]
467 IGNORED_TESTS = {
468 'test_suite_config': [
469 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
470 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
471 ],
472 'test_suite_platform': [
473 # Incompatible with sanitizers (e.g. ASan). If the driver
474 # component uses a sanitizer but the reference component
475 # doesn't, we have a PASS vs SKIP mismatch.
476 'Check mbedtls_calloc overallocation',
477 ],
478 # See ecp_light_only
479 'test_suite_random': [
480 'PSA classic wrapper: ECDSA signature (SECP256R1)',
481 ],
482 'test_suite_pkparse': [
483 # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED
484 # is automatically enabled in build_info.h (backward compatibility)
485 # even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a
486 # consequence compressed points are supported in the reference
487 # component but not in the accelerated one, so they should be skipped
488 # while checking driver's coverage.
489 re.compile(r'Parse EC Key .*compressed\)'),
490 re.compile(r'Parse Public EC Key .*compressed\)'),
491 ],
492 # See ecp_light_only
493 'test_suite_ssl': [
494 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
495 ],
496 }
497
Gilles Peskine082eade2024-10-03 18:42:37 +0200498class DriverVSReference_ecc_no_bignum(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200499 REFERENCE = 'test_psa_crypto_config_reference_ecc_no_bignum'
500 DRIVER = 'test_psa_crypto_config_accel_ecc_no_bignum'
501 IGNORED_SUITES = [
502 # Modules replaced by drivers
503 'ecp', 'ecdsa', 'ecdh', 'ecjpake',
504 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
505 'bignum.generated', 'bignum.misc',
Gilles Peskine77587ce2024-10-29 20:55:11 +0100506 # Unit tests for the built-in implementation
507 'psa_crypto_ecp',
Gilles Peskine9df375b2024-09-16 20:14:26 +0200508 ]
509 IGNORED_TESTS = {
510 'test_suite_config': [
511 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
512 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
513 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
514 ],
515 'test_suite_platform': [
516 # Incompatible with sanitizers (e.g. ASan). If the driver
517 # component uses a sanitizer but the reference component
518 # doesn't, we have a PASS vs SKIP mismatch.
519 'Check mbedtls_calloc overallocation',
520 ],
521 # See ecp_light_only
522 'test_suite_random': [
523 'PSA classic wrapper: ECDSA signature (SECP256R1)',
524 ],
525 # See no_ecp_at_all
526 'test_suite_pkparse': [
527 re.compile(r'Parse EC Key .*compressed\)'),
528 re.compile(r'Parse Public EC Key .*compressed\)'),
529 ],
530 'test_suite_asn1parse': [
531 'INTEGER too large for mpi',
532 ],
533 'test_suite_asn1write': [
534 re.compile(r'ASN.1 Write mpi.*'),
535 ],
536 'test_suite_debug': [
537 re.compile(r'Debug print mbedtls_mpi.*'),
538 ],
539 # See ecp_light_only
540 'test_suite_ssl': [
541 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
542 ],
543 }
544
Gilles Peskine082eade2024-10-03 18:42:37 +0200545class DriverVSReference_ecc_ffdh_no_bignum(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200546 REFERENCE = 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum'
547 DRIVER = 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum'
548 IGNORED_SUITES = [
549 # Modules replaced by drivers
550 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm',
551 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
552 'bignum.generated', 'bignum.misc',
Gilles Peskine77587ce2024-10-29 20:55:11 +0100553 # Unit tests for the built-in implementation
554 'psa_crypto_ecp',
Gilles Peskine9df375b2024-09-16 20:14:26 +0200555 ]
556 IGNORED_TESTS = {
557 'ssl-opt': [
558 # DHE support in TLS 1.2 requires built-in MBEDTLS_DHM_C
559 # (because it needs custom groups, which PSA does not
560 # provide), even with MBEDTLS_USE_PSA_CRYPTO.
561 re.compile(r'PSK callback:.*\bdhe-psk\b.*'),
562 ],
563 'test_suite_config': [
564 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
565 re.compile(r'.*\bMBEDTLS_DHM_C\b.*'),
566 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
567 re.compile(r'.*\bMBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED\b.*'),
568 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
569 ],
570 'test_suite_platform': [
571 # Incompatible with sanitizers (e.g. ASan). If the driver
572 # component uses a sanitizer but the reference component
573 # doesn't, we have a PASS vs SKIP mismatch.
574 'Check mbedtls_calloc overallocation',
575 ],
576 # See ecp_light_only
577 'test_suite_random': [
578 'PSA classic wrapper: ECDSA signature (SECP256R1)',
579 ],
580 # See no_ecp_at_all
581 'test_suite_pkparse': [
582 re.compile(r'Parse EC Key .*compressed\)'),
583 re.compile(r'Parse Public EC Key .*compressed\)'),
584 ],
585 'test_suite_asn1parse': [
586 'INTEGER too large for mpi',
587 ],
588 'test_suite_asn1write': [
589 re.compile(r'ASN.1 Write mpi.*'),
590 ],
591 'test_suite_debug': [
592 re.compile(r'Debug print mbedtls_mpi.*'),
593 ],
594 # See ecp_light_only
595 'test_suite_ssl': [
596 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
597 ],
598 }
599
Gilles Peskine082eade2024-10-03 18:42:37 +0200600class DriverVSReference_ffdh_alg(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200601 REFERENCE = 'test_psa_crypto_config_reference_ffdh'
602 DRIVER = 'test_psa_crypto_config_accel_ffdh'
603 IGNORED_SUITES = ['dhm']
604 IGNORED_TESTS = {
605 'test_suite_config': [
606 re.compile(r'.*\bMBEDTLS_DHM_C\b.*'),
607 ],
608 'test_suite_platform': [
609 # Incompatible with sanitizers (e.g. ASan). If the driver
610 # component uses a sanitizer but the reference component
611 # doesn't, we have a PASS vs SKIP mismatch.
612 'Check mbedtls_calloc overallocation',
613 ],
614 }
615
Gilles Peskine082eade2024-10-03 18:42:37 +0200616class DriverVSReference_tfm_config(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200617 REFERENCE = 'test_tfm_config_no_p256m'
618 DRIVER = 'test_tfm_config_p256m_driver_accel_ec'
619 IGNORED_SUITES = [
620 # Modules replaced by drivers
621 'asn1parse', 'asn1write',
622 'ecp', 'ecdsa', 'ecdh', 'ecjpake',
623 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
624 'bignum.generated', 'bignum.misc',
Gilles Peskine77587ce2024-10-29 20:55:11 +0100625 # Unit tests for the built-in implementation
626 'psa_crypto_ecp',
Gilles Peskine9df375b2024-09-16 20:14:26 +0200627 ]
628 IGNORED_TESTS = {
629 'test_suite_config': [
630 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
631 re.compile(r'.*\bMBEDTLS_(ASN1\w+)_C\b.*'),
632 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECP)_.*'),
633 re.compile(r'.*\bMBEDTLS_PSA_P256M_DRIVER_ENABLED\b.*')
634 ],
635 'test_suite_config.crypto_combinations': [
636 'Config: ECC: Weierstrass curves only',
637 ],
638 'test_suite_platform': [
639 # Incompatible with sanitizers (e.g. ASan). If the driver
640 # component uses a sanitizer but the reference component
641 # doesn't, we have a PASS vs SKIP mismatch.
642 'Check mbedtls_calloc overallocation',
643 ],
644 # See ecp_light_only
645 'test_suite_random': [
646 'PSA classic wrapper: ECDSA signature (SECP256R1)',
647 ],
648 }
649
Gilles Peskine082eade2024-10-03 18:42:37 +0200650class DriverVSReference_rsa(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200651 REFERENCE = 'test_psa_crypto_config_reference_rsa_crypto'
652 DRIVER = 'test_psa_crypto_config_accel_rsa_crypto'
653 IGNORED_SUITES = [
654 # Modules replaced by drivers.
655 'rsa', 'pkcs1_v15', 'pkcs1_v21',
656 # We temporarily don't care about PK stuff.
657 'pk', 'pkwrite', 'pkparse'
658 ]
659 IGNORED_TESTS = {
660 'test_suite_config': [
661 re.compile(r'.*\bMBEDTLS_(PKCS1|RSA)_.*'),
662 re.compile(r'.*\bMBEDTLS_GENPRIME\b.*')
663 ],
664 'test_suite_platform': [
665 # Incompatible with sanitizers (e.g. ASan). If the driver
666 # component uses a sanitizer but the reference component
667 # doesn't, we have a PASS vs SKIP mismatch.
668 'Check mbedtls_calloc overallocation',
669 ],
670 # Following tests depend on RSA_C but are not about
671 # them really, just need to know some error code is there.
672 'test_suite_error': [
673 'Low and high error',
674 'Single high error'
675 ],
676 # Constant time operations only used for PKCS1_V15
677 'test_suite_constant_time': [
678 re.compile(r'mbedtls_ct_zeroize_if .*'),
679 re.compile(r'mbedtls_ct_memmove_left .*')
680 ],
681 'test_suite_psa_crypto': [
682 # We don't support generate_key_custom entry points
683 # in drivers yet.
684 re.compile(r'PSA generate key custom: RSA, e=.*'),
685 re.compile(r'PSA generate key ext: RSA, e=.*'),
686 ],
687 }
688
Gilles Peskine082eade2024-10-03 18:42:37 +0200689class DriverVSReference_block_cipher_dispatch(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200690 REFERENCE = 'test_full_block_cipher_legacy_dispatch'
691 DRIVER = 'test_full_block_cipher_psa_dispatch'
692 IGNORED_SUITES = [
693 # Skipped in the accelerated component
694 'aes', 'aria', 'camellia',
695 # These require AES_C, ARIA_C or CAMELLIA_C to be enabled in
696 # order for the cipher module (actually cipher_wrapper) to work
697 # properly. However these symbols are disabled in the accelerated
698 # component so we ignore them.
699 'cipher.ccm', 'cipher.gcm', 'cipher.aes', 'cipher.aria',
700 'cipher.camellia',
701 ]
702 IGNORED_TESTS = {
703 'test_suite_config': [
704 re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA)_.*'),
705 re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'),
706 ],
707 'test_suite_cmac': [
708 # Following tests require AES_C/ARIA_C/CAMELLIA_C to be enabled,
709 # but these are not available in the accelerated component.
710 'CMAC null arguments',
711 re.compile('CMAC.* (AES|ARIA|Camellia).*'),
712 ],
713 'test_suite_cipher.padding': [
714 # Following tests require AES_C/CAMELLIA_C to be enabled,
715 # but these are not available in the accelerated component.
716 re.compile('Set( non-existent)? padding with (AES|CAMELLIA).*'),
717 ],
718 'test_suite_pkcs5': [
719 # The AES part of PKCS#5 PBES2 is not yet supported.
720 # The rest of PKCS#5 (PBKDF2) works, though.
721 re.compile(r'PBES2 .* AES-.*')
722 ],
723 'test_suite_pkparse': [
724 # PEM (called by pkparse) requires AES_C in order to decrypt
725 # the key, but this is not available in the accelerated
726 # component.
727 re.compile('Parse RSA Key.*(password|AES-).*'),
728 ],
729 'test_suite_pem': [
730 # Following tests require AES_C, but this is diabled in the
731 # accelerated component.
732 re.compile('PEM read .*AES.*'),
733 'PEM read (unknown encryption algorithm)',
734 ],
735 'test_suite_error': [
736 # Following tests depend on AES_C but are not about them
737 # really, just need to know some error code is there.
738 'Single low error',
739 'Low and high error',
740 ],
741 'test_suite_version': [
742 # Similar to test_suite_error above.
743 'Check for MBEDTLS_AES_C when already present',
744 ],
745 'test_suite_platform': [
746 # Incompatible with sanitizers (e.g. ASan). If the driver
747 # component uses a sanitizer but the reference component
748 # doesn't, we have a PASS vs SKIP mismatch.
749 'Check mbedtls_calloc overallocation',
750 ],
751 }
752
753#pylint: enable=invalid-name,missing-class-docstring
754
755
Przemek Stekiel6856f4c2022-11-09 10:50:29 +0100756# List of tasks with a function that can handle this task and additional arguments if required
Valerio Settidfd7ca62023-10-09 16:30:11 +0200757KNOWN_TASKS = {
Gilles Peskinef646dbf2024-09-16 19:15:29 +0200758 'analyze_coverage': CoverageTask,
Gilles Peskine9df375b2024-09-16 20:14:26 +0200759 'analyze_driver_vs_reference_hash': DriverVSReference_hash,
760 'analyze_driver_vs_reference_hmac': DriverVSReference_hmac,
761 'analyze_driver_vs_reference_cipher_aead_cmac': DriverVSReference_cipher_aead_cmac,
762 'analyze_driver_vs_reference_ecp_light_only': DriverVSReference_ecp_light_only,
763 'analyze_driver_vs_reference_no_ecp_at_all': DriverVSReference_no_ecp_at_all,
764 'analyze_driver_vs_reference_ecc_no_bignum': DriverVSReference_ecc_no_bignum,
765 'analyze_driver_vs_reference_ecc_ffdh_no_bignum': DriverVSReference_ecc_ffdh_no_bignum,
766 'analyze_driver_vs_reference_ffdh_alg': DriverVSReference_ffdh_alg,
767 'analyze_driver_vs_reference_tfm_config': DriverVSReference_tfm_config,
768 'analyze_driver_vs_reference_rsa': DriverVSReference_rsa,
769 'analyze_block_cipher_dispatch': DriverVSReference_block_cipher_dispatch,
Przemek Stekiel4d13c832022-10-26 16:11:26 +0200770}
Przemek Stekiel4d13c832022-10-26 16:11:26 +0200771
Gilles Peskine15c2cbf2020-06-25 18:36:28 +0200772if __name__ == '__main__':
Gilles Peskine082eade2024-10-03 18:42:37 +0200773 outcome_analysis.main(KNOWN_TASKS)