blob: 698358dbb9c2d81e52372d347c28033ecbd18595 [file] [log] [blame]
Gilles Peskine15c2cbf2020-06-25 18:36:28 +02001#!/usr/bin/env python3
2
3"""Analyze the test outcomes from a full CI run.
4
5This script can also run on outcomes from a partial run, but the results are
6less likely to be useful.
7"""
8
Przemek Stekiel85c54ea2022-11-17 11:50:23 +01009import re
Gilles Peskine2a71fac2024-09-17 15:07:22 +020010import typing
Gilles Peskine15c2cbf2020-06-25 18:36:28 +020011
Gilles Peskine31467722024-10-03 18:52:58 +020012import scripts_path # pylint: disable=unused-import
13from mbedtls_framework import outcome_analysis
Gilles Peskine8d3c70a2020-06-25 18:37:43 +020014
Pengyu Lvc2e8f3a2023-11-28 17:22:04 +080015
Gilles Peskine082eade2024-10-03 18:42:37 +020016class CoverageTask(outcome_analysis.CoverageTask):
Gilles Peskine96db2cc2024-10-04 15:52:01 +020017 # We'll populate IGNORED_TESTS soon. In the meantime, lack of coverage
18 # is just a warning.
19 outcome_analysis.FULL_COVERAGE_BY_DEFAULT = False
Gilles Peskine3f5022e2024-09-16 20:23:40 +020020
Gilles Peskine2a71fac2024-09-17 15:07:22 +020021 @staticmethod
22 def _has_word_re(words: typing.Iterable[str]) -> typing.Pattern:
23 """Construct a regex that matches if any of the words appears.
24
25 The occurrence must start and end at a word boundary.
26 """
27 return re.compile(r'.*\b(?:' + r'|'.join(words) + r')\b.*')
28
29 # generate_psa_tests.py generates test cases involving cryptographic
30 # mechanisms (key types, families, algorithms) that are declared but
31 # not implemented. Until we improve the Python scripts, ignore those
32 # test cases in the analysis.
33 # https://github.com/Mbed-TLS/mbedtls/issues/9572
34 _PSA_MECHANISMS_NOT_IMPLEMENTED = [
35 r'CBC_MAC',
36 r'DETERMINISTIC_DSA',
37 r'DET_DSA',
38 r'DSA',
39 r'ECC_KEY_PAIR\(BRAINPOOL_P_R1\) (?:160|192|224|320)-bit',
40 r'ECC_KEY_PAIR\(SECP_K1\) 225-bit',
41 r'ECC_PAIR\(BP_R1\) (?:160|192|224|320)-bit',
42 r'ECC_PAIR\(SECP_K1\) 225-bit',
43 r'ECC_PUBLIC_KEY\(BRAINPOOL_P_R1\) (?:160|192|224|320)-bit',
44 r'ECC_PUBLIC_KEY\(SECP_K1\) 225-bit',
45 r'ECC_PUB\(BP_R1\) (?:160|192|224|320)-bit',
46 r'ECC_PUB\(SECP_K1\) 225-bit',
47 r'ED25519PH',
48 r'ED448PH',
49 r'PEPPER',
50 r'PURE_EDDSA',
51 r'SECP_R2',
52 r'SECT_K1',
53 r'SECT_R1',
54 r'SECT_R2',
55 r'SHAKE256_512',
56 r'SHA_512_224',
57 r'SHA_512_256',
58 r'TWISTED_EDWARDS',
59 r'XTS',
60 ]
61 PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE = \
62 _has_word_re(_PSA_MECHANISMS_NOT_IMPLEMENTED)
63
64 IGNORED_TESTS = {
65 'test_suite_psa_crypto_generate_key.generated': [
66 PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE,
67 ],
68 'test_suite_psa_crypto_not_supported.generated': [
69 PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE,
70 ],
71 'test_suite_psa_crypto_op_fail.generated': [
72 PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE,
73 ],
74 'test_suite_psa_crypto_storage_format.current': [
75 PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE,
76 ],
77 'test_suite_psa_crypto_storage_format.v0': [
78 PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE,
79 ],
80 }
81
Gilles Peskine82b16722024-09-16 19:57:10 +020082
Gilles Peskine9df375b2024-09-16 20:14:26 +020083# The names that we give to classes derived from DriverVSReference do not
84# follow the usual naming convention, because it's more readable to use
85# underscores and parts of the configuration names. Also, these classes
86# are just there to specify some data, so they don't need repetitive
87# documentation.
88#pylint: disable=invalid-name,missing-class-docstring
89
Gilles Peskine082eade2024-10-03 18:42:37 +020090class DriverVSReference_hash(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +020091 REFERENCE = 'test_psa_crypto_config_reference_hash_use_psa'
92 DRIVER = 'test_psa_crypto_config_accel_hash_use_psa'
93 IGNORED_SUITES = [
94 'shax', 'mdx', # the software implementations that are being excluded
95 'md.psa', # purposefully depends on whether drivers are present
96 'psa_crypto_low_hash.generated', # testing the builtins
97 ]
98 IGNORED_TESTS = {
99 'test_suite_config': [
100 re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'),
101 ],
102 'test_suite_platform': [
103 # Incompatible with sanitizers (e.g. ASan). If the driver
104 # component uses a sanitizer but the reference component
105 # doesn't, we have a PASS vs SKIP mismatch.
106 'Check mbedtls_calloc overallocation',
107 ],
108 }
109
Gilles Peskine082eade2024-10-03 18:42:37 +0200110class DriverVSReference_hmac(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200111 REFERENCE = 'test_psa_crypto_config_reference_hmac'
112 DRIVER = 'test_psa_crypto_config_accel_hmac'
113 IGNORED_SUITES = [
114 # These suites require legacy hash support, which is disabled
115 # in the accelerated component.
116 'shax', 'mdx',
117 # This suite tests builtins directly, but these are missing
118 # in the accelerated case.
119 'psa_crypto_low_hash.generated',
120 ]
121 IGNORED_TESTS = {
122 'test_suite_config': [
123 re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'),
124 re.compile(r'.*\bMBEDTLS_MD_C\b')
125 ],
126 'test_suite_md': [
127 # Builtin HMAC is not supported in the accelerate component.
128 re.compile('.*HMAC.*'),
129 # Following tests make use of functions which are not available
130 # when MD_C is disabled, as it happens in the accelerated
131 # test component.
132 re.compile('generic .* Hash file .*'),
133 'MD list',
134 ],
135 'test_suite_md.psa': [
136 # "legacy only" tests require hash algorithms to be NOT
137 # accelerated, but this of course false for the accelerated
138 # test component.
139 re.compile('PSA dispatch .* legacy only'),
140 ],
141 'test_suite_platform': [
142 # Incompatible with sanitizers (e.g. ASan). If the driver
143 # component uses a sanitizer but the reference component
144 # doesn't, we have a PASS vs SKIP mismatch.
145 'Check mbedtls_calloc overallocation',
146 ],
147 }
148
Gilles Peskine082eade2024-10-03 18:42:37 +0200149class DriverVSReference_cipher_aead_cmac(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200150 REFERENCE = 'test_psa_crypto_config_reference_cipher_aead_cmac'
151 DRIVER = 'test_psa_crypto_config_accel_cipher_aead_cmac'
152 # Modules replaced by drivers.
153 IGNORED_SUITES = [
154 # low-level (block/stream) cipher modules
155 'aes', 'aria', 'camellia', 'des', 'chacha20',
156 # AEAD modes and CMAC
157 'ccm', 'chachapoly', 'cmac', 'gcm',
158 # The Cipher abstraction layer
159 'cipher',
160 ]
161 IGNORED_TESTS = {
162 'test_suite_config': [
163 re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA|CHACHA20|DES)_.*'),
164 re.compile(r'.*\bMBEDTLS_(CCM|CHACHAPOLY|CMAC|GCM)_.*'),
165 re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'),
166 re.compile(r'.*\bMBEDTLS_CIPHER_.*'),
167 ],
168 # PEM decryption is not supported so far.
169 # The rest of PEM (write, unencrypted read) works though.
170 'test_suite_pem': [
171 re.compile(r'PEM read .*(AES|DES|\bencrypt).*'),
172 ],
173 'test_suite_platform': [
174 # Incompatible with sanitizers (e.g. ASan). If the driver
175 # component uses a sanitizer but the reference component
176 # doesn't, we have a PASS vs SKIP mismatch.
177 'Check mbedtls_calloc overallocation',
178 ],
179 # Following tests depend on AES_C/DES_C but are not about
180 # them really, just need to know some error code is there.
181 'test_suite_error': [
182 'Low and high error',
183 'Single low error'
184 ],
185 # Similar to test_suite_error above.
186 'test_suite_version': [
187 'Check for MBEDTLS_AES_C when already present',
188 ],
189 # The en/decryption part of PKCS#12 is not supported so far.
190 # The rest of PKCS#12 (key derivation) works though.
191 'test_suite_pkcs12': [
192 re.compile(r'PBE Encrypt, .*'),
193 re.compile(r'PBE Decrypt, .*'),
194 ],
195 # The en/decryption part of PKCS#5 is not supported so far.
196 # The rest of PKCS#5 (PBKDF2) works though.
197 'test_suite_pkcs5': [
198 re.compile(r'PBES2 Encrypt, .*'),
199 re.compile(r'PBES2 Decrypt .*'),
200 ],
201 # Encrypted keys are not supported so far.
202 # pylint: disable=line-too-long
203 'test_suite_pkparse': [
204 'Key ASN1 (Encrypted key PKCS12, trailing garbage data)',
205 'Key ASN1 (Encrypted key PKCS5, trailing garbage data)',
206 re.compile(r'Parse (RSA|EC) Key .*\(.* ([Ee]ncrypted|password).*\)'),
207 ],
208 # Encrypted keys are not supported so far.
209 'ssl-opt': [
210 'TLS: password protected server key',
211 'TLS: password protected client key',
212 'TLS: password protected server key, two certificates',
213 ],
214 }
215
Gilles Peskine082eade2024-10-03 18:42:37 +0200216class DriverVSReference_ecp_light_only(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200217 REFERENCE = 'test_psa_crypto_config_reference_ecc_ecp_light_only'
218 DRIVER = 'test_psa_crypto_config_accel_ecc_ecp_light_only'
219 IGNORED_SUITES = [
220 # Modules replaced by drivers
221 'ecdsa', 'ecdh', 'ecjpake',
222 ]
223 IGNORED_TESTS = {
224 'test_suite_config': [
225 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
226 ],
227 'test_suite_platform': [
228 # Incompatible with sanitizers (e.g. ASan). If the driver
229 # component uses a sanitizer but the reference component
230 # doesn't, we have a PASS vs SKIP mismatch.
231 'Check mbedtls_calloc overallocation',
232 ],
233 # This test wants a legacy function that takes f_rng, p_rng
234 # arguments, and uses legacy ECDSA for that. The test is
235 # really about the wrapper around the PSA RNG, not ECDSA.
236 'test_suite_random': [
237 'PSA classic wrapper: ECDSA signature (SECP256R1)',
238 ],
239 # In the accelerated test ECP_C is not set (only ECP_LIGHT is)
240 # so we must ignore disparities in the tests for which ECP_C
241 # is required.
242 'test_suite_ecp': [
243 re.compile(r'ECP check public-private .*'),
244 re.compile(r'ECP calculate public: .*'),
245 re.compile(r'ECP gen keypair .*'),
246 re.compile(r'ECP point muladd .*'),
247 re.compile(r'ECP point multiplication .*'),
248 re.compile(r'ECP test vectors .*'),
249 ],
250 'test_suite_ssl': [
251 # This deprecated function is only present when ECP_C is On.
252 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
253 ],
254 }
255
Gilles Peskine082eade2024-10-03 18:42:37 +0200256class DriverVSReference_no_ecp_at_all(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200257 REFERENCE = 'test_psa_crypto_config_reference_ecc_no_ecp_at_all'
258 DRIVER = 'test_psa_crypto_config_accel_ecc_no_ecp_at_all'
259 IGNORED_SUITES = [
260 # Modules replaced by drivers
261 'ecp', 'ecdsa', 'ecdh', 'ecjpake',
262 ]
263 IGNORED_TESTS = {
264 'test_suite_config': [
265 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
266 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
267 ],
268 'test_suite_platform': [
269 # Incompatible with sanitizers (e.g. ASan). If the driver
270 # component uses a sanitizer but the reference component
271 # doesn't, we have a PASS vs SKIP mismatch.
272 'Check mbedtls_calloc overallocation',
273 ],
274 # See ecp_light_only
275 'test_suite_random': [
276 'PSA classic wrapper: ECDSA signature (SECP256R1)',
277 ],
278 'test_suite_pkparse': [
279 # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED
280 # is automatically enabled in build_info.h (backward compatibility)
281 # even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a
282 # consequence compressed points are supported in the reference
283 # component but not in the accelerated one, so they should be skipped
284 # while checking driver's coverage.
285 re.compile(r'Parse EC Key .*compressed\)'),
286 re.compile(r'Parse Public EC Key .*compressed\)'),
287 ],
288 # See ecp_light_only
289 'test_suite_ssl': [
290 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
291 ],
292 }
293
Gilles Peskine082eade2024-10-03 18:42:37 +0200294class DriverVSReference_ecc_no_bignum(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200295 REFERENCE = 'test_psa_crypto_config_reference_ecc_no_bignum'
296 DRIVER = 'test_psa_crypto_config_accel_ecc_no_bignum'
297 IGNORED_SUITES = [
298 # Modules replaced by drivers
299 'ecp', 'ecdsa', 'ecdh', 'ecjpake',
300 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
301 'bignum.generated', 'bignum.misc',
302 ]
303 IGNORED_TESTS = {
304 'test_suite_config': [
305 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
306 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
307 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
308 ],
309 'test_suite_platform': [
310 # Incompatible with sanitizers (e.g. ASan). If the driver
311 # component uses a sanitizer but the reference component
312 # doesn't, we have a PASS vs SKIP mismatch.
313 'Check mbedtls_calloc overallocation',
314 ],
315 # See ecp_light_only
316 'test_suite_random': [
317 'PSA classic wrapper: ECDSA signature (SECP256R1)',
318 ],
319 # See no_ecp_at_all
320 'test_suite_pkparse': [
321 re.compile(r'Parse EC Key .*compressed\)'),
322 re.compile(r'Parse Public EC Key .*compressed\)'),
323 ],
324 'test_suite_asn1parse': [
325 'INTEGER too large for mpi',
326 ],
327 'test_suite_asn1write': [
328 re.compile(r'ASN.1 Write mpi.*'),
329 ],
330 'test_suite_debug': [
331 re.compile(r'Debug print mbedtls_mpi.*'),
332 ],
333 # See ecp_light_only
334 'test_suite_ssl': [
335 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
336 ],
337 }
338
Gilles Peskine082eade2024-10-03 18:42:37 +0200339class DriverVSReference_ecc_ffdh_no_bignum(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200340 REFERENCE = 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum'
341 DRIVER = 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum'
342 IGNORED_SUITES = [
343 # Modules replaced by drivers
344 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm',
345 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
346 'bignum.generated', 'bignum.misc',
347 ]
348 IGNORED_TESTS = {
349 'ssl-opt': [
350 # DHE support in TLS 1.2 requires built-in MBEDTLS_DHM_C
351 # (because it needs custom groups, which PSA does not
352 # provide), even with MBEDTLS_USE_PSA_CRYPTO.
353 re.compile(r'PSK callback:.*\bdhe-psk\b.*'),
354 ],
355 'test_suite_config': [
356 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
357 re.compile(r'.*\bMBEDTLS_DHM_C\b.*'),
358 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
359 re.compile(r'.*\bMBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED\b.*'),
360 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
361 ],
362 'test_suite_platform': [
363 # Incompatible with sanitizers (e.g. ASan). If the driver
364 # component uses a sanitizer but the reference component
365 # doesn't, we have a PASS vs SKIP mismatch.
366 'Check mbedtls_calloc overallocation',
367 ],
368 # See ecp_light_only
369 'test_suite_random': [
370 'PSA classic wrapper: ECDSA signature (SECP256R1)',
371 ],
372 # See no_ecp_at_all
373 'test_suite_pkparse': [
374 re.compile(r'Parse EC Key .*compressed\)'),
375 re.compile(r'Parse Public EC Key .*compressed\)'),
376 ],
377 'test_suite_asn1parse': [
378 'INTEGER too large for mpi',
379 ],
380 'test_suite_asn1write': [
381 re.compile(r'ASN.1 Write mpi.*'),
382 ],
383 'test_suite_debug': [
384 re.compile(r'Debug print mbedtls_mpi.*'),
385 ],
386 # See ecp_light_only
387 'test_suite_ssl': [
388 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
389 ],
390 }
391
Gilles Peskine082eade2024-10-03 18:42:37 +0200392class DriverVSReference_ffdh_alg(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200393 REFERENCE = 'test_psa_crypto_config_reference_ffdh'
394 DRIVER = 'test_psa_crypto_config_accel_ffdh'
395 IGNORED_SUITES = ['dhm']
396 IGNORED_TESTS = {
397 'test_suite_config': [
398 re.compile(r'.*\bMBEDTLS_DHM_C\b.*'),
399 ],
400 'test_suite_platform': [
401 # Incompatible with sanitizers (e.g. ASan). If the driver
402 # component uses a sanitizer but the reference component
403 # doesn't, we have a PASS vs SKIP mismatch.
404 'Check mbedtls_calloc overallocation',
405 ],
406 }
407
Gilles Peskine082eade2024-10-03 18:42:37 +0200408class DriverVSReference_tfm_config(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200409 REFERENCE = 'test_tfm_config_no_p256m'
410 DRIVER = 'test_tfm_config_p256m_driver_accel_ec'
411 IGNORED_SUITES = [
412 # Modules replaced by drivers
413 'asn1parse', 'asn1write',
414 'ecp', 'ecdsa', 'ecdh', 'ecjpake',
415 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
416 'bignum.generated', 'bignum.misc',
417 ]
418 IGNORED_TESTS = {
419 'test_suite_config': [
420 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
421 re.compile(r'.*\bMBEDTLS_(ASN1\w+)_C\b.*'),
422 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECP)_.*'),
423 re.compile(r'.*\bMBEDTLS_PSA_P256M_DRIVER_ENABLED\b.*')
424 ],
425 'test_suite_config.crypto_combinations': [
426 'Config: ECC: Weierstrass curves only',
427 ],
428 'test_suite_platform': [
429 # Incompatible with sanitizers (e.g. ASan). If the driver
430 # component uses a sanitizer but the reference component
431 # doesn't, we have a PASS vs SKIP mismatch.
432 'Check mbedtls_calloc overallocation',
433 ],
434 # See ecp_light_only
435 'test_suite_random': [
436 'PSA classic wrapper: ECDSA signature (SECP256R1)',
437 ],
438 }
439
Gilles Peskine082eade2024-10-03 18:42:37 +0200440class DriverVSReference_rsa(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200441 REFERENCE = 'test_psa_crypto_config_reference_rsa_crypto'
442 DRIVER = 'test_psa_crypto_config_accel_rsa_crypto'
443 IGNORED_SUITES = [
444 # Modules replaced by drivers.
445 'rsa', 'pkcs1_v15', 'pkcs1_v21',
446 # We temporarily don't care about PK stuff.
447 'pk', 'pkwrite', 'pkparse'
448 ]
449 IGNORED_TESTS = {
450 'test_suite_config': [
451 re.compile(r'.*\bMBEDTLS_(PKCS1|RSA)_.*'),
452 re.compile(r'.*\bMBEDTLS_GENPRIME\b.*')
453 ],
454 'test_suite_platform': [
455 # Incompatible with sanitizers (e.g. ASan). If the driver
456 # component uses a sanitizer but the reference component
457 # doesn't, we have a PASS vs SKIP mismatch.
458 'Check mbedtls_calloc overallocation',
459 ],
460 # Following tests depend on RSA_C but are not about
461 # them really, just need to know some error code is there.
462 'test_suite_error': [
463 'Low and high error',
464 'Single high error'
465 ],
466 # Constant time operations only used for PKCS1_V15
467 'test_suite_constant_time': [
468 re.compile(r'mbedtls_ct_zeroize_if .*'),
469 re.compile(r'mbedtls_ct_memmove_left .*')
470 ],
471 'test_suite_psa_crypto': [
472 # We don't support generate_key_custom entry points
473 # in drivers yet.
474 re.compile(r'PSA generate key custom: RSA, e=.*'),
475 re.compile(r'PSA generate key ext: RSA, e=.*'),
476 ],
477 }
478
Gilles Peskine082eade2024-10-03 18:42:37 +0200479class DriverVSReference_block_cipher_dispatch(outcome_analysis.DriverVSReference):
Gilles Peskine9df375b2024-09-16 20:14:26 +0200480 REFERENCE = 'test_full_block_cipher_legacy_dispatch'
481 DRIVER = 'test_full_block_cipher_psa_dispatch'
482 IGNORED_SUITES = [
483 # Skipped in the accelerated component
484 'aes', 'aria', 'camellia',
485 # These require AES_C, ARIA_C or CAMELLIA_C to be enabled in
486 # order for the cipher module (actually cipher_wrapper) to work
487 # properly. However these symbols are disabled in the accelerated
488 # component so we ignore them.
489 'cipher.ccm', 'cipher.gcm', 'cipher.aes', 'cipher.aria',
490 'cipher.camellia',
491 ]
492 IGNORED_TESTS = {
493 'test_suite_config': [
494 re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA)_.*'),
495 re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'),
496 ],
497 'test_suite_cmac': [
498 # Following tests require AES_C/ARIA_C/CAMELLIA_C to be enabled,
499 # but these are not available in the accelerated component.
500 'CMAC null arguments',
501 re.compile('CMAC.* (AES|ARIA|Camellia).*'),
502 ],
503 'test_suite_cipher.padding': [
504 # Following tests require AES_C/CAMELLIA_C to be enabled,
505 # but these are not available in the accelerated component.
506 re.compile('Set( non-existent)? padding with (AES|CAMELLIA).*'),
507 ],
508 'test_suite_pkcs5': [
509 # The AES part of PKCS#5 PBES2 is not yet supported.
510 # The rest of PKCS#5 (PBKDF2) works, though.
511 re.compile(r'PBES2 .* AES-.*')
512 ],
513 'test_suite_pkparse': [
514 # PEM (called by pkparse) requires AES_C in order to decrypt
515 # the key, but this is not available in the accelerated
516 # component.
517 re.compile('Parse RSA Key.*(password|AES-).*'),
518 ],
519 'test_suite_pem': [
520 # Following tests require AES_C, but this is diabled in the
521 # accelerated component.
522 re.compile('PEM read .*AES.*'),
523 'PEM read (unknown encryption algorithm)',
524 ],
525 'test_suite_error': [
526 # Following tests depend on AES_C but are not about them
527 # really, just need to know some error code is there.
528 'Single low error',
529 'Low and high error',
530 ],
531 'test_suite_version': [
532 # Similar to test_suite_error above.
533 'Check for MBEDTLS_AES_C when already present',
534 ],
535 'test_suite_platform': [
536 # Incompatible with sanitizers (e.g. ASan). If the driver
537 # component uses a sanitizer but the reference component
538 # doesn't, we have a PASS vs SKIP mismatch.
539 'Check mbedtls_calloc overallocation',
540 ],
541 }
542
543#pylint: enable=invalid-name,missing-class-docstring
544
545
Przemek Stekiel6856f4c2022-11-09 10:50:29 +0100546# List of tasks with a function that can handle this task and additional arguments if required
Valerio Settidfd7ca62023-10-09 16:30:11 +0200547KNOWN_TASKS = {
Gilles Peskinef646dbf2024-09-16 19:15:29 +0200548 'analyze_coverage': CoverageTask,
Gilles Peskine9df375b2024-09-16 20:14:26 +0200549 'analyze_driver_vs_reference_hash': DriverVSReference_hash,
550 'analyze_driver_vs_reference_hmac': DriverVSReference_hmac,
551 'analyze_driver_vs_reference_cipher_aead_cmac': DriverVSReference_cipher_aead_cmac,
552 'analyze_driver_vs_reference_ecp_light_only': DriverVSReference_ecp_light_only,
553 'analyze_driver_vs_reference_no_ecp_at_all': DriverVSReference_no_ecp_at_all,
554 'analyze_driver_vs_reference_ecc_no_bignum': DriverVSReference_ecc_no_bignum,
555 'analyze_driver_vs_reference_ecc_ffdh_no_bignum': DriverVSReference_ecc_ffdh_no_bignum,
556 'analyze_driver_vs_reference_ffdh_alg': DriverVSReference_ffdh_alg,
557 'analyze_driver_vs_reference_tfm_config': DriverVSReference_tfm_config,
558 'analyze_driver_vs_reference_rsa': DriverVSReference_rsa,
559 'analyze_block_cipher_dispatch': DriverVSReference_block_cipher_dispatch,
Przemek Stekiel4d13c832022-10-26 16:11:26 +0200560}
Przemek Stekiel4d13c832022-10-26 16:11:26 +0200561
Gilles Peskine15c2cbf2020-06-25 18:36:28 +0200562if __name__ == '__main__':
Gilles Peskine082eade2024-10-03 18:42:37 +0200563 outcome_analysis.main(KNOWN_TASKS)