blob: fde07159eda8dc0ae97a05dcc1878851b1475007 [file] [log] [blame]
Gilles Peskine15c2cbf2020-06-25 18:36:28 +02001#!/usr/bin/env python3
2
3"""Analyze the test outcomes from a full CI run.
4
5This script can also run on outcomes from a partial run, but the results are
6less likely to be useful.
7"""
8
9import argparse
10import sys
11import traceback
Przemek Stekiel85c54ea2022-11-17 11:50:23 +010012import re
Valerio Settia2663322023-03-24 08:20:18 +010013import subprocess
14import os
Gilles Peskine15c2cbf2020-06-25 18:36:28 +020015
Gilles Peskine8d3c70a2020-06-25 18:37:43 +020016import check_test_cases
17
Gilles Peskine15c2cbf2020-06-25 18:36:28 +020018class Results:
19 """Process analysis results."""
20
21 def __init__(self):
22 self.error_count = 0
23 self.warning_count = 0
24
25 @staticmethod
26 def log(fmt, *args, **kwargs):
27 sys.stderr.write((fmt + '\n').format(*args, **kwargs))
28
29 def error(self, fmt, *args, **kwargs):
30 self.log('Error: ' + fmt, *args, **kwargs)
31 self.error_count += 1
32
33 def warning(self, fmt, *args, **kwargs):
34 self.log('Warning: ' + fmt, *args, **kwargs)
35 self.warning_count += 1
36
37class TestCaseOutcomes:
38 """The outcomes of one test case across many configurations."""
39 # pylint: disable=too-few-public-methods
40
41 def __init__(self):
Gilles Peskine3d863f22020-06-26 13:02:30 +020042 # Collect a list of witnesses of the test case succeeding or failing.
43 # Currently we don't do anything with witnesses except count them.
44 # The format of a witness is determined by the read_outcome_file
45 # function; it's the platform and configuration joined by ';'.
Gilles Peskine15c2cbf2020-06-25 18:36:28 +020046 self.successes = []
47 self.failures = []
48
49 def hits(self):
50 """Return the number of times a test case has been run.
51
52 This includes passes and failures, but not skips.
53 """
54 return len(self.successes) + len(self.failures)
55
Valerio Settia2663322023-03-24 08:20:18 +010056def execute_reference_driver_tests(ref_component, driver_component, outcome_file):
Valerio Setti22992a02023-03-29 11:15:28 +020057 """Run the tests specified in ref_component and driver_component. Results
58 are stored in the output_file and they will be used for the following
Valerio Settia2663322023-03-24 08:20:18 +010059 coverage analysis"""
60 # If the outcome file already exists, we assume that the user wants to
61 # perform the comparison analysis again without repeating the tests.
62 if os.path.exists(outcome_file):
63 Results.log("Outcome file (" + outcome_file + ") already exists. " + \
64 "Tests will be skipped.")
65 return
66
67 shell_command = "tests/scripts/all.sh --outcome-file " + outcome_file + \
68 " " + ref_component + " " + driver_component
Valerio Settif109c662023-03-29 11:15:44 +020069 Results.log("Running: " + shell_command)
Valerio Settia2663322023-03-24 08:20:18 +010070 ret_val = subprocess.run(shell_command.split(), check=False).returncode
71
72 if ret_val != 0:
73 Results.log("Error: failed to run reference/driver components")
74 sys.exit(ret_val)
75
Tomás González07bdcc22023-08-11 14:59:03 +010076def analyze_coverage(results, outcomes, allow_list):
Gilles Peskine8d3c70a2020-06-25 18:37:43 +020077 """Check that all available test cases are executed at least once."""
Gilles Peskine686c2922022-01-07 15:58:38 +010078 available = check_test_cases.collect_available_test_cases()
Gilles Peskine8d3c70a2020-06-25 18:37:43 +020079 for key in available:
80 hits = outcomes[key].hits() if key in outcomes else 0
Tomás González07bdcc22023-08-11 14:59:03 +010081 if hits == 0 and key not in allow_list:
Gilles Peskine8d3c70a2020-06-25 18:37:43 +020082 # Make this a warning, not an error, as long as we haven't
83 # fixed this branch to have full coverage of test cases.
84 results.warning('Test case not executed: {}', key)
Tomás González07bdcc22023-08-11 14:59:03 +010085 elif hits != 0 and key in allow_list:
86 # Test Case should be removed from the allow list.
87 results.warning('Allow listed test case was executed: {}', key)
Gilles Peskine8d3c70a2020-06-25 18:37:43 +020088
Valerio Setti3002c992023-01-18 17:28:36 +010089def analyze_driver_vs_reference(outcomes, component_ref, component_driver,
90 ignored_suites, ignored_test=None):
Przemek Stekiel4e955902022-10-21 13:42:08 +020091 """Check that all tests executed in the reference component are also
92 executed in the corresponding driver component.
Valerio Setti3002c992023-01-18 17:28:36 +010093 Skip:
94 - full test suites provided in ignored_suites list
95 - only some specific test inside a test suite, for which the corresponding
96 output string is provided
Przemek Stekiel4e955902022-10-21 13:42:08 +020097 """
Przemek Stekiel4e955902022-10-21 13:42:08 +020098 available = check_test_cases.collect_available_test_cases()
99 result = True
100
101 for key in available:
Przemek Stekiel4e955902022-10-21 13:42:08 +0200102 # Continue if test was not executed by any component
103 hits = outcomes[key].hits() if key in outcomes else 0
Przemek Stekielc86dedf2022-10-24 09:16:04 +0200104 if hits == 0:
Przemek Stekiel4e955902022-10-21 13:42:08 +0200105 continue
Valerio Setti00c1ccb2023-02-02 11:33:31 +0100106 # Skip ignored test suites
107 full_test_suite = key.split(';')[0] # retrieve full test suite name
108 test_string = key.split(';')[1] # retrieve the text string of this test
109 test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name
Manuel Pégourié-Gonnard7d381f52023-03-17 15:13:08 +0100110 if test_suite in ignored_suites or full_test_suite in ignored_suites:
Valerio Setti00c1ccb2023-02-02 11:33:31 +0100111 continue
Valerio Setti3002c992023-01-18 17:28:36 +0100112 if ((full_test_suite in ignored_test) and
113 (test_string in ignored_test[full_test_suite])):
114 continue
Przemek Stekiel4e955902022-10-21 13:42:08 +0200115 # Search for tests that run in reference component and not in driver component
116 driver_test_passed = False
117 reference_test_passed = False
118 for entry in outcomes[key].successes:
Przemek Stekiel51f30ff2022-11-09 12:07:29 +0100119 if component_driver in entry:
Przemek Stekiel4e955902022-10-21 13:42:08 +0200120 driver_test_passed = True
Przemek Stekiel51f30ff2022-11-09 12:07:29 +0100121 if component_ref in entry:
Przemek Stekiel4e955902022-10-21 13:42:08 +0200122 reference_test_passed = True
Manuel Pégourié-Gonnardc6967d22022-12-30 13:40:34 +0100123 if(reference_test_passed and not driver_test_passed):
Valerio Setti3951d1b2023-03-13 18:37:34 +0100124 Results.log(key)
Przemek Stekiel4e955902022-10-21 13:42:08 +0200125 result = False
126 return result
127
Tomás González07bdcc22023-08-11 14:59:03 +0100128def analyze_outcomes(outcomes, allow_list):
Gilles Peskine15c2cbf2020-06-25 18:36:28 +0200129 """Run all analyses on the given outcome collection."""
130 results = Results()
Tomás González07bdcc22023-08-11 14:59:03 +0100131 analyze_coverage(results, outcomes, allow_list)
Gilles Peskine15c2cbf2020-06-25 18:36:28 +0200132 return results
133
134def read_outcome_file(outcome_file):
135 """Parse an outcome file and return an outcome collection.
136
137An outcome collection is a dictionary mapping keys to TestCaseOutcomes objects.
138The keys are the test suite name and the test case description, separated
139by a semicolon.
140"""
141 outcomes = {}
142 with open(outcome_file, 'r', encoding='utf-8') as input_file:
143 for line in input_file:
144 (platform, config, suite, case, result, _cause) = line.split(';')
145 key = ';'.join([suite, case])
146 setup = ';'.join([platform, config])
147 if key not in outcomes:
148 outcomes[key] = TestCaseOutcomes()
149 if result == 'PASS':
150 outcomes[key].successes.append(setup)
151 elif result == 'FAIL':
152 outcomes[key].failures.append(setup)
153 return outcomes
154
Przemek Stekiel4d13c832022-10-26 16:11:26 +0200155def do_analyze_coverage(outcome_file, args):
Przemek Stekiel6856f4c2022-11-09 10:50:29 +0100156 """Perform coverage analysis."""
Gilles Peskine15c2cbf2020-06-25 18:36:28 +0200157 outcomes = read_outcome_file(outcome_file)
Valerio Setti3951d1b2023-03-13 18:37:34 +0100158 Results.log("\n*** Analyze coverage ***\n")
Tomás González07bdcc22023-08-11 14:59:03 +0100159 results = analyze_outcomes(outcomes, args['allow_list'])
Przemek Stekielc86dedf2022-10-24 09:16:04 +0200160 return results.error_count == 0
Przemek Stekiel4e955902022-10-21 13:42:08 +0200161
Przemek Stekiel4d13c832022-10-26 16:11:26 +0200162def do_analyze_driver_vs_reference(outcome_file, args):
Przemek Stekiel4e955902022-10-21 13:42:08 +0200163 """Perform driver vs reference analyze."""
Valerio Settia2663322023-03-24 08:20:18 +0100164 execute_reference_driver_tests(args['component_ref'], \
165 args['component_driver'], outcome_file)
166
Valerio Setti3002c992023-01-18 17:28:36 +0100167 ignored_suites = ['test_suite_' + x for x in args['ignored_suites']]
Przemek Stekiel51f30ff2022-11-09 12:07:29 +0100168
Przemek Stekiel4e955902022-10-21 13:42:08 +0200169 outcomes = read_outcome_file(outcome_file)
Valerio Setti3951d1b2023-03-13 18:37:34 +0100170 Results.log("\n*** Analyze driver {} vs reference {} ***\n".format(
Manuel Pégourié-Gonnardc6967d22022-12-30 13:40:34 +0100171 args['component_driver'], args['component_ref']))
Przemek Stekiel51f30ff2022-11-09 12:07:29 +0100172 return analyze_driver_vs_reference(outcomes, args['component_ref'],
Valerio Setti3002c992023-01-18 17:28:36 +0100173 args['component_driver'], ignored_suites,
174 args['ignored_tests'])
Gilles Peskine15c2cbf2020-06-25 18:36:28 +0200175
Przemek Stekiel6856f4c2022-11-09 10:50:29 +0100176# List of tasks with a function that can handle this task and additional arguments if required
Przemek Stekiel4d13c832022-10-26 16:11:26 +0200177TASKS = {
178 'analyze_coverage': {
179 'test_function': do_analyze_coverage,
Tomás González07bdcc22023-08-11 14:59:03 +0100180 'args': {
181 'allow_list': [],
182 }
Manuel Pégourié-Gonnard10e39632022-12-29 12:29:09 +0100183 },
Valerio Settia2663322023-03-24 08:20:18 +0100184 # There are 2 options to use analyze_driver_vs_reference_xxx locally:
185 # 1. Run tests and then analysis:
186 # - tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver>
187 # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
188 # 2. Let this script run both automatically:
189 # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
Przemek Stekiel4d13c832022-10-26 16:11:26 +0200190 'analyze_driver_vs_reference_hash': {
191 'test_function': do_analyze_driver_vs_reference,
192 'args': {
Przemek Stekiel51f30ff2022-11-09 12:07:29 +0100193 'component_ref': 'test_psa_crypto_config_reference_hash_use_psa',
194 'component_driver': 'test_psa_crypto_config_accel_hash_use_psa',
Manuel Pégourié-Gonnard10e39632022-12-29 12:29:09 +0100195 'ignored_suites': [
196 'shax', 'mdx', # the software implementations that are being excluded
Manuel Pégourié-Gonnard7d381f52023-03-17 15:13:08 +0100197 'md.psa', # purposefully depends on whether drivers are present
Valerio Setti3002c992023-01-18 17:28:36 +0100198 ],
199 'ignored_tests': {
200 }
201 }
202 },
Valerio Setti4d25a8d2023-06-14 10:33:10 +0200203 'analyze_driver_vs_reference_ecp_light_only': {
Valerio Setti42d5f192023-03-20 13:54:41 +0100204 'test_function': do_analyze_driver_vs_reference,
205 'args': {
Valerio Setti4d25a8d2023-06-14 10:33:10 +0200206 'component_ref': 'test_psa_crypto_config_reference_ecc_ecp_light_only',
207 'component_driver': 'test_psa_crypto_config_accel_ecc_ecp_light_only',
Valerio Setti42d5f192023-03-20 13:54:41 +0100208 'ignored_suites': [
209 'ecdsa',
210 'ecdh',
211 'ecjpake',
212 ],
213 'ignored_tests': {
214 'test_suite_random': [
215 'PSA classic wrapper: ECDSA signature (SECP256R1)',
216 ],
Valerio Setti0c477d32023-04-07 15:54:20 +0200217 # In the accelerated test ECP_C is not set (only ECP_LIGHT is)
218 # so we must ignore disparities in the tests for which ECP_C
219 # is required.
220 'test_suite_ecp': [
221 'ECP check public-private #1 (OK)',
222 'ECP check public-private #2 (group none)',
223 'ECP check public-private #3 (group mismatch)',
224 'ECP check public-private #4 (Qx mismatch)',
225 'ECP check public-private #5 (Qy mismatch)',
226 'ECP check public-private #6 (wrong Qx)',
227 'ECP check public-private #7 (wrong Qy)',
228 'ECP gen keypair [#1]',
229 'ECP gen keypair [#2]',
230 'ECP gen keypair [#3]',
231 'ECP gen keypair wrapper',
232 'ECP point muladd secp256r1 #1',
233 'ECP point muladd secp256r1 #2',
234 'ECP point multiplication Curve25519 (element of order 2: origin) #3',
235 'ECP point multiplication Curve25519 (element of order 4: 1) #4',
236 'ECP point multiplication Curve25519 (element of order 8) #5',
237 'ECP point multiplication Curve25519 (normalized) #1',
238 'ECP point multiplication Curve25519 (not normalized) #2',
239 'ECP point multiplication rng fail Curve25519',
240 'ECP point multiplication rng fail secp256r1',
241 'ECP test vectors Curve25519',
242 'ECP test vectors Curve448 (RFC 7748 6.2, after decodeUCoordinate)',
243 'ECP test vectors brainpoolP256r1 rfc 7027',
244 'ECP test vectors brainpoolP384r1 rfc 7027',
245 'ECP test vectors brainpoolP512r1 rfc 7027',
246 'ECP test vectors secp192k1',
247 'ECP test vectors secp192r1 rfc 5114',
248 'ECP test vectors secp224k1',
249 'ECP test vectors secp224r1 rfc 5114',
250 'ECP test vectors secp256k1',
251 'ECP test vectors secp256r1 rfc 5114',
252 'ECP test vectors secp384r1 rfc 5114',
253 'ECP test vectors secp521r1 rfc 5114',
Valerio Settie50a75f2023-05-19 17:43:06 +0200254 ],
Valerio Setti5f540202023-06-30 17:20:49 +0200255 }
Valerio Setti42d5f192023-03-20 13:54:41 +0100256 }
257 },
Valerio Setti4d25a8d2023-06-14 10:33:10 +0200258 'analyze_driver_vs_reference_no_ecp_at_all': {
Valerio Settie618cb02023-04-12 14:59:16 +0200259 'test_function': do_analyze_driver_vs_reference,
260 'args': {
Valerio Setti4d25a8d2023-06-14 10:33:10 +0200261 'component_ref': 'test_psa_crypto_config_reference_ecc_no_ecp_at_all',
262 'component_driver': 'test_psa_crypto_config_accel_ecc_no_ecp_at_all',
Valerio Settie618cb02023-04-12 14:59:16 +0200263 'ignored_suites': [
264 # Ignore test suites for the modules that are disabled in the
265 # accelerated test case.
266 'ecp',
267 'ecdsa',
268 'ecdh',
269 'ecjpake',
270 ],
271 'ignored_tests': {
272 'test_suite_random': [
273 'PSA classic wrapper: ECDSA signature (SECP256R1)',
274 ],
275 'test_suite_psa_crypto': [
276 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1',
277 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)',
278 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA',
279 'PSA key derivation: HKDF-SHA-256 -> ECC secp384r1',
280 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #0',
281 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #1',
282 'PSA key derivation: bits=7 invalid for ECC BRAINPOOL_P_R1 (ECC enabled)',
283 'PSA key derivation: bits=7 invalid for ECC SECP_K1 (ECC enabled)',
284 'PSA key derivation: bits=7 invalid for ECC SECP_R1 (ECC enabled)',
285 'PSA key derivation: bits=7 invalid for ECC SECP_R2 (ECC enabled)',
286 'PSA key derivation: bits=7 invalid for ECC SECT_K1 (ECC enabled)',
287 'PSA key derivation: bits=7 invalid for ECC SECT_R1 (ECC enabled)',
288 'PSA key derivation: bits=7 invalid for ECC SECT_R2 (ECC enabled)',
Valerio Settiaddeee42023-06-14 10:46:55 +0200289 ],
290 'test_suite_pkparse': [
Valerio Setti5bd25232023-06-19 19:32:14 +0200291 # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED
292 # is automatically enabled in build_info.h (backward compatibility)
293 # even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a
294 # consequence compressed points are supported in the reference
295 # component but not in the accelerated one, so they should be skipped
296 # while checking driver's coverage.
297 'Parse EC Key #10a (SEC1 PEM, secp384r1, compressed)',
298 'Parse EC Key #11a (SEC1 PEM, secp521r1, compressed)',
299 'Parse EC Key #12a (SEC1 PEM, bp256r1, compressed)',
300 'Parse EC Key #13a (SEC1 PEM, bp384r1, compressed)',
301 'Parse EC Key #14a (SEC1 PEM, bp512r1, compressed)',
302 'Parse EC Key #2a (SEC1 PEM, secp192r1, compressed)',
303 'Parse EC Key #8a (SEC1 PEM, secp224r1, compressed)',
304 'Parse EC Key #9a (SEC1 PEM, secp256r1, compressed)',
305 'Parse Public EC Key #2a (RFC 5480, PEM, secp192r1, compressed)',
306 'Parse Public EC Key #3a (RFC 5480, secp224r1, compressed)',
307 'Parse Public EC Key #4a (RFC 5480, secp256r1, compressed)',
308 'Parse Public EC Key #5a (RFC 5480, secp384r1, compressed)',
309 'Parse Public EC Key #6a (RFC 5480, secp521r1, compressed)',
310 'Parse Public EC Key #7a (RFC 5480, brainpoolP256r1, compressed)',
311 'Parse Public EC Key #8a (RFC 5480, brainpoolP384r1, compressed)',
312 'Parse Public EC Key #9a (RFC 5480, brainpoolP512r1, compressed)',
Valerio Settiaddeee42023-06-14 10:46:55 +0200313 ],
Valerio Settie618cb02023-04-12 14:59:16 +0200314 }
315 }
316 },
Manuel Pégourié-Gonnardabd00d02023-06-12 17:51:33 +0200317 'analyze_driver_vs_reference_no_bignum': {
318 'test_function': do_analyze_driver_vs_reference,
319 'args': {
320 'component_ref': 'test_psa_crypto_config_reference_ecc_no_bignum',
321 'component_driver': 'test_psa_crypto_config_accel_ecc_no_bignum',
322 'ignored_suites': [
323 # Ignore test suites for the modules that are disabled in the
324 # accelerated test case.
325 'ecp',
326 'ecdsa',
327 'ecdh',
328 'ecjpake',
Valerio Setti9b3dbcc2023-07-26 18:00:31 +0200329 'bignum_core',
330 'bignum_random',
331 'bignum_mod',
332 'bignum_mod_raw',
333 'bignum.generated',
334 'bignum.misc',
Manuel Pégourié-Gonnardabd00d02023-06-12 17:51:33 +0200335 ],
336 'ignored_tests': {
337 'test_suite_random': [
338 'PSA classic wrapper: ECDSA signature (SECP256R1)',
339 ],
340 'test_suite_psa_crypto': [
341 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1',
342 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1 (1 redraw)',
343 'PSA key derivation: HKDF-SHA-256 -> ECC secp256r1, exercise ECDSA',
344 'PSA key derivation: HKDF-SHA-256 -> ECC secp384r1',
345 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #0',
346 'PSA key derivation: HKDF-SHA-256 -> ECC secp521r1 #1',
347 'PSA key derivation: bits=7 invalid for ECC BRAINPOOL_P_R1 (ECC enabled)',
348 'PSA key derivation: bits=7 invalid for ECC SECP_K1 (ECC enabled)',
349 'PSA key derivation: bits=7 invalid for ECC SECP_R1 (ECC enabled)',
350 'PSA key derivation: bits=7 invalid for ECC SECP_R2 (ECC enabled)',
351 'PSA key derivation: bits=7 invalid for ECC SECT_K1 (ECC enabled)',
352 'PSA key derivation: bits=7 invalid for ECC SECT_R1 (ECC enabled)',
353 'PSA key derivation: bits=7 invalid for ECC SECT_R2 (ECC enabled)',
354 ],
355 'test_suite_pkparse': [
356 # See the description provided above in the
357 # analyze_driver_vs_reference_no_ecp_at_all component.
358 'Parse EC Key #10a (SEC1 PEM, secp384r1, compressed)',
359 'Parse EC Key #11a (SEC1 PEM, secp521r1, compressed)',
360 'Parse EC Key #12a (SEC1 PEM, bp256r1, compressed)',
361 'Parse EC Key #13a (SEC1 PEM, bp384r1, compressed)',
362 'Parse EC Key #14a (SEC1 PEM, bp512r1, compressed)',
363 'Parse EC Key #2a (SEC1 PEM, secp192r1, compressed)',
364 'Parse EC Key #8a (SEC1 PEM, secp224r1, compressed)',
365 'Parse EC Key #9a (SEC1 PEM, secp256r1, compressed)',
366 'Parse Public EC Key #2a (RFC 5480, PEM, secp192r1, compressed)',
367 'Parse Public EC Key #3a (RFC 5480, secp224r1, compressed)',
368 'Parse Public EC Key #4a (RFC 5480, secp256r1, compressed)',
369 'Parse Public EC Key #5a (RFC 5480, secp384r1, compressed)',
370 'Parse Public EC Key #6a (RFC 5480, secp521r1, compressed)',
371 'Parse Public EC Key #7a (RFC 5480, brainpoolP256r1, compressed)',
372 'Parse Public EC Key #8a (RFC 5480, brainpoolP384r1, compressed)',
373 'Parse Public EC Key #9a (RFC 5480, brainpoolP512r1, compressed)',
374 ],
Valerio Setti9b3dbcc2023-07-26 18:00:31 +0200375 'test_suite_asn1parse': [
376 # This test depends on BIGNUM_C
377 'INTEGER too large for mpi',
378 ],
379 'test_suite_asn1write': [
380 # Following tests depends on BIGNUM_C
381 'ASN.1 Write mpi 0 (1 limb)',
382 'ASN.1 Write mpi 0 (null)',
383 'ASN.1 Write mpi 0x100',
384 'ASN.1 Write mpi 0x7f',
385 'ASN.1 Write mpi 0x7f with leading 0 limb',
386 'ASN.1 Write mpi 0x80',
387 'ASN.1 Write mpi 0x80 with leading 0 limb',
388 'ASN.1 Write mpi 0xff',
389 'ASN.1 Write mpi 1',
390 'ASN.1 Write mpi, 127*8 bits',
391 'ASN.1 Write mpi, 127*8+1 bits',
392 'ASN.1 Write mpi, 127*8-1 bits',
393 'ASN.1 Write mpi, 255*8 bits',
394 'ASN.1 Write mpi, 255*8-1 bits',
395 'ASN.1 Write mpi, 256*8-1 bits',
396 ],
Valerio Settie0be95e2023-08-01 09:07:43 +0200397 'test_suite_debug': [
398 # Following tests depends on BIGNUM_C
399 'Debug print mbedtls_mpi #2: 3 bits',
400 'Debug print mbedtls_mpi: 0 (empty representation)',
401 'Debug print mbedtls_mpi: 0 (non-empty representation)',
402 'Debug print mbedtls_mpi: 49 bits',
403 'Debug print mbedtls_mpi: 759 bits',
404 'Debug print mbedtls_mpi: 764 bits #1',
405 'Debug print mbedtls_mpi: 764 bits #2',
406 ],
Manuel Pégourié-Gonnardabd00d02023-06-12 17:51:33 +0200407 }
408 }
409 },
Przemek Stekiel85b64422023-05-26 09:55:23 +0200410 'analyze_driver_vs_reference_ffdh_alg': {
411 'test_function': do_analyze_driver_vs_reference,
412 'args': {
413 'component_ref': 'test_psa_crypto_config_reference_ffdh',
414 'component_driver': 'test_psa_crypto_config_accel_ffdh',
Przemek Stekiel84f4ff12023-07-04 12:35:31 +0200415 'ignored_suites': ['dhm'],
Przemek Stekiel565353e2023-07-05 11:07:07 +0200416 'ignored_tests': {}
Przemek Stekiel85b64422023-05-26 09:55:23 +0200417 }
418 },
Przemek Stekiel4d13c832022-10-26 16:11:26 +0200419}
Przemek Stekiel4d13c832022-10-26 16:11:26 +0200420
Gilles Peskine15c2cbf2020-06-25 18:36:28 +0200421def main():
422 try:
423 parser = argparse.ArgumentParser(description=__doc__)
Przemek Stekiel58bbc232022-10-24 08:10:10 +0200424 parser.add_argument('outcomes', metavar='OUTCOMES.CSV',
Gilles Peskine15c2cbf2020-06-25 18:36:28 +0200425 help='Outcome file to analyze')
Przemek Stekiel542d9322022-11-17 09:43:34 +0100426 parser.add_argument('task', default='all', nargs='?',
Przemek Stekiel992de3c2022-11-09 13:54:49 +0100427 help='Analysis to be done. By default, run all tasks. '
428 'With one or more TASK, run only those. '
429 'TASK can be the name of a single task or '
Przemek Stekiel85c54ea2022-11-17 11:50:23 +0100430 'comma/space-separated list of tasks. ')
Przemek Stekiel992de3c2022-11-09 13:54:49 +0100431 parser.add_argument('--list', action='store_true',
432 help='List all available tasks and exit.')
Gilles Peskine15c2cbf2020-06-25 18:36:28 +0200433 options = parser.parse_args()
Przemek Stekiel4e955902022-10-21 13:42:08 +0200434
Przemek Stekiel992de3c2022-11-09 13:54:49 +0100435 if options.list:
436 for task in TASKS:
Valerio Setti3951d1b2023-03-13 18:37:34 +0100437 Results.log(task)
Przemek Stekiel992de3c2022-11-09 13:54:49 +0100438 sys.exit(0)
439
Przemek Stekiel4d13c832022-10-26 16:11:26 +0200440 result = True
Przemek Stekiel4e955902022-10-21 13:42:08 +0200441
Przemek Stekiel4d13c832022-10-26 16:11:26 +0200442 if options.task == 'all':
Przemek Stekield3068af2022-11-14 16:15:19 +0100443 tasks = TASKS.keys()
Przemek Stekiel992de3c2022-11-09 13:54:49 +0100444 else:
Przemek Stekiel85c54ea2022-11-17 11:50:23 +0100445 tasks = re.split(r'[, ]+', options.task)
Przemek Stekiel992de3c2022-11-09 13:54:49 +0100446
Przemek Stekield3068af2022-11-14 16:15:19 +0100447 for task in tasks:
448 if task not in TASKS:
Valerio Setti3951d1b2023-03-13 18:37:34 +0100449 Results.log('Error: invalid task: {}'.format(task))
Przemek Stekield3068af2022-11-14 16:15:19 +0100450 sys.exit(1)
Przemek Stekiel992de3c2022-11-09 13:54:49 +0100451
452 for task in TASKS:
453 if task in tasks:
Przemek Stekiel4d13c832022-10-26 16:11:26 +0200454 if not TASKS[task]['test_function'](options.outcomes, TASKS[task]['args']):
455 result = False
Przemek Stekiel4e955902022-10-21 13:42:08 +0200456
Przemek Stekielc86dedf2022-10-24 09:16:04 +0200457 if result is False:
Gilles Peskine15c2cbf2020-06-25 18:36:28 +0200458 sys.exit(1)
Valerio Setti3951d1b2023-03-13 18:37:34 +0100459 Results.log("SUCCESS :-)")
Gilles Peskine15c2cbf2020-06-25 18:36:28 +0200460 except Exception: # pylint: disable=broad-except
461 # Print the backtrace and exit explicitly with our chosen status.
462 traceback.print_exc()
463 sys.exit(120)
464
465if __name__ == '__main__':
466 main()