blob: b0b84213aa60cc0202dd866fae105ebef011e23d [file] [log] [blame]
Gilles Peskineba94b582019-09-16 19:18:40 +02001#!/usr/bin/env python3
2
3"""Sanity checks for test data.
Gilles Peskinebbb36642020-07-03 00:30:12 +02004
5This program contains a class for traversing test cases that can be used
6independently of the checks.
Gilles Peskineba94b582019-09-16 19:18:40 +02007"""
8
Bence Szépkúti1e148272020-08-07 13:07:28 +02009# Copyright The Mbed TLS Contributors
Gilles Peskineba94b582019-09-16 19:18:40 +020010# SPDX-License-Identifier: Apache-2.0
11#
12# Licensed under the Apache License, Version 2.0 (the "License"); you may
13# not use this file except in compliance with the License.
14# You may obtain a copy of the License at
15#
16# http://www.apache.org/licenses/LICENSE-2.0
17#
18# Unless required by applicable law or agreed to in writing, software
19# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
20# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21# See the License for the specific language governing permissions and
22# limitations under the License.
Gilles Peskineba94b582019-09-16 19:18:40 +020023
Gilles Peskine1fb7aea2019-12-02 14:26:04 +010024import argparse
Gilles Peskineba94b582019-09-16 19:18:40 +020025import glob
26import os
27import re
Yanray Wang23546932023-02-24 14:53:29 +080028import subprocess
Gilles Peskineba94b582019-09-16 19:18:40 +020029import sys
30
31class Results:
Darryl Green18220612019-12-17 15:03:59 +000032 """Store file and line information about errors or warnings in test suites."""
Gilles Peskine1fb7aea2019-12-02 14:26:04 +010033
34 def __init__(self, options):
Gilles Peskineba94b582019-09-16 19:18:40 +020035 self.errors = 0
36 self.warnings = 0
Gilles Peskine1fb7aea2019-12-02 14:26:04 +010037 self.ignore_warnings = options.quiet
Gilles Peskineba94b582019-09-16 19:18:40 +020038
39 def error(self, file_name, line_number, fmt, *args):
40 sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n').
41 format(file_name, line_number, *args))
42 self.errors += 1
43
44 def warning(self, file_name, line_number, fmt, *args):
Gilles Peskine1fb7aea2019-12-02 14:26:04 +010045 if not self.ignore_warnings:
46 sys.stderr.write(('{}:{}:Warning:' + fmt + '\n')
47 .format(file_name, line_number, *args))
48 self.warnings += 1
Gilles Peskineba94b582019-09-16 19:18:40 +020049
Gilles Peskine78c45db2020-06-25 16:34:11 +020050class TestDescriptionExplorer:
51 """An iterator over test cases with descriptions.
Gilles Peskined34e9e42020-06-25 16:16:25 +020052
Gilles Peskine78c45db2020-06-25 16:34:11 +020053The test cases that have descriptions are:
54* Individual unit tests (entries in a .data file) in test suites.
55* Individual test cases in ssl-opt.sh.
56
57This is an abstract class. To use it, derive a class that implements
58the process_test_case method, and call walk_all().
Gilles Peskined34e9e42020-06-25 16:16:25 +020059"""
Gilles Peskineba94b582019-09-16 19:18:40 +020060
Gilles Peskine78c45db2020-06-25 16:34:11 +020061 def process_test_case(self, per_file_state,
62 file_name, line_number, description):
63 """Process a test case.
Gilles Peskined34e9e42020-06-25 16:16:25 +020064
Gilles Peskinebbb36642020-07-03 00:30:12 +020065per_file_state: an object created by new_per_file_state() at the beginning
66 of each file.
Gilles Peskine78c45db2020-06-25 16:34:11 +020067file_name: a relative path to the file containing the test case.
68line_number: the line number in the given file.
69description: the test case description as a byte string.
Gilles Peskined34e9e42020-06-25 16:16:25 +020070"""
Gilles Peskine78c45db2020-06-25 16:34:11 +020071 raise NotImplementedError
Gilles Peskined34e9e42020-06-25 16:16:25 +020072
Gilles Peskinebbb36642020-07-03 00:30:12 +020073 def new_per_file_state(self):
Gilles Peskine78c45db2020-06-25 16:34:11 +020074 """Return a new per-file state object.
Gilles Peskined34e9e42020-06-25 16:16:25 +020075
Gilles Peskine78c45db2020-06-25 16:34:11 +020076The default per-file state object is None. Child classes that require per-file
77state may override this method.
Gilles Peskined34e9e42020-06-25 16:16:25 +020078"""
Gilles Peskine78c45db2020-06-25 16:34:11 +020079 #pylint: disable=no-self-use
80 return None
81
82 def walk_test_suite(self, data_file_name):
83 """Iterate over the test cases in the given unit test data file."""
84 in_paragraph = False
Gilles Peskinebbb36642020-07-03 00:30:12 +020085 descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
Gilles Peskine78c45db2020-06-25 16:34:11 +020086 with open(data_file_name, 'rb') as data_file:
87 for line_number, line in enumerate(data_file, 1):
88 line = line.rstrip(b'\r\n')
89 if not line:
90 in_paragraph = False
91 continue
92 if line.startswith(b'#'):
93 continue
94 if not in_paragraph:
95 # This is a test case description line.
96 self.process_test_case(descriptions,
97 data_file_name, line_number, line)
98 in_paragraph = True
99
100 def walk_ssl_opt_sh(self, file_name):
101 """Iterate over the test cases in ssl-opt.sh or a file with a similar format."""
Gilles Peskinebbb36642020-07-03 00:30:12 +0200102 descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
Gilles Peskine78c45db2020-06-25 16:34:11 +0200103 with open(file_name, 'rb') as file_contents:
104 for line_number, line in enumerate(file_contents, 1):
105 # Assume that all run_test calls have the same simple form
106 # with the test description entirely on the same line as the
107 # function name.
108 m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line)
109 if not m:
110 continue
111 description = m.group(1)
112 self.process_test_case(descriptions,
113 file_name, line_number, description)
114
Yanray Wang23546932023-02-24 14:53:29 +0800115 def walk_compat_sh(self, file_name):
116 """Iterate over the test cases compat.sh with a similar format."""
117 descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
118 compat_cmd = ['sh', file_name, '--list-test-case']
119 result = subprocess.run(compat_cmd,
120 stdout=subprocess.PIPE,
121 check=False)
122 if result.returncode != 0:
123 print(*compat_cmd, 'returned', str(result.returncode))
124 return
125 else:
Yanray Wang521710e2023-03-02 14:45:01 +0800126 # Assume compat.sh is responsible for printing identical format of
127 # test case description between --list-test-case and its OUTCOME.CSV
128 description = result.stdout.strip().split(b'\n')
129 for idx, descrip in enumerate(description):
130 self.process_test_case(descriptions, file_name, idx, descrip)
Yanray Wang23546932023-02-24 14:53:29 +0800131
Gilles Peskine6f6ff332020-06-25 16:40:10 +0200132 @staticmethod
133 def collect_test_directories():
134 """Get the relative path for the TLS and Crypto test directories."""
135 if os.path.isdir('tests'):
136 tests_dir = 'tests'
137 elif os.path.isdir('suites'):
138 tests_dir = '.'
139 elif os.path.isdir('../suites'):
140 tests_dir = '..'
141 directories = [tests_dir]
142 return directories
143
Gilles Peskine78c45db2020-06-25 16:34:11 +0200144 def walk_all(self):
145 """Iterate over all named test cases."""
Gilles Peskine6f6ff332020-06-25 16:40:10 +0200146 test_directories = self.collect_test_directories()
Gilles Peskine78c45db2020-06-25 16:34:11 +0200147 for directory in test_directories:
148 for data_file_name in glob.glob(os.path.join(directory, 'suites',
149 '*.data')):
150 self.walk_test_suite(data_file_name)
151 ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh')
152 if os.path.exists(ssl_opt_sh):
153 self.walk_ssl_opt_sh(ssl_opt_sh)
Jerry Yuf17a60f2021-11-26 20:40:17 +0800154 for ssl_opt_file_name in glob.glob(os.path.join(directory, 'opt-testcases',
155 '*.sh')):
156 self.walk_ssl_opt_sh(ssl_opt_file_name)
Yanray Wang23546932023-02-24 14:53:29 +0800157 compat_sh = os.path.join(directory, 'compat.sh')
158 if os.path.exists(compat_sh):
159 self.walk_compat_sh(compat_sh)
Gilles Peskine78c45db2020-06-25 16:34:11 +0200160
Gilles Peskine686c2922022-01-07 15:58:38 +0100161class TestDescriptions(TestDescriptionExplorer):
162 """Collect the available test cases."""
163
164 def __init__(self):
165 super().__init__()
166 self.descriptions = set()
167
168 def process_test_case(self, _per_file_state,
169 file_name, _line_number, description):
170 """Record an available test case."""
171 base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name))
172 key = ';'.join([base_name, description.decode('utf-8')])
173 self.descriptions.add(key)
174
175def collect_available_test_cases():
176 """Collect the available test cases."""
177 explorer = TestDescriptions()
178 explorer.walk_all()
179 return sorted(explorer.descriptions)
180
Gilles Peskine78c45db2020-06-25 16:34:11 +0200181class DescriptionChecker(TestDescriptionExplorer):
182 """Check all test case descriptions.
183
184* Check that each description is valid (length, allowed character set, etc.).
185* Check that there is no duplicated description inside of one test suite.
186"""
187
188 def __init__(self, results):
189 self.results = results
190
Gilles Peskinebbb36642020-07-03 00:30:12 +0200191 def new_per_file_state(self):
Gilles Peskine6f6ff332020-06-25 16:40:10 +0200192 """Dictionary mapping descriptions to their line number."""
Gilles Peskine78c45db2020-06-25 16:34:11 +0200193 return {}
194
195 def process_test_case(self, per_file_state,
196 file_name, line_number, description):
Gilles Peskine6f6ff332020-06-25 16:40:10 +0200197 """Check test case descriptions for errors."""
198 results = self.results
199 seen = per_file_state
200 if description in seen:
201 results.error(file_name, line_number,
202 'Duplicate description (also line {})',
203 seen[description])
204 return
205 if re.search(br'[\t;]', description):
206 results.error(file_name, line_number,
207 'Forbidden character \'{}\' in description',
208 re.search(br'[\t;]', description).group(0).decode('ascii'))
209 if re.search(br'[^ -~]', description):
210 results.error(file_name, line_number,
211 'Non-ASCII character in description')
212 if len(description) > 66:
213 results.warning(file_name, line_number,
214 'Test description too long ({} > 66)',
215 len(description))
216 seen[description] = line_number
Gilles Peskineba94b582019-09-16 19:18:40 +0200217
218def main():
Gilles Peskine1fb7aea2019-12-02 14:26:04 +0100219 parser = argparse.ArgumentParser(description=__doc__)
Gilles Peskine7e091052022-01-07 15:58:55 +0100220 parser.add_argument('--list-all',
221 action='store_true',
222 help='List all test cases, without doing checks')
Gilles Peskine1fb7aea2019-12-02 14:26:04 +0100223 parser.add_argument('--quiet', '-q',
224 action='store_true',
225 help='Hide warnings')
226 parser.add_argument('--verbose', '-v',
227 action='store_false', dest='quiet',
228 help='Show warnings (default: on; undoes --quiet)')
229 options = parser.parse_args()
Gilles Peskine7e091052022-01-07 15:58:55 +0100230 if options.list_all:
231 descriptions = collect_available_test_cases()
232 sys.stdout.write('\n'.join(descriptions + ['']))
233 return
Gilles Peskine1fb7aea2019-12-02 14:26:04 +0100234 results = Results(options)
Gilles Peskine78c45db2020-06-25 16:34:11 +0200235 checker = DescriptionChecker(results)
236 checker.walk_all()
Gilles Peskine1fb7aea2019-12-02 14:26:04 +0100237 if (results.warnings or results.errors) and not options.quiet:
Gilles Peskineba94b582019-09-16 19:18:40 +0200238 sys.stderr.write('{}: {} errors, {} warnings\n'
239 .format(sys.argv[0], results.errors, results.warnings))
240 sys.exit(1 if results.errors else 0)
241
242if __name__ == '__main__':
243 main()