blob: b68fdf78c53730c679ea12e9679a5c0f3632ef07 [file] [log] [blame]
Gilles Peskine33406b62023-11-02 18:48:39 +01001/** \file metatest.c
2 *
3 * \brief Test features of the test framework.
Gilles Peskinecce00122023-11-10 15:36:15 +01004 *
5 * When you run this program, it runs a single "meta-test". A meta-test
6 * performs an operation which should be caught as a failure by our
7 * test framework. The meta-test passes if this program calls `exit` with
8 * a nonzero status, or aborts, or is terminated by a signal, or if the
9 * framework running the program considers the run an error (this happens
10 * with Valgrind for a memory leak). The non-success of the meta-test
11 * program means that the test failure has been caught correctly.
12 *
13 * Some failures are purely functional: the logic of the code causes the
14 * test result to be set to FAIL. Other failures come from extra
15 * instrumentation which is not present in a normal build; for example,
16 * Asan or Valgrind to detect memory leaks. This is reflected by the
17 * "platform" associated with each meta-test.
18 *
19 * Use the companion script `tests/scripts/run-metatests.sh` to run all
20 * the meta-tests for a given platform and validate that they trigger a
21 * detected failure as expected.
Gilles Peskine33406b62023-11-02 18:48:39 +010022 */
23
24/*
25 * Copyright The Mbed TLS Contributors
26 * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
27 */
28
29#define MBEDTLS_ALLOW_PRIVATE_ACCESS
30
31#include <mbedtls/platform.h>
Gilles Peskine69e8db02023-11-02 19:52:32 +010032#include <mbedtls/platform_util.h>
Gilles Peskine33406b62023-11-02 18:48:39 +010033#include "test/helpers.h"
Paul Elliott17c119a2023-12-08 16:55:03 +000034#include "test/threading_helpers.h"
Gilles Peskine102aea22023-11-03 18:05:38 +010035#include "test/macros.h"
Gilles Peskine33406b62023-11-02 18:48:39 +010036
37#include <stdio.h>
38#include <string.h>
39
Gilles Peskine102aea22023-11-03 18:05:38 +010040#if defined(MBEDTLS_THREADING_C)
41#include <mbedtls/threading.h>
42#endif
43
Gilles Peskine33406b62023-11-02 18:48:39 +010044
Gilles Peskine69e8db02023-11-02 19:52:32 +010045/* This is an external variable, so the compiler doesn't know that we're never
46 * changing its value.
Gilles Peskine69e8db02023-11-02 19:52:32 +010047 */
Gilles Peskined2fa6982023-11-09 21:46:24 +010048volatile int false_but_the_compiler_does_not_know = 0;
49
Gilles Peskine7a715c42023-11-21 13:42:40 +010050/* Hide calls to calloc/free from static checkers such as
51 * `gcc-12 -Wuse-after-free`, to avoid compile-time complaints about
52 * code where we do mean to cause a runtime error. */
53void * (* volatile calloc_but_the_compiler_does_not_know)(size_t, size_t) = mbedtls_calloc;
54void(*volatile free_but_the_compiler_does_not_know)(void *) = mbedtls_free;
55
Gilles Peskined2fa6982023-11-09 21:46:24 +010056/* Set n bytes at the address p to all-bits-zero, in such a way that
57 * the compiler should not know that p is all-bits-zero. */
Gilles Peskineda6e7a22023-11-10 10:09:27 +010058static void set_to_zero_but_the_compiler_does_not_know(volatile void *p, size_t n)
Gilles Peskined2fa6982023-11-09 21:46:24 +010059{
Gilles Peskineda6e7a22023-11-10 10:09:27 +010060 memset((void *) p, false_but_the_compiler_does_not_know, n);
Gilles Peskined2fa6982023-11-09 21:46:24 +010061}
Gilles Peskine69e8db02023-11-02 19:52:32 +010062
63
Gilles Peskine33406b62023-11-02 18:48:39 +010064/****************************************************************/
Gilles Peskinef309fbf2023-11-02 18:49:52 +010065/* Test framework features */
66/****************************************************************/
67
68void meta_test_fail(const char *name)
69{
70 (void) name;
71 mbedtls_test_fail("Forced test failure", __LINE__, __FILE__);
72}
73
Paul Elliott60bbfe62024-02-13 15:06:10 +000074void meta_test_not_equal(const char *name)
75{
76 int left = 20;
77 int right = 10;
78
79 (void) name;
80
81 TEST_EQUAL(left, right);
82exit:
83 ;
84}
85
86void meta_test_not_le_s(const char *name)
87{
88 int left = 20;
89 int right = 10;
90
91 (void) name;
92
93 TEST_LE_S(left, right);
94exit:
95 ;
96}
97
98void meta_test_not_le_u(const char *name)
99{
100 size_t left = 20;
101 size_t right = 10;
102
103 (void) name;
104
105 TEST_LE_U(left, right);
106exit:
107 ;
108}
Gilles Peskinef309fbf2023-11-02 18:49:52 +0100109
110/****************************************************************/
Gilles Peskine80ba8322023-11-02 19:23:26 +0100111/* Platform features */
112/****************************************************************/
113
114void null_pointer_dereference(const char *name)
115{
116 (void) name;
Gilles Peskineda6e7a22023-11-10 10:09:27 +0100117 volatile char *volatile p;
Gilles Peskined2fa6982023-11-09 21:46:24 +0100118 set_to_zero_but_the_compiler_does_not_know(&p, sizeof(p));
Gilles Peskine2f40cc02023-11-16 15:11:44 +0100119 /* Undefined behavior (read from null data pointer) */
Gilles Peskine69e8db02023-11-02 19:52:32 +0100120 mbedtls_printf("%p -> %u\n", p, (unsigned) *p);
Gilles Peskine80ba8322023-11-02 19:23:26 +0100121}
122
123void null_pointer_call(const char *name)
124{
125 (void) name;
Gilles Peskineda6e7a22023-11-10 10:09:27 +0100126 unsigned(*volatile p)(void);
Gilles Peskined2fa6982023-11-09 21:46:24 +0100127 set_to_zero_but_the_compiler_does_not_know(&p, sizeof(p));
Gilles Peskine2f40cc02023-11-16 15:11:44 +0100128 /* Undefined behavior (execute null function pointer) */
Gilles Peskinef0d5cf92023-11-03 10:58:57 +0100129 /* The pointer representation may be truncated, but we don't care:
130 * the only point of printing it is to have some use of the pointer
131 * to dissuade the compiler from optimizing it away. */
132 mbedtls_printf("%lx() -> %u\n", (unsigned long) (uintptr_t) p, p());
Gilles Peskine80ba8322023-11-02 19:23:26 +0100133}
134
135
136/****************************************************************/
Gilles Peskine102aea22023-11-03 18:05:38 +0100137/* Memory */
Gilles Peskineb0f0a642023-11-02 19:42:13 +0100138/****************************************************************/
139
140void read_after_free(const char *name)
141{
142 (void) name;
Gilles Peskine7a715c42023-11-21 13:42:40 +0100143 volatile char *p = calloc_but_the_compiler_does_not_know(1, 1);
Gilles Peskineb0f0a642023-11-02 19:42:13 +0100144 *p = 'a';
Gilles Peskine7a715c42023-11-21 13:42:40 +0100145 free_but_the_compiler_does_not_know((void *) p);
Gilles Peskine2f40cc02023-11-16 15:11:44 +0100146 /* Undefined behavior (read after free) */
Gilles Peskineb0f0a642023-11-02 19:42:13 +0100147 mbedtls_printf("%u\n", (unsigned) *p);
148}
149
150void double_free(const char *name)
151{
152 (void) name;
Gilles Peskine7a715c42023-11-21 13:42:40 +0100153 volatile char *p = calloc_but_the_compiler_does_not_know(1, 1);
Gilles Peskineb0f0a642023-11-02 19:42:13 +0100154 *p = 'a';
Gilles Peskine7a715c42023-11-21 13:42:40 +0100155 free_but_the_compiler_does_not_know((void *) p);
Gilles Peskine2f40cc02023-11-16 15:11:44 +0100156 /* Undefined behavior (double free) */
Gilles Peskine7a715c42023-11-21 13:42:40 +0100157 free_but_the_compiler_does_not_know((void *) p);
Gilles Peskineb0f0a642023-11-02 19:42:13 +0100158}
159
160void read_uninitialized_stack(const char *name)
161{
162 (void) name;
Gilles Peskineccb12152023-11-10 11:35:36 +0100163 char buf[1];
Gilles Peskineb0f0a642023-11-02 19:42:13 +0100164 if (false_but_the_compiler_does_not_know) {
165 buf[0] = '!';
166 }
Gilles Peskineccb12152023-11-10 11:35:36 +0100167 char *volatile p = buf;
168 if (*p != 0) {
Gilles Peskine2f40cc02023-11-16 15:11:44 +0100169 /* Unspecified result (read from uninitialized memory) */
Gilles Peskineccb12152023-11-10 11:35:36 +0100170 mbedtls_printf("%u\n", (unsigned) *p);
Gilles Peskineb0f0a642023-11-02 19:42:13 +0100171 }
172}
173
174void memory_leak(const char *name)
175{
176 (void) name;
Gilles Peskine7a715c42023-11-21 13:42:40 +0100177 volatile char *p = calloc_but_the_compiler_does_not_know(1, 1);
Gilles Peskined2fa6982023-11-09 21:46:24 +0100178 mbedtls_printf("%u\n", (unsigned) *p);
Gilles Peskine2f40cc02023-11-16 15:11:44 +0100179 /* Leak of a heap object */
Gilles Peskineb0f0a642023-11-02 19:42:13 +0100180}
181
182
183/****************************************************************/
Gilles Peskine102aea22023-11-03 18:05:38 +0100184/* Threading */
185/****************************************************************/
186
187void mutex_lock_not_initialized(const char *name)
188{
189 (void) name;
Gilles Peskinead2a17e2023-11-16 15:09:48 +0100190#if defined(MBEDTLS_THREADING_C)
Gilles Peskine102aea22023-11-03 18:05:38 +0100191 mbedtls_threading_mutex_t mutex;
192 memset(&mutex, 0, sizeof(mutex));
Gilles Peskine2f40cc02023-11-16 15:11:44 +0100193 /* This mutex usage error is detected by our test framework's mutex usage
194 * verification framework. See tests/src/threading_helpers.c. Other
195 * threading implementations (e.g. pthread without our instrumentation)
196 * might consider this normal usage. */
Gilles Peskine102aea22023-11-03 18:05:38 +0100197 TEST_ASSERT(mbedtls_mutex_lock(&mutex) == 0);
198exit:
199 ;
200#endif
201}
202
203void mutex_unlock_not_initialized(const char *name)
204{
205 (void) name;
Gilles Peskine102aea22023-11-03 18:05:38 +0100206#if defined(MBEDTLS_THREADING_C)
207 mbedtls_threading_mutex_t mutex;
208 memset(&mutex, 0, sizeof(mutex));
Gilles Peskine2f40cc02023-11-16 15:11:44 +0100209 /* This mutex usage error is detected by our test framework's mutex usage
210 * verification framework. See tests/src/threading_helpers.c. Other
211 * threading implementations (e.g. pthread without our instrumentation)
212 * might consider this normal usage. */
Gilles Peskine102aea22023-11-03 18:05:38 +0100213 TEST_ASSERT(mbedtls_mutex_unlock(&mutex) == 0);
214exit:
215 ;
216#endif
217}
218
219void mutex_free_not_initialized(const char *name)
220{
221 (void) name;
Gilles Peskine102aea22023-11-03 18:05:38 +0100222#if defined(MBEDTLS_THREADING_C)
223 mbedtls_threading_mutex_t mutex;
224 memset(&mutex, 0, sizeof(mutex));
Gilles Peskine2f40cc02023-11-16 15:11:44 +0100225 /* This mutex usage error is detected by our test framework's mutex usage
226 * verification framework. See tests/src/threading_helpers.c. Other
227 * threading implementations (e.g. pthread without our instrumentation)
228 * might consider this normal usage. */
Gilles Peskine102aea22023-11-03 18:05:38 +0100229 mbedtls_mutex_free(&mutex);
230#endif
231}
232
233void mutex_double_init(const char *name)
234{
235 (void) name;
236#if defined(MBEDTLS_THREADING_C)
237 mbedtls_threading_mutex_t mutex;
238 mbedtls_mutex_init(&mutex);
Gilles Peskine2f40cc02023-11-16 15:11:44 +0100239 /* This mutex usage error is detected by our test framework's mutex usage
240 * verification framework. See tests/src/threading_helpers.c. Other
241 * threading implementations (e.g. pthread without our instrumentation)
242 * might consider this normal usage. */
Gilles Peskine102aea22023-11-03 18:05:38 +0100243 mbedtls_mutex_init(&mutex);
244 mbedtls_mutex_free(&mutex);
245#endif
246}
247
248void mutex_double_free(const char *name)
249{
250 (void) name;
251#if defined(MBEDTLS_THREADING_C)
252 mbedtls_threading_mutex_t mutex;
253 mbedtls_mutex_init(&mutex);
254 mbedtls_mutex_free(&mutex);
Gilles Peskine2f40cc02023-11-16 15:11:44 +0100255 /* This mutex usage error is detected by our test framework's mutex usage
256 * verification framework. See tests/src/threading_helpers.c. Other
257 * threading implementations (e.g. pthread without our instrumentation)
258 * might consider this normal usage. */
Gilles Peskine102aea22023-11-03 18:05:38 +0100259 mbedtls_mutex_free(&mutex);
260#endif
261}
262
263void mutex_leak(const char *name)
264{
265 (void) name;
Gilles Peskinead2a17e2023-11-16 15:09:48 +0100266#if defined(MBEDTLS_THREADING_C)
Gilles Peskine102aea22023-11-03 18:05:38 +0100267 mbedtls_threading_mutex_t mutex;
268 mbedtls_mutex_init(&mutex);
269#endif
Gilles Peskine2f40cc02023-11-16 15:11:44 +0100270 /* This mutex usage error is detected by our test framework's mutex usage
271 * verification framework. See tests/src/threading_helpers.c. Other
272 * threading implementations (e.g. pthread without our instrumentation)
273 * might consider this normal usage. */
Gilles Peskine102aea22023-11-03 18:05:38 +0100274}
275
276
277/****************************************************************/
Gilles Peskine33406b62023-11-02 18:48:39 +0100278/* Command line entry point */
279/****************************************************************/
280
281typedef struct {
Gilles Peskinecce00122023-11-10 15:36:15 +0100282 /** Command line argument that will trigger that metatest.
283 *
284 * Conventionally matches "[a-z0-9_]+". */
Gilles Peskine33406b62023-11-02 18:48:39 +0100285 const char *name;
Gilles Peskinecce00122023-11-10 15:36:15 +0100286
287 /** Platform under which that metatest is valid.
288 *
289 * - "any": should work anywhere.
290 * - "asan": triggers ASan (Address Sanitizer).
291 * - "msan": triggers MSan (Memory Sanitizer).
Gilles Peskine2f40cc02023-11-16 15:11:44 +0100292 * - "pthread": requires MBEDTLS_THREADING_PTHREAD and MBEDTLS_TEST_HOOKS,
293 * which enables MBEDTLS_TEST_MUTEX_USAGE internally in the test
294 * framework (see tests/src/threading_helpers.c).
Gilles Peskinecce00122023-11-10 15:36:15 +0100295 */
Gilles Peskine33406b62023-11-02 18:48:39 +0100296 const char *platform;
Gilles Peskinecce00122023-11-10 15:36:15 +0100297
298 /** Function that performs the metatest.
299 *
300 * The function receives the name as an argument. This allows using the
301 * same function to perform multiple variants of a test based on the name.
302 *
303 * When executed on a conforming platform, the function is expected to
304 * either cause a test failure (mbedtls_test_fail()), or cause the
305 * program to abort in some way (e.g. by causing a segfault or by
306 * triggering a sanitizer).
307 *
308 * When executed on a non-conforming platform, the function may return
309 * normally or may have unpredictable behavior.
310 */
Gilles Peskine33406b62023-11-02 18:48:39 +0100311 void (*entry_point)(const char *name);
312} metatest_t;
313
Gilles Peskinecce00122023-11-10 15:36:15 +0100314/* The list of availble meta-tests. Remember to register new functions here!
315 *
316 * Note that we always compile all the functions, so that `metatest --list`
317 * will always list all the available meta-tests.
Gilles Peskine2f40cc02023-11-16 15:11:44 +0100318 *
319 * See the documentation of metatest_t::platform for the meaning of
320 * platform values.
Gilles Peskinecce00122023-11-10 15:36:15 +0100321 */
Gilles Peskine33406b62023-11-02 18:48:39 +0100322metatest_t metatests[] = {
Gilles Peskinef309fbf2023-11-02 18:49:52 +0100323 { "test_fail", "any", meta_test_fail },
Paul Elliott60bbfe62024-02-13 15:06:10 +0000324 { "test_not_equal", "any", meta_test_not_equal },
325 { "test_not_le_s", "any", meta_test_not_le_s },
326 { "test_not_le_u", "any", meta_test_not_le_u },
Gilles Peskine80ba8322023-11-02 19:23:26 +0100327 { "null_dereference", "any", null_pointer_dereference },
328 { "null_call", "any", null_pointer_call },
Gilles Peskineb0f0a642023-11-02 19:42:13 +0100329 { "read_after_free", "asan", read_after_free },
330 { "double_free", "asan", double_free },
331 { "read_uninitialized_stack", "msan", read_uninitialized_stack },
332 { "memory_leak", "asan", memory_leak },
Gilles Peskine102aea22023-11-03 18:05:38 +0100333 { "mutex_lock_not_initialized", "pthread", mutex_lock_not_initialized },
334 { "mutex_unlock_not_initialized", "pthread", mutex_unlock_not_initialized },
335 { "mutex_free_not_initialized", "pthread", mutex_free_not_initialized },
336 { "mutex_double_init", "pthread", mutex_double_init },
337 { "mutex_double_free", "pthread", mutex_double_free },
338 { "mutex_leak", "pthread", mutex_leak },
Gilles Peskine33406b62023-11-02 18:48:39 +0100339 { NULL, NULL, NULL }
340};
341
342static void help(FILE *out, const char *argv0)
343{
344 mbedtls_fprintf(out, "Usage: %s list|TEST\n", argv0);
345 mbedtls_fprintf(out, "Run a meta-test that should cause a test failure.\n");
346 mbedtls_fprintf(out, "With 'list', list the available tests and their platform requirement.\n");
347}
348
349int main(int argc, char *argv[])
350{
351 const char *argv0 = argc > 0 ? argv[0] : "metatest";
352 if (argc != 2) {
353 help(stderr, argv0);
354 mbedtls_exit(MBEDTLS_EXIT_FAILURE);
355 }
356
357 /* Support "-help", "--help", "--list", etc. */
358 const char *command = argv[1];
359 while (*command == '-') {
360 ++command;
361 }
362
363 if (strcmp(argv[1], "help") == 0) {
364 help(stdout, argv0);
365 mbedtls_exit(MBEDTLS_EXIT_SUCCESS);
366 }
367 if (strcmp(argv[1], "list") == 0) {
368 for (const metatest_t *p = metatests; p->name != NULL; p++) {
369 mbedtls_printf("%s %s\n", p->name, p->platform);
370 }
371 mbedtls_exit(MBEDTLS_EXIT_SUCCESS);
372 }
373
Gilles Peskine102aea22023-11-03 18:05:38 +0100374#if defined(MBEDTLS_TEST_MUTEX_USAGE)
375 mbedtls_test_mutex_usage_init();
376#endif
377
Gilles Peskine33406b62023-11-02 18:48:39 +0100378 for (const metatest_t *p = metatests; p->name != NULL; p++) {
379 if (strcmp(argv[1], p->name) == 0) {
380 mbedtls_printf("Running metatest %s...\n", argv[1]);
381 p->entry_point(argv[1]);
Gilles Peskine102aea22023-11-03 18:05:38 +0100382#if defined(MBEDTLS_TEST_MUTEX_USAGE)
383 mbedtls_test_mutex_usage_check();
384#endif
Paul Elliott4580d4d2023-10-27 18:41:02 +0100385 int result = (int) mbedtls_test_get_result();
386
Gilles Peskine33406b62023-11-02 18:48:39 +0100387 mbedtls_printf("Running metatest %s... done, result=%d\n",
Paul Elliott4580d4d2023-10-27 18:41:02 +0100388 argv[1], result);
389 mbedtls_exit(result == MBEDTLS_TEST_RESULT_SUCCESS ?
Gilles Peskine33406b62023-11-02 18:48:39 +0100390 MBEDTLS_EXIT_SUCCESS :
391 MBEDTLS_EXIT_FAILURE);
392 }
393 }
394
395 mbedtls_fprintf(stderr, "%s: FATAL: No such metatest: %s\n",
396 argv0, command);
397 mbedtls_exit(MBEDTLS_EXIT_FAILURE);
398}