Merge pull request #8463 from gilles-peskine-arm/metatest-create
Create a metatest program
diff --git a/programs/.gitignore b/programs/.gitignore
index a641c31..e0c4987 100644
--- a/programs/.gitignore
+++ b/programs/.gitignore
@@ -38,6 +38,7 @@
psa/hmac_demo
psa/key_ladder_demo
psa/psa_constant_names
+psa/psa_hash
random/gen_entropy
random/gen_random_ctr_drbg
ssl/dtls_client
@@ -56,6 +57,7 @@
test/cpp_dummy_build.cpp
test/dlopen
test/ecp-bench
+test/metatest
test/query_compile_time_config
test/query_included_headers
test/selftest
diff --git a/programs/Makefile b/programs/Makefile
index 116883b..a3fa816 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -123,6 +123,7 @@
ssl/ssl_server \
ssl/ssl_server2 \
test/benchmark \
+ test/metatest \
test/query_compile_time_config \
test/query_included_headers \
test/selftest \
@@ -413,6 +414,10 @@
$(CC) $(LOCAL_CFLAGS) $(CFLAGS) test/dlopen.c $(LDFLAGS) $(DLOPEN_LDFLAGS) -o $@
endif
+test/metatest$(EXEXT): test/metatest.c $(DEP)
+ echo " CC test/metatest.c"
+ $(CC) $(LOCAL_CFLAGS) $(CFLAGS) -I ../library test/metatest.c $(LOCAL_LDFLAGS) $(LDFLAGS) -o $@
+
test/query_config.o: test/query_config.c test/query_config.h $(DEP)
echo " CC test/query_config.c"
$(CC) $(LOCAL_CFLAGS) $(CFLAGS) -c test/query_config.c -o $@
diff --git a/programs/test/CMakeLists.txt b/programs/test/CMakeLists.txt
index a75f8d9..0778731 100644
--- a/programs/test/CMakeLists.txt
+++ b/programs/test/CMakeLists.txt
@@ -3,6 +3,7 @@
)
set(executables_libs
+ metatest
query_included_headers
selftest
udp_proxy
@@ -72,6 +73,7 @@
add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>
${extra_sources})
target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../tests/include)
+ target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../library)
if(exe STREQUAL "query_compile_time_config")
target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
endif()
diff --git a/programs/test/metatest.c b/programs/test/metatest.c
new file mode 100644
index 0000000..2973cce
--- /dev/null
+++ b/programs/test/metatest.c
@@ -0,0 +1,351 @@
+/** \file metatest.c
+ *
+ * \brief Test features of the test framework.
+ *
+ * When you run this program, it runs a single "meta-test". A meta-test
+ * performs an operation which should be caught as a failure by our
+ * test framework. The meta-test passes if this program calls `exit` with
+ * a nonzero status, or aborts, or is terminated by a signal, or if the
+ * framework running the program considers the run an error (this happens
+ * with Valgrind for a memory leak). The non-success of the meta-test
+ * program means that the test failure has been caught correctly.
+ *
+ * Some failures are purely functional: the logic of the code causes the
+ * test result to be set to FAIL. Other failures come from extra
+ * instrumentation which is not present in a normal build; for example,
+ * Asan or Valgrind to detect memory leaks. This is reflected by the
+ * "platform" associated with each meta-test.
+ *
+ * Use the companion script `tests/scripts/run-metatests.sh` to run all
+ * the meta-tests for a given platform and validate that they trigger a
+ * detected failure as expected.
+ */
+
+/*
+ * Copyright The Mbed TLS Contributors
+ * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+ */
+
+#define MBEDTLS_ALLOW_PRIVATE_ACCESS
+
+#include <mbedtls/platform.h>
+#include <mbedtls/platform_util.h>
+#include "test/helpers.h"
+#include "test/macros.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#if defined(MBEDTLS_THREADING_C)
+#include <mbedtls/threading.h>
+#endif
+
+
+/* This is an external variable, so the compiler doesn't know that we're never
+ * changing its value.
+ */
+volatile int false_but_the_compiler_does_not_know = 0;
+
+/* Set n bytes at the address p to all-bits-zero, in such a way that
+ * the compiler should not know that p is all-bits-zero. */
+static void set_to_zero_but_the_compiler_does_not_know(volatile void *p, size_t n)
+{
+ memset((void *) p, false_but_the_compiler_does_not_know, n);
+}
+
+
+/****************************************************************/
+/* Test framework features */
+/****************************************************************/
+
+void meta_test_fail(const char *name)
+{
+ (void) name;
+ mbedtls_test_fail("Forced test failure", __LINE__, __FILE__);
+}
+
+
+/****************************************************************/
+/* Platform features */
+/****************************************************************/
+
+void null_pointer_dereference(const char *name)
+{
+ (void) name;
+ volatile char *volatile p;
+ set_to_zero_but_the_compiler_does_not_know(&p, sizeof(p));
+ /* Undefined behavior (read from null data pointer) */
+ mbedtls_printf("%p -> %u\n", p, (unsigned) *p);
+}
+
+void null_pointer_call(const char *name)
+{
+ (void) name;
+ unsigned(*volatile p)(void);
+ set_to_zero_but_the_compiler_does_not_know(&p, sizeof(p));
+ /* Undefined behavior (execute null function pointer) */
+ /* The pointer representation may be truncated, but we don't care:
+ * the only point of printing it is to have some use of the pointer
+ * to dissuade the compiler from optimizing it away. */
+ mbedtls_printf("%lx() -> %u\n", (unsigned long) (uintptr_t) p, p());
+}
+
+
+/****************************************************************/
+/* Memory */
+/****************************************************************/
+
+void read_after_free(const char *name)
+{
+ (void) name;
+ volatile char *p = mbedtls_calloc(1, 1);
+ *p = 'a';
+ mbedtls_free((void *) p);
+ /* Undefined behavior (read after free) */
+ mbedtls_printf("%u\n", (unsigned) *p);
+}
+
+void double_free(const char *name)
+{
+ (void) name;
+ volatile char *p = mbedtls_calloc(1, 1);
+ *p = 'a';
+ mbedtls_free((void *) p);
+ /* Undefined behavior (double free) */
+ mbedtls_free((void *) p);
+}
+
+void read_uninitialized_stack(const char *name)
+{
+ (void) name;
+ char buf[1];
+ if (false_but_the_compiler_does_not_know) {
+ buf[0] = '!';
+ }
+ char *volatile p = buf;
+ if (*p != 0) {
+ /* Unspecified result (read from uninitialized memory) */
+ mbedtls_printf("%u\n", (unsigned) *p);
+ }
+}
+
+void memory_leak(const char *name)
+{
+ (void) name;
+ volatile char *p = mbedtls_calloc(1, 1);
+ mbedtls_printf("%u\n", (unsigned) *p);
+ /* Leak of a heap object */
+}
+
+
+/****************************************************************/
+/* Threading */
+/****************************************************************/
+
+void mutex_lock_not_initialized(const char *name)
+{
+ (void) name;
+#if defined(MBEDTLS_THREADING_C)
+ mbedtls_threading_mutex_t mutex;
+ memset(&mutex, 0, sizeof(mutex));
+ /* This mutex usage error is detected by our test framework's mutex usage
+ * verification framework. See tests/src/threading_helpers.c. Other
+ * threading implementations (e.g. pthread without our instrumentation)
+ * might consider this normal usage. */
+ TEST_ASSERT(mbedtls_mutex_lock(&mutex) == 0);
+exit:
+ ;
+#endif
+}
+
+void mutex_unlock_not_initialized(const char *name)
+{
+ (void) name;
+#if defined(MBEDTLS_THREADING_C)
+ mbedtls_threading_mutex_t mutex;
+ memset(&mutex, 0, sizeof(mutex));
+ /* This mutex usage error is detected by our test framework's mutex usage
+ * verification framework. See tests/src/threading_helpers.c. Other
+ * threading implementations (e.g. pthread without our instrumentation)
+ * might consider this normal usage. */
+ TEST_ASSERT(mbedtls_mutex_unlock(&mutex) == 0);
+exit:
+ ;
+#endif
+}
+
+void mutex_free_not_initialized(const char *name)
+{
+ (void) name;
+#if defined(MBEDTLS_THREADING_C)
+ mbedtls_threading_mutex_t mutex;
+ memset(&mutex, 0, sizeof(mutex));
+ /* This mutex usage error is detected by our test framework's mutex usage
+ * verification framework. See tests/src/threading_helpers.c. Other
+ * threading implementations (e.g. pthread without our instrumentation)
+ * might consider this normal usage. */
+ mbedtls_mutex_free(&mutex);
+#endif
+}
+
+void mutex_double_init(const char *name)
+{
+ (void) name;
+#if defined(MBEDTLS_THREADING_C)
+ mbedtls_threading_mutex_t mutex;
+ mbedtls_mutex_init(&mutex);
+ /* This mutex usage error is detected by our test framework's mutex usage
+ * verification framework. See tests/src/threading_helpers.c. Other
+ * threading implementations (e.g. pthread without our instrumentation)
+ * might consider this normal usage. */
+ mbedtls_mutex_init(&mutex);
+ mbedtls_mutex_free(&mutex);
+#endif
+}
+
+void mutex_double_free(const char *name)
+{
+ (void) name;
+#if defined(MBEDTLS_THREADING_C)
+ mbedtls_threading_mutex_t mutex;
+ mbedtls_mutex_init(&mutex);
+ mbedtls_mutex_free(&mutex);
+ /* This mutex usage error is detected by our test framework's mutex usage
+ * verification framework. See tests/src/threading_helpers.c. Other
+ * threading implementations (e.g. pthread without our instrumentation)
+ * might consider this normal usage. */
+ mbedtls_mutex_free(&mutex);
+#endif
+}
+
+void mutex_leak(const char *name)
+{
+ (void) name;
+#if defined(MBEDTLS_THREADING_C)
+ mbedtls_threading_mutex_t mutex;
+ mbedtls_mutex_init(&mutex);
+#endif
+ /* This mutex usage error is detected by our test framework's mutex usage
+ * verification framework. See tests/src/threading_helpers.c. Other
+ * threading implementations (e.g. pthread without our instrumentation)
+ * might consider this normal usage. */
+}
+
+
+/****************************************************************/
+/* Command line entry point */
+/****************************************************************/
+
+typedef struct {
+ /** Command line argument that will trigger that metatest.
+ *
+ * Conventionally matches "[a-z0-9_]+". */
+ const char *name;
+
+ /** Platform under which that metatest is valid.
+ *
+ * - "any": should work anywhere.
+ * - "asan": triggers ASan (Address Sanitizer).
+ * - "msan": triggers MSan (Memory Sanitizer).
+ * - "pthread": requires MBEDTLS_THREADING_PTHREAD and MBEDTLS_TEST_HOOKS,
+ * which enables MBEDTLS_TEST_MUTEX_USAGE internally in the test
+ * framework (see tests/src/threading_helpers.c).
+ */
+ const char *platform;
+
+ /** Function that performs the metatest.
+ *
+ * The function receives the name as an argument. This allows using the
+ * same function to perform multiple variants of a test based on the name.
+ *
+ * When executed on a conforming platform, the function is expected to
+ * either cause a test failure (mbedtls_test_fail()), or cause the
+ * program to abort in some way (e.g. by causing a segfault or by
+ * triggering a sanitizer).
+ *
+ * When executed on a non-conforming platform, the function may return
+ * normally or may have unpredictable behavior.
+ */
+ void (*entry_point)(const char *name);
+} metatest_t;
+
+/* The list of availble meta-tests. Remember to register new functions here!
+ *
+ * Note that we always compile all the functions, so that `metatest --list`
+ * will always list all the available meta-tests.
+ *
+ * See the documentation of metatest_t::platform for the meaning of
+ * platform values.
+ */
+metatest_t metatests[] = {
+ { "test_fail", "any", meta_test_fail },
+ { "null_dereference", "any", null_pointer_dereference },
+ { "null_call", "any", null_pointer_call },
+ { "read_after_free", "asan", read_after_free },
+ { "double_free", "asan", double_free },
+ { "read_uninitialized_stack", "msan", read_uninitialized_stack },
+ { "memory_leak", "asan", memory_leak },
+ { "mutex_lock_not_initialized", "pthread", mutex_lock_not_initialized },
+ { "mutex_unlock_not_initialized", "pthread", mutex_unlock_not_initialized },
+ { "mutex_free_not_initialized", "pthread", mutex_free_not_initialized },
+ { "mutex_double_init", "pthread", mutex_double_init },
+ { "mutex_double_free", "pthread", mutex_double_free },
+ { "mutex_leak", "pthread", mutex_leak },
+ { NULL, NULL, NULL }
+};
+
+static void help(FILE *out, const char *argv0)
+{
+ mbedtls_fprintf(out, "Usage: %s list|TEST\n", argv0);
+ mbedtls_fprintf(out, "Run a meta-test that should cause a test failure.\n");
+ mbedtls_fprintf(out, "With 'list', list the available tests and their platform requirement.\n");
+}
+
+int main(int argc, char *argv[])
+{
+ const char *argv0 = argc > 0 ? argv[0] : "metatest";
+ if (argc != 2) {
+ help(stderr, argv0);
+ mbedtls_exit(MBEDTLS_EXIT_FAILURE);
+ }
+
+ /* Support "-help", "--help", "--list", etc. */
+ const char *command = argv[1];
+ while (*command == '-') {
+ ++command;
+ }
+
+ if (strcmp(argv[1], "help") == 0) {
+ help(stdout, argv0);
+ mbedtls_exit(MBEDTLS_EXIT_SUCCESS);
+ }
+ if (strcmp(argv[1], "list") == 0) {
+ for (const metatest_t *p = metatests; p->name != NULL; p++) {
+ mbedtls_printf("%s %s\n", p->name, p->platform);
+ }
+ mbedtls_exit(MBEDTLS_EXIT_SUCCESS);
+ }
+
+#if defined(MBEDTLS_TEST_MUTEX_USAGE)
+ mbedtls_test_mutex_usage_init();
+#endif
+
+ for (const metatest_t *p = metatests; p->name != NULL; p++) {
+ if (strcmp(argv[1], p->name) == 0) {
+ mbedtls_printf("Running metatest %s...\n", argv[1]);
+ p->entry_point(argv[1]);
+#if defined(MBEDTLS_TEST_MUTEX_USAGE)
+ mbedtls_test_mutex_usage_check();
+#endif
+ mbedtls_printf("Running metatest %s... done, result=%d\n",
+ argv[1], (int) mbedtls_test_info.result);
+ mbedtls_exit(mbedtls_test_info.result == MBEDTLS_TEST_RESULT_SUCCESS ?
+ MBEDTLS_EXIT_SUCCESS :
+ MBEDTLS_EXIT_FAILURE);
+ }
+ }
+
+ mbedtls_fprintf(stderr, "%s: FATAL: No such metatest: %s\n",
+ argv0, command);
+ mbedtls_exit(MBEDTLS_EXIT_FAILURE);
+}
diff --git a/scripts/generate_visualc_files.pl b/scripts/generate_visualc_files.pl
index 7f56098..96ade2f 100755
--- a/scripts/generate_visualc_files.pl
+++ b/scripts/generate_visualc_files.pl
@@ -144,6 +144,7 @@
my $guid = gen_app_guid( $path );
$path =~ s!/!\\!g;
(my $appname = $path) =~ s/.*\\//;
+ my $is_test_app = ($path =~ m/^test\\/);
my $srcs = "<ClCompile Include=\"..\\..\\programs\\$path.c\" \/>";
if( $appname eq "ssl_client2" or $appname eq "ssl_server2" or
@@ -158,7 +159,9 @@
$content =~ s/<SOURCES>/$srcs/g;
$content =~ s/<APPNAME>/$appname/g;
$content =~ s/<GUID>/$guid/g;
- $content =~ s/INCLUDE_DIRECTORIES\n/$include_directories/g;
+ $content =~ s/INCLUDE_DIRECTORIES\n/($is_test_app ?
+ $library_include_directories :
+ $include_directories)/ge;
content_to_file( $content, "$dir/$appname.$ext" );
}
diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh
index ed6a0b7..b3bb3f1 100755
--- a/tests/scripts/all.sh
+++ b/tests/scripts/all.sh
@@ -1120,6 +1120,9 @@
msg "test: selftest (ASan build)" # ~ 10s
programs/test/selftest
+ msg "test: metatests (GCC, ASan build)"
+ tests/scripts/run-metatests.sh any asan
+
msg "test: ssl-opt.sh (ASan build)" # ~ 1 min
tests/ssl-opt.sh
@@ -1885,6 +1888,9 @@
msg "test: Everest ECDH context - main suites (inc. selftests) (ASan build)" # ~ 50s
make test
+ msg "test: metatests (clang, ASan)"
+ tests/scripts/run-metatests.sh any asan
+
msg "test: Everest ECDH context - ECDH-related part of ssl-opt.sh (ASan build)" # ~ 5s
tests/ssl-opt.sh -f ECDH
@@ -1973,6 +1979,9 @@
msg "test: cpp_dummy_build (full config, clang)" # ~ 1s
programs/test/cpp_dummy_build
+ msg "test: metatests (clang)"
+ tests/scripts/run-metatests.sh any pthread
+
msg "program demos (full config, clang)" # ~10s
tests/scripts/run_demos.py
@@ -5465,6 +5474,9 @@
msg "test: main suites (MSan)" # ~ 10s
make test
+ msg "test: metatests (MSan)"
+ tests/scripts/run-metatests.sh any msan
+
msg "program demos (MSan)" # ~20s
tests/scripts/run_demos.py
diff --git a/tests/scripts/run-metatests.sh b/tests/scripts/run-metatests.sh
new file mode 100755
index 0000000..22a302c
--- /dev/null
+++ b/tests/scripts/run-metatests.sh
@@ -0,0 +1,89 @@
+#!/bin/sh
+
+help () {
+ cat <<EOF
+Usage: $0 [OPTION] [PLATFORM]...
+Run all the metatests whose platform matches any of the given PLATFORM.
+A PLATFORM can contain shell wildcards.
+
+Expected output: a lot of scary-looking error messages, since each
+metatest is expected to report a failure. The final line should be
+"Ran N metatests, all good."
+
+If something goes wrong: the final line should be
+"Ran N metatests, X unexpected successes". Look for "Unexpected success"
+in the logs above.
+
+ -l List the available metatests, don't run them.
+EOF
+}
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+
+set -e -u
+
+if [ -d programs ]; then
+ METATEST_PROGRAM=programs/test/metatest
+elif [ -d ../programs ]; then
+ METATEST_PROGRAM=../programs/test/metatest
+elif [ -d ../../programs ]; then
+ METATEST_PROGRAM=../../programs/test/metatest
+else
+ echo >&2 "$0: FATAL: programs/test/metatest not found"
+ exit 120
+fi
+
+LIST_ONLY=
+while getopts hl OPTLET; do
+ case $OPTLET in
+ h) help; exit;;
+ l) LIST_ONLY=1;;
+ \?) help >&2; exit 120;;
+ esac
+done
+shift $((OPTIND - 1))
+
+list_matches () {
+ while read name platform junk; do
+ for pattern in "$@"; do
+ case $platform in
+ $pattern) echo "$name"; break;;
+ esac
+ done
+ done
+}
+
+count=0
+errors=0
+run_metatest () {
+ ret=0
+ "$METATEST_PROGRAM" "$1" || ret=$?
+ if [ $ret -eq 0 ]; then
+ echo >&2 "$0: Unexpected success: $1"
+ errors=$((errors + 1))
+ fi
+ count=$((count + 1))
+}
+
+# Don't pipe the output of metatest so that if it fails, this script exits
+# immediately with a failure status.
+full_list=$("$METATEST_PROGRAM" list)
+matching_list=$(printf '%s\n' "$full_list" | list_matches "$@")
+
+if [ -n "$LIST_ONLY" ]; then
+ printf '%s\n' $matching_list
+ exit
+fi
+
+for name in $matching_list; do
+ run_metatest "$name"
+done
+
+if [ $errors -eq 0 ]; then
+ echo "Ran $count metatests, all good."
+ exit 0
+else
+ echo "Ran $count metatests, $errors unexpected successes."
+ exit 1
+fi