Test: test framework and test suites
-- Test framework
-- Test secure service
-- TFM Core secure/non-secure test suites
-- SST secure/non-secure test suites
Change-Id: I5868d9a65a740b984748d975a3a5bd486cbfafc4
Signed-off-by: Abhishek Pandit <abhishek.pandit@arm.com>
Co-Authored-By: Mate Toth-pal <mate.toth-pal@arm.com>
Co-Authored-By: Antonio de Angelis <antonio.deangelis@arm.com>
Co-Authored-By: Miklos Balint <miklos.balint@arm.com>
Co-Authored-By: Jamie Fox <jamie.fox@arm.com>
Co-Authored-By: Ashutosh Singh <ashutosh.singh@arm.com>
diff --git a/test/framework/helpers.c b/test/framework/helpers.c
new file mode 100755
index 0000000..f6d5df7
--- /dev/null
+++ b/test/framework/helpers.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2017, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+
+#include "helpers.h"
+
+#include <stdio.h>
+
+const char *sst_err_to_str(enum tfm_sst_err_t err)
+{
+ switch (err) {
+ case TFM_SST_ERR_SUCCESS:
+ return "TFM_SST_ERR_SUCCESS";
+ case TFM_SST_ERR_ASSET_NOT_PREPARED:
+ return "TFM_SST_ERR_ASSET_NOT_PREPARED";
+ case TFM_SST_ERR_ASSET_NOT_FOUND:
+ return "TFM_SST_ERR_ASSET_NOT_FOUND";
+ case TFM_SST_ERR_PARAM_ERROR:
+ return "TFM_SST_ERR_PARAM_ERROR";
+ case TFM_SST_ERR_INVALID_HANDLE:
+ return "TFM_SST_ERR_INVALID_HANDLE";
+ case TFM_SST_ERR_STORAGE_SYSTEM_FULL:
+ return "TFM_SST_ERR_STORAGE_SYSTEM_FULL";
+ case TFM_SST_ERR_SYSTEM_ERROR:
+ return "TFM_SST_ERR_SYSTEM_ERROR";
+ case TFM_SST_ERR_FORCE_INT_SIZE:
+ return "TFM_SST_ERR_FORCE_INT_SIZE";
+ /* default: The default is not defined intentionally to force the
+ * compiler to check that all the enumeration values are
+ * covered in the switch.
+ */
+ }
+}
+
+const char *asset_perms_to_str(uint8_t permissions)
+{
+ switch (permissions) {
+ case 0:
+ return "No permissions";
+ case 1:
+ return "SECURE_ASSET_REFERENCE";
+ case 2:
+ return "SECURE_ASSET_WRITE";
+ case 3:
+ return "SECURE_ASSET_REFERENCE | SECURE_ASSET_WRITE";
+ case 4:
+ return "SECURE_ASSET_READ";
+ case 5:
+ return "SECURE_ASSET_REFERENCE | SECURE_ASSET_READ";
+ case 6:
+ return "SECURE_ASSET_WRITE | SECURE_ASSET_READ";
+ case 7:
+ return "SECURE_ASSET_REFERENCE | SECURE_ASSET_WRITE | "
+ "SECURE_ASSET_READ";
+ default:
+ return "Unknown permissions";
+ }
+}
+
+void printf_set_color(enum serial_color_t color_id)
+{
+ printf("\33[3%dm", color_id);
+}
diff --git a/test/framework/helpers.h b/test/framework/helpers.h
new file mode 100755
index 0000000..517c4b0
--- /dev/null
+++ b/test/framework/helpers.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2017, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+
+#ifndef __HELPERS_H__
+#define __HELPERS_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include "tfm_sst_defs.h"
+#include "secure_fw/services/secure_storage/sst_asset_management.h"
+#include "test_framework.h"
+
+enum serial_color_t {
+ BLACK = 0,
+ RED = 1,
+ GREEN = 2,
+ YELLOW = 3,
+ BLUE = 4,
+ MAGENDA = 5,
+ CYAN = 6,
+ WHITE = 7,
+};
+
+/**
+ * \brief Translates tfm_sst_err_t into a string.
+ *
+ * \param[in] err tfm_sst_err_t error value.
+ *
+ * \return tfm_sst_err_t as string.
+ */
+const char *sst_err_to_str(enum tfm_sst_err_t err);
+
+/**
+ * \brief Translates asset permissions into a string.
+ *
+ * \param[in] permissions Asset permissions value.
+ *
+ * \return asset permissions as string.
+ */
+const char *asset_perms_to_str(uint8_t permissions);
+
+/**
+ * \brief Sets the the text color in the serial port.
+ *
+ * \param[in] color_id Serial foreground color.
+ *
+ * \return 0 if the message is send successfully, 1 otherwise.
+ */
+void printf_set_color(enum serial_color_t color_id);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __HELPERS_H__ */
diff --git a/test/framework/integ_test.h b/test/framework/integ_test.h
new file mode 100644
index 0000000..d00fa39
--- /dev/null
+++ b/test/framework/integ_test.h
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2017, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+
+#ifndef __INTEG_TEST_H__
+#define __INTEG_TEST_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * \brief Executes integration tests.
+ */
+void start_integ_test(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __INTEG_TEST_H__ */
diff --git a/test/framework/integ_test_helper.c b/test/framework/integ_test_helper.c
new file mode 100644
index 0000000..264b197
--- /dev/null
+++ b/test/framework/integ_test_helper.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2017, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+
+#include <stdio.h>
+
+#include "test_framework.h"
+#include "integ_test_helper.h"
+
+void integ_test(const char *suite_type,
+ struct test_suite_t test_suites[],
+ uint32_t test_suite_cnt)
+{
+ uint32_t i;
+
+ printf_set_color(YELLOW);
+ printf("\r\n#### Execute test suites for the %s area ####\r\n", suite_type);
+
+ /* Executes test suites */
+ for (i = 0; i < test_suite_cnt; i++) {
+ if (run_testsuite(&test_suites[i]) != TEST_SUITE_ERR_NO_ERROR) {
+ /* End function execution */
+ return;
+ }
+ }
+
+ /* Prints test suites summary */
+ printf_set_color(YELLOW);
+ printf("\r\n*** %s test suites summary ***\r\n", suite_type);
+ for (i = 0; i < test_suite_cnt; i++) {
+ printf_set_color(WHITE);
+ printf("Test suite '%s' has ", test_suites[i].name);
+ if (test_suites[i].val == TEST_PASSED) {
+ printf_set_color(GREEN);
+ printf(" PASSED");
+ } else {
+ printf_set_color(RED);
+ printf(" FAILED");
+ }
+ printf("\r\n");
+ }
+
+ printf_set_color(YELLOW);
+ printf("\r\n*** End of %s test suites ***\r\n", suite_type);
+}
diff --git a/test/framework/integ_test_helper.h b/test/framework/integ_test_helper.h
new file mode 100644
index 0000000..cfe7ace
--- /dev/null
+++ b/test/framework/integ_test_helper.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2017, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+
+#ifndef __INTEG_TEST_HELPER_H__
+#define __INTEG_TEST_HELPER_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "test_framework.h"
+
+/**
+ * \brief Executes integration test suites provided in the parameters.
+ *
+ * \param[in] suite_type A string containing the type of the suite
+ * (used for printing).
+ * \param[in] test_suites The suites to be executed.
+ * \param[in] test_suite_cnt The number of test suites to be executed.
+ */
+void integ_test(const char *suite_type,
+ struct test_suite_t test_suites[],
+ uint32_t test_suite_cnt);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __INTEG_TEST_HELPER_H__ */
diff --git a/test/framework/non_secure_suites.c b/test/framework/non_secure_suites.c
new file mode 100644
index 0000000..b14c685
--- /dev/null
+++ b/test/framework/non_secure_suites.c
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2017, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+
+#include "integ_test.h"
+#include "integ_test_helper.h"
+#include "test_framework.h"
+
+/* Service specific includes */
+#include "test/suites/sst/non_secure/sst_ns_tests.h"
+#include "test/suites/invert/non_secure/invert_ns_tests.h"
+#include "test/suites/core/non_secure/core_ns_tests.h"
+
+static struct test_suite_t test_suites[] = {
+#ifdef SERVICES_TEST_NS
+ /* Non-secure SST test cases */
+ {®ister_testsuite_ns_sst_interface, 0, 0, 0},
+
+#ifdef CORE_TEST_SERVICES
+ /* Non-secure invert test cases */
+ /* Note: since this is sample code, only run if test services are enabled */
+ {®ister_testsuite_ns_invert_interface, 0, 0, 0},
+#endif
+#endif
+
+#ifdef CORE_TEST_POSITIVE
+ /* Non-secure core test cases */
+ {®ister_testsuite_ns_core_positive, 0, 0, 0},
+#endif
+
+#ifdef CORE_TEST_INTERACTIVE
+ /* Non-secure interactive test cases */
+ {®ister_testsuite_ns_core_interactive, 0, 0, 0},
+#endif
+};
+
+void start_integ_test(void)
+{
+ integ_test("Non-secure", test_suites,
+ sizeof(test_suites)/sizeof(test_suites[0]));
+}
diff --git a/test/framework/secure_suites.c b/test/framework/secure_suites.c
new file mode 100644
index 0000000..1b5360c
--- /dev/null
+++ b/test/framework/secure_suites.c
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2017, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+
+#include "integ_test.h"
+#include "integ_test_helper.h"
+#include "test_framework.h"
+
+/* Service specific includes */
+#include "test/suites/sst/secure/sst_tests.h"
+#include "test/suites/invert/secure/invert_s_tests.h"
+#include "secure_fw/services/secure_storage/sst_core_interface.h"
+
+static struct test_suite_t test_suites[] = {
+#ifdef SERVICES_TEST_S
+ /* Secure SST test cases */
+ {®ister_testsuite_s_sst_sec_interface, 0, 0, 0},
+ {®ister_testsuite_s_sst_reliability, 0, 0, 0},
+
+ /* FIXME: this test suite needs to be refactored to accomodate it to
+ * use only secure application ID.
+ *
+ * {®ister_testsuite_s_sst_interface, 0, 0, 0},
+ */
+
+#ifdef CORE_TEST_SERVICES
+ /* Secure invert test cases */
+ /* Note: since this is sample code, only run if test services are enabled */
+ {®ister_testsuite_s_invert_interface, 0, 0, 0},
+#endif
+#endif /* SERVICES_TEST_S */
+};
+
+static void setup_integ_test(void)
+{
+ /* Left empty intentionally, currently implemented
+ * test suites require no setup
+ */
+}
+
+static void tear_down_integ_test(void)
+{
+ /* Leave the SST area clean after execute the tests */
+ sst_object_wipe_all();
+ sst_object_prepare();
+}
+
+void start_integ_test(void)
+{
+ setup_integ_test();
+ integ_test("Secure",
+ test_suites,
+ sizeof(test_suites)/sizeof(test_suites[0]));
+ tear_down_integ_test();
+}
diff --git a/test/framework/test_framework.c b/test/framework/test_framework.c
new file mode 100644
index 0000000..fcb7451
--- /dev/null
+++ b/test/framework/test_framework.c
@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) 2017, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+
+#include "test_framework.h"
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "helpers.h"
+
+static void test_failed(const struct test_result_t *ret)
+{
+ printf_set_color(RED);
+ if (ret->info_msg != 0) {
+ printf(" %s", ret->info_msg);
+ if (ret->filename != 0) {
+ printf(" (Failed at %s:%d)\r\n", ret->filename, ret->line);
+ }
+ } else {
+ if (ret->filename != 0) {
+ printf(" Failed at %s:%d\r\n", ret->filename, ret->line);
+ }
+ }
+
+ printf(" TEST FAILED!");
+ printf("\r\n");
+}
+
+static void print_error(const char *err_msg)
+{
+ printf_set_color(RED);
+ printf("Error ( %s )\r\n", err_msg);
+}
+
+const char *test_err_to_str(enum test_suite_err_t err)
+{
+ switch (err) {
+ case TEST_SUITE_ERR_NO_ERROR:
+ return "TEST_SUITE_ERR_NO_ERROR";
+ case TEST_SUITE_ERR_INVALID_DATA:
+ return "TEST_SUITE_ERR_INVALID_DATA";
+ case TEST_SUITE_ERR_INVALID_TEST_DATA:
+ return "TEST_SUITE_ERR_INVALID_TEST_DATA";
+ case TEST_SUITE_ERR_TEST_FAILED:
+ return "TEST_SUITE_ERR_TEST_FAILED";
+ /* default: The default is not defined intentionally to force the
+ * compiler to check that all the enumeration values are
+ * covered in the switch.
+ */
+ }
+}
+
+enum test_suite_err_t set_testsuite(const char *name,
+ struct test_t *test_list, uint32_t size,
+ struct test_suite_t *p_ts)
+{
+ if (p_ts == 0) {
+ print_error("TEST_SUITE_ERR_INVALID_DATA!");
+ return TEST_SUITE_ERR_INVALID_DATA;
+ }
+
+ p_ts->name = name;
+ p_ts->test_list = test_list;
+ p_ts->list_size = size;
+
+ return TEST_SUITE_ERR_NO_ERROR;
+}
+
+void set_test_failed(const char *info_msg, const char *filename, uint32_t line,
+ struct test_result_t *ret)
+{
+ if (ret == 0) {
+ print_error("TEST_SUITE_ERR_INVALID_TEST_DATA!");
+ return;
+ }
+
+ ret->val = TEST_FAILED;
+ ret->info_msg = info_msg;
+ ret->filename = filename;
+ ret->line = line;
+}
+
+enum test_suite_err_t run_testsuite(struct test_suite_t *test_suite)
+{
+ uint32_t failed_tests = 0;
+ uint32_t i;
+ struct test_t *p_test;
+
+ if (test_suite == 0 || test_suite->freg == 0) {
+ print_error("TEST_SUITE_ERR_INVALID_DATA!");
+ return TEST_SUITE_ERR_INVALID_DATA;
+ }
+
+ /* Sets test suite parameters */
+ test_suite->freg(test_suite);
+ if (test_suite->name == 0 || test_suite->list_size == 0) {
+ print_error("TEST_SUITE_ERR_INVALID_DATA!");
+ return TEST_SUITE_ERR_INVALID_DATA;
+ }
+
+ printf_set_color(YELLOW);
+ printf("Running Test Suite %s...\r\n", test_suite->name);
+
+ /* Sets pointer to the first test */
+ p_test = test_suite->test_list;
+
+ for (i = 0; i < test_suite->list_size; i++) {
+
+ if (p_test->test == 0 || p_test->name == 0) {
+ print_error("TEST_SUITE_ERR_INVALID_TEST_DATA!");
+ return TEST_SUITE_ERR_INVALID_TEST_DATA;
+ }
+
+ printf_set_color(WHITE);
+ printf("> Executing '%s' \r\n Description: '%s'\r\n",
+ p_test->name, p_test->desc);
+
+ /* Sets the default value before the test */
+ p_test->ret.val = TEST_PASSED;
+
+ /* Executes the test */
+ p_test->test(&p_test->ret);
+ if (p_test->ret.val == TEST_FAILED) {
+ test_failed(&p_test->ret);
+ failed_tests++;
+ } else {
+ printf_set_color(GREEN);
+ printf(" TEST PASSED!\r\n");
+ }
+
+ /* Sets pointer to the next test */
+ p_test++;
+ }
+
+
+ if (failed_tests == 0) {
+ printf_set_color(GREEN);
+ printf("TESTSUITE PASSED!\r\n");
+ test_suite->val = TEST_PASSED;
+ } else {
+ printf_set_color(RED);
+ printf("TESTSUITE FAILED!\r\n");
+ printf_set_color(YELLOW);
+ printf("Number of failed tests: %d of %d\r\n",
+ failed_tests, test_suite->list_size);
+ test_suite->val = TEST_FAILED;
+ }
+
+ return TEST_SUITE_ERR_NO_ERROR;
+}
diff --git a/test/framework/test_framework.h b/test/framework/test_framework.h
new file mode 100644
index 0000000..acd0315
--- /dev/null
+++ b/test/framework/test_framework.h
@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2017, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+
+#ifndef __TEST_FRAMEWORK_H__
+#define __TEST_FRAMEWORK_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+#include "helpers.h"
+
+enum test_status_t {
+ TEST_PASSED = 0, /*!< Test has passed */
+ TEST_FAILED = 1, /*!< Test has failed */
+};
+
+struct test_result_t {
+ enum test_status_t val; /*!< Test result \ref test_status_t */
+ const char *info_msg; /*!< Information message to show in case of
+ * failure
+ */
+ const char *filename; /*!< Filename where the failure has occured */
+ uint32_t line; /*!< Line where the failure has occured */
+};
+
+/**
+ * \brief Runs the test.
+ *
+ * \param[out] ret Test result value
+ */
+typedef void TEST_FUN(struct test_result_t *ret);
+
+struct test_t {
+ TEST_FUN * const test; /*!< Test function to call */
+ const char *name; /*!< Test name */
+ const char *desc; /*!< Test description */
+ struct test_result_t ret; /*!< Test result */
+};
+
+struct test_suite_t;
+
+/**
+ * \brief Registers test in the testsuite structure and sets the name.
+ *
+ * \param[in] p_test_suite Pointer to the p_test_suite_location.
+ */
+typedef void TESTSUITE_REG(struct test_suite_t *p_test_suite);
+
+struct test_suite_t {
+ TESTSUITE_REG * const freg; /*!< Function to set all follow fields
+ * of the current test suite
+ */
+ struct test_t *test_list; /*!< List of tests */
+ uint32_t list_size; /*!< List size */
+ const char *name; /*!< Test suite name */
+ enum test_status_t val; /*!< Test suite result \ref test_result_t */
+};
+
+enum test_suite_err_t {
+ TEST_SUITE_ERR_NO_ERROR = 0, /*!< No error */
+ TEST_SUITE_ERR_INVALID_DATA = 1, /*!< Invalid test suite if any of the
+ * pointers is NULL
+ */
+ TEST_SUITE_ERR_INVALID_TEST_DATA = 2, /*!< Invalid test if any of the
+ * pointers is NULL
+ */
+ TEST_SUITE_ERR_TEST_FAILED = 3, /*!< Last executed test has failed */
+};
+
+/**
+ * \brief Translates the test suite error into a string.
+ *
+ * \param[in] err Error value \ref test_suite_err_t
+ *
+ * \returns error as string.
+ */
+const char *test_err_to_str(enum test_suite_err_t err);
+
+/**
+ * \brief Sets test suite parameters in the \ref structure.
+ *
+ * \param[in] name Test suite name
+ * \param[in] test_list Pointer to the test list
+ * \param[in] size Test list size
+ * \param[in/out] p_ts Pointer to test suite object to fill in the
+ * parameters
+ *
+ * \returns Returns error code as specified in \ref test_suite_err_t
+ */
+enum test_suite_err_t set_testsuite(const char *name,
+ struct test_t *test_list, uint32_t size,
+ struct test_suite_t *p_ts);
+
+/**
+ * \brief Runs the given test suite.
+ *
+ * \param[in/out] test_suite Test suite to run the list of tests and
+ * store test results.
+ *
+ * \returns Returns error code as specified in \ref test_suite_err_t
+ */
+enum test_suite_err_t run_testsuite(struct test_suite_t *test_suite);
+
+/**
+ * \brief Prints all test in the the given test suite.
+ *
+ * \param[in] ts Test suite to print the list of tests
+ */
+void show_tests(const struct test_suite_t *ts);
+
+/**
+ * \brief Sets test failure state and information in the \ref sst_test_result_t
+ * structure.
+ *
+ * \param[in] info_msg Information message to show
+ * \param[in] filename Filename where the error has ocurred
+ * \param[in] line Line in the file where the error has ocurred
+ * \param[out] ret Pointer to \ref sst_test_result_t structure to
+ * set the values
+ *
+ * \note: If info_msg is "" or , info message is not shown. If filename is "",
+ * filename and line are not shown.
+ */
+void set_test_failed(const char *info_msg, const char *filename, uint32_t line,
+ struct test_result_t *ret);
+
+#define TEST_FAIL(info_msg) set_test_failed(info_msg, __FILE__, __LINE__, ret)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __TEST_FRAMEWORK_H__ */