aboutsummaryrefslogtreecommitdiff
path: root/tftf/framework
diff options
context:
space:
mode:
authorSandrine Bailleux <sandrine.bailleux@arm.com>2018-11-07 17:11:59 +0100
committerSandrine Bailleux <sandrine.bailleux@arm.com>2018-12-13 16:07:05 +0100
commit125d58c4c065e27c7764ed9f05236f70fc7a5788 (patch)
tree9594adfa2f9032765680ff1f6c59f79d14b204f1 /tftf/framework
parent2b001329a3d7ba88a3304b7effa48b753181ccea (diff)
downloadtf-a-tests-125d58c4c065e27c7764ed9f05236f70fc7a5788.tar.gz
Make tests output more concise and clearer
Also refactor the code that generates this output. This is only a first step, subsequent patches will further improve the output. Here is a sample of the old output: [cpu 0x0000] NOTICE: Starting unittest 'Template - Single core test' [cpu 0x0000] NOTICE: Unittest 'Template - Single core test' complete. Result: Passed [cpu 0x0000] NOTICE: Starting unittest 'Template - Multi core test' [cpu 0x0000] NOTICE: Unittest 'Template - Multi core test' complete. Result: Passed ========== TEST REPORT ========== # Test suite 'Template': - Single core test: Passed - Multi core test: Passed ================================= Tests Skipped : 0 Tests Passed : 2 Tests Failed : 0 Tests Crashed : 0 Total tests : 2 ================================= [cpu 0x0000] NOTICE: Exiting tests. And now the new output: [cpu 0x0000] -- [cpu 0x0000] Running test suite 'Template' [cpu 0x0000] Description: Template test code [cpu 0x0000] [cpu 0x0000] > Executing 'Single core test' [cpu 0x0000] TEST COMPLETE Passed [cpu 0x0000] [cpu 0x0000] > Executing 'Multi core test' [cpu 0x0000] TEST COMPLETE Passed [cpu 0x0000] [cpu 0x0000] ******************************* Summary ******************************* [cpu 0x0000] > Test suite 'Template' [cpu 0x0000] Passed [cpu 0x0000] ================================= [cpu 0x0000] Tests Skipped : 0 [cpu 0x0000] Tests Passed : 2 [cpu 0x0000] Tests Failed : 0 [cpu 0x0000] Tests Crashed : 0 [cpu 0x0000] Total tests : 2 [cpu 0x0000] ================================= [cpu 0x0000] NOTICE: Exiting tests. Change-Id: I9d52f2da8905962ab1df73d0691846d88622d3b5 Signed-off-by: Sandrine Bailleux <sandrine.bailleux@arm.com>
Diffstat (limited to 'tftf/framework')
-rw-r--r--tftf/framework/include/tftf.h5
-rw-r--r--tftf/framework/main.c21
-rw-r--r--tftf/framework/report.c113
3 files changed, 86 insertions, 53 deletions
diff --git a/tftf/framework/include/tftf.h b/tftf/framework/include/tftf.h
index 8231e284..43f1e7ea 100644
--- a/tftf/framework/include/tftf.h
+++ b/tftf/framework/include/tftf.h
@@ -130,7 +130,10 @@ STATUS tftf_testcase_set_result(const test_case_t *testcase,
*/
STATUS tftf_testcase_get_result(const test_case_t *testcase, TESTCASE_RESULT *result, char *test_output);
-void tftf_report_generate(void);
+void print_testsuite_start(const test_suite_t *testsuite);
+void print_test_start(const test_case_t *test);
+void print_test_end(const test_case_t *test);
+void print_tests_summary(void);
/*
* Exit the TFTF.
diff --git a/tftf/framework/main.c b/tftf/framework/main.c
index 3f94dc94..ea56f2eb 100644
--- a/tftf/framework/main.c
+++ b/tftf/framework/main.c
@@ -148,8 +148,14 @@ static void prepare_next_test(void)
for (unsigned int i = 0; i < PLATFORM_CORE_COUNT; ++i)
test_results[i] = TEST_RESULT_NA;
- NOTICE("Starting unittest '%s - %s'\n",
- current_testsuite()->name, current_testcase()->name);
+ /* If we're starting a new testsuite, announce it. */
+ test_ref_t test_to_run;
+ tftf_get_test_to_run(&test_to_run);
+ if (test_to_run.testcase_idx == 0) {
+ print_testsuite_start(current_testsuite());
+ }
+
+ print_test_start(current_testcase());
/* Program the watchdog */
tftf_platform_watchdog_set();
@@ -263,21 +269,18 @@ static unsigned int close_test(void)
assert(tftf_get_ref_cnt() == 0);
/* Save test result in NVM */
- test_result_t overall_test_result = get_overall_test_result();
tftf_testcase_set_result(current_testcase(),
- overall_test_result,
+ get_overall_test_result(),
0);
- NOTICE("Unittest '%s - %s' complete. Result: %s\n",
- current_testsuite()->name, current_testcase()->name,
- test_result_to_string(overall_test_result));
+ print_test_end(current_testcase());
/* The test is finished, let's move to the next one (if any) */
next_test = advance_to_next_test();
/* If this was the last test then report all results */
if (!next_test) {
- tftf_report_generate();
+ print_tests_summary();
tftf_clean_nvm();
return 1;
} else {
@@ -551,7 +554,7 @@ void __dead2 tftf_cold_boot_main(void)
NOTICE("Resuming interrupted test session\n");
rc = resume_test_session();
if (rc < 0) {
- tftf_report_generate();
+ print_tests_summary();
tftf_clean_nvm();
tftf_exit();
}
diff --git a/tftf/framework/report.c b/tftf/framework/report.c
index faae34b8..663caf36 100644
--- a/tftf/framework/report.c
+++ b/tftf/framework/report.c
@@ -6,70 +6,97 @@
#include <assert.h>
#include <debug.h>
-#include <platform_def.h> /* For TESTCASE_OUTPUT_MAX_SIZE */
#include <stdio.h>
-#include <string.h>
+#include <stdbool.h>
#include <tftf.h>
-static unsigned int total_tests;
-static unsigned int tests_stats[TEST_RESULT_MAX];
-
-static void tftf_update_tests_statistics(test_result_t result)
-{
- assert(TEST_RESULT_IS_VALID(result));
- total_tests++;
- tests_stats[result]++;
-}
-
static const char *test_result_strings[TEST_RESULT_MAX] = {
"Skipped", "Passed", "Failed", "Crashed",
};
-const char *test_result_to_string(test_result_t result)
+static const char *test_result_to_string(test_result_t result)
{
assert(TEST_RESULT_IS_VALID(result));
return test_result_strings[result];
}
-void tftf_report_generate(void)
+void print_testsuite_start(const test_suite_t *testsuite)
{
- unsigned i, j;
- const test_case_t *testcases;
- TESTCASE_RESULT testcase_result;
- char test_output[TESTCASE_OUTPUT_MAX_SIZE];
- STATUS status;
-
- /* Extract the result of all the testcases */
- printf("========== TEST REPORT ==========\n");
- for (i = 0; testsuites[i].name != NULL; i++) {
- printf("# Test suite '%s':\n", testsuites[i].name);
- testcases = testsuites[i].testcases;
-
- for (j = 0; testcases[j].name != NULL; j++) {
- status = tftf_testcase_get_result(&testcases[j], &testcase_result, test_output);
- if (status != STATUS_SUCCESS) {
- printf("Failed to get test result.\n");
+ mp_printf("--\n");
+ mp_printf("Running test suite '%s'\n", testsuite->name);
+ mp_printf("Description: %s\n", testsuite->description);
+ mp_printf("\n");
+}
+
+void print_test_start(const test_case_t *test)
+{
+ mp_printf("> Executing '%s'\n", test->name);
+}
+
+void print_test_end(const test_case_t *test)
+{
+ TESTCASE_RESULT result;
+ char output[TESTCASE_OUTPUT_MAX_SIZE];
+
+ tftf_testcase_get_result(test, &result, output);
+
+ mp_printf(" TEST COMPLETE %54s\n",
+ test_result_to_string(result.result));
+ if (strlen(output) != 0) {
+ mp_printf("%s", output);
+ }
+ mp_printf("\n");
+}
+
+void print_tests_summary(void)
+{
+ int total_tests = 0;
+ int tests_stats[TEST_RESULT_MAX] = { 0 };
+
+ mp_printf("******************************* Summary *******************************\n");
+
+ /* Go through the list of test suites. */
+ for (int i = 0; testsuites[i].name != NULL; i++) {
+ bool passed = true;
+
+ mp_printf("> Test suite '%s'\n", testsuites[i].name);
+
+ const test_case_t *testcases = testsuites[i].testcases;
+
+ /* Go through the list of tests inside this test suite. */
+ for (int j = 0; testcases[j].name != NULL; j++) {
+ TESTCASE_RESULT result;
+ char output[TESTCASE_OUTPUT_MAX_SIZE];
+
+ if (tftf_testcase_get_result(&testcases[j], &result,
+ output) != STATUS_SUCCESS) {
+ mp_printf("Failed to get test result.\n");
continue;
}
- tftf_update_tests_statistics(testcase_result.result);
- /* TODO: print test duration */
- printf("\t - %s: %s\n", testcases[j].name,
- test_result_to_string(testcase_result.result));
+ assert(TEST_RESULT_IS_VALID(result.result));
- if (strlen(test_output) != 0) {
- printf("--- output ---\n");
- printf("%s", test_output);
- printf("--------------\n");
+ /*
+ * Consider that a test suite passed if all of its
+ * tests passed or were skipped.
+ */
+ if ((result.result != TEST_RESULT_SUCCESS) &&
+ (result.result != TEST_RESULT_SKIPPED)) {
+ passed = false;
}
+
+ total_tests++;
+ tests_stats[result.result]++;
}
+ mp_printf("%70s\n", passed ? "Passed" : "Failed");
}
- printf("=================================\n");
- for (i = TEST_RESULT_MIN; i < TEST_RESULT_MAX; i++) {
- printf("Tests %-8s: %d\n",
+ mp_printf("=================================\n");
+
+ for (int i = TEST_RESULT_MIN; i < TEST_RESULT_MAX; i++) {
+ mp_printf("Tests %-8s: %d\n",
test_result_to_string(i), tests_stats[i]);
}
- printf("%-14s: %d\n", "Total tests", total_tests);
- printf("=================================\n");
+ mp_printf("%-14s: %d\n", "Total tests", total_tests);
+ mp_printf("=================================\n");
}