Add Benchmark Test Suite
For now we only have one test suite in xtest.
This patch provides another test suite for benchmark,
but the benchmark test cases are not implemented yet.
User can still run original xtest test suite with the
same command as before.
The command to run benchmark tests as follows:
xtest -t benchmark <test id>
Tested-by: James Kung <james.kung@linaro.org> (QEMU)
Tested-by: Pascal Brand <pascal.brand@linaro.org> (QEMU)
Signed-off-by: James Kung <james.kung@linaro.org>
Reviewed-by: Pascal Brand <pascal.brand@linaro.org>
Reviewed-by: Jens Wiklander <jens.wiklander@linaro.org>
diff --git a/host/xtest/Makefile b/host/xtest/Makefile
index fa71d83..3f626b4 100644
--- a/host/xtest/Makefile
+++ b/host/xtest/Makefile
@@ -32,6 +32,7 @@
xtest_7000$(GP).c \
xtest_10000.c \
xtest_20000.c \
+ xtest_benchmark_1000.c \
xtest_helpers.c \
xtest_main.c \
xtest_test.c \
@@ -71,6 +72,7 @@
CFLAGS += -I../../ta/rpc_test/include
CFLAGS += -I../../ta/sims/include
CFLAGS += -I../../ta/storage/include
+CFLAGS += -I../../ta/bonnie/include
CFLAGS += -I../../ta/concurrent/include
ifdef CFG_GP_PACKAGE_PATH
CFLAGS += -I../../ta/GP_TTA_Arithmetical
diff --git a/host/xtest/xtest_benchmark_1000.c b/host/xtest/xtest_benchmark_1000.c
new file mode 100644
index 0000000..fa2beeb
--- /dev/null
+++ b/host/xtest/xtest_benchmark_1000.c
@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2015, Linaro Limited
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License Version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <stdio.h>
+#include <string.h>
+
+#include "xtest_test.h"
+#include "xtest_helpers.h"
+
+#include <ta_bonnie.h>
+
+static void xtest_tee_benchmark_1001(ADBG_Case_t *Case_p);
+static void xtest_tee_benchmark_1002(ADBG_Case_t *Case_p);
+static void xtest_tee_benchmark_1003(ADBG_Case_t *Case_p);
+static void xtest_tee_benchmark_1004(ADBG_Case_t *Case_p);
+static void xtest_tee_benchmark_1005(ADBG_Case_t *Case_p);
+static void xtest_tee_benchmark_1006(ADBG_Case_t *Case_p);
+
+
+static TEEC_Result run_test(enum bonnie_cmd cmd)
+{
+ TEEC_Operation op = TEEC_OPERATION_INITIALIZER;
+ TEEC_Result res;
+ TEEC_Session sess;
+ uint32_t orig;
+
+ res = xtest_teec_open_session(&sess, &bonnie_ta_uuid, NULL, &orig);
+ if (res != TEEC_SUCCESS)
+ return res;
+
+ res = TEEC_InvokeCommand(&sess, cmd, &op, &orig);
+
+ TEEC_CloseSession(&sess);
+
+ return res;
+}
+
+static void xtest_tee_benchmark_1001(ADBG_Case_t *c)
+{
+ ADBG_EXPECT_TEEC_SUCCESS(c, run_test(TA_BONNIE_CMD_TEST_PUTC));
+}
+
+static void xtest_tee_benchmark_1002(ADBG_Case_t *c)
+{
+ ADBG_EXPECT_TEEC_SUCCESS(c, run_test(TA_BONNIE_CMD_TEST_REWRITE));
+}
+
+static void xtest_tee_benchmark_1003(ADBG_Case_t *c)
+{
+ ADBG_EXPECT_TEEC_SUCCESS(c, run_test(TA_BONNIE_CMD_TEST_FASTWRITE));
+}
+
+static void xtest_tee_benchmark_1004(ADBG_Case_t *c)
+{
+ ADBG_EXPECT_TEEC_SUCCESS(c, run_test(TA_BONNIE_CMD_TEST_GETC));
+}
+
+static void xtest_tee_benchmark_1005(ADBG_Case_t *c)
+{
+ ADBG_EXPECT_TEEC_SUCCESS(c, run_test(TA_BONNIE_CMD_TEST_FASTREAD));
+}
+
+static void xtest_tee_benchmark_1006(ADBG_Case_t *c)
+{
+ ADBG_EXPECT_TEEC_SUCCESS(c, run_test(TA_BONNIE_CMD_TEST_LSEEK));
+}
+
+
+ADBG_CASE_DEFINE(XTEST_TEE_BENCHMARK_1001, xtest_tee_benchmark_1001,
+ /* Title */ "TEE Trusted Storage Performance Test (PUTC)",
+ /* Short description */ "",
+ /* Requirement IDs */ "",
+ /* How to implement */ ""
+ );
+
+ADBG_CASE_DEFINE(XTEST_TEE_BENCHMARK_1002, xtest_tee_benchmark_1002,
+ /* Title */ "TEE Trusted Storage Performance Test (REWRITE)",
+ /* Short description */ "",
+ /* Requirement IDs */ "",
+ /* How to implement */ ""
+ );
+
+ADBG_CASE_DEFINE(XTEST_TEE_BENCHMARK_1003, xtest_tee_benchmark_1003,
+ /* Title */ "TEE Trusted Storage Performance Test (FASTWRITE)",
+ /* Short description */ "",
+ /* Requirement IDs */ "",
+ /* How to implement */ ""
+ );
+
+ADBG_CASE_DEFINE(XTEST_TEE_BENCHMARK_1004, xtest_tee_benchmark_1004,
+ /* Title */ "TEE Trusted Storage Performance Test (GETC)",
+ /* Short description */ "",
+ /* Requirement IDs */ "",
+ /* How to implement */ ""
+ );
+
+ADBG_CASE_DEFINE(XTEST_TEE_BENCHMARK_1005, xtest_tee_benchmark_1005,
+ /* Title */ "TEE Trusted Storage Performance Test (FASTREAD)",
+ /* Short description */ "",
+ /* Requirement IDs */ "",
+ /* How to implement */ ""
+ );
+
+ADBG_CASE_DEFINE(XTEST_TEE_BENCHMARK_1006, xtest_tee_benchmark_1006,
+ /* Title */ "TEE Trusted Storage Performance Test (LSEEK)",
+ /* Short description */ "",
+ /* Requirement IDs */ "",
+ /* How to implement */ ""
+ );
+
diff --git a/host/xtest/xtest_main.c b/host/xtest/xtest_main.c
index 098dc53..ee0f6df 100644
--- a/host/xtest/xtest_main.c
+++ b/host/xtest/xtest_main.c
@@ -104,10 +104,23 @@
ADBG_SUITE_DEFINE_END()
+
+ADBG_SUITE_DECLARE(XTEST_TEE_BENCHMARK)
+
+ADBG_SUITE_DEFINE_BEGIN(XTEST_TEE_BENCHMARK, NULL)
+ADBG_SUITE_ENTRY(XTEST_TEE_BENCHMARK_1001, NULL)
+ADBG_SUITE_ENTRY(XTEST_TEE_BENCHMARK_1002, NULL)
+ADBG_SUITE_ENTRY(XTEST_TEE_BENCHMARK_1003, NULL)
+ADBG_SUITE_ENTRY(XTEST_TEE_BENCHMARK_1004, NULL)
+ADBG_SUITE_ENTRY(XTEST_TEE_BENCHMARK_1005, NULL)
+ADBG_SUITE_ENTRY(XTEST_TEE_BENCHMARK_1006, NULL)
+ADBG_SUITE_DEFINE_END()
+
static const char gdevname_tz[] = "opteearmtz00";
char *_device = (char *)gdevname_tz;
unsigned int level = 0;
static const char glevel[] = "0";
+static const char gsuitename[] = "regression";
void usage(char *program);
@@ -118,7 +131,8 @@
printf("options:\n");
printf("\t-d <device> default value = %s\n", gdevname_tz);
printf("\t-l <level> test suite level: [0-15]\n");
- printf("\t default value = %s\n", glevel);
+ printf("\t-t <test_suite> available test suite: sanity, benchmark\n");
+ printf("\t default value = %s\n", gsuitename);
printf("\t-h show usage\n");
printf("\n");
}
@@ -129,8 +143,11 @@
int index;
int ret;
char *p = (char *)glevel;
+ char *test_suite = (char *)gsuitename;
- while ((opt = getopt(argc, argv, "d:l:h")) != -1) {
+ opterr = 0;
+
+ while ((opt = getopt(argc, argv, "d:l:t:h")) != -1)
switch (opt) {
case 'd':
_device = optarg;
@@ -138,6 +155,9 @@
case 'l':
p = optarg;
break;
+ case 't':
+ test_suite = optarg;
+ break;
case 'h':
usage(argv[0]);
return 0;
@@ -145,7 +165,6 @@
usage(argv[0]);
return -1;
}
- }
for (index = optind; index < argc; index++)
printf("Test ID: %s\n", argv[index]);
@@ -160,7 +179,14 @@
xtest_teec_ctx_init();
- ret = Do_ADBG_RunSuite(&ADBG_Suite_XTEST_TEE_TEST, argc - optind, (argv + optind));
+ if (strcmp(test_suite, "sanity") == 0)
+ ret = Do_ADBG_RunSuite(&ADBG_Suite_XTEST_TEE_TEST, argc - optind, (argv + optind));
+ else if (strcmp(test_suite, "benchmark") == 0)
+ ret = Do_ADBG_RunSuite(&ADBG_Suite_XTEST_TEE_BENCHMARK, argc - optind, (argv + optind));
+ else {
+ fprintf(stderr, "No test suite found: %s\n", test_suite);
+ ret = -1;
+ }
xtest_teec_ctx_deinit();
diff --git a/host/xtest/xtest_test.c b/host/xtest/xtest_test.c
index f66f879..28603e9 100644
--- a/host/xtest/xtest_test.c
+++ b/host/xtest/xtest_test.c
@@ -21,6 +21,7 @@
#include <ta_storage.h>
#include <ta_concurrent.h>
#include <enc_fs_key_manager_test.h>
+#include <ta_bonnie.h>
#include <tee_api_defines.h>
#ifdef WITH_GP_TESTS
#include <tee_api_types.h>
@@ -80,6 +81,7 @@
const TEEC_UUID storage_ta_uuid = TA_STORAGE_UUID;
const TEEC_UUID enc_fs_key_manager_test_ta_uuid = ENC_FS_KEY_MANAGER_TEST_UUID;
const TEEC_UUID concurrent_ta_uuid = TA_CONCURRENT_UUID;
+const TEEC_UUID bonnie_ta_uuid = TA_BONNIE_UUID;
#ifdef WITH_GP_TESTS
const TEEC_UUID gp_tta_ds_uuid = TA_TTA_DS_UUID;
#endif
diff --git a/host/xtest/xtest_test.h b/host/xtest/xtest_test.h
index 12623e9..d07f451 100644
--- a/host/xtest/xtest_test.h
+++ b/host/xtest/xtest_test.h
@@ -201,6 +201,7 @@
extern const TEEC_UUID ecc_test_ta_uuid;
extern const TEEC_UUID gp_tta_time_uuid;
extern const TEEC_UUID concurrent_ta_uuid;
+extern const TEEC_UUID bonnie_ta_uuid;
extern char *_device;
#endif /*XTEST_TEST_H*/