diff options
author | julhal01 <julian.hall@arm.com> | 2021-02-05 17:30:49 +0000 |
---|---|---|
committer | Gyorgy Szing <Gyorgy.Szing@arm.com> | 2021-04-14 16:59:24 +0200 |
commit | 3ec4c32dc835bffa0f5af07b3e170f4b8586a799 (patch) | |
tree | ba25f104b66f4a22c1364e991c0f726ac4ff4efe /components | |
parent | 2c18fbfda57fdc7b3971bffbe5902caf52a9530a (diff) | |
download | trusted-services-3ec4c32dc835bffa0f5af07b3e170f4b8586a799.tar.gz |
Add test_runner service
This is a new service with client and provider that can be
used for running tests in a secure processing environment
and retrieving the results. The test_runner provider allows
for arbitrary test farmework backends. The goal is to
have a cpputest backend. In this commit, a mock backend
is included for testing the service itself. The service
has its own access protocol defined under the protocols
top-level directory.
Signed-off-by: Julian Hall <julian.hall@arm.com>
Change-Id: If4e965c110763bd805abbdcb87e7e03cd76248b2
Diffstat (limited to 'components')
29 files changed, 1690 insertions, 21 deletions
diff --git a/components/app/remote-test-runner/component.cmake b/components/app/remote-test-runner/component.cmake new file mode 100644 index 000000000..a47ec425e --- /dev/null +++ b/components/app/remote-test-runner/component.cmake @@ -0,0 +1,14 @@ +#------------------------------------------------------------------------------- +# Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# +#------------------------------------------------------------------------------- +if (NOT DEFINED TGT) + message(FATAL_ERROR "mandatory parameter TGT is not defined.") +endif() + +target_sources(${TGT} PRIVATE + "${CMAKE_CURRENT_LIST_DIR}/remote_test_runner.cpp" + ) + diff --git a/components/app/remote-test-runner/remote_test_runner.cpp b/components/app/remote-test-runner/remote_test_runner.cpp new file mode 100644 index 000000000..681c7467f --- /dev/null +++ b/components/app/remote-test-runner/remote_test_runner.cpp @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "remote_test_runner.h" +#include <protocols/service/test_runner/packed-c/status.h> +#include <vector> +#include <string> +#include <cstring> +#include <cstdio> + +remote_test_runner::remote_test_runner() : + m_client(NULL) +{ + +} + +remote_test_runner::remote_test_runner(test_runner_client *client) : + m_client(client) +{ + +} + +remote_test_runner::~remote_test_runner() +{ + +} + +void remote_test_runner::set_client(test_runner_client *client) +{ + m_client = client; +} + +int remote_test_runner::execute(int argc, char *argv[]) +{ + int test_status = TS_TEST_RUNNER_STATUS_ERROR; + struct test_spec spec; + + /* Parse command line parameters */ + bool list_only = option_selected("-l", argc, argv); + parse_test_spec_params(argc, argv, spec); + + /* Run or list tests qualified bu spec */ + struct test_summary summary; + std::vector<struct test_result> results; + + if (list_only) { + + test_status = m_client->list_tests(spec, summary, results); + output_list(summary, results); + } + else { + + test_status = m_client->run_tests(spec, summary, results); + output_results(summary, results); + } + + if (test_status != TS_TEST_RUNNER_STATUS_SUCCESS) { + + printf("Tests failed to run with error: %d\n", test_status); + } + + return test_status; +} + +void remote_test_runner::parse_test_spec_params(int argc, char *argv[], struct test_spec &spec) const +{ + std::string name = parse_option("-n", argc, argv); + std::string group = parse_option("-g", argc, argv); + + memset(spec.name, 0, TEST_NAME_MAX_LEN); + name.copy(spec.name, TEST_NAME_MAX_LEN - 1); + + memset(spec.group, 0, TEST_GROUP_MAX_LEN); + group.copy(spec.group, TEST_GROUP_MAX_LEN - 1); +} + +std::string remote_test_runner::parse_option(const char *option_switch, int argc, char *argv[]) const +{ + std::string option; + + for (int i = 1; i + 1 < argc; ++i) { + + if (strcmp(argv[i], option_switch) == 0) { + + option = std::string(argv[i +1]); + break; + } + } + + return option; +} + +bool remote_test_runner::option_selected(const char *option_switch, int argc, char *argv[]) const +{ + bool selected = false; + + for (int i = 1; (i < argc) && !selected; ++i) { + + selected = (strcmp(argv[i], option_switch) == 0); + } + + return selected; +} + +void remote_test_runner::output_summary(const struct test_summary &summary) +{ + printf("\n"); + + if (summary.num_failed == 0) printf("OK ("); + else printf("Errors (%d failures, ", summary.num_failed); + + printf("%d tests, %d ran)\n", summary.num_tests, summary.num_failed + summary.num_passed); +} + + +void remote_test_runner::output_list(const struct test_summary &summary, + const std::vector<struct test_result> &results) +{ + +} + +void remote_test_runner::output_results(const struct test_summary &summary, + const std::vector<struct test_result> &results) +{ + for (int i = 0; i < results.size(); ++i) { + + printf("TEST(%s, %s) ", results[i].group, results[i].name); + + if (results[i].run_state == TEST_RUN_STATE_PASSED) { + + printf("OK\n"); + } + else if (results[i].run_state == TEST_RUN_STATE_FAILED) { + + printf("error\n"); + } + else { + + printf("did not run\n"); + } + } + + output_summary(summary); +} diff --git a/components/app/remote-test-runner/remote_test_runner.h b/components/app/remote-test-runner/remote_test_runner.h new file mode 100644 index 000000000..9d1fca483 --- /dev/null +++ b/components/app/remote-test-runner/remote_test_runner.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef REMOTE_TEST_RUNNER_H +#define REMOTE_TEST_RUNNER_H + +#include <service/test_runner/client/cpp/test_runner_client.h> +#include <service/test_runner/common/test_runner.h> + +/* + * Provides a command line interface for running remote tests. + */ +class remote_test_runner +{ +public: + remote_test_runner(); + remote_test_runner(test_runner_client *client); + virtual ~remote_test_runner(); + + void set_client(test_runner_client *client); + + int execute(int argc, char *argv[]); + +private: + + void parse_test_spec_params(int argc, char *argv[], struct test_spec &spec) const; + std::string parse_option(const char *option_switch, int argc, char *argv[]) const; + bool option_selected(const char *option_switch, int argc, char *argv[]) const; + + void output_summary(const struct test_summary &summary); + void output_list(const struct test_summary &summary, const std::vector<struct test_result> &results); + void output_results(const struct test_summary &summary, const std::vector<struct test_result> &results); + + test_runner_client *m_client; +}; + +#endif /* REMOTE_TEST_RUNNER_H */ diff --git a/components/common/tlv/test/tlv_tests.cpp b/components/common/tlv/test/tlv_tests.cpp index 661a81024..5d8a50d21 100644 --- a/components/common/tlv/test/tlv_tests.cpp +++ b/components/common/tlv/test/tlv_tests.cpp @@ -76,6 +76,26 @@ TEST(TlvTests, findAndDecode) UNSIGNED_LONGS_EQUAL(1, decoded_record.length); } +TEST(TlvTests, findAndDecodeMissingOptional) +{ + struct tlv_const_iterator iter; + struct tlv_record decoded_record; + + /* + * Checks finding a missing record is correctly + * identified as not present but that the following + * record is found. + */ + const uint8_t encoded_records[] = { + 0x00, 0x07, 0x00, 0x02, 0x77, 0x77 + }; + + tlv_const_iterator_begin(&iter, encoded_records, sizeof(encoded_records)); + CHECK(!tlv_find_decode(&iter, 0x0001, &decoded_record)); + CHECK(tlv_find_decode(&iter, 0x0007, &decoded_record)); + CHECK_EQUAL(2, decoded_record.length); +} + TEST(TlvTests, decodeBadRecords) { struct tlv_const_iterator iter; diff --git a/components/common/tlv/tlv.c b/components/common/tlv/tlv.c index 13088321c..80c1edbdd 100644 --- a/components/common/tlv/tlv.c +++ b/components/common/tlv/tlv.c @@ -82,10 +82,13 @@ bool tlv_decode(struct tlv_const_iterator *iter, struct tlv_record *output) bool tlv_find_decode(struct tlv_const_iterator *iter, uint16_t tag, struct tlv_record *output) { - while (tlv_decode(iter, output)) { + struct tlv_const_iterator temp_iter = *iter; + + while (tlv_decode(&temp_iter, output)) { if (output->tag == tag) { - /* Found a record */ + /* Found a record - update input iterator to next record */ + *iter = temp_iter; return true; } else if (output->tag > tag) { diff --git a/components/service/crypto/provider/serializer/packed-c/packedc_crypto_provider_serializer.c b/components/service/crypto/provider/serializer/packed-c/packedc_crypto_provider_serializer.c index f39aa1cff..078a88f0b 100644 --- a/components/service/crypto/provider/serializer/packed-c/packedc_crypto_provider_serializer.c +++ b/components/service/crypto/provider/serializer/packed-c/packedc_crypto_provider_serializer.c @@ -233,7 +233,7 @@ static rpc_status_t deserialize_import_key_req(const struct call_param_buf *req_ if (expected_fixed_len <= req_buf->data_len) { - struct tlv_const_iterator resp_iter; + struct tlv_const_iterator req_iter; struct tlv_record decoded_record; rpc_status = TS_RPC_CALL_ACCEPTED; @@ -241,11 +241,11 @@ static rpc_status_t deserialize_import_key_req(const struct call_param_buf *req_ memcpy(&recv_msg, req_buf->data, expected_fixed_len); packedc_crypto_provider_translate_key_attributes(attributes, &recv_msg.attributes); - tlv_const_iterator_begin(&resp_iter, + tlv_const_iterator_begin(&req_iter, (uint8_t*)req_buf->data + expected_fixed_len, req_buf->data_len - expected_fixed_len); - if (tlv_find_decode(&resp_iter, TS_CRYPTO_IMPORT_KEY_IN_TAG_DATA, &decoded_record)) { + if (tlv_find_decode(&req_iter, TS_CRYPTO_IMPORT_KEY_IN_TAG_DATA, &decoded_record)) { if (decoded_record.length <= *data_len) { @@ -296,7 +296,7 @@ static rpc_status_t deserialize_sign_hash_req(const struct call_param_buf *req_b if (expected_fixed_len <= req_buf->data_len) { - struct tlv_const_iterator resp_iter; + struct tlv_const_iterator req_iter; struct tlv_record decoded_record; rpc_status = TS_RPC_CALL_ACCEPTED; @@ -306,11 +306,11 @@ static rpc_status_t deserialize_sign_hash_req(const struct call_param_buf *req_b *handle = recv_msg.handle; *alg = recv_msg.alg; - tlv_const_iterator_begin(&resp_iter, + tlv_const_iterator_begin(&req_iter, (uint8_t*)req_buf->data + expected_fixed_len, req_buf->data_len - expected_fixed_len); - if (tlv_find_decode(&resp_iter, TS_CRYPTO_SIGN_HASH_IN_TAG_HASH, &decoded_record)) { + if (tlv_find_decode(&req_iter, TS_CRYPTO_SIGN_HASH_IN_TAG_HASH, &decoded_record)) { if (decoded_record.length <= *hash_len) { @@ -365,7 +365,7 @@ static rpc_status_t deserialize_verify_hash_req(const struct call_param_buf *req if (expected_fixed_len <= req_buf->data_len) { - struct tlv_const_iterator resp_iter; + struct tlv_const_iterator req_iter; struct tlv_record decoded_record; rpc_status = TS_RPC_CALL_ACCEPTED; @@ -375,11 +375,11 @@ static rpc_status_t deserialize_verify_hash_req(const struct call_param_buf *req *handle = recv_msg.handle; *alg = recv_msg.alg; - tlv_const_iterator_begin(&resp_iter, + tlv_const_iterator_begin(&req_iter, (uint8_t*)req_buf->data + expected_fixed_len, req_buf->data_len - expected_fixed_len); - if (tlv_find_decode(&resp_iter, TS_CRYPTO_VERIFY_HASH_IN_TAG_HASH, &decoded_record)) { + if (tlv_find_decode(&req_iter, TS_CRYPTO_VERIFY_HASH_IN_TAG_HASH, &decoded_record)) { if (decoded_record.length <= *hash_len) { @@ -396,7 +396,7 @@ static rpc_status_t deserialize_verify_hash_req(const struct call_param_buf *req *hash_len = 0; } - if (tlv_find_decode(&resp_iter, TS_CRYPTO_VERIFY_HASH_IN_TAG_SIGNATURE, &decoded_record)) { + if (tlv_find_decode(&req_iter, TS_CRYPTO_VERIFY_HASH_IN_TAG_SIGNATURE, &decoded_record)) { if (decoded_record.length <= *sig_len) { @@ -429,7 +429,7 @@ static rpc_status_t deserialize_asymmetric_decrypt_req(const struct call_param_b if (expected_fixed_len <= req_buf->data_len) { - struct tlv_const_iterator resp_iter; + struct tlv_const_iterator req_iter; struct tlv_record decoded_record; rpc_status = TS_RPC_CALL_ACCEPTED; @@ -439,11 +439,11 @@ static rpc_status_t deserialize_asymmetric_decrypt_req(const struct call_param_b *handle = recv_msg.handle; *alg = recv_msg.alg; - tlv_const_iterator_begin(&resp_iter, + tlv_const_iterator_begin(&req_iter, (uint8_t*)req_buf->data + expected_fixed_len, req_buf->data_len - expected_fixed_len); - if (tlv_find_decode(&resp_iter, TS_CRYPTO_ASYMMETRIC_DECRYPT_IN_TAG_CIPHERTEXT, &decoded_record)) { + if (tlv_find_decode(&req_iter, TS_CRYPTO_ASYMMETRIC_DECRYPT_IN_TAG_CIPHERTEXT, &decoded_record)) { if (decoded_record.length <= *ciphertext_len) { @@ -460,7 +460,7 @@ static rpc_status_t deserialize_asymmetric_decrypt_req(const struct call_param_b *ciphertext_len = 0; } - if (tlv_find_decode(&resp_iter, TS_CRYPTO_ASYMMETRIC_DECRYPT_IN_TAG_SALT, &decoded_record)) { + if (tlv_find_decode(&req_iter, TS_CRYPTO_ASYMMETRIC_DECRYPT_IN_TAG_SALT, &decoded_record)) { if (decoded_record.length <= *salt_len) { @@ -515,7 +515,7 @@ static rpc_status_t deserialize_asymmetric_encrypt_req(const struct call_param_b if (expected_fixed_len <= req_buf->data_len) { - struct tlv_const_iterator resp_iter; + struct tlv_const_iterator req_iter; struct tlv_record decoded_record; rpc_status = TS_RPC_CALL_ACCEPTED; @@ -525,11 +525,11 @@ static rpc_status_t deserialize_asymmetric_encrypt_req(const struct call_param_b *handle = recv_msg.handle; *alg = recv_msg.alg; - tlv_const_iterator_begin(&resp_iter, + tlv_const_iterator_begin(&req_iter, (uint8_t*)req_buf->data + expected_fixed_len, req_buf->data_len - expected_fixed_len); - if (tlv_find_decode(&resp_iter, TS_CRYPTO_ASYMMETRIC_ENCRYPT_IN_TAG_PLAINTEXT, &decoded_record)) { + if (tlv_find_decode(&req_iter, TS_CRYPTO_ASYMMETRIC_ENCRYPT_IN_TAG_PLAINTEXT, &decoded_record)) { if (decoded_record.length <= *plaintext_len) { @@ -546,7 +546,7 @@ static rpc_status_t deserialize_asymmetric_encrypt_req(const struct call_param_b *plaintext_len = 0; } - if (tlv_find_decode(&resp_iter, TS_CRYPTO_ASYMMETRIC_ENCRYPT_IN_TAG_SALT, &decoded_record)) { + if (tlv_find_decode(&req_iter, TS_CRYPTO_ASYMMETRIC_ENCRYPT_IN_TAG_SALT, &decoded_record)) { if (decoded_record.length <= *salt_len) { diff --git a/components/service/locator/linux/ffa/linuxffa_location_strategy.c b/components/service/locator/linux/ffa/linuxffa_location_strategy.c index e9650769f..f7e78ad92 100644 --- a/components/service/locator/linux/ffa/linuxffa_location_strategy.c +++ b/components/service/locator/linux/ffa/linuxffa_location_strategy.c @@ -90,6 +90,7 @@ static size_t suggest_tf_org_partition_uuids(const char *sn, struct uuid_canonic { {"crypto", "d9df52d5-16a2-4bb2-9aa4-d26d3b84e8c0"}, {"secure-storage", "dc1eef48-b17a-4ccf-ac8b-dfcff7711b14"}, + {"test-runner", "33c75baf-ac6a-4fe4-8ac7-e9909bee2d17"}, {NULL, NULL} }; diff --git a/components/service/locator/standalone/services/test-runner/component.cmake b/components/service/locator/standalone/services/test-runner/component.cmake new file mode 100644 index 000000000..e30f2c641 --- /dev/null +++ b/components/service/locator/standalone/services/test-runner/component.cmake @@ -0,0 +1,14 @@ +#------------------------------------------------------------------------------- +# Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# +#------------------------------------------------------------------------------- +if (NOT DEFINED TGT) + message(FATAL_ERROR "mandatory parameter TGT is not defined.") +endif() + +target_sources(${TGT} PRIVATE + "${CMAKE_CURRENT_LIST_DIR}/test_runner_service_context.cpp" + ) + diff --git a/components/service/locator/standalone/services/test-runner/test_runner_service_context.cpp b/components/service/locator/standalone/services/test-runner/test_runner_service_context.cpp new file mode 100644 index 000000000..103f18806 --- /dev/null +++ b/components/service/locator/standalone/services/test-runner/test_runner_service_context.cpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2020-2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "test_runner_service_context.h" +#include <service/test_runner/provider/serializer/packed-c/packedc_test_runner_provider_serializer.h> + +test_runner_service_context::test_runner_service_context(const char *sn) : + standalone_service_context(sn), + m_test_runner_provider() +{ + +} + +test_runner_service_context::~test_runner_service_context() +{ + +} + +void test_runner_service_context::do_init() +{ + struct rpc_interface *test_runner_ep = test_runner_provider_init(&m_test_runner_provider); + + test_runner_provider_register_serializer(&m_test_runner_provider, + TS_RPC_ENCODING_PACKED_C, packedc_test_runner_provider_serializer_instance()); + + standalone_service_context::set_rpc_interface(test_runner_ep); +} + +void test_runner_service_context::do_deinit() +{ + test_runner_provider_deinit(&m_test_runner_provider); +} diff --git a/components/service/locator/standalone/services/test-runner/test_runner_service_context.h b/components/service/locator/standalone/services/test-runner/test_runner_service_context.h new file mode 100644 index 000000000..8474ec22f --- /dev/null +++ b/components/service/locator/standalone/services/test-runner/test_runner_service_context.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef STANDALONE_TEST_RUNNER_SERVICE_CONTEXT_H +#define STANDALONE_TEST_RUNNER_SERVICE_CONTEXT_H + +#include <service/locator/standalone/standalone_service_context.h> +#include <rpc/direct/direct_caller.h> +#include <service/test_runner/provider/test_runner_provider.h> + +class test_runner_service_context : public standalone_service_context +{ +public: + test_runner_service_context(const char *sn); + virtual ~test_runner_service_context(); + +private: + + void do_init(); + void do_deinit(); + + struct test_runner_provider m_test_runner_provider; +}; + +#endif /* STANDALONE_TEST_RUNNER_SERVICE_CONTEXT_H */ diff --git a/components/service/locator/standalone/standalone_env.cpp b/components/service/locator/standalone/standalone_env.cpp index 80d1777c2..41dd206ab 100644 --- a/components/service/locator/standalone/standalone_env.cpp +++ b/components/service/locator/standalone/standalone_env.cpp @@ -1,11 +1,12 @@ /* - * Copyright (c) 2020, Arm Limited and Contributors. All rights reserved. + * Copyright (c) 2020-2021, Arm Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ #include <service_locator.h> #include <service/locator/standalone/services/crypto/crypto_service_context.h> +#include <service/locator/standalone/services/test-runner/test_runner_service_context.h> #include "standalone_location_strategy.h" #include "standalone_service_registry.h" @@ -13,5 +14,9 @@ void service_locator_envinit(void) { static crypto_service_context crypto_context("sn:trustedfirmware.org:crypto:0"); standalone_service_registry::instance()->regsiter_service_instance(&crypto_context); + + static test_runner_service_context test_runner_context("sn:trustedfirmware.org:test-runner:0"); + standalone_service_registry::instance()->regsiter_service_instance(&test_runner_context); + service_locator_register_strategy(standalone_location_strategy()); }
\ No newline at end of file diff --git a/components/service/test_runner/client/cpp/component.cmake b/components/service/test_runner/client/cpp/component.cmake new file mode 100644 index 000000000..8bd9e51ec --- /dev/null +++ b/components/service/test_runner/client/cpp/component.cmake @@ -0,0 +1,14 @@ +#------------------------------------------------------------------------------- +# Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# +#------------------------------------------------------------------------------- +if (NOT DEFINED TGT) + message(FATAL_ERROR "mandatory parameter TGT is not defined.") +endif() + +target_sources(${TGT} PRIVATE + "${CMAKE_CURRENT_LIST_DIR}/test_runner_client.cpp" + ) + diff --git a/components/service/test_runner/client/cpp/test_runner_client.cpp b/components/service/test_runner/client/cpp/test_runner_client.cpp new file mode 100644 index 000000000..a78000255 --- /dev/null +++ b/components/service/test_runner/client/cpp/test_runner_client.cpp @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "test_runner_client.h" +#include <protocols/rpc/common/packed-c/status.h> +#include <protocols/service/test_runner/packed-c/opcodes.h> +#include <protocols/service/test_runner/packed-c/status.h> +#include <protocols/service/test_runner/packed-c/run_tests.h> +#include <protocols/service/test_runner/packed-c/list_tests.h> +#include <rpc_caller.h> +#include <common/tlv/tlv.h> +#include <cstddef> +#include <cstring> +#include <string> + +test_runner_client::test_runner_client() : + m_caller(NULL), + m_err_rpc_status(TS_RPC_CALL_ACCEPTED) +{ + +} + +test_runner_client::test_runner_client(struct rpc_caller *caller) : + m_caller(caller), + m_err_rpc_status(TS_RPC_CALL_ACCEPTED) +{ + +} + +test_runner_client::~test_runner_client() +{ + +} + +void test_runner_client::set_caller(struct rpc_caller *caller) +{ + m_caller = caller; +} + +int test_runner_client::err_rpc_status() const +{ + return m_err_rpc_status; +} + +int test_runner_client::run_tests( + const struct test_spec &spec, + struct test_summary &summary, + std::vector<struct test_result> &results) +{ + return iterate_over_tests(spec, false, summary, results); +} + +int test_runner_client::list_tests( + const struct test_spec &spec, + struct test_summary &summary, + std::vector<struct test_result> &results) +{ + return iterate_over_tests(spec, true, summary, results); +} + +int test_runner_client::iterate_over_tests( + const struct test_spec &spec, bool list_only, + struct test_summary &summary, + std::vector<struct test_result> &results) +{ + int test_status = TS_TEST_RUNNER_STATUS_ERROR; + m_err_rpc_status = TS_RPC_ERROR_RESOURCE_FAILURE; + rpc_call_handle call_handle; + uint8_t *req_buf; + std::vector<uint8_t> req_param; + + serialize_test_spec(req_param, spec); + + size_t req_len = req_param.size(); + call_handle = rpc_caller_begin(m_caller, &req_buf, req_len); + + if (call_handle) { + + uint8_t *resp_buf; + size_t resp_len; + int opstatus; + + memcpy(req_buf, req_param.data(), req_len); + + uint32_t opcode = (list_only) ? + TS_TEST_RUNNER_OPCODE_LIST_TESTS : + TS_TEST_RUNNER_OPCODE_RUN_TESTS; + + m_err_rpc_status = rpc_caller_invoke(m_caller, call_handle, + opcode, &opstatus, &resp_buf, &resp_len); + + if (m_err_rpc_status == TS_RPC_CALL_ACCEPTED) { + + test_status = opstatus; + + if (opstatus == TS_TEST_RUNNER_STATUS_SUCCESS) { + + test_status = deserialize_results(resp_buf, resp_len, summary, results); + } + } + + rpc_caller_end(m_caller, call_handle); + } + + return test_status; +} + +void test_runner_client::serialize_test_spec( + std::vector<uint8_t> &serialized_data, + const struct test_spec &spec) const +{ + size_t name_len = strlen(spec.name); + size_t group_len = strlen(spec.group); + size_t tlv_space = 0; + + /* First determine buffer space needed for TLV parameters */ + if (name_len) tlv_space += tlv_required_space(name_len); + if (group_len) tlv_space += tlv_required_space(group_len); + + /* Extend the params vector and write the tlv records */ + if (tlv_space) { + + serialized_data.resize(tlv_space); + struct tlv_iterator iter; + uint8_t *buf = serialized_data.data(); + + tlv_iterator_begin(&iter, buf, tlv_space); + + if (name_len) { + + struct tlv_record record; + record.tag = TS_TEST_RUNNER_TEST_SPEC_TAG_NAME; + record.length = name_len; + record.value = (const uint8_t*)spec.name; + tlv_encode(&iter, &record); + } + + if (group_len) { + + struct tlv_record record; + record.tag = TS_TEST_RUNNER_TEST_SPEC_TAG_GROUP; + record.length = group_len; + record.value = (const uint8_t*)spec.group; + tlv_encode(&iter, &record); + } + } +} + +int test_runner_client::deserialize_results( + const uint8_t *resp_buf, size_t resp_len, + struct test_summary &summary, + std::vector<struct test_result> &results) const +{ + int test_status = TS_TEST_RUNNER_STATUS_SUCCESS; + size_t fixed_size = sizeof(ts_test_runner_result_summary); + + if (resp_len >= fixed_size) { + + /* Deserialize fixed size summary structure */ + struct ts_test_runner_result_summary packed_summary; + memcpy(&packed_summary, resp_buf, fixed_size); + + summary.num_tests = packed_summary.num_tests; + summary.num_passed = packed_summary.num_passed; + summary.num_failed = packed_summary.num_failed; + summary.num_results = 0; + + /* Deserialize any test result records */ + struct tlv_const_iterator tlv_iter; + struct tlv_record result_record; + tlv_const_iterator_begin(&tlv_iter, &resp_buf[fixed_size], resp_len - fixed_size); + + while (tlv_find_decode(&tlv_iter, TS_TEST_RUNNER_TEST_RESULT_TAG, &result_record)) { + + struct test_result result; + + test_status = deserialize_result(result_record.value, result_record.length, result); + + if (test_status == TS_TEST_RUNNER_STATUS_SUCCESS) { + + results.push_back(result); + ++summary.num_results; + } + else { + /* Failed to decode result record */ + break; + } + } + } + else { + /* Failed to mandatory test summary */ + test_status = TS_TEST_RUNNER_STATUS_INVALID_TEST_RESULTS; + } + + return test_status; +} + +int test_runner_client::deserialize_result( + const uint8_t *value_buf, size_t value_len, + struct test_result &result) const +{ + int test_status = TS_TEST_RUNNER_STATUS_SUCCESS; + size_t fixed_size = sizeof(ts_test_runner_test_result); + + if (value_len >= fixed_size) { + + struct ts_test_runner_test_result packed_result; + memcpy(&packed_result, value_buf, fixed_size); + + result.run_state = (enum test_run_state)packed_result.run_state; + result.fail_line = packed_result.fail_line; + result.name[0] = 0; + result.group[0] = 0; + + /* Deserialize name and group if present */ + struct tlv_const_iterator req_iter; + struct tlv_record decoded_record; + + tlv_const_iterator_begin(&req_iter, &value_buf[fixed_size], value_len - fixed_size); + + if (tlv_find_decode(&req_iter, TS_TEST_RUNNER_TEST_RESULT_TAG_NAME, &decoded_record)) { + + if ((decoded_record.length > 0) && (decoded_record.length < TEST_NAME_MAX_LEN)) { + + memcpy(result.name, decoded_record.value, decoded_record.length); + result.name[decoded_record.length] = 0; + } + } + + if (tlv_find_decode(&req_iter, TS_TEST_RUNNER_TEST_RESULT_TAG_GROUP, &decoded_record)) { + + if ((decoded_record.length > 0) && (decoded_record.length < TEST_GROUP_MAX_LEN)) { + + memcpy(result.group, decoded_record.value, decoded_record.length); + result.group[decoded_record.length] = 0; + } + } + } + else { + /* Invalid test result */ + test_status = TS_TEST_RUNNER_STATUS_INVALID_TEST_RESULTS; + } + + return test_status; +} diff --git a/components/service/test_runner/client/cpp/test_runner_client.h b/components/service/test_runner/client/cpp/test_runner_client.h new file mode 100644 index 000000000..b86b93e09 --- /dev/null +++ b/components/service/test_runner/client/cpp/test_runner_client.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef TEST_RUNNER_CLIENT_H +#define TEST_RUNNER_CLIENT_H + +#include <cstdint> +#include <vector> +#include <string> +#include <service/test_runner/common/test_runner.h> + +struct rpc_caller; + +/* + * Provides a client interface for running remote tests using the test-runner + * service access protocol. + */ +class test_runner_client +{ +public: + test_runner_client(); + test_runner_client(struct rpc_caller *caller); + virtual ~test_runner_client(); + + void set_caller(struct rpc_caller *caller); + int err_rpc_status() const; + + int run_tests(const struct test_spec &spec, + struct test_summary &summary, + std::vector<struct test_result> &results); + + int list_tests(const struct test_spec &spec, + struct test_summary &summary, + std::vector<struct test_result> &results); + +private: + + int iterate_over_tests(const struct test_spec &spec, bool list_only, + struct test_summary &summary, + std::vector<struct test_result> &results); + + void serialize_test_spec(std::vector<uint8_t> &serialized_data, + const struct test_spec &spec) const; + + int deserialize_results(const uint8_t *resp_buf, size_t resp_len, + struct test_summary &summary, + std::vector<struct test_result> &results) const; + + int deserialize_result(const uint8_t *value_buf, size_t value_len, + struct test_result &result) const; + + struct rpc_caller *m_caller; + int m_err_rpc_status; +}; + +#endif /* TEST_RUNNER_CLIENT_H */ diff --git a/components/service/test_runner/common/test_runner.h b/components/service/test_runner/common/test_runner.h new file mode 100644 index 000000000..c891918cb --- /dev/null +++ b/components/service/test_runner/common/test_runner.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2020-2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef CPPUTEST_TEST_RUNNER_H +#define CPPUTEST_TEST_RUNNER_H + +#include <protocols/service/test_runner/packed-c/test_result.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Constants + */ +#define TEST_NAME_MAX_LEN (30) +#define TEST_GROUP_MAX_LEN (30) + +/** + * Specifies a set of tests for running or listing. + */ +struct test_spec +{ + char name[TEST_NAME_MAX_LEN]; + char group[TEST_GROUP_MAX_LEN]; +}; + +/** + * A summary of a set of tests qualified by a test_spec. + */ +struct test_summary +{ + int num_tests; /* Number of qualifying tests */ + int num_results; /* Number of available test result objects */ + int num_passed; /* Number that ran and passed */ + int num_failed; /* Number that ran and failed */ +}; + +/** + * The run state of a test case + */ +enum test_run_state +{ + TEST_RUN_STATE_NOT_RUN = TS_TEST_RUNNER_TEST_RESULT_RUN_STATE_NOT_RUN, + TEST_RUN_STATE_PASSED = TS_TEST_RUNNER_TEST_RESULT_RUN_STATE_PASSED, + TEST_RUN_STATE_FAILED = TS_TEST_RUNNER_TEST_RESULT_RUN_STATE_FAILED +}; + +/** + * The result for a particular test case. + */ +struct test_result +{ + char name[TEST_NAME_MAX_LEN]; + char group[TEST_GROUP_MAX_LEN]; + enum test_run_state run_state; + unsigned int fail_line; +}; + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* CPPUTEST_TEST_RUNNER_H */ diff --git a/components/service/test_runner/provider/backend/mock/component.cmake b/components/service/test_runner/provider/backend/mock/component.cmake new file mode 100644 index 000000000..4666b2b61 --- /dev/null +++ b/components/service/test_runner/provider/backend/mock/component.cmake @@ -0,0 +1,14 @@ +#------------------------------------------------------------------------------- +# Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# +#------------------------------------------------------------------------------- +if (NOT DEFINED TGT) + message(FATAL_ERROR "mandatory parameter TGT is not defined.") +endif() + +target_sources(${TGT} PRIVATE + "${CMAKE_CURRENT_LIST_DIR}/mock_test_runner.c" + ) + diff --git a/components/service/test_runner/provider/backend/mock/mock_test_runner.c b/components/service/test_runner/provider/backend/mock/mock_test_runner.c new file mode 100644 index 000000000..9ea593631 --- /dev/null +++ b/components/service/test_runner/provider/backend/mock/mock_test_runner.c @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <service/test_runner/provider/test_runner_backend.h> +#include <service/test_runner/provider/test_runner_provider.h> +#include <stdbool.h> +#include <string.h> + +/** + * The mock backend is a test_runner that provides some mock test cases + * that can be used for testing the test_runner service iteslf. + */ + +struct mock_test_case +{ + const char *group; + const char *name; + bool (*test_func)(void); +}; + +/* Mock test test functions */ +static bool test_that_passes(void) { return true; } +static bool test_that_fails(void) { return false; } + +/* Mock test suite */ +const struct mock_test_case mock_test_suite[] = +{ + {.group = "PlatformTests", .name = "Trng", .test_func = test_that_passes}, + {.group = "PlatformTests", .name = "CheckIOmap", .test_func = test_that_passes}, + {.group = "ConfigTests", .name = "ValidateConfig", .test_func = test_that_fails}, + {.group = "ConfigTests", .name = "ApplyConfig", .test_func = test_that_passes} +}; + + +static bool does_qualify(const struct mock_test_case *test_case, const struct test_spec *spec) +{ + return + ((strlen(spec->group) == 0) || (strcmp(spec->group, test_case->group) == 0)) && + ((strlen(spec->name) == 0) || (strcmp(spec->name, test_case->name) == 0)); +} + +static size_t count_tests(const struct test_spec *spec) +{ + size_t count = 0; + + for (size_t i = 0; i < sizeof(mock_test_suite)/sizeof(struct mock_test_case); ++i) { + + if (does_qualify(&mock_test_suite[i], spec)) ++count; + } + + return count; +} + +static int run_tests(const struct test_spec *spec, struct test_summary *summary, + struct test_result *results, size_t result_limit) +{ + summary->num_tests = 0; + summary->num_results = 0; + summary->num_passed = 0; + summary->num_failed = 0; + + for (size_t i = 0; i < sizeof(mock_test_suite)/sizeof(struct mock_test_case); ++i) { + + if (does_qualify(&mock_test_suite[i], spec)) { + + bool did_pass = mock_test_suite[i].test_func(); + + if (did_pass) + ++summary->num_passed; + else + ++summary->num_failed; + + if (summary->num_tests < result_limit) { + + struct test_result *new_result = &results[summary->num_results]; + + new_result->run_state = (did_pass) ? TEST_RUN_STATE_PASSED : TEST_RUN_STATE_FAILED; + new_result->fail_line = 0; + strcpy(new_result->group, mock_test_suite[i].group); + strcpy(new_result->name, mock_test_suite[i].name); + + ++summary->num_results; + } + + ++summary->num_tests; + } + } + + return 0; +} + +static void list_tests(const struct test_spec *spec, struct test_summary *summary, + struct test_result *results, size_t result_limit) +{ + summary->num_tests = 0; + summary->num_results = 0; + summary->num_passed = 0; + summary->num_failed = 0; + + for (size_t i = 0; i < sizeof(mock_test_suite)/sizeof(struct mock_test_case); ++i) { + + if (does_qualify(&mock_test_suite[i], spec)) { + + if (summary->num_tests < result_limit) { + + struct test_result *new_result = &results[summary->num_results]; + + new_result->run_state = TEST_RUN_STATE_NOT_RUN; + new_result->fail_line = 0; + strcpy(new_result->group, mock_test_suite[i].group); + strcpy(new_result->name, mock_test_suite[i].name); + + ++summary->num_results; + } + + ++summary->num_tests; + } + } +} + +void test_runner_register_default_backend(struct test_runner_provider *context) +{ + static struct test_runner_backend this_instance; + + this_instance.count_tests = count_tests; + this_instance.run_tests = run_tests; + this_instance.list_tests = list_tests; + this_instance.next = NULL; + + test_runner_provider_register_backend(context, &this_instance); +}
\ No newline at end of file diff --git a/components/service/test_runner/provider/backend/null/component.cmake b/components/service/test_runner/provider/backend/null/component.cmake new file mode 100644 index 000000000..81974121e --- /dev/null +++ b/components/service/test_runner/provider/backend/null/component.cmake @@ -0,0 +1,14 @@ +#------------------------------------------------------------------------------- +# Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# +#------------------------------------------------------------------------------- +if (NOT DEFINED TGT) + message(FATAL_ERROR "mandatory parameter TGT is not defined.") +endif() + +target_sources(${TGT} PRIVATE + "${CMAKE_CURRENT_LIST_DIR}/null_test_runner.c" + ) + diff --git a/components/service/test_runner/provider/backend/null/null_test_runner.c b/components/service/test_runner/provider/backend/null/null_test_runner.c new file mode 100644 index 000000000..9d4afb509 --- /dev/null +++ b/components/service/test_runner/provider/backend/null/null_test_runner.c @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <service/test_runner/provider/test_runner_backend.h> + +/** + * The null backend is a non-existent test_runner. An implementation of + * the test_runner_register_default_backend() function is provided but it does + * nothing. This component should be used in deployments where there is + * no need for a default test_runner. + */ +void test_runner_register_default_backend(struct test_runner_provider *context) +{ + /* Don't register anything */ + (void)context; +}
\ No newline at end of file diff --git a/components/service/test_runner/provider/component.cmake b/components/service/test_runner/provider/component.cmake new file mode 100644 index 000000000..4a3764833 --- /dev/null +++ b/components/service/test_runner/provider/component.cmake @@ -0,0 +1,13 @@ +#------------------------------------------------------------------------------- +# Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# +#------------------------------------------------------------------------------- +if (NOT DEFINED TGT) + message(FATAL_ERROR "mandatory parameter TGT is not defined.") +endif() + +target_sources(${TGT} PRIVATE + "${CMAKE_CURRENT_LIST_DIR}/test_runner_provider.c" + ) diff --git a/components/service/test_runner/provider/serializer/packed-c/component.cmake b/components/service/test_runner/provider/serializer/packed-c/component.cmake new file mode 100644 index 000000000..1180035b4 --- /dev/null +++ b/components/service/test_runner/provider/serializer/packed-c/component.cmake @@ -0,0 +1,13 @@ +#------------------------------------------------------------------------------- +# Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# +#------------------------------------------------------------------------------- +if (NOT DEFINED TGT) + message(FATAL_ERROR "mandatory parameter TGT is not defined.") +endif() + +target_sources(${TGT} PRIVATE + "${CMAKE_CURRENT_LIST_DIR}/packedc_test_runner_provider_serializer.c" + ) diff --git a/components/service/test_runner/provider/serializer/packed-c/packedc_test_runner_provider_serializer.c b/components/service/test_runner/provider/serializer/packed-c/packedc_test_runner_provider_serializer.c new file mode 100644 index 000000000..577bfdc8f --- /dev/null +++ b/components/service/test_runner/provider/serializer/packed-c/packedc_test_runner_provider_serializer.c @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <stddef.h> +#include <stdlib.h> +#include <string.h> +#include <common/tlv/tlv.h> +#include <protocols/rpc/common/packed-c/status.h> +#include <protocols/service/test_runner/packed-c/run_tests.h> +#include <protocols/service/test_runner/packed-c/list_tests.h> +#include "packedc_test_runner_provider_serializer.h" + +/* Common rerialization methods used for different operations */ +static rpc_status_t deserialize_test_spec(const struct call_param_buf *req_buf, + struct test_spec *test_spec) +{ + struct tlv_const_iterator req_iter; + struct tlv_record decoded_record; + + test_spec->name[0] = 0; + test_spec->group[0] = 0; + + tlv_const_iterator_begin(&req_iter, (uint8_t*)req_buf->data, req_buf->data_len); + + if (tlv_find_decode(&req_iter, TS_TEST_RUNNER_TEST_SPEC_TAG_NAME, &decoded_record)) { + + if ((decoded_record.length > 0) && (decoded_record.length < TEST_NAME_MAX_LEN)) { + + memcpy(test_spec->name, decoded_record.value, decoded_record.length); + test_spec->name[decoded_record.length] = 0; + } + } + + if (tlv_find_decode(&req_iter, TS_TEST_RUNNER_TEST_SPEC_TAG_GROUP, &decoded_record)) { + + if ((decoded_record.length > 0) && (decoded_record.length < TEST_GROUP_MAX_LEN)) { + + memcpy(test_spec->group, decoded_record.value, decoded_record.length); + test_spec->group[decoded_record.length] = 0; + } + } + + return TS_RPC_CALL_ACCEPTED; +} + +static uint8_t *serialize_test_result(const struct test_result *result, size_t *serialized_len) +{ + uint8_t *out_buf; + size_t fixed_len = sizeof(struct ts_test_runner_test_result); + size_t required_space = fixed_len; + size_t name_len = strlen(result->name); + size_t group_len = strlen(result->group); + + if (name_len) required_space += tlv_required_space(name_len); + if (group_len) required_space += tlv_required_space(group_len); + + *serialized_len = required_space; + + out_buf = malloc(required_space); + + if (out_buf) { + + struct ts_test_runner_test_result result_msg; + result_msg.run_state = result->run_state; + result_msg.fail_line = result->fail_line; + + memcpy(out_buf, &result_msg, fixed_len); + + struct tlv_iterator tlv_iter; + tlv_iterator_begin(&tlv_iter, (uint8_t*)out_buf + fixed_len, required_space - fixed_len); + + if (name_len) { + + struct tlv_record record; + record.tag = TS_TEST_RUNNER_TEST_RESULT_TAG_NAME; + record.length = name_len; + record.value = result->name; + tlv_encode(&tlv_iter, &record); + } + + if (group_len) { + + struct tlv_record record; + record.tag = TS_TEST_RUNNER_TEST_RESULT_TAG_GROUP; + record.length = group_len; + record.value = result->group; + tlv_encode(&tlv_iter, &record); + } + } + + return out_buf; +} + +static rpc_status_t serialize_test_results(struct call_param_buf *resp_buf, + const struct test_summary *summary, + const struct test_result *results) +{ + size_t space_used = 0; + rpc_status_t rpc_status = TS_RPC_CALL_ACCEPTED; + + /* Serialize fixed size summary */ + struct ts_test_runner_result_summary summary_msg; + size_t fixed_len = sizeof(struct ts_test_runner_result_summary); + + summary_msg.num_tests = summary->num_tests; + summary_msg.num_passed = summary->num_passed; + summary_msg.num_failed = summary->num_failed; + + if (fixed_len + space_used <= resp_buf->size) { + + memcpy((uint8_t*)resp_buf->data + space_used, &summary_msg, fixed_len); + space_used += fixed_len; + + /* Serialize test result objects */ + struct tlv_iterator resp_iter; + tlv_iterator_begin(&resp_iter, (uint8_t*)resp_buf->data + space_used, resp_buf->size - space_used); + + for (int i = 0; (i < summary->num_results) && (rpc_status == TS_RPC_CALL_ACCEPTED); ++i) { + + size_t serialised_len; + uint8_t *serialize_buf = serialize_test_result(&results[i], &serialised_len); + + if (serialize_buf) { + + struct tlv_record result_record; + result_record.tag = TS_TEST_RUNNER_TEST_RESULT_TAG; + result_record.length = serialised_len; + result_record.value = serialize_buf; + + if (tlv_encode(&resp_iter, &result_record)) { + + space_used += tlv_required_space(serialised_len); + } + else { + /* Insufficient buffer space */ + rpc_status = TS_RPC_ERROR_RESOURCE_FAILURE; + } + + free(serialize_buf); + } + } + } + + resp_buf->data_len = space_used; + + return rpc_status; +} + +/* Operation: run_tests */ +static rpc_status_t deserialize_run_tests_req(const struct call_param_buf *req_buf, + struct test_spec *test_spec) +{ + return deserialize_test_spec(req_buf, test_spec); +} + +static rpc_status_t serialize_run_tests_resp(struct call_param_buf *resp_buf, + const struct test_summary *summary, + const struct test_result *results) +{ + return serialize_test_results(resp_buf, summary, results); +} + +/* Operation: list_tests */ +static rpc_status_t deserialize_list_tests_req(const struct call_param_buf *req_buf, + struct test_spec *test_spec) +{ + return deserialize_test_spec(req_buf, test_spec); +} + +static rpc_status_t serialize_list_tests_resp(struct call_param_buf *resp_buf, + const struct test_summary *summary, + const struct test_result *results) +{ + return serialize_test_results(resp_buf, summary, results); +} + +/* Singleton method to provide access to the serializer instance */ +const struct test_runner_provider_serializer *packedc_test_runner_provider_serializer_instance(void) +{ + static const struct test_runner_provider_serializer instance = { + deserialize_run_tests_req, + serialize_run_tests_resp, + deserialize_list_tests_req, + serialize_list_tests_resp + }; + + return &instance; +} diff --git a/components/service/test_runner/provider/serializer/packed-c/packedc_test_runner_provider_serializer.h b/components/service/test_runner/provider/serializer/packed-c/packedc_test_runner_provider_serializer.h new file mode 100644 index 000000000..c956bc30a --- /dev/null +++ b/components/service/test_runner/provider/serializer/packed-c/packedc_test_runner_provider_serializer.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef PACKEDC_TEST_RUNNER_PROVIDER_SERIALIZER_H +#define PACKEDC_TEST_RUNNER_PROVIDER_SERIALIZER_H + +#include <service/test_runner/provider/serializer/test_runner_provider_serializer.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/* Singleton method to provide access to the packed-c serializer + * for the test_runner service provider. + */ +const struct test_runner_provider_serializer *packedc_test_runner_provider_serializer_instance(void); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* PACKEDC_TEST_RUNNER_PROVIDER_SERIALIZER_H */ diff --git a/components/service/test_runner/provider/serializer/test_runner_provider_serializer.h b/components/service/test_runner/provider/serializer/test_runner_provider_serializer.h new file mode 100644 index 000000000..bd457f77b --- /dev/null +++ b/components/service/test_runner/provider/serializer/test_runner_provider_serializer.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef TEST_RUNNER_PROVIDER_SERIALIZER_H +#define TEST_RUNNER_PROVIDER_SERIALIZER_H + +#include <rpc/common/endpoint/rpc_interface.h> +#include <service/test_runner/common/test_runner.h> + +/* Provides a common interface for parameter serialization operations + * for the test_runner service provider. Allows alternative serialization + * protocols to be used without hard-wiring a particular protocol + * into the service provider code. A concrete serializer must + * implement this interface. + */ +struct test_runner_provider_serializer { + + /* Operation: run_tests */ + rpc_status_t (*deserialize_run_tests_req)(const struct call_param_buf *req_buf, + struct test_spec *test_spec); + + rpc_status_t (*serialize_run_tests_resp)(struct call_param_buf *resp_buf, + const struct test_summary *summary, + const struct test_result *results); + + /* Operation: list_tests */ + rpc_status_t (*deserialize_list_tests_req)(const struct call_param_buf *req_buf, + struct test_spec *test_spec); + + rpc_status_t (*serialize_list_tests_resp)(struct call_param_buf *resp_buf, + const struct test_summary *summary, + const struct test_result *results); +}; + +#endif /* TEST_RUNNER_PROVIDER_SERIALIZER_H */ diff --git a/components/service/test_runner/provider/test_runner_backend.h b/components/service/test_runner/provider/test_runner_backend.h new file mode 100644 index 000000000..9c9cbc815 --- /dev/null +++ b/components/service/test_runner/provider/test_runner_backend.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef CPPUTEST_TEST_RUNNER_BACKEND_H +#define CPPUTEST_TEST_RUNNER_BACKEND_H + +#include <stddef.h> +#include <service/test_runner/common/test_runner.h> + +#ifdef __cplusplus +extern "C" { +#endif + +struct test_runner_provider; + +/** + * Provides an abstract interface for a backend test runner. A + * concrete implementation will map methods to a test framework + * such as CppUtets. test_runner objects may be linked to + * accommodate a mix of backend test frameworks. + */ +struct test_runner_backend +{ + /* Return the number of tests that are qualified by the test spec */ + size_t (*count_tests)(const struct test_spec *spec); + + /* Run a set of tests according to the provided test_spec */ + int (*run_tests)(const struct test_spec *spec, + struct test_summary *summary, struct test_result *results, size_t result_limit); + + /* List a set of tests according to the provided test_spec */ + void (*list_tests)(const struct test_spec *spec, + struct test_summary *summary, struct test_result *results, size_t result_limit); + + /* Used by the test_runner_provider to maintain a linked list */ + struct test_runner_backend *next; +}; + +/** + * A concrete test_runner may provide an implementation of this function if it + * is to be the default test runner for a deployment. Additional test runners + * may be registered but there can only be one default for a deployment. + */ +void test_runner_register_default_backend(struct test_runner_provider *context); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* CPPUTEST_TEST_RUNNER_BACKEND_H */ diff --git a/components/service/test_runner/provider/test_runner_provider.c b/components/service/test_runner/provider/test_runner_provider.c new file mode 100644 index 000000000..4b61bc1d7 --- /dev/null +++ b/components/service/test_runner/provider/test_runner_provider.c @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <stdlib.h> +#include <stdbool.h> +#include <protocols/service/test_runner/packed-c/opcodes.h> +#include <protocols/service/test_runner/packed-c/status.h> +#include <protocols/rpc/common/packed-c/status.h> +#include "test_runner_provider.h" +#include "test_runner_backend.h" + +/* Service request handlers */ +static rpc_status_t run_tests_handler(void *context, struct call_req* req); +static rpc_status_t list_tests_handler(void *context, struct call_req* req); + +/* Handler mapping table for service */ +static const struct service_handler handler_table[] = { + {TS_TEST_RUNNER_OPCODE_RUN_TESTS, run_tests_handler}, + {TS_TEST_RUNNER_OPCODE_LIST_TESTS, list_tests_handler} +}; + +struct rpc_interface *test_runner_provider_init(struct test_runner_provider *context) +{ + struct rpc_interface *rpc_interface = NULL; + + if (context) { + + for (size_t encoding = 0; encoding < TS_RPC_ENCODING_LIMIT; ++encoding) + context->serializers[encoding] = NULL; + + context->backend_list = NULL; + + service_provider_init(&context->base_provider, context, + handler_table, sizeof(handler_table)/sizeof(struct service_handler)); + + rpc_interface = service_provider_get_rpc_interface(&context->base_provider); + + /* Allow a deployment specific test_runner backend to be registrered */ + test_runner_register_default_backend(context); + } + + return rpc_interface; +} + +void test_runner_provider_deinit(struct test_runner_provider *context) +{ + (void)context; +} + +void test_runner_provider_register_serializer(struct test_runner_provider *context, + unsigned int encoding, const struct test_runner_provider_serializer *serializer) +{ + if (encoding < TS_RPC_ENCODING_LIMIT) + context->serializers[encoding] = serializer; +} + +void test_runner_provider_register_backend(struct test_runner_provider *context, + struct test_runner_backend *backend) +{ + /* Insert into list of backend test runners */ + backend->next = context->backend_list; + context->backend_list = backend; +} + +static const struct test_runner_provider_serializer* get_test_runner_serializer( + struct test_runner_provider *context, const struct call_req *req) +{ + const struct test_runner_provider_serializer* serializer = NULL; + unsigned int encoding = call_req_get_encoding(req); + + if (encoding < TS_RPC_ENCODING_LIMIT) serializer = context->serializers[encoding]; + + return serializer; +} + +static struct test_result *alloc_result_buf(struct test_runner_provider *context, + const struct test_spec *test_spec, size_t *result_limit) +{ + struct test_result *space = NULL; + size_t total_tests = 0; + struct test_runner_backend *backend = context->backend_list; + + while (backend) { + + total_tests += backend->count_tests(test_spec); + backend = backend->next; + } + + space = malloc(total_tests * sizeof(struct test_result)); + + *result_limit = total_tests; + return space; +} + +static int run_qualifying_tests(struct test_runner_provider *context, bool list_only, const struct test_spec *spec, + struct test_summary *summary, struct test_result *results, size_t result_limit) +{ + int test_status = TS_TEST_RUNNER_STATUS_SUCCESS; + size_t results_used = 0; + struct test_runner_backend *backend = context->backend_list; + + summary->num_tests = 0; + summary->num_results = 0; + summary->num_passed = 0; + summary->num_failed = 0; + + while (backend && (test_status == TS_TEST_RUNNER_STATUS_SUCCESS)) { + + struct test_summary interim_summary; + + if (list_only) { + + backend->list_tests(spec, &interim_summary, + &results[summary->num_results], + result_limit - summary->num_results); + } + else { + + test_status = backend->run_tests(spec, &interim_summary, + &results[summary->num_results], + result_limit - summary->num_results); + } + + summary->num_tests += interim_summary.num_tests; + summary->num_results += interim_summary.num_results; + summary->num_passed += interim_summary.num_passed; + summary->num_failed += interim_summary.num_failed; + + backend = backend->next; + } + + return test_status; +} + +static rpc_status_t run_tests_handler(void *context, struct call_req* req) +{ + struct test_runner_provider *this_instance = (struct test_runner_provider*)context; + rpc_status_t rpc_status = TS_RPC_ERROR_SERIALIZATION_NOT_SUPPORTED; + struct test_spec test_spec; + + struct call_param_buf *req_buf = call_req_get_req_buf(req); + const struct test_runner_provider_serializer *serializer = get_test_runner_serializer(this_instance, req); + + if (serializer) + rpc_status = serializer->deserialize_run_tests_req(req_buf, &test_spec); + + if (rpc_status == TS_RPC_CALL_ACCEPTED) { + + int test_status; + struct test_summary summary; + size_t result_limit = 0; + struct test_result *result_buf = alloc_result_buf(this_instance, &test_spec, &result_limit); + + test_status = run_qualifying_tests(this_instance, false, &test_spec, &summary, result_buf, result_limit); + + call_req_set_opstatus(req, test_status); + + if (test_status == TS_TEST_RUNNER_STATUS_SUCCESS) { + + struct call_param_buf *resp_buf = call_req_get_resp_buf(req); + rpc_status = serializer->serialize_run_tests_resp(resp_buf, &summary, result_buf); + + free(result_buf); + } + } + + return rpc_status; +} + +static rpc_status_t list_tests_handler(void *context, struct call_req* req) +{ + struct test_runner_provider *this_instance = (struct test_runner_provider*)context; + rpc_status_t rpc_status = TS_RPC_ERROR_SERIALIZATION_NOT_SUPPORTED; + struct test_spec test_spec; + + struct call_param_buf *req_buf = call_req_get_req_buf(req); + const struct test_runner_provider_serializer *serializer = get_test_runner_serializer(this_instance, req); + + if (serializer) + rpc_status = serializer->deserialize_list_tests_req(req_buf, &test_spec); + + if (rpc_status == TS_RPC_CALL_ACCEPTED) { + + int test_status; + struct test_summary summary; + size_t result_limit = 0; + struct test_result *result_buf = alloc_result_buf(this_instance, &test_spec, &result_limit); + + test_status = run_qualifying_tests(this_instance, true, &test_spec, &summary, result_buf, result_limit); + + call_req_set_opstatus(req, test_status); + + if (test_status == TS_TEST_RUNNER_STATUS_SUCCESS) { + + struct call_param_buf *resp_buf = call_req_get_resp_buf(req); + rpc_status = serializer->serialize_list_tests_resp(resp_buf, &summary, result_buf); + + free(result_buf); + } + } + + return rpc_status; +}
\ No newline at end of file diff --git a/components/service/test_runner/provider/test_runner_provider.h b/components/service/test_runner/provider/test_runner_provider.h new file mode 100644 index 000000000..e5c92304b --- /dev/null +++ b/components/service/test_runner/provider/test_runner_provider.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef TEST_RUNNER_PROVIDER_H +#define TEST_RUNNER_PROVIDER_H + +#include <rpc/common/endpoint/rpc_interface.h> +#include <rpc_caller.h> +#include <service/common/provider/service_provider.h> +#include <service/test_runner/provider/serializer/test_runner_provider_serializer.h> +#include <protocols/rpc/common/packed-c/encoding.h> +#include "test_runner_backend.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* test_runner_provider serice provider state */ +struct test_runner_provider +{ + struct service_provider base_provider; + const struct test_runner_provider_serializer *serializers[TS_RPC_ENCODING_LIMIT]; + struct test_runner_backend *backend_list; +}; + +struct rpc_interface *test_runner_provider_init(struct test_runner_provider *context); + +void test_runner_provider_deinit(struct test_runner_provider *context); + +void test_runner_provider_register_serializer(struct test_runner_provider *context, + unsigned int encoding, const struct test_runner_provider_serializer *serializer); + +void test_runner_provider_register_backend(struct test_runner_provider *context, + struct test_runner_backend *backend); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* CPPUTEST_TEST_RUNNER_PROVIDER_H */ diff --git a/components/service/test_runner/test/service/component.cmake b/components/service/test_runner/test/service/component.cmake new file mode 100644 index 000000000..7cbd452c9 --- /dev/null +++ b/components/service/test_runner/test/service/component.cmake @@ -0,0 +1,14 @@ +#------------------------------------------------------------------------------- +# Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# +#------------------------------------------------------------------------------- +if (NOT DEFINED TGT) + message(FATAL_ERROR "mandatory parameter TGT is not defined.") +endif() + +target_sources(${TGT} PRIVATE + "${CMAKE_CURRENT_LIST_DIR}/test_runner_service_tests.cpp" + ) + diff --git a/components/service/test_runner/test/service/test_runner_service_tests.cpp b/components/service/test_runner/test/service/test_runner_service_tests.cpp new file mode 100644 index 000000000..c4ba0e07f --- /dev/null +++ b/components/service/test_runner/test/service/test_runner_service_tests.cpp @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2021, Arm Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <service/test_runner/client/cpp/test_runner_client.h> +#include <protocols/rpc/common/packed-c/encoding.h> +#include <protocols/service/test_runner/packed-c/status.h> +#include <service_locator.h> +#include <CppUTest/TestHarness.h> +#include <vector> +#include <cstring> + +/* + * Service-level tests for the test_runner service. These tests assume + * that the mock_test_runner backend is used as the only registered + * backend. It includes some referenece test cases that are assumed + * by these tests. + */ +TEST_GROUP(TestRunnerServiceTests) +{ + void setup() + { + struct rpc_caller *caller; + int status; + + m_rpc_session_handle = NULL; + m_test_runner_service_context = NULL; + m_test_runner_client = NULL; + + service_locator_init(); + + m_test_runner_service_context = service_locator_query("sn:trustedfirmware.org:test-runner:0", &status); + CHECK(m_test_runner_service_context); + + m_rpc_session_handle = service_context_open(m_test_runner_service_context, TS_RPC_ENCODING_PACKED_C, &caller); + CHECK(m_rpc_session_handle); + + m_test_runner_client = new test_runner_client(caller); + } + + void teardown() + { + delete m_test_runner_client; + m_test_runner_client = NULL; + + service_context_close(m_test_runner_service_context, m_rpc_session_handle); + m_rpc_session_handle = NULL; + + service_context_relinquish(m_test_runner_service_context); + m_test_runner_service_context = NULL; + } + + rpc_session_handle m_rpc_session_handle; + struct service_context *m_test_runner_service_context; + test_runner_client *m_test_runner_client; +}; + +TEST(TestRunnerServiceTests, listAllTests) +{ + int test_status; + struct test_spec spec; + struct test_summary summary; + std::vector<struct test_result> results; + + /* Create spec that qualifies all tests */ + spec.name[0] = 0; + spec.group[0] = 0; + + test_status = m_test_runner_client->list_tests(spec, summary, results); + + CHECK_EQUAL(TS_TEST_RUNNER_STATUS_SUCCESS, test_status); + + /* Check test summary */ + CHECK_EQUAL(4, summary.num_tests); + CHECK_EQUAL(4, summary.num_results); + CHECK_EQUAL(0, summary.num_passed); + CHECK_EQUAL(0, summary.num_failed); + + /* Check each test result is listed but not run */ + CHECK(strcmp(results[0].group, "PlatformTests") == 0); + CHECK(strcmp(results[0].name, "Trng") == 0); + CHECK_EQUAL(TEST_RUN_STATE_NOT_RUN, results[0].run_state); + + CHECK(strcmp(results[1].group, "PlatformTests") == 0); + CHECK(strcmp(results[1].name, "CheckIOmap") == 0); + CHECK_EQUAL(TEST_RUN_STATE_NOT_RUN, results[1].run_state); + + CHECK(strcmp(results[2].group, "ConfigTests") == 0); + CHECK(strcmp(results[2].name, "ValidateConfig") == 0); + CHECK_EQUAL(TEST_RUN_STATE_NOT_RUN, results[2].run_state); + + CHECK(strcmp(results[3].group, "ConfigTests") == 0); + CHECK(strcmp(results[3].name, "ApplyConfig") == 0); + CHECK_EQUAL(TEST_RUN_STATE_NOT_RUN, results[3].run_state); +} + +TEST(TestRunnerServiceTests, runAllTests) +{ + int test_status; + struct test_spec spec; + struct test_summary summary; + std::vector<struct test_result> results; + + /* Create spec that qualifies all tests */ + spec.name[0] = 0; + spec.group[0] = 0; + + test_status = m_test_runner_client->run_tests(spec, summary, results); + + CHECK_EQUAL(TS_TEST_RUNNER_STATUS_SUCCESS, test_status); + + CHECK_EQUAL(4, summary.num_tests); + CHECK_EQUAL(4, summary.num_results); + CHECK_EQUAL(3, summary.num_passed); + CHECK_EQUAL(1, summary.num_failed); + + /* Check each test result has run with the expected outcome */ + CHECK(strcmp(results[0].group, "PlatformTests") == 0); + CHECK(strcmp(results[0].name, "Trng") == 0); + CHECK_EQUAL(TEST_RUN_STATE_PASSED, results[0].run_state); + + CHECK(strcmp(results[1].group, "PlatformTests") == 0); + CHECK(strcmp(results[1].name, "CheckIOmap") == 0); + CHECK_EQUAL(TEST_RUN_STATE_PASSED, results[1].run_state); + + CHECK(strcmp(results[2].group, "ConfigTests") == 0); + CHECK(strcmp(results[2].name, "ValidateConfig") == 0); + CHECK_EQUAL(TEST_RUN_STATE_FAILED, results[2].run_state); + + CHECK(strcmp(results[3].group, "ConfigTests") == 0); + CHECK(strcmp(results[3].name, "ApplyConfig") == 0); + CHECK_EQUAL(TEST_RUN_STATE_PASSED, results[3].run_state); +} + +TEST(TestRunnerServiceTests, listPlatformTests) +{ + int test_status; + struct test_spec spec; + struct test_summary summary; + std::vector<struct test_result> results; + + /* Create spec that qualifies all tests in a group*/ + spec.name[0] = 0; + strcpy(spec.group, "PlatformTests"); + + test_status = m_test_runner_client->list_tests(spec, summary, results); + + CHECK_EQUAL(TS_TEST_RUNNER_STATUS_SUCCESS, test_status); + + /* Check test summary */ + CHECK_EQUAL(2, summary.num_tests); + CHECK_EQUAL(2, summary.num_results); + CHECK_EQUAL(0, summary.num_passed); + CHECK_EQUAL(0, summary.num_failed); +} + +TEST(TestRunnerServiceTests, runConfigTests) +{ + int test_status; + struct test_spec spec; + struct test_summary summary; + std::vector<struct test_result> results; + + /* Create spec that qualifies all tests in a group*/ + spec.name[0] = 0; + strcpy(spec.group, "ConfigTests"); + + test_status = m_test_runner_client->run_tests(spec, summary, results); + + CHECK_EQUAL(TS_TEST_RUNNER_STATUS_SUCCESS, test_status); + + /* Check test summary */ + CHECK_EQUAL(2, summary.num_tests); + CHECK_EQUAL(2, summary.num_results); + CHECK_EQUAL(1, summary.num_passed); + CHECK_EQUAL(1, summary.num_failed); +} + +TEST(TestRunnerServiceTests, runSpecificTest) +{ + int test_status; + struct test_spec spec; + struct test_summary summary; + std::vector<struct test_result> results; + + /* Create spec that qualifies a specific test */ + strcpy(spec.name, "ValidateConfig"); + strcpy(spec.group, "ConfigTests"); + + test_status = m_test_runner_client->run_tests(spec, summary, results); + + CHECK_EQUAL(TS_TEST_RUNNER_STATUS_SUCCESS, test_status); + + /* Check test summary */ + CHECK_EQUAL(1, summary.num_tests); + CHECK_EQUAL(1, summary.num_results); + CHECK_EQUAL(0, summary.num_passed); + CHECK_EQUAL(1, summary.num_failed); +} |