Add serializer service for protobuf

Implements a serializer service for protobuf using nanopb. Adds external
component to fetch nanopb.

Change-Id: I31464be0e38d78bdf543e9f30bd51218e935dc9d
Signed-off-by: Julian Hall <julian.hall@arm.com>
diff --git a/components/service/common/component.cmake b/components/service/common/component.cmake
new file mode 100644
index 0000000..041f7d5
--- /dev/null
+++ b/components/service/common/component.cmake
@@ -0,0 +1,14 @@
+#-------------------------------------------------------------------------------
+# Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+#-------------------------------------------------------------------------------
+if (NOT DEFINED TGT)
+	message(FATAL_ERROR "mandatory parameter TGT is not defined.")
+endif()
+
+target_include_directories(${TGT}
+	 PRIVATE
+		"${CMAKE_CURRENT_LIST_DIR}"
+	)
diff --git a/components/service/common/serializer/protobuf/component.cmake b/components/service/common/serializer/protobuf/component.cmake
new file mode 100644
index 0000000..b9f8d97
--- /dev/null
+++ b/components/service/common/serializer/protobuf/component.cmake
@@ -0,0 +1,13 @@
+#-------------------------------------------------------------------------------
+# Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+#-------------------------------------------------------------------------------
+if (NOT DEFINED TGT)
+	message(FATAL_ERROR "mandatory parameter TGT is not defined.")
+endif()
+
+target_sources(${TGT} PRIVATE
+	"${CMAKE_CURRENT_LIST_DIR}/pb_helper.c"
+	)
diff --git a/components/service/common/serializer/protobuf/pb_helper.c b/components/service/common/serializer/protobuf/pb_helper.c
new file mode 100644
index 0000000..d32b30c
--- /dev/null
+++ b/components/service/common/serializer/protobuf/pb_helper.c
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include "pb_helper.h"
+#include <pb_encode.h>
+#include <pb_decode.h>
+
+static bool pb_encode_byte_array(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) {
+
+    const pb_bytes_array_t *byte_array = (const pb_bytes_array_t *)*arg;
+    if (!pb_encode_tag_for_field(stream, field)) return false;
+
+    return pb_encode_string(stream, byte_array->bytes, byte_array->size);
+}
+
+static bool pb_decode_byte_array(pb_istream_t *stream, const pb_field_t *field, void **arg) {
+
+    (void)field;
+    pb_bytes_array_t *byte_array = (pb_bytes_array_t *)*arg;
+    if (stream->bytes_left > byte_array->size) return false;
+
+    byte_array->size = stream->bytes_left;
+
+    return pb_read(stream, byte_array->bytes, stream->bytes_left);
+}
+
+pb_callback_t pb_out_byte_array(const pb_bytes_array_t *byte_array) {
+
+    pb_callback_t callback;
+    callback.funcs.encode = pb_encode_byte_array;
+    callback.arg = (void*)byte_array;
+
+    return callback;
+}
+
+pb_callback_t pb_in_byte_array(pb_bytes_array_t *byte_array) {
+
+    pb_callback_t callback;
+    callback.funcs.decode = pb_decode_byte_array;
+    callback.arg = (void*)byte_array;
+    return callback;
+}
+
+pb_bytes_array_t *pb_malloc_byte_array(size_t num_bytes) {
+
+    pb_bytes_array_t *byte_array = (pb_bytes_array_t*)malloc(offsetof(pb_bytes_array_t, bytes) + num_bytes);
+
+    if (byte_array) {
+
+        byte_array->size = num_bytes;
+    }
+
+    return byte_array;
+}
+
+pb_bytes_array_t *pb_malloc_byte_array_containing_string(const char *str) {
+
+    pb_bytes_array_t *byte_array;
+    size_t required_space = strlen(str) + 1;
+
+    byte_array = pb_malloc_byte_array(required_space);
+
+    if (byte_array) {
+
+        memcpy(byte_array->bytes, str, required_space);
+    }
+
+    return byte_array;
+}
+
+pb_bytes_array_t *pb_malloc_byte_array_containing_bytes(const uint8_t *buf, size_t num_bytes) {
+
+    pb_bytes_array_t *byte_array = pb_malloc_byte_array(num_bytes);
+
+    if (byte_array) {
+
+        memcpy(byte_array->bytes, buf, num_bytes);
+    }
+
+    return byte_array;
+}
diff --git a/components/service/common/serializer/protobuf/pb_helper.h b/components/service/common/serializer/protobuf/pb_helper.h
new file mode 100644
index 0000000..1f109b0
--- /dev/null
+++ b/components/service/common/serializer/protobuf/pb_helper.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef PB_HELPER_H
+#define PB_HELPER_H
+
+#include <pb.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Returns an initialised pb_callback_t structure for encoding a variable length byte array */
+extern pb_callback_t pb_out_byte_array(const pb_bytes_array_t *byte_array);
+
+/* Returns an initialised pb_callback_t structure for decoding a variable length byte array */
+extern pb_callback_t pb_in_byte_array(pb_bytes_array_t *byte_array);
+
+/* Malloc space for a pb_bytes_array_t object with space for the requested number of bytes */
+extern pb_bytes_array_t *pb_malloc_byte_array(size_t num_bytes);
+
+/* Malloc space for a pb_bytes_array_t object containing the given string */
+extern pb_bytes_array_t *pb_malloc_byte_array_containing_string(const char *str);
+
+/* Malloc space for a pb_bytes_array_t object containing the given bytes */
+extern pb_bytes_array_t *pb_malloc_byte_array_containing_bytes(const uint8_t *buf, size_t num_bytes);
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* PB_HELPER_H */
diff --git a/external/nanopb/fix-pyhon-name.patch b/external/nanopb/fix-pyhon-name.patch
new file mode 100644
index 0000000..ab0e84c
--- /dev/null
+++ b/external/nanopb/fix-pyhon-name.patch
@@ -0,0 +1,41 @@
+This patch fixes two issues:
+
+1. On windows the python3 executable is not allways called "python3". As a result
+    "protoc" execution can fail due to the shebang in the file. This patch fixes
+    this by running protoc with the intepreter.
+
+2. In addition when not running from a virtualenv, the install path for python file
+    is set to the "user site-packages" to avoid needing elevated access rights.
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 31c86e7..e827015 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -54,13 +54,25 @@ if(nanopb_BUILD_GENERATOR)
+         string(REGEX REPLACE "([^;]+)" "\\1_pb2.py" generator_proto_py_file "${generator_proto}")
+         add_custom_command(
+             OUTPUT ${generator_proto_py_file}
+-            COMMAND ${nanopb_PROTOC_PATH} --python_out=${PROJECT_BINARY_DIR} -I${PROJECT_SOURCE_DIR}/generator/proto ${generator_proto_file}
++            COMMAND ${Python_EXECUTABLE} ${nanopb_PROTOC_PATH} --python_out=${PROJECT_BINARY_DIR} -I${PROJECT_SOURCE_DIR}/generator/proto ${generator_proto_file}
+             DEPENDS ${generator_proto_file}
+         )
+         add_custom_target("generate_${generator_proto_py_file}" ALL DEPENDS ${generator_proto_py_file})
++
++        if (DEFINED ENV{VIRTUAL_ENV})
++            set(PYTHON_INSTALL_DIR ${Python_SITELIB} CACHE PATH "Install location for generated python modules.")
++        else()
++            execute_process(
++                COMMAND ${Python_EXECUTABLE} -m site --user-site
++                OUTPUT_VARIABLE PYTHON_USER_SITE
++                OUTPUT_STRIP_TRAILING_WHITESPACE
++            )
++            set(PYTHON_INSTALL_DIR ${PYTHON_USER_SITE} CACHE PATH "Install location for generated python modules.")
++        endif()
++
+         install(
+             FILES ${PROJECT_BINARY_DIR}/${generator_proto_py_file}
+-            DESTINATION ${Python_SITELIB}
++            DESTINATION ${PYTHON_INSTALL_DIR}
+         )
+     endforeach()
+ endif()
diff --git a/external/nanopb/nanopb.cmake b/external/nanopb/nanopb.cmake
new file mode 100644
index 0000000..eda00e4
--- /dev/null
+++ b/external/nanopb/nanopb.cmake
@@ -0,0 +1,294 @@
+#-------------------------------------------------------------------------------
+# Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+#-------------------------------------------------------------------------------
+
+#[===[.rst:
+NonoPB integration for cmake
+----------------------------
+
+This module will:
+	- download nanopb if not available locally
+	- build the runtime static library and the generator
+	- import the static library to the build
+	- define a function to provide access to the generator
+
+Note: the python module created by the generator build will be installed under
+Python_SITELIB ("Third-party platform independent installation directory.")
+This means the build may alter the state of your system. Please use virtualenv.
+
+Note: see requirements.txt for dependnecies which need to be installed before
+running this module.
+
+#]===]
+
+#### Get the dependency
+
+set(NANOPB_URL "https://github.com/nanopb/nanopb.git" CACHE STRING "nanopb repository URL")
+set(NANOPB_REFSPEC "nanopb-0.4.2" CACHE STRING "nanopb git refspec")
+set(NANOPB_INSTALL_PATH "${CMAKE_CURRENT_BINARY_DIR}/nanopb_install" CACHE PATH "nanopb installation directory")
+set(NANOPB_PACKAGE_PATH "${NANOPB_INSTALL_PATH}/libnanopb/cmake" CACHE PATH "nanopb CMake package directory")
+
+include(FetchContent)
+
+# Checking git
+find_program(GIT_COMMAND "git")
+if (NOT GIT_COMMAND)
+	message(FATAL_ERROR "Please install git")
+endif()
+
+# Fetching nanopb
+FetchContent_Declare(
+	nanopb
+	GIT_REPOSITORY ${NANOPB_URL}
+	GIT_TAG ${NANOPB_REFSPEC}
+	GIT_SHALLOW TRUE
+	#See the .patch file for details on why it is needed.
+	PATCH_COMMAND git stash
+		COMMAND git apply ${CMAKE_CURRENT_LIST_DIR}/fix-pyhon-name.patch
+)
+
+# FetchContent_GetProperties exports nanopb_SOURCE_DIR and nanopb_BINARY_DIR variables
+FetchContent_GetProperties(nanopb)
+if(NOT nanopb_POPULATED)
+	message(STATUS "Fetching nanopb")
+	FetchContent_Populate(nanopb)
+endif()
+
+#### Build the runtime and the generator.
+if( NOT CMAKE_CROSSCOMPILING)
+	execute_process(COMMAND
+		${CMAKE_COMMAND}
+				-DBUILD_SHARED_LIBS=Off
+				-DBUILD_STATIC_LIBS=On
+				-Dnanopb_BUILD_RUNTIME=On
+				-Dnanopb_BUILD_GENERATOR=On
+				-Dnanopb_PROTOC_PATH=${nanopb_SOURCE_DIR}/generator/protoc
+				-Dnanopb_MSVC_STATIC_RUNTIME=Off
+				-DCMAKE_INSTALL_PREFIX=${NANOPB_INSTALL_PATH}
+				-DCMAKE_TOOLCHAIN_FILE=${TS_EXTERNAL_LIB_TOOLCHAIN_FILE}
+				-GUnix\ Makefiles
+				${nanopb_SOURCE_DIR}
+			WORKING_DIRECTORY
+				${nanopb_BINARY_DIR}
+			RESULT_VARIABLE _exec_error
+	)
+else()
+	execute_process(COMMAND
+		${CMAKE_COMMAND}
+				-DBUILD_SHARED_LIBS=Off
+				-DBUILD_STATIC_LIBS=On
+				-Dnanopb_BUILD_RUNTIME=On
+				-Dnanopb_BUILD_GENERATOR=On
+				-Dnanopb_PROTOC_PATH=${nanopb_SOURCE_DIR}/generator/protoc
+				-Dnanopb_MSVC_STATIC_RUNTIME=Off
+				-DCMAKE_INSTALL_PREFIX=${NANOPB_INSTALL_PATH}
+				-DCMAKE_TOOLCHAIN_FILE=${TS_EXTERNAL_LIB_TOOLCHAIN_FILE}
+				-DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY
+				-GUnix\ Makefiles
+				${nanopb_SOURCE_DIR}
+			WORKING_DIRECTORY
+				${nanopb_BINARY_DIR}
+			RESULT_VARIABLE _exec_error
+	)
+endif()
+
+if (_exec_error)
+	message(FATAL_ERROR "Configuration step of nanopb runtime failed with ${_exec_error}.")
+endif()
+
+execute_process(COMMAND
+		${CMAKE_COMMAND} --build ${nanopb_BINARY_DIR} -- install -j8
+		RESULT_VARIABLE _exec_error
+	)
+if (_exec_error)
+	message(FATAL_ERROR "Build step of nanopb runtime failed with ${_exec_error}.")
+endif()
+
+#### Include Nanopb runtime in the build.
+find_package(Nanopb
+			PATHS "${NANOPB_INSTALL_PATH}"
+			NO_DEFAULT_PATH
+		)
+
+#### Build access to the protobuf compiler
+#TODO: verify protoc dependencies: python3-protobuf
+find_package(Python3 COMPONENTS Interpreter)
+
+if (NOT Python3_Interpreter_FOUND)
+	message(FATAL_ERROR "Failed to find python3 interpreter.")
+endif()
+
+find_file(NANOPB_GENERATOR_PATH
+			NAMES nanopb_generator.py
+			PATHS ${nanopb_SOURCE_DIR}/generator
+			DOC "nanopb protobuf compiler"
+			NO_DEFAULT_PATH
+		)
+
+if (NOT NANOPB_GENERATOR_PATH)
+	message(FATAL_ERROR "Nanopb generator was not found!")
+endif()
+
+#[===[.rst:
+.. cmake:command:: protobuf_generate
+
+  .. code-block:: cmake
+
+	 protobuf_generate(SRC file.proto
+					 TGT foo
+					 NAMESPACE bar
+					 BASE_DIR "proto/definitions")
+
+  Run the ``nanopb_generator`` to compile a protobuf definition file into C source.
+  Generated source file will be added to the source list of ``TGT``. Protobuf
+  compilation will take part before TGT+NAMESPACE is built.
+
+  Protobuf file names added to the same TGT must not collide.
+
+  Inputs:
+
+  ``SRC``
+	Path to of the protobuf file to process. Either absoluto or relative to the
+	callers location.
+
+  ``TGT``
+	Name of target to compile generated source files.
+
+  ``NAMESPACE``
+	Namespace to put generated files under. Specifies include path and allows
+	separating colliding protobuf files.
+
+  ``BASE_DIR``
+	Base directory. Generated files are located reletive to this base.
+
+#]===]
+function(protobuf_generate)
+	set(_options )
+	set(_oneValueArgs SRC TGT NAMESPACE BASE_DIR)
+	set(_multiValueArgs )
+
+	cmake_parse_arguments(PARAMS "${_options}" "${_oneValueArgs}" "${_multiValueArgs}" ${ARGN})
+
+	#Verify mandatory parameters
+	if (NOT DEFINED PARAMS_SRC)
+		message(FATAL_ERROR "nanopb_generate(): mandatory parameter SRC missing.")
+	endif()
+	if (NOT DEFINED PARAMS_TGT)
+		message(FATAL_ERROR "nanopb_generate(): mandatory parameter TGT missing.")
+	endif()
+	if (NOT DEFINED PARAMS_NAMESPACE)
+		message(FATAL_ERROR "nanopb_generate(): mandatory parameter NAMESPACE missing.")
+	endif()
+	if (NOT DEFINED PARAMS_BASE_DIR)
+		message(FATAL_ERROR "nanopb_generate(): mandatory parameter BASE_DIR missing.")
+	endif()
+
+	#If SRC is not abolute make it relative to the callers location.
+	if (NOT IS_ABSOLUTE ${PARAMS_SRC})
+		set(PARAMS_SRC "${CMAKE_CURRENT_LIST_DIR}/${PARAMS_SRC}")
+	endif()
+
+	#Calculate the output directory
+	set(_OUT_DIR_BASE ${CMAKE_BINARY_DIR}/src/${PARAMS_NAMESPACE})
+	#Calculate output file names
+	get_filename_component(_BASENAME ${PARAMS_SRC} NAME_WE)
+
+	#Get relative path or SRC to BASE_DIR
+	file(RELATIVE_PATH _SRC_REL ${PARAMS_BASE_DIR} ${PARAMS_SRC})
+	get_filename_component(_OUT_DIR_REL ${_SRC_REL} DIRECTORY )
+
+	#Calculate output file paths
+	set(_OUT_C "${_OUT_DIR_BASE}/${_OUT_DIR_REL}/${_BASENAME}.pb.c")
+	set(_OUT_H "${_OUT_DIR_BASE}/${_OUT_DIR_REL}/${_BASENAME}.pb.h")
+
+	#some helper variables for the purpose of readability
+	set(_nanopb_target "nanopb_generate_${PARAMS_TGT}_${PARAMS_NAMESPACE}")
+	set(_nanopb_fake_file "nanopb_generate_ff_${PARAMS_TGT}")
+
+	if (NOT TARGET "${_nanopb_target}")
+		#Create a custom target which depends on a "fake" file.
+		add_custom_target("${_nanopb_target}"
+							DEPENDS "${_nanopb_fake_file}")
+		#Tell cmake the dependency (source) file is fake.
+		set_source_files_properties("${_nanopb_fake_file}" PROPERTIES SYMBOLIC "true")
+		#Add a cutom command to the target to create output directory.
+		add_custom_command(OUTPUT "${_nanopb_fake_file}"
+			COMMAND ${CMAKE_COMMAND} -E make_directory ${_OUT_DIR_BASE}
+			COMMENT "Generating source from protobuf definitions for target ${PARAMS_TGT}")
+		#Ensure protobuf build happens before test target.
+		add_dependencies(${PARAMS_TGT} ${_nanopb_target})
+		#Add include path to protobuf output.
+		target_include_directories(${PARAMS_TGT} PRIVATE ${_OUT_DIR_BASE})
+	endif()
+
+	#Append a protobuf generator command to the nanopb_generate target.
+	add_custom_command(OUTPUT "${_nanopb_fake_file}" "${_OUT_C}" "${_OUT_H}"
+					   APPEND
+					   COMMAND ${Python3_EXECUTABLE} ${NANOPB_GENERATOR_PATH}
+						  -I ${PARAMS_BASE_DIR}
+						  -D ${_OUT_DIR_BASE}
+						  ${_SRC_REL}
+					   DEPENDS "${PARAMS_SRC}")
+
+	#Add generated file to the target
+	set_property(SOURCE "${_OUT_C}" PROPERTY GENERATED TRUE)
+	target_sources(${PARAMS_TGT} PRIVATE "${_OUT_C}")
+endfunction()
+
+#[===[.rst:
+.. cmake:command:: protobuf_generate_all
+
+  .. code-block:: cmake
+
+	 protobuf_generate_all(TGT foo
+					 NAMESPACE bar
+					 BASE_DIR "proto/definitions")
+
+  Generates C code from all .proto files listed in the target
+  property PROTOBUF_FILES.
+
+  Inputs:
+
+   ``TGT``
+	Name of target to compile generated source files.
+
+  ``NAMESPACE``
+	Namespace to put generated files under. Specifies include path and allows
+	separating colliding protobuf files.
+
+  ``BASE_DIR``
+	Base directory. Generated files are located reletive to this base.
+
+#]===]
+function(protobuf_generate_all)
+	set(_options )
+	set(_oneValueArgs TGT NAMESPACE BASE_DIR)
+	set(_multiValueArgs )
+
+	cmake_parse_arguments(PARAMS "${_options}" "${_oneValueArgs}" "${_multiValueArgs}" ${ARGN})
+
+	#Verify mandatory parameters
+	if (NOT DEFINED PARAMS_TGT)
+		message(FATAL_ERROR "nanopb_generate_all(): mandatory parameter TGT missing.")
+	endif()
+	if (NOT DEFINED PARAMS_NAMESPACE)
+		message(FATAL_ERROR "nanopb_generate_all(): mandatory parameter NAMESPACE missing.")
+	endif()
+	if (NOT DEFINED PARAMS_BASE_DIR)
+		message(FATAL_ERROR "nanopb_generate_all(): mandatory parameter BASE_DIR missing.")
+	endif()
+
+	get_property(_protolist TARGET ${PARAMS_TGT} PROPERTY PROTOBUF_FILES)
+
+	#Build of each .proto file
+	foreach(_file IN LISTS _protolist)
+		protobuf_generate(
+				TGT ${PARAMS_TGT}
+				SRC "${_file}"
+				NAMESPACE ${PARAMS_NAMESPACE}
+				BASE_DIR ${PARAMS_BASE_DIR})
+	endforeach()
+endfunction()
\ No newline at end of file
diff --git a/external/nanopb/requirements.txt b/external/nanopb/requirements.txt
new file mode 100644
index 0000000..8319923
--- /dev/null
+++ b/external/nanopb/requirements.txt
@@ -0,0 +1,9 @@
+#-------------------------------------------------------------------------------
+# Copyright (c) 2020, Arm Limited and Contributors. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+#-------------------------------------------------------------------------------
+
+python3-protobuf==2.5.0
+grpcio-tools==1.32.0
diff --git a/requirements.txt b/requirements.txt
index b2cebb4..78caa84 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,5 +7,8 @@
 # Include packages needed for socument generation
 -r docs/requirements.txt
 
+# Include packages needed for nanopb
+-r external/nanopb/requirements.txt
+
 # Include packages needed for build test tool
 -r tools/b-test/requirements.txt