aboutsummaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorMate Toth-Pal <mate.toth-pal@arm.com>2020-03-06 15:16:51 +0100
committerAnton Komlev <Anton.Komlev@arm.com>2021-05-21 13:18:07 +0200
commit2a1d01b804ed69b2b9672e1ca8b3303190cd6bd3 (patch)
tree4c2a47f70ea7d60c50ff7ee7da36a361e9e59989 /tools
parent9974f0a675c1423573d9c6a70d54a54688df6a34 (diff)
downloadtrusted-firmware-m-2a1d01b804ed69b2b9672e1ca8b3303190cd6bd3.tar.gz
TF-Fuzz: Add test suite generator script
Change-Id: I148b34b5cc3c028cc5392acc08d758d81a31ff61 Signed-off-by: Mate Toth-Pal <mate.toth-pal@arm.com> Signed-off-by: Karl Zhang <karl.zhang@arm.com>
Diffstat (limited to 'tools')
-rw-r--r--tools/tf_fuzz/README154
-rw-r--r--tools/tf_fuzz/generate_test_suite.sh209
2 files changed, 363 insertions, 0 deletions
diff --git a/tools/tf_fuzz/README b/tools/tf_fuzz/README
new file mode 100644
index 000000000..32754c216
--- /dev/null
+++ b/tools/tf_fuzz/README
@@ -0,0 +1,154 @@
+.../tf_fuzz directory contents:
+
+assets calls demo parser tests regression
+backupStuff class_forwards.hpp lib README tf_fuzz.cpp utility
+boilerplate commands Makefile template tf_fuzz.hpp visualStudio
+
+TF-Fuzz root directory.
+
+--------------------------------------------------------------------------------
+
+TF-Fuzz is a TF-M fuzzing tool, at the PSA-call level. At the time of writing
+this at least, presentations available at:
+ https://www.trustedfirmware.org/docs/TF-M_Fuzzing_Tool_TFOrg.pdf
+ https://zoom.us/rec/share/1dxZcZit111IadadyFqFU7IoP5X5aaa8gXUdr_UInxmMbyLzEqEmXQdx79-IWQ9p
+(These presentation materials may not be viewable by all parties.)
+
+--------------------------------------------------------------------------------
+
+To build TF-Fuzz, simply type "make" in this directory. Executable, called
+"tfz," is placed in this directory.
+
+To run tfz, two environment variables must first be assigned. In bash syntax:
+export TF_FUZZ_LIB_DIR=<path to this TF-M installation>/tools/tf_fuzz/lib
+export TF_FUZZ_BPLATE=tfm_boilerplate.txt
+
+Examples of usage can be found in the demo directory.
+
+--------------------------------------------------------------------------------
+
+To generate a testsuite for TF-M from a set of template files, use
+generate_test_suite.sh.
+
+.. code-block:: bash
+
+ Usage: generate_test_suite.sh <template_dir> <suites_dir>
+
+ Where:
+ template_dir: The directory containing template files for the
+ fuzzing tool
+ suites_dir: The suites directory in the TF-M working copy.
+ i.e.: $TF-M_ROOT/test/suites
+ Example:
+ cd tools/tf_fuzz
+ ./generate_test_suite.sh $TF-M_ROOT/tools/tf_fuzz/tests/ $TF-M_ROOT/../tf-m-tests/test/suites/
+
+
+After the test suite is generated, the new test suite needs to be added to the
+TF-M build by providing the following options to the CMake generate command
+(The path needs to be aligned with the test suite dir provided for the shell
+script above):
+
+.. code-block:: bash
+
+ -DTFM_FUZZER_TOOL_TESTS=1
+ -DTFM_FUZZER_TOOL_TESTS_CMAKE_INC_PATH=$TF-M_ROOT/test/suites/tf_fuzz
+
+--------------------------------------------------------------------------------
+
+To help understand the code, below is a C++-class hierarchy used in this code
+base. They are explained further in the READMEs in their respective direc-
+tories, so the file names where the classes are defined is listed below (this,
+very roughly in order of functional interactions, of chronological usage during
+execution, and of most-to-least importance):
+
+ template_line ./template/template_line.hpp
+ sst_template_line ./template/template_line.hpp
+ read_sst_template_line ./template/sst_template_line.hpp
+ remove_sst_template_line ./template/sst_template_line.hpp
+ set_sst_template_line ./template/sst_template_line.hpp
+ policy_template_line ./template/template_line.hpp
+ read_policy_template_line ./template/crypto_template_line.hpp
+ set_policy_template_line ./template/crypto_template_line.hpp
+ key_template_line ./template/template_line.hpp
+ read_key_template_line ./template/crypto_template_line.hpp
+ remove_key_template_line ./template/crypto_template_line.hpp
+ set_key_template_line ./template/crypto_template_line.hpp
+ security_template_line ./template/template_line.hpp
+ security_hash_template_line ./template/secure_template_line.hpp
+
+ psa_call ./calls/psa_call.hpp
+ crypto_call ./calls/psa_call.hpp
+ policy_call ./calls/crypto_call.hpp
+ policy_get_call ./calls/crypto_call.hpp
+ policy_set_call ./calls/crypto_call.hpp
+ key_call ./calls/crypto_call.hpp
+ get_key_info_call ./calls/crypto_call.hpp
+ set_key_call ./calls/crypto_call.hpp
+ destroy_key_call ./calls/crypto_call.hpp
+ sst_call ./calls/psa_call.hpp
+ sst_remove_call ./calls/sst_call.hpp
+ sst_get_call ./calls/sst_call.hpp
+ sst_set_call ./calls/sst_call.hpp
+ security_call ./calls/psa_call.hpp
+ hash_call ./calls/security_call.hpp
+
+ boilerplate ./boilerplate/boilerplate.hpp
+
+ psa_asset ./assets/psa_asset.hpp
+ crypto_asset ./assets/crypto_asset.hpp
+ policy_asset ./assets/crypto_asset.hpp
+ key_asset ./assets/crypto_asset.hpp
+ sst_asset ./assets/sst_asset.hpp
+
+ tf_fuzz_info ./tf_fuzz.hpp
+
+ crc32 ./utility/compute.hpp
+
+ gibberish ./utility/gibberish.hpp
+
+ expect_info ./utility/data_blocks.hpp
+ set_data_info ./utility/data_blocks.hpp
+ asset_name_id_info ./utility/data_blocks.hpp
+
+--------------------------------------------------------------------------------
+
+There are currently two especially annoying warts on the design of TF-Fuzz:
+* Need better management of variables in the generated code. Currently,
+ for example, upon "read"ing a value from a PSA asset more than once, it
+ creates a same-named (i.e., duplicate) variable for each such time, which
+ is obviously not right.
+* Upon adding the ability to do "do any N of these PSA calls at random,"
+ in hindsight, a fundamental flaw was uncovered in the top-level flow of
+ how TF-Fuzz generates the code. High-level summary:
+ * It should have completely distinct Parse, Simulate, then Code-generation
+ stages.
+ * Currently, the Parse and Simulate stages aren't really completely
+ distinct, so there's a bunch of complicated Boolean flags traffic-
+ copping between what in hindsight should be completely-separate Parse
+ vs. Code-generation functionality.
+ The function, interpret_template_line(), currently in
+ .../tf_fuzz/parser/tf_fuzz_grammar.y (which may be moved to the its own file
+ with randomize_template_lines()), has the lion's share of such Booleans,
+ such as fill_in_template, create_call_bool, and create_asset_bool.
+ The way it *should* work is:
+ * The parser in .../tf_fuzz_grammar.y should generate an STL vector (or
+ list) of psa_call-subclass "tracker" objects. It should not generate
+ PSA-asset tracker objects (subclasses of psa_asset).
+ * There should then be an organized Simulate stage, that sequences through
+ the psa_call-subclass list, creating and accumulating/maintaining current
+ state in psa_asset-subclass objects, using that current state to
+ determine expected results of each PSA call, which get annotated back
+ into the psa_call-tracker objects.
+ * Finally, there already is, and should continue to be, a Code-generation
+ phase that writes out the code, based upon text substitutions of
+ "boilerplate" code snippets.
+ * Currently, (hindsight obvious) the Parse and Simulate phases got somewhat
+ muddled together. This shouldn't be super-hard to fix.
+ That final Code-generation phase, conceptually at least, could be replaced
+ instead with simply executing those commands directly, for targets that
+ sufficient space to run TF-Fuzz in real-time.
+
+--------------
+
+*Copyright (c) 2019-2020, Arm Limited. All rights reserved.*
diff --git a/tools/tf_fuzz/generate_test_suite.sh b/tools/tf_fuzz/generate_test_suite.sh
new file mode 100644
index 000000000..80e0b38bc
--- /dev/null
+++ b/tools/tf_fuzz/generate_test_suite.sh
@@ -0,0 +1,209 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2020-2021, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -e
+
+SUITE_DIR_NAME=tfm_fuzz
+
+show_help()
+{
+ echo "Usage: $0 <template_dir> <suites_dir> "
+ echo ""
+ echo "Where: "
+ echo " template_dir: The directory containing template files for the"
+ echo " fuzzing tool"
+ echo " suites_dir: The directory to generate the test suite to."
+}
+
+if [ $# != 2 ]
+then
+ echo "Invalid number of parameters."
+ show_help
+ exit 1
+fi
+
+INPUT_DIR=$1
+SUITES_DIR=$2
+
+# Check that the input directory exists
+if [ ! -d "$INPUT_DIR" ]
+then
+ echo "Template directory '$INPUT_DIR' doesn't exist"
+ exit 1
+fi
+
+# Check that the input directory contains files
+if [ `ls "$INPUT_DIR"| wc -l` == 0 ]
+then
+ echo "Template directory '$INPUT_DIR' doesn't contain files"
+ exit 1
+fi
+
+if [ ! -d "$SUITES_DIR" ]
+then
+ echo "Suites directory '$SUITES_DIR' doesn't exist"
+ exit 1
+fi
+
+# Check whether a fuzz test suite was generated before
+if [ -d "$SUITES_DIR/$SUITE_DIR_NAME" ]
+then
+ echo "'$SUITE_DIR_NAME' exists."
+ echo "A TF fuzz test suite is already generated, please delete it before running this script"
+ exit 1
+fi
+
+#get absolute path of the suit dir TF_FUZZ
+pushd $SUITES_DIR > /dev/null
+SUITE_ABSOLUTE_DIR=`pwd`/$SUITE_DIR_NAME
+popd
+
+# creating directory for the fuzz tests tool
+echo "Creating '$SUITES_DIR/$SUITE_DIR_NAME/non_secure'"
+mkdir -p $SUITES_DIR/$SUITE_DIR_NAME/non_secure
+
+# generate additional files for the suite:
+CMAKELIST=$SUITES_DIR/$SUITE_DIR_NAME/CMakeLists.txt
+TESTSUITE=$SUITES_DIR/$SUITE_DIR_NAME/non_secure/tf_fuzz_testsuite.c
+TESTSUITE_HEADER=$SUITES_DIR/$SUITE_DIR_NAME/non_secure/tf_fuzz_testsuite.h
+
+# generate data for the testcases
+# Iterate over the testcase files and
+# - Run the fuzzer on the test template
+# - Generate a test function name, and replace the generic test_thread to that
+# in the generated c file
+# - append the generated c file to the CmakeList file.
+declare -A FILENAMES
+declare -A FUNC_NAMES
+declare -A PURPOSES
+for f in `ls $INPUT_DIR`
+do
+ FILE="$SUITES_DIR/$SUITE_DIR_NAME/non_secure/$f.c"
+ FILENAMES[$f]=$FILE
+
+ SEED=$RANDOM
+ echo "Generating testcase $f with seed $SEED"
+ ./tfz -z $INPUT_DIR/$f $FILE $SEED
+
+ echo "in File ${FILENAMES[$f]}"
+
+ PURPOSE=`grep -A 1 'Test purpose' $FILE | tail -n 1 | cut -d '*' -f 2 | sed -e 's/^[[:space:]]*//'`
+ PURPOSES[$f]=$PURPOSE
+
+ FUNC_NAME="$PURPOSE"
+ FUNC_NAME=`echo "$FUNC_NAME" | \
+ sed 's/^\s*//g; s/\s*$//g; s/\s\s*/_/g'`
+ FUNC_NAME=`echo "$FUNC_NAME" | \
+ sed 's/\-/_/g; s/"//g; s/\\\//g'`
+ FUNC_NAME=`echo "$FUNC_NAME" | \
+ sed "s/#//g; s/'//g; s/,//g; s/(//g; s/)//g"`
+ FUNC_NAMES[$f]=$FUNC_NAME
+
+ echo " Changing 'test_thread' to '${FUNC_NAMES[$f]}'"
+
+ sed -i "s/test_thread/$FUNC_NAME/g" $FILE
+
+done
+
+# generate the non-secure testsuite file
+echo '#include <stdio.h>' >> $TESTSUITE
+echo '#include <string.h>' >> $TESTSUITE
+echo '' >> $TESTSUITE
+echo '#include "tfm_api.h"' >> $TESTSUITE
+echo '#include "psa_manifest/sid.h"' >> $TESTSUITE
+echo '#include "test_framework.h"' >> $TESTSUITE
+echo '' >> $TESTSUITE
+echo '/* Forward declaring static test functions */' >> $TESTSUITE
+for f in `ls $INPUT_DIR`
+do
+ echo "static void test_${FUNC_NAMES[$f]}(struct test_result_t *ret);" >> $TESTSUITE
+done
+echo '/* Forward declaring functions */' >> $TESTSUITE
+for f in `ls $INPUT_DIR`
+do
+ echo "void ${FUNC_NAMES[$f]}(struct test_result_t *ret);" >> $TESTSUITE
+done
+echo '' >> $TESTSUITE
+echo 'static struct test_t tf_fuzz_tests[] = {' >> $TESTSUITE
+for f in `ls $INPUT_DIR`
+do
+ echo " {&test_${FUNC_NAMES[$f]}, \"${FUNC_NAMES[$f]}\", \"${PURPOSES[$f]}\", {0} }," >> $TESTSUITE
+done
+echo '}; ' >> $TESTSUITE
+echo '' >> $TESTSUITE
+echo 'void register_testsuite_tf_fuzz_test(struct test_suite_t *p_test_suite)' >> $TESTSUITE
+echo '{' >> $TESTSUITE
+echo ' uint32_t list_size;' >> $TESTSUITE
+echo '' >> $TESTSUITE
+echo ' list_size = (sizeof(tf_fuzz_tests) / sizeof(tf_fuzz_tests[0]));' >> $TESTSUITE
+echo '' >> $TESTSUITE
+echo ' set_testsuite("TF-M fuzz tests (TF_FUZZ_TEST)",' >> $TESTSUITE
+echo ' tf_fuzz_tests, list_size, p_test_suite);' >> $TESTSUITE
+echo '}' >> $TESTSUITE
+echo '' >> $TESTSUITE
+for f in `ls $INPUT_DIR`
+do
+ echo "static void test_${FUNC_NAMES[$f]}(struct test_result_t *ret)" >> $TESTSUITE
+ echo '{' >> $TESTSUITE
+ echo " ${FUNC_NAMES[$f]}(ret);" >> $TESTSUITE
+ echo ' if (ret->val != TEST_PASSED) {' >> $TESTSUITE
+ echo ' return;' >> $TESTSUITE
+ echo ' }' >> $TESTSUITE
+ echo '' >> $TESTSUITE
+ echo '}' >> $TESTSUITE
+done
+
+# generate the CmakeList file
+echo "cmake_policy(SET CMP0079 NEW)" >> $CMAKELIST
+echo "" >> $CMAKELIST
+echo "add_library(tfm_test_suite_tf_fuzz_ns STATIC EXCLUDE_FROM_ALL)" >> $CMAKELIST
+echo "" >> $CMAKELIST
+echo "target_sources(tfm_test_suite_tf_fuzz_ns" >> $CMAKELIST
+echo " PRIVATE" >> $CMAKELIST
+for f in `ls $INPUT_DIR`
+do
+ echo " $SUITE_ABSOLUTE_DIR/non_secure/$f.c" >> $CMAKELIST
+done
+echo " $SUITE_ABSOLUTE_DIR/non_secure/tf_fuzz_testsuite.c" >> $CMAKELIST
+echo ")" >> $CMAKELIST
+echo "" >> $CMAKELIST
+echo "target_include_directories(tfm_test_suite_tf_fuzz_ns" >> $CMAKELIST
+echo " PUBLIC" >> $CMAKELIST
+echo " ./non_secure" >> $CMAKELIST
+echo " PRIVATE" >> $CMAKELIST
+echo " ." >> $CMAKELIST
+echo ")" >> $CMAKELIST
+echo "" >> $CMAKELIST
+echo "target_link_libraries(tfm_test_suite_tf_fuzz_ns" >> $CMAKELIST
+echo " PRIVATE" >> $CMAKELIST
+echo " tfm_test_framework_ns" >> $CMAKELIST
+echo " tfm_test_suite_its_ns" >> $CMAKELIST
+echo ")" >> $CMAKELIST
+echo "" >> $CMAKELIST
+echo "target_link_libraries(tfm_ns_tests" >> $CMAKELIST
+echo " INTERFACE" >> $CMAKELIST
+echo " tfm_test_suite_tf_fuzz_ns" >> $CMAKELIST
+echo ")" >> $CMAKELIST
+
+# generate the testsuite header file
+echo '#ifndef __PSA_API_FUZZ_TESTSUITE_H__' >> $TESTSUITE_HEADER
+echo '#define __PSA_API_FUZZ_TESTSUITE_H__' >> $TESTSUITE_HEADER
+echo '' >> $TESTSUITE_HEADER
+echo 'void register_testsuite_tf_fuzz_test(struct test_suite_t *p_test_suite);' >> $TESTSUITE_HEADER
+echo '' >> $TESTSUITE_HEADER
+echo '#endif /* __PSA_API_FUZZ_TESTSUITE_H__ */' >> $TESTSUITE_HEADER
+
+# print some instruction on the screen
+echo ''
+echo ''
+echo '========================================================================='
+echo '= The test suite generation is done. It can be compiled to TF-M by'
+echo '= providing the following options to the CMake generate command:'
+echo '='
+echo '= -DTFM_FUZZER_TOOL_TESTS=1'
+echo '= -DTFM_FUZZER_TOOL_TESTS_CMAKE_INC_PATH='$SUITE_ABSOLUTE_DIR
+echo '========================================================================='