compat.sh: add support to record outcome of test cases
If the environment variable MBEDTLS_TEST_OUTCOME_FILE is set,
the test outcome file records each test case in a single line
with the format of
PLATFORM;CONFIGURATION;compat;TEST CASE DESCRIPTION;RESULT;[CAUSE]
- CONFIGURATION comes from MBEDTLS_TEST_CONFIGURATION to record
configuration of each test case
- PLATFORM is either set by users or calculated from test
platform
- RESULT is one of PASS, FAIL or SKIP. If test case fails,
srv_out/cli_out follows as FAILURE CAUSE.
Signed-off-by: Yanray Wang <yanray.wang@arm.com>
diff --git a/tests/compat.sh b/tests/compat.sh
index 98433ec..75d5461 100755
--- a/tests/compat.sh
+++ b/tests/compat.sh
@@ -30,6 +30,11 @@
# where it may output seemingly unlimited length error logs.
ulimit -f 20971520
+ORIGINAL_PWD=$PWD
+if ! cd "$(dirname "$0")"; then
+ exit 125
+fi
+
# initialise counters
TESTS=0
FAILED=0
@@ -77,6 +82,17 @@
PEER_GNUTLS=""
fi
+guess_config_name() {
+ if git diff --quiet ../include/mbedtls/config.h 2>/dev/null; then
+ echo "default"
+ else
+ echo "unknown"
+ fi
+}
+: ${MBEDTLS_TEST_OUTCOME_FILE=}
+: ${MBEDTLS_TEST_CONFIGURATION:="$(guess_config_name)"}
+: ${MBEDTLS_TEST_PLATFORM:="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"}
+
# default values for options
# /!\ keep this synchronised with:
# - basic-build-test.sh
@@ -112,6 +128,8 @@
printf " \tAlso available: GnuTLS (needs v3.2.15 or higher)\n"
printf " -M|--memcheck\tCheck memory leaks and errors.\n"
printf " -v|--verbose\tSet verbose output.\n"
+ printf " --outcome-file\tFile where test outcomes are written\n"
+ printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n"
}
get_options() {
@@ -141,6 +159,9 @@
-M|--memcheck)
MEMCHECK=1
;;
+ --outcome-file)
+ shift; MBEDTLS_TEST_OUTCOME_FILE=$1
+ ;;
-h|--help)
print_usage
exit 0
@@ -1145,12 +1166,29 @@
echo "EXIT: $EXIT" >> $CLI_OUT
}
+# record_outcome <outcome> [<failure-reason>]
+record_outcome() {
+ echo "$1"
+ if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ]; then
+ # The test outcome file has the format (in single line):
+ # platform;configuration;
+ # test suite name;test case description;
+ # PASS/FAIL/SKIP;[failure cause]
+ printf '%s;%s;%s;%s;%s;%s\n' \
+ "$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \
+ "compat" "$TITLE" \
+ "$1" "${2-}" \
+ >> "$MBEDTLS_TEST_OUTCOME_FILE"
+ fi
+}
+
# display additional information if test case fails
report_fail() {
- echo "FAIL"
+ FAIL_PROMPT="outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log"
+ record_outcome "FAIL" "$FAIL_PROMPT"
cp $SRV_OUT c-srv-${TESTS}.log
cp $CLI_OUT c-cli-${TESTS}.log
- echo " ! outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log"
+ echo " ! $FAIL_PROMPT"
if [ "${LOG_FAILURE_ON_STDOUT:-0}" != 0 ]; then
echo " ! server output:"
@@ -1174,7 +1212,7 @@
# should we skip?
if [ "X$SKIP_NEXT" = "XYES" ]; then
SKIP_NEXT="NO"
- echo "SKIP"
+ record_outcome "SKIP"
SKIPPED=$(( $SKIPPED + 1 ))
return
fi
@@ -1268,10 +1306,10 @@
# report and count result
case $RESULT in
"0")
- echo PASS
+ record_outcome "PASS"
;;
"1")
- echo SKIP
+ record_outcome "SKIP"
SKIPPED=$(( $SKIPPED + 1 ))
;;
"2")
@@ -1287,13 +1325,16 @@
# MAIN
#
-if cd $( dirname $0 ); then :; else
- echo "cd $( dirname $0 ) failed" >&2
- exit 1
-fi
-
get_options "$@"
+# Make the outcome file path relative to the original directory, not
+# to .../tests
+case "$MBEDTLS_TEST_OUTCOME_FILE" in
+ [!/]*)
+ MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE"
+ ;;
+esac
+
# sanity checks, avoid an avalanche of errors
if [ ! -x "$M_SRV" ]; then
echo "Command '$M_SRV' is not an executable file" >&2