Merge pull request #7901 from tgonzalezorlandoarm/tg/7879-backport
Backport 2.28: tests/test_suite_pem: Augment DES test cases with AES: PEM
diff --git a/docs/architecture/testing/test-framework.md b/docs/architecture/testing/test-framework.md
index c537951..8baa390 100644
--- a/docs/architecture/testing/test-framework.md
+++ b/docs/architecture/testing/test-framework.md
@@ -30,17 +30,23 @@
#### SSL test case descriptions
-Each test case in `ssl-opt.sh` has a description which succinctly describes for a human audience what the test does. The test description is the first parameter to `run_tests`.
+Each test case in `ssl-opt.sh` has a description which succinctly describes for a human audience what the test does. The test description is the first parameter to `run_test`.
The same rules and guidelines apply as for [unit test descriptions](#unit-test-descriptions). In addition, the description must be written on the same line as `run_test`, in double quotes, for the sake of `check_test_cases.py`.
+### SSL cipher suite tests
+
+Each test case in `compat.sh` has a description which succinctly describes for a human audience what the test does. The test description is `$TITLE` defined in `run_client`.
+
+The same rules and guidelines apply as for [unit test descriptions](#unit-test-descriptions). In addition, failure cause in `compat.sh` is not classified as `ssl-opt.sh`, so the information of failed log files are followed as prompt.
+
## Running tests
### Outcome file
#### Generating an outcome file
-Unit tests and `ssl-opt.sh` record the outcome of each test case in a **test outcome file**. This feature is enabled if the environment variable `MBEDTLS_TEST_OUTCOME_FILE` is set. Set it to the path of the desired file.
+Unit tests, `ssl-opt.sh` and `compat.sh` record the outcome of each test case in a **test outcome file**. This feature is enabled if the environment variable `MBEDTLS_TEST_OUTCOME_FILE` is set. Set it to the path of the desired file.
If you run `all.sh --outcome-file test-outcome.csv`, this collects the outcome of all the test cases in `test-outcome.csv`.
@@ -52,7 +58,7 @@
* **Platform**: a description of the platform, e.g. `Linux-x86_64` or `Linux-x86_64-gcc7-msan`.
* **Configuration**: a unique description of the configuration (`config.h`).
-* **Test suite**: `test_suite_xxx` or `ssl-opt`.
+* **Test suite**: `test_suite_xxx`, `ssl-opt` or `compat`.
* **Test case**: the description of the test case.
* **Result**: one of `PASS`, `SKIP` or `FAIL`.
* **Cause**: more information explaining the result.
diff --git a/tests/compat.sh b/tests/compat.sh
index e7f9d49..75d5461 100755
--- a/tests/compat.sh
+++ b/tests/compat.sh
@@ -30,6 +30,11 @@
# where it may output seemingly unlimited length error logs.
ulimit -f 20971520
+ORIGINAL_PWD=$PWD
+if ! cd "$(dirname "$0")"; then
+ exit 125
+fi
+
# initialise counters
TESTS=0
FAILED=0
@@ -77,6 +82,17 @@
PEER_GNUTLS=""
fi
+guess_config_name() {
+ if git diff --quiet ../include/mbedtls/config.h 2>/dev/null; then
+ echo "default"
+ else
+ echo "unknown"
+ fi
+}
+: ${MBEDTLS_TEST_OUTCOME_FILE=}
+: ${MBEDTLS_TEST_CONFIGURATION:="$(guess_config_name)"}
+: ${MBEDTLS_TEST_PLATFORM:="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"}
+
# default values for options
# /!\ keep this synchronised with:
# - basic-build-test.sh
@@ -112,6 +128,8 @@
printf " \tAlso available: GnuTLS (needs v3.2.15 or higher)\n"
printf " -M|--memcheck\tCheck memory leaks and errors.\n"
printf " -v|--verbose\tSet verbose output.\n"
+ printf " --outcome-file\tFile where test outcomes are written\n"
+ printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n"
}
get_options() {
@@ -141,6 +159,9 @@
-M|--memcheck)
MEMCHECK=1
;;
+ --outcome-file)
+ shift; MBEDTLS_TEST_OUTCOME_FILE=$1
+ ;;
-h|--help)
print_usage
exit 0
@@ -1145,6 +1166,39 @@
echo "EXIT: $EXIT" >> $CLI_OUT
}
+# record_outcome <outcome> [<failure-reason>]
+record_outcome() {
+ echo "$1"
+ if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ]; then
+ # The test outcome file has the format (in single line):
+ # platform;configuration;
+ # test suite name;test case description;
+ # PASS/FAIL/SKIP;[failure cause]
+ printf '%s;%s;%s;%s;%s;%s\n' \
+ "$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \
+ "compat" "$TITLE" \
+ "$1" "${2-}" \
+ >> "$MBEDTLS_TEST_OUTCOME_FILE"
+ fi
+}
+
+# display additional information if test case fails
+report_fail() {
+ FAIL_PROMPT="outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log"
+ record_outcome "FAIL" "$FAIL_PROMPT"
+ cp $SRV_OUT c-srv-${TESTS}.log
+ cp $CLI_OUT c-cli-${TESTS}.log
+ echo " ! $FAIL_PROMPT"
+
+ if [ "${LOG_FAILURE_ON_STDOUT:-0}" != 0 ]; then
+ echo " ! server output:"
+ cat c-srv-${TESTS}.log
+ echo " ! ==================================================="
+ echo " ! client output:"
+ cat c-cli-${TESTS}.log
+ fi
+}
+
# run_client <name> <cipher>
run_client() {
# announce what we're going to do
@@ -1158,7 +1212,7 @@
# should we skip?
if [ "X$SKIP_NEXT" = "XYES" ]; then
SKIP_NEXT="NO"
- echo "SKIP"
+ record_outcome "SKIP"
SKIPPED=$(( $SKIPPED + 1 ))
return
fi
@@ -1252,26 +1306,14 @@
# report and count result
case $RESULT in
"0")
- echo PASS
+ record_outcome "PASS"
;;
"1")
- echo SKIP
+ record_outcome "SKIP"
SKIPPED=$(( $SKIPPED + 1 ))
;;
"2")
- echo FAIL
- cp $SRV_OUT c-srv-${TESTS}.log
- cp $CLI_OUT c-cli-${TESTS}.log
- echo " ! outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log"
-
- if [ "${LOG_FAILURE_ON_STDOUT:-0}" != 0 ]; then
- echo " ! server output:"
- cat c-srv-${TESTS}.log
- echo " ! ==================================================="
- echo " ! client output:"
- cat c-cli-${TESTS}.log
- fi
-
+ report_fail
FAILED=$(( $FAILED + 1 ))
;;
esac
@@ -1283,13 +1325,16 @@
# MAIN
#
-if cd $( dirname $0 ); then :; else
- echo "cd $( dirname $0 ) failed" >&2
- exit 1
-fi
-
get_options "$@"
+# Make the outcome file path relative to the original directory, not
+# to .../tests
+case "$MBEDTLS_TEST_OUTCOME_FILE" in
+ [!/]*)
+ MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE"
+ ;;
+esac
+
# sanity checks, avoid an avalanche of errors
if [ ! -x "$M_SRV" ]; then
echo "Command '$M_SRV' is not an executable file" >&2
diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh
index 5a563ab..3a27aac 100755
--- a/tests/ssl-opt.sh
+++ b/tests/ssl-opt.sh
@@ -167,6 +167,9 @@
-p|--preserve-logs)
PRESERVE_LOGS=1
;;
+ --outcome-file)
+ shift; MBEDTLS_TEST_OUTCOME_FILE=$1
+ ;;
--port)
shift; SRV_PORT=$1
;;
@@ -190,14 +193,6 @@
done
}
-# Make the outcome file path relative to the original directory, not
-# to .../tests
-case "$MBEDTLS_TEST_OUTCOME_FILE" in
- [!/]*)
- MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE"
- ;;
-esac
-
# Read boolean configuration options from config.h for easy and quick
# testing. Skip non-boolean options (with something other than spaces
# and a comment after "#define SYMBOL"). The variable contains a
@@ -1392,6 +1387,14 @@
get_options "$@"
+# Make the outcome file path relative to the original directory, not
+# to .../tests
+case "$MBEDTLS_TEST_OUTCOME_FILE" in
+ [!/]*)
+ MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE"
+ ;;
+esac
+
# Optimize filters: if $FILTER and $EXCLUDE can be expressed as shell
# patterns rather than regular expressions, use a case statement instead
# of calling grep. To keep the optimizer simple, it is incomplete and only