Merge pull request #5098 from gilles-peskine-arm/ssl-opt-resend-retry-2.16
Backport 2.16: Retry a test case if it fails due to an unexpected resend
diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh
index dac9942..daf03b6 100755
--- a/tests/ssl-opt.sh
+++ b/tests/ssl-opt.sh
@@ -439,9 +439,40 @@
}
+# Trivial function for compatibility with later Mbed TLS versions
+record_outcome() {
+ echo "$1"
+}
+
+# True if the presence of the given pattern in a log definitely indicates
+# that the test has failed. False if the presence is inconclusive.
+#
+# Inputs:
+# * $1: pattern found in the logs
+# * $TIMES_LEFT: >0 if retrying is an option
+#
+# Outputs:
+# * $outcome: set to a retry reason if the pattern is inconclusive,
+# unchanged otherwise.
+# * Return value: 1 if the pattern is inconclusive,
+# 0 if the failure is definitive.
+log_pattern_presence_is_conclusive() {
+ # If we've run out of attempts, then don't retry no matter what.
+ if [ $TIMES_LEFT -eq 0 ]; then
+ return 0
+ fi
+ case $1 in
+ "resend")
+ # An undesired resend may have been caused by the OS dropping or
+ # delaying a packet at an inopportune time.
+ outcome="RETRY(resend)"
+ return 1;;
+ esac
+}
+
# fail <message>
fail() {
- echo "FAIL"
+ record_outcome "FAIL" "$1"
echo " ! $1"
mv $SRV_OUT o-srv-${TESTS}.log
@@ -616,70 +647,12 @@
esac
}
-# Usage: run_test name [-p proxy_cmd] srv_cmd cli_cmd cli_exit [option [...]]
-# Options: -s pattern pattern that must be present in server output
-# -c pattern pattern that must be present in client output
-# -u pattern lines after pattern must be unique in client output
-# -f call shell function on client output
-# -S pattern pattern that must be absent in server output
-# -C pattern pattern that must be absent in client output
-# -U pattern lines after pattern must be unique in server output
-# -F call shell function on server output
-run_test() {
- NAME="$1"
- shift 1
-
- if is_excluded "$NAME"; then
- SKIP_NEXT="NO"
- return
- fi
-
- print_name "$NAME"
-
- # Do we only run numbered tests?
- if [ -n "$RUN_TEST_NUMBER" ]; then
- case ",$RUN_TEST_NUMBER," in
- *",$TESTS,"*) :;;
- *) SKIP_NEXT="YES";;
- esac
- fi
-
- # should we skip?
- if [ "X$SKIP_NEXT" = "XYES" ]; then
- SKIP_NEXT="NO"
- echo "SKIP"
- SKIPS=$(( $SKIPS + 1 ))
- return
- fi
-
- # does this test use a proxy?
- if [ "X$1" = "X-p" ]; then
- PXY_CMD="$2"
- shift 2
- else
- PXY_CMD=""
- fi
-
- # get commands and client output
- SRV_CMD="$1"
- CLI_CMD="$2"
- CLI_EXPECT="$3"
- shift 3
-
- # Check if test uses files
- case "$SRV_CMD $CLI_CMD" in
- *data_files/*)
- requires_config_enabled MBEDTLS_FS_IO;;
- esac
-
- # should we skip?
- if [ "X$SKIP_NEXT" = "XYES" ]; then
- SKIP_NEXT="NO"
- echo "SKIP"
- SKIPS=$(( $SKIPS + 1 ))
- return
- fi
-
+# Analyze the commands that will be used in a test.
+#
+# Analyze and possibly instrument $PXY_CMD, $CLI_CMD, $SRV_CMD to pass
+# extra arguments or go through wrappers.
+# Set $DTLS (0=TLS, 1=DTLS).
+analyze_test_commands() {
# update DTLS variable
detect_dtls "$SRV_CMD"
@@ -709,48 +682,29 @@
CLI_CMD="valgrind --leak-check=full $CLI_CMD"
fi
fi
+}
- TIMES_LEFT=2
- while [ $TIMES_LEFT -gt 0 ]; do
- TIMES_LEFT=$(( $TIMES_LEFT - 1 ))
+# Check for failure conditions after a test case.
+#
+# Inputs from run_test:
+# * positional parameters: test options (see run_test documentation)
+# * $CLI_EXIT: client return code
+# * $CLI_EXPECT: expected client return code
+# * $SRV_RET: server return code
+# * $CLI_OUT, $SRV_OUT, $PXY_OUT: files containing client/server/proxy logs
+# * $TIMES_LEFT: if nonzero, a RETRY outcome is allowed
+#
+# Outputs:
+# * $outcome: one of PASS/RETRY*/FAIL
+check_test_failure() {
+ outcome=FAIL
- # run the commands
- if [ -n "$PXY_CMD" ]; then
- printf "# %s\n%s\n" "$NAME" "$PXY_CMD" > $PXY_OUT
- $PXY_CMD >> $PXY_OUT 2>&1 &
- PXY_PID=$!
- wait_proxy_start "$PXY_PORT" "$PXY_PID"
- fi
-
- check_osrv_dtls
- printf '# %s\n%s\n' "$NAME" "$SRV_CMD" > $SRV_OUT
- provide_input | $SRV_CMD >> $SRV_OUT 2>&1 &
- SRV_PID=$!
- wait_server_start "$SRV_PORT" "$SRV_PID"
-
- printf '# %s\n%s\n' "$NAME" "$CLI_CMD" > $CLI_OUT
- eval "$CLI_CMD" >> $CLI_OUT 2>&1 &
- wait_client_done
-
- sleep 0.05
-
- # terminate the server (and the proxy)
- kill $SRV_PID
- wait $SRV_PID
- SRV_RET=$?
-
- if [ -n "$PXY_CMD" ]; then
- kill $PXY_PID >/dev/null 2>&1
- wait $PXY_PID
- fi
-
- # retry only on timeouts
- if grep '===CLIENT_TIMEOUT===' $CLI_OUT >/dev/null; then
- printf "RETRY "
- else
- TIMES_LEFT=0
- fi
- done
+ if [ $TIMES_LEFT -gt 0 ] &&
+ grep '===CLIENT_TIMEOUT===' $CLI_OUT >/dev/null
+ then
+ outcome="RETRY(client-timeout)"
+ return
+ fi
# check if the client and server went at least to the handshake stage
# (useful to avoid tests with only negative assertions and non-zero
@@ -809,14 +763,18 @@
"-S")
if grep -v '^==' $SRV_OUT | grep -v 'Serious error when reading debug info' | grep "$2" >/dev/null; then
- fail "pattern '$2' MUST NOT be present in the Server output"
+ if log_pattern_presence_is_conclusive "$2"; then
+ fail "pattern '$2' MUST NOT be present in the Server output"
+ fi
return
fi
;;
"-C")
if grep -v '^==' $CLI_OUT | grep -v 'Serious error when reading debug info' | grep "$2" >/dev/null; then
- fail "pattern '$2' MUST NOT be present in the Client output"
+ if log_pattern_presence_is_conclusive "$2"; then
+ fail "pattern '$2' MUST NOT be present in the Client output"
+ fi
return
fi
;;
@@ -874,7 +832,126 @@
fi
# if we're here, everything is ok
- echo "PASS"
+ outcome=PASS
+}
+
+# Run the current test case: start the server and if applicable the proxy, run
+# the client, wait for all processes to finish or time out.
+#
+# Inputs:
+# * $NAME: test case name
+# * $CLI_CMD, $SRV_CMD, $PXY_CMD: commands to run
+# * $CLI_OUT, $SRV_OUT, $PXY_OUT: files to contain client/server/proxy logs
+#
+# Outputs:
+# * $CLI_EXIT: client return code
+# * $SRV_RET: server return code
+do_run_test_once() {
+ # run the commands
+ if [ -n "$PXY_CMD" ]; then
+ printf "# %s\n%s\n" "$NAME" "$PXY_CMD" > $PXY_OUT
+ $PXY_CMD >> $PXY_OUT 2>&1 &
+ PXY_PID=$!
+ wait_proxy_start "$PXY_PORT" "$PXY_PID"
+ fi
+
+ check_osrv_dtls
+ printf '# %s\n%s\n' "$NAME" "$SRV_CMD" > $SRV_OUT
+ provide_input | $SRV_CMD >> $SRV_OUT 2>&1 &
+ SRV_PID=$!
+ wait_server_start "$SRV_PORT" "$SRV_PID"
+
+ printf '# %s\n%s\n' "$NAME" "$CLI_CMD" > $CLI_OUT
+ eval "$CLI_CMD" >> $CLI_OUT 2>&1 &
+ wait_client_done
+
+ sleep 0.05
+
+ # terminate the server (and the proxy)
+ kill $SRV_PID
+ wait $SRV_PID
+ SRV_RET=$?
+
+ if [ -n "$PXY_CMD" ]; then
+ kill $PXY_PID >/dev/null 2>&1
+ wait $PXY_PID
+ fi
+}
+
+# Usage: run_test name [-p proxy_cmd] srv_cmd cli_cmd cli_exit [option [...]]
+# Options: -s pattern pattern that must be present in server output
+# -c pattern pattern that must be present in client output
+# -u pattern lines after pattern must be unique in client output
+# -f call shell function on client output
+# -S pattern pattern that must be absent in server output
+# -C pattern pattern that must be absent in client output
+# -U pattern lines after pattern must be unique in server output
+# -F call shell function on server output
+run_test() {
+ NAME="$1"
+ shift 1
+
+ if is_excluded "$NAME"; then
+ SKIP_NEXT="NO"
+ return
+ fi
+
+ print_name "$NAME"
+
+ # Do we only run numbered tests?
+ if [ -n "$RUN_TEST_NUMBER" ]; then
+ case ",$RUN_TEST_NUMBER," in
+ *",$TESTS,"*) :;;
+ *) SKIP_NEXT="YES";;
+ esac
+ fi
+
+ # does this test use a proxy?
+ if [ "X$1" = "X-p" ]; then
+ PXY_CMD="$2"
+ shift 2
+ else
+ PXY_CMD=""
+ fi
+
+ # get commands and client output
+ SRV_CMD="$1"
+ CLI_CMD="$2"
+ CLI_EXPECT="$3"
+ shift 3
+
+ # Check if test uses files
+ case "$SRV_CMD $CLI_CMD" in
+ *data_files/*)
+ requires_config_enabled MBEDTLS_FS_IO;;
+ esac
+
+ # should we skip?
+ if [ "X$SKIP_NEXT" = "XYES" ]; then
+ SKIP_NEXT="NO"
+ record_outcome "SKIP"
+ SKIPS=$(( $SKIPS + 1 ))
+ return
+ fi
+
+ analyze_test_commands "$@"
+
+ TIMES_LEFT=2
+ while [ $TIMES_LEFT -gt 0 ]; do
+ TIMES_LEFT=$(( $TIMES_LEFT - 1 ))
+
+ do_run_test_once
+
+ check_test_failure "$@"
+ case $outcome in
+ PASS) break;;
+ RETRY*) printf "$outcome ";;
+ FAIL) return;;
+ esac
+ done
+
+ # If we get this far, the test case passed.
+ record_outcome "PASS"
if [ "$PRESERVE_LOGS" -gt 0 ]; then
mv $SRV_OUT o-srv-${TESTS}.log
mv $CLI_OUT o-cli-${TESTS}.log