Merge improvements to SSL test scripts
diff --git a/tests/compat.sh b/tests/compat.sh
index 724311c..06243bd 100755
--- a/tests/compat.sh
+++ b/tests/compat.sh
@@ -645,14 +645,14 @@
exit 1;
esac
- P_SERVER_ARGS="server_addr=0.0.0.0 force_version=$MODE"
- O_SERVER_ARGS="-www -cipher NULL,ALL -$MODE"
- G_SERVER_ARGS="-p 4433 --http"
+ P_SERVER_ARGS="server_port=$PORT server_addr=0.0.0.0 force_version=$MODE"
+ O_SERVER_ARGS="-accept $PORT -www -cipher NULL,ALL -$MODE"
+ G_SERVER_ARGS="-p $PORT --http"
G_SERVER_PRIO="EXPORT:+NULL:+MD5:+PSK:+DHE-PSK:+ECDHE-PSK:+RSA-PSK:-VERS-TLS-ALL:$G_PRIO_MODE"
- P_CLIENT_ARGS="force_version=$MODE"
- O_CLIENT_ARGS="-$MODE"
- G_CLIENT_ARGS="-p 4433 --debug 3"
+ P_CLIENT_ARGS="server_port=$PORT force_version=$MODE"
+ O_CLIENT_ARGS="-connect localhost:$PORT -$MODE"
+ G_CLIENT_ARGS="-p $PORT --debug 3"
G_CLIENT_PRIO="NONE:$G_PRIO_MODE:+COMP-NULL:+CURVE-ALL:+SIGN-ALL"
if [ "X$VERIFY" = "XYES" ];
@@ -757,8 +757,8 @@
SERVER_NAME=$1
log "$SERVER_CMD"
- echo "$SERVER_CMD" > srv_out
- $SERVER_CMD >> srv_out 2>&1 &
+ echo "$SERVER_CMD" > $SRV_OUT
+ $SERVER_CMD >> $SRV_OUT 2>&1 &
PROCESS_ID=$!
sleep 1
@@ -768,6 +768,10 @@
stop_server() {
case $SERVER_NAME in
[Pp]olar*)
+ # start watchdog in case SERVERQUIT fails
+ ( sleep 20; echo "SERVERQUIT TIMEOUT"; kill $MAIN_PID ) &
+ WATCHDOG_PID=$!
+
# we must force a PSK suite when in PSK mode (otherwise client
# auth will fail), so try every entry in $P_CIPHERS in turn (in
# case the first one is not implemented in this configuration)
@@ -779,27 +783,31 @@
break
fi
done
+
+ wait $PROCESS_ID 2>/dev/null
+ kill $WATCHDOG_PID 2>/dev/null
+ wait $WATCHDOG_PID 2>/dev/null
;;
*)
kill $PROCESS_ID 2>/dev/null
+ wait $PROCESS_ID 2>/dev/null
esac
- wait $PROCESS_ID 2>/dev/null
if [ "$MEMCHECK" -gt 0 ]; then
- if is_polar "$SERVER_CMD" && has_mem_err srv_out; then
+ if is_polar "$SERVER_CMD" && has_mem_err $SRV_OUT; then
echo " ! Server had memory errors"
let "srvmem++"
return
fi
fi
- rm -f srv_out
+ rm -f $SRV_OUT
}
# kill the running server (used when killed by signal)
cleanup() {
- rm -f srv_out cli_out
+ rm -f $SRV_OUT $CLI_OUT
kill $PROCESS_ID
exit 1
}
@@ -820,14 +828,14 @@
[Oo]pen*)
CLIENT_CMD="$OPENSSL_CMD s_client $O_CLIENT_ARGS -cipher $2"
log "$CLIENT_CMD"
- echo "$CLIENT_CMD" > cli_out
- ( echo -e 'GET HTTP/1.0'; echo; ) | $CLIENT_CMD >> cli_out 2>&1
+ echo "$CLIENT_CMD" > $CLI_OUT
+ ( echo -e 'GET HTTP/1.0'; echo; ) | $CLIENT_CMD >> $CLI_OUT 2>&1
EXIT=$?
if [ "$EXIT" == "0" ]; then
RESULT=0
else
- if grep 'Cipher is (NONE)' cli_out >/dev/null; then
+ if grep 'Cipher is (NONE)' $CLI_OUT >/dev/null; then
RESULT=1
else
RESULT=2
@@ -838,8 +846,8 @@
[Gg]nu*)
CLIENT_CMD="$GNUTLS_CLI $G_CLIENT_ARGS --priority $G_PRIO_MODE:$2 localhost"
log "$CLIENT_CMD"
- echo "$CLIENT_CMD" > cli_out
- ( echo -e 'GET HTTP/1.0'; echo; ) | $CLIENT_CMD >> cli_out 2>&1
+ echo "$CLIENT_CMD" > $CLI_OUT
+ ( echo -e 'GET HTTP/1.0'; echo; ) | $CLIENT_CMD >> $CLI_OUT 2>&1
EXIT=$?
if [ "$EXIT" == "0" ]; then
@@ -848,8 +856,8 @@
RESULT=2
# interpret early failure, with a handshake_failure alert
# before the server hello, as "no ciphersuite in common"
- if grep -F 'Received alert [40]: Handshake failed' cli_out; then
- if grep -i 'SERVER HELLO .* was received' cli_out; then :
+ if grep -F 'Received alert [40]: Handshake failed' $CLI_OUT; then
+ if grep -i 'SERVER HELLO .* was received' $CLI_OUT; then :
else
RESULT=1
fi
@@ -863,8 +871,8 @@
CLIENT_CMD="valgrind --leak-check=full $CLIENT_CMD"
fi
log "$CLIENT_CMD"
- echo "$CLIENT_CMD" > cli_out
- $CLIENT_CMD >> cli_out 2>&1
+ echo "$CLIENT_CMD" > $CLI_OUT
+ $CLIENT_CMD >> $CLI_OUT 2>&1
EXIT=$?
case $EXIT in
@@ -874,7 +882,7 @@
esac
if [ "$MEMCHECK" -gt 0 ]; then
- if is_polar "$CLIENT_CMD" && has_mem_err cli_out; then
+ if is_polar "$CLIENT_CMD" && has_mem_err $CLI_OUT; then
RESULT=2
fi
fi
@@ -887,7 +895,7 @@
;;
esac
- echo "EXIT: $EXIT" >> cli_out
+ echo "EXIT: $EXIT" >> $CLI_OUT
# report and count result
case $RESULT in
@@ -900,14 +908,14 @@
;;
"2")
echo FAIL
- cp srv_out c-srv-${tests}.log
- cp cli_out c-cli-${tests}.log
+ cp $SRV_OUT c-srv-${tests}.log
+ cp $CLI_OUT c-cli-${tests}.log
echo " ! outputs saved to c-srv-${tests}.log, c-cli-${tests}.log"
let "failed++"
;;
esac
- rm -f cli_out
+ rm -f $CLI_OUT
}
#
@@ -952,7 +960,17 @@
esac
done
-killall -q gnutls-serv openssl ssl_server ssl_server2
+# used by watchdog
+MAIN_PID="$$"
+
+# Pick a "unique" port in the range 10000-19999.
+PORT="0000$$"
+PORT="1$(echo $PORT | tail -c 4)"
+
+# Also pick a unique name for intermediate files
+SRV_OUT="srv_out.$$"
+CLI_OUT="cli_out.$$"
+
trap cleanup INT TERM HUP
for VERIFY in $VERIFIES; do
diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh
index 1e11d78..d01dc25 100755
--- a/tests/scripts/all.sh
+++ b/tests/scripts/all.sh
@@ -6,8 +6,8 @@
# CMake configuration. After this script is run, the CMake cache is lost and
# CMake is not initialised any more!
#
-# Assumes gcc, clang (recent enough for using ASan) are available, as weel as
-# cmake. Also assumes valgrind is available if --memcheck is used.
+# Assumes gcc and clang (recent enough for using ASan) are available,
+# as well as cmake and valgrind.
# Abort on errors (and uninitiliased variables)
set -eu
@@ -54,54 +54,72 @@
echo "******************************************************************"
}
-# Step 1: various build types
+# The test ordering tries to optimize for the following criteria:
+# 1. Catch possible problems early, by running first test that run quickly
+# and/or are more likely to fail than others.
+# 2. Minimize total running time, by avoiding useless rebuilds
+#
+# Indicative running times are given for reference.
-msg "Unix make, default compiler and flags"
-cleanup
-make
-
-msg "cmake, gcc with lots of warnings"
+msg "build: cmake, gcc with lots of warnings" # ~ 1 min
cleanup
CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Check .
make
-msg "cmake, clang with lots of warnings"
+msg "test: main suites with valgrind" # ~ 2 min 10s
+make memcheck
+
+msg "build: with ASan" # ~ 1 min
+cleanup
+cmake -D CMAKE_BUILD_TYPE:String=ASan .
+make
+
+msg "test: ssl-opt.sh (ASan build)" # ~ 1 min 10s
+cd tests
+./ssl-opt.sh
+cd ..
+
+msg "test: main suites and selftest (ASan build)" # ~ 10s + 30s
+make test
+programs/test/selftest
+
+msg "test: ref-configs (ASan build)" # ~ 4 min 45 s
+tests/scripts/test-ref-configs.pl
+
+# Most issues are likely to be caught at this point
+
+msg "build: with ASan (rebuild after ref-configs)" # ~ 1 min
+make
+
+msg "test: compat.sh (ASan build)" # ~ 7 min 30s
+cd tests
+./compat.sh
+cd ..
+
+msg "build: cmake, clang with lots of warnings" # ~ 40s
cleanup
CC=clang cmake -D CMAKE_BUILD_TYPE:String=Check .
make
-# Step 2: Full tests, with ASan
-
-msg "ASan build and full tests"
+msg "build: Unix make, -O2" # ~ 30s
cleanup
-cmake -D CMAKE_BUILD_TYPE:String=ASan .
make
-make test
-programs/test/selftest
-cd tests
-./compat.sh
-./ssl-opt.sh
-cd ..
-tests/scripts/test-ref-configs.pl
-# Step 3: using valgrind's memcheck
-
-msg "Release build, test suites with valgrind's memcheck"
-cleanup
-# optimized build to compensate a bit for valgrind slowdown
-cmake -D CMAKE_BUILD_TYPE:String=Release .
-make
-make memcheck
+# Optional parts that take a long time to run
if [ "$MEMORY" -gt 0 ]; then
+ msg "test: ssl-opt --memcheck (-02 build)" # ~ 8 min
cd tests
./ssl-opt.sh --memcheck
- [ "$MEMORY" -gt 1 ] && ./compat.sh --memcheck
cd ..
- # no test-ref-configs: doesn't have a memcheck option (yet?)
-fi
-# Done
+ if [ "$MEMORY" -gt 1 ]; then
+ msg "test: compat --memcheck (-02 build)" # ~ 42 min
+ cd tests
+ ./compat.sh --memcheck
+ cd ..
+ fi
+fi
echo "Done."
cleanup
diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh
index f4fbc01..238995f 100755
--- a/tests/ssl-opt.sh
+++ b/tests/ssl-opt.sh
@@ -89,8 +89,8 @@
echo "FAIL"
echo " ! $1"
- cp srv_out o-srv-${TESTS}.log
- cp cli_out o-cli-${TESTS}.log
+ cp $SRV_OUT o-srv-${TESTS}.log
+ cp $CLI_OUT o-cli-${TESTS}.log
echo " ! outputs saved to o-srv-${TESTS}.log and o-cli-${TESTS}.log"
FAILS=`echo $FAILS + 1 | bc`
@@ -112,6 +112,28 @@
fi
}
+# wait for server to be ready
+wait_srv_ready() {
+ if is_polar "$SRV_CMD"; then
+ READY_MSG="Waiting for a remote connection"
+ else
+ READY_MSG="ACCEPT"
+ fi
+
+ # If the server isn't ready after 10 secs, something probably went wrong
+ ( sleep 10; echo "SERVERSTART TIMEOUT"; kill $MAIN_PID ) &
+ WATCHDOG_PID=$!
+
+ while ! grep "$READY_MSG" $SRV_OUT >/dev/null; do
+ # don't use sleep, since the whole goal is to avoid wasting time,
+ # and 1 second is usually way more than the server needs to start
+ true
+ done
+
+ kill $WATCHDOG_PID
+ wait $WATCHDOG_PID
+}
+
# Usage: run_test name srv_cmd cli_cmd cli_exit [option [...]]
# Options: -s pattern pattern that must be present in server output
# -c pattern pattern that must be present in client output
@@ -142,38 +164,46 @@
fi
# run the commands
- echo "$SRV_CMD" > srv_out
- $SRV_CMD >> srv_out 2>&1 &
+ echo "$SRV_CMD" > $SRV_OUT
+ $SRV_CMD >> $SRV_OUT 2>&1 &
SRV_PID=$!
- sleep 1
- echo "$CLI_CMD" > cli_out
- eval "$CLI_CMD" >> cli_out 2>&1
+ wait_srv_ready
+ echo "$CLI_CMD" > $CLI_OUT
+ eval "$CLI_CMD" >> $CLI_OUT 2>&1
CLI_EXIT=$?
- echo "EXIT: $CLI_EXIT" >> cli_out
+ echo "EXIT: $CLI_EXIT" >> $CLI_OUT
- # psk is useful when server only has bad certs
if is_polar "$SRV_CMD"; then
- "$P_CLI" request_page=SERVERQUIT tickets=0 auth_mode=none psk=abc123 \
+ # start watchdog in case SERVERQUIT fails
+ ( sleep 10; echo "SERVERQUIT TIMEOUT"; kill $MAIN_PID ) &
+ WATCHDOG_PID=$!
+
+ # psk is useful when server only has bad certs
+ $P_CLI request_page=SERVERQUIT tickets=0 auth_mode=none psk=abc123 \
crt_file=data_files/cli2.crt key_file=data_files/cli2.key \
>/dev/null
+
+ wait $SRV_PID
+ kill $WATCHDOG_PID
+ wait $WATCHDOG_PID
else
kill $SRV_PID
+ wait $SRV_PID
fi
- wait $SRV_PID
# check if the client and server went at least to the handshake stage
# (useful to avoid tests with only negative assertions and non-zero
# expected client exit to incorrectly succeed in case of catastrophic
# failure)
if is_polar "$SRV_CMD"; then
- if grep "Performing the SSL/TLS handshake" srv_out >/dev/null; then :;
+ if grep "Performing the SSL/TLS handshake" $SRV_OUT >/dev/null; then :;
else
fail "server failed to start"
return
fi
fi
if is_polar "$CLI_CMD"; then
- if grep "Performing the SSL/TLS handshake" cli_out >/dev/null; then :;
+ if grep "Performing the SSL/TLS handshake" $CLI_OUT >/dev/null; then :;
else
fail "client failed to start"
return
@@ -199,28 +229,28 @@
do
case $1 in
"-s")
- if grep "$2" srv_out >/dev/null; then :; else
+ if grep "$2" $SRV_OUT >/dev/null; then :; else
fail "-s $2"
return
fi
;;
"-c")
- if grep "$2" cli_out >/dev/null; then :; else
+ if grep "$2" $CLI_OUT >/dev/null; then :; else
fail "-c $2"
return
fi
;;
"-S")
- if grep "$2" srv_out >/dev/null; then
+ if grep "$2" $SRV_OUT >/dev/null; then
fail "-S $2"
return
fi
;;
"-C")
- if grep "$2" cli_out >/dev/null; then
+ if grep "$2" $CLI_OUT >/dev/null; then
fail "-C $2"
return
fi
@@ -235,11 +265,11 @@
# check valgrind's results
if [ "$MEMCHECK" -gt 0 ]; then
- if is_polar "$SRV_CMD" && has_mem_err srv_out; then
+ if is_polar "$SRV_CMD" && has_mem_err $SRV_OUT; then
fail "Server has memory errors"
return
fi
- if is_polar "$CLI_CMD" && has_mem_err cli_out; then
+ if is_polar "$CLI_CMD" && has_mem_err $CLI_OUT; then
fail "Client has memory errors"
return
fi
@@ -247,11 +277,11 @@
# if we're here, everything is ok
echo "PASS"
- rm -f srv_out cli_out
+ rm -f $SRV_OUT $CLI_OUT
}
cleanup() {
- rm -f cli_out srv_out sess
+ rm -f $CLI_OUT $SRV_OUT $SESSION
kill $SRV_PID
exit 1
}
@@ -276,7 +306,24 @@
exit 1
fi
-killall -q openssl ssl_server ssl_server2
+# used by watchdog
+MAIN_PID="$$"
+
+# Pick a "unique" port in the range 10000-19999.
+PORT="0000$$"
+PORT="1$(echo $PORT | tail -c 4)"
+
+# fix commands to use this port
+P_SRV="$P_SRV server_port=$PORT"
+P_CLI="$P_CLI server_port=$PORT"
+O_SRV="$O_SRV -accept $PORT"
+O_CLI="$O_CLI -connect localhost:$PORT"
+
+# Also pick a unique name for intermediate files
+SRV_OUT="srv_out.$$"
+CLI_OUT="cli_out.$$"
+SESSION="session.$$"
+
trap cleanup INT TERM HUP
# Test for SSLv2 ClientHello
@@ -365,7 +412,9 @@
run_test "Session resume using tickets #5 (openssl client)" \
"$P_SRV debug_level=4 tickets=1" \
- "($O_CLI -sess_out sess; $O_CLI -sess_in sess; rm -f sess)" \
+ "( $O_CLI -sess_out $SESSION; \
+ $O_CLI -sess_in $SESSION; \
+ rm -f $SESSION )" \
0 \
-s "found session ticket extension" \
-s "server hello, adding session ticket extension" \
@@ -450,7 +499,9 @@
run_test "Session resume using cache #8 (openssl client)" \
"$P_SRV debug_level=4 tickets=0" \
- "($O_CLI -sess_out sess; $O_CLI -sess_in sess; rm -f sess)" \
+ "( $O_CLI -sess_out $SESSION; \
+ $O_CLI -sess_in $SESSION; \
+ rm -f $SESSION )" \
0 \
-s "found session ticket extension" \
-S "server hello, adding session ticket extension" \