From 8066b81a548fab373df68adb02a4436bc91bb565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Wed, 28 May 2014 22:59:30 +0200 Subject: [PATCH 1/5] Pick a "unique" port in SSL test scripts --- tests/compat.sh | 17 ++++++++++------- tests/ssl-opt.sh | 13 +++++++++++-- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index 724311cf77..1eea23cbdc 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -645,14 +645,14 @@ setup_arguments() exit 1; esac - P_SERVER_ARGS="server_addr=0.0.0.0 force_version=$MODE" - O_SERVER_ARGS="-www -cipher NULL,ALL -$MODE" - G_SERVER_ARGS="-p 4433 --http" + P_SERVER_ARGS="server_port=$PORT server_addr=0.0.0.0 force_version=$MODE" + O_SERVER_ARGS="-accept $PORT -www -cipher NULL,ALL -$MODE" + G_SERVER_ARGS="-p $PORT --http" G_SERVER_PRIO="EXPORT:+NULL:+MD5:+PSK:+DHE-PSK:+ECDHE-PSK:+RSA-PSK:-VERS-TLS-ALL:$G_PRIO_MODE" - P_CLIENT_ARGS="force_version=$MODE" - O_CLIENT_ARGS="-$MODE" - G_CLIENT_ARGS="-p 4433 --debug 3" + P_CLIENT_ARGS="server_port=$PORT force_version=$MODE" + O_CLIENT_ARGS="-connect localhost:$PORT -$MODE" + G_CLIENT_ARGS="-p $PORT --debug 3" G_CLIENT_PRIO="NONE:$G_PRIO_MODE:+COMP-NULL:+CURVE-ALL:+SIGN-ALL" if [ "X$VERIFY" = "XYES" ]; @@ -952,7 +952,10 @@ for PEER in $PEERS; do esac done -killall -q gnutls-serv openssl ssl_server ssl_server2 +# Pick a "unique" port in the range 10000-19999. +PORT="0000$$" +PORT="1$(echo $PORT | tail -c 4)" + trap cleanup INT TERM HUP for VERIFY in $VERIFIES; do diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index f4fbc019bf..4df999eb64 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -153,7 +153,7 @@ run_test() { # psk is useful when server only has bad certs if is_polar "$SRV_CMD"; then - "$P_CLI" request_page=SERVERQUIT tickets=0 auth_mode=none psk=abc123 \ + $P_CLI request_page=SERVERQUIT tickets=0 auth_mode=none psk=abc123 \ crt_file=data_files/cli2.crt key_file=data_files/cli2.key \ >/dev/null else @@ -276,7 +276,16 @@ if which $OPENSSL_CMD >/dev/null 2>&1; then :; else exit 1 fi -killall -q openssl ssl_server ssl_server2 +# Pick a "unique" port in the range 10000-19999. +PORT="0000$$" +PORT="1$(echo $PORT | tail -c 4)" + +# fix commands to use this port +P_SRV="$P_SRV server_port=$PORT" +P_CLI="$P_CLI server_port=$PORT" +O_SRV="$O_SRV -accept $PORT" +O_CLI="$O_CLI -connect localhost:$PORT" + trap cleanup INT TERM HUP # Test for SSLv2 ClientHello From bc3b16c7e2027cc18aa9d8f90f7b884cc50e6804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Wed, 28 May 2014 23:06:50 +0200 Subject: [PATCH 2/5] Also use unique names for temp files --- tests/compat.sh | 42 +++++++++++++++++++++++------------------- tests/ssl-opt.sh | 47 ++++++++++++++++++++++++++++------------------- 2 files changed, 51 insertions(+), 38 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index 1eea23cbdc..0146e15e8c 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -757,8 +757,8 @@ start_server() { SERVER_NAME=$1 log "$SERVER_CMD" - echo "$SERVER_CMD" > srv_out - $SERVER_CMD >> srv_out 2>&1 & + echo "$SERVER_CMD" > $SRV_OUT + $SERVER_CMD >> $SRV_OUT 2>&1 & PROCESS_ID=$! sleep 1 @@ -787,19 +787,19 @@ stop_server() { wait $PROCESS_ID 2>/dev/null if [ "$MEMCHECK" -gt 0 ]; then - if is_polar "$SERVER_CMD" && has_mem_err srv_out; then + if is_polar "$SERVER_CMD" && has_mem_err $SRV_OUT; then echo " ! Server had memory errors" let "srvmem++" return fi fi - rm -f srv_out + rm -f $SRV_OUT } # kill the running server (used when killed by signal) cleanup() { - rm -f srv_out cli_out + rm -f $SRV_OUT $CLI_OUT kill $PROCESS_ID exit 1 } @@ -820,14 +820,14 @@ run_client() { [Oo]pen*) CLIENT_CMD="$OPENSSL_CMD s_client $O_CLIENT_ARGS -cipher $2" log "$CLIENT_CMD" - echo "$CLIENT_CMD" > cli_out - ( echo -e 'GET HTTP/1.0'; echo; ) | $CLIENT_CMD >> cli_out 2>&1 + echo "$CLIENT_CMD" > $CLI_OUT + ( echo -e 'GET HTTP/1.0'; echo; ) | $CLIENT_CMD >> $CLI_OUT 2>&1 EXIT=$? if [ "$EXIT" == "0" ]; then RESULT=0 else - if grep 'Cipher is (NONE)' cli_out >/dev/null; then + if grep 'Cipher is (NONE)' $CLI_OUT >/dev/null; then RESULT=1 else RESULT=2 @@ -838,8 +838,8 @@ run_client() { [Gg]nu*) CLIENT_CMD="$GNUTLS_CLI $G_CLIENT_ARGS --priority $G_PRIO_MODE:$2 localhost" log "$CLIENT_CMD" - echo "$CLIENT_CMD" > cli_out - ( echo -e 'GET HTTP/1.0'; echo; ) | $CLIENT_CMD >> cli_out 2>&1 + echo "$CLIENT_CMD" > $CLI_OUT + ( echo -e 'GET HTTP/1.0'; echo; ) | $CLIENT_CMD >> $CLI_OUT 2>&1 EXIT=$? if [ "$EXIT" == "0" ]; then @@ -848,8 +848,8 @@ run_client() { RESULT=2 # interpret early failure, with a handshake_failure alert # before the server hello, as "no ciphersuite in common" - if grep -F 'Received alert [40]: Handshake failed' cli_out; then - if grep -i 'SERVER HELLO .* was received' cli_out; then : + if grep -F 'Received alert [40]: Handshake failed' $CLI_OUT; then + if grep -i 'SERVER HELLO .* was received' $CLI_OUT; then : else RESULT=1 fi @@ -863,8 +863,8 @@ run_client() { CLIENT_CMD="valgrind --leak-check=full $CLIENT_CMD" fi log "$CLIENT_CMD" - echo "$CLIENT_CMD" > cli_out - $CLIENT_CMD >> cli_out 2>&1 + echo "$CLIENT_CMD" > $CLI_OUT + $CLIENT_CMD >> $CLI_OUT 2>&1 EXIT=$? case $EXIT in @@ -874,7 +874,7 @@ run_client() { esac if [ "$MEMCHECK" -gt 0 ]; then - if is_polar "$CLIENT_CMD" && has_mem_err cli_out; then + if is_polar "$CLIENT_CMD" && has_mem_err $CLI_OUT; then RESULT=2 fi fi @@ -887,7 +887,7 @@ run_client() { ;; esac - echo "EXIT: $EXIT" >> cli_out + echo "EXIT: $EXIT" >> $CLI_OUT # report and count result case $RESULT in @@ -900,14 +900,14 @@ run_client() { ;; "2") echo FAIL - cp srv_out c-srv-${tests}.log - cp cli_out c-cli-${tests}.log + cp $SRV_OUT c-srv-${tests}.log + cp $CLI_OUT c-cli-${tests}.log echo " ! outputs saved to c-srv-${tests}.log, c-cli-${tests}.log" let "failed++" ;; esac - rm -f cli_out + rm -f $CLI_OUT } # @@ -956,6 +956,10 @@ done PORT="0000$$" PORT="1$(echo $PORT | tail -c 4)" +# Also pick a unique name for intermediate files +SRV_OUT="srv_out.$$" +CLI_OUT="cli_out.$$" + trap cleanup INT TERM HUP for VERIFY in $VERIFIES; do diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 4df999eb64..19be5ef2b2 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -89,8 +89,8 @@ fail() { echo "FAIL" echo " ! $1" - cp srv_out o-srv-${TESTS}.log - cp cli_out o-cli-${TESTS}.log + cp $SRV_OUT o-srv-${TESTS}.log + cp $CLI_OUT o-cli-${TESTS}.log echo " ! outputs saved to o-srv-${TESTS}.log and o-cli-${TESTS}.log" FAILS=`echo $FAILS + 1 | bc` @@ -142,14 +142,14 @@ run_test() { fi # run the commands - echo "$SRV_CMD" > srv_out - $SRV_CMD >> srv_out 2>&1 & + echo "$SRV_CMD" > $SRV_OUT + $SRV_CMD >> $SRV_OUT 2>&1 & SRV_PID=$! sleep 1 - echo "$CLI_CMD" > cli_out - eval "$CLI_CMD" >> cli_out 2>&1 + echo "$CLI_CMD" > $CLI_OUT + eval "$CLI_CMD" >> $CLI_OUT 2>&1 CLI_EXIT=$? - echo "EXIT: $CLI_EXIT" >> cli_out + echo "EXIT: $CLI_EXIT" >> $CLI_OUT # psk is useful when server only has bad certs if is_polar "$SRV_CMD"; then @@ -166,14 +166,14 @@ run_test() { # expected client exit to incorrectly succeed in case of catastrophic # failure) if is_polar "$SRV_CMD"; then - if grep "Performing the SSL/TLS handshake" srv_out >/dev/null; then :; + if grep "Performing the SSL/TLS handshake" $SRV_OUT >/dev/null; then :; else fail "server failed to start" return fi fi if is_polar "$CLI_CMD"; then - if grep "Performing the SSL/TLS handshake" cli_out >/dev/null; then :; + if grep "Performing the SSL/TLS handshake" $CLI_OUT >/dev/null; then :; else fail "client failed to start" return @@ -199,28 +199,28 @@ run_test() { do case $1 in "-s") - if grep "$2" srv_out >/dev/null; then :; else + if grep "$2" $SRV_OUT >/dev/null; then :; else fail "-s $2" return fi ;; "-c") - if grep "$2" cli_out >/dev/null; then :; else + if grep "$2" $CLI_OUT >/dev/null; then :; else fail "-c $2" return fi ;; "-S") - if grep "$2" srv_out >/dev/null; then + if grep "$2" $SRV_OUT >/dev/null; then fail "-S $2" return fi ;; "-C") - if grep "$2" cli_out >/dev/null; then + if grep "$2" $CLI_OUT >/dev/null; then fail "-C $2" return fi @@ -235,11 +235,11 @@ run_test() { # check valgrind's results if [ "$MEMCHECK" -gt 0 ]; then - if is_polar "$SRV_CMD" && has_mem_err srv_out; then + if is_polar "$SRV_CMD" && has_mem_err $SRV_OUT; then fail "Server has memory errors" return fi - if is_polar "$CLI_CMD" && has_mem_err cli_out; then + if is_polar "$CLI_CMD" && has_mem_err $CLI_OUT; then fail "Client has memory errors" return fi @@ -247,11 +247,11 @@ run_test() { # if we're here, everything is ok echo "PASS" - rm -f srv_out cli_out + rm -f $SRV_OUT $CLI_OUT } cleanup() { - rm -f cli_out srv_out sess + rm -f $CLI_OUT $SRV_OUT $SESSION kill $SRV_PID exit 1 } @@ -286,6 +286,11 @@ P_CLI="$P_CLI server_port=$PORT" O_SRV="$O_SRV -accept $PORT" O_CLI="$O_CLI -connect localhost:$PORT" +# Also pick a unique name for intermediate files +SRV_OUT="srv_out.$$" +CLI_OUT="cli_out.$$" +SESSION="session.$$" + trap cleanup INT TERM HUP # Test for SSLv2 ClientHello @@ -374,7 +379,9 @@ run_test "Session resume using tickets #4 (openssl server)" \ run_test "Session resume using tickets #5 (openssl client)" \ "$P_SRV debug_level=4 tickets=1" \ - "($O_CLI -sess_out sess; $O_CLI -sess_in sess; rm -f sess)" \ + "( $O_CLI -sess_out $SESSION; \ + $O_CLI -sess_in $SESSION; \ + rm -f $SESSION )" \ 0 \ -s "found session ticket extension" \ -s "server hello, adding session ticket extension" \ @@ -459,7 +466,9 @@ run_test "Session resume using cache #7 (no timeout)" \ run_test "Session resume using cache #8 (openssl client)" \ "$P_SRV debug_level=4 tickets=0" \ - "($O_CLI -sess_out sess; $O_CLI -sess_in sess; rm -f sess)" \ + "( $O_CLI -sess_out $SESSION; \ + $O_CLI -sess_in $SESSION; \ + rm -f $SESSION )" \ 0 \ -s "found session ticket extension" \ -S "server hello, adding session ticket extension" \ From 32f8f4d1a0fb12ee29fb8501e3bef1846abb580a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Thu, 29 May 2014 11:31:20 +0200 Subject: [PATCH 3/5] Catch SERVERQUIT timeout in ssl test scripts --- tests/compat.sh | 13 ++++++++++++- tests/ssl-opt.sh | 15 +++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index 0146e15e8c..06243bde48 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -768,6 +768,10 @@ start_server() { stop_server() { case $SERVER_NAME in [Pp]olar*) + # start watchdog in case SERVERQUIT fails + ( sleep 20; echo "SERVERQUIT TIMEOUT"; kill $MAIN_PID ) & + WATCHDOG_PID=$! + # we must force a PSK suite when in PSK mode (otherwise client # auth will fail), so try every entry in $P_CIPHERS in turn (in # case the first one is not implemented in this configuration) @@ -779,12 +783,16 @@ stop_server() { break fi done + + wait $PROCESS_ID 2>/dev/null + kill $WATCHDOG_PID 2>/dev/null + wait $WATCHDOG_PID 2>/dev/null ;; *) kill $PROCESS_ID 2>/dev/null + wait $PROCESS_ID 2>/dev/null esac - wait $PROCESS_ID 2>/dev/null if [ "$MEMCHECK" -gt 0 ]; then if is_polar "$SERVER_CMD" && has_mem_err $SRV_OUT; then @@ -952,6 +960,9 @@ for PEER in $PEERS; do esac done +# used by watchdog +MAIN_PID="$$" + # Pick a "unique" port in the range 10000-19999. PORT="0000$$" PORT="1$(echo $PORT | tail -c 4)" diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 19be5ef2b2..1f7a4adb35 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -151,15 +151,23 @@ run_test() { CLI_EXIT=$? echo "EXIT: $CLI_EXIT" >> $CLI_OUT - # psk is useful when server only has bad certs if is_polar "$SRV_CMD"; then + # start watchdog in case SERVERQUIT fails + ( sleep 10; echo "SERVERQUIT TIMEOUT"; kill $MAIN_PID ) & + WATCHDOG_PID=$! + + # psk is useful when server only has bad certs $P_CLI request_page=SERVERQUIT tickets=0 auth_mode=none psk=abc123 \ crt_file=data_files/cli2.crt key_file=data_files/cli2.key \ >/dev/null + + wait $SRV_PID + kill $WATCHDOG_PID + wait $WATCHDOG_PID else kill $SRV_PID + wait $SRV_PID fi - wait $SRV_PID # check if the client and server went at least to the handshake stage # (useful to avoid tests with only negative assertions and non-zero @@ -276,6 +284,9 @@ if which $OPENSSL_CMD >/dev/null 2>&1; then :; else exit 1 fi +# used by watchdog +MAIN_PID="$$" + # Pick a "unique" port in the range 10000-19999. PORT="0000$$" PORT="1$(echo $PORT | tail -c 4)" From db2a6c1a20085afc5c61a70d6d95e7e3ef7be61a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Thu, 29 May 2014 12:15:40 +0200 Subject: [PATCH 4/5] Avoid sleep 1 at server start in ssl-opt.sh On my machine, brings running time from 135 to 45 seconds... 3 times faster :) --- tests/ssl-opt.sh | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 1f7a4adb35..238995ff8c 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -112,6 +112,28 @@ has_mem_err() { fi } +# wait for server to be ready +wait_srv_ready() { + if is_polar "$SRV_CMD"; then + READY_MSG="Waiting for a remote connection" + else + READY_MSG="ACCEPT" + fi + + # If the server isn't ready after 10 secs, something probably went wrong + ( sleep 10; echo "SERVERSTART TIMEOUT"; kill $MAIN_PID ) & + WATCHDOG_PID=$! + + while ! grep "$READY_MSG" $SRV_OUT >/dev/null; do + # don't use sleep, since the whole goal is to avoid wasting time, + # and 1 second is usually way more than the server needs to start + true + done + + kill $WATCHDOG_PID + wait $WATCHDOG_PID +} + # Usage: run_test name srv_cmd cli_cmd cli_exit [option [...]] # Options: -s pattern pattern that must be present in server output # -c pattern pattern that must be present in client output @@ -145,7 +167,7 @@ run_test() { echo "$SRV_CMD" > $SRV_OUT $SRV_CMD >> $SRV_OUT 2>&1 & SRV_PID=$! - sleep 1 + wait_srv_ready echo "$CLI_CMD" > $CLI_OUT eval "$CLI_CMD" >> $CLI_OUT 2>&1 CLI_EXIT=$? From 57255b147d47454c5effa4801196aa9208a8e6f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20P=C3=A9gouri=C3=A9-Gonnard?= Date: Mon, 9 Jun 2014 11:21:49 +0200 Subject: [PATCH 5/5] Tweak test ordering in all.sh --- tests/scripts/all.sh | 82 +++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 32 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 1e11d78c8f..d01dc259e3 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -6,8 +6,8 @@ # CMake configuration. After this script is run, the CMake cache is lost and # CMake is not initialised any more! # -# Assumes gcc, clang (recent enough for using ASan) are available, as weel as -# cmake. Also assumes valgrind is available if --memcheck is used. +# Assumes gcc and clang (recent enough for using ASan) are available, +# as well as cmake and valgrind. # Abort on errors (and uninitiliased variables) set -eu @@ -54,54 +54,72 @@ msg() echo "******************************************************************" } -# Step 1: various build types +# The test ordering tries to optimize for the following criteria: +# 1. Catch possible problems early, by running first test that run quickly +# and/or are more likely to fail than others. +# 2. Minimize total running time, by avoiding useless rebuilds +# +# Indicative running times are given for reference. -msg "Unix make, default compiler and flags" -cleanup -make - -msg "cmake, gcc with lots of warnings" +msg "build: cmake, gcc with lots of warnings" # ~ 1 min cleanup CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Check . make -msg "cmake, clang with lots of warnings" +msg "test: main suites with valgrind" # ~ 2 min 10s +make memcheck + +msg "build: with ASan" # ~ 1 min +cleanup +cmake -D CMAKE_BUILD_TYPE:String=ASan . +make + +msg "test: ssl-opt.sh (ASan build)" # ~ 1 min 10s +cd tests +./ssl-opt.sh +cd .. + +msg "test: main suites and selftest (ASan build)" # ~ 10s + 30s +make test +programs/test/selftest + +msg "test: ref-configs (ASan build)" # ~ 4 min 45 s +tests/scripts/test-ref-configs.pl + +# Most issues are likely to be caught at this point + +msg "build: with ASan (rebuild after ref-configs)" # ~ 1 min +make + +msg "test: compat.sh (ASan build)" # ~ 7 min 30s +cd tests +./compat.sh +cd .. + +msg "build: cmake, clang with lots of warnings" # ~ 40s cleanup CC=clang cmake -D CMAKE_BUILD_TYPE:String=Check . make -# Step 2: Full tests, with ASan - -msg "ASan build and full tests" +msg "build: Unix make, -O2" # ~ 30s cleanup -cmake -D CMAKE_BUILD_TYPE:String=ASan . make -make test -programs/test/selftest -cd tests -./compat.sh -./ssl-opt.sh -cd .. -tests/scripts/test-ref-configs.pl -# Step 3: using valgrind's memcheck - -msg "Release build, test suites with valgrind's memcheck" -cleanup -# optimized build to compensate a bit for valgrind slowdown -cmake -D CMAKE_BUILD_TYPE:String=Release . -make -make memcheck +# Optional parts that take a long time to run if [ "$MEMORY" -gt 0 ]; then + msg "test: ssl-opt --memcheck (-02 build)" # ~ 8 min cd tests ./ssl-opt.sh --memcheck - [ "$MEMORY" -gt 1 ] && ./compat.sh --memcheck cd .. - # no test-ref-configs: doesn't have a memcheck option (yet?) -fi -# Done + if [ "$MEMORY" -gt 1 ]; then + msg "test: compat --memcheck (-02 build)" # ~ 42 min + cd tests + ./compat.sh --memcheck + cd .. + fi +fi echo "Done." cleanup