diff --git a/tests/compat.sh b/tests/compat.sh index 6506e6c09d..a98ed0eec5 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -127,7 +127,7 @@ print_usage() { printf " \tAlso available: GnuTLS (needs v3.2.15 or higher)\n" printf " -M|--memcheck\tCheck memory leaks and errors.\n" printf " -v|--verbose\tSet verbose output.\n" - printf " --list-test-case\tList all potential test cases (No Execution)\n" + printf " --list-test-cases\tList all potential test cases (No Execution)\n" printf " --outcome-file\tFile where test outcomes are written\n" printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n" printf " --preserve-logs\tPreserve logs of successful tests as well\n" @@ -141,8 +141,8 @@ print_test_case() { done } -# list_test_case lists all potential test cases in compat.sh without execution -list_test_case() { +# list_test_cases lists all potential test cases in compat.sh without execution +list_test_cases() { reset_ciphersuites for TYPE in $TYPES; do add_common_ciphersuites @@ -191,9 +191,9 @@ get_options() { MEMCHECK=1 ;; # Please check scripts/check_test_cases.py correspondingly - # if you have to modify option, --list-test-case - --list-test-case) - list_test_case + # if you have to modify option, --list-test-cases + --list-test-cases) + list_test_cases exit $? ;; --outcome-file) @@ -869,7 +869,7 @@ wait_client_done() { } # uniform_title -# $TITLE is considered as test case description for both --list-test-case and +# $TITLE is considered as test case description for both --list-test-cases and # MBEDTLS_TEST_OUTCOME_FILE. This function aims to control the format of # each test case description. uniform_title() { diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 1395d4d901..30879d7635 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -28,6 +28,7 @@ import re import subprocess import sys + class Results: """Store file and line information about errors or warnings in test suites.""" @@ -97,33 +98,21 @@ state may override this method. data_file_name, line_number, line) in_paragraph = True - def walk_ssl_opt_sh(self, file_name): - """Iterate over the test cases in ssl-opt.sh or a file with a similar format.""" + def collect_from_script(self, file_name): + """Collect the test cases in a script by calling its listing test cases +option""" descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none - with open(file_name, 'rb') as file_contents: - for line_number, line in enumerate(file_contents, 1): - # Assume that all run_test calls have the same simple form - # with the test description entirely on the same line as the - # function name. - m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line) - if not m: - continue - description = m.group(1) - self.process_test_case(descriptions, - file_name, line_number, description) - - def walk_compat_sh(self, file_name): - """Iterate over the test cases compat.sh with a similar format.""" - descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none - compat_cmd = ['sh', file_name, '--list-test-case'] - compat_output = subprocess.check_output(compat_cmd) - # Assume compat.sh is responsible for printing identical format of - # test case description between --list-test-case and its OUTCOME.CSV - description = compat_output.strip().split(b'\n') + listed = subprocess.check_output(['sh', file_name, '--list-test-cases']) + # Assume test file is responsible for printing identical format of + # test case description between --list-test-cases and its OUTCOME.CSV + # # idx indicates the number of test case since there is no line number - # in `compat.sh` for each test case. - for idx, descrip in enumerate(description): - self.process_test_case(descriptions, file_name, idx, descrip) + # in the script for each test case. + for idx, description in enumerate(listed.splitlines()): + self.process_test_case(descriptions, + file_name, + idx, + description.rstrip()) @staticmethod def collect_test_directories(): @@ -144,15 +133,11 @@ state may override this method. for data_file_name in glob.glob(os.path.join(directory, 'suites', '*.data')): self.walk_test_suite(data_file_name) - ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh') - if os.path.exists(ssl_opt_sh): - self.walk_ssl_opt_sh(ssl_opt_sh) - for ssl_opt_file_name in glob.glob(os.path.join(directory, 'opt-testcases', - '*.sh')): - self.walk_ssl_opt_sh(ssl_opt_file_name) - compat_sh = os.path.join(directory, 'compat.sh') - if os.path.exists(compat_sh): - self.walk_compat_sh(compat_sh) + + for sh_file in ['ssl-opt.sh', 'compat.sh']: + sh_file = os.path.join(directory, sh_file) + if os.path.exists(sh_file): + self.collect_from_script(sh_file) class TestDescriptions(TestDescriptionExplorer): """Collect the available test cases.""" diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 0dd7fe6d36..36753af8a8 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -123,6 +123,7 @@ FILTER='.*' EXCLUDE='^$' SHOW_TEST_NUMBER=0 +LIST_TESTS=0 RUN_TEST_NUMBER='' PRESERVE_LOGS=0 @@ -142,6 +143,7 @@ print_usage() { printf " -n|--number\tExecute only numbered test (comma-separated, e.g. '245,256')\n" printf " -s|--show-numbers\tShow test numbers in front of test names\n" printf " -p|--preserve-logs\tPreserve logs of successful tests as well\n" + printf " --list-test-cases\tList all potential test cases (No Execution)\n" printf " --outcome-file\tFile where test outcomes are written\n" printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n" printf " --port \tTCP/UDP port (default: randomish 1xxxx)\n" @@ -167,6 +169,9 @@ get_options() { -s|--show-numbers) SHOW_TEST_NUMBER=1 ;; + -l|--list-test-cases) + LIST_TESTS=1 + ;; -p|--preserve-logs) PRESERVE_LOGS=1 ;; @@ -196,11 +201,18 @@ get_options() { done } +get_options "$@" + # Read boolean configuration options from mbedtls_config.h for easy and quick # testing. Skip non-boolean options (with something other than spaces # and a comment after "#define SYMBOL"). The variable contains a # space-separated list of symbols. -CONFIGS_ENABLED=" $(echo `$P_QUERY -l` )" +if [ "$LIST_TESTS" -eq 0 ];then + CONFIGS_ENABLED=" $(echo `$P_QUERY -l` )" +else + P_QUERY=":" + CONFIGS_ENABLED="" +fi # Skip next test; use this macro to skip tests which are legitimate # in theory and expected to be re-introduced at some point, but # aren't expected to succeed at the moment due to problems outside @@ -296,7 +308,12 @@ get_config_value_or_default() { # # Note that if the configuration is not defined or is defined to nothing, # the output of this function will be an empty string. - ${P_SRV} "query_config=${1}" + if [ "$LIST_TESTS" -eq 0 ];then + ${P_SRV} "query_config=${1}" + else + echo "1" + fi + } requires_config_value_at_least() { @@ -807,19 +824,20 @@ requires_not_i686() { fi } -# Calculate the input & output maximum content lengths set in the config MAX_CONTENT_LEN=16384 MAX_IN_LEN=$( get_config_value_or_default "MBEDTLS_SSL_IN_CONTENT_LEN" ) MAX_OUT_LEN=$( get_config_value_or_default "MBEDTLS_SSL_OUT_CONTENT_LEN" ) +if [ "$LIST_TESTS" -eq 0 ];then + # Calculate the input & output maximum content lengths set in the config -# Calculate the maximum content length that fits both -if [ "$MAX_IN_LEN" -lt "$MAX_CONTENT_LEN" ]; then - MAX_CONTENT_LEN="$MAX_IN_LEN" + # Calculate the maximum content length that fits both + if [ "$MAX_IN_LEN" -lt "$MAX_CONTENT_LEN" ]; then + MAX_CONTENT_LEN="$MAX_IN_LEN" + fi + if [ "$MAX_OUT_LEN" -lt "$MAX_CONTENT_LEN" ]; then + MAX_CONTENT_LEN="$MAX_OUT_LEN" + fi fi -if [ "$MAX_OUT_LEN" -lt "$MAX_CONTENT_LEN" ]; then - MAX_CONTENT_LEN="$MAX_OUT_LEN" -fi - # skip the next test if the SSL output buffer is less than 16KB requires_full_size_output_buffer() { if [ "$MAX_OUT_LEN" -ne 16384 ]; then @@ -861,6 +879,7 @@ print_name() { fi LINE="$LINE$1" + printf "%s " "$LINE" LEN=$(( 72 - `echo "$LINE" | wc -c` )) for i in `seq 1 $LEN`; do printf '.'; done @@ -876,7 +895,7 @@ record_outcome() { if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ]; then printf '%s;%s;%s;%s;%s;%s\n' \ "$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \ - "${TEST_SUITE_NAME:-ssl-opt}" "$NAME" \ + "ssl-opt" "$NAME" \ "$1" "${2-}" \ >>"$MBEDTLS_TEST_OUTCOME_FILE" fi @@ -1578,6 +1597,11 @@ run_test() { return fi + if [ "$LIST_TESTS" -gt 0 ]; then + printf "%s\n" "$NAME" + return + fi + print_name "$NAME" # Do we only run numbered tests? @@ -1775,8 +1799,6 @@ cleanup() { # MAIN # -get_options "$@" - # Make the outcome file path relative to the original directory, not # to .../tests case "$MBEDTLS_TEST_OUTCOME_FILE" in @@ -1827,109 +1849,112 @@ else } fi -# sanity checks, avoid an avalanche of errors -P_SRV_BIN="${P_SRV%%[ ]*}" -P_CLI_BIN="${P_CLI%%[ ]*}" -P_PXY_BIN="${P_PXY%%[ ]*}" -if [ ! -x "$P_SRV_BIN" ]; then - echo "Command '$P_SRV_BIN' is not an executable file" - exit 1 -fi -if [ ! -x "$P_CLI_BIN" ]; then - echo "Command '$P_CLI_BIN' is not an executable file" - exit 1 -fi -if [ ! -x "$P_PXY_BIN" ]; then - echo "Command '$P_PXY_BIN' is not an executable file" - exit 1 -fi -if [ "$MEMCHECK" -gt 0 ]; then - if which valgrind >/dev/null 2>&1; then :; else - echo "Memcheck not possible. Valgrind not found" +if [ "$LIST_TESTS" -eq 0 ];then + + # sanity checks, avoid an avalanche of errors + P_SRV_BIN="${P_SRV%%[ ]*}" + P_CLI_BIN="${P_CLI%%[ ]*}" + P_PXY_BIN="${P_PXY%%[ ]*}" + if [ ! -x "$P_SRV_BIN" ]; then + echo "Command '$P_SRV_BIN' is not an executable file" exit 1 fi + if [ ! -x "$P_CLI_BIN" ]; then + echo "Command '$P_CLI_BIN' is not an executable file" + exit 1 + fi + if [ ! -x "$P_PXY_BIN" ]; then + echo "Command '$P_PXY_BIN' is not an executable file" + exit 1 + fi + if [ "$MEMCHECK" -gt 0 ]; then + if which valgrind >/dev/null 2>&1; then :; else + echo "Memcheck not possible. Valgrind not found" + exit 1 + fi + fi + if which $OPENSSL >/dev/null 2>&1; then :; else + echo "Command '$OPENSSL' not found" + exit 1 + fi + + # used by watchdog + MAIN_PID="$$" + + # We use somewhat arbitrary delays for tests: + # - how long do we wait for the server to start (when lsof not available)? + # - how long do we allow for the client to finish? + # (not to check performance, just to avoid waiting indefinitely) + # Things are slower with valgrind, so give extra time here. + # + # Note: without lsof, there is a trade-off between the running time of this + # script and the risk of spurious errors because we didn't wait long enough. + # The watchdog delay on the other hand doesn't affect normal running time of + # the script, only the case where a client or server gets stuck. + if [ "$MEMCHECK" -gt 0 ]; then + START_DELAY=6 + DOG_DELAY=60 + else + START_DELAY=2 + DOG_DELAY=20 + fi + + # some particular tests need more time: + # - for the client, we multiply the usual watchdog limit by a factor + # - for the server, we sleep for a number of seconds after the client exits + # see client_need_more_time() and server_needs_more_time() + CLI_DELAY_FACTOR=1 + SRV_DELAY_SECONDS=0 + + # fix commands to use this port, force IPv4 while at it + # +SRV_PORT will be replaced by either $SRV_PORT or $PXY_PORT later + # Note: Using 'localhost' rather than 127.0.0.1 here is unwise, as on many + # machines that will resolve to ::1, and we don't want ipv6 here. + P_SRV="$P_SRV server_addr=127.0.0.1 server_port=$SRV_PORT" + P_CLI="$P_CLI server_addr=127.0.0.1 server_port=+SRV_PORT" + P_PXY="$P_PXY server_addr=127.0.0.1 server_port=$SRV_PORT listen_addr=127.0.0.1 listen_port=$PXY_PORT ${SEED:+"seed=$SEED"}" + O_SRV="$O_SRV -accept $SRV_PORT" + O_CLI="$O_CLI -connect 127.0.0.1:+SRV_PORT" + G_SRV="$G_SRV -p $SRV_PORT" + G_CLI="$G_CLI -p +SRV_PORT" + + # Newer versions of OpenSSL have a syntax to enable all "ciphers", even + # low-security ones. This covers not just cipher suites but also protocol + # versions. It is necessary, for example, to use (D)TLS 1.0/1.1 on + # OpenSSL 1.1.1f from Ubuntu 20.04. The syntax was only introduced in + # OpenSSL 1.1.0 (21e0c1d23afff48601eb93135defddae51f7e2e3) and I can't find + # a way to discover it from -help, so check the openssl version. + case $($OPENSSL version) in + "OpenSSL 0"*|"OpenSSL 1.0"*) :;; + *) + O_CLI="$O_CLI -cipher ALL@SECLEVEL=0" + O_SRV="$O_SRV -cipher ALL@SECLEVEL=0" + ;; + esac + + if [ -n "${OPENSSL_NEXT:-}" ]; then + O_NEXT_SRV="$O_NEXT_SRV -accept $SRV_PORT" + O_NEXT_SRV_NO_CERT="$O_NEXT_SRV_NO_CERT -accept $SRV_PORT" + O_NEXT_SRV_EARLY_DATA="$O_NEXT_SRV_EARLY_DATA -accept $SRV_PORT" + O_NEXT_CLI="$O_NEXT_CLI -connect 127.0.0.1:+SRV_PORT" + O_NEXT_CLI_NO_CERT="$O_NEXT_CLI_NO_CERT -connect 127.0.0.1:+SRV_PORT" + fi + + if [ -n "${GNUTLS_NEXT_SERV:-}" ]; then + G_NEXT_SRV="$G_NEXT_SRV -p $SRV_PORT" + G_NEXT_SRV_NO_CERT="$G_NEXT_SRV_NO_CERT -p $SRV_PORT" + fi + + if [ -n "${GNUTLS_NEXT_CLI:-}" ]; then + G_NEXT_CLI="$G_NEXT_CLI -p +SRV_PORT" + G_NEXT_CLI_NO_CERT="$G_NEXT_CLI_NO_CERT -p +SRV_PORT localhost" + fi + + # Allow SHA-1, because many of our test certificates use it + P_SRV="$P_SRV allow_sha1=1" + P_CLI="$P_CLI allow_sha1=1" + fi -if which $OPENSSL >/dev/null 2>&1; then :; else - echo "Command '$OPENSSL' not found" - exit 1 -fi - -# used by watchdog -MAIN_PID="$$" - -# We use somewhat arbitrary delays for tests: -# - how long do we wait for the server to start (when lsof not available)? -# - how long do we allow for the client to finish? -# (not to check performance, just to avoid waiting indefinitely) -# Things are slower with valgrind, so give extra time here. -# -# Note: without lsof, there is a trade-off between the running time of this -# script and the risk of spurious errors because we didn't wait long enough. -# The watchdog delay on the other hand doesn't affect normal running time of -# the script, only the case where a client or server gets stuck. -if [ "$MEMCHECK" -gt 0 ]; then - START_DELAY=6 - DOG_DELAY=60 -else - START_DELAY=2 - DOG_DELAY=20 -fi - -# some particular tests need more time: -# - for the client, we multiply the usual watchdog limit by a factor -# - for the server, we sleep for a number of seconds after the client exits -# see client_need_more_time() and server_needs_more_time() -CLI_DELAY_FACTOR=1 -SRV_DELAY_SECONDS=0 - -# fix commands to use this port, force IPv4 while at it -# +SRV_PORT will be replaced by either $SRV_PORT or $PXY_PORT later -# Note: Using 'localhost' rather than 127.0.0.1 here is unwise, as on many -# machines that will resolve to ::1, and we don't want ipv6 here. -P_SRV="$P_SRV server_addr=127.0.0.1 server_port=$SRV_PORT" -P_CLI="$P_CLI server_addr=127.0.0.1 server_port=+SRV_PORT" -P_PXY="$P_PXY server_addr=127.0.0.1 server_port=$SRV_PORT listen_addr=127.0.0.1 listen_port=$PXY_PORT ${SEED:+"seed=$SEED"}" -O_SRV="$O_SRV -accept $SRV_PORT" -O_CLI="$O_CLI -connect 127.0.0.1:+SRV_PORT" -G_SRV="$G_SRV -p $SRV_PORT" -G_CLI="$G_CLI -p +SRV_PORT" - -# Newer versions of OpenSSL have a syntax to enable all "ciphers", even -# low-security ones. This covers not just cipher suites but also protocol -# versions. It is necessary, for example, to use (D)TLS 1.0/1.1 on -# OpenSSL 1.1.1f from Ubuntu 20.04. The syntax was only introduced in -# OpenSSL 1.1.0 (21e0c1d23afff48601eb93135defddae51f7e2e3) and I can't find -# a way to discover it from -help, so check the openssl version. -case $($OPENSSL version) in - "OpenSSL 0"*|"OpenSSL 1.0"*) :;; - *) - O_CLI="$O_CLI -cipher ALL@SECLEVEL=0" - O_SRV="$O_SRV -cipher ALL@SECLEVEL=0" - ;; -esac - -if [ -n "${OPENSSL_NEXT:-}" ]; then - O_NEXT_SRV="$O_NEXT_SRV -accept $SRV_PORT" - O_NEXT_SRV_NO_CERT="$O_NEXT_SRV_NO_CERT -accept $SRV_PORT" - O_NEXT_SRV_EARLY_DATA="$O_NEXT_SRV_EARLY_DATA -accept $SRV_PORT" - O_NEXT_CLI="$O_NEXT_CLI -connect 127.0.0.1:+SRV_PORT" - O_NEXT_CLI_NO_CERT="$O_NEXT_CLI_NO_CERT -connect 127.0.0.1:+SRV_PORT" -fi - -if [ -n "${GNUTLS_NEXT_SERV:-}" ]; then - G_NEXT_SRV="$G_NEXT_SRV -p $SRV_PORT" - G_NEXT_SRV_NO_CERT="$G_NEXT_SRV_NO_CERT -p $SRV_PORT" -fi - -if [ -n "${GNUTLS_NEXT_CLI:-}" ]; then - G_NEXT_CLI="$G_NEXT_CLI -p +SRV_PORT" - G_NEXT_CLI_NO_CERT="$G_NEXT_CLI_NO_CERT -p +SRV_PORT localhost" -fi - -# Allow SHA-1, because many of our test certificates use it -P_SRV="$P_SRV allow_sha1=1" -P_CLI="$P_CLI allow_sha1=1" - # Also pick a unique name for intermediate files SRV_OUT="srv_out.$$" CLI_OUT="cli_out.$$" @@ -13375,17 +13400,21 @@ requires_config_enabled MBEDTLS_SSL_MAX_FRAGMENT_LENGTH requires_max_content_len 16384 run_tests_memory_after_hanshake -# Final report +if [ "$LIST_TESTS" -eq 0 ]; then -echo "------------------------------------------------------------------------" + # Final report + + echo "------------------------------------------------------------------------" + + if [ $FAILS = 0 ]; then + printf "PASSED" + else + printf "FAILED" + fi + PASSES=$(( $TESTS - $FAILS )) + echo " ($PASSES / $TESTS tests ($SKIPS skipped))" -if [ $FAILS = 0 ]; then - printf "PASSED" -else - printf "FAILED" fi -PASSES=$(( $TESTS - $FAILS )) -echo " ($PASSES / $TESTS tests ($SKIPS skipped))" if [ $FAILS -gt 255 ]; then # Clamp at 255 as caller gets exit code & 0xFF