From a12baf8c5fd7968859f1c4c9c0bb319d8cf08285 Mon Sep 17 00:00:00 2001 From: Gowtham Suresh Kumar Date: Wed, 19 Jul 2023 08:39:20 +0100 Subject: [PATCH 1/6] Update test scripts to use latest/earliest compilers The Ubuntu 16.04 and 22.04 docker images have been updated with earliest and latest versions of gcc and clang respectively. This patch adds the necessary component and support functions required for the CI to run these compilers. For FreeBSD we invoke the function by name so a condition is added to disable the existing test_clang_opt function for linux. Signed-off-by: Gowtham Suresh Kumar --- scripts/output_env.sh | 12 +++++++++++ tests/scripts/all.sh | 47 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/scripts/output_env.sh b/scripts/output_env.sh index 1d9e0faebe..ce2c500656 100755 --- a/scripts/output_env.sh +++ b/scripts/output_env.sh @@ -105,9 +105,21 @@ echo print_version "gcc" "--version" "" "head -n 1" echo +print_version "gcc-earliest" "--version" "" "head -n 1" +echo + +print_version "gcc-latest" "--version" "" "head -n 1" +echo + print_version "clang" "--version" "" "head -n 2" echo +print_version "clang-earliest" "--version" "" "head -n 2" +echo + +print_version "clang-latest" "--version" "" "head -n 2" +echo + print_version "ldd" "--version" "" "head -n 1" echo diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index a747c9d1ea..21f537106c 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -195,6 +195,13 @@ pre_initialize_variables () { # they are defined. ALL_COMPONENTS=$(sed -n 's/^ *component_\([0-9A-Z_a-z]*\) *().*/\1/p' <"$0") + # For Linux platforms we run latest/earliest versions of clang and the + # test_clang_opt function is only for FreeBSD. This condition removes + # test_clang_opt element from the ALL_COMPONENTS array for Linux. + if [[ $(uname) == "Linux" ]]; then + ALL_COMPONENTS=( "${ALL_COMPONENTS[@]/test_clang_opt}" ) + fi + # Exclude components that are not supported on this platform. SUPPORTED_COMPONENTS= for component in $ALL_COMPONENTS; do @@ -3993,6 +4000,7 @@ component_test_cmake_shared () { test_build_opt () { info=$1 cc=$2; shift 2 + $cc --version for opt in "$@"; do msg "build/test: $cc $opt, $info" # ~ 30s make CC="$cc" CFLAGS="$opt -std=c99 -pedantic -Wall -Wextra -Werror" @@ -4005,14 +4013,45 @@ test_build_opt () { done } -component_test_clang_opt () { +# For FreeBSD we invoke the function by name so this condition is added +# to disable the existing test_clang_opt function for linux. +if [[ $(uname) != "Linux" ]]; then + component_test_clang_opt () { + scripts/config.py full + test_build_opt 'full config' clang -O0 -Os -O2 + } +fi + +component_test_clang_latest_opt () { scripts/config.py full - test_build_opt 'full config' clang -O0 -Os -O2 + test_build_opt 'full config' clang-latest -O0 -Os -O2 +} +support_test_clang_latest_opt () { + type clang-latest >/dev/null 2>/dev/null } -component_test_gcc_opt () { +component_test_clang_earliest_opt () { scripts/config.py full - test_build_opt 'full config' gcc -O0 -Os -O2 + test_build_opt 'full config' clang-earliest -O0 +} +support_test_clang_earliest_opt () { + type clang-earliest >/dev/null 2>/dev/null +} + +component_test_gcc_latest_opt () { + scripts/config.py full + test_build_opt 'full config' gcc-latest -O0 -Os -O2 +} +support_test_gcc_latest_opt () { + type gcc-latest >/dev/null 2>/dev/null +} + +component_test_gcc_earliest_opt () { + scripts/config.py full + test_build_opt 'full config' gcc-earliest -O0 +} +support_test_gcc_earliest_opt () { + type gcc-earliest >/dev/null 2>/dev/null } component_build_mbedtls_config_file () { From 186731b22acea22c0af32e2577c7ce1cb6e2491e Mon Sep 17 00:00:00 2001 From: Gowtham Suresh Kumar Date: Wed, 26 Jul 2023 15:47:45 +0100 Subject: [PATCH 2/6] Fix warnings from clang-16 Running clang-16 on mbedtls reports warnings of type "-Wstrict-prototypes". This patch fixes these warnings by adding void to functions with no arguments. The generate_test_code.py is modified to insert void into test functions with no arguments in *.function files. Signed-off-by: Gowtham Suresh Kumar --- library/ecp.c | 2 +- programs/fuzz/common.c | 2 +- programs/fuzz/common.h | 2 +- programs/ssl/ssl_context_info.c | 6 +++--- programs/test/udp_proxy.c | 2 +- tests/scripts/generate_test_code.py | 5 +++++ tests/scripts/test_generate_test_code.py | 12 ++++++------ 7 files changed, 18 insertions(+), 13 deletions(-) diff --git a/library/ecp.c b/library/ecp.c index 049a1e0151..29f5bc77fa 100644 --- a/library/ecp.c +++ b/library/ecp.c @@ -3639,7 +3639,7 @@ cleanup: #if defined(MBEDTLS_TEST_HOOKS) MBEDTLS_STATIC_TESTABLE -mbedtls_ecp_variant mbedtls_ecp_get_variant() +mbedtls_ecp_variant mbedtls_ecp_get_variant(void) { return MBEDTLS_ECP_VARIANT_WITH_MPI_STRUCT; } diff --git a/programs/fuzz/common.c b/programs/fuzz/common.c index 75021bb2f0..98aa4037b3 100644 --- a/programs/fuzz/common.c +++ b/programs/fuzz/common.c @@ -13,7 +13,7 @@ mbedtls_time_t dummy_constant_time(mbedtls_time_t *time) } #endif -void dummy_init() +void dummy_init(void) { #if defined(MBEDTLS_PLATFORM_TIME_ALT) mbedtls_platform_set_time(dummy_constant_time); diff --git a/programs/fuzz/common.h b/programs/fuzz/common.h index bc817bd769..094383c7a4 100644 --- a/programs/fuzz/common.h +++ b/programs/fuzz/common.h @@ -15,7 +15,7 @@ typedef struct fuzzBufferOffset { #if defined(MBEDTLS_HAVE_TIME) mbedtls_time_t dummy_constant_time(mbedtls_time_t *time); #endif -void dummy_init(); +void dummy_init(void); int dummy_send(void *ctx, const unsigned char *buf, size_t len); int fuzz_recv(void *ctx, unsigned char *buf, size_t len); diff --git a/programs/ssl/ssl_context_info.c b/programs/ssl/ssl_context_info.c index a5f0650322..855b0911fd 100644 --- a/programs/ssl/ssl_context_info.c +++ b/programs/ssl/ssl_context_info.c @@ -125,12 +125,12 @@ const char buf_ln_err[] = "Buffer does not have enough data to complete the pars /* * Basic printing functions */ -void print_version() +void print_version(void) { printf("%s v%d.%d\n", PROG_NAME, VER_MAJOR, VER_MINOR); } -void print_usage() +void print_usage(void) { print_version(); printf("\nThis program is used to deserialize an Mbed TLS SSL session from the base64 code provided\n" @@ -179,7 +179,7 @@ void printf_err(const char *str, ...) /* * Exit from the program in case of error */ -void error_exit() +void error_exit(void) { if (NULL != b64_file) { fclose(b64_file); diff --git a/programs/test/udp_proxy.c b/programs/test/udp_proxy.c index ac527f2573..685e336e67 100644 --- a/programs/test/udp_proxy.c +++ b/programs/test/udp_proxy.c @@ -644,7 +644,7 @@ void delay_packet(packet *delay) memcpy(&prev[prev_len++], delay, sizeof(packet)); } -int send_delayed() +int send_delayed(void) { uint8_t offset; int ret; diff --git a/tests/scripts/generate_test_code.py b/tests/scripts/generate_test_code.py index ff7f9b9976..c42f9a84ba 100755 --- a/tests/scripts/generate_test_code.py +++ b/tests/scripts/generate_test_code.py @@ -667,6 +667,11 @@ def parse_function_code(funcs_f, dependencies, suite_dependencies): code = code.replace(name, 'test_' + name, 1) name = 'test_' + name + # If a test function has no arguments then add 'void' argument to + # avoid "-Wstrict-prototypes" warnings from clang-16 + if len(args) == 0: + code = code.replace('()', '(void)', 1) + for line in funcs_f: if re.search(END_CASE_REGEX, line): break diff --git a/tests/scripts/test_generate_test_code.py b/tests/scripts/test_generate_test_code.py index fe748aeb46..b32d18423b 100755 --- a/tests/scripts/test_generate_test_code.py +++ b/tests/scripts/test_generate_test_code.py @@ -647,7 +647,7 @@ void func() self.assertEqual(arg, []) expected = '''#line 1 "test_suite_ut.function" -void test_func() +void test_func(void) { ba ba black sheep have you any wool @@ -690,7 +690,7 @@ exit: expected = '''#line 1 "test_suite_ut.function" -void test_func() +void test_func(void) { ba ba black sheep have you any wool @@ -750,7 +750,7 @@ exit: void -test_func() +test_func(void) { ba ba black sheep have you any wool @@ -803,7 +803,7 @@ exit: -void test_func() +void test_func(void) { ba ba black sheep have you any wool @@ -1139,7 +1139,7 @@ void func2() #if defined(MBEDTLS_ENTROPY_NV_SEED) #if defined(MBEDTLS_FS_IO) #line 13 "test_suite_ut.function" -void test_func1() +void test_func1(void) { exit: ; @@ -1156,7 +1156,7 @@ void test_func1_wrapper( void ** params ) #if defined(MBEDTLS_ENTROPY_NV_SEED) #if defined(MBEDTLS_FS_IO) #line 19 "test_suite_ut.function" -void test_func2() +void test_func2(void) { exit: ; From 8d45ec8b976b70675083140bf68b6b7956fd7c79 Mon Sep 17 00:00:00 2001 From: Gowtham Suresh Kumar Date: Fri, 28 Jul 2023 16:36:25 +0100 Subject: [PATCH 3/6] Remove test_clang_opt check The component functions in all.sh will be listed using compgen instead of sed so this check is not needed. Signed-off-by: Gowtham Suresh Kumar --- tests/scripts/all.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 21f537106c..f1da9d2cd8 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -195,13 +195,6 @@ pre_initialize_variables () { # they are defined. ALL_COMPONENTS=$(sed -n 's/^ *component_\([0-9A-Z_a-z]*\) *().*/\1/p' <"$0") - # For Linux platforms we run latest/earliest versions of clang and the - # test_clang_opt function is only for FreeBSD. This condition removes - # test_clang_opt element from the ALL_COMPONENTS array for Linux. - if [[ $(uname) == "Linux" ]]; then - ALL_COMPONENTS=( "${ALL_COMPONENTS[@]/test_clang_opt}" ) - fi - # Exclude components that are not supported on this platform. SUPPORTED_COMPONENTS= for component in $ALL_COMPONENTS; do From 1e829a403f7d08bb36162367369b97a1508c6764 Mon Sep 17 00:00:00 2001 From: Gowtham Suresh Kumar Date: Fri, 28 Jul 2023 16:41:21 +0100 Subject: [PATCH 4/6] Use compgen to gather components in all.sh Signed-off-by: Gowtham Suresh Kumar --- tests/scripts/all.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index f1da9d2cd8..ad31e5c6d7 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -191,9 +191,7 @@ pre_initialize_variables () { # Gather the list of available components. These are the functions # defined in this script whose name starts with "component_". - # Parse the script with sed. This way we get the functions in the order - # they are defined. - ALL_COMPONENTS=$(sed -n 's/^ *component_\([0-9A-Z_a-z]*\) *().*/\1/p' <"$0") + ALL_COMPONENTS=$(compgen -A function component_ | sed 's/component_//') # Exclude components that are not supported on this platform. SUPPORTED_COMPONENTS= From 6f1977bf208b3361e3651412ce889e1bcdd7e3fb Mon Sep 17 00:00:00 2001 From: Gowtham Suresh Kumar Date: Fri, 28 Jul 2023 17:04:47 +0100 Subject: [PATCH 5/6] Use variables for selecting compilers The latest and earliest clang/GCC compilers are now used through variables instead of symlinks and also the all.sh script is updated to support options for overriding the default values. Signed-off-by: Gowtham Suresh Kumar --- scripts/output_env.sh | 24 ++++++++++++++++++++---- tests/scripts/all.sh | 29 ++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/scripts/output_env.sh b/scripts/output_env.sh index ce2c500656..535613298e 100755 --- a/scripts/output_env.sh +++ b/scripts/output_env.sh @@ -105,19 +105,35 @@ echo print_version "gcc" "--version" "" "head -n 1" echo -print_version "gcc-earliest" "--version" "" "head -n 1" +if [ -n "${GCC_EARLIEST+set}" ]; then + print_version "${GCC_EARLIEST}" "--version" "" "head -n 1" +else + echo " GCC_EARLIEST : Not configured." +fi echo -print_version "gcc-latest" "--version" "" "head -n 1" +if [ -n "${GCC_LATEST+set}" ]; then + print_version "${GCC_LATEST}" "--version" "" "head -n 1" +else + echo " GCC_LATEST : Not configured." +fi echo print_version "clang" "--version" "" "head -n 2" echo -print_version "clang-earliest" "--version" "" "head -n 2" +if [ -n "${CLANG_EARLIEST+set}" ]; then + print_version "${CLANG_EARLIEST}" "--version" "" "head -n 2" +else + echo " CLANG_EARLIEST : Not configured." +fi echo -print_version "clang-latest" "--version" "" "head -n 2" +if [ -n "${CLANG_LATEST+set}" ]; then + print_version "${CLANG_LATEST}" "--version" "" "head -n 2" +else + echo " CLANG_LATEST : Not configured." +fi echo print_version "ldd" "--version" "" "head -n 1" diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index ad31e5c6d7..8caec486a2 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -176,7 +176,10 @@ pre_initialize_variables () { : ${ARMC6_BIN_DIR:=/usr/bin} : ${ARM_NONE_EABI_GCC_PREFIX:=arm-none-eabi-} : ${ARM_LINUX_GNUEABI_GCC_PREFIX:=arm-linux-gnueabi-} - + : ${CLANG_LATEST:="clang-16"} + : ${CLANG_EARLIEST:="clang-3.5"} + : ${GCC_LATEST:="gcc-12"} + : ${GCC_EARLIEST:="gcc-4.7"} # if MAKEFLAGS is not set add the -j option to speed up invocations of make if [ -z "${MAKEFLAGS+set}" ]; then export MAKEFLAGS="-j$(all_sh_nproc)" @@ -273,6 +276,10 @@ General options: Tool path options: --armc5-bin-dir= ARM Compiler 5 bin directory. --armc6-bin-dir= ARM Compiler 6 bin directory. + --clang-earliest= Earliest version of clang available + --clang-latest= Latest version of clang available + --gcc-earliest= Earliest version of GCC available + --gcc-latest= Latest version of GCC available --gnutls-cli= GnuTLS client executable to use for most tests. --gnutls-serv= GnuTLS server executable to use for most tests. --gnutls-legacy-cli= GnuTLS client executable to use for legacy tests. @@ -439,9 +446,13 @@ pre_parse_command_line () { --armcc) no_armcc=;; --armc5-bin-dir) shift; ;; # assignment to ARMC5_BIN_DIR done in pre_parse_command_line_for_dirs --armc6-bin-dir) shift; ;; # assignment to ARMC6_BIN_DIR done in pre_parse_command_line_for_dirs + --clang-earliest) shift; CLANG_EARLIEST="$1";; + --clang-latest) shift; CLANG_LATEST="$1";; --error-test) error_test=$((error_test + 1));; --except) all_except=1;; --force|-f) FORCE=1;; + --gcc-earliest) shift; GCC_EARLIEST="$1";; + --gcc-latest) shift; GCC_LATEST="$1";; --gnutls-cli) shift; GNUTLS_CLI="$1";; --gnutls-legacy-cli) shift; GNUTLS_LEGACY_CLI="$1";; --gnutls-legacy-serv) shift; GNUTLS_LEGACY_SERV="$1";; @@ -4015,34 +4026,34 @@ fi component_test_clang_latest_opt () { scripts/config.py full - test_build_opt 'full config' clang-latest -O0 -Os -O2 + test_build_opt 'full config' "$CLANG_LATEST" -O0 -Os -O2 } support_test_clang_latest_opt () { - type clang-latest >/dev/null 2>/dev/null + type "$CLANG_LATEST" >/dev/null 2>/dev/null } component_test_clang_earliest_opt () { scripts/config.py full - test_build_opt 'full config' clang-earliest -O0 + test_build_opt 'full config' "$CLANG_EARLIEST" -O0 } support_test_clang_earliest_opt () { - type clang-earliest >/dev/null 2>/dev/null + type "$CLANG_EARLIEST" >/dev/null 2>/dev/null } component_test_gcc_latest_opt () { scripts/config.py full - test_build_opt 'full config' gcc-latest -O0 -Os -O2 + test_build_opt 'full config' "$GCC_LATEST" -O0 -Os -O2 } support_test_gcc_latest_opt () { - type gcc-latest >/dev/null 2>/dev/null + type "$GCC_LATEST" >/dev/null 2>/dev/null } component_test_gcc_earliest_opt () { scripts/config.py full - test_build_opt 'full config' gcc-earliest -O0 + test_build_opt 'full config' "$GCC_EARLIEST" -O0 } support_test_gcc_earliest_opt () { - type gcc-earliest >/dev/null 2>/dev/null + type "$GCC_EARLIEST" >/dev/null 2>/dev/null } component_build_mbedtls_config_file () { From 9da40b89270228c7a036cead713f777b6e886c72 Mon Sep 17 00:00:00 2001 From: Gowtham Suresh Kumar Date: Mon, 31 Jul 2023 16:38:10 +0100 Subject: [PATCH 6/6] Update default variable values for compilers Signed-off-by: Gowtham Suresh Kumar --- tests/scripts/all.sh | 8 ++++---- tests/scripts/generate_test_code.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 8caec486a2..03782636e5 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -176,10 +176,10 @@ pre_initialize_variables () { : ${ARMC6_BIN_DIR:=/usr/bin} : ${ARM_NONE_EABI_GCC_PREFIX:=arm-none-eabi-} : ${ARM_LINUX_GNUEABI_GCC_PREFIX:=arm-linux-gnueabi-} - : ${CLANG_LATEST:="clang-16"} - : ${CLANG_EARLIEST:="clang-3.5"} - : ${GCC_LATEST:="gcc-12"} - : ${GCC_EARLIEST:="gcc-4.7"} + : ${CLANG_LATEST:="clang-latest"} + : ${CLANG_EARLIEST:="clang-earliest"} + : ${GCC_LATEST:="gcc-latest"} + : ${GCC_EARLIEST:="gcc-earliest"} # if MAKEFLAGS is not set add the -j option to speed up invocations of make if [ -z "${MAKEFLAGS+set}" ]; then export MAKEFLAGS="-j$(all_sh_nproc)" diff --git a/tests/scripts/generate_test_code.py b/tests/scripts/generate_test_code.py index c42f9a84ba..76806de95f 100755 --- a/tests/scripts/generate_test_code.py +++ b/tests/scripts/generate_test_code.py @@ -668,7 +668,7 @@ def parse_function_code(funcs_f, dependencies, suite_dependencies): name = 'test_' + name # If a test function has no arguments then add 'void' argument to - # avoid "-Wstrict-prototypes" warnings from clang-16 + # avoid "-Wstrict-prototypes" warnings from clang if len(args) == 0: code = code.replace('()', '(void)', 1)