Merge pull request #5463 from gilles-peskine-arm/cmake-test-suite-enumeration

CMake: generate the list of test suites automatically

tests/CMakeLists.txt

@@ -154,96 +154,19 @@ if(MSVC)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /WX-")
 endif(MSVC)
 
-add_test_suite(aes aes.cbc)
+file(GLOB test_suites RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" suites/*.data)
-add_test_suite(aes aes.cfb)
+list(APPEND test_suites ${base_generated_data_files})
-add_test_suite(aes aes.ecb)
+# If the generated .data files are present in the source tree, we just added
-add_test_suite(aes aes.ofb)
+# them twice, both through GLOB and through ${base_generated_data_files}.
-add_test_suite(aes aes.rest)
+list(REMOVE_DUPLICATES test_suites)
-add_test_suite(aes aes.xts)
+list(SORT test_suites)
-add_test_suite(aria)
+foreach(test_suite ${test_suites})
-add_test_suite(asn1parse)
+    get_filename_component(data_name ${test_suite} NAME)
-add_test_suite(asn1write)
+    string(REGEX REPLACE "\\.data\$" "" data_name "${data_name}")
-add_test_suite(base64)
+    string(REPLACE "test_suite_" "" data_name "${data_name}")
-add_test_suite(camellia)
+    string(REGEX MATCH "[^.]*" function_name "${data_name}")
-add_test_suite(ccm)
+    add_test_suite(${function_name} ${data_name})
-add_test_suite(chacha20)
+endforeach(test_suite)
-add_test_suite(chachapoly)
-add_test_suite(cipher cipher.aes)
-add_test_suite(cipher cipher.camellia)
-add_test_suite(cipher cipher.ccm)
-add_test_suite(cipher cipher.chacha20)
-add_test_suite(cipher cipher.chachapoly)
-add_test_suite(cipher cipher.des)
-add_test_suite(cipher cipher.gcm)
-add_test_suite(cipher cipher.misc)
-add_test_suite(cipher cipher.nist_kw)
-add_test_suite(cipher cipher.null)
-add_test_suite(cipher cipher.padding)
-add_test_suite(cmac)
-add_test_suite(ctr_drbg)
-add_test_suite(debug)
-add_test_suite(des)
-add_test_suite(dhm)
-add_test_suite(ecdh)
-add_test_suite(ecdsa)
-add_test_suite(ecjpake)
-add_test_suite(ecp)
-add_test_suite(entropy)
-add_test_suite(error)
-add_test_suite(gcm gcm.aes128_de)
-add_test_suite(gcm gcm.aes128_en)
-add_test_suite(gcm gcm.aes192_de)
-add_test_suite(gcm gcm.aes192_en)
-add_test_suite(gcm gcm.aes256_de)
-add_test_suite(gcm gcm.aes256_en)
-add_test_suite(gcm gcm.camellia)
-add_test_suite(gcm gcm.misc)
-add_test_suite(hkdf)
-add_test_suite(hmac_drbg hmac_drbg.misc)
-add_test_suite(hmac_drbg hmac_drbg.no_reseed)
-add_test_suite(hmac_drbg hmac_drbg.nopr)
-add_test_suite(hmac_drbg hmac_drbg.pr)
-add_test_suite(md)
-add_test_suite(mdx)
-add_test_suite(memory_buffer_alloc)
-add_test_suite(mpi)
-add_test_suite(mps)
-add_test_suite(net)
-add_test_suite(nist_kw)
-add_test_suite(oid)
-add_test_suite(pem)
-add_test_suite(pk)
-add_test_suite(pkcs1_v15)
-add_test_suite(pkcs1_v21)
-add_test_suite(pkcs5)
-add_test_suite(pkcs12)
-add_test_suite(pkparse)
-add_test_suite(pkwrite)
-add_test_suite(poly1305)
-add_test_suite(psa_crypto)
-add_test_suite(psa_crypto_attributes)
-add_test_suite(psa_crypto_entropy)
-add_test_suite(psa_crypto_hash)
-add_test_suite(psa_crypto_init)
-add_test_suite(psa_crypto_metadata)
-add_test_suite(psa_crypto_not_supported psa_crypto_not_supported.generated)
-add_test_suite(psa_crypto_not_supported psa_crypto_not_supported.misc)
-add_test_suite(psa_crypto_persistent_key)
-add_test_suite(psa_crypto_se_driver_hal)
-add_test_suite(psa_crypto_se_driver_hal_mocks)
-add_test_suite(psa_crypto_slot_management)
-add_test_suite(psa_crypto_storage_format psa_crypto_storage_format.misc)
-add_test_suite(psa_crypto_storage_format psa_crypto_storage_format.current)
-add_test_suite(psa_crypto_storage_format psa_crypto_storage_format.v0)
-add_test_suite(psa_its)
-add_test_suite(random)
-add_test_suite(rsa)
-add_test_suite(shax)
-add_test_suite(ssl)
-add_test_suite(timing)
-add_test_suite(version)
-add_test_suite(x509parse)
-add_test_suite(x509write)
 
 # Make scripts and data files needed for testing available in an
 # out-of-source build.

tests/scripts/all.sh

@@ -883,7 +883,6 @@ component_test_default_out_of_box () {
     msg "build: make, default config (out-of-box)" # ~1min
     make
     # Disable fancy stuff
-    SAVE_MBEDTLS_TEST_OUTCOME_FILE="$MBEDTLS_TEST_OUTCOME_FILE"
     unset MBEDTLS_TEST_OUTCOME_FILE
 
     msg "test: main suites make, default config (out-of-box)" # ~10s
@@ -891,9 +890,6 @@ component_test_default_out_of_box () {
 
     msg "selftest: make, default config (out-of-box)" # ~10s
     programs/test/selftest
-
-    export MBEDTLS_TEST_OUTCOME_FILE="$SAVE_MBEDTLS_TEST_OUTCOME_FILE"
-    unset SAVE_MBEDTLS_TEST_OUTCOME_FILE
 }
 
 component_test_default_cmake_gcc_asan () {
@@ -1582,10 +1578,7 @@ component_test_psa_crypto_config_accel_ecdsa () {
     loc_accel_flags="$loc_accel_flags $( echo "$loc_accel_list" | sed 's/[^ ]* */-DMBEDTLS_PSA_ACCEL_&/g' )"
     make CFLAGS="$ASAN_CFLAGS -O -Werror -I../tests/include -I../tests -DPSA_CRYPTO_DRIVER_TEST -DMBEDTLS_TEST_LIBTESTDRIVER1 $loc_accel_flags" LDFLAGS="-ltestdriver1 $ASAN_CFLAGS"
 
-    unset loc_accel_flags
+    not grep mbedtls_ecdsa_ library/ecdsa.o
-    unset loc_accel_list
-
-    if_build_succeeded not grep mbedtls_ecdsa_ library/ecdsa.o
 
     msg "test: MBEDTLS_PSA_CRYPTO_CONFIG with accelerated ECDSA"
     make test
@@ -1664,11 +1657,8 @@ component_test_psa_crypto_config_accel_rsa_signature () {
     loc_accel_flags="$loc_accel_flags $( echo "$loc_accel_list" | sed 's/[^ ]* */-DMBEDTLS_PSA_ACCEL_&/g' )"
     make CFLAGS="$ASAN_CFLAGS -Werror -I../tests/include -I../tests -DPSA_CRYPTO_DRIVER_TEST -DMBEDTLS_TEST_LIBTESTDRIVER1 $loc_accel_flags" LDFLAGS="-ltestdriver1 $ASAN_CFLAGS"
 
-    unset loc_accel_flags
+    not grep mbedtls_rsa_rsassa_pkcs1_v15_sign library/rsa.o
-    unset loc_accel_list
+    not grep mbedtls_rsa_rsassa_pss_sign_ext library/rsa.o
-
-    if_build_succeeded not grep mbedtls_rsa_rsassa_pkcs1_v15_sign library/rsa.o
-    if_build_succeeded not grep mbedtls_rsa_rsassa_pss_sign_ext library/rsa.o
 
     msg "test: MBEDTLS_PSA_CRYPTO_CONFIG with accelerated RSA signature"
     make test
@@ -1703,11 +1693,8 @@ component_test_psa_crypto_config_accel_hash () {
     loc_accel_flags="$loc_accel_flags $( echo "$loc_accel_list" | sed 's/[^ ]* */-DMBEDTLS_PSA_ACCEL_&/g' )"
     make CFLAGS="$ASAN_CFLAGS -Werror -I../tests/include -I../tests -DPSA_CRYPTO_DRIVER_TEST -DMBEDTLS_TEST_LIBTESTDRIVER1 $loc_accel_flags" LDFLAGS="-ltestdriver1 $ASAN_CFLAGS"
 
-    unset loc_accel_flags
+    not grep mbedtls_sha512_init library/sha512.o
-    unset loc_accel_list
+    not grep mbedtls_sha1_init library/sha1.o
-
-    if_build_succeeded not grep mbedtls_sha512_init library/sha512.o
-    if_build_succeeded not grep mbedtls_sha1_init library/sha1.o
 
     msg "test: MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash"
     make test
@@ -1743,10 +1730,7 @@ component_test_psa_crypto_config_accel_cipher () {
     loc_accel_flags="$loc_accel_flags $( echo "$loc_accel_list" | sed 's/[^ ]* */-DMBEDTLS_PSA_ACCEL_&/g' )"
     make CFLAGS="$ASAN_CFLAGS -Werror -I../tests/include -I../tests -DPSA_CRYPTO_DRIVER_TEST -DMBEDTLS_TEST_LIBTESTDRIVER1 $loc_accel_flags" LDFLAGS="-ltestdriver1 $ASAN_CFLAGS"
 
-    unset loc_accel_flags
+    not grep mbedtls_des* library/des.o
-    unset loc_accel_list
-
-    if_build_succeeded not grep mbedtls_des* library/des.o
 
     msg "test: MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash"
     make test
@@ -2426,7 +2410,6 @@ component_test_psa_crypto_drivers () {
     loc_cflags="${loc_cflags} -I../tests/include -O2"
 
     make CC=gcc CFLAGS="${loc_cflags}" LDFLAGS="$ASAN_CFLAGS"
-    unset loc_cflags
 
     msg "test: full + MBEDTLS_PSA_CRYPTO_DRIVERS"
     make test
@@ -2732,7 +2715,7 @@ component_test_tls13 () {
     msg "test: default config with MBEDTLS_SSL_PROTO_TLS1_3 enabled, without padding"
     make test
     msg "ssl-opt.sh (TLS 1.3)"
-    if_build_succeeded tests/ssl-opt.sh
+    tests/ssl-opt.sh
 }
 
 component_test_tls13_no_compatibility_mode () {
@@ -2745,7 +2728,7 @@ component_test_tls13_no_compatibility_mode () {
     msg "test: default config with MBEDTLS_SSL_PROTO_TLS1_3 enabled, without padding"
     make test
     msg "ssl-opt.sh (TLS 1.3 no compatibility mode)"
-    if_build_succeeded tests/ssl-opt.sh
+    tests/ssl-opt.sh
 }
 
 component_test_tls13_with_padding () {
@@ -2758,7 +2741,7 @@ component_test_tls13_with_padding () {
     msg "test: default config with MBEDTLS_SSL_PROTO_TLS1_3 enabled, with padding"
     make test
     msg "ssl-opt.sh (TLS 1.3 with padding)"
-    if_build_succeeded tests/ssl-opt.sh
+    tests/ssl-opt.sh
 }
 
 component_test_tls13_with_ecp_restartable () {
@@ -2771,7 +2754,7 @@ component_test_tls13_with_ecp_restartable () {
     msg "test: default config with MBEDTLS_SSL_PROTO_TLS1_3 enabled, with ecp_restartable"
     make test
     msg "ssl-opt.sh (TLS 1.3 with ecp_restartable)"
-    if_build_succeeded tests/ssl-opt.sh
+    tests/ssl-opt.sh
 }
 
 component_test_tls13_with_everest () {
@@ -2785,7 +2768,7 @@ component_test_tls13_with_everest () {
     msg "test: default config with MBEDTLS_SSL_PROTO_TLS1_3 enabled, with Everest"
     make test
     msg "ssl-opt.sh (TLS 1.3 with everest)"
-    if_build_succeeded tests/ssl-opt.sh
+    tests/ssl-opt.sh
 }
 
 component_build_mingw () {
@@ -2910,41 +2893,35 @@ component_test_cmake_out_of_source () {
 
 component_test_cmake_as_subdirectory () {
     msg "build: cmake 'as-subdirectory' build"
-    MBEDTLS_ROOT_DIR="$PWD"
-
     cd programs/test/cmake_subproject
     cmake .
     make
     ./cmake_subproject
-
+}
-    cd "$MBEDTLS_ROOT_DIR"
+support_test_cmake_as_subdirectory () {
-    unset MBEDTLS_ROOT_DIR
+    support_test_cmake_out_of_source
 }
 
 component_test_cmake_as_package () {
     msg "build: cmake 'as-package' build"
-    MBEDTLS_ROOT_DIR="$PWD"
-
     cd programs/test/cmake_package
     cmake .
     make
     ./cmake_package
-
+}
-    cd "$MBEDTLS_ROOT_DIR"
+support_test_cmake_as_package () {
-    unset MBEDTLS_ROOT_DIR
+    support_test_cmake_out_of_source
 }
 
 component_test_cmake_as_package_install () {
     msg "build: cmake 'as-installed-package' build"
-    MBEDTLS_ROOT_DIR="$PWD"
-
     cd programs/test/cmake_package_install
     cmake .
     make
     ./cmake_package_install
-
+}
-    cd "$MBEDTLS_ROOT_DIR"
+support_test_cmake_as_package_install () {
-    unset MBEDTLS_ROOT_DIR
+    support_test_cmake_out_of_source
 }
 
 component_test_zeroize () {
@@ -2973,8 +2950,6 @@ component_test_zeroize () {
             make clean
         done
     done
-
-    unset gdb_disable_aslr
 }
 
 component_test_psa_compliance () {