diff --git a/tests/scripts/components-sanitizers.sh b/tests/scripts/components-sanitizers.sh index 03cd50453c..5b79d2b778 100644 --- a/tests/scripts/components-sanitizers.sh +++ b/tests/scripts/components-sanitizers.sh @@ -8,3 +8,201 @@ ################################################################ #### Sanitizer Testing ################################################################ + +skip_suites_without_constant_flow () { + # Skip the test suites that don't have any constant-flow annotations. + # This will need to be adjusted if we ever start declaring things as + # secret from macros or functions inside tests/include or tests/src. + SKIP_TEST_SUITES=$( + git -C tests/suites grep -L TEST_CF_ 'test_suite_*.function' | + sed 's/test_suite_//; s/\.function$//' | + tr '\n' ,) + export SKIP_TEST_SUITES +} + +skip_all_except_given_suite () { + # Skip all but the given test suite + SKIP_TEST_SUITES=$( + ls -1 tests/suites/test_suite_*.function | + grep -v $1.function | + sed 's/tests.suites.test_suite_//; s/\.function$//' | + tr '\n' ,) + export SKIP_TEST_SUITES +} + +component_test_memsan_constant_flow () { + # This tests both (1) accesses to undefined memory, and (2) branches or + # memory access depending on secret values. To distinguish between those: + # - unset MBEDTLS_TEST_CONSTANT_FLOW_MEMSAN - does the failure persist? + # - or alternatively, change the build type to MemSanDbg, which enables + # origin tracking and nicer stack traces (which are useful for debugging + # anyway), and check if the origin was TEST_CF_SECRET() or something else. + msg "build: cmake MSan (clang), full config minus MBEDTLS_USE_PSA_CRYPTO with constant flow testing" + scripts/config.py full + scripts/config.py set MBEDTLS_TEST_CONSTANT_FLOW_MEMSAN + scripts/config.py unset MBEDTLS_USE_PSA_CRYPTO + scripts/config.py unset MBEDTLS_AESNI_C # memsan doesn't grok asm + CC=clang cmake -D CMAKE_BUILD_TYPE:String=MemSan . + make + + msg "test: main suites (full minus MBEDTLS_USE_PSA_CRYPTO, Msan + constant flow)" + make test +} + +component_test_memsan_constant_flow_psa () { + # This tests both (1) accesses to undefined memory, and (2) branches or + # memory access depending on secret values. To distinguish between those: + # - unset MBEDTLS_TEST_CONSTANT_FLOW_MEMSAN - does the failure persist? + # - or alternatively, change the build type to MemSanDbg, which enables + # origin tracking and nicer stack traces (which are useful for debugging + # anyway), and check if the origin was TEST_CF_SECRET() or something else. + msg "build: cmake MSan (clang), full config with constant flow testing" + scripts/config.py full + scripts/config.py set MBEDTLS_TEST_CONSTANT_FLOW_MEMSAN + scripts/config.py unset MBEDTLS_AESNI_C # memsan doesn't grok asm + CC=clang cmake -D CMAKE_BUILD_TYPE:String=MemSan . + make + + msg "test: main suites (Msan + constant flow)" + make test +} + +component_release_test_valgrind_constant_flow () { + # This tests both (1) everything that valgrind's memcheck usually checks + # (heap buffer overflows, use of uninitialized memory, use-after-free, + # etc.) and (2) branches or memory access depending on secret values, + # which will be reported as uninitialized memory. To distinguish between + # secret and actually uninitialized: + # - unset MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND - does the failure persist? + # - or alternatively, build with debug info and manually run the offending + # test suite with valgrind --track-origins=yes, then check if the origin + # was TEST_CF_SECRET() or something else. + msg "build: cmake release GCC, full config minus MBEDTLS_USE_PSA_CRYPTO with constant flow testing" + scripts/config.py full + scripts/config.py set MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND + scripts/config.py unset MBEDTLS_USE_PSA_CRYPTO + skip_suites_without_constant_flow + cmake -D CMAKE_BUILD_TYPE:String=Release . + make + + # this only shows a summary of the results (how many of each type) + # details are left in Testing//DynamicAnalysis.xml + msg "test: some suites (full minus MBEDTLS_USE_PSA_CRYPTO, valgrind + constant flow)" + make memcheck + + # Test asm path in constant time module - by default, it will test the plain C + # path under Valgrind or Memsan. Running only the constant_time tests is fast (<1s) + msg "test: valgrind asm constant_time" + scripts/config.py --force set MBEDTLS_TEST_CONSTANT_FLOW_ASM + skip_all_except_given_suite test_suite_constant_time + cmake -D CMAKE_BUILD_TYPE:String=Release . + make clean + make + make memcheck +} + +component_release_test_valgrind_constant_flow_psa () { + # This tests both (1) everything that valgrind's memcheck usually checks + # (heap buffer overflows, use of uninitialized memory, use-after-free, + # etc.) and (2) branches or memory access depending on secret values, + # which will be reported as uninitialized memory. To distinguish between + # secret and actually uninitialized: + # - unset MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND - does the failure persist? + # - or alternatively, build with debug info and manually run the offending + # test suite with valgrind --track-origins=yes, then check if the origin + # was TEST_CF_SECRET() or something else. + msg "build: cmake release GCC, full config with constant flow testing" + scripts/config.py full + scripts/config.py set MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND + skip_suites_without_constant_flow + cmake -D CMAKE_BUILD_TYPE:String=Release . + make + + # this only shows a summary of the results (how many of each type) + # details are left in Testing//DynamicAnalysis.xml + msg "test: some suites (valgrind + constant flow)" + make memcheck +} + +component_test_tsan () { + msg "build: TSan (clang)" + scripts/config.py full + scripts/config.py set MBEDTLS_THREADING_C + scripts/config.py set MBEDTLS_THREADING_PTHREAD + # Self-tests do not currently use multiple threads. + scripts/config.py unset MBEDTLS_SELF_TEST + + # The deprecated MBEDTLS_PSA_CRYPTO_SE_C interface is not thread safe. + scripts/config.py unset MBEDTLS_PSA_CRYPTO_SE_C + + CC=clang cmake -D CMAKE_BUILD_TYPE:String=TSan . + make + + msg "test: main suites (TSan)" + make test +} + +component_test_memsan () { + msg "build: MSan (clang)" # ~ 1 min 20s + scripts/config.py unset MBEDTLS_AESNI_C # memsan doesn't grok asm + CC=clang cmake -D CMAKE_BUILD_TYPE:String=MemSan . + make + + msg "test: main suites (MSan)" # ~ 10s + make test + + msg "test: metatests (MSan)" + tests/scripts/run-metatests.sh any msan + + msg "program demos (MSan)" # ~20s + tests/scripts/run_demos.py + + msg "test: ssl-opt.sh (MSan)" # ~ 1 min + tests/ssl-opt.sh + + # Optional part(s) + + if [ "$MEMORY" -gt 0 ]; then + msg "test: compat.sh (MSan)" # ~ 6 min 20s + tests/compat.sh + fi +} + +component_release_test_valgrind () { + msg "build: Release (clang)" + # default config, in particular without MBEDTLS_USE_PSA_CRYPTO + CC=clang cmake -D CMAKE_BUILD_TYPE:String=Release . + make + + msg "test: main suites, Valgrind (default config)" + make memcheck + + # Optional parts (slow; currently broken on OS X because programs don't + # seem to receive signals under valgrind on OS X). + # These optional parts don't run on the CI. + if [ "$MEMORY" -gt 0 ]; then + msg "test: ssl-opt.sh --memcheck (default config)" + tests/ssl-opt.sh --memcheck + fi + + if [ "$MEMORY" -gt 1 ]; then + msg "test: compat.sh --memcheck (default config)" + tests/compat.sh --memcheck + fi + + if [ "$MEMORY" -gt 0 ]; then + msg "test: context-info.sh --memcheck (default config)" + tests/context-info.sh --memcheck + fi +} + +component_release_test_valgrind_psa () { + msg "build: Release, full (clang)" + # full config, in particular with MBEDTLS_USE_PSA_CRYPTO + scripts/config.py full + CC=clang cmake -D CMAKE_BUILD_TYPE:String=Release . + make + + msg "test: main suites, Valgrind (full config)" + make memcheck +} diff --git a/tests/scripts/components.sh b/tests/scripts/components.sh index 7ddddbe30b..2e14866c08 100644 --- a/tests/scripts/components.sh +++ b/tests/scripts/components.sh @@ -359,139 +359,6 @@ component_test_tls1_2_default_cbc_legacy_cbc_etm_cipher_only_use_psa () { tests/ssl-opt.sh -f "TLS 1.2" } -skip_suites_without_constant_flow () { - # Skip the test suites that don't have any constant-flow annotations. - # This will need to be adjusted if we ever start declaring things as - # secret from macros or functions inside tests/include or tests/src. - SKIP_TEST_SUITES=$( - git -C tests/suites grep -L TEST_CF_ 'test_suite_*.function' | - sed 's/test_suite_//; s/\.function$//' | - tr '\n' ,) - export SKIP_TEST_SUITES -} - -skip_all_except_given_suite () { - # Skip all but the given test suite - SKIP_TEST_SUITES=$( - ls -1 tests/suites/test_suite_*.function | - grep -v $1.function | - sed 's/tests.suites.test_suite_//; s/\.function$//' | - tr '\n' ,) - export SKIP_TEST_SUITES -} - -component_test_memsan_constant_flow () { - # This tests both (1) accesses to undefined memory, and (2) branches or - # memory access depending on secret values. To distinguish between those: - # - unset MBEDTLS_TEST_CONSTANT_FLOW_MEMSAN - does the failure persist? - # - or alternatively, change the build type to MemSanDbg, which enables - # origin tracking and nicer stack traces (which are useful for debugging - # anyway), and check if the origin was TEST_CF_SECRET() or something else. - msg "build: cmake MSan (clang), full config minus MBEDTLS_USE_PSA_CRYPTO with constant flow testing" - scripts/config.py full - scripts/config.py set MBEDTLS_TEST_CONSTANT_FLOW_MEMSAN - scripts/config.py unset MBEDTLS_USE_PSA_CRYPTO - scripts/config.py unset MBEDTLS_AESNI_C # memsan doesn't grok asm - CC=clang cmake -D CMAKE_BUILD_TYPE:String=MemSan . - make - - msg "test: main suites (full minus MBEDTLS_USE_PSA_CRYPTO, Msan + constant flow)" - make test -} - -component_test_memsan_constant_flow_psa () { - # This tests both (1) accesses to undefined memory, and (2) branches or - # memory access depending on secret values. To distinguish between those: - # - unset MBEDTLS_TEST_CONSTANT_FLOW_MEMSAN - does the failure persist? - # - or alternatively, change the build type to MemSanDbg, which enables - # origin tracking and nicer stack traces (which are useful for debugging - # anyway), and check if the origin was TEST_CF_SECRET() or something else. - msg "build: cmake MSan (clang), full config with constant flow testing" - scripts/config.py full - scripts/config.py set MBEDTLS_TEST_CONSTANT_FLOW_MEMSAN - scripts/config.py unset MBEDTLS_AESNI_C # memsan doesn't grok asm - CC=clang cmake -D CMAKE_BUILD_TYPE:String=MemSan . - make - - msg "test: main suites (Msan + constant flow)" - make test -} - -component_release_test_valgrind_constant_flow () { - # This tests both (1) everything that valgrind's memcheck usually checks - # (heap buffer overflows, use of uninitialized memory, use-after-free, - # etc.) and (2) branches or memory access depending on secret values, - # which will be reported as uninitialized memory. To distinguish between - # secret and actually uninitialized: - # - unset MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND - does the failure persist? - # - or alternatively, build with debug info and manually run the offending - # test suite with valgrind --track-origins=yes, then check if the origin - # was TEST_CF_SECRET() or something else. - msg "build: cmake release GCC, full config minus MBEDTLS_USE_PSA_CRYPTO with constant flow testing" - scripts/config.py full - scripts/config.py set MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND - scripts/config.py unset MBEDTLS_USE_PSA_CRYPTO - skip_suites_without_constant_flow - cmake -D CMAKE_BUILD_TYPE:String=Release . - make - - # this only shows a summary of the results (how many of each type) - # details are left in Testing//DynamicAnalysis.xml - msg "test: some suites (full minus MBEDTLS_USE_PSA_CRYPTO, valgrind + constant flow)" - make memcheck - - # Test asm path in constant time module - by default, it will test the plain C - # path under Valgrind or Memsan. Running only the constant_time tests is fast (<1s) - msg "test: valgrind asm constant_time" - scripts/config.py --force set MBEDTLS_TEST_CONSTANT_FLOW_ASM - skip_all_except_given_suite test_suite_constant_time - cmake -D CMAKE_BUILD_TYPE:String=Release . - make clean - make - make memcheck -} - -component_release_test_valgrind_constant_flow_psa () { - # This tests both (1) everything that valgrind's memcheck usually checks - # (heap buffer overflows, use of uninitialized memory, use-after-free, - # etc.) and (2) branches or memory access depending on secret values, - # which will be reported as uninitialized memory. To distinguish between - # secret and actually uninitialized: - # - unset MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND - does the failure persist? - # - or alternatively, build with debug info and manually run the offending - # test suite with valgrind --track-origins=yes, then check if the origin - # was TEST_CF_SECRET() or something else. - msg "build: cmake release GCC, full config with constant flow testing" - scripts/config.py full - scripts/config.py set MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND - skip_suites_without_constant_flow - cmake -D CMAKE_BUILD_TYPE:String=Release . - make - - # this only shows a summary of the results (how many of each type) - # details are left in Testing//DynamicAnalysis.xml - msg "test: some suites (valgrind + constant flow)" - make memcheck -} - -component_test_tsan () { - msg "build: TSan (clang)" - scripts/config.py full - scripts/config.py set MBEDTLS_THREADING_C - scripts/config.py set MBEDTLS_THREADING_PTHREAD - # Self-tests do not currently use multiple threads. - scripts/config.py unset MBEDTLS_SELF_TEST - - # The deprecated MBEDTLS_PSA_CRYPTO_SE_C interface is not thread safe. - scripts/config.py unset MBEDTLS_PSA_CRYPTO_SE_C - - CC=clang cmake -D CMAKE_BUILD_TYPE:String=TSan . - make - - msg "test: main suites (TSan)" - make test -} - component_build_dhm_alt () { msg "build: MBEDTLS_DHM_ALT" # ~30s scripts/config.py full @@ -788,69 +655,3 @@ support_build_aes_via_padlock_only () { [ "`dpkg --print-foreign-architectures`" == "i386" ] } -component_test_memsan () { - msg "build: MSan (clang)" # ~ 1 min 20s - scripts/config.py unset MBEDTLS_AESNI_C # memsan doesn't grok asm - CC=clang cmake -D CMAKE_BUILD_TYPE:String=MemSan . - make - - msg "test: main suites (MSan)" # ~ 10s - make test - - msg "test: metatests (MSan)" - tests/scripts/run-metatests.sh any msan - - msg "program demos (MSan)" # ~20s - tests/scripts/run_demos.py - - msg "test: ssl-opt.sh (MSan)" # ~ 1 min - tests/ssl-opt.sh - - # Optional part(s) - - if [ "$MEMORY" -gt 0 ]; then - msg "test: compat.sh (MSan)" # ~ 6 min 20s - tests/compat.sh - fi -} - -component_release_test_valgrind () { - msg "build: Release (clang)" - # default config, in particular without MBEDTLS_USE_PSA_CRYPTO - CC=clang cmake -D CMAKE_BUILD_TYPE:String=Release . - make - - msg "test: main suites, Valgrind (default config)" - make memcheck - - # Optional parts (slow; currently broken on OS X because programs don't - # seem to receive signals under valgrind on OS X). - # These optional parts don't run on the CI. - if [ "$MEMORY" -gt 0 ]; then - msg "test: ssl-opt.sh --memcheck (default config)" - tests/ssl-opt.sh --memcheck - fi - - if [ "$MEMORY" -gt 1 ]; then - msg "test: compat.sh --memcheck (default config)" - tests/compat.sh --memcheck - fi - - if [ "$MEMORY" -gt 0 ]; then - msg "test: context-info.sh --memcheck (default config)" - tests/context-info.sh --memcheck - fi -} - -component_release_test_valgrind_psa () { - msg "build: Release, full (clang)" - # full config, in particular with MBEDTLS_USE_PSA_CRYPTO - scripts/config.py full - CC=clang cmake -D CMAKE_BUILD_TYPE:String=Release . - make - - msg "test: main suites, Valgrind (full config)" - make memcheck -} - -