diff --git a/programs/.gitignore b/programs/.gitignore index a641c31c45..e0c49873ee 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -38,6 +38,7 @@ psa/crypto_examples psa/hmac_demo psa/key_ladder_demo psa/psa_constant_names +psa/psa_hash random/gen_entropy random/gen_random_ctr_drbg ssl/dtls_client @@ -56,6 +57,7 @@ test/cpp_dummy_build test/cpp_dummy_build.cpp test/dlopen test/ecp-bench +test/metatest test/query_compile_time_config test/query_included_headers test/selftest diff --git a/programs/Makefile b/programs/Makefile index 116883b836..a3fa81679f 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -123,6 +123,7 @@ APPS = \ ssl/ssl_server \ ssl/ssl_server2 \ test/benchmark \ + test/metatest \ test/query_compile_time_config \ test/query_included_headers \ test/selftest \ @@ -413,6 +414,10 @@ test/dlopen$(EXEXT): test/dlopen.c $(DEP) $(CC) $(LOCAL_CFLAGS) $(CFLAGS) test/dlopen.c $(LDFLAGS) $(DLOPEN_LDFLAGS) -o $@ endif +test/metatest$(EXEXT): test/metatest.c $(DEP) + echo " CC test/metatest.c" + $(CC) $(LOCAL_CFLAGS) $(CFLAGS) -I ../library test/metatest.c $(LOCAL_LDFLAGS) $(LDFLAGS) -o $@ + test/query_config.o: test/query_config.c test/query_config.h $(DEP) echo " CC test/query_config.c" $(CC) $(LOCAL_CFLAGS) $(CFLAGS) -c test/query_config.c -o $@ diff --git a/programs/test/CMakeLists.txt b/programs/test/CMakeLists.txt index a75f8d9239..0778731125 100644 --- a/programs/test/CMakeLists.txt +++ b/programs/test/CMakeLists.txt @@ -3,6 +3,7 @@ set(libs ) set(executables_libs + metatest query_included_headers selftest udp_proxy @@ -72,6 +73,7 @@ foreach(exe IN LISTS executables_libs executables_mbedcrypto) add_executable(${exe} ${exe}.c $ ${extra_sources}) target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../tests/include) + target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../library) if(exe STREQUAL "query_compile_time_config") target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) endif() diff --git a/programs/test/metatest.c b/programs/test/metatest.c new file mode 100644 index 0000000000..2973cce3fa --- /dev/null +++ b/programs/test/metatest.c @@ -0,0 +1,351 @@ +/** \file metatest.c + * + * \brief Test features of the test framework. + * + * When you run this program, it runs a single "meta-test". A meta-test + * performs an operation which should be caught as a failure by our + * test framework. The meta-test passes if this program calls `exit` with + * a nonzero status, or aborts, or is terminated by a signal, or if the + * framework running the program considers the run an error (this happens + * with Valgrind for a memory leak). The non-success of the meta-test + * program means that the test failure has been caught correctly. + * + * Some failures are purely functional: the logic of the code causes the + * test result to be set to FAIL. Other failures come from extra + * instrumentation which is not present in a normal build; for example, + * Asan or Valgrind to detect memory leaks. This is reflected by the + * "platform" associated with each meta-test. + * + * Use the companion script `tests/scripts/run-metatests.sh` to run all + * the meta-tests for a given platform and validate that they trigger a + * detected failure as expected. + */ + +/* + * Copyright The Mbed TLS Contributors + * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + */ + +#define MBEDTLS_ALLOW_PRIVATE_ACCESS + +#include +#include +#include "test/helpers.h" +#include "test/macros.h" + +#include +#include + +#if defined(MBEDTLS_THREADING_C) +#include +#endif + + +/* This is an external variable, so the compiler doesn't know that we're never + * changing its value. + */ +volatile int false_but_the_compiler_does_not_know = 0; + +/* Set n bytes at the address p to all-bits-zero, in such a way that + * the compiler should not know that p is all-bits-zero. */ +static void set_to_zero_but_the_compiler_does_not_know(volatile void *p, size_t n) +{ + memset((void *) p, false_but_the_compiler_does_not_know, n); +} + + +/****************************************************************/ +/* Test framework features */ +/****************************************************************/ + +void meta_test_fail(const char *name) +{ + (void) name; + mbedtls_test_fail("Forced test failure", __LINE__, __FILE__); +} + + +/****************************************************************/ +/* Platform features */ +/****************************************************************/ + +void null_pointer_dereference(const char *name) +{ + (void) name; + volatile char *volatile p; + set_to_zero_but_the_compiler_does_not_know(&p, sizeof(p)); + /* Undefined behavior (read from null data pointer) */ + mbedtls_printf("%p -> %u\n", p, (unsigned) *p); +} + +void null_pointer_call(const char *name) +{ + (void) name; + unsigned(*volatile p)(void); + set_to_zero_but_the_compiler_does_not_know(&p, sizeof(p)); + /* Undefined behavior (execute null function pointer) */ + /* The pointer representation may be truncated, but we don't care: + * the only point of printing it is to have some use of the pointer + * to dissuade the compiler from optimizing it away. */ + mbedtls_printf("%lx() -> %u\n", (unsigned long) (uintptr_t) p, p()); +} + + +/****************************************************************/ +/* Memory */ +/****************************************************************/ + +void read_after_free(const char *name) +{ + (void) name; + volatile char *p = mbedtls_calloc(1, 1); + *p = 'a'; + mbedtls_free((void *) p); + /* Undefined behavior (read after free) */ + mbedtls_printf("%u\n", (unsigned) *p); +} + +void double_free(const char *name) +{ + (void) name; + volatile char *p = mbedtls_calloc(1, 1); + *p = 'a'; + mbedtls_free((void *) p); + /* Undefined behavior (double free) */ + mbedtls_free((void *) p); +} + +void read_uninitialized_stack(const char *name) +{ + (void) name; + char buf[1]; + if (false_but_the_compiler_does_not_know) { + buf[0] = '!'; + } + char *volatile p = buf; + if (*p != 0) { + /* Unspecified result (read from uninitialized memory) */ + mbedtls_printf("%u\n", (unsigned) *p); + } +} + +void memory_leak(const char *name) +{ + (void) name; + volatile char *p = mbedtls_calloc(1, 1); + mbedtls_printf("%u\n", (unsigned) *p); + /* Leak of a heap object */ +} + + +/****************************************************************/ +/* Threading */ +/****************************************************************/ + +void mutex_lock_not_initialized(const char *name) +{ + (void) name; +#if defined(MBEDTLS_THREADING_C) + mbedtls_threading_mutex_t mutex; + memset(&mutex, 0, sizeof(mutex)); + /* This mutex usage error is detected by our test framework's mutex usage + * verification framework. See tests/src/threading_helpers.c. Other + * threading implementations (e.g. pthread without our instrumentation) + * might consider this normal usage. */ + TEST_ASSERT(mbedtls_mutex_lock(&mutex) == 0); +exit: + ; +#endif +} + +void mutex_unlock_not_initialized(const char *name) +{ + (void) name; +#if defined(MBEDTLS_THREADING_C) + mbedtls_threading_mutex_t mutex; + memset(&mutex, 0, sizeof(mutex)); + /* This mutex usage error is detected by our test framework's mutex usage + * verification framework. See tests/src/threading_helpers.c. Other + * threading implementations (e.g. pthread without our instrumentation) + * might consider this normal usage. */ + TEST_ASSERT(mbedtls_mutex_unlock(&mutex) == 0); +exit: + ; +#endif +} + +void mutex_free_not_initialized(const char *name) +{ + (void) name; +#if defined(MBEDTLS_THREADING_C) + mbedtls_threading_mutex_t mutex; + memset(&mutex, 0, sizeof(mutex)); + /* This mutex usage error is detected by our test framework's mutex usage + * verification framework. See tests/src/threading_helpers.c. Other + * threading implementations (e.g. pthread without our instrumentation) + * might consider this normal usage. */ + mbedtls_mutex_free(&mutex); +#endif +} + +void mutex_double_init(const char *name) +{ + (void) name; +#if defined(MBEDTLS_THREADING_C) + mbedtls_threading_mutex_t mutex; + mbedtls_mutex_init(&mutex); + /* This mutex usage error is detected by our test framework's mutex usage + * verification framework. See tests/src/threading_helpers.c. Other + * threading implementations (e.g. pthread without our instrumentation) + * might consider this normal usage. */ + mbedtls_mutex_init(&mutex); + mbedtls_mutex_free(&mutex); +#endif +} + +void mutex_double_free(const char *name) +{ + (void) name; +#if defined(MBEDTLS_THREADING_C) + mbedtls_threading_mutex_t mutex; + mbedtls_mutex_init(&mutex); + mbedtls_mutex_free(&mutex); + /* This mutex usage error is detected by our test framework's mutex usage + * verification framework. See tests/src/threading_helpers.c. Other + * threading implementations (e.g. pthread without our instrumentation) + * might consider this normal usage. */ + mbedtls_mutex_free(&mutex); +#endif +} + +void mutex_leak(const char *name) +{ + (void) name; +#if defined(MBEDTLS_THREADING_C) + mbedtls_threading_mutex_t mutex; + mbedtls_mutex_init(&mutex); +#endif + /* This mutex usage error is detected by our test framework's mutex usage + * verification framework. See tests/src/threading_helpers.c. Other + * threading implementations (e.g. pthread without our instrumentation) + * might consider this normal usage. */ +} + + +/****************************************************************/ +/* Command line entry point */ +/****************************************************************/ + +typedef struct { + /** Command line argument that will trigger that metatest. + * + * Conventionally matches "[a-z0-9_]+". */ + const char *name; + + /** Platform under which that metatest is valid. + * + * - "any": should work anywhere. + * - "asan": triggers ASan (Address Sanitizer). + * - "msan": triggers MSan (Memory Sanitizer). + * - "pthread": requires MBEDTLS_THREADING_PTHREAD and MBEDTLS_TEST_HOOKS, + * which enables MBEDTLS_TEST_MUTEX_USAGE internally in the test + * framework (see tests/src/threading_helpers.c). + */ + const char *platform; + + /** Function that performs the metatest. + * + * The function receives the name as an argument. This allows using the + * same function to perform multiple variants of a test based on the name. + * + * When executed on a conforming platform, the function is expected to + * either cause a test failure (mbedtls_test_fail()), or cause the + * program to abort in some way (e.g. by causing a segfault or by + * triggering a sanitizer). + * + * When executed on a non-conforming platform, the function may return + * normally or may have unpredictable behavior. + */ + void (*entry_point)(const char *name); +} metatest_t; + +/* The list of availble meta-tests. Remember to register new functions here! + * + * Note that we always compile all the functions, so that `metatest --list` + * will always list all the available meta-tests. + * + * See the documentation of metatest_t::platform for the meaning of + * platform values. + */ +metatest_t metatests[] = { + { "test_fail", "any", meta_test_fail }, + { "null_dereference", "any", null_pointer_dereference }, + { "null_call", "any", null_pointer_call }, + { "read_after_free", "asan", read_after_free }, + { "double_free", "asan", double_free }, + { "read_uninitialized_stack", "msan", read_uninitialized_stack }, + { "memory_leak", "asan", memory_leak }, + { "mutex_lock_not_initialized", "pthread", mutex_lock_not_initialized }, + { "mutex_unlock_not_initialized", "pthread", mutex_unlock_not_initialized }, + { "mutex_free_not_initialized", "pthread", mutex_free_not_initialized }, + { "mutex_double_init", "pthread", mutex_double_init }, + { "mutex_double_free", "pthread", mutex_double_free }, + { "mutex_leak", "pthread", mutex_leak }, + { NULL, NULL, NULL } +}; + +static void help(FILE *out, const char *argv0) +{ + mbedtls_fprintf(out, "Usage: %s list|TEST\n", argv0); + mbedtls_fprintf(out, "Run a meta-test that should cause a test failure.\n"); + mbedtls_fprintf(out, "With 'list', list the available tests and their platform requirement.\n"); +} + +int main(int argc, char *argv[]) +{ + const char *argv0 = argc > 0 ? argv[0] : "metatest"; + if (argc != 2) { + help(stderr, argv0); + mbedtls_exit(MBEDTLS_EXIT_FAILURE); + } + + /* Support "-help", "--help", "--list", etc. */ + const char *command = argv[1]; + while (*command == '-') { + ++command; + } + + if (strcmp(argv[1], "help") == 0) { + help(stdout, argv0); + mbedtls_exit(MBEDTLS_EXIT_SUCCESS); + } + if (strcmp(argv[1], "list") == 0) { + for (const metatest_t *p = metatests; p->name != NULL; p++) { + mbedtls_printf("%s %s\n", p->name, p->platform); + } + mbedtls_exit(MBEDTLS_EXIT_SUCCESS); + } + +#if defined(MBEDTLS_TEST_MUTEX_USAGE) + mbedtls_test_mutex_usage_init(); +#endif + + for (const metatest_t *p = metatests; p->name != NULL; p++) { + if (strcmp(argv[1], p->name) == 0) { + mbedtls_printf("Running metatest %s...\n", argv[1]); + p->entry_point(argv[1]); +#if defined(MBEDTLS_TEST_MUTEX_USAGE) + mbedtls_test_mutex_usage_check(); +#endif + mbedtls_printf("Running metatest %s... done, result=%d\n", + argv[1], (int) mbedtls_test_info.result); + mbedtls_exit(mbedtls_test_info.result == MBEDTLS_TEST_RESULT_SUCCESS ? + MBEDTLS_EXIT_SUCCESS : + MBEDTLS_EXIT_FAILURE); + } + } + + mbedtls_fprintf(stderr, "%s: FATAL: No such metatest: %s\n", + argv0, command); + mbedtls_exit(MBEDTLS_EXIT_FAILURE); +} diff --git a/scripts/generate_visualc_files.pl b/scripts/generate_visualc_files.pl index 7f5609820b..96ade2fa76 100755 --- a/scripts/generate_visualc_files.pl +++ b/scripts/generate_visualc_files.pl @@ -144,6 +144,7 @@ sub gen_app { my $guid = gen_app_guid( $path ); $path =~ s!/!\\!g; (my $appname = $path) =~ s/.*\\//; + my $is_test_app = ($path =~ m/^test\\/); my $srcs = ""; if( $appname eq "ssl_client2" or $appname eq "ssl_server2" or @@ -158,7 +159,9 @@ sub gen_app { $content =~ s//$srcs/g; $content =~ s//$appname/g; $content =~ s//$guid/g; - $content =~ s/INCLUDE_DIRECTORIES\n/$include_directories/g; + $content =~ s/INCLUDE_DIRECTORIES\n/($is_test_app ? + $library_include_directories : + $include_directories)/ge; content_to_file( $content, "$dir/$appname.$ext" ); } diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index ed6a0b7265..b3bb3f1594 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -1120,6 +1120,9 @@ component_test_default_cmake_gcc_asan () { msg "test: selftest (ASan build)" # ~ 10s programs/test/selftest + msg "test: metatests (GCC, ASan build)" + tests/scripts/run-metatests.sh any asan + msg "test: ssl-opt.sh (ASan build)" # ~ 1 min tests/ssl-opt.sh @@ -1885,6 +1888,9 @@ component_test_everest () { msg "test: Everest ECDH context - main suites (inc. selftests) (ASan build)" # ~ 50s make test + msg "test: metatests (clang, ASan)" + tests/scripts/run-metatests.sh any asan + msg "test: Everest ECDH context - ECDH-related part of ssl-opt.sh (ASan build)" # ~ 5s tests/ssl-opt.sh -f ECDH @@ -1973,6 +1979,9 @@ component_test_full_cmake_clang () { msg "test: cpp_dummy_build (full config, clang)" # ~ 1s programs/test/cpp_dummy_build + msg "test: metatests (clang)" + tests/scripts/run-metatests.sh any pthread + msg "program demos (full config, clang)" # ~10s tests/scripts/run_demos.py @@ -5465,6 +5474,9 @@ component_test_memsan () { msg "test: main suites (MSan)" # ~ 10s make test + msg "test: metatests (MSan)" + tests/scripts/run-metatests.sh any msan + msg "program demos (MSan)" # ~20s tests/scripts/run_demos.py diff --git a/tests/scripts/run-metatests.sh b/tests/scripts/run-metatests.sh new file mode 100755 index 0000000000..22a302c62f --- /dev/null +++ b/tests/scripts/run-metatests.sh @@ -0,0 +1,89 @@ +#!/bin/sh + +help () { + cat <&2 "$0: FATAL: programs/test/metatest not found" + exit 120 +fi + +LIST_ONLY= +while getopts hl OPTLET; do + case $OPTLET in + h) help; exit;; + l) LIST_ONLY=1;; + \?) help >&2; exit 120;; + esac +done +shift $((OPTIND - 1)) + +list_matches () { + while read name platform junk; do + for pattern in "$@"; do + case $platform in + $pattern) echo "$name"; break;; + esac + done + done +} + +count=0 +errors=0 +run_metatest () { + ret=0 + "$METATEST_PROGRAM" "$1" || ret=$? + if [ $ret -eq 0 ]; then + echo >&2 "$0: Unexpected success: $1" + errors=$((errors + 1)) + fi + count=$((count + 1)) +} + +# Don't pipe the output of metatest so that if it fails, this script exits +# immediately with a failure status. +full_list=$("$METATEST_PROGRAM" list) +matching_list=$(printf '%s\n' "$full_list" | list_matches "$@") + +if [ -n "$LIST_ONLY" ]; then + printf '%s\n' $matching_list + exit +fi + +for name in $matching_list; do + run_metatest "$name" +done + +if [ $errors -eq 0 ]; then + echo "Ran $count metatests, all good." + exit 0 +else + echo "Ran $count metatests, $errors unexpected successes." + exit 1 +fi