Merge pull request #8463 from gilles-peskine-arm/metatest-create

Create a metatest program

programs/.gitignore

@@ -38,6 +38,7 @@ psa/crypto_examples
 psa/hmac_demo
 psa/key_ladder_demo
 psa/psa_constant_names
+psa/psa_hash
 random/gen_entropy
 random/gen_random_ctr_drbg
 ssl/dtls_client
@@ -56,6 +57,7 @@ test/cpp_dummy_build
 test/cpp_dummy_build.cpp
 test/dlopen
 test/ecp-bench
+test/metatest
 test/query_compile_time_config
 test/query_included_headers
 test/selftest

programs/Makefile

@@ -123,6 +123,7 @@ APPS = \
 	ssl/ssl_server \
 	ssl/ssl_server2 \
 	test/benchmark \
+	test/metatest \
 	test/query_compile_time_config \
 	test/query_included_headers \
 	test/selftest \
@@ -413,6 +414,10 @@ test/dlopen$(EXEXT): test/dlopen.c $(DEP)
 	$(CC) $(LOCAL_CFLAGS) $(CFLAGS) test/dlopen.c $(LDFLAGS) $(DLOPEN_LDFLAGS) -o $@
 endif
 
+test/metatest$(EXEXT): test/metatest.c $(DEP)
+	echo "  CC    test/metatest.c"
+	$(CC) $(LOCAL_CFLAGS) $(CFLAGS) -I ../library test/metatest.c    $(LOCAL_LDFLAGS) $(LDFLAGS) -o $@
+
 test/query_config.o: test/query_config.c test/query_config.h $(DEP)
 	echo "  CC    test/query_config.c"
 	$(CC) $(LOCAL_CFLAGS) $(CFLAGS) -c test/query_config.c -o $@

programs/test/CMakeLists.txt

@@ -3,6 +3,7 @@ set(libs
 )
 
 set(executables_libs
+    metatest
     query_included_headers
     selftest
     udp_proxy
@@ -72,6 +73,7 @@ foreach(exe IN LISTS executables_libs executables_mbedcrypto)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>
         ${extra_sources})
     target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../tests/include)
+    target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../library)
     if(exe STREQUAL "query_compile_time_config")
         target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     endif()

programs/test/metatest.c

@@ -0,0 +1,351 @@
+/** \file metatest.c
+ *
+ *  \brief Test features of the test framework.
+ *
+ * When you run this program, it runs a single "meta-test". A meta-test
+ * performs an operation which should be caught as a failure by our
+ * test framework. The meta-test passes if this program calls `exit` with
+ * a nonzero status, or aborts, or is terminated by a signal, or if the
+ * framework running the program considers the run an error (this happens
+ * with Valgrind for a memory leak). The non-success of the meta-test
+ * program means that the test failure has been caught correctly.
+ *
+ * Some failures are purely functional: the logic of the code causes the
+ * test result to be set to FAIL. Other failures come from extra
+ * instrumentation which is not present in a normal build; for example,
+ * Asan or Valgrind to detect memory leaks. This is reflected by the
+ * "platform" associated with each meta-test.
+ *
+ * Use the companion script `tests/scripts/run-metatests.sh` to run all
+ * the meta-tests for a given platform and validate that they trigger a
+ * detected failure as expected.
+ */
+
+/*
+ *  Copyright The Mbed TLS Contributors
+ *  SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+ */
+
+#define MBEDTLS_ALLOW_PRIVATE_ACCESS
+
+#include <mbedtls/platform.h>
+#include <mbedtls/platform_util.h>
+#include "test/helpers.h"
+#include "test/macros.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#if defined(MBEDTLS_THREADING_C)
+#include <mbedtls/threading.h>
+#endif
+
+
+/* This is an external variable, so the compiler doesn't know that we're never
+ * changing its value.
+ */
+volatile int false_but_the_compiler_does_not_know = 0;
+
+/* Set n bytes at the address p to all-bits-zero, in such a way that
+ * the compiler should not know that p is all-bits-zero. */
+static void set_to_zero_but_the_compiler_does_not_know(volatile void *p, size_t n)
+{
+    memset((void *) p, false_but_the_compiler_does_not_know, n);
+}
+
+
+/****************************************************************/
+/* Test framework features */
+/****************************************************************/
+
+void meta_test_fail(const char *name)
+{
+    (void) name;
+    mbedtls_test_fail("Forced test failure", __LINE__, __FILE__);
+}
+
+
+/****************************************************************/
+/* Platform features */
+/****************************************************************/
+
+void null_pointer_dereference(const char *name)
+{
+    (void) name;
+    volatile char *volatile p;
+    set_to_zero_but_the_compiler_does_not_know(&p, sizeof(p));
+    /* Undefined behavior (read from null data pointer) */
+    mbedtls_printf("%p -> %u\n", p, (unsigned) *p);
+}
+
+void null_pointer_call(const char *name)
+{
+    (void) name;
+    unsigned(*volatile p)(void);
+    set_to_zero_but_the_compiler_does_not_know(&p, sizeof(p));
+    /* Undefined behavior (execute null function pointer) */
+    /* The pointer representation may be truncated, but we don't care:
+     * the only point of printing it is to have some use of the pointer
+     * to dissuade the compiler from optimizing it away. */
+    mbedtls_printf("%lx() -> %u\n", (unsigned long) (uintptr_t) p, p());
+}
+
+
+/****************************************************************/
+/* Memory */
+/****************************************************************/
+
+void read_after_free(const char *name)
+{
+    (void) name;
+    volatile char *p = mbedtls_calloc(1, 1);
+    *p = 'a';
+    mbedtls_free((void *) p);
+    /* Undefined behavior (read after free) */
+    mbedtls_printf("%u\n", (unsigned) *p);
+}
+
+void double_free(const char *name)
+{
+    (void) name;
+    volatile char *p = mbedtls_calloc(1, 1);
+    *p = 'a';
+    mbedtls_free((void *) p);
+    /* Undefined behavior (double free) */
+    mbedtls_free((void *) p);
+}
+
+void read_uninitialized_stack(const char *name)
+{
+    (void) name;
+    char buf[1];
+    if (false_but_the_compiler_does_not_know) {
+        buf[0] = '!';
+    }
+    char *volatile p = buf;
+    if (*p != 0) {
+        /* Unspecified result (read from uninitialized memory) */
+        mbedtls_printf("%u\n", (unsigned) *p);
+    }
+}
+
+void memory_leak(const char *name)
+{
+    (void) name;
+    volatile char *p = mbedtls_calloc(1, 1);
+    mbedtls_printf("%u\n", (unsigned) *p);
+    /* Leak of a heap object */
+}
+
+
+/****************************************************************/
+/* Threading */
+/****************************************************************/
+
+void mutex_lock_not_initialized(const char *name)
+{
+    (void) name;
+#if defined(MBEDTLS_THREADING_C)
+    mbedtls_threading_mutex_t mutex;
+    memset(&mutex, 0, sizeof(mutex));
+    /* This mutex usage error is detected by our test framework's mutex usage
+     * verification framework. See tests/src/threading_helpers.c. Other
+     * threading implementations (e.g. pthread without our instrumentation)
+     * might consider this normal usage. */
+    TEST_ASSERT(mbedtls_mutex_lock(&mutex) == 0);
+exit:
+    ;
+#endif
+}
+
+void mutex_unlock_not_initialized(const char *name)
+{
+    (void) name;
+#if defined(MBEDTLS_THREADING_C)
+    mbedtls_threading_mutex_t mutex;
+    memset(&mutex, 0, sizeof(mutex));
+    /* This mutex usage error is detected by our test framework's mutex usage
+     * verification framework. See tests/src/threading_helpers.c. Other
+     * threading implementations (e.g. pthread without our instrumentation)
+     * might consider this normal usage. */
+    TEST_ASSERT(mbedtls_mutex_unlock(&mutex) == 0);
+exit:
+    ;
+#endif
+}
+
+void mutex_free_not_initialized(const char *name)
+{
+    (void) name;
+#if defined(MBEDTLS_THREADING_C)
+    mbedtls_threading_mutex_t mutex;
+    memset(&mutex, 0, sizeof(mutex));
+    /* This mutex usage error is detected by our test framework's mutex usage
+     * verification framework. See tests/src/threading_helpers.c. Other
+     * threading implementations (e.g. pthread without our instrumentation)
+     * might consider this normal usage. */
+    mbedtls_mutex_free(&mutex);
+#endif
+}
+
+void mutex_double_init(const char *name)
+{
+    (void) name;
+#if defined(MBEDTLS_THREADING_C)
+    mbedtls_threading_mutex_t mutex;
+    mbedtls_mutex_init(&mutex);
+    /* This mutex usage error is detected by our test framework's mutex usage
+     * verification framework. See tests/src/threading_helpers.c. Other
+     * threading implementations (e.g. pthread without our instrumentation)
+     * might consider this normal usage. */
+    mbedtls_mutex_init(&mutex);
+    mbedtls_mutex_free(&mutex);
+#endif
+}
+
+void mutex_double_free(const char *name)
+{
+    (void) name;
+#if defined(MBEDTLS_THREADING_C)
+    mbedtls_threading_mutex_t mutex;
+    mbedtls_mutex_init(&mutex);
+    mbedtls_mutex_free(&mutex);
+    /* This mutex usage error is detected by our test framework's mutex usage
+     * verification framework. See tests/src/threading_helpers.c. Other
+     * threading implementations (e.g. pthread without our instrumentation)
+     * might consider this normal usage. */
+    mbedtls_mutex_free(&mutex);
+#endif
+}
+
+void mutex_leak(const char *name)
+{
+    (void) name;
+#if defined(MBEDTLS_THREADING_C)
+    mbedtls_threading_mutex_t mutex;
+    mbedtls_mutex_init(&mutex);
+#endif
+    /* This mutex usage error is detected by our test framework's mutex usage
+     * verification framework. See tests/src/threading_helpers.c. Other
+     * threading implementations (e.g. pthread without our instrumentation)
+     * might consider this normal usage. */
+}
+
+
+/****************************************************************/
+/* Command line entry point */
+/****************************************************************/
+
+typedef struct {
+    /** Command line argument that will trigger that metatest.
+     *
+     * Conventionally matches "[a-z0-9_]+". */
+    const char *name;
+
+    /** Platform under which that metatest is valid.
+     *
+     * - "any": should work anywhere.
+     * - "asan": triggers ASan (Address Sanitizer).
+     * - "msan": triggers MSan (Memory Sanitizer).
+     * - "pthread": requires MBEDTLS_THREADING_PTHREAD and MBEDTLS_TEST_HOOKS,
+     *   which enables MBEDTLS_TEST_MUTEX_USAGE internally in the test
+     *   framework (see tests/src/threading_helpers.c).
+     */
+    const char *platform;
+
+    /** Function that performs the metatest.
+     *
+     * The function receives the name as an argument. This allows using the
+     * same function to perform multiple variants of a test based on the name.
+     *
+     * When executed on a conforming platform, the function is expected to
+     * either cause a test failure (mbedtls_test_fail()), or cause the
+     * program to abort in some way (e.g. by causing a segfault or by
+     * triggering a sanitizer).
+     *
+     * When executed on a non-conforming platform, the function may return
+     * normally or may have unpredictable behavior.
+     */
+    void (*entry_point)(const char *name);
+} metatest_t;
+
+/* The list of availble meta-tests. Remember to register new functions here!
+ *
+ * Note that we always compile all the functions, so that `metatest --list`
+ * will always list all the available meta-tests.
+ *
+ * See the documentation of metatest_t::platform for the meaning of
+ * platform values.
+ */
+metatest_t metatests[] = {
+    { "test_fail", "any", meta_test_fail },
+    { "null_dereference", "any", null_pointer_dereference },
+    { "null_call", "any", null_pointer_call },
+    { "read_after_free", "asan", read_after_free },
+    { "double_free", "asan", double_free },
+    { "read_uninitialized_stack", "msan", read_uninitialized_stack },
+    { "memory_leak", "asan", memory_leak },
+    { "mutex_lock_not_initialized", "pthread", mutex_lock_not_initialized },
+    { "mutex_unlock_not_initialized", "pthread", mutex_unlock_not_initialized },
+    { "mutex_free_not_initialized", "pthread", mutex_free_not_initialized },
+    { "mutex_double_init", "pthread", mutex_double_init },
+    { "mutex_double_free", "pthread", mutex_double_free },
+    { "mutex_leak", "pthread", mutex_leak },
+    { NULL, NULL, NULL }
+};
+
+static void help(FILE *out, const char *argv0)
+{
+    mbedtls_fprintf(out, "Usage: %s list|TEST\n", argv0);
+    mbedtls_fprintf(out, "Run a meta-test that should cause a test failure.\n");
+    mbedtls_fprintf(out, "With 'list', list the available tests and their platform requirement.\n");
+}
+
+int main(int argc, char *argv[])
+{
+    const char *argv0 = argc > 0 ? argv[0] : "metatest";
+    if (argc != 2) {
+        help(stderr, argv0);
+        mbedtls_exit(MBEDTLS_EXIT_FAILURE);
+    }
+
+    /* Support "-help", "--help", "--list", etc. */
+    const char *command = argv[1];
+    while (*command == '-') {
+        ++command;
+    }
+
+    if (strcmp(argv[1], "help") == 0) {
+        help(stdout, argv0);
+        mbedtls_exit(MBEDTLS_EXIT_SUCCESS);
+    }
+    if (strcmp(argv[1], "list") == 0) {
+        for (const metatest_t *p = metatests; p->name != NULL; p++) {
+            mbedtls_printf("%s %s\n", p->name, p->platform);
+        }
+        mbedtls_exit(MBEDTLS_EXIT_SUCCESS);
+    }
+
+#if defined(MBEDTLS_TEST_MUTEX_USAGE)
+    mbedtls_test_mutex_usage_init();
+#endif
+
+    for (const metatest_t *p = metatests; p->name != NULL; p++) {
+        if (strcmp(argv[1], p->name) == 0) {
+            mbedtls_printf("Running metatest %s...\n", argv[1]);
+            p->entry_point(argv[1]);
+#if defined(MBEDTLS_TEST_MUTEX_USAGE)
+            mbedtls_test_mutex_usage_check();
+#endif
+            mbedtls_printf("Running metatest %s... done, result=%d\n",
+                           argv[1], (int) mbedtls_test_info.result);
+            mbedtls_exit(mbedtls_test_info.result == MBEDTLS_TEST_RESULT_SUCCESS ?
+                         MBEDTLS_EXIT_SUCCESS :
+                         MBEDTLS_EXIT_FAILURE);
+        }
+    }
+
+    mbedtls_fprintf(stderr, "%s: FATAL: No such metatest: %s\n",
+                    argv0, command);
+    mbedtls_exit(MBEDTLS_EXIT_FAILURE);
+}

scripts/generate_visualc_files.pl

@@ -144,6 +144,7 @@ sub gen_app {
     my $guid = gen_app_guid( $path );
     $path =~ s!/!\\!g;
     (my $appname = $path) =~ s/.*\\//;
+    my $is_test_app = ($path =~ m/^test\\/);
 
     my $srcs = "<ClCompile Include=\"..\\..\\programs\\$path.c\" \/>";
     if( $appname eq "ssl_client2" or $appname eq "ssl_server2" or
@@ -158,7 +159,9 @@ sub gen_app {
     $content =~ s/<SOURCES>/$srcs/g;
     $content =~ s/<APPNAME>/$appname/g;
     $content =~ s/<GUID>/$guid/g;
-    $content =~ s/INCLUDE_DIRECTORIES\n/$include_directories/g;
+    $content =~ s/INCLUDE_DIRECTORIES\n/($is_test_app ?
+                                         $library_include_directories :
+                                         $include_directories)/ge;
 
     content_to_file( $content, "$dir/$appname.$ext" );
 }

tests/scripts/all.sh

@@ -1120,6 +1120,9 @@ component_test_default_cmake_gcc_asan () {
     msg "test: selftest (ASan build)" # ~ 10s
     programs/test/selftest
 
+    msg "test: metatests (GCC, ASan build)"
+    tests/scripts/run-metatests.sh any asan
+
     msg "test: ssl-opt.sh (ASan build)" # ~ 1 min
     tests/ssl-opt.sh
 
@@ -1885,6 +1888,9 @@ component_test_everest () {
     msg "test: Everest ECDH context - main suites (inc. selftests) (ASan build)" # ~ 50s
     make test
 
+    msg "test: metatests (clang, ASan)"
+    tests/scripts/run-metatests.sh any asan
+
     msg "test: Everest ECDH context - ECDH-related part of ssl-opt.sh (ASan build)" # ~ 5s
     tests/ssl-opt.sh -f ECDH
 
@@ -1973,6 +1979,9 @@ component_test_full_cmake_clang () {
     msg "test: cpp_dummy_build (full config, clang)" # ~ 1s
     programs/test/cpp_dummy_build
 
+    msg "test: metatests (clang)"
+    tests/scripts/run-metatests.sh any pthread
+
     msg "program demos (full config, clang)" # ~10s
     tests/scripts/run_demos.py
 
@@ -5465,6 +5474,9 @@ component_test_memsan () {
     msg "test: main suites (MSan)" # ~ 10s
     make test
 
+    msg "test: metatests (MSan)"
+    tests/scripts/run-metatests.sh any msan
+
     msg "program demos (MSan)" # ~20s
     tests/scripts/run_demos.py
 

tests/scripts/run-metatests.sh

@@ -0,0 +1,89 @@
+#!/bin/sh
+
+help () {
+    cat <<EOF
+Usage: $0 [OPTION] [PLATFORM]...
+Run all the metatests whose platform matches any of the given PLATFORM.
+A PLATFORM can contain shell wildcards.
+
+Expected output: a lot of scary-looking error messages, since each
+metatest is expected to report a failure. The final line should be
+"Ran N metatests, all good."
+
+If something goes wrong: the final line should be
+"Ran N metatests, X unexpected successes". Look for "Unexpected success"
+in the logs above.
+
+  -l  List the available metatests, don't run them.
+EOF
+}
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+
+set -e -u
+
+if [ -d programs ]; then
+    METATEST_PROGRAM=programs/test/metatest
+elif [ -d ../programs ]; then
+    METATEST_PROGRAM=../programs/test/metatest
+elif [ -d ../../programs ]; then
+    METATEST_PROGRAM=../../programs/test/metatest
+else
+    echo >&2 "$0: FATAL: programs/test/metatest not found"
+    exit 120
+fi
+
+LIST_ONLY=
+while getopts hl OPTLET; do
+    case $OPTLET in
+        h) help; exit;;
+        l) LIST_ONLY=1;;
+        \?) help >&2; exit 120;;
+    esac
+done
+shift $((OPTIND - 1))
+
+list_matches () {
+    while read name platform junk; do
+        for pattern in "$@"; do
+            case $platform in
+                $pattern) echo "$name"; break;;
+            esac
+        done
+    done
+}
+
+count=0
+errors=0
+run_metatest () {
+    ret=0
+    "$METATEST_PROGRAM" "$1" || ret=$?
+    if [ $ret -eq 0 ]; then
+        echo >&2 "$0: Unexpected success: $1"
+        errors=$((errors + 1))
+    fi
+    count=$((count + 1))
+}
+
+# Don't pipe the output of metatest so that if it fails, this script exits
+# immediately with a failure status.
+full_list=$("$METATEST_PROGRAM" list)
+matching_list=$(printf '%s\n' "$full_list" | list_matches "$@")
+
+if [ -n "$LIST_ONLY" ]; then
+    printf '%s\n' $matching_list
+    exit
+fi
+
+for name in $matching_list; do
+    run_metatest "$name"
+done
+
+if [ $errors -eq 0 ]; then
+    echo "Ran $count metatests, all good."
+    exit 0
+else
+    echo "Ran $count metatests, $errors unexpected successes."
+    exit 1
+fi