Merge pull request #9394 from ronald-cron-arm/move-crypto-test-suites

Move crypto test suites

CMakeLists.txt

@@ -22,6 +22,10 @@ cmake_minimum_required(VERSION 3.5.1)
 
 include(CMakePackageConfigHelpers)
 
+# Include convenience functions for printing properties and variables, like
+# cmake_print_properties(), cmake_print_variables().
+include(CMakePrintHelpers)
+
 # https://cmake.org/cmake/help/latest/policy/CMP0011.html
 # Setting this policy is required in CMake >= 3.18.0, otherwise a warning is generated. The OLD
 # policy setting is deprecated, and will be removed in future versions.
@@ -401,6 +405,7 @@ if(ENABLE_TESTING)
     enable_testing()
 
     add_subdirectory(tests)
+    add_subdirectory(tf-psa-crypto/tests)
 
     # additional convenience targets for Unix only
     if(UNIX)

CONTRIBUTING.md

@@ -57,7 +57,7 @@ Tests
 -----
 As mentioned, tests that show the correctness of the feature or bug fix should be added to the pull request, if no such tests exist.
 
-Mbed TLS includes a comprehensive set of test suites in the `tests/` directory that are dynamically generated to produce the actual test source files (e.g. `test_suite_rsa.c`). These files are generated from a `function file` (e.g. `suites/test_suite_rsa.function`) and a `data file` (e.g. `suites/test_suite_rsa.data`). The function file contains the test functions. The data file contains the test cases, specified as parameters that will be passed to the test function.
+Mbed TLS includes a comprehensive set of test suites in the `tests/` directory that are dynamically generated to produce the actual test source files (e.g. `test_suite_ssl.c`). These files are generated from a `function file` (e.g. `suites/test_suite_ssl.function`) and a `data file` (e.g. `suites/test_suite_ssl.data`). The function file contains the test functions. The data file contains the test cases, specified as parameters that will be passed to the test function.
 
 [A Knowledge Base article describing how to add additional tests is available on the Mbed TLS website](https://mbed-tls.readthedocs.io/en/latest/kb/development/test_suites/).
 

Makefile

@@ -216,6 +216,7 @@ C_SOURCE_FILES = $(wildcard \
 	tests/include/*/*.h tests/include/*/*/*.h \
 	tests/src/*.c tests/src/*/*.c \
 	tests/suites/*.function \
+	tf-psa-crypto/tests/suites/*.function \
 )
 # Exuberant-ctags invocation. Other ctags implementations may require different options.
 CTAGS = ctags --langmap=c:+.h.function --line-directives=no -o

README.md

@@ -240,7 +240,7 @@ Please note that the goal of these sample programs is to demonstrate specific fe
 Tests
 -----
 
-Mbed TLS includes an elaborate test suite in `tests/` that initially requires Python to generate the tests files (e.g. `test\_suite\_mpi.c`). These files are generated from a `function file` (e.g. `suites/test\_suite\_mpi.function`) and a `data file` (e.g. `suites/test\_suite\_mpi.data`). The `function file` contains the test functions. The `data file` contains the test cases, specified as parameters that will be passed to the test function.
+Mbed TLS includes an elaborate test suite in `tests/` that initially requires Python to generate the tests files (e.g. `test\_suite\_ssl.c`). These files are generated from a `function file` (e.g. `suites/test\_suite\_ssl.function`) and a `data file` (e.g. `suites/test\_suite\_ssl.data`). The `function file` contains the test functions. The `data file` contains the test cases, specified as parameters that will be passed to the test function.
 
 For machines with a Unix shell and OpenSSL (and optionally GnuTLS) installed, additional test scripts are available:
 

framework

@@ -1 +1 @@
-Subproject commit 9eeacb7125d6630a11e29d8a7aab5873b3638b6d
+Subproject commit e8b4ae9bc4bf7e643ee46bf8ff4ef613be2de86f

library/Makefile

@@ -243,6 +243,7 @@ endif
 
 static: libmbedcrypto.a libmbedx509.a libmbedtls.a
 	cd ../tests && echo "This is a seedfile that contains 64 bytes (65 on Windows)......" > seedfile
+	cd ../tf-psa-crypto/tests && echo "This is a seedfile that contains 64 bytes (65 on Windows)......" > seedfile
 
 shared: libmbedcrypto.$(DLEXT) libmbedx509.$(DLEXT) libmbedtls.$(DLEXT)
 
@@ -397,11 +398,10 @@ clean:
 ifndef WINDOWS
 	rm -f *.o libmbed*
 	rm -f $(OBJS_CRYPTO)
-	rm -f $(THIRDPARTY_CRYPTO_OBJECTS)
 else
 	if exist *.o del /Q /F *.o
 	if exist libmbed* del /Q /F libmbed*
-	del /Q /F del_errors_out_if_the_file_list_is_empty_but_not_if_a_file_does_not_exist $(subst /,\,$(THIRDPARTY_CRYPTO_OBJECTS))
+	del /Q /F del_errors_out_if_the_file_list_is_empty_but_not_if_a_file_does_not_exist $(subst /,\,$(OBJS_CRYPTO))
 endif
 
 neat: clean

programs/Makefile

@@ -129,7 +129,7 @@ generated_files: $(GENERATED_FILES)
 psa/psa_constant_names_generated.c: $(gen_file_dep) ../scripts/generate_psa_constants.py
 psa/psa_constant_names_generated.c: $(gen_file_dep) ../tf-psa-crypto/include/psa/crypto_values.h
 psa/psa_constant_names_generated.c: $(gen_file_dep) ../tf-psa-crypto/include/psa/crypto_extra.h
-psa/psa_constant_names_generated.c: $(gen_file_dep) ../tests/suites/test_suite_psa_crypto_metadata.data
+psa/psa_constant_names_generated.c: $(gen_file_dep) ../tf-psa-crypto/tests/suites/test_suite_psa_crypto_metadata.data
 psa/psa_constant_names_generated.c:
 	echo "  Gen   $@"
 	$(PYTHON) ../scripts/generate_psa_constants.py
@@ -417,6 +417,7 @@ x509/req_app$(EXEXT): x509/req_app.c $(DEP)
 clean:
 ifndef WINDOWS
 	rm -f $(EXES)
+	rm -f */*.o
 	-rm -f ssl/ssl_pthread_server$(EXEXT)
 	-rm -f test/cpp_dummy_build.cpp test/cpp_dummy_build$(EXEXT)
 	-rm -f test/dlopen$(EXEXT)

programs/fuzz/Makefile

@@ -9,9 +9,7 @@ ifdef FUZZINGENGINE
 LOCAL_LDFLAGS += -lFuzzingEngine
 endif
 
-# A test application is built for each suites/test_suite_*.data file.
+# A test application is built for each fuzz_*.c file.
-# Application name is same as .data file's base name and can be
-# constructed by stripping path 'suites/' and extension .data.
 APPS = $(basename $(wildcard fuzz_*.c))
 
 # Construct executable name by adding OS specific suffix $(EXEXT).

scripts/abi_check.py

@@ -350,9 +350,15 @@ class AbiChecker:
         """
         # Existing test data files. This may be missing some automatically
         # generated files if they haven't been generated yet.
-        storage_data_files = set(glob.glob(
+        if os.path.isdir(os.path.join(git_worktree_path, 'tf-psa-crypto',
-            'tests/suites/test_suite_*storage_format*.data'
+                                      'tests', 'suites')):
-        ))
+            storage_data_files = set(glob.glob(
+                'tf-psa-crypto/tests/suites/test_suite_*storage_format*.data'
+            ))
+        else:
+            storage_data_files = set(glob.glob(
+                'tests/suites/test_suite_*storage_format*.data'
+            ))
         # Discover and (re)generate automatically generated data files.
         to_be_generated = set()
         for filename in self._list_generated_test_data_files(git_worktree_path):

scripts/bump_version.sh

@@ -124,8 +124,8 @@ cat include/mbedtls/build_info.h |                                    \
 mv tmp include/mbedtls/build_info.h
 
 [ $VERBOSE ] && echo "Bumping version in tests/suites/test_suite_version.data"
-sed -e "s/version:\".\{1,\}/version:\"$VERSION\"/g" < tests/suites/test_suite_version.data > tmp
+sed -e "s/version:\".\{1,\}/version:\"$VERSION\"/g" < tf-psa-crypto/tests/suites/test_suite_version.data > tmp
-mv tmp tests/suites/test_suite_version.data
+mv tmp tf-psa-crypto/tests/suites/test_suite_version.data
 
 [ $VERBOSE ] && echo "Bumping PROJECT_NAME in doxygen/mbedtls.doxyfile and doxygen/input/doc_mainpage.h"
 for i in doxygen/mbedtls.doxyfile doxygen/input/doc_mainpage.h;

scripts/code_style.py

@@ -71,6 +71,7 @@ def get_src_files(since: Optional[str]) -> List[str]:
     """
     file_patterns = ["*.[hc]",
                      "tests/suites/*.function",
+                     "tf-psa-crypto/tests/suites/*.function",
                      "scripts/data_files/*.fmt"]
     output = subprocess.check_output(["git", "ls-files"] + file_patterns,
                                      universal_newlines=True)

scripts/make_generated_files.bat

@@ -10,9 +10,10 @@ perl scripts\generate_features.pl || exit /b 1
 python scripts\generate_ssl_debug_helpers.py || exit /b 1
 perl scripts\generate_visualc_files.pl || exit /b 1
 python scripts\generate_psa_constants.py || exit /b 1
-python framework\scripts\generate_bignum_tests.py || exit /b 1
+python framework\scripts\generate_bignum_tests.py --directory tf-psa-crypto\tests\suites || exit /b 1
-python framework\scripts\generate_config_tests.py || exit /b 1
+python framework\scripts\generate_config_tests.py tests\suites\test_suite_config.mbedtls_boolean.data || exit /b 1
-python framework\scripts\generate_ecp_tests.py || exit /b 1
+python framework\scripts\generate_config_tests.py --directory tf-psa-crypto\tests\suites tests\suites\test_suite_config.psa_boolean.data || exit /b 1
-python framework\scripts\generate_psa_tests.py || exit /b 1
+python framework\scripts\generate_ecp_tests.py --directory tf-psa-crypto\tests\suites || exit /b 1
+python framework\scripts\generate_psa_tests.py --directory tf-psa-crypto\tests\suites || exit /b 1
 python framework\scripts\generate_test_keys.py --output tests\src\test_keys.h || exit /b 1
 python framework\scripts\generate_test_cert_macros.py --output tests\src\test_certs.h || exit /b 1

tests/.gitignore

@@ -20,9 +20,6 @@
 # Generated source files
 /suites/*.generated.data
 /suites/test_suite_config.mbedtls_boolean.data
-/suites/test_suite_config.psa_boolean.data
-/suites/test_suite_psa_crypto_storage_format.v[0-9]*.data
-/suites/test_suite_psa_crypto_storage_format.current.data
 /src/test_keys.h
 /src/test_certs.h
 ###END_GENERATED_FILES###

tests/CMakeLists.txt

@@ -18,18 +18,6 @@ endif()
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/suites)
 
 # Get base names for generated files
-execute_process(
-    COMMAND
-        ${MBEDTLS_PYTHON_EXECUTABLE}
-        ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_bignum_tests.py
-        --list-for-cmake
-    WORKING_DIRECTORY
-        ${CMAKE_CURRENT_SOURCE_DIR}/..
-    OUTPUT_VARIABLE
-        base_bignum_generated_data_files)
-string(REGEX REPLACE "[^;]*/" ""
-       base_bignum_generated_data_files "${base_bignum_generated_data_files}")
-
 execute_process(
     COMMAND
         ${MBEDTLS_PYTHON_EXECUTABLE}
@@ -38,76 +26,28 @@ execute_process(
     WORKING_DIRECTORY
         ${CMAKE_CURRENT_SOURCE_DIR}/..
     OUTPUT_VARIABLE
-        base_config_generated_data_files)
+        base_config_generated_data_files_raw)
 string(REGEX REPLACE "[^;]*/" ""
-       base_config_generated_data_files "${base_config_generated_data_files}")
+       base_config_generated_data_files_raw "${base_config_generated_data_files_raw}")
-
+# Can be replace by list(FILTER ...) when CI CMake version is >=3.6
-execute_process(
+foreach(file ${base_config_generated_data_files_raw})
-    COMMAND
+    if(${file} MATCHES "mbedtls")
-        ${MBEDTLS_PYTHON_EXECUTABLE}
+        list(APPEND base_config_generated_data_files ${file})
-        ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_ecp_tests.py
+    endif()
-        --list-for-cmake
+endforeach()
-    WORKING_DIRECTORY
-        ${CMAKE_CURRENT_SOURCE_DIR}/..
-    OUTPUT_VARIABLE
-        base_ecp_generated_data_files)
-string(REGEX REPLACE "[^;]*/" ""
-       base_ecp_generated_data_files "${base_ecp_generated_data_files}")
-
-execute_process(
-    COMMAND
-        ${MBEDTLS_PYTHON_EXECUTABLE}
-        ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_psa_tests.py
-        --list-for-cmake
-    WORKING_DIRECTORY
-        ${CMAKE_CURRENT_SOURCE_DIR}/..
-    OUTPUT_VARIABLE
-        base_psa_generated_data_files)
-string(REGEX REPLACE "[^;]*/" ""
-       base_psa_generated_data_files "${base_psa_generated_data_files}")
 
 # Derive generated file paths in the build directory. The generated data
 # files go into the suites/ subdirectory.
 set(base_generated_data_files
-    ${base_bignum_generated_data_files} ${base_ecp_generated_data_files} ${base_psa_generated_data_files})
+    ${base_config_generated_data_files})
 string(REGEX REPLACE "([^;]+)" "suites/\\1"
        all_generated_data_files "${base_generated_data_files}")
-set(bignum_generated_data_files "")
 set(config_generated_data_files "")
-set(ecp_generated_data_files "")
-set(psa_generated_data_files "")
-foreach(file ${base_bignum_generated_data_files})
-    list(APPEND bignum_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
-endforeach()
 foreach(file ${base_config_generated_data_files})
     list(APPEND config_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
 endforeach()
-foreach(file ${base_ecp_generated_data_files})
-    list(APPEND ecp_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
-endforeach()
-foreach(file ${base_psa_generated_data_files})
-    list(APPEND psa_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
-endforeach()
 
 if(GEN_FILES)
-    add_custom_command(
-        OUTPUT
-            ${bignum_generated_data_files}
-        WORKING_DIRECTORY
-            ${CMAKE_CURRENT_SOURCE_DIR}/..
-        COMMAND
-            ${MBEDTLS_PYTHON_EXECUTABLE}
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_bignum_tests.py
-            --directory ${CMAKE_CURRENT_BINARY_DIR}/suites
-        DEPENDS
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_bignum_tests.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/bignum_common.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/bignum_core.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/bignum_mod_raw.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/bignum_mod.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/test_case.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/test_data_generation.py
-    )
     add_custom_command(
         OUTPUT
             ${config_generated_data_files}
@@ -117,51 +57,13 @@ if(GEN_FILES)
             ${MBEDTLS_PYTHON_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_config_tests.py
             --directory ${CMAKE_CURRENT_BINARY_DIR}/suites
+            ${config_generated_data_files}
         DEPENDS
             ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_config_tests.py
             # Do not declare the configuration files as dependencies: they
             # change too often in ways that don't affect the result
             # ((un)commenting some options).
     )
-    add_custom_command(
-        OUTPUT
-            ${ecp_generated_data_files}
-        WORKING_DIRECTORY
-            ${CMAKE_CURRENT_SOURCE_DIR}/..
-        COMMAND
-            ${MBEDTLS_PYTHON_EXECUTABLE}
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_ecp_tests.py
-            --directory ${CMAKE_CURRENT_BINARY_DIR}/suites
-        DEPENDS
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_ecp_tests.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/bignum_common.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/ecp.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/test_case.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/test_data_generation.py
-    )
-    add_custom_command(
-        OUTPUT
-            ${psa_generated_data_files}
-        WORKING_DIRECTORY
-            ${CMAKE_CURRENT_SOURCE_DIR}/..
-        COMMAND
-            ${MBEDTLS_PYTHON_EXECUTABLE}
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_psa_tests.py
-            --directory ${CMAKE_CURRENT_BINARY_DIR}/suites
-        DEPENDS
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_psa_tests.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/crypto_data_tests.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/crypto_knowledge.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/macro_collector.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/psa_information.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/psa_storage.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/test_case.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/test_data_generation.py
-            ${CMAKE_CURRENT_SOURCE_DIR}/../tf-psa-crypto/include/psa/crypto_config.h
-            ${CMAKE_CURRENT_SOURCE_DIR}/../tf-psa-crypto/include/psa/crypto_values.h
-            ${CMAKE_CURRENT_SOURCE_DIR}/../tf-psa-crypto/include/psa/crypto_extra.h
-    )
-
 else()
     foreach(file ${all_generated_data_files})
         link_to_source(${file})
@@ -172,10 +74,7 @@ endif()
 # they can cause race conditions in parallel builds.
 # With this line, only 4 sub-makefiles include the above command, that reduces
 # the risk of a race.
-add_custom_target(test_suite_bignum_generated_data DEPENDS ${bignum_generated_data_files})
 add_custom_target(test_suite_config_generated_data DEPENDS ${config_generated_data_files})
-add_custom_target(test_suite_ecp_generated_data DEPENDS ${ecp_generated_data_files})
-add_custom_target(test_suite_psa_generated_data DEPENDS ${psa_generated_data_files})
 # If SKIP_TEST_SUITES is not defined with -D, get it from the environment.
 if((NOT DEFINED SKIP_TEST_SUITES) AND (DEFINED ENV{SKIP_TEST_SUITES}))
     set(SKIP_TEST_SUITES $ENV{SKIP_TEST_SUITES})
@@ -196,61 +95,26 @@ function(add_test_suite suite_name)
 
     # Get the test names of the tests with generated .data files
     # from the generated_data_files list in parent scope.
-    set(bignum_generated_data_names "")
+    set(config_generated_data_names "")
-    set(ecp_generated_data_names "")
+    foreach(generated_data_file ${config_generated_data_files})
-    set(psa_generated_data_names "")
-    foreach(generated_data_file ${bignum_generated_data_files})
         # Get the plain filename
         get_filename_component(generated_data_name ${generated_data_file} NAME)
         # Remove the ".data" extension
         get_name_without_last_ext(generated_data_name ${generated_data_name})
         # Remove leading "test_suite_"
         string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name)
-        list(APPEND bignum_generated_data_names ${generated_data_name})
+        list(APPEND config_generated_data_names ${generated_data_name})
-    endforeach()
-    foreach(generated_data_file ${ecp_generated_data_files})
-        # Get the plain filename
-        get_filename_component(generated_data_name ${generated_data_file} NAME)
-        # Remove the ".data" extension
-        get_name_without_last_ext(generated_data_name ${generated_data_name})
-        # Remove leading "test_suite_"
-        string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name)
-        list(APPEND ecp_generated_data_names ${generated_data_name})
-    endforeach()
-    foreach(generated_data_file ${psa_generated_data_files})
-        # Get the plain filename
-        get_filename_component(generated_data_name ${generated_data_file} NAME)
-        # Remove the ".data" extension
-        get_name_without_last_ext(generated_data_name ${generated_data_name})
-        # Remove leading "test_suite_"
-        string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name)
-        list(APPEND psa_generated_data_names ${generated_data_name})
     endforeach()
 
-    if(";${bignum_generated_data_names};" MATCHES ";${data_name};")
+    if(";${config_generated_data_names};" MATCHES ";${data_name};")
         set(data_file
             ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
-        set(dependency test_suite_bignum_generated_data)
+        set(dependency test_suite_config_generated_data)
-    elseif(";${config_generated_data_names};" MATCHES ";${data_name};")
-        set(data_file
-            ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
-        set(dependency test_suite_bignum_generated_data)
-    elseif(";${ecp_generated_data_names};" MATCHES ";${data_name};")
-        set(data_file
-            ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
-        set(dependency test_suite_ecp_generated_data)
-    elseif(";${psa_generated_data_names};" MATCHES ";${data_name};")
-        set(data_file
-            ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
-        set(dependency test_suite_psa_generated_data)
     else()
         set(data_file
             ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${data_name}.data)
         set(dependency
-            test_suite_bignum_generated_data
+            test_suite_config_generated_data)
-            test_suite_config_generated_data
-            test_suite_ecp_generated_data
-            test_suite_psa_generated_data)
     endif()
 
     add_custom_command(
@@ -263,18 +127,18 @@ function(add_test_suite suite_name)
             ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_test_code.py
             -f ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${suite_name}.function
             -d ${data_file}
-            -t ${CMAKE_CURRENT_SOURCE_DIR}/suites/main_test.function
+            -t ${CMAKE_CURRENT_SOURCE_DIR}/../tf-psa-crypto/tests/suites/main_test.function
-            -p ${CMAKE_CURRENT_SOURCE_DIR}/suites/host_test.function
+            -p ${CMAKE_CURRENT_SOURCE_DIR}/../tf-psa-crypto/tests/suites/host_test.function
             -s ${CMAKE_CURRENT_SOURCE_DIR}/suites
-            --helpers-file ${CMAKE_CURRENT_SOURCE_DIR}/suites/helpers.function
+            --helpers-file ${CMAKE_CURRENT_SOURCE_DIR}/../tf-psa-crypto/tests/suites/helpers.function
             -o .
         DEPENDS
             ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_test_code.py
             ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${suite_name}.function
             ${data_file}
-            ${CMAKE_CURRENT_SOURCE_DIR}/suites/main_test.function
+            ${CMAKE_CURRENT_SOURCE_DIR}/../tf-psa-crypto/tests/suites/main_test.function
-            ${CMAKE_CURRENT_SOURCE_DIR}/suites/host_test.function
+            ${CMAKE_CURRENT_SOURCE_DIR}/../tf-psa-crypto/tests/suites/host_test.function
-            ${CMAKE_CURRENT_SOURCE_DIR}/suites/helpers.function
+            ${CMAKE_CURRENT_SOURCE_DIR}/../tf-psa-crypto/tests/suites/helpers.function
             ${mbedtls_target}
         BYPRODUCTS
             test_suite_${data_name}.datax

tests/Makefile

@@ -5,8 +5,7 @@ include ../scripts/common.make
 TEST_FLAGS ?= $(if $(filter-out 0 OFF Off off NO No no FALSE False false N n,$(CTEST_OUTPUT_ON_FAILURE)),-v,)
 
 # Also include private headers, for the sake of invasive tests.
-LOCAL_CFLAGS += -I../library -I../tf-psa-crypto/core -I../tf-psa-crypto/drivers/builtin/src
+LOCAL_CFLAGS += -I$(MBEDTLS_PATH)/library -I$(MBEDTLS_PATH)/tf-psa-crypto/core -I$(MBEDTLS_PATH)/tf-psa-crypto/drivers/builtin/src
-
 
 # Enable definition of various functions used throughout the testsuite
 # (gethostname, strdup, fileno...) even when compiling with -std=c99. Harmless
@@ -17,43 +16,47 @@ ifdef RECORD_PSA_STATUS_COVERAGE_LOG
 LOCAL_CFLAGS += -Werror -DRECORD_PSA_STATUS_COVERAGE_LOG
 endif
 
-GENERATED_BIGNUM_DATA_FILES := $(patsubst tests/%,%,$(shell \
+GENERATED_BIGNUM_DATA_FILES := $(addprefix ../tf-psa-crypto/,$(shell \
 	$(PYTHON) ../framework/scripts/generate_bignum_tests.py --list || \
 	echo FAILED \
 ))
 ifeq ($(GENERATED_BIGNUM_DATA_FILES),FAILED)
 $(error "$(PYTHON) ../framework/scripts/generate_bignum_tests.py --list" failed)
 endif
-GENERATED_DATA_FILES += $(GENERATED_BIGNUM_DATA_FILES)
+GENERATED_CRYPTO_DATA_FILES += $(GENERATED_BIGNUM_DATA_FILES)
 
-GENERATED_CONFIG_DATA_FILES := $(patsubst tests/%,%,$(shell \
+GENERATED_CONFIG_DATA_FILES_RAW := $(patsubst tests/%,%,$(shell \
 	$(PYTHON) ../framework/scripts/generate_config_tests.py --list || \
 	echo FAILED \
 ))
 ifeq ($(GENERATED_CONFIG_DATA_FILES),FAILED)
 $(error "$(PYTHON) ../framework/scripts/generate_config_tests.py --list" failed)
 endif
-GENERATED_DATA_FILES += $(GENERATED_CONFIG_DATA_FILES)
+GENERATED_MBEDTLS_CONFIG_DATA_FILES := $(foreach file,$(GENERATED_CONFIG_DATA_FILES_RAW),$(if $(findstring mbedtls,$(file)),$(file),))
+GENERATED_PSA_CONFIG_DATA_FILES := $(foreach file,$(GENERATED_CONFIG_DATA_FILES_RAW),$(if $(findstring psa,$(file)),$(addprefix ../tf-psa-crypto/tests/,$(file)),))
+GENERATED_CONFIG_DATA_FILES := $(GENERATED_MBEDTLS_CONFIG_DATA_FILES)$(GENERATED_PSA_CONFIG_DATA_FILES)
+GENERATED_DATA_FILES += $(GENERATED_MBEDTLS_CONFIG_DATA_FILES)
+GENERATED_CRYPTO_DATA_FILES += $(GENERATED_PSA_CONFIG_DATA_FILES)
 
-GENERATED_ECP_DATA_FILES := $(patsubst tests/%,%,$(shell \
+GENERATED_ECP_DATA_FILES := $(addprefix ../tf-psa-crypto/,$(shell \
 	$(PYTHON) ../framework/scripts/generate_ecp_tests.py --list || \
 	echo FAILED \
 ))
 ifeq ($(GENERATED_ECP_DATA_FILES),FAILED)
 $(error "$(PYTHON) ../framework/scripts/generate_ecp_tests.py --list" failed)
 endif
-GENERATED_DATA_FILES += $(GENERATED_ECP_DATA_FILES)
+GENERATED_CRYPTO_DATA_FILES += $(GENERATED_ECP_DATA_FILES)
 
-GENERATED_PSA_DATA_FILES := $(patsubst tests/%,%,$(shell \
+GENERATED_PSA_DATA_FILES := $(addprefix ../tf-psa-crypto/,$(shell \
 	$(PYTHON) ../framework/scripts/generate_psa_tests.py --list || \
 	echo FAILED \
 ))
 ifeq ($(GENERATED_PSA_DATA_FILES),FAILED)
 $(error "$(PYTHON) ../framework/scripts/generate_psa_tests.py --list" failed)
 endif
-GENERATED_DATA_FILES += $(GENERATED_PSA_DATA_FILES)
+GENERATED_CRYPTO_DATA_FILES += $(GENERATED_PSA_DATA_FILES)
 
-GENERATED_FILES = $(GENERATED_DATA_FILES)
+GENERATED_FILES = $(GENERATED_DATA_FILES) $(GENERATED_CRYPTO_DATA_FILES)
 GENERATED_FILES += src/test_keys.h src/test_certs.h
 
 .PHONY: generated_files
@@ -75,7 +78,7 @@ generated_bignum_test_data: ../framework/scripts/mbedtls_framework/test_case.py
 generated_bignum_test_data: ../framework/scripts/mbedtls_framework/test_data_generation.py
 generated_bignum_test_data:
 	echo "  Gen   $(GENERATED_BIGNUM_DATA_FILES)"
-	$(PYTHON) ../framework/scripts/generate_bignum_tests.py
+	$(PYTHON) ../framework/scripts/generate_bignum_tests.py --directory ../tf-psa-crypto/tests/suites
 .SECONDARY: generated_bignum_test_data
 
 # We deliberately omit the configuration files (mbedtls_config.h,
@@ -91,7 +94,8 @@ generated_config_test_data: ../framework/scripts/mbedtls_framework/test_case.py
 generated_config_test_data: ../framework/scripts/mbedtls_framework/test_data_generation.py
 generated_config_test_data:
 	echo "  Gen   $(GENERATED_CONFIG_DATA_FILES)"
-	$(PYTHON) ../framework/scripts/generate_config_tests.py
+	$(PYTHON) ../framework/scripts/generate_config_tests.py $(GENERATED_MBEDTLS_CONFIG_DATA_FILES)
+	$(PYTHON) ../framework/scripts/generate_config_tests.py --directory ../tf-psa-crypto/tests/suites $(GENERATED_PSA_CONFIG_DATA_FILES)
 .SECONDARY: generated_config_test_data
 
 $(GENERATED_ECP_DATA_FILES): $(gen_file_dep) generated_ecp_test_data
@@ -102,7 +106,7 @@ generated_ecp_test_data: ../framework/scripts/mbedtls_framework/test_case.py
 generated_ecp_test_data: ../framework/scripts/mbedtls_framework/test_data_generation.py
 generated_ecp_test_data:
 	echo "  Gen   $(GENERATED_ECP_DATA_FILES)"
-	$(PYTHON) ../framework/scripts/generate_ecp_tests.py
+	$(PYTHON) ../framework/scripts/generate_ecp_tests.py --directory ../tf-psa-crypto/tests/suites
 .SECONDARY: generated_ecp_test_data
 
 $(GENERATED_PSA_DATA_FILES): $(gen_file_dep) generated_psa_test_data
@@ -122,29 +126,35 @@ generated_psa_test_data: ../framework/scripts/mbedtls_framework/test_data_genera
 #generated_psa_test_data: ../tf-psa-crypto/include/psa/crypto_config.h
 generated_psa_test_data: ../tf-psa-crypto/include/psa/crypto_values.h
 generated_psa_test_data: ../tf-psa-crypto/include/psa/crypto_extra.h
-generated_psa_test_data: suites/test_suite_psa_crypto_metadata.data
+generated_psa_test_data: ../tf-psa-crypto/tests/suites/test_suite_psa_crypto_metadata.data
 generated_psa_test_data:
 	echo "  Gen   $(GENERATED_PSA_DATA_FILES) ..."
-	$(PYTHON) ../framework/scripts/generate_psa_tests.py
+	$(PYTHON) ../framework/scripts/generate_psa_tests.py --directory ../tf-psa-crypto/tests/suites
 .SECONDARY: generated_psa_test_data
 
 # A test application is built for each suites/test_suite_*.data file.
 # Application name is same as .data file's base name and can be
 # constructed by stripping path 'suites/' and extension .data.
-DATA_FILES := $(wildcard suites/test_suite_*.data)
+DATA_FILES = $(filter-out $(GENERATED_DATA_FILES), $(wildcard suites/test_suite_*.data))
+CRYPTO_DATA_FILES = $(filter-out $(GENERATED_CRYPTO_DATA_FILES), $(wildcard ../tf-psa-crypto/tests/suites/test_suite_*.data))
+
 # Make sure that generated data files are included even if they don't
 # exist yet when the makefile is parsed.
-DATA_FILES += $(filter-out $(DATA_FILES),$(GENERATED_DATA_FILES))
+DATA_FILES += $(GENERATED_DATA_FILES)
+CRYPTO_DATA_FILES += $(GENERATED_CRYPTO_DATA_FILES)
+
 APPS = $(basename $(subst suites/,,$(DATA_FILES)))
+CRYPTO_APPS = $(basename $(subst suites/,,$(CRYPTO_DATA_FILES)))
 
 # Construct executable name by adding OS specific suffix $(EXEXT).
 BINARIES := $(addsuffix $(EXEXT),$(APPS))
+CRYPTO_BINARIES := $(addsuffix $(EXEXT),$(CRYPTO_APPS))
 
 .SILENT:
 
 .PHONY: all check test clean
 
-all: $(BINARIES)
+all: $(BINARIES) $(CRYPTO_BINARIES)
 
 mbedtls_test: $(MBEDTLS_TEST_OBJS)
 
@@ -177,7 +187,7 @@ src/test_helpers/%.o : src/test_helpers/%.c
 	echo "  CC    $<"
 	$(CC) $(LOCAL_CFLAGS) $(CFLAGS) -o $@ -c $<
 
-C_FILES := $(addsuffix .c,$(APPS))
+C_FILES := $(addsuffix .c,$(APPS)) $(addsuffix .c,$(CRYPTO_APPS))
 c: $(C_FILES)
 
 # Wildcard target for test code generation:
@@ -195,9 +205,19 @@ c: $(C_FILES)
 # dot in .c file's base name.
 #
 .SECONDEXPANSION:
-%.c: suites/$$(firstword $$(subst ., ,$$*)).function suites/%.data ../framework/scripts/generate_test_code.py suites/helpers.function suites/main_test.function suites/host_test.function
+%.c: suites/$$(firstword $$(subst ., ,$$*)).function suites/%.data ../framework/scripts/generate_test_code.py ../tf-psa-crypto/tests/suites/helpers.function ../tf-psa-crypto/tests/suites/main_test.function ../tf-psa-crypto/tests/suites/host_test.function
 	echo "  Gen   $@"
 	$(PYTHON) ../framework/scripts/generate_test_code.py -f suites/$(firstword $(subst ., ,$*)).function \
+		-d suites/$*.data \
+		-t ../tf-psa-crypto/tests/suites/main_test.function \
+		-p ../tf-psa-crypto/tests/suites/host_test.function \
+		-s suites  \
+		--helpers-file ../tf-psa-crypto/tests/suites/helpers.function \
+		-o .
+
+../tf-psa-crypto/tests/%.c: ../tf-psa-crypto/tests/suites/$$(firstword $$(subst ., ,$$*)).function ../tf-psa-crypto/tests/suites/%.data ../framework/scripts/generate_test_code.py ../tf-psa-crypto/tests/suites/helpers.function ../tf-psa-crypto/tests/suites/main_test.function ../tf-psa-crypto/tests/suites/host_test.function
+	echo "  Gen   $@"
+	cd ../tf-psa-crypto/tests && $(PYTHON) ../../framework/scripts/generate_test_code.py -f suites/$(firstword $(subst ., ,$*)).function \
 		-d suites/$*.data \
 		-t suites/main_test.function \
 		-p suites/host_test.function \
@@ -205,15 +225,23 @@ c: $(C_FILES)
 		--helpers-file suites/helpers.function \
 		-o .
 
-
 $(BINARIES): %$(EXEXT): %.c $(MBEDLIBS) $(TEST_OBJS_DEPS) $(MBEDTLS_TEST_OBJS)
-	echo "  CC    $<"
+	echo "  CC   $<"
 	$(CC) $(LOCAL_CFLAGS) $(CFLAGS) $< $(LOCAL_LDFLAGS) $(LDFLAGS) -o $@
 
+LOCAL_CRYPTO_CFLAGS = $(patsubst -I./include, -I../../tests/include, $(patsubst -I../%,-I../../%, $(LOCAL_CFLAGS)))
+LOCAL_CRYPTO_LDFLAGS = $(patsubst -L../library, -L../../library, \
+                       $(patsubst -L../tests/%, -L../../tests/%, \
+                       $(patsubst ./src/%,../../tests/src/%, $(LOCAL_LDFLAGS))))
+$(CRYPTO_BINARIES): %$(EXEXT): %.c $(MBEDLIBS) $(TEST_OBJS_DEPS) $(MBEDTLS_TEST_OBJS)
+	echo "  CC   $<"
+	cd ../tf-psa-crypto/tests && $(CC) $(LOCAL_CRYPTO_CFLAGS) $(CFLAGS) $(subst $(EXEXT),,$(@F)).c $(LOCAL_CRYPTO_LDFLAGS) $(LDFLAGS) -o $(@F)
+
 clean:
 ifndef WINDOWS
 	$(MAKE) -C psa-client-server/psasim clean
 	rm -rf $(BINARIES) *.c *.datax
+	rm -rf $(CRYPTO_BINARIES) ../tf-psa-crypto/tests/*.c ../tf-psa-crypto/tests/*.datax
 	rm -f src/*.o src/drivers/*.o src/test_helpers/*.o src/libmbed* src/test_keys.h src/test_certs.h
 	rm -f src/test_keys.h src/test_certs.h
 	rm -f include/test/instrument_record_status.h
@@ -225,6 +253,9 @@ else
 	if exist *.c del /Q /F *.c
 	if exist *.exe del /Q /F *.exe
 	if exist *.datax del /Q /F *.datax
+	if exist ../tf-psa-crypto/tests/*.c del /Q /F ../tf-psa-crypto/tests/*.c
+	if exist ../tf-psa-crypto/tests/*.exe del /Q /F ../tf-psa-crypto/tests/*.exe
+	if exist ../tf-psa-crypto/tests/*.datax del /Q /F ../tf-psa-crypto/tests/*.datax
 	if exist src/*.o del /Q /F src/*.o
 	if exist src/drivers/*.o del /Q /F src/drivers/*.o
 	if exist src/test_keys.h del /Q /F src/test_keys.h
@@ -235,8 +266,9 @@ else
 endif
 
 # Test suites caught by SKIP_TEST_SUITES are built but not executed.
-check: $(BINARIES)
+check: $(BINARIES) $(CRYPTO_BINARIES)
 	perl scripts/run-test-suites.pl $(TEST_FLAGS) --skip=$(SKIP_TEST_SUITES)
+	cd ../tf-psa-crypto/tests && perl ../../tests/scripts/run-test-suites.pl $(TEST_FLAGS) --skip=$(SKIP_TEST_SUITES)
 
 test: check
 

tests/include/test/psa_test_wrappers.h

@@ -17,7 +17,6 @@ extern "C" {
     !defined(RECORD_PSA_STATUS_COVERAGE_LOG)
 
 #include <psa/crypto.h>
-
 #include <test/memory.h>
 #include <test/psa_crypto_helpers.h>
 #include <test/psa_test_wrappers.h>

tests/psa-client-server/psasim/src/psa_ff_server.c

@@ -605,9 +605,9 @@ void __init_psasim(const char **array,
     key_t key;
     int qid;
     FILE *fp;
-    char doorbell_path[PATHNAMESIZE] = { 0 };
+    char doorbell_file[PATHNAMESIZE] = { 0 };
     char queue_path[PATHNAMESIZE];
-    snprintf(doorbell_path, PATHNAMESIZE, TMP_FILE_BASE_PATH "psa_notify_%u", getpid());
+    snprintf(doorbell_file, PATHNAMESIZE, "psa_notify_%u", getpid());
 
     if (library_initialised > 0) {
         return;
@@ -619,7 +619,7 @@ void __init_psasim(const char **array,
         FATAL("Unsupported value. Aborting.");
     }
 
-    array[3] = doorbell_path;
+    array[3] = doorbell_file;
 
     for (int i = 0; i < 32; i++) {
         if (strncmp(array[i], "", 1) != 0) {

tests/psa-client-server/psasim/test/kill_server.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-# Copyright The Mbed TLS Contributors
-# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
-
-set -e
-
-pkill psa_server || true
-
-# Remove temporary files and logs
-rm -f psa_notify_*
-rm -f psa_service_*
-rm -f psa_server.log
-
-# Remove all IPCs
-ipcs -q | awk '{ printf " -q " $2 }' | xargs ipcrm > /dev/null 2>&1 || true

tests/psa-client-server/psasim/test/kill_servers.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+
+set -e
+
+pkill psa_server || true
+
+# Remove all IPCs
+# Not just ipcrm -all=msg as it is not supported on macOS.
+# Filter out header and empty lines, choosing to select based on keys being
+# output in hex.
+ipcs -q | fgrep 0x | awk '{ printf " -q " $2 }' | xargs ipcrm > /dev/null 2>&1 || true

tests/psa-client-server/psasim/test/start_server.sh

@@ -13,7 +13,5 @@ function wait_for_server_startup() {
     done
 }
 
-$(dirname "$0")/kill_server.sh
-
 $(dirname "$0")/psa_server &
 wait_for_server_startup

tests/scripts/all.sh

@@ -343,6 +343,7 @@ cleanup()
     rm -f include/Makefile programs/!(fuzz)/Makefile
     rm -f tf-psa-crypto/Makefile tf-psa-crypto/include/Makefile
     rm -f tf-psa-crypto/core/Makefile tf-psa-crypto/drivers/Makefile
+    rm -f tf-psa-crypto/tests/Makefile
     rm -f tf-psa-crypto/drivers/everest/Makefile
     rm -f tf-psa-crypto/drivers/p256-m/Makefile
     rm -f tf-psa-crypto/drivers/builtin/Makefile
@@ -1097,14 +1098,27 @@ helper_psasim_server() {
     if [ "$OPERATION" == "start" ]; then
     (
         cd tests
-        msg "start server"
+        msg "start server in tests"
         psa-client-server/psasim/test/start_server.sh
+        msg "start server in tf-psa-crypto/tests"
+        cd ../tf-psa-crypto/tests
+        ../../tests/psa-client-server/psasim/test/start_server.sh
     )
     else
     (
+        msg "terminate servers and cleanup"
+        tests/psa-client-server/psasim//test/kill_servers.sh
+
+        # Remove temporary files and logs
         cd tests
-        msg "terminate server and cleanup"
+        rm -f psa_notify_*
-        psa-client-server/psasim//test/kill_server.sh
+        rm -f psa_service_*
+        rm -f psa_server.log
+
+        cd ../tf-psa-crypto/tests
+        rm -f psa_notify_*
+        rm -f psa_service_*
+        rm -f psa_server.log
     )
     fi
 }
@@ -1206,7 +1220,8 @@ component_check_test_dependencies () {
 
     # Find legacy dependencies in PSA tests
     grep 'depends_on' \
-        tests/suites/test_suite_psa*.data tests/suites/test_suite_psa*.function |
+        tf-psa-crypto/tests/suites/test_suite_psa*.data \
+        tf-psa-crypto/tests/suites/test_suite_psa*.function |
         grep -Eo '!?MBEDTLS_[^: ]*' |
         grep -v -e MBEDTLS_PSA_ -e MBEDTLS_TEST_ |
         sort -u > $found
@@ -1446,7 +1461,7 @@ component_test_ref_configs () {
     # dependency resolution for generated files and just rely on them being
     # present (thanks to pre_generate_files) by turning GEN_FILES off.
     CC=$ASAN_CC cmake -D GEN_FILES=Off -D CMAKE_BUILD_TYPE:String=Asan .
-    tests/scripts/test-ref-configs.pl
+    tests/scripts/test-ref-configs.pl config-tfm.h
 }
 
 component_test_no_renegotiation () {
@@ -2081,6 +2096,9 @@ skip_suites_without_constant_flow () {
     # secret from macros or functions inside tests/include or tests/src.
     SKIP_TEST_SUITES=$(
         git -C tests/suites grep -L TEST_CF_ 'test_suite_*.function' |
+            sed 's/test_suite_//; s/\.function$//' |
+            tr '\n' ,),$(
+        git -C tf-psa-crypto/tests/suites grep -L TEST_CF_ 'test_suite_*.function' |
             sed 's/test_suite_//; s/\.function$//' |
             tr '\n' ,)
     export SKIP_TEST_SUITES
@@ -2092,6 +2110,10 @@ skip_all_except_given_suite () {
         ls -1 tests/suites/test_suite_*.function |
         grep -v $1.function |
          sed 's/tests.suites.test_suite_//; s/\.function$//' |
+        tr '\n' ,),$(
+        ls -1 tf-psa-crypto/tests/suites/test_suite_*.function |
+        grep -v $1.function |
+         sed 's/tf-psa-crypto.tests.suites.test_suite_//; s/\.function$//' |
         tr '\n' ,)
     export SKIP_TEST_SUITES
 }
@@ -4511,13 +4533,13 @@ END
 
     msg "all loops unrolled"
     make clean
-    make -C tests test_suite_shax CFLAGS="-DMBEDTLS_SHA3_THETA_UNROLL=1 -DMBEDTLS_SHA3_PI_UNROLL=1 -DMBEDTLS_SHA3_CHI_UNROLL=1 -DMBEDTLS_SHA3_RHO_UNROLL=1"
+    make -C tests ../tf-psa-crypto/tests/test_suite_shax CFLAGS="-DMBEDTLS_SHA3_THETA_UNROLL=1 -DMBEDTLS_SHA3_PI_UNROLL=1 -DMBEDTLS_SHA3_CHI_UNROLL=1 -DMBEDTLS_SHA3_RHO_UNROLL=1"
-    ./tests/test_suite_shax
+    ./tf-psa-crypto/tests/test_suite_shax
 
     msg "all loops rolled up"
     make clean
-    make -C tests test_suite_shax CFLAGS="-DMBEDTLS_SHA3_THETA_UNROLL=0 -DMBEDTLS_SHA3_PI_UNROLL=0 -DMBEDTLS_SHA3_CHI_UNROLL=0 -DMBEDTLS_SHA3_RHO_UNROLL=0"
+    make -C tests ../tf-psa-crypto/tests/test_suite_shax CFLAGS="-DMBEDTLS_SHA3_THETA_UNROLL=0 -DMBEDTLS_SHA3_PI_UNROLL=0 -DMBEDTLS_SHA3_CHI_UNROLL=0 -DMBEDTLS_SHA3_RHO_UNROLL=0"
-    ./tests/test_suite_shax
+    ./tf-psa-crypto/tests/test_suite_shax
 }
 
 support_test_aesni_m32() {
@@ -6085,6 +6107,7 @@ component_test_suite_with_psasim()
     msg "build test suites"
     make PSASIM=1 CFLAGS="$ASAN_CFLAGS" LDFLAGS="$ASAN_CFLAGS" tests
 
+    helper_psasim_server kill
     helper_psasim_server start
 
     # psasim takes an extremely long execution time on some test suites so we
@@ -6148,6 +6171,14 @@ run_component () {
     esac
     "${dd_cmd[@]}"
 
+    if [ -d tf-psa-crypto ]; then
+        dd_cmd=(dd if=/dev/urandom of=./tf-psa-crypto/tests/seedfile bs=64 count=1)
+        case $OSTYPE in
+            linux*|freebsd*|openbsd*) dd_cmd+=(status=none)
+        esac
+        "${dd_cmd[@]}"
+    fi
+
     # Run the component in a subshell, with error trapping and output
     # redirection set up based on the relevant options.
     if [ $KEEP_GOING -eq 1 ]; then

tests/scripts/basic-build-test.sh

@@ -35,6 +35,8 @@ if [ -d library -a -d include -a -d tests ]; then :; else
     exit 1
 fi
 
+MBEDTLS_ROOT_DIR="$PWD"
+
 : ${OPENSSL:="openssl"}
 : ${GNUTLS_CLI:="gnutls-cli"}
 : ${GNUTLS_SERV:="gnutls-serv"}
@@ -74,12 +76,16 @@ export LDFLAGS=' --coverage'
 make clean
 cp "$CONFIG_H" "$CONFIG_BAK"
 scripts/config.py full
-make
+make -j
 
 
 # Step 2 - Execute the tests
 TEST_OUTPUT=out_${PPID}
-cd tests
+cd $MBEDTLS_ROOT_DIR/tests
+if [ ! -f "seedfile" ]; then
+    dd if=/dev/urandom of="seedfile" bs=64 count=1
+fi
+cd $MBEDTLS_ROOT_DIR/tf-psa-crypto/tests
 if [ ! -f "seedfile" ]; then
     dd if=/dev/urandom of="seedfile" bs=64 count=1
 fi
@@ -87,10 +93,14 @@ echo
 
 # Step 2a - Unit Tests (keep going even if some tests fail)
 echo '################ Unit tests ################'
-perl scripts/run-test-suites.pl -v 2 |tee unit-test-$TEST_OUTPUT
+cd $MBEDTLS_ROOT_DIR/tests
+perl scripts/run-test-suites.pl -v 2 |tee tls-x509-unit-test-$TEST_OUTPUT
+cd $MBEDTLS_ROOT_DIR/tf-psa-crypto/tests
+perl $MBEDTLS_ROOT_DIR/tests/scripts/run-test-suites.pl -v 2 |tee ../../tests/crypto-unit-test-$TEST_OUTPUT
 echo '^^^^^^^^^^^^^^^^ Unit tests ^^^^^^^^^^^^^^^^'
 echo
 
+cd $MBEDTLS_ROOT_DIR/tests
 # Step 2b - System Tests (keep going even if some tests fail)
 echo
 echo '################ ssl-opt.sh ################'
@@ -141,13 +151,13 @@ rm -f "tests/basic-build-test-$$.ok"
 
     cd tests
 
-    # Step 4a - Unit tests
+    # Step 4a - TLS and x509 unit tests
-    echo "Unit tests - tests/scripts/run-test-suites.pl"
+    echo "TLS and x509 unit tests - tests/scripts/run-test-suites.pl"
 
-    PASSED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/test cases passed :[\t]*\([0-9]*\)/\1/p'| tr -d ' ')
+    PASSED_TESTS=$(tail -n6 tls-x509-unit-test-$TEST_OUTPUT|sed -n -e 's/test cases passed :[\t]*\([0-9]*\)/\1/p'| tr -d ' ')
-    SKIPPED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/skipped :[ \t]*\([0-9]*\)/\1/p'| tr -d ' ')
+    SKIPPED_TESTS=$(tail -n6 tls-x509-unit-test-$TEST_OUTPUT|sed -n -e 's/skipped :[ \t]*\([0-9]*\)/\1/p'| tr -d ' ')
-    TOTAL_SUITES=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) .*, [0-9]* tests run)/\1/p'| tr -d ' ')
+    TOTAL_SUITES=$(tail -n6 tls-x509-unit-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) .*, [0-9]* tests run)/\1/p'| tr -d ' ')
-    FAILED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/failed :[\t]*\([0-9]*\)/\1/p' |tr -d ' ')
+    FAILED_TESTS=$(tail -n6 tls-x509-unit-test-$TEST_OUTPUT|sed -n -e 's/failed :[\t]*\([0-9]*\)/\1/p' |tr -d ' ')
 
     echo "No test suites     : $TOTAL_SUITES"
     echo "Passed             : $PASSED_TESTS"
@@ -163,7 +173,29 @@ rm -f "tests/basic-build-test-$$.ok"
     TOTAL_AVAIL=$(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS))
     TOTAL_EXED=$(($PASSED_TESTS + $FAILED_TESTS))
 
-    # Step 4b - TLS Options tests
+    # Step 4b - Crypto unit tests
+    echo "Crypto unit tests - tests/scripts/run-test-suites.pl"
+
+    PASSED_TESTS=$(tail -n6 crypto-unit-test-$TEST_OUTPUT|sed -n -e 's/test cases passed :[\t]*\([0-9]*\)/\1/p'| tr -d ' ')
+    SKIPPED_TESTS=$(tail -n6 crypto-unit-test-$TEST_OUTPUT|sed -n -e 's/skipped :[ \t]*\([0-9]*\)/\1/p'| tr -d ' ')
+    TOTAL_SUITES=$(tail -n6 crypto-unit-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) .*, [0-9]* tests run)/\1/p'| tr -d ' ')
+    FAILED_TESTS=$(tail -n6 crypto-unit-test-$TEST_OUTPUT|sed -n -e 's/failed :[\t]*\([0-9]*\)/\1/p' |tr -d ' ')
+
+    echo "No test suites     : $TOTAL_SUITES"
+    echo "Passed             : $PASSED_TESTS"
+    echo "Failed             : $FAILED_TESTS"
+    echo "Skipped            : $SKIPPED_TESTS"
+    echo "Total exec'd tests : $(($PASSED_TESTS + $FAILED_TESTS))"
+    echo "Total avail tests  : $(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS))"
+    echo
+
+    TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS))
+    TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS))
+    TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS))
+    TOTAL_AVAIL=$(($TOTAL_AVAIL + $PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS))
+    TOTAL_EXED=$(($TOTAL_EXED + $PASSED_TESTS + $FAILED_TESTS))
+
+    # Step 4c - TLS Options tests
     echo "TLS Options tests - tests/ssl-opt.sh"
 
     PASSED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p')
@@ -185,7 +217,7 @@ rm -f "tests/basic-build-test-$$.ok"
     TOTAL_EXED=$(($TOTAL_EXED + $TOTAL_TESTS))
 
 
-    # Step 4c - System Compatibility tests
+    # Step 4d - System Compatibility tests
     echo "System/Compatibility tests - tests/compat.sh"
 
     PASSED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }')
@@ -207,7 +239,7 @@ rm -f "tests/basic-build-test-$$.ok"
     TOTAL_EXED=$(($TOTAL_EXED + $EXED_TESTS))
 
 
-    # Step 4d - Grand totals
+    # Step 4e - Grand totals
     echo "-------------------------------------------------------------------------"
     echo "Total tests"
 
@@ -219,12 +251,13 @@ rm -f "tests/basic-build-test-$$.ok"
     echo
 
 
-    # Step 4e - Coverage report
+    # Step 4f - Coverage report
     echo "Coverage statistics:"
     sed -n '1,/^Overall coverage/d; /%/p' cov-$TEST_OUTPUT
     echo
 
-    rm unit-test-$TEST_OUTPUT
+    rm tls-x509-unit-test-$TEST_OUTPUT
+    rm crypto-unit-test-$TEST_OUTPUT
     rm sys-test-$TEST_OUTPUT
     rm compat-test-$TEST_OUTPUT
     rm cov-$TEST_OUTPUT

tests/scripts/check-generated-files.sh

@@ -89,7 +89,14 @@ check()
         fi
     done
 
-    "$SCRIPT"
+    # In the case of the config tests, generate only the files to be checked
+    # by the caller as they are divided into Mbed TLS and TF-PSA-Crypto
+    # specific ones.
+    if [ "${SCRIPT##*/}" = "generate_config_tests.py" ]; then
+        "$SCRIPT" "$@"
+    else
+        "$SCRIPT"
+    fi
 
     # Compare the script output to the old files and remove backups
     for FILE in "$@"; do
@@ -134,11 +141,29 @@ check()
 #   - scripts/make_generated_files.bat (to generate them under Windows)
 
 # These checks are common to Mbed TLS and TF-PSA-Crypto
+
+# The first case is temporary for the hybrid situation with a tf-psa-crypto
+# directory in Mbed TLS that is not just a TF-PSA-Crypto submodule.
+if [ -d tf-psa-crypto ]; then
+    cd tf-psa-crypto
+    check ../framework/scripts/generate_bignum_tests.py $(../framework/scripts/generate_bignum_tests.py --list)
+    check ../framework/scripts/generate_config_tests.py tests/suites/test_suite_config.psa_boolean.data
+    check ../framework/scripts/generate_ecp_tests.py $(../framework/scripts/generate_ecp_tests.py --list)
+    check ../framework/scripts/generate_psa_tests.py $(../framework/scripts/generate_psa_tests.py --list)
+    cd ..
+    check framework/scripts/generate_config_tests.py tests/suites/test_suite_config.mbedtls_boolean.data
+else
+    check framework/scripts/generate_bignum_tests.py $(framework/scripts/generate_bignum_tests.py --list)
+    if in_tf_psa_crypto_repo; then
+        check framework/scripts/generate_config_tests.py tests/suites/test_suite_config.psa_boolean.data
+    else
+        check framework/scripts/generate_config_tests.py tests/suites/test_suite_config.mbedtls_boolean.data
+    fi
+    check framework/scripts/generate_ecp_tests.py $(framework/scripts/generate_ecp_tests.py --list)
+    check framework/scripts/generate_psa_tests.py $(framework/scripts/generate_psa_tests.py --list)
+fi
+
 check scripts/generate_psa_constants.py programs/psa/psa_constant_names_generated.c
-check framework/scripts/generate_bignum_tests.py $(framework/scripts/generate_bignum_tests.py --list)
-check framework/scripts/generate_config_tests.py $(framework/scripts/generate_config_tests.py --list)
-check framework/scripts/generate_ecp_tests.py $(framework/scripts/generate_ecp_tests.py --list)
-check framework/scripts/generate_psa_tests.py $(framework/scripts/generate_psa_tests.py --list)
 check framework/scripts/generate_test_keys.py tests/src/test_keys.h
 check scripts/generate_driver_wrappers.py ${crypto_core_dir}/psa_crypto_driver_wrappers.h \
                                           ${crypto_core_dir}/psa_crypto_driver_wrappers_no_static.c

tests/scripts/check_test_cases.py

@@ -16,6 +16,9 @@ import re
 import subprocess
 import sys
 
+import scripts_path # pylint: disable=unused-import
+from mbedtls_framework import build_tree
+
 class ScriptOutputError(ValueError):
     """A kind of ValueError that indicates we found
     the script doesn't list test cases in an expected
@@ -130,13 +133,10 @@ option"""
     @staticmethod
     def collect_test_directories():
         """Get the relative path for the TLS and Crypto test directories."""
-        if os.path.isdir('tests'):
+        mbedtls_root = build_tree.guess_mbedtls_root()
-            tests_dir = 'tests'
+        directories = [os.path.join(mbedtls_root, 'tests'),
-        elif os.path.isdir('suites'):
+                       os.path.join(mbedtls_root, 'tf-psa-crypto', 'tests')]
-            tests_dir = '.'
+        directories = [os.path.relpath(p) for p in directories]
-        elif os.path.isdir('../suites'):
-            tests_dir = '..'
-        directories = [tests_dir]
         return directories
 
     def walk_all(self):
@@ -149,7 +149,8 @@ option"""
 
             for sh_file in ['ssl-opt.sh', 'compat.sh']:
                 sh_file = os.path.join(directory, sh_file)
-                self.collect_from_script(sh_file)
+                if os.path.isfile(sh_file):
+                    self.collect_from_script(sh_file)
 
 class TestDescriptions(TestDescriptionExplorer):
     """Collect the available test cases."""

tests/scripts/run-test-suites.pl

@@ -59,8 +59,15 @@ my $skip_re =
       ')(\z|\.)' );
 
 # in case test suites are linked dynamically
-$ENV{'LD_LIBRARY_PATH'} = '../library';
+if (-d '../../tf-psa-crypto') {
-$ENV{'DYLD_LIBRARY_PATH'} = '../library';
+    $ENV{'LD_LIBRARY_PATH'} = '../../library';
+    $ENV{'DYLD_LIBRARY_PATH'} = '../../library';
+}
+else
+{
+    $ENV{'LD_LIBRARY_PATH'} = '../library';
+    $ENV{'DYLD_LIBRARY_PATH'} = '../library';
+}
 
 my $prefix = $^O eq "MSWin32" ? '' : './';
 

tests/scripts/test_psa_constant_names.py

@@ -156,7 +156,7 @@ class Tests:
             out.write(' PASS\n')
 
 HEADERS = ['psa/crypto.h', 'psa/crypto_extra.h', 'psa/crypto_values.h']
-TEST_SUITES = ['tests/suites/test_suite_psa_crypto_metadata.data']
+TEST_SUITES = ['tf-psa-crypto/tests/suites/test_suite_psa_crypto_metadata.data']
 
 def main():
     parser = argparse.ArgumentParser(description=globals()['__doc__'])

tests/src/psa_test_wrappers.c

@@ -10,7 +10,6 @@
     !defined(RECORD_PSA_STATUS_COVERAGE_LOG)
 
 #include <psa/crypto.h>
-
 #include <test/memory.h>
 #include <test/psa_crypto_helpers.h>
 #include <test/psa_test_wrappers.h>

tf-psa-crypto/tests/.gitignore

@@ -0,0 +1,10 @@
+/test_suite*
+/libtestdriver1/*
+
+###START_GENERATED_FILES###
+# Generated source files
+/suites/*.generated.data
+/suites/test_suite_config.psa_boolean.data
+/suites/test_suite_psa_crypto_storage_format.v[0-9]*.data
+/suites/test_suite_psa_crypto_storage_format.current.data
+###END_GENERATED_FILES###

tf-psa-crypto/tests/CMakeLists.txt

@@ -0,0 +1,362 @@
+set(libs
+    ${mbedtls_target}
+    ${CMAKE_THREAD_LIBS_INIT}
+)
+
+# Set the project root directory if it's not already defined, as may happen if
+# the tests folder is included directly by a parent project, without including
+# the top level CMakeLists.txt.
+if(NOT DEFINED MBEDTLS_DIR)
+    set(MBEDTLS_DIR ${CMAKE_SOURCE_DIR})
+endif()
+
+if(NOT MBEDTLS_PYTHON_EXECUTABLE)
+    message(FATAL_ERROR "Cannot build test suites without Python 3")
+endif()
+
+# generated .data files will go there
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/suites)
+
+# Get base names for generated files
+execute_process(
+    COMMAND
+        ${MBEDTLS_PYTHON_EXECUTABLE}
+        ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/generate_bignum_tests.py
+        --list-for-cmake
+    WORKING_DIRECTORY
+        ${CMAKE_CURRENT_SOURCE_DIR}/..
+    OUTPUT_VARIABLE
+        base_bignum_generated_data_files)
+string(REGEX REPLACE "[^;]*/" ""
+       base_bignum_generated_data_files "${base_bignum_generated_data_files}")
+
+execute_process(
+    COMMAND
+        ${MBEDTLS_PYTHON_EXECUTABLE}
+        ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/generate_config_tests.py
+        --list-for-cmake
+    WORKING_DIRECTORY
+        ${CMAKE_CURRENT_SOURCE_DIR}/..
+    OUTPUT_VARIABLE
+        base_config_generated_data_files_raw)
+string(REGEX REPLACE "[^;]*/" ""
+       base_config_generated_data_files_raw
+       "${base_config_generated_data_files_raw}")
+# Can be replace by list(FILTER ...) when CI CMake version is >=3.6
+foreach(file ${base_config_generated_data_files_raw})
+    if(${file} MATCHES "psa")
+        list(APPEND base_config_generated_data_files ${file})
+    endif()
+endforeach()
+
+execute_process(
+    COMMAND
+        ${MBEDTLS_PYTHON_EXECUTABLE}
+        ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/generate_ecp_tests.py
+        --list-for-cmake
+    WORKING_DIRECTORY
+        ${CMAKE_CURRENT_SOURCE_DIR}/..
+    OUTPUT_VARIABLE
+        base_ecp_generated_data_files)
+string(REGEX REPLACE "[^;]*/" ""
+       base_ecp_generated_data_files "${base_ecp_generated_data_files}")
+
+execute_process(
+    COMMAND
+        ${MBEDTLS_PYTHON_EXECUTABLE}
+        ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/generate_psa_tests.py
+        --list-for-cmake
+    WORKING_DIRECTORY
+        ${CMAKE_CURRENT_SOURCE_DIR}/..
+    OUTPUT_VARIABLE
+        base_psa_generated_data_files)
+string(REGEX REPLACE "[^;]*/" ""
+       base_psa_generated_data_files "${base_psa_generated_data_files}")
+
+# Derive generated file paths in the build directory. The generated data
+# files go into the suites/ subdirectory.
+set(base_generated_data_files
+    ${base_bignum_generated_data_files} ${base_config_generated_data_files}
+    ${base_ecp_generated_data_files} ${base_psa_generated_data_files})
+string(REGEX REPLACE "([^;]+)" "suites/\\1"
+       all_generated_data_files "${base_generated_data_files}")
+set(bignum_generated_data_files "")
+set(config_generated_data_files "")
+set(ecp_generated_data_files "")
+set(psa_generated_data_files "")
+foreach(file ${base_bignum_generated_data_files})
+    list(APPEND bignum_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
+endforeach()
+foreach(file ${base_config_generated_data_files})
+    list(APPEND config_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
+endforeach()
+foreach(file ${base_ecp_generated_data_files})
+    list(APPEND ecp_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
+endforeach()
+foreach(file ${base_psa_generated_data_files})
+    list(APPEND psa_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
+endforeach()
+
+if(GEN_FILES)
+    add_custom_command(
+        OUTPUT
+            ${bignum_generated_data_files}
+        WORKING_DIRECTORY
+            ${CMAKE_CURRENT_SOURCE_DIR}/..
+        COMMAND
+            ${MBEDTLS_PYTHON_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/generate_bignum_tests.py
+            --directory ${CMAKE_CURRENT_BINARY_DIR}/suites
+        DEPENDS
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/generate_bignum_tests.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/bignum_common.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/bignum_core.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/bignum_mod_raw.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/bignum_mod.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/test_case.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/test_data_generation.py
+    )
+    add_custom_command(
+        OUTPUT
+            ${config_generated_data_files}
+        WORKING_DIRECTORY
+            ${CMAKE_CURRENT_SOURCE_DIR}/..
+        COMMAND
+            ${MBEDTLS_PYTHON_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/generate_config_tests.py
+            --directory ${CMAKE_CURRENT_BINARY_DIR}/suites
+            ${config_generated_data_files}
+        DEPENDS
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/generate_config_tests.py
+            # Do not declare the configuration files as dependencies: they
+            # change too often in ways that don't affect the result
+            # ((un)commenting some options).
+    )
+    add_custom_command(
+        OUTPUT
+            ${ecp_generated_data_files}
+        WORKING_DIRECTORY
+            ${CMAKE_CURRENT_SOURCE_DIR}/..
+        COMMAND
+            ${MBEDTLS_PYTHON_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/generate_ecp_tests.py
+            --directory ${CMAKE_CURRENT_BINARY_DIR}/suites
+        DEPENDS
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/generate_ecp_tests.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/bignum_common.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/ecp.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/test_case.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/test_data_generation.py
+    )
+    add_custom_command(
+        OUTPUT
+            ${psa_generated_data_files}
+        WORKING_DIRECTORY
+            ${CMAKE_CURRENT_SOURCE_DIR}/..
+        COMMAND
+            ${MBEDTLS_PYTHON_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/generate_psa_tests.py
+            --directory ${CMAKE_CURRENT_BINARY_DIR}/suites
+        DEPENDS
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/generate_psa_tests.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/crypto_data_tests.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/crypto_knowledge.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/macro_collector.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/psa_information.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/psa_storage.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/test_case.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/mbedtls_framework/test_data_generation.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_config.h
+            ${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_values.h
+            ${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_extra.h
+    )
+
+else()
+    foreach(file ${all_generated_data_files})
+        link_to_source(${file})
+    endforeach()
+endif()
+# CMake generates sub-makefiles for each target and calls them in subprocesses.
+# Without this command, cmake will generate rules in each sub-makefile. As a result,
+# they can cause race conditions in parallel builds.
+# With this line, only 4 sub-makefiles include the above command, that reduces
+# the risk of a race.
+add_custom_target(test_suite_bignum_generated_data DEPENDS ${bignum_generated_data_files})
+add_custom_target(test_suite_tf_psa_crypto_config_generated_data DEPENDS ${config_generated_data_files})
+add_custom_target(test_suite_ecp_generated_data DEPENDS ${ecp_generated_data_files})
+add_custom_target(test_suite_psa_generated_data DEPENDS ${psa_generated_data_files})
+# If SKIP_TEST_SUITES is not defined with -D, get it from the environment.
+if((NOT DEFINED SKIP_TEST_SUITES) AND (DEFINED ENV{SKIP_TEST_SUITES}))
+    set(SKIP_TEST_SUITES $ENV{SKIP_TEST_SUITES})
+endif()
+# Test suites caught by SKIP_TEST_SUITES are built but not executed.
+# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar"
+# but not "test_suite_foobar".
+string(REGEX REPLACE "[ ,;]" "|" SKIP_TEST_SUITES_REGEX "${SKIP_TEST_SUITES}")
+string(REPLACE "." "\\." SKIP_TEST_SUITES_REGEX "${SKIP_TEST_SUITES_REGEX}")
+set(SKIP_TEST_SUITES_REGEX "^(${SKIP_TEST_SUITES_REGEX})(\$|\\.)")
+
+function(add_test_suite suite_name)
+    if(ARGV1)
+        set(data_name ${ARGV1})
+    else()
+        set(data_name ${suite_name})
+    endif()
+
+    # Get the test names of the tests with generated .data files
+    # from the generated_data_files list in parent scope.
+    set(bignum_generated_data_names "")
+    set(config_generated_data_names "")
+    set(ecp_generated_data_names "")
+    set(psa_generated_data_names "")
+    foreach(generated_data_file ${bignum_generated_data_files})
+        # Get the plain filename
+        get_filename_component(generated_data_name ${generated_data_file} NAME)
+        # Remove the ".data" extension
+        get_name_without_last_ext(generated_data_name ${generated_data_name})
+        # Remove leading "test_suite_"
+        string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name)
+        list(APPEND bignum_generated_data_names ${generated_data_name})
+    endforeach()
+    foreach(generated_data_file ${config_generated_data_files})
+        # Get the plain filename
+        get_filename_component(generated_data_name ${generated_data_file} NAME)
+        # Remove the ".data" extension
+        get_name_without_last_ext(generated_data_name ${generated_data_name})
+        # Remove leading "test_suite_"
+        string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name)
+        list(APPEND config_generated_data_names ${generated_data_name})
+    endforeach()
+    foreach(generated_data_file ${ecp_generated_data_files})
+        # Get the plain filename
+        get_filename_component(generated_data_name ${generated_data_file} NAME)
+        # Remove the ".data" extension
+        get_name_without_last_ext(generated_data_name ${generated_data_name})
+        # Remove leading "test_suite_"
+        string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name)
+        list(APPEND ecp_generated_data_names ${generated_data_name})
+    endforeach()
+    foreach(generated_data_file ${psa_generated_data_files})
+        # Get the plain filename
+        get_filename_component(generated_data_name ${generated_data_file} NAME)
+        # Remove the ".data" extension
+        get_name_without_last_ext(generated_data_name ${generated_data_name})
+        # Remove leading "test_suite_"
+        string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name)
+        list(APPEND psa_generated_data_names ${generated_data_name})
+    endforeach()
+
+    if(";${bignum_generated_data_names};" MATCHES ";${data_name};")
+        set(data_file
+            ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
+        set(dependency test_suite_bignum_generated_data)
+    elseif(";${config_generated_data_names};" MATCHES ";${data_name};")
+        set(data_file
+            ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
+        set(dependency test_suite_tf_psa_crypto_config_generated_data)
+    elseif(";${ecp_generated_data_names};" MATCHES ";${data_name};")
+        set(data_file
+            ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
+        set(dependency test_suite_ecp_generated_data)
+    elseif(";${psa_generated_data_names};" MATCHES ";${data_name};")
+        set(data_file
+            ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
+        set(dependency test_suite_psa_generated_data)
+    else()
+        set(data_file
+            ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${data_name}.data)
+        set(dependency
+            test_suite_bignum_generated_data
+            test_suite_tf_psa_crypto_config_generated_data
+            test_suite_ecp_generated_data
+            test_suite_psa_generated_data)
+    endif()
+
+    add_custom_command(
+        OUTPUT
+            # The output filename of generate_test_code.py is derived from the -d
+            # input argument.
+            test_suite_${data_name}.c
+        COMMAND
+            ${MBEDTLS_PYTHON_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/generate_test_code.py
+            -f ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${suite_name}.function
+            -d ${data_file}
+            -t ${CMAKE_CURRENT_SOURCE_DIR}/suites/main_test.function
+            -p ${CMAKE_CURRENT_SOURCE_DIR}/suites/host_test.function
+            -s ${CMAKE_CURRENT_SOURCE_DIR}/suites
+            --helpers-file ${CMAKE_CURRENT_SOURCE_DIR}/suites/helpers.function
+            -o .
+        DEPENDS
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../framework/scripts/generate_test_code.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${suite_name}.function
+            ${data_file}
+            ${CMAKE_CURRENT_SOURCE_DIR}/suites/main_test.function
+            ${CMAKE_CURRENT_SOURCE_DIR}/suites/host_test.function
+            ${CMAKE_CURRENT_SOURCE_DIR}/suites/helpers.function
+            ${mbedtls_target}
+        BYPRODUCTS
+            test_suite_${data_name}.datax
+    )
+
+    add_executable(test_suite_${data_name} test_suite_${data_name}.c
+                   $<TARGET_OBJECTS:mbedtls_test>
+                   $<TARGET_OBJECTS:mbedtls_test_helpers>)
+    add_dependencies(test_suite_${data_name} ${dependency})
+    target_link_libraries(test_suite_${data_name} ${libs})
+    # Include test-specific header files from ./include and private header
+    # files (used by some invasive tests) from ../library. Public header
+    # files are automatically included because the library targets declare
+    # them as PUBLIC.
+    target_include_directories(test_suite_${data_name}
+        PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../tests/include
+        PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../library
+        PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../core
+        PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../drivers/builtin/src)
+    # Request C11, which is needed for memory poisoning tests
+    set_target_properties(test_suite_${data_name} PROPERTIES C_STANDARD 11)
+
+    if(${data_name} MATCHES ${SKIP_TEST_SUITES_REGEX})
+        message(STATUS "The test suite ${data_name} will not be executed.")
+    else()
+        add_test(${data_name}-suite test_suite_${data_name} --verbose)
+    endif()
+endfunction(add_test_suite)
+
+# Enable definition of various functions used throughout the testsuite
+# (gethostname, strdup, fileno...) even when compiling with -std=c99. Harmless
+# on non-POSIX platforms.
+add_definitions("-D_POSIX_C_SOURCE=200809L")
+
+if(CMAKE_COMPILER_IS_CLANG)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wdocumentation -Wno-documentation-deprecated-sync -Wunreachable-code")
+endif(CMAKE_COMPILER_IS_CLANG)
+
+if(MSVC)
+    # If a warning level has been defined, suppress all warnings for test code
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W0")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /WX-")
+endif(MSVC)
+
+file(GLOB test_suites RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" suites/*.data)
+list(APPEND test_suites ${all_generated_data_files})
+# If the generated .data files are present in the source tree, we just added
+# them twice, both through GLOB and through ${all_generated_data_files}.
+list(REMOVE_DUPLICATES test_suites)
+list(SORT test_suites)
+foreach(test_suite ${test_suites})
+    get_filename_component(data_name ${test_suite} NAME)
+    string(REGEX REPLACE "\\.data\$" "" data_name "${data_name}")
+    string(REPLACE "test_suite_" "" data_name "${data_name}")
+    string(REGEX MATCH "[^.]*" function_name "${data_name}")
+    add_test_suite(${function_name} ${data_name})
+endforeach(test_suite)
+
+# Make scripts and data files needed for testing available in an
+# out-of-source build.
+if (NOT ${CMAKE_CURRENT_BINARY_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR})
+    if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/seedfile")
+        link_to_source(seedfile)
+    endif()
+    link_to_source(../../framework/data_files)
+endif()

tf-psa-crypto/tests/suites/test_suite_bignum.misc.data

@@ -110,19 +110,19 @@ Test mbedtls_mpi_write_binary_le #2 (Buffer too small)
 mpi_write_binary_le:"123123123123123123123123123":"23311223311223311223311223":13:MBEDTLS_ERR_MPI_BUFFER_TOO_SMALL
 
 Base test mbedtls_mpi_read_file #1
-mpi_read_file:"../framework/data_files/mpi_16":"01f55332c3a48b910f9942f6c914e58bef37a47ee45cb164a5b6b8d1006bf59a059c21449939ebebfdf517d2e1dbac88010d7b1f141e997bd6801ddaec9d05910f4f2de2b2c4d714e2c14a72fc7f17aa428d59c531627f09":0
+mpi_read_file:"../../framework/data_files/mpi_16":"01f55332c3a48b910f9942f6c914e58bef37a47ee45cb164a5b6b8d1006bf59a059c21449939ebebfdf517d2e1dbac88010d7b1f141e997bd6801ddaec9d05910f4f2de2b2c4d714e2c14a72fc7f17aa428d59c531627f09":0
 
 Test mbedtls_mpi_read_file #1 (Empty file)
-mpi_read_file:"../framework/data_files/hash_file_4":"":MBEDTLS_ERR_MPI_FILE_IO_ERROR
+mpi_read_file:"../../framework/data_files/hash_file_4":"":MBEDTLS_ERR_MPI_FILE_IO_ERROR
 
 Test mbedtls_mpi_read_file #2 (Illegal input)
-mpi_read_file:"../framework/data_files/hash_file_2":"":0
+mpi_read_file:"../../framework/data_files/hash_file_2":"":0
 
 Test mbedtls_mpi_read_file #3 (Input too big)
-mpi_read_file:"../framework/data_files/mpi_too_big":"":MBEDTLS_ERR_MPI_BUFFER_TOO_SMALL
+mpi_read_file:"../../framework/data_files/mpi_too_big":"":MBEDTLS_ERR_MPI_BUFFER_TOO_SMALL
 
 Base test mbedtls_mpi_write_file #1
-mpi_write_file:"941379d00fed1491fe15df284dfde4a142f68aa8d412023195cee66883e6290ffe703f4ea5963bf212713cee46b107c09182b5edcd955adac418bf4918e2889af48e1099d513830cec85c26ac1e158b52620e33ba8692f893efbb2f958b4424":"../framework/data_files/mpi_write"
+mpi_write_file:"941379d00fed1491fe15df284dfde4a142f68aa8d412023195cee66883e6290ffe703f4ea5963bf212713cee46b107c09182b5edcd955adac418bf4918e2889af48e1099d513830cec85c26ac1e158b52620e33ba8692f893efbb2f958b4424":"../../framework/data_files/mpi_write"
 
 Test mbedtls_mpi_lsb: 0 (null)
 mpi_lsb:"":0

tf-psa-crypto/tests/suites/test_suite_config.function

@@ -0,0 +1,14 @@
+/* BEGIN_HEADER */
+
+/* END_HEADER */
+
+/* BEGIN_CASE */
+/* This test case always passes. It is intended solely for configuration
+ * reporting in the outcome file. Write test cases using this function
+ * with dependencies to record in which configurations the dependencies
+ * are met. */
+void pass()
+{
+    goto exit;
+}
+/* END_CASE */

tf-psa-crypto/tests/suites/test_suite_ctr_drbg.data

@@ -1088,7 +1088,7 @@ depends_on:!MBEDTLS_CTR_DRBG_USE_128_BIT_KEY
 ctr_drbg_entropy_strength:256
 
 CTR_DRBG write/update seed file [#1]
-ctr_drbg_seed_file:"../framework/data_files/ctr_drbg_seed":0
+ctr_drbg_seed_file:"../../framework/data_files/ctr_drbg_seed":0
 
 CTR_DRBG write/update seed file [#2]
 ctr_drbg_seed_file:"no_such_dir/file":MBEDTLS_ERR_CTR_DRBG_FILE_IO_ERROR