Generate config test cases for single options

Generate option-on and option-off cases for test_suite_config, for all
boolean options (MBEDTLS_xxx and PSA_WANT_xxx, collected from the mbedtls
and PSA config files).

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>

framework

@@ -1 +1 @@
-Subproject commit 04847216ab964b9bdce41f1e61ccc6d8f5d2a139
+Subproject commit 1893bfb1ee4b1d971db38bf3ea1f8005fe3c8d9a

scripts/make_generated_files.bat

@@ -11,6 +11,7 @@ python scripts\generate_ssl_debug_helpers.py || exit /b 1
 perl scripts\generate_visualc_files.pl || exit /b 1
 python scripts\generate_psa_constants.py || exit /b 1
 python framework\scripts\generate_bignum_tests.py || exit /b 1
+python framework\scripts\generate_config_tests.py || exit /b 1
 python framework\scripts\generate_ecp_tests.py || exit /b 1
 python framework\scripts\generate_psa_tests.py || exit /b 1
 python framework\scripts\generate_test_keys.py --output tests\src\test_keys.h || exit /b 1

tests/.gitignore

@@ -19,6 +19,8 @@ libtestdriver1/*
 ###START_GENERATED_FILES###
 # Generated source files
 /suites/*.generated.data
+suites/test_suite_config.mbedtls_boolean.data
+suites/test_suite_config.psa_boolean.data
 /suites/test_suite_psa_crypto_storage_format.v[0-9]*.data
 /suites/test_suite_psa_crypto_storage_format.current.data
 /src/test_keys.h

tests/CMakeLists.txt

@@ -30,6 +30,18 @@ execute_process(
 string(REGEX REPLACE "[^;]*/" ""
        base_bignum_generated_data_files "${base_bignum_generated_data_files}")
 
+execute_process(
+    COMMAND
+        ${MBEDTLS_PYTHON_EXECUTABLE}
+        ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_config_tests.py
+        --list-for-cmake
+    WORKING_DIRECTORY
+        ${CMAKE_CURRENT_SOURCE_DIR}/..
+    OUTPUT_VARIABLE
+        base_config_generated_data_files)
+string(REGEX REPLACE "[^;]*/" ""
+       base_config_generated_data_files "${base_config_generated_data_files}")
+
 execute_process(
     COMMAND
         ${MBEDTLS_PYTHON_EXECUTABLE}
@@ -61,11 +73,15 @@ set(base_generated_data_files
 string(REGEX REPLACE "([^;]+)" "suites/\\1"
        all_generated_data_files "${base_generated_data_files}")
 set(bignum_generated_data_files "")
+set(config_generated_data_files "")
 set(ecp_generated_data_files "")
 set(psa_generated_data_files "")
 foreach(file ${base_bignum_generated_data_files})
     list(APPEND bignum_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
 endforeach()
+foreach(file ${base_config_generated_data_files})
+    list(APPEND config_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
+endforeach()
 foreach(file ${base_ecp_generated_data_files})
     list(APPEND ecp_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
 endforeach()
@@ -92,6 +108,21 @@ if(GEN_FILES)
             ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/test_case.py
             ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/mbedtls_framework/test_data_generation.py
     )
+    add_custom_command(
+        OUTPUT
+            ${config_generated_data_files}
+        WORKING_DIRECTORY
+            ${CMAKE_CURRENT_SOURCE_DIR}/..
+        COMMAND
+            ${MBEDTLS_PYTHON_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_config_tests.py
+            --directory ${CMAKE_CURRENT_BINARY_DIR}/suites
+        DEPENDS
+            ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_config_tests.py
+            # Do not declare the configuration files as dependencies: they
+            # change too often in ways that don't affect the result
+            # ((un)commenting some options).
+    )
     add_custom_command(
         OUTPUT
             ${ecp_generated_data_files}

tests/Makefile

@@ -25,6 +25,15 @@ $(error "$(PYTHON) ../framework/scripts/generate_bignum_tests.py --list" failed)
 endif
 GENERATED_DATA_FILES += $(GENERATED_BIGNUM_DATA_FILES)
 
+GENERATED_CONFIG_DATA_FILES := $(patsubst tests/%,%,$(shell \
+	$(PYTHON) ../framework/scripts/generate_config_tests.py --list || \
+	echo FAILED \
+))
+ifeq ($(GENERATED_CONFIG_DATA_FILES),FAILED)
+$(error "$(PYTHON) ../framework/scripts/generate_config_tests.py --list" failed)
+endif
+GENERATED_DATA_FILES += $(GENERATED_CONFIG_DATA_FILES)
+
 GENERATED_ECP_DATA_FILES := $(patsubst tests/%,%,$(shell \
 	$(PYTHON) ../framework/scripts/generate_ecp_tests.py --list || \
 	echo FAILED \
@@ -68,6 +77,22 @@ generated_bignum_test_data:
 	$(PYTHON) ../framework/scripts/generate_bignum_tests.py
 .SECONDARY: generated_bignum_test_data
 
+# We deliberately omit the configuration files (mbedtls_config.h,
+# crypto_config.h) from the depenency list because during development
+# and on the CI, we often edit those in a way that doesn't change the
+# output, to comment out certain options, or even to remove certain
+# lines which do affect the output negatively (it will miss the
+# corresponding test cases).
+$(GENERATED_CONFIG_DATA_FILES): $(gen_file_dep) generated_config_test_data
+generated_config_test_data: ../framework/scripts/generate_config_tests.py
+generated_config_test_data: ../scripts/config.py
+generated_config_test_data: ../framework/scripts/mbedtls_framework/test_case.py
+generated_config_test_data: ../framework/scripts/mbedtls_framework/test_data_generation.py
+generated_config_test_data:
+	echo "  Gen   $(GENERATED_CONFIG_DATA_FILES)"
+	$(PYTHON) ../framework/scripts/generate_config_tests.py
+.SECONDARY: generated_config_test_data
+
 $(GENERATED_ECP_DATA_FILES): $(gen_file_dep) generated_ecp_test_data
 generated_ecp_test_data: ../framework/scripts/generate_ecp_tests.py
 generated_ecp_test_data: ../framework/scripts/mbedtls_framework/bignum_common.py

tests/scripts/check-generated-files.sh

@@ -129,6 +129,7 @@ check()
 # These checks are common to Mbed TLS and TF-PSA-Crypto
 check scripts/generate_psa_constants.py programs/psa/psa_constant_names_generated.c
 check framework/scripts/generate_bignum_tests.py $(framework/scripts/generate_bignum_tests.py --list)
+check framework/scripts/generate_config_tests.py $(framework/scripts/generate_config_tests.py --list)
 check framework/scripts/generate_ecp_tests.py $(framework/scripts/generate_ecp_tests.py --list)
 check framework/scripts/generate_psa_tests.py $(framework/scripts/generate_psa_tests.py --list)
 check framework/scripts/generate_test_keys.py tests/src/test_keys.h