Merge pull request #9565 from gilles-peskine-arm/test-ref-configs-go-away

Switch from test-ref-configs.pl to separate components

Makefile

@@ -28,6 +28,9 @@ no_test: programs
 programs: lib mbedtls_test
 	$(MAKE) -C programs
 
+ssl-opt: lib mbedtls_test
+	$(MAKE) -C programs ssl-opt
+
 lib:
 	$(MAKE) -C library
 

programs/CMakeLists.txt

@@ -1,3 +1,9 @@
+set(programs_target "${MBEDTLS_TARGET_PREFIX}programs")
+add_custom_target(${programs_target})
+
+set(ssl_opt_target "${MBEDTLS_TARGET_PREFIX}ssl-opt")
+add_custom_target(${ssl_opt_target})
+
 add_subdirectory(aes)
 add_subdirectory(cipher)
 if (NOT WIN32)

programs/Makefile

@@ -116,6 +116,12 @@ ifndef WINDOWS
 all: fuzz
 endif
 
+SSL_OPT_APPS = $(filter ssl/%,$(APPS))
+SSL_OPT_APPS += test/query_compile_time_config test/udp_proxy
+# Just the programs needed to run ssl-opt.sh (and compat.sh)
+ssl-opt: $(patsubst %,%$(EXEXT),$(SSL_OPT_APPS))
+.PHONY: ssl-opt
+
 fuzz: ${MBEDTLS_TEST_OBJS}
 	$(MAKE) -C fuzz
 

programs/aes/CMakeLists.txt

@@ -1,6 +1,7 @@
 set(executables
     crypt_and_hash
 )
+add_dependencies(${programs_target} ${executables})
 
 foreach(exe IN LISTS executables)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)

programs/cipher/CMakeLists.txt

@@ -1,6 +1,7 @@
 set(executables
     cipher_aead_demo
 )
+add_dependencies(${programs_target} ${executables})
 
 foreach(exe IN LISTS executables)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)

programs/fuzz/CMakeLists.txt

@@ -15,6 +15,7 @@ set(executables_no_common_c
     fuzz_x509csr
     fuzz_pkcs7
 )
+add_dependencies(${programs_target} ${executables_no_common_c})
 
 set(executables_with_common_c
     fuzz_privkey
@@ -23,6 +24,7 @@ set(executables_with_common_c
     fuzz_dtlsserver
     fuzz_server
 )
+add_dependencies(${programs_target} ${executables_with_common_c})
 
 foreach(exe IN LISTS executables_no_common_c executables_with_common_c)
 

programs/hash/CMakeLists.txt

@@ -3,6 +3,7 @@ set(executables
     hello
     md_hmac_demo
 )
+add_dependencies(${programs_target} ${executables})
 
 foreach(exe IN LISTS executables)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)

programs/pkey/CMakeLists.txt

@@ -2,6 +2,7 @@ set(executables_mbedtls
     dh_client
     dh_server
 )
+add_dependencies(${programs_target} ${executables_mbedtls})
 
 foreach(exe IN LISTS executables_mbedtls)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)
@@ -29,6 +30,7 @@ set(executables_mbedcrypto
     rsa_verify
     rsa_verify_pss
 )
+add_dependencies(${programs_target} ${executables_mbedcrypto})
 
 foreach(exe IN LISTS executables_mbedcrypto)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)

programs/psa/CMakeLists.txt

@@ -6,6 +6,7 @@ set(executables
     psa_constant_names
     psa_hash
 )
+add_dependencies(${programs_target} ${executables})
 
 if(GEN_FILES)
     add_custom_command(

programs/random/CMakeLists.txt

@@ -2,6 +2,7 @@ set(executables
     gen_entropy
     gen_random_ctr_drbg
 )
+add_dependencies(${programs_target} ${executables})
 
 foreach(exe IN LISTS executables)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)

programs/ssl/CMakeLists.txt

@@ -16,6 +16,8 @@ set(executables
     ssl_server
     ssl_server2
 )
+add_dependencies(${programs_target} ${executables})
+add_dependencies(${ssl_opt_target} ${executables})
 
 if(GEN_FILES)
     # Inform CMake that the following file will be generated as part of the build

programs/test/CMakeLists.txt

@@ -8,12 +8,16 @@ set(executables_libs
     selftest
     udp_proxy
 )
+add_dependencies(${programs_target} ${executables_libs})
+add_dependencies(${ssl_opt_target} udp_proxy)
 
 set(executables_mbedcrypto
     benchmark
     query_compile_time_config
     zeroize
 )
+add_dependencies(${programs_target} ${executables_mbedcrypto})
+add_dependencies(${ssl_opt_target} query_compile_time_config)
 
 if(TEST_CPP)
     set(cpp_dummy_build_cpp "${CMAKE_CURRENT_BINARY_DIR}/cpp_dummy_build.cpp")

programs/util/CMakeLists.txt

@@ -6,6 +6,7 @@ set(executables
     pem2der
     strerror
 )
+add_dependencies(${programs_target} ${executables})
 
 foreach(exe IN LISTS executables)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)

programs/x509/CMakeLists.txt

@@ -10,6 +10,7 @@ set(executables
     load_roots
     req_app
 )
+add_dependencies(${programs_target} ${executables})
 
 foreach(exe IN LISTS executables)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)

tests/scripts/analyze_outcomes.py

@@ -576,7 +576,7 @@ KNOWN_TASKS = {
     'analyze_driver_vs_reference_tfm_config': {
         'test_function':  do_analyze_driver_vs_reference,
         'args': {
-            'component_ref': 'test_tfm_config',
+            'component_ref': 'test_tfm_config_no_p256m',
             'component_driver': 'test_tfm_config_p256m_driver_accel_ec',
             'ignored_suites': [
                 # Modules replaced by drivers

tests/scripts/basic-in-docker.sh

@@ -31,7 +31,6 @@ for compiler in clang gcc; do
     run_in_docker programs/test/selftest
     run_in_docker -e OSSL_NO_DTLS=1 tests/compat.sh
     run_in_docker tests/ssl-opt.sh -e '\(DTLS\|SCSV\).*openssl'
-    run_in_docker tests/scripts/test-ref-configs.pl
     run_in_docker tests/scripts/depends.py curves
     run_in_docker tests/scripts/depends.py kex
 done

tests/scripts/components-configuration-crypto.sh

@@ -408,6 +408,25 @@ component_test_full_no_ccm_star_no_tag () {
     make test
 }
 
+component_test_config_symmetric_only () {
+    msg "build: configs/config-symmetric-only.h"
+    cp configs/config-symmetric-only.h "$CONFIG_H"
+    # test-ref-configs works by overwriting mbedtls_config.h; this makes cmake
+    # want to re-generate generated files that depend on it, quite correctly.
+    # However this doesn't work as the generation script expects a specific
+    # format for mbedtls_config.h, which the other files don't follow. Also,
+    # cmake can't know this, but re-generation is actually not necessary as
+    # the generated files only depend on the list of available options, not
+    # whether they're on or off. So, disable cmake's (over-sensitive here)
+    # dependency resolution for generated files and just rely on them being
+    # present (thanks to pre_generate_files) by turning GEN_FILES off.
+    CC=$ASAN_CC cmake -D GEN_FILES=Off -D CMAKE_BUILD_TYPE:String=Asan .
+    make
+
+    msg "test: configs/config-symmetric-only.h - unit tests"
+    make test
+}
+
 component_test_everest () {
     msg "build: Everest ECDH context (ASan build)" # ~ 6 min
     scripts/config.py set MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED
@@ -1415,9 +1434,19 @@ component_test_psa_crypto_config_reference_ecc_ffdh_no_bignum () {
     common_test_psa_crypto_config_reference_ecc_ffdh_no_bignum "ECC_DH"
 }
 
+component_test_tfm_config_as_is () {
+    msg "build: configs/config-tfm.h"
+    cp configs/config-tfm.h "$CONFIG_H"
+    CC=$ASAN_CC cmake -D CMAKE_BUILD_TYPE:String=Asan .
+    make
+
+    msg "test: configs/config-tfm.h - unit tests"
+    make test
+}
+
 # Helper for setting common configurations between:
 # - component_test_tfm_config_p256m_driver_accel_ec()
-# - component_test_tfm_config()
+# - component_test_tfm_config_no_p256m()
 common_tfm_config () {
     # Enable TF-M config
     cp configs/config-tfm.h "$CONFIG_H"
@@ -1471,14 +1500,14 @@ component_test_tfm_config_p256m_driver_accel_ec () {
 # Keep this in sync with component_test_tfm_config_p256m_driver_accel_ec() as
 # they are both meant to be used in analyze_outcomes.py for driver's coverage
 # analysis.
-component_test_tfm_config () {
+component_test_tfm_config_no_p256m () {
     common_tfm_config
 
     # Disable P256M driver, which is on by default, so that analyze_outcomes
     # can compare this test with test_tfm_config_p256m_driver_accel_ec
     echo "#undef MBEDTLS_PSA_P256M_DRIVER_ENABLED" >> "$CONFIG_H"
 
-    msg "build: TF-M config"
+    msg "build: TF-M config without p256m"
     make CFLAGS='-Werror -Wall -Wextra -I../tests/include/spe' tests
 
     # Check that p256m was not built
@@ -1488,7 +1517,7 @@ component_test_tfm_config () {
     # files, so we want to ensure that it has not be re-enabled accidentally.
     not grep mbedtls_cipher ${BUILTIN_SRC_PATH}/cipher.o
 
-    msg "test: TF-M config"
+    msg "test: TF-M config without p256m"
     make test
 }
 

tests/scripts/components-configuration-tls.sh

@@ -9,6 +9,38 @@
 #### Configuration Testing - TLS
 ################################################################
 
+component_test_config_suite_b () {
+    msg "build: configs/config-suite-b.h"
+    cp configs/config-suite-b.h "$CONFIG_H"
+    # test-ref-configs works by overwriting mbedtls_config.h; this makes cmake
+    # want to re-generate generated files that depend on it, quite correctly.
+    # However this doesn't work as the generation script expects a specific
+    # format for mbedtls_config.h, which the other files don't follow. Also,
+    # cmake can't know this, but re-generation is actually not necessary as
+    # the generated files only depend on the list of available options, not
+    # whether they're on or off. So, disable cmake's (over-sensitive here)
+    # dependency resolution for generated files and just rely on them being
+    # present (thanks to pre_generate_files) by turning GEN_FILES off.
+    CC=$ASAN_CC cmake -D GEN_FILES=Off -D CMAKE_BUILD_TYPE:String=Asan .
+    make
+
+    msg "test: configs/config-suite-b.h - unit tests"
+    make test
+
+    msg "test: configs/config-suite-b.h - compat.sh"
+    tests/compat.sh -m tls12 -f 'ECDHE_ECDSA.*AES.*GCM' -p mbedTLS
+
+    msg "build: configs/config-suite-b.h + DEBUG"
+    MBEDTLS_TEST_CONFIGURATION="$MBEDTLS_TEST_CONFIGURATION+DEBUG"
+    make clean
+    scripts/config.py set MBEDTLS_DEBUG_C
+    scripts/config.py set MBEDTLS_ERROR_C
+    make ssl-opt
+
+    msg "test: configs/config-suite-b.h + DEBUG - ssl-opt.sh"
+    tests/ssl-opt.sh
+}
+
 component_test_no_renegotiation () {
     msg "build: Default + !MBEDTLS_SSL_RENEGOTIATION (ASan build)" # ~ 6 min
     scripts/config.py unset MBEDTLS_SSL_RENEGOTIATION
@@ -130,6 +162,28 @@ component_test_tls1_2_default_cbc_legacy_cbc_etm_cipher_only () {
     tests/ssl-opt.sh -f "TLS 1.2"
 }
 
+component_test_config_thread () {
+    msg "build: configs/config-thread.h"
+    cp configs/config-thread.h "$CONFIG_H"
+    # test-ref-configs works by overwriting mbedtls_config.h; this makes cmake
+    # want to re-generate generated files that depend on it, quite correctly.
+    # However this doesn't work as the generation script expects a specific
+    # format for mbedtls_config.h, which the other files don't follow. Also,
+    # cmake can't know this, but re-generation is actually not necessary as
+    # the generated files only depend on the list of available options, not
+    # whether they're on or off. So, disable cmake's (over-sensitive here)
+    # dependency resolution for generated files and just rely on them being
+    # present (thanks to pre_generate_files) by turning GEN_FILES off.
+    CC=$ASAN_CC cmake -D GEN_FILES=Off -D CMAKE_BUILD_TYPE:String=Asan .
+    make
+
+    msg "test: configs/config-thread.h - unit tests"
+    make test
+
+    msg "test: configs/config-thread.h - ssl-opt.sh"
+    tests/ssl-opt.sh -f 'ECJPAKE.*nolog'
+}
+
 # We're not aware of any other (open source) implementation of EC J-PAKE in TLS
 # that we could use for interop testing. However, we now have sort of two
 # implementations ourselves: one using PSA, the other not. At least test that
@@ -163,6 +217,60 @@ component_test_tls1_2_ecjpake_compatibility () {
     rm s2_no_use_psa c2_no_use_psa
 }
 
+component_test_tls1_2_ccm_psk () {
+    msg "build: configs/config-ccm-psk-tls1_2.h"
+    cp configs/config-ccm-psk-tls1_2.h "$CONFIG_H"
+    # test-ref-configs works by overwriting mbedtls_config.h; this makes cmake
+    # want to re-generate generated files that depend on it, quite correctly.
+    # However this doesn't work as the generation script expects a specific
+    # format for mbedtls_config.h, which the other files don't follow. Also,
+    # cmake can't know this, but re-generation is actually not necessary as
+    # the generated files only depend on the list of available options, not
+    # whether they're on or off. So, disable cmake's (over-sensitive here)
+    # dependency resolution for generated files and just rely on them being
+    # present (thanks to pre_generate_files) by turning GEN_FILES off.
+    CC=$ASAN_CC cmake -D GEN_FILES=Off -D CMAKE_BUILD_TYPE:String=Asan .
+    make
+
+    msg "test: configs/config-ccm-psk-tls1_2.h - unit tests"
+    make test
+
+    msg "test: configs/config-ccm-psk-tls1_2.h - compat.sh"
+    tests/compat.sh -m tls12 -f '^TLS_PSK_WITH_AES_..._CCM_8'
+}
+
+component_test_tls1_2_ccm_psk_dtls () {
+    msg "build: configs/config-ccm-psk-dtls1_2.h"
+    cp configs/config-ccm-psk-dtls1_2.h "$CONFIG_H"
+    # test-ref-configs works by overwriting mbedtls_config.h; this makes cmake
+    # want to re-generate generated files that depend on it, quite correctly.
+    # However this doesn't work as the generation script expects a specific
+    # format for mbedtls_config.h, which the other files don't follow. Also,
+    # cmake can't know this, but re-generation is actually not necessary as
+    # the generated files only depend on the list of available options, not
+    # whether they're on or off. So, disable cmake's (over-sensitive here)
+    # dependency resolution for generated files and just rely on them being
+    # present (thanks to pre_generate_files) by turning GEN_FILES off.
+    CC=$ASAN_CC cmake -D GEN_FILES=Off -D CMAKE_BUILD_TYPE:String=Asan .
+    make
+
+    msg "test: configs/config-ccm-psk-dtls1_2.h - unit tests"
+    make test
+
+    msg "test: configs/config-ccm-psk-dtls1_2.h - compat.sh"
+    tests/compat.sh -m dtls12 -f '^TLS_PSK_WITH_AES_..._CCM_8'
+
+    msg "build: configs/config-ccm-psk-dtls1_2.h + DEBUG"
+    MBEDTLS_TEST_CONFIGURATION="$MBEDTLS_TEST_CONFIGURATION+DEBUG"
+    make clean
+    scripts/config.py set MBEDTLS_DEBUG_C
+    scripts/config.py set MBEDTLS_ERROR_C
+    make ssl-opt
+
+    msg "test: configs/config-ccm-psk-dtls1_2.h + DEBUG - ssl-opt.sh"
+    tests/ssl-opt.sh
+}
+
 component_test_small_ssl_out_content_len () {
     msg "build: small SSL_OUT_CONTENT_LEN (ASan build)"
     scripts/config.py set MBEDTLS_SSL_IN_CONTENT_LEN 16384

tests/scripts/components-configuration.sh

@@ -129,21 +129,6 @@ component_test_full_cmake_gcc_asan_new_bignum () {
     tests/context-info.sh
 }
 
-component_test_ref_configs () {
-    msg "test/build: ref-configs (ASan build)" # ~ 6 min 20s
-    # test-ref-configs works by overwriting mbedtls_config.h; this makes cmake
-    # want to re-generate generated files that depend on it, quite correctly.
-    # However this doesn't work as the generation script expects a specific
-    # format for mbedtls_config.h, which the other files don't follow. Also,
-    # cmake can't know this, but re-generation is actually not necessary as
-    # the generated files only depend on the list of available options, not
-    # whether they're on or off. So, disable cmake's (over-sensitive here)
-    # dependency resolution for generated files and just rely on them being
-    # present (thanks to pre_generate_files) by turning GEN_FILES off.
-    CC=$ASAN_CC cmake -D GEN_FILES=Off -D CMAKE_BUILD_TYPE:String=Asan .
-    tests/scripts/test-ref-configs.pl
-}
-
 component_test_full_cmake_clang () {
     msg "build: cmake, full config, clang" # ~ 50s
     scripts/config.py full

tests/scripts/test-ref-configs.pl

@@ -1,156 +0,0 @@
-#!/usr/bin/env perl
-
-# test-ref-configs.pl
-#
-# Copyright The Mbed TLS Contributors
-# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
-#
-# Purpose
-#
-# For each reference configuration file in the configs directory, build the
-# configuration, run the test suites and compat.sh
-#
-# Usage: tests/scripts/test-ref-configs.pl [config-name [...]]
-
-use warnings;
-use strict;
-
-my %configs = (
-    'config-ccm-psk-tls1_2.h' => {
-        'compat' => '-m tls12 -f \'^TLS_PSK_WITH_AES_..._CCM_8\'',
-    },
-    'config-ccm-psk-dtls1_2.h' => {
-        'compat' => '-m dtls12 -f \'^TLS_PSK_WITH_AES_..._CCM_8\'',
-        'opt' => ' ',
-        'opt_needs_debug' => 1,
-    },
-    'config-suite-b.h' => {
-        'compat' => "-m tls12 -f 'ECDHE_ECDSA.*AES.*GCM' -p mbedTLS",
-        'opt' => ' ',
-        'opt_needs_debug' => 1,
-    },
-    'config-symmetric-only.h' => {
-    },
-    'config-tfm.h' => {
-    },
-    'config-thread.h' => {
-        'opt' => '-f ECJPAKE.*nolog',
-    },
-);
-
-# If no config-name is provided, use all known configs.
-# Otherwise, use the provided names only.
-my @configs_to_test = sort keys %configs;
-if ($#ARGV >= 0) {
-    foreach my $conf_name ( @ARGV ) {
-        if( ! exists $configs{$conf_name} ) {
-            die "Unknown configuration: $conf_name\n";
-        }
-    }
-    @configs_to_test = @ARGV;
-}
-
--d 'library' && -d 'include' && -d 'tests' or die "Must be run from root\n";
-
-my $config_h = 'include/mbedtls/mbedtls_config.h';
-
-system( "cp $config_h $config_h.bak" ) and die;
-sub abort {
-    system( "mv $config_h.bak $config_h" ) and warn "$config_h not restored\n";
-    # use an exit code between 1 and 124 for git bisect (die returns 255)
-    warn $_[0];
-    exit 1;
-}
-
-# Create a seedfile for configurations that enable MBEDTLS_ENTROPY_NV_SEED.
-# For test purposes, this doesn't have to be cryptographically random.
-if (!-e "tests/seedfile" || -s "tests/seedfile" < 64) {
-    local *SEEDFILE;
-    open SEEDFILE, ">tests/seedfile" or die;
-    print SEEDFILE "*" x 64 or die;
-    close SEEDFILE or die;
-}
-
-sub perform_test {
-    my $conf_file = $_[0];
-    my $data = $_[1];
-    my $test_with_psa = $_[2];
-
-    my $conf_name = $conf_file;
-    if ( $test_with_psa )
-    {
-        $conf_name .= "+PSA";
-    }
-
-    system( "cp $config_h.bak $config_h" ) and die;
-    system( "make clean" ) and die;
-
-    print "\n******************************************\n";
-    print "* Testing configuration: $conf_name\n";
-    print "******************************************\n";
-
-    $ENV{MBEDTLS_TEST_CONFIGURATION} = $conf_name;
-
-    system( "cp configs/$conf_file $config_h" )
-        and abort "Failed to activate $conf_file\n";
-
-    if ( $test_with_psa )
-    {
-        system( "scripts/config.py set MBEDTLS_PSA_CRYPTO_C" );
-        system( "scripts/config.py set MBEDTLS_USE_PSA_CRYPTO" );
-    }
-
-    system( "CFLAGS='-Os -Werror -Wall -Wextra' make" ) and abort "Failed to build: $conf_name\n";
-    system( "make test" ) and abort "Failed test suite: $conf_name\n";
-
-    my $compat = $data->{'compat'};
-    if( $compat )
-    {
-        print "\nrunning compat.sh $compat ($conf_name)\n";
-        system( "tests/compat.sh $compat" )
-            and abort "Failed compat.sh: $conf_name\n";
-    }
-    else
-    {
-        print "\nskipping compat.sh ($conf_name)\n";
-    }
-
-    my $opt = $data->{'opt'};
-    if( $opt )
-    {
-        if( $data->{'opt_needs_debug'} )
-        {
-            print "\nrebuilding with debug traces for ssl-opt ($conf_name)\n";
-            $conf_name .= '+DEBUG';
-            $ENV{MBEDTLS_TEST_CONFIGURATION} = $conf_name;
-            system( "make clean" );
-            system( "scripts/config.py set MBEDTLS_DEBUG_C" );
-            system( "scripts/config.py set MBEDTLS_ERROR_C" );
-            system( "CFLAGS='-Os -Werror -Wall -Wextra' make" ) and abort "Failed to build: $conf_name\n";
-        }
-
-        print "\nrunning ssl-opt.sh $opt ($conf_name)\n";
-        system( "tests/ssl-opt.sh $opt" )
-            and abort "Failed ssl-opt.sh: $conf_name\n";
-    }
-    else
-    {
-        print "\nskipping ssl-opt.sh ($conf_name)\n";
-    }
-}
-
-foreach my $conf ( @configs_to_test ) {
-    system("grep '//#define MBEDTLS_USE_PSA_CRYPTO' configs/$conf > /dev/null");
-    die "grep ... configs/$conf: $!" if $? != 0 && $? != 0x100;
-    my $test_with_psa = $? == 0;
-
-    if ( $test_with_psa )
-    {
-        perform_test( $conf, $configs{$conf}, $test_with_psa );
-    }
-    perform_test( $conf, $configs{$conf}, 0 );
-}
-
-system( "mv $config_h.bak $config_h" ) and warn "$config_h not restored\n";
-system( "make clean" );
-exit 0;