From d8da2fcc7abcbfb0cb7e750b244beaa341a78e9d Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 17 Sep 2024 15:07:22 +0200 Subject: [PATCH 01/14] In PSA generated tests, ignore mechanisms that are not implemented This clears more than half of the test cases that are not executed. This also captures a few negative test cases that are executed. Subsequent commits will refine the filtering. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 62 +++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 72dba99f7c..698358dbb9 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -7,6 +7,7 @@ less likely to be useful. """ import re +import typing import scripts_path # pylint: disable=unused-import from mbedtls_framework import outcome_analysis @@ -17,6 +18,67 @@ class CoverageTask(outcome_analysis.CoverageTask): # is just a warning. outcome_analysis.FULL_COVERAGE_BY_DEFAULT = False + @staticmethod + def _has_word_re(words: typing.Iterable[str]) -> typing.Pattern: + """Construct a regex that matches if any of the words appears. + + The occurrence must start and end at a word boundary. + """ + return re.compile(r'.*\b(?:' + r'|'.join(words) + r')\b.*') + + # generate_psa_tests.py generates test cases involving cryptographic + # mechanisms (key types, families, algorithms) that are declared but + # not implemented. Until we improve the Python scripts, ignore those + # test cases in the analysis. + # https://github.com/Mbed-TLS/mbedtls/issues/9572 + _PSA_MECHANISMS_NOT_IMPLEMENTED = [ + r'CBC_MAC', + r'DETERMINISTIC_DSA', + r'DET_DSA', + r'DSA', + r'ECC_KEY_PAIR\(BRAINPOOL_P_R1\) (?:160|192|224|320)-bit', + r'ECC_KEY_PAIR\(SECP_K1\) 225-bit', + r'ECC_PAIR\(BP_R1\) (?:160|192|224|320)-bit', + r'ECC_PAIR\(SECP_K1\) 225-bit', + r'ECC_PUBLIC_KEY\(BRAINPOOL_P_R1\) (?:160|192|224|320)-bit', + r'ECC_PUBLIC_KEY\(SECP_K1\) 225-bit', + r'ECC_PUB\(BP_R1\) (?:160|192|224|320)-bit', + r'ECC_PUB\(SECP_K1\) 225-bit', + r'ED25519PH', + r'ED448PH', + r'PEPPER', + r'PURE_EDDSA', + r'SECP_R2', + r'SECT_K1', + r'SECT_R1', + r'SECT_R2', + r'SHAKE256_512', + r'SHA_512_224', + r'SHA_512_256', + r'TWISTED_EDWARDS', + r'XTS', + ] + PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE = \ + _has_word_re(_PSA_MECHANISMS_NOT_IMPLEMENTED) + + IGNORED_TESTS = { + 'test_suite_psa_crypto_generate_key.generated': [ + PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + ], + 'test_suite_psa_crypto_not_supported.generated': [ + PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + ], + 'test_suite_psa_crypto_op_fail.generated': [ + PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + ], + 'test_suite_psa_crypto_storage_format.current': [ + PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + ], + 'test_suite_psa_crypto_storage_format.v0': [ + PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + ], + } + # The names that we give to classes derived from DriverVSReference do not # follow the usual naming convention, because it's more readable to use From 72396da267fd8938bc6dd69092a365a356922949 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 17 Sep 2024 17:15:29 +0200 Subject: [PATCH 02/14] In PSA generated tests, don't ignore not-implemented in some negative tests Some negative tests involving unsupported mechanisms are executed, because they're testing what happens if the mechanism is unsupported. Refine the ignore list for `test_suite_psa_crypto_generate_key.generated` and `test_suite_psa_crypto_op_fail.generated` accordingly. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 698358dbb9..b0b16c54d7 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -19,12 +19,21 @@ class CoverageTask(outcome_analysis.CoverageTask): outcome_analysis.FULL_COVERAGE_BY_DEFAULT = False @staticmethod - def _has_word_re(words: typing.Iterable[str]) -> typing.Pattern: + def _has_word_re(words: typing.Iterable[str], + exclude: typing.Optional[str] = None) -> typing.Pattern: """Construct a regex that matches if any of the words appears. The occurrence must start and end at a word boundary. + + If exclude is specified, strings containing a match for that + regular expression will not match the returned pattern. """ - return re.compile(r'.*\b(?:' + r'|'.join(words) + r')\b.*') + exclude_clause = r'' + if exclude: + exclude_clause = r'(?!.*' + exclude + ')' + return re.compile(exclude_clause + + r'.*\b(?:' + r'|'.join(words) + r')\b.*', + re.S) # generate_psa_tests.py generates test cases involving cryptographic # mechanisms (key types, families, algorithms) that are declared but @@ -63,13 +72,25 @@ class CoverageTask(outcome_analysis.CoverageTask): IGNORED_TESTS = { 'test_suite_psa_crypto_generate_key.generated': [ - PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + # Ignore mechanisms that are not implemented, except + # for public keys for which we always test that + # psa_generate_key() returns PSA_ERROR_INVALID_ARGUMENT + # regardless of whether the specific key type is supported. + _has_word_re((mech + for mech in _PSA_MECHANISMS_NOT_IMPLEMENTED + if not mech.startswith('ECC_PUB')), + exclude=r'ECC_PUB'), ], 'test_suite_psa_crypto_not_supported.generated': [ PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, ], 'test_suite_psa_crypto_op_fail.generated': [ - PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + # Ignore mechanisms that are not implemented, except + # for test cases that assume the mechanism is not supported. + _has_word_re(_PSA_MECHANISMS_NOT_IMPLEMENTED, + exclude=(r'.*: !(?:' + + r'|'.join(_PSA_MECHANISMS_NOT_IMPLEMENTED) + + r')\b')), ], 'test_suite_psa_crypto_storage_format.current': [ PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, From 1fac371ebe52125d8e41af187090a108be3bf60b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 17 Sep 2024 17:57:11 +0200 Subject: [PATCH 03/14] In PSA generated tests, ignore patterns for which an issue has been raised Ignore certain test cases which either should not be generated or should be executed. For each ignore list entry, link to a GitHub issue whose definition of done includes removing the entry. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 41 +++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index b0b16c54d7..4ee0c40cf4 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -82,7 +82,21 @@ class CoverageTask(outcome_analysis.CoverageTask): exclude=r'ECC_PUB'), ], 'test_suite_psa_crypto_not_supported.generated': [ + # It is a bug that not-supported test cases aren't getting + # run for never-implemented key types. + # https://github.com/Mbed-TLS/mbedtls/issues/7915 PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, + # We mever test with DH key support disabled but support + # for a DH group enabled. The dependencies of these test + # cases don't really make sense. + # https://github.com/Mbed-TLS/mbedtls/issues/9574 + re.compile(r'PSA \w+ DH_.*type not supported'), + # We only test partial support for DH with the 2048-bit group + # enabled and the other groups disabled. + # https://github.com/Mbed-TLS/mbedtls/issues/9575 + 'PSA generate DH_KEY_PAIR(RFC7919) 2048-bit group not supported', + 'PSA import DH_KEY_PAIR(RFC7919) 2048-bit group not supported', + 'PSA import DH_PUBLIC_KEY(RFC7919) 2048-bit group not supported', ], 'test_suite_psa_crypto_op_fail.generated': [ # Ignore mechanisms that are not implemented, except @@ -91,6 +105,33 @@ class CoverageTask(outcome_analysis.CoverageTask): exclude=(r'.*: !(?:' + r'|'.join(_PSA_MECHANISMS_NOT_IMPLEMENTED) + r')\b')), + # Incorrect dependency generation. To be fixed as part of the + # resolution of https://github.com/Mbed-TLS/mbedtls/issues/9167 + # by forward-porting the commit + # "PSA test case generation: dependency inference class: operation fail" + # from https://github.com/Mbed-TLS/mbedtls/pull/9025 . + re.compile(r'.* with (?:DH|ECC)_(?:KEY_PAIR|PUBLIC_KEY)\(.*'), + # PBKDF2_HMAC is not in the default configuration, so we don't + # enable it in depends.py where we remove hashes. + # https://github.com/Mbed-TLS/mbedtls/issues/9576 + re.compile(r'PSA key_derivation PBKDF2_HMAC\(\w+\): !(?!PBKDF2_HMAC\Z).*'), + # We never test with TLS12_PRF or TLS12_PSK_TO_MS disabled + # but certain other things enabled. + # https://github.com/Mbed-TLS/mbedtls/issues/9577 + re.compile(r'PSA key_derivation TLS12_PRF\(\w+\): !TLS12_PRF'), + re.compile(r'PSA key_derivation TLS12_PSK_TO_MS' + r'\((?!SHA_256|SHA_384|SHA_512)\w+\): !TLS12_PSK_TO_MS'), + 'PSA key_derivation KEY_AGREEMENT(ECDH,TLS12_PRF(SHA_256)): !TLS12_PRF', + 'PSA key_derivation KEY_AGREEMENT(ECDH,TLS12_PRF(SHA_384)): !TLS12_PRF', + + # We never test with the HMAC algorithm enabled but the HMAC + # key type disabled. Those dependencies don't really make sense. + # https://github.com/Mbed-TLS/mbedtls/issues/9573 + re.compile(r'.* !HMAC with HMAC'), + # There's something wrong with PSA_WANT_ALG_RSA_PSS_ANY_SALT + # differing from PSA_WANT_ALG_RSA_PSS. + # https://github.com/Mbed-TLS/mbedtls/issues/9578 + re.compile(r'PSA sign RSA_PSS_ANY_SALT.*!(?:MD|RIPEMD|SHA).*'), ], 'test_suite_psa_crypto_storage_format.current': [ PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, From f7784160e69a9f9c9a824aa8735c208d399c9e02 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 17 Sep 2024 18:24:44 +0200 Subject: [PATCH 04/14] Remove test case involving SECP224K1 in PSA Our PSA crypto implementation does not and will not support SECP224K1. Signed-off-by: Gilles Peskine --- tests/suites/test_suite_psa_crypto.data | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/suites/test_suite_psa_crypto.data b/tests/suites/test_suite_psa_crypto.data index 4149fdbfc9..e69469b958 100644 --- a/tests/suites/test_suite_psa_crypto.data +++ b/tests/suites/test_suite_psa_crypto.data @@ -7859,9 +7859,7 @@ ECP group ID <-> PSA family - SECP192K1 depends_on:PSA_WANT_ECC_SECP_K1_192 ecc_conversion_functions:MBEDTLS_ECP_DP_SECP192K1:PSA_ECC_FAMILY_SECP_K1:192 -ECP group ID <-> PSA family - SECP224K1 -depends_on:PSA_WANT_ECC_SECP_K1_224 -ecc_conversion_functions:MBEDTLS_ECP_DP_SECP224K1:PSA_ECC_FAMILY_SECP_K1:224 +# No test case for SECP224K1, which is not implemented in the PSA API. ECP group ID <-> PSA family - SECP256K1 depends_on:PSA_WANT_ECC_SECP_K1_256 From 419a5841a433da4206ec63bee4d1efbbbd88106d Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 17 Sep 2024 18:32:05 +0200 Subject: [PATCH 05/14] Add ignore list entries for TLS tests that are not executed For each ignore list entry, link to a GitHub issue for its resolution, except for ssl-opt Valgrind tests which we never intend to run on the CI. Signed-off-by: Gilles Peskine --- tests/opt-testcases/tls13-misc.sh | 1 + tests/scripts/analyze_outcomes.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tests/opt-testcases/tls13-misc.sh b/tests/opt-testcases/tls13-misc.sh index 003401c10a..76cbeec5fa 100644 --- a/tests/opt-testcases/tls13-misc.sh +++ b/tests/opt-testcases/tls13-misc.sh @@ -948,6 +948,7 @@ run_test "TLS 1.3 m->O: resumption fails, ticket handling disabled (default)" # ephemeral then ticket based scenario we use for early data testing the first # handshake fails. The following skipped test is here to illustrate the kind # of testing we would like to do. +# https://github.com/Mbed-TLS/mbedtls/issues/9582 skip_next_test requires_openssl_tls1_3_with_compatible_ephemeral requires_config_enabled MBEDTLS_SSL_CLI_C diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 4ee0c40cf4..8cd0c160ec 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -71,6 +71,27 @@ class CoverageTask(outcome_analysis.CoverageTask): _has_word_re(_PSA_MECHANISMS_NOT_IMPLEMENTED) IGNORED_TESTS = { + 'ssl-opt': [ + # We don't run ssl-opt.sh with Valgrind on the CI because + # it's extremely slow. We don't intend to change this. + 'DTLS client reconnect from same port: reconnect, nbio, valgrind', + + # We don't have IPv6 in our CI environment. + # https://github.com/Mbed-TLS/mbedtls-test/issues/176 + 'DTLS cookie: enabled, IPv6', + # Disabled due to OpenSSL bug. + # https://github.com/openssl/openssl/issues/18887 + 'DTLS fragmenting: 3d, openssl client, DTLS 1.2', + # We don't run ssl-opt.sh with Valgrind on the CI because + # it's extremely slow. We don't intend to change this. + 'DTLS fragmenting: proxy MTU: auto-reduction (with valgrind)', + # It seems that we don't run `ssl-opt.sh` with + # `MBEDTLS_USE_PSA_CRYPTO` enabled but `MBEDTLS_SSL_ASYNC_PRIVATE` + # disabled. + # https://github.com/Mbed-TLS/mbedtls/issues/9581 + 'Opaque key for server authentication: invalid key: decrypt with ECC key, no async', + 'Opaque key for server authentication: invalid key: ecdh with RSA key, no async', + ], 'test_suite_psa_crypto_generate_key.generated': [ # Ignore mechanisms that are not implemented, except # for public keys for which we always test that @@ -139,6 +160,14 @@ class CoverageTask(outcome_analysis.CoverageTask): 'test_suite_psa_crypto_storage_format.v0': [ PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, ], + 'tls13-misc': [ + # Disabled due to OpenSSL bug. + # https://github.com/openssl/openssl/issues/10714 + 'TLS 1.3 O->m: resumption', + # Disabled due to OpenSSL command line limitation. + # https://github.com/Mbed-TLS/mbedtls/issues/9582 + 'TLS 1.3 m->O: resumption with early data', + ], } From 1a17627a338752c69c563dbee5475b9b84f01e01 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 17 Sep 2024 18:33:29 +0200 Subject: [PATCH 06/14] Add ignore list entries for crypto tests that are not executed For each ignore list entry, link to a GitHub issue for its resolution. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 8cd0c160ec..8dd812f1d7 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -92,6 +92,19 @@ class CoverageTask(outcome_analysis.CoverageTask): 'Opaque key for server authentication: invalid key: decrypt with ECC key, no async', 'Opaque key for server authentication: invalid key: ecdh with RSA key, no async', ], + 'test_suite_pkcs12': [ + # Probably a missing configuration in the CI. + # https://github.com/Mbed-TLS/mbedtls/issues/9580 + 'PBE Decrypt, (Invalid padding & PKCS7 padding disabled)', + 'PBE Encrypt, pad = 8 (PKCS7 padding disabled)', + ], + 'test_suite_pkcs5': [ + # Probably a missing configuration in the CI. + # https://github.com/Mbed-TLS/mbedtls/issues/9580 + 'PBES2 Decrypt (Invalid padding & PKCS7 padding disabled)', + 'PBES2 Encrypt, pad=6 (PKCS7 padding disabled)', + 'PBES2 Encrypt, pad=8 (PKCS7 padding disabled)', + ], 'test_suite_psa_crypto_generate_key.generated': [ # Ignore mechanisms that are not implemented, except # for public keys for which we always test that @@ -102,6 +115,18 @@ class CoverageTask(outcome_analysis.CoverageTask): if not mech.startswith('ECC_PUB')), exclude=r'ECC_PUB'), ], + 'test_suite_psa_crypto_metadata': [ + # Algorithms declared but not supported. + # https://github.com/Mbed-TLS/mbedtls/issues/9579 + 'Asymmetric signature: Ed25519ph', + 'Asymmetric signature: Ed448ph', + 'Asymmetric signature: pure EdDSA', + 'Cipher: XTS', + 'MAC: CBC_MAC-3DES', + 'MAC: CBC_MAC-AES-128', + 'MAC: CBC_MAC-AES-192', + 'MAC: CBC_MAC-AES-256', + ], 'test_suite_psa_crypto_not_supported.generated': [ # It is a bug that not-supported test cases aren't getting # run for never-implemented key types. From 47243fd88f5cdbcfdb90291b11471aca851348a9 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 17 Sep 2024 19:46:18 +0200 Subject: [PATCH 07/14] Add ignore list entries for configurations that are not tested For each ignore list entry, link to a GitHub issue for its resolution, except for a few configurations which there is a good reason to leave uncovered. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 104 +++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 2 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 8dd812f1d7..75dc41d6f8 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -92,14 +92,114 @@ class CoverageTask(outcome_analysis.CoverageTask): 'Opaque key for server authentication: invalid key: decrypt with ECC key, no async', 'Opaque key for server authentication: invalid key: ecdh with RSA key, no async', ], + 'test_suite_config.mbedtls_boolean': [ + # We never test with CBC/PKCS5/PKCS12 enabled but + # PKCS7 padding disabled. + # https://github.com/Mbed-TLS/mbedtls/issues/9580 + 'Config: !MBEDTLS_CIPHER_PADDING_PKCS7', + # https://github.com/Mbed-TLS/mbedtls/issues/9583 + 'Config: !MBEDTLS_ECP_NIST_OPTIM', + # Missing coverage of test configurations. + # https://github.com/Mbed-TLS/mbedtls/issues/9585 + 'Config: !MBEDTLS_SSL_DTLS_ANTI_REPLAY', + # Missing coverage of test configurations. + # https://github.com/Mbed-TLS/mbedtls/issues/9585 + 'Config: !MBEDTLS_SSL_DTLS_HELLO_VERIFY', + # We don't run test_suite_config when we test this. + # https://github.com/Mbed-TLS/mbedtls/issues/9586 + 'Config: !MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ENABLED', + # We only test multithreading with pthreads. + # https://github.com/Mbed-TLS/mbedtls/issues/9584 + 'Config: !MBEDTLS_THREADING_PTHREAD', + # Built but not tested. + # https://github.com/Mbed-TLS/mbedtls/issues/9587 + 'Config: MBEDTLS_AES_USE_HARDWARE_ONLY', + # Untested platform-specific optimizations. + # https://github.com/Mbed-TLS/mbedtls/issues/9588 + 'Config: MBEDTLS_HAVE_SSE2', + # Obsolete configuration option, to be replaced by + # PSA entropy drivers. + # https://github.com/Mbed-TLS/mbedtls/issues/8150 + 'Config: MBEDTLS_NO_PLATFORM_ENTROPY', + # Untested aspect of the platform interface. + # https://github.com/Mbed-TLS/mbedtls/issues/9589 + 'Config: MBEDTLS_PLATFORM_NO_STD_FUNCTIONS', + # In a client-server build, test_suite_config runs in the + # client configuration, so it will never report + # MBEDTLS_PSA_CRYPTO_SPM as enabled. That's ok. + 'Config: MBEDTLS_PSA_CRYPTO_SPM', + # We don't test on armv8 yet. + 'Config: MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT', + 'Config: MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY', + 'Config: MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY', + 'Config: MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY', + # We don't run test_suite_config when we test this. + # https://github.com/Mbed-TLS/mbedtls/issues/9586 + 'Config: MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND', + ], + 'test_suite_config.psa_boolean': [ + # We don't test with HMAC disabled. + # https://github.com/Mbed-TLS/mbedtls/issues/9591 + 'Config: !PSA_WANT_ALG_HMAC', + # We don't test with HMAC disabled. + # https://github.com/Mbed-TLS/mbedtls/issues/9591 + 'Config: !PSA_WANT_ALG_TLS12_PRF', + # The DERIVE key type is always enabled. + 'Config: !PSA_WANT_KEY_TYPE_DERIVE', + # More granularity of key pair type enablement macros + # than we care to test. + # https://github.com/Mbed-TLS/mbedtls/issues/9590 + 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_EXPORT', + 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_GENERATE', + 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_IMPORT', + # More granularity of key pair type enablement macros + # than we care to test. + # https://github.com/Mbed-TLS/mbedtls/issues/9590 + 'Config: !PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT', + 'Config: !PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT', + # We don't test with HMAC disabled. + # https://github.com/Mbed-TLS/mbedtls/issues/9591 + 'Config: !PSA_WANT_KEY_TYPE_HMAC', + # The PASSWORD key type is always enabled. + 'Config: !PSA_WANT_KEY_TYPE_PASSWORD', + # The PASSWORD_HASH key type is always enabled. + 'Config: !PSA_WANT_KEY_TYPE_PASSWORD_HASH', + # The RAW_DATA key type is always enabled. + 'Config: !PSA_WANT_KEY_TYPE_RAW_DATA', + # More granularity of key pair type enablement macros + # than we care to test. + # https://github.com/Mbed-TLS/mbedtls/issues/9590 + 'Config: !PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_EXPORT', + 'Config: !PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_IMPORT', + # Algorithm declared but not supported. + 'Config: PSA_WANT_ALG_CBC_MAC', + # Algorithm declared but not supported. + 'Config: PSA_WANT_ALG_XTS', + # Family declared but not supported. + 'Config: PSA_WANT_ECC_SECP_K1_224', + # More granularity of key pair type enablement macros + # than we care to test. + # https://github.com/Mbed-TLS/mbedtls/issues/9590 + 'Config: PSA_WANT_KEY_TYPE_DH_KEY_PAIR_DERIVE', + 'Config: PSA_WANT_KEY_TYPE_ECC_KEY_PAIR', + 'Config: PSA_WANT_KEY_TYPE_RSA_KEY_PAIR', + 'Config: PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_DERIVE', + ], + 'test_suite_config.psa_combinations': [ + # We don't test this unusual, but sensible configuration. + # https://github.com/Mbed-TLS/mbedtls/issues/9592 + 'Config: PSA_WANT_ALG_DETERMINSTIC_ECDSA without PSA_WANT_ALG_ECDSA', + ], 'test_suite_pkcs12': [ - # Probably a missing configuration in the CI. + # We never test with CBC/PKCS5/PKCS12 enabled but + # PKCS7 padding disabled. # https://github.com/Mbed-TLS/mbedtls/issues/9580 'PBE Decrypt, (Invalid padding & PKCS7 padding disabled)', 'PBE Encrypt, pad = 8 (PKCS7 padding disabled)', ], 'test_suite_pkcs5': [ - # Probably a missing configuration in the CI. + # We never test with CBC/PKCS5/PKCS12 enabled but + # PKCS7 padding disabled. # https://github.com/Mbed-TLS/mbedtls/issues/9580 'PBES2 Decrypt (Invalid padding & PKCS7 padding disabled)', 'PBES2 Encrypt, pad=6 (PKCS7 padding disabled)', From 3b427c72be4e52bbbff1bfff6c9c0d353f7b5a6c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 10 Oct 2024 18:18:45 +0200 Subject: [PATCH 08/14] Add missing algorithm in the test driver configuration Signed-off-by: Gilles Peskine --- tests/configs/user-config-for-test.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/configs/user-config-for-test.h b/tests/configs/user-config-for-test.h index 639496be60..e187ae2aab 100644 --- a/tests/configs/user-config-for-test.h +++ b/tests/configs/user-config-for-test.h @@ -39,6 +39,7 @@ /* Use the accelerator driver for all cryptographic mechanisms for which * the test driver implemented. */ #define MBEDTLS_PSA_ACCEL_KEY_TYPE_AES +#define MBEDTLS_PSA_ACCEL_KEY_TYPE_ARIA #define MBEDTLS_PSA_ACCEL_KEY_TYPE_CAMELLIA #define MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_PUBLIC_KEY #define MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_BASIC From 44fdd9293c077e8c8953dde00c255b84c8ff7c09 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 10 Oct 2024 18:19:23 +0200 Subject: [PATCH 09/14] Add ALT-adjacent config option to the test coverage ignore list MBEDTLS_ECP_NO_FALLBACK is only relevant when an ALT implementation of ECP is used. We don't test ALT implementations so we don't test MBEDTLS_ECP_NO_FALLBACK either. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 75dc41d6f8..5df9c213b2 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -99,6 +99,13 @@ class CoverageTask(outcome_analysis.CoverageTask): 'Config: !MBEDTLS_CIPHER_PADDING_PKCS7', # https://github.com/Mbed-TLS/mbedtls/issues/9583 'Config: !MBEDTLS_ECP_NIST_OPTIM', + # MBEDTLS_ECP_NO_FALLBACK only affects builds using a partial + # alternative implementation of ECP arithmetic (with + # MBEDTLS_ECP_INTERNAL_ALT enabled). We don't test those builds. + # The configuration enumeration script skips xxx_ALT options + # but not MBEDTLS_ECP_NO_FALLBACK, so it appears in the report, + # but we don't care about it. + 'Config: MBEDTLS_ECP_NO_FALLBACK', # Missing coverage of test configurations. # https://github.com/Mbed-TLS/mbedtls/issues/9585 'Config: !MBEDTLS_SSL_DTLS_ANTI_REPLAY', From 4e606db7c2bf3535b6de2a9e1137ddadb405fb62 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 4 Oct 2024 16:24:26 +0200 Subject: [PATCH 10/14] Switch outcome analysis to enforcing that all tests are executed Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 5df9c213b2..42f51e6906 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -14,9 +14,7 @@ from mbedtls_framework import outcome_analysis class CoverageTask(outcome_analysis.CoverageTask): - # We'll populate IGNORED_TESTS soon. In the meantime, lack of coverage - # is just a warning. - outcome_analysis.FULL_COVERAGE_BY_DEFAULT = False + """Justify test cases that are never executed.""" @staticmethod def _has_word_re(words: typing.Iterable[str], From 8f886456f90c31290849f3449b9f5e427450b68a Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 10 Oct 2024 18:33:54 +0200 Subject: [PATCH 11/14] framework: Fix overly lenient config tests for PSA_WANT_xxx Signed-off-by: Gilles Peskine --- framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework b/framework index 1de0641e78..239c0d8d17 160000 --- a/framework +++ b/framework @@ -1 +1 @@ -Subproject commit 1de0641e789d3c38b3ce99d7922002992cbe816c +Subproject commit 239c0d8d17884d192654519ce0de4d4c5ee4ae09 From ccd1440c484b110dfdbc41d7e332ab3d1892597a Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 11 Oct 2024 10:02:59 +0200 Subject: [PATCH 12/14] Remove ignore list entries that don't apply in 3.6 In 3.6, compared with the state of development that this work was based on, we're doing more testing with TLS12_PRF and TLS12_PSK_TO_MS disabled, thanks to some configurations with MBEDTLS_PSA_CRYPTO_CONFIG disabled. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 42f51e6906..c4581e677b 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -146,9 +146,6 @@ class CoverageTask(outcome_analysis.CoverageTask): # We don't test with HMAC disabled. # https://github.com/Mbed-TLS/mbedtls/issues/9591 'Config: !PSA_WANT_ALG_HMAC', - # We don't test with HMAC disabled. - # https://github.com/Mbed-TLS/mbedtls/issues/9591 - 'Config: !PSA_WANT_ALG_TLS12_PRF', # The DERIVE key type is always enabled. 'Config: !PSA_WANT_KEY_TYPE_DERIVE', # More granularity of key pair type enablement macros @@ -266,14 +263,6 @@ class CoverageTask(outcome_analysis.CoverageTask): # enable it in depends.py where we remove hashes. # https://github.com/Mbed-TLS/mbedtls/issues/9576 re.compile(r'PSA key_derivation PBKDF2_HMAC\(\w+\): !(?!PBKDF2_HMAC\Z).*'), - # We never test with TLS12_PRF or TLS12_PSK_TO_MS disabled - # but certain other things enabled. - # https://github.com/Mbed-TLS/mbedtls/issues/9577 - re.compile(r'PSA key_derivation TLS12_PRF\(\w+\): !TLS12_PRF'), - re.compile(r'PSA key_derivation TLS12_PSK_TO_MS' - r'\((?!SHA_256|SHA_384|SHA_512)\w+\): !TLS12_PSK_TO_MS'), - 'PSA key_derivation KEY_AGREEMENT(ECDH,TLS12_PRF(SHA_256)): !TLS12_PRF', - 'PSA key_derivation KEY_AGREEMENT(ECDH,TLS12_PRF(SHA_384)): !TLS12_PRF', # We never test with the HMAC algorithm enabled but the HMAC # key type disabled. Those dependencies don't really make sense. From 1abc800adc678eabd7a1d7d11d13b035292be5f3 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 11 Oct 2024 12:00:44 +0200 Subject: [PATCH 13/14] Minor readability improvements Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index c4581e677b..1f7e98c525 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -31,7 +31,7 @@ class CoverageTask(outcome_analysis.CoverageTask): exclude_clause = r'(?!.*' + exclude + ')' return re.compile(exclude_clause + r'.*\b(?:' + r'|'.join(words) + r')\b.*', - re.S) + re.DOTALL) # generate_psa_tests.py generates test cases involving cryptographic # mechanisms (key types, families, algorithms) that are declared but @@ -73,7 +73,6 @@ class CoverageTask(outcome_analysis.CoverageTask): # We don't run ssl-opt.sh with Valgrind on the CI because # it's extremely slow. We don't intend to change this. 'DTLS client reconnect from same port: reconnect, nbio, valgrind', - # We don't have IPv6 in our CI environment. # https://github.com/Mbed-TLS/mbedtls-test/issues/176 'DTLS cookie: enabled, IPv6', @@ -234,7 +233,7 @@ class CoverageTask(outcome_analysis.CoverageTask): # run for never-implemented key types. # https://github.com/Mbed-TLS/mbedtls/issues/7915 PSA_MECHANISM_NOT_IMPLEMENTED_SEARCH_RE, - # We mever test with DH key support disabled but support + # We never test with DH key support disabled but support # for a DH group enabled. The dependencies of these test # cases don't really make sense. # https://github.com/Mbed-TLS/mbedtls/issues/9574 From 6d268e3fe10f9e21220603d14ea9fa4faf0feb4b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 16 Oct 2024 19:33:59 +0200 Subject: [PATCH 14/14] Update framework to main Signed-off-by: Gilles Peskine --- framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework b/framework index 239c0d8d17..3eafac12ae 160000 --- a/framework +++ b/framework @@ -1 +1 @@ -Subproject commit 239c0d8d17884d192654519ce0de4d4c5ee4ae09 +Subproject commit 3eafac12ae1ddc68cc1f0aefdff540d6d3d5a2fb