From 419bacc0494c7857bdb755221a8111e9d9506fc5 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Sep 2024 18:50:27 +0200 Subject: [PATCH 01/12] Allow running pylint and mypy on a single file Fix `mypy scripts/xxx.py`, `mypy tests/scripts/xxx.py`, `pylint scripts/xxx.py`, `pylint tests/scripts/xxx.py` failing to find `mbedtls_framework`. Signed-off-by: Gilles Peskine --- .mypy.ini | 2 +- .pylintrc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.mypy.ini b/.mypy.ini index 6b831ddb77..f727cc20e7 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -1,4 +1,4 @@ [mypy] -mypy_path = scripts +mypy_path = framework/scripts:scripts namespace_packages = True warn_unused_configs = True diff --git a/.pylintrc b/.pylintrc index f395fb91e7..f9c97d55ea 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,5 +1,5 @@ [MASTER] -init-hook='import sys; sys.path.append("scripts")' +init-hook='import sys; sys.path.append("scripts"); sys.path.append("framework/scripts")' min-similarity-lines=10 [BASIC] From 19ef1ae72ed9fd3b0d65317d182734a7eaca6819 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Sep 2024 19:12:09 +0200 Subject: [PATCH 02/12] Replace stringly typed data by class: prepare Start replacing the stringly typed KNOWN_TASKS by classes for each category of tasks, with a structure that matches the behavior. This commit introduces some transition code. No intended behavior change. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 35 ++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index e78e90c1f5..5835c80948 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -211,6 +211,25 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) - args['component_ref'], args['component_driver'], ignored_suites, args['ignored_tests']) + +class Task: + """Base class for outcome analysis tasks.""" + + def __init__(self, options) -> None: + """Pass command line options to the tasks. + + Each task decides which command line options it cares about. + """ + pass + + def run(self, results: Results, outcomes: Outcomes): + """Run the analysis on the specified outcomes. + + Signal errors via the results objects + """ + raise NotImplementedError + + # List of tasks with a function that can handle this task and additional arguments if required KNOWN_TASKS = { 'analyze_coverage': { @@ -766,7 +785,8 @@ def main(): task_name = tasks_list[0] task = KNOWN_TASKS[task_name] - if task['test_function'] != do_analyze_driver_vs_reference: # pylint: disable=comparison-with-callable + if isinstance(task, dict) and \ + task['test_function'] != do_analyze_driver_vs_reference: # pylint: disable=comparison-with-callable sys.stderr.write("please provide valid outcomes file for {}.\n".format(task_name)) sys.exit(2) @@ -777,10 +797,15 @@ def main(): outcomes = read_outcome_file(options.outcomes) - for task in tasks_list: - test_function = KNOWN_TASKS[task]['test_function'] - test_args = KNOWN_TASKS[task]['args'] - test_function(main_results, outcomes, test_args) + for task_name in tasks_list: + task_constructor = KNOWN_TASKS[task_name] + if isinstance(task_constructor, dict): + test_function = task_constructor['test_function'] + test_args = task_constructor['args'] + test_function(main_results, outcomes, test_args) + else: + task = task_constructor(options) + task.run(main_results, outcomes) main_results.info("Overall results: {} warnings and {} errors", main_results.warning_count, main_results.error_count) From f646dbf71dc4ab6b66557c980adb3f3c7ee35198 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Sep 2024 19:15:29 +0200 Subject: [PATCH 03/12] Replace stringly typed data by class: coverage Work on replacing the stringly typed KNOWN_TASKS by classes for each category of tasks, with a structure that matches the behavior. This commit migrates test coverage analysis. No intended behavior change. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 54 +++++++++++++++++-------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 5835c80948..98c4afdbb9 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -170,11 +170,6 @@ def analyze_driver_vs_reference(results: Results, outcomes: Outcomes, if ignored and suite_case in driver_outcomes.successes: results.error("uselessly ignored: {}", suite_case) -def analyze_outcomes(results: Results, outcomes: Outcomes, args) -> None: - """Run all analyses on the given outcome collection.""" - analyze_coverage(results, outcomes, args['allow_list'], - args['full_coverage']) - def read_outcome_file(outcome_file: str) -> Outcomes: """Parse an outcome file and return an outcome collection. """ @@ -195,11 +190,6 @@ def read_outcome_file(outcome_file: str) -> Outcomes: return outcomes -def do_analyze_coverage(results: Results, outcomes: Outcomes, args) -> None: - """Perform coverage analysis.""" - results.new_section("Analyze coverage") - analyze_outcomes(results, outcomes, args) - def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -> None: """Perform driver vs reference analyze.""" results.new_section("Analyze driver {} vs reference {}", @@ -222,6 +212,9 @@ class Task: """ pass + def section_name(self) -> str: + """The section name to use in results.""" + def run(self, results: Results, outcomes: Outcomes): """Run the analysis on the specified outcomes. @@ -230,20 +223,34 @@ class Task: raise NotImplementedError +class CoverageTask(Task): + """Analyze test coverage.""" + + ALLOW_LIST = [ + # Algorithm not supported yet + 'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA', + # Algorithm not supported yet + 'test_suite_psa_crypto_metadata;Cipher: XTS', + ] + + def __init__(self, options) -> None: + super().__init__(options) + self.full_coverage = options.full_coverage #type: bool + + @staticmethod + def section_name() -> str: + return "Analyze coverage" + + def run(self, results: Results, outcomes: Outcomes): + """Check that all test cases are executed at least once.""" + analyze_coverage(results, outcomes, + self.ALLOW_LIST, self.full_coverage) + + # List of tasks with a function that can handle this task and additional arguments if required KNOWN_TASKS = { - 'analyze_coverage': { - 'test_function': do_analyze_coverage, - 'args': { - 'allow_list': [ - # Algorithm not supported yet - 'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA', - # Algorithm not supported yet - 'test_suite_psa_crypto_metadata;Cipher: XTS', - ], - 'full_coverage': False, - } - }, + 'analyze_coverage': CoverageTask, + # There are 2 options to use analyze_driver_vs_reference_xxx locally: # 1. Run tests and then analysis: # - tests/scripts/all.sh --outcome-file "$PWD/out.csv" @@ -773,8 +780,6 @@ def main(): sys.stderr.write('invalid task: {}\n'.format(task)) sys.exit(2) - KNOWN_TASKS['analyze_coverage']['args']['full_coverage'] = options.full_coverage - # If the outcome file exists, parse it once and share the result # among tasks to improve performance. # Otherwise, it will be generated by execute_reference_driver_tests. @@ -805,6 +810,7 @@ def main(): test_function(main_results, outcomes, test_args) else: task = task_constructor(options) + main_results.new_section(task.section_name()) task.run(main_results, outcomes) main_results.info("Overall results: {} warnings and {} errors", From 82b16721bd398880eab5694d16831b36d6d46fe2 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Sep 2024 19:57:10 +0200 Subject: [PATCH 04/12] Replace stringly typed data by class: driver vs reference (code) Work on the stringly typed KNOWN_TASKS by classes for each category of tasks, with a structure that matches the behavior. This commit migrates the code for driver-vs-reference analysis. To facilitate review, this commit preserves the layout of the data that parametrizes each task. The next commit will migrate the data. No intended behavior change. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 142 ++++++++++++++++-------------- 1 file changed, 75 insertions(+), 67 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 98c4afdbb9..076a29d738 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -114,7 +114,9 @@ def analyze_coverage(results: Results, outcomes: Outcomes, else: results.warning('Allow listed test case was executed: {}', suite_case) -def name_matches_pattern(name: str, str_or_re) -> bool: +IgnoreEntry = typing.Union[str, typing.Pattern] + +def name_matches_pattern(name: str, str_or_re: IgnoreEntry) -> bool: """Check if name matches a pattern, that may be a string or regex. - If the pattern is a string, name must be equal to match. - If the pattern is a regex, name must fully match. @@ -190,17 +192,6 @@ def read_outcome_file(outcome_file: str) -> Outcomes: return outcomes -def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -> None: - """Perform driver vs reference analyze.""" - results.new_section("Analyze driver {} vs reference {}", - args['component_driver'], args['component_ref']) - - ignored_suites = ['test_suite_' + x for x in args['ignored_suites']] - - analyze_driver_vs_reference(results, outcomes, - args['component_ref'], args['component_driver'], - ignored_suites, args['ignored_tests']) - class Task: """Base class for outcome analysis tasks.""" @@ -247,19 +238,58 @@ class CoverageTask(Task): self.ALLOW_LIST, self.full_coverage) +class DriverVSReference(Task): + """Compare outcomes from testing with and without a driver. + + There are 2 options to use analyze_driver_vs_reference_xxx locally: + 1. Run tests and then analysis: + - tests/scripts/all.sh --outcome-file "$PWD/out.csv" + - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx + 2. Let this script run both automatically: + - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx + """ + + # Override the following in child classes. + # Configuration name (all.sh component) used as the reference. + REFERENCE = '' + # Configuration name (all.sh component) used as the driver. + DRIVER = '' + # Ignored test suites (without the test_suite_ prefix). + IGNORED_SUITES = [] #type: typing.List[str] + # Map test suite names (with the test_suite_prefix) to a list of ignored + # test cases. Each element in the list can be either a string or a regex; + # see the `name_matches_pattern` function. + IGNORED_TESTS = {} #type: typing.Dict[str, typing.List[IgnoreEntry]] + + def section_name(self) -> str: + return f"Analyze driver {self.DRIVER} vs reference {self.REFERENCE}" + + def run(self, results: Results, outcomes: Outcomes) -> None: + """Compare driver test outcomes with reference outcomes.""" + ignored_suites = ['test_suite_' + x for x in self.IGNORED_SUITES] + analyze_driver_vs_reference(results, outcomes, + self.REFERENCE, self.DRIVER, + ignored_suites, self.IGNORED_TESTS) + + +def driver_vs_reference_factory(name: str, + args: typing.Dict[str, typing.Any] + ) -> typing.Type[DriverVSReference]: + """Build a driver-vs-reference class from dynamic data.""" + return type('Drivervsreference_' + name, + (DriverVSReference,), + { + 'REFERENCE': args['component_ref'], + 'DRIVER': args['component_driver'], + 'IGNORED_SUITES': args['ignored_suites'], + 'IGNORED_TESTS': args['ignored_tests'], + }) + # List of tasks with a function that can handle this task and additional arguments if required KNOWN_TASKS = { 'analyze_coverage': CoverageTask, - # There are 2 options to use analyze_driver_vs_reference_xxx locally: - # 1. Run tests and then analysis: - # - tests/scripts/all.sh --outcome-file "$PWD/out.csv" - # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx - # 2. Let this script run both automatically: - # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx - 'analyze_driver_vs_reference_hash': { - 'test_function': do_analyze_driver_vs_reference, - 'args': { + 'analyze_driver_vs_reference_hash': driver_vs_reference_factory('hash', { 'component_ref': 'test_psa_crypto_config_reference_hash_use_psa', 'component_driver': 'test_psa_crypto_config_accel_hash_use_psa', 'ignored_suites': [ @@ -279,10 +309,8 @@ KNOWN_TASKS = { ], } } - }, - 'analyze_driver_vs_reference_hmac': { - 'test_function': do_analyze_driver_vs_reference, - 'args': { + ), + 'analyze_driver_vs_reference_hmac': driver_vs_reference_factory('hmac', { 'component_ref': 'test_psa_crypto_config_reference_hmac', 'component_driver': 'test_psa_crypto_config_accel_hmac', 'ignored_suites': [ @@ -321,10 +349,8 @@ KNOWN_TASKS = { ], } } - }, - 'analyze_driver_vs_reference_cipher_aead_cmac': { - 'test_function': do_analyze_driver_vs_reference, - 'args': { + ), + 'analyze_driver_vs_reference_cipher_aead_cmac': driver_vs_reference_factory('cipher_aead_cmac', { 'component_ref': 'test_psa_crypto_config_reference_cipher_aead_cmac', 'component_driver': 'test_psa_crypto_config_accel_cipher_aead_cmac', # Modules replaced by drivers. @@ -391,10 +417,8 @@ KNOWN_TASKS = { ], } } - }, - 'analyze_driver_vs_reference_ecp_light_only': { - 'test_function': do_analyze_driver_vs_reference, - 'args': { + ), + 'analyze_driver_vs_reference_ecp_light_only': driver_vs_reference_factory('ecp_light_only', { 'component_ref': 'test_psa_crypto_config_reference_ecc_ecp_light_only', 'component_driver': 'test_psa_crypto_config_accel_ecc_ecp_light_only', 'ignored_suites': [ @@ -434,10 +458,8 @@ KNOWN_TASKS = { ], } } - }, - 'analyze_driver_vs_reference_no_ecp_at_all': { - 'test_function': do_analyze_driver_vs_reference, - 'args': { + ), + 'analyze_driver_vs_reference_no_ecp_at_all': driver_vs_reference_factory('no_ecp_at_all', { 'component_ref': 'test_psa_crypto_config_reference_ecc_no_ecp_at_all', 'component_driver': 'test_psa_crypto_config_accel_ecc_no_ecp_at_all', 'ignored_suites': [ @@ -475,10 +497,8 @@ KNOWN_TASKS = { ], } } - }, - 'analyze_driver_vs_reference_ecc_no_bignum': { - 'test_function': do_analyze_driver_vs_reference, - 'args': { + ), + 'analyze_driver_vs_reference_ecc_no_bignum': driver_vs_reference_factory('ecc_no_bignum', { 'component_ref': 'test_psa_crypto_config_reference_ecc_no_bignum', 'component_driver': 'test_psa_crypto_config_accel_ecc_no_bignum', 'ignored_suites': [ @@ -523,10 +543,8 @@ KNOWN_TASKS = { ], } } - }, - 'analyze_driver_vs_reference_ecc_ffdh_no_bignum': { - 'test_function': do_analyze_driver_vs_reference, - 'args': { + ), + 'analyze_driver_vs_reference_ecc_ffdh_no_bignum': driver_vs_reference_factory('ecc_ffdh_no_bignum', { 'component_ref': 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum', 'component_driver': 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum', 'ignored_suites': [ @@ -579,10 +597,8 @@ KNOWN_TASKS = { ], } } - }, - 'analyze_driver_vs_reference_ffdh_alg': { - 'test_function': do_analyze_driver_vs_reference, - 'args': { + ), + 'analyze_driver_vs_reference_ffdh_alg': driver_vs_reference_factory('ffdh_alg', { 'component_ref': 'test_psa_crypto_config_reference_ffdh', 'component_driver': 'test_psa_crypto_config_accel_ffdh', 'ignored_suites': ['dhm'], @@ -598,10 +614,8 @@ KNOWN_TASKS = { ], } } - }, - 'analyze_driver_vs_reference_tfm_config': { - 'test_function': do_analyze_driver_vs_reference, - 'args': { + ), + 'analyze_driver_vs_reference_tfm_config': driver_vs_reference_factory('tfm_config', { 'component_ref': 'test_tfm_config_no_p256m', 'component_driver': 'test_tfm_config_p256m_driver_accel_ec', 'ignored_suites': [ @@ -633,10 +647,8 @@ KNOWN_TASKS = { ], } } - }, - 'analyze_driver_vs_reference_rsa': { - 'test_function': do_analyze_driver_vs_reference, - 'args': { + ), + 'analyze_driver_vs_reference_rsa': driver_vs_reference_factory('rsa', { 'component_ref': 'test_psa_crypto_config_reference_rsa_crypto', 'component_driver': 'test_psa_crypto_config_accel_rsa_crypto', 'ignored_suites': [ @@ -675,10 +687,8 @@ KNOWN_TASKS = { ], } } - }, - 'analyze_block_cipher_dispatch': { - 'test_function': do_analyze_driver_vs_reference, - 'args': { + ), + 'analyze_block_cipher_dispatch': driver_vs_reference_factory('block_cipher_dispatch', { 'component_ref': 'test_full_block_cipher_legacy_dispatch', 'component_driver': 'test_full_block_cipher_psa_dispatch', 'ignored_suites': [ @@ -742,7 +752,7 @@ KNOWN_TASKS = { ], } } - } + ), } def main(): @@ -790,14 +800,12 @@ def main(): task_name = tasks_list[0] task = KNOWN_TASKS[task_name] - if isinstance(task, dict) and \ - task['test_function'] != do_analyze_driver_vs_reference: # pylint: disable=comparison-with-callable + if not issubclass(task, DriverVSReference): sys.stderr.write("please provide valid outcomes file for {}.\n".format(task_name)) sys.exit(2) - execute_reference_driver_tests(main_results, - task['args']['component_ref'], - task['args']['component_driver'], + task.REFERENCE, + task.DRIVER, options.outcomes) outcomes = read_outcome_file(options.outcomes) From 9df375b018f43e60c5901fa88655965499c50004 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Sep 2024 20:14:26 +0200 Subject: [PATCH 05/12] Replace stringly typed data by class: driver vs reference (data) Work on the stringly typed KNOWN_TASKS by classes for each category of tasks, with a structure that matches the behavior. This commit migrates the data for driver-vs-reference analysis and gets rid of the transitional code that was using the old form of the data. No intended behavior change. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 952 +++++++++++++++--------------- 1 file changed, 475 insertions(+), 477 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 076a29d738..bfddf9830b 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -272,489 +272,487 @@ class DriverVSReference(Task): ignored_suites, self.IGNORED_TESTS) -def driver_vs_reference_factory(name: str, - args: typing.Dict[str, typing.Any] - ) -> typing.Type[DriverVSReference]: - """Build a driver-vs-reference class from dynamic data.""" - return type('Drivervsreference_' + name, - (DriverVSReference,), - { - 'REFERENCE': args['component_ref'], - 'DRIVER': args['component_driver'], - 'IGNORED_SUITES': args['ignored_suites'], - 'IGNORED_TESTS': args['ignored_tests'], - }) +# The names that we give to classes derived from DriverVSReference do not +# follow the usual naming convention, because it's more readable to use +# underscores and parts of the configuration names. Also, these classes +# are just there to specify some data, so they don't need repetitive +# documentation. +#pylint: disable=invalid-name,missing-class-docstring + +class DriverVSReference_hash(DriverVSReference): + REFERENCE = 'test_psa_crypto_config_reference_hash_use_psa' + DRIVER = 'test_psa_crypto_config_accel_hash_use_psa' + IGNORED_SUITES = [ + 'shax', 'mdx', # the software implementations that are being excluded + 'md.psa', # purposefully depends on whether drivers are present + 'psa_crypto_low_hash.generated', # testing the builtins + ] + IGNORED_TESTS = { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'), + ], + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + } + +class DriverVSReference_hmac(DriverVSReference): + REFERENCE = 'test_psa_crypto_config_reference_hmac' + DRIVER = 'test_psa_crypto_config_accel_hmac' + IGNORED_SUITES = [ + # These suites require legacy hash support, which is disabled + # in the accelerated component. + 'shax', 'mdx', + # This suite tests builtins directly, but these are missing + # in the accelerated case. + 'psa_crypto_low_hash.generated', + ] + IGNORED_TESTS = { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'), + re.compile(r'.*\bMBEDTLS_MD_C\b') + ], + 'test_suite_md': [ + # Builtin HMAC is not supported in the accelerate component. + re.compile('.*HMAC.*'), + # Following tests make use of functions which are not available + # when MD_C is disabled, as it happens in the accelerated + # test component. + re.compile('generic .* Hash file .*'), + 'MD list', + ], + 'test_suite_md.psa': [ + # "legacy only" tests require hash algorithms to be NOT + # accelerated, but this of course false for the accelerated + # test component. + re.compile('PSA dispatch .* legacy only'), + ], + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + } + +class DriverVSReference_cipher_aead_cmac(DriverVSReference): + REFERENCE = 'test_psa_crypto_config_reference_cipher_aead_cmac' + DRIVER = 'test_psa_crypto_config_accel_cipher_aead_cmac' + # Modules replaced by drivers. + IGNORED_SUITES = [ + # low-level (block/stream) cipher modules + 'aes', 'aria', 'camellia', 'des', 'chacha20', + # AEAD modes and CMAC + 'ccm', 'chachapoly', 'cmac', 'gcm', + # The Cipher abstraction layer + 'cipher', + ] + IGNORED_TESTS = { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA|CHACHA20|DES)_.*'), + re.compile(r'.*\bMBEDTLS_(CCM|CHACHAPOLY|CMAC|GCM)_.*'), + re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'), + re.compile(r'.*\bMBEDTLS_CIPHER_.*'), + ], + # PEM decryption is not supported so far. + # The rest of PEM (write, unencrypted read) works though. + 'test_suite_pem': [ + re.compile(r'PEM read .*(AES|DES|\bencrypt).*'), + ], + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + # Following tests depend on AES_C/DES_C but are not about + # them really, just need to know some error code is there. + 'test_suite_error': [ + 'Low and high error', + 'Single low error' + ], + # Similar to test_suite_error above. + 'test_suite_version': [ + 'Check for MBEDTLS_AES_C when already present', + ], + # The en/decryption part of PKCS#12 is not supported so far. + # The rest of PKCS#12 (key derivation) works though. + 'test_suite_pkcs12': [ + re.compile(r'PBE Encrypt, .*'), + re.compile(r'PBE Decrypt, .*'), + ], + # The en/decryption part of PKCS#5 is not supported so far. + # The rest of PKCS#5 (PBKDF2) works though. + 'test_suite_pkcs5': [ + re.compile(r'PBES2 Encrypt, .*'), + re.compile(r'PBES2 Decrypt .*'), + ], + # Encrypted keys are not supported so far. + # pylint: disable=line-too-long + 'test_suite_pkparse': [ + 'Key ASN1 (Encrypted key PKCS12, trailing garbage data)', + 'Key ASN1 (Encrypted key PKCS5, trailing garbage data)', + re.compile(r'Parse (RSA|EC) Key .*\(.* ([Ee]ncrypted|password).*\)'), + ], + # Encrypted keys are not supported so far. + 'ssl-opt': [ + 'TLS: password protected server key', + 'TLS: password protected client key', + 'TLS: password protected server key, two certificates', + ], + } + +class DriverVSReference_ecp_light_only(DriverVSReference): + REFERENCE = 'test_psa_crypto_config_reference_ecc_ecp_light_only' + DRIVER = 'test_psa_crypto_config_accel_ecc_ecp_light_only' + IGNORED_SUITES = [ + # Modules replaced by drivers + 'ecdsa', 'ecdh', 'ecjpake', + ] + IGNORED_TESTS = { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), + ], + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + # This test wants a legacy function that takes f_rng, p_rng + # arguments, and uses legacy ECDSA for that. The test is + # really about the wrapper around the PSA RNG, not ECDSA. + 'test_suite_random': [ + 'PSA classic wrapper: ECDSA signature (SECP256R1)', + ], + # In the accelerated test ECP_C is not set (only ECP_LIGHT is) + # so we must ignore disparities in the tests for which ECP_C + # is required. + 'test_suite_ecp': [ + re.compile(r'ECP check public-private .*'), + re.compile(r'ECP calculate public: .*'), + re.compile(r'ECP gen keypair .*'), + re.compile(r'ECP point muladd .*'), + re.compile(r'ECP point multiplication .*'), + re.compile(r'ECP test vectors .*'), + ], + 'test_suite_ssl': [ + # This deprecated function is only present when ECP_C is On. + 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', + ], + } + +class DriverVSReference_no_ecp_at_all(DriverVSReference): + REFERENCE = 'test_psa_crypto_config_reference_ecc_no_ecp_at_all' + DRIVER = 'test_psa_crypto_config_accel_ecc_no_ecp_at_all' + IGNORED_SUITES = [ + # Modules replaced by drivers + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', + ] + IGNORED_TESTS = { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), + re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), + ], + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + # See ecp_light_only + 'test_suite_random': [ + 'PSA classic wrapper: ECDSA signature (SECP256R1)', + ], + 'test_suite_pkparse': [ + # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED + # is automatically enabled in build_info.h (backward compatibility) + # even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a + # consequence compressed points are supported in the reference + # component but not in the accelerated one, so they should be skipped + # while checking driver's coverage. + re.compile(r'Parse EC Key .*compressed\)'), + re.compile(r'Parse Public EC Key .*compressed\)'), + ], + # See ecp_light_only + 'test_suite_ssl': [ + 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', + ], + } + +class DriverVSReference_ecc_no_bignum(DriverVSReference): + REFERENCE = 'test_psa_crypto_config_reference_ecc_no_bignum' + DRIVER = 'test_psa_crypto_config_accel_ecc_no_bignum' + IGNORED_SUITES = [ + # Modules replaced by drivers + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', + 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', + 'bignum.generated', 'bignum.misc', + ] + IGNORED_TESTS = { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), + re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), + re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), + ], + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + # See ecp_light_only + 'test_suite_random': [ + 'PSA classic wrapper: ECDSA signature (SECP256R1)', + ], + # See no_ecp_at_all + 'test_suite_pkparse': [ + re.compile(r'Parse EC Key .*compressed\)'), + re.compile(r'Parse Public EC Key .*compressed\)'), + ], + 'test_suite_asn1parse': [ + 'INTEGER too large for mpi', + ], + 'test_suite_asn1write': [ + re.compile(r'ASN.1 Write mpi.*'), + ], + 'test_suite_debug': [ + re.compile(r'Debug print mbedtls_mpi.*'), + ], + # See ecp_light_only + 'test_suite_ssl': [ + 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', + ], + } + +class DriverVSReference_ecc_ffdh_no_bignum(DriverVSReference): + REFERENCE = 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum' + DRIVER = 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum' + IGNORED_SUITES = [ + # Modules replaced by drivers + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm', + 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', + 'bignum.generated', 'bignum.misc', + ] + IGNORED_TESTS = { + 'ssl-opt': [ + # DHE support in TLS 1.2 requires built-in MBEDTLS_DHM_C + # (because it needs custom groups, which PSA does not + # provide), even with MBEDTLS_USE_PSA_CRYPTO. + re.compile(r'PSK callback:.*\bdhe-psk\b.*'), + ], + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), + re.compile(r'.*\bMBEDTLS_DHM_C\b.*'), + re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), + re.compile(r'.*\bMBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED\b.*'), + re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), + ], + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + # See ecp_light_only + 'test_suite_random': [ + 'PSA classic wrapper: ECDSA signature (SECP256R1)', + ], + # See no_ecp_at_all + 'test_suite_pkparse': [ + re.compile(r'Parse EC Key .*compressed\)'), + re.compile(r'Parse Public EC Key .*compressed\)'), + ], + 'test_suite_asn1parse': [ + 'INTEGER too large for mpi', + ], + 'test_suite_asn1write': [ + re.compile(r'ASN.1 Write mpi.*'), + ], + 'test_suite_debug': [ + re.compile(r'Debug print mbedtls_mpi.*'), + ], + # See ecp_light_only + 'test_suite_ssl': [ + 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', + ], + } + +class DriverVSReference_ffdh_alg(DriverVSReference): + REFERENCE = 'test_psa_crypto_config_reference_ffdh' + DRIVER = 'test_psa_crypto_config_accel_ffdh' + IGNORED_SUITES = ['dhm'] + IGNORED_TESTS = { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_DHM_C\b.*'), + ], + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + } + +class DriverVSReference_tfm_config(DriverVSReference): + REFERENCE = 'test_tfm_config_no_p256m' + DRIVER = 'test_tfm_config_p256m_driver_accel_ec' + IGNORED_SUITES = [ + # Modules replaced by drivers + 'asn1parse', 'asn1write', + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', + 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', + 'bignum.generated', 'bignum.misc', + ] + IGNORED_TESTS = { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), + re.compile(r'.*\bMBEDTLS_(ASN1\w+)_C\b.*'), + re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECP)_.*'), + re.compile(r'.*\bMBEDTLS_PSA_P256M_DRIVER_ENABLED\b.*') + ], + 'test_suite_config.crypto_combinations': [ + 'Config: ECC: Weierstrass curves only', + ], + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + # See ecp_light_only + 'test_suite_random': [ + 'PSA classic wrapper: ECDSA signature (SECP256R1)', + ], + } + +class DriverVSReference_rsa(DriverVSReference): + REFERENCE = 'test_psa_crypto_config_reference_rsa_crypto' + DRIVER = 'test_psa_crypto_config_accel_rsa_crypto' + IGNORED_SUITES = [ + # Modules replaced by drivers. + 'rsa', 'pkcs1_v15', 'pkcs1_v21', + # We temporarily don't care about PK stuff. + 'pk', 'pkwrite', 'pkparse' + ] + IGNORED_TESTS = { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_(PKCS1|RSA)_.*'), + re.compile(r'.*\bMBEDTLS_GENPRIME\b.*') + ], + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + # Following tests depend on RSA_C but are not about + # them really, just need to know some error code is there. + 'test_suite_error': [ + 'Low and high error', + 'Single high error' + ], + # Constant time operations only used for PKCS1_V15 + 'test_suite_constant_time': [ + re.compile(r'mbedtls_ct_zeroize_if .*'), + re.compile(r'mbedtls_ct_memmove_left .*') + ], + 'test_suite_psa_crypto': [ + # We don't support generate_key_custom entry points + # in drivers yet. + re.compile(r'PSA generate key custom: RSA, e=.*'), + re.compile(r'PSA generate key ext: RSA, e=.*'), + ], + } + +class DriverVSReference_block_cipher_dispatch(DriverVSReference): + REFERENCE = 'test_full_block_cipher_legacy_dispatch' + DRIVER = 'test_full_block_cipher_psa_dispatch' + IGNORED_SUITES = [ + # Skipped in the accelerated component + 'aes', 'aria', 'camellia', + # These require AES_C, ARIA_C or CAMELLIA_C to be enabled in + # order for the cipher module (actually cipher_wrapper) to work + # properly. However these symbols are disabled in the accelerated + # component so we ignore them. + 'cipher.ccm', 'cipher.gcm', 'cipher.aes', 'cipher.aria', + 'cipher.camellia', + ] + IGNORED_TESTS = { + 'test_suite_config': [ + re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA)_.*'), + re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'), + ], + 'test_suite_cmac': [ + # Following tests require AES_C/ARIA_C/CAMELLIA_C to be enabled, + # but these are not available in the accelerated component. + 'CMAC null arguments', + re.compile('CMAC.* (AES|ARIA|Camellia).*'), + ], + 'test_suite_cipher.padding': [ + # Following tests require AES_C/CAMELLIA_C to be enabled, + # but these are not available in the accelerated component. + re.compile('Set( non-existent)? padding with (AES|CAMELLIA).*'), + ], + 'test_suite_pkcs5': [ + # The AES part of PKCS#5 PBES2 is not yet supported. + # The rest of PKCS#5 (PBKDF2) works, though. + re.compile(r'PBES2 .* AES-.*') + ], + 'test_suite_pkparse': [ + # PEM (called by pkparse) requires AES_C in order to decrypt + # the key, but this is not available in the accelerated + # component. + re.compile('Parse RSA Key.*(password|AES-).*'), + ], + 'test_suite_pem': [ + # Following tests require AES_C, but this is diabled in the + # accelerated component. + re.compile('PEM read .*AES.*'), + 'PEM read (unknown encryption algorithm)', + ], + 'test_suite_error': [ + # Following tests depend on AES_C but are not about them + # really, just need to know some error code is there. + 'Single low error', + 'Low and high error', + ], + 'test_suite_version': [ + # Similar to test_suite_error above. + 'Check for MBEDTLS_AES_C when already present', + ], + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + } + +#pylint: enable=invalid-name,missing-class-docstring + + # List of tasks with a function that can handle this task and additional arguments if required KNOWN_TASKS = { 'analyze_coverage': CoverageTask, - - 'analyze_driver_vs_reference_hash': driver_vs_reference_factory('hash', { - 'component_ref': 'test_psa_crypto_config_reference_hash_use_psa', - 'component_driver': 'test_psa_crypto_config_accel_hash_use_psa', - 'ignored_suites': [ - 'shax', 'mdx', # the software implementations that are being excluded - 'md.psa', # purposefully depends on whether drivers are present - 'psa_crypto_low_hash.generated', # testing the builtins - ], - 'ignored_tests': { - 'test_suite_config': [ - re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'), - ], - 'test_suite_platform': [ - # Incompatible with sanitizers (e.g. ASan). If the driver - # component uses a sanitizer but the reference component - # doesn't, we have a PASS vs SKIP mismatch. - 'Check mbedtls_calloc overallocation', - ], - } - } - ), - 'analyze_driver_vs_reference_hmac': driver_vs_reference_factory('hmac', { - 'component_ref': 'test_psa_crypto_config_reference_hmac', - 'component_driver': 'test_psa_crypto_config_accel_hmac', - 'ignored_suites': [ - # These suites require legacy hash support, which is disabled - # in the accelerated component. - 'shax', 'mdx', - # This suite tests builtins directly, but these are missing - # in the accelerated case. - 'psa_crypto_low_hash.generated', - ], - 'ignored_tests': { - 'test_suite_config': [ - re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'), - re.compile(r'.*\bMBEDTLS_MD_C\b') - ], - 'test_suite_md': [ - # Builtin HMAC is not supported in the accelerate component. - re.compile('.*HMAC.*'), - # Following tests make use of functions which are not available - # when MD_C is disabled, as it happens in the accelerated - # test component. - re.compile('generic .* Hash file .*'), - 'MD list', - ], - 'test_suite_md.psa': [ - # "legacy only" tests require hash algorithms to be NOT - # accelerated, but this of course false for the accelerated - # test component. - re.compile('PSA dispatch .* legacy only'), - ], - 'test_suite_platform': [ - # Incompatible with sanitizers (e.g. ASan). If the driver - # component uses a sanitizer but the reference component - # doesn't, we have a PASS vs SKIP mismatch. - 'Check mbedtls_calloc overallocation', - ], - } - } - ), - 'analyze_driver_vs_reference_cipher_aead_cmac': driver_vs_reference_factory('cipher_aead_cmac', { - 'component_ref': 'test_psa_crypto_config_reference_cipher_aead_cmac', - 'component_driver': 'test_psa_crypto_config_accel_cipher_aead_cmac', - # Modules replaced by drivers. - 'ignored_suites': [ - # low-level (block/stream) cipher modules - 'aes', 'aria', 'camellia', 'des', 'chacha20', - # AEAD modes and CMAC - 'ccm', 'chachapoly', 'cmac', 'gcm', - # The Cipher abstraction layer - 'cipher', - ], - 'ignored_tests': { - 'test_suite_config': [ - re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA|CHACHA20|DES)_.*'), - re.compile(r'.*\bMBEDTLS_(CCM|CHACHAPOLY|CMAC|GCM)_.*'), - re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'), - re.compile(r'.*\bMBEDTLS_CIPHER_.*'), - ], - # PEM decryption is not supported so far. - # The rest of PEM (write, unencrypted read) works though. - 'test_suite_pem': [ - re.compile(r'PEM read .*(AES|DES|\bencrypt).*'), - ], - 'test_suite_platform': [ - # Incompatible with sanitizers (e.g. ASan). If the driver - # component uses a sanitizer but the reference component - # doesn't, we have a PASS vs SKIP mismatch. - 'Check mbedtls_calloc overallocation', - ], - # Following tests depend on AES_C/DES_C but are not about - # them really, just need to know some error code is there. - 'test_suite_error': [ - 'Low and high error', - 'Single low error' - ], - # Similar to test_suite_error above. - 'test_suite_version': [ - 'Check for MBEDTLS_AES_C when already present', - ], - # The en/decryption part of PKCS#12 is not supported so far. - # The rest of PKCS#12 (key derivation) works though. - 'test_suite_pkcs12': [ - re.compile(r'PBE Encrypt, .*'), - re.compile(r'PBE Decrypt, .*'), - ], - # The en/decryption part of PKCS#5 is not supported so far. - # The rest of PKCS#5 (PBKDF2) works though. - 'test_suite_pkcs5': [ - re.compile(r'PBES2 Encrypt, .*'), - re.compile(r'PBES2 Decrypt .*'), - ], - # Encrypted keys are not supported so far. - # pylint: disable=line-too-long - 'test_suite_pkparse': [ - 'Key ASN1 (Encrypted key PKCS12, trailing garbage data)', - 'Key ASN1 (Encrypted key PKCS5, trailing garbage data)', - re.compile(r'Parse (RSA|EC) Key .*\(.* ([Ee]ncrypted|password).*\)'), - ], - # Encrypted keys are not supported so far. - 'ssl-opt': [ - 'TLS: password protected server key', - 'TLS: password protected client key', - 'TLS: password protected server key, two certificates', - ], - } - } - ), - 'analyze_driver_vs_reference_ecp_light_only': driver_vs_reference_factory('ecp_light_only', { - 'component_ref': 'test_psa_crypto_config_reference_ecc_ecp_light_only', - 'component_driver': 'test_psa_crypto_config_accel_ecc_ecp_light_only', - 'ignored_suites': [ - # Modules replaced by drivers - 'ecdsa', 'ecdh', 'ecjpake', - ], - 'ignored_tests': { - 'test_suite_config': [ - re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), - ], - 'test_suite_platform': [ - # Incompatible with sanitizers (e.g. ASan). If the driver - # component uses a sanitizer but the reference component - # doesn't, we have a PASS vs SKIP mismatch. - 'Check mbedtls_calloc overallocation', - ], - # This test wants a legacy function that takes f_rng, p_rng - # arguments, and uses legacy ECDSA for that. The test is - # really about the wrapper around the PSA RNG, not ECDSA. - 'test_suite_random': [ - 'PSA classic wrapper: ECDSA signature (SECP256R1)', - ], - # In the accelerated test ECP_C is not set (only ECP_LIGHT is) - # so we must ignore disparities in the tests for which ECP_C - # is required. - 'test_suite_ecp': [ - re.compile(r'ECP check public-private .*'), - re.compile(r'ECP calculate public: .*'), - re.compile(r'ECP gen keypair .*'), - re.compile(r'ECP point muladd .*'), - re.compile(r'ECP point multiplication .*'), - re.compile(r'ECP test vectors .*'), - ], - 'test_suite_ssl': [ - # This deprecated function is only present when ECP_C is On. - 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', - ], - } - } - ), - 'analyze_driver_vs_reference_no_ecp_at_all': driver_vs_reference_factory('no_ecp_at_all', { - 'component_ref': 'test_psa_crypto_config_reference_ecc_no_ecp_at_all', - 'component_driver': 'test_psa_crypto_config_accel_ecc_no_ecp_at_all', - 'ignored_suites': [ - # Modules replaced by drivers - 'ecp', 'ecdsa', 'ecdh', 'ecjpake', - ], - 'ignored_tests': { - 'test_suite_config': [ - re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), - re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), - ], - 'test_suite_platform': [ - # Incompatible with sanitizers (e.g. ASan). If the driver - # component uses a sanitizer but the reference component - # doesn't, we have a PASS vs SKIP mismatch. - 'Check mbedtls_calloc overallocation', - ], - # See ecp_light_only - 'test_suite_random': [ - 'PSA classic wrapper: ECDSA signature (SECP256R1)', - ], - 'test_suite_pkparse': [ - # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED - # is automatically enabled in build_info.h (backward compatibility) - # even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a - # consequence compressed points are supported in the reference - # component but not in the accelerated one, so they should be skipped - # while checking driver's coverage. - re.compile(r'Parse EC Key .*compressed\)'), - re.compile(r'Parse Public EC Key .*compressed\)'), - ], - # See ecp_light_only - 'test_suite_ssl': [ - 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', - ], - } - } - ), - 'analyze_driver_vs_reference_ecc_no_bignum': driver_vs_reference_factory('ecc_no_bignum', { - 'component_ref': 'test_psa_crypto_config_reference_ecc_no_bignum', - 'component_driver': 'test_psa_crypto_config_accel_ecc_no_bignum', - 'ignored_suites': [ - # Modules replaced by drivers - 'ecp', 'ecdsa', 'ecdh', 'ecjpake', - 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', - 'bignum.generated', 'bignum.misc', - ], - 'ignored_tests': { - 'test_suite_config': [ - re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), - re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), - re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), - ], - 'test_suite_platform': [ - # Incompatible with sanitizers (e.g. ASan). If the driver - # component uses a sanitizer but the reference component - # doesn't, we have a PASS vs SKIP mismatch. - 'Check mbedtls_calloc overallocation', - ], - # See ecp_light_only - 'test_suite_random': [ - 'PSA classic wrapper: ECDSA signature (SECP256R1)', - ], - # See no_ecp_at_all - 'test_suite_pkparse': [ - re.compile(r'Parse EC Key .*compressed\)'), - re.compile(r'Parse Public EC Key .*compressed\)'), - ], - 'test_suite_asn1parse': [ - 'INTEGER too large for mpi', - ], - 'test_suite_asn1write': [ - re.compile(r'ASN.1 Write mpi.*'), - ], - 'test_suite_debug': [ - re.compile(r'Debug print mbedtls_mpi.*'), - ], - # See ecp_light_only - 'test_suite_ssl': [ - 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', - ], - } - } - ), - 'analyze_driver_vs_reference_ecc_ffdh_no_bignum': driver_vs_reference_factory('ecc_ffdh_no_bignum', { - 'component_ref': 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum', - 'component_driver': 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum', - 'ignored_suites': [ - # Modules replaced by drivers - 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm', - 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', - 'bignum.generated', 'bignum.misc', - ], - 'ignored_tests': { - 'ssl-opt': [ - # DHE support in TLS 1.2 requires built-in MBEDTLS_DHM_C - # (because it needs custom groups, which PSA does not - # provide), even with MBEDTLS_USE_PSA_CRYPTO. - re.compile(r'PSK callback:.*\bdhe-psk\b.*'), - ], - 'test_suite_config': [ - re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), - re.compile(r'.*\bMBEDTLS_DHM_C\b.*'), - re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), - re.compile(r'.*\bMBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED\b.*'), - re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), - ], - 'test_suite_platform': [ - # Incompatible with sanitizers (e.g. ASan). If the driver - # component uses a sanitizer but the reference component - # doesn't, we have a PASS vs SKIP mismatch. - 'Check mbedtls_calloc overallocation', - ], - # See ecp_light_only - 'test_suite_random': [ - 'PSA classic wrapper: ECDSA signature (SECP256R1)', - ], - # See no_ecp_at_all - 'test_suite_pkparse': [ - re.compile(r'Parse EC Key .*compressed\)'), - re.compile(r'Parse Public EC Key .*compressed\)'), - ], - 'test_suite_asn1parse': [ - 'INTEGER too large for mpi', - ], - 'test_suite_asn1write': [ - re.compile(r'ASN.1 Write mpi.*'), - ], - 'test_suite_debug': [ - re.compile(r'Debug print mbedtls_mpi.*'), - ], - # See ecp_light_only - 'test_suite_ssl': [ - 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', - ], - } - } - ), - 'analyze_driver_vs_reference_ffdh_alg': driver_vs_reference_factory('ffdh_alg', { - 'component_ref': 'test_psa_crypto_config_reference_ffdh', - 'component_driver': 'test_psa_crypto_config_accel_ffdh', - 'ignored_suites': ['dhm'], - 'ignored_tests': { - 'test_suite_config': [ - re.compile(r'.*\bMBEDTLS_DHM_C\b.*'), - ], - 'test_suite_platform': [ - # Incompatible with sanitizers (e.g. ASan). If the driver - # component uses a sanitizer but the reference component - # doesn't, we have a PASS vs SKIP mismatch. - 'Check mbedtls_calloc overallocation', - ], - } - } - ), - 'analyze_driver_vs_reference_tfm_config': driver_vs_reference_factory('tfm_config', { - 'component_ref': 'test_tfm_config_no_p256m', - 'component_driver': 'test_tfm_config_p256m_driver_accel_ec', - 'ignored_suites': [ - # Modules replaced by drivers - 'asn1parse', 'asn1write', - 'ecp', 'ecdsa', 'ecdh', 'ecjpake', - 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', - 'bignum.generated', 'bignum.misc', - ], - 'ignored_tests': { - 'test_suite_config': [ - re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), - re.compile(r'.*\bMBEDTLS_(ASN1\w+)_C\b.*'), - re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECP)_.*'), - re.compile(r'.*\bMBEDTLS_PSA_P256M_DRIVER_ENABLED\b.*') - ], - 'test_suite_config.crypto_combinations': [ - 'Config: ECC: Weierstrass curves only', - ], - 'test_suite_platform': [ - # Incompatible with sanitizers (e.g. ASan). If the driver - # component uses a sanitizer but the reference component - # doesn't, we have a PASS vs SKIP mismatch. - 'Check mbedtls_calloc overallocation', - ], - # See ecp_light_only - 'test_suite_random': [ - 'PSA classic wrapper: ECDSA signature (SECP256R1)', - ], - } - } - ), - 'analyze_driver_vs_reference_rsa': driver_vs_reference_factory('rsa', { - 'component_ref': 'test_psa_crypto_config_reference_rsa_crypto', - 'component_driver': 'test_psa_crypto_config_accel_rsa_crypto', - 'ignored_suites': [ - # Modules replaced by drivers. - 'rsa', 'pkcs1_v15', 'pkcs1_v21', - # We temporarily don't care about PK stuff. - 'pk', 'pkwrite', 'pkparse' - ], - 'ignored_tests': { - 'test_suite_config': [ - re.compile(r'.*\bMBEDTLS_(PKCS1|RSA)_.*'), - re.compile(r'.*\bMBEDTLS_GENPRIME\b.*') - ], - 'test_suite_platform': [ - # Incompatible with sanitizers (e.g. ASan). If the driver - # component uses a sanitizer but the reference component - # doesn't, we have a PASS vs SKIP mismatch. - 'Check mbedtls_calloc overallocation', - ], - # Following tests depend on RSA_C but are not about - # them really, just need to know some error code is there. - 'test_suite_error': [ - 'Low and high error', - 'Single high error' - ], - # Constant time operations only used for PKCS1_V15 - 'test_suite_constant_time': [ - re.compile(r'mbedtls_ct_zeroize_if .*'), - re.compile(r'mbedtls_ct_memmove_left .*') - ], - 'test_suite_psa_crypto': [ - # We don't support generate_key_custom entry points - # in drivers yet. - re.compile(r'PSA generate key custom: RSA, e=.*'), - re.compile(r'PSA generate key ext: RSA, e=.*'), - ], - } - } - ), - 'analyze_block_cipher_dispatch': driver_vs_reference_factory('block_cipher_dispatch', { - 'component_ref': 'test_full_block_cipher_legacy_dispatch', - 'component_driver': 'test_full_block_cipher_psa_dispatch', - 'ignored_suites': [ - # Skipped in the accelerated component - 'aes', 'aria', 'camellia', - # These require AES_C, ARIA_C or CAMELLIA_C to be enabled in - # order for the cipher module (actually cipher_wrapper) to work - # properly. However these symbols are disabled in the accelerated - # component so we ignore them. - 'cipher.ccm', 'cipher.gcm', 'cipher.aes', 'cipher.aria', - 'cipher.camellia', - ], - 'ignored_tests': { - 'test_suite_config': [ - re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA)_.*'), - re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'), - ], - 'test_suite_cmac': [ - # Following tests require AES_C/ARIA_C/CAMELLIA_C to be enabled, - # but these are not available in the accelerated component. - 'CMAC null arguments', - re.compile('CMAC.* (AES|ARIA|Camellia).*'), - ], - 'test_suite_cipher.padding': [ - # Following tests require AES_C/CAMELLIA_C to be enabled, - # but these are not available in the accelerated component. - re.compile('Set( non-existent)? padding with (AES|CAMELLIA).*'), - ], - 'test_suite_pkcs5': [ - # The AES part of PKCS#5 PBES2 is not yet supported. - # The rest of PKCS#5 (PBKDF2) works, though. - re.compile(r'PBES2 .* AES-.*') - ], - 'test_suite_pkparse': [ - # PEM (called by pkparse) requires AES_C in order to decrypt - # the key, but this is not available in the accelerated - # component. - re.compile('Parse RSA Key.*(password|AES-).*'), - ], - 'test_suite_pem': [ - # Following tests require AES_C, but this is diabled in the - # accelerated component. - re.compile('PEM read .*AES.*'), - 'PEM read (unknown encryption algorithm)', - ], - 'test_suite_error': [ - # Following tests depend on AES_C but are not about them - # really, just need to know some error code is there. - 'Single low error', - 'Low and high error', - ], - 'test_suite_version': [ - # Similar to test_suite_error above. - 'Check for MBEDTLS_AES_C when already present', - ], - 'test_suite_platform': [ - # Incompatible with sanitizers (e.g. ASan). If the driver - # component uses a sanitizer but the reference component - # doesn't, we have a PASS vs SKIP mismatch. - 'Check mbedtls_calloc overallocation', - ], - } - } - ), + 'analyze_driver_vs_reference_hash': DriverVSReference_hash, + 'analyze_driver_vs_reference_hmac': DriverVSReference_hmac, + 'analyze_driver_vs_reference_cipher_aead_cmac': DriverVSReference_cipher_aead_cmac, + 'analyze_driver_vs_reference_ecp_light_only': DriverVSReference_ecp_light_only, + 'analyze_driver_vs_reference_no_ecp_at_all': DriverVSReference_no_ecp_at_all, + 'analyze_driver_vs_reference_ecc_no_bignum': DriverVSReference_ecc_no_bignum, + 'analyze_driver_vs_reference_ecc_ffdh_no_bignum': DriverVSReference_ecc_ffdh_no_bignum, + 'analyze_driver_vs_reference_ffdh_alg': DriverVSReference_ffdh_alg, + 'analyze_driver_vs_reference_tfm_config': DriverVSReference_tfm_config, + 'analyze_driver_vs_reference_rsa': DriverVSReference_rsa, + 'analyze_block_cipher_dispatch': DriverVSReference_block_cipher_dispatch, } + def main(): main_results = Results() From 0f31f76f83103ac213b55b4748ebfc8c578ef60e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Sep 2024 20:15:58 +0200 Subject: [PATCH 06/12] Remove dead code that was handling stringly typed data Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index bfddf9830b..70fc0984a9 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -810,14 +810,9 @@ def main(): for task_name in tasks_list: task_constructor = KNOWN_TASKS[task_name] - if isinstance(task_constructor, dict): - test_function = task_constructor['test_function'] - test_args = task_constructor['args'] - test_function(main_results, outcomes, test_args) - else: - task = task_constructor(options) - main_results.new_section(task.section_name()) - task.run(main_results, outcomes) + task = task_constructor(options) + main_results.new_section(task.section_name()) + task.run(main_results, outcomes) main_results.info("Overall results: {} warnings and {} errors", main_results.warning_count, main_results.error_count) From 3f5022e66d31b167b2e5c1fd9ee0a0511ed21e04 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Sep 2024 20:23:40 +0200 Subject: [PATCH 07/12] Move analysis functions into their respective classes No intended behavior change. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 164 +++++++++++++++--------------- 1 file changed, 83 insertions(+), 81 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 70fc0984a9..32c400f032 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -82,38 +82,6 @@ def execute_reference_driver_tests(results: Results, ref_component: str, driver_ if ret_val != 0: results.error("failed to run reference/driver components") -def analyze_coverage(results: Results, outcomes: Outcomes, - allow_list: typing.List[str], full_coverage: bool) -> None: - """Check that all available test cases are executed at least once.""" - # Make sure that the generated data files are present (and up-to-date). - # This allows analyze_outcomes.py to run correctly on a fresh Git - # checkout. - cp = subprocess.run(['make', 'generated_files'], - cwd='tests', - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - check=False) - if cp.returncode != 0: - sys.stderr.write(cp.stdout.decode('utf-8')) - results.error("Failed \"make generated_files\" in tests. " - "Coverage analysis may be incorrect.") - available = check_test_cases.collect_available_test_cases() - for suite_case in available: - hit = any(suite_case in comp_outcomes.successes or - suite_case in comp_outcomes.failures - for comp_outcomes in outcomes.values()) - - if not hit and suite_case not in allow_list: - if full_coverage: - results.error('Test case not executed: {}', suite_case) - else: - results.warning('Test case not executed: {}', suite_case) - elif hit and suite_case in allow_list: - # Test Case should be removed from the allow list. - if full_coverage: - results.error('Allow listed test case was executed: {}', suite_case) - else: - results.warning('Allow listed test case was executed: {}', suite_case) - IgnoreEntry = typing.Union[str, typing.Pattern] def name_matches_pattern(name: str, str_or_re: IgnoreEntry) -> bool: @@ -128,50 +96,6 @@ def name_matches_pattern(name: str, str_or_re: IgnoreEntry) -> bool: else: return str_or_re == name -def analyze_driver_vs_reference(results: Results, outcomes: Outcomes, - component_ref: str, component_driver: str, - ignored_suites: typing.List[str], ignored_tests=None) -> None: - """Check that all tests passing in the driver component are also - passing in the corresponding reference component. - Skip: - - full test suites provided in ignored_suites list - - only some specific test inside a test suite, for which the corresponding - output string is provided - """ - ref_outcomes = outcomes.get("component_" + component_ref) - driver_outcomes = outcomes.get("component_" + component_driver) - - if ref_outcomes is None or driver_outcomes is None: - results.error("required components are missing: bad outcome file?") - return - - if not ref_outcomes.successes: - results.error("no passing test in reference component: bad outcome file?") - return - - for suite_case in ref_outcomes.successes: - # suite_case is like "test_suite_foo.bar;Description of test case" - (full_test_suite, test_string) = suite_case.split(';') - test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name - - # Immediately skip fully-ignored test suites - if test_suite in ignored_suites or full_test_suite in ignored_suites: - continue - - # For ignored test cases inside test suites, just remember and: - # don't issue an error if they're skipped with drivers, - # but issue an error if they're not (means we have a bad entry). - ignored = False - for str_or_re in (ignored_tests.get(full_test_suite, []) + - ignored_tests.get(test_suite, [])): - if name_matches_pattern(test_string, str_or_re): - ignored = True - - if not ignored and not suite_case in driver_outcomes.successes: - results.error("SKIP/FAIL -> PASS: {}", suite_case) - if ignored and suite_case in driver_outcomes.successes: - results.error("uselessly ignored: {}", suite_case) - def read_outcome_file(outcome_file: str) -> Outcomes: """Parse an outcome file and return an outcome collection. """ @@ -232,10 +156,43 @@ class CoverageTask(Task): def section_name() -> str: return "Analyze coverage" + @staticmethod + def analyze_coverage(results: Results, outcomes: Outcomes, + allow_list: typing.List[str], full_coverage: bool) -> None: + """Check that all available test cases are executed at least once.""" + # Make sure that the generated data files are present (and up-to-date). + # This allows analyze_outcomes.py to run correctly on a fresh Git + # checkout. + cp = subprocess.run(['make', 'generated_files'], + cwd='tests', + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + check=False) + if cp.returncode != 0: + sys.stderr.write(cp.stdout.decode('utf-8')) + results.error("Failed \"make generated_files\" in tests. " + "Coverage analysis may be incorrect.") + available = check_test_cases.collect_available_test_cases() + for suite_case in available: + hit = any(suite_case in comp_outcomes.successes or + suite_case in comp_outcomes.failures + for comp_outcomes in outcomes.values()) + + if not hit and suite_case not in allow_list: + if full_coverage: + results.error('Test case not executed: {}', suite_case) + else: + results.warning('Test case not executed: {}', suite_case) + elif hit and suite_case in allow_list: + # Test Case should be removed from the allow list. + if full_coverage: + results.error('Allow listed test case was executed: {}', suite_case) + else: + results.warning('Allow listed test case was executed: {}', suite_case) + def run(self, results: Results, outcomes: Outcomes): """Check that all test cases are executed at least once.""" - analyze_coverage(results, outcomes, - self.ALLOW_LIST, self.full_coverage) + self.analyze_coverage(results, outcomes, + self.ALLOW_LIST, self.full_coverage) class DriverVSReference(Task): @@ -264,12 +221,57 @@ class DriverVSReference(Task): def section_name(self) -> str: return f"Analyze driver {self.DRIVER} vs reference {self.REFERENCE}" + @staticmethod + def analyze_driver_vs_reference(results: Results, outcomes: Outcomes, + component_ref: str, component_driver: str, + ignored_suites: typing.List[str], ignored_tests=None) -> None: + """Check that all tests passing in the driver component are also + passing in the corresponding reference component. + Skip: + - full test suites provided in ignored_suites list + - only some specific test inside a test suite, for which the corresponding + output string is provided + """ + ref_outcomes = outcomes.get("component_" + component_ref) + driver_outcomes = outcomes.get("component_" + component_driver) + + if ref_outcomes is None or driver_outcomes is None: + results.error("required components are missing: bad outcome file?") + return + + if not ref_outcomes.successes: + results.error("no passing test in reference component: bad outcome file?") + return + + for suite_case in ref_outcomes.successes: + # suite_case is like "test_suite_foo.bar;Description of test case" + (full_test_suite, test_string) = suite_case.split(';') + test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name + + # Immediately skip fully-ignored test suites + if test_suite in ignored_suites or full_test_suite in ignored_suites: + continue + + # For ignored test cases inside test suites, just remember and: + # don't issue an error if they're skipped with drivers, + # but issue an error if they're not (means we have a bad entry). + ignored = False + for str_or_re in (ignored_tests.get(full_test_suite, []) + + ignored_tests.get(test_suite, [])): + if name_matches_pattern(test_string, str_or_re): + ignored = True + + if not ignored and not suite_case in driver_outcomes.successes: + results.error("SKIP/FAIL -> PASS: {}", suite_case) + if ignored and suite_case in driver_outcomes.successes: + results.error("uselessly ignored: {}", suite_case) + def run(self, results: Results, outcomes: Outcomes) -> None: """Compare driver test outcomes with reference outcomes.""" ignored_suites = ['test_suite_' + x for x in self.IGNORED_SUITES] - analyze_driver_vs_reference(results, outcomes, - self.REFERENCE, self.DRIVER, - ignored_suites, self.IGNORED_TESTS) + self.analyze_driver_vs_reference(results, outcomes, + self.REFERENCE, self.DRIVER, + ignored_suites, self.IGNORED_TESTS) # The names that we give to classes derived from DriverVSReference do not From b4daeb4fd26c62f7d451d6bee06e2a96d1fc81fc Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Sep 2024 20:32:59 +0200 Subject: [PATCH 08/12] Remove now-useless level of method call indirection No intended behavior change. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 45 ++++++++++++------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 32c400f032..fa5079e0b4 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -156,9 +156,7 @@ class CoverageTask(Task): def section_name() -> str: return "Analyze coverage" - @staticmethod - def analyze_coverage(results: Results, outcomes: Outcomes, - allow_list: typing.List[str], full_coverage: bool) -> None: + def run(self, results: Results, outcomes: Outcomes) -> None: """Check that all available test cases are executed at least once.""" # Make sure that the generated data files are present (and up-to-date). # This allows analyze_outcomes.py to run correctly on a fresh Git @@ -177,23 +175,18 @@ class CoverageTask(Task): suite_case in comp_outcomes.failures for comp_outcomes in outcomes.values()) - if not hit and suite_case not in allow_list: - if full_coverage: + if not hit and suite_case not in self.ALLOW_LIST: + if self.full_coverage: results.error('Test case not executed: {}', suite_case) else: results.warning('Test case not executed: {}', suite_case) - elif hit and suite_case in allow_list: + elif hit and suite_case in self.ALLOW_LIST: # Test Case should be removed from the allow list. - if full_coverage: + if self.full_coverage: results.error('Allow listed test case was executed: {}', suite_case) else: results.warning('Allow listed test case was executed: {}', suite_case) - def run(self, results: Results, outcomes: Outcomes): - """Check that all test cases are executed at least once.""" - self.analyze_coverage(results, outcomes, - self.ALLOW_LIST, self.full_coverage) - class DriverVSReference(Task): """Compare outcomes from testing with and without a driver. @@ -218,13 +211,15 @@ class DriverVSReference(Task): # see the `name_matches_pattern` function. IGNORED_TESTS = {} #type: typing.Dict[str, typing.List[IgnoreEntry]] + def __init__(self, options) -> None: + super().__init__(options) + self.ignored_suites = frozenset('test_suite_' + x + for x in self.IGNORED_SUITES) + def section_name(self) -> str: return f"Analyze driver {self.DRIVER} vs reference {self.REFERENCE}" - @staticmethod - def analyze_driver_vs_reference(results: Results, outcomes: Outcomes, - component_ref: str, component_driver: str, - ignored_suites: typing.List[str], ignored_tests=None) -> None: + def run(self, results: Results, outcomes: Outcomes) -> None: """Check that all tests passing in the driver component are also passing in the corresponding reference component. Skip: @@ -232,8 +227,8 @@ class DriverVSReference(Task): - only some specific test inside a test suite, for which the corresponding output string is provided """ - ref_outcomes = outcomes.get("component_" + component_ref) - driver_outcomes = outcomes.get("component_" + component_driver) + ref_outcomes = outcomes.get("component_" + self.REFERENCE) + driver_outcomes = outcomes.get("component_" + self.DRIVER) if ref_outcomes is None or driver_outcomes is None: results.error("required components are missing: bad outcome file?") @@ -249,15 +244,16 @@ class DriverVSReference(Task): test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name # Immediately skip fully-ignored test suites - if test_suite in ignored_suites or full_test_suite in ignored_suites: + if test_suite in self.ignored_suites or \ + full_test_suite in self.ignored_suites: continue # For ignored test cases inside test suites, just remember and: # don't issue an error if they're skipped with drivers, # but issue an error if they're not (means we have a bad entry). ignored = False - for str_or_re in (ignored_tests.get(full_test_suite, []) + - ignored_tests.get(test_suite, [])): + for str_or_re in (self.IGNORED_TESTS.get(full_test_suite, []) + + self.IGNORED_TESTS.get(test_suite, [])): if name_matches_pattern(test_string, str_or_re): ignored = True @@ -266,13 +262,6 @@ class DriverVSReference(Task): if ignored and suite_case in driver_outcomes.successes: results.error("uselessly ignored: {}", suite_case) - def run(self, results: Results, outcomes: Outcomes) -> None: - """Compare driver test outcomes with reference outcomes.""" - ignored_suites = ['test_suite_' + x for x in self.IGNORED_SUITES] - self.analyze_driver_vs_reference(results, outcomes, - self.REFERENCE, self.DRIVER, - ignored_suites, self.IGNORED_TESTS) - # The names that we give to classes derived from DriverVSReference do not # follow the usual naming convention, because it's more readable to use From 02976056410e1cf20758b6fa274c808f8e6ce4fa Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Sep 2024 20:44:15 +0200 Subject: [PATCH 09/12] Move test case ignore list to the master Task class No intended behavior change. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index fa5079e0b4..cbe4ed4256 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -120,6 +120,12 @@ def read_outcome_file(outcome_file: str) -> Outcomes: class Task: """Base class for outcome analysis tasks.""" + # Override the following in child classes. + # Map test suite names (with the test_suite_prefix) to a list of ignored + # test cases. Each element in the list can be either a string or a regex; + # see the `name_matches_pattern` function. + IGNORED_TESTS = {} #type: typing.Dict[str, typing.List[IgnoreEntry]] + def __init__(self, options) -> None: """Pass command line options to the tasks. @@ -130,6 +136,15 @@ class Task: def section_name(self) -> str: """The section name to use in results.""" + def is_test_case_ignored(self, full_test_suite: str, test_string: str) -> bool: + """Check if the specified test case is ignored.""" + test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name + for str_or_re in (self.IGNORED_TESTS.get(full_test_suite, []) + + self.IGNORED_TESTS.get(test_suite, [])): + if name_matches_pattern(test_string, str_or_re): + return True + return False + def run(self, results: Results, outcomes: Outcomes): """Run the analysis on the specified outcomes. @@ -206,10 +221,6 @@ class DriverVSReference(Task): DRIVER = '' # Ignored test suites (without the test_suite_ prefix). IGNORED_SUITES = [] #type: typing.List[str] - # Map test suite names (with the test_suite_prefix) to a list of ignored - # test cases. Each element in the list can be either a string or a regex; - # see the `name_matches_pattern` function. - IGNORED_TESTS = {} #type: typing.Dict[str, typing.List[IgnoreEntry]] def __init__(self, options) -> None: super().__init__(options) @@ -251,11 +262,7 @@ class DriverVSReference(Task): # For ignored test cases inside test suites, just remember and: # don't issue an error if they're skipped with drivers, # but issue an error if they're not (means we have a bad entry). - ignored = False - for str_or_re in (self.IGNORED_TESTS.get(full_test_suite, []) + - self.IGNORED_TESTS.get(test_suite, [])): - if name_matches_pattern(test_string, str_or_re): - ignored = True + ignored = self.is_test_case_ignored(full_test_suite, test_string) if not ignored and not suite_case in driver_outcomes.successes: results.error("SKIP/FAIL -> PASS: {}", suite_case) From dba8010384e99828ae1cab5757f7e22c989ca550 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Sep 2024 20:52:58 +0200 Subject: [PATCH 10/12] Simplify sub-test-suite handling in is_test_case_ignored No intended behavior change. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index cbe4ed4256..32f4ae3bb7 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -136,11 +136,19 @@ class Task: def section_name(self) -> str: """The section name to use in results.""" - def is_test_case_ignored(self, full_test_suite: str, test_string: str) -> bool: + def ignored_tests(self, test_suite: str) -> typing.Iterator[IgnoreEntry]: + """Generate the ignore list for the specified test suite.""" + if test_suite in self.IGNORED_TESTS: + yield from self.IGNORED_TESTS[test_suite] + pos = test_suite.find('.') + if pos != -1: + base_test_suite = test_suite[:pos] + if base_test_suite in self.IGNORED_TESTS: + yield from self.IGNORED_TESTS[base_test_suite] + + def is_test_case_ignored(self, test_suite: str, test_string: str) -> bool: """Check if the specified test case is ignored.""" - test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name - for str_or_re in (self.IGNORED_TESTS.get(full_test_suite, []) + - self.IGNORED_TESTS.get(test_suite, [])): + for str_or_re in self.ignored_tests(test_suite): if name_matches_pattern(test_string, str_or_re): return True return False From 54cfe779512ce0527364eeada8df572131040a71 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 16 Sep 2024 20:56:43 +0200 Subject: [PATCH 11/12] Switch coverage analysis to IGNORE_TESTS for its allowlist No intended behavior change. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 32f4ae3bb7..ef0db33275 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -164,12 +164,14 @@ class Task: class CoverageTask(Task): """Analyze test coverage.""" - ALLOW_LIST = [ - # Algorithm not supported yet - 'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA', - # Algorithm not supported yet - 'test_suite_psa_crypto_metadata;Cipher: XTS', - ] + IGNORED_TESTS = { + 'test_suite_psa_crypto_metadata': [ + # Algorithm not supported yet + 'Asymmetric signature: pure EdDSA', + # Algorithm not supported yet + 'Cipher: XTS', + ], + } def __init__(self, options) -> None: super().__init__(options) @@ -197,13 +199,15 @@ class CoverageTask(Task): hit = any(suite_case in comp_outcomes.successes or suite_case in comp_outcomes.failures for comp_outcomes in outcomes.values()) + (test_suite, test_description) = suite_case.split(';') + ignored = self.is_test_case_ignored(test_suite, test_description) - if not hit and suite_case not in self.ALLOW_LIST: + if not hit and not ignored: if self.full_coverage: results.error('Test case not executed: {}', suite_case) else: results.warning('Test case not executed: {}', suite_case) - elif hit and suite_case in self.ALLOW_LIST: + elif hit and ignored: # Test Case should be removed from the allow list. if self.full_coverage: results.error('Allow listed test case was executed: {}', suite_case) From 0930b331c00dd733a74f81df53c47f89df98202f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 26 Sep 2024 19:54:38 +0200 Subject: [PATCH 12/12] Don't use the "allow list" terminology any longer What was formerly called an allow list is now an ignore table. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index ef0db33275..188b68d1d5 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -164,6 +164,9 @@ class Task: class CoverageTask(Task): """Analyze test coverage.""" + # Test cases whose suite and description are matched by an entry in + # IGNORED_TESTS are expected to be never executed. + # All other test cases are expected to be executed at least once. IGNORED_TESTS = { 'test_suite_psa_crypto_metadata': [ # Algorithm not supported yet @@ -208,11 +211,14 @@ class CoverageTask(Task): else: results.warning('Test case not executed: {}', suite_case) elif hit and ignored: - # Test Case should be removed from the allow list. + # If a test case is no longer always skipped, we should remove + # it from the ignore list. if self.full_coverage: - results.error('Allow listed test case was executed: {}', suite_case) + results.error('Test case was executed but marked as ignored for coverage: {}', + suite_case) else: - results.warning('Allow listed test case was executed: {}', suite_case) + results.warning('Test case was executed but marked as ignored for coverage: {}', + suite_case) class DriverVSReference(Task):