mirror/mbedtls
1
0
Fork 0
mirror of https://github.com/Mbed-TLS/mbedtls.git synced 2025-03-22 04:20:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #9594 from gilles-peskine-arm/analyze_outcomes-classes-development

Browse Source 
analyze_outcomes.py refactoring: change stringly typed data to classes
This commit is contained in:
Gilles Peskine 2024-10-03 10:38:04 +00:00 committed by GitHub
commit 38d4c91b06
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 657 additions and 609 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.mypy.ini
View File

@ -1,4 +1,4 @@
[mypy]
mypy_path = scripts
mypy_path = framework/scripts:scripts
namespace_packages = True
warn_unused_configs = True

2
.pylintrc
View File

@ -1,5 +1,5 @@
[MASTER]
init-hook='import sys; sys.path.append("scripts")'
init-hook='import sys; sys.path.append("scripts"); sys.path.append("framework/scripts")'
min-similarity-lines=10
[BASIC]

504
tests/scripts/analyze_outcomes.py
View File

@ -82,39 +82,9 @@ def execute_reference_driver_tests(results: Results, ref_component: str, driver_
if ret_val != 0:
results.error("failed to run reference/driver components")
def analyze_coverage(results: Results, outcomes: Outcomes,
allow_list: typing.List[str], full_coverage: bool) -> None:
"""Check that all available test cases are executed at least once."""
# Make sure that the generated data files are present (and up-to-date).
# This allows analyze_outcomes.py to run correctly on a fresh Git
# checkout.
cp = subprocess.run(['make', 'generated_files'],
cwd='tests',
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
check=False)
if cp.returncode != 0:
sys.stderr.write(cp.stdout.decode('utf-8'))
results.error("Failed \"make generated_files\" in tests. "
"Coverage analysis may be incorrect.")
available = check_test_cases.collect_available_test_cases()
for suite_case in available:
hit = any(suite_case in comp_outcomes.successes or
suite_case in comp_outcomes.failures
for comp_outcomes in outcomes.values())
IgnoreEntry = typing.Union[str, typing.Pattern]
if not hit and suite_case not in allow_list:
if full_coverage:
results.error('Test case not executed: {}', suite_case)
else:
results.warning('Test case not executed: {}', suite_case)
elif hit and suite_case in allow_list:
# Test Case should be removed from the allow list.
if full_coverage:
results.error('Allow listed test case was executed: {}', suite_case)
else:
results.warning('Allow listed test case was executed: {}', suite_case)
def name_matches_pattern(name: str, str_or_re) -> bool:
def name_matches_pattern(name: str, str_or_re: IgnoreEntry) -> bool:
"""Check if name matches a pattern, that may be a string or regex.
- If the pattern is a string, name must be equal to match.
- If the pattern is a regex, name must fully match.
@ -126,55 +96,6 @@ def name_matches_pattern(name: str, str_or_re) -> bool:
else:
return str_or_re == name
def analyze_driver_vs_reference(results: Results, outcomes: Outcomes,
component_ref: str, component_driver: str,
ignored_suites: typing.List[str], ignored_tests=None) -> None:
"""Check that all tests passing in the driver component are also
passing in the corresponding reference component.
Skip:
- full test suites provided in ignored_suites list
- only some specific test inside a test suite, for which the corresponding
output string is provided
"""
ref_outcomes = outcomes.get("component_" + component_ref)
driver_outcomes = outcomes.get("component_" + component_driver)
if ref_outcomes is None or driver_outcomes is None:
results.error("required components are missing: bad outcome file?")
return
if not ref_outcomes.successes:
results.error("no passing test in reference component: bad outcome file?")
return
for suite_case in ref_outcomes.successes:
# suite_case is like "test_suite_foo.bar;Description of test case"
(full_test_suite, test_string) = suite_case.split(';')
test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name
# Immediately skip fully-ignored test suites
if test_suite in ignored_suites or full_test_suite in ignored_suites:
continue
# For ignored test cases inside test suites, just remember and:
# don't issue an error if they're skipped with drivers,
# but issue an error if they're not (means we have a bad entry).
ignored = False
for str_or_re in (ignored_tests.get(full_test_suite, []) +
ignored_tests.get(test_suite, [])):
if name_matches_pattern(test_string, str_or_re):
ignored = True
if not ignored and not suite_case in driver_outcomes.successes:
results.error("SKIP/FAIL -> PASS: {}", suite_case)
if ignored and suite_case in driver_outcomes.successes:
results.error("uselessly ignored: {}", suite_case)
def analyze_outcomes(results: Results, outcomes: Outcomes, args) -> None:
"""Run all analyses on the given outcome collection."""
analyze_coverage(results, outcomes, args['allow_list'],
args['full_coverage'])
def read_outcome_file(outcome_file: str) -> Outcomes:
"""Parse an outcome file and return an outcome collection.
"""
@ -195,53 +116,194 @@ def read_outcome_file(outcome_file: str) -> Outcomes:
return outcomes
def do_analyze_coverage(results: Results, outcomes: Outcomes, args) -> None:
"""Perform coverage analysis."""
results.new_section("Analyze coverage")
analyze_outcomes(results, outcomes, args)
def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -> None:
"""Perform driver vs reference analyze."""
results.new_section("Analyze driver {} vs reference {}",
args['component_driver'], args['component_ref'])
class Task:
"""Base class for outcome analysis tasks."""
ignored_suites = ['test_suite_' + x for x in args['ignored_suites']]
# Override the following in child classes.
# Map test suite names (with the test_suite_prefix) to a list of ignored
# test cases. Each element in the list can be either a string or a regex;
# see the `name_matches_pattern` function.
IGNORED_TESTS = {} #type: typing.Dict[str, typing.List[IgnoreEntry]]
analyze_driver_vs_reference(results, outcomes,
args['component_ref'], args['component_driver'],
ignored_suites, args['ignored_tests'])
def __init__(self, options) -> None:
"""Pass command line options to the tasks.
# List of tasks with a function that can handle this task and additional arguments if required
KNOWN_TASKS = {
'analyze_coverage': {
'test_function': do_analyze_coverage,
'args': {
'allow_list': [
Each task decides which command line options it cares about.
"""
pass
def section_name(self) -> str:
"""The section name to use in results."""
def ignored_tests(self, test_suite: str) -> typing.Iterator[IgnoreEntry]:
"""Generate the ignore list for the specified test suite."""
if test_suite in self.IGNORED_TESTS:
yield from self.IGNORED_TESTS[test_suite]
pos = test_suite.find('.')
if pos != -1:
base_test_suite = test_suite[:pos]
if base_test_suite in self.IGNORED_TESTS:
yield from self.IGNORED_TESTS[base_test_suite]
def is_test_case_ignored(self, test_suite: str, test_string: str) -> bool:
"""Check if the specified test case is ignored."""
for str_or_re in self.ignored_tests(test_suite):
if name_matches_pattern(test_string, str_or_re):
return True
return False
def run(self, results: Results, outcomes: Outcomes):
"""Run the analysis on the specified outcomes.
Signal errors via the results objects
"""
raise NotImplementedError
class CoverageTask(Task):
"""Analyze test coverage."""
# Test cases whose suite and description are matched by an entry in
# IGNORED_TESTS are expected to be never executed.
# All other test cases are expected to be executed at least once.
IGNORED_TESTS = {
'test_suite_psa_crypto_metadata': [
# Algorithm not supported yet
'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA',
'Asymmetric signature: pure EdDSA',
# Algorithm not supported yet
'test_suite_psa_crypto_metadata;Cipher: XTS',
'Cipher: XTS',
],
'full_coverage': False,
}
},
# There are 2 options to use analyze_driver_vs_reference_xxx locally:
# 1. Run tests and then analysis:
# - tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver>
# - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
# 2. Let this script run both automatically:
# - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
'analyze_driver_vs_reference_hash': {
'test_function': do_analyze_driver_vs_reference,
'args': {
'component_ref': 'test_psa_crypto_config_reference_hash_use_psa',
'component_driver': 'test_psa_crypto_config_accel_hash_use_psa',
'ignored_suites': [
def __init__(self, options) -> None:
super().__init__(options)
self.full_coverage = options.full_coverage #type: bool
@staticmethod
def section_name() -> str:
return "Analyze coverage"
def run(self, results: Results, outcomes: Outcomes) -> None:
"""Check that all available test cases are executed at least once."""
# Make sure that the generated data files are present (and up-to-date).
# This allows analyze_outcomes.py to run correctly on a fresh Git
# checkout.
cp = subprocess.run(['make', 'generated_files'],
cwd='tests',
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
check=False)
if cp.returncode != 0:
sys.stderr.write(cp.stdout.decode('utf-8'))
results.error("Failed \"make generated_files\" in tests. "
"Coverage analysis may be incorrect.")
available = check_test_cases.collect_available_test_cases()
for suite_case in available:
hit = any(suite_case in comp_outcomes.successes or
suite_case in comp_outcomes.failures
for comp_outcomes in outcomes.values())
(test_suite, test_description) = suite_case.split(';')
ignored = self.is_test_case_ignored(test_suite, test_description)
if not hit and not ignored:
if self.full_coverage:
results.error('Test case not executed: {}', suite_case)
else:
results.warning('Test case not executed: {}', suite_case)
elif hit and ignored:
# If a test case is no longer always skipped, we should remove
# it from the ignore list.
if self.full_coverage:
results.error('Test case was executed but marked as ignored for coverage: {}',
suite_case)
else:
results.warning('Test case was executed but marked as ignored for coverage: {}',
suite_case)
class DriverVSReference(Task):
"""Compare outcomes from testing with and without a driver.
There are 2 options to use analyze_driver_vs_reference_xxx locally:
1. Run tests and then analysis:
- tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver>
- tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
2. Let this script run both automatically:
- tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx
"""
# Override the following in child classes.
# Configuration name (all.sh component) used as the reference.
REFERENCE = ''
# Configuration name (all.sh component) used as the driver.
DRIVER = ''
# Ignored test suites (without the test_suite_ prefix).
IGNORED_SUITES = [] #type: typing.List[str]
def __init__(self, options) -> None:
super().__init__(options)
self.ignored_suites = frozenset('test_suite_' + x
for x in self.IGNORED_SUITES)
def section_name(self) -> str:
return f"Analyze driver {self.DRIVER} vs reference {self.REFERENCE}"
def run(self, results: Results, outcomes: Outcomes) -> None:
"""Check that all tests passing in the driver component are also
passing in the corresponding reference component.
Skip:
- full test suites provided in ignored_suites list
- only some specific test inside a test suite, for which the corresponding
output string is provided
"""
ref_outcomes = outcomes.get("component_" + self.REFERENCE)
driver_outcomes = outcomes.get("component_" + self.DRIVER)
if ref_outcomes is None or driver_outcomes is None:
results.error("required components are missing: bad outcome file?")
return
if not ref_outcomes.successes:
results.error("no passing test in reference component: bad outcome file?")
return
for suite_case in ref_outcomes.successes:
# suite_case is like "test_suite_foo.bar;Description of test case"
(full_test_suite, test_string) = suite_case.split(';')
test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name
# Immediately skip fully-ignored test suites
if test_suite in self.ignored_suites or \
full_test_suite in self.ignored_suites:
continue
# For ignored test cases inside test suites, just remember and:
# don't issue an error if they're skipped with drivers,
# but issue an error if they're not (means we have a bad entry).
ignored = self.is_test_case_ignored(full_test_suite, test_string)
if not ignored and not suite_case in driver_outcomes.successes:
results.error("SKIP/FAIL -> PASS: {}", suite_case)
if ignored and suite_case in driver_outcomes.successes:
results.error("uselessly ignored: {}", suite_case)
# The names that we give to classes derived from DriverVSReference do not
# follow the usual naming convention, because it's more readable to use
# underscores and parts of the configuration names. Also, these classes
# are just there to specify some data, so they don't need repetitive
# documentation.
#pylint: disable=invalid-name,missing-class-docstring
class DriverVSReference_hash(DriverVSReference):
REFERENCE = 'test_psa_crypto_config_reference_hash_use_psa'
DRIVER = 'test_psa_crypto_config_accel_hash_use_psa'
IGNORED_SUITES = [
'shax', 'mdx', # the software implementations that are being excluded
'md.psa', # purposefully depends on whether drivers are present
'psa_crypto_low_hash.generated', # testing the builtins
],
'ignored_tests': {
]
IGNORED_TESTS = {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'),
],
@ -252,22 +314,19 @@ KNOWN_TASKS = {
'Check mbedtls_calloc overallocation',
],
}
}
},
'analyze_driver_vs_reference_hmac': {
'test_function': do_analyze_driver_vs_reference,
'args': {
'component_ref': 'test_psa_crypto_config_reference_hmac',
'component_driver': 'test_psa_crypto_config_accel_hmac',
'ignored_suites': [
class DriverVSReference_hmac(DriverVSReference):
REFERENCE = 'test_psa_crypto_config_reference_hmac'
DRIVER = 'test_psa_crypto_config_accel_hmac'
IGNORED_SUITES = [
# These suites require legacy hash support, which is disabled
# in the accelerated component.
'shax', 'mdx',
# This suite tests builtins directly, but these are missing
# in the accelerated case.
'psa_crypto_low_hash.generated',
],
'ignored_tests': {
]
IGNORED_TESTS = {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'),
re.compile(r'.*\bMBEDTLS_MD_C\b')
@ -294,23 +353,20 @@ KNOWN_TASKS = {
'Check mbedtls_calloc overallocation',
],
}
}
},
'analyze_driver_vs_reference_cipher_aead_cmac': {
'test_function': do_analyze_driver_vs_reference,
'args': {
'component_ref': 'test_psa_crypto_config_reference_cipher_aead_cmac',
'component_driver': 'test_psa_crypto_config_accel_cipher_aead_cmac',
class DriverVSReference_cipher_aead_cmac(DriverVSReference):
REFERENCE = 'test_psa_crypto_config_reference_cipher_aead_cmac'
DRIVER = 'test_psa_crypto_config_accel_cipher_aead_cmac'
# Modules replaced by drivers.
'ignored_suites': [
IGNORED_SUITES = [
# low-level (block/stream) cipher modules
'aes', 'aria', 'camellia', 'des', 'chacha20',
# AEAD modes and CMAC
'ccm', 'chachapoly', 'cmac', 'gcm',
# The Cipher abstraction layer
'cipher',
],
'ignored_tests': {
]
IGNORED_TESTS = {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA|CHACHA20|DES)_.*'),
re.compile(r'.*\bMBEDTLS_(CCM|CHACHAPOLY|CMAC|GCM)_.*'),
@ -364,18 +420,15 @@ KNOWN_TASKS = {
'TLS: password protected server key, two certificates',
],
}
}
},
'analyze_driver_vs_reference_ecp_light_only': {
'test_function': do_analyze_driver_vs_reference,
'args': {
'component_ref': 'test_psa_crypto_config_reference_ecc_ecp_light_only',
'component_driver': 'test_psa_crypto_config_accel_ecc_ecp_light_only',
'ignored_suites': [
class DriverVSReference_ecp_light_only(DriverVSReference):
REFERENCE = 'test_psa_crypto_config_reference_ecc_ecp_light_only'
DRIVER = 'test_psa_crypto_config_accel_ecc_ecp_light_only'
IGNORED_SUITES = [
# Modules replaced by drivers
'ecdsa', 'ecdh', 'ecjpake',
],
'ignored_tests': {
]
IGNORED_TESTS = {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
],
@ -407,18 +460,15 @@ KNOWN_TASKS = {
'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
],
}
}
},
'analyze_driver_vs_reference_no_ecp_at_all': {
'test_function': do_analyze_driver_vs_reference,
'args': {
'component_ref': 'test_psa_crypto_config_reference_ecc_no_ecp_at_all',
'component_driver': 'test_psa_crypto_config_accel_ecc_no_ecp_at_all',
'ignored_suites': [
class DriverVSReference_no_ecp_at_all(DriverVSReference):
REFERENCE = 'test_psa_crypto_config_reference_ecc_no_ecp_at_all'
DRIVER = 'test_psa_crypto_config_accel_ecc_no_ecp_at_all'
IGNORED_SUITES = [
# Modules replaced by drivers
'ecp', 'ecdsa', 'ecdh', 'ecjpake',
],
'ignored_tests': {
]
IGNORED_TESTS = {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
@ -448,20 +498,17 @@ KNOWN_TASKS = {
'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
],
}
}
},
'analyze_driver_vs_reference_ecc_no_bignum': {
'test_function': do_analyze_driver_vs_reference,
'args': {
'component_ref': 'test_psa_crypto_config_reference_ecc_no_bignum',
'component_driver': 'test_psa_crypto_config_accel_ecc_no_bignum',
'ignored_suites': [
class DriverVSReference_ecc_no_bignum(DriverVSReference):
REFERENCE = 'test_psa_crypto_config_reference_ecc_no_bignum'
DRIVER = 'test_psa_crypto_config_accel_ecc_no_bignum'
IGNORED_SUITES = [
# Modules replaced by drivers
'ecp', 'ecdsa', 'ecdh', 'ecjpake',
'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
'bignum.generated', 'bignum.misc',
],
'ignored_tests': {
]
IGNORED_TESTS = {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
@ -496,20 +543,17 @@ KNOWN_TASKS = {
'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
],
}
}
},
'analyze_driver_vs_reference_ecc_ffdh_no_bignum': {
'test_function': do_analyze_driver_vs_reference,
'args': {
'component_ref': 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum',
'component_driver': 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum',
'ignored_suites': [
class DriverVSReference_ecc_ffdh_no_bignum(DriverVSReference):
REFERENCE = 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum'
DRIVER = 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum'
IGNORED_SUITES = [
# Modules replaced by drivers
'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm',
'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
'bignum.generated', 'bignum.misc',
],
'ignored_tests': {
]
IGNORED_TESTS = {
'ssl-opt': [
# DHE support in TLS 1.2 requires built-in MBEDTLS_DHM_C
# (because it needs custom groups, which PSA does not
@ -552,15 +596,12 @@ KNOWN_TASKS = {
'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
],
}
}
},
'analyze_driver_vs_reference_ffdh_alg': {
'test_function': do_analyze_driver_vs_reference,
'args': {
'component_ref': 'test_psa_crypto_config_reference_ffdh',
'component_driver': 'test_psa_crypto_config_accel_ffdh',
'ignored_suites': ['dhm'],
'ignored_tests': {
class DriverVSReference_ffdh_alg(DriverVSReference):
REFERENCE = 'test_psa_crypto_config_reference_ffdh'
DRIVER = 'test_psa_crypto_config_accel_ffdh'
IGNORED_SUITES = ['dhm']
IGNORED_TESTS = {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_DHM_C\b.*'),
],
@ -571,21 +612,18 @@ KNOWN_TASKS = {
'Check mbedtls_calloc overallocation',
],
}
}
},
'analyze_driver_vs_reference_tfm_config': {
'test_function': do_analyze_driver_vs_reference,
'args': {
'component_ref': 'test_tfm_config_no_p256m',
'component_driver': 'test_tfm_config_p256m_driver_accel_ec',
'ignored_suites': [
class DriverVSReference_tfm_config(DriverVSReference):
REFERENCE = 'test_tfm_config_no_p256m'
DRIVER = 'test_tfm_config_p256m_driver_accel_ec'
IGNORED_SUITES = [
# Modules replaced by drivers
'asn1parse', 'asn1write',
'ecp', 'ecdsa', 'ecdh', 'ecjpake',
'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
'bignum.generated', 'bignum.misc',
],
'ignored_tests': {
]
IGNORED_TESTS = {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
re.compile(r'.*\bMBEDTLS_(ASN1\w+)_C\b.*'),
@ -606,20 +644,17 @@ KNOWN_TASKS = {
'PSA classic wrapper: ECDSA signature (SECP256R1)',
],
}
}
},
'analyze_driver_vs_reference_rsa': {
'test_function': do_analyze_driver_vs_reference,
'args': {
'component_ref': 'test_psa_crypto_config_reference_rsa_crypto',
'component_driver': 'test_psa_crypto_config_accel_rsa_crypto',
'ignored_suites': [
class DriverVSReference_rsa(DriverVSReference):
REFERENCE = 'test_psa_crypto_config_reference_rsa_crypto'
DRIVER = 'test_psa_crypto_config_accel_rsa_crypto'
IGNORED_SUITES = [
# Modules replaced by drivers.
'rsa', 'pkcs1_v15', 'pkcs1_v21',
# We temporarily don't care about PK stuff.
'pk', 'pkwrite', 'pkparse'
],
'ignored_tests': {
]
IGNORED_TESTS = {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_(PKCS1|RSA)_.*'),
re.compile(r'.*\bMBEDTLS_GENPRIME\b.*')
@ -648,14 +683,11 @@ KNOWN_TASKS = {
re.compile(r'PSA generate key ext: RSA, e=.*'),
],
}
}
},
'analyze_block_cipher_dispatch': {
'test_function': do_analyze_driver_vs_reference,
'args': {
'component_ref': 'test_full_block_cipher_legacy_dispatch',
'component_driver': 'test_full_block_cipher_psa_dispatch',
'ignored_suites': [
class DriverVSReference_block_cipher_dispatch(DriverVSReference):
REFERENCE = 'test_full_block_cipher_legacy_dispatch'
DRIVER = 'test_full_block_cipher_psa_dispatch'
IGNORED_SUITES = [
# Skipped in the accelerated component
'aes', 'aria', 'camellia',
# These require AES_C, ARIA_C or CAMELLIA_C to be enabled in
@ -664,8 +696,8 @@ KNOWN_TASKS = {
# component so we ignore them.
'cipher.ccm', 'cipher.gcm', 'cipher.aes', 'cipher.aria',
'cipher.camellia',
],
'ignored_tests': {
]
IGNORED_TESTS = {
'test_suite_config': [
re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA)_.*'),
re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'),
@ -715,10 +747,28 @@ KNOWN_TASKS = {
'Check mbedtls_calloc overallocation',
],
}
}
}
#pylint: enable=invalid-name,missing-class-docstring
# List of tasks with a function that can handle this task and additional arguments if required
KNOWN_TASKS = {
'analyze_coverage': CoverageTask,
'analyze_driver_vs_reference_hash': DriverVSReference_hash,
'analyze_driver_vs_reference_hmac': DriverVSReference_hmac,
'analyze_driver_vs_reference_cipher_aead_cmac': DriverVSReference_cipher_aead_cmac,
'analyze_driver_vs_reference_ecp_light_only': DriverVSReference_ecp_light_only,
'analyze_driver_vs_reference_no_ecp_at_all': DriverVSReference_no_ecp_at_all,
'analyze_driver_vs_reference_ecc_no_bignum': DriverVSReference_ecc_no_bignum,
'analyze_driver_vs_reference_ecc_ffdh_no_bignum': DriverVSReference_ecc_ffdh_no_bignum,
'analyze_driver_vs_reference_ffdh_alg': DriverVSReference_ffdh_alg,
'analyze_driver_vs_reference_tfm_config': DriverVSReference_tfm_config,
'analyze_driver_vs_reference_rsa': DriverVSReference_rsa,
'analyze_block_cipher_dispatch': DriverVSReference_block_cipher_dispatch,
}
def main():
main_results = Results()
@ -754,8 +804,6 @@ def main():
sys.stderr.write('invalid task: {}\n'.format(task))
sys.exit(2)
KNOWN_TASKS['analyze_coverage']['args']['full_coverage'] = options.full_coverage
# If the outcome file exists, parse it once and share the result
# among tasks to improve performance.
# Otherwise, it will be generated by execute_reference_driver_tests.
@ -766,21 +814,21 @@ def main():
task_name = tasks_list[0]
task = KNOWN_TASKS[task_name]
if task['test_function'] != do_analyze_driver_vs_reference: # pylint: disable=comparison-with-callable
if not issubclass(task, DriverVSReference):
sys.stderr.write("please provide valid outcomes file for {}.\n".format(task_name))
sys.exit(2)
execute_reference_driver_tests(main_results,
task['args']['component_ref'],
task['args']['component_driver'],
task.REFERENCE,
task.DRIVER,
options.outcomes)
outcomes = read_outcome_file(options.outcomes)
for task in tasks_list:
test_function = KNOWN_TASKS[task]['test_function']
test_args = KNOWN_TASKS[task]['args']
test_function(main_results, outcomes, test_args)
for task_name in tasks_list:
task_constructor = KNOWN_TASKS[task_name]
task = task_constructor(options)
main_results.new_section(task.section_name())
task.run(main_results, outcomes)
main_results.info("Overall results: {} warnings and {} errors",
main_results.warning_count, main_results.error_count)