mirror/mbedtls
1
0
Fork 0
mirror of https://github.com/Mbed-TLS/mbedtls.git synced 2025-04-25 09:02:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add allow list for non-executed test cases

Browse Source 
The allow list explicits which test cases are allowed to not be
executed when testing. This may be, for example, because a feature
is yet to be developed but the test for that feature is already in
our code base.

Signed-off-by: Tomás González <tomasagustin.gonzalezorlando@arm.com>
This commit is contained in:
Tomás González 2023-08-11 14:59:03 +01:00
parent a797f152ee
commit 07bdcc2b0d
1 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
tests/scripts/analyze_outcomes.py
View File

@ -73,15 +73,18 @@ def execute_reference_driver_tests(ref_component, driver_component, outcome_file
Results.log("Error: failed to run reference/driver components") Results.log("Error: failed to run reference/driver components")
sys.exit(ret_val) sys.exit(ret_val)
def analyze_coverage(results, outcomes): def analyze_coverage(results, outcomes, allow_list):
"""Check that all available test cases are executed at least once.""" """Check that all available test cases are executed at least once."""
available = check_test_cases.collect_available_test_cases() available = check_test_cases.collect_available_test_cases()
for key in available: for key in available:
hits = outcomes[key].hits() if key in outcomes else 0 hits = outcomes[key].hits() if key in outcomes else 0
if hits == 0: if hits == 0 and key not in allow_list:
# Make this a warning, not an error, as long as we haven't # Make this a warning, not an error, as long as we haven't
# fixed this branch to have full coverage of test cases. # fixed this branch to have full coverage of test cases.
results.warning('Test case not executed: {}', key) results.warning('Test case not executed: {}', key)
elif hits != 0 and key in allow_list:
# Test Case should be removed from the allow list.
results.warning('Allow listed test case was executed: {}', key)
def analyze_driver_vs_reference(outcomes, component_ref, component_driver, def analyze_driver_vs_reference(outcomes, component_ref, component_driver,
ignored_suites, ignored_test=None): ignored_suites, ignored_test=None):
@ -122,10 +125,10 @@ def analyze_driver_vs_reference(outcomes, component_ref, component_driver,
result = False result = False
return result return result
def analyze_outcomes(outcomes): def analyze_outcomes(outcomes, allow_list):
"""Run all analyses on the given outcome collection.""" """Run all analyses on the given outcome collection."""
results = Results() results = Results()
analyze_coverage(results, outcomes) analyze_coverage(results, outcomes, allow_list)
return results return results
def read_outcome_file(outcome_file): def read_outcome_file(outcome_file):
@ -151,10 +154,9 @@ by a semicolon.
def do_analyze_coverage(outcome_file, args): def do_analyze_coverage(outcome_file, args):
"""Perform coverage analysis.""" """Perform coverage analysis."""
del args # unused
outcomes = read_outcome_file(outcome_file) outcomes = read_outcome_file(outcome_file)
Results.log("\n*** Analyze coverage ***\n") Results.log("\n*** Analyze coverage ***\n")
results = analyze_outcomes(outcomes) results = analyze_outcomes(outcomes, args['allow_list'])
return results.error_count == 0 return results.error_count == 0
def do_analyze_driver_vs_reference(outcome_file, args): def do_analyze_driver_vs_reference(outcome_file, args):
@ -175,7 +177,9 @@ def do_analyze_driver_vs_reference(outcome_file, args):
TASKS = { TASKS = {
'analyze_coverage': { 'analyze_coverage': {
'test_function': do_analyze_coverage, 'test_function': do_analyze_coverage,
'args': {} 'args': {
'allow_list': [],
}
}, },
# There are 2 options to use analyze_driver_vs_reference_xxx locally: # There are 2 options to use analyze_driver_vs_reference_xxx locally:
# 1. Run tests and then analysis: # 1. Run tests and then analysis: