mirror/mbedtls
1
0
Fork 0
mirror of https://github.com/Mbed-TLS/mbedtls.git synced 2025-03-21 19:20:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Replace stringly typed data by class: coverage

Browse Source 
Work on replacing the stringly typed KNOWN_TASKS by classes for each category
of tasks, with a structure that matches the behavior.

This commit migrates test coverage analysis.

No intended behavior change.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
This commit is contained in:
Gilles Peskine 2024-09-16 19:15:29 +02:00
parent 00ed057738
commit 0316f1006c
1 changed files with 30 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
tests/scripts/analyze_outcomes.py
View File

@ -170,11 +170,6 @@ def analyze_driver_vs_reference(results: Results, outcomes: Outcomes,
if ignored and suite_case in driver_outcomes.successes:
results.error("uselessly ignored: {}", suite_case)
def analyze_outcomes(results: Results, outcomes: Outcomes, args) -> None:
"""Run all analyses on the given outcome collection."""
analyze_coverage(results, outcomes, args['allow_list'],
args['full_coverage'])
def read_outcome_file(outcome_file: str) -> Outcomes:
"""Parse an outcome file and return an outcome collection.
"""
@ -195,11 +190,6 @@ def read_outcome_file(outcome_file: str) -> Outcomes:
return outcomes
def do_analyze_coverage(results: Results, outcomes: Outcomes, args) -> None:
"""Perform coverage analysis."""
results.new_section("Analyze coverage")
analyze_outcomes(results, outcomes, args)
def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -> None:
"""Perform driver vs reference analyze."""
results.new_section("Analyze driver {} vs reference {}",
@ -222,6 +212,9 @@ class Task:
"""
pass
def section_name(self) -> str:
"""The section name to use in results."""
def run(self, results: Results, outcomes: Outcomes):
"""Run the analysis on the specified outcomes.
@ -230,20 +223,34 @@ class Task:
raise NotImplementedError
class CoverageTask(Task):
"""Analyze test coverage."""
ALLOW_LIST = [
# Algorithm not supported yet
'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA',
# Algorithm not supported yet
'test_suite_psa_crypto_metadata;Cipher: XTS',
]
def __init__(self, options) -> None:
super().__init__(options)
self.full_coverage = options.full_coverage #type: bool
@staticmethod
def section_name() -> str:
return "Analyze coverage"
def run(self, results: Results, outcomes: Outcomes):
"""Check that all test cases are executed at least once."""
analyze_coverage(results, outcomes,
self.ALLOW_LIST, self.full_coverage)
# List of tasks with a function that can handle this task and additional arguments if required
KNOWN_TASKS = {
'analyze_coverage': {
'test_function': do_analyze_coverage,
'args': {
'allow_list': [
# Algorithm not supported yet
'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA',
# Algorithm not supported yet
'test_suite_psa_crypto_metadata;Cipher: XTS',
],
'full_coverage': False,
}
},
'analyze_coverage': CoverageTask,
# There are 2 options to use analyze_driver_vs_reference_xxx locally:
# 1. Run tests and then analysis:
# - tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver>
@ -773,8 +780,6 @@ def main():
sys.stderr.write('invalid task: {}\n'.format(task))
sys.exit(2)
KNOWN_TASKS['analyze_coverage']['args']['full_coverage'] = options.full_coverage
# If the outcome file exists, parse it once and share the result
# among tasks to improve performance.
# Otherwise, it will be generated by execute_reference_driver_tests.
@ -805,6 +810,7 @@ def main():
test_function(main_results, outcomes, test_args)
else:
task = task_constructor(options)
main_results.new_section(task.section_name())
task.run(main_results, outcomes)
main_results.info("Overall results: {} warnings and {} errors",