diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 5835c80948..98c4afdbb9 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -170,11 +170,6 @@ def analyze_driver_vs_reference(results: Results, outcomes: Outcomes, if ignored and suite_case in driver_outcomes.successes: results.error("uselessly ignored: {}", suite_case) -def analyze_outcomes(results: Results, outcomes: Outcomes, args) -> None: - """Run all analyses on the given outcome collection.""" - analyze_coverage(results, outcomes, args['allow_list'], - args['full_coverage']) - def read_outcome_file(outcome_file: str) -> Outcomes: """Parse an outcome file and return an outcome collection. """ @@ -195,11 +190,6 @@ def read_outcome_file(outcome_file: str) -> Outcomes: return outcomes -def do_analyze_coverage(results: Results, outcomes: Outcomes, args) -> None: - """Perform coverage analysis.""" - results.new_section("Analyze coverage") - analyze_outcomes(results, outcomes, args) - def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -> None: """Perform driver vs reference analyze.""" results.new_section("Analyze driver {} vs reference {}", @@ -222,6 +212,9 @@ class Task: """ pass + def section_name(self) -> str: + """The section name to use in results.""" + def run(self, results: Results, outcomes: Outcomes): """Run the analysis on the specified outcomes. @@ -230,20 +223,34 @@ class Task: raise NotImplementedError +class CoverageTask(Task): + """Analyze test coverage.""" + + ALLOW_LIST = [ + # Algorithm not supported yet + 'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA', + # Algorithm not supported yet + 'test_suite_psa_crypto_metadata;Cipher: XTS', + ] + + def __init__(self, options) -> None: + super().__init__(options) + self.full_coverage = options.full_coverage #type: bool + + @staticmethod + def section_name() -> str: + return "Analyze coverage" + + def run(self, results: Results, outcomes: Outcomes): + """Check that all test cases are executed at least once.""" + analyze_coverage(results, outcomes, + self.ALLOW_LIST, self.full_coverage) + + # List of tasks with a function that can handle this task and additional arguments if required KNOWN_TASKS = { - 'analyze_coverage': { - 'test_function': do_analyze_coverage, - 'args': { - 'allow_list': [ - # Algorithm not supported yet - 'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA', - # Algorithm not supported yet - 'test_suite_psa_crypto_metadata;Cipher: XTS', - ], - 'full_coverage': False, - } - }, + 'analyze_coverage': CoverageTask, + # There are 2 options to use analyze_driver_vs_reference_xxx locally: # 1. Run tests and then analysis: # - tests/scripts/all.sh --outcome-file "$PWD/out.csv" @@ -773,8 +780,6 @@ def main(): sys.stderr.write('invalid task: {}\n'.format(task)) sys.exit(2) - KNOWN_TASKS['analyze_coverage']['args']['full_coverage'] = options.full_coverage - # If the outcome file exists, parse it once and share the result # among tasks to improve performance. # Otherwise, it will be generated by execute_reference_driver_tests. @@ -805,6 +810,7 @@ def main(): test_function(main_results, outcomes, test_args) else: task = task_constructor(options) + main_results.new_section(task.section_name()) task.run(main_results, outcomes) main_results.info("Overall results: {} warnings and {} errors",