From eba00974d60642c8440a9f0550aa395894ff1137 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 3 Oct 2024 17:35:52 +0200 Subject: [PATCH] Split test case collection from checks Move the test case collection code out of check_test_cases.py and into its own module. This allows outcome analysis to depend only on the new module and not on check_test_cases.py. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 4 +- tests/scripts/check_test_cases.py | 164 +--------------------------- tests/scripts/collect_test_cases.py | 163 +++++++++++++++++++++++++++ 3 files changed, 169 insertions(+), 162 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 188b68d1d5..87efb1b89c 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -14,7 +14,7 @@ import subprocess import os import typing -import check_test_cases +import collect_test_cases # `ComponentOutcomes` is a named tuple which is defined as: @@ -197,7 +197,7 @@ class CoverageTask(Task): sys.stderr.write(cp.stdout.decode('utf-8')) results.error("Failed \"make generated_files\" in tests. " "Coverage analysis may be incorrect.") - available = check_test_cases.collect_available_test_cases() + available = collect_test_cases.collect_available_test_cases() for suite_case in available: hit = any(suite_case in comp_outcomes.successes or suite_case in comp_outcomes.failures diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index a7ad9aebe3..b00319bad7 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -10,170 +10,14 @@ independently of the checks. # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later import argparse -import glob -import os import re -import subprocess import sys import scripts_path # pylint: disable=unused-import -from mbedtls_framework import build_tree import collect_test_cases -class ScriptOutputError(ValueError): - """A kind of ValueError that indicates we found - the script doesn't list test cases in an expected - pattern. - """ - @property - def script_name(self): - return super().args[0] - - @property - def idx(self): - return super().args[1] - - @property - def line(self): - return super().args[2] - -class Results: - """Store file and line information about errors or warnings in test suites.""" - - def __init__(self, options): - self.errors = 0 - self.warnings = 0 - self.ignore_warnings = options.quiet - - def error(self, file_name, line_number, fmt, *args): - sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n'). - format(file_name, line_number, *args)) - self.errors += 1 - - def warning(self, file_name, line_number, fmt, *args): - if not self.ignore_warnings: - sys.stderr.write(('{}:{}:Warning:' + fmt + '\n') - .format(file_name, line_number, *args)) - self.warnings += 1 - -class TestDescriptionExplorer: - """An iterator over test cases with descriptions. - -The test cases that have descriptions are: -* Individual unit tests (entries in a .data file) in test suites. -* Individual test cases in ssl-opt.sh. - -This is an abstract class. To use it, derive a class that implements -the process_test_case method, and call walk_all(). -""" - - def process_test_case(self, per_file_state, - file_name, line_number, description): - """Process a test case. - -per_file_state: an object created by new_per_file_state() at the beginning - of each file. -file_name: a relative path to the file containing the test case. -line_number: the line number in the given file. -description: the test case description as a byte string. -""" - raise NotImplementedError - - def new_per_file_state(self): - """Return a new per-file state object. - -The default per-file state object is None. Child classes that require per-file -state may override this method. -""" - #pylint: disable=no-self-use - return None - - def walk_test_suite(self, data_file_name): - """Iterate over the test cases in the given unit test data file.""" - in_paragraph = False - descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none - with open(data_file_name, 'rb') as data_file: - for line_number, line in enumerate(data_file, 1): - line = line.rstrip(b'\r\n') - if not line: - in_paragraph = False - continue - if line.startswith(b'#'): - continue - if not in_paragraph: - # This is a test case description line. - self.process_test_case(descriptions, - data_file_name, line_number, line) - in_paragraph = True - - def collect_from_script(self, script_name): - """Collect the test cases in a script by calling its listing test cases -option""" - descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none - listed = subprocess.check_output(['sh', script_name, '--list-test-cases']) - # Assume test file is responsible for printing identical format of - # test case description between --list-test-cases and its OUTCOME.CSV - # - # idx indicates the number of test case since there is no line number - # in the script for each test case. - for idx, line in enumerate(listed.splitlines()): - # We are expecting the script to list the test cases in - # `;` pattern. - script_outputs = line.split(b';', 1) - if len(script_outputs) == 2: - suite_name, description = script_outputs - else: - raise ScriptOutputError(script_name, idx, line.decode("utf-8")) - - self.process_test_case(descriptions, - suite_name.decode('utf-8'), - idx, - description.rstrip()) - - @staticmethod - def collect_test_directories(): - """Get the relative path for the TLS and Crypto test directories.""" - mbedtls_root = build_tree.guess_mbedtls_root() - directories = [os.path.join(mbedtls_root, 'tests'), - os.path.join(mbedtls_root, 'tf-psa-crypto', 'tests')] - directories = [os.path.relpath(p) for p in directories] - return directories - - def walk_all(self): - """Iterate over all named test cases.""" - test_directories = self.collect_test_directories() - for directory in test_directories: - for data_file_name in glob.glob(os.path.join(directory, 'suites', - '*.data')): - self.walk_test_suite(data_file_name) - - for sh_file in ['ssl-opt.sh', 'compat.sh']: - sh_file = os.path.join(directory, sh_file) - if os.path.isfile(sh_file): - self.collect_from_script(sh_file) - -class TestDescriptions(TestDescriptionExplorer): - """Collect the available test cases.""" - - def __init__(self): - super().__init__() - self.descriptions = set() - - def process_test_case(self, _per_file_state, - file_name, _line_number, description): - """Record an available test case.""" - base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name)) - key = ';'.join([base_name, description.decode('utf-8')]) - self.descriptions.add(key) - -def collect_available_test_cases(): - """Collect the available test cases.""" - explorer = TestDescriptions() - explorer.walk_all() - return sorted(explorer.descriptions) - -class DescriptionChecker(TestDescriptionExplorer): +class DescriptionChecker(collect_test_cases.TestDescriptionExplorer): """Check all test case descriptions. * Check that each description is valid (length, allowed character set, etc.). @@ -223,14 +67,14 @@ def main(): help='Show warnings (default: on; undoes --quiet)') options = parser.parse_args() if options.list_all: - descriptions = collect_available_test_cases() + descriptions = collect_test_cases.collect_available_test_cases() sys.stdout.write('\n'.join(descriptions + [''])) return - results = Results(options) + results = collect_test_cases.Results(options) checker = DescriptionChecker(results) try: checker.walk_all() - except ScriptOutputError as e: + except collect_test_cases.ScriptOutputError as e: results.error(e.script_name, e.idx, '"{}" should be listed as ";"', e.line) diff --git a/tests/scripts/collect_test_cases.py b/tests/scripts/collect_test_cases.py index 7e9031c970..c326710b67 100644 --- a/tests/scripts/collect_test_cases.py +++ b/tests/scripts/collect_test_cases.py @@ -2,3 +2,166 @@ # Copyright The Mbed TLS Contributors # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later + +import glob +import os +import re +import subprocess +import sys + +import scripts_path # pylint: disable=unused-import +from mbedtls_framework import build_tree + + +class ScriptOutputError(ValueError): + """A kind of ValueError that indicates we found + the script doesn't list test cases in an expected + pattern. + """ + + @property + def script_name(self): + return super().args[0] + + @property + def idx(self): + return super().args[1] + + @property + def line(self): + return super().args[2] + +class Results: + """Store file and line information about errors or warnings in test suites.""" + + def __init__(self, options): + self.errors = 0 + self.warnings = 0 + self.ignore_warnings = options.quiet + + def error(self, file_name, line_number, fmt, *args): + sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n'). + format(file_name, line_number, *args)) + self.errors += 1 + + def warning(self, file_name, line_number, fmt, *args): + if not self.ignore_warnings: + sys.stderr.write(('{}:{}:Warning:' + fmt + '\n') + .format(file_name, line_number, *args)) + self.warnings += 1 + +class TestDescriptionExplorer: + """An iterator over test cases with descriptions. + +The test cases that have descriptions are: +* Individual unit tests (entries in a .data file) in test suites. +* Individual test cases in ssl-opt.sh. + +This is an abstract class. To use it, derive a class that implements +the process_test_case method, and call walk_all(). +""" + + def process_test_case(self, per_file_state, + file_name, line_number, description): + """Process a test case. + +per_file_state: an object created by new_per_file_state() at the beginning + of each file. +file_name: a relative path to the file containing the test case. +line_number: the line number in the given file. +description: the test case description as a byte string. +""" + raise NotImplementedError + + def new_per_file_state(self): + """Return a new per-file state object. + +The default per-file state object is None. Child classes that require per-file +state may override this method. +""" + #pylint: disable=no-self-use + return None + + def walk_test_suite(self, data_file_name): + """Iterate over the test cases in the given unit test data file.""" + in_paragraph = False + descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none + with open(data_file_name, 'rb') as data_file: + for line_number, line in enumerate(data_file, 1): + line = line.rstrip(b'\r\n') + if not line: + in_paragraph = False + continue + if line.startswith(b'#'): + continue + if not in_paragraph: + # This is a test case description line. + self.process_test_case(descriptions, + data_file_name, line_number, line) + in_paragraph = True + + def collect_from_script(self, script_name): + """Collect the test cases in a script by calling its listing test cases +option""" + descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none + listed = subprocess.check_output(['sh', script_name, '--list-test-cases']) + # Assume test file is responsible for printing identical format of + # test case description between --list-test-cases and its OUTCOME.CSV + # + # idx indicates the number of test case since there is no line number + # in the script for each test case. + for idx, line in enumerate(listed.splitlines()): + # We are expecting the script to list the test cases in + # `;` pattern. + script_outputs = line.split(b';', 1) + if len(script_outputs) == 2: + suite_name, description = script_outputs + else: + raise ScriptOutputError(script_name, idx, line.decode("utf-8")) + + self.process_test_case(descriptions, + suite_name.decode('utf-8'), + idx, + description.rstrip()) + + @staticmethod + def collect_test_directories(): + """Get the relative path for the TLS and Crypto test directories.""" + mbedtls_root = build_tree.guess_mbedtls_root() + directories = [os.path.join(mbedtls_root, 'tests'), + os.path.join(mbedtls_root, 'tf-psa-crypto', 'tests')] + directories = [os.path.relpath(p) for p in directories] + return directories + + def walk_all(self): + """Iterate over all named test cases.""" + test_directories = self.collect_test_directories() + for directory in test_directories: + for data_file_name in glob.glob(os.path.join(directory, 'suites', + '*.data')): + self.walk_test_suite(data_file_name) + + for sh_file in ['ssl-opt.sh', 'compat.sh']: + sh_file = os.path.join(directory, sh_file) + if os.path.isfile(sh_file): + self.collect_from_script(sh_file) + +class TestDescriptions(TestDescriptionExplorer): + """Collect the available test cases.""" + + def __init__(self): + super().__init__() + self.descriptions = set() + + def process_test_case(self, _per_file_state, + file_name, _line_number, description): + """Record an available test case.""" + base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name)) + key = ';'.join([base_name, description.decode('utf-8')]) + self.descriptions.add(key) + +def collect_available_test_cases(): + """Collect the available test cases.""" + explorer = TestDescriptions() + explorer.walk_all() + return sorted(explorer.descriptions)