diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py
index 188b68d1d5..87efb1b89c 100755
--- a/tests/scripts/analyze_outcomes.py
+++ b/tests/scripts/analyze_outcomes.py
@@ -14,7 +14,7 @@ import subprocess
import os
import typing
-import check_test_cases
+import collect_test_cases
# `ComponentOutcomes` is a named tuple which is defined as:
@@ -197,7 +197,7 @@ class CoverageTask(Task):
sys.stderr.write(cp.stdout.decode('utf-8'))
results.error("Failed \"make generated_files\" in tests. "
"Coverage analysis may be incorrect.")
- available = check_test_cases.collect_available_test_cases()
+ available = collect_test_cases.collect_available_test_cases()
for suite_case in available:
hit = any(suite_case in comp_outcomes.successes or
suite_case in comp_outcomes.failures
diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py
index a7ad9aebe3..b00319bad7 100755
--- a/tests/scripts/check_test_cases.py
+++ b/tests/scripts/check_test_cases.py
@@ -10,170 +10,14 @@ independently of the checks.
# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
import argparse
-import glob
-import os
import re
-import subprocess
import sys
import scripts_path # pylint: disable=unused-import
-from mbedtls_framework import build_tree
import collect_test_cases
-class ScriptOutputError(ValueError):
- """A kind of ValueError that indicates we found
- the script doesn't list test cases in an expected
- pattern.
- """
- @property
- def script_name(self):
- return super().args[0]
-
- @property
- def idx(self):
- return super().args[1]
-
- @property
- def line(self):
- return super().args[2]
-
-class Results:
- """Store file and line information about errors or warnings in test suites."""
-
- def __init__(self, options):
- self.errors = 0
- self.warnings = 0
- self.ignore_warnings = options.quiet
-
- def error(self, file_name, line_number, fmt, *args):
- sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n').
- format(file_name, line_number, *args))
- self.errors += 1
-
- def warning(self, file_name, line_number, fmt, *args):
- if not self.ignore_warnings:
- sys.stderr.write(('{}:{}:Warning:' + fmt + '\n')
- .format(file_name, line_number, *args))
- self.warnings += 1
-
-class TestDescriptionExplorer:
- """An iterator over test cases with descriptions.
-
-The test cases that have descriptions are:
-* Individual unit tests (entries in a .data file) in test suites.
-* Individual test cases in ssl-opt.sh.
-
-This is an abstract class. To use it, derive a class that implements
-the process_test_case method, and call walk_all().
-"""
-
- def process_test_case(self, per_file_state,
- file_name, line_number, description):
- """Process a test case.
-
-per_file_state: an object created by new_per_file_state() at the beginning
- of each file.
-file_name: a relative path to the file containing the test case.
-line_number: the line number in the given file.
-description: the test case description as a byte string.
-"""
- raise NotImplementedError
-
- def new_per_file_state(self):
- """Return a new per-file state object.
-
-The default per-file state object is None. Child classes that require per-file
-state may override this method.
-"""
- #pylint: disable=no-self-use
- return None
-
- def walk_test_suite(self, data_file_name):
- """Iterate over the test cases in the given unit test data file."""
- in_paragraph = False
- descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
- with open(data_file_name, 'rb') as data_file:
- for line_number, line in enumerate(data_file, 1):
- line = line.rstrip(b'\r\n')
- if not line:
- in_paragraph = False
- continue
- if line.startswith(b'#'):
- continue
- if not in_paragraph:
- # This is a test case description line.
- self.process_test_case(descriptions,
- data_file_name, line_number, line)
- in_paragraph = True
-
- def collect_from_script(self, script_name):
- """Collect the test cases in a script by calling its listing test cases
-option"""
- descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
- listed = subprocess.check_output(['sh', script_name, '--list-test-cases'])
- # Assume test file is responsible for printing identical format of
- # test case description between --list-test-cases and its OUTCOME.CSV
- #
- # idx indicates the number of test case since there is no line number
- # in the script for each test case.
- for idx, line in enumerate(listed.splitlines()):
- # We are expecting the script to list the test cases in
- # `<suite_name>;<description>` pattern.
- script_outputs = line.split(b';', 1)
- if len(script_outputs) == 2:
- suite_name, description = script_outputs
- else:
- raise ScriptOutputError(script_name, idx, line.decode("utf-8"))
-
- self.process_test_case(descriptions,
- suite_name.decode('utf-8'),
- idx,
- description.rstrip())
-
- @staticmethod
- def collect_test_directories():
- """Get the relative path for the TLS and Crypto test directories."""
- mbedtls_root = build_tree.guess_mbedtls_root()
- directories = [os.path.join(mbedtls_root, 'tests'),
- os.path.join(mbedtls_root, 'tf-psa-crypto', 'tests')]
- directories = [os.path.relpath(p) for p in directories]
- return directories
-
- def walk_all(self):
- """Iterate over all named test cases."""
- test_directories = self.collect_test_directories()
- for directory in test_directories:
- for data_file_name in glob.glob(os.path.join(directory, 'suites',
- '*.data')):
- self.walk_test_suite(data_file_name)
-
- for sh_file in ['ssl-opt.sh', 'compat.sh']:
- sh_file = os.path.join(directory, sh_file)
- if os.path.isfile(sh_file):
- self.collect_from_script(sh_file)
-
-class TestDescriptions(TestDescriptionExplorer):
- """Collect the available test cases."""
-
- def __init__(self):
- super().__init__()
- self.descriptions = set()
-
- def process_test_case(self, _per_file_state,
- file_name, _line_number, description):
- """Record an available test case."""
- base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name))
- key = ';'.join([base_name, description.decode('utf-8')])
- self.descriptions.add(key)
-
-def collect_available_test_cases():
- """Collect the available test cases."""
- explorer = TestDescriptions()
- explorer.walk_all()
- return sorted(explorer.descriptions)
-
-class DescriptionChecker(TestDescriptionExplorer):
+class DescriptionChecker(collect_test_cases.TestDescriptionExplorer):
"""Check all test case descriptions.
* Check that each description is valid (length, allowed character set, etc.).
@@ -223,14 +67,14 @@ def main():
help='Show warnings (default: on; undoes --quiet)')
options = parser.parse_args()
if options.list_all:
- descriptions = collect_available_test_cases()
+ descriptions = collect_test_cases.collect_available_test_cases()
sys.stdout.write('\n'.join(descriptions + ['']))
return
- results = Results(options)
+ results = collect_test_cases.Results(options)
checker = DescriptionChecker(results)
try:
checker.walk_all()
- except ScriptOutputError as e:
+ except collect_test_cases.ScriptOutputError as e:
results.error(e.script_name, e.idx,
'"{}" should be listed as "<suite_name>;<description>"',
e.line)
diff --git a/tests/scripts/collect_test_cases.py b/tests/scripts/collect_test_cases.py
index 7e9031c970..c326710b67 100644
--- a/tests/scripts/collect_test_cases.py
+++ b/tests/scripts/collect_test_cases.py
@@ -2,3 +2,166 @@
# Copyright The Mbed TLS Contributors
# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+
+import glob
+import os
+import re
+import subprocess
+import sys
+
+import scripts_path # pylint: disable=unused-import
+from mbedtls_framework import build_tree
+
+
+class ScriptOutputError(ValueError):
+ """A kind of ValueError that indicates we found
+ the script doesn't list test cases in an expected
+ pattern.
+ """
+
+ @property
+ def script_name(self):
+ return super().args[0]
+
+ @property
+ def idx(self):
+ return super().args[1]
+
+ @property
+ def line(self):
+ return super().args[2]
+
+class Results:
+ """Store file and line information about errors or warnings in test suites."""
+
+ def __init__(self, options):
+ self.errors = 0
+ self.warnings = 0
+ self.ignore_warnings = options.quiet
+
+ def error(self, file_name, line_number, fmt, *args):
+ sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n').
+ format(file_name, line_number, *args))
+ self.errors += 1
+
+ def warning(self, file_name, line_number, fmt, *args):
+ if not self.ignore_warnings:
+ sys.stderr.write(('{}:{}:Warning:' + fmt + '\n')
+ .format(file_name, line_number, *args))
+ self.warnings += 1
+
+class TestDescriptionExplorer:
+ """An iterator over test cases with descriptions.
+
+The test cases that have descriptions are:
+* Individual unit tests (entries in a .data file) in test suites.
+* Individual test cases in ssl-opt.sh.
+
+This is an abstract class. To use it, derive a class that implements
+the process_test_case method, and call walk_all().
+"""
+
+ def process_test_case(self, per_file_state,
+ file_name, line_number, description):
+ """Process a test case.
+
+per_file_state: an object created by new_per_file_state() at the beginning
+ of each file.
+file_name: a relative path to the file containing the test case.
+line_number: the line number in the given file.
+description: the test case description as a byte string.
+"""
+ raise NotImplementedError
+
+ def new_per_file_state(self):
+ """Return a new per-file state object.
+
+The default per-file state object is None. Child classes that require per-file
+state may override this method.
+"""
+ #pylint: disable=no-self-use
+ return None
+
+ def walk_test_suite(self, data_file_name):
+ """Iterate over the test cases in the given unit test data file."""
+ in_paragraph = False
+ descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
+ with open(data_file_name, 'rb') as data_file:
+ for line_number, line in enumerate(data_file, 1):
+ line = line.rstrip(b'\r\n')
+ if not line:
+ in_paragraph = False
+ continue
+ if line.startswith(b'#'):
+ continue
+ if not in_paragraph:
+ # This is a test case description line.
+ self.process_test_case(descriptions,
+ data_file_name, line_number, line)
+ in_paragraph = True
+
+ def collect_from_script(self, script_name):
+ """Collect the test cases in a script by calling its listing test cases
+option"""
+ descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
+ listed = subprocess.check_output(['sh', script_name, '--list-test-cases'])
+ # Assume test file is responsible for printing identical format of
+ # test case description between --list-test-cases and its OUTCOME.CSV
+ #
+ # idx indicates the number of test case since there is no line number
+ # in the script for each test case.
+ for idx, line in enumerate(listed.splitlines()):
+ # We are expecting the script to list the test cases in
+ # `<suite_name>;<description>` pattern.
+ script_outputs = line.split(b';', 1)
+ if len(script_outputs) == 2:
+ suite_name, description = script_outputs
+ else:
+ raise ScriptOutputError(script_name, idx, line.decode("utf-8"))
+
+ self.process_test_case(descriptions,
+ suite_name.decode('utf-8'),
+ idx,
+ description.rstrip())
+
+ @staticmethod
+ def collect_test_directories():
+ """Get the relative path for the TLS and Crypto test directories."""
+ mbedtls_root = build_tree.guess_mbedtls_root()
+ directories = [os.path.join(mbedtls_root, 'tests'),
+ os.path.join(mbedtls_root, 'tf-psa-crypto', 'tests')]
+ directories = [os.path.relpath(p) for p in directories]
+ return directories
+
+ def walk_all(self):
+ """Iterate over all named test cases."""
+ test_directories = self.collect_test_directories()
+ for directory in test_directories:
+ for data_file_name in glob.glob(os.path.join(directory, 'suites',
+ '*.data')):
+ self.walk_test_suite(data_file_name)
+
+ for sh_file in ['ssl-opt.sh', 'compat.sh']:
+ sh_file = os.path.join(directory, sh_file)
+ if os.path.isfile(sh_file):
+ self.collect_from_script(sh_file)
+
+class TestDescriptions(TestDescriptionExplorer):
+ """Collect the available test cases."""
+
+ def __init__(self):
+ super().__init__()
+ self.descriptions = set()
+
+ def process_test_case(self, _per_file_state,
+ file_name, _line_number, description):
+ """Record an available test case."""
+ base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name))
+ key = ';'.join([base_name, description.decode('utf-8')])
+ self.descriptions.add(key)
+
+def collect_available_test_cases():
+ """Collect the available test cases."""
+ explorer = TestDescriptions()
+ explorer.walk_all()
+ return sorted(explorer.descriptions)