From 3c170d32985e65f51cf86de4b5ae195bc274802e Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Wed, 29 Nov 2023 13:53:34 +0800 Subject: [PATCH 1/7] Print suite name when listing test cases When a test script has multiple suites, it is not true to determine the suite name from the file name of the script. We need the script to list the suite name for every test cases. Signed-off-by: Pengyu Lv --- tests/compat.sh | 2 +- tests/ssl-opt.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/compat.sh b/tests/compat.sh index ac29e50c35..a101ffd138 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -125,7 +125,7 @@ print_usage() { print_test_case() { for i in $3; do uniform_title $1 $2 $i - echo $TITLE + echo "compat;$TITLE" done } diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 42f9f5e5a7..43a8863558 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -1620,7 +1620,7 @@ run_test() { fi if [ "$LIST_TESTS" -gt 0 ]; then - printf "%s\n" "$NAME" + printf "%s\n" "${TEST_SUITE_NAME:-ssl-opt};$NAME" return fi From 443c479fafa80489fbd579259c93a60f63dfa745 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Wed, 29 Nov 2023 14:24:52 +0800 Subject: [PATCH 2/7] Use the outputs as keys if the test case is defined in a script Signed-off-by: Pengyu Lv --- tests/scripts/check_test_cases.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 68e7e690a8..deebbd3882 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -137,8 +137,14 @@ class TestDescriptions(TestDescriptionExplorer): def process_test_case(self, _per_file_state, file_name, _line_number, description): """Record an available test case.""" - base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name)) - key = ';'.join([base_name, description.decode('utf-8')]) + if file_name.endswith('.data'): + base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name)) + key = ';'.join([base_name, description.decode('utf-8')]) + else: + # For test cases defined in scripts (i.e. ssl-op.sh and compat.sh), + # we need the script to list the suite name, and use the outputs + # as keys directly. + key = description.decode('utf-8') self.descriptions.add(key) def collect_available_test_cases(): From 2978c6c24ea1521fdc5b021c3b2fdedbe494f007 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Wed, 29 Nov 2023 17:35:38 +0800 Subject: [PATCH 3/7] Add rules to check script test case listing Signed-off-by: Pengyu Lv --- tests/scripts/check_test_cases.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index deebbd3882..5cac96049e 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -172,6 +172,15 @@ class DescriptionChecker(TestDescriptionExplorer): """Check test case descriptions for errors.""" results = self.results seen = per_file_state + if not file_name.endswith('.data'): + script_output = description.split(b';', 1) + if len(script_output) == 2: + description = script_output[1] + else: + results.error(file_name, line_number, + '"{}" should be listed in ' + '";" format', + description.decode('ascii')) if description in seen: results.error(file_name, line_number, 'Duplicate description (also line {})', From ce980e61cc40123a37b2af05f1de5021698f5224 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Thu, 30 Nov 2023 16:53:31 +0800 Subject: [PATCH 4/7] Move script outputs handling to collect_from_script To simplify the logic, `collect_from_script` should take the responsiblity to parse script outputs to suite name and test case description. Also define new Error class ScriptOutputError for error handling. Signed-off-by: Pengyu Lv --- tests/scripts/check_test_cases.py | 59 +++++++++++++++++++------------ 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 5cac96049e..39bf73c838 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -16,6 +16,23 @@ import re import subprocess import sys +class ScriptOutputError(ValueError): + """A kind of ValueError that indicates we found + the script doesn't list test cases in an expected + pattern. + """ + + @property + def script_name(self): + return super().args[0] + + @property + def idx(self): + return super().args[1] + + @property + def line(self): + return super().args[2] class Results: """Store file and line information about errors or warnings in test suites.""" @@ -86,19 +103,27 @@ state may override this method. data_file_name, line_number, line) in_paragraph = True - def collect_from_script(self, file_name): + def collect_from_script(self, script_name): """Collect the test cases in a script by calling its listing test cases option""" descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none - listed = subprocess.check_output(['sh', file_name, '--list-test-cases']) + listed = subprocess.check_output(['sh', script_name, '--list-test-cases']) # Assume test file is responsible for printing identical format of # test case description between --list-test-cases and its OUTCOME.CSV # # idx indicates the number of test case since there is no line number # in the script for each test case. - for idx, description in enumerate(listed.splitlines()): + for idx, line in enumerate(listed.splitlines()): + # We are expecting the script to list the test cases in + # `;` pattern. + script_outputs = line.split(b';', 1) + if len(script_outputs) == 2: + suite_name, description = script_outputs + else: + raise ScriptOutputError(script_name, idx, line.decode("utf-8")) + self.process_test_case(descriptions, - file_name, + suite_name.decode('utf-8'), idx, description.rstrip()) @@ -137,14 +162,8 @@ class TestDescriptions(TestDescriptionExplorer): def process_test_case(self, _per_file_state, file_name, _line_number, description): """Record an available test case.""" - if file_name.endswith('.data'): - base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name)) - key = ';'.join([base_name, description.decode('utf-8')]) - else: - # For test cases defined in scripts (i.e. ssl-op.sh and compat.sh), - # we need the script to list the suite name, and use the outputs - # as keys directly. - key = description.decode('utf-8') + base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name)) + key = ';'.join([base_name, description.decode('utf-8')]) self.descriptions.add(key) def collect_available_test_cases(): @@ -172,15 +191,6 @@ class DescriptionChecker(TestDescriptionExplorer): """Check test case descriptions for errors.""" results = self.results seen = per_file_state - if not file_name.endswith('.data'): - script_output = description.split(b';', 1) - if len(script_output) == 2: - description = script_output[1] - else: - results.error(file_name, line_number, - '"{}" should be listed in ' - '";" format', - description.decode('ascii')) if description in seen: results.error(file_name, line_number, 'Duplicate description (also line {})', @@ -217,7 +227,12 @@ def main(): return results = Results(options) checker = DescriptionChecker(results) - checker.walk_all() + try: + checker.walk_all() + except ScriptOutputError as e: + results.error(e.script_name, e.idx, + '"{}" should be listed as ";"', + e.line) if (results.warnings or results.errors) and not options.quiet: sys.stderr.write('{}: {} errors, {} warnings\n' .format(sys.argv[0], results.errors, results.warnings)) From c353c5cfd5adecbb8774780bf3202a3a31473470 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Thu, 30 Nov 2023 16:57:08 +0800 Subject: [PATCH 5/7] Catch ScriptOutputError in analyze_outcomes.py Signed-off-by: Pengyu Lv --- tests/scripts/analyze_outcomes.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index ca349d38e9..ab983ec697 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -85,7 +85,12 @@ def execute_reference_driver_tests(results: Results, ref_component: str, driver_ def analyze_coverage(results: Results, outcomes: Outcomes, allow_list: typing.List[str], full_coverage: bool) -> None: """Check that all available test cases are executed at least once.""" - available = check_test_cases.collect_available_test_cases() + try: + available = check_test_cases.collect_available_test_cases() + except check_test_cases.ScriptOutputError: + results.error("fail to collect available test cases") + return + for suite_case in available: hit = any(suite_case in comp_outcomes.successes or suite_case in comp_outcomes.failures From 5bde6bd8a6da0628233e9169e6ec24a09cde683e Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Fri, 8 Dec 2023 12:22:56 +0800 Subject: [PATCH 6/7] Revert "Catch ScriptOutputError in analyze_outcomes.py" Just abort the program if there is exception raised when collecting available test cases, so that we could easily found out what's going wrong. This reverts commit c353c5cfd5adecbb8774780bf3202a3a31473470. Signed-off-by: Pengyu Lv --- tests/scripts/analyze_outcomes.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index ab983ec697..ca349d38e9 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -85,12 +85,7 @@ def execute_reference_driver_tests(results: Results, ref_component: str, driver_ def analyze_coverage(results: Results, outcomes: Outcomes, allow_list: typing.List[str], full_coverage: bool) -> None: """Check that all available test cases are executed at least once.""" - try: - available = check_test_cases.collect_available_test_cases() - except check_test_cases.ScriptOutputError: - results.error("fail to collect available test cases") - return - + available = check_test_cases.collect_available_test_cases() for suite_case in available: hit = any(suite_case in comp_outcomes.successes or suite_case in comp_outcomes.failures From 7166434ad82a8fe4aa506242d05c541b9a14bb20 Mon Sep 17 00:00:00 2001 From: Pengyu Lv Date: Fri, 8 Dec 2023 13:06:54 +0800 Subject: [PATCH 7/7] Error out if script is missing when collecting test cases Signed-off-by: Pengyu Lv --- tests/scripts/check_test_cases.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 39bf73c838..d67e6781b4 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -149,8 +149,7 @@ option""" for sh_file in ['ssl-opt.sh', 'compat.sh']: sh_file = os.path.join(directory, sh_file) - if os.path.exists(sh_file): - self.collect_from_script(sh_file) + self.collect_from_script(sh_file) class TestDescriptions(TestDescriptionExplorer): """Collect the available test cases."""