From 68a98516f61b2e48ad91cbd74210d9503419d19b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 25 Jun 2020 14:19:09 +0200 Subject: [PATCH 1/9] basic-in-docker: call all.sh for sanity checks Call all.sh for sanity checks, rather than maintain an explicit list. This was done in .travis.yml in 3c7ffd7a4091916db501d41c8e9ce6bc7e2f0586 Travis has diverged from basic-in-docker. This commit updates the description of basic-in-docker to no longer refer to Travis. Alignment with Travis may be desirable but that is beyond the scope of this commit. Signed-off-by: Gilles Peskine --- tests/scripts/basic-in-docker.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/scripts/basic-in-docker.sh b/tests/scripts/basic-in-docker.sh index 37ed5ea50b..83d665598e 100755 --- a/tests/scripts/basic-in-docker.sh +++ b/tests/scripts/basic-in-docker.sh @@ -4,8 +4,10 @@ # # Purpose # ------- -# This runs a rough equivalent of the travis.yml in a Docker container. -# The tests are run for both clang and gcc. +# This runs sanity checks and library tests in a Docker container. The tests +# are run for both clang and gcc. The testing includes a full test run +# in the default configuration, partial test runs in the reference +# configurations, and some dependency tests. # # Notes for users # --------------- @@ -30,12 +32,7 @@ source tests/scripts/docker_env.sh -run_in_docker tests/scripts/recursion.pl library/*.c -run_in_docker tests/scripts/check-generated-files.sh -run_in_docker tests/scripts/check-doxy-blocks.pl -run_in_docker tests/scripts/check-names.sh -run_in_docker tests/scripts/check-files.py -run_in_docker tests/scripts/doxygen.sh +run_in_docker tests/scripts/all.sh 'check_*' for compiler in clang gcc; do run_in_docker -e CC=${compiler} cmake -D CMAKE_BUILD_TYPE:String="Check" . From fb4f933f8e47c86bad133a3a5bb6c492f89267e1 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 25 Jun 2020 14:18:34 +0200 Subject: [PATCH 2/9] Rename Python scripts to use '_' and not '-' You can't import a Python script whose name includes '-'. Signed-off-by: Gilles Peskine --- docs/architecture/testing/test-framework.md | 4 ++-- tests/scripts/all.sh | 4 ++-- tests/scripts/{check-files.py => check_files.py} | 0 tests/scripts/{check-test-cases.py => check_test_cases.py} | 0 4 files changed, 4 insertions(+), 4 deletions(-) rename tests/scripts/{check-files.py => check_files.py} (100%) rename tests/scripts/{check-test-cases.py => check_test_cases.py} (100%) diff --git a/docs/architecture/testing/test-framework.md b/docs/architecture/testing/test-framework.md index e0e960f87c..c4178fa170 100644 --- a/docs/architecture/testing/test-framework.md +++ b/docs/architecture/testing/test-framework.md @@ -22,7 +22,7 @@ Each test case has a description which succinctly describes for a human audience * Make the description descriptive. “foo: x=2, y=4” is more descriptive than “foo #2”. “foo: 0 Date: Thu, 25 Jun 2020 16:16:25 +0200 Subject: [PATCH 3/9] check_test_cases: parametrize iteration functions by the action Parametrize the code that iterates over test case descriptions by the function to apply on each description. No behavior change. Signed-off-by: Gilles Peskine --- tests/scripts/check_test_cases.py | 50 ++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 35a9987497..f25b602c7d 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -76,10 +76,13 @@ def check_description(results, seen, file_name, line_number, description): len(description)) seen[description] = line_number -def check_test_suite(results, data_file_name): - """Check the test cases in the given unit test data file.""" +def walk_test_suite(function, results, descriptions, data_file_name): + """Iterate over the test cases in the given unit test data file. + +Call function(results, descriptions, data_file_name, line_number, description) +on each description. +""" in_paragraph = False - descriptions = {} with open(data_file_name, 'rb') as data_file: for line_number, line in enumerate(data_file, 1): line = line.rstrip(b'\r\n') @@ -90,13 +93,16 @@ def check_test_suite(results, data_file_name): continue if not in_paragraph: # This is a test case description line. - check_description(results, descriptions, - data_file_name, line_number, line) + function(results, descriptions, + data_file_name, line_number, line) in_paragraph = True -def check_ssl_opt_sh(results, file_name): - """Check the test cases in ssl-opt.sh or a file with a similar format.""" - descriptions = {} +def walk_ssl_opt_sh(function, results, descriptions, file_name): + """Iterate over the test cases in ssl-opt.sh or a file with a similar format. + +Call function(results, descriptions, file_name, line_number, description) +on each description. +""" with open(file_name, 'rb') as file_contents: for line_number, line in enumerate(file_contents, 1): # Assume that all run_test calls have the same simple form @@ -106,8 +112,23 @@ def check_ssl_opt_sh(results, file_name): if not m: continue description = m.group(1) - check_description(results, descriptions, - file_name, line_number, description) + function(results, descriptions, + file_name, line_number, description) + +def walk_all(function, results): + """Iterate over all named test cases. + +Call function(results, {}, file_name, line_number, description) +on each description. +""" + test_directories = collect_test_directories() + for directory in test_directories: + for data_file_name in glob.glob(os.path.join(directory, 'suites', + '*.data')): + walk_test_suite(function, results, {}, data_file_name) + ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh') + if os.path.exists(ssl_opt_sh): + walk_ssl_opt_sh(function, results, {}, ssl_opt_sh) def main(): parser = argparse.ArgumentParser(description=__doc__) @@ -118,15 +139,8 @@ def main(): action='store_false', dest='quiet', help='Show warnings (default: on; undoes --quiet)') options = parser.parse_args() - test_directories = collect_test_directories() results = Results(options) - for directory in test_directories: - for data_file_name in glob.glob(os.path.join(directory, 'suites', - '*.data')): - check_test_suite(results, data_file_name) - ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh') - if os.path.exists(ssl_opt_sh): - check_ssl_opt_sh(results, ssl_opt_sh) + walk_all(check_description, results) if (results.warnings or results.errors) and not options.quiet: sys.stderr.write('{}: {} errors, {} warnings\n' .format(sys.argv[0], results.errors, results.warnings)) From 78c45dbb0f74f7cba5e19de9e2f98dcd6ca91d68 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 25 Jun 2020 16:34:11 +0200 Subject: [PATCH 4/9] check_test_cases: move "walk" functions into a class Make the structure more Pythonic: use classes for abstraction and refinement, rather than higher-order functions. Convert walk(function, state, data) into instance.walk(data) where instance has a method that implements function and state is a field of instance. No behavior change. Signed-off-by: Gilles Peskine --- tests/scripts/check_test_cases.py | 132 +++++++++++++++++++----------- 1 file changed, 86 insertions(+), 46 deletions(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index f25b602c7d..04ade631a0 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -76,59 +76,98 @@ def check_description(results, seen, file_name, line_number, description): len(description)) seen[description] = line_number -def walk_test_suite(function, results, descriptions, data_file_name): - """Iterate over the test cases in the given unit test data file. +class TestDescriptionExplorer: + """An iterator over test cases with descriptions. -Call function(results, descriptions, data_file_name, line_number, description) -on each description. +The test cases that have descriptions are: +* Individual unit tests (entries in a .data file) in test suites. +* Individual test cases in ssl-opt.sh. + +This is an abstract class. To use it, derive a class that implements +the process_test_case method, and call walk_all(). """ - in_paragraph = False - with open(data_file_name, 'rb') as data_file: - for line_number, line in enumerate(data_file, 1): - line = line.rstrip(b'\r\n') - if not line: - in_paragraph = False - continue - if line.startswith(b'#'): - continue - if not in_paragraph: - # This is a test case description line. - function(results, descriptions, - data_file_name, line_number, line) - in_paragraph = True -def walk_ssl_opt_sh(function, results, descriptions, file_name): - """Iterate over the test cases in ssl-opt.sh or a file with a similar format. + def process_test_case(self, per_file_state, + file_name, line_number, description): + """Process a test case. -Call function(results, descriptions, file_name, line_number, description) -on each description. +per_file_state: a new object returned by per_file_state() for each file. +file_name: a relative path to the file containing the test case. +line_number: the line number in the given file. +description: the test case description as a byte string. """ - with open(file_name, 'rb') as file_contents: - for line_number, line in enumerate(file_contents, 1): - # Assume that all run_test calls have the same simple form - # with the test description entirely on the same line as the - # function name. - m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line) - if not m: - continue - description = m.group(1) - function(results, descriptions, - file_name, line_number, description) + raise NotImplementedError -def walk_all(function, results): - """Iterate over all named test cases. + def per_file_state(self): + """Return a new per-file state object. -Call function(results, {}, file_name, line_number, description) -on each description. +The default per-file state object is None. Child classes that require per-file +state may override this method. """ - test_directories = collect_test_directories() - for directory in test_directories: - for data_file_name in glob.glob(os.path.join(directory, 'suites', - '*.data')): - walk_test_suite(function, results, {}, data_file_name) - ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh') - if os.path.exists(ssl_opt_sh): - walk_ssl_opt_sh(function, results, {}, ssl_opt_sh) + #pylint: disable=no-self-use + return None + + def walk_test_suite(self, data_file_name): + """Iterate over the test cases in the given unit test data file.""" + in_paragraph = False + descriptions = self.per_file_state() # pylint: disable=assignment-from-none + with open(data_file_name, 'rb') as data_file: + for line_number, line in enumerate(data_file, 1): + line = line.rstrip(b'\r\n') + if not line: + in_paragraph = False + continue + if line.startswith(b'#'): + continue + if not in_paragraph: + # This is a test case description line. + self.process_test_case(descriptions, + data_file_name, line_number, line) + in_paragraph = True + + def walk_ssl_opt_sh(self, file_name): + """Iterate over the test cases in ssl-opt.sh or a file with a similar format.""" + descriptions = self.per_file_state() # pylint: disable=assignment-from-none + with open(file_name, 'rb') as file_contents: + for line_number, line in enumerate(file_contents, 1): + # Assume that all run_test calls have the same simple form + # with the test description entirely on the same line as the + # function name. + m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line) + if not m: + continue + description = m.group(1) + self.process_test_case(descriptions, + file_name, line_number, description) + + def walk_all(self): + """Iterate over all named test cases.""" + test_directories = collect_test_directories() + for directory in test_directories: + for data_file_name in glob.glob(os.path.join(directory, 'suites', + '*.data')): + self.walk_test_suite(data_file_name) + ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh') + if os.path.exists(ssl_opt_sh): + self.walk_ssl_opt_sh(ssl_opt_sh) + +class DescriptionChecker(TestDescriptionExplorer): + """Check all test case descriptions. + +* Check that each description is valid (length, allowed character set, etc.). +* Check that there is no duplicated description inside of one test suite. +""" + + def __init__(self, results): + self.results = results + + def per_file_state(self): + return {} + + def process_test_case(self, per_file_state, + file_name, line_number, description): + check_description(self.results, per_file_state, + file_name, line_number, description) def main(): parser = argparse.ArgumentParser(description=__doc__) @@ -140,7 +179,8 @@ def main(): help='Show warnings (default: on; undoes --quiet)') options = parser.parse_args() results = Results(options) - walk_all(check_description, results) + checker = DescriptionChecker(results) + checker.walk_all() if (results.warnings or results.errors) and not options.quiet: sys.stderr.write('{}: {} errors, {} warnings\n' .format(sys.argv[0], results.errors, results.warnings)) From 6f6ff3346d971a049def9730c32ee4d7cf93f935 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 25 Jun 2020 16:40:10 +0200 Subject: [PATCH 5/9] check_test_cases: move some functions into the logical class With previous refactorings, some functions are now solely meant to be called from other functions in a particular class. Move them into this class. No behavior change. Signed-off-by: Gilles Peskine --- tests/scripts/check_test_cases.py | 68 +++++++++++++++---------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 04ade631a0..2df4c7a68d 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -45,37 +45,6 @@ class Results: .format(file_name, line_number, *args)) self.warnings += 1 -def collect_test_directories(): - """Get the relative path for the TLS and Crypto test directories.""" - if os.path.isdir('tests'): - tests_dir = 'tests' - elif os.path.isdir('suites'): - tests_dir = '.' - elif os.path.isdir('../suites'): - tests_dir = '..' - directories = [tests_dir] - return directories - -def check_description(results, seen, file_name, line_number, description): - """Check test case descriptions for errors.""" - if description in seen: - results.error(file_name, line_number, - 'Duplicate description (also line {})', - seen[description]) - return - if re.search(br'[\t;]', description): - results.error(file_name, line_number, - 'Forbidden character \'{}\' in description', - re.search(br'[\t;]', description).group(0).decode('ascii')) - if re.search(br'[^ -~]', description): - results.error(file_name, line_number, - 'Non-ASCII character in description') - if len(description) > 66: - results.warning(file_name, line_number, - 'Test description too long ({} > 66)', - len(description)) - seen[description] = line_number - class TestDescriptionExplorer: """An iterator over test cases with descriptions. @@ -140,9 +109,21 @@ state may override this method. self.process_test_case(descriptions, file_name, line_number, description) + @staticmethod + def collect_test_directories(): + """Get the relative path for the TLS and Crypto test directories.""" + if os.path.isdir('tests'): + tests_dir = 'tests' + elif os.path.isdir('suites'): + tests_dir = '.' + elif os.path.isdir('../suites'): + tests_dir = '..' + directories = [tests_dir] + return directories + def walk_all(self): """Iterate over all named test cases.""" - test_directories = collect_test_directories() + test_directories = self.collect_test_directories() for directory in test_directories: for data_file_name in glob.glob(os.path.join(directory, 'suites', '*.data')): @@ -162,12 +143,31 @@ class DescriptionChecker(TestDescriptionExplorer): self.results = results def per_file_state(self): + """Dictionary mapping descriptions to their line number.""" return {} def process_test_case(self, per_file_state, file_name, line_number, description): - check_description(self.results, per_file_state, - file_name, line_number, description) + """Check test case descriptions for errors.""" + results = self.results + seen = per_file_state + if description in seen: + results.error(file_name, line_number, + 'Duplicate description (also line {})', + seen[description]) + return + if re.search(br'[\t;]', description): + results.error(file_name, line_number, + 'Forbidden character \'{}\' in description', + re.search(br'[\t;]', description).group(0).decode('ascii')) + if re.search(br'[^ -~]', description): + results.error(file_name, line_number, + 'Non-ASCII character in description') + if len(description) > 66: + results.warning(file_name, line_number, + 'Test description too long ({} > 66)', + len(description)) + seen[description] = line_number def main(): parser = argparse.ArgumentParser(description=__doc__) From 15c2cbfed51f8d24b1c18a0eff4fa52415374a2f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 25 Jun 2020 18:36:28 +0200 Subject: [PATCH 6/9] New script for test outcome analysis This is a new script designed to analyze test outcomes collected during a whole CI run. This commit introduces the script, the code to read the outcome file, and a very simple framework to report errors. It does not perform any actual analysis yet. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 93 +++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100755 tests/scripts/analyze_outcomes.py diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py new file mode 100755 index 0000000000..9d011db55c --- /dev/null +++ b/tests/scripts/analyze_outcomes.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 + +"""Analyze the test outcomes from a full CI run. + +This script can also run on outcomes from a partial run, but the results are +less likely to be useful. +""" + +import argparse +import sys +import traceback + +class Results: + """Process analysis results.""" + + def __init__(self): + self.error_count = 0 + self.warning_count = 0 + + @staticmethod + def log(fmt, *args, **kwargs): + sys.stderr.write((fmt + '\n').format(*args, **kwargs)) + + def error(self, fmt, *args, **kwargs): + self.log('Error: ' + fmt, *args, **kwargs) + self.error_count += 1 + + def warning(self, fmt, *args, **kwargs): + self.log('Warning: ' + fmt, *args, **kwargs) + self.warning_count += 1 + +class TestCaseOutcomes: + """The outcomes of one test case across many configurations.""" + # pylint: disable=too-few-public-methods + + def __init__(self): + self.successes = [] + self.failures = [] + + def hits(self): + """Return the number of times a test case has been run. + + This includes passes and failures, but not skips. + """ + return len(self.successes) + len(self.failures) + +def analyze_outcomes(outcomes): + """Run all analyses on the given outcome collection.""" + results = Results() + return results + +def read_outcome_file(outcome_file): + """Parse an outcome file and return an outcome collection. + +An outcome collection is a dictionary mapping keys to TestCaseOutcomes objects. +The keys are the test suite name and the test case description, separated +by a semicolon. +""" + outcomes = {} + with open(outcome_file, 'r', encoding='utf-8') as input_file: + for line in input_file: + (platform, config, suite, case, result, _cause) = line.split(';') + key = ';'.join([suite, case]) + setup = ';'.join([platform, config]) + if key not in outcomes: + outcomes[key] = TestCaseOutcomes() + if result == 'PASS': + outcomes[key].successes.append(setup) + elif result == 'FAIL': + outcomes[key].failures.append(setup) + return outcomes + +def analyze_outcome_file(outcome_file): + """Analyze the given outcome file.""" + outcomes = read_outcome_file(outcome_file) + return analyze_outcomes(outcomes) + +def main(): + try: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('outcomes', metavar='OUTCOMES.CSV', + help='Outcome file to analyze') + options = parser.parse_args() + results = analyze_outcome_file(options.outcomes) + if results.error_count > 0: + sys.exit(1) + except Exception: # pylint: disable=broad-except + # Print the backtrace and exit explicitly with our chosen status. + traceback.print_exc() + sys.exit(120) + +if __name__ == '__main__': + main() From 8d3c70a279917c87d46845fb909032ce5351874d Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 25 Jun 2020 18:37:43 +0200 Subject: [PATCH 7/9] Check test case coverage Check that every available test case in the test suites and ssl-opt.sh has been executed at least once. For the time being, only report a warning, because our coverage is incomplete. Once we've updated all.sh to have full coverage, this warning should become an error. Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 9d011db55c..96599bd530 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -7,9 +7,12 @@ less likely to be useful. """ import argparse +import re import sys import traceback +import check_test_cases + class Results: """Process analysis results.""" @@ -44,9 +47,40 @@ class TestCaseOutcomes: """ return len(self.successes) + len(self.failures) +class TestDescriptions(check_test_cases.TestDescriptionExplorer): + """Collect the available test cases.""" + + def __init__(self): + super().__init__() + self.descriptions = set() + + def process_test_case(self, _per_file_state, + file_name, _line_number, description): + """Record an available test case.""" + base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name)) + key = ';'.join([base_name, description.decode('utf-8')]) + self.descriptions.add(key) + +def collect_available_test_cases(): + """Collect the available test cases.""" + explorer = TestDescriptions() + explorer.walk_all() + return sorted(explorer.descriptions) + +def analyze_coverage(results, outcomes): + """Check that all available test cases are executed at least once.""" + available = collect_available_test_cases() + for key in available: + hits = outcomes[key].hits() if key in outcomes else 0 + if hits == 0: + # Make this a warning, not an error, as long as we haven't + # fixed this branch to have full coverage of test cases. + results.warning('Test case not executed: {}', key) + def analyze_outcomes(outcomes): """Run all analyses on the given outcome collection.""" results = Results() + analyze_coverage(results, outcomes) return results def read_outcome_file(outcome_file): From 3d863f263136525d62d7617f34038139bc87b153 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 26 Jun 2020 13:02:30 +0200 Subject: [PATCH 8/9] Document the fields of TestCasesOutcomes Signed-off-by: Gilles Peskine --- tests/scripts/analyze_outcomes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 96599bd530..73f16bdb25 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -37,6 +37,10 @@ class TestCaseOutcomes: # pylint: disable=too-few-public-methods def __init__(self): + # Collect a list of witnesses of the test case succeeding or failing. + # Currently we don't do anything with witnesses except count them. + # The format of a witness is determined by the read_outcome_file + # function; it's the platform and configuration joined by ';'. self.successes = [] self.failures = [] From bbb36649578ff4a5c020db44b0127dd8ca093149 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 3 Jul 2020 00:30:12 +0200 Subject: [PATCH 9/9] Documentation improvements Signed-off-by: Gilles Peskine --- tests/scripts/check_test_cases.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/scripts/check_test_cases.py b/tests/scripts/check_test_cases.py index 2df4c7a68d..3360d2817e 100755 --- a/tests/scripts/check_test_cases.py +++ b/tests/scripts/check_test_cases.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 """Sanity checks for test data. + +This program contains a class for traversing test cases that can be used +independently of the checks. """ # Copyright (C) 2019, Arm Limited, All Rights Reserved @@ -60,14 +63,15 @@ the process_test_case method, and call walk_all(). file_name, line_number, description): """Process a test case. -per_file_state: a new object returned by per_file_state() for each file. +per_file_state: an object created by new_per_file_state() at the beginning + of each file. file_name: a relative path to the file containing the test case. line_number: the line number in the given file. description: the test case description as a byte string. """ raise NotImplementedError - def per_file_state(self): + def new_per_file_state(self): """Return a new per-file state object. The default per-file state object is None. Child classes that require per-file @@ -79,7 +83,7 @@ state may override this method. def walk_test_suite(self, data_file_name): """Iterate over the test cases in the given unit test data file.""" in_paragraph = False - descriptions = self.per_file_state() # pylint: disable=assignment-from-none + descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none with open(data_file_name, 'rb') as data_file: for line_number, line in enumerate(data_file, 1): line = line.rstrip(b'\r\n') @@ -96,7 +100,7 @@ state may override this method. def walk_ssl_opt_sh(self, file_name): """Iterate over the test cases in ssl-opt.sh or a file with a similar format.""" - descriptions = self.per_file_state() # pylint: disable=assignment-from-none + descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none with open(file_name, 'rb') as file_contents: for line_number, line in enumerate(file_contents, 1): # Assume that all run_test calls have the same simple form @@ -142,7 +146,7 @@ class DescriptionChecker(TestDescriptionExplorer): def __init__(self, results): self.results = results - def per_file_state(self): + def new_per_file_state(self): """Dictionary mapping descriptions to their line number.""" return {}