diff --git a/scripts/mbedtls_dev/test_generation.py b/scripts/mbedtls_dev/test_generation.py index bb70b9c72e..712c7996bf 100644 --- a/scripts/mbedtls_dev/test_generation.py +++ b/scripts/mbedtls_dev/test_generation.py @@ -23,6 +23,8 @@ import argparse import os import posixpath import re + +from abc import abstractmethod from typing import Callable, Dict, Iterable, List, Type, TypeVar from mbedtls_dev import build_tree @@ -53,15 +55,34 @@ class BaseTarget: def __init__(self) -> None: type(self).count += 1 + @abstractmethod def arguments(self) -> List[str]: - return [] + """Get the list of arguments for the test case. + + Override this method to provide the list of arguments required for + generating the test_function. + + Returns: + List of arguments required for the test function. + """ + pass def description(self) -> str: - """Create a numbered test description.""" + """Create a test description. + + Creates a description of the test case, including a name for the test + function, and describing the specific test case. This should inform a + reader of the purpose of the case. The case description may be + generated in the class, or provided manually as needed. + + Returns: + Description for the test case. + """ return "{} #{} {}".format(self.test_name, self.count, self.case_description) + def create_test_case(self) -> test_case.TestCase: - """Generate test case from the current object.""" + """Generate TestCase from the current object.""" tc = test_case.TestCase() tc.set_description(self.description()) tc.set_function(self.test_function) @@ -71,7 +92,16 @@ class BaseTarget: @classmethod def generate_tests(cls): - """Generate test cases for the target subclasses.""" + """Generate test cases for the target subclasses. + + Classes will iterate over its subclasses, calling this method in each. + In abstract classes, no further changes are needed, as there is no + function to generate tests for. + In classes which do implement a test function, this should be overrided + and a means to use `create_test_case()` should be added. In most cases + the subclasses can still be iterated over, as either the class will + have none, or it may continue. + """ for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): yield from subclass.generate_tests() diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py index 471fd77245..7a8ebd1d8a 100755 --- a/tests/scripts/generate_bignum_tests.py +++ b/tests/scripts/generate_bignum_tests.py @@ -3,6 +3,31 @@ With no arguments, generate all test data. With non-option arguments, generate only the specified files. + +Class structure: + +Target classes are directly derived from test_generation.BaseTarget, +representing a target file. These indicate where test cases will be written +to in classes derived from the Target. Multiple Target classes must not +represent the same target_basename. + +Each subclass derived from a Target can either be: + - A concrete class, representing a test function, which generates test cases. + - An abstract class containing shared methods and attributes, not associated + with a test function. An example is BignumOperation, which provides common + features used in binary bignum operations. + + +Adding test generation for a function: + +A subclass representing the test function should be added, deriving from a +Target class or a descendant. This subclass must set/implement the following: + - test_function: the function name from the associated .function file. + - arguments(): generation of the arguments required for the test_function. + - generate_function_test(): generation of the test cases for the function. + +Additional details and other attributes/methods are given in the documentation +of BaseTarget in test_generation.py. """ # Copyright The Mbed TLS Contributors @@ -22,6 +47,8 @@ generate only the specified files. import itertools import sys + +from abc import abstractmethod from typing import Callable, Dict, Iterator, List, Optional, Tuple, TypeVar import scripts_path # pylint: disable=unused-import @@ -43,11 +70,16 @@ class BignumTarget(test_generation.BaseTarget): class BignumOperation(BignumTarget): - """Common features for test cases covering bignum operations. + """Common features for test cases covering binary bignum operations. + + This adds functionality common in binary operation tests. This includes + generation of case descriptions, using descriptions of values and symbols + to represent the operation or result. Attributes: - symbol: Symbol used for operation in description. - input_values: List of values to use as test case inputs. + symbol: Symbol used for the operation in case description. + input_values: List of values to use as test case inputs. These are + combined to produce pairs of values. input_cases: List of tuples containing pairs of test case inputs. This can be used to implement specific pairs of inputs. """ @@ -71,6 +103,12 @@ class BignumOperation(BignumTarget): return [quote_str(self.arg_l), quote_str(self.arg_r), self.result()] def description(self): + """Generate a description for the test case. + + If not set, case_description uses the form A `symbol` B, where symbol + is used to represent the operation. Descriptions of each value are + generated to provide some context to the test case. + """ if not self.case_description: self.case_description = "{} {} {}".format( self.value_description(self.arg_l), @@ -79,11 +117,22 @@ class BignumOperation(BignumTarget): ) return super().description() + @abstractmethod def result(self) -> Optional[str]: - return None + """Get the result of the operation. + + This may be calculated during initialization and stored as `_result`, + or calculated when the method is called. + """ + pass @staticmethod def value_description(val) -> str: + """Generate a description of the argument val. + + This produces a simple description of the value, which are used in test + case naming, to avoid most generated cases only being numbered. + """ if val == "": return "0 (null)" if val == "0": @@ -102,7 +151,11 @@ class BignumOperation(BignumTarget): @classmethod def get_value_pairs(cls) -> Iterator[Tuple[str, ...]]: - """Generate value pairs.""" + """Generator for pairs of inputs. + + Combinations are first generated from all input values, and then + specific cases provided. + """ yield from itertools.combinations(cls.input_values, 2) yield from cls.input_cases @@ -139,7 +192,7 @@ class BignumCmp(BignumOperation): class BignumCmpAbs(BignumCmp): - """Target for abs comparison variant.""" + """Target for bignum comparison, absolute variant.""" count = 0 test_function = "mbedtls_mpi_cmp_abs" test_name = "MPI compare (abs)"