From 7297e8d4406d36cfbf20431f3e6882889bcfd242 Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Wed, 29 Jan 2025 15:12:09 +0100 Subject: [PATCH 1/4] Move files out of Mbed TLS The following files are moved to the framework repo (deleted here): tests/scripts/test_psa_compliance.py tests/scripts/test_psa_constant_names.py Signed-off-by: Valerio Setti --- tests/scripts/test_psa_compliance.py | 159 ------------------- tests/scripts/test_psa_constant_names.py | 191 ----------------------- 2 files changed, 350 deletions(-) delete mode 100755 tests/scripts/test_psa_compliance.py delete mode 100755 tests/scripts/test_psa_constant_names.py diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py deleted file mode 100755 index f7d18954ca..0000000000 --- a/tests/scripts/test_psa_compliance.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python3 -"""Run the PSA Crypto API compliance test suite. -Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF, -then compile and run the test suite. The clone is stored at /psa-arch-tests. -Known defects in either the test suite or mbedtls / TF-PSA-Crypto - identified by their test -number - are ignored, while unexpected failures AND successes are reported as errors, to help -keep the list of known defects as up to date as possible. -""" - -# Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - -import argparse -import os -import re -import shutil -import subprocess -import sys -from typing import List - -#pylint: disable=unused-import -import scripts_path -from mbedtls_framework import build_tree - -# PSA Compliance tests we expect to fail due to known defects in Mbed TLS / -# TF-PSA-Crypto (or the test suite). -# The test numbers correspond to the numbers used by the console output of the test suite. -# Test number 2xx corresponds to the files in the folder -# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx -EXPECTED_FAILURES = {} # type: dict - -PSA_ARCH_TESTS_REPO = 'https://github.com/ARM-software/psa-arch-tests.git' -PSA_ARCH_TESTS_REF = 'v23.06_API1.5_ADAC_EAC' - -#pylint: disable=too-many-branches,too-many-statements,too-many-locals -def main(library_build_dir: str): - root_dir = os.getcwd() - - in_tf_psa_crypto_repo = build_tree.looks_like_tf_psa_crypto_root(root_dir) - - crypto_name = build_tree.crypto_library_filename(root_dir) - library_subdir = build_tree.crypto_core_directory(root_dir, relative=True) - - crypto_lib_filename = (library_build_dir + '/' + - library_subdir + '/' + - 'lib' + crypto_name + '.a') - - if not os.path.exists(crypto_lib_filename): - #pylint: disable=bad-continuation - subprocess.check_call([ - 'cmake', '.', - '-GUnix Makefiles', - '-B' + library_build_dir - ]) - subprocess.check_call(['cmake', '--build', library_build_dir, - '--target', crypto_name]) - - psa_arch_tests_dir = 'psa-arch-tests' - os.makedirs(psa_arch_tests_dir, exist_ok=True) - try: - os.chdir(psa_arch_tests_dir) - - # Reuse existing local clone - subprocess.check_call(['git', 'init']) - subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) - subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) - - build_dir = 'api-tests/build' - try: - shutil.rmtree(build_dir) - except FileNotFoundError: - pass - os.mkdir(build_dir) - os.chdir(build_dir) - - extra_includes = (';{}/drivers/builtin/include'.format(root_dir) - if in_tf_psa_crypto_repo else '') - - #pylint: disable=bad-continuation - subprocess.check_call([ - 'cmake', '..', - '-GUnix Makefiles', - '-DTARGET=tgt_dev_apis_stdc', - '-DTOOLCHAIN=HOST_GCC', - '-DSUITE=CRYPTO', - '-DPSA_CRYPTO_LIB_FILENAME={}/{}'.format(root_dir, - crypto_lib_filename), - ('-DPSA_INCLUDE_PATHS={}/include' + extra_includes).format(root_dir) - ]) - subprocess.check_call(['cmake', '--build', '.']) - - proc = subprocess.Popen(['./psa-arch-tests-crypto'], - bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) - - test_re = re.compile( - '^TEST: (?P[0-9]*)|' - '^TEST RESULT: (?PFAILED|PASSED)' - ) - test = -1 - unexpected_successes = set(EXPECTED_FAILURES) - expected_failures = [] # type: List[int] - unexpected_failures = [] # type: List[int] - if proc.stdout is None: - return 1 - - for line in proc.stdout: - print(line, end='') - match = test_re.match(line) - if match is not None: - groupdict = match.groupdict() - test_num = groupdict['test_num'] - if test_num is not None: - test = int(test_num) - elif groupdict['test_result'] == 'FAILED': - try: - unexpected_successes.remove(test) - expected_failures.append(test) - print('Expected failure, ignoring') - except KeyError: - unexpected_failures.append(test) - print('ERROR: Unexpected failure') - elif test in unexpected_successes: - print('ERROR: Unexpected success') - proc.wait() - - print() - print('***** test_psa_compliance.py report ******') - print() - print('Expected failures:', ', '.join(str(i) for i in expected_failures)) - print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) - print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) - print() - if unexpected_successes or unexpected_failures: - if unexpected_successes: - print('Unexpected successes encountered.') - print('Please remove the corresponding tests from ' - 'EXPECTED_FAILURES in tests/scripts/compliance_test.py') - print() - print('FAILED') - return 1 - else: - print('SUCCESS') - return 0 - finally: - os.chdir(root_dir) - -if __name__ == '__main__': - BUILD_DIR = 'out_of_source_build' - - # pylint: disable=invalid-name - parser = argparse.ArgumentParser() - parser.add_argument('--build-dir', nargs=1, - help='path to Mbed TLS / TF-PSA-Crypto build directory') - args = parser.parse_args() - - if args.build_dir is not None: - BUILD_DIR = args.build_dir[0] - - sys.exit(main(BUILD_DIR)) diff --git a/tests/scripts/test_psa_constant_names.py b/tests/scripts/test_psa_constant_names.py deleted file mode 100755 index 86d9e6f2be..0000000000 --- a/tests/scripts/test_psa_constant_names.py +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env python3 -"""Test the program psa_constant_names. -Gather constant names from header files and test cases. Compile a C program -to print out their numerical values, feed these numerical values to -psa_constant_names, and check that the output is the original name. -Return 0 if all test cases pass, 1 if the output was not always as expected, -or 1 (with a Python backtrace) if there was an operational error. -""" - -# Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - -import argparse -from collections import namedtuple -import os -import re -import subprocess -import sys -from typing import Iterable, List, Optional, Tuple - -import scripts_path # pylint: disable=unused-import -from mbedtls_framework import c_build_helper -from mbedtls_framework.macro_collector import InputsForTest, PSAMacroEnumerator -from mbedtls_framework import typing_util - -def gather_inputs(headers: Iterable[str], - test_suites: Iterable[str], - inputs_class=InputsForTest) -> PSAMacroEnumerator: - """Read the list of inputs to test psa_constant_names with.""" - inputs = inputs_class() - for header in headers: - inputs.parse_header(header) - for test_cases in test_suites: - inputs.parse_test_cases(test_cases) - inputs.add_numerical_values() - inputs.gather_arguments() - return inputs - -def run_c(type_word: str, - expressions: Iterable[str], - include_path: Optional[str] = None, - keep_c: bool = False) -> List[str]: - """Generate and run a program to print out numerical values of C expressions.""" - if type_word == 'status': - cast_to = 'long' - printf_format = '%ld' - else: - cast_to = 'unsigned long' - printf_format = '0x%08lx' - return c_build_helper.get_c_expression_values( - cast_to, printf_format, - expressions, - caller='test_psa_constant_names.py for {} values'.format(type_word), - file_label=type_word, - header='#include ', - include_path=include_path, - keep_c=keep_c - ) - -NORMALIZE_STRIP_RE = re.compile(r'\s+') -def normalize(expr: str) -> str: - """Normalize the C expression so as not to care about trivial differences. - - Currently "trivial differences" means whitespace. - """ - return re.sub(NORMALIZE_STRIP_RE, '', expr) - -ALG_TRUNCATED_TO_SELF_RE = \ - re.compile(r'PSA_ALG_AEAD_WITH_SHORTENED_TAG\(' - r'PSA_ALG_(?:CCM|CHACHA20_POLY1305|GCM)' - r', *16\)\Z') - -def is_simplifiable(expr: str) -> bool: - """Determine whether an expression is simplifiable. - - Simplifiable expressions can't be output in their input form, since - the output will be the simple form. Therefore they must be excluded - from testing. - """ - if ALG_TRUNCATED_TO_SELF_RE.match(expr): - return True - return False - -def collect_values(inputs: InputsForTest, - type_word: str, - include_path: Optional[str] = None, - keep_c: bool = False) -> Tuple[List[str], List[str]]: - """Generate expressions using known macro names and calculate their values. - - Return a list of pairs of (expr, value) where expr is an expression and - value is a string representation of its integer value. - """ - names = inputs.get_names(type_word) - expressions = sorted(expr - for expr in inputs.generate_expressions(names) - if not is_simplifiable(expr)) - values = run_c(type_word, expressions, - include_path=include_path, keep_c=keep_c) - return expressions, values - -class Tests: - """An object representing tests and their results.""" - - Error = namedtuple('Error', - ['type', 'expression', 'value', 'output']) - - def __init__(self, options) -> None: - self.options = options - self.count = 0 - self.errors = [] #type: List[Tests.Error] - - def run_one(self, inputs: InputsForTest, type_word: str) -> None: - """Test psa_constant_names for the specified type. - - Run the program on the names for this type. - Use the inputs to figure out what arguments to pass to macros that - take arguments. - """ - expressions, values = collect_values(inputs, type_word, - include_path=self.options.include, - keep_c=self.options.keep_c) - output_bytes = subprocess.check_output([self.options.program, - type_word] + values) - output = output_bytes.decode('ascii') - outputs = output.strip().split('\n') - self.count += len(expressions) - for expr, value, output in zip(expressions, values, outputs): - if self.options.show: - sys.stdout.write('{} {}\t{}\n'.format(type_word, value, output)) - if normalize(expr) != normalize(output): - self.errors.append(self.Error(type=type_word, - expression=expr, - value=value, - output=output)) - - def run_all(self, inputs: InputsForTest) -> None: - """Run psa_constant_names on all the gathered inputs.""" - for type_word in ['status', 'algorithm', 'ecc_curve', 'dh_group', - 'key_type', 'key_usage']: - self.run_one(inputs, type_word) - - def report(self, out: typing_util.Writable) -> None: - """Describe each case where the output is not as expected. - - Write the errors to ``out``. - Also write a total. - """ - for error in self.errors: - out.write('For {} "{}", got "{}" (value: {})\n' - .format(error.type, error.expression, - error.output, error.value)) - out.write('{} test cases'.format(self.count)) - if self.errors: - out.write(', {} FAIL\n'.format(len(self.errors))) - else: - out.write(' PASS\n') - -HEADERS = ['psa/crypto.h', 'psa/crypto_extra.h', 'psa/crypto_values.h'] -TEST_SUITES = ['tests/suites/test_suite_psa_crypto_metadata.data'] - -def main(): - parser = argparse.ArgumentParser(description=globals()['__doc__']) - parser.add_argument('--include', '-I', - action='append', default=['include'], - help='Directory for header files') - parser.add_argument('--keep-c', - action='store_true', dest='keep_c', default=False, - help='Keep the intermediate C file') - parser.add_argument('--no-keep-c', - action='store_false', dest='keep_c', - help='Don\'t keep the intermediate C file (default)') - parser.add_argument('--program', - default='programs/psa/psa_constant_names', - help='Program to test') - parser.add_argument('--show', - action='store_true', - help='Show tested values on stdout') - parser.add_argument('--no-show', - action='store_false', dest='show', - help='Don\'t show tested values (default)') - options = parser.parse_args() - headers = [os.path.join(options.include[0], h) for h in HEADERS] - inputs = gather_inputs(headers, TEST_SUITES) - tests = Tests(options) - tests.run_all(inputs) - tests.report(sys.stdout) - if tests.errors: - sys.exit(1) - -if __name__ == '__main__': - main() From d673acf89e7ab994eaa13694b2047d96248ce44a Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Wed, 29 Jan 2025 15:16:50 +0100 Subject: [PATCH 2/4] components-configuration.sh: update references to test_psa_constant_names.py Signed-off-by: Valerio Setti --- tests/scripts/components-configuration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/components-configuration.sh b/tests/scripts/components-configuration.sh index feedbcfedf..a581e6bffa 100644 --- a/tests/scripts/components-configuration.sh +++ b/tests/scripts/components-configuration.sh @@ -148,7 +148,7 @@ component_test_full_cmake_clang () { tests/scripts/run_demos.py msg "test: psa_constant_names (full config, clang)" # ~ 1s - tests/scripts/test_psa_constant_names.py + $FRAMEWORK/scripts/test_psa_constant_names.py msg "test: ssl-opt.sh default, ECJPAKE, SSL async (full config)" # ~ 1s tests/ssl-opt.sh -f 'Default\|ECJPAKE\|SSL async private' From aa7bd59dbe22dd6c9f0bf90f6a154f4246bde7ba Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Wed, 29 Jan 2025 15:17:26 +0100 Subject: [PATCH 3/4] components-compliance.sh: update references to test_psa_compliance.py Signed-off-by: Valerio Setti --- tests/scripts/components-compliance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/components-compliance.sh b/tests/scripts/components-compliance.sh index 38bcd01430..ec61b10c45 100644 --- a/tests/scripts/components-compliance.sh +++ b/tests/scripts/components-compliance.sh @@ -15,7 +15,7 @@ component_test_psa_compliance () { CC=gcc make -C library libmbedcrypto.a msg "unit test: test_psa_compliance.py" - CC=gcc ./tests/scripts/test_psa_compliance.py + CC=gcc $FRAMEWORK/scripts/test_psa_compliance.py } support_test_psa_compliance () { From 411f7dc1670084fccaf2e5d145e52494d34d2b90 Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Wed, 29 Jan 2025 15:32:45 +0100 Subject: [PATCH 4/4] framework: update reference Signed-off-by: Valerio Setti --- framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework b/framework index 8296a73ce0..2000db4295 160000 --- a/framework +++ b/framework @@ -1 +1 @@ -Subproject commit 8296a73ce0cb31fadf411b6929a3201beece37a5 +Subproject commit 2000db429553aa38e5875c621daf32aa8b63c340