mirror/mbedtls
1
0
Fork 0
mirror of https://github.com/Mbed-TLS/mbedtls.git synced 2025-03-06 19:13:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #9941 from valeriosetti/issue94-3.6

Browse Source 
[Backport 3.6] Move test_psa_*.py scripts to the framework
This commit is contained in:
Ronald Cron 2025-02-05 13:55:44 +00:00 committed by GitHub
commit c811fb79ad
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 3 additions and 353 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
framework

@ -1 +1 @@
Subproject commit 8296a73ce0cb31fadf411b6929a3201beece37a5
Subproject commit 2000db429553aa38e5875c621daf32aa8b63c340

2
tests/scripts/components-compliance.sh
View File

@ -15,7 +15,7 @@ component_test_psa_compliance () {
CC=gcc make -C library libmbedcrypto.a
msg "unit test: test_psa_compliance.py"
CC=gcc ./tests/scripts/test_psa_compliance.py
CC=gcc $FRAMEWORK/scripts/test_psa_compliance.py
}
support_test_psa_compliance () {

2
tests/scripts/components-configuration.sh
View File

@ -148,7 +148,7 @@ component_test_full_cmake_clang () {
tests/scripts/run_demos.py
msg "test: psa_constant_names (full config, clang)" # ~ 1s
tests/scripts/test_psa_constant_names.py
$FRAMEWORK/scripts/test_psa_constant_names.py
msg "test: ssl-opt.sh default, ECJPAKE, SSL async (full config)" # ~ 1s
tests/ssl-opt.sh -f 'Default\|ECJPAKE\|SSL async private'

159
tests/scripts/test_psa_compliance.py
View File

@ -1,159 +0,0 @@
#!/usr/bin/env python3
"""Run the PSA Crypto API compliance test suite.
Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF,
then compile and run the test suite. The clone is stored at <repository root>/psa-arch-tests.
Known defects in either the test suite or mbedtls / TF-PSA-Crypto - identified by their test
number - are ignored, while unexpected failures AND successes are reported as errors, to help
keep the list of known defects as up to date as possible.
"""
# Copyright The Mbed TLS Contributors
# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
import argparse
import os
import re
import shutil
import subprocess
import sys
from typing import List
#pylint: disable=unused-import
import scripts_path
from mbedtls_framework import build_tree
# PSA Compliance tests we expect to fail due to known defects in Mbed TLS /
# TF-PSA-Crypto (or the test suite).
# The test numbers correspond to the numbers used by the console output of the test suite.
# Test number 2xx corresponds to the files in the folder
# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx
EXPECTED_FAILURES = {} # type: dict
PSA_ARCH_TESTS_REPO = 'https://github.com/ARM-software/psa-arch-tests.git'
PSA_ARCH_TESTS_REF = 'v23.06_API1.5_ADAC_EAC'
#pylint: disable=too-many-branches,too-many-statements,too-many-locals
def main(library_build_dir: str):
root_dir = os.getcwd()
in_tf_psa_crypto_repo = build_tree.looks_like_tf_psa_crypto_root(root_dir)
crypto_name = build_tree.crypto_library_filename(root_dir)
library_subdir = build_tree.crypto_core_directory(root_dir, relative=True)
crypto_lib_filename = (library_build_dir + '/' +
library_subdir + '/' +
'lib' + crypto_name + '.a')
if not os.path.exists(crypto_lib_filename):
#pylint: disable=bad-continuation
subprocess.check_call([
'cmake', '.',
'-GUnix Makefiles',
'-B' + library_build_dir
])
subprocess.check_call(['cmake', '--build', library_build_dir,
'--target', crypto_name])
psa_arch_tests_dir = 'psa-arch-tests'
os.makedirs(psa_arch_tests_dir, exist_ok=True)
try:
os.chdir(psa_arch_tests_dir)
# Reuse existing local clone
subprocess.check_call(['git', 'init'])
subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF])
subprocess.check_call(['git', 'checkout', 'FETCH_HEAD'])
build_dir = 'api-tests/build'
try:
shutil.rmtree(build_dir)
except FileNotFoundError:
pass
os.mkdir(build_dir)
os.chdir(build_dir)
extra_includes = (';{}/drivers/builtin/include'.format(root_dir)
if in_tf_psa_crypto_repo else '')
#pylint: disable=bad-continuation
subprocess.check_call([
'cmake', '..',
'-GUnix Makefiles',
'-DTARGET=tgt_dev_apis_stdc',
'-DTOOLCHAIN=HOST_GCC',
'-DSUITE=CRYPTO',
'-DPSA_CRYPTO_LIB_FILENAME={}/{}'.format(root_dir,
crypto_lib_filename),
('-DPSA_INCLUDE_PATHS={}/include' + extra_includes).format(root_dir)
])
subprocess.check_call(['cmake', '--build', '.'])
proc = subprocess.Popen(['./psa-arch-tests-crypto'],
bufsize=1, stdout=subprocess.PIPE, universal_newlines=True)
test_re = re.compile(
'^TEST: (?P<test_num>[0-9]*)|'
'^TEST RESULT: (?P<test_result>FAILED|PASSED)'
)
test = -1
unexpected_successes = set(EXPECTED_FAILURES)
expected_failures = [] # type: List[int]
unexpected_failures = [] # type: List[int]
if proc.stdout is None:
return 1
for line in proc.stdout:
print(line, end='')
match = test_re.match(line)
if match is not None:
groupdict = match.groupdict()
test_num = groupdict['test_num']
if test_num is not None:
test = int(test_num)
elif groupdict['test_result'] == 'FAILED':
try:
unexpected_successes.remove(test)
expected_failures.append(test)
print('Expected failure, ignoring')
except KeyError:
unexpected_failures.append(test)
print('ERROR: Unexpected failure')
elif test in unexpected_successes:
print('ERROR: Unexpected success')
proc.wait()
print()
print('***** test_psa_compliance.py report ******')
print()
print('Expected failures:', ', '.join(str(i) for i in expected_failures))
print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures))
print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes)))
print()
if unexpected_successes or unexpected_failures:
if unexpected_successes:
print('Unexpected successes encountered.')
print('Please remove the corresponding tests from '
'EXPECTED_FAILURES in tests/scripts/compliance_test.py')
print()
print('FAILED')
return 1
else:
print('SUCCESS')
return 0
finally:
os.chdir(root_dir)
if __name__ == '__main__':
BUILD_DIR = 'out_of_source_build'
# pylint: disable=invalid-name
parser = argparse.ArgumentParser()
parser.add_argument('--build-dir', nargs=1,
help='path to Mbed TLS / TF-PSA-Crypto build directory')
args = parser.parse_args()
if args.build_dir is not None:
BUILD_DIR = args.build_dir[0]
sys.exit(main(BUILD_DIR))

191
tests/scripts/test_psa_constant_names.py
View File

@ -1,191 +0,0 @@
#!/usr/bin/env python3
"""Test the program psa_constant_names.
Gather constant names from header files and test cases. Compile a C program
to print out their numerical values, feed these numerical values to
psa_constant_names, and check that the output is the original name.
Return 0 if all test cases pass, 1 if the output was not always as expected,
or 1 (with a Python backtrace) if there was an operational error.
"""
# Copyright The Mbed TLS Contributors
# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
import argparse
from collections import namedtuple
import os
import re
import subprocess
import sys
from typing import Iterable, List, Optional, Tuple
import scripts_path # pylint: disable=unused-import
from mbedtls_framework import c_build_helper
from mbedtls_framework.macro_collector import InputsForTest, PSAMacroEnumerator
from mbedtls_framework import typing_util
def gather_inputs(headers: Iterable[str],
test_suites: Iterable[str],
inputs_class=InputsForTest) -> PSAMacroEnumerator:
"""Read the list of inputs to test psa_constant_names with."""
inputs = inputs_class()
for header in headers:
inputs.parse_header(header)
for test_cases in test_suites:
inputs.parse_test_cases(test_cases)
inputs.add_numerical_values()
inputs.gather_arguments()
return inputs
def run_c(type_word: str,
expressions: Iterable[str],
include_path: Optional[str] = None,
keep_c: bool = False) -> List[str]:
"""Generate and run a program to print out numerical values of C expressions."""
if type_word == 'status':
cast_to = 'long'
printf_format = '%ld'
else:
cast_to = 'unsigned long'
printf_format = '0x%08lx'
return c_build_helper.get_c_expression_values(
cast_to, printf_format,
expressions,
caller='test_psa_constant_names.py for {} values'.format(type_word),
file_label=type_word,
header='#include <psa/crypto.h>',
include_path=include_path,
keep_c=keep_c
)
NORMALIZE_STRIP_RE = re.compile(r'\s+')
def normalize(expr: str) -> str:
"""Normalize the C expression so as not to care about trivial differences.
Currently "trivial differences" means whitespace.
"""
return re.sub(NORMALIZE_STRIP_RE, '', expr)
ALG_TRUNCATED_TO_SELF_RE = \
re.compile(r'PSA_ALG_AEAD_WITH_SHORTENED_TAG\('
r'PSA_ALG_(?:CCM|CHACHA20_POLY1305|GCM)'
r', *16\)\Z')
def is_simplifiable(expr: str) -> bool:
"""Determine whether an expression is simplifiable.
Simplifiable expressions can't be output in their input form, since
the output will be the simple form. Therefore they must be excluded
from testing.
"""
if ALG_TRUNCATED_TO_SELF_RE.match(expr):
return True
return False
def collect_values(inputs: InputsForTest,
type_word: str,
include_path: Optional[str] = None,
keep_c: bool = False) -> Tuple[List[str], List[str]]:
"""Generate expressions using known macro names and calculate their values.
Return a list of pairs of (expr, value) where expr is an expression and
value is a string representation of its integer value.
"""
names = inputs.get_names(type_word)
expressions = sorted(expr
for expr in inputs.generate_expressions(names)
if not is_simplifiable(expr))
values = run_c(type_word, expressions,
include_path=include_path, keep_c=keep_c)
return expressions, values
class Tests:
"""An object representing tests and their results."""
Error = namedtuple('Error',
['type', 'expression', 'value', 'output'])
def __init__(self, options) -> None:
self.options = options
self.count = 0
self.errors = [] #type: List[Tests.Error]
def run_one(self, inputs: InputsForTest, type_word: str) -> None:
"""Test psa_constant_names for the specified type.
Run the program on the names for this type.
Use the inputs to figure out what arguments to pass to macros that
take arguments.
"""
expressions, values = collect_values(inputs, type_word,
include_path=self.options.include,
keep_c=self.options.keep_c)
output_bytes = subprocess.check_output([self.options.program,
type_word] + values)
output = output_bytes.decode('ascii')
outputs = output.strip().split('\n')
self.count += len(expressions)
for expr, value, output in zip(expressions, values, outputs):
if self.options.show:
sys.stdout.write('{} {}\t{}\n'.format(type_word, value, output))
if normalize(expr) != normalize(output):
self.errors.append(self.Error(type=type_word,
expression=expr,
value=value,
output=output))
def run_all(self, inputs: InputsForTest) -> None:
"""Run psa_constant_names on all the gathered inputs."""
for type_word in ['status', 'algorithm', 'ecc_curve', 'dh_group',
'key_type', 'key_usage']:
self.run_one(inputs, type_word)
def report(self, out: typing_util.Writable) -> None:
"""Describe each case where the output is not as expected.
Write the errors to ``out``.
Also write a total.
"""
for error in self.errors:
out.write('For {} "{}", got "{}" (value: {})\n'
.format(error.type, error.expression,
error.output, error.value))
out.write('{} test cases'.format(self.count))
if self.errors:
out.write(', {} FAIL\n'.format(len(self.errors)))
else:
out.write(' PASS\n')
HEADERS = ['psa/crypto.h', 'psa/crypto_extra.h', 'psa/crypto_values.h']
TEST_SUITES = ['tests/suites/test_suite_psa_crypto_metadata.data']
def main():
parser = argparse.ArgumentParser(description=globals()['__doc__'])
parser.add_argument('--include', '-I',
action='append', default=['include'],
help='Directory for header files')
parser.add_argument('--keep-c',
action='store_true', dest='keep_c', default=False,
help='Keep the intermediate C file')
parser.add_argument('--no-keep-c',
action='store_false', dest='keep_c',
help='Don\'t keep the intermediate C file (default)')
parser.add_argument('--program',
default='programs/psa/psa_constant_names',
help='Program to test')
parser.add_argument('--show',
action='store_true',
help='Show tested values on stdout')
parser.add_argument('--no-show',
action='store_false', dest='show',
help='Don\'t show tested values (default)')
options = parser.parse_args()
headers = [os.path.join(options.include[0], h) for h in HEADERS]
inputs = gather_inputs(headers, TEST_SUITES)
tests = Tests(options)
tests.run_all(inputs)
tests.report(sys.stdout)
if tests.errors:
sys.exit(1)
if __name__ == '__main__':
main()