mirror of
https://github.com/Mbed-TLS/mbedtls.git
synced 2025-03-28 08:37:25 +00:00
12438969fc
This document is mainly about a plan that was driven by backward compatibility constraints that no longer exist in 4.0. Although some of it is still of interest to explain why 4.0 is the way it is (and more complicated than one would expect based on the APIs that are left in 4.0). But for this it should suffice to consult earlier versions and does not worth to maintain it. Signed-off-by: Janos Follath <janos.follath@arm.com>
140 lines
3.6 KiB
Bash
Executable File
140 lines
3.6 KiB
Bash
Executable File
#!/bin/sh
|
|
|
|
# This script runs tests before and after a PR and analyzes the results in
|
|
# order to highlight any difference in the set of tests skipped.
|
|
#
|
|
# It can be used to check for unintended consequences when making non-trivial
|
|
# changes to compile time guards: the sets of tests skipped in the default
|
|
# config and the full config must be the same before and after the PR.
|
|
#
|
|
# USAGE:
|
|
# - First, commit any uncommited changes. (Also, see warning below.)
|
|
# - Then launch --> [SKIP_SSL_OPT=1] docs/architecture/psa-migration/outcome-analysis.sh
|
|
# - SKIP_SSL_OPT=1 can optionally be set to skip ssl-opt.sh tests
|
|
#
|
|
# WARNING: this script checks out a commit other than the head of the current
|
|
# branch; it checks out the current branch again when running successfully,
|
|
# but while the script is running, or if it terminates early in error, you
|
|
# should be aware that you might be at a different commit than expected.
|
|
#
|
|
# NOTE: you can comment out parts that don't need to be re-done when
|
|
# re-running this script (for example "get numbers before this PR").
|
|
|
|
set -eu
|
|
|
|
: ${SKIP_SSL_OPT:=0}
|
|
|
|
cleanup() {
|
|
make clean
|
|
git checkout -- include/mbedtls/mbedtls_config.h tf-psa-crypto/include/psa/crypto_config.h
|
|
}
|
|
|
|
record() {
|
|
export MBEDTLS_TEST_OUTCOME_FILE="$PWD/outcome-$1.csv"
|
|
rm -f $MBEDTLS_TEST_OUTCOME_FILE
|
|
|
|
make check
|
|
|
|
if [ $SKIP_SSL_OPT -eq 0 ]; then
|
|
make -C programs ssl/ssl_server2 ssl/ssl_client2 \
|
|
test/udp_proxy test/query_compile_time_config
|
|
tests/ssl-opt.sh
|
|
fi
|
|
}
|
|
|
|
# save current HEAD.
|
|
# Note: this can optionally be updated to
|
|
# HEAD=$(git branch --show-current)
|
|
# when using a Git version above 2.22
|
|
HEAD=$(git rev-parse --abbrev-ref HEAD)
|
|
|
|
# get the numbers before this PR for default and full
|
|
cleanup
|
|
git checkout $(git merge-base HEAD development)
|
|
|
|
record "before-default"
|
|
|
|
cleanup
|
|
|
|
scripts/config.py full
|
|
record "before-full"
|
|
|
|
# get the numbers now for default and full
|
|
cleanup
|
|
git checkout $HEAD
|
|
|
|
record "after-default"
|
|
|
|
cleanup
|
|
|
|
scripts/config.py full
|
|
record "after-full"
|
|
|
|
cleanup
|
|
|
|
# analysis
|
|
|
|
populate_suites () {
|
|
SUITES=''
|
|
make generated_files >/dev/null
|
|
data_files=$(cd tests/suites && echo *.data)
|
|
for data in $data_files; do
|
|
suite=${data%.data}
|
|
SUITES="$SUITES $suite"
|
|
done
|
|
make neat
|
|
|
|
if [ $SKIP_SSL_OPT -eq 0 ]; then
|
|
SUITES="$SUITES ssl-opt"
|
|
extra_files=$(cd tests/opt-testcases && echo *.sh)
|
|
for extra in $extra_files; do
|
|
suite=${extra%.sh}
|
|
SUITES="$SUITES $suite"
|
|
done
|
|
fi
|
|
}
|
|
|
|
compare_suite () {
|
|
ref="outcome-$1.csv"
|
|
new="outcome-$2.csv"
|
|
suite="$3"
|
|
|
|
pattern_suite=";$suite;"
|
|
total=$(grep -c "$pattern_suite" "$ref")
|
|
sed_cmd="s/^.*$pattern_suite\(.*\);SKIP.*/\1/p"
|
|
sed -n "$sed_cmd" "$ref" > skipped-ref
|
|
sed -n "$sed_cmd" "$new" > skipped-new
|
|
nb_ref=$(wc -l <skipped-ref)
|
|
nb_new=$(wc -l <skipped-new)
|
|
|
|
name=${suite#test_suite_}
|
|
printf "%40s: total %4d; skipped %4d -> %4d\n" \
|
|
$name $total $nb_ref $nb_new
|
|
if diff skipped-ref skipped-new | grep '^> '; then
|
|
ret=1
|
|
else
|
|
ret=0
|
|
fi
|
|
rm skipped-ref skipped-new
|
|
return $ret
|
|
}
|
|
|
|
compare_builds () {
|
|
printf "\n*** Comparing $1 -> $2 ***\n"
|
|
failed=''
|
|
for suite in $SUITES; do
|
|
if compare_suite "$1" "$2" "$suite"; then :; else
|
|
failed="$failed $suite"
|
|
fi
|
|
done
|
|
if [ -z "$failed" ]; then
|
|
printf "No coverage gap found.\n"
|
|
else
|
|
printf "Suites with less coverage:%s\n" "$failed"
|
|
fi
|
|
}
|
|
|
|
populate_suites
|
|
compare_builds before-default after-default
|
|
compare_builds before-full after-full
|