From 39c5207d79c640a7ee1d75af59ab57890b46172c Mon Sep 17 00:00:00 2001
From: Gilles Peskine <Gilles.Peskine@arm.com>
Date: Fri, 17 May 2024 11:55:15 +0200
Subject: [PATCH] ssl-opt.sh, compat.sh: Error out if not executing any tests

Alert if all tests are filtered out or skipped: that probably indicates a
test script that set up an unintended configuration or an overly strict
filter. You can pass `--min 0` to bypass this check. You can pass `--min`
with a larger value to require that many test cases to run.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
---
 tests/compat.sh  | 15 +++++++++++++++
 tests/ssl-opt.sh | 14 ++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/tests/compat.sh b/tests/compat.sh
index 20f2dbda61..073258ed1b 100755
--- a/tests/compat.sh
+++ b/tests/compat.sh
@@ -96,6 +96,7 @@ FILTER=""
 EXCLUDE='NULL\|ARIA\|CHACHA20_POLY1305'
 VERBOSE=""
 MEMCHECK=0
+MIN_TESTS=1
 PRESERVE_LOGS=0
 PEERS="OpenSSL$PEER_GNUTLS mbedTLS"
 
@@ -116,6 +117,7 @@ print_usage() {
     printf "  -M|--memcheck\tCheck memory leaks and errors.\n"
     printf "  -v|--verbose\tSet verbose output.\n"
     printf "     --list-test-cases\tList all potential test cases (No Execution)\n"
+    printf "     --min      \tMinimum number of non-skipped tests (default 1)\n"
     printf "     --outcome-file\tFile where test outcomes are written\n"
     printf "                   \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n"
     printf "     --preserve-logs\tPreserve logs of successful tests as well\n"
@@ -190,6 +192,9 @@ get_options() {
                 list_test_cases
                 exit $?
                 ;;
+            --min)
+                shift; MIN_TESTS=$1
+                ;;
             --outcome-file)
                 shift; MBEDTLS_TEST_OUTCOME_FILE=$1
                 ;;
@@ -1238,6 +1243,16 @@ fi
 PASSED=$(( $TESTS - $FAILED ))
 echo " ($PASSED / $TESTS tests ($SKIPPED skipped$MEMREPORT))"
 
+if [ $((TESTS - SKIPPED)) -lt $MIN_TESTS ]; then
+    cat <<EOF
+Error: Expected to run at least $MIN_TESTS, but only ran $((TESTS - SKIPPED)).
+Maybe a bad filter ('$FILTER' excluding '$EXCLUDE') or a bad configuration?
+EOF
+    if [ $FAILED -eq 0 ]; then
+        FAILED=1
+    fi
+fi
+
 FAILED=$(( $FAILED + $SRVMEM ))
 if [ $FAILED -gt 255 ]; then
     # Clamp at 255 as caller gets exit code & 0xFF
diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh
index f4c7f6d49f..646daad199 100755
--- a/tests/ssl-opt.sh
+++ b/tests/ssl-opt.sh
@@ -141,6 +141,7 @@ LIST_TESTS=0
 RUN_TEST_NUMBER=''
 RUN_TEST_SUITE=''
 
+MIN_TESTS=1
 PRESERVE_LOGS=0
 
 # Pick a "unique" server port in the range 10000-19999, and a proxy
@@ -159,6 +160,7 @@ print_usage() {
     printf "  -s|--show-numbers\tShow test numbers in front of test names\n"
     printf "  -p|--preserve-logs\tPreserve logs of successful tests as well\n"
     printf "     --list-test-cases\tList all potential test cases (No Execution)\n"
+    printf "     --min      \tMinimum number of non-skipped tests (default 1)\n"
     printf "     --outcome-file\tFile where test outcomes are written\n"
     printf "                \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n"
     printf "     --port     \tTCP/UDP port (default: randomish 1xxxx)\n"
@@ -192,6 +194,9 @@ get_options() {
             -p|--preserve-logs)
                 PRESERVE_LOGS=1
                 ;;
+            --min)
+                shift; MIN_TESTS=$1
+                ;;
             --outcome-file)
                 shift; MBEDTLS_TEST_OUTCOME_FILE=$1
                 ;;
@@ -14152,6 +14157,15 @@ if [ "$LIST_TESTS" -eq 0 ]; then
     PASSES=$(( $TESTS - $FAILS ))
     echo " ($PASSES / $TESTS tests ($SKIPS skipped))"
 
+    if [ $((TESTS - SKIPS)) -lt $MIN_TESTS ]; then
+        cat <<EOF
+Error: Expected to run at least $MIN_TESTS, but only ran $((TESTS - SKIPS)).
+Maybe a bad filter ('$FILTER') or a bad configuration?
+EOF
+        if [ $FAILS -eq 0 ]; then
+            FAILS=1
+        fi
+    fi
 fi
 
 if [ $FAILS -gt 255 ]; then