Merge pull request #6699 from gilles-peskine-arm/lcov-script

lcov script

CMakeLists.txt

@@ -304,22 +304,15 @@ if(ENABLE_TESTING)
     # additional convenience targets for Unix only
     if(UNIX)
 
-        ADD_CUSTOM_TARGET(covtest
+        # For coverage testing:
-            COMMAND make test
+        # 1. Build with:
-            COMMAND programs/test/selftest
+        #         cmake -D CMAKE_BUILD_TYPE=Coverage /path/to/source && make
-            COMMAND tests/compat.sh
+        # 2. Run the relevant tests for the part of the code you're interested in.
-            COMMAND tests/ssl-opt.sh
+        #    For the reference coverage measurement, see
-        )
+        #    tests/scripts/basic-build-test.sh
-
+        # 3. Run scripts/lcov.sh to generate an HTML report.
         ADD_CUSTOM_TARGET(lcov
-            COMMAND rm -rf Coverage
+            COMMAND scripts/lcov.sh
-            COMMAND lcov --capture --initial --directory library/CMakeFiles/mbedtls.dir -o files.info
-            COMMAND lcov --capture --directory library/CMakeFiles/mbedtls.dir -o tests.info
-            COMMAND lcov --add-tracefile files.info --add-tracefile tests.info -o all.info
-            COMMAND lcov --remove all.info -o final.info '*.h'
-            COMMAND gendesc tests/Descriptions.txt -o descriptions
-            COMMAND genhtml --title "mbed TLS" --description-file descriptions --keep-descriptions --legend --no-branch-coverage -o Coverage final.info
-            COMMAND rm -f files.info tests.info all.info final.info descriptions
         )
 
         ADD_CUSTOM_TARGET(memcheck

Makefile

@@ -4,7 +4,7 @@ PERL ?= perl
 
 .SILENT:
 
-.PHONY: all no_test programs lib tests install uninstall clean test check covtest lcov apidoc apidoc_clean
+.PHONY: all no_test programs lib tests install uninstall clean test check lcov apidoc apidoc_clean
 
 all: programs tests
 	$(MAKE) post_build
@@ -136,23 +136,15 @@ check: lib tests
 test: check
 
 ifndef WINDOWS
-# note: for coverage testing, build with:
+# For coverage testing:
-# make CFLAGS='--coverage -g3 -O0'
+# 1. Build with:
-covtest:
+#         make CFLAGS='--coverage -g3 -O0' LDFLAGS='--coverage'
-	$(MAKE) check
+# 2. Run the relevant tests for the part of the code you're interested in.
-	programs/test/selftest
+#    For the reference coverage measurement, see
-	tests/compat.sh
+#    tests/scripts/basic-build-test.sh
-	tests/ssl-opt.sh
+# 3. Run scripts/lcov.sh to generate an HTML report.
-
 lcov:
-	rm -rf Coverage
+	scripts/lcov.sh
-	lcov --capture --initial --directory library -o files.info
-	lcov --rc lcov_branch_coverage=1 --capture --directory library -o tests.info
-	lcov --rc lcov_branch_coverage=1 --add-tracefile files.info --add-tracefile tests.info -o all.info
-	lcov --rc lcov_branch_coverage=1 --remove all.info -o final.info '*.h'
-	gendesc tests/Descriptions.txt -o descriptions
-	genhtml --title "mbed TLS" --description-file descriptions --keep-descriptions --legend --branch-coverage -o Coverage final.info
-	rm -f files.info tests.info all.info final.info descriptions
 
 apidoc:
 	mkdir -p apidoc

scripts/lcov.sh

@@ -0,0 +1,81 @@
+#!/bin/sh
+
+help () {
+    cat <<EOF
+Usage: $0 [-r]
+Collect coverage statistics of library code into an HTML report.
+
+General instructions:
+1. Build the library with CFLAGS="--coverage -O0 -g3" and link the test
+   programs with LDFLAGS="--coverage".
+   This can be an out-of-tree build.
+   For example (in-tree):
+        make CFLAGS="--coverage -O0 -g3" LDFLAGS="--coverage"
+   Or (out-of-tree):
+        mkdir build-coverage && cd build-coverage &&
+        cmake -D CMAKE_BUILD_TYPE=Coverage .. && make
+2. Run whatever tests you want.
+3. Run this script from the parent of the directory containing the library
+   object files and coverage statistics files.
+4. Browse the coverage report in Coverage/index.html.
+5. After rework, run "$0 -r", then re-test and run "$0" to get a fresh report.
+
+Options
+  -r    Reset traces. Run this before re-testing to get fresh measurements.
+EOF
+}
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+# Collect stats and build a HTML report.
+lcov_library_report () {
+    rm -rf Coverage
+    mkdir Coverage Coverage/tmp
+    lcov --capture --initial --directory library -o Coverage/tmp/files.info
+    lcov --rc lcov_branch_coverage=1 --capture --directory library -o Coverage/tmp/tests.info
+    lcov --rc lcov_branch_coverage=1 --add-tracefile Coverage/tmp/files.info --add-tracefile Coverage/tmp/tests.info -o Coverage/tmp/all.info
+    lcov --rc lcov_branch_coverage=1 --remove Coverage/tmp/all.info -o Coverage/tmp/final.info '*.h'
+    gendesc tests/Descriptions.txt -o Coverage/tmp/descriptions
+    genhtml --title "mbed TLS" --description-file Coverage/tmp/descriptions --keep-descriptions --legend --branch-coverage -o Coverage Coverage/tmp/final.info
+    rm -f Coverage/tmp/*.info Coverage/tmp/descriptions
+    echo "Coverage report in: Coverage/index.html"
+}
+
+# Reset the traces to 0.
+lcov_reset_traces () {
+    # Location with plain make
+    rm -f library/*.gcda
+    # Location with CMake
+    rm -f library/CMakeFiles/*.dir/*.gcda
+}
+
+if [ $# -gt 0 ] && [ "$1" = "--help" ]; then
+    help
+    exit
+fi
+
+main=lcov_library_report
+while getopts r OPTLET; do
+    case $OPTLET in
+        r) main=lcov_reset_traces;;
+        *) help 2>&1; exit 120;;
+    esac
+done
+shift $((OPTIND - 1))
+
+"$main" "$@"

tests/CMakeLists.txt

@@ -243,6 +243,7 @@ if (NOT ${CMAKE_CURRENT_BINARY_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR})
     if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/seedfile")
         link_to_source(seedfile)
     endif()
+    link_to_source(Descriptions.txt)
     link_to_source(compat.sh)
     link_to_source(context-info.sh)
     link_to_source(data_files)