diff --git a/CMakeLists.txt b/CMakeLists.txt
index a56ecdc0c9..306cf0261d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -316,6 +316,34 @@ if(ENABLE_TESTING OR ENABLE_PROGRAMS)
          ${CMAKE_CURRENT_SOURCE_DIR}/tests/src/drivers/*.c)
     add_library(mbedtls_test OBJECT ${MBEDTLS_TEST_FILES})
     if(GEN_FILES)
+        add_custom_command(
+            OUTPUT
+                ${CMAKE_CURRENT_SOURCE_DIR}/tests/src/test_keys.h
+            WORKING_DIRECTORY
+                ${CMAKE_CURRENT_SOURCE_DIR}/tests
+            COMMAND
+                "${MBEDTLS_PYTHON_EXECUTABLE}"
+                "${CMAKE_CURRENT_SOURCE_DIR}/tests/scripts/generate_test_keys.py"
+                "--output"
+                "${CMAKE_CURRENT_SOURCE_DIR}/tests/src/test_keys.h"
+            DEPENDS
+                ${CMAKE_CURRENT_SOURCE_DIR}/tests/scripts/generate_test_keys.py
+        )
+        add_custom_target(test_keys_header DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/tests/src/test_keys.h)
+        add_custom_command(
+            OUTPUT
+                ${CMAKE_CURRENT_SOURCE_DIR}/tests/src/test_certs.h
+            WORKING_DIRECTORY
+                ${CMAKE_CURRENT_SOURCE_DIR}/tests
+            COMMAND
+                "${MBEDTLS_PYTHON_EXECUTABLE}"
+                "${CMAKE_CURRENT_SOURCE_DIR}/tests/scripts/generate_test_cert_macros.py"
+                "--output"
+                "${CMAKE_CURRENT_SOURCE_DIR}/tests/src/test_certs.h"
+            DEPENDS
+                ${CMAKE_CURRENT_SOURCE_DIR}/tests/scripts/generate_test_cert_macros.py
+        )
+        add_custom_target(test_certs_header DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/tests/src/test_certs.h)
         add_dependencies(mbedtls_test test_keys_header test_certs_header)
     endif()
     target_include_directories(mbedtls_test
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 531404fc1d..589643a806 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -74,34 +74,6 @@ foreach(file ${base_psa_generated_data_files})
 endforeach()
 
 if(GEN_FILES)
-    add_custom_command(
-        OUTPUT
-            ${CMAKE_CURRENT_SOURCE_DIR}/src/test_keys.h
-        WORKING_DIRECTORY
-            ${CMAKE_CURRENT_SOURCE_DIR}
-        COMMAND
-            "${MBEDTLS_PYTHON_EXECUTABLE}"
-            "${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_test_keys.py"
-            "--output"
-            "${CMAKE_CURRENT_SOURCE_DIR}/src/test_keys.h"
-        DEPENDS
-            ${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_test_keys.py
-    )
-    add_custom_target(test_keys_header DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/src/test_keys.h)
-    add_custom_command(
-        OUTPUT
-            ${CMAKE_CURRENT_SOURCE_DIR}/src/test_certs.h
-        WORKING_DIRECTORY
-            ${CMAKE_CURRENT_SOURCE_DIR}
-        COMMAND
-            "${MBEDTLS_PYTHON_EXECUTABLE}"
-            "${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_test_cert_macros.py"
-            "--output"
-            "${CMAKE_CURRENT_SOURCE_DIR}/src/test_certs.h"
-        DEPENDS
-            ${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_test_cert_macros.py
-    )
-    add_custom_target(test_certs_header DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/src/test_certs.h)
     add_custom_command(
         OUTPUT
             ${bignum_generated_data_files}
diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh
index 5343337b43..8158c8d976 100755
--- a/tests/scripts/all.sh
+++ b/tests/scripts/all.sh
@@ -6077,6 +6077,15 @@ support_build_cmake_custom_config_file () {
     support_test_cmake_out_of_source
 }
 
+component_build_cmake_programs_no_testing () {
+    # Verify that the type of builds performed by oss-fuzz don't get accidentally broken
+    msg "build: cmake with -DENABLE_PROGRAMS=ON and -DENABLE_TESTING=OFF"
+    cmake -DENABLE_PROGRAMS=ON -DENABLE_TESTING=OFF .
+    make
+}
+support_build_cmake_programs_no_testing () {
+    support_test_cmake_out_of_source
+}
 
 component_build_zeroize_checks () {
     msg "build: check for obviously wrong calls to mbedtls_platform_zeroize()"