Merge pull request #152 from ARMmbed/psa-test-psa_constant_names

Test psa_constant_names

include/CMakeLists.txt

@@ -18,4 +18,5 @@ endif(INSTALL_MBEDTLS_HEADERS)
 # Make config.h available in an out-of-source build. ssl-opt.sh requires it.
 if (NOT ${CMAKE_CURRENT_BINARY_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR})
     link_to_source(mbedtls)
+    link_to_source(psa)
 endif()

include/psa/crypto_values.h

@@ -766,7 +766,7 @@
  *          algorithm is considered identical to the untruncated algorithm
  *          for policy comparison purposes.
  *
- * \param alg           A MAC algorithm identifier (value of type
+ * \param mac_alg       A MAC algorithm identifier (value of type
  *                      #psa_algorithm_t such that #PSA_ALG_IS_MAC(\p alg)
  *                      is true). This may be a truncated or untruncated
  *                      MAC algorithm.
@@ -782,14 +782,14 @@
  *                      MAC algorithm or if \p mac_length is too small or
  *                      too large for the specified MAC algorithm.
  */
-#define PSA_ALG_TRUNCATED_MAC(alg, mac_length)                          \
+#define PSA_ALG_TRUNCATED_MAC(mac_alg, mac_length)                      \
-    (((alg) & ~PSA_ALG_MAC_TRUNCATION_MASK) |                           \
+    (((mac_alg) & ~PSA_ALG_MAC_TRUNCATION_MASK) |                       \
      ((mac_length) << PSA_MAC_TRUNCATION_OFFSET & PSA_ALG_MAC_TRUNCATION_MASK))
 
 /** Macro to build the base MAC algorithm corresponding to a truncated
  * MAC algorithm.
  *
- * \param alg           A MAC algorithm identifier (value of type
+ * \param mac_alg       A MAC algorithm identifier (value of type
  *                      #psa_algorithm_t such that #PSA_ALG_IS_MAC(\p alg)
  *                      is true). This may be a truncated or untruncated
  *                      MAC algorithm.
@@ -798,12 +798,12 @@
  * \return              Unspecified if \p alg is not a supported
  *                      MAC algorithm.
  */
-#define PSA_ALG_FULL_LENGTH_MAC(alg)            \
+#define PSA_ALG_FULL_LENGTH_MAC(mac_alg)        \
-    ((alg) & ~PSA_ALG_MAC_TRUNCATION_MASK)
+    ((mac_alg) & ~PSA_ALG_MAC_TRUNCATION_MASK)
 
 /** Length to which a MAC algorithm is truncated.
  *
- * \param alg           A MAC algorithm identifier (value of type
+ * \param mac_alg       A MAC algorithm identifier (value of type
  *                      #psa_algorithm_t such that #PSA_ALG_IS_MAC(\p alg)
  *                      is true).
  *
@@ -812,8 +812,8 @@
  * \return              Unspecified if \p alg is not a supported
  *                      MAC algorithm.
  */
-#define PSA_MAC_TRUNCATED_LENGTH(alg)           \
+#define PSA_MAC_TRUNCATED_LENGTH(mac_alg)                               \
-    (((alg) & PSA_ALG_MAC_TRUNCATION_MASK) >> PSA_MAC_TRUNCATION_OFFSET)
+    (((mac_alg) & PSA_ALG_MAC_TRUNCATION_MASK) >> PSA_MAC_TRUNCATION_OFFSET)
 
 #define PSA_ALG_CIPHER_MAC_BASE                 ((psa_algorithm_t)0x02c00000)
 #define PSA_ALG_CBC_MAC                         ((psa_algorithm_t)0x02c00001)
@@ -910,7 +910,7 @@
  * Depending on the algorithm, the tag length may affect the calculation
  * of the ciphertext.
  *
- * \param alg           A AEAD algorithm identifier (value of type
+ * \param aead_alg      An AEAD algorithm identifier (value of type
  *                      #psa_algorithm_t such that #PSA_ALG_IS_AEAD(\p alg)
  *                      is true).
  * \param tag_length    Desired length of the authentication tag in bytes.
@@ -921,26 +921,26 @@
  *                      AEAD algorithm or if \p tag_length is not valid
  *                      for the specified AEAD algorithm.
  */
-#define PSA_ALG_AEAD_WITH_TAG_LENGTH(alg, tag_length)                   \
+#define PSA_ALG_AEAD_WITH_TAG_LENGTH(aead_alg, tag_length)              \
-    (((alg) & ~PSA_ALG_AEAD_TAG_LENGTH_MASK) |                          \
+    (((aead_alg) & ~PSA_ALG_AEAD_TAG_LENGTH_MASK) |                     \
      ((tag_length) << PSA_AEAD_TAG_LENGTH_OFFSET &                      \
       PSA_ALG_AEAD_TAG_LENGTH_MASK))
 
 /** Calculate the corresponding AEAD algorithm with the default tag length.
  *
- * \param alg   An AEAD algorithm (\c PSA_ALG_XXX value such that
+ * \param aead_alg      An AEAD algorithm (\c PSA_ALG_XXX value such that
  *                      #PSA_ALG_IS_AEAD(\p alg) is true).
  *
- * \return      The corresponding AEAD algorithm with the default tag length
+ * \return              The corresponding AEAD algorithm with the default
- *              for that algorithm.
+ *                      tag length for that algorithm.
  */
-#define PSA_ALG_AEAD_WITH_DEFAULT_TAG_LENGTH(alg)                       \
+#define PSA_ALG_AEAD_WITH_DEFAULT_TAG_LENGTH(aead_alg)                  \
     (                                                                   \
-        PSA__ALG_AEAD_WITH_DEFAULT_TAG_LENGTH__CASE(alg, PSA_ALG_CCM)   \
+        PSA__ALG_AEAD_WITH_DEFAULT_TAG_LENGTH__CASE(aead_alg, PSA_ALG_CCM) \
-        PSA__ALG_AEAD_WITH_DEFAULT_TAG_LENGTH__CASE(alg, PSA_ALG_GCM)   \
+        PSA__ALG_AEAD_WITH_DEFAULT_TAG_LENGTH__CASE(aead_alg, PSA_ALG_GCM) \
         0)
-#define PSA__ALG_AEAD_WITH_DEFAULT_TAG_LENGTH__CASE(alg, ref) \
+#define PSA__ALG_AEAD_WITH_DEFAULT_TAG_LENGTH__CASE(aead_alg, ref)      \
-    PSA_ALG_AEAD_WITH_TAG_LENGTH(alg, 0) == \
+    PSA_ALG_AEAD_WITH_TAG_LENGTH(aead_alg, 0) ==                        \
     PSA_ALG_AEAD_WITH_TAG_LENGTH(ref, 0) ?  \
     ref :
 

programs/Makefile

@@ -103,7 +103,7 @@ EXTRA_GENERATED += psa/psa_constant_names_generated.c
 endif
 
 psa/psa_constant_names$(EXEXT): psa/psa_constant_names_generated.c
-psa/psa_constant_names_generated.c: ../scripts/generate_psa_constants.py ../include/psa/crypto_values.h
+psa/psa_constant_names_generated.c: ../scripts/generate_psa_constants.py ../include/psa/crypto_values.h ../include/psa/crypto_extra.h
 	../scripts/generate_psa_constants.py
 
 aes/aescrypt2$(EXEXT): aes/aescrypt2.c $(DEP)

programs/psa/CMakeLists.txt

@@ -1,7 +1,26 @@
 add_executable(crypto_examples crypto_examples.c)
 target_link_libraries(crypto_examples mbedtls)
 
-install(TARGETS crypto_examples
+add_executable(key_ladder_demo key_ladder_demo.c)
+target_link_libraries(key_ladder_demo mbedtls)
+
+add_executable(psa_constant_names psa_constant_names.c)
+target_link_libraries(psa_constant_names mbedtls)
+
+add_custom_target(
+    psa_constant_names_generated
+    COMMAND ${PYTHON_EXECUTABLE} scripts/generate_psa_constants.py
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../
+)
+add_dependencies(psa_constant_names psa_constant_names_generated)
+
+install(TARGETS
+            crypto_examples
+            key_ladder_demo
+            psa_constant_names
         DESTINATION "bin"
         PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
 
+install(PROGRAMS
+            key_ladder_demo.sh
+        DESTINATION "bin")

programs/psa/key_ladder_demo.c

@@ -620,9 +620,9 @@ static void usage( void )
 
 int main( int argc, char *argv[] )
 {
-    char *key_file_name = "master.key";
+    const char *key_file_name = "master.key";
-    char *input_file_name = NULL;
+    const char *input_file_name = NULL;
-    char *output_file_name = NULL;
+    const char *output_file_name = NULL;
     const char *ladder[MAX_LADDER_DEPTH];
     size_t ladder_depth = 0;
     int i;

programs/psa/psa_constant_names.c

@@ -138,7 +138,7 @@ static int psa_snprint_ecc_curve(char *buffer, size_t buffer_size,
 
 static void usage(const char *program_name)
 {
-    printf("Usage: %s TYPE VALUE\n",
+    printf("Usage: %s TYPE VALUE [VALUE...]\n",
            program_name == NULL ? "psa_constant_names" : program_name);
     printf("Print the symbolic name whose numerical value is VALUE in TYPE.\n");
     printf("Supported types (with = between aliases):\n");
@@ -149,11 +149,19 @@ static void usage(const char *program_name)
     printf("  error=status          Status code (psa_status_t)\n");
 }
 
+typedef enum {
+    TYPE_STATUS,
+    TYPE_ALGORITHM,
+    TYPE_ECC_CURVE,
+    TYPE_KEY_TYPE,
+    TYPE_KEY_USAGE,
+} value_type;
+
 int main(int argc, char *argv[])
 {
-    char buffer[200];
+    value_type type;
-    unsigned long value;
+    unsigned long max;
-    char *end;
+    int i;
 
     if (argc <= 1 ||
         !strcmp(argv[1], "help") ||
@@ -162,31 +170,64 @@ int main(int argc, char *argv[])
         usage(argv[0]);
         return EXIT_FAILURE;
     }
-    if (argc != 3) {
-        usage(argv[0]);
-        return EXIT_FAILURE;
-    }
-    value = strtoul(argv[2], &end, 0);
-    if (*end) {
-        printf("Non-numeric value: %s\n", argv[2]);
-        return EXIT_FAILURE;
-    }
 
-    if (!strcmp(argv[1], "error") || !strcmp(argv[1], "status"))
+    if (!strcmp(argv[1], "error") || !strcmp(argv[1], "status")) {
-        psa_snprint_status(buffer, sizeof(buffer), (psa_status_t) value);
+        type = TYPE_STATUS;
-    else if (!strcmp(argv[1], "alg") || !strcmp(argv[1], "algorithm"))
+        max = 0x7fffffff; /* hard-coded because psa_status_t is signed */
-        psa_snprint_algorithm(buffer, sizeof(buffer), (psa_algorithm_t) value);
+    } else if (!strcmp(argv[1], "alg") || !strcmp(argv[1], "algorithm")) {
-    else if (!strcmp(argv[1], "curve") || !strcmp(argv[1], "ecc_curve"))
+        type = TYPE_ALGORITHM;
-        psa_snprint_ecc_curve(buffer, sizeof(buffer), (psa_ecc_curve_t) value);
+        max = (psa_algorithm_t)( -1 );
-    else if (!strcmp(argv[1], "type") || !strcmp(argv[1], "key_type"))
+    } else if (!strcmp(argv[1], "curve") || !strcmp(argv[1], "ecc_curve")) {
-        psa_snprint_key_type(buffer, sizeof(buffer), (psa_key_type_t) value);
+        type = TYPE_ECC_CURVE;
-    else if (!strcmp(argv[1], "usage") || !strcmp(argv[1], "key_usage"))
+        max = (psa_ecc_curve_t)( -1 );
-        psa_snprint_key_usage(buffer, sizeof(buffer), (psa_key_usage_t) value);
+    } else if (!strcmp(argv[1], "type") || !strcmp(argv[1], "key_type")) {
-    else {
+        type = TYPE_KEY_TYPE;
+        max = (psa_key_type_t)( -1 );
+    } else if (!strcmp(argv[1], "usage") || !strcmp(argv[1], "key_usage")) {
+        type = TYPE_KEY_USAGE;
+        max = (psa_key_usage_t)( -1 );
+    } else {
         printf("Unknown type: %s\n", argv[1]);
         return EXIT_FAILURE;
     }
 
+    for (i = 2; i < argc; i++) {
+        char buffer[200];
+        char *end;
+        unsigned long value = strtoul(argv[i], &end, 0);
+        if (*end) {
+            printf("Non-numeric value: %s\n", argv[i]);
+            return EXIT_FAILURE;
+        }
+        if (value > max) {
+            printf("Value out of range: %s\n", argv[i]);
+            return EXIT_FAILURE;
+        }
+
+        switch (type) {
+            case TYPE_STATUS:
+                psa_snprint_status(buffer, sizeof(buffer),
+                                   (psa_status_t) value);
+                break;
+            case TYPE_ALGORITHM:
+                psa_snprint_algorithm(buffer, sizeof(buffer),
+                                      (psa_algorithm_t) value);
+                break;
+            case TYPE_ECC_CURVE:
+                psa_snprint_ecc_curve(buffer, sizeof(buffer),
+                                      (psa_ecc_curve_t) value);
+                break;
+            case TYPE_KEY_TYPE:
+                psa_snprint_key_type(buffer, sizeof(buffer),
+                                     (psa_key_type_t) value);
+                break;
+            case TYPE_KEY_USAGE:
+                psa_snprint_key_usage(buffer, sizeof(buffer),
+                                      (psa_key_usage_t) value);
+                break;
+        }
         puts(buffer);
+    }
+
     return EXIT_SUCCESS;
 }

scripts/generate_psa_constants.py

@@ -62,7 +62,10 @@ static int psa_snprint_algorithm(char *buffer, size_t buffer_size,
         }
     } else if (PSA_ALG_IS_AEAD(alg)) {
         core_alg = PSA_ALG_AEAD_WITH_DEFAULT_TAG_LENGTH(alg);
-        if (core_alg != alg) {
+        if (core_alg == 0) {
+            /* For unknown AEAD algorithms, there is no "default tag length". */
+            core_alg = alg;
+        } else if (core_alg != alg) {
             append(&buffer, buffer_size, &required_size,
                    "PSA_ALG_AEAD_WITH_TAG_LENGTH(", 29);
             length_modifier = PSA_AEAD_TAG_LENGTH(alg);
@@ -73,7 +76,7 @@ static int psa_snprint_algorithm(char *buffer, size_t buffer_size,
     default:
         %(algorithm_code)s{
             append_integer(&buffer, buffer_size, &required_size,
-                           "0x%%08lx", (unsigned long) alg);
+                           "0x%%08lx", (unsigned long) core_alg);
         }
         break;
     }
@@ -273,8 +276,9 @@ class MacroCollector:
         data['key_usage_code'] = self.make_key_usage_code()
         output_file.write(output_template % data)
 
-def generate_psa_constants(header_file_name, output_file_name):
+def generate_psa_constants(header_file_names, output_file_name):
     collector = MacroCollector()
+    for header_file_name in header_file_names:
         with open(header_file_name) as header_file:
             collector.read_file(header_file)
     temp_file_name = output_file_name + '.tmp'
@@ -285,5 +289,6 @@ def generate_psa_constants(header_file_name, output_file_name):
 if __name__ == '__main__':
     if not os.path.isdir('programs') and os.path.isdir('../programs'):
         os.chdir('..')
-    generate_psa_constants('include/psa/crypto_values.h',
+    generate_psa_constants(['include/psa/crypto_values.h',
+                            'include/psa/crypto_extra.h'],
                            'programs/psa/psa_constant_names_generated.c')

tests/CMakeLists.txt

@@ -134,4 +134,5 @@ if (NOT ${CMAKE_CURRENT_BINARY_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR})
     link_to_source(data_files)
     link_to_source(scripts)
     link_to_source(ssl-opt.sh)
+    link_to_source(suites)
 endif()

tests/scripts/all.sh

@@ -711,6 +711,9 @@ component_test_full_cmake_clang () {
     msg "test: main suites (full config)" # ~ 5s
     make test
 
+    msg "test: psa_constant_names (full config)" # ~ 1s
+    record_status tests/scripts/test_psa_constant_names.py
+
     msg "test: ssl-opt.sh default, ECJPAKE, SSL async (full config)" # ~ 1s
     if_build_succeeded tests/ssl-opt.sh -f 'Default\|ECJPAKE\|SSL async private'
 

tests/scripts/test_psa_constant_names.py

@@ -0,0 +1,320 @@
+#!/usr/bin/env python3
+'''Test the program psa_constant_names.
+Gather constant names from header files and test cases. Compile a C program
+to print out their numerical values, feed these numerical values to
+psa_constant_names, and check that the output is the original name.
+Return 0 if all test cases pass, 1 if the output was not always as expected,
+or 1 (with a Python backtrace) if there was an operational error.'''
+
+import argparse
+import itertools
+import os
+import platform
+import re
+import subprocess
+import sys
+import tempfile
+
+class ReadFileLineException(Exception):
+    def __init__(self, filename, line_number):
+        message = 'in {} at {}'.format(filename, line_number)
+        super(ReadFileLineException, self).__init__(message)
+        self.filename = filename
+        self.line_number = line_number
+
+class read_file_lines:
+    '''Context manager to read a text file line by line.
+with read_file_lines(filename) as lines:
+    for line in lines:
+        process(line)
+is equivalent to
+with open(filename, 'r') as input_file:
+    for line in input_file:
+        process(line)
+except that if process(line) raises an exception, then the read_file_lines
+snippet annotates the exception with the file name and line number.'''
+    def __init__(self, filename):
+        self.filename = filename
+        self.line_number = 'entry'
+    def __enter__(self):
+        self.generator = enumerate(open(self.filename, 'r'))
+        return self
+    def __iter__(self):
+        for line_number, content in self.generator:
+            self.line_number = line_number
+            yield content
+        self.line_number = 'exit'
+    def __exit__(self, type, value, traceback):
+        if type is not None:
+            raise ReadFileLineException(self.filename, self.line_number) \
+                from value
+
+class Inputs:
+    '''Accumulate information about macros to test.
+This includes macro names as well as information about their arguments
+when applicable.'''
+    def __init__(self):
+        # Sets of names per type
+        self.statuses = set(['PSA_SUCCESS'])
+        self.algorithms = set(['0xffffffff'])
+        self.ecc_curves = set(['0xffff'])
+        self.key_types = set(['0xffffffff'])
+        self.key_usage_flags = set(['0x80000000'])
+        # Hard-coded value for unknown algorithms
+        self.hash_algorithms = set(['0x010000fe'])
+        self.mac_algorithms = set(['0x02ff00ff'])
+        self.kdf_algorithms = set(['0x300000ff', '0x310000ff'])
+        # For AEAD algorithms, the only variability is over the tag length,
+        # and this only applies to known algorithms, so don't test an
+        # unknown algorithm.
+        self.aead_algorithms = set()
+        # Identifier prefixes
+        self.table_by_prefix = {
+            'ERROR': self.statuses,
+            'ALG': self.algorithms,
+            'CURVE': self.ecc_curves,
+            'KEY_TYPE': self.key_types,
+            'KEY_USAGE': self.key_usage_flags,
+        }
+        # macro name -> list of argument names
+        self.argspecs = {}
+        # argument name -> list of values
+        self.arguments_for = {
+            'mac_length': ['1', '63'],
+            'tag_length': ['1', '63'],
+        }
+
+    def gather_arguments(self):
+        '''Populate the list of values for macro arguments.
+Call this after parsing all the inputs.'''
+        self.arguments_for['hash_alg'] = sorted(self.hash_algorithms)
+        self.arguments_for['mac_alg'] = sorted(self.mac_algorithms)
+        self.arguments_for['kdf_alg'] = sorted(self.kdf_algorithms)
+        self.arguments_for['aead_alg'] = sorted(self.aead_algorithms)
+        self.arguments_for['curve'] = sorted(self.ecc_curves)
+
+    def format_arguments(self, name, arguments):
+        '''Format a macro call with arguments..'''
+        return name + '(' + ', '.join(arguments) + ')'
+
+    def distribute_arguments(self, name):
+        '''Generate macro calls with each tested argument set.
+If name is a macro without arguments, just yield "name".
+If name is a macro with arguments, yield a series of "name(arg1,...,argN)"
+where each argument takes each possible value at least once.'''
+        try:
+            if name not in self.argspecs:
+                yield name
+                return
+            argspec = self.argspecs[name]
+            if argspec == []:
+                yield name + '()'
+                return
+            argument_lists = [self.arguments_for[arg] for arg in argspec]
+            arguments = [values[0] for values in argument_lists]
+            yield self.format_arguments(name, arguments)
+            for i in range(len(arguments)):
+                for value in argument_lists[i][1:]:
+                    arguments[i] = value
+                    yield self.format_arguments(name, arguments)
+                arguments[i] = argument_lists[0][0]
+        except BaseException as e:
+            raise Exception('distribute_arguments({})'.format(name)) from e
+
+    # Regex for interesting header lines.
+    # Groups: 1=macro name, 2=type, 3=argument list (optional).
+    header_line_re = \
+        re.compile(r'#define +' +
+                   r'(PSA_((?:KEY_)?[A-Z]+)_\w+)' +
+                   r'(?:\(([^\n()]*)\))?')
+    # Regex of macro names to exclude.
+    excluded_name_re = re.compile('_(?:GET|IS|OF)_|_(?:BASE|FLAG|MASK)\Z')
+    # Additional excluded macros.
+    # PSA_ALG_ECDH and PSA_ALG_FFDH are excluded for now as the script
+    # currently doesn't support them.
+    excluded_names = set(['PSA_ALG_AEAD_WITH_DEFAULT_TAG_LENGTH',
+                          'PSA_ALG_FULL_LENGTH_MAC',
+                          'PSA_ALG_ECDH',
+                          'PSA_ALG_FFDH'])
+    argument_split_re = re.compile(r' *, *')
+    def parse_header_line(self, line):
+        '''Parse a C header line, looking for "#define PSA_xxx".'''
+        m = re.match(self.header_line_re, line)
+        if not m:
+            return
+        name = m.group(1)
+        if re.search(self.excluded_name_re, name) or \
+           name in self.excluded_names:
+            return
+        dest = self.table_by_prefix.get(m.group(2))
+        if dest is None:
+            return
+        dest.add(name)
+        if m.group(3):
+            self.argspecs[name] = re.split(self.argument_split_re, m.group(3))
+
+    def parse_header(self, filename):
+        '''Parse a C header file, looking for "#define PSA_xxx".'''
+        with read_file_lines(filename) as lines:
+            for line in lines:
+                self.parse_header_line(line)
+
+    def add_test_case_line(self, function, argument):
+        '''Parse a test case data line, looking for algorithm metadata tests.'''
+        if function.endswith('_algorithm'):
+            # As above, ECDH and FFDH algorithms are excluded for now.
+            # Support for them will be added in the future.
+            if 'ECDH' in argument or 'FFDH' in argument:
+                return
+            self.algorithms.add(argument)
+            if function == 'hash_algorithm':
+                self.hash_algorithms.add(argument)
+            elif function in ['mac_algorithm', 'hmac_algorithm']:
+                self.mac_algorithms.add(argument)
+            elif function == 'aead_algorithm':
+                self.aead_algorithms.add(argument)
+        elif function == 'key_type':
+            self.key_types.add(argument)
+        elif function == 'ecc_key_types':
+            self.ecc_curves.add(argument)
+
+    # Regex matching a *.data line containing a test function call and
+    # its arguments. The actual definition is partly positional, but this
+    # regex is good enough in practice.
+    test_case_line_re = re.compile('(?!depends_on:)(\w+):([^\n :][^:\n]*)')
+    def parse_test_cases(self, filename):
+        '''Parse a test case file (*.data), looking for algorithm metadata tests.'''
+        with read_file_lines(filename) as lines:
+            for line in lines:
+                m = re.match(self.test_case_line_re, line)
+                if m:
+                    self.add_test_case_line(m.group(1), m.group(2))
+
+def gather_inputs(headers, test_suites):
+    '''Read the list of inputs to test psa_constant_names with.'''
+    inputs = Inputs()
+    for header in headers:
+        inputs.parse_header(header)
+    for test_cases in test_suites:
+        inputs.parse_test_cases(test_cases)
+    inputs.gather_arguments()
+    return inputs
+
+def remove_file_if_exists(filename):
+    '''Remove the specified file, ignoring errors.'''
+    if not filename:
+        return
+    try:
+        os.remove(filename)
+    except:
+        pass
+
+def run_c(options, type, names):
+    '''Generate and run a program to print out numerical values for names.'''
+    c_name = None
+    exe_name = None
+    try:
+        c_fd, c_name = tempfile.mkstemp(prefix='tmp-{}-'.format(type),
+                                        suffix='.c',
+                                        dir='programs/psa')
+        exe_suffix = '.exe' if platform.system() == 'Windows' else ''
+        exe_name = c_name[:-2] + exe_suffix
+        remove_file_if_exists(exe_name)
+        c_file = os.fdopen(c_fd, 'w', encoding='ascii')
+        c_file.write('/* Generated by test_psa_constant_names.py for {} values */'
+                     .format(type))
+        c_file.write('''
+#include <stdio.h>
+#include <psa/crypto.h>
+int main(void)
+{
+''')
+        for name in names:
+            c_file.write('    printf("0x%08x\\n", {});\n'.format(name))
+        c_file.write('''    return 0;
+}
+''')
+        c_file.close()
+        cc = os.getenv('CC', 'cc')
+        subprocess.check_call([cc] +
+                              ['-I' + dir for dir in options.include] +
+                              ['-o', exe_name, c_name])
+        if options.keep_c:
+            sys.stderr.write('List of {} tests kept at {}\n'
+                             .format(type, c_name))
+        else:
+            os.remove(c_name)
+        output = subprocess.check_output([exe_name])
+        return output.decode('ascii').strip().split('\n')
+    finally:
+        remove_file_if_exists(exe_name)
+
+normalize_strip_re = re.compile(r'\s+')
+def normalize(expr):
+    '''Normalize the C expression so as not to care about trivial differences.
+Currently "trivial differences" means whitespace.'''
+    expr = re.sub(normalize_strip_re, '', expr, len(expr))
+    return expr.strip().split('\n')
+
+def do_test(options, inputs, type, names):
+    '''Test psa_constant_names for the specified type.
+Run program on names.
+Use inputs to figure out what arguments to pass to macros that take arguments.'''
+    names = sorted(itertools.chain(*map(inputs.distribute_arguments, names)))
+    values = run_c(options, type, names)
+    output = subprocess.check_output([options.program, type] + values)
+    outputs = output.decode('ascii').strip().split('\n')
+    errors = [(type, name, value, output)
+              for (name, value, output) in zip(names, values, outputs)
+              if normalize(name) != normalize(output)]
+    return len(names), errors
+
+def report_errors(errors):
+    '''Describe each case where the output is not as expected.'''
+    for type, name, value, output in errors:
+        print('For {} "{}", got "{}" (value: {})'
+              .format(type, name, output, value))
+
+def run_tests(options, inputs):
+    '''Run psa_constant_names on all the gathered inputs.
+Return a tuple (count, errors) where count is the total number of inputs
+that were tested and errors is the list of cases where the output was
+not as expected.'''
+    count = 0
+    errors = []
+    for type, names in [('status', inputs.statuses),
+                        ('algorithm', inputs.algorithms),
+                        ('ecc_curve', inputs.ecc_curves),
+                        ('key_type', inputs.key_types),
+                        ('key_usage', inputs.key_usage_flags)]:
+        c, e = do_test(options, inputs, type, names)
+        count += c
+        errors += e
+    return count, errors
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=globals()['__doc__'])
+    parser.add_argument('--include', '-I',
+                        action='append', default=['include'],
+                        help='Directory for header files')
+    parser.add_argument('--program',
+                        default='programs/psa/psa_constant_names',
+                        help='Program to test')
+    parser.add_argument('--keep-c',
+                        action='store_true', dest='keep_c', default=False,
+                        help='Keep the intermediate C file')
+    parser.add_argument('--no-keep-c',
+                        action='store_false', dest='keep_c',
+                        help='Don\'t keep the intermediate C file (default)')
+    options = parser.parse_args()
+    headers = [os.path.join(options.include[0], 'psa', h)
+               for h in ['crypto.h', 'crypto_extra.h', 'crypto_values.h']]
+    test_suites = ['tests/suites/test_suite_psa_crypto_metadata.data']
+    inputs = gather_inputs(headers, test_suites)
+    count, errors = run_tests(options, inputs)
+    report_errors(errors)
+    if errors == []:
+        print('{} test cases PASS'.format(count))
+    else:
+        print('{} test cases, {} FAIL'.format(count, len(errors)))
+        exit(1)