mirror of
https://github.com/Mbed-TLS/mbedtls.git
synced 2025-02-04 15:39:53 +00:00
94 lines
2.8 KiB
Python
94 lines
2.8 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
|
||
|
|
"""Analyze the test outcomes from a full CI run.
|
||
|
|
|
||
|
|
This script can also run on outcomes from a partial run, but the results are
|
||
|
|
less likely to be useful.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import sys
|
||
|
|
import traceback
|
||
|
|
|
||
|
|
class Results:
|
||
|
|
"""Process analysis results."""
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
self.error_count = 0
|
||
|
|
self.warning_count = 0
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def log(fmt, *args, **kwargs):
|
||
|
|
sys.stderr.write((fmt + '\n').format(*args, **kwargs))
|
||
|
|
|
||
|
|
def error(self, fmt, *args, **kwargs):
|
||
|
|
self.log('Error: ' + fmt, *args, **kwargs)
|
||
|
|
self.error_count += 1
|
||
|
|
|
||
|
|
def warning(self, fmt, *args, **kwargs):
|
||
|
|
self.log('Warning: ' + fmt, *args, **kwargs)
|
||
|
|
self.warning_count += 1
|
||
|
|
|
||
|
|
class TestCaseOutcomes:
|
||
|
|
"""The outcomes of one test case across many configurations."""
|
||
|
|
# pylint: disable=too-few-public-methods
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
self.successes = []
|
||
|
|
self.failures = []
|
||
|
|
|
||
|
|
def hits(self):
|
||
|
|
"""Return the number of times a test case has been run.
|
||
|
|
|
||
|
|
This includes passes and failures, but not skips.
|
||
|
|
"""
|
||
|
|
return len(self.successes) + len(self.failures)
|
||
|
|
|
||
|
|
def analyze_outcomes(outcomes):
|
||
|
|
"""Run all analyses on the given outcome collection."""
|
||
|
|
results = Results()
|
||
|
|
return results
|
||
|
|
|
||
|
|
def read_outcome_file(outcome_file):
|
||
|
|
"""Parse an outcome file and return an outcome collection.
|
||
|
|
|
||
|
|
An outcome collection is a dictionary mapping keys to TestCaseOutcomes objects.
|
||
|
|
The keys are the test suite name and the test case description, separated
|
||
|
|
by a semicolon.
|
||
|
|
"""
|
||
|
|
outcomes = {}
|
||
|
|
with open(outcome_file, 'r', encoding='utf-8') as input_file:
|
||
|
|
for line in input_file:
|
||
|
|
(platform, config, suite, case, result, _cause) = line.split(';')
|
||
|
|
key = ';'.join([suite, case])
|
||
|
|
setup = ';'.join([platform, config])
|
||
|
|
if key not in outcomes:
|
||
|
|
outcomes[key] = TestCaseOutcomes()
|
||
|
|
if result == 'PASS':
|
||
|
|
outcomes[key].successes.append(setup)
|
||
|
|
elif result == 'FAIL':
|
||
|
|
outcomes[key].failures.append(setup)
|
||
|
|
return outcomes
|
||
|
|
|
||
|
|
def analyze_outcome_file(outcome_file):
|
||
|
|
"""Analyze the given outcome file."""
|
||
|
|
outcomes = read_outcome_file(outcome_file)
|
||
|
|
return analyze_outcomes(outcomes)
|
||
|
|
|
||
|
|
def main():
|
||
|
|
try:
|
||
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
||
|
|
parser.add_argument('outcomes', metavar='OUTCOMES.CSV',
|
||
|
|
help='Outcome file to analyze')
|
||
|
|
options = parser.parse_args()
|
||
|
|
results = analyze_outcome_file(options.outcomes)
|
||
|
|
if results.error_count > 0:
|
||
|
|
sys.exit(1)
|
||
|
|
except Exception: # pylint: disable=broad-except
|
||
|
|
# Print the backtrace and exit explicitly with our chosen status.
|
||
|
|
traceback.print_exc()
|
||
|
|
sys.exit(120)
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
main()
|