146 lines
5.0 KiB
Python
146 lines
5.0 KiB
Python
# Copyright 2021 The Chromium Authors
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
"""This is a library for printing test result as specified by
|
|
//docs/testing/test_executable_api.md (e.g. --isolated-script-test-output)
|
|
and //docs/testing/json_test_results_format.md
|
|
|
|
Typical usage:
|
|
import argparse
|
|
import test_results
|
|
|
|
cmdline_parser = argparse.ArgumentParser()
|
|
test_results.add_cmdline_args(cmdline_parser)
|
|
... adding other cmdline parameter definitions ...
|
|
parsed_cmdline_args = cmdline_parser.parse_args()
|
|
|
|
test_results = []
|
|
test_results.append(test_results.TestResult(
|
|
'test-suite/test-name', 'PASS'))
|
|
...
|
|
|
|
test_results.print_test_results(parsed_cmdline_args, test_results)
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
|
|
|
|
class TestResult:
|
|
"""TestResult represents a result of executing a single test once.
|
|
"""
|
|
|
|
def __init__(self,
|
|
test_name,
|
|
actual_test_result,
|
|
expected_test_result='PASS'):
|
|
self.test_name = test_name
|
|
self.actual_test_result = actual_test_result
|
|
self.expected_test_result = expected_test_result
|
|
|
|
def __eq__(self, other):
|
|
self_tuple = tuple(sorted(self.__dict__.items()))
|
|
other_tuple = tuple(sorted(other.__dict__.items()))
|
|
return self_tuple == other_tuple
|
|
|
|
def __hash__(self):
|
|
return hash(tuple(sorted(self.__dict__.items())))
|
|
|
|
def __repr__(self):
|
|
result = 'TestResult[{}: {}'.format(self.test_name,
|
|
self.actual_test_result)
|
|
if self.expected_test_result != 'PASS':
|
|
result += ' (expecting: {})]'.format(self.expected_test_result)
|
|
else:
|
|
result += ']'
|
|
return result
|
|
|
|
|
|
def _validate_output_directory(outdir):
|
|
if not os.path.isdir(outdir):
|
|
raise argparse.ArgumentTypeError('No such directory: ' + outdir)
|
|
return outdir
|
|
|
|
|
|
def add_cmdline_args(argparse_parser):
|
|
"""Adds test-result-specific cmdline parameter definitions to
|
|
`argparse_parser`.
|
|
|
|
Args:
|
|
argparse_parser: An object of argparse.ArgumentParser type.
|
|
"""
|
|
outdir_help = 'Directory where test results will be written into.'
|
|
argparse_parser.add_argument('--isolated-outdir',
|
|
dest='outdir',
|
|
help=outdir_help,
|
|
metavar='DIRPATH',
|
|
type=_validate_output_directory)
|
|
|
|
outfile_help = 'If this argument is provided, then test results in the ' \
|
|
'JSON Test Results Format will be written here. See also ' \
|
|
'//docs/testing/json_test_results_format.md'
|
|
argparse_parser.add_argument('--isolated-script-test-output',
|
|
dest='test_output',
|
|
default=None,
|
|
help=outfile_help,
|
|
metavar='FILENAME')
|
|
|
|
argparse_parser.add_argument('--isolated-script-test-perf-output',
|
|
dest='ignored_perf_output',
|
|
default=None,
|
|
help='Deprecated and ignored.',
|
|
metavar='IGNORED')
|
|
|
|
|
|
def _build_json_data(list_of_test_results, seconds_since_epoch):
|
|
num_failures_by_type = {}
|
|
tests = {}
|
|
for res in list_of_test_results:
|
|
old_count = num_failures_by_type.get(res.actual_test_result, 0)
|
|
num_failures_by_type[res.actual_test_result] = old_count + 1
|
|
|
|
path = res.test_name.split('//')
|
|
group = tests
|
|
for group_name in path[:-1]:
|
|
if not group_name in group:
|
|
group[group_name] = {}
|
|
group = group[group_name]
|
|
|
|
group[path[-1]] = {
|
|
'expected': res.expected_test_result,
|
|
'actual': res.actual_test_result,
|
|
}
|
|
|
|
return {
|
|
'interrupted': False,
|
|
'path_delimiter': '//',
|
|
'seconds_since_epoch': seconds_since_epoch,
|
|
'version': 3,
|
|
'tests': tests,
|
|
'num_failures_by_type': num_failures_by_type,
|
|
}
|
|
|
|
|
|
def print_test_results(argparse_parsed_args, list_of_test_results,
|
|
testing_start_time_as_seconds_since_epoch):
|
|
"""Prints `list_of_test_results` to a file specified on the cmdline.
|
|
|
|
Args:
|
|
argparse_parsed_arg: A result of an earlier call to
|
|
argparse_parser.parse_args() call (where `argparse_parser` has been
|
|
populated via an even earlier call to add_cmdline_args).
|
|
list_of_test_results: A list of TestResult objects.
|
|
testing_start_time_as_seconds_since_epoch: A number from an earlier
|
|
`time.time()` call.
|
|
"""
|
|
if argparse_parsed_args.test_output is None:
|
|
return
|
|
|
|
json_data = _build_json_data(list_of_test_results,
|
|
testing_start_time_as_seconds_since_epoch)
|
|
|
|
filepath = argparse_parsed_args.test_output
|
|
with open(filepath, mode='w', encoding='utf-8') as f:
|
|
json.dump(json_data, f, indent=2)
|