841 lines
33 KiB
Python
Executable File
841 lines
33 KiB
Python
Executable File
#!/usr/bin/env vpython3
|
|
# Copyright 2017 The Chromium Authors
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""Runs telemetry benchmarks and gtest perf tests.
|
|
|
|
If optional argument --isolated-script-test-output=[FILENAME] is passed
|
|
to the script, json is written to that file in the format detailed in
|
|
//docs/testing/json-test-results-format.md.
|
|
|
|
If optional argument --isolated-script-test-filter=[TEST_NAMES] is passed to
|
|
the script, it should be a double-colon-separated ("::") list of test names,
|
|
to run just that subset of tests.
|
|
|
|
This script is intended to be the base command invoked by the isolate,
|
|
followed by a subsequent Python script. It could be generalized to
|
|
invoke an arbitrary executable.
|
|
It currently runs several benchmarks. The benchmarks it will execute are
|
|
based on the shard it is running on and the sharding_map_path.
|
|
|
|
If this is executed with a gtest perf test, the flag --non-telemetry
|
|
has to be passed in to the script so the script knows it is running
|
|
an executable and not the run_benchmark command.
|
|
|
|
This script merges test results from all the benchmarks into the one
|
|
output.json file. The test results and perf results are also put in separate
|
|
directories per benchmark. Two files will be present in each directory;
|
|
perf_results.json, which is the perf specific results (with unenforced format,
|
|
could be histogram or graph json), and test_results.json.
|
|
|
|
TESTING:
|
|
To test changes to this script, please run
|
|
cd tools/perf
|
|
./run_tests ScriptsSmokeTest.testRunPerformanceTests
|
|
"""
|
|
|
|
from __future__ import print_function
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import requests
|
|
import shutil
|
|
import sys
|
|
import time
|
|
import tempfile
|
|
import traceback
|
|
import six
|
|
|
|
from collections import OrderedDict
|
|
|
|
CHROMIUM_SRC_DIR = os.path.abspath(
|
|
os.path.join(os.path.dirname(__file__),
|
|
os.path.pardir, os.path.pardir))
|
|
|
|
PERF_DIR = os.path.join(CHROMIUM_SRC_DIR, 'tools', 'perf')
|
|
sys.path.append(PERF_DIR)
|
|
import generate_legacy_perf_dashboard_json
|
|
from core import path_util
|
|
|
|
PERF_CORE_DIR = os.path.join(PERF_DIR, 'core')
|
|
sys.path.append(PERF_CORE_DIR)
|
|
import results_merger
|
|
|
|
# Add src/testing/ into sys.path for importing xvfb, test_env, and common.
|
|
sys.path.append(
|
|
os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)))
|
|
import xvfb
|
|
import test_env
|
|
from scripts import common
|
|
|
|
SHARD_MAPS_DIRECTORY = os.path.abspath(
|
|
os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir,
|
|
'tools', 'perf', 'core', 'shard_maps'))
|
|
|
|
# See https://crbug.com/923564.
|
|
# We want to switch over to using histograms for everything, but converting from
|
|
# the format output by gtest perf tests to histograms has introduced several
|
|
# problems. So, only perform the conversion on tests that are whitelisted and
|
|
# are okay with potentially encountering issues.
|
|
GTEST_CONVERSION_WHITELIST = [
|
|
'angle_perftests',
|
|
'base_perftests',
|
|
'blink_heap_perftests',
|
|
'blink_platform_perftests',
|
|
'cc_perftests',
|
|
'components_perftests',
|
|
'command_buffer_perftests',
|
|
'dawn_perf_tests',
|
|
'gpu_perftests',
|
|
'load_library_perf_tests',
|
|
'net_perftests',
|
|
'browser_tests',
|
|
'services_perftests',
|
|
# TODO(jmadill): Remove once migrated. http://anglebug.com/5124
|
|
'standalone_angle_perftests',
|
|
'sync_performance_tests',
|
|
'tracing_perftests',
|
|
'views_perftests',
|
|
'viz_perftests',
|
|
'wayland_client_perftests',
|
|
'xr.vr.common_perftests',
|
|
]
|
|
|
|
# pylint: disable=useless-object-inheritance
|
|
|
|
|
|
class OutputFilePaths(object):
|
|
"""Provide paths to where results outputs should be written.
|
|
|
|
The process_perf_results.py merge script later will pull all of these
|
|
together, so that's why they aren't in the standard locations. Also,
|
|
note that because of the OBBS (One Build Bot Step), Telemetry
|
|
has multiple tests running on a single shard, so we need to prefix
|
|
these locations with a directory named by the benchmark name.
|
|
"""
|
|
|
|
def __init__(self, isolated_out_dir, perf_test_name):
|
|
self.name = perf_test_name
|
|
self.benchmark_path = os.path.join(isolated_out_dir, perf_test_name)
|
|
|
|
def SetUp(self):
|
|
os.makedirs(self.benchmark_path)
|
|
return self
|
|
|
|
@property
|
|
def perf_results(self):
|
|
return os.path.join(self.benchmark_path, 'perf_results.json')
|
|
|
|
@property
|
|
def test_results(self):
|
|
return os.path.join(self.benchmark_path, 'test_results.json')
|
|
|
|
@property
|
|
def logs(self):
|
|
return os.path.join(self.benchmark_path, 'benchmark_log.txt')
|
|
|
|
@property
|
|
def csv_perf_results(self):
|
|
"""Path for csv perf results.
|
|
|
|
Note that the chrome.perf waterfall uses the json histogram perf results
|
|
exclusively. csv_perf_results are implemented here in case a user script
|
|
passes --output-format=csv.
|
|
"""
|
|
return os.path.join(self.benchmark_path, 'perf_results.csv')
|
|
|
|
|
|
def print_duration(step, start):
|
|
print('Duration of %s: %d seconds' % (step, time.time() - start))
|
|
|
|
|
|
def IsWindows():
|
|
return sys.platform == 'cygwin' or sys.platform.startswith('win')
|
|
|
|
|
|
class GtestCommandGenerator(object):
|
|
def __init__(self, options, override_executable=None, additional_flags=None,
|
|
ignore_shard_env_vars=False):
|
|
self._options = options
|
|
self._override_executable = override_executable
|
|
self._additional_flags = additional_flags or []
|
|
self._ignore_shard_env_vars = ignore_shard_env_vars
|
|
|
|
def generate(self, output_dir):
|
|
"""Generate the command to run to start the gtest perf test.
|
|
|
|
Returns:
|
|
list of strings, the executable and its arguments.
|
|
"""
|
|
return ([self._get_executable()] +
|
|
self._generate_filter_args() +
|
|
self._generate_repeat_args() +
|
|
self._generate_also_run_disabled_tests_args() +
|
|
self._generate_output_args(output_dir) +
|
|
self._generate_shard_args() +
|
|
self._get_additional_flags()
|
|
)
|
|
|
|
@property
|
|
def executable_name(self):
|
|
"""Gets the platform-independent name of the executable."""
|
|
return self._override_executable or self._options.executable
|
|
|
|
def _get_executable(self):
|
|
executable = str(self.executable_name)
|
|
if IsWindows():
|
|
return r'.\%s.exe' % executable
|
|
return './%s' % executable
|
|
|
|
def _get_additional_flags(self):
|
|
return self._additional_flags
|
|
|
|
def _generate_shard_args(self):
|
|
"""Teach the gtest to ignore the environment variables.
|
|
|
|
GTEST_SHARD_INDEX and GTEST_TOTAL_SHARDS will confuse the gtest
|
|
and convince it to only run some of its tests. Instead run all
|
|
of them.
|
|
"""
|
|
if self._ignore_shard_env_vars:
|
|
return ['--test-launcher-total-shards=1', '--test-launcher-shard-index=0']
|
|
return []
|
|
|
|
def _generate_filter_args(self):
|
|
if self._options.isolated_script_test_filter:
|
|
filter_list = common.extract_filter_list(
|
|
self._options.isolated_script_test_filter)
|
|
return ['--gtest_filter=' + ':'.join(filter_list)]
|
|
return []
|
|
|
|
def _generate_repeat_args(self):
|
|
# TODO(crbug.com/920002): Support --isolated-script-test-repeat.
|
|
return []
|
|
|
|
def _generate_also_run_disabled_tests_args(self):
|
|
# TODO(crbug.com/920002): Support
|
|
# --isolated-script-test-also-run-disabled-tests.
|
|
return []
|
|
|
|
def _generate_output_args(self, output_dir):
|
|
output_args = []
|
|
if self._options.use_gtest_benchmark_script:
|
|
output_args.append('--output-dir=' + output_dir)
|
|
# These flags are to make sure that test output perf metrics in the log.
|
|
if not '--verbose' in self._get_additional_flags():
|
|
output_args.append('--verbose')
|
|
if (not '--test-launcher-print-test-stdio=always'
|
|
in self._get_additional_flags()):
|
|
output_args.append('--test-launcher-print-test-stdio=always')
|
|
return output_args
|
|
|
|
|
|
def write_simple_test_results(return_code, output_filepath, benchmark_name):
|
|
# TODO(crbug.com/1115658): Fix to output
|
|
# https://chromium.googlesource.com/chromium/src/+/main/docs/testing/json_test_results_format.md
|
|
# for each test rather than this summary.
|
|
# Append the shard index to the end of the name so that the merge script
|
|
# doesn't blow up trying to merge unmergeable results.
|
|
benchmark_name += '_shard_%s' % os.environ.get('GTEST_SHARD_INDEX', '0')
|
|
output_json = {
|
|
'tests': {
|
|
benchmark_name: {
|
|
'expected': 'PASS',
|
|
'actual': 'FAIL' if return_code else 'PASS',
|
|
'is_unexpected': bool(return_code),
|
|
},
|
|
},
|
|
'interrupted': False,
|
|
'path_delimiter': '/',
|
|
'version': 3,
|
|
'seconds_since_epoch': time.time(),
|
|
'num_failures_by_type': {
|
|
'FAIL': 1 if return_code else 0,
|
|
'PASS': 0 if return_code else 1,
|
|
},
|
|
}
|
|
with open(output_filepath, 'w') as fh:
|
|
json.dump(output_json, fh)
|
|
|
|
|
|
def upload_simple_test_results(return_code, benchmark_name):
|
|
# TODO(crbug.com/1115658): Fix to upload results for each test rather than
|
|
# this summary.
|
|
try:
|
|
with open(os.environ['LUCI_CONTEXT']) as f:
|
|
sink = json.load(f)['result_sink']
|
|
except KeyError:
|
|
return
|
|
|
|
if return_code:
|
|
summary = '<p>Benchmark failed with status code %d</p>' % return_code
|
|
else:
|
|
summary = '<p>Benchmark passed</p>'
|
|
|
|
result_json = {
|
|
'testResults': [{
|
|
'testId': benchmark_name,
|
|
'expected': not return_code,
|
|
'status': 'FAIL' if return_code else 'PASS',
|
|
'summaryHtml': summary,
|
|
'tags': [{'key': 'exit_code', 'value': str(return_code)}],
|
|
}]
|
|
}
|
|
|
|
res = requests.post(
|
|
url='http://%s/prpc/luci.resultsink.v1.Sink/ReportTestResults' %
|
|
sink['address'],
|
|
headers={
|
|
'Content-Type': 'application/json',
|
|
'Accept': 'application/json',
|
|
'Authorization': 'ResultSink %s' % sink['auth_token'],
|
|
},
|
|
data=json.dumps(result_json))
|
|
res.raise_for_status()
|
|
|
|
|
|
def execute_gtest_perf_test(command_generator, output_paths, use_xvfb=False,
|
|
is_unittest=False):
|
|
start = time.time()
|
|
|
|
env = os.environ.copy()
|
|
env['CHROME_HEADLESS'] = '1'
|
|
#TODO(crbug/1138988): Some gtests do not implements the unit_test_launcher.cc.
|
|
# As a result, they will not respect the arguments added by
|
|
# _generate_shard_args() and will still use the values of GTEST_SHARD_INDEX
|
|
# and GTEST_TOTAL_SHARDS to run part of the tests.
|
|
# Removing those environment variables as a workaround.
|
|
if command_generator._ignore_shard_env_vars:
|
|
if 'GTEST_TOTAL_SHARDS' in env:
|
|
env.pop('GTEST_TOTAL_SHARDS')
|
|
if 'GTEST_SHARD_INDEX' in env:
|
|
env.pop('GTEST_SHARD_INDEX')
|
|
|
|
return_code = 0
|
|
try:
|
|
command = command_generator.generate(output_paths.benchmark_path)
|
|
if use_xvfb:
|
|
# When running with xvfb, we currently output both to stdout and to the
|
|
# file. It would be better to only output to the file to keep the logs
|
|
# clean.
|
|
return_code = xvfb.run_executable(
|
|
command, env, stdoutfile=output_paths.logs)
|
|
else:
|
|
with open(output_paths.logs, 'w') as handle:
|
|
try:
|
|
return_code = test_env.run_command_output_to_handle(
|
|
command, handle, env=env)
|
|
except OSError as e:
|
|
print('Command to run gtest perf test %s failed with an OSError: %s' %
|
|
(output_paths.name, e))
|
|
return_code = 1
|
|
if (not os.path.exists(output_paths.perf_results) and
|
|
os.path.exists(output_paths.logs)):
|
|
# Get the correct json format from the stdout to write to the perf
|
|
# results file if gtest does not generate one.
|
|
results_processor = generate_legacy_perf_dashboard_json.\
|
|
LegacyResultsProcessor()
|
|
graph_json_string = results_processor.GenerateJsonResults(
|
|
output_paths.logs)
|
|
with open(output_paths.perf_results, 'w') as fh:
|
|
fh.write(graph_json_string)
|
|
except Exception:
|
|
traceback.print_exc()
|
|
return_code = 1
|
|
if os.path.exists(output_paths.perf_results):
|
|
executable_name = command_generator.executable_name
|
|
if executable_name.startswith('bin/run_'):
|
|
# The executable is a wrapper used by Fuchsia. Remove the prefix to get
|
|
# the actual executable name.
|
|
executable_name = executable_name[8:]
|
|
if executable_name in GTEST_CONVERSION_WHITELIST:
|
|
with path_util.SysPath(path_util.GetTracingDir()):
|
|
# pylint: disable=no-name-in-module,import-outside-toplevel
|
|
from tracing.value import gtest_json_converter
|
|
# pylint: enable=no-name-in-module,import-outside-toplevel
|
|
gtest_json_converter.ConvertGtestJsonFile(output_paths.perf_results)
|
|
else:
|
|
print('ERROR: gtest perf test %s did not generate perf output' %
|
|
output_paths.name)
|
|
return_code = 1
|
|
write_simple_test_results(return_code, output_paths.test_results,
|
|
output_paths.name)
|
|
if not is_unittest:
|
|
upload_simple_test_results(return_code, output_paths.name)
|
|
|
|
print_duration(
|
|
'executing gtest %s' % command_generator.executable_name, start)
|
|
|
|
return return_code
|
|
|
|
|
|
class _TelemetryFilterArgument(object):
|
|
def __init__(self, filter_string):
|
|
self.benchmark, self.story = filter_string.split('/')
|
|
|
|
|
|
class TelemetryCommandGenerator(object):
|
|
def __init__(self, benchmark, options,
|
|
story_selection_config=None, is_reference=False):
|
|
self.benchmark = benchmark
|
|
self._options = options
|
|
self._story_selection_config = story_selection_config
|
|
self._is_reference = is_reference
|
|
|
|
def generate(self, output_dir):
|
|
"""Generate the command to run to start the benchmark.
|
|
|
|
Args:
|
|
output_dir: The directory to configure the command to put output files
|
|
into.
|
|
|
|
Returns:
|
|
list of strings, the executable and its arguments.
|
|
"""
|
|
return ([sys.executable] + self._options.executable.split(' ') +
|
|
[self.benchmark] +
|
|
self._generate_filter_args() +
|
|
self._generate_also_run_disabled_tests_args() +
|
|
self._generate_output_args(output_dir) +
|
|
self._generate_story_selection_args() +
|
|
# passthrough args must be before reference args and repeat args:
|
|
# crbug.com/928928, crbug.com/894254#c78
|
|
self._get_passthrough_args() +
|
|
self._generate_syslog_args() +
|
|
self._generate_repeat_args() +
|
|
self._generate_reference_build_args()
|
|
)
|
|
|
|
def _get_passthrough_args(self):
|
|
return self._options.passthrough_args
|
|
|
|
def _generate_filter_args(self):
|
|
if self._options.isolated_script_test_filter:
|
|
filter_list = common.extract_filter_list(
|
|
self._options.isolated_script_test_filter)
|
|
filter_arguments = [_TelemetryFilterArgument(f) for f in filter_list]
|
|
applicable_stories = [
|
|
f.story for f in filter_arguments if f.benchmark == self.benchmark]
|
|
# Need to convert this to a valid regex.
|
|
filter_regex = '(' + '|'.join(applicable_stories) + ')'
|
|
return ['--story-filter=' + filter_regex]
|
|
return []
|
|
|
|
def _generate_repeat_args(self):
|
|
if self._options.isolated_script_test_repeat:
|
|
return ['--pageset-repeat=' + str(
|
|
self._options.isolated_script_test_repeat)]
|
|
return []
|
|
|
|
def _generate_also_run_disabled_tests_args(self):
|
|
if self._options.isolated_script_test_also_run_disabled_tests:
|
|
return ['--also-run-disabled-tests']
|
|
return []
|
|
|
|
def _generate_output_args(self, output_dir):
|
|
return ['--output-format=json-test-results',
|
|
'--output-format=histograms',
|
|
'--output-dir=' + output_dir]
|
|
|
|
def _generate_story_selection_args(self):
|
|
"""Returns arguments that limit the stories to be run inside the benchmark.
|
|
"""
|
|
selection_args = []
|
|
if self._story_selection_config:
|
|
if 'begin' in self._story_selection_config:
|
|
selection_args.append('--story-shard-begin-index=%d' % (
|
|
self._story_selection_config['begin']))
|
|
if 'end' in self._story_selection_config:
|
|
selection_args.append('--story-shard-end-index=%d' % (
|
|
self._story_selection_config['end']))
|
|
if 'sections' in self._story_selection_config:
|
|
range_string = self._generate_story_index_ranges(
|
|
self._story_selection_config['sections'])
|
|
if range_string:
|
|
selection_args.append('--story-shard-indexes=%s' % range_string)
|
|
if self._story_selection_config.get('abridged', True):
|
|
selection_args.append('--run-abridged-story-set')
|
|
return selection_args
|
|
|
|
def _generate_syslog_args(self):
|
|
if self._options.per_test_logs_dir:
|
|
isolated_out_dir = os.path.dirname(
|
|
self._options.isolated_script_test_output)
|
|
return ['--logs-dir', os.path.join(
|
|
isolated_out_dir,
|
|
self.benchmark)]
|
|
return []
|
|
|
|
|
|
def _generate_story_index_ranges(self, sections):
|
|
range_string = ''
|
|
for section in sections:
|
|
begin = section.get('begin', '')
|
|
end = section.get('end', '')
|
|
# If there only one story in the range, we only keep its index.
|
|
# In general, we expect either begin or end, or both.
|
|
if begin != '' and end != '' and end - begin == 1:
|
|
new_range = str(begin)
|
|
elif begin != '' or end != '':
|
|
new_range = '%s-%s' % (str(begin), str(end))
|
|
else:
|
|
raise ValueError('Index ranges in "sections" in shard map should have'
|
|
'at least one of "begin" and "end": %s' % str(section))
|
|
if range_string:
|
|
range_string += ',%s' % new_range
|
|
else:
|
|
range_string = new_range
|
|
return range_string
|
|
|
|
|
|
def _generate_reference_build_args(self):
|
|
if self._is_reference:
|
|
reference_browser_flag = '--browser=reference'
|
|
# TODO(crbug.com/1038137): Make the logic generic once more reference
|
|
# settings are added
|
|
if '--browser=android-chrome-bundle' in self._get_passthrough_args():
|
|
reference_browser_flag = '--browser=reference-android-chrome-bundle'
|
|
return [reference_browser_flag,
|
|
'--max-failures=5']
|
|
return []
|
|
|
|
|
|
def execute_telemetry_benchmark(
|
|
command_generator, output_paths, use_xvfb=False,
|
|
return_exit_code_zero=False):
|
|
start = time.time()
|
|
|
|
env = os.environ.copy()
|
|
env['CHROME_HEADLESS'] = '1'
|
|
|
|
return_code = 1
|
|
temp_dir = tempfile.mkdtemp('telemetry')
|
|
infra_failure = False
|
|
try:
|
|
command = command_generator.generate(temp_dir)
|
|
if use_xvfb:
|
|
# When running with xvfb, we currently output both to stdout and to the
|
|
# file. It would be better to only output to the file to keep the logs
|
|
# clean.
|
|
return_code = xvfb.run_executable(
|
|
command, env=env, stdoutfile=output_paths.logs)
|
|
else:
|
|
with open(output_paths.logs, 'w') as handle:
|
|
return_code = test_env.run_command_output_to_handle(
|
|
command, handle, env=env)
|
|
expected_results_filename = os.path.join(temp_dir, 'test-results.json')
|
|
if os.path.exists(expected_results_filename):
|
|
shutil.move(expected_results_filename, output_paths.test_results)
|
|
else:
|
|
common.write_interrupted_test_results_to(output_paths.test_results, start)
|
|
expected_perf_filename = os.path.join(temp_dir, 'histograms.json')
|
|
shutil.move(expected_perf_filename, output_paths.perf_results)
|
|
|
|
csv_file_path = os.path.join(temp_dir, 'results.csv')
|
|
if os.path.isfile(csv_file_path):
|
|
shutil.move(csv_file_path, output_paths.csv_perf_results)
|
|
except Exception:
|
|
print('The following exception may have prevented the code from '
|
|
'outputing structured test results and perf results output:')
|
|
print(traceback.format_exc())
|
|
infra_failure = True
|
|
finally:
|
|
# On swarming bots, don't remove output directory, since Result Sink might
|
|
# still be uploading files to Result DB. Also, swarming bots automatically
|
|
# clean up at the end of each task.
|
|
if 'SWARMING_TASK_ID' not in os.environ:
|
|
# Add ignore_errors=True because otherwise rmtree may fail due to leaky
|
|
# processes of tests are still holding opened handles to files under
|
|
# |tempfile_dir|. For example, see crbug.com/865896
|
|
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
|
|
print_duration('executing benchmark %s' % command_generator.benchmark, start)
|
|
|
|
if infra_failure:
|
|
print('There was an infrastructure error encountered during the run. '
|
|
'Please check the logs above for details')
|
|
return 1
|
|
|
|
# Telemetry sets exit code to -1 to indicate that no stories were run. This
|
|
# becomes 255 on linux because linux doesn't support -1 so it does modulo:
|
|
# -1 % 256 == 255.
|
|
# TODO(crbug.com/1019139): Make 111 be the exit code that means
|
|
# "no stories were run.".
|
|
if return_code in (111, -1, 255):
|
|
print('Exit code %s indicates that no stories were run, so we are marking '
|
|
'this as a success.' % return_code)
|
|
return 0
|
|
if return_code:
|
|
if return_exit_code_zero:
|
|
print ('run_benchmark returned exit code ' + str(return_code)
|
|
+ ' which indicates there were test failures in the run.')
|
|
return 0
|
|
return return_code
|
|
return 0
|
|
|
|
def parse_arguments(args):
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('executable', help='The name of the executable to run.')
|
|
parser.add_argument(
|
|
'--isolated-script-test-output', required=True)
|
|
# The following two flags may be passed in sometimes by Pinpoint
|
|
# or by the recipe, but they don't do anything. crbug.com/927482.
|
|
parser.add_argument(
|
|
'--isolated-script-test-chartjson-output', required=False)
|
|
parser.add_argument(
|
|
'--isolated-script-test-perf-output', required=False)
|
|
|
|
parser.add_argument(
|
|
'--isolated-script-test-filter', type=str, required=False)
|
|
|
|
# Note that the following three arguments are only supported by Telemetry
|
|
# tests right now. See crbug.com/920002.
|
|
parser.add_argument(
|
|
'--isolated-script-test-repeat', type=int, required=False)
|
|
parser.add_argument(
|
|
'--isolated-script-test-launcher-retry-limit', type=int, required=False,
|
|
choices=[0]) # Telemetry does not support retries. crbug.com/894254#c21
|
|
parser.add_argument(
|
|
'--isolated-script-test-also-run-disabled-tests',
|
|
default=False, action='store_true', required=False)
|
|
parser.add_argument('--xvfb', help='Start xvfb.', action='store_true')
|
|
parser.add_argument('--non-telemetry',
|
|
help='Type of perf test', type=bool, default=False)
|
|
parser.add_argument('--gtest-benchmark-name',
|
|
help='Name of the gtest benchmark', type=str,
|
|
required=False)
|
|
parser.add_argument('--use-gtest-benchmark-script',
|
|
help='Whether gtest is invoked via benchmark script.',
|
|
default=False, action='store_true')
|
|
|
|
parser.add_argument('--benchmarks',
|
|
help='Comma separated list of benchmark names'
|
|
' to run in lieu of indexing into our benchmark bot maps',
|
|
required=False)
|
|
# crbug.com/1236245: This allows for per-benchmark device logs.
|
|
parser.add_argument('--per-test-logs-dir',
|
|
help='Require --logs-dir args for test', required=False,
|
|
default=False, action='store_true')
|
|
# Some executions may have a different sharding scheme and/or set of tests.
|
|
# These files must live in src/tools/perf/core/shard_maps
|
|
parser.add_argument('--test-shard-map-filename', type=str, required=False)
|
|
parser.add_argument('--run-ref-build',
|
|
help='Run test on reference browser', action='store_true')
|
|
parser.add_argument('--passthrough-arg',
|
|
help='Arguments to pass directly through to the test '
|
|
'executable.', action='append',
|
|
dest='passthrough_args',
|
|
default=[])
|
|
parser.add_argument('--use-dynamic-shards',
|
|
help='If set, use dynamic shardmap instead of the file.',
|
|
action='store_true',
|
|
required=False
|
|
)
|
|
parser.add_argument('--dynamic-shardmap',
|
|
help='The dynamically generated shardmap string used to '
|
|
'replace the static shardmap file.',
|
|
type=str,
|
|
required=False)
|
|
parser.add_argument('--ignore-benchmark-exit-code',
|
|
help='If set, return an exit code 0 even if there'
|
|
+ ' are benchmark failures',
|
|
action='store_true',
|
|
required=False
|
|
)
|
|
options, leftover_args = parser.parse_known_args(args)
|
|
options.passthrough_args.extend(leftover_args)
|
|
return options
|
|
|
|
|
|
def main(sys_args):
|
|
args = sys_args[1:] # Skip program name.
|
|
options = parse_arguments(args)
|
|
isolated_out_dir = os.path.dirname(options.isolated_script_test_output)
|
|
overall_return_code = 0
|
|
# This is a list of test results files to be merged into a standard
|
|
# output.json file for use by infrastructure including FindIt.
|
|
# This list should not contain reference build runs
|
|
# since we do not monitor those. Also, merging test reference build results
|
|
# with standard build results may not work properly.
|
|
test_results_files = []
|
|
|
|
print('Running a series of performance test subprocesses. Logs, performance\n'
|
|
'results, and test results JSON will be saved in a subfolder of the\n'
|
|
'isolated output directory. Inside the hash marks in the following\n'
|
|
'lines is the name of the subfolder to find results in.\n')
|
|
|
|
if options.non_telemetry:
|
|
benchmark_name = options.gtest_benchmark_name
|
|
passthrough_args = options.passthrough_args
|
|
# crbug/1146949#c15
|
|
# In the case that pinpoint passes all arguments to swarming through http
|
|
# request, the passthrough_args are converted into a comma-separated string.
|
|
if passthrough_args and isinstance(passthrough_args, six.text_type):
|
|
passthrough_args = passthrough_args.split(',')
|
|
# With --non-telemetry, the gtest executable file path will be passed in as
|
|
# options.executable, which is different from running on shard map. Thus,
|
|
# we don't override executable as we do in running on shard map.
|
|
command_generator = GtestCommandGenerator(
|
|
options, additional_flags=passthrough_args, ignore_shard_env_vars=True)
|
|
# Fallback to use the name of the executable if flag isn't set.
|
|
# TODO(crbug.com/870899): remove fallback logic and raise parser error if
|
|
# --non-telemetry is set but --gtest-benchmark-name is not set once pinpoint
|
|
# is converted to always pass --gtest-benchmark-name flag.
|
|
if not benchmark_name:
|
|
benchmark_name = options.executable
|
|
output_paths = OutputFilePaths(isolated_out_dir, benchmark_name).SetUp()
|
|
print('\n### {folder} ###'.format(folder=benchmark_name))
|
|
overall_return_code = execute_gtest_perf_test(
|
|
command_generator, output_paths, options.xvfb)
|
|
test_results_files.append(output_paths.test_results)
|
|
else:
|
|
if options.use_dynamic_shards:
|
|
shard_map_str = options.dynamic_shardmap
|
|
shard_map = json.loads(shard_map_str, object_pairs_hook=OrderedDict)
|
|
shard_map_path = os.path.join(SHARD_MAPS_DIRECTORY,
|
|
options.test_shard_map_filename)
|
|
with open(shard_map_path, 'w') as f:
|
|
json.dump(shard_map, f, indent=4, separators=(',', ': '))
|
|
shutil.copyfile(
|
|
shard_map_path,
|
|
os.path.join(isolated_out_dir, 'benchmarks_shard_map.json'))
|
|
overall_return_code = _run_benchmarks_on_shardmap(
|
|
shard_map, options, isolated_out_dir, test_results_files
|
|
)
|
|
# If the user has supplied a list of benchmark names, execute those instead
|
|
# of using the shard map.
|
|
elif options.benchmarks:
|
|
benchmarks = options.benchmarks.split(',')
|
|
for benchmark in benchmarks:
|
|
output_paths = OutputFilePaths(isolated_out_dir, benchmark).SetUp()
|
|
command_generator = TelemetryCommandGenerator(
|
|
benchmark, options)
|
|
print('\n### {folder} ###'.format(folder=benchmark))
|
|
return_code = execute_telemetry_benchmark(
|
|
command_generator, output_paths, options.xvfb,
|
|
options.ignore_benchmark_exit_code)
|
|
overall_return_code = return_code or overall_return_code
|
|
test_results_files.append(output_paths.test_results)
|
|
if options.run_ref_build:
|
|
print('Not running reference build. --run-ref-build argument is only '
|
|
'supported for sharded benchmarks. It is simple to support '
|
|
'this for unsharded --benchmarks if needed.')
|
|
elif options.test_shard_map_filename:
|
|
# First determine what shard we are running on to know how to
|
|
# index into the bot map to get list of telemetry benchmarks to run.
|
|
shard_map_path = os.path.join(SHARD_MAPS_DIRECTORY,
|
|
options.test_shard_map_filename)
|
|
# Copy sharding map file to isolated_out_dir so that the merge script
|
|
# can collect it later.
|
|
shutil.copyfile(
|
|
shard_map_path,
|
|
os.path.join(isolated_out_dir, 'benchmarks_shard_map.json'))
|
|
with open(shard_map_path) as f:
|
|
shard_map = json.load(f)
|
|
overall_return_code = _run_benchmarks_on_shardmap(
|
|
shard_map, options, isolated_out_dir, test_results_files
|
|
)
|
|
else:
|
|
raise Exception('Telemetry tests must provide either a shard map or a '
|
|
'--benchmarks list so that we know which stories to run.')
|
|
|
|
test_results_list = []
|
|
for test_results_file in test_results_files:
|
|
if os.path.exists(test_results_file):
|
|
with open(test_results_file, 'r') as fh:
|
|
test_results_list.append(json.load(fh))
|
|
merged_test_results = results_merger.merge_test_results(test_results_list)
|
|
with open(options.isolated_script_test_output, 'w') as f:
|
|
json.dump(merged_test_results, f)
|
|
|
|
return overall_return_code
|
|
|
|
def _run_benchmarks_on_shardmap(
|
|
shard_map, options, isolated_out_dir, test_results_files):
|
|
overall_return_code = 0
|
|
shard_index = None
|
|
env = os.environ.copy()
|
|
if 'GTEST_SHARD_INDEX' in env:
|
|
shard_index = env['GTEST_SHARD_INDEX']
|
|
# TODO(crbug.com/972844): shard environment variables are not specified
|
|
# for single-shard shard runs.
|
|
if not shard_index:
|
|
shard_map_has_multiple_shards = bool(shard_map.get('1', False))
|
|
if not shard_map_has_multiple_shards:
|
|
shard_index = '0'
|
|
if not shard_index:
|
|
raise Exception(
|
|
'Sharded Telemetry perf tests must either specify --benchmarks '
|
|
'list or have GTEST_SHARD_INDEX environment variable present.')
|
|
shard_configuration = shard_map[shard_index]
|
|
assert ('benchmarks' in shard_configuration or
|
|
'executables' in shard_configuration), (
|
|
'Every shard must have benchmarks or executables associated '
|
|
'with it.')
|
|
if 'benchmarks' in shard_configuration:
|
|
benchmarks_and_configs = shard_configuration['benchmarks']
|
|
for (benchmark, story_selection_config) in benchmarks_and_configs.items():
|
|
# Need to run the benchmark on both latest browser and reference
|
|
# build.
|
|
output_paths = OutputFilePaths(isolated_out_dir, benchmark).SetUp()
|
|
command_generator = TelemetryCommandGenerator(
|
|
benchmark, options,
|
|
story_selection_config=story_selection_config)
|
|
print('\n### {folder} ###'.format(folder=benchmark))
|
|
return_code = execute_telemetry_benchmark(
|
|
command_generator, output_paths, options.xvfb,
|
|
options.ignore_benchmark_exit_code)
|
|
overall_return_code = return_code or overall_return_code
|
|
test_results_files.append(output_paths.test_results)
|
|
if options.run_ref_build:
|
|
reference_benchmark_foldername = benchmark + '.reference'
|
|
reference_output_paths = OutputFilePaths(
|
|
isolated_out_dir, reference_benchmark_foldername).SetUp()
|
|
reference_command_generator = TelemetryCommandGenerator(
|
|
benchmark, options,
|
|
story_selection_config=story_selection_config,
|
|
is_reference=True)
|
|
print('\n### {folder} ###'.format(
|
|
folder=reference_benchmark_foldername))
|
|
# We intentionally ignore the return code and test results of the
|
|
# reference build.
|
|
execute_telemetry_benchmark(
|
|
reference_command_generator, reference_output_paths,
|
|
options.xvfb, options.ignore_benchmark_exit_code)
|
|
if 'executables' in shard_configuration:
|
|
names_and_configs = shard_configuration['executables']
|
|
for (name, configuration) in names_and_configs.items():
|
|
additional_flags = []
|
|
if 'arguments' in configuration:
|
|
additional_flags = configuration['arguments']
|
|
command_generator = GtestCommandGenerator(
|
|
options, override_executable=configuration['path'],
|
|
additional_flags=additional_flags, ignore_shard_env_vars=True)
|
|
output_paths = OutputFilePaths(isolated_out_dir, name).SetUp()
|
|
print('\n### {folder} ###'.format(folder=name))
|
|
return_code = execute_gtest_perf_test(
|
|
command_generator, output_paths, options.xvfb)
|
|
overall_return_code = return_code or overall_return_code
|
|
test_results_files.append(output_paths.test_results)
|
|
|
|
return overall_return_code
|
|
|
|
# This is not really a "script test" so does not need to manually add
|
|
# any additional compile targets.
|
|
def main_compile_targets(args):
|
|
json.dump([], args.output)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
# Conform minimally to the protocol defined by ScriptTest.
|
|
if 'compile_targets' in sys.argv:
|
|
funcs = {
|
|
'run': None,
|
|
'compile_targets': main_compile_targets,
|
|
}
|
|
sys.exit(common.run_script(sys.argv[1:], funcs))
|
|
|
|
sys.exit(main(sys.argv))
|