89 lines
4.3 KiB
Python
89 lines
4.3 KiB
Python
|
# Copyright 2021 The Chromium Authors
|
||
|
|
# Use of this source code is governed by a BSD-style license that can be
|
||
|
|
# found in the LICENSE file.
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
|
||
|
|
|
||
|
|
def ParseArgs():
|
||
|
|
parser = argparse.ArgumentParser(
|
||
|
|
description=('Script for automatically suppressing flaky/failing '
|
||
|
|
'tests.'))
|
||
|
|
parser.add_argument('--project',
|
||
|
|
required=True,
|
||
|
|
help=('The billing project to use for BigQuery queries. '
|
||
|
|
'Must have access to the ResultDB BQ tables, e.g. '
|
||
|
|
'"chrome-luci-data.chromium.gpu_ci_test_results".'))
|
||
|
|
parser.add_argument('--sample-period',
|
||
|
|
type=int,
|
||
|
|
default=1,
|
||
|
|
help=('The number of days to sample data from.'))
|
||
|
|
parser.add_argument('--no-group-by-tags',
|
||
|
|
action='store_false',
|
||
|
|
default=True,
|
||
|
|
dest='group_by_tags',
|
||
|
|
help=('Append added expectations to the end of the file '
|
||
|
|
'instead of attempting to automatically group with '
|
||
|
|
'similar expectations.'))
|
||
|
|
parser.add_argument('--no-prompt-for-user-input',
|
||
|
|
action='store_false',
|
||
|
|
default=True,
|
||
|
|
dest='prompt_for_user_input',
|
||
|
|
help=('Generate expectations automatically based on '
|
||
|
|
'thresholds instead of prompting the user each '
|
||
|
|
'time. The user will still need to add associated '
|
||
|
|
'bugs to generated expectations afterwards.'))
|
||
|
|
parser.add_argument('--ignore-threshold',
|
||
|
|
type=float,
|
||
|
|
default=0.01,
|
||
|
|
help=('The fraction of failed tests under which flakes '
|
||
|
|
'will be ignored instead of having an expectation '
|
||
|
|
'added when --no-prompt-for-user-input is used.'))
|
||
|
|
parser.add_argument('--flaky-threshold',
|
||
|
|
type=float,
|
||
|
|
default=0.5,
|
||
|
|
help=('The fraction of failed tests under which flakes '
|
||
|
|
'will be marked as RetryOnFailure when '
|
||
|
|
'--no-prompt-for-user-input is used. Above this, '
|
||
|
|
'failures will be marked as Failure.'))
|
||
|
|
parser.add_argument('--include-all-tags',
|
||
|
|
action='store_true',
|
||
|
|
default=False,
|
||
|
|
help=('Use all tags generated by a configuration when '
|
||
|
|
'creating an expectation rather than attempting '
|
||
|
|
'to only use the most specific one. This should '
|
||
|
|
'only need to be passed if the tags in the '
|
||
|
|
'expectation files are not ordered from least '
|
||
|
|
'specific to most specific.'))
|
||
|
|
parser.add_argument('--result-output-file',
|
||
|
|
help=('Output file to store the generated results. If '
|
||
|
|
'not specified, will use a temporary file.'))
|
||
|
|
parser.add_argument('--bypass-up-to-date-check',
|
||
|
|
action='store_true',
|
||
|
|
default=False,
|
||
|
|
help=('Bypasses the check that the local expectation '
|
||
|
|
'files are up to date. Only intended for use on '
|
||
|
|
'bots to avoid failures due to potential race '
|
||
|
|
'conditions between updating the checkout and '
|
||
|
|
'running the script.'))
|
||
|
|
parser.add_argument(
|
||
|
|
'--non-hidden-failures',
|
||
|
|
action='store_true',
|
||
|
|
default=False,
|
||
|
|
help=
|
||
|
|
('Enable this option to only targeting visible failures on CI builders. '
|
||
|
|
'The test results will fail the builder runs, flaky results will '
|
||
|
|
'consider as pass in this option.'))
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
if not args.prompt_for_user_input:
|
||
|
|
if args.ignore_threshold < 0:
|
||
|
|
raise ValueError('--ignore-threshold must be positive')
|
||
|
|
if args.flaky_threshold < 0:
|
||
|
|
raise ValueError('--flaky-threshold must be positive')
|
||
|
|
if args.flaky_threshold <= args.ignore_threshold:
|
||
|
|
raise ValueError(
|
||
|
|
'--flaky-threshold must be greater than --ignore-threshold')
|
||
|
|
|
||
|
|
return args
|