381 lines
14 KiB
Python
381 lines
14 KiB
Python
|
# Copyright 2015 The Chromium Authors
|
||
|
|
# Use of this source code is governed by a BSD-style license that can be
|
||
|
|
# found in the LICENSE file.
|
||
|
|
"""Presubmit script validating field trial configs.
|
||
|
|
|
||
|
|
See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
|
||
|
|
for more details on the presubmit API built into depot_tools.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import copy
|
||
|
|
import io
|
||
|
|
import json
|
||
|
|
import sys
|
||
|
|
|
||
|
|
from collections import OrderedDict
|
||
|
|
|
||
|
|
USE_PYTHON3 = True
|
||
|
|
|
||
|
|
VALID_EXPERIMENT_KEYS = [
|
||
|
|
'name', 'forcing_flag', 'params', 'enable_features', 'disable_features',
|
||
|
|
'min_os_version', '//0', '//1', '//2', '//3', '//4', '//5', '//6', '//7',
|
||
|
|
'//8', '//9'
|
||
|
|
]
|
||
|
|
|
||
|
|
FIELDTRIAL_CONFIG_FILE_NAME = 'fieldtrial_testing_config.json'
|
||
|
|
|
||
|
|
|
||
|
|
def PrettyPrint(contents):
|
||
|
|
"""Pretty prints a fieldtrial configuration.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
contents: File contents as a string.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
Pretty printed file contents.
|
||
|
|
"""
|
||
|
|
|
||
|
|
# We have a preferred ordering of the fields (e.g. platforms on top). This
|
||
|
|
# code loads everything into OrderedDicts and then tells json to dump it out.
|
||
|
|
# The JSON dumper will respect the dict ordering.
|
||
|
|
#
|
||
|
|
# The ordering is as follows:
|
||
|
|
# {
|
||
|
|
# 'StudyName Alphabetical': [
|
||
|
|
# {
|
||
|
|
# 'platforms': [sorted platforms]
|
||
|
|
# 'groups': [
|
||
|
|
# {
|
||
|
|
# name: ...
|
||
|
|
# forcing_flag: "forcing flag string"
|
||
|
|
# params: {sorted dict}
|
||
|
|
# enable_features: [sorted features]
|
||
|
|
# disable_features: [sorted features]
|
||
|
|
# (Unexpected extra keys will be caught by the validator)
|
||
|
|
# }
|
||
|
|
# ],
|
||
|
|
# ....
|
||
|
|
# },
|
||
|
|
# ...
|
||
|
|
# ]
|
||
|
|
# ...
|
||
|
|
# }
|
||
|
|
config = json.loads(contents)
|
||
|
|
ordered_config = OrderedDict()
|
||
|
|
for key in sorted(config.keys()):
|
||
|
|
study = copy.deepcopy(config[key])
|
||
|
|
ordered_study = []
|
||
|
|
for experiment_config in study:
|
||
|
|
ordered_experiment_config = OrderedDict([('platforms',
|
||
|
|
experiment_config['platforms']),
|
||
|
|
('experiments', [])])
|
||
|
|
for experiment in experiment_config['experiments']:
|
||
|
|
ordered_experiment = OrderedDict()
|
||
|
|
for index in range(0, 10):
|
||
|
|
comment_key = '//' + str(index)
|
||
|
|
if comment_key in experiment:
|
||
|
|
ordered_experiment[comment_key] = experiment[comment_key]
|
||
|
|
ordered_experiment['name'] = experiment['name']
|
||
|
|
if 'forcing_flag' in experiment:
|
||
|
|
ordered_experiment['forcing_flag'] = experiment['forcing_flag']
|
||
|
|
if 'params' in experiment:
|
||
|
|
ordered_experiment['params'] = OrderedDict(
|
||
|
|
sorted(experiment['params'].items(), key=lambda t: t[0]))
|
||
|
|
if 'enable_features' in experiment:
|
||
|
|
ordered_experiment['enable_features'] = \
|
||
|
|
sorted(experiment['enable_features'])
|
||
|
|
if 'disable_features' in experiment:
|
||
|
|
ordered_experiment['disable_features'] = \
|
||
|
|
sorted(experiment['disable_features'])
|
||
|
|
ordered_experiment_config['experiments'].append(ordered_experiment)
|
||
|
|
if 'min_os_version' in experiment:
|
||
|
|
ordered_experiment['min_os_version'] = experiment['min_os_version']
|
||
|
|
ordered_study.append(ordered_experiment_config)
|
||
|
|
ordered_config[key] = ordered_study
|
||
|
|
return json.dumps(
|
||
|
|
ordered_config, sort_keys=False, indent=4, separators=(',', ': ')) + '\n'
|
||
|
|
|
||
|
|
|
||
|
|
def ValidateData(json_data, file_path, message_type):
|
||
|
|
"""Validates the format of a fieldtrial configuration.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
json_data: Parsed JSON object representing the fieldtrial config.
|
||
|
|
file_path: String representing the path to the JSON file.
|
||
|
|
message_type: Type of message from |output_api| to return in the case of
|
||
|
|
errors/warnings.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
A list of |message_type| messages. In the case of all tests passing with no
|
||
|
|
warnings/errors, this will return [].
|
||
|
|
"""
|
||
|
|
|
||
|
|
def _CreateMessage(message_format, *args):
|
||
|
|
return _CreateMalformedConfigMessage(message_type, file_path,
|
||
|
|
message_format, *args)
|
||
|
|
|
||
|
|
if not isinstance(json_data, dict):
|
||
|
|
return _CreateMessage('Expecting dict')
|
||
|
|
for (study, experiment_configs) in iter(json_data.items()):
|
||
|
|
warnings = _ValidateEntry(study, experiment_configs, _CreateMessage)
|
||
|
|
if warnings:
|
||
|
|
return warnings
|
||
|
|
|
||
|
|
return []
|
||
|
|
|
||
|
|
|
||
|
|
def _ValidateEntry(study, experiment_configs, create_message_fn):
|
||
|
|
"""Validates one entry of the field trial configuration."""
|
||
|
|
if not isinstance(study, str):
|
||
|
|
return create_message_fn('Expecting keys to be string, got %s', type(study))
|
||
|
|
if not isinstance(experiment_configs, list):
|
||
|
|
return create_message_fn('Expecting list for study %s', study)
|
||
|
|
|
||
|
|
# Add context to other messages.
|
||
|
|
def _CreateStudyMessage(message_format, *args):
|
||
|
|
suffix = ' in Study[%s]' % study
|
||
|
|
return create_message_fn(message_format + suffix, *args)
|
||
|
|
|
||
|
|
for experiment_config in experiment_configs:
|
||
|
|
warnings = _ValidateExperimentConfig(experiment_config, _CreateStudyMessage)
|
||
|
|
if warnings:
|
||
|
|
return warnings
|
||
|
|
return []
|
||
|
|
|
||
|
|
|
||
|
|
def _ValidateExperimentConfig(experiment_config, create_message_fn):
|
||
|
|
"""Validates one config in a configuration entry."""
|
||
|
|
if not isinstance(experiment_config, dict):
|
||
|
|
return create_message_fn('Expecting dict for experiment config')
|
||
|
|
if not 'experiments' in experiment_config:
|
||
|
|
return create_message_fn('Missing valid experiments for experiment config')
|
||
|
|
if not isinstance(experiment_config['experiments'], list):
|
||
|
|
return create_message_fn('Expecting list for experiments')
|
||
|
|
for experiment_group in experiment_config['experiments']:
|
||
|
|
warnings = _ValidateExperimentGroup(experiment_group, create_message_fn)
|
||
|
|
if warnings:
|
||
|
|
return warnings
|
||
|
|
if not 'platforms' in experiment_config:
|
||
|
|
return create_message_fn('Missing valid platforms for experiment config')
|
||
|
|
if not isinstance(experiment_config['platforms'], list):
|
||
|
|
return create_message_fn('Expecting list for platforms')
|
||
|
|
supported_platforms = [
|
||
|
|
'android', 'android_weblayer', 'android_webview', 'chromeos',
|
||
|
|
'chromeos_lacros', 'fuchsia', 'ios', 'linux', 'mac', 'windows'
|
||
|
|
]
|
||
|
|
experiment_platforms = experiment_config['platforms']
|
||
|
|
unsupported_platforms = list(
|
||
|
|
set(experiment_platforms).difference(supported_platforms))
|
||
|
|
if unsupported_platforms:
|
||
|
|
return create_message_fn('Unsupported platforms %s', unsupported_platforms)
|
||
|
|
return []
|
||
|
|
|
||
|
|
|
||
|
|
def _ValidateExperimentGroup(experiment_group, create_message_fn):
|
||
|
|
"""Validates one group of one config in a configuration entry."""
|
||
|
|
name = experiment_group.get('name', '')
|
||
|
|
if not name or not isinstance(name, str):
|
||
|
|
return create_message_fn('Missing valid name for experiment')
|
||
|
|
|
||
|
|
# Add context to other messages.
|
||
|
|
def _CreateGroupMessage(message_format, *args):
|
||
|
|
suffix = ' in Group[%s]' % name
|
||
|
|
return create_message_fn(message_format + suffix, *args)
|
||
|
|
|
||
|
|
if 'params' in experiment_group:
|
||
|
|
params = experiment_group['params']
|
||
|
|
if not isinstance(params, dict):
|
||
|
|
return _CreateGroupMessage('Expected dict for params')
|
||
|
|
for (key, value) in iter(params.items()):
|
||
|
|
if not isinstance(key, str) or not isinstance(value, str):
|
||
|
|
return _CreateGroupMessage('Invalid param (%s: %s)', key, value)
|
||
|
|
for key in experiment_group.keys():
|
||
|
|
if key not in VALID_EXPERIMENT_KEYS:
|
||
|
|
return _CreateGroupMessage('Key[%s] is not a valid key', key)
|
||
|
|
return []
|
||
|
|
|
||
|
|
|
||
|
|
def _CreateMalformedConfigMessage(message_type, file_path, message_format,
|
||
|
|
*args):
|
||
|
|
"""Returns a list containing one |message_type| with the error message.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
message_type: Type of message from |output_api| to return in the case of
|
||
|
|
errors/warnings.
|
||
|
|
message_format: The error message format string.
|
||
|
|
file_path: The path to the config file.
|
||
|
|
*args: The args for message_format.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
A list containing a message_type with a formatted error message and
|
||
|
|
'Malformed config file [file]: ' prepended to it.
|
||
|
|
"""
|
||
|
|
error_message_format = 'Malformed config file %s: ' + message_format
|
||
|
|
format_args = (file_path,) + args
|
||
|
|
return [message_type(error_message_format % format_args)]
|
||
|
|
|
||
|
|
|
||
|
|
def CheckPretty(contents, file_path, message_type):
|
||
|
|
"""Validates the pretty printing of fieldtrial configuration.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
contents: File contents as a string.
|
||
|
|
file_path: String representing the path to the JSON file.
|
||
|
|
message_type: Type of message from |output_api| to return in the case of
|
||
|
|
errors/warnings.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
A list of |message_type| messages. In the case of all tests passing with no
|
||
|
|
warnings/errors, this will return [].
|
||
|
|
"""
|
||
|
|
pretty = PrettyPrint(contents)
|
||
|
|
if contents != pretty:
|
||
|
|
return [
|
||
|
|
message_type('Pretty printing error: Run '
|
||
|
|
'python3 testing/variations/PRESUBMIT.py %s' % file_path)
|
||
|
|
]
|
||
|
|
return []
|
||
|
|
|
||
|
|
def _GetStudyConfigFeatures(study_config):
|
||
|
|
"""Gets the set of features overridden in a study config."""
|
||
|
|
features = set()
|
||
|
|
for experiment in study_config.get("experiments", []):
|
||
|
|
features.update(experiment.get("enable_features", []))
|
||
|
|
features.update(experiment.get("disable_features", []))
|
||
|
|
return features
|
||
|
|
|
||
|
|
def _GetDuplicatedFeatures(study1, study2):
|
||
|
|
"""Gets the set of features that are overridden in two overlapping studies."""
|
||
|
|
duplicated_features = set()
|
||
|
|
for study_config1 in study1:
|
||
|
|
features = _GetStudyConfigFeatures(study_config1)
|
||
|
|
platforms = set(study_config1.get("platforms", []))
|
||
|
|
for study_config2 in study2:
|
||
|
|
# If the study configs do not specify any common platform, they do not
|
||
|
|
# overlap, so we can skip them.
|
||
|
|
if platforms.isdisjoint(set(study_config2.get("platforms", []))):
|
||
|
|
continue
|
||
|
|
|
||
|
|
common_features = features & _GetStudyConfigFeatures(study_config2)
|
||
|
|
duplicated_features.update(common_features)
|
||
|
|
|
||
|
|
return duplicated_features
|
||
|
|
|
||
|
|
def CheckDuplicatedFeatures(new_json_data, old_json_data, message_type):
|
||
|
|
"""Validates that features are not specified in multiple studies.
|
||
|
|
|
||
|
|
Note that a feature may be specified in different studies that do not overlap.
|
||
|
|
For example, if they specify different platforms. In such a case, this will
|
||
|
|
not give a warning/error. However, it is possible that this incorrectly
|
||
|
|
gives an error, as it is possible for studies to have complex filters (e.g.,
|
||
|
|
if they make use of additional filters such as form_factors,
|
||
|
|
is_low_end_device, etc.). In those cases, the PRESUBMIT check can be bypassed.
|
||
|
|
Since this will only check for studies that were changed in this particular
|
||
|
|
commit, bypassing the PRESUBMIT check will not block future commits.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
new_json_data: Parsed JSON object representing the new fieldtrial config.
|
||
|
|
old_json_data: Parsed JSON object representing the old fieldtrial config.
|
||
|
|
message_type: Type of message from |output_api| to return in the case of
|
||
|
|
errors/warnings.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
A list of |message_type| messages. In the case of all tests passing with no
|
||
|
|
warnings/errors, this will return [].
|
||
|
|
"""
|
||
|
|
# Get list of studies that changed.
|
||
|
|
changed_studies = []
|
||
|
|
for study_name in new_json_data:
|
||
|
|
if (study_name not in old_json_data or
|
||
|
|
new_json_data[study_name] != old_json_data[study_name]):
|
||
|
|
changed_studies.append(study_name)
|
||
|
|
|
||
|
|
# A map between a feature name and the name of studies that use it. E.g.,
|
||
|
|
# duplicated_features_to_studies_map["FeatureA"] = {"StudyA", "StudyB"}.
|
||
|
|
# Only features that are defined in multiple studies are added to this map.
|
||
|
|
duplicated_features_to_studies_map = dict()
|
||
|
|
|
||
|
|
# Compare the changed studies against all studies defined.
|
||
|
|
for changed_study_name in changed_studies:
|
||
|
|
for study_name in new_json_data:
|
||
|
|
if changed_study_name == study_name:
|
||
|
|
continue
|
||
|
|
|
||
|
|
duplicated_features = _GetDuplicatedFeatures(
|
||
|
|
new_json_data[changed_study_name], new_json_data[study_name])
|
||
|
|
|
||
|
|
for feature in duplicated_features:
|
||
|
|
if feature not in duplicated_features_to_studies_map:
|
||
|
|
duplicated_features_to_studies_map[feature] = set()
|
||
|
|
duplicated_features_to_studies_map[feature].update(
|
||
|
|
[changed_study_name, study_name])
|
||
|
|
|
||
|
|
if len(duplicated_features_to_studies_map) == 0:
|
||
|
|
return []
|
||
|
|
|
||
|
|
duplicated_features_strings = [
|
||
|
|
"%s (in studies %s)" % (feature, ', '.join(studies))
|
||
|
|
for feature, studies in duplicated_features_to_studies_map.items()
|
||
|
|
]
|
||
|
|
|
||
|
|
return [
|
||
|
|
message_type('The following feature(s) were specified in multiple '
|
||
|
|
'studies: %s' % ', '.join(duplicated_features_strings))
|
||
|
|
]
|
||
|
|
|
||
|
|
def CommonChecks(input_api, output_api):
|
||
|
|
affected_files = input_api.AffectedFiles(
|
||
|
|
include_deletes=False,
|
||
|
|
file_filter=lambda x: x.LocalPath().endswith('.json'))
|
||
|
|
for f in affected_files:
|
||
|
|
if not f.LocalPath().endswith(FIELDTRIAL_CONFIG_FILE_NAME):
|
||
|
|
return [
|
||
|
|
output_api.PresubmitError(
|
||
|
|
'%s is the only json file expected in this folder. If new jsons '
|
||
|
|
'are added, please update the presubmit process with proper '
|
||
|
|
'validation. ' % FIELDTRIAL_CONFIG_FILE_NAME
|
||
|
|
)
|
||
|
|
]
|
||
|
|
contents = input_api.ReadFile(f)
|
||
|
|
try:
|
||
|
|
json_data = input_api.json.loads(contents)
|
||
|
|
result = ValidateData(
|
||
|
|
json_data,
|
||
|
|
f.AbsoluteLocalPath(),
|
||
|
|
output_api.PresubmitError)
|
||
|
|
if result:
|
||
|
|
return result
|
||
|
|
result = CheckPretty(contents, f.LocalPath(), output_api.PresubmitError)
|
||
|
|
if result:
|
||
|
|
return result
|
||
|
|
result = CheckDuplicatedFeatures(
|
||
|
|
json_data,
|
||
|
|
input_api.json.loads('\n'.join(f.OldContents())),
|
||
|
|
output_api.PresubmitError)
|
||
|
|
if result:
|
||
|
|
return result
|
||
|
|
except ValueError:
|
||
|
|
return [
|
||
|
|
output_api.PresubmitError('Malformed JSON file: %s' % f.LocalPath())
|
||
|
|
]
|
||
|
|
return []
|
||
|
|
|
||
|
|
|
||
|
|
def CheckChangeOnUpload(input_api, output_api):
|
||
|
|
return CommonChecks(input_api, output_api)
|
||
|
|
|
||
|
|
|
||
|
|
def CheckChangeOnCommit(input_api, output_api):
|
||
|
|
return CommonChecks(input_api, output_api)
|
||
|
|
|
||
|
|
|
||
|
|
def main(argv):
|
||
|
|
with io.open(argv[1], encoding='utf-8') as f:
|
||
|
|
content = f.read()
|
||
|
|
pretty = PrettyPrint(content)
|
||
|
|
io.open(argv[1], 'wb').write(pretty.encode('utf-8'))
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
sys.exit(main(sys.argv))
|