1160 lines
51 KiB
Python
1160 lines
51 KiB
Python
|
#!/usr/bin/env python3.4
|
||
|
|
#
|
||
|
|
# Copyright 2017 - The Android Open Source Project
|
||
|
|
#
|
||
|
|
# Licensed under the Apache License, Version 2.0 (the 'License');
|
||
|
|
# you may not use this file except in compliance with the License.
|
||
|
|
# You may obtain a copy of the License at
|
||
|
|
#
|
||
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
#
|
||
|
|
# Unless required by applicable law or agreed to in writing, software
|
||
|
|
# distributed under the License is distributed on an 'AS IS' BASIS,
|
||
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
# See the License for the specific language governing permissions and
|
||
|
|
# limitations under the License.
|
||
|
|
|
||
|
|
import collections
|
||
|
|
import itertools
|
||
|
|
import json
|
||
|
|
import logging
|
||
|
|
import numpy
|
||
|
|
import os
|
||
|
|
import time
|
||
|
|
from acts import asserts
|
||
|
|
from acts import base_test
|
||
|
|
from acts import utils
|
||
|
|
from acts.controllers import iperf_server as ipf
|
||
|
|
from acts.controllers.utils_lib import ssh
|
||
|
|
from acts.metrics.loggers.blackbox import BlackboxMappedMetricLogger
|
||
|
|
from acts_contrib.test_utils.wifi import ota_chamber
|
||
|
|
from acts_contrib.test_utils.wifi import ota_sniffer
|
||
|
|
from acts_contrib.test_utils.wifi import wifi_performance_test_utils as wputils
|
||
|
|
from acts_contrib.test_utils.wifi.wifi_performance_test_utils.bokeh_figure import BokehFigure
|
||
|
|
from acts_contrib.test_utils.wifi import wifi_retail_ap as retail_ap
|
||
|
|
from acts_contrib.test_utils.wifi import wifi_test_utils as wutils
|
||
|
|
from functools import partial
|
||
|
|
|
||
|
|
|
||
|
|
class WifiRvrTest(base_test.BaseTestClass):
|
||
|
|
"""Class to test WiFi rate versus range.
|
||
|
|
|
||
|
|
This class implements WiFi rate versus range tests on single AP single STA
|
||
|
|
links. The class setups up the AP in the desired configurations, configures
|
||
|
|
and connects the phone to the AP, and runs iperf throughput test while
|
||
|
|
sweeping attenuation. For an example config file to run this test class see
|
||
|
|
example_connectivity_performance_ap_sta.json.
|
||
|
|
"""
|
||
|
|
|
||
|
|
TEST_TIMEOUT = 6
|
||
|
|
MAX_CONSECUTIVE_ZEROS = 3
|
||
|
|
|
||
|
|
def __init__(self, controllers):
|
||
|
|
base_test.BaseTestClass.__init__(self, controllers)
|
||
|
|
self.testcase_metric_logger = (
|
||
|
|
BlackboxMappedMetricLogger.for_test_case())
|
||
|
|
self.testclass_metric_logger = (
|
||
|
|
BlackboxMappedMetricLogger.for_test_class())
|
||
|
|
self.publish_testcase_metrics = True
|
||
|
|
|
||
|
|
def setup_class(self):
|
||
|
|
"""Initializes common test hardware and parameters.
|
||
|
|
|
||
|
|
This function initializes hardwares and compiles parameters that are
|
||
|
|
common to all tests in this class.
|
||
|
|
"""
|
||
|
|
self.sta_dut = self.android_devices[0]
|
||
|
|
req_params = [
|
||
|
|
'RetailAccessPoints', 'rvr_test_params', 'testbed_params',
|
||
|
|
'RemoteServer', 'main_network'
|
||
|
|
]
|
||
|
|
opt_params = ['golden_files_list', 'OTASniffer']
|
||
|
|
self.unpack_userparams(req_params, opt_params)
|
||
|
|
self.testclass_params = self.rvr_test_params
|
||
|
|
self.num_atten = self.attenuators[0].instrument.num_atten
|
||
|
|
self.iperf_server = self.iperf_servers[0]
|
||
|
|
self.remote_server = ssh.connection.SshConnection(
|
||
|
|
ssh.settings.from_config(self.RemoteServer[0]['ssh_config']))
|
||
|
|
self.iperf_client = self.iperf_clients[0]
|
||
|
|
self.access_point = retail_ap.create(self.RetailAccessPoints)[0]
|
||
|
|
if hasattr(self,
|
||
|
|
'OTASniffer') and self.testbed_params['sniffer_enable']:
|
||
|
|
try:
|
||
|
|
self.sniffer = ota_sniffer.create(self.OTASniffer)[0]
|
||
|
|
except:
|
||
|
|
self.log.warning('Could not start sniffer. Disabling sniffs.')
|
||
|
|
self.testbed_params['sniffer_enable'] = 0
|
||
|
|
self.log.info('Access Point Configuration: {}'.format(
|
||
|
|
self.access_point.ap_settings))
|
||
|
|
self.log_path = os.path.join(logging.log_path, 'results')
|
||
|
|
os.makedirs(self.log_path, exist_ok=True)
|
||
|
|
if not hasattr(self, 'golden_files_list'):
|
||
|
|
if 'golden_results_path' in self.testbed_params:
|
||
|
|
self.golden_files_list = [
|
||
|
|
os.path.join(self.testbed_params['golden_results_path'],
|
||
|
|
file) for file in
|
||
|
|
os.listdir(self.testbed_params['golden_results_path'])
|
||
|
|
]
|
||
|
|
else:
|
||
|
|
self.log.warning('No golden files found.')
|
||
|
|
self.golden_files_list = []
|
||
|
|
self.testclass_results = []
|
||
|
|
|
||
|
|
# Turn WiFi ON
|
||
|
|
if self.testclass_params.get('airplane_mode', 1):
|
||
|
|
for dev in self.android_devices:
|
||
|
|
self.log.info('Turning on airplane mode.')
|
||
|
|
asserts.assert_true(utils.force_airplane_mode(dev, True),
|
||
|
|
'Can not turn on airplane mode.')
|
||
|
|
wutils.reset_wifi(dev)
|
||
|
|
wutils.wifi_toggle_state(dev, True)
|
||
|
|
|
||
|
|
def teardown_test(self):
|
||
|
|
self.iperf_server.stop()
|
||
|
|
|
||
|
|
def teardown_class(self):
|
||
|
|
# Turn WiFi OFF
|
||
|
|
self.access_point.teardown()
|
||
|
|
for dev in self.android_devices:
|
||
|
|
wutils.wifi_toggle_state(dev, False)
|
||
|
|
dev.go_to_sleep()
|
||
|
|
self.process_testclass_results()
|
||
|
|
|
||
|
|
def process_testclass_results(self):
|
||
|
|
"""Saves plot with all test results to enable comparison."""
|
||
|
|
# Plot and save all results
|
||
|
|
plots = collections.OrderedDict()
|
||
|
|
for result in self.testclass_results:
|
||
|
|
plot_id = (result['testcase_params']['channel'],
|
||
|
|
result['testcase_params']['mode'])
|
||
|
|
if plot_id not in plots:
|
||
|
|
plots[plot_id] = BokehFigure(
|
||
|
|
title='Channel {} {} ({})'.format(
|
||
|
|
result['testcase_params']['channel'],
|
||
|
|
result['testcase_params']['mode'],
|
||
|
|
result['testcase_params']['traffic_type']),
|
||
|
|
x_label='Attenuation (dB)',
|
||
|
|
primary_y_label='Throughput (Mbps)')
|
||
|
|
plots[plot_id].add_line(result['total_attenuation'],
|
||
|
|
result['throughput_receive'],
|
||
|
|
result['test_name'].strip('test_rvr_'),
|
||
|
|
hover_text=result['hover_text'],
|
||
|
|
marker='circle')
|
||
|
|
plots[plot_id].add_line(result['total_attenuation'],
|
||
|
|
result['rx_phy_rate'],
|
||
|
|
result['test_name'].strip('test_rvr_') +
|
||
|
|
' (Rx PHY)',
|
||
|
|
hover_text=result['hover_text'],
|
||
|
|
style='dashed',
|
||
|
|
marker='inverted_triangle')
|
||
|
|
plots[plot_id].add_line(result['total_attenuation'],
|
||
|
|
result['tx_phy_rate'],
|
||
|
|
result['test_name'].strip('test_rvr_') +
|
||
|
|
' (Tx PHY)',
|
||
|
|
hover_text=result['hover_text'],
|
||
|
|
style='dashed',
|
||
|
|
marker='triangle')
|
||
|
|
|
||
|
|
figure_list = []
|
||
|
|
for plot_id, plot in plots.items():
|
||
|
|
plot.generate_figure()
|
||
|
|
figure_list.append(plot)
|
||
|
|
output_file_path = os.path.join(self.log_path, 'results.html')
|
||
|
|
BokehFigure.save_figures(figure_list, output_file_path)
|
||
|
|
|
||
|
|
def pass_fail_check(self, rvr_result):
|
||
|
|
"""Check the test result and decide if it passed or failed.
|
||
|
|
|
||
|
|
Checks the RvR test result and compares to a throughput limites for
|
||
|
|
the same configuration. The pass/fail tolerances are provided in the
|
||
|
|
config file.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
rvr_result: dict containing attenuation, throughput and other data
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
throughput_limits = self.compute_throughput_limits(rvr_result)
|
||
|
|
except:
|
||
|
|
asserts.explicit_pass(
|
||
|
|
'Test passed by default. Golden file not found')
|
||
|
|
|
||
|
|
failure_count = 0
|
||
|
|
for idx, current_throughput in enumerate(
|
||
|
|
rvr_result['throughput_receive']):
|
||
|
|
if (current_throughput < throughput_limits['lower_limit'][idx]
|
||
|
|
or current_throughput >
|
||
|
|
throughput_limits['upper_limit'][idx]):
|
||
|
|
failure_count = failure_count + 1
|
||
|
|
|
||
|
|
# Set test metrics
|
||
|
|
rvr_result['metrics']['failure_count'] = failure_count
|
||
|
|
if self.publish_testcase_metrics:
|
||
|
|
self.testcase_metric_logger.add_metric('failure_count',
|
||
|
|
failure_count)
|
||
|
|
|
||
|
|
# Assert pass or fail
|
||
|
|
if failure_count >= self.testclass_params['failure_count_tolerance']:
|
||
|
|
asserts.fail('Test failed. Found {} points outside limits.'.format(
|
||
|
|
failure_count))
|
||
|
|
asserts.explicit_pass(
|
||
|
|
'Test passed. Found {} points outside throughput limits.'.format(
|
||
|
|
failure_count))
|
||
|
|
|
||
|
|
def compute_throughput_limits(self, rvr_result):
|
||
|
|
"""Compute throughput limits for current test.
|
||
|
|
|
||
|
|
Checks the RvR test result and compares to a throughput limites for
|
||
|
|
the same configuration. The pass/fail tolerances are provided in the
|
||
|
|
config file.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
rvr_result: dict containing attenuation, throughput and other meta
|
||
|
|
data
|
||
|
|
Returns:
|
||
|
|
throughput_limits: dict containing attenuation and throughput limit data
|
||
|
|
"""
|
||
|
|
test_name = self.current_test_name
|
||
|
|
golden_path = next(file_name for file_name in self.golden_files_list
|
||
|
|
if test_name in file_name)
|
||
|
|
with open(golden_path, 'r') as golden_file:
|
||
|
|
golden_results = json.load(golden_file)
|
||
|
|
golden_attenuation = [
|
||
|
|
att + golden_results['fixed_attenuation']
|
||
|
|
for att in golden_results['attenuation']
|
||
|
|
]
|
||
|
|
attenuation = []
|
||
|
|
lower_limit = []
|
||
|
|
upper_limit = []
|
||
|
|
for idx, current_throughput in enumerate(
|
||
|
|
rvr_result['throughput_receive']):
|
||
|
|
current_att = rvr_result['attenuation'][idx] + rvr_result[
|
||
|
|
'fixed_attenuation']
|
||
|
|
att_distances = [
|
||
|
|
abs(current_att - golden_att)
|
||
|
|
for golden_att in golden_attenuation
|
||
|
|
]
|
||
|
|
sorted_distances = sorted(enumerate(att_distances),
|
||
|
|
key=lambda x: x[1])
|
||
|
|
closest_indeces = [dist[0] for dist in sorted_distances[0:3]]
|
||
|
|
closest_throughputs = [
|
||
|
|
golden_results['throughput_receive'][index]
|
||
|
|
for index in closest_indeces
|
||
|
|
]
|
||
|
|
closest_throughputs.sort()
|
||
|
|
|
||
|
|
attenuation.append(current_att)
|
||
|
|
lower_limit.append(
|
||
|
|
max(
|
||
|
|
closest_throughputs[0] - max(
|
||
|
|
self.testclass_params['abs_tolerance'],
|
||
|
|
closest_throughputs[0] *
|
||
|
|
self.testclass_params['pct_tolerance'] / 100), 0))
|
||
|
|
upper_limit.append(closest_throughputs[-1] + max(
|
||
|
|
self.testclass_params['abs_tolerance'], closest_throughputs[-1]
|
||
|
|
* self.testclass_params['pct_tolerance'] / 100))
|
||
|
|
throughput_limits = {
|
||
|
|
'attenuation': attenuation,
|
||
|
|
'lower_limit': lower_limit,
|
||
|
|
'upper_limit': upper_limit
|
||
|
|
}
|
||
|
|
return throughput_limits
|
||
|
|
|
||
|
|
def plot_rvr_result(self, rvr_result):
|
||
|
|
"""Saves plots and JSON formatted results.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
rvr_result: dict containing attenuation, throughput and other meta
|
||
|
|
data
|
||
|
|
"""
|
||
|
|
# Save output as text file
|
||
|
|
results_file_path = os.path.join(
|
||
|
|
self.log_path, '{}.json'.format(self.current_test_name))
|
||
|
|
with open(results_file_path, 'w') as results_file:
|
||
|
|
json.dump(wputils.serialize_dict(rvr_result),
|
||
|
|
results_file,
|
||
|
|
indent=4)
|
||
|
|
# Plot and save
|
||
|
|
figure = BokehFigure(title=self.current_test_name,
|
||
|
|
x_label='Attenuation (dB)',
|
||
|
|
primary_y_label='Throughput (Mbps)')
|
||
|
|
try:
|
||
|
|
golden_path = next(file_name
|
||
|
|
for file_name in self.golden_files_list
|
||
|
|
if self.current_test_name in file_name)
|
||
|
|
with open(golden_path, 'r') as golden_file:
|
||
|
|
golden_results = json.load(golden_file)
|
||
|
|
golden_attenuation = [
|
||
|
|
att + golden_results['fixed_attenuation']
|
||
|
|
for att in golden_results['attenuation']
|
||
|
|
]
|
||
|
|
throughput_limits = self.compute_throughput_limits(rvr_result)
|
||
|
|
shaded_region = {
|
||
|
|
'x_vector': throughput_limits['attenuation'],
|
||
|
|
'lower_limit': throughput_limits['lower_limit'],
|
||
|
|
'upper_limit': throughput_limits['upper_limit']
|
||
|
|
}
|
||
|
|
figure.add_line(golden_attenuation,
|
||
|
|
golden_results['throughput_receive'],
|
||
|
|
'Golden Results',
|
||
|
|
color='green',
|
||
|
|
marker='circle',
|
||
|
|
shaded_region=shaded_region)
|
||
|
|
except:
|
||
|
|
self.log.warning('ValueError: Golden file not found')
|
||
|
|
|
||
|
|
# Generate graph annotatios
|
||
|
|
rvr_result['hover_text'] = {
|
||
|
|
'llstats': [
|
||
|
|
'TX MCS = {0} ({1:.1f}%). RX MCS = {2} ({3:.1f}%)'.format(
|
||
|
|
curr_llstats['summary']['common_tx_mcs'],
|
||
|
|
curr_llstats['summary']['common_tx_mcs_freq'] * 100,
|
||
|
|
curr_llstats['summary']['common_rx_mcs'],
|
||
|
|
curr_llstats['summary']['common_rx_mcs_freq'] * 100)
|
||
|
|
for curr_llstats in rvr_result['llstats']
|
||
|
|
],
|
||
|
|
'rssi': [
|
||
|
|
'{0:.2f} [{1:.2f},{2:.2f}]'.format(
|
||
|
|
rssi['signal_poll_rssi'],
|
||
|
|
rssi['chain_0_rssi'],
|
||
|
|
rssi['chain_1_rssi'],
|
||
|
|
) for rssi in rvr_result['rssi']
|
||
|
|
]
|
||
|
|
}
|
||
|
|
|
||
|
|
figure.add_line(rvr_result['total_attenuation'],
|
||
|
|
rvr_result['throughput_receive'],
|
||
|
|
'Measured Throughput',
|
||
|
|
hover_text=rvr_result['hover_text'],
|
||
|
|
color='black',
|
||
|
|
marker='circle')
|
||
|
|
figure.add_line(
|
||
|
|
rvr_result['total_attenuation'][0:len(rvr_result['rx_phy_rate'])],
|
||
|
|
rvr_result['rx_phy_rate'],
|
||
|
|
'Rx PHY Rate',
|
||
|
|
hover_text=rvr_result['hover_text'],
|
||
|
|
color='blue',
|
||
|
|
style='dashed',
|
||
|
|
marker='inverted_triangle')
|
||
|
|
figure.add_line(
|
||
|
|
rvr_result['total_attenuation'][0:len(rvr_result['rx_phy_rate'])],
|
||
|
|
rvr_result['tx_phy_rate'],
|
||
|
|
'Tx PHY Rate',
|
||
|
|
hover_text=rvr_result['hover_text'],
|
||
|
|
color='red',
|
||
|
|
style='dashed',
|
||
|
|
marker='triangle')
|
||
|
|
|
||
|
|
output_file_path = os.path.join(
|
||
|
|
self.log_path, '{}.html'.format(self.current_test_name))
|
||
|
|
figure.generate_figure(output_file_path)
|
||
|
|
|
||
|
|
def compute_test_metrics(self, rvr_result):
|
||
|
|
# Set test metrics
|
||
|
|
rvr_result['metrics'] = {}
|
||
|
|
rvr_result['metrics']['peak_tput'] = max(
|
||
|
|
rvr_result['throughput_receive'])
|
||
|
|
if self.publish_testcase_metrics:
|
||
|
|
self.testcase_metric_logger.add_metric(
|
||
|
|
'peak_tput', rvr_result['metrics']['peak_tput'])
|
||
|
|
|
||
|
|
test_mode = rvr_result['ap_settings'][rvr_result['testcase_params']
|
||
|
|
['band']]['bandwidth']
|
||
|
|
tput_below_limit = [
|
||
|
|
tput <
|
||
|
|
self.testclass_params['tput_metric_targets'][test_mode]['high']
|
||
|
|
for tput in rvr_result['throughput_receive']
|
||
|
|
]
|
||
|
|
rvr_result['metrics']['high_tput_range'] = -1
|
||
|
|
for idx in range(len(tput_below_limit)):
|
||
|
|
if all(tput_below_limit[idx:]):
|
||
|
|
if idx == 0:
|
||
|
|
# Throughput was never above limit
|
||
|
|
rvr_result['metrics']['high_tput_range'] = -1
|
||
|
|
else:
|
||
|
|
rvr_result['metrics']['high_tput_range'] = rvr_result[
|
||
|
|
'total_attenuation'][max(idx, 1) - 1]
|
||
|
|
break
|
||
|
|
if self.publish_testcase_metrics:
|
||
|
|
self.testcase_metric_logger.add_metric(
|
||
|
|
'high_tput_range', rvr_result['metrics']['high_tput_range'])
|
||
|
|
|
||
|
|
tput_below_limit = [
|
||
|
|
tput <
|
||
|
|
self.testclass_params['tput_metric_targets'][test_mode]['low']
|
||
|
|
for tput in rvr_result['throughput_receive']
|
||
|
|
]
|
||
|
|
for idx in range(len(tput_below_limit)):
|
||
|
|
if all(tput_below_limit[idx:]):
|
||
|
|
rvr_result['metrics']['low_tput_range'] = rvr_result[
|
||
|
|
'total_attenuation'][max(idx, 1) - 1]
|
||
|
|
break
|
||
|
|
else:
|
||
|
|
rvr_result['metrics']['low_tput_range'] = -1
|
||
|
|
if self.publish_testcase_metrics:
|
||
|
|
self.testcase_metric_logger.add_metric(
|
||
|
|
'low_tput_range', rvr_result['metrics']['low_tput_range'])
|
||
|
|
|
||
|
|
def process_test_results(self, rvr_result):
|
||
|
|
self.plot_rvr_result(rvr_result)
|
||
|
|
self.compute_test_metrics(rvr_result)
|
||
|
|
|
||
|
|
def run_rvr_test(self, testcase_params):
|
||
|
|
"""Test function to run RvR.
|
||
|
|
|
||
|
|
The function runs an RvR test in the current device/AP configuration.
|
||
|
|
Function is called from another wrapper function that sets up the
|
||
|
|
testbed for the RvR test
|
||
|
|
|
||
|
|
Args:
|
||
|
|
testcase_params: dict containing test-specific parameters
|
||
|
|
Returns:
|
||
|
|
rvr_result: dict containing rvr_results and meta data
|
||
|
|
"""
|
||
|
|
self.log.info('Start running RvR')
|
||
|
|
# Refresh link layer stats before test
|
||
|
|
llstats_obj = wputils.LinkLayerStats(
|
||
|
|
self.monitored_dut,
|
||
|
|
self.testclass_params.get('monitor_llstats', 1))
|
||
|
|
zero_counter = 0
|
||
|
|
throughput = []
|
||
|
|
rx_phy_rate = []
|
||
|
|
tx_phy_rate = []
|
||
|
|
llstats = []
|
||
|
|
rssi = []
|
||
|
|
for atten in testcase_params['atten_range']:
|
||
|
|
for dev in self.android_devices:
|
||
|
|
if not wputils.health_check(dev, 5, 50):
|
||
|
|
asserts.skip('DUT health check failed. Skipping test.')
|
||
|
|
# Set Attenuation
|
||
|
|
for attenuator in self.attenuators:
|
||
|
|
attenuator.set_atten(atten, strict=False, retry=True)
|
||
|
|
# Refresh link layer stats
|
||
|
|
llstats_obj.update_stats()
|
||
|
|
# Setup sniffer
|
||
|
|
if self.testbed_params['sniffer_enable']:
|
||
|
|
self.sniffer.start_capture(
|
||
|
|
network=testcase_params['test_network'],
|
||
|
|
chan=testcase_params['channel'],
|
||
|
|
bw=testcase_params['bandwidth'],
|
||
|
|
duration=self.testclass_params['iperf_duration'] / 5)
|
||
|
|
# Start iperf session
|
||
|
|
if self.testclass_params.get('monitor_rssi', 1):
|
||
|
|
rssi_future = wputils.get_connected_rssi_nb(
|
||
|
|
self.monitored_dut,
|
||
|
|
self.testclass_params['iperf_duration'] - 1,
|
||
|
|
1,
|
||
|
|
1,
|
||
|
|
interface=self.monitored_interface)
|
||
|
|
self.iperf_server.start(tag=str(atten))
|
||
|
|
client_output_path = self.iperf_client.start(
|
||
|
|
testcase_params['iperf_server_address'],
|
||
|
|
testcase_params['iperf_args'], str(atten),
|
||
|
|
self.testclass_params['iperf_duration'] + self.TEST_TIMEOUT)
|
||
|
|
server_output_path = self.iperf_server.stop()
|
||
|
|
if self.testclass_params.get('monitor_rssi', 1):
|
||
|
|
rssi_result = rssi_future.result()
|
||
|
|
current_rssi = {
|
||
|
|
'signal_poll_rssi':
|
||
|
|
rssi_result['signal_poll_rssi']['mean'],
|
||
|
|
'chain_0_rssi': rssi_result['chain_0_rssi']['mean'],
|
||
|
|
'chain_1_rssi': rssi_result['chain_1_rssi']['mean']
|
||
|
|
}
|
||
|
|
else:
|
||
|
|
current_rssi = {
|
||
|
|
'signal_poll_rssi': float('nan'),
|
||
|
|
'chain_0_rssi': float('nan'),
|
||
|
|
'chain_1_rssi': float('nan')
|
||
|
|
}
|
||
|
|
rssi.append(current_rssi)
|
||
|
|
# Stop sniffer
|
||
|
|
if self.testbed_params['sniffer_enable']:
|
||
|
|
self.sniffer.stop_capture(tag=str(atten))
|
||
|
|
# Parse and log result
|
||
|
|
if testcase_params['use_client_output']:
|
||
|
|
iperf_file = client_output_path
|
||
|
|
else:
|
||
|
|
iperf_file = server_output_path
|
||
|
|
try:
|
||
|
|
iperf_result = ipf.IPerfResult(iperf_file)
|
||
|
|
curr_throughput = numpy.mean(iperf_result.instantaneous_rates[
|
||
|
|
self.testclass_params['iperf_ignored_interval']:-1]
|
||
|
|
) * 8 * (1.024**2)
|
||
|
|
except:
|
||
|
|
self.log.warning(
|
||
|
|
'ValueError: Cannot get iperf result. Setting to 0')
|
||
|
|
curr_throughput = 0
|
||
|
|
throughput.append(curr_throughput)
|
||
|
|
llstats_obj.update_stats()
|
||
|
|
curr_llstats = llstats_obj.llstats_incremental.copy()
|
||
|
|
llstats.append(curr_llstats)
|
||
|
|
rx_phy_rate.append(curr_llstats['summary'].get(
|
||
|
|
'mean_rx_phy_rate', 0))
|
||
|
|
tx_phy_rate.append(curr_llstats['summary'].get(
|
||
|
|
'mean_tx_phy_rate', 0))
|
||
|
|
self.log.info(
|
||
|
|
('Throughput at {0:.2f} dB is {1:.2f} Mbps. '
|
||
|
|
'RSSI = {2:.2f} [{3:.2f}, {4:.2f}].').format(
|
||
|
|
atten, curr_throughput, current_rssi['signal_poll_rssi'],
|
||
|
|
current_rssi['chain_0_rssi'],
|
||
|
|
current_rssi['chain_1_rssi']))
|
||
|
|
if curr_throughput == 0:
|
||
|
|
zero_counter = zero_counter + 1
|
||
|
|
else:
|
||
|
|
zero_counter = 0
|
||
|
|
if zero_counter == self.MAX_CONSECUTIVE_ZEROS:
|
||
|
|
self.log.info(
|
||
|
|
'Throughput stable at 0 Mbps. Stopping test now.')
|
||
|
|
zero_padding = len(
|
||
|
|
testcase_params['atten_range']) - len(throughput)
|
||
|
|
throughput.extend([0] * zero_padding)
|
||
|
|
rx_phy_rate.extend([0] * zero_padding)
|
||
|
|
tx_phy_rate.extend([0] * zero_padding)
|
||
|
|
break
|
||
|
|
for attenuator in self.attenuators:
|
||
|
|
attenuator.set_atten(0, strict=False, retry=True)
|
||
|
|
# Compile test result and meta data
|
||
|
|
rvr_result = collections.OrderedDict()
|
||
|
|
rvr_result['test_name'] = self.current_test_name
|
||
|
|
rvr_result['testcase_params'] = testcase_params.copy()
|
||
|
|
rvr_result['ap_settings'] = self.access_point.ap_settings.copy()
|
||
|
|
rvr_result['fixed_attenuation'] = self.testbed_params[
|
||
|
|
'fixed_attenuation'][str(testcase_params['channel'])]
|
||
|
|
rvr_result['attenuation'] = list(testcase_params['atten_range'])
|
||
|
|
rvr_result['total_attenuation'] = [
|
||
|
|
att + rvr_result['fixed_attenuation']
|
||
|
|
for att in rvr_result['attenuation']
|
||
|
|
]
|
||
|
|
rvr_result['rssi'] = rssi
|
||
|
|
rvr_result['throughput_receive'] = throughput
|
||
|
|
rvr_result['rx_phy_rate'] = rx_phy_rate
|
||
|
|
rvr_result['tx_phy_rate'] = tx_phy_rate
|
||
|
|
rvr_result['llstats'] = llstats
|
||
|
|
return rvr_result
|
||
|
|
|
||
|
|
def setup_ap(self, testcase_params):
|
||
|
|
"""Sets up the access point in the configuration required by the test.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
testcase_params: dict containing AP and other test params
|
||
|
|
"""
|
||
|
|
band = self.access_point.band_lookup_by_channel(
|
||
|
|
testcase_params['channel'])
|
||
|
|
if '6G' in band:
|
||
|
|
frequency = wutils.WifiEnums.channel_6G_to_freq[int(
|
||
|
|
testcase_params['channel'].strip('6g'))]
|
||
|
|
else:
|
||
|
|
if testcase_params['channel'] < 13:
|
||
|
|
frequency = wutils.WifiEnums.channel_2G_to_freq[
|
||
|
|
testcase_params['channel']]
|
||
|
|
else:
|
||
|
|
frequency = wutils.WifiEnums.channel_5G_to_freq[
|
||
|
|
testcase_params['channel']]
|
||
|
|
if frequency in wutils.WifiEnums.DFS_5G_FREQUENCIES:
|
||
|
|
self.access_point.set_region(self.testbed_params['DFS_region'])
|
||
|
|
else:
|
||
|
|
self.access_point.set_region(self.testbed_params['default_region'])
|
||
|
|
self.access_point.set_channel_and_bandwidth(testcase_params['band'],
|
||
|
|
testcase_params['channel'],
|
||
|
|
testcase_params['mode'])
|
||
|
|
self.log.info('Access Point Configuration: {}'.format(
|
||
|
|
self.access_point.ap_settings))
|
||
|
|
|
||
|
|
def setup_dut(self, testcase_params):
|
||
|
|
"""Sets up the DUT in the configuration required by the test.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
testcase_params: dict containing AP and other test params
|
||
|
|
"""
|
||
|
|
# Turn screen off to preserve battery
|
||
|
|
if self.testbed_params.get('screen_on',
|
||
|
|
False) or self.testclass_params.get(
|
||
|
|
'screen_on', False):
|
||
|
|
self.sta_dut.droid.wakeLockAcquireDim()
|
||
|
|
else:
|
||
|
|
self.sta_dut.go_to_sleep()
|
||
|
|
if (wputils.validate_network(self.sta_dut,
|
||
|
|
testcase_params['test_network']['SSID'])
|
||
|
|
and not self.testclass_params.get('force_reconnect', 0)):
|
||
|
|
self.log.info('Already connected to desired network')
|
||
|
|
else:
|
||
|
|
wutils.wifi_toggle_state(self.sta_dut, False)
|
||
|
|
wutils.set_wifi_country_code(self.sta_dut,
|
||
|
|
self.testclass_params['country_code'])
|
||
|
|
wutils.wifi_toggle_state(self.sta_dut, True)
|
||
|
|
wutils.reset_wifi(self.sta_dut)
|
||
|
|
if self.testbed_params.get('txbf_off', False):
|
||
|
|
wputils.disable_beamforming(self.sta_dut)
|
||
|
|
wutils.set_wifi_country_code(self.sta_dut,
|
||
|
|
self.testclass_params['country_code'])
|
||
|
|
if self.testbed_params['sniffer_enable']:
|
||
|
|
self.sniffer.start_capture(
|
||
|
|
network={'SSID': testcase_params['test_network']['SSID']},
|
||
|
|
chan=testcase_params['channel'],
|
||
|
|
bw=testcase_params['bandwidth'],
|
||
|
|
duration=180)
|
||
|
|
try:
|
||
|
|
wutils.wifi_connect(self.sta_dut,
|
||
|
|
testcase_params['test_network'],
|
||
|
|
num_of_tries=5,
|
||
|
|
check_connectivity=True)
|
||
|
|
if self.testclass_params.get('num_streams', 2) == 1:
|
||
|
|
wputils.set_nss_capability(self.sta_dut, 1)
|
||
|
|
finally:
|
||
|
|
if self.testbed_params['sniffer_enable']:
|
||
|
|
self.sniffer.stop_capture(tag='connection_setup')
|
||
|
|
|
||
|
|
def setup_rvr_test(self, testcase_params):
|
||
|
|
"""Function that gets devices ready for the test.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
testcase_params: dict containing test-specific parameters
|
||
|
|
"""
|
||
|
|
# Configure AP
|
||
|
|
self.setup_ap(testcase_params)
|
||
|
|
# Set attenuator to 0 dB
|
||
|
|
for attenuator in self.attenuators:
|
||
|
|
attenuator.set_atten(0, strict=False, retry=True)
|
||
|
|
# Reset, configure, and connect DUT
|
||
|
|
self.setup_dut(testcase_params)
|
||
|
|
# Wait before running the first wifi test
|
||
|
|
first_test_delay = self.testclass_params.get('first_test_delay', 600)
|
||
|
|
if first_test_delay > 0 and len(self.testclass_results) == 0:
|
||
|
|
self.log.info('Waiting before the first RvR test.')
|
||
|
|
time.sleep(first_test_delay)
|
||
|
|
self.setup_dut(testcase_params)
|
||
|
|
# Get iperf_server address
|
||
|
|
sta_dut_ip = self.sta_dut.droid.connectivityGetIPv4Addresses(
|
||
|
|
'wlan0')[0]
|
||
|
|
if isinstance(self.iperf_server, ipf.IPerfServerOverAdb):
|
||
|
|
testcase_params['iperf_server_address'] = sta_dut_ip
|
||
|
|
else:
|
||
|
|
if self.testbed_params.get('lan_traffic_only', True):
|
||
|
|
testcase_params[
|
||
|
|
'iperf_server_address'] = wputils.get_server_address(
|
||
|
|
self.remote_server, sta_dut_ip, '255.255.255.0')
|
||
|
|
else:
|
||
|
|
testcase_params[
|
||
|
|
'iperf_server_address'] = wputils.get_server_address(
|
||
|
|
self.remote_server, sta_dut_ip, 'public')
|
||
|
|
# Set DUT to monitor RSSI and LLStats on
|
||
|
|
self.monitored_dut = self.sta_dut
|
||
|
|
self.monitored_interface = 'wlan0'
|
||
|
|
|
||
|
|
def compile_test_params(self, testcase_params):
|
||
|
|
"""Function that completes all test params based on the test name.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
testcase_params: dict containing test-specific parameters
|
||
|
|
"""
|
||
|
|
# Check if test should be skipped based on parameters.
|
||
|
|
wputils.check_skip_conditions(testcase_params, self.sta_dut,
|
||
|
|
self.access_point,
|
||
|
|
getattr(self, 'ota_chamber', None))
|
||
|
|
|
||
|
|
band = wputils.CHANNEL_TO_BAND_MAP[testcase_params['channel']]
|
||
|
|
start_atten = self.testclass_params['atten_start'].get(band, 0)
|
||
|
|
num_atten_steps = int(
|
||
|
|
(self.testclass_params['atten_stop'] - start_atten) /
|
||
|
|
self.testclass_params['atten_step'])
|
||
|
|
testcase_params['atten_range'] = [
|
||
|
|
start_atten + x * self.testclass_params['atten_step']
|
||
|
|
for x in range(0, num_atten_steps)
|
||
|
|
]
|
||
|
|
band = self.access_point.band_lookup_by_channel(
|
||
|
|
testcase_params['channel'])
|
||
|
|
testcase_params['band'] = band
|
||
|
|
testcase_params['test_network'] = self.main_network[band]
|
||
|
|
if testcase_params['traffic_type'] == 'TCP':
|
||
|
|
testcase_params['iperf_socket_size'] = self.testclass_params.get(
|
||
|
|
'tcp_socket_size', None)
|
||
|
|
testcase_params['iperf_processes'] = self.testclass_params.get(
|
||
|
|
'tcp_processes', 1)
|
||
|
|
elif testcase_params['traffic_type'] == 'UDP':
|
||
|
|
testcase_params['iperf_socket_size'] = self.testclass_params.get(
|
||
|
|
'udp_socket_size', None)
|
||
|
|
testcase_params['iperf_processes'] = self.testclass_params.get(
|
||
|
|
'udp_processes', 1)
|
||
|
|
if (testcase_params['traffic_direction'] == 'DL'
|
||
|
|
and not isinstance(self.iperf_server, ipf.IPerfServerOverAdb)
|
||
|
|
) or (testcase_params['traffic_direction'] == 'UL'
|
||
|
|
and isinstance(self.iperf_server, ipf.IPerfServerOverAdb)):
|
||
|
|
testcase_params['iperf_args'] = wputils.get_iperf_arg_string(
|
||
|
|
duration=self.testclass_params['iperf_duration'],
|
||
|
|
reverse_direction=1,
|
||
|
|
traffic_type=testcase_params['traffic_type'],
|
||
|
|
socket_size=testcase_params['iperf_socket_size'],
|
||
|
|
num_processes=testcase_params['iperf_processes'],
|
||
|
|
udp_throughput=self.testclass_params['UDP_rates'][
|
||
|
|
testcase_params['mode']])
|
||
|
|
testcase_params['use_client_output'] = True
|
||
|
|
else:
|
||
|
|
testcase_params['iperf_args'] = wputils.get_iperf_arg_string(
|
||
|
|
duration=self.testclass_params['iperf_duration'],
|
||
|
|
reverse_direction=0,
|
||
|
|
traffic_type=testcase_params['traffic_type'],
|
||
|
|
socket_size=testcase_params['iperf_socket_size'],
|
||
|
|
num_processes=testcase_params['iperf_processes'],
|
||
|
|
udp_throughput=self.testclass_params['UDP_rates'][
|
||
|
|
testcase_params['mode']])
|
||
|
|
testcase_params['use_client_output'] = False
|
||
|
|
return testcase_params
|
||
|
|
|
||
|
|
def _test_rvr(self, testcase_params):
|
||
|
|
""" Function that gets called for each test case
|
||
|
|
|
||
|
|
Args:
|
||
|
|
testcase_params: dict containing test-specific parameters
|
||
|
|
"""
|
||
|
|
# Compile test parameters from config and test name
|
||
|
|
testcase_params = self.compile_test_params(testcase_params)
|
||
|
|
|
||
|
|
# Prepare devices and run test
|
||
|
|
self.setup_rvr_test(testcase_params)
|
||
|
|
rvr_result = self.run_rvr_test(testcase_params)
|
||
|
|
|
||
|
|
# Post-process results
|
||
|
|
self.testclass_results.append(rvr_result)
|
||
|
|
self.process_test_results(rvr_result)
|
||
|
|
self.pass_fail_check(rvr_result)
|
||
|
|
|
||
|
|
def generate_test_cases(self, channels, modes, traffic_types,
|
||
|
|
traffic_directions):
|
||
|
|
"""Function that auto-generates test cases for a test class."""
|
||
|
|
test_cases = []
|
||
|
|
allowed_configs = {
|
||
|
|
20: [
|
||
|
|
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 36, 40, 44, 48, 64, 100,
|
||
|
|
116, 132, 140, 149, 153, 157, 161, '6g37', '6g117', '6g213'
|
||
|
|
],
|
||
|
|
40: [36, 44, 100, 149, 157, '6g37', '6g117', '6g213'],
|
||
|
|
80: [36, 100, 149, '6g37', '6g117', '6g213'],
|
||
|
|
160: [36, '6g37', '6g117', '6g213']
|
||
|
|
}
|
||
|
|
|
||
|
|
for channel, mode, traffic_type, traffic_direction in itertools.product(
|
||
|
|
channels, modes, traffic_types, traffic_directions):
|
||
|
|
bandwidth = int(''.join([x for x in mode if x.isdigit()]))
|
||
|
|
if channel not in allowed_configs[bandwidth]:
|
||
|
|
continue
|
||
|
|
test_name = 'test_rvr_{}_{}_ch{}_{}'.format(
|
||
|
|
traffic_type, traffic_direction, channel, mode)
|
||
|
|
test_params = collections.OrderedDict(
|
||
|
|
channel=channel,
|
||
|
|
mode=mode,
|
||
|
|
bandwidth=bandwidth,
|
||
|
|
traffic_type=traffic_type,
|
||
|
|
traffic_direction=traffic_direction)
|
||
|
|
setattr(self, test_name, partial(self._test_rvr, test_params))
|
||
|
|
test_cases.append(test_name)
|
||
|
|
return test_cases
|
||
|
|
|
||
|
|
|
||
|
|
class WifiRvr_TCP_Test(WifiRvrTest):
|
||
|
|
|
||
|
|
def __init__(self, controllers):
|
||
|
|
super().__init__(controllers)
|
||
|
|
self.tests = self.generate_test_cases(
|
||
|
|
channels=[
|
||
|
|
1, 6, 11, 36, 40, 44, 48, 149, 153, 157, 161, '6g37', '6g117',
|
||
|
|
'6g213'
|
||
|
|
],
|
||
|
|
modes=['bw20', 'bw40', 'bw80', 'bw160'],
|
||
|
|
traffic_types=['TCP'],
|
||
|
|
traffic_directions=['DL', 'UL'])
|
||
|
|
|
||
|
|
|
||
|
|
class WifiRvr_VHT_TCP_Test(WifiRvrTest):
|
||
|
|
|
||
|
|
def __init__(self, controllers):
|
||
|
|
super().__init__(controllers)
|
||
|
|
self.tests = self.generate_test_cases(
|
||
|
|
channels=[1, 6, 11, 36, 40, 44, 48, 149, 153, 157, 161],
|
||
|
|
modes=['VHT20', 'VHT40', 'VHT80'],
|
||
|
|
traffic_types=['TCP'],
|
||
|
|
traffic_directions=['DL', 'UL'])
|
||
|
|
|
||
|
|
|
||
|
|
class WifiRvr_HE_TCP_Test(WifiRvrTest):
|
||
|
|
|
||
|
|
def __init__(self, controllers):
|
||
|
|
super().__init__(controllers)
|
||
|
|
self.tests = self.generate_test_cases(
|
||
|
|
channels=[
|
||
|
|
1, 6, 11, 36, 40, 44, 48, 149, 153, 157, 161, '6g37', '6g117',
|
||
|
|
'6g213'
|
||
|
|
],
|
||
|
|
modes=['HE20', 'HE40', 'HE80', 'HE160'],
|
||
|
|
traffic_types=['TCP'],
|
||
|
|
traffic_directions=['DL', 'UL'])
|
||
|
|
|
||
|
|
|
||
|
|
class WifiRvr_SampleUDP_Test(WifiRvrTest):
|
||
|
|
|
||
|
|
def __init__(self, controllers):
|
||
|
|
super().__init__(controllers)
|
||
|
|
self.tests = self.generate_test_cases(
|
||
|
|
channels=[6, 36, 149, '6g37'],
|
||
|
|
modes=['bw20', 'bw40', 'bw80', 'bw160'],
|
||
|
|
traffic_types=['UDP'],
|
||
|
|
traffic_directions=['DL', 'UL'])
|
||
|
|
|
||
|
|
|
||
|
|
class WifiRvr_VHT_SampleUDP_Test(WifiRvrTest):
|
||
|
|
|
||
|
|
def __init__(self, controllers):
|
||
|
|
super().__init__(controllers)
|
||
|
|
self.tests = self.generate_test_cases(
|
||
|
|
channels=[6, 36, 149],
|
||
|
|
modes=['VHT20', 'VHT40', 'VHT80', 'VHT160'],
|
||
|
|
traffic_types=['UDP'],
|
||
|
|
traffic_directions=['DL', 'UL'])
|
||
|
|
|
||
|
|
|
||
|
|
class WifiRvr_HE_SampleUDP_Test(WifiRvrTest):
|
||
|
|
|
||
|
|
def __init__(self, controllers):
|
||
|
|
super().__init__(controllers)
|
||
|
|
self.tests = self.generate_test_cases(
|
||
|
|
channels=[6, 36, 149],
|
||
|
|
modes=['HE20', 'HE40', 'HE80', 'HE160', '6g37'],
|
||
|
|
traffic_types=['UDP'],
|
||
|
|
traffic_directions=['DL', 'UL'])
|
||
|
|
|
||
|
|
|
||
|
|
class WifiRvr_SampleDFS_Test(WifiRvrTest):
|
||
|
|
|
||
|
|
def __init__(self, controllers):
|
||
|
|
super().__init__(controllers)
|
||
|
|
self.tests = self.generate_test_cases(
|
||
|
|
channels=[64, 100, 116, 132, 140],
|
||
|
|
modes=['bw20', 'bw40', 'bw80'],
|
||
|
|
traffic_types=['TCP'],
|
||
|
|
traffic_directions=['DL', 'UL'])
|
||
|
|
|
||
|
|
|
||
|
|
class WifiRvr_SingleChain_TCP_Test(WifiRvrTest):
|
||
|
|
|
||
|
|
def __init__(self, controllers):
|
||
|
|
super().__init__(controllers)
|
||
|
|
self.tests = self.generate_test_cases(
|
||
|
|
channels=[
|
||
|
|
1, 6, 11, 36, 40, 44, 48, 149, 153, 157, 161, '6g37', '6g117',
|
||
|
|
'6g213'
|
||
|
|
],
|
||
|
|
modes=['bw20', 'bw40', 'bw80', 'bw160'],
|
||
|
|
traffic_types=['TCP'],
|
||
|
|
traffic_directions=['DL', 'UL'],
|
||
|
|
chains=[0, 1, '2x2'])
|
||
|
|
|
||
|
|
def setup_dut(self, testcase_params):
|
||
|
|
self.sta_dut = self.android_devices[0]
|
||
|
|
wputils.set_chain_mask(self.sta_dut, testcase_params['chain'])
|
||
|
|
WifiRvrTest.setup_dut(self, testcase_params)
|
||
|
|
|
||
|
|
def generate_test_cases(self, channels, modes, traffic_types,
|
||
|
|
traffic_directions, chains):
|
||
|
|
"""Function that auto-generates test cases for a test class."""
|
||
|
|
test_cases = []
|
||
|
|
allowed_configs = {
|
||
|
|
20: [
|
||
|
|
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 36, 40, 44, 48, 64, 100,
|
||
|
|
116, 132, 140, 149, 153, 157, 161, '6g37', '6g117', '6g213'
|
||
|
|
],
|
||
|
|
40: [36, 44, 100, 149, 157, '6g37', '6g117', '6g213'],
|
||
|
|
80: [36, 100, 149, '6g37', '6g117', '6g213'],
|
||
|
|
160: [36, '6g37', '6g117', '6g213']
|
||
|
|
}
|
||
|
|
|
||
|
|
for channel, mode, chain, traffic_type, traffic_direction in itertools.product(
|
||
|
|
channels, modes, chains, traffic_types, traffic_directions):
|
||
|
|
bandwidth = int(''.join([x for x in mode if x.isdigit()]))
|
||
|
|
if channel not in allowed_configs[bandwidth]:
|
||
|
|
continue
|
||
|
|
test_name = 'test_rvr_{}_{}_ch{}_{}_ch{}'.format(
|
||
|
|
traffic_type, traffic_direction, channel, mode, chain)
|
||
|
|
test_params = collections.OrderedDict(
|
||
|
|
channel=channel,
|
||
|
|
mode=mode,
|
||
|
|
bandwidth=bandwidth,
|
||
|
|
traffic_type=traffic_type,
|
||
|
|
traffic_direction=traffic_direction,
|
||
|
|
chain=chain)
|
||
|
|
setattr(self, test_name, partial(self._test_rvr, test_params))
|
||
|
|
test_cases.append(test_name)
|
||
|
|
return test_cases
|
||
|
|
|
||
|
|
|
||
|
|
# Over-the air version of RVR tests
|
||
|
|
class WifiOtaRvrTest(WifiRvrTest):
|
||
|
|
"""Class to test over-the-air RvR
|
||
|
|
|
||
|
|
This class implements measures WiFi RvR tests in an OTA chamber. It enables
|
||
|
|
setting turntable orientation and other chamber parameters to study
|
||
|
|
performance in varying channel conditions
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, controllers):
|
||
|
|
base_test.BaseTestClass.__init__(self, controllers)
|
||
|
|
self.testcase_metric_logger = (
|
||
|
|
BlackboxMappedMetricLogger.for_test_case())
|
||
|
|
self.testclass_metric_logger = (
|
||
|
|
BlackboxMappedMetricLogger.for_test_class())
|
||
|
|
self.publish_testcase_metrics = False
|
||
|
|
|
||
|
|
def setup_class(self):
|
||
|
|
WifiRvrTest.setup_class(self)
|
||
|
|
self.ota_chamber = ota_chamber.create(
|
||
|
|
self.user_params['OTAChamber'])[0]
|
||
|
|
|
||
|
|
def teardown_class(self):
|
||
|
|
WifiRvrTest.teardown_class(self)
|
||
|
|
self.ota_chamber.reset_chamber()
|
||
|
|
|
||
|
|
def extract_test_id(self, testcase_params, id_fields):
|
||
|
|
test_id = collections.OrderedDict(
|
||
|
|
(param, testcase_params.get(param, None)) for param in id_fields)
|
||
|
|
return test_id
|
||
|
|
|
||
|
|
def process_testclass_results(self):
|
||
|
|
"""Saves plot with all test results to enable comparison."""
|
||
|
|
# Plot individual test id results raw data and compile metrics
|
||
|
|
plots = collections.OrderedDict()
|
||
|
|
compiled_data = collections.OrderedDict()
|
||
|
|
for result in self.testclass_results:
|
||
|
|
test_id = tuple(
|
||
|
|
self.extract_test_id(result['testcase_params'], [
|
||
|
|
'channel', 'mode', 'traffic_type', 'traffic_direction',
|
||
|
|
'chain'
|
||
|
|
]).items())
|
||
|
|
if test_id not in plots:
|
||
|
|
# Initialize test id data when not present
|
||
|
|
compiled_data[test_id] = {
|
||
|
|
'throughput': [],
|
||
|
|
'rx_phy_rate': [],
|
||
|
|
'tx_phy_rate': [],
|
||
|
|
'metrics': {}
|
||
|
|
}
|
||
|
|
compiled_data[test_id]['metrics'] = {
|
||
|
|
key: []
|
||
|
|
for key in result['metrics'].keys()
|
||
|
|
}
|
||
|
|
plots[test_id] = BokehFigure(
|
||
|
|
title='Channel {} {} ({} {})'.format(
|
||
|
|
result['testcase_params']['channel'],
|
||
|
|
result['testcase_params']['mode'],
|
||
|
|
result['testcase_params']['traffic_type'],
|
||
|
|
result['testcase_params']['traffic_direction']),
|
||
|
|
x_label='Attenuation (dB)',
|
||
|
|
primary_y_label='Throughput (Mbps)')
|
||
|
|
test_id_phy = test_id + tuple('PHY')
|
||
|
|
plots[test_id_phy] = BokehFigure(
|
||
|
|
title='Channel {} {} ({} {}) (PHY Rate)'.format(
|
||
|
|
result['testcase_params']['channel'],
|
||
|
|
result['testcase_params']['mode'],
|
||
|
|
result['testcase_params']['traffic_type'],
|
||
|
|
result['testcase_params']['traffic_direction']),
|
||
|
|
x_label='Attenuation (dB)',
|
||
|
|
primary_y_label='PHY Rate (Mbps)')
|
||
|
|
# Compile test id data and metrics
|
||
|
|
compiled_data[test_id]['throughput'].append(
|
||
|
|
result['throughput_receive'])
|
||
|
|
compiled_data[test_id]['rx_phy_rate'].append(result['rx_phy_rate'])
|
||
|
|
compiled_data[test_id]['tx_phy_rate'].append(result['tx_phy_rate'])
|
||
|
|
compiled_data[test_id]['total_attenuation'] = result[
|
||
|
|
'total_attenuation']
|
||
|
|
for metric_key, metric_value in result['metrics'].items():
|
||
|
|
compiled_data[test_id]['metrics'][metric_key].append(
|
||
|
|
metric_value)
|
||
|
|
# Add test id to plots
|
||
|
|
plots[test_id].add_line(result['total_attenuation'],
|
||
|
|
result['throughput_receive'],
|
||
|
|
result['test_name'].strip('test_rvr_'),
|
||
|
|
hover_text=result['hover_text'],
|
||
|
|
width=1,
|
||
|
|
style='dashed',
|
||
|
|
marker='circle')
|
||
|
|
plots[test_id_phy].add_line(
|
||
|
|
result['total_attenuation'],
|
||
|
|
result['rx_phy_rate'],
|
||
|
|
result['test_name'].strip('test_rvr_') + ' Rx PHY Rate',
|
||
|
|
hover_text=result['hover_text'],
|
||
|
|
width=1,
|
||
|
|
style='dashed',
|
||
|
|
marker='inverted_triangle')
|
||
|
|
plots[test_id_phy].add_line(
|
||
|
|
result['total_attenuation'],
|
||
|
|
result['tx_phy_rate'],
|
||
|
|
result['test_name'].strip('test_rvr_') + ' Tx PHY Rate',
|
||
|
|
hover_text=result['hover_text'],
|
||
|
|
width=1,
|
||
|
|
style='dashed',
|
||
|
|
marker='triangle')
|
||
|
|
|
||
|
|
# Compute average RvRs and compute metrics over orientations
|
||
|
|
for test_id, test_data in compiled_data.items():
|
||
|
|
test_id_dict = dict(test_id)
|
||
|
|
metric_tag = '{}_{}_ch{}_{}'.format(
|
||
|
|
test_id_dict['traffic_type'],
|
||
|
|
test_id_dict['traffic_direction'], test_id_dict['channel'],
|
||
|
|
test_id_dict['mode'])
|
||
|
|
high_tput_hit_freq = numpy.mean(
|
||
|
|
numpy.not_equal(test_data['metrics']['high_tput_range'], -1))
|
||
|
|
self.testclass_metric_logger.add_metric(
|
||
|
|
'{}.high_tput_hit_freq'.format(metric_tag), high_tput_hit_freq)
|
||
|
|
for metric_key, metric_value in test_data['metrics'].items():
|
||
|
|
metric_key = '{}.avg_{}'.format(metric_tag, metric_key)
|
||
|
|
metric_value = numpy.mean(metric_value)
|
||
|
|
self.testclass_metric_logger.add_metric(
|
||
|
|
metric_key, metric_value)
|
||
|
|
test_data['avg_rvr'] = numpy.mean(test_data['throughput'], 0)
|
||
|
|
test_data['median_rvr'] = numpy.median(test_data['throughput'], 0)
|
||
|
|
test_data['avg_rx_phy_rate'] = numpy.mean(test_data['rx_phy_rate'],
|
||
|
|
0)
|
||
|
|
test_data['avg_tx_phy_rate'] = numpy.mean(test_data['tx_phy_rate'],
|
||
|
|
0)
|
||
|
|
plots[test_id].add_line(test_data['total_attenuation'],
|
||
|
|
test_data['avg_rvr'],
|
||
|
|
legend='Average Throughput',
|
||
|
|
marker='circle')
|
||
|
|
plots[test_id].add_line(test_data['total_attenuation'],
|
||
|
|
test_data['median_rvr'],
|
||
|
|
legend='Median Throughput',
|
||
|
|
marker='square')
|
||
|
|
test_id_phy = test_id + tuple('PHY')
|
||
|
|
plots[test_id_phy].add_line(test_data['total_attenuation'],
|
||
|
|
test_data['avg_rx_phy_rate'],
|
||
|
|
legend='Average Rx Rate',
|
||
|
|
marker='inverted_triangle')
|
||
|
|
plots[test_id_phy].add_line(test_data['total_attenuation'],
|
||
|
|
test_data['avg_tx_phy_rate'],
|
||
|
|
legend='Average Tx Rate',
|
||
|
|
marker='triangle')
|
||
|
|
|
||
|
|
figure_list = []
|
||
|
|
for plot_id, plot in plots.items():
|
||
|
|
plot.generate_figure()
|
||
|
|
figure_list.append(plot)
|
||
|
|
output_file_path = os.path.join(self.log_path, 'results.html')
|
||
|
|
BokehFigure.save_figures(figure_list, output_file_path)
|
||
|
|
|
||
|
|
def setup_rvr_test(self, testcase_params):
|
||
|
|
# Continue test setup
|
||
|
|
WifiRvrTest.setup_rvr_test(self, testcase_params)
|
||
|
|
# Set turntable orientation
|
||
|
|
self.ota_chamber.set_orientation(testcase_params['orientation'])
|
||
|
|
|
||
|
|
def generate_test_cases(self, channels, modes, angles, traffic_types,
|
||
|
|
directions):
|
||
|
|
test_cases = []
|
||
|
|
allowed_configs = {
|
||
|
|
20: [
|
||
|
|
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 36, 40, 44, 48, 64, 100,
|
||
|
|
116, 132, 140, 149, 153, 157, 161
|
||
|
|
],
|
||
|
|
40: [36, 44, 100, 149, 157],
|
||
|
|
80: [36, 100, 149],
|
||
|
|
160: [36, '6g37', '6g117', '6g213']
|
||
|
|
}
|
||
|
|
for channel, mode, angle, traffic_type, direction in itertools.product(
|
||
|
|
channels, modes, angles, traffic_types, directions):
|
||
|
|
bandwidth = int(''.join([x for x in mode if x.isdigit()]))
|
||
|
|
if channel not in allowed_configs[bandwidth]:
|
||
|
|
continue
|
||
|
|
testcase_name = 'test_rvr_{}_{}_ch{}_{}_{}deg'.format(
|
||
|
|
traffic_type, direction, channel, mode, angle)
|
||
|
|
test_params = collections.OrderedDict(channel=channel,
|
||
|
|
mode=mode,
|
||
|
|
bandwidth=bandwidth,
|
||
|
|
traffic_type=traffic_type,
|
||
|
|
traffic_direction=direction,
|
||
|
|
orientation=angle)
|
||
|
|
setattr(self, testcase_name, partial(self._test_rvr, test_params))
|
||
|
|
test_cases.append(testcase_name)
|
||
|
|
return test_cases
|
||
|
|
|
||
|
|
|
||
|
|
class WifiOtaRvr_StandardOrientation_Test(WifiOtaRvrTest):
|
||
|
|
|
||
|
|
def __init__(self, controllers):
|
||
|
|
WifiOtaRvrTest.__init__(self, controllers)
|
||
|
|
self.tests = self.generate_test_cases(
|
||
|
|
[1, 6, 11, 36, 40, 44, 48, 149, 153, 157, 161, '6g37'],
|
||
|
|
['bw20', 'bw40', 'bw80', 'bw160'], list(range(0, 360, 45)),
|
||
|
|
['TCP'], ['DL', 'UL'])
|
||
|
|
|
||
|
|
|
||
|
|
class WifiOtaRvr_SampleChannel_Test(WifiOtaRvrTest):
|
||
|
|
|
||
|
|
def __init__(self, controllers):
|
||
|
|
WifiOtaRvrTest.__init__(self, controllers)
|
||
|
|
self.tests = self.generate_test_cases([6], ['bw20'],
|
||
|
|
list(range(0, 360, 45)), ['TCP'],
|
||
|
|
['DL'])
|
||
|
|
self.tests.extend(
|
||
|
|
self.generate_test_cases([36, 149], ['bw80', 'bw160'],
|
||
|
|
list(range(0, 360, 45)), ['TCP'], ['DL']))
|
||
|
|
self.tests.extend(
|
||
|
|
self.generate_test_cases(['6g37'], ['bw160'],
|
||
|
|
list(range(0, 360, 45)), ['TCP'], ['DL']))
|
||
|
|
|
||
|
|
|
||
|
|
class WifiOtaRvr_SingleOrientation_Test(WifiOtaRvrTest):
|
||
|
|
|
||
|
|
def __init__(self, controllers):
|
||
|
|
WifiOtaRvrTest.__init__(self, controllers)
|
||
|
|
self.tests = self.generate_test_cases(
|
||
|
|
[6, 36, 40, 44, 48, 149, 153, 157, 161, '6g37'],
|
||
|
|
['bw20', 'bw40', 'bw80', 'bw160'], [0], ['TCP'], ['DL', 'UL'])
|
||
|
|
|
||
|
|
|
||
|
|
class WifiOtaRvr_SingleChain_Test(WifiOtaRvrTest):
|
||
|
|
|
||
|
|
def __init__(self, controllers):
|
||
|
|
WifiOtaRvrTest.__init__(self, controllers)
|
||
|
|
self.tests = self.generate_test_cases([6], ['bw20'],
|
||
|
|
list(range(0, 360, 45)), ['TCP'],
|
||
|
|
['DL', 'UL'], [0, 1])
|
||
|
|
self.tests.extend(
|
||
|
|
self.generate_test_cases([36, 149], ['bw20', 'bw80', 'bw160'],
|
||
|
|
list(range(0, 360, 45)), ['TCP'],
|
||
|
|
['DL', 'UL'], [0, 1, '2x2']))
|
||
|
|
self.tests.extend(
|
||
|
|
self.generate_test_cases(['6g37'], ['bw20', 'bw80', 'bw160'],
|
||
|
|
list(range(0, 360, 45)), ['TCP'],
|
||
|
|
['DL', 'UL'], [0, 1, '2x2']))
|
||
|
|
|
||
|
|
def setup_dut(self, testcase_params):
|
||
|
|
self.sta_dut = self.android_devices[0]
|
||
|
|
wputils.set_chain_mask(self.sta_dut, testcase_params['chain'])
|
||
|
|
WifiRvrTest.setup_dut(self, testcase_params)
|
||
|
|
|
||
|
|
def generate_test_cases(self, channels, modes, angles, traffic_types,
|
||
|
|
directions, chains):
|
||
|
|
test_cases = []
|
||
|
|
allowed_configs = {
|
||
|
|
20: [
|
||
|
|
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 36, 40, 44, 48, 64, 100,
|
||
|
|
116, 132, 140, 149, 153, 157, 161
|
||
|
|
],
|
||
|
|
40: [36, 44, 100, 149, 157],
|
||
|
|
80: [36, 100, 149],
|
||
|
|
160: [36, '6g37', '6g117', '6g213']
|
||
|
|
}
|
||
|
|
for channel, mode, chain, angle, traffic_type, direction in itertools.product(
|
||
|
|
channels, modes, chains, angles, traffic_types, directions):
|
||
|
|
bandwidth = int(''.join([x for x in mode if x.isdigit()]))
|
||
|
|
if channel not in allowed_configs[bandwidth]:
|
||
|
|
continue
|
||
|
|
testcase_name = 'test_rvr_{}_{}_ch{}_{}_ch{}_{}deg'.format(
|
||
|
|
traffic_type, direction, channel, mode, chain, angle)
|
||
|
|
test_params = collections.OrderedDict(channel=channel,
|
||
|
|
mode=mode,
|
||
|
|
bandwidth=bandwidth,
|
||
|
|
chain=chain,
|
||
|
|
traffic_type=traffic_type,
|
||
|
|
traffic_direction=direction,
|
||
|
|
orientation=angle)
|
||
|
|
setattr(self, testcase_name, partial(self._test_rvr, test_params))
|
||
|
|
test_cases.append(testcase_name)
|
||
|
|
return test_cases
|