468 lines
13 KiB
Python
Executable File
468 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Copyright 2022, The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Integration tests for the Atest Bazel mode feature."""
|
|
|
|
# pylint: disable=invalid-name
|
|
# pylint: disable=missing-class-docstring
|
|
# pylint: disable=missing-function-docstring
|
|
|
|
import dataclasses
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Tuple
|
|
|
|
|
|
_ENV_BUILD_TOP = 'ANDROID_BUILD_TOP'
|
|
_PASSING_CLASS_NAME = 'PassingHostTest'
|
|
_FAILING_CLASS_NAME = 'FailingHostTest'
|
|
_PASSING_METHOD_NAME = 'testPass'
|
|
_FAILING_METHOD_NAME = 'testFAIL'
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class JavaSourceFile:
|
|
class_name: str
|
|
src_body: str
|
|
|
|
|
|
class BazelModeTest(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.src_root_path = Path(os.environ['ANDROID_BUILD_TOP'])
|
|
self.test_dir = self.src_root_path.joinpath('atest_bazel_mode_test')
|
|
if self.test_dir.exists():
|
|
shutil.rmtree(self.test_dir)
|
|
self.out_dir_path = Path(tempfile.mkdtemp())
|
|
self.test_env = self.setup_test_env()
|
|
|
|
def tearDown(self):
|
|
shutil.rmtree(self.test_dir)
|
|
shutil.rmtree(self.out_dir_path)
|
|
|
|
def test_passing_test_returns_zero_exit_code(self):
|
|
module_name = 'passing_java_host_test'
|
|
self.add_passing_test(module_name)
|
|
|
|
completed_process = self.run_shell_command(
|
|
f'atest -c -m --bazel-mode {module_name}')
|
|
|
|
self.assertEqual(completed_process.returncode, 0)
|
|
|
|
def test_failing_test_returns_nonzero_exit_code(self):
|
|
module_name = 'failing_java_host_test'
|
|
self.add_failing_test(module_name)
|
|
|
|
completed_process = self.run_shell_command(
|
|
f'atest -c -m --bazel-mode {module_name}')
|
|
|
|
self.assertNotEqual(completed_process.returncode, 0)
|
|
|
|
def test_passing_test_is_cached_when_rerun(self):
|
|
module_name = 'passing_java_host_test'
|
|
self.add_passing_test(module_name)
|
|
|
|
completed_process = self.run_shell_command(
|
|
f'atest -c -m --bazel-mode {module_name} && '
|
|
f'atest --bazel-mode {module_name}')
|
|
|
|
self.assert_in_stdout(f':{module_name}_host (cached) PASSED',
|
|
completed_process)
|
|
|
|
def test_cached_test_reruns_when_modified(self):
|
|
module_name = 'passing_java_host_test'
|
|
java_test_file, _ = self.write_java_test_module(
|
|
module_name, passing_java_test_source())
|
|
self.run_shell_command(
|
|
f'atest -c -m --bazel-mode {module_name}')
|
|
|
|
java_test_file.write_text(
|
|
failing_java_test_source(
|
|
test_class_name=_PASSING_CLASS_NAME).src_body)
|
|
completed_process = self.run_shell_command(
|
|
f'atest --bazel-mode {module_name}')
|
|
|
|
self.assert_in_stdout(f':{module_name}_host FAILED',
|
|
completed_process)
|
|
|
|
def test_only_supported_test_run_with_bazel(self):
|
|
module_name = 'passing_java_host_test'
|
|
unsupported_module_name = 'unsupported_passing_java_test'
|
|
self.add_passing_test(module_name)
|
|
self.add_unsupported_passing_test(unsupported_module_name)
|
|
|
|
completed_process = self.run_shell_command(
|
|
f'atest -c -m --host --bazel-mode {module_name} '
|
|
f'{unsupported_module_name}')
|
|
|
|
self.assert_in_stdout(f':{module_name}_host PASSED',
|
|
completed_process)
|
|
self.assert_in_stdout(
|
|
f'{_PASSING_CLASS_NAME}#{_PASSING_METHOD_NAME}: PASSED',
|
|
completed_process)
|
|
|
|
def test_defaults_to_device_variant(self):
|
|
module_name = 'passing_cc_host_test'
|
|
self.write_cc_test_module(module_name, passing_cc_test_source())
|
|
|
|
completed_process = self.run_shell_command(
|
|
f'atest -c -m --bazel-mode {module_name}')
|
|
|
|
self.assert_in_stdout('AtestTradefedTestRunner:',
|
|
completed_process)
|
|
|
|
def test_runs_host_variant_when_requested(self):
|
|
module_name = 'passing_cc_host_test'
|
|
self.write_cc_test_module(module_name, passing_cc_test_source())
|
|
|
|
completed_process = self.run_shell_command(
|
|
f'atest -c -m --host --bazel-mode {module_name}')
|
|
|
|
self.assert_in_stdout(f':{module_name}_host PASSED',
|
|
completed_process)
|
|
|
|
def test_ignores_host_arg_for_device_only_test(self):
|
|
module_name = 'passing_cc_device_test'
|
|
self.write_cc_test_module(module_name, passing_cc_test_source(),
|
|
host_supported=False)
|
|
|
|
completed_process = self.run_shell_command(
|
|
f'atest -c -m --host --bazel-mode {module_name}')
|
|
|
|
self.assert_in_stdout('Specified --host, but the following tests are '
|
|
'device-only', completed_process)
|
|
|
|
def test_supports_extra_tradefed_reporters(self):
|
|
test_module_name = 'passing_java_host_test'
|
|
self.add_passing_test(test_module_name)
|
|
|
|
reporter_module_name = 'test-result-reporter'
|
|
reporter_class_name = 'TestResultReporter'
|
|
expected_output_string = '0xFEEDF00D'
|
|
|
|
self.write_java_reporter_module(
|
|
reporter_module_name,
|
|
java_reporter_source(
|
|
reporter_class_name,
|
|
expected_output_string
|
|
)
|
|
)
|
|
|
|
self.run_shell_command(
|
|
f'm {reporter_module_name}', check=True)
|
|
self.run_shell_command(
|
|
f'atest -c -m --bazel-mode {test_module_name} --dry-run',
|
|
check=True)
|
|
self.run_shell_command(
|
|
f'cp ${{ANDROID_HOST_OUT}}/framework/{reporter_module_name}.jar '
|
|
f'{self.out_dir_path}/atest_bazel_workspace/tools/asuite/atest/'
|
|
'bazel/reporter/bazel-result-reporter/host/framework/.',
|
|
check=True)
|
|
|
|
completed_process = self.run_shell_command(
|
|
f'atest --bazel-mode {test_module_name} --bazel-arg='
|
|
'--//bazel/rules:extra_tradefed_result_reporters=android.'
|
|
f'{reporter_class_name} --bazel-arg=--test_output=all', check=True)
|
|
|
|
self.assert_in_stdout(
|
|
expected_output_string, completed_process)
|
|
|
|
def setup_test_env(self) -> Dict[str, Any]:
|
|
test_env = {
|
|
'PATH': os.environ['PATH'],
|
|
'HOME': os.environ['HOME'],
|
|
'OUT_DIR': str(self.out_dir_path),
|
|
}
|
|
return test_env
|
|
|
|
def run_shell_command(
|
|
self,
|
|
shell_command: str,
|
|
check: bool=False
|
|
) -> subprocess.CompletedProcess:
|
|
return subprocess.run(
|
|
'. build/envsetup.sh && '
|
|
'lunch aosp_cf_x86_64_pc-userdebug && '
|
|
f'{shell_command}',
|
|
env=self.test_env,
|
|
cwd=self.src_root_path,
|
|
shell=True,
|
|
check=check,
|
|
stderr=subprocess.STDOUT,
|
|
stdout=subprocess.PIPE)
|
|
|
|
def add_passing_test(self, module_name: str):
|
|
self.write_java_test_module(
|
|
module_name, passing_java_test_source())
|
|
|
|
def add_failing_test(self, module_name: str):
|
|
self.write_java_test_module(
|
|
module_name, failing_java_test_source())
|
|
|
|
def add_unsupported_passing_test(self, module_name: str):
|
|
self.write_java_test_module(
|
|
module_name, passing_java_test_source(), unit_test=False)
|
|
|
|
def write_java_test_module(
|
|
self,
|
|
module_name: str,
|
|
test_src: JavaSourceFile,
|
|
unit_test: bool=True,
|
|
) -> Tuple[Path, Path]:
|
|
test_dir = self.test_dir.joinpath(module_name)
|
|
test_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
src_file_name = f'{test_src.class_name}.java'
|
|
src_file_path = test_dir.joinpath(f'{src_file_name}')
|
|
src_file_path.write_text(test_src.src_body, encoding='utf8')
|
|
|
|
bp_file_path = test_dir.joinpath('Android.bp')
|
|
bp_file_path.write_text(
|
|
android_bp(
|
|
java_test_host(
|
|
name=module_name,
|
|
srcs=[
|
|
str(src_file_name),
|
|
],
|
|
unit_test=unit_test,
|
|
),
|
|
),
|
|
encoding='utf8')
|
|
return (src_file_path, bp_file_path)
|
|
|
|
def write_cc_test_module(
|
|
self,
|
|
module_name: str,
|
|
test_src: str,
|
|
host_supported: bool=True,
|
|
) -> Tuple[Path, Path]:
|
|
test_dir = self.test_dir.joinpath(module_name)
|
|
test_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
src_file_name = f'{module_name}.cpp'
|
|
src_file_path = test_dir.joinpath(f'{src_file_name}')
|
|
src_file_path.write_text(test_src, encoding='utf8')
|
|
|
|
bp_file_path = test_dir.joinpath('Android.bp')
|
|
bp_file_path.write_text(
|
|
android_bp(
|
|
cc_test(
|
|
name=module_name,
|
|
srcs=[
|
|
str(src_file_name),
|
|
],
|
|
host_supported=host_supported,
|
|
),
|
|
),
|
|
encoding='utf8')
|
|
return (src_file_path, bp_file_path)
|
|
|
|
def write_java_reporter_module(
|
|
self,
|
|
module_name: str,
|
|
reporter_src: JavaSourceFile,
|
|
) -> Tuple[Path, Path]:
|
|
test_dir = self.test_dir.joinpath(module_name)
|
|
test_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
src_file_name = f'{reporter_src.class_name}.java'
|
|
src_file_path = test_dir.joinpath(f'{src_file_name}')
|
|
src_file_path.write_text(reporter_src.src_body, encoding='utf8')
|
|
|
|
bp_file_path = test_dir.joinpath('Android.bp')
|
|
bp_file_path.write_text(
|
|
android_bp(
|
|
java_library(
|
|
name=module_name,
|
|
srcs=[
|
|
str(src_file_name),
|
|
],
|
|
),
|
|
),
|
|
encoding='utf8')
|
|
return (src_file_path, bp_file_path)
|
|
|
|
def assert_in_stdout(
|
|
self,
|
|
message: str,
|
|
completed_process: subprocess.CompletedProcess,
|
|
):
|
|
self.assertIn(message, completed_process.stdout.decode())
|
|
|
|
|
|
def passing_java_test_source() -> JavaSourceFile:
|
|
return java_test_source(
|
|
test_class_name=_PASSING_CLASS_NAME,
|
|
test_method_name=_PASSING_METHOD_NAME,
|
|
test_method_body='Assert.assertEquals("Pass", "Pass");')
|
|
|
|
|
|
def failing_java_test_source(
|
|
test_class_name=_FAILING_CLASS_NAME
|
|
)-> JavaSourceFile:
|
|
return java_test_source(
|
|
test_class_name=test_class_name,
|
|
test_method_name=_FAILING_METHOD_NAME,
|
|
test_method_body='Assert.assertEquals("Pass", "Fail");')
|
|
|
|
|
|
def java_test_source(
|
|
test_class_name: str,
|
|
test_method_name: str,
|
|
test_method_body: str,
|
|
) -> JavaSourceFile:
|
|
return JavaSourceFile(test_class_name, f"""\
|
|
package android;
|
|
|
|
import org.junit.Assert;
|
|
import org.junit.Test;
|
|
import org.junit.runners.JUnit4;
|
|
import org.junit.runner.RunWith;
|
|
|
|
@RunWith(JUnit4.class)
|
|
public final class {test_class_name} {{
|
|
|
|
@Test
|
|
public void {test_method_name}() {{
|
|
{test_method_body}
|
|
}}
|
|
}}
|
|
""")
|
|
|
|
def java_reporter_source(
|
|
reporter_class_name: str,
|
|
output_string: str,
|
|
) -> JavaSourceFile:
|
|
return JavaSourceFile(reporter_class_name, f"""\
|
|
package android;
|
|
|
|
import com.android.tradefed.result.ITestInvocationListener;
|
|
|
|
public final class {reporter_class_name} implements ITestInvocationListener {{
|
|
|
|
@Override
|
|
public void invocationEnded(long elapsedTime) {{
|
|
System.out.println("{output_string}");
|
|
}}
|
|
}}
|
|
""")
|
|
|
|
def passing_cc_test_source() -> str:
|
|
return cc_test_source(
|
|
test_suite_name='TestSuite',
|
|
test_name='PassingTest',
|
|
test_body='')
|
|
|
|
|
|
def cc_test_source(
|
|
test_suite_name: str,
|
|
test_name: str,
|
|
test_body: str,
|
|
) -> str:
|
|
return f"""\
|
|
#include <gtest/gtest.h>
|
|
|
|
TEST({test_suite_name}, {test_name}) {{
|
|
{test_body}
|
|
}}
|
|
"""
|
|
|
|
|
|
def android_bp(
|
|
modules: str='',
|
|
) -> str:
|
|
return f"""\
|
|
package {{
|
|
default_applicable_licenses: ["Android-Apache-2.0"],
|
|
}}
|
|
|
|
{modules}
|
|
"""
|
|
|
|
|
|
def cc_test(
|
|
name: str,
|
|
srcs: List[str],
|
|
host_supported: bool,
|
|
) -> str:
|
|
src_files = ',\n'.join(
|
|
[f'"{f}"' for f in srcs])
|
|
|
|
return f"""\
|
|
cc_test {{
|
|
name: "{name}",
|
|
srcs: [
|
|
{src_files},
|
|
],
|
|
test_options: {{
|
|
unit_test: true,
|
|
}},
|
|
host_supported: {str(host_supported).lower()},
|
|
}}
|
|
"""
|
|
|
|
|
|
def java_test_host(
|
|
name: str,
|
|
srcs: List[str],
|
|
unit_test: bool,
|
|
) -> str:
|
|
src_files = ',\n'.join(
|
|
[f'"{f}"' for f in srcs])
|
|
|
|
return f"""\
|
|
java_test_host {{
|
|
name: "{name}",
|
|
srcs: [
|
|
{src_files},
|
|
],
|
|
test_options: {{
|
|
unit_test: {str(unit_test).lower()},
|
|
}},
|
|
static_libs: [
|
|
"junit",
|
|
],
|
|
}}
|
|
"""
|
|
|
|
def java_library(
|
|
name: str,
|
|
srcs: List[str],
|
|
) -> str:
|
|
src_files = ',\n'.join(
|
|
[f'"{f}"' for f in srcs])
|
|
|
|
return f"""\
|
|
java_library_host {{
|
|
name: "{name}",
|
|
srcs: [
|
|
{src_files},
|
|
],
|
|
libs: [
|
|
"tradefed",
|
|
],
|
|
}}
|
|
"""
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main(verbosity=2)
|