222 lines
6.2 KiB
Python
222 lines
6.2 KiB
Python
#!/usr/bin/env python3
|
||
# Copyright (C) 2023 The Android Open Source Project
|
||
#
|
||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
# you may not use this file except in compliance with the License.
|
||
# You may obtain a copy of the License at
|
||
#
|
||
# http://www.apache.org/licenses/LICENSE-2.0
|
||
#
|
||
# Unless required by applicable law or agreed to in writing, software
|
||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
# See the License for the specific language governing permissions and
|
||
# limitations under the License.
|
||
|
||
import inspect
|
||
import os
|
||
from dataclasses import dataclass
|
||
from typing import List, Union, Callable
|
||
from enum import Enum
|
||
import re
|
||
|
||
from google.protobuf import text_format
|
||
|
||
TestName = str
|
||
|
||
|
||
@dataclass
|
||
class Path:
|
||
filename: str
|
||
|
||
|
||
@dataclass
|
||
class DataPath(Path):
|
||
filename: str
|
||
|
||
|
||
@dataclass
|
||
class Metric:
|
||
name: str
|
||
|
||
|
||
@dataclass
|
||
class Json:
|
||
contents: str
|
||
|
||
|
||
@dataclass
|
||
class Csv:
|
||
contents: str
|
||
|
||
|
||
@dataclass
|
||
class TextProto:
|
||
contents: str
|
||
|
||
|
||
@dataclass
|
||
class BinaryProto:
|
||
message_type: str
|
||
contents: str
|
||
# Comparing protos is tricky. For example, repeated fields might be written in
|
||
# any order. To help with that you can specify a `post_processing` function
|
||
# that will be called with the actual proto message object before converting
|
||
# it to text representation and doing the comparison with `contents`. This
|
||
# gives us a chance to e.g. sort messages in a repeated field.
|
||
post_processing: Callable = text_format.MessageToString
|
||
|
||
|
||
@dataclass
|
||
class Systrace:
|
||
contents: str
|
||
|
||
|
||
class TestType(Enum):
|
||
QUERY = 1
|
||
METRIC = 2
|
||
|
||
|
||
# Blueprint for running the diff test. 'query' is being run over data from the
|
||
# 'trace 'and result will be compared to the 'out. Each test (function in class
|
||
# inheriting from TestSuite) returns a DiffTestBlueprint.
|
||
@dataclass
|
||
class DiffTestBlueprint:
|
||
|
||
trace: Union[Path, DataPath, Json, Systrace, TextProto]
|
||
query: Union[str, Path, DataPath, Metric]
|
||
out: Union[Path, DataPath, Json, Csv, TextProto, BinaryProto]
|
||
|
||
def is_trace_file(self):
|
||
return isinstance(self.trace, Path)
|
||
|
||
def is_trace_textproto(self):
|
||
return isinstance(self.trace, TextProto)
|
||
|
||
def is_trace_json(self):
|
||
return isinstance(self.trace, Json)
|
||
|
||
def is_trace_systrace(self):
|
||
return isinstance(self.trace, Systrace)
|
||
|
||
def is_query_file(self):
|
||
return isinstance(self.query, Path)
|
||
|
||
def is_metric(self):
|
||
return isinstance(self.query, Metric)
|
||
|
||
def is_out_file(self):
|
||
return isinstance(self.out, Path)
|
||
|
||
def is_out_json(self):
|
||
return isinstance(self.out, Json)
|
||
|
||
def is_out_texproto(self):
|
||
return isinstance(self.out, TextProto)
|
||
|
||
def is_out_binaryproto(self):
|
||
return isinstance(self.out, BinaryProto)
|
||
|
||
def is_out_csv(self):
|
||
return isinstance(self.out, Csv)
|
||
|
||
|
||
# Description of a diff test. Created in `fetch_diff_tests()` in
|
||
# TestSuite: each test (function starting with `test_`) returns
|
||
# DiffTestBlueprint and function name is a TestCase name. Used by diff test
|
||
# script.
|
||
class TestCase:
|
||
|
||
def __get_query_path(self) -> str:
|
||
if not self.blueprint.is_query_file():
|
||
return None
|
||
|
||
if isinstance(self.blueprint.query, DataPath):
|
||
path = os.path.join(self.test_dir, 'data', self.blueprint.query.filename)
|
||
else:
|
||
path = os.path.abspath(
|
||
os.path.join(self.index_dir, self.blueprint.query.filename))
|
||
|
||
if not os.path.exists(path):
|
||
raise AssertionError(
|
||
f"Query file ({path}) for test '{self.name}' does not exist.")
|
||
return path
|
||
|
||
def __get_trace_path(self) -> str:
|
||
if not self.blueprint.is_trace_file():
|
||
return None
|
||
|
||
if isinstance(self.blueprint.trace, DataPath):
|
||
path = os.path.join(self.test_dir, 'data', self.blueprint.trace.filename)
|
||
else:
|
||
path = os.path.abspath(
|
||
os.path.join(self.index_dir, self.blueprint.trace.filename))
|
||
|
||
if not os.path.exists(path):
|
||
raise AssertionError(
|
||
f"Trace file ({path}) for test '{self.name}' does not exist.")
|
||
return path
|
||
|
||
def __get_out_path(self) -> str:
|
||
if not self.blueprint.is_out_file():
|
||
return None
|
||
|
||
if isinstance(self.blueprint.out, DataPath):
|
||
path = os.path.join(self.test_dir, 'data', self.blueprint.out.filename)
|
||
else:
|
||
path = os.path.abspath(
|
||
os.path.join(self.index_dir, self.blueprint.out.filename))
|
||
|
||
if not os.path.exists(path):
|
||
raise AssertionError(
|
||
f"Out file ({path}) for test '{self.name}' does not exist.")
|
||
return path
|
||
|
||
def __init__(self, name: str, blueprint: DiffTestBlueprint,
|
||
index_dir: str) -> None:
|
||
self.name = name
|
||
self.blueprint = blueprint
|
||
self.index_dir = index_dir
|
||
self.test_dir = os.path.dirname(os.path.dirname(os.path.dirname(index_dir)))
|
||
|
||
if blueprint.is_metric():
|
||
self.type = TestType.METRIC
|
||
else:
|
||
self.type = TestType.QUERY
|
||
|
||
self.query_path = self.__get_query_path()
|
||
self.trace_path = self.__get_trace_path()
|
||
self.expected_path = self.__get_out_path()
|
||
|
||
# Verifies that the test should be in test suite. If False, test will not be
|
||
# executed.
|
||
def validate(self, name_filter: str):
|
||
query_metric_pattern = re.compile(name_filter)
|
||
return bool(query_metric_pattern.match(os.path.basename(self.name)))
|
||
|
||
|
||
# Virtual class responsible for fetching diff tests.
|
||
# All functions with name starting with `test_` have to return
|
||
# DiffTestBlueprint and function name is a test name. All DiffTestModules have
|
||
# to be included in `test/diff_tests/trace_processor/include_index.py`.
|
||
# `fetch_diff_test` function should not be overwritten.
|
||
class TestSuite:
|
||
|
||
def __init__(self, include_index_dir: str, dir_name: str,
|
||
class_name: str) -> None:
|
||
self.dir_name = dir_name
|
||
self.index_dir = os.path.join(include_index_dir, dir_name)
|
||
self.class_name = class_name
|
||
|
||
def __test_name(self, method_name):
|
||
return f"{self.class_name}:{method_name.split('test_',1)[1]}"
|
||
|
||
def fetch(self) -> List['TestCase']:
|
||
attrs = (getattr(self, name) for name in dir(self))
|
||
methods = [attr for attr in attrs if inspect.ismethod(attr)]
|
||
return [
|
||
TestCase(self.__test_name(method.__name__), method(), self.index_dir)
|
||
for method in methods
|
||
if method.__name__.startswith('test_')
|
||
]
|