467 lines
14 KiB
Python
467 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
#
|
|
# Copyright (C) 2022 The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Utility functions to produce module or module type dependency graphs using json-module-graph or queryview."""
|
|
|
|
from typing import Set
|
|
import collections
|
|
import dataclasses
|
|
import json
|
|
import os
|
|
import os.path
|
|
import subprocess
|
|
import sys
|
|
import xml.etree.ElementTree
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True, order=True)
|
|
class _ModuleKey:
|
|
"""_ModuleKey uniquely identifies a module by name nad variations."""
|
|
name: str
|
|
variations: list
|
|
|
|
def __str__(self):
|
|
return f"{self.name}, {self.variations}"
|
|
|
|
def __hash__(self):
|
|
return (self.name + str(self.variations)).__hash__()
|
|
|
|
|
|
# This list of module types are omitted from the report and graph
|
|
# for brevity and simplicity. Presence in this list doesn't mean
|
|
# that they shouldn't be converted, but that they are not that useful
|
|
# to be recorded in the graph or report currently.
|
|
IGNORED_KINDS = set([
|
|
"cc_defaults",
|
|
"hidl_package_root", # not being converted, contents converted as part of hidl_interface
|
|
"java_defaults",
|
|
"license",
|
|
"license_kind",
|
|
])
|
|
|
|
# queryview doesn't have information on the type of deps, so we explicitly skip
|
|
# prebuilt types
|
|
_QUERYVIEW_IGNORE_KINDS = set([
|
|
"android_app_import",
|
|
"android_library_import",
|
|
"cc_prebuilt_library",
|
|
"cc_prebuilt_library_headers",
|
|
"cc_prebuilt_library_shared",
|
|
"cc_prebuilt_library_static",
|
|
"cc_prebuilt_library_static",
|
|
"cc_prebuilt_object",
|
|
"java_import",
|
|
"java_import_host",
|
|
"java_sdk_library_import",
|
|
])
|
|
|
|
SRC_ROOT_DIR = os.path.abspath(__file__ + "/../../../../..")
|
|
|
|
LUNCH_ENV = {
|
|
# Use aosp_arm as the canonical target product.
|
|
"TARGET_PRODUCT": "aosp_arm",
|
|
"TARGET_BUILD_VARIANT": "userdebug",
|
|
}
|
|
|
|
BANCHAN_ENV = {
|
|
# Use module_arm64 as the canonical banchan target product.
|
|
"TARGET_PRODUCT": "module_arm64",
|
|
"TARGET_BUILD_VARIANT": "eng",
|
|
# just needs to be non-empty, not the specific module for Soong
|
|
# analysis purposes
|
|
"TARGET_BUILD_APPS": "all",
|
|
}
|
|
|
|
|
|
def _build_with_soong(target, banchan_mode=False):
|
|
subprocess.check_output(
|
|
[
|
|
"build/soong/soong_ui.bash",
|
|
"--make-mode",
|
|
"--skip-soong-tests",
|
|
target,
|
|
],
|
|
cwd=SRC_ROOT_DIR,
|
|
env=BANCHAN_ENV if banchan_mode else LUNCH_ENV,
|
|
)
|
|
|
|
|
|
def get_properties(json_module):
|
|
set_properties = {}
|
|
if "Module" not in json_module:
|
|
return set_properties
|
|
if "Android" not in json_module["Module"]:
|
|
return set_properties
|
|
if "SetProperties" not in json_module["Module"]["Android"]:
|
|
return set_properties
|
|
|
|
for prop in json_module['Module']['Android']['SetProperties']:
|
|
if prop["Values"]:
|
|
value = prop["Values"]
|
|
else:
|
|
value = prop["Value"]
|
|
set_properties[prop["Name"]] = value
|
|
return set_properties
|
|
|
|
|
|
def get_property_names(json_module):
|
|
return get_properties(json_module).keys()
|
|
|
|
|
|
def get_queryview_module_info(modules, banchan_mode):
|
|
"""Returns the list of transitive dependencies of input module as built by queryview."""
|
|
_build_with_soong("queryview", banchan_mode)
|
|
|
|
queryview_xml = subprocess.check_output(
|
|
[
|
|
"build/bazel/bin/bazel",
|
|
"query",
|
|
"--config=ci",
|
|
"--config=queryview",
|
|
"--output=xml",
|
|
# union of queries to get the deps of all Soong modules with the give names
|
|
" + ".join(f'deps(attr("soong_module_name", "^{m}$", //...))'
|
|
for m in modules)
|
|
],
|
|
cwd=SRC_ROOT_DIR,
|
|
)
|
|
try:
|
|
return xml.etree.ElementTree.fromstring(queryview_xml)
|
|
except xml.etree.ElementTree.ParseError as err:
|
|
sys.exit(f"""Could not parse XML:
|
|
{queryview_xml}
|
|
ParseError: {err}""")
|
|
|
|
|
|
def get_json_module_info(banchan_mode=False):
|
|
"""Returns the list of transitive dependencies of input module as provided by Soong's json module graph."""
|
|
_build_with_soong("json-module-graph", banchan_mode)
|
|
try:
|
|
with open(os.path.join(SRC_ROOT_DIR,"out/soong/module-graph.json")) as f:
|
|
return json.load(f)
|
|
except json.JSONDecodeError as err:
|
|
sys.exit(f"""Could not decode json:
|
|
out/soong/module-graph.json
|
|
JSONDecodeError: {err}""")
|
|
|
|
|
|
def _ignore_json_module(json_module, ignore_by_name):
|
|
# windows is not a priority currently
|
|
if is_windows_variation(json_module):
|
|
return True
|
|
if ignore_kind(json_module['Type']):
|
|
return True
|
|
if json_module['Name'] in ignore_by_name:
|
|
return True
|
|
# for filegroups with a name the same as the source, we are not migrating the
|
|
# filegroup and instead just rely on the filename being exported
|
|
if json_module['Type'] == 'filegroup':
|
|
set_properties = get_properties(json_module)
|
|
srcs = set_properties.get('Srcs', [])
|
|
if len(srcs) == 1:
|
|
return json_module['Name'] in srcs
|
|
return False
|
|
|
|
|
|
def visit_json_module_graph_post_order(
|
|
module_graph, ignore_by_name, ignore_java_auto_deps, filter_predicate, visit
|
|
):
|
|
# The set of ignored modules. These modules (and their dependencies) are not shown
|
|
# in the graph or report.
|
|
ignored = set()
|
|
|
|
# name to all module variants
|
|
module_graph_map = {}
|
|
root_module_keys = []
|
|
name_to_keys = collections.defaultdict(set)
|
|
|
|
# Do a single pass to find all top-level modules to be ignored
|
|
for module in module_graph:
|
|
name = module["Name"]
|
|
key = _ModuleKey(name, module["Variations"])
|
|
if _ignore_json_module(module, ignore_by_name):
|
|
ignored.add(key)
|
|
continue
|
|
name_to_keys[name].add(key)
|
|
module_graph_map[key] = module
|
|
if filter_predicate(module):
|
|
root_module_keys.append(key)
|
|
|
|
visited = set()
|
|
|
|
def json_module_graph_post_traversal(module_key):
|
|
if module_key in ignored or module_key in visited:
|
|
return
|
|
visited.add(module_key)
|
|
|
|
deps = set()
|
|
module = module_graph_map[module_key]
|
|
created_by = module["CreatedBy"]
|
|
to_visit = set()
|
|
|
|
if created_by:
|
|
for key in name_to_keys[created_by]:
|
|
if key in ignored:
|
|
continue
|
|
# treat created by as a dep so it appears as a blocker, otherwise the
|
|
# module will be disconnected from the traversal graph despite having a
|
|
# direct relationship to a module and must addressed in the migration
|
|
deps.add(created_by)
|
|
json_module_graph_post_traversal(key)
|
|
|
|
for dep in module["Deps"]:
|
|
if ignore_json_dep(
|
|
dep, module["Name"], ignored, ignore_java_auto_deps
|
|
):
|
|
continue
|
|
|
|
dep_name = dep["Name"]
|
|
deps.add(dep_name)
|
|
dep_key = _ModuleKey(dep_name, dep["Variations"])
|
|
|
|
if dep_key not in visited:
|
|
json_module_graph_post_traversal(dep_key)
|
|
|
|
visit(module, deps)
|
|
|
|
for module_key in root_module_keys:
|
|
json_module_graph_post_traversal(module_key)
|
|
|
|
|
|
QueryviewModule = collections.namedtuple("QueryviewModule", [
|
|
"name",
|
|
"kind",
|
|
"variant",
|
|
"dirname",
|
|
"deps",
|
|
"srcs",
|
|
])
|
|
|
|
|
|
def _bazel_target_to_dir(full_target):
|
|
dirname, _ = full_target.split(":")
|
|
return dirname[len("//"):] # discard prefix
|
|
|
|
|
|
def _get_queryview_module(name_with_variant, module, kind):
|
|
name = None
|
|
variant = ""
|
|
deps = []
|
|
srcs = []
|
|
for attr in module:
|
|
attr_name = attr.attrib["name"]
|
|
if attr.tag == "rule-input":
|
|
deps.append(attr_name)
|
|
elif attr_name == "soong_module_name":
|
|
name = attr.attrib["value"]
|
|
elif attr_name == "soong_module_variant":
|
|
variant = attr.attrib["value"]
|
|
elif attr_name == "soong_module_type" and kind == "generic_soong_module":
|
|
kind = attr.attrib["value"]
|
|
elif attr_name == "srcs":
|
|
for item in attr:
|
|
srcs.append(item.attrib["value"])
|
|
|
|
return QueryviewModule(
|
|
name=name,
|
|
kind=kind,
|
|
variant=variant,
|
|
dirname=_bazel_target_to_dir(name_with_variant),
|
|
deps=deps,
|
|
srcs=srcs,
|
|
)
|
|
|
|
|
|
def _ignore_queryview_module(module, ignore_by_name):
|
|
if module.name in ignore_by_name:
|
|
return True
|
|
if ignore_kind(module.kind, queryview=True):
|
|
return True
|
|
# special handling for filegroup srcs, if a source has the same name as
|
|
# the filegroup module, we don't convert it
|
|
if module.kind == "filegroup" and module.name in module.srcs:
|
|
return True
|
|
return module.variant.startswith("windows")
|
|
|
|
|
|
def visit_queryview_xml_module_graph_post_order(module_graph, ignored_by_name,
|
|
filter_predicate, visit):
|
|
# The set of ignored modules. These modules (and their dependencies) are
|
|
# not shown in the graph or report.
|
|
ignored = set()
|
|
|
|
# queryview embeds variant in long name, keep a map of the name with vaiarnt
|
|
# to just name
|
|
name_with_variant_to_name = dict()
|
|
|
|
module_graph_map = dict()
|
|
to_visit = []
|
|
|
|
for module in module_graph:
|
|
ignore = False
|
|
if module.tag != "rule":
|
|
continue
|
|
kind = module.attrib["class"]
|
|
name_with_variant = module.attrib["name"]
|
|
|
|
qv_module = _get_queryview_module(name_with_variant, module, kind)
|
|
|
|
if _ignore_queryview_module(qv_module, ignored_by_name):
|
|
ignored.add(name_with_variant)
|
|
continue
|
|
|
|
if filter_predicate(qv_module):
|
|
to_visit.append(name_with_variant)
|
|
|
|
name_with_variant_to_name.setdefault(name_with_variant, qv_module.name)
|
|
module_graph_map[name_with_variant] = qv_module
|
|
|
|
visited = set()
|
|
|
|
def queryview_module_graph_post_traversal(name_with_variant):
|
|
module = module_graph_map[name_with_variant]
|
|
if name_with_variant in ignored or name_with_variant in visited:
|
|
return
|
|
visited.add(name_with_variant)
|
|
|
|
name = name_with_variant_to_name[name_with_variant]
|
|
|
|
deps = set()
|
|
for dep_name_with_variant in module.deps:
|
|
if dep_name_with_variant in ignored:
|
|
continue
|
|
dep_name = name_with_variant_to_name[dep_name_with_variant]
|
|
if dep_name == "prebuilt_" + name:
|
|
continue
|
|
if dep_name_with_variant not in visited:
|
|
queryview_module_graph_post_traversal(dep_name_with_variant)
|
|
|
|
if name != dep_name:
|
|
deps.add(dep_name)
|
|
|
|
visit(module, deps)
|
|
|
|
for name_with_variant in to_visit:
|
|
queryview_module_graph_post_traversal(name_with_variant)
|
|
|
|
|
|
def get_bp2build_converted_modules() -> Set[str]:
|
|
""" Returns the list of modules that bp2build can currently convert. """
|
|
_build_with_soong("bp2build")
|
|
# Parse the list of converted module names from bp2build
|
|
with open(
|
|
os.path.join(SRC_ROOT_DIR,
|
|
"out/soong/soong_injection/metrics/converted_modules.txt"),
|
|
"r") as f:
|
|
# Read line by line, excluding comments.
|
|
# Each line is a module name.
|
|
ret = set(line.strip() for line in f if not line.strip().startswith("#"))
|
|
return ret
|
|
|
|
|
|
def get_json_module_type_info(module_type):
|
|
"""Returns the combined transitive dependency closures of all modules of module_type."""
|
|
_build_with_soong("json-module-graph")
|
|
# Run query.sh on the module graph for the top level module type
|
|
result = subprocess.check_output(
|
|
[
|
|
"build/bazel/json_module_graph/query.sh",
|
|
"fullTransitiveModuleTypeDeps", "out/soong/module-graph.json",
|
|
module_type
|
|
],
|
|
cwd=SRC_ROOT_DIR,
|
|
)
|
|
return json.loads(result)
|
|
|
|
|
|
def is_windows_variation(module):
|
|
"""Returns True if input module's variant is Windows.
|
|
|
|
Args:
|
|
module: an entry parsed from Soong's json-module-graph
|
|
"""
|
|
dep_variations = module.get("Variations")
|
|
dep_variation_os = ""
|
|
if dep_variations != None:
|
|
for v in dep_variations:
|
|
if v["Mutator"] == "os":
|
|
dep_variation_os = v["Variation"]
|
|
return dep_variation_os == "windows"
|
|
|
|
|
|
def ignore_kind(kind, queryview=False):
|
|
if queryview and kind in _QUERYVIEW_IGNORE_KINDS:
|
|
return True
|
|
return kind in IGNORED_KINDS or "defaults" in kind
|
|
|
|
|
|
def is_prebuilt_to_source_dep(dep):
|
|
# Soong always adds a dependency from a source module to its corresponding
|
|
# prebuilt module, if it exists.
|
|
# https://cs.android.com/android/platform/superproject/+/master:build/soong/android/prebuilt.go;l=395-396;drc=5d6fa4d8571d01a6e5a63a8b7aa15e61f45737a9
|
|
# This makes it appear that the prebuilt is a transitive dependency regardless
|
|
# of whether it is actually necessary. Skip these to keep the graph to modules
|
|
# used to build.
|
|
return dep["Tag"] == "android.prebuiltDependencyTag {BaseDependencyTag:{}}"
|
|
|
|
|
|
def _is_java_auto_dep(dep):
|
|
# Soong adds a number of dependencies automatically for Java deps, making it
|
|
# difficult to understand the actual dependencies, remove the
|
|
# non-user-specified deps
|
|
tag = dep["Tag"]
|
|
if not tag:
|
|
return False
|
|
return (
|
|
(
|
|
tag.startswith("java.dependencyTag")
|
|
and (
|
|
"name:proguard-raise" in tag
|
|
or "name:bootclasspath" in tag
|
|
or "name:system modules" in tag
|
|
or "name:framework-res" in tag
|
|
or "name:sdklib" in tag
|
|
or "name:java9lib" in tag
|
|
)
|
|
or (
|
|
tag.startswith("java.usesLibraryDependencyTag")
|
|
or tag.startswith("java.hiddenAPIStubsDependencyTag")
|
|
)
|
|
)
|
|
or (
|
|
tag.startswith("android.sdkMemberDependencyTag")
|
|
or tag.startswith("java.scopeDependencyTag")
|
|
)
|
|
or tag.startswith("dexpreopt.dex2oatDependencyTag")
|
|
)
|
|
|
|
|
|
def ignore_json_dep(dep, module_name, ignored_keys, ignore_java_auto_deps):
|
|
"""Whether to ignore a json dependency based on heuristics.
|
|
|
|
Args:
|
|
dep: dependency struct from an entry in Soogn's json-module-graph
|
|
module_name: name of the module this is a dependency of
|
|
ignored_names: a set of _ModuleKey to ignore
|
|
"""
|
|
if is_prebuilt_to_source_dep(dep):
|
|
return True
|
|
if ignore_java_auto_deps and _is_java_auto_dep(dep):
|
|
return True
|
|
name = dep["Name"]
|
|
return (
|
|
_ModuleKey(name, dep["Variations"]) in ignored_keys or name == module_name
|
|
)
|