werunplugged/unplugged-system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
06ee9615ad
unplugged-system/external/perfetto/tools/install-build-deps

751 lines
30 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import dataclasses as dc
import hashlib
import logging
import os
import shutil
import subprocess
import stat
import sys
import tempfile
import time
import zipfile
from collections import namedtuple
from platform import system, machine
# The format for the deps below is the following:
# (target_folder, source_url, sha1, target_os, target_arch)
# |source_url| can be either a git repo or a http url.
# If a git repo, |checksum| is the SHA1 committish that will be checked out.
# If a http url, |checksum| is the SHA256 of the downloaded file.
# If the url is a .zip or .tgz file it will be automatically deflated under
# |target_folder|, taking care of stripping the root folder if it's a single
# root (to avoid ending up with buildtools/protobuf/protobuf-1.2.3/... and have
# instead just buildtools/protobuf).
# |target_os| is either 'darwin', 'linux', 'windows' or 'all'
# |target_arch| is either 'x64', 'arm64' or 'all'
# in both cases the dep only applies on matching platforms
# |target_arch| can be 'all' when 'target_os' is not 'all' for example in the
# case of MacOS universal binaries.
@dc.dataclass
class Dependency:
target_folder: str
source_url: str
checksum: str
target_os: str
target_arch: str
submodules: bool = False
# This is to remove old directories when build tools get {re,}moved. This is to
# avoid accidentally referring to stale dir in custom user scripts.
CLEANUP_OLD_DIRS = [
'buildtools/nodejs', # Moved to buildtools/{mac,linux64}/nodejs
'buildtools/emsdk', # Moved to buildtools/{mac,linux64}/emsdk
'buildtools/test_data', # Moved to test/data by r.android.com/1539381 .
'buildtools/d8', # Removed by r.android.com/1424334 .
# Build toools moved to third_party/ by r.android.com/2327602 .
'buildtools/mac/clang-format',
'buildtools/mac/gn',
'buildtools/mac/ninja',
'buildtools/linux64/clang-format',
'buildtools/linux64/gn',
'buildtools/linux64/ninja',
'buildtools/win/clang-format.exe',
'buildtools/win/gn.exe',
'buildtools/win/ninja.exe',
]
# Dependencies required to build code on the host or when targeting desktop OS.
BUILD_DEPS_TOOLCHAIN_HOST = [
# GN. From https://chrome-infra-packages.appspot.com/dl/gn/gn/.
# git_revision:0725d7827575b239594fbc8fd5192873a1d62f44 .
Dependency(
'third_party/gn/gn',
'https://storage.googleapis.com/perfetto/gn-mac-1968-0725d782',
'9ced623a664560bba38bbadb9b91158ca4186358c847e17ab7d982b351373c2e',
'darwin', 'x64'),
Dependency(
'third_party/gn/gn',
'https://storage.googleapis.com/perfetto/gn-mac-arm64-1968-0725d782',
'd22336b5210b4dad5e36e8c28ce81187f491822cf4d8fd0a257b30d6bee3fd3f',
'darwin', 'arm64'),
Dependency(
'third_party/gn/gn',
'https://storage.googleapis.com/perfetto/gn-linux64-1968-0725d782',
'f706aaa0676e3e22f5fc9ca482295d7caee8535d1869f99efa2358177b64f5cd',
'linux', 'x64'),
Dependency(
'third_party/gn/gn.exe',
'https://storage.googleapis.com/perfetto/gn-win-1968-0725d782',
'001f777f023c7a6959c778fb3a6b6cfc63f6baef953410ecdeaec350fb12285b',
'windows', 'x64'),
# clang-format
# From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/mac/clang-format.sha1
Dependency(
'third_party/clang-format/clang-format',
'https://storage.googleapis.com/chromium-clang-format/62bde1baa7196ad9df969fc1f06b66360b1a927b',
'6df686a937443cbe6efc013467a7ba5f98d3f187eb7765bb7abc6ce47626cf66',
'darwin', 'all'),
# From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/linux64/clang-format.sha1
Dependency(
'third_party/clang-format/clang-format',
'https://storage.googleapis.com/chromium-clang-format/1baf0089e895c989a311b6a38ed94d0e8be4c0a7',
'd02a97a87e8c28898033aaf5986967b24dc47ebd5b376e1cd93e5009f22cd75e',
'linux', 'x64'),
# From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/win/clang-format.exe.sha1
Dependency(
'third_party/clang-format/clang-format.exe',
'https://storage.googleapis.com/chromium-clang-format/d4afd4eba27022f5f6d518133aebde57281677c9',
'2ba1b4d3ade90ea80316890b598ab5fc16777572be26afec6ce23117da121b80',
'windows', 'x64'),
# Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS.
Dependency(
'buildtools/clang_format/script',
'https://chromium.googlesource.com/chromium/llvm-project/cfe/tools/clang-format.git',
'96636aa0e9f047f17447f2d45a094d0b59ed7917', 'all', 'all'),
# Ninja
Dependency(
'third_party/ninja/ninja',
'https://storage.googleapis.com/perfetto/ninja-mac-x64_and_arm64-182',
'36e8b7aaa06911e1334feb664dd731a1cd69a15eb916a231a3d10ff65fca2c73',
'darwin', 'all'),
Dependency(
'third_party/ninja/ninja',
'https://storage.googleapis.com/perfetto/ninja-linux64-182',
'54ac6a01362190aaabf4cf276f9c8982cdf11b225438940fdde3339be0f2ecdc',
'linux', 'x64'),
Dependency(
'third_party/ninja/ninja.exe',
'https://storage.googleapis.com/perfetto/ninja-win-182',
'09ced0fcd1a4dec7d1b798a2cf9ce5d20e5d2fbc2337343827f192ce47d0f491',
'windows', 'x64'),
# Keep the revision in sync with Chrome's PACKAGE_VERSION in
# tools/clang/scripts/update.py.
Dependency(
'buildtools/linux64/clang.tgz',
'https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64/clang-llvmorg-16-init-8697-g60809cd2-1.tgz',
'5ae35f85e0d32136795c6b223bf64263d46678dd4a24fea4e9039e58a32670de',
'linux', 'x64'),
Dependency(
'buildtools/win/clang.tgz',
'https://commondatastorage.googleapis.com/chromium-browser-clang/Win/clang-llvmorg-16-init-8697-g60809cd2-1.tgz',
'086faec822acba5b9c0308c6a8be34424031027d757efa2b81805aed18ffc521',
'windows', 'x64'),
]
BUILD_DEPS_HOST = [
# Keep in sync with Android's //external/googletest/METADATA.
Dependency(
'buildtools/googletest',
'https://android.googlesource.com/platform/external/googletest.git',
'609281088cfefc76f9d0ce82e1ff6c30cc3591e5', 'all', 'all'),
# Keep in sync with Chromium's //third_party/protobuf.
Dependency(
'buildtools/protobuf',
'https://chromium.googlesource.com/external/github.com/protocolbuffers/protobuf.git',
'fe271ab76f2ad2b2b28c10443865d2af21e27e0e', # refs/tags/v3.20.3
'all',
'all'),
# libc++, libc++abi and libunwind for Linux where we need to rebuild the C++
# lib from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS.
Dependency(
'buildtools/libcxx',
'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxx.git',
'f8571eaba606bde2eb8cd34b30104ca33e7c207e', 'all', 'all'),
Dependency(
'buildtools/libcxxabi',
'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxxabi.git',
'8dd405113a4f3694e910b79785dd7fb7535a888a', 'all', 'all'),
Dependency(
'buildtools/libunwind',
'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libunwind.git',
'aabcd8753678f1536e15eb6385a948470debdae4', 'all', 'all'),
# Keep in sync with chromium DEPS.
Dependency(
'buildtools/libfuzzer',
'https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer.git',
'debe7d2d1982e540fbd6bd78604bf001753f9e74', 'linux', 'all'),
# Benchmarking tool.
Dependency(
'buildtools/benchmark',
'https://chromium.googlesource.com/external/github.com/google/benchmark.git',
'e991355c02b93fe17713efe04cbc2e278e00fdbd', 'all', 'all'),
# Libbacktrace, for stacktraces in Linux/Android debug builds.
# From https://github.com/ianlancetaylor/libbacktrace/archive/177940370e4a6b2509e92a0aaa9749184e64af43.zip
Dependency(
'buildtools/libbacktrace.zip',
'https://storage.googleapis.com/perfetto/libbacktrace-177940370e4a6b2509e92a0aaa9749184e64af43.zip',
'21ac9a4209f7aeef766c482be53a7fa365063c031c7077e2070b491202983b31',
'all', 'all'),
# Sqlite for the trace processing library.
# This is the amalgamated source whose compiled output is meant to be faster.
# We still pull the full source for the extensions (which are not available
# in the amalgamation).
# If updating the version, also update bazel/deps.bzl.
Dependency(
'buildtools/sqlite.zip',
'https://storage.googleapis.com/perfetto/sqlite-amalgamation-3390200.zip',
'87775784f8b22d0d0f1d7811870d39feaa7896319c7c20b849a4181c5a50609b',
'all', 'all'),
Dependency(
'buildtools/sqlite_src',
'https://chromium.googlesource.com/external/github.com/sqlite/sqlite.git',
'202b2a7b54ea2dd13a8a5adfd75523abe4dcf17f', # refs/tags/version-3.39.2.
'all',
'all'),
# JsonCpp for legacy json import. Used only by the trace processor in
# standalone builds.
# If updating the version, also update bazel/deps.bzl.
Dependency(
'buildtools/jsoncpp',
'https://chromium.googlesource.com/external/github.com/open-source-parsers/jsoncpp.git',
'6aba23f4a8628d599a9ef7fa4811c4ff6e4070e2', # refs/tags/1.9.3.
'all',
'all'),
# Archive with only the demangling sources from llvm-project.
# See tools/repackage_llvm_demangler.sh on how to update this.
# File suffix is the git reference to the commit at which we rearchived the
# sources, as hosted on https://llvm.googlesource.com/llvm-project.
# If updating the version, also update bazel/deps.bzl.
Dependency(
'buildtools/llvm-project.tgz',
'https://storage.googleapis.com/perfetto/llvm-project-3b4c59c156919902c785ce3cbae0eee2ee53064d.tgz',
'f4a52e7f36edd7cacc844d5ae0e5f60b6f57c5afc40683e99f295886c9ce8ff4',
'all', 'all'),
# These dependencies are for libunwindstack, which is used by src/profiling.
Dependency('buildtools/android-core',
'https://android.googlesource.com/platform/system/core.git',
'9e6cef7f07d8c11b3ea820938aeb7ff2e9dbaa52', 'all', 'all'),
Dependency(
'buildtools/android-unwinding',
'https://android.googlesource.com/platform/system/unwinding.git',
'11cf5564fbff659b5cdba9c3e977d4fc51bab4b3', 'all', 'all'),
Dependency('buildtools/android-logging',
'https://android.googlesource.com/platform/system/logging.git',
'7b36b566c9113fc703d68f76e8f40c0c2432481c', 'all', 'all'),
Dependency('buildtools/android-libbase',
'https://android.googlesource.com/platform/system/libbase.git',
'78f1c2f83e625bdf66d55b48bdb3a301c20d2fb3', 'all', 'all'),
Dependency(
'buildtools/android-libprocinfo',
'https://android.googlesource.com/platform/system/libprocinfo.git',
'fd214c13ededecae97a3b15b5fccc8925a749a84', 'all', 'all'),
Dependency('buildtools/lzma',
'https://android.googlesource.com/platform/external/lzma.git',
'7851dce6f4ca17f5caa1c93a4e0a45686b1d56c3', 'all', 'all'),
Dependency('buildtools/bionic',
'https://android.googlesource.com/platform/bionic.git',
'4b0e16bc72a82a63c699977376a7d6eadca1b206', 'all', 'all'),
# Zlib used both in the tracing binaries, as well as the trace processor and
# assorted tools.
# If updating the version, also update bazel/deps.bzl.
Dependency('buildtools/zlib',
'https://android.googlesource.com/platform/external/zlib.git',
'6d3f6aa0f87c9791ca7724c279ef61384f331dfd', 'all', 'all'),
# Linenoise, used only by trace_processor in standalone builds.
# If updating the version, also update bazel/deps.bzl.
Dependency('buildtools/linenoise',
'https://fuchsia.googlesource.com/third_party/linenoise.git',
'c894b9e59f02203dbe4e2be657572cf88c4230c3', 'all', 'all'),
# Bloaty, used to investigate binary size
Dependency(
'buildtools/bloaty.zip',
'https://storage.googleapis.com/perfetto/bloaty-1.1-b3b829de35babc2fe831b9488ad2e50bca939412-mac.zip',
'2d301bd72a20e3f42888c9274ceb4dca76c103608053572322412c2c65ab8cb8',
'darwin', 'all'),
]
# Dependencies required to build Android code.
# URLs and SHA1s taken from:
# - https://dl.google.com/android/repository/repository-11.xml
# - https://dl.google.com/android/repository/sys-img/android/sys-img.xml
BUILD_DEPS_ANDROID = [
# Android NDK
Dependency(
'buildtools/ndk.zip',
'https://dl.google.com/android/repository/android-ndk-r21e-darwin-x86_64.zip',
'437278103a3db12632c05b1be5c41bbb8522791a67e415cc54411a65366f499d',
'darwin', 'all'),
Dependency(
'buildtools/ndk.zip',
'https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip',
'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e',
'linux', 'x64'),
]
# Dependencies required to run Android tests.
TEST_DEPS_ANDROID = [
# Android emulator images.
Dependency(
'buildtools/aosp-arm.zip',
'https://storage.googleapis.com/perfetto/aosp-02022018-arm.zip',
'f5c7a3a22ad7aa0bd14ba467e8697e1e917d306699bd25622aa4419a413b9b67',
'all', 'all'),
# platform-tools.zip contains adb binaries.
Dependency(
'buildtools/android_sdk/platform-tools.zip',
'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip',
'98d392cbd21ca20d643c7e1605760cc49075611e317c534096b5564053f4ac8e',
'darwin', 'all'),
Dependency(
'buildtools/android_sdk/platform-tools.zip',
'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip',
'90208207521d85abf0d46e3374aa4e04b7aff74e4f355c792ac334de7a77e50b',
'linux', 'x64'),
# Android emulator binaries.
Dependency(
'buildtools/emulator',
'https://android.googlesource.com/platform/prebuilts/android-emulator.git',
'4b260028dc27bc92c39bee9129cb2ba839970956', 'all', 'x64'),
]
# This variable is updated by tools/roll-catapult-trace-viewer.
CATAPULT_SHA256 = 'b30108e05268ce6c65bb4126b65f6bfac165d17f5c1fd285046e7e6fd76c209f'
TYPEFACES_SHA256 = '1065172aaf0e9c22bc4f206ed9fdf5f1b4355d233fb21f9f26a89cd66c941940'
UI_DEPS = [
Dependency(
'buildtools/mac/nodejs.tgz',
'https://storage.googleapis.com/chromium-nodejs/16.13.0/31859fc1fa0994a95f44f09c367d6ff63607cfde',
'd9cf1f36b16e08ce52cfbdef427c6596eed2bd2f3608a79112d25b4ec8f75753',
'darwin', 'arm64'),
Dependency(
'buildtools/mac/nodejs.tgz',
'https://storage.googleapis.com/chromium-nodejs/16.13.0/16dfd094763b71988933a31735f9dea966f9abd6',
'065ced7e93f0e26276cb74d9688706674323865c4a8fc89e4adfa6868d4ebb4d',
'darwin', 'x64'),
Dependency(
'buildtools/linux64/nodejs.tgz',
'https://storage.googleapis.com/chromium-nodejs/16.13.0/ab9544e24e752d3d17f335fb7b2055062e582d11',
'3b5ca150a55f3aadfa18f3a10a4495aaf9653614ba3e460647170fef5287ec4f',
'linux', 'x64'),
Dependency(
'buildtools/mac/emsdk.tgz',
'https://storage.googleapis.com/perfetto/emscripten-2.0.12-mac.tgz',
'aa125f8c8ff8a386d43e18c8ea0c98c875cc19160a899403e8967a5478f96f31',
'darwin', 'all'),
Dependency(
'buildtools/linux64/emsdk.tgz',
'https://storage.googleapis.com/perfetto/emscripten-2.0.12-linux.tgz',
'bfff9fb0326363c12e19b542f27a5f12cedbfc310f30621dc497c9af51d2d2e3',
'linux', 'x64'),
Dependency(
'buildtools/catapult_trace_viewer.tgz',
'https://storage.googleapis.com/perfetto/catapult_trace_viewer-%s.tar.gz'
% CATAPULT_SHA256, CATAPULT_SHA256, 'all', 'all'),
Dependency(
'buildtools/typefaces.tgz',
'https://storage.googleapis.com/perfetto/typefaces-%s.tar.gz' %
TYPEFACES_SHA256, TYPEFACES_SHA256, 'all', 'all')
]
# Dependencies to build gRPC.
GRPC_DEPS = [
Dependency(
'buildtools/grpc/src',
'https://chromium.googlesource.com/external/github.com/grpc/grpc.git',
'6943c1841f57cac4666b165aea4f618fe73b3ff1', 'all', 'all', True),
]
# Sysroots required to cross-compile Linux targets (linux-arm{,64}).
# These are taken from Chromium's build/linux/sysroot_scripts/sysroots.json.
BUILD_DEPS_LINUX_CROSS_SYSROOTS = [
Dependency(
'buildtools/debian_sid_arm-sysroot.tgz',
'https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/11d6f690ca49e8ba01a1d8c5346cedad2cf308fd/debian_sid_arm_sysroot.tar.xz',
'ff192fe073d140d836c9ca1e68f7200d558bb9aa6c5c8f4f76f794f82890f99a',
'linux', 'all'),
Dependency(
'buildtools/debian_sid_arm64-sysroot.tgz',
'https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2befe8ce3e88be6080e4fb7e6d412278ea6a7625/debian_sid_arm64_sysroot.tar.xz',
'e4389eab2fe363f3fbdfa4d3ce9d94457d78fd2c0e62171a7534867623eadc90',
'linux', 'all'),
]
ALL_DEPS = (
BUILD_DEPS_HOST + BUILD_DEPS_ANDROID + BUILD_DEPS_LINUX_CROSS_SYSROOTS +
TEST_DEPS_ANDROID + UI_DEPS)
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
UI_DIR = os.path.join(ROOT_DIR, 'ui')
TOOLS_DIR = os.path.join(ROOT_DIR, 'tools')
NODE_MODULES_STATUS_FILE = os.path.join(UI_DIR, 'node_modules', '.last_install')
TEST_DATA_SCRIPT = os.path.join(TOOLS_DIR, 'test_data')
def CheckCallRetry(*args, **kwargs):
""" Like subprocess.check_call, with retries up to 5 times. """
MAX_ATTEMPTS = 5
for attempt in range(1, MAX_ATTEMPTS + 1):
try:
return subprocess.check_call(*args, **kwargs)
except subprocess.CalledProcessError as error:
if attempt == MAX_ATTEMPTS:
raise error
else:
logging.error(error)
time.sleep(attempt * 3)
def DownloadURL(url, out_file):
CheckCallRetry(['curl', '-L', '-#', '-o', out_file, url])
def GetArch():
arch = machine()
if arch == 'arm64':
return 'arm64'
else:
# Assume everything else is x64 matching previous behaviour.
return 'x64'
def ReadFile(path):
if not os.path.exists(path):
return None
with open(path) as f:
return f.read().strip()
def MkdirRecursive(path):
# Works with both relative and absolute paths
cwd = '/' if path.startswith('/') else ROOT_DIR
for part in path.split('/'):
cwd = os.path.join(cwd, part)
if not os.path.exists(cwd):
os.makedirs(cwd)
else:
assert (os.path.isdir(cwd))
def HashLocalFile(path):
if not os.path.exists(path):
return None
with open(path, 'rb') as f:
return hashlib.sha256(f.read()).hexdigest()
def ExtractZipfilePreservePermissions(zf, info, path):
zf.extract(info.filename, path=path)
target_path = os.path.join(path, info.filename)
min_acls = 0o755 if info.filename.endswith('/') else 0o644
os.chmod(target_path, (info.external_attr >> 16) | min_acls)
def IsGitRepoCheckoutOutAtRevision(path, revision):
return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision
def RmtreeIfExists(path):
# Git creates read-only files on windows, which cause failures with rmtree.
# This seems the socially accepted way to deal with it.
# See https://bugs.python.org/issue19643 .
def del_read_only_for_windows(_action, name, _exc):
os.chmod(name, stat.S_IWRITE)
os.remove(name)
if not os.path.exists(path):
return
third_party_path = os.path.abspath(os.path.join(ROOT_DIR, 'third_party'))
buildtools_path = os.path.abspath(os.path.join(ROOT_DIR, 'buildtools'))
test_path = os.path.abspath(os.path.join(ROOT_DIR, 'test', 'data'))
if (not os.path.abspath(path).startswith(buildtools_path) and
not os.path.abspath(path).startswith(test_path) and
not os.path.abspath(path).startswith(third_party_path)):
# Safety check to prevent that some merge confilct ends up doing some
# rm -rf / or similar.
logging.fatal(
'Cannot remove %s: outside of {buildtools, test/data, third_party}',
path)
sys.exit(1)
logging.info('Removing %s' % path)
if os.path.isdir(path):
shutil.rmtree(path, onerror=del_read_only_for_windows)
else:
os.remove(path)
def CheckoutGitRepo(path, git_url, revision, check_only):
if IsGitRepoCheckoutOutAtRevision(path, revision):
return False
if check_only:
return True
path = path.replace('/', os.sep)
RmtreeIfExists(path)
MkdirRecursive(path)
logging.info('Fetching %s @ %s into %s', git_url, revision, path)
subprocess.check_call(['git', 'init', path], cwd=path)
CheckCallRetry(['git', 'fetch', '--quiet', '--depth', '1', git_url, revision],
cwd=path)
subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path)
CheckCallRetry(
['git', 'submodule', 'update', '--init', '--recursive', '--quiet'],
cwd=path)
assert (IsGitRepoCheckoutOutAtRevision(path, revision))
return True
def InstallNodeModules(force_clean=False):
if force_clean:
node_modules = os.path.join(UI_DIR, 'node_modules')
logging.info('Clearing %s', node_modules)
subprocess.check_call(['git', 'clean', '-qxffd', node_modules],
cwd=ROOT_DIR)
logging.info("Running `npm ci` in {0}".format(UI_DIR))
# `npm ci` is like `npm install` but respects package-lock.json.
subprocess.check_call([os.path.join(TOOLS_DIR, 'npm'), 'ci'], cwd=UI_DIR)
# pbjs has the bad habit of installing extra packages on its first run. Run
# it here, so we avoid fetches while building.
node_bin = os.path.join(TOOLS_DIR, 'node')
pbjs = [node_bin, 'node_modules/.bin/pbjs', '/dev/null', '-o', '/dev/null']
subprocess.call(pbjs, cwd=UI_DIR)
with open(NODE_MODULES_STATUS_FILE, 'w') as f:
f.write(HashLocalFile(os.path.join(UI_DIR, 'package-lock.json')))
def CheckNodeModules():
"""Returns True if the modules are up-to-date.
There doesn't seem to be an easy way to check node modules versions. Instead
just check if package-lock.json changed since the last `npm install` call.
"""
if not os.path.exists(NODE_MODULES_STATUS_FILE):
return False
with open(NODE_MODULES_STATUS_FILE, 'r') as f:
actual = f.read()
expected = HashLocalFile(os.path.join(UI_DIR, 'package-lock.json'))
return expected == actual
def CheckHashes():
for dep in ALL_DEPS:
if dep.source_url.endswith('.git'):
continue
logging.info('Downloading %s for %s-%s', dep.source_url, dep.target_os,
dep.target_arch)
with tempfile.NamedTemporaryFile(delete=False) as f:
f.close()
DownloadURL(dep.source_url, f.name)
actual_checksum = HashLocalFile(f.name)
os.unlink(f.name)
if (actual_checksum != dep.checksum):
logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
dep.source_url, dep.checksum, actual_checksum))
def CheckDepotToolsIsRecent():
gn_py_path = shutil.which('gn.py')
if gn_py_path is None:
return True # depot_tools doesn't seem to be installed in the PATH.
dt_dir = os.path.abspath(os.path.dirname(gn_py_path))
cmd = ['git', '-C', dt_dir, 'merge-base', '--is-ancestor', 'a0cf4321', 'HEAD']
git_ret = subprocess.call(cmd, stderr=subprocess.DEVNULL)
if git_ret == 0:
return True
print('\033[91mYour depot_tools revision is too old. Please run:\033[0m')
print('git -C %s fetch origin && git -C %s checkout -B main -t origin/main' %
(dt_dir, dt_dir))
return False
def Main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--android',
action='store_true',
help='NDK and emulator images target_os="android"')
parser.add_argument(
'--linux-arm',
action='store_true',
help='Debian sysroots for target_os="linux" target_cpu="arm|arm64"')
parser.add_argument(
'--ui',
action='store_true',
help='Node and NPM packages to Build the Web-based UI via ./ui/build')
parser.add_argument(
'--grpc', action='store_true', help='Packages to build gRPC')
parser.add_argument('--check-only')
parser.add_argument('--filter', action='append')
parser.add_argument('--verify', help='Check all URLs', action='store_true')
parser.add_argument(
'--no-toolchain', help='Do not download toolchain', action='store_true')
args = parser.parse_args()
if args.verify:
CheckHashes()
return 0
target_os = system().lower()
if args.ui and target_os == 'windows':
print('Building the UI on Windows is unsupported')
return 1
if not CheckDepotToolsIsRecent():
return 1
deps = BUILD_DEPS_HOST
if not args.no_toolchain:
deps += BUILD_DEPS_TOOLCHAIN_HOST
if args.android:
deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID
if args.linux_arm:
deps += BUILD_DEPS_LINUX_CROSS_SYSROOTS
if args.ui:
deps += UI_DEPS
if args.grpc:
deps += GRPC_DEPS
deps_updated = False
nodejs_updated = False
for old_dir in CLEANUP_OLD_DIRS:
RmtreeIfExists(os.path.join(ROOT_DIR, old_dir))
for dep in deps:
target_arch = GetArch()
matches_os = dep.target_os == 'all' or target_os == dep.target_os
matches_arch = dep.target_arch == 'all' or target_arch == dep.target_arch
if not matches_os or not matches_arch:
continue
if args.filter and not any(f in dep.target_folder for f in args.filter):
continue
local_path = os.path.join(ROOT_DIR, dep.target_folder)
if dep.source_url.endswith('.git'):
deps_updated |= CheckoutGitRepo(local_path, dep.source_url, dep.checksum,
args.check_only)
continue
is_zip = local_path.endswith('.zip') or local_path.endswith('.tgz')
zip_target_dir = local_path[:-4] if is_zip else None
zip_dir_stamp = os.path.join(zip_target_dir, '.stamp') if is_zip else None
if ((not is_zip and HashLocalFile(local_path) == dep.checksum) or
(is_zip and ReadFile(zip_dir_stamp) == dep.checksum)):
continue
deps_updated = True
if args.check_only:
continue
MkdirRecursive(os.path.dirname(dep.target_folder))
if HashLocalFile(local_path) != dep.checksum:
download_path = local_path + '.tmp'
logging.info('Downloading %s from %s', local_path, dep.source_url)
DownloadURL(dep.source_url, download_path)
os.chmod(download_path, 0o755)
actual_checksum = HashLocalFile(download_path)
if (actual_checksum != dep.checksum):
os.remove(download_path)
logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
download_path, dep.checksum, actual_checksum))
return 1
shutil.move(download_path, local_path)
if 'nodejs' in dep.target_folder:
nodejs_updated = True
assert (HashLocalFile(local_path) == dep.checksum)
if is_zip:
logging.info('Extracting %s into %s' % (local_path, zip_target_dir))
assert (os.path.commonprefix((ROOT_DIR, zip_target_dir)) == ROOT_DIR)
RmtreeIfExists(zip_target_dir)
# Decompress the archive.
if local_path.endswith('.tgz'):
MkdirRecursive(zip_target_dir)
subprocess.check_call(['tar', '-oxf', local_path], cwd=zip_target_dir)
elif local_path.endswith('.zip'):
with zipfile.ZipFile(local_path, 'r') as zf:
for info in zf.infolist():
ExtractZipfilePreservePermissions(zf, info, zip_target_dir)
# If the zip contains one root folder, rebase one level up moving all
# its sub files and folders inside |target_dir|.
subdir = os.listdir(zip_target_dir)
if len(subdir) == 1:
subdir = os.path.join(zip_target_dir, subdir[0])
if os.path.isdir(subdir):
for subf in os.listdir(subdir):
shutil.move(os.path.join(subdir, subf), zip_target_dir)
os.rmdir(subdir)
# Create stamp and remove the archive.
with open(zip_dir_stamp, 'w') as stamp_file:
stamp_file.write(dep.checksum)
os.remove(local_path)
if args.ui:
# Needs to happen after nodejs is installed above.
if args.check_only:
deps_updated |= not CheckNodeModules()
else:
InstallNodeModules(force_clean=nodejs_updated)
cur_python_interpreter = sys.executable
test_data_synced = 0 == subprocess.call([
cur_python_interpreter, TEST_DATA_SCRIPT, 'status', '--quiet',
'--ignore-new'
])
if args.check_only:
if not deps_updated and test_data_synced:
with open(args.check_only, 'w') as f:
f.write('OK') # The content is irrelevant, just keep GN happy.
return 0
argz = ' '.join(
[x for x in sys.argv[1:] if not x.startswith('--check-only')])
print('\033[91mBuild deps are stale. ' +
'Please run tools/install-build-deps %s\033[0m' % argz)
if not test_data_synced:
print('//test/data/ is out of sync. `tools/test_data status` for details')
return 1
if not test_data_synced:
cmd = [cur_python_interpreter, TEST_DATA_SCRIPT, 'download', '--overwrite']
if not sys.stdout.isatty():
cmd += ['--verbose'] # For CI bots
subprocess.check_call(cmd)
if deps_updated:
# Stale binary files may be compiled against old sysroot headers that aren't
# tracked by gn.
logging.warning('Remember to run "gn clean <output_directory>" ' +
'to avoid stale binary files.')
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
sys.exit(Main())
Reference in New Issue View Git Blame Copy Permalink