577 lines
22 KiB
Python
577 lines
22 KiB
Python
# Copyright 2020 The Chromium Authors
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
"""Class for interacting with the Skia Gold image diffing service."""
|
|
|
|
import logging
|
|
import os
|
|
import platform
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
from skia_gold_common import skia_gold_properties
|
|
|
|
CHROMIUM_SRC = os.path.realpath(
|
|
os.path.join(os.path.dirname(__file__), '..', '..'))
|
|
|
|
GOLDCTL_BINARY = os.path.join(CHROMIUM_SRC, 'tools', 'skia_goldctl')
|
|
if sys.platform == 'win32':
|
|
GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'win', 'goldctl') + '.exe'
|
|
elif sys.platform == 'darwin':
|
|
machine = platform.machine().lower()
|
|
if any(machine.startswith(m) for m in ('arm64', 'aarch64')):
|
|
GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'mac_arm64', 'goldctl')
|
|
else:
|
|
GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'mac_amd64', 'goldctl')
|
|
else:
|
|
GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'linux', 'goldctl')
|
|
|
|
|
|
StepRetVal = Tuple[int, Optional[str]]
|
|
|
|
|
|
class SkiaGoldSession():
|
|
class StatusCodes():
|
|
"""Status codes for RunComparison."""
|
|
SUCCESS = 0
|
|
AUTH_FAILURE = 1
|
|
INIT_FAILURE = 2
|
|
COMPARISON_FAILURE_REMOTE = 3
|
|
COMPARISON_FAILURE_LOCAL = 4
|
|
LOCAL_DIFF_FAILURE = 5
|
|
NO_OUTPUT_MANAGER = 6
|
|
|
|
class ComparisonResults():
|
|
"""Struct-like object for storing results of an image comparison."""
|
|
|
|
def __init__(self):
|
|
self.public_triage_link: Optional[str] = None
|
|
self.internal_triage_link: Optional[str] = None
|
|
self.triage_link_omission_reason: Optional[str] = None
|
|
self.local_diff_given_image: Optional[str] = None
|
|
self.local_diff_closest_image: Optional[str] = None
|
|
self.local_diff_diff_image: Optional[str] = None
|
|
|
|
def __init__(self,
|
|
working_dir: str,
|
|
gold_properties: skia_gold_properties.SkiaGoldProperties,
|
|
keys_file: str,
|
|
corpus: str,
|
|
instance: str,
|
|
bucket: Optional[str] = None):
|
|
"""Abstract class to handle all aspects of image comparison via Skia Gold.
|
|
|
|
A single SkiaGoldSession is valid for a single instance/corpus/keys_file
|
|
combination.
|
|
|
|
Args:
|
|
working_dir: The directory to store config files, etc.
|
|
gold_properties: A skia_gold_properties.SkiaGoldProperties instance for
|
|
the current test run.
|
|
keys_file: A path to a JSON file containing various comparison config data
|
|
such as corpus and debug information like the hardware/software
|
|
configuration the images will be produced on.
|
|
corpus: The corpus that images that will be compared belong to.
|
|
instance: The name of the Skia Gold instance to interact with.
|
|
bucket: Overrides the formulaic Google Storage bucket name generated by
|
|
goldctl
|
|
"""
|
|
self._working_dir = working_dir
|
|
self._gold_properties = gold_properties
|
|
self._corpus = corpus
|
|
self._instance = instance
|
|
self._bucket = bucket
|
|
self._local_png_directory = (self._gold_properties.local_png_directory
|
|
or tempfile.mkdtemp())
|
|
with tempfile.NamedTemporaryFile(suffix='.txt',
|
|
dir=working_dir,
|
|
delete=False) as triage_link_file:
|
|
self._triage_link_file = triage_link_file.name
|
|
# A map of image name (string) to ComparisonResults for that image.
|
|
self._comparison_results = {}
|
|
self._authenticated = False
|
|
self._initialized = False
|
|
|
|
# Copy the given keys file to the working directory in case it ends up
|
|
# getting deleted before we try to use it.
|
|
self._keys_file = os.path.join(working_dir, 'gold_keys.json')
|
|
shutil.copy(keys_file, self._keys_file)
|
|
|
|
def RunComparison(self,
|
|
name: str,
|
|
png_file: str,
|
|
output_manager: Any,
|
|
inexact_matching_args: Optional[List[str]] = None,
|
|
use_luci: bool = True,
|
|
service_account: Optional[str] = None,
|
|
optional_keys: Optional[Dict[str, str]] = None,
|
|
force_dryrun: bool = False) -> StepRetVal:
|
|
"""Helper method to run all steps to compare a produced image.
|
|
|
|
Handles authentication, itnitialization, comparison, and, if necessary,
|
|
local diffing.
|
|
|
|
Args:
|
|
name: The name of the image being compared.
|
|
png_file: A path to a PNG file containing the image to be compared.
|
|
output_manager: An output manager to use to store diff links. The
|
|
argument's type depends on what type a subclasses' _StoreDiffLinks
|
|
implementation expects. Can be None even if _StoreDiffLinks expects
|
|
a valid input, but will fail if it ever actually needs to be used.
|
|
inexact_matching_args: A list of strings containing extra command line
|
|
arguments to pass to Gold for inexact matching. Can be omitted to use
|
|
exact matching.
|
|
use_luci: If true, authentication will use the service account provided by
|
|
the LUCI context. If false, will attempt to use whatever is set up in
|
|
gsutil, which is only supported for local runs.
|
|
service_account: If set, uses the provided service account instead of
|
|
LUCI_CONTEXT or whatever is set in gsutil.
|
|
optional_keys: A dict containing optional key/value pairs to pass to Gold
|
|
for this comparison. Optional keys are keys unrelated to the
|
|
configuration the image was produced on, e.g. a comment or whether
|
|
Gold should treat the image as ignored.
|
|
force_dryrun: A boolean denoting whether dryrun should be forced on
|
|
regardless of whether this is a local comparison or not.
|
|
|
|
Returns:
|
|
A tuple (status, error). |status| is a value from
|
|
SkiaGoldSession.StatusCodes signifying the result of the comparison.
|
|
|error| is an error message describing the status if not successful.
|
|
"""
|
|
auth_rc, auth_stdout = self.Authenticate(use_luci=use_luci,
|
|
service_account=service_account)
|
|
if auth_rc:
|
|
return self.StatusCodes.AUTH_FAILURE, auth_stdout
|
|
|
|
init_rc, init_stdout = self.Initialize()
|
|
if init_rc:
|
|
return self.StatusCodes.INIT_FAILURE, init_stdout
|
|
|
|
compare_rc, compare_stdout = self.Compare(
|
|
name=name,
|
|
png_file=png_file,
|
|
inexact_matching_args=inexact_matching_args,
|
|
optional_keys=optional_keys,
|
|
force_dryrun=force_dryrun)
|
|
if not compare_rc:
|
|
return self.StatusCodes.SUCCESS, None
|
|
|
|
logging.error('Gold comparison failed: %s', compare_stdout)
|
|
if not self._gold_properties.local_pixel_tests:
|
|
return self.StatusCodes.COMPARISON_FAILURE_REMOTE, compare_stdout
|
|
|
|
if not output_manager:
|
|
return (self.StatusCodes.NO_OUTPUT_MANAGER,
|
|
'No output manager for local diff images')
|
|
|
|
diff_rc, diff_stdout = self.Diff(name=name,
|
|
png_file=png_file,
|
|
output_manager=output_manager)
|
|
if diff_rc:
|
|
return self.StatusCodes.LOCAL_DIFF_FAILURE, diff_stdout
|
|
return self.StatusCodes.COMPARISON_FAILURE_LOCAL, compare_stdout
|
|
|
|
def Authenticate(self,
|
|
use_luci: bool = True,
|
|
service_account: Optional[str] = None) -> StepRetVal:
|
|
"""Authenticates with Skia Gold for this session.
|
|
|
|
Args:
|
|
use_luci: If true, authentication will use the service account provided
|
|
by the LUCI context. If false, will attempt to use whatever is set up
|
|
in gsutil, which is only supported for local runs.
|
|
service_account: If set, uses the provided service account instead of
|
|
LUCI_CONTEXT or whatever is set in gsutil.
|
|
|
|
Returns:
|
|
A tuple (return_code, output). |return_code| is the return code of the
|
|
authentication process. |output| is the stdout + stderr of the
|
|
authentication process.
|
|
"""
|
|
if self._authenticated:
|
|
return 0, None
|
|
if self._gold_properties.bypass_skia_gold_functionality:
|
|
logging.warning('Not actually authenticating with Gold due to '
|
|
'--bypass-skia-gold-functionality being present.')
|
|
return 0, None
|
|
assert not (use_luci and service_account)
|
|
|
|
auth_cmd = [GOLDCTL_BINARY, 'auth', '--work-dir', self._working_dir]
|
|
if use_luci:
|
|
auth_cmd.append('--luci')
|
|
elif service_account:
|
|
auth_cmd.extend(['--service-account', service_account])
|
|
elif not self._gold_properties.local_pixel_tests:
|
|
raise RuntimeError(
|
|
'Cannot authenticate to Skia Gold with use_luci=False without a '
|
|
'service account unless running local pixel tests')
|
|
|
|
rc, stdout = self._RunCmdForRcAndOutput(auth_cmd)
|
|
if rc == 0:
|
|
self._authenticated = True
|
|
return rc, stdout
|
|
|
|
def Initialize(self) -> StepRetVal:
|
|
"""Initializes the working directory if necessary.
|
|
|
|
This can technically be skipped if the same information is passed to the
|
|
command used for image comparison, but that is less efficient under the
|
|
hood. Doing it that way effectively requires an initialization for every
|
|
comparison (~250 ms) instead of once at the beginning.
|
|
|
|
Returns:
|
|
A tuple (return_code, output). |return_code| is the return code of the
|
|
initialization process. |output| is the stdout + stderr of the
|
|
initialization process.
|
|
"""
|
|
if self._initialized:
|
|
return 0, None
|
|
if self._gold_properties.bypass_skia_gold_functionality:
|
|
logging.warning('Not actually initializing Gold due to '
|
|
'--bypass-skia-gold-functionality being present.')
|
|
return 0, None
|
|
|
|
init_cmd = [
|
|
GOLDCTL_BINARY,
|
|
'imgtest',
|
|
'init',
|
|
'--passfail',
|
|
'--instance',
|
|
self._instance,
|
|
'--corpus',
|
|
self._corpus,
|
|
'--keys-file',
|
|
self._keys_file,
|
|
'--work-dir',
|
|
self._working_dir,
|
|
'--failure-file',
|
|
self._triage_link_file,
|
|
'--commit',
|
|
self._gold_properties.git_revision,
|
|
]
|
|
if self._bucket:
|
|
init_cmd.extend(['--bucket', self._bucket])
|
|
if self._gold_properties.IsTryjobRun():
|
|
init_cmd.extend([
|
|
'--issue',
|
|
str(self._gold_properties.issue),
|
|
'--patchset',
|
|
str(self._gold_properties.patchset),
|
|
'--jobid',
|
|
str(self._gold_properties.job_id),
|
|
'--crs',
|
|
str(self._gold_properties.code_review_system),
|
|
'--cis',
|
|
str(self._gold_properties.continuous_integration_system),
|
|
])
|
|
|
|
rc, stdout = self._RunCmdForRcAndOutput(init_cmd)
|
|
if rc == 0:
|
|
self._initialized = True
|
|
return rc, stdout
|
|
|
|
def Compare(self,
|
|
name: str,
|
|
png_file: str,
|
|
inexact_matching_args: Optional[List[str]] = None,
|
|
optional_keys: Optional[Dict[str, str]] = None,
|
|
force_dryrun: bool = False) -> StepRetVal:
|
|
"""Compares the given image to images known to Gold.
|
|
|
|
Triage links can later be retrieved using GetTriageLinks().
|
|
|
|
Args:
|
|
name: The name of the image being compared.
|
|
png_file: A path to a PNG file containing the image to be compared.
|
|
inexact_matching_args: A list of strings containing extra command line
|
|
arguments to pass to Gold for inexact matching. Can be omitted to use
|
|
exact matching.
|
|
optional_keys: A dict containing optional key/value pairs to pass to Gold
|
|
for this comparison. Optional keys are keys unrelated to the
|
|
configuration the image was produced on, e.g. a comment or whether
|
|
Gold should treat the image as ignored.
|
|
force_dryrun: A boolean denoting whether dryrun should be forced on
|
|
regardless of whether this is a local comparison or not.
|
|
|
|
Returns:
|
|
A tuple (return_code, output). |return_code| is the return code of the
|
|
comparison process. |output| is the stdout + stderr of the comparison
|
|
process.
|
|
"""
|
|
if self._gold_properties.bypass_skia_gold_functionality:
|
|
logging.warning('Not actually comparing with Gold due to '
|
|
'--bypass-skia-gold-functionality being present.')
|
|
return 0, None
|
|
|
|
compare_cmd = [
|
|
GOLDCTL_BINARY,
|
|
'imgtest',
|
|
'add',
|
|
'--test-name',
|
|
name,
|
|
'--png-file',
|
|
png_file,
|
|
'--work-dir',
|
|
self._working_dir,
|
|
]
|
|
if self._gold_properties.local_pixel_tests or force_dryrun:
|
|
compare_cmd.append('--dryrun')
|
|
if inexact_matching_args:
|
|
logging.info('Using inexact matching arguments for image %s: %s', name,
|
|
inexact_matching_args)
|
|
compare_cmd.extend(inexact_matching_args)
|
|
|
|
optional_keys = optional_keys or {}
|
|
for k, v in optional_keys.items():
|
|
compare_cmd.extend([
|
|
'--add-test-optional-key',
|
|
'%s:%s' % (k, v),
|
|
])
|
|
|
|
self._ClearTriageLinkFile()
|
|
rc, stdout = self._RunCmdForRcAndOutput(compare_cmd)
|
|
|
|
self._comparison_results[name] = self.ComparisonResults()
|
|
if rc == 0:
|
|
self._comparison_results[name].triage_link_omission_reason = (
|
|
'Comparison succeeded, no triage link')
|
|
elif self._gold_properties.IsTryjobRun():
|
|
cl_triage_link = ('https://{instance}-gold.skia.org/cl/{crs}/{issue}')
|
|
cl_triage_link = cl_triage_link.format(
|
|
instance=self._instance,
|
|
crs=self._gold_properties.code_review_system,
|
|
issue=self._gold_properties.issue)
|
|
self._comparison_results[name].internal_triage_link = cl_triage_link
|
|
self._comparison_results[name].public_triage_link =\
|
|
self._GeneratePublicTriageLink(cl_triage_link)
|
|
else:
|
|
try:
|
|
with open(self._triage_link_file) as tlf:
|
|
triage_link = tlf.read().strip()
|
|
if not triage_link:
|
|
self._comparison_results[name].triage_link_omission_reason = (
|
|
'Gold did not provide a triage link. This is likely a bug on '
|
|
"Gold's end.")
|
|
self._comparison_results[name].internal_triage_link = None
|
|
self._comparison_results[name].public_triage_link = None
|
|
else:
|
|
self._comparison_results[name].internal_triage_link = triage_link
|
|
self._comparison_results[name].public_triage_link =\
|
|
self._GeneratePublicTriageLink(triage_link)
|
|
except IOError:
|
|
self._comparison_results[name].triage_link_omission_reason = (
|
|
'Failed to read triage link from file')
|
|
return rc, stdout
|
|
|
|
def Diff(self, name: str, png_file: str, output_manager: Any) -> StepRetVal:
|
|
"""Performs a local image diff against the closest known positive in Gold.
|
|
|
|
This is used for running tests on a workstation, where uploading data to
|
|
Gold for ingestion is not allowed, and thus the web UI is not available.
|
|
|
|
Image links can later be retrieved using Get*ImageLink().
|
|
|
|
Args:
|
|
name: The name of the image being compared.
|
|
png_file: The path to a PNG file containing the image to be diffed.
|
|
output_manager: An output manager to use to store diff links. The
|
|
argument's type depends on what type a subclasses' _StoreDiffLinks
|
|
implementation expects.
|
|
|
|
Returns:
|
|
A tuple (return_code, output). |return_code| is the return code of the
|
|
diff process. |output| is the stdout + stderr of the diff process.
|
|
"""
|
|
# Instead of returning that everything is okay and putting in dummy links,
|
|
# just fail since this should only be called when running locally and
|
|
# --bypass-skia-gold-functionality is only meant for use on the bots.
|
|
if self._gold_properties.bypass_skia_gold_functionality:
|
|
raise RuntimeError(
|
|
'--bypass-skia-gold-functionality is not supported when running '
|
|
'tests locally.')
|
|
|
|
output_dir = self._CreateDiffOutputDir(name)
|
|
# TODO(skbug.com/10611): Remove this temporary work dir and instead just use
|
|
# self._working_dir once `goldctl diff` stops clobbering the auth files in
|
|
# the provided work directory.
|
|
temp_work_dir = tempfile.mkdtemp()
|
|
# shutil.copytree() fails if the destination already exists, so use a
|
|
# subdirectory of the temporary directory.
|
|
temp_work_dir = os.path.join(temp_work_dir, 'diff_work_dir')
|
|
try:
|
|
shutil.copytree(self._working_dir, temp_work_dir)
|
|
diff_cmd = [
|
|
GOLDCTL_BINARY,
|
|
'diff',
|
|
'--corpus',
|
|
self._corpus,
|
|
'--instance',
|
|
self._GetDiffGoldInstance(),
|
|
'--input',
|
|
png_file,
|
|
'--test',
|
|
name,
|
|
'--work-dir',
|
|
temp_work_dir,
|
|
'--out-dir',
|
|
output_dir,
|
|
]
|
|
rc, stdout = self._RunCmdForRcAndOutput(diff_cmd)
|
|
self._StoreDiffLinks(name, output_manager, output_dir)
|
|
return rc, stdout
|
|
finally:
|
|
shutil.rmtree(os.path.realpath(os.path.join(temp_work_dir, '..')))
|
|
|
|
def GetTriageLinks(self, name: str) -> Tuple[str, str]:
|
|
"""Gets the triage links for the given image.
|
|
|
|
Args:
|
|
name: The name of the image to retrieve the triage link for.
|
|
|
|
Returns:
|
|
A tuple (public, internal). |public| is a string containing the triage
|
|
link for the public Gold instance if it is available, or None if it is not
|
|
available for some reason. |internal| is the same as |public|, but
|
|
containing a link to the internal Gold instance. The reason for links not
|
|
being available can be retrieved using GetTriageLinkOmissionReason.
|
|
"""
|
|
comparison_results = self._comparison_results.get(name,
|
|
self.ComparisonResults())
|
|
return (comparison_results.public_triage_link,
|
|
comparison_results.internal_triage_link)
|
|
|
|
def GetTriageLinkOmissionReason(self, name: str) -> str:
|
|
"""Gets the reason why a triage link is not available for an image.
|
|
|
|
Args:
|
|
name: The name of the image whose triage link does not exist.
|
|
|
|
Returns:
|
|
A string containing the reason why a triage link is not available.
|
|
"""
|
|
if name not in self._comparison_results:
|
|
return 'No image comparison performed for %s' % name
|
|
results = self._comparison_results[name]
|
|
# This method should not be called if there is a valid triage link.
|
|
assert results.public_triage_link is None
|
|
assert results.internal_triage_link is None
|
|
if results.triage_link_omission_reason:
|
|
return results.triage_link_omission_reason
|
|
if results.local_diff_given_image:
|
|
return 'Gold only used to do a local image diff'
|
|
raise RuntimeError(
|
|
'Somehow have a ComparisonResults instance for %s that should not '
|
|
'exist' % name)
|
|
|
|
def GetGivenImageLink(self, name: str) -> str:
|
|
"""Gets the link to the given image used for local diffing.
|
|
|
|
Args:
|
|
name: The name of the image that was diffed.
|
|
|
|
Returns:
|
|
A string containing the link to where the image is saved, or None if it
|
|
does not exist.
|
|
"""
|
|
assert name in self._comparison_results
|
|
return self._comparison_results[name].local_diff_given_image
|
|
|
|
def GetClosestImageLink(self, name: str) -> str:
|
|
"""Gets the link to the closest known image used for local diffing.
|
|
|
|
Args:
|
|
name: The name of the image that was diffed.
|
|
|
|
Returns:
|
|
A string containing the link to where the image is saved, or None if it
|
|
does not exist.
|
|
"""
|
|
assert name in self._comparison_results
|
|
return self._comparison_results[name].local_diff_closest_image
|
|
|
|
def GetDiffImageLink(self, name: str) -> str:
|
|
"""Gets the link to the diff between the given and closest images.
|
|
|
|
Args:
|
|
name: The name of the image that was diffed.
|
|
|
|
Returns:
|
|
A string containing the link to where the image is saved, or None if it
|
|
does not exist.
|
|
"""
|
|
assert name in self._comparison_results
|
|
return self._comparison_results[name].local_diff_diff_image
|
|
|
|
def _GeneratePublicTriageLink(self, internal_link: str) -> str:
|
|
"""Generates a public triage link given an internal one.
|
|
|
|
Args:
|
|
internal_link: A string containing a triage link pointing to an internal
|
|
Gold instance.
|
|
|
|
Returns:
|
|
A string containing a triage link pointing to the public mirror of the
|
|
link pointed to by |internal_link|.
|
|
"""
|
|
return internal_link.replace('%s-gold' % self._instance,
|
|
'%s-public-gold' % self._instance)
|
|
|
|
def _ClearTriageLinkFile(self) -> None:
|
|
"""Clears the contents of the triage link file.
|
|
|
|
This should be done before every comparison since goldctl appends to the
|
|
file instead of overwriting its contents, which results in multiple triage
|
|
links getting concatenated together if there are multiple failures.
|
|
"""
|
|
open(self._triage_link_file, 'w').close()
|
|
|
|
def _CreateDiffOutputDir(self, _name: str) -> str:
|
|
# We don't use self._local_png_directory here since we want it to be
|
|
# automatically cleaned up with the working directory. Any subclasses that
|
|
# want to keep it around can override this method.
|
|
return tempfile.mkdtemp(dir=self._working_dir)
|
|
|
|
def _GetDiffGoldInstance(self) -> str:
|
|
"""Gets the Skia Gold instance to use for the Diff step.
|
|
|
|
This can differ based on how a particular instance is set up, mainly
|
|
depending on whether it is set up for internal results or not.
|
|
"""
|
|
# TODO(skbug.com/10610): Decide whether to use the public or
|
|
# non-public instance once authentication is fixed for the non-public
|
|
# instance.
|
|
return str(self._instance) + '-public'
|
|
|
|
def _StoreDiffLinks(self, image_name: str, output_manager: Any,
|
|
output_dir: str) -> None:
|
|
"""Stores the local diff files as links.
|
|
|
|
The ComparisonResults entry for |image_name| should have its *_image fields
|
|
filled after this unless corresponding images were not found on disk.
|
|
|
|
Args:
|
|
image_name: A string containing the name of the image that was diffed.
|
|
output_manager: An output manager used used to surface links to users,
|
|
if necessary. The expected argument type depends on each subclasses'
|
|
implementation of this method.
|
|
output_dir: A string containing the path to the directory where diff
|
|
output image files where saved.
|
|
"""
|
|
raise NotImplementedError()
|
|
|
|
@staticmethod
|
|
def _RunCmdForRcAndOutput(cmd: List[str]) -> Tuple[int, str]:
|
|
"""Runs |cmd| and returns its returncode and output.
|
|
|
|
Args:
|
|
cmd: A list containing the command line to run.
|
|
|
|
Returns:
|
|
A tuple (rc, output), where, |rc| is the returncode of the command and
|
|
|output| is the stdout + stderr of the command.
|
|
"""
|
|
raise NotImplementedError()
|