482 lines
16 KiB
Python
482 lines
16 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright 2014 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""Classes of failure types."""
|
|
|
|
from __future__ import print_function
|
|
|
|
import collections
|
|
import json
|
|
import sys
|
|
import traceback
|
|
|
|
from autotest_lib.utils.frozen_chromite.lib import constants
|
|
from autotest_lib.utils.frozen_chromite.lib import cros_build_lib
|
|
from autotest_lib.utils.frozen_chromite.lib import failure_message_lib
|
|
from autotest_lib.utils.frozen_chromite.lib import metrics
|
|
|
|
|
|
class StepFailure(Exception):
|
|
"""StepFailure exceptions indicate that a cbuildbot step failed.
|
|
|
|
Exceptions that derive from StepFailure should meet the following
|
|
criteria:
|
|
1) The failure indicates that a cbuildbot step failed.
|
|
2) The necessary information to debug the problem has already been
|
|
printed in the logs for the stage that failed.
|
|
3) __str__() should be brief enough to include in a Commit Queue
|
|
failure message.
|
|
"""
|
|
|
|
# The constants.EXCEPTION_CATEGORY_ALL_CATEGORIES values that this exception
|
|
# maps to. Subclasses should redefine this class constant to map to a
|
|
# different category.
|
|
EXCEPTION_CATEGORY = constants.EXCEPTION_CATEGORY_UNKNOWN
|
|
|
|
def EncodeExtraInfo(self):
|
|
"""Encode extra_info into a json string, can be overwritten by subclasses"""
|
|
|
|
def ConvertToStageFailureMessage(self, build_stage_id, stage_name,
|
|
stage_prefix_name=None):
|
|
"""Convert StepFailure to StageFailureMessage.
|
|
|
|
Args:
|
|
build_stage_id: The id of the build stage.
|
|
stage_name: The name (string) of the failed stage.
|
|
stage_prefix_name: The prefix name (string) of the failed stage,
|
|
default to None.
|
|
|
|
Returns:
|
|
An instance of failure_message_lib.StageFailureMessage.
|
|
"""
|
|
stage_failure = failure_message_lib.StageFailure(
|
|
None, build_stage_id, None, self.__class__.__name__, str(self),
|
|
self.EXCEPTION_CATEGORY, self.EncodeExtraInfo(), None, stage_name,
|
|
None, None, None, None, None, None, None, None, None, None)
|
|
return failure_message_lib.StageFailureMessage(
|
|
stage_failure, stage_prefix_name=stage_prefix_name)
|
|
|
|
|
|
# A namedtuple to hold information of an exception.
|
|
ExceptInfo = collections.namedtuple(
|
|
'ExceptInfo', ['type', 'str', 'traceback'])
|
|
|
|
|
|
def CreateExceptInfo(exception, tb):
|
|
"""Creates a list of ExceptInfo objects from |exception| and |tb|.
|
|
|
|
Creates an ExceptInfo object from |exception| and |tb|. If
|
|
|exception| is a CompoundFailure with non-empty list of exc_infos,
|
|
simly returns exception.exc_infos. Note that we do not preserve type
|
|
of |exception| in this case.
|
|
|
|
Args:
|
|
exception: The exception.
|
|
tb: The textual traceback.
|
|
|
|
Returns:
|
|
A list of ExceptInfo objects.
|
|
"""
|
|
if isinstance(exception, CompoundFailure) and exception.exc_infos:
|
|
return exception.exc_infos
|
|
|
|
return [ExceptInfo(exception.__class__, str(exception), tb)]
|
|
|
|
|
|
class CompoundFailure(StepFailure):
|
|
"""An exception that contains a list of ExceptInfo objects."""
|
|
|
|
def __init__(self, message='', exc_infos=None):
|
|
"""Initializes an CompoundFailure instance.
|
|
|
|
Args:
|
|
message: A string describing the failure.
|
|
exc_infos: A list of ExceptInfo objects.
|
|
"""
|
|
self.exc_infos = exc_infos if exc_infos else []
|
|
if not message:
|
|
# By default, print all stored ExceptInfo objects. This is the
|
|
# preferred behavior because we'd always have the full
|
|
# tracebacks to debug the failure.
|
|
message = '\n'.join('{e.type}: {e.str}\n{e.traceback}'.format(e=ex)
|
|
for ex in self.exc_infos)
|
|
self.msg = message
|
|
|
|
super(CompoundFailure, self).__init__(message)
|
|
|
|
def ToSummaryString(self):
|
|
"""Returns a string with type and string of each ExceptInfo object.
|
|
|
|
This does not include the textual tracebacks on purpose, so the
|
|
message is more readable on the waterfall.
|
|
"""
|
|
if self.HasEmptyList():
|
|
# Fall back to return self.message if list is empty.
|
|
return self.msg
|
|
else:
|
|
return '\n'.join(['%s: %s' % (e.type, e.str) for e in self.exc_infos])
|
|
|
|
def HasEmptyList(self):
|
|
"""Returns True if self.exc_infos is empty."""
|
|
return not bool(self.exc_infos)
|
|
|
|
def HasFailureType(self, cls):
|
|
"""Returns True if any of the failures matches |cls|."""
|
|
return any(issubclass(x.type, cls) for x in self.exc_infos)
|
|
|
|
def MatchesFailureType(self, cls):
|
|
"""Returns True if all failures matches |cls|."""
|
|
return (not self.HasEmptyList() and
|
|
all(issubclass(x.type, cls) for x in self.exc_infos))
|
|
|
|
def HasFatalFailure(self, whitelist=None):
|
|
"""Determine if there are non-whitlisted failures.
|
|
|
|
Args:
|
|
whitelist: A list of whitelisted exception types.
|
|
|
|
Returns:
|
|
Returns True if any failure is not in |whitelist|.
|
|
"""
|
|
if not whitelist:
|
|
return not self.HasEmptyList()
|
|
|
|
for ex in self.exc_infos:
|
|
if all(not issubclass(ex.type, cls) for cls in whitelist):
|
|
return True
|
|
|
|
return False
|
|
|
|
def ConvertToStageFailureMessage(self, build_stage_id, stage_name,
|
|
stage_prefix_name=None):
|
|
"""Convert CompoundFailure to StageFailureMessage.
|
|
|
|
Args:
|
|
build_stage_id: The id of the build stage.
|
|
stage_name: The name (string) of the failed stage.
|
|
stage_prefix_name: The prefix name (string) of the failed stage,
|
|
default to None.
|
|
|
|
Returns:
|
|
An instance of failure_message_lib.StageFailureMessage.
|
|
"""
|
|
stage_failure = failure_message_lib.StageFailure(
|
|
None, build_stage_id, None, self.__class__.__name__, str(self),
|
|
self.EXCEPTION_CATEGORY, self.EncodeExtraInfo(), None, stage_name,
|
|
None, None, None, None, None, None, None, None, None, None)
|
|
compound_failure_message = failure_message_lib.CompoundFailureMessage(
|
|
stage_failure, stage_prefix_name=stage_prefix_name)
|
|
|
|
for exc_class, exc_str, _ in self.exc_infos:
|
|
inner_failure = failure_message_lib.StageFailure(
|
|
None, build_stage_id, None, exc_class.__name__, exc_str,
|
|
_GetExceptionCategory(exc_class), None, None, stage_name,
|
|
None, None, None, None, None, None, None, None, None, None)
|
|
innner_failure_message = failure_message_lib.StageFailureMessage(
|
|
inner_failure, stage_prefix_name=stage_prefix_name)
|
|
compound_failure_message.inner_failures.append(innner_failure_message)
|
|
|
|
return compound_failure_message
|
|
|
|
|
|
class ExitEarlyException(Exception):
|
|
"""Exception when a stage finishes and exits early."""
|
|
|
|
# ExitEarlyException is to simulate sys.exit(0), and SystemExit derives
|
|
# from BaseException, so should not catch ExitEarlyException as Exception
|
|
# and reset type to re-raise.
|
|
EXCEPTIONS_TO_EXCLUDE = (ExitEarlyException,)
|
|
|
|
class SetFailureType(object):
|
|
"""A wrapper to re-raise the exception as the pre-set type."""
|
|
|
|
def __init__(self, category_exception, source_exception=None,
|
|
exclude_exceptions=EXCEPTIONS_TO_EXCLUDE):
|
|
"""Initializes the decorator.
|
|
|
|
Args:
|
|
category_exception: The exception type to re-raise as. It must be
|
|
a subclass of CompoundFailure.
|
|
source_exception: The exception types to re-raise. By default, re-raise
|
|
all Exception classes.
|
|
exclude_exceptions: Do not set the type of the exception if it's subclass
|
|
of one exception in exclude_exceptions. Default to EXCLUSIVE_EXCEPTIONS.
|
|
"""
|
|
assert issubclass(category_exception, CompoundFailure)
|
|
self.category_exception = category_exception
|
|
self.source_exception = source_exception
|
|
if self.source_exception is None:
|
|
self.source_exception = Exception
|
|
self.exclude_exceptions = exclude_exceptions
|
|
|
|
def __call__(self, functor):
|
|
"""Returns a wrapped function."""
|
|
def wrapped_functor(*args, **kwargs):
|
|
try:
|
|
return functor(*args, **kwargs)
|
|
except self.source_exception:
|
|
# Get the information about the original exception.
|
|
exc_type, exc_value, _ = sys.exc_info()
|
|
exc_traceback = traceback.format_exc()
|
|
if self.exclude_exceptions is not None:
|
|
for exclude_exception in self.exclude_exceptions:
|
|
if issubclass(exc_type, exclude_exception):
|
|
raise
|
|
if issubclass(exc_type, self.category_exception):
|
|
# Do not re-raise if the exception is a subclass of the set
|
|
# exception type because it offers more information.
|
|
raise
|
|
else:
|
|
exc_infos = CreateExceptInfo(exc_value, exc_traceback)
|
|
raise self.category_exception(exc_infos=exc_infos)
|
|
|
|
return wrapped_functor
|
|
|
|
|
|
class RetriableStepFailure(StepFailure):
|
|
"""This exception is thrown when a step failed, but should be retried."""
|
|
|
|
|
|
# TODO(nxia): Everytime the class name is changed, add the new class name to
|
|
# BUILD_SCRIPT_FAILURE_TYPES.
|
|
class BuildScriptFailure(StepFailure):
|
|
"""This exception is thrown when a build command failed.
|
|
|
|
It is intended to provide a shorter summary of what command failed,
|
|
for usage in failure messages from the Commit Queue, so as to ensure
|
|
that developers aren't spammed with giant error messages when common
|
|
commands (e.g. build_packages) fail.
|
|
"""
|
|
|
|
EXCEPTION_CATEGORY = constants.EXCEPTION_CATEGORY_BUILD
|
|
|
|
def __init__(self, exception, shortname):
|
|
"""Construct a BuildScriptFailure object.
|
|
|
|
Args:
|
|
exception: A RunCommandError object.
|
|
shortname: Short name for the command we're running.
|
|
"""
|
|
StepFailure.__init__(self)
|
|
assert isinstance(exception, cros_build_lib.RunCommandError)
|
|
self.exception = exception
|
|
self.shortname = shortname
|
|
self.args = (exception, shortname)
|
|
|
|
def __str__(self):
|
|
"""Summarize a build command failure briefly."""
|
|
result = self.exception.result
|
|
if result.returncode:
|
|
return '%s failed (code=%s)' % (self.shortname, result.returncode)
|
|
else:
|
|
return self.exception.msg
|
|
|
|
def EncodeExtraInfo(self):
|
|
"""Encode extra_info into a json string.
|
|
|
|
Returns:
|
|
A json string containing shortname.
|
|
"""
|
|
extra_info_dict = {
|
|
'shortname': self.shortname,
|
|
}
|
|
return json.dumps(extra_info_dict)
|
|
|
|
|
|
# TODO(nxia): Everytime the class name is changed, add the new class name to
|
|
# PACKAGE_BUILD_FAILURE_TYPES
|
|
class PackageBuildFailure(BuildScriptFailure):
|
|
"""This exception is thrown when packages fail to build."""
|
|
|
|
def __init__(self, exception, shortname, failed_packages):
|
|
"""Construct a PackageBuildFailure object.
|
|
|
|
Args:
|
|
exception: The underlying exception.
|
|
shortname: Short name for the command we're running.
|
|
failed_packages: List of packages that failed to build.
|
|
"""
|
|
BuildScriptFailure.__init__(self, exception, shortname)
|
|
self.failed_packages = set(failed_packages)
|
|
self.args = (exception, shortname, failed_packages)
|
|
|
|
def __str__(self):
|
|
return ('Packages failed in %s: %s'
|
|
% (self.shortname, ' '.join(sorted(self.failed_packages))))
|
|
|
|
def EncodeExtraInfo(self):
|
|
"""Encode extra_info into a json string.
|
|
|
|
Returns:
|
|
A json string containing shortname and failed_packages.
|
|
"""
|
|
extra_info_dict = {
|
|
'shortname': self.shortname,
|
|
'failed_packages': list(self.failed_packages)
|
|
}
|
|
return json.dumps(extra_info_dict)
|
|
|
|
def BuildCompileFailureOutputJson(self):
|
|
"""Build proto BuildCompileFailureOutput compatible JSON output.
|
|
|
|
Returns:
|
|
A json string with BuildCompileFailureOutput proto as json.
|
|
"""
|
|
failures = []
|
|
for pkg in self.failed_packages:
|
|
failures.append({'rule': 'emerge', 'output_targets': pkg})
|
|
wrapper = {'failures': failures}
|
|
return json.dumps(wrapper, indent=2)
|
|
|
|
class InfrastructureFailure(CompoundFailure):
|
|
"""Raised if a stage fails due to infrastructure issues."""
|
|
|
|
EXCEPTION_CATEGORY = constants.EXCEPTION_CATEGORY_INFRA
|
|
|
|
|
|
# ChromeOS Test Lab failures.
|
|
class TestLabFailure(InfrastructureFailure):
|
|
"""Raised if a stage fails due to hardware lab infrastructure issues."""
|
|
|
|
EXCEPTION_CATEGORY = constants.EXCEPTION_CATEGORY_LAB
|
|
|
|
|
|
class SuiteTimedOut(TestLabFailure):
|
|
"""Raised if a test suite timed out with no test failures."""
|
|
|
|
|
|
class BoardNotAvailable(TestLabFailure):
|
|
"""Raised if the board is not available in the lab."""
|
|
|
|
|
|
class SwarmingProxyFailure(TestLabFailure):
|
|
"""Raised when error related to swarming proxy occurs."""
|
|
|
|
|
|
# Gerrit-on-Borg failures.
|
|
class GoBFailure(InfrastructureFailure):
|
|
"""Raised if a stage fails due to Gerrit-on-Borg (GoB) issues."""
|
|
|
|
|
|
class GoBQueryFailure(GoBFailure):
|
|
"""Raised if a stage fails due to Gerrit-on-Borg (GoB) query errors."""
|
|
|
|
|
|
class GoBSubmitFailure(GoBFailure):
|
|
"""Raised if a stage fails due to Gerrit-on-Borg (GoB) submission errors."""
|
|
|
|
|
|
class GoBFetchFailure(GoBFailure):
|
|
"""Raised if a stage fails due to Gerrit-on-Borg (GoB) fetch errors."""
|
|
|
|
|
|
# Google Storage failures.
|
|
class GSFailure(InfrastructureFailure):
|
|
"""Raised if a stage fails due to Google Storage (GS) issues."""
|
|
|
|
|
|
class GSUploadFailure(GSFailure):
|
|
"""Raised if a stage fails due to Google Storage (GS) upload issues."""
|
|
|
|
|
|
class GSDownloadFailure(GSFailure):
|
|
"""Raised if a stage fails due to Google Storage (GS) download issues."""
|
|
|
|
|
|
# Builder failures.
|
|
class BuilderFailure(InfrastructureFailure):
|
|
"""Raised if a stage fails due to builder issues."""
|
|
|
|
|
|
class MasterSlaveVersionMismatchFailure(BuilderFailure):
|
|
"""Raised if a slave build has a different full_version than its master."""
|
|
|
|
# Crash collection service failures.
|
|
class CrashCollectionFailure(InfrastructureFailure):
|
|
"""Raised if a stage fails due to crash collection services."""
|
|
|
|
|
|
class TestFailure(StepFailure):
|
|
"""Raised if a test stage (e.g. VMTest) fails."""
|
|
|
|
EXCEPTION_CATEGORY = constants.EXCEPTION_CATEGORY_TEST
|
|
|
|
|
|
class TestWarning(StepFailure):
|
|
"""Raised if a test stage (e.g. VMTest) returns a warning code."""
|
|
|
|
|
|
def ReportStageFailure(exception, metrics_fields=None):
|
|
"""Reports stage failure to Mornach along with inner exceptions.
|
|
|
|
Args:
|
|
exception: The failure exception to report.
|
|
metrics_fields: (Optional) Fields for ts_mon metric.
|
|
"""
|
|
_InsertFailureToMonarch(
|
|
exception_category=_GetExceptionCategory(type(exception)),
|
|
metrics_fields=metrics_fields)
|
|
|
|
# This assumes that CompoundFailure can't be nested.
|
|
if isinstance(exception, CompoundFailure):
|
|
for exc_class, _, _ in exception.exc_infos:
|
|
_InsertFailureToMonarch(
|
|
exception_category=_GetExceptionCategory(exc_class),
|
|
metrics_fields=metrics_fields)
|
|
|
|
|
|
def _InsertFailureToMonarch(
|
|
exception_category=constants.EXCEPTION_CATEGORY_UNKNOWN,
|
|
metrics_fields=None):
|
|
"""Report a single stage failure to Mornach if needed.
|
|
|
|
Args:
|
|
exception_category: (Optional) one of
|
|
constants.EXCEPTION_CATEGORY_ALL_CATEGORIES,
|
|
Default: 'unknown'.
|
|
metrics_fields: (Optional) Fields for ts_mon metric.
|
|
"""
|
|
if (metrics_fields is not None and
|
|
exception_category != constants.EXCEPTION_CATEGORY_UNKNOWN):
|
|
counter = metrics.Counter(constants.MON_STAGE_FAILURE_COUNT)
|
|
metrics_fields['exception_category'] = exception_category
|
|
counter.increment(fields=metrics_fields)
|
|
|
|
|
|
def GetStageFailureMessageFromException(stage_name, build_stage_id,
|
|
exception, stage_prefix_name=None):
|
|
"""Get StageFailureMessage from an exception.
|
|
|
|
Args:
|
|
stage_name: The name (string) of the failed stage.
|
|
build_stage_id: The id of the failed build stage.
|
|
exception: The BaseException instance to convert to StageFailureMessage.
|
|
stage_prefix_name: The prefix name (string) of the failed stage,
|
|
default to None.
|
|
|
|
Returns:
|
|
An instance of failure_message_lib.StageFailureMessage.
|
|
"""
|
|
if isinstance(exception, StepFailure):
|
|
return exception.ConvertToStageFailureMessage(
|
|
build_stage_id, stage_name, stage_prefix_name=stage_prefix_name)
|
|
else:
|
|
stage_failure = failure_message_lib.StageFailure(
|
|
None, build_stage_id, None, type(exception).__name__, str(exception),
|
|
_GetExceptionCategory(type(exception)), None, None, stage_name,
|
|
None, None, None, None, None, None, None, None, None, None)
|
|
|
|
return failure_message_lib.StageFailureMessage(
|
|
stage_failure, stage_prefix_name=stage_prefix_name)
|
|
|
|
|
|
def _GetExceptionCategory(exception_class):
|
|
# Do not use try/catch. If a subclass of StepFailure does not have a valid
|
|
# EXCEPTION_CATEGORY, it is a programming error, not a runtime error.
|
|
if issubclass(exception_class, StepFailure):
|
|
return exception_class.EXCEPTION_CATEGORY
|
|
else:
|
|
return constants.EXCEPTION_CATEGORY_UNKNOWN
|