244 lines
11 KiB
C++
244 lines
11 KiB
C++
/*
|
|
* Copyright (C) 2022 The Android Open Source Project
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
#include "Conv2DOperationConverter.h"
|
|
|
|
#include <vector>
|
|
|
|
#include "OperationConverterResolver.h"
|
|
#include "SubGraphContext.h"
|
|
|
|
namespace android {
|
|
namespace nn {
|
|
|
|
Result<std::vector<int32_t>> Conv2DOperationConverter::getConv2DInputs(
|
|
const Operation& operation, SubGraphContext* context) const {
|
|
NN_RET_CHECK(isOperandConstant(
|
|
context->getSubgraph()->operands[operation.inputs[kFilterTensorIdx]]));
|
|
|
|
NN_TRY(context->createTensorFlatbufferFromOperand(operation.inputs[kInputTensorIdx]));
|
|
// TFLite does not support asymmetric tensors for convolution filters
|
|
NN_TRY(context->createTensorFlatbufferFromOperand(operation.inputs[kFilterTensorIdx],
|
|
true /* makeSymmetric */));
|
|
NN_TRY(context->createTensorFlatbufferFromOperand(operation.inputs[kBiasTensorIdx]));
|
|
std::vector<int32_t> inputs{
|
|
context->getTensorIdxFromOperandIdx(operation.inputs[kInputTensorIdx]),
|
|
context->getTensorIdxFromOperandIdx(operation.inputs[kFilterTensorIdx]),
|
|
context->getTensorIdxFromOperandIdx(operation.inputs[kBiasTensorIdx])};
|
|
return inputs;
|
|
}
|
|
|
|
Result<std::vector<int32_t>> Conv2DOperationConverter::getConv2DOutputs(
|
|
const Operation& operation, SubGraphContext* context) const {
|
|
NN_TRY(context->createTensorFlatbufferFromOperand(operation.outputs[kOutputTensorIdx]));
|
|
std::vector<int32_t> outputs{
|
|
context->getTensorIdxFromOperandIdx(operation.outputs[kOutputTensorIdx])};
|
|
return outputs;
|
|
}
|
|
|
|
Result<int> Conv2DOperationConverter::decomposeExplicitPadding(const Operation& operation,
|
|
SubGraphContext* context) const {
|
|
const Model::Subgraph* subgraph = context->getSubgraph();
|
|
const Operand& inputOperand = subgraph->operands[operation.inputs[0]];
|
|
|
|
// add opcode for PAD if it does not exist yet
|
|
uint32_t opCodeIdx = context->addOpCode(OperationType::PAD);
|
|
|
|
// pad options
|
|
auto padOptionsFlatbuffer = tflite::CreatePadOptions(context->getBuilder());
|
|
|
|
// check to make sure padding Operands are constants
|
|
const Operand& frontWidthPaddingOperand = subgraph->operands[operation.inputs[3]];
|
|
const Operand& backWidthPaddingOperand = subgraph->operands[operation.inputs[4]];
|
|
const Operand& frontHeightPaddingOperand = subgraph->operands[operation.inputs[5]];
|
|
const Operand& backHeightPaddingOperand = subgraph->operands[operation.inputs[6]];
|
|
NN_RET_CHECK(isOperandConstant(frontWidthPaddingOperand));
|
|
NN_RET_CHECK(isOperandConstant(backWidthPaddingOperand));
|
|
NN_RET_CHECK(isOperandConstant(frontHeightPaddingOperand));
|
|
NN_RET_CHECK(isOperandConstant(backHeightPaddingOperand));
|
|
|
|
// get padding params
|
|
int32_t frontHeightPadding = context->getConstantScalar<int32_t>(frontHeightPaddingOperand);
|
|
int32_t backHeightPadding = context->getConstantScalar<int32_t>(backHeightPaddingOperand);
|
|
int32_t frontWidthPadding = context->getConstantScalar<int32_t>(frontWidthPaddingOperand);
|
|
int32_t backWidthPadding = context->getConstantScalar<int32_t>(backWidthPaddingOperand);
|
|
|
|
// build padding buffer
|
|
const Dimensions& dims = inputOperand.dimensions;
|
|
int numDimensionsInput = static_cast<int>(dims.size());
|
|
std::vector<int32_t> paddingData(numDimensionsInput * 2, 0);
|
|
paddingData[2] = frontHeightPadding;
|
|
paddingData[3] = backHeightPadding;
|
|
paddingData[4] = frontWidthPadding;
|
|
paddingData[5] = backWidthPadding;
|
|
uint32_t paddingBufferIdx = context->addBufferFromData(
|
|
reinterpret_cast<uint8_t*>(paddingData.data()), paddingData.size() * sizeof(int32_t));
|
|
|
|
// create new tensor for padding
|
|
std::vector<int32_t> padShape{numDimensionsInput, 2};
|
|
auto padTensor = tflite::CreateTensorDirect(context->getBuilder(), &padShape /* shape */,
|
|
tflite::TensorType::TensorType_INT32 /* type */,
|
|
paddingBufferIdx /* buffer */);
|
|
int padTensorIdx = context->addTensorFlatbuffer(padTensor);
|
|
|
|
// add inputs for padding operation
|
|
std::vector<int32_t> padInputs = {context->getTensorIdxFromOperandIdx(operation.inputs[0]),
|
|
padTensorIdx};
|
|
|
|
// get dimensions of output of pad operation
|
|
std::vector<int32_t> padToConv2dShape(dims.begin(), dims.end());
|
|
// keep unknown height and width dimensions unknown
|
|
padToConv2dShape[1] = padToConv2dShape[1] != 0
|
|
? frontHeightPadding + padToConv2dShape[1] + backHeightPadding
|
|
: -1;
|
|
padToConv2dShape[2] = padToConv2dShape[2] != 0
|
|
? frontWidthPadding + padToConv2dShape[2] + backWidthPadding
|
|
: -1;
|
|
replaceZeroDimensions(&padToConv2dShape);
|
|
|
|
// build quantization parameters
|
|
std::vector<float> scaleVector{inputOperand.scale};
|
|
std::vector<int64_t> zeroPointVector{inputOperand.zeroPoint};
|
|
// min and max used to convert TFLite models to TF models, so it is unused in this case and can
|
|
// be set to 0
|
|
std::vector<float> minVector{0};
|
|
std::vector<float> maxVector{0};
|
|
auto quantizationParams = tflite::CreateQuantizationParametersDirect(
|
|
context->getBuilder(), &minVector /* min */, &maxVector /* max */,
|
|
&scaleVector /* scale */, &zeroPointVector /* zero_point */,
|
|
tflite::QuantizationDetails::QuantizationDetails_NONE /* details_type */);
|
|
|
|
// create new tensor to be output of pad & input for conv2d
|
|
auto padToConv2dTensor = tflite::CreateTensorDirect(
|
|
context->getBuilder(), &padToConv2dShape /* shape */,
|
|
NN_TRY(getTensorFlatbufferOperandType(inputOperand.type)) /* type */, 0 /* buffer */,
|
|
0 /* name */, quantizationParams /* quantization */);
|
|
int padToConv2dTensorIdx = context->addTensorFlatbuffer(padToConv2dTensor);
|
|
|
|
// set output for padding operation and add to operators
|
|
std::vector<int32_t> padOutputs{padToConv2dTensorIdx};
|
|
|
|
OperatorFlatbuffer padOp = tflite::CreateOperatorDirect(
|
|
context->getBuilder(), opCodeIdx, &padInputs, &padOutputs,
|
|
tflite::BuiltinOptions::BuiltinOptions_PadOptions, padOptionsFlatbuffer.Union());
|
|
context->addOperatorFlatbuffer(padOp);
|
|
|
|
// Return tensor index of pad output created
|
|
return padToConv2dTensorIdx;
|
|
}
|
|
|
|
Result<void> Conv2DOperationConverter::convert(const Operation& operation,
|
|
SubGraphContext* context) const {
|
|
const Model::Subgraph* subgraph = context->getSubgraph();
|
|
|
|
// add opcode for CONV_2D if not added yet
|
|
uint32_t opCodeIdx = context->addOpCode(OperationType::CONV_2D);
|
|
|
|
// if there are less than 8 inputs or the input at the 7th index is a BOOL, there is implicit
|
|
// padding
|
|
bool isImplicitPadding = false;
|
|
if (operation.inputs.size() < 8 ||
|
|
subgraph->operands[operation.inputs[7]].type == OperandType::BOOL) {
|
|
isImplicitPadding = true;
|
|
}
|
|
|
|
std::vector<int32_t> inputs = NN_TRY(getConv2DInputs(operation, context));
|
|
std::vector<int32_t> outputs = NN_TRY(getConv2DOutputs(operation, context));
|
|
|
|
// if explicit padding, we need to decompose the operation to a separate padding op and a conv2d
|
|
// op
|
|
if (!isImplicitPadding) {
|
|
auto padOpIdx = NN_TRY(decomposeExplicitPadding(operation, context));
|
|
inputs[0] = padOpIdx;
|
|
}
|
|
|
|
int baseOptionsIdx = 4;
|
|
tflite::Padding padding;
|
|
if (isImplicitPadding) {
|
|
const Operand& paddingTypeOperand = subgraph->operands[operation.inputs[3]];
|
|
NN_RET_CHECK(isOperandConstant(paddingTypeOperand));
|
|
|
|
int32_t paddingType = context->getConstantScalar<int32_t>(paddingTypeOperand);
|
|
padding = getTFLitePadding(paddingType);
|
|
} else {
|
|
padding = tflite::Padding::Padding_VALID;
|
|
baseOptionsIdx = 7;
|
|
}
|
|
|
|
// check if stride and activation Operands are constant
|
|
const Operand& strideWOperand =
|
|
subgraph->operands[operation.inputs[baseOptionsIdx + kStrideWOffset]];
|
|
const Operand& strideHOperand =
|
|
subgraph->operands[operation.inputs[baseOptionsIdx + kStrideHOffset]];
|
|
const Operand& activationOperand =
|
|
subgraph->operands[operation.inputs[baseOptionsIdx + kActivationOffset]];
|
|
NN_RET_CHECK(isOperandConstant(strideWOperand));
|
|
NN_RET_CHECK(isOperandConstant(strideHOperand));
|
|
NN_RET_CHECK(isOperandConstant(activationOperand));
|
|
|
|
// get strides and activation
|
|
int32_t strideW = context->getConstantScalar<int32_t>(strideWOperand);
|
|
int32_t strideH = context->getConstantScalar<int32_t>(strideHOperand);
|
|
FusedActivationFunc activation = static_cast<FusedActivationFunc>(
|
|
context->getConstantScalar<int32_t>(activationOperand));
|
|
|
|
// check for nchw
|
|
int isNchwIdx = baseOptionsIdx + kIsNchwOffset;
|
|
if (operation.inputs.size() > static_cast<uint32_t>(isNchwIdx)) {
|
|
const Operand& isNchwOperand = subgraph->operands[operation.inputs[isNchwIdx]];
|
|
NN_RET_CHECK(isOperandConstant(isNchwOperand));
|
|
|
|
bool isNchw = context->getConstantScalar<bool>(isNchwOperand);
|
|
NN_RET_CHECK(!isNchw) << "TFLite does not support NCHW formatted input tensors";
|
|
}
|
|
|
|
// dilations
|
|
int dilationWIdx = baseOptionsIdx + kDilationWOffset;
|
|
int dilationHIdx = baseOptionsIdx + kDilationHOffset;
|
|
// default dilation factors are 1
|
|
int32_t dilationW = 1;
|
|
int32_t dilationH = 1;
|
|
if (operation.inputs.size() > static_cast<uint32_t>(dilationWIdx)) {
|
|
const Operand& dilationWOperand = subgraph->operands[operation.inputs[dilationWIdx]];
|
|
NN_RET_CHECK(isOperandConstant(dilationWOperand));
|
|
|
|
dilationW = context->getConstantScalar<int32_t>(dilationWOperand);
|
|
}
|
|
if (operation.inputs.size() > static_cast<uint32_t>(dilationHIdx)) {
|
|
const Operand& dilationHOperand = subgraph->operands[operation.inputs[dilationHIdx]];
|
|
NN_RET_CHECK(isOperandConstant(dilationHOperand));
|
|
|
|
dilationH = context->getConstantScalar<int32_t>(dilationHOperand);
|
|
}
|
|
|
|
flatbuffers::Offset<tflite::Conv2DOptions> optionsFlatbuffer = tflite::CreateConv2DOptions(
|
|
context->getBuilder(), padding, strideW, strideH,
|
|
NN_TRY(getTfliteActivation(activation)) /* fused_activation_function */, dilationW,
|
|
dilationH);
|
|
auto operatorFlatbuffer = tflite::CreateOperatorDirect(
|
|
context->getBuilder() /* builder */, opCodeIdx /* opcode_index */, &inputs /* inputs */,
|
|
&outputs /* outputs */,
|
|
tflite::BuiltinOptions::BuiltinOptions_Conv2DOptions /* builtin_options_type */,
|
|
optionsFlatbuffer.Union() /* builtin_options */);
|
|
context->addOperatorFlatbuffer(operatorFlatbuffer);
|
|
|
|
return {};
|
|
}
|
|
|
|
NN_REGISTER_OPERATION_CONVERTER(CONV_2D, Conv2DOperationConverter);
|
|
|
|
} // namespace nn
|
|
} // namespace android
|