From 411fd2b761c7ad054d1bb9eeb496cdbd881da3fa Mon Sep 17 00:00:00 2001 From: Tiago De Gaspari Date: Mon, 7 Jun 2021 10:55:23 -0300 Subject: [PATCH 001/376] Add Thickness parameter in drawMatches function This commit adds the feature of selecting the thickness of the matches drawn by the drawMatches function. In larger images, the default thickness of 1 pixel creates images that are hard to visualize. --- .../features2d/include/opencv2/features2d.hpp | 7 ++++++ modules/features2d/src/draw.cpp | 25 ++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/modules/features2d/include/opencv2/features2d.hpp b/modules/features2d/include/opencv2/features2d.hpp index 86b5e935c8ad..cff09170c500 100644 --- a/modules/features2d/include/opencv2/features2d.hpp +++ b/modules/features2d/include/opencv2/features2d.hpp @@ -1314,6 +1314,13 @@ CV_EXPORTS_W void drawMatches( InputArray img1, const std::vector& key const std::vector& matchesMask=std::vector(), int flags=DrawMatchesFlags::DEFAULT ); /** @overload */ +CV_EXPORTS_W void drawMatches( InputArray img1, const std::vector& keypoints1, + InputArray img2, const std::vector& keypoints2, + const std::vector& matches1to2, InputOutputArray outImg, + const int matchesThickness, const Scalar& matchColor=Scalar::all(-1), + const Scalar& singlePointColor=Scalar::all(-1), const std::vector& matchesMask=std::vector(), + int flags=DrawMatchesFlags::DEFAULT ); + CV_EXPORTS_AS(drawMatchesKnn) void drawMatches( InputArray img1, const std::vector& keypoints1, InputArray img2, const std::vector& keypoints2, const std::vector >& matches1to2, InputOutputArray outImg, diff --git a/modules/features2d/src/draw.cpp b/modules/features2d/src/draw.cpp index dc74ecb0809c..e4c75144fbec 100644 --- a/modules/features2d/src/draw.cpp +++ b/modules/features2d/src/draw.cpp @@ -183,7 +183,8 @@ static void _prepareImgAndDrawKeypoints( InputArray img1, const std::vector& keypoints1, @@ -207,6 +208,21 @@ void drawMatches( InputArray img1, const std::vector& keypoints1, const std::vector& matches1to2, InputOutputArray outImg, const Scalar& matchColor, const Scalar& singlePointColor, const std::vector& matchesMask, int flags ) +{ + drawMatches( img1, keypoints1, + img2, keypoints2, + matches1to2, outImg, + 1, matchColor, + singlePointColor, matchesMask, + flags); +} + +void drawMatches( InputArray img1, const std::vector& keypoints1, + InputArray img2, const std::vector& keypoints2, + const std::vector& matches1to2, InputOutputArray outImg, + const int matchesThickness, const Scalar& matchColor, + const Scalar& singlePointColor, const std::vector& matchesMask, + int flags ) { if( !matchesMask.empty() && matchesMask.size() != matches1to2.size() ) CV_Error( Error::StsBadSize, "matchesMask must have the same size as matches1to2" ); @@ -226,11 +242,12 @@ void drawMatches( InputArray img1, const std::vector& keypoints1, CV_Assert(i2 >= 0 && i2 < static_cast(keypoints2.size())); const KeyPoint &kp1 = keypoints1[i1], &kp2 = keypoints2[i2]; - _drawMatch( outImg, outImg1, outImg2, kp1, kp2, matchColor, flags ); + _drawMatch( outImg, outImg1, outImg2, kp1, kp2, matchColor, flags, matchesThickness ); } } } + void drawMatches( InputArray img1, const std::vector& keypoints1, InputArray img2, const std::vector& keypoints2, const std::vector >& matches1to2, InputOutputArray outImg, @@ -254,7 +271,7 @@ void drawMatches( InputArray img1, const std::vector& keypoints1, if( matchesMask.empty() || matchesMask[i][j] ) { const KeyPoint &kp1 = keypoints1[i1], &kp2 = keypoints2[i2]; - _drawMatch( outImg, outImg1, outImg2, kp1, kp2, matchColor, flags ); + _drawMatch( outImg, outImg1, outImg2, kp1, kp2, matchColor, flags, 1 ); } } } From 8f4f834ce6ff054b16aeb3c92f6ea1279185b632 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Wed, 9 Jun 2021 18:43:42 +0300 Subject: [PATCH 002/376] applied modifier mask to the state --- modules/highgui/src/window_gtk.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/highgui/src/window_gtk.cpp b/modules/highgui/src/window_gtk.cpp index 17307ea7f988..78e78e12a290 100644 --- a/modules/highgui/src/window_gtk.cpp +++ b/modules/highgui/src/window_gtk.cpp @@ -1881,6 +1881,7 @@ static gboolean icvOnMouse( GtkWidget *widget, GdkEvent *event, gpointer user_da (unsigned)pt.y < (unsigned)(image_widget->original_image->height) )) { + state &= gtk_accelerator_get_default_mod_mask(); flags |= BIT_MAP(state, GDK_SHIFT_MASK, CV_EVENT_FLAG_SHIFTKEY) | BIT_MAP(state, GDK_CONTROL_MASK, CV_EVENT_FLAG_CTRLKEY) | BIT_MAP(state, GDK_MOD1_MASK, CV_EVENT_FLAG_ALTKEY) | From 7ee181661231a949d7998f3caaa9e4cba6250e3e Mon Sep 17 00:00:00 2001 From: rogday Date: Tue, 1 Jun 2021 17:05:27 +0300 Subject: [PATCH 003/376] split if into map of functions --- modules/dnn/src/tensorflow/tf_importer.cpp | 3789 +++++++++++--------- 1 file changed, 2006 insertions(+), 1783 deletions(-) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 084e4ac6daa5..39c230939474 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -510,2051 +510,2274 @@ class TFImporter private: void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId); + + typedef void (TFImporter::*TFImporterNodeParser)(tensorflow::GraphDef&, const tensorflow::NodeDef&, LayerParams&); + typedef std::map DispatchMap; + + const DispatchMap dispatch; + static const DispatchMap buildDispatchMap(); + + void parseConvolution (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseBias (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseMatMul (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseReshape (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseFlatten (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseTranspose (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseConstant (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseLrn (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseConcat (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseMaxPool (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseAvgPool (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseMaxPoolGrad (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parsePlaceholder (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseSplit (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseSlice (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseStridedSlice (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseMul (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseFusedBatchNorm (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseConv2DBackpropInput(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseBlockLSTM (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseResize (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseL2Normalize (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parsePriorBox (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseSoftmax (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseCropAndResize (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseMean (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parsePack (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseClipByValue (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseLeakyRelu (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseActivation (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + + void parseCustomLayer (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); }; -TFImporter::TFImporter(Net& net, const char *model, const char *config) - : dstNet(net) +const TFImporter::DispatchMap TFImporter::buildDispatchMap() { - if (model && model[0]) - { - CV_LOG_DEBUG(NULL, "DNN/TF: processing TensorFlow model from file: " << model); - ReadTFNetParamsFromBinaryFileOrDie(model, &netBin); - } - if (config && config[0]) - { - CV_LOG_DEBUG(NULL, "DNN/TF: processing TensorFlow config from file: " << config); - ReadTFNetParamsFromTextFileOrDie(config, &netTxt); - } - - populateNet(); + static DispatchMap dispatch; + dispatch["Conv2D"] = dispatch["SpaceToBatchND"] = dispatch["DepthwiseConv2dNative"] = + dispatch["Pad"] = dispatch["MirrorPad"] = dispatch["Conv3D"] = &TFImporter::parseConvolution; + dispatch["BiasAdd"] = dispatch["Add"] = dispatch["AddV2"] = dispatch["Sub"] = dispatch["AddN"] = &TFImporter::parseBias; + dispatch["MatMul"] = &TFImporter::parseMatMul; + dispatch["Reshape"] = &TFImporter::parseReshape; + dispatch["Flatten"] = dispatch["Squeeze"] = &TFImporter::parseFlatten; + dispatch["Transpose"] = &TFImporter::parseTranspose; + dispatch["Const"] = &TFImporter::parseConstant; + dispatch["LRN"] = &TFImporter::parseLrn; + dispatch["Concat"] = dispatch["ConcatV2"] = &TFImporter::parseConcat; + dispatch["MaxPool"] = dispatch["MaxPool3D"] = &TFImporter::parseMaxPool; + dispatch["AvgPool"] = dispatch["AvgPool3D"] = &TFImporter::parseAvgPool; + dispatch["MaxPoolGrad"] = &TFImporter::parseMaxPoolGrad; + dispatch["Placeholder"] = &TFImporter::parsePlaceholder; + dispatch["Split"] = &TFImporter::parseSplit; + dispatch["Slice"] = &TFImporter::parseSlice; + dispatch["StridedSlice"] = &TFImporter::parseStridedSlice; + dispatch["Mul"] = dispatch["RealDiv"] = &TFImporter::parseMul; + dispatch["FusedBatchNorm"] = dispatch["FusedBatchNormV3"] = &TFImporter::parseFusedBatchNorm; + dispatch["Conv2DBackpropInput"] = &TFImporter::parseConv2DBackpropInput; + dispatch["BlockLSTM"] = &TFImporter::parseBlockLSTM; + dispatch["ResizeNearestNeighbor"] = dispatch["ResizeBilinear"] = dispatch["FusedResizeAndPadConv2D"] = &TFImporter::parseResize; + dispatch["L2Normalize"] = &TFImporter::parseL2Normalize; + dispatch["PriorBox"] = &TFImporter::parsePriorBox; + dispatch["Softmax"] = &TFImporter::parseSoftmax; + dispatch["CropAndResize"] = &TFImporter::parseCropAndResize; + dispatch["Mean"] = dispatch["Sum"] = &TFImporter::parseMean; + dispatch["Pack"] = &TFImporter::parsePack; + dispatch["ClipByValue"] = &TFImporter::parseClipByValue; + dispatch["LeakyRelu"] = &TFImporter::parseLeakyRelu; + dispatch["Abs"] = dispatch["Tanh"] = dispatch["Sigmoid"] = dispatch["Relu"] = + dispatch["Elu"] = dispatch["Exp"] = dispatch["Identity"] = dispatch["Relu6"] = &TFImporter::parseActivation; + + return dispatch; } -TFImporter::TFImporter( - Net& net, - const char *dataModel, size_t lenModel, - const char *dataConfig, size_t lenConfig -) - : dstNet(net) +void TFImporter::parseConvolution(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer_, LayerParams& layerParams) { - if (dataModel != NULL && lenModel > 0) - { - CV_LOG_DEBUG(NULL, "DNN/TF: processing TensorFlow model from memory (" << lenModel << " bytes)"); - ReadTFNetParamsFromBinaryBufferOrDie(dataModel, lenModel, &netBin); - } - if (dataConfig != NULL && lenConfig > 0) + tensorflow::NodeDef layer = layer_; + std::string name = layer.name(); + std::string type = layer.op(); + int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + // The first node of dilated convolution subgraph. + // Extract input node, dilation rate and paddings. + std::string input = layer.input(0); + StrIntVector next_layers; + if (type == "SpaceToBatchND" || type == "Pad") { - CV_LOG_DEBUG(NULL, "DNN/TF: processing TensorFlow config from memory (" << lenConfig << " bytes)"); - ReadTFNetParamsFromTextBufferOrDie(dataConfig, lenConfig, &netTxt); + next_layers = getNextLayers(net, name, "Conv2D"); + if (next_layers.empty()) + next_layers = getNextLayers(net, name, "DepthwiseConv2dNative"); } - populateNet(); -} - -void TFImporter::kernelFromTensor(const tensorflow::TensorProto &tensor, Mat &dstBlob) -{ - MatShape shape; - blobShapeFromTensor(tensor, shape); - int dims = (int)shape.size(); - - // TODO: other blob types - CV_Assert(tensor.dtype() == tensorflow::DT_FLOAT || - tensor.dtype() == tensorflow::DT_HALF); - CV_Assert(dims == 4 || dims == 5); - int out_c, input_c, depth, height, width; - if (dims == 4) + if (type == "SpaceToBatchND") { - // REORDER kernel HWIO to OIHW - swap(shape[0], shape[2]); // IWHO - swap(shape[1], shape[3]); // IOHW - swap(shape[0], shape[1]); // OIHW - depth = 1; height = shape[2]; width = shape[3]; + // op: "SpaceToBatchND" + // input: "input" + // input: "SpaceToBatchND/block_shape" + // input: "SpaceToBatchND/paddings" + CV_CheckEQ(num_inputs, 3, ""); + + DictValue dilation = parseDims(getConstBlob(layer, value_id, 1)); + CV_Assert(dilation.size() == 2); + layerParams.set("dilation_h", dilation.get(0)); + layerParams.set("dilation_w", dilation.get(1)); + + Mat paddings; + parseTensor(getConstBlob(layer, value_id, 2), paddings); + + // paddings is a 2x2 matrix: [[top, bot], [left, right]] + layerParams.set("pad_h", paddings.at(0)); + layerParams.set("pad_w", paddings.at(2)); + + CV_Assert(next_layers.size() == 1); + layers_to_ignore.insert(next_layers[0].first); + + // FIXIT don't override, rewrite this code + layer = net.node(next_layers[0].second); + name = layer.name(); + type = layer.op(); + num_inputs = layer.input_size(); + CV_LOG_DEBUG(NULL, "DNN/TF: switched to layer " << name << " @ " << type << ") with " << num_inputs << " inputs"); } - else + else if (type == "Pad" || type == "MirrorPad") { - // REORDER kernel DHWIO to OIDHW - swap(shape[0], shape[4]); // OHWID - swap(shape[1], shape[3]); // OIWHD - swap(shape[2], shape[4]); // OIDHW - depth = shape[2]; height = shape[3]; width = shape[4]; - } - out_c = shape[0]; input_c = shape[1]; + Mat paddings = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_Assert(paddings.type() == CV_32SC1); + if (paddings.total() == 8) + { + // Perhaps, we have NHWC padding dimensions order. + // N H W C + // 0 1 2 3 4 5 6 7 + std::swap(paddings.at(2), paddings.at(6)); + std::swap(paddings.at(3), paddings.at(7)); + // N C W H + // 0 1 2 3 4 5 6 7 + std::swap(paddings.at(4), paddings.at(6)); + std::swap(paddings.at(5), paddings.at(7)); + // N C H W + // 0 1 2 3 4 5 6 7 + } - dstBlob.create(shape, CV_32F); + if (next_layers.empty() || paddings.total() != 8 || + paddings.at(4) != paddings.at(5) || + paddings.at(6) != paddings.at(7) || type == "MirrorPad") + { + // Just a single padding layer. + layerParams.set("paddings", DictValue::arrayInt((int*)paddings.data, paddings.total())); + if (type == "MirrorPad") + layerParams.set("type", "reflect"); - Mat tensorContent = getTensorContent(tensor, /*no copy*/false); - int size = tensorContent.total(); - CV_Assert(size == (int)dstBlob.total()); + int id = dstNet.addLayer(name, "Padding", layerParams); + layer_id[name] = id; - float *dstData = dstBlob.ptr(); - const float *data = reinterpret_cast(tensorContent.data); + connect(layer_id, dstNet, parsePin(input), id, 0); + return; + } + else + { + // Merge with subsequent convolutional layer. + CV_Assert(next_layers.size() == 1); - int total = out_c * input_c * depth * height * width; - for (int i_oc = 0; i_oc < out_c; i_oc++) { - for (int i_ic = 0; i_ic < input_c; i_ic++) { - for (int i_d = 0; i_d < depth; i_d++) { - for (int i_h = 0; i_h < height; i_h++) { - for (int i_w = 0; i_w < width; i_w++) { - int dst_i = input_c * depth * height * width * i_oc + - depth * height * width * i_ic + height * width * i_d + width * i_h + i_w; - int src_i = out_c * input_c * width * height * i_d + - out_c * input_c * width * i_h + out_c * input_c * i_w + out_c * i_ic + i_oc; - CV_Assert(dst_i < total); - CV_Assert(src_i < total); - dstData[dst_i] = data[src_i]; - } - } - } + layerParams.set("pad_h", paddings.at(4)); + layerParams.set("pad_w", paddings.at(6)); + + layers_to_ignore.insert(next_layers[0].first); + + // FIXIT don't override, rewrite this code + layer = net.node(next_layers[0].second); + name = layer.name(); + type = layer.op(); + num_inputs = layer.input_size(); + CV_LOG_DEBUG(NULL, "DNN/TF: switched to layer " << name << " @ " << type << ") with " << num_inputs << " inputs"); } } -} -void TFImporter::connect(const std::map& layers_name_id_map, Net& network, const Pin& outPin, - const int input_layer_id, const int input_blob_id) -{ - std::map::const_iterator it = layers_name_id_map.find(outPin.name); - if (it == layers_name_id_map.end()) - CV_Error(Error::StsError, "Input layer not found: " + outPin.name); + // For the object detection networks, TensorFlow Object Detection API + // predicts deltas for bounding boxes in yxYX (ymin, xmin, ymax, xmax) + // order. We can manage it at DetectionOutput layer parsing predictions + // or shuffle last convolution's weights. + bool locPredTransposed = hasLayerAttr(layer, "loc_pred_transposed") && + getLayerAttr(layer, "loc_pred_transposed").b(); - std::vector::iterator inpNameIt = std::find(netInputsNames.begin(), netInputsNames.end(), outPin.name); - int blobIndex; - if (inpNameIt == netInputsNames.end()) - blobIndex = outPin.blobIndex; - else - blobIndex = inpNameIt - netInputsNames.begin(); - network.connect(it->second, blobIndex, input_layer_id, input_blob_id); -} + layerParams.set("bias_term", false); + layerParams.blobs.resize(1); -void TFImporter::connectToAllBlobs(const std::map& layer_id, Net& network, const Pin& outPin, - const int input_layer_id, const int input_blobs_count) -{ - for (int input_blob_id = 0; input_blob_id < input_blobs_count; input_blob_id++) - connect(layer_id, network, outPin, input_layer_id, input_blob_id); -} + next_layers = getNextLayers(net, name, "BiasAdd"); + if (next_layers.size() == 1) { + layerParams.set("bias_term", true); + layerParams.blobs.resize(2); -const tensorflow::TensorProto& TFImporter::getConstBlob(const tensorflow::NodeDef &layer, std::map const_layers, - int input_blob_index, int* actual_inp_blob_idx) { - if (input_blob_index == -1) { - for(int i = 0; i < layer.input_size(); i++) { - Pin input = parsePin(layer.input(i)); - if (const_layers.find(input.name) != const_layers.end()) { - if (input_blob_index != -1) - CV_Error(Error::StsError, "More than one input is Const op"); + int weights_layer_index = next_layers[0].second; - input_blob_index = i; + blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); + ExcludeLayer(net, weights_layer_index, 0, false); + layers_to_ignore.insert(next_layers[0].first); + + // Shuffle bias from yxYX to xyXY. + if (locPredTransposed) + { + const int numWeights = layerParams.blobs[1].total(); + float* biasData = reinterpret_cast(layerParams.blobs[1].data); + CV_Assert(numWeights % 4 == 0); + for (int i = 0; i < numWeights; i += 2) + { + std::swap(biasData[i], biasData[i + 1]); } } } - if (input_blob_index == -1) - CV_Error(Error::StsError, "Const input blob for weights not found"); - - Pin kernel_inp = parsePin(layer.input(input_blob_index)); - if (const_layers.find(kernel_inp.name) == const_layers.end()) - CV_Error(Error::StsError, "Input [" + layer.input(input_blob_index) + - "] for node [" + layer.name() + "] not found"); - if (kernel_inp.blobIndex != 0) - CV_Error(Error::StsError, "Unsupported kernel input"); - - if(actual_inp_blob_idx) { - *actual_inp_blob_idx = input_blob_index; - } - - int nodeIdx = const_layers.at(kernel_inp.name); - if (nodeIdx < netBin.node_size() && netBin.node(nodeIdx).name() == kernel_inp.name) + int kernelTensorInpId = -1; + const tensorflow::TensorProto& kernelTensor = getConstBlob(layer, value_id, -1, &kernelTensorInpId); + const String kernelTensorName = layer.input(kernelTensorInpId); + std::map::iterator sharedWeightsIt = sharedWeights.find(kernelTensorName); + if (sharedWeightsIt == sharedWeights.end()) { - return netBin.node(nodeIdx).attr().at("value").tensor(); + kernelFromTensor(kernelTensor, layerParams.blobs[0]); + releaseTensor(const_cast(&kernelTensor)); + + int* kshape = layerParams.blobs[0].size.p; + const int outCh = kshape[0]; + const int inCh = kshape[1]; + const int height = kshape[2]; + const int width = kshape[3]; + if (type == "DepthwiseConv2dNative") + { + CV_Assert(!locPredTransposed); + const int chMultiplier = kshape[0]; + + Mat copy = layerParams.blobs[0].clone(); + float* src = (float*)copy.data; + float* dst = (float*)layerParams.blobs[0].data; + for (int i = 0; i < chMultiplier; ++i) + for (int j = 0; j < inCh; ++j) + for (int s = 0; s < height * width; ++s) + { + int src_i = (i * inCh + j) * height * width + s; + int dst_i = (j * chMultiplier + i) * height* width + s; + dst[dst_i] = src[src_i]; + } + // TODO Use reshape instead + kshape[0] = inCh * chMultiplier; + kshape[1] = 1; + size_t* kstep = layerParams.blobs[0].step.p; + kstep[0] = kstep[1]; // fix steps too + } + + // Shuffle output channels from yxYX to xyXY. + if (locPredTransposed) + { + const int slice = height * width * inCh; + for (int i = 0; i < outCh; i += 2) + { + cv::Mat src(1, slice, CV_32F, layerParams.blobs[0].ptr(i)); + cv::Mat dst(1, slice, CV_32F, layerParams.blobs[0].ptr(i + 1)); + std::swap_ranges(src.begin(), src.end(), dst.begin()); + } + } + sharedWeights[kernelTensorName] = layerParams.blobs[0]; } else { - CV_Assert_N(nodeIdx < netTxt.node_size(), - netTxt.node(nodeIdx).name() == kernel_inp.name); - return netTxt.node(nodeIdx).attr().at("value").tensor(); + layerParams.blobs[0] = sharedWeightsIt->second; } -} + Mat weights = layerParams.blobs[0]; + layerParams.set("kernel_size", DictValue::arrayInt(&weights.size[2], weights.dims - 2)); -static void addConstNodes(tensorflow::GraphDef& net, std::map& const_layers, - std::set& layers_to_ignore) -{ - CV_LOG_DEBUG(NULL, "DNN/TF: addConstNodes(): handling " << net.node_size() << " nodes..."); - for (int li = 0; li < net.node_size(); li++) - { - const tensorflow::NodeDef &layer = net.node(li); - String name = layer.name(); - String type = layer.op(); + layerParams.set("num_output", layerParams.blobs[0].size[0]); - //CV_LOG_DEBUG(NULL, "DNN/TF: layer_id=" << li << " - '" << name << "' @ " << type); + setStrides(layerParams, layer); + if (!layerParams.has("pad_w") && !layerParams.has("pad_h")) + setPadding(layerParams, layer); - try - { - if (type == "Dequantize") - { - // Example of Dequantize node: - // name: "conv2d_1/bias" - // op: "Dequantize" - // input: "conv2d_1/bias_quantized_const" (tensor of dtype DT_QUINT8) - // input: "conv2d_1/bias_quantized_min" - // input: "conv2d_1/bias_quantized_max" - // attr { key: "T" value { type: DT_QUINT8 } } (quantized type) - // attr { key: "mode" value { s: "MIN_FIRST" } } (quantization technique) - CV_CheckEQ(layer.input_size(), 3, "Dequantize: 3 inputs is supported only"); - for (int i = 0; i < 3; ++i) - CV_Assert(const_layers.find(layer.input(i)) != const_layers.end()); - CV_Assert(hasLayerAttr(layer, "mode") && - getLayerAttr(layer, "mode").s() == "MIN_FIRST"); + // The final node of dilated convolution subgraph. + next_layers = getNextLayers(net, name, "BatchToSpaceND"); + if (!next_layers.empty()) + { + CV_Assert(next_layers.size() == 1); + ExcludeLayer(net, next_layers[0].second, 0, false); + layers_to_ignore.insert(next_layers[0].first); + } - int tensorId = const_layers[layer.input(0)]; - int minId = const_layers[layer.input(1)]; - int maxId = const_layers[layer.input(2)]; + int id = dstNet.addLayer(name, "Convolution", layerParams); + layer_id[name] = id; - tensorflow::TensorProto* tensor = net.mutable_node(tensorId) - ->mutable_attr()->at("value") - .mutable_tensor(); - CV_CheckEQ((int)tensor->dtype(), (int)tensorflow::DT_QUINT8, ""); + // one input only + connect(layer_id, dstNet, parsePin(input), id, 0); - Mat qMin = getTensorContent(net.node(minId).attr().at("value").tensor()); - Mat qMax = getTensorContent(net.node(maxId).attr().at("value").tensor()); - CV_CheckEQ(qMin.total(), (size_t)1, ""); - CV_CheckTypeEQ(qMin.type(), CV_32FC1, ""); - CV_CheckEQ(qMax.total(), (size_t)1, ""); - CV_CheckTypeEQ(qMax.type(), CV_32FC1, ""); - Mat content = getTensorContent(*tensor); + if (getDataLayout(name, data_layouts) == DATA_LAYOUT_UNKNOWN) + data_layouts[name] = DATA_LAYOUT_NHWC; +} - float minVal = qMin.at(0); - float rangeScale = (qMax.at(0) - minVal) / 255; - CV_Assert(rangeScale >= 0); - content.convertTo(content, CV_32FC1, rangeScale, - rangeScale * cvRound(minVal / rangeScale)); +void TFImporter::parseBias(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); - tensor->set_dtype(tensorflow::DT_FLOAT); - tensor->set_tensor_content(content.data, content.total() * content.elemSize1()); + CV_CheckGT(num_inputs, 0, ""); + bool haveConst = false; + for(int ii = 0; !haveConst && ii < num_inputs; ++ii) + { + Pin input = parsePin(layer.input(ii)); + haveConst = value_id.find(input.name) != value_id.end(); + } + CV_Assert(!haveConst || num_inputs == 2); - net.mutable_node(tensorId)->set_name(name); - CV_Assert(const_layers.insert(std::make_pair(name, tensorId)).second); - layers_to_ignore.insert(name); - continue; - } - else if (type != "Const") - continue; // only Const parameters are supported + if (haveConst) + { + Mat values = getTensorContent(getConstBlob(layer, value_id)); + CV_Assert(values.type() == CV_32FC1); + if (type == "Sub") + values *= -1.0f; - if (layer.attr().find("value") != layer.attr().end()) - { - CV_Assert(const_layers.insert(std::make_pair(name, li)).second); - } - layers_to_ignore.insert(name); + int id; + if (values.total() == 1) // is a scalar. + { + layerParams.set("shift", values.at(0)); + id = dstNet.addLayer(name, "Power", layerParams); } - catch (const std::exception& e) + else // is a vector { - CV_LOG_ERROR(NULL, "DNN/TF: Can't handle node='" << name << "'. Exception: " << e.what()); - throw; + layerParams.blobs.resize(1, values); + id = dstNet.addLayer(name, "Shift", layerParams); + } + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + } + else + { + layerParams.set("operation", "sum"); + if (type == "Sub") + { + static float subCoeffs[] = {1.f, -1.f}; + layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); + } + + int id = dstNet.addLayer(name, "Eltwise", layerParams); + layer_id[name] = id; + + for (int ii = 0; ii < num_inputs; ii++) + { + Pin inp = parsePin(layer.input(ii)); + if (layer_id.find(inp.name) == layer_id.end()) + CV_Error(Error::StsError, "Input layer not found: " + inp.name); + connect(layer_id, dstNet, inp, id, ii); } } - CV_LOG_DEBUG(NULL, "DNN/TF: layers_to_ignore.size() = " << layers_to_ignore.size()); } -// If all inputs of specific layer have the same data layout we can say that -// this layer's output has this data layout too. Returns DATA_LAYOUT_UNKNOWN otherwise. -DataLayout TFImporter::predictOutputDataLayout(const tensorflow::NodeDef& layer) +void TFImporter::parseMatMul(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { - DataLayout layout = getDataLayout(layer); - if (layout != DATA_LAYOUT_UNKNOWN) + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 2, ""); + + // For the object detection networks, TensorFlow Object Detection API + // predicts deltas for bounding boxes in yxYX (ymin, xmin, ymax, xmax) + // order. We can manage it at DetectionOutput layer parsing predictions + // or shuffle last Faster-RCNN's matmul weights. + bool locPredTransposed = hasLayerAttr(layer, "loc_pred_transposed") && + getLayerAttr(layer, "loc_pred_transposed").b(); + + layerParams.set("bias_term", false); + layerParams.blobs.resize(1); + + StrIntVector next_layers = getNextLayers(net, name, "BiasAdd"); // FIXIT Use layers fusion instead + if (next_layers.empty()) { - CV_LOG_DEBUG(NULL, "DNN/TF: predictOutputDataLayout(" << layer.name() << " @ " << layer.op() << ") => " << (int)layout << " (from attrs)"); - return layout; + next_layers = getNextLayers(net, name, "Add"); } + if (next_layers.size() == 1) { + layerParams.set("bias_term", true); + layerParams.blobs.resize(2); - // Determine layout by layer's inputs - for (int i = 0, n = layer.input_size(); i < n; ++i) - { - std::map::const_iterator it = data_layouts.find(getNodeName(layer.input(i))); - if (it != data_layouts.end()) + int weights_layer_index = next_layers[0].second; + blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); + ExcludeLayer(net, weights_layer_index, 0, false); + layers_to_ignore.insert(next_layers[0].first); + + if (locPredTransposed) { - if (layout != DATA_LAYOUT_UNKNOWN) + const int numWeights = layerParams.blobs[1].total(); + float* biasData = reinterpret_cast(layerParams.blobs[1].data); + CV_Assert(numWeights % 4 == 0); + for (int i = 0; i < numWeights; i += 2) { - if (it->second != layout && it->second != DATA_LAYOUT_UNKNOWN) - return DATA_LAYOUT_UNKNOWN; + std::swap(biasData[i], biasData[i + 1]); } - else - layout = it->second; } } - if (layout != DATA_LAYOUT_UNKNOWN) - { - CV_LOG_DEBUG(NULL, "DNN/TF: predictOutputDataLayout(" << layer.name() << " @ " << layer.op() << ") => " << (int)layout << " (from inputs)"); - return layout; - } - - // Determine layout by layer's consumers recursively. - std::map::const_iterator it = data_layouts.find(layer.name()); - CV_Assert(it != data_layouts.end()); - return it->second; -} - -void TFImporter::populateNet() -{ - CV_Assert(netBin.ByteSize() || netTxt.ByteSize()); - - CV_LOG_INFO(NULL, "DNN/TF: parsing model" - << (netBin.has_versions() ? cv::format(" produced by TF v%d (min_consumer=%d)", (int)netBin.versions().producer(), (int)netBin.versions().min_consumer()) : cv::String(" (N/A version info)")) - << ". Number of nodes = " << netBin.node_size() - ); - - if (netTxt.ByteSize()) + int kernel_blob_index = -1; + const tensorflow::TensorProto& kernelTensor = getConstBlob(layer, value_id, -1, &kernel_blob_index); + const String kernelTensorName = layer.input(kernel_blob_index); + std::map::iterator sharedWeightsIt = sharedWeights.find(kernelTensorName); + if (sharedWeightsIt == sharedWeights.end()) { - CV_LOG_INFO(NULL, "DNN/TF: parsing config" - << (netTxt.has_versions() ? cv::format(" produced by TF v%d (min_consumer=%d)", (int)netTxt.versions().producer(), (int)netTxt.versions().min_consumer()) : cv::String(" (N/A version info)")) - << ". Number of nodes = " << netTxt.node_size() - ); - - RemoveIdentityOps(netBin); - CV_LOG_DEBUG(NULL, "DNN/TF: RemoveIdentityOps(model) => " << netBin.node_size() << " nodes"); - RemoveIdentityOps(netTxt); - CV_LOG_DEBUG(NULL, "DNN/TF: RemoveIdentityOps(config) => " << netTxt.node_size() << " nodes"); - - sortByExecutionOrder(netTxt); - CV_LOG_DEBUG(NULL, "DNN/TF: sortByExecutionOrder(config) => " << netTxt.node_size() << " nodes"); + blobFromTensor(kernelTensor, layerParams.blobs[0]); + releaseTensor(const_cast(&kernelTensor)); + sharedWeights[kernelTensorName] = layerParams.blobs[0]; } else { - removePhaseSwitches(netBin); - CV_LOG_DEBUG(NULL, "DNN/TF: removePhaseSwitches(model) => " << netBin.node_size() << " nodes"); + layerParams.blobs[0] = sharedWeightsIt->second; + } - RemoveIdentityOps(netBin); - CV_LOG_DEBUG(NULL, "DNN/TF: RemoveIdentityOps(model) => " << netBin.node_size() << " nodes"); + if (kernel_blob_index == 1) { // In this case output is computed by x*W formula - W should be transposed + Mat data = layerParams.blobs[0].t(); + layerParams.blobs[0] = data.clone(); + } - simplifySubgraphs(netBin); - CV_LOG_DEBUG(NULL, "DNN/TF: simplifySubgraphs(model) => " << netBin.node_size() << " nodes"); - sortByExecutionOrder(netBin); - CV_LOG_DEBUG(NULL, "DNN/TF: sortByExecutionOrder(model) => " << netBin.node_size() << " nodes"); + layerParams.set("num_output", layerParams.blobs[0].size[0]); + if (locPredTransposed) + { + CV_Assert(layerParams.blobs[0].dims == 2); + for (int i = 0; i < layerParams.blobs[0].size[0]; i += 2) + { + cv::Mat src = layerParams.blobs[0].row(i); + cv::Mat dst = layerParams.blobs[0].row(i + 1); + std::swap_ranges(src.begin(), src.end(), dst.begin()); + } } - tensorflow::GraphDef& net = netTxt.ByteSize() != 0 ? netTxt : netBin; + int id = dstNet.addLayer(name, "InnerProduct", layerParams); + layer_id[name] = id; - int layersSize = net.node_size(); + // one input only + int input_blob_index = kernel_blob_index == 0 ? 1 : 0; + connect(layer_id, dstNet, parsePin(layer.input(input_blob_index)), id, 0); + data_layouts[name] = DATA_LAYOUT_PLANAR; +} - // Pre-fill data layouts where they are set explicitly. - // Assuming that nodes are in topological order - for (int i = layersSize - 1; i >= 0; --i) +void TFImporter::parseReshape(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + Pin inpId = parsePin(layer.input(0)); + DataLayout inpLayout = getDataLayout(layer.input(0), data_layouts); + // There are two possible implementations: reshape an input using + // predefined sizes or use a second input blob as a source of new shape. + if (value_id.find(layer.input(1)) != value_id.end()) { - const tensorflow::NodeDef& layer = net.node(i); - std::string name = layer.name(); - - CV_LOG_DEBUG(NULL, "DNN/TF: node(" << i << " - '" << name << "') propagating layout..."); - - try + Mat newShape = getTensorContent(getConstBlob(layer, value_id, 1)); + int newShapeSize = newShape.total(); + bool hasSwap = false; + if (newShapeSize == 4 && hasAllOnes(newShape, 0, 2)) { - DataLayout layout = getDataLayout(layer); - std::map::iterator it = data_layouts.find(name); - if (it != data_layouts.end()) + // NHWC->NCHW + std::swap(*newShape.ptr(0, 2), *newShape.ptr(0, 3)); + std::swap(*newShape.ptr(0, 1), *newShape.ptr(0, 2)); + hasSwap = true; + } + if (inpLayout == DATA_LAYOUT_NHWC) + { + if (newShapeSize >= 2 || newShape.at(1) == 1) { - if (layout != DATA_LAYOUT_UNKNOWN) + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + addPermuteLayer(order, name + "/nhwc", inpId); + if (newShapeSize < 4) { - if (it->second == DATA_LAYOUT_UNKNOWN) - it->second = layout; - else if (it->second != layout) - { - it->second = DATA_LAYOUT_UNKNOWN; - layout = DATA_LAYOUT_UNKNOWN; - } + inpLayout = DATA_LAYOUT_NCHW; } else - layout = it->second; - } - else - data_layouts[name] = layout; - - // Specify input layers to have the same data layout. - for (int j = 0; j < layer.input_size(); ++j) - { - name = getNodeName(layer.input(j)); - it = data_layouts.find(name); - if (it != data_layouts.end()) { - if (layout != DATA_LAYOUT_UNKNOWN) - { - if (it->second == DATA_LAYOUT_UNKNOWN) - it->second = layout; - else if (it->second != layout) - it->second = DATA_LAYOUT_UNKNOWN; - } + inpLayout = DATA_LAYOUT_NHWC; } - else - data_layouts[name] = layout; } } - catch (const std::exception& e) - { - CV_LOG_ERROR(NULL, "DNN/TF: Can't propagate layout for node='" << name << "'. Exception: " << e.what()); - throw; - } - } - - addConstNodes(netBin, value_id, layers_to_ignore); - addConstNodes(netTxt, value_id, layers_to_ignore); + layerParams.set("dim", DictValue::arrayInt(newShape.ptr(), newShapeSize)); + int id = dstNet.addLayer(name, "Reshape", layerParams); + layer_id[name] = id; - for (int li = 0; li < layersSize; li++) - { - const tensorflow::NodeDef& layer = net.node(li); + // one input only + connect(layer_id, dstNet, inpId, id, 0); + inpId = Pin(name); - const std::string name = layer.name(); - const std::string type = layer.op(); - const int ninputs = layer.input_size(); - CV_LOG_DEBUG(NULL, "DNN/TF: (" << li << "/" << layersSize << ") Parse layer " << name << " @ " << type << " with " << ninputs << " inputs"); + if ((inpLayout == DATA_LAYOUT_NHWC || inpLayout == DATA_LAYOUT_UNKNOWN || inpLayout == DATA_LAYOUT_PLANAR) && + newShapeSize == 4 && !hasSwap) + { + int order[] = {0, 3, 1, 2}; // Transform back to OpenCV's NCHW. + addPermuteLayer(order, name + "/nchw", inpId); + inpLayout = DATA_LAYOUT_NCHW; + } - parseNode(layer); + data_layouts[name] = newShapeSize == 2 ? DATA_LAYOUT_PLANAR : inpLayout; } - - for (size_t i = 0; i < netInputsNames.size(); i++) + else { - CV_LOG_DEBUG(NULL, "DNN/TF: Model input: " << i << " - '" << netInputsNames[i] << "'"); - CV_Assert(!netInputsNames[i].empty()); + int id = dstNet.addLayer(name, "Reshape", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, inpId, id, 0); + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); + data_layouts[name] = inpLayout; } - dstNet.setInputsNames(netInputsNames); - CV_LOG_DEBUG(NULL, "DNN/TF: ===================== Import completed ====================="); } -void TFImporter::addPermuteLayer(const int* order, const std::string& permName, Pin& inpId) +void TFImporter::parseFlatten(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { - LayerParams permLP; - permLP.set("order", DictValue::arrayInt(order, 4)); - CV_Assert(layer_id.find(permName) == layer_id.end()); - int permId = dstNet.addLayer(permName, "Permute", permLP); - layer_id[permName] = permId; - connect(layer_id, dstNet, inpId, permId, 0); - inpId = Pin(permName); + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + Pin inpId = parsePin(layer.input(0)); + int inpLayout = getDataLayout(layer.input(0), data_layouts); + if (type == "Squeeze") + { + CV_Assert(hasLayerAttr(layer, "squeeze_dims")); + const tensorflow::AttrValue& dims = getLayerAttr(layer, "squeeze_dims"); + std::vector dimsVector(dims.list().i_size()); + for (int i = 0; i < dimsVector.size(); ++i) + dimsVector[i] = dims.list().i(i); + + // Flatten layer can squeeze dimensions range into one. + std::sort(dimsVector.begin(), dimsVector.end()); + for (int i = 1; i < dimsVector.size(); ++i) + { + if (dimsVector[i] != dimsVector[i - 1] + 1) + CV_Error(Error::StsNotImplemented, "Unsupported squeeze configuration"); + } + int start = dimsVector.front() - 1, end = dimsVector.back(); + if (start == -1 && end == 0) // squeeze 0th dimension + { + start = 0; + end = 1; + } + layerParams.set("axis", start); + layerParams.set("end_axis", end); + } + if (inpLayout == DATA_LAYOUT_NHWC) + { + LayerParams permLP; + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + permLP.set("order", DictValue::arrayInt(order, 4)); + + std::string permName = name + "/nchw"; + CV_Assert(layer_id.find(permName) == layer_id.end()); + int permId = dstNet.addLayer(permName, "Permute", permLP); + layer_id[permName] = permId; + connect(layer_id, dstNet, inpId, permId, 0); + inpId = Pin(permName); + } + int id = dstNet.addLayer(name, "Flatten", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, inpId, id, 0); + data_layouts[name] = DATA_LAYOUT_PLANAR; } -void TFImporter::parseNode(const tensorflow::NodeDef& layer_) +void TFImporter::parseTranspose(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { - tensorflow::NodeDef layer = layer_; - - tensorflow::GraphDef& net = netTxt.ByteSize() != 0 ? netTxt : netBin; - - /*const*/ std::string name = layer.name(); - /*const*/ std::string type = layer.op(); - /*const*/ int num_inputs = layer.input_size(); - - try + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + Mat perm = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_Assert(perm.type() == CV_32SC1); + int* permData = (int*)perm.data; + if (perm.total() == 4) { - LayerParams layerParams; - - if (layers_to_ignore.find(name) != layers_to_ignore.end()) + // Only NHWC <-> NCHW permutations are allowed. OpenCV is always + // keep NCHW layout this way. + int inpLayout = getDataLayout(layer.input(0), data_layouts); + std::string type = "Identity"; + if (inpLayout == DATA_LAYOUT_NHWC) { - CV_LOG_DEBUG(NULL, "DNN/TF: ignored"); - return; + if (permData[0] == 0 && permData[1] == 3 && permData[2] == 1 && permData[3] == 2) + { + // in TensorFlow: NHWC->NCHW + // in OpenCV: NCHW->NCHW + data_layouts[name] = DATA_LAYOUT_NCHW; + } + else if (permData[0] == 0 && permData[1] == 1 && permData[2] == 2 && permData[3] == 3) + { + // in TensorFlow: NHWC->NHWC + // in OpenCV: NCHW->NCHW + data_layouts[name] = DATA_LAYOUT_NHWC; + } + else if (permData[0] == 0 && permData[1] == 3 && permData[2] == 2 && permData[3] == 1) + { + // in TensorFlow: NHWC->NCWH + // in OpenCV: NCHW->NCWH + int permData[] = {0, 1, 3, 2}; + layerParams.set("order", DictValue::arrayInt(permData, perm.total())); + data_layouts[name] = DATA_LAYOUT_NCHW; // we keep track NCHW because channels position only matters + type = "Permute"; + } + else + CV_Error(Error::StsParseError, "Only NHWC <-> NCHW permutations are allowed."); } - - DataLayout predictedLayout = predictOutputDataLayout(layer); - data_layouts[name] = predictedLayout; - - if (type == "Conv2D" || type == "SpaceToBatchND" || type == "DepthwiseConv2dNative" || type == "Pad" || type == "MirrorPad" || type == "Conv3D") + else if (inpLayout == DATA_LAYOUT_NCHW) { - CV_CheckGT(num_inputs, 0, ""); - // The first node of dilated convolution subgraph. - // Extract input node, dilation rate and paddings. - std::string input = layer.input(0); - StrIntVector next_layers; - if (type == "SpaceToBatchND" || type == "Pad") + if (permData[0] == 0 && permData[1] == 2 && permData[2] == 3 && permData[3] == 1) { - next_layers = getNextLayers(net, name, "Conv2D"); - if (next_layers.empty()) - next_layers = getNextLayers(net, name, "DepthwiseConv2dNative"); + // in TensorFlow: NCHW->NHWC + // in OpenCV: NCHW->NCHW + data_layouts[name] = DATA_LAYOUT_NHWC; } - - if (type == "SpaceToBatchND") + else if (permData[0] == 0 && permData[1] == 1 && permData[2] == 2 && permData[3] == 3) { - // op: "SpaceToBatchND" - // input: "input" - // input: "SpaceToBatchND/block_shape" - // input: "SpaceToBatchND/paddings" - CV_CheckEQ(num_inputs, 3, ""); + // in TensorFlow: NCHW->NCHW + // in OpenCV: NCHW->NCHW + data_layouts[name] = DATA_LAYOUT_NCHW; + } + else + CV_Error(Error::StsParseError, "Only NHWC <-> NCHW permutations are allowed."); + } + int id = dstNet.addLayer(name, type, layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + } + else + { + layerParams.set("order", DictValue::arrayInt(permData, perm.total())); - DictValue dilation = parseDims(getConstBlob(layer, value_id, 1)); - CV_Assert(dilation.size() == 2); - layerParams.set("dilation_h", dilation.get(0)); - layerParams.set("dilation_w", dilation.get(1)); + int id = dstNet.addLayer(name, "Permute", layerParams); + layer_id[name] = id; - Mat paddings; - parseTensor(getConstBlob(layer, value_id, 2), paddings); + // one input only + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + data_layouts[name] = DATA_LAYOUT_UNKNOWN; + } +} - // paddings is a 2x2 matrix: [[top, bot], [left, right]] - layerParams.set("pad_h", paddings.at(0)); - layerParams.set("pad_w", paddings.at(2)); +void TFImporter::parseConstant(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ +} - CV_Assert(next_layers.size() == 1); - layers_to_ignore.insert(next_layers[0].first); +void TFImporter::parseLrn(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); - // FIXIT don't override, rewrite this code - layer = net.node(next_layers[0].second); - name = layer.name(); - type = layer.op(); - num_inputs = layer.input_size(); - CV_LOG_DEBUG(NULL, "DNN/TF: switched to layer " << name << " @ " << type << ") with " << num_inputs << " inputs"); - } - else if (type == "Pad" || type == "MirrorPad") - { - Mat paddings = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_Assert(paddings.type() == CV_32SC1); - if (paddings.total() == 8) - { - // Perhaps, we have NHWC padding dimensions order. - // N H W C - // 0 1 2 3 4 5 6 7 - std::swap(paddings.at(2), paddings.at(6)); - std::swap(paddings.at(3), paddings.at(7)); - // N C W H - // 0 1 2 3 4 5 6 7 - std::swap(paddings.at(4), paddings.at(6)); - std::swap(paddings.at(5), paddings.at(7)); - // N C H W - // 0 1 2 3 4 5 6 7 - } + CV_CheckGT(num_inputs, 0, ""); + if(hasLayerAttr(layer, "alpha")) { + layerParams.set("alpha", getLayerAttr(layer, "alpha").f()); + } + if(hasLayerAttr(layer, "beta")) { + layerParams.set("beta", getLayerAttr(layer, "beta").f()); + } + if(hasLayerAttr(layer, "depth_radius")) { + int radius = (int)getLayerAttr(layer, "depth_radius").i(); + layerParams.set("local_size", 2*radius + 1); + } + if(hasLayerAttr(layer, "bias")) { + layerParams.set("bias", getLayerAttr(layer, "bias").f()); + } + layerParams.set("norm_by_size", false); - if (next_layers.empty() || paddings.total() != 8 || - paddings.at(4) != paddings.at(5) || - paddings.at(6) != paddings.at(7) || type == "MirrorPad") - { - // Just a single padding layer. - layerParams.set("paddings", DictValue::arrayInt((int*)paddings.data, paddings.total())); - if (type == "MirrorPad") - layerParams.set("type", "reflect"); + int id = dstNet.addLayer(name, "LRN", layerParams); + layer_id[name] = id; - int id = dstNet.addLayer(name, "Padding", layerParams); - layer_id[name] = id; + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} - connect(layer_id, dstNet, parsePin(input), id, 0); - return; - } - else - { - // Merge with subsequent convolutional layer. - CV_Assert(next_layers.size() == 1); +void TFImporter::parseConcat(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); - layerParams.set("pad_h", paddings.at(4)); - layerParams.set("pad_w", paddings.at(6)); + CV_CheckGT(num_inputs, 0, ""); + int axisId = (type == "Concat" ? 0 : num_inputs - 1); + int axis = getConstBlob(layer, value_id, axisId).int_val().Get(0); - layers_to_ignore.insert(next_layers[0].first); + if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) + axis = toNCHW(axis); + else if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NDHWC) + axis = toNCDHW(axis); + layerParams.set("axis", axis); - // FIXIT don't override, rewrite this code - layer = net.node(next_layers[0].second); - name = layer.name(); - type = layer.op(); - num_inputs = layer.input_size(); - CV_LOG_DEBUG(NULL, "DNN/TF: switched to layer " << name << " @ " << type << ") with " << num_inputs << " inputs"); - } - } + // input(0) or input(n-1) is concat_dim + int from = (type == "Concat" ? 1 : 0); + int to = (type == "Concat" ? num_inputs : num_inputs - 1); - // For the object detection networks, TensorFlow Object Detection API - // predicts deltas for bounding boxes in yxYX (ymin, xmin, ymax, xmax) - // order. We can manage it at DetectionOutput layer parsing predictions - // or shuffle last convolution's weights. - bool locPredTransposed = hasLayerAttr(layer, "loc_pred_transposed") && - getLayerAttr(layer, "loc_pred_transposed").b(); + for (int ii = from; ii < to; ii++) + { + Pin inp = parsePin(layer.input(ii)); + if (layer_id.find(inp.name) == layer_id.end()) + { + // There are constant inputs. + LayerParams lp; + lp.name = inp.name; + lp.type = "Const"; + lp.blobs.resize(1); + blobFromTensor(getConstBlob(layer, value_id, ii), lp.blobs.back()); + CV_Assert_N(!lp.blobs[0].empty(), lp.blobs[0].type() == CV_32F); + + int constInpId = dstNet.addLayer(lp.name, lp.type, lp); + layer_id[lp.name] = constInpId; + } + } - layerParams.set("bias_term", false); - layerParams.blobs.resize(1); + int id = dstNet.addLayer(name, "Concat", layerParams); + layer_id[name] = id; - next_layers = getNextLayers(net, name, "BiasAdd"); - if (next_layers.size() == 1) { - layerParams.set("bias_term", true); - layerParams.blobs.resize(2); + for (int ii = from; ii < to; ii++) + { + Pin inp = parsePin(layer.input(ii)); + if (layer_id.find(inp.name) == layer_id.end()) + CV_Error(Error::StsError, "Input layer not found: " + inp.name); + connect(layer_id, dstNet, inp, id, ii - from); + } +} - int weights_layer_index = next_layers[0].second; +void TFImporter::parseMaxPool(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); - blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); - ExcludeLayer(net, weights_layer_index, 0, false); - layers_to_ignore.insert(next_layers[0].first); + CV_CheckGT(num_inputs, 0, ""); + layerParams.set("pool", "max"); - // Shuffle bias from yxYX to xyXY. - if (locPredTransposed) - { - const int numWeights = layerParams.blobs[1].total(); - float* biasData = reinterpret_cast(layerParams.blobs[1].data); - CV_Assert(numWeights % 4 == 0); - for (int i = 0; i < numWeights; i += 2) - { - std::swap(biasData[i], biasData[i + 1]); - } - } - } + setKSize(layerParams, layer); + setStrides(layerParams, layer); + setPadding(layerParams, layer); + // Test_TensorFlow_nets.EAST_text_detection/1, NGRAPH/CPU + layerParams.set("ceil_mode", false); - int kernelTensorInpId = -1; - const tensorflow::TensorProto& kernelTensor = getConstBlob(layer, value_id, -1, &kernelTensorInpId); - const String kernelTensorName = layer.input(kernelTensorInpId); - std::map::iterator sharedWeightsIt = sharedWeights.find(kernelTensorName); - if (sharedWeightsIt == sharedWeights.end()) - { - kernelFromTensor(kernelTensor, layerParams.blobs[0]); - releaseTensor(const_cast(&kernelTensor)); - - int* kshape = layerParams.blobs[0].size.p; - const int outCh = kshape[0]; - const int inCh = kshape[1]; - const int height = kshape[2]; - const int width = kshape[3]; - if (type == "DepthwiseConv2dNative") - { - CV_Assert(!locPredTransposed); - const int chMultiplier = kshape[0]; - - Mat copy = layerParams.blobs[0].clone(); - float* src = (float*)copy.data; - float* dst = (float*)layerParams.blobs[0].data; - for (int i = 0; i < chMultiplier; ++i) - for (int j = 0; j < inCh; ++j) - for (int s = 0; s < height * width; ++s) - { - int src_i = (i * inCh + j) * height * width + s; - int dst_i = (j * chMultiplier + i) * height* width + s; - dst[dst_i] = src[src_i]; - } - // TODO Use reshape instead - kshape[0] = inCh * chMultiplier; - kshape[1] = 1; - size_t* kstep = layerParams.blobs[0].step.p; - kstep[0] = kstep[1]; // fix steps too - } + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; - // Shuffle output channels from yxYX to xyXY. - if (locPredTransposed) - { - const int slice = height * width * inCh; - for (int i = 0; i < outCh; i += 2) - { - cv::Mat src(1, slice, CV_32F, layerParams.blobs[0].ptr(i)); - cv::Mat dst(1, slice, CV_32F, layerParams.blobs[0].ptr(i + 1)); - std::swap_ranges(src.begin(), src.end(), dst.begin()); - } - } - sharedWeights[kernelTensorName] = layerParams.blobs[0]; - } - else - { - layerParams.blobs[0] = sharedWeightsIt->second; - } - Mat weights = layerParams.blobs[0]; - layerParams.set("kernel_size", DictValue::arrayInt(&weights.size[2], weights.dims - 2)); + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} - layerParams.set("num_output", layerParams.blobs[0].size[0]); +void TFImporter::parseAvgPool(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); - setStrides(layerParams, layer); - if (!layerParams.has("pad_w") && !layerParams.has("pad_h")) - setPadding(layerParams, layer); + CV_CheckGT(num_inputs, 0, ""); + layerParams.set("pool", "ave"); + layerParams.set("ave_pool_padded_area", false); + setKSize(layerParams, layer); + setStrides(layerParams, layer); + setPadding(layerParams, layer); - // The final node of dilated convolution subgraph. - next_layers = getNextLayers(net, name, "BatchToSpaceND"); - if (!next_layers.empty()) - { - CV_Assert(next_layers.size() == 1); - ExcludeLayer(net, next_layers[0].second, 0, false); - layers_to_ignore.insert(next_layers[0].first); - } + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; - int id = dstNet.addLayer(name, "Convolution", layerParams); - layer_id[name] = id; + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} - // one input only - connect(layer_id, dstNet, parsePin(input), id, 0); +void TFImporter::parseMaxPoolGrad(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + CV_CheckEQ(num_inputs, 3, ""); - if (getDataLayout(name, data_layouts) == DATA_LAYOUT_UNKNOWN) - data_layouts[name] = DATA_LAYOUT_NHWC; - } - else if (type == "BiasAdd" || type == "Add" || type == "AddV2" || type == "Sub" || type=="AddN") - { - CV_CheckGT(num_inputs, 0, ""); - bool haveConst = false; - for(int ii = 0; !haveConst && ii < num_inputs; ++ii) - { - Pin input = parsePin(layer.input(ii)); - haveConst = value_id.find(input.name) != value_id.end(); - } - CV_Assert(!haveConst || num_inputs == 2); + layerParams.set("pool_k_h", 0); + layerParams.set("pool_k_w", 0); + layerParams.set("pool_stride_h", 0); + layerParams.set("pool_stride_w", 0); + layerParams.set("pool_pad_h", 0); + layerParams.set("pool_pad_w", 0); - if (haveConst) - { - Mat values = getTensorContent(getConstBlob(layer, value_id)); - CV_Assert(values.type() == CV_32FC1); - if (type == "Sub") - values *= -1.0f; + int id = dstNet.addLayer(name, "MaxUnpool", layerParams); + layer_id[name] = id; - int id; - if (values.total() == 1) // is a scalar. - { - layerParams.set("shift", values.at(0)); - id = dstNet.addLayer(name, "Power", layerParams); - } - else // is a vector - { - layerParams.blobs.resize(1, values); - id = dstNet.addLayer(name, "Shift", layerParams); - } - layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); + connect(layer_id, dstNet, parsePin(layer.input(1) + ":1"), id, 1); + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 2); +} - // one input only - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else - { - layerParams.set("operation", "sum"); - if (type == "Sub") - { - static float subCoeffs[] = {1.f, -1.f}; - layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); - } +void TFImporter::parsePlaceholder(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); - int id = dstNet.addLayer(name, "Eltwise", layerParams); - layer_id[name] = id; + DataLayout predictedLayout = data_layouts[name]; - for (int ii = 0; ii < num_inputs; ii++) - { - Pin inp = parsePin(layer.input(ii)); - if (layer_id.find(inp.name) == layer_id.end()) - CV_Error(Error::StsError, "Input layer not found: " + inp.name); - connect(layer_id, dstNet, inp, id, ii); - } - } + if (!hasLayerAttr(layer, "dtype") || + getLayerAttr(layer, "dtype").type() != tensorflow::DT_BOOL) // If input is not a train/test flag. + { + netInputsNames.push_back(name); + layer_id[name] = 0; + } + tensorflow::TensorShapeProto shape; + if (hasLayerAttr(layer, "shape")) + shape = getLayerAttr(layer, "shape").shape(); + else if (hasLayerAttr(layer, "_output_shapes")) + { + tensorflow::AttrValue_ListValue list = getLayerAttr(layer, "_output_shapes").list(); + if (list.shape_size()) + shape = list.shape()[0]; + } + if (shape.dim_size()) + { + MatShape dims(shape.dim_size()); + for (int i = 0; i < dims.size(); ++i) + dims[i] = shape.dim(i).size(); + if (dims.size() == 4 && predictedLayout == DATA_LAYOUT_NHWC) + { + std::swap(dims[1], dims[3]); // NHWC->NCWH + std::swap(dims[2], dims[3]); // NCWH->NCHW + if (dims[0] == -1) // It's OK to have undetermined batch size + dims[0] = 1; } - else if (type == "MatMul") + bool hasNeg = false; + for (int i = 0; i < dims.size() && !hasNeg; ++i) { - CV_CheckEQ(num_inputs, 2, ""); + hasNeg = dims[i] < 0; + } + if (!hasNeg) + netInputShapes.push_back(dims); + } +} - // For the object detection networks, TensorFlow Object Detection API - // predicts deltas for bounding boxes in yxYX (ymin, xmin, ymax, xmax) - // order. We can manage it at DetectionOutput layer parsing predictions - // or shuffle last Faster-RCNN's matmul weights. - bool locPredTransposed = hasLayerAttr(layer, "loc_pred_transposed") && - getLayerAttr(layer, "loc_pred_transposed").b(); +void TFImporter::parseSplit(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // TODO: determining axis index remapping by input dimensions order of input blob + // TODO: slicing input may be Const op + // TODO: slicing kernels for convolutions - in current implementation it is impossible + // TODO: add parsing num of slices parameter + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 2, ""); + // num_split + // 1st blob is dims tensor + int axis = getConstBlob(layer, value_id, 0).int_val().Get(0); + if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) + axis = toNCHW(axis); + layerParams.set("axis", axis); + + if (hasLayerAttr(layer, "num_split")) + layerParams.set("num_split", getLayerAttr(layer, "num_split").i()); + + int id = dstNet.addLayer(name, "Slice", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); +} - layerParams.set("bias_term", false); - layerParams.blobs.resize(1); +void TFImporter::parseSlice(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "Slice" + // input: "input_node" + // input: "Slice/begin" + // input: "Slice/size" + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 3, ""); + Mat begins = getTensorContent(getConstBlob(layer, value_id, 1)); + Mat sizes = getTensorContent(getConstBlob(layer, value_id, 2)); + CV_Assert_N(!begins.empty(), !sizes.empty()); + CV_CheckTypeEQ(begins.type(), CV_32SC1, ""); + CV_CheckTypeEQ(sizes.type(), CV_32SC1, ""); + + if (begins.total() == 4 && getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) + { + // Swap NHWC parameters' order to NCHW. + std::swap(*begins.ptr(0, 2), *begins.ptr(0, 3)); + std::swap(*begins.ptr(0, 1), *begins.ptr(0, 2)); + std::swap(*sizes.ptr(0, 2), *sizes.ptr(0, 3)); + std::swap(*sizes.ptr(0, 1), *sizes.ptr(0, 2)); + } + layerParams.set("begin", DictValue::arrayInt((int*)begins.data, begins.total())); + layerParams.set("size", DictValue::arrayInt((int*)sizes.data, sizes.total())); - StrIntVector next_layers = getNextLayers(net, name, "BiasAdd"); // FIXIT Use layers fusion instead - if (next_layers.empty()) - { - next_layers = getNextLayers(net, name, "Add"); - } - if (next_layers.size() == 1) { - layerParams.set("bias_term", true); - layerParams.blobs.resize(2); + int id = dstNet.addLayer(name, "Slice", layerParams); + layer_id[name] = id; - int weights_layer_index = next_layers[0].second; - blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); - ExcludeLayer(net, weights_layer_index, 0, false); - layers_to_ignore.insert(next_layers[0].first); + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); +} - if (locPredTransposed) - { - const int numWeights = layerParams.blobs[1].total(); - float* biasData = reinterpret_cast(layerParams.blobs[1].data); - CV_Assert(numWeights % 4 == 0); - for (int i = 0; i < numWeights; i += 2) - { - std::swap(biasData[i], biasData[i + 1]); - } - } - } +void TFImporter::parseStridedSlice(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 4, ""); + Mat begins = getTensorContent(getConstBlob(layer, value_id, 1)); + Mat ends = getTensorContent(getConstBlob(layer, value_id, 2)); + Mat strides = getTensorContent(getConstBlob(layer, value_id, 3)); + CV_CheckTypeEQ(begins.type(), CV_32SC1, ""); + CV_CheckTypeEQ(ends.type(), CV_32SC1, ""); + CV_CheckTypeEQ(strides.type(), CV_32SC1, ""); + const int num = begins.total(); + CV_Assert_N(num == ends.total(), num == strides.total()); + + int end_mask = getLayerAttr(layer, "end_mask").i(); + for (int i = 0; i < num; ++i) + { + if (ends.at(i) < 0) + ends.at(i) -= 1; + if (end_mask & (1 << i)) + ends.at(i) = -1; + if (strides.at(i) != 1) + CV_Error(Error::StsNotImplemented, + format("StridedSlice with stride %d", strides.at(i))); + } + if (begins.total() == 4 && getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) + { + // Swap NHWC parameters' order to NCHW. + std::swap(begins.at(2), begins.at(3)); + std::swap(begins.at(1), begins.at(2)); + std::swap(ends.at(2), ends.at(3)); + std::swap(ends.at(1), ends.at(2)); + } + layerParams.set("begin", DictValue::arrayInt((int*)begins.data, begins.total())); + layerParams.set("end", DictValue::arrayInt((int*)ends.data, ends.total())); - int kernel_blob_index = -1; - const tensorflow::TensorProto& kernelTensor = getConstBlob(layer, value_id, -1, &kernel_blob_index); - const String kernelTensorName = layer.input(kernel_blob_index); - std::map::iterator sharedWeightsIt = sharedWeights.find(kernelTensorName); - if (sharedWeightsIt == sharedWeights.end()) - { - blobFromTensor(kernelTensor, layerParams.blobs[0]); - releaseTensor(const_cast(&kernelTensor)); - sharedWeights[kernelTensorName] = layerParams.blobs[0]; - } - else - { - layerParams.blobs[0] = sharedWeightsIt->second; - } + int id = dstNet.addLayer(name, "Slice", layerParams); + layer_id[name] = id; - if (kernel_blob_index == 1) { // In this case output is computed by x*W formula - W should be transposed - Mat data = layerParams.blobs[0].t(); - layerParams.blobs[0] = data.clone(); - } + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); +} - layerParams.set("num_output", layerParams.blobs[0].size[0]); - if (locPredTransposed) - { - CV_Assert(layerParams.blobs[0].dims == 2); - for (int i = 0; i < layerParams.blobs[0].size[0]; i += 2) - { - cv::Mat src = layerParams.blobs[0].row(i); - cv::Mat dst = layerParams.blobs[0].row(i + 1); - std::swap_ranges(src.begin(), src.end(), dst.begin()); - } - } +void TFImporter::parseMul(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); - int id = dstNet.addLayer(name, "InnerProduct", layerParams); - layer_id[name] = id; + CV_CheckGT(num_inputs, 0, ""); + int constId = -1; + for(int ii = 0; ii < num_inputs; ++ii) + { + Pin input = parsePin(layer.input(ii)); + if (value_id.find(input.name) != value_id.end()) + { + constId = ii; + break; + } + } + CV_Assert((constId != -1) || (num_inputs == 2)); - // one input only - int input_blob_index = kernel_blob_index == 0 ? 1 : 0; - connect(layer_id, dstNet, parsePin(layer.input(input_blob_index)), id, 0); - data_layouts[name] = DATA_LAYOUT_PLANAR; + if (constId != -1) + { + // Multiplication by constant. + CV_CheckEQ(num_inputs, 2, ""); + Mat scaleMat = getTensorContent(getConstBlob(layer, value_id)); + CV_Assert(scaleMat.type() == CV_32FC1); + if (type == "RealDiv") + { + if (constId == 0) + CV_Error(Error::StsNotImplemented, "Division of constant over variable"); + scaleMat = 1.0f / scaleMat; } - else if (type == "Reshape") + + int id; + if (scaleMat.total() == 1) // is a scalar. { - CV_CheckGT(num_inputs, 0, ""); - Pin inpId = parsePin(layer.input(0)); - DataLayout inpLayout = getDataLayout(layer.input(0), data_layouts); - // There are two possible implementations: reshape an input using - // predefined sizes or use a second input blob as a source of new shape. - if (value_id.find(layer.input(1)) != value_id.end()) + // Try to match with a LeakyRelu: + // node { + // name: "LeakyRelu/mul" + // op: "Mul" + // input: "LeakyRelu/alpha" + // input: "input" + // } + // node { + // name: "LeakyRelu/Maximum" + // op: "Maximum" + // input: "LeakyRelu/mul" + // input: "input" + // } + StrIntVector next_layers = getNextLayers(net, name, "Maximum"); + if (!next_layers.empty()) { - Mat newShape = getTensorContent(getConstBlob(layer, value_id, 1)); - int newShapeSize = newShape.total(); - bool hasSwap = false; - if (newShapeSize == 4 && hasAllOnes(newShape, 0, 2)) - { - // NHWC->NCHW - std::swap(*newShape.ptr(0, 2), *newShape.ptr(0, 3)); - std::swap(*newShape.ptr(0, 1), *newShape.ptr(0, 2)); - hasSwap = true; - } - if (inpLayout == DATA_LAYOUT_NHWC) - { - if (newShapeSize >= 2 || newShape.at(1) == 1) - { - int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - addPermuteLayer(order, name + "/nhwc", inpId); - if (newShapeSize < 4) - { - inpLayout = DATA_LAYOUT_NCHW; - } - else - { - inpLayout = DATA_LAYOUT_NHWC; - } - } - } - layerParams.set("dim", DictValue::arrayInt(newShape.ptr(), newShapeSize)); + int maximumLayerIdx = next_layers[0].second; - int id = dstNet.addLayer(name, "Reshape", layerParams); - layer_id[name] = id; + CV_Assert(net.node(maximumLayerIdx).input_size() == 2); - // one input only - connect(layer_id, dstNet, inpId, id, 0); - inpId = Pin(name); + // The input from the Mul layer can also be at index 1. + int mulInputIdx = (net.node(maximumLayerIdx).input(0) == name) ? 0 : 1; - if ((inpLayout == DATA_LAYOUT_NHWC || inpLayout == DATA_LAYOUT_UNKNOWN || inpLayout == DATA_LAYOUT_PLANAR) && - newShapeSize == 4 && !hasSwap) - { - int order[] = {0, 3, 1, 2}; // Transform back to OpenCV's NCHW. - addPermuteLayer(order, name + "/nchw", inpId); - inpLayout = DATA_LAYOUT_NCHW; - } + ExcludeLayer(net, maximumLayerIdx, mulInputIdx, false); + layers_to_ignore.insert(next_layers[0].first); - data_layouts[name] = newShapeSize == 2 ? DATA_LAYOUT_PLANAR : inpLayout; + layerParams.set("negative_slope", scaleMat.at(0)); + id = dstNet.addLayer(name, "ReLU", layerParams); } else { - int id = dstNet.addLayer(name, "Reshape", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, inpId, id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); - data_layouts[name] = inpLayout; + // Just a multiplication. + layerParams.set("scale", scaleMat.at(0)); + id = dstNet.addLayer(name, "Power", layerParams); } } - else if (type == "Flatten" || type == "Squeeze") + else // is a vector { - CV_CheckGT(num_inputs, 0, ""); - Pin inpId = parsePin(layer.input(0)); - int inpLayout = getDataLayout(layer.input(0), data_layouts); - if (type == "Squeeze") - { - CV_Assert(hasLayerAttr(layer, "squeeze_dims")); - const tensorflow::AttrValue& dims = getLayerAttr(layer, "squeeze_dims"); - std::vector dimsVector(dims.list().i_size()); - for (int i = 0; i < dimsVector.size(); ++i) - dimsVector[i] = dims.list().i(i); - - // Flatten layer can squeeze dimensions range into one. - std::sort(dimsVector.begin(), dimsVector.end()); - for (int i = 1; i < dimsVector.size(); ++i) - { - if (dimsVector[i] != dimsVector[i - 1] + 1) - CV_Error(Error::StsNotImplemented, "Unsupported squeeze configuration"); - } - int start = dimsVector.front() - 1, end = dimsVector.back(); - if (start == -1 && end == 0) // squeeze 0th dimension - { - start = 0; - end = 1; - } - layerParams.set("axis", start); - layerParams.set("end_axis", end); - } - if (inpLayout == DATA_LAYOUT_NHWC) + layerParams.blobs.resize(1, scaleMat); + + StrIntVector next_layers = getNextLayers(net, name, "Add"); + if (!next_layers.empty()) { - LayerParams permLP; - int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - permLP.set("order", DictValue::arrayInt(order, 4)); + layerParams.set("bias_term", true); + layerParams.blobs.resize(2); - std::string permName = name + "/nchw"; - CV_Assert(layer_id.find(permName) == layer_id.end()); - int permId = dstNet.addLayer(permName, "Permute", permLP); - layer_id[permName] = permId; - connect(layer_id, dstNet, inpId, permId, 0); - inpId = Pin(permName); + int weights_layer_index = next_layers[0].second; + blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs.back()); + ExcludeLayer(net, weights_layer_index, 0, false); + layers_to_ignore.insert(next_layers[0].first); } - int id = dstNet.addLayer(name, "Flatten", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, inpId, id, 0); - data_layouts[name] = DATA_LAYOUT_PLANAR; + + if (hasLayerAttr(layer, "axis")) + layerParams.set("axis", getLayerAttr(layer, "axis").i()); + + id = dstNet.addLayer(name, "Scale", layerParams); } - else if (type == "Transpose") + layer_id[name] = id; + + Pin inp0 = parsePin(layer.input(0)); + if (layer_id.find(inp0.name) != layer_id.end()) + // First operand is a constant. + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + else + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); + } + else + { + // Check if all the inputs have the same shape. + bool equalInpShapes = true; + bool isShapeOnes = false; + MatShape outShape0; + for (int ii = 0; ii < num_inputs && !netInputShapes.empty(); ii++) { - CV_CheckGT(num_inputs, 0, ""); - Mat perm = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_Assert(perm.type() == CV_32SC1); - int* permData = (int*)perm.data; - if (perm.total() == 4) + Pin pin = parsePin(layer.input(ii)); + int inpId = layer_id.find(pin.name)->second; + + // Get input shape + MatShape outShape; + std::vector inpShapes, outShapes; + dstNet.getLayerShapes(netInputShapes, inpId, inpShapes, outShapes); + CV_CheckGT(static_cast(outShapes.size()), pin.blobIndex, ""); + outShape = outShapes[pin.blobIndex]; + + if (ii == 0) { - // Only NHWC <-> NCHW permutations are allowed. OpenCV is always - // keep NCHW layout this way. - int inpLayout = getDataLayout(layer.input(0), data_layouts); - std::string type = "Identity"; - if (inpLayout == DATA_LAYOUT_NHWC) - { - if (permData[0] == 0 && permData[1] == 3 && permData[2] == 1 && permData[3] == 2) - { - // in TensorFlow: NHWC->NCHW - // in OpenCV: NCHW->NCHW - data_layouts[name] = DATA_LAYOUT_NCHW; - } - else if (permData[0] == 0 && permData[1] == 1 && permData[2] == 2 && permData[3] == 3) - { - // in TensorFlow: NHWC->NHWC - // in OpenCV: NCHW->NCHW - data_layouts[name] = DATA_LAYOUT_NHWC; - } - else if (permData[0] == 0 && permData[1] == 3 && permData[2] == 2 && permData[3] == 1) - { - // in TensorFlow: NHWC->NCWH - // in OpenCV: NCHW->NCWH - int permData[] = {0, 1, 3, 2}; - layerParams.set("order", DictValue::arrayInt(permData, perm.total())); - data_layouts[name] = DATA_LAYOUT_NCHW; // we keep track NCHW because channels position only matters - type = "Permute"; - } - else - CV_Error(Error::StsParseError, "Only NHWC <-> NCHW permutations are allowed."); - } - else if (inpLayout == DATA_LAYOUT_NCHW) - { - if (permData[0] == 0 && permData[1] == 2 && permData[2] == 3 && permData[3] == 1) - { - // in TensorFlow: NCHW->NHWC - // in OpenCV: NCHW->NCHW - data_layouts[name] = DATA_LAYOUT_NHWC; - } - else if (permData[0] == 0 && permData[1] == 1 && permData[2] == 2 && permData[3] == 3) - { - // in TensorFlow: NCHW->NCHW - // in OpenCV: NCHW->NCHW - data_layouts[name] = DATA_LAYOUT_NCHW; - } - else - CV_Error(Error::StsParseError, "Only NHWC <-> NCHW permutations are allowed."); - } - int id = dstNet.addLayer(name, type, layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + outShape0 = outShape; } - else + else if (outShape != outShape0) { - layerParams.set("order", DictValue::arrayInt(permData, perm.total())); - - int id = dstNet.addLayer(name, "Permute", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - data_layouts[name] = DATA_LAYOUT_UNKNOWN; + equalInpShapes = false; + isShapeOnes = isAllOnes(outShape, 2, outShape.size()) || + isAllOnes(outShape0, 2, outShape0.size()); + break; } } - else if (type == "Const") + + int id; + if (equalInpShapes || netInputShapes.empty() || (!equalInpShapes && isShapeOnes)) { + layerParams.set("operation", type == "RealDiv" ? "div" : "prod"); + id = dstNet.addLayer(name, "Eltwise", layerParams); } - else if (type == "LRN") + else { - CV_CheckGT(num_inputs, 0, ""); - if(hasLayerAttr(layer, "alpha")) { - layerParams.set("alpha", getLayerAttr(layer, "alpha").f()); - } - if(hasLayerAttr(layer, "beta")) { - layerParams.set("beta", getLayerAttr(layer, "beta").f()); - } - if(hasLayerAttr(layer, "depth_radius")) { - int radius = (int)getLayerAttr(layer, "depth_radius").i(); - layerParams.set("local_size", 2*radius + 1); - } - if(hasLayerAttr(layer, "bias")) { - layerParams.set("bias", getLayerAttr(layer, "bias").f()); - } - layerParams.set("norm_by_size", false); + if (type == "RealDiv") + CV_Error(Error::StsNotImplemented, "Division of non equal tensors"); + id = dstNet.addLayer(name, "Scale", layerParams); + } - int id = dstNet.addLayer(name, "LRN", layerParams); - layer_id[name] = id; + layer_id[name] = id; - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else if (type == "Concat" || type == "ConcatV2") + for (int ii = 0; ii < num_inputs; ii++) { - CV_CheckGT(num_inputs, 0, ""); - int axisId = (type == "Concat" ? 0 : num_inputs - 1); - int axis = getConstBlob(layer, value_id, axisId).int_val().Get(0); + Pin inp = parsePin(layer.input(ii)); + if (layer_id.find(inp.name) == layer_id.end()) + CV_Error(Error::StsError, "Input layer not found: " + inp.name); + connect(layer_id, dstNet, inp, id, ii); + } + } +} - if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) - axis = toNCHW(axis); - else if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NDHWC) - axis = toNCDHW(axis); - layerParams.set("axis", axis); +void TFImporter::parseFusedBatchNorm(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "FusedBatchNorm" + // input: "input" + // input: "BatchNorm/gamma" + // input: "BatchNorm/beta" + // input: "BatchNorm/moving_mean" + // input: "BatchNorm/moving_variance" - // input(0) or input(n-1) is concat_dim - int from = (type == "Concat" ? 1 : 0); - int to = (type == "Concat" ? num_inputs : num_inputs - 1); + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); - for (int ii = from; ii < to; ii++) - { - Pin inp = parsePin(layer.input(ii)); - if (layer_id.find(inp.name) == layer_id.end()) - { - // There are constant inputs. - LayerParams lp; - lp.name = inp.name; - lp.type = "Const"; - lp.blobs.resize(1); - blobFromTensor(getConstBlob(layer, value_id, ii), lp.blobs.back()); - CV_Assert_N(!lp.blobs[0].empty(), lp.blobs[0].type() == CV_32F); - - int constInpId = dstNet.addLayer(lp.name, lp.type, lp); - layer_id[lp.name] = constInpId; - } - } + CV_CheckEQ(num_inputs, 5, "Expected gamma, beta, mean and std"); + Pin inpId = parsePin(layer.input(0)); - int id = dstNet.addLayer(name, "Concat", layerParams); - layer_id[name] = id; + bool isTraining = hasLayerAttr(layer, "is_training") && getLayerAttr(layer, "is_training").b(); - for (int ii = from; ii < to; ii++) - { - Pin inp = parsePin(layer.input(ii)); - if (layer_id.find(inp.name) == layer_id.end()) - CV_Error(Error::StsError, "Input layer not found: " + inp.name); - connect(layer_id, dstNet, inp, id, ii - from); - } - } - else if (type == "MaxPool" || type == "MaxPool3D") - { - CV_CheckGT(num_inputs, 0, ""); - layerParams.set("pool", "max"); + layerParams.blobs.resize(2); - setKSize(layerParams, layer); - setStrides(layerParams, layer); - setPadding(layerParams, layer); - // Test_TensorFlow_nets.EAST_text_detection/1, NGRAPH/CPU - layerParams.set("ceil_mode", false); + const tensorflow::TensorProto& gammaTensor = getConstBlob(layer, value_id, 1); + if (!gammaTensor.tensor_content().empty()) + { + layerParams.blobs.resize(layerParams.blobs.size() + 1); + layerParams.set("has_weight", true); + blobFromTensor(gammaTensor, layerParams.blobs.back()); + } + else + layerParams.set("has_weight", false); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; + const tensorflow::TensorProto& betaTensor = getConstBlob(layer, value_id, 2); + if (!betaTensor.tensor_content().empty()) + { + layerParams.blobs.resize(layerParams.blobs.size() + 1); + layerParams.set("has_bias", true); + blobFromTensor(betaTensor, layerParams.blobs.back()); + } + else + layerParams.set("has_bias", false); - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else if (type == "AvgPool" || type == "AvgPool3D") - { - CV_CheckGT(num_inputs, 0, ""); - layerParams.set("pool", "ave"); - layerParams.set("ave_pool_padded_area", false); - setKSize(layerParams, layer); - setStrides(layerParams, layer); - setPadding(layerParams, layer); + Mat mean, std; + if (isTraining) + { + if (layerParams.blobs.size() == 2) + CV_Error(Error::StsNotImplemented, "Cannot determine number " + "of parameters for batch normalization layer."); + mean = Mat::zeros(1, layerParams.blobs[2].total(), CV_32F); + std = Mat::ones(1, layerParams.blobs[2].total(), CV_32F); + + // Add an extra layer: Mean-Variance normalization + LayerParams mvnParams; + std::string mvnName = name + "/MVN"; + CV_Assert(layer_id.find(mvnName) == layer_id.end()); + int mvnId = dstNet.addLayer(mvnName, "MVN", mvnParams); + layer_id[mvnName] = mvnId; + connect(layer_id, dstNet, inpId, mvnId, 0); + inpId = Pin(mvnName); + } + else + { + blobFromTensor(getConstBlob(layer, value_id, 3), mean); + blobFromTensor(getConstBlob(layer, value_id, 4), std); + } + layerParams.blobs[0] = mean; + layerParams.blobs[1] = std; - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; + if (hasLayerAttr(layer, "epsilon")) + layerParams.set("eps", getLayerAttr(layer, "epsilon").f()); - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else if (type == "MaxPoolGrad") - { - CV_CheckEQ(num_inputs, 3, ""); + int id = dstNet.addLayer(name, "BatchNorm", layerParams); + layer_id[name] = id; - layerParams.set("pool_k_h", 0); - layerParams.set("pool_k_w", 0); - layerParams.set("pool_stride_h", 0); - layerParams.set("pool_stride_w", 0); - layerParams.set("pool_pad_h", 0); - layerParams.set("pool_pad_w", 0); + // one input only + connect(layer_id, dstNet, inpId, id, 0); +} - int id = dstNet.addLayer(name, "MaxUnpool", layerParams); - layer_id[name] = id; +void TFImporter::parseConv2DBackpropInput(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "Conv2DBackpropInput" + // input: "conv2d_transpose/output_shape" + // input: "weights" + // input: "input" - connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1) + ":1"), id, 1); - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 2); - } - else if (type == "Placeholder") - { - if (!hasLayerAttr(layer, "dtype") || - getLayerAttr(layer, "dtype").type() != tensorflow::DT_BOOL) // If input is not a train/test flag. - { - netInputsNames.push_back(name); - layer_id[name] = 0; - } - tensorflow::TensorShapeProto shape; - if (hasLayerAttr(layer, "shape")) - shape = getLayerAttr(layer, "shape").shape(); - else if (hasLayerAttr(layer, "_output_shapes")) - { - tensorflow::AttrValue_ListValue list = getLayerAttr(layer, "_output_shapes").list(); - if (list.shape_size()) - shape = list.shape()[0]; - } - if (shape.dim_size()) - { - MatShape dims(shape.dim_size()); - for (int i = 0; i < dims.size(); ++i) - dims[i] = shape.dim(i).size(); - if (dims.size() == 4 && predictedLayout == DATA_LAYOUT_NHWC) - { - std::swap(dims[1], dims[3]); // NHWC->NCWH - std::swap(dims[2], dims[3]); // NCWH->NCHW - if (dims[0] == -1) // It's OK to have undetermined batch size - dims[0] = 1; - } - bool hasNeg = false; - for (int i = 0; i < dims.size() && !hasNeg; ++i) - { - hasNeg = dims[i] < 0; - } - if (!hasNeg) - netInputShapes.push_back(dims); - } - } - else if (type == "Split") { - // TODO: determining axis index remapping by input dimensions order of input blob - // TODO: slicing input may be Const op - // TODO: slicing kernels for convolutions - in current implementation it is impossible - // TODO: add parsing num of slices parameter - CV_CheckEQ(num_inputs, 2, ""); - // num_split - // 1st blob is dims tensor - int axis = getConstBlob(layer, value_id, 0).int_val().Get(0); - if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) - axis = toNCHW(axis); - layerParams.set("axis", axis); - - if (hasLayerAttr(layer, "num_split")) - layerParams.set("num_split", getLayerAttr(layer, "num_split").i()); - - int id = dstNet.addLayer(name, "Slice", layerParams); - layer_id[name] = id; + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); - // one input only - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); - } - else if (type == "Slice") - { - // op: "Slice" - // input: "input_node" - // input: "Slice/begin" - // input: "Slice/size" - CV_CheckEQ(num_inputs, 3, ""); - Mat begins = getTensorContent(getConstBlob(layer, value_id, 1)); - Mat sizes = getTensorContent(getConstBlob(layer, value_id, 2)); - CV_Assert_N(!begins.empty(), !sizes.empty()); - CV_CheckTypeEQ(begins.type(), CV_32SC1, ""); - CV_CheckTypeEQ(sizes.type(), CV_32SC1, ""); - - if (begins.total() == 4 && getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) - { - // Swap NHWC parameters' order to NCHW. - std::swap(*begins.ptr(0, 2), *begins.ptr(0, 3)); - std::swap(*begins.ptr(0, 1), *begins.ptr(0, 2)); - std::swap(*sizes.ptr(0, 2), *sizes.ptr(0, 3)); - std::swap(*sizes.ptr(0, 1), *sizes.ptr(0, 2)); - } - layerParams.set("begin", DictValue::arrayInt((int*)begins.data, begins.total())); - layerParams.set("size", DictValue::arrayInt((int*)sizes.data, sizes.total())); + CV_CheckEQ(num_inputs, 3, "Expected output shape, weights and input nodes"); - int id = dstNet.addLayer(name, "Slice", layerParams); - layer_id[name] = id; + layerParams.set("bias_term", false); + layerParams.blobs.resize(1); - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else if (type == "StridedSlice") + StrIntVector next_layers = getNextLayers(net, name, "BiasAdd"); + if (next_layers.size() == 1) + { + layerParams.set("bias_term", true); + layerParams.blobs.resize(2); + + int weights_layer_index = next_layers[0].second; + + blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); + ExcludeLayer(net, weights_layer_index, 0, false); + layers_to_ignore.insert(next_layers[0].first); + } + + kernelFromTensor(getConstBlob(layer, value_id, 1), layerParams.blobs[0]); + + const int* kshape = layerParams.blobs[0].size.p; + const int kernelH = kshape[2]; + const int kernelW = kshape[3]; + layerParams.set("kernel_h", kernelH); + layerParams.set("kernel_w", kernelW); + layerParams.set("num_output", kshape[1]); + + setStrides(layerParams, layer); + setPadding(layerParams, layer); + + // For convolution layer, output shape computes as + // o = 1 + (i - k + 2*p) / s + // i - input size, o - output size, k - kernel size, p - pad, s - stride + // In TensorFlow, p == 0 is padMode == 'VALID' or p == (k - 1) / 2 + // considering that k is odd. + // SAME: o = 1 + (i - 1) / s + // VALID: o = 1 + i / s + // Deconvolution's layer output shape computes as + // SAME: o = 1 + (i - 1)*s + // VALID: o = (i - 1)*s + // If output_shape differs from formulas above then adjust padding is applied. + + const int strideY = layerParams.get("stride_h"); + const int strideX = layerParams.get("stride_w"); + Mat outShape = getTensorContent(getConstBlob(layer, value_id, 0)); + const int outH = outShape.at(1); + const int outW = outShape.at(2); + if (layerParams.get("pad_mode") == "SAME") + { + layerParams.set("adj_w", (outW - 1) % strideX); + layerParams.set("adj_h", (outH - 1) % strideY); + } + else if (layerParams.get("pad_mode") == "VALID") + { + layerParams.set("adj_w", (outW - kernelW) % strideX); + layerParams.set("adj_h", (outH - kernelH) % strideY); + } + int id = dstNet.addLayer(name, "Deconvolution", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); +} + +void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "BlockLSTM" + // input: "lstm_block_wrapper/ToInt64/x" (ignore, number of time stamps) + // input: "input" + // input: "lstm_block_wrapper/zeros" (ignore) + // input: "lstm_block_wrapper/zeros" (ignore) + // input: "lstm_block_wrapper/kernel" + // input: "lstm_block_wrapper/w_i_diag" + // input: "lstm_block_wrapper/w_f_diag" + // input: "lstm_block_wrapper/w_o_diag" + // input: "lstm_block_wrapper/bias" + + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 9, "Unexpected number of input nodes"); + + if (hasLayerAttr(layer, "forget_bias")) + layerParams.set("forget_bias", getLayerAttr(layer, "forget_bias").f()); + + if (hasLayerAttr(layer, "forget_bias")) + { + float cellClip = getLayerAttr(layer, "cell_clip").f(); + // Cell clip disabled if it's negative. + if (cellClip >= 0) { - CV_CheckEQ(num_inputs, 4, ""); - Mat begins = getTensorContent(getConstBlob(layer, value_id, 1)); - Mat ends = getTensorContent(getConstBlob(layer, value_id, 2)); - Mat strides = getTensorContent(getConstBlob(layer, value_id, 3)); - CV_CheckTypeEQ(begins.type(), CV_32SC1, ""); - CV_CheckTypeEQ(ends.type(), CV_32SC1, ""); - CV_CheckTypeEQ(strides.type(), CV_32SC1, ""); - const int num = begins.total(); - CV_Assert_N(num == ends.total(), num == strides.total()); - - int end_mask = getLayerAttr(layer, "end_mask").i(); - for (int i = 0; i < num; ++i) - { - if (ends.at(i) < 0) - ends.at(i) -= 1; - if (end_mask & (1 << i)) - ends.at(i) = -1; - if (strides.at(i) != 1) - CV_Error(Error::StsNotImplemented, - format("StridedSlice with stride %d", strides.at(i))); - } - if (begins.total() == 4 && getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) - { - // Swap NHWC parameters' order to NCHW. - std::swap(begins.at(2), begins.at(3)); - std::swap(begins.at(1), begins.at(2)); - std::swap(ends.at(2), ends.at(3)); - std::swap(ends.at(1), ends.at(2)); - } - layerParams.set("begin", DictValue::arrayInt((int*)begins.data, begins.total())); - layerParams.set("end", DictValue::arrayInt((int*)ends.data, ends.total())); + layerParams.set("use_cell_clip", true); + layerParams.set("cell_clip", cellClip); + } + } - int id = dstNet.addLayer(name, "Slice", layerParams); - layer_id[name] = id; + Mat W, Wh, Wx, b; + blobFromTensor(getConstBlob(layer, value_id, 4), W); + blobFromTensor(getConstBlob(layer, value_id, 8), b); + const int outSize = W.cols / 4; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + // IGFO->IFOG + float* weightData = (float*)W.data; + for (int i = 0; i < W.rows; ++i) + for (int j = 0; j < outSize; ++j) + { + std::swap(weightData[i * W.cols + 1 * outSize + j], + weightData[i * W.cols + 2 * outSize + j]); + std::swap(weightData[i * W.cols + 2 * outSize + j], + weightData[i * W.cols + 3 * outSize + j]); } - else if (type == "Mul" || type == "RealDiv") + Wx = W.rowRange(0, W.rows - outSize).t(); + Wh = W.rowRange(W.rows - outSize, W.rows).t(); + + layerParams.blobs.resize(3); + layerParams.blobs[0] = Wh; + layerParams.blobs[1] = Wx; + layerParams.blobs[2] = b; + + if (hasLayerAttr(layer, "use_peephole")) + { + bool usePeephole = getLayerAttr(layer, "use_peephole").b(); + if (usePeephole) { - CV_CheckGT(num_inputs, 0, ""); - int constId = -1; - for(int ii = 0; ii < num_inputs; ++ii) + layerParams.set("use_peephole", true); + layerParams.blobs.resize(6); + for (int i = 0; i < 3; ++i) { - Pin input = parsePin(layer.input(ii)); - if (value_id.find(input.name) != value_id.end()) - { - constId = ii; - break; - } + Mat w; + blobFromTensor(getConstBlob(layer, value_id, 5 + i), w); + w = w.reshape(1, w.total()); // Single column. + w = Mat::diag(w); // Make a diagonal matrix. + layerParams.blobs[3 + i] = w; } - CV_Assert((constId != -1) || (num_inputs == 2)); + } + } - if (constId != -1) - { - // Multiplication by constant. - CV_CheckEQ(num_inputs, 2, ""); - Mat scaleMat = getTensorContent(getConstBlob(layer, value_id)); - CV_Assert(scaleMat.type() == CV_32FC1); - if (type == "RealDiv") - { - if (constId == 0) - CV_Error(Error::StsNotImplemented, "Division of constant over variable"); - scaleMat = 1.0f / scaleMat; - } + int id = dstNet.addLayer(name, "LSTM", layerParams); + layer_id[name] = id; - int id; - if (scaleMat.total() == 1) // is a scalar. - { - // Try to match with a LeakyRelu: - // node { - // name: "LeakyRelu/mul" - // op: "Mul" - // input: "LeakyRelu/alpha" - // input: "input" - // } - // node { - // name: "LeakyRelu/Maximum" - // op: "Maximum" - // input: "LeakyRelu/mul" - // input: "input" - // } - StrIntVector next_layers = getNextLayers(net, name, "Maximum"); - if (!next_layers.empty()) - { - int maximumLayerIdx = next_layers[0].second; + // one input only + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); + data_layouts[name] = DATA_LAYOUT_UNKNOWN; +} - CV_Assert(net.node(maximumLayerIdx).input_size() == 2); +void TFImporter::parseResize(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer_, LayerParams& layerParams) +{ + tensorflow::NodeDef layer = layer_; + std::string name = layer.name(); + const std::string& type = layer.op(); + int num_inputs = layer.input_size(); - // The input from the Mul layer can also be at index 1. - int mulInputIdx = (net.node(maximumLayerIdx).input(0) == name) ? 0 : 1; + CV_CheckGT(num_inputs, 0, ""); + std::string convWeights = ""; + if (type == "FusedResizeAndPadConv2D") + { + // input: "mul_1" + // input: "decoder/ResizeBilinear/size" + // input: "decoder/decoder_conv0/Conv2D_dummy_paddings" + // input: "decoder/decoder_conv0/weights" + CV_CheckEQ(num_inputs, 4, "Number of input for FusedResizeAndPadConv2D"); - ExcludeLayer(net, maximumLayerIdx, mulInputIdx, false); - layers_to_ignore.insert(next_layers[0].first); + Mat paddings = getTensorContent(getConstBlob(layer, value_id, 2)); + CV_CheckEQ(countNonZero(paddings), 0, "Unsupported mode"); - layerParams.set("negative_slope", scaleMat.at(0)); - id = dstNet.addLayer(name, "ReLU", layerParams); - } - else - { - // Just a multiplication. - layerParams.set("scale", scaleMat.at(0)); - id = dstNet.addLayer(name, "Power", layerParams); - } - } - else // is a vector - { - layerParams.blobs.resize(1, scaleMat); - - StrIntVector next_layers = getNextLayers(net, name, "Add"); - if (!next_layers.empty()) - { - layerParams.set("bias_term", true); - layerParams.blobs.resize(2); - - int weights_layer_index = next_layers[0].second; - blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs.back()); - ExcludeLayer(net, weights_layer_index, 0, false); - layers_to_ignore.insert(next_layers[0].first); - } + convWeights = layer.input(3); + layer.mutable_input()->DeleteSubrange(2, 2); // FIXIT do NOT modify input model + num_inputs = layer.input_size(); + name = name + "/resize"; - if (hasLayerAttr(layer, "axis")) - layerParams.set("axis", getLayerAttr(layer, "axis").i()); + if (hasLayerAttr(layer, "resize_align_corners")) + { + // FIXIT do NOT modify input model + layer.mutable_attr()->insert( + ::google::protobuf::MapPair("align_corners", + getLayerAttr(layer, "resize_align_corners"))); + } + } + if (num_inputs == 2) + { + Mat outSize = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_CheckTypeEQ(outSize.type(), CV_32SC1, ""); CV_CheckEQ(outSize.total(), (size_t)2, ""); + layerParams.set("height", outSize.at(0, 0)); + layerParams.set("width", outSize.at(0, 1)); + } + else if (num_inputs == 3) + { + Mat factorHeight = getTensorContent(getConstBlob(layer, value_id, 1)); + Mat factorWidth = getTensorContent(getConstBlob(layer, value_id, 2)); + factorHeight.convertTo(factorHeight, CV_32F); + factorWidth.convertTo(factorWidth, CV_32F); + layerParams.set("zoom_factor_x", factorWidth.at(0)); + layerParams.set("zoom_factor_y", factorHeight.at(0)); + } + else + CV_Check(num_inputs, num_inputs == 2 || num_inputs == 3, ""); - id = dstNet.addLayer(name, "Scale", layerParams); - } - layer_id[name] = id; + if (type == "ResizeNearestNeighbor") + layerParams.set("interpolation", "nearest"); + else + layerParams.set("interpolation", "bilinear"); - Pin inp0 = parsePin(layer.input(0)); - if (layer_id.find(inp0.name) != layer_id.end()) - // First operand is a constant. - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - else - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); - } - else - { - // Check if all the inputs have the same shape. - bool equalInpShapes = true; - bool isShapeOnes = false; - MatShape outShape0; - for (int ii = 0; ii < num_inputs && !netInputShapes.empty(); ii++) - { - Pin pin = parsePin(layer.input(ii)); - int inpId = layer_id.find(pin.name)->second; + if (hasLayerAttr(layer, "align_corners")) + layerParams.set("align_corners", getLayerAttr(layer, "align_corners").b()); - // Get input shape - MatShape outShape; - std::vector inpShapes, outShapes; - dstNet.getLayerShapes(netInputShapes, inpId, inpShapes, outShapes); - CV_CheckGT(static_cast(outShapes.size()), pin.blobIndex, ""); - outShape = outShapes[pin.blobIndex]; + if (hasLayerAttr(layer, "half_pixel_centers")) + layerParams.set("half_pixel_centers", getLayerAttr(layer, "half_pixel_centers").b()); - if (ii == 0) - { - outShape0 = outShape; - } - else if (outShape != outShape0) - { - equalInpShapes = false; - isShapeOnes = isAllOnes(outShape, 2, outShape.size()) || - isAllOnes(outShape0, 2, outShape0.size()); - break; - } - } + int id = dstNet.addLayer(name, "Resize", layerParams); + layer_id[name] = id; - int id; - if (equalInpShapes || netInputShapes.empty() || (!equalInpShapes && isShapeOnes)) - { - layerParams.set("operation", type == "RealDiv" ? "div" : "prod"); - id = dstNet.addLayer(name, "Eltwise", layerParams); - } - else - { - if (type == "RealDiv") - CV_Error(Error::StsNotImplemented, "Division of non equal tensors"); - id = dstNet.addLayer(name, "Scale", layerParams); - } + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - layer_id[name] = id; + // Step back to add convolution + if (type == "FusedResizeAndPadConv2D") + { + tensorflow::NodeDef conv = layer_; + conv.clear_input(); + conv.add_input(name); + conv.add_input(convWeights); + conv.set_op("Conv2D"); + parseNode(conv); + } +} - for (int ii = 0; ii < num_inputs; ii++) - { - Pin inp = parsePin(layer.input(ii)); - if (layer_id.find(inp.name) == layer_id.end()) - CV_Error(Error::StsError, "Input layer not found: " + inp.name); - connect(layer_id, dstNet, inp, id, ii); - } - } - } - else if (type == "FusedBatchNorm" || type == "FusedBatchNormV3") - { - // op: "FusedBatchNorm" - // input: "input" - // input: "BatchNorm/gamma" - // input: "BatchNorm/beta" - // input: "BatchNorm/moving_mean" - // input: "BatchNorm/moving_variance" - CV_CheckEQ(num_inputs, 5, "Expected gamma, beta, mean and std"); - Pin inpId = parsePin(layer.input(0)); +void TFImporter::parseL2Normalize(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "L2Normalize" + // input: "input" + // input: "reduction_indices" (axis) - bool isTraining = hasLayerAttr(layer, "is_training") && getLayerAttr(layer, "is_training").b(); + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); - layerParams.blobs.resize(2); + CV_CheckEQ(num_inputs, 2, ""); + Mat reductionIndices = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_Assert(reductionIndices.type() == CV_32SC1); - const tensorflow::TensorProto& gammaTensor = getConstBlob(layer, value_id, 1); - if (!gammaTensor.tensor_content().empty()) - { - layerParams.blobs.resize(layerParams.blobs.size() + 1); - layerParams.set("has_weight", true); - blobFromTensor(gammaTensor, layerParams.blobs.back()); - } - else - layerParams.set("has_weight", false); - - const tensorflow::TensorProto& betaTensor = getConstBlob(layer, value_id, 2); - if (!betaTensor.tensor_content().empty()) - { - layerParams.blobs.resize(layerParams.blobs.size() + 1); - layerParams.set("has_bias", true); - blobFromTensor(betaTensor, layerParams.blobs.back()); - } - else - layerParams.set("has_bias", false); - - Mat mean, std; - if (isTraining) - { - if (layerParams.blobs.size() == 2) - CV_Error(Error::StsNotImplemented, "Cannot determine number " - "of parameters for batch normalization layer."); - mean = Mat::zeros(1, layerParams.blobs[2].total(), CV_32F); - std = Mat::ones(1, layerParams.blobs[2].total(), CV_32F); - - // Add an extra layer: Mean-Variance normalization - LayerParams mvnParams; - std::string mvnName = name + "/MVN"; - CV_Assert(layer_id.find(mvnName) == layer_id.end()); - int mvnId = dstNet.addLayer(mvnName, "MVN", mvnParams); - layer_id[mvnName] = mvnId; - connect(layer_id, dstNet, inpId, mvnId, 0); - inpId = Pin(mvnName); - } - else - { - blobFromTensor(getConstBlob(layer, value_id, 3), mean); - blobFromTensor(getConstBlob(layer, value_id, 4), std); - } - layerParams.blobs[0] = mean; - layerParams.blobs[1] = std; + const int numAxes = reductionIndices.total(); + if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) + for (int i = 0; i < numAxes; ++i) + reductionIndices.at(i) = toNCHW(reductionIndices.at(i)); - if (hasLayerAttr(layer, "epsilon")) - layerParams.set("eps", getLayerAttr(layer, "epsilon").f()); + cv::sort(reductionIndices, reductionIndices, SORT_ASCENDING); + for (int i = 1; i < numAxes; ++i) + { + CV_Assert(reductionIndices.at(i) == reductionIndices.at(i - 1) + 1); + // Axes have the same sign. + CV_Assert(reductionIndices.at(i) * reductionIndices.at(i - 1) >= 0); + } + layerParams.set("start_axis", reductionIndices.at(0)); + layerParams.set("end_axis", reductionIndices.at(numAxes - 1)); - int id = dstNet.addLayer(name, "BatchNorm", layerParams); - layer_id[name] = id; + int id = dstNet.addLayer(name, "Normalize", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); +} - // one input only - connect(layer_id, dstNet, inpId, id, 0); - } - else if (type == "Conv2DBackpropInput") +void TFImporter::parsePriorBox(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 2, ""); + if (hasLayerAttr(layer, "min_size")) + layerParams.set("min_size", getLayerAttr(layer, "min_size").i()); + if (hasLayerAttr(layer, "max_size")) + layerParams.set("max_size", getLayerAttr(layer, "max_size").i()); + if (hasLayerAttr(layer, "flip")) + layerParams.set("flip", getLayerAttr(layer, "flip").b()); + if (hasLayerAttr(layer, "clip")) + layerParams.set("clip", getLayerAttr(layer, "clip").b()); + if (hasLayerAttr(layer, "offset")) + layerParams.set("offset", getLayerAttr(layer, "offset").f()); + if (hasLayerAttr(layer, "step")) + layerParams.set("step", getLayerAttr(layer, "step").f()); + + const std::string paramNames[] = {"variance", "aspect_ratio", "scales", + "width", "height"}; + for (int i = 0; i < 5; ++i) + { + if (hasLayerAttr(layer, paramNames[i])) { - // op: "Conv2DBackpropInput" - // input: "conv2d_transpose/output_shape" - // input: "weights" - // input: "input" - CV_CheckEQ(num_inputs, 3, "Expected output shape, weights and input nodes"); + Mat values = getTensorContent(getLayerAttr(layer, paramNames[i]).tensor()); + layerParams.set(paramNames[i], + DictValue::arrayReal((float*)values.data, values.total())); + } + } + int id = dstNet.addLayer(name, "PriorBox", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); + data_layouts[name] = DATA_LAYOUT_UNKNOWN; +} - layerParams.set("bias_term", false); - layerParams.blobs.resize(1); +void TFImporter::parseSoftmax(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); - StrIntVector next_layers = getNextLayers(net, name, "BiasAdd"); - if (next_layers.size() == 1) - { - layerParams.set("bias_term", true); - layerParams.blobs.resize(2); + CV_CheckGT(num_inputs, 0, ""); + if (hasLayerAttr(layer, "axis")) + layerParams.set("axis", getLayerAttr(layer, "axis").i()); - int weights_layer_index = next_layers[0].second; + int id = dstNet.addLayer(name, "Softmax", layerParams); + layer_id[name] = id; + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} - blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); - ExcludeLayer(net, weights_layer_index, 0, false); - layers_to_ignore.insert(next_layers[0].first); - } +void TFImporter::parseCropAndResize(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "CropAndResize" + // input: "input" + // input: "boxes" + // input: "sizes" - kernelFromTensor(getConstBlob(layer, value_id, 1), layerParams.blobs[0]); - - const int* kshape = layerParams.blobs[0].size.p; - const int kernelH = kshape[2]; - const int kernelW = kshape[3]; - layerParams.set("kernel_h", kernelH); - layerParams.set("kernel_w", kernelW); - layerParams.set("num_output", kshape[1]); - - setStrides(layerParams, layer); - setPadding(layerParams, layer); - - // For convolution layer, output shape computes as - // o = 1 + (i - k + 2*p) / s - // i - input size, o - output size, k - kernel size, p - pad, s - stride - // In TensorFlow, p == 0 is padMode == 'VALID' or p == (k - 1) / 2 - // considering that k is odd. - // SAME: o = 1 + (i - 1) / s - // VALID: o = 1 + i / s - // Deconvolution's layer output shape computes as - // SAME: o = 1 + (i - 1)*s - // VALID: o = (i - 1)*s - // If output_shape differs from formulas above then adjust padding is applied. - - const int strideY = layerParams.get("stride_h"); - const int strideX = layerParams.get("stride_w"); - Mat outShape = getTensorContent(getConstBlob(layer, value_id, 0)); - const int outH = outShape.at(1); - const int outW = outShape.at(2); - if (layerParams.get("pad_mode") == "SAME") - { - layerParams.set("adj_w", (outW - 1) % strideX); - layerParams.set("adj_h", (outH - 1) % strideY); - } - else if (layerParams.get("pad_mode") == "VALID") - { - layerParams.set("adj_w", (outW - kernelW) % strideX); - layerParams.set("adj_h", (outH - kernelH) % strideY); - } - int id = dstNet.addLayer(name, "Deconvolution", layerParams); - layer_id[name] = id; + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + CV_CheckEQ(num_inputs, 3, ""); - // one input only - connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); - } - else if (type == "BlockLSTM") - { - // op: "BlockLSTM" - // input: "lstm_block_wrapper/ToInt64/x" (ignore, number of time stamps) - // input: "input" - // input: "lstm_block_wrapper/zeros" (ignore) - // input: "lstm_block_wrapper/zeros" (ignore) - // input: "lstm_block_wrapper/kernel" - // input: "lstm_block_wrapper/w_i_diag" - // input: "lstm_block_wrapper/w_f_diag" - // input: "lstm_block_wrapper/w_o_diag" - // input: "lstm_block_wrapper/bias" - CV_CheckEQ(num_inputs, 9, "Unexpected number of input nodes"); - - if (hasLayerAttr(layer, "forget_bias")) - layerParams.set("forget_bias", getLayerAttr(layer, "forget_bias").f()); - - if (hasLayerAttr(layer, "forget_bias")) - { - float cellClip = getLayerAttr(layer, "cell_clip").f(); - // Cell clip disabled if it's negative. - if (cellClip >= 0) - { - layerParams.set("use_cell_clip", true); - layerParams.set("cell_clip", cellClip); - } - } + Mat cropSize = getTensorContent(getConstBlob(layer, value_id, 2)); + CV_CheckTypeEQ(cropSize.type(), CV_32SC1, ""); CV_CheckEQ(cropSize.total(), (size_t)2, ""); - Mat W, Wh, Wx, b; - blobFromTensor(getConstBlob(layer, value_id, 4), W); - blobFromTensor(getConstBlob(layer, value_id, 8), b); - const int outSize = W.cols / 4; + layerParams.set("height", cropSize.at(0)); + layerParams.set("width", cropSize.at(1)); - // IGFO->IFOG - float* weightData = (float*)W.data; - for (int i = 0; i < W.rows; ++i) - for (int j = 0; j < outSize; ++j) - { - std::swap(weightData[i * W.cols + 1 * outSize + j], - weightData[i * W.cols + 2 * outSize + j]); - std::swap(weightData[i * W.cols + 2 * outSize + j], - weightData[i * W.cols + 3 * outSize + j]); - } - Wx = W.rowRange(0, W.rows - outSize).t(); - Wh = W.rowRange(W.rows - outSize, W.rows).t(); + int id = dstNet.addLayer(name, "CropAndResize", layerParams); + layer_id[name] = id; - layerParams.blobs.resize(3); - layerParams.blobs[0] = Wh; - layerParams.blobs[1] = Wx; - layerParams.blobs[2] = b; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); +} - if (hasLayerAttr(layer, "use_peephole")) - { - bool usePeephole = getLayerAttr(layer, "use_peephole").b(); - if (usePeephole) - { - layerParams.set("use_peephole", true); - layerParams.blobs.resize(6); - for (int i = 0; i < 3; ++i) - { - Mat w; - blobFromTensor(getConstBlob(layer, value_id, 5 + i), w); - w = w.reshape(1, w.total()); // Single column. - w = Mat::diag(w); // Make a diagonal matrix. - layerParams.blobs[3 + i] = w; - } - } - } +void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // Computes the mean of elements across dimensions of a tensor. + // If keepdims is false (default) reduces input_tensor along the dimensions given in axis, + // else the reduced dimensions are retained with length 1. + // if indices = [1, 2] in NHWC layout we use global pooling: NxCxHxW --Pooling--> NxCx1x1 + // if keepdims is false we use Flatten after Pooling: out_shape = NxC + // if indices = [0] we use a global pooling by indices. + // To return correct shape, we use Reshape after Pooling. To determine input shape use Slice for input, + // if keepdims is false we use Flatten after Slice. + // Example: input_shape = NxCxHxW + // determine out shape: NxCxHxW --Slice--> 1xCxHxW + // out_shape = 1xCxHxW if keepDims else (1xCxHxW --Flatten--> CxHxW) + // global pool: NxCxHxW --Flatten--> Nx(C*H*W) --Reshape--> 1x1xNx(C*H*W) --Pooling--> 1x1x1x(C*H*W) --Reshape--> out_shape + + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + + Mat indices = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_Assert(indices.type() == CV_32SC1); + + // There are two attributes, "keepdims" and a deprecated "keep_dims". + bool keepDims = false; + if (hasLayerAttr(layer, "keepdims")) + keepDims = getLayerAttr(layer, "keepdims").b(); + else if (hasLayerAttr(layer, "keep_dims")) + keepDims = getLayerAttr(layer, "keep_dims").b(); + + if (indices.total() == 1 && indices.at(0) == 0) + { + LayerParams flattenLp; + std::string flattenName = name + "/flatten"; + CV_Assert(layer_id.find(flattenName) == layer_id.end()); + int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); + layer_id[flattenName] = flattenId; + connect(layer_id, dstNet, parsePin(layer.input(0)), flattenId, 0); + + LayerParams reshapeLp; + std::string reshapeName = name + "/reshape"; + CV_Assert(layer_id.find(reshapeName) == layer_id.end()); + reshapeLp.set("axis", 0); + reshapeLp.set("num_axes", 1); + int newShape[] = {1, 1, -1}; + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 3)); + + int reshapeId = dstNet.addLayer(reshapeName, "Reshape", reshapeLp); + layer_id[reshapeName] = reshapeId; + connect(layer_id, dstNet, Pin(flattenName), reshapeId, 0); + + LayerParams avgLp; + std::string avgName = name + "/avg"; + CV_Assert(layer_id.find(avgName) == layer_id.end()); + avgLp.set("pool", type == "Mean" ? "ave" : "sum"); + // pooling kernel H x 1 + avgLp.set("global_pooling_h", true); + avgLp.set("kernel_w", 1); + int avgId = dstNet.addLayer(avgName, "Pooling", avgLp); + layer_id[avgName] = avgId; + connect(layer_id, dstNet, Pin(reshapeName), avgId, 0); + + LayerParams sliceLp; + std::string layerShapeName = name + "/slice"; + CV_Assert(layer_id.find(layerShapeName) == layer_id.end()); + sliceLp.set("axis", 0); + int begin[] = {0}; + int size[] = {1}; + sliceLp.set("begin", DictValue::arrayInt(&begin[0], 1)); + sliceLp.set("size", DictValue::arrayInt(&size[0], 1)); + int sliceId = dstNet.addLayer(layerShapeName, "Slice", sliceLp); + layer_id[layerShapeName] = sliceId; + connect(layer_id, dstNet, Pin(layer.input(0)), sliceId, 0); + + if (!keepDims) + { + LayerParams squeezeLp; + std::string squeezeName = name + "/squeeze"; + CV_Assert(layer_id.find(squeezeName) == layer_id.end()); + squeezeLp.set("axis", 0); + squeezeLp.set("end_axis", 1); + int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); + layer_id[squeezeName] = squeezeId; + connect(layer_id, dstNet, Pin(layerShapeName), squeezeId, 0); + layerShapeName = squeezeName; + } - int id = dstNet.addLayer(name, "LSTM", layerParams); + int id = dstNet.addLayer(name, "Reshape", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, Pin(avgName), id, 0); + connect(layer_id, dstNet, Pin(layerShapeName), id, 1); + } else if (indices.total() == 1) { + int axis = toNCHW(indices.at(0)); + if (axis == 2 || axis == 3) + { + layerParams.set("pool", type == "Mean" ? "ave" : "sum"); + layerParams.set(axis == 2 ? "kernel_w" : "kernel_h", 1); + layerParams.set(axis == 2 ? "global_pooling_h" : "global_pooling_w", true); + int id = dstNet.addLayer(name, "Pooling", layerParams); layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - // one input only - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); - data_layouts[name] = DATA_LAYOUT_UNKNOWN; + if (!keepDims) + { + // To keep correct order after squeeze dims we first need to change layout from NCHW to NHWC + LayerParams permLP; + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + std::string permName = name + "/nchw"; + Pin inpId = Pin(name); + addPermuteLayer(order, permName, inpId); + + LayerParams squeezeLp; + std::string squeezeName = name + "/squeeze"; + CV_Assert(layer_id.find(squeezeName) == layer_id.end()); + squeezeLp.set("axis", indices.at(0)); + squeezeLp.set("end_axis", indices.at(0) + 1); + int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); + layer_id[squeezeName] = squeezeId; + connect(layer_id, dstNet, Pin(permName), squeezeId, 0); + } } - else if (type == "ResizeNearestNeighbor" || type == "ResizeBilinear" || type == "FusedResizeAndPadConv2D") + else if (axis == 1) { - CV_CheckGT(num_inputs, 0, ""); - std::string convWeights = ""; - if (type == "FusedResizeAndPadConv2D") - { - // input: "mul_1" - // input: "decoder/ResizeBilinear/size" - // input: "decoder/decoder_conv0/Conv2D_dummy_paddings" - // input: "decoder/decoder_conv0/weights" - CV_CheckEQ(num_inputs, 4, "Number of input for FusedResizeAndPadConv2D"); - - Mat paddings = getTensorContent(getConstBlob(layer, value_id, 2)); - CV_CheckEQ(countNonZero(paddings), 0, "Unsupported mode"); + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + Pin inpId = parsePin(layer.input(0)); + addPermuteLayer(order, name + "/nhwc", inpId); - convWeights = layer.input(3); - layer.mutable_input()->DeleteSubrange(2, 2); // FIXIT do NOT modify input model - num_inputs = layer.input_size(); - name = name + "/resize"; + layerParams.set("pool", type == "Mean" ? "ave" : "sum"); + layerParams.set("kernel_h", 1); + layerParams.set("global_pooling_w", true); + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, inpId, id, 0); - if (hasLayerAttr(layer, "resize_align_corners")) - { - // FIXIT do NOT modify input model - layer.mutable_attr()->insert( - ::google::protobuf::MapPair("align_corners", - getLayerAttr(layer, "resize_align_corners"))); - } - } - if (num_inputs == 2) - { - Mat outSize = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_CheckTypeEQ(outSize.type(), CV_32SC1, ""); CV_CheckEQ(outSize.total(), (size_t)2, ""); - layerParams.set("height", outSize.at(0, 0)); - layerParams.set("width", outSize.at(0, 1)); - } - else if (num_inputs == 3) + if (!keepDims) { - Mat factorHeight = getTensorContent(getConstBlob(layer, value_id, 1)); - Mat factorWidth = getTensorContent(getConstBlob(layer, value_id, 2)); - factorHeight.convertTo(factorHeight, CV_32F); - factorWidth.convertTo(factorWidth, CV_32F); - layerParams.set("zoom_factor_x", factorWidth.at(0)); - layerParams.set("zoom_factor_y", factorHeight.at(0)); + LayerParams squeezeLp; + std::string squeezeName = name + "/squeeze"; + CV_Assert(layer_id.find(squeezeName) == layer_id.end()); + int channel_id = 3; // TF NHWC layout + squeezeLp.set("axis", channel_id - 1); + squeezeLp.set("end_axis", channel_id); + int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); + layer_id[squeezeName] = squeezeId; + connect(layer_id, dstNet, Pin(name), squeezeId, 0); } else - CV_Check(num_inputs, num_inputs == 2 || num_inputs == 3, ""); - - if (type == "ResizeNearestNeighbor") - layerParams.set("interpolation", "nearest"); - else - layerParams.set("interpolation", "bilinear"); - - if (hasLayerAttr(layer, "align_corners")) - layerParams.set("align_corners", getLayerAttr(layer, "align_corners").b()); - - if (hasLayerAttr(layer, "half_pixel_centers")) - layerParams.set("half_pixel_centers", getLayerAttr(layer, "half_pixel_centers").b()); - - int id = dstNet.addLayer(name, "Resize", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - - // Step back to add convolution - if (type == "FusedResizeAndPadConv2D") { - tensorflow::NodeDef conv = layer_; - conv.clear_input(); - conv.add_input(name); - conv.add_input(convWeights); - conv.set_op("Conv2D"); - parseNode(conv); + int order[] = {0, 3, 1, 2}; // From NHWC to OpenCV's NCHW. + Pin inpId = parsePin(name); + addPermuteLayer(order, name + "/nchw", inpId); } } - else if (type == "L2Normalize") - { - // op: "L2Normalize" - // input: "input" - // input: "reduction_indices" (axis) - CV_CheckEQ(num_inputs, 2, ""); - Mat reductionIndices = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_Assert(reductionIndices.type() == CV_32SC1); - - const int numAxes = reductionIndices.total(); - if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) - for (int i = 0; i < numAxes; ++i) - reductionIndices.at(i) = toNCHW(reductionIndices.at(i)); - - cv::sort(reductionIndices, reductionIndices, SORT_ASCENDING); - for (int i = 1; i < numAxes; ++i) - { - CV_Assert(reductionIndices.at(i) == reductionIndices.at(i - 1) + 1); - // Axes have the same sign. - CV_Assert(reductionIndices.at(i) * reductionIndices.at(i - 1) >= 0); - } - layerParams.set("start_axis", reductionIndices.at(0)); - layerParams.set("end_axis", reductionIndices.at(numAxes - 1)); + } else { + if (indices.total() != 2 || indices.at(0) != 1 || indices.at(1) != 2) + CV_Error(Error::StsNotImplemented, "Unsupported mode of reduce_mean or reduce_sum operation."); - int id = dstNet.addLayer(name, "Normalize", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else if (type == "PriorBox") + layerParams.set("pool", type == "Mean" ? "ave" : "sum"); + layerParams.set("global_pooling", true); + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + + if (!keepDims) { - CV_CheckEQ(num_inputs, 2, ""); - if (hasLayerAttr(layer, "min_size")) - layerParams.set("min_size", getLayerAttr(layer, "min_size").i()); - if (hasLayerAttr(layer, "max_size")) - layerParams.set("max_size", getLayerAttr(layer, "max_size").i()); - if (hasLayerAttr(layer, "flip")) - layerParams.set("flip", getLayerAttr(layer, "flip").b()); - if (hasLayerAttr(layer, "clip")) - layerParams.set("clip", getLayerAttr(layer, "clip").b()); - if (hasLayerAttr(layer, "offset")) - layerParams.set("offset", getLayerAttr(layer, "offset").f()); - if (hasLayerAttr(layer, "step")) - layerParams.set("step", getLayerAttr(layer, "step").f()); - - const std::string paramNames[] = {"variance", "aspect_ratio", "scales", - "width", "height"}; - for (int i = 0; i < 5; ++i) - { - if (hasLayerAttr(layer, paramNames[i])) - { - Mat values = getTensorContent(getLayerAttr(layer, paramNames[i]).tensor()); - layerParams.set(paramNames[i], - DictValue::arrayReal((float*)values.data, values.total())); - } - } - int id = dstNet.addLayer(name, "PriorBox", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); - data_layouts[name] = DATA_LAYOUT_UNKNOWN; + LayerParams flattenLp; + std::string flattenName = name + "/flatten"; + CV_Assert(layer_id.find(flattenName) == layer_id.end()); + int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); + layer_id[flattenName] = flattenId; + connect(layer_id, dstNet, Pin(name), flattenId, 0); } - else if (type == "Softmax") - { - CV_CheckGT(num_inputs, 0, ""); - if (hasLayerAttr(layer, "axis")) - layerParams.set("axis", getLayerAttr(layer, "axis").i()); + } +} - int id = dstNet.addLayer(name, "Softmax", layerParams); - layer_id[name] = id; - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else if (type == "CropAndResize") - { - // op: "CropAndResize" - // input: "input" - // input: "boxes" - // input: "sizes" - CV_CheckEQ(num_inputs, 3, ""); +void TFImporter::parsePack(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: tf.stack(list of tensors, axis=0) + // Join a list of inputs along a new axis. + // The "axis" specifies the index of the new axis in the dimensions of the output. + // Example: given a list with "N" tensors of shape (C, H, W): + // if axis == 0 then the output tensor will have the shape (N, C, H, W), + // if axis == 1 then the output tensor will have the shape (C, N, H, W). + + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + CV_Assert(hasLayerAttr(layer, "axis")); + int dim = (int)getLayerAttr(layer, "axis").i(); + if (dim != 0) + CV_Error(Error::StsNotImplemented, "Unsupported mode of pack operation."); + + CV_Assert(hasLayerAttr(layer, "N")); + int num = (int)getLayerAttr(layer, "N").i(); + CV_CheckEQ(num_inputs, num, ""); + std::string base_name = name + "/reshape_"; + std::vector reshape_ids; + for (int i = 0; i < num; i++) { + std::ostringstream ss; + ss << i; + std::string reshape_name = base_name + ss.str(); + LayerParams reshapeLP; + reshapeLP.set("axis", dim); + reshapeLP.set("num_axes", 1); + int outShape[] = {1, -1}; + reshapeLP.set("dim", DictValue::arrayInt(&outShape[0], 2)); + int id = dstNet.addLayer(reshape_name, "Reshape", reshapeLP); + layer_id[reshape_name] = id; + reshape_ids.push_back(id); + connect(layer_id, dstNet, parsePin(layer.input(i)), id, 0); + } - Mat cropSize = getTensorContent(getConstBlob(layer, value_id, 2)); - CV_CheckTypeEQ(cropSize.type(), CV_32SC1, ""); CV_CheckEQ(cropSize.total(), (size_t)2, ""); + layerParams.set("axis", dim); + int id = dstNet.addLayer(name, "Concat", layerParams); + layer_id[name] = id; - layerParams.set("height", cropSize.at(0)); - layerParams.set("width", cropSize.at(1)); + for (int li = 0; li < num; li++) + dstNet.connect(reshape_ids[li], 0, id, li); +} - int id = dstNet.addLayer(name, "CropAndResize", layerParams); - layer_id[name] = id; +void TFImporter::parseClipByValue(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "ClipByValue" + // input: "input" + // input: "mix" + // input: "max" - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); - } - else if (type == "Mean" || type == "Sum") - { - // Computes the mean of elements across dimensions of a tensor. - // If keepdims is false (default) reduces input_tensor along the dimensions given in axis, - // else the reduced dimensions are retained with length 1. - // if indices = [1, 2] in NHWC layout we use global pooling: NxCxHxW --Pooling--> NxCx1x1 - // if keepdims is false we use Flatten after Pooling: out_shape = NxC - // if indices = [0] we use a global pooling by indices. - // To return correct shape, we use Reshape after Pooling. To determine input shape use Slice for input, - // if keepdims is false we use Flatten after Slice. - // Example: input_shape = NxCxHxW - // determine out shape: NxCxHxW --Slice--> 1xCxHxW - // out_shape = 1xCxHxW if keepDims else (1xCxHxW --Flatten--> CxHxW) - // global pool: NxCxHxW --Flatten--> Nx(C*H*W) --Reshape--> 1x1xNx(C*H*W) --Pooling--> 1x1x1x(C*H*W) --Reshape--> out_shape - CV_CheckGT(num_inputs, 0, ""); - - Mat indices = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_Assert(indices.type() == CV_32SC1); - - // There are two attributes, "keepdims" and a deprecated "keep_dims". - bool keepDims = false; - if (hasLayerAttr(layer, "keepdims")) - keepDims = getLayerAttr(layer, "keepdims").b(); - else if (hasLayerAttr(layer, "keep_dims")) - keepDims = getLayerAttr(layer, "keep_dims").b(); - - if (indices.total() == 1 && indices.at(0) == 0) - { - LayerParams flattenLp; - std::string flattenName = name + "/flatten"; - CV_Assert(layer_id.find(flattenName) == layer_id.end()); - int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); - layer_id[flattenName] = flattenId; - connect(layer_id, dstNet, parsePin(layer.input(0)), flattenId, 0); - - LayerParams reshapeLp; - std::string reshapeName = name + "/reshape"; - CV_Assert(layer_id.find(reshapeName) == layer_id.end()); - reshapeLp.set("axis", 0); - reshapeLp.set("num_axes", 1); - int newShape[] = {1, 1, -1}; - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 3)); - - int reshapeId = dstNet.addLayer(reshapeName, "Reshape", reshapeLp); - layer_id[reshapeName] = reshapeId; - connect(layer_id, dstNet, Pin(flattenName), reshapeId, 0); - - LayerParams avgLp; - std::string avgName = name + "/avg"; - CV_Assert(layer_id.find(avgName) == layer_id.end()); - avgLp.set("pool", type == "Mean" ? "ave" : "sum"); - // pooling kernel H x 1 - avgLp.set("global_pooling_h", true); - avgLp.set("kernel_w", 1); - int avgId = dstNet.addLayer(avgName, "Pooling", avgLp); - layer_id[avgName] = avgId; - connect(layer_id, dstNet, Pin(reshapeName), avgId, 0); - - LayerParams sliceLp; - std::string layerShapeName = name + "/slice"; - CV_Assert(layer_id.find(layerShapeName) == layer_id.end()); - sliceLp.set("axis", 0); - int begin[] = {0}; - int size[] = {1}; - sliceLp.set("begin", DictValue::arrayInt(&begin[0], 1)); - sliceLp.set("size", DictValue::arrayInt(&size[0], 1)); - int sliceId = dstNet.addLayer(layerShapeName, "Slice", sliceLp); - layer_id[layerShapeName] = sliceId; - connect(layer_id, dstNet, Pin(layer.input(0)), sliceId, 0); - - if (!keepDims) - { - LayerParams squeezeLp; - std::string squeezeName = name + "/squeeze"; - CV_Assert(layer_id.find(squeezeName) == layer_id.end()); - squeezeLp.set("axis", 0); - squeezeLp.set("end_axis", 1); - int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); - layer_id[squeezeName] = squeezeId; - connect(layer_id, dstNet, Pin(layerShapeName), squeezeId, 0); - layerShapeName = squeezeName; - } + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); - int id = dstNet.addLayer(name, "Reshape", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, Pin(avgName), id, 0); - connect(layer_id, dstNet, Pin(layerShapeName), id, 1); - } else if (indices.total() == 1) { - int axis = toNCHW(indices.at(0)); - if (axis == 2 || axis == 3) - { - layerParams.set("pool", type == "Mean" ? "ave" : "sum"); - layerParams.set(axis == 2 ? "kernel_w" : "kernel_h", 1); - layerParams.set(axis == 2 ? "global_pooling_h" : "global_pooling_w", true); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - - if (!keepDims) - { - // To keep correct order after squeeze dims we first need to change layout from NCHW to NHWC - LayerParams permLP; - int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - std::string permName = name + "/nchw"; - Pin inpId = Pin(name); - addPermuteLayer(order, permName, inpId); - - LayerParams squeezeLp; - std::string squeezeName = name + "/squeeze"; - CV_Assert(layer_id.find(squeezeName) == layer_id.end()); - squeezeLp.set("axis", indices.at(0)); - squeezeLp.set("end_axis", indices.at(0) + 1); - int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); - layer_id[squeezeName] = squeezeId; - connect(layer_id, dstNet, Pin(permName), squeezeId, 0); - } - } - else if (axis == 1) - { - int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - Pin inpId = parsePin(layer.input(0)); - addPermuteLayer(order, name + "/nhwc", inpId); - - layerParams.set("pool", type == "Mean" ? "ave" : "sum"); - layerParams.set("kernel_h", 1); - layerParams.set("global_pooling_w", true); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, inpId, id, 0); - - if (!keepDims) - { - LayerParams squeezeLp; - std::string squeezeName = name + "/squeeze"; - CV_Assert(layer_id.find(squeezeName) == layer_id.end()); - int channel_id = 3; // TF NHWC layout - squeezeLp.set("axis", channel_id - 1); - squeezeLp.set("end_axis", channel_id); - int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); - layer_id[squeezeName] = squeezeId; - connect(layer_id, dstNet, Pin(name), squeezeId, 0); - } - else - { - int order[] = {0, 3, 1, 2}; // From NHWC to OpenCV's NCHW. - Pin inpId = parsePin(name); - addPermuteLayer(order, name + "/nchw", inpId); - } - } - } else { - if (indices.total() != 2 || indices.at(0) != 1 || indices.at(1) != 2) - CV_Error(Error::StsNotImplemented, "Unsupported mode of reduce_mean or reduce_sum operation."); + CV_CheckEQ(num_inputs, 3, ""); - layerParams.set("pool", type == "Mean" ? "ave" : "sum"); - layerParams.set("global_pooling", true); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + Mat minValue = getTensorContent(getConstBlob(layer, value_id, 1)); + Mat maxValue = getTensorContent(getConstBlob(layer, value_id, 2)); + CV_CheckEQ(minValue.total(), (size_t)1, ""); CV_CheckTypeEQ(minValue.type(), CV_32FC1, ""); + CV_CheckEQ(maxValue.total(), (size_t)1, ""); CV_CheckTypeEQ(maxValue.type(), CV_32FC1, ""); - if (!keepDims) - { - LayerParams flattenLp; - std::string flattenName = name + "/flatten"; - CV_Assert(layer_id.find(flattenName) == layer_id.end()); - int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); - layer_id[flattenName] = flattenId; - connect(layer_id, dstNet, Pin(name), flattenId, 0); - } - } - } - else if (type == "Pack") - { - // op: tf.stack(list of tensors, axis=0) - // Join a list of inputs along a new axis. - // The "axis" specifies the index of the new axis in the dimensions of the output. - // Example: given a list with "N" tensors of shape (C, H, W): - // if axis == 0 then the output tensor will have the shape (N, C, H, W), - // if axis == 1 then the output tensor will have the shape (C, N, H, W). - CV_CheckGT(num_inputs, 0, ""); - CV_Assert(hasLayerAttr(layer, "axis")); - int dim = (int)getLayerAttr(layer, "axis").i(); - if (dim != 0) - CV_Error(Error::StsNotImplemented, "Unsupported mode of pack operation."); - - CV_Assert(hasLayerAttr(layer, "N")); - int num = (int)getLayerAttr(layer, "N").i(); - CV_CheckEQ(num_inputs, num, ""); - std::string base_name = name + "/reshape_"; - std::vector reshape_ids; - for (int i = 0; i < num; i++) { - std::ostringstream ss; - ss << i; - std::string reshape_name = base_name + ss.str(); - LayerParams reshapeLP; - reshapeLP.set("axis", dim); - reshapeLP.set("num_axes", 1); - int outShape[] = {1, -1}; - reshapeLP.set("dim", DictValue::arrayInt(&outShape[0], 2)); - int id = dstNet.addLayer(reshape_name, "Reshape", reshapeLP); - layer_id[reshape_name] = id; - reshape_ids.push_back(id); - connect(layer_id, dstNet, parsePin(layer.input(i)), id, 0); - } + layerParams.set("min_value", minValue.at(0)); + layerParams.set("max_value", maxValue.at(0)); - layerParams.set("axis", dim); - int id = dstNet.addLayer(name, "Concat", layerParams); - layer_id[name] = id; + int id = dstNet.addLayer(name, "ReLU6", layerParams); + layer_id[name] = id; - for (int li = 0; li < num; li++) - dstNet.connect(reshape_ids[li], 0, id, li); - } - else if (type == "ClipByValue") - { - // op: "ClipByValue" - // input: "input" - // input: "mix" - // input: "max" - CV_CheckEQ(num_inputs, 3, ""); + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); +} - Mat minValue = getTensorContent(getConstBlob(layer, value_id, 1)); - Mat maxValue = getTensorContent(getConstBlob(layer, value_id, 2)); - CV_CheckEQ(minValue.total(), (size_t)1, ""); CV_CheckTypeEQ(minValue.type(), CV_32FC1, ""); - CV_CheckEQ(maxValue.total(), (size_t)1, ""); CV_CheckTypeEQ(maxValue.type(), CV_32FC1, ""); +void TFImporter::parseLeakyRelu(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); - layerParams.set("min_value", minValue.at(0)); - layerParams.set("max_value", maxValue.at(0)); + CV_CheckGT(num_inputs, 0, ""); + CV_Assert(hasLayerAttr(layer, "alpha")); + layerParams.set("negative_slope", getLayerAttr(layer, "alpha").f()); - int id = dstNet.addLayer(name, "ReLU6", layerParams); - layer_id[name] = id; + int id = dstNet.addLayer(name, "ReLU", layerParams); + layer_id[name] = id; + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else if (type == "LeakyRelu") - { - CV_CheckGT(num_inputs, 0, ""); - CV_Assert(hasLayerAttr(layer, "alpha")); - layerParams.set("negative_slope", getLayerAttr(layer, "alpha").f()); +void TFImporter::parseActivation(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + std::string dnnType = type; + if (type == "Abs") dnnType = "AbsVal"; + else if (type == "Tanh") dnnType = "TanH"; + else if (type == "Relu") dnnType = "ReLU"; + else if (type == "Relu6") dnnType = "ReLU6"; + else if (type == "Elu") dnnType = "ELU"; + + int id = dstNet.addLayer(name, dnnType, layerParams); + layer_id[name] = id; + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} - int id = dstNet.addLayer(name, "ReLU", layerParams); - layer_id[name] = id; - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else if (type == "Abs" || type == "Tanh" || type == "Sigmoid" || - type == "Relu" || type == "Elu" || type == "Exp" || - type == "Identity" || type == "Relu6") +void TFImporter::parseCustomLayer(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // Importer does not know how to map this TensorFlow's operation onto OpenCV's layer. + // However we create a layer with the same type and rely that user defined a custom layer. + + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + // All the attributes are added to LayerParams. + google::protobuf::Map attr = layer.attr(); + for (google::protobuf::Map::const_iterator ai = attr.begin(); + ai != attr.end(); ++ai) + { + if (ai->second.value_case() == tensorflow::AttrValue::kS) // string + layerParams.set(ai->first, ai->second.s()); + if (ai->second.value_case() == tensorflow::AttrValue::kI) // int64 + layerParams.set(ai->first, ai->second.i()); + if (ai->second.value_case() == tensorflow::AttrValue::kF) // float + layerParams.set(ai->first, ai->second.f()); + if (ai->second.value_case() == tensorflow::AttrValue::kB) // bool + layerParams.set(ai->first, ai->second.b()); + } + + // All the Const input nodes are added to layer's blobs. + std::vector inputsNames; + for (int i = 0; i < num_inputs; ++i) + { + // Check if input is a Const node. + if (value_id.find(layer.input(i)) != value_id.end()) { - CV_CheckGT(num_inputs, 0, ""); - std::string dnnType = type; - if (type == "Abs") dnnType = "AbsVal"; - else if (type == "Tanh") dnnType = "TanH"; - else if (type == "Relu") dnnType = "ReLU"; - else if (type == "Relu6") dnnType = "ReLU6"; - else if (type == "Elu") dnnType = "ELU"; - - int id = dstNet.addLayer(name, dnnType, layerParams); - layer_id[name] = id; - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); + Mat blob = getTensorContent(getConstBlob(layer, value_id, i)); + layerParams.blobs.push_back(blob); } else - { - // Importer does not know how to map this TensorFlow's operation onto OpenCV's layer. - // However we create a layer with the same type and rely that user defined a custom layer. + inputsNames.push_back(layer.input(i)); + } + int id = dstNet.addLayer(name, type, layerParams); + layer_id[name] = id; - // All the attributes are added to LayerParams. - google::protobuf::Map attr = layer.attr(); - for (google::protobuf::Map::const_iterator ai = attr.begin(); - ai != attr.end(); ++ai) - { - if (ai->second.value_case() == tensorflow::AttrValue::kS) // string - layerParams.set(ai->first, ai->second.s()); - if (ai->second.value_case() == tensorflow::AttrValue::kI) // int64 - layerParams.set(ai->first, ai->second.i()); - if (ai->second.value_case() == tensorflow::AttrValue::kF) // float - layerParams.set(ai->first, ai->second.f()); - if (ai->second.value_case() == tensorflow::AttrValue::kB) // bool - layerParams.set(ai->first, ai->second.b()); - } + for (int i = 0; i < inputsNames.size(); ++i) + { + connect(layer_id, dstNet, parsePin(inputsNames[i]), id, i); + } +} - // All the Const input nodes are added to layer's blobs. - std::vector inputsNames; - for (int i = 0; i < num_inputs; ++i) - { - // Check if input is a Const node. - if (value_id.find(layer.input(i)) != value_id.end()) - { - Mat blob = getTensorContent(getConstBlob(layer, value_id, i)); - layerParams.blobs.push_back(blob); - } - else - inputsNames.push_back(layer.input(i)); - } - int id = dstNet.addLayer(name, type, layerParams); - layer_id[name] = id; +TFImporter::TFImporter(Net& net, const char *model, const char *config) + : dstNet(net), dispatch(buildDispatchMap()) +{ + if (model && model[0]) + { + CV_LOG_DEBUG(NULL, "DNN/TF: processing TensorFlow model from file: " << model); + ReadTFNetParamsFromBinaryFileOrDie(model, &netBin); + } + if (config && config[0]) + { + CV_LOG_DEBUG(NULL, "DNN/TF: processing TensorFlow config from file: " << config); + ReadTFNetParamsFromTextFileOrDie(config, &netTxt); + } - for (int i = 0; i < inputsNames.size(); ++i) - { - connect(layer_id, dstNet, parsePin(inputsNames[i]), id, i); - } + populateNet(); +} + +TFImporter::TFImporter( + Net& net, + const char *dataModel, size_t lenModel, + const char *dataConfig, size_t lenConfig +) + : dstNet(net), dispatch(buildDispatchMap()) +{ + if (dataModel != NULL && lenModel > 0) + { + CV_LOG_DEBUG(NULL, "DNN/TF: processing TensorFlow model from memory (" << lenModel << " bytes)"); + ReadTFNetParamsFromBinaryBufferOrDie(dataModel, lenModel, &netBin); + } + if (dataConfig != NULL && lenConfig > 0) + { + CV_LOG_DEBUG(NULL, "DNN/TF: processing TensorFlow config from memory (" << lenConfig << " bytes)"); + ReadTFNetParamsFromTextBufferOrDie(dataConfig, lenConfig, &netTxt); + } + populateNet(); +} + +void TFImporter::kernelFromTensor(const tensorflow::TensorProto &tensor, Mat &dstBlob) +{ + MatShape shape; + blobShapeFromTensor(tensor, shape); + int dims = (int)shape.size(); + + // TODO: other blob types + CV_Assert(tensor.dtype() == tensorflow::DT_FLOAT || + tensor.dtype() == tensorflow::DT_HALF); + CV_Assert(dims == 4 || dims == 5); + + int out_c, input_c, depth, height, width; + if (dims == 4) + { + // REORDER kernel HWIO to OIHW + swap(shape[0], shape[2]); // IWHO + swap(shape[1], shape[3]); // IOHW + swap(shape[0], shape[1]); // OIHW + depth = 1; height = shape[2]; width = shape[3]; + } + else + { + // REORDER kernel DHWIO to OIDHW + swap(shape[0], shape[4]); // OHWID + swap(shape[1], shape[3]); // OIWHD + swap(shape[2], shape[4]); // OIDHW + depth = shape[2]; height = shape[3]; width = shape[4]; + } + out_c = shape[0]; input_c = shape[1]; + + dstBlob.create(shape, CV_32F); + + Mat tensorContent = getTensorContent(tensor, /*no copy*/false); + int size = tensorContent.total(); + CV_Assert(size == (int)dstBlob.total()); + + float *dstData = dstBlob.ptr(); + const float *data = reinterpret_cast(tensorContent.data); + + int total = out_c * input_c * depth * height * width; + for (int i_oc = 0; i_oc < out_c; i_oc++) { + for (int i_ic = 0; i_ic < input_c; i_ic++) { + for (int i_d = 0; i_d < depth; i_d++) { + for (int i_h = 0; i_h < height; i_h++) { + for (int i_w = 0; i_w < width; i_w++) { + int dst_i = input_c * depth * height * width * i_oc + + depth * height * width * i_ic + height * width * i_d + width * i_h + i_w; + int src_i = out_c * input_c * width * height * i_d + + out_c * input_c * width * i_h + out_c * input_c * i_w + out_c * i_ic + i_oc; + CV_Assert(dst_i < total); + CV_Assert(src_i < total); + dstData[dst_i] = data[src_i]; + } + } + } + } + } +} + +void TFImporter::connect(const std::map& layers_name_id_map, Net& network, const Pin& outPin, + const int input_layer_id, const int input_blob_id) +{ + std::map::const_iterator it = layers_name_id_map.find(outPin.name); + if (it == layers_name_id_map.end()) + CV_Error(Error::StsError, "Input layer not found: " + outPin.name); + + std::vector::iterator inpNameIt = std::find(netInputsNames.begin(), netInputsNames.end(), outPin.name); + int blobIndex; + if (inpNameIt == netInputsNames.end()) + blobIndex = outPin.blobIndex; + else + blobIndex = inpNameIt - netInputsNames.begin(); + network.connect(it->second, blobIndex, input_layer_id, input_blob_id); +} + +void TFImporter::connectToAllBlobs(const std::map& layer_id, Net& network, const Pin& outPin, + const int input_layer_id, const int input_blobs_count) +{ + for (int input_blob_id = 0; input_blob_id < input_blobs_count; input_blob_id++) + connect(layer_id, network, outPin, input_layer_id, input_blob_id); +} + +const tensorflow::TensorProto& TFImporter::getConstBlob(const tensorflow::NodeDef &layer, std::map const_layers, + int input_blob_index, int* actual_inp_blob_idx) { + if (input_blob_index == -1) { + for(int i = 0; i < layer.input_size(); i++) { + Pin input = parsePin(layer.input(i)); + if (const_layers.find(input.name) != const_layers.end()) { + if (input_blob_index != -1) + CV_Error(Error::StsError, "More than one input is Const op"); + + input_blob_index = i; + } + } + } + + if (input_blob_index == -1) + CV_Error(Error::StsError, "Const input blob for weights not found"); + + Pin kernel_inp = parsePin(layer.input(input_blob_index)); + if (const_layers.find(kernel_inp.name) == const_layers.end()) + CV_Error(Error::StsError, "Input [" + layer.input(input_blob_index) + + "] for node [" + layer.name() + "] not found"); + if (kernel_inp.blobIndex != 0) + CV_Error(Error::StsError, "Unsupported kernel input"); + + if(actual_inp_blob_idx) { + *actual_inp_blob_idx = input_blob_index; + } + + int nodeIdx = const_layers.at(kernel_inp.name); + if (nodeIdx < netBin.node_size() && netBin.node(nodeIdx).name() == kernel_inp.name) + { + return netBin.node(nodeIdx).attr().at("value").tensor(); + } + else + { + CV_Assert_N(nodeIdx < netTxt.node_size(), + netTxt.node(nodeIdx).name() == kernel_inp.name); + return netTxt.node(nodeIdx).attr().at("value").tensor(); + } +} + +static void addConstNodes(tensorflow::GraphDef& net, std::map& const_layers, + std::set& layers_to_ignore) +{ + CV_LOG_DEBUG(NULL, "DNN/TF: addConstNodes(): handling " << net.node_size() << " nodes..."); + for (int li = 0; li < net.node_size(); li++) + { + const tensorflow::NodeDef &layer = net.node(li); + String name = layer.name(); + String type = layer.op(); + + //CV_LOG_DEBUG(NULL, "DNN/TF: layer_id=" << li << " - '" << name << "' @ " << type); + + try + { + if (type == "Dequantize") + { + // Example of Dequantize node: + // name: "conv2d_1/bias" + // op: "Dequantize" + // input: "conv2d_1/bias_quantized_const" (tensor of dtype DT_QUINT8) + // input: "conv2d_1/bias_quantized_min" + // input: "conv2d_1/bias_quantized_max" + // attr { key: "T" value { type: DT_QUINT8 } } (quantized type) + // attr { key: "mode" value { s: "MIN_FIRST" } } (quantization technique) + CV_CheckEQ(layer.input_size(), 3, "Dequantize: 3 inputs is supported only"); + for (int i = 0; i < 3; ++i) + CV_Assert(const_layers.find(layer.input(i)) != const_layers.end()); + CV_Assert(hasLayerAttr(layer, "mode") && + getLayerAttr(layer, "mode").s() == "MIN_FIRST"); + + int tensorId = const_layers[layer.input(0)]; + int minId = const_layers[layer.input(1)]; + int maxId = const_layers[layer.input(2)]; + + tensorflow::TensorProto* tensor = net.mutable_node(tensorId) + ->mutable_attr()->at("value") + .mutable_tensor(); + CV_CheckEQ((int)tensor->dtype(), (int)tensorflow::DT_QUINT8, ""); + + Mat qMin = getTensorContent(net.node(minId).attr().at("value").tensor()); + Mat qMax = getTensorContent(net.node(maxId).attr().at("value").tensor()); + CV_CheckEQ(qMin.total(), (size_t)1, ""); + CV_CheckTypeEQ(qMin.type(), CV_32FC1, ""); + CV_CheckEQ(qMax.total(), (size_t)1, ""); + CV_CheckTypeEQ(qMax.type(), CV_32FC1, ""); + + Mat content = getTensorContent(*tensor); + + float minVal = qMin.at(0); + float rangeScale = (qMax.at(0) - minVal) / 255; + CV_Assert(rangeScale >= 0); + content.convertTo(content, CV_32FC1, rangeScale, + rangeScale * cvRound(minVal / rangeScale)); + + tensor->set_dtype(tensorflow::DT_FLOAT); + tensor->set_tensor_content(content.data, content.total() * content.elemSize1()); + + net.mutable_node(tensorId)->set_name(name); + CV_Assert(const_layers.insert(std::make_pair(name, tensorId)).second); + layers_to_ignore.insert(name); + continue; + } + else if (type != "Const") + continue; // only Const parameters are supported + + if (layer.attr().find("value") != layer.attr().end()) + { + CV_Assert(const_layers.insert(std::make_pair(name, li)).second); + } + layers_to_ignore.insert(name); + } + catch (const std::exception& e) + { + CV_LOG_ERROR(NULL, "DNN/TF: Can't handle node='" << name << "'. Exception: " << e.what()); + throw; + } + } + CV_LOG_DEBUG(NULL, "DNN/TF: layers_to_ignore.size() = " << layers_to_ignore.size()); +} + +// If all inputs of specific layer have the same data layout we can say that +// this layer's output has this data layout too. Returns DATA_LAYOUT_UNKNOWN otherwise. +DataLayout TFImporter::predictOutputDataLayout(const tensorflow::NodeDef& layer) +{ + DataLayout layout = getDataLayout(layer); + if (layout != DATA_LAYOUT_UNKNOWN) + { + CV_LOG_DEBUG(NULL, "DNN/TF: predictOutputDataLayout(" << layer.name() << " @ " << layer.op() << ") => " << (int)layout << " (from attrs)"); + return layout; + } + + // Determine layout by layer's inputs + for (int i = 0, n = layer.input_size(); i < n; ++i) + { + std::map::const_iterator it = data_layouts.find(getNodeName(layer.input(i))); + if (it != data_layouts.end()) + { + if (layout != DATA_LAYOUT_UNKNOWN) + { + if (it->second != layout && it->second != DATA_LAYOUT_UNKNOWN) + return DATA_LAYOUT_UNKNOWN; + } + else + layout = it->second; + } + } + + if (layout != DATA_LAYOUT_UNKNOWN) + { + CV_LOG_DEBUG(NULL, "DNN/TF: predictOutputDataLayout(" << layer.name() << " @ " << layer.op() << ") => " << (int)layout << " (from inputs)"); + return layout; + } + + // Determine layout by layer's consumers recursively. + std::map::const_iterator it = data_layouts.find(layer.name()); + CV_Assert(it != data_layouts.end()); + return it->second; +} + +void TFImporter::populateNet() +{ + CV_Assert(netBin.ByteSize() || netTxt.ByteSize()); + + CV_LOG_INFO(NULL, "DNN/TF: parsing model" + << (netBin.has_versions() ? cv::format(" produced by TF v%d (min_consumer=%d)", (int)netBin.versions().producer(), (int)netBin.versions().min_consumer()) : cv::String(" (N/A version info)")) + << ". Number of nodes = " << netBin.node_size() + ); + + if (netTxt.ByteSize()) + { + CV_LOG_INFO(NULL, "DNN/TF: parsing config" + << (netTxt.has_versions() ? cv::format(" produced by TF v%d (min_consumer=%d)", (int)netTxt.versions().producer(), (int)netTxt.versions().min_consumer()) : cv::String(" (N/A version info)")) + << ". Number of nodes = " << netTxt.node_size() + ); + + RemoveIdentityOps(netBin); + CV_LOG_DEBUG(NULL, "DNN/TF: RemoveIdentityOps(model) => " << netBin.node_size() << " nodes"); + RemoveIdentityOps(netTxt); + CV_LOG_DEBUG(NULL, "DNN/TF: RemoveIdentityOps(config) => " << netTxt.node_size() << " nodes"); + + sortByExecutionOrder(netTxt); + CV_LOG_DEBUG(NULL, "DNN/TF: sortByExecutionOrder(config) => " << netTxt.node_size() << " nodes"); + } + else + { + removePhaseSwitches(netBin); + CV_LOG_DEBUG(NULL, "DNN/TF: removePhaseSwitches(model) => " << netBin.node_size() << " nodes"); + + RemoveIdentityOps(netBin); + CV_LOG_DEBUG(NULL, "DNN/TF: RemoveIdentityOps(model) => " << netBin.node_size() << " nodes"); + + simplifySubgraphs(netBin); + CV_LOG_DEBUG(NULL, "DNN/TF: simplifySubgraphs(model) => " << netBin.node_size() << " nodes"); + sortByExecutionOrder(netBin); + CV_LOG_DEBUG(NULL, "DNN/TF: sortByExecutionOrder(model) => " << netBin.node_size() << " nodes"); + } + + tensorflow::GraphDef& net = netTxt.ByteSize() != 0 ? netTxt : netBin; + + int layersSize = net.node_size(); + + // Pre-fill data layouts where they are set explicitly. + // Assuming that nodes are in topological order + for (int i = layersSize - 1; i >= 0; --i) + { + const tensorflow::NodeDef& layer = net.node(i); + std::string name = layer.name(); + + CV_LOG_DEBUG(NULL, "DNN/TF: node(" << i << " - '" << name << "') propagating layout..."); + + try + { + DataLayout layout = getDataLayout(layer); + std::map::iterator it = data_layouts.find(name); + if (it != data_layouts.end()) + { + if (layout != DATA_LAYOUT_UNKNOWN) + { + if (it->second == DATA_LAYOUT_UNKNOWN) + it->second = layout; + else if (it->second != layout) + { + it->second = DATA_LAYOUT_UNKNOWN; + layout = DATA_LAYOUT_UNKNOWN; + } + } + else + layout = it->second; + } + else + data_layouts[name] = layout; + + // Specify input layers to have the same data layout. + for (int j = 0; j < layer.input_size(); ++j) + { + name = getNodeName(layer.input(j)); + it = data_layouts.find(name); + if (it != data_layouts.end()) + { + if (layout != DATA_LAYOUT_UNKNOWN) + { + if (it->second == DATA_LAYOUT_UNKNOWN) + it->second = layout; + else if (it->second != layout) + it->second = DATA_LAYOUT_UNKNOWN; + } + } + else + data_layouts[name] = layout; + } + } + catch (const std::exception& e) + { + CV_LOG_ERROR(NULL, "DNN/TF: Can't propagate layout for node='" << name << "'. Exception: " << e.what()); + throw; + } + } + + addConstNodes(netBin, value_id, layers_to_ignore); + addConstNodes(netTxt, value_id, layers_to_ignore); + + + for (int li = 0; li < layersSize; li++) + { + const tensorflow::NodeDef& layer = net.node(li); + + const std::string name = layer.name(); + const std::string type = layer.op(); + const int ninputs = layer.input_size(); + CV_LOG_DEBUG(NULL, "DNN/TF: (" << li << "/" << layersSize << ") Parse layer " << name << " @ " << type << " with " << ninputs << " inputs"); + + parseNode(layer); + } + + for (size_t i = 0; i < netInputsNames.size(); i++) + { + CV_LOG_DEBUG(NULL, "DNN/TF: Model input: " << i << " - '" << netInputsNames[i] << "'"); + CV_Assert(!netInputsNames[i].empty()); + } + dstNet.setInputsNames(netInputsNames); + CV_LOG_DEBUG(NULL, "DNN/TF: ===================== Import completed ====================="); +} + +void TFImporter::addPermuteLayer(const int* order, const std::string& permName, Pin& inpId) +{ + LayerParams permLP; + permLP.set("order", DictValue::arrayInt(order, 4)); + CV_Assert(layer_id.find(permName) == layer_id.end()); + int permId = dstNet.addLayer(permName, "Permute", permLP); + layer_id[permName] = permId; + connect(layer_id, dstNet, inpId, permId, 0); + inpId = Pin(permName); +} + +void TFImporter::parseNode(const tensorflow::NodeDef& layer) +{ + tensorflow::GraphDef& net = netTxt.ByteSize() != 0 ? netTxt : netBin; + + const std::string& name = layer.name(); + const std::string& type = layer.op(); + + try + { + LayerParams layerParams; + + if (layers_to_ignore.find(name) != layers_to_ignore.end()) + { + CV_LOG_DEBUG(NULL, "DNN/TF: ignored"); + return; + } + + DataLayout predictedLayout = predictOutputDataLayout(layer); + data_layouts[name] = predictedLayout; + + DispatchMap::const_iterator iter = dispatch.find(type); + if (iter != dispatch.end()) + { + ((*this).*(iter->second))(net, layer, layerParams); + } + else + { + parseCustomLayer(net, layer, layerParams); } } catch (const std::exception& e) From 3cf43753876e3c2c2676283d4fd10a6018b5a264 Mon Sep 17 00:00:00 2001 From: Tiago De Gaspari Date: Sat, 12 Jun 2021 17:28:54 -0300 Subject: [PATCH 004/376] Merge pull request #19842 from gasparitiago:3.4 Update rotatedRectangleIntersection function to calculate near to origin * Change type used in points function from RotatedRect In the function that sets the points of a RotatedRect, the types should be double in order to keep the precision when dealing with RotatedRects that are defined far from the origin. This commit solves the problem in some assertions from rotatedRectangleIntersection when dealing with rectangles far from origin. * added proper type casts * Update rotatedRectangleIntersection function to calculate near to origin This commit changes the rotatedRectangleIntersection function in order to calculate the intersection of two rectangles considering that they are shifted near the coordinates origin (0, 0). This commit solves the problem in some assertions from rotatedRectangleIntersection when dealing with rectangles far from origin. * Revert type changes in types.cpp and adequate code to c++98 * Revert unnecessary casts on types.cpp Co-authored-by: Vadim Pisarevsky --- modules/imgproc/src/intersection.cpp | 57 +++++++++++++++++----- modules/imgproc/test/test_intersection.cpp | 17 +++++++ 2 files changed, 62 insertions(+), 12 deletions(-) diff --git a/modules/imgproc/src/intersection.cpp b/modules/imgproc/src/intersection.cpp index 3f749896a42c..47d3f3f457b5 100644 --- a/modules/imgproc/src/intersection.cpp +++ b/modules/imgproc/src/intersection.cpp @@ -47,24 +47,16 @@ namespace cv { -int rotatedRectangleIntersection( const RotatedRect& rect1, const RotatedRect& rect2, OutputArray intersectingRegion ) +static int _rotatedRectangleIntersection( const RotatedRect& rect1, const RotatedRect& rect2, std::vector &intersection ) { CV_INSTRUMENT_REGION(); // L2 metric const float samePointEps = std::max(1e-16f, 1e-6f * (float)std::max(rect1.size.area(), rect2.size.area())); - if (rect1.size.empty() || rect2.size.empty()) - { - intersectingRegion.release(); - return INTERSECT_NONE; - } - Point2f vec1[4], vec2[4]; Point2f pts1[4], pts2[4]; - std::vector intersection; intersection.reserve(24); - rect1.points(pts1); rect2.points(pts2); @@ -92,8 +84,6 @@ int rotatedRectangleIntersection( const RotatedRect& rect1, const RotatedRect& r intersection[i] = pts1[i]; } - Mat(intersection).copyTo(intersectingRegion); - return INTERSECT_FULL; } } @@ -300,7 +290,50 @@ int rotatedRectangleIntersection( const RotatedRect& rect1, const RotatedRect& r } intersection.resize(N); - Mat(intersection).copyTo(intersectingRegion); + + return ret; +} + +int rotatedRectangleIntersection( const RotatedRect& rect1, const RotatedRect& rect2, OutputArray intersectingRegion ) +{ + CV_INSTRUMENT_REGION(); + + if (rect1.size.empty() || rect2.size.empty()) + { + intersectingRegion.release(); + return INTERSECT_NONE; + } + + // Shift rectangles closer to origin (0, 0) to improve the calculation of the intesection region + // To do that, the average center of the rectangles is moved to the origin + const Point2f averageCenter = (rect1.center + rect2.center) / 2.0f; + + RotatedRect shiftedRect1(rect1); + RotatedRect shiftedRect2(rect2); + + // Move rectangles closer to origin + shiftedRect1.center -= averageCenter; + shiftedRect2.center -= averageCenter; + + std::vector intersection; intersection.reserve(24); + + const int ret = _rotatedRectangleIntersection(shiftedRect1, shiftedRect2, intersection); + + // If return is not None, the intersection Points are shifted back to the original position + // and copied to the interesectingRegion + if (ret != INTERSECT_NONE) + { + for (size_t i = 0; i < intersection.size(); ++i) + { + intersection[i] += averageCenter; + } + + Mat(intersection).copyTo(intersectingRegion); + } + else + { + intersectingRegion.release(); + } return ret; } diff --git a/modules/imgproc/test/test_intersection.cpp b/modules/imgproc/test/test_intersection.cpp index 7527dd9a22cc..c455c439fce1 100644 --- a/modules/imgproc/test/test_intersection.cpp +++ b/modules/imgproc/test/test_intersection.cpp @@ -391,4 +391,21 @@ TEST(Imgproc_RotatedRectangleIntersection, regression_18520) } } +TEST(Imgproc_RotatedRectangleIntersection, regression_19824) +{ + RotatedRect r1( + Point2f(246805.033f, 4002326.94f), + Size2f(26.40587f, 6.20026f), + -62.10156f); + RotatedRect r2( + Point2f(246805.122f, 4002326.59f), + Size2f(27.4821f, 8.5361f), + -56.33761f); + + std::vector intersections; + int interType = cv::rotatedRectangleIntersection(r1, r2, intersections); + EXPECT_EQ(INTERSECT_PARTIAL, interType); + EXPECT_LE(intersections.size(), (size_t)7); +} + }} // namespace From c8268e65fd6c3a8fba7a5a1f4b830222b9bc9d66 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Fri, 11 Jun 2021 22:03:33 +0200 Subject: [PATCH 005/376] Fix potential NaN in cv::norm. There can be an int overflow. cv::norm( InputArray _src, int normType, InputArray _mask ) is fine, not cv::norm( InputArray _src1, InputArray _src2, int normType, InputArray _mask ). --- modules/core/src/norm.cpp | 2 +- modules/core/test/test_arithm.cpp | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/core/src/norm.cpp b/modules/core/src/norm.cpp index 601082783e3c..fad641554dac 100644 --- a/modules/core/src/norm.cpp +++ b/modules/core/src/norm.cpp @@ -1171,7 +1171,7 @@ double norm( InputArray _src1, InputArray _src2, int normType, InputArray _mask // special case to handle "integer" overflow in accumulator const size_t esz = src1.elemSize(); const int total = (int)it.size; - const int intSumBlockSize = normType == NORM_L1 && depth <= CV_8S ? (1 << 23) : (1 << 15); + const int intSumBlockSize = (normType == NORM_L1 && depth <= CV_8S ? (1 << 23) : (1 << 15))/cn; const int blockSize = std::min(total, intSumBlockSize); int isum = 0; int count = 0; diff --git a/modules/core/test/test_arithm.cpp b/modules/core/test/test_arithm.cpp index 2746feb2f245..74bf39fbc7a2 100644 --- a/modules/core/test/test_arithm.cpp +++ b/modules/core/test/test_arithm.cpp @@ -2117,6 +2117,15 @@ TEST(Core_Norm, IPP_regression_NORM_L1_16UC3_small) EXPECT_EQ((double)20*cn, cv::norm(a, b, NORM_L1, mask)); } +TEST(Core_Norm, NORM_L2_8UC4) +{ + // Tests there is no integer overflow in norm computation for multiple channels. + const int kSide = 100; + cv::Mat4b a(kSide, kSide, cv::Scalar(255, 255, 255, 255)); + cv::Mat4b b = cv::Mat4b::zeros(kSide, kSide); + const double kNorm = 2.*kSide*255.; + EXPECT_EQ(kNorm, cv::norm(a, b, NORM_L2)); +} TEST(Core_ConvertTo, regression_12121) { From 464441d8c3943126a5238b34ee34633dcfbf399e Mon Sep 17 00:00:00 2001 From: Ian Maquignaz <9im14@queensu.ca> Date: Thu, 10 Jun 2021 20:13:06 -0400 Subject: [PATCH 006/376] Added new unit test for initInverseRectificationMap() Function is validated. Included an update to DISABLED_Calib3d_InitInverseRectificationMap. Includes updates per input from @alalek and unit test regression # to reflect PR # --- modules/calib3d/test/test_undistort.cpp | 82 +++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 4 deletions(-) diff --git a/modules/calib3d/test/test_undistort.cpp b/modules/calib3d/test/test_undistort.cpp index 9663d36b7862..ea1a95207954 100644 --- a/modules/calib3d/test/test_undistort.cpp +++ b/modules/calib3d/test/test_undistort.cpp @@ -897,7 +897,7 @@ void CV_InitInverseRectificationMapTest::prepare_to_validation(int/* test_case_i Mat _new_cam0 = zero_new_cam ? test_mat[INPUT][0] : test_mat[INPUT][3]; Mat _mapx(img_size, CV_32F), _mapy(img_size, CV_32F); - double a[9], d[5]={0,0,0,0,0}, R[9]={1, 0, 0, 0, 1, 0, 0, 0, 1}, a1[9]; + double a[9], d[5]={0., 0., 0., 0. , 0.}, R[9]={1., 0., 0., 0., 1., 0., 0., 0., 1.}, a1[9]; Mat _a(3, 3, CV_64F, a), _a1(3, 3, CV_64F, a1); Mat _d(_d0.rows,_d0.cols, CV_MAKETYPE(CV_64F,_d0.channels()),d); Mat _R(3, 3, CV_64F, R); @@ -951,9 +951,9 @@ void CV_InitInverseRectificationMapTest::prepare_to_validation(int/* test_case_i // Undistort double x2 = x*x, y2 = y*y; double r2 = x2 + y2; - double cdist = 1./(1 + (d[0] + (d[1] + d[4]*r2)*r2)*r2); // (1 + (d[5] + (d[6] + d[7]*r2)*r2)*r2) == 1 as d[5-7]=0; - double x_ = x*cdist - d[2]*2*x*y + d[3]*(r2 + 2*x2); - double y_ = y*cdist - d[3]*2*x*y + d[2]*(r2 + 2*y2); + double cdist = 1./(1. + (d[0] + (d[1] + d[4]*r2)*r2)*r2); // (1. + (d[5] + (d[6] + d[7]*r2)*r2)*r2) == 1 as d[5-7]=0; + double x_ = (x - (d[2]*2.*x*y + d[3]*(r2 + 2.*x2)))*cdist; + double y_ = (y - (d[3]*2.*x*y + d[2]*(r2 + 2.*y2)))*cdist; // Rectify double X = R[0]*x_ + R[1]*y_ + R[2]; @@ -1807,4 +1807,78 @@ TEST(Calib3d_initUndistortRectifyMap, regression_14467) EXPECT_LE(cvtest::norm(dst, mesh_uv, NORM_INF), 1e-3); } +TEST(Calib3d_initInverseRectificationMap, regression_20165) +{ + Size size_w_h(1280, 800); + Mat dst(size_w_h, CV_32FC2); // Reference for validation + Mat mapxy; // Output of initInverseRectificationMap() + + // Camera Matrix + double k[9]={ + 1.5393951443032472e+03, 0., 6.7491727003047140e+02, + 0., 1.5400748240626747e+03, 5.1226968329123963e+02, + 0., 0., 1. + }; + Mat _K(3, 3, CV_64F, k); + + // Distortion + // double d[5]={0,0,0,0,0}; // Zero Distortion + double d[5]={ // Non-zero distortion + -3.4134571357400023e-03, 2.9733267766101856e-03, // K1, K2 + 3.6653586399031184e-03, -3.1960714017365702e-03, // P1, P2 + 0. // K3 + }; + Mat _d(1, 5, CV_64F, d); + + // Rotation + //double R[9]={1., 0., 0., 0., 1., 0., 0., 0., 1.}; // Identity transform (none) + double R[9]={ // Random transform + 9.6625486010428052e-01, 1.6055789378989216e-02, 2.5708706103628531e-01, + -8.0300261706161002e-03, 9.9944797497929860e-01, -3.2237617614807819e-02, + -2.5746274294459848e-01, 2.9085338870243265e-02, 9.6585039165403186e-01 + }; + Mat _R(3, 3, CV_64F, R); + + // --- Validation --- // + initInverseRectificationMap(_K, _d, _R, _K, size_w_h, CV_32FC2, mapxy, noArray()); + + // Copy camera matrix + double fx, fy, cx, cy, ifx, ify, cxn, cyn; + fx = k[0]; fy = k[4]; cx = k[2]; cy = k[5]; + + // Copy new camera matrix + ifx = k[0]; ify = k[4]; cxn = k[2]; cyn = k[5]; + + // Distort Points + for( int v = 0; v < size_w_h.height; v++ ) + { + for( int u = 0; u < size_w_h.width; u++ ) + { + // Convert from image to pin-hole coordinates + double x = (u - cx)/fx; + double y = (v - cy)/fy; + + // Undistort + double x2 = x*x, y2 = y*y; + double r2 = x2 + y2; + double cdist = 1./(1. + (d[0] + (d[1] + d[4]*r2)*r2)*r2); // (1. + (d[5] + (d[6] + d[7]*r2)*r2)*r2) == 1 as d[5-7]=0; + double x_ = (x - (d[2]*2.*x*y + d[3]*(r2 + 2.*x2)))*cdist; + double y_ = (y - (d[3]*2.*x*y + d[2]*(r2 + 2.*y2)))*cdist; + + // Rectify + double X = R[0]*x_ + R[1]*y_ + R[2]; + double Y = R[3]*x_ + R[4]*y_ + R[5]; + double Z = R[6]*x_ + R[7]*y_ + R[8]; + double x__ = X/Z; + double y__ = Y/Z; + + // Convert from pin-hole to image coordinates + dst.at(v, u) = Vec2f((float)(x__*ifx + cxn), (float)(y__*ify + cyn)); + } + } + + // Check Result + EXPECT_LE(cvtest::norm(dst, mapxy, NORM_INF), 2e-1); +} + }} // namespace From 9557b9f70f18785d44e3f21155f4f9f8859e7c0f Mon Sep 17 00:00:00 2001 From: Developer-Ecosystem-Engineering <65677710+Developer-Ecosystem-Engineering@users.noreply.github.com> Date: Thu, 17 Jun 2021 10:14:48 -0700 Subject: [PATCH 007/376] Improve SIFT for arm64/Apple silicon - Reduce branch density by collapsing compares. - Fix windows build errors - Use OpenCV universal intrinsics - Use v_check_any and v_signmask as requested --- modules/features2d/src/sift.simd.hpp | 195 ++++++++++++++++++++++++--- 1 file changed, 174 insertions(+), 21 deletions(-) diff --git a/modules/features2d/src/sift.simd.hpp b/modules/features2d/src/sift.simd.hpp index b5033459b957..60129b1535b5 100644 --- a/modules/features2d/src/sift.simd.hpp +++ b/modules/features2d/src/sift.simd.hpp @@ -450,31 +450,184 @@ class findScaleSpaceExtremaT const sift_wt* currptr = img.ptr(r); const sift_wt* prevptr = prev.ptr(r); const sift_wt* nextptr = next.ptr(r); + int c = SIFT_IMG_BORDER; - for( int c = SIFT_IMG_BORDER; c < cols-SIFT_IMG_BORDER; c++) +#if CV_SIMD && !(DoG_TYPE_SHORT) + const int vecsize = v_float32::nlanes; + for( ; c <= cols-SIFT_IMG_BORDER - vecsize; c += vecsize) + { + v_float32 val = vx_load(&currptr[c]); + v_float32 _00,_01,_02; + v_float32 _10, _12; + v_float32 _20,_21,_22; + + v_float32 vmin,vmax; + + + v_float32 cond = v_abs(val) > vx_setall_f32((float)threshold); + if (!v_check_any(cond)) + { + continue; + } + + _00 = vx_load(&currptr[c-step-1]); _01 = vx_load(&currptr[c-step]); _02 = vx_load(&currptr[c-step+1]); + _10 = vx_load(&currptr[c -1]); _12 = vx_load(&currptr[c +1]); + _20 = vx_load(&currptr[c+step-1]); _21 = vx_load(&currptr[c+step]); _22 = vx_load(&currptr[c+step+1]); + + vmax = v_max(v_max(v_max(_00,_01),v_max(_02,_10)),v_max(v_max(_12,_20),v_max(_21,_22))); + vmin = v_min(v_min(v_min(_00,_01),v_min(_02,_10)),v_min(v_min(_12,_20),v_min(_21,_22))); + + v_float32 condp = cond & (val > vx_setall_f32(0)) & (val >= vmax); + v_float32 condm = cond & (val < vx_setall_f32(0)) & (val <= vmin); + + cond = condp | condm; + if (!v_check_any(cond)) + { + continue; + } + + _00 = vx_load(&prevptr[c-step-1]); _01 = vx_load(&prevptr[c-step]); _02 = vx_load(&prevptr[c-step+1]); + _10 = vx_load(&prevptr[c -1]); _12 = vx_load(&prevptr[c +1]); + _20 = vx_load(&prevptr[c+step-1]); _21 = vx_load(&prevptr[c+step]); _22 = vx_load(&prevptr[c+step+1]); + + vmax = v_max(v_max(v_max(_00,_01),v_max(_02,_10)),v_max(v_max(_12,_20),v_max(_21,_22))); + vmin = v_min(v_min(v_min(_00,_01),v_min(_02,_10)),v_min(v_min(_12,_20),v_min(_21,_22))); + + condp &= (val >= vmax); + condm &= (val <= vmin); + + cond = condp | condm; + if (!v_check_any(cond)) + { + continue; + } + + v_float32 _11p = vx_load(&prevptr[c]); + v_float32 _11n = vx_load(&nextptr[c]); + + v_float32 max_middle = v_max(_11n,_11p); + v_float32 min_middle = v_min(_11n,_11p); + + _00 = vx_load(&nextptr[c-step-1]); _01 = vx_load(&nextptr[c-step]); _02 = vx_load(&nextptr[c-step+1]); + _10 = vx_load(&nextptr[c -1]); _12 = vx_load(&nextptr[c +1]); + _20 = vx_load(&nextptr[c+step-1]); _21 = vx_load(&nextptr[c+step]); _22 = vx_load(&nextptr[c+step+1]); + + vmax = v_max(v_max(v_max(_00,_01),v_max(_02,_10)),v_max(v_max(_12,_20),v_max(_21,_22))); + vmin = v_min(v_min(v_min(_00,_01),v_min(_02,_10)),v_min(v_min(_12,_20),v_min(_21,_22))); + + condp &= (val >= v_max(vmax,max_middle)); + condm &= (val <= v_min(vmin,min_middle)); + + cond = condp | condm; + if (!v_check_any(cond)) + { + continue; + } + + int mask = v_signmask(cond); + for (int k = 0; k 0 ? j - 1 : n - 1; + int r2 = j < n-1 ? j + 1 : 0; + + if( hist[j] > hist[l] && hist[j] > hist[r2] && hist[j] >= mag_thr ) + { + float bin = j + 0.5f * (hist[l]-hist[r2]) / (hist[l] - 2*hist[j] + hist[r2]); + bin = bin < 0 ? n + bin : bin >= n ? bin - n : bin; + kpt.angle = 360.f - (float)((360.f/n) * bin); + if(std::abs(kpt.angle - 360.f) < FLT_EPSILON) + kpt.angle = 0.f; + + kpts_.push_back(kpt); + } + } + } + } + +#endif //CV_SIMD && !(DoG_TYPE_SHORT) + + // vector loop reminder, better predictibility and less branch density + for( ; c < cols-SIFT_IMG_BORDER; c++) { sift_wt val = currptr[c]; + if (std::abs(val) <= threshold) + continue; + + sift_wt _00,_01,_02; + sift_wt _10, _12; + sift_wt _20,_21,_22; + _00 = currptr[c-step-1]; _01 = currptr[c-step]; _02 = currptr[c-step+1]; + _10 = currptr[c -1]; _12 = currptr[c +1]; + _20 = currptr[c+step-1]; _21 = currptr[c+step]; _22 = currptr[c+step+1]; + + bool calculate = false; + if (val > 0) + { + sift_wt vmax = std::max(std::max(std::max(_00,_01),std::max(_02,_10)),std::max(std::max(_12,_20),std::max(_21,_22))); + if (val >= vmax) + { + _00 = prevptr[c-step-1]; _01 = prevptr[c-step]; _02 = prevptr[c-step+1]; + _10 = prevptr[c -1]; _12 = prevptr[c +1]; + _20 = prevptr[c+step-1]; _21 = prevptr[c+step]; _22 = prevptr[c+step+1]; + vmax = std::max(std::max(std::max(_00,_01),std::max(_02,_10)),std::max(std::max(_12,_20),std::max(_21,_22))); + if (val >= vmax) + { + _00 = nextptr[c-step-1]; _01 = nextptr[c-step]; _02 = nextptr[c-step+1]; + _10 = nextptr[c -1]; _12 = nextptr[c +1]; + _20 = nextptr[c+step-1]; _21 = nextptr[c+step]; _22 = nextptr[c+step+1]; + vmax = std::max(std::max(std::max(_00,_01),std::max(_02,_10)),std::max(std::max(_12,_20),std::max(_21,_22))); + if (val >= vmax) + { + sift_wt _11p = prevptr[c], _11n = nextptr[c]; + calculate = (val >= std::max(_11p,_11n)); + } + } + } + + } else { // val cant be zero here (first abs took care of zero), must be negative + sift_wt vmin = std::min(std::min(std::min(_00,_01),std::min(_02,_10)),std::min(std::min(_12,_20),std::min(_21,_22))); + if (val <= vmin) + { + _00 = prevptr[c-step-1]; _01 = prevptr[c-step]; _02 = prevptr[c-step+1]; + _10 = prevptr[c -1]; _12 = prevptr[c +1]; + _20 = prevptr[c+step-1]; _21 = prevptr[c+step]; _22 = prevptr[c+step+1]; + vmin = std::min(std::min(std::min(_00,_01),std::min(_02,_10)),std::min(std::min(_12,_20),std::min(_21,_22))); + if (val <= vmin) + { + _00 = nextptr[c-step-1]; _01 = nextptr[c-step]; _02 = nextptr[c-step+1]; + _10 = nextptr[c -1]; _12 = nextptr[c +1]; + _20 = nextptr[c+step-1]; _21 = nextptr[c+step]; _22 = nextptr[c+step+1]; + vmin = std::min(std::min(std::min(_00,_01),std::min(_02,_10)),std::min(std::min(_12,_20),std::min(_21,_22))); + if (val <= vmin) + { + sift_wt _11p = prevptr[c], _11n = nextptr[c]; + calculate = (val <= std::min(_11p,_11n)); + } + } + } + } - // find local extrema with pixel accuracy - if( std::abs(val) > threshold && - ((val > 0 && val >= currptr[c-1] && val >= currptr[c+1] && - val >= currptr[c-step-1] && val >= currptr[c-step] && val >= currptr[c-step+1] && - val >= currptr[c+step-1] && val >= currptr[c+step] && val >= currptr[c+step+1] && - val >= nextptr[c] && val >= nextptr[c-1] && val >= nextptr[c+1] && - val >= nextptr[c-step-1] && val >= nextptr[c-step] && val >= nextptr[c-step+1] && - val >= nextptr[c+step-1] && val >= nextptr[c+step] && val >= nextptr[c+step+1] && - val >= prevptr[c] && val >= prevptr[c-1] && val >= prevptr[c+1] && - val >= prevptr[c-step-1] && val >= prevptr[c-step] && val >= prevptr[c-step+1] && - val >= prevptr[c+step-1] && val >= prevptr[c+step] && val >= prevptr[c+step+1]) || - (val < 0 && val <= currptr[c-1] && val <= currptr[c+1] && - val <= currptr[c-step-1] && val <= currptr[c-step] && val <= currptr[c-step+1] && - val <= currptr[c+step-1] && val <= currptr[c+step] && val <= currptr[c+step+1] && - val <= nextptr[c] && val <= nextptr[c-1] && val <= nextptr[c+1] && - val <= nextptr[c-step-1] && val <= nextptr[c-step] && val <= nextptr[c-step+1] && - val <= nextptr[c+step-1] && val <= nextptr[c+step] && val <= nextptr[c+step+1] && - val <= prevptr[c] && val <= prevptr[c-1] && val <= prevptr[c+1] && - val <= prevptr[c-step-1] && val <= prevptr[c-step] && val <= prevptr[c-step+1] && - val <= prevptr[c+step-1] && val <= prevptr[c+step] && val <= prevptr[c+step+1]))) + if (calculate) { CV_TRACE_REGION("pixel_candidate"); From 3a15a3821ac147e2715c2a9364bc02c6d789177d Mon Sep 17 00:00:00 2001 From: Zhang Yin Date: Thu, 10 Jun 2021 08:08:20 +0000 Subject: [PATCH 008/376] Update RISC-V back-end to RVV 0.10 --- cmake/checks/cpu_rvv.cpp | 2 +- .../include/opencv2/core/hal/intrin_rvv.hpp | 1292 ++++++++--------- platforms/linux/riscv64-gcc.toolchain.cmake | 4 +- 3 files changed, 582 insertions(+), 716 deletions(-) diff --git a/cmake/checks/cpu_rvv.cpp b/cmake/checks/cpu_rvv.cpp index a3eab2abc44e..684b2ecbebfa 100644 --- a/cmake/checks/cpu_rvv.cpp +++ b/cmake/checks/cpu_rvv.cpp @@ -9,7 +9,7 @@ int test() { const float src[] = { 0.0f, 0.0f, 0.0f, 0.0f }; - vfloat32m1_t val = vle32_v_f32m1((const float*)(src)); + vfloat32m1_t val = vle32_v_f32m1((const float*)(src), 4); return (int)vfmv_f_s_f32m1_f32(val); } #else diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp index cb2140df585b..4a3455b07385 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp @@ -151,12 +151,14 @@ struct vint8mf4_t }; #define OPENCV_HAL_IMPL_RVV_NATIVE_LOADSTORE_MF2(_Tpvec, _Tp, suffix, width, n) \ -inline _Tpvec vle##width##_v_##suffix##mf2(const _Tp* ptr) \ +inline _Tpvec vle##width##_v_##suffix##mf2(const _Tp* ptr, size_t vl) \ { \ + CV_UNUSED(vl); \ return _Tpvec(ptr); \ } \ -inline void vse##width##_v_##suffix##mf2(_Tp* ptr, _Tpvec v) \ +inline void vse##width##_v_##suffix##mf2(_Tp* ptr, _Tpvec v, size_t vl) \ { \ + CV_UNUSED(vl); \ for (int i = 0; i < n; ++i) \ { \ ptr[i] = v.val[i]; \ @@ -176,15 +178,14 @@ OPENCV_HAL_IMPL_RVV_NATIVE_LOADSTORE_MF2(vfloat64mf2_t, float64_t, f64, 64, 1) #define OPENCV_HAL_IMPL_RVV_NATIVE_WCVT(_Tpwvec, _Tpvec, _wTp, wcvt, suffix, width, n) \ -inline _Tpwvec wcvt (_Tpvec v) \ +inline _Tpwvec wcvt (_Tpvec v, size_t vl) \ { \ _wTp tmp[n]; \ for (int i = 0; i < n; ++i) \ { \ tmp[i] = (_wTp)v.val[i]; \ } \ - vsetvlmax_e##width##m1(); \ - return vle##width##_v_##suffix##m1(tmp); \ + return vle##width##_v_##suffix##m1(tmp, vl); \ } OPENCV_HAL_IMPL_RVV_NATIVE_WCVT(vuint16m1_t, vuint8mf2_t, ushort, vwcvtu_x_x_v_u16m1, u16, 16, 8) @@ -194,32 +195,34 @@ OPENCV_HAL_IMPL_RVV_NATIVE_WCVT(vint32m1_t, vint16mf2_t, int, vwcvt_x_x_v_i32m1, OPENCV_HAL_IMPL_RVV_NATIVE_WCVT(vuint64m1_t, vuint32mf2_t, uint64, vwcvtu_x_x_v_u64m1, u64, 64, 2) OPENCV_HAL_IMPL_RVV_NATIVE_WCVT(vint64m1_t, vint32mf2_t, int64, vwcvt_x_x_v_i64m1, i64, 64, 2) -inline vuint8mf4_t vle8_v_u8mf4 (const uint8_t *base) +inline vuint8mf4_t vle8_v_u8mf4 (const uint8_t *base, size_t vl) { + CV_UNUSED(vl); return vuint8mf4_t(base); } -inline vint8mf4_t vle8_v_i8mf4 (const int8_t *base) +inline vint8mf4_t vle8_v_i8mf4 (const int8_t *base, size_t vl) { + CV_UNUSED(vl); return vint8mf4_t(base); } -inline vuint16mf2_t vwcvtu_x_x_v_u16mf2 (vuint8mf4_t src) +inline vuint16mf2_t vwcvtu_x_x_v_u16mf2 (vuint8mf4_t src, size_t vl) { ushort tmp[4]; for (int i = 0; i < 4; ++i) { tmp[i] = (ushort)src.val[i]; } - return vle16_v_u16mf2(tmp); + return vle16_v_u16mf2(tmp, vl); } -inline vint16mf2_t vwcvt_x_x_v_i16mf2 (vint8mf4_t src) +inline vint16mf2_t vwcvt_x_x_v_i16mf2 (vint8mf4_t src, size_t vl) { short tmp[4]; for (int i = 0; i < 4; ++i) { tmp[i] = (short)src.val[i]; } - return vle16_v_i16mf2(tmp); + return vle16_v_i16mf2(tmp, vl); } //////////// Types //////////// @@ -232,8 +235,7 @@ struct v_uint8x16 v_uint8x16() {} explicit v_uint8x16(vuint8m1_t v) { - vsetvlmax_e8m1(); - vse8_v_u8m1(val, v); + vse8_v_u8m1(val, v, nlanes); } v_uint8x16(uchar v0, uchar v1, uchar v2, uchar v3, uchar v4, uchar v5, uchar v6, uchar v7, uchar v8, uchar v9, uchar v10, uchar v11, uchar v12, uchar v13, uchar v14, uchar v15) @@ -246,8 +248,7 @@ struct v_uint8x16 } operator vuint8m1_t() const { - vsetvlmax_e8m1(); - return vle8_v_u8m1(val); + return vle8_v_u8m1(val, nlanes); } uchar get0() const { @@ -265,8 +266,7 @@ struct v_int8x16 v_int8x16() {} explicit v_int8x16(vint8m1_t v) { - vsetvlmax_e8m1(); - vse8_v_i8m1(val, v); + vse8_v_i8m1(val, v, nlanes); } v_int8x16(schar v0, schar v1, schar v2, schar v3, schar v4, schar v5, schar v6, schar v7, schar v8, schar v9, schar v10, schar v11, schar v12, schar v13, schar v14, schar v15) @@ -279,8 +279,7 @@ struct v_int8x16 } operator vint8m1_t() const { - vsetvlmax_e8m1(); - return vle8_v_i8m1(val); + return vle8_v_i8m1(val, nlanes); } schar get0() const { @@ -298,8 +297,7 @@ struct v_uint16x8 v_uint16x8() {} explicit v_uint16x8(vuint16m1_t v) { - vsetvlmax_e16m1(); - vse16_v_u16m1(val, v); + vse16_v_u16m1(val, v, nlanes); } v_uint16x8(ushort v0, ushort v1, ushort v2, ushort v3, ushort v4, ushort v5, ushort v6, ushort v7) { @@ -311,8 +309,7 @@ struct v_uint16x8 } operator vuint16m1_t() const { - vsetvlmax_e16m1(); - return vle16_v_u16m1(val); + return vle16_v_u16m1(val, nlanes); } ushort get0() const { @@ -330,8 +327,7 @@ struct v_int16x8 v_int16x8() {} explicit v_int16x8(vint16m1_t v) { - vsetvlmax_e16m1(); - vse16_v_i16m1(val, v); + vse16_v_i16m1(val, v, nlanes); } v_int16x8(short v0, short v1, short v2, short v3, short v4, short v5, short v6, short v7) { @@ -343,8 +339,7 @@ struct v_int16x8 } operator vint16m1_t() const { - vsetvlmax_e16m1(); - return vle16_v_i16m1(val); + return vle16_v_i16m1(val, nlanes); } short get0() const { @@ -362,8 +357,7 @@ struct v_uint32x4 v_uint32x4() {} explicit v_uint32x4(vuint32m1_t v) { - vsetvlmax_e32m1(); - vse32_v_u32m1(val, v); + vse32_v_u32m1(val, v, nlanes); } v_uint32x4(unsigned v0, unsigned v1, unsigned v2, unsigned v3) { @@ -375,8 +369,7 @@ struct v_uint32x4 } operator vuint32m1_t() const { - vsetvlmax_e32m1(); - return vle32_v_u32m1(val); + return vle32_v_u32m1(val, nlanes); } unsigned get0() const { @@ -394,8 +387,7 @@ struct v_int32x4 v_int32x4() {} explicit v_int32x4(vint32m1_t v) { - vsetvlmax_e32m1(); - vse32_v_i32m1(val, v); + vse32_v_i32m1(val, v, nlanes); } v_int32x4(int v0, int v1, int v2, int v3) { @@ -407,8 +399,7 @@ struct v_int32x4 } operator vint32m1_t() const { - vsetvlmax_e32m1(); - return vle32_v_i32m1(val); + return vle32_v_i32m1(val, nlanes); } int get0() const { @@ -425,8 +416,7 @@ struct v_float32x4 v_float32x4() {} explicit v_float32x4(vfloat32m1_t v) { - vsetvlmax_e32m1(); - vse32_v_f32m1(val, v); + vse32_v_f32m1(val, v, nlanes); } v_float32x4(float v0, float v1, float v2, float v3) { @@ -438,8 +428,7 @@ struct v_float32x4 } operator vfloat32m1_t() const { - vsetvlmax_e32m1(); - return vle32_v_f32m1(val); + return vle32_v_f32m1(val, nlanes); } float get0() const { @@ -456,8 +445,7 @@ struct v_uint64x2 v_uint64x2() {} explicit v_uint64x2(vuint64m1_t v) { - vsetvlmax_e64m1(); - vse64_v_u64m1(val, v); + vse64_v_u64m1(val, v, nlanes); } v_uint64x2(uint64 v0, uint64 v1) { @@ -469,8 +457,7 @@ struct v_uint64x2 } operator vuint64m1_t() const { - vsetvlmax_e64m1(); - return vle64_v_u64m1(val); + return vle64_v_u64m1(val, nlanes); } uint64 get0() const { @@ -488,8 +475,7 @@ struct v_int64x2 v_int64x2() {} explicit v_int64x2(vint64m1_t v) { - vsetvlmax_e64m1(); - vse64_v_i64m1(val, v); + vse64_v_i64m1(val, v, nlanes); } v_int64x2(int64 v0, int64 v1) { @@ -501,8 +487,7 @@ struct v_int64x2 } operator vint64m1_t() const { - vsetvlmax_e64m1(); - return vle64_v_i64m1(val); + return vle64_v_i64m1(val, nlanes); } int64 get0() const { @@ -521,8 +506,7 @@ struct v_float64x2 v_float64x2() {} explicit v_float64x2(vfloat64m1_t v) { - vsetvlmax_e64m1(); - vse64_v_f64m1(val, v); + vse64_v_f64m1(val, v, nlanes); } v_float64x2(double v0, double v1) { @@ -534,8 +518,7 @@ struct v_float64x2 } operator vfloat64m1_t() const { - vsetvlmax_e64m1(); - return vle64_v_f64m1(val); + return vle64_v_f64m1(val, nlanes); } double get0() const { @@ -549,42 +532,38 @@ struct v_float64x2 //////////// Initial //////////// -#define OPENCV_HAL_IMPL_RVV_INIT_INTEGER(_Tpvec, _Tp, width, suffix1, suffix2) \ +#define OPENCV_HAL_IMPL_RVV_INIT_INTEGER(_Tpvec, _Tp, suffix1, suffix2, vl) \ inline v_##_Tpvec v_setzero_##suffix1() \ { \ - vsetvlmax_e##width##m1(); \ - return v_##_Tpvec(vzero_##suffix2##m1()); \ + return v_##_Tpvec(vmv_v_x_##suffix2##m1(0, vl)); \ } \ inline v_##_Tpvec v_setall_##suffix1(_Tp v) \ { \ - vsetvlmax_e##width##m1(); \ - return v_##_Tpvec(vmv_v_x_##suffix2##m1(v)); \ + return v_##_Tpvec(vmv_v_x_##suffix2##m1(v, vl)); \ } -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint8x16, uchar, 8, u8, u8) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int8x16, schar, 8, s8, i8) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint16x8, ushort, 16, u16, u16) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int16x8, short, 16, s16, i16) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint32x4, unsigned, 32, u32, u32) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int32x4, int, 32, s32, i32) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint64x2, uint64, 64, u64, u64) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int64x2, int64, 64, s64, i64) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint8x16, uchar, u8, u8, 16) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int8x16, schar, s8, i8, 16) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint16x8, ushort, u16, u16, 8) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int16x8, short, s16, i16, 8) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint32x4, unsigned, u32, u32, 4) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int32x4, int, s32, i32, 4) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint64x2, uint64, u64, u64, 2) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int64x2, int64, s64, i64, 2) -#define OPENCV_HAL_IMPL_RVV_INIT_FP(_Tpv, _Tp, width, suffix) \ +#define OPENCV_HAL_IMPL_RVV_INIT_FP(_Tpv, _Tp, suffix, vl) \ inline v_##_Tpv v_setzero_##suffix() \ { \ - vsetvlmax_e##width##m1(); \ - return v_##_Tpv(vzero_##suffix##m1()); \ + return v_##_Tpv(vfmv_v_f_##suffix##m1(0, vl)); \ } \ inline v_##_Tpv v_setall_##suffix(_Tp v) \ { \ - vsetvlmax_e##width##m1(); \ - return v_##_Tpv(vfmv_v_f_##suffix##m1(v)); \ + return v_##_Tpv(vfmv_v_f_##suffix##m1(v, vl)); \ } -OPENCV_HAL_IMPL_RVV_INIT_FP(float32x4, float, 32, f32) +OPENCV_HAL_IMPL_RVV_INIT_FP(float32x4, float, f32, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_INIT_FP(float64x2, double, 64, f64) +OPENCV_HAL_IMPL_RVV_INIT_FP(float64x2, double, f64, 2) #endif //////////// Reinterpret //////////// @@ -605,167 +584,155 @@ OPENCV_HAL_IMPL_RVV_SELF_REINTERPRET(int64x2, s64) OPENCV_HAL_IMPL_RVV_SELF_REINTERPRET(float64x2, f64) #endif -#define OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(_Tpvec1, _Tpvec2, _nTpvec1, _nTpvec2, suffix1, suffix2, nsuffix1, nsuffix2, width1, width2) \ +#define OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(_Tpvec1, _Tpvec2, _nTpvec1, _nTpvec2, suffix1, suffix2, nsuffix1, nsuffix2, width1, width2, vl1, vl2) \ inline v_##_Tpvec1 v_reinterpret_as_##suffix1(const v_##_Tpvec2& v) \ { \ - vsetvlmax_e##width2##m1(); \ - return v_##_Tpvec1((_nTpvec1)vle##width2##_v_##nsuffix2##m1(v.val)); \ + return v_##_Tpvec1((_nTpvec1)vle##width2##_v_##nsuffix2##m1(v.val, vl2)); \ } \ inline v_##_Tpvec2 v_reinterpret_as_##suffix2(const v_##_Tpvec1& v) \ { \ - vsetvlmax_e##width1##m1(); \ - return v_##_Tpvec2((_nTpvec2)vle##width1##_v_##nsuffix1##m1(v.val)); \ -} - -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int8x16, vuint8m1_t, vint8m1_t, u8, s8, u8, i8, 8, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int16x8, vuint16m1_t, vint16m1_t, u16, s16, u16, i16, 16, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int32x4, vuint32m1_t, vint32m1_t, u32, s32, u32, i32, 32, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, float32x4, vuint32m1_t, vfloat32m1_t, u32, f32, u32, f32, 32, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, float32x4, vint32m1_t, vfloat32m1_t, s32, f32, i32, f32, 32, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int64x2, vuint64m1_t, vint64m1_t, u64, s64, u64, i64, 64, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint16x8, vuint8m1_t, vuint16m1_t, u8, u16, u8, u16, 8, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint32x4, vuint8m1_t, vuint32m1_t, u8, u32, u8, u32, 8, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint64x2, vuint8m1_t, vuint64m1_t, u8, u64, u8, u64, 8, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, uint32x4, vuint16m1_t, vuint32m1_t, u16, u32, u16, u32, 16, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, uint64x2, vuint16m1_t, vuint64m1_t, u16, u64, u16, u64, 16, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, uint64x2, vuint32m1_t, vuint64m1_t, u32, u64, u32, u64, 32, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int16x8, vint8m1_t, vint16m1_t, s8, s16, i8, i16, 8, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int32x4, vint8m1_t, vint32m1_t, s8, s32, i8, i32, 8, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int64x2, vint8m1_t, vint64m1_t, s8, s64, i8, i64, 8, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, int32x4, vint16m1_t, vint32m1_t, s16, s32, i16, i32, 16, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, int64x2, vint16m1_t, vint64m1_t, s16, s64, i16, i64, 16, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, int64x2, vint32m1_t, vint64m1_t, s32, s64, i32, i64, 32, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int16x8, vuint8m1_t, vint16m1_t, u8, s16, u8, i16, 8, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int32x4, vuint8m1_t, vint32m1_t, u8, s32, u8, i32, 8, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int64x2, vuint8m1_t, vint64m1_t, u8, s64, u8, i64, 8, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int8x16, vuint16m1_t, vint8m1_t, u16, s8, u16, i8, 16, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int32x4, vuint16m1_t, vint32m1_t, u16, s32, u16, i32, 16, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int64x2, vuint16m1_t, vint64m1_t, u16, s64, u16, i64, 16, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int8x16, vuint32m1_t, vint8m1_t, u32, s8, u32, i8, 32, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int16x8, vuint32m1_t, vint16m1_t, u32, s16, u32, i16, 32, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int64x2, vuint32m1_t, vint64m1_t, u32, s64, u32, i64, 32, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int8x16, vuint64m1_t, vint8m1_t, u64, s8, u64, i8, 64, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int16x8, vuint64m1_t, vint16m1_t, u64, s16, u64, i16, 64, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int32x4, vuint64m1_t, vint32m1_t, u64, s32, u64, i32, 64, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, float32x4, vuint8m1_t, vfloat32m1_t, u8, f32, u8, f32, 8, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, float32x4, vuint16m1_t, vfloat32m1_t, u16, f32, u16, f32, 16, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, float32x4, vuint64m1_t, vfloat32m1_t, u64, f32, u64, f32, 64, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, float32x4, vint8m1_t, vfloat32m1_t, s8, f32, i8, f32, 8, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, float32x4, vint16m1_t, vfloat32m1_t, s16, f32, i16, f32, 16, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int64x2, float32x4, vint64m1_t, vfloat32m1_t, s64, f32, i64, f32, 64, 32) + return v_##_Tpvec2((_nTpvec2)vle##width1##_v_##nsuffix1##m1(v.val, vl1)); \ +} + +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int8x16, vuint8m1_t, vint8m1_t, u8, s8, u8, i8, 8, 8, 16, 16) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int16x8, vuint16m1_t, vint16m1_t, u16, s16, u16, i16, 16, 16, 8, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int32x4, vuint32m1_t, vint32m1_t, u32, s32, u32, i32, 32, 32, 4, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, float32x4, vuint32m1_t, vfloat32m1_t, u32, f32, u32, f32, 32, 32, 4, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, float32x4, vint32m1_t, vfloat32m1_t, s32, f32, i32, f32, 32, 32, 4, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int64x2, vuint64m1_t, vint64m1_t, u64, s64, u64, i64, 64, 64, 2, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint16x8, vuint8m1_t, vuint16m1_t, u8, u16, u8, u16, 8, 16, 16, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint32x4, vuint8m1_t, vuint32m1_t, u8, u32, u8, u32, 8, 32, 16, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint64x2, vuint8m1_t, vuint64m1_t, u8, u64, u8, u64, 8, 64, 16, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, uint32x4, vuint16m1_t, vuint32m1_t, u16, u32, u16, u32, 16, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, uint64x2, vuint16m1_t, vuint64m1_t, u16, u64, u16, u64, 16, 64, 8, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, uint64x2, vuint32m1_t, vuint64m1_t, u32, u64, u32, u64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int16x8, vint8m1_t, vint16m1_t, s8, s16, i8, i16, 8, 16, 16, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int32x4, vint8m1_t, vint32m1_t, s8, s32, i8, i32, 8, 32, 16, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int64x2, vint8m1_t, vint64m1_t, s8, s64, i8, i64, 8, 64, 16, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, int32x4, vint16m1_t, vint32m1_t, s16, s32, i16, i32, 16, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, int64x2, vint16m1_t, vint64m1_t, s16, s64, i16, i64, 16, 64, 8, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, int64x2, vint32m1_t, vint64m1_t, s32, s64, i32, i64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int16x8, vuint8m1_t, vint16m1_t, u8, s16, u8, i16, 8, 16, 16, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int32x4, vuint8m1_t, vint32m1_t, u8, s32, u8, i32, 8, 32, 16, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int64x2, vuint8m1_t, vint64m1_t, u8, s64, u8, i64, 8, 64, 16, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int8x16, vuint16m1_t, vint8m1_t, u16, s8, u16, i8, 16, 8, 8, 16) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int32x4, vuint16m1_t, vint32m1_t, u16, s32, u16, i32, 16, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int64x2, vuint16m1_t, vint64m1_t, u16, s64, u16, i64, 16, 64, 8, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int8x16, vuint32m1_t, vint8m1_t, u32, s8, u32, i8, 32, 8, 4, 16) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int16x8, vuint32m1_t, vint16m1_t, u32, s16, u32, i16, 32, 16, 4, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int64x2, vuint32m1_t, vint64m1_t, u32, s64, u32, i64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int8x16, vuint64m1_t, vint8m1_t, u64, s8, u64, i8, 64, 8, 2, 16) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int16x8, vuint64m1_t, vint16m1_t, u64, s16, u64, i16, 64, 16, 2, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int32x4, vuint64m1_t, vint32m1_t, u64, s32, u64, i32, 64, 32, 2, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, float32x4, vuint8m1_t, vfloat32m1_t, u8, f32, u8, f32, 8, 32, 16, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, float32x4, vuint16m1_t, vfloat32m1_t, u16, f32, u16, f32, 16, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, float32x4, vuint64m1_t, vfloat32m1_t, u64, f32, u64, f32, 64, 32, 2, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, float32x4, vint8m1_t, vfloat32m1_t, s8, f32, i8, f32, 8, 32, 16, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, float32x4, vint16m1_t, vfloat32m1_t, s16, f32, i16, f32, 16, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int64x2, float32x4, vint64m1_t, vfloat32m1_t, s64, f32, i64, f32, 64, 32, 2, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, float64x2, vuint64m1_t, vfloat64m1_t, u64, f64, u64, f64, 64, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int64x2, float64x2, vint64m1_t, vfloat64m1_t, s64, f64, i64, f64, 64, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, float64x2, vuint8m1_t, vfloat64m1_t, u8, f64, u8, f64, 8, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, float64x2, vuint16m1_t, vfloat64m1_t, u16, f64, u16, f64, 16, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, float64x2, vuint32m1_t, vfloat64m1_t, u32, f64, u32, f64, 32, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, float64x2, vint8m1_t, vfloat64m1_t, s8, f64, i8, f64, 8, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, float64x2, vint16m1_t, vfloat64m1_t, s16, f64, i16, f64, 16, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, float64x2, vint32m1_t, vfloat64m1_t, s32, f64, i32, f64, 32, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(float32x4, float64x2, vfloat32m1_t, vfloat64m1_t, f32, f64, f32, f64, 32, 64) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, float64x2, vuint64m1_t, vfloat64m1_t, u64, f64, u64, f64, 64, 64, 2, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int64x2, float64x2, vint64m1_t, vfloat64m1_t, s64, f64, i64, f64, 64, 64, 2, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, float64x2, vuint8m1_t, vfloat64m1_t, u8, f64, u8, f64, 8, 64, 16, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, float64x2, vuint16m1_t, vfloat64m1_t, u16, f64, u16, f64, 16, 64, 6, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, float64x2, vuint32m1_t, vfloat64m1_t, u32, f64, u32, f64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, float64x2, vint8m1_t, vfloat64m1_t, s8, f64, i8, f64, 8, 64, 16, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, float64x2, vint16m1_t, vfloat64m1_t, s16, f64, i16, f64, 16, 64, 8, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, float64x2, vint32m1_t, vfloat64m1_t, s32, f64, i32, f64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(float32x4, float64x2, vfloat32m1_t, vfloat64m1_t, f32, f64, f32, f64, 32, 64, 4, 2) #endif ////////////// Extract ////////////// -#define OPENCV_HAL_IMPL_RVV_EXTRACT(_Tpvec, _Tp, suffix, width, vmv) \ +#define OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(_Tpvec, _Tp, suffix, vmv, vl) \ template \ inline _Tpvec v_extract(const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vzero_##suffix##m1(), a, s), b, _Tpvec::nlanes - s)); \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), a, s, vl), b, _Tpvec::nlanes - s, vl)); \ } \ template inline _Tp v_extract_n(_Tpvec v) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tp(vmv(vslidedown_vx_##suffix##m1(vzero_##suffix##m1(), v, i))); \ + return _Tp(vmv(vslidedown_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), v, i, vl))); \ } -OPENCV_HAL_IMPL_RVV_EXTRACT(v_uint8x16, uchar, u8, 8, vmv_x_s_u8m1_u8) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_int8x16, schar, i8, 8, vmv_x_s_i8m1_i8) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_uint16x8, ushort, u16, 16, vmv_x_s_u16m1_u16) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_int16x8, short, i16, 16, vmv_x_s_i16m1_i16) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_uint32x4, uint, u32, 32, vmv_x_s_u32m1_u32) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_int32x4, int, i32, 32, vmv_x_s_i32m1_i32) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_uint64x2, uint64, u64, 64, vmv_x_s_u64m1_u64) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_int64x2, int64, i64, 64, vmv_x_s_i64m1_i64) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_float32x4, float, f32, 32, vfmv_f_s_f32m1_f32) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_uint8x16, uchar, u8, vmv_x_s_u8m1_u8, 16) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_int8x16, schar, i8, vmv_x_s_i8m1_i8, 16) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_uint16x8, ushort, u16, vmv_x_s_u16m1_u16, 8) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_int16x8, short, i16, vmv_x_s_i16m1_i16, 8) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_uint32x4, uint, u32, vmv_x_s_u32m1_u32, 4) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_int32x4, int, i32, vmv_x_s_i32m1_i32, 4) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_uint64x2, uint64, u64, vmv_x_s_u64m1_u64, 2) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_int64x2, int64, i64, vmv_x_s_i64m1_i64, 2) + +#define OPENCV_HAL_IMPL_RVV_EXTRACT_FP(_Tpvec, _Tp, suffix, vmv, vl) \ +template \ +inline _Tpvec v_extract(const _Tpvec& a, const _Tpvec& b) \ +{ \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), a, s, vl), b, _Tpvec::nlanes - s, vl)); \ +} \ +template inline _Tp v_extract_n(_Tpvec v) \ +{ \ + return _Tp(vmv(vslidedown_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), v, i, vl))); \ +} + +OPENCV_HAL_IMPL_RVV_EXTRACT_FP(v_float32x4, float, f32, vfmv_f_s_f32m1_f32, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_EXTRACT(v_float64x2, double, f64, 64, vfmv_f_s_f64m1_f64) +OPENCV_HAL_IMPL_RVV_EXTRACT_FP(v_float64x2, double, f64, vfmv_f_s_f64m1_f64, 2) #endif ////////////// Load/Store ////////////// -#define OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(_Tpvec, _nTpvec, _Tp, hvl, width, suffix) \ +#define OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(_Tpvec, _nTpvec, _Tp, hvl, vl, width, suffix, vmv) \ inline _Tpvec v_load(const _Tp* ptr) \ { \ - vsetvlmax_e8m1(); \ - return _Tpvec((_nTpvec)vle8_v_u8m1((uchar*)ptr)); \ + return _Tpvec((_nTpvec)vle8_v_u8m1((uchar*)ptr, 16)); \ } \ inline _Tpvec v_load_aligned(const _Tp* ptr) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vle##width##_v_##suffix##m1(ptr)); \ + return _Tpvec(vle##width##_v_##suffix##m1(ptr, vl)); \ } \ inline _Tpvec v_load_low(const _Tp* ptr) \ { \ - vsetvl_e##width##m1(hvl); \ - _Tpvec res = _Tpvec(vle##width##_v_##suffix##m1(ptr)); \ - vsetvlmax_e##width##m1(); \ + _Tpvec res = _Tpvec(vle##width##_v_##suffix##m1(ptr, hvl)); \ return res; \ } \ inline void v_store(_Tp* ptr, const _Tpvec& a) \ { \ - vsetvlmax_e8m1(); \ - vse8_v_u8m1((uchar*)ptr, vle8_v_u8m1((uchar*)a.val)); \ + vse8_v_u8m1((uchar*)ptr, vle8_v_u8m1((uchar*)a.val, 16), 16); \ } \ inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - vse##width##_v_##suffix##m1(ptr, a); \ + vse##width##_v_##suffix##m1(ptr, a, vl); \ } \ inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - vse##width##_v_##suffix##m1(ptr, a); \ + vse##width##_v_##suffix##m1(ptr, a, vl); \ } \ inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode /*mode*/) \ { \ - vsetvlmax_e##width##m1(); \ - vse##width##_v_##suffix##m1(ptr, a); \ + vse##width##_v_##suffix##m1(ptr, a, vl); \ } \ inline void v_store_low(_Tp* ptr, const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) tmp_ptr[_Tpvec::nlanes] = {0}; \ - vsetvlmax_e##width##m1(); \ - vse##width##_v_##suffix##m1(tmp_ptr, a); \ - for(int i = 0; i < _Tpvec::nlanes/2; ++i) \ - { \ - ptr[i] = tmp_ptr[i]; \ - } \ + vse##width##_v_##suffix##m1(ptr, a, hvl); \ } \ inline void v_store_high(_Tp* ptr, const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) tmp_ptr[_Tpvec::nlanes] = {0}; \ - vsetvlmax_e##width##m1(); \ - vse##width##_v_##suffix##m1(tmp_ptr, a); \ - for(int i = 0; i < _Tpvec::nlanes/2; ++i) \ - { \ - ptr[i] = tmp_ptr[i+_Tpvec::nlanes/2]; \ - } \ + vse##width##_v_##suffix##m1(ptr, vslidedown_vx_##suffix##m1(vmv(0, vl), a, hvl, vl), hvl); \ } -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint8x16, vuint8m1_t, uchar, 8, 8, u8) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int8x16, vint8m1_t, schar, 8, 8, i8) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint16x8, vuint16m1_t, ushort, 4, 16, u16) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int16x8, vint16m1_t, short, 4, 16, i16) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint32x4, vuint32m1_t, unsigned, 2, 32, u32) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int32x4, vint32m1_t, int, 2, 32, i32) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint64x2, vuint64m1_t, uint64, 1, 64, u64) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int64x2, vint64m1_t, int64, 1, 64, i64) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float32x4, vfloat32m1_t, float, 2, 32, f32) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint8x16, vuint8m1_t, uchar, 8, 16, 8, u8, vmv_v_x_u8m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int8x16, vint8m1_t, schar, 8, 16, 8, i8, vmv_v_x_i8m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint16x8, vuint16m1_t, ushort, 4, 8, 16, u16, vmv_v_x_u16m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int16x8, vint16m1_t, short, 4, 8, 16, i16, vmv_v_x_i16m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint32x4, vuint32m1_t, unsigned, 2, 4, 32, u32, vmv_v_x_u32m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int32x4, vint32m1_t, int, 2, 4, 32, i32, vmv_v_x_i32m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint64x2, vuint64m1_t, uint64, 1, 2, 64, u64, vmv_v_x_u64m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int64x2, vint64m1_t, int64, 1, 2, 64, i64, vmv_v_x_i64m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float32x4, vfloat32m1_t, float, 2, 4, 32, f32, vfmv_v_f_f32m1) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float64x2, vfloat64m1_t, double, 1, 64, f64) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float64x2, vfloat64m1_t, double, 1, 2, 64, f64, vfmv_v_f_f64m1) #endif inline v_int8x16 v_load_halves(const schar* ptr0, const schar* ptr1) @@ -775,8 +742,7 @@ inline v_int8x16 v_load_halves(const schar* ptr0, const schar* ptr1) ptr0[0], ptr0[1], ptr0[2], ptr0[3], ptr0[4], ptr0[5], ptr0[6], ptr0[7], ptr1[0], ptr1[1], ptr1[2], ptr1[3], ptr1[4], ptr1[5], ptr1[6], ptr1[7] }; - vsetvlmax_e8m1(); - return v_int8x16(vle8_v_i8m1(elems)); + return v_int8x16(vle8_v_i8m1(elems, 16)); } inline v_uint8x16 v_load_halves(const uchar* ptr0, const uchar* ptr1) { return v_reinterpret_as_u8(v_load_halves((schar*)ptr0, (schar*)ptr1)); } @@ -786,8 +752,7 @@ inline v_int16x8 v_load_halves(const short* ptr0, const short* ptr1) { ptr0[0], ptr0[1], ptr0[2], ptr0[3], ptr1[0], ptr1[1], ptr1[2], ptr1[3] }; - vsetvlmax_e16m1(); - return v_int16x8(vle16_v_i16m1(elems)); + return v_int16x8(vle16_v_i16m1(elems, 8)); } inline v_uint16x8 v_load_halves(const ushort* ptr0, const ushort* ptr1) { return v_reinterpret_as_u16(v_load_halves((short*)ptr0, (short*)ptr1)); } @@ -797,8 +762,7 @@ inline v_int32x4 v_load_halves(const int* ptr0, const int* ptr1) { ptr0[0], ptr0[1], ptr1[0], ptr1[1] }; - vsetvlmax_e32m1(); - return v_int32x4(vle32_v_i32m1(elems)); + return v_int32x4(vle32_v_i32m1(elems, 4)); } inline v_float32x4 v_load_halves(const float* ptr0, const float* ptr1) { @@ -806,8 +770,7 @@ inline v_float32x4 v_load_halves(const float* ptr0, const float* ptr1) { ptr0[0], ptr0[1], ptr1[0], ptr1[1] }; - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(elems)); + return v_float32x4(vle32_v_f32m1(elems, 4)); } inline v_uint32x4 v_load_halves(const unsigned* ptr0, const unsigned* ptr1) { return v_reinterpret_as_u32(v_load_halves((int*)ptr0, (int*)ptr1)); } @@ -817,8 +780,7 @@ inline v_int64x2 v_load_halves(const int64* ptr0, const int64* ptr1) { ptr0[0], ptr1[0] }; - vsetvlmax_e64m1(); - return v_int64x2(vle64_v_i64m1(elems)); + return v_int64x2(vle64_v_i64m1(elems, 2)); } inline v_uint64x2 v_load_halves(const uint64* ptr0, const uint64* ptr1) { return v_reinterpret_as_u64(v_load_halves((int64*)ptr0, (int64*)ptr1)); } @@ -829,8 +791,7 @@ inline v_float64x2 v_load_halves(const double* ptr0, const double* ptr1) { ptr0[0], ptr1[0] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } #endif @@ -858,8 +819,7 @@ inline v_int8x16 v_lut(const schar* tab, const int* idx) tab[idx[14]], tab[idx[15]] }; - vsetvlmax_e8m1(); - return v_int8x16(vle8_v_i8m1(elems)); + return v_int8x16(vle8_v_i8m1(elems, 16)); } inline v_int8x16 v_lut_pairs(const schar* tab, const int* idx) { @@ -882,8 +842,7 @@ inline v_int8x16 v_lut_pairs(const schar* tab, const int* idx) tab[idx[7]], tab[idx[7] + 1] }; - vsetvlmax_e8m1(); - return v_int8x16(vle8_v_i8m1(elems)); + return v_int8x16(vle8_v_i8m1(elems, 16)); } inline v_int8x16 v_lut_quads(const schar* tab, const int* idx) { @@ -906,8 +865,7 @@ inline v_int8x16 v_lut_quads(const schar* tab, const int* idx) tab[idx[3] + 2], tab[idx[3] + 3] }; - vsetvlmax_e8m1(); - return v_int8x16(vle8_v_i8m1(elems)); + return v_int8x16(vle8_v_i8m1(elems, 16)); } inline v_uint8x16 v_lut(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut((schar*)tab, idx)); } inline v_uint8x16 v_lut_pairs(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut_pairs((schar*)tab, idx)); } @@ -926,8 +884,7 @@ inline v_int16x8 v_lut(const short* tab, const int* idx) tab[idx[6]], tab[idx[7]] }; - vsetvlmax_e16m1(); - return v_int16x8(vle16_v_i16m1(elems)); + return v_int16x8(vle16_v_i16m1(elems, 8)); } inline v_int16x8 v_lut_pairs(const short* tab, const int* idx) { @@ -942,8 +899,7 @@ inline v_int16x8 v_lut_pairs(const short* tab, const int* idx) tab[idx[3]], tab[idx[3] + 1] }; - vsetvlmax_e16m1(); - return v_int16x8(vle16_v_i16m1(elems)); + return v_int16x8(vle16_v_i16m1(elems, 8)); } inline v_int16x8 v_lut_quads(const short* tab, const int* idx) { @@ -958,8 +914,7 @@ inline v_int16x8 v_lut_quads(const short* tab, const int* idx) tab[idx[1] + 2], tab[idx[1] + 3] }; - vsetvlmax_e16m1(); - return v_int16x8(vle16_v_i16m1(elems)); + return v_int16x8(vle16_v_i16m1(elems, 8)); } inline v_uint16x8 v_lut(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v_lut((short*)tab, idx)); } inline v_uint16x8 v_lut_pairs(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v_lut_pairs((short*)tab, idx)); } @@ -974,8 +929,7 @@ inline v_int32x4 v_lut(const int* tab, const int* idx) tab[idx[2]], tab[idx[3]] }; - vsetvlmax_e32m1(); - return v_int32x4(vle32_v_i32m1(elems)); + return v_int32x4(vle32_v_i32m1(elems, 4)); } inline v_int32x4 v_lut_pairs(const int* tab, const int* idx) { @@ -986,13 +940,11 @@ inline v_int32x4 v_lut_pairs(const int* tab, const int* idx) tab[idx[1]], tab[idx[1] + 1] }; - vsetvlmax_e32m1(); - return v_int32x4(vle32_v_i32m1(elems)); + return v_int32x4(vle32_v_i32m1(elems, 4)); } inline v_int32x4 v_lut_quads(const int* tab, const int* idx) { - vsetvlmax_e32m1(); - return v_int32x4(vle32_v_i32m1(tab + idx[0])); + return v_int32x4(vle32_v_i32m1(tab + idx[0], 4)); } inline v_uint32x4 v_lut(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v_lut((int*)tab, idx)); } @@ -1006,13 +958,11 @@ inline v_int64x2 v_lut(const int64_t* tab, const int* idx) tab[idx[0]], tab[idx[1]] }; - vsetvlmax_e64m1(); - return v_int64x2(vle64_v_i64m1(elems)); + return v_int64x2(vle64_v_i64m1(elems, 2)); } inline v_int64x2 v_lut_pairs(const int64* tab, const int* idx) { - vsetvlmax_e64m1(); - return v_int64x2(vle64_v_i64m1(tab + idx[0])); + return v_int64x2(vle64_v_i64m1(tab + idx[0], 2)); } inline v_uint64x2 v_lut(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v_lut((const int64_t *)tab, idx)); } inline v_uint64x2 v_lut_pairs(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v_lut_pairs((const int64_t *)tab, idx)); } @@ -1026,8 +976,7 @@ inline v_float32x4 v_lut(const float* tab, const int* idx) tab[idx[2]], tab[idx[3]] }; - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(elems)); + return v_float32x4(vle32_v_f32m1(elems, 4)); } inline v_float32x4 v_lut_pairs(const float* tab, const int* idx) { @@ -1038,13 +987,11 @@ inline v_float32x4 v_lut_pairs(const float* tab, const int* idx) tab[idx[1]], tab[idx[1] + 1] }; - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(elems)); + return v_float32x4(vle32_v_f32m1(elems, 4)); } inline v_float32x4 v_lut_quads(const float* tab, const int* idx) { - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(tab + idx[0])); + return v_float32x4(vle32_v_f32m1(tab + idx[0], 4)); } inline v_int32x4 v_lut(const int* tab, const v_int32x4& idxvec) @@ -1056,8 +1003,7 @@ inline v_int32x4 v_lut(const int* tab, const v_int32x4& idxvec) tab[v_extract_n<2>(idxvec)], tab[v_extract_n<3>(idxvec)] }; - vsetvlmax_e32m1(); - return v_int32x4(vle32_v_i32m1(elems)); + return v_int32x4(vle32_v_i32m1(elems, 4)); } inline v_uint32x4 v_lut(const unsigned* tab, const v_int32x4& idxvec) @@ -1069,8 +1015,7 @@ inline v_uint32x4 v_lut(const unsigned* tab, const v_int32x4& idxvec) tab[v_extract_n<2>(idxvec)], tab[v_extract_n<3>(idxvec)] }; - vsetvlmax_e32m1(); - return v_uint32x4(vle32_v_u32m1(elems)); + return v_uint32x4(vle32_v_u32m1(elems, 4)); } inline v_float32x4 v_lut(const float* tab, const v_int32x4& idxvec) @@ -1082,8 +1027,7 @@ inline v_float32x4 v_lut(const float* tab, const v_int32x4& idxvec) tab[v_extract_n<2>(idxvec)], tab[v_extract_n<3>(idxvec)] }; - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(elems)); + return v_float32x4(vle32_v_f32m1(elems, 4)); } inline void v_lut_deinterleave(const float* tab, const v_int32x4& idxvec, v_float32x4& x, v_float32x4& y) @@ -1103,14 +1047,12 @@ inline v_float64x2 v_lut(const double* tab, const int* idx) tab[idx[0]], tab[idx[1]] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline v_float64x2 v_lut_pairs(const double* tab, const int* idx) { - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(tab + idx[0])); + return v_float64x2(vle64_v_f64m1(tab + idx[0], 2)); } inline v_float64x2 v_lut(const double* tab, const v_int32x4& idxvec) @@ -1120,8 +1062,7 @@ inline v_float64x2 v_lut(const double* tab, const v_int32x4& idxvec) tab[v_extract_n<0>(idxvec)], tab[v_extract_n<1>(idxvec)] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline void v_lut_deinterleave(const double* tab, const v_int32x4& idxvec, v_float64x2& x, v_float64x2& y) @@ -1141,8 +1082,7 @@ inline v_uint8x16 v_pack_b(const v_uint16x8& a, const v_uint16x8& b) ushort CV_DECL_ALIGNED(32) ptr[16] = {0}; v_store(ptr, a); v_store(ptr + 8, b); - vsetvlmax_e8m1(); - return v_uint8x16(vnsrl_wx_u8m1(vle16_v_u16m2(ptr), 0)); + return v_uint8x16(vnsrl_wx_u8m1(vle16_v_u16m2(ptr, 16), 0, 16)); } inline v_uint8x16 v_pack_b(const v_uint32x4& a, const v_uint32x4& b, @@ -1153,8 +1093,7 @@ inline v_uint8x16 v_pack_b(const v_uint32x4& a, const v_uint32x4& b, v_store(ptr + 4, b); v_store(ptr + 8, c); v_store(ptr + 12, d); - vsetvlmax_e8m1(); - return v_uint8x16(vnsrl_wx_u8m1(vnsrl_wx_u16m2(vle32_v_u32m4(ptr), 0), 0)); + return v_uint8x16(vnsrl_wx_u8m1(vnsrl_wx_u16m2(vle32_v_u32m4(ptr, 16), 0, 16), 0, 16)); } inline v_uint8x16 v_pack_b(const v_uint64x2& a, const v_uint64x2& b, const v_uint64x2& c, @@ -1170,95 +1109,89 @@ inline v_uint8x16 v_pack_b(const v_uint64x2& a, const v_uint64x2& b, const v_uin v_store(ptr + 10, f); v_store(ptr + 12, g); v_store(ptr + 14, h); - vsetvlmax_e8m1(); - return v_uint8x16(vnsrl_wx_u8m1(vnsrl_wx_u16m2(vnsrl_wx_u32m4(vle64_v_u64m8(ptr), 0), 0), 0)); + return v_uint8x16(vnsrl_wx_u8m1(vnsrl_wx_u16m2(vnsrl_wx_u32m4(vle64_v_u64m8(ptr, 16), 0, 16), 0, 16), 0, 16)); } ////////////// Arithmetics ////////////// -#define OPENCV_HAL_IMPL_RVV_BIN_OP(bin_op, _Tpvec, intrin, width) \ +#define OPENCV_HAL_IMPL_RVV_BIN_OP(bin_op, _Tpvec, intrin, vl) \ inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(intrin(a, b)); \ + return _Tpvec(intrin(a, b, vl)); \ } \ inline _Tpvec& operator bin_op##= (_Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - a = _Tpvec(intrin(a, b)); \ + a = _Tpvec(intrin(a, b, vl)); \ return a; \ } -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint8x16, vsaddu_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint8x16, vssubu_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint8x16, vdivu_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int8x16, vsadd_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int8x16, vssub_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int8x16, vdiv_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint16x8, vsaddu_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint16x8, vssubu_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint16x8, vdivu_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int16x8, vsadd_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int16x8, vssub_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int16x8, vdiv_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint32x4, vadd_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint32x4, vsub_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_uint32x4, vmul_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint32x4, vdivu_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int32x4, vadd_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int32x4, vsub_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_int32x4, vmul_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int32x4, vdiv_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_float32x4, vfadd_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_float32x4, vfsub_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_float32x4, vfmul_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_float32x4, vfdiv_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint64x2, vadd_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint64x2, vsub_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_uint64x2, vmul_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint64x2, vdivu_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int64x2, vadd_vv_i64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int64x2, vsub_vv_i64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_int64x2, vmul_vv_i64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int64x2, vdiv_vv_i64m1, 64) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint8x16, vsaddu_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint8x16, vssubu_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint8x16, vdivu_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int8x16, vsadd_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int8x16, vssub_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int8x16, vdiv_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint16x8, vsaddu_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint16x8, vssubu_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint16x8, vdivu_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int16x8, vsadd_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int16x8, vssub_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int16x8, vdiv_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint32x4, vadd_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint32x4, vsub_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_uint32x4, vmul_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint32x4, vdivu_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int32x4, vadd_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int32x4, vsub_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_int32x4, vmul_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int32x4, vdiv_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_float32x4, vfadd_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_float32x4, vfsub_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_float32x4, vfmul_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_float32x4, vfdiv_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint64x2, vadd_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint64x2, vsub_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_uint64x2, vmul_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint64x2, vdivu_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int64x2, vadd_vv_i64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int64x2, vsub_vv_i64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_int64x2, vmul_vv_i64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int64x2, vdiv_vv_i64m1, 2) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_float64x2, vfadd_vv_f64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_float64x2, vfsub_vv_f64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_float64x2, vfmul_vv_f64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_float64x2, vfdiv_vv_f64m1, 64) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_float64x2, vfadd_vv_f64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_float64x2, vfsub_vv_f64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_float64x2, vfmul_vv_f64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_float64x2, vfdiv_vv_f64m1, 2) #endif ////////////// Bitwise logic ////////////// -#define OPENCV_HAL_IMPL_RVV_LOGIC_OP(_Tpvec, suffix, width) \ -OPENCV_HAL_IMPL_RVV_BIN_OP(&, _Tpvec, vand_vv_##suffix##m1, width) \ -OPENCV_HAL_IMPL_RVV_BIN_OP(|, _Tpvec, vor_vv_##suffix##m1, width) \ -OPENCV_HAL_IMPL_RVV_BIN_OP(^, _Tpvec, vxor_vv_##suffix##m1, width) \ +#define OPENCV_HAL_IMPL_RVV_LOGIC_OP(_Tpvec, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_BIN_OP(&, _Tpvec, vand_vv_##suffix##m1, vl) \ +OPENCV_HAL_IMPL_RVV_BIN_OP(|, _Tpvec, vor_vv_##suffix##m1, vl) \ +OPENCV_HAL_IMPL_RVV_BIN_OP(^, _Tpvec, vxor_vv_##suffix##m1, vl) \ inline _Tpvec operator ~ (const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vnot_v_##suffix##m1(a)); \ + return _Tpvec(vnot_v_##suffix##m1(a, vl)); \ } -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint8x16, u8, 8) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int8x16, i8, 8) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint16x8, u16, 16) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int16x8, i16, 16) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint32x4, u32, 32) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int32x4, i32, 32) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint64x2, u64, 64) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int64x2, i64, 64) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint8x16, u8, 16) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int8x16, i8, 16) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint16x8, u16, 8) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int16x8, i16, 8) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint32x4, u32, 4) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int32x4, i32, 4) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint64x2, u64, 2) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int64x2, i64, 2) #define OPENCV_HAL_IMPL_RVV_FLT_BIT_OP(bin_op, intrin) \ inline v_float32x4 operator bin_op (const v_float32x4& a, const v_float32x4& b) \ { \ - vsetvlmax_e32m1(); \ - return v_float32x4(vreinterpret_v_i32m1_f32m1(intrin(vreinterpret_v_f32m1_i32m1(a), vreinterpret_v_f32m1_i32m1(b)))); \ + return v_float32x4(vreinterpret_v_i32m1_f32m1(intrin(vreinterpret_v_f32m1_i32m1(a), vreinterpret_v_f32m1_i32m1(b), 4))); \ } \ inline v_float32x4& operator bin_op##= (v_float32x4& a, const v_float32x4& b) \ { \ - vsetvlmax_e32m1(); \ - a = v_float32x4(vreinterpret_v_i32m1_f32m1(intrin(vreinterpret_v_f32m1_i32m1(a), vreinterpret_v_f32m1_i32m1(b)))); \ + a = v_float32x4(vreinterpret_v_i32m1_f32m1(intrin(vreinterpret_v_f32m1_i32m1(a), vreinterpret_v_f32m1_i32m1(b), 4))); \ return a; \ } @@ -1268,21 +1201,18 @@ OPENCV_HAL_IMPL_RVV_FLT_BIT_OP(^, vxor_vv_i32m1) inline v_float32x4 operator ~ (const v_float32x4& a) { - vsetvlmax_e32m1(); - return v_float32x4(vreinterpret_v_i32m1_f32m1(vnot_v_i32m1(vreinterpret_v_f32m1_i32m1(a)))); + return v_float32x4(vreinterpret_v_i32m1_f32m1(vnot_v_i32m1(vreinterpret_v_f32m1_i32m1(a), 4))); } #if CV_SIMD128_64F #define OPENCV_HAL_IMPL_RVV_FLT64_BIT_OP(bin_op, intrin) \ inline v_float64x2 operator bin_op (const v_float64x2& a, const v_float64x2& b) \ { \ - vsetvlmax_e64m1(); \ - return v_float64x2(vreinterpret_v_i64m1_f64m1(intrin(vreinterpret_v_f64m1_i64m1(a), vreinterpret_v_f64m1_i64m1(b)))); \ + return v_float64x2(vreinterpret_v_i64m1_f64m1(intrin(vreinterpret_v_f64m1_i64m1(a), vreinterpret_v_f64m1_i64m1(b), 2))); \ } \ inline v_float64x2& operator bin_op##= (v_float64x2& a, const v_float64x2& b) \ { \ - vsetvlmax_e64m1(); \ - a = v_float64x2(vreinterpret_v_i64m1_f64m1(intrin(vreinterpret_v_f64m1_i64m1(a), vreinterpret_v_f64m1_i64m1(b)))); \ + a = v_float64x2(vreinterpret_v_i64m1_f64m1(intrin(vreinterpret_v_f64m1_i64m1(a), vreinterpret_v_f64m1_i64m1(b), 2))); \ return a; \ } @@ -1292,119 +1222,108 @@ OPENCV_HAL_IMPL_RVV_FLT64_BIT_OP(^, vxor_vv_i64m1) inline v_float64x2 operator ~ (const v_float64x2& a) { - vsetvlmax_e64m1(); - return v_float64x2(vreinterpret_v_i64m1_f64m1(vnot_v_i64m1(vreinterpret_v_f64m1_i64m1(a)))); + return v_float64x2(vreinterpret_v_i64m1_f64m1(vnot_v_i64m1(vreinterpret_v_f64m1_i64m1(a), 2))); } #endif ////////////// Bitwise shifts ////////////// -#define OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(_Tpvec, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(_Tpvec, suffix, vl) \ inline _Tpvec operator << (const _Tpvec& a, int n) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ inline _Tpvec operator >> (const _Tpvec& a, int n) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsrl_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsrl_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ template inline _Tpvec v_shl(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ template inline _Tpvec v_shr(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsrl_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsrl_vx_##suffix##m1(a, uint8_t(n), vl)); \ } -#define OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(_Tpvec, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(_Tpvec, suffix, vl) \ inline _Tpvec operator << (const _Tpvec& a, int n) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ inline _Tpvec operator >> (const _Tpvec& a, int n) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsra_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsra_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ template inline _Tpvec v_shl(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ template inline _Tpvec v_shr(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsra_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsra_vx_##suffix##m1(a, uint8_t(n), vl)); \ } -OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint8x16, u8, 8) -OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint16x8, u16, 16) -OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint32x4, u32, 32) -OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint64x2, u64, 64) -OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int8x16, i8, 8) -OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int16x8, i16, 16) -OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int32x4, i32, 32) -OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int64x2, i64, 64) +OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint8x16, u8, 16) +OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint16x8, u16, 8) +OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint32x4, u32, 4) +OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint64x2, u64, 2) +OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int8x16, i8, 16) +OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int16x8, i16, 8) +OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int32x4, i32, 4) +OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int64x2, i64, 2) ////////////// Comparison ////////////// -#define OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, op, intrin, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, op, intrin, suffix, vl) \ inline _Tpvec operator op (const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vmerge_vxm_##suffix##m1(intrin(a, b), vzero_##suffix##m1(), 1)); \ + return _Tpvec(vmerge_vxm_##suffix##m1(intrin(a, b, vl), vmv_v_x_##suffix##m1(0, vl), 1, vl)); \ } -#define OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, op, intrin, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, op, intrin, suffix, vl) \ inline _Tpvec operator op (const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vfmerge_vfm_##suffix##m1(intrin(a, b), vzero_##suffix##m1(), 1)); \ -} - -#define OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(_Tpvec, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, ==, vmseq_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, !=, vmsne_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <, vmsltu_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >, vmsgtu_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <=, vmsleu_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >=, vmsgeu_vv_##suffix##m1_b##width, suffix, width) - -#define OPENCV_HAL_IMPL_RVV_SIGNED_CMP(_Tpvec, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, ==, vmseq_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, !=, vmsne_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <, vmslt_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >, vmsgt_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <=, vmsle_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >=, vmsge_vv_##suffix##m1_b##width, suffix, width) - -#define OPENCV_HAL_IMPL_RVV_FLOAT_CMP(_Tpvec, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, ==, vmfeq_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, !=, vmfne_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, <, vmflt_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, >, vmfgt_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, <=, vmfle_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, >=, vmfge_vv_##suffix##m1_b##width, suffix, width) - - -OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint8x16, u8, 8) -OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint16x8, u16, 16) -OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint32x4, u32, 32) -OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint64x2, u64, 64) -OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int8x16, i8, 8) -OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int16x8, i16, 16) -OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int32x4, i32, 32) -OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int64x2, i64, 64) -OPENCV_HAL_IMPL_RVV_FLOAT_CMP(v_float32x4, f32, 32) + return _Tpvec(vfmerge_vfm_##suffix##m1(intrin(a, b, vl), vfmv_v_f_##suffix##m1(0, vl), 1, vl)); \ +} + +#define OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(_Tpvec, suffix, width, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, ==, vmseq_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, !=, vmsne_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <, vmsltu_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >, vmsgtu_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <=, vmsleu_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >=, vmsgeu_vv_##suffix##m1_b##width, suffix, vl) + +#define OPENCV_HAL_IMPL_RVV_SIGNED_CMP(_Tpvec, suffix, width, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, ==, vmseq_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, !=, vmsne_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <, vmslt_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >, vmsgt_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <=, vmsle_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >=, vmsge_vv_##suffix##m1_b##width, suffix, vl) + +#define OPENCV_HAL_IMPL_RVV_FLOAT_CMP(_Tpvec, suffix, width, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, ==, vmfeq_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, !=, vmfne_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, <, vmflt_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, >, vmfgt_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, <=, vmfle_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, >=, vmfge_vv_##suffix##m1_b##width, suffix, vl) + + +OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint8x16, u8, 8, 16) +OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint16x8, u16, 16, 8) +OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint32x4, u32, 32, 4) +OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint64x2, u64, 64, 2) +OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int8x16, i8, 8, 16) +OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int16x8, i16, 16, 8) +OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int32x4, i32, 32, 4) +OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int64x2, i64, 64, 2) +OPENCV_HAL_IMPL_RVV_FLOAT_CMP(v_float32x4, f32, 32, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_FLOAT_CMP(v_float64x2, f64, 64) +OPENCV_HAL_IMPL_RVV_FLOAT_CMP(v_float64x2, f64, 64, 2) #endif inline v_float32x4 v_not_nan(const v_float32x4& a) @@ -1417,99 +1336,106 @@ inline v_float64x2 v_not_nan(const v_float64x2& a) ////////////// Min/Max ////////////// -#define OPENCV_HAL_IMPL_RVV_BIN_FUNC(_Tpvec, func, intrin, width) \ +#define OPENCV_HAL_IMPL_RVV_BIN_FUNC(_Tpvec, func, intrin, vl) \ inline _Tpvec func(const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(intrin(a, b)); \ -} - -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_min, vminu_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_max, vmaxu_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_min, vmin_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_max, vmax_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_min, vminu_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_max, vmaxu_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_min, vmin_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_max, vmax_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint32x4, v_min, vminu_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint32x4, v_max, vmaxu_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int32x4, v_min, vmin_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int32x4, v_max, vmax_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float32x4, v_min, vfmin_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float32x4, v_max, vfmax_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint64x2, v_min, vminu_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint64x2, v_max, vmaxu_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int64x2, v_min, vmin_vv_i64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int64x2, v_max, vmax_vv_i64m1, 64) + return _Tpvec(intrin(a, b, vl)); \ +} + +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_min, vminu_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_max, vmaxu_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_min, vmin_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_max, vmax_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_min, vminu_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_max, vmaxu_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_min, vmin_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_max, vmax_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint32x4, v_min, vminu_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint32x4, v_max, vmaxu_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int32x4, v_min, vmin_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int32x4, v_max, vmax_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float32x4, v_min, vfmin_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float32x4, v_max, vfmax_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint64x2, v_min, vminu_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint64x2, v_max, vmaxu_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int64x2, v_min, vmin_vv_i64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int64x2, v_max, vmax_vv_i64m1, 2) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float64x2, v_min, vfmin_vv_f64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float64x2, v_max, vfmax_vv_f64m1, 64) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float64x2, v_min, vfmin_vv_f64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float64x2, v_max, vfmax_vv_f64m1, 2) #endif ////////////// Arithmetics wrap ////////////// -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_add_wrap, vadd_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_add_wrap, vadd_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_add_wrap, vadd_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_add_wrap, vadd_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_sub_wrap, vsub_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_sub_wrap, vsub_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_sub_wrap, vsub_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_sub_wrap, vsub_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_mul_wrap, vmul_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_mul_wrap, vmul_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_mul_wrap, vmul_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_mul_wrap, vmul_vv_i16m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_add_wrap, vadd_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_add_wrap, vadd_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_add_wrap, vadd_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_add_wrap, vadd_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_sub_wrap, vsub_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_sub_wrap, vsub_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_sub_wrap, vsub_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_sub_wrap, vsub_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_mul_wrap, vmul_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_mul_wrap, vmul_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_mul_wrap, vmul_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_mul_wrap, vmul_vv_i16m1, 8) ////////////// Reduce ////////////// -#define OPENCV_HAL_IMPL_RVV_REDUCE_SUM(_Tpvec, _wTpvec, _nwTpvec, scalartype, suffix, wsuffix, wwidth, red) \ +#define OPENCV_HAL_IMPL_RVV_REDUCE_SUM(_Tpvec, _wTpvec, _nwTpvec, scalartype, suffix, wsuffix, vl, red) \ inline scalartype v_reduce_sum(const _Tpvec& a) \ { \ - vsetvlmax_e##wwidth##m1(); \ - _nwTpvec zero = vzero_##wsuffix##m1(); \ - _nwTpvec res = vzero_##wsuffix##m1(); \ - res = v##red##_vs_##suffix##m1_##wsuffix##m1(res, a, zero); \ + _nwTpvec zero = vmv_v_x_##wsuffix##m1(0, vl); \ + _nwTpvec res = vmv_v_x_##wsuffix##m1(0, vl); \ + res = v##red##_vs_##suffix##m1_##wsuffix##m1(res, a, zero, vl); \ return (scalartype)(_wTpvec(res).get0()); \ } OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint8x16, v_uint16x8, vuint16m1_t, unsigned, u8, u16, 16, wredsumu) OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int8x16, v_int16x8, vint16m1_t, int, i8, i16, 16, wredsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint16x8, v_uint32x4, vuint32m1_t, unsigned, u16, u32, 32, wredsumu) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int16x8, v_int32x4, vint32m1_t, int, i16, i32, 32, wredsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint32x4, v_uint64x2, vuint64m1_t, unsigned, u32, u64, 64, wredsumu) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int32x4, v_int64x2, vint64m1_t, int, i32, i64, 64, wredsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_float32x4, v_float32x4, vfloat32m1_t, float, f32, f32, 32, fredsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint64x2, v_uint64x2, vuint64m1_t, uint64, u64, u64, 64, redsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int64x2, v_int64x2, vint64m1_t, int64, i64, i64, 64, redsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint16x8, v_uint32x4, vuint32m1_t, unsigned, u16, u32, 8, wredsumu) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int16x8, v_int32x4, vint32m1_t, int, i16, i32, 8, wredsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint32x4, v_uint64x2, vuint64m1_t, unsigned, u32, u64, 4, wredsumu) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int32x4, v_int64x2, vint64m1_t, int, i32, i64, 4, wredsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint64x2, v_uint64x2, vuint64m1_t, uint64, u64, u64, 4, redsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int64x2, v_int64x2, vint64m1_t, int64, i64, i64, 4, redsum) + +#define OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(_Tpvec, _wTpvec, _nwTpvec, scalartype, suffix, wsuffix, vl, red) \ +inline scalartype v_reduce_sum(const _Tpvec& a) \ +{ \ + _nwTpvec zero = vfmv_v_f_##wsuffix##m1(0, vl); \ + _nwTpvec res = vfmv_v_f_##wsuffix##m1(0, vl); \ + res = v##red##_vs_##suffix##m1_##wsuffix##m1(res, a, zero, vl); \ + return (scalartype)(_wTpvec(res).get0()); \ +} + +OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float32x4, v_float32x4, vfloat32m1_t, float, f32, f32, 8, fredsum) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_float64x2, v_float64x2, vfloat64m1_t, double, f64, f64, 64, fredsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float64x2, v_float64x2, vfloat64m1_t, double, f64, f64, 4, fredsum) #endif -#define OPENCV_HAL_IMPL_RVV_REDUCE(_Tpvec, func, scalartype, suffix, width, red) \ +#define OPENCV_HAL_IMPL_RVV_REDUCE(_Tpvec, func, scalartype, suffix, vl, red) \ inline scalartype v_reduce_##func(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - _Tpvec res = _Tpvec(v##red##_vs_##suffix##m1_##suffix##m1(a, a, a)); \ + _Tpvec res = _Tpvec(v##red##_vs_##suffix##m1_##suffix##m1(a, a, a, vl)); \ return scalartype(res.get0()); \ } -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint8x16, min, uchar, u8, 8, redminu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int8x16, min, schar, i8, 8, redmin) -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint16x8, min, ushort, u16, 16, redminu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int16x8, min, short, i16, 16, redmin) -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint32x4, min, unsigned, u32, 32, redminu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int32x4, min, int, i32, 32, redmin) -OPENCV_HAL_IMPL_RVV_REDUCE(v_float32x4, min, float, f32, 32, fredmin) -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint8x16, max, uchar, u8, 8, redmaxu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int8x16, max, schar, i8, 8, redmax) -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint16x8, max, ushort, u16, 16, redmaxu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int16x8, max, short, i16, 16, redmax) -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint32x4, max, unsigned, u32, 32, redmaxu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int32x4, max, int, i32, 32, redmax) -OPENCV_HAL_IMPL_RVV_REDUCE(v_float32x4, max, float, f32, 32, fredmax) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint8x16, min, uchar, u8, 16, redminu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int8x16, min, schar, i8, 16, redmin) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint16x8, min, ushort, u16, 8, redminu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int16x8, min, short, i16, 8, redmin) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint32x4, min, unsigned, u32, 4, redminu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int32x4, min, int, i32, 4, redmin) +OPENCV_HAL_IMPL_RVV_REDUCE(v_float32x4, min, float, f32, 4, fredmin) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint8x16, max, uchar, u8, 16, redmaxu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int8x16, max, schar, i8, 16, redmax) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint16x8, max, ushort, u16, 8, redmaxu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int16x8, max, short, i16, 8, redmax) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint32x4, max, unsigned, u32, 4, redmaxu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int32x4, max, int, i32, 4, redmax) +OPENCV_HAL_IMPL_RVV_REDUCE(v_float32x4, max, float, f32, 4, fredmax) inline v_float32x4 v_reduce_sum4(const v_float32x4& a, const v_float32x4& b, @@ -1522,16 +1448,14 @@ inline v_float32x4 v_reduce_sum4(const v_float32x4& a, const v_float32x4& b, v_reduce_sum(c), v_reduce_sum(d) }; - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(elems)); + return v_float32x4(vle32_v_f32m1(elems, 4)); } ////////////// Square-Root ////////////// inline v_float32x4 v_sqrt(const v_float32x4& x) { - vsetvlmax_e32m1(); - return v_float32x4(vfsqrt_v_f32m1(x)); + return v_float32x4(vfsqrt_v_f32m1(x, 4)); } inline v_float32x4 v_invsqrt(const v_float32x4& x) @@ -1543,8 +1467,7 @@ inline v_float32x4 v_invsqrt(const v_float32x4& x) #if CV_SIMD128_64F inline v_float64x2 v_sqrt(const v_float64x2& x) { - vsetvlmax_e64m1(); - return v_float64x2(vfsqrt_v_f64m1(x)); + return v_float64x2(vfsqrt_v_f64m1(x, 4)); } inline v_float64x2 v_invsqrt(const v_float64x2& x) @@ -1556,29 +1479,25 @@ inline v_float64x2 v_invsqrt(const v_float64x2& x) inline v_float32x4 v_magnitude(const v_float32x4& a, const v_float32x4& b) { - vsetvlmax_e32m1(); - v_float32x4 x(vfmacc_vv_f32m1(vfmul_vv_f32m1(a, a), b, b)); + v_float32x4 x(vfmacc_vv_f32m1(vfmul_vv_f32m1(a, a, 4), b, b, 4)); return v_sqrt(x); } inline v_float32x4 v_sqr_magnitude(const v_float32x4& a, const v_float32x4& b) { - vsetvlmax_e32m1(); - return v_float32x4(vfmacc_vv_f32m1(vfmul_vv_f32m1(a, a), b, b)); + return v_float32x4(vfmacc_vv_f32m1(vfmul_vv_f32m1(a, a, 4), b, b, 4)); } #if CV_SIMD128_64F inline v_float64x2 v_magnitude(const v_float64x2& a, const v_float64x2& b) { - vsetvlmax_e64m1(); - v_float64x2 x(vfmacc_vv_f64m1(vfmul_vv_f64m1(a, a), b, b)); + v_float64x2 x(vfmacc_vv_f64m1(vfmul_vv_f64m1(a, a, 2), b, b, 2)); return v_sqrt(x); } inline v_float64x2 v_sqr_magnitude(const v_float64x2& a, const v_float64x2& b) { - vsetvlmax_e64m1(); - return v_float64x2(vfmacc_vv_f64m1(vfmul_vv_f64m1(a, a), b, b)); + return v_float64x2(vfmacc_vv_f64m1(vfmul_vv_f64m1(a, a, 2), b, b, 2)); } #endif @@ -1586,13 +1505,11 @@ inline v_float64x2 v_sqr_magnitude(const v_float64x2& a, const v_float64x2& b) inline v_float32x4 v_fma(const v_float32x4& a, const v_float32x4& b, const v_float32x4& c) { - vsetvlmax_e32m1(); - return v_float32x4(vfmacc_vv_f32m1(c, a, b)); + return v_float32x4(vfmacc_vv_f32m1(c, a, b, 4)); } inline v_int32x4 v_fma(const v_int32x4& a, const v_int32x4& b, const v_int32x4& c) { - vsetvlmax_e32m1(); - return v_int32x4(vmacc_vv_i32m1(c, a, b)); + return v_int32x4(vmacc_vv_i32m1(c, a, b, 4)); } inline v_float32x4 v_muladd(const v_float32x4& a, const v_float32x4& b, const v_float32x4& c) @@ -1608,8 +1525,7 @@ inline v_int32x4 v_muladd(const v_int32x4& a, const v_int32x4& b, const v_int32x #if CV_SIMD128_64F inline v_float64x2 v_fma(const v_float64x2& a, const v_float64x2& b, const v_float64x2& c) { - vsetvlmax_e64m1(); - return v_float64x2(vfmacc_vv_f64m1(c, a, b)); + return v_float64x2(vfmacc_vv_f64m1(c, a, b, 2)); } inline v_float64x2 v_muladd(const v_float64x2& a, const v_float64x2& b, const v_float64x2& c) @@ -1620,24 +1536,22 @@ inline v_float64x2 v_muladd(const v_float64x2& a, const v_float64x2& b, const v_ ////////////// Check all/any ////////////// -#define OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(_Tpvec, suffix, shift, width) \ +#define OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(_Tpvec, suffix, shift, vl) \ inline bool v_check_all(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - v_uint64x2 v = v_uint64x2((vuint64m1_t)vsrl_vx_##suffix##m1(vnot_v_##suffix##m1(a), shift)); \ + v_uint64x2 v = v_uint64x2((vuint64m1_t)vsrl_vx_##suffix##m1(vnot_v_##suffix##m1(a, vl), shift, vl)); \ return (v.val[0] | v.val[1]) == 0; \ } \ inline bool v_check_any(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - v_uint64x2 v = v_uint64x2((vuint64m1_t)vsrl_vx_##suffix##m1(a, shift)); \ + v_uint64x2 v = v_uint64x2((vuint64m1_t)vsrl_vx_##suffix##m1(a, shift, vl)); \ return (v.val[0] | v.val[1]) != 0; \ } -OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint8x16, u8, 7, 8) -OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint16x8, u16, 15, 16) -OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint32x4, u32, 31, 32) -OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint64x2, u64, 63, 64) +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint8x16, u8, 7, 16) +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint16x8, u16, 15, 8) +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint32x4, u32, 31, 4) +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint64x2, u64, 63, 2) inline bool v_check_all(const v_int8x16& a) @@ -1690,16 +1604,15 @@ OPENCV_HAL_IMPL_RVV_ABSDIFF(v_float64x2, absdiff) OPENCV_HAL_IMPL_RVV_ABSDIFF(v_int8x16, absdiffs) OPENCV_HAL_IMPL_RVV_ABSDIFF(v_int16x8, absdiffs) -#define OPENCV_HAL_IMPL_RVV_ABSDIFF_S(_Tpvec, _rTpvec, _nwTpvec, sub, rshr, width) \ +#define OPENCV_HAL_IMPL_RVV_ABSDIFF_S(_Tpvec, _rTpvec, _nwTpvec, sub, rshr, vl) \ inline _rTpvec v_absdiff(const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _rTpvec(rshr((_nwTpvec)sub(v_max(a, b), v_min(a, b)), 0)); \ + return _rTpvec(rshr((_nwTpvec)sub(v_max(a, b), v_min(a, b), vl), 0, vl)); \ } -OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int8x16, v_uint8x16, vuint16m2_t, vwsub_vv_i16m2, vnclipu_wx_u8m1, 8) -OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int16x8, v_uint16x8, vuint32m2_t, vwsub_vv_i32m2, vnclipu_wx_u16m1, 16) -OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int32x4, v_uint32x4, vuint64m2_t, vwsub_vv_i64m2, vnclipu_wx_u32m1, 32) +OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int8x16, v_uint8x16, vuint16m2_t, vwsub_vv_i16m2, vnclipu_wx_u8m1, 16) +OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int16x8, v_uint16x8, vuint32m2_t, vwsub_vv_i32m2, vnclipu_wx_u16m1, 8) +OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int32x4, v_uint32x4, vuint64m2_t, vwsub_vv_i64m2, vnclipu_wx_u32m1, 4) #define OPENCV_HAL_IMPL_RVV_ABS(_Tprvec, _Tpvec, suffix) \ inline _Tprvec v_abs(const _Tpvec& a) \ @@ -1732,149 +1645,152 @@ OPENCV_HAL_IMPL_RVV_REDUCE_SAD(v_float32x4, float) ////////////// Select ////////////// -#define OPENCV_HAL_IMPL_RVV_SELECT(_Tpvec, merge, ne, width) \ +#define OPENCV_HAL_IMPL_RVV_SELECT(_Tpvec, merge, ne, vl) \ inline _Tpvec v_select(const _Tpvec& mask, const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(merge(ne(mask, 0), b, a)); \ + return _Tpvec(merge(ne(mask, 0, vl), b, a, vl)); \ } -OPENCV_HAL_IMPL_RVV_SELECT(v_uint8x16, vmerge_vvm_u8m1, vmsne_vx_u8m1_b8, 8) -OPENCV_HAL_IMPL_RVV_SELECT(v_int8x16, vmerge_vvm_i8m1, vmsne_vx_i8m1_b8, 8) -OPENCV_HAL_IMPL_RVV_SELECT(v_uint16x8, vmerge_vvm_u16m1, vmsne_vx_u16m1_b16, 16) -OPENCV_HAL_IMPL_RVV_SELECT(v_int16x8, vmerge_vvm_i16m1, vmsne_vx_i16m1_b16, 16) -OPENCV_HAL_IMPL_RVV_SELECT(v_uint32x4, vmerge_vvm_u32m1, vmsne_vx_u32m1_b32, 32) -OPENCV_HAL_IMPL_RVV_SELECT(v_int32x4, vmerge_vvm_i32m1, vmsne_vx_i32m1_b32, 32) -OPENCV_HAL_IMPL_RVV_SELECT(v_float32x4, vmerge_vvm_f32m1, vmfne_vf_f32m1_b32, 32) +OPENCV_HAL_IMPL_RVV_SELECT(v_uint8x16, vmerge_vvm_u8m1, vmsne_vx_u8m1_b8, 16) +OPENCV_HAL_IMPL_RVV_SELECT(v_int8x16, vmerge_vvm_i8m1, vmsne_vx_i8m1_b8, 16) +OPENCV_HAL_IMPL_RVV_SELECT(v_uint16x8, vmerge_vvm_u16m1, vmsne_vx_u16m1_b16, 8) +OPENCV_HAL_IMPL_RVV_SELECT(v_int16x8, vmerge_vvm_i16m1, vmsne_vx_i16m1_b16, 8) +OPENCV_HAL_IMPL_RVV_SELECT(v_uint32x4, vmerge_vvm_u32m1, vmsne_vx_u32m1_b32, 4) +OPENCV_HAL_IMPL_RVV_SELECT(v_int32x4, vmerge_vvm_i32m1, vmsne_vx_i32m1_b32, 4) +OPENCV_HAL_IMPL_RVV_SELECT(v_float32x4, vmerge_vvm_f32m1, vmfne_vf_f32m1_b32, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_SELECT(v_float64x2, vmerge_vvm_f64m1, vmfne_vf_f64m1_b64, 64) +OPENCV_HAL_IMPL_RVV_SELECT(v_float64x2, vmerge_vvm_f64m1, vmfne_vf_f64m1_b64, 2) #endif ////////////// Rotate shift ////////////// -#define OPENCV_HAL_IMPL_RVV_ROTATE_OP(_Tpvec, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(_Tpvec, suffix, vl) \ template inline _Tpvec v_rotate_right(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vslidedown_vx_##suffix##m1(vzero_##suffix##m1(), a, n)); \ + return _Tpvec(vslidedown_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), a, n, vl)); \ } \ template inline _Tpvec v_rotate_left(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vslideup_vx_##suffix##m1(vzero_##suffix##m1(), a, n)); \ + return _Tpvec(vslideup_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), a, n, vl)); \ } \ template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a) \ { return a; } \ template inline _Tpvec v_rotate_right(const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vzero_##suffix##m1(), a, n), b, _Tpvec::nlanes - n)); \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), a, n, vl), b, _Tpvec::nlanes - n, vl)); \ } \ template inline _Tpvec v_rotate_left(const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vzero_##suffix##m1(), b, _Tpvec::nlanes - n), a, n)); \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), b, _Tpvec::nlanes - n, vl), a, n, vl)); \ } \ template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a, const _Tpvec& b) \ { CV_UNUSED(b); return a; } +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_uint8x16, u8, 16) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_int8x16, i8, 16) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_uint16x8, u16, 8) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_int16x8, i16, 8) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_uint32x4, u32, 4) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_int32x4, i32, 4) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_uint64x2, u64, 2) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_int64x2, i64, 2) + +#define OPENCV_HAL_IMPL_RVV_ROTATE_FP(_Tpvec, suffix, vl) \ +template inline _Tpvec v_rotate_right(const _Tpvec& a) \ +{ \ + return _Tpvec(vslidedown_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), a, n, vl)); \ +} \ +template inline _Tpvec v_rotate_left(const _Tpvec& a) \ +{ \ + return _Tpvec(vslideup_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), a, n, vl)); \ +} \ +template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a) \ +{ return a; } \ +template inline _Tpvec v_rotate_right(const _Tpvec& a, const _Tpvec& b) \ +{ \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), a, n, vl), b, _Tpvec::nlanes - n, vl)); \ +} \ +template inline _Tpvec v_rotate_left(const _Tpvec& a, const _Tpvec& b) \ +{ \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), b, _Tpvec::nlanes - n, vl), a, n, vl)); \ +} \ +template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a, const _Tpvec& b) \ +{ CV_UNUSED(b); return a; } -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_uint8x16, u8, 8) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_int8x16, i8, 8) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_uint16x8, u16, 16) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_int16x8, i16, 16) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_uint32x4, u32, 32) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_int32x4, i32, 32) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_float32x4, f32, 32) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_uint64x2, u64, 64) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_int64x2, i64, 64) +OPENCV_HAL_IMPL_RVV_ROTATE_FP(v_float32x4, f32, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_float64x2, f64, 64) +OPENCV_HAL_IMPL_RVV_ROTATE_FP(v_float64x2, f64, 2) #endif ////////////// Convert to float ////////////// inline v_float32x4 v_cvt_f32(const v_int32x4& a) { - vsetvlmax_e32m1(); - return v_float32x4(vfcvt_f_x_v_f32m1(a)); + return v_float32x4(vfcvt_f_x_v_f32m1(a, 4)); } #if CV_SIMD128_64F inline v_float32x4 v_cvt_f32(const v_float64x2& a) { double arr[4] = {a.val[0], a.val[1], 0, 0}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - vsetvlmax_e32m1(); - return v_float32x4(vfncvt_f_f_w_f32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_float32x4(vfncvt_f_f_w_f32m1(tmp, 4)); } inline v_float32x4 v_cvt_f32(const v_float64x2& a, const v_float64x2& b) { double arr[4] = {a.val[0], a.val[1], b.val[0], b.val[1]}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - vsetvlmax_e32m1(); - return v_float32x4(vfncvt_f_f_w_f32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_float32x4(vfncvt_f_f_w_f32m1(tmp, 4)); } inline v_float64x2 v_cvt_f64(const v_int32x4& a) { double CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a)); + vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a, 4), 4); double CV_DECL_ALIGNED(32) elems[2] = { ptr[0], ptr[1] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline v_float64x2 v_cvt_f64_high(const v_int32x4& a) { double CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a)); + vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a, 4), 4); double CV_DECL_ALIGNED(32) elems[2] = { ptr[2], ptr[3] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline v_float64x2 v_cvt_f64(const v_float32x4& a) { double CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a)); + vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a, 4), 4); double CV_DECL_ALIGNED(32) elems[2] = { ptr[0], ptr[1] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline v_float64x2 v_cvt_f64_high(const v_float32x4& a) { double CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a)); + vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a, 4), 4); double CV_DECL_ALIGNED(32) elems[2] = { ptr[2], ptr[3] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline v_float64x2 v_cvt_f64(const v_int64x2& a) { - vsetvlmax_e64m1(); - return v_float64x2(vfcvt_f_x_v_f64m1(a)); + return v_float64x2(vfcvt_f_x_v_f64m1(a, 2)); } #endif @@ -1947,7 +1863,7 @@ OPENCV_HAL_IMPL_RVV_TRANSPOSE4x4(float32x4, float, f32) ////////////// Reverse ////////////// -#define OPENCV_HAL_IMPL_RVV_REVERSE(_Tpvec, _Tp, width, suffix) \ +#define OPENCV_HAL_IMPL_RVV_REVERSE(_Tpvec, _Tp, suffix) \ inline _Tpvec v_reverse(const _Tpvec& a) \ { \ _Tp CV_DECL_ALIGNED(32) ptr[_Tpvec::nlanes] = {0}; \ @@ -1960,84 +1876,80 @@ inline _Tpvec v_reverse(const _Tpvec& a) \ return v_load(ptr); \ } -OPENCV_HAL_IMPL_RVV_REVERSE(v_uint8x16, uchar, 8, u8) -OPENCV_HAL_IMPL_RVV_REVERSE(v_int8x16, schar, 8, i8) -OPENCV_HAL_IMPL_RVV_REVERSE(v_uint16x8, ushort, 16, u16) -OPENCV_HAL_IMPL_RVV_REVERSE(v_int16x8, short, 16, i16) -OPENCV_HAL_IMPL_RVV_REVERSE(v_uint32x4, unsigned, 32, u32) -OPENCV_HAL_IMPL_RVV_REVERSE(v_int32x4, int, 32, i32) -OPENCV_HAL_IMPL_RVV_REVERSE(v_float32x4, float, 32, f32) -OPENCV_HAL_IMPL_RVV_REVERSE(v_uint64x2, uint64, 64, u64) -OPENCV_HAL_IMPL_RVV_REVERSE(v_int64x2, int64, 64, i64) +OPENCV_HAL_IMPL_RVV_REVERSE(v_uint8x16, uchar, u8) +OPENCV_HAL_IMPL_RVV_REVERSE(v_int8x16, schar, i8) +OPENCV_HAL_IMPL_RVV_REVERSE(v_uint16x8, ushort, u16) +OPENCV_HAL_IMPL_RVV_REVERSE(v_int16x8, short, i16) +OPENCV_HAL_IMPL_RVV_REVERSE(v_uint32x4, unsigned, u32) +OPENCV_HAL_IMPL_RVV_REVERSE(v_int32x4, int, i32) +OPENCV_HAL_IMPL_RVV_REVERSE(v_float32x4, float, f32) +OPENCV_HAL_IMPL_RVV_REVERSE(v_uint64x2, uint64, u64) +OPENCV_HAL_IMPL_RVV_REVERSE(v_int64x2, int64, i64) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_REVERSE(v_float64x2, double, 64, f64) +OPENCV_HAL_IMPL_RVV_REVERSE(v_float64x2, double, f64) #endif //////////// Value reordering //////////// -#define OPENCV_HAL_IMPL_RVV_EXPAND(_Tpwvec, _Tp, _Tpvec, width, suffix, wcvt) \ +#define OPENCV_HAL_IMPL_RVV_EXPAND(_Tpwvec, _Tp, _Tpvec, width, suffix, wcvt, vl) \ inline void v_expand(const _Tpvec& a, _Tpwvec& b0, _Tpwvec& b1) \ { \ _Tp CV_DECL_ALIGNED(32) lptr[_Tpvec::nlanes/2] = {0}; \ _Tp CV_DECL_ALIGNED(32) hptr[_Tpvec::nlanes/2] = {0}; \ v_store_low(lptr, a); \ v_store_high(hptr, a); \ - b0 = _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr))); \ - b1 = _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(hptr))); \ + b0 = _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr, vl), vl)); \ + b1 = _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(hptr, vl), vl)); \ } \ inline _Tpwvec v_expand_low(const _Tpvec& a) \ { \ _Tp CV_DECL_ALIGNED(32) lptr[_Tpvec::nlanes/2] = {0}; \ v_store_low(lptr, a); \ - return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr))); \ + return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr, vl), vl)); \ } \ inline _Tpwvec v_expand_high(const _Tpvec& a) \ { \ _Tp CV_DECL_ALIGNED(32) hptr[_Tpvec::nlanes/2] = {0}; \ v_store_high(hptr, a); \ - return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(hptr))); \ + return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(hptr, vl), vl)); \ } \ inline _Tpwvec v_load_expand(const _Tp* ptr) \ { \ - return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(ptr))); \ + return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(ptr, vl), vl)); \ } -OPENCV_HAL_IMPL_RVV_EXPAND(v_uint16x8, uchar, v_uint8x16, 8, u8, vwcvtu_x_x_v_u16m1) -OPENCV_HAL_IMPL_RVV_EXPAND(v_int16x8, schar, v_int8x16, 8, i8, vwcvt_x_x_v_i16m1) -OPENCV_HAL_IMPL_RVV_EXPAND(v_uint32x4, ushort, v_uint16x8, 16, u16, vwcvtu_x_x_v_u32m1) -OPENCV_HAL_IMPL_RVV_EXPAND(v_int32x4, short, v_int16x8, 16, i16, vwcvt_x_x_v_i32m1) -OPENCV_HAL_IMPL_RVV_EXPAND(v_uint64x2, uint, v_uint32x4, 32, u32, vwcvtu_x_x_v_u64m1) -OPENCV_HAL_IMPL_RVV_EXPAND(v_int64x2, int, v_int32x4, 32, i32, vwcvt_x_x_v_i64m1) +OPENCV_HAL_IMPL_RVV_EXPAND(v_uint16x8, uchar, v_uint8x16, 8, u8, vwcvtu_x_x_v_u16m1, 8) +OPENCV_HAL_IMPL_RVV_EXPAND(v_int16x8, schar, v_int8x16, 8, i8, vwcvt_x_x_v_i16m1, 8) +OPENCV_HAL_IMPL_RVV_EXPAND(v_uint32x4, ushort, v_uint16x8, 16, u16, vwcvtu_x_x_v_u32m1, 4) +OPENCV_HAL_IMPL_RVV_EXPAND(v_int32x4, short, v_int16x8, 16, i16, vwcvt_x_x_v_i32m1, 4) +OPENCV_HAL_IMPL_RVV_EXPAND(v_uint64x2, uint, v_uint32x4, 32, u32, vwcvtu_x_x_v_u64m1, 2) +OPENCV_HAL_IMPL_RVV_EXPAND(v_int64x2, int, v_int32x4, 32, i32, vwcvt_x_x_v_i64m1, 2) inline v_uint32x4 v_load_expand_q(const uchar* ptr) { - vsetvlmax_e32m1(); - return v_uint32x4(vwcvtu_x_x_v_u32m1(vwcvtu_x_x_v_u16mf2(vle8_v_u8mf4(ptr)))); + return v_uint32x4(vwcvtu_x_x_v_u32m1(vwcvtu_x_x_v_u16mf2(vle8_v_u8mf4(ptr, 4), 4), 4)); } inline v_int32x4 v_load_expand_q(const schar* ptr) { - vsetvlmax_e32m1(); - return v_int32x4(vwcvt_x_x_v_i32m1(vwcvt_x_x_v_i16mf2(vle8_v_i8mf4(ptr)))); + return v_int32x4(vwcvt_x_x_v_i32m1(vwcvt_x_x_v_i16mf2(vle8_v_i8mf4(ptr, 4), 4), 4)); } -#define OPENCV_HAL_IMPL_RVV_PACK(_Tpvec, _Tp, _wTpvec, _wTp, width, suffix, rshr, shr) \ +#define OPENCV_HAL_IMPL_RVV_PACK(_Tpvec, _Tp, _wTpvec, _wTp, width, suffix, rshr, shr, hvl, vl) \ inline _Tpvec v_pack(const _wTpvec& a, const _wTpvec& b) \ { \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ - vsetvlmax_e##width##m2(); \ - return _Tpvec(shr(vle##width##_v_##suffix##m2(arr), 0)); \ + return _Tpvec(shr(vle##width##_v_##suffix##m2(arr, vl), 0, vl)); \ } \ inline void v_pack_store(_Tp* ptr, const _wTpvec& a) \ { \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vzero_##suffix##m1())); \ - vsetvlmax_e##width##m2(); \ - v_store(ptr, _Tpvec(shr(vle##width##_v_##suffix##m2(arr), 0))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ + v_store(ptr, _Tpvec(shr(vle##width##_v_##suffix##m2(arr, vl), 0, vl))); \ } \ template inline \ _Tpvec v_rshr_pack(const _wTpvec& a, const _wTpvec& b) \ @@ -2045,43 +1957,39 @@ _Tpvec v_rshr_pack(const _wTpvec& a, const _wTpvec& b) \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ - vsetvlmax_e##width##m2(); \ - return _Tpvec(rshr(vle##width##_v_##suffix##m2(arr), n)); \ + return _Tpvec(rshr(vle##width##_v_##suffix##m2(arr, vl), n, vl)); \ } \ template inline \ void v_rshr_pack_store(_Tp* ptr, const _wTpvec& a) \ { \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vzero_##suffix##m1())); \ - vsetvlmax_e##width##m2(); \ - v_store(ptr, _Tpvec(rshr(vle##width##_v_##suffix##m2(arr), n))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ + v_store(ptr, _Tpvec(rshr(vle##width##_v_##suffix##m2(arr, vl), n, vl))); \ } -OPENCV_HAL_IMPL_RVV_PACK(v_uint8x16, uchar, v_uint16x8, ushort, 16, u16, vnclipu_wx_u8m1, vnclipu_wx_u8m1) -OPENCV_HAL_IMPL_RVV_PACK(v_int8x16, schar, v_int16x8, short, 16, i16, vnclip_wx_i8m1, vnclip_wx_i8m1) -OPENCV_HAL_IMPL_RVV_PACK(v_uint16x8, ushort, v_uint32x4, unsigned, 32, u32, vnclipu_wx_u16m1, vnclipu_wx_u16m1) -OPENCV_HAL_IMPL_RVV_PACK(v_int16x8, short, v_int32x4, int, 32, i32, vnclip_wx_i16m1, vnclip_wx_i16m1) -OPENCV_HAL_IMPL_RVV_PACK(v_uint32x4, unsigned, v_uint64x2, uint64, 64, u64, vnclipu_wx_u32m1, vnsrl_wx_u32m1) -OPENCV_HAL_IMPL_RVV_PACK(v_int32x4, int, v_int64x2, int64, 64, i64, vnclip_wx_i32m1, vnsra_wx_i32m1) +OPENCV_HAL_IMPL_RVV_PACK(v_uint8x16, uchar, v_uint16x8, ushort, 16, u16, vnclipu_wx_u8m1, vnclipu_wx_u8m1, 8, 16) +OPENCV_HAL_IMPL_RVV_PACK(v_int8x16, schar, v_int16x8, short, 16, i16, vnclip_wx_i8m1, vnclip_wx_i8m1, 8, 16) +OPENCV_HAL_IMPL_RVV_PACK(v_uint16x8, ushort, v_uint32x4, unsigned, 32, u32, vnclipu_wx_u16m1, vnclipu_wx_u16m1, 4, 8) +OPENCV_HAL_IMPL_RVV_PACK(v_int16x8, short, v_int32x4, int, 32, i32, vnclip_wx_i16m1, vnclip_wx_i16m1, 4, 8) +OPENCV_HAL_IMPL_RVV_PACK(v_uint32x4, unsigned, v_uint64x2, uint64, 64, u64, vnclipu_wx_u32m1, vnsrl_wx_u32m1, 2, 4) +OPENCV_HAL_IMPL_RVV_PACK(v_int32x4, int, v_int64x2, int64, 64, i64, vnclip_wx_i32m1, vnsra_wx_i32m1, 2, 4) -#define OPENCV_HAL_IMPL_RVV_PACK_U(_Tpvec, _Tp, _wTpvec, _wTp, width, suffix, rshr, cast) \ +#define OPENCV_HAL_IMPL_RVV_PACK_U(_Tpvec, _Tp, _wTpvec, _wTp, width, suffix, rshr, cast, vl) \ inline _Tpvec v_pack_u(const _wTpvec& a, const _wTpvec& b) \ { \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ - vsetvlmax_e##width##m2(); \ - return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr), 0)), 0)); \ + return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), 0, vl)); \ } \ inline void v_pack_u_store(_Tp* ptr, const _wTpvec& a) \ { \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vzero_##suffix##m1())); \ - vsetvlmax_e##width##m2(); \ - v_store(ptr, _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr), 0)), 0))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, vl))); \ + v_store(ptr, _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), 0, vl))); \ } \ template inline \ _Tpvec v_rshr_pack_u(const _wTpvec& a, const _wTpvec& b) \ @@ -2089,24 +1997,22 @@ _Tpvec v_rshr_pack_u(const _wTpvec& a, const _wTpvec& b) \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ - vsetvlmax_e##width##m2(); \ - return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr), 0)), n)); \ + return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), n, vl)); \ } \ template inline \ void v_rshr_pack_u_store(_Tp* ptr, const _wTpvec& a) \ { \ _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vzero_##suffix##m1())); \ - vsetvlmax_e##width##m2(); \ - v_store(ptr, _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr), 0)), n))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, vl))); \ + v_store(ptr, _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), n, vl))); \ } -OPENCV_HAL_IMPL_RVV_PACK_U(v_uint8x16, uchar, v_int16x8, short, 16, i16, vnclipu_wx_u8m1, vreinterpret_v_i16m2_u16m2) -OPENCV_HAL_IMPL_RVV_PACK_U(v_uint16x8, ushort, v_int32x4, int, 32, i32, vnclipu_wx_u16m1, vreinterpret_v_i32m2_u32m2) +OPENCV_HAL_IMPL_RVV_PACK_U(v_uint8x16, uchar, v_int16x8, short, 16, i16, vnclipu_wx_u8m1, vreinterpret_v_i16m2_u16m2, 16) +OPENCV_HAL_IMPL_RVV_PACK_U(v_uint16x8, ushort, v_int32x4, int, 32, i32, vnclipu_wx_u16m1, vreinterpret_v_i32m2_u32m2, 8) -#define OPENCV_HAL_IMPL_RVV_UNPACKS(_Tpvec, _Tp, width, suffix) \ +#define OPENCV_HAL_IMPL_RVV_UNPACKS(_Tpvec, _Tp, suffix) \ inline void v_zip(const v_##_Tpvec& a0, const v_##_Tpvec& a1, v_##_Tpvec& b0, v_##_Tpvec& b1) \ { \ _Tp CV_DECL_ALIGNED(32) ptra0[v_##_Tpvec::nlanes] = {0}; \ @@ -2151,19 +2057,19 @@ inline void v_recombine(const v_##_Tpvec& a, const v_##_Tpvec& b, v_##_Tpvec& c, d = v_combine_high(a, b); \ } -OPENCV_HAL_IMPL_RVV_UNPACKS(uint8x16, uchar, 8, u8) -OPENCV_HAL_IMPL_RVV_UNPACKS(int8x16, schar, 8, i8) -OPENCV_HAL_IMPL_RVV_UNPACKS(uint16x8, ushort, 16, u16) -OPENCV_HAL_IMPL_RVV_UNPACKS(int16x8, short, 16, i16) -OPENCV_HAL_IMPL_RVV_UNPACKS(uint32x4, unsigned, 32, u32) -OPENCV_HAL_IMPL_RVV_UNPACKS(int32x4, int, 32, i32) -OPENCV_HAL_IMPL_RVV_UNPACKS(float32x4, float, 32, f32) +OPENCV_HAL_IMPL_RVV_UNPACKS(uint8x16, uchar, u8) +OPENCV_HAL_IMPL_RVV_UNPACKS(int8x16, schar, i8) +OPENCV_HAL_IMPL_RVV_UNPACKS(uint16x8, ushort, u16) +OPENCV_HAL_IMPL_RVV_UNPACKS(int16x8, short, i16) +OPENCV_HAL_IMPL_RVV_UNPACKS(uint32x4, unsigned, u32) +OPENCV_HAL_IMPL_RVV_UNPACKS(int32x4, int, i32) +OPENCV_HAL_IMPL_RVV_UNPACKS(float32x4, float, f32) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_UNPACKS(float64x2, double, 64, f64) +OPENCV_HAL_IMPL_RVV_UNPACKS(float64x2, double, f64) #endif -#define OPENCV_HAL_IMPL_RVV_INTERLEAVED(_Tpvec, _Tp, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_INTERLEAVED(_Tpvec, _Tp) \ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b) \ { \ _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ @@ -2298,17 +2204,17 @@ inline v_##_Tpvec v_interleave_quads(const v_##_Tpvec& vec) \ return v_load(ptr); \ } -OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint8x16, uchar, u8, 8) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(int8x16, schar, i8, 8) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint16x8, ushort, u16, 16) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(int16x8, short, i16, 16) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint32x4, unsigned, u32, 32) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(int32x4, int, i32, 32) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(float32x4, float, f32, 32) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint64x2, uint64, u64, 64) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(int64x2, int64, i64, 64) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint8x16, uchar) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(int8x16, schar) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint16x8, ushort) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(int16x8, short) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint32x4, unsigned) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(int32x4, int) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(float32x4, float) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint64x2, uint64) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(int64x2, int64) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_INTERLEAVED(float64x2, double, f64, 64) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(float64x2, double) #endif //////////// PopCount //////////// @@ -2356,21 +2262,20 @@ OPENCV_HAL_IMPL_RVV_POPCOUNT_OP(v_uint64x2, v_int64x2, uint64, int64, u64) //////////// SignMask //////////// -#define OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(_Tpvec, _Tp, suffix, width, shift) \ +#define OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(_Tpvec, _Tp, suffix, vl, shift) \ inline int v_signmask(const _Tpvec& a) \ { \ int mask = 0; \ - vsetvlmax_e##width##m1(); \ - _Tpvec tmp = _Tpvec(vsrl_vx_##suffix##m1(a, shift)); \ + _Tpvec tmp = _Tpvec(vsrl_vx_##suffix##m1(a, shift, vl)); \ for( int i = 0; i < _Tpvec::nlanes; i++ ) \ mask |= (int)(tmp.val[i]) << i; \ return mask; \ } -OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint8x16, uchar, u8, 8, 7) -OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint16x8, ushort, u16, 16, 15) -OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint32x4, unsigned, u32, 32, 31) -OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint64x2, uint64, u64, 64, 63) +OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint8x16, uchar, u8, 16, 7) +OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint16x8, ushort, u16, 8, 15) +OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint32x4, unsigned, u32, 4, 31) +OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint64x2, uint64, u64, 2, 63) inline int v_signmask(const v_int8x16& a) { return v_signmask(v_reinterpret_as_u8(a)); } @@ -2445,12 +2350,12 @@ OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_float32x4, float) #if CV_FP16 inline v_float32x4 v_load_expand(const float16_t* ptr) { - return v_float32x4(vfwcvt_f_f_v_f32m1(vle16_v_f16mf2(ptr))); + return v_float32x4(vfwcvt_f_f_v_f32m1(vle16_v_f16mf2(ptr, 4), 4)); } inline void v_pack_store(float16_t* ptr, const v_float32x4& v) { - vse16_v_f16mf2(ptr, vfncvt_f_f_w_f16mf2(v)); + vse16_v_f16mf2(ptr, vfncvt_f_f_w_f16mf2(v, 4), 4); } #else inline v_float32x4 v_load_expand(const float16_t* ptr) @@ -2474,70 +2379,61 @@ inline void v_pack_store(float16_t* ptr, const v_float32x4& v) inline v_int32x4 v_round(const v_float32x4& a) { - vsetvlmax_e32m1(); - return v_int32x4(vfcvt_x_f_v_i32m1(a)); + return v_int32x4(vfcvt_x_f_v_i32m1(a, 4)); } inline v_int32x4 v_floor(const v_float32x4& a) { v_float32x4 ZP5 = v_setall_f32(0.5f); v_float32x4 t = a - ZP5; - vsetvlmax_e32m1(); - return v_int32x4(vfcvt_x_f_v_i32m1(t)); + return v_int32x4(vfcvt_x_f_v_i32m1(t, 4)); } inline v_int32x4 v_ceil(const v_float32x4& a) { v_float32x4 ZP5 = v_setall_f32(0.5f); v_float32x4 t = a + ZP5; - vsetvlmax_e32m1(); - return v_int32x4(vfcvt_x_f_v_i32m1(t)); + return v_int32x4(vfcvt_x_f_v_i32m1(t, 4)); } inline v_int32x4 v_trunc(const v_float32x4& a) { - vsetvlmax_e32m1(); - return v_int32x4(vfcvt_rtz_x_f_v_i32m1(a)); + return v_int32x4(vfcvt_rtz_x_f_v_i32m1(a, 4)); } #if CV_SIMD128_64F inline v_int32x4 v_round(const v_float64x2& a) { double arr[4] = {a.val[0], a.val[1], 0, 0}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - return v_int32x4(vfncvt_x_f_w_i32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_int32x4(vfncvt_x_f_w_i32m1(tmp, 4)); } inline v_int32x4 v_round(const v_float64x2& a, const v_float64x2& b) { double arr[4] = {a.val[0], a.val[1], b.val[0], b.val[1]}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - return v_int32x4(vfncvt_x_f_w_i32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_int32x4(vfncvt_x_f_w_i32m1(tmp, 4)); } inline v_int32x4 v_floor(const v_float64x2& a) { double arr[4] = {a.val[0]-0.5f, a.val[1]-0.5f, 0, 0}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - return v_int32x4(vfncvt_x_f_w_i32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_int32x4(vfncvt_x_f_w_i32m1(tmp, 4)); } inline v_int32x4 v_ceil(const v_float64x2& a) { double arr[4] = {a.val[0]+0.5f, a.val[1]+0.5f, 0, 0}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - return v_int32x4(vfncvt_x_f_w_i32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_int32x4(vfncvt_x_f_w_i32m1(tmp, 4)); } inline v_int32x4 v_trunc(const v_float64x2& a) { double arr[4] = {a.val[0], a.val[1], 0, 0}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - return v_int32x4(vfncvt_rtz_x_f_w_i32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_int32x4(vfncvt_rtz_x_f_w_i32m1(tmp, 4)); } #endif @@ -2549,8 +2445,7 @@ inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b) { int CV_DECL_ALIGNED(32) ptr[8] = {0}; v_int32x4 t1, t2; - vsetvlmax_e32m2(); - vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b)); + vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_load_deinterleave(ptr, t1, t2); return t1 + t2; } @@ -2558,8 +2453,7 @@ inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b, const v_int32 { int CV_DECL_ALIGNED(32) ptr[8] = {0}; v_int32x4 t1, t2; - vsetvlmax_e32m2(); - vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b)); + vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_load_deinterleave(ptr, t1, t2); return t1 + t2 + c; } @@ -2569,8 +2463,7 @@ inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b) { int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; v_int64x2 t1, t2; - vsetvlmax_e64m2(); - vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b)); + vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_load_deinterleave(ptr, t1, t2); return t1 + t2; } @@ -2578,8 +2471,7 @@ inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b, const v_int64 { int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; v_int64x2 t1, t2; - vsetvlmax_e64m2(); - vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b)); + vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_load_deinterleave(ptr, t1, t2); return t1 + t2 + c; } @@ -2589,8 +2481,7 @@ inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b) { unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; v_uint32x4 t1, t2, t3, t4; - vsetvlmax_e32m4(); - vse32_v_u32m4(ptr, vqmaccu_vv_u32m4(vzero_u32m4(), a, b)); + vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4; } @@ -2599,8 +2490,7 @@ inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b, { unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; v_uint32x4 t1, t2, t3, t4; - vsetvlmax_e32m4(); - vse32_v_u32m4(ptr, vqmaccu_vv_u32m4(vzero_u32m4(), a, b)); + vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4 + c; } @@ -2609,8 +2499,7 @@ inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b) { int CV_DECL_ALIGNED(32) ptr[16] = {0}; v_int32x4 t1, t2, t3, t4; - vsetvlmax_e32m4(); - vse32_v_i32m4(ptr, vqmacc_vv_i32m4(vzero_i32m4(), a, b)); + vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4; } @@ -2619,8 +2508,7 @@ inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b, { int CV_DECL_ALIGNED(32) ptr[16] = {0}; v_int32x4 t1, t2, t3, t4; - vsetvlmax_e32m4(); - vse32_v_i32m4(ptr, vqmacc_vv_i32m4(vzero_i32m4(), a, b)); + vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4 + c; } @@ -2630,8 +2518,7 @@ inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b) { uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; v_uint64x2 t1, t2, t3, t4; - vsetvlmax_e64m4(); - vse64_v_u64m4(ptr, vqmaccu_vv_u64m4(vzero_u64m4(), a, b)); + vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4; } @@ -2639,8 +2526,7 @@ inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b, con { uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; v_uint64x2 t1, t2, t3, t4; - vsetvlmax_e64m4(); - vse64_v_u64m4(ptr, vqmaccu_vv_u64m4(vzero_u64m4(), a, b)); + vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4 + c; } @@ -2649,8 +2535,7 @@ inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b) { int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; v_int64x2 t1, t2, t3, t4; - vsetvlmax_e64m4(); - vse64_v_i64m4(ptr, vqmacc_vv_i64m4(vzero_i64m4(), a, b)); + vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4; } @@ -2659,8 +2544,7 @@ inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b, { int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; v_int64x2 t1, t2, t3, t4; - vsetvlmax_e64m4(); - vse64_v_i64m4(ptr, vqmacc_vv_i64m4(vzero_i64m4(), a, b)); + vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4 + c; } @@ -2680,8 +2564,7 @@ inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b, inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b) { int CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e32m2(); - vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b)); + vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); return t1 + t2; @@ -2689,8 +2572,7 @@ inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b) inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b, const v_int32x4& c) { int CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e32m2(); - vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b)); + vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); return t1 + t2 + c; @@ -2700,8 +2582,7 @@ inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b, const v_ inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b) { int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b)); + vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); return t1 + t2; @@ -2709,8 +2590,7 @@ inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b) inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b, const v_int64x2& c) { int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b)); + vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); return t1 + t2 + c; @@ -2721,8 +2601,7 @@ inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b, const v_ inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b) { unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; - vsetvlmax_e32m4(); - vse32_v_u32m4(ptr, vqmaccu_vv_u32m4(vzero_u32m4(), a, b)); + vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_uint32x4 t1 = v_load(ptr); v_uint32x4 t2 = v_load(ptr+4); v_uint32x4 t3 = v_load(ptr+8); @@ -2732,8 +2611,7 @@ inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b, const v_uint32x4& c) { unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; - vsetvlmax_e32m4(); - vse32_v_u32m4(ptr, vqmaccu_vv_u32m4(vzero_u32m4(), a, b)); + vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_uint32x4 t1 = v_load(ptr); v_uint32x4 t2 = v_load(ptr+4); v_uint32x4 t3 = v_load(ptr+8); @@ -2743,8 +2621,7 @@ inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b) { int CV_DECL_ALIGNED(32) ptr[16] = {0}; - vsetvlmax_e32m4(); - vse32_v_i32m4(ptr, vqmacc_vv_i32m4(vzero_i32m4(), a, b)); + vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); v_int32x4 t3 = v_load(ptr+8); @@ -2754,8 +2631,7 @@ inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b) inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b, const v_int32x4& c) { int CV_DECL_ALIGNED(32) ptr[16] = {0}; - vsetvlmax_e32m4(); - vse32_v_i32m4(ptr, vqmacc_vv_i32m4(vzero_i32m4(), a, b)); + vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); v_int32x4 t3 = v_load(ptr+8); @@ -2767,8 +2643,7 @@ inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b, c inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b) { uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e64m4(); - vse64_v_u64m4(ptr, vqmaccu_vv_u64m4(vzero_u64m4(), a, b)); + vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_uint64x2 t1 = v_load(ptr); v_uint64x2 t2 = v_load(ptr+2); v_uint64x2 t3 = v_load(ptr+4); @@ -2778,8 +2653,7 @@ inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c) { uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e64m4(); - vse64_v_u64m4(ptr, vqmaccu_vv_u64m4(vzero_u64m4(), a, b)); + vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_uint64x2 t1 = v_load(ptr); v_uint64x2 t2 = v_load(ptr+2); v_uint64x2 t3 = v_load(ptr+4); @@ -2789,8 +2663,7 @@ inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b) { int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e64m4(); - vse64_v_i64m4(ptr, vqmacc_vv_i64m4(vzero_i64m4(), a, b)); + vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); v_int64x2 t3 = v_load(ptr+4); @@ -2800,8 +2673,7 @@ inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b) inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c) { int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e64m4(); - vse64_v_i64m4(ptr, vqmacc_vv_i64m4(vzero_i64m4(), a, b)); + vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); v_int64x2 t3 = v_load(ptr+4); @@ -2822,11 +2694,10 @@ inline v_float32x4 v_matmul(const v_float32x4& v, const v_float32x4& m0, const v_float32x4& m1, const v_float32x4& m2, const v_float32x4& m3) { - vsetvlmax_e32m1(); - vfloat32m1_t res = vfmul_vf_f32m1(m0, v_extract_n<0>(v)); - res = vfmacc_vf_f32m1(res, v_extract_n<1>(v), m1); - res = vfmacc_vf_f32m1(res, v_extract_n<2>(v), m2); - res = vfmacc_vf_f32m1(res, v_extract_n<3>(v), m3); + vfloat32m1_t res = vfmul_vf_f32m1(m0, v_extract_n<0>(v), 4); + res = vfmacc_vf_f32m1(res, v_extract_n<1>(v), m1, 4); + res = vfmacc_vf_f32m1(res, v_extract_n<2>(v), m2, 4); + res = vfmacc_vf_f32m1(res, v_extract_n<3>(v), m3, 4); return v_float32x4(res); } @@ -2834,40 +2705,35 @@ inline v_float32x4 v_matmuladd(const v_float32x4& v, const v_float32x4& m0, const v_float32x4& m1, const v_float32x4& m2, const v_float32x4& a) { - vsetvlmax_e32m1(); - vfloat32m1_t res = vfmul_vf_f32m1(m0, v_extract_n<0>(v)); - res = vfmacc_vf_f32m1(res, v_extract_n<1>(v), m1); - res = vfmacc_vf_f32m1(res, v_extract_n<2>(v), m2); + vfloat32m1_t res = vfmul_vf_f32m1(m0, v_extract_n<0>(v), 4); + res = vfmacc_vf_f32m1(res, v_extract_n<1>(v), m1, 4); + res = vfmacc_vf_f32m1(res, v_extract_n<2>(v), m2, 4); return v_float32x4(res) + a; } -#define OPENCV_HAL_IMPL_RVV_MUL_EXPAND(_Tpvec, _Tpwvec, _Tpw, suffix, wmul, width) \ +#define OPENCV_HAL_IMPL_RVV_MUL_EXPAND(_Tpvec, _Tpwvec, _Tpw, suffix, wmul, width, vl, hvl) \ inline void v_mul_expand(const _Tpvec& a, const _Tpvec& b, _Tpwvec& c, _Tpwvec& d) \ { \ _Tpw CV_DECL_ALIGNED(32) ptr[_Tpwvec::nlanes*2] = {0}; \ - vsetvlmax_e##width##m2(); \ - vse##width##_v_##suffix##m2(ptr, wmul(a, b)); \ - vsetvlmax_e##width##m1(); \ - c = _Tpwvec(vle##width##_v_##suffix##m1(ptr)); \ - d = _Tpwvec(vle##width##_v_##suffix##m1(ptr+_Tpwvec::nlanes)); \ + vse##width##_v_##suffix##m2(ptr, wmul(a, b, vl), vl); \ + c = _Tpwvec(vle##width##_v_##suffix##m1(ptr, hvl)); \ + d = _Tpwvec(vle##width##_v_##suffix##m1(ptr+_Tpwvec::nlanes, hvl)); \ } -OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint8x16, v_uint16x8, ushort, u16, vwmulu_vv_u16m2, 16) -OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_int8x16, v_int16x8, short, i16, vwmul_vv_i16m2, 16) -OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint16x8, v_uint32x4, unsigned, u32, vwmulu_vv_u32m2, 32) -OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_int16x8, v_int32x4, int, i32, vwmul_vv_i32m2, 32) -OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint32x4, v_uint64x2, uint64, u64, vwmulu_vv_u64m2, 64) +OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint8x16, v_uint16x8, ushort, u16, vwmulu_vv_u16m2, 16, 16, 8) +OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_int8x16, v_int16x8, short, i16, vwmul_vv_i16m2, 16, 16, 8) +OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint16x8, v_uint32x4, unsigned, u32, vwmulu_vv_u32m2, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_int16x8, v_int32x4, int, i32, vwmul_vv_i32m2, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint32x4, v_uint64x2, uint64, u64, vwmulu_vv_u64m2, 64, 4, 2) inline v_int16x8 v_mul_hi(const v_int16x8& a, const v_int16x8& b) { - vsetvlmax_e16m1(); - return v_int16x8(vnsra_wx_i16m1(vwmul_vv_i32m2(a, b), 16)); + return v_int16x8(vnsra_wx_i16m1(vwmul_vv_i32m2(a, b, 8), 16, 8)); } inline v_uint16x8 v_mul_hi(const v_uint16x8& a, const v_uint16x8& b) { - vsetvlmax_e16m1(); - return v_uint16x8(vnsrl_wx_u16m1(vwmulu_vv_u32m2(a, b), 16)); + return v_uint16x8(vnsrl_wx_u16m1(vwmulu_vv_u32m2(a, b, 8), 16, 8)); } diff --git a/platforms/linux/riscv64-gcc.toolchain.cmake b/platforms/linux/riscv64-gcc.toolchain.cmake index c46d62a360d3..675879f86b9f 100644 --- a/platforms/linux/riscv64-gcc.toolchain.cmake +++ b/platforms/linux/riscv64-gcc.toolchain.cmake @@ -10,8 +10,8 @@ set(CMAKE_CXX_COMPILER ${RISCV_GCC_INSTALL_ROOT}/bin/riscv64-unknown-linux-gnu-g # Don't run the linker on compiler check set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) -set(CMAKE_C_FLAGS "-march=rv64gcv_zvqmac ${CMAKE_C_FLAGS}") -set(CMAKE_CXX_FLAGS "-march=rv64gcv_zvqmac ${CXX_FLAGS}") +set(CMAKE_C_FLAGS "-march=rv64gcv_zfh ${CMAKE_C_FLAGS}") +set(CMAKE_CXX_FLAGS "-march=rv64gcv_zfh ${CXX_FLAGS}") set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT}) set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) From 53eca2ff5b24a6088647632cb598d732fc582ce6 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Fri, 18 Jun 2021 20:16:07 +0300 Subject: [PATCH 009/376] Merge pull request #20196 from TolyaTalamanov:at/support-vaargs-compile-args G-API: Support vaargs for cv.compile_args * Support cv.compile_args to work with variadic number of inputs * Disable python2.x G-API * Move compile_args to gapi pkg --- .../gapi/misc/python/package/gapi/__init__.py | 5 + modules/gapi/misc/python/shadow_gapi.hpp | 9 +- .../gapi/misc/python/test/test_gapi_core.py | 332 +++++----- .../misc/python/test/test_gapi_imgproc.py | 163 ++--- .../gapi/misc/python/test/test_gapi_infer.py | 580 +++++++++--------- .../python/test/test_gapi_sample_pipelines.py | 36 +- .../misc/python/test/test_gapi_streaming.py | 314 +++++----- .../gapi/misc/python/test/test_gapi_types.py | 52 +- 8 files changed, 802 insertions(+), 689 deletions(-) diff --git a/modules/gapi/misc/python/package/gapi/__init__.py b/modules/gapi/misc/python/package/gapi/__init__.py index 733c980010af..23f5f41846f3 100644 --- a/modules/gapi/misc/python/package/gapi/__init__.py +++ b/modules/gapi/misc/python/package/gapi/__init__.py @@ -11,6 +11,11 @@ def parameterized(func): return parameterized +@register('cv2.gapi') +def compile_args(*args): + return list(map(cv.GCompileArg, args)) + + @register('cv2') class GOpaque(): # NB: Inheritance from c++ class cause segfault. diff --git a/modules/gapi/misc/python/shadow_gapi.hpp b/modules/gapi/misc/python/shadow_gapi.hpp index 40dab4158141..941250c2fb45 100644 --- a/modules/gapi/misc/python/shadow_gapi.hpp +++ b/modules/gapi/misc/python/shadow_gapi.hpp @@ -3,11 +3,10 @@ namespace cv { - struct GAPI_EXPORTS_W_SIMPLE GCompileArg { }; - - GAPI_EXPORTS_W GCompileArgs compile_args(gapi::GKernelPackage pkg); - GAPI_EXPORTS_W GCompileArgs compile_args(gapi::GNetPackage pkg); - GAPI_EXPORTS_W GCompileArgs compile_args(gapi::GKernelPackage kernels, gapi::GNetPackage nets); + struct GAPI_EXPORTS_W_SIMPLE GCompileArg { + GAPI_WRAP GCompileArg(gapi::GKernelPackage pkg); + GAPI_WRAP GCompileArg(gapi::GNetPackage pkg); + }; // NB: This classes doesn't exist in *.so // HACK: Mark them as a class to force python wrapper generate code for this entities diff --git a/modules/gapi/misc/python/test/test_gapi_core.py b/modules/gapi/misc/python/test/test_gapi_core.py index 814d05d7cde4..780558d98b1a 100644 --- a/modules/gapi/misc/python/test/test_gapi_core.py +++ b/modules/gapi/misc/python/test/test_gapi_core.py @@ -3,187 +3,209 @@ import numpy as np import cv2 as cv import os +import sys +import unittest from tests_common import NewOpenCVTests -# Plaidml is an optional backend -pkgs = [ - ('ocl' , cv.gapi.core.ocl.kernels()), - ('cpu' , cv.gapi.core.cpu.kernels()), - ('fluid' , cv.gapi.core.fluid.kernels()) - # ('plaidml', cv.gapi.core.plaidml.kernels()) - ] +try: + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') -class gapi_core_test(NewOpenCVTests): + # Plaidml is an optional backend + pkgs = [ + ('ocl' , cv.gapi.core.ocl.kernels()), + ('cpu' , cv.gapi.core.cpu.kernels()), + ('fluid' , cv.gapi.core.fluid.kernels()) + # ('plaidml', cv.gapi.core.plaidml.kernels()) + ] - def test_add(self): - # TODO: Extend to use any type and size here - sz = (720, 1280) - in1 = np.full(sz, 100) - in2 = np.full(sz, 50) - # OpenCV - expected = cv.add(in1, in2) + class gapi_core_test(NewOpenCVTests): - # G-API - g_in1 = cv.GMat() - g_in2 = cv.GMat() - g_out = cv.gapi.add(g_in1, g_in2) - comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) + def test_add(self): + # TODO: Extend to use any type and size here + sz = (720, 1280) + in1 = np.full(sz, 100) + in2 = np.full(sz, 50) - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in1, in2), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') - self.assertEqual(expected.dtype, actual.dtype, 'Failed on ' + pkg_name + ' backend') + # OpenCV + expected = cv.add(in1, in2) + # G-API + g_in1 = cv.GMat() + g_in2 = cv.GMat() + g_out = cv.gapi.add(g_in1, g_in2) + comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) + + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in1, in2), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected.dtype, actual.dtype, 'Failed on ' + pkg_name + ' backend') + + + def test_add_uint8(self): + sz = (720, 1280) + in1 = np.full(sz, 100, dtype=np.uint8) + in2 = np.full(sz, 50 , dtype=np.uint8) + + # OpenCV + expected = cv.add(in1, in2) + + # G-API + g_in1 = cv.GMat() + g_in2 = cv.GMat() + g_out = cv.gapi.add(g_in1, g_in2) + comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) + + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in1, in2), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected.dtype, actual.dtype, 'Failed on ' + pkg_name + ' backend') - def test_add_uint8(self): - sz = (720, 1280) - in1 = np.full(sz, 100, dtype=np.uint8) - in2 = np.full(sz, 50 , dtype=np.uint8) - # OpenCV - expected = cv.add(in1, in2) + def test_mean(self): + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in_mat = cv.imread(img_path) - # G-API - g_in1 = cv.GMat() - g_in2 = cv.GMat() - g_out = cv.gapi.add(g_in1, g_in2) - comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) + # OpenCV + expected = cv.mean(in_mat) - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in1, in2), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') - self.assertEqual(expected.dtype, actual.dtype, 'Failed on ' + pkg_name + ' backend') + # G-API + g_in = cv.GMat() + g_out = cv.gapi.mean(g_in) + comp = cv.GComputation(g_in, g_out) + + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') - def test_mean(self): - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - in_mat = cv.imread(img_path) + def test_split3(self): + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in_mat = cv.imread(img_path) - # OpenCV - expected = cv.mean(in_mat) + # OpenCV + expected = cv.split(in_mat) - # G-API - g_in = cv.GMat() - g_out = cv.gapi.mean(g_in) - comp = cv.GComputation(g_in, g_out) + # G-API + g_in = cv.GMat() + b, g, r = cv.gapi.split3(g_in) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(b, g, r)) - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) + # Comparison + for e, a in zip(expected, actual): + self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(e.dtype, a.dtype, 'Failed on ' + pkg_name + ' backend') - def test_split3(self): - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - in_mat = cv.imread(img_path) + def test_threshold(self): + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in_mat = cv.cvtColor(cv.imread(img_path), cv.COLOR_RGB2GRAY) + maxv = (30, 30) - # OpenCV - expected = cv.split(in_mat) + # OpenCV + expected_thresh, expected_mat = cv.threshold(in_mat, maxv[0], maxv[0], cv.THRESH_TRIANGLE) - # G-API - g_in = cv.GMat() - b, g, r = cv.gapi.split3(g_in) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(b, g, r)) + # G-API + g_in = cv.GMat() + g_sc = cv.GScalar() + mat, threshold = cv.gapi.threshold(g_in, g_sc, cv.THRESH_TRIANGLE) + comp = cv.GComputation(cv.GIn(g_in, g_sc), cv.GOut(mat, threshold)) - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) - # Comparison - for e, a in zip(expected, actual): - self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF), + for pkg_name, pkg in pkgs: + actual_mat, actual_thresh = comp.apply(cv.gin(in_mat, maxv), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected_mat, actual_mat, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected_mat.dtype, actual_mat.dtype, + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected_thresh, actual_thresh[0], 'Failed on ' + pkg_name + ' backend') - self.assertEqual(e.dtype, a.dtype, 'Failed on ' + pkg_name + ' backend') - - - def test_threshold(self): - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - in_mat = cv.cvtColor(cv.imread(img_path), cv.COLOR_RGB2GRAY) - maxv = (30, 30) - - # OpenCV - expected_thresh, expected_mat = cv.threshold(in_mat, maxv[0], maxv[0], cv.THRESH_TRIANGLE) - - # G-API - g_in = cv.GMat() - g_sc = cv.GScalar() - mat, threshold = cv.gapi.threshold(g_in, g_sc, cv.THRESH_TRIANGLE) - comp = cv.GComputation(cv.GIn(g_in, g_sc), cv.GOut(mat, threshold)) - - for pkg_name, pkg in pkgs: - actual_mat, actual_thresh = comp.apply(cv.gin(in_mat, maxv), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected_mat, actual_mat, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') - self.assertEqual(expected_mat.dtype, actual_mat.dtype, - 'Failed on ' + pkg_name + ' backend') - self.assertEqual(expected_thresh, actual_thresh[0], - 'Failed on ' + pkg_name + ' backend') - - def test_kmeans(self): - # K-means params - count = 100 - sz = (count, 2) - in_mat = np.random.random(sz).astype(np.float32) - K = 5 - flags = cv.KMEANS_RANDOM_CENTERS - attempts = 1; - criteria = (cv.TERM_CRITERIA_MAX_ITER + cv.TERM_CRITERIA_EPS, 30, 0) - - # G-API - g_in = cv.GMat() - compactness, out_labels, centers = cv.gapi.kmeans(g_in, K, criteria, attempts, flags) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(compactness, out_labels, centers)) - - compact, labels, centers = comp.apply(cv.gin(in_mat)) - - # Assert - self.assertTrue(compact >= 0) - self.assertEqual(sz[0], labels.shape[0]) - self.assertEqual(1, labels.shape[1]) - self.assertTrue(labels.size != 0) - self.assertEqual(centers.shape[1], sz[1]); - self.assertEqual(centers.shape[0], K); - self.assertTrue(centers.size != 0); - - - def generate_random_points(self, sz): - arr = np.random.random(sz).astype(np.float32).T - return list(zip(arr[0], arr[1])) - - - def test_kmeans_2d(self): - # K-means 2D params - count = 100 - sz = (count, 2) - amount = sz[0] - K = 5 - flags = cv.KMEANS_RANDOM_CENTERS - attempts = 1; - criteria = (cv.TERM_CRITERIA_MAX_ITER + cv.TERM_CRITERIA_EPS, 30, 0); - in_vector = self.generate_random_points(sz) - in_labels = [] - - # G-API - data = cv.GArrayT(cv.gapi.CV_POINT2F) - best_labels = cv.GArrayT(cv.gapi.CV_INT) - - compactness, out_labels, centers = cv.gapi.kmeans(data, K, best_labels, criteria, attempts, flags); - comp = cv.GComputation(cv.GIn(data, best_labels), cv.GOut(compactness, out_labels, centers)); - - compact, labels, centers = comp.apply(cv.gin(in_vector, in_labels)); - - # Assert - self.assertTrue(compact >= 0) - self.assertEqual(amount, len(labels)) - self.assertEqual(K, len(centers)) + + + def test_kmeans(self): + # K-means params + count = 100 + sz = (count, 2) + in_mat = np.random.random(sz).astype(np.float32) + K = 5 + flags = cv.KMEANS_RANDOM_CENTERS + attempts = 1 + criteria = (cv.TERM_CRITERIA_MAX_ITER + cv.TERM_CRITERIA_EPS, 30, 0) + + # G-API + g_in = cv.GMat() + compactness, out_labels, centers = cv.gapi.kmeans(g_in, K, criteria, attempts, flags) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(compactness, out_labels, centers)) + + compact, labels, centers = comp.apply(cv.gin(in_mat)) + + # Assert + self.assertTrue(compact >= 0) + self.assertEqual(sz[0], labels.shape[0]) + self.assertEqual(1, labels.shape[1]) + self.assertTrue(labels.size != 0) + self.assertEqual(centers.shape[1], sz[1]) + self.assertEqual(centers.shape[0], K) + self.assertTrue(centers.size != 0) + + + def generate_random_points(self, sz): + arr = np.random.random(sz).astype(np.float32).T + return list(zip(arr[0], arr[1])) + + + def test_kmeans_2d(self): + # K-means 2D params + count = 100 + sz = (count, 2) + amount = sz[0] + K = 5 + flags = cv.KMEANS_RANDOM_CENTERS + attempts = 1 + criteria = (cv.TERM_CRITERIA_MAX_ITER + cv.TERM_CRITERIA_EPS, 30, 0) + in_vector = self.generate_random_points(sz) + in_labels = [] + + # G-API + data = cv.GArrayT(cv.gapi.CV_POINT2F) + best_labels = cv.GArrayT(cv.gapi.CV_INT) + + compactness, out_labels, centers = cv.gapi.kmeans(data, K, best_labels, criteria, attempts, flags) + comp = cv.GComputation(cv.GIn(data, best_labels), cv.GOut(compactness, out_labels, centers)) + + compact, labels, centers = comp.apply(cv.gin(in_vector, in_labels)) + + # Assert + self.assertTrue(compact >= 0) + self.assertEqual(amount, len(labels)) + self.assertEqual(K, len(centers)) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass if __name__ == '__main__': diff --git a/modules/gapi/misc/python/test/test_gapi_imgproc.py b/modules/gapi/misc/python/test/test_gapi_imgproc.py index ed6f883fe55f..365a5a8cca74 100644 --- a/modules/gapi/misc/python/test/test_gapi_imgproc.py +++ b/modules/gapi/misc/python/test/test_gapi_imgproc.py @@ -3,103 +3,124 @@ import numpy as np import cv2 as cv import os +import sys +import unittest from tests_common import NewOpenCVTests -# Plaidml is an optional backend -pkgs = [ - ('ocl' , cv.gapi.core.ocl.kernels()), - ('cpu' , cv.gapi.core.cpu.kernels()), - ('fluid' , cv.gapi.core.fluid.kernels()) - # ('plaidml', cv.gapi.core.plaidml.kernels()) - ] +try: + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') -class gapi_imgproc_test(NewOpenCVTests): + # Plaidml is an optional backend + pkgs = [ + ('ocl' , cv.gapi.core.ocl.kernels()), + ('cpu' , cv.gapi.core.cpu.kernels()), + ('fluid' , cv.gapi.core.fluid.kernels()) + # ('plaidml', cv.gapi.core.plaidml.kernels()) + ] - def test_good_features_to_track(self): - # TODO: Extend to use any type and size here - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - in1 = cv.cvtColor(cv.imread(img_path), cv.COLOR_RGB2GRAY) - # NB: goodFeaturesToTrack configuration - max_corners = 50 - quality_lvl = 0.01 - min_distance = 10 - block_sz = 3 - use_harris_detector = True - k = 0.04 - mask = None + class gapi_imgproc_test(NewOpenCVTests): - # OpenCV - expected = cv.goodFeaturesToTrack(in1, max_corners, quality_lvl, - min_distance, mask=mask, - blockSize=block_sz, useHarrisDetector=use_harris_detector, k=k) + def test_good_features_to_track(self): + # TODO: Extend to use any type and size here + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in1 = cv.cvtColor(cv.imread(img_path), cv.COLOR_RGB2GRAY) - # G-API - g_in = cv.GMat() - g_out = cv.gapi.goodFeaturesToTrack(g_in, max_corners, quality_lvl, - min_distance, mask, block_sz, use_harris_detector, k) + # NB: goodFeaturesToTrack configuration + max_corners = 50 + quality_lvl = 0.01 + min_distance = 10 + block_sz = 3 + use_harris_detector = True + k = 0.04 + mask = None - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + # OpenCV + expected = cv.goodFeaturesToTrack(in1, max_corners, quality_lvl, + min_distance, mask=mask, + blockSize=block_sz, useHarrisDetector=use_harris_detector, k=k) - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in1), args=cv.compile_args(pkg)) - # NB: OpenCV & G-API have different output shapes: - # OpenCV - (num_points, 1, 2) - # G-API - (num_points, 2) - # Comparison - self.assertEqual(0.0, cv.norm(expected.flatten(), - np.array(actual, dtype=np.float32).flatten(), - cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') + # G-API + g_in = cv.GMat() + g_out = cv.gapi.goodFeaturesToTrack(g_in, max_corners, quality_lvl, + min_distance, mask, block_sz, use_harris_detector, k) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) - def test_rgb2gray(self): - # TODO: Extend to use any type and size here - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - in1 = cv.imread(img_path) + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in1), args=cv.gapi.compile_args(pkg)) + # NB: OpenCV & G-API have different output shapes: + # OpenCV - (num_points, 1, 2) + # G-API - (num_points, 2) + # Comparison + self.assertEqual(0.0, cv.norm(expected.flatten(), + np.array(actual, dtype=np.float32).flatten(), + cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') - # OpenCV - expected = cv.cvtColor(in1, cv.COLOR_RGB2GRAY) - # G-API - g_in = cv.GMat() - g_out = cv.gapi.RGB2Gray(g_in) + def test_rgb2gray(self): + # TODO: Extend to use any type and size here + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in1 = cv.imread(img_path) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + # OpenCV + expected = cv.cvtColor(in1, cv.COLOR_RGB2GRAY) - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in1), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') + # G-API + g_in = cv.GMat() + g_out = cv.gapi.RGB2Gray(g_in) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) - def test_bounding_rect(self): - sz = 1280 - fscale = 256 + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in1), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') - def sample_value(fscale): - return np.random.uniform(0, 255 * fscale) / fscale - points = np.array([(sample_value(fscale), sample_value(fscale)) for _ in range(1280)], np.float32) + def test_bounding_rect(self): + sz = 1280 + fscale = 256 - # OpenCV - expected = cv.boundingRect(points) + def sample_value(fscale): + return np.random.uniform(0, 255 * fscale) / fscale - # G-API - g_in = cv.GMat() - g_out = cv.gapi.boundingRect(g_in) + points = np.array([(sample_value(fscale), sample_value(fscale)) for _ in range(1280)], np.float32) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + # OpenCV + expected = cv.boundingRect(points) - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(points), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') + # G-API + g_in = cv.GMat() + g_out = cv.gapi.boundingRect(g_in) + + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(points), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass if __name__ == '__main__': diff --git a/modules/gapi/misc/python/test/test_gapi_infer.py b/modules/gapi/misc/python/test/test_gapi_infer.py index db048f57866c..8ecc957e416d 100644 --- a/modules/gapi/misc/python/test/test_gapi_infer.py +++ b/modules/gapi/misc/python/test/test_gapi_infer.py @@ -3,318 +3,338 @@ import numpy as np import cv2 as cv import os +import sys +import unittest from tests_common import NewOpenCVTests -class test_gapi_infer(NewOpenCVTests): +try: - def infer_reference_network(self, model_path, weights_path, img): - net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) - net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) - net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') - blob = cv.dnn.blobFromImage(img) - net.setInput(blob) - return net.forward(net.getUnconnectedOutLayersNames()) + class test_gapi_infer(NewOpenCVTests): + def infer_reference_network(self, model_path, weights_path, img): + net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) + net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) + net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) - def make_roi(self, img, roi): - return img[roi[1]:roi[1] + roi[3], roi[0]:roi[0] + roi[2], ...] + blob = cv.dnn.blobFromImage(img) + net.setInput(blob) + return net.forward(net.getUnconnectedOutLayersNames()) - def test_age_gender_infer(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return - root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - device_id = 'CPU' + def make_roi(self, img, roi): + return img[roi[1]:roi[1] + roi[3], roi[0]:roi[0] + roi[2], ...] - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - img = cv.resize(cv.imread(img_path), (62,62)) - # OpenCV DNN - dnn_age, dnn_gender = self.infer_reference_network(model_path, weights_path, img) + def test_age_gender_infer(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return - # OpenCV G-API - g_in = cv.GMat() - inputs = cv.GInferInputs() - inputs.setInput('data', g_in) + root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + device_id = 'CPU' - outputs = cv.gapi.infer("net", inputs) - age_g = outputs.at("age_conv3") - gender_g = outputs.at("prob") + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + img = cv.resize(cv.imread(img_path), (62,62)) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(age_g, gender_g)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + # OpenCV DNN + dnn_age, dnn_gender = self.infer_reference_network(model_path, weights_path, img) - gapi_age, gapi_gender = comp.apply(cv.gin(img), args=cv.compile_args(cv.gapi.networks(pp))) + # OpenCV G-API + g_in = cv.GMat() + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) - # Check - self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) - self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + outputs = cv.gapi.infer("net", inputs) + age_g = outputs.at("age_conv3") + gender_g = outputs.at("prob") + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(age_g, gender_g)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - def test_age_gender_infer_roi(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return + gapi_age, gapi_gender = comp.apply(cv.gin(img), args=cv.gapi.compile_args(cv.gapi.networks(pp))) - root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - device_id = 'CPU' + # Check + self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - img = cv.imread(img_path) - roi = (10, 10, 62, 62) - # OpenCV DNN - dnn_age, dnn_gender = self.infer_reference_network(model_path, - weights_path, - self.make_roi(img, roi)) + def test_age_gender_infer_roi(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return - # OpenCV G-API - g_in = cv.GMat() - g_roi = cv.GOpaqueT(cv.gapi.CV_RECT) - inputs = cv.GInferInputs() - inputs.setInput('data', g_in) - - outputs = cv.gapi.infer("net", g_roi, inputs) - age_g = outputs.at("age_conv3") - gender_g = outputs.at("prob") - - comp = cv.GComputation(cv.GIn(g_in, g_roi), cv.GOut(age_g, gender_g)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - - gapi_age, gapi_gender = comp.apply(cv.gin(img, roi), args=cv.compile_args(cv.gapi.networks(pp))) - - # Check - self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) - self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) - - - def test_age_gender_infer_roi_list(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return - - root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - device_id = 'CPU' - - rois = [(10, 15, 62, 62), (23, 50, 62, 62), (14, 100, 62, 62), (80, 50, 62, 62)] - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - img = cv.imread(img_path) - - # OpenCV DNN - dnn_age_list = [] - dnn_gender_list = [] - for roi in rois: - age, gender = self.infer_reference_network(model_path, - weights_path, - self.make_roi(img, roi)) - dnn_age_list.append(age) - dnn_gender_list.append(gender) - - # OpenCV G-API - g_in = cv.GMat() - g_rois = cv.GArrayT(cv.gapi.CV_RECT) - inputs = cv.GInferInputs() - inputs.setInput('data', g_in) - - outputs = cv.gapi.infer("net", g_rois, inputs) - age_g = outputs.at("age_conv3") - gender_g = outputs.at("prob") - - comp = cv.GComputation(cv.GIn(g_in, g_rois), cv.GOut(age_g, gender_g)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - - gapi_age_list, gapi_gender_list = comp.apply(cv.gin(img, rois), - args=cv.compile_args(cv.gapi.networks(pp))) - - # Check - for gapi_age, gapi_gender, dnn_age, dnn_gender in zip(gapi_age_list, - gapi_gender_list, - dnn_age_list, - dnn_gender_list): - self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) - self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + device_id = 'CPU' + + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + img = cv.imread(img_path) + roi = (10, 10, 62, 62) + + # OpenCV DNN + dnn_age, dnn_gender = self.infer_reference_network(model_path, + weights_path, + self.make_roi(img, roi)) + + # OpenCV G-API + g_in = cv.GMat() + g_roi = cv.GOpaqueT(cv.gapi.CV_RECT) + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) + + outputs = cv.gapi.infer("net", g_roi, inputs) + age_g = outputs.at("age_conv3") + gender_g = outputs.at("prob") + + comp = cv.GComputation(cv.GIn(g_in, g_roi), cv.GOut(age_g, gender_g)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + gapi_age, gapi_gender = comp.apply(cv.gin(img, roi), args=cv.gapi.compile_args(cv.gapi.networks(pp))) - def test_age_gender_infer2_roi(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return - - root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - device_id = 'CPU' - - rois = [(10, 15, 62, 62), (23, 50, 62, 62), (14, 100, 62, 62), (80, 50, 62, 62)] - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - img = cv.imread(img_path) - - # OpenCV DNN - dnn_age_list = [] - dnn_gender_list = [] - for roi in rois: - age, gender = self.infer_reference_network(model_path, - weights_path, - self.make_roi(img, roi)) - dnn_age_list.append(age) - dnn_gender_list.append(gender) - - # OpenCV G-API - g_in = cv.GMat() - g_rois = cv.GArrayT(cv.gapi.CV_RECT) - inputs = cv.GInferListInputs() - inputs.setInput('data', g_rois) - - outputs = cv.gapi.infer2("net", g_in, inputs) - age_g = outputs.at("age_conv3") - gender_g = outputs.at("prob") - - comp = cv.GComputation(cv.GIn(g_in, g_rois), cv.GOut(age_g, gender_g)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - - gapi_age_list, gapi_gender_list = comp.apply(cv.gin(img, rois), - args=cv.compile_args(cv.gapi.networks(pp))) - - # Check - for gapi_age, gapi_gender, dnn_age, dnn_gender in zip(gapi_age_list, - gapi_gender_list, - dnn_age_list, - dnn_gender_list): + # Check self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + def test_age_gender_infer_roi_list(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return + + root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + device_id = 'CPU' + + rois = [(10, 15, 62, 62), (23, 50, 62, 62), (14, 100, 62, 62), (80, 50, 62, 62)] + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + img = cv.imread(img_path) + + # OpenCV DNN + dnn_age_list = [] + dnn_gender_list = [] + for roi in rois: + age, gender = self.infer_reference_network(model_path, + weights_path, + self.make_roi(img, roi)) + dnn_age_list.append(age) + dnn_gender_list.append(gender) + + # OpenCV G-API + g_in = cv.GMat() + g_rois = cv.GArrayT(cv.gapi.CV_RECT) + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) + + outputs = cv.gapi.infer("net", g_rois, inputs) + age_g = outputs.at("age_conv3") + gender_g = outputs.at("prob") + + comp = cv.GComputation(cv.GIn(g_in, g_rois), cv.GOut(age_g, gender_g)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + + gapi_age_list, gapi_gender_list = comp.apply(cv.gin(img, rois), + args=cv.gapi.compile_args(cv.gapi.networks(pp))) + + # Check + for gapi_age, gapi_gender, dnn_age, dnn_gender in zip(gapi_age_list, + gapi_gender_list, + dnn_age_list, + dnn_gender_list): + self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + + + def test_age_gender_infer2_roi(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return + + root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + device_id = 'CPU' + + rois = [(10, 15, 62, 62), (23, 50, 62, 62), (14, 100, 62, 62), (80, 50, 62, 62)] + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + img = cv.imread(img_path) + + # OpenCV DNN + dnn_age_list = [] + dnn_gender_list = [] + for roi in rois: + age, gender = self.infer_reference_network(model_path, + weights_path, + self.make_roi(img, roi)) + dnn_age_list.append(age) + dnn_gender_list.append(gender) + + # OpenCV G-API + g_in = cv.GMat() + g_rois = cv.GArrayT(cv.gapi.CV_RECT) + inputs = cv.GInferListInputs() + inputs.setInput('data', g_rois) + + outputs = cv.gapi.infer2("net", g_in, inputs) + age_g = outputs.at("age_conv3") + gender_g = outputs.at("prob") + + comp = cv.GComputation(cv.GIn(g_in, g_rois), cv.GOut(age_g, gender_g)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + + gapi_age_list, gapi_gender_list = comp.apply(cv.gin(img, rois), + args=cv.gapi.compile_args(cv.gapi.networks(pp))) + + # Check + for gapi_age, gapi_gender, dnn_age, dnn_gender in zip(gapi_age_list, + gapi_gender_list, + dnn_age_list, + dnn_gender_list): + self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + + + + def test_person_detection_retail_0013(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return + + root_path = '/omz_intel_models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + img_path = self.find_file('gpu/lbpcascade/er.png', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + device_id = 'CPU' + img = cv.resize(cv.imread(img_path), (544, 320)) + + # OpenCV DNN + net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) + net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) + net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) + + blob = cv.dnn.blobFromImage(img) + + def parseSSD(detections, size): + h, w = size + bboxes = [] + detections = detections.reshape(-1, 7) + for sample_id, class_id, confidence, xmin, ymin, xmax, ymax in detections: + if confidence >= 0.5: + x = int(xmin * w) + y = int(ymin * h) + width = int(xmax * w - x) + height = int(ymax * h - y) + bboxes.append((x, y, width, height)) + + return bboxes + + net.setInput(blob) + dnn_detections = net.forward() + dnn_boxes = parseSSD(np.array(dnn_detections), img.shape[:2]) + + # OpenCV G-API + g_in = cv.GMat() + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) + + g_sz = cv.gapi.streaming.size(g_in) + outputs = cv.gapi.infer("net", inputs) + detections = outputs.at("detection_out") + bboxes = cv.gapi.parseSSD(detections, g_sz, 0.5, False, False) + + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(bboxes)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + + gapi_boxes = comp.apply(cv.gin(img.astype(np.float32)), + args=cv.gapi.compile_args(cv.gapi.networks(pp))) + + # Comparison + self.assertEqual(0.0, cv.norm(np.array(dnn_boxes).flatten(), + np.array(gapi_boxes).flatten(), + cv.NORM_INF)) + + + def test_person_detection_retail_0013(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return + + root_path = '/omz_intel_models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + img_path = self.find_file('gpu/lbpcascade/er.png', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + device_id = 'CPU' + img = cv.resize(cv.imread(img_path), (544, 320)) + + # OpenCV DNN + net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) + net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) + net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) + + blob = cv.dnn.blobFromImage(img) + + def parseSSD(detections, size): + h, w = size + bboxes = [] + detections = detections.reshape(-1, 7) + for sample_id, class_id, confidence, xmin, ymin, xmax, ymax in detections: + if confidence >= 0.5: + x = int(xmin * w) + y = int(ymin * h) + width = int(xmax * w - x) + height = int(ymax * h - y) + bboxes.append((x, y, width, height)) + + return bboxes + + net.setInput(blob) + dnn_detections = net.forward() + dnn_boxes = parseSSD(np.array(dnn_detections), img.shape[:2]) + + # OpenCV G-API + g_in = cv.GMat() + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) + + g_sz = cv.gapi.streaming.size(g_in) + outputs = cv.gapi.infer("net", inputs) + detections = outputs.at("detection_out") + bboxes = cv.gapi.parseSSD(detections, g_sz, 0.5, False, False) + + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(bboxes)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + + gapi_boxes = comp.apply(cv.gin(img.astype(np.float32)), + args=cv.gapi.compile_args(cv.gapi.networks(pp))) + + # Comparison + self.assertEqual(0.0, cv.norm(np.array(dnn_boxes).flatten(), + np.array(gapi_boxes).flatten(), + cv.NORM_INF)) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass - def test_person_detection_retail_0013(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return - - root_path = '/omz_intel_models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - img_path = self.find_file('gpu/lbpcascade/er.png', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - device_id = 'CPU' - img = cv.resize(cv.imread(img_path), (544, 320)) - - # OpenCV DNN - net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) - net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) - net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) - - blob = cv.dnn.blobFromImage(img) - - def parseSSD(detections, size): - h, w = size - bboxes = [] - detections = detections.reshape(-1, 7) - for sample_id, class_id, confidence, xmin, ymin, xmax, ymax in detections: - if confidence >= 0.5: - x = int(xmin * w) - y = int(ymin * h) - width = int(xmax * w - x) - height = int(ymax * h - y) - bboxes.append((x, y, width, height)) - - return bboxes - - net.setInput(blob) - dnn_detections = net.forward() - dnn_boxes = parseSSD(np.array(dnn_detections), img.shape[:2]) - - # OpenCV G-API - g_in = cv.GMat() - inputs = cv.GInferInputs() - inputs.setInput('data', g_in) - - g_sz = cv.gapi.streaming.size(g_in) - outputs = cv.gapi.infer("net", inputs) - detections = outputs.at("detection_out") - bboxes = cv.gapi.parseSSD(detections, g_sz, 0.5, False, False) - - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(bboxes)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - - gapi_age, gapi_gender = comp.apply(cv.gin(img), args=cv.compile_args(cv.gapi.networks(pp))) - - gapi_boxes = comp.apply(cv.gin(img.astype(np.float32)), - args=cv.compile_args(cv.gapi.networks(pp))) - - # Comparison - self.assertEqual(0.0, cv.norm(np.array(dnn_boxes).flatten(), - np.array(gapi_boxes).flatten(), - cv.NORM_INF)) - - - def test_person_detection_retail_0013(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return - - root_path = '/omz_intel_models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - img_path = self.find_file('gpu/lbpcascade/er.png', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - device_id = 'CPU' - img = cv.resize(cv.imread(img_path), (544, 320)) - - # OpenCV DNN - net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) - net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) - net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) - - blob = cv.dnn.blobFromImage(img) - - def parseSSD(detections, size): - h, w = size - bboxes = [] - detections = detections.reshape(-1, 7) - for sample_id, class_id, confidence, xmin, ymin, xmax, ymax in detections: - if confidence >= 0.5: - x = int(xmin * w) - y = int(ymin * h) - width = int(xmax * w - x) - height = int(ymax * h - y) - bboxes.append((x, y, width, height)) - - return bboxes - - net.setInput(blob) - dnn_detections = net.forward() - dnn_boxes = parseSSD(np.array(dnn_detections), img.shape[:2]) - - # OpenCV G-API - g_in = cv.GMat() - inputs = cv.GInferInputs() - inputs.setInput('data', g_in) - - g_sz = cv.gapi.streaming.size(g_in) - outputs = cv.gapi.infer("net", inputs) - detections = outputs.at("detection_out") - bboxes = cv.gapi.parseSSD(detections, g_sz, 0.5, False, False) - - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(bboxes)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - - gapi_boxes = comp.apply(cv.gin(img.astype(np.float32)), - args=cv.compile_args(cv.gapi.networks(pp))) - - # Comparison - self.assertEqual(0.0, cv.norm(np.array(dnn_boxes).flatten(), - np.array(gapi_boxes).flatten(), - cv.NORM_INF)) + pass if __name__ == '__main__': diff --git a/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py b/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py index 2f921901db7a..a10d63f09ef2 100644 --- a/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py +++ b/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py @@ -225,7 +225,7 @@ def test_custom_op_add(self): comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) pkg = cv.gapi.kernels(GAddImpl) - actual = comp.apply(cv.gin(in_mat1, in_mat2), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(in_mat1, in_mat2), args=cv.gapi.compile_args(pkg)) self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) @@ -245,7 +245,7 @@ def test_custom_op_split3(self): comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_ch1, g_ch2, g_ch3)) pkg = cv.gapi.kernels(GSplit3Impl) - ch1, ch2, ch3 = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) + ch1, ch2, ch3 = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) self.assertEqual(0.0, cv.norm(in_ch1, ch1, cv.NORM_INF)) self.assertEqual(0.0, cv.norm(in_ch2, ch2, cv.NORM_INF)) @@ -266,7 +266,7 @@ def test_custom_op_mean(self): comp = cv.GComputation(g_in, g_out) pkg = cv.gapi.kernels(GMeanImpl) - actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) # Comparison self.assertEqual(expected, actual) @@ -287,7 +287,7 @@ def test_custom_op_addC(self): comp = cv.GComputation(cv.GIn(g_in, g_sc), cv.GOut(g_out)) pkg = cv.gapi.kernels(GAddCImpl) - actual = comp.apply(cv.gin(in_mat, sc), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(in_mat, sc), args=cv.gapi.compile_args(pkg)) self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) @@ -305,7 +305,7 @@ def test_custom_op_size(self): comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_sz)) pkg = cv.gapi.kernels(GSizeImpl) - actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) @@ -322,7 +322,7 @@ def test_custom_op_sizeR(self): comp = cv.GComputation(cv.GIn(g_r), cv.GOut(g_sz)) pkg = cv.gapi.kernels(GSizeRImpl) - actual = comp.apply(cv.gin(roi), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(roi), args=cv.gapi.compile_args(pkg)) # cv.norm works with tuples ? self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) @@ -340,7 +340,7 @@ def test_custom_op_boundingRect(self): comp = cv.GComputation(cv.GIn(g_pts), cv.GOut(g_br)) pkg = cv.gapi.kernels(GBoundingRectImpl) - actual = comp.apply(cv.gin(points), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(points), args=cv.gapi.compile_args(pkg)) # cv.norm works with tuples ? self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) @@ -371,7 +371,7 @@ def test_custom_op_goodFeaturesToTrack(self): comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) pkg = cv.gapi.kernels(GGoodFeaturesImpl) - actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) # NB: OpenCV & G-API have different output types. # OpenCV - numpy array with shape (num_points, 1, 2) @@ -453,10 +453,10 @@ def run(arr): g_in = cv.GArray.Int() comp = cv.GComputation(cv.GIn(g_in), cv.GOut(GSum.on(g_in))) - s = comp.apply(cv.gin([1, 2, 3, 4]), args=cv.compile_args(cv.gapi.kernels(GSumImpl))) + s = comp.apply(cv.gin([1, 2, 3, 4]), args=cv.gapi.compile_args(cv.gapi.kernels(GSumImpl))) self.assertEqual(10, s) - s = comp.apply(cv.gin([1, 2, 8, 7]), args=cv.compile_args(cv.gapi.kernels(GSumImpl))) + s = comp.apply(cv.gin([1, 2, 8, 7]), args=cv.gapi.compile_args(cv.gapi.kernels(GSumImpl))) self.assertEqual(18, s) self.assertEqual(18, GSumImpl.last_result) @@ -488,13 +488,13 @@ def run(table, key): 'tuple': (42, 42) } - out = comp.apply(cv.gin(table, 'int'), args=cv.compile_args(cv.gapi.kernels(GLookUpImpl))) + out = comp.apply(cv.gin(table, 'int'), args=cv.gapi.compile_args(cv.gapi.kernels(GLookUpImpl))) self.assertEqual(42, out) - out = comp.apply(cv.gin(table, 'str'), args=cv.compile_args(cv.gapi.kernels(GLookUpImpl))) + out = comp.apply(cv.gin(table, 'str'), args=cv.gapi.compile_args(cv.gapi.kernels(GLookUpImpl))) self.assertEqual('hello, world!', out) - out = comp.apply(cv.gin(table, 'tuple'), args=cv.compile_args(cv.gapi.kernels(GLookUpImpl))) + out = comp.apply(cv.gin(table, 'tuple'), args=cv.gapi.compile_args(cv.gapi.kernels(GLookUpImpl))) self.assertEqual((42, 42), out) @@ -521,7 +521,7 @@ def run(arr0, arr1): arr1 = [3, 'str'] out = comp.apply(cv.gin(arr0, arr1), - args=cv.compile_args(cv.gapi.kernels(GConcatImpl))) + args=cv.gapi.compile_args(cv.gapi.kernels(GConcatImpl))) self.assertEqual(arr0 + arr1, out) @@ -550,7 +550,7 @@ def run(img0, img1): img1 = np.array([1, 2, 3]) with self.assertRaises(Exception): comp.apply(cv.gin(img0, img1), - args=cv.compile_args( + args=cv.gapi.compile_args( cv.gapi.kernels(GAddImpl))) @@ -577,7 +577,7 @@ def run(img0, img1): img1 = np.array([1, 2, 3]) with self.assertRaises(Exception): comp.apply(cv.gin(img0, img1), - args=cv.compile_args( + args=cv.gapi.compile_args( cv.gapi.kernels(GAddImpl))) @@ -607,7 +607,7 @@ def run(img0, img1): # FIXME: Cause Bad variant access. # Need to provide more descriptive error messsage. with self.assertRaises(Exception): comp.apply(cv.gin(img0, img1), - args=cv.compile_args( + args=cv.gapi.compile_args( cv.gapi.kernels(GAddImpl))) def test_pipeline_with_custom_kernels(self): @@ -657,7 +657,7 @@ def run(img, order): g_mean = cv.gapi.mean(g_transposed) comp = cv.GComputation(cv.GIn(g_bgr), cv.GOut(g_mean)) - actual = comp.apply(cv.gin(img), args=cv.compile_args( + actual = comp.apply(cv.gin(img), args=cv.gapi.compile_args( cv.gapi.kernels(GResizeImpl, GTransposeImpl))) self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) diff --git a/modules/gapi/misc/python/test/test_gapi_streaming.py b/modules/gapi/misc/python/test/test_gapi_streaming.py index 5356abc76afd..f1cce4fb72fc 100644 --- a/modules/gapi/misc/python/test/test_gapi_streaming.py +++ b/modules/gapi/misc/python/test/test_gapi_streaming.py @@ -3,201 +3,225 @@ import numpy as np import cv2 as cv import os +import sys +import unittest from tests_common import NewOpenCVTests -class test_gapi_streaming(NewOpenCVTests): - def test_image_input(self): - sz = (1280, 720) - in_mat = np.random.randint(0, 100, sz).astype(np.uint8) +try: - # OpenCV - expected = cv.medianBlur(in_mat, 3) + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') - # G-API - g_in = cv.GMat() - g_out = cv.gapi.medianBlur(g_in, 3) - c = cv.GComputation(g_in, g_out) - ccomp = c.compileStreaming(cv.descr_of(in_mat)) - ccomp.setSource(cv.gin(in_mat)) - ccomp.start() - _, actual = ccomp.pull() + class test_gapi_streaming(NewOpenCVTests): - # Assert - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + def test_image_input(self): + sz = (1280, 720) + in_mat = np.random.randint(0, 100, sz).astype(np.uint8) + # OpenCV + expected = cv.medianBlur(in_mat, 3) + + # G-API + g_in = cv.GMat() + g_out = cv.gapi.medianBlur(g_in, 3) + c = cv.GComputation(g_in, g_out) + ccomp = c.compileStreaming(cv.descr_of(in_mat)) + ccomp.setSource(cv.gin(in_mat)) + ccomp.start() - def test_video_input(self): - ksize = 3 - path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + _, actual = ccomp.pull() - # OpenCV - cap = cv.VideoCapture(path) + # Assert + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) - # G-API - g_in = cv.GMat() - g_out = cv.gapi.medianBlur(g_in, ksize) - c = cv.GComputation(g_in, g_out) - ccomp = c.compileStreaming() - source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(source) - ccomp.start() + def test_video_input(self): + ksize = 3 + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # OpenCV + cap = cv.VideoCapture(path) - # Assert - max_num_frames = 10 - proc_num_frames = 0 - while cap.isOpened(): - has_expected, expected = cap.read() - has_actual, actual = ccomp.pull() + # G-API + g_in = cv.GMat() + g_out = cv.gapi.medianBlur(g_in, ksize) + c = cv.GComputation(g_in, g_out) - self.assertEqual(has_expected, has_actual) + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(source) + ccomp.start() - if not has_actual: - break + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, expected = cap.read() + has_actual, actual = ccomp.pull() - self.assertEqual(0.0, cv.norm(cv.medianBlur(expected, ksize), actual, cv.NORM_INF)) + self.assertEqual(has_expected, has_actual) - proc_num_frames += 1 - if proc_num_frames == max_num_frames: - break; + if not has_actual: + break + self.assertEqual(0.0, cv.norm(cv.medianBlur(expected, ksize), actual, cv.NORM_INF)) - def test_video_split3(self): - path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break - # OpenCV - cap = cv.VideoCapture(path) - # G-API - g_in = cv.GMat() - b, g, r = cv.gapi.split3(g_in) - c = cv.GComputation(cv.GIn(g_in), cv.GOut(b, g, r)) + def test_video_split3(self): + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) - ccomp = c.compileStreaming() - source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(source) - ccomp.start() + # OpenCV + cap = cv.VideoCapture(path) - # Assert - max_num_frames = 10 - proc_num_frames = 0 - while cap.isOpened(): - has_expected, frame = cap.read() - has_actual, actual = ccomp.pull() + # G-API + g_in = cv.GMat() + b, g, r = cv.gapi.split3(g_in) + c = cv.GComputation(cv.GIn(g_in), cv.GOut(b, g, r)) - self.assertEqual(has_expected, has_actual) + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(source) + ccomp.start() - if not has_actual: - break + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, frame = cap.read() + has_actual, actual = ccomp.pull() - expected = cv.split(frame) - for e, a in zip(expected, actual): - self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF)) + self.assertEqual(has_expected, has_actual) - proc_num_frames += 1 - if proc_num_frames == max_num_frames: - break; + if not has_actual: + break + expected = cv.split(frame) + for e, a in zip(expected, actual): + self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF)) - def test_video_add(self): - sz = (576, 768, 3) - in_mat = np.random.randint(0, 100, sz).astype(np.uint8) + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break - path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) - # OpenCV - cap = cv.VideoCapture(path) + def test_video_add(self): + sz = (576, 768, 3) + in_mat = np.random.randint(0, 100, sz).astype(np.uint8) - # G-API - g_in1 = cv.GMat() - g_in2 = cv.GMat() - out = cv.gapi.add(g_in1, g_in2) - c = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(out)) + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) - ccomp = c.compileStreaming() - source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(cv.gin(source, in_mat)) - ccomp.start() + # OpenCV + cap = cv.VideoCapture(path) - # Assert - max_num_frames = 10 - proc_num_frames = 0 - while cap.isOpened(): - has_expected, frame = cap.read() - has_actual, actual = ccomp.pull() + # G-API + g_in1 = cv.GMat() + g_in2 = cv.GMat() + out = cv.gapi.add(g_in1, g_in2) + c = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(out)) - self.assertEqual(has_expected, has_actual) + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(cv.gin(source, in_mat)) + ccomp.start() - if not has_actual: - break + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, frame = cap.read() + has_actual, actual = ccomp.pull() - expected = cv.add(frame, in_mat) - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + self.assertEqual(has_expected, has_actual) - proc_num_frames += 1 - if proc_num_frames == max_num_frames: - break; + if not has_actual: + break + expected = cv.add(frame, in_mat) + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) - def test_video_good_features_to_track(self): - path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break; - # NB: goodFeaturesToTrack configuration - max_corners = 50 - quality_lvl = 0.01 - min_distance = 10 - block_sz = 3 - use_harris_detector = True - k = 0.04 - mask = None - # OpenCV - cap = cv.VideoCapture(path) + def test_video_good_features_to_track(self): + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) - # G-API - g_in = cv.GMat() - g_gray = cv.gapi.RGB2Gray(g_in) - g_out = cv.gapi.goodFeaturesToTrack(g_gray, max_corners, quality_lvl, - min_distance, mask, block_sz, use_harris_detector, k) + # NB: goodFeaturesToTrack configuration + max_corners = 50 + quality_lvl = 0.01 + min_distance = 10 + block_sz = 3 + use_harris_detector = True + k = 0.04 + mask = None - c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + # OpenCV + cap = cv.VideoCapture(path) - ccomp = c.compileStreaming() - source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(source) - ccomp.start() + # G-API + g_in = cv.GMat() + g_gray = cv.gapi.RGB2Gray(g_in) + g_out = cv.gapi.goodFeaturesToTrack(g_gray, max_corners, quality_lvl, + min_distance, mask, block_sz, use_harris_detector, k) - # Assert - max_num_frames = 10 - proc_num_frames = 0 - while cap.isOpened(): - has_expected, frame = cap.read() - has_actual, actual = ccomp.pull() + c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) - self.assertEqual(has_expected, has_actual) + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(source) + ccomp.start() - if not has_actual: - break + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, frame = cap.read() + has_actual, actual = ccomp.pull() + + self.assertEqual(has_expected, has_actual) + + if not has_actual: + break + + # OpenCV + frame = cv.cvtColor(frame, cv.COLOR_RGB2GRAY) + expected = cv.goodFeaturesToTrack(frame, max_corners, quality_lvl, + min_distance, mask=mask, + blockSize=block_sz, useHarrisDetector=use_harris_detector, k=k) + for e, a in zip(expected, actual): + # NB: OpenCV & G-API have different output shapes: + # OpenCV - (num_points, 1, 2) + # G-API - (num_points, 2) + self.assertEqual(0.0, cv.norm(e.flatten(), + np.array(a, np.float32).flatten(), + cv.NORM_INF)) + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass - # OpenCV - frame = cv.cvtColor(frame, cv.COLOR_RGB2GRAY) - expected = cv.goodFeaturesToTrack(frame, max_corners, quality_lvl, - min_distance, mask=mask, - blockSize=block_sz, useHarrisDetector=use_harris_detector, k=k) - for e, a in zip(expected, actual): - # NB: OpenCV & G-API have different output shapes: - # OpenCV - (num_points, 1, 2) - # G-API - (num_points, 2) - self.assertEqual(0.0, cv.norm(e.flatten(), - np.array(a, np.float32).flatten(), - cv.NORM_INF)) - - proc_num_frames += 1 - if proc_num_frames == max_num_frames: - break; if __name__ == '__main__': NewOpenCVTests.bootstrap() diff --git a/modules/gapi/misc/python/test/test_gapi_types.py b/modules/gapi/misc/python/test/test_gapi_types.py index 0f3b194a2f97..dde554f5e10a 100644 --- a/modules/gapi/misc/python/test/test_gapi_types.py +++ b/modules/gapi/misc/python/test/test_gapi_types.py @@ -3,29 +3,51 @@ import numpy as np import cv2 as cv import os +import sys +import unittest from tests_common import NewOpenCVTests -class gapi_types_test(NewOpenCVTests): - def test_garray_type(self): - types = [cv.gapi.CV_BOOL , cv.gapi.CV_INT , cv.gapi.CV_DOUBLE , cv.gapi.CV_FLOAT, - cv.gapi.CV_STRING, cv.gapi.CV_POINT , cv.gapi.CV_POINT2F, cv.gapi.CV_SIZE , - cv.gapi.CV_RECT , cv.gapi.CV_SCALAR, cv.gapi.CV_MAT , cv.gapi.CV_GMAT] +try: - for t in types: - g_array = cv.GArrayT(t) - self.assertEqual(t, g_array.type()) + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') + class gapi_types_test(NewOpenCVTests): - def test_gopaque_type(self): - types = [cv.gapi.CV_BOOL , cv.gapi.CV_INT , cv.gapi.CV_DOUBLE , cv.gapi.CV_FLOAT, - cv.gapi.CV_STRING, cv.gapi.CV_POINT , cv.gapi.CV_POINT2F, cv.gapi.CV_SIZE , - cv.gapi.CV_RECT] + def test_garray_type(self): + types = [cv.gapi.CV_BOOL , cv.gapi.CV_INT , cv.gapi.CV_DOUBLE , cv.gapi.CV_FLOAT, + cv.gapi.CV_STRING, cv.gapi.CV_POINT , cv.gapi.CV_POINT2F, cv.gapi.CV_SIZE , + cv.gapi.CV_RECT , cv.gapi.CV_SCALAR, cv.gapi.CV_MAT , cv.gapi.CV_GMAT] - for t in types: - g_opaque = cv.GOpaqueT(t) - self.assertEqual(t, g_opaque.type()) + for t in types: + g_array = cv.GArrayT(t) + self.assertEqual(t, g_array.type()) + + + def test_gopaque_type(self): + types = [cv.gapi.CV_BOOL , cv.gapi.CV_INT , cv.gapi.CV_DOUBLE , cv.gapi.CV_FLOAT, + cv.gapi.CV_STRING, cv.gapi.CV_POINT , cv.gapi.CV_POINT2F, cv.gapi.CV_SIZE , + cv.gapi.CV_RECT] + + for t in types: + g_opaque = cv.GOpaqueT(t) + self.assertEqual(t, g_opaque.type()) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass if __name__ == '__main__': From ef2b400c61ee61211ffaa4233dd44b65974ae2aa Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 19 Jun 2021 09:16:23 +0000 Subject: [PATCH 010/376] highgui: win32ui plugin --- cmake/OpenCVFindLibsGUI.cmake | 9 - cmake/templates/cvconfig.h.in | 3 - modules/highgui/CMakeLists.txt | 16 +- modules/highgui/cmake/detect_win32ui.cmake | 17 + modules/highgui/cmake/init.cmake | 3 +- modules/highgui/src/backend.hpp | 4 + modules/highgui/src/precomp.hpp | 8 +- modules/highgui/src/registry.impl.hpp | 8 + modules/highgui/src/window.cpp | 65 +- modules/highgui/src/window_QT.cpp | 3 +- modules/highgui/src/window_cocoa.mm | 6 +- modules/highgui/src/window_gtk.cpp | 4 +- modules/highgui/src/window_w32.cpp | 2331 ++++++++++++-------- 13 files changed, 1500 insertions(+), 977 deletions(-) create mode 100644 modules/highgui/cmake/detect_win32ui.cmake diff --git a/cmake/OpenCVFindLibsGUI.cmake b/cmake/OpenCVFindLibsGUI.cmake index 8030e8b0c0fc..c8ec55b58864 100644 --- a/cmake/OpenCVFindLibsGUI.cmake +++ b/cmake/OpenCVFindLibsGUI.cmake @@ -2,15 +2,6 @@ # Detect 3rd-party GUI libraries # ---------------------------------------------------------------------------- -#--- Win32 UI --- -ocv_clear_vars(HAVE_WIN32UI) -if(WITH_WIN32UI) - try_compile(HAVE_WIN32UI - "${OpenCV_BINARY_DIR}" - "${OpenCV_SOURCE_DIR}/cmake/checks/win32uitest.cpp" - CMAKE_FLAGS "-DLINK_LIBRARIES:STRING=user32;gdi32") -endif() - # --- QT4/5 --- ocv_clear_vars(HAVE_QT HAVE_QT5) if(WITH_QT) diff --git a/cmake/templates/cvconfig.h.in b/cmake/templates/cvconfig.h.in index e79e1ec0a1bc..6439d8b43f06 100644 --- a/cmake/templates/cvconfig.h.in +++ b/cmake/templates/cvconfig.h.in @@ -121,9 +121,6 @@ /* TIFF codec */ #cmakedefine HAVE_TIFF -/* Win32 UI */ -#cmakedefine HAVE_WIN32UI - /* Define if your processor stores words with the most significant byte first (like Motorola and SPARC, unlike Intel and VAX). */ #cmakedefine WORDS_BIGENDIAN diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index b4d4b9f50384..5eb9f5ab5e6b 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -131,12 +131,6 @@ elseif(WINRT) message(STATUS " ${name}: Removing 'comctl32.lib, gdi32.lib, ole32.lib, setupapi.lib'") message(STATUS " ${name}: Leaving '${HIGHGUI_LIBRARIES}'") endif() -elseif(HAVE_WIN32UI) - set(OPENCV_HIGHGUI_BUILTIN_BACKEND "WIN32UI") - list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_w32.cpp) - if(OpenCV_ARCH STREQUAL "ARM64") - list(APPEND HIGHGUI_LIBRARIES "comdlg32" "advapi32") - endif() elseif(HAVE_COCOA) set(OPENCV_HIGHGUI_BUILTIN_BACKEND "COCOA") add_definitions(-DHAVE_COCOA) @@ -144,6 +138,16 @@ elseif(HAVE_COCOA) list(APPEND HIGHGUI_LIBRARIES "-framework Cocoa") endif() +if(TARGET ocv.3rdparty.win32ui) + if("win32ui" IN_LIST HIGHGUI_PLUGIN_LIST OR HIGHGUI_PLUGIN_LIST STREQUAL "all") + ocv_create_builtin_highgui_plugin(opencv_highgui_win32 ocv.3rdparty.win32ui "window_w32.cpp") + else() + set(OPENCV_HIGHGUI_BUILTIN_BACKEND "WIN32UI") + list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_w32.cpp) + list(APPEND tgts ocv.3rdparty.win32ui) + endif() +endif() + if(TARGET ocv.3rdparty.gtk3 OR TARGET ocv.3rdparty.gtk2) if(TARGET ocv.3rdparty.gtk3 AND NOT WITH_GTK_2_X) set(__gtk_dependency "ocv.3rdparty.gtk3") diff --git a/modules/highgui/cmake/detect_win32ui.cmake b/modules/highgui/cmake/detect_win32ui.cmake new file mode 100644 index 000000000000..1d2fdc5d4654 --- /dev/null +++ b/modules/highgui/cmake/detect_win32ui.cmake @@ -0,0 +1,17 @@ +#--- Win32 UI --- +ocv_clear_vars(HAVE_WIN32UI) +if(WITH_WIN32UI) + try_compile(HAVE_WIN32UI + "${CMAKE_CURRENT_BINARY_DIR}" + "${OpenCV_SOURCE_DIR}/cmake/checks/win32uitest.cpp" + CMAKE_FLAGS "-DLINK_LIBRARIES:STRING=user32;gdi32") + if(HAVE_WIN32UI) + set(__libs "user32" "gdi32") + if(OpenCV_ARCH STREQUAL "ARM64") + list(APPEND __libs "comdlg32" "advapi32") + endif() + ocv_add_external_target(win32ui "" "${__libs}" "HAVE_WIN32UI") + endif() +endif() + +set(HAVE_WIN32UI "${HAVE_WIN32UI}" PARENT_SCOPE) # informational diff --git a/modules/highgui/cmake/init.cmake b/modules/highgui/cmake/init.cmake index 3b766b3758c0..1626d254daf9 100644 --- a/modules/highgui/cmake/init.cmake +++ b/modules/highgui/cmake/init.cmake @@ -43,8 +43,7 @@ else() endif() add_backend("gtk" WITH_GTK) - -# TODO win32 +add_backend("win32ui" WITH_WIN32UI) # TODO cocoa # TODO qt # TODO opengl diff --git a/modules/highgui/src/backend.hpp b/modules/highgui/src/backend.hpp index 14c88b238761..7c32846ce4a3 100644 --- a/modules/highgui/src/backend.hpp +++ b/modules/highgui/src/backend.hpp @@ -114,6 +114,10 @@ bool setUIBackend(const std::string& backendName); #ifndef BUILD_PLUGIN +#ifdef HAVE_WIN32UI +std::shared_ptr createUIBackendWin32UI(); +#endif + #ifdef HAVE_GTK std::shared_ptr createUIBackendGTK(); #endif diff --git a/modules/highgui/src/precomp.hpp b/modules/highgui/src/precomp.hpp index 6ad5bce8b465..0d26b957ad71 100644 --- a/modules/highgui/src/precomp.hpp +++ b/modules/highgui/src/precomp.hpp @@ -67,7 +67,6 @@ #include #include #include -#include #if defined _WIN32 || defined WINCE #include @@ -127,6 +126,13 @@ void cvSetPropTopmost_COCOA(const char* name, const bool topmost); double cvGetPropVsync_W32(const char* name); void cvSetPropVsync_W32(const char* name, const bool enabled); +void setWindowTitle_W32(const cv::String& name, const cv::String& title); +void setWindowTitle_GTK(const cv::String& name, const cv::String& title); +void setWindowTitle_QT(const cv::String& name, const cv::String& title); +void setWindowTitle_COCOA(const cv::String& name, const cv::String& title); + +int pollKey_W32(); + //for QT #if defined (HAVE_QT) CvRect cvGetWindowRect_QT(const char* name); diff --git a/modules/highgui/src/registry.impl.hpp b/modules/highgui/src/registry.impl.hpp index ccf81f928002..66693f1b07e0 100644 --- a/modules/highgui/src/registry.impl.hpp +++ b/modules/highgui/src/registry.impl.hpp @@ -50,6 +50,14 @@ std::vector& getBuiltinBackendsInfo() #elif defined(ENABLE_PLUGINS) DECLARE_DYNAMIC_BACKEND("QT") #endif +#endif + +#ifdef _WIN32 +#ifdef HAVE_WIN32UI + DECLARE_STATIC_BACKEND("WIN32", createUIBackendWin32UI) +#elif defined(ENABLE_PLUGINS) + DECLARE_DYNAMIC_BACKEND("WIN32") +#endif #endif }; return g_backends; diff --git a/modules/highgui/src/window.cpp b/modules/highgui/src/window.cpp index 56c1456a5d95..d1ccd1dbc3a9 100644 --- a/modules/highgui/src/window.cpp +++ b/modules/highgui/src/window.cpp @@ -586,6 +586,46 @@ void cv::moveWindow( const String& winname, int x, int y ) #endif } +void cv::setWindowTitle(const String& winname, const String& title) +{ + CV_TRACE_FUNCTION(); + + { + cv::AutoLock lock(cv::getWindowMutex()); + auto window = findWindow_(winname); + if (window) + { + return window->setTitle(title); + } + } + +#if defined(OPENCV_HIGHGUI_WITHOUT_BUILTIN_BACKEND) && defined(ENABLE_PLUGINS) + auto backend = getCurrentUIBackend(); + if (backend) + { + CV_LOG_WARNING(NULL, "Can't find window with name: '" << winname << "'. Do nothing"); + CV_NOT_FOUND_DEPRECATION; + } + else + { + CV_LOG_WARNING(NULL, "No UI backends available. Use OPENCV_LOG_LEVEL=DEBUG for investigation"); + } + return; +#elif defined(HAVE_WIN32UI) + return setWindowTitle_W32(winname, title); +#elif defined (HAVE_GTK) + return setWindowTitle_GTK(winname, title); +#elif defined (HAVE_QT) + return setWindowTitle_QT(winname, title); +#elif defined (HAVE_COCOA) + return setWindowTitle_COCOA(winname, title); +#else + CV_Error(Error::StsNotImplemented, "The function is not implemented. " + "Rebuild the library with Windows, GTK+ 2.x or Cocoa support. " + "If you are on Ubuntu or Debian, install libgtk2.0-dev and pkg-config, then re-run cmake or configure script"); +#endif +} + void cv::setWindowProperty(const String& winname, int prop_id, double prop_value) { CV_TRACE_FUNCTION(); @@ -630,9 +670,9 @@ int cv::waitKey(int delay) return (code != -1) ? (code & 0xff) : -1; } -#if defined(HAVE_QT) || (defined (WINRT) && !defined (WINRT_8_0)) || \ - !defined(HAVE_WIN32UI) && (defined(HAVE_GTK) || defined(HAVE_COCOA)) -// pollKey() fallback implementation +/* + * process until queue is empty but don't wait. + */ int cv::pollKey() { CV_TRACE_FUNCTION(); @@ -646,12 +686,13 @@ int cv::pollKey() } } +#if defined(HAVE_WIN32UI) + return pollKey_W32(); +#else // fallback. please implement a proper polling function return cvWaitKey(1); -} -#elif defined(HAVE_WIN32UI) -// pollKey() implemented in window_w32.cpp #endif +} int cv::createTrackbar(const String& trackbarName, const String& winName, int* value, int count, TrackbarCallback callback, @@ -1203,13 +1244,6 @@ int cv::createButton(const String&, ButtonCallback, void*, int , bool ) // version with a more capable one without a need to recompile dependent // applications or libraries. -void cv::setWindowTitle(const String&, const String&) -{ - CV_Error(Error::StsNotImplemented, "The function is not implemented. " - "Rebuild the library with Windows, GTK+ 2.x or Cocoa support. " - "If you are on Ubuntu or Debian, install libgtk2.0-dev and pkg-config, then re-run cmake or configure script"); -} - #define CV_NO_GUI_ERROR(funcname) \ cv::error(cv::Error::StsError, \ "The function is not implemented. " \ @@ -1360,11 +1394,6 @@ CV_IMPL int cvCreateButton(const char*, void (*)(int, void*), void*, int, int) CV_NO_GUI_ERROR("cvCreateButton"); } -int cv::pollKey() -{ - CV_NO_GUI_ERROR("cv::pollKey()"); -} - #endif /* End of file. */ diff --git a/modules/highgui/src/window_QT.cpp b/modules/highgui/src/window_QT.cpp index 60d7d69a5979..9899dfdcf0f1 100644 --- a/modules/highgui/src/window_QT.cpp +++ b/modules/highgui/src/window_QT.cpp @@ -63,6 +63,7 @@ #endif #endif +using namespace cv; //Static and global first static GuiReceiver *guiMainThread = NULL; @@ -197,7 +198,7 @@ void cvSetPropWindow_QT(const char* name,double prop_value) Q_ARG(double, prop_value)); } -void cv::setWindowTitle(const String& winname, const String& title) +void setWindowTitle_QT(const String& winname, const String& title) { if (!guiMainThread) CV_Error(Error::StsNullPtr, "NULL guiReceiver (please create a window)"); diff --git a/modules/highgui/src/window_cocoa.mm b/modules/highgui/src/window_cocoa.mm index 29a0278c982e..e8e903440675 100644 --- a/modules/highgui/src/window_cocoa.mm +++ b/modules/highgui/src/window_cocoa.mm @@ -795,18 +795,18 @@ void cvSetPropTopmost_COCOA( const char* name, const bool topmost ) __END__; } -void cv::setWindowTitle(const String& winname, const String& title) +void setWindowTitle_COCOA(const cv::String& winname, const cv::String& title) { CVWindow *window = cvGetWindow(winname.c_str()); if (window == NULL) { - namedWindow(winname); + cv::namedWindow(winname); window = cvGetWindow(winname.c_str()); } if (window == NULL) - CV_Error(Error::StsNullPtr, "NULL window"); + CV_Error(cv::Error::StsNullPtr, "NULL window"); NSAutoreleasePool* localpool = [[NSAutoreleasePool alloc] init]; diff --git a/modules/highgui/src/window_gtk.cpp b/modules/highgui/src/window_gtk.cpp index efa3fbd96f56..8eaf98fb3612 100644 --- a/modules/highgui/src/window_gtk.cpp +++ b/modules/highgui/src/window_gtk.cpp @@ -364,7 +364,7 @@ static void cvImageWidget_set_size(GtkWidget * widget, int max_width, int max_he } - assert( image_widget->scaled_image ); + CV_Assert(image_widget->scaled_image); } static void @@ -849,7 +849,7 @@ static bool setModeWindow_(const std::shared_ptr& window, int mode) return false; } -void cv::setWindowTitle(const String& winname, const String& title) +void setWindowTitle_GTK(const String& winname, const String& title) { CV_LOCK_MUTEX(); diff --git a/modules/highgui/src/window_w32.cpp b/modules/highgui/src/window_w32.cpp index c4f2ddd2a603..d9a9d732227a 100644 --- a/modules/highgui/src/window_w32.cpp +++ b/modules/highgui/src/window_w32.cpp @@ -41,12 +41,17 @@ #include "precomp.hpp" +#ifdef HAVE_WIN32UI + +#include +#include + +#include "backend.hpp" + using namespace cv; #include // required for GET_X_LPARAM() and GET_Y_LPARAM() macros -#if defined _WIN32 - #ifdef __GNUC__ # pragma GCC diagnostic ignored "-Wmissing-declarations" #endif @@ -60,14 +65,12 @@ using namespace cv; #include #include #include -#include #ifdef HAVE_OPENGL #include #include #include #include -#include "opencv2/highgui.hpp" #include #include "opencv2/core/opengl.hpp" #endif @@ -78,7 +81,7 @@ static const char* trackbar_text = #if defined _M_X64 || defined __x86_64 || defined _M_ARM64 #define icvGetWindowLongPtr GetWindowLongPtr -#define icvSetWindowLongPtr( hwnd, id, ptr ) SetWindowLongPtr( hwnd, id, (LONG_PTR)(ptr) ) +#define icvSetWindowLongPtr(hwnd, id, ptr) SetWindowLongPtr(hwnd, id, (LONG_PTR)(ptr)) #define icvGetClassLongPtr GetClassLongPtr #define CV_USERDATA GWLP_USERDATA @@ -89,7 +92,7 @@ static const char* trackbar_text = #else #define icvGetWindowLongPtr GetWindowLong -#define icvSetWindowLongPtr( hwnd, id, ptr ) SetWindowLong( hwnd, id, (size_t)ptr ) +#define icvSetWindowLongPtr(hwnd, id, ptr) SetWindowLong(hwnd, id, (size_t)ptr) #define icvGetClassLongPtr GetClassLong #define CV_USERDATA GWL_USERDATA @@ -116,13 +119,13 @@ static inline void mingw_strcat_s(char *dest, size_t destsz, const char *src){ #define strcat_s mingw_strcat_s #endif -static void FillBitmapInfo( BITMAPINFO* bmi, int width, int height, int bpp, int origin ) +static void FillBitmapInfo(BITMAPINFO* bmi, int width, int height, int bpp, int origin) { - assert( bmi && width >= 0 && height >= 0 && (bpp == 8 || bpp == 24 || bpp == 32)); + CV_Assert(bmi && width >= 0 && height >= 0 && (bpp == 8 || bpp == 24 || bpp == 32)); BITMAPINFOHEADER* bmih = &(bmi->bmiHeader); - memset( bmih, 0, sizeof(*bmih)); + memset(bmih, 0, sizeof(*bmih)); bmih->biSize = sizeof(BITMAPINFOHEADER); bmih->biWidth = width; bmih->biHeight = origin ? abs(height) : -abs(height); @@ -130,11 +133,11 @@ static void FillBitmapInfo( BITMAPINFO* bmi, int width, int height, int bpp, int bmih->biBitCount = (unsigned short)bpp; bmih->biCompression = BI_RGB; - if( bpp == 8 ) + if (bpp == 8) { RGBQUAD* palette = bmi->bmiColors; int i; - for( i = 0; i < 256; i++ ) + for (i = 0; i < 256; i++) { palette[i].rgbBlue = palette[i].rgbGreen = palette[i].rgbRed = (BYTE)i; palette[i].rgbReserved = 0; @@ -144,68 +147,91 @@ static void FillBitmapInfo( BITMAPINFO* bmi, int width, int height, int bpp, int struct CvWindow; -typedef struct CvTrackbar +struct CvTrackbar : public std::enable_shared_from_this { + CvTrackbar(CvWindow& window, const std::string& name_) + : signature(CV_TRACKBAR_MAGIC_VAL) + , name(name_) + , parent(&window) + { + // nothing + } + ~CvTrackbar() + { + signature = -1; + } + int signature; - HWND hwnd; - char* name; - CvTrackbar* next; - CvWindow* parent; - HWND buddy; - int* data; - int pos; - int maxval; - int minval; - void (*notify)(int); - void (*notify2)(int, void*); - void* userdata; - int id; -} -CvTrackbar; + HWND hwnd = 0; + std::string name; + CvWindow* parent; // TODO weak_ptr + HWND buddy = 0; + int* data = nullptr; + int pos = 0; + int maxval = 0; + int minval = 0; + void (*notify)(int) = nullptr; // deprecated + void (*notify2)(int, void*) = nullptr; // deprecated + TrackbarCallback onChangeCallback = nullptr; + void* userdata = nullptr; + int id = -1; +}; -typedef struct CvWindow +struct CvWindow : public std::enable_shared_from_this { + CvWindow(const std::string& name_) + : signature(CV_WINDOW_MAGIC_VAL) + , name(name_) + { + // nothing + } + + ~CvWindow() + { + signature = -1; + } + + void destroy(); + int signature; - HWND hwnd; - char* name; - CvWindow* prev; - CvWindow* next; - HWND frame; + cv::Mutex mutex; + HWND hwnd = 0; + std::string name; + HWND frame = 0; - HDC dc; - HGDIOBJ image; - int last_key; - int flags; - int status;//0 normal, 1 fullscreen (YV) + HDC dc = 0; + HGDIOBJ image = 0; + int last_key = 0; + int flags = 0; + int status = 0;//0 normal, 1 fullscreen (YV) - CvMouseCallback on_mouse; - void* on_mouse_param; + CvMouseCallback on_mouse = nullptr; + void* on_mouse_param = nullptr; struct { - HWND toolbar; - int pos; - int rows; - WNDPROC toolBarProc; - CvTrackbar* first; + HWND toolbar = 0; + int pos = 0; + int rows = 0; + WNDPROC toolBarProc = nullptr; + std::vector< std::shared_ptr > trackbars; } toolbar; - int width; - int height; + int width = -1; + int height = -1; // OpenGL support #ifdef HAVE_OPENGL - bool useGl; - HGLRC hGLRC; + bool useGl = false; + HGLRC hGLRC = 0; - CvOpenGlDrawCallback glDrawCallback; - void* glDrawData; + CvOpenGlDrawCallback glDrawCallback = nullptr; + void* glDrawData = nullptr; #endif -} -CvWindow; +}; #define HG_BUDDY_WIDTH 130 @@ -221,19 +247,50 @@ CvWindow; #define TBM_GETTOOLTIPS (WM_USER + 30) #endif -static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); -static LRESULT CALLBACK WindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); -static LRESULT CALLBACK MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); -static void icvUpdateWindowPos( CvWindow* window ); +static +std::vector< std::shared_ptr >& getWindowsList() +{ + static std::vector< std::shared_ptr > g_windows; + return g_windows; +} -static CvWindow* hg_windows = 0; + +// Mutex must be locked +static +std::shared_ptr icvFindWindowByName(const std::string& name) +{ + auto& g_windows = getWindowsList(); + for (auto it = g_windows.begin(); it != g_windows.end(); ++it) + { + auto window = *it; + if (!window) + continue; + if (window->name == name) + return window; + } + return std::shared_ptr(); +} + +static inline +std::shared_ptr icvFindWindowByName(const char* name) +{ + CV_Assert(name); + return icvFindWindowByName(std::string(name)); +} + + + +static LRESULT CALLBACK HighGUIProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); +static LRESULT CALLBACK WindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); +static LRESULT CALLBACK MainWindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); +static void icvUpdateWindowPos(CvWindow& window); typedef int (CV_CDECL * CvWin32WindowCallback)(HWND, UINT, WPARAM, LPARAM, int*); static CvWin32WindowCallback hg_on_preprocess = 0, hg_on_postprocess = 0; static HINSTANCE hg_hinstance = 0; -static const char* highGUIclassName = "HighGUI class"; -static const char* mainHighGUIclassName = "Main HighGUI class"; +static const char* const highGUIclassName = "HighGUI class"; +static const char* const mainHighGUIclassName = "Main HighGUI class"; static void icvCleanupHighgui() { @@ -242,15 +299,15 @@ static void icvCleanupHighgui() UnregisterClass(mainHighGUIclassName, hg_hinstance); } -CV_IMPL int cvInitSystem( int, char** ) +CV_IMPL int cvInitSystem(int, char**) { static int wasInitialized = 0; // check initialization status - if( !wasInitialized ) + if (!wasInitialized) { - // Initialize the storage - hg_windows = 0; + (void)getWindowMutex(); // force mutex initialization + (void)getWindowsList(); // Initialize the storage // Register the class WNDCLASS wndc; @@ -262,7 +319,7 @@ CV_IMPL int cvInitSystem( int, char** ) wndc.lpszClassName = highGUIclassName; wndc.lpszMenuName = highGUIclassName; wndc.hIcon = LoadIcon(0, IDI_APPLICATION); - wndc.hCursor = (HCURSOR)LoadCursor(0, (LPSTR)(size_t)IDC_CROSS ); + wndc.hCursor = (HCURSOR)LoadCursor(0, (LPSTR)(size_t)IDC_CROSS); wndc.hbrBackground = (HBRUSH)GetStockObject(DKGRAY_BRUSH); RegisterClass(&wndc); @@ -273,12 +330,12 @@ CV_IMPL int cvInitSystem( int, char** ) wndc.lpfnWndProc = MainWindowProc; RegisterClass(&wndc); - atexit( icvCleanupHighgui ); + atexit(icvCleanupHighgui); wasInitialized = 1; } - setlocale(LC_NUMERIC,"C"); + setlocale(LC_NUMERIC,"C"); // FIXIT must be removed return 0; } @@ -287,50 +344,58 @@ CV_IMPL int cvStartWindowThread(){ return 0; } -static CvWindow* icvFindWindowByName( const char* name ) -{ - CvWindow* window = hg_windows; - - for( ; window != 0 && strcmp( name, window->name) != 0; window = window->next ) - ; - return window; -} - - -static CvWindow* icvWindowByHWND( HWND hwnd ) +static std::shared_ptr icvWindowByHWND(HWND hwnd) { - CvWindow* window = (CvWindow*)icvGetWindowLongPtr( hwnd, CV_USERDATA ); - return window != 0 && hg_windows != 0 && + AutoLock lock(getWindowMutex()); + CvWindow* window = (CvWindow*)icvGetWindowLongPtr(hwnd, CV_USERDATA); + window = window != 0 && window->signature == CV_WINDOW_MAGIC_VAL ? window : 0; + if (window) + { + return window->shared_from_this(); + } + else + { + return std::shared_ptr(); + } } -static CvTrackbar* icvTrackbarByHWND( HWND hwnd ) +static std::shared_ptr icvTrackbarByHWND(HWND hwnd) { - CvTrackbar* trackbar = (CvTrackbar*)icvGetWindowLongPtr( hwnd, CV_USERDATA ); - return trackbar != 0 && trackbar->signature == CV_TRACKBAR_MAGIC_VAL && + AutoLock lock(getWindowMutex()); + CvTrackbar* trackbar = (CvTrackbar*)icvGetWindowLongPtr(hwnd, CV_USERDATA); + trackbar = trackbar != 0 && trackbar->signature == CV_TRACKBAR_MAGIC_VAL && trackbar->hwnd == hwnd ? trackbar : 0; + if (trackbar) + { + return trackbar->shared_from_this(); + } + else + { + return std::shared_ptr(); + } } -static const char* icvWindowPosRootKey = "Software\\OpenCV\\HighGUI\\Windows\\"; +static const char* const icvWindowPosRootKey = "Software\\OpenCV\\HighGUI\\Windows\\"; // Window positions saving/loading added by Philip Gruebele. //pgruebele@cox.net // Restores the window position from the registry saved position. static void -icvLoadWindowPos( const char* name, CvRect& rect ) +icvLoadWindowPos(const char* name, CvRect& rect) { HKEY hkey; char szKey[1024]; - strcpy_s( szKey, 1024, icvWindowPosRootKey ); - strcat_s( szKey, 1024, name ); + strcpy_s(szKey, 1024, icvWindowPosRootKey); + strcat_s(szKey, 1024, name); rect.x = rect.y = CW_USEDEFAULT; rect.width = rect.height = 320; - if( RegOpenKeyEx(HKEY_CURRENT_USER,szKey,0,KEY_QUERY_VALUE,&hkey) == ERROR_SUCCESS ) + if (RegOpenKeyEx(HKEY_CURRENT_USER,szKey,0,KEY_QUERY_VALUE,&hkey) == ERROR_SUCCESS) { // Yes we are installed. DWORD dwType = 0; @@ -379,16 +444,16 @@ icvLoadWindowPos( const char* name, CvRect& rect ) //pgruebele@cox.net // philipg. Saves the window position in the registry static void -icvSaveWindowPos( const char* name, CvRect rect ) +icvSaveWindowPos(const char* name, CvRect rect) { static const DWORD MAX_RECORD_COUNT = 100; HKEY hkey; char szKey[1024]; char rootKey[1024]; - strcpy_s( szKey, 1024, icvWindowPosRootKey ); - strcat_s( szKey, 1024, name ); + strcpy_s(szKey, 1024, icvWindowPosRootKey); + strcat_s(szKey, 1024, name); - if( RegOpenKeyEx( HKEY_CURRENT_USER,szKey,0,KEY_READ,&hkey) != ERROR_SUCCESS ) + if (RegOpenKeyEx(HKEY_CURRENT_USER,szKey,0,KEY_READ,&hkey) != ERROR_SUCCESS) { HKEY hroot; DWORD count = 0; @@ -396,40 +461,40 @@ icvSaveWindowPos( const char* name, CvRect rect ) char oldestKey[1024]; char currentKey[1024]; - strcpy_s( rootKey, 1024, icvWindowPosRootKey ); + strcpy_s(rootKey, 1024, icvWindowPosRootKey); rootKey[strlen(rootKey)-1] = '\0'; - if( RegCreateKeyEx(HKEY_CURRENT_USER, rootKey, 0, NULL, REG_OPTION_NON_VOLATILE, KEY_READ+KEY_WRITE, 0, &hroot, NULL) != ERROR_SUCCESS ) - //RegOpenKeyEx( HKEY_CURRENT_USER,rootKey,0,KEY_READ,&hroot) != ERROR_SUCCESS ) + if (RegCreateKeyEx(HKEY_CURRENT_USER, rootKey, 0, NULL, REG_OPTION_NON_VOLATILE, KEY_READ+KEY_WRITE, 0, &hroot, NULL) != ERROR_SUCCESS) + //RegOpenKeyEx(HKEY_CURRENT_USER,rootKey,0,KEY_READ,&hroot) != ERROR_SUCCESS) return; for(;;) { DWORD csize = sizeof(currentKey); FILETIME accesstime = { 0, 0 }; - LONG code = RegEnumKeyEx( hroot, count, currentKey, &csize, NULL, NULL, NULL, &accesstime ); - if( code != ERROR_SUCCESS && code != ERROR_MORE_DATA ) + LONG code = RegEnumKeyEx(hroot, count, currentKey, &csize, NULL, NULL, NULL, &accesstime); + if (code != ERROR_SUCCESS && code != ERROR_MORE_DATA) break; count++; - if( oldestTime.dwHighDateTime > accesstime.dwHighDateTime || + if (oldestTime.dwHighDateTime > accesstime.dwHighDateTime || (oldestTime.dwHighDateTime == accesstime.dwHighDateTime && - oldestTime.dwLowDateTime > accesstime.dwLowDateTime) ) + oldestTime.dwLowDateTime > accesstime.dwLowDateTime)) { oldestTime = accesstime; - strcpy_s( oldestKey, 1024, currentKey ); + strcpy_s(oldestKey, 1024, currentKey); } } - if( count >= MAX_RECORD_COUNT ) - RegDeleteKey( hroot, oldestKey ); - RegCloseKey( hroot ); + if (count >= MAX_RECORD_COUNT) + RegDeleteKey(hroot, oldestKey); + RegCloseKey(hroot); - if( RegCreateKeyEx(HKEY_CURRENT_USER,szKey,0,NULL,REG_OPTION_NON_VOLATILE, KEY_WRITE, 0, &hkey, NULL) != ERROR_SUCCESS ) + if (RegCreateKeyEx(HKEY_CURRENT_USER,szKey,0,NULL,REG_OPTION_NON_VOLATILE, KEY_WRITE, 0, &hkey, NULL) != ERROR_SUCCESS) return; } else { - RegCloseKey( hkey ); - if( RegOpenKeyEx( HKEY_CURRENT_USER,szKey,0,KEY_WRITE,&hkey) != ERROR_SUCCESS ) + RegCloseKey(hkey); + if (RegOpenKeyEx(HKEY_CURRENT_USER,szKey,0,KEY_WRITE,&hkey) != ERROR_SUCCESS) return; } @@ -440,96 +505,101 @@ icvSaveWindowPos( const char* name, CvRect rect ) RegCloseKey(hkey); } +static Rect getImageRect_(CvWindow& window); + CvRect cvGetWindowRect_W32(const char* name) { - RECT rect = { 0 }; - CvRect result = cvRect(-1, -1, -1, -1); - - CV_FUNCNAME( "cvGetWindowRect_W32" ); + CV_FUNCNAME("cvGetWindowRect_W32"); - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); - window = icvFindWindowByName( name ); + CV_Error(Error::StsNullPtr, "NULL name string"); + + auto window = icvFindWindowByName(name); if (!window) - EXIT; // keep silence here + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); - GetClientRect(window->hwnd, &rect); - { + Rect r = getImageRect_(*window); + + CvRect result = cvRect(r.x, r.y, r.width, r.height); + return result; +} + +static Rect getImageRect_(CvWindow& window) +{ + RECT rect = { 0 }; + GetClientRect(window.hwnd, &rect); POINT pt = {rect.left, rect.top}; - ClientToScreen(window->hwnd, &pt); - result = cvRect(pt.x, pt.y, rect.right - rect.left, rect.bottom - rect.top); - } - __END__; + ClientToScreen(window.hwnd, &pt); + Rect result(pt.x, pt.y, rect.right - rect.left, rect.bottom - rect.top); return result; } double cvGetModeWindow_W32(const char* name)//YV { - double result = -1; - - CV_FUNCNAME( "cvGetModeWindow_W32" ); - - __BEGIN__; + CV_FUNCNAME("cvGetModeWindow_W32"); - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - EXIT; // keep silence here + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); - result = window->status; - - __END__; - return result; + return window->status; } -void cvSetModeWindow_W32( const char* name, double prop_value)//Yannick Verdie +static bool setModeWindow_(CvWindow& window, int mode); + +void cvSetModeWindow_W32(const char* name, double prop_value)//Yannick Verdie { - CV_FUNCNAME( "cvSetModeWindow_W32" ); + CV_FUNCNAME("cvSetModeWindow_W32"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name string"); - if(!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); - window = icvFindWindowByName( name ); - if( !window ) - CV_ERROR( CV_StsNullPtr, "NULL window" ); + (void)setModeWindow_(*window, (int)prop_value); +} - if(window->flags & CV_WINDOW_AUTOSIZE)//if the flag CV_WINDOW_AUTOSIZE is set - EXIT; +static bool setModeWindow_(CvWindow& window, int mode) +{ + if (window.flags & CV_WINDOW_AUTOSIZE)//if the flag CV_WINDOW_AUTOSIZE is set + return false; + + if (window.status == mode) + return true; { - DWORD dwStyle = (DWORD)GetWindowLongPtr(window->frame, GWL_STYLE); + DWORD dwStyle = (DWORD)GetWindowLongPtr(window.frame, GWL_STYLE); CvRect position; - if (window->status==CV_WINDOW_FULLSCREEN && prop_value==CV_WINDOW_NORMAL) + if (window.status == CV_WINDOW_FULLSCREEN && mode == CV_WINDOW_NORMAL) { - icvLoadWindowPos(window->name,position ); - SetWindowLongPtr(window->frame, GWL_STYLE, dwStyle | WS_CAPTION | WS_THICKFRAME); + icvLoadWindowPos(window.name.c_str(), position); + SetWindowLongPtr(window.frame, GWL_STYLE, dwStyle | WS_CAPTION | WS_THICKFRAME); - SetWindowPos(window->frame, HWND_TOP, position.x, position.y , position.width,position.height, SWP_NOZORDER | SWP_FRAMECHANGED); - window->status=CV_WINDOW_NORMAL; + SetWindowPos(window.frame, HWND_TOP, position.x, position.y , position.width,position.height, SWP_NOZORDER | SWP_FRAMECHANGED); + window.status=CV_WINDOW_NORMAL; - EXIT; + return true; } - if (window->status==CV_WINDOW_NORMAL && prop_value==CV_WINDOW_FULLSCREEN) + if (window.status == CV_WINDOW_NORMAL && mode == CV_WINDOW_FULLSCREEN) { //save dimension RECT rect = { 0 }; - GetWindowRect(window->frame, &rect); - CvRect RectCV = cvRect(rect.left, rect.top,rect.right - rect.left, rect.bottom - rect.top); - icvSaveWindowPos(window->name,RectCV ); + GetWindowRect(window.frame, &rect); + CvRect rectCV = cvRect(rect.left, rect.top,rect.right - rect.left, rect.bottom - rect.top); + icvSaveWindowPos(window.name.c_str(), rectCV); //Look at coordinate for fullscreen HMONITOR hMonitor; @@ -542,60 +612,75 @@ void cvSetModeWindow_W32( const char* name, double prop_value)//Yannick Verdie //fullscreen position.x=mi.rcMonitor.left;position.y=mi.rcMonitor.top; position.width=mi.rcMonitor.right - mi.rcMonitor.left;position.height=mi.rcMonitor.bottom - mi.rcMonitor.top; - SetWindowLongPtr(window->frame, GWL_STYLE, dwStyle & ~WS_CAPTION & ~WS_THICKFRAME); + SetWindowLongPtr(window.frame, GWL_STYLE, dwStyle & ~WS_CAPTION & ~WS_THICKFRAME); - SetWindowPos(window->frame, HWND_TOP, position.x, position.y , position.width,position.height, SWP_NOZORDER | SWP_FRAMECHANGED); - window->status=CV_WINDOW_FULLSCREEN; + SetWindowPos(window.frame, HWND_TOP, position.x, position.y , position.width,position.height, SWP_NOZORDER | SWP_FRAMECHANGED); + window.status=CV_WINDOW_FULLSCREEN; - EXIT; + return true; } } - __END__; + return false; } +static double getPropTopmost_(CvWindow& window); + double cvGetPropTopmost_W32(const char* name) { - double result = -1; - CV_Assert(name); - CvWindow* window = icvFindWindowByName(name); + auto window = icvFindWindowByName(name); if (!window) CV_Error(Error::StsNullPtr, "NULL window"); - LONG style = GetWindowLongA(window->frame, GWL_EXSTYLE); // -20 + return getPropTopmost_(*window); +} + +static double getPropTopmost_(CvWindow& window) +{ + LONG style = GetWindowLongA(window.frame, GWL_EXSTYLE); // -20 if (!style) { std::ostringstream errorMsg; - errorMsg << "window(" << name << "): failed to retrieve extended window style using GetWindowLongA(); error code: " << GetLastError(); - CV_Error(Error::StsError, errorMsg.str().c_str()); + errorMsg << "window(" << window.name << "): failed to retrieve extended window style using GetWindowLongA(); error code: " << GetLastError(); + CV_Error(Error::StsError, errorMsg.str()); } - result = (style & WS_EX_TOPMOST) == WS_EX_TOPMOST; - - return result; + bool result = (style & WS_EX_TOPMOST) == WS_EX_TOPMOST; + return result ? 1.0 : 0.0; } +static bool setPropTopmost_(CvWindow& window, bool topmost); + void cvSetPropTopmost_W32(const char* name, const bool topmost) { CV_Assert(name); - CvWindow* window = icvFindWindowByName(name); + auto window = icvFindWindowByName(name); if (!window) CV_Error(Error::StsNullPtr, "NULL window"); + (void)setPropTopmost_(*window, topmost); +} + +static bool setPropTopmost_(CvWindow& window, bool topmost) +{ HWND flag = topmost ? HWND_TOPMOST : HWND_TOP; - BOOL success = SetWindowPos(window->frame, flag, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE); + BOOL success = SetWindowPos(window.frame, flag, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE); if (!success) { std::ostringstream errorMsg; - errorMsg << "window(" << name << "): error reported by SetWindowPos(" << (topmost ? "HWND_TOPMOST" : "HWND_TOP") << "), error code: " << GetLastError(); - CV_Error(Error::StsError, errorMsg.str().c_str()); + errorMsg << "window(" << window.name << "): error reported by SetWindowPos(" << (topmost ? "HWND_TOPMOST" : "HWND_TOP") << "), error code: " << GetLastError(); + CV_Error(Error::StsError, errorMsg.str()); + return false; } + return true; } +static double getPropVsync_(CvWindow& window); + double cvGetPropVsync_W32(const char* name) { #ifndef HAVE_OPENGL @@ -605,40 +690,53 @@ double cvGetPropVsync_W32(const char* name) if (!name) CV_Error(Error::StsNullPtr, "'name' argument must not be NULL"); - CvWindow* window = icvFindWindowByName(name); + auto window = icvFindWindowByName(name); if (!window) CV_Error_(Error::StsBadArg, ("there is no window named '%s'", name)); + double result = getPropVsync_(*window); + return cvIsNaN(result) ? -1.0 : result; +#endif +} + +static double getPropVsync_(CvWindow& window) +{ +#ifndef HAVE_OPENGL + CV_UNUSED(window); + CV_Error(Error::OpenGlNotSupported, "Library was built without OpenGL support"); +#else // https://www.khronos.org/opengl/wiki/Swap_Interval // https://www.khronos.org/registry/OpenGL/extensions/EXT/WGL_EXT_extensions_string.txt // https://www.khronos.org/registry/OpenGL/extensions/EXT/WGL_EXT_swap_control.txt - if (!wglMakeCurrent(window->dc, window->hGLRC)) + if (!wglMakeCurrent(window.dc, window.hGLRC)) CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); typedef const char* (APIENTRY* PFNWGLGETEXTENSIONSSTRINGEXTPROC)(void); PFNWGLGETEXTENSIONSSTRINGEXTPROC wglGetExtensionsString = NULL; wglGetExtensionsString = (PFNWGLGETEXTENSIONSSTRINGEXTPROC)wglGetProcAddress("wglGetExtensionsStringEXT"); if (wglGetExtensionsString == NULL) - return -1; // wglGetProcAddress failed to get wglGetExtensionsStringEXT + return std::numeric_limits::quiet_NaN(); // wglGetProcAddress failed to get wglGetExtensionsStringEXT const char* wgl_extensions = wglGetExtensionsString(); if (wgl_extensions == NULL) - return -1; // Can't get WGL extensions string + return std::numeric_limits::quiet_NaN(); // Can't get WGL extensions string if (strstr(wgl_extensions, "WGL_EXT_swap_control") == NULL) - return -1; // WGL extensions don't contain WGL_EXT_swap_control + return std::numeric_limits::quiet_NaN(); // WGL extensions don't contain WGL_EXT_swap_control typedef int (APIENTRY* PFNWGLGETSWAPINTERVALPROC)(void); PFNWGLGETSWAPINTERVALPROC wglGetSwapInterval = 0; wglGetSwapInterval = (PFNWGLGETSWAPINTERVALPROC)wglGetProcAddress("wglGetSwapIntervalEXT"); if (wglGetSwapInterval == NULL) - return -1; // wglGetProcAddress failed to get wglGetSwapIntervalEXT + return std::numeric_limits::quiet_NaN(); // wglGetProcAddress failed to get wglGetSwapIntervalEXT return wglGetSwapInterval(); #endif } +static bool setPropVsync_(CvWindow& window, bool enable_vsync); + void cvSetPropVsync_W32(const char* name, const bool enable_vsync) { #ifndef HAVE_OPENGL @@ -649,11 +747,22 @@ void cvSetPropVsync_W32(const char* name, const bool enable_vsync) if (!name) CV_Error(Error::StsNullPtr, "'name' argument must not be NULL"); - CvWindow* window = icvFindWindowByName(name); + auto window = icvFindWindowByName(name); if (!window) CV_Error_(Error::StsBadArg, ("there is no window named '%s'", name)); - if (!wglMakeCurrent(window->dc, window->hGLRC)) + (void)setPropVsync_(*window, enable_vsync); +#endif +} + +static bool setPropVsync_(CvWindow& window, bool enable_vsync) +{ +#ifndef HAVE_OPENGL + CV_UNUSED(window); + CV_UNUSED(enable_vsync); + CV_Error(Error::OpenGlNotSupported, "Library was built without OpenGL support"); +#else + if (!wglMakeCurrent(window.dc, window.hGLRC)) CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); typedef const char* (APIENTRY* PFNWGLGETEXTENSIONSSTRINGEXTPROC)(void); @@ -676,47 +785,44 @@ void cvSetPropVsync_W32(const char* name, const bool enable_vsync) CV_Error(Error::OpenGlApiCallError, "wglGetProcAddress failed to get wglSwapIntervalEXT"); wglSwapInterval(enable_vsync); + return true; #endif } -void cv::setWindowTitle(const String& winname, const String& title) +void setWindowTitle_W32(const std::string& name, const std::string& title) { - CvWindow* window = icvFindWindowByName(winname.c_str()); + auto window = icvFindWindowByName(name); if (!window) { - namedWindow(winname); - window = icvFindWindowByName(winname.c_str()); + namedWindow(name); + window = icvFindWindowByName(name); } if (!window) CV_Error(Error::StsNullPtr, "NULL window"); if (!SetWindowText(window->frame, title.c_str())) - CV_Error_(Error::StsError, ("Failed to set \"%s\" window title to \"%s\"", winname.c_str(), title.c_str())); + CV_Error_(Error::StsError, ("Failed to set \"%s\" window title to \"%s\"", name.c_str(), title.c_str())); } double cvGetPropWindowAutoSize_W32(const char* name) { double result = -1; - CV_FUNCNAME( "cvSetCloseCallback" ); + CV_FUNCNAME("cvSetCloseCallback"); - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - EXIT; // keep silence here + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); result = window->flags & CV_WINDOW_AUTOSIZE; - __END__; - return result; } @@ -724,23 +830,19 @@ double cvGetRatioWindow_W32(const char* name) { double result = -1; - CV_FUNCNAME( "cvGetRatioWindow_W32" ); + CV_FUNCNAME("cvGetRatioWindow_W32"); - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - EXIT; // keep silence here + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); result = static_cast(window->width) / window->height; - __END__; - return result; } @@ -749,23 +851,20 @@ double cvGetOpenGlProp_W32(const char* name) double result = -1; #ifdef HAVE_OPENGL - CV_FUNCNAME( "cvGetOpenGlProp_W32" ); + CV_FUNCNAME("cvGetOpenGlProp_W32"); - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - EXIT; // keep silence here + return -1; result = window->useGl; - - __END__; #endif + CV_UNUSED(name); return result; @@ -775,16 +874,15 @@ double cvGetPropVisible_W32(const char* name) { double result = -1; - CV_FUNCNAME( "cvGetPropVisible_W32" ); + CV_FUNCNAME("cvGetPropVisible_W32"); - __BEGIN__; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); - - result = (icvFindWindowByName( name ) != NULL); + CV_Error(Error::StsNullPtr, "NULL name string"); - __END__; + auto window = icvFindWindowByName(name); + result = (bool)window ? 1.0 : 0.0; return result; } @@ -798,9 +896,9 @@ namespace { void createGlContext(HWND hWnd, HDC& hGLDC, HGLRC& hGLRC, bool& useGl) { - CV_FUNCNAME( "createGlContext" ); + CV_FUNCNAME("createGlContext"); - __BEGIN__; + AutoLock lock(getWindowMutex()); useGl = false; @@ -830,120 +928,119 @@ namespace hGLDC = GetDC(hWnd); if (!hGLDC) - CV_ERROR( CV_OpenGlApiCallError, "Can't Create A GL Device Context" ); + CV_Error(Error::OpenGlApiCallError, "Can't Create A GL Device Context"); PixelFormat = ChoosePixelFormat(hGLDC, &pfd); if (!PixelFormat) - CV_ERROR( CV_OpenGlApiCallError, "Can't Find A Suitable PixelFormat" ); + CV_Error(Error::OpenGlApiCallError, "Can't Find A Suitable PixelFormat"); if (!SetPixelFormat(hGLDC, PixelFormat, &pfd)) - CV_ERROR( CV_OpenGlApiCallError, "Can't Set The PixelFormat" ); + CV_Error(Error::OpenGlApiCallError, "Can't Set The PixelFormat"); hGLRC = wglCreateContext(hGLDC); if (!hGLRC) - CV_ERROR( CV_OpenGlApiCallError, "Can't Create A GL Rendering Context" ); + CV_Error(Error::OpenGlApiCallError, "Can't Create A GL Rendering Context"); if (!wglMakeCurrent(hGLDC, hGLRC)) - CV_ERROR( CV_OpenGlApiCallError, "Can't Activate The GL Rendering Context" ); + CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); useGl = true; - - __END__; } - void releaseGlContext(CvWindow* window) + void releaseGlContext(CvWindow& window) { - //CV_FUNCNAME( "releaseGlContext" ); + //CV_FUNCNAME("releaseGlContext"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - if (window->hGLRC) + if (window.hGLRC) { - wglDeleteContext(window->hGLRC); - window->hGLRC = NULL; + wglDeleteContext(window.hGLRC); + window.hGLRC = NULL; } - if (window->dc) + if (window.dc) { - ReleaseDC(window->hwnd, window->dc); - window->dc = NULL; + ReleaseDC(window.hwnd, window.dc); + window.dc = NULL; } - window->useGl = false; - - __END__; + window.useGl = false; } - void drawGl(CvWindow* window) + void drawGl(CvWindow& window) { - CV_FUNCNAME( "drawGl" ); + CV_FUNCNAME("drawGl"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - if (!wglMakeCurrent(window->dc, window->hGLRC)) - CV_ERROR( CV_OpenGlApiCallError, "Can't Activate The GL Rendering Context" ); + if (!wglMakeCurrent(window.dc, window.hGLRC)) + CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); - if (window->glDrawCallback) - window->glDrawCallback(window->glDrawData); + if (window.glDrawCallback) + window.glDrawCallback(window.glDrawData); - if (!SwapBuffers(window->dc)) - CV_ERROR( CV_OpenGlApiCallError, "Can't swap OpenGL buffers" ); - - __END__; + if (!SwapBuffers(window.dc)) + CV_Error(Error::OpenGlApiCallError, "Can't swap OpenGL buffers"); } - void resizeGl(CvWindow* window) + void resizeGl(CvWindow& window) { - CV_FUNCNAME( "resizeGl" ); - - __BEGIN__; + CV_FUNCNAME("resizeGl"); - if (!wglMakeCurrent(window->dc, window->hGLRC)) - CV_ERROR( CV_OpenGlApiCallError, "Can't Activate The GL Rendering Context" ); + AutoLock lock(getWindowMutex()); - glViewport(0, 0, window->width, window->height); + if (!wglMakeCurrent(window.dc, window.hGLRC)) + CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); - __END__; + glViewport(0, 0, window.width, window.height); } } #endif // HAVE_OPENGL +static std::shared_ptr namedWindow_(const std::string& name, int flags); + +CV_IMPL int cvNamedWindow(const char* name, int flags) +{ + CV_FUNCNAME("cvNamedWindow"); + + AutoLock lock(getWindowMutex()); + + if (!name) + CV_Error(Error::StsNullPtr, "NULL name string"); + + // Check the name in the storage + auto window = icvFindWindowByName(name); + if (window) + { + return 1; + } -CV_IMPL int cvNamedWindow( const char* name, int flags ) + window = namedWindow_(name, flags); + return (bool)window; +} + +static std::shared_ptr namedWindow_(const std::string& name, int flags) { - int result = 0; - CV_FUNCNAME( "cvNamedWindow" ); + AutoLock lock(getWindowMutex()); - __BEGIN__; + cvInitSystem(0,0); HWND hWnd, mainhWnd; - CvWindow* window; DWORD defStyle = WS_VISIBLE | WS_MINIMIZEBOX | WS_MAXIMIZEBOX | WS_SYSMENU; - int len; - CvRect rect; #ifdef HAVE_OPENGL bool useGl; HDC hGLDC; HGLRC hGLRC; #endif - cvInitSystem(0,0); - - if( !name ) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); - - // Check the name in the storage - window = icvFindWindowByName( name ); - if (window != 0) - { - result = 1; - EXIT; - } + CvRect rect; + icvLoadWindowPos(name.c_str(), rect); - if( !(flags & CV_WINDOW_AUTOSIZE))//YV add border in order to resize the window + if (!(flags & CV_WINDOW_AUTOSIZE))//YV add border in order to resize the window defStyle |= WS_SIZEBOX; #ifdef HAVE_OPENGL @@ -951,23 +1048,21 @@ CV_IMPL int cvNamedWindow( const char* name, int flags ) defStyle |= WS_CLIPCHILDREN | WS_CLIPSIBLINGS; #endif - icvLoadWindowPos( name, rect ); - - mainhWnd = CreateWindow( "Main HighGUI class", name, defStyle | WS_OVERLAPPED, - rect.x, rect.y, rect.width, rect.height, 0, 0, hg_hinstance, 0 ); - if( !mainhWnd ) - CV_ERROR( CV_StsError, "Frame window can not be created" ); + mainhWnd = CreateWindow(mainHighGUIclassName, name.c_str(), defStyle | WS_OVERLAPPED, + rect.x, rect.y, rect.width, rect.height, 0, 0, hg_hinstance, 0); + if (!mainhWnd) + CV_Error_(Error::StsError, ("Frame window can not be created: '%s'", name.c_str())); ShowWindow(mainhWnd, SW_SHOW); //YV- remove one border by changing the style - hWnd = CreateWindow("HighGUI class", "", (defStyle & ~WS_SIZEBOX) | WS_CHILD, CW_USEDEFAULT, 0, rect.width, rect.height, mainhWnd, 0, hg_hinstance, 0); - if( !hWnd ) - CV_ERROR( CV_StsError, "Frame window can not be created" ); + hWnd = CreateWindow(highGUIclassName, "", (defStyle & ~WS_SIZEBOX) | WS_CHILD, CW_USEDEFAULT, 0, rect.width, rect.height, mainhWnd, 0, hg_hinstance, 0); + if (!hWnd) + CV_Error(Error::StsError, "Frame window can not be created"); #ifndef HAVE_OPENGL if (flags & CV_WINDOW_OPENGL) - CV_ERROR( CV_OpenGlNotSupported, "Library was built without OpenGL support" ); + CV_Error(Error::OpenGlNotSupported, "Library was built without OpenGL support"); #else useGl = false; hGLDC = 0; @@ -979,14 +1074,10 @@ CV_IMPL int cvNamedWindow( const char* name, int flags ) ShowWindow(hWnd, SW_SHOW); - len = (int)strlen(name); - CV_CALL( window = (CvWindow*)cvAlloc(sizeof(CvWindow) + len + 1)); + auto window = std::make_shared(name); - window->signature = CV_WINDOW_MAGIC_VAL; window->hwnd = hWnd; window->frame = mainhWnd; - window->name = (char*)(window + 1); - memcpy( window->name, name, len + 1 ); window->flags = flags; window->image = 0; @@ -1016,200 +1107,175 @@ CV_IMPL int cvNamedWindow( const char* name, int flags ) window->on_mouse = 0; window->on_mouse_param = 0; - memset( &window->toolbar, 0, sizeof(window->toolbar)); + icvSetWindowLongPtr(hWnd, CV_USERDATA, window.get()); + icvSetWindowLongPtr(mainhWnd, CV_USERDATA, window.get()); - window->next = hg_windows; - window->prev = 0; - if( hg_windows ) - hg_windows->prev = window; - hg_windows = window; - icvSetWindowLongPtr( hWnd, CV_USERDATA, window ); - icvSetWindowLongPtr( mainhWnd, CV_USERDATA, window ); + auto& g_windows = getWindowsList(); + g_windows.push_back(window); // Recalculate window pos - icvUpdateWindowPos( window ); + icvUpdateWindowPos(*window); - result = 1; - __END__; - - return result; + return window; } #ifdef HAVE_OPENGL CV_IMPL void cvSetOpenGlContext(const char* name) { - CV_FUNCNAME( "cvSetOpenGlContext" ); - - __BEGIN__; + CV_FUNCNAME("cvSetOpenGlContext"); - CvWindow* window; + AutoLock lock(getWindowMutex()); - if(!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + if (!name) + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - CV_ERROR( CV_StsNullPtr, "NULL window" ); + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); if (!window->useGl) - CV_ERROR( CV_OpenGlNotSupported, "Window doesn't support OpenGL" ); + CV_Error(Error::OpenGlNotSupported, "Window doesn't support OpenGL"); if (!wglMakeCurrent(window->dc, window->hGLRC)) - CV_ERROR( CV_OpenGlApiCallError, "Can't Activate The GL Rendering Context" ); - - __END__; + CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); } CV_IMPL void cvUpdateWindow(const char* name) { - CV_FUNCNAME( "cvUpdateWindow" ); + CV_FUNCNAME("cvUpdateWindow"); - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - EXIT; + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); InvalidateRect(window->hwnd, 0, 0); - - __END__; } CV_IMPL void cvSetOpenGlDrawCallback(const char* name, CvOpenGlDrawCallback callback, void* userdata) { - CV_FUNCNAME( "cvCreateOpenGLCallback" ); + CV_FUNCNAME("cvCreateOpenGLCallback"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; - - if(!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + if (!name) + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); - if( !window ) - EXIT; + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); if (!window->useGl) - CV_ERROR( CV_OpenGlNotSupported, "Window was created without OpenGL context" ); + CV_Error(Error::OpenGlNotSupported, "Window was created without OpenGL context"); window->glDrawCallback = callback; window->glDrawData = userdata; - - __END__; } #endif // HAVE_OPENGL -static void icvRemoveWindow( CvWindow* window ) +static void icvRemoveWindow(const std::shared_ptr& window_) { - CvTrackbar* trackbar = NULL; + CV_Assert(window_); + AutoLock lock(getWindowMutex()); + CvWindow& window = *window_; + RECT wrect={0,0,0,0}; + auto& g_windows = getWindowsList(); + for (auto it = g_windows.begin(); it != g_windows.end(); ++it) + { + const std::shared_ptr& w = *it; + if (w.get() == &window) + { + g_windows.erase(it); + break; + } + } + #ifdef HAVE_OPENGL - if (window->useGl) + if (window.useGl) releaseGlContext(window); #endif - if( window->frame ) - GetWindowRect( window->frame, &wrect ); - if( window->name ) - icvSaveWindowPos( window->name, cvRect(wrect.left, wrect.top, - wrect.right-wrect.left, wrect.bottom-wrect.top) ); - - if( window->hwnd ) - icvSetWindowLongPtr( window->hwnd, CV_USERDATA, 0 ); - if( window->frame ) - icvSetWindowLongPtr( window->frame, CV_USERDATA, 0 ); + if (window.frame) + GetWindowRect(window.frame, &wrect); + icvSaveWindowPos(window.name.c_str(), cvRect(wrect.left, wrect.top, wrect.right-wrect.left, wrect.bottom-wrect.top)); - if( window->toolbar.toolbar ) - icvSetWindowLongPtr(window->toolbar.toolbar, CV_USERDATA, 0); + if (window.hwnd) + icvSetWindowLongPtr(window.hwnd, CV_USERDATA, 0); + if (window.frame) + icvSetWindowLongPtr(window.frame, CV_USERDATA, 0); - if( window->prev ) - window->prev->next = window->next; - else - hg_windows = window->next; - - if( window->next ) - window->next->prev = window->prev; + if (window.toolbar.toolbar) + icvSetWindowLongPtr(window.toolbar.toolbar, CV_USERDATA, 0); - window->prev = window->next = 0; + if (window.dc && window.image) + DeleteObject(SelectObject(window.dc, window.image)); - if( window->dc && window->image ) - DeleteObject(SelectObject(window->dc,window->image)); + if (window.dc) + DeleteDC(window.dc); - if( window->dc ) - DeleteDC(window->dc); - - for( trackbar = window->toolbar.first; trackbar != 0; ) + for (auto it = window.toolbar.trackbars.begin(); it != window.toolbar.trackbars.end(); ++it) { - CvTrackbar* next = trackbar->next; - if( trackbar->hwnd ) + auto trackbar = (*it).get(); + if (trackbar && trackbar->hwnd) { - icvSetWindowLongPtr( trackbar->hwnd, CV_USERDATA, 0 ); - cvFree( &trackbar ); + icvSetWindowLongPtr(trackbar->hwnd, CV_USERDATA, 0); } - trackbar = next; } - - cvFree( &window ); } -CV_IMPL void cvDestroyWindow( const char* name ) +CV_IMPL void cvDestroyWindow(const char* name) { - CV_FUNCNAME( "cvDestroyWindow" ); + CV_FUNCNAME("cvDestroyWindow"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; - HWND mainhWnd; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name string"); - if(!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); - window = icvFindWindowByName( name ); - if( !window ) - EXIT; + window->destroy(); +} - mainhWnd = window->frame; - SendMessage(window->hwnd, WM_CLOSE, 0, 0); - SendMessage( mainhWnd, WM_CLOSE, 0, 0); +void CvWindow::destroy() +{ + SendMessage(hwnd, WM_CLOSE, 0, 0); + SendMessage(frame, WM_CLOSE, 0, 0); // Do NOT call _remove_window -- CvWindow list will be updated automatically ... - - __END__; } - -static void icvScreenToClient( HWND hwnd, RECT* rect ) +static void icvScreenToClient(HWND hwnd, RECT* rect) { POINT p; p.x = rect->left; p.y = rect->top; ScreenToClient(hwnd, &p); - OffsetRect( rect, p.x - rect->left, p.y - rect->top ); + OffsetRect(rect, p.x - rect->left, p.y - rect->top); } /* Calculatess the window coordinates relative to the upper left corner of the mainhWnd window */ -static RECT icvCalcWindowRect( CvWindow* window ) +static RECT icvCalcWindowRect(CvWindow& window) { RECT crect = { 0 }, trect = { 0 }, rect = { 0 }; - assert(window); - - GetClientRect(window->frame, &crect); - if (window->toolbar.toolbar) + GetClientRect(window.frame, &crect); + if (window.toolbar.toolbar) { - GetWindowRect(window->toolbar.toolbar, &trect); - icvScreenToClient(window->frame, &trect); + GetWindowRect(window.toolbar.toolbar, &trect); + icvScreenToClient(window.frame, &trect); SubtractRect(&rect, &crect, &trect); } else @@ -1217,138 +1283,153 @@ static RECT icvCalcWindowRect( CvWindow* window ) return rect; } +static inline RECT icvCalcWindowRect(CvWindow* window) { CV_Assert(window); return icvCalcWindowRect(*window); } + -// returns TRUE if there is a problem such as ERROR_IO_PENDING. -static bool icvGetBitmapData( CvWindow* window, SIZE* size, int* channels, void** data ) +// returns FALSE if there is a problem such as ERROR_IO_PENDING. +static bool icvGetBitmapData(CvWindow& window, SIZE& size, int& channels, void*& data) { - BITMAP bmp; GdiFlush(); - HGDIOBJ h = GetCurrentObject( window->dc, OBJ_BITMAP ); - if( size ) - size->cx = size->cy = 0; - if( data ) - *data = 0; + + HGDIOBJ h = GetCurrentObject(window.dc, OBJ_BITMAP); + size.cx = size.cy = 0; + data = 0; if (h == NULL) - return true; + return false; + + BITMAP bmp = {}; if (GetObject(h, sizeof(bmp), &bmp) == 0) - return true; + return false; - if( size ) - { - size->cx = abs(bmp.bmWidth); - size->cy = abs(bmp.bmHeight); - } + size.cx = abs(bmp.bmWidth); + size.cy = abs(bmp.bmHeight); - if( channels ) - *channels = bmp.bmBitsPixel/8; + channels = bmp.bmBitsPixel/8; - if( data ) - *data = bmp.bmBits; + data = bmp.bmBits; - return false; + return true; +} +static bool icvGetBitmapData(CvWindow& window, SIZE& size) +{ + int channels = 0; + void* data = nullptr; + return icvGetBitmapData(window, size, channels, data); } -static void icvUpdateWindowPos( CvWindow* window ) +static void icvUpdateWindowPos(CvWindow& window) { RECT rect = { 0 }; - assert(window); - if( (window->flags & CV_WINDOW_AUTOSIZE) && window->image ) + if ((window.flags & CV_WINDOW_AUTOSIZE) && window.image) { int i; SIZE size = {0,0}; - icvGetBitmapData( window, &size, 0, 0 ); + icvGetBitmapData(window, size); // TODO check return value? // Repeat two times because after the first resizing of the mainhWnd window // toolbar may resize too - for(i = 0; i < (window->toolbar.toolbar ? 2 : 1); i++) + for(i = 0; i < (window.toolbar.toolbar ? 2 : 1); i++) { - RECT rmw = { 0 }, rw = icvCalcWindowRect(window ); - MoveWindow(window->hwnd, rw.left, rw.top, + RECT rmw = { 0 }, rw = icvCalcWindowRect(&window); + MoveWindow(window.hwnd, rw.left, rw.top, rw.right - rw.left, rw.bottom - rw.top, FALSE); - GetClientRect(window->hwnd, &rw); - GetWindowRect(window->frame, &rmw); + GetClientRect(window.hwnd, &rw); + GetWindowRect(window.frame, &rmw); // Resize the mainhWnd window in order to make the bitmap fit into the child window - MoveWindow(window->frame, rmw.left, rmw.top, + MoveWindow(window.frame, rmw.left, rmw.top, size.cx + (rmw.right - rmw.left) - (rw.right - rw.left), - size.cy + (rmw.bottom - rmw.top) - (rw.bottom - rw.top), TRUE ); + size.cy + (rmw.bottom - rmw.top) - (rw.bottom - rw.top), TRUE); } } rect = icvCalcWindowRect(window); - MoveWindow(window->hwnd, rect.left, rect.top, + MoveWindow(window.hwnd, rect.left, rect.top, rect.right - rect.left, - rect.bottom - rect.top, TRUE ); + rect.bottom - rect.top, TRUE); } +static void showImage_(CvWindow& window, const Mat& image); + CV_IMPL void -cvShowImage( const char* name, const CvArr* arr ) +cvShowImage(const char* name, const CvArr* arr) { - CV_FUNCNAME( "cvShowImage" ); - - __BEGIN__; - - CvWindow* window; - SIZE size = { 0, 0 }; - int channels = 0; - void* dst_ptr = 0; - const int channels0 = 3; - CvMat stub, *image; - bool changed_size = false; // philipg + CV_FUNCNAME("cvShowImage"); - if( !name ) - CV_ERROR( CV_StsNullPtr, "NULL name" ); + if (!name) + CV_Error(Error::StsNullPtr, "NULL name"); - window = icvFindWindowByName(name); - if(!window) + std::shared_ptr window; { - cvNamedWindow(name, CV_WINDOW_AUTOSIZE); + AutoLock lock(getWindowMutex()); + window = icvFindWindowByName(name); + if (!window) + { + cvNamedWindow(name, CV_WINDOW_AUTOSIZE); + window = icvFindWindowByName(name); + } } - if( !window || !arr ) - EXIT; // keep silence here. - - CV_CALL( image = cvGetMat( arr, &stub )); + if (!window || !arr) + return; // keep silence here. + CvMat stub = {}; + CvMat* image_c = cvGetMat(arr, &stub); + Mat image = cv::cvarrToMat(image_c); #ifdef HAVE_OPENGL if (window->useGl) { - cv::imshow(name, cv::cvarrToMat(image)); + cv::imshow(name, image); return; } #endif + return showImage_(*window, image); +} + +static void showImage_(CvWindow& window, const Mat& image) +{ + AutoLock lock(window.mutex); - if (window->image) + SIZE size = { 0, 0 }; + int channels = 0; + void* dst_ptr = 0; + const int channels0 = 3; + bool changed_size = false; // philipg + + if (window.image) + { // if there is something wrong with these system calls, we cannot display image... - if (icvGetBitmapData( window, &size, &channels, &dst_ptr )) + if (!icvGetBitmapData(window, size, channels, dst_ptr)) return; + } - if( size.cx != image->width || size.cy != image->height || channels != channels0 ) + if (size.cx != image.cols || size.cy != image.rows || channels != channels0) { changed_size = true; uchar buffer[sizeof(BITMAPINFO) + 255*sizeof(RGBQUAD)]; BITMAPINFO* binfo = (BITMAPINFO*)buffer; - DeleteObject( SelectObject( window->dc, window->image )); - window->image = 0; + DeleteObject(SelectObject(window.dc, window.image)); + window.image = 0; - size.cx = image->width; - size.cy = image->height; + size.cx = image.cols; + size.cy = image.rows; channels = channels0; - FillBitmapInfo( binfo, size.cx, size.cy, channels*8, 1 ); + FillBitmapInfo(binfo, size.cx, size.cy, channels*8, 1); - window->image = SelectObject( window->dc, CreateDIBSection(window->dc, binfo, - DIB_RGB_COLORS, &dst_ptr, 0, 0)); + window.image = SelectObject(window.dc, + CreateDIBSection(window.dc, binfo, DIB_RGB_COLORS, &dst_ptr, 0, 0) + ); } { cv::Mat dst(size.cy, size.cx, CV_8UC3, dst_ptr, (size.cx * channels + 3) & -4); - convertToShow(cv::cvarrToMat(image), dst, false); + convertToShow(image, dst, false); CV_Assert(dst.data == (uchar*)dst_ptr); cv::flip(dst, dst, 0); } @@ -1356,98 +1437,103 @@ cvShowImage( const char* name, const CvArr* arr ) // only resize window if needed if (changed_size) icvUpdateWindowPos(window); - InvalidateRect(window->hwnd, 0, 0); + InvalidateRect(window.hwnd, 0, 0); // philipg: this is not needed and just slows things down // UpdateWindow(window->hwnd); - - __END__; } -CV_IMPL void cvResizeWindow(const char* name, int width, int height ) +static void resizeWindow_(CvWindow& window, const Size& size); + +CV_IMPL void cvResizeWindow(const char* name, int width, int height) { - CV_FUNCNAME( "cvResizeWindow" ); + CV_FUNCNAME("cvResizeWindow"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - int i; - CvWindow* window; - RECT rmw = { 0 }, rw = { 0 }, rect = { 0 }; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name"); - if( !name ) - CV_ERROR( CV_StsNullPtr, "NULL name" ); + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); - window = icvFindWindowByName(name); - if(!window) - EXIT; + return resizeWindow_(*window, Size(width, height)); +} + +static void resizeWindow_(CvWindow& window, const Size& size) +{ + RECT rmw = { 0 }, rw = { 0 }, rect = { 0 }; // Repeat two times because after the first resizing of the mainhWnd window // toolbar may resize too - for(i = 0; i < (window->toolbar.toolbar ? 2 : 1); i++) + for (int i = 0; i < (window.toolbar.toolbar ? 2 : 1); i++) { rw = icvCalcWindowRect(window); - MoveWindow(window->hwnd, rw.left, rw.top, + MoveWindow(window.hwnd, rw.left, rw.top, rw.right - rw.left, rw.bottom - rw.top, FALSE); - GetClientRect(window->hwnd, &rw); - GetWindowRect(window->frame, &rmw); + GetClientRect(window.hwnd, &rw); + GetWindowRect(window.frame, &rmw); // Resize the mainhWnd window in order to make the bitmap fit into the child window - MoveWindow(window->frame, rmw.left, rmw.top, - width + (rmw.right - rmw.left) - (rw.right - rw.left), - height + (rmw.bottom - rmw.top) - (rw.bottom - rw.top), TRUE); + MoveWindow(window.frame, rmw.left, rmw.top, + size.width + (rmw.right - rmw.left) - (rw.right - rw.left), + size.height + (rmw.bottom - rmw.top) - (rw.bottom - rw.top), TRUE); } rect = icvCalcWindowRect(window); - MoveWindow(window->hwnd, rect.left, rect.top, + MoveWindow(window.hwnd, rect.left, rect.top, rect.right - rect.left, rect.bottom - rect.top, TRUE); - - __END__; } +static void moveWindow_(CvWindow& window, const Point& pt); -CV_IMPL void cvMoveWindow( const char* name, int x, int y ) +CV_IMPL void cvMoveWindow(const char* name, int x, int y) { - CV_FUNCNAME( "cvMoveWindow" ); + CV_FUNCNAME("cvMoveWindow"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; - RECT rect = { 0 }; - - if( !name ) - CV_ERROR( CV_StsNullPtr, "NULL name" ); + if (!name) + CV_Error(Error::StsNullPtr, "NULL name"); - window = icvFindWindowByName(name); - if(!window) - EXIT; + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); - GetWindowRect( window->frame, &rect ); - MoveWindow( window->frame, x, y, rect.right - rect.left, rect.bottom - rect.top, TRUE); + (void)moveWindow_(*window, Point(x, y)); +} - __END__; +static void moveWindow_(CvWindow& window, const Point& pt) +{ + RECT rect = { 0 }; + GetWindowRect(window.frame, &rect); // TODO check return value + MoveWindow(window.frame, pt.x, pt.y, rect.right - rect.left, rect.bottom - rect.top, TRUE); } static LRESULT CALLBACK -MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) +MainWindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { - CvWindow* window = icvWindowByHWND( hwnd ); - if( !window ) + auto window_ = icvWindowByHWND(hwnd); + if (!window_) return DefWindowProc(hwnd, uMsg, wParam, lParam); + CvWindow& window = *window_; + switch(uMsg) { case WM_COPY: - ::SendMessage(window->hwnd, uMsg, wParam, lParam); + ::SendMessage(window.hwnd, uMsg, wParam, lParam); break; case WM_DESTROY: - icvRemoveWindow(window); + icvRemoveWindow(window_); // Do nothing!!! //PostQuitMessage(0); break; case WM_GETMINMAXINFO: - if( !(window->flags & CV_WINDOW_AUTOSIZE) ) + if (!(window.flags & CV_WINDOW_AUTOSIZE)) { MINMAXINFO* minmax = (MINMAXINFO*)lParam; RECT rect = { 0 }; @@ -1456,10 +1542,10 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) minmax->ptMinTrackSize.y = 100; minmax->ptMinTrackSize.x = 100; - if( window->toolbar.first ) + if (!window.toolbar.trackbars.empty()) { - GetWindowRect( window->toolbar.first->hwnd, &rect ); - minmax->ptMinTrackSize.y += window->toolbar.rows*(rect.bottom - rect.top); + GetWindowRect(window.toolbar.trackbars[0]->hwnd, &rect); + minmax->ptMinTrackSize.y += window.toolbar.rows*(rect.bottom - rect.top); minmax->ptMinTrackSize.x = MAX(rect.right - rect.left + HG_BUDDY_WIDTH, HG_BUDDY_WIDTH*2); } return retval; @@ -1471,14 +1557,14 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) WINDOWPOS* pos = (WINDOWPOS*)lParam; // Update the toolbar pos/size - if(window->toolbar.toolbar) + if (window.toolbar.toolbar) { RECT rect = { 0 }; - GetWindowRect(window->toolbar.toolbar, &rect); - MoveWindow(window->toolbar.toolbar, 0, 0, pos->cx, rect.bottom - rect.top, TRUE); + GetWindowRect(window.toolbar.toolbar, &rect); + MoveWindow(window.toolbar.toolbar, 0, 0, pos->cx, rect.bottom - rect.top, TRUE); } - if(!(window->flags & CV_WINDOW_AUTOSIZE)) + if (!(window.flags & CV_WINDOW_AUTOSIZE)) icvUpdateWindowPos(window); break; @@ -1490,7 +1576,7 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) LPWINDOWPOS pos = (LPWINDOWPOS)lParam; RECT rect = { 0 }; - GetWindowRect(window->frame, &rect); + GetWindowRect(window.frame, &rect); HMONITOR hMonitor; hMonitor = MonitorFromRect(&rect, MONITOR_DEFAULTTONEAREST); @@ -1515,13 +1601,13 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) } case WM_ACTIVATE: - if(LOWORD(wParam) == WA_ACTIVE || LOWORD(wParam) == WA_CLICKACTIVE) - SetFocus(window->hwnd); + if (LOWORD(wParam) == WA_ACTIVE || LOWORD(wParam) == WA_CLICKACTIVE) + SetFocus(window.hwnd); break; case WM_MOUSEWHEEL: case WM_MOUSEHWHEEL: - if( window->on_mouse ) + if (window.on_mouse) { int flags = (wParam & MK_LBUTTON ? CV_EVENT_FLAG_LBUTTON : 0)| (wParam & MK_RBUTTON ? CV_EVENT_FLAG_RBUTTON : 0)| @@ -1536,32 +1622,32 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) flags |= (delta << 16); POINT pt; - pt.x = GET_X_LPARAM( lParam ); - pt.y = GET_Y_LPARAM( lParam ); + pt.x = GET_X_LPARAM(lParam); + pt.y = GET_Y_LPARAM(lParam); ::ScreenToClient(hwnd, &pt); // Convert screen coordinates to client coordinates. RECT rect = { 0 }; - GetClientRect( window->hwnd, &rect ); + GetClientRect(window.hwnd, &rect); SIZE size = {0,0}; #ifdef HAVE_OPENGL - if (window->useGl) + if (window.useGl) { - cv::ogl::Texture2D* texObj = static_cast(window->glDrawData); + cv::ogl::Texture2D* texObj = static_cast(window.glDrawData); size.cx = texObj->cols(); size.cy = texObj->rows(); } else { - icvGetBitmapData(window, &size, 0, 0); + icvGetBitmapData(window, size); } #else - icvGetBitmapData(window, &size, 0, 0); + icvGetBitmapData(window, size); #endif - window->on_mouse( event, pt.x*size.cx/MAX(rect.right - rect.left,1), - pt.y*size.cy/MAX(rect.bottom - rect.top,1), flags, - window->on_mouse_param ); + int x = cvRound((float)pt.x*size.cx/MAX(rect.right - rect.left,1)); + int y = cvRound((float)pt.y*size.cy/MAX(rect.bottom - rect.top,1)); + window.on_mouse(event, x, y, flags, window.on_mouse_param); } break; @@ -1571,17 +1657,17 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) HRGN rgn, rgn1, rgn2; int ret; HDC hdc = (HDC)wParam; - GetWindowRect(window->hwnd, &cr); - icvScreenToClient(window->frame, &cr); - if(window->toolbar.toolbar) + GetWindowRect(window.hwnd, &cr); + icvScreenToClient(window.frame, &cr); + if (window.toolbar.toolbar) { - GetWindowRect(window->toolbar.toolbar, &tr); - icvScreenToClient(window->frame, &tr); + GetWindowRect(window.toolbar.toolbar, &tr); + icvScreenToClient(window.frame, &tr); } else tr.left = tr.top = tr.right = tr.bottom = 0; - GetClientRect(window->frame, &wrc); + GetClientRect(window.frame, &wrc); rgn = CreateRectRgn(0, 0, wrc.right, wrc.bottom); rgn1 = CreateRectRgn(cr.left, cr.top, cr.right, cr.bottom); @@ -1591,7 +1677,7 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) ret = CombineRgn(rgn, rgn, rgn1, RGN_DIFF); ret = CombineRgn(rgn, rgn, rgn2, RGN_DIFF); - if(ret != NULLREGION && ret != ERROR) + if (ret != NULLREGION && ret != ERROR) FillRgn(hdc, rgn, (HBRUSH)icvGetClassLongPtr(hwnd, CV_HBRBACKGROUND)); DeleteObject(rgn); @@ -1605,20 +1691,24 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) } -static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) +static LRESULT CALLBACK HighGUIProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { - CvWindow* window = icvWindowByHWND(hwnd); - if( !window ) + auto window_ = icvWindowByHWND(hwnd); + if (!window_) + { // This window is not mentioned in HighGUI storage // Actually, this should be error except for the case of calls to CreateWindow return DefWindowProc(hwnd, uMsg, wParam, lParam); + } + + CvWindow& window = *window_; // Process the message switch(uMsg) { case WM_COPY: { - if (!::OpenClipboard(hwnd) ) + if (!::OpenClipboard(hwnd)) break; HDC hDC = 0; @@ -1632,7 +1722,7 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM if (!::EmptyClipboard()) break; - if(!window->image) + if (!window.image) break; // Get window device context @@ -1640,19 +1730,20 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM break; // Create another DC compatible with hDC - if (0 == (memDC = ::CreateCompatibleDC( hDC ))) + if (0 == (memDC = ::CreateCompatibleDC(hDC))) break; // Determine the bitmap's dimensions - int nchannels = 3; SIZE size = {0,0}; - icvGetBitmapData( window, &size, &nchannels, 0 ); + int nchannels = 3; + void* data = NULL; // unused + icvGetBitmapData(window, size, nchannels, data); // Create bitmap to draw on and it in the new DC - if (0 == (memBM = ::CreateCompatibleBitmap ( hDC, size.cx, size.cy))) + if (0 == (memBM = ::CreateCompatibleBitmap(hDC, size.cx, size.cy))) break; - if (!::SelectObject( memDC, memBM )) + if (!::SelectObject(memDC, memBM)) break; // Begin drawing to DC @@ -1660,7 +1751,7 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM break; RGBQUAD table[256]; - if( 1 == nchannels ) + if (1 == nchannels) { for(int i = 0; i < 256; ++i) { @@ -1668,14 +1759,14 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM table[i].rgbGreen = (unsigned char)i; table[i].rgbRed = (unsigned char)i; } - if (!::SetDIBColorTable(window->dc, 0, 255, table)) + if (!::SetDIBColorTable(window.dc, 0, 255, table)) break; } // The image copied to the clipboard will be in its original size, regardless if the window itself was resized. // Render the image to the dc/bitmap (at original size). - if (!::BitBlt( memDC, 0, 0, size.cx, size.cy, window->dc, 0, 0, SRCCOPY )) + if (!::BitBlt(memDC, 0, 0, size.cx, size.cy, window.dc, 0, 0, SRCCOPY)) break; // Finally, set bitmap to clipboard @@ -1712,7 +1803,7 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM case WM_RBUTTONUP: case WM_MBUTTONUP: case WM_MOUSEMOVE: - if( window->on_mouse ) + if (window.on_mouse) { POINT pt; @@ -1732,50 +1823,50 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM uMsg == WM_RBUTTONDBLCLK ? CV_EVENT_RBUTTONDBLCLK : uMsg == WM_MBUTTONDBLCLK ? CV_EVENT_MBUTTONDBLCLK : CV_EVENT_MOUSEMOVE; - if( uMsg == WM_LBUTTONDOWN || uMsg == WM_RBUTTONDOWN || uMsg == WM_MBUTTONDOWN ) - SetCapture( hwnd ); - if( uMsg == WM_LBUTTONUP || uMsg == WM_RBUTTONUP || uMsg == WM_MBUTTONUP ) + if (uMsg == WM_LBUTTONDOWN || uMsg == WM_RBUTTONDOWN || uMsg == WM_MBUTTONDOWN) + SetCapture(hwnd); + if (uMsg == WM_LBUTTONUP || uMsg == WM_RBUTTONUP || uMsg == WM_MBUTTONUP) ReleaseCapture(); - pt.x = GET_X_LPARAM( lParam ); - pt.y = GET_Y_LPARAM( lParam ); + pt.x = GET_X_LPARAM(lParam); + pt.y = GET_Y_LPARAM(lParam); - if (window->flags & CV_WINDOW_AUTOSIZE) + if (window.flags & CV_WINDOW_AUTOSIZE) { // As user can't change window size, do not scale window coordinates. Underlying windowing system // may prevent full window from being displayed and in this case coordinates should not be scaled. - window->on_mouse( event, pt.x, pt.y, flags, window->on_mouse_param ); + window.on_mouse(event, pt.x, pt.y, flags, window.on_mouse_param); } else { // Full window is displayed using different size. Scale coordinates to match underlying positions. RECT rect = { 0 }; SIZE size = {0, 0}; - GetClientRect( window->hwnd, &rect ); + GetClientRect(window.hwnd, &rect); #ifdef HAVE_OPENGL - if (window->useGl) + if (window.useGl) { - cv::ogl::Texture2D* texObj = static_cast(window->glDrawData); + cv::ogl::Texture2D* texObj = static_cast(window.glDrawData); size.cx = texObj->cols(); size.cy = texObj->rows(); } else { - icvGetBitmapData(window, &size, 0, 0); + icvGetBitmapData(window, size); } #else - icvGetBitmapData( window, &size, 0, 0 ); + icvGetBitmapData(window, size); #endif - window->on_mouse( event, pt.x*size.cx/MAX(rect.right - rect.left,1), - pt.y*size.cy/MAX(rect.bottom - rect.top,1), flags, - window->on_mouse_param ); + int x = cvRound((float)pt.x*size.cx/MAX(rect.right - rect.left,1)); + int y = cvRound((float)pt.y*size.cy/MAX(rect.bottom - rect.top,1)); + window.on_mouse(event, x, y, flags, window.on_mouse_param); } } break; case WM_PAINT: - if(window->image != 0) + if (window.image != 0) { int nchannels = 3; SIZE size = {0,0}; @@ -1784,12 +1875,13 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM RGBQUAD table[256]; // Determine the bitmap's dimensions - icvGetBitmapData( window, &size, &nchannels, 0 ); + void* data = 0; // unused + icvGetBitmapData(window, size, nchannels, data); hdc = BeginPaint(hwnd, &paint); SetStretchBltMode(hdc, COLORONCOLOR); - if( nchannels == 1 ) + if (nchannels == 1) { int i; for(i = 0; i < 256; i++) @@ -1798,25 +1890,25 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM table[i].rgbGreen = (unsigned char)i; table[i].rgbRed = (unsigned char)i; } - SetDIBColorTable(window->dc, 0, 255, table); + SetDIBColorTable(window.dc, 0, 255, table); } - if(window->flags & CV_WINDOW_AUTOSIZE) + if (window.flags & CV_WINDOW_AUTOSIZE) { - BitBlt( hdc, 0, 0, size.cx, size.cy, window->dc, 0, 0, SRCCOPY ); + BitBlt(hdc, 0, 0, size.cx, size.cy, window.dc, 0, 0, SRCCOPY); } else { RECT rect = { 0 }; - GetClientRect(window->hwnd, &rect); - StretchBlt( hdc, 0, 0, rect.right - rect.left, rect.bottom - rect.top, - window->dc, 0, 0, size.cx, size.cy, SRCCOPY ); + GetClientRect(window.hwnd, &rect); + StretchBlt(hdc, 0, 0, rect.right - rect.left, rect.bottom - rect.top, + window.dc, 0, 0, size.cx, size.cy, SRCCOPY); } //DeleteDC(hdc); EndPaint(hwnd, &paint); } #ifdef HAVE_OPENGL - else if(window->useGl) + else if (window.useGl) { drawGl(window); return DefWindowProc(hwnd, uMsg, wParam, lParam); @@ -1829,13 +1921,13 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM return 0; case WM_ERASEBKGND: - if(window->image) + if (window.image) return 0; break; case WM_DESTROY: - icvRemoveWindow(window); + icvRemoveWindow(window_); // Do nothing!!! //PostQuitMessage(0); break; @@ -1845,15 +1937,15 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM return 0; case WM_KEYDOWN: - window->last_key = (int)wParam; + window.last_key = (int)wParam; return 0; case WM_SIZE: - window->width = LOWORD(lParam); - window->height = HIWORD(lParam); + window.width = LOWORD(lParam); + window.height = HIWORD(lParam); #ifdef HAVE_OPENGL - if (window->useGl) + if (window.useGl) resizeGl(window); #endif } @@ -1862,24 +1954,24 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM } -static LRESULT CALLBACK WindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) +static LRESULT CALLBACK WindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { LRESULT ret; - if( hg_on_preprocess ) + if (hg_on_preprocess) { int was_processed = 0; int rethg = hg_on_preprocess(hwnd, uMsg, wParam, lParam, &was_processed); - if( was_processed ) + if (was_processed) return rethg; } ret = HighGUIProc(hwnd, uMsg, wParam, lParam); - if(hg_on_postprocess) + if (hg_on_postprocess) { int was_processed = 0; int rethg = hg_on_postprocess(hwnd, uMsg, wParam, lParam, &was_processed); - if( was_processed ) + if (was_processed) return rethg; } @@ -1887,51 +1979,56 @@ static LRESULT CALLBACK WindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM } -static void icvUpdateTrackbar( CvTrackbar* trackbar, int pos ) +static void icvUpdateTrackbar(CvTrackbar& trackbar, int pos) { const int max_name_len = 10; const char* suffix = ""; char pos_text[32]; int name_len; - if( trackbar->data ) - *trackbar->data = pos; + if (trackbar.data) + *trackbar.data = pos; - if( trackbar->pos != pos ) + if (trackbar.pos != pos) { - trackbar->pos = pos; - if( trackbar->notify2 ) - trackbar->notify2(pos, trackbar->userdata); - if( trackbar->notify ) - trackbar->notify(pos); - - name_len = (int)strlen(trackbar->name); - - if( name_len > max_name_len ) + trackbar.pos = pos; + if (trackbar.onChangeCallback) + trackbar.onChangeCallback(pos, trackbar.userdata); + if (trackbar.notify2) + trackbar.notify2(pos, trackbar.userdata); + if (trackbar.notify) + trackbar.notify(pos); + + name_len = (int)trackbar.name.size(); + + // TODO replace C strings manipulation + if (name_len > max_name_len) { int start_len = max_name_len*2/3; int end_len = max_name_len - start_len - 2; - memcpy( pos_text, trackbar->name, start_len ); - memcpy( pos_text + start_len, "...", 3 ); - memcpy( pos_text + start_len + 3, trackbar->name + name_len - end_len, end_len + 1 ); + memcpy(pos_text, trackbar.name.c_str(), start_len); + memcpy(pos_text + start_len, "...", 3); + memcpy(pos_text + start_len + 3, trackbar.name.c_str() + name_len - end_len, end_len + 1); } else { - memcpy( pos_text, trackbar->name, name_len + 1); + memcpy(pos_text, trackbar.name.c_str(), name_len + 1); } - sprintf( pos_text + strlen(pos_text), "%s: %d\n", suffix, pos ); - SetWindowText( trackbar->buddy, pos_text ); + sprintf(pos_text + strlen(pos_text), "%s: %d\n", suffix, pos); + SetWindowText(trackbar.buddy, pos_text); } } -static LRESULT CALLBACK HGToolbarProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) +static LRESULT CALLBACK HGToolbarProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { - CvWindow* window = icvWindowByHWND( hwnd ); - if(!window) + auto window_ = icvWindowByHWND(hwnd); + if (!window_) return DefWindowProc(hwnd, uMsg, wParam, lParam); + CvWindow& window = *window_; + // Control messages processing switch(uMsg) { @@ -1940,32 +2037,34 @@ static LRESULT CALLBACK HGToolbarProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPAR { HWND slider = (HWND)lParam; int pos = (int)SendMessage(slider, TBM_GETPOS, 0, 0); - CvTrackbar* trackbar = icvTrackbarByHWND( slider ); + auto trackbar = icvTrackbarByHWND(slider); - if( trackbar ) + if (trackbar) { - if( trackbar->pos != pos ) - icvUpdateTrackbar( trackbar, pos ); + if (trackbar->pos != pos) + icvUpdateTrackbar(*trackbar, pos); } - SetFocus( window->hwnd ); + SetFocus(window.hwnd); return 0; } case WM_NCCALCSIZE: { - LRESULT ret = CallWindowProc(window->toolbar.toolBarProc, hwnd, uMsg, wParam, lParam); + LRESULT ret = CallWindowProc(window.toolbar.toolBarProc, hwnd, uMsg, wParam, lParam); int rows = (int)SendMessage(hwnd, TB_GETROWS, 0, 0); - if(window->toolbar.rows != rows) + if (window.toolbar.rows != rows) { - SendMessage(window->toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); - CvTrackbar* trackbar = window->toolbar.first; + SendMessage(window.toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); + auto& trakbars = window.toolbar.trackbars; - for( ; trackbar != 0; trackbar = trackbar->next ) + for (auto it = trakbars.begin(); it != trakbars.end(); ++it) { + auto trackbar = *it; + CV_Assert(trackbar); RECT rect = { 0 }; - SendMessage(window->toolbar.toolbar, TB_GETITEMRECT, + SendMessage(window.toolbar.toolbar, TB_GETITEMRECT, (WPARAM)trackbar->id, (LPARAM)&rect); MoveWindow(trackbar->hwnd, rect.left + HG_BUDDY_WIDTH, rect.top, rect.right - rect.left - HG_BUDDY_WIDTH, @@ -1973,46 +2072,63 @@ static LRESULT CALLBACK HGToolbarProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPAR MoveWindow(trackbar->buddy, rect.left, rect.top, HG_BUDDY_WIDTH, rect.bottom - rect.top, FALSE); } - window->toolbar.rows = rows; + window.toolbar.rows = rows; } return ret; } } - return CallWindowProc(window->toolbar.toolBarProc, hwnd, uMsg, wParam, lParam); + return CallWindowProc(window.toolbar.toolBarProc, hwnd, uMsg, wParam, lParam); } CV_IMPL void cvDestroyAllWindows(void) { - CvWindow* window = hg_windows; - - while( window ) + std::vector< std::shared_ptr > g_windows; { - HWND mainhWnd = window->frame; - HWND hwnd = window->hwnd; - window = window->next; + AutoLock lock(getWindowMutex()); + g_windows = getWindowsList(); // copy + } + for (auto it = g_windows.begin(); it != g_windows.end(); ++it) + { + auto window_ = *it; + if (!window_) + continue; + + { + CvWindow& window = *window_; + + HWND mainhWnd = window.frame; + HWND hwnd = window.hwnd; + + SendMessage(hwnd, WM_CLOSE, 0, 0); + SendMessage(mainhWnd, WM_CLOSE, 0, 0); + } - SendMessage( hwnd, WM_CLOSE, 0, 0 ); - SendMessage( mainhWnd, WM_CLOSE, 0, 0 ); + window_.reset(); + } + // TODO needed? + { + AutoLock lock(getWindowMutex()); + getWindowsList().clear(); } } -static void showSaveDialog(CvWindow* window) +static void showSaveDialog(CvWindow& window) { - if (!window || !window->image) + if (!window.image) return; SIZE sz; int channels; void* data; - if (icvGetBitmapData(window, &sz, &channels, &data)) + if (icvGetBitmapData(window, sz, channels, data)) return; // nothing to save char szFileName[MAX_PATH] = ""; // try to use window title as file name - GetWindowText(window->frame, szFileName, MAX_PATH); + GetWindowText(window.frame, szFileName, MAX_PATH); OPENFILENAME ofn; ZeroMemory(&ofn, sizeof(ofn)); @@ -2022,7 +2138,7 @@ static void showSaveDialog(CvWindow* window) #else ofn.lStructSize = sizeof(ofn); #endif - ofn.hwndOwner = window->hwnd; + ofn.hwndOwner = window.hwnd; ofn.lpstrFilter = #ifdef HAVE_PNG "Portable Network Graphics files (*.png)\0*.png\0" @@ -2075,9 +2191,15 @@ static bool handleMessage(MSG& message, int& keyCode) // otherwise the message was handled specifically bool is_processed = false; - for (CvWindow* window = hg_windows; window != 0 && is_processed == 0; window = window->next) + AutoLock lock(getWindowMutex()); + auto& g_windows = getWindowsList(); + for (auto it = g_windows.begin(); it != g_windows.end() && !is_processed; ++it) { - if (!(window->hwnd == message.hwnd || window->frame == message.hwnd)) + auto window_ = *it; + if (!window_) + continue; + CvWindow& window = *window_; + if (!(window.hwnd == message.hwnd || window.frame == message.hwnd)) continue; is_processed = true; @@ -2140,7 +2262,7 @@ static bool handleMessage(MSG& message, int& keyCode) /* * process until queue is empty but don't wait. */ -int cv::pollKey() +int pollKey_W32() { CV_TRACE_FUNCTION(); for(;;) @@ -2156,7 +2278,7 @@ int cv::pollKey() } CV_IMPL int -cvWaitKey( int delay ) +cvWaitKey(int delay) { int64 time0 = cv::getTickCount(); int64 timeEnd = time0 + (int64)(delay * 0.001f * cv::getTickFrequency()); @@ -2165,9 +2287,9 @@ cvWaitKey( int delay ) { MSG message; - if( (delay <= 0) && hg_windows) + if ((delay <= 0) && !getWindowsList().empty()) GetMessage(&message, 0, 0, 0); - else if( PeekMessage(&message, 0, 0, 0, PM_REMOVE) == FALSE ) + else if (PeekMessage(&message, 0, 0, 0, PM_REMOVE) == FALSE) { int64 t = cv::getTickCount(); if (t - timeEnd >= 0) @@ -2183,110 +2305,135 @@ cvWaitKey( int delay ) } -static CvTrackbar* -icvFindTrackbarByName( const CvWindow* window, const char* name ) +static +std::shared_ptr icvFindTrackbarByName(CvWindow& window, const std::string& name) { - CvTrackbar* trackbar = window->toolbar.first; - - for( ; trackbar != 0 && strcmp( trackbar->name, name ) != 0; trackbar = trackbar->next ) - ; - - return trackbar; + auto trackbars = window.toolbar.trackbars; + for (auto it = trackbars.begin(); it != trackbars.end(); ++it) + { + auto& trackbar = *it; + CV_Assert(trackbar); + if (trackbar->name == name) + return trackbar; + } + return std::shared_ptr(); +} +static inline +std::shared_ptr icvFindTrackbarByName(const std::shared_ptr& window, const std::string& name) +{ + CV_Assert(window); + return icvFindTrackbarByName(window, name); } +static +std::shared_ptr createTrackbar_(CvWindow& window, const std::string& trackbar_name, + int count, + TrackbarCallback onChange, void* userdata); static int -icvCreateTrackbar( const char* trackbar_name, const char* window_name, - int* val, int count, CvTrackbarCallback on_notify, - CvTrackbarCallback2 on_notify2, void* userdata ) +icvCreateTrackbar(const char* trackbar_name, const char* window_name, + int* val, int count, CvTrackbarCallback on_notify, + CvTrackbarCallback2 on_notify2, void* userdata) { - int result = 0; + CV_FUNCNAME("icvCreateTrackbar"); - CV_FUNCNAME( "icvCreateTrackbar" ); + AutoLock lock(getWindowMutex()); - __BEGIN__; + if (!window_name || !trackbar_name) + CV_Error(Error::StsNullPtr, "NULL window or trackbar name"); - char slider_name[32]; - CvWindow* window = 0; - CvTrackbar* trackbar = 0; - int pos = 0; + if (count < 0) + CV_Error(Error::StsOutOfRange, "Bad trackbar maximal value"); - if( !window_name || !trackbar_name ) - CV_ERROR( CV_StsNullPtr, "NULL window or trackbar name" ); + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); - if( count < 0 ) - CV_ERROR( CV_StsOutOfRange, "Bad trackbar maximal value" ); + auto trackbar = icvFindTrackbarByName(*window, trackbar_name); + if (!trackbar) + trackbar = createTrackbar_(*window, trackbar_name, count, nullptr, userdata); + CV_Assert(trackbar); - window = icvFindWindowByName(window_name); - if( !window ) - EXIT; + trackbar->notify = on_notify; + trackbar->notify2 = on_notify2; + trackbar->userdata = userdata; + trackbar->data = val; - trackbar = icvFindTrackbarByName(window,trackbar_name); - if( !trackbar ) - { - TBBUTTON tbs = {}; - TBBUTTONINFO tbis = {}; - RECT rect = { 0 }; - int bcount; - int len = (int)strlen( trackbar_name ); + return 1; +} - // create toolbar if it is not created yet - if( !window->toolbar.toolbar ) - { - const int default_height = 30; - - // CreateToolbarEx is deprecated and forces linking against Comctl32.lib. - window->toolbar.toolbar = CreateWindowEx(0, TOOLBARCLASSNAME, NULL, - WS_CHILD | CCS_TOP | TBSTYLE_WRAPABLE | BTNS_AUTOSIZE | BTNS_BUTTON, - 0, 0, 0, 0, - window->frame, NULL, GetModuleHandle(NULL), NULL); - // CreateToolbarEx automatically sends this but CreateWindowEx doesn't. - SendMessage(window->toolbar.toolbar, TB_BUTTONSTRUCTSIZE, (WPARAM)sizeof(TBBUTTON), 0); - - GetClientRect(window->frame, &rect); - MoveWindow( window->toolbar.toolbar, 0, 0, - rect.right - rect.left, default_height, TRUE); - SendMessage(window->toolbar.toolbar, TB_AUTOSIZE, 0, 0); - ShowWindow(window->toolbar.toolbar, SW_SHOW); - - window->toolbar.first = 0; - window->toolbar.pos = 0; - window->toolbar.rows = 0; - window->toolbar.toolBarProc = - (WNDPROC)icvGetWindowLongPtr(window->toolbar.toolbar, CV_WNDPROC); - - icvUpdateWindowPos(window); - - // Subclassing from toolbar - icvSetWindowLongPtr(window->toolbar.toolbar, CV_WNDPROC, HGToolbarProc); - icvSetWindowLongPtr(window->toolbar.toolbar, CV_USERDATA, window); - } +static void createToolbar_(CvWindow& window) +{ + CV_Assert(!window.toolbar.toolbar); + + const int default_height = 30; + + // CreateToolbarEx is deprecated and forces linking against Comctl32.lib. + window.toolbar.toolbar = CreateWindowEx(0, TOOLBARCLASSNAME, NULL, + WS_CHILD | CCS_TOP | TBSTYLE_WRAPABLE | BTNS_AUTOSIZE | BTNS_BUTTON, + 0, 0, 0, 0, + window.frame, NULL, GetModuleHandle(NULL), NULL); + // CreateToolbarEx automatically sends this but CreateWindowEx doesn't. + SendMessage(window.toolbar.toolbar, TB_BUTTONSTRUCTSIZE, (WPARAM)sizeof(TBBUTTON), 0); + + RECT rect; + GetClientRect(window.frame, &rect); + MoveWindow(window.toolbar.toolbar, 0, 0, + rect.right - rect.left, default_height, TRUE); + SendMessage(window.toolbar.toolbar, TB_AUTOSIZE, 0, 0); + ShowWindow(window.toolbar.toolbar, SW_SHOW); + + window.toolbar.pos = 0; + window.toolbar.rows = 0; + window.toolbar.toolBarProc = + (WNDPROC)icvGetWindowLongPtr(window.toolbar.toolbar, CV_WNDPROC); - /* Retrieve current buttons count */ - bcount = (int)SendMessage(window->toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); + icvUpdateWindowPos(window); - if(bcount > 1) - { - /* If this is not the first button then we need to - separate it from the previous one */ - tbs.iBitmap = 0; - tbs.idCommand = bcount; // Set button id to it's number - tbs.iString = 0; - tbs.fsStyle = TBSTYLE_SEP; - tbs.fsState = TBSTATE_ENABLED; - SendMessage(window->toolbar.toolbar, TB_ADDBUTTONS, 1, (LPARAM)&tbs); - - // Retrieve current buttons count - bcount = (int)SendMessage(window->toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); - } + // Subclassing from toolbar + icvSetWindowLongPtr(window.toolbar.toolbar, CV_WNDPROC, HGToolbarProc); + icvSetWindowLongPtr(window.toolbar.toolbar, CV_USERDATA, (void*)&window); + +} + +static +std::shared_ptr createTrackbar_(CvWindow& window, const std::string& trackbar_name, + int count, + TrackbarCallback onChange, void* userdata) +{ + // create toolbar if it is not created yet + if (!window.toolbar.toolbar) + { + createToolbar_(window); + } + + TBBUTTON tbs = {}; - /* Add a button which we're going to cover with the slider */ + /* Retrieve current buttons count */ + int bcount = (int)SendMessage(window.toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); + + if (bcount > 1) + { + /* If this is not the first button then we need to + separate it from the previous one */ tbs.iBitmap = 0; tbs.idCommand = bcount; // Set button id to it's number + tbs.iString = 0; + tbs.fsStyle = TBSTYLE_SEP; tbs.fsState = TBSTATE_ENABLED; + SendMessage(window.toolbar.toolbar, TB_ADDBUTTONS, 1, (LPARAM)&tbs); + + // Retrieve current buttons count + bcount = (int)SendMessage(window.toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); + } + + /* Add a button which we're going to cover with the slider */ + tbs.iBitmap = 0; + tbs.idCommand = bcount; // Set button id to it's number + tbs.fsState = TBSTATE_ENABLED; #if 0/*!defined WIN64 && !defined EM64T*/ - tbs.fsStyle = 0; - tbs.iString = 0; + tbs.fsStyle = 0; + tbs.iString = 0; #else #ifndef TBSTYLE_AUTOSIZE @@ -2296,320 +2443,640 @@ icvCreateTrackbar( const char* trackbar_name, const char* window_name, #ifndef TBSTYLE_GROUP #define TBSTYLE_GROUP 0x0004 #endif - //tbs.fsStyle = TBSTYLE_AUTOSIZE; - tbs.fsStyle = TBSTYLE_GROUP; - tbs.iString = (INT_PTR)trackbar_text; + //tbs.fsStyle = TBSTYLE_AUTOSIZE; + tbs.fsStyle = TBSTYLE_GROUP; + tbs.iString = (INT_PTR)trackbar_text; #endif - SendMessage(window->toolbar.toolbar, TB_ADDBUTTONS, 1, (LPARAM)&tbs); - - /* Adjust button size to the slider */ - tbis.cbSize = sizeof(tbis); - tbis.dwMask = TBIF_SIZE; - - GetClientRect(window->hwnd, &rect); - tbis.cx = (unsigned short)(rect.right - rect.left); - - SendMessage(window->toolbar.toolbar, TB_SETBUTTONINFO, - (WPARAM)tbs.idCommand, (LPARAM)&tbis); - - /* Get button pos */ - SendMessage(window->toolbar.toolbar, TB_GETITEMRECT, - (WPARAM)tbs.idCommand, (LPARAM)&rect); - - /* Create a slider */ - trackbar = (CvTrackbar*)cvAlloc( sizeof(CvTrackbar) + len + 1 ); - trackbar->signature = CV_TRACKBAR_MAGIC_VAL; - trackbar->notify = 0; - trackbar->notify2 = 0; - trackbar->parent = window; - trackbar->pos = 0; - trackbar->data = 0; - trackbar->id = bcount; - trackbar->next = window->toolbar.first; - trackbar->name = (char*)(trackbar + 1); - memcpy( trackbar->name, trackbar_name, len + 1 ); - window->toolbar.first = trackbar; - - sprintf(slider_name, "Trackbar%p", val); - trackbar->hwnd = CreateWindowEx(0, TRACKBAR_CLASS, slider_name, - WS_CHILD | WS_VISIBLE | TBS_AUTOTICKS | - TBS_FIXEDLENGTH | TBS_HORZ | TBS_BOTTOM, - rect.left + HG_BUDDY_WIDTH, rect.top, - rect.right - rect.left - HG_BUDDY_WIDTH, - rect.bottom - rect.top, window->toolbar.toolbar, - (HMENU)(size_t)bcount, hg_hinstance, 0); - - sprintf(slider_name,"Buddy%p", val); - trackbar->buddy = CreateWindowEx(0, "STATIC", slider_name, - WS_CHILD | SS_RIGHT, - rect.left, rect.top, - HG_BUDDY_WIDTH, rect.bottom - rect.top, - window->toolbar.toolbar, 0, hg_hinstance, 0); - - icvSetWindowLongPtr( trackbar->hwnd, CV_USERDATA, trackbar ); - - /* Minimize the number of rows */ - SendMessage( window->toolbar.toolbar, TB_SETROWS, - MAKEWPARAM(1, FALSE), (LPARAM)&rect ); - } - else - { - trackbar->data = 0; - trackbar->notify = 0; - trackbar->notify2 = 0; - } + SendMessage(window.toolbar.toolbar, TB_ADDBUTTONS, 1, (LPARAM)&tbs); + + TBBUTTONINFO tbis = {}; + + /* Adjust button size to the slider */ + tbis.cbSize = sizeof(tbis); + tbis.dwMask = TBIF_SIZE; + + RECT rect = { 0 }; + GetClientRect(window.hwnd, &rect); + tbis.cx = (unsigned short)(rect.right - rect.left); + + SendMessage(window.toolbar.toolbar, TB_SETBUTTONINFO, + (WPARAM)tbs.idCommand, (LPARAM)&tbis); + + /* Get button pos */ + SendMessage(window.toolbar.toolbar, TB_GETITEMRECT, + (WPARAM)tbs.idCommand, (LPARAM)&rect); + + /* Create a slider */ + auto trackbar = std::make_shared(window, trackbar_name); + trackbar->id = bcount; + window.toolbar.trackbars.push_back(trackbar); + + auto slider_name = cv::format("Trackbar%p", trackbar.get()); + trackbar->hwnd = CreateWindowEx(0, TRACKBAR_CLASS, slider_name.c_str(), + WS_CHILD | WS_VISIBLE | TBS_AUTOTICKS | + TBS_FIXEDLENGTH | TBS_HORZ | TBS_BOTTOM, + rect.left + HG_BUDDY_WIDTH, rect.top, + rect.right - rect.left - HG_BUDDY_WIDTH, + rect.bottom - rect.top, window.toolbar.toolbar, + (HMENU)(size_t)bcount, hg_hinstance, 0); + + slider_name = cv::format("Buddy%p", trackbar.get()); + trackbar->buddy = CreateWindowEx(0, "STATIC", slider_name.c_str(), + WS_CHILD | SS_RIGHT, + rect.left, rect.top, + HG_BUDDY_WIDTH, rect.bottom - rect.top, + window.toolbar.toolbar, 0, hg_hinstance, 0); + + icvSetWindowLongPtr(trackbar->hwnd, CV_USERDATA, (void*)trackbar.get()); + + /* Minimize the number of rows */ + SendMessage(window.toolbar.toolbar, TB_SETROWS, + MAKEWPARAM(1, FALSE), (LPARAM)&rect); trackbar->maxval = count; /* Adjust slider parameters */ SendMessage(trackbar->hwnd, TBM_SETRANGEMIN, (WPARAM)TRUE, (LPARAM)0); SendMessage(trackbar->hwnd, TBM_SETRANGEMAX, (WPARAM)TRUE, (LPARAM)count); - SendMessage(trackbar->hwnd, TBM_SETTICFREQ, (WPARAM)1, (LPARAM)0 ); - if( val ) - pos = *val; + SendMessage(trackbar->hwnd, TBM_SETTICFREQ, (WPARAM)1, (LPARAM)0); - SendMessage(trackbar->hwnd, TBM_SETPOS, (WPARAM)TRUE, (LPARAM)pos ); - SendMessage(window->toolbar.toolbar, TB_AUTOSIZE, 0, 0); + int pos = 0; + SendMessage(trackbar->hwnd, TBM_SETPOS, (WPARAM)TRUE, (LPARAM)pos); + SendMessage(window.toolbar.toolbar, TB_AUTOSIZE, 0, 0); trackbar->pos = -1; - icvUpdateTrackbar( trackbar, pos ); - ShowWindow( trackbar->buddy, SW_SHOW ); - ShowWindow( trackbar->hwnd, SW_SHOW ); - - trackbar->notify = on_notify; - trackbar->notify2 = on_notify2; - trackbar->userdata = userdata; - trackbar->data = val; + icvUpdateTrackbar(*trackbar, pos); + ShowWindow(trackbar->buddy, SW_SHOW); + ShowWindow(trackbar->hwnd, SW_SHOW); /* Resize the window to reflect the toolbar resizing*/ icvUpdateWindowPos(window); - result = 1; - - __END__; + trackbar->onChangeCallback = onChange; + trackbar->userdata = userdata; - return result; + return trackbar; } CV_IMPL int -cvCreateTrackbar( const char* trackbar_name, const char* window_name, - int* val, int count, CvTrackbarCallback on_notify ) +cvCreateTrackbar(const char* trackbar_name, const char* window_name, + int* val, int count, CvTrackbarCallback on_notify) { - return icvCreateTrackbar( trackbar_name, window_name, val, count, - on_notify, 0, 0 ); + return icvCreateTrackbar(trackbar_name, window_name, val, count, + on_notify, 0, 0); } CV_IMPL int -cvCreateTrackbar2( const char* trackbar_name, const char* window_name, - int* val, int count, CvTrackbarCallback2 on_notify2, - void* userdata ) +cvCreateTrackbar2(const char* trackbar_name, const char* window_name, + int* val, int count, CvTrackbarCallback2 on_notify2, + void* userdata) { - return icvCreateTrackbar( trackbar_name, window_name, val, count, - 0, on_notify2, userdata ); + return icvCreateTrackbar(trackbar_name, window_name, val, count, + 0, on_notify2, userdata); } CV_IMPL void -cvSetMouseCallback( const char* window_name, CvMouseCallback on_mouse, void* param ) +cvSetMouseCallback(const char* name, CvMouseCallback on_mouse, void* param) { - CV_FUNCNAME( "cvSetMouseCallback" ); - - __BEGIN__; + CV_FUNCNAME("cvSetMouseCallback"); - CvWindow* window = 0; + if (!name) + CV_Error(Error::StsNullPtr, "NULL window name"); - if( !window_name ) - CV_ERROR( CV_StsNullPtr, "NULL window name" ); + AutoLock lock(getWindowMutex()); - window = icvFindWindowByName(window_name); - if( !window ) - EXIT; + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); window->on_mouse = on_mouse; window->on_mouse_param = param; - - __END__; } -CV_IMPL int cvGetTrackbarPos( const char* trackbar_name, const char* window_name ) +CV_IMPL int cvGetTrackbarPos(const char* trackbar_name, const char* window_name) { - int pos = -1; - - CV_FUNCNAME( "cvGetTrackbarPos" ); + CV_FUNCNAME("cvGetTrackbarPos"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; - CvTrackbar* trackbar = 0; + if (trackbar_name == 0 || window_name == 0) + CV_Error(Error::StsNullPtr, "NULL trackbar or window name"); - if( trackbar_name == 0 || window_name == 0 ) - CV_ERROR( CV_StsNullPtr, "NULL trackbar or window name" ); - - window = icvFindWindowByName( window_name ); - if( window ) - trackbar = icvFindTrackbarByName( window, trackbar_name ); - - if( trackbar ) - pos = trackbar->pos; + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); - __END__; + auto trackbar = icvFindTrackbarByName(window, trackbar_name); + if (!trackbar) + CV_Error_(Error::StsNullPtr, ("NULL trackbar: '%s'", trackbar_name)); - return pos; + return trackbar->pos; } -CV_IMPL void cvSetTrackbarPos( const char* trackbar_name, const char* window_name, int pos ) +CV_IMPL void cvSetTrackbarPos(const char* trackbar_name, const char* window_name, int pos) { - CV_FUNCNAME( "cvSetTrackbarPos" ); + CV_FUNCNAME("cvSetTrackbarPos"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; - CvTrackbar* trackbar = 0; + if (trackbar_name == 0 || window_name == 0) + CV_Error(Error::StsNullPtr, "NULL trackbar or window name"); - if( trackbar_name == 0 || window_name == 0 ) - CV_ERROR( CV_StsNullPtr, "NULL trackbar or window name" ); + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); - window = icvFindWindowByName( window_name ); - if( window ) - trackbar = icvFindTrackbarByName( window, trackbar_name ); + auto trackbar = icvFindTrackbarByName(window, trackbar_name); + if (!trackbar) + CV_Error_(Error::StsNullPtr, ("NULL trackbar: '%s'", trackbar_name)); - if( trackbar ) { - if( pos < 0 ) + if (pos < 0) pos = 0; - if( pos > trackbar->maxval ) + if (pos > trackbar->maxval) pos = trackbar->maxval; - SendMessage( trackbar->hwnd, TBM_SETPOS, (WPARAM)TRUE, (LPARAM)pos ); - icvUpdateTrackbar( trackbar, pos ); + SendMessage(trackbar->hwnd, TBM_SETPOS, (WPARAM)TRUE, (LPARAM)pos); + icvUpdateTrackbar(*trackbar, pos); } - - __END__; } CV_IMPL void cvSetTrackbarMax(const char* trackbar_name, const char* window_name, int maxval) { - CV_FUNCNAME( "cvSetTrackbarMax" ); + CV_FUNCNAME("cvSetTrackbarMax"); + + if (trackbar_name == 0 || window_name == 0) + { + CV_Error(Error::StsNullPtr, "NULL trackbar or window name"); + } + + AutoLock lock(getWindowMutex()); + + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); - __BEGIN__; + auto trackbar = icvFindTrackbarByName(window, trackbar_name); + if (!trackbar) + CV_Error_(Error::StsNullPtr, ("NULL trackbar: '%s'", trackbar_name)); + // FIXIT if (maxval >= 0) { - CvWindow* window = 0; - CvTrackbar* trackbar = 0; - if (trackbar_name == 0 || window_name == 0) + // The position will be min(pos, maxval). + trackbar->maxval = (trackbar->minval>maxval)?trackbar->minval:maxval; + SendMessage(trackbar->hwnd, TBM_SETRANGEMAX, (WPARAM)TRUE, (LPARAM)maxval); + } +} + + +CV_IMPL void cvSetTrackbarMin(const char* trackbar_name, const char* window_name, int minval) +{ + CV_FUNCNAME("cvSetTrackbarMin"); + + if (trackbar_name == 0 || window_name == 0) + { + CV_Error(Error::StsNullPtr, "NULL trackbar or window name"); + } + + AutoLock lock(getWindowMutex()); + + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); + + auto trackbar = icvFindTrackbarByName(window, trackbar_name); + if (!trackbar) + CV_Error_(Error::StsNullPtr, ("NULL trackbar: '%s'", trackbar_name)); + + // FIXIT + if (minval >= 0) + { + // The position will be min(pos, maxval). + trackbar->minval = (minvalmaxval)?minval:trackbar->maxval; + SendMessage(trackbar->hwnd, TBM_SETRANGEMIN, (WPARAM)TRUE, (LPARAM)minval); + } +} + + +CV_IMPL void* cvGetWindowHandle(const char* window_name) +{ + CV_FUNCNAME("cvGetWindowHandle"); + + AutoLock lock(getWindowMutex()); + + if (window_name == 0) + CV_Error(Error::StsNullPtr, "NULL window name"); + + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); + + return (void*)window->hwnd; +} + +// FIXIT: result is not safe to use +CV_IMPL const char* cvGetWindowName(void* window_handle) +{ + CV_FUNCNAME("cvGetWindowName"); + + AutoLock lock(getWindowMutex()); + + if (window_handle == 0) + CV_Error(Error::StsNullPtr, "NULL window handle"); + + auto window = icvWindowByHWND((HWND)window_handle); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%p'", window_handle)); + + return window->name.c_str(); +} + + +CV_IMPL void +cvSetPreprocessFuncWin32_(const void* callback) +{ + hg_on_preprocess = (CvWin32WindowCallback)callback; +} + +CV_IMPL void +cvSetPostprocessFuncWin32_(const void* callback) +{ + hg_on_postprocess = (CvWin32WindowCallback)callback; +} + + + +namespace cv { namespace impl { + +using namespace cv::highgui_backend; + +class Win32UITrackbar; + +class Win32UIWindow + : public UIWindow + , public std::enable_shared_from_this +{ +protected: + const std::string name_; + std::weak_ptr window_; + std::map > trackbars_; +public: + Win32UIWindow(const std::string& name, const std::shared_ptr& window) + : name_(name) + , window_(window) + { + // nothing + } + + ~Win32UIWindow() CV_OVERRIDE + { + if (!window_.expired()) + destroy(); + CV_LOG_DEBUG(NULL, "OpenCV/UI/Win32UI: Win32UIWindow(" << name_ << ") is disposed"); + } + + const std::string& getID() const CV_OVERRIDE { return name_; } + + bool isActive() const CV_OVERRIDE { return !window_.expired(); } + + void destroy() CV_OVERRIDE + { + cv::AutoLock lock(getWindowMutex()); + if (!window_.expired()) { - CV_ERROR(CV_StsNullPtr, "NULL trackbar or window name"); + auto window = window_.lock(); + if (window) + window->destroy(); + window_.reset(); } + } - window = icvFindWindowByName(window_name); - if (window) + void imshow(InputArray image) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + Mat image_mat = image.getMat(); + showImage_(window, image_mat); + } + + double getProperty(int prop) const CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + // see cvGetWindowProperty + switch ((WindowPropertyFlags)prop) { - trackbar = icvFindTrackbarByName(window, trackbar_name); - if (trackbar) - { - // The position will be min(pos, maxval). - trackbar->maxval = (trackbar->minval>maxval)?trackbar->minval:maxval; - SendMessage(trackbar->hwnd, TBM_SETRANGEMAX, (WPARAM)TRUE, (LPARAM)maxval); - } + case WND_PROP_FULLSCREEN: + return (double)window.status; + + case WND_PROP_AUTOSIZE: + return (window.flags & WINDOW_AUTOSIZE) ? 1.0 : 0.0; + + case WND_PROP_ASPECT_RATIO: + return static_cast(window.width) / window.height; + +#ifdef HAVE_OPENGL + case WND_PROP_OPENGL: + return window.useGl ? 1.0 : 0.0; +#endif + + case WND_PROP_VISIBLE: + return 1.0; + + case WND_PROP_TOPMOST: + return getPropTopmost_(window); + + case WND_PROP_VSYNC: + return getPropVsync_(window); + + // don't use default, add unsupported cases below: + // case WND_PROP_UNSUPPORTED: // fallthru + // break; } + return std::numeric_limits::quiet_NaN(); } - __END__; -} + bool setProperty(int prop, double value) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + // see cvSetWindowProperty + switch ((WindowPropertyFlags)prop) + { + case WND_PROP_FULLSCREEN: + if (value != WINDOW_NORMAL && value != WINDOW_FULLSCREEN) // bad arg + break; + setModeWindow_(window, (int)value); + return true; + case WND_PROP_TOPMOST: + return setPropTopmost_(window, value != 0.0); -CV_IMPL void cvSetTrackbarMin(const char* trackbar_name, const char* window_name, int minval) -{ - CV_FUNCNAME( "cvSetTrackbarMin" ); + case WND_PROP_VSYNC: + return setPropVsync_(window, value != 0.0); - __BEGIN__; + // don't use default, add unsupported cases below: + // case WND_PROP_UNSUPPORTED: // fallthru + case WND_PROP_AUTOSIZE: // fallthru + case WND_PROP_ASPECT_RATIO: // fallthru + case WND_PROP_OPENGL: // fallthru + case WND_PROP_VISIBLE: // fallthru + break; + } + return false; + } - if (minval >= 0) + void resize(int width, int height) CV_OVERRIDE { - CvWindow* window = 0; - CvTrackbar* trackbar = 0; - if (trackbar_name == 0 || window_name == 0) + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + resizeWindow_(window, Size(width, height)); + } + + void move(int x, int y) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + moveWindow_(window, Point(x, y)); + } + + Rect getImageRect() const CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + return getImageRect_(window); + } + + void setTitle(const std::string& title) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + if (!SetWindowText(window.frame, title.c_str())) + CV_Error_(Error::StsError, ("Failed to set \"%s\" window title to \"%s\"", window.name.c_str(), title.c_str())); + } + + void setMouseCallback(MouseCallback onMouse, void* userdata /*= 0*/) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + window.on_mouse = onMouse; + window.on_mouse_param = userdata; + } + + std::shared_ptr createTrackbar( + const std::string& name, + int count, + TrackbarCallback onChange /*= 0*/, + void* userdata /*= 0*/ + ) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + CV_LOG_INFO(NULL, "OpenCV/UI: Creating Win32UI trackbar at '" << name_ << "': '" << name << "'"); + auto trackbar = createTrackbar_(window, name, count, onChange, userdata); + auto ui_trackbar = std::make_shared(name, trackbar, shared_from_this()); { - CV_ERROR(CV_StsNullPtr, "NULL trackbar or window name"); + cv::AutoLock lock(getWindowMutex()); + trackbars_.emplace(name, ui_trackbar); } + return std::static_pointer_cast(ui_trackbar); + } - window = icvFindWindowByName(window_name); - if (window) + std::shared_ptr findTrackbar(const std::string& name) CV_OVERRIDE + { + cv::AutoLock lock(getWindowMutex()); + auto i = trackbars_.find(name); + if (i != trackbars_.end()) { - trackbar = icvFindTrackbarByName(window, trackbar_name); - if (trackbar) - { - // The position will be min(pos, maxval). - trackbar->minval = (minvalmaxval)?minval:trackbar->maxval; - SendMessage(trackbar->hwnd, TBM_SETRANGEMIN, (WPARAM)TRUE, (LPARAM)minval); - } + return std::static_pointer_cast(i->second); } + return std::shared_ptr(); } - - __END__; -} +}; // Win32UIWindow -CV_IMPL void* cvGetWindowHandle( const char* window_name ) +class Win32UITrackbar : public UITrackbar { - void* hwnd = 0; +protected: + /*const*/ std::string name_; + std::weak_ptr trackbar_; + std::weak_ptr parent_; + std::map > trackbars_; +public: + Win32UITrackbar(const std::string& name, const std::shared_ptr& trackbar, const std::shared_ptr& parent) + : trackbar_(trackbar) + , parent_(parent) + { + name_ = std::string("<") + name + ">@" + parent->getID(); + } - CV_FUNCNAME( "cvGetWindowHandle" ); + ~Win32UITrackbar() CV_OVERRIDE + { + if (!trackbar_.expired()) + destroy(); + CV_LOG_DEBUG(NULL, "OpenCV/UI/Win32UI: Win32UITrackbar(" << name_ << ") is disposed"); + } - __BEGIN__; + const std::string& getID() const CV_OVERRIDE { return name_; } - CvWindow* window; + bool isActive() const CV_OVERRIDE { return !trackbar_.expired(); } - if( window_name == 0 ) - CV_ERROR( CV_StsNullPtr, "NULL window name" ); + void destroy() CV_OVERRIDE + { + // nothing (destroyed with parent window, dedicated trackbar removal is not supported) + } - window = icvFindWindowByName( window_name ); - if( window ) - hwnd = (void*)window->hwnd; + int getPos() const CV_OVERRIDE + { + auto trackbar_ptr = trackbar_.lock(); + CV_Assert(trackbar_ptr); + CvTrackbar& trackbar = *trackbar_ptr; + return trackbar.pos; + } + void setPos(int pos) CV_OVERRIDE + { + auto trackbar_ptr = trackbar_.lock(); + CV_Assert(trackbar_ptr); + CvTrackbar& trackbar = *trackbar_ptr; + SendMessage(trackbar.hwnd, TBM_SETPOS, (WPARAM)TRUE, (LPARAM)pos); + icvUpdateTrackbar(trackbar, pos); + } - __END__; + cv::Range getRange() const CV_OVERRIDE + { + auto trackbar_ptr = trackbar_.lock(); + CV_Assert(trackbar_ptr); + CvTrackbar& trackbar = *trackbar_ptr; + return cv::Range(trackbar.minval, trackbar.maxval); + } - return hwnd; -} + void setRange(const cv::Range& range) CV_OVERRIDE + { + auto trackbar_ptr = trackbar_.lock(); + CV_Assert(trackbar_ptr); + CvTrackbar& trackbar = *trackbar_ptr; + CV_CheckLE(range.start, range.end, "Invalid trackbar range"); + trackbar.minval = range.start; + trackbar.maxval = range.start; + SendMessage(trackbar.hwnd, TBM_SETRANGEMIN, (WPARAM)TRUE, (LPARAM)trackbar.minval); + SendMessage(trackbar.hwnd, TBM_SETRANGEMAX, (WPARAM)TRUE, (LPARAM)trackbar.maxval); + } +}; // Win32UITrackbar -CV_IMPL const char* cvGetWindowName( void* window_handle ) +class Win32BackendUI : public UIBackend { - const char* window_name = ""; +public: + ~Win32BackendUI() CV_OVERRIDE + { + destroyAllWindows(); + } - CV_FUNCNAME( "cvGetWindowName" ); + void destroyAllWindows() CV_OVERRIDE + { + cvDestroyAllWindows(); + } - __BEGIN__; + // namedWindow + virtual std::shared_ptr createWindow( + const std::string& winname, + int flags + ) CV_OVERRIDE + { + CV_LOG_INFO(NULL, "OpenCV/UI: Creating Win32UI window: " << winname << " (" << flags << ")"); + auto window = namedWindow_(winname, flags); + auto ui_window = std::make_shared(winname, window); + return ui_window; + } - CvWindow* window; + int waitKeyEx(int delay) CV_OVERRIDE + { + return cvWaitKey(delay); + } + int pollKey() CV_OVERRIDE + { + return pollKey_W32(); + } +}; // Win32BackendUI - if( window_handle == 0 ) - CV_ERROR( CV_StsNullPtr, "NULL window" ); +static +std::shared_ptr& getInstance() +{ + static std::shared_ptr g_instance = std::make_shared(); + return g_instance; +} - window = icvWindowByHWND( (HWND)window_handle ); - if( window ) - window_name = window->name; +} // namespace impl - __END__; +#ifndef BUILD_PLUGIN +namespace highgui_backend { - return window_name; +std::shared_ptr createUIBackendWin32UI() +{ + return impl::getInstance(); } +} // namespace highgui_backend +#endif -CV_IMPL void -cvSetPreprocessFuncWin32_(const void* callback) +} // namespace + + +#ifdef BUILD_PLUGIN + +#define ABI_VERSION 0 +#define API_VERSION 0 +#include "plugin_api.hpp" + +static +CvResult cv_getInstance(CV_OUT CvPluginUIBackend* handle) CV_NOEXCEPT { - hg_on_preprocess = (CvWin32WindowCallback)callback; + try + { + if (!handle) + return CV_ERROR_FAIL; + *handle = cv::impl::getInstance().get(); + return CV_ERROR_OK; + } + catch (...) + { + return CV_ERROR_FAIL; + } } -CV_IMPL void -cvSetPostprocessFuncWin32_(const void* callback) +static const OpenCV_UI_Plugin_API plugin_api = { - hg_on_postprocess = (CvWin32WindowCallback)callback; + { + sizeof(OpenCV_UI_Plugin_API), ABI_VERSION, API_VERSION, + CV_VERSION_MAJOR, CV_VERSION_MINOR, CV_VERSION_REVISION, CV_VERSION_STATUS, + "Win32 OpenCV UI plugin" + }, + { + /* 1*/cv_getInstance + } +}; + +const OpenCV_UI_Plugin_API* CV_API_CALL opencv_ui_plugin_init_v0(int requested_abi_version, int requested_api_version, void* /*reserved=NULL*/) CV_NOEXCEPT +{ + if (requested_abi_version == ABI_VERSION && requested_api_version <= API_VERSION) + return &plugin_api; + return NULL; } -#endif //_WIN32 +#endif // BUILD_PLUGIN + +#endif // HAVE_WIN32UI From 55e1dfb778b806272262871c3060ae32a4ff20d0 Mon Sep 17 00:00:00 2001 From: SamFC10 Date: Sun, 20 Jun 2021 13:19:29 +0530 Subject: [PATCH 011/376] Fix BatchNorm reinitialization --- modules/dnn/src/layers/batch_norm_layer.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index 27c3db6c4414..42676c79386f 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -29,6 +29,7 @@ namespace dnn class BatchNormLayerImpl CV_FINAL : public BatchNormLayer { public: + Mat origin_weights, origin_bias; Mat weights_, bias_; UMat umat_weight, umat_bias; mutable int dims; @@ -82,11 +83,11 @@ class BatchNormLayerImpl CV_FINAL : public BatchNormLayer const float* weightsData = hasWeights ? blobs[weightsBlobIndex].ptr() : 0; const float* biasData = hasBias ? blobs[biasBlobIndex].ptr() : 0; - weights_.create(1, (int)n, CV_32F); - bias_.create(1, (int)n, CV_32F); + origin_weights.create(1, (int)n, CV_32F); + origin_bias.create(1, (int)n, CV_32F); - float* dstWeightsData = weights_.ptr(); - float* dstBiasData = bias_.ptr(); + float* dstWeightsData = origin_weights.ptr(); + float* dstBiasData = origin_bias.ptr(); for (size_t i = 0; i < n; ++i) { @@ -94,15 +95,12 @@ class BatchNormLayerImpl CV_FINAL : public BatchNormLayer dstWeightsData[i] = w; dstBiasData[i] = (hasBias ? biasData[i] : 0.0f) - w * meanData[i] * varMeanScale; } - // We will use blobs to store origin weights and bias to restore them in case of reinitialization. - weights_.copyTo(blobs[0].reshape(1, 1)); - bias_.copyTo(blobs[1].reshape(1, 1)); } virtual void finalize(InputArrayOfArrays, OutputArrayOfArrays) CV_OVERRIDE { - blobs[0].reshape(1, 1).copyTo(weights_); - blobs[1].reshape(1, 1).copyTo(bias_); + origin_weights.reshape(1, 1).copyTo(weights_); + origin_bias.reshape(1, 1).copyTo(bias_); } void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE From bb60cb0bf97338848859e5402518ff9b53c2a6d1 Mon Sep 17 00:00:00 2001 From: kikaxa Date: Thu, 20 May 2021 16:01:36 +0300 Subject: [PATCH 012/376] Reenable filesystem for ios builds --- .../core/include/opencv2/core/utils/filesystem.private.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/include/opencv2/core/utils/filesystem.private.hpp b/modules/core/include/opencv2/core/utils/filesystem.private.hpp index ea2591c9de1d..72b2bb947968 100644 --- a/modules/core/include/opencv2/core/utils/filesystem.private.hpp +++ b/modules/core/include/opencv2/core/utils/filesystem.private.hpp @@ -16,8 +16,8 @@ # define OPENCV_HAVE_FILESYSTEM_SUPPORT 1 # elif defined(__APPLE__) # include -# if (defined(TARGET_OS_OSX) && TARGET_OS_OSX) || (!defined(TARGET_OS_OSX) && !TARGET_OS_IPHONE) -# define OPENCV_HAVE_FILESYSTEM_SUPPORT 1 // OSX only +# if (defined(TARGET_OS_OSX) && TARGET_OS_OSX) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS) +# define OPENCV_HAVE_FILESYSTEM_SUPPORT 1 // OSX, iOS only # endif # else /* unknown */ From 8be86cbdfd3c941fc12500bcb61a034a9fa9e148 Mon Sep 17 00:00:00 2001 From: Dale Phurrough Date: Tue, 22 Jun 2021 05:32:54 +0200 Subject: [PATCH 013/376] add usageFlags to UMat static factories - add abi compatible overloads - add test case --- modules/core/include/opencv2/core/mat.hpp | 27 ++++++++++++++------- modules/core/src/matrix_operations.cpp | 8 +++---- modules/core/src/umatrix.cpp | 28 +++++++++++----------- modules/core/test/ocl/test_matrix_expr.cpp | 18 ++++++++++++++ 4 files changed, 54 insertions(+), 27 deletions(-) diff --git a/modules/core/include/opencv2/core/mat.hpp b/modules/core/include/opencv2/core/mat.hpp index 5e667b213242..6768be76834b 100644 --- a/modules/core/include/opencv2/core/mat.hpp +++ b/modules/core/include/opencv2/core/mat.hpp @@ -2451,7 +2451,8 @@ class CV_EXPORTS UMat //! <0 - a diagonal from the lower half) UMat diag(int d=0) const; //! constructs a square diagonal matrix which main diagonal is vector "d" - static UMat diag(const UMat& d); + static UMat diag(const UMat& d, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat diag(const UMat& d) { return diag(d, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload //! returns deep copy of the matrix, i.e. the data is copied UMat clone() const CV_NODISCARD; @@ -2485,14 +2486,22 @@ class CV_EXPORTS UMat double dot(InputArray m) const; //! Matlab-style matrix initialization - static UMat zeros(int rows, int cols, int type); - static UMat zeros(Size size, int type); - static UMat zeros(int ndims, const int* sz, int type); - static UMat ones(int rows, int cols, int type); - static UMat ones(Size size, int type); - static UMat ones(int ndims, const int* sz, int type); - static UMat eye(int rows, int cols, int type); - static UMat eye(Size size, int type); + static UMat zeros(int rows, int cols, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat zeros(Size size, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat zeros(int ndims, const int* sz, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat zeros(int rows, int cols, int type) { return zeros(rows, cols, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + static UMat zeros(Size size, int type) { return zeros(size, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + static UMat zeros(int ndims, const int* sz, int type) { return zeros(ndims, sz, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + static UMat ones(int rows, int cols, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat ones(Size size, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat ones(int ndims, const int* sz, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat ones(int rows, int cols, int type) { return ones(rows, cols, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + static UMat ones(Size size, int type) { return ones(size, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + static UMat ones(int ndims, const int* sz, int type) { return ones(ndims, sz, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + static UMat eye(int rows, int cols, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat eye(Size size, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + static UMat eye(int rows, int cols, int type) { return eye(rows, cols, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + static UMat eye(Size size, int type) { return eye(size, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload //! allocates new matrix data unless the matrix already has specified size and type. // previous data is unreferenced if needed. diff --git a/modules/core/src/matrix_operations.cpp b/modules/core/src/matrix_operations.cpp index 83c8aaeb5705..227c7aaef774 100644 --- a/modules/core/src/matrix_operations.cpp +++ b/modules/core/src/matrix_operations.cpp @@ -229,14 +229,14 @@ void cv::setIdentity( InputOutputArray _m, const Scalar& s ) namespace cv { -UMat UMat::eye(int rows, int cols, int type) +UMat UMat::eye(int rows, int cols, int type, UMatUsageFlags usageFlags) { - return UMat::eye(Size(cols, rows), type); + return UMat::eye(Size(cols, rows), type, usageFlags); } -UMat UMat::eye(Size size, int type) +UMat UMat::eye(Size size, int type, UMatUsageFlags usageFlags) { - UMat m(size, type); + UMat m(size, type, usageFlags); setIdentity(m); return m; } diff --git a/modules/core/src/umatrix.cpp b/modules/core/src/umatrix.cpp index c80d240ecc02..bf5dfb68a318 100644 --- a/modules/core/src/umatrix.cpp +++ b/modules/core/src/umatrix.cpp @@ -951,11 +951,11 @@ UMat UMat::reshape(int new_cn, int new_rows) const return hdr; } -UMat UMat::diag(const UMat& d) +UMat UMat::diag(const UMat& d, UMatUsageFlags usageFlags) { CV_Assert( d.cols == 1 || d.rows == 1 ); int len = d.rows + d.cols - 1; - UMat m(len, len, d.type(), Scalar(0)); + UMat m(len, len, d.type(), Scalar(0), usageFlags); UMat md = m.diag(); if( d.cols == 1 ) d.copyTo(md); @@ -1323,34 +1323,34 @@ UMat UMat::t() const return m; } -UMat UMat::zeros(int rows, int cols, int type) +UMat UMat::zeros(int rows, int cols, int type, UMatUsageFlags usageFlags) { - return UMat(rows, cols, type, Scalar::all(0)); + return UMat(rows, cols, type, Scalar::all(0), usageFlags); } -UMat UMat::zeros(Size size, int type) +UMat UMat::zeros(Size size, int type, UMatUsageFlags usageFlags) { - return UMat(size, type, Scalar::all(0)); + return UMat(size, type, Scalar::all(0), usageFlags); } -UMat UMat::zeros(int ndims, const int* sz, int type) +UMat UMat::zeros(int ndims, const int* sz, int type, UMatUsageFlags usageFlags) { - return UMat(ndims, sz, type, Scalar::all(0)); + return UMat(ndims, sz, type, Scalar::all(0), usageFlags); } -UMat UMat::ones(int rows, int cols, int type) +UMat UMat::ones(int rows, int cols, int type, UMatUsageFlags usageFlags) { - return UMat::ones(Size(cols, rows), type); + return UMat(rows, cols, type, Scalar(1), usageFlags); } -UMat UMat::ones(Size size, int type) +UMat UMat::ones(Size size, int type, UMatUsageFlags usageFlags) { - return UMat(size, type, Scalar(1)); + return UMat(size, type, Scalar(1), usageFlags); } -UMat UMat::ones(int ndims, const int* sz, int type) +UMat UMat::ones(int ndims, const int* sz, int type, UMatUsageFlags usageFlags) { - return UMat(ndims, sz, type, Scalar(1)); + return UMat(ndims, sz, type, Scalar(1), usageFlags); } } diff --git a/modules/core/test/ocl/test_matrix_expr.cpp b/modules/core/test/ocl/test_matrix_expr.cpp index 7a5ff72cb24e..f11c0a6ebb6d 100644 --- a/modules/core/test/ocl/test_matrix_expr.cpp +++ b/modules/core/test/ocl/test_matrix_expr.cpp @@ -76,6 +76,24 @@ OCL_TEST_P(UMatExpr, Ones) } } +//////////////////////////////// with usageFlags ///////////////////////////////////////////////// + +OCL_TEST_P(UMatExpr, WithUsageFlags) +{ + for (int j = 0; j < test_loop_times; j++) + { + generateTestData(); + + UMat u0 = UMat::zeros(size, type, cv::USAGE_ALLOCATE_HOST_MEMORY); + UMat u1 = UMat::ones(size, type, cv::USAGE_ALLOCATE_HOST_MEMORY); + UMat u8 = UMat::eye(size, type, cv::USAGE_ALLOCATE_HOST_MEMORY); + + EXPECT_EQ(cv::USAGE_ALLOCATE_HOST_MEMORY, u0.usageFlags); + EXPECT_EQ(cv::USAGE_ALLOCATE_HOST_MEMORY, u1.usageFlags); + EXPECT_EQ(cv::USAGE_ALLOCATE_HOST_MEMORY, u8.usageFlags); + } +} + //////////////////////////////// Instantiation ///////////////////////////////////////////////// OCL_INSTANTIATE_TEST_CASE_P(MatrixOperation, UMatExpr, Combine(OCL_ALL_DEPTHS_16F, OCL_ALL_CHANNELS)); From b68057d92701e24765d9ff199011f5171d320143 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Wed, 23 Jun 2021 21:27:54 +0200 Subject: [PATCH 014/376] Do not use = 0 for a cv::Mat. There are several operator= overloads and some compilers can be confused. --- modules/calib3d/src/chessboard.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/calib3d/src/chessboard.cpp b/modules/calib3d/src/chessboard.cpp index dbc47722cba9..18e2605f53b5 100644 --- a/modules/calib3d/src/chessboard.cpp +++ b/modules/calib3d/src/chessboard.cpp @@ -3924,7 +3924,7 @@ bool findChessboardCornersSB(cv::InputArray image_, cv::Size pattern_size, { meta_.create(int(board.rowCount()),int(board.colCount()),CV_8UC1); cv::Mat meta = meta_.getMat(); - meta = 0; + meta.setTo(cv::Scalar::all(0)); for(int row =0;row < meta.rows-1;++row) { for(int col=0;col< meta.cols-1;++col) From dc5199feeae9cef33bd55cc8c161917bc3ef367b Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Mon, 21 Jun 2021 11:11:14 +0300 Subject: [PATCH 015/376] skipping missing layers and layer failures --- apps/model-diagnostics/model_diagnostics.cpp | 6 +-- modules/dnn/src/tensorflow/tf_importer.cpp | 43 +++++++++++++++++--- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/apps/model-diagnostics/model_diagnostics.cpp b/apps/model-diagnostics/model_diagnostics.cpp index 2ffeaa1ea5b9..d3934577aec6 100644 --- a/apps/model-diagnostics/model_diagnostics.cpp +++ b/apps/model-diagnostics/model_diagnostics.cpp @@ -1,6 +1,6 @@ /************************************************* USAGE: -./model_diagnostics -m +./model_diagnostics -m **************************************************/ #include #include @@ -32,7 +32,7 @@ static std::string checkFileExists(const std::string& fileName) } std::string diagnosticKeys = - "{ model m | | Path to the model .onnx file. }" + "{ model m | | Path to the model file. }" "{ config c | | Path to the model configuration file. }" "{ framework f | | [Optional] Name of the model framework. }"; @@ -41,7 +41,7 @@ std::string diagnosticKeys = int main( int argc, const char** argv ) { CommandLineParser argParser(argc, argv, diagnosticKeys); - argParser.about("Use this tool to run the diagnostics of provided ONNX model" + argParser.about("Use this tool to run the diagnostics of provided ONNX/TF model" "to obtain the information about its support (supported layers)."); if (argc == 1) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index ed6a40792053..15f88007b4d1 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -32,6 +32,8 @@ namespace cv { namespace dnn { CV__DNN_INLINE_NS_BEGIN +extern bool DNN_DIAGNOSTICS_RUN; + #if HAVE_PROTOBUF using ::google::protobuf::RepeatedField; @@ -471,6 +473,7 @@ class TFImporter TFImporter(Net& net, const char *dataModel, size_t lenModel, const char *dataConfig = NULL, size_t lenConfig = 0); protected: + std::unique_ptr utilNet; Net& dstNet; void populateNet(); @@ -2337,7 +2340,8 @@ void TFImporter::parseCustomLayer(tensorflow::GraphDef& net, const tensorflow::N } TFImporter::TFImporter(Net& net, const char *model, const char *config) - : dstNet(net), dispatch(buildDispatchMap()) + : utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), + dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap()) { if (model && model[0]) { @@ -2358,7 +2362,8 @@ TFImporter::TFImporter( const char *dataModel, size_t lenModel, const char *dataConfig, size_t lenConfig ) - : dstNet(net), dispatch(buildDispatchMap()) + : utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), + dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap()) { if (dataModel != NULL && lenModel > 0) { @@ -2615,6 +2620,11 @@ DataLayout TFImporter::predictOutputDataLayout(const tensorflow::NodeDef& layer) return it->second; } +Ptr dummy_constructor(LayerParams & params) +{ + return new Layer(params); +} + void TFImporter::populateNet() { CV_Assert(netBin.ByteSize() || netTxt.ByteSize()); @@ -2757,9 +2767,9 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer) const std::string& name = layer.name(); const std::string& type = layer.op(); + LayerParams layerParams; try { - LayerParams layerParams; if (layers_to_ignore.find(name) != layers_to_ignore.end()) { @@ -2777,13 +2787,36 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer) } else { + if (DNN_DIAGNOSTICS_RUN && !LayerFactory::createLayerInstance(type, layerParams)) + { + CV_LOG_ERROR(NULL, "DNN/TF: Node='" << name << "' of type='"<< type + << "' is not supported. This error won't be displayed again."); + LayerFactory::registerLayer(type, dummy_constructor); + } + parseCustomLayer(net, layer, layerParams); } } catch (const std::exception& e) { - CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "'. Exception: " << e.what()); - throw; + if (!DNN_DIAGNOSTICS_RUN) + { + CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type + << "'. Exception: " << e.what()); + throw; + } + else + { + CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type + << "'. Exception: " << e.what()); + + // internal layer failure (didnt call addLayer) + if (dstNet.getLayerId(name) == -1) + { + int id = dstNet.addLayer(name, type, layerParams); + layer_id[name] = id; + } + } } } From c95a56450dd8c612b3f8e83d680a0a6c6533acf0 Mon Sep 17 00:00:00 2001 From: Alexey Smirnov Date: Sat, 26 Jun 2021 00:09:33 +0300 Subject: [PATCH 016/376] Merge pull request #20156 from smirnov-alexey:as/gapi_remote_infer G-API: Support remote inference * Extend MediaFrame to be able to extract additional info besides access * Add API for remote inference * Add default implementation for blobParams() * Add default implementation for blobParams() * Address review comments * Fix any_cast usage * Add comment on the default blobParams() * Address review comments * Add missing rctx * Minor fix * Fix indentation and comment * Address review comments * Add documentation --- .../gapi/include/opencv2/gapi/infer/ie.hpp | 44 +++++++++++++++++-- modules/gapi/src/backends/ie/giebackend.cpp | 40 ++++++++++++++++- .../src/backends/ie/giebackend/giewrapper.hpp | 26 +++++++++-- 3 files changed, 100 insertions(+), 10 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp index 70712ba74039..2be739e51840 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ie.hpp @@ -74,7 +74,11 @@ struct ParamDesc { std::map> reshape_table; std::unordered_set layer_names_to_reshape; + // NB: Number of asyncrhonious infer requests size_t nireq; + + // NB: An optional config to setup RemoteContext for IE + cv::util::any context_config; }; } // namespace detail @@ -115,7 +119,8 @@ template class Params { , {} , {} , {} - , 1u} { + , 1u + , {}} { }; /** @overload @@ -135,7 +140,8 @@ template class Params { , {} , {} , {} - , 1u} { + , 1u + , {}} { }; /** @brief Specifies sequence of network input layers names for inference. @@ -217,6 +223,30 @@ template class Params { return *this; } + /** @brief Specifies configuration for RemoteContext in InferenceEngine. + + When RemoteContext is configured the backend imports the networks using the context. + It also expects cv::MediaFrames to be actually remote, to operate with blobs via the context. + + @param ctx_cfg cv::util::any value which holds InferenceEngine::ParamMap. + @return reference to this parameter structure. + */ + Params& cfgContextParams(const cv::util::any& ctx_cfg) { + desc.context_config = ctx_cfg; + return *this; + } + + /** @overload + Function with an rvalue parameter. + + @param ctx_cfg cv::util::any value which holds InferenceEngine::ParamMap. + @return reference to this parameter structure. + */ + Params& cfgContextParams(cv::util::any&& ctx_cfg) { + desc.context_config = std::move(ctx_cfg); + return *this; + } + /** @brief Specifies number of asynchronous inference requests. @param nireq Number of inference asynchronous requests. @@ -318,7 +348,10 @@ class Params { const std::string &model, const std::string &weights, const std::string &device) - : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u}, m_tag(tag) { + : desc{ model, weights, device, {}, {}, {}, 0u, 0u, + detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u, + {}}, + m_tag(tag) { }; /** @overload @@ -333,7 +366,10 @@ class Params { Params(const std::string &tag, const std::string &model, const std::string &device) - : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u}, m_tag(tag) { + : desc{ model, {}, device, {}, {}, {}, 0u, 0u, + detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u, + {}}, + m_tag(tag) { }; /** @see ie::Params::pluginConfig. */ diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 46b6bdbb97ab..77a6515f8530 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -222,8 +222,17 @@ struct IEUnit { IE::ExecutableNetwork this_network; cv::gimpl::ie::wrap::Plugin this_plugin; + InferenceEngine::RemoteContext::Ptr rctx = nullptr; + explicit IEUnit(const cv::gapi::ie::detail::ParamDesc &pp) : params(pp) { + InferenceEngine::ParamMap* ctx_params = + cv::util::any_cast(¶ms.context_config); + if (ctx_params != nullptr) { + auto ie_core = cv::gimpl::ie::wrap::getCore(); + rctx = ie_core.CreateContext(params.device_id, *ctx_params); + } + if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { net = cv::gimpl::ie::wrap::readNetwork(params); inputs = net.getInputsInfo(); @@ -231,7 +240,7 @@ struct IEUnit { } else if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import) { this_plugin = cv::gimpl::ie::wrap::getPlugin(params); this_plugin.SetConfig(params.config); - this_network = cv::gimpl::ie::wrap::importNetwork(this_plugin, params); + this_network = cv::gimpl::ie::wrap::importNetwork(this_plugin, params, rctx); // FIXME: ICNNetwork returns InputsDataMap/OutputsDataMap, // but ExecutableNetwork returns ConstInputsDataMap/ConstOutputsDataMap inputs = cv::gimpl::ie::wrap::toInputsDataMap(this_network.GetInputsInfo()); @@ -279,7 +288,8 @@ struct IEUnit { // for loadNetwork they can be obtained by using readNetwork non_const_this->this_plugin = cv::gimpl::ie::wrap::getPlugin(params); non_const_this->this_plugin.SetConfig(params.config); - non_const_this->this_network = cv::gimpl::ie::wrap::loadNetwork(non_const_this->this_plugin, net, params); + non_const_this->this_network = cv::gimpl::ie::wrap::loadNetwork(non_const_this->this_plugin, + net, params, rctx); } return {params, this_plugin, this_network}; @@ -481,7 +491,32 @@ using GConstGIEModel = ade::ConstTypedGraph , IECallable >; +inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i) { + GAPI_Assert(ctx.inShape(i) == cv::GShape::GFRAME && + "Remote blob is supported for MediaFrame only"); + + cv::util::any any_blob_params = ctx.inFrame(i).blobParams(); + auto ie_core = cv::gimpl::ie::wrap::getCore(); + + using ParamType = std::pair; + + ParamType* blob_params = cv::util::any_cast(&any_blob_params); + if (blob_params == nullptr) { + GAPI_Assert(false && "Incorrect type of blobParams: " + "expected std::pair"); + } + + return ctx.uu.rctx->CreateBlob(blob_params->first, + blob_params->second); +} + inline IE::Blob::Ptr extractBlob(IECallContext& ctx, std::size_t i) { + if (ctx.uu.rctx != nullptr) { + return extractRemoteBlob(ctx, i); + } + switch (ctx.inShape(i)) { case cv::GShape::GFRAME: { const auto& frame = ctx.inFrame(i); @@ -1060,6 +1095,7 @@ struct InferList: public cv::detail::KernelTag { } IE::Blob::Ptr this_blob = extractBlob(*ctx, 1); + std::vector> cached_dims(ctx->uu.params.num_out); for (auto i : ade::util::iota(ctx->uu.params.num_out)) { const IE::DataPtr& ie_out = ctx->uu.outputs.at(ctx->uu.params.output_names[i]); diff --git a/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp b/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp index 3927c802b713..7e67cb8989d6 100644 --- a/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp +++ b/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp @@ -13,6 +13,7 @@ #include #include +#include #include "opencv2/gapi/infer/ie.hpp" @@ -50,12 +51,29 @@ GAPI_EXPORTS IE::Core getCore(); GAPI_EXPORTS IE::Core getPlugin(const GIEParam& params); GAPI_EXPORTS inline IE::ExecutableNetwork loadNetwork( IE::Core& core, const IE::CNNNetwork& net, - const GIEParam& params) { - return core.LoadNetwork(net, params.device_id); + const GIEParam& params, + IE::RemoteContext::Ptr rctx = nullptr) { + if (rctx != nullptr) { + return core.LoadNetwork(net, rctx); + } else { + return core.LoadNetwork(net, params.device_id); + } } GAPI_EXPORTS inline IE::ExecutableNetwork importNetwork( IE::Core& core, - const GIEParam& param) { - return core.ImportNetwork(param.model_path, param.device_id, {}); + const GIEParam& params, + IE::RemoteContext::Ptr rctx = nullptr) { + if (rctx != nullptr) { + std::filebuf blobFile; + if (!blobFile.open(params.model_path, std::ios::in | std::ios::binary)) + { + blobFile.close(); + throw std::runtime_error("Could not open file"); + } + std::istream graphBlob(&blobFile); + return core.ImportNetwork(graphBlob, rctx); + } else { + return core.ImportNetwork(params.model_path, params.device_id, {}); + } } #endif // INF_ENGINE_RELEASE < 2019020000 }}}} From 42d644ef9134bcb620dd00f7ab7a6d7d039bfdf2 Mon Sep 17 00:00:00 2001 From: xzvno Date: Sun, 27 Jun 2021 05:01:31 +0800 Subject: [PATCH 017/376] Merge pull request #20293 from endjkv:fix-mem-leak-when-throw * fix memory leak when exception is thrown --- modules/core/src/system.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index af4a62181697..441457d50fd2 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -1835,7 +1835,15 @@ void* TLSDataContainer::getData() const { // Create new data instance and save it to TLS storage pData = createDataInstance(); - getTlsStorage().setData(key_, pData); + try + { + getTlsStorage().setData(key_, pData); + } + catch (...) + { + deleteDataInstance(pData); + throw; + } } return pData; } From 61a5378aeb3a6be13cbca1a1e6a6874358996fb4 Mon Sep 17 00:00:00 2001 From: Giles Payne Date: Sun, 27 Jun 2021 21:08:25 +0900 Subject: [PATCH 018/376] Improvements/fixes for unsigned type handling in Swift/Kotlin --- modules/core/misc/java/src/java/core+MatAt.kt | 93 ++++++++-- modules/core/misc/objc/common/Mat.mm | 2 +- modules/core/misc/objc/common/MatExt.swift | 146 +++++++++++++-- modules/core/misc/objc/test/MatTest.swift | 172 ++++++++++++++++-- 4 files changed, 366 insertions(+), 47 deletions(-) diff --git a/modules/core/misc/java/src/java/core+MatAt.kt b/modules/core/misc/java/src/java/core+MatAt.kt index f48a3deaedf9..c81e21057f27 100644 --- a/modules/core/misc/java/src/java/core+MatAt.kt +++ b/modules/core/misc/java/src/java/core+MatAt.kt @@ -3,6 +3,16 @@ package org.opencv.core import org.opencv.core.Mat.* import java.lang.RuntimeException +fun Mat.get(row: Int, col: Int, data: UByteArray) = this.get(row, col, data.asByteArray()) +fun Mat.get(indices: IntArray, data: UByteArray) = this.get(indices, data.asByteArray()) +fun Mat.put(row: Int, col: Int, data: UByteArray) = this.put(row, col, data.asByteArray()) +fun Mat.put(indices: IntArray, data: UByteArray) = this.put(indices, data.asByteArray()) + +fun Mat.get(row: Int, col: Int, data: UShortArray) = this.get(row, col, data.asShortArray()) +fun Mat.get(indices: IntArray, data: UShortArray) = this.get(indices, data.asShortArray()) +fun Mat.put(row: Int, col: Int, data: UShortArray) = this.put(row, col, data.asShortArray()) +fun Mat.put(indices: IntArray, data: UShortArray) = this.put(indices, data.asShortArray()) + /*** * Example use: * @@ -19,6 +29,7 @@ inline fun Mat.at(row: Int, col: Int) : Atable = col ) UByte::class -> AtableUByte(this, row, col) as Atable + UShort::class -> AtableUShort(this, row, col) as Atable else -> throw RuntimeException("Unsupported class type") } @@ -30,6 +41,7 @@ inline fun Mat.at(idx: IntArray) : Atable = idx ) UByte::class -> AtableUByte(this, idx) as Atable + UShort::class -> AtableUShort(this, idx) as Atable else -> throw RuntimeException("Unsupported class type") } @@ -38,46 +50,95 @@ class AtableUByte(val mat: Mat, val indices: IntArray): Atable { constructor(mat: Mat, row: Int, col: Int) : this(mat, intArrayOf(row, col)) override fun getV(): UByte { - val data = ByteArray(1) - mat[indices, data] - return data[0].toUByte() + val data = UByteArray(1) + mat.get(indices, data) + return data[0] } override fun setV(v: UByte) { - val data = byteArrayOf(v.toByte()) + val data = ubyteArrayOf(v) mat.put(indices, data) } override fun getV2c(): Tuple2 { - val data = ByteArray(2) - mat[indices, data] - return Tuple2(data[0].toUByte(), data[1].toUByte()) + val data = UByteArray(2) + mat.get(indices, data) + return Tuple2(data[0], data[1]) } override fun setV2c(v: Tuple2) { - val data = byteArrayOf(v._0.toByte(), v._1.toByte()) + val data = ubyteArrayOf(v._0, v._1) mat.put(indices, data) } override fun getV3c(): Tuple3 { - val data = ByteArray(3) - mat[indices, data] - return Tuple3(data[0].toUByte(), data[1].toUByte(), data[2].toUByte()) + val data = UByteArray(3) + mat.get(indices, data) + return Tuple3(data[0], data[1], data[2]) } override fun setV3c(v: Tuple3) { - val data = byteArrayOf(v._0.toByte(), v._1.toByte(), v._2.toByte()) + val data = ubyteArrayOf(v._0, v._1, v._2) mat.put(indices, data) } override fun getV4c(): Tuple4 { - val data = ByteArray(4) - mat[indices, data] - return Tuple4(data[0].toUByte(), data[1].toUByte(), data[2].toUByte(), data[3].toUByte()) + val data = UByteArray(4) + mat.get(indices, data) + return Tuple4(data[0], data[1], data[2], data[3]) } override fun setV4c(v: Tuple4) { - val data = byteArrayOf(v._0.toByte(), v._1.toByte(), v._2.toByte(), v._3.toByte()) + val data = ubyteArrayOf(v._0, v._1, v._2, v._3) + mat.put(indices, data) + } +} + +class AtableUShort(val mat: Mat, val indices: IntArray): Atable { + + constructor(mat: Mat, row: Int, col: Int) : this(mat, intArrayOf(row, col)) + + override fun getV(): UShort { + val data = UShortArray(1) + mat.get(indices, data) + return data[0] + } + + override fun setV(v: UShort) { + val data = ushortArrayOf(v) + mat.put(indices, data) + } + + override fun getV2c(): Tuple2 { + val data = UShortArray(2) + mat.get(indices, data) + return Tuple2(data[0], data[1]) + } + + override fun setV2c(v: Tuple2) { + val data = ushortArrayOf(v._0, v._1) + mat.put(indices, data) + } + + override fun getV3c(): Tuple3 { + val data = UShortArray(3) + mat.get(indices, data) + return Tuple3(data[0], data[1], data[2]) + } + + override fun setV3c(v: Tuple3) { + val data = ushortArrayOf(v._0, v._1, v._2) + mat.put(indices, data) + } + + override fun getV4c(): Tuple4 { + val data = UShortArray(4) + mat.get(indices, data) + return Tuple4(data[0], data[1], data[2], data[3]) + } + + override fun setV4c(v: Tuple4) { + val data = ushortArrayOf(v._0, v._1, v._2, v._3) mat.put(indices, data) } } diff --git a/modules/core/misc/objc/common/Mat.mm b/modules/core/misc/objc/common/Mat.mm index 5d41a3622e71..045bd8393ea3 100644 --- a/modules/core/misc/objc/common/Mat.mm +++ b/modules/core/misc/objc/common/Mat.mm @@ -548,7 +548,7 @@ - (void)put:(uchar*)dest data:(NSArray*)data offset:(int)offset count if (depth == CV_8U) { putData(dest, count, ^uchar (int index) { return cv::saturate_cast(data[offset + index].doubleValue);} ); } else if (depth == CV_8S) { - putData(dest, count, ^char (int index) { return cv::saturate_cast(data[offset + index].doubleValue);} ); + putData(dest, count, ^schar (int index) { return cv::saturate_cast(data[offset + index].doubleValue);} ); } else if (depth == CV_16U) { putData(dest, count, ^ushort (int index) { return cv::saturate_cast(data[offset + index].doubleValue);} ); } else if (depth == CV_16S) { diff --git a/modules/core/misc/objc/common/MatExt.swift b/modules/core/misc/objc/common/MatExt.swift index 5ce3a5e6fb56..a6ba548599d8 100644 --- a/modules/core/misc/objc/common/MatExt.swift +++ b/modules/core/misc/objc/common/MatExt.swift @@ -62,6 +62,21 @@ public extension Mat { } } + @discardableResult func get(indices:[Int32], data:inout [UInt8]) throws -> Int32 { + let channels = CvType.channels(Int32(type())) + if Int32(data.count) % channels != 0 { + try throwIncompatibleBufferSize(count: data.count, channels: channels) + } else if depth() != CvType.CV_8U { + try throwIncompatibleDataType(typeName: CvType.type(toString: type())) + } + let count = Int32(data.count) + return data.withUnsafeMutableBufferPointer { body in + body.withMemoryRebound(to: Int8.self) { reboundBody in + return __get(indices as [NSNumber], count: count, byteBuffer: reboundBody.baseAddress!) + } + } + } + @discardableResult func get(indices:[Int32], data:inout [Double]) throws -> Int32 { let channels = CvType.channels(Int32(type())) if Int32(data.count) % channels != 0 { @@ -114,10 +129,29 @@ public extension Mat { } } + @discardableResult func get(indices:[Int32], data:inout [UInt16]) throws -> Int32 { + let channels = CvType.channels(Int32(type())) + if Int32(data.count) % channels != 0 { + try throwIncompatibleBufferSize(count: data.count, channels: channels) + } else if depth() != CvType.CV_16U { + try throwIncompatibleDataType(typeName: CvType.type(toString: type())) + } + let count = Int32(data.count) + return data.withUnsafeMutableBufferPointer { body in + body.withMemoryRebound(to: Int16.self) { reboundBody in + return __get(indices as [NSNumber], count: count, shortBuffer: reboundBody.baseAddress!) + } + } + } + @discardableResult func get(row: Int32, col: Int32, data:inout [Int8]) throws -> Int32 { return try get(indices: [row, col], data: &data) } + @discardableResult func get(row: Int32, col: Int32, data:inout [UInt8]) throws -> Int32 { + return try get(indices: [row, col], data: &data) + } + @discardableResult func get(row: Int32, col: Int32, data:inout [Double]) throws -> Int32 { return try get(indices: [row, col], data: &data) } @@ -134,6 +168,10 @@ public extension Mat { return try get(indices: [row, col], data: &data) } + @discardableResult func get(row: Int32, col: Int32, data:inout [UInt16]) throws -> Int32 { + return try get(indices: [row, col], data: &data) + } + @discardableResult func put(indices:[Int32], data:[Int8]) throws -> Int32 { let channels = CvType.channels(Int32(type())) if Int32(data.count) % channels != 0 { @@ -147,6 +185,21 @@ public extension Mat { } } + @discardableResult func put(indices:[Int32], data:[UInt8]) throws -> Int32 { + let channels = CvType.channels(Int32(type())) + if Int32(data.count) % channels != 0 { + try throwIncompatibleBufferSize(count: data.count, channels: channels) + } else if depth() != CvType.CV_8U { + try throwIncompatibleDataType(typeName: CvType.type(toString: type())) + } + let count = Int32(data.count) + return data.withUnsafeBufferPointer { body in + body.withMemoryRebound(to: Int8.self) { reboundBody in + return __put(indices as [NSNumber], count: count, byteBuffer: reboundBody.baseAddress!) + } + } + } + @discardableResult func put(indices:[Int32], data:[Int8], offset: Int, length: Int32) throws -> Int32 { let channels = CvType.channels(Int32(type())) if Int32(data.count) % channels != 0 { @@ -214,10 +267,29 @@ public extension Mat { } } + @discardableResult func put(indices:[Int32], data:[UInt16]) throws -> Int32 { + let channels = CvType.channels(Int32(type())) + if Int32(data.count) % channels != 0 { + try throwIncompatibleBufferSize(count: data.count, channels: channels) + } else if depth() != CvType.CV_16U { + try throwIncompatibleDataType(typeName: CvType.type(toString: type())) + } + let count = Int32(data.count) + return data.withUnsafeBufferPointer { body in + body.withMemoryRebound(to: Int16.self) { reboundBody in + return __put(indices as [NSNumber], count: count, shortBuffer: reboundBody.baseAddress!) + } + } + } + @discardableResult func put(row: Int32, col: Int32, data:[Int8]) throws -> Int32 { return try put(indices: [row, col], data: data) } + @discardableResult func put(row: Int32, col: Int32, data:[UInt8]) throws -> Int32 { + return try put(indices: [row, col], data: data) + } + @discardableResult func put(row: Int32, col: Int32, data: [Int8], offset: Int, length: Int32) throws -> Int32 { return try put(indices: [row, col], data: data, offset: offset, length: length) } @@ -238,6 +310,10 @@ public extension Mat { return try put(indices: [row, col], data: data) } + @discardableResult func put(row: Int32, col: Int32, data: [UInt16]) throws -> Int32 { + return try put(indices: [row, col], data: data) + } + @discardableResult func get(row: Int32, col: Int32) -> [Double] { return get(indices: [row, col]) } @@ -303,46 +379,46 @@ public class MatAt { extension UInt8: Atable { public static func getAt(m: Mat, indices:[Int32]) -> UInt8 { - var tmp = [Int8](repeating: 0, count: 1) + var tmp = [UInt8](repeating: 0, count: 1) try! m.get(indices: indices, data: &tmp) - return UInt8(bitPattern: tmp[0]) + return tmp[0] } public static func putAt(m: Mat, indices: [Int32], v: UInt8) { - let tmp = [Int8(bitPattern: v)] + let tmp = [v] try! m.put(indices: indices, data: tmp) } public static func getAt2c(m: Mat, indices:[Int32]) -> (UInt8, UInt8) { - var tmp = [Int8](repeating: 0, count: 2) + var tmp = [UInt8](repeating: 0, count: 2) try! m.get(indices: indices, data: &tmp) - return (UInt8(bitPattern: tmp[0]), UInt8(bitPattern: tmp[1])) + return (tmp[0], tmp[1]) } public static func putAt2c(m: Mat, indices: [Int32], v: (UInt8, UInt8)) { - let tmp = [Int8(bitPattern: v.0), Int8(bitPattern: v.1)] + let tmp = [v.0, v.1] try! m.put(indices: indices, data: tmp) } public static func getAt3c(m: Mat, indices:[Int32]) -> (UInt8, UInt8, UInt8) { - var tmp = [Int8](repeating: 0, count: 3) + var tmp = [UInt8](repeating: 0, count: 3) try! m.get(indices: indices, data: &tmp) - return (UInt8(bitPattern: tmp[0]), UInt8(bitPattern: tmp[1]), UInt8(bitPattern: tmp[2])) + return (tmp[0], tmp[1], tmp[2]) } public static func putAt3c(m: Mat, indices: [Int32], v: (UInt8, UInt8, UInt8)) { - let tmp = [Int8(bitPattern: v.0), Int8(bitPattern: v.1), Int8(bitPattern: v.2)] + let tmp = [v.0, v.1, v.2] try! m.put(indices: indices, data: tmp) } public static func getAt4c(m: Mat, indices:[Int32]) -> (UInt8, UInt8, UInt8, UInt8) { - var tmp = [Int8](repeating: 0, count: 4) + var tmp = [UInt8](repeating: 0, count: 4) try! m.get(indices: indices, data: &tmp) - return (UInt8(bitPattern: tmp[0]), UInt8(bitPattern: tmp[1]), UInt8(bitPattern: tmp[2]), UInt8(bitPattern: tmp[3])) + return (tmp[0], tmp[1], tmp[2], tmp[3]) } public static func putAt4c(m: Mat, indices: [Int32], v: (UInt8, UInt8, UInt8, UInt8)) { - let tmp = [Int8(bitPattern: v.0), Int8(bitPattern: v.1), Int8(bitPattern: v.2), Int8(bitPattern: v.3)] + let tmp = [v.0, v.1, v.2, v.3] try! m.put(indices: indices, data: tmp) } } @@ -531,6 +607,52 @@ extension Int32: Atable { } } +extension UInt16: Atable { + public static func getAt(m: Mat, indices:[Int32]) -> UInt16 { + var tmp = [UInt16](repeating: 0, count: 1) + try! m.get(indices: indices, data: &tmp) + return tmp[0] + } + + public static func putAt(m: Mat, indices: [Int32], v: UInt16) { + let tmp = [v] + try! m.put(indices: indices, data: tmp) + } + + public static func getAt2c(m: Mat, indices:[Int32]) -> (UInt16, UInt16) { + var tmp = [UInt16](repeating: 0, count: 2) + try! m.get(indices: indices, data: &tmp) + return (tmp[0], tmp[1]) + } + + public static func putAt2c(m: Mat, indices: [Int32], v: (UInt16, UInt16)) { + let tmp = [v.0, v.1] + try! m.put(indices: indices, data: tmp) + } + + public static func getAt3c(m: Mat, indices:[Int32]) -> (UInt16, UInt16, UInt16) { + var tmp = [UInt16](repeating: 0, count: 3) + try! m.get(indices: indices, data: &tmp) + return (tmp[0], tmp[1], tmp[2]) + } + + public static func putAt3c(m: Mat, indices: [Int32], v: (UInt16, UInt16, UInt16)) { + let tmp = [v.0, v.1, v.2] + try! m.put(indices: indices, data: tmp) + } + + public static func getAt4c(m: Mat, indices:[Int32]) -> (UInt16, UInt16, UInt16, UInt16) { + var tmp = [UInt16](repeating: 0, count: 4) + try! m.get(indices: indices, data: &tmp) + return (tmp[0], tmp[1], tmp[2], tmp[3]) + } + + public static func putAt4c(m: Mat, indices: [Int32], v: (UInt16, UInt16, UInt16, UInt16)) { + let tmp = [v.0, v.1, v.2, v.3] + try! m.put(indices: indices, data: tmp) + } +} + extension Int16: Atable { public static func getAt(m: Mat, indices:[Int32]) -> Int16 { var tmp = [Int16](repeating: 0, count: 1) diff --git a/modules/core/misc/objc/test/MatTest.swift b/modules/core/misc/objc/test/MatTest.swift index 14c440b5eb88..8a513505cc14 100644 --- a/modules/core/misc/objc/test/MatTest.swift +++ b/modules/core/misc/objc/test/MatTest.swift @@ -308,15 +308,15 @@ class MatTests: OpenCVTestCase { XCTAssert([340] == sm.get(row: 1, col: 1)) } - func testGetIntIntByteArray() throws { - let m = try getTestMat(size: 5, type: CvType.CV_8UC3) + func testGetIntIntInt8Array() throws { + let m = try getTestMat(size: 5, type: CvType.CV_8SC3) var goodData = [Int8](repeating: 0, count: 9) // whole Mat var bytesNum = try m.get(row: 1, col: 1, data: &goodData) XCTAssertEqual(9, bytesNum) - XCTAssert([110, 111, 112, 120, 121, 122, -126, -125, -124] == goodData) + XCTAssert([110, 111, 112, 120, 121, 122, 127, 127, 127] == goodData) var badData = [Int8](repeating: 0, count: 7) XCTAssertThrowsError(bytesNum = try m.get(row: 0, col: 0, data: &badData)) @@ -326,11 +326,36 @@ class MatTests: OpenCVTestCase { var buff00 = [Int8](repeating: 0, count: 3) bytesNum = try sm.get(row: 0, col: 0, data: &buff00) XCTAssertEqual(3, bytesNum) - XCTAssert(buff00 == [-26, -25, -24]) + XCTAssert(buff00 == [127, 127, 127]) var buff11 = [Int8](repeating: 0, count: 3) bytesNum = try sm.get(row: 1, col: 1, data: &buff11) XCTAssertEqual(3, bytesNum) - XCTAssert(buff11 == [-1, -1, -1]) + XCTAssert(buff11 == [127, 127, 127]) + } + + func testGetIntIntUInt8Array() throws { + let m = try getTestMat(size: 5, type: CvType.CV_8UC3) + var goodData = [UInt8](repeating: 0, count: 9) + + // whole Mat + var bytesNum = try m.get(row: 1, col: 1, data: &goodData) + + XCTAssertEqual(9, bytesNum) + XCTAssert([110, 111, 112, 120, 121, 122, 130, 131, 132] == goodData) + + var badData = [UInt8](repeating: 0, count: 7) + XCTAssertThrowsError(bytesNum = try m.get(row: 0, col: 0, data: &badData)) + + // sub-Mat + let sm = m.submat(rowStart: 2, rowEnd: 4, colStart: 3, colEnd: 5) + var buff00 = [UInt8](repeating: 0, count: 3) + bytesNum = try sm.get(row: 0, col: 0, data: &buff00) + XCTAssertEqual(3, bytesNum) + XCTAssert(buff00 == [230, 231, 232]) + var buff11 = [UInt8](repeating: 0, count: 3) + bytesNum = try sm.get(row: 1, col: 1, data: &buff11) + XCTAssertEqual(3, bytesNum) + XCTAssert(buff11 == [255, 255, 255]) } func testGetIntIntDoubleArray() throws { @@ -399,7 +424,7 @@ class MatTests: OpenCVTestCase { XCTAssert(buff11 == [340, 341, 0, 0]) } - func testGetIntIntShortArray() throws { + func testGetIntIntInt16Array() throws { let m = try getTestMat(size: 5, type: CvType.CV_16SC2) var buff = [Int16](repeating: 0, count: 6) @@ -421,6 +446,28 @@ class MatTests: OpenCVTestCase { XCTAssert(buff11 == [340, 341, 0, 0]) } + func testGetIntIntUInt16Array() throws { + let m = try getTestMat(size: 5, type: CvType.CV_16UC2) + var buff = [UInt16](repeating: 0, count: 6) + + // whole Mat + var bytesNum = try m.get(row: 1, col: 1, data: &buff) + + XCTAssertEqual(12, bytesNum); + XCTAssert(buff == [110, 111, 120, 121, 130, 131]) + + // sub-Mat + let sm = m.submat(rowStart: 2, rowEnd: 4, colStart: 3, colEnd: 5) + var buff00 = [UInt16](repeating: 0, count: 4) + bytesNum = try sm.get(row: 0, col: 0, data: &buff00) + XCTAssertEqual(8, bytesNum) + XCTAssert(buff00 == [230, 231, 240, 241]) + var buff11 = [UInt16](repeating: 0, count: 4) + bytesNum = try sm.get(row: 1, col: 1, data: &buff11) + XCTAssertEqual(4, bytesNum); + XCTAssert(buff11 == [340, 341, 0, 0]) + } + func testHeight() { XCTAssertEqual(gray0.rows(), gray0.height()) XCTAssertEqual(rgbLena.rows(), rgbLena.height()) @@ -653,7 +700,7 @@ class MatTests: OpenCVTestCase { try assertMatEqual(truth!, m1, OpenCVTestCase.EPS) } - func testPutIntIntByteArray() throws { + func testPutIntIntInt8Array() throws { let m = Mat(rows: 5, cols: 5, type: CvType.CV_8SC3, scalar: Scalar(1, 2, 3)) let sm = m.submat(rowStart: 2, rowEnd: 4, colStart: 3, colEnd: 5) var buff = [Int8](repeating: 0, count: 6) @@ -683,7 +730,37 @@ class MatTests: OpenCVTestCase { XCTAssert(buff == buff0) } - func testPutIntArrayByteArray() throws { + func testPutIntIntUInt8Array() throws { + let m = Mat(rows: 5, cols: 5, type: CvType.CV_8UC3, scalar: Scalar(1, 2, 3)) + let sm = m.submat(rowStart: 2, rowEnd: 4, colStart: 3, colEnd: 5) + var buff = [UInt8](repeating: 0, count: 6) + let buff0:[UInt8] = [10, 20, 30, 40, 50, 60] + let buff1:[UInt8] = [255, 254, 253, 252, 251, 250] + + var bytesNum = try m.put(row:1, col:2, data:buff0) + + XCTAssertEqual(6, bytesNum) + bytesNum = try m.get(row: 1, col: 2, data: &buff) + XCTAssertEqual(6, bytesNum) + XCTAssert(buff == buff0) + + bytesNum = try sm.put(row:0, col:0, data:buff1) + + XCTAssertEqual(6, bytesNum) + bytesNum = try sm.get(row: 0, col: 0, data: &buff) + XCTAssertEqual(6, bytesNum) + XCTAssert(buff == buff1) + bytesNum = try m.get(row: 2, col: 3, data: &buff) + XCTAssertEqual(6, bytesNum); + XCTAssert(buff == buff1) + + let m1 = m.row(1) + bytesNum = try m1.get(row: 0, col: 2, data: &buff) + XCTAssertEqual(6, bytesNum) + XCTAssert(buff == buff0) + } + + func testPutIntArrayInt8Array() throws { let m = Mat(sizes: [5, 5, 5], type: CvType.CV_8SC3, scalar: Scalar(1, 2, 3)) let sm = m.submat(ranges: [Range(start: 0, end: 2), Range(start: 1, end: 3), Range(start: 2, end: 4)]) var buff = [Int8](repeating: 0, count: 6) @@ -714,10 +791,41 @@ class MatTests: OpenCVTestCase { XCTAssert(buff == buff0) } + func testPutIntArrayUInt8Array() throws { + let m = Mat(sizes: [5, 5, 5], type: CvType.CV_8UC3, scalar: Scalar(1, 2, 3)) + let sm = m.submat(ranges: [Range(start: 0, end: 2), Range(start: 1, end: 3), Range(start: 2, end: 4)]) + var buff = [UInt8](repeating: 0, count: 6) + let buff0:[UInt8] = [10, 20, 30, 40, 50, 60] + let buff1:[UInt8] = [255, 254, 253, 252, 251, 250] + + var bytesNum = try m.put(indices:[1, 2, 0], data:buff0) + + XCTAssertEqual(6, bytesNum) + bytesNum = try m.get(indices: [1, 2, 0], data: &buff) + XCTAssertEqual(6, bytesNum) + XCTAssert(buff == buff0) + + bytesNum = try sm.put(indices: [0, 0, 0], data: buff1) + + XCTAssertEqual(6, bytesNum) + bytesNum = try sm.get(indices: [0, 0, 0], data: &buff) + XCTAssertEqual(6, bytesNum) + XCTAssert(buff == buff1) + + bytesNum = try m.get(indices: [0, 1, 2], data: &buff) + XCTAssertEqual(6, bytesNum) + XCTAssert(buff == buff1) + + let m1 = m.submat(ranges: [Range(start: 1,end: 2), Range.all(), Range.all()]) + bytesNum = try m1.get(indices: [0, 2, 0], data: &buff) + XCTAssertEqual(6, bytesNum) + XCTAssert(buff == buff0) + } + func testPutIntIntDoubleArray() throws { - let m = Mat(rows: 5, cols: 5, type: CvType.CV_8SC3, scalar: Scalar(1, 2, 3)) + let m = Mat(rows: 5, cols: 5, type: CvType.CV_8UC3, scalar: Scalar(1, 2, 3)) let sm = m.submat(rowStart: 2, rowEnd: 4, colStart: 3, colEnd: 5) - var buff = [Int8](repeating: 0, count: 6) + var buff = [UInt8](repeating: 0, count: 6) var bytesNum = try m.put(row: 1, col: 2, data: [10, 20, 30, 40, 50, 60] as [Double]) @@ -731,16 +839,16 @@ class MatTests: OpenCVTestCase { XCTAssertEqual(6, bytesNum) bytesNum = try sm.get(row: 0, col: 0, data: &buff) XCTAssertEqual(6, bytesNum); - XCTAssert(buff == [-1, -2, -3, -4, -5, -6]) + XCTAssert(buff == [255, 254, 253, 252, 251, 250]) bytesNum = try m.get(row: 2, col: 3, data: &buff) XCTAssertEqual(6, bytesNum); - XCTAssert(buff == [-1, -2, -3, -4, -5, -6]) + XCTAssert(buff == [255, 254, 253, 252, 251, 250]) } func testPutIntArrayDoubleArray() throws { - let m = Mat(sizes: [5, 5, 5], type: CvType.CV_8SC3, scalar: Scalar(1, 2, 3)) + let m = Mat(sizes: [5, 5, 5], type: CvType.CV_8UC3, scalar: Scalar(1, 2, 3)) let sm = m.submat(ranges: [Range(start: 0, end: 2), Range(start: 1, end: 3), Range(start: 2, end: 4)]) - var buff = [Int8](repeating: 0, count: 6) + var buff = [UInt8](repeating: 0, count: 6) var bytesNum = try m.put(indices: [1, 2, 0], data: [10, 20, 30, 40, 50, 60] as [Double]) @@ -754,10 +862,10 @@ class MatTests: OpenCVTestCase { XCTAssertEqual(6, bytesNum); bytesNum = try sm.get(indices: [0, 0, 0], data: &buff) XCTAssertEqual(6, bytesNum); - XCTAssert(buff == [-1, -2, -3, -4, -5, -6]) + XCTAssert(buff == [255, 254, 253, 252, 251, 250]) bytesNum = try m.get(indices: [0, 1, 2], data: &buff) XCTAssertEqual(6, bytesNum) - XCTAssert(buff == [-1, -2, -3, -4, -5, -6]) + XCTAssert(buff == [255, 254, 253, 252, 251, 250]) } func testPutIntIntFloatArray() throws { @@ -820,7 +928,7 @@ class MatTests: OpenCVTestCase { XCTAssert([40, 50, 60] == m.get(indices: [0, 1, 0])) } - func testPutIntIntShortArray() throws { + func testPutIntIntInt16Array() throws { let m = Mat(rows: 5, cols: 5, type: CvType.CV_16SC3, scalar: Scalar(-1, -2, -3)) let elements: [Int16] = [ 10, 20, 30, 40, 50, 60] @@ -834,7 +942,21 @@ class MatTests: OpenCVTestCase { XCTAssert([40, 50, 60] == m.get(row: 2, col: 4)) } - func testPutIntArrayShortArray() throws { + func testPutIntIntUInt16Array() throws { + let m = Mat(rows: 5, cols: 5, type: CvType.CV_16UC3, scalar: Scalar(-1, -2, -3)) + let elements: [UInt16] = [ 10, 20, 30, 40, 50, 60] + + var bytesNum = try m.put(row: 2, col: 3, data: elements) + + XCTAssertEqual(Int32(elements.count * 2), bytesNum) + let m1 = m.col(3) + var buff = [UInt16](repeating: 0, count: 3) + bytesNum = try m1.get(row: 2, col: 0, data: &buff) + XCTAssert(buff == [10, 20, 30]) + XCTAssert([40, 50, 60] == m.get(row: 2, col: 4)) + } + + func testPutIntArrayInt16Array() throws { let m = Mat(sizes: [5, 5, 5], type: CvType.CV_16SC3, scalar: Scalar(-1, -2, -3)) let elements: [Int16] = [ 10, 20, 30, 40, 50, 60] @@ -848,6 +970,20 @@ class MatTests: OpenCVTestCase { XCTAssert([40, 50, 60] == m.get(indices: [0, 2, 4])) } + func testPutIntArrayUInt16Array() throws { + let m = Mat(sizes: [5, 5, 5], type: CvType.CV_16UC3, scalar: Scalar(-1, -2, -3)) + let elements: [UInt16] = [ 10, 20, 30, 40, 50, 60] + + var bytesNum = try m.put(indices: [0, 2, 3], data: elements) + + XCTAssertEqual(Int32(elements.count * 2), bytesNum) + let m1 = m.submat(ranges: [Range.all(), Range.all(), Range(start: 3, end: 4)]) + var buff = [UInt16](repeating: 0, count: 3) + bytesNum = try m1.get(indices: [0, 2, 0], data: &buff) + XCTAssert(buff == [10, 20, 30]) + XCTAssert([40, 50, 60] == m.get(indices: [0, 2, 4])) + } + func testReshapeInt() throws { let src = Mat(rows: 4, cols: 4, type: CvType.CV_8U, scalar: Scalar(0)) dst = src.reshape(channels: 4) From 4eac198270783d8924ed26ecfb82f8aa54d9e67d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 29 Jun 2021 09:00:10 +0000 Subject: [PATCH 019/376] core(persistence): fix types format handling, fix 16F support --- modules/core/src/persistence.cpp | 19 +++++++--- modules/core/test/test_io.cpp | 65 ++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 4bf52a3134df..32328361e874 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -143,17 +143,17 @@ static const char symbols[9] = "ucwsifdh"; static char typeSymbol(int depth) { CV_StaticAssert(CV_64F == 6, ""); - CV_Assert(depth >=0 && depth <= CV_64F); + CV_CheckDepth(depth, depth >=0 && depth <= CV_16F, ""); return symbols[depth]; } static int symbolToType(char c) { + if (c == 'r') + return CV_SEQ_ELTYPE_PTR; const char* pos = strchr( symbols, c ); if( !pos ) CV_Error( CV_StsBadArg, "Invalid data type specification" ); - if (c == 'r') - return CV_SEQ_ELTYPE_PTR; return static_cast(pos - symbols); } @@ -245,8 +245,12 @@ int calcStructSize( const char* dt, int initial_size ) { int size = calcElemSize( dt, initial_size ); size_t elem_max_size = 0; - for ( const char * type = dt; *type != '\0'; type++ ) { - switch ( *type ) + for ( const char * type = dt; *type != '\0'; type++ ) + { + char v = *type; + if (v >= '0' && v <= '9') + continue; // skip vector size + switch (v) { case 'u': { elem_max_size = std::max( elem_max_size, sizeof(uchar ) ); break; } case 'c': { elem_max_size = std::max( elem_max_size, sizeof(schar ) ); break; } @@ -255,7 +259,9 @@ int calcStructSize( const char* dt, int initial_size ) case 'i': { elem_max_size = std::max( elem_max_size, sizeof(int ) ); break; } case 'f': { elem_max_size = std::max( elem_max_size, sizeof(float ) ); break; } case 'd': { elem_max_size = std::max( elem_max_size, sizeof(double) ); break; } - default: break; + case 'h': { elem_max_size = std::max(elem_max_size, sizeof(float16_t)); break; } + default: + CV_Error_(Error::StsNotImplemented, ("Unknown type identifier: '%c' in '%s'", (char)(*type), dt)); } } size = cvAlign( size, static_cast(elem_max_size) ); @@ -1054,6 +1060,7 @@ class FileStorage::Impl : public FileStorage_API CV_Assert(write_mode); size_t elemSize = fs::calcStructSize(dt.c_str(), 0); + CV_Assert(elemSize); CV_Assert( len % elemSize == 0 ); len /= elemSize; diff --git a/modules/core/test/test_io.cpp b/modules/core/test/test_io.cpp index d30c48536888..82bd05372da7 100644 --- a/modules/core/test/test_io.cpp +++ b/modules/core/test/test_io.cpp @@ -1837,4 +1837,69 @@ TEST(Core_InputOutput, FileStorage_copy_constructor_17412_heap) EXPECT_EQ(0, remove(fname.c_str())); } + +static void test_20279(FileStorage& fs) +{ + Mat m32fc1(5, 10, CV_32FC1, Scalar::all(0)); + for (size_t i = 0; i < m32fc1.total(); i++) + { + float v = (float)i; + m32fc1.at((int)i) = v * 0.5f; + } + Mat m16fc1; + // produces CV_16S output: convertFp16(m32fc1, m16fc1); + m32fc1.convertTo(m16fc1, CV_16FC1); + EXPECT_EQ(CV_16FC1, m16fc1.type()) << typeToString(m16fc1.type()); + //std::cout << m16fc1 << std::endl; + + Mat m32fc3(4, 3, CV_32FC3, Scalar::all(0)); + for (size_t i = 0; i < m32fc3.total(); i++) + { + float v = (float)i; + m32fc3.at((int)i) = Vec3f(v, v * 0.2f, -v); + } + Mat m16fc3; + m32fc3.convertTo(m16fc3, CV_16FC3); + EXPECT_EQ(CV_16FC3, m16fc3.type()) << typeToString(m16fc3.type()); + //std::cout << m16fc3 << std::endl; + + fs << "m16fc1" << m16fc1; + fs << "m16fc3" << m16fc3; + + string content = fs.releaseAndGetString(); + if (cvtest::debugLevel > 0) std::cout << content << std::endl; + + FileStorage fs_read(content, FileStorage::READ + FileStorage::MEMORY); + Mat m16fc1_result; + Mat m16fc3_result; + fs_read["m16fc1"] >> m16fc1_result; + ASSERT_FALSE(m16fc1_result.empty()); + EXPECT_EQ(CV_16FC1, m16fc1_result.type()) << typeToString(m16fc1_result.type()); + EXPECT_LE(cvtest::norm(m16fc1_result, m16fc1, NORM_INF), 1e-2); + + fs_read["m16fc3"] >> m16fc3_result; + ASSERT_FALSE(m16fc3_result.empty()); + EXPECT_EQ(CV_16FC3, m16fc3_result.type()) << typeToString(m16fc3_result.type()); + EXPECT_LE(cvtest::norm(m16fc3_result, m16fc3, NORM_INF), 1e-2); +} + +TEST(Core_InputOutput, FileStorage_16F_xml) +{ + FileStorage fs("test.xml", cv::FileStorage::WRITE | cv::FileStorage::MEMORY); + test_20279(fs); +} + +TEST(Core_InputOutput, FileStorage_16F_yml) +{ + FileStorage fs("test.yml", cv::FileStorage::WRITE | cv::FileStorage::MEMORY); + test_20279(fs); +} + +TEST(Core_InputOutput, FileStorage_16F_json) +{ + FileStorage fs("test.json", cv::FileStorage::WRITE | cv::FileStorage::MEMORY); + test_20279(fs); +} + + }} // namespace From 7d842f5bcffc54e25b365935a79b99b96c49e01d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 29 Jun 2021 18:48:21 +0000 Subject: [PATCH 020/376] dnn: use OpenVINO 2021.4 defines --- cmake/OpenCVDetectInferenceEngine.cmake | 4 ++-- modules/dnn/src/op_inf_engine.hpp | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cmake/OpenCVDetectInferenceEngine.cmake b/cmake/OpenCVDetectInferenceEngine.cmake index aa4bb4a864ae..829ddbfe7e2a 100644 --- a/cmake/OpenCVDetectInferenceEngine.cmake +++ b/cmake/OpenCVDetectInferenceEngine.cmake @@ -141,9 +141,9 @@ endif() if(INF_ENGINE_TARGET) if(NOT INF_ENGINE_RELEASE) - message(WARNING "InferenceEngine version has not been set, 2021.3 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") + message(WARNING "InferenceEngine version has not been set, 2021.4 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") endif() - set(INF_ENGINE_RELEASE "2021030000" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") + set(INF_ENGINE_RELEASE "2021040000" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") set_target_properties(${INF_ENGINE_TARGET} PROPERTIES INTERFACE_COMPILE_DEFINITIONS "HAVE_INF_ENGINE=1;INF_ENGINE_RELEASE=${INF_ENGINE_RELEASE}" ) diff --git a/modules/dnn/src/op_inf_engine.hpp b/modules/dnn/src/op_inf_engine.hpp index 42008b0f10b8..a825431627bd 100644 --- a/modules/dnn/src/op_inf_engine.hpp +++ b/modules/dnn/src/op_inf_engine.hpp @@ -30,10 +30,11 @@ #define INF_ENGINE_RELEASE_2021_1 2021010000 #define INF_ENGINE_RELEASE_2021_2 2021020000 #define INF_ENGINE_RELEASE_2021_3 2021030000 +#define INF_ENGINE_RELEASE_2021_4 2021040000 #ifndef INF_ENGINE_RELEASE -#warning("IE version have not been provided via command-line. Using 2021.3 by default") -#define INF_ENGINE_RELEASE INF_ENGINE_RELEASE_2021_3 +#warning("IE version have not been provided via command-line. Using 2021.4 by default") +#define INF_ENGINE_RELEASE INF_ENGINE_RELEASE_2021_4 #endif #define INF_ENGINE_VER_MAJOR_GT(ver) (((INF_ENGINE_RELEASE) / 10000) > ((ver) / 10000)) From db4b1e613ccdb449fad83c0711a3d44cb21662bc Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 29 Jun 2021 09:00:10 +0000 Subject: [PATCH 021/376] core(persistence): fix types format handling partial backport of 4eac198270783d8924ed26ecfb82f8aa54d9e67d --- modules/core/src/persistence.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 40d1cdfa0796..7e9d107c3524 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -6,6 +6,8 @@ #include "precomp.hpp" #include "persistence.hpp" +using namespace cv; + char* icv_itoa( int _val, char* buffer, int /*radix*/ ) { const int radix = 10; @@ -519,12 +521,16 @@ static const char symbols[9] = "ucwsifdr"; char icvTypeSymbol(int depth) { - CV_Assert(depth >=0 && depth < 9); + CV_StaticAssert(CV_64F == 6, ""); + CV_Assert(depth >=0 && depth <= CV_64F); + CV_CheckDepth(depth, depth >=0 && depth <= CV_64F, ""); return symbols[depth]; } static int icvSymbolToType(char c) { + if (c == 'r') + return CV_SEQ_ELTYPE_PTR; const char* pos = strchr( symbols, c ); if( !pos ) CV_Error( CV_StsBadArg, "Invalid data type specification" ); @@ -618,8 +624,12 @@ int icvCalcStructSize( const char* dt, int initial_size ) { int size = icvCalcElemSize( dt, initial_size ); size_t elem_max_size = 0; - for ( const char * type = dt; *type != '\0'; type++ ) { - switch ( *type ) + for ( const char * type = dt; *type != '\0'; type++ ) + { + char v = *type; + if (v >= '0' && v <= '9') + continue; // skip vector size + switch (v) { case 'u': { elem_max_size = std::max( elem_max_size, sizeof(uchar ) ); break; } case 'c': { elem_max_size = std::max( elem_max_size, sizeof(schar ) ); break; } @@ -628,7 +638,8 @@ int icvCalcStructSize( const char* dt, int initial_size ) case 'i': { elem_max_size = std::max( elem_max_size, sizeof(int ) ); break; } case 'f': { elem_max_size = std::max( elem_max_size, sizeof(float ) ); break; } case 'd': { elem_max_size = std::max( elem_max_size, sizeof(double) ); break; } - default: break; + default: + CV_Error_(Error::StsNotImplemented, ("Unknown type identifier: '%c' in '%s'", (char)(*type), dt)); } } size = cvAlign( size, static_cast(elem_max_size) ); From fb7ef76e742f408cc4e23ba7b67c99db06d0140d Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Wed, 30 Jun 2021 12:04:09 +0300 Subject: [PATCH 022/376] Merge pull request #20271 from TolyaTalamanov:at/extend-python-bindings G-API: Extend python bindings * Extend G-API bindings * Wrap timestamp, seqNo, seq_id * Wrap copy * Wrap parseSSD, parseYolo * Rewrap cv.gapi.networks * Add test for metabackend in pytnon * Remove int64 pyopencv_to --- modules/gapi/include/opencv2/gapi/gcommon.hpp | 2 + .../gapi/include/opencv2/gapi/gstreaming.hpp | 2 +- modules/gapi/include/opencv2/gapi/infer.hpp | 6 +- .../include/opencv2/gapi/infer/parsers.hpp | 20 +- .../include/opencv2/gapi/streaming/format.hpp | 2 +- .../gapi/misc/python/package/gapi/__init__.py | 25 ++ modules/gapi/misc/python/pyopencv_gapi.hpp | 220 ++++++++++-------- modules/gapi/misc/python/python_bridge.hpp | 3 + modules/gapi/misc/python/shadow_gapi.hpp | 29 +-- .../misc/python/test/test_gapi_streaming.py | 42 +++- modules/gapi/src/api/ginfer.cpp | 4 + .../gapi/src/backends/common/gmetabackend.cpp | 13 ++ modules/python/src2/cv2.cpp | 6 - 13 files changed, 235 insertions(+), 139 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/gcommon.hpp b/modules/gapi/include/opencv2/gapi/gcommon.hpp index a9cb0159014e..d3c280816ff9 100644 --- a/modules/gapi/include/opencv2/gapi/gcommon.hpp +++ b/modules/gapi/include/opencv2/gapi/gcommon.hpp @@ -44,6 +44,7 @@ namespace detail CV_UNKNOWN, // Unknown, generic, opaque-to-GAPI data type unsupported in graph seriallization CV_BOOL, // bool user G-API data CV_INT, // int user G-API data + CV_INT64, // int64_t user G-API data CV_DOUBLE, // double user G-API data CV_FLOAT, // float user G-API data CV_UINT64, // uint64_t user G-API data @@ -61,6 +62,7 @@ namespace detail template struct GOpaqueTraits; template struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_UNKNOWN; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_INT; }; + template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_INT64; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_DOUBLE; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_FLOAT; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_UINT64; }; diff --git a/modules/gapi/include/opencv2/gapi/gstreaming.hpp b/modules/gapi/include/opencv2/gapi/gstreaming.hpp index 5bbed5e12dda..47e103fd0ea7 100644 --- a/modules/gapi/include/opencv2/gapi/gstreaming.hpp +++ b/modules/gapi/include/opencv2/gapi/gstreaming.hpp @@ -196,7 +196,7 @@ class GAPI_EXPORTS_W_SIMPLE GStreamingCompiled * @param s a shared pointer to IStreamSource representing the * input video stream. */ - GAPI_WRAP void setSource(const gapi::wip::IStreamSource::Ptr& s); + void setSource(const gapi::wip::IStreamSource::Ptr& s); /** * @brief Constructs and specifies an input video stream for a diff --git a/modules/gapi/include/opencv2/gapi/infer.hpp b/modules/gapi/include/opencv2/gapi/infer.hpp index 93701856bbdb..807c82d31f89 100644 --- a/modules/gapi/include/opencv2/gapi/infer.hpp +++ b/modules/gapi/include/opencv2/gapi/infer.hpp @@ -136,11 +136,12 @@ class GInferInputsTyped } template - void setInput(const std::string& name, U in) + GInferInputsTyped& setInput(const std::string& name, U in) { m_priv->blobs.emplace(std::piecewise_construct, std::forward_as_tuple(name), std::forward_as_tuple(in)); + return *this; } using StorageT = cv::util::variant; @@ -654,7 +655,7 @@ namespace gapi { // A type-erased form of network parameters. // Similar to how a type-erased GKernel is represented and used. /// @private -struct GAPI_EXPORTS GNetParam { +struct GAPI_EXPORTS_W_SIMPLE GNetParam { std::string tag; // FIXME: const? GBackend backend; // Specifies the execution model util::any params; // Backend-interpreted parameter structure @@ -671,6 +672,7 @@ struct GAPI_EXPORTS GNetParam { */ struct GAPI_EXPORTS_W_SIMPLE GNetPackage { GAPI_WRAP GNetPackage() = default; + GAPI_WRAP explicit GNetPackage(std::vector nets); explicit GNetPackage(std::initializer_list ii); std::vector backends() const; std::vector networks; diff --git a/modules/gapi/include/opencv2/gapi/infer/parsers.hpp b/modules/gapi/include/opencv2/gapi/infer/parsers.hpp index 22c8701a6c2e..c7308dd39f47 100644 --- a/modules/gapi/include/opencv2/gapi/infer/parsers.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/parsers.hpp @@ -64,10 +64,10 @@ detection is smaller than confidence threshold, detection is rejected. given label will get to the output. @return a tuple with a vector of detected boxes and a vector of appropriate labels. */ -GAPI_EXPORTS std::tuple, GArray> parseSSD(const GMat& in, - const GOpaque& inSz, - const float confidenceThreshold = 0.5f, - const int filterLabel = -1); +GAPI_EXPORTS_W std::tuple, GArray> parseSSD(const GMat& in, + const GOpaque& inSz, + const float confidenceThreshold = 0.5f, + const int filterLabel = -1); /** @brief Parses output of SSD network. @@ -113,12 +113,12 @@ If 1.f, nms is not performed and no boxes are rejected. documentation. @return a tuple with a vector of detected boxes and a vector of appropriate labels. */ -GAPI_EXPORTS std::tuple, GArray> parseYolo(const GMat& in, - const GOpaque& inSz, - const float confidenceThreshold = 0.5f, - const float nmsThreshold = 0.5f, - const std::vector& anchors - = nn::parsers::GParseYolo::defaultAnchors()); +GAPI_EXPORTS_W std::tuple, GArray> parseYolo(const GMat& in, + const GOpaque& inSz, + const float confidenceThreshold = 0.5f, + const float nmsThreshold = 0.5f, + const std::vector& anchors + = nn::parsers::GParseYolo::defaultAnchors()); } // namespace gapi } // namespace cv diff --git a/modules/gapi/include/opencv2/gapi/streaming/format.hpp b/modules/gapi/include/opencv2/gapi/streaming/format.hpp index c9d2fa3e0a29..f7c3bd457dfb 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/format.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/format.hpp @@ -74,7 +74,7 @@ e.g when graph's input needs to be passed directly to output, like in Streaming @param in Input image @return Copy of the input */ -GAPI_EXPORTS GMat copy(const GMat& in); +GAPI_EXPORTS_W GMat copy(const GMat& in); /** @brief Makes a copy of the input frame. Note that this copy may be not real (no actual data copied). Use this function to maintain graph contracts, diff --git a/modules/gapi/misc/python/package/gapi/__init__.py b/modules/gapi/misc/python/package/gapi/__init__.py index 23f5f41846f3..587f641fd33a 100644 --- a/modules/gapi/misc/python/package/gapi/__init__.py +++ b/modules/gapi/misc/python/package/gapi/__init__.py @@ -11,11 +11,36 @@ def parameterized(func): return parameterized +@register('cv2.gapi') +def networks(*args): + return cv.gapi_GNetPackage(list(map(cv.detail.strip, args))) + + @register('cv2.gapi') def compile_args(*args): return list(map(cv.GCompileArg, args)) +@register('cv2') +def GIn(*args): + return [*args] + + +@register('cv2') +def GOut(*args): + return [*args] + + +@register('cv2') +def gin(*args): + return [*args] + + +@register('cv2.gapi') +def descr_of(*args): + return [*args] + + @register('cv2') class GOpaque(): # NB: Inheritance from c++ class cause segfault. diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index 6b782cfc8dd8..6cd79e4a7318 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -17,6 +17,7 @@ using gapi_ie_PyParams = cv::gapi::ie::PyParams; using gapi_wip_IStreamSource_Ptr = cv::Ptr; using detail_ExtractArgsCallback = cv::detail::ExtractArgsCallback; using detail_ExtractMetaCallback = cv::detail::ExtractMetaCallback; +using vector_GNetParam = std::vector; // NB: Python wrapper generate T_U for T // This behavior is only observed for inputs @@ -138,6 +139,7 @@ PyObject* pyopencv_from(const cv::GArg& value) { HANDLE_CASE(BOOL, bool); HANDLE_CASE(INT, int); + HANDLE_CASE(INT64, int64_t); HANDLE_CASE(DOUBLE, double); HANDLE_CASE(FLOAT, float); HANDLE_CASE(STRING, std::string); @@ -164,23 +166,29 @@ bool pyopencv_to(PyObject* obj, cv::GArg& value, const ArgInfo& info) } template <> -bool pyopencv_to(PyObject* obj, std::vector& value, const ArgInfo& info) +bool pyopencv_to(PyObject* obj, std::vector& value, const ArgInfo& info) { return pyopencv_to_generic_vec(obj, value, info); } template <> -PyObject* pyopencv_from(const std::vector& value) +PyObject* pyopencv_from(const std::vector& value) { return pyopencv_from_generic_vec(value); } template <> -bool pyopencv_to(PyObject* obj, GRunArgs& value, const ArgInfo& info) +bool pyopencv_to(PyObject* obj, std::vector& value, const ArgInfo& info) { return pyopencv_to_generic_vec(obj, value, info); } +template <> +PyObject* pyopencv_from(const std::vector& value) +{ + return pyopencv_from_generic_vec(value); +} + template<> PyObject* pyopencv_from(const cv::detail::OpaqueRef& o) { @@ -188,6 +196,7 @@ PyObject* pyopencv_from(const cv::detail::OpaqueRef& o) { case cv::detail::OpaqueKind::CV_BOOL : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_INT : return pyopencv_from(o.rref()); + case cv::detail::OpaqueKind::CV_INT64 : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_DOUBLE : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_FLOAT : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_STRING : return pyopencv_from(o.rref()); @@ -213,6 +222,7 @@ PyObject* pyopencv_from(const cv::detail::VectorRef& v) { case cv::detail::OpaqueKind::CV_BOOL : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_INT : return pyopencv_from_generic_vec(v.rref()); + case cv::detail::OpaqueKind::CV_INT64 : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_DOUBLE : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_FLOAT : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_STRING : return pyopencv_from_generic_vec(v.rref()); @@ -285,18 +295,6 @@ PyObject* pyopencv_from(const GRunArgs& value) return list; } -template<> -bool pyopencv_to(PyObject* obj, GMetaArgs& value, const ArgInfo& info) -{ - return pyopencv_to_generic_vec(obj, value, info); -} - -template<> -PyObject* pyopencv_from(const GMetaArgs& value) -{ - return pyopencv_from_generic_vec(value); -} - template void pyopencv_to_with_check(PyObject* from, T& to, const std::string& msg = "") { @@ -318,16 +316,16 @@ void pyopencv_to_generic_vec_with_check(PyObject* from, } template -static PyObject* extract_proto_args(PyObject* py_args, PyObject* kw) +static T extract_proto_args(PyObject* py_args) { using namespace cv; GProtoArgs args; - Py_ssize_t size = PyTuple_Size(py_args); + Py_ssize_t size = PyList_Size(py_args); args.reserve(size); for (int i = 0; i < size; ++i) { - PyObject* item = PyTuple_GetItem(py_args, i); + PyObject* item = PyList_GetItem(py_args, i); if (PyObject_TypeCheck(item, reinterpret_cast(pyopencv_GScalar_TypePtr))) { args.emplace_back(reinterpret_cast(item)->v); @@ -346,22 +344,11 @@ static PyObject* extract_proto_args(PyObject* py_args, PyObject* kw) } else { - PyErr_SetString(PyExc_TypeError, "Unsupported type for cv.GIn()/cv.GOut()"); - return NULL; + util::throw_error(std::logic_error("Unsupported type for GProtoArgs")); } } - return pyopencv_from(T{std::move(args)}); -} - -static PyObject* pyopencv_cv_GIn(PyObject* , PyObject* py_args, PyObject* kw) -{ - return extract_proto_args(py_args, kw); -} - -static PyObject* pyopencv_cv_GOut(PyObject* , PyObject* py_args, PyObject* kw) -{ - return extract_proto_args(py_args, kw); + return T(std::move(args)); } static cv::detail::OpaqueRef extract_opaque_ref(PyObject* from, cv::detail::OpaqueKind kind) @@ -386,6 +373,7 @@ static cv::detail::OpaqueRef extract_opaque_ref(PyObject* from, cv::detail::Opaq HANDLE_CASE(RECT, cv::Rect); HANDLE_CASE(UNKNOWN, cv::GArg); UNSUPPORTED(UINT64); + UNSUPPORTED(INT64); UNSUPPORTED(SCALAR); UNSUPPORTED(MAT); UNSUPPORTED(DRAW_PRIM); @@ -419,6 +407,7 @@ static cv::detail::VectorRef extract_vector_ref(PyObject* from, cv::detail::Opaq HANDLE_CASE(MAT, cv::Mat); HANDLE_CASE(UNKNOWN, cv::GArg); UNSUPPORTED(UINT64); + UNSUPPORTED(INT64); UNSUPPORTED(DRAW_PRIM); #undef HANDLE_CASE #undef UNSUPPORTED @@ -470,13 +459,15 @@ static cv::GRunArg extract_run_arg(const cv::GTypeInfo& info, PyObject* item) static cv::GRunArgs extract_run_args(const cv::GTypesInfo& info, PyObject* py_args) { + GAPI_Assert(PyList_Check(py_args)); + cv::GRunArgs args; - Py_ssize_t tuple_size = PyTuple_Size(py_args); - args.reserve(tuple_size); + Py_ssize_t list_size = PyList_Size(py_args); + args.reserve(list_size); - for (int i = 0; i < tuple_size; ++i) + for (int i = 0; i < list_size; ++i) { - args.push_back(extract_run_arg(info[i], PyTuple_GetItem(py_args, i))); + args.push_back(extract_run_arg(info[i], PyList_GetItem(py_args, i))); } return args; @@ -517,13 +508,15 @@ static cv::GMetaArg extract_meta_arg(const cv::GTypeInfo& info, PyObject* item) static cv::GMetaArgs extract_meta_args(const cv::GTypesInfo& info, PyObject* py_args) { + GAPI_Assert(PyList_Check(py_args)); + cv::GMetaArgs metas; - Py_ssize_t tuple_size = PyTuple_Size(py_args); - metas.reserve(tuple_size); + Py_ssize_t list_size = PyList_Size(py_args); + metas.reserve(list_size); - for (int i = 0; i < tuple_size; ++i) + for (int i = 0; i < list_size; ++i) { - metas.push_back(extract_meta_arg(info[i], PyTuple_GetItem(py_args, i))); + metas.push_back(extract_meta_arg(info[i], PyList_GetItem(py_args, i))); } return metas; @@ -589,8 +582,27 @@ static cv::GRunArgs run_py_kernel(cv::detail::PyObjectHolder kernel, // NB: In fact it's impossible situation, becase errors were handled above. GAPI_Assert(result.get() && "Python kernel returned NULL!"); - outs = out_info.size() == 1 ? cv::GRunArgs{extract_run_arg(out_info[0], result.get())} - : extract_run_args(out_info, result.get()); + if (out_info.size() == 1) + { + outs = cv::GRunArgs{extract_run_arg(out_info[0], result.get())}; + } + else if (out_info.size() > 1) + { + GAPI_Assert(PyTuple_Check(result.get())); + + Py_ssize_t tuple_size = PyTuple_Size(result.get()); + outs.reserve(tuple_size); + + for (int i = 0; i < tuple_size; ++i) + { + outs.push_back(extract_run_arg(out_info[i], PyTuple_GetItem(result.get(), i))); + } + } + else + { + // Seems to be impossible case. + GAPI_Assert(false); + } } catch (...) { @@ -756,23 +768,6 @@ static PyObject* pyopencv_cv_gapi_kernels(PyObject* , PyObject* py_args, PyObjec return pyopencv_from(pkg); } -static PyObject* pyopencv_cv_gapi_networks(PyObject*, PyObject* py_args, PyObject*) -{ - using namespace cv; - gapi::GNetPackage pkg; - Py_ssize_t size = PyTuple_Size(py_args); - for (int i = 0; i < size; ++i) - { - gapi_ie_PyParams params; - PyObject* item = PyTuple_GetItem(py_args, i); - if (pyopencv_to(item, params, ArgInfo("PyParams", false))) - { - pkg += gapi::networks(params); - } - } - return pyopencv_from(pkg); -} - static PyObject* pyopencv_cv_gapi_op(PyObject* , PyObject* py_args, PyObject*) { using namespace cv; @@ -834,53 +829,54 @@ static PyObject* pyopencv_cv_gapi_op(PyObject* , PyObject* py_args, PyObject*) return pyopencv_from(cv::gapi::wip::op(id, outMetaWrapper, std::move(args))); } -static PyObject* pyopencv_cv_gin(PyObject*, PyObject* py_args, PyObject*) +template<> +bool pyopencv_to(PyObject* obj, cv::detail::ExtractArgsCallback& value, const ArgInfo&) { - cv::detail::PyObjectHolder holder{py_args}; - auto callback = cv::detail::ExtractArgsCallback{[=](const cv::GTypesInfo& info) - { - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); + cv::detail::PyObjectHolder holder{obj}; + value = cv::detail::ExtractArgsCallback{[=](const cv::GTypesInfo& info) + { + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); - cv::GRunArgs args; - try - { - args = extract_run_args(info, holder.get()); - } - catch (...) - { - PyGILState_Release(gstate); - throw; - } + cv::GRunArgs args; + try + { + args = extract_run_args(info, holder.get()); + } + catch (...) + { PyGILState_Release(gstate); - return args; - }}; - - return pyopencv_from(callback); + throw; + } + PyGILState_Release(gstate); + return args; + }}; + return true; } -static PyObject* pyopencv_cv_descr_of(PyObject*, PyObject* py_args, PyObject*) +template<> +bool pyopencv_to(PyObject* obj, cv::detail::ExtractMetaCallback& value, const ArgInfo&) { - Py_INCREF(py_args); - auto callback = cv::detail::ExtractMetaCallback{[=](const cv::GTypesInfo& info) - { - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); + cv::detail::PyObjectHolder holder{obj}; + value = cv::detail::ExtractMetaCallback{[=](const cv::GTypesInfo& info) + { + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); - cv::GMetaArgs args; - try - { - args = extract_meta_args(info, py_args); - } - catch (...) - { - PyGILState_Release(gstate); - throw; - } + cv::GMetaArgs args; + try + { + args = extract_meta_args(info, holder.get()); + } + catch (...) + { PyGILState_Release(gstate); - return args; - }}; - return pyopencv_from(callback); + throw; + } + PyGILState_Release(gstate); + return args; + }}; + return true; } template @@ -929,11 +925,39 @@ struct PyOpenCV_Converter> } }; +template<> +bool pyopencv_to(PyObject* obj, cv::GProtoInputArgs& value, const ArgInfo& info) +{ + try + { + value = extract_proto_args(obj); + return true; + } + catch (...) + { + failmsg("Can't parse cv::GProtoInputArgs"); + return false; + } +} + +template<> +bool pyopencv_to(PyObject* obj, cv::GProtoOutputArgs& value, const ArgInfo& info) +{ + try + { + value = extract_proto_args(obj); + return true; + } + catch (...) + { + failmsg("Can't parse cv::GProtoOutputArgs"); + return false; + } +} // extend cv.gapi methods #define PYOPENCV_EXTRA_METHODS_GAPI \ {"kernels", CV_PY_FN_WITH_KW(pyopencv_cv_gapi_kernels), "kernels(...) -> GKernelPackage"}, \ - {"networks", CV_PY_FN_WITH_KW(pyopencv_cv_gapi_networks), "networks(...) -> GNetPackage"}, \ {"__op", CV_PY_FN_WITH_KW(pyopencv_cv_gapi_op), "__op(...) -> retval\n"}, diff --git a/modules/gapi/misc/python/python_bridge.hpp b/modules/gapi/misc/python/python_bridge.hpp index 0d1c6d51c574..b212babe4599 100644 --- a/modules/gapi/misc/python/python_bridge.hpp +++ b/modules/gapi/misc/python/python_bridge.hpp @@ -27,6 +27,7 @@ #define GARRAY_TYPE_LIST_G(G, G2) \ WRAP_ARGS(bool , cv::gapi::ArgType::CV_BOOL, G) \ WRAP_ARGS(int , cv::gapi::ArgType::CV_INT, G) \ +WRAP_ARGS(int64_t , cv::gapi::ArgType::CV_INT64, G) \ WRAP_ARGS(double , cv::gapi::ArgType::CV_DOUBLE, G) \ WRAP_ARGS(float , cv::gapi::ArgType::CV_FLOAT, G) \ WRAP_ARGS(std::string , cv::gapi::ArgType::CV_STRING, G) \ @@ -42,6 +43,7 @@ WRAP_ARGS(cv::GMat , cv::gapi::ArgType::CV_GMAT, G2) \ #define GOPAQUE_TYPE_LIST_G(G, G2) \ WRAP_ARGS(bool , cv::gapi::ArgType::CV_BOOL, G) \ WRAP_ARGS(int , cv::gapi::ArgType::CV_INT, G) \ +WRAP_ARGS(int64_t , cv::gapi::ArgType::CV_INT64, G) \ WRAP_ARGS(double , cv::gapi::ArgType::CV_DOUBLE, G) \ WRAP_ARGS(float , cv::gapi::ArgType::CV_FLOAT, G) \ WRAP_ARGS(std::string , cv::gapi::ArgType::CV_STRING, G) \ @@ -58,6 +60,7 @@ namespace gapi { enum ArgType { CV_BOOL, CV_INT, + CV_INT64, CV_DOUBLE, CV_FLOAT, CV_STRING, diff --git a/modules/gapi/misc/python/shadow_gapi.hpp b/modules/gapi/misc/python/shadow_gapi.hpp index 941250c2fb45..e777aa5d934b 100644 --- a/modules/gapi/misc/python/shadow_gapi.hpp +++ b/modules/gapi/misc/python/shadow_gapi.hpp @@ -8,31 +8,20 @@ namespace cv GAPI_WRAP GCompileArg(gapi::GNetPackage pkg); }; - // NB: This classes doesn't exist in *.so - // HACK: Mark them as a class to force python wrapper generate code for this entities - class GAPI_EXPORTS_W_SIMPLE GProtoArg { }; - class GAPI_EXPORTS_W_SIMPLE GProtoInputArgs { }; - class GAPI_EXPORTS_W_SIMPLE GProtoOutputArgs { }; - class GAPI_EXPORTS_W_SIMPLE GRunArg { }; - class GAPI_EXPORTS_W_SIMPLE GMetaArg { GAPI_WRAP GMetaArg(); }; - - using GProtoInputArgs = GIOProtoArgs; - using GProtoOutputArgs = GIOProtoArgs; - class GAPI_EXPORTS_W_SIMPLE GInferInputs { public: GAPI_WRAP GInferInputs(); - GAPI_WRAP void setInput(const std::string& name, const cv::GMat& value); - GAPI_WRAP void setInput(const std::string& name, const cv::GFrame& value); + GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GMat& value); + GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GFrame& value); }; class GAPI_EXPORTS_W_SIMPLE GInferListInputs { public: GAPI_WRAP GInferListInputs(); - GAPI_WRAP void setInput(const std::string& name, const cv::GArray& value); - GAPI_WRAP void setInput(const std::string& name, const cv::GArray& value); + GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); + GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); }; class GAPI_EXPORTS_W_SIMPLE GInferOutputs @@ -51,12 +40,18 @@ namespace cv namespace detail { - struct GAPI_EXPORTS_W_SIMPLE ExtractArgsCallback { }; - struct GAPI_EXPORTS_W_SIMPLE ExtractMetaCallback { }; + gapi::GNetParam GAPI_EXPORTS_W strip(gapi::ie::PyParams params); } // namespace detail namespace gapi { + namespace streaming + { + // FIXME: Extend to work with an arbitrary G-type. + cv::GOpaque GAPI_EXPORTS_W timestamp(cv::GMat); + cv::GOpaque GAPI_EXPORTS_W seqNo(cv::GMat); + cv::GOpaque GAPI_EXPORTS_W seq_id(cv::GMat); + } // namespace streaming namespace wip { class GAPI_EXPORTS_W IStreamSource { }; diff --git a/modules/gapi/misc/python/test/test_gapi_streaming.py b/modules/gapi/misc/python/test/test_gapi_streaming.py index f1cce4fb72fc..4ea88878eeab 100644 --- a/modules/gapi/misc/python/test/test_gapi_streaming.py +++ b/modules/gapi/misc/python/test/test_gapi_streaming.py @@ -28,7 +28,7 @@ def test_image_input(self): g_in = cv.GMat() g_out = cv.gapi.medianBlur(g_in, 3) c = cv.GComputation(g_in, g_out) - ccomp = c.compileStreaming(cv.descr_of(in_mat)) + ccomp = c.compileStreaming(cv.gapi.descr_of(in_mat)) ccomp.setSource(cv.gin(in_mat)) ccomp.start() @@ -52,7 +52,7 @@ def test_video_input(self): ccomp = c.compileStreaming() source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(source) + ccomp.setSource(cv.gin(source)) ccomp.start() # Assert @@ -87,7 +87,7 @@ def test_video_split3(self): ccomp = c.compileStreaming() source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(source) + ccomp.setSource(cv.gin(source)) ccomp.start() # Assert @@ -176,7 +176,7 @@ def test_video_good_features_to_track(self): ccomp = c.compileStreaming() source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(source) + ccomp.setSource(cv.gin(source)) ccomp.start() # Assert @@ -209,6 +209,40 @@ def test_video_good_features_to_track(self): break + def test_gapi_streaming_meta(self): + ksize = 3 + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # G-API + g_in = cv.GMat() + g_ts = cv.gapi.streaming.timestamp(g_in) + g_seqno = cv.gapi.streaming.seqNo(g_in) + g_seqid = cv.gapi.streaming.seq_id(g_in) + + c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_ts, g_seqno, g_seqid)) + + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(cv.gin(source)) + ccomp.start() + + # Assert + max_num_frames = 10 + curr_frame_number = 0 + while True: + has_frame, (ts, seqno, seqid) = ccomp.pull() + + if not has_frame: + break + + self.assertEqual(curr_frame_number, seqno) + self.assertEqual(curr_frame_number, seqid) + + curr_frame_number += 1 + if curr_frame_number == max_num_frames: + break + + except unittest.SkipTest as e: message = str(e) diff --git a/modules/gapi/src/api/ginfer.cpp b/modules/gapi/src/api/ginfer.cpp index e3cc94041c32..9db05a43c369 100644 --- a/modules/gapi/src/api/ginfer.cpp +++ b/modules/gapi/src/api/ginfer.cpp @@ -15,6 +15,10 @@ cv::gapi::GNetPackage::GNetPackage(std::initializer_list ii) : networks(ii) { } +cv::gapi::GNetPackage::GNetPackage(std::vector nets) + : networks(nets) { +} + std::vector cv::gapi::GNetPackage::backends() const { std::unordered_set unique_set; for (const auto &nn : networks) unique_set.insert(nn.backend); diff --git a/modules/gapi/src/backends/common/gmetabackend.cpp b/modules/gapi/src/backends/common/gmetabackend.cpp index c535569b0cef..40e87c3ea0aa 100644 --- a/modules/gapi/src/backends/common/gmetabackend.cpp +++ b/modules/gapi/src/backends/common/gmetabackend.cpp @@ -85,6 +85,19 @@ class GGraphMetaBackendImpl final: public cv::gapi::GBackend::Priv { const std::vector&) const override { return EPtr{new GraphMetaExecutable(graph, nodes)}; } + + virtual bool controlsMerge() const override + { + return true; + } + + virtual bool allowsMerge(const cv::gimpl::GIslandModel::Graph &, + const ade::NodeHandle &, + const ade::NodeHandle &, + const ade::NodeHandle &) const override + { + return false; + } }; cv::gapi::GBackend graph_meta_backend() { diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index 9e8a6ee13bd9..795afb13f276 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -2219,12 +2219,6 @@ static PyMethodDef special_methods[] = { #ifdef HAVE_OPENCV_DNN {"dnn_registerLayer", CV_PY_FN_WITH_KW(pyopencv_cv_dnn_registerLayer), "registerLayer(type, class) -> None"}, {"dnn_unregisterLayer", CV_PY_FN_WITH_KW(pyopencv_cv_dnn_unregisterLayer), "unregisterLayer(type) -> None"}, -#endif -#ifdef HAVE_OPENCV_GAPI - {"GIn", CV_PY_FN_WITH_KW(pyopencv_cv_GIn), "GIn(...) -> GInputProtoArgs"}, - {"GOut", CV_PY_FN_WITH_KW(pyopencv_cv_GOut), "GOut(...) -> GOutputProtoArgs"}, - {"gin", CV_PY_FN_WITH_KW(pyopencv_cv_gin), "gin(...) -> ExtractArgsCallback"}, - {"descr_of", CV_PY_FN_WITH_KW(pyopencv_cv_descr_of), "descr_of(...) -> ExtractMetaCallback"}, #endif {NULL, NULL}, }; From 5e80bd3cc922bad5eb1fa02b56bc3db44abb96e9 Mon Sep 17 00:00:00 2001 From: APrigarina Date: Wed, 30 Jun 2021 12:50:21 +0300 Subject: [PATCH 023/376] fix samples 3.4 --- samples/python/camera_calibration_show_extrinsics.py | 2 +- samples/python/gaussian_mix.py | 2 +- samples/python/hist.py | 2 +- samples/python/lk_homography.py | 6 +++--- samples/python/lk_track.py | 2 +- samples/python/video_v4l2.py | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/samples/python/camera_calibration_show_extrinsics.py b/samples/python/camera_calibration_show_extrinsics.py index 0118b5b913d5..d676691f15d7 100755 --- a/samples/python/camera_calibration_show_extrinsics.py +++ b/samples/python/camera_calibration_show_extrinsics.py @@ -188,7 +188,7 @@ def main(): fig = plt.figure() ax = fig.gca(projection='3d') - ax.set_aspect("equal") + ax.set_aspect("auto") cam_width = args.cam_width cam_height = args.cam_height diff --git a/samples/python/gaussian_mix.py b/samples/python/gaussian_mix.py index 5f2dfcc44093..6a656647ddcf 100755 --- a/samples/python/gaussian_mix.py +++ b/samples/python/gaussian_mix.py @@ -32,7 +32,7 @@ def draw_gaussain(img, mean, cov, color): w, u, _vt = cv.SVDecomp(cov) ang = np.arctan2(u[1, 0], u[0, 0])*(180/np.pi) s1, s2 = np.sqrt(w)*3.0 - cv.ellipse(img, (x, y), (s1, s2), ang, 0, 360, color, 1, cv.LINE_AA) + cv.ellipse(img, (int(x), int(y)), (int(s1), int(s2)), ang, 0, 360, color, 1, cv.LINE_AA) def main(): diff --git a/samples/python/hist.py b/samples/python/hist.py index 4c2c1ad395ef..157d5ff0ba3e 100755 --- a/samples/python/hist.py +++ b/samples/python/hist.py @@ -48,7 +48,7 @@ def hist_lines(im): cv.normalize(hist_item,hist_item,0,255,cv.NORM_MINMAX) hist=np.int32(np.around(hist_item)) for x,y in enumerate(hist): - cv.line(h,(x,0),(x,y),(255,255,255)) + cv.line(h,(x,0),(x,y[0]),(255,255,255)) y = np.flipud(h) return y diff --git a/samples/python/lk_homography.py b/samples/python/lk_homography.py index 808f30965f0d..38a05f63b6a5 100755 --- a/samples/python/lk_homography.py +++ b/samples/python/lk_homography.py @@ -77,8 +77,8 @@ def run(self): for (x0, y0), (x1, y1), good in zip(self.p0[:,0], self.p1[:,0], status[:,0]): if good: - cv.line(vis, (x0, y0), (x1, y1), (0, 128, 0)) - cv.circle(vis, (x1, y1), 2, (red, green)[good], -1) + cv.line(vis, (int(x0), int(y0)), (int(x1), int(y1)), (0, 128, 0)) + cv.circle(vis, (int(x1), int(y1)), 2, (red, green)[good], -1) draw_str(vis, (20, 20), 'track count: %d' % len(self.p1)) if self.use_ransac: draw_str(vis, (20, 40), 'RANSAC') @@ -86,7 +86,7 @@ def run(self): p = cv.goodFeaturesToTrack(frame_gray, **feature_params) if p is not None: for x, y in p[:,0]: - cv.circle(vis, (x, y), 2, green, -1) + cv.circle(vis, (int(x), int(y)), 2, green, -1) draw_str(vis, (20, 20), 'feature count: %d' % len(p)) cv.imshow('lk_homography', vis) diff --git a/samples/python/lk_track.py b/samples/python/lk_track.py index 7b77f1b33595..97a8c40241e2 100755 --- a/samples/python/lk_track.py +++ b/samples/python/lk_track.py @@ -65,7 +65,7 @@ def run(self): if len(tr) > self.track_len: del tr[0] new_tracks.append(tr) - cv.circle(vis, (x, y), 2, (0, 255, 0), -1) + cv.circle(vis, (int(x), int(y)), 2, (0, 255, 0), -1) self.tracks = new_tracks cv.polylines(vis, [np.int32(tr) for tr in self.tracks], False, (0, 255, 0)) draw_str(vis, (20, 20), 'track count: %d' % len(self.tracks)) diff --git a/samples/python/video_v4l2.py b/samples/python/video_v4l2.py index 61b1e3580483..abebb2a2cacc 100644 --- a/samples/python/video_v4l2.py +++ b/samples/python/video_v4l2.py @@ -30,7 +30,7 @@ def decode_fourcc(v): color = (0, 255, 0) cap = cv.VideoCapture(0) - cap.set(cv.CAP_PROP_AUTOFOCUS, False) # Known bug: https://github.com/opencv/opencv/pull/5474 + cap.set(cv.CAP_PROP_AUTOFOCUS, 0) # Known bug: https://github.com/opencv/opencv/pull/5474 cv.namedWindow("Video") @@ -67,7 +67,7 @@ def decode_fourcc(v): break elif k == ord('g'): convert_rgb = not convert_rgb - cap.set(cv.CAP_PROP_CONVERT_RGB, convert_rgb) + cap.set(cv.CAP_PROP_CONVERT_RGB, 1 if convert_rgb else 0) print('Done') From 90be83ae99cb719619f26fedf6d199f6422b551e Mon Sep 17 00:00:00 2001 From: Vladimir <10669582+Wovchena@users.noreply.github.com> Date: Wed, 30 Jun 2021 10:15:58 +0300 Subject: [PATCH 024/376] Fix an arg for calcHist() in demos `float* histRange = { range };` doesn't make much sense. `histRange` is an array of array(s), so it should have a type of ptr to ptr. Strangely some domos are correct as well as the example for the function https://docs.opencv.org/master/d6/dc7/group__imgproc__hist.html#ga4b2b5fd75503ff9e6844cc4dcdaed35d --- .../Histograms_Matching/MatchTemplate_Demo.cpp | 2 +- .../Histograms_Matching/calcBackProject_Demo1.cpp | 6 +++--- .../tutorial_code/Histograms_Matching/calcHist_Demo.cpp | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp b/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp index 5bcc878965a2..f9abbae94527 100644 --- a/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp +++ b/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp @@ -89,7 +89,7 @@ void MatchingMethod( int, void* ) //! [create_result_matrix] /// Create the result matrix - int result_cols = img.cols - templ.cols + 1; + int result_cols = img.cols - templ.cols + 1; int result_rows = img.rows - templ.rows + 1; result.create( result_rows, result_cols, CV_32FC1 ); diff --git a/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo1.cpp b/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo1.cpp index 61b6d607ceb6..bcb547a2fb9f 100644 --- a/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo1.cpp +++ b/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo1.cpp @@ -72,18 +72,18 @@ void Hist_and_Backproj(int, void* ) //! [initialize] int histSize = MAX( bins, 2 ); float hue_range[] = { 0, 180 }; - const float* ranges = { hue_range }; + const float* ranges[] = { hue_range }; //! [initialize] //! [Get the Histogram and normalize it] Mat hist; - calcHist( &hue, 1, 0, Mat(), hist, 1, &histSize, &ranges, true, false ); + calcHist( &hue, 1, 0, Mat(), hist, 1, &histSize, ranges, true, false ); normalize( hist, hist, 0, 255, NORM_MINMAX, -1, Mat() ); //! [Get the Histogram and normalize it] //! [Get Backprojection] Mat backproj; - calcBackProject( &hue, 1, 0, hist, backproj, &ranges, 1, true ); + calcBackProject( &hue, 1, 0, hist, backproj, ranges, 1, true ); //! [Get Backprojection] //! [Draw the backproj] diff --git a/samples/cpp/tutorial_code/Histograms_Matching/calcHist_Demo.cpp b/samples/cpp/tutorial_code/Histograms_Matching/calcHist_Demo.cpp index 86167e519a2f..a7582e42820a 100644 --- a/samples/cpp/tutorial_code/Histograms_Matching/calcHist_Demo.cpp +++ b/samples/cpp/tutorial_code/Histograms_Matching/calcHist_Demo.cpp @@ -37,7 +37,7 @@ int main(int argc, char** argv) //! [Set the ranges ( for B,G,R) )] float range[] = { 0, 256 }; //the upper boundary is exclusive - const float* histRange = { range }; + const float* histRange[] = { range }; //! [Set the ranges ( for B,G,R) )] //! [Set histogram param] @@ -46,9 +46,9 @@ int main(int argc, char** argv) //! [Compute the histograms] Mat b_hist, g_hist, r_hist; - calcHist( &bgr_planes[0], 1, 0, Mat(), b_hist, 1, &histSize, &histRange, uniform, accumulate ); - calcHist( &bgr_planes[1], 1, 0, Mat(), g_hist, 1, &histSize, &histRange, uniform, accumulate ); - calcHist( &bgr_planes[2], 1, 0, Mat(), r_hist, 1, &histSize, &histRange, uniform, accumulate ); + calcHist( &bgr_planes[0], 1, 0, Mat(), b_hist, 1, &histSize, histRange, uniform, accumulate ); + calcHist( &bgr_planes[1], 1, 0, Mat(), g_hist, 1, &histSize, histRange, uniform, accumulate ); + calcHist( &bgr_planes[2], 1, 0, Mat(), r_hist, 1, &histSize, histRange, uniform, accumulate ); //! [Compute the histograms] //! [Draw the histograms for B, G and R] From 6a3d925a47d54945eab7b50a769531703cde99a2 Mon Sep 17 00:00:00 2001 From: Joe Howse Date: Mon, 21 Jun 2021 00:46:32 -0300 Subject: [PATCH 025/376] OpenCL: core support for FP16, more channel orders * Support cl_image conversion for CL_HALF_FLOAT (float16) * Support cl_image conversion for additional channel orders: CL_A, CL_INTENSITY, CL_LUMINANCE, CL_RG, CL_RA * Comment on why cl_image conversion is unsupported for CL_RGB * Predict optimal vector width for float16 * ocl::kernelToStr: support float16 * ocl::Device::halfFPConfig: drop artificial requirement for OpenCL version >= 1.2. Even OpenCL 1.0 supports the underlying config property, CL_DEVICE_HALF_FP_CONFIG. * dumpOpenCLInformation: provide info on OpenCL half-float support and preferred half-float vector width * randu: support default range [-1.0, 1.0] for float16 * TestBase::warmup: support float16 --- .../opencv2/core/opencl/opencl_info.hpp | 7 +++ modules/core/src/ocl.cpp | 45 ++++++++++++++----- modules/ts/src/ocl_perf.cpp | 2 +- modules/ts/src/ts_perf.cpp | 2 +- 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/modules/core/include/opencv2/core/opencl/opencl_info.hpp b/modules/core/include/opencv2/core/opencl/opencl_info.hpp index 5e5c846ad059..3ead76e5c46e 100644 --- a/modules/core/include/opencv2/core/opencl/opencl_info.hpp +++ b/modules/core/include/opencv2/core/opencl/opencl_info.hpp @@ -144,6 +144,10 @@ static void dumpOpenCLInformation() DUMP_MESSAGE_STDOUT(" Double support = " << doubleSupportStr); DUMP_CONFIG_PROPERTY("cv_ocl_current_haveDoubleSupport", device.doubleFPConfig() > 0); + const char* halfSupportStr = device.halfFPConfig() > 0 ? "Yes" : "No"; + DUMP_MESSAGE_STDOUT(" Half support = " << halfSupportStr); + DUMP_CONFIG_PROPERTY("cv_ocl_current_haveHalfSupport", device.halfFPConfig() > 0); + const char* isUnifiedMemoryStr = device.hostUnifiedMemory() ? "Yes" : "No"; DUMP_MESSAGE_STDOUT(" Host unified memory = " << isUnifiedMemoryStr); DUMP_CONFIG_PROPERTY("cv_ocl_current_hostUnifiedMemory", device.hostUnifiedMemory()); @@ -191,6 +195,9 @@ static void dumpOpenCLInformation() DUMP_MESSAGE_STDOUT(" Preferred vector width double = " << device.preferredVectorWidthDouble()); DUMP_CONFIG_PROPERTY("cv_ocl_current_preferredVectorWidthDouble", device.preferredVectorWidthDouble()); + + DUMP_MESSAGE_STDOUT(" Preferred vector width half = " << device.preferredVectorWidthHalf()); + DUMP_CONFIG_PROPERTY("cv_ocl_current_preferredVectorWidthHalf", device.preferredVectorWidthHalf()); } catch (...) { diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index 0e97cf52feb3..46185446f726 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -1566,6 +1566,7 @@ struct Device::Impl version_ = getStrProp(CL_DEVICE_VERSION); extensions_ = getStrProp(CL_DEVICE_EXTENSIONS); doubleFPConfig_ = getProp(CL_DEVICE_DOUBLE_FP_CONFIG); + halfFPConfig_ = getProp(CL_DEVICE_HALF_FP_CONFIG); hostUnifiedMemory_ = getBoolProp(CL_DEVICE_HOST_UNIFIED_MEMORY); maxComputeUnits_ = getProp(CL_DEVICE_MAX_COMPUTE_UNITS); maxWorkGroupSize_ = getProp(CL_DEVICE_MAX_WORK_GROUP_SIZE); @@ -1678,6 +1679,7 @@ struct Device::Impl String version_; std::string extensions_; int doubleFPConfig_; + int halfFPConfig_; bool hostUnifiedMemory_; int maxComputeUnits_; size_t maxWorkGroupSize_; @@ -1827,11 +1829,7 @@ int Device::singleFPConfig() const { return p ? p->getProp(CL_DEVICE_SINGLE_FP_CONFIG) : 0; } int Device::halfFPConfig() const -#ifdef CL_VERSION_1_2 -{ return p ? p->getProp(CL_DEVICE_HALF_FP_CONFIG) : 0; } -#else -{ CV_REQUIRE_OPENCL_1_2_ERROR; } -#endif +{ return p ? p->halfFPConfig_ : 0; } bool Device::endianLittle() const { return p ? p->getBoolProp(CL_DEVICE_ENDIAN_LITTLE) : false; } @@ -6668,6 +6666,10 @@ void convertFromImage(void* cl_mem_image, UMat& dst) depth = CV_32F; break; + case CL_HALF_FLOAT: + depth = CV_16F; + break; + default: CV_Error(cv::Error::OpenCLApiCallError, "Not supported image_channel_data_type"); } @@ -6676,9 +6678,23 @@ void convertFromImage(void* cl_mem_image, UMat& dst) switch (fmt.image_channel_order) { case CL_R: + case CL_A: + case CL_INTENSITY: + case CL_LUMINANCE: type = CV_MAKE_TYPE(depth, 1); break; + case CL_RG: + case CL_RA: + type = CV_MAKE_TYPE(depth, 2); + break; + + // CL_RGB has no mappings to OpenCV types because CL_RGB can only be used with + // CL_UNORM_SHORT_565, CL_UNORM_SHORT_555, or CL_UNORM_INT_101010. + /*case CL_RGB: + type = CV_MAKE_TYPE(depth, 3); + break;*/ + case CL_RGBA: case CL_BGRA: case CL_ARGB: @@ -7068,6 +7084,13 @@ static std::string kerToStr(const Mat & k) stream << "DIG(" << data[i] << "f)"; stream << "DIG(" << data[width] << "f)"; } + else if (depth == CV_16F) + { + stream.setf(std::ios_base::showpoint); + for (int i = 0; i < width; ++i) + stream << "DIG(" << (float)data[i] << "h)"; + stream << "DIG(" << (float)data[width] << "h)"; + } else { for (int i = 0; i < width; ++i) @@ -7091,7 +7114,7 @@ String kernelToStr(InputArray _kernel, int ddepth, const char * name) typedef std::string (* func_t)(const Mat &); static const func_t funcs[] = { kerToStr, kerToStr, kerToStr, kerToStr, - kerToStr, kerToStr, kerToStr, 0 }; + kerToStr, kerToStr, kerToStr, kerToStr }; const func_t func = funcs[ddepth]; CV_Assert(func != 0); @@ -7130,14 +7153,14 @@ int predictOptimalVectorWidth(InputArray src1, InputArray src2, InputArray src3, int vectorWidths[] = { d.preferredVectorWidthChar(), d.preferredVectorWidthChar(), d.preferredVectorWidthShort(), d.preferredVectorWidthShort(), d.preferredVectorWidthInt(), d.preferredVectorWidthFloat(), - d.preferredVectorWidthDouble(), -1 }; + d.preferredVectorWidthDouble(), d.preferredVectorWidthHalf() }; // if the device says don't use vectors if (vectorWidths[0] == 1) { // it's heuristic vectorWidths[CV_8U] = vectorWidths[CV_8S] = 4; - vectorWidths[CV_16U] = vectorWidths[CV_16S] = 2; + vectorWidths[CV_16U] = vectorWidths[CV_16S] = vectorWidths[CV_16F] = 2; vectorWidths[CV_32S] = vectorWidths[CV_32F] = vectorWidths[CV_64F] = 1; } @@ -7225,10 +7248,12 @@ struct Image2D::Impl { cl_image_format format; static const int channelTypes[] = { CL_UNSIGNED_INT8, CL_SIGNED_INT8, CL_UNSIGNED_INT16, - CL_SIGNED_INT16, CL_SIGNED_INT32, CL_FLOAT, -1, -1 }; + CL_SIGNED_INT16, CL_SIGNED_INT32, CL_FLOAT, -1, CL_HALF_FLOAT }; static const int channelTypesNorm[] = { CL_UNORM_INT8, CL_SNORM_INT8, CL_UNORM_INT16, CL_SNORM_INT16, -1, -1, -1, -1 }; - static const int channelOrders[] = { -1, CL_R, CL_RG, -1, CL_RGBA }; + // CL_RGB has no mappings to OpenCV types because CL_RGB can only be used with + // CL_UNORM_SHORT_565, CL_UNORM_SHORT_555, or CL_UNORM_INT_101010. + static const int channelOrders[] = { -1, CL_R, CL_RG, /*CL_RGB*/ -1, CL_RGBA }; int channelType = norm ? channelTypesNorm[depth] : channelTypes[depth]; int channelOrder = channelOrders[cn]; diff --git a/modules/ts/src/ocl_perf.cpp b/modules/ts/src/ocl_perf.cpp index 8dacf219f64b..fe521f2c00d9 100644 --- a/modules/ts/src/ocl_perf.cpp +++ b/modules/ts/src/ocl_perf.cpp @@ -70,7 +70,7 @@ void randu(InputOutputArray dst) cv::randu(dst, -128, 128); else if (dst.depth() == CV_16U) cv::randu(dst, 0, 1024); - else if (dst.depth() == CV_32F || dst.depth() == CV_64F) + else if (dst.depth() == CV_32F || dst.depth() == CV_64F || dst.depth() == CV_16F) cv::randu(dst, -1.0, 1.0); else if (dst.depth() == CV_16S || dst.depth() == CV_32S) cv::randu(dst, -4096, 4096); diff --git a/modules/ts/src/ts_perf.cpp b/modules/ts/src/ts_perf.cpp index 2a9169fd13a5..5a42ca01cdc4 100644 --- a/modules/ts/src/ts_perf.cpp +++ b/modules/ts/src/ts_perf.cpp @@ -1297,7 +1297,7 @@ void TestBase::warmup(cv::InputOutputArray a, WarmUpType wtype) cv::randu(a, -128, 128); else if (depth == CV_16U) cv::randu(a, 0, 1024); - else if (depth == CV_32F || depth == CV_64F) + else if (depth == CV_32F || depth == CV_64F || depth == CV_16F) cv::randu(a, -1.0, 1.0); else if (depth == CV_16S || depth == CV_32S) cv::randu(a, -4096, 4096); From bf489feef11f365f8bd911bf4491d207d04ca46f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20H=20Tib=C3=A3es?= Date: Wed, 30 Jun 2021 19:08:24 -0300 Subject: [PATCH 026/376] Merge pull request #20327 from tibaes:MSMF-Slow-Webcam-Startup * fixes MSMF slow webcam startup * add variable to change MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS at runtime --- modules/videoio/src/cap_msmf.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/videoio/src/cap_msmf.cpp b/modules/videoio/src/cap_msmf.cpp index 73288c3d03b1..9e45fd1bacce 100644 --- a/modules/videoio/src/cap_msmf.cpp +++ b/modules/videoio/src/cap_msmf.cpp @@ -708,9 +708,10 @@ bool CvCapture_MSMF::initStream(DWORD streamID, const MediaType& mt) _ComPtr CvCapture_MSMF::getDefaultSourceConfig(UINT32 num) { CV_Assert(num > 0); + const bool OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS = utils::getConfigurationParameterBool("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", true); _ComPtr res; if (FAILED(MFCreateAttributes(&res, num)) || - FAILED(res->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, true)) || + FAILED(res->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS)) || FAILED(res->SetUINT32(MF_SOURCE_READER_DISABLE_DXVA, false)) || FAILED(res->SetUINT32(MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING, false)) || FAILED(res->SetUINT32(MF_SOURCE_READER_ENABLE_ADVANCED_VIDEO_PROCESSING, true)) From 6797fd65a5de608d30273ace911c5de14ac7693e Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 29 Jun 2021 20:25:22 +0000 Subject: [PATCH 027/376] dnn(test): update tests for OpenVINO 2021.4 --- modules/dnn/test/test_backends.cpp | 10 +++++----- modules/dnn/test/test_ie_models.cpp | 9 +++++++++ modules/dnn/test/test_torch_importer.cpp | 9 +++++++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp index b9958c107ebc..5426a11a3f51 100644 --- a/modules/dnn/test/test_backends.cpp +++ b/modules/dnn/test/test_backends.cpp @@ -196,7 +196,7 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe) Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false); float diffScores = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 1.5e-2 : 0.0; float diffSquares = (target == DNN_TARGET_MYRIAD) ? 0.063 : 0.0; - float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.252 : FLT_MIN; + float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.262 : FLT_MIN; processNet("dnn/MobileNetSSD_deploy.caffemodel", "dnn/MobileNetSSD_deploy.prototxt", inp, "detection_out", "", diffScores, diffSquares, detectionConfThresh); expectNoFallbacksFromIE(net); @@ -301,8 +301,8 @@ TEST_P(DNNTestNetwork, OpenPose_pose_coco) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif - const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.0056 : 0.0; - const float lInf = (target == DNN_TARGET_MYRIAD) ? 0.072 : 0.0; + const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.009 : 0.0; + const float lInf = (target == DNN_TARGET_MYRIAD) ? 0.09 : 0.0; processNet("dnn/openpose_pose_coco.caffemodel", "dnn/openpose_pose_coco.prototxt", Size(46, 46), "", "", l1, lInf); expectNoFallbacksFromIE(net); @@ -321,8 +321,8 @@ TEST_P(DNNTestNetwork, OpenPose_pose_mpi) #endif // output range: [-0.001, 0.97] - const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.012 : 0.0; - const float lInf = (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_OPENCL_FP16) ? 0.16 : 0.0; + const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.02 : 0.0; + const float lInf = (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_OPENCL_FP16) ? 0.2 : 0.0; processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi.prototxt", Size(46, 46), "", "", l1, lInf); expectNoFallbacksFromIE(net); diff --git a/modules/dnn/test/test_ie_models.cpp b/modules/dnn/test/test_ie_models.cpp index 2ba7d80f5865..da6cbd6fbc2f 100644 --- a/modules/dnn/test/test_ie_models.cpp +++ b/modules/dnn/test/test_ie_models.cpp @@ -288,6 +288,15 @@ TEST_P(DNNTestOpenVINO, models) ASSERT_FALSE(backendId != DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && backendId != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) << "Inference Engine backend is required"; +#if INF_ENGINE_VER_MAJOR_EQ(2021040000) + if (targetId == DNN_TARGET_MYRIAD && ( + modelName == "person-detection-retail-0013" || // ncDeviceOpen:1013 Failed to find booted device after boot + modelName == "age-gender-recognition-retail-0013" // ncDeviceOpen:1013 Failed to find booted device after boot + ) + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + #if INF_ENGINE_VER_MAJOR_GE(2020020000) if (targetId == DNN_TARGET_MYRIAD && backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) { diff --git a/modules/dnn/test/test_torch_importer.cpp b/modules/dnn/test/test_torch_importer.cpp index 8738e5e25cc4..7316a0685630 100644 --- a/modules/dnn/test/test_torch_importer.cpp +++ b/modules/dnn/test/test_torch_importer.cpp @@ -254,9 +254,14 @@ TEST_P(Test_Torch_layers, net_padding) TEST_P(Test_Torch_layers, net_non_spatial) { -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021030000) +#if defined(INF_ENGINE_RELEASE) && ( \ + INF_ENGINE_VER_MAJOR_EQ(2021030000) || \ + INF_ENGINE_VER_MAJOR_EQ(2021040000) \ +) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // crash + // 2021.3: crash + // 2021.4: [ GENERAL_ERROR ] AssertionFailed: !out.networkInputs.empty() + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // exception if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) From f2057ce1ab7d92f2deb4de29e7a81f401965a127 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 29 Jun 2021 21:55:18 +0000 Subject: [PATCH 028/376] dnn(ie): replace deprecated calls --- modules/dnn/src/ie_ngraph.cpp | 51 +++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/modules/dnn/src/ie_ngraph.cpp b/modules/dnn/src/ie_ngraph.cpp index 3b87243717a4..e6c219f13e5a 100644 --- a/modules/dnn/src/ie_ngraph.cpp +++ b/modules/dnn/src/ie_ngraph.cpp @@ -654,7 +654,11 @@ void InfEngineNgraphNet::initPlugin(InferenceEngine::CNNNetwork& net) try { InferenceEngine::IExtensionPtr extension = +#if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2021_4) + std::make_shared(libName); +#else InferenceEngine::make_so_pointer(libName); +#endif ie.AddExtension(extension, "CPU"); CV_LOG_INFO(NULL, "DNN-IE: Loaded extension plugin: " << libName); @@ -1002,35 +1006,54 @@ void InfEngineNgraphNet::forward(const std::vector >& outBlo reqWrapper->req.SetInput(inpBlobs); reqWrapper->req.SetOutput(outBlobs); +#if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2021_4) + InferenceEngine::InferRequest infRequest = reqWrapper->req; + NgraphReqWrapper* wrapperPtr = reqWrapper.get(); + CV_Assert(wrapperPtr && "Internal error"); +#else InferenceEngine::IInferRequest::Ptr infRequestPtr = reqWrapper->req; - infRequestPtr->SetUserData(reqWrapper.get(), 0); + CV_Assert(infRequestPtr); + InferenceEngine::IInferRequest& infRequest = *infRequestPtr.get(); + infRequest.SetUserData(reqWrapper.get(), 0); +#endif - infRequestPtr->SetCompletionCallback( - [](InferenceEngine::IInferRequest::Ptr request, InferenceEngine::StatusCode status) +#if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2021_4) + // do NOT capture 'reqWrapper' (smart ptr) in the lambda callback + infRequest.SetCompletionCallback>( + [wrapperPtr](InferenceEngine::InferRequest /*request*/, InferenceEngine::StatusCode status) +#else + infRequest.SetCompletionCallback( + [](InferenceEngine::IInferRequest::Ptr requestPtr, InferenceEngine::StatusCode status) +#endif { CV_LOG_DEBUG(NULL, "DNN(nGraph): completionCallback(" << (int)status << ")"); +#if !INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2021_4) + CV_Assert(requestPtr); + InferenceEngine::IInferRequest& request = *requestPtr.get(); - NgraphReqWrapper* wrapper; - request->GetUserData((void**)&wrapper, 0); - CV_Assert(wrapper && "Internal error"); + NgraphReqWrapper* wrapperPtr; + request.GetUserData((void**)&wrapperPtr, 0); + CV_Assert(wrapperPtr && "Internal error"); +#endif + NgraphReqWrapper& wrapper = *wrapperPtr; size_t processedOutputs = 0; try { - for (; processedOutputs < wrapper->outProms.size(); ++processedOutputs) + for (; processedOutputs < wrapper.outProms.size(); ++processedOutputs) { - const std::string& name = wrapper->outsNames[processedOutputs]; - Mat m = ngraphBlobToMat(wrapper->req.GetBlob(name)); + const std::string& name = wrapper.outsNames[processedOutputs]; + Mat m = ngraphBlobToMat(wrapper.req.GetBlob(name)); try { CV_Assert(status == InferenceEngine::StatusCode::OK); - wrapper->outProms[processedOutputs].setValue(m.clone()); + wrapper.outProms[processedOutputs].setValue(m.clone()); } catch (...) { try { - wrapper->outProms[processedOutputs].setException(std::current_exception()); + wrapper.outProms[processedOutputs].setException(std::current_exception()); } catch(...) { CV_LOG_ERROR(NULL, "DNN: Exception occurred during async inference exception propagation"); } @@ -1040,16 +1063,16 @@ void InfEngineNgraphNet::forward(const std::vector >& outBlo catch (...) { std::exception_ptr e = std::current_exception(); - for (; processedOutputs < wrapper->outProms.size(); ++processedOutputs) + for (; processedOutputs < wrapper.outProms.size(); ++processedOutputs) { try { - wrapper->outProms[processedOutputs].setException(e); + wrapper.outProms[processedOutputs].setException(e); } catch(...) { CV_LOG_ERROR(NULL, "DNN: Exception occurred during async inference exception propagation"); } } } - wrapper->isReady = true; + wrapper.isReady = true; } ); } From 5b8c10f2f85eb54fe3c945d1b5c8d0e4344ddbbd Mon Sep 17 00:00:00 2001 From: SamFC10 Date: Wed, 30 Jun 2021 21:55:42 +0530 Subject: [PATCH 029/376] modified onnx importer to concat const input blobs --- modules/dnn/src/onnx/onnx_importer.cpp | 17 +++++++++++++++++ modules/dnn/test/test_onnx_importer.cpp | 1 + 2 files changed, 18 insertions(+) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 651a2ab33344..3668c9b51e5d 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1792,6 +1792,23 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) addConstant(layerParams.name, concatenated[0]); return; } + else + { + for (int i = 0; i < node_proto.input_size(); ++i) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + { + LayerParams constParams; + constParams.name = node_proto.input(i); + constParams.type = "Const"; + constParams.blobs.push_back(getBlob(node_proto, i)); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + } + } + } } else if (layer_type == "Resize") { diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index c4cb87717200..600f727d7db4 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -327,6 +327,7 @@ TEST_P(Test_ONNX_layers, Concatenation) if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); } testONNXModels("concatenation"); + testONNXModels("concat_const_blobs"); } TEST_P(Test_ONNX_layers, Eltwise3D) From 9fe49497bb8b3e9c5b2043b4c74d3af495a66927 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Thu, 1 Jul 2021 12:36:19 +0300 Subject: [PATCH 030/376] Merge pull request #20284 from TolyaTalamanov:at/wrap-render G-API: Wrap render functionality to python * Wrap render Rect prim * Add all primitives and tests * Cover mosaic and image * Handle error in pyopencv_to(Prim) * Move Mosaic and Rect ctors wrappers to shadow file * Use GAPI_PROP_RW * Fix indent --- .../gapi/include/opencv2/gapi/own/exports.hpp | 2 + .../include/opencv2/gapi/render/render.hpp | 24 +- .../opencv2/gapi/render/render_types.hpp | 100 ++++---- .../gapi/misc/python/package/gapi/__init__.py | 45 +++- modules/gapi/misc/python/pyopencv_gapi.hpp | 116 ++++++--- modules/gapi/misc/python/python_bridge.hpp | 33 +-- modules/gapi/misc/python/shadow_gapi.hpp | 127 +++++----- .../gapi/misc/python/test/test_gapi_render.py | 227 ++++++++++++++++++ modules/gapi/src/api/render_ocv.cpp | 6 +- .../test/render/gapi_render_tests_ocv.cpp | 8 +- modules/python/src2/hdr_parser.py | 1 + 11 files changed, 518 insertions(+), 171 deletions(-) create mode 100644 modules/gapi/misc/python/test/test_gapi_render.py diff --git a/modules/gapi/include/opencv2/gapi/own/exports.hpp b/modules/gapi/include/opencv2/gapi/own/exports.hpp index 1978991b7518..c36f4003d0fb 100644 --- a/modules/gapi/include/opencv2/gapi/own/exports.hpp +++ b/modules/gapi/include/opencv2/gapi/own/exports.hpp @@ -13,11 +13,13 @@ # define GAPI_EXPORTS CV_EXPORTS /* special informative macros for wrapper generators */ # define GAPI_PROP CV_PROP +# define GAPI_PROP_RW CV_PROP_RW # define GAPI_WRAP CV_WRAP # define GAPI_EXPORTS_W_SIMPLE CV_EXPORTS_W_SIMPLE # define GAPI_EXPORTS_W CV_EXPORTS_W # else # define GAPI_PROP +# define GAPI_PROP_RW # define GAPI_WRAP # define GAPI_EXPORTS # define GAPI_EXPORTS_W_SIMPLE diff --git a/modules/gapi/include/opencv2/gapi/render/render.hpp b/modules/gapi/include/opencv2/gapi/render/render.hpp index 6bfe92388a12..537541222414 100644 --- a/modules/gapi/include/opencv2/gapi/render/render.hpp +++ b/modules/gapi/include/opencv2/gapi/render/render.hpp @@ -81,9 +81,9 @@ using GMatDesc2 = std::tuple; @param prims vector of drawing primitivies @param args graph compile time parameters */ -void GAPI_EXPORTS render(cv::Mat& bgr, - const Prims& prims, - cv::GCompileArgs&& args = {}); +void GAPI_EXPORTS_W render(cv::Mat& bgr, + const Prims& prims, + cv::GCompileArgs&& args = {}); /** @brief The function renders on two NV12 planes passed drawing primitivies @@ -92,10 +92,10 @@ void GAPI_EXPORTS render(cv::Mat& bgr, @param prims vector of drawing primitivies @param args graph compile time parameters */ -void GAPI_EXPORTS render(cv::Mat& y_plane, - cv::Mat& uv_plane, - const Prims& prims, - cv::GCompileArgs&& args = {}); +void GAPI_EXPORTS_W render(cv::Mat& y_plane, + cv::Mat& uv_plane, + const Prims& prims, + cv::GCompileArgs&& args = {}); /** @brief The function renders on the input media frame passed drawing primitivies @@ -139,7 +139,7 @@ Output image must be 8-bit unsigned planar 3-channel image @param src input image: 8-bit unsigned 3-channel image @ref CV_8UC3 @param prims draw primitives */ -GAPI_EXPORTS GMat render3ch(const GMat& src, const GArray& prims); +GAPI_EXPORTS_W GMat render3ch(const GMat& src, const GArray& prims); /** @brief Renders on two planes @@ -150,9 +150,9 @@ uv image must be 8-bit unsigned planar 2-channel image @ref CV_8UC2 @param uv input image: 8-bit unsigned 2-channel image @ref CV_8UC2 @param prims draw primitives */ -GAPI_EXPORTS GMat2 renderNV12(const GMat& y, - const GMat& uv, - const GArray& prims); +GAPI_EXPORTS_W GMat2 renderNV12(const GMat& y, + const GMat& uv, + const GArray& prims); /** @brief Renders Media Frame @@ -177,7 +177,7 @@ namespace render { namespace ocv { - GAPI_EXPORTS cv::gapi::GKernelPackage kernels(); + GAPI_EXPORTS_W cv::gapi::GKernelPackage kernels(); } // namespace ocv } // namespace render diff --git a/modules/gapi/include/opencv2/gapi/render/render_types.hpp b/modules/gapi/include/opencv2/gapi/render/render_types.hpp index ca403be361ee..6d70e3a877dd 100644 --- a/modules/gapi/include/opencv2/gapi/render/render_types.hpp +++ b/modules/gapi/include/opencv2/gapi/render/render_types.hpp @@ -41,7 +41,7 @@ struct freetype_font * * Parameters match cv::putText(). */ -struct Text +struct GAPI_EXPORTS_W_SIMPLE Text { /** * @brief Text constructor @@ -55,6 +55,7 @@ struct Text * @param lt_ The line type. See #LineTypes * @param bottom_left_origin_ When true, the image data origin is at the bottom-left corner. Otherwise, it is at the top-left corner */ + GAPI_WRAP Text(const std::string& text_, const cv::Point& org_, int ff_, @@ -68,17 +69,18 @@ struct Text { } + GAPI_WRAP Text() = default; /*@{*/ - std::string text; //!< The text string to be drawn - cv::Point org; //!< The bottom-left corner of the text string in the image - int ff; //!< The font type, see #HersheyFonts - double fs; //!< The font scale factor that is multiplied by the font-specific base size - cv::Scalar color; //!< The text color - int thick; //!< The thickness of the lines used to draw a text - int lt; //!< The line type. See #LineTypes - bool bottom_left_origin; //!< When true, the image data origin is at the bottom-left corner. Otherwise, it is at the top-left corner + GAPI_PROP_RW std::string text; //!< The text string to be drawn + GAPI_PROP_RW cv::Point org; //!< The bottom-left corner of the text string in the image + GAPI_PROP_RW int ff; //!< The font type, see #HersheyFonts + GAPI_PROP_RW double fs; //!< The font scale factor that is multiplied by the font-specific base size + GAPI_PROP_RW cv::Scalar color; //!< The text color + GAPI_PROP_RW int thick; //!< The thickness of the lines used to draw a text + GAPI_PROP_RW int lt; //!< The line type. See #LineTypes + GAPI_PROP_RW bool bottom_left_origin; //!< When true, the image data origin is at the bottom-left corner. Otherwise, it is at the top-left corner /*@{*/ }; @@ -122,7 +124,7 @@ struct FText * * Parameters match cv::rectangle(). */ -struct Rect +struct GAPI_EXPORTS_W_SIMPLE Rect { /** * @brief Rect constructor @@ -142,14 +144,15 @@ struct Rect { } + GAPI_WRAP Rect() = default; /*@{*/ - cv::Rect rect; //!< Coordinates of the rectangle - cv::Scalar color; //!< The rectangle color or brightness (grayscale image) - int thick; //!< The thickness of lines that make up the rectangle. Negative values, like #FILLED, mean that the function has to draw a filled rectangle - int lt; //!< The type of the line. See #LineTypes - int shift; //!< The number of fractional bits in the point coordinates + GAPI_PROP_RW cv::Rect rect; //!< Coordinates of the rectangle + GAPI_PROP_RW cv::Scalar color; //!< The rectangle color or brightness (grayscale image) + GAPI_PROP_RW int thick; //!< The thickness of lines that make up the rectangle. Negative values, like #FILLED, mean that the function has to draw a filled rectangle + GAPI_PROP_RW int lt; //!< The type of the line. See #LineTypes + GAPI_PROP_RW int shift; //!< The number of fractional bits in the point coordinates /*@{*/ }; @@ -158,7 +161,7 @@ struct Rect * * Parameters match cv::circle(). */ -struct Circle +struct GAPI_EXPORTS_W_SIMPLE Circle { /** * @brief Circle constructor @@ -170,6 +173,7 @@ struct Circle * @param lt_ The Type of the circle boundary. See #LineTypes * @param shift_ The Number of fractional bits in the coordinates of the center and in the radius value */ + GAPI_WRAP Circle(const cv::Point& center_, int radius_, const cv::Scalar& color_, @@ -180,15 +184,16 @@ struct Circle { } + GAPI_WRAP Circle() = default; /*@{*/ - cv::Point center; //!< The center of the circle - int radius; //!< The radius of the circle - cv::Scalar color; //!< The color of the circle - int thick; //!< The thickness of the circle outline, if positive. Negative values, like #FILLED, mean that a filled circle is to be drawn - int lt; //!< The Type of the circle boundary. See #LineTypes - int shift; //!< The Number of fractional bits in the coordinates of the center and in the radius value + GAPI_PROP_RW cv::Point center; //!< The center of the circle + GAPI_PROP_RW int radius; //!< The radius of the circle + GAPI_PROP_RW cv::Scalar color; //!< The color of the circle + GAPI_PROP_RW int thick; //!< The thickness of the circle outline, if positive. Negative values, like #FILLED, mean that a filled circle is to be drawn + GAPI_PROP_RW int lt; //!< The Type of the circle boundary. See #LineTypes + GAPI_PROP_RW int shift; //!< The Number of fractional bits in the coordinates of the center and in the radius value /*@{*/ }; @@ -197,7 +202,7 @@ struct Circle * * Parameters match cv::line(). */ -struct Line +struct GAPI_EXPORTS_W_SIMPLE Line { /** * @brief Line constructor @@ -209,6 +214,7 @@ struct Line * @param lt_ The Type of the line. See #LineTypes * @param shift_ The number of fractional bits in the point coordinates */ + GAPI_WRAP Line(const cv::Point& pt1_, const cv::Point& pt2_, const cv::Scalar& color_, @@ -219,15 +225,16 @@ struct Line { } + GAPI_WRAP Line() = default; /*@{*/ - cv::Point pt1; //!< The first point of the line segment - cv::Point pt2; //!< The second point of the line segment - cv::Scalar color; //!< The line color - int thick; //!< The thickness of line - int lt; //!< The Type of the line. See #LineTypes - int shift; //!< The number of fractional bits in the point coordinates + GAPI_PROP_RW cv::Point pt1; //!< The first point of the line segment + GAPI_PROP_RW cv::Point pt2; //!< The second point of the line segment + GAPI_PROP_RW cv::Scalar color; //!< The line color + GAPI_PROP_RW int thick; //!< The thickness of line + GAPI_PROP_RW int lt; //!< The Type of the line. See #LineTypes + GAPI_PROP_RW int shift; //!< The number of fractional bits in the point coordinates /*@{*/ }; @@ -236,7 +243,7 @@ struct Line * * Mosaicing is a very basic method to obfuscate regions in the image. */ -struct Mosaic +struct GAPI_EXPORTS_W_SIMPLE Mosaic { /** * @brief Mosaic constructor @@ -252,12 +259,13 @@ struct Mosaic { } + GAPI_WRAP Mosaic() : cellSz(0), decim(0) {} /*@{*/ - cv::Rect mos; //!< Coordinates of the mosaic - int cellSz; //!< Cell size (same for X, Y) - int decim; //!< Decimation (0 stands for no decimation) + GAPI_PROP_RW cv::Rect mos; //!< Coordinates of the mosaic + GAPI_PROP_RW int cellSz; //!< Cell size (same for X, Y) + GAPI_PROP_RW int decim; //!< Decimation (0 stands for no decimation) /*@{*/ }; @@ -266,7 +274,7 @@ struct Mosaic * * Image is blended on a frame using the specified mask. */ -struct Image +struct GAPI_EXPORTS_W_SIMPLE Image { /** * @brief Mosaic constructor @@ -275,6 +283,7 @@ struct Image * @param img_ Image to draw * @param alpha_ Alpha channel for image to draw (same size and number of channels) */ + GAPI_WRAP Image(const cv::Point& org_, const cv::Mat& img_, const cv::Mat& alpha_) : @@ -282,19 +291,20 @@ struct Image { } + GAPI_WRAP Image() = default; /*@{*/ - cv::Point org; //!< The bottom-left corner of the image - cv::Mat img; //!< Image to draw - cv::Mat alpha; //!< Alpha channel for image to draw (same size and number of channels) + GAPI_PROP_RW cv::Point org; //!< The bottom-left corner of the image + GAPI_PROP_RW cv::Mat img; //!< Image to draw + GAPI_PROP_RW cv::Mat alpha; //!< Alpha channel for image to draw (same size and number of channels) /*@{*/ }; /** * @brief This structure represents a polygon to draw. */ -struct Poly +struct GAPI_EXPORTS_W_SIMPLE Poly { /** * @brief Mosaic constructor @@ -305,6 +315,7 @@ struct Poly * @param lt_ The Type of the line. See #LineTypes * @param shift_ The number of fractional bits in the point coordinate */ + GAPI_WRAP Poly(const std::vector& points_, const cv::Scalar& color_, int thick_ = 1, @@ -314,14 +325,15 @@ struct Poly { } + GAPI_WRAP Poly() = default; /*@{*/ - std::vector points; //!< Points to connect - cv::Scalar color; //!< The line color - int thick; //!< The thickness of line - int lt; //!< The Type of the line. See #LineTypes - int shift; //!< The number of fractional bits in the point coordinate + GAPI_PROP_RW std::vector points; //!< Points to connect + GAPI_PROP_RW cv::Scalar color; //!< The line color + GAPI_PROP_RW int thick; //!< The thickness of line + GAPI_PROP_RW int lt; //!< The Type of the line. See #LineTypes + GAPI_PROP_RW int shift; //!< The number of fractional bits in the point coordinate /*@{*/ }; @@ -336,7 +348,7 @@ using Prim = util::variant , Poly >; -using Prims = std::vector; +using Prims = std::vector; //! @} gapi_draw_prims } // namespace draw diff --git a/modules/gapi/misc/python/package/gapi/__init__.py b/modules/gapi/misc/python/package/gapi/__init__.py index 587f641fd33a..dc874f0b0ca5 100644 --- a/modules/gapi/misc/python/package/gapi/__init__.py +++ b/modules/gapi/misc/python/package/gapi/__init__.py @@ -84,6 +84,10 @@ class Rect(): def __new__(self): return cv.GOpaqueT(cv.gapi.CV_RECT) + class Prim(): + def __new__(self): + return cv.GOpaqueT(cv.gapi.CV_DRAW_PRIM) + class Any(): def __new__(self): return cv.GOpaqueT(cv.gapi.CV_ANY) @@ -143,6 +147,10 @@ class GMat(): def __new__(self): return cv.GArrayT(cv.gapi.CV_GMAT) + class Prim(): + def __new__(self): + return cv.GArray(cv.gapi.CV_DRAW_PRIM) + class Any(): def __new__(self): return cv.GArray(cv.gapi.CV_ANY) @@ -164,6 +172,7 @@ def op(op_id, in_types, out_types): cv.GArray.Scalar: cv.gapi.CV_SCALAR, cv.GArray.Mat: cv.gapi.CV_MAT, cv.GArray.GMat: cv.gapi.CV_GMAT, + cv.GArray.Prim: cv.gapi.CV_DRAW_PRIM, cv.GArray.Any: cv.gapi.CV_ANY } @@ -179,22 +188,24 @@ def op(op_id, in_types, out_types): cv.GOpaque.Point2f: cv.gapi.CV_POINT2F, cv.GOpaque.Size: cv.gapi.CV_SIZE, cv.GOpaque.Rect: cv.gapi.CV_RECT, + cv.GOpaque.Prim: cv.gapi.CV_DRAW_PRIM, cv.GOpaque.Any: cv.gapi.CV_ANY } type2str = { - cv.gapi.CV_BOOL: 'cv.gapi.CV_BOOL' , - cv.gapi.CV_INT: 'cv.gapi.CV_INT' , - cv.gapi.CV_DOUBLE: 'cv.gapi.CV_DOUBLE' , - cv.gapi.CV_FLOAT: 'cv.gapi.CV_FLOAT' , - cv.gapi.CV_STRING: 'cv.gapi.CV_STRING' , - cv.gapi.CV_POINT: 'cv.gapi.CV_POINT' , - cv.gapi.CV_POINT2F: 'cv.gapi.CV_POINT2F' , - cv.gapi.CV_SIZE: 'cv.gapi.CV_SIZE', - cv.gapi.CV_RECT: 'cv.gapi.CV_RECT', - cv.gapi.CV_SCALAR: 'cv.gapi.CV_SCALAR', - cv.gapi.CV_MAT: 'cv.gapi.CV_MAT', - cv.gapi.CV_GMAT: 'cv.gapi.CV_GMAT' + cv.gapi.CV_BOOL: 'cv.gapi.CV_BOOL' , + cv.gapi.CV_INT: 'cv.gapi.CV_INT' , + cv.gapi.CV_DOUBLE: 'cv.gapi.CV_DOUBLE' , + cv.gapi.CV_FLOAT: 'cv.gapi.CV_FLOAT' , + cv.gapi.CV_STRING: 'cv.gapi.CV_STRING' , + cv.gapi.CV_POINT: 'cv.gapi.CV_POINT' , + cv.gapi.CV_POINT2F: 'cv.gapi.CV_POINT2F' , + cv.gapi.CV_SIZE: 'cv.gapi.CV_SIZE', + cv.gapi.CV_RECT: 'cv.gapi.CV_RECT', + cv.gapi.CV_SCALAR: 'cv.gapi.CV_SCALAR', + cv.gapi.CV_MAT: 'cv.gapi.CV_MAT', + cv.gapi.CV_GMAT: 'cv.gapi.CV_GMAT', + cv.gapi.CV_DRAW_PRIM: 'cv.gapi.CV_DRAW_PRIM' } # NB: Second lvl decorator takes class to decorate @@ -274,3 +285,13 @@ def kernel_with_params(cls): return cls return kernel_with_params + + +# FIXME: On the c++ side every class is placed in cv2 module. +cv.gapi.wip.draw.Rect = cv.gapi_wip_draw_Rect +cv.gapi.wip.draw.Text = cv.gapi_wip_draw_Text +cv.gapi.wip.draw.Circle = cv.gapi_wip_draw_Circle +cv.gapi.wip.draw.Line = cv.gapi_wip_draw_Line +cv.gapi.wip.draw.Mosaic = cv.gapi_wip_draw_Mosaic +cv.gapi.wip.draw.Image = cv.gapi_wip_draw_Image +cv.gapi.wip.draw.Poly = cv.gapi_wip_draw_Poly diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index 6cd79e4a7318..3c428dde6d82 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -43,6 +43,7 @@ using GArray_Rect = cv::GArray; using GArray_Scalar = cv::GArray; using GArray_Mat = cv::GArray; using GArray_GMat = cv::GArray; +using GArray_Prim = cv::GArray; // FIXME: Python wrapper generate code without namespace std, // so it cause error: "string wasn't declared" @@ -125,6 +126,65 @@ PyObject* pyopencv_from(const cv::detail::PyObjectHolder& v) return o; } +// #FIXME: Is it possible to implement pyopencv_from/pyopencv_to for generic +// cv::variant ? +template <> +PyObject* pyopencv_from(const cv::gapi::wip::draw::Prim& prim) +{ + switch (prim.index()) { + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + } + + util::throw_error(std::logic_error("Unsupported draw primitive type")); +} + +template <> +PyObject* pyopencv_from(const cv::gapi::wip::draw::Prims& value) +{ + return pyopencv_from_generic_vec(value); +} + +template<> +bool pyopencv_to(PyObject* obj, cv::gapi::wip::draw::Prim& value, const ArgInfo& info) +{ +#define TRY_EXTRACT(Prim) \ + if (PyObject_TypeCheck(obj, reinterpret_cast(pyopencv_gapi_wip_draw_##Prim##_TypePtr))) \ + { \ + value = reinterpret_cast(obj)->v; \ + return true; \ + } \ + + TRY_EXTRACT(Rect) + TRY_EXTRACT(Text) + TRY_EXTRACT(Circle) + TRY_EXTRACT(Line) + TRY_EXTRACT(Mosaic) + TRY_EXTRACT(Image) + TRY_EXTRACT(Poly) + + failmsg("Unsupported primitive type"); + return false; +} + +template <> +bool pyopencv_to(PyObject* obj, cv::gapi::wip::draw::Prims& value, const ArgInfo& info) +{ + return pyopencv_to_generic_vec(obj, value, info); +} + template<> PyObject* pyopencv_from(const cv::GArg& value) { @@ -137,21 +197,21 @@ PyObject* pyopencv_from(const cv::GArg& value) #define UNSUPPORTED(T) case cv::detail::OpaqueKind::CV_##T: break switch (value.opaque_kind) { - HANDLE_CASE(BOOL, bool); - HANDLE_CASE(INT, int); + HANDLE_CASE(BOOL, bool); + HANDLE_CASE(INT, int); HANDLE_CASE(INT64, int64_t); - HANDLE_CASE(DOUBLE, double); - HANDLE_CASE(FLOAT, float); - HANDLE_CASE(STRING, std::string); - HANDLE_CASE(POINT, cv::Point); - HANDLE_CASE(POINT2F, cv::Point2f); - HANDLE_CASE(SIZE, cv::Size); - HANDLE_CASE(RECT, cv::Rect); - HANDLE_CASE(SCALAR, cv::Scalar); - HANDLE_CASE(MAT, cv::Mat); - HANDLE_CASE(UNKNOWN, cv::detail::PyObjectHolder); + HANDLE_CASE(DOUBLE, double); + HANDLE_CASE(FLOAT, float); + HANDLE_CASE(STRING, std::string); + HANDLE_CASE(POINT, cv::Point); + HANDLE_CASE(POINT2F, cv::Point2f); + HANDLE_CASE(SIZE, cv::Size); + HANDLE_CASE(RECT, cv::Rect); + HANDLE_CASE(SCALAR, cv::Scalar); + HANDLE_CASE(MAT, cv::Mat); + HANDLE_CASE(UNKNOWN, cv::detail::PyObjectHolder); + HANDLE_CASE(DRAW_PRIM, cv::gapi::wip::draw::Prim); UNSUPPORTED(UINT64); - UNSUPPORTED(DRAW_PRIM); #undef HANDLE_CASE #undef UNSUPPORTED } @@ -205,10 +265,10 @@ PyObject* pyopencv_from(const cv::detail::OpaqueRef& o) case cv::detail::OpaqueKind::CV_SIZE : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_RECT : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_UNKNOWN : return pyopencv_from(o.rref()); + case cv::detail::OpaqueKind::CV_DRAW_PRIM : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_UINT64 : break; case cv::detail::OpaqueKind::CV_SCALAR : break; case cv::detail::OpaqueKind::CV_MAT : break; - case cv::detail::OpaqueKind::CV_DRAW_PRIM : break; } PyErr_SetString(PyExc_TypeError, "Unsupported GOpaque type"); @@ -233,8 +293,8 @@ PyObject* pyopencv_from(const cv::detail::VectorRef& v) case cv::detail::OpaqueKind::CV_SCALAR : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_MAT : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_UNKNOWN : return pyopencv_from_generic_vec(v.rref()); + case cv::detail::OpaqueKind::CV_DRAW_PRIM : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_UINT64 : break; - case cv::detail::OpaqueKind::CV_DRAW_PRIM : break; } PyErr_SetString(PyExc_TypeError, "Unsupported GArray type"); @@ -394,21 +454,21 @@ static cv::detail::VectorRef extract_vector_ref(PyObject* from, cv::detail::Opaq #define UNSUPPORTED(T) case cv::detail::OpaqueKind::CV_##T: break switch (kind) { - HANDLE_CASE(BOOL, bool); - HANDLE_CASE(INT, int); - HANDLE_CASE(DOUBLE, double); - HANDLE_CASE(FLOAT, float); - HANDLE_CASE(STRING, std::string); - HANDLE_CASE(POINT, cv::Point); - HANDLE_CASE(POINT2F, cv::Point2f); - HANDLE_CASE(SIZE, cv::Size); - HANDLE_CASE(RECT, cv::Rect); - HANDLE_CASE(SCALAR, cv::Scalar); - HANDLE_CASE(MAT, cv::Mat); - HANDLE_CASE(UNKNOWN, cv::GArg); + HANDLE_CASE(BOOL, bool); + HANDLE_CASE(INT, int); + HANDLE_CASE(DOUBLE, double); + HANDLE_CASE(FLOAT, float); + HANDLE_CASE(STRING, std::string); + HANDLE_CASE(POINT, cv::Point); + HANDLE_CASE(POINT2F, cv::Point2f); + HANDLE_CASE(SIZE, cv::Size); + HANDLE_CASE(RECT, cv::Rect); + HANDLE_CASE(SCALAR, cv::Scalar); + HANDLE_CASE(MAT, cv::Mat); + HANDLE_CASE(UNKNOWN, cv::GArg); + HANDLE_CASE(DRAW_PRIM, cv::gapi::wip::draw::Prim); UNSUPPORTED(UINT64); UNSUPPORTED(INT64); - UNSUPPORTED(DRAW_PRIM); #undef HANDLE_CASE #undef UNSUPPORTED } diff --git a/modules/gapi/misc/python/python_bridge.hpp b/modules/gapi/misc/python/python_bridge.hpp index b212babe4599..11d17287308e 100644 --- a/modules/gapi/misc/python/python_bridge.hpp +++ b/modules/gapi/misc/python/python_bridge.hpp @@ -10,6 +10,7 @@ #include #include #include +#include // Prim #define ID(T, E) T #define ID_(T, E) ID(T, E), @@ -24,21 +25,24 @@ GAPI_Assert(false && "Unsupported type"); \ } +using cv::gapi::wip::draw::Prim; + #define GARRAY_TYPE_LIST_G(G, G2) \ -WRAP_ARGS(bool , cv::gapi::ArgType::CV_BOOL, G) \ -WRAP_ARGS(int , cv::gapi::ArgType::CV_INT, G) \ -WRAP_ARGS(int64_t , cv::gapi::ArgType::CV_INT64, G) \ -WRAP_ARGS(double , cv::gapi::ArgType::CV_DOUBLE, G) \ -WRAP_ARGS(float , cv::gapi::ArgType::CV_FLOAT, G) \ -WRAP_ARGS(std::string , cv::gapi::ArgType::CV_STRING, G) \ -WRAP_ARGS(cv::Point , cv::gapi::ArgType::CV_POINT, G) \ -WRAP_ARGS(cv::Point2f , cv::gapi::ArgType::CV_POINT2F, G) \ -WRAP_ARGS(cv::Size , cv::gapi::ArgType::CV_SIZE, G) \ -WRAP_ARGS(cv::Rect , cv::gapi::ArgType::CV_RECT, G) \ -WRAP_ARGS(cv::Scalar , cv::gapi::ArgType::CV_SCALAR, G) \ -WRAP_ARGS(cv::Mat , cv::gapi::ArgType::CV_MAT, G) \ -WRAP_ARGS(cv::GArg , cv::gapi::ArgType::CV_ANY, G) \ -WRAP_ARGS(cv::GMat , cv::gapi::ArgType::CV_GMAT, G2) \ +WRAP_ARGS(bool , cv::gapi::ArgType::CV_BOOL, G) \ +WRAP_ARGS(int , cv::gapi::ArgType::CV_INT, G) \ +WRAP_ARGS(int64_t , cv::gapi::ArgType::CV_INT64, G) \ +WRAP_ARGS(double , cv::gapi::ArgType::CV_DOUBLE, G) \ +WRAP_ARGS(float , cv::gapi::ArgType::CV_FLOAT, G) \ +WRAP_ARGS(std::string , cv::gapi::ArgType::CV_STRING, G) \ +WRAP_ARGS(cv::Point , cv::gapi::ArgType::CV_POINT, G) \ +WRAP_ARGS(cv::Point2f , cv::gapi::ArgType::CV_POINT2F, G) \ +WRAP_ARGS(cv::Size , cv::gapi::ArgType::CV_SIZE, G) \ +WRAP_ARGS(cv::Rect , cv::gapi::ArgType::CV_RECT, G) \ +WRAP_ARGS(cv::Scalar , cv::gapi::ArgType::CV_SCALAR, G) \ +WRAP_ARGS(cv::Mat , cv::gapi::ArgType::CV_MAT, G) \ +WRAP_ARGS(Prim , cv::gapi::ArgType::CV_DRAW_PRIM, G) \ +WRAP_ARGS(cv::GArg , cv::gapi::ArgType::CV_ANY, G) \ +WRAP_ARGS(cv::GMat , cv::gapi::ArgType::CV_GMAT, G2) \ #define GOPAQUE_TYPE_LIST_G(G, G2) \ WRAP_ARGS(bool , cv::gapi::ArgType::CV_BOOL, G) \ @@ -71,6 +75,7 @@ enum ArgType { CV_SCALAR, CV_MAT, CV_GMAT, + CV_DRAW_PRIM, CV_ANY, }; diff --git a/modules/gapi/misc/python/shadow_gapi.hpp b/modules/gapi/misc/python/shadow_gapi.hpp index e777aa5d934b..41d0f1973223 100644 --- a/modules/gapi/misc/python/shadow_gapi.hpp +++ b/modules/gapi/misc/python/shadow_gapi.hpp @@ -3,58 +3,77 @@ namespace cv { - struct GAPI_EXPORTS_W_SIMPLE GCompileArg { - GAPI_WRAP GCompileArg(gapi::GKernelPackage pkg); - GAPI_WRAP GCompileArg(gapi::GNetPackage pkg); - }; - - class GAPI_EXPORTS_W_SIMPLE GInferInputs - { - public: - GAPI_WRAP GInferInputs(); - GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GMat& value); - GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GFrame& value); - }; - - class GAPI_EXPORTS_W_SIMPLE GInferListInputs - { - public: - GAPI_WRAP GInferListInputs(); - GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); - GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); - }; - - class GAPI_EXPORTS_W_SIMPLE GInferOutputs - { - public: - GAPI_WRAP GInferOutputs(); - GAPI_WRAP cv::GMat at(const std::string& name); - }; - - class GAPI_EXPORTS_W_SIMPLE GInferListOutputs - { - public: - GAPI_WRAP GInferListOutputs(); - GAPI_WRAP cv::GArray at(const std::string& name); - }; - - namespace detail - { - gapi::GNetParam GAPI_EXPORTS_W strip(gapi::ie::PyParams params); - } // namespace detail - - namespace gapi - { - namespace streaming - { - // FIXME: Extend to work with an arbitrary G-type. - cv::GOpaque GAPI_EXPORTS_W timestamp(cv::GMat); - cv::GOpaque GAPI_EXPORTS_W seqNo(cv::GMat); - cv::GOpaque GAPI_EXPORTS_W seq_id(cv::GMat); - } // namespace streaming - namespace wip - { - class GAPI_EXPORTS_W IStreamSource { }; - } // namespace wip - } // namespace gapi +struct GAPI_EXPORTS_W_SIMPLE GCompileArg { + GAPI_WRAP GCompileArg(gapi::GKernelPackage pkg); + GAPI_WRAP GCompileArg(gapi::GNetPackage pkg); +}; + +class GAPI_EXPORTS_W_SIMPLE GInferInputs +{ +public: + GAPI_WRAP GInferInputs(); + GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GMat& value); + GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GFrame& value); +}; + +class GAPI_EXPORTS_W_SIMPLE GInferListInputs +{ +public: + GAPI_WRAP GInferListInputs(); + GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); + GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); +}; + +class GAPI_EXPORTS_W_SIMPLE GInferOutputs +{ +public: + GAPI_WRAP GInferOutputs(); + GAPI_WRAP cv::GMat at(const std::string& name); +}; + +class GAPI_EXPORTS_W_SIMPLE GInferListOutputs +{ +public: + GAPI_WRAP GInferListOutputs(); + GAPI_WRAP cv::GArray at(const std::string& name); +}; + +namespace gapi +{ +namespace wip +{ +class GAPI_EXPORTS_W IStreamSource { }; +namespace draw +{ + // NB: These render primitives are partially wrapped in shadow file + // because cv::Rect conflicts with cv::gapi::wip::draw::Rect in python generator + // and cv::Rect2i breaks standalone mode. + struct Rect + { + GAPI_WRAP Rect(const cv::Rect2i& rect_, + const cv::Scalar& color_, + int thick_ = 1, + int lt_ = 8, + int shift_ = 0); + }; + + struct Mosaic + { + GAPI_WRAP Mosaic(const cv::Rect2i& mos_, int cellSz_, int decim_); + }; +} // namespace draw +} // namespace wip +namespace streaming +{ + // FIXME: Extend to work with an arbitrary G-type. + cv::GOpaque GAPI_EXPORTS_W timestamp(cv::GMat); + cv::GOpaque GAPI_EXPORTS_W seqNo(cv::GMat); + cv::GOpaque GAPI_EXPORTS_W seq_id(cv::GMat); +} // namespace streaming +} // namespace gapi + +namespace detail +{ + gapi::GNetParam GAPI_EXPORTS_W strip(gapi::ie::PyParams params); +} // namespace detail } // namespace cv diff --git a/modules/gapi/misc/python/test/test_gapi_render.py b/modules/gapi/misc/python/test/test_gapi_render.py new file mode 100644 index 000000000000..70601a72e57d --- /dev/null +++ b/modules/gapi/misc/python/test/test_gapi_render.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python + +import numpy as np +import cv2 as cv +import os +import sys +import unittest + +from tests_common import NewOpenCVTests + +try: + + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') + + # FIXME: FText isn't supported yet. + class gapi_render_test(NewOpenCVTests): + def __init__(self, *args): + super().__init__(*args) + + self.size = (300, 300, 3) + + # Rect + self.rect = (30, 30, 50, 50) + self.rcolor = (0, 255, 0) + self.rlt = cv.LINE_4 + self.rthick = 2 + self.rshift = 3 + + # Text + self.text = 'Hello, world!' + self.org = (100, 100) + self.ff = cv.FONT_HERSHEY_SIMPLEX + self.fs = 1.0 + self.tthick = 2 + self.tlt = cv.LINE_8 + self.tcolor = (255, 255, 255) + self.blo = False + + # Circle + self.center = (200, 200) + self.radius = 200 + self.ccolor = (255, 255, 0) + self.cthick = 2 + self.clt = cv.LINE_4 + self.cshift = 1 + + # Line + self.pt1 = (50, 50) + self.pt2 = (200, 200) + self.lcolor = (0, 255, 128) + self.lthick = 5 + self.llt = cv.LINE_8 + self.lshift = 2 + + # Poly + self.pts = [(50, 100), (100, 200), (25, 250)] + self.pcolor = (0, 0, 255) + self.pthick = 3 + self.plt = cv.LINE_4 + self.pshift = 1 + + # Image + self.iorg = (150, 150) + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + self.img = cv.resize(cv.imread(img_path), (50, 50)) + self.alpha = np.full(self.img.shape[:2], 0.8, dtype=np.float32) + + # Mosaic + self.mos = (100, 100, 100, 100) + self.cell_sz = 25 + self.decim = 0 + + # Render primitives + self.prims = [cv.gapi.wip.draw.Rect(self.rect, self.rcolor, self.rthick, self.rlt, self.rshift), + cv.gapi.wip.draw.Text(self.text, self.org, self.ff, self.fs, self.tcolor, self.tthick, self.tlt, self.blo), + cv.gapi.wip.draw.Circle(self.center, self.radius, self.ccolor, self.cthick, self.clt, self.cshift), + cv.gapi.wip.draw.Line(self.pt1, self.pt2, self.lcolor, self.lthick, self.llt, self.lshift), + cv.gapi.wip.draw.Mosaic(self.mos, self.cell_sz, self.decim), + cv.gapi.wip.draw.Image(self.iorg, self.img, self.alpha), + cv.gapi.wip.draw.Poly(self.pts, self.pcolor, self.pthick, self.plt, self.pshift)] + + def cvt_nv12_to_yuv(self, y, uv): + h,w,_ = uv.shape + upsample_uv = cv.resize(uv, (h * 2, w * 2)) + return cv.merge([y, upsample_uv]) + + def cvt_yuv_to_nv12(self, yuv, y_out, uv_out): + chs = cv.split(yuv, [y_out, None, None]) + uv = cv.merge([chs[1], chs[2]]) + uv_out = cv.resize(uv, (uv.shape[0] // 2, uv.shape[1] // 2), dst=uv_out) + return y_out, uv_out + + def cvt_bgr_to_yuv_color(self, bgr): + y = bgr[2] * 0.299000 + bgr[1] * 0.587000 + bgr[0] * 0.114000; + u = bgr[2] * -0.168736 + bgr[1] * -0.331264 + bgr[0] * 0.500000 + 128; + v = bgr[2] * 0.500000 + bgr[1] * -0.418688 + bgr[0] * -0.081312 + 128; + return (y, u, v) + + def blend_img(self, background, org, img, alpha): + x, y = org + h, w, _ = img.shape + roi_img = background[x:x+w, y:y+h, :] + img32f_w = cv.merge([alpha] * 3).astype(np.float32) + roi32f_w = np.full(roi_img.shape, 1.0, dtype=np.float32) + roi32f_w -= img32f_w + img32f = (img / 255).astype(np.float32) + roi32f = (roi_img / 255).astype(np.float32) + cv.multiply(img32f, img32f_w, dst=img32f) + cv.multiply(roi32f, roi32f_w, dst=roi32f) + roi32f += img32f + roi_img[...] = np.round(roi32f * 255) + + # This is quite naive implementations used as a simple reference + # doesn't consider corner cases. + def draw_mosaic(self, img, mos, cell_sz, decim): + x,y,w,h = mos + mosaic_area = img[x:x+w, y:y+h, :] + for i in range(0, mosaic_area.shape[0], cell_sz): + for j in range(0, mosaic_area.shape[1], cell_sz): + cell_roi = mosaic_area[j:j+cell_sz, i:i+cell_sz, :] + s0, s1, s2 = cv.mean(cell_roi)[:3] + mosaic_area[j:j+cell_sz, i:i+cell_sz] = (round(s0), round(s1), round(s2)) + + def render_primitives_bgr_ref(self, img): + cv.rectangle(img, self.rect, self.rcolor, self.rthick, self.rlt, self.rshift) + cv.putText(img, self.text, self.org, self.ff, self.fs, self.tcolor, self.tthick, self.tlt, self.blo) + cv.circle(img, self.center, self.radius, self.ccolor, self.cthick, self.clt, self.cshift) + cv.line(img, self.pt1, self.pt2, self.lcolor, self.lthick, self.llt, self.lshift) + cv.fillPoly(img, np.expand_dims(np.array([self.pts]), axis=0), self.pcolor, self.plt, self.pshift) + self.draw_mosaic(img, self.mos, self.cell_sz, self.decim) + self.blend_img(img, self.iorg, self.img, self.alpha) + + def render_primitives_nv12_ref(self, y_plane, uv_plane): + yuv = self.cvt_nv12_to_yuv(y_plane, uv_plane) + cv.rectangle(yuv, self.rect, self.cvt_bgr_to_yuv_color(self.rcolor), self.rthick, self.rlt, self.rshift) + cv.putText(yuv, self.text, self.org, self.ff, self.fs, self.cvt_bgr_to_yuv_color(self.tcolor), self.tthick, self.tlt, self.blo) + cv.circle(yuv, self.center, self.radius, self.cvt_bgr_to_yuv_color(self.ccolor), self.cthick, self.clt, self.cshift) + cv.line(yuv, self.pt1, self.pt2, self.cvt_bgr_to_yuv_color(self.lcolor), self.lthick, self.llt, self.lshift) + cv.fillPoly(yuv, np.expand_dims(np.array([self.pts]), axis=0), self.cvt_bgr_to_yuv_color(self.pcolor), self.plt, self.pshift) + self.draw_mosaic(yuv, self.mos, self.cell_sz, self.decim) + self.blend_img(yuv, self.iorg, cv.cvtColor(self.img, cv.COLOR_BGR2YUV), self.alpha) + self.cvt_yuv_to_nv12(yuv, y_plane, uv_plane) + + def test_render_primitives_on_bgr_graph(self): + expected = np.zeros(self.size, dtype=np.uint8) + actual = np.array(expected, copy=True) + + # OpenCV + self.render_primitives_bgr_ref(expected) + + # G-API + g_in = cv.GMat() + g_prims = cv.GArray.Prim() + g_out = cv.gapi.wip.draw.render3ch(g_in, g_prims) + + + comp = cv.GComputation(cv.GIn(g_in, g_prims), cv.GOut(g_out)) + actual = comp.apply(cv.gin(actual, self.prims)) + + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + + def test_render_primitives_on_bgr_function(self): + expected = np.zeros(self.size, dtype=np.uint8) + actual = np.array(expected, copy=True) + + # OpenCV + self.render_primitives_bgr_ref(expected) + + # G-API + cv.gapi.wip.draw.render(actual, self.prims) + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + + def test_render_primitives_on_nv12_graph(self): + y_expected = np.zeros((self.size[0], self.size[1], 1), dtype=np.uint8) + uv_expected = np.zeros((self.size[0] // 2, self.size[1] // 2, 2), dtype=np.uint8) + + y_actual = np.array(y_expected, copy=True) + uv_actual = np.array(uv_expected, copy=True) + + # OpenCV + self.render_primitives_nv12_ref(y_expected, uv_expected) + + # G-API + g_y = cv.GMat() + g_uv = cv.GMat() + g_prims = cv.GArray.Prim() + g_out_y, g_out_uv = cv.gapi.wip.draw.renderNV12(g_y, g_uv, g_prims) + + comp = cv.GComputation(cv.GIn(g_y, g_uv, g_prims), cv.GOut(g_out_y, g_out_uv)) + y_actual, uv_actual = comp.apply(cv.gin(y_actual, uv_actual, self.prims)) + + self.assertEqual(0.0, cv.norm(y_expected, y_actual, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(uv_expected, uv_actual, cv.NORM_INF)) + + def test_render_primitives_on_nv12_function(self): + y_expected = np.zeros((self.size[0], self.size[1], 1), dtype=np.uint8) + uv_expected = np.zeros((self.size[0] // 2, self.size[1] // 2, 2), dtype=np.uint8) + + y_actual = np.array(y_expected, copy=True) + uv_actual = np.array(uv_expected, copy=True) + + # OpenCV + self.render_primitives_nv12_ref(y_expected, uv_expected) + + # G-API + cv.gapi.wip.draw.render(y_actual, uv_actual, self.prims) + + self.assertEqual(0.0, cv.norm(y_expected, y_actual, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(uv_expected, uv_actual, cv.NORM_INF)) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/modules/gapi/src/api/render_ocv.cpp b/modules/gapi/src/api/render_ocv.cpp index 5ab2e1dd07c2..f1e9be4b4893 100644 --- a/modules/gapi/src/api/render_ocv.cpp +++ b/modules/gapi/src/api/render_ocv.cpp @@ -159,7 +159,7 @@ void drawPrimitivesOCV(cv::Mat& in, { const auto& rp = cv::util::get(p); const auto color = converter.cvtColor(rp.color); - cv::rectangle(in, rp.rect, color , rp.thick); + cv::rectangle(in, rp.rect, color, rp.thick, rp.lt, rp.shift); break; } @@ -198,7 +198,7 @@ void drawPrimitivesOCV(cv::Mat& in, { const auto& cp = cv::util::get(p); const auto color = converter.cvtColor(cp.color); - cv::circle(in, cp.center, cp.radius, color, cp.thick); + cv::circle(in, cp.center, cp.radius, color, cp.thick, cp.lt, cp.shift); break; } @@ -206,7 +206,7 @@ void drawPrimitivesOCV(cv::Mat& in, { const auto& lp = cv::util::get(p); const auto color = converter.cvtColor(lp.color); - cv::line(in, lp.pt1, lp.pt2, color, lp.thick); + cv::line(in, lp.pt1, lp.pt2, color, lp.thick, lp.lt, lp.shift); break; } diff --git a/modules/gapi/test/render/gapi_render_tests_ocv.cpp b/modules/gapi/test/render/gapi_render_tests_ocv.cpp index 010df5dff75b..95f695415609 100644 --- a/modules/gapi/test/render/gapi_render_tests_ocv.cpp +++ b/modules/gapi/test/render/gapi_render_tests_ocv.cpp @@ -639,8 +639,8 @@ INSTANTIATE_TEST_CASE_P(RenderBGROCVTestRectsImpl, RenderBGROCVTestRects, Values(cv::Rect(100, 100, 200, 200)), Values(cv::Scalar(100, 50, 150)), Values(2), - Values(LINE_8), - Values(0))); + Values(LINE_8, LINE_4), + Values(0, 1))); INSTANTIATE_TEST_CASE_P(RenderNV12OCVTestRectsImpl, RenderNV12OCVTestRects, Combine(Values(cv::Size(1280, 720)), @@ -673,8 +673,8 @@ INSTANTIATE_TEST_CASE_P(RenderNV12OCVTestCirclesImpl, RenderNV12OCVTestCircles, Values(10), Values(cv::Scalar(100, 50, 150)), Values(2), - Values(LINE_8), - Values(0))); + Values(LINE_8, LINE_4), + Values(0, 1))); INSTANTIATE_TEST_CASE_P(RenderMFrameOCVTestCirclesImpl, RenderMFrameOCVTestCircles, Combine(Values(cv::Size(1280, 720)), diff --git a/modules/python/src2/hdr_parser.py b/modules/python/src2/hdr_parser.py index 412d41a4df3f..1e0f9b3a954d 100755 --- a/modules/python/src2/hdr_parser.py +++ b/modules/python/src2/hdr_parser.py @@ -832,6 +832,7 @@ def parse(self, hname, wmode=True): ("GAPI_EXPORTS_W_SIMPLE","CV_EXPORTS_W_SIMPLE"), ("GAPI_WRAP", "CV_WRAP"), ("GAPI_PROP", "CV_PROP"), + ("GAPI_PROP_RW", "CV_PROP_RW"), ('defined(GAPI_STANDALONE)', '0'), ]) From d70053aba528ec5ddf8d6906109c7cc9310654a5 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Thu, 1 Jul 2021 13:27:28 +0300 Subject: [PATCH 031/376] Merge pull request #20144 from mpashchenkov:mp/python-ge G-API: Python. Gaze Estimation sample. * GE pep8 * Added function description, wrapped copy * Applying review comments * One more change * Added gin * Rstrt bb --- .../misc/python/samples/gaze_estimation.py | 467 ++++++++++++++++++ 1 file changed, 467 insertions(+) create mode 100644 modules/gapi/misc/python/samples/gaze_estimation.py diff --git a/modules/gapi/misc/python/samples/gaze_estimation.py b/modules/gapi/misc/python/samples/gaze_estimation.py new file mode 100644 index 000000000000..db190f67bb99 --- /dev/null +++ b/modules/gapi/misc/python/samples/gaze_estimation.py @@ -0,0 +1,467 @@ +import argparse +import time +import numpy as np +import cv2 as cv + +# ------------------------Service operations------------------------ +def weight_path(model_path): + """ Get path of weights based on path to IR + + Params: + model_path: the string contains path to IR file + + Return: + Path to weights file + """ + assert model_path.endswith('.xml'), "Wrong topology path was provided" + return model_path[:-3] + 'bin' + + +def build_argparser(): + """ Parse arguments from command line + + Return: + Pack of arguments from command line + """ + parser = argparse.ArgumentParser(description='This is an OpenCV-based version of Gaze Estimation example') + + parser.add_argument('--input', + help='Path to the input video file') + parser.add_argument('--out', + help='Path to the output video file') + parser.add_argument('--facem', + default='face-detection-retail-0005.xml', + help='Path to OpenVINO face detection model (.xml)') + parser.add_argument('--faced', + default='CPU', + help='Target device for the face detection' + + '(e.g. CPU, GPU, VPU, ...)') + parser.add_argument('--headm', + default='head-pose-estimation-adas-0001.xml', + help='Path to OpenVINO head pose estimation model (.xml)') + parser.add_argument('--headd', + default='CPU', + help='Target device for the head pose estimation inference ' + + '(e.g. CPU, GPU, VPU, ...)') + parser.add_argument('--landm', + default='facial-landmarks-35-adas-0002.xml', + help='Path to OpenVINO landmarks detector model (.xml)') + parser.add_argument('--landd', + default='CPU', + help='Target device for the landmarks detector (e.g. CPU, GPU, VPU, ...)') + parser.add_argument('--gazem', + default='gaze-estimation-adas-0002.xml', + help='Path to OpenVINO gaze vector estimaiton model (.xml)') + parser.add_argument('--gazed', + default='CPU', + help='Target device for the gaze vector estimation inference ' + + '(e.g. CPU, GPU, VPU, ...)') + parser.add_argument('--eyem', + default='open-closed-eye-0001.xml', + help='Path to OpenVINO open closed eye model (.xml)') + parser.add_argument('--eyed', + default='CPU', + help='Target device for the eyes state inference (e.g. CPU, GPU, VPU, ...)') + return parser + + +# ------------------------Support functions for custom kernels------------------------ +def intersection(surface, rect): + """ Remove zone of out of bound from ROI + + Params: + surface: image bounds is rect representation (top left coordinates and width and height) + rect: region of interest is also has rect representation + + Return: + Modified ROI with correct bounds + """ + l_x = max(surface[0], rect[0]) + l_y = max(surface[1], rect[1]) + width = min(surface[0] + surface[2], rect[0] + rect[2]) - l_x + height = min(surface[1] + surface[3], rect[1] + rect[3]) - l_y + if width < 0 or height < 0: + return (0, 0, 0, 0) + return (l_x, l_y, width, height) + + +def process_landmarks(r_x, r_y, r_w, r_h, landmarks): + """ Create points from result of inference of facial-landmarks network and size of input image + + Params: + r_x: x coordinate of top left corner of input image + r_y: y coordinate of top left corner of input image + r_w: width of input image + r_h: height of input image + landmarks: result of inference of facial-landmarks network + + Return: + Array of landmarks points for one face + """ + lmrks = landmarks[0] + raw_x = lmrks[::2] * r_w + r_x + raw_y = lmrks[1::2] * r_h + r_y + return np.array([[int(x), int(y)] for x, y in zip(raw_x, raw_y)]) + + +def eye_box(p_1, p_2, scale=1.8): + """ Get bounding box of eye + + Params: + p_1: point of left edge of eye + p_2: point of right edge of eye + scale: change size of box with this value + + Return: + Bounding box of eye and its midpoint + """ + + size = np.linalg.norm(p_1 - p_2) + midpoint = (p_1 + p_2) / 2 + width = scale * size + height = width + p_x = midpoint[0] - (width / 2) + p_y = midpoint[1] - (height / 2) + return (int(p_x), int(p_y), int(width), int(height)), list(map(int, midpoint)) + + +# ------------------------Custom graph operations------------------------ +@cv.gapi.op('custom.GProcessPoses', + in_types=[cv.GArray.GMat, cv.GArray.GMat, cv.GArray.GMat], + out_types=[cv.GArray.GMat]) +class GProcessPoses: + @staticmethod + def outMeta(arr_desc0, arr_desc1, arr_desc2): + return cv.empty_array_desc() + + +@cv.gapi.op('custom.GParseEyes', + in_types=[cv.GArray.GMat, cv.GArray.Rect, cv.GOpaque.Size], + out_types=[cv.GArray.Rect, cv.GArray.Rect, cv.GArray.Point, cv.GArray.Point]) +class GParseEyes: + @staticmethod + def outMeta(arr_desc0, arr_desc1, arr_desc2): + return cv.empty_array_desc(), cv.empty_array_desc(), \ + cv.empty_array_desc(), cv.empty_array_desc() + + +@cv.gapi.op('custom.GGetStates', + in_types=[cv.GArray.GMat, cv.GArray.GMat], + out_types=[cv.GArray.Int, cv.GArray.Int]) +class GGetStates: + @staticmethod + def outMeta(arr_desc0, arr_desc1): + return cv.empty_array_desc(), cv.empty_array_desc() + + +# ------------------------Custom kernels------------------------ +@cv.gapi.kernel(GProcessPoses) +class GProcessPosesImpl: + """ Custom kernel. Processed poses of heads + """ + @staticmethod + def run(in_ys, in_ps, in_rs): + """ Сustom kernel executable code + + Params: + in_ys: yaw angle of head + in_ps: pitch angle of head + in_rs: roll angle of head + + Return: + Arrays with heads poses + """ + out_poses = [] + size = len(in_ys) + for i in range(size): + out_poses.append(np.array([in_ys[i][0], in_ps[i][0], in_rs[i][0]]).T) + return out_poses + + +@cv.gapi.kernel(GParseEyes) +class GParseEyesImpl: + """ Custom kernel. Get information about eyes + """ + @staticmethod + def run(in_landm_per_face, in_face_rcs, frame_size): + """ Сustom kernel executable code + + Params: + in_landm_per_face: landmarks from inference of facial-landmarks network for each face + in_face_rcs: bounding boxes for each face + frame_size: size of input image + + Return: + Arrays of ROI for left and right eyes, array of midpoints and + array of landmarks points + """ + left_eyes = [] + right_eyes = [] + midpoints = [] + lmarks = [] + num_faces = len(in_landm_per_face) + surface = (0, 0, *frame_size) + for i in range(num_faces): + rect = in_face_rcs[i] + points = process_landmarks(*rect, in_landm_per_face[i]) + for p in points: + lmarks.append(p) + size = int(len(in_landm_per_face[i][0]) / 2) + + rect, midpoint_l = eye_box(lmarks[0 + i * size], lmarks[1 + i * size]) + left_eyes.append(intersection(surface, rect)) + rect, midpoint_r = eye_box(lmarks[2 + i * size], lmarks[3 + i * size]) + right_eyes.append(intersection(surface, rect)) + midpoints += [midpoint_l, midpoint_r] + return left_eyes, right_eyes, midpoints, lmarks + + +@cv.gapi.kernel(GGetStates) +class GGetStatesImpl: + """ Custom kernel. Get state of eye - open or closed + """ + @staticmethod + def run(eyesl, eyesr): + """ Сustom kernel executable code + + Params: + eyesl: result of inference of open-closed-eye network for left eye + eyesr: result of inference of open-closed-eye network for right eye + + Return: + States of left eyes and states of right eyes + """ + size = len(eyesl) + out_l_st = [] + out_r_st = [] + for i in range(size): + for st in eyesl[i]: + out_l_st += [1 if st[0] < st[1] else 0] + for st in eyesr[i]: + out_r_st += [1 if st[0] < st[1] else 0] + return out_l_st, out_r_st + + +if __name__ == '__main__': + ARGUMENTS = build_argparser().parse_args() + + # ------------------------Demo's graph------------------------ + g_in = cv.GMat() + + # Detect faces + face_inputs = cv.GInferInputs() + face_inputs.setInput('data', g_in) + face_outputs = cv.gapi.infer('face-detection', face_inputs) + faces = face_outputs.at('detection_out') + + # Parse faces + sz = cv.gapi.streaming.size(g_in) + faces_rc = cv.gapi.parseSSD(faces, sz, 0.5, False, False) + + # Detect poses + head_inputs = cv.GInferInputs() + head_inputs.setInput('data', g_in) + face_outputs = cv.gapi.infer('head-pose', faces_rc, head_inputs) + angles_y = face_outputs.at('angle_y_fc') + angles_p = face_outputs.at('angle_p_fc') + angles_r = face_outputs.at('angle_r_fc') + + # Parse poses + heads_pos = GProcessPoses.on(angles_y, angles_p, angles_r) + + # Detect landmarks + landmark_inputs = cv.GInferInputs() + landmark_inputs.setInput('data', g_in) + landmark_outputs = cv.gapi.infer('facial-landmarks', faces_rc, + landmark_inputs) + landmark = landmark_outputs.at('align_fc3') + + # Parse landmarks + left_eyes, right_eyes, mids, lmarks = GParseEyes.on(landmark, faces_rc, sz) + + # Detect eyes + eyes_inputs = cv.GInferInputs() + eyes_inputs.setInput('input.1', g_in) + eyesl_outputs = cv.gapi.infer('open-closed-eye', left_eyes, eyes_inputs) + eyesr_outputs = cv.gapi.infer('open-closed-eye', right_eyes, eyes_inputs) + eyesl = eyesl_outputs.at('19') + eyesr = eyesr_outputs.at('19') + + # Process eyes states + l_eye_st, r_eye_st = GGetStates.on(eyesl, eyesr) + + # Gaze estimation + gaze_inputs = cv.GInferListInputs() + gaze_inputs.setInput('left_eye_image', left_eyes) + gaze_inputs.setInput('right_eye_image', right_eyes) + gaze_inputs.setInput('head_pose_angles', heads_pos) + gaze_outputs = cv.gapi.infer2('gaze-estimation', g_in, gaze_inputs) + gaze_vectors = gaze_outputs.at('gaze_vector') + + out = cv.gapi.copy(g_in) + # ------------------------End of graph------------------------ + + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(out, + faces_rc, + left_eyes, + right_eyes, + gaze_vectors, + angles_y, + angles_p, + angles_r, + l_eye_st, + r_eye_st, + mids, + lmarks)) + + # Networks + face_net = cv.gapi.ie.params('face-detection', ARGUMENTS.facem, + weight_path(ARGUMENTS.facem), ARGUMENTS.faced) + head_pose_net = cv.gapi.ie.params('head-pose', ARGUMENTS.headm, + weight_path(ARGUMENTS.headm), ARGUMENTS.headd) + landmarks_net = cv.gapi.ie.params('facial-landmarks', ARGUMENTS.landm, + weight_path(ARGUMENTS.landm), ARGUMENTS.landd) + gaze_net = cv.gapi.ie.params('gaze-estimation', ARGUMENTS.gazem, + weight_path(ARGUMENTS.gazem), ARGUMENTS.gazed) + eye_net = cv.gapi.ie.params('open-closed-eye', ARGUMENTS.eyem, + weight_path(ARGUMENTS.eyem), ARGUMENTS.eyed) + + nets = cv.gapi.networks(face_net, head_pose_net, landmarks_net, gaze_net, eye_net) + + # Kernels pack + kernels = cv.gapi.kernels(GParseEyesImpl, GProcessPosesImpl, GGetStatesImpl) + + # ------------------------Execution part------------------------ + ccomp = comp.compileStreaming(args=cv.gapi.compile_args(kernels, nets)) + source = cv.gapi.wip.make_capture_src(ARGUMENTS.input) + ccomp.setSource(cv.gin(source)) + ccomp.start() + + frames = 0 + fps = 0 + print('Processing') + START_TIME = time.time() + + while True: + start_time_cycle = time.time() + has_frame, (oimg, + outr, + l_eyes, + r_eyes, + outg, + out_y, + out_p, + out_r, + out_st_l, + out_st_r, + out_mids, + outl) = ccomp.pull() + + if not has_frame: + break + + # Draw + GREEN = (0, 255, 0) + RED = (0, 0, 255) + WHITE = (255, 255, 255) + BLUE = (255, 0, 0) + PINK = (255, 0, 255) + YELLOW = (0, 255, 255) + + M_PI_180 = np.pi / 180 + M_PI_2 = np.pi / 2 + M_PI = np.pi + + FACES_SIZE = len(outr) + + for i, out_rect in enumerate(outr): + # Face box + cv.rectangle(oimg, out_rect, WHITE, 1) + rx, ry, rwidth, rheight = out_rect + + # Landmarks + lm_radius = int(0.01 * rwidth + 1) + lmsize = int(len(outl) / FACES_SIZE) + for j in range(lmsize): + cv.circle(oimg, outl[j + i * lmsize], lm_radius, YELLOW, -1) + + # Headposes + yaw = out_y[i] + pitch = out_p[i] + roll = out_r[i] + sin_y = np.sin(yaw[:] * M_PI_180) + sin_p = np.sin(pitch[:] * M_PI_180) + sin_r = np.sin(roll[:] * M_PI_180) + + cos_y = np.cos(yaw[:] * M_PI_180) + cos_p = np.cos(pitch[:] * M_PI_180) + cos_r = np.cos(roll[:] * M_PI_180) + + axis_length = 0.4 * rwidth + x_center = int(rx + rwidth / 2) + y_center = int(ry + rheight / 2) + + # center to right + cv.line(oimg, [x_center, y_center], + [int(x_center + axis_length * (cos_r * cos_y + sin_y * sin_p * sin_r)), + int(y_center + axis_length * cos_p * sin_r)], + RED, 2) + + # center to top + cv.line(oimg, [x_center, y_center], + [int(x_center + axis_length * (cos_r * sin_y * sin_p + cos_y * sin_r)), + int(y_center - axis_length * cos_p * cos_r)], + GREEN, 2) + + # center to forward + cv.line(oimg, [x_center, y_center], + [int(x_center + axis_length * sin_y * cos_p), + int(y_center + axis_length * sin_p)], + PINK, 2) + + scale_box = 0.002 * rwidth + cv.putText(oimg, "head pose: (y=%0.0f, p=%0.0f, r=%0.0f)" % + (np.round(yaw), np.round(pitch), np.round(roll)), + [int(rx), int(ry + rheight + 5 * rwidth / 100)], + cv.FONT_HERSHEY_PLAIN, scale_box * 2, WHITE, 1) + + # Eyes boxes + color_l = GREEN if out_st_l[i] else RED + cv.rectangle(oimg, l_eyes[i], color_l, 1) + color_r = GREEN if out_st_r[i] else RED + cv.rectangle(oimg, r_eyes[i], color_r, 1) + + # Gaze vectors + norm_gazes = np.linalg.norm(outg[i][0]) + gaze_vector = outg[i][0] / norm_gazes + + arrow_length = 0.4 * rwidth + gaze_arrow = [arrow_length * gaze_vector[0], -arrow_length * gaze_vector[1]] + left_arrow = [int(a+b) for a, b in zip(out_mids[0 + i * 2], gaze_arrow)] + right_arrow = [int(a+b) for a, b in zip(out_mids[1 + i * 2], gaze_arrow)] + if out_st_l[i]: + cv.arrowedLine(oimg, out_mids[0 + i * 2], left_arrow, BLUE, 2) + if out_st_r[i]: + cv.arrowedLine(oimg, out_mids[1 + i * 2], right_arrow, BLUE, 2) + + v0, v1, v2 = outg[i][0] + + gaze_angles = [180 / M_PI * (M_PI_2 + np.arctan2(v2, v0)), + 180 / M_PI * (M_PI_2 - np.arccos(v1 / norm_gazes))] + cv.putText(oimg, "gaze angles: (h=%0.0f, v=%0.0f)" % + (np.round(gaze_angles[0]), np.round(gaze_angles[1])), + [int(rx), int(ry + rheight + 12 * rwidth / 100)], + cv.FONT_HERSHEY_PLAIN, scale_box * 2, WHITE, 1) + + # Add FPS value to frame + cv.putText(oimg, "FPS: %0i" % (fps), [int(20), int(40)], + cv.FONT_HERSHEY_PLAIN, 2, RED, 2) + + # Show result + cv.imshow('Gaze Estimation', oimg) + + fps = int(1. / (time.time() - start_time_cycle)) + frames += 1 + EXECUTION_TIME = time.time() - START_TIME + print('Execution successful') + print('Mean FPS is ', int(frames / EXECUTION_TIME)) From fc799191f4f19c523343c41a99ec9e6e4269da9e Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 1 Jul 2021 13:48:11 +0000 Subject: [PATCH 032/376] gapi(ie): replace deprecated calls --- modules/gapi/src/backends/ie/giebackend/giewrapper.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp b/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp index ba0632d4f0f2..d4ec806e4846 100644 --- a/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp +++ b/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp @@ -124,7 +124,11 @@ IE::Core giewrap::getPlugin(const GIEParam& params) { { try { +#if INF_ENGINE_RELEASE >= 2021040000 + plugin.AddExtension(std::make_shared(extlib), params.device_id); +#else plugin.AddExtension(IE::make_so_pointer(extlib), params.device_id); +#endif CV_LOG_INFO(NULL, "DNN-IE: Loaded extension plugin: " << extlib); break; } From 0f24d4d2a14d8f79eaa4ca0710d7770aeeb833f2 Mon Sep 17 00:00:00 2001 From: APrigarina Date: Tue, 29 Jun 2021 22:15:37 +0300 Subject: [PATCH 033/376] fix samples --- samples/python/gaussian_mix.py | 2 +- samples/python/hist.py | 2 +- samples/python/morphology.py | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/samples/python/gaussian_mix.py b/samples/python/gaussian_mix.py index 6a656647ddcf..4c1f86794cd6 100755 --- a/samples/python/gaussian_mix.py +++ b/samples/python/gaussian_mix.py @@ -28,7 +28,7 @@ def make_gaussians(cluster_n, img_size): return points, ref_distrs def draw_gaussain(img, mean, cov, color): - x, y = np.int32(mean) + x, y = mean w, u, _vt = cv.SVDecomp(cov) ang = np.arctan2(u[1, 0], u[0, 0])*(180/np.pi) s1, s2 = np.sqrt(w)*3.0 diff --git a/samples/python/hist.py b/samples/python/hist.py index 157d5ff0ba3e..8c1f4546a817 100755 --- a/samples/python/hist.py +++ b/samples/python/hist.py @@ -46,7 +46,7 @@ def hist_lines(im): im = cv.cvtColor(im,cv.COLOR_BGR2GRAY) hist_item = cv.calcHist([im],[0],None,[256],[0,256]) cv.normalize(hist_item,hist_item,0,255,cv.NORM_MINMAX) - hist=np.int32(np.around(hist_item)) + hist = np.int32(np.around(hist_item)) for x,y in enumerate(hist): cv.line(h,(x,0),(x,y[0]),(255,255,255)) y = np.flipud(h) diff --git a/samples/python/morphology.py b/samples/python/morphology.py index 9ecf5b0682e7..183f5e828815 100755 --- a/samples/python/morphology.py +++ b/samples/python/morphology.py @@ -50,8 +50,11 @@ def main(): cur_str_mode = str_modes.next() def update(dummy=None): - sz = cv.getTrackbarPos('op/size', 'morphology') - iters = cv.getTrackbarPos('iters', 'morphology') + try: # do not get trackbar position while trackbar is not created + sz = cv.getTrackbarPos('op/size', 'morphology') + iters = cv.getTrackbarPos('iters', 'morphology') + except: + return opers = cur_mode.split('/') if len(opers) > 1: sz = sz - 10 From 05f1939b0284c55f987b6caa92f34affd50dd454 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Thu, 1 Jul 2021 22:06:35 +0300 Subject: [PATCH 034/376] Merge pull request #20298 from mpashchenkov:mp/python-desync G-API: Python. Desync. * Desync. GMat. * Alignment --- .../gapi/include/opencv2/gapi/gstreaming.hpp | 11 +- modules/gapi/misc/python/pyopencv_gapi.hpp | 112 ++++++++++++------ modules/gapi/misc/python/shadow_gapi.hpp | 31 ++--- .../misc/python/test/test_gapi_streaming.py | 68 ++++++++++- modules/gapi/src/compiler/gstreaming.cpp | 18 ++- modules/gapi/src/compiler/gstreaming_priv.hpp | 1 + .../gapi/src/executor/gstreamingexecutor.cpp | 78 ++++++++++++ .../gapi/src/executor/gstreamingexecutor.hpp | 3 + .../test/streaming/gapi_streaming_tests.cpp | 81 ++++++++++++- 9 files changed, 334 insertions(+), 69 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/gstreaming.hpp b/modules/gapi/include/opencv2/gapi/gstreaming.hpp index 47e103fd0ea7..50abe69f87b7 100644 --- a/modules/gapi/include/opencv2/gapi/gstreaming.hpp +++ b/modules/gapi/include/opencv2/gapi/gstreaming.hpp @@ -71,6 +71,15 @@ using GOptRunArgP = util::variant< >; using GOptRunArgsP = std::vector; +using GOptRunArg = util::variant< + optional, + optional, + optional, + optional, + optional +>; +using GOptRunArgs = std::vector; + namespace detail { template inline GOptRunArgP wrap_opt_arg(optional& arg) { @@ -255,7 +264,7 @@ class GAPI_EXPORTS_W_SIMPLE GStreamingCompiled // NB: Used from python /// @private -- Exclude this function from OpenCV documentation - GAPI_WRAP std::tuple pull(); + GAPI_WRAP std::tuple> pull(); /** * @brief Get some next available data from the pipeline. diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index 3c428dde6d82..d378a91b5fd6 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -131,7 +131,8 @@ PyObject* pyopencv_from(const cv::detail::PyObjectHolder& v) template <> PyObject* pyopencv_from(const cv::gapi::wip::draw::Prim& prim) { - switch (prim.index()) { + switch (prim.index()) + { case cv::gapi::wip::draw::Prim::index_of(): return pyopencv_from(cv::util::get(prim)); case cv::gapi::wip::draw::Prim::index_of(): @@ -319,40 +320,69 @@ PyObject* pyopencv_from(const GRunArg& v) return pyopencv_from(util::get(v)); } - PyErr_SetString(PyExc_TypeError, "Failed to unpack GRunArgs"); + PyErr_SetString(PyExc_TypeError, "Failed to unpack GRunArgs. Index of variant is unknown"); return NULL; } -template<> -PyObject* pyopencv_from(const GRunArgs& value) +template +PyObject* pyopencv_from(const cv::optional& opt) { - size_t i, n = value.size(); + if (!opt.has_value()) + { + Py_RETURN_NONE; + } + return pyopencv_from(*opt); +} - // NB: It doesn't make sense to return list with a single element - if (n == 1) +template <> +PyObject* pyopencv_from(const GOptRunArg& v) +{ + switch (v.index()) { - PyObject* item = pyopencv_from(value[0]); - if(!item) - { - return NULL; - } - return item; + case GOptRunArg::index_of>(): + return pyopencv_from(util::get>(v)); + + case GOptRunArg::index_of>(): + return pyopencv_from(util::get>(v)); + + case GOptRunArg::index_of>(): + return pyopencv_from(util::get>(v)); + + case GOptRunArg::index_of>(): + return pyopencv_from(util::get>(v)); } - PyObject* list = PyList_New(n); - for(i = 0; i < n; ++i) + PyErr_SetString(PyExc_TypeError, "Failed to unpack GOptRunArg. Index of variant is unknown"); + return NULL; +} + +template<> +PyObject* pyopencv_from(const GRunArgs& value) +{ + return value.size() == 1 ? pyopencv_from(value[0]) : pyopencv_from_generic_vec(value); +} + +template<> +PyObject* pyopencv_from(const GOptRunArgs& value) +{ + return value.size() == 1 ? pyopencv_from(value[0]) : pyopencv_from_generic_vec(value); +} + +// FIXME: cv::variant should be wrapped once for all types. +template <> +PyObject* pyopencv_from(const cv::util::variant& v) +{ + using RunArgs = cv::util::variant; + switch (v.index()) { - PyObject* item = pyopencv_from(value[i]); - if(!item) - { - Py_DECREF(list); - PyErr_SetString(PyExc_TypeError, "Failed to unpack GRunArgs"); - return NULL; - } - PyList_SetItem(list, i, item); + case RunArgs::index_of(): + return pyopencv_from(util::get(v)); + case RunArgs::index_of(): + return pyopencv_from(util::get(v)); } - return list; + PyErr_SetString(PyExc_TypeError, "Failed to recognize kind of RunArgs. Index of variant is unknown"); + return NULL; } template @@ -634,7 +664,8 @@ static cv::GRunArgs run_py_kernel(cv::detail::PyObjectHolder kernel, cv::detail::PyObjectHolder result( PyObject_CallObject(kernel.get(), args.get()), false); - if (PyErr_Occurred()) { + if (PyErr_Occurred()) + { PyErr_PrintEx(0); PyErr_Clear(); throw std::logic_error("Python kernel failed with error!"); @@ -717,8 +748,9 @@ static cv::GMetaArgs get_meta_args(PyObject* tuple) } static GMetaArgs run_py_meta(cv::detail::PyObjectHolder out_meta, - const cv::GMetaArgs &meta, - const cv::GArgs &gargs) { + const cv::GMetaArgs &meta, + const cv::GArgs &gargs) +{ PyGILState_STATE gstate; gstate = PyGILState_Ensure(); @@ -760,7 +792,8 @@ static GMetaArgs run_py_meta(cv::detail::PyObjectHolder out_meta, cv::detail::PyObjectHolder result( PyObject_CallObject(out_meta.get(), args.get()), false); - if (PyErr_Occurred()) { + if (PyErr_Occurred()) + { PyErr_PrintEx(0); PyErr_Clear(); throw std::logic_error("Python outMeta failed with error!"); @@ -792,21 +825,24 @@ static PyObject* pyopencv_cv_gapi_kernels(PyObject* , PyObject* py_args, PyObjec PyObject* user_kernel = PyTuple_GetItem(py_args, i); PyObject* id_obj = PyObject_GetAttrString(user_kernel, "id"); - if (!id_obj) { + if (!id_obj) + { PyErr_SetString(PyExc_TypeError, "Python kernel should contain id, please use cv.gapi.kernel to define kernel"); return NULL; } PyObject* out_meta = PyObject_GetAttrString(user_kernel, "outMeta"); - if (!out_meta) { + if (!out_meta) + { PyErr_SetString(PyExc_TypeError, "Python kernel should contain outMeta, please use cv.gapi.kernel to define kernel"); return NULL; } PyObject* run = PyObject_GetAttrString(user_kernel, "run"); - if (!run) { + if (!run) + { PyErr_SetString(PyExc_TypeError, "Python kernel should contain run, please use cv.gapi.kernel to define kernel"); return NULL; @@ -951,9 +987,12 @@ struct PyOpenCV_Converter> if (PyObject_TypeCheck(obj, reinterpret_cast(pyopencv_GArrayT_TypePtr))) { auto& array = reinterpret_cast(obj)->v; - try { + try + { value = cv::util::get>(array.arg()); - } catch (...) { + } + catch (...) + { return false; } return true; @@ -974,9 +1013,12 @@ struct PyOpenCV_Converter> if (PyObject_TypeCheck(obj, reinterpret_cast(pyopencv_GOpaqueT_TypePtr))) { auto& opaque = reinterpret_cast(obj)->v; - try { + try + { value = cv::util::get>(opaque.arg()); - } catch (...) { + } + catch (...) + { return false; } return true; diff --git a/modules/gapi/misc/python/shadow_gapi.hpp b/modules/gapi/misc/python/shadow_gapi.hpp index 41d0f1973223..0b489dde0f55 100644 --- a/modules/gapi/misc/python/shadow_gapi.hpp +++ b/modules/gapi/misc/python/shadow_gapi.hpp @@ -3,39 +3,40 @@ namespace cv { -struct GAPI_EXPORTS_W_SIMPLE GCompileArg { - GAPI_WRAP GCompileArg(gapi::GKernelPackage pkg); - GAPI_WRAP GCompileArg(gapi::GNetPackage pkg); +struct GAPI_EXPORTS_W_SIMPLE GCompileArg +{ + GAPI_WRAP GCompileArg(gapi::GKernelPackage pkg); + GAPI_WRAP GCompileArg(gapi::GNetPackage pkg); }; class GAPI_EXPORTS_W_SIMPLE GInferInputs { public: - GAPI_WRAP GInferInputs(); - GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GMat& value); - GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GFrame& value); + GAPI_WRAP GInferInputs(); + GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GMat& value); + GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GFrame& value); }; class GAPI_EXPORTS_W_SIMPLE GInferListInputs { public: - GAPI_WRAP GInferListInputs(); - GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); - GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); + GAPI_WRAP GInferListInputs(); + GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); + GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); }; class GAPI_EXPORTS_W_SIMPLE GInferOutputs { public: - GAPI_WRAP GInferOutputs(); - GAPI_WRAP cv::GMat at(const std::string& name); + GAPI_WRAP GInferOutputs(); + GAPI_WRAP cv::GMat at(const std::string& name); }; class GAPI_EXPORTS_W_SIMPLE GInferListOutputs { public: - GAPI_WRAP GInferListOutputs(); - GAPI_WRAP cv::GArray at(const std::string& name); + GAPI_WRAP GInferListOutputs(); + GAPI_WRAP cv::GArray at(const std::string& name); }; namespace gapi @@ -69,11 +70,13 @@ namespace streaming cv::GOpaque GAPI_EXPORTS_W timestamp(cv::GMat); cv::GOpaque GAPI_EXPORTS_W seqNo(cv::GMat); cv::GOpaque GAPI_EXPORTS_W seq_id(cv::GMat); + + GAPI_EXPORTS_W cv::GMat desync(const cv::GMat &g); } // namespace streaming } // namespace gapi namespace detail { - gapi::GNetParam GAPI_EXPORTS_W strip(gapi::ie::PyParams params); + gapi::GNetParam GAPI_EXPORTS_W strip(gapi::ie::PyParams params); } // namespace detail } // namespace cv diff --git a/modules/gapi/misc/python/test/test_gapi_streaming.py b/modules/gapi/misc/python/test/test_gapi_streaming.py index 4ea88878eeab..7ede1b5cf38d 100644 --- a/modules/gapi/misc/python/test/test_gapi_streaming.py +++ b/modules/gapi/misc/python/test/test_gapi_streaming.py @@ -5,16 +5,35 @@ import os import sys import unittest +import time from tests_common import NewOpenCVTests try: - if sys.version_info[:2] < (3, 0): raise unittest.SkipTest('Python 2.x is not supported') + @cv.gapi.op('custom.delay', in_types=[cv.GMat], out_types=[cv.GMat]) + class GDelay: + """Delay for 10 ms.""" + + @staticmethod + def outMeta(desc): + return desc + + + @cv.gapi.kernel(GDelay) + class GDelayImpl: + """Implementation for GDelay operation.""" + + @staticmethod + def run(img): + time.sleep(0.01) + return img + + class test_gapi_streaming(NewOpenCVTests): def test_image_input(self): @@ -148,7 +167,7 @@ def test_video_add(self): proc_num_frames += 1 if proc_num_frames == max_num_frames: - break; + break def test_video_good_features_to_track(self): @@ -242,6 +261,51 @@ def test_gapi_streaming_meta(self): if curr_frame_number == max_num_frames: break + def test_desync(self): + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # G-API + g_in = cv.GMat() + g_out1 = cv.gapi.copy(g_in) + des = cv.gapi.streaming.desync(g_in) + g_out2 = GDelay.on(des) + + c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out1, g_out2)) + + kernels = cv.gapi.kernels(GDelayImpl) + ccomp = c.compileStreaming(args=cv.gapi.compile_args(kernels)) + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(cv.gin(source)) + ccomp.start() + + # Assert + max_num_frames = 10 + proc_num_frames = 0 + + out_counter = 0 + desync_out_counter = 0 + none_counter = 0 + while True: + has_frame, (out1, out2) = ccomp.pull() + if not has_frame: + break + + if not out1 is None: + out_counter += 1 + if not out2 is None: + desync_out_counter += 1 + else: + none_counter += 1 + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + ccomp.stop() + break + + self.assertLess(0, proc_num_frames) + self.assertLess(desync_out_counter, out_counter) + self.assertLess(0, none_counter) + except unittest.SkipTest as e: diff --git a/modules/gapi/src/compiler/gstreaming.cpp b/modules/gapi/src/compiler/gstreaming.cpp index 3bdc0323b5c7..e45e77042755 100644 --- a/modules/gapi/src/compiler/gstreaming.cpp +++ b/modules/gapi/src/compiler/gstreaming.cpp @@ -75,6 +75,11 @@ bool cv::GStreamingCompiled::Priv::pull(cv::GOptRunArgsP &&outs) return m_exec->pull(std::move(outs)); } +std::tuple> cv::GStreamingCompiled::Priv::pull() +{ + return m_exec->pull(); +} + bool cv::GStreamingCompiled::Priv::try_pull(cv::GRunArgsP &&outs) { return m_exec->try_pull(std::move(outs)); @@ -123,18 +128,9 @@ bool cv::GStreamingCompiled::pull(cv::GRunArgsP &&outs) return m_priv->pull(std::move(outs)); } -std::tuple cv::GStreamingCompiled::pull() +std::tuple> cv::GStreamingCompiled::pull() { - GRunArgs run_args; - GRunArgsP outs; - const auto& out_info = m_priv->outInfo(); - run_args.reserve(out_info.size()); - outs.reserve(out_info.size()); - - cv::detail::constructGraphOutputs(m_priv->outInfo(), run_args, outs); - - bool is_over = m_priv->pull(std::move(outs)); - return std::make_tuple(is_over, run_args); + return m_priv->pull(); } bool cv::GStreamingCompiled::pull(cv::GOptRunArgsP &&outs) diff --git a/modules/gapi/src/compiler/gstreaming_priv.hpp b/modules/gapi/src/compiler/gstreaming_priv.hpp index 59b19d425261..1b559ba31030 100644 --- a/modules/gapi/src/compiler/gstreaming_priv.hpp +++ b/modules/gapi/src/compiler/gstreaming_priv.hpp @@ -46,6 +46,7 @@ class GAPI_EXPORTS GStreamingCompiled::Priv void start(); bool pull(cv::GRunArgsP &&outs); bool pull(cv::GOptRunArgsP &&outs); + std::tuple> pull(); bool try_pull(cv::GRunArgsP &&outs); void stop(); diff --git a/modules/gapi/src/executor/gstreamingexecutor.cpp b/modules/gapi/src/executor/gstreamingexecutor.cpp index 74c96bdf3ef3..27049aef6327 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.cpp +++ b/modules/gapi/src/executor/gstreamingexecutor.cpp @@ -1017,6 +1017,49 @@ void check_DesyncObjectConsumedByMultipleIslands(const cv::gimpl::GIslandModel:: } // for(nodes) } +// NB: Construct GRunArgsP based on passed info and store the memory in passed cv::GRunArgs. +// Needed for python bridge, because in case python user doesn't pass output arguments to apply. +void constructOptGraphOutputs(const cv::GTypesInfo &out_info, + cv::GOptRunArgs &args, + cv::GOptRunArgsP &outs) +{ + for (auto&& info : out_info) + { + switch (info.shape) + { + case cv::GShape::GMAT: + { + args.emplace_back(cv::optional{}); + outs.emplace_back(&cv::util::get>(args.back())); + break; + } + case cv::GShape::GSCALAR: + { + args.emplace_back(cv::optional{}); + outs.emplace_back(&cv::util::get>(args.back())); + break; + } + case cv::GShape::GARRAY: + { + cv::detail::VectorRef ref; + cv::util::get(info.ctor)(ref); + args.emplace_back(cv::util::make_optional(std::move(ref))); + outs.emplace_back(wrap_opt_arg(cv::util::get>(args.back()))); + break; + } + case cv::GShape::GOPAQUE: + { + cv::detail::OpaqueRef ref; + cv::util::get(info.ctor)(ref); + args.emplace_back(cv::util::make_optional(std::move(ref))); + outs.emplace_back(wrap_opt_arg(cv::util::get>(args.back()))); + break; + } + default: + cv::util::throw_error(std::logic_error("Unsupported optional output shape for Python")); + } + } +} } // anonymous namespace class cv::gimpl::GStreamingExecutor::Synchronizer final { @@ -1320,6 +1363,16 @@ cv::gimpl::GStreamingExecutor::GStreamingExecutor(std::unique_ptr && // per the same input frame, so the output traffic multiplies) GAPI_Assert(m_collector_map.size() > 0u); m_out_queue.set_capacity(queue_capacity * m_collector_map.size()); + + // FIXME: The code duplicates logic of collectGraphInfo() + cv::gimpl::GModel::ConstGraph cgr(*m_orig_graph); + auto meta = cgr.metadata().get().out_nhs; + out_info.reserve(meta.size()); + + ade::util::transform(meta, std::back_inserter(out_info), [&cgr](const ade::NodeHandle& nh) { + const auto& data = cgr.metadata(nh).get(); + return cv::GTypeInfo{data.shape, data.kind, data.ctor}; + }); } cv::gimpl::GStreamingExecutor::~GStreamingExecutor() @@ -1653,6 +1706,31 @@ bool cv::gimpl::GStreamingExecutor::pull(cv::GOptRunArgsP &&outs) return true; } +std::tuple> cv::gimpl::GStreamingExecutor::pull() +{ + using RunArgs = cv::util::variant; + bool is_over = false; + + if (m_desync) { + GOptRunArgs opt_run_args; + GOptRunArgsP opt_outs; + opt_outs.reserve(out_info.size()); + opt_run_args.reserve(out_info.size()); + + constructOptGraphOutputs(out_info, opt_run_args, opt_outs); + is_over = pull(std::move(opt_outs)); + return std::make_tuple(is_over, RunArgs(opt_run_args)); + } + + GRunArgs run_args; + GRunArgsP outs; + run_args.reserve(out_info.size()); + outs.reserve(out_info.size()); + + constructGraphOutputs(out_info, run_args, outs); + is_over = pull(std::move(outs)); + return std::make_tuple(is_over, RunArgs(run_args)); +} bool cv::gimpl::GStreamingExecutor::try_pull(cv::GRunArgsP &&outs) { diff --git a/modules/gapi/src/executor/gstreamingexecutor.hpp b/modules/gapi/src/executor/gstreamingexecutor.hpp index 40b787268228..b4aadcbbaf4d 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.hpp +++ b/modules/gapi/src/executor/gstreamingexecutor.hpp @@ -195,6 +195,8 @@ class GStreamingExecutor final void wait_shutdown(); + cv::GTypesInfo out_info; + public: explicit GStreamingExecutor(std::unique_ptr &&g_model, const cv::GCompileArgs &comp_args); @@ -203,6 +205,7 @@ class GStreamingExecutor final void start(); bool pull(cv::GRunArgsP &&outs); bool pull(cv::GOptRunArgsP &&outs); + std::tuple> pull(); bool try_pull(cv::GRunArgsP &&outs); void stop(); bool running() const; diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp index f3179a70813a..5386d1736f67 100644 --- a/modules/gapi/test/streaming/gapi_streaming_tests.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_tests.cpp @@ -244,6 +244,35 @@ class NV12Source : public cv::gapi::wip::GCaptureSource { } }; +void checkPullOverload(const cv::Mat& ref, + const bool has_output, + cv::util::variant& args) { + EXPECT_TRUE(has_output); + using runArgs = cv::util::variant; + cv::Mat out_mat; + switch (args.index()) { + case runArgs::index_of(): + { + auto outputs = util::get(args); + EXPECT_EQ(1u, outputs.size()); + out_mat = cv::util::get(outputs[0]); + break; + } + case runArgs::index_of(): + { + auto outputs = util::get(args); + EXPECT_EQ(1u, outputs.size()); + auto opt_mat = cv::util::get>(outputs[0]); + ASSERT_TRUE(opt_mat.has_value()); + out_mat = *opt_mat; + break; + } + default: GAPI_Assert(false && "Incorrect type of Args"); + } + + EXPECT_EQ(0., cv::norm(ref, out_mat, cv::NORM_INF)); +} + } // anonymous namespace TEST_P(GAPI_Streaming, SmokeTest_ConstInput_GMat) @@ -1336,13 +1365,45 @@ TEST(Streaming, Python_Pull_Overload) bool has_output; cv::GRunArgs outputs; - std::tie(has_output, outputs) = ccomp.pull(); + using RunArgs = cv::util::variant; + RunArgs args; - EXPECT_TRUE(has_output); - EXPECT_EQ(1u, outputs.size()); + std::tie(has_output, args) = ccomp.pull(); + + checkPullOverload(in_mat, has_output, args); + + ccomp.stop(); + EXPECT_FALSE(ccomp.running()); +} + +TEST(GAPI_Streaming_Desync, Python_Pull_Overload) +{ + cv::GMat in; + cv::GMat out = cv::gapi::streaming::desync(in); + cv::GComputation c(in, out); + + cv::Size sz(3,3); + cv::Mat in_mat(sz, CV_8UC3); + cv::randu(in_mat, cv::Scalar::all(0), cv::Scalar(255)); - auto out_mat = cv::util::get(outputs[0]); - EXPECT_EQ(0., cv::norm(in_mat, out_mat, cv::NORM_INF)); + auto ccomp = c.compileStreaming(); + + EXPECT_TRUE(ccomp); + EXPECT_FALSE(ccomp.running()); + + ccomp.setSource(cv::gin(in_mat)); + + ccomp.start(); + EXPECT_TRUE(ccomp.running()); + + bool has_output; + cv::GRunArgs outputs; + using RunArgs = cv::util::variant; + RunArgs args; + + std::tie(has_output, args) = ccomp.pull(); + + checkPullOverload(in_mat, has_output, args); ccomp.stop(); EXPECT_FALSE(ccomp.running()); @@ -2132,9 +2193,17 @@ TEST(GAPI_Streaming, TestPythonAPI) bool is_over = false; cv::GRunArgs out_args; + using RunArgs = cv::util::variant; + RunArgs args; // NB: Used by python bridge - std::tie(is_over, out_args) = cc.pull(); + std::tie(is_over, args) = cc.pull(); + + switch (args.index()) { + case RunArgs::index_of(): + out_args = util::get(args); break; + default: GAPI_Assert(false && "Incorrect type of return value"); + } ASSERT_EQ(1u, out_args.size()); ASSERT_TRUE(cv::util::holds_alternative(out_args[0])); From 90df3af6cffec4a3255ae0f217471d013fd68492 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 30 Jun 2021 21:41:25 +0000 Subject: [PATCH 035/376] build: winpack_dldt with dldt 2021.4.0 --- ...-dldt-disable-multidevice-autoplugin.patch | 16 ++ ...20210630-dldt-disable-unused-targets.patch | 219 ++++++++++++++++++ .../2021.4/20210630-dldt-pdb.patch | 15 ++ .../2021.4/20210630-dldt-vs-version.patch | 16 ++ platforms/winpack_dldt/2021.4/build.config.py | 1 + platforms/winpack_dldt/2021.4/patch.config.py | 4 + .../winpack_dldt/2021.4/sysroot.config.py | 56 +++++ platforms/winpack_dldt/build_package.py | 14 +- 8 files changed, 336 insertions(+), 5 deletions(-) create mode 100644 platforms/winpack_dldt/2021.4/20210630-dldt-disable-multidevice-autoplugin.patch create mode 100644 platforms/winpack_dldt/2021.4/20210630-dldt-disable-unused-targets.patch create mode 100644 platforms/winpack_dldt/2021.4/20210630-dldt-pdb.patch create mode 100644 platforms/winpack_dldt/2021.4/20210630-dldt-vs-version.patch create mode 100644 platforms/winpack_dldt/2021.4/build.config.py create mode 100644 platforms/winpack_dldt/2021.4/patch.config.py create mode 100644 platforms/winpack_dldt/2021.4/sysroot.config.py diff --git a/platforms/winpack_dldt/2021.4/20210630-dldt-disable-multidevice-autoplugin.patch b/platforms/winpack_dldt/2021.4/20210630-dldt-disable-multidevice-autoplugin.patch new file mode 100644 index 000000000000..f1e748744277 --- /dev/null +++ b/platforms/winpack_dldt/2021.4/20210630-dldt-disable-multidevice-autoplugin.patch @@ -0,0 +1,16 @@ +diff --git a/inference-engine/src/CMakeLists.txt b/inference-engine/src/CMakeLists.txt +index 0ba0dd78..7d34e7cb 100644 +--- a/inference-engine/src/CMakeLists.txt ++++ b/inference-engine/src/CMakeLists.txt +@@ -26,9 +26,9 @@ endif() + + add_subdirectory(hetero_plugin) + +-add_subdirectory(auto_plugin) ++#add_subdirectory(auto_plugin) + +-add_subdirectory(multi_device) ++#add_subdirectory(multi_device) + + add_subdirectory(transformations) + diff --git a/platforms/winpack_dldt/2021.4/20210630-dldt-disable-unused-targets.patch b/platforms/winpack_dldt/2021.4/20210630-dldt-disable-unused-targets.patch new file mode 100644 index 000000000000..9d44cdadc6cd --- /dev/null +++ b/platforms/winpack_dldt/2021.4/20210630-dldt-disable-unused-targets.patch @@ -0,0 +1,219 @@ +diff --git a/cmake/developer_package/add_ie_target.cmake b/cmake/developer_package/add_ie_target.cmake +index d49f16a4d..2726ca787 100644 +--- a/cmake/developer_package/add_ie_target.cmake ++++ b/cmake/developer_package/add_ie_target.cmake +@@ -92,7 +92,7 @@ function(addIeTarget) + if (ARG_TYPE STREQUAL EXECUTABLE) + add_executable(${ARG_NAME} ${all_sources}) + elseif(ARG_TYPE STREQUAL STATIC OR ARG_TYPE STREQUAL SHARED) +- add_library(${ARG_NAME} ${ARG_TYPE} ${all_sources}) ++ add_library(${ARG_NAME} ${ARG_TYPE} EXCLUDE_FROM_ALL ${all_sources}) + else() + message(SEND_ERROR "Invalid target type ${ARG_TYPE} specified for target name ${ARG_NAME}") + endif() +diff --git a/inference-engine/CMakeLists.txt b/inference-engine/CMakeLists.txt +index 1ac7fd8bf..df7091e51 100644 +--- a/inference-engine/CMakeLists.txt ++++ b/inference-engine/CMakeLists.txt +@@ -39,7 +39,7 @@ if(ENABLE_TESTS) + add_subdirectory(tests) + endif() + +-add_subdirectory(tools) ++#add_subdirectory(tools) + + function(ie_build_samples) + # samples should be build with the same flags as from OpenVINO package, +@@ -58,7 +58,7 @@ endfunction() + + # gflags and format_reader targets are kept inside of samples directory and + # they must be built even if samples build is disabled (required for tests and tools). +-ie_build_samples() ++#ie_build_samples() + + if(ENABLE_PYTHON) + add_subdirectory(ie_bridges/python) +@@ -142,7 +142,7 @@ endif() + # Developer package + # + +-openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) ++#openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) + + # for Template plugin + if(NGRAPH_INTERPRETER_ENABLE) +@@ -166,7 +166,7 @@ function(ie_generate_dev_package_config) + @ONLY) + endfunction() + +-ie_generate_dev_package_config() ++#ie_generate_dev_package_config() + + # + # Coverage +diff --git a/inference-engine/src/inference_engine/CMakeLists.txt b/inference-engine/src/inference_engine/CMakeLists.txt +index e8ed1a5c4..1fc9fc3ff 100644 +--- a/inference-engine/src/inference_engine/CMakeLists.txt ++++ b/inference-engine/src/inference_engine/CMakeLists.txt +@@ -110,7 +110,7 @@ add_cpplint_target(${TARGET_NAME}_plugin_api_cpplint FOR_SOURCES ${plugin_api_sr + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS} + ${PUBLIC_HEADERS}) +@@ -181,7 +181,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $ + $ + ${IE_STATIC_DEPENDENT_FILES}) +diff --git a/inference-engine/src/legacy_api/CMakeLists.txt b/inference-engine/src/legacy_api/CMakeLists.txt +index 8eae82bd2..e0e6745b1 100644 +--- a/inference-engine/src/legacy_api/CMakeLists.txt ++++ b/inference-engine/src/legacy_api/CMakeLists.txt +@@ -26,7 +26,7 @@ endif() + + file(TOUCH ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp) + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${PUBLIC_HEADERS}) + +diff --git a/inference-engine/src/mkldnn_plugin/CMakeLists.txt b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +index fe57b29dd..07831e2fb 100644 +--- a/inference-engine/src/mkldnn_plugin/CMakeLists.txt ++++ b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +@@ -67,7 +67,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # add test object library + +-add_library(${TARGET_NAME}_obj OBJECT ${SOURCES} ${HEADERS}) ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL ${SOURCES} ${HEADERS}) + target_link_libraries(${TARGET_NAME}_obj PUBLIC mkldnn) + + target_include_directories(${TARGET_NAME}_obj PRIVATE $ +diff --git a/inference-engine/src/preprocessing/CMakeLists.txt b/inference-engine/src/preprocessing/CMakeLists.txt +index f9548339d..ef962145a 100644 +--- a/inference-engine/src/preprocessing/CMakeLists.txt ++++ b/inference-engine/src/preprocessing/CMakeLists.txt +@@ -101,7 +101,7 @@ endif() + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS}) + +@@ -153,7 +153,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $) + + set_ie_threading_interface_for(${TARGET_NAME}_s) +diff --git a/inference-engine/src/vpu/common/CMakeLists.txt b/inference-engine/src/vpu/common/CMakeLists.txt +index 249e47c28..4ddf63049 100644 +--- a/inference-engine/src/vpu/common/CMakeLists.txt ++++ b/inference-engine/src/vpu/common/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h) + + function(add_common_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + ie_faster_build(${TARGET_NAME} + UNITY +@@ -60,7 +60,7 @@ add_common_target("vpu_common_lib" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_common_target("vpu_common_lib_test_static" TRUE) ++ #add_common_target("vpu_common_lib_test_static" TRUE) + else() + add_library("vpu_common_lib_test_static" ALIAS "vpu_common_lib") + endif() +diff --git a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +index bc73ab5b1..b4c1547fc 100644 +--- a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt ++++ b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h *.inc) + + function(add_graph_transformer_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + set_ie_threading_interface_for(${TARGET_NAME}) + +@@ -70,7 +70,7 @@ add_graph_transformer_target("vpu_graph_transformer" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) ++ #add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) + else() + add_library("vpu_graph_transformer_test_static" ALIAS "vpu_graph_transformer") + endif() +diff --git a/inference-engine/thirdparty/pugixml/CMakeLists.txt b/inference-engine/thirdparty/pugixml/CMakeLists.txt +index 8bcb2801a..f7e031c01 100644 +--- a/inference-engine/thirdparty/pugixml/CMakeLists.txt ++++ b/inference-engine/thirdparty/pugixml/CMakeLists.txt +@@ -41,7 +41,7 @@ if(BUILD_SHARED_LIBS) + else() + add_library(pugixml STATIC ${SOURCES}) + if (MSVC) +- add_library(pugixml_mt STATIC ${SOURCES}) ++ #add_library(pugixml_mt STATIC ${SOURCES}) + #if (WIN32) + # set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") + # set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") +diff --git a/ngraph/core/builder/CMakeLists.txt b/ngraph/core/builder/CMakeLists.txt +index ff5c381e7..2797ec9ab 100644 +--- a/ngraph/core/builder/CMakeLists.txt ++++ b/ngraph/core/builder/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/ngraph/core/reference/CMakeLists.txt b/ngraph/core/reference/CMakeLists.txt +index ef4a764ab..f6d3172e2 100644 +--- a/ngraph/core/reference/CMakeLists.txt ++++ b/ngraph/core/reference/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/openvino/itt/CMakeLists.txt b/openvino/itt/CMakeLists.txt +index e9f880b8c..c63f4df63 100644 +--- a/openvino/itt/CMakeLists.txt ++++ b/openvino/itt/CMakeLists.txt +@@ -6,7 +6,7 @@ set(TARGET_NAME itt) + + file(GLOB_RECURSE SOURCES "src/*.cpp" "src/*.hpp") + +-add_library(${TARGET_NAME} STATIC ${SOURCES}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + add_library(openvino::itt ALIAS ${TARGET_NAME}) + diff --git a/platforms/winpack_dldt/2021.4/20210630-dldt-pdb.patch b/platforms/winpack_dldt/2021.4/20210630-dldt-pdb.patch new file mode 100644 index 000000000000..65e6f84dc80b --- /dev/null +++ b/platforms/winpack_dldt/2021.4/20210630-dldt-pdb.patch @@ -0,0 +1,15 @@ +iff --git a/CMakeLists.txt b/CMakeLists.txt +index e0706a72e..9a053b1e4 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -6,6 +6,10 @@ cmake_minimum_required(VERSION 3.13) + + project(OpenVINO) + ++set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi /FS") ++set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++ + set(OpenVINO_MAIN_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + set(IE_MAIN_SOURCE_DIR ${OpenVINO_MAIN_SOURCE_DIR}/inference-engine) + diff --git a/platforms/winpack_dldt/2021.4/20210630-dldt-vs-version.patch b/platforms/winpack_dldt/2021.4/20210630-dldt-vs-version.patch new file mode 100644 index 000000000000..36b0068775eb --- /dev/null +++ b/platforms/winpack_dldt/2021.4/20210630-dldt-vs-version.patch @@ -0,0 +1,16 @@ +diff --git a/cmake/developer_package/vs_version/vs_version.cmake b/cmake/developer_package/vs_version/vs_version.cmake +index 14d4c0e1e..6a44f73b9 100644 +--- a/cmake/developer_package/vs_version/vs_version.cmake ++++ b/cmake/developer_package/vs_version/vs_version.cmake +@@ -8,9 +8,9 @@ set(IE_VS_VER_FILEVERSION_STR "${IE_VERSION_MAJOR}.${IE_VERSION_MINOR}.${IE_VERS + + set(IE_VS_VER_COMPANY_NAME_STR "Intel Corporation") + set(IE_VS_VER_PRODUCTVERSION_STR "${CI_BUILD_NUMBER}") +-set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit") ++set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit (for OpenCV Windows package)") + set(IE_VS_VER_COPYRIGHT_STR "Copyright (C) 2018-2021, Intel Corporation") +-set(IE_VS_VER_COMMENTS_STR "https://docs.openvinotoolkit.org/") ++set(IE_VS_VER_COMMENTS_STR "https://github.com/opencv/opencv/wiki/Intel%27s-Deep-Learning-Inference-Engine-backend") + + # + # ie_add_vs_version_file(NAME diff --git a/platforms/winpack_dldt/2021.4/build.config.py b/platforms/winpack_dldt/2021.4/build.config.py new file mode 100644 index 000000000000..33ef1050cad4 --- /dev/null +++ b/platforms/winpack_dldt/2021.4/build.config.py @@ -0,0 +1 @@ +os.environ['CI_BUILD_NUMBER'] = '2021.4.0-opencv_winpack_dldt' diff --git a/platforms/winpack_dldt/2021.4/patch.config.py b/platforms/winpack_dldt/2021.4/patch.config.py new file mode 100644 index 000000000000..7f8715aae2da --- /dev/null +++ b/platforms/winpack_dldt/2021.4/patch.config.py @@ -0,0 +1,4 @@ +applyPatch('20210630-dldt-disable-unused-targets.patch') +applyPatch('20210630-dldt-pdb.patch') +applyPatch('20210630-dldt-disable-multidevice-autoplugin.patch') +applyPatch('20210630-dldt-vs-version.patch') diff --git a/platforms/winpack_dldt/2021.4/sysroot.config.py b/platforms/winpack_dldt/2021.4/sysroot.config.py new file mode 100644 index 000000000000..fa4281107d23 --- /dev/null +++ b/platforms/winpack_dldt/2021.4/sysroot.config.py @@ -0,0 +1,56 @@ +sysroot_bin_dir = prepare_dir(self.sysrootdir / 'bin') +copytree(self.build_dir / 'install', self.sysrootdir / 'ngraph') +#rm_one(self.sysrootdir / 'ngraph' / 'lib' / 'ngraph.dll') + +build_config = 'Release' if not self.config.build_debug else 'Debug' +build_bin_dir = self.build_dir / 'bin' / 'intel64' / build_config + +def copy_bin(name): + global build_bin_dir, sysroot_bin_dir + copytree(build_bin_dir / name, sysroot_bin_dir / name) + +dll_suffix = 'd' if self.config.build_debug else '' +def copy_dll(name): + global copy_bin, dll_suffix + copy_bin(name + dll_suffix + '.dll') + copy_bin(name + dll_suffix + '.pdb') + +copy_bin('cache.json') +copy_dll('clDNNPlugin') +copy_dll('HeteroPlugin') +copy_dll('inference_engine') +copy_dll('inference_engine_ir_reader') +#copy_dll('inference_engine_ir_v7_reader') +copy_dll('inference_engine_legacy') +copy_dll('inference_engine_transformations') # runtime +copy_dll('inference_engine_lp_transformations') # runtime +#copy_dll('inference_engine_preproc') # runtime +copy_dll('MKLDNNPlugin') # runtime +copy_dll('myriadPlugin') # runtime +#copy_dll('MultiDevicePlugin') # runtime, not used +copy_dll('ngraph') +copy_bin('plugins.xml') +copy_bin('pcie-ma2x8x.elf') +copy_bin('usb-ma2x8x.mvcmd') + +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb' / 'bin', sysroot_bin_dir) +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb', self.sysrootdir / 'tbb') + +sysroot_ie_dir = prepare_dir(self.sysrootdir / 'deployment_tools' / 'inference_engine') +sysroot_ie_lib_dir = prepare_dir(sysroot_ie_dir / 'lib' / 'intel64') + +copytree(self.srcdir / 'inference-engine' / 'include', sysroot_ie_dir / 'include') +if not self.config.build_debug: + copytree(build_bin_dir / 'ngraph.lib', sysroot_ie_lib_dir / 'ngraph.lib') + copytree(build_bin_dir / 'inference_engine.lib', sysroot_ie_lib_dir / 'inference_engine.lib') + copytree(build_bin_dir / 'inference_engine_ir_reader.lib', sysroot_ie_lib_dir / 'inference_engine_ir_reader.lib') + copytree(build_bin_dir / 'inference_engine_legacy.lib', sysroot_ie_lib_dir / 'inference_engine_legacy.lib') +else: + copytree(build_bin_dir / 'ngraphd.lib', sysroot_ie_lib_dir / 'ngraphd.lib') + copytree(build_bin_dir / 'inference_engined.lib', sysroot_ie_lib_dir / 'inference_engined.lib') + copytree(build_bin_dir / 'inference_engine_ir_readerd.lib', sysroot_ie_lib_dir / 'inference_engine_ir_readerd.lib') + copytree(build_bin_dir / 'inference_engine_legacyd.lib', sysroot_ie_lib_dir / 'inference_engine_legacyd.lib') + +sysroot_license_dir = prepare_dir(self.sysrootdir / 'etc' / 'licenses') +copytree(self.srcdir / 'LICENSE', sysroot_license_dir / 'dldt-LICENSE') +copytree(self.sysrootdir / 'tbb/LICENSE', sysroot_license_dir / 'tbb-LICENSE') diff --git a/platforms/winpack_dldt/build_package.py b/platforms/winpack_dldt/build_package.py index b993f076d901..6fde62241a5e 100644 --- a/platforms/winpack_dldt/build_package.py +++ b/platforms/winpack_dldt/build_package.py @@ -214,7 +214,7 @@ def init_patchset(self): patch_hashsum = hashlib.md5(self.patch_file_contents.encode('utf-8')).hexdigest() except: log.warn("Can't compute hashsum of patches: %s", self.patch_file) - self.patch_hashsum = patch_hashsum + self.patch_hashsum = self.config.override_patch_hashsum if self.config.override_patch_hashsum else patch_hashsum def prepare_sources(self): @@ -355,7 +355,6 @@ def build(self, builderDLDT): BUILD_PERF_TESTS='OFF', ENABLE_CXX11='ON', WITH_INF_ENGINE='ON', - INF_ENGINE_RELEASE=str(self.config.dldt_release), WITH_TBB='ON', CPU_BASELINE='AVX2', CMAKE_INSTALL_PREFIX=str(self.install_dir), @@ -381,6 +380,9 @@ def build(self, builderDLDT): OPENCV_PYTHON_INSTALL_PATH='python', ) + if self.config.dldt_release: + cmake_vars['INF_ENGINE_RELEASE'] = str(self.config.dldt_release) + cmake_vars['INF_ENGINE_LIB_DIRS:PATH'] = str(builderDLDT.sysrootdir / 'deployment_tools/inference_engine/lib/intel64') assert os.path.exists(cmake_vars['INF_ENGINE_LIB_DIRS:PATH']), cmake_vars['INF_ENGINE_LIB_DIRS:PATH'] cmake_vars['INF_ENGINE_INCLUDE_DIRS:PATH'] = str(builderDLDT.sysrootdir / 'deployment_tools/inference_engine/include') @@ -464,8 +466,8 @@ def package_sources(self): def main(): dldt_src_url = 'https://github.com/openvinotoolkit/openvino' - dldt_src_commit = '2021.3' - dldt_release = '2021030000' + dldt_src_commit = '2021.4' + dldt_release = None build_cache_dir_default = os.environ.get('BUILD_CACHE_DIR', '.build_cache') build_subst_drive = os.environ.get('BUILD_SUBST_DRIVE', None) @@ -492,13 +494,15 @@ def main(): parser.add_argument('--dldt_src_branch', help='DLDT checkout branch') parser.add_argument('--dldt_src_commit', default=dldt_src_commit, help='DLDT source commit / tag (default: %s)' % dldt_src_commit) parser.add_argument('--dldt_src_git_clone_extra', action='append', help='DLDT git clone extra args') - parser.add_argument('--dldt_release', default=dldt_release, help='DLDT release code for INF_ENGINE_RELEASE (default: %s)' % dldt_release) + parser.add_argument('--dldt_release', default=dldt_release, help='DLDT release code for INF_ENGINE_RELEASE, e.g 2021030000 (default: %s)' % dldt_release) parser.add_argument('--dldt_reference_dir', help='DLDT reference git repository (optional)') parser.add_argument('--dldt_src_dir', help='DLDT custom source repository (skip git checkout and patching, use for TESTING only)') parser.add_argument('--dldt_config', help='Specify DLDT build configuration (defaults to evaluate from DLDT commit/branch)') + parser.add_argument('--override_patch_hashsum', default='', help='(script debug mode)') + args = parser.parse_args() log.basicConfig( From e5841d3126527ff1151ab480ba8e291e29942e07 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 2 Jul 2021 10:41:41 +0000 Subject: [PATCH 036/376] java: force using of 'Ptr<>' for OpenCV classes --- modules/dnn/misc/java/gen_dict.json | 2 +- modules/java/generator/gen_java.py | 34 +++++++++++-------- .../misc/java/test/TrackerCreateTest.java | 7 ++++ 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/modules/dnn/misc/java/gen_dict.json b/modules/dnn/misc/java/gen_dict.json index 5a397eac51c0..65ecfdc25ea7 100644 --- a/modules/dnn/misc/java/gen_dict.json +++ b/modules/dnn/misc/java/gen_dict.json @@ -54,7 +54,7 @@ ] ], - "jni_name": "(*(cv::dnn::DictValue*)%(n)s_nativeObj)", + "jni_name": "(*(*(Ptr*)%(n)s_nativeObj))", "jni_type": "jlong", "suffix": "J", "j_import": "org.opencv.dnn.DictValue" diff --git a/modules/java/generator/gen_java.py b/modules/java/generator/gen_java.py index 6019ca340d25..c5b4f34a8f2b 100755 --- a/modules/java/generator/gen_java.py +++ b/modules/java/generator/gen_java.py @@ -258,6 +258,8 @@ def __init__(self, decl, namespaces=[]): # [ 'class/struct cname', ': base', [mo for m in decl[2]: if m.startswith("="): self.jname = m[1:] + if m == '/Simple': + self.smart = False if self.classpath: prefix = self.classpath.replace('.', '_') @@ -445,7 +447,7 @@ def __init__(self): def clear(self): self.namespaces = ["cv"] - classinfo_Mat = ClassInfo([ 'class cv.Mat', '', [], [] ], self.namespaces) + classinfo_Mat = ClassInfo([ 'class cv.Mat', '', ['/Simple'], [] ], self.namespaces) self.classes = { "Mat" : classinfo_Mat } self.module = "" self.Module = "" @@ -466,10 +468,15 @@ def add_class(self, decl): if name in type_dict and not classinfo.base: logging.warning('duplicated: %s', classinfo) return + if self.isSmartClass(classinfo): + jni_name = "*((*(Ptr<"+classinfo.fullNameCPP()+">*)%(n)s_nativeObj).get())" + else: + jni_name = "(*("+classinfo.fullNameCPP()+"*)%(n)s_nativeObj)" type_dict.setdefault(name, {}).update( { "j_type" : classinfo.jname, "jn_type" : "long", "jn_args" : (("__int64", ".nativeObj"),), - "jni_name" : "(*("+classinfo.fullNameCPP()+"*)%(n)s_nativeObj)", "jni_type" : "jlong", + "jni_name" : jni_name, + "jni_type" : "jlong", "suffix" : "J", "j_import" : "org.opencv.%s.%s" % (self.module, classinfo.jname) } @@ -477,7 +484,8 @@ def add_class(self, decl): type_dict.setdefault(name+'*', {}).update( { "j_type" : classinfo.jname, "jn_type" : "long", "jn_args" : (("__int64", ".nativeObj"),), - "jni_name" : "("+classinfo.fullNameCPP()+"*)%(n)s_nativeObj", "jni_type" : "jlong", + "jni_name" : "&("+jni_name+")", + "jni_type" : "jlong", "suffix" : "J", "j_import" : "org.opencv.%s.%s" % (self.module, classinfo.jname) } @@ -966,7 +974,13 @@ def gen_func(self, ci, fi, prop_name=''): ret = "return env->NewStringUTF(_retval_.c_str());" default = 'return env->NewStringUTF("");' elif self.isWrapped(fi.ctype): # wrapped class: - ret = "return (jlong) new %s(_retval_);" % self.fullTypeNameCPP(fi.ctype) + ret = None + if fi.ctype in self.classes: + ret_ci = self.classes[fi.ctype] + if self.isSmartClass(ret_ci): + ret = "return (jlong)(new Ptr<%(ctype)s>(new %(ctype)s(_retval_)));" % { 'ctype': ret_ci.fullNameCPP() } + if ret is None: + ret = "return (jlong) new %s(_retval_);" % self.fullTypeNameCPP(fi.ctype) elif fi.ctype.startswith('Ptr_'): c_prologue.append("typedef Ptr<%s> %s;" % (self.fullTypeNameCPP(fi.ctype[4:]), fi.ctype)) ret = "return (jlong)(new %(ctype)s(_retval_));" % { 'ctype':fi.ctype } @@ -1207,17 +1221,7 @@ def isSmartClass(self, ci): if ci.smart != None: return ci.smart - # if parents are smart (we hope) then children are! - # if not we believe the class is smart if it has "create" method - ci.smart = False - if ci.base or ci.name == 'Algorithm': - ci.smart = True - else: - for fi in ci.methods: - if fi.name == "create": - ci.smart = True - break - + ci.smart = True # smart class is not properly handled in case of base/derived classes return ci.smart def smartWrap(self, ci, fullname): diff --git a/modules/video/misc/java/test/TrackerCreateTest.java b/modules/video/misc/java/test/TrackerCreateTest.java index dad696bebfa2..83bbd0b5d5ce 100644 --- a/modules/video/misc/java/test/TrackerCreateTest.java +++ b/modules/video/misc/java/test/TrackerCreateTest.java @@ -1,7 +1,10 @@ package org.opencv.test.video; import org.opencv.core.Core; +import org.opencv.core.CvType; import org.opencv.core.CvException; +import org.opencv.core.Mat; +import org.opencv.core.Rect; import org.opencv.test.OpenCVTestCase; import org.opencv.video.Tracker; @@ -27,6 +30,10 @@ public void testCreateTrackerGOTURN() { public void testCreateTrackerMIL() { Tracker tracker = TrackerMIL.create(); + assert(tracker != null); + Mat mat = new Mat(100, 100, CvType.CV_8UC1); + Rect rect = new Rect(10, 10, 30, 30); + tracker.init(mat, rect); // should not crash (https://github.com/opencv/opencv/issues/19915) } } From 8d1f254dcc2b4f413b1a610ec77b92896702cfca Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 2 Jul 2021 10:41:41 +0000 Subject: [PATCH 037/376] java: force using of 'Ptr<>' for OpenCV classes backport of commit: e5841d3126527ff1151ab480ba8e291e29942e07 --- modules/dnn/misc/java/gen_dict.json | 2 +- modules/java/generator/gen_java.py | 35 ++++++++++++++++------------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/modules/dnn/misc/java/gen_dict.json b/modules/dnn/misc/java/gen_dict.json index 5a397eac51c0..65ecfdc25ea7 100644 --- a/modules/dnn/misc/java/gen_dict.json +++ b/modules/dnn/misc/java/gen_dict.json @@ -54,7 +54,7 @@ ] ], - "jni_name": "(*(cv::dnn::DictValue*)%(n)s_nativeObj)", + "jni_name": "(*(*(Ptr*)%(n)s_nativeObj))", "jni_type": "jlong", "suffix": "J", "j_import": "org.opencv.dnn.DictValue" diff --git a/modules/java/generator/gen_java.py b/modules/java/generator/gen_java.py index 8e5c69e78861..f3b5e132d1c6 100755 --- a/modules/java/generator/gen_java.py +++ b/modules/java/generator/gen_java.py @@ -224,6 +224,9 @@ def __init__(self, decl, namespaces=[]): # [ 'class/struct cname', ': base', [mo for m in decl[2]: if m.startswith("="): self.jname = m[1:] + if m == '/Simple': + self.smart = False + self.base = '' if decl[1]: #self.base = re.sub(r"\b"+self.jname+r"\b", "", decl[1].replace(":", "")).strip() @@ -370,7 +373,7 @@ def __init__(self): def clear(self): self.namespaces = ["cv"] - self.classes = { "Mat" : ClassInfo([ 'class Mat', '', [], [] ], self.namespaces) } + self.classes = { "Mat" : ClassInfo([ 'class Mat', '', ['/Simple'], [] ], self.namespaces) } self.module = "" self.Module = "" self.ported_func_list = [] @@ -390,10 +393,15 @@ def add_class(self, decl): if name in type_dict and not classinfo.base: logging.warning('duplicated: %s', classinfo) return + if self.isSmartClass(classinfo): + jni_name = "*((*(Ptr<"+classinfo.fullName(isCPP=True)+">*)%(n)s_nativeObj).get())" + else: + jni_name = "(*("+classinfo.fullName(isCPP=True)+"*)%(n)s_nativeObj)" type_dict.setdefault(name, {}).update( { "j_type" : classinfo.jname, "jn_type" : "long", "jn_args" : (("__int64", ".nativeObj"),), - "jni_name" : "(*("+classinfo.fullName(isCPP=True)+"*)%(n)s_nativeObj)", "jni_type" : "jlong", + "jni_name" : jni_name, + "jni_type" : "jlong", "suffix" : "J", "j_import" : "org.opencv.%s.%s" % (self.module, classinfo.jname) } @@ -401,7 +409,8 @@ def add_class(self, decl): type_dict.setdefault(name+'*', {}).update( { "j_type" : classinfo.jname, "jn_type" : "long", "jn_args" : (("__int64", ".nativeObj"),), - "jni_name" : "("+classinfo.fullName(isCPP=True)+"*)%(n)s_nativeObj", "jni_type" : "jlong", + "jni_name" : "&("+jni_name+")", + "jni_type" : "jlong", "suffix" : "J", "j_import" : "org.opencv.%s.%s" % (self.module, classinfo.jname) } @@ -889,7 +898,13 @@ def gen_func(self, ci, fi, prop_name=''): ret = "return env->NewStringUTF(_retval_.c_str());" default = 'return env->NewStringUTF("");' elif self.isWrapped(fi.ctype): # wrapped class: - ret = "return (jlong) new %s(_retval_);" % self.fullTypeName(fi.ctype) + ret = None + if fi.ctype in self.classes: + ret_ci = self.classes[fi.ctype] + if self.isSmartClass(ret_ci): + ret = "return (jlong)(new Ptr<%(ctype)s>(new %(ctype)s(_retval_)));" % { 'ctype': self.fullTypeName(fi.ctype) } + if ret is None: + ret = "return (jlong) new %s(_retval_);" % self.fullTypeName(fi.ctype) elif fi.ctype.startswith('Ptr_'): c_prologue.append("typedef Ptr<%s> %s;" % (self.fullTypeName(fi.ctype[4:]), fi.ctype)) ret = "return (jlong)(new %(ctype)s(_retval_));" % { 'ctype':fi.ctype } @@ -1128,17 +1143,7 @@ def isSmartClass(self, ci): if ci.smart != None: return ci.smart - # if parents are smart (we hope) then children are! - # if not we believe the class is smart if it has "create" method - ci.smart = False - if ci.base or ci.name == 'Algorithm': - ci.smart = True - else: - for fi in ci.methods: - if fi.name == "create": - ci.smart = True - break - + ci.smart = True # smart class is not properly handled in case of base/derived classes return ci.smart def smartWrap(self, ci, fullname): From 18dbac203f3b55ad445df26a5728798e26f78633 Mon Sep 17 00:00:00 2001 From: mitruska Date: Fri, 2 Jul 2021 15:29:34 +0200 Subject: [PATCH 038/376] Use explicit version of ngraph NormalizeL2 --- modules/dnn/src/layers/normalize_bbox_layer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dnn/src/layers/normalize_bbox_layer.cpp b/modules/dnn/src/layers/normalize_bbox_layer.cpp index cdaa87bde568..001ea3bd16b0 100644 --- a/modules/dnn/src/layers/normalize_bbox_layer.cpp +++ b/modules/dnn/src/layers/normalize_bbox_layer.cpp @@ -328,7 +328,7 @@ class NormalizeBBoxLayerImpl CV_FINAL : public NormalizeBBoxLayer std::iota(axes_data.begin(), axes_data.end(), 1); } auto axes = std::make_shared(ngraph::element::i64, ngraph::Shape{axes_data.size()}, axes_data); - auto norm = std::make_shared(ieInpNode, axes, epsilon, ngraph::op::EpsMode::ADD); + auto norm = std::make_shared(ieInpNode, axes, epsilon, ngraph::op::EpsMode::ADD); CV_Assert(blobs.empty() || numChannels == blobs[0].total()); std::vector shape(ieInpNode->get_shape().size(), 1); From 9b0d6862c474b4edb012d76492834d58451d5fec Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 2 Jul 2021 21:37:37 +0000 Subject: [PATCH 039/376] cmake(IE): extract INF_ENGINE_RELEASE from InferenceEngine package --- cmake/OpenCVDetectInferenceEngine.cmake | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/cmake/OpenCVDetectInferenceEngine.cmake b/cmake/OpenCVDetectInferenceEngine.cmake index 829ddbfe7e2a..6308d1b42480 100644 --- a/cmake/OpenCVDetectInferenceEngine.cmake +++ b/cmake/OpenCVDetectInferenceEngine.cmake @@ -140,12 +140,21 @@ endif() # Add more features to the target if(INF_ENGINE_TARGET) - if(NOT INF_ENGINE_RELEASE) + if(DEFINED InferenceEngine_VERSION) + message(STATUS "InferenceEngine: ${InferenceEngine_VERSION}") + if(NOT INF_ENGINE_RELEASE AND NOT (InferenceEngine_VERSION VERSION_LESS "2021.4")) + math(EXPR INF_ENGINE_RELEASE_INIT "${InferenceEngine_VERSION_MAJOR} * 1000000 + ${InferenceEngine_VERSION_MINOR} * 10000 + ${InferenceEngine_VERSION_PATCH} * 100") + endif() + endif() + if(NOT INF_ENGINE_RELEASE AND NOT INF_ENGINE_RELEASE_INIT) message(WARNING "InferenceEngine version has not been set, 2021.4 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") + set(INF_ENGINE_RELEASE_INIT "2021040000") + elseif(DEFINED INF_ENGINE_RELEASE) + set(INF_ENGINE_RELEASE_INIT "${INF_ENGINE_RELEASE}") endif() - set(INF_ENGINE_RELEASE "2021040000" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") + set(INF_ENGINE_RELEASE "${INF_ENGINE_RELEASE_INIT}" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") set_target_properties(${INF_ENGINE_TARGET} PROPERTIES - INTERFACE_COMPILE_DEFINITIONS "HAVE_INF_ENGINE=1;INF_ENGINE_RELEASE=${INF_ENGINE_RELEASE}" + INTERFACE_COMPILE_DEFINITIONS "HAVE_INF_ENGINE=1;INF_ENGINE_RELEASE=${INF_ENGINE_RELEASE}" ) endif() From 0e523618a17a99647fc13a2be1577b9c887d6d64 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 3 Jul 2021 10:57:18 +0000 Subject: [PATCH 040/376] cmake: exclude -pthread from Emscripten default build --- cmake/OpenCVCompilerOptions.cmake | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmake/OpenCVCompilerOptions.cmake b/cmake/OpenCVCompilerOptions.cmake index 0a10bfffcfed..303d4f451e64 100644 --- a/cmake/OpenCVCompilerOptions.cmake +++ b/cmake/OpenCVCompilerOptions.cmake @@ -177,7 +177,13 @@ if(CV_GCC OR CV_CLANG) endif() # We need pthread's - if(UNIX AND NOT ANDROID AND NOT (APPLE AND CV_CLANG)) # TODO + if((UNIX + AND NOT ANDROID + AND NOT (APPLE AND CV_CLANG) + AND NOT EMSCRIPTEN + ) + OR (EMSCRIPTEN AND WITH_PTHREADS_PF) # https://github.com/opencv/opencv/issues/20285 + ) add_extra_compiler_option(-pthread) endif() From 5d0cfa252797e115010eec269043f958c3305a51 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 3 Jul 2021 11:36:29 +0000 Subject: [PATCH 041/376] cmake(highgui): don't allow multiple builtin backends --- modules/highgui/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index 5eb9f5ab5e6b..bc31b84c74e1 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -141,7 +141,7 @@ endif() if(TARGET ocv.3rdparty.win32ui) if("win32ui" IN_LIST HIGHGUI_PLUGIN_LIST OR HIGHGUI_PLUGIN_LIST STREQUAL "all") ocv_create_builtin_highgui_plugin(opencv_highgui_win32 ocv.3rdparty.win32ui "window_w32.cpp") - else() + elseif(NOT OPENCV_HIGHGUI_BUILTIN_BACKEND) set(OPENCV_HIGHGUI_BUILTIN_BACKEND "WIN32UI") list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_w32.cpp) list(APPEND tgts ocv.3rdparty.win32ui) From 4c3f9b2ef4ca1a74c0fd15f1747bd131e249c57f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sun, 4 Jul 2021 13:07:34 +0300 Subject: [PATCH 042/376] cmake: update Halide detection --- cmake/OpenCVDetectHalide.cmake | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/cmake/OpenCVDetectHalide.cmake b/cmake/OpenCVDetectHalide.cmake index 790f69205662..4828c299aead 100644 --- a/cmake/OpenCVDetectHalide.cmake +++ b/cmake/OpenCVDetectHalide.cmake @@ -9,9 +9,14 @@ set(HALIDE_ROOT_DIR "${HALIDE_ROOT_DIR}" CACHE PATH "Halide root directory") if(NOT HAVE_HALIDE) find_package(Halide QUIET) # Try CMake-based config files if(Halide_FOUND) - set(HALIDE_INCLUDE_DIRS "${Halide_INCLUDE_DIRS}" CACHE PATH "Halide include directories" FORCE) - set(HALIDE_LIBRARIES "${Halide_LIBRARIES}" CACHE PATH "Halide libraries" FORCE) - set(HAVE_HALIDE TRUE) + if(TARGET Halide::Halide) # modern Halide scripts defines imported target + set(HALIDE_INCLUDE_DIRS "") + set(HALIDE_LIBRARIES "Halide::Halide") + set(HAVE_HALIDE TRUE) + else() + # using HALIDE_INCLUDE_DIRS / Halide_LIBRARIES + set(HAVE_HALIDE TRUE) + endif() endif() endif() @@ -28,18 +33,15 @@ if(NOT HAVE_HALIDE AND HALIDE_ROOT_DIR) ) if(HALIDE_LIBRARY AND HALIDE_INCLUDE_DIR) # TODO try_compile - set(HALIDE_INCLUDE_DIRS "${HALIDE_INCLUDE_DIR}" CACHE PATH "Halide include directories" FORCE) - set(HALIDE_LIBRARIES "${HALIDE_LIBRARY}" CACHE PATH "Halide libraries" FORCE) + set(HALIDE_INCLUDE_DIRS "${HALIDE_INCLUDE_DIR}") + set(HALIDE_LIBRARIES "${HALIDE_LIBRARY}") set(HAVE_HALIDE TRUE) endif() - if(NOT HAVE_HALIDE) - ocv_clear_vars(HALIDE_LIBRARIES HALIDE_INCLUDE_DIRS CACHE) - endif() endif() if(HAVE_HALIDE) - include_directories(${HALIDE_INCLUDE_DIRS}) + if(HALIDE_INCLUDE_DIRS) + include_directories(${HALIDE_INCLUDE_DIRS}) + endif() list(APPEND OPENCV_LINKER_LIBS ${HALIDE_LIBRARIES}) -else() - ocv_clear_vars(HALIDE_INCLUDE_DIRS HALIDE_LIBRARIES) endif() From cbff19ff1a8be873c3c2aa0cafa2ff77738cc437 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sun, 4 Jul 2021 17:33:18 +0300 Subject: [PATCH 043/376] highgui: fix win32 backend behavior --- modules/highgui/src/window.cpp | 2 +- modules/highgui/src/window_w32.cpp | 25 +++++++++++++++++-------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/modules/highgui/src/window.cpp b/modules/highgui/src/window.cpp index d1ccd1dbc3a9..d9481de6da24 100644 --- a/modules/highgui/src/window.cpp +++ b/modules/highgui/src/window.cpp @@ -990,7 +990,7 @@ void cv::imshow( const String& winname, InputArray _img ) auto backend = getCurrentUIBackend(); if (backend) { - auto window = backend->createWindow(winname, WINDOW_NORMAL); + auto window = backend->createWindow(winname, WINDOW_AUTOSIZE); if (!window) { CV_LOG_ERROR(NULL, "OpenCV/UI: Can't create window: '" << winname << "'"); diff --git a/modules/highgui/src/window_w32.cpp b/modules/highgui/src/window_w32.cpp index d9a9d732227a..716af1094c29 100644 --- a/modules/highgui/src/window_w32.cpp +++ b/modules/highgui/src/window_w32.cpp @@ -2123,7 +2123,7 @@ static void showSaveDialog(CvWindow& window) SIZE sz; int channels; void* data; - if (icvGetBitmapData(window, sz, channels, data)) + if (!icvGetBitmapData(window, sz, channels, data)) return; // nothing to save char szFileName[MAX_PATH] = ""; @@ -2206,6 +2206,7 @@ static bool handleMessage(MSG& message, int& keyCode) switch (message.message) { case WM_DESTROY: + // fallthru case WM_CHAR: DispatchMessage(&message); keyCode = (int)message.wParam; @@ -2221,6 +2222,20 @@ static bool handleMessage(MSG& message, int& keyCode) break; case WM_KEYDOWN: + // Intercept Ctrl+C for copy to clipboard + if ('C' == message.wParam && (::GetKeyState(VK_CONTROL) >> 15)) + { + ::SendMessage(message.hwnd, WM_COPY, 0, 0); + return false; + } + + // Intercept Ctrl+S for "save as" dialog + if ('S' == message.wParam && (::GetKeyState(VK_CONTROL) >> 15)) + { + showSaveDialog(window); + return false; + } + TranslateMessage(&message); if ((message.wParam >= VK_F1 && message.wParam <= VK_F24) || message.wParam == VK_HOME || message.wParam == VK_END || @@ -2235,13 +2250,7 @@ static bool handleMessage(MSG& message, int& keyCode) return true; } - // Intercept Ctrl+C for copy to clipboard - if ('C' == message.wParam && (::GetKeyState(VK_CONTROL) >> 15)) - ::SendMessage(message.hwnd, WM_COPY, 0, 0); - - // Intercept Ctrl+S for "save as" dialog - if ('S' == message.wParam && (::GetKeyState(VK_CONTROL) >> 15)) - showSaveDialog(window); + // fallthru default: DispatchMessage(&message); From 591708903b4393b5d33c3a2f4af5be4daeb94c4d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sun, 4 Jul 2021 21:10:13 +0000 Subject: [PATCH 044/376] release: OpenCV 3.4.15 --- modules/core/include/opencv2/core/version.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index a71c13ebd1c4..ecd8e733c7b6 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -8,7 +8,7 @@ #define CV_VERSION_MAJOR 3 #define CV_VERSION_MINOR 4 #define CV_VERSION_REVISION 15 -#define CV_VERSION_STATUS "-pre" +#define CV_VERSION_STATUS "" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) From ad6e82942b37be8ee2c71c1d9bc7fe79cd16f7ab Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 5 Jul 2021 12:03:22 +0000 Subject: [PATCH 045/376] release: OpenCV 4.5.3 --- modules/core/include/opencv2/core/version.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index 4757e30f9b30..c9f4b7ed1165 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -8,7 +8,7 @@ #define CV_VERSION_MAJOR 4 #define CV_VERSION_MINOR 5 #define CV_VERSION_REVISION 3 -#define CV_VERSION_STATUS "-pre" +#define CV_VERSION_STATUS "" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) From ed2a69839293625ecae9f2b9e53eeefc23ceedd6 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Tue, 6 Jul 2021 21:35:41 +0300 Subject: [PATCH 046/376] Merge pull request #20359 from mpashchenkov:mp/onnx-tests G-API: ONNX. Skip tests. * imread for every test * Changed name for Yolo function --- .../gapi/test/infer/gapi_infer_onnx_test.cpp | 107 ++++++++++-------- 1 file changed, 62 insertions(+), 45 deletions(-) diff --git a/modules/gapi/test/infer/gapi_infer_onnx_test.cpp b/modules/gapi/test/infer/gapi_infer_onnx_test.cpp index ef192b9d6a96..b1bf9c935694 100644 --- a/modules/gapi/test/infer/gapi_infer_onnx_test.cpp +++ b/modules/gapi/test/infer/gapi_infer_onnx_test.cpp @@ -67,17 +67,17 @@ struct ONNXInitPath { static ONNXInitPath g_init_path; cv::Mat initMatrixRandU(const int type, const cv::Size& sz_in) { - const cv::Mat in_mat1 = cv::Mat(sz_in, type); + const cv::Mat in_mat = cv::Mat(sz_in, type); if (CV_MAT_DEPTH(type) < CV_32F) { - cv::randu(in_mat1, cv::Scalar::all(0), cv::Scalar::all(255)); + cv::randu(in_mat, cv::Scalar::all(0), cv::Scalar::all(255)); } else { const int fscale = 256; // avoid bits near ULP, generate stable test input - cv::Mat in_mat32s(in_mat1.size(), CV_MAKE_TYPE(CV_32S, CV_MAT_CN(type))); + cv::Mat in_mat32s(in_mat.size(), CV_MAKE_TYPE(CV_32S, CV_MAT_CN(type))); cv::randu(in_mat32s, cv::Scalar::all(0), cv::Scalar::all(255 * fscale)); - in_mat32s.convertTo(in_mat1, type, 1.0f / fscale, 0); + in_mat32s.convertTo(in_mat, type, 1.0f / fscale, 0); } - return in_mat1; + return in_mat; } } // anonymous namespace namespace opencv_test @@ -319,15 +319,13 @@ class ONNXtest : public ::testing::Test { size_t num_in, num_out; std::vector out_gapi; std::vector out_onnx; - cv::Mat in_mat1; + cv::Mat in_mat; ONNXtest() { initTestDataPath(); env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "test"); memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); out_gapi.resize(1); - // FIXME: It should be an image from own (gapi) directory in opencv extra - in_mat1 = cv::imread(findDataFile("cv/dpm/cat.png")); } template @@ -463,13 +461,9 @@ class ONNXMediaFrame : public ONNXClassification { cv::Rect(cv::Point{70, 10}, cv::Size{20, 260}), cv::Rect(cv::Point{5, 15}, cv::Size{200, 160}), }; - cv::Mat m_in_y; - cv::Mat m_in_uv; - virtual void SetUp() { - cv::Size sz{640, 480}; - m_in_y = initMatrixRandU(CV_8UC1, sz); - m_in_uv = initMatrixRandU(CV_8UC2, sz / 2); - } + const cv::Size sz{640, 480}; + const cv::Mat m_in_y = initMatrixRandU(CV_8UC1, sz); + const cv::Mat m_in_uv = initMatrixRandU(CV_8UC2, sz / 2); }; class ONNXGRayScale : public ONNXtest { @@ -545,20 +539,20 @@ class ONNXYoloV3 : public ONNXWithRemap { public: std::vector ins; -private: - virtual void SetUp() { + void constructYoloInputs(const cv::Mat& src) { const int yolo_in_h = 416; const int yolo_in_w = 416; cv::Mat yolov3_input, shape, prep_mat; - cv::resize(in_mat1, yolov3_input, cv::Size(yolo_in_w, yolo_in_h)); + cv::resize(src, yolov3_input, cv::Size(yolo_in_w, yolo_in_h)); shape.create(cv::Size(2, 1), CV_32F); float* ptr = shape.ptr(); - ptr[0] = in_mat1.cols; - ptr[1] = in_mat1.rows; + ptr[0] = src.cols; + ptr[1] = src.rows; preprocess(yolov3_input, prep_mat); ins = {prep_mat, shape}; } +private: void preprocess(const cv::Mat& src, cv::Mat& dst) { cv::Mat cvt; src.convertTo(cvt, CV_32F, 1.f / 255.f); @@ -571,9 +565,10 @@ class ONNXYoloV3 : public ONNXWithRemap { TEST_F(ONNXClassification, Infer) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code cv::Mat processed_mat; - preprocess(in_mat1, processed_mat); + preprocess(in_mat, processed_mat); infer(processed_mat, out_onnx); // G_API code G_API_NET(SqueezNet, , "squeeznet"); @@ -583,7 +578,7 @@ TEST_F(ONNXClassification, Infer) // NOTE: We have to normalize U8 tensor // so cfgMeanStd() is here auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({ mean }, { std }); - comp.apply(cv::gin(in_mat1), + comp.apply(cv::gin(in_mat), cv::gout(out_gapi.front()), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -593,9 +588,10 @@ TEST_F(ONNXClassification, Infer) TEST_F(ONNXClassification, InferTensor) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // Create tensor cv::Mat tensor; - preprocess(in_mat1, tensor); + preprocess(in_mat, tensor); // ONNX_API code infer(tensor, out_onnx); // G_API code @@ -614,10 +610,11 @@ TEST_F(ONNXClassification, InferTensor) TEST_F(ONNXClassification, InferROI) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); const auto ROI = rois.at(0); // ONNX_API code cv::Mat roi_mat; - preprocess(in_mat1(ROI), roi_mat); + preprocess(in_mat(ROI), roi_mat); infer(roi_mat, out_onnx); // G_API code G_API_NET(SqueezNet, , "squeeznet"); @@ -628,7 +625,7 @@ TEST_F(ONNXClassification, InferROI) // NOTE: We have to normalize U8 tensor // so cfgMeanStd() is here auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({ mean }, { std }); - comp.apply(cv::gin(in_mat1, ROI), + comp.apply(cv::gin(in_mat, ROI), cv::gout(out_gapi.front()), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -638,10 +635,11 @@ TEST_F(ONNXClassification, InferROI) TEST_F(ONNXClassification, InferROIList) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code for (size_t i = 0; i < rois.size(); ++i) { cv::Mat roi_mat; - preprocess(in_mat1(rois[i]), roi_mat); + preprocess(in_mat(rois[i]), roi_mat); infer(roi_mat, out_onnx); } // G_API code @@ -653,7 +651,7 @@ TEST_F(ONNXClassification, InferROIList) // NOTE: We have to normalize U8 tensor // so cfgMeanStd() is here auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({ mean }, { std }); - comp.apply(cv::gin(in_mat1, rois), + comp.apply(cv::gin(in_mat, rois), cv::gout(out_gapi), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -663,10 +661,11 @@ TEST_F(ONNXClassification, InferROIList) TEST_F(ONNXClassification, Infer2ROIList) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code for (size_t i = 0; i < rois.size(); ++i) { cv::Mat roi_mat; - preprocess(in_mat1(rois[i]), roi_mat); + preprocess(in_mat(rois[i]), roi_mat); infer(roi_mat, out_onnx); } // G_API code @@ -678,7 +677,7 @@ TEST_F(ONNXClassification, Infer2ROIList) // NOTE: We have to normalize U8 tensor // so cfgMeanStd() is here auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({ mean }, { std }); - comp.apply(cv::gin(in_mat1, rois), + comp.apply(cv::gin(in_mat, rois), cv::gout(out_gapi), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -688,9 +687,10 @@ TEST_F(ONNXClassification, Infer2ROIList) TEST_F(ONNXWithRemap, InferDynamicInputTensor) { useModel("object_detection_segmentation/tiny-yolov2/model/tinyyolov2-8"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // Create tensor cv::Mat cvt, rsz, tensor; - cv::resize(in_mat1, rsz, cv::Size{416, 416}); + cv::resize(in_mat, rsz, cv::Size{416, 416}); rsz.convertTo(cvt, CV_32F, 1.f / 255.f); toCHW(cvt, tensor); tensor = tensor.reshape(1, {1, 3, 416, 416}); @@ -714,9 +714,10 @@ TEST_F(ONNXWithRemap, InferDynamicInputTensor) TEST_F(ONNXGRayScale, InferImage) { useModel("body_analysis/emotion_ferplus/model/emotion-ferplus-8"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code cv::Mat prep_mat; - preprocess(in_mat1, prep_mat); + preprocess(in_mat, prep_mat); infer(prep_mat, out_onnx); // G_API code G_API_NET(EmotionNet, , "emotion-ferplus"); @@ -725,7 +726,7 @@ TEST_F(ONNXGRayScale, InferImage) cv::GComputation comp(cv::GIn(in), cv::GOut(out)); auto net = cv::gapi::onnx::Params { model_path } .cfgNormalize({ false }); // model accepts 0..255 range in FP32; - comp.apply(cv::gin(in_mat1), + comp.apply(cv::gin(in_mat), cv::gout(out_gapi.front()), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -735,8 +736,9 @@ TEST_F(ONNXGRayScale, InferImage) TEST_F(ONNXWithRemap, InferMultiOutput) { useModel("object_detection_segmentation/ssd-mobilenetv1/model/ssd_mobilenet_v1_10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code - const auto prep_mat = in_mat1.reshape(1, {1, in_mat1.rows, in_mat1.cols, in_mat1.channels()}); + const auto prep_mat = in_mat.reshape(1, {1, in_mat.rows, in_mat.cols, in_mat.channels()}); infer(prep_mat, out_onnx); cv::Mat onnx_conv_out({1, 1, 200, 7}, CV_32F); remapToIESSDOut({out_onnx[3], out_onnx[0], out_onnx[2], out_onnx[1]}, onnx_conv_out); @@ -750,7 +752,7 @@ TEST_F(ONNXWithRemap, InferMultiOutput) auto net = cv::gapi::onnx::Params{ model_path } .cfgOutputLayers({"detection_output"}) .cfgPostProc({cv::GMatDesc{CV_32F, {1, 1, 200, 7}}}, remapSSDPorts); - comp.apply(cv::gin(in_mat1), + comp.apply(cv::gin(in_mat), cv::gout(out_gapi.front()), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -760,12 +762,13 @@ TEST_F(ONNXWithRemap, InferMultiOutput) TEST_F(ONNXMediaFrame, InferBGR) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code cv::Mat processed_mat; - preprocess(in_mat1, processed_mat); + preprocess(in_mat, processed_mat); infer(processed_mat, out_onnx); // G_API code - auto frame = MediaFrame::Create(in_mat1); + auto frame = MediaFrame::Create(in_mat); G_API_NET(SqueezNet, , "squeeznet"); cv::GFrame in; cv::GMat out = cv::gapi::infer(in); @@ -783,6 +786,7 @@ TEST_F(ONNXMediaFrame, InferBGR) TEST_F(ONNXMediaFrame, InferYUV) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); const auto frame = MediaFrame::Create(m_in_y, m_in_uv); // ONNX_API code cv::Mat pp; @@ -808,10 +812,11 @@ TEST_F(ONNXMediaFrame, InferYUV) TEST_F(ONNXMediaFrame, InferROIBGR) { useModel("classification/squeezenet/model/squeezenet1.0-9"); - auto frame = MediaFrame::Create(in_mat1); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); + auto frame = MediaFrame::Create(in_mat); // ONNX_API code cv::Mat roi_mat; - preprocess(in_mat1(rois.front()), roi_mat); + preprocess(in_mat(rois.front()), roi_mat); infer(roi_mat, out_onnx); // G_API code G_API_NET(SqueezNet, , "squeeznet"); @@ -832,6 +837,7 @@ TEST_F(ONNXMediaFrame, InferROIBGR) TEST_F(ONNXMediaFrame, InferROIYUV) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); const auto frame = MediaFrame::Create(m_in_y, m_in_uv); // ONNX_API code cv::Mat pp; @@ -858,11 +864,12 @@ TEST_F(ONNXMediaFrame, InferROIYUV) TEST_F(ONNXMediaFrame, InferListBGR) { useModel("classification/squeezenet/model/squeezenet1.0-9"); - const auto frame = MediaFrame::Create(in_mat1); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); + const auto frame = MediaFrame::Create(in_mat); // ONNX_API code for (size_t i = 0; i < rois.size(); ++i) { cv::Mat roi_mat; - preprocess(in_mat1(rois[i]), roi_mat); + preprocess(in_mat(rois[i]), roi_mat); infer(roi_mat, out_onnx); } // G_API code @@ -884,6 +891,7 @@ TEST_F(ONNXMediaFrame, InferListBGR) TEST_F(ONNXMediaFrame, InferListYUV) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); const auto frame = MediaFrame::Create(m_in_y, m_in_uv); // ONNX_API code cv::Mat pp; @@ -911,8 +919,9 @@ TEST_F(ONNXMediaFrame, InferListYUV) TEST_F(ONNXRCNN, InferWithDisabledOut) { useModel("object_detection_segmentation/faster-rcnn/model/FasterRCNN-10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); cv::Mat pp; - preprocess(in_mat1, pp); + preprocess(in_mat, pp); // ONNX_API code infer(pp, out_onnx, {"6379", "6383"}); // G_API code @@ -937,11 +946,12 @@ TEST_F(ONNXRCNN, InferWithDisabledOut) TEST_F(ONNXMediaFrame, InferList2BGR) { useModel("classification/squeezenet/model/squeezenet1.0-9"); - const auto frame = MediaFrame::Create(in_mat1); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); + const auto frame = MediaFrame::Create(in_mat); // ONNX_API code for (size_t i = 0; i < rois.size(); ++i) { cv::Mat roi_mat; - preprocess(in_mat1(rois[i]), roi_mat); + preprocess(in_mat(rois[i]), roi_mat); infer(roi_mat, out_onnx); } // G_API code @@ -963,6 +973,7 @@ TEST_F(ONNXMediaFrame, InferList2BGR) TEST_F(ONNXMediaFrame, InferList2YUV) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); const auto frame = MediaFrame::Create(m_in_y, m_in_uv); // ONNX_API code cv::Mat pp; @@ -991,6 +1002,8 @@ TEST_F(ONNXMediaFrame, InferList2YUV) TEST_F(ONNXYoloV3, InferConstInput) { useModel("object_detection_segmentation/yolov3/model/yolov3-10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); + constructYoloInputs(in_mat); // ONNX_API code infer(ins, out_onnx); // G_API code @@ -1022,6 +1035,8 @@ TEST_F(ONNXYoloV3, InferBSConstInput) // and all input layer names are specified. // Const input has the advantage. It is expected behavior. useModel("object_detection_segmentation/yolov3/model/yolov3-10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); + constructYoloInputs(in_mat); // Tensor with incorrect image size // is used for check case when InputLayers and constInput have same names cv::Mat bad_shape; @@ -1059,8 +1074,9 @@ TEST_F(ONNXYoloV3, InferBSConstInput) TEST_F(ONNXRCNN, ConversionInt64to32) { useModel("object_detection_segmentation/faster-rcnn/model/FasterRCNN-10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); cv::Mat dst; - preprocess(in_mat1, dst); + preprocess(in_mat, dst); // ONNX_API code infer(dst, out_onnx); // G_API code @@ -1087,6 +1103,7 @@ TEST_F(ONNXRCNN, ConversionInt64to32) TEST_F(ONNXWithRemap, InferOutReallocation) { useModel("object_detection_segmentation/ssd-mobilenetv1/model/ssd_mobilenet_v1_10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // G_API code G_API_NET(MobileNet, , "ssd_mobilenet"); auto net = cv::gapi::onnx::Params{model_path} @@ -1096,7 +1113,7 @@ TEST_F(ONNXWithRemap, InferOutReallocation) cv::GMat out1; out1 = cv::gapi::infer(in); cv::GComputation comp(cv::GIn(in), cv::GOut(out1)); - EXPECT_THROW(comp.apply(cv::gin(in_mat1), + EXPECT_THROW(comp.apply(cv::gin(in_mat), cv::gout(out_gapi[0]), cv::compile_args(cv::gapi::networks(net))), std::exception); } From 5627a0cbdf94a053e55aa4749a5c83d8b18ad34c Mon Sep 17 00:00:00 2001 From: Xinguang Bian Date: Wed, 7 Jul 2021 12:35:11 +0800 Subject: [PATCH 047/376] fix scale problem in DefaultViewPort::controlImagePosition() --- modules/highgui/src/window_QT.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/highgui/src/window_QT.cpp b/modules/highgui/src/window_QT.cpp index 68289eb87620..8dff03117e1b 100644 --- a/modules/highgui/src/window_QT.cpp +++ b/modules/highgui/src/window_QT.cpp @@ -2883,18 +2883,19 @@ inline bool DefaultViewPort::isSameSize(IplImage* img1, IplImage* img2) void DefaultViewPort::controlImagePosition() { qreal left, top, right, bottom; + qreal factor = 1.0 / param_matrixWorld.m11(); //after check top-left, bottom right corner to avoid getting "out" during zoom/panning param_matrixWorld.map(0,0,&left,&top); if (left > 0) { - param_matrixWorld.translate(-left,0); + param_matrixWorld.translate(-left * factor, 0); left = 0; } if (top > 0) { - param_matrixWorld.translate(0,-top); + param_matrixWorld.translate(0, -top * factor); top = 0; } //------- @@ -2903,12 +2904,12 @@ void DefaultViewPort::controlImagePosition() param_matrixWorld.map(sizeImage.width(),sizeImage.height(),&right,&bottom); if (right < sizeImage.width()) { - param_matrixWorld.translate(sizeImage.width()-right,0); + param_matrixWorld.translate((sizeImage.width() - right) * factor, 0); right = sizeImage.width(); } if (bottom < sizeImage.height()) { - param_matrixWorld.translate(0,sizeImage.height()-bottom); + param_matrixWorld.translate(0, (sizeImage.height() - bottom) * factor); bottom = sizeImage.height(); } From c0f63eb21f044415bf7a21cb98fa6f4ae2586e1b Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Wed, 7 Jul 2021 15:33:40 +0300 Subject: [PATCH 048/376] Merge pull request #20039 from sivanov-work:gapi_empty_input G-API: Implement variant visit() * Add variant visitor, use visitor for check compile args * Fix GAPI UT: variant *compiler * Aling apply_visior with std, fix indentations * Fix compilation (included compiler_hints.hpp) * Fix compilation (due gapi standalone) * Fix compilation2 (Docs) * Add Lambdas overload, Refactor visit() * Add ReturnType auto deduction * Fix comilation * Fix compilation * Fix warnings * Try to fix MSVC14 * Fix docs * Try fix Win compile * Fix Docs again * Revert GAPI empty input fix * Apply comment for `tuple_element` * Add std::decay for std::base_of to work arounf armv7 problem * Apply review comments * Apply review comments: added comment & removed unused args * Fix docs compilation --- .../include/opencv2/gapi/gtype_traits.hpp | 13 - modules/gapi/include/opencv2/gapi/gtyped.hpp | 1 - .../gapi/include/opencv2/gapi/util/util.hpp | 37 +++ .../include/opencv2/gapi/util/variant.hpp | 226 +++++++++++++++- modules/gapi/src/api/gproto.cpp | 5 - modules/gapi/test/util/variant_tests.cpp | 250 ++++++++++++++++++ 6 files changed, 512 insertions(+), 20 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/gtype_traits.hpp b/modules/gapi/include/opencv2/gapi/gtype_traits.hpp index 0b11b18485c0..2e8dcb1aec7d 100644 --- a/modules/gapi/include/opencv2/gapi/gtype_traits.hpp +++ b/modules/gapi/include/opencv2/gapi/gtype_traits.hpp @@ -43,19 +43,6 @@ namespace detail GOPAQUE, // a cv::GOpaqueU (note - exactly GOpaqueU, not GOpaque!) }; - template - constexpr const char* meta_to_string() noexcept; - template<> - constexpr const char* meta_to_string() noexcept { return "GMatDesc"; } - template<> - constexpr const char* meta_to_string() noexcept { return "GScalarDesc"; } - template<> - constexpr const char* meta_to_string() noexcept { return "GArrayDesc"; } - template<> - constexpr const char* meta_to_string() noexcept { return "GOpaqueDesc"; } - template<> - constexpr const char* meta_to_string() noexcept { return "GFrameDesc";} - // Describe G-API types (G-types) with traits. Mostly used by // cv::GArg to store meta information about types passed into // operation arguments. Please note that cv::GComputation is diff --git a/modules/gapi/include/opencv2/gapi/gtyped.hpp b/modules/gapi/include/opencv2/gapi/gtyped.hpp index 27d977794454..6fe52a62e16b 100644 --- a/modules/gapi/include/opencv2/gapi/gtyped.hpp +++ b/modules/gapi/include/opencv2/gapi/gtyped.hpp @@ -35,7 +35,6 @@ namespace detail template<> struct ProtoToMeta { using type = cv::GScalarDesc; }; template struct ProtoToMeta > { using type = cv::GArrayDesc; }; template struct ProtoToMeta > { using type = cv::GOpaqueDesc; }; - template<> struct ProtoToMeta { using type = cv::GFrameDesc; }; template using ProtoToMetaT = typename ProtoToMeta::type; //workaround for MSVC 19.0 bug diff --git a/modules/gapi/include/opencv2/gapi/util/util.hpp b/modules/gapi/include/opencv2/gapi/util/util.hpp index afcf5596fd60..c6ad0632e268 100644 --- a/modules/gapi/include/opencv2/gapi/util/util.hpp +++ b/modules/gapi/include/opencv2/gapi/util/util.hpp @@ -117,6 +117,43 @@ namespace detail static type get(std::tuple&& objs) { return std::forward>(objs); } }; } // namespace detail + +namespace util +{ +template +struct overload_lamba_set; + +template +struct overload_lamba_set : public L1 +{ + overload_lamba_set(L1&& lambda) : L1(std::move(lambda)) {} + overload_lamba_set(const L1& lambda) : L1(lambda) {} + + using L1::operator(); +}; + +template +struct overload_lamba_set : public L1, public overload_lamba_set +{ + using base_type = overload_lamba_set; + overload_lamba_set(L1 &&lambda1, L&& ...lambdas): + L1(std::move(lambda1)), + base_type(std::forward(lambdas)...) {} + + overload_lamba_set(const L1 &lambda1, L&& ...lambdas): + L1(lambda1), + base_type(std::forward(lambdas)...) {} + + using L1::operator(); + using base_type::operator(); +}; + +template +overload_lamba_set overload_lambdas(L&& ...lambdas) +{ + return overload_lamba_set(std::forward(lambdas)...); +} +} } // namespace cv // \endcond diff --git a/modules/gapi/include/opencv2/gapi/util/variant.hpp b/modules/gapi/include/opencv2/gapi/util/variant.hpp index 71a06d2dcf22..f412110deb76 100644 --- a/modules/gapi/include/opencv2/gapi/util/variant.hpp +++ b/modules/gapi/include/opencv2/gapi/util/variant.hpp @@ -11,6 +11,7 @@ #include #include +#include #include #include // max_of_t #include @@ -44,6 +45,12 @@ namespace util static const constexpr std::size_t value = detail::type_list_index_helper<0, Target, Types...>::value; }; + template + struct type_list_element + { + using type = typename std::tuple_element >::type; + }; + class bad_variant_access: public std::exception { public: @@ -233,9 +240,87 @@ namespace util template const T& get(const util::variant &v); + template + typename util::type_list_element::type& get(util::variant &v); + + template + const typename util::type_list_element::type& get(const util::variant &v); + template bool holds_alternative(const util::variant &v) noexcept; + + // Visitor + namespace detail + { + struct visitor_interface {}; + + // Class `visitor_return_type_deduction_helper` + // introduces solution for deduction `return_type` in `visit` function in common way + // for both Lambda and class Visitor and keep one interface invocation point: `visit` only + // his helper class is required to unify return_type deduction mechanism because + // for Lambda it is possible to take type of `decltype(visitor(get<0>(var)))` + // but for class Visitor there is no operator() in base case, + // because it provides `operator() (std::size_t index, ...)` + // So `visitor_return_type_deduction_helper` expose `operator()` + // uses only for class Visitor only for deduction `return type` in visit() + template + struct visitor_return_type_deduction_helper + { + using return_type = R; + + // to be used in Lambda return type deduction context only + template + return_type operator() (T&&); + }; + } + + // Special purpose `static_visitor` can receive additional arguments + template + struct static_visitor : public detail::visitor_interface, + public detail::visitor_return_type_deduction_helper { + + // assign responsibility for return type deduction to helper class + using return_type = typename detail::visitor_return_type_deduction_helper::return_type; + using detail::visitor_return_type_deduction_helper::operator(); + friend Impl; + + template + return_type operator() (std::size_t index, VariantValue&& value, Args&& ...args) + { + suppress_unused_warning(index); + return static_cast(this)-> visit( + std::forward(value), + std::forward(args)...); + } + }; + + // Special purpose `static_indexed_visitor` can receive additional arguments + // And make forwarding current variant index as runtime function argument to its `Impl` + template + struct static_indexed_visitor : public detail::visitor_interface, + public detail::visitor_return_type_deduction_helper { + + // assign responsibility for return type deduction to helper class + using return_type = typename detail::visitor_return_type_deduction_helper::return_type; + using detail::visitor_return_type_deduction_helper::operator(); + friend Impl; + + template + return_type operator() (std::size_t Index, VariantValue&& value, Args&& ...args) + { + return static_cast(this)-> visit(Index, + std::forward(value), + std::forward(args)...); + } + }; + + template + struct variant_size; + + template + struct variant_size> + : std::integral_constant { }; // FIXME: T&&, const TT&& versions. // Implementation ////////////////////////////////////////////////////////// @@ -402,6 +487,22 @@ namespace util throw_error(bad_variant_access()); } + template + typename util::type_list_element::type& get(util::variant &v) + { + using ReturnType = typename util::type_list_element::type; + return const_cast(get(static_cast &>(v))); + } + + template + const typename util::type_list_element::type& get(const util::variant &v) + { + static_assert(Index < sizeof...(Types), + "`Index` it out of bound of `util::variant` type list"); + using ReturnType = typename util::type_list_element::type; + return get(v); + } + template bool holds_alternative(const util::variant &v) noexcept { @@ -428,7 +529,130 @@ namespace util { return !(lhs == rhs); } -} // namespace cv + +namespace detail +{ + // terminate recursion implementation for `non-void` ReturnType + template + ReturnType apply_visitor_impl(Visitor&&, Variant&, + std::true_type, std::false_type, + VisitorArgs&& ...) + { + return {}; + } + + // terminate recursion implementation for `void` ReturnType + template + void apply_visitor_impl(Visitor&&, Variant&, + std::true_type, std::true_type, + VisitorArgs&& ...) + { + } + + // Intermediate resursion processor for Lambda Visitors + template + typename std::enable_if::type>::value, ReturnType>::type + apply_visitor_impl(Visitor&& visitor, Variant&& v, std::false_type not_processed, + std::integral_constant should_no_return, + VisitorArgs&& ...args) + { + static_assert(std::is_same(v)))>::value, + "Different `ReturnType`s detected! All `Visitor::visit` or `overload_lamba_set`" + " must return the same type"); + suppress_unused_warning(not_processed); + if (v.index() == CurIndex) + { + return visitor.operator()(get(v), std::forward(args)... ); + } + + using is_variant_processed_t = std::integral_constant= ElemCount>; + return apply_visitor_impl( + std::forward(visitor), + std::forward(v), + is_variant_processed_t{}, + should_no_return, + std::forward(args)...); + } + + //Visual Studio 2014 compilation fix: cast visitor to base class before invoke operator() + template + typename std::enable_if::type>, + typename std::decay::type>::value, ReturnType>::type + invoke_class_visitor(Visitor& visitor, Value&& v, VisitorArgs&&...args) + { + return static_cast::type>&>(visitor).operator() (CurIndex, std::forward(v), std::forward(args)... ); + } + + //Visual Studio 2014 compilation fix: cast visitor to base class before invoke operator() + template + typename std::enable_if::type>, + typename std::decay::type>::value, ReturnType>::type + invoke_class_visitor(Visitor& visitor, Value&& v, VisitorArgs&&...args) + { + return static_cast::type>&>(visitor).operator() (CurIndex, std::forward(v), std::forward(args)... ); + } + + // Intermediate recursion processor for special case `visitor_interface` derived Visitors + template + typename std::enable_if::type>::value, ReturnType>::type + apply_visitor_impl(Visitor&& visitor, Variant&& v, std::false_type not_processed, + std::integral_constant should_no_return, + VisitorArgs&& ...args) + { + static_assert(std::is_same(v)))>::value, + "Different `ReturnType`s detected! All `Visitor::visit` or `overload_lamba_set`" + " must return the same type"); + suppress_unused_warning(not_processed); + if (v.index() == CurIndex) + { + return invoke_class_visitor(visitor, get(v), std::forward(args)... ); + } + + using is_variant_processed_t = std::integral_constant= ElemCount>; + return apply_visitor_impl( + std::forward(visitor), + std::forward(v), + is_variant_processed_t{}, + should_no_return, + std::forward(args)...); + } +} // namespace detail + + template + auto visit(Visitor &visitor, const Variant& var, VisitorArg &&...args) -> decltype(visitor(get<0>(var))) + { + constexpr std::size_t varsize = util::variant_size::value; + static_assert(varsize != 0, "utils::variant must contains one type at least "); + using is_variant_processed_t = std::false_type; + + using ReturnType = decltype(visitor(get<0>(var))); + using return_t = std::is_same; + return detail::apply_visitor_impl( + std::forward(visitor), + var, is_variant_processed_t{}, + return_t{}, + std::forward(args)...); + } + + template + auto visit(Visitor&& visitor, const Variant& var) -> decltype(visitor(get<0>(var))) + { + constexpr std::size_t varsize = util::variant_size::value; + static_assert(varsize != 0, "utils::variant must contains one type at least "); + using is_variant_processed_t = std::false_type; + + using ReturnType = decltype(visitor(get<0>(var))); + using return_t = std::is_same; + return detail::apply_visitor_impl( + std::forward(visitor), + var, is_variant_processed_t{}, + return_t{}); + } } // namespace util +} // namespace cv #endif // OPENCV_GAPI_UTIL_VARIANT_HPP diff --git a/modules/gapi/src/api/gproto.cpp b/modules/gapi/src/api/gproto.cpp index 94234c9b4d70..9b012770caee 100644 --- a/modules/gapi/src/api/gproto.cpp +++ b/modules/gapi/src/api/gproto.cpp @@ -14,7 +14,6 @@ #include "api/gorigin.hpp" #include "api/gproto_priv.hpp" -#include "logger.hpp" // FIXME: it should be a visitor! // FIXME: Reimplement with traits? @@ -277,13 +276,9 @@ void cv::validate_input_arg(const GRunArg& arg) void cv::validate_input_args(const GRunArgs& args) { - GAPI_LOG_DEBUG(nullptr, "Total count: " << args.size()); - size_t index = 0; for (const auto& arg : args) { - GAPI_LOG_DEBUG(nullptr, "Process index: " << index); validate_input_arg(arg); - index ++; } } diff --git a/modules/gapi/test/util/variant_tests.cpp b/modules/gapi/test/util/variant_tests.cpp index 65d5e579f81b..7725f9a70211 100644 --- a/modules/gapi/test/util/variant_tests.cpp +++ b/modules/gapi/test/util/variant_tests.cpp @@ -354,6 +354,20 @@ TEST(Variant, Get) EXPECT_THROW(util::get(cv2), util::bad_variant_access); } +TEST(Variant, GetIndexed) +{ + const TestVar cv(42); + + // Test const& get() + EXPECT_EQ(42, util::get<0>(cv)); + EXPECT_THROW(util::get<1>(cv), util::bad_variant_access); + + // Test &get + TestVar cv2(std::string("42")); + EXPECT_EQ("42", util::get<1>(cv2)); + EXPECT_THROW(util::get<0>(cv2), util::bad_variant_access); +} + TEST(Variant, GetWrite) { util::variant v(42); @@ -486,4 +500,240 @@ TEST(Variant, EXT_IndexOf) static_assert(6u == V::index_of(), "Index is incorrect"); } +namespace test_validation +{ +struct MyType +{ + friend std::ostream& operator<<(std::ostream& out, const MyType& src) + { + return out << "MyType"; (void) src; + } +}; +class MyClass +{ + friend std::ostream& operator<<(std::ostream& out, const MyClass& src) + { + return out << "MyClass"; (void) src; + } +}; + +struct MyBoolParamIndexedVisitor : cv::util::static_indexed_visitor +{ + MyBoolParamIndexedVisitor(std::ostream &output) : out(output) {} + + template + bool visit(std::size_t index, Type val, int check) + { + bool result = false; + out << index << ":" << val <<","; + if(std::is_same::value) + { + result = !memcmp(&val, &check, sizeof(int)); + } + return result; + } + + std::ostream &out; +}; + +struct MyBoolNoParamNonIndexedVisitor : cv::util::static_indexed_visitor +{ + MyBoolNoParamNonIndexedVisitor(std::ostream &output) : out(output) {} + + template + bool visit(std::size_t index, Type val) + { + out << index << ":" << val <<","; + return true; + } + std::ostream &out; +}; + + +struct MyVoidNoParamNonIndexedVisitor : cv::util::static_visitor +{ + MyVoidNoParamNonIndexedVisitor(std::ostream &output) : out(output) {} + + template + void visit(Type val) + { + out << val << ","; + } + + std::ostream &out; +}; + + +struct MyVoidNoParamIndexedVisitor : cv::util::static_indexed_visitor +{ + MyVoidNoParamIndexedVisitor(std::ostream &output) : out(output) {} + + template + void visit(std::size_t Index, Type val) + { + out << Index << ":" << val <<","; + } + + std::ostream &out; +}; +} + +TEST(Variant, DynamicVisitor) +{ + using V = cv::util::variant; + V var{42}; + { + std::stringstream ss; + test_validation::MyBoolParamIndexedVisitor visitor(ss); + + EXPECT_TRUE(cv::util::visit(visitor, var, int{42})); + EXPECT_EQ(ss.str(), std::string("0:42,")); + } + + std::stringstream ss; + test_validation::MyBoolNoParamNonIndexedVisitor visitor(ss); + + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("0:42,")); + + var = double{1.0}; + EXPECT_TRUE(cv::util::visit(visitor, var)); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,")); + + var = char{'a'}; + EXPECT_TRUE(cv::util::visit(visitor, var)); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,")); + + var = float{6.0}; + EXPECT_TRUE(cv::util::visit(visitor, var)); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,")); + + var = test_validation::MyType{}; + EXPECT_TRUE(cv::util::visit(visitor, var)); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,")); + + var = test_validation::MyClass{}; + EXPECT_TRUE(cv::util::visit(visitor, var)); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,5:MyClass,")); +} + +TEST(Variant, StaticVisitor) +{ + using V = cv::util::variant; + V var{42}; + std::stringstream ss; + test_validation::MyVoidNoParamNonIndexedVisitor visitor(ss); + + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,")); + + var = double{1.0}; + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,1,")); + + var = char{'a'}; + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,1,a,")); + + var = float{6.0}; + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,1,a,6,")); + + var = test_validation::MyType{}; + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,1,a,6,MyType,")); + + var = test_validation::MyClass{}; + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,1,a,6,MyType,MyClass,")); +} + +TEST(Variant, StaticIndexedVisitor) +{ + using V = cv::util::variant; + V var{42}; + + std::stringstream ss; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor {ss}, var); + EXPECT_EQ(ss.str(), std::string("0:42,")); + + var = double{1.0}; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,")); + + var = char{'a'}; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,")); + + var = float{6.0}; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,")); + + var = test_validation::MyType{}; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,")); + + var = test_validation::MyClass{}; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,5:MyClass,")); +} + + +TEST(Variant, LambdaVisitor) +{ + using V = cv::util::variant; + V var{42}; + { + cv::util::visit(cv::util::overload_lambdas( + [](int value) { + EXPECT_EQ(value, 42); + }, + [](double) { + ADD_FAILURE() << "can't be called for `double`"; + }, + [](char) { + ADD_FAILURE() << "can't be called for `char`"; + }, + [](float) { + ADD_FAILURE() << "can't be called for `float`"; + }, + [](test_validation::MyType) { + ADD_FAILURE() << "can't be called for `MyType`"; + }, + [](test_validation::MyClass) { + ADD_FAILURE() << "can't be called for `MyClass`"; + }, + [](std::string) { + ADD_FAILURE() << "can't be called for `std::string`, invalid type"; + } + ), var); + } + + var = 'c'; + { + cv::util::visit(cv::util::overload_lambdas( + [](int) { + ADD_FAILURE() << "can't be called for `int`"; + }, + [](double) { + ADD_FAILURE() << "can't be called for `double`"; + }, + [](char value) { + EXPECT_EQ(value, 'c'); + }, + [](float) { + ADD_FAILURE() << "can't be called for `float`"; + }, + [](test_validation::MyType) { + ADD_FAILURE() << "can't be called for `MyType`"; + }, + [](test_validation::MyClass) { + ADD_FAILURE() << "can't be called for `MyClass`"; + }, + [](std::string) { + ADD_FAILURE() << "can't be called for `std::string`, invalid type"; + } + ), var); + } +} } // namespace opencv_test From 926535469d82649204a9b2819f3f7341c140b7cb Mon Sep 17 00:00:00 2001 From: kikaxa Date: Wed, 7 Jul 2021 18:31:53 +0300 Subject: [PATCH 049/376] fix videoio/src/container_avi.cpp VideoInputStream alignment --- modules/videoio/src/container_avi.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/videoio/src/container_avi.cpp b/modules/videoio/src/container_avi.cpp index 8664d198df1b..c0c3234f6f95 100644 --- a/modules/videoio/src/container_avi.cpp +++ b/modules/videoio/src/container_avi.cpp @@ -124,6 +124,7 @@ struct RiffList uint32_t m_size; uint32_t m_list_type_cc; }; +#pragma pack(pop) class VideoInputStream { @@ -149,7 +150,6 @@ class VideoInputStream String m_fname; }; -#pragma pack(pop) inline VideoInputStream& operator >> (VideoInputStream& is, AviMainHeader& avih) { From 59ae0e0013d85bf4a6d064e7c258c00e56858b35 Mon Sep 17 00:00:00 2001 From: Alexey Smirnov Date: Wed, 7 Jul 2021 22:07:59 +0300 Subject: [PATCH 050/376] Merge pull request #20163 from smirnov-alexey:as/gapi_serialization_docs G-API: add documentation on serialization functionality * Add documentation on serialization/deserialization * Add docs on bind() methods * Fix typo * Docs refactoring * Fix s11n docs * Fix deserialize() docs * Change deserialize docs * Fix warning * Address review comments * Fix sample * Fix warnings and errors * Fix docs warnings * Fix warnings * Address review comments * Add prefixes to snippets and fix indentation * Address review comments and move snippets to a single file --- modules/gapi/include/opencv2/gapi.hpp | 3 +- modules/gapi/include/opencv2/gapi/garg.hpp | 41 +++- modules/gapi/include/opencv2/gapi/gproto.hpp | 2 +- modules/gapi/include/opencv2/gapi/s11n.hpp | 185 ++++++++++++++---- .../gapi/include/opencv2/gapi/s11n/base.hpp | 35 +++- modules/gapi/samples/api_ref_snippets.cpp | 121 ++++++++++++ modules/gapi/src/api/s11n.cpp | 6 +- modules/gapi/test/s11n/gapi_s11n_tests.cpp | 2 - 8 files changed, 339 insertions(+), 56 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi.hpp b/modules/gapi/include/opencv2/gapi.hpp index e4b20214796a..f10dfd471dbf 100644 --- a/modules/gapi/include/opencv2/gapi.hpp +++ b/modules/gapi/include/opencv2/gapi.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2018 Intel Corporation +// Copyright (C) 2018-2021 Intel Corporation #ifndef OPENCV_GAPI_HPP @@ -19,6 +19,7 @@ @} @defgroup gapi_std_backends G-API Standard Backends @defgroup gapi_compile_args G-API Graph Compilation Arguments + @defgroup gapi_serialization G-API Serialization functionality @} */ diff --git a/modules/gapi/include/opencv2/gapi/garg.hpp b/modules/gapi/include/opencv2/gapi/garg.hpp index 20f2233bf9c3..ee6ee81e1cc6 100644 --- a/modules/gapi/include/opencv2/gapi/garg.hpp +++ b/modules/gapi/include/opencv2/gapi/garg.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2018-2020 Intel Corporation +// Copyright (C) 2018-2021 Intel Corporation #ifndef OPENCV_GAPI_GARG_HPP @@ -171,7 +171,7 @@ using GRunArgs = std::vector; * It's an ordinary overload of addition assignment operator. * * Example of usage: - * @snippet dynamic_graph.cpp GRunArgs usage + * @snippet modules/gapi/samples/dynamic_graph.cpp GRunArgs usage * */ inline GRunArgs& operator += (GRunArgs &lhs, const GRunArgs &rhs) @@ -223,7 +223,7 @@ using GRunArgsP = std::vector; * It's an ordinary overload of addition assignment operator. * * Example of usage: - * @snippet dynamic_graph.cpp GRunArgsP usage + * @snippet modules/gapi/samples/dynamic_graph.cpp GRunArgsP usage * */ inline GRunArgsP& operator += (GRunArgsP &lhs, const GRunArgsP &rhs) @@ -235,8 +235,39 @@ inline GRunArgsP& operator += (GRunArgsP &lhs, const GRunArgsP &rhs) namespace gapi { - GAPI_EXPORTS cv::GRunArgsP bind(cv::GRunArgs &results); - GAPI_EXPORTS cv::GRunArg bind(cv::GRunArgP &out); // FIXME: think more about it +/** + * \addtogroup gapi_serialization + * @{ + * + * @brief G-API functions and classes for serialization and deserialization. + */ +/** @brief Wraps deserialized output GRunArgs to GRunArgsP which can be used by GCompiled. + * + * Since it's impossible to get modifiable output arguments from deserialization + * it needs to be wrapped by this function. + * + * Example of usage: + * @snippet modules/gapi/samples/api_ref_snippets.cpp bind after deserialization + * + * @param out_args deserialized GRunArgs. + * @return the same GRunArgs wrapped in GRunArgsP. + * @see deserialize + */ +GAPI_EXPORTS cv::GRunArgsP bind(cv::GRunArgs &out_args); +/** @brief Wraps output GRunArgsP available during graph execution to GRunArgs which can be serialized. + * + * GRunArgsP is pointer-to-value, so to be serialized they need to be binded to real values + * which this function does. + * + * Example of usage: + * @snippet modules/gapi/samples/api_ref_snippets.cpp bind before serialization + * + * @param out output GRunArgsP available during graph execution. + * @return the same GRunArgsP wrapped in serializable GRunArgs. + * @see serialize + */ +GAPI_EXPORTS cv::GRunArg bind(cv::GRunArgP &out); // FIXME: think more about it +/** @} */ } template inline GRunArgs gin(const Ts&... args) diff --git a/modules/gapi/include/opencv2/gapi/gproto.hpp b/modules/gapi/include/opencv2/gapi/gproto.hpp index fbcccb38ea71..6271e470b076 100644 --- a/modules/gapi/include/opencv2/gapi/gproto.hpp +++ b/modules/gapi/include/opencv2/gapi/gproto.hpp @@ -71,7 +71,7 @@ struct GIOProtoArgs * It's an ordinary overload of addition assignment operator. * * Example of usage: - * @snippet dynamic_graph.cpp GIOProtoArgs usage + * @snippet modules/gapi/samples/dynamic_graph.cpp GIOProtoArgs usage * */ template diff --git a/modules/gapi/include/opencv2/gapi/s11n.hpp b/modules/gapi/include/opencv2/gapi/s11n.hpp index 5a64410e5abe..ca8e32c98bf9 100644 --- a/modules/gapi/include/opencv2/gapi/s11n.hpp +++ b/modules/gapi/include/opencv2/gapi/s11n.hpp @@ -17,65 +17,135 @@ namespace cv { namespace gapi { +/** +* \addtogroup gapi_serialization +* @{ +*/ + namespace detail { - GAPI_EXPORTS cv::GComputation getGraph(const std::vector &p); + GAPI_EXPORTS cv::GComputation getGraph(const std::vector &bytes); - GAPI_EXPORTS cv::GMetaArgs getMetaArgs(const std::vector &p); + GAPI_EXPORTS cv::GMetaArgs getMetaArgs(const std::vector &bytes); - GAPI_EXPORTS cv::GRunArgs getRunArgs(const std::vector &p); + GAPI_EXPORTS cv::GRunArgs getRunArgs(const std::vector &bytes); - GAPI_EXPORTS std::vector getVectorOfStrings(const std::vector &p); + GAPI_EXPORTS std::vector getVectorOfStrings(const std::vector &bytes); template - cv::GCompileArgs getCompileArgs(const std::vector &p); + cv::GCompileArgs getCompileArgs(const std::vector &bytes); template - cv::GRunArgs getRunArgsWithRMats(const std::vector &p); + cv::GRunArgs getRunArgsWithRMats(const std::vector &bytes); } // namespace detail +/** @brief Serialize a graph represented by GComputation into an array of bytes. + * + * Check different overloads for more examples. + * @param c GComputation to serialize. + * @return serialized vector of bytes. + */ GAPI_EXPORTS std::vector serialize(const cv::GComputation &c); -//namespace{ +/** @overload + * @param ca GCompileArgs to serialize. + */ +GAPI_EXPORTS std::vector serialize(const cv::GCompileArgs& ca); + +/** @overload + * @param ma GMetaArgs to serialize. + */ +GAPI_EXPORTS std::vector serialize(const cv::GMetaArgs& ma); + +/** @overload + * @param ra GRunArgs to serialize. + */ +GAPI_EXPORTS std::vector serialize(const cv::GRunArgs& ra); + +/** @overload + * @param vs std::vector to serialize. + */ +GAPI_EXPORTS std::vector serialize(const std::vector& vs); + +/** + * @private + */ template static inline -T deserialize(const std::vector &p); - -//} //ananymous namespace - -GAPI_EXPORTS std::vector serialize(const cv::GCompileArgs&); -GAPI_EXPORTS std::vector serialize(const cv::GMetaArgs&); -GAPI_EXPORTS std::vector serialize(const cv::GRunArgs&); -GAPI_EXPORTS std::vector serialize(const std::vector&); - +T deserialize(const std::vector &bytes); + +/** @brief Deserialize GComputation from a byte array. + * + * Check different overloads for more examples. + * @param bytes serialized vector of bytes. + * @return deserialized GComputation object. + */ template<> inline -cv::GComputation deserialize(const std::vector &p) { - return detail::getGraph(p); +cv::GComputation deserialize(const std::vector &bytes) { + return detail::getGraph(bytes); } +/** @brief Deserialize GMetaArgs from a byte array. + * + * Check different overloads for more examples. + * @param bytes serialized vector of bytes. + * @return deserialized GMetaArgs object. + */ template<> inline -cv::GMetaArgs deserialize(const std::vector &p) { - return detail::getMetaArgs(p); +cv::GMetaArgs deserialize(const std::vector &bytes) { + return detail::getMetaArgs(bytes); } +/** @brief Deserialize GRunArgs from a byte array. + * + * Check different overloads for more examples. + * @param bytes serialized vector of bytes. + * @return deserialized GRunArgs object. + */ template<> inline -cv::GRunArgs deserialize(const std::vector &p) { - return detail::getRunArgs(p); +cv::GRunArgs deserialize(const std::vector &bytes) { + return detail::getRunArgs(bytes); } +/** @brief Deserialize std::vector from a byte array. + * + * Check different overloads for more examples. + * @param bytes serialized vector of bytes. + * @return deserialized std::vector object. + */ template<> inline -std::vector deserialize(const std::vector &p) { - return detail::getVectorOfStrings(p); +std::vector deserialize(const std::vector &bytes) { + return detail::getVectorOfStrings(bytes); } +/** + * @brief Deserialize GCompileArgs which types were specified in the template from a byte array. + * + * @note cv::gapi::s11n::detail::S11N template specialization must be provided to make a custom type + * in GCompileArgs deserializable. + * + * @param bytes vector of bytes to deserialize GCompileArgs object from. + * @return GCompileArgs object. + * @see GCompileArgs cv::gapi::s11n::detail::S11N + */ template inline typename std::enable_if::value, GCompileArgs>:: -type deserialize(const std::vector &p) { - return detail::getCompileArgs(p); +type deserialize(const std::vector &bytes) { + return detail::getCompileArgs(bytes); } +/** + * @brief Deserialize GRunArgs including RMat objects if any from a byte array. + * + * RMat adapter type is specified in the template. + * @note To be used properly specified adapter type must overload its serialize() and + * deserialize() methods. + * @param bytes vector of bytes to deserialize GRunArgs object from. + * @return GRunArgs including RMat objects if any. + * @see RMat + */ template inline typename std::enable_if::value, GRunArgs>:: -type deserialize(const std::vector &p) { - return detail::getRunArgsWithRMats(p); +type deserialize(const std::vector &bytes) { + return detail::getRunArgsWithRMats(bytes); } } // namespace gapi } // namespace cv @@ -83,6 +153,17 @@ type deserialize(const std::vector &p) { namespace cv { namespace gapi { namespace s11n { + +/** @brief This structure is an interface for serialization routines. + * + * It's main purpose is to provide multiple overloads for operator<<() + * with basic C++ in addition to OpenCV/G-API types. + * + * This sctructure can be inherited and further extended with additional types. + * + * For example, it is utilized in cv::gapi::s11n::detail::S11N as input parameter + * in serialize() method. + */ struct GAPI_EXPORTS IOStream { virtual ~IOStream() = default; // Define the native support for basic C++ types at the API level: @@ -99,6 +180,16 @@ struct GAPI_EXPORTS IOStream { virtual IOStream& operator<< (const std::string&) = 0; }; +/** @brief This structure is an interface for deserialization routines. + * + * It's main purpose is to provide multiple overloads for operator>>() + * with basic C++ in addition to OpenCV/G-API types. + * + * This structure can be inherited and further extended with additional types. + * + * For example, it is utilized in cv::gapi::s11n::detail::S11N as input parameter + * in deserialize() method. + */ struct GAPI_EXPORTS IIStream { virtual ~IIStream() = default; virtual IIStream& operator>> (bool &) = 0; @@ -116,7 +207,7 @@ struct GAPI_EXPORTS IIStream { }; namespace detail { -GAPI_EXPORTS std::unique_ptr getInStream(const std::vector &p); +GAPI_EXPORTS std::unique_ptr getInStream(const std::vector &bytes); } // namespace detail //////////////////////////////////////////////////////////////////////////////// @@ -146,24 +237,26 @@ GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::Mat &m); // FIXME: for GRunArgs serailization #if !defined(GAPI_STANDALONE) -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::UMat &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::UMat &); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::UMat & um); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::UMat & um); #endif // !defined(GAPI_STANDALONE) GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::RMat &r); GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::RMat &r); -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gapi::wip::IStreamSource::Ptr &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gapi::wip::IStreamSource::Ptr &); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gapi::wip::IStreamSource::Ptr &issptr); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gapi::wip::IStreamSource::Ptr &issptr); -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::VectorRef &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::VectorRef &); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::VectorRef &vr); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::VectorRef &vr); -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::OpaqueRef &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::OpaqueRef &); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::OpaqueRef &opr); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::OpaqueRef &opr); -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::MediaFrame &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::MediaFrame &); +/// @private -- Exclude this function from OpenCV documentation +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::MediaFrame &mf); +/// @private -- Exclude this function from OpenCV documentation +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::MediaFrame &mf); // Generic STL types //////////////////////////////////////////////////////////////// template @@ -186,6 +279,7 @@ IIStream& operator>> (IIStream& is, std::map &m) { } return is; } + template IOStream& operator<< (IOStream& os, const std::unordered_map &m) { const uint32_t sz = static_cast(m.size()); @@ -206,6 +300,7 @@ IIStream& operator>> (IIStream& is, std::unordered_map &m) { } return is; } + template IOStream& operator<< (IOStream& os, const std::vector &ts) { const uint32_t sz = static_cast(ts.size()); @@ -233,16 +328,19 @@ template IOStream& put_v(IOStream&, const V&, std::size_t) { GAPI_Assert(false && "variant>>: requested index is invalid"); }; + template IOStream& put_v(IOStream& os, const V& v, std::size_t x) { return (x == 0u) ? os << cv::util::get(v) : put_v(os, v, x-1); } + template IIStream& get_v(IIStream&, V&, std::size_t, std::size_t) { GAPI_Assert(false && "variant<<: requested index is invalid"); } + template IIStream& get_v(IIStream& is, V& v, std::size_t i, std::size_t gi) { if (i == gi) { @@ -254,11 +352,13 @@ IIStream& get_v(IIStream& is, V& v, std::size_t i, std::size_t gi) { } } // namespace detail +//! @overload template IOStream& operator<< (IOStream& os, const cv::util::variant &v) { os << static_cast(v.index()); return detail::put_v, Ts...>(os, v, v.index()); } +//! @overload template IIStream& operator>> (IIStream& is, cv::util::variant &v) { int idx = -1; @@ -268,6 +368,7 @@ IIStream& operator>> (IIStream& is, cv::util::variant &v) { } // FIXME: consider a better solution +/// @private -- Exclude this function from OpenCV documentation template void getRunArgByIdx (IIStream& is, cv::util::variant &v, uint32_t idx) { is = detail::get_v, Ts...>(is, v, 0u, idx); @@ -351,8 +452,8 @@ cv::GCompileArgs getCompileArgs(const std::vector &sArgs) { } template -cv::GRunArgs getRunArgsWithRMats(const std::vector &p) { - std::unique_ptr pIs = cv::gapi::s11n::detail::getInStream(p); +cv::GRunArgs getRunArgsWithRMats(const std::vector &bytes) { + std::unique_ptr pIs = cv::gapi::s11n::detail::getInStream(bytes); cv::gapi::s11n::IIStream& is = *pIs; cv::GRunArgs args; @@ -367,6 +468,8 @@ cv::GRunArgs getRunArgsWithRMats(const std::vector &p) { return args; } } // namespace detail +/** @} */ + } // namespace gapi } // namespace cv diff --git a/modules/gapi/include/opencv2/gapi/s11n/base.hpp b/modules/gapi/include/opencv2/gapi/s11n/base.hpp index b8ec8cfaff73..11440b27e5f8 100644 --- a/modules/gapi/include/opencv2/gapi/s11n/base.hpp +++ b/modules/gapi/include/opencv2/gapi/s11n/base.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation #ifndef OPENCV_GAPI_S11N_BASE_HPP #define OPENCV_GAPI_S11N_BASE_HPP @@ -23,25 +23,54 @@ struct IIStream; namespace detail { +//! @addtogroup gapi_serialization +//! @{ + struct NotImplemented { }; -// The default S11N for custom types is NotImplemented -// Don't! sublass from NotImplemented if you actually implement S11N. +/** @brief This structure allows to implement serialization routines for custom types. + * + * The default S11N for custom types is not implemented. + * + * @note When providing an overloaded implementation for S11N with your type + * don't inherit it from NotImplemented structure. + * + * @note There are lots of overloaded >> and << operators for basic and OpenCV/G-API types + * which can be utilized when serializing a custom type. + * + * Example of usage: + * @snippet modules/gapi/samples/api_ref_snippets.cpp S11N usage + * + */ template struct S11N: public NotImplemented { + /** + * @brief This function allows user to serialize their custom type. + * + * @note The default overload throws an exception if called. User need to + * properly overload the function to use it. + */ static void serialize(IOStream &, const T &) { GAPI_Assert(false && "No serialization routine is provided!"); } + /** + * @brief This function allows user to deserialize their custom type. + * + * @note The default overload throws an exception if called. User need to + * properly overload the function to use it. + */ static T deserialize(IIStream &) { GAPI_Assert(false && "No deserialization routine is provided!"); } }; +/// @private -- Exclude this struct from OpenCV documentation template struct has_S11N_spec { static constexpr bool value = !std::is_base_of::type>>::value; }; +//! @} gapi_serialization } // namespace detail } // namespace s11n diff --git a/modules/gapi/samples/api_ref_snippets.cpp b/modules/gapi/samples/api_ref_snippets.cpp index 6c660fb8fa2e..0abcab89b383 100644 --- a/modules/gapi/samples/api_ref_snippets.cpp +++ b/modules/gapi/samples/api_ref_snippets.cpp @@ -4,6 +4,10 @@ #include #include +#include +#include +#include + #include #include @@ -55,6 +59,120 @@ static void typed_example() //! [Typed_Example] } +static void bind_serialization_example() +{ + // ! [bind after deserialization] + cv::GCompiled compd; + std::vector bytes; + auto graph = cv::gapi::deserialize(bytes); + auto meta = cv::gapi::deserialize(bytes); + + compd = graph.compile(std::move(meta), cv::compile_args()); + auto in_args = cv::gapi::deserialize(bytes); + auto out_args = cv::gapi::deserialize(bytes); + compd(std::move(in_args), cv::gapi::bind(out_args)); + // ! [bind after deserialization] +} + +static void bind_deserialization_example() +{ + // ! [bind before serialization] + std::vector graph_outs; + cv::GRunArgs out_args; + + for (auto &&out : graph_outs) { + out_args.emplace_back(cv::gapi::bind(out)); + } + const auto sargsout = cv::gapi::serialize(out_args); + // ! [bind before serialization] +} + +struct SimpleCustomType { + bool val; + bool operator==(const SimpleCustomType& other) const { + return val == other.val; + } +}; + +struct SimpleCustomType2 { + int val; + std::string name; + std::vector vec; + std::map mmap; + bool operator==(const SimpleCustomType2& other) const { + return val == other.val && name == other.name && + vec == other.vec && mmap == other.mmap; + } +}; + +// ! [S11N usage] +namespace cv { +namespace gapi { +namespace s11n { +namespace detail { +template<> struct S11N { + static void serialize(IOStream &os, const SimpleCustomType &p) { + os << p.val; + } + static SimpleCustomType deserialize(IIStream &is) { + SimpleCustomType p; + is >> p.val; + return p; + } +}; + +template<> struct S11N { + static void serialize(IOStream &os, const SimpleCustomType2 &p) { + os << p.val << p.name << p.vec << p.mmap; + } + static SimpleCustomType2 deserialize(IIStream &is) { + SimpleCustomType2 p; + is >> p.val >> p.name >> p.vec >> p.mmap; + return p; + } +}; +} // namespace detail +} // namespace s11n +} // namespace gapi +} // namespace cv +// ! [S11N usage] + +namespace cv { +namespace detail { +template<> struct CompileArgTag { + static const char* tag() { + return "org.opencv.test.simple_custom_type"; + } +}; + +template<> struct CompileArgTag { + static const char* tag() { + return "org.opencv.test.simple_custom_type_2"; + } +}; +} // namespace detail +} // namespace cv + +static void s11n_example() +{ + SimpleCustomType customVar1 { false }; + SimpleCustomType2 customVar2 { 1248, "World", {1280, 720, 640, 480}, + { {5, 32434142342}, {7, 34242432} } }; + + std::vector sArgs = cv::gapi::serialize( + cv::compile_args(customVar1, customVar2)); + + cv::GCompileArgs dArgs = cv::gapi::deserialize(sArgs); + + SimpleCustomType dCustomVar1 = cv::gapi::getCompileArg(dArgs).value(); + SimpleCustomType2 dCustomVar2 = cv::gapi::getCompileArg(dArgs).value(); + + (void) dCustomVar1; + (void) dCustomVar2; +} + G_TYPED_KERNEL(IAdd, , "test.custom.add") { static cv::GMatDesc outMeta(const cv::GMatDesc &in) { return in; } }; @@ -128,5 +246,8 @@ int main(int argc, char *argv[]) // unused functions typed_example(); gscalar_example(); + bind_serialization_example(); + bind_deserialization_example(); + s11n_example(); return 0; } diff --git a/modules/gapi/src/api/s11n.cpp b/modules/gapi/src/api/s11n.cpp index d08f47fd26a7..97f5a95c42a6 100644 --- a/modules/gapi/src/api/s11n.cpp +++ b/modules/gapi/src/api/s11n.cpp @@ -65,11 +65,11 @@ std::vector cv::gapi::serialize(const std::vector& vs) // FIXME: This function should move from S11N to GRunArg-related entities. // it has nothing to do with the S11N as it is -cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &results) +cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &out_args) { cv::GRunArgsP outputs; - outputs.reserve(results.size()); - for (cv::GRunArg &res_obj : results) + outputs.reserve(out_args.size()); + for (cv::GRunArg &res_obj : out_args) { using T = cv::GRunArg; switch (res_obj.index()) diff --git a/modules/gapi/test/s11n/gapi_s11n_tests.cpp b/modules/gapi/test/s11n/gapi_s11n_tests.cpp index f4a30b394631..c2b17521d966 100644 --- a/modules/gapi/test/s11n/gapi_s11n_tests.cpp +++ b/modules/gapi/test/s11n/gapi_s11n_tests.cpp @@ -754,8 +754,6 @@ TEST_F(S11N_Basic, Test_Deserialize_CompileArgs_RandomOrder) { std::vector sArgs = cv::gapi::serialize( cv::compile_args(simpleCustomVar, simpleCustomVar2)); GCompileArgs dArgs = cv::gapi::deserialize(sArgs); From b42623ff9d6b1baf9fda11b16a8d2f512720d424 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Thu, 8 Jul 2021 10:42:44 +0300 Subject: [PATCH 051/376] port base64 encoding from 3.4 --- modules/core/src/persistence.cpp | 2393 +++++++++-------- modules/core/src/persistence.hpp | 18 + .../core/src/persistence_base64_encoding.cpp | 370 +++ .../core/src/persistence_base64_encoding.hpp | 127 + modules/core/src/persistence_impl.hpp | 231 ++ modules/core/src/persistence_json.cpp | 69 +- modules/core/src/persistence_xml.cpp | 30 +- modules/core/src/persistence_yml.cpp | 28 +- modules/core/test/test_io.cpp | 29 +- modules/python/test/test_filestorage_io.py | 93 + 10 files changed, 2156 insertions(+), 1232 deletions(-) create mode 100644 modules/core/src/persistence_base64_encoding.cpp create mode 100644 modules/core/src/persistence_base64_encoding.hpp create mode 100644 modules/core/src/persistence_impl.hpp diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 32328361e874..291931b5ae01 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -4,6 +4,8 @@ #include "precomp.hpp" #include "persistence.hpp" +#include "persistence_impl.hpp" +#include "persistence_base64_encoding.hpp" #include #include @@ -153,7 +155,7 @@ static int symbolToType(char c) return CV_SEQ_ELTYPE_PTR; const char* pos = strchr( symbols, c ); if( !pos ) - CV_Error( CV_StsBadArg, "Invalid data type specification" ); + CV_Error( cv::Error::StsBadArg, "Invalid data type specification" ); return static_cast(pos - symbols); } @@ -192,7 +194,7 @@ int decodeFormat( const char* dt, int* fmt_pairs, int max_len ) } if( count <= 0 ) - CV_Error( CV_StsBadArg, "Invalid data type specification" ); + CV_Error( cv::Error::StsBadArg, "Invalid data type specification" ); fmt_pairs[i] = count; } @@ -208,7 +210,7 @@ int decodeFormat( const char* dt, int* fmt_pairs, int max_len ) { i += 2; if( i >= max_len ) - CV_Error( CV_StsBadArg, "Too long data type specification" ); + CV_Error( cv::Error::StsBadArg, "Too long data type specification" ); } fmt_pairs[i] = 0; } @@ -275,7 +277,7 @@ int decodeSimpleFormat( const char* dt ) fmt_pair_count = decodeFormat( dt, fmt_pairs, CV_FS_MAX_FMT_PAIRS ); if( fmt_pair_count != 1 || fmt_pairs[0] >= CV_CN_MAX) - CV_Error( CV_StsError, "Too complex format for the matrix" ); + CV_Error( cv::Error::StsError, "Too complex format for the matrix" ); elem_type = CV_MAKETYPE( fmt_pairs[1], fmt_pairs[0] ); @@ -345,1450 +347,1483 @@ static inline void writeReal(uchar* p, double fval) #endif } -class FileStorage::Impl : public FileStorage_API -{ -public: - void init() - { - flags = 0; - buffer.clear(); - bufofs = 0; - state = UNDEFINED; - is_opened = false; - dummy_eof = false; - write_mode = false; - mem_mode = false; - space = 0; - wrap_margin = 71; - fmt = 0; - file = 0; - gzfile = 0; - empty_stream = true; - strbufv.clear(); - strbuf = 0; - strbufsize = strbufpos = 0; - roots.clear(); - - fs_data.clear(); - fs_data_ptrs.clear(); - fs_data_blksz.clear(); - freeSpaceOfs = 0; - - str_hash.clear(); - str_hash_data.clear(); - str_hash_data.resize(1); - str_hash_data[0] = '\0'; - - filename.clear(); - lineno = 0; - } - - Impl(FileStorage* _fs) - { - fs_ext = _fs; - init(); - } - - virtual ~Impl() - { - release(); - } - void release(String* out=0) - { - if( is_opened ) - { - if(out) - out->clear(); - if( write_mode ) - { - while( write_stack.size() > 1 ) - { - endWriteStruct(); - } - flush(); - if( fmt == FileStorage::FORMAT_XML ) - puts( "\n" ); - else if ( fmt == FileStorage::FORMAT_JSON ) - puts( "}\n" ); - } - if( mem_mode && out ) - { - *out = cv::String(outbuf.begin(), outbuf.end()); +void FileStorage::Impl::init() { + flags = 0; + buffer.clear(); + bufofs = 0; + state = UNDEFINED; + is_using_base64 = false; + state_of_writing_base64 = FileStorage_API::Base64State::Uncertain; + is_write_struct_delayed = false; + delayed_struct_key = nullptr; + delayed_struct_flags = 0; + delayed_type_name = nullptr; + base64_writer = nullptr; + is_opened = false; + dummy_eof = false; + write_mode = false; + mem_mode = false; + space = 0; + wrap_margin = 71; + fmt = 0; + file = 0; + gzfile = 0; + empty_stream = true; + + strbufv.clear(); + strbuf = 0; + strbufsize = strbufpos = 0; + roots.clear(); + + fs_data.clear(); + fs_data_ptrs.clear(); + fs_data_blksz.clear(); + freeSpaceOfs = 0; + + str_hash.clear(); + str_hash_data.clear(); + str_hash_data.resize(1); + str_hash_data[0] = '\0'; + + filename.clear(); + lineno = 0; +} + +FileStorage::Impl::Impl(FileStorage *_fs) { + fs_ext = _fs; + init(); +} + +FileStorage::Impl::~Impl() { + release(); +} + +void FileStorage::Impl::release(String *out) { + if (is_opened) { + if (out) + out->clear(); + if (write_mode) { + while (write_stack.size() > 1) { + endWriteStruct(); } + flush(); + if (fmt == FileStorage::FORMAT_XML) + puts("\n"); + else if (fmt == FileStorage::FORMAT_JSON) + puts("}\n"); + } + if (mem_mode && out) { + *out = cv::String(outbuf.begin(), outbuf.end()); } - closeFile(); - init(); } + closeFile(); + init(); +} - void analyze_file_name( const std::string& file_name, std::vector& params ) - { - params.clear(); - static const char not_file_name = '\n'; - static const char parameter_begin = '?'; - static const char parameter_separator = '&'; +void FileStorage::Impl::analyze_file_name(const std::string &file_name, std::vector ¶ms) { + params.clear(); + static const char not_file_name = '\n'; + static const char parameter_begin = '?'; + static const char parameter_separator = '&'; - if( file_name.find(not_file_name, (size_t)0) != std::string::npos ) - return; + if (file_name.find(not_file_name, (size_t) 0) != std::string::npos) + return; - size_t beg = file_name.find_last_of(parameter_begin); - params.push_back(file_name.substr((size_t)0, beg)); + size_t beg = file_name.find_last_of(parameter_begin); + params.push_back(file_name.substr((size_t) 0, beg)); - if( beg != std::string::npos ) - { - size_t end = file_name.size(); - beg++; - for( size_t param_beg = beg, param_end = beg; - param_end < end; - param_beg = param_end + 1 ) - { - param_end = file_name.find_first_of( parameter_separator, param_beg ); - if( (param_end == std::string::npos || param_end != param_beg) && param_beg + 1 < end ) - { - params.push_back( file_name.substr( param_beg, param_end - param_beg ) ); - } + if (beg != std::string::npos) { + size_t end = file_name.size(); + beg++; + for (size_t param_beg = beg, param_end = beg; + param_end < end; + param_beg = param_end + 1) { + param_end = file_name.find_first_of(parameter_separator, param_beg); + if ((param_end == std::string::npos || param_end != param_beg) && param_beg + 1 < end) { + params.push_back(file_name.substr(param_beg, param_end - param_beg)); } } } +} - bool open( const char* filename_or_buf, int _flags, const char* encoding ) - { - _flags &= ~FileStorage::BASE64; - - bool ok = true; - release(); +bool FileStorage::Impl::open(const char *filename_or_buf, int _flags, const char *encoding) { + bool ok = true; + release(); - bool append = (_flags & 3) == FileStorage::APPEND; - mem_mode = (_flags & FileStorage::MEMORY) != 0; + bool append = (_flags & 3) == FileStorage::APPEND; + mem_mode = (_flags & FileStorage::MEMORY) != 0; - write_mode = (_flags & 3) != 0; + write_mode = (_flags & 3) != 0; + bool write_base64 = (write_mode || append) && (_flags & FileStorage::BASE64) != 0; - bool isGZ = false; - size_t fnamelen = 0; + bool isGZ = false; + size_t fnamelen = 0; - std::vector params; - //if ( !mem_mode ) - { - analyze_file_name( filename_or_buf, params ); - if( !params.empty() ) - filename = params[0]; + std::vector params; + //if ( !mem_mode ) + { + analyze_file_name(filename_or_buf, params); + if (!params.empty()) + filename = params[0]; - /*if( !write_base64 && params.size() >= 2 && - std::find(params.begin()+1, params.end(), std::string("base64")) != params.end()) - write_base64 = (write_mode || append);*/ - } + if (!write_base64 && params.size() >= 2 && + std::find(params.begin() + 1, params.end(), std::string("base64")) != params.end()) + write_base64 = (write_mode || append); + } - if( filename.size() == 0 && !mem_mode && !write_mode ) - CV_Error( CV_StsNullPtr, "NULL or empty filename" ); + if (filename.size() == 0 && !mem_mode && !write_mode) + CV_Error(cv::Error::StsNullPtr, "NULL or empty filename"); - if( mem_mode && append ) - CV_Error( CV_StsBadFlag, "FileStorage::APPEND and FileStorage::MEMORY are not currently compatible" ); + if (mem_mode && append) + CV_Error(cv::Error::StsBadFlag, "FileStorage::APPEND and FileStorage::MEMORY are not currently compatible"); - flags = _flags; + flags = _flags; - if( !mem_mode ) - { - char* dot_pos = strrchr((char*)filename.c_str(), '.'); - char compression = '\0'; + if (!mem_mode) { + char *dot_pos = strrchr((char *) filename.c_str(), '.'); + char compression = '\0'; - if( dot_pos && dot_pos[1] == 'g' && dot_pos[2] == 'z' && - (dot_pos[3] == '\0' || (cv_isdigit(dot_pos[3]) && dot_pos[4] == '\0')) ) - { - if( append ) - { - CV_Error(CV_StsNotImplemented, "Appending data to compressed file is not implemented" ); - } - isGZ = true; - compression = dot_pos[3]; - if( compression ) - dot_pos[3] = '\0', fnamelen--; + if (dot_pos && dot_pos[1] == 'g' && dot_pos[2] == 'z' && + (dot_pos[3] == '\0' || (cv_isdigit(dot_pos[3]) && dot_pos[4] == '\0'))) { + if (append) { + CV_Error(cv::Error::StsNotImplemented, "Appending data to compressed file is not implemented"); } + isGZ = true; + compression = dot_pos[3]; + if (compression) + dot_pos[3] = '\0', fnamelen--; + } - if( !isGZ ) - { - file = fopen(filename.c_str(), !write_mode ? "rt" : !append ? "wt" : "a+t" ); - if( !file ) - return false; - } - else - { + if (!isGZ) { + file = fopen(filename.c_str(), !write_mode ? "rt" : !append ? "wt" : "a+t"); + if (!file) + return false; + } else { #if USE_ZLIB - char mode[] = { write_mode ? 'w' : 'r', 'b', compression ? compression : '3', '\0' }; - gzfile = gzopen(filename.c_str(), mode); - if( !gzfile ) - return false; + char mode[] = {write_mode ? 'w' : 'r', 'b', compression ? compression : '3', '\0'}; + gzfile = gzopen(filename.c_str(), mode); + if (!gzfile) + return false; #else - CV_Error(CV_StsNotImplemented, "There is no compressed file storage support in this configuration"); + CV_Error(cv::Error::StsNotImplemented, "There is no compressed file storage support in this configuration"); #endif - } } + } - roots.clear(); - fs_data.clear(); - wrap_margin = 71; - fmt = FileStorage::FORMAT_AUTO; + roots.clear(); + fs_data.clear(); + wrap_margin = 71; + fmt = FileStorage::FORMAT_AUTO; - if( write_mode ) - { - fmt = flags & FileStorage::FORMAT_MASK; + if (write_mode) { + fmt = flags & FileStorage::FORMAT_MASK; - if( mem_mode ) - outbuf.clear(); + if (mem_mode) + outbuf.clear(); - if( fmt == FileStorage::FORMAT_AUTO && !filename.empty() ) - { - const char* dot_pos = NULL; - const char* dot_pos2 = NULL; - // like strrchr() implementation, but save two last positions simultaneously - for (const char* pos = &filename[0]; pos[0] != 0; pos++) - { - if( pos[0] == '.' ) - { - dot_pos2 = dot_pos; - dot_pos = pos; - } - } - if (fs::strcasecmp(dot_pos, ".gz") == 0 && dot_pos2 != NULL) - { - dot_pos = dot_pos2; + if (fmt == FileStorage::FORMAT_AUTO && !filename.empty()) { + const char *dot_pos = NULL; + const char *dot_pos2 = NULL; + // like strrchr() implementation, but save two last positions simultaneously + for (const char *pos = &filename[0]; pos[0] != 0; pos++) { + if (pos[0] == '.') { + dot_pos2 = dot_pos; + dot_pos = pos; } - fmt = (fs::strcasecmp(dot_pos, ".xml") == 0 || fs::strcasecmp(dot_pos, ".xml.gz") == 0 ) - ? FileStorage::FORMAT_XML - : (fs::strcasecmp(dot_pos, ".json") == 0 || fs::strcasecmp(dot_pos, ".json.gz") == 0) - ? FileStorage::FORMAT_JSON - : FileStorage::FORMAT_YAML; - } - else if( fmt == FileStorage::FORMAT_AUTO ) - { - fmt = FileStorage::FORMAT_XML; } - - // we use factor=6 for XML (the longest characters (' and ") are encoded with 6 bytes (' and ") - // and factor=4 for YAML ( as we use 4 bytes for non ASCII characters (e.g. \xAB)) - int buf_size = CV_FS_MAX_LEN*(fmt == FileStorage::FORMAT_XML ? 6 : 4) + 1024; - - if (append) - { - fseek( file, 0, SEEK_END ); - if (ftell(file) == 0) - append = false; + if (fs::strcasecmp(dot_pos, ".gz") == 0 && dot_pos2 != NULL) { + dot_pos = dot_pos2; } + fmt = (fs::strcasecmp(dot_pos, ".xml") == 0 || fs::strcasecmp(dot_pos, ".xml.gz") == 0) + ? FileStorage::FORMAT_XML + : (fs::strcasecmp(dot_pos, ".json") == 0 || fs::strcasecmp(dot_pos, ".json.gz") == 0) + ? FileStorage::FORMAT_JSON + : FileStorage::FORMAT_YAML; + } else if (fmt == FileStorage::FORMAT_AUTO) { + fmt = FileStorage::FORMAT_XML; + } - write_stack.clear(); - empty_stream = true; - write_stack.push_back(FStructData("", FileNode::MAP | FileNode::EMPTY, 0)); - buffer.reserve(buf_size + 1024); - buffer.resize(buf_size); - bufofs = 0; + // we use factor=6 for XML (the longest characters (' and ") are encoded with 6 bytes (' and ") + // and factor=4 for YAML ( as we use 4 bytes for non ASCII characters (e.g. \xAB)) + int buf_size = CV_FS_MAX_LEN * (fmt == FileStorage::FORMAT_XML ? 6 : 4) + 1024; - if( fmt == FileStorage::FORMAT_XML ) - { - size_t file_size = file ? (size_t)ftell(file) : (size_t)0; - if( !append || file_size == 0 ) - { - if( encoding && *encoding != '\0' ) - { - if( fs::strcasecmp(encoding, "UTF-16" ) == 0 ) - { - release(); - CV_Error( CV_StsBadArg, "UTF-16 XML encoding is not supported! Use 8-bit encoding\n"); - } + if (append) { + fseek(file, 0, SEEK_END); + if (ftell(file) == 0) + append = false; + } - CV_Assert( strlen(encoding) < 1000 ); - char buf[1100]; - sprintf(buf, "\n", encoding); - puts( buf ); + write_stack.clear(); + empty_stream = true; + write_stack.push_back(FStructData("", FileNode::MAP | FileNode::EMPTY, 0)); + buffer.reserve(buf_size + 1024); + buffer.resize(buf_size); + bufofs = 0; + is_using_base64 = write_base64; + state_of_writing_base64 = FileStorage_API::Base64State::Uncertain; + + if (fmt == FileStorage::FORMAT_XML) { + size_t file_size = file ? (size_t) ftell(file) : (size_t) 0; + if (!append || file_size == 0) { + if (encoding && *encoding != '\0') { + if (fs::strcasecmp(encoding, "UTF-16") == 0) { + release(); + CV_Error(cv::Error::StsBadArg, "UTF-16 XML encoding is not supported! Use 8-bit encoding\n"); } - else - puts( "\n" ); - puts( "\n" ); - } - else - { - int xml_buf_size = 1 << 10; - char substr[] = ""; - int last_occurrence = -1; - xml_buf_size = MIN(xml_buf_size, int(file_size)); - fseek( file, -xml_buf_size, SEEK_END ); - // find the last occurrence of - for(;;) - { - int line_offset = (int)ftell( file ); - const char* ptr0 = this->gets(xml_buf_size); - const char* ptr = NULL; - if( !ptr0 ) + + CV_Assert(strlen(encoding) < 1000); + char buf[1100]; + sprintf(buf, "\n", encoding); + puts(buf); + } else + puts("\n"); + puts("\n"); + } else { + int xml_buf_size = 1 << 10; + char substr[] = ""; + int last_occurrence = -1; + xml_buf_size = MIN(xml_buf_size, int(file_size)); + fseek(file, -xml_buf_size, SEEK_END); + // find the last occurrence of + for (;;) { + int line_offset = (int) ftell(file); + const char *ptr0 = this->gets(xml_buf_size); + const char *ptr = NULL; + if (!ptr0) + break; + ptr = ptr0; + for (;;) { + ptr = strstr(ptr, substr); + if (!ptr) break; - ptr = ptr0; - for(;;) - { - ptr = strstr( ptr, substr ); - if( !ptr ) - break; - last_occurrence = line_offset + (int)(ptr - ptr0); - ptr += strlen(substr); - } + last_occurrence = line_offset + (int) (ptr - ptr0); + ptr += strlen(substr); } - if( last_occurrence < 0 ) - { - release(); - CV_Error( CV_StsError, "Could not find in the end of file.\n" ); - } - closeFile(); - file = fopen( filename.c_str(), "r+t" ); - CV_Assert(file != 0); - fseek( file, last_occurrence, SEEK_SET ); - // replace the last "" with " ", which has the same length - puts( " " ); - fseek( file, 0, SEEK_END ); - puts( "\n" ); } - - emitter = createXMLEmitter(this); + if (last_occurrence < 0) { + release(); + CV_Error(cv::Error::StsError, "Could not find in the end of file.\n"); + } + closeFile(); + file = fopen(filename.c_str(), "r+t"); + CV_Assert(file != 0); + fseek(file, last_occurrence, SEEK_SET); + // replace the last "" with " ", which has the same length + puts(" "); + fseek(file, 0, SEEK_END); + puts("\n"); } - else if( fmt == FileStorage::FORMAT_YAML ) - { - if( !append) - puts( "%YAML:1.0\n---\n" ); - else - puts( "...\n---\n" ); - emitter = createYAMLEmitter(this); - } + emitter = createXMLEmitter(this); + } else if (fmt == FileStorage::FORMAT_YAML) { + if (!append) + puts("%YAML:1.0\n---\n"); else - { - CV_Assert( fmt == FileStorage::FORMAT_JSON ); - if( !append ) - puts( "{\n" ); - else - { - bool valid = false; - long roffset = 0; - for ( ; - fseek( file, roffset, SEEK_END ) == 0; - roffset -= 1 ) - { - const char end_mark = '}'; - if ( fgetc( file ) == end_mark ) - { - fseek( file, roffset, SEEK_END ); - valid = true; - break; - } + puts("...\n---\n"); + + emitter = createYAMLEmitter(this); + } else { + CV_Assert(fmt == FileStorage::FORMAT_JSON); + if (!append) + puts("{\n"); + else { + bool valid = false; + long roffset = 0; + for (; + fseek(file, roffset, SEEK_END) == 0; + roffset -= 1) { + const char end_mark = '}'; + if (fgetc(file) == end_mark) { + fseek(file, roffset, SEEK_END); + valid = true; + break; } + } - if ( valid ) - { - closeFile(); - file = fopen( filename.c_str(), "r+t" ); - CV_Assert(file != 0); - fseek( file, roffset, SEEK_END ); - fputs( ",", file ); - } - else - { - CV_Error( CV_StsError, "Could not find '}' in the end of file.\n" ); - } + if (valid) { + closeFile(); + file = fopen(filename.c_str(), "r+t"); + CV_Assert(file != 0); + fseek(file, roffset, SEEK_END); + fputs(",", file); + } else { + CV_Error(cv::Error::StsError, "Could not find '}' in the end of file.\n"); } - write_stack.back().indent = 4; - emitter = createJSONEmitter(this); } - is_opened = true; + write_stack.back().indent = 4; + emitter = createJSONEmitter(this); + } + is_opened = true; + } else { + const size_t buf_size0 = 40; + buffer.resize(buf_size0); + if (mem_mode) { + strbuf = (char *) filename_or_buf; + strbufsize = strlen(strbuf); } + + const char *yaml_signature = "%YAML"; + const char *json_signature = "{"; + const char *xml_signature = "gets(16); + CV_Assert(buf); + char *bufPtr = cv_skip_BOM(buf); + size_t bufOffset = bufPtr - buf; + + if (strncmp(bufPtr, yaml_signature, strlen(yaml_signature)) == 0) + fmt = FileStorage::FORMAT_YAML; + else if (strncmp(bufPtr, json_signature, strlen(json_signature)) == 0) + fmt = FileStorage::FORMAT_JSON; + else if (strncmp(bufPtr, xml_signature, strlen(xml_signature)) == 0) + fmt = FileStorage::FORMAT_XML; + else if (strbufsize == bufOffset) + CV_Error(cv::Error::StsBadArg, "Input file is invalid"); else - { - const size_t buf_size0 = 40; - buffer.resize(buf_size0); - if( mem_mode ) - { - strbuf = (char*)filename_or_buf; - strbufsize = strlen(strbuf); - } + CV_Error(cv::Error::StsBadArg, "Unsupported file storage format"); - const char* yaml_signature = "%YAML"; - const char* json_signature = "{"; - const char* xml_signature = "gets(16); - CV_Assert(buf); - char* bufPtr = cv_skip_BOM(buf); - size_t bufOffset = bufPtr - buf; - - if(strncmp( bufPtr, yaml_signature, strlen(yaml_signature) ) == 0) - fmt = FileStorage::FORMAT_YAML; - else if(strncmp( bufPtr, json_signature, strlen(json_signature) ) == 0) - fmt = FileStorage::FORMAT_JSON; - else if(strncmp( bufPtr, xml_signature, strlen(xml_signature) ) == 0) - fmt = FileStorage::FORMAT_XML; - else if(strbufsize == bufOffset) - CV_Error(CV_BADARG_ERR, "Input file is invalid"); - else - CV_Error(CV_BADARG_ERR, "Unsupported file storage format"); + rewind(); + strbufpos = bufOffset; + bufofs = 0; - rewind(); - strbufpos = bufOffset; - bufofs = 0; + try { + char *ptr = bufferStart(); + ptr[0] = ptr[1] = ptr[2] = '\0'; + FileNode root_nodes(fs_ext, 0, 0); - try - { - char* ptr = bufferStart(); - ptr[0] = ptr[1] = ptr[2] = '\0'; - FileNode root_nodes(fs_ext, 0, 0); + uchar *rptr = reserveNodeSpace(root_nodes, 9); + *rptr = FileNode::SEQ; + writeInt(rptr + 1, 4); + writeInt(rptr + 5, 0); - uchar* rptr = reserveNodeSpace(root_nodes, 9); - *rptr = FileNode::SEQ; - writeInt(rptr + 1, 4); - writeInt(rptr + 5, 0); + roots.clear(); - roots.clear(); + switch (fmt) { + case FileStorage::FORMAT_XML: + parser = createXMLParser(this); + break; + case FileStorage::FORMAT_YAML: + parser = createYAMLParser(this); + break; + case FileStorage::FORMAT_JSON: + parser = createJSONParser(this); + break; + default: + parser = Ptr(); + } - switch (fmt) - { - case FileStorage::FORMAT_XML: parser = createXMLParser(this); break; - case FileStorage::FORMAT_YAML: parser = createYAMLParser(this); break; - case FileStorage::FORMAT_JSON: parser = createJSONParser(this); break; - default: parser = Ptr(); - } + if (!parser.empty()) { + ok = parser->parse(ptr); + if (ok) { + finalizeCollection(root_nodes); - if( !parser.empty() ) - { - ok = parser->parse(ptr); - if( ok ) - { - finalizeCollection(root_nodes); + CV_Assert(!fs_data_ptrs.empty()); + FileNode roots_node(fs_ext, 0, 0); + size_t i, nroots = roots_node.size(); + FileNodeIterator it = roots_node.begin(); - CV_Assert( !fs_data_ptrs.empty() ); - FileNode roots_node(fs_ext, 0, 0); - size_t i, nroots = roots_node.size(); - FileNodeIterator it = roots_node.begin(); - - for( i = 0; i < nroots; i++, ++it ) - roots.push_back(*it); - } + for (i = 0; i < nroots; i++, ++it) + roots.push_back(*it); } } - catch(...) - { - is_opened = true; - release(); - throw; - } - - // release resources that we do not need anymore - closeFile(); + } + catch (...) { is_opened = true; - std::vector tmpbuf; - std::swap(buffer, tmpbuf); - bufofs = 0; + release(); + throw; } - return ok; + + // release resources that we do not need anymore + closeFile(); + is_opened = true; + std::vector tmpbuf; + std::swap(buffer, tmpbuf); + bufofs = 0; } + return ok; +} - void puts( const char* str ) - { - CV_Assert( write_mode ); - if( mem_mode ) - std::copy(str, str + strlen(str), std::back_inserter(outbuf)); - else if( file ) - fputs( str, file ); +void FileStorage::Impl::puts(const char *str) { + CV_Assert(write_mode); + if (mem_mode) + std::copy(str, str + strlen(str), std::back_inserter(outbuf)); + else if (file) + fputs(str, file); #if USE_ZLIB - else if( gzfile ) - gzputs( gzfile, str ); + else if (gzfile) + gzputs(gzfile, str); #endif - else - CV_Error( CV_StsError, "The storage is not opened" ); - } - - char* getsFromFile( char* buf, int count ) - { - if( file ) - return fgets( buf, count, file ); - #if USE_ZLIB - if( gzfile ) - return gzgets( gzfile, buf, count ); - #endif - CV_Error(CV_StsError, "The storage is not opened"); - } + else + CV_Error(cv::Error::StsError, "The storage is not opened"); +} - char* gets( size_t maxCount ) - { - if( strbuf ) - { - size_t i = strbufpos, len = strbufsize; - const char* instr = strbuf; - for( ; i < len; i++ ) - { - char c = instr[i]; - if( c == '\0' || c == '\n' ) - { - if( c == '\n' ) - i++; - break; - } +char *FileStorage::Impl::getsFromFile(char *buf, int count) { + if (file) + return fgets(buf, count, file); +#if USE_ZLIB + if (gzfile) + return gzgets(gzfile, buf, count); +#endif + CV_Error(cv::Error::StsError, "The storage is not opened"); +} + +char *FileStorage::Impl::gets(size_t maxCount) { + if (strbuf) { + size_t i = strbufpos, len = strbufsize; + const char *instr = strbuf; + for (; i < len; i++) { + char c = instr[i]; + if (c == '\0' || c == '\n') { + if (c == '\n') + i++; + break; } - size_t count = i - strbufpos; - if( maxCount == 0 || maxCount > count ) - maxCount = count; - buffer.resize(std::max(buffer.size(), maxCount + 8)); - memcpy(&buffer[0], instr + strbufpos, maxCount); - buffer[maxCount] = '\0'; - strbufpos = i; - return maxCount > 0 ? &buffer[0] : 0; } + size_t count = i - strbufpos; + if (maxCount == 0 || maxCount > count) + maxCount = count; + buffer.resize(std::max(buffer.size(), maxCount + 8)); + memcpy(&buffer[0], instr + strbufpos, maxCount); + buffer[maxCount] = '\0'; + strbufpos = i; + return maxCount > 0 ? &buffer[0] : 0; + } + + const size_t MAX_BLOCK_SIZE = INT_MAX / 2; // hopefully, that will be enough + if (maxCount == 0) + maxCount = MAX_BLOCK_SIZE; + else + CV_Assert(maxCount < MAX_BLOCK_SIZE); + size_t ofs = 0; - const size_t MAX_BLOCK_SIZE = INT_MAX/2; // hopefully, that will be enough - if( maxCount == 0 ) - maxCount = MAX_BLOCK_SIZE; - else - CV_Assert(maxCount < MAX_BLOCK_SIZE); - size_t ofs = 0; - - for(;;) - { - int count = (int)std::min(buffer.size() - ofs - 16, maxCount); - char* ptr = getsFromFile( &buffer[ofs], count+1 ); - if( !ptr ) - break; - int delta = (int)strlen(ptr); - ofs += delta; - maxCount -= delta; - if( ptr[delta-1] == '\n' || maxCount == 0 ) - break; - if( delta == count ) - buffer.resize((size_t)(buffer.size()*1.5)); - } - return ofs > 0 ? &buffer[0] : 0; + for (;;) { + int count = (int) std::min(buffer.size() - ofs - 16, maxCount); + char *ptr = getsFromFile(&buffer[ofs], count + 1); + if (!ptr) + break; + int delta = (int) strlen(ptr); + ofs += delta; + maxCount -= delta; + if (ptr[delta - 1] == '\n' || maxCount == 0) + break; + if (delta == count) + buffer.resize((size_t) (buffer.size() * 1.5)); } + return ofs > 0 ? &buffer[0] : 0; +} - char* gets() - { - char* ptr = this->gets(0); - if( !ptr ) - { - ptr = bufferStart(); // FIXIT Why do we need this hack? What is about other parsers JSON/YAML? - *ptr = '\0'; - setEof(); - return 0; - } - else - { - size_t l = strlen(ptr); - if( l > 0 && ptr[l-1] != '\n' && ptr[l-1] != '\r' && !eof() ) - { - ptr[l] = '\n'; - ptr[l+1] = '\0'; - } +char *FileStorage::Impl::gets() { + char *ptr = this->gets(0); + if (!ptr) { + ptr = bufferStart(); // FIXIT Why do we need this hack? What is about other parsers JSON/YAML? + *ptr = '\0'; + setEof(); + return 0; + } else { + size_t l = strlen(ptr); + if (l > 0 && ptr[l - 1] != '\n' && ptr[l - 1] != '\r' && !eof()) { + ptr[l] = '\n'; + ptr[l + 1] = '\0'; } - lineno++; - return ptr; } + lineno++; + return ptr; +} - bool eof() - { - if( dummy_eof ) - return true; - if( strbuf ) - return strbufpos >= strbufsize; - if( file ) - return feof(file) != 0; +bool FileStorage::Impl::eof() { + if (dummy_eof) + return true; + if (strbuf) + return strbufpos >= strbufsize; + if (file) + return feof(file) != 0; #if USE_ZLIB - if( gzfile ) - return gzeof(gzfile) != 0; + if (gzfile) + return gzeof(gzfile) != 0; #endif - return false; - } + return false; +} - void setEof() - { - dummy_eof = true; - } +void FileStorage::Impl::setEof() { + dummy_eof = true; +} - void closeFile() - { - if( file ) - fclose( file ); +void FileStorage::Impl::closeFile() { + if (file) + fclose(file); #if USE_ZLIB - else if( gzfile ) - gzclose( gzfile ); + else if (gzfile) + gzclose(gzfile); #endif - file = 0; - gzfile = 0; - strbuf = 0; - strbufpos = 0; - is_opened = false; - } + file = 0; + gzfile = 0; + strbuf = 0; + strbufpos = 0; + is_opened = false; +} - void rewind() - { - if( file ) - ::rewind(file); +void FileStorage::Impl::rewind() { + if (file) + ::rewind(file); #if USE_ZLIB - else if( gzfile ) - gzrewind(gzfile); + else if (gzfile) + gzrewind(gzfile); #endif - strbufpos = 0; - } + strbufpos = 0; +} - char* resizeWriteBuffer( char* ptr, int len ) - { - const char* buffer_end = &buffer[0] + buffer.size(); - if( ptr + len < buffer_end ) - return ptr; +char *FileStorage::Impl::resizeWriteBuffer(char *ptr, int len) { + const char *buffer_end = &buffer[0] + buffer.size(); + if (ptr + len < buffer_end) + return ptr; - const char* buffer_start = &buffer[0]; - int written_len = (int)(ptr - buffer_start); + const char *buffer_start = &buffer[0]; + int written_len = (int) (ptr - buffer_start); - CV_Assert(written_len <= (int)buffer.size()); - int new_size = (int)((buffer_end - buffer_start)*3/2); - new_size = MAX( written_len + len, new_size ); - buffer.reserve( new_size + 256 ); - buffer.resize( new_size ); - bufofs = written_len; - return &buffer[0] + bufofs; + CV_Assert(written_len <= (int) buffer.size()); + int new_size = (int) ((buffer_end - buffer_start) * 3 / 2); + new_size = MAX(written_len + len, new_size); + buffer.reserve(new_size + 256); + buffer.resize(new_size); + bufofs = written_len; + return &buffer[0] + bufofs; +} + +char *FileStorage::Impl::flush() { + char *buffer_start = &buffer[0]; + char *ptr = buffer_start + bufofs; + + if (ptr > buffer_start + space) { + ptr[0] = '\n'; + ptr[1] = '\0'; + puts(buffer_start); + bufofs = 0; } - char* flush() - { - char* buffer_start = &buffer[0]; - char* ptr = buffer_start + bufofs; + int indent = write_stack.back().indent; - if( ptr > buffer_start + space ) - { - ptr[0] = '\n'; - ptr[1] = '\0'; - puts( buffer_start ); - bufofs = 0; - } + if (space != indent) { + memset(buffer_start, ' ', indent); + space = indent; + } + bufofs = space; + ptr = buffer_start + bufofs; - int indent = write_stack.back().indent; + return ptr; +} - if( space != indent ) - { - memset( buffer_start, ' ', indent ); - space = indent; - } - bufofs = space; - ptr = buffer_start + bufofs; +void FileStorage::Impl::endWriteStruct() { + CV_Assert(write_mode); - return ptr; - } + check_if_write_struct_is_delayed(false); + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); - void endWriteStruct() - { - CV_Assert( write_mode ); - CV_Assert( !write_stack.empty() ); + CV_Assert(!write_stack.empty()); - FStructData& current_struct = write_stack.back(); - if( fmt == FileStorage::FORMAT_JSON && !FileNode::isFlow(current_struct.flags) && write_stack.size() > 1 ) - current_struct.indent = write_stack[write_stack.size() - 2].indent; + FStructData ¤t_struct = write_stack.back(); + if (fmt == FileStorage::FORMAT_JSON && !FileNode::isFlow(current_struct.flags) && write_stack.size() > 1) + current_struct.indent = write_stack[write_stack.size() - 2].indent; - emitter->endWriteStruct(current_struct); + emitter->endWriteStruct(current_struct); - write_stack.pop_back(); - if( !write_stack.empty() ) - write_stack.back().flags &= ~FileNode::EMPTY; - } + write_stack.pop_back(); + if (!write_stack.empty()) + write_stack.back().flags &= ~FileNode::EMPTY; +} - void startWriteStruct( const char* key, int struct_flags, - const char* type_name ) - { - CV_Assert( write_mode ); +void FileStorage::Impl::startWriteStruct_helper(const char *key, int struct_flags, + const char *type_name) { + CV_Assert(write_mode); - struct_flags = (struct_flags & (FileNode::TYPE_MASK|FileNode::FLOW)) | FileNode::EMPTY; - if( !FileNode::isCollection(struct_flags)) - CV_Error( CV_StsBadArg, - "Some collection type: FileNode::SEQ or FileNode::MAP must be specified" ); + struct_flags = (struct_flags & (FileNode::TYPE_MASK | FileNode::FLOW)) | FileNode::EMPTY; + if (!FileNode::isCollection(struct_flags)) + CV_Error(cv::Error::StsBadArg, + "Some collection type: FileNode::SEQ or FileNode::MAP must be specified"); - if( type_name && type_name[0] == '\0' ) - type_name = 0; + if (type_name && type_name[0] == '\0') + type_name = 0; - FStructData s = emitter->startWriteStruct( write_stack.back(), key, struct_flags, type_name ); - write_stack.push_back(s); - size_t write_stack_size = write_stack.size(); - if( write_stack_size > 1 ) - write_stack[write_stack_size-2].flags &= ~FileNode::EMPTY; + FStructData s = emitter->startWriteStruct(write_stack.back(), key, struct_flags, type_name); - if( !FileNode::isFlow(s.flags) ) - flush(); + write_stack.push_back(s); + size_t write_stack_size = write_stack.size(); + if (write_stack_size > 1) + write_stack[write_stack_size - 2].flags &= ~FileNode::EMPTY; - if( fmt == FileStorage::FORMAT_JSON && type_name && type_name[0] && FileNode::isMap(struct_flags)) - { - emitter->write("type_id", type_name, false); - } - } + if (fmt != FileStorage::FORMAT_JSON && !FileNode::isFlow(s.flags)) + flush(); - void writeComment( const char* comment, bool eol_comment ) - { - CV_Assert(write_mode); - emitter->writeComment( comment, eol_comment ); + if (fmt == FileStorage::FORMAT_JSON && type_name && type_name[0] && FileNode::isMap(struct_flags)) { + emitter->write("type_id", type_name, false); } +} - void startNextStream() - { - CV_Assert(write_mode); - if( !empty_stream ) - { - while( !write_stack.empty() ) - endWriteStruct(); - flush(); - emitter->startNextStream(); - empty_stream = true; - write_stack.push_back(FStructData("", FileNode::EMPTY, 0)); - bufofs = 0; - } - } +void FileStorage::Impl::startWriteStruct(const char *key, int struct_flags, + const char *type_name) { + check_if_write_struct_is_delayed(false); + if (state_of_writing_base64 == FileStorage_API::NotUse) + switch_to_Base64_state(FileStorage_API::Uncertain); - void write( const String& key, int value ) - { - CV_Assert(write_mode); - emitter->write(key.c_str(), value); - } + if (state_of_writing_base64 == FileStorage_API::Uncertain && FileNode::isSeq(struct_flags) + && is_using_base64 && type_name == 0) { + /* Uncertain whether output Base64 data */ + make_write_struct_delayed(key, struct_flags, type_name); + } else if (type_name && memcmp(type_name, "binary", 6) == 0) { + /* Must output Base64 data */ + if ((FileNode::TYPE_MASK & struct_flags) != FileNode::SEQ) + CV_Error(cv::Error::StsBadArg, "must set 'struct_flags |= CV_NODE_SEQ' if using Base64."); + else if (state_of_writing_base64 != FileStorage_API::Uncertain) + CV_Error(cv::Error::StsError, "function \'cvStartWriteStruct\' calls cannot be nested if using Base64."); - void write( const String& key, double value ) - { - CV_Assert(write_mode); - emitter->write(key.c_str(), value); + startWriteStruct_helper(key, struct_flags, "binary"); + + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::InUse); + } else { + /* Won't output Base64 data */ + if (state_of_writing_base64 == FileStorage_API::InUse) + CV_Error(cv::Error::StsError, "At the end of the output Base64, `cvEndWriteStruct` is needed."); + + startWriteStruct_helper(key, struct_flags, type_name); + + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::NotUse); } +} - void write( const String& key, const String& value ) - { - CV_Assert(write_mode); - emitter->write(key.c_str(), value.c_str(), false); +void FileStorage::Impl::writeComment(const char *comment, bool eol_comment) { + CV_Assert(write_mode); + emitter->writeComment(comment, eol_comment); +} + +void FileStorage::Impl::startNextStream() { + CV_Assert(write_mode); + if (!empty_stream) { + while (!write_stack.empty()) + endWriteStruct(); + flush(); + emitter->startNextStream(); + empty_stream = true; + write_stack.push_back(FStructData("", FileNode::EMPTY, 0)); + bufofs = 0; } +} - void writeRawData( const std::string& dt, const void* _data, size_t len ) - { - CV_Assert(write_mode); +void FileStorage::Impl::write(const String &key, int value) { + CV_Assert(write_mode); + emitter->write(key.c_str(), value); +} - size_t elemSize = fs::calcStructSize(dt.c_str(), 0); - CV_Assert(elemSize); - CV_Assert( len % elemSize == 0 ); - len /= elemSize; +void FileStorage::Impl::write(const String &key, double value) { + CV_Assert(write_mode); + emitter->write(key.c_str(), value); +} - bool explicitZero = fmt == FileStorage::FORMAT_JSON; - const uchar* data0 = (const uchar*)_data; - int fmt_pairs[CV_FS_MAX_FMT_PAIRS*2], k, fmt_pair_count; - char buf[256] = ""; +void FileStorage::Impl::write(const String &key, const String &value) { + CV_Assert(write_mode); + emitter->write(key.c_str(), value.c_str(), false); +} - fmt_pair_count = fs::decodeFormat( dt.c_str(), fmt_pairs, CV_FS_MAX_FMT_PAIRS ); +void FileStorage::Impl::writeRawData(const std::string &dt, const void *_data, size_t len) { + CV_Assert(write_mode); - if( !len ) - return; + if (is_using_base64 || state_of_writing_base64 == FileStorage_API::Base64State::InUse) { + writeRawDataBase64(_data, len, dt.c_str()); + return; + } else if (state_of_writing_base64 == FileStorage_API::Base64State::Uncertain) { + switch_to_Base64_state(FileStorage_API::Base64State::NotUse); + } - if( !data0 ) - CV_Error( CV_StsNullPtr, "Null data pointer" ); + size_t elemSize = fs::calcStructSize(dt.c_str(), 0); + CV_Assert(elemSize); + CV_Assert(len % elemSize == 0); + len /= elemSize; - if( fmt_pair_count == 1 ) - { - fmt_pairs[0] *= (int)len; - len = 1; - } + bool explicitZero = fmt == FileStorage::FORMAT_JSON; + const uchar *data0 = (const uchar *) _data; + int fmt_pairs[CV_FS_MAX_FMT_PAIRS * 2], k, fmt_pair_count; + char buf[256] = ""; - for(;len--; data0 += elemSize) - { - int offset = 0; - for( k = 0; k < fmt_pair_count; k++ ) - { - int i, count = fmt_pairs[k*2]; - int elem_type = fmt_pairs[k*2+1]; - int elem_size = CV_ELEM_SIZE(elem_type); - const char *ptr; + fmt_pair_count = fs::decodeFormat(dt.c_str(), fmt_pairs, CV_FS_MAX_FMT_PAIRS); - offset = cvAlign( offset, elem_size ); - const uchar* data = data0 + offset; + if (!len) + return; - for( i = 0; i < count; i++ ) - { - switch( elem_type ) - { + if (!data0) + CV_Error(cv::Error::StsNullPtr, "Null data pointer"); + + if (fmt_pair_count == 1) { + fmt_pairs[0] *= (int) len; + len = 1; + } + + for (; len--; data0 += elemSize) { + int offset = 0; + for (k = 0; k < fmt_pair_count; k++) { + int i, count = fmt_pairs[k * 2]; + int elem_type = fmt_pairs[k * 2 + 1]; + int elem_size = CV_ELEM_SIZE(elem_type); + const char *ptr; + + offset = cvAlign(offset, elem_size); + const uchar *data = data0 + offset; + + for (i = 0; i < count; i++) { + switch (elem_type) { case CV_8U: - ptr = fs::itoa( *(uchar*)data, buf, 10 ); + ptr = fs::itoa(*(uchar *) data, buf, 10); data++; break; case CV_8S: - ptr = fs::itoa( *(char*)data, buf, 10 ); + ptr = fs::itoa(*(char *) data, buf, 10); data++; break; case CV_16U: - ptr = fs::itoa( *(ushort*)data, buf, 10 ); + ptr = fs::itoa(*(ushort *) data, buf, 10); data += sizeof(ushort); break; case CV_16S: - ptr = fs::itoa( *(short*)data, buf, 10 ); + ptr = fs::itoa(*(short *) data, buf, 10); data += sizeof(short); break; case CV_32S: - ptr = fs::itoa( *(int*)data, buf, 10 ); + ptr = fs::itoa(*(int *) data, buf, 10); data += sizeof(int); break; case CV_32F: - ptr = fs::floatToString( buf, *(float*)data, false, explicitZero ); + ptr = fs::floatToString(buf, *(float *) data, false, explicitZero); data += sizeof(float); break; case CV_64F: - ptr = fs::doubleToString( buf, *(double*)data, explicitZero ); + ptr = fs::doubleToString(buf, *(double *) data, explicitZero); data += sizeof(double); break; case CV_16F: /* reference */ - ptr = fs::floatToString( buf, (float)*(float16_t*)data, true, explicitZero ); + ptr = fs::floatToString(buf, (float) *(float16_t *) data, true, explicitZero); data += sizeof(float16_t); break; default: - CV_Error( CV_StsUnsupportedFormat, "Unsupported type" ); + CV_Error(cv::Error::StsUnsupportedFormat, "Unsupported type"); return; - } - - emitter->writeScalar(0, ptr); } - offset = (int)(data - data0); + emitter->writeScalar(0, ptr); } + + offset = (int) (data - data0); } } +} - void writeRawDataBase64(const void* /*data*/, int /*len*/, const char* /*dt*/ ) - { +void FileStorage::Impl::workaround() { + check_if_write_struct_is_delayed(false); - } + if (state_of_writing_base64 != FileStorage_API::Base64State::Uncertain) + switch_to_Base64_state(FileStorage_API::Base64State::Uncertain); +} - String releaseAndGetString(); +void FileStorage::Impl::switch_to_Base64_state(FileStorage_API::Base64State new_state) { + const char *err_unkonwn_state = "Unexpected error, unable to determine the Base64 state."; + const char *err_unable_to_switch = "Unexpected error, unable to switch to this state."; - FileNode getFirstTopLevelNode() const - { - return roots.empty() ? FileNode() : roots[0]; + /* like a finite state machine */ + switch (state_of_writing_base64) { + case FileStorage_API::Base64State::Uncertain: + switch (new_state) { + case FileStorage_API::Base64State::InUse: + { + CV_DbgAssert(base64_writer == 0); + bool can_indent = (fmt != cv::FileStorage::Mode::FORMAT_JSON); + base64_writer = new base64::Base64Writer(*this, can_indent); + if (!can_indent) { + char *ptr = bufferPtr(); + *ptr++ = '\0'; + puts(bufferStart()); + setBufferPtr(bufferStart()); + memset(bufferStart(), 0, static_cast(space)); + puts("\"$base64$"); + } + break; + } + case FileStorage_API::Base64State::Uncertain: + break; + case FileStorage_API::Base64State::NotUse: + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; + } + break; + case FileStorage_API::Base64State::InUse: + switch (new_state) { + case FileStorage_API::Base64State::InUse: + case FileStorage_API::Base64State::NotUse: + CV_Error(cv::Error::StsError, err_unable_to_switch); + break; + case FileStorage_API::Base64State::Uncertain: + delete base64_writer; + base64_writer = 0; + if ( fmt == cv::FileStorage::FORMAT_JSON ) + { + puts("\""); + setBufferPtr(bufferStart()); + flush(); + memset(bufferStart(), 0, static_cast(space) ); + setBufferPtr(bufferStart()); + } + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; + } + break; + case FileStorage_API::Base64State::NotUse: + switch (new_state) { + case FileStorage_API::Base64State::InUse: + case FileStorage_API::Base64State::NotUse: + CV_Error(cv::Error::StsError, err_unable_to_switch); + break; + case FileStorage_API::Base64State::Uncertain: + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; + } + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; } - FileNode root(int streamIdx=0) const - { - return streamIdx >= 0 && streamIdx < (int)roots.size() ? roots[streamIdx] : FileNode(); - } + state_of_writing_base64 = new_state; +} - FileNode operator[](const String& nodename) const - { - return this->operator[](nodename.c_str()); +void FileStorage::Impl::make_write_struct_delayed(const char *key, int struct_flags, const char *type_name) { + CV_Assert(is_write_struct_delayed == false); + CV_DbgAssert(delayed_struct_key == nullptr); + CV_DbgAssert(delayed_struct_flags == 0); + CV_DbgAssert(delayed_type_name == nullptr); + + delayed_struct_flags = struct_flags; + + if (key != nullptr) { + delayed_struct_key = new char[strlen(key) + 1U]; + strcpy(delayed_struct_key, key); } - FileNode operator[](const char* /*nodename*/) const - { - return FileNode(); + if (type_name != nullptr) { + delayed_type_name = new char[strlen(type_name) + 1U]; + strcpy(delayed_type_name, type_name); } - int getFormat() const { return fmt; } + is_write_struct_delayed = true; +} - char* bufferPtr() const { return (char*)(&buffer[0] + bufofs); } - char* bufferStart() const { return (char*)&buffer[0]; } - char* bufferEnd() const { return (char*)(&buffer[0] + buffer.size()); } - void setBufferPtr(char* ptr) - { - char* bufferstart = bufferStart(); - CV_Assert( ptr >= bufferstart && ptr <= bufferEnd() ); - bufofs = ptr - bufferstart; - } - int wrapMargin() const { return wrap_margin; } +void FileStorage::Impl::check_if_write_struct_is_delayed(bool change_type_to_base64) { + if (is_write_struct_delayed) { + /* save data to prevent recursive call errors */ + std::string struct_key; + std::string type_name; + int struct_flags = delayed_struct_flags; - FStructData& getCurrentStruct() - { - CV_Assert(!write_stack.empty()); - return write_stack.back(); + if (delayed_struct_key != nullptr && *delayed_struct_key != '\0') { + struct_key.assign(delayed_struct_key); + } + if (delayed_type_name != nullptr && *delayed_type_name != '\0') { + type_name.assign(delayed_type_name); + } + + /* reset */ + delete[] delayed_struct_key; + delete[] delayed_type_name; + delayed_struct_key = nullptr; + delayed_struct_flags = 0; + delayed_type_name = nullptr; + + is_write_struct_delayed = false; + + /* call */ + if (change_type_to_base64) { + startWriteStruct_helper(struct_key.c_str(), struct_flags, "binary"); + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::InUse); + } else { + startWriteStruct_helper(struct_key.c_str(), struct_flags, type_name.c_str()); + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::NotUse); + } } +} - void setNonEmpty() - { - empty_stream = false; +void FileStorage::Impl::writeRawDataBase64(const void *_data, size_t len, const char *dt) { + CV_Assert(write_mode); + + check_if_write_struct_is_delayed(true); + + if (state_of_writing_base64 == FileStorage_API::Base64State::Uncertain) { + switch_to_Base64_state(FileStorage_API::Base64State::InUse); + } else if (state_of_writing_base64 != FileStorage_API::Base64State::InUse) { + CV_Error(cv::Error::StsError, "Base64 should not be used at present."); } - void processSpecialDouble( char* buf, double* value, char** endptr ) - { - FileStorage_API* fs = this; - char c = buf[0]; - int inf_hi = 0x7ff00000; + base64_writer->write(_data, len, dt); +} - if( c == '-' || c == '+' ) - { - inf_hi = c == '-' ? 0xfff00000 : 0x7ff00000; - c = *++buf; - } +FileNode FileStorage::Impl::getFirstTopLevelNode() const { + return roots.empty() ? FileNode() : roots[0]; +} - if( c != '.' ) - CV_PARSE_ERROR_CPP( "Bad format of floating-point constant" ); +FileNode FileStorage::Impl::root(int streamIdx) const { + return streamIdx >= 0 && streamIdx < (int) roots.size() ? roots[streamIdx] : FileNode(); +} - Cv64suf v; - v.f = 0.; - if( toupper(buf[1]) == 'I' && toupper(buf[2]) == 'N' && toupper(buf[3]) == 'F' ) - v.u = (uint64)inf_hi << 32; - else if( toupper(buf[1]) == 'N' && toupper(buf[2]) == 'A' && toupper(buf[3]) == 'N' ) - v.u = (uint64)-1; - else - CV_PARSE_ERROR_CPP( "Bad format of floating-point constant" ); - *value = v.f; - *endptr = buf + 4; - } +FileNode FileStorage::Impl::operator[](const String &nodename) const { + return this->operator[](nodename.c_str()); +} - double strtod( char* ptr, char** endptr ) - { - double fval = ::strtod( ptr, endptr ); - if( **endptr == '.' ) - { - char* dot_pos = *endptr; - *dot_pos = ','; - double fval2 = ::strtod( ptr, endptr ); - *dot_pos = '.'; - if( *endptr > dot_pos ) - fval = fval2; - else - *endptr = dot_pos; - } +FileNode FileStorage::Impl::operator[](const char * /*nodename*/) const { + return FileNode(); +} - if( *endptr == ptr || cv_isalpha(**endptr) ) - processSpecialDouble( ptr, &fval, endptr ); +int FileStorage::Impl::getFormat() const { return fmt; } - return fval; - } +char *FileStorage::Impl::bufferPtr() const { return (char *) (&buffer[0] + bufofs); } - void convertToCollection(int type, FileNode& node) - { - CV_Assert( type == FileNode::SEQ || type == FileNode::MAP ); +char *FileStorage::Impl::bufferStart() const { return (char *) &buffer[0]; } - int node_type = node.type(); - if( node_type == type ) - return; +char *FileStorage::Impl::bufferEnd() const { return (char *) (&buffer[0] + buffer.size()); } - bool named = node.isNamed(); - uchar* ptr = node.ptr() + 1 + (named ? 4 : 0); +void FileStorage::Impl::setBufferPtr(char *ptr) { + char *bufferstart = bufferStart(); + CV_Assert(ptr >= bufferstart && ptr <= bufferEnd()); + bufofs = ptr - bufferstart; +} - int ival = 0; - double fval = 0; - std::string sval; - bool add_first_scalar = false; +int FileStorage::Impl::wrapMargin() const { return wrap_margin; } - if( node_type != FileNode::NONE ) - { - // scalar nodes can only be converted to sequences, e.g. in XML: - // 5[parser_position]... => create 5 with name "a" - // 5 6[parser_position]... => 5 is converted to [5] and then 6 is added to it - // - // otherwise we don't know where to get the element names from - CV_Assert( type == FileNode::SEQ ); - if( node_type == FileNode::INT ) - { - ival = readInt(ptr); - add_first_scalar = true; - } - else if( node_type == FileNode::REAL ) - { - fval = readReal(ptr); - add_first_scalar = true; - } - else if( node_type == FileNode::STRING ) - { - sval = std::string(node); - add_first_scalar = true; - } - else - CV_Error_(Error::StsError, ("The node of type %d cannot be converted to collection", node_type)); - } +FStructData &FileStorage::Impl::getCurrentStruct() { + CV_Assert(!write_stack.empty()); + return write_stack.back(); +} - ptr = reserveNodeSpace(node, 1 + (named ? 4 : 0) + 4 + 4); - *ptr++ = (uchar)(type | (named ? FileNode::NAMED : 0)); - // name has been copied automatically - if( named ) - ptr += 4; - // set raw_size(collection)==4, nelems(collection)==1 - writeInt(ptr, 4); - writeInt(ptr + 4, 0); - - if( add_first_scalar ) - addNode(node, std::string(), node_type, - node_type == FileNode::INT ? (const void*)&ival : - node_type == FileNode::REAL ? (const void*)&fval : - node_type == FileNode::STRING ? (const void*)sval.c_str() : 0, - -1); - } - - // a) allocates new FileNode (for that just set blockIdx to the last block and ofs to freeSpaceOfs) or - // b) reallocates just created new node (blockIdx and ofs must be taken from FileNode). - // If there is no enough space in the current block (it should be the last block added so far), - // the last block is shrunk so that it ends immediately before the reallocated node. Then, - // a new block of sufficient size is allocated and the FileNode is placed in the beginning of it. - // The case (a) can be used to allocate the very first node by setting blockIdx == ofs == 0. - // In the case (b) the existing tag and the name are copied automatically. - uchar* reserveNodeSpace(FileNode& node, size_t sz) - { - bool shrinkBlock = false; - size_t shrinkBlockIdx = 0, shrinkSize = 0; +void FileStorage::Impl::setNonEmpty() { + empty_stream = false; +} - uchar *ptr = 0, *blockEnd = 0; +void FileStorage::Impl::processSpecialDouble(char *buf, double *value, char **endptr) { + FileStorage_API *fs = this; + char c = buf[0]; + int inf_hi = 0x7ff00000; - if( !fs_data_ptrs.empty() ) - { - size_t blockIdx = node.blockIdx; - size_t ofs = node.ofs; - CV_Assert( blockIdx == fs_data_ptrs.size()-1 ); - CV_Assert( ofs <= fs_data_blksz[blockIdx] ); - CV_Assert( freeSpaceOfs <= fs_data_blksz[blockIdx] ); - //CV_Assert( freeSpaceOfs <= ofs + sz ); - - ptr = fs_data_ptrs[blockIdx] + ofs; - blockEnd = fs_data_ptrs[blockIdx] + fs_data_blksz[blockIdx]; - - CV_Assert(ptr >= fs_data_ptrs[blockIdx] && ptr <= blockEnd); - if( ptr + sz <= blockEnd ) - { - freeSpaceOfs = ofs + sz; - return ptr; - } + if (c == '-' || c == '+') { + inf_hi = c == '-' ? 0xfff00000 : 0x7ff00000; + c = *++buf; + } - if (ofs == 0) // FileNode is a first component of this block. Resize current block instead of allocation of new one. - { - fs_data[blockIdx]->resize(sz); - ptr = &fs_data[blockIdx]->at(0); - fs_data_ptrs[blockIdx] = ptr; - fs_data_blksz[blockIdx] = sz; - freeSpaceOfs = sz; - return ptr; - } + if (c != '.') + CV_PARSE_ERROR_CPP("Bad format of floating-point constant"); - shrinkBlock = true; - shrinkBlockIdx = blockIdx; - shrinkSize = ofs; + Cv64suf v; + v.f = 0.; + if (toupper(buf[1]) == 'I' && toupper(buf[2]) == 'N' && toupper(buf[3]) == 'F') + v.u = (uint64) inf_hi << 32; + else if (toupper(buf[1]) == 'N' && toupper(buf[2]) == 'A' && toupper(buf[3]) == 'N') + v.u = (uint64) -1; + else + CV_PARSE_ERROR_CPP("Bad format of floating-point constant"); + *value = v.f; + *endptr = buf + 4; +} + +double FileStorage::Impl::strtod(char *ptr, char **endptr) { + double fval = ::strtod(ptr, endptr); + if (**endptr == '.') { + char *dot_pos = *endptr; + *dot_pos = ','; + double fval2 = ::strtod(ptr, endptr); + *dot_pos = '.'; + if (*endptr > dot_pos) + fval = fval2; + else + *endptr = dot_pos; + } + + if (*endptr == ptr || cv_isalpha(**endptr)) + processSpecialDouble(ptr, &fval, endptr); + + return fval; +} + +void FileStorage::Impl::convertToCollection(int type, FileNode &node) { + CV_Assert(type == FileNode::SEQ || type == FileNode::MAP); + + int node_type = node.type(); + if (node_type == type) + return; + + bool named = node.isNamed(); + uchar *ptr = node.ptr() + 1 + (named ? 4 : 0); + + int ival = 0; + double fval = 0; + std::string sval; + bool add_first_scalar = false; + + if (node_type != FileNode::NONE) { + // scalar nodes can only be converted to sequences, e.g. in XML: + // 5[parser_position]... => create 5 with name "a" + // 5 6[parser_position]... => 5 is converted to [5] and then 6 is added to it + // + // otherwise we don't know where to get the element names from + CV_Assert(type == FileNode::SEQ); + if (node_type == FileNode::INT) { + ival = readInt(ptr); + add_first_scalar = true; + } else if (node_type == FileNode::REAL) { + fval = readReal(ptr); + add_first_scalar = true; + } else if (node_type == FileNode::STRING) { + sval = std::string(node); + add_first_scalar = true; + } else + CV_Error_(Error::StsError, ("The node of type %d cannot be converted to collection", node_type)); + } + + ptr = reserveNodeSpace(node, 1 + (named ? 4 : 0) + 4 + 4); + *ptr++ = (uchar) (type | (named ? FileNode::NAMED : 0)); + // name has been copied automatically + if (named) + ptr += 4; + // set raw_size(collection)==4, nelems(collection)==1 + writeInt(ptr, 4); + writeInt(ptr + 4, 0); + + if (add_first_scalar) + addNode(node, std::string(), node_type, + node_type == FileNode::INT ? (const void *) &ival : + node_type == FileNode::REAL ? (const void *) &fval : + node_type == FileNode::STRING ? (const void *) sval.c_str() : 0, + -1); +} + +// a) allocates new FileNode (for that just set blockIdx to the last block and ofs to freeSpaceOfs) or +// b) reallocates just created new node (blockIdx and ofs must be taken from FileNode). +// If there is no enough space in the current block (it should be the last block added so far), +// the last block is shrunk so that it ends immediately before the reallocated node. Then, +// a new block of sufficient size is allocated and the FileNode is placed in the beginning of it. +// The case (a) can be used to allocate the very first node by setting blockIdx == ofs == 0. +// In the case (b) the existing tag and the name are copied automatically. +uchar *FileStorage::Impl::reserveNodeSpace(FileNode &node, size_t sz) { + bool shrinkBlock = false; + size_t shrinkBlockIdx = 0, shrinkSize = 0; + + uchar *ptr = 0, *blockEnd = 0; + + if (!fs_data_ptrs.empty()) { + size_t blockIdx = node.blockIdx; + size_t ofs = node.ofs; + CV_Assert(blockIdx == fs_data_ptrs.size() - 1); + CV_Assert(ofs <= fs_data_blksz[blockIdx]); + CV_Assert(freeSpaceOfs <= fs_data_blksz[blockIdx]); + //CV_Assert( freeSpaceOfs <= ofs + sz ); + + ptr = fs_data_ptrs[blockIdx] + ofs; + blockEnd = fs_data_ptrs[blockIdx] + fs_data_blksz[blockIdx]; + + CV_Assert(ptr >= fs_data_ptrs[blockIdx] && ptr <= blockEnd); + if (ptr + sz <= blockEnd) { + freeSpaceOfs = ofs + sz; + return ptr; } - size_t blockSize = std::max((size_t)CV_FS_MAX_LEN*4 - 256, sz) + 256; - Ptr > pv = makePtr >(blockSize); - fs_data.push_back(pv); - uchar* new_ptr = &pv->at(0); - fs_data_ptrs.push_back(new_ptr); - fs_data_blksz.push_back(blockSize); - node.blockIdx = fs_data_ptrs.size()-1; - node.ofs = 0; - freeSpaceOfs = sz; - - if( ptr && ptr + 5 <= blockEnd ) + if (ofs == + 0) // FileNode is a first component of this block. Resize current block instead of allocation of new one. { - new_ptr[0] = ptr[0]; - if( ptr[0] & FileNode::NAMED ) - { - new_ptr[1] = ptr[1]; - new_ptr[2] = ptr[2]; - new_ptr[3] = ptr[3]; - new_ptr[4] = ptr[4]; - } + fs_data[blockIdx]->resize(sz); + ptr = &fs_data[blockIdx]->at(0); + fs_data_ptrs[blockIdx] = ptr; + fs_data_blksz[blockIdx] = sz; + freeSpaceOfs = sz; + return ptr; } - if (shrinkBlock) - { - fs_data[shrinkBlockIdx]->resize(shrinkSize); - fs_data_blksz[shrinkBlockIdx] = shrinkSize; + shrinkBlock = true; + shrinkBlockIdx = blockIdx; + shrinkSize = ofs; + } + + size_t blockSize = std::max((size_t) CV_FS_MAX_LEN * 4 - 256, sz) + 256; + Ptr > pv = makePtr >(blockSize); + fs_data.push_back(pv); + uchar *new_ptr = &pv->at(0); + fs_data_ptrs.push_back(new_ptr); + fs_data_blksz.push_back(blockSize); + node.blockIdx = fs_data_ptrs.size() - 1; + node.ofs = 0; + freeSpaceOfs = sz; + + if (ptr && ptr + 5 <= blockEnd) { + new_ptr[0] = ptr[0]; + if (ptr[0] & FileNode::NAMED) { + new_ptr[1] = ptr[1]; + new_ptr[2] = ptr[2]; + new_ptr[3] = ptr[3]; + new_ptr[4] = ptr[4]; } - - return new_ptr; } - unsigned getStringOfs( const std::string& key ) const - { - str_hash_t::const_iterator it = str_hash.find(key); - return it != str_hash.end() ? it->second : 0; + if (shrinkBlock) { + fs_data[shrinkBlockIdx]->resize(shrinkSize); + fs_data_blksz[shrinkBlockIdx] = shrinkSize; } - FileNode addNode( FileNode& collection, const std::string& key, - int elem_type, const void* value, int len ) - { - FileStorage_API* fs = this; - bool noname = key.empty() || (fmt == FileStorage::FORMAT_XML && strcmp(key.c_str(), "_") == 0); - convertToCollection( noname ? FileNode::SEQ : FileNode::MAP, collection ); - - bool isseq = collection.empty() ? false : collection.isSeq(); - if( noname != isseq ) - CV_PARSE_ERROR_CPP( noname ? "Map element should have a name" : - "Sequence element should not have name (use <_>)" ); - unsigned strofs = 0; - if( !noname ) - { - strofs = getStringOfs(key); - if( !strofs ) - { - strofs = (unsigned)str_hash_data.size(); - size_t keysize = key.size() + 1; - str_hash_data.resize(strofs + keysize); - memcpy(&str_hash_data[0] + strofs, &key[0], keysize); - str_hash.insert(std::make_pair(key, strofs)); - } - } + return new_ptr; +} - uchar* cp = collection.ptr(); +unsigned FileStorage::Impl::getStringOfs(const std::string &key) const { + str_hash_t::const_iterator it = str_hash.find(key); + return it != str_hash.end() ? it->second : 0; +} - size_t blockIdx = fs_data_ptrs.size() - 1; - size_t ofs = freeSpaceOfs; - FileNode node(fs_ext, blockIdx, ofs); +FileNode FileStorage::Impl::addNode(FileNode &collection, const std::string &key, + int elem_type, const void *value, int len) { + FileStorage_API *fs = this; + bool noname = key.empty() || (fmt == FileStorage::FORMAT_XML && strcmp(key.c_str(), "_") == 0); + convertToCollection(noname ? FileNode::SEQ : FileNode::MAP, collection); - size_t sz0 = 1 + (noname ? 0 : 4) + 8; - uchar* ptr = reserveNodeSpace(node, sz0); + bool isseq = collection.empty() ? false : collection.isSeq(); + if (noname != isseq) + CV_PARSE_ERROR_CPP(noname ? "Map element should have a name" : + "Sequence element should not have name (use <_>)"); + unsigned strofs = 0; + if (!noname) { + strofs = getStringOfs(key); + if (!strofs) { + strofs = (unsigned) str_hash_data.size(); + size_t keysize = key.size() + 1; + str_hash_data.resize(strofs + keysize); + memcpy(&str_hash_data[0] + strofs, &key[0], keysize); + str_hash.insert(std::make_pair(key, strofs)); + } + } - *ptr++ = (uchar)(elem_type | (noname ? 0 : FileNode::NAMED)); - if( elem_type == FileNode::NONE ) - freeSpaceOfs -= 8; + uchar *cp = collection.ptr(); - if( !noname ) - { - writeInt(ptr, (int)strofs); - ptr += 4; - } + size_t blockIdx = fs_data_ptrs.size() - 1; + size_t ofs = freeSpaceOfs; + FileNode node(fs_ext, blockIdx, ofs); - if( elem_type == FileNode::SEQ || elem_type == FileNode::MAP ) - { - writeInt(ptr, 4); - writeInt(ptr, 0); - } + size_t sz0 = 1 + (noname ? 0 : 4) + 8; + uchar *ptr = reserveNodeSpace(node, sz0); - if( value ) - node.setValue(elem_type, value, len); + *ptr++ = (uchar) (elem_type | (noname ? 0 : FileNode::NAMED)); + if (elem_type == FileNode::NONE) + freeSpaceOfs -= 8; - if( collection.isNamed() ) - cp += 4; - int nelems = readInt(cp + 5); - writeInt(cp + 5, nelems + 1); + if (!noname) { + writeInt(ptr, (int) strofs); + ptr += 4; + } - return node; + if (elem_type == FileNode::SEQ || elem_type == FileNode::MAP) { + writeInt(ptr, 4); + writeInt(ptr, 0); } - void finalizeCollection( FileNode& collection ) - { - if( !collection.isSeq() && !collection.isMap() ) - return; - uchar* ptr0 = collection.ptr(), *ptr = ptr0 + 1; - if( *ptr0 & FileNode::NAMED ) - ptr += 4; - size_t blockIdx = collection.blockIdx; - size_t ofs = collection.ofs + (size_t)(ptr + 8 - ptr0); - size_t rawSize = 4; - unsigned sz = (unsigned)readInt(ptr + 4); - if( sz > 0 ) - { - size_t lastBlockIdx = fs_data_ptrs.size() - 1; + if (value) + node.setValue(elem_type, value, len); - for( ; blockIdx < lastBlockIdx; blockIdx++ ) - { - rawSize += fs_data_blksz[blockIdx] - ofs; - ofs = 0; - } + if (collection.isNamed()) + cp += 4; + int nelems = readInt(cp + 5); + writeInt(cp + 5, nelems + 1); + + return node; +} + +void FileStorage::Impl::finalizeCollection(FileNode &collection) { + if (!collection.isSeq() && !collection.isMap()) + return; + uchar *ptr0 = collection.ptr(), *ptr = ptr0 + 1; + if (*ptr0 & FileNode::NAMED) + ptr += 4; + size_t blockIdx = collection.blockIdx; + size_t ofs = collection.ofs + (size_t) (ptr + 8 - ptr0); + size_t rawSize = 4; + unsigned sz = (unsigned) readInt(ptr + 4); + if (sz > 0) { + size_t lastBlockIdx = fs_data_ptrs.size() - 1; + + for (; blockIdx < lastBlockIdx; blockIdx++) { + rawSize += fs_data_blksz[blockIdx] - ofs; + ofs = 0; } - rawSize += freeSpaceOfs - ofs; - writeInt(ptr, (int)rawSize); } + rawSize += freeSpaceOfs - ofs; + writeInt(ptr, (int) rawSize); +} - void normalizeNodeOfs(size_t& blockIdx, size_t& ofs) const - { - while( ofs >= fs_data_blksz[blockIdx] ) - { - if( blockIdx == fs_data_blksz.size() - 1 ) - { - CV_Assert( ofs == fs_data_blksz[blockIdx] ); - break; - } - ofs -= fs_data_blksz[blockIdx]; - blockIdx++; +void FileStorage::Impl::normalizeNodeOfs(size_t &blockIdx, size_t &ofs) const { + while (ofs >= fs_data_blksz[blockIdx]) { + if (blockIdx == fs_data_blksz.size() - 1) { + CV_Assert(ofs == fs_data_blksz[blockIdx]); + break; } + ofs -= fs_data_blksz[blockIdx]; + blockIdx++; } +} - class Base64Decoder - { - public: - Base64Decoder() { ofs = 0; ptr = 0; indent = 0; totalchars = 0; eos = true; } - void init(Ptr& _parser, char* _ptr, int _indent) - { - parser = _parser; - ptr = _ptr; - indent = _indent; - encoded.clear(); - decoded.clear(); - ofs = 0; - totalchars = 0; - eos = false; - } +FileStorage::Impl::Base64State FileStorage::Impl::get_state_of_writing_base64() { + return state_of_writing_base64; +} - bool readMore(int needed) - { - static const uchar base64tab[] = +int FileStorage::Impl::get_space() { + return space; +} + + +FileStorage::Impl::Base64Decoder::Base64Decoder() { + ofs = 0; + ptr = 0; + indent = 0; + totalchars = 0; + eos = true; +} + +void FileStorage::Impl::Base64Decoder::init(Ptr &_parser, char *_ptr, int _indent) { + parser = _parser; + ptr = _ptr; + indent = _indent; + encoded.clear(); + decoded.clear(); + ofs = 0; + totalchars = 0; + eos = false; +} + +bool FileStorage::Impl::Base64Decoder::readMore(int needed) { + static const uchar base64tab[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 63, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, - 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, + 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - if( eos ) - return false; - - size_t sz = decoded.size(); - CV_Assert( ofs <= sz ); - sz -= ofs; - for( size_t i = 0; i < sz; i++ ) - decoded[i] = decoded[ofs + i]; + if (eos) + return false; - decoded.resize(sz); - ofs = 0; + size_t sz = decoded.size(); + CV_Assert(ofs <= sz); + sz -= ofs; + for (size_t i = 0; i < sz; i++) + decoded[i] = decoded[ofs + i]; - CV_Assert( !parser.empty() && ptr ); - char *beg = 0, *end = 0; - bool ok = parser->getBase64Row(ptr, indent, beg, end); - ptr = end; - std::copy(beg, end, std::back_inserter(encoded)); - totalchars += end - beg; + decoded.resize(sz); + ofs = 0; - if( !ok || beg == end ) - { - // in the end of base64 sequence pad it with '=' characters so that - // its total length is multiple of - eos = true; - size_t tc = totalchars; - for( ; tc % 4 != 0; tc++ ) - encoded.push_back('='); - } + CV_Assert(!parser.empty() && ptr); + char *beg = 0, *end = 0; + bool ok = parser->getBase64Row(ptr, indent, beg, end); + ptr = end; + std::copy(beg, end, std::back_inserter(encoded)); + totalchars += end - beg; + + if (!ok || beg == end) { + // in the end of base64 sequence pad it with '=' characters so that + // its total length is multiple of + eos = true; + size_t tc = totalchars; + for (; tc % 4 != 0; tc++) + encoded.push_back('='); + } + + int i = 0, j, n = (int) encoded.size(); + if (n > 0) { + const uchar *tab = base64tab; + char *src = &encoded[0]; + + for (; i <= n - 4; i += 4) { + // dddddd cccccc bbbbbb aaaaaa => ddddddcc ccccbbbb bbaaaaaa + uchar d = tab[(int) (uchar) src[i]], c = tab[(int) (uchar) src[i + 1]]; + uchar b = tab[(int) (uchar) src[i + 2]], a = tab[(int) (uchar) src[i + 3]]; + + decoded.push_back((uchar) ((d << 2) | (c >> 4))); + decoded.push_back((uchar) ((c << 4) | (b >> 2))); + decoded.push_back((uchar) ((b << 6) | a)); + } + } - int i = 0, j, n = (int)encoded.size(); - if( n > 0 ) - { - const uchar* tab = base64tab; - char* src = &encoded[0]; + if (i > 0 && encoded[i - 1] == '=') { + if (i > 1 && encoded[i - 2] == '=' && !decoded.empty()) + decoded.pop_back(); + if (!decoded.empty()) + decoded.pop_back(); + } - for( ; i <= n - 4; i += 4 ) - { - // dddddd cccccc bbbbbb aaaaaa => ddddddcc ccccbbbb bbaaaaaa - uchar d = tab[(int)(uchar)src[i]], c = tab[(int)(uchar)src[i+1]]; - uchar b = tab[(int)(uchar)src[i+2]], a = tab[(int)(uchar)src[i+3]]; + n -= i; + for (j = 0; j < n; j++) + encoded[j] = encoded[i + j]; + encoded.resize(n); - decoded.push_back((uchar)((d << 2) | (c >> 4))); - decoded.push_back((uchar)((c << 4) | (b >> 2))); - decoded.push_back((uchar)((b << 6) | a)); - } - } + return (int) decoded.size() >= needed; +} - if( i > 0 && encoded[i-1] == '=' ) - { - if( i > 1 && encoded[i-2] == '=' && !decoded.empty() ) - decoded.pop_back(); - if( !decoded.empty() ) - decoded.pop_back(); - } +uchar FileStorage::Impl::Base64Decoder::getUInt8() { + size_t sz = decoded.size(); + if (ofs >= sz && !readMore(1)) + return (uchar) 0; + return decoded[ofs++]; +} - n -= i; - for( j = 0; j < n; j++ ) - encoded[j] = encoded[i + j]; - encoded.resize(n); +ushort FileStorage::Impl::Base64Decoder::getUInt16() { + size_t sz = decoded.size(); + if (ofs + 2 > sz && !readMore(2)) + return (ushort) 0; + ushort val = (decoded[ofs] + (decoded[ofs + 1] << 8)); + ofs += 2; + return val; +} - return (int)decoded.size() >= needed; - } +int FileStorage::Impl::Base64Decoder::getInt32() { + size_t sz = decoded.size(); + if (ofs + 4 > sz && !readMore(4)) + return 0; + int ival = readInt(&decoded[ofs]); + ofs += 4; + return ival; +} - uchar getUInt8() - { - size_t sz = decoded.size(); - if( ofs >= sz && !readMore(1) ) - return (uchar)0; - return decoded[ofs++]; - } +double FileStorage::Impl::Base64Decoder::getFloat64() { + size_t sz = decoded.size(); + if (ofs + 8 > sz && !readMore(8)) + return 0; + double fval = readReal(&decoded[ofs]); + ofs += 8; + return fval; +} - ushort getUInt16() - { - size_t sz = decoded.size(); - if( ofs + 2 > sz && !readMore(2) ) - return (ushort)0; - ushort val = (decoded[ofs] + (decoded[ofs + 1] << 8)); - ofs += 2; - return val; - } +bool FileStorage::Impl::Base64Decoder::endOfStream() const { return eos; } - int getInt32() - { - size_t sz = decoded.size(); - if( ofs + 4 > sz && !readMore(4) ) - return 0; - int ival = readInt(&decoded[ofs]); - ofs += 4; - return ival; - } +char *FileStorage::Impl::Base64Decoder::getPtr() const { return ptr; } - double getFloat64() - { - size_t sz = decoded.size(); - if( ofs + 8 > sz && !readMore(8) ) - return 0; - double fval = readReal(&decoded[ofs]); - ofs += 8; - return fval; - } - bool endOfStream() const { return eos; } - char* getPtr() const { return ptr; } - protected: - - Ptr parser; - char* ptr; - int indent; - std::vector encoded; - std::vector decoded; - size_t ofs; - size_t totalchars; - bool eos; - }; - - char* parseBase64(char* ptr, int indent, FileNode& collection) - { - const int BASE64_HDR_SIZE = 24; - char dt[BASE64_HDR_SIZE+1] = {0}; - base64decoder.init(parser, ptr, indent); +char *FileStorage::Impl::parseBase64(char *ptr, int indent, FileNode &collection) { + const int BASE64_HDR_SIZE = 24; + char dt[BASE64_HDR_SIZE + 1] = {0}; + base64decoder.init(parser, ptr, indent); - int i, k; + int i, k; - for( i = 0; i < BASE64_HDR_SIZE; i++ ) - dt[i] = (char)base64decoder.getUInt8(); - for( i = 0; i < BASE64_HDR_SIZE; i++ ) - if( isspace(dt[i])) - break; - dt[i] = '\0'; + for (i = 0; i < BASE64_HDR_SIZE; i++) + dt[i] = (char) base64decoder.getUInt8(); + for (i = 0; i < BASE64_HDR_SIZE; i++) + if (isspace(dt[i])) + break; + dt[i] = '\0'; - CV_Assert( !base64decoder.endOfStream() ); + CV_Assert(!base64decoder.endOfStream()); - int fmt_pairs[CV_FS_MAX_FMT_PAIRS*2]; - int fmt_pair_count = fs::decodeFormat( dt, fmt_pairs, CV_FS_MAX_FMT_PAIRS ); - int ival = 0; - double fval = 0; + int fmt_pairs[CV_FS_MAX_FMT_PAIRS * 2]; + int fmt_pair_count = fs::decodeFormat(dt, fmt_pairs, CV_FS_MAX_FMT_PAIRS); + int ival = 0; + double fval = 0; - for(;;) - { - for( k = 0; k < fmt_pair_count; k++ ) - { - int elem_type = fmt_pairs[k*2+1]; - int count = fmt_pairs[k*2]; + for (;;) { + for (k = 0; k < fmt_pair_count; k++) { + int elem_type = fmt_pairs[k * 2 + 1]; + int count = fmt_pairs[k * 2]; - for( i = 0; i < count; i++ ) - { - int node_type = FileNode::INT; - switch( elem_type ) - { + for (i = 0; i < count; i++) { + int node_type = FileNode::INT; + switch (elem_type) { case CV_8U: ival = base64decoder.getUInt8(); break; case CV_8S: - ival = (char)base64decoder.getUInt8(); + ival = (char) base64decoder.getUInt8(); break; case CV_16U: ival = base64decoder.getUInt16(); break; case CV_16S: - ival = (short)base64decoder.getUInt16(); + ival = (short) base64decoder.getUInt16(); break; case CV_32S: ival = base64decoder.getInt32(); break; - case CV_32F: - { + case CV_32F: { Cv32suf v; v.i = base64decoder.getInt32(); fval = v.f; node_type = FileNode::REAL; - } + } break; case CV_64F: fval = base64decoder.getFloat64(); node_type = FileNode::REAL; break; case CV_16F: - fval = (float)float16_t::fromBits(base64decoder.getUInt16()); + fval = (float) float16_t::fromBits(base64decoder.getUInt16()); node_type = FileNode::REAL; break; default: - CV_Error( Error::StsUnsupportedFormat, "Unsupported type" ); - } - - if( base64decoder.endOfStream() ) - break; - addNode(collection, std::string(), node_type, - node_type == FileNode::INT ? (void*)&ival : (void*)&fval, -1); + CV_Error(Error::StsUnsupportedFormat, "Unsupported type"); } + + if (base64decoder.endOfStream()) + break; + addNode(collection, std::string(), node_type, + node_type == FileNode::INT ? (void *) &ival : (void *) &fval, -1); } - if( base64decoder.endOfStream() ) - break; } - - finalizeCollection(collection); - return base64decoder.getPtr(); - } - - void parseError( const char* func_name, const std::string& err_msg, const char* source_file, int source_line ) - { - std::string msg = format("%s(%d): %s", filename.c_str(), lineno, err_msg.c_str()); - error(Error::StsParseError, func_name, msg.c_str(), source_file, source_line ); - } - - const uchar* getNodePtr(size_t blockIdx, size_t ofs) const - { - CV_Assert( blockIdx < fs_data_ptrs.size()); - CV_Assert( ofs < fs_data_blksz[blockIdx]); - - return fs_data_ptrs[blockIdx] + ofs; - } - - std::string getName( size_t nameofs ) const - { - CV_Assert( nameofs < str_hash_data.size() ); - return std::string(&str_hash_data[nameofs]); + if (base64decoder.endOfStream()) + break; } - FileStorage* getFS() { return fs_ext; } - - FileStorage* fs_ext; - - std::string filename; - int flags; - bool empty_stream; - - FILE* file; - gzFile gzfile; + finalizeCollection(collection); + return base64decoder.getPtr(); +} - bool is_opened; - bool dummy_eof; - bool write_mode; - bool mem_mode; - int fmt; +void FileStorage::Impl::parseError(const char *func_name, const std::string &err_msg, const char *source_file, + int source_line) { + std::string msg = format("%s(%d): %s", filename.c_str(), lineno, err_msg.c_str()); + error(Error::StsParseError, func_name, msg.c_str(), source_file, source_line); +} - State state; //!< current state of the FileStorage (used only for writing) - int space, wrap_margin; - std::deque write_stack; - std::vector buffer; - size_t bufofs; +const uchar *FileStorage::Impl::getNodePtr(size_t blockIdx, size_t ofs) const { + CV_Assert(blockIdx < fs_data_ptrs.size()); + CV_Assert(ofs < fs_data_blksz[blockIdx]); - std::deque outbuf; + return fs_data_ptrs[blockIdx] + ofs; +} - Ptr emitter; - Ptr parser; - Base64Decoder base64decoder; +std::string FileStorage::Impl::getName(size_t nameofs) const { + CV_Assert(nameofs < str_hash_data.size()); + return std::string(&str_hash_data[nameofs]); +} - std::vector roots; - std::vector > > fs_data; - std::vector fs_data_ptrs; - std::vector fs_data_blksz; - size_t freeSpaceOfs; - typedef std::unordered_map str_hash_t; - str_hash_t str_hash; - std::vector str_hash_data; +FileStorage *FileStorage::Impl::getFS() { return fs_ext; } - std::vector strbufv; - char* strbuf; - size_t strbufsize; - size_t strbufpos; - int lineno; -}; FileStorage::FileStorage() : state(0) @@ -1807,7 +1842,7 @@ FileStorage::FileStorage(const String& filename, int flags, const String& encodi void FileStorage::startWriteStruct(const String& name, int struct_flags, const String& typeName) { - p->startWriteStruct(name.c_str(), struct_flags, typeName.c_str()); + p->startWriteStruct(name.size() ? name.c_str() : 0, struct_flags, typeName.size() ? typeName.c_str() : 0); elname = String(); if ((struct_flags & FileNode::TYPE_MASK) == FileNode::SEQ) state = FileStorage::VALUE_EXPECTED; @@ -1882,7 +1917,7 @@ std::string FileStorage::getDefaultObjectName(const std::string& _filename) } ptr++; if( ptr == ptr2 ) - CV_Error( CV_StsBadArg, "Invalid filename" ); + CV_Error( cv::Error::StsBadArg, "Invalid filename" ); char* name = name_buf.data(); @@ -2005,12 +2040,14 @@ FileStorage& operator << (FileStorage& fs, const String& str) if( c == '}' || c == ']' ) { if( fs_impl->write_stack.empty() ) - CV_Error_( CV_StsError, ("Extra closing '%c'", *_str) ); + CV_Error_( cv::Error::StsError, ("Extra closing '%c'", *_str) ); + + fs_impl->workaround(); int struct_flags = fs_impl->write_stack.back().flags; char expected_bracket = FileNode::isMap(struct_flags) ? '}' : ']'; if( c != expected_bracket ) - CV_Error_( CV_StsError, ("The closing '%c' does not match the opening '%c'", c, expected_bracket)); + CV_Error_( cv::Error::StsError, ("The closing '%c' does not match the opening '%c'", c, expected_bracket)); fs_impl->endWriteStruct(); CV_Assert(!fs_impl->write_stack.empty()); struct_flags = fs_impl->write_stack.back().flags; @@ -2020,7 +2057,7 @@ FileStorage& operator << (FileStorage& fs, const String& str) else if( fs.state == NAME_EXPECTED + INSIDE_MAP ) { if (!cv_isalpha(c) && c != '_') - CV_Error_( CV_StsError, ("Incorrect element name %s; should start with a letter or '_'", _str) ); + CV_Error_( cv::Error::StsError, ("Incorrect element name %s; should start with a letter or '_'", _str) ); fs.elname = str; fs.state = VALUE_EXPECTED + INSIDE_MAP; } @@ -2049,7 +2086,7 @@ FileStorage& operator << (FileStorage& fs, const String& str) } } else - CV_Error( CV_StsError, "Invalid fs.state" ); + CV_Error( cv::Error::StsError, "Invalid fs.state" ); return fs; } diff --git a/modules/core/src/persistence.hpp b/modules/core/src/persistence.hpp index 05c7adc17ce3..1a9dbecf7c5b 100644 --- a/modules/core/src/persistence.hpp +++ b/modules/core/src/persistence.hpp @@ -163,6 +163,24 @@ class FileStorage_API CV_NORETURN virtual void parseError(const char* funcname, const std::string& msg, const char* filename, int lineno) = 0; + +private: + enum Base64State{ + Uncertain, + NotUse, + InUse, + }; + + friend class cv::FileStorage::Impl; + friend class cv::FileStorage; + friend class JSONEmitter; + friend class XMLEmitter; + friend class YAMLEmitter; + + virtual void check_if_write_struct_is_delayed(bool change_type_to_base64 = false) = 0; + virtual void switch_to_Base64_state(Base64State state) = 0; + virtual Base64State get_state_of_writing_base64() = 0; + virtual int get_space() = 0; }; class FileStorageEmitter diff --git a/modules/core/src/persistence_base64_encoding.cpp b/modules/core/src/persistence_base64_encoding.cpp new file mode 100644 index 000000000000..7d90fd422b2d --- /dev/null +++ b/modules/core/src/persistence_base64_encoding.cpp @@ -0,0 +1,370 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include "precomp.hpp" +#include "persistence_impl.hpp" +#include "persistence_base64_encoding.hpp" + +namespace cv +{ + +class base64::Base64ContextEmitter +{ +public: + explicit Base64ContextEmitter(cv::FileStorage::Impl& fs, bool needs_indent_) + : file_storage(fs) + , needs_indent(needs_indent_) + , binary_buffer(BUFFER_LEN) + , base64_buffer(base64_encode_buffer_size(BUFFER_LEN)) + , src_beg(0) + , src_cur(0) + , src_end(0) + { + src_beg = binary_buffer.data(); + src_end = src_beg + BUFFER_LEN; + src_cur = src_beg; + + CV_Assert(fs.write_mode); + + if (needs_indent) + { + file_storage.flush(); + } + } + + ~Base64ContextEmitter() + { + /* cleaning */ + if (src_cur != src_beg) + flush(); /* encode the rest binary data to base64 buffer */ + } + + Base64ContextEmitter & write(const uchar * beg, const uchar * end) + { + if (beg >= end) + return *this; + + while (beg < end) { + /* collect binary data and copy to binary buffer */ + size_t len = std::min(end - beg, src_end - src_cur); + std::memcpy(src_cur, beg, len); + beg += len; + src_cur += len; + + if (src_cur >= src_end) { + /* binary buffer is full. */ + /* encode it to base64 and send result to fs */ + flush(); + } + } + + return *this; + } + + /* + * a convertor must provide : + * - `operator >> (uchar * & dst)` for writing current binary data to `dst` and moving to next data. + * - `operator bool` for checking if current loaction is valid and not the end. + */ + template inline + Base64ContextEmitter & write(_to_binary_convertor_t & convertor) + { + static const size_t BUFFER_MAX_LEN = 1024U; + + std::vector buffer(BUFFER_MAX_LEN); + uchar * beg = buffer.data(); + uchar * end = beg; + + while (convertor) { + convertor >> end; + write(beg, end); + end = beg; + } + + return *this; + } + + bool flush() + { + /* control line width, so on. */ + size_t len = base64_encode(src_beg, base64_buffer.data(), 0U, src_cur - src_beg); + if (len == 0U) + return false; + + src_cur = src_beg; + + if ( !needs_indent) + { + file_storage.puts((const char*)base64_buffer.data()); + } + else + { + const char newline[] = "\n"; + char space[80]; + int ident = file_storage.write_stack.back().indent; + memset(space, ' ', static_cast(ident)); + space[ident] = '\0'; + + file_storage.puts(space); + file_storage.puts((const char*)base64_buffer.data()); + file_storage.puts(newline); + file_storage.flush(); + } + + return true; + } + +private: + /* because of Base64, we must keep its length a multiple of 3 */ + static const size_t BUFFER_LEN = 48U; + // static_assert(BUFFER_LEN % 3 == 0, "BUFFER_LEN is invalid"); + +private: + cv::FileStorage::Impl& file_storage; + bool needs_indent; + + std::vector binary_buffer; + std::vector base64_buffer; + uchar * src_beg; + uchar * src_cur; + uchar * src_end; +}; + +std::string base64::make_base64_header(const char *dt) { + std::ostringstream oss; + oss << dt << ' '; + std::string buffer(oss.str()); + CV_Assert(buffer.size() < ::base64::HEADER_SIZE); + + buffer.reserve(::base64::HEADER_SIZE); + while (buffer.size() < ::base64::HEADER_SIZE) + buffer += ' '; + + return buffer; +} + +size_t base64::base64_encode(const uint8_t *src, uint8_t *dst, size_t off, size_t cnt) { + if (!src || !dst || !cnt) + return 0; + + /* initialize beginning and end */ + uint8_t * dst_beg = dst; + uint8_t * dst_cur = dst_beg; + + uint8_t const * src_beg = src + off; + uint8_t const * src_cur = src_beg; + uint8_t const * src_end = src_cur + cnt / 3U * 3U; + + /* integer multiples part */ + while (src_cur < src_end) { + uint8_t _2 = *src_cur++; + uint8_t _1 = *src_cur++; + uint8_t _0 = *src_cur++; + *dst_cur++ = base64_mapping[ _2 >> 2U]; + *dst_cur++ = base64_mapping[(_1 & 0xF0U) >> 4U | (_2 & 0x03U) << 4U]; + *dst_cur++ = base64_mapping[(_0 & 0xC0U) >> 6U | (_1 & 0x0FU) << 2U]; + *dst_cur++ = base64_mapping[ _0 & 0x3FU]; + } + + /* remainder part */ + size_t rst = src_beg + cnt - src_cur; + if (rst == 1U) { + uint8_t _2 = *src_cur++; + *dst_cur++ = base64_mapping[ _2 >> 2U]; + *dst_cur++ = base64_mapping[(_2 & 0x03U) << 4U]; + } else if (rst == 2U) { + uint8_t _2 = *src_cur++; + uint8_t _1 = *src_cur++; + *dst_cur++ = base64_mapping[ _2 >> 2U]; + *dst_cur++ = base64_mapping[(_2 & 0x03U) << 4U | (_1 & 0xF0U) >> 4U]; + *dst_cur++ = base64_mapping[(_1 & 0x0FU) << 2U]; + } + + /* padding */ + switch (rst) + { + case 1U: *dst_cur++ = base64_padding; + /* fallthrough */ + case 2U: *dst_cur++ = base64_padding; + /* fallthrough */ + default: *dst_cur = 0; + break; + } + + return static_cast(dst_cur - dst_beg); +} + +int base64::icvCalcStructSize(const char *dt, int initial_size) { + int size = cv::fs::calcElemSize( dt, initial_size ); + size_t elem_max_size = 0; + for ( const char * type = dt; *type != '\0'; type++ ) { + switch ( *type ) + { + case 'u': { elem_max_size = std::max( elem_max_size, sizeof(uchar ) ); break; } + case 'c': { elem_max_size = std::max( elem_max_size, sizeof(schar ) ); break; } + case 'w': { elem_max_size = std::max( elem_max_size, sizeof(ushort) ); break; } + case 's': { elem_max_size = std::max( elem_max_size, sizeof(short ) ); break; } + case 'i': { elem_max_size = std::max( elem_max_size, sizeof(int ) ); break; } + case 'f': { elem_max_size = std::max( elem_max_size, sizeof(float ) ); break; } + case 'd': { elem_max_size = std::max( elem_max_size, sizeof(double) ); break; } + default: break; + } + } + size = cvAlign( size, static_cast(elem_max_size) ); + return size; +} + +size_t base64::base64_encode_buffer_size(size_t cnt, bool is_end_with_zero) { + size_t additional = static_cast(is_end_with_zero == true); + return (cnt + 2U) / 3U * 4U + additional; +} + +base64::Base64Writer::Base64Writer(cv::FileStorage::Impl& fs, bool can_indent) + : emitter(new Base64ContextEmitter(fs, can_indent)) + , data_type_string() +{ + CV_Assert(fs.write_mode); +} + +void base64::Base64Writer::write(const void* _data, size_t len, const char* dt) +{ + check_dt(dt); + RawDataToBinaryConvertor convertor(_data, static_cast(len), data_type_string); + emitter->write(convertor); +} + +template inline +void base64::Base64Writer::write(_to_binary_convertor_t & convertor, const char* dt) +{ + check_dt(dt); + emitter->write(convertor); +} + +base64::Base64Writer::~Base64Writer() +{ + delete emitter; +} + +void base64::Base64Writer::check_dt(const char* dt) +{ + if ( dt == 0 ) + CV_Error( cv::Error::StsBadArg, "Invalid \'dt\'." ); + else if (data_type_string.empty()) { + data_type_string = dt; + + /* output header */ + std::string buffer = make_base64_header(dt); + const uchar * beg = reinterpret_cast(buffer.data()); + const uchar * end = beg + buffer.size(); + + emitter->write(beg, end); + } else if ( data_type_string != dt ) + CV_Error( cv::Error::StsBadArg, "\'dt\' does not match." ); +} + +base64::RawDataToBinaryConvertor::RawDataToBinaryConvertor(const void* src, int len, const std::string & dt) + : beg(reinterpret_cast(src)) + , cur(0) + , end(0) +{ + CV_Assert(src); + CV_Assert(!dt.empty()); + CV_Assert(len > 0); + + /* calc step and to_binary_funcs */ + step_packed = make_to_binary_funcs(dt); + + end = beg; + cur = beg; + + step = icvCalcStructSize(dt.c_str(), 0); + end = beg + static_cast(len); +} + +inline base64::RawDataToBinaryConvertor& base64::RawDataToBinaryConvertor::operator >>(uchar * & dst) +{ + CV_DbgAssert(*this); + + for (size_t i = 0U, n = to_binary_funcs.size(); i < n; i++) { + elem_to_binary_t & pack = to_binary_funcs[i]; + pack.func(cur + pack.offset, dst + pack.offset_packed); + } + cur += step; + dst += step_packed; + + return *this; +} + +inline base64::RawDataToBinaryConvertor::operator bool() const +{ + return cur < end; +} + +size_t base64::RawDataToBinaryConvertor::make_to_binary_funcs(const std::string &dt) +{ + size_t cnt = 0; + size_t offset = 0; + size_t offset_packed = 0; + char type = '\0'; + + std::istringstream iss(dt); + while (!iss.eof()) { + if (!(iss >> cnt)) { + iss.clear(); + cnt = 1; + } + CV_Assert(cnt > 0U); + if (!(iss >> type)) + break; + + while (cnt-- > 0) + { + elem_to_binary_t pack; + + size_t size = 0; + switch (type) + { + case 'u': + case 'c': + size = sizeof(uchar); + pack.func = to_binary; + break; + case 'w': + case 's': + size = sizeof(ushort); + pack.func = to_binary; + break; + case 'i': + size = sizeof(uint); + pack.func = to_binary; + break; + case 'f': + size = sizeof(float); + pack.func = to_binary; + break; + case 'd': + size = sizeof(double); + pack.func = to_binary; + break; + case 'r': + default: + CV_Error(cv::Error::StsError, "type is not supported"); + }; + + offset = static_cast(cvAlign(static_cast(offset), static_cast(size))); + pack.offset = offset; + offset += size; + + pack.offset_packed = offset_packed; + offset_packed += size; + + to_binary_funcs.push_back(pack); + } + } + + CV_Assert(iss.eof()); + return offset_packed; +} + +} \ No newline at end of file diff --git a/modules/core/src/persistence_base64_encoding.hpp b/modules/core/src/persistence_base64_encoding.hpp new file mode 100644 index 000000000000..1ee5201e141f --- /dev/null +++ b/modules/core/src/persistence_base64_encoding.hpp @@ -0,0 +1,127 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#ifndef OPENCV_CORE_BASE64_ENCODING_HPP +#define OPENCV_CORE_BASE64_ENCODING_HPP + +namespace cv +{ + +namespace base64 +{ +/* A decorator for CvFileStorage +* - no copyable +* - not safe for now +* - move constructor may be needed if C++11 +*/ +uint8_t const base64_mapping[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + +uint8_t const base64_padding = '='; + +std::string make_base64_header(const char * dt); + +size_t base64_encode(uint8_t const * src, uint8_t * dst, size_t off, size_t cnt); + + +int icvCalcStructSize( const char* dt, int initial_size ); + +class Base64ContextEmitter; +class Impl; + +class Base64Writer +{ +public: + Base64Writer(cv::FileStorage::Impl& fs, bool can_indent); + ~Base64Writer(); + void write(const void* _data, size_t len, const char* dt); + template void write(_to_binary_convertor_t & convertor, const char* dt); + +private: + void check_dt(const char* dt); + +private: + // disable copy and assignment + Base64Writer(const Base64Writer &); + Base64Writer & operator=(const Base64Writer &); + +private: + + Base64ContextEmitter * emitter; + std::string data_type_string; +}; + +size_t base64_encode_buffer_size(size_t cnt, bool is_end_with_zero = true); + +template inline size_t +to_binary(_uint_t val, uchar * cur) +{ + size_t delta = CHAR_BIT; + size_t cnt = sizeof(_uint_t); + while (cnt --> static_cast(0U)) { + *cur++ = static_cast(val); + val >>= delta; + } + return sizeof(_uint_t); +} + +template<> inline size_t to_binary(double val, uchar * cur) +{ + Cv64suf bit64; + bit64.f = val; + return to_binary(bit64.u, cur); +} + +template<> inline size_t to_binary(float val, uchar * cur) +{ + Cv32suf bit32; + bit32.f = val; + return to_binary(bit32.u, cur); +} + +template inline size_t +to_binary(uchar const * val, uchar * cur) +{ + return to_binary<_primitive_t>(*reinterpret_cast<_primitive_t const *>(val), cur); +} + + + +class RawDataToBinaryConvertor +{ +public: + // NOTE: len is already multiplied by element size here + RawDataToBinaryConvertor(const void* src, int len, const std::string & dt); + + inline RawDataToBinaryConvertor & operator >>(uchar * & dst); + inline operator bool() const; + +private: + typedef size_t(*to_binary_t)(const uchar *, uchar *); + struct elem_to_binary_t + { + size_t offset; + size_t offset_packed; + to_binary_t func; + }; + +private: + size_t make_to_binary_funcs(const std::string &dt); + +private: + const uchar * beg; + const uchar * cur; + const uchar * end; + + size_t step; + size_t step_packed; + std::vector to_binary_funcs; +}; + +} + +} +#endif \ No newline at end of file diff --git a/modules/core/src/persistence_impl.hpp b/modules/core/src/persistence_impl.hpp new file mode 100644 index 000000000000..4ea2dc350282 --- /dev/null +++ b/modules/core/src/persistence_impl.hpp @@ -0,0 +1,231 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#ifndef OPENCV_CORE_PERSISTENCE_IMPL_HPP +#define OPENCV_CORE_PERSISTENCE_IMPL_HPP + +#include "persistence.hpp" +#include "persistence_base64_encoding.hpp" +#include +#include + + +namespace cv +{ + +enum Base64State{ + Uncertain, + NotUse, + InUse, +}; + +class cv::FileStorage::Impl : public FileStorage_API +{ +public: + void init(); + + Impl(FileStorage* _fs); + + virtual ~Impl(); + + void release(String* out=0); + + void analyze_file_name( const std::string& file_name, std::vector& params ); + + bool open( const char* filename_or_buf, int _flags, const char* encoding ); + + void puts( const char* str ); + + char* getsFromFile( char* buf, int count ); + + char* gets( size_t maxCount ); + + char* gets(); + + bool eof(); + + void setEof(); + + void closeFile(); + + void rewind(); + + char* resizeWriteBuffer( char* ptr, int len ); + + char* flush(); + + void endWriteStruct(); + + void startWriteStruct_helper( const char* key, int struct_flags, + const char* type_name ); + + void startWriteStruct( const char* key, int struct_flags, + const char* type_name ); + + void writeComment( const char* comment, bool eol_comment ); + + void startNextStream(); + + void write( const String& key, int value ); + + void write( const String& key, double value ); + + void write( const String& key, const String& value ); + + void writeRawData( const std::string& dt, const void* _data, size_t len ); + + void workaround(); + + void switch_to_Base64_state( FileStorage_API::Base64State new_state); + + void make_write_struct_delayed( const char* key, int struct_flags, const char* type_name ); + + void check_if_write_struct_is_delayed( bool change_type_to_base64 ); + + void writeRawDataBase64(const void* _data, size_t len, const char* dt ); + + String releaseAndGetString(); + + FileNode getFirstTopLevelNode() const; + + FileNode root(int streamIdx=0) const; + + FileNode operator[](const String& nodename) const; + + FileNode operator[](const char* /*nodename*/) const; + + int getFormat() const; + + char* bufferPtr() const; + char* bufferStart() const; + char* bufferEnd() const; + void setBufferPtr(char* ptr); + int wrapMargin() const; + + FStructData& getCurrentStruct(); + + void setNonEmpty(); + + void processSpecialDouble( char* buf, double* value, char** endptr ); + + double strtod( char* ptr, char** endptr ); + + void convertToCollection(int type, FileNode& node); + + // a) allocates new FileNode (for that just set blockIdx to the last block and ofs to freeSpaceOfs) or + // b) reallocates just created new node (blockIdx and ofs must be taken from FileNode). + // If there is no enough space in the current block (it should be the last block added so far), + // the last block is shrunk so that it ends immediately before the reallocated node. Then, + // a new block of sufficient size is allocated and the FileNode is placed in the beginning of it. + // The case (a) can be used to allocate the very first node by setting blockIdx == ofs == 0. + // In the case (b) the existing tag and the name are copied automatically. + uchar* reserveNodeSpace(FileNode& node, size_t sz); + + unsigned getStringOfs( const std::string& key ) const; + + FileNode addNode( FileNode& collection, const std::string& key, + int elem_type, const void* value, int len ); + + void finalizeCollection( FileNode& collection ); + + void normalizeNodeOfs(size_t& blockIdx, size_t& ofs) const; + + Base64State get_state_of_writing_base64(); + + int get_space(); + + class Base64Decoder + { + public: + Base64Decoder(); + void init(Ptr& _parser, char* _ptr, int _indent); + + bool readMore(int needed); + + uchar getUInt8(); + + ushort getUInt16(); + + int getInt32(); + + double getFloat64(); + + bool endOfStream() const; + char* getPtr() const; + protected: + + Ptr parser; + char* ptr; + int indent; + std::vector encoded; + std::vector decoded; + size_t ofs; + size_t totalchars; + bool eos; + }; + + char* parseBase64(char* ptr, int indent, FileNode& collection); + + void parseError( const char* func_name, const std::string& err_msg, const char* source_file, int source_line ); + + const uchar* getNodePtr(size_t blockIdx, size_t ofs) const; + + std::string getName( size_t nameofs ) const; + + FileStorage* getFS(); + + FileStorage* fs_ext; + + std::string filename; + int flags; + bool empty_stream; + + FILE* file; + gzFile gzfile; + + bool is_opened; + bool dummy_eof; + bool write_mode; + bool mem_mode; + int fmt; + + State state; //!< current state of the FileStorage (used only for writing) + bool is_using_base64; + bool is_write_struct_delayed; + char* delayed_struct_key; + int delayed_struct_flags; + char* delayed_type_name; + FileStorage_API::Base64State state_of_writing_base64; + + int space, wrap_margin; + std::deque write_stack; + std::vector buffer; + size_t bufofs; + + std::deque outbuf; + + Ptr emitter; + Ptr parser; + Base64Decoder base64decoder; + base64::Base64Writer* base64_writer; + + std::vector roots; + std::vector > > fs_data; + std::vector fs_data_ptrs; + std::vector fs_data_blksz; + size_t freeSpaceOfs; + typedef std::unordered_map str_hash_t; + str_hash_t str_hash; + std::vector str_hash_data; + + std::vector strbufv; + char* strbuf; + size_t strbufsize; + size_t strbufpos; + int lineno; +}; + +} + +#endif \ No newline at end of file diff --git a/modules/core/src/persistence_json.cpp b/modules/core/src/persistence_json.cpp index 667895fbc5e7..12a58e80bfa0 100644 --- a/modules/core/src/persistence_json.cpp +++ b/modules/core/src/persistence_json.cpp @@ -23,7 +23,7 @@ class JSONEmitter : public FileStorageEmitter struct_flags = (struct_flags & (FileNode::TYPE_MASK|FileNode::FLOW)) | FileNode::EMPTY; if( !FileNode::isCollection(struct_flags)) - CV_Error( CV_StsBadArg, + CV_Error( cv::Error::StsBadArg, "Some collection type - FileNode::SEQ or FileNode::MAP, must be specified" ); if( type_name && *type_name == '\0' ) @@ -53,29 +53,26 @@ class JSONEmitter : public FileStorageEmitter void endWriteStruct(const FStructData& current_struct) { int struct_flags = current_struct.flags; - CV_Assert( FileNode::isCollection(struct_flags) ); - if( !FileNode::isFlow(struct_flags) ) - { -#if 0 - if ( fs->bufferPtr() <= fs->bufferStart() + fs->space ) - { - /* some bad code for base64_writer... */ - ptr = fs->bufferPtr(); - *ptr++ = '\n'; - *ptr++ = '\0'; - fs->puts( fs->bufferStart() ); - fs->setBufferPtr(fs->bufferStart()); + if (FileNode::isCollection(struct_flags)) { + if (!FileNode::isFlow(struct_flags)) { + if (fs->bufferPtr() <= fs->bufferStart() + fs->get_space()) { + /* some bad code for base64_writer... */ + char *ptr = fs->bufferPtr(); + *ptr++ = '\n'; + *ptr++ = '\0'; + fs->puts(fs->bufferStart()); + fs->setBufferPtr(fs->bufferStart()); + } + fs->flush(); } -#endif - fs->flush(); - } - char* ptr = fs->bufferPtr(); - if( ptr > fs->bufferStart() + current_struct.indent && !FileNode::isEmptyCollection(struct_flags) ) - *ptr++ = ' '; - *ptr++ = FileNode::isMap(struct_flags) ? '}' : ']'; - fs->setBufferPtr(ptr); + char *ptr = fs->bufferPtr(); + if (ptr > fs->bufferStart() + current_struct.indent && !FileNode::isEmptyCollection(struct_flags)) + *ptr++ = ' '; + *ptr++ = FileNode::isMap(struct_flags) ? '}' : ']'; + fs->setBufferPtr(ptr); + } } void write(const char* key, int value) @@ -97,11 +94,11 @@ class JSONEmitter : public FileStorageEmitter int i, len; if( !str ) - CV_Error( CV_StsNullPtr, "Null string pointer" ); + CV_Error( cv::Error::StsNullPtr, "Null string pointer" ); len = (int)strlen(str); if( len > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The written string is too long" ); + CV_Error( cv::Error::StsBadArg, "The written string is too long" ); if( quote || len == 0 || str[0] != str[len-1] || (str[0] != '\"' && str[0] != '\'') ) { @@ -136,6 +133,20 @@ class JSONEmitter : public FileStorageEmitter void writeScalar(const char* key, const char* data) { + /* check write_struct */ + + fs->check_if_write_struct_is_delayed(false); + if ( fs->get_state_of_writing_base64() == FileStorage_API::Uncertain ) + { + fs->switch_to_Base64_state( FileStorage_API::NotUse ); + } + else if ( fs->get_state_of_writing_base64() == FileStorage_API::InUse ) + { + CV_Error( cv::Error::StsError, "At present, output Base64 data only." ); + } + + /* check parameters */ + size_t key_len = 0u; if( key && *key == '\0' ) key = 0; @@ -143,9 +154,9 @@ class JSONEmitter : public FileStorageEmitter { key_len = strlen(key); if ( key_len == 0u ) - CV_Error( CV_StsBadArg, "The key is an empty" ); + CV_Error( cv::Error::StsBadArg, "The key is an empty" ); else if ( static_cast(key_len) > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The key is too long" ); + CV_Error( cv::Error::StsBadArg, "The key is too long" ); } size_t data_len = 0u; @@ -157,7 +168,7 @@ class JSONEmitter : public FileStorageEmitter if( FileNode::isCollection(struct_flags) ) { if ( (FileNode::isMap(struct_flags) ^ (key != 0)) ) - CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, " + CV_Error( cv::Error::StsBadArg, "An attempt to add element without a key to a map, " "or add element with key to sequence" ); } else { fs->setNonEmpty(); @@ -199,7 +210,7 @@ class JSONEmitter : public FileStorageEmitter if( key ) { if( !cv_isalpha(key[0]) && key[0] != '_' ) - CV_Error( CV_StsBadArg, "Key must start with a letter or _" ); + CV_Error( cv::Error::StsBadArg, "Key must start with a letter or _" ); ptr = fs->resizeWriteBuffer( ptr, static_cast(key_len) ); *ptr++ = '\"'; @@ -210,7 +221,7 @@ class JSONEmitter : public FileStorageEmitter ptr[i] = c; if( !cv_isalnum(c) && c != '-' && c != '_' && c != ' ' ) - CV_Error( CV_StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); + CV_Error( cv::Error::StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); } ptr += key_len; @@ -233,7 +244,7 @@ class JSONEmitter : public FileStorageEmitter void writeComment(const char* comment, bool eol_comment) { if( !comment ) - CV_Error( CV_StsNullPtr, "Null comment" ); + CV_Error( cv::Error::StsNullPtr, "Null comment" ); int len = static_cast(strlen(comment)); char* ptr = fs->bufferPtr(); diff --git a/modules/core/src/persistence_xml.cpp b/modules/core/src/persistence_xml.cpp index 52b53744254e..62b7b1eb59c6 100644 --- a/modules/core/src/persistence_xml.cpp +++ b/modules/core/src/persistence_xml.cpp @@ -45,7 +45,7 @@ class XMLEmitter : public FileStorageEmitter if( FileNode::isCollection(struct_flags) ) { if( FileNode::isMap(struct_flags) ^ (key != 0) ) - CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, " + CV_Error( cv::Error::StsBadArg, "An attempt to add element without a key to a map, " "or add element with key to sequence" ); } else @@ -61,26 +61,26 @@ class XMLEmitter : public FileStorageEmitter if( !key ) key = "_"; else if( key[0] == '_' && key[1] == '\0' ) - CV_Error( CV_StsBadArg, "A single _ is a reserved tag name" ); + CV_Error( cv::Error::StsBadArg, "A single _ is a reserved tag name" ); len = (int)strlen( key ); *ptr++ = '<'; if( tag_type == CV_XML_CLOSING_TAG ) { if( !attrlist.empty() ) - CV_Error( CV_StsBadArg, "Closing tag should not include any attributes" ); + CV_Error( cv::Error::StsBadArg, "Closing tag should not include any attributes" ); *ptr++ = '/'; } if( !cv_isalpha(key[0]) && key[0] != '_' ) - CV_Error( CV_StsBadArg, "Key should start with a letter or _" ); + CV_Error( cv::Error::StsBadArg, "Key should start with a letter or _" ); ptr = fs->resizeWriteBuffer( ptr, len ); for( i = 0; i < len; i++ ) { char c = key[i]; if( !cv_isalnum(c) && c != '_' && c != '-' ) - CV_Error( CV_StsBadArg, "Key name may only contain alphanumeric characters [a-zA-Z0-9], '-' and '_'" ); + CV_Error( cv::Error::StsBadArg, "Key name may only contain alphanumeric characters [a-zA-Z0-9], '-' and '_'" ); ptr[i] = c; } ptr += len; @@ -158,11 +158,11 @@ class XMLEmitter : public FileStorageEmitter int i, len; if( !str ) - CV_Error( CV_StsNullPtr, "Null string pointer" ); + CV_Error( cv::Error::StsNullPtr, "Null string pointer" ); len = (int)strlen(str); if( len > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The written string is too long" ); + CV_Error( cv::Error::StsBadArg, "The written string is too long" ); if( quote || len == 0 || str[0] != '\"' || str[0] != str[len-1] ) { @@ -233,6 +233,16 @@ class XMLEmitter : public FileStorageEmitter void writeScalar(const char* key, const char* data) { + fs->check_if_write_struct_is_delayed(false); + if ( fs->get_state_of_writing_base64() == FileStorage_API::Uncertain ) + { + fs->switch_to_Base64_state( FileStorage_API::NotUse ); + } + else if ( fs->get_state_of_writing_base64() == FileStorage_API::InUse ) + { + CV_Error( cv::Error::StsError, "At present, output Base64 data only." ); + } + int len = (int)strlen(data); if( key && *key == '\0' ) key = 0; @@ -255,7 +265,7 @@ class XMLEmitter : public FileStorageEmitter int new_offset = (int)(ptr - fs->bufferStart()) + len; if( key ) - CV_Error( CV_StsBadArg, "elements with keys can not be written to sequence" ); + CV_Error( cv::Error::StsBadArg, "elements with keys can not be written to sequence" ); current_struct.flags = FileNode::SEQ; @@ -281,10 +291,10 @@ class XMLEmitter : public FileStorageEmitter char* ptr; if( !comment ) - CV_Error( CV_StsNullPtr, "Null comment" ); + CV_Error( cv::Error::StsNullPtr, "Null comment" ); if( strstr(comment, "--") != 0 ) - CV_Error( CV_StsBadArg, "Double hyphen \'--\' is not allowed in the comments" ); + CV_Error( cv::Error::StsBadArg, "Double hyphen \'--\' is not allowed in the comments" ); len = (int)strlen(comment); eol = strchr(comment, '\n'); diff --git a/modules/core/src/persistence_yml.cpp b/modules/core/src/persistence_yml.cpp index 3f3742b8d18e..95db1450c62e 100644 --- a/modules/core/src/persistence_yml.cpp +++ b/modules/core/src/persistence_yml.cpp @@ -33,7 +33,7 @@ class YAMLEmitter : public FileStorageEmitter struct_flags = (struct_flags & (FileNode::TYPE_MASK|FileNode::FLOW)) | FileNode::EMPTY; if( !FileNode::isCollection(struct_flags)) - CV_Error( CV_StsBadArg, + CV_Error( cv::Error::StsBadArg, "Some collection type - FileNode::SEQ or FileNode::MAP, must be specified" ); if (type_name && memcmp(type_name, "binary", 6) == 0) @@ -120,11 +120,11 @@ class YAMLEmitter : public FileStorageEmitter int i, len; if( !str ) - CV_Error( CV_StsNullPtr, "Null string pointer" ); + CV_Error( cv::Error::StsNullPtr, "Null string pointer" ); len = (int)strlen(str); if( len > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The written string is too long" ); + CV_Error( cv::Error::StsBadArg, "The written string is too long" ); if( quote || len == 0 || str[0] != str[len-1] || (str[0] != '\"' && str[0] != '\'') ) { @@ -174,6 +174,16 @@ class YAMLEmitter : public FileStorageEmitter void writeScalar(const char* key, const char* data) { + fs->check_if_write_struct_is_delayed(false); + if ( fs->get_state_of_writing_base64() == FileStorage_API::Uncertain ) + { + fs->switch_to_Base64_state( FileStorage_API::NotUse ); + } + else if ( fs->get_state_of_writing_base64() == FileStorage_API::InUse ) + { + CV_Error( cv::Error::StsError, "At present, output Base64 data only." ); + } + int i, keylen = 0; int datalen = 0; char* ptr; @@ -188,7 +198,7 @@ class YAMLEmitter : public FileStorageEmitter if( FileNode::isCollection(struct_flags) ) { if( (FileNode::isMap(struct_flags) ^ (key != 0)) ) - CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, " + CV_Error( cv::Error::StsBadArg, "An attempt to add element without a key to a map, " "or add element with key to sequence" ); } else @@ -201,10 +211,10 @@ class YAMLEmitter : public FileStorageEmitter { keylen = (int)strlen(key); if( keylen == 0 ) - CV_Error( CV_StsBadArg, "The key is an empty" ); + CV_Error( cv::Error::StsBadArg, "The key is an empty" ); if( keylen > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The key is too long" ); + CV_Error( cv::Error::StsBadArg, "The key is too long" ); } if( data ) @@ -238,7 +248,7 @@ class YAMLEmitter : public FileStorageEmitter if( key ) { if( !cv_isalpha(key[0]) && key[0] != '_' ) - CV_Error( CV_StsBadArg, "Key must start with a letter or _" ); + CV_Error( cv::Error::StsBadArg, "Key must start with a letter or _" ); ptr = fs->resizeWriteBuffer( ptr, keylen ); @@ -248,7 +258,7 @@ class YAMLEmitter : public FileStorageEmitter ptr[i] = c; if( !cv_isalnum(c) && c != '-' && c != '_' && c != ' ' ) - CV_Error( CV_StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); + CV_Error( cv::Error::StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); } ptr += keylen; @@ -271,7 +281,7 @@ class YAMLEmitter : public FileStorageEmitter void writeComment(const char* comment, bool eol_comment) { if( !comment ) - CV_Error( CV_StsNullPtr, "Null comment" ); + CV_Error( cv::Error::StsNullPtr, "Null comment" ); int len = (int)strlen(comment); const char* eol = strchr(comment, '\n'); diff --git a/modules/core/test/test_io.cpp b/modules/core/test/test_io.cpp index 82bd05372da7..3712be9f2e39 100644 --- a/modules/core/test/test_io.cpp +++ b/modules/core/test/test_io.cpp @@ -586,6 +586,7 @@ static void test_filestorage_basic(int write_flags, const char* suffix_name, boo const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); CV_Assert(test_info); std::string name = (std::string(test_info->test_case_name()) + "--" + test_info->name() + suffix_name); + std::string name_34 = string(cvtest::TS::ptr()->get_data_path()) + "io/3_4/" + name; if (!testReadWrite) name = string(cvtest::TS::ptr()->get_data_path()) + "io/" + name; @@ -661,7 +662,23 @@ static void test_filestorage_basic(int write_flags, const char* suffix_name, boo std::ifstream f(name.c_str(), std::ios::in|std::ios::binary); f.seekg(0, std::fstream::end); sz = (size_t)f.tellg(); + + f.seekg(0, std::ios::beg); + std::vector test_data(sz); + f.read(&test_data[0], sz); f.close(); + + std::ifstream reference(name_34.c_str(), std::ios::in|std::ios::binary); + ASSERT_TRUE(reference.is_open()); + reference.seekg(0, std::fstream::end); + size_t ref_sz = (size_t)reference.tellg(); + + reference.seekg(0, std::ios::beg); + std::vector reference_data(ref_sz); + reference.read(&reference_data[0], ref_sz); + reference.close(); + + EXPECT_EQ(reference_data, test_data); } std::cout << "Storage size: " << sz << std::endl; EXPECT_LE(sz, (size_t)6000); @@ -757,27 +774,27 @@ TEST(Core_InputOutput, filestorage_base64_basic_read_JSON) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", false); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_rw_XML) +TEST(Core_InputOutput, filestorage_base64_basic_rw_XML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_rw_YAML) +TEST(Core_InputOutput, filestorage_base64_basic_rw_YAML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_rw_JSON) +TEST(Core_InputOutput, filestorage_base64_basic_rw_JSON) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_memory_XML) +TEST(Core_InputOutput, filestorage_base64_basic_memory_XML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true, true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_memory_YAML) +TEST(Core_InputOutput, filestorage_base64_basic_memory_YAML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true, true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_memory_JSON) +TEST(Core_InputOutput, filestorage_base64_basic_memory_JSON) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true, true); } diff --git a/modules/python/test/test_filestorage_io.py b/modules/python/test/test_filestorage_io.py index 62b540d79cd8..01e0a72300cc 100755 --- a/modules/python/test/test_filestorage_io.py +++ b/modules/python/test/test_filestorage_io.py @@ -1,6 +1,8 @@ #!/usr/bin/env python """Algorithm serialization test.""" from __future__ import print_function +import base64 +import json import tempfile import os import cv2 as cv @@ -109,5 +111,96 @@ def test_yml(self): def test_json(self): self.run_fs_test(".json") + def test_base64(self): + fd, fname = tempfile.mkstemp(prefix="opencv_python_sample_filestorage_base64", suffix=".json") + os.close(fd) + np.random.seed(42) + self.write_base64_json(fname) + os.remove(fname) + + @staticmethod + def get_normal_2d_mat(): + rows = 10 + cols = 20 + cn = 3 + + image = np.zeros((rows, cols, cn), np.uint8) + image[:] = (1, 2, 127) + + for i in range(rows): + for j in range(cols): + image[i, j, 1] = (i + j) % 256 + + return image + + @staticmethod + def get_normal_nd_mat(): + shape = (2, 2, 1, 2) + cn = 4 + + image = np.zeros(shape + (cn,), np.float64) + image[:] = (0.888, 0.111, 0.666, 0.444) + + return image + + @staticmethod + def get_empty_2d_mat(): + shape = (0, 0) + cn = 1 + + image = np.zeros(shape + (cn,), np.uint8) + + return image + + @staticmethod + def get_random_mat(): + rows = 8 + cols = 16 + cn = 1 + + image = np.random.rand(rows, cols, cn) + + return image + + @staticmethod + def decode(data): + # strip $base64$ + encoded = data[8:] + + if len(encoded) == 0: + return b'' + + # strip info about datatype and padding + return base64.b64decode(encoded)[24:] + + def write_base64_json(self, fname): + fs = cv.FileStorage(fname, cv.FileStorage_WRITE_BASE64) + + mats = {'normal_2d_mat': self.get_normal_2d_mat(), + 'normal_nd_mat': self.get_normal_nd_mat(), + 'empty_2d_mat': self.get_empty_2d_mat(), + 'random_mat': self.get_random_mat()} + + for name, mat in mats.items(): + fs.write(name, mat) + + fs.release() + + data = {} + with open(fname) as file: + data = json.load(file) + + for name, mat in mats.items(): + buffer = b'' + + if mat.size != 0: + if hasattr(mat, 'tobytes'): + buffer = mat.tobytes() + else: + buffer = mat.tostring() + + self.assertEqual(buffer, self.decode(data[name]['data'])) + + if __name__ == '__main__': NewOpenCVTests.bootstrap() From b928ebdd53b3db10864ff675461c44f04eef3e8c Mon Sep 17 00:00:00 2001 From: Francesco Petrogalli <25690309+fpetrogalli@users.noreply.github.com> Date: Thu, 8 Jul 2021 21:21:21 +0100 Subject: [PATCH 052/376] Merge pull request #19985 from fpetrogalli:disable_threads * [build][option] Introduce `OPENCV_DISABLE_THREAD_SUPPORT` option. The option forces the library to build without thread support. * update handling of OPENCV_DISABLE_THREAD_SUPPORT - reduce amount of #if conditions * [to squash] cmake: apply mode vars in toolchains too Co-authored-by: Alexander Alekhin --- 3rdparty/libwebp/CMakeLists.txt | 4 +- CMakeLists.txt | 15 ++ cmake/OpenCVCompilerOptions.cmake | 11 +- cmake/OpenCVUtils.cmake | 6 + cmake/vars/EnableModeVars.cmake | 18 ++ .../vars/OPENCV_DISABLE_THREAD_SUPPORT.cmake | 28 +++ modules/core/CMakeLists.txt | 4 + modules/core/include/opencv2/core/utility.hpp | 20 ++- modules/core/src/async.cpp | 166 ++++++++++++++++++ modules/core/src/parallel.cpp | 10 +- modules/core/src/system.cpp | 112 ++++++++++++ modules/core/src/umatrix.cpp | 37 +++- modules/core/src/utils/logtagmanager.hpp | 4 +- modules/core/test/test_async.cpp | 5 +- modules/core/test/test_utils.cpp | 5 +- modules/ts/CMakeLists.txt | 6 + 16 files changed, 435 insertions(+), 16 deletions(-) create mode 100644 cmake/vars/EnableModeVars.cmake create mode 100644 cmake/vars/OPENCV_DISABLE_THREAD_SUPPORT.cmake diff --git a/3rdparty/libwebp/CMakeLists.txt b/3rdparty/libwebp/CMakeLists.txt index 80ab0b86ab76..9160e2024ca0 100644 --- a/3rdparty/libwebp/CMakeLists.txt +++ b/3rdparty/libwebp/CMakeLists.txt @@ -32,7 +32,9 @@ endif() # Define the library target: # ---------------------------------------------------------------------------------- -add_definitions(-DWEBP_USE_THREAD) +if(NOT OPENCV_DISABLE_THREAD_SUPPORT) + add_definitions(-DWEBP_USE_THREAD) +endif() add_library(${WEBP_LIBRARY} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${lib_srcs} ${lib_hdrs}) if(ANDROID) diff --git a/CMakeLists.txt b/CMakeLists.txt index 49abe017a5ee..dd862bb1549e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -512,6 +512,7 @@ OCV_OPTION(OPENCV_GENERATE_SETUPVARS "Generate setup_vars* scripts" ON IF (NOT OCV_OPTION(ENABLE_CONFIG_VERIFICATION "Fail build if actual configuration doesn't match requested (WITH_XXX != HAVE_XXX)" OFF) OCV_OPTION(OPENCV_ENABLE_MEMALIGN "Enable posix_memalign or memalign usage" ON) OCV_OPTION(OPENCV_DISABLE_FILESYSTEM_SUPPORT "Disable filesystem support" OFF) +OCV_OPTION(OPENCV_DISABLE_THREAD_SUPPORT "Build the library without multi-threaded code." OFF) OCV_OPTION(ENABLE_PYLINT "Add target with Pylint checks" (BUILD_DOCS OR BUILD_EXAMPLES) IF (NOT CMAKE_CROSSCOMPILING AND NOT APPLE_FRAMEWORK) ) OCV_OPTION(ENABLE_FLAKE8 "Add target with Python flake8 checker" (BUILD_DOCS OR BUILD_EXAMPLES) IF (NOT CMAKE_CROSSCOMPILING AND NOT APPLE_FRAMEWORK) ) @@ -666,6 +667,11 @@ if(UNIX) set(HAVE_PTHREAD 1) endif() + # Ensure that libpthread is not listed as one of the libraries to pass to the linker. + if (OPENCV_DISABLE_THREAD_SUPPORT) + list(REMOVE_ITEM OPENCV_LINKER_LIBS pthread) + endif() + if(OPENCV_ENABLE_MEMALIGN) CHECK_SYMBOL_EXISTS(posix_memalign stdlib.h HAVE_POSIX_MEMALIGN) CHECK_INCLUDE_FILE(malloc.h HAVE_MALLOC_H) @@ -1459,6 +1465,15 @@ ocv_build_features_string(parallel_status EXCLUSIVE ELSE "none") status("") status(" Parallel framework:" "${parallel_status}") +if (OPENCV_DISABLE_THREAD_SUPPORT) + status("" "Multi thread code explicitly disabled with OPENCV_DISABLE_THREAD_SUPPORT.") + if(HAVE_PTHREADS_PF OR HAVE_HPX OR HAVE_OPENMP OR HAVE_GCD OR HAVE_CONCURRENCY) + message(FATAL_ERROR "Not all parallel frameworks have been disabled (using ${parallel_status}).") + endif() + if(HAVE_PTHREAD) + message(FATAL_ERROR "Thread execution might be in use in some component.") + endif() +endif() if(CV_TRACE OR OPENCV_TRACE) ocv_build_features_string(trace_status EXCLUSIVE diff --git a/cmake/OpenCVCompilerOptions.cmake b/cmake/OpenCVCompilerOptions.cmake index 6e56a2e34aa0..2917dd33d5ee 100644 --- a/cmake/OpenCVCompilerOptions.cmake +++ b/cmake/OpenCVCompilerOptions.cmake @@ -178,14 +178,17 @@ if(CV_GCC OR CV_CLANG) add_extra_compiler_option(-Wno-long-long) endif() - # We need pthread's - if((UNIX + # We need pthread's, unless we have explicitly disabled multi-thread execution. + if(NOT OPENCV_DISABLE_THREAD_SUPPORT + AND ( + (UNIX AND NOT ANDROID AND NOT (APPLE AND CV_CLANG) AND NOT EMSCRIPTEN + ) + OR (EMSCRIPTEN AND WITH_PTHREADS_PF) # https://github.com/opencv/opencv/issues/20285 ) - OR (EMSCRIPTEN AND WITH_PTHREADS_PF) # https://github.com/opencv/opencv/issues/20285 - ) + ) # TODO add_extra_compiler_option(-pthread) endif() diff --git a/cmake/OpenCVUtils.cmake b/cmake/OpenCVUtils.cmake index 252078bdf776..39445150a911 100644 --- a/cmake/OpenCVUtils.cmake +++ b/cmake/OpenCVUtils.cmake @@ -1973,3 +1973,9 @@ if(NOT BUILD_SHARED_LIBS AND (CMAKE_VERSION VERSION_LESS "3.14.0")) else() ocv_update(OPENCV_3RDPARTY_EXCLUDE_FROM_ALL "EXCLUDE_FROM_ALL") endif() + + +# +# Include configuration override settings +# +include(cmake/vars/EnableModeVars.cmake) diff --git a/cmake/vars/EnableModeVars.cmake b/cmake/vars/EnableModeVars.cmake new file mode 100644 index 000000000000..b3c4e79c46d1 --- /dev/null +++ b/cmake/vars/EnableModeVars.cmake @@ -0,0 +1,18 @@ +set(__OCV_MODE_VARS_DIR "${CMAKE_CURRENT_LIST_DIR}") + +macro(ocv_change_mode_var) + set(__var "${ARGV0}") + set(__mode "${ARGV1}") + set(__value "${ARGV2}") + if(__mode STREQUAL "MODIFIED_ACCESS" AND __value) + if(NOT __applied_mode_${__var}) + include("${__OCV_MODE_VARS_DIR}/${__var}.cmake") + set(__applied_mode_${__var} 1) + else() + #message("Mode is already applied: ${__var}") + endif() + endif() +endmacro() + +variable_watch(OPENCV_DISABLE_THREAD_SUPPORT ocv_change_mode_var) +set(OPENCV_DISABLE_THREAD_SUPPORT "${OPENCV_DISABLE_THREAD_SUPPORT}") diff --git a/cmake/vars/OPENCV_DISABLE_THREAD_SUPPORT.cmake b/cmake/vars/OPENCV_DISABLE_THREAD_SUPPORT.cmake new file mode 100644 index 000000000000..5f5fc0204dfc --- /dev/null +++ b/cmake/vars/OPENCV_DISABLE_THREAD_SUPPORT.cmake @@ -0,0 +1,28 @@ +# Force removal of code conditionally compiled with `#if +# HAVE_PTHREAD`. +ocv_update(HAVE_PTHREAD 0) + +# There components are disabled because they require +# multi-threaded execution. +ocv_update(WITH_PROTOBUF OFF) +ocv_update(WITH_GSTREAMER OFF) +ocv_update(WITH_IPP OFF) +ocv_update(WITH_ITT OFF) +ocv_update(WITH_OPENCL OFF) +ocv_update(WITH_VA OFF) +ocv_update(WITH_VA_INTEL OFF) + +# Disable bindings +ocv_update(BUILD_opencv_python2 OFF) +ocv_update(BUILD_opencv_python3 OFF) +ocv_update(BUILD_JAVA OFF) +ocv_update(BUILD_opencv_java OFF) + +# These modules require `#include +# <[thread|mutex|condition_variable|future]>` and linkage into +# `libpthread` to work. +ocv_update(BUILD_opencv_objdetect OFF) +ocv_update(BUILD_opencv_gapi OFF) +ocv_update(BUILD_opencv_dnn OFF) + +set(OPJ_USE_THREAD "OFF" CACHE INTERNAL "") diff --git a/modules/core/CMakeLists.txt b/modules/core/CMakeLists.txt index b2797ab31fc1..6a969e5fc358 100644 --- a/modules/core/CMakeLists.txt +++ b/modules/core/CMakeLists.txt @@ -153,6 +153,10 @@ if(OPENCV_CORE_EXCLUDE_C_API) ocv_target_compile_definitions(${the_module} PRIVATE "OPENCV_EXCLUDE_C_API=1") endif() +if(OPENCV_DISABLE_THREAD_SUPPORT) + ocv_target_compile_definitions(${the_module} PUBLIC "OPENCV_DISABLE_THREAD_SUPPORT=1") +endif() + if(HAVE_HPX) ocv_target_link_libraries(${the_module} LINK_PRIVATE "${HPX_LIBRARIES}") endif() diff --git a/modules/core/include/opencv2/core/utility.hpp b/modules/core/include/opencv2/core/utility.hpp index f0368027aa6a..108c0d93e749 100644 --- a/modules/core/include/opencv2/core/utility.hpp +++ b/modules/core/include/opencv2/core/utility.hpp @@ -714,9 +714,27 @@ void Mat::forEach_impl(const Functor& operation) { /////////////////////////// Synchronization Primitives /////////////////////////////// #if !defined(_M_CEE) +#ifndef OPENCV_DISABLE_THREAD_SUPPORT typedef std::recursive_mutex Mutex; typedef std::lock_guard AutoLock; -#endif +#else // OPENCV_DISABLE_THREAD_SUPPORT +// Custom (failing) implementation of `std::recursive_mutex`. +struct Mutex { + void lock(){ + CV_Error(cv::Error::StsNotImplemented, + "cv::Mutex is disabled by OPENCV_DISABLE_THREAD_SUPPORT=ON"); + } + void unlock(){ + CV_Error(cv::Error::StsNotImplemented, + "cv::Mutex is disabled by OPENCV_DISABLE_THREAD_SUPPORT=ON"); + } +}; +// Stub for cv::AutoLock when threads are disabled. +struct AutoLock { + AutoLock(Mutex &) { } +}; +#endif // OPENCV_DISABLE_THREAD_SUPPORT +#endif // !defined(_M_CEE) /** @brief Designed for command line parsing diff --git a/modules/core/src/async.cpp b/modules/core/src/async.cpp index a2f4612365b9..78c0a1ee8116 100644 --- a/modules/core/src/async.cpp +++ b/modules/core/src/async.cpp @@ -14,6 +14,7 @@ #define CV_LOG_STRIP_LEVEL CV_LOG_LEVEL_DEBUG + 1 #include +#ifndef OPENCV_DISABLE_THREAD_SUPPORT #ifdef CV_CXX11 #include @@ -236,6 +237,171 @@ struct AsyncArray::Impl } }; +} // namespace + +#else // OPENCV_DISABLE_THREAD_SUPPORT + +namespace cv { + +// no threading +struct AsyncArray::Impl +{ + int refcount; + void addrefFuture() CV_NOEXCEPT { refcount_future++; refcount++; } + void releaseFuture() CV_NOEXCEPT { refcount_future--; if (0 == --refcount) delete this; } + int refcount_future; + void addrefPromise() CV_NOEXCEPT { refcount_promise++; refcount++; } \ + void releasePromise() CV_NOEXCEPT { refcount_promise--; if (0 == --refcount) delete this; } + int refcount_promise; + + mutable bool has_result; // Mat, UMat or exception + + mutable cv::Ptr result_mat; + mutable cv::Ptr result_umat; + + + bool has_exception; +#if CV__EXCEPTION_PTR + std::exception_ptr exception; +#endif + cv::Exception cv_exception; + + mutable bool result_is_fetched; + + bool future_is_returned; + + Impl() + : refcount(1), refcount_future(0), refcount_promise(1) + , has_result(false) + , has_exception(false) + , result_is_fetched(false) + , future_is_returned(false) + { + // nothing + } + + ~Impl() + { + if (has_result && !result_is_fetched) + { + CV_LOG_INFO(NULL, "Asynchronous result has not been fetched"); + } + } + + bool get(OutputArray dst, int64 timeoutNs) const + { + CV_Assert(!result_is_fetched); + if (!has_result) + { + CV_UNUSED(timeoutNs); + CV_Error(Error::StsError, "Result is not produced (unable to wait for result in OPENCV_DISABLE_THREAD_SUPPORT mode)"); + } + if (!result_mat.empty()) + { + dst.move(*result_mat.get()); + result_mat.release(); + result_is_fetched = true; + return true; + } + if (!result_umat.empty()) + { + dst.move(*result_umat.get()); + result_umat.release(); + result_is_fetched = true; + return true; + } +#if CV__EXCEPTION_PTR + if (has_exception && exception) + { + result_is_fetched = true; + std::rethrow_exception(exception); + } +#endif + if (has_exception) + { + result_is_fetched = true; + throw cv_exception; + } + CV_Error(Error::StsInternal, "AsyncArray: invalid state of 'has_result = true'"); + return false; + } + + bool valid() const CV_NOEXCEPT + { + if (result_is_fetched) + return false; + if (refcount_promise == 0 && !has_result) + return false; + return true; + } + + bool wait_for(int64 timeoutNs) const + { + CV_Assert(valid()); + if (has_result) + return has_result; + if (timeoutNs == 0) + return has_result; + CV_Error(Error::StsError, "Unable to wait in OPENCV_DISABLE_THREAD_SUPPORT mode"); + } + + AsyncArray getArrayResult() + { + CV_Assert(refcount_future == 0); + AsyncArray result; + addrefFuture(); + result.p = this; + future_is_returned = true; + return result; + } + + void setValue(InputArray value) + { + if (future_is_returned && refcount_future == 0) + CV_Error(Error::StsError, "Associated AsyncArray has been destroyed"); + CV_Assert(!has_result); + int k = value.kind(); + if (k == _InputArray::UMAT) + { + result_umat = makePtr(); + value.copyTo(*result_umat.get()); + } + else + { + result_mat = makePtr(); + value.copyTo(*result_mat.get()); + } + has_result = true; + } + +#if CV__EXCEPTION_PTR + void setException(std::exception_ptr e) + { + if (future_is_returned && refcount_future == 0) + CV_Error(Error::StsError, "Associated AsyncArray has been destroyed"); + CV_Assert(!has_result); + has_exception = true; + exception = e; + has_result = true; + } +#endif + + void setException(const cv::Exception e) + { + if (future_is_returned && refcount_future == 0) + CV_Error(Error::StsError, "Associated AsyncArray has been destroyed"); + CV_Assert(!has_result); + has_exception = true; + cv_exception = e; + has_result = true; + } +}; + +} + +#endif // OPENCV_DISABLE_THREAD_SUPPORT + +namespace cv { AsyncArray::AsyncArray() CV_NOEXCEPT : p(NULL) diff --git a/modules/core/src/parallel.cpp b/modules/core/src/parallel.cpp index 7bb7e4633dcd..81ddd0c5ddce 100644 --- a/modules/core/src/parallel.cpp +++ b/modules/core/src/parallel.cpp @@ -72,7 +72,7 @@ #endif #endif -#if defined CV_CXX11 +#ifndef OPENCV_DISABLE_THREAD_SUPPORT #include #endif @@ -884,6 +884,7 @@ T minNonZero(const T& val_1, const T& val_2) return (val_1 != 0) ? val_1 : val_2; } +#ifndef OPENCV_DISABLE_THREAD_SUPPORT static int getNumberOfCPUs_() { @@ -986,6 +987,13 @@ int getNumberOfCPUs() return nCPUs; // cached value } +#else // OPENCV_DISABLE_THREAD_SUPPORT +int getNumberOfCPUs() +{ + return 1; +} +#endif // OPENCV_DISABLE_THREAD_SUPPORT + const char* currentParallelFramework() { std::shared_ptr& api = getCurrentParallelForAPI(); diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index 441457d50fd2..777efceca021 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -216,7 +216,9 @@ std::wstring GetTempFileNameWinRT(std::wstring prefix) #endif #else +#ifndef OPENCV_DISABLE_THREAD_SUPPORT #include +#endif #include #include @@ -1366,6 +1368,8 @@ bool __termination = false; namespace details { +#ifndef OPENCV_DISABLE_THREAD_SUPPORT + #ifdef _WIN32 #ifdef _MSC_VER #pragma warning(disable:4505) // unreferenced local function has been removed @@ -1778,14 +1782,122 @@ static void WINAPI opencv_fls_destructor(void* pData) #endif // CV_USE_FLS #endif // _WIN32 +#else // OPENCV_DISABLE_THREAD_SUPPORT + +// no threading (OPENCV_DISABLE_THREAD_SUPPORT=ON) +class TlsStorage +{ +public: + TlsStorage() + { + slots.reserve(32); + } + ~TlsStorage() + { + for (size_t slotIdx = 0; slotIdx < slots.size(); slotIdx++) + { + SlotInfo& s = slots[slotIdx]; + TLSDataContainer* container = s.container; + if (container && s.data) + { + container->deleteDataInstance(s.data); // Can't use from SlotInfo destructor + s.data = nullptr; + } + } + } + + // Reserve TLS storage index + size_t reserveSlot(TLSDataContainer* container) + { + size_t slotsSize = slots.size(); + for (size_t slot = 0; slot < slotsSize; slot++) + { + SlotInfo& s = slots[slot]; + if (s.container == NULL) + { + CV_Assert(!s.data); + s.container = container; + return slot; + } + } + + // create new slot + slots.push_back(SlotInfo(container)); + return slotsSize; + } + + // Release TLS storage index and pass associated data to caller + void releaseSlot(size_t slotIdx, std::vector &dataVec, bool keepSlot = false) + { + CV_Assert(slotIdx < slots.size()); + SlotInfo& s = slots[slotIdx]; + void* data = s.data; + if (data) + { + dataVec.push_back(data); + s.data = nullptr; + } + if (!keepSlot) + { + s.container = NULL; // mark slot as free (see reserveSlot() implementation) + } + } + + // Get data by TLS storage index + void* getData(size_t slotIdx) const + { + CV_Assert(slotIdx < slots.size()); + const SlotInfo& s = slots[slotIdx]; + return s.data; + } + + // Gather data from threads by TLS storage index + void gather(size_t slotIdx, std::vector &dataVec) + { + CV_Assert(slotIdx < slots.size()); + SlotInfo& s = slots[slotIdx]; + void* data = s.data; + if (data) + dataVec.push_back(data); + return; + } + + // Set data to storage index + void setData(size_t slotIdx, void* pData) + { + CV_Assert(slotIdx < slots.size()); + SlotInfo& s = slots[slotIdx]; + s.data = pData; + } + +private: + struct SlotInfo + { + SlotInfo(TLSDataContainer* _container) : container(_container), data(nullptr) {} + TLSDataContainer* container; // attached container (to dispose data) + void* data; + }; + std::vector slots; +}; + +static TlsStorage& getTlsStorage() +{ + static TlsStorage g_storage; // no threading + return g_storage; +} + +#endif // OPENCV_DISABLE_THREAD_SUPPORT + } // namespace details using namespace details; void releaseTlsStorageThread() { +#ifndef OPENCV_DISABLE_THREAD_SUPPORT if (!g_isTlsStorageInitialized) return; // nothing to release, so prefer to avoid creation of new global structures getTlsStorage().releaseThread(); +#endif } TLSDataContainer::TLSDataContainer() diff --git a/modules/core/src/umatrix.cpp b/modules/core/src/umatrix.cpp index bf5dfb68a318..bbb34a725604 100644 --- a/modules/core/src/umatrix.cpp +++ b/modules/core/src/umatrix.cpp @@ -56,10 +56,6 @@ void setSize(UMat& m, int _dims, const int* _sz, const size_t* _steps, void updateContinuityFlag(UMat& m); void finalizeHdr(UMat& m); -// it should be a prime number for the best hash function -enum { UMAT_NLOCKS = 31 }; -static Mutex umatLocks[UMAT_NLOCKS]; - UMatData::UMatData(const MatAllocator* allocator) { prevAllocator = currAllocator = allocator; @@ -131,6 +127,12 @@ UMatData::~UMatData() } } +#ifndef OPENCV_DISABLE_THREAD_SUPPORT + +// it should be a prime number for the best hash function +enum { UMAT_NLOCKS = 31 }; +static Mutex umatLocks[UMAT_NLOCKS]; + static size_t getUMatDataLockIndex(const UMatData* u) { size_t idx = ((size_t)(void*)u) % UMAT_NLOCKS; @@ -228,6 +230,33 @@ UMatDataAutoLock::~UMatDataAutoLock() getUMatDataAutoLocker().release(u1, u2); } +#else + +void UMatData::lock() +{ + // nothing in OPENCV_DISABLE_THREAD_SUPPORT mode +} + +void UMatData::unlock() +{ + // nothing in OPENCV_DISABLE_THREAD_SUPPORT mode +} + +UMatDataAutoLock::UMatDataAutoLock(UMatData* u) : u1(u), u2(NULL) +{ + // nothing in OPENCV_DISABLE_THREAD_SUPPORT mode +} +UMatDataAutoLock::UMatDataAutoLock(UMatData* u1_, UMatData* u2_) : u1(u1_), u2(u2_) +{ + // nothing in OPENCV_DISABLE_THREAD_SUPPORT mode +} +UMatDataAutoLock::~UMatDataAutoLock() +{ + // nothing in OPENCV_DISABLE_THREAD_SUPPORT mode +} + +#endif // OPENCV_DISABLE_THREAD_SUPPORT + //////////////////////////////// UMat //////////////////////////////// UMat::UMat(UMatUsageFlags _usageFlags) CV_NOEXCEPT diff --git a/modules/core/src/utils/logtagmanager.hpp b/modules/core/src/utils/logtagmanager.hpp index 29a1776ada21..ab4bb9b7d3d4 100644 --- a/modules/core/src/utils/logtagmanager.hpp +++ b/modules/core/src/utils/logtagmanager.hpp @@ -37,8 +37,8 @@ class LogTagManager // also, extensible functions (accepting user-provided callback) are not allowed // to call LogTagManger (to prevent iterator invalidation), which needs enforced // with a non-recursive mutex. - using MutexType = std::mutex; - using LockType = std::lock_guard; + using MutexType = cv::Mutex; + using LockType = cv::AutoLock; enum class MatchingScope { diff --git a/modules/core/test/test_async.cpp b/modules/core/test/test_async.cpp index f898a22878d2..58bcfddcd769 100644 --- a/modules/core/test/test_async.cpp +++ b/modules/core/test/test_async.cpp @@ -7,7 +7,7 @@ #include -#ifdef CV_CXX11 +#if defined(CV_CXX11) && !defined(OPENCV_DISABLE_THREAD_SUPPORT) #include #include #endif @@ -85,7 +85,8 @@ TEST(Core_Async, LikePythonTest) } -#ifdef CV_CXX11 +#if defined(CV_CXX11) && !defined(OPENCV_DISABLE_THREAD_SUPPORT) + TEST(Core_Async, AsyncThread_Simple) { Mat m(3, 3, CV_32FC1, Scalar::all(5.0f)); diff --git a/modules/core/test/test_utils.cpp b/modules/core/test/test_utils.cpp index ed5f34603de5..c31ca75667e9 100644 --- a/modules/core/test/test_utils.cpp +++ b/modules/core/test/test_utils.cpp @@ -8,9 +8,12 @@ #include "opencv2/core/utils/logger.hpp" #include "opencv2/core/utils/buffer_area.private.hpp" -#include "test_utils_tls.impl.hpp" #include "opencv2/core/utils/filesystem.private.hpp" +#ifndef OPENCV_DISABLE_THREAD_SUPPORT +#include "test_utils_tls.impl.hpp" +#endif + namespace opencv_test { namespace { static const char * const keys = diff --git a/modules/ts/CMakeLists.txt b/modules/ts/CMakeLists.txt index f95bed079383..c1d249ea149a 100644 --- a/modules/ts/CMakeLists.txt +++ b/modules/ts/CMakeLists.txt @@ -41,3 +41,9 @@ endif() if(NOT OPENCV_TESTS_CONFIG_STR STREQUAL "${__content}") file(WRITE "${OPENCV_TESTS_CONFIG_FILE}" "${OPENCV_TESTS_CONFIG_STR}") endif() + +if(OPENCV_DISABLE_THREAD_SUPPORT) + # This is required to disable threads in the ts module, as + # described in `ts_gtest.h`. + ocv_target_compile_definitions(${the_module} PUBLIC GTEST_HAS_PTHREAD=0) +endif() From fd16222613f06201fa1c9b503aee3cdb1b13fa8b Mon Sep 17 00:00:00 2001 From: berak Date: Fri, 9 Jul 2021 13:21:44 +0200 Subject: [PATCH 053/376] dnn: update links for the colorization samples --- samples/dnn/colorization.cpp | 2 +- samples/dnn/colorization.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/dnn/colorization.cpp b/samples/dnn/colorization.cpp index b68e0ec4d8bf..6d751590d10d 100644 --- a/samples/dnn/colorization.cpp +++ b/samples/dnn/colorization.cpp @@ -50,7 +50,7 @@ int main(int argc, char **argv) " https://github.com/richzhang/colorization\n" "Download caffemodel and prototxt files:\n" " http://eecs.berkeley.edu/~rich.zhang/projects/2016_colorization/files/demo_v2/colorization_release_v2.caffemodel\n" - " https://raw.githubusercontent.com/richzhang/colorization/master/colorization/models/colorization_deploy_v2.prototxt\n"; + " https://raw.githubusercontent.com/richzhang/colorization/caffe/models/colorization_deploy_v2.prototxt\n"; const string keys = "{ h help | | print this help message }" "{ proto | colorization_deploy_v2.prototxt | model configuration }" diff --git a/samples/dnn/colorization.py b/samples/dnn/colorization.py index c9eb2af3b668..5bdef9793e30 100644 --- a/samples/dnn/colorization.py +++ b/samples/dnn/colorization.py @@ -1,6 +1,6 @@ # Script is based on https://github.com/richzhang/colorization/blob/master/colorization/colorize.py -# To download the caffemodel and the prototxt, see: https://github.com/richzhang/colorization/tree/master/colorization/models -# To download pts_in_hull.npy, see: https://github.com/richzhang/colorization/blob/master/colorization/resources/pts_in_hull.npy +# To download the caffemodel and the prototxt, see: https://github.com/richzhang/colorization/tree/caffe/colorization/models +# To download pts_in_hull.npy, see: https://github.com/richzhang/colorization/tree/caffe/colorization/resources/pts_in_hull.npy import numpy as np import argparse import cv2 as cv From 34b65be44a265cac7921a63bb20b09786802de99 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Fri, 9 Jul 2021 19:15:45 +0300 Subject: [PATCH 054/376] fix find_package cache pollution --- cmake/OpenCVDetectVTK.cmake | 60 ++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/cmake/OpenCVDetectVTK.cmake b/cmake/OpenCVDetectVTK.cmake index b8cf36007cf2..57c154475c67 100644 --- a/cmake/OpenCVDetectVTK.cmake +++ b/cmake/OpenCVDetectVTK.cmake @@ -1,34 +1,34 @@ -# VTK 9.0 if(NOT VTK_FOUND) - find_package(VTK 9 QUIET NAMES vtk COMPONENTS - FiltersExtraction - FiltersSources - FiltersTexture - IOExport - IOGeometry - IOPLY - InteractionStyle - RenderingCore - RenderingLOD - RenderingOpenGL2 - NO_MODULE) -endif() - -# VTK 6.x components -if(NOT VTK_FOUND) - find_package(VTK QUIET COMPONENTS vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport NO_MODULE) - IF(VTK_FOUND) - IF(VTK_RENDERING_BACKEND) #in vtk 7, the rendering backend is exported as a var. - find_package(VTK QUIET COMPONENTS vtkRendering${VTK_RENDERING_BACKEND} vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport vtkIOGeometry NO_MODULE) - ELSE(VTK_RENDERING_BACKEND) - find_package(VTK QUIET COMPONENTS vtkRenderingOpenGL vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport NO_MODULE) - ENDIF(VTK_RENDERING_BACKEND) - ENDIF(VTK_FOUND) -endif() - -# VTK 5.x components -if(NOT VTK_FOUND) - find_package(VTK QUIET COMPONENTS vtkCommon NO_MODULE) + find_package(VTK QUIET NAMES vtk VTK) + if(VTK_FOUND) + if(VTK_VERSION VERSION_EQUAL "9") # VTK 9.0 + find_package(VTK 9 QUIET NAMES vtk COMPONENTS + FiltersExtraction + FiltersSources + FiltersTexture + IOExport + IOGeometry + IOPLY + InteractionStyle + RenderingCore + RenderingLOD + RenderingOpenGL2 + NO_MODULE) + elseif(VTK_VERSION VERSION_GREATER "5") # VTK 6.x components + find_package(VTK QUIET COMPONENTS vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport NO_MODULE) + IF(VTK_FOUND) + IF(VTK_RENDERING_BACKEND) #in vtk 7, the rendering backend is exported as a var. + find_package(VTK QUIET COMPONENTS vtkRendering${VTK_RENDERING_BACKEND} vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport vtkIOGeometry NO_MODULE) + ELSE(VTK_RENDERING_BACKEND) + find_package(VTK QUIET COMPONENTS vtkRenderingOpenGL vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport NO_MODULE) + ENDIF(VTK_RENDERING_BACKEND) + ENDIF(VTK_FOUND) + elseif(VTK_VERSION VERSION_EQUAL "5") # VTK 5.x components + find_package(VTK QUIET COMPONENTS vtkCommon NO_MODULE) + else() + set(VTK_FOUND FALSE) + endif() + endif() endif() if(NOT VTK_FOUND) From 167a12028daa7b8b0f52fc828b9693c161cc01f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9sar=20Gouveia?= <33461054+cesarpgouveia@users.noreply.github.com> Date: Fri, 9 Jul 2021 19:21:56 +0100 Subject: [PATCH 055/376] Merge pull request #20374 from cesarpgouveia:bugfix/fix_load_onnxModel_debug * Fix bug while loading onnx model in debug * dnn: fix other .at using Co-authored-by: Alexander Alekhin --- modules/dnn/src/onnx/onnx_importer.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 3668c9b51e5d..db16cfd56d8b 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -788,7 +788,7 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) int blob_total = blob.total(); if (blob_total == 1) { layerParams.type = "Power"; - layerParams.set("shift", (isSub ? -1 : 1) * blob.at(0)); + layerParams.set("shift", (isSub ? -1 : 1) * blob.ptr()[0]); } else { MatShape inpShape = outShapes[node_proto.input(1 - const_blob_id)]; @@ -871,7 +871,7 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) blob.convertTo(blob, CV_32F); layerParams.type = "Power"; - layerParams.set("power", blob.at(0)); + layerParams.set("power", blob.ptr()[0]); } else if (layer_type == "Max") { @@ -1150,7 +1150,8 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) Mat blob = getBlob(node_proto, constId); blob = blob.reshape(1, 1); if (blob.total() == 1) { - float coeff = isDiv ? 1.0 / blob.at(0) : blob.at(0); + float blob_value = blob.ptr()[0]; + float coeff = isDiv ? 1.0 / blob_value : blob_value; layerParams.set("scale", coeff); layerParams.type = "Power"; } @@ -1188,12 +1189,14 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) { if (inp0.total() == 1) { - float coeff = isDiv ? 1.0 / inp0.at(0) : inp0.at(0); + float inp0_value = inp0.ptr()[0]; + float coeff = isDiv ? 1.0 / inp0_value : inp0_value; multiply(inp1, coeff, out); } else { - float coeff = isDiv ? 1.0 / inp1.at(0) : inp1.at(0); + float inp1_value = inp1.ptr()[0]; + float coeff = isDiv ? 1.0 / inp1_value : inp1_value; multiply(inp0, coeff, out); } @@ -1605,7 +1608,7 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) if (node_proto.input_size() == 3) { Mat value = getBlob(node_proto, 2); - layerParams.set("value", value.at(0)); + layerParams.set("value", value.ptr()[0]); } } } From 3f3c5de851dc8e28240a5fa292708f3bdcfe8851 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Fri, 9 Jul 2021 21:46:38 +0300 Subject: [PATCH 056/376] Merge pull request #20372 from sivanov-work:serialize GAPI: Implement ConstValue serialize/deserialize * Implement ConstValue ser/deser * Fix MacOs compile issue * Fix Docs compile * Change uint32 -> uint64 for serialize tag --- .../src/backends/common/serialization.cpp | 48 +++++++++++++++++-- .../src/backends/common/serialization.hpp | 6 +++ .../test/s11n/gapi_sample_pipelines_s11n.cpp | 29 +++++++++++ 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/modules/gapi/src/backends/common/serialization.cpp b/modules/gapi/src/backends/common/serialization.cpp index 7389bacb02f0..f2c956874c1e 100644 --- a/modules/gapi/src/backends/common/serialization.cpp +++ b/modules/gapi/src/backends/common/serialization.cpp @@ -32,6 +32,14 @@ void putData(GSerialized& s, const cv::gimpl::GModel::ConstGraph& cg, const ade: }); if (s.m_datas.end() == it) { s.m_datas.push_back(gdata); + + if (cg.metadata(nh).contains()) { + size_t datas_num = s.m_datas.size() - 1; + GAPI_DbgAssert(datas_num <= static_cast(std::numeric_limits::max())); + GSerialized::data_tag_t tag = static_cast(datas_num); + s.m_const_datas.emplace(tag, + cg.metadata(nh).get()); + } } } @@ -42,11 +50,20 @@ void putOp(GSerialized& s, const cv::gimpl::GModel::ConstGraph& cg, const ade::N s.m_ops.push_back(op); } -void mkDataNode(ade::Graph& g, const cv::gimpl::Data& data) { +ade::NodeHandle mkDataNode(ade::Graph& g, const cv::gimpl::Data& data) { cv::gimpl::GModel::Graph gm(g); auto nh = gm.createNode(); gm.metadata(nh).set(cv::gimpl::NodeType{cv::gimpl::NodeType::DATA}); gm.metadata(nh).set(data); + return nh; +} + +ade::NodeHandle mkConstDataNode(ade::Graph& g, const cv::gimpl::Data& data, const cv::gimpl::ConstValue& const_data) { + auto nh = mkDataNode(g, data); + + cv::gimpl::GModel::Graph gm(g); + gm.metadata(nh).set(const_data); + return nh; } void mkOpNode(ade::Graph& g, const cv::gimpl::Op& op) { @@ -624,6 +641,10 @@ IOStream& operator<< (IOStream& os, const cv::gimpl::Data &d) { return os << d.shape << d.rc << d.meta << d.storage << d.kind; } +IOStream& operator<< (IOStream& os, const cv::gimpl::ConstValue &cd) { + return os << cd.arg; +} + namespace { template @@ -667,6 +688,9 @@ IIStream& operator>> (IIStream& is, cv::gimpl::Data &d) { return is; } +IIStream& operator>> (IIStream& is, cv::gimpl::ConstValue &cd) { + return is >> cd.arg; +} IOStream& operator<< (IOStream& os, const cv::gimpl::DataObjectCounter &c) { return os << c.m_next_data_id; @@ -709,18 +733,34 @@ void serialize( IOStream& os } s.m_counter = cg.metadata().get(); s.m_proto = p; - os << s.m_ops << s.m_datas << s.m_counter << s.m_proto; + os << s.m_ops << s.m_datas << s.m_counter << s.m_proto << s.m_const_datas; } GSerialized deserialize(IIStream &is) { GSerialized s; - is >> s.m_ops >> s.m_datas >> s.m_counter >> s.m_proto; + is >> s.m_ops >> s.m_datas >> s.m_counter >> s.m_proto >> s.m_const_datas; return s; } void reconstruct(const GSerialized &s, ade::Graph &g) { GAPI_Assert(g.nodes().empty()); - for (const auto& d : s.m_datas) cv::gapi::s11n::mkDataNode(g, d); + + GSerialized::data_tag_t tag = 0; + for (const auto& d : s.m_datas) { + if (d.storage == gimpl::Data::Storage::CONST_VAL) { + auto cit = s.m_const_datas.find(tag); + if (cit == s.m_const_datas.end()) { + util::throw_error(std::logic_error("Cannot reconstruct graph: Data::Storage::CONST_VAL by tag: " + + std::to_string(tag) + " requires ConstValue")); + } + + mkConstDataNode(g, d, cit->second); + } else { + cv::gapi::s11n::mkDataNode(g, d); + } + + tag ++; + } for (const auto& op : s.m_ops) cv::gapi::s11n::mkOpNode(g, op); cv::gapi::s11n::linkNodes(g); diff --git a/modules/gapi/src/backends/common/serialization.hpp b/modules/gapi/src/backends/common/serialization.hpp index b4204ca64e38..529fdc635d5e 100644 --- a/modules/gapi/src/backends/common/serialization.hpp +++ b/modules/gapi/src/backends/common/serialization.hpp @@ -31,6 +31,9 @@ struct GSerialized { std::vector m_datas; cv::gimpl::DataObjectCounter m_counter; cv::gimpl::Protocol m_proto; + + using data_tag_t = uint64_t; + std::map m_const_datas; }; //////////////////////////////////////////////////////////////////////////////// @@ -97,6 +100,9 @@ GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gimpl::Op &op); GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gimpl::Data &op); GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gimpl::Data &op); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gimpl::ConstValue &cd); +GAPI_EXPORTS IIStream& operator>> (IIStream& os, cv::gimpl::ConstValue &cd); + // Render types //////////////////////////////////////////////////////////////// GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gapi::wip::draw::Text &t); diff --git a/modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp b/modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp index 885457cd9063..c3d21a3f6f8c 100644 --- a/modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp +++ b/modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp @@ -806,4 +806,33 @@ TEST(S11N, Pipeline_Render_RGB) EXPECT_EQ(cv::norm(input, ref_mat), 0); } + +TEST(S11N, Pipeline_Const_GScalar) +{ + static constexpr auto in_scalar = 10; + + cv::GMat a; + cv::GScalar s; + + cv::GComputation computation(GIn(a), GOut(cv::gapi::addC(a, in_scalar))); + auto p = cv::gapi::serialize(computation); + auto deserialized_computation = cv::gapi::deserialize(p); + + cv::Mat in_mat = cv::Mat::eye(32, 32, CV_8UC1); + cv::Mat ref_mat; + cv::add(in_mat, in_scalar, ref_mat); + + cv::Mat out_mat; + computation.apply(cv::gin(in_mat/*, in_scalar*/), cv::gout(out_mat)); + EXPECT_EQ(0, cvtest::norm(out_mat, ref_mat, NORM_INF)); + + out_mat = cv::Mat(); + deserialized_computation.apply(cv::gin(in_mat/*, in_scalar*/), cv::gout(out_mat)); + EXPECT_EQ(0, cvtest::norm(out_mat, ref_mat, NORM_INF)); + + out_mat = cv::Mat(); + auto cc = deserialized_computation.compile(cv::descr_of(in_mat)); + cc(cv::gin(in_mat/*, in_scalar*/), cv::gout(out_mat)); + EXPECT_EQ(0, cvtest::norm(out_mat, ref_mat, NORM_INF)); +} } // namespace opencv_test From fd22e9829815860d25142f9e42aa68fb8f192b3f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 12 Jul 2021 19:32:11 +0000 Subject: [PATCH 057/376] build(winpack_dldt): avoid stale sysroot contents --- platforms/winpack_dldt/build_package.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/platforms/winpack_dldt/build_package.py b/platforms/winpack_dldt/build_package.py index 6fde62241a5e..bd4355e1cdf4 100644 --- a/platforms/winpack_dldt/build_package.py +++ b/platforms/winpack_dldt/build_package.py @@ -189,7 +189,10 @@ def __init__(self, config): if self.srcdir is None: self.srcdir = prepare_dir(self.outdir / 'sources', clean=clean_src_dir) self.build_dir = prepare_dir(self.outdir / 'build', clean=self.config.clean_dldt) - self.sysrootdir = prepare_dir(self.outdir / 'sysroot', clean=self.config.clean_dldt) + self.sysrootdir = prepare_dir(self.outdir / 'sysroot', clean=self.config.clean_dldt or self.config.clean_dldt_sysroot) + if not (self.config.clean_dldt or self.config.clean_dldt_sysroot): + _ = prepare_dir(self.sysrootdir / 'bin', clean=True) # always clean sysroot/bin (package files) + _ = prepare_dir(self.sysrootdir / 'etc', clean=True) # always clean sysroot/etc (package files) if self.config.build_subst_drive: if os.path.exists(self.config.build_subst_drive + ':\\'): @@ -483,8 +486,9 @@ def main(): parser.add_argument('--cmake_option', action='append', help='Append OpenCV CMake option') parser.add_argument('--cmake_option_dldt', action='append', help='Append CMake option for DLDT project') - parser.add_argument('--clean_dldt', action='store_true', help='Clear DLDT build and sysroot directories') - parser.add_argument('--clean_opencv', action='store_true', help='Clear OpenCV build directory') + parser.add_argument('--clean_dldt', action='store_true', help='Clean DLDT build and sysroot directories') + parser.add_argument('--clean_dldt_sysroot', action='store_true', help='Clean DLDT sysroot directories') + parser.add_argument('--clean_opencv', action='store_true', help='Clean OpenCV build directory') parser.add_argument('--build_debug', action='store_true', help='Build debug binaries') parser.add_argument('--build_tests', action='store_true', help='Build OpenCV tests') From 4af1f31a3fb3530499826e516fdc5b5121cdf600 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 13 Jul 2021 09:15:03 +0000 Subject: [PATCH 058/376] cmake: use relative path for mode vars --- cmake/OpenCVUtils.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/OpenCVUtils.cmake b/cmake/OpenCVUtils.cmake index 39445150a911..da0ee3b36bc6 100644 --- a/cmake/OpenCVUtils.cmake +++ b/cmake/OpenCVUtils.cmake @@ -1978,4 +1978,4 @@ endif() # # Include configuration override settings # -include(cmake/vars/EnableModeVars.cmake) +include("${CMAKE_CURRENT_LIST_DIR}/vars/EnableModeVars.cmake") From 5179e37bd1670e025735ee9ed9aa7e4cbe38ccd9 Mon Sep 17 00:00:00 2001 From: Alexey Smirnov Date: Tue, 13 Jul 2021 22:31:46 +0300 Subject: [PATCH 059/376] Merge pull request #20329 from smirnov-alexey:as/mediaframe_serialization [G-API]: Add serialization mechanism for cv::MediaFrame * Stub initial interface * Fix templates for deserialization * Fix tests * Disable a warning on windows * Address review comments * Change enable_ifs to other template helpers * Resolve ambiguous template * Fix warnings in docs --- modules/gapi/include/opencv2/gapi/media.hpp | 28 +++++ modules/gapi/include/opencv2/gapi/rmat.hpp | 10 +- modules/gapi/include/opencv2/gapi/s11n.hpp | 67 +++++++--- .../gapi/include/opencv2/gapi/util/util.hpp | 24 +++- modules/gapi/src/api/media.cpp | 4 + modules/gapi/src/api/s11n.cpp | 14 ++- .../src/backends/common/serialization.cpp | 14 ++- modules/gapi/test/s11n/gapi_s11n_tests.cpp | 116 ++++++++++++++++++ 8 files changed, 243 insertions(+), 34 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/media.hpp b/modules/gapi/include/opencv2/gapi/media.hpp index aa7d6d6a1f4e..19aaef3fd1a5 100644 --- a/modules/gapi/include/opencv2/gapi/media.hpp +++ b/modules/gapi/include/opencv2/gapi/media.hpp @@ -15,6 +15,16 @@ #include #include +// Forward declaration +namespace cv { +namespace gapi { +namespace s11n { +struct IOStream; +struct IIStream; +} // namespace s11n +} // namespace gapi +} // namespace cv + namespace cv { /** \addtogroup gapi_data_structures @@ -125,6 +135,16 @@ class GAPI_EXPORTS MediaFrame { return dynamic_cast(adapter); } + /** + * @brief Serialize MediaFrame's data to a byte array. + * + * @note The actual logic is implemented by frame's adapter class. + * Does nothing by default. + * + * @param os Bytestream to store serialized MediaFrame data in. + */ + void serialize(cv::gapi::s11n::IOStream& os) const; + private: struct Priv; std::shared_ptr m; @@ -221,6 +241,14 @@ class GAPI_EXPORTS MediaFrame::IAdapter { // FIXME: design a better solution // The default implementation does nothing virtual cv::util::any blobParams() const; + virtual void serialize(cv::gapi::s11n::IOStream&) { + GAPI_Assert(false && "Generic serialize method of MediaFrame::IAdapter does nothing by default. " + "Please, implement it in derived class to properly serialize the object."); + } + virtual void deserialize(cv::gapi::s11n::IIStream&) { + GAPI_Assert(false && "Generic deserialize method of MediaFrame::IAdapter does nothing by default. " + "Please, implement it in derived class to properly deserialize the object."); + } }; /** @} */ diff --git a/modules/gapi/include/opencv2/gapi/rmat.hpp b/modules/gapi/include/opencv2/gapi/rmat.hpp index cc27f48664cd..6b289001e7f3 100644 --- a/modules/gapi/include/opencv2/gapi/rmat.hpp +++ b/modules/gapi/include/opencv2/gapi/rmat.hpp @@ -14,8 +14,8 @@ namespace cv { namespace gapi { namespace s11n { - struct IOStream; - struct IIStream; +struct IOStream; +struct IIStream; } // namespace s11n } // namespace gapi } // namespace cv @@ -111,10 +111,12 @@ class GAPI_EXPORTS RMat // is transferred to the device when the view is destroyed virtual View access(Access) = 0; virtual void serialize(cv::gapi::s11n::IOStream&) { - GAPI_Assert(false && "Generic serialize method should never be called for RMat adapter"); + GAPI_Assert(false && "Generic serialize method of RMat::Adapter does nothing by default. " + "Please, implement it in derived class to properly serialize the object."); } virtual void deserialize(cv::gapi::s11n::IIStream&) { - GAPI_Assert(false && "Generic deserialize method should never be called for RMat adapter"); + GAPI_Assert(false && "Generic deserialize method of RMat::Adapter does nothing by default. " + "Please, implement it in derived class to properly deserialize the object."); } }; using AdapterP = std::shared_ptr; diff --git a/modules/gapi/include/opencv2/gapi/s11n.hpp b/modules/gapi/include/opencv2/gapi/s11n.hpp index ca8e32c98bf9..53800970d1cb 100644 --- a/modules/gapi/include/opencv2/gapi/s11n.hpp +++ b/modules/gapi/include/opencv2/gapi/s11n.hpp @@ -13,6 +13,13 @@ #include #include #include +#include +#include + +// FIXME: caused by deserialize_runarg +#if (defined _WIN32 || defined _WIN64) && defined _MSC_VER +#pragma warning(disable: 4702) +#endif namespace cv { namespace gapi { @@ -34,8 +41,8 @@ namespace detail { template cv::GCompileArgs getCompileArgs(const std::vector &bytes); - template - cv::GRunArgs getRunArgsWithRMats(const std::vector &bytes); + template + cv::GRunArgs getRunArgsWithAdapters(const std::vector &bytes); } // namespace detail /** @brief Serialize a graph represented by GComputation into an array of bytes. @@ -133,19 +140,18 @@ type deserialize(const std::vector &bytes) { } /** - * @brief Deserialize GRunArgs including RMat objects if any from a byte array. + * @brief Deserialize GRunArgs including RMat and MediaFrame objects if any from a byte array. * - * RMat adapter type is specified in the template. - * @note To be used properly specified adapter type must overload its serialize() and - * deserialize() methods. + * Adapter types are specified in the template. + * @note To be used properly specified adapter types must overload their deserialize() method. * @param bytes vector of bytes to deserialize GRunArgs object from. - * @return GRunArgs including RMat objects if any. - * @see RMat + * @return GRunArgs including RMat and MediaFrame objects if any. + * @see RMat MediaFrame */ -template inline +template inline typename std::enable_if::value, GRunArgs>:: type deserialize(const std::vector &bytes) { - return detail::getRunArgsWithRMats(bytes); + return detail::getRunArgsWithAdapters(bytes); } } // namespace gapi } // namespace cv @@ -399,16 +405,39 @@ static cv::util::optional exec(const std::string& tag, cv::gapi::s1 } }; -template struct deserialize_runarg; +template +struct deserialize_arg_with_adapter; + +template +struct deserialize_arg_with_adapter { +static GRunArg exec(cv::gapi::s11n::IIStream& is) { + std::unique_ptr ptr(new TA); + ptr->deserialize(is); + return GRunArg { RA(std::move(ptr)) }; +} +}; + +template +struct deserialize_arg_with_adapter { +static GRunArg exec(cv::gapi::s11n::IIStream&) { + GAPI_Assert(false && "No suitable adapter class found during RMat/MediaFrame deserialization. " + "Please, make sure you've passed them in cv::gapi::deserialize() template"); + return GRunArg{}; +} +}; -template +template struct deserialize_runarg { static GRunArg exec(cv::gapi::s11n::IIStream& is, uint32_t idx) { if (idx == GRunArg::index_of()) { - auto ptr = std::make_shared(); - ptr->deserialize(is); - return GRunArg { RMat(std::move(ptr)) }; - } else { // non-RMat arg - use default deserialization + // Type or void (if not found) + using TA = typename cv::util::find_adapter_impl::type; + return deserialize_arg_with_adapter::exec(is); + } else if (idx == GRunArg::index_of()) { + // Type or void (if not found) + using TA = typename cv::util::find_adapter_impl::type; + return deserialize_arg_with_adapter::exec(is); + } else { // not an adapter holding type runarg - use default deserialization GRunArg arg; getRunArgByIdx(is, arg, idx); return arg; @@ -451,8 +480,8 @@ cv::GCompileArgs getCompileArgs(const std::vector &sArgs) { return args; } -template -cv::GRunArgs getRunArgsWithRMats(const std::vector &bytes) { +template +cv::GRunArgs getRunArgsWithAdapters(const std::vector &bytes) { std::unique_ptr pIs = cv::gapi::s11n::detail::getInStream(bytes); cv::gapi::s11n::IIStream& is = *pIs; cv::GRunArgs args; @@ -462,7 +491,7 @@ cv::GRunArgs getRunArgsWithRMats(const std::vector &bytes) { for (uint32_t i = 0; i < sz; ++i) { uint32_t idx = 0; is >> idx; - args.push_back(cv::gapi::detail::deserialize_runarg::exec(is, idx)); + args.push_back(cv::gapi::detail::deserialize_runarg::exec(is, idx)); } return args; diff --git a/modules/gapi/include/opencv2/gapi/util/util.hpp b/modules/gapi/include/opencv2/gapi/util/util.hpp index c6ad0632e268..eb435a3eeff0 100644 --- a/modules/gapi/include/opencv2/gapi/util/util.hpp +++ b/modules/gapi/include/opencv2/gapi/util/util.hpp @@ -153,7 +153,29 @@ overload_lamba_set overload_lambdas(L&& ...lambdas) { return overload_lamba_set(std::forward(lambdas)...); } -} + +template +struct find_adapter_impl; + +template +struct find_adapter_impl +{ + using type = typename std::conditional::value, + T, + void>::type; + static constexpr bool found = std::is_base_of::value; +}; + +template +struct find_adapter_impl +{ + using type = typename std::conditional::value, + T, + typename find_adapter_impl::type>::type; + static constexpr bool found = std::is_base_of::value || + find_adapter_impl::found; +}; +} // namespace util } // namespace cv // \endcond diff --git a/modules/gapi/src/api/media.cpp b/modules/gapi/src/api/media.cpp index 884fc9e83d79..b1c455d40aef 100644 --- a/modules/gapi/src/api/media.cpp +++ b/modules/gapi/src/api/media.cpp @@ -35,6 +35,10 @@ cv::MediaFrame::IAdapter* cv::MediaFrame::getAdapter() const { return m->adapter.get(); } +void cv::MediaFrame::serialize(cv::gapi::s11n::IOStream& os) const { + return m->adapter->serialize(os); +} + cv::MediaFrame::View::View(Ptrs&& ptrs, Strides&& strs, Callback &&cb) : ptr (std::move(ptrs)) , stride(std::move(strs)) diff --git a/modules/gapi/src/api/s11n.cpp b/modules/gapi/src/api/s11n.cpp index 97f5a95c42a6..bd7f46c88aec 100644 --- a/modules/gapi/src/api/s11n.cpp +++ b/modules/gapi/src/api/s11n.cpp @@ -76,14 +76,14 @@ cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &out_args) { #if !defined(GAPI_STANDALONE) case T::index_of() : - outputs.emplace_back((cv::UMat*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); break; #endif case cv::GRunArg::index_of() : - outputs.emplace_back((cv::Mat*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); break; case cv::GRunArg::index_of() : - outputs.emplace_back((cv::Scalar*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); break; case T::index_of() : outputs.emplace_back(cv::util::get(res_obj)); @@ -92,7 +92,10 @@ cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &out_args) outputs.emplace_back(cv::util::get(res_obj)); break; case cv::GRunArg::index_of() : - outputs.emplace_back((cv::RMat*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); + break; + case cv::GRunArg::index_of() : + outputs.emplace_back(&(cv::util::get(res_obj))); break; default: GAPI_Assert(false && "This value type is not supported!"); // ...maybe because of STANDALONE mode. @@ -130,6 +133,9 @@ cv::GRunArg cv::gapi::bind(cv::GRunArgP &out) case T::index_of() : return cv::GRunArg(*cv::util::get(out)); + case T::index_of() : + return cv::GRunArg(*cv::util::get(out)); + default: // ...maybe our types were extended GAPI_Assert(false && "This value type is UNKNOWN!"); diff --git a/modules/gapi/src/backends/common/serialization.cpp b/modules/gapi/src/backends/common/serialization.cpp index f2c956874c1e..619b2feb7417 100644 --- a/modules/gapi/src/backends/common/serialization.cpp +++ b/modules/gapi/src/backends/common/serialization.cpp @@ -201,18 +201,20 @@ IOStream& operator<< (IOStream& os, const cv::RMat& mat) { return os; } IIStream& operator>> (IIStream& is, cv::RMat&) { - util::throw_error(std::logic_error("operator>> for RMat should never be called")); + util::throw_error(std::logic_error("operator>> for RMat should never be called. " + "Instead, cv::gapi::deserialize() " + "should be used")); return is; } -IOStream& operator<< (IOStream& os, const cv::MediaFrame &) { - // Stub - GAPI_Assert(false && "cv::MediaFrame serialization is not supported!"); +IOStream& operator<< (IOStream& os, const cv::MediaFrame &frame) { + frame.serialize(os); return os; } IIStream& operator>> (IIStream& is, cv::MediaFrame &) { - // Stub - GAPI_Assert(false && "cv::MediaFrame serialization is not supported!"); + util::throw_error(std::logic_error("operator>> for MediaFrame should never be called. " + "Instead, cv::gapi::deserialize() " + "should be used")); return is; } diff --git a/modules/gapi/test/s11n/gapi_s11n_tests.cpp b/modules/gapi/test/s11n/gapi_s11n_tests.cpp index c2b17521d966..4c6e63b55204 100644 --- a/modules/gapi/test/s11n/gapi_s11n_tests.cpp +++ b/modules/gapi/test/s11n/gapi_s11n_tests.cpp @@ -2,6 +2,7 @@ #include "backends/common/serialization.hpp" #include +#include #include <../src/backends/common/gbackend.hpp> // asView namespace { @@ -148,6 +149,29 @@ class MyRMatAdapter : public cv::RMat::Adapter { int getVal() { return m_value; } std::string getStr() { return m_str; } }; + +class MyMediaFrameAdapter : public cv::MediaFrame::IAdapter { + cv::Mat m_mat; + int m_value; + std::string m_str; +public: + MyMediaFrameAdapter() = default; + MyMediaFrameAdapter(cv::Mat m, int value, const std::string& str) + : m_mat(m), m_value(value), m_str(str) + {} + virtual cv::MediaFrame::View access(cv::MediaFrame::Access) override { + return cv::MediaFrame::View({m_mat.data}, {m_mat.step}); + } + virtual cv::GFrameDesc meta() const override { return {cv::MediaFormat::BGR, m_mat.size()}; } + virtual void serialize(cv::gapi::s11n::IOStream& os) override { + os << m_value << m_str; + } + virtual void deserialize(cv::gapi::s11n::IIStream& is) override { + is >> m_value >> m_str; + } + int getVal() { return m_value; } + std::string getStr() { return m_str; } +}; } namespace opencv_test { @@ -581,6 +605,17 @@ TEST_F(S11N_Basic, Test_Vector_Of_Strings) { EXPECT_EQ("42", des[2]); } +TEST_F(S11N_Basic, Test_RunArg) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + auto v = cv::GRunArgs{ cv::GRunArg{ mat } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + cv::Mat out_mat = cv::util::get(out[0]); + + EXPECT_EQ(0, cv::norm(mat, out_mat)); +} + TEST_F(S11N_Basic, Test_RunArg_RMat) { cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); cv::RMat rmat = cv::make_rmat(mat, 42, "It actually works"); @@ -614,6 +649,87 @@ TEST_F(S11N_Basic, Test_RunArg_RMat_Scalar_Mat) { EXPECT_EQ(0, cv::norm(mat, out_mat)); } +TEST_F(S11N_Basic, Test_RunArg_MediaFrame) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + auto v = cv::GRunArgs{ cv::GRunArg{ frame } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + cv::MediaFrame out_mat = cv::util::get(out[0]); + auto adapter = out_mat.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); +} + +TEST_F(S11N_Basic, Test_RunArg_MediaFrame_Scalar_Mat) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + cv::Scalar sc(111); + auto v = cv::GRunArgs{ cv::GRunArg{ frame }, cv::GRunArg{ sc }, cv::GRunArg{ mat } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + cv::MediaFrame out_frame = cv::util::get(out[0]); + auto adapter = out_frame.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); + + cv::Scalar out_sc = cv::util::get(out[1]); + EXPECT_EQ(sc, out_sc); + + cv::Mat out_mat = cv::util::get(out[2]); + EXPECT_EQ(0, cv::norm(mat, out_mat)); +} + +TEST_F(S11N_Basic, Test_RunArg_MediaFrame_RMat) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + cv::Mat mat2 = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + auto rmat = cv::make_rmat(mat2, 24, "Hello there"); + + auto v = cv::GRunArgs{ cv::GRunArg{ frame }, cv::GRunArg{ rmat } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + + cv::MediaFrame out_frame = cv::util::get(out[0]); + cv::RMat out_rmat = cv::util::get(out[1]); + + auto adapter = out_frame.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); + + auto adapter2 = out_rmat.get(); + EXPECT_EQ(24, adapter2->getVal()); + EXPECT_EQ("Hello there", adapter2->getStr()); +} + +TEST_F(S11N_Basic, Test_RunArg_RMat_MediaFrame) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + cv::Mat mat2 = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + auto rmat = cv::make_rmat(mat2, 24, "Hello there"); + + auto v = cv::GRunArgs{ cv::GRunArg{ rmat }, cv::GRunArg{ frame } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + + cv::RMat out_rmat = cv::util::get(out[0]); + cv::MediaFrame out_frame = cv::util::get(out[1]); + + auto adapter = out_frame.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); + + auto adapter2 = out_rmat.get(); + EXPECT_EQ(24, adapter2->getVal()); + EXPECT_EQ("Hello there", adapter2->getStr()); +} + namespace { template bool verifyOpaqueKind(T&& in) { From a7742d7d631b00346aed8a681e1bbc338d58651d Mon Sep 17 00:00:00 2001 From: Dmitry Budnikov Date: Tue, 13 Jul 2021 22:33:13 +0300 Subject: [PATCH 060/376] Merge pull request #20383 from dbudniko:dbudniko/mtcnn_1st_pnet_simplification MTCNN 1st pnet simplification to ensure single graph input * 1st pnet simplification to ensure single graph input * address comment from Dmitry M regarding unused variable --- modules/gapi/samples/face_detection_mtcnn.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/gapi/samples/face_detection_mtcnn.cpp b/modules/gapi/samples/face_detection_mtcnn.cpp index c437bdbba46c..d679ba0529b1 100644 --- a/modules/gapi/samples/face_detection_mtcnn.cpp +++ b/modules/gapi/samples/face_detection_mtcnn.cpp @@ -596,7 +596,6 @@ int main(int argc, char* argv[]) { cv::GMat scores[MAX_PYRAMID_LEVELS]; cv::GArray nms_p_faces[MAX_PYRAMID_LEVELS]; cv::GArray total_faces[MAX_PYRAMID_LEVELS]; - cv::GArray faces_init(std::vector{}); //The very first PNet pyramid layer to init total_faces[0] in_resized[0] = cv::gapi::resize(in_originalRGB, level_size[0]); @@ -605,8 +604,7 @@ int main(int argc, char* argv[]) { cv::GArray faces0 = custom::BuildFaces::on(scores[0], regressions[0], static_cast(scales[0]), conf_thresh_p); cv::GArray final_p_faces_for_bb2squares = custom::ApplyRegression::on(faces0, true); cv::GArray final_faces_pnet0 = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares); - nms_p_faces[0] = custom::RunNMS::on(final_faces_pnet0, 0.5f, false); - total_faces[0] = custom::AccumulatePyramidOutputs::on(faces_init, nms_p_faces[0]); + total_faces[0] = custom::RunNMS::on(final_faces_pnet0, 0.5f, false); //The rest PNet pyramid layers to accumlate all layers result in total_faces[PYRAMID_LEVELS - 1]] for (int i = 1; i < pyramid_levels; ++i) { From 6f417b57c1bbb3b113104b3023ae9f2361b4618d Mon Sep 17 00:00:00 2001 From: Pablo Romero Date: Tue, 13 Jul 2021 21:40:15 +0200 Subject: [PATCH 061/376] Merge pull request #20399 from pablorcum:3.4 Improves support for Unix non-Linux systems, including QNX * Fixes #20395. Improves support for Unix non-Linux systems. Focus on QNX Neutrino. Signed-off-by: promero * Update system.cpp --- CMakeLists.txt | 2 ++ modules/core/src/parallel.cpp | 2 +- modules/core/src/system.cpp | 22 +++++++++++++--------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f6a2da53103f..94ef43fcb46f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -648,6 +648,8 @@ if(UNIX) set(OPENCV_LINKER_LIBS ${OPENCV_LINKER_LIBS} m pthread) elseif(EMSCRIPTEN) # no need to link to system libs with emscripten + elseif(QNXNTO) + set(OPENCV_LINKER_LIBS ${OPENCV_LINKER_LIBS} m) else() set(OPENCV_LINKER_LIBS ${OPENCV_LINKER_LIBS} dl m pthread rt) endif() diff --git a/modules/core/src/parallel.cpp b/modules/core/src/parallel.cpp index 9ac7d3e4c093..cfff4cea4bce 100644 --- a/modules/core/src/parallel.cpp +++ b/modules/core/src/parallel.cpp @@ -53,7 +53,7 @@ #undef abs #endif -#if defined __linux__ || defined __APPLE__ || defined __GLIBC__ \ +#if defined __unix__ || defined __APPLE__ || defined __GLIBC__ \ || defined __HAIKU__ || defined __EMSCRIPTEN__ || defined __FreeBSD__ \ || defined __OpenBSD__ #include diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index b6810fa9f5fa..d8b8f6755950 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -114,10 +114,14 @@ void* allocSingletonNewBuffer(size_t size) { return malloc(size); } #include // std::abort #endif -#if defined __ANDROID__ || defined __linux__ || defined __FreeBSD__ || defined __OpenBSD__ || defined __HAIKU__ +#if defined __ANDROID__ || defined __unix__ || defined __FreeBSD__ || defined __OpenBSD__ || defined __HAIKU__ # include # include +#if defined __QNXNTO__ +# include +#else # include +#endif #if defined __ANDROID__ || defined __linux__ # include #endif @@ -128,7 +132,7 @@ void* allocSingletonNewBuffer(size_t size) { return malloc(size); } #endif -#if (defined __ppc64__ || defined __PPC64__) && defined __linux__ +#if (defined __ppc64__ || defined __PPC64__) && defined __unix__ # include "sys/auxv.h" # ifndef AT_HWCAP2 # define AT_HWCAP2 26 @@ -229,7 +233,7 @@ std::wstring GetTempFileNameWinRT(std::wstring prefix) #include "omp.h" #endif -#if defined __linux__ || defined __APPLE__ || defined __EMSCRIPTEN__ || defined __FreeBSD__ || defined __GLIBC__ || defined __HAIKU__ +#if defined __unix__ || defined __APPLE__ || defined __EMSCRIPTEN__ || defined __FreeBSD__ || defined __GLIBC__ || defined __HAIKU__ #include #include #include @@ -591,7 +595,7 @@ struct HWFeatures have[CV_CPU_MSA] = true; #endif - #if (defined __ppc64__ || defined __PPC64__) && defined __linux__ + #if (defined __ppc64__ || defined __PPC64__) && defined __unix__ unsigned int hwcap = getauxval(AT_HWCAP); if (hwcap & PPC_FEATURE_HAS_VSX) { hwcap = getauxval(AT_HWCAP2); @@ -804,12 +808,12 @@ int64 getTickCount(void) LARGE_INTEGER counter; QueryPerformanceCounter( &counter ); return (int64)counter.QuadPart; -#elif defined __linux || defined __linux__ +#elif defined __MACH__ && defined __APPLE__ + return (int64)mach_absolute_time(); +#elif defined __unix__ struct timespec tp; clock_gettime(CLOCK_MONOTONIC, &tp); return (int64)tp.tv_sec*1000000000 + tp.tv_nsec; -#elif defined __MACH__ && defined __APPLE__ - return (int64)mach_absolute_time(); #else struct timeval tv; gettimeofday(&tv, NULL); @@ -823,8 +827,6 @@ double getTickFrequency(void) LARGE_INTEGER freq; QueryPerformanceFrequency(&freq); return (double)freq.QuadPart; -#elif defined __linux || defined __linux__ - return 1e9; #elif defined __MACH__ && defined __APPLE__ static double freq = 0; if( freq == 0 ) @@ -834,6 +836,8 @@ double getTickFrequency(void) freq = sTimebaseInfo.denom*1e9/sTimebaseInfo.numer; } return freq; +#elif defined __unix__ + return 1e9; #else return 1e6; #endif From 2113af9c52b4756809004dc932489e518dea4837 Mon Sep 17 00:00:00 2001 From: Roland Meertens Date: Thu, 3 Jun 2021 20:59:22 +0200 Subject: [PATCH 062/376] Updated grabcut example to show the background in a transparant way --- samples/cpp/grabcut.cpp | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/samples/cpp/grabcut.cpp b/samples/cpp/grabcut.cpp index d3e3db49f9d1..25492166a781 100644 --- a/samples/cpp/grabcut.cpp +++ b/samples/cpp/grabcut.cpp @@ -107,12 +107,14 @@ void GCApplication::showImage() const Mat res; Mat binMask; - if( !isInitialized ) - image->copyTo( res ); - else - { - getBinMask( mask, binMask ); - image->copyTo( res, binMask ); + image->copyTo( res ); + if( isInitialized ){ + getBinMask( mask, binMask); + + Mat black (binMask.rows, binMask.cols, CV_8UC3, cv::Scalar(0,0,0)); + black.setTo(Scalar::all(255), binMask); + + addWeighted(black, 0.5, res, 0.5, 0.0, res); } vector::const_iterator it; @@ -201,24 +203,39 @@ void GCApplication::mouseClick( int event, int x, int y, int flags, void* ) case EVENT_LBUTTONUP: if( rectState == IN_PROCESS ) { - rect = Rect( Point(rect.x, rect.y), Point(x,y) ); - rectState = SET; - setRectInMask(); - CV_Assert( bgdPxls.empty() && fgdPxls.empty() && prBgdPxls.empty() && prFgdPxls.empty() ); + if(rect.x == x || rect.y == y){ + rectState = NOT_SET; + } + else{ + rect = Rect( Point(rect.x, rect.y), Point(x,y) ); + rectState = SET; + setRectInMask(); + CV_Assert( bgdPxls.empty() && fgdPxls.empty() && prBgdPxls.empty() && prFgdPxls.empty() ); + } showImage(); } if( lblsState == IN_PROCESS ) { setLblsInMask(flags, Point(x,y), false); lblsState = SET; + nextIter(); showImage(); } + else{ + if(rectState == SET){ + nextIter(); + showImage(); + } + } break; case EVENT_RBUTTONUP: if( prLblsState == IN_PROCESS ) { setLblsInMask(flags, Point(x,y), true); prLblsState = SET; + } + if(rectState == SET){ + nextIter(); showImage(); } break; From bc210b292b7df85d0582211e85c8017aca165444 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 9 Jul 2021 16:22:13 +0000 Subject: [PATCH 063/376] dnn(test): backport test_ie_models.cpp from 4.5.3 --- modules/dnn/test/test_ie_models.cpp | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/modules/dnn/test/test_ie_models.cpp b/modules/dnn/test/test_ie_models.cpp index da6cbd6fbc2f..06d2e1776dd7 100644 --- a/modules/dnn/test/test_ie_models.cpp +++ b/modules/dnn/test/test_ie_models.cpp @@ -103,11 +103,15 @@ static const std::map& getOpenVINOTestMo #if INF_ENGINE_RELEASE >= 2020010000 // Downloaded using these parameters for Open Model Zoo downloader (2020.1): // ./downloader.py -o ${OPENCV_DNN_TEST_DATA_PATH}/omz_intel_models --cache_dir ${OPENCV_DNN_TEST_DATA_PATH}/.omz_cache/ \ - // --name person-detection-retail-0013 + // --name person-detection-retail-0013,age-gender-recognition-retail-0013 { "person-detection-retail-0013", { // IRv10 "intel/person-detection-retail-0013/FP32/person-detection-retail-0013", "intel/person-detection-retail-0013/FP16/person-detection-retail-0013" }}, + { "age-gender-recognition-retail-0013", { + "intel/age-gender-recognition-retail-0013/FP16/age-gender-recognition-retail-0013", + "intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013" + }}, #endif }; @@ -123,6 +127,21 @@ static const std::vector getOpenVINOTestModelsList() return result; } +inline static std::string getOpenVINOModel(const std::string &modelName, bool isFP16) +{ + const std::map& models = getOpenVINOTestModels(); + const auto it = models.find(modelName); + if (it != models.end()) + { + OpenVINOModelTestCaseInfo modelInfo = it->second; + if (isFP16 && modelInfo.modelPathFP16) + return std::string(modelInfo.modelPathFP16); + else if (!isFP16 && modelInfo.modelPathFP32) + return std::string(modelInfo.modelPathFP32); + } + return std::string(); +} + static inline void genData(const InferenceEngine::TensorDesc& desc, Mat& m, Blob::Ptr& dataPtr) { const std::vector& dims = desc.getDims(); @@ -319,11 +338,8 @@ TEST_P(DNNTestOpenVINO, models) bool isFP16 = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_MYRIAD); - const std::map& models = getOpenVINOTestModels(); - const auto it = models.find(modelName); - ASSERT_TRUE(it != models.end()) << modelName; - OpenVINOModelTestCaseInfo modelInfo = it->second; - std::string modelPath = isFP16 ? modelInfo.modelPathFP16 : modelInfo.modelPathFP32; + const std::string modelPath = getOpenVINOModel(modelName, isFP16); + ASSERT_FALSE(modelPath.empty()) << modelName; std::string xmlPath = findDataFile(modelPath + ".xml", false); std::string binPath = findDataFile(modelPath + ".bin", false); From 9f2dcc3f13d8484d8f97df866f8bd51be4875c36 Mon Sep 17 00:00:00 2001 From: berak Date: Thu, 15 Jul 2021 17:02:23 +0200 Subject: [PATCH 064/376] python: fix trackbar warning --- modules/python/src2/cv2.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index 4bdb0fcc14bc..6c5e6463d2fb 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -1971,15 +1971,23 @@ static void OnChange(int pos, void *param) } #ifdef HAVE_OPENCV_HIGHGUI +// workaround for #20408, use nullptr, set value later +static int _createTrackbar(const String &trackbar_name, const String &window_name, int value, int count, + TrackbarCallback onChange, PyObject* py_callback_info) +{ + int n = createTrackbar(trackbar_name, window_name, NULL, count, onChange, py_callback_info); + setTrackbarPos(trackbar_name, window_name, value); + return n; +} static PyObject *pycvCreateTrackbar(PyObject*, PyObject *args) { PyObject *on_change; char* trackbar_name; char* window_name; - int *value = new int; + int value; int count; - if (!PyArg_ParseTuple(args, "ssiiO", &trackbar_name, &window_name, value, &count, &on_change)) + if (!PyArg_ParseTuple(args, "ssiiO", &trackbar_name, &window_name, &value, &count, &on_change)) return NULL; if (!PyCallable_Check(on_change)) { PyErr_SetString(PyExc_TypeError, "on_change must be callable"); @@ -1998,7 +2006,7 @@ static PyObject *pycvCreateTrackbar(PyObject*, PyObject *args) { registered_callbacks.insert(std::pair(name, py_callback_info)); } - ERRWRAP2(createTrackbar(trackbar_name, window_name, value, count, OnChange, py_callback_info)); + ERRWRAP2(_createTrackbar(trackbar_name, window_name, value, count, OnChange, py_callback_info)); Py_RETURN_NONE; } From 602e7c83e2ea3cf7d094e883c69f0911a37e6d1a Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 10 Jul 2021 13:06:33 +0000 Subject: [PATCH 065/376] dnn(test): add extra IR models, more checks in IE testing code --- modules/dnn/test/test_ie_models.cpp | 98 ++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 10 deletions(-) diff --git a/modules/dnn/test/test_ie_models.cpp b/modules/dnn/test/test_ie_models.cpp index 06d2e1776dd7..3407e95e9bd2 100644 --- a/modules/dnn/test/test_ie_models.cpp +++ b/modules/dnn/test/test_ie_models.cpp @@ -112,6 +112,25 @@ static const std::map& getOpenVINOTestMo "intel/age-gender-recognition-retail-0013/FP16/age-gender-recognition-retail-0013", "intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013" }}, +#endif +#if INF_ENGINE_RELEASE >= 2021020000 + // OMZ: 2020.2 + { "face-detection-0105", { + "intel/face-detection-0105/FP32/face-detection-0105", + "intel/face-detection-0105/FP16/face-detection-0105" + }}, + { "face-detection-0106", { + "intel/face-detection-0106/FP32/face-detection-0106", + "intel/face-detection-0106/FP16/face-detection-0106" + }}, +#endif +#if INF_ENGINE_RELEASE >= 2021040000 + // OMZ: 2021.4 + { "person-vehicle-bike-detection-2004", { + "intel/person-vehicle-bike-detection-2004/FP32/person-vehicle-bike-detection-2004", + "intel/person-vehicle-bike-detection-2004/FP16/person-vehicle-bike-detection-2004" + //"intel/person-vehicle-bike-detection-2004/FP16-INT8/person-vehicle-bike-detection-2004" + }}, #endif }; @@ -145,10 +164,22 @@ inline static std::string getOpenVINOModel(const std::string &modelName, bool is static inline void genData(const InferenceEngine::TensorDesc& desc, Mat& m, Blob::Ptr& dataPtr) { const std::vector& dims = desc.getDims(); - m.create(std::vector(dims.begin(), dims.end()), CV_32F); - randu(m, -1, 1); - - dataPtr = make_shared_blob(desc, (float*)m.data); + if (desc.getPrecision() == InferenceEngine::Precision::FP32) + { + m.create(std::vector(dims.begin(), dims.end()), CV_32F); + randu(m, -1, 1); + dataPtr = make_shared_blob(desc, (float*)m.data); + } + else if (desc.getPrecision() == InferenceEngine::Precision::I32) + { + m.create(std::vector(dims.begin(), dims.end()), CV_32S); + randu(m, -100, 100); + dataPtr = make_shared_blob(desc, (int*)m.data); + } + else + { + FAIL() << "Unsupported precision: " << desc.getPrecision(); + } } void runIE(Target target, const std::string& xmlPath, const std::string& binPath, @@ -254,7 +285,16 @@ void runIE(Target target, const std::string& xmlPath, const std::string& binPath BlobMap inputBlobs; for (auto& it : net.getInputsInfo()) { - genData(it.second->getTensorDesc(), inputsMap[it.first], inputBlobs[it.first]); + const InferenceEngine::TensorDesc& desc = it.second->getTensorDesc(); + genData(desc, inputsMap[it.first], inputBlobs[it.first]); + if (cvtest::debugLevel > 0) + { + const std::vector& dims = desc.getDims(); + std::cout << "Input: '" << it.first << "' precison=" << desc.getPrecision() << " dims=" << dims.size() << " ["; + for (auto d : dims) + std::cout << " " << d; + std::cout << "] ocv_mat=" << inputsMap[it.first].size << " of " << typeToString(inputsMap[it.first].type()) << std::endl; + } } infRequest.SetInput(inputBlobs); @@ -263,7 +303,16 @@ void runIE(Target target, const std::string& xmlPath, const std::string& binPath BlobMap outputBlobs; for (auto& it : net.getOutputsInfo()) { - genData(it.second->getTensorDesc(), outputsMap[it.first], outputBlobs[it.first]); + const InferenceEngine::TensorDesc& desc = it.second->getTensorDesc(); + genData(desc, outputsMap[it.first], outputBlobs[it.first]); + if (cvtest::debugLevel > 0) + { + const std::vector& dims = desc.getDims(); + std::cout << "Output: '" << it.first << "' precison=" << desc.getPrecision() << " dims=" << dims.size() << " ["; + for (auto d : dims) + std::cout << " " << d; + std::cout << "] ocv_mat=" << outputsMap[it.first].size << " of " << typeToString(outputsMap[it.first].type()) << std::endl; + } } infRequest.SetOutput(outputBlobs); @@ -284,6 +333,12 @@ void runCV(Backend backendId, Target targetId, const std::string& xmlPath, const net.setPreferableTarget(targetId); std::vector outNames = net.getUnconnectedOutLayersNames(); + if (cvtest::debugLevel > 0) + { + std::cout << "OpenCV output names: " << outNames.size() << std::endl; + for (auto name : outNames) + std::cout << "- " << name << std::endl; + } std::vector outs; net.forward(outs, outNames); @@ -307,13 +362,26 @@ TEST_P(DNNTestOpenVINO, models) ASSERT_FALSE(backendId != DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && backendId != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) << "Inference Engine backend is required"; -#if INF_ENGINE_VER_MAJOR_EQ(2021040000) - if (targetId == DNN_TARGET_MYRIAD && ( - modelName == "person-detection-retail-0013" || // ncDeviceOpen:1013 Failed to find booted device after boot - modelName == "age-gender-recognition-retail-0013" // ncDeviceOpen:1013 Failed to find booted device after boot +#if INF_ENGINE_VER_MAJOR_GE(2021030000) + if (targetId == DNN_TARGET_MYRIAD && (false + || modelName == "person-detection-retail-0013" // ncDeviceOpen:1013 Failed to find booted device after boot + || modelName == "age-gender-recognition-retail-0013" // ncDeviceOpen:1013 Failed to find booted device after boot + || modelName == "face-detection-0105" // get_element_type() must be called on a node with exactly one output + || modelName == "face-detection-0106" // get_element_type() must be called on a node with exactly one output + || modelName == "person-vehicle-bike-detection-2004" // 2021.4+: ncDeviceOpen:1013 Failed to find booted device after boot ) ) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (targetId == DNN_TARGET_OPENCL && (false + || modelName == "face-detection-0106" // Operation: 2278 of type ExperimentalDetectronPriorGridGenerator(op::v6) is not supported + ) + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (targetId == DNN_TARGET_OPENCL_FP16 && (false + || modelName == "face-detection-0106" // Operation: 2278 of type ExperimentalDetectronPriorGridGenerator(op::v6) is not supported + ) + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif #if INF_ENGINE_VER_MAJOR_GE(2020020000) @@ -350,6 +418,8 @@ TEST_P(DNNTestOpenVINO, models) if (targetId == DNN_TARGET_MYRIAD) resetMyriadDevice(); EXPECT_NO_THROW(runIE(targetId, xmlPath, binPath, inputsMap, ieOutputsMap)) << "runIE"; + if (targetId == DNN_TARGET_MYRIAD) + resetMyriadDevice(); EXPECT_NO_THROW(runCV(backendId, targetId, xmlPath, binPath, inputsMap, cvOutputsMap)) << "runCV"; double eps = 0; @@ -357,6 +427,14 @@ TEST_P(DNNTestOpenVINO, models) if (targetId == DNN_TARGET_CPU && checkHardwareSupport(CV_CPU_AVX_512F)) eps = 1e-5; #endif +#if INF_ENGINE_VER_MAJOR_GE(2021030000) + if (targetId == DNN_TARGET_CPU && modelName == "face-detection-0105") + eps = 2e-4; +#endif +#if INF_ENGINE_VER_MAJOR_GE(2021040000) + if (targetId == DNN_TARGET_CPU && modelName == "person-vehicle-bike-detection-2004") + eps = 1e-6; +#endif EXPECT_EQ(ieOutputsMap.size(), cvOutputsMap.size()); for (auto& srcIt : ieOutputsMap) From fbde0c6c961e631c4c91ce4c94d6c1a891e282dd Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 14 Jul 2021 23:31:41 +0000 Subject: [PATCH 066/376] dnn(ie): fix handling of 1D and non-32F outputs of InferenceEngine --- modules/dnn/src/dnn.cpp | 29 ++++++++--- modules/dnn/src/ie_ngraph.cpp | 97 ++++++++++++++++++++++++++++++++--- modules/dnn/src/ie_ngraph.hpp | 10 +++- 3 files changed, 120 insertions(+), 16 deletions(-) diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 45be6eb97ca9..8182394387ab 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -1944,7 +1944,10 @@ struct Net::Impl : public detail::NetImplBase Ptr ieNode = node.dynamicCast(); CV_Assert(!ieNode.empty()); - ieNode->net->reset(); + + CV_Assert(ieNode->net); + InfEngineNgraphNet& ienet = *ieNode->net; + ienet.reset(); for (it = layers.begin(); it != layers.end(); ++it) { @@ -1961,16 +1964,26 @@ struct Net::Impl : public detail::NetImplBase { for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i) { - InferenceEngine::DataPtr dataPtr = ngraphDataNode(ld.outputBlobsWrappers[i]); - dataPtr->setName(ld.name); + auto it = ienet.outputsDesc.find(ld.name); + if (it != ienet.outputsDesc.end()) + { + const InferenceEngine::TensorDesc& descriptor = it->second; + InferenceEngine::DataPtr dataPtr = ngraphDataOutputNode(ld.outputBlobsWrappers[i], descriptor, ld.name); + dataPtr->setName(ld.name); + } + else + { + InferenceEngine::DataPtr dataPtr = ngraphDataNode(ld.outputBlobsWrappers[i]); + dataPtr->setName(ld.name); + } } } - ieNode->net->addBlobs(ld.inputBlobsWrappers); - ieNode->net->addBlobs(ld.outputBlobsWrappers); + ienet.addBlobs(ld.inputBlobsWrappers); + ienet.addBlobs(ld.outputBlobsWrappers); ld.skip = true; } layers[lastLayerId].skip = false; - ieNode->net->init((Target)preferableTarget); + ienet.init((Target)preferableTarget); return; } @@ -3719,8 +3732,8 @@ void Net::forward(OutputArrayOfArrays outputBlobs, matvec.push_back(impl->getBlob(pins[i])); } - std::vector & outputvec = *(std::vector *)outputBlobs.getObj(); - outputvec = matvec; + outputBlobs.create((int)matvec.size(), 1, CV_32F/*FIXIT*/, -1); // allocate vector + outputBlobs.assign(matvec); } void Net::forward(std::vector >& outputBlobs, diff --git a/modules/dnn/src/ie_ngraph.cpp b/modules/dnn/src/ie_ngraph.cpp index e6c219f13e5a..6736590161c6 100644 --- a/modules/dnn/src/ie_ngraph.cpp +++ b/modules/dnn/src/ie_ngraph.cpp @@ -789,21 +789,32 @@ void NgraphBackendLayer::forward(InputArrayOfArrays inputs, OutputArrayOfArrays } -static InferenceEngine::Layout estimateLayout(const Mat& m) +static InferenceEngine::Layout estimateLayout(int dims) { - if (m.dims == 4) + if (dims == 4) return InferenceEngine::Layout::NCHW; - else if (m.dims == 3) + else if (dims == 3) return InferenceEngine::Layout::CHW; - else if (m.dims == 2) + else if (dims == 2) return InferenceEngine::Layout::NC; - else if (m.dims == 1) + else if (dims == 1) return InferenceEngine::Layout::C; - else if (m.dims == 5) + else if (dims == 5) return InferenceEngine::Layout::NCDHW; else return InferenceEngine::Layout::ANY; } +static inline +InferenceEngine::Layout estimateLayout(size_t dims) +{ + return estimateLayout((int)dims); +} + +static inline +InferenceEngine::Layout estimateLayout(const Mat& m) +{ + return estimateLayout(m.dims); +} static InferenceEngine::DataPtr wrapToInfEngineDataNode(const Mat& m, const std::string& name = "") { @@ -839,6 +850,7 @@ InferenceEngine::Blob::Ptr wrapToNgraphBlob(const Mat& m, InferenceEngine::Layou NgraphBackendWrapper::NgraphBackendWrapper(int targetId, const cv::Mat& m) : BackendWrapper(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, targetId) + , host((Mat*)&m) { dataPtr = wrapToInfEngineDataNode(m); blob = wrapToNgraphBlob(m, estimateLayout(m)); @@ -890,7 +902,11 @@ InferenceEngine::Blob::Ptr copyBlob(const InferenceEngine::Blob::Ptr& blob) copy = InferenceEngine::make_shared_blob(description); } else - CV_Error(Error::StsNotImplemented, "Unsupported blob precision"); + { + std::ostringstream msg; + msg << precision; + CV_Error_(Error::StsNotImplemented, ("Unsupported blob precision: %s", msg.str().c_str())); + } copy->allocate(); return copy; } @@ -903,6 +919,66 @@ InferenceEngine::DataPtr ngraphDataNode(const Ptr& ptr) return p->dataPtr; } +static +InferenceEngine::Blob::Ptr reallocateBlob(Mat &m, const InferenceEngine::TensorDesc& description) +{ + auto dims = description.getDims(); + auto layout = estimateLayout(dims.size()); + MatShape matShape(dims.begin(), dims.end()); + if (description.getPrecision() == InferenceEngine::Precision::FP32) + { + m.create(matShape, CV_32FC1); + return InferenceEngine::make_shared_blob( + {description.getPrecision(), dims, layout}, (float*)m.data); + } + else if (description.getPrecision() == InferenceEngine::Precision::I32) + { + m.create(matShape, CV_32SC1); + return InferenceEngine::make_shared_blob( + {description.getPrecision(), dims, layout}, (int*)m.data); + } + else if (description.getPrecision() == InferenceEngine::Precision::U8) + { + m.create(matShape, CV_8UC1); + return InferenceEngine::make_shared_blob( + {description.getPrecision(), dims, layout}, (uchar*)m.data); + } + std::ostringstream msg; + msg << "Unsupported IE precision: " << description.getPrecision(); + CV_Error(Error::StsNotImplemented, msg.str()); +} + +InferenceEngine::DataPtr ngraphDataOutputNode( + const Ptr& ptr, + const InferenceEngine::TensorDesc& description, + const std::string name) +{ + CV_Assert(!ptr.empty()); + Ptr p = ptr.dynamicCast(); + CV_Assert(!p.empty()); + NgraphBackendWrapper& w = *p; + const InferenceEngine::TensorDesc& blobDesc = w.blob.get()->getTensorDesc(); + auto dims = description.getDims(); + bool reallocate = false; + if (blobDesc.getPrecision() != description.getPrecision()) + { + reallocate = true; + CV_LOG_WARNING(NULL, "Reallocate output '" << name << "' blob due to wrong precision: " << blobDesc.getPrecision() << " => " << description.getPrecision() << " ndims=" << dims.size()); + } + if (dims.size() != blobDesc.getDims().size()) + { + reallocate = true; + CV_LOG_WARNING(NULL, "Reallocate output '" << name << "' blob due to wrong dims: " << blobDesc.getDims().size() << " => " << dims.size()); + } + if (reallocate) + { + auto layout = estimateLayout(dims.size()); + w.dataPtr = InferenceEngine::DataPtr(new InferenceEngine::Data(name, + {description.getPrecision(), dims, layout})); + w.blob = reallocateBlob(*w.host, description); + } + return w.dataPtr; +} void forwardNgraph(const std::vector >& outBlobsWrappers, Ptr& node, bool isAsync) @@ -918,6 +994,13 @@ void InfEngineNgraphNet::reset() allBlobs.clear(); infRequests.clear(); isInit = false; + + outputsDesc.clear(); + for (const auto& it : cnn.getOutputsInfo()) + { + const std::string& name = it.first; + outputsDesc.insert({name, it.second->getTensorDesc()}); + } } void InfEngineNgraphNet::addBlobs(const std::vector >& ptrs) diff --git a/modules/dnn/src/ie_ngraph.hpp b/modules/dnn/src/ie_ngraph.hpp index 7a8c4bef8d5c..617f1d454232 100644 --- a/modules/dnn/src/ie_ngraph.hpp +++ b/modules/dnn/src/ie_ngraph.hpp @@ -54,7 +54,8 @@ class InfEngineNgraphNet void setNodePtr(std::shared_ptr* ptr); void reset(); -private: + +//private: detail::NetImplBase& netImpl_; void release(); @@ -89,6 +90,8 @@ class InfEngineNgraphNet bool hasNetOwner; std::vector requestedOutputs; std::unordered_set> unconnectedNodes; + + std::map outputsDesc; }; class InfEngineNgraphNode : public BackendNode @@ -121,12 +124,17 @@ class NgraphBackendWrapper : public BackendWrapper virtual void copyToHost() CV_OVERRIDE; virtual void setHostDirty() CV_OVERRIDE; + Mat* host; InferenceEngine::DataPtr dataPtr; InferenceEngine::Blob::Ptr blob; AsyncArray futureMat; }; InferenceEngine::DataPtr ngraphDataNode(const Ptr& ptr); +InferenceEngine::DataPtr ngraphDataOutputNode( + const Ptr& ptr, + const InferenceEngine::TensorDesc& description, + const std::string name); // This is a fake class to run networks from Model Optimizer. Objects of that // class simulate responses of layers are imported by OpenCV and supported by From 96d35f7c54e6482e2c041c20dbc78bb3ef568a88 Mon Sep 17 00:00:00 2001 From: SamFC10 Date: Fri, 16 Jul 2021 09:39:41 +0530 Subject: [PATCH 067/376] Fix convolution asymmetric padding bug in onnx importer --- modules/dnn/src/onnx/onnx_importer.cpp | 39 +++++++++++++++++++++++++ modules/dnn/test/test_onnx_importer.cpp | 1 + 2 files changed, 40 insertions(+) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index db16cfd56d8b..ec61a9707eb9 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1263,6 +1263,45 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) } int outCn = layerParams.blobs.empty() ? outShapes[node_proto.input(1)][0] : layerParams.blobs[0].size[0]; layerParams.set("num_output", outCn); + + // Check for asymmetric padding in Conv2D + if (layerParams.has("pad")) + { + bool asymmetricPadding = false; + DictValue pads = layerParams.get("pad"); + const int dims = pads.size() / 2; + for (int i = 0; i < dims; ++i) + { + if (pads.get(i) != pads.get(i + dims)) + { + asymmetricPadding = true; + break; + } + } + if (asymmetricPadding && pads.size() == 4) // [pad_t, pad_l, pad_b, pad_r] + { + layerParams.erase("pad"); + // No paddings required for N, C axis + std::vector paddings(4, 0); + // Add paddings for H, W axis + for (int i = 0; i < dims; ++i) + { + paddings.push_back(pads.get(i)); + paddings.push_back(pads.get(dims + i)); + } + LayerParams padLp; + padLp.name = layerParams.name + "/pad"; + padLp.type = "Padding"; + padLp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size())); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(padLp.name); + + addLayer(padLp, proto); + node_proto.set_input(0, padLp.name); + } + } } else if (layer_type == "ConvTranspose") { diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 600f727d7db4..3923068dbf17 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -109,6 +109,7 @@ TEST_P(Test_ONNX_layers, MaxPooling_2) TEST_P(Test_ONNX_layers, Convolution) { testONNXModels("convolution"); + testONNXModels("conv_asymmetric_pads"); } TEST_P(Test_ONNX_layers, Convolution_variable_weight) From c30078c5a3e852e6df7ec825280ecd8bc03e3107 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Tue, 13 Jul 2021 12:20:35 +0300 Subject: [PATCH 068/376] add NotImplemented layer --- modules/dnn/src/dnn.cpp | 28 ++- modules/dnn/src/dnn_common.hpp | 9 + .../dnn/src/layers/not_implemented_layer.cpp | 194 ++++++++++++++++++ modules/dnn/src/tensorflow/tf_importer.cpp | 95 ++++++--- modules/dnn/test/test_tf_importer.cpp | 33 +++ 5 files changed, 326 insertions(+), 33 deletions(-) create mode 100644 modules/dnn/src/layers/not_implemented_layer.cpp diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 668cce8fa671..2d1a093ef47a 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -99,6 +99,15 @@ bool DNN_DIAGNOSTICS_RUN = false; void enableModelDiagnostics(bool isDiagnosticsMode) { DNN_DIAGNOSTICS_RUN = isDiagnosticsMode; + + if (DNN_DIAGNOSTICS_RUN) + { + detail::NotImplemented::Register(); + } + else + { + detail::NotImplemented::unRegister(); + } } using std::vector; @@ -4001,13 +4010,24 @@ int Net::addLayer(const String &name, const String &type, LayerParams ¶ms) { CV_TRACE_FUNCTION(); - if (impl->getLayerId(name) >= 0) + int id = impl->getLayerId(name); + if (id >= 0) { - CV_Error(Error::StsBadArg, "Layer \"" + name + "\" already into net"); - return -1; + if (!DNN_DIAGNOSTICS_RUN || type != "NotImplemented") + { + CV_Error(Error::StsBadArg, "Layer \"" + name + "\" already into net"); + return -1; + } + else + { + LayerData& ld = impl->layers.find(id)->second; + ld.type = type; + ld.params = params; + return -1; + } } - int id = ++impl->lastLayerId; + id = ++impl->lastLayerId; impl->layerNameToId.insert(std::make_pair(name, id)); impl->layers.insert(std::make_pair(id, LayerData(id, name, type, params))); if (params.get("has_dynamic_shapes", false)) diff --git a/modules/dnn/src/dnn_common.hpp b/modules/dnn/src/dnn_common.hpp index ff8f5e846724..46fae41cc217 100644 --- a/modules/dnn/src/dnn_common.hpp +++ b/modules/dnn/src/dnn_common.hpp @@ -15,6 +15,15 @@ void initializeLayerFactory(); namespace detail { +class NotImplemented : public Layer +{ +public: + static Ptr create(const LayerParams ¶ms); + + static void Register(); + static void unRegister(); +}; + struct NetImplBase { const int networkId; // network global identifier diff --git a/modules/dnn/src/layers/not_implemented_layer.cpp b/modules/dnn/src/layers/not_implemented_layer.cpp new file mode 100644 index 000000000000..c4b134390222 --- /dev/null +++ b/modules/dnn/src/layers/not_implemented_layer.cpp @@ -0,0 +1,194 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "../dnn_common.hpp" + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +namespace detail { + +class NotImplementedImpl CV_FINAL : public NotImplemented +{ +public: + NotImplementedImpl(const LayerParams& params) + { + setParamsFrom(params); + CV_Assert(params.has("type")); + std::stringstream ss; + ss << "Node for layer '" << params.name << "' of type '" << params.get("type") << "' wasn't initialized."; + msg = ss.str(); + } + + CV_DEPRECATED_EXTERNAL + virtual void finalize(const std::vector &input, std::vector &output) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void finalize(InputArrayOfArrays inputs, OutputArrayOfArrays outputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED_EXTERNAL + virtual void forward(std::vector &input, std::vector &output, std::vector &internals) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + void forward_fallback(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals) + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED_EXTERNAL + void finalize(const std::vector &inputs, CV_OUT std::vector &outputs) + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED std::vector finalize(const std::vector &inputs) + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED void run(const std::vector &inputs, + CV_OUT std::vector &outputs, + CV_IN_OUT std::vector &internals) + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual int inputNameToIndex(String inputName) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual int outputNameToIndex(const String& outputName) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initHalide(const std::vector > &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initInfEngine(const std::vector > &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initNgraph(const std::vector > &inputs, + const std::vector >& nodes) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initVkCom(const std::vector > &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initCUDA( + void *context, + const std::vector>& inputs, + const std::vector>& outputs + ) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void applyHalideScheduler(Ptr& node, + const std::vector &inputs, + const std::vector &outputs, + int targetId) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr tryAttach(const Ptr& node) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool setActivation(const Ptr& layer) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void unsetAttached() CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool updateMemoryShapes(const std::vector &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + +private: + std::string msg; +}; + +Ptr NotImplemented::create(const LayerParams& params) +{ + return makePtr(params); +} + +Ptr notImplementedRegisterer(LayerParams ¶ms) +{ + return detail::NotImplemented::create(params); +} + +void NotImplemented::Register() +{ + LayerFactory::registerLayer("NotImplemented", detail::notImplementedRegisterer); +} + +void NotImplemented::unRegister() +{ + LayerFactory::unregisterLayer("NotImplemented"); +} + +} // namespace detail + +CV__DNN_INLINE_NS_END +}} // namespace cv::dnn diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 15f88007b4d1..10670bfef9df 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -466,6 +466,8 @@ void ExcludeLayer(tensorflow::GraphDef& net, const int layer_index, const int in net.mutable_node()->DeleteSubrange(layer_index, 1); } +class LayerHandler; + class TFImporter { public: @@ -473,6 +475,7 @@ class TFImporter TFImporter(Net& net, const char *dataModel, size_t lenModel, const char *dataConfig = NULL, size_t lenConfig = 0); protected: + std::unique_ptr layerHandler; std::unique_ptr utilNet; Net& dstNet; void populateNet(); @@ -514,6 +517,7 @@ class TFImporter private: void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId); + friend class LayerHandler; typedef void (TFImporter::*TFImporterNodeParser)(tensorflow::GraphDef&, const tensorflow::NodeDef&, LayerParams&); typedef std::map DispatchMap; @@ -554,6 +558,20 @@ class TFImporter void parseCustomLayer (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); }; +class LayerHandler +{ +public: + LayerHandler(TFImporter* importer_); + ~LayerHandler() = default; + + bool handleMissing(const opencv_tensorflow::NodeDef& layer); + void handleFailed(const opencv_tensorflow::NodeDef& layer); + +private: + TFImporter* importer; + std::set layers; +}; + const TFImporter::DispatchMap TFImporter::buildDispatchMap() { static DispatchMap dispatch; @@ -2340,7 +2358,8 @@ void TFImporter::parseCustomLayer(tensorflow::GraphDef& net, const tensorflow::N } TFImporter::TFImporter(Net& net, const char *model, const char *config) - : utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), + : layerHandler(DNN_DIAGNOSTICS_RUN ? new LayerHandler(this) : nullptr), + utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap()) { if (model && model[0]) @@ -2362,7 +2381,8 @@ TFImporter::TFImporter( const char *dataModel, size_t lenModel, const char *dataConfig, size_t lenConfig ) - : utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), + : layerHandler(DNN_DIAGNOSTICS_RUN ? new LayerHandler(this) : nullptr), + utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap()) { if (dataModel != NULL && lenModel > 0) @@ -2620,11 +2640,6 @@ DataLayout TFImporter::predictOutputDataLayout(const tensorflow::NodeDef& layer) return it->second; } -Ptr dummy_constructor(LayerParams & params) -{ - return new Layer(params); -} - void TFImporter::populateNet() { CV_Assert(netBin.ByteSize() || netTxt.ByteSize()); @@ -2727,7 +2742,6 @@ void TFImporter::populateNet() addConstNodes(netBin, value_id, layers_to_ignore); addConstNodes(netTxt, value_id, layers_to_ignore); - for (int li = 0; li < layersSize; li++) { const tensorflow::NodeDef& layer = net.node(li); @@ -2785,41 +2799,64 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer) { ((*this).*(iter->second))(net, layer, layerParams); } - else + else if (!DNN_DIAGNOSTICS_RUN || !layerHandler->handleMissing(layer)) { - if (DNN_DIAGNOSTICS_RUN && !LayerFactory::createLayerInstance(type, layerParams)) - { - CV_LOG_ERROR(NULL, "DNN/TF: Node='" << name << "' of type='"<< type - << "' is not supported. This error won't be displayed again."); - LayerFactory::registerLayer(type, dummy_constructor); - } - parseCustomLayer(net, layer, layerParams); } } catch (const std::exception& e) { - if (!DNN_DIAGNOSTICS_RUN) + CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type + << "'. Exception: " << e.what()); + + if (DNN_DIAGNOSTICS_RUN) { - CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type - << "'. Exception: " << e.what()); - throw; + layerHandler->handleFailed(layer); } else { - CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type - << "'. Exception: " << e.what()); - - // internal layer failure (didnt call addLayer) - if (dstNet.getLayerId(name) == -1) - { - int id = dstNet.addLayer(name, type, layerParams); - layer_id[name] = id; - } + throw; } } } +LayerHandler::LayerHandler(TFImporter* importer_) : importer(importer_) {} + +void LayerHandler::handleFailed(const opencv_tensorflow::NodeDef& layer) +{ + LayerParams lp; + lp.name = layer.name(); + lp.type = "NotImplemented"; + lp.set("type", layer.op()); + + // the layer will be created or its params and type will be replaced + int id = importer->dstNet.addLayer(lp.name, "NotImplemented", lp); + if (id != -1) // internal layer failure before the call to addLayer() + { + importer->layer_id[lp.name] = id; + } +} + +bool LayerHandler::handleMissing(const opencv_tensorflow::NodeDef& layer) +{ + LayerParams lp; + // If we didn't add it, but can create it, it's custom and not missing. + if (layers.find(layer.op()) == layers.end() && LayerFactory::createLayerInstance(layer.op(), lp)) + { + return false; + } + + if (layers.insert(layer.op()).second) + { + CV_LOG_ERROR(NULL, "DNN/TF: Node='" << layer.name() << "' of type='"<< layer.op() + << "' is not supported. This error won't be displayed again."); + } + + handleFailed(layer); + + return true; +} + } // namespace #endif //HAVE_PROTOBUF diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 2c3613472451..35751b482467 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -568,6 +568,39 @@ TEST_P(Test_TensorFlow_layers, l2_normalize_3d) runTensorFlowNet("l2_normalize_3d"); } +class Test_TensorFlow_diagnostics : public DNNTestLayer { +public: + Test_TensorFlow_diagnostics() + { + enableModelDiagnostics(true); + } + + ~Test_TensorFlow_diagnostics() + { + enableModelDiagnostics(false); + } + + void runFailingTensorFlowNet(const std::string& prefix, bool hasText = false) + { + std::string netPath = path(prefix + "_net.pb"); + std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : ""); + + Net net = readNetFromTensorflow(netPath, netConfig); + } +}; + +TEST_P(Test_TensorFlow_diagnostics, not_implemented_layer) +{ + runFailingTensorFlowNet("not_implemented_layer"); +} + +TEST_P(Test_TensorFlow_diagnostics, broken_parameters) +{ + runFailingTensorFlowNet("broken_layer"); +} + +INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_diagnostics, dnnBackendsAndTargets()); + class Test_TensorFlow_nets : public DNNTestLayer {}; TEST_P(Test_TensorFlow_nets, MobileNet_SSD) From 2062a7ca8fd313af7ebcd510d8dca10b77858ec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C5=A1a=20Bajtl?= Date: Sun, 18 Jul 2021 10:12:39 +0200 Subject: [PATCH 069/376] Bugfix on import script with web worker. --- modules/js/src/make_umd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/js/src/make_umd.py b/modules/js/src/make_umd.py index bed6ee9bcc0f..1096a8eb31b0 100644 --- a/modules/js/src/make_umd.py +++ b/modules/js/src/make_umd.py @@ -95,7 +95,7 @@ def make_umd(opencvjs, cvjs): root.cv = factory(); } else if (typeof importScripts === 'function') { // Web worker - root.cv = factory; + root.cv = factory(); } else { // Other shells, e.g. d8 root.cv = factory(); From 863ab0e72ee9aa530cbf936489be639837739b86 Mon Sep 17 00:00:00 2001 From: Lukas-Alexander Weber <32765578+lukasalexanderweber@users.noreply.github.com> Date: Tue, 20 Jul 2021 10:59:15 +0200 Subject: [PATCH 070/376] fix TypeError when specifying compose_megapix without rounding the composed image sizes (variable "sz") they will be odly fractions of a pixel (e.g. (5300.965, 3772.897)) and therefore cause a "TypeError: integer argument expected, got float" in line 456 roi = warper.warpRoi(sz, K, cameras[i].R) --- samples/python/stitching_detailed.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/python/stitching_detailed.py b/samples/python/stitching_detailed.py index a7e316105edd..4ee29048d118 100644 --- a/samples/python/stitching_detailed.py +++ b/samples/python/stitching_detailed.py @@ -450,7 +450,8 @@ def main(): cameras[i].focal *= compose_work_aspect cameras[i].ppx *= compose_work_aspect cameras[i].ppy *= compose_work_aspect - sz = (full_img_sizes[i][0] * compose_scale, full_img_sizes[i][1] * compose_scale) + sz = (int(round(full_img_sizes[i][0] * compose_scale)), + int(round(full_img_sizes[i][1] * compose_scale))) K = cameras[i].K().astype(np.float32) roi = warper.warpRoi(sz, K, cameras[i].R) corners.append(roi[0:2]) From 3817f3a89bec25347568dab775fdfa1eea4704cd Mon Sep 17 00:00:00 2001 From: Xiaoxiao Tian Date: Wed, 21 Jul 2021 13:32:50 +0800 Subject: [PATCH 071/376] fix: ocv_target_link_libraries could not handle the keyword rightly #20430 --- cmake/OpenCVUtils.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/OpenCVUtils.cmake b/cmake/OpenCVUtils.cmake index d03dc9c55191..1e0ea947ef77 100644 --- a/cmake/OpenCVUtils.cmake +++ b/cmake/OpenCVUtils.cmake @@ -1488,8 +1488,8 @@ function(ocv_target_link_libraries target) if(NOT LINK_PENDING STREQUAL "") __ocv_push_target_link_libraries(${LINK_MODE} ${LINK_PENDING}) set(LINK_PENDING "") - set(LINK_MODE "${dep}") endif() + set(LINK_MODE "${dep}") else() if(BUILD_opencv_world) if(OPENCV_MODULE_${dep}_IS_PART_OF_WORLD) From d29c7e787159fb01ea31b4673a07a52d752aa66d Mon Sep 17 00:00:00 2001 From: Francesco Petrogalli <25690309+fpetrogalli@users.noreply.github.com> Date: Wed, 21 Jul 2021 16:46:05 +0100 Subject: [PATCH 072/376] Merge pull request #20392 from fpetrogalli:aarch64-semihosting AArch64 semihosting * [ts] Disable filesystem support in the TS module. Because of this change, all the tests loading data will file, but tat least the core module can be tested with the following line: opencv_test_core --gtest_filter=-"*Core_InputOutput*:*Core_globbing.accuracy*" * [aarch64] Build OpenCV for AArch64 semihosting. This patch provide a toolchain file that allows to build the library for semihosting applications [1]. Minimal changes have been applied to the code to be able to compile with a baremetal toolchain. [1] https://developer.arm.com/documentation/100863/latest The option `CV_SEMIHOSTING` is used to guard the bits in the code that are specific to the target. To build the code: cmake ../opencv/ \ -DCMAKE_TOOLCHAIN_FILE=../opencv/platforms/semihosting/aarch64-semihosting.toolchain.cmake \ -DSEMIHOSTING_TOOLCHAIN_PATH=/path/to/baremetal-toolchain/bin/ \ -DBUILD_EXAMPLES=ON -GNinja A barematel toolchain for targeting aarch64 semihosting can be found at [2], under `aarch64-none-elf`. [2] https://developer.arm.com/tools-and-software/open-source-software/developer-tools/gnu-toolchain/gnu-a/downloads The folder `samples/semihosting` provides two example semihosting applications. The two binaries can be executed on the host platform with: qemu-aarch64 ./bin/example_semihosting_histogram qemu-aarch64 ./bin/example_semihosting_norm Similarly, the test and perf executables of the modules can be run with: qemu-aarch64 ./bin/opecv_[test|perf]_ Notice that filesystem support is disabled by the toolchain file, hence some of the test that depend on filesystem support will fail. * [semihosting] Remove blank like at the end of file. [NFC] The spurious blankline was reported by https://pullrequest.opencv.org/buildbot/builders/precommit_docs/builds/31158. * [semihosting] Make the raw pixel file generation OS independent. Use the facilities provided by Cmake to generate the header file instead of a shell script, so that the build doesn't fail on systems that do not have a unix shell. * [semihosting] Rename variable for semihosting compilation. * [semihosting] Move the cmake configuration to a variable file. * [semihosting] Make the guard macro private for the core module. * [semihosting] Remove space. [NFC] * [semihosting] Improve comment with information about semihosting. [NFC] * [semihosting] Update license statement on top of sourvce file. [NFC] * [semihosting] Replace BM_SUFFIX with SEMIHOSTING_SUFFIX. [NFC] * [semihosting] Remove double space. [NFC] * [semihosting] Add some text output to the sample applications. * [semihosting] Remove duplicate entry in cmake configuration. [NFCI] * [semihosting] Replace `long` with `int` in sample apps. [NFCI] * [semihosting] Use `configure_file` to create the random pixels. [NFCI] * [semihosting][bugfix] Fix name of cmakedefine variable. * [semihosting][samples] Use CV_8UC1 for grayscale images. [NFCI] * [semihosting] Add readme file. * [semihosting] Remove blank like at the end of README. [NFC] This fixes the failure at https://pullrequest.opencv.org/buildbot/builders/precommit_docs/builds/31272. --- CMakeLists.txt | 1 + cmake/vars/EnableModeVars.cmake | 3 ++ cmake/vars/OPENCV_SEMIHOSTING.cmake | 10 +++++ modules/calib3d/src/ap3p.cpp | 18 ++++---- modules/core/CMakeLists.txt | 4 ++ modules/core/src/parallel.cpp | 4 ++ modules/ts/src/ts.cpp | 2 + modules/ts/src/ts_gtest.cpp | 10 +++++ .../aarch64-semihosting.toolchain.cmake | 40 +++++++++++++++++ .../include/aarch64_semihosting_port.hpp | 42 ++++++++++++++++++ samples/CMakeLists.txt | 7 ++- samples/semihosting/CMakeLists.txt | 10 +++++ samples/semihosting/README.md | 27 ++++++++++++ samples/semihosting/histogram/CMakeLists.txt | 26 +++++++++++ samples/semihosting/histogram/histogram.cpp | 43 +++++++++++++++++++ samples/semihosting/include/CMakeLists.txt | 16 +++++++ samples/semihosting/include/raw_pixels.hpp.in | 11 +++++ samples/semihosting/norm/CMakeLists.txt | 25 +++++++++++ samples/semihosting/norm/norm.cpp | 33 ++++++++++++++ 19 files changed, 321 insertions(+), 11 deletions(-) create mode 100644 cmake/vars/OPENCV_SEMIHOSTING.cmake create mode 100644 platforms/semihosting/aarch64-semihosting.toolchain.cmake create mode 100644 platforms/semihosting/include/aarch64_semihosting_port.hpp create mode 100644 samples/semihosting/CMakeLists.txt create mode 100644 samples/semihosting/README.md create mode 100644 samples/semihosting/histogram/CMakeLists.txt create mode 100644 samples/semihosting/histogram/histogram.cpp create mode 100644 samples/semihosting/include/CMakeLists.txt create mode 100644 samples/semihosting/include/raw_pixels.hpp.in create mode 100644 samples/semihosting/norm/CMakeLists.txt create mode 100644 samples/semihosting/norm/norm.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f4fe0385d12e..b7e5b58837bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -513,6 +513,7 @@ OCV_OPTION(ENABLE_CONFIG_VERIFICATION "Fail build if actual configuration doesn' OCV_OPTION(OPENCV_ENABLE_MEMALIGN "Enable posix_memalign or memalign usage" ON) OCV_OPTION(OPENCV_DISABLE_FILESYSTEM_SUPPORT "Disable filesystem support" OFF) OCV_OPTION(OPENCV_DISABLE_THREAD_SUPPORT "Build the library without multi-threaded code." OFF) +OCV_OPTION(OPENCV_SEMIHOSTING "Build the library for semihosting target (Arm). See https://developer.arm.com/documentation/100863/latest." OFF) OCV_OPTION(ENABLE_PYLINT "Add target with Pylint checks" (BUILD_DOCS OR BUILD_EXAMPLES) IF (NOT CMAKE_CROSSCOMPILING AND NOT APPLE_FRAMEWORK) ) OCV_OPTION(ENABLE_FLAKE8 "Add target with Python flake8 checker" (BUILD_DOCS OR BUILD_EXAMPLES) IF (NOT CMAKE_CROSSCOMPILING AND NOT APPLE_FRAMEWORK) ) diff --git a/cmake/vars/EnableModeVars.cmake b/cmake/vars/EnableModeVars.cmake index b3c4e79c46d1..3f017af496f2 100644 --- a/cmake/vars/EnableModeVars.cmake +++ b/cmake/vars/EnableModeVars.cmake @@ -16,3 +16,6 @@ endmacro() variable_watch(OPENCV_DISABLE_THREAD_SUPPORT ocv_change_mode_var) set(OPENCV_DISABLE_THREAD_SUPPORT "${OPENCV_DISABLE_THREAD_SUPPORT}") + +variable_watch(OPENCV_SEMIHOSTING ocv_change_mode_var) +set(OPENCV_SEMIHOSTING "${OPENCV_SEMIHOSTING}") diff --git a/cmake/vars/OPENCV_SEMIHOSTING.cmake b/cmake/vars/OPENCV_SEMIHOSTING.cmake new file mode 100644 index 000000000000..66f21c7ebddc --- /dev/null +++ b/cmake/vars/OPENCV_SEMIHOSTING.cmake @@ -0,0 +1,10 @@ +set(CV_TRACE OFF) + +# These third parties libraries are incompatible with the semihosting +# toolchain. +set(WITH_JPEG OFF) +set(WITH_OPENEXR OFF) +set(WITH_TIFF OFF) + +# Turn off `libpng` for some linking issues. +set(WITH_PNG OFF) diff --git a/modules/calib3d/src/ap3p.cpp b/modules/calib3d/src/ap3p.cpp index 386a4499efbe..582b201b36a1 100644 --- a/modules/calib3d/src/ap3p.cpp +++ b/modules/calib3d/src/ap3p.cpp @@ -7,8 +7,6 @@ static inline double cbrt(double x) { return (double)cv::cubeRoot((float)x); }; #endif -using namespace std; - namespace { void solveQuartic(const double *factors, double *realRoots) { const double &a4 = factors[0]; @@ -30,29 +28,29 @@ void solveQuartic(const double *factors, double *realRoots) { double q3 = (72 * r4 * p4 - 2 * p4 * p4 * p4 - 27 * q4 * q4) / 432; // /=2 double t; // *=2 - complex w; + std::complex w; if (q3 >= 0) - w = -sqrt(static_cast >(q3 * q3 - p3 * p3 * p3)) - q3; + w = -std::sqrt(static_cast >(q3 * q3 - p3 * p3 * p3)) - q3; else - w = sqrt(static_cast >(q3 * q3 - p3 * p3 * p3)) - q3; + w = std::sqrt(static_cast >(q3 * q3 - p3 * p3 * p3)) - q3; if (w.imag() == 0.0) { - w.real(cbrt(w.real())); + w.real(std::cbrt(w.real())); t = 2.0 * (w.real() + p3 / w.real()); } else { w = pow(w, 1.0 / 3); t = 4.0 * w.real(); } - complex sqrt_2m = sqrt(static_cast >(-2 * p4 / 3 + t)); + std::complex sqrt_2m = sqrt(static_cast >(-2 * p4 / 3 + t)); double B_4A = -a3 / (4 * a4); double complex1 = 4 * p4 / 3 + t; #if defined(__clang__) && defined(__arm__) && (__clang_major__ == 3 || __clang_major__ == 4) && !defined(__ANDROID__) // details: https://github.com/opencv/opencv/issues/11135 // details: https://github.com/opencv/opencv/issues/11056 - complex complex2 = 2 * q4; - complex2 = complex(complex2.real() / sqrt_2m.real(), 0); + std::complex complex2 = 2 * q4; + complex2 = std::complex(complex2.real() / sqrt_2m.real(), 0); #else - complex complex2 = 2 * q4 / sqrt_2m; + std::complex complex2 = 2 * q4 / sqrt_2m; #endif double sqrt_2m_rh = sqrt_2m.real() / 2; double sqrt1 = sqrt(-(complex1 + complex2)).real() / 2; diff --git a/modules/core/CMakeLists.txt b/modules/core/CMakeLists.txt index 6a969e5fc358..13d0af4db82f 100644 --- a/modules/core/CMakeLists.txt +++ b/modules/core/CMakeLists.txt @@ -157,6 +157,10 @@ if(OPENCV_DISABLE_THREAD_SUPPORT) ocv_target_compile_definitions(${the_module} PUBLIC "OPENCV_DISABLE_THREAD_SUPPORT=1") endif() +if(OPENCV_SEMIHOSTING) + ocv_target_compile_definitions(${the_module} PRIVATE "-DOPENCV_SEMIHOSTING") +endif(OPENCV_SEMIHOSTING) + if(HAVE_HPX) ocv_target_link_libraries(${the_module} LINK_PRIVATE "${HPX_LIBRARIES}") endif() diff --git a/modules/core/src/parallel.cpp b/modules/core/src/parallel.cpp index 1d4179b7b417..8fccd19798ae 100644 --- a/modules/core/src/parallel.cpp +++ b/modules/core/src/parallel.cpp @@ -888,6 +888,7 @@ T minNonZero(const T& val_1, const T& val_2) static int getNumberOfCPUs_() { +#ifndef OPENCV_SEMIHOSTING /* * Logic here is to try different methods of getting CPU counts and return * the minimum most value as it has high probablity of being right and safe. @@ -979,6 +980,9 @@ int getNumberOfCPUs_() #endif return ncpus != 0 ? ncpus : 1; +#else // OPENCV_SEMIHOSTING + return 1; +#endif //OPENCV_SEMIHOSTING } int getNumberOfCPUs() diff --git a/modules/ts/src/ts.cpp b/modules/ts/src/ts.cpp index 3aa403ad87e8..3af3a7b8d5af 100644 --- a/modules/ts/src/ts.cpp +++ b/modules/ts/src/ts.cpp @@ -72,7 +72,9 @@ #if defined _WIN32 || defined WINCE # include #else +#if OPENCV_HAVE_FILESYSTEM_SUPPORT # include +#endif # include #endif diff --git a/modules/ts/src/ts_gtest.cpp b/modules/ts/src/ts_gtest.cpp index a65ef721a2c6..b3debd54d2ed 100644 --- a/modules/ts/src/ts_gtest.cpp +++ b/modules/ts/src/ts_gtest.cpp @@ -1067,6 +1067,7 @@ class GTEST_API_ UnitTestImpl { void AddTestInfo(Test::SetUpTestCaseFunc set_up_tc, Test::TearDownTestCaseFunc tear_down_tc, TestInfo* test_info) { +#if OPENCV_HAVE_FILESYSTEM_SUPPORT // In order to support thread-safe death tests, we need to // remember the original working directory when the test program // was first invoked. We cannot do this in RUN_ALL_TESTS(), as @@ -1079,6 +1080,7 @@ class GTEST_API_ UnitTestImpl { GTEST_CHECK_(!original_working_dir_.IsEmpty()) << "Failed to get the current working directory."; } +#endif GetTestCase(test_info->test_case_name(), test_info->type_param(), @@ -9165,6 +9167,7 @@ static bool IsPathSeparator(char c) { // Returns the current working directory, or "" if unsuccessful. FilePath FilePath::GetCurrentDir() { +#if OPENCV_HAVE_FILESYSTEM_SUPPORT #if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT // Windows CE doesn't have a current directory, so we just return // something reasonable. @@ -9183,6 +9186,9 @@ FilePath FilePath::GetCurrentDir() { # endif // GTEST_OS_NACL return FilePath(result == NULL ? "" : cwd); #endif // GTEST_OS_WINDOWS_MOBILE +#else // OPENCV_HAVE_FILESYSTEM_SUPPORT + return FilePath(""); +#endif // OPENCV_HAVE_FILESYSTEM_SUPPORT } // Returns a copy of the FilePath with the case-insensitive extension removed. @@ -9391,6 +9397,7 @@ bool FilePath::CreateDirectoriesRecursively() const { // directory for any reason, including if the parent directory does not // exist. Not named "CreateDirectory" because that's a macro on Windows. bool FilePath::CreateFolder() const { +#if OPENCV_HAVE_FILESYSTEM_SUPPORT #if GTEST_OS_WINDOWS_MOBILE FilePath removed_sep(this->RemoveTrailingPathSeparator()); LPCWSTR unicode = String::AnsiToUtf16(removed_sep.c_str()); @@ -9406,6 +9413,9 @@ bool FilePath::CreateFolder() const { return this->DirectoryExists(); // An error is OK if the directory exists. } return true; // No error. +#else // OPENCV_HAVE_FILESYSTEM_SUPPORT + return false; +#endif // OPENCV_HAVE_FILESYSTEM_SUPPORT } // If input name has a trailing separator character, remove it and return the diff --git a/platforms/semihosting/aarch64-semihosting.toolchain.cmake b/platforms/semihosting/aarch64-semihosting.toolchain.cmake new file mode 100644 index 000000000000..95bbda3bedba --- /dev/null +++ b/platforms/semihosting/aarch64-semihosting.toolchain.cmake @@ -0,0 +1,40 @@ +# This file is part of OpenCV project. +# It is subject to the license terms in the LICENSE file found in the top-level directory +# of this distribution and at http://opencv.org/license.html + +set(CMAKE_SYSTEM_NAME Generic) +set(CMAKE_SYSTEM_PROCESSOR AArch64) + +set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) + +set(PORT_FILE ${CMAKE_SOURCE_DIR}/platforms/semihosting/include/aarch64_semihosting_port.hpp) + +set(COMMON_FLAGS "--specs=rdimon.specs -DOPENCV_INCLUDE_PORT_FILE=\\\"${PORT_FILE}\\\"") + +set(CMAKE_AR ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-ar${CMAKE_EXECUTABLE_SUFFIX}) +set(CMAKE_ASM_COMPILER ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-gcc${CMAKE_EXECUTABLE_SUFFIX}) +set(CMAKE_C_COMPILER ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-gcc${CMAKE_EXECUTABLE_SUFFIX}) +set(CMAKE_CXX_COMPILER ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-g++${CMAKE_EXECUTABLE_SUFFIX}) +set(CMAKE_LINKER ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-ld${CMAKE_EXECUTABLE_SUFFIX}) +set(CMAKE_OBJCOPY ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-objcopy${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "") +set(CMAKE_RANLIB ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-ranlib${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "") +set(CMAKE_SIZE ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-size${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "") +set(CMAKE_STRIP ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-strip${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "") +set(CMAKE_C_FLAGS ${COMMON_FLAGS} CACHE INTERNAL "") +set(CMAKE_CXX_FLAGS ${COMMON_FLAGS} CACHE INTERNAL "") + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + +set(OPENCV_SEMIHOSTING ON) +set(OPENCV_DISABLE_THREAD_SUPPORT ON) +set(OPENCV_DISABLE_FILESYSTEM_SUPPORT ON) +set(BUILD_SHARED_LIBS OFF) +set(OPENCV_FORCE_3RDPARTY_BUILD OFF) + + +# Enable newlib. +add_definitions(-D_GNU_SOURCE) + +add_definitions(-D_POSIX_PATH_MAX=0) diff --git a/platforms/semihosting/include/aarch64_semihosting_port.hpp b/platforms/semihosting/include/aarch64_semihosting_port.hpp new file mode 100644 index 000000000000..d3151c240a30 --- /dev/null +++ b/platforms/semihosting/include/aarch64_semihosting_port.hpp @@ -0,0 +1,42 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef AARCH64_BAREMETAL_PORT_HPP +#define AARCH64_BAREMETAL_PORT_HPP + +#include // Needed for `memalign`. +#include // Needed for `ENOMEM`. + +// -std=c++11 is missing the following definitions when targeting +// semihosting on aarch64. +#if __cplusplus == 201103L +#include +#define M_PI 3.14159265358979323846 +#define M_SQRT2 1.41421356237309504880 + +namespace std { +inline double cbrt(double x) { + return ::cbrt(x); +} +inline double copysign(double mag, double sgn) { + return ::copysign(mag, sgn); +} +} //namespace std +#endif // __cplusplus == 201103L + +extern "C" { +// Redirect the implementation of `posix_memalign` to `memalign` +// as the former is +// missing at link time. https://pubs.opengroup.org/onlinepubs/9699919799/functions/posix_memalign.html +__attribute__((weak)) int posix_memalign(void **memptr, size_t alignment, size_t size) { + void * ptr = memalign(alignment, size); + if (ptr != NULL) { + *memptr = ptr; + return 0; + } + return ENOMEM; +} +} // extern "C" + +#endif diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 0c70698ccbf6..9bfc2bf8ada4 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -45,7 +45,12 @@ endif() if(INSTALL_PYTHON_EXAMPLES) add_subdirectory(python) endif() - +# The examples in this folder will work with a semihosting version of +# OpenCV. For more information about semihosting, see +# https://developer.arm.com/documentation/100863/latest +if(OPENCV_SEMIHOSTING) + add_subdirectory(semihosting) +endif() ocv_install_example_src("." CMakeLists.txt samples_utils.cmake) if(INSTALL_C_EXAMPLES) install(DIRECTORY data DESTINATION "${OPENCV_SAMPLES_SRC_INSTALL_PATH}" COMPONENT samples_data) diff --git a/samples/semihosting/CMakeLists.txt b/samples/semihosting/CMakeLists.txt new file mode 100644 index 000000000000..9fddb0587b43 --- /dev/null +++ b/samples/semihosting/CMakeLists.txt @@ -0,0 +1,10 @@ +# This file is part of OpenCV project. +# It is subject to the license terms in the LICENSE file found in the top-level directory +# of this distribution and at http://opencv.org/license.html + +set(SEMIHOSTING_SUFFIX semihosting) + +add_subdirectory(include) +set(RAW_PIXEL_INCLUDE ${CMAKE_CURRENT_BINARY_DIR}/include) +add_subdirectory(histogram) +add_subdirectory(norm) diff --git a/samples/semihosting/README.md b/samples/semihosting/README.md new file mode 100644 index 000000000000..881b09b735b8 --- /dev/null +++ b/samples/semihosting/README.md @@ -0,0 +1,27 @@ +# Arm semihosting + +This folder contain a toolchain file and a couple of examples for +building OpenCV based applications that can run in an [Arm +semihosting](https://developer.arm.com/documentation/100863/latest) +setup. + +OpenCV can be compiled to target a semihosting platform as follows: + +``` +cmake ../opencv/ \ + -DCMAKE_TOOLCHAIN_FILE=../opencv/platforms/semihosting/aarch64-semihosting.toolchain.cmake \ + -DSEMIHOSTING_TOOLCHAIN_PATH=/path/to/baremetal-toolchain/bin/ \ + -DBUILD_EXAMPLES=ON -GNinja +``` + +A barematel toolchain for targeting aarch64 semihosting can be found +[here](https://developer.arm.com/tools-and-software/open-source-software/developer-tools/gnu-toolchain/gnu-a/downloads), +under `aarch64-none-elf`. + +The code of the examples in the `norm` and `histogram` folders can be +executed with qemu in Linux userspace: + +``` + qemu-aarch64 ./bin/example_semihosting_histogram + qemu-aarch64 ./bin/example_semihosting_norm +``` diff --git a/samples/semihosting/histogram/CMakeLists.txt b/samples/semihosting/histogram/CMakeLists.txt new file mode 100644 index 000000000000..d2f065d1b9c8 --- /dev/null +++ b/samples/semihosting/histogram/CMakeLists.txt @@ -0,0 +1,26 @@ +# This file is part of OpenCV project. +# It is subject to the license terms in the LICENSE file found in the top-level directory +# of this distribution and at http://opencv.org/license.html + +set(PROJECT_NAME histogram) +project(${PROJECT_NAME}) + +ocv_install_example_src(histogram *.cpp *.hpp CMakeLists.txt) + +set(LOCAL_DEPS + opencv_core + opencv_imgproc + ${OPENCV_MODULES_PUBLIC} + ${OpenCV_LIB_COMPONENTS}) +ocv_check_dependencies(${LOCAL_DEPS}) + +if(NOT OCV_DEPENDENCIES_FOUND) + return() +endif() + +ocv_define_sample(histogram histogram.cpp ${SEMIHOSTING_SUFFIX}) +ocv_include_modules_recurse(${LOCAL_DEPS}) +target_include_directories(${histogram} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(${histogram} PRIVATE ${RAW_PIXEL_INCLUDE}) +ocv_target_link_libraries(${histogram} PRIVATE ${OPENCV_LINKER_LIBS} + ${LOCAL_DEPS}) diff --git a/samples/semihosting/histogram/histogram.cpp b/samples/semihosting/histogram/histogram.cpp new file mode 100644 index 000000000000..daa568d0bbb0 --- /dev/null +++ b/samples/semihosting/histogram/histogram.cpp @@ -0,0 +1,43 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include +#include + +#include +#include +#include +#include "raw_pixels.hpp" + +#define IMG_ROWS 100 +#define IMG_COLS 100 + +static_assert(IMG_ROWS * IMG_COLS <= RAW_PIXELS_SIZE, "Incompatible size"); + +int main(void) +{ + // Number of experiment runs + int no_runs = 2; + + // https://docs.opencv.org/master/d3/d63/classcv_1_1Mat.html + cv::Mat src_new(IMG_ROWS, IMG_COLS, CV_8UC1, (void *)raw_pixels); + + // Set parameters + int imgCount = 1; + const int channels[] = {0}; + cv::Mat mask = cv::Mat(); + cv::Mat hist; + int dims = 1; + const int hist_sizes[] = {256}; + float Range[] = {0,256}; + const float *ranges[] = {Range}; + + // Run calc Hist + for(int i=0; i < no_runs; i++){ + std::cout << "Running iteration # "<< i << std::endl; + cv::calcHist(&src_new, imgCount, channels, mask, hist, dims, hist_sizes, ranges); + } + + return 0; +} diff --git a/samples/semihosting/include/CMakeLists.txt b/samples/semihosting/include/CMakeLists.txt new file mode 100644 index 000000000000..3c429b8adf6c --- /dev/null +++ b/samples/semihosting/include/CMakeLists.txt @@ -0,0 +1,16 @@ +# Populate a C array with random data. +set(RAW_PIXELS_SIZE 102400) +set(RAW_PIXELS_HEADER ${CMAKE_CURRENT_BINARY_DIR}/raw_pixels.hpp) +set(RAW_PIXELS_HEADER_IN ${CMAKE_CURRENT_SOURCE_DIR}/raw_pixels.hpp.in) + +set(RAW_PIXEL_VALUES "") +# Seed the random number generator. +string(RANDOM LENGTH 8 ALPHABET 0123456789abcdf RANDOM_SEED 314 number) +math(EXPR LOOP_RANGE "${RAW_PIXELS_SIZE} - 1") + +foreach(i RANGE ${LOOP_RANGE}) + string(RANDOM LENGTH 8 ALPHABET 0123456789abcdf number) + string(CONCAT RAW_PIXEL_VALUES ${RAW_PIXEL_VALUES} "0x${number}, \\\n") +endforeach() + +configure_file(${RAW_PIXELS_HEADER_IN} ${RAW_PIXELS_HEADER}) diff --git a/samples/semihosting/include/raw_pixels.hpp.in b/samples/semihosting/include/raw_pixels.hpp.in new file mode 100644 index 000000000000..6ee98222cc1b --- /dev/null +++ b/samples/semihosting/include/raw_pixels.hpp.in @@ -0,0 +1,11 @@ +#ifndef RAW_PIXELS_HPP +#define RAW_PIXELS_HP +#include + +#cmakedefine RAW_PIXEL_VALUES @RAW_PIXEL_VALUES@ +#cmakedefine RAW_PIXELS_SIZE @RAW_PIXELS_SIZE@ + +static std::uint32_t raw_pixels[RAW_PIXELS_SIZE] = { + RAW_PIXEL_VALUES +}; +#endif //RAW_PIXELS_HPP diff --git a/samples/semihosting/norm/CMakeLists.txt b/samples/semihosting/norm/CMakeLists.txt new file mode 100644 index 000000000000..6f23d74627d2 --- /dev/null +++ b/samples/semihosting/norm/CMakeLists.txt @@ -0,0 +1,25 @@ +# This file is part of OpenCV project. +# It is subject to the license terms in the LICENSE file found in the top-level directory +# of this distribution and at http://opencv.org/license.html + +set(PROJECT_NAME norm) +project(${PROJECT_NAME}) + +ocv_install_example_src(norm *.cpp *.hpp CMakeLists.txt) + +set(LOCAL_DEPS + opencv_core + ${OPENCV_MODULES_PUBLIC} + ${OpenCV_LIB_COMPONENTS}) +ocv_check_dependencies(${LOCAL_DEPS}) + +if(NOT OCV_DEPENDENCIES_FOUND) + return() +endif() + +ocv_define_sample(norm norm.cpp ${SEMIHOSTING_SUFFIX}) +ocv_include_modules_recurse(${LOCAL_DEPS}) +target_include_directories(${norm} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(${norm} PRIVATE ${RAW_PIXEL_INCLUDE}) +ocv_target_link_libraries(${norm} PRIVATE ${OPENCV_LINKER_LIBS} + ${LOCAL_DEPS}) diff --git a/samples/semihosting/norm/norm.cpp b/samples/semihosting/norm/norm.cpp new file mode 100644 index 000000000000..f911754be132 --- /dev/null +++ b/samples/semihosting/norm/norm.cpp @@ -0,0 +1,33 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include +#include + +#include +#include +#include +#include "raw_pixels.hpp" + +#define IMG_ROWS 100 +#define IMG_COLS 100 + +static_assert(IMG_ROWS * IMG_COLS <= RAW_PIXELS_SIZE, "Incompatible size"); + +int main(void) +{ + // Number of experiment runs + int no_runs = 2; + + // https://docs.opencv.org/master/d3/d63/classcv_1_1Mat.html + cv::Mat src(IMG_ROWS, IMG_COLS, CV_8UC1, (void *)raw_pixels); + + // Run calc Hist + for(int i=0; i < no_runs; i++){ + std::cout << "Running iteration # "<< i << std::endl; + cv::norm(src); + } + + return 0; +} From aae48e6fd77a30e45b56c60ceef20de21886d5a9 Mon Sep 17 00:00:00 2001 From: Dmitry Budnikov Date: Thu, 22 Jul 2021 16:52:11 +0300 Subject: [PATCH 073/376] single transpose MTCNN version --- modules/gapi/samples/face_detection_mtcnn.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/modules/gapi/samples/face_detection_mtcnn.cpp b/modules/gapi/samples/face_detection_mtcnn.cpp index d679ba0529b1..ad7c20c1175c 100644 --- a/modules/gapi/samples/face_detection_mtcnn.cpp +++ b/modules/gapi/samples/face_detection_mtcnn.cpp @@ -589,18 +589,18 @@ int main(int argc, char* argv[]) { //Preprocessing BGR2RGB + transpose (NCWH is expected instead of NCHW) cv::GMat in_original; cv::GMat in_originalRGB = cv::gapi::BGR2RGB(in_original); + cv::GMat in_transposedRGB = cv::gapi::transpose(in_originalRGB); cv::GOpaque in_sz = cv::gapi::streaming::size(in_original); cv::GMat in_resized[MAX_PYRAMID_LEVELS]; - cv::GMat in_transposed[MAX_PYRAMID_LEVELS]; cv::GMat regressions[MAX_PYRAMID_LEVELS]; cv::GMat scores[MAX_PYRAMID_LEVELS]; cv::GArray nms_p_faces[MAX_PYRAMID_LEVELS]; cv::GArray total_faces[MAX_PYRAMID_LEVELS]; //The very first PNet pyramid layer to init total_faces[0] - in_resized[0] = cv::gapi::resize(in_originalRGB, level_size[0]); - in_transposed[0] = cv::gapi::transpose(in_resized[0]); - std::tie(regressions[0], scores[0]) = run_mtcnn_p(in_transposed[0], get_pnet_level_name(level_size[0])); + cv::Size currentSize = cv::Size(level_size[0].height, level_size[0].width); + in_resized[0] = cv::gapi::resize(in_transposedRGB, currentSize); + std::tie(regressions[0], scores[0]) = run_mtcnn_p(in_resized[0], get_pnet_level_name(level_size[0])); cv::GArray faces0 = custom::BuildFaces::on(scores[0], regressions[0], static_cast(scales[0]), conf_thresh_p); cv::GArray final_p_faces_for_bb2squares = custom::ApplyRegression::on(faces0, true); cv::GArray final_faces_pnet0 = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares); @@ -608,9 +608,9 @@ int main(int argc, char* argv[]) { //The rest PNet pyramid layers to accumlate all layers result in total_faces[PYRAMID_LEVELS - 1]] for (int i = 1; i < pyramid_levels; ++i) { - in_resized[i] = cv::gapi::resize(in_originalRGB, level_size[i]); - in_transposed[i] = cv::gapi::transpose(in_resized[i]); - std::tie(regressions[i], scores[i]) = run_mtcnn_p(in_transposed[i], get_pnet_level_name(level_size[i])); + currentSize = cv::Size(level_size[i].height, level_size[i].width); + in_resized[i] = cv::gapi::resize(in_transposedRGB, currentSize); + std::tie(regressions[i], scores[i]) = run_mtcnn_p(in_resized[i], get_pnet_level_name(level_size[i])); cv::GArray faces = custom::BuildFaces::on(scores[i], regressions[i], static_cast(scales[i]), conf_thresh_p); cv::GArray final_p_faces_for_bb2squares_i = custom::ApplyRegression::on(faces, true); cv::GArray final_faces_pnet_i = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares_i); @@ -624,8 +624,7 @@ int main(int argc, char* argv[]) { //Refinement part of MTCNN graph cv::GArray faces_roi_pnet = custom::R_O_NetPreProcGetROIs::on(final_faces_pnet, in_sz); cv::GArray regressionsRNet, scoresRNet; - cv::GMat in_originalRGB_transposed = cv::gapi::transpose(in_originalRGB); - std::tie(regressionsRNet, scoresRNet) = cv::gapi::infer(faces_roi_pnet, in_originalRGB_transposed); + std::tie(regressionsRNet, scoresRNet) = cv::gapi::infer(faces_roi_pnet, in_transposedRGB); //Refinement post-processing cv::GArray rnet_post_proc_faces = custom::RNetPostProc::on(final_faces_pnet, scoresRNet, regressionsRNet, conf_thresh_r); @@ -636,7 +635,7 @@ int main(int argc, char* argv[]) { //Output part of MTCNN graph cv::GArray faces_roi_rnet = custom::R_O_NetPreProcGetROIs::on(final_faces_rnet, in_sz); cv::GArray regressionsONet, scoresONet, landmarksONet; - std::tie(regressionsONet, landmarksONet, scoresONet) = cv::gapi::infer(faces_roi_rnet, in_originalRGB_transposed); + std::tie(regressionsONet, landmarksONet, scoresONet) = cv::gapi::infer(faces_roi_rnet, in_transposedRGB); //Output post-processing cv::GArray onet_post_proc_faces = custom::ONetPostProc::on(final_faces_rnet, scoresONet, regressionsONet, landmarksONet, conf_thresh_o); From 024b43ca06dc7ab5e612aeee55fb44cdb534c49d Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Mon, 19 Jul 2021 18:24:15 +0300 Subject: [PATCH 074/376] implement asymmetric padding for conv2d, max_pool and conv2d_backprop_input --- modules/dnn/src/tensorflow/tf_importer.cpp | 112 +++++++++++++++++++-- modules/dnn/test/test_tf_importer.cpp | 10 ++ 2 files changed, 112 insertions(+), 10 deletions(-) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 39c230939474..426710989e48 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -404,12 +404,53 @@ void setKSize(LayerParams &layerParams, const tensorflow::NodeDef &layer) } } -void setPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer) +void setPadMode(LayerParams &layerParams, const tensorflow::NodeDef &layer) { if (hasLayerAttr(layer, "padding")) layerParams.set("pad_mode", getLayerAttr(layer, "padding").s()); } +bool getExplicitPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer, int64_t (&pads)[8]) +{ + if (!layerParams.has("pad_mode") || + layerParams.get("pad_mode").getStringValue() != "EXPLICIT") + { + return false; + } + + CV_Assert(hasLayerAttr(layer, "explicit_paddings")); + + const tensorflow::AttrValue& protoPads = getLayerAttr(layer, "explicit_paddings"); + if (protoPads.list().i_size() != 8) + { + CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding configuration."); + } + + int n = sizeof(pads) / sizeof(pads[0]); + for (int i = 0; i < n; ++i) + { + pads[i] = protoPads.list().i(i); + } + + if (getDataLayout(layer) != DATA_LAYOUT_NCHW) + { + CV_LOG_DEBUG(NULL, "DNN/TF: Data format " << getLayerAttr(layer, "data_format").s() << ", assuming NHWC."); + // Perhaps, we have NHWC padding dimensions order. + // N H W C + // 0 1 2 3 4 5 6 7 + std::swap(pads[2], pads[6]); + std::swap(pads[3], pads[7]); + // N C W H + // 0 1 2 3 4 5 6 7 + std::swap(pads[4], pads[6]); + std::swap(pads[5], pads[7]); + // N C H W + // 0 1 2 3 4 5 6 7 + } + + return true; +} + Pin parsePin(const std::string &name) { Pin pin(name); @@ -510,6 +551,7 @@ class TFImporter private: void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId); + void setPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer, std::string& inputName, float value = 0.); typedef void (TFImporter::*TFImporterNodeParser)(tensorflow::GraphDef&, const tensorflow::NodeDef&, LayerParams&); typedef std::map DispatchMap; @@ -551,6 +593,31 @@ class TFImporter void parseCustomLayer (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); }; +void TFImporter::setPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer, std::string& inputName, float value) +{ + setPadMode(layerParams, layer); + int64_t pads[8]; + + if (!getExplicitPadding(layerParams, layer, pads)) + { + return; + } + + LayerParams padLp; + padLp.name = layer.name() + "/pad"; + padLp.type = "Padding"; + padLp.set("paddings", DictValue::arrayInt(pads, sizeof(pads) / sizeof(pads[0]))); + padLp.set("value", value); + + int id = dstNet.addLayer(padLp.name, padLp.type, padLp); + layer_id[padLp.name] = id; + + connect(layer_id, dstNet, parsePin(inputName), id, 0); + inputName = padLp.name; + + layerParams.set("pad_mode", "VALID"); +} + const TFImporter::DispatchMap TFImporter::buildDispatchMap() { static DispatchMap dispatch; @@ -787,7 +854,7 @@ void TFImporter::parseConvolution(tensorflow::GraphDef& net, const tensorflow::N setStrides(layerParams, layer); if (!layerParams.has("pad_w") && !layerParams.has("pad_h")) - setPadding(layerParams, layer); + setPadding(layerParams, layer, input); // The final node of dilated convolution subgraph. next_layers = getNextLayers(net, name, "BatchToSpaceND"); @@ -1232,20 +1299,21 @@ void TFImporter::parseMaxPool(tensorflow::GraphDef& net, const tensorflow::NodeD { const std::string& name = layer.name(); const int num_inputs = layer.input_size(); + std::string inputName = layer.input(0); CV_CheckGT(num_inputs, 0, ""); layerParams.set("pool", "max"); setKSize(layerParams, layer); setStrides(layerParams, layer); - setPadding(layerParams, layer); + setPadding(layerParams, layer, inputName, -std::numeric_limits::infinity()); // Test_TensorFlow_nets.EAST_text_detection/1, NGRAPH/CPU layerParams.set("ceil_mode", false); int id = dstNet.addLayer(name, "Pooling", layerParams); layer_id[name] = id; - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); + connectToAllBlobs(layer_id, dstNet, parsePin(inputName), id, num_inputs); } void TFImporter::parseAvgPool(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) @@ -1258,7 +1326,7 @@ void TFImporter::parseAvgPool(tensorflow::GraphDef& net, const tensorflow::NodeD layerParams.set("ave_pool_padded_area", false); setKSize(layerParams, layer); setStrides(layerParams, layer); - setPadding(layerParams, layer); + setPadMode(layerParams, layer); int id = dstNet.addLayer(name, "Pooling", layerParams); layer_id[name] = id; @@ -1673,7 +1741,7 @@ void TFImporter::parseConv2DBackpropInput(tensorflow::GraphDef& net, const tenso // input: "weights" // input: "input" - const std::string& name = layer.name(); + std::string name = layer.name(); const int num_inputs = layer.input_size(); CV_CheckEQ(num_inputs, 3, "Expected output shape, weights and input nodes"); @@ -1704,7 +1772,21 @@ void TFImporter::parseConv2DBackpropInput(tensorflow::GraphDef& net, const tenso layerParams.set("num_output", kshape[1]); setStrides(layerParams, layer); - setPadding(layerParams, layer); + setPadMode(layerParams, layer); + int64_t pads[8]; + bool explicit_pads = getExplicitPadding(layerParams, layer, pads); + int64_t begs[4] = {}; + int64_t ends[4] = {-1, -1, -1, -1}; + if (explicit_pads) + { + name += "/deconv"; + layerParams.set("pad_mode", "VALID"); + for (int i = 2; i < 4; ++i) // begins=[0, 0, a, b], ends=[-1, -1, c, d] + { + begs[i] = pads[2*i]; + ends[i] = -1 - pads[2*i + 1]; + } + } // For convolution layer, output shape computes as // o = 1 + (i - k + 2*p) / s @@ -1721,8 +1803,9 @@ void TFImporter::parseConv2DBackpropInput(tensorflow::GraphDef& net, const tenso const int strideY = layerParams.get("stride_h"); const int strideX = layerParams.get("stride_w"); Mat outShape = getTensorContent(getConstBlob(layer, value_id, 0)); - const int outH = outShape.at(1); - const int outW = outShape.at(2); + int shift = (getDataLayout(layer) == DATA_LAYOUT_NCHW); + const int outH = outShape.at(1 + shift) + begs[2] - 1 - ends[2]; + const int outW = outShape.at(2 + shift) + begs[3] - 1 - ends[3]; if (layerParams.get("pad_mode") == "SAME") { layerParams.set("adj_w", (outW - 1) % strideX); @@ -1738,6 +1821,16 @@ void TFImporter::parseConv2DBackpropInput(tensorflow::GraphDef& net, const tenso // one input only connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); + if (explicit_pads) // If we have explicit paddings, remove extra data + { + layerParams.set("begin", DictValue::arrayInt(begs, sizeof(begs) / sizeof(begs[0]))); + layerParams.set("end", DictValue::arrayInt(ends, sizeof(ends) / sizeof(ends[0]))); + + int id = dstNet.addLayer(layer.name(), "Slice", layerParams); + layer_id[layer.name()] = id; + + connect(layer_id, dstNet, parsePin(name), id, 0); + } } void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) @@ -2717,7 +2810,6 @@ void TFImporter::populateNet() addConstNodes(netBin, value_id, layers_to_ignore); addConstNodes(netTxt, value_id, layers_to_ignore); - for (int li = 0; li < layersSize; li++) { const tensorflow::NodeDef& layer = net.node(li); diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 4ba4f29322d1..4f7840f9e4eb 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -203,6 +203,16 @@ TEST_P(Test_TensorFlow_layers, padding) runTensorFlowNet("keras_pad_concat"); } +TEST_P(Test_TensorFlow_layers, padding_asymmetric) +{ + runTensorFlowNet("conv2d_asymmetric_pads_nchw"); + runTensorFlowNet("conv2d_asymmetric_pads_nhwc"); + runTensorFlowNet("max_pool2d_asymmetric_pads_nchw"); + runTensorFlowNet("max_pool2d_asymmetric_pads_nhwc"); + runTensorFlowNet("conv2d_backprop_input_asymmetric_pads_nchw"); + runTensorFlowNet("conv2d_backprop_input_asymmetric_pads_nhwc"); +} + TEST_P(Test_TensorFlow_layers, padding_same) { // Reference output values are in range [0.0006, 2.798] From 4015a5486c5cd831923c6d8019233228c06b99d1 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 21 Jul 2021 04:06:31 +0000 Subject: [PATCH 075/376] cmake: process modules in the same CMake scope --- CMakeLists.txt | 4 +- cmake/OpenCVModule.cmake | 39 +++++++++++++++++-- modules/CMakeLists.txt | 28 ------------- modules/highgui/CMakeLists.txt | 3 +- modules/highgui/cmake/detect_gtk.cmake | 11 ------ modules/highgui/cmake/detect_win32ui.cmake | 2 - modules/highgui/cmake/init.cmake | 19 +++------ .../videoio/cmake/detect_android_camera.cmake | 2 - .../cmake/detect_android_mediandk.cmake | 2 - modules/videoio/cmake/detect_aravis.cmake | 4 +- .../videoio/cmake/detect_avfoundation.cmake | 2 - modules/videoio/cmake/detect_dc1394.cmake | 5 +-- modules/videoio/cmake/detect_dshow.cmake | 2 - modules/videoio/cmake/detect_ffmpeg.cmake | 7 ---- modules/videoio/cmake/detect_gphoto.cmake | 2 - modules/videoio/cmake/detect_gstreamer.cmake | 6 +-- modules/videoio/cmake/detect_ios.cmake | 2 - modules/videoio/cmake/detect_msdk.cmake | 2 - modules/videoio/cmake/detect_msmf.cmake | 3 -- modules/videoio/cmake/detect_openni2.cmake | 4 +- modules/videoio/cmake/detect_pvapi.cmake | 2 - modules/videoio/cmake/detect_realsense.cmake | 4 +- modules/videoio/cmake/detect_ueye.cmake | 2 - modules/videoio/cmake/detect_v4l.cmake | 2 - modules/videoio/cmake/detect_ximea.cmake | 2 - modules/videoio/cmake/detect_xine.cmake | 2 - modules/videoio/cmake/init.cmake | 19 +++------ 27 files changed, 56 insertions(+), 126 deletions(-) delete mode 100644 modules/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index f4fe0385d12e..48bce581f468 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -913,7 +913,7 @@ add_subdirectory(include) ocv_add_modules_compiler_options() # OpenCV modules -add_subdirectory(modules) +ocv_register_modules() # Generate targets for documentation add_subdirectory(doc) @@ -1243,7 +1243,7 @@ endif(WIN32) # ========================== GUI ========================== status("") -status(" GUI: ") +status(" GUI: " "${OPENCV_HIGHGUI_BUILTIN_BACKEND}") if(WITH_QT OR HAVE_QT) if(HAVE_QT5) diff --git a/cmake/OpenCVModule.cmake b/cmake/OpenCVModule.cmake index 7c48aad9c295..9981620f2560 100644 --- a/cmake/OpenCVModule.cmake +++ b/cmake/OpenCVModule.cmake @@ -254,7 +254,7 @@ function(_glob_locations out_paths out_names) list(LENGTH paths before) get_filename_component(path "${path}" ABSOLUTE) # Either module itself - if(NOT path STREQUAL CMAKE_CURRENT_SOURCE_DIR AND EXISTS "${path}/CMakeLists.txt") + if(NOT path STREQUAL "${OpenCV_SOURCE_DIR}/modules" AND EXISTS "${path}/CMakeLists.txt") get_filename_component(name "${path}" NAME) list(APPEND paths "${path}") list(APPEND names "${name}") @@ -296,7 +296,7 @@ macro(_add_modules_1 paths names) list(GET ${names} ${i} __name) #message(STATUS "First pass: ${__name} => ${__path}") include("${__path}/cmake/init.cmake" OPTIONAL) - add_subdirectory("${__path}" "${CMAKE_CURRENT_BINARY_DIR}/.firstpass/${__name}") + add_subdirectory("${__path}" "${OpenCV_BINARY_DIR}/modules/.firstpass/${__name}") endforeach() endif() endmacro() @@ -316,7 +316,7 @@ macro(_add_modules_2) endif() string(REGEX REPLACE "^opencv_" "" name "${m}") #message(STATUS "Second pass: ${name} => ${OPENCV_MODULE_${m}_LOCATION}") - add_subdirectory("${OPENCV_MODULE_${m}_LOCATION}" "${CMAKE_CURRENT_BINARY_DIR}/${name}") + add_subdirectory("${OPENCV_MODULE_${m}_LOCATION}" "${OpenCV_BINARY_DIR}/modules/${name}") endif() ocv_cmake_hook(POST_MODULES_CREATE_${the_module}) endforeach() @@ -369,7 +369,6 @@ macro(ocv_glob_modules main_root) __ocv_resolve_dependencies() # create modules - set(OPENCV_INITIAL_PASS OFF PARENT_SCOPE) set(OPENCV_INITIAL_PASS OFF) ocv_cmake_hook(PRE_MODULES_CREATE) _add_modules_2(${OPENCV_MODULES_BUILD}) @@ -377,6 +376,37 @@ macro(ocv_glob_modules main_root) endmacro() +# called by root CMakeLists.txt +macro(ocv_register_modules) + if(NOT OPENCV_MODULES_PATH) + set(OPENCV_MODULES_PATH "${OpenCV_SOURCE_DIR}/modules") + endif() + + ocv_glob_modules(${OPENCV_MODULES_PATH} ${OPENCV_EXTRA_MODULES_PATH}) + + # build lists of modules to be documented + set(OPENCV_MODULES_MAIN "") + set(OPENCV_MODULES_EXTRA "") + + foreach(mod ${OPENCV_MODULES_BUILD} ${OPENCV_MODULES_DISABLED_USER} ${OPENCV_MODULES_DISABLED_AUTO} ${OPENCV_MODULES_DISABLED_FORCE}) + string(REGEX REPLACE "^opencv_" "" mod "${mod}") + if("${OPENCV_MODULE_opencv_${mod}_LOCATION}" STREQUAL "${OpenCV_SOURCE_DIR}/modules/${mod}") + list(APPEND OPENCV_MODULES_MAIN ${mod}) + else() + list(APPEND OPENCV_MODULES_EXTRA ${mod}) + endif() + endforeach() + ocv_list_sort(OPENCV_MODULES_MAIN) + ocv_list_sort(OPENCV_MODULES_EXTRA) + set(FIXED_ORDER_MODULES core imgproc imgcodecs videoio highgui video calib3d features2d objdetect dnn ml flann photo stitching) + list(REMOVE_ITEM OPENCV_MODULES_MAIN ${FIXED_ORDER_MODULES}) + set(OPENCV_MODULES_MAIN ${FIXED_ORDER_MODULES} ${OPENCV_MODULES_MAIN}) + + set(OPENCV_MODULES_MAIN ${OPENCV_MODULES_MAIN} CACHE INTERNAL "List of main modules" FORCE) + set(OPENCV_MODULES_EXTRA ${OPENCV_MODULES_EXTRA} CACHE INTERNAL "List of extra modules" FORCE) +endmacro() + + # disables OpenCV module with missing dependencies function(__ocv_module_turn_off the_module) list(REMOVE_ITEM OPENCV_MODULES_DISABLED_AUTO "${the_module}") @@ -877,6 +907,7 @@ macro(ocv_create_module) endmacro() macro(_ocv_create_module) + add_definitions(-D__OPENCV_BUILD=1) ocv_compiler_optimization_process_sources(OPENCV_MODULE_${the_module}_SOURCES OPENCV_MODULE_${the_module}_DEPS_EXT ${the_module}) set(__module_headers ${OPENCV_MODULE_${the_module}_HEADERS}) diff --git a/modules/CMakeLists.txt b/modules/CMakeLists.txt deleted file mode 100644 index 6a8004036b28..000000000000 --- a/modules/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -add_definitions(-D__OPENCV_BUILD=1) - -if(NOT OPENCV_MODULES_PATH) - set(OPENCV_MODULES_PATH "${CMAKE_CURRENT_SOURCE_DIR}") -endif() - -ocv_glob_modules(${OPENCV_MODULES_PATH} ${OPENCV_EXTRA_MODULES_PATH}) - -# build lists of modules to be documented -set(OPENCV_MODULES_MAIN "") -set(OPENCV_MODULES_EXTRA "") - -foreach(mod ${OPENCV_MODULES_BUILD} ${OPENCV_MODULES_DISABLED_USER} ${OPENCV_MODULES_DISABLED_AUTO} ${OPENCV_MODULES_DISABLED_FORCE}) - string(REGEX REPLACE "^opencv_" "" mod "${mod}") - if("${OPENCV_MODULE_opencv_${mod}_LOCATION}" STREQUAL "${OpenCV_SOURCE_DIR}/modules/${mod}") - list(APPEND OPENCV_MODULES_MAIN ${mod}) - else() - list(APPEND OPENCV_MODULES_EXTRA ${mod}) - endif() -endforeach() -ocv_list_sort(OPENCV_MODULES_MAIN) -ocv_list_sort(OPENCV_MODULES_EXTRA) -set(FIXED_ORDER_MODULES core imgproc imgcodecs videoio highgui video calib3d features2d objdetect dnn ml flann photo stitching) -list(REMOVE_ITEM OPENCV_MODULES_MAIN ${FIXED_ORDER_MODULES}) -set(OPENCV_MODULES_MAIN ${FIXED_ORDER_MODULES} ${OPENCV_MODULES_MAIN}) - -set(OPENCV_MODULES_MAIN ${OPENCV_MODULES_MAIN} CACHE INTERNAL "List of main modules" FORCE) -set(OPENCV_MODULES_EXTRA ${OPENCV_MODULES_EXTRA} CACHE INTERNAL "List of extra modules" FORCE) diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index bc31b84c74e1..2b630bfed80d 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -218,7 +218,8 @@ endif() if(NOT OPENCV_HIGHGUI_BUILTIN_BACKEND) set(OPENCV_HIGHGUI_BUILTIN_BACKEND "NONE") endif() -message(STATUS "highgui: using builtin backend: ${OPENCV_HIGHGUI_BUILTIN_BACKEND}") # FIXIT: propagate to root CMake +message(STATUS "highgui: using builtin backend: ${OPENCV_HIGHGUI_BUILTIN_BACKEND}") +set(OPENCV_HIGHGUI_BUILTIN_BACKEND "${OPENCV_HIGHGUI_BUILTIN_BACKEND}" PARENT_SCOPE) # informational if(TRUE) # these variables are set by 'ocv_append_build_options(HIGHGUI ...)' diff --git a/modules/highgui/cmake/detect_gtk.cmake b/modules/highgui/cmake/detect_gtk.cmake index cdc054fad0c5..c58246ac5414 100644 --- a/modules/highgui/cmake/detect_gtk.cmake +++ b/modules/highgui/cmake/detect_gtk.cmake @@ -6,8 +6,6 @@ if(WITH_GTK) if(HAVE_GTK3) ocv_add_external_target(gtk3 "${GTK3_INCLUDE_DIRS}" "${GTK3_LIBRARIES}" "HAVE_GTK3;HAVE_GTK") set(HAVE_GTK TRUE) - set(HAVE_GTK3 ${HAVE_GTK3} PARENT_SCOPE) - set(GTK3_VERSION "${GTK3_VERSION}" PARENT_SCOPE) # informational endif() endif() if((PROJECT_NAME STREQUAL "OpenCV" AND HIGHGUI_ENABLE_PLUGINS) OR NOT HAVE_GTK3) @@ -19,8 +17,6 @@ if(WITH_GTK) else() ocv_add_external_target(gtk2 "${GTK2_INCLUDE_DIRS}" "${GTK2_LIBRARIES}" "HAVE_GTK2;HAVE_GTK") set(HAVE_GTK TRUE) - set(HAVE_GTK2 ${HAVE_GTK2} PARENT_SCOPE) - set(GTK2_VERSION "${GTK2_VERSION}" PARENT_SCOPE) # informational endif() endif() endif() @@ -29,15 +25,11 @@ if(WITH_GTK) message(FATAL_ERROR "gthread not found. This library is required when building with GTK support") else() ocv_add_external_target(gthread "${GTHREAD_INCLUDE_DIRS}" "${GTHREAD_LIBRARIES}" "HAVE_GTHREAD") - set(HAVE_GTHREAD "${HAVE_GTHREAD}" PARENT_SCOPE) # informational - set(GTHREAD_VERSION "${GTHREAD_VERSION}" PARENT_SCOPE) # informational endif() if((WITH_OPENGL OR HAVE_OPENGL) AND HAVE_GTK2) ocv_check_modules(GTKGLEXT gtkglext-1.0) if(HAVE_GTKGLEXT) ocv_add_external_target(gtkglext "${GTKGLEXT_INCLUDE_DIRS}" "${GTKGLEXT_LIBRARIES}" "HAVE_GTKGLEXT") - set(HAVE_GTKGLEXT "${HAVE_GTKGLEXT}" PARENT_SCOPE) # informational - set(GTKGLEXT_VERSION "${GTKGLEXT_VERSION}" PARENT_SCOPE) # informational endif() endif() elseif(HAVE_GTK) @@ -48,9 +40,6 @@ if(WITH_OPENGL AND HAVE_GTKGLEXT) find_package(OpenGL QUIET) if(OPENGL_FOUND) set(HAVE_OPENGL TRUE) - #set(HAVE_OPENGL ${HAVE_OPENGL} PARENT_SCOPE) ocv_add_external_target(gtk_opengl "${OPENGL_INCLUDE_DIRS}" "${OPENGL_LIBRARIES}" "HAVE_OPENGL") endif() endif() - -set(HAVE_GTK ${HAVE_GTK} PARENT_SCOPE) diff --git a/modules/highgui/cmake/detect_win32ui.cmake b/modules/highgui/cmake/detect_win32ui.cmake index 1d2fdc5d4654..c5e358ffa710 100644 --- a/modules/highgui/cmake/detect_win32ui.cmake +++ b/modules/highgui/cmake/detect_win32ui.cmake @@ -13,5 +13,3 @@ if(WITH_WIN32UI) ocv_add_external_target(win32ui "" "${__libs}" "HAVE_WIN32UI") endif() endif() - -set(HAVE_WIN32UI "${HAVE_WIN32UI}" PARENT_SCOPE) # informational diff --git a/modules/highgui/cmake/init.cmake b/modules/highgui/cmake/init.cmake index 1626d254daf9..a302c4d534a6 100644 --- a/modules/highgui/cmake/init.cmake +++ b/modules/highgui/cmake/init.cmake @@ -27,20 +27,11 @@ endif() include(FindPkgConfig) -# FIXIT: stop using PARENT_SCOPE in dependencies -if(PROJECT_NAME STREQUAL "OpenCV") - macro(add_backend backend_id cond_var) - if(${cond_var}) - include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") - endif() - endmacro() -else() - function(add_backend backend_id cond_var) - if(${cond_var}) - include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") - endif() - endfunction() -endif() +macro(add_backend backend_id cond_var) + if(${cond_var}) + include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") + endif() +endmacro() add_backend("gtk" WITH_GTK) add_backend("win32ui" WITH_WIN32UI) diff --git a/modules/videoio/cmake/detect_android_camera.cmake b/modules/videoio/cmake/detect_android_camera.cmake index ded4c91ccf17..a465751334fd 100644 --- a/modules/videoio/cmake/detect_android_camera.cmake +++ b/modules/videoio/cmake/detect_android_camera.cmake @@ -4,5 +4,3 @@ if(ANDROID AND ANDROID_NATIVE_API_LEVEL GREATER 23) set(libs "-landroid -llog -lcamera2ndk") ocv_add_external_target(android_native_camera "" "${libs}" "HAVE_ANDROID_NATIVE_CAMERA") endif() - -set(HAVE_ANDROID_NATIVE_CAMERA ${HAVE_ANDROID_NATIVE_CAMERA} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_android_mediandk.cmake b/modules/videoio/cmake/detect_android_mediandk.cmake index edfb4bbbc5c3..cee64ab54991 100644 --- a/modules/videoio/cmake/detect_android_mediandk.cmake +++ b/modules/videoio/cmake/detect_android_mediandk.cmake @@ -4,5 +4,3 @@ if(ANDROID AND ANDROID_NATIVE_API_LEVEL GREATER 20) set(libs "-landroid -llog -lmediandk") ocv_add_external_target(android_mediandk "" "${libs}" "HAVE_ANDROID_MEDIANDK") endif() - -set(HAVE_ANDROID_MEDIANDK ${HAVE_ANDROID_MEDIANDK} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_aravis.cmake b/modules/videoio/cmake/detect_aravis.cmake index 79d6a217db2e..e7b382899343 100644 --- a/modules/videoio/cmake/detect_aravis.cmake +++ b/modules/videoio/cmake/detect_aravis.cmake @@ -21,7 +21,7 @@ if(NOT HAVE_ARAVIS_API) string(REGEX REPLACE ".*ARAVIS_MAJOR_VERSION[^0-9]+([0-9]+).*" "\\1" ver_major "${ver_strings}") string(REGEX REPLACE ".*ARAVIS_MINOR_VERSION[^0-9]+([0-9]+).*" "\\1" ver_minor "${ver_strings}") string(REGEX REPLACE ".*ARAVIS_MICRO_VERSION[^0-9]+([0-9]+).*" "\\1" ver_micro "${ver_strings}") - set(ARAVIS_VERSION "${ver_major}.${ver_minor}.${ver_micro}" PARENT_SCOPE) # informational + set(ARAVIS_VERSION "${ver_major}.${ver_minor}.${ver_micro}") # informational set(ARAVIS_INCLUDE_DIRS "${ARAVIS_INCLUDE}") set(ARAVIS_LIBRARIES "${ARAVIS_LIBRARY}") endif() @@ -30,5 +30,3 @@ endif() if(HAVE_ARAVIS_API) ocv_add_external_target(aravis "${ARAVIS_INCLUDE_DIRS}" "${ARAVIS_LIBRARIES}" "HAVE_ARAVIS_API") endif() - -set(HAVE_ARAVIS_API ${HAVE_ARAVIS_API} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_avfoundation.cmake b/modules/videoio/cmake/detect_avfoundation.cmake index a341f587a199..2da4fabfab44 100644 --- a/modules/videoio/cmake/detect_avfoundation.cmake +++ b/modules/videoio/cmake/detect_avfoundation.cmake @@ -14,5 +14,3 @@ if(APPLE) endif() ocv_add_external_target(avfoundation "" "${libs}" "HAVE_AVFOUNDATION") endif() - -set(HAVE_AVFOUNDATION ${HAVE_AVFOUNDATION} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_dc1394.cmake b/modules/videoio/cmake/detect_dc1394.cmake index 51ab2dd80eb4..8bcee4bf7098 100644 --- a/modules/videoio/cmake/detect_dc1394.cmake +++ b/modules/videoio/cmake/detect_dc1394.cmake @@ -2,7 +2,6 @@ if(NOT HAVE_DC1394_2 AND PKG_CONFIG_FOUND) ocv_check_modules(DC1394_2 libdc1394-2) if(DC1394_2_FOUND) - set(DC1394_2_VERSION "${DC1394_2_VERSION}" PARENT_SCOPE) # informational set(HAVE_DC1394_2 TRUE) endif() endif() @@ -20,12 +19,10 @@ if(NOT HAVE_DC1394_2) set(HAVE_DC1394_2 TRUE) set(DC1394_2_INCLUDE_DIRS "${DC1394_INCLUDE}") set(DC1394_2_LIBRARIES "${DC1394_LIBRARY}") - set(DC1394_2_VERSION "unknown" PARENT_SCOPE) # informational + set(DC1394_2_VERSION "unknown") # informational endif() endif() if(HAVE_DC1394_2) ocv_add_external_target(dc1394_2 "${DC1394_2_INCLUDE_DIRS}" "${DC1394_2_LIBRARIES}" "HAVE_DC1394_2") endif() - -set(HAVE_DC1394_2 ${HAVE_DC1394_2} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_dshow.cmake b/modules/videoio/cmake/detect_dshow.cmake index 3f41b3fd34e2..928134c08c54 100644 --- a/modules/videoio/cmake/detect_dshow.cmake +++ b/modules/videoio/cmake/detect_dshow.cmake @@ -10,5 +10,3 @@ endif() if(HAVE_DSHOW) ocv_add_external_target(dshow "" "" "HAVE_DSHOW") endif() - -set(HAVE_DSHOW ${HAVE_DSHOW} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_ffmpeg.cmake b/modules/videoio/cmake/detect_ffmpeg.cmake index 58de4b9515ac..c33eaf221b8a 100644 --- a/modules/videoio/cmake/detect_ffmpeg.cmake +++ b/modules/videoio/cmake/detect_ffmpeg.cmake @@ -14,11 +14,6 @@ if(NOT HAVE_FFMPEG AND WIN32 AND NOT ARM AND NOT OPENCV_FFMPEG_SKIP_DOWNLOAD) download_win_ffmpeg(FFMPEG_CMAKE_SCRIPT) if(FFMPEG_CMAKE_SCRIPT) include("${FFMPEG_CMAKE_SCRIPT}") - set(FFMPEG_libavcodec_VERSION ${FFMPEG_libavcodec_VERSION} PARENT_SCOPE) # info - set(FFMPEG_libavformat_VERSION ${FFMPEG_libavformat_VERSION} PARENT_SCOPE) # info - set(FFMPEG_libavutil_VERSION ${FFMPEG_libavutil_VERSION} PARENT_SCOPE) # info - set(FFMPEG_libswscale_VERSION ${FFMPEG_libswscale_VERSION} PARENT_SCOPE) # info - set(FFMPEG_libavresample_VERSION ${FFMPEG_libavresample_VERSION} PARENT_SCOPE) # info set(HAVE_FFMPEG TRUE) set(HAVE_FFMPEG_WRAPPER TRUE) endif() @@ -132,5 +127,3 @@ elseif(HAVE_FFMPEG) ocv_add_external_target(ffmpeg.plugin_deps "${__plugin_include_dirs}" "${__plugin_include_libs}" "${__plugin_defines}") endif() endif() - -set(HAVE_FFMPEG ${HAVE_FFMPEG} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_gphoto.cmake b/modules/videoio/cmake/detect_gphoto.cmake index 0d6f1212eb37..2cb23c00335f 100644 --- a/modules/videoio/cmake/detect_gphoto.cmake +++ b/modules/videoio/cmake/detect_gphoto.cmake @@ -9,5 +9,3 @@ endif() if(HAVE_GPHOTO2) ocv_add_external_target(gphoto2 "${GPHOTO2_INCLUDE_DIRS}" "${GPHOTO2_LIBRARIES}" "HAVE_GPHOTO2") endif() - -set(HAVE_GPHOTO2 ${HAVE_GPHOTO2} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_gstreamer.cmake b/modules/videoio/cmake/detect_gstreamer.cmake index 219878616175..47ea7a0b3071 100644 --- a/modules/videoio/cmake/detect_gstreamer.cmake +++ b/modules/videoio/cmake/detect_gstreamer.cmake @@ -69,7 +69,7 @@ if(NOT HAVE_GSTREAMER AND WIN32) string(REGEX REPLACE ".*GST_VERSION_MAJOR[^0-9]+([0-9]+).*" "\\1" ver_major "${ver_strings}") string(REGEX REPLACE ".*GST_VERSION_MINOR[^0-9]+([0-9]+).*" "\\1" ver_minor "${ver_strings}") string(REGEX REPLACE ".*GST_VERSION_MICRO[^0-9]+([0-9]+).*" "\\1" ver_micro "${ver_strings}") - set(GSTREAMER_VERSION "${ver_major}.${ver_minor}.${ver_micro}" PARENT_SCOPE) # informational + set(GSTREAMER_VERSION "${ver_major}.${ver_minor}.${ver_micro}") # informational set(HAVE_GSTREAMER TRUE) set(GSTREAMER_LIBRARIES ${GSTREAMER_gstreamer_LIBRARY} @@ -95,7 +95,7 @@ if(NOT HAVE_GSTREAMER AND PKG_CONFIG_FOUND) ocv_check_modules(GSTREAMER_video gstreamer-video-1.0) if(GSTREAMER_base_FOUND AND GSTREAMER_app_FOUND AND GSTREAMER_riff_FOUND AND GSTREAMER_pbutils_FOUND AND GSTREAMER_video_FOUND) set(HAVE_GSTREAMER TRUE) - set(GSTREAMER_VERSION ${GSTREAMER_base_VERSION} PARENT_SCOPE) # informational + set(GSTREAMER_VERSION ${GSTREAMER_base_VERSION}) # informational set(GSTREAMER_LIBRARIES ${GSTREAMER_base_LIBRARIES} ${GSTREAMER_app_LIBRARIES} ${GSTREAMER_riff_LIBRARIES} ${GSTREAMER_pbutils_LIBRARIES} ${GSTREAMER_video_LIBRARIES}) set(GSTREAMER_INCLUDE_DIRS ${GSTREAMER_base_INCLUDE_DIRS} ${GSTREAMER_app_INCLUDE_DIRS} ${GSTREAMER_riff_INCLUDE_DIRS} ${GSTREAMER_pbutils_INCLUDE_DIRS} ${GSTREAMER_video_INCLUDE_DIRS}) endif() @@ -104,5 +104,3 @@ endif() if(HAVE_GSTREAMER) ocv_add_external_target(gstreamer "${GSTREAMER_INCLUDE_DIRS}" "${GSTREAMER_LIBRARIES}" "HAVE_GSTREAMER") endif() - -set(HAVE_GSTREAMER ${HAVE_GSTREAMER} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_ios.cmake b/modules/videoio/cmake/detect_ios.cmake index c75426060b0b..8d48dd6f3bea 100644 --- a/modules/videoio/cmake/detect_ios.cmake +++ b/modules/videoio/cmake/detect_ios.cmake @@ -11,5 +11,3 @@ if(APPLE AND IOS) "-framework UIKit") ocv_add_external_target(cap_ios "" "${libs}" "HAVE_CAP_IOS") endif() - -set(HAVE_CAP_IOS ${HAVE_CAP_IOS} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_msdk.cmake b/modules/videoio/cmake/detect_msdk.cmake index d035c3f5cc11..83701425e1f8 100644 --- a/modules/videoio/cmake/detect_msdk.cmake +++ b/modules/videoio/cmake/detect_msdk.cmake @@ -70,5 +70,3 @@ if(HAVE_MFX) list(APPEND MFX_DEFS "HAVE_MFX") ocv_add_external_target(mediasdk "${MFX_INCLUDE_DIRS}" "${MFX_LIBRARIES}" "${MFX_DEFS}") endif() - -set(HAVE_MFX ${HAVE_MFX} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_msmf.cmake b/modules/videoio/cmake/detect_msmf.cmake index a1c91dab670a..aebc226bcfc9 100644 --- a/modules/videoio/cmake/detect_msmf.cmake +++ b/modules/videoio/cmake/detect_msmf.cmake @@ -20,6 +20,3 @@ if(HAVE_MSMF) endif() ocv_add_external_target(msmf "" "" "${defs}") endif() - -set(HAVE_MSMF ${HAVE_MSMF} PARENT_SCOPE) -set(HAVE_MSMF_DXVA ${HAVE_MSMF_DXVA} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_openni2.cmake b/modules/videoio/cmake/detect_openni2.cmake index 76c31454da81..54a5c62beddc 100644 --- a/modules/videoio/cmake/detect_openni2.cmake +++ b/modules/videoio/cmake/detect_openni2.cmake @@ -42,8 +42,6 @@ if(HAVE_OPENNI2) string(REGEX REPLACE ".*ONI_VERSION_MAJOR[^0-9]+([0-9]+).*" "\\1" ver_major "${ver_strings}") string(REGEX REPLACE ".*ONI_VERSION_MINOR[^0-9]+([0-9]+).*" "\\1" ver_minor "${ver_strings}") string(REGEX REPLACE ".*ONI_VERSION_MAINTENANCE[^0-9]+([0-9]+).*" "\\1" ver_maint "${ver_strings}") - set(OPENNI2_VERSION "${ver_major}.${ver_minor}.${ver_maint}" PARENT_SCOPE) # informational + set(OPENNI2_VERSION "${ver_major}.${ver_minor}.${ver_maint}") # informational ocv_add_external_target(openni2 "${OPENNI2_INCLUDE_DIRS}" "${OPENNI2_LIBRARIES}" "HAVE_OPENNI2") endif() - -set(HAVE_OPENNI2 ${HAVE_OPENNI2} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_pvapi.cmake b/modules/videoio/cmake/detect_pvapi.cmake index a0f4673fdc1d..f2c6d4bceaa5 100644 --- a/modules/videoio/cmake/detect_pvapi.cmake +++ b/modules/videoio/cmake/detect_pvapi.cmake @@ -19,5 +19,3 @@ endif() if(HAVE_PVAPI) ocv_add_external_target(pvapi "${PVAPI_INCLUDE}" "${PVAPI_LIBRARY}" "HAVE_PVAPI") endif() - -set(HAVE_PVAPI ${HAVE_PVAPI} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_realsense.cmake b/modules/videoio/cmake/detect_realsense.cmake index 32e5e02c9e7b..065f5488301f 100644 --- a/modules/videoio/cmake/detect_realsense.cmake +++ b/modules/videoio/cmake/detect_realsense.cmake @@ -4,7 +4,7 @@ if(NOT HAVE_LIBREALSENSE) find_package(realsense2 QUIET) if(realsense2_FOUND) set(HAVE_LIBREALSENSE TRUE) - set(LIBREALSENSE_VERSION "${realsense2_VERSION}" PARENT_SCOPE) # informational + set(LIBREALSENSE_VERSION "${realsense2_VERSION}") # informational ocv_add_external_target(librealsense "" "${realsense2_LIBRARY}" "HAVE_LIBREALSENSE") endif() endif() @@ -20,7 +20,7 @@ if(NOT HAVE_LIBREALSENSE) string(REGEX REPLACE ".*RS2_API_MAJOR_VERSION[^0-9]+([0-9]+).*" "\\1" ver_major "${ver_strings}") string(REGEX REPLACE ".*RS2_API_MINOR_VERSION[^0-9]+([0-9]+).*" "\\1" ver_minor "${ver_strings}") string(REGEX REPLACE ".*RS2_API_PATCH_VERSION[^0-9]+([0-9]+).*" "\\1" ver_patch "${ver_strings}") - set(LIBREALSENSE_VERSION "${ver_major}.${ver_minor}.${ver_patch}" PARENT_SCOPE) # informational + set(LIBREALSENSE_VERSION "${ver_major}.${ver_minor}.${ver_patch}") # informational ocv_add_external_target(librealsense "${LIBREALSENSE_INCLUDE_DIR}" "${LIBREALSENSE_LIBRARIES}" "HAVE_LIBREALSENSE") endif() endif() diff --git a/modules/videoio/cmake/detect_ueye.cmake b/modules/videoio/cmake/detect_ueye.cmake index 495e9c245023..9428f9e59647 100644 --- a/modules/videoio/cmake/detect_ueye.cmake +++ b/modules/videoio/cmake/detect_ueye.cmake @@ -21,5 +21,3 @@ unset(_WIN_LIB_SUFFIX) if(HAVE_UEYE) ocv_add_external_target(ueye "${UEYE_INCLUDE}" "${UEYE_LIBRARY}" "HAVE_UEYE") endif() - -set(HAVE_UEYE ${HAVE_UEYE} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_v4l.cmake b/modules/videoio/cmake/detect_v4l.cmake index 05b73b003c4f..e413dae9ca4e 100644 --- a/modules/videoio/cmake/detect_v4l.cmake +++ b/modules/videoio/cmake/detect_v4l.cmake @@ -15,5 +15,3 @@ if(NOT HAVE_V4L) ocv_add_external_target(v4l "" "" "${defs}") endif() endif() - -set(HAVE_V4L ${HAVE_V4L} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_ximea.cmake b/modules/videoio/cmake/detect_ximea.cmake index 9cf295e3529b..7521e619b036 100644 --- a/modules/videoio/cmake/detect_ximea.cmake +++ b/modules/videoio/cmake/detect_ximea.cmake @@ -28,5 +28,3 @@ endif() if(HAVE_XIMEA) ocv_add_external_target(ximea "${XIMEA_INCLUDE}" "${XIMEA_LIBRARY}" "HAVE_XIMEA") endif() - -set(HAVE_XIMEA ${HAVE_XIMEA} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_xine.cmake b/modules/videoio/cmake/detect_xine.cmake index 3e1f3010a431..0a6f64235349 100644 --- a/modules/videoio/cmake/detect_xine.cmake +++ b/modules/videoio/cmake/detect_xine.cmake @@ -5,5 +5,3 @@ endif() if(HAVE_XINE) ocv_add_external_target(xine "${XINE_INCLUDE_DIRS}" "${XINE_LIBRARIES}" "HAVE_XINE") endif() - -set(HAVE_XINE ${HAVE_XINE} PARENT_SCOPE) diff --git a/modules/videoio/cmake/init.cmake b/modules/videoio/cmake/init.cmake index 310df2d249e3..68838790b8a2 100644 --- a/modules/videoio/cmake/init.cmake +++ b/modules/videoio/cmake/init.cmake @@ -1,19 +1,10 @@ include(FindPkgConfig) -# FIXIT: stop using PARENT_SCOPE in dependencies -if(PROJECT_NAME STREQUAL "OpenCV") - macro(add_backend backend_id cond_var) - if(${cond_var}) - include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") - endif() - endmacro() -else() - function(add_backend backend_id cond_var) - if(${cond_var}) - include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") - endif() - endfunction() -endif() +macro(add_backend backend_id cond_var) + if(${cond_var}) + include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") + endif() +endmacro() add_backend("ffmpeg" WITH_FFMPEG) add_backend("gstreamer" WITH_GSTREAMER) From 803ff8ebb9ed7960c28e28c6e11e77a783f77d0a Mon Sep 17 00:00:00 2001 From: Shreyas Taware Date: Wed, 7 Jul 2021 14:40:08 +0530 Subject: [PATCH 076/376] Update py_canny.markdown Fixed a word that was previously written as third argument but it is instead the fourth argument of cv.Canny() function --- doc/py_tutorials/py_imgproc/py_canny/py_canny.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/py_tutorials/py_imgproc/py_canny/py_canny.markdown b/doc/py_tutorials/py_imgproc/py_canny/py_canny.markdown index cbc2a72eecc7..d36e5784ebc8 100644 --- a/doc/py_tutorials/py_imgproc/py_canny/py_canny.markdown +++ b/doc/py_tutorials/py_imgproc/py_canny/py_canny.markdown @@ -74,7 +74,7 @@ Canny Edge Detection in OpenCV OpenCV puts all the above in single function, **cv.Canny()**. We will see how to use it. First argument is our input image. Second and third arguments are our minVal and maxVal respectively. -Third argument is aperture_size. It is the size of Sobel kernel used for find image gradients. By +Fourth argument is aperture_size. It is the size of Sobel kernel used for find image gradients. By default it is 3. Last argument is L2gradient which specifies the equation for finding gradient magnitude. If it is True, it uses the equation mentioned above which is more accurate, otherwise it uses this function: \f$Edge\_Gradient \; (G) = |G_x| + |G_y|\f$. By default, it is False. From a76274b549632464987805a29472a9520fb45389 Mon Sep 17 00:00:00 2001 From: Scott Noyes Date: Thu, 22 Jul 2021 14:58:20 -0500 Subject: [PATCH 077/376] minor grammar edits --- .../mat_the_basic_image_container.markdown | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown b/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown index f6a1a0a4fb2c..aafa9687d930 100644 --- a/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown +++ b/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown @@ -84,8 +84,8 @@ a new header with the new boundaries: Mat D (A, Rect(10, 10, 100, 100) ); // using a rectangle Mat E = A(Range::all(), Range(1,3)); // using row and column boundaries @endcode -Now you may ask -- if the matrix itself may belong to multiple *Mat* objects who takes responsibility -for cleaning it up when it's no longer needed. The short answer is: the last object that used it. +Now you may ask -- if the matrix itself may belong to multiple *Mat* objects, who takes responsibility +for cleaning it up when it's no longer needed? The short answer is: the last object that used it. This is handled by using a reference counting mechanism. Whenever somebody copies a header of a *Mat* object, a counter is increased for the matrix. Whenever a header is cleaned, this counter is decreased. When the counter reaches zero the matrix is freed. Sometimes you will want to copy @@ -95,12 +95,12 @@ Mat F = A.clone(); Mat G; A.copyTo(G); @endcode -Now modifying *F* or *G* will not affect the matrix pointed by the *A*'s header. What you need to +Now modifying *F* or *G* will not affect the matrix pointed to by the *A*'s header. What you need to remember from all this is that: - Output image allocation for OpenCV functions is automatic (unless specified otherwise). - You do not need to think about memory management with OpenCV's C++ interface. -- The assignment operator and the copy constructor only copies the header. +- The assignment operator and the copy constructor only copy the header. - The underlying matrix of an image may be copied using the @ref cv::Mat::clone() and @ref cv::Mat::copyTo() functions. @@ -115,10 +115,10 @@ of these allows us to create many shades of gray. For *colorful* ways we have a lot more methods to choose from. Each of them breaks it down to three or four basic components and we can use the combination of these to create the others. The most popular one is RGB, mainly because this is also how our eye builds up colors. Its base colors are -red, green and blue. To code the transparency of a color sometimes a fourth element: alpha (A) is +red, green and blue. To code the transparency of a color sometimes a fourth element, alpha (A), is added. -There are, however, many other color systems each with their own advantages: +There are, however, many other color systems, each with their own advantages: - RGB is the most common as our eyes use something similar, however keep in mind that OpenCV standard display system composes colors using the BGR color space (red and blue channels are swapped places). @@ -132,11 +132,11 @@ There are, however, many other color systems each with their own advantages: Each of the building components has its own valid domains. This leads to the data type used. How we store a component defines the control we have over its domain. The smallest data type possible is *char*, which means one byte or 8 bits. This may be unsigned (so can store values from 0 to 255) or -signed (values from -127 to +127). Although in case of three components this already gives 16 -million possible colors to represent (like in case of RGB) we may acquire an even finer control by +signed (values from -127 to +127). Although this width, in the case of three components (like RGB), already gives 16 +million possible colors to represent, we may acquire an even finer control by using the float (4 byte = 32 bit) or double (8 byte = 64 bit) data types for each component. Nevertheless, remember that increasing the size of a component also increases the size of the whole -picture in the memory. +picture in memory. Creating a Mat object explicitly ---------------------------------- From acc576658ad628d46fd4e79c68c1419d438ce716 Mon Sep 17 00:00:00 2001 From: ZhangYin Date: Fri, 23 Jul 2021 22:08:43 +0800 Subject: [PATCH 078/376] Merge pull request #20412 from joy2myself:rvv-0.10 bug fixes for universal intrinsics of RISC-V back-end * Align universal intrinsic comparator behaviour with other platforms Set all bits to one for return value of int and fp comparators. * fix v_pack_triplets, v_pack_store and v_pack_u_store * Remove redundant CV_DECL_ALIGNED statements Co-authored-by: Alexander Smorkalov --- .../include/opencv2/core/hal/intrin_rvv.hpp | 299 +++++++++--------- 1 file changed, 151 insertions(+), 148 deletions(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp index 4a3455b07385..51433cdbae72 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp @@ -737,7 +737,7 @@ OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float64x2, vfloat64m1_t, double, 1, 2, 64, f6 inline v_int8x16 v_load_halves(const schar* ptr0, const schar* ptr1) { - schar CV_DECL_ALIGNED(32) elems[16] = + schar elems[16] = { ptr0[0], ptr0[1], ptr0[2], ptr0[3], ptr0[4], ptr0[5], ptr0[6], ptr0[7], ptr1[0], ptr1[1], ptr1[2], ptr1[3], ptr1[4], ptr1[5], ptr1[6], ptr1[7] @@ -748,7 +748,7 @@ inline v_uint8x16 v_load_halves(const uchar* ptr0, const uchar* ptr1) { return v inline v_int16x8 v_load_halves(const short* ptr0, const short* ptr1) { - short CV_DECL_ALIGNED(32) elems[8] = + short elems[8] = { ptr0[0], ptr0[1], ptr0[2], ptr0[3], ptr1[0], ptr1[1], ptr1[2], ptr1[3] }; @@ -758,7 +758,7 @@ inline v_uint16x8 v_load_halves(const ushort* ptr0, const ushort* ptr1) { return inline v_int32x4 v_load_halves(const int* ptr0, const int* ptr1) { - int CV_DECL_ALIGNED(32) elems[4] = + int elems[4] = { ptr0[0], ptr0[1], ptr1[0], ptr1[1] }; @@ -766,7 +766,7 @@ inline v_int32x4 v_load_halves(const int* ptr0, const int* ptr1) } inline v_float32x4 v_load_halves(const float* ptr0, const float* ptr1) { - float CV_DECL_ALIGNED(32) elems[4] = + float elems[4] = { ptr0[0], ptr0[1], ptr1[0], ptr1[1] }; @@ -776,7 +776,7 @@ inline v_uint32x4 v_load_halves(const unsigned* ptr0, const unsigned* ptr1) { re inline v_int64x2 v_load_halves(const int64* ptr0, const int64* ptr1) { - int64 CV_DECL_ALIGNED(32) elems[2] = + int64 elems[2] = { ptr0[0], ptr1[0] }; @@ -787,7 +787,7 @@ inline v_uint64x2 v_load_halves(const uint64* ptr0, const uint64* ptr1) { return #if CV_SIMD128_64F inline v_float64x2 v_load_halves(const double* ptr0, const double* ptr1) { - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { ptr0[0], ptr1[0] }; @@ -800,7 +800,7 @@ inline v_float64x2 v_load_halves(const double* ptr0, const double* ptr1) inline v_int8x16 v_lut(const schar* tab, const int* idx) { - schar CV_DECL_ALIGNED(32) elems[16] = + schar elems[16] = { tab[idx[ 0]], tab[idx[ 1]], @@ -823,7 +823,7 @@ inline v_int8x16 v_lut(const schar* tab, const int* idx) } inline v_int8x16 v_lut_pairs(const schar* tab, const int* idx) { - schar CV_DECL_ALIGNED(32) elems[16] = + schar elems[16] = { tab[idx[0]], tab[idx[0] + 1], @@ -846,7 +846,7 @@ inline v_int8x16 v_lut_pairs(const schar* tab, const int* idx) } inline v_int8x16 v_lut_quads(const schar* tab, const int* idx) { - schar CV_DECL_ALIGNED(32) elems[16] = + schar elems[16] = { tab[idx[0]], tab[idx[0] + 1], @@ -873,7 +873,7 @@ inline v_uint8x16 v_lut_quads(const uchar* tab, const int* idx) { return v_reint inline v_int16x8 v_lut(const short* tab, const int* idx) { - short CV_DECL_ALIGNED(32) elems[8] = + short elems[8] = { tab[idx[0]], tab[idx[1]], @@ -888,7 +888,7 @@ inline v_int16x8 v_lut(const short* tab, const int* idx) } inline v_int16x8 v_lut_pairs(const short* tab, const int* idx) { - short CV_DECL_ALIGNED(32) elems[8] = + short elems[8] = { tab[idx[0]], tab[idx[0] + 1], @@ -903,7 +903,7 @@ inline v_int16x8 v_lut_pairs(const short* tab, const int* idx) } inline v_int16x8 v_lut_quads(const short* tab, const int* idx) { - short CV_DECL_ALIGNED(32) elems[8] = + short elems[8] = { tab[idx[0]], tab[idx[0] + 1], @@ -922,7 +922,7 @@ inline v_uint16x8 v_lut_quads(const ushort* tab, const int* idx) { return v_rein inline v_int32x4 v_lut(const int* tab, const int* idx) { - int CV_DECL_ALIGNED(32) elems[4] = + int elems[4] = { tab[idx[0]], tab[idx[1]], @@ -933,7 +933,7 @@ inline v_int32x4 v_lut(const int* tab, const int* idx) } inline v_int32x4 v_lut_pairs(const int* tab, const int* idx) { - int CV_DECL_ALIGNED(32) elems[4] = + int elems[4] = { tab[idx[0]], tab[idx[0] + 1], @@ -953,7 +953,7 @@ inline v_uint32x4 v_lut_quads(const unsigned* tab, const int* idx) { return v_re inline v_int64x2 v_lut(const int64_t* tab, const int* idx) { - int64_t CV_DECL_ALIGNED(32) elems[2] = + int64_t elems[2] = { tab[idx[0]], tab[idx[1]] @@ -969,7 +969,7 @@ inline v_uint64x2 v_lut_pairs(const uint64* tab, const int* idx) { return v_rein inline v_float32x4 v_lut(const float* tab, const int* idx) { - float CV_DECL_ALIGNED(32) elems[4] = + float elems[4] = { tab[idx[0]], tab[idx[1]], @@ -980,7 +980,7 @@ inline v_float32x4 v_lut(const float* tab, const int* idx) } inline v_float32x4 v_lut_pairs(const float* tab, const int* idx) { - float CV_DECL_ALIGNED(32) elems[4] = + float elems[4] = { tab[idx[0]], tab[idx[0] + 1], @@ -996,7 +996,7 @@ inline v_float32x4 v_lut_quads(const float* tab, const int* idx) inline v_int32x4 v_lut(const int* tab, const v_int32x4& idxvec) { - int CV_DECL_ALIGNED(32) elems[4] = + int elems[4] = { tab[v_extract_n<0>(idxvec)], tab[v_extract_n<1>(idxvec)], @@ -1008,7 +1008,7 @@ inline v_int32x4 v_lut(const int* tab, const v_int32x4& idxvec) inline v_uint32x4 v_lut(const unsigned* tab, const v_int32x4& idxvec) { - unsigned CV_DECL_ALIGNED(32) elems[4] = + unsigned elems[4] = { tab[v_extract_n<0>(idxvec)], tab[v_extract_n<1>(idxvec)], @@ -1020,7 +1020,7 @@ inline v_uint32x4 v_lut(const unsigned* tab, const v_int32x4& idxvec) inline v_float32x4 v_lut(const float* tab, const v_int32x4& idxvec) { - float CV_DECL_ALIGNED(32) elems[4] = + float elems[4] = { tab[v_extract_n<0>(idxvec)], tab[v_extract_n<1>(idxvec)], @@ -1032,7 +1032,7 @@ inline v_float32x4 v_lut(const float* tab, const v_int32x4& idxvec) inline void v_lut_deinterleave(const float* tab, const v_int32x4& idxvec, v_float32x4& x, v_float32x4& y) { - int CV_DECL_ALIGNED(32) idx[4]; + int idx[4]; v_store_aligned(idx, idxvec); x = v_float32x4(tab[idx[0]], tab[idx[1]], tab[idx[2]], tab[idx[3]]); @@ -1042,7 +1042,7 @@ inline void v_lut_deinterleave(const float* tab, const v_int32x4& idxvec, v_floa #if CV_SIMD128_64F inline v_float64x2 v_lut(const double* tab, const int* idx) { - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { tab[idx[0]], tab[idx[1]] @@ -1057,7 +1057,7 @@ inline v_float64x2 v_lut_pairs(const double* tab, const int* idx) inline v_float64x2 v_lut(const double* tab, const v_int32x4& idxvec) { - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { tab[v_extract_n<0>(idxvec)], tab[v_extract_n<1>(idxvec)] @@ -1067,7 +1067,7 @@ inline v_float64x2 v_lut(const double* tab, const v_int32x4& idxvec) inline void v_lut_deinterleave(const double* tab, const v_int32x4& idxvec, v_float64x2& x, v_float64x2& y) { - int CV_DECL_ALIGNED(32) idx[4] = {0}; + int idx[4] = {0}; v_store_aligned(idx, idxvec); x = v_float64x2(tab[idx[0]], tab[idx[1]]); @@ -1079,7 +1079,7 @@ inline void v_lut_deinterleave(const double* tab, const v_int32x4& idxvec, v_flo inline v_uint8x16 v_pack_b(const v_uint16x8& a, const v_uint16x8& b) { - ushort CV_DECL_ALIGNED(32) ptr[16] = {0}; + ushort ptr[16] = {0}; v_store(ptr, a); v_store(ptr + 8, b); return v_uint8x16(vnsrl_wx_u8m1(vle16_v_u16m2(ptr, 16), 0, 16)); @@ -1088,7 +1088,7 @@ inline v_uint8x16 v_pack_b(const v_uint16x8& a, const v_uint16x8& b) inline v_uint8x16 v_pack_b(const v_uint32x4& a, const v_uint32x4& b, const v_uint32x4& c, const v_uint32x4& d) { - unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; + unsigned ptr[16] = {0}; v_store(ptr, a); v_store(ptr + 4, b); v_store(ptr + 8, c); @@ -1100,7 +1100,7 @@ inline v_uint8x16 v_pack_b(const v_uint64x2& a, const v_uint64x2& b, const v_uin const v_uint64x2& d, const v_uint64x2& e, const v_uint64x2& f, const v_uint64x2& g, const v_uint64x2& h) { - uint64 CV_DECL_ALIGNED(32) ptr[16] = {0}; + uint64 ptr[16] = {0}; v_store(ptr, a); v_store(ptr + 2, b); v_store(ptr + 4, c); @@ -1279,13 +1279,15 @@ OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int64x2, i64, 2) #define OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, op, intrin, suffix, vl) \ inline _Tpvec operator op (const _Tpvec& a, const _Tpvec& b) \ { \ - return _Tpvec(vmerge_vxm_##suffix##m1(intrin(a, b, vl), vmv_v_x_##suffix##m1(0, vl), 1, vl)); \ + uint64_t ones = -1; \ + return _Tpvec(vmerge_vxm_##suffix##m1(intrin(a, b, vl), vmv_v_x_##suffix##m1(0, vl), ones, vl)); \ } #define OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, op, intrin, suffix, vl) \ inline _Tpvec operator op (const _Tpvec& a, const _Tpvec& b) \ { \ - return _Tpvec(vfmerge_vfm_##suffix##m1(intrin(a, b, vl), vfmv_v_f_##suffix##m1(0, vl), 1, vl)); \ + union { uint64 u; double d; } ones; ones.u = -1; \ + return _Tpvec(vfmerge_vfm_##suffix##m1(intrin(a, b, vl), vfmv_v_f_##suffix##m1(0, vl), ones.d, vl)); \ } #define OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(_Tpvec, suffix, width, vl) \ @@ -1441,7 +1443,7 @@ OPENCV_HAL_IMPL_RVV_REDUCE(v_float32x4, max, float, f32, 4, fredmax) inline v_float32x4 v_reduce_sum4(const v_float32x4& a, const v_float32x4& b, const v_float32x4& c, const v_float32x4& d) { - float CV_DECL_ALIGNED(32) elems[4] = + float elems[4] = { v_reduce_sum(a), v_reduce_sum(b), @@ -1746,9 +1748,9 @@ inline v_float32x4 v_cvt_f32(const v_float64x2& a, const v_float64x2& b) inline v_float64x2 v_cvt_f64(const v_int32x4& a) { - double CV_DECL_ALIGNED(32) ptr[4] = {0}; + double ptr[4] = {0}; vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a, 4), 4); - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { ptr[0], ptr[1] }; @@ -1757,9 +1759,9 @@ inline v_float64x2 v_cvt_f64(const v_int32x4& a) inline v_float64x2 v_cvt_f64_high(const v_int32x4& a) { - double CV_DECL_ALIGNED(32) ptr[4] = {0}; + double ptr[4] = {0}; vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a, 4), 4); - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { ptr[2], ptr[3] }; @@ -1768,9 +1770,9 @@ inline v_float64x2 v_cvt_f64_high(const v_int32x4& a) inline v_float64x2 v_cvt_f64(const v_float32x4& a) { - double CV_DECL_ALIGNED(32) ptr[4] = {0}; + double ptr[4] = {0}; vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a, 4), 4); - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { ptr[0], ptr[1] }; @@ -1779,9 +1781,9 @@ inline v_float64x2 v_cvt_f64(const v_float32x4& a) inline v_float64x2 v_cvt_f64_high(const v_float32x4& a) { - double CV_DECL_ALIGNED(32) ptr[4] = {0}; + double ptr[4] = {0}; vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a, 4), 4); - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { ptr[2], ptr[3] }; @@ -1823,7 +1825,7 @@ inline void v_transpose4x4(const v_##_Tpvec& a0, const v_##_Tpvec& a1, \ v_##_Tpvec& b0, v_##_Tpvec& b1, \ v_##_Tpvec& b2, v_##_Tpvec& b3) \ { \ - _Tp CV_DECL_ALIGNED(32) elems0[4] = \ + _Tp elems0[4] = \ { \ v_extract_n<0>(a0), \ v_extract_n<0>(a1), \ @@ -1831,7 +1833,7 @@ inline void v_transpose4x4(const v_##_Tpvec& a0, const v_##_Tpvec& a1, \ v_extract_n<0>(a3) \ }; \ b0 = v_load(elems0); \ - _Tp CV_DECL_ALIGNED(32) elems1[4] = \ + _Tp elems1[4] = \ { \ v_extract_n<1>(a0), \ v_extract_n<1>(a1), \ @@ -1839,7 +1841,7 @@ inline void v_transpose4x4(const v_##_Tpvec& a0, const v_##_Tpvec& a1, \ v_extract_n<1>(a3) \ }; \ b1 = v_load(elems1); \ - _Tp CV_DECL_ALIGNED(32) elems2[4] = \ + _Tp elems2[4] = \ { \ v_extract_n<2>(a0), \ v_extract_n<2>(a1), \ @@ -1847,7 +1849,7 @@ inline void v_transpose4x4(const v_##_Tpvec& a0, const v_##_Tpvec& a1, \ v_extract_n<2>(a3) \ }; \ b2 = v_load(elems2); \ - _Tp CV_DECL_ALIGNED(32) elems3[4] = \ + _Tp elems3[4] = \ { \ v_extract_n<3>(a0), \ v_extract_n<3>(a1), \ @@ -1866,8 +1868,8 @@ OPENCV_HAL_IMPL_RVV_TRANSPOSE4x4(float32x4, float, f32) #define OPENCV_HAL_IMPL_RVV_REVERSE(_Tpvec, _Tp, suffix) \ inline _Tpvec v_reverse(const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) ptr[_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptra[_Tpvec::nlanes] = {0}; \ + _Tp ptr[_Tpvec::nlanes] = {0}; \ + _Tp ptra[_Tpvec::nlanes] = {0}; \ v_store(ptra, a); \ for (int i = 0; i < _Tpvec::nlanes; i++) \ { \ @@ -1894,8 +1896,8 @@ OPENCV_HAL_IMPL_RVV_REVERSE(v_float64x2, double, f64) #define OPENCV_HAL_IMPL_RVV_EXPAND(_Tpwvec, _Tp, _Tpvec, width, suffix, wcvt, vl) \ inline void v_expand(const _Tpvec& a, _Tpwvec& b0, _Tpwvec& b1) \ { \ - _Tp CV_DECL_ALIGNED(32) lptr[_Tpvec::nlanes/2] = {0}; \ - _Tp CV_DECL_ALIGNED(32) hptr[_Tpvec::nlanes/2] = {0}; \ + _Tp lptr[_Tpvec::nlanes/2] = {0}; \ + _Tp hptr[_Tpvec::nlanes/2] = {0}; \ v_store_low(lptr, a); \ v_store_high(hptr, a); \ b0 = _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr, vl), vl)); \ @@ -1903,13 +1905,13 @@ inline void v_expand(const _Tpvec& a, _Tpwvec& b0, _Tpwvec& b1) \ } \ inline _Tpwvec v_expand_low(const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) lptr[_Tpvec::nlanes/2] = {0}; \ + _Tp lptr[_Tpvec::nlanes/2] = {0}; \ v_store_low(lptr, a); \ return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr, vl), vl)); \ } \ inline _Tpwvec v_expand_high(const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) hptr[_Tpvec::nlanes/2] = {0}; \ + _Tp hptr[_Tpvec::nlanes/2] = {0}; \ v_store_high(hptr, a); \ return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(hptr, vl), vl)); \ } \ @@ -1936,25 +1938,25 @@ inline v_int32x4 v_load_expand_q(const schar* ptr) } -#define OPENCV_HAL_IMPL_RVV_PACK(_Tpvec, _Tp, _wTpvec, _wTp, width, suffix, rshr, shr, hvl, vl) \ +#define OPENCV_HAL_IMPL_RVV_PACK(_Tpvec, _Tp, _wTpvec, _wTp, hwidth, width, hsuffix, suffix, rshr, shr, hvl, vl) \ inline _Tpvec v_pack(const _wTpvec& a, const _wTpvec& b) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ return _Tpvec(shr(vle##width##_v_##suffix##m2(arr, vl), 0, vl)); \ } \ inline void v_pack_store(_Tp* ptr, const _wTpvec& a) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ - v_store(ptr, _Tpvec(shr(vle##width##_v_##suffix##m2(arr, vl), 0, vl))); \ + vse##hwidth##_v_##hsuffix##m1(ptr, shr(vle##width##_v_##suffix##m2(arr, vl), 0, vl), hvl); \ } \ template inline \ _Tpvec v_rshr_pack(const _wTpvec& a, const _wTpvec& b) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ return _Tpvec(rshr(vle##width##_v_##suffix##m2(arr, vl), n, vl)); \ @@ -1962,39 +1964,39 @@ _Tpvec v_rshr_pack(const _wTpvec& a, const _wTpvec& b) \ template inline \ void v_rshr_pack_store(_Tp* ptr, const _wTpvec& a) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ v_store(ptr, _Tpvec(rshr(vle##width##_v_##suffix##m2(arr, vl), n, vl))); \ } -OPENCV_HAL_IMPL_RVV_PACK(v_uint8x16, uchar, v_uint16x8, ushort, 16, u16, vnclipu_wx_u8m1, vnclipu_wx_u8m1, 8, 16) -OPENCV_HAL_IMPL_RVV_PACK(v_int8x16, schar, v_int16x8, short, 16, i16, vnclip_wx_i8m1, vnclip_wx_i8m1, 8, 16) -OPENCV_HAL_IMPL_RVV_PACK(v_uint16x8, ushort, v_uint32x4, unsigned, 32, u32, vnclipu_wx_u16m1, vnclipu_wx_u16m1, 4, 8) -OPENCV_HAL_IMPL_RVV_PACK(v_int16x8, short, v_int32x4, int, 32, i32, vnclip_wx_i16m1, vnclip_wx_i16m1, 4, 8) -OPENCV_HAL_IMPL_RVV_PACK(v_uint32x4, unsigned, v_uint64x2, uint64, 64, u64, vnclipu_wx_u32m1, vnsrl_wx_u32m1, 2, 4) -OPENCV_HAL_IMPL_RVV_PACK(v_int32x4, int, v_int64x2, int64, 64, i64, vnclip_wx_i32m1, vnsra_wx_i32m1, 2, 4) +OPENCV_HAL_IMPL_RVV_PACK(v_uint8x16, uchar, v_uint16x8, ushort, 8, 16, u8, u16, vnclipu_wx_u8m1, vnclipu_wx_u8m1, 8, 16) +OPENCV_HAL_IMPL_RVV_PACK(v_int8x16, schar, v_int16x8, short, 8, 16, i8, i16, vnclip_wx_i8m1, vnclip_wx_i8m1, 8, 16) +OPENCV_HAL_IMPL_RVV_PACK(v_uint16x8, ushort, v_uint32x4, unsigned, 16, 32, u16, u32, vnclipu_wx_u16m1, vnclipu_wx_u16m1, 4, 8) +OPENCV_HAL_IMPL_RVV_PACK(v_int16x8, short, v_int32x4, int, 16, 32, i16, i32, vnclip_wx_i16m1, vnclip_wx_i16m1, 4, 8) +OPENCV_HAL_IMPL_RVV_PACK(v_uint32x4, unsigned, v_uint64x2, uint64, 32, 64, u32, u64, vnclipu_wx_u32m1, vnsrl_wx_u32m1, 2, 4) +OPENCV_HAL_IMPL_RVV_PACK(v_int32x4, int, v_int64x2, int64, 32, 64, i32, i64, vnclip_wx_i32m1, vnsra_wx_i32m1, 2, 4) -#define OPENCV_HAL_IMPL_RVV_PACK_U(_Tpvec, _Tp, _wTpvec, _wTp, width, suffix, rshr, cast, vl) \ +#define OPENCV_HAL_IMPL_RVV_PACK_U(_Tpvec, _Tp, _wTpvec, _wTp, hwidth, width, hsuffix, suffix, rshr, cast, hvl, vl) \ inline _Tpvec v_pack_u(const _wTpvec& a, const _wTpvec& b) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), 0, vl)); \ } \ inline void v_pack_u_store(_Tp* ptr, const _wTpvec& a) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, vl))); \ - v_store(ptr, _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), 0, vl))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ + vse##hwidth##_v_##hsuffix##m1(ptr, rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), 0, vl), hvl); \ } \ template inline \ _Tpvec v_rshr_pack_u(const _wTpvec& a, const _wTpvec& b) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), n, vl)); \ @@ -2002,23 +2004,23 @@ _Tpvec v_rshr_pack_u(const _wTpvec& a, const _wTpvec& b) \ template inline \ void v_rshr_pack_u_store(_Tp* ptr, const _wTpvec& a) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, vl))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ v_store(ptr, _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), n, vl))); \ } -OPENCV_HAL_IMPL_RVV_PACK_U(v_uint8x16, uchar, v_int16x8, short, 16, i16, vnclipu_wx_u8m1, vreinterpret_v_i16m2_u16m2, 16) -OPENCV_HAL_IMPL_RVV_PACK_U(v_uint16x8, ushort, v_int32x4, int, 32, i32, vnclipu_wx_u16m1, vreinterpret_v_i32m2_u32m2, 8) +OPENCV_HAL_IMPL_RVV_PACK_U(v_uint8x16, uchar, v_int16x8, short, 8, 16, u8, i16, vnclipu_wx_u8m1, vreinterpret_v_i16m2_u16m2, 8, 16) +OPENCV_HAL_IMPL_RVV_PACK_U(v_uint16x8, ushort, v_int32x4, int, 16, 32, u16, i32, vnclipu_wx_u16m1, vreinterpret_v_i32m2_u32m2, 4, 8) #define OPENCV_HAL_IMPL_RVV_UNPACKS(_Tpvec, _Tp, suffix) \ inline void v_zip(const v_##_Tpvec& a0, const v_##_Tpvec& a1, v_##_Tpvec& b0, v_##_Tpvec& b1) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra0[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptra1[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb0[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb1[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra0[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra1[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb0[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb1[v_##_Tpvec::nlanes] = {0}; \ v_store(ptra0, a0); \ v_store(ptra1, a1); \ int i; \ @@ -2037,16 +2039,16 @@ inline void v_zip(const v_##_Tpvec& a0, const v_##_Tpvec& a1, v_##_Tpvec& b0, v_ } \ inline v_##_Tpvec v_combine_low(const v_##_Tpvec& a, const v_##_Tpvec& b) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes/2] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes/2] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes/2] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes/2] = {0}; \ v_store_low(ptra, a); \ v_store_low(ptrb, b); \ return v_load_halves(ptra, ptrb); \ } \ inline v_##_Tpvec v_combine_high(const v_##_Tpvec& a, const v_##_Tpvec& b) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes/2] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes/2] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes/2] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes/2] = {0}; \ v_store_high(ptra, a); \ v_store_high(ptrb, b); \ return v_load_halves(ptra, ptrb); \ @@ -2072,8 +2074,8 @@ OPENCV_HAL_IMPL_RVV_UNPACKS(float64x2, double, f64) #define OPENCV_HAL_IMPL_RVV_INTERLEAVED(_Tpvec, _Tp) \ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ int i, i2; \ for( i = i2 = 0; i < v_##_Tpvec::nlanes; i++, i2 += 2 ) \ { \ @@ -2085,9 +2087,9 @@ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b) \ } \ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b, v_##_Tpvec& c) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrc[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrc[v_##_Tpvec::nlanes] = {0}; \ int i, i3; \ for( i = i3 = 0; i < v_##_Tpvec::nlanes; i++, i3 += 3 ) \ { \ @@ -2102,10 +2104,10 @@ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b, v_ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b, \ v_##_Tpvec& c, v_##_Tpvec& d) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrc[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrd[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrc[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrd[v_##_Tpvec::nlanes] = {0}; \ int i, i4; \ for( i = i4 = 0; i < v_##_Tpvec::nlanes; i++, i4 += 4 ) \ { \ @@ -2123,8 +2125,8 @@ inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \ { \ int i, i2; \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ v_store(ptra, a); \ v_store(ptrb, b); \ for( i = i2 = 0; i < v_##_Tpvec::nlanes; i++, i2 += 2 ) \ @@ -2137,9 +2139,9 @@ inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& const v_##_Tpvec& c, hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \ { \ int i, i3; \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrc[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrc[v_##_Tpvec::nlanes] = {0}; \ v_store(ptra, a); \ v_store(ptrb, b); \ v_store(ptrc, c); \ @@ -2155,10 +2157,10 @@ inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& hal::StoreMode /*mode*/=hal::STORE_UNALIGNED ) \ { \ int i, i4; \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrc[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrd[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrc[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrd[v_##_Tpvec::nlanes] = {0}; \ v_store(ptra, a); \ v_store(ptrb, b); \ v_store(ptrc, c); \ @@ -2173,8 +2175,8 @@ inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& } \ inline v_##_Tpvec v_interleave_pairs(const v_##_Tpvec& vec) \ { \ - _Tp CV_DECL_ALIGNED(32) ptr[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrvec[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptr[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrvec[v_##_Tpvec::nlanes] = {0}; \ v_store(ptrvec, vec); \ for (int i = 0; i < v_##_Tpvec::nlanes/4; i++) \ { \ @@ -2187,8 +2189,8 @@ inline v_##_Tpvec v_interleave_pairs(const v_##_Tpvec& vec) \ } \ inline v_##_Tpvec v_interleave_quads(const v_##_Tpvec& vec) \ { \ - _Tp CV_DECL_ALIGNED(32) ptr[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrvec[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptr[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrvec[v_##_Tpvec::nlanes] = {0}; \ v_store(ptrvec, vec); \ for (int i = 0; i < v_##_Tpvec::nlanes/8; i++) \ { \ @@ -2242,9 +2244,9 @@ static const unsigned char popCountTable[] = #define OPENCV_HAL_IMPL_RVV_POPCOUNT_OP(_rTpvec, _Tpvec, _rTp, _Tp, suffix) \ inline _rTpvec v_popcount(const _Tpvec& a) \ { \ - uchar CV_DECL_ALIGNED(32) ptra[16] = {0}; \ + uchar ptra[16] = {0}; \ v_store(ptra, v_reinterpret_as_u8(a)); \ - _rTp CV_DECL_ALIGNED(32) ptr[_Tpvec::nlanes] = {0}; \ + _rTp ptr[_Tpvec::nlanes] = {0}; \ v_store(ptr, v_setzero_##suffix()); \ for (int i = 0; i < _Tpvec::nlanes*(int)sizeof(_Tp); i++) \ ptr[i/sizeof(_Tp)] += popCountTable[ptra[i]]; \ @@ -2298,7 +2300,7 @@ inline int v_signmask(const v_float64x2& a) #define OPENCV_HAL_IMPL_RVV_SCAN_FORWOARD_OP(_Tpvec, _Tp, suffix) \ inline int v_scan_forward(const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) ptr[_Tpvec::nlanes] = {0}; \ + _Tp ptr[_Tpvec::nlanes] = {0}; \ v_store(ptr, v_reinterpret_as_##suffix(a)); \ for (int i = 0; i < _Tpvec::nlanes; i++) \ if(int(ptr[i]) < 0) \ @@ -2321,28 +2323,29 @@ OPENCV_HAL_IMPL_RVV_SCAN_FORWOARD_OP(v_float64x2, double, f64) //////////// Pack triplets //////////// -#define OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(_Tpvec, _Tp) \ -inline _Tpvec v_pack_triplets(const _Tpvec& vec) \ -{ \ - _Tp CV_DECL_ALIGNED(32) ptr[_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrvec[_Tpvec::nlanes] = {0}; \ - v_store(ptrvec, vec); \ - for (int i = 0; i < _Tpvec::nlanes/4; i++) \ - { \ - ptr[3*i ] = ptrvec[4*i ]; \ - ptr[3*i+1] = ptrvec[4*i+2]; \ - ptr[3*i+2] = ptrvec[4*i+2]; \ - } \ - return v_load(ptr); \ +inline v_int8x16 v_pack_triplets(const v_int8x16& vec) +{ + uint64 ptr[2] = {0x0908060504020100, 0xFFFFFFFF0E0D0C0A}; + return v_int8x16((vint8m1_t)vrgather_vv_u8m1((vuint8m1_t)vint8m1_t(vec), (vuint8m1_t)vle64_v_u64m1(ptr, 2), 16)); +} +inline v_uint8x16 v_pack_triplets(const v_uint8x16& vec) +{ + return v_reinterpret_as_u8(v_pack_triplets(v_reinterpret_as_s8(vec))); +} + +inline v_int16x8 v_pack_triplets(const v_int16x8& vec) +{ + uint64 ptr[2] = {0x0908060504020100, 0xFFFFFFFF0E0D0C0A}; + return v_int16x8((vint16m1_t)vrgather_vv_u8m1((vuint8m1_t)vint16m1_t(vec), (vuint8m1_t)vle64_v_u64m1(ptr, 2), 16)); +} +inline v_uint16x8 v_pack_triplets(const v_uint16x8& vec) +{ + return v_reinterpret_as_u16(v_pack_triplets(v_reinterpret_as_s16(vec))); } -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_uint8x16, uchar) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_int8x16, schar) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_uint16x8, ushort) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_int16x8, short) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_uint32x4, unsigned) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_int32x4, int) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_float32x4, float) +inline v_int32x4 v_pack_triplets(const v_int32x4& vec) { return vec; } +inline v_uint32x4 v_pack_triplets(const v_uint32x4& vec) { return vec; } +inline v_float32x4 v_pack_triplets(const v_float32x4& vec) { return vec; } ////// FP16 support /////// @@ -2443,7 +2446,7 @@ inline v_int32x4 v_trunc(const v_float64x2& a) // 16 >> 32 inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b) { - int CV_DECL_ALIGNED(32) ptr[8] = {0}; + int ptr[8] = {0}; v_int32x4 t1, t2; vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_load_deinterleave(ptr, t1, t2); @@ -2451,7 +2454,7 @@ inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b) } inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b, const v_int32x4& c) { - int CV_DECL_ALIGNED(32) ptr[8] = {0}; + int ptr[8] = {0}; v_int32x4 t1, t2; vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_load_deinterleave(ptr, t1, t2); @@ -2461,7 +2464,7 @@ inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b, const v_int32 // 32 >> 64 inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b) { - int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; + int64 ptr[4] = {0}; v_int64x2 t1, t2; vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_load_deinterleave(ptr, t1, t2); @@ -2469,7 +2472,7 @@ inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b) } inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b, const v_int64x2& c) { - int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; + int64 ptr[4] = {0}; v_int64x2 t1, t2; vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_load_deinterleave(ptr, t1, t2); @@ -2479,7 +2482,7 @@ inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b, const v_int64 // 8 >> 32 inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b) { - unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; + unsigned ptr[16] = {0}; v_uint32x4 t1, t2, t3, t4; vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2488,7 +2491,7 @@ inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b) inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b, const v_uint32x4& c) { - unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; + unsigned ptr[16] = {0}; v_uint32x4 t1, t2, t3, t4; vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2497,7 +2500,7 @@ inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b, inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b) { - int CV_DECL_ALIGNED(32) ptr[16] = {0}; + int ptr[16] = {0}; v_int32x4 t1, t2, t3, t4; vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2506,7 +2509,7 @@ inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b) inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b, const v_int32x4& c) { - int CV_DECL_ALIGNED(32) ptr[16] = {0}; + int ptr[16] = {0}; v_int32x4 t1, t2, t3, t4; vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2516,7 +2519,7 @@ inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b, // 16 >> 64 inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b) { - uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + uint64 ptr[8] = {0}; v_uint64x2 t1, t2, t3, t4; vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2524,7 +2527,7 @@ inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b) } inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c) { - uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + uint64 ptr[8] = {0}; v_uint64x2 t1, t2, t3, t4; vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2533,7 +2536,7 @@ inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b, con inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b) { - int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + int64 ptr[8] = {0}; v_int64x2 t1, t2, t3, t4; vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2542,7 +2545,7 @@ inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b) inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c) { - int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + int64 ptr[8] = {0}; v_int64x2 t1, t2, t3, t4; vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); @@ -2563,7 +2566,7 @@ inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b, // 16 >> 32 inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b) { - int CV_DECL_ALIGNED(32) ptr[8] = {0}; + int ptr[8] = {0}; vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); @@ -2571,7 +2574,7 @@ inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b) } inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b, const v_int32x4& c) { - int CV_DECL_ALIGNED(32) ptr[8] = {0}; + int ptr[8] = {0}; vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); @@ -2581,7 +2584,7 @@ inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b, const v_ // 32 >> 64 inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b) { - int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; + int64 ptr[4] = {0}; vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); @@ -2589,7 +2592,7 @@ inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b) } inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b, const v_int64x2& c) { - int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; + int64 ptr[4] = {0}; vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); @@ -2600,7 +2603,7 @@ inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b, const v_ // 8 >> 32 inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b) { - unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; + unsigned ptr[16] = {0}; vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_uint32x4 t1 = v_load(ptr); v_uint32x4 t2 = v_load(ptr+4); @@ -2610,7 +2613,7 @@ inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b } inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b, const v_uint32x4& c) { - unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; + unsigned ptr[16] = {0}; vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_uint32x4 t1 = v_load(ptr); v_uint32x4 t2 = v_load(ptr+4); @@ -2620,7 +2623,7 @@ inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b } inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b) { - int CV_DECL_ALIGNED(32) ptr[16] = {0}; + int ptr[16] = {0}; vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); @@ -2630,7 +2633,7 @@ inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b) } inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b, const v_int32x4& c) { - int CV_DECL_ALIGNED(32) ptr[16] = {0}; + int ptr[16] = {0}; vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); @@ -2642,7 +2645,7 @@ inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b, c // 16 >> 64 inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b) { - uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + uint64 ptr[8] = {0}; vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_uint64x2 t1 = v_load(ptr); v_uint64x2 t2 = v_load(ptr+2); @@ -2652,7 +2655,7 @@ inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b } inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c) { - uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + uint64 ptr[8] = {0}; vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_uint64x2 t1 = v_load(ptr); v_uint64x2 t2 = v_load(ptr+2); @@ -2662,7 +2665,7 @@ inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b } inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b) { - int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + int64 ptr[8] = {0}; vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); @@ -2672,7 +2675,7 @@ inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b) } inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c) { - int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + int64 ptr[8] = {0}; vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); @@ -2714,7 +2717,7 @@ inline v_float32x4 v_matmuladd(const v_float32x4& v, const v_float32x4& m0, #define OPENCV_HAL_IMPL_RVV_MUL_EXPAND(_Tpvec, _Tpwvec, _Tpw, suffix, wmul, width, vl, hvl) \ inline void v_mul_expand(const _Tpvec& a, const _Tpvec& b, _Tpwvec& c, _Tpwvec& d) \ { \ - _Tpw CV_DECL_ALIGNED(32) ptr[_Tpwvec::nlanes*2] = {0}; \ + _Tpw ptr[_Tpwvec::nlanes*2] = {0}; \ vse##width##_v_##suffix##m2(ptr, wmul(a, b, vl), vl); \ c = _Tpwvec(vle##width##_v_##suffix##m1(ptr, hvl)); \ d = _Tpwvec(vle##width##_v_##suffix##m1(ptr+_Tpwvec::nlanes, hvl)); \ From 4e5699fa716a3e5b0faddf1d6a00213aeb5c60cc Mon Sep 17 00:00:00 2001 From: Julia Bareeva <34717687+JulieBar@users.noreply.github.com> Date: Fri, 23 Jul 2021 17:11:50 +0300 Subject: [PATCH 079/376] Merge pull request #20450 from JulieBar:lstm_inside Support non-zero hidden state for LSTM * fully support non-zero hidden state for LSTM * check dims of hidden state for LSTM * fix failed test Test_Model.TextRecognition * add new tests for LSTM w/ non-zero hidden params Co-authored-by: Julie Bareeva --- modules/dnn/src/layers/recurrent_layers.cpp | 28 +++++--- modules/dnn/src/onnx/onnx_importer.cpp | 11 ++- modules/dnn/src/tensorflow/tf_importer.cpp | 16 +++-- modules/dnn/test/test_layers.cpp | 80 ++++++++++++++++++++- modules/dnn/test/test_onnx_importer.cpp | 10 +++ 5 files changed, 122 insertions(+), 23 deletions(-) diff --git a/modules/dnn/src/layers/recurrent_layers.cpp b/modules/dnn/src/layers/recurrent_layers.cpp index 69606a6b4ef5..a6715aefca92 100644 --- a/modules/dnn/src/layers/recurrent_layers.cpp +++ b/modules/dnn/src/layers/recurrent_layers.cpp @@ -112,19 +112,24 @@ class LSTMLayerImpl CV_FINAL : public LSTMLayer const Mat& Wh = blobs[0]; const Mat& Wx = blobs[1]; const Mat& bias = blobs[2]; + const Mat& hInternal = blobs[3]; + const Mat& cInternal = blobs[4]; CV_CheckEQ(Wh.dims, 2, ""); CV_CheckEQ(Wx.dims, 2, ""); CV_CheckEQ(Wh.rows, Wx.rows, ""); CV_CheckEQ(Wh.rows, (1 + static_cast(bidirectional))*4*Wh.cols, ""); CV_CheckEQ(Wh.rows, (int)bias.total(), ""); + CV_CheckEQ(hInternal.cols, Wh.cols, ""); + CV_CheckEQ(hInternal.cols, cInternal.cols, ""); + CV_CheckEQ(hInternal.rows, cInternal.rows, ""); CV_Assert(Wh.type() == Wx.type() && Wx.type() == bias.type()); // Peephole weights. - if (blobs.size() > 3) + if (blobs.size() > 5) { - CV_Assert(blobs.size() == 6); + CV_Assert(blobs.size() == 8); const int N = Wh.cols; - for (int i = 3; i < 6; ++i) + for (int i = 5; i < 8; ++i) { CV_Assert(blobs[i].rows == N && blobs[i].cols == N); CV_Assert(blobs[i].type() == bias.type()); @@ -181,7 +186,7 @@ class LSTMLayerImpl CV_FINAL : public LSTMLayer std::vector &outputs, std::vector &internals) const CV_OVERRIDE { - CV_Assert((!usePeephole && blobs.size() == 3) || (usePeephole && blobs.size() == 6)); + CV_Assert((!usePeephole && blobs.size() == 5) || (usePeephole && blobs.size() == 8)); CV_Assert(inputs.size() == 1); const MatShape& inp0 = inputs[0]; @@ -228,7 +233,7 @@ class LSTMLayerImpl CV_FINAL : public LSTMLayer std::vector input; inputs_arr.getMatVector(input); - CV_Assert((!usePeephole && blobs.size() == 3) || (usePeephole && blobs.size() == 6)); + CV_Assert((!usePeephole && blobs.size() == 5) || (usePeephole && blobs.size() == 8)); CV_Assert(input.size() == 1); const Mat& inp0 = input[0]; @@ -284,13 +289,14 @@ class LSTMLayerImpl CV_FINAL : public LSTMLayer const Mat &Wh = blobs[0].rowRange(i * blobs[0].rows / numDirs, (i + 1) * blobs[0].rows / numDirs); const Mat &Wx = blobs[1].rowRange(i * blobs[1].rows / numDirs, (i + 1) * blobs[1].rows / numDirs); const Mat &bias = blobs[2].colRange(i * blobs[2].cols / numDirs, (i + 1) * blobs[2].cols / numDirs); + const Mat &h_0 = blobs[3].rowRange(i * blobs[3].rows / numDirs, (i + 1) * blobs[3].rows / numDirs); + const Mat &c_0 = blobs[4].rowRange(i * blobs[4].rows / numDirs, (i + 1) * blobs[4].rows / numDirs); int numOut = Wh.size[1]; - Mat hInternal = internals[0], cInternal = internals[1], dummyOnes = internals[2], gates = internals[3]; - hInternal.setTo(0.); - cInternal.setTo(0.); + h_0.copyTo(hInternal); + c_0.copyTo(cInternal); dummyOnes.setTo(1.); int numSamplesTotal = numTimeStamps*numSamples; @@ -331,8 +337,8 @@ class LSTMLayerImpl CV_FINAL : public LSTMLayer if (usePeephole) { Mat gatesIF = gates.colRange(0, 2*numOut); - gemm(cInternal, blobs[3], 1, gateI, 1, gateI); - gemm(cInternal, blobs[4], 1, gateF, 1, gateF); + gemm(cInternal, blobs[5], 1, gateI, 1, gateI); + gemm(cInternal, blobs[6], 1, gateF, 1, gateF); sigmoid(gatesIF, gatesIF); } else @@ -355,7 +361,7 @@ class LSTMLayerImpl CV_FINAL : public LSTMLayer } if (usePeephole) { - gemm(cInternal, blobs[5], 1, gateO, 1, gateO); + gemm(cInternal, blobs[7], 1, gateO, 1, gateO); sigmoid(gateO, gateO); } diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index ec61a9707eb9..4ad0fd496e37 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -900,8 +900,9 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) Mat Wx = getBlob(node_proto, 1); Mat Wh = getBlob(node_proto, 2); Mat b = getBlob(node_proto, 3); - CV_CheckEQ(countNonZero(getBlob(node_proto, 5)), 0, "Unsupported non zero initial_h"); - CV_CheckEQ(countNonZero(getBlob(node_proto, 6)), 0, "Unsupported non zero initial_c"); + Mat h0 = getBlob(node_proto, 5); + Mat c0 = getBlob(node_proto, 6); + b = b.reshape(1, b.size[0]); const int numHidden = lstmParams.get("hidden_size"); @@ -934,11 +935,15 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) } Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + c0 = c0.reshape(1, c0.size[0] * c0.size[1]); - lstmParams.blobs.resize(3); + lstmParams.blobs.resize(5); lstmParams.blobs[0] = Wh; lstmParams.blobs[1] = Wx; lstmParams.blobs[2] = b; + lstmParams.blobs[3] = h0; + lstmParams.blobs[4] = c0; lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 426710989e48..01fa0df985b7 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -1838,8 +1838,8 @@ void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::Nod // op: "BlockLSTM" // input: "lstm_block_wrapper/ToInt64/x" (ignore, number of time stamps) // input: "input" - // input: "lstm_block_wrapper/zeros" (ignore) - // input: "lstm_block_wrapper/zeros" (ignore) + // input: "lstm_block_wrapper/zeros" + // input: "lstm_block_wrapper/zeros" // input: "lstm_block_wrapper/kernel" // input: "lstm_block_wrapper/w_i_diag" // input: "lstm_block_wrapper/w_f_diag" @@ -1865,9 +1865,11 @@ void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::Nod } } - Mat W, Wh, Wx, b; + Mat W, Wh, Wx, b, cs_prev, h_prev; blobFromTensor(getConstBlob(layer, value_id, 4), W); blobFromTensor(getConstBlob(layer, value_id, 8), b); + blobFromTensor(getConstBlob(layer, value_id, 2), cs_prev); + blobFromTensor(getConstBlob(layer, value_id, 3), h_prev); const int outSize = W.cols / 4; // IGFO->IFOG @@ -1883,10 +1885,12 @@ void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::Nod Wx = W.rowRange(0, W.rows - outSize).t(); Wh = W.rowRange(W.rows - outSize, W.rows).t(); - layerParams.blobs.resize(3); + layerParams.blobs.resize(5); layerParams.blobs[0] = Wh; layerParams.blobs[1] = Wx; layerParams.blobs[2] = b; + layerParams.blobs[3] = h_prev; + layerParams.blobs[4] = cs_prev; if (hasLayerAttr(layer, "use_peephole")) { @@ -1894,14 +1898,14 @@ void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::Nod if (usePeephole) { layerParams.set("use_peephole", true); - layerParams.blobs.resize(6); + layerParams.blobs.resize(8); for (int i = 0; i < 3; ++i) { Mat w; blobFromTensor(getConstBlob(layer, value_id, 5 + i), w); w = w.reshape(1, w.total()); // Single column. w = Mat::diag(w); // Make a diagonal matrix. - layerParams.blobs[3 + i] = w; + layerParams.blobs[5 + i] = w; } } } diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp index 897603d274ca..fbe9605e7f9c 100644 --- a/modules/dnn/test/test_layers.cpp +++ b/modules/dnn/test/test_layers.cpp @@ -434,7 +434,7 @@ class Layer_LSTM_Test : public ::testing::Test { public: int numInp, numOut; - Mat Wh, Wx, b; + Mat Wh, Wx, b, h, c; Ptr layer; std::vector inputs, outputs; @@ -449,12 +449,17 @@ class Layer_LSTM_Test : public ::testing::Test Wh = Mat::ones(4 * numOut, numOut, CV_32F); Wx = Mat::ones(4 * numOut, numInp, CV_32F); b = Mat::ones(4 * numOut, 1, CV_32F); + h = Mat::ones(4, numOut, CV_32F); + c = Mat::ones(4, numOut, CV_32F); LayerParams lp; - lp.blobs.resize(3); + lp.blobs.resize(5); lp.blobs[0] = Wh; lp.blobs[1] = Wx; lp.blobs[2] = b; + lp.blobs[3] = h; + lp.blobs[4] = c; + lp.set("produce_cell_output", produceCellOutput); lp.set("use_timestamp_dim", useTimestampDim); @@ -502,10 +507,12 @@ TEST_F(Layer_LSTM_Test, get_set_test) TEST(Layer_LSTM_Test_Accuracy_with_, CaffeRecurrent) { LayerParams lp; - lp.blobs.resize(3); + lp.blobs.resize(5); lp.blobs[0] = blobFromNPY(_tf("lstm.prototxt.w_2.npy")); // Wh lp.blobs[1] = blobFromNPY(_tf("lstm.prototxt.w_0.npy")); // Wx lp.blobs[2] = blobFromNPY(_tf("lstm.prototxt.w_1.npy")); // bias + lp.blobs[3] = Mat::zeros(2, 17, CV_32F); // h_0 + lp.blobs[4] = Mat::zeros(2, 17, CV_32F); // c_0 Ptr layer = LSTMLayer::create(lp); Mat inp = blobFromNPY(_tf("recurrent.input.npy")); @@ -516,6 +523,68 @@ TEST(Layer_LSTM_Test_Accuracy_with_, CaffeRecurrent) normAssert(h_t_reference, outputs[0]); } +TEST(Layer_LSTM_Test_Accuracy_with_, HiddenParams) +{ + Mat Wx = blobFromNPY(_tf("lstm.hidden.W.npy")); + Mat Wh = blobFromNPY(_tf("lstm.hidden.R.npy")); + Mat b = blobFromNPY(_tf("lstm.hidden.B.npy")); + Mat h0 = blobFromNPY(_tf("lstm.hidden.h0.npy")); + Mat c0 = blobFromNPY(_tf("lstm.hidden.c0.npy")); + + const int numHidden = 3; + const int numDirs = Wx.size[0]; + const int numFeatures = Wx.size[2]; + + b = b.reshape(1, b.size[0]); + Mat bx = b.colRange(0, b.cols / 2); + Mat bh = b.colRange(b.cols / 2, b.cols); + b = bx + bh; + + // IFGO->IGFO + for (int k = 0; k < numDirs; ++k) + { + float* WxData = Wx.ptr(k); + float* WhData = Wh.ptr(k); + float* biasData = b.ptr(k); + for (int j = 0; j < numHidden; ++j) + { + for (int i = 0; i < numFeatures; ++i) + { + std::swap(WxData[(numHidden + j) * numFeatures + i], + WxData[(numHidden * 2 + j) * numFeatures + i]); + } + for (int i = 0; i < numHidden; ++i) + { + std::swap(WhData[(numHidden + j) * numHidden + i], + WhData[(numHidden * 2 + j) * numHidden + i]); + } + std::swap(biasData[numHidden + j], biasData[numHidden * 2 + j]); + } + } + + Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); + Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + c0 = c0.reshape(1, c0.size[0] * c0.size[1]); + + LayerParams lstmParams; + lstmParams.blobs.resize(5); + lstmParams.blobs[0] = Wh; + lstmParams.blobs[1] = Wx; + lstmParams.blobs[2] = b; + lstmParams.blobs[3] = h0; + lstmParams.blobs[4] = c0; + lstmParams.set("bidirectional", false); + Ptr layer = LSTMLayer::create(lstmParams); + + Mat inp = blobFromNPY(_tf("lstm.hidden.input.npy")); + std::vector inputs(1, inp), outputs; + runLayer(layer, inputs, outputs); + + Mat h_t_reference = blobFromNPY(_tf("lstm.hidden.output.npy")); + normAssert(h_t_reference, outputs[0]); +} + TEST(Layer_RNN_Test_Accuracy_with_, CaffeRecurrent) { Ptr layer = RNNLayer::create(LayerParams()); @@ -560,6 +629,9 @@ TEST(Layer_LSTM_Test_Accuracy_, Reverse) bias.at(2, 0) = 1e10f; // Output gate - always output everything bias.at(3, 0) = 0.f; // Update signal + cv::Mat hInternal = cv::Mat::zeros(1, 1, CV_32FC1); + cv::Mat cInternal = cv::Mat::zeros(1, 1, CV_32FC1); + LayerParams lp; lp.set("reverse", true); lp.set("use_timestamp_dim", true); @@ -567,6 +639,8 @@ TEST(Layer_LSTM_Test_Accuracy_, Reverse) lp.blobs.push_back(Wh); lp.blobs.push_back(Wx); lp.blobs.push_back(bias); + lp.blobs.push_back(hInternal); + lp.blobs.push_back(cInternal); cv::Ptr layer = LSTMLayer::create(lp); std::vector outputs; diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 3923068dbf17..05f77730af07 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -675,6 +675,16 @@ TEST_P(Test_ONNX_layers, LSTM_bidirectional) testONNXModels("lstm_bidirectional", npy, 0, 0, false, false); } +TEST_P(Test_ONNX_layers, LSTM_hidden) +{ + testONNXModels("hidden_lstm", npy, 0, 0, false, false); +} + +TEST_P(Test_ONNX_layers, LSTM_hidden_bidirectional) +{ + testONNXModels("hidden_lstm_bi", npy, 0, 0, false, false); +} + TEST_P(Test_ONNX_layers, Pad2d_Unfused) { testONNXModels("ReflectionPad2d"); From 7aa922ceac7d61c0a2b6c8c76debb70e256f6e2f Mon Sep 17 00:00:00 2001 From: Parsa Date: Sat, 24 Jul 2021 19:59:24 +0430 Subject: [PATCH 080/376] Merge pull request #20440 from parsa-ra:patch-1 * Update config_reference.markdown Added description for `WITH_CLP` build option. * Added extra description Can't cross-reference with anchors to other sections of the markdown file due to the presence of markdown link extension in the form of `## Header {#id-of-header}` * Fixed trailing space issue --- .../config_reference/config_reference.markdown | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/tutorials/introduction/config_reference/config_reference.markdown b/doc/tutorials/introduction/config_reference/config_reference.markdown index 58b4ed55ca41..0ba5627249ec 100644 --- a/doc/tutorials/introduction/config_reference/config_reference.markdown +++ b/doc/tutorials/introduction/config_reference/config_reference.markdown @@ -589,6 +589,14 @@ Some features have been added specifically for automated build environments, lik | `OPENCV_CMAKE_HOOKS_DIR` | _empty_ | OpenCV allows to customize configuration process by adding custom hook scripts at each stage and substage. cmake scripts with predefined names located in the directory set by this variable will be included before and after various configuration stages. Examples of file names: _CMAKE_INIT.cmake_, _PRE_CMAKE_BOOTSTRAP.cmake_, _POST_CMAKE_BOOTSTRAP.cmake_, etc.. Other names are not documented and can be found in the project cmake files by searching for the _ocv_cmake_hook_ macro calls. | | `OPENCV_DUMP_HOOKS_FLOW` | _OFF_ | Enables a debug message print on each cmake hook script call. | +## Contrib Modules + +Following build options are utilized in `opencv_contrib` modules, as stated [previously](#tutorial_config_reference_general_contrib), these extra modules can be added to your final build by setting `DOPENCV_EXTRA_MODULES_PATH` option. + +| Option | Default | Description | +| ------ | ------- | ----------- | +| `WITH_CLP` | _OFF_ | Will add [coinor](https://projects.coin-or.org/Clp) linear programming library build support which is required in `videostab` module. Make sure to install the development libraries of coinor-clp. | + # Other non-documented options @@ -605,7 +613,6 @@ Some features have been added specifically for automated build environments, lik `WITH_CPUFEATURES` `WITH_EIGEN` `WITH_OPENVX` -`WITH_CLP` `WITH_DIRECTX` `WITH_VA` `WITH_LAPACK` From 2f180cea7f6e2519aed9665c37dafdf4ab0e80ba Mon Sep 17 00:00:00 2001 From: Giles Payne Date: Sun, 25 Jul 2021 14:23:52 +0900 Subject: [PATCH 081/376] Add Quicklook for Mat on iOS and macOS --- .../imgcodecs/misc/objc/ios/Mat+Converters.h | 2 +- .../imgcodecs/misc/objc/ios/Mat+Converters.mm | 2 +- .../imgcodecs/misc/objc/ios/Mat+QuickLook.h | 27 +++ .../imgcodecs/misc/objc/ios/Mat+QuickLook.mm | 155 ++++++++++++++++++ .../misc/objc/macosx/Mat+QuickLook.h | 27 +++ .../misc/objc/macosx/Mat+QuickLook.mm | 154 +++++++++++++++++ 6 files changed, 365 insertions(+), 2 deletions(-) create mode 100644 modules/imgcodecs/misc/objc/ios/Mat+QuickLook.h create mode 100644 modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm create mode 100644 modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.h create mode 100644 modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm diff --git a/modules/imgcodecs/misc/objc/ios/Mat+Converters.h b/modules/imgcodecs/misc/objc/ios/Mat+Converters.h index a3ee005c18be..0f74bb2f5dc7 100644 --- a/modules/imgcodecs/misc/objc/ios/Mat+Converters.h +++ b/modules/imgcodecs/misc/objc/ios/Mat+Converters.h @@ -1,5 +1,5 @@ // -// Mat+UIImage.h +// Mat+Converters.h // // Created by Giles Payne on 2020/03/03. // diff --git a/modules/imgcodecs/misc/objc/ios/Mat+Converters.mm b/modules/imgcodecs/misc/objc/ios/Mat+Converters.mm index 69250eb99415..79358cb6de7f 100644 --- a/modules/imgcodecs/misc/objc/ios/Mat+Converters.mm +++ b/modules/imgcodecs/misc/objc/ios/Mat+Converters.mm @@ -1,5 +1,5 @@ // -// Mat+UIImage.mm +// Mat+Converters.mm // // Created by Giles Payne on 2020/03/03. // diff --git a/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.h b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.h new file mode 100644 index 000000000000..341172798ed4 --- /dev/null +++ b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.h @@ -0,0 +1,27 @@ +// +// Mat+QuickLook.h +// +// Created by Giles Payne on 2021/07/18. +// + +#pragma once + +#ifdef __cplusplus +#import "opencv2/core.hpp" +#else +#define CV_EXPORTS +#endif + +#import "Mat.h" +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +CV_EXPORTS @interface Mat (QuickLook) + +- (id)debugQuickLookObject; + +@end + +NS_ASSUME_NONNULL_END diff --git a/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm new file mode 100644 index 000000000000..7bfee07eb131 --- /dev/null +++ b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm @@ -0,0 +1,155 @@ +// +// Mat+QuickLook.mm +// +// Created by Giles Payne on 2021/07/18. +// + +#import "Mat+QuickLook.h" +#import "Mat+Converters.h" +#import "Rect2i.h" +#import "Core.h" +#import "Imgproc.h" +#import + +#define SIZE 20 + +static UIFont* getCMU() { + return [UIFont fontWithName:@"CMU Serif" size:SIZE]; +} + +static UIFont* getBodoni72() { + return [UIFont fontWithName:@"Bodoni 72" size:SIZE]; +} + +static UIFont* getAnySerif() { + if (@available(iOS 13.0, *)) { + return [UIFont fontWithDescriptor:[[UIFontDescriptor preferredFontDescriptorWithTextStyle:UIFontTextStyleBody] fontDescriptorWithDesign:UIFontDescriptorSystemDesignSerif] size:SIZE]; + } else { + return nil; + } +} + +static UIFont* getSystemFont() { + return [UIFont systemFontOfSize:SIZE]; +} + +typedef UIFont* (*FontGetter)(); + +@implementation Mat (QuickLook) + +- (NSString*)makeLabel:(BOOL)isIntType val:(NSNumber*)num { + if (isIntType) { + return [NSString stringWithFormat:@"%d", num.intValue]; + } else { + int exponent = 1 + (int)log10(abs(num.doubleValue)); + if (num.doubleValue == (double)num.intValue && num.doubleValue < 10000 && num.doubleValue > -10000) { + return [NSString stringWithFormat:@"%d", num.intValue];; + } else if (exponent <= 5 && exponent >= -1) { + return [NSString stringWithFormat:[NSString stringWithFormat:@"%%%d.%df", 6, MIN(5 - exponent, 4)], num.doubleValue]; + } else { + return [[[NSString stringWithFormat:@"%.2e", num.doubleValue] stringByReplacingOccurrencesOfString:@"e+0" withString:@"e"] stringByReplacingOccurrencesOfString:@"e-0" withString:@"e-"]; + } + } +} + +- (void)relativeLine:(UIBezierPath*)path relX:(CGFloat)x relY:(CGFloat)y { + CGPoint curr = path.currentPoint; + [path addLineToPoint:CGPointMake(curr.x + x, curr.y + y)]; +} + +- (id)debugQuickLookObject { + if ([self dims] == 2 && [self rows] <= 10 && [self cols] <= 10) { + FontGetter fontGetters[] = { getCMU, getBodoni72, getAnySerif, getSystemFont }; + UIFont* font = nil; + for (int fontGetterIndex = 0; font==nil && fontGetterIndex < (sizeof(fontGetters)) / (sizeof(fontGetters[0])); fontGetterIndex++) { + font = fontGetters[fontGetterIndex](); + } + int elements = [self rows] * [self cols]; + NSDictionary* textFontAttributes = @{ NSFontAttributeName: font, NSForegroundColorAttributeName: UIColor.blackColor }; + NSMutableArray* rawData = [NSMutableArray new]; + for (int dataIndex = 0; dataIndex < elements; dataIndex++) { + [rawData addObject:[NSNumber numberWithDouble:0]]; + } + [self get:0 col: 0 data: rawData]; + BOOL isIntType = [self depth] <= CV_32S; + NSMutableArray* labels = [NSMutableArray new]; + NSMutableDictionary* boundingRects = [NSMutableDictionary dictionaryWithCapacity:elements]; + int maxWidth = 0, maxHeight = 0; + for (NSNumber* number in rawData) { + NSString* label = [self makeLabel:isIntType val:number]; + [labels addObject:label]; + CGRect boundingRect = [label boundingRectWithSize:CGSizeMake(CGFLOAT_MAX, CGFLOAT_MAX) options:NSStringDrawingUsesLineFragmentOrigin attributes:textFontAttributes context:nil]; + if (boundingRect.size.width > maxWidth) { + maxWidth = boundingRect.size.width; + } + if (boundingRect.size.height > maxHeight) { + maxHeight = boundingRect.size.height; + } + boundingRects[label] = [NSValue valueWithCGRect:boundingRect]; + } + + int rowGap = 6; + int colGap = 6; + int borderGap = 8; + int lineThickness = 3; + int lipWidth = 6; + int imageWidth = 2 * (borderGap + lipWidth) + maxWidth * [self cols] + colGap * ([self cols] - 1); + int imageHeight = 2 * (borderGap + lipWidth) + maxHeight * [self rows] + rowGap * ([self rows] - 1); + + UIBezierPath* leftBracket = [UIBezierPath new]; + [leftBracket moveToPoint:CGPointMake(borderGap, borderGap)]; + [self relativeLine:leftBracket relX:0 relY:imageHeight - 2 * borderGap]; + [self relativeLine:leftBracket relX:lineThickness + lipWidth relY:0]; + [self relativeLine:leftBracket relX:0 relY:-lineThickness]; + [self relativeLine:leftBracket relX:-lipWidth relY:0]; + [self relativeLine:leftBracket relX:0 relY:-(imageHeight - 2 * (borderGap + lineThickness))]; + [self relativeLine:leftBracket relX:lipWidth relY:0]; + [self relativeLine:leftBracket relX:0 relY:-lineThickness]; + [leftBracket closePath]; + CGAffineTransform reflect = CGAffineTransformConcat(CGAffineTransformMakeTranslation(-imageWidth, 0), CGAffineTransformMakeScale(-1, 1)); + UIBezierPath* rightBracket = [leftBracket copy]; + [rightBracket applyTransform:reflect]; + + CGRect rect = CGRectMake(0, 0, imageWidth, imageHeight); + UIGraphicsBeginImageContextWithOptions(rect.size, false, 0.0); + [UIColor.whiteColor setFill]; + UIRectFill(rect); + [UIColor.blackColor setFill]; + [leftBracket fill]; + [rightBracket fill]; + [labels enumerateObjectsUsingBlock:^(id label, NSUInteger index, BOOL *stop) + { + CGRect boundingRect = boundingRects[label].CGRectValue; + int row = (int)index / [self cols]; + int col = (int)index % [self cols]; + int x = borderGap + lipWidth + col * (maxWidth + colGap) + (maxWidth - boundingRect.size.width) / 2; + int y = borderGap + lipWidth + row * (maxHeight + rowGap) + (maxHeight - boundingRect.size.height) / 2; + CGRect textRect = CGRectMake(x, y, boundingRect.size.width, boundingRect.size.height); + [label drawInRect:textRect withAttributes:textFontAttributes]; + }]; + UIImage* image = UIGraphicsGetImageFromCurrentImageContext(); + UIGraphicsEndImageContext(); + return image; + } else if (([self dims] == 2) && ([self type] == CV_8U || [self type] == CV_8UC3 || [self type] == CV_8UC4)) { + return [self toUIImage]; + } else if ([self dims] == 2 && [self channels] == 1) { + Mat* normalized = [Mat new]; + [Core normalize:self dst:normalized alpha:0 beta:255 norm_type:NORM_MINMAX dtype:CV_8U]; + Mat* normalizedKey = [[Mat alloc] initWithRows:[self rows] + 10 cols:[self cols] type:CV_8U]; + std::vector key; + for (int index = 0; index < [self cols]; index++) { + key.push_back((char)(index * 256 / [self cols])); + } + for (int index = 0; index < 10; index++) { + [normalizedKey put:@[[NSNumber numberWithInt:index], [NSNumber numberWithInt:0]] count:[self cols] byteBuffer:key.data()]; + } + [normalized copyTo:[normalizedKey submatRoi:[[Rect2i alloc] initWithX:0 y:10 width:[self cols] height:[self rows]]]]; + Mat* colorMap = [Mat new]; + [Imgproc applyColorMap:normalizedKey dst:colorMap colormap:COLORMAP_JET]; + [Imgproc cvtColor:colorMap dst:colorMap code:COLOR_BGR2RGB]; + return [colorMap toUIImage]; + } + return [self description]; +} + +@end diff --git a/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.h b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.h new file mode 100644 index 000000000000..9fa31aba399e --- /dev/null +++ b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.h @@ -0,0 +1,27 @@ +// +// Mat+QuickLook.h +// +// Created by Giles Payne on 2021/07/18. +// + +#pragma once + +#ifdef __cplusplus +#import "opencv2/core.hpp" +#else +#define CV_EXPORTS +#endif + +#import "Mat.h" +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +CV_EXPORTS @interface Mat (QuickLook) + +- (id)debugQuickLookObject; + +@end + +NS_ASSUME_NONNULL_END diff --git a/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm new file mode 100644 index 000000000000..6775f817806c --- /dev/null +++ b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm @@ -0,0 +1,154 @@ +// +// Mat+QuickLook.mm +// +// Created by Giles Payne on 2021/07/18. +// + +#import "Mat+QuickLook.h" +#import "Mat+Converters.h" +#import "Rect2i.h" +#import "Core.h" +#import "Imgproc.h" +#import + +#define SIZE 20 + +static NSFont* getCMU() { + return [NSFont fontWithName:@"CMU Serif" size:SIZE]; +} + +static NSFont* getBodoni72() { + return [NSFont fontWithName:@"Bodoni 72" size:SIZE]; +} + +static NSFont* getAnySerif() { + if (@available(macOS 11.0, *)) { + return [NSFont fontWithDescriptor:[[NSFontDescriptor preferredFontDescriptorForTextStyle:NSFontTextStyleBody options:@{}] fontDescriptorWithDesign:NSFontDescriptorSystemDesignSerif] size:SIZE]; + } else { + return nil; + } +} + +static NSFont* getSystemFont() { + return [NSFont systemFontOfSize:SIZE]; +} + +typedef NSFont* (*FontGetter)(); + +@implementation Mat (QuickLook) + +- (NSString*)makeLabel:(BOOL)isIntType val:(NSNumber*)num { + if (isIntType) { + return [NSString stringWithFormat:@"%d", num.intValue]; + } else { + int exponent = 1 + (int)log10(abs(num.doubleValue)); + if (num.doubleValue == (double)num.intValue && num.doubleValue < 10000 && num.doubleValue > -10000) { + return [NSString stringWithFormat:@"%d", num.intValue];; + } else if (exponent <= 5 && exponent >= -1) { + return [NSString stringWithFormat:[NSString stringWithFormat:@"%%%d.%df", 6, MIN(5 - exponent, 4)], num.doubleValue]; + } else { + return [[[NSString stringWithFormat:@"%.2e", num.doubleValue] stringByReplacingOccurrencesOfString:@"e+0" withString:@"e"] stringByReplacingOccurrencesOfString:@"e-0" withString:@"e-"]; + } + } +} + +- (id)debugQuickLookObject { + // for smallish Mat objects display as a matrix + if ([self dims] == 2 && [self rows] <= 10 && [self cols] <= 10) { + FontGetter fontGetters[] = { getCMU, getBodoni72, getAnySerif, getSystemFont }; + NSFont* font = nil; + for (int fontGetterIndex = 0; font==nil && fontGetterIndex < (sizeof(fontGetters)) / (sizeof(fontGetters[0])); fontGetterIndex++) { + font = fontGetters[fontGetterIndex](); + } + int elements = [self rows] * [self cols]; + NSDictionary* textFontAttributes = @{ NSFontAttributeName: font, NSForegroundColorAttributeName: NSColor.blackColor }; + NSMutableArray* rawData = [NSMutableArray new]; + for (int dataIndex = 0; dataIndex < elements; dataIndex++) { + [rawData addObject:[NSNumber numberWithDouble:0]]; + } + [self get:0 col: 0 data: rawData]; + BOOL isIntType = [self depth] <= CV_32S; + NSMutableArray* labels = [NSMutableArray new]; + NSMutableDictionary* boundingRects = [NSMutableDictionary dictionaryWithCapacity:elements]; + int maxWidth = 0, maxHeight = 0; + for (NSNumber* number in rawData) { + NSString* label = [self makeLabel:isIntType val:number]; + [labels addObject:label]; + NSRect boundingRect = [label boundingRectWithSize:NSMakeSize(CGFLOAT_MAX, CGFLOAT_MAX) options:NSStringDrawingUsesLineFragmentOrigin attributes:textFontAttributes]; + if (boundingRect.size.width > maxWidth) { + maxWidth = boundingRect.size.width; + } + if (boundingRect.size.height > maxHeight) { + maxHeight = boundingRect.size.height; + } + boundingRects[label] = [NSValue valueWithRect:boundingRect]; + } + + int rowGap = 8; + int colGap = 8; + int borderGap = 9; + int lineThickness = 4; + int lipWidth = 8; + int imageWidth = 2 * (borderGap + lipWidth) + maxWidth * [self cols] + colGap * ([self cols] - 1); + int imageHeight = 2 * (borderGap + lipWidth) + maxHeight * [self rows] + rowGap * ([self rows] - 1); + NSImage* image = [[NSImage alloc] initWithSize:NSMakeSize(imageWidth, imageHeight)]; + NSBezierPath* leftBracket = [NSBezierPath new]; + [leftBracket moveToPoint:NSMakePoint(borderGap, borderGap)]; + [leftBracket relativeLineToPoint:NSMakePoint(0, imageHeight - 2 * borderGap)]; + [leftBracket relativeLineToPoint:NSMakePoint(lineThickness + lipWidth, 0)]; + [leftBracket relativeLineToPoint:NSMakePoint(0, -lineThickness)]; + [leftBracket relativeLineToPoint:NSMakePoint(-lipWidth, 0)]; + [leftBracket relativeLineToPoint:NSMakePoint(0, -(imageHeight - 2 * (borderGap + lineThickness)))]; + [leftBracket relativeLineToPoint:NSMakePoint(lipWidth, 0)]; + [leftBracket relativeLineToPoint:NSMakePoint(0, -lineThickness)]; + [leftBracket relativeLineToPoint:NSMakePoint(-(lineThickness + lipWidth), 0)]; + NSAffineTransform* reflect = [NSAffineTransform new]; + [reflect scaleXBy:-1 yBy:1]; + [reflect translateXBy:-imageWidth yBy:0]; + NSBezierPath* rightBracket = [leftBracket copy]; + [rightBracket transformUsingAffineTransform:reflect]; + + [image lockFocus]; + [NSColor.whiteColor drawSwatchInRect:NSMakeRect(0, 0, imageWidth, imageHeight)]; + [NSColor.blackColor set]; + [leftBracket fill]; + [rightBracket fill]; + + [labels enumerateObjectsUsingBlock:^(id label, NSUInteger index, BOOL *stop) + { + NSRect boundingRect = boundingRects[label].rectValue; + int row = [self rows] - 1 - ((int)index / [self cols]); + int col = (int)index % [self cols]; + int x = borderGap + lipWidth + col * (maxWidth + colGap) + (maxWidth - boundingRect.size.width) / 2; + int y = borderGap + lipWidth + row * (maxHeight + rowGap) + (maxHeight - boundingRect.size.height) / 2; + NSRect textRect = NSMakeRect(x, y, boundingRect.size.width, boundingRect.size.height); + [label drawInRect:textRect withAttributes:textFontAttributes]; + }]; + [image unlockFocus]; + return image; + } else if (([self dims] == 2) && ([self type] == CV_8U || [self type] == CV_8UC3 || [self type] == CV_8UC4)) { + // convert to NSImage if the Mats has 2 dimensions and a type and number of channels consistent with it being a image + return [self toNSImage]; + } else if ([self dims] == 2 && [self channels] == 1) { + // for other Mats with 2 dimensions and one channel - generate heat map + Mat* normalized = [Mat new]; + [Core normalize:self dst:normalized alpha:0 beta:255 norm_type:NORM_MINMAX dtype:CV_8U]; + Mat* normalizedKey = [[Mat alloc] initWithRows:[self rows] + 10 cols:[self cols] type:CV_8U]; + std::vector key; + for (int index = 0; index < [self cols]; index++) { + key.push_back((char)(index * 256 / [self cols])); + } + for (int index = 0; index < 10; index++) { + [normalizedKey put:@[[NSNumber numberWithInt:index], [NSNumber numberWithInt:0]] count:[self cols] byteBuffer:key.data()]; + } + [normalized copyTo:[normalizedKey submatRoi:[[Rect2i alloc] initWithX:0 y:10 width:[self cols] height:[self rows]]]]; + Mat* colorMap = [Mat new]; + [Imgproc applyColorMap:normalizedKey dst:colorMap colormap:COLORMAP_JET]; + [Imgproc cvtColor:colorMap dst:colorMap code:COLOR_BGR2RGB]; + return [colorMap toNSImage]; + } + //everything just return the Mat description + return [self description]; +} + +@end From 1e1984a586028ab233d26f0a4a5668653fc410ba Mon Sep 17 00:00:00 2001 From: Xerxes Battiwalla Date: Mon, 26 Jul 2021 14:54:27 +1000 Subject: [PATCH 082/376] Fixed typo in error message in OpenCVDetectCUDA.cmake There was a minor typo in the FATAL error message when the specified CUDA generation does not match any known generation --- cmake/OpenCVDetectCUDA.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/OpenCVDetectCUDA.cmake b/cmake/OpenCVDetectCUDA.cmake index c7cfebe50f51..ac29e600d379 100644 --- a/cmake/OpenCVDetectCUDA.cmake +++ b/cmake/OpenCVDetectCUDA.cmake @@ -102,7 +102,7 @@ if(CUDA_FOUND) if(CUDA_GENERATION) if(NOT ";${_generations};" MATCHES ";${CUDA_GENERATION};") string(REPLACE ";" ", " _generations "${_generations}") - message(FATAL_ERROR "ERROR: ${_generations} Generations are suppered.") + message(FATAL_ERROR "ERROR: ${_generations} Generations are supported.") endif() unset(CUDA_ARCH_BIN CACHE) unset(CUDA_ARCH_PTX CACHE) From cff0168f3a136b86ac4f415c8332abcd39212f67 Mon Sep 17 00:00:00 2001 From: rogday Date: Wed, 28 Jul 2021 18:06:24 +0300 Subject: [PATCH 083/376] Merge pull request #20453 from rogday:onnx_importer_fix Split layer dispatch into functions in ONNXImporter * split layer dispatch into functions * fixes * identation and comment fixes * fix constness --- modules/dnn/src/dnn_common.hpp | 1 + modules/dnn/src/onnx/onnx_importer.cpp | 2922 +++++++++++--------- modules/dnn/src/tensorflow/tf_importer.cpp | 2 +- 3 files changed, 1566 insertions(+), 1359 deletions(-) diff --git a/modules/dnn/src/dnn_common.hpp b/modules/dnn/src/dnn_common.hpp index cd6cea0c6b09..0f3feda91b4a 100644 --- a/modules/dnn/src/dnn_common.hpp +++ b/modules/dnn/src/dnn_common.hpp @@ -14,6 +14,7 @@ Mutex& getInitializationMutex(); void initializeLayerFactory(); namespace detail { +#define CALL_MEMBER_FN(object, ptrToMemFn) ((object).*(ptrToMemFn)) struct NetImplBase { diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 4ad0fd496e37..b833b2ea443f 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -62,7 +62,7 @@ class ONNXImporter public: ONNXImporter(Net& net, const char *onnxFile) - : dstNet(net) + : dstNet(net), dispatch(buildDispatchMap()) { hasDynamicShapes = false; CV_Assert(onnxFile); @@ -83,7 +83,7 @@ class ONNXImporter } ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer) - : dstNet(net) + : dstNet(net), dispatch(buildDispatchMap()) { hasDynamicShapes = false; CV_LOG_DEBUG(NULL, "DNN/ONNX: processing in-memory ONNX model (" << sizeBuffer << " bytes)"); @@ -124,6 +124,57 @@ class ONNXImporter typedef std::map::iterator IterLayerId_t; void handleNode(const opencv_onnx::NodeProto& node_proto); + +private: + typedef void (ONNXImporter::*ONNXImporterNodeParser)(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + typedef std::map DispatchMap; + + void parseMaxPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseAveragePool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseReduce (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSlice (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSplit (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseBias (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parsePow (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseNeg (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConstant (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLSTM (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseImageScaler (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseClip (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLeakyRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseElu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseTanh (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parsePRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLRN (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseBatchNormalization (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseGemm (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMatMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConv (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConvTranspose (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseTranspose (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSqueeze (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseFlatten (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseUnsqueeze (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseExpand (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseReshape (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parsePad (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseShape (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseCast (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConstantFill (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseGather (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseResize (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseUpsample (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSoftMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseCustom (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + + const DispatchMap dispatch; + static const DispatchMap buildDispatchMap(); }; inline void replaceLayerParam(LayerParams& layerParams, const String& oldKey, const String& newKey) @@ -448,13 +499,11 @@ void ONNXImporter::populateNet() CV_LOG_DEBUG(NULL, "DNN/ONNX: import completed!"); } -void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) +void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) { - opencv_onnx::NodeProto node_proto = node_proto_; // TODO FIXIT - CV_Assert(node_proto.output_size() >= 1); std::string name = node_proto.output(0); - std::string layer_type = node_proto.op_type(); + const std::string& layer_type = node_proto.op_type(); CV_LOG_DEBUG(NULL, "DNN/ONNX: processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) ); @@ -468,1537 +517,1694 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) layerParams.type = layer_type; layerParams.set("has_dynamic_shapes", hasDynamicShapes); - if (layer_type == "MaxPool") + DispatchMap::const_iterator iter = dispatch.find(layer_type); + if (iter != dispatch.end()) { - layerParams.type = "Pooling"; - layerParams.set("pool", "MAX"); - layerParams.set("ceil_mode", layerParams.has("pad_mode")); + CALL_MEMBER_FN(*this, iter->second)(layerParams, node_proto); } - else if (layer_type == "AveragePool") + else { - layerParams.type = "Pooling"; - layerParams.set("pool", "AVE"); - layerParams.set("ceil_mode", layerParams.has("pad_mode")); - layerParams.set("ave_pool_padded_area", framework_name == "pytorch"); + parseCustom(layerParams, node_proto); } - else if (layer_type == "GlobalAveragePool" || layer_type == "GlobalMaxPool" || - layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax") + } + catch (const cv::Exception& e) + { + CV_LOG_ERROR(NULL, "DNN/ONNX: ERROR during processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " + << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) + ); + for (int i = 0; i < node_proto.input_size(); i++) { - CV_Assert(node_proto.input_size() == 1); - layerParams.type = "Pooling"; - String pool; - if (layer_type == "GlobalMaxPool" || layer_type == "ReduceMax") - pool = "MAX"; - else if (layer_type == "ReduceSum") - pool = "SUM"; - else - pool = "AVE"; - layerParams.set("pool", pool); - layerParams.set("global_pooling", !layerParams.has("axes")); - if (layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) - { - MatShape inpShape = outShapes[node_proto.input(0)]; - DictValue axes = layerParams.get("axes"); - bool keepdims = layerParams.get("keepdims"); - MatShape targetShape; - std::vector shouldDelete(inpShape.size(), false); - for (int i = 0; i < axes.size(); i++) { - int axis = normalize_axis(axes.get(i), inpShape.size()); - shouldDelete[axis] = true; - } - for (int axis = 0; axis < inpShape.size(); ++axis){ - if (!shouldDelete[axis]) - targetShape.push_back(inpShape[axis]); - else if (keepdims) - targetShape.push_back(1); - } - - if (inpShape.size() == 3 && axes.size() <= 2) - { - int axis = normalize_axis(axes.get(0), inpShape.size()); - CV_CheckNE(axis, 0, ""); - - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("axis", 0); - reshapeLp.set("num_axes", 1); - int newShape[] = {1, -1}; - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 2)); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); - - LayerParams avgLp; - avgLp.name = layerParams.name + "/avg"; - avgLp.type = "Pooling"; - CV_Assert(layer_id.find(avgLp.name) == layer_id.end()); - avgLp.set("pool", pool); - if (axes.size() == 2) - { - CV_CheckEQ(normalize_axis(axes.get(0), inpShape.size()), 1, "Unsupported mode"); - CV_CheckEQ(normalize_axis(axes.get(1), inpShape.size()), 2, "Unsupported mode"); - avgLp.set("global_pooling", true); - } - else - { - avgLp.set(axis == 2 ? "global_pooling_w" : "global_pooling_h", true); - avgLp.set(axis == 2 ? "kernel_h" : "kernel_w", 1); - } - - node_proto.set_input(0, reshapeLp.name); - node_proto.set_output(0, avgLp.name); - addLayer(avgLp, node_proto); - } - else - { - if (inpShape.size() != 4 && inpShape.size() != 5) - CV_Error(Error::StsNotImplemented, "Unsupported input shape of " + layer_type + " operation."); + CV_LOG_INFO(NULL, " Input[" << i << "] = '" << node_proto.input(i) << "'"); + } + for (int i = 0; i < node_proto.output_size(); i++) + { + CV_LOG_INFO(NULL, " Output[" << i << "] = '" << node_proto.output(i) << "'"); + } + CV_Error(Error::StsError, cv::format("Node [%s]:(%s) parse error: %s", layer_type.c_str(), name.c_str(), e.what())); + } +} - CV_Assert(axes.size() <= inpShape.size() - 2); - std::vector kernel_size(inpShape.size() - 2, 1); - if (axes.size() == 1 && (normalize_axis(axes.get(0), inpShape.size()) <= 1)) - { - int axis = normalize_axis(axes.get(0), inpShape.size()); - MatShape newShape = inpShape; - newShape[axis + 1] = total(newShape, axis + 1); - newShape.resize(axis + 2); - newShape.insert(newShape.begin(), 2 - axis, 1); - - LayerParams reshapeLp; - reshapeLp.type = "Reshape"; - reshapeLp.name = layerParams.name + "/reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], newShape.size())); - - node_proto.set_output(0, reshapeLp.name); - addLayer(reshapeLp, node_proto); - - kernel_size.resize(2); - kernel_size[0] = inpShape[axis]; - node_proto.set_input(0, node_proto.output(0)); - } - else - { - for (int i = 0; i < axes.size(); i++) { - int axis = normalize_axis(axes.get(i), inpShape.size()); - CV_Assert_N(axis >= 2 + i, axis < inpShape.size()); - kernel_size[axis - 2] = inpShape[axis]; - } - } +void ONNXImporter::parseMaxPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Pooling"; + layerParams.set("pool", "MAX"); + layerParams.set("ceil_mode", layerParams.has("pad_mode")); + addLayer(layerParams, node_proto); +} - LayerParams poolLp = layerParams; - poolLp.name = layerParams.name + "/avg"; - CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); - poolLp.set("kernel_size", DictValue::arrayInt(&kernel_size[0], kernel_size.size())); +void ONNXImporter::parseAveragePool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Pooling"; + layerParams.set("pool", "AVE"); + layerParams.set("ceil_mode", layerParams.has("pad_mode")); + layerParams.set("ave_pool_padded_area", framework_name == "pytorch"); + addLayer(layerParams, node_proto); +} - node_proto.set_output(0, poolLp.name); - addLayer(poolLp, node_proto); - } +// "GlobalAveragePool" "GlobalMaxPool" "ReduceMean" "ReduceSum" "ReduceMax" +void ONNXImporter::parseReduce(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + const std::string& layer_type = node_proto.op_type(); + + CV_Assert(node_proto.input_size() == 1); + layerParams.type = "Pooling"; + String pool; + if (layer_type == "GlobalMaxPool" || layer_type == "ReduceMax") + pool = "MAX"; + else if (layer_type == "ReduceSum") + pool = "SUM"; + else + pool = "AVE"; + layerParams.set("pool", pool); + layerParams.set("global_pooling", !layerParams.has("axes")); + if (layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) + { + MatShape inpShape = outShapes[node_proto.input(0)]; + DictValue axes = layerParams.get("axes"); + bool keepdims = layerParams.get("keepdims"); + MatShape targetShape; + std::vector shouldDelete(inpShape.size(), false); + for (int i = 0; i < axes.size(); i++) { + int axis = normalize_axis(axes.get(i), inpShape.size()); + shouldDelete[axis] = true; + } + for (int axis = 0; axis < inpShape.size(); ++axis){ + if (!shouldDelete[axis]) + targetShape.push_back(inpShape[axis]); + else if (keepdims) + targetShape.push_back(1); + } + + if (inpShape.size() == 3 && axes.size() <= 2) + { + int axis = normalize_axis(axes.get(0), inpShape.size()); + CV_CheckNE(axis, 0, ""); + + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + reshapeLp.set("axis", 0); + reshapeLp.set("num_axes", 1); + int newShape[] = {1, -1}; + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 2)); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + + LayerParams avgLp; + avgLp.name = layerParams.name + "/avg"; + avgLp.type = "Pooling"; + CV_Assert(layer_id.find(avgLp.name) == layer_id.end()); + avgLp.set("pool", pool); + if (axes.size() == 2) + { + CV_CheckEQ(normalize_axis(axes.get(0), inpShape.size()), 1, "Unsupported mode"); + CV_CheckEQ(normalize_axis(axes.get(1), inpShape.size()), 2, "Unsupported mode"); + avgLp.set("global_pooling", true); + } + else + { + avgLp.set(axis == 2 ? "global_pooling_w" : "global_pooling_h", true); + avgLp.set(axis == 2 ? "kernel_h" : "kernel_w", 1); + } - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&targetShape[0], targetShape.size())); + node_proto.set_input(0, reshapeLp.name); + node_proto.set_output(0, avgLp.name); + addLayer(avgLp, node_proto); + } + else + { + if (inpShape.size() != 4 && inpShape.size() != 5) + CV_Error(Error::StsNotImplemented, "Unsupported input shape of " + layer_type + " operation."); - node_proto.set_input(0, node_proto.output(0)); - node_proto.set_output(0, layerParams.name); - } - else if (!layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) + CV_Assert(axes.size() <= inpShape.size() - 2); + std::vector kernel_size(inpShape.size() - 2, 1); + if (axes.size() == 1 && (normalize_axis(axes.get(0), inpShape.size()) <= 1)) { - CV_CheckEQ(layerParams.get("keepdims"), 0, "layer only supports keepdims = false"); + int axis = normalize_axis(axes.get(0), inpShape.size()); + MatShape newShape = inpShape; + newShape[axis + 1] = total(newShape, axis + 1); + newShape.resize(axis + 2); + newShape.insert(newShape.begin(), 2 - axis, 1); + LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; reshapeLp.type = "Reshape"; + reshapeLp.name = layerParams.name + "/reshape"; CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - int newShape[] = {1, 1, 1, -1}; - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 4)); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], newShape.size())); - LayerParams poolLp = layerParams; - poolLp.name = layerParams.name + "/pool"; - CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); - - node_proto.set_input(0, reshapeLp.name); - node_proto.set_output(0, poolLp.name); - addLayer(poolLp, node_proto); - - layerParams.type = "Reshape"; - int targetShape[] = {1}; - layerParams.set("dim", DictValue::arrayInt(&targetShape[0], 1)); + node_proto.set_output(0, reshapeLp.name); + addLayer(reshapeLp, node_proto); + kernel_size.resize(2); + kernel_size[0] = inpShape[axis]; node_proto.set_input(0, node_proto.output(0)); - node_proto.set_output(0, layerParams.name); } - } - else if (layer_type == "Slice") - { - int axis = 0; - std::vector begin; - std::vector end; - std::vector steps; - int inp_size = node_proto.input_size(); - - if (inp_size == 1) + else { - if (layerParams.has("axes")) { - DictValue axes = layerParams.get("axes"); - for (int i = 1; i < axes.size(); ++i) { - CV_Assert(axes.get(i - 1) == axes.get(i) - 1); - } - axis = axes.get(0); + for (int i = 0; i < axes.size(); i++) { + int axis = normalize_axis(axes.get(i), inpShape.size()); + CV_Assert_N(axis >= 2 + i, axis < inpShape.size()); + kernel_size[axis - 2] = inpShape[axis]; } + } - DictValue starts = layerParams.get("starts"); - DictValue ends = layerParams.get("ends"); - CV_Assert(starts.size() == ends.size()); - - if (axis > 0) { - begin.resize(axis, 0); - end.resize(axis, -1); - } - for (int i = 0; i < starts.size(); ++i) - { - begin.push_back(starts.get(i)); - int finish = ends.get(i); - end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim - } - } else { // inp_size > 1 - CV_Assert(inp_size >= 3); - for (int i = 1; i < inp_size; i++) { - CV_Assert(constBlobs.find(node_proto.input(i)) != constBlobs.end()); - } - Mat start_blob = getBlob(node_proto, 1); - Mat end_blob = getBlob(node_proto, 2); - CV_Assert(start_blob.total() == end_blob.total()); - - if (inp_size > 3) { - Mat axes_blob = getBlob(node_proto, 3); - const int* axes = (int*)axes_blob.data; - for (int i = 1; i < axes_blob.total(); ++i) { - CV_Assert(axes[i - 1] == axes[i] - 1); - } - axis = axes[0]; - } + LayerParams poolLp = layerParams; + poolLp.name = layerParams.name + "/avg"; + CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); + poolLp.set("kernel_size", DictValue::arrayInt(&kernel_size[0], kernel_size.size())); - const int* starts = start_blob.ptr(); - const int* ends = end_blob.ptr(); - if (axis > 0) { - begin.resize(axis, 0); - end.resize(axis, -1); - } - std::copy(starts, starts + start_blob.total(), std::back_inserter(begin)); - for (int i = 0; i < end_blob.total(); ++i) - { - int finish = ends[i]; - end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim - } + node_proto.set_output(0, poolLp.name); + addLayer(poolLp, node_proto); + } - if (inp_size == 5) { - CV_Assert(constBlobs.find(node_proto.input(4)) != constBlobs.end()); - Mat step_blob = getBlob(node_proto, 4); - const int* steps_ptr = step_blob.ptr(); + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&targetShape[0], targetShape.size())); - if (axis > 0) - steps.resize(axis, 1); + node_proto.set_input(0, node_proto.output(0)); + node_proto.set_output(0, layerParams.name); + } + else if (!layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) + { + CV_CheckEQ(layerParams.get("keepdims"), 0, "layer only supports keepdims = false"); + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + int newShape[] = {1, 1, 1, -1}; + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 4)); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + + LayerParams poolLp = layerParams; + poolLp.name = layerParams.name + "/pool"; + CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); + + node_proto.set_input(0, reshapeLp.name); + node_proto.set_output(0, poolLp.name); + addLayer(poolLp, node_proto); + + layerParams.type = "Reshape"; + int targetShape[] = {1}; + layerParams.set("dim", DictValue::arrayInt(&targetShape[0], 1)); + + node_proto.set_input(0, node_proto.output(0)); + node_proto.set_output(0, layerParams.name); + } + addLayer(layerParams, node_proto); +} - std::copy(steps_ptr, steps_ptr + step_blob.total(), std::back_inserter(steps)); +void ONNXImporter::parseSlice(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int axis = 0; + std::vector begin; + std::vector end; + std::vector steps; + int inp_size = node_proto.input_size(); - // Very strange application for Slice op with tensor reversing. - // We just workaround it for 2d constants. - if (constBlobs.find(node_proto.input(0)) != constBlobs.end() && - axis == 0 && - start_blob.at(0) == -1 && step_blob.at(0) == -1 && - end_blob.at(0) == std::numeric_limits::min()) - { - Mat inp = getBlob(node_proto, 0); - if (inp.dims == 2) - { - Mat flipped; - flip(inp, flipped, 0); - addConstant(layerParams.name, flipped); - return; - } - } - } + if (inp_size == 1) + { + if (layerParams.has("axes")) { + DictValue axes = layerParams.get("axes"); + for (int i = 1; i < axes.size(); ++i) { + CV_Assert(axes.get(i - 1) == axes.get(i) - 1); } - layerParams.set("begin", DictValue::arrayInt(&begin[0], begin.size())); - layerParams.set("end", DictValue::arrayInt(&end[0], end.size())); - layerParams.set("axis", axis); + axis = axes.get(0); + } - if (!steps.empty()) - layerParams.set("steps", DictValue::arrayInt(&steps[0], steps.size())); + DictValue starts = layerParams.get("starts"); + DictValue ends = layerParams.get("ends"); + CV_Assert(starts.size() == ends.size()); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat inp = getBlob(node_proto, 0); - std::vector inputs, sliced; - inputs.push_back(inp); - runLayer(layerParams, inputs, sliced); - CV_Assert(sliced.size() == 1); - addConstant(layerParams.name, sliced[0]); - return; - } + if (axis > 0) { + begin.resize(axis, 0); + end.resize(axis, -1); } - else if (layer_type == "Split") + for (int i = 0; i < starts.size(); ++i) { - if (layerParams.has("split")) - { - DictValue splits = layerParams.get("split"); - const int numSplits = splits.size(); - CV_Assert(numSplits > 1); - - std::vector slicePoints(numSplits - 1, splits.get(0)); - for (int i = 1; i < splits.size() - 1; ++i) - { - slicePoints[i] = slicePoints[i - 1] + splits.get(i - 1); - } - layerParams.set("slice_point", DictValue::arrayInt(&slicePoints[0], slicePoints.size())); - } - else - { - layerParams.set("num_split", node_proto.output_size()); - } - layerParams.type = "Slice"; + begin.push_back(starts.get(i)); + int finish = ends.get(i); + end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim } - else if (layer_type == "Add" || layer_type == "Sum" || layer_type == "Sub") - { - bool isSub = layer_type == "Sub"; - CV_CheckEQ(node_proto.input_size(), 2, ""); - bool is_const_0 = layer_id.find(node_proto.input(0)) == layer_id.end(); - bool is_const_1 = layer_id.find(node_proto.input(1)) == layer_id.end(); - if (is_const_0 && is_const_1) - { - Mat blob_0 = getBlob(node_proto, 0); - Mat blob_1 = getBlob(node_proto, 1); - CV_Assert(blob_0.size == blob_1.size); - Mat output = isSub ? (blob_0 - blob_1) : (blob_0 + blob_1); - addConstant(layerParams.name, output); - return; - } - else if (is_const_0 || is_const_1) - { - int const_blob_id = is_const_0 ? 0 : 1; - Mat blob = getBlob(node_proto, const_blob_id); - int blob_total = blob.total(); - if (blob_total == 1) { - layerParams.type = "Power"; - layerParams.set("shift", (isSub ? -1 : 1) * blob.ptr()[0]); - } - else { - MatShape inpShape = outShapes[node_proto.input(1 - const_blob_id)]; - if (shape(blob) == inpShape) - { - LayerParams constParams; - constParams.name = layerParams.name + "/const"; - constParams.type = "Const"; - constParams.blobs.push_back((isSub ? -1 : 1) * blob); - int id = dstNet.addLayer(constParams.name, constParams.type, constParams); - layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); - outShapes[constParams.name] = shape(blob); - - layerParams.type = "Eltwise"; - node_proto.set_input(const_blob_id, constParams.name); - } - else - { - layerParams.type = "Scale"; - layerParams.set("bias_term", true); - int axis = 1; - for (int i = 0; i < graph_proto.initializer_size(); i++) - { - opencv_onnx::TensorProto tensor_proto = graph_proto.initializer(i); - if (tensor_proto.name() == node_proto.input(const_blob_id)) - { - axis = inpShape.size() - tensor_proto.dims_size(); - break; - } - } - layerParams.set("axis", axis); - blob = blob.reshape(1, 1); - layerParams.blobs.push_back((isSub ? -1 : 1) * blob); - } - } - } - else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) - { - layerParams.type = "Eltwise"; - if (isSub) - { - static float subCoeffs[] = {1.f, -1.f}; - layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); - } - } - else - { - if (isSub) - { - LayerParams powerParams; - powerParams.name = layerParams.name + "/neg"; - powerParams.type = "Power"; - powerParams.set("scale", -1); - - //Create Power layer - int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(1)); - CV_Assert(layerId != layer_id.end()); - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); - outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - - //Replace input to Power - node_proto.set_input(1, powerParams.name); - } - layerParams.type = "Scale"; - layerParams.set("bias_term", true); - } + } else { // inp_size > 1 + CV_Assert(inp_size >= 3); + for (int i = 1; i < inp_size; i++) { + CV_Assert(constBlobs.find(node_proto.input(i)) != constBlobs.end()); } - else if (layer_type == "Pow") - { - if (layer_id.find(node_proto.input(1)) != layer_id.end()) - CV_Error(Error::StsNotImplemented, "Unsupported Pow op with variable power"); - - Mat blob = getBlob(node_proto, 1); - if (blob.total() != 1) - CV_Error(Error::StsNotImplemented, "Pow op supports only scalar power"); + Mat start_blob = getBlob(node_proto, 1); + Mat end_blob = getBlob(node_proto, 2); + CV_Assert(start_blob.total() == end_blob.total()); - blob.convertTo(blob, CV_32F); - layerParams.type = "Power"; - layerParams.set("power", blob.ptr()[0]); - } - else if (layer_type == "Max") - { - layerParams.type = "Eltwise"; - layerParams.set("operation", "max"); + if (inp_size > 3) { + Mat axes_blob = getBlob(node_proto, 3); + const int* axes = (int*)axes_blob.data; + for (int i = 1; i < axes_blob.total(); ++i) { + CV_Assert(axes[i - 1] == axes[i] - 1); + } + axis = axes[0]; } - else if (layer_type == "Neg") - { - layerParams.type = "Power"; - layerParams.set("scale", -1); + + const int* starts = start_blob.ptr(); + const int* ends = end_blob.ptr(); + if (axis > 0) { + begin.resize(axis, 0); + end.resize(axis, -1); } - else if (layer_type == "Constant") + std::copy(starts, starts + start_blob.total(), std::back_inserter(begin)); + for (int i = 0; i < end_blob.total(); ++i) { - CV_Assert(node_proto.input_size() == 0); - CV_Assert(layerParams.blobs.size() == 1); - addConstant(layerParams.name, layerParams.blobs[0]); - return; + int finish = ends[i]; + end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim } - else if (layer_type == "LSTM") - { - LayerParams lstmParams = layerParams; - lstmParams.name += "/lstm"; - - // https://pytorch.org/docs/stable/nn.html#lstm - CV_Assert(node_proto.input_size() == 7); - Mat Wx = getBlob(node_proto, 1); - Mat Wh = getBlob(node_proto, 2); - Mat b = getBlob(node_proto, 3); - Mat h0 = getBlob(node_proto, 5); - Mat c0 = getBlob(node_proto, 6); - - b = b.reshape(1, b.size[0]); - - const int numHidden = lstmParams.get("hidden_size"); - const int numDirs = Wx.size[0]; // Is 1 for forward only and 2 for bidirectional LSTM. - const int numFeatures = Wx.size[2]; - Mat bx = b.colRange(0, b.cols / 2); - Mat bh = b.colRange(b.cols / 2, b.cols); - b = bx + bh; - - // IFGO->IGFO - for (int k = 0; k < numDirs; ++k) + + if (inp_size == 5) { + CV_Assert(constBlobs.find(node_proto.input(4)) != constBlobs.end()); + Mat step_blob = getBlob(node_proto, 4); + const int* steps_ptr = step_blob.ptr(); + + if (axis > 0) + steps.resize(axis, 1); + + std::copy(steps_ptr, steps_ptr + step_blob.total(), std::back_inserter(steps)); + + // Very strange application for Slice op with tensor reversing. + // We just workaround it for 2d constants. + if (constBlobs.find(node_proto.input(0)) != constBlobs.end() && + axis == 0 && + start_blob.at(0) == -1 && step_blob.at(0) == -1 && + end_blob.at(0) == std::numeric_limits::min()) { - float* WxData = Wx.ptr(k); - float* WhData = Wh.ptr(k); - float* biasData = b.ptr(k); - for (int j = 0; j < numHidden; ++j) + Mat inp = getBlob(node_proto, 0); + if (inp.dims == 2) { - for (int i = 0; i < numFeatures; ++i) - { - std::swap(WxData[(numHidden + j) * numFeatures + i], - WxData[(numHidden * 2 + j) * numFeatures + i]); - } - for (int i = 0; i < numHidden; ++i) - { - std::swap(WhData[(numHidden + j) * numHidden + i], - WhData[(numHidden * 2 + j) * numHidden + i]); - } - std::swap(biasData[numHidden + j], biasData[numHidden * 2 + j]); + Mat flipped; + flip(inp, flipped, 0); + addConstant(layerParams.name, flipped); + return; } } - Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); - Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); - h0 = h0.reshape(1, h0.size[0] * h0.size[1]); - c0 = c0.reshape(1, c0.size[0] * c0.size[1]); + } + } + layerParams.set("begin", DictValue::arrayInt(&begin[0], begin.size())); + layerParams.set("end", DictValue::arrayInt(&end[0], end.size())); + layerParams.set("axis", axis); - lstmParams.blobs.resize(5); - lstmParams.blobs[0] = Wh; - lstmParams.blobs[1] = Wx; - lstmParams.blobs[2] = b; - lstmParams.blobs[3] = h0; - lstmParams.blobs[4] = c0; - lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); + if (!steps.empty()) + layerParams.set("steps", DictValue::arrayInt(&steps[0], steps.size())); - node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name - addLayer(lstmParams, node_proto); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inp = getBlob(node_proto, 0); + std::vector inputs, sliced; + inputs.push_back(inp); + runLayer(layerParams, inputs, sliced); + CV_Assert(sliced.size() == 1); + addConstant(layerParams.name, sliced[0]); + return; + } + addLayer(layerParams, node_proto); +} - MatShape lstmShape = outShapes[node_proto.output(0)]; +void ONNXImporter::parseSplit(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (layerParams.has("split")) + { + DictValue splits = layerParams.get("split"); + const int numSplits = splits.size(); + CV_Assert(numSplits > 1); - // Add fake 1 as it is done in ONNX - lstmShape.insert(lstmShape.begin() + 1, 1); + std::vector slicePoints(numSplits - 1, splits.get(0)); + for (int i = 1; i < splits.size() - 1; ++i) + { + slicePoints[i] = slicePoints[i - 1] + splits.get(i - 1); + } + layerParams.set("slice_point", DictValue::arrayInt(&slicePoints[0], slicePoints.size())); + } + else + { + layerParams.set("num_split", node_proto.output_size()); + } + layerParams.type = "Slice"; + addLayer(layerParams, node_proto); +} - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&lstmShape[0], lstmShape.size())); - node_proto.set_input(0, lstmParams.name); // redirect input to LSTM - node_proto.set_output(0, layerParams.name); // keep origin LSTM's name +// "Add" "Sum" "Sub" +void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + const std::string& layer_type = node_proto.op_type(); + bool isSub = layer_type == "Sub"; + CV_CheckEQ(node_proto.input_size(), 2, ""); + bool is_const_0 = layer_id.find(node_proto.input(0)) == layer_id.end(); + bool is_const_1 = layer_id.find(node_proto.input(1)) == layer_id.end(); + if (is_const_0 && is_const_1) + { + Mat blob_0 = getBlob(node_proto, 0); + Mat blob_1 = getBlob(node_proto, 1); + CV_Assert(blob_0.size == blob_1.size); + Mat output = isSub ? (blob_0 - blob_1) : (blob_0 + blob_1); + addConstant(layerParams.name, output); + return; + } + else if (is_const_0 || is_const_1) + { + int const_blob_id = is_const_0 ? 0 : 1; + Mat blob = getBlob(node_proto, const_blob_id); + int blob_total = blob.total(); + if (blob_total == 1) { + layerParams.type = "Power"; + layerParams.set("shift", (isSub ? -1 : 1) * blob.ptr()[0]); } - else if (layer_type == "ImageScaler") - { - const float scale = layerParams.has("scale") ? layerParams.get("scale") : 1.0f; - layerParams.erase("scale"); + else { + MatShape inpShape = outShapes[node_proto.input(1 - const_blob_id)]; + if (shape(blob) == inpShape) + { + LayerParams constParams; + constParams.name = layerParams.name + "/const"; + constParams.type = "Const"; + constParams.blobs.push_back((isSub ? -1 : 1) * blob); + int id = dstNet.addLayer(constParams.name, constParams.type, constParams); + layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); + outShapes[constParams.name] = shape(blob); - if (layerParams.has("bias")) + layerParams.type = "Eltwise"; + node_proto.set_input(const_blob_id, constParams.name); + } + else { layerParams.type = "Scale"; - layerParams.blobs.push_back( - Mat(Size(1, layerParams.get("bias").size()), CV_32FC1, scale)); - layerParams.set("bias_term", true); - Mat bias(1, layerParams.get("bias").size(), CV_32FC1); - for (int j = 0; j < bias.total(); j++) { - bias.at(0, j) = layerParams.get("bias").getRealValue(j); + int axis = 1; + for (int i = 0; i < graph_proto.initializer_size(); i++) + { + opencv_onnx::TensorProto tensor_proto = graph_proto.initializer(i); + if (tensor_proto.name() == node_proto.input(const_blob_id)) + { + axis = inpShape.size() - tensor_proto.dims_size(); + break; + } } - layerParams.blobs.push_back(bias); - layerParams.erase("bias"); - } - else { - layerParams.set("scale", scale); - layerParams.type = "Power"; + layerParams.set("axis", axis); + blob = blob.reshape(1, 1); + layerParams.blobs.push_back((isSub ? -1 : 1) * blob); } } - else if (layer_type == "Clip") - { - layerParams.type = "ReLU6"; - replaceLayerParam(layerParams, "min", "min_value"); - replaceLayerParam(layerParams, "max", "max_value"); - - } - else if (layer_type == "LeakyRelu") - { - layerParams.type = "ReLU"; - replaceLayerParam(layerParams, "alpha", "negative_slope"); - } - else if (layer_type == "Relu") - { - layerParams.type = "ReLU"; - } - else if (layer_type == "Elu") - { - layerParams.type = "ELU"; - } - else if (layer_type == "Tanh") - { - layerParams.type = "TanH"; - } - else if (layer_type == "PRelu") - { - layerParams.type = "PReLU"; - layerParams.blobs.push_back(getBlob(node_proto, 1)); - } - else if (layer_type == "LRN") + } + else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) + { + layerParams.type = "Eltwise"; + if (isSub) { - replaceLayerParam(layerParams, "size", "local_size"); + static float subCoeffs[] = {1.f, -1.f}; + layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); } - else if (layer_type == "InstanceNormalization") + } + else + { + if (isSub) { - if (node_proto.input_size() != 3) - CV_Error(Error::StsNotImplemented, - "Expected input, scale, bias"); - - layerParams.blobs.resize(4); - layerParams.blobs[2] = getBlob(node_proto, 1); // weightData - layerParams.blobs[3] = getBlob(node_proto, 2); // biasData - layerParams.set("has_bias", true); - layerParams.set("has_weight", true); - - // Get number of channels in input - int size = layerParams.blobs[2].total(); - layerParams.blobs[0] = Mat::zeros(size, 1, CV_32F); // mean - layerParams.blobs[1] = Mat::ones(size, 1, CV_32F); // std - - LayerParams mvnParams; - mvnParams.name = layerParams.name + "/MVN"; - mvnParams.type = "MVN"; - mvnParams.set("eps", layerParams.get("epsilon")); - layerParams.erase("epsilon"); - - //Create MVN layer - int id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); + LayerParams powerParams; + powerParams.name = layerParams.name + "/neg"; + powerParams.type = "Power"; + powerParams.set("scale", -1); + + //Create Power layer + int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(0)); + IterLayerId_t layerId = layer_id.find(node_proto.input(1)); CV_Assert(layerId != layer_id.end()); dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); //Add shape - layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0))); - outShapes[mvnParams.name] = outShapes[node_proto.input(0)]; + layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); + outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - //Replace Batch Norm's input to MVN - node_proto.set_input(0, mvnParams.name); - layerParams.type = "BatchNorm"; + //Replace input to Power + node_proto.set_input(1, powerParams.name); } - else if (layer_type == "BatchNormalization") - { - if (node_proto.input_size() != 5) - CV_Error(Error::StsNotImplemented, - "Expected input, scale, bias, mean and var"); - - layerParams.type = "BatchNorm"; - replaceLayerParam(layerParams, "epsilon", "eps"); - replaceLayerParam(layerParams, "spatial", "use_global_stats"); - - Mat meanData = getBlob(node_proto, 3); - Mat stdData = getBlob(node_proto, 4); + layerParams.type = "Scale"; + layerParams.set("bias_term", true); + } + addLayer(layerParams, node_proto); +} - layerParams.blobs.push_back(meanData); - layerParams.blobs.push_back(stdData); +void ONNXImporter::parsePow(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (layer_id.find(node_proto.input(1)) != layer_id.end()) + CV_Error(Error::StsNotImplemented, "Unsupported Pow op with variable power"); - if (!node_proto.input(1).empty()) { - layerParams.set("has_weight", true); - layerParams.blobs.push_back(getBlob(node_proto, 1)); // weightData - } else { - layerParams.set("has_weight", false); - } + Mat blob = getBlob(node_proto, 1); + if (blob.total() != 1) + CV_Error(Error::StsNotImplemented, "Pow op supports only scalar power"); - if (!node_proto.input(2).empty()) { - layerParams.set("has_bias", true); - layerParams.blobs.push_back(getBlob(node_proto, 2)); // biasData - } else { - layerParams.set("has_bias", false); - } - } - else if (layer_type == "Gemm") - { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "InnerProduct"; - Mat weights = getBlob(node_proto, 1); - int ind_num_out = 0; - if (layerParams.has("transB") && !layerParams.get("transB")) { - transpose(weights, weights); - ind_num_out = 1; - } - layerParams.blobs.push_back(weights); + blob.convertTo(blob, CV_32F); + layerParams.type = "Power"; + layerParams.set("power", blob.ptr()[0]); + addLayer(layerParams, node_proto); +} - if (node_proto.input_size() == 3) { - Mat bias = getBlob(node_proto, 2); - layerParams.blobs.push_back(bias); - } - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat inputBuf = getBlob(node_proto, 0); +void ONNXImporter::parseMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Eltwise"; + layerParams.set("operation", "max"); + addLayer(layerParams, node_proto); +} - LayerParams constParams; - constParams.name = node_proto.input(0); - constParams.type = "Const"; - constParams.blobs.push_back(inputBuf); +void ONNXImporter::parseNeg(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Power"; + layerParams.set("scale", -1); + addLayer(layerParams, node_proto); +} - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); - } +void ONNXImporter::parseConstant(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 0); + CV_Assert(layerParams.blobs.size() == 1); + addConstant(layerParams.name, layerParams.blobs[0]); +} - layerParams.set("num_output", layerParams.blobs[0].size[ind_num_out]); - layerParams.set("bias_term", node_proto.input_size() == 3); - } - else if (layer_type == "MatMul") - { - CV_Assert(node_proto.input_size() == 2); - layerParams.type = "InnerProduct"; - layerParams.set("bias_term", false); - CV_Assert(constBlobs.find(node_proto.input(0)) == constBlobs.end()); - int firstInpDims = outShapes[node_proto.input(0)].size(); - int secondInpDims; - - if (constBlobs.find(node_proto.input(1)) != constBlobs.end()) - { - Mat blob = getBlob(node_proto, 1); - secondInpDims = blob.dims; - layerParams.blobs.push_back(blob.t()); - layerParams.set("num_output", layerParams.blobs[0].size[0]); - } else { - secondInpDims = outShapes[node_proto.input(1)].size(); - } - layerParams.set("axis", firstInpDims - secondInpDims + 1); - } - else if (layer_type == "Mul" || layer_type == "Div") +void ONNXImporter::parseLSTM(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + LayerParams lstmParams = layerParams; + lstmParams.name += "/lstm"; + + // https://pytorch.org/docs/stable/nn.html#lstm + CV_Assert(node_proto.input_size() == 7); + Mat Wx = getBlob(node_proto, 1); + Mat Wh = getBlob(node_proto, 2); + Mat b = getBlob(node_proto, 3); + Mat h0 = getBlob(node_proto, 5); + Mat c0 = getBlob(node_proto, 6); + + b = b.reshape(1, b.size[0]); + + const int numHidden = lstmParams.get("hidden_size"); + const int numDirs = Wx.size[0]; // Is 1 for forward only and 2 for bidirectional LSTM. + const int numFeatures = Wx.size[2]; + Mat bx = b.colRange(0, b.cols / 2); + Mat bh = b.colRange(b.cols / 2, b.cols); + b = bx + bh; + + // IFGO->IGFO + for (int k = 0; k < numDirs; ++k) + { + float* WxData = Wx.ptr(k); + float* WhData = Wh.ptr(k); + float* biasData = b.ptr(k); + for (int j = 0; j < numHidden; ++j) { - CV_Assert(node_proto.input_size() == 2); - - bool isDiv = layer_type == "Div"; - int constId = -1; - bool haveVariables = false; - for (int i = 0; i < 2; ++i) + for (int i = 0; i < numFeatures; ++i) { - if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) - constId = i; - else - haveVariables = true; + std::swap(WxData[(numHidden + j) * numFeatures + i], + WxData[(numHidden * 2 + j) * numFeatures + i]); } - if (constId != -1 && haveVariables) + for (int i = 0; i < numHidden; ++i) { - Mat blob = getBlob(node_proto, constId); - blob = blob.reshape(1, 1); - if (blob.total() == 1) { - float blob_value = blob.ptr()[0]; - float coeff = isDiv ? 1.0 / blob_value : blob_value; - layerParams.set("scale", coeff); - layerParams.type = "Power"; - } - else { - if (isDiv) - divide(1.0, blob, blob); - layerParams.blobs.push_back(blob); - layerParams.type = "Scale"; - } + std::swap(WhData[(numHidden + j) * numHidden + i], + WhData[(numHidden * 2 + j) * numHidden + i]); } - else if (!haveVariables) - { - Mat inp0 = getBlob(node_proto, 0); - Mat inp1 = getBlob(node_proto, 1); - - if (inp0.size != inp1.size && (inp0.total() != 1 || inp1.total() != 1)) - CV_Error_(Error::StsNotImplemented, ("Different shapes case is not supported with constant inputs: %s", layer_type.c_str())); + std::swap(biasData[numHidden + j], biasData[numHidden * 2 + j]); + } + } + Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); + Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + c0 = c0.reshape(1, c0.size[0] * c0.size[1]); + + lstmParams.blobs.resize(5); + lstmParams.blobs[0] = Wh; + lstmParams.blobs[1] = Wx; + lstmParams.blobs[2] = b; + lstmParams.blobs[3] = h0; + lstmParams.blobs[4] = c0; + lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); + + node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name + addLayer(lstmParams, node_proto); + + MatShape lstmShape = outShapes[node_proto.output(0)]; + + // Add fake 1 as it is done in ONNX + lstmShape.insert(lstmShape.begin() + 1, 1); + + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&lstmShape[0], lstmShape.size())); + node_proto.set_input(0, lstmParams.name); // redirect input to LSTM + node_proto.set_output(0, layerParams.name); // keep origin LSTM's name + addLayer(layerParams, node_proto); +} - if (inp0.total() == 1 && inp1.total() == 1 && inp0.dims != inp1.dims) - { - if (inp0.dims < inp1.dims) - { - inp0 = inp0.reshape(1, inp1.dims, inp1.size); - inp0.dims = inp1.dims; - } - else - { - inp1 = inp1.reshape(1, inp0.dims, inp0.size); - inp1.dims = inp0.dims; - } - } +void ONNXImporter::parseImageScaler(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + const float scale = layerParams.has("scale") ? layerParams.get("scale") : 1.0f; + layerParams.erase("scale"); - Mat out; - if (inp0.total() != inp1.total()) - { - if (inp0.total() == 1) - { - float inp0_value = inp0.ptr()[0]; - float coeff = isDiv ? 1.0 / inp0_value : inp0_value; - multiply(inp1, coeff, out); - } - else - { - float inp1_value = inp1.ptr()[0]; - float coeff = isDiv ? 1.0 / inp1_value : inp1_value; - multiply(inp0, coeff, out); - } + if (layerParams.has("bias")) + { + layerParams.type = "Scale"; + layerParams.blobs.push_back( + Mat(Size(1, layerParams.get("bias").size()), CV_32FC1, scale)); - } - else - { - out = isDiv ? inp0 / inp1 : inp0.mul(inp1); - } + layerParams.set("bias_term", true); + Mat bias(1, layerParams.get("bias").size(), CV_32FC1); + for (int j = 0; j < bias.total(); j++) { + bias.at(0, j) = layerParams.get("bias").getRealValue(j); + } + layerParams.blobs.push_back(bias); + layerParams.erase("bias"); + } + else { + layerParams.set("scale", scale); + layerParams.type = "Power"; + } + addLayer(layerParams, node_proto); +} - if (inp0.dims == 1 && inp1.dims == 1) - out.dims = 1; // to workaround dims == 1 - addConstant(layerParams.name, out); - return; - } - else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) - { - layerParams.type = "Eltwise"; - layerParams.set("operation", isDiv ? "div" : "prod"); - } - else - { - // Scale layer allocate output with the first input shape - if (total(outShapes[node_proto.input(0)]) < total(outShapes[node_proto.input(1)])) - { - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(1)); - proto.add_input(node_proto.input(0)); - proto.add_output(layerParams.name); - node_proto = proto; - } +void ONNXImporter::parseClip(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ReLU6"; + replaceLayerParam(layerParams, "min", "min_value"); + replaceLayerParam(layerParams, "max", "max_value"); + addLayer(layerParams, node_proto); +} - if (isDiv) - { - LayerParams powerParams; - powerParams.name = layerParams.name + "/inv"; - powerParams.type = "Power"; - powerParams.set("power", -1); - - //Create Power layer - int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(1)); - CV_Assert(layerId != layer_id.end()); - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); - outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - - //Replace input to Power - node_proto.set_input(1, powerParams.name); - } - layerParams.type = "Scale"; - } - } - else if (layer_type == "Conv") - { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "Convolution"; - for (int j = 1; j < node_proto.input_size(); j++) { - if (constBlobs.find(node_proto.input(j)) != constBlobs.end()) - { - layerParams.blobs.push_back(getBlob(node_proto, j)); - } - } - int outCn = layerParams.blobs.empty() ? outShapes[node_proto.input(1)][0] : layerParams.blobs[0].size[0]; - layerParams.set("num_output", outCn); +void ONNXImporter::parseLeakyRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ReLU"; + replaceLayerParam(layerParams, "alpha", "negative_slope"); + addLayer(layerParams, node_proto); +} - // Check for asymmetric padding in Conv2D - if (layerParams.has("pad")) - { - bool asymmetricPadding = false; - DictValue pads = layerParams.get("pad"); - const int dims = pads.size() / 2; - for (int i = 0; i < dims; ++i) - { - if (pads.get(i) != pads.get(i + dims)) - { - asymmetricPadding = true; - break; - } - } - if (asymmetricPadding && pads.size() == 4) // [pad_t, pad_l, pad_b, pad_r] - { - layerParams.erase("pad"); - // No paddings required for N, C axis - std::vector paddings(4, 0); - // Add paddings for H, W axis - for (int i = 0; i < dims; ++i) - { - paddings.push_back(pads.get(i)); - paddings.push_back(pads.get(dims + i)); - } - LayerParams padLp; - padLp.name = layerParams.name + "/pad"; - padLp.type = "Padding"; - padLp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size())); +void ONNXImporter::parseRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ReLU"; + addLayer(layerParams, node_proto); +} - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(padLp.name); +void ONNXImporter::parseElu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ELU"; + addLayer(layerParams, node_proto); +} - addLayer(padLp, proto); - node_proto.set_input(0, padLp.name); - } - } - } - else if (layer_type == "ConvTranspose") - { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "Deconvolution"; - for (int j = 1; j < node_proto.input_size(); j++) { - layerParams.blobs.push_back(getBlob(node_proto, j)); - } - layerParams.set("num_output", layerParams.blobs[0].size[1] * layerParams.get("group", 1)); - layerParams.set("bias_term", node_proto.input_size() == 3); +void ONNXImporter::parseTanh(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "TanH"; + addLayer(layerParams, node_proto); +} - if (!layerParams.has("kernel_size")) - CV_Error(Error::StsNotImplemented, - "Required attribute 'kernel_size' is not present."); +void ONNXImporter::parsePRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "PReLU"; + layerParams.blobs.push_back(getBlob(node_proto, 1)); + addLayer(layerParams, node_proto); +} - if (layerParams.has("output_shape")) - { - const DictValue& outShape = layerParams.get("output_shape"); - DictValue strides = layerParams.get("stride"); - DictValue kernel = layerParams.get("kernel_size"); +void ONNXImporter::parseLRN(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + replaceLayerParam(layerParams, "size", "local_size"); + addLayer(layerParams, node_proto); +} - String padMode; - std::vector adjust_pads; - if (layerParams.has("pad_mode")) - { - padMode = toUpperCase(layerParams.get("pad_mode")); - if (padMode != "SAME" && padMode != "VALID") - CV_Error(Error::StsError, "Unsupported padding mode " + padMode); +void ONNXImporter::parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + if (node_proto.input_size() != 3) + CV_Error(Error::StsNotImplemented, + "Expected input, scale, bias"); + + layerParams.blobs.resize(4); + layerParams.blobs[2] = getBlob(node_proto, 1); // weightData + layerParams.blobs[3] = getBlob(node_proto, 2); // biasData + layerParams.set("has_bias", true); + layerParams.set("has_weight", true); + + // Get number of channels in input + int size = layerParams.blobs[2].total(); + layerParams.blobs[0] = Mat::zeros(size, 1, CV_32F); // mean + layerParams.blobs[1] = Mat::ones(size, 1, CV_32F); // std + + LayerParams mvnParams; + mvnParams.name = layerParams.name + "/MVN"; + mvnParams.type = "MVN"; + mvnParams.set("eps", layerParams.get("epsilon")); + layerParams.erase("epsilon"); + + //Create MVN layer + int id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); + //Connect to input + IterLayerId_t layerId = layer_id.find(node_proto.input(0)); + CV_Assert(layerId != layer_id.end()); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0))); + outShapes[mvnParams.name] = outShapes[node_proto.input(0)]; + + //Replace Batch Norm's input to MVN + node_proto.set_input(0, mvnParams.name); + layerParams.type = "BatchNorm"; + addLayer(layerParams, node_proto); +} - for (int i = 0; i < strides.size(); i++) - { - int sz = outShape.get(2 + i); - int stride = strides.get(i); - adjust_pads.push_back(padMode == "SAME"? (sz - 1) % stride : - (sz - kernel.get(i)) % stride); - } - layerParams.set("adj", DictValue::arrayInt(&adjust_pads[0], adjust_pads.size())); - } - } - else if (layerParams.has("output_padding")) - { - replaceLayerParam(layerParams, "output_padding", "adj"); - } - } - else if (layer_type == "Transpose") - { - layerParams.type = "Permute"; - replaceLayerParam(layerParams, "perm", "order"); +void ONNXImporter::parseBatchNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (node_proto.input_size() != 5) + CV_Error(Error::StsNotImplemented, + "Expected input, scale, bias, mean and var"); - CV_Assert(node_proto.input_size() == 1); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - std::vector inputs(1, getBlob(node_proto, 0)), transposed; - runLayer(layerParams, inputs, transposed); - CV_Assert(transposed.size() == 1); - addConstant(layerParams.name, transposed[0]); - return; - } + layerParams.type = "BatchNorm"; + replaceLayerParam(layerParams, "epsilon", "eps"); + replaceLayerParam(layerParams, "spatial", "use_global_stats"); + + Mat meanData = getBlob(node_proto, 3); + Mat stdData = getBlob(node_proto, 4); + + layerParams.blobs.push_back(meanData); + layerParams.blobs.push_back(stdData); + + if (!node_proto.input(1).empty()) { + layerParams.set("has_weight", true); + layerParams.blobs.push_back(getBlob(node_proto, 1)); // weightData + } else { + layerParams.set("has_weight", false); + } + + if (!node_proto.input(2).empty()) { + layerParams.set("has_bias", true); + layerParams.blobs.push_back(getBlob(node_proto, 2)); // biasData + } else { + layerParams.set("has_bias", false); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseGemm(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() >= 2); + layerParams.type = "InnerProduct"; + Mat weights = getBlob(node_proto, 1); + int ind_num_out = 0; + if (layerParams.has("transB") && !layerParams.get("transB")) { + transpose(weights, weights); + ind_num_out = 1; + } + layerParams.blobs.push_back(weights); + + if (node_proto.input_size() == 3) { + Mat bias = getBlob(node_proto, 2); + layerParams.blobs.push_back(bias); + } + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inputBuf = getBlob(node_proto, 0); + + LayerParams constParams; + constParams.name = node_proto.input(0); + constParams.type = "Const"; + constParams.blobs.push_back(inputBuf); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + } + + layerParams.set("num_output", layerParams.blobs[0].size[ind_num_out]); + layerParams.set("bias_term", node_proto.input_size() == 3); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseMatMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 2); + layerParams.type = "InnerProduct"; + layerParams.set("bias_term", false); + CV_Assert(constBlobs.find(node_proto.input(0)) == constBlobs.end()); + int firstInpDims = outShapes[node_proto.input(0)].size(); + int secondInpDims; + + if (constBlobs.find(node_proto.input(1)) != constBlobs.end()) + { + Mat blob = getBlob(node_proto, 1); + secondInpDims = blob.dims; + layerParams.blobs.push_back(blob.t()); + layerParams.set("num_output", layerParams.blobs[0].size[0]); + } else { + secondInpDims = outShapes[node_proto.input(1)].size(); + } + layerParams.set("axis", firstInpDims - secondInpDims + 1); + addLayer(layerParams, node_proto); +} + +// "Mul" "Div" +void ONNXImporter::parseMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + const std::string& layer_type = node_proto.op_type(); + CV_Assert(node_proto.input_size() == 2); + + bool isDiv = layer_type == "Div"; + int constId = -1; + bool haveVariables = false; + for (int i = 0; i < 2; ++i) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + constId = i; + else + haveVariables = true; + } + if (constId != -1 && haveVariables) + { + Mat blob = getBlob(node_proto, constId); + blob = blob.reshape(1, 1); + if (blob.total() == 1) { + float blob_value = blob.ptr()[0]; + float coeff = isDiv ? 1.0 / blob_value : blob_value; + layerParams.set("scale", coeff); + layerParams.type = "Power"; } - else if (layer_type == "Squeeze") - { - CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes")); - DictValue axes_dict = layerParams.get("axes"); - MatShape inpShape = outShapes[node_proto.input(0)]; + else { + if (isDiv) + divide(1.0, blob, blob); + layerParams.blobs.push_back(blob); + layerParams.type = "Scale"; + } + } + else if (!haveVariables) + { + Mat inp0 = getBlob(node_proto, 0); + Mat inp1 = getBlob(node_proto, 1); - std::vector maskedAxes(inpShape.size(), false); - for (int i = 0; i < axes_dict.size(); ++i) + if (inp0.size != inp1.size && (inp0.total() != 1 || inp1.total() != 1)) + CV_Error_(Error::StsNotImplemented, ("Different shapes case is not supported with constant inputs: %s", layer_type.c_str())); + + if (inp0.total() == 1 && inp1.total() == 1 && inp0.dims != inp1.dims) + { + if (inp0.dims < inp1.dims) { - int axis = axes_dict.getIntValue(i); - CV_CheckLE(axis, static_cast(inpShape.size()), "Squeeze axis"); - maskedAxes[axis] = inpShape[axis] == 1; + inp0 = inp0.reshape(1, inp1.dims, inp1.size); + inp0.dims = inp1.dims; } - MatShape outShape; - for (int i = 0; i < inpShape.size(); ++i) + else { - if (!maskedAxes[i]) - outShape.push_back(inpShape[i]); + inp1 = inp1.reshape(1, inp0.dims, inp0.size); + inp1.dims = inp0.dims; } - if (outShape.size() != inpShape.size()) + } + + Mat out; + if (inp0.total() != inp1.total()) + { + if (inp0.total() == 1) { - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); - if (hasDynamicShapes) - { - std::vector dynamicAxes; - std::vector inputIndices; - for (int index = 0; index < inpShape.size(); ++index) - { - if (!maskedAxes[index]) - inputIndices.push_back(index); - } - for (int index = 0; index < outShape.size(); ++index) - dynamicAxes.push_back(index); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } + float inp0_value = inp0.ptr()[0]; + float coeff = isDiv ? 1.0 / inp0_value : inp0_value; + multiply(inp1, coeff, out); } else - layerParams.type = "Identity"; - - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) { - Mat inp = getBlob(node_proto, 0); - Mat out = inp.reshape(1, outShape); - out.dims = outShape.size(); // to workaround dims == 1 - addConstant(layerParams.name, out); - return; + float inp1_value = inp1.ptr()[0]; + float coeff = isDiv ? 1.0 / inp1_value : inp1_value; + multiply(inp0, coeff, out); } + } - else if (layer_type == "Flatten") + else { - CV_CheckEQ(node_proto.input_size(), 1, ""); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat input = getBlob(node_proto, 0); - int axis = normalize_axis(layerParams.get("axis", 1), input.dims); - - std::vector out_size(&input.size[0], &input.size[0] + axis); - out_size.push_back(input.total(axis)); - Mat output = input.reshape(1, out_size); - addConstant(layerParams.name, output); - return; - } + out = isDiv ? inp0 / inp1 : inp0.mul(inp1); } - else if (layer_type == "Unsqueeze") + + if (inp0.dims == 1 && inp1.dims == 1) + out.dims = 1; // to workaround dims == 1 + addConstant(layerParams.name, out); + return; + } + else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) + { + layerParams.type = "Eltwise"; + layerParams.set("operation", isDiv ? "div" : "prod"); + } + else + { + // Scale layer allocate output with the first input shape + if (total(outShapes[node_proto.input(0)]) < total(outShapes[node_proto.input(1)])) { - CV_Assert(node_proto.input_size() == 1); - DictValue axes = layerParams.get("axes"); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - // Constant input. - Mat input = getBlob(node_proto, 0); + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(1)); + proto.add_input(node_proto.input(0)); + proto.add_output(layerParams.name); + node_proto = proto; + } - std::vector dims; - for (int j = 0; j < input.dims; j++) { - dims.push_back(input.size[j]); - } - CV_Assert(axes.getIntValue(axes.size()-1) <= dims.size()); - for (int j = 0; j < axes.size(); j++) { - dims.insert(dims.begin() + axes.getIntValue(j), 1); - } + if (isDiv) + { + LayerParams powerParams; + powerParams.name = layerParams.name + "/inv"; + powerParams.type = "Power"; + powerParams.set("power", -1); - Mat out = input.reshape(0, dims); - addConstant(layerParams.name, out); - return; - } + //Create Power layer + int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + //Connect to input + IterLayerId_t layerId = layer_id.find(node_proto.input(1)); + CV_Assert(layerId != layer_id.end()); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); + outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - // Variable input. - if (axes.size() != 1) - CV_Error(Error::StsNotImplemented, "Multidimensional unsqueeze"); + //Replace input to Power + node_proto.set_input(1, powerParams.name); + } + layerParams.type = "Scale"; + } + addLayer(layerParams, node_proto); +} - MatShape inpShape = outShapes[node_proto.input(0)]; - int axis = axes.getIntValue(0); - CV_Assert(0 <= axis && axis <= inpShape.size()); - std::vector outShape = inpShape; - outShape.insert(outShape.begin() + axis, 1); - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); - if (hasDynamicShapes) - { - std::vector dynamicAxes; - std::vector inputIndices; - for (int index = 0; index < outShape.size(); ++index) { - if (index != axis) - dynamicAxes.push_back(index); - } - for (int index = 0; index < inpShape.size(); ++index) - inputIndices.push_back(index); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } +void ONNXImporter::parseConv(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_Assert(node_proto.input_size() >= 2); + layerParams.type = "Convolution"; + for (int j = 1; j < node_proto.input_size(); j++) { + if (constBlobs.find(node_proto.input(j)) != constBlobs.end()) + { + layerParams.blobs.push_back(getBlob(node_proto, j)); } - else if (layer_type == "Expand") + } + int outCn = layerParams.blobs.empty() ? outShapes[node_proto.input(1)][0] : layerParams.blobs[0].size[0]; + layerParams.set("num_output", outCn); + + // Check for asymmetric padding in Conv2D + if (layerParams.has("pad")) + { + bool asymmetricPadding = false; + DictValue pads = layerParams.get("pad"); + const int dims = pads.size() / 2; + for (int i = 0; i < dims; ++i) { - CV_CheckEQ(node_proto.input_size(), 2, ""); - const std::string& input0 = node_proto.input(0); - const std::string& input1 = node_proto.input(1); - Mat newShapeMat = getBlob(input1); - MatShape targetShape(newShapeMat.ptr(), newShapeMat.ptr() + newShapeMat.total()); - - MatShape inpShape; - bool haveVariables = constBlobs.find(input0) == constBlobs.end(); - if (haveVariables) + if (pads.get(i) != pads.get(i + dims)) { - IterShape_t shapeIt = outShapes.find(input0); - CV_Assert(shapeIt != outShapes.end()); - inpShape = shapeIt->second; + asymmetricPadding = true; + break; } - else + } + if (asymmetricPadding && pads.size() == 4) // [pad_t, pad_l, pad_b, pad_r] + { + layerParams.erase("pad"); + // No paddings required for N, C axis + std::vector paddings(4, 0); + // Add paddings for H, W axis + for (int i = 0; i < dims; ++i) { - inpShape = shape(getBlob(input0)); + paddings.push_back(pads.get(i)); + paddings.push_back(pads.get(dims + i)); } + LayerParams padLp; + padLp.name = layerParams.name + "/pad"; + padLp.type = "Padding"; + padLp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size())); - String srcName = input0; - // Unsqueeze and repeat along new axis - if (targetShape.size() == inpShape.size() + 1) - { - for (int i = 0; i < targetShape.size(); i++) - { - if (targetShape[i] == -1 && i < inpShape.size()) - targetShape[i] = inpShape[i]; - else if (i < inpShape.size() && targetShape[i] != inpShape[i]) - inpShape.insert(inpShape.begin() + i, 1); - } - if (haveVariables) - { - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); - srcName = reshapeLp.name; - } - } - CV_CheckEQ(inpShape.size(), targetShape.size(), "Unsupported Expand op with different dims"); + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(padLp.name); - std::vector broadcast_axes; - for (int i = 0; i < targetShape.size(); i++) - { - if (targetShape[i] != inpShape[i]) - { - if (inpShape[i] == 1) - broadcast_axes.push_back(i); - else - CV_Error(Error::StsError, format("Could not be broadcast by axis: %d", i)); - } - } + addLayer(padLp, proto); + node_proto.set_input(0, padLp.name); + } + } + addLayer(layerParams, node_proto); +} - if (!haveVariables) - { - if (broadcast_axes.size() != 1) - CV_Error(Error::StsNotImplemented, "Expand op doesn't support multiple axes for constant input"); - - Mat input = getBlob(node_proto, 0); - input = input.reshape(0, total(inpShape, 0, broadcast_axes[0])); - Mat output = cv::repeat(input, 1, targetShape[broadcast_axes[0]]); - output = output.reshape(0, targetShape); - addConstant(layerParams.name, output); - return; - } +void ONNXImporter::parseConvTranspose(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() >= 2); + layerParams.type = "Deconvolution"; + for (int j = 1; j < node_proto.input_size(); j++) { + layerParams.blobs.push_back(getBlob(node_proto, j)); + } + layerParams.set("num_output", layerParams.blobs[0].size[1] * layerParams.get("group", 1)); + layerParams.set("bias_term", node_proto.input_size() == 3); - if (broadcast_axes.size() == 2 && - broadcast_axes[0] == broadcast_axes[1] - 1 && broadcast_axes[1] == inpShape.size() - 1) - { - LayerParams constParams; - constParams.name = layerParams.name + "/const"; - CV_Assert(layer_id.find(constParams.name) == layer_id.end()); - constParams.type = "Const"; + if (!layerParams.has("kernel_size")) + CV_Error(Error::StsNotImplemented, + "Required attribute 'kernel_size' is not present."); - Mat inp = Mat::ones(newShapeMat.total(), newShapeMat.ptr(), CV_32F); - constParams.blobs.push_back(inp); + if (layerParams.has("output_shape")) + { + const DictValue& outShape = layerParams.get("output_shape"); + DictValue strides = layerParams.get("stride"); + DictValue kernel = layerParams.get("kernel_size"); - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); + String padMode; + std::vector adjust_pads; + if (layerParams.has("pad_mode")) + { + padMode = toUpperCase(layerParams.get("pad_mode")); + if (padMode != "SAME" && padMode != "VALID") + CV_Error(Error::StsError, "Unsupported padding mode " + padMode); - layerParams.type = "Scale"; - layerParams.set("bias_term", false); - node_proto.set_input(0, constParams.name); - node_proto.set_input(1, srcName); - } - else if (broadcast_axes.size() == 1 && broadcast_axes[0] <= 1) + for (int i = 0; i < strides.size(); i++) { - String base_name = layerParams.name + "/copy_"; - std::vector input_names; - for (int j = 0; j < targetShape[broadcast_axes[0]]; j++) - { - std::ostringstream ss; - ss << j; - LayerParams copyLP; - copyLP.name = base_name + ss.str(); - copyLP.type = "Identity"; - CV_Assert(layer_id.find(copyLP.name) == layer_id.end()); - input_names.push_back(copyLP.name); - - node_proto.set_input(0, srcName); - node_proto.set_output(0, copyLP.name); - addLayer(copyLP, node_proto); - } - node_proto.clear_input(); - for (int i = 0; i < input_names.size(); i++) - { - node_proto.add_input(input_names[i]); - } - layerParams.set("axis", broadcast_axes[0]); - layerParams.type = "Concat"; - node_proto.set_output(0, layerParams.name); + int sz = outShape.get(2 + i); + int stride = strides.get(i); + adjust_pads.push_back(padMode == "SAME"? (sz - 1) % stride : + (sz - kernel.get(i)) % stride); } - else - CV_Error(Error::StsNotImplemented, "Unsupported Expand op"); + layerParams.set("adj", DictValue::arrayInt(&adjust_pads[0], adjust_pads.size())); } - else if (layer_type == "Reshape") - { - CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape")); + } + else if (layerParams.has("output_padding")) + { + replaceLayerParam(layerParams, "output_padding", "adj"); + } + addLayer(layerParams, node_proto); +} - if (node_proto.input_size() == 2) { - Mat blob = getBlob(node_proto, 1); - CV_Assert(blob.type() == CV_32SC1); +void ONNXImporter::parseTranspose(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Permute"; + replaceLayerParam(layerParams, "perm", "order"); - layerParams.set("dim", DictValue::arrayInt( - blob.ptr(), blob.total() )); + CV_Assert(node_proto.input_size() == 1); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + std::vector inputs(1, getBlob(node_proto, 0)), transposed; + runLayer(layerParams, inputs, transposed); + CV_Assert(transposed.size() == 1); + addConstant(layerParams.name, transposed[0]); + return; + } + addLayer(layerParams, node_proto); +} - if (layer_id.find(node_proto.input(0)) == layer_id.end()) { - std::vector inputs(1, getBlob(node_proto, 0)), outputs; - runLayer(layerParams, inputs, outputs); - addConstant(layerParams.name, outputs[0]); - return; - } - } - else { - DictValue shape = layerParams.get("shape"); - std::vector dim; - for (int j = 0; j < shape.size(); j++) { - dim.push_back(shape.getIntValue(j)); - } +void ONNXImporter::parseSqueeze(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes")); + DictValue axes_dict = layerParams.get("axes"); + MatShape inpShape = outShapes[node_proto.input(0)]; - if (layer_id.find(node_proto.input(0)) == layer_id.end()) { - Mat input = getBlob(node_proto, 0); - Mat out = input.reshape(0, dim); - addConstant(layerParams.name, out); - return; - } - replaceLayerParam(layerParams, "shape", "dim"); - } - } - else if (layer_type == "Pad") + std::vector maskedAxes(inpShape.size(), false); + for (int i = 0; i < axes_dict.size(); ++i) + { + int axis = axes_dict.getIntValue(i); + CV_CheckLE(axis, static_cast(inpShape.size()), "Squeeze axis"); + maskedAxes[axis] = inpShape[axis] == 1; + } + MatShape outShape; + for (int i = 0; i < inpShape.size(); ++i) + { + if (!maskedAxes[i]) + outShape.push_back(inpShape[i]); + } + if (outShape.size() != inpShape.size()) + { + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + if (hasDynamicShapes) { - layerParams.type = "Padding"; - replaceLayerParam(layerParams, "mode", "type"); - if (node_proto.input_size() == 3 || node_proto.input_size() == 2) + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < inpShape.size(); ++index) { - // Paddings are in order begin0, begin1, .. beginN, end0, end1, ..., endN. - // We need to shuffle it to begin0, end0, begin1, end1, ... - Mat paddings = getBlob(node_proto, 1).reshape(1, 2); - paddings = paddings.t(); - layerParams.set("paddings", DictValue::arrayInt(paddings.ptr(), paddings.total())); - - if (node_proto.input_size() == 3) - { - Mat value = getBlob(node_proto, 2); - layerParams.set("value", value.ptr()[0]); - } + if (!maskedAxes[index]) + inputIndices.push_back(index); } + for (int index = 0; index < outShape.size(); ++index) + dynamicAxes.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); } - else if (layer_type == "Shape") - { - CV_Assert(node_proto.input_size() == 1); - IterShape_t shapeIt = outShapes.find(node_proto.input(0)); - CV_Assert(shapeIt != outShapes.end()); - const MatShape& inpShape = shapeIt->second; + } + else + layerParams.type = "Identity"; - Mat shapeMat(inpShape.size(), 1, CV_32S); - for (int j = 0; j < inpShape.size(); ++j) - shapeMat.at(j) = inpShape[j]; - shapeMat.dims = 1; + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inp = getBlob(node_proto, 0); + Mat out = inp.reshape(1, outShape); + out.dims = outShape.size(); // to workaround dims == 1 + addConstant(layerParams.name, out); + return; + } + addLayer(layerParams, node_proto); +} - addConstant(layerParams.name, shapeMat); - return; +void ONNXImporter::parseFlatten(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_CheckEQ(node_proto.input_size(), 1, ""); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat input = getBlob(node_proto, 0); + int axis = normalize_axis(layerParams.get("axis", 1), input.dims); + + std::vector out_size(&input.size[0], &input.size[0] + axis); + out_size.push_back(input.total(axis)); + Mat output = input.reshape(1, out_size); + addConstant(layerParams.name, output); + return; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseUnsqueeze(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 1); + DictValue axes = layerParams.get("axes"); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + // Constant input. + Mat input = getBlob(node_proto, 0); + + std::vector dims; + for (int j = 0; j < input.dims; j++) { + dims.push_back(input.size[j]); + } + CV_Assert(axes.getIntValue(axes.size()-1) <= dims.size()); + for (int j = 0; j < axes.size(); j++) { + dims.insert(dims.begin() + axes.getIntValue(j), 1); } - else if (layer_type == "Cast") + + Mat out = input.reshape(0, dims); + addConstant(layerParams.name, out); + return; + } + + // Variable input. + if (axes.size() != 1) + CV_Error(Error::StsNotImplemented, "Multidimensional unsqueeze"); + + MatShape inpShape = outShapes[node_proto.input(0)]; + int axis = axes.getIntValue(0); + CV_Assert(0 <= axis && axis <= inpShape.size()); + std::vector outShape = inpShape; + outShape.insert(outShape.begin() + axis, 1); + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + if (hasDynamicShapes) + { + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < outShape.size(); ++index) { + if (index != axis) + dynamicAxes.push_back(index); + } + for (int index = 0; index < inpShape.size(); ++index) + inputIndices.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseExpand(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_CheckEQ(node_proto.input_size(), 2, ""); + const std::string& input0 = node_proto.input(0); + const std::string& input1 = node_proto.input(1); + Mat newShapeMat = getBlob(input1); + MatShape targetShape(newShapeMat.ptr(), newShapeMat.ptr() + newShapeMat.total()); + + MatShape inpShape; + bool haveVariables = constBlobs.find(input0) == constBlobs.end(); + if (haveVariables) + { + IterShape_t shapeIt = outShapes.find(input0); + CV_Assert(shapeIt != outShapes.end()); + inpShape = shapeIt->second; + } + else + { + inpShape = shape(getBlob(input0)); + } + + String srcName = input0; + // Unsqueeze and repeat along new axis + if (targetShape.size() == inpShape.size() + 1) + { + for (int i = 0; i < targetShape.size(); i++) { - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat blob = getBlob(node_proto, 0); - int type; - switch (layerParams.get("to")) - { - case opencv_onnx::TensorProto_DataType_FLOAT: type = CV_32F; break; - case opencv_onnx::TensorProto_DataType_UINT8: type = CV_8U; break; - case opencv_onnx::TensorProto_DataType_UINT16: type = CV_16U; break; - case opencv_onnx::TensorProto_DataType_FLOAT16: type = CV_16S; break; - case opencv_onnx::TensorProto_DataType_INT8: - case opencv_onnx::TensorProto_DataType_INT16: - case opencv_onnx::TensorProto_DataType_INT32: - case opencv_onnx::TensorProto_DataType_INT64: type = CV_32S; break; - default: type = blob.type(); - } - Mat dst; - blob.convertTo(dst, type); - dst.dims = blob.dims; - addConstant(layerParams.name, dst); - return; - } - else - layerParams.type = "Identity"; + if (targetShape[i] == -1 && i < inpShape.size()) + targetShape[i] = inpShape[i]; + else if (i < inpShape.size() && targetShape[i] != inpShape[i]) + inpShape.insert(inpShape.begin() + i, 1); } - else if (layer_type == "ConstantOfShape" || layer_type == "ConstantFill") + if (haveVariables) { - int depth = CV_32F; - float fill_value; - if (!layerParams.blobs.empty()) - { - CV_Assert(!layerParams.has("value")); - depth = layerParams.blobs[0].depth(); - Mat floats; - layerParams.blobs[0].convertTo(floats, CV_32F); - fill_value = floats.at(0, 0); - } - else - fill_value = layerParams.get("value", 0); + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + reshapeLp.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); - MatShape inpShape = getBlob(node_proto, 0); - for (int i = 0; i < inpShape.size(); i++) - CV_CheckGT(inpShape[i], 0, ""); - Mat tensor(inpShape.size(), &inpShape[0], depth, Scalar(fill_value)); - addConstant(layerParams.name, tensor); - return; + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + srcName = reshapeLp.name; } - else if (layer_type == "Gather") - { - CV_Assert(node_proto.input_size() == 2); - Mat indexMat = getBlob(node_proto, 1); - CV_Assert_N(indexMat.type() == CV_32S, indexMat.total() == 1); - int index = indexMat.at(0); - int axis = layerParams.get("axis", 0); + } + CV_CheckEQ(inpShape.size(), targetShape.size(), "Unsupported Expand op with different dims"); - if ((constBlobs.find(node_proto.input(0)) != constBlobs.end())) - { - Mat input = getBlob(node_proto, 0); - Mat out; - std::vector ranges(input.dims, Range::all()); - ranges[axis] = Range(index, index + 1); - - out = input(ranges); - MatShape outShape = shape(out); - if (outShape.size() > 1) - { - outShape.erase(outShape.begin() + axis); - out.reshape(0, outShape); - } else { - out.dims = 1; - } - addConstant(layerParams.name, out); - return; - } + std::vector broadcast_axes; + for (int i = 0; i < targetShape.size(); i++) + { + if (targetShape[i] != inpShape[i]) + { + if (inpShape[i] == 1) + broadcast_axes.push_back(i); else - { - IterShape_t shapeIt = outShapes.find(node_proto.input(0)); - CV_Assert(shapeIt != outShapes.end()); - MatShape inpShape = shapeIt->second; - - LayerParams sliceLp; - sliceLp.type = "Slice"; - sliceLp.name = inpShape.size() > 1 ? layerParams.name + "/slice" : layerParams.name; - std::vector begin(inpShape.size(), 0); - std::vector end(inpShape.size(), -1); - begin[axis] = index; - end[axis] = index + 1; - - cv::dnn::DictValue paramBegin = cv::dnn::DictValue::arrayInt(begin.data(), begin.size()); - cv::dnn::DictValue paramEnd = cv::dnn::DictValue::arrayInt(end.data(), end.size()); - sliceLp.set("begin", paramBegin); - sliceLp.set("end", paramEnd); - sliceLp.set("has_dynamic_shapes", hasDynamicShapes); - - if (inpShape.size() > 1) - { - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(sliceLp.name); - addLayer(sliceLp, proto); - - inpShape.erase(inpShape.begin() + axis); - layerParams.type = "Reshape"; - layerParams.set("axis", 0); - layerParams.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); - if (hasDynamicShapes) - { - std::vector dynamicAxes; - std::vector inputIndices; - for (int index = 0; index < inpShape.size(); ++index) - dynamicAxes.push_back(index); - for (int index = 0; index < inpShape.size(); ++index) - inputIndices.push_back(index); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } - node_proto.set_input(0, sliceLp.name); - } - else - { - layerParams = sliceLp; - } - } + CV_Error(Error::StsError, format("Could not be broadcast by axis: %d", i)); } - else if (layer_type == "Concat") - { - bool hasVariableInps = false; - for (int i = 0; i < node_proto.input_size(); ++i) - { - if (layer_id.find(node_proto.input(i)) != layer_id.end()) - { - hasVariableInps = true; - break; - } - } + } - if (!hasVariableInps) - { - std::vector inputs(node_proto.input_size()), concatenated; - // Due constant folding we can get inputs with different number of dimensions - // Insert the missing dimension to inputs - MatShape inputShape; - for (size_t i = 0; i < inputs.size(); ++i) - { - inputs[i] = getBlob(node_proto, i); - if (inputs[i].size.dims() > inputShape.size()) - { - inputShape = shape(inputs[i]); - } - } + if (!haveVariables) + { + if (broadcast_axes.size() != 1) + CV_Error(Error::StsNotImplemented, "Expand op doesn't support multiple axes for constant input"); + + Mat input = getBlob(node_proto, 0); + input = input.reshape(0, total(inpShape, 0, broadcast_axes[0])); + Mat output = cv::repeat(input, 1, targetShape[broadcast_axes[0]]); + output = output.reshape(0, targetShape); + addConstant(layerParams.name, output); + return; + } - // Concat-1 has default value for axis is 1: https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-1 - int axis = layerParams.get("axis", 1); - for (size_t i = 0; i < inputs.size(); ++i) - { - MatShape targetShape = inputShape; - targetShape[axis] = shape(inputs[i])[axis]; - CV_CheckEQ(total(targetShape), total(shape(inputs[i])), ""); - inputs[i] = inputs[i].reshape(0, targetShape); - } - runLayer(layerParams, inputs, concatenated); + if (broadcast_axes.size() == 2 && + broadcast_axes[0] == broadcast_axes[1] - 1 && broadcast_axes[1] == inpShape.size() - 1) + { + LayerParams constParams; + constParams.name = layerParams.name + "/const"; + CV_Assert(layer_id.find(constParams.name) == layer_id.end()); + constParams.type = "Const"; + + Mat inp = Mat::ones(newShapeMat.total(), newShapeMat.ptr(), CV_32F); + constParams.blobs.push_back(inp); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + + layerParams.type = "Scale"; + layerParams.set("bias_term", false); + node_proto.set_input(0, constParams.name); + node_proto.set_input(1, srcName); + } + else if (broadcast_axes.size() == 1 && broadcast_axes[0] <= 1) + { + String base_name = layerParams.name + "/copy_"; + std::vector input_names; + for (int j = 0; j < targetShape[broadcast_axes[0]]; j++) + { + std::ostringstream ss; + ss << j; + LayerParams copyLP; + copyLP.name = base_name + ss.str(); + copyLP.type = "Identity"; + CV_Assert(layer_id.find(copyLP.name) == layer_id.end()); + input_names.push_back(copyLP.name); + + node_proto.set_input(0, srcName); + node_proto.set_output(0, copyLP.name); + addLayer(copyLP, node_proto); + } + node_proto.clear_input(); + for (int i = 0; i < input_names.size(); i++) + { + node_proto.add_input(input_names[i]); + } + layerParams.set("axis", broadcast_axes[0]); + layerParams.type = "Concat"; + node_proto.set_output(0, layerParams.name); + } + else + CV_Error(Error::StsNotImplemented, "Unsupported Expand op"); + addLayer(layerParams, node_proto); +} - CV_Assert(concatenated.size() == 1); - addConstant(layerParams.name, concatenated[0]); - return; - } - else - { - for (int i = 0; i < node_proto.input_size(); ++i) - { - if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) - { - LayerParams constParams; - constParams.name = node_proto.input(i); - constParams.type = "Const"; - constParams.blobs.push_back(getBlob(node_proto, i)); - - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); - } - } - } +void ONNXImporter::parseReshape(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape")); + + if (node_proto.input_size() == 2) { + Mat blob = getBlob(node_proto, 1); + CV_Assert(blob.type() == CV_32SC1); + + layerParams.set("dim", DictValue::arrayInt( + blob.ptr(), blob.total() )); + + if (layer_id.find(node_proto.input(0)) == layer_id.end()) { + std::vector inputs(1, getBlob(node_proto, 0)), outputs; + runLayer(layerParams, inputs, outputs); + addConstant(layerParams.name, outputs[0]); + return; + } + } + else { + DictValue shape = layerParams.get("shape"); + std::vector dim; + for (int j = 0; j < shape.size(); j++) { + dim.push_back(shape.getIntValue(j)); + } + + if (layer_id.find(node_proto.input(0)) == layer_id.end()) { + Mat input = getBlob(node_proto, 0); + Mat out = input.reshape(0, dim); + addConstant(layerParams.name, out); + return; } - else if (layer_type == "Resize") + replaceLayerParam(layerParams, "shape", "dim"); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parsePad(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Padding"; + replaceLayerParam(layerParams, "mode", "type"); + if (node_proto.input_size() == 3 || node_proto.input_size() == 2) + { + // Paddings are in order begin0, begin1, .. beginN, end0, end1, ..., endN. + // We need to shuffle it to begin0, end0, begin1, end1, ... + Mat paddings = getBlob(node_proto, 1).reshape(1, 2); + paddings = paddings.t(); + layerParams.set("paddings", DictValue::arrayInt(paddings.ptr(), paddings.total())); + + if (node_proto.input_size() == 3) { - for (int i = 1; i < node_proto.input_size(); i++) - CV_Assert(layer_id.find(node_proto.input(i)) == layer_id.end()); + Mat value = getBlob(node_proto, 2); + layerParams.set("value", value.ptr()[0]); + } + } + addLayer(layerParams, node_proto); +} - if (layerParams.has("coordinate_transformation_mode")) - { - String interp_mode = layerParams.get("coordinate_transformation_mode"); - CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); +void ONNXImporter::parseShape(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 1); + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); + CV_Assert(shapeIt != outShapes.end()); + const MatShape& inpShape = shapeIt->second; - layerParams.set("align_corners", interp_mode == "align_corners"); - if (layerParams.get("mode") == "linear") - { - layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? - "opencv_linear" : "bilinear"); - } - } - if (layerParams.get("mode") == "linear" && framework_name == "pytorch") - layerParams.set("mode", "opencv_linear"); + Mat shapeMat(inpShape.size(), 1, CV_32S); + for (int j = 0; j < inpShape.size(); ++j) + shapeMat.at(j) = inpShape[j]; + shapeMat.dims = 1; - // input = [X, scales], [X, roi, scales] or [x, roi, scales, sizes] - int foundScaleId = hasDynamicShapes ? node_proto.input_size() - 1 - : node_proto.input_size() > 2 ? 2 : 1; + addConstant(layerParams.name, shapeMat); +} - Mat scales = getBlob(node_proto, foundScaleId); - if (scales.total() == 4) - { - layerParams.set("zoom_factor_y", scales.at(2)); - layerParams.set("zoom_factor_x", scales.at(3)); - } - else +void ONNXImporter::parseCast(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat blob = getBlob(node_proto, 0); + int type; + switch (layerParams.get("to")) + { + case opencv_onnx::TensorProto_DataType_FLOAT: type = CV_32F; break; + case opencv_onnx::TensorProto_DataType_UINT8: type = CV_8U; break; + case opencv_onnx::TensorProto_DataType_UINT16: type = CV_16U; break; + case opencv_onnx::TensorProto_DataType_FLOAT16: type = CV_16S; break; + case opencv_onnx::TensorProto_DataType_INT8: + case opencv_onnx::TensorProto_DataType_INT16: + case opencv_onnx::TensorProto_DataType_INT32: + case opencv_onnx::TensorProto_DataType_INT64: type = CV_32S; break; + default: type = blob.type(); + } + Mat dst; + blob.convertTo(dst, type); + dst.dims = blob.dims; + addConstant(layerParams.name, dst); + return; + } + else + layerParams.type = "Identity"; + addLayer(layerParams, node_proto); +} + +// "ConstantOfShape" "ConstantFill" +void ONNXImporter::parseConstantFill(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int depth = CV_32F; + float fill_value; + if (!layerParams.blobs.empty()) + { + CV_Assert(!layerParams.has("value")); + depth = layerParams.blobs[0].depth(); + Mat floats; + layerParams.blobs[0].convertTo(floats, CV_32F); + fill_value = floats.at(0, 0); + } + else + fill_value = layerParams.get("value", 0); + + MatShape inpShape = getBlob(node_proto, 0); + for (int i = 0; i < inpShape.size(); i++) + CV_CheckGT(inpShape[i], 0, ""); + Mat tensor(inpShape.size(), &inpShape[0], depth, Scalar(fill_value)); + addConstant(layerParams.name, tensor); +} + +void ONNXImporter::parseGather(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_Assert(node_proto.input_size() == 2); + Mat indexMat = getBlob(node_proto, 1); + CV_Assert_N(indexMat.type() == CV_32S, indexMat.total() == 1); + int index = indexMat.at(0); + int axis = layerParams.get("axis", 0); + + if ((constBlobs.find(node_proto.input(0)) != constBlobs.end())) + { + Mat input = getBlob(node_proto, 0); + Mat out; + std::vector ranges(input.dims, Range::all()); + ranges[axis] = Range(index, index + 1); + + out = input(ranges); + MatShape outShape = shape(out); + if (outShape.size() > 1) + { + outShape.erase(outShape.begin() + axis); + out.reshape(0, outShape); + } else { + out.dims = 1; + } + addConstant(layerParams.name, out); + return; + } + else + { + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); + CV_Assert(shapeIt != outShapes.end()); + MatShape inpShape = shapeIt->second; + + LayerParams sliceLp; + sliceLp.type = "Slice"; + sliceLp.name = inpShape.size() > 1 ? layerParams.name + "/slice" : layerParams.name; + std::vector begin(inpShape.size(), 0); + std::vector end(inpShape.size(), -1); + begin[axis] = index; + end[axis] = index + 1; + + cv::dnn::DictValue paramBegin = cv::dnn::DictValue::arrayInt(begin.data(), begin.size()); + cv::dnn::DictValue paramEnd = cv::dnn::DictValue::arrayInt(end.data(), end.size()); + sliceLp.set("begin", paramBegin); + sliceLp.set("end", paramEnd); + sliceLp.set("has_dynamic_shapes", hasDynamicShapes); + + if (inpShape.size() > 1) + { + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(sliceLp.name); + addLayer(sliceLp, proto); + + inpShape.erase(inpShape.begin() + axis); + layerParams.type = "Reshape"; + layerParams.set("axis", 0); + layerParams.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); + if (hasDynamicShapes) { - const std::string& inputLast = node_proto.input(node_proto.input_size() - 1); - if (constBlobs.find(inputLast) != constBlobs.end()) - { - Mat shapes = getBlob(inputLast); - CV_CheckEQ(shapes.size[0], 4, ""); - CV_CheckEQ(shapes.size[1], 1, ""); - CV_CheckDepth(shapes.depth(), shapes.depth() == CV_32S || shapes.depth() == CV_32F, ""); - if (shapes.depth() == CV_32F) - shapes.convertTo(shapes, CV_32S); - layerParams.set("width", shapes.at(3)); - layerParams.set("height", shapes.at(2)); - } + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < inpShape.size(); ++index) + dynamicAxes.push_back(index); + for (int index = 0; index < inpShape.size(); ++index) + inputIndices.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); } - replaceLayerParam(layerParams, "mode", "interpolation"); + node_proto.set_input(0, sliceLp.name); } - else if (layer_type == "Upsample") + else { - //fused from Resize Subgraph - if (layerParams.has("coordinate_transformation_mode")) - { - String interp_mode = layerParams.get("coordinate_transformation_mode"); - CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); + layerParams = sliceLp; + } + } + addLayer(layerParams, node_proto); +} - layerParams.set("align_corners", interp_mode == "align_corners"); - if (layerParams.get("mode") == "linear") - { - layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? - "opencv_linear" : "bilinear"); - } - } - if (layerParams.get("mode") == "linear" && framework_name == "pytorch") - layerParams.set("mode", "opencv_linear"); +void ONNXImporter::parseConcat(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + bool hasVariableInps = false; + for (int i = 0; i < node_proto.input_size(); ++i) + { + if (layer_id.find(node_proto.input(i)) != layer_id.end()) + { + hasVariableInps = true; + break; + } + } - layerParams.type = "Resize"; - if (layerParams.has("scales")) - { - // Pytorch layer - DictValue scales = layerParams.get("scales"); - CV_Assert(scales.size() == 4); - layerParams.set("zoom_factor_y", scales.getIntValue(2)); - layerParams.set("zoom_factor_x", scales.getIntValue(3)); - } - else if (layerParams.has("height_scale") && layerParams.has("width_scale")) - { - // Caffe2 layer - replaceLayerParam(layerParams, "height_scale", "zoom_factor_y"); - replaceLayerParam(layerParams, "width_scale", "zoom_factor_x"); - } - else + if (!hasVariableInps) + { + std::vector inputs(node_proto.input_size()), concatenated; + // Due constant folding we can get inputs with different number of dimensions + // Insert the missing dimension to inputs + MatShape inputShape; + for (size_t i = 0; i < inputs.size(); ++i) + { + inputs[i] = getBlob(node_proto, i); + if (inputs[i].size.dims() > inputShape.size()) { - // scales as input - const std::string& input1 = node_proto.input(1); - if (constBlobs.find(input1) != constBlobs.end()) - { - Mat scales = getBlob(input1); - CV_Assert(scales.total() == 4); - layerParams.set("zoom_factor_y", scales.at(2)); - layerParams.set("zoom_factor_x", scales.at(3)); - } + inputShape = shape(inputs[i]); } - replaceLayerParam(layerParams, "mode", "interpolation"); } - else if (layer_type == "SoftMax" || layer_type == "LogSoftmax") + + // Concat-1 has default value for axis is 1: https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-1 + int axis = layerParams.get("axis", 1); + for (size_t i = 0; i < inputs.size(); ++i) { - layerParams.type = "Softmax"; - layerParams.set("log_softmax", layer_type == "LogSoftmax"); + MatShape targetShape = inputShape; + targetShape[axis] = shape(inputs[i])[axis]; + CV_CheckEQ(total(targetShape), total(shape(inputs[i])), ""); + inputs[i] = inputs[i].reshape(0, targetShape); } - else if (layer_type == "DetectionOutput") + runLayer(layerParams, inputs, concatenated); + + CV_Assert(concatenated.size() == 1); + addConstant(layerParams.name, concatenated[0]); + return; + } + else + { + for (int i = 0; i < node_proto.input_size(); ++i) { - CV_CheckEQ(node_proto.input_size(), 3, ""); - if (constBlobs.find(node_proto.input(2)) != constBlobs.end()) + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) { - Mat priors = getBlob(node_proto, 2); - LayerParams constParams; - constParams.name = layerParams.name + "/priors"; + constParams.name = node_proto.input(i); constParams.type = "Const"; - constParams.blobs.push_back(priors); - - opencv_onnx::NodeProto priorsProto; - priorsProto.add_output(constParams.name); - addLayer(constParams, priorsProto); + constParams.blobs.push_back(getBlob(node_proto, i)); - node_proto.set_input(2, constParams.name); + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); } } - else + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseResize(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + for (int i = 1; i < node_proto.input_size(); i++) + CV_Assert(layer_id.find(node_proto.input(i)) == layer_id.end()); + + if (layerParams.has("coordinate_transformation_mode")) + { + String interp_mode = layerParams.get("coordinate_transformation_mode"); + CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); + + layerParams.set("align_corners", interp_mode == "align_corners"); + if (layerParams.get("mode") == "linear") { - for (int j = 0; j < node_proto.input_size(); j++) { - if (layer_id.find(node_proto.input(j)) == layer_id.end()) - layerParams.blobs.push_back(getBlob(node_proto, j)); - } + layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? + "opencv_linear" : "bilinear"); } - addLayer(layerParams, node_proto); } - catch (const cv::Exception& e) + if (layerParams.get("mode") == "linear" && framework_name == "pytorch") + layerParams.set("mode", "opencv_linear"); + + // input = [X, scales], [X, roi, scales] or [x, roi, scales, sizes] + int foundScaleId = hasDynamicShapes ? node_proto.input_size() - 1 + : node_proto.input_size() > 2 ? 2 : 1; + + Mat scales = getBlob(node_proto, foundScaleId); + if (scales.total() == 4) { - CV_LOG_ERROR(NULL, "DNN/ONNX: ERROR during processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " - << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) - ); - for (int i = 0; i < node_proto.input_size(); i++) + layerParams.set("zoom_factor_y", scales.at(2)); + layerParams.set("zoom_factor_x", scales.at(3)); + } + else + { + const std::string& inputLast = node_proto.input(node_proto.input_size() - 1); + if (constBlobs.find(inputLast) != constBlobs.end()) + { + Mat shapes = getBlob(inputLast); + CV_CheckEQ(shapes.size[0], 4, ""); + CV_CheckEQ(shapes.size[1], 1, ""); + CV_CheckDepth(shapes.depth(), shapes.depth() == CV_32S || shapes.depth() == CV_32F, ""); + if (shapes.depth() == CV_32F) + shapes.convertTo(shapes, CV_32S); + layerParams.set("width", shapes.at(3)); + layerParams.set("height", shapes.at(2)); + } + } + replaceLayerParam(layerParams, "mode", "interpolation"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseUpsample(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + //fused from Resize Subgraph + if (layerParams.has("coordinate_transformation_mode")) + { + String interp_mode = layerParams.get("coordinate_transformation_mode"); + CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); + + layerParams.set("align_corners", interp_mode == "align_corners"); + if (layerParams.get("mode") == "linear") { - CV_LOG_INFO(NULL, " Input[" << i << "] = '" << node_proto.input(i) << "'"); + layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? + "opencv_linear" : "bilinear"); } - for (int i = 0; i < node_proto.output_size(); i++) + } + if (layerParams.get("mode") == "linear" && framework_name == "pytorch") + layerParams.set("mode", "opencv_linear"); + + layerParams.type = "Resize"; + if (layerParams.has("scales")) + { + // Pytorch layer + DictValue scales = layerParams.get("scales"); + CV_Assert(scales.size() == 4); + layerParams.set("zoom_factor_y", scales.getIntValue(2)); + layerParams.set("zoom_factor_x", scales.getIntValue(3)); + } + else if (layerParams.has("height_scale") && layerParams.has("width_scale")) + { + // Caffe2 layer + replaceLayerParam(layerParams, "height_scale", "zoom_factor_y"); + replaceLayerParam(layerParams, "width_scale", "zoom_factor_x"); + } + else + { + // scales as input + const std::string& input1 = node_proto.input(1); + if (constBlobs.find(input1) != constBlobs.end()) { - CV_LOG_INFO(NULL, " Output[" << i << "] = '" << node_proto.output(i) << "'"); + Mat scales = getBlob(input1); + CV_Assert(scales.total() == 4); + layerParams.set("zoom_factor_y", scales.at(2)); + layerParams.set("zoom_factor_x", scales.at(3)); } - CV_Error(Error::StsError, cv::format("Node [%s]:(%s) parse error: %s", layer_type.c_str(), name.c_str(), e.what())); } + replaceLayerParam(layerParams, "mode", "interpolation"); + addLayer(layerParams, node_proto); +} + +// "SoftMax" "LogSoftmax" +void ONNXImporter::parseSoftMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + const std::string& layer_type = node_proto.op_type(); + layerParams.type = "Softmax"; + layerParams.set("log_softmax", layer_type == "LogSoftmax"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseDetectionOutput(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_CheckEQ(node_proto.input_size(), 3, ""); + if (constBlobs.find(node_proto.input(2)) != constBlobs.end()) + { + Mat priors = getBlob(node_proto, 2); + + LayerParams constParams; + constParams.name = layerParams.name + "/priors"; + constParams.type = "Const"; + constParams.blobs.push_back(priors); + + opencv_onnx::NodeProto priorsProto; + priorsProto.add_output(constParams.name); + addLayer(constParams, priorsProto); + + node_proto.set_input(2, constParams.name); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseCustom(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + for (int j = 0; j < node_proto.input_size(); j++) { + if (layer_id.find(node_proto.input(j)) == layer_id.end()) + layerParams.blobs.push_back(getBlob(node_proto, j)); + } + addLayer(layerParams, node_proto); +} + +const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() +{ + DispatchMap dispatch; + + dispatch["MaxPool"] = &ONNXImporter::parseMaxPool; + dispatch["AveragePool"] = &ONNXImporter::parseAveragePool; + dispatch["GlobalAveragePool"] = dispatch["GlobalMaxPool"] = dispatch["ReduceMean"] = dispatch["ReduceSum"] = + dispatch["ReduceMax"] = &ONNXImporter::parseReduce; + dispatch["Slice"] = &ONNXImporter::parseSlice; + dispatch["Split"] = &ONNXImporter::parseSplit; + dispatch["Add"] = dispatch["Sum"] = dispatch["Sub"] = &ONNXImporter::parseBias; + dispatch["Pow"] = &ONNXImporter::parsePow; + dispatch["Max"] = &ONNXImporter::parseMax; + dispatch["Neg"] = &ONNXImporter::parseNeg; + dispatch["Constant"] = &ONNXImporter::parseConstant; + dispatch["LSTM"] = &ONNXImporter::parseLSTM; + dispatch["ImageScaler"] = &ONNXImporter::parseImageScaler; + dispatch["Clip"] = &ONNXImporter::parseClip; + dispatch["LeakyRelu"] = &ONNXImporter::parseLeakyRelu; + dispatch["Relu"] = &ONNXImporter::parseRelu; + dispatch["Elu"] = &ONNXImporter::parseElu; + dispatch["Tanh"] = &ONNXImporter::parseTanh; + dispatch["PRelu"] = &ONNXImporter::parsePRelu; + dispatch["LRN"] = &ONNXImporter::parseLRN; + dispatch["InstanceNormalization"] = &ONNXImporter::parseInstanceNormalization; + dispatch["BatchNormalization"] = &ONNXImporter::parseBatchNormalization; + dispatch["Gemm"] = &ONNXImporter::parseGemm; + dispatch["MatMul"] = &ONNXImporter::parseMatMul; + dispatch["Mul"] = dispatch["Div"] = &ONNXImporter::parseMul; + dispatch["Conv"] = &ONNXImporter::parseConv; + dispatch["ConvTranspose"] = &ONNXImporter::parseConvTranspose; + dispatch["Transpose"] = &ONNXImporter::parseTranspose; + dispatch["Squeeze"] = &ONNXImporter::parseSqueeze; + dispatch["Flatten"] = &ONNXImporter::parseFlatten; + dispatch["Unsqueeze"] = &ONNXImporter::parseUnsqueeze; + dispatch["Expand"] = &ONNXImporter::parseExpand; + dispatch["Reshape"] = &ONNXImporter::parseReshape; + dispatch["Pad"] = &ONNXImporter::parsePad; + dispatch["Shape"] = &ONNXImporter::parseShape; + dispatch["Cast"] = &ONNXImporter::parseCast; + dispatch["ConstantFill"] = dispatch["ConstantOfShape"] = &ONNXImporter::parseConstantFill; + dispatch["Gather"] = &ONNXImporter::parseGather; + dispatch["Concat"] = &ONNXImporter::parseConcat; + dispatch["Resize"] = &ONNXImporter::parseResize; + dispatch["Upsample"] = &ONNXImporter::parseUpsample; + dispatch["SoftMax"] = dispatch["LogSoftmax"] = &ONNXImporter::parseSoftMax; + dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput; + dispatch["Custom"] = &ONNXImporter::parseCustom; + + return dispatch; } Net readNetFromONNX(const String& onnxFile) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 01fa0df985b7..fa33211a50e1 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -2869,7 +2869,7 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer) DispatchMap::const_iterator iter = dispatch.find(type); if (iter != dispatch.end()) { - ((*this).*(iter->second))(net, layer, layerParams); + CALL_MEMBER_FN(*this, iter->second)(net, layer, layerParams); } else { From bdd3930855c06cf9e48b7b836837414c36660c9a Mon Sep 17 00:00:00 2001 From: Zhuo Zhang Date: Thu, 29 Jul 2021 09:34:09 +0800 Subject: [PATCH 084/376] Fix typo in comment, OpenMP => TBB --- .../include/opencv2/core/parallel/backend/parallel_for.tbb.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/parallel/backend/parallel_for.tbb.hpp b/modules/core/include/opencv2/core/parallel/backend/parallel_for.tbb.hpp index 264def5f508b..04b0c4c6cb59 100644 --- a/modules/core/include/opencv2/core/parallel/backend/parallel_for.tbb.hpp +++ b/modules/core/include/opencv2/core/parallel/backend/parallel_for.tbb.hpp @@ -38,7 +38,7 @@ static tbb::task_scheduler_init& getScheduler() } #endif -/** OpenMP parallel_for API implementation +/** TBB parallel_for API implementation * * @sa setParallelForBackend * @ingroup core_parallel_backend From 27392f832d99714e5134f877b7b8f72716c916f0 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Fri, 30 Jul 2021 13:00:13 +0300 Subject: [PATCH 085/376] reimplement onnx refactor for master --- modules/dnn/src/dnn_common.hpp | 1 + modules/dnn/src/onnx/onnx_importer.cpp | 2999 +++++++++++--------- modules/dnn/src/tensorflow/tf_importer.cpp | 2 +- 3 files changed, 1602 insertions(+), 1400 deletions(-) diff --git a/modules/dnn/src/dnn_common.hpp b/modules/dnn/src/dnn_common.hpp index 46fae41cc217..591be88079f3 100644 --- a/modules/dnn/src/dnn_common.hpp +++ b/modules/dnn/src/dnn_common.hpp @@ -14,6 +14,7 @@ Mutex& getInitializationMutex(); void initializeLayerFactory(); namespace detail { +#define CALL_MEMBER_FN(object, ptrToMemFn) ((object).*(ptrToMemFn)) class NotImplemented : public Layer { diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index d33fb68ac17f..33dc648b2c8d 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -66,7 +66,7 @@ class ONNXImporter public: ONNXImporter(Net& net, const char *onnxFile) - : dstNet(net), utilNet() + : dstNet(net), utilNet(), dispatch(buildDispatchMap()) { hasDynamicShapes = false; CV_Assert(onnxFile); @@ -87,7 +87,7 @@ class ONNXImporter } ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer) - : dstNet(net), utilNet() + : dstNet(net), utilNet(), dispatch(buildDispatchMap()) { hasDynamicShapes = false; CV_LOG_DEBUG(NULL, "DNN/ONNX: processing in-memory ONNX model (" << sizeBuffer << " bytes)"); @@ -129,6 +129,57 @@ class ONNXImporter typedef std::map::iterator IterLayerId_t; void handleNode(const opencv_onnx::NodeProto& node_proto); + +private: + typedef void (ONNXImporter::*ONNXImporterNodeParser)(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + typedef std::map DispatchMap; + + void parseMaxPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseAveragePool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseReduce (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSlice (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSplit (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseBias (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parsePow (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseNeg (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConstant (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLSTM (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseImageScaler (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseClip (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLeakyRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseElu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseTanh (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parsePRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLRN (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseBatchNormalization (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseGemm (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMatMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConv (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConvTranspose (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseTranspose (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSqueeze (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseFlatten (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseUnsqueeze (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseExpand (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseReshape (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parsePad (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseShape (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseCast (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConstantFill (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseGather (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseResize (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseUpsample (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSoftMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseCustom (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + + const DispatchMap dispatch; + static const DispatchMap buildDispatchMap(); }; inline void replaceLayerParam(LayerParams& layerParams, const String& oldKey, const String& newKey) @@ -585,13 +636,11 @@ const std::set& ONNXImporter::getSupportedTypes() return layerTypes; } -void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) +void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) { - opencv_onnx::NodeProto node_proto = node_proto_; // TODO FIXIT - CV_Assert(node_proto.output_size() >= 1); std::string name = node_proto.output(0); - std::string layer_type = node_proto.op_type(); + const std::string& layer_type = node_proto.op_type(); CV_LOG_DEBUG(NULL, "DNN/ONNX: processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) ); @@ -605,1593 +654,1745 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) layerParams.type = layer_type; layerParams.set("has_dynamic_shapes", hasDynamicShapes); - if (layer_type == "MaxPool") + DispatchMap::const_iterator iter = dispatch.find(layer_type); + if (iter != dispatch.end()) { - layerParams.type = "Pooling"; - layerParams.set("pool", "MAX"); - layerParams.set("ceil_mode", layerParams.has("pad_mode")); + CALL_MEMBER_FN(*this, iter->second)(layerParams, node_proto); } - else if (layer_type == "AveragePool") + else { - layerParams.type = "Pooling"; - layerParams.set("pool", "AVE"); - layerParams.set("ceil_mode", layerParams.has("pad_mode")); - layerParams.set("ave_pool_padded_area", framework_name == "pytorch"); + parseCustom(layerParams, node_proto); } - else if (layer_type == "GlobalAveragePool" || layer_type == "GlobalMaxPool" || - layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax") + } + catch (const cv::Exception& e) + { + if (DNN_DIAGNOSTICS_RUN) { - CV_Assert(node_proto.input_size() == 1); - layerParams.type = "Pooling"; - String pool; - if (layer_type == "GlobalMaxPool" || layer_type == "ReduceMax") - pool = "MAX"; - else if (layer_type == "ReduceSum") - pool = "SUM"; - else - pool = "AVE"; - layerParams.set("pool", pool); - layerParams.set("global_pooling", !layerParams.has("axes")); - if (layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) + CV_LOG_ERROR(NULL, "DNN/ONNX: Potential problem during processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " + << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) << "\n" << e.msg + ); + auto registeredLayers = getLayerFactoryImpl(); + if (registeredLayers.find(layerParams.type) != registeredLayers.end()) { - MatShape inpShape = outShapes[node_proto.input(0)]; - DictValue axes = layerParams.get("axes"); - bool keepdims = layerParams.get("keepdims"); - MatShape targetShape; - std::vector shouldDelete(inpShape.size(), false); - for (int i = 0; i < axes.size(); i++) { - int axis = normalize_axis(axes.get(i), inpShape.size()); - shouldDelete[axis] = true; - } - for (int axis = 0; axis < inpShape.size(); ++axis){ - if (!shouldDelete[axis]) - targetShape.push_back(inpShape[axis]); - else if (keepdims) - targetShape.push_back(1); - } - - if (inpShape.size() == 3 && axes.size() <= 2) + try { - int axis = normalize_axis(axes.get(0), inpShape.size()); - CV_CheckNE(axis, 0, ""); - - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("axis", 0); - reshapeLp.set("num_axes", 1); - int newShape[] = {1, -1}; - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 2)); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); - - LayerParams avgLp; - avgLp.name = layerParams.name + "/avg"; - avgLp.type = "Pooling"; - CV_Assert(layer_id.find(avgLp.name) == layer_id.end()); - avgLp.set("pool", pool); - if (axes.size() == 2) - { - CV_CheckEQ(normalize_axis(axes.get(0), inpShape.size()), 1, "Unsupported mode"); - CV_CheckEQ(normalize_axis(axes.get(1), inpShape.size()), 2, "Unsupported mode"); - avgLp.set("global_pooling", true); - } - else - { - avgLp.set(axis == 2 ? "global_pooling_w" : "global_pooling_h", true); - avgLp.set(axis == 2 ? "kernel_h" : "kernel_w", 1); - } - - node_proto.set_input(0, reshapeLp.name); - node_proto.set_output(0, avgLp.name); - addLayer(avgLp, node_proto); + Ptr layer = LayerFactory::createLayerInstance(layerParams.type, layerParams); } - else + catch (const std::exception& e) { - if (inpShape.size() != 4 && inpShape.size() != 5) - CV_Error(Error::StsNotImplemented, "Unsupported input shape of " + layer_type + " operation."); - - CV_Assert(axes.size() <= inpShape.size() - 2); - std::vector kernel_size(inpShape.size() - 2, 1); - if (axes.size() == 1 && (normalize_axis(axes.get(0), inpShape.size()) <= 1)) - { - int axis = normalize_axis(axes.get(0), inpShape.size()); - MatShape newShape = inpShape; - newShape[axis + 1] = total(newShape, axis + 1); - newShape.resize(axis + 2); - newShape.insert(newShape.begin(), 2 - axis, 1); - - LayerParams reshapeLp; - reshapeLp.type = "Reshape"; - reshapeLp.name = layerParams.name + "/reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], newShape.size())); - - node_proto.set_output(0, reshapeLp.name); - addLayer(reshapeLp, node_proto); - - kernel_size.resize(2); - kernel_size[0] = inpShape[axis]; - node_proto.set_input(0, node_proto.output(0)); - } - else - { - for (int i = 0; i < axes.size(); i++) { - int axis = normalize_axis(axes.get(i), inpShape.size()); - CV_Assert_N(axis >= 2 + i, axis < inpShape.size()); - kernel_size[axis - 2] = inpShape[axis]; - } - } - - LayerParams poolLp = layerParams; - poolLp.name = layerParams.name + "/avg"; - CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); - poolLp.set("kernel_size", DictValue::arrayInt(&kernel_size[0], kernel_size.size())); - - node_proto.set_output(0, poolLp.name); - addLayer(poolLp, node_proto); + CV_LOG_ERROR(NULL, "DNN/ONNX: Layer of type " << layerParams.type << "(" << layer_type << ") cannot be created with parameters " << layerParams << ". Error: " << e.what() + ); } - - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&targetShape[0], targetShape.size())); - - node_proto.set_input(0, node_proto.output(0)); - node_proto.set_output(0, layerParams.name); } - else if (!layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) + } + else + { + CV_LOG_ERROR(NULL, "DNN/ONNX: ERROR during processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " + << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) + ); + } + for (int i = 0; i < node_proto.input_size(); i++) + { + CV_LOG_INFO(NULL, " Input[" << i << "] = '" << node_proto.input(i) << "'"); + } + for (int i = 0; i < node_proto.output_size(); i++) + { + CV_LOG_INFO(NULL, " Output[" << i << "] = '" << node_proto.output(i) << "'"); + } + if (DNN_DIAGNOSTICS_RUN) + { + for (int i = 0; i < node_proto.output_size(); ++i) { - CV_CheckEQ(layerParams.get("keepdims"), 0, "layer only supports keepdims = false"); - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - int newShape[] = {1, 1, 1, -1}; - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 4)); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); + layer_id.insert(std::make_pair(node_proto.output(i), LayerInfo(0, i))); + outShapes[node_proto.output(i)] = outShapes[node_proto.input(0)]; + } + } + else + CV_Error(Error::StsError, cv::format("Node [%s]:(%s) parse error: %s", layer_type.c_str(), name.c_str(), e.what())); + } +} - LayerParams poolLp = layerParams; - poolLp.name = layerParams.name + "/pool"; - CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); +void ONNXImporter::parseMaxPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Pooling"; + layerParams.set("pool", "MAX"); + layerParams.set("ceil_mode", layerParams.has("pad_mode")); + addLayer(layerParams, node_proto); +} - node_proto.set_input(0, reshapeLp.name); - node_proto.set_output(0, poolLp.name); - addLayer(poolLp, node_proto); +void ONNXImporter::parseAveragePool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Pooling"; + layerParams.set("pool", "AVE"); + layerParams.set("ceil_mode", layerParams.has("pad_mode")); + layerParams.set("ave_pool_padded_area", framework_name == "pytorch"); + addLayer(layerParams, node_proto); +} - layerParams.type = "Reshape"; - int targetShape[] = {1}; - layerParams.set("dim", DictValue::arrayInt(&targetShape[0], 1)); +void ONNXImporter::parseReduce(LayerParams &layerParams, const opencv_onnx::NodeProto &node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + const std::string& layer_type = node_proto.op_type(); + + CV_Assert(node_proto.input_size() == 1); + layerParams.type = "Pooling"; + String pool; + if (layer_type == "GlobalMaxPool" || layer_type == "ReduceMax") + pool = "MAX"; + else if (layer_type == "ReduceSum") + pool = "SUM"; + else + pool = "AVE"; + layerParams.set("pool", pool); + layerParams.set("global_pooling", !layerParams.has("axes")); + if (layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) + { + MatShape inpShape = outShapes[node_proto.input(0)]; + DictValue axes = layerParams.get("axes"); + bool keepdims = layerParams.get("keepdims"); + MatShape targetShape; + std::vector shouldDelete(inpShape.size(), false); + for (int i = 0; i < axes.size(); i++) { + int axis = normalize_axis(axes.get(i), inpShape.size()); + shouldDelete[axis] = true; + } + for (int axis = 0; axis < inpShape.size(); ++axis){ + if (!shouldDelete[axis]) + targetShape.push_back(inpShape[axis]); + else if (keepdims) + targetShape.push_back(1); + } - node_proto.set_input(0, node_proto.output(0)); - node_proto.set_output(0, layerParams.name); + if (inpShape.size() == 3 && axes.size() <= 2) + { + int axis = normalize_axis(axes.get(0), inpShape.size()); + CV_CheckNE(axis, 0, ""); + + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + reshapeLp.set("axis", 0); + reshapeLp.set("num_axes", 1); + int newShape[] = {1, -1}; + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 2)); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + + LayerParams avgLp; + avgLp.name = layerParams.name + "/avg"; + avgLp.type = "Pooling"; + CV_Assert(layer_id.find(avgLp.name) == layer_id.end()); + avgLp.set("pool", pool); + if (axes.size() == 2) + { + CV_CheckEQ(normalize_axis(axes.get(0), inpShape.size()), 1, "Unsupported mode"); + CV_CheckEQ(normalize_axis(axes.get(1), inpShape.size()), 2, "Unsupported mode"); + avgLp.set("global_pooling", true); + } + else + { + avgLp.set(axis == 2 ? "global_pooling_w" : "global_pooling_h", true); + avgLp.set(axis == 2 ? "kernel_h" : "kernel_w", 1); } + + node_proto.set_input(0, reshapeLp.name); + node_proto.set_output(0, avgLp.name); + addLayer(avgLp, node_proto); } - else if (layer_type == "Slice") + else { - int axis = 0; - std::vector begin; - std::vector end; - std::vector steps; - int inp_size = node_proto.input_size(); + if (inpShape.size() != 4 && inpShape.size() != 5) + CV_Error(Error::StsNotImplemented, "Unsupported input shape of " + layer_type + " operation."); - if (inp_size == 1) + CV_Assert(axes.size() <= inpShape.size() - 2); + std::vector kernel_size(inpShape.size() - 2, 1); + if (axes.size() == 1 && (normalize_axis(axes.get(0), inpShape.size()) <= 1)) { - if (layerParams.has("axes")) { - DictValue axes = layerParams.get("axes"); - for (int i = 1; i < axes.size(); ++i) { - CV_Assert(axes.get(i - 1) == axes.get(i) - 1); - } - axis = axes.get(0); - } + int axis = normalize_axis(axes.get(0), inpShape.size()); + MatShape newShape = inpShape; + newShape[axis + 1] = total(newShape, axis + 1); + newShape.resize(axis + 2); + newShape.insert(newShape.begin(), 2 - axis, 1); - DictValue starts = layerParams.get("starts"); - DictValue ends = layerParams.get("ends"); - CV_Assert(starts.size() == ends.size()); + LayerParams reshapeLp; + reshapeLp.type = "Reshape"; + reshapeLp.name = layerParams.name + "/reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], newShape.size())); - if (axis > 0) { - begin.resize(axis, 0); - end.resize(axis, -1); - } - for (int i = 0; i < starts.size(); ++i) - { - begin.push_back(starts.get(i)); - int finish = ends.get(i); - end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim - } - } else { // inp_size > 1 - CV_Assert(inp_size >= 3); - for (int i = 1; i < inp_size; i++) { - CV_Assert(constBlobs.find(node_proto.input(i)) != constBlobs.end()); - } - Mat start_blob = getBlob(node_proto, 1); - Mat end_blob = getBlob(node_proto, 2); - CV_Assert(start_blob.total() == end_blob.total()); - - if (inp_size > 3) { - Mat axes_blob = getBlob(node_proto, 3); - const int* axes = (int*)axes_blob.data; - for (int i = 1; i < axes_blob.total(); ++i) { - CV_Assert(axes[i - 1] == axes[i] - 1); - } - axis = axes[0]; - } + node_proto.set_output(0, reshapeLp.name); + addLayer(reshapeLp, node_proto); - const int* starts = start_blob.ptr(); - const int* ends = end_blob.ptr(); - if (axis > 0) { - begin.resize(axis, 0); - end.resize(axis, -1); - } - std::copy(starts, starts + start_blob.total(), std::back_inserter(begin)); - for (int i = 0; i < end_blob.total(); ++i) - { - int finish = ends[i]; - end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim + kernel_size.resize(2); + kernel_size[0] = inpShape[axis]; + node_proto.set_input(0, node_proto.output(0)); + } + else + { + for (int i = 0; i < axes.size(); i++) { + int axis = normalize_axis(axes.get(i), inpShape.size()); + CV_Assert_N(axis >= 2 + i, axis < inpShape.size()); + kernel_size[axis - 2] = inpShape[axis]; } + } - if (inp_size == 5) { - CV_Assert(constBlobs.find(node_proto.input(4)) != constBlobs.end()); - Mat step_blob = getBlob(node_proto, 4); - const int* steps_ptr = step_blob.ptr(); + LayerParams poolLp = layerParams; + poolLp.name = layerParams.name + "/avg"; + CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); + poolLp.set("kernel_size", DictValue::arrayInt(&kernel_size[0], kernel_size.size())); - if (axis > 0) - steps.resize(axis, 1); + node_proto.set_output(0, poolLp.name); + addLayer(poolLp, node_proto); + } - std::copy(steps_ptr, steps_ptr + step_blob.total(), std::back_inserter(steps)); + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&targetShape[0], targetShape.size())); - // Very strange application for Slice op with tensor reversing. - // We just workaround it for 2d constants. - if (constBlobs.find(node_proto.input(0)) != constBlobs.end() && - axis == 0 && - start_blob.at(0) == -1 && step_blob.at(0) == -1 && - end_blob.at(0) == std::numeric_limits::min()) - { - Mat inp = getBlob(node_proto, 0); - if (inp.dims == 2) - { - Mat flipped; - flip(inp, flipped, 0); - addConstant(layerParams.name, flipped); - return; - } - } - } - } - layerParams.set("begin", DictValue::arrayInt(&begin[0], begin.size())); - layerParams.set("end", DictValue::arrayInt(&end[0], end.size())); - layerParams.set("axis", axis); + node_proto.set_input(0, node_proto.output(0)); + node_proto.set_output(0, layerParams.name); + } + else if (!layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) + { + CV_CheckEQ(layerParams.get("keepdims"), 0, "layer only supports keepdims = false"); + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + int newShape[] = {1, 1, 1, -1}; + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 4)); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + + LayerParams poolLp = layerParams; + poolLp.name = layerParams.name + "/pool"; + CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); + + node_proto.set_input(0, reshapeLp.name); + node_proto.set_output(0, poolLp.name); + addLayer(poolLp, node_proto); + + layerParams.type = "Reshape"; + int targetShape[] = {1}; + layerParams.set("dim", DictValue::arrayInt(&targetShape[0], 1)); + + node_proto.set_input(0, node_proto.output(0)); + node_proto.set_output(0, layerParams.name); + } + addLayer(layerParams, node_proto); +} - if (!steps.empty()) - layerParams.set("steps", DictValue::arrayInt(&steps[0], steps.size())); +void ONNXImporter::parseSlice(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int axis = 0; + std::vector begin; + std::vector end; + std::vector steps; + int inp_size = node_proto.input_size(); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat inp = getBlob(node_proto, 0); - std::vector inputs, sliced; - inputs.push_back(inp); - runLayer(layerParams, inputs, sliced); - CV_Assert(sliced.size() == 1); - addConstant(layerParams.name, sliced[0]); - return; + if (inp_size == 1) + { + if (layerParams.has("axes")) { + DictValue axes = layerParams.get("axes"); + for (int i = 1; i < axes.size(); ++i) { + CV_Assert(axes.get(i - 1) == axes.get(i) - 1); } + axis = axes.get(0); } - else if (layer_type == "Split") - { - if (layerParams.has("split")) - { - DictValue splits = layerParams.get("split"); - const int numSplits = splits.size(); - CV_Assert(numSplits > 1); - std::vector slicePoints(numSplits - 1, splits.get(0)); - for (int i = 1; i < splits.size() - 1; ++i) - { - slicePoints[i] = slicePoints[i - 1] + splits.get(i - 1); - } - layerParams.set("slice_point", DictValue::arrayInt(&slicePoints[0], slicePoints.size())); - } - else - { - layerParams.set("num_split", node_proto.output_size()); - } - layerParams.type = "Slice"; + DictValue starts = layerParams.get("starts"); + DictValue ends = layerParams.get("ends"); + CV_Assert(starts.size() == ends.size()); + + if (axis > 0) { + begin.resize(axis, 0); + end.resize(axis, -1); } - else if (layer_type == "Add" || layer_type == "Sum" || layer_type == "Sub") + for (int i = 0; i < starts.size(); ++i) { - bool isSub = layer_type == "Sub"; - CV_CheckEQ(node_proto.input_size(), 2, ""); - bool is_const_0 = layer_id.find(node_proto.input(0)) == layer_id.end(); - bool is_const_1 = layer_id.find(node_proto.input(1)) == layer_id.end(); - if (is_const_0 && is_const_1) - { - Mat blob_0 = getBlob(node_proto, 0); - Mat blob_1 = getBlob(node_proto, 1); - CV_Assert(blob_0.size == blob_1.size); - Mat output = isSub ? (blob_0 - blob_1) : (blob_0 + blob_1); - addConstant(layerParams.name, output); - return; - } - else if (is_const_0 || is_const_1) - { - int const_blob_id = is_const_0 ? 0 : 1; - Mat blob = getBlob(node_proto, const_blob_id); - int blob_total = blob.total(); - if (blob_total == 1) { - layerParams.type = "Power"; - layerParams.set("shift", (isSub ? -1 : 1) * blob.ptr()[0]); - } - else { - MatShape inpShape = outShapes[node_proto.input(1 - const_blob_id)]; - if (shape(blob) == inpShape) - { - LayerParams constParams; - constParams.name = layerParams.name + "/const"; - constParams.type = "Const"; - constParams.blobs.push_back((isSub ? -1 : 1) * blob); - int id; - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(constParams.name, constParams.type, constParams); - else - id = dstNet.addLayer(constParams.name, constParams.type, constParams); - layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); - outShapes[constParams.name] = shape(blob); - - layerParams.type = "Eltwise"; - node_proto.set_input(const_blob_id, constParams.name); - } - else - { - layerParams.type = "Scale"; - layerParams.set("bias_term", true); - int axis = 1; - for (int i = 0; i < graph_proto.initializer_size(); i++) - { - opencv_onnx::TensorProto tensor_proto = graph_proto.initializer(i); - if (tensor_proto.name() == node_proto.input(const_blob_id)) - { - axis = inpShape.size() - tensor_proto.dims_size(); - break; - } - } - layerParams.set("axis", axis); - blob = blob.reshape(1, 1); - layerParams.blobs.push_back((isSub ? -1 : 1) * blob); - } - } - } - else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) - { - layerParams.type = "Eltwise"; - if (isSub) - { - static float subCoeffs[] = {1.f, -1.f}; - layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); - } - } - else - { - if (isSub) - { - LayerParams powerParams; - powerParams.name = layerParams.name + "/neg"; - powerParams.type = "Power"; - powerParams.set("scale", -1); - - int id; - //Create Power layer - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(powerParams.name, powerParams.type, powerParams); - else - id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(1)); - CV_Assert(layerId != layer_id.end()); - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); - outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - - //Replace input to Power - node_proto.set_input(1, powerParams.name); - } - layerParams.type = "Scale"; - layerParams.set("bias_term", true); - } + begin.push_back(starts.get(i)); + int finish = ends.get(i); + end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim } - else if (layer_type == "Pow") - { - if (layer_id.find(node_proto.input(1)) != layer_id.end()) - CV_Error(Error::StsNotImplemented, "Unsupported Pow op with variable power"); - - Mat blob = getBlob(node_proto, 1); - if (blob.total() != 1) - CV_Error(Error::StsNotImplemented, "Pow op supports only scalar power"); - - blob.convertTo(blob, CV_32F); - layerParams.type = "Power"; - layerParams.set("power", blob.ptr()[0]); + } else { // inp_size > 1 + CV_Assert(inp_size >= 3); + for (int i = 1; i < inp_size; i++) { + CV_Assert(constBlobs.find(node_proto.input(i)) != constBlobs.end()); } - else if (layer_type == "Max") - { - layerParams.type = "Eltwise"; - layerParams.set("operation", "max"); + Mat start_blob = getBlob(node_proto, 1); + Mat end_blob = getBlob(node_proto, 2); + CV_Assert(start_blob.total() == end_blob.total()); + + if (inp_size > 3) { + Mat axes_blob = getBlob(node_proto, 3); + const int* axes = (int*)axes_blob.data; + for (int i = 1; i < axes_blob.total(); ++i) { + CV_Assert(axes[i - 1] == axes[i] - 1); + } + axis = axes[0]; } - else if (layer_type == "Neg") - { - layerParams.type = "Power"; - layerParams.set("scale", -1); + + const int* starts = start_blob.ptr(); + const int* ends = end_blob.ptr(); + if (axis > 0) { + begin.resize(axis, 0); + end.resize(axis, -1); } - else if (layer_type == "Constant") + std::copy(starts, starts + start_blob.total(), std::back_inserter(begin)); + for (int i = 0; i < end_blob.total(); ++i) { - CV_Assert(node_proto.input_size() == 0); - CV_Assert(layerParams.blobs.size() == 1); - addConstant(layerParams.name, layerParams.blobs[0]); - return; + int finish = ends[i]; + end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim } - else if (layer_type == "LSTM") - { - LayerParams lstmParams = layerParams; - lstmParams.name += "/lstm"; - - // https://pytorch.org/docs/stable/nn.html#lstm - CV_Assert(node_proto.input_size() == 7); - Mat Wx = getBlob(node_proto, 1); - Mat Wh = getBlob(node_proto, 2); - Mat b = getBlob(node_proto, 3); - Mat h0 = getBlob(node_proto, 5); - Mat c0 = getBlob(node_proto, 6); - - b = b.reshape(1, b.size[0]); - - const int numHidden = lstmParams.get("hidden_size"); - const int numDirs = Wx.size[0]; // Is 1 for forward only and 2 for bidirectional LSTM. - const int numFeatures = Wx.size[2]; - Mat bx = b.colRange(0, b.cols / 2); - Mat bh = b.colRange(b.cols / 2, b.cols); - b = bx + bh; - - // IFGO->IGFO - for (int k = 0; k < numDirs; ++k) + + if (inp_size == 5) { + CV_Assert(constBlobs.find(node_proto.input(4)) != constBlobs.end()); + Mat step_blob = getBlob(node_proto, 4); + const int* steps_ptr = step_blob.ptr(); + + if (axis > 0) + steps.resize(axis, 1); + + std::copy(steps_ptr, steps_ptr + step_blob.total(), std::back_inserter(steps)); + + // Very strange application for Slice op with tensor reversing. + // We just workaround it for 2d constants. + if (constBlobs.find(node_proto.input(0)) != constBlobs.end() && + axis == 0 && + start_blob.at(0) == -1 && step_blob.at(0) == -1 && + end_blob.at(0) == std::numeric_limits::min()) { - float* WxData = Wx.ptr(k); - float* WhData = Wh.ptr(k); - float* biasData = b.ptr(k); - for (int j = 0; j < numHidden; ++j) + Mat inp = getBlob(node_proto, 0); + if (inp.dims == 2) { - for (int i = 0; i < numFeatures; ++i) - { - std::swap(WxData[(numHidden + j) * numFeatures + i], - WxData[(numHidden * 2 + j) * numFeatures + i]); - } - for (int i = 0; i < numHidden; ++i) - { - std::swap(WhData[(numHidden + j) * numHidden + i], - WhData[(numHidden * 2 + j) * numHidden + i]); - } - std::swap(biasData[numHidden + j], biasData[numHidden * 2 + j]); + Mat flipped; + flip(inp, flipped, 0); + addConstant(layerParams.name, flipped); + return; } } - Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); - Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); - h0 = h0.reshape(1, h0.size[0] * h0.size[1]); - c0 = c0.reshape(1, c0.size[0] * c0.size[1]); + } + } + layerParams.set("begin", DictValue::arrayInt(&begin[0], begin.size())); + layerParams.set("end", DictValue::arrayInt(&end[0], end.size())); + layerParams.set("axis", axis); - lstmParams.blobs.resize(5); - lstmParams.blobs[0] = Wh; - lstmParams.blobs[1] = Wx; - lstmParams.blobs[2] = b; - lstmParams.blobs[3] = h0; - lstmParams.blobs[4] = c0; - lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); + if (!steps.empty()) + layerParams.set("steps", DictValue::arrayInt(&steps[0], steps.size())); - node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name - addLayer(lstmParams, node_proto); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inp = getBlob(node_proto, 0); + std::vector inputs, sliced; + inputs.push_back(inp); + runLayer(layerParams, inputs, sliced); + CV_Assert(sliced.size() == 1); + addConstant(layerParams.name, sliced[0]); + return; + } + addLayer(layerParams, node_proto); +} - MatShape lstmShape = outShapes[node_proto.output(0)]; +void ONNXImporter::parseSplit(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (layerParams.has("split")) + { + DictValue splits = layerParams.get("split"); + const int numSplits = splits.size(); + CV_Assert(numSplits > 1); - // Add fake 1 as it is done in ONNX - lstmShape.insert(lstmShape.begin() + 1, 1); + std::vector slicePoints(numSplits - 1, splits.get(0)); + for (int i = 1; i < splits.size() - 1; ++i) + { + slicePoints[i] = slicePoints[i - 1] + splits.get(i - 1); + } + layerParams.set("slice_point", DictValue::arrayInt(&slicePoints[0], slicePoints.size())); + } + else + { + layerParams.set("num_split", node_proto.output_size()); + } + layerParams.type = "Slice"; + addLayer(layerParams, node_proto); +} - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&lstmShape[0], lstmShape.size())); - node_proto.set_input(0, lstmParams.name); // redirect input to LSTM - node_proto.set_output(0, layerParams.name); // keep origin LSTM's name +void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + const std::string& layer_type = node_proto.op_type(); + bool isSub = layer_type == "Sub"; + CV_CheckEQ(node_proto.input_size(), 2, ""); + bool is_const_0 = layer_id.find(node_proto.input(0)) == layer_id.end(); + bool is_const_1 = layer_id.find(node_proto.input(1)) == layer_id.end(); + if (is_const_0 && is_const_1) + { + Mat blob_0 = getBlob(node_proto, 0); + Mat blob_1 = getBlob(node_proto, 1); + CV_Assert(blob_0.size == blob_1.size); + Mat output = isSub ? (blob_0 - blob_1) : (blob_0 + blob_1); + addConstant(layerParams.name, output); + return; + } + else if (is_const_0 || is_const_1) + { + int const_blob_id = is_const_0 ? 0 : 1; + Mat blob = getBlob(node_proto, const_blob_id); + int blob_total = blob.total(); + if (blob_total == 1) { + layerParams.type = "Power"; + layerParams.set("shift", (isSub ? -1 : 1) * blob.ptr()[0]); } - else if (layer_type == "ImageScaler") - { - const float scale = layerParams.has("scale") ? layerParams.get("scale") : 1.0f; - layerParams.erase("scale"); + else { + MatShape inpShape = outShapes[node_proto.input(1 - const_blob_id)]; + if (shape(blob) == inpShape) + { + LayerParams constParams; + constParams.name = layerParams.name + "/const"; + constParams.type = "Const"; + constParams.blobs.push_back((isSub ? -1 : 1) * blob); + int id; + if (DNN_DIAGNOSTICS_RUN) + id = utilNet.addLayer(constParams.name, constParams.type, constParams); + else + id = dstNet.addLayer(constParams.name, constParams.type, constParams); + layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); + outShapes[constParams.name] = shape(blob); - if (layerParams.has("bias")) + layerParams.type = "Eltwise"; + node_proto.set_input(const_blob_id, constParams.name); + } + else { layerParams.type = "Scale"; - layerParams.blobs.push_back( - Mat(Size(1, layerParams.get("bias").size()), CV_32FC1, scale)); - layerParams.set("bias_term", true); - Mat bias(1, layerParams.get("bias").size(), CV_32FC1); - for (int j = 0; j < bias.total(); j++) { - bias.at(0, j) = layerParams.get("bias").getRealValue(j); + int axis = 1; + for (int i = 0; i < graph_proto.initializer_size(); i++) + { + opencv_onnx::TensorProto tensor_proto = graph_proto.initializer(i); + if (tensor_proto.name() == node_proto.input(const_blob_id)) + { + axis = inpShape.size() - tensor_proto.dims_size(); + break; + } } - layerParams.blobs.push_back(bias); - layerParams.erase("bias"); - } - else { - layerParams.set("scale", scale); - layerParams.type = "Power"; + layerParams.set("axis", axis); + blob = blob.reshape(1, 1); + layerParams.blobs.push_back((isSub ? -1 : 1) * blob); } } - else if (layer_type == "Clip") - { - layerParams.type = "ReLU6"; - replaceLayerParam(layerParams, "min", "min_value"); - replaceLayerParam(layerParams, "max", "max_value"); - - } - else if (layer_type == "LeakyRelu") - { - layerParams.type = "ReLU"; - replaceLayerParam(layerParams, "alpha", "negative_slope"); - } - else if (layer_type == "Relu") - { - layerParams.type = "ReLU"; - } - else if (layer_type == "Elu") - { - layerParams.type = "ELU"; - } - else if (layer_type == "Tanh") - { - layerParams.type = "TanH"; - } - else if (layer_type == "PRelu") - { - layerParams.type = "PReLU"; - layerParams.blobs.push_back(getBlob(node_proto, 1)); - } - else if (layer_type == "LRN") + } + else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) + { + layerParams.type = "Eltwise"; + if (isSub) { - replaceLayerParam(layerParams, "size", "local_size"); + static float subCoeffs[] = {1.f, -1.f}; + layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); } - else if (layer_type == "InstanceNormalization") + } + else + { + if (isSub) { - if (node_proto.input_size() != 3) - CV_Error(Error::StsNotImplemented, - "Expected input, scale, bias"); - - layerParams.blobs.resize(4); - layerParams.blobs[2] = getBlob(node_proto, 1); // weightData - layerParams.blobs[3] = getBlob(node_proto, 2); // biasData - layerParams.set("has_bias", true); - layerParams.set("has_weight", true); - - // Get number of channels in input - int size = layerParams.blobs[2].total(); - layerParams.blobs[0] = Mat::zeros(size, 1, CV_32F); // mean - layerParams.blobs[1] = Mat::ones(size, 1, CV_32F); // std - - LayerParams mvnParams; - mvnParams.name = layerParams.name + "/MVN"; - mvnParams.type = "MVN"; - mvnParams.set("eps", layerParams.get("epsilon")); - layerParams.erase("epsilon"); - - //Create MVN layer + LayerParams powerParams; + powerParams.name = layerParams.name + "/neg"; + powerParams.type = "Power"; + powerParams.set("scale", -1); + int id; + //Create Power layer if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); + id = utilNet.addLayer(powerParams.name, powerParams.type, powerParams); else - id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); + id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(0)); + IterLayerId_t layerId = layer_id.find(node_proto.input(1)); CV_Assert(layerId != layer_id.end()); if (DNN_DIAGNOSTICS_RUN) utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); else dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); //Add shape - layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0))); - outShapes[mvnParams.name] = outShapes[node_proto.input(0)]; + layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); + outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - //Replace Batch Norm's input to MVN - node_proto.set_input(0, mvnParams.name); - layerParams.type = "BatchNorm"; + //Replace input to Power + node_proto.set_input(1, powerParams.name); } - else if (layer_type == "BatchNormalization") - { - if (node_proto.input_size() != 5) - CV_Error(Error::StsNotImplemented, - "Expected input, scale, bias, mean and var"); + layerParams.type = "Scale"; + layerParams.set("bias_term", true); + } + addLayer(layerParams, node_proto); +} - layerParams.type = "BatchNorm"; - replaceLayerParam(layerParams, "epsilon", "eps"); - replaceLayerParam(layerParams, "spatial", "use_global_stats"); +void ONNXImporter::parsePow(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (layer_id.find(node_proto.input(1)) != layer_id.end()) + CV_Error(Error::StsNotImplemented, "Unsupported Pow op with variable power"); - Mat meanData = getBlob(node_proto, 3); - Mat stdData = getBlob(node_proto, 4); + Mat blob = getBlob(node_proto, 1); + if (blob.total() != 1) + CV_Error(Error::StsNotImplemented, "Pow op supports only scalar power"); - layerParams.blobs.push_back(meanData); - layerParams.blobs.push_back(stdData); + blob.convertTo(blob, CV_32F); + layerParams.type = "Power"; + layerParams.set("power", blob.ptr()[0]); + addLayer(layerParams, node_proto); +} - if (!node_proto.input(1).empty()) { - layerParams.set("has_weight", true); - layerParams.blobs.push_back(getBlob(node_proto, 1)); // weightData - } else { - layerParams.set("has_weight", false); - } +void ONNXImporter::parseMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Eltwise"; + layerParams.set("operation", "max"); + addLayer(layerParams, node_proto); +} - if (!node_proto.input(2).empty()) { - layerParams.set("has_bias", true); - layerParams.blobs.push_back(getBlob(node_proto, 2)); // biasData - } else { - layerParams.set("has_bias", false); - } - } - else if (layer_type == "Gemm") - { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "InnerProduct"; - Mat weights = getBlob(node_proto, 1); - int ind_num_out = 0; - if (layerParams.has("transB") && !layerParams.get("transB")) { - transpose(weights, weights); - ind_num_out = 1; - } - layerParams.blobs.push_back(weights); +void ONNXImporter::parseNeg(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Power"; + layerParams.set("scale", -1); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseConstant(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 0); + CV_Assert(layerParams.blobs.size() == 1); + addConstant(layerParams.name, layerParams.blobs[0]); +} - if (node_proto.input_size() == 3) { - Mat bias = getBlob(node_proto, 2); - layerParams.blobs.push_back(bias); +void ONNXImporter::parseLSTM(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + LayerParams lstmParams = layerParams; + lstmParams.name += "/lstm"; + + // https://pytorch.org/docs/stable/nn.html#lstm + CV_Assert(node_proto.input_size() == 7); + Mat Wx = getBlob(node_proto, 1); + Mat Wh = getBlob(node_proto, 2); + Mat b = getBlob(node_proto, 3); + Mat h0 = getBlob(node_proto, 5); + Mat c0 = getBlob(node_proto, 6); + + b = b.reshape(1, b.size[0]); + + const int numHidden = lstmParams.get("hidden_size"); + const int numDirs = Wx.size[0]; // Is 1 for forward only and 2 for bidirectional LSTM. + const int numFeatures = Wx.size[2]; + Mat bx = b.colRange(0, b.cols / 2); + Mat bh = b.colRange(b.cols / 2, b.cols); + b = bx + bh; + + // IFGO->IGFO + for (int k = 0; k < numDirs; ++k) + { + float* WxData = Wx.ptr(k); + float* WhData = Wh.ptr(k); + float* biasData = b.ptr(k); + for (int j = 0; j < numHidden; ++j) + { + for (int i = 0; i < numFeatures; ++i) + { + std::swap(WxData[(numHidden + j) * numFeatures + i], + WxData[(numHidden * 2 + j) * numFeatures + i]); } - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + for (int i = 0; i < numHidden; ++i) { - Mat inputBuf = getBlob(node_proto, 0); - - LayerParams constParams; - constParams.name = node_proto.input(0); - constParams.type = "Const"; - constParams.blobs.push_back(inputBuf); - - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); + std::swap(WhData[(numHidden + j) * numHidden + i], + WhData[(numHidden * 2 + j) * numHidden + i]); } - - layerParams.set("num_output", layerParams.blobs[0].size[ind_num_out]); - layerParams.set("bias_term", node_proto.input_size() == 3); + std::swap(biasData[numHidden + j], biasData[numHidden * 2 + j]); } - else if (layer_type == "MatMul") - { - CV_Assert(node_proto.input_size() == 2); - layerParams.type = "InnerProduct"; - layerParams.set("bias_term", false); - CV_Assert(constBlobs.find(node_proto.input(0)) == constBlobs.end()); - int firstInpDims = outShapes[node_proto.input(0)].size(); - int secondInpDims; - - if (constBlobs.find(node_proto.input(1)) != constBlobs.end()) - { - Mat blob = getBlob(node_proto, 1); - secondInpDims = blob.dims; - layerParams.blobs.push_back(blob.t()); - layerParams.set("num_output", layerParams.blobs[0].size[0]); - } else { - secondInpDims = outShapes[node_proto.input(1)].size(); - } - layerParams.set("axis", firstInpDims - secondInpDims + 1); + } + Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); + Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + c0 = c0.reshape(1, c0.size[0] * c0.size[1]); + + lstmParams.blobs.resize(5); + lstmParams.blobs[0] = Wh; + lstmParams.blobs[1] = Wx; + lstmParams.blobs[2] = b; + lstmParams.blobs[3] = h0; + lstmParams.blobs[4] = c0; + lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); + + node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name + addLayer(lstmParams, node_proto); + + MatShape lstmShape = outShapes[node_proto.output(0)]; + + // Add fake 1 as it is done in ONNX + lstmShape.insert(lstmShape.begin() + 1, 1); + + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&lstmShape[0], lstmShape.size())); + node_proto.set_input(0, lstmParams.name); // redirect input to LSTM + node_proto.set_output(0, layerParams.name); // keep origin LSTM's name + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseImageScaler(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + const float scale = layerParams.has("scale") ? layerParams.get("scale") : 1.0f; + layerParams.erase("scale"); + + if (layerParams.has("bias")) + { + layerParams.type = "Scale"; + layerParams.blobs.push_back( + Mat(Size(1, layerParams.get("bias").size()), CV_32FC1, scale)); + + layerParams.set("bias_term", true); + Mat bias(1, layerParams.get("bias").size(), CV_32FC1); + for (int j = 0; j < bias.total(); j++) { + bias.at(0, j) = layerParams.get("bias").getRealValue(j); } - else if (layer_type == "Mul" || layer_type == "Div") - { - CV_Assert(node_proto.input_size() == 2); + layerParams.blobs.push_back(bias); + layerParams.erase("bias"); + } + else { + layerParams.set("scale", scale); + layerParams.type = "Power"; + } + addLayer(layerParams, node_proto); +} - bool isDiv = layer_type == "Div"; - int constId = -1; - bool haveVariables = false; - for (int i = 0; i < 2; ++i) - { - if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) - constId = i; - else - haveVariables = true; - } - if (constId != -1 && haveVariables) - { - Mat blob = getBlob(node_proto, constId); - blob = blob.reshape(1, 1); - if (blob.total() == 1) { - float blob_value = blob.ptr()[0]; - float coeff = isDiv ? 1.0 / blob_value : blob_value; - layerParams.set("scale", coeff); - layerParams.type = "Power"; - } - else { - if (isDiv) - divide(1.0, blob, blob); - layerParams.blobs.push_back(blob); - layerParams.type = "Scale"; - } - } - else if (!haveVariables) - { - Mat inp0 = getBlob(node_proto, 0); - Mat inp1 = getBlob(node_proto, 1); +void ONNXImporter::parseClip(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ReLU6"; + replaceLayerParam(layerParams, "min", "min_value"); + replaceLayerParam(layerParams, "max", "max_value"); + addLayer(layerParams, node_proto); +} - if (inp0.size != inp1.size && (inp0.total() != 1 || inp1.total() != 1)) - CV_Error_(Error::StsNotImplemented, ("Different shapes case is not supported with constant inputs: %s", layer_type.c_str())); +void ONNXImporter::parseLeakyRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ReLU"; + replaceLayerParam(layerParams, "alpha", "negative_slope"); + addLayer(layerParams, node_proto); +} - if (inp0.total() == 1 && inp1.total() == 1 && inp0.dims != inp1.dims) - { - if (inp0.dims < inp1.dims) - { - inp0 = inp0.reshape(1, inp1.dims, inp1.size); - inp0.dims = inp1.dims; - } - else - { - inp1 = inp1.reshape(1, inp0.dims, inp0.size); - inp1.dims = inp0.dims; - } - } +void ONNXImporter::parseRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ReLU"; + addLayer(layerParams, node_proto); +} - Mat out; - if (inp0.total() != inp1.total()) - { - if (inp0.total() == 1) - { - float inp0_value = inp0.ptr()[0]; - float coeff = isDiv ? 1.0 / inp0_value : inp0_value; - multiply(inp1, coeff, out); - } - else - { - float inp1_value = inp1.ptr()[0]; - float coeff = isDiv ? 1.0 / inp1_value : inp1_value; - multiply(inp0, coeff, out); - } +void ONNXImporter::parseElu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ELU"; + addLayer(layerParams, node_proto); +} - } - else - { - out = isDiv ? inp0 / inp1 : inp0.mul(inp1); - } +void ONNXImporter::parseTanh(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "TanH"; + addLayer(layerParams, node_proto); +} - if (inp0.dims == 1 && inp1.dims == 1) - out.dims = 1; // to workaround dims == 1 - addConstant(layerParams.name, out); - return; - } - else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) - { - layerParams.type = "Eltwise"; - layerParams.set("operation", isDiv ? "div" : "prod"); - } - else - { - // Scale layer allocate output with the first input shape - if (total(outShapes[node_proto.input(0)]) < total(outShapes[node_proto.input(1)])) - { - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(1)); - proto.add_input(node_proto.input(0)); - proto.add_output(layerParams.name); - node_proto = proto; - } +void ONNXImporter::parsePRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "PReLU"; + layerParams.blobs.push_back(getBlob(node_proto, 1)); + addLayer(layerParams, node_proto); +} - if (isDiv) - { - LayerParams powerParams; - powerParams.name = layerParams.name + "/inv"; - powerParams.type = "Power"; - powerParams.set("power", -1); - - int id; - //Create Power layer - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(powerParams.name, powerParams.type, powerParams); - else - id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(1)); - CV_Assert(layerId != layer_id.end()); - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); - outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - - //Replace input to Power - node_proto.set_input(1, powerParams.name); - } - layerParams.type = "Scale"; - } - } - else if (layer_type == "Conv") - { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "Convolution"; - for (int j = 1; j < node_proto.input_size(); j++) { - if (constBlobs.find(node_proto.input(j)) != constBlobs.end()) - { - layerParams.blobs.push_back(getBlob(node_proto, j)); - } - } - int outCn = layerParams.blobs.empty() ? outShapes[node_proto.input(1)][0] : layerParams.blobs[0].size[0]; - layerParams.set("num_output", outCn); +void ONNXImporter::parseLRN(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + replaceLayerParam(layerParams, "size", "local_size"); + addLayer(layerParams, node_proto); +} - // Check for asymmetric padding in Conv2D - if (layerParams.has("pad")) - { - bool asymmetricPadding = false; - DictValue pads = layerParams.get("pad"); - const int dims = pads.size() / 2; - for (int i = 0; i < dims; ++i) - { - if (pads.get(i) != pads.get(i + dims)) - { - asymmetricPadding = true; - break; - } - } - if (asymmetricPadding && pads.size() == 4) // [pad_t, pad_l, pad_b, pad_r] - { - layerParams.erase("pad"); - // No paddings required for N, C axis - std::vector paddings(4, 0); - // Add paddings for H, W axis - for (int i = 0; i < dims; ++i) - { - paddings.push_back(pads.get(i)); - paddings.push_back(pads.get(dims + i)); - } - LayerParams padLp; - padLp.name = layerParams.name + "/pad"; - padLp.type = "Padding"; - padLp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size())); +void ONNXImporter::parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + if (node_proto.input_size() != 3) + CV_Error(Error::StsNotImplemented, + "Expected input, scale, bias"); + + layerParams.blobs.resize(4); + layerParams.blobs[2] = getBlob(node_proto, 1); // weightData + layerParams.blobs[3] = getBlob(node_proto, 2); // biasData + layerParams.set("has_bias", true); + layerParams.set("has_weight", true); + + // Get number of channels in input + int size = layerParams.blobs[2].total(); + layerParams.blobs[0] = Mat::zeros(size, 1, CV_32F); // mean + layerParams.blobs[1] = Mat::ones(size, 1, CV_32F); // std + + LayerParams mvnParams; + mvnParams.name = layerParams.name + "/MVN"; + mvnParams.type = "MVN"; + mvnParams.set("eps", layerParams.get("epsilon")); + layerParams.erase("epsilon"); + + //Create MVN layer + int id; + if (DNN_DIAGNOSTICS_RUN) + id = utilNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); + else + id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); + //Connect to input + IterLayerId_t layerId = layer_id.find(node_proto.input(0)); + CV_Assert(layerId != layer_id.end()); + if (DNN_DIAGNOSTICS_RUN) + utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + else + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0))); + outShapes[mvnParams.name] = outShapes[node_proto.input(0)]; + + //Replace Batch Norm's input to MVN + node_proto.set_input(0, mvnParams.name); + layerParams.type = "BatchNorm"; + addLayer(layerParams, node_proto); +} - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(padLp.name); +void ONNXImporter::parseBatchNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (node_proto.input_size() != 5) + CV_Error(Error::StsNotImplemented, + "Expected input, scale, bias, mean and var"); - addLayer(padLp, proto); - node_proto.set_input(0, padLp.name); - } - } - } - else if (layer_type == "ConvTranspose") - { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "Deconvolution"; - for (int j = 1; j < node_proto.input_size(); j++) { - layerParams.blobs.push_back(getBlob(node_proto, j)); - } - layerParams.set("num_output", layerParams.blobs[0].size[1] * layerParams.get("group", 1)); - layerParams.set("bias_term", node_proto.input_size() == 3); + layerParams.type = "BatchNorm"; + replaceLayerParam(layerParams, "epsilon", "eps"); + replaceLayerParam(layerParams, "spatial", "use_global_stats"); - if (!layerParams.has("kernel_size")) - CV_Error(Error::StsNotImplemented, - "Required attribute 'kernel_size' is not present."); + Mat meanData = getBlob(node_proto, 3); + Mat stdData = getBlob(node_proto, 4); - if (layerParams.has("output_shape")) - { - const DictValue& outShape = layerParams.get("output_shape"); - DictValue strides = layerParams.get("stride"); - DictValue kernel = layerParams.get("kernel_size"); + layerParams.blobs.push_back(meanData); + layerParams.blobs.push_back(stdData); - String padMode; - std::vector adjust_pads; - if (layerParams.has("pad_mode")) - { - padMode = toUpperCase(layerParams.get("pad_mode")); - if (padMode != "SAME" && padMode != "VALID") - CV_Error(Error::StsError, "Unsupported padding mode " + padMode); + if (!node_proto.input(1).empty()) { + layerParams.set("has_weight", true); + layerParams.blobs.push_back(getBlob(node_proto, 1)); // weightData + } else { + layerParams.set("has_weight", false); + } - for (int i = 0; i < strides.size(); i++) - { - int sz = outShape.get(2 + i); - int stride = strides.get(i); - adjust_pads.push_back(padMode == "SAME"? (sz - 1) % stride : - (sz - kernel.get(i)) % stride); - } - layerParams.set("adj", DictValue::arrayInt(&adjust_pads[0], adjust_pads.size())); - } - } - else if (layerParams.has("output_padding")) - { - replaceLayerParam(layerParams, "output_padding", "adj"); - } - } - else if (layer_type == "Transpose") - { - layerParams.type = "Permute"; - replaceLayerParam(layerParams, "perm", "order"); + if (!node_proto.input(2).empty()) { + layerParams.set("has_bias", true); + layerParams.blobs.push_back(getBlob(node_proto, 2)); // biasData + } else { + layerParams.set("has_bias", false); + } + addLayer(layerParams, node_proto); +} - CV_Assert(node_proto.input_size() == 1); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - std::vector inputs(1, getBlob(node_proto, 0)), transposed; - runLayer(layerParams, inputs, transposed); - CV_Assert(transposed.size() == 1); - addConstant(layerParams.name, transposed[0]); - return; - } +void ONNXImporter::parseGemm(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() >= 2); + layerParams.type = "InnerProduct"; + Mat weights = getBlob(node_proto, 1); + int ind_num_out = 0; + if (layerParams.has("transB") && !layerParams.get("transB")) { + transpose(weights, weights); + ind_num_out = 1; + } + layerParams.blobs.push_back(weights); + + if (node_proto.input_size() == 3) { + Mat bias = getBlob(node_proto, 2); + layerParams.blobs.push_back(bias); + } + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inputBuf = getBlob(node_proto, 0); + + LayerParams constParams; + constParams.name = node_proto.input(0); + constParams.type = "Const"; + constParams.blobs.push_back(inputBuf); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + } + + layerParams.set("num_output", layerParams.blobs[0].size[ind_num_out]); + layerParams.set("bias_term", node_proto.input_size() == 3); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseMatMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 2); + layerParams.type = "InnerProduct"; + layerParams.set("bias_term", false); + CV_Assert(constBlobs.find(node_proto.input(0)) == constBlobs.end()); + int firstInpDims = outShapes[node_proto.input(0)].size(); + int secondInpDims; + + if (constBlobs.find(node_proto.input(1)) != constBlobs.end()) + { + Mat blob = getBlob(node_proto, 1); + secondInpDims = blob.dims; + layerParams.blobs.push_back(blob.t()); + layerParams.set("num_output", layerParams.blobs[0].size[0]); + } else { + secondInpDims = outShapes[node_proto.input(1)].size(); + } + layerParams.set("axis", firstInpDims - secondInpDims + 1); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + const std::string& layer_type = node_proto.op_type(); + CV_Assert(node_proto.input_size() == 2); + + bool isDiv = layer_type == "Div"; + int constId = -1; + bool haveVariables = false; + for (int i = 0; i < 2; ++i) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + constId = i; + else + haveVariables = true; + } + if (constId != -1 && haveVariables) + { + Mat blob = getBlob(node_proto, constId); + blob = blob.reshape(1, 1); + if (blob.total() == 1) { + float blob_value = blob.ptr()[0]; + float coeff = isDiv ? 1.0 / blob_value : blob_value; + layerParams.set("scale", coeff); + layerParams.type = "Power"; } - else if (layer_type == "Squeeze") - { - CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes")); - DictValue axes_dict = layerParams.get("axes"); - MatShape inpShape = outShapes[node_proto.input(0)]; + else { + if (isDiv) + divide(1.0, blob, blob); + layerParams.blobs.push_back(blob); + layerParams.type = "Scale"; + } + } + else if (!haveVariables) + { + Mat inp0 = getBlob(node_proto, 0); + Mat inp1 = getBlob(node_proto, 1); - std::vector maskedAxes(inpShape.size(), false); - for (int i = 0; i < axes_dict.size(); ++i) + if (inp0.size != inp1.size && (inp0.total() != 1 || inp1.total() != 1)) + CV_Error_(Error::StsNotImplemented, ("Different shapes case is not supported with constant inputs: %s", layer_type.c_str())); + + if (inp0.total() == 1 && inp1.total() == 1 && inp0.dims != inp1.dims) + { + if (inp0.dims < inp1.dims) { - int axis = axes_dict.getIntValue(i); - CV_CheckLE(axis, static_cast(inpShape.size()), "Squeeze axis"); - maskedAxes[axis] = inpShape[axis] == 1; + inp0 = inp0.reshape(1, inp1.dims, inp1.size); + inp0.dims = inp1.dims; } - MatShape outShape; - for (int i = 0; i < inpShape.size(); ++i) + else { - if (!maskedAxes[i]) - outShape.push_back(inpShape[i]); + inp1 = inp1.reshape(1, inp0.dims, inp0.size); + inp1.dims = inp0.dims; } - if (outShape.size() != inpShape.size()) + } + + Mat out; + if (inp0.total() != inp1.total()) + { + if (inp0.total() == 1) { - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); - if (hasDynamicShapes) - { - std::vector dynamicAxes; - std::vector inputIndices; - for (int index = 0; index < inpShape.size(); ++index) - { - if (!maskedAxes[index]) - inputIndices.push_back(index); - } - for (int index = 0; index < outShape.size(); ++index) - dynamicAxes.push_back(index); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } + float inp0_value = inp0.ptr()[0]; + float coeff = isDiv ? 1.0 / inp0_value : inp0_value; + multiply(inp1, coeff, out); } else - layerParams.type = "Identity"; - - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) { - Mat inp = getBlob(node_proto, 0); - Mat out = inp.reshape(1, outShape); - out.dims = outShape.size(); // to workaround dims == 1 - addConstant(layerParams.name, out); - return; + float inp1_value = inp1.ptr()[0]; + float coeff = isDiv ? 1.0 / inp1_value : inp1_value; + multiply(inp0, coeff, out); } + } - else if (layer_type == "Flatten") + else { - CV_CheckEQ(node_proto.input_size(), 1, ""); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat input = getBlob(node_proto, 0); - int axis = normalize_axis(layerParams.get("axis", 1), input.dims); - - std::vector out_size(&input.size[0], &input.size[0] + axis); - out_size.push_back(input.total(axis)); - Mat output = input.reshape(1, out_size); - addConstant(layerParams.name, output); - return; - } + out = isDiv ? inp0 / inp1 : inp0.mul(inp1); } - else if (layer_type == "Unsqueeze") + + if (inp0.dims == 1 && inp1.dims == 1) + out.dims = 1; // to workaround dims == 1 + addConstant(layerParams.name, out); + return; + } + else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) + { + layerParams.type = "Eltwise"; + layerParams.set("operation", isDiv ? "div" : "prod"); + } + else + { + // Scale layer allocate output with the first input shape + if (total(outShapes[node_proto.input(0)]) < total(outShapes[node_proto.input(1)])) { - CV_Assert(node_proto.input_size() == 1); - DictValue axes = layerParams.get("axes"); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - // Constant input. - Mat input = getBlob(node_proto, 0); + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(1)); + proto.add_input(node_proto.input(0)); + proto.add_output(layerParams.name); + node_proto = proto; + } - std::vector dims; - for (int j = 0; j < input.dims; j++) { - dims.push_back(input.size[j]); - } - CV_Assert(axes.getIntValue(axes.size()-1) <= dims.size()); - for (int j = 0; j < axes.size(); j++) { - dims.insert(dims.begin() + axes.getIntValue(j), 1); - } + if (isDiv) + { + LayerParams powerParams; + powerParams.name = layerParams.name + "/inv"; + powerParams.type = "Power"; + powerParams.set("power", -1); - Mat out = input.reshape(0, dims); - addConstant(layerParams.name, out); - return; - } + int id; + //Create Power layer + if (DNN_DIAGNOSTICS_RUN) + id = utilNet.addLayer(powerParams.name, powerParams.type, powerParams); + else + id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + //Connect to input + IterLayerId_t layerId = layer_id.find(node_proto.input(1)); + CV_Assert(layerId != layer_id.end()); + if (DNN_DIAGNOSTICS_RUN) + utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + else + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); + outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - // Variable input. - if (axes.size() != 1) - CV_Error(Error::StsNotImplemented, "Multidimensional unsqueeze"); + //Replace input to Power + node_proto.set_input(1, powerParams.name); + } + layerParams.type = "Scale"; + } + addLayer(layerParams, node_proto); +} - MatShape inpShape = outShapes[node_proto.input(0)]; - int axis = axes.getIntValue(0); - CV_Assert(0 <= axis && axis <= inpShape.size()); - std::vector outShape = inpShape; - outShape.insert(outShape.begin() + axis, 1); - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); - if (hasDynamicShapes) - { - std::vector dynamicAxes; - std::vector inputIndices; - for (int index = 0; index < outShape.size(); ++index) { - if (index != axis) - dynamicAxes.push_back(index); - } - for (int index = 0; index < inpShape.size(); ++index) - inputIndices.push_back(index); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } +void ONNXImporter::parseConv(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_Assert(node_proto.input_size() >= 2); + layerParams.type = "Convolution"; + for (int j = 1; j < node_proto.input_size(); j++) { + if (constBlobs.find(node_proto.input(j)) != constBlobs.end()) + { + layerParams.blobs.push_back(getBlob(node_proto, j)); } - else if (layer_type == "Expand") + } + int outCn = layerParams.blobs.empty() ? outShapes[node_proto.input(1)][0] : layerParams.blobs[0].size[0]; + layerParams.set("num_output", outCn); + + // Check for asymmetric padding in Conv2D + if (layerParams.has("pad")) + { + bool asymmetricPadding = false; + DictValue pads = layerParams.get("pad"); + const int dims = pads.size() / 2; + for (int i = 0; i < dims; ++i) { - CV_CheckEQ(node_proto.input_size(), 2, ""); - const std::string& input0 = node_proto.input(0); - const std::string& input1 = node_proto.input(1); - Mat newShapeMat = getBlob(input1); - MatShape targetShape(newShapeMat.ptr(), newShapeMat.ptr() + newShapeMat.total()); - - MatShape inpShape; - bool haveVariables = constBlobs.find(input0) == constBlobs.end(); - if (haveVariables) + if (pads.get(i) != pads.get(i + dims)) { - IterShape_t shapeIt = outShapes.find(input0); - CV_Assert(shapeIt != outShapes.end()); - inpShape = shapeIt->second; + asymmetricPadding = true; + break; } - else + } + if (asymmetricPadding && pads.size() == 4) // [pad_t, pad_l, pad_b, pad_r] { - inpShape = shape(getBlob(input0)); - } - - String srcName = input0; - // Unsqueeze and repeat along new axis - if (targetShape.size() == inpShape.size() + 1) + layerParams.erase("pad"); + // No paddings required for N, C axis + std::vector paddings(4, 0); + // Add paddings for H, W axis + for (int i = 0; i < dims; ++i) { - for (int i = 0; i < targetShape.size(); i++) - { - if (targetShape[i] == -1 && i < inpShape.size()) - targetShape[i] = inpShape[i]; - else if (i < inpShape.size() && targetShape[i] != inpShape[i]) - inpShape.insert(inpShape.begin() + i, 1); - } - if (haveVariables) - { - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); - srcName = reshapeLp.name; - } + paddings.push_back(pads.get(i)); + paddings.push_back(pads.get(dims + i)); } - CV_CheckEQ(inpShape.size(), targetShape.size(), "Unsupported Expand op with different dims"); + LayerParams padLp; + padLp.name = layerParams.name + "/pad"; + padLp.type = "Padding"; + padLp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size())); - std::vector broadcast_axes; - for (int i = 0; i < targetShape.size(); i++) - { - if (targetShape[i] != inpShape[i]) - { - if (inpShape[i] == 1) - broadcast_axes.push_back(i); - else - CV_Error(Error::StsError, format("Could not be broadcast by axis: %d", i)); - } - } + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(padLp.name); - if (!haveVariables) - { - if (broadcast_axes.size() != 1) - CV_Error(Error::StsNotImplemented, "Expand op doesn't support multiple axes for constant input"); - - Mat input = getBlob(node_proto, 0); - input = input.reshape(0, total(inpShape, 0, broadcast_axes[0])); - Mat output = cv::repeat(input, 1, targetShape[broadcast_axes[0]]); - output = output.reshape(0, targetShape); - addConstant(layerParams.name, output); - return; + addLayer(padLp, proto); + node_proto.set_input(0, padLp.name); } + } + addLayer(layerParams, node_proto); +} - if (broadcast_axes.size() == 2 && - broadcast_axes[0] == broadcast_axes[1] - 1 && broadcast_axes[1] == inpShape.size() - 1) - { - LayerParams constParams; - constParams.name = layerParams.name + "/const"; - CV_Assert(layer_id.find(constParams.name) == layer_id.end()); - constParams.type = "Const"; +void ONNXImporter::parseConvTranspose(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() >= 2); + layerParams.type = "Deconvolution"; + for (int j = 1; j < node_proto.input_size(); j++) { + layerParams.blobs.push_back(getBlob(node_proto, j)); + } + layerParams.set("num_output", layerParams.blobs[0].size[1] * layerParams.get("group", 1)); + layerParams.set("bias_term", node_proto.input_size() == 3); - Mat inp = Mat::ones(newShapeMat.total(), newShapeMat.ptr(), CV_32F); - constParams.blobs.push_back(inp); + if (!layerParams.has("kernel_size")) + CV_Error(Error::StsNotImplemented, + "Required attribute 'kernel_size' is not present."); - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); + if (layerParams.has("output_shape")) + { + const DictValue& outShape = layerParams.get("output_shape"); + DictValue strides = layerParams.get("stride"); + DictValue kernel = layerParams.get("kernel_size"); - layerParams.type = "Scale"; - layerParams.set("bias_term", false); - node_proto.set_input(0, constParams.name); - node_proto.set_input(1, srcName); - } - else if (broadcast_axes.size() == 1 && broadcast_axes[0] <= 1) + String padMode; + std::vector adjust_pads; + if (layerParams.has("pad_mode")) + { + padMode = toUpperCase(layerParams.get("pad_mode")); + if (padMode != "SAME" && padMode != "VALID") + CV_Error(Error::StsError, "Unsupported padding mode " + padMode); + + for (int i = 0; i < strides.size(); i++) { - String base_name = layerParams.name + "/copy_"; - std::vector input_names; - for (int j = 0; j < targetShape[broadcast_axes[0]]; j++) - { - std::ostringstream ss; - ss << j; - LayerParams copyLP; - copyLP.name = base_name + ss.str(); - copyLP.type = "Identity"; - CV_Assert(layer_id.find(copyLP.name) == layer_id.end()); - input_names.push_back(copyLP.name); - - node_proto.set_input(0, srcName); - node_proto.set_output(0, copyLP.name); - addLayer(copyLP, node_proto); - } - node_proto.clear_input(); - for (int i = 0; i < input_names.size(); i++) - { - node_proto.add_input(input_names[i]); - } - layerParams.set("axis", broadcast_axes[0]); - layerParams.type = "Concat"; - node_proto.set_output(0, layerParams.name); + int sz = outShape.get(2 + i); + int stride = strides.get(i); + adjust_pads.push_back(padMode == "SAME"? (sz - 1) % stride : + (sz - kernel.get(i)) % stride); } - else - CV_Error(Error::StsNotImplemented, "Unsupported Expand op"); + layerParams.set("adj", DictValue::arrayInt(&adjust_pads[0], adjust_pads.size())); } - else if (layer_type == "Reshape") - { - CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape")); + } + else if (layerParams.has("output_padding")) + { + replaceLayerParam(layerParams, "output_padding", "adj"); + } + addLayer(layerParams, node_proto); +} - if (node_proto.input_size() == 2) { - Mat blob = getBlob(node_proto, 1); - CV_Assert(blob.type() == CV_32SC1); +void ONNXImporter::parseTranspose(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Permute"; + replaceLayerParam(layerParams, "perm", "order"); - layerParams.set("dim", DictValue::arrayInt( - blob.ptr(), blob.total() )); + CV_Assert(node_proto.input_size() == 1); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + std::vector inputs(1, getBlob(node_proto, 0)), transposed; + runLayer(layerParams, inputs, transposed); + CV_Assert(transposed.size() == 1); + addConstant(layerParams.name, transposed[0]); + return; + } + addLayer(layerParams, node_proto); +} - if (layer_id.find(node_proto.input(0)) == layer_id.end()) { - std::vector inputs(1, getBlob(node_proto, 0)), outputs; - runLayer(layerParams, inputs, outputs); - addConstant(layerParams.name, outputs[0]); - return; - } - } - else { - DictValue shape = layerParams.get("shape"); - std::vector dim; - for (int j = 0; j < shape.size(); j++) { - dim.push_back(shape.getIntValue(j)); - } +void ONNXImporter::parseSqueeze(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes")); + DictValue axes_dict = layerParams.get("axes"); + MatShape inpShape = outShapes[node_proto.input(0)]; - if (layer_id.find(node_proto.input(0)) == layer_id.end()) { - Mat input = getBlob(node_proto, 0); - Mat out = input.reshape(0, dim); - addConstant(layerParams.name, out); - return; - } - replaceLayerParam(layerParams, "shape", "dim"); - } - } - else if (layer_type == "Pad") + std::vector maskedAxes(inpShape.size(), false); + for (int i = 0; i < axes_dict.size(); ++i) + { + int axis = axes_dict.getIntValue(i); + CV_CheckLE(axis, static_cast(inpShape.size()), "Squeeze axis"); + maskedAxes[axis] = inpShape[axis] == 1; + } + MatShape outShape; + for (int i = 0; i < inpShape.size(); ++i) + { + if (!maskedAxes[i]) + outShape.push_back(inpShape[i]); + } + if (outShape.size() != inpShape.size()) + { + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + if (hasDynamicShapes) { - layerParams.type = "Padding"; - replaceLayerParam(layerParams, "mode", "type"); - if (node_proto.input_size() == 3 || node_proto.input_size() == 2) + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < inpShape.size(); ++index) { - // Paddings are in order begin0, begin1, .. beginN, end0, end1, ..., endN. - // We need to shuffle it to begin0, end0, begin1, end1, ... - Mat paddings = getBlob(node_proto, 1).reshape(1, 2); - paddings = paddings.t(); - layerParams.set("paddings", DictValue::arrayInt(paddings.ptr(), paddings.total())); - - if (node_proto.input_size() == 3) - { - Mat value = getBlob(node_proto, 2); - layerParams.set("value", value.ptr()[0]); - } + if (!maskedAxes[index]) + inputIndices.push_back(index); } + for (int index = 0; index < outShape.size(); ++index) + dynamicAxes.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); } - else if (layer_type == "Shape") - { - CV_Assert(node_proto.input_size() == 1); - IterShape_t shapeIt = outShapes.find(node_proto.input(0)); - CV_Assert(shapeIt != outShapes.end()); - const MatShape& inpShape = shapeIt->second; + } + else + layerParams.type = "Identity"; - Mat shapeMat(inpShape.size(), 1, CV_32S); - for (int j = 0; j < inpShape.size(); ++j) - shapeMat.at(j) = inpShape[j]; - shapeMat.dims = 1; + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inp = getBlob(node_proto, 0); + Mat out = inp.reshape(1, outShape); + out.dims = outShape.size(); // to workaround dims == 1 + addConstant(layerParams.name, out); + return; + } + addLayer(layerParams, node_proto); +} - addConstant(layerParams.name, shapeMat); - return; +void ONNXImporter::parseFlatten(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_CheckEQ(node_proto.input_size(), 1, ""); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat input = getBlob(node_proto, 0); + int axis = normalize_axis(layerParams.get("axis", 1), input.dims); + + std::vector out_size(&input.size[0], &input.size[0] + axis); + out_size.push_back(input.total(axis)); + Mat output = input.reshape(1, out_size); + addConstant(layerParams.name, output); + return; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseUnsqueeze(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 1); + DictValue axes = layerParams.get("axes"); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + // Constant input. + Mat input = getBlob(node_proto, 0); + + std::vector dims; + for (int j = 0; j < input.dims; j++) { + dims.push_back(input.size[j]); + } + CV_Assert(axes.getIntValue(axes.size()-1) <= dims.size()); + for (int j = 0; j < axes.size(); j++) { + dims.insert(dims.begin() + axes.getIntValue(j), 1); + } + + Mat out = input.reshape(0, dims); + addConstant(layerParams.name, out); + return; + } + + // Variable input. + if (axes.size() != 1) + CV_Error(Error::StsNotImplemented, "Multidimensional unsqueeze"); + + MatShape inpShape = outShapes[node_proto.input(0)]; + int axis = axes.getIntValue(0); + CV_Assert(0 <= axis && axis <= inpShape.size()); + std::vector outShape = inpShape; + outShape.insert(outShape.begin() + axis, 1); + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + if (hasDynamicShapes) + { + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < outShape.size(); ++index) { + if (index != axis) + dynamicAxes.push_back(index); + } + for (int index = 0; index < inpShape.size(); ++index) + inputIndices.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseExpand(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_CheckEQ(node_proto.input_size(), 2, ""); + const std::string& input0 = node_proto.input(0); + const std::string& input1 = node_proto.input(1); + Mat newShapeMat = getBlob(input1); + MatShape targetShape(newShapeMat.ptr(), newShapeMat.ptr() + newShapeMat.total()); + + MatShape inpShape; + bool haveVariables = constBlobs.find(input0) == constBlobs.end(); + if (haveVariables) + { + IterShape_t shapeIt = outShapes.find(input0); + CV_Assert(shapeIt != outShapes.end()); + inpShape = shapeIt->second; + } + else + { + inpShape = shape(getBlob(input0)); + } + + String srcName = input0; + // Unsqueeze and repeat along new axis + if (targetShape.size() == inpShape.size() + 1) + { + for (int i = 0; i < targetShape.size(); i++) + { + if (targetShape[i] == -1 && i < inpShape.size()) + targetShape[i] = inpShape[i]; + else if (i < inpShape.size() && targetShape[i] != inpShape[i]) + inpShape.insert(inpShape.begin() + i, 1); } - else if (layer_type == "Cast") + if (haveVariables) { - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat blob = getBlob(node_proto, 0); - int type; - switch (layerParams.get("to")) - { - case opencv_onnx::TensorProto_DataType_FLOAT: type = CV_32F; break; - case opencv_onnx::TensorProto_DataType_UINT8: type = CV_8U; break; - case opencv_onnx::TensorProto_DataType_UINT16: type = CV_16U; break; - case opencv_onnx::TensorProto_DataType_FLOAT16: type = CV_16S; break; - case opencv_onnx::TensorProto_DataType_INT8: - case opencv_onnx::TensorProto_DataType_INT16: - case opencv_onnx::TensorProto_DataType_INT32: - case opencv_onnx::TensorProto_DataType_INT64: type = CV_32S; break; - default: type = blob.type(); - } - Mat dst; - blob.convertTo(dst, type); - dst.dims = blob.dims; - addConstant(layerParams.name, dst); - return; - } - else - layerParams.type = "Identity"; + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + reshapeLp.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + srcName = reshapeLp.name; } - else if (layer_type == "ConstantOfShape" || layer_type == "ConstantFill") + } + CV_CheckEQ(inpShape.size(), targetShape.size(), "Unsupported Expand op with different dims"); + + std::vector broadcast_axes; + for (int i = 0; i < targetShape.size(); i++) + { + if (targetShape[i] != inpShape[i]) { - int depth = CV_32F; - float fill_value; - if (!layerParams.blobs.empty()) - { - CV_Assert(!layerParams.has("value")); - depth = layerParams.blobs[0].depth(); - Mat floats; - layerParams.blobs[0].convertTo(floats, CV_32F); - fill_value = floats.at(0, 0); - } + if (inpShape[i] == 1) + broadcast_axes.push_back(i); else - fill_value = layerParams.get("value", 0); + CV_Error(Error::StsError, format("Could not be broadcast by axis: %d", i)); + } + } - MatShape inpShape = getBlob(node_proto, 0); - for (int i = 0; i < inpShape.size(); i++) - CV_CheckGT(inpShape[i], 0, ""); - Mat tensor(inpShape.size(), &inpShape[0], depth, Scalar(fill_value)); - addConstant(layerParams.name, tensor); - return; + if (!haveVariables) + { + if (broadcast_axes.size() != 1) + CV_Error(Error::StsNotImplemented, "Expand op doesn't support multiple axes for constant input"); + + Mat input = getBlob(node_proto, 0); + input = input.reshape(0, total(inpShape, 0, broadcast_axes[0])); + Mat output = cv::repeat(input, 1, targetShape[broadcast_axes[0]]); + output = output.reshape(0, targetShape); + addConstant(layerParams.name, output); + return; + } + + if (broadcast_axes.size() == 2 && + broadcast_axes[0] == broadcast_axes[1] - 1 && broadcast_axes[1] == inpShape.size() - 1) + { + LayerParams constParams; + constParams.name = layerParams.name + "/const"; + CV_Assert(layer_id.find(constParams.name) == layer_id.end()); + constParams.type = "Const"; + + Mat inp = Mat::ones(newShapeMat.total(), newShapeMat.ptr(), CV_32F); + constParams.blobs.push_back(inp); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + + layerParams.type = "Scale"; + layerParams.set("bias_term", false); + node_proto.set_input(0, constParams.name); + node_proto.set_input(1, srcName); + } + else if (broadcast_axes.size() == 1 && broadcast_axes[0] <= 1) + { + String base_name = layerParams.name + "/copy_"; + std::vector input_names; + for (int j = 0; j < targetShape[broadcast_axes[0]]; j++) + { + std::ostringstream ss; + ss << j; + LayerParams copyLP; + copyLP.name = base_name + ss.str(); + copyLP.type = "Identity"; + CV_Assert(layer_id.find(copyLP.name) == layer_id.end()); + input_names.push_back(copyLP.name); + + node_proto.set_input(0, srcName); + node_proto.set_output(0, copyLP.name); + addLayer(copyLP, node_proto); } - else if (layer_type == "Gather") + node_proto.clear_input(); + for (int i = 0; i < input_names.size(); i++) { - CV_Assert(node_proto.input_size() == 2); - Mat indexMat = getBlob(node_proto, 1); - CV_Assert_N(indexMat.type() == CV_32S, indexMat.total() == 1); - int index = indexMat.at(0); - int axis = layerParams.get("axis", 0); + node_proto.add_input(input_names[i]); + } + layerParams.set("axis", broadcast_axes[0]); + layerParams.type = "Concat"; + node_proto.set_output(0, layerParams.name); + } + else + CV_Error(Error::StsNotImplemented, "Unsupported Expand op"); + addLayer(layerParams, node_proto); +} - if ((constBlobs.find(node_proto.input(0)) != constBlobs.end())) - { - Mat input = getBlob(node_proto, 0); - Mat out; - std::vector ranges(input.dims, Range::all()); - ranges[axis] = Range(index, index + 1); - - out = input(ranges); - MatShape outShape = shape(out); - if (outShape.size() > 1) - { - outShape.erase(outShape.begin() + axis); - out.reshape(0, outShape); - } else { - out.dims = 1; - } - addConstant(layerParams.name, out); - return; - } - else - { - IterShape_t shapeIt = outShapes.find(node_proto.input(0)); - CV_Assert(shapeIt != outShapes.end()); - MatShape inpShape = shapeIt->second; - - LayerParams sliceLp; - sliceLp.type = "Slice"; - sliceLp.name = inpShape.size() > 1 ? layerParams.name + "/slice" : layerParams.name; - std::vector begin(inpShape.size(), 0); - std::vector end(inpShape.size(), -1); - begin[axis] = index; - end[axis] = index + 1; - - cv::dnn::DictValue paramBegin = cv::dnn::DictValue::arrayInt(begin.data(), begin.size()); - cv::dnn::DictValue paramEnd = cv::dnn::DictValue::arrayInt(end.data(), end.size()); - sliceLp.set("begin", paramBegin); - sliceLp.set("end", paramEnd); - sliceLp.set("has_dynamic_shapes", hasDynamicShapes); - - if (inpShape.size() > 1) - { - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(sliceLp.name); - addLayer(sliceLp, proto); - - inpShape.erase(inpShape.begin() + axis); - layerParams.type = "Reshape"; - layerParams.set("axis", 0); - layerParams.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); - if (hasDynamicShapes) - { - std::vector dynamicAxes; - std::vector inputIndices; - for (int index = 0; index < inpShape.size(); ++index) - dynamicAxes.push_back(index); - for (int index = 0; index < inpShape.size(); ++index) - inputIndices.push_back(index); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } - node_proto.set_input(0, sliceLp.name); - } - else - { - layerParams = sliceLp; - } - } +void ONNXImporter::parseReshape(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape")); + + if (node_proto.input_size() == 2) { + Mat blob = getBlob(node_proto, 1); + CV_Assert(blob.type() == CV_32SC1); + + layerParams.set("dim", DictValue::arrayInt( + blob.ptr(), blob.total() )); + + if (layer_id.find(node_proto.input(0)) == layer_id.end()) { + std::vector inputs(1, getBlob(node_proto, 0)), outputs; + runLayer(layerParams, inputs, outputs); + addConstant(layerParams.name, outputs[0]); + return; + } + } + else { + DictValue shape = layerParams.get("shape"); + std::vector dim; + for (int j = 0; j < shape.size(); j++) { + dim.push_back(shape.getIntValue(j)); } - else if (layer_type == "Concat") - { - bool hasVariableInps = false; - for (int i = 0; i < node_proto.input_size(); ++i) - { - if (layer_id.find(node_proto.input(i)) != layer_id.end()) - { - hasVariableInps = true; - break; - } - } - if (!hasVariableInps) - { - std::vector inputs(node_proto.input_size()), concatenated; - // Due constant folding we can get inputs with different number of dimensions - // Insert the missing dimension to inputs - MatShape inputShape; - for (size_t i = 0; i < inputs.size(); ++i) - { - inputs[i] = getBlob(node_proto, i); - if (inputs[i].size.dims() > inputShape.size()) - { - inputShape = shape(inputs[i]); - } - } + if (layer_id.find(node_proto.input(0)) == layer_id.end()) { + Mat input = getBlob(node_proto, 0); + Mat out = input.reshape(0, dim); + addConstant(layerParams.name, out); + return; + } + replaceLayerParam(layerParams, "shape", "dim"); + } + addLayer(layerParams, node_proto); +} - // Concat-1 has default value for axis is 1: https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-1 - int axis = layerParams.get("axis", 1); - for (size_t i = 0; i < inputs.size(); ++i) - { - MatShape targetShape = inputShape; - targetShape[axis] = shape(inputs[i])[axis]; - CV_CheckEQ(total(targetShape), total(shape(inputs[i])), ""); - inputs[i] = inputs[i].reshape(0, targetShape); - } - runLayer(layerParams, inputs, concatenated); +void ONNXImporter::parsePad(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Padding"; + replaceLayerParam(layerParams, "mode", "type"); + if (node_proto.input_size() == 3 || node_proto.input_size() == 2) + { + // Paddings are in order begin0, begin1, .. beginN, end0, end1, ..., endN. + // We need to shuffle it to begin0, end0, begin1, end1, ... + Mat paddings = getBlob(node_proto, 1).reshape(1, 2); + paddings = paddings.t(); + layerParams.set("paddings", DictValue::arrayInt(paddings.ptr(), paddings.total())); - CV_Assert(concatenated.size() == 1); - addConstant(layerParams.name, concatenated[0]); - return; - } - else - { - for (int i = 0; i < node_proto.input_size(); ++i) - { - if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) - { - LayerParams constParams; - constParams.name = node_proto.input(i); - constParams.type = "Const"; - constParams.blobs.push_back(getBlob(node_proto, i)); - - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); - } - } - } + if (node_proto.input_size() == 3) + { + Mat value = getBlob(node_proto, 2); + layerParams.set("value", value.ptr()[0]); } - else if (layer_type == "Resize") + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseShape(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 1); + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); + CV_Assert(shapeIt != outShapes.end()); + const MatShape& inpShape = shapeIt->second; + + Mat shapeMat(inpShape.size(), 1, CV_32S); + for (int j = 0; j < inpShape.size(); ++j) + shapeMat.at(j) = inpShape[j]; + shapeMat.dims = 1; + + addConstant(layerParams.name, shapeMat); +} + +void ONNXImporter::parseCast(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat blob = getBlob(node_proto, 0); + int type; + switch (layerParams.get("to")) { - for (int i = 1; i < node_proto.input_size(); i++) - CV_Assert(layer_id.find(node_proto.input(i)) == layer_id.end()); + case opencv_onnx::TensorProto_DataType_FLOAT: type = CV_32F; break; + case opencv_onnx::TensorProto_DataType_UINT8: type = CV_8U; break; + case opencv_onnx::TensorProto_DataType_UINT16: type = CV_16U; break; + case opencv_onnx::TensorProto_DataType_FLOAT16: type = CV_16S; break; + case opencv_onnx::TensorProto_DataType_INT8: + case opencv_onnx::TensorProto_DataType_INT16: + case opencv_onnx::TensorProto_DataType_INT32: + case opencv_onnx::TensorProto_DataType_INT64: type = CV_32S; break; + default: type = blob.type(); + } + Mat dst; + blob.convertTo(dst, type); + dst.dims = blob.dims; + addConstant(layerParams.name, dst); + return; + } + else + layerParams.type = "Identity"; + addLayer(layerParams, node_proto); +} - if (layerParams.has("coordinate_transformation_mode")) - { - String interp_mode = layerParams.get("coordinate_transformation_mode"); - CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); +void ONNXImporter::parseConstantFill(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int depth = CV_32F; + float fill_value; + if (!layerParams.blobs.empty()) + { + CV_Assert(!layerParams.has("value")); + depth = layerParams.blobs[0].depth(); + Mat floats; + layerParams.blobs[0].convertTo(floats, CV_32F); + fill_value = floats.at(0, 0); + } + else + fill_value = layerParams.get("value", 0); - layerParams.set("align_corners", interp_mode == "align_corners"); - if (layerParams.get("mode") == "linear") - { - layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? - "opencv_linear" : "bilinear"); - } - } - if (layerParams.get("mode") == "linear" && framework_name == "pytorch") - layerParams.set("mode", "opencv_linear"); + MatShape inpShape = getBlob(node_proto, 0); + for (int i = 0; i < inpShape.size(); i++) + CV_CheckGT(inpShape[i], 0, ""); + Mat tensor(inpShape.size(), &inpShape[0], depth, Scalar(fill_value)); + addConstant(layerParams.name, tensor); +} - // input = [X, scales], [X, roi, scales] or [x, roi, scales, sizes] - int foundScaleId = hasDynamicShapes ? node_proto.input_size() - 1 - : node_proto.input_size() > 2 ? 2 : 1; +void ONNXImporter::parseGather(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_Assert(node_proto.input_size() == 2); + Mat indexMat = getBlob(node_proto, 1); + CV_Assert_N(indexMat.type() == CV_32S, indexMat.total() == 1); + int index = indexMat.at(0); + int axis = layerParams.get("axis", 0); + + if ((constBlobs.find(node_proto.input(0)) != constBlobs.end())) + { + Mat input = getBlob(node_proto, 0); + Mat out; + std::vector ranges(input.dims, Range::all()); + ranges[axis] = Range(index, index + 1); + + out = input(ranges); + MatShape outShape = shape(out); + if (outShape.size() > 1) + { + outShape.erase(outShape.begin() + axis); + out.reshape(0, outShape); + } else { + out.dims = 1; + } + addConstant(layerParams.name, out); + return; + } + else + { + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); + CV_Assert(shapeIt != outShapes.end()); + MatShape inpShape = shapeIt->second; + + LayerParams sliceLp; + sliceLp.type = "Slice"; + sliceLp.name = inpShape.size() > 1 ? layerParams.name + "/slice" : layerParams.name; + std::vector begin(inpShape.size(), 0); + std::vector end(inpShape.size(), -1); + begin[axis] = index; + end[axis] = index + 1; + + cv::dnn::DictValue paramBegin = cv::dnn::DictValue::arrayInt(begin.data(), begin.size()); + cv::dnn::DictValue paramEnd = cv::dnn::DictValue::arrayInt(end.data(), end.size()); + sliceLp.set("begin", paramBegin); + sliceLp.set("end", paramEnd); + sliceLp.set("has_dynamic_shapes", hasDynamicShapes); + + if (inpShape.size() > 1) + { + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(sliceLp.name); + addLayer(sliceLp, proto); - Mat scales = getBlob(node_proto, foundScaleId); - if (scales.total() == 4) - { - layerParams.set("zoom_factor_y", scales.at(2)); - layerParams.set("zoom_factor_x", scales.at(3)); - } - else + inpShape.erase(inpShape.begin() + axis); + layerParams.type = "Reshape"; + layerParams.set("axis", 0); + layerParams.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); + if (hasDynamicShapes) { - const std::string& inputLast = node_proto.input(node_proto.input_size() - 1); - if (constBlobs.find(inputLast) != constBlobs.end()) - { - Mat shapes = getBlob(inputLast); - CV_CheckEQ(shapes.size[0], 4, ""); - CV_CheckEQ(shapes.size[1], 1, ""); - CV_CheckDepth(shapes.depth(), shapes.depth() == CV_32S || shapes.depth() == CV_32F, ""); - if (shapes.depth() == CV_32F) - shapes.convertTo(shapes, CV_32S); - layerParams.set("width", shapes.at(3)); - layerParams.set("height", shapes.at(2)); - } + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < inpShape.size(); ++index) + dynamicAxes.push_back(index); + for (int index = 0; index < inpShape.size(); ++index) + inputIndices.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); } - replaceLayerParam(layerParams, "mode", "interpolation"); + node_proto.set_input(0, sliceLp.name); } - else if (layer_type == "Upsample") + else { - //fused from Resize Subgraph - if (layerParams.has("coordinate_transformation_mode")) - { - String interp_mode = layerParams.get("coordinate_transformation_mode"); - CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); + layerParams = sliceLp; + } + } + addLayer(layerParams, node_proto); +} - layerParams.set("align_corners", interp_mode == "align_corners"); - if (layerParams.get("mode") == "linear") - { - layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? - "opencv_linear" : "bilinear"); - } - } - if (layerParams.get("mode") == "linear" && framework_name == "pytorch") - layerParams.set("mode", "opencv_linear"); +void ONNXImporter::parseConcat(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + bool hasVariableInps = false; + for (int i = 0; i < node_proto.input_size(); ++i) + { + if (layer_id.find(node_proto.input(i)) != layer_id.end()) + { + hasVariableInps = true; + break; + } + } - layerParams.type = "Resize"; - if (layerParams.has("scales")) - { - // Pytorch layer - DictValue scales = layerParams.get("scales"); - CV_Assert(scales.size() == 4); - layerParams.set("zoom_factor_y", scales.getIntValue(2)); - layerParams.set("zoom_factor_x", scales.getIntValue(3)); - } - else if (layerParams.has("height_scale") && layerParams.has("width_scale")) + if (!hasVariableInps) + { + std::vector inputs(node_proto.input_size()), concatenated; + // Due constant folding we can get inputs with different number of dimensions + // Insert the missing dimension to inputs + MatShape inputShape; + for (size_t i = 0; i < inputs.size(); ++i) + { + inputs[i] = getBlob(node_proto, i); + if (inputs[i].size.dims() > inputShape.size()) { - // Caffe2 layer - replaceLayerParam(layerParams, "height_scale", "zoom_factor_y"); - replaceLayerParam(layerParams, "width_scale", "zoom_factor_x"); + inputShape = shape(inputs[i]); } - else - { - // scales as input - const std::string& input1 = node_proto.input(1); - if (constBlobs.find(input1) != constBlobs.end()) - { - Mat scales = getBlob(input1); - CV_Assert(scales.total() == 4); - layerParams.set("zoom_factor_y", scales.at(2)); - layerParams.set("zoom_factor_x", scales.at(3)); - } - } - replaceLayerParam(layerParams, "mode", "interpolation"); } - else if (layer_type == "SoftMax" || layer_type == "LogSoftmax") + + // Concat-1 has default value for axis is 1: https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-1 + int axis = layerParams.get("axis", 1); + for (size_t i = 0; i < inputs.size(); ++i) { - layerParams.type = "Softmax"; - layerParams.set("log_softmax", layer_type == "LogSoftmax"); + MatShape targetShape = inputShape; + targetShape[axis] = shape(inputs[i])[axis]; + CV_CheckEQ(total(targetShape), total(shape(inputs[i])), ""); + inputs[i] = inputs[i].reshape(0, targetShape); } - else if (layer_type == "DetectionOutput") + runLayer(layerParams, inputs, concatenated); + + CV_Assert(concatenated.size() == 1); + addConstant(layerParams.name, concatenated[0]); + return; + } + else + { + for (int i = 0; i < node_proto.input_size(); ++i) { - CV_CheckEQ(node_proto.input_size(), 3, ""); - if (constBlobs.find(node_proto.input(2)) != constBlobs.end()) + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) { - Mat priors = getBlob(node_proto, 2); - LayerParams constParams; - constParams.name = layerParams.name + "/priors"; + constParams.name = node_proto.input(i); constParams.type = "Const"; - constParams.blobs.push_back(priors); + constParams.blobs.push_back(getBlob(node_proto, i)); - opencv_onnx::NodeProto priorsProto; - priorsProto.add_output(constParams.name); - addLayer(constParams, priorsProto); - - node_proto.set_input(2, constParams.name); - } - } - else - { - for (int j = 0; j < node_proto.input_size(); j++) { - if (layer_id.find(node_proto.input(j)) == layer_id.end()) - layerParams.blobs.push_back(getBlob(node_proto, j)); + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); } } - addLayer(layerParams, node_proto); } - catch (const cv::Exception& e) + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseResize(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + for (int i = 1; i < node_proto.input_size(); i++) + CV_Assert(layer_id.find(node_proto.input(i)) == layer_id.end()); + + if (layerParams.has("coordinate_transformation_mode")) { - if (DNN_DIAGNOSTICS_RUN) - { - CV_LOG_ERROR(NULL, "DNN/ONNX: Potential problem during processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " - << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) << "\n" << e.msg - ); - auto registeredLayers = getLayerFactoryImpl(); - if (registeredLayers.find(layerParams.type) != registeredLayers.end()) - { - try - { - Ptr layer = LayerFactory::createLayerInstance(layerParams.type, layerParams); - } - catch (const std::exception& e) - { - CV_LOG_ERROR(NULL, "DNN/ONNX: Layer of type " << layerParams.type << "(" << layer_type << ") cannot be created with parameters " << layerParams << ". Error: " << e.what() - ); - } - } - } - else + String interp_mode = layerParams.get("coordinate_transformation_mode"); + CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); + + layerParams.set("align_corners", interp_mode == "align_corners"); + if (layerParams.get("mode") == "linear") { - CV_LOG_ERROR(NULL, "DNN/ONNX: ERROR during processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " - << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) - ); + layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? + "opencv_linear" : "bilinear"); } - for (int i = 0; i < node_proto.input_size(); i++) + } + if (layerParams.get("mode") == "linear" && framework_name == "pytorch") + layerParams.set("mode", "opencv_linear"); + + // input = [X, scales], [X, roi, scales] or [x, roi, scales, sizes] + int foundScaleId = hasDynamicShapes ? node_proto.input_size() - 1 + : node_proto.input_size() > 2 ? 2 : 1; + + Mat scales = getBlob(node_proto, foundScaleId); + if (scales.total() == 4) + { + layerParams.set("zoom_factor_y", scales.at(2)); + layerParams.set("zoom_factor_x", scales.at(3)); + } + else + { + const std::string& inputLast = node_proto.input(node_proto.input_size() - 1); + if (constBlobs.find(inputLast) != constBlobs.end()) { - CV_LOG_INFO(NULL, " Input[" << i << "] = '" << node_proto.input(i) << "'"); + Mat shapes = getBlob(inputLast); + CV_CheckEQ(shapes.size[0], 4, ""); + CV_CheckEQ(shapes.size[1], 1, ""); + CV_CheckDepth(shapes.depth(), shapes.depth() == CV_32S || shapes.depth() == CV_32F, ""); + if (shapes.depth() == CV_32F) + shapes.convertTo(shapes, CV_32S); + layerParams.set("width", shapes.at(3)); + layerParams.set("height", shapes.at(2)); } - for (int i = 0; i < node_proto.output_size(); i++) + } + replaceLayerParam(layerParams, "mode", "interpolation"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseUpsample(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + //fused from Resize Subgraph + if (layerParams.has("coordinate_transformation_mode")) + { + String interp_mode = layerParams.get("coordinate_transformation_mode"); + CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); + + layerParams.set("align_corners", interp_mode == "align_corners"); + if (layerParams.get("mode") == "linear") { - CV_LOG_INFO(NULL, " Output[" << i << "] = '" << node_proto.output(i) << "'"); + layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? + "opencv_linear" : "bilinear"); } - if (DNN_DIAGNOSTICS_RUN) + } + if (layerParams.get("mode") == "linear" && framework_name == "pytorch") + layerParams.set("mode", "opencv_linear"); + + layerParams.type = "Resize"; + if (layerParams.has("scales")) + { + // Pytorch layer + DictValue scales = layerParams.get("scales"); + CV_Assert(scales.size() == 4); + layerParams.set("zoom_factor_y", scales.getIntValue(2)); + layerParams.set("zoom_factor_x", scales.getIntValue(3)); + } + else if (layerParams.has("height_scale") && layerParams.has("width_scale")) + { + // Caffe2 layer + replaceLayerParam(layerParams, "height_scale", "zoom_factor_y"); + replaceLayerParam(layerParams, "width_scale", "zoom_factor_x"); + } + else + { + // scales as input + const std::string& input1 = node_proto.input(1); + if (constBlobs.find(input1) != constBlobs.end()) { - for (int i = 0; i < node_proto.output_size(); ++i) - { - layer_id.insert(std::make_pair(node_proto.output(i), LayerInfo(0, i))); - outShapes[node_proto.output(i)] = outShapes[node_proto.input(0)]; - } + Mat scales = getBlob(input1); + CV_Assert(scales.total() == 4); + layerParams.set("zoom_factor_y", scales.at(2)); + layerParams.set("zoom_factor_x", scales.at(3)); } - else - CV_Error(Error::StsError, cv::format("Node [%s]:(%s) parse error: %s", layer_type.c_str(), name.c_str(), e.what())); } + replaceLayerParam(layerParams, "mode", "interpolation"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseSoftMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + const std::string& layer_type = node_proto.op_type(); + layerParams.type = "Softmax"; + layerParams.set("log_softmax", layer_type == "LogSoftmax"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseDetectionOutput(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_CheckEQ(node_proto.input_size(), 3, ""); + if (constBlobs.find(node_proto.input(2)) != constBlobs.end()) + { + Mat priors = getBlob(node_proto, 2); + + LayerParams constParams; + constParams.name = layerParams.name + "/priors"; + constParams.type = "Const"; + constParams.blobs.push_back(priors); + + opencv_onnx::NodeProto priorsProto; + priorsProto.add_output(constParams.name); + addLayer(constParams, priorsProto); + + node_proto.set_input(2, constParams.name); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseCustom(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + for (int j = 0; j < node_proto.input_size(); j++) { + if (layer_id.find(node_proto.input(j)) == layer_id.end()) + layerParams.blobs.push_back(getBlob(node_proto, j)); + } + addLayer(layerParams, node_proto); +} + +const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() +{ + DispatchMap dispatch; + + dispatch["MaxPool"] = &ONNXImporter::parseMaxPool; + dispatch["AveragePool"] = &ONNXImporter::parseAveragePool; + dispatch["GlobalAveragePool"] = dispatch["GlobalMaxPool"] = dispatch["ReduceMean"] = dispatch["ReduceSum"] = + dispatch["ReduceMax"] = &ONNXImporter::parseReduce; + dispatch["Slice"] = &ONNXImporter::parseSlice; + dispatch["Split"] = &ONNXImporter::parseSplit; + dispatch["Add"] = dispatch["Sum"] = dispatch["Sub"] = &ONNXImporter::parseBias; + dispatch["Pow"] = &ONNXImporter::parsePow; + dispatch["Max"] = &ONNXImporter::parseMax; + dispatch["Neg"] = &ONNXImporter::parseNeg; + dispatch["Constant"] = &ONNXImporter::parseConstant; + dispatch["LSTM"] = &ONNXImporter::parseLSTM; + dispatch["ImageScaler"] = &ONNXImporter::parseImageScaler; + dispatch["Clip"] = &ONNXImporter::parseClip; + dispatch["LeakyRelu"] = &ONNXImporter::parseLeakyRelu; + dispatch["Relu"] = &ONNXImporter::parseRelu; + dispatch["Elu"] = &ONNXImporter::parseElu; + dispatch["Tanh"] = &ONNXImporter::parseTanh; + dispatch["PRelu"] = &ONNXImporter::parsePRelu; + dispatch["LRN"] = &ONNXImporter::parseLRN; + dispatch["InstanceNormalization"] = &ONNXImporter::parseInstanceNormalization; + dispatch["BatchNormalization"] = &ONNXImporter::parseBatchNormalization; + dispatch["Gemm"] = &ONNXImporter::parseGemm; + dispatch["MatMul"] = &ONNXImporter::parseMatMul; + dispatch["Mul"] = dispatch["Div"] = &ONNXImporter::parseMul; + dispatch["Conv"] = &ONNXImporter::parseConv; + dispatch["ConvTranspose"] = &ONNXImporter::parseConvTranspose; + dispatch["Transpose"] = &ONNXImporter::parseTranspose; + dispatch["Squeeze"] = &ONNXImporter::parseSqueeze; + dispatch["Flatten"] = &ONNXImporter::parseFlatten; + dispatch["Unsqueeze"] = &ONNXImporter::parseUnsqueeze; + dispatch["Expand"] = &ONNXImporter::parseExpand; + dispatch["Reshape"] = &ONNXImporter::parseReshape; + dispatch["Pad"] = &ONNXImporter::parsePad; + dispatch["Shape"] = &ONNXImporter::parseShape; + dispatch["Cast"] = &ONNXImporter::parseCast; + dispatch["ConstantFill"] = dispatch["ConstantOfShape"] = &ONNXImporter::parseConstantFill; + dispatch["Gather"] = &ONNXImporter::parseGather; + dispatch["Concat"] = &ONNXImporter::parseConcat; + dispatch["Resize"] = &ONNXImporter::parseResize; + dispatch["Upsample"] = &ONNXImporter::parseUpsample; + dispatch["SoftMax"] = dispatch["LogSoftmax"] = &ONNXImporter::parseSoftMax; + dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput; + dispatch["Custom"] = &ONNXImporter::parseCustom; + + return dispatch; } Net readNetFromONNX(const String& onnxFile) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 4bd09adda043..38a55d12b9a9 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -2894,7 +2894,7 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer) DispatchMap::const_iterator iter = dispatch.find(type); if (iter != dispatch.end()) { - ((*this).*(iter->second))(net, layer, layerParams); + CALL_MEMBER_FN(*this, iter->second)(net, layer, layerParams); } else if (!DNN_DIAGNOSTICS_RUN || !layerHandler->handleMissing(layer)) { From b468468e7e24d78387fe1cf75ec7b46ae2319457 Mon Sep 17 00:00:00 2001 From: Mahendra Kumar <66687425+kumar-mahendra@users.noreply.github.com> Date: Fri, 23 Jul 2021 16:44:48 +0530 Subject: [PATCH 086/376] Closing brackets missing In line 94, closing brackets are added which were missing . --- .../py_contour_features/py_contour_features.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown index ecd0e97af2de..d0e6c4b2ac11 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown @@ -91,7 +91,7 @@ convexity defects, which are the local maximum deviations of hull from contours. There is a little bit things to discuss about it its syntax: @code{.py} -hull = cv.convexHull(points[, hull[, clockwise[, returnPoints]] +hull = cv.convexHull(points[, hull[, clockwise[, returnPoints]]]) @endcode Arguments details: From 531ea5b3a21a5632789e592cfd71e9515849523b Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Sun, 1 Aug 2021 11:59:16 +0300 Subject: [PATCH 087/376] fix: convert arguments names that are keywords reserved by Python --- .../include/opencv2/core/bindings_utils.hpp | 6 ++++++ modules/python/src2/gen2.py | 12 ++++++++++++ modules/python/test/test_misc.py | 17 +++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/modules/core/include/opencv2/core/bindings_utils.hpp b/modules/core/include/opencv2/core/bindings_utils.hpp index 98a4a2b78539..a3f83d9c2cf5 100644 --- a/modules/core/include/opencv2/core/bindings_utils.hpp +++ b/modules/core/include/opencv2/core/bindings_utils.hpp @@ -116,6 +116,12 @@ String dumpRange(const Range& argument) } } +CV_WRAP static inline +String testReservedKeywordConversion(int positional_argument, int lambda = 2, int from = 3) +{ + return format("arg=%d, lambda=%d, from=%d", positional_argument, lambda, from); +} + CV_WRAP static inline void testRaiseGeneralException() { diff --git a/modules/python/src2/gen2.py b/modules/python/src2/gen2.py index bccf0d27677b..51566fc24844 100755 --- a/modules/python/src2/gen2.py +++ b/modules/python/src2/gen2.py @@ -212,6 +212,16 @@ class FormatStrings: "c_string": ArgTypeInfo("char*", FormatStrings.string, '(char*)""') } +# Set of reserved keywords for Python. Can be acquired via the following call +# $ python -c "help('keywords')" +# Keywords that are reserved in C/C++ are excluded because they can not be +# used as variables identifiers +python_reserved_keywords = { + "True", "None", "False", "as", "assert", "def", "del", "elif", "except", "exec", + "finally", "from", "global", "import", "in", "is", "lambda", "nonlocal", + "pass", "print", "raise", "with", "yield" +} + def normalize_class_name(name): return re.sub(r"^cv\.", "", name).replace(".", "_") @@ -369,6 +379,8 @@ class ArgInfo(object): def __init__(self, arg_tuple): self.tp = handle_ptr(arg_tuple[0]) self.name = arg_tuple[1] + if self.name in python_reserved_keywords: + self.name += "_" self.defval = arg_tuple[2] self.isarray = False self.arraylen = 0 diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index 121e86a64c62..4d435a46b668 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -463,6 +463,23 @@ def test_parse_to_range_not_convertible(self): with self.assertRaises((TypeError), msg=get_no_exception_msg(not_convertible)): _ = cv.utils.dumpRange(not_convertible) + def test_reserved_keywords_are_transformed(self): + default_lambda_value = 2 + default_from_value = 3 + format_str = "arg={}, lambda={}, from={}" + self.assertEqual( + cv.utils.testReservedKeywordConversion(20), format_str.format(20, default_lambda_value, default_from_value) + ) + self.assertEqual( + cv.utils.testReservedKeywordConversion(10, lambda_=10), format_str.format(10, 10, default_from_value) + ) + self.assertEqual( + cv.utils.testReservedKeywordConversion(10, from_=10), format_str.format(10, default_lambda_value, 10) + ) + self.assertEqual( + cv.utils.testReservedKeywordConversion(20, lambda_=-4, from_=12), format_str.format(20, -4, 12) + ) + class SamplesFindFile(NewOpenCVTests): From ba0cea6826a90689d4d58efcd28d1aa53f3478cc Mon Sep 17 00:00:00 2001 From: Dmitry Budnikov Date: Mon, 2 Aug 2021 13:58:18 +0300 Subject: [PATCH 088/376] Merge pull request #20474 from dbudniko:dbudniko/mtcnn_graph_without_resizes Remove explicit PNet resizes from MTCNN graph * remove PNet resizes * address comment from Ruslan --- modules/gapi/samples/face_detection_mtcnn.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/modules/gapi/samples/face_detection_mtcnn.cpp b/modules/gapi/samples/face_detection_mtcnn.cpp index ad7c20c1175c..50cb666a90f5 100644 --- a/modules/gapi/samples/face_detection_mtcnn.cpp +++ b/modules/gapi/samples/face_detection_mtcnn.cpp @@ -591,16 +591,13 @@ int main(int argc, char* argv[]) { cv::GMat in_originalRGB = cv::gapi::BGR2RGB(in_original); cv::GMat in_transposedRGB = cv::gapi::transpose(in_originalRGB); cv::GOpaque in_sz = cv::gapi::streaming::size(in_original); - cv::GMat in_resized[MAX_PYRAMID_LEVELS]; cv::GMat regressions[MAX_PYRAMID_LEVELS]; cv::GMat scores[MAX_PYRAMID_LEVELS]; cv::GArray nms_p_faces[MAX_PYRAMID_LEVELS]; cv::GArray total_faces[MAX_PYRAMID_LEVELS]; //The very first PNet pyramid layer to init total_faces[0] - cv::Size currentSize = cv::Size(level_size[0].height, level_size[0].width); - in_resized[0] = cv::gapi::resize(in_transposedRGB, currentSize); - std::tie(regressions[0], scores[0]) = run_mtcnn_p(in_resized[0], get_pnet_level_name(level_size[0])); + std::tie(regressions[0], scores[0]) = run_mtcnn_p(in_transposedRGB, get_pnet_level_name(level_size[0])); cv::GArray faces0 = custom::BuildFaces::on(scores[0], regressions[0], static_cast(scales[0]), conf_thresh_p); cv::GArray final_p_faces_for_bb2squares = custom::ApplyRegression::on(faces0, true); cv::GArray final_faces_pnet0 = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares); @@ -608,9 +605,7 @@ int main(int argc, char* argv[]) { //The rest PNet pyramid layers to accumlate all layers result in total_faces[PYRAMID_LEVELS - 1]] for (int i = 1; i < pyramid_levels; ++i) { - currentSize = cv::Size(level_size[i].height, level_size[i].width); - in_resized[i] = cv::gapi::resize(in_transposedRGB, currentSize); - std::tie(regressions[i], scores[i]) = run_mtcnn_p(in_resized[i], get_pnet_level_name(level_size[i])); + std::tie(regressions[i], scores[i]) = run_mtcnn_p(in_transposedRGB, get_pnet_level_name(level_size[i])); cv::GArray faces = custom::BuildFaces::on(scores[i], regressions[i], static_cast(scales[i]), conf_thresh_p); cv::GArray final_p_faces_for_bb2squares_i = custom::ApplyRegression::on(faces, true); cv::GArray final_faces_pnet_i = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares_i); From 2d8ce500fa4961a9122346e51c2d6d421f0b8cef Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Mon, 2 Aug 2021 18:41:53 +0300 Subject: [PATCH 089/376] add note about using version 3.4 to traincascade documentation --- doc/tutorials/objdetect/traincascade.markdown | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/tutorials/objdetect/traincascade.markdown b/doc/tutorials/objdetect/traincascade.markdown index d7acdca5539f..d78de2ec9a6f 100644 --- a/doc/tutorials/objdetect/traincascade.markdown +++ b/doc/tutorials/objdetect/traincascade.markdown @@ -10,6 +10,8 @@ Working with a boosted cascade of weak classifiers includes two major stages: th To support this tutorial, several official OpenCV applications will be used: [opencv_createsamples](https://github.com/opencv/opencv/tree/3.4/apps/createsamples), [opencv_annotation](https://github.com/opencv/opencv/tree/3.4/apps/annotation), [opencv_traincascade](https://github.com/opencv/opencv/tree/3.4/apps/traincascade) and [opencv_visualisation](https://github.com/opencv/opencv/tree/3.4/apps/visualisation). +@note Createsamples and traincascade are disabled since OpenCV 4.0. Consider using these apps for training from 3.4 branch for Cascade Classifier. Model format is the same between 3.4 and 4.x. + ### Important notes - If you come across any tutorial mentioning the old opencv_haartraining tool (which is deprecated and still using the OpenCV1.x interface), then please ignore that tutorial and stick to the opencv_traincascade tool. This tool is a newer version, written in C++ in accordance to the OpenCV 2.x and OpenCV 3.x API. The opencv_traincascade supports both HAAR like wavelet features @cite Viola01 and LBP (Local Binary Patterns) @cite Liao2007 features. LBP features yield integer precision in contrast to HAAR features, yielding floating point precision, so both training and detection with LBP are several times faster then with HAAR features. Regarding the LBP and HAAR detection quality, it mainly depends on the training data used and the training parameters selected. It's possible to train a LBP-based classifier that will provide almost the same quality as HAAR-based one, within a percentage of the training time. From d773691848f6850eb1b21b2b6031a5f64c321efa Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Wed, 4 Aug 2021 15:37:20 +0300 Subject: [PATCH 090/376] add note about hierarchy in python version --- modules/imgproc/include/opencv2/imgproc.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index a1cfff991da8..5e66b14e3b2b 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -3951,6 +3951,7 @@ hierarchy[i][0] , hierarchy[i][1] , hierarchy[i][2] , and hierarchy[i][3] are se in contours of the next and previous contours at the same hierarchical level, the first child contour and the parent contour, respectively. If for the contour i there are no next, previous, parent, or nested contours, the corresponding elements of hierarchy[i] will be negative. +@note In Python, hierarchy is nested inside a top level array. Use hierarchy[0][i] to access hierarchical elements of i-th contour. @param mode Contour retrieval mode, see #RetrievalModes @param method Contour approximation method, see #ContourApproximationModes @param offset Optional offset by which every contour point is shifted. This is useful if the From cefa60260105aaa33d54daddacd3c60a1ce93e57 Mon Sep 17 00:00:00 2001 From: Duong Dac Date: Wed, 4 Aug 2021 15:17:25 +0200 Subject: [PATCH 091/376] Avoid adding false UMat/GpuMat declaration --- modules/python/src2/hdr_parser.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/python/src2/hdr_parser.py b/modules/python/src2/hdr_parser.py index ac3f383dc85b..749a9033eed6 100755 --- a/modules/python/src2/hdr_parser.py +++ b/modules/python/src2/hdr_parser.py @@ -954,7 +954,8 @@ def parse(self, hname, wmode=True): has_mat = len(list(filter(lambda x: x[0] in {"Mat", "vector_Mat"}, args))) > 0 if has_mat: _, _, _, gpumat_decl = self.parse_stmt(stmt, token, mat="cuda::GpuMat", docstring=docstring) - decls.append(gpumat_decl) + if gpumat_decl != decl: + decls.append(gpumat_decl) if self._generate_umat_decls: # If function takes as one of arguments Mat or vector - we want to create the @@ -963,7 +964,8 @@ def parse(self, hname, wmode=True): has_mat = len(list(filter(lambda x: x[0] in {"Mat", "vector_Mat"}, args))) > 0 if has_mat: _, _, _, umat_decl = self.parse_stmt(stmt, token, mat="UMat", docstring=docstring) - decls.append(umat_decl) + if umat_decl != decl: + decls.append(umat_decl) docstring = "" if stmt_type == "namespace": From 6a2e559222e3950d7323e5703b3f5857f59dafce Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Wed, 4 Aug 2021 20:04:03 +0300 Subject: [PATCH 092/376] Fixed memory access issue in v_rshr_pack_store intrinsic on RISC-V RVV. --- modules/core/include/opencv2/core/hal/intrin_rvv.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp index 51433cdbae72..5b3319378103 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp @@ -1967,7 +1967,7 @@ void v_rshr_pack_store(_Tp* ptr, const _wTpvec& a) \ _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ - v_store(ptr, _Tpvec(rshr(vle##width##_v_##suffix##m2(arr, vl), n, vl))); \ + vse##hwidth##_v_##hsuffix##m1(ptr, _Tpvec(rshr(vle##width##_v_##suffix##m2(arr, vl), n, vl)), hvl); \ } OPENCV_HAL_IMPL_RVV_PACK(v_uint8x16, uchar, v_uint16x8, ushort, 8, 16, u8, u16, vnclipu_wx_u8m1, vnclipu_wx_u8m1, 8, 16) From d5f34cf34c7088ed69e3e3a922435743e3d95d81 Mon Sep 17 00:00:00 2001 From: Abhinav Sharma <63901956+abhi-bhatra@users.noreply.github.com> Date: Wed, 4 Aug 2021 22:58:59 +0530 Subject: [PATCH 093/376] Merge pull request #20493 from abhi-bhatra:patch-1 Fix Broken Link * Update README.md Fix broken link * Update Readme.txt * Update readme.txt Add missing link * Update readme.txt Fix names --- 3rdparty/readme.txt | 6 ++++-- platforms/winrt/readme.txt | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/3rdparty/readme.txt b/3rdparty/readme.txt index 3b961782b097..4e4a6ba0a653 100644 --- a/3rdparty/readme.txt +++ b/3rdparty/readme.txt @@ -31,7 +31,7 @@ libpng Portable Network Graphics library. libtiff Tag Image File Format (TIFF) Software Copyright (c) 1988-1997 Sam Leffler Copyright (c) 1991-1997 Silicon Graphics, Inc. - See libtiff home page http://www.remotesensing.org/libtiff/ + See libtiff home page http://www.libtiff.org/ for details and links to the source code WITH_TIFF CMake option must be ON to add libtiff & zlib support to imgcodecs. @@ -51,7 +51,9 @@ jasper JasPer is a collection of software Copyright (c) 1999-2000 The University of British Columbia Copyright (c) 2001-2003 Michael David Adams - The JasPer license can be found in libjasper. + See JasPer official GitHub repository + https://github.com/jasper-software/jasper.git + for details and links to source code ------------------------------------------------------------------------------------ openexr OpenEXR is a high dynamic-range (HDR) image file format developed by Industrial Light & Magic for use in computer imaging applications. diff --git a/platforms/winrt/readme.txt b/platforms/winrt/readme.txt index 2fb4ce1f54c4..2d1b4e6c30c1 100644 --- a/platforms/winrt/readme.txt +++ b/platforms/winrt/readme.txt @@ -13,7 +13,7 @@ Install Visual Studio 2013 Community Edition http://go.microsoft.com/?linkid=9863608 Install Visual Studio Express 2012 for Windows Desktop - http://www.microsoft.com/en-us/download/details.aspx?id=34673 + https://devblogs.microsoft.com/visualstudio/visual-studio-express-2012-for-windows-desktop-is-here/ @@ -156,4 +156,4 @@ Manual build cmake -G "Visual Studio 12 2013 Win64" -DCMAKE_SYSTEM_NAME:String=WindowsStore -DCMAKE_SYSTEM_VERSION:String=8.1 -DCMAKE_VS_EFFECTIVE_PLATFORMS:String=x64 -DCMAKE_INSTALL_PREFIX:PATH=.\install\WS\8.1\x64\ .. -Return to "Running tests for Windows Store", list item 4. \ No newline at end of file +Return to "Running tests for Windows Store", list item 4. From 5af09e73f250946e3e5447099a23c1e07131b1fe Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Thu, 5 Aug 2021 11:59:24 +0300 Subject: [PATCH 094/376] Merge pull request #20490 from TolyaTalamanov:at/support-fp16-input-precision [G-API] Support FP16 input precision for IE backend * Support fp16 input precision IE backend * Add support to wrapIE --- modules/gapi/src/backends/ie/giebackend.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 77a6515f8530..fc9fc502ef6d 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -108,6 +108,7 @@ inline IE::Precision toIE(int depth) { case CV_8U: return IE::Precision::U8; case CV_32S: return IE::Precision::I32; case CV_32F: return IE::Precision::FP32; + case CV_16F: return IE::Precision::FP16; default: GAPI_Assert(false && "IE. Unsupported data type"); } return IE::Precision::UNSPECIFIED; @@ -161,6 +162,7 @@ inline IE::Blob::Ptr wrapIE(const cv::Mat &mat, cv::gapi::ie::TraitAs hint) { HANDLE(8U, uint8_t); HANDLE(32F, float); HANDLE(32S, int); + HANDLE(16F, int16_t); #undef HANDLE default: GAPI_Assert(false && "IE. Unsupported data type"); } From 65134c793bc7c80cf63f7a4d32158fa4e17ffed3 Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Thu, 5 Aug 2021 14:21:34 +0300 Subject: [PATCH 095/376] add links in python and js contours tutorial --- .../js_contour_features/js_contour_features.markdown | 3 +++ .../js_contour_properties/js_contour_properties.markdown | 3 +++ .../js_contours/js_contours_begin/js_contours_begin.markdown | 2 ++ .../js_contours_hierarchy/js_contours_hierarchy.markdown | 2 ++ .../js_contours_more_functions.markdown | 3 +++ .../py_contour_features/py_contour_features.markdown | 3 +++ .../py_contour_properties/py_contour_properties.markdown | 3 +++ .../py_contours/py_contours_begin/py_contours_begin.markdown | 2 ++ .../py_contours_hierarchy/py_contours_hierarchy.markdown | 2 ++ .../py_contours_more_functions.markdown | 4 ++++ 10 files changed, 27 insertions(+) diff --git a/doc/js_tutorials/js_imgproc/js_contours/js_contour_features/js_contour_features.markdown b/doc/js_tutorials/js_imgproc/js_contours/js_contour_features/js_contour_features.markdown index 22544b280c60..842126958731 100644 --- a/doc/js_tutorials/js_imgproc/js_contours/js_contour_features/js_contour_features.markdown +++ b/doc/js_tutorials/js_imgproc/js_contours/js_contour_features/js_contour_features.markdown @@ -1,6 +1,9 @@ Contour Features {#tutorial_js_contour_features} ================ +@prev_tutorial{tutorial_js_contours_begin} +@next_tutorial{tutorial_js_contour_properties} + Goal ---- diff --git a/doc/js_tutorials/js_imgproc/js_contours/js_contour_properties/js_contour_properties.markdown b/doc/js_tutorials/js_imgproc/js_contours/js_contour_properties/js_contour_properties.markdown index 1dbb15c4cf3e..3392283305c0 100644 --- a/doc/js_tutorials/js_imgproc/js_contours/js_contour_properties/js_contour_properties.markdown +++ b/doc/js_tutorials/js_imgproc/js_contours/js_contour_properties/js_contour_properties.markdown @@ -1,6 +1,9 @@ Contour Properties {#tutorial_js_contour_properties} ================== +@prev_tutorial{tutorial_js_contour_features} +@next_tutorial{tutorial_js_contours_more_functions} + Goal ---- diff --git a/doc/js_tutorials/js_imgproc/js_contours/js_contours_begin/js_contours_begin.markdown b/doc/js_tutorials/js_imgproc/js_contours/js_contours_begin/js_contours_begin.markdown index 9678a7c9f05d..3caf17f873a4 100644 --- a/doc/js_tutorials/js_imgproc/js_contours/js_contours_begin/js_contours_begin.markdown +++ b/doc/js_tutorials/js_imgproc/js_contours/js_contours_begin/js_contours_begin.markdown @@ -1,6 +1,8 @@ Contours : Getting Started {#tutorial_js_contours_begin} ========================== +@next_tutorial{tutorial_js_contour_features} + Goal ---- diff --git a/doc/js_tutorials/js_imgproc/js_contours/js_contours_hierarchy/js_contours_hierarchy.markdown b/doc/js_tutorials/js_imgproc/js_contours/js_contours_hierarchy/js_contours_hierarchy.markdown index c98628e48648..c2e408a96292 100644 --- a/doc/js_tutorials/js_imgproc/js_contours/js_contours_hierarchy/js_contours_hierarchy.markdown +++ b/doc/js_tutorials/js_imgproc/js_contours/js_contours_hierarchy/js_contours_hierarchy.markdown @@ -1,6 +1,8 @@ Contours Hierarchy {#tutorial_js_contours_hierarchy} ================== +@prev_tutorial{tutorial_js_contours_more_functions} + Goal ---- diff --git a/doc/js_tutorials/js_imgproc/js_contours/js_contours_more_functions/js_contours_more_functions.markdown b/doc/js_tutorials/js_imgproc/js_contours/js_contours_more_functions/js_contours_more_functions.markdown index b75311666271..941f0c486a1f 100644 --- a/doc/js_tutorials/js_imgproc/js_contours/js_contours_more_functions/js_contours_more_functions.markdown +++ b/doc/js_tutorials/js_imgproc/js_contours/js_contours_more_functions/js_contours_more_functions.markdown @@ -1,6 +1,9 @@ Contours : More Functions {#tutorial_js_contours_more_functions} ========================= +@prev_tutorial{tutorial_js_contour_properties} +@next_tutorial{tutorial_js_contours_hierarchy} + Goal ---- diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown index d0e6c4b2ac11..e8cfbd659715 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown @@ -1,6 +1,9 @@ Contour Features {#tutorial_py_contour_features} ================ +@prev_tutorial{tutorial_py_contours_begin} +@next_tutorial{tutorial_py_contour_properties} + Goal ---- diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contour_properties/py_contour_properties.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contour_properties/py_contour_properties.markdown index 461c87034398..282f62ddf98e 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contour_properties/py_contour_properties.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contour_properties/py_contour_properties.markdown @@ -1,6 +1,9 @@ Contour Properties {#tutorial_py_contour_properties} ================== +@prev_tutorial{tutorial_py_contour_features} +@next_tutorial{tutorial_py_contours_more_functions} + Here we will learn to extract some frequently used properties of objects like Solidity, Equivalent Diameter, Mask image, Mean Intensity etc. More features can be found at [Matlab regionprops documentation](http://www.mathworks.in/help/images/ref/regionprops.html). diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contours_begin/py_contours_begin.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contours_begin/py_contours_begin.markdown index c2055f75af68..74d7b252a542 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contours_begin/py_contours_begin.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contours_begin/py_contours_begin.markdown @@ -1,6 +1,8 @@ Contours : Getting Started {#tutorial_py_contours_begin} ========================== +@next_tutorial{tutorial_py_contour_features} + Goal ---- diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contours_hierarchy/py_contours_hierarchy.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contours_hierarchy/py_contours_hierarchy.markdown index 2619ea2a7095..075e6ec81f93 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contours_hierarchy/py_contours_hierarchy.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contours_hierarchy/py_contours_hierarchy.markdown @@ -1,6 +1,8 @@ Contours Hierarchy {#tutorial_py_contours_hierarchy} ================== +@prev_tutorial{tutorial_py_contours_more_functions} + Goal ---- diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown index fb5f59bef661..397a2a63a06f 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown @@ -1,6 +1,10 @@ Contours : More Functions {#tutorial_py_contours_more_functions} ========================= +@prev_tutorial{tutorial_py_contour_properties} +@next_tutorial{tutorial_py_contours_hierarchy} + + Goal ---- From 4ff76cad2a0b204a43ec4d57aebe65aa2a616543 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 5 Aug 2021 11:42:58 +0000 Subject: [PATCH 096/376] cmake: fix cross-compilation problems - unexpected pkg-config module (we should not use host binary) - bump cmake_minimum_required to 3.5 in toolchain files --- modules/highgui/cmake/init.cmake | 4 +++- modules/videoio/cmake/init.cmake | 4 +++- platforms/linux/gnu.toolchain.cmake | 2 +- platforms/linux/riscv.toolchain.cmake | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/highgui/cmake/init.cmake b/modules/highgui/cmake/init.cmake index a302c4d534a6..2002ff0e9d70 100644 --- a/modules/highgui/cmake/init.cmake +++ b/modules/highgui/cmake/init.cmake @@ -25,7 +25,9 @@ endif() # Detect available dependencies # -include(FindPkgConfig) +if(NOT PROJECT_NAME STREQUAL "OpenCV") + include(FindPkgConfig) +endif() macro(add_backend backend_id cond_var) if(${cond_var}) diff --git a/modules/videoio/cmake/init.cmake b/modules/videoio/cmake/init.cmake index 68838790b8a2..af664f94df37 100644 --- a/modules/videoio/cmake/init.cmake +++ b/modules/videoio/cmake/init.cmake @@ -1,4 +1,6 @@ -include(FindPkgConfig) +if(NOT PROJECT_NAME STREQUAL "OpenCV") + include(FindPkgConfig) +endif() macro(add_backend backend_id cond_var) if(${cond_var}) diff --git a/platforms/linux/gnu.toolchain.cmake b/platforms/linux/gnu.toolchain.cmake index cba08e7fbbf4..64258e65b3f5 100644 --- a/platforms/linux/gnu.toolchain.cmake +++ b/platforms/linux/gnu.toolchain.cmake @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 3.5) # load settings in case of "try compile" set(TOOLCHAIN_CONFIG_FILE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/toolchain.config.cmake") diff --git a/platforms/linux/riscv.toolchain.cmake b/platforms/linux/riscv.toolchain.cmake index 2a69d7e0048d..cea80bd9ba90 100644 --- a/platforms/linux/riscv.toolchain.cmake +++ b/platforms/linux/riscv.toolchain.cmake @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 3.5) if(COMMAND toolchain_save_config) return() # prevent recursive call From 633fedaa96d1a6e48db5f0897d82fa5d2c5f4560 Mon Sep 17 00:00:00 2001 From: Julia Bareeva <34717687+JulieBar@users.noreply.github.com> Date: Thu, 5 Aug 2021 18:13:17 +0300 Subject: [PATCH 097/376] Merge pull request #20480 from JulieBar:lstm_pytest Add Python's test for LSTM layer * Add Python's test for LSTM layer * Set different test threshold for FP16 target * rename test to test_input_3d Co-authored-by: Julie Bareeva --- modules/dnn/misc/python/test/test_dnn.py | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/modules/dnn/misc/python/test/test_dnn.py b/modules/dnn/misc/python/test/test_dnn.py index 932984f1c201..f7bfc0111940 100644 --- a/modules/dnn/misc/python/test/test_dnn.py +++ b/modules/dnn/misc/python/test/test_dnn.py @@ -62,6 +62,12 @@ def printParams(backend, target): } print('%s/%s' % (backendNames[backend], targetNames[target])) +def getDefaultThreshold(target): + if target == cv.dnn.DNN_TARGET_OPENCL_FP16 or target == cv.dnn.DNN_TARGET_MYRIAD: + return 4e-3 + else: + return 1e-5 + testdata_required = bool(os.environ.get('OPENCV_DNN_TEST_REQUIRE_TESTDATA', False)) g_dnnBackendsAndTargets = None @@ -305,5 +311,35 @@ def forward(self, inputs): cv.dnn_unregisterLayer('CropCaffe') + # check that dnn module can work with 3D tensor as input for network + def test_input_3d(self): + model = self.find_dnn_file('dnn/onnx/models/hidden_lstm.onnx') + input_file = self.find_dnn_file('dnn/onnx/data/input_hidden_lstm.npy') + output_file = self.find_dnn_file('dnn/onnx/data/output_hidden_lstm.npy') + if model is None: + raise unittest.SkipTest("Missing DNN test files (dnn/onnx/models/hidden_lstm.onnx). " + "Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.") + if input_file is None or output_file is None: + raise unittest.SkipTest("Missing DNN test files (dnn/onnx/data/{input/output}_hidden_lstm.npy). " + "Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.") + + net = cv.dnn.readNet(model) + input = np.load(input_file) + # we have to expand the shape of input tensor because Python bindings cut 3D tensors to 2D + # it should be fixed in future. see : https://github.com/opencv/opencv/issues/19091 + # please remove `expand_dims` after that + input = np.expand_dims(input, axis=3) + gold_output = np.load(output_file) + net.setInput(input) + + for backend, target in self.dnnBackendsAndTargets: + printParams(backend, target) + + net.setPreferableBackend(backend) + net.setPreferableTarget(target) + real_output = net.forward() + + normAssert(self, real_output, gold_output, "", getDefaultThreshold(target)) + if __name__ == '__main__': NewOpenCVTests.bootstrap() From 6306bc3ddcbe3480a6d113081bebedb5745dff24 Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Fri, 6 Aug 2021 09:52:46 +0300 Subject: [PATCH 098/376] Applying of comments --- .../misc/python/samples/gaze_estimation.py | 39 +++++++------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/modules/gapi/misc/python/samples/gaze_estimation.py b/modules/gapi/misc/python/samples/gaze_estimation.py index db190f67bb99..5536787e608c 100644 --- a/modules/gapi/misc/python/samples/gaze_estimation.py +++ b/modules/gapi/misc/python/samples/gaze_estimation.py @@ -3,6 +3,7 @@ import numpy as np import cv2 as cv + # ------------------------Service operations------------------------ def weight_path(model_path): """ Get path of weights based on path to IR @@ -171,11 +172,7 @@ def run(in_ys, in_ps, in_rs): Return: Arrays with heads poses """ - out_poses = [] - size = len(in_ys) - for i in range(size): - out_poses.append(np.array([in_ys[i][0], in_ps[i][0], in_rs[i][0]]).T) - return out_poses + return [np.array([ys[0], ps[0], rs[0]]).T for ys, ps, rs in zip(in_ys, in_ps, in_rs)] @cv.gapi.kernel(GParseEyes) @@ -199,20 +196,19 @@ def run(in_landm_per_face, in_face_rcs, frame_size): right_eyes = [] midpoints = [] lmarks = [] - num_faces = len(in_landm_per_face) surface = (0, 0, *frame_size) - for i in range(num_faces): - rect = in_face_rcs[i] - points = process_landmarks(*rect, in_landm_per_face[i]) - for p in points: - lmarks.append(p) - size = int(len(in_landm_per_face[i][0]) / 2) - - rect, midpoint_l = eye_box(lmarks[0 + i * size], lmarks[1 + i * size]) + for landm_face, rect in zip(in_landm_per_face, in_face_rcs): + points = process_landmarks(*rect, landm_face) + lmarks.extend(points) + + rect, midpoint_l = eye_box(points[0], points[1]) left_eyes.append(intersection(surface, rect)) - rect, midpoint_r = eye_box(lmarks[2 + i * size], lmarks[3 + i * size]) + + rect, midpoint_r = eye_box(points[2], points[3]) right_eyes.append(intersection(surface, rect)) - midpoints += [midpoint_l, midpoint_r] + + midpoints.append(midpoint_l) + midpoints.append(midpoint_r) return left_eyes, right_eyes, midpoints, lmarks @@ -231,14 +227,8 @@ def run(eyesl, eyesr): Return: States of left eyes and states of right eyes """ - size = len(eyesl) - out_l_st = [] - out_r_st = [] - for i in range(size): - for st in eyesl[i]: - out_l_st += [1 if st[0] < st[1] else 0] - for st in eyesr[i]: - out_r_st += [1 if st[0] < st[1] else 0] + out_l_st = [int(st) for eye_l in eyesl for st in (eye_l[:, 0] < eye_l[:, 1]).ravel()] + out_r_st = [int(st) for eye_r in eyesr for st in (eye_r[:, 0] < eye_r[:, 1]).ravel()] return out_l_st, out_r_st @@ -459,6 +449,7 @@ def run(eyesl, eyesr): # Show result cv.imshow('Gaze Estimation', oimg) + cv.waitKey(1) fps = int(1. / (time.time() - start_time_cycle)) frames += 1 From 24de676a6480aae151b1a2c80a483ec3ef98ab2a Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Fri, 6 Aug 2021 13:26:49 +0300 Subject: [PATCH 099/376] Merge pull request #20476 from TolyaTalamanov:at/support-unet-camvid-0001-segm-sample [G-API] Support postprocessing for not argmaxed outputs * Support postprocessing for not argmaxed outputs * Fix typo * Add assert * Remove static cast * CamelCast to snake_case * Fix windows warning * Add static_cast to uint8_t * Add const to variables --- .../gapi/samples/semantic_segmentation.cpp | 83 ++++++++++++++----- 1 file changed, 63 insertions(+), 20 deletions(-) diff --git a/modules/gapi/samples/semantic_segmentation.cpp b/modules/gapi/samples/semantic_segmentation.cpp index 0a6e7231c4f2..4cdb14cc5c1a 100644 --- a/modules/gapi/samples/semantic_segmentation.cpp +++ b/modules/gapi/samples/semantic_segmentation.cpp @@ -47,6 +47,53 @@ std::string get_weights_path(const std::string &model_path) { CV_Assert(ext == ".xml"); return model_path.substr(0u, sz - EXT_LEN) + ".bin"; } + +void classesToColors(const cv::Mat &out_blob, + cv::Mat &mask_img) { + const int H = out_blob.size[0]; + const int W = out_blob.size[1]; + + mask_img.create(H, W, CV_8UC3); + GAPI_Assert(out_blob.type() == CV_8UC1); + const uint8_t* const classes = out_blob.ptr(); + + for (int rowId = 0; rowId < H; ++rowId) { + for (int colId = 0; colId < W; ++colId) { + uint8_t class_id = classes[rowId * W + colId]; + mask_img.at(rowId, colId) = + class_id < colors.size() + ? colors[class_id] + : cv::Vec3b{0, 0, 0}; // NB: sample supports 20 classes + } + } +} + +void probsToClasses(const cv::Mat& probs, cv::Mat& classes) { + const int C = probs.size[1]; + const int H = probs.size[2]; + const int W = probs.size[3]; + + classes.create(H, W, CV_8UC1); + GAPI_Assert(probs.depth() == CV_32F); + float* out_p = reinterpret_cast(probs.data); + uint8_t* classes_p = reinterpret_cast(classes.data); + + for (int h = 0; h < H; ++h) { + for (int w = 0; w < W; ++w) { + double max = 0; + int class_id = 0; + for (int c = 0; c < C; ++c) { + int idx = c * H * W + h * W + w; + if (out_p[idx] > max) { + max = out_p[idx]; + class_id = c; + } + } + classes_p[h * W + w] = static_cast(class_id); + } + } +} + } // anonymous namespace namespace custom { @@ -57,25 +104,21 @@ G_API_OP(PostProcessing, , "sample.custom.post_pro }; GAPI_OCV_KERNEL(OCVPostProcessing, PostProcessing) { - static void run(const cv::Mat &in, const cv::Mat &detected_classes, cv::Mat &out) { - // This kernel constructs output image by class table and colors vector - - // The semantic-segmentation-adas-0001 output a blob with the shape - // [B, C=1, H=1024, W=2048] - const int outHeight = 1024; - const int outWidth = 2048; - cv::Mat maskImg(outHeight, outWidth, CV_8UC3); - const int* const classes = detected_classes.ptr(); - for (int rowId = 0; rowId < outHeight; ++rowId) { - for (int colId = 0; colId < outWidth; ++colId) { - size_t classId = static_cast(classes[rowId * outWidth + colId]); - maskImg.at(rowId, colId) = - classId < colors.size() - ? colors[classId] - : cv::Vec3b{0, 0, 0}; // sample detects 20 classes - } + static void run(const cv::Mat &in, const cv::Mat &out_blob, cv::Mat &out) { + cv::Mat classes; + // NB: If output has more than single plane, it contains probabilities + // otherwise class id. + if (out_blob.size[1] > 1) { + probsToClasses(out_blob, classes); + } else { + out_blob.convertTo(classes, CV_8UC1); + classes = classes.reshape(1, out_blob.size[2]); } - cv::resize(maskImg, out, in.size()); + + cv::Mat mask_img; + classesToColors(classes, mask_img); + + cv::resize(mask_img, out, in.size()); const float blending = 0.3f; out = in * blending + out * (1 - blending); } @@ -104,8 +147,8 @@ int main(int argc, char *argv[]) { // Now build the graph cv::GMat in; - cv::GMat detected_classes = cv::gapi::infer(in); - cv::GMat out = custom::PostProcessing::on(in, detected_classes); + cv::GMat out_blob = cv::gapi::infer(in); + cv::GMat out = custom::PostProcessing::on(in, out_blob); cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) .compileStreaming(cv::compile_args(kernels, networks)); From ba539eb9aad7571361657ea7e00a7c3efcc2f9ba Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Fri, 6 Aug 2021 14:41:56 +0300 Subject: [PATCH 100/376] Merge pull request #20508 from TolyaTalamanov:at/expand-python-pyparams [G-API] Expand PyParams to support constInput * Wrap constInputs to python * Wrap cfgNumRequests * Fix alignment * Move macro to the line above --- .../gapi/include/opencv2/gapi/infer/bindings_ie.hpp | 11 +++++++++++ modules/gapi/src/backends/ie/bindings_ie.cpp | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp b/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp index fdd4128b1ae2..92ef2101a179 100644 --- a/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp @@ -22,17 +22,28 @@ namespace ie { // This class can be marked as SIMPLE, because it's implemented as pimpl class GAPI_EXPORTS_W_SIMPLE PyParams { public: + GAPI_WRAP PyParams() = default; + GAPI_WRAP PyParams(const std::string &tag, const std::string &model, const std::string &weights, const std::string &device); + GAPI_WRAP PyParams(const std::string &tag, const std::string &model, const std::string &device); + GAPI_WRAP + PyParams& constInput(const std::string &layer_name, + const cv::Mat &data, + TraitAs hint = TraitAs::TENSOR); + + GAPI_WRAP + PyParams& cfgNumRequests(size_t nireq); + GBackend backend() const; std::string tag() const; cv::util::any params() const; diff --git a/modules/gapi/src/backends/ie/bindings_ie.cpp b/modules/gapi/src/backends/ie/bindings_ie.cpp index 35191d7bcb53..5874fe137864 100644 --- a/modules/gapi/src/backends/ie/bindings_ie.cpp +++ b/modules/gapi/src/backends/ie/bindings_ie.cpp @@ -37,3 +37,15 @@ cv::gapi::ie::PyParams cv::gapi::ie::params(const std::string &tag, const std::string &device) { return {tag, model, device}; } + +cv::gapi::ie::PyParams& cv::gapi::ie::PyParams::constInput(const std::string &layer_name, + const cv::Mat &data, + TraitAs hint) { + m_priv->constInput(layer_name, data, hint); + return *this; +} + +cv::gapi::ie::PyParams& cv::gapi::ie::PyParams::cfgNumRequests(size_t nireq) { + m_priv->cfgNumRequests(nireq); + return *this; +} From e1cafa383431a65e4ca0493a21668444d10a14b3 Mon Sep 17 00:00:00 2001 From: Julia Bareeva <34717687+JulieBar@users.noreply.github.com> Date: Sat, 7 Aug 2021 10:07:37 +0300 Subject: [PATCH 101/376] Merge pull request #20442 from JulieBar:gru_layer * Add initialization and inference for GRU layer * fix issues found on review --- .../dnn/include/opencv2/dnn/all_layers.hpp | 34 +++ modules/dnn/src/init.cpp | 1 + modules/dnn/src/layers/recurrent_layers.cpp | 209 ++++++++++++++++++ modules/dnn/src/onnx/onnx_importer.cpp | 43 ++++ modules/dnn/test/test_layers.cpp | 29 +++ modules/dnn/test/test_onnx_importer.cpp | 10 + 6 files changed, 326 insertions(+) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 24d35646df17..9c96c5a5f187 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -165,6 +165,40 @@ CV__DNN_INLINE_NS_BEGIN int outputNameToIndex(const String& outputName) CV_OVERRIDE; }; + /** @brief GRU recurrent one-layer + * + * Accepts input sequence and computes the final hidden state for each element in the batch. + * + * - input[0] containing the features of the input sequence. + * input[0] should have shape [`T`, `N`, `data_dims`] where `T` is sequence length, `N` is batch size, `data_dims` is input size + * - output would have shape [`T`, `N`, `D` * `hidden_size`] where `D = 2` if layer is bidirectional otherwise `D = 1` + * + * Depends on the following attributes: + * - hidden_size - Number of neurons in the hidden layer + * - direction - RNN could be bidirectional or forward + * + * The final hidden state @f$ h_t @f$ computes by the following formulas: + * + @f{eqnarray*}{ + r_t = \sigma(W_{ir} x_t + b_{ir} + W_{hr} h_{(t-1)} + b_{hr}) \\ + z_t = \sigma(W_{iz} x_t + b_{iz} + W_{hz} h_{(t-1)} + b_{hz}) \\ + n_t = \tanh(W_{in} x_t + b_{in} + r_t \odot (W_{hn} h_{(t-1)}+ b_{hn})) \\ + h_t = (1 - z_t) \odot n_t + z_t \odot h_{(t-1)} \\ + @f} + * Where @f$x_t@f$ is current input, @f$h_{(t-1)}@f$ is previous or initial hidden state. + * + * @f$W_{x?}@f$, @f$W_{h?}@f$ and @f$b_{?}@f$ are learned weights represented as matrices: + * @f$W_{x?} \in R^{N_h \times N_x}@f$, @f$W_{h?} \in R^{N_h \times N_h}@f$, @f$b_? \in R^{N_h}@f$. + * + * @f$\odot@f$ is per-element multiply operation. + */ + class CV_EXPORTS GRULayer : public Layer + { + public: + /** Creates instance of GRU layer */ + static Ptr create(const LayerParams& params); + }; + /** @brief Classical recurrent layer Accepts two inputs @f$x_t@f$ and @f$h_{t-1}@f$ and compute two outputs @f$o_t@f$ and @f$h_t@f$. diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 698168817f5f..ebd887999b83 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -139,6 +139,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(FlowWarp, FlowWarpLayer); CV_DNN_REGISTER_LAYER_CLASS(LSTM, LSTMLayer); + CV_DNN_REGISTER_LAYER_CLASS(GRU, GRULayer); } CV__DNN_INLINE_NS_END diff --git a/modules/dnn/src/layers/recurrent_layers.cpp b/modules/dnn/src/layers/recurrent_layers.cpp index feae35dac01b..cb2ffb8cc99c 100644 --- a/modules/dnn/src/layers/recurrent_layers.cpp +++ b/modules/dnn/src/layers/recurrent_layers.cpp @@ -563,5 +563,214 @@ CV_EXPORTS_W Ptr RNNLayer::create(const LayerParams& params) return Ptr(new RNNLayerImpl(params)); } +class GRULayerImpl CV_FINAL : public GRULayer +{ + int numTimeStamps, numSamples; + bool allocated; + + MatShape outTailShape; //shape of single output sample + MatShape outTsShape; //shape of N output samples + bool bidirectional; // If true, produces both forward and reversed directions along time axis + +public: + + GRULayerImpl(const LayerParams& params) : numTimeStamps(0), numSamples(0) + { + setParamsFrom(params); + + bidirectional = params.get("bidirectional", false); + if (!blobs.empty()) + { + CV_Assert(blobs.size() >= 3); + + blobs[2] = blobs[2].reshape(1, 1); + + const Mat& Wh = blobs[0]; + const Mat& Wx = blobs[1]; + const Mat& bias = blobs[2]; + const Mat& hInternal = blobs[3]; + CV_CheckEQ(Wh.dims, 2, ""); + CV_CheckEQ(Wx.dims, 2, ""); + CV_CheckEQ(Wh.rows, Wx.rows, ""); + CV_CheckEQ(Wh.rows, (1 + static_cast(bidirectional)) * 3 * Wh.cols, ""); + CV_CheckEQ(Wh.rows * 2, (int)bias.total(), ""); + CV_CheckEQ(hInternal.cols, Wh.cols, ""); + CV_CheckTypeEQ(Wh.type(), Wx.type(), ""); + CV_CheckTypeEQ(Wx.type(), bias.type(), ""); + } + + allocated = false; + outTailShape.clear(); + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() == 1); + const MatShape& inp0 = inputs[0]; + + const Mat &Wh = blobs[0], &Wx = blobs[1]; + int _numOut = Wh.size[1]; + int _numInp = Wx.size[1]; + MatShape outTailShape_(outTailShape), outResShape; + + if (!outTailShape_.empty()) + CV_Assert(total(outTailShape_) == _numOut); + else + outTailShape_.assign(1, _numOut); + + int _numSamples; + CV_Assert(inp0.size() >= 2 && total(inp0, 2) == _numInp); + _numSamples = inp0[1]; + outResShape.push_back(inp0[0]); + + outResShape.push_back(_numSamples); + outResShape.insert(outResShape.end(), outTailShape_.begin(), outTailShape_.end()); + outResShape.back() *= (1 + static_cast(bidirectional)); + + outputs.assign(1, outResShape); + + internals.assign(1, shape(_numSamples, _numOut)); // hInternal + internals.push_back(shape(_numSamples, 1)); // dummyOnes + internals.push_back(shape(_numSamples, 2 * _numOut)); // gates + internals.push_back(shape(_numSamples, 2 * _numOut)); // gates_b + internals.push_back(shape(_numSamples, 1 * _numOut)); // h_linear + internals.push_back(shape(_numSamples, _numOut)); // ones + + return false; + } + + void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays) CV_OVERRIDE + { + std::vector input; + inputs_arr.getMatVector(input); + + CV_Assert(input.size() == 1); + const Mat& inp0 = input[0]; + + Mat &Wh = blobs[0], &Wx = blobs[1]; + int numOut = Wh.size[1]; + int numInp = Wx.size[1]; + + if (!outTailShape.empty()) + CV_Assert(total(outTailShape) == numOut); + else + outTailShape.assign(1, numOut); + + CV_Assert(inp0.dims >= 2 && (int)inp0.total(2) == numInp); + numTimeStamps = inp0.size[0]; + numSamples = inp0.size[1]; + + outTsShape.clear(); + outTsShape.push_back(numSamples); + outTsShape.insert(outTsShape.end(), outTailShape.begin(), outTailShape.end()); + outTsShape.back() *= (1 + static_cast(bidirectional)); + + allocated = true; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + if (inputs_arr.depth() == CV_16S) + { + forward_fallback(inputs_arr, outputs_arr, internals_arr); + return; + } + + std::vector input, output, internals; + inputs_arr.getMatVector(input); + outputs_arr.getMatVector(output); + internals_arr.getMatVector(internals); + + const int numDirs = 1 + static_cast(bidirectional); + for (int i = 0; i < numDirs; ++i) + { + const Mat &Wh = blobs[0].rowRange(i * blobs[0].rows / numDirs, (i + 1) * blobs[0].rows / numDirs); + const Mat &Wx = blobs[1].rowRange(i * blobs[1].rows / numDirs, (i + 1) * blobs[1].rows / numDirs); + const Mat &bias = blobs[2].colRange(i * blobs[2].cols / numDirs, (i + 1) * blobs[2].cols / numDirs); + const Mat &h_0 = blobs[3].rowRange(i * blobs[3].rows / numDirs, (i + 1) * blobs[3].rows / numDirs); + + const Mat &bx = bias.colRange(0, bias.cols / 2); + const Mat &bh = bias.colRange(bias.cols / 2, bias.cols); + + Mat hInternal = internals[0], dummyOnes = internals[1], gates = internals[2], + b_rz = internals[3], n_t = internals[4], ones = internals[5]; + h_0.copyTo(hInternal); + dummyOnes.setTo(1.); + ones.setTo(1.); + + int numOut = Wh.size[1]; + const Mat& wx_rz = Wx.rowRange(0, 2 * numOut); + const Mat& wh_rz = Wh.rowRange(0, 2 * numOut); + b_rz = bx.colRange(0, 2 * numOut) + bh.colRange(0, 2 * numOut); + const Mat& wx_n = Wx.rowRange(2 * numOut, 3 * numOut); + const Mat& wh_n = Wh.rowRange(2 * numOut, 3 * numOut); + const Mat& b_in = bx.colRange(2 * numOut, 3 * numOut); + const Mat& b_hn = bh.colRange(2 * numOut, 3 * numOut); + + int numSamplesTotal = numTimeStamps * numSamples; + Mat xTs = input[0].reshape(1, numSamplesTotal); + + Mat hOutTs = output[0].reshape(1, numSamplesTotal); + hOutTs = hOutTs.colRange(i * hOutTs.cols / numDirs, (i + 1) * hOutTs.cols / numDirs); + Mat cOutTs = Mat(); + + int tsStart, tsEnd, tsInc; + if (i == 1) { + tsStart = numTimeStamps - 1; + tsEnd = -1; + tsInc = -1; + } + else { + tsStart = 0; + tsEnd = numTimeStamps; + tsInc = 1; + } + for (int ts = tsStart; ts != tsEnd; ts += tsInc) + { + Range curRowRange(ts * numSamples, (ts + 1) * numSamples); + Mat xCurr = xTs.rowRange(curRowRange); + + // calculate r_t = sigmoid(x * Wx_r + h_(t-1) * Wh_r + b_r) + // calculate z_t = sigmoid(x * Wx_z + h_(t-1) * Wh_z + b_z) + gemm(xCurr, wx_rz, 1, gates, 0, gates, GEMM_2_T); // x * Wx_rz + gemm(hInternal, wh_rz, 1, gates, 1, gates, GEMM_2_T); // + h_(t-1) * Wh_rz + gemm(dummyOnes, b_rz, 1, gates, 1, gates); // + b_rz + sigmoid(gates, gates); // sigmoid() + + Mat z = gates.colRange(0, gates.cols / 2); + Mat r = gates.colRange(gates.cols / 2, gates.cols); + + // calculate n_t = tanh(r (*) (h_(t-1) * Wh_n + b_hn) + x * Wx_n + b_in) + gemm(hInternal, wh_n, 1, n_t, 0, n_t, GEMM_2_T); // h_(t-1) * Wh_n + gemm(dummyOnes, b_hn, 1, n_t, 1, n_t); // + b_hn + multiply(r, n_t, n_t); // r (*) (h_(t-1) * Wh_n + b_hn) + + gemm(xCurr, wx_n, 1, n_t, 1, n_t, GEMM_2_T); // + x * Wx_n + gemm(dummyOnes, b_in, 1, n_t, 1, n_t); // + b_in + tanh(n_t, n_t); // tanh() + + //compute next h_t = z (*) h_(t-1) + (1 - z) (*) n_t + multiply(z, hInternal, hInternal); // z (*) h_{t-1} + subtract(ones, z, z); // 1 - z + multiply(z, n_t, z); // (1 - z) * n + add(z, hInternal, hInternal); // z (*) h_(t-1) + (1 - z) (*) n_t + + //save results in output blobs + hInternal.copyTo(hOutTs.rowRange(curRowRange)); + } + } + } +}; + +Ptr GRULayer::create(const LayerParams ¶ms) { + return Ptr(new GRULayerImpl(params)); +} + } } diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 2573d783d84c..dee7e128fa02 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -145,6 +145,7 @@ class ONNXImporter void parseNeg (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseConstant (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseLSTM (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseGRU (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseImageScaler (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseClip (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseLeakyRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -582,6 +583,7 @@ const std::set& ONNXImporter::getSupportedTypes() "Neg", "Constant", "LSTM", + "GRU", "ImageScaler", "Clip", "LeakyRelu", @@ -1239,6 +1241,46 @@ void ONNXImporter::parseLSTM(LayerParams& layerParams, const opencv_onnx::NodePr addLayer(layerParams, node_proto); } +void ONNXImporter::parseGRU(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + LayerParams gruParams = layerParams; + gruParams.name += "/gru"; + + // https://pytorch.org/docs/stable/generated/torch.nn.GRU.html?highlight=gru# + CV_Assert(node_proto.input_size() == 6); + Mat Wx = getBlob(node_proto, 1); + Mat Wh = getBlob(node_proto, 2); + Mat b = getBlob(node_proto, 3); + Mat h0 = getBlob(node_proto, 5); + + Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); + Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + b = b.reshape(1, b.size[0]); + + gruParams.blobs.resize(4); + gruParams.blobs[0] = Wh; + gruParams.blobs[1] = Wx; + gruParams.blobs[2] = b; + gruParams.blobs[3] = h0; + gruParams.set("bidirectional", gruParams.get("direction", "") == "bidirectional"); + + node_proto.set_output(0, gruParams.name); // set different name so output shapes will be registered on that name + addLayer(gruParams, node_proto); + + MatShape gruShape = outShapes[node_proto.output(0)]; + + // Add fake 1 as it is done in ONNX + gruShape.insert(gruShape.begin() + 1, 1); + + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&gruShape[0], gruShape.size())); + node_proto.set_input(0, gruParams.name); // redirect input to GRU + node_proto.set_output(0, layerParams.name); // keep origin GRU's name + addLayer(layerParams, node_proto); +} + void ONNXImporter::parseImageScaler(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { const float scale = layerParams.has("scale") ? layerParams.get("scale") : 1.0f; @@ -2358,6 +2400,7 @@ const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() dispatch["Neg"] = &ONNXImporter::parseNeg; dispatch["Constant"] = &ONNXImporter::parseConstant; dispatch["LSTM"] = &ONNXImporter::parseLSTM; + dispatch["GRU"] = &ONNXImporter::parseGRU; dispatch["ImageScaler"] = &ONNXImporter::parseImageScaler; dispatch["Clip"] = &ONNXImporter::parseClip; dispatch["LeakyRelu"] = &ONNXImporter::parseLeakyRelu; diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp index 1383c59e28d7..04d5fa63559e 100644 --- a/modules/dnn/test/test_layers.cpp +++ b/modules/dnn/test/test_layers.cpp @@ -596,6 +596,35 @@ TEST(Layer_LSTM_Test_Accuracy_with_, HiddenParams) normAssert(h_t_reference, outputs[0]); } +TEST(Layer_GRU_Test_Accuracy_with_, Pytorch) +{ + Mat Wx = blobFromNPY(_tf("gru.W.npy")); + Mat Wh = blobFromNPY(_tf("gru.R.npy")); + Mat b = blobFromNPY(_tf("gru.B.npy")); + Mat h0 = blobFromNPY(_tf("gru.h0.npy")); + + Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); + Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + b = b.reshape(1, b.size[0]); + + LayerParams gruParams; + gruParams.blobs.resize(4); + gruParams.blobs[0] = Wh; + gruParams.blobs[1] = Wx; + gruParams.blobs[2] = b; + gruParams.blobs[3] = h0; + gruParams.set("bidirectional", false); + Ptr layer = GRULayer::create(gruParams); + + Mat inp = blobFromNPY(_tf("gru.input.npy")); + std::vector inputs(1, inp), outputs; + runLayer(layer, inputs, outputs); + + Mat h_t_reference = blobFromNPY(_tf("gru.output.npy")); + normAssert(h_t_reference, outputs[0]); +} + TEST(Layer_RNN_Test_Accuracy_with_, CaffeRecurrent) { Ptr layer = RNNLayer::create(LayerParams()); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 8bfd86495508..0e252cca8312 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -720,6 +720,16 @@ TEST_P(Test_ONNX_layers, LSTM_hidden_bidirectional) testONNXModels("hidden_lstm_bi", npy, 0, 0, false, false); } +TEST_P(Test_ONNX_layers, GRU) +{ + testONNXModels("gru", npy, 0, 0, false, false); +} + +TEST_P(Test_ONNX_layers, GRU_bidirectional) +{ + testONNXModels("gru_bi", npy, 0, 0, false, false); +} + TEST_P(Test_ONNX_layers, Pad2d_Unfused) { testONNXModels("ReflectionPad2d"); From 2a177052de55c85554194a2464a91e6e09c7f768 Mon Sep 17 00:00:00 2001 From: SamFC10 Date: Mon, 9 Aug 2021 12:08:55 +0530 Subject: [PATCH 102/376] fix bug in prior-box variances --- .../dnn/src/layers/detection_output_layer.cpp | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/modules/dnn/src/layers/detection_output_layer.cpp b/modules/dnn/src/layers/detection_output_layer.cpp index 8374d74293be..614b3a646266 100644 --- a/modules/dnn/src/layers/detection_output_layer.cpp +++ b/modules/dnn/src/layers/detection_output_layer.cpp @@ -456,7 +456,7 @@ class DetectionOutputLayerImpl CV_FINAL : public DetectionOutputLayer // Retrieve all prior bboxes std::vector priorBBoxes; std::vector > priorVariances; - GetPriorBBoxes(priorData, numPriors, _bboxesNormalized, priorBBoxes, priorVariances); + GetPriorBBoxes(priorData, numPriors, _bboxesNormalized, _varianceEncodedInTarget, priorBBoxes, priorVariances); // Decode all loc predictions to bboxes util::NormalizedBBox clipBounds; @@ -750,7 +750,7 @@ class DetectionOutputLayerImpl CV_FINAL : public DetectionOutputLayer CV_Assert(prior_bboxes.size() == prior_variances.size()); CV_Assert(prior_bboxes.size() == bboxes.size()); size_t num_bboxes = prior_bboxes.size(); - CV_Assert(num_bboxes == 0 || prior_variances[0].size() == 4); + CV_Assert(num_bboxes == 0 || prior_variances[0].size() == 4 || variance_encoded_in_target); decode_bboxes.clear(); decode_bboxes.resize(num_bboxes); if(variance_encoded_in_target) { @@ -802,12 +802,13 @@ class DetectionOutputLayerImpl CV_FINAL : public DetectionOutputLayer } // Get prior bounding boxes from prior_data - // prior_data: 1 x 2 x num_priors * 4 x 1 blob. + // prior_data: 1 x 1 x num_priors * 4 x 1 blob or 1 x 2 x num_priors * 4 x 1 blob. // num_priors: number of priors. // prior_bboxes: stores all the prior bboxes in the format of util::NormalizedBBox. // prior_variances: stores all the variances needed by prior bboxes. static void GetPriorBBoxes(const float* priorData, const int& numPriors, - bool normalized_bbox, std::vector& priorBBoxes, + bool normalized_bbox, bool variance_encoded_in_target, + std::vector& priorBBoxes, std::vector >& priorVariances) { priorBBoxes.clear(); priorBBoxes.resize(numPriors); @@ -823,13 +824,16 @@ class DetectionOutputLayerImpl CV_FINAL : public DetectionOutputLayer bbox.set_size(BBoxSize(bbox, normalized_bbox)); } - for (int i = 0; i < numPriors; ++i) + if (!variance_encoded_in_target) { - int startIdx = (numPriors + i) * 4; - // not needed here: priorVariances[i].clear(); - for (int j = 0; j < 4; ++j) + for (int i = 0; i < numPriors; ++i) { - priorVariances[i].push_back(priorData[startIdx + j]); + int startIdx = (numPriors + i) * 4; + // not needed here: priorVariances[i].clear(); + for (int j = 0; j < 4; ++j) + { + priorVariances[i].push_back(priorData[startIdx + j]); + } } } } From 739ff84732f6385a43b32b37d875921b9073d009 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Mon, 9 Aug 2021 13:28:33 +0300 Subject: [PATCH 103/376] add Max layer to TFImporter --- modules/dnn/src/tensorflow/tf_importer.cpp | 26 +++++++++++++++++----- modules/dnn/test/test_tf_importer.cpp | 22 ++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index fa33211a50e1..ca9d7c5e2174 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -647,7 +647,7 @@ const TFImporter::DispatchMap TFImporter::buildDispatchMap() dispatch["PriorBox"] = &TFImporter::parsePriorBox; dispatch["Softmax"] = &TFImporter::parseSoftmax; dispatch["CropAndResize"] = &TFImporter::parseCropAndResize; - dispatch["Mean"] = dispatch["Sum"] = &TFImporter::parseMean; + dispatch["Mean"] = dispatch["Sum"] = dispatch["Max"] = &TFImporter::parseMean; dispatch["Pack"] = &TFImporter::parsePack; dispatch["ClipByValue"] = &TFImporter::parseClipByValue; dispatch["LeakyRelu"] = &TFImporter::parseLeakyRelu; @@ -657,6 +657,7 @@ const TFImporter::DispatchMap TFImporter::buildDispatchMap() return dispatch; } +// "Conv2D" "SpaceToBatchND" "DepthwiseConv2dNative" "Pad" "MirrorPad" "Conv3D" void TFImporter::parseConvolution(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer_, LayerParams& layerParams) { tensorflow::NodeDef layer = layer_; @@ -876,6 +877,7 @@ void TFImporter::parseConvolution(tensorflow::GraphDef& net, const tensorflow::N data_layouts[name] = DATA_LAYOUT_NHWC; } +// "BiasAdd" "Add" "AddV2" "Sub" "AddN" void TFImporter::parseBias(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); @@ -1087,6 +1089,7 @@ void TFImporter::parseReshape(tensorflow::GraphDef& net, const tensorflow::NodeD } } +// "Flatten" "Squeeze" void TFImporter::parseFlatten(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); @@ -1245,6 +1248,7 @@ void TFImporter::parseLrn(tensorflow::GraphDef& net, const tensorflow::NodeDef& connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); } +// "Concat" "ConcatV2" void TFImporter::parseConcat(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); @@ -1295,6 +1299,7 @@ void TFImporter::parseConcat(tensorflow::GraphDef& net, const tensorflow::NodeDe } } +// "MaxPool" "MaxPool3D" void TFImporter::parseMaxPool(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); @@ -1316,6 +1321,7 @@ void TFImporter::parseMaxPool(tensorflow::GraphDef& net, const tensorflow::NodeD connectToAllBlobs(layer_id, dstNet, parsePin(inputName), id, num_inputs); } +// "AvgPool" "AvgPool3D" void TFImporter::parseAvgPool(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); @@ -1502,6 +1508,7 @@ void TFImporter::parseStridedSlice(tensorflow::GraphDef& net, const tensorflow:: connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); } +// "Mul" "RealDiv" void TFImporter::parseMul(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); @@ -1659,6 +1666,7 @@ void TFImporter::parseMul(tensorflow::GraphDef& net, const tensorflow::NodeDef& } } +// "FusedBatchNorm" "FusedBatchNormV3" void TFImporter::parseFusedBatchNorm(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { // op: "FusedBatchNorm" @@ -1918,6 +1926,7 @@ void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::Nod data_layouts[name] = DATA_LAYOUT_UNKNOWN; } +// "ResizeNearestNeighbor" "ResizeBilinear" "FusedResizeAndPadConv2D" void TFImporter::parseResize(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer_, LayerParams& layerParams) { tensorflow::NodeDef layer = layer_; @@ -2106,6 +2115,7 @@ void TFImporter::parseCropAndResize(tensorflow::GraphDef& net, const tensorflow: connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); } +// "Mean" "Sum" "Max" void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { // Computes the mean of elements across dimensions of a tensor. @@ -2124,7 +2134,12 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& const std::string& name = layer.name(); const std::string& type = layer.op(); const int num_inputs = layer.input_size(); + std::string pool_type = cv::toLowerCase(type); + if (pool_type == "mean") + { + pool_type = "ave"; + } CV_CheckGT(num_inputs, 0, ""); Mat indices = getTensorContent(getConstBlob(layer, value_id, 1)); @@ -2161,7 +2176,7 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& LayerParams avgLp; std::string avgName = name + "/avg"; CV_Assert(layer_id.find(avgName) == layer_id.end()); - avgLp.set("pool", type == "Mean" ? "ave" : "sum"); + avgLp.set("pool", pool_type); // pooling kernel H x 1 avgLp.set("global_pooling_h", true); avgLp.set("kernel_w", 1); @@ -2202,7 +2217,7 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& int axis = toNCHW(indices.at(0)); if (axis == 2 || axis == 3) { - layerParams.set("pool", type == "Mean" ? "ave" : "sum"); + layerParams.set("pool", pool_type); layerParams.set(axis == 2 ? "kernel_w" : "kernel_h", 1); layerParams.set(axis == 2 ? "global_pooling_h" : "global_pooling_w", true); int id = dstNet.addLayer(name, "Pooling", layerParams); @@ -2234,7 +2249,7 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& Pin inpId = parsePin(layer.input(0)); addPermuteLayer(order, name + "/nhwc", inpId); - layerParams.set("pool", type == "Mean" ? "ave" : "sum"); + layerParams.set("pool", pool_type); layerParams.set("kernel_h", 1); layerParams.set("global_pooling_w", true); int id = dstNet.addLayer(name, "Pooling", layerParams); @@ -2264,7 +2279,7 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& if (indices.total() != 2 || indices.at(0) != 1 || indices.at(1) != 2) CV_Error(Error::StsNotImplemented, "Unsupported mode of reduce_mean or reduce_sum operation."); - layerParams.set("pool", type == "Mean" ? "ave" : "sum"); + layerParams.set("pool", pool_type); layerParams.set("global_pooling", true); int id = dstNet.addLayer(name, "Pooling", layerParams); layer_id[name] = id; @@ -2368,6 +2383,7 @@ void TFImporter::parseLeakyRelu(tensorflow::GraphDef& net, const tensorflow::Nod connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); } +// "Abs" "Tanh" "Sigmoid" "Relu" "Elu" "Exp" "Identity" "Relu6" void TFImporter::parseActivation(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 4f7840f9e4eb..68d6e88a6642 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -128,6 +128,13 @@ TEST_P(Test_TensorFlow_layers, reduce_mean) runTensorFlowNet("global_pool_by_axis"); } +TEST_P(Test_TensorFlow_layers, reduce_max) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + runTensorFlowNet("max_pool_by_axis"); +} + TEST_P(Test_TensorFlow_layers, reduce_sum) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) @@ -135,11 +142,21 @@ TEST_P(Test_TensorFlow_layers, reduce_sum) runTensorFlowNet("sum_pool_by_axis"); } +TEST_P(Test_TensorFlow_layers, reduce_max_channel) +{ + runTensorFlowNet("reduce_max_channel"); +} + TEST_P(Test_TensorFlow_layers, reduce_sum_channel) { runTensorFlowNet("reduce_sum_channel"); } +TEST_P(Test_TensorFlow_layers, reduce_max_channel_keep_dims) +{ + runTensorFlowNet("reduce_max_channel", false, 0.0, 0.0, false, "_keep_dims"); +} + TEST_P(Test_TensorFlow_layers, reduce_sum_channel_keep_dims) { runTensorFlowNet("reduce_sum_channel", false, 0.0, 0.0, false, "_keep_dims"); @@ -386,6 +403,11 @@ TEST_P(Test_TensorFlow_layers, pooling_reduce_mean) runTensorFlowNet("reduce_mean"); // an average pooling over all spatial dimensions. } +TEST_P(Test_TensorFlow_layers, pooling_reduce_max) +{ + runTensorFlowNet("reduce_max"); // a MAX pooling over all spatial dimensions. +} + TEST_P(Test_TensorFlow_layers, pooling_reduce_sum) { runTensorFlowNet("reduce_sum"); // a SUM pooling over all spatial dimensions. From 21d0f4075141568f7bf6616c5f8e79c06987283e Mon Sep 17 00:00:00 2001 From: Saikat Nanda Date: Mon, 9 Aug 2021 21:41:00 -0400 Subject: [PATCH 104/376] Fix YUV indexes + YUV Planner detection condition --- modules/videoio/src/cap_android_camera.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/videoio/src/cap_android_camera.cpp b/modules/videoio/src/cap_android_camera.cpp index b369a12a6861..5952b6f08c48 100644 --- a/modules/videoio/src/cap_android_camera.cpp +++ b/modules/videoio/src/cap_android_camera.cpp @@ -304,8 +304,8 @@ class AndroidCameraCapture : public IVideoCapture AImage_getPlaneRowStride(image.get(), 0, &yStride); AImage_getPlaneRowStride(image.get(), 1, &uvStride); AImage_getPlaneData(image.get(), 0, &yPixel, &yLen); - AImage_getPlaneData(image.get(), 1, &vPixel, &vLen); - AImage_getPlaneData(image.get(), 2, &uPixel, &uLen); + AImage_getPlaneData(image.get(), 1, &uPixel, &uLen); + AImage_getPlaneData(image.get(), 2, &vPixel, &vLen); AImage_getPlanePixelStride(image.get(), 1, &uvPixelStride); if ( (uvPixelStride == 2) && (vPixel == uPixel + 1) && (yLen == frameWidth * frameHeight) && (uLen == ((yLen / 2) - 1)) && (vLen == uLen) ) { @@ -313,7 +313,7 @@ class AndroidCameraCapture : public IVideoCapture if (fourCC == FOURCC_UNKNOWN) { fourCC = FOURCC_NV21; } - } else if ( (uvPixelStride == 1) && (vPixel = uPixel + uLen) && (yLen == frameWidth * frameHeight) && (uLen == yLen / 4) && (vLen == uLen) ) { + } else if ( (uvPixelStride == 1) && (vPixel == uPixel + uLen) && (yLen == frameWidth * frameHeight) && (uLen == yLen / 4) && (vLen == uLen) ) { colorFormat = COLOR_FormatYUV420Planar; if (fourCC == FOURCC_UNKNOWN) { fourCC = FOURCC_YV12; From 992b47b9916f9dbdfee16ed1a59ba64cda0779bb Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Tue, 10 Aug 2021 18:53:28 +0300 Subject: [PATCH 105/376] add 19769 and 19769_lightweight tests --- modules/imgproc/test/test_convhull.cpp | 73 ++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/modules/imgproc/test/test_convhull.cpp b/modules/imgproc/test/test_convhull.cpp index f1d739b0e059..dee3769762c9 100644 --- a/modules/imgproc/test/test_convhull.cpp +++ b/modules/imgproc/test/test_convhull.cpp @@ -2384,5 +2384,78 @@ TEST(Imgproc_minAreaRect, reproducer_18157) EXPECT_TRUE(checkMinAreaRect(rr, contour)) << rr.center << " " << rr.size << " " << rr.angle; } +TEST(Imgproc_minAreaRect, reproducer_19769_lightweight) +{ + const int N = 23; + float pts_[N][2] = { + {1325, 732}, {1248, 808}, {582, 1510}, {586, 1524}, + {595, 1541}, {599, 1547}, {789, 1745}, {829, 1786}, + {997, 1958}, {1116, 2074}, {1207, 2066}, {1216, 2058}, + {1231, 2044}, {1265, 2011}, {2036, 1254}, {2100, 1191}, + {2169, 1123}, {2315, 979}, {2395, 900}, {2438, 787}, + {2434, 782}, {2416, 762}, {2266, 610} + }; + Mat contour(N, 1, CV_32FC2, (void*)pts_); + + RotatedRect rr = cv::minAreaRect(contour); + + EXPECT_TRUE(checkMinAreaRect(rr, contour)) << rr.center << " " << rr.size << " " << rr.angle; +} + +TEST(Imgproc_minAreaRect, reproducer_19769) +{ + const int N = 169; + float pts_[N][2] = { + {1854, 227}, {1850, 228}, {1847, 229}, {1835, 235}, + {1832, 237}, {1829, 239}, {1825, 242}, {1818, 248}, + {1807, 258}, {1759, 306}, {1712, 351}, {1708, 356}, + {1658, 404}, {1655, 408}, {1602, 459}, {1599, 463}, + {1542, 518}, {1477, 582}, {1402, 656}, {1325, 732}, + {1248, 808}, {1161, 894}, {1157, 898}, {1155, 900}, + {1068, 986}, {1060, 995}, {1058, 997}, {957, 1097}, + {956, 1097}, {814, 1238}, {810, 1242}, {805, 1248}, + {610, 1442}, {603, 1450}, {599, 1455}, {596, 1459}, + {594, 1462}, {592, 1465}, {590, 1470}, {588, 1472}, + {586, 1476}, {586, 1478}, {584, 1481}, {583, 1485}, + {582, 1490}, {582, 1510}, {583, 1515}, {584, 1518}, + {585, 1521}, {586, 1524}, {593, 1538}, {595, 1541}, + {597, 1544}, {599, 1547}, {603, 1552}, {609, 1559}, + {623, 1574}, {645, 1597}, {677, 1630}, {713, 1667}, + {753, 1707}, {789, 1744}, {789, 1745}, {829, 1786}, + {871, 1828}, {909, 1867}, {909, 1868}, {950, 1910}, + {953, 1912}, {997, 1958}, {1047, 2009}, {1094, 2056}, + {1105, 2066}, {1110, 2070}, {1113, 2072}, {1116, 2074}, + {1119, 2076}, {1122, 2077}, {1124, 2079}, {1130, 2082}, + {1133, 2083}, {1136, 2084}, {1139, 2085}, {1142, 2086}, + {1148, 2087}, {1166, 2087}, {1170, 2086}, {1174, 2085}, + {1177, 2084}, {1180, 2083}, {1188, 2079}, {1190, 2077}, + {1193, 2076}, {1196, 2074}, {1199, 2072}, {1202, 2070}, + {1207, 2066}, {1216, 2058}, {1231, 2044}, {1265, 2011}, + {1314, 1962}, {1360, 1917}, {1361, 1917}, {1408, 1871}, + {1457, 1822}, {1508, 1773}, {1512, 1768}, {1560, 1722}, + {1617, 1665}, {1671, 1613}, {1730, 1554}, {1784, 1502}, + {1786, 1500}, {1787, 1498}, {1846, 1440}, {1850, 1437}, + {1908, 1380}, {1974, 1314}, {2034, 1256}, {2036, 1254}, + {2100, 1191}, {2169, 1123}, {2242, 1051}, {2315, 979}, + {2395, 900}, {2426, 869}, {2435, 859}, {2438, 855}, + {2440, 852}, {2442, 849}, {2443, 846}, {2445, 844}, + {2446, 842}, {2446, 840}, {2448, 837}, {2449, 834}, + {2450, 829}, {2450, 814}, {2449, 809}, {2448, 806}, + {2447, 803}, {2442, 793}, {2440, 790}, {2438, 787}, + {2434, 782}, {2428, 775}, {2416, 762}, {2411, 758}, + {2342, 688}, {2340, 686}, {2338, 684}, {2266, 610}, + {2260, 605}, {2170, 513}, {2075, 417}, {2073, 415}, + {2069, 412}, {1955, 297}, {1955, 296}, {1913, 254}, + {1904, 246}, {1897, 240}, {1894, 238}, {1891, 236}, + {1888, 234}, {1880, 230}, {1877, 229}, {1874, 228}, + {1870, 227} + }; + Mat contour(N, 1, CV_32FC2, (void*)pts_); + + RotatedRect rr = cv::minAreaRect(contour); + + EXPECT_TRUE(checkMinAreaRect(rr, contour)) << rr.center << " " << rr.size << " " << rr.angle; +} + }} // namespace /* End of file. */ From aaca4987c9ffe12e9e486d40ae264859caed89df Mon Sep 17 00:00:00 2001 From: HAN Liutong Date: Wed, 11 Aug 2021 06:16:03 +0800 Subject: [PATCH 106/376] Merge pull request #20287 from hanliutong:dev-rvv-0.10 Optimization of DNN using native RISC-V vector intrinsics. * Use RVV to optimize fastGEMM (FP32) in DNN. * Use RVV to optimize fastGEMM1T in DNN. * Use RVV to optimize fastConv in DNN. * Use RVV to optimize fastDepthwiseConv in DNN. * Vectorize tails using vl. * Use "vl" instead of scalar to handle small block in fastConv. * Fix memory access out of bound in "fastGEMM1T". * Remove setvl. * Remove useless initialization. * Use loop unrolling to handle tail part instead of switch. --- modules/dnn/CMakeLists.txt | 2 +- modules/dnn/src/layers/convolution_layer.cpp | 25 +- .../dnn/src/layers/fully_connected_layer.cpp | 9 +- modules/dnn/src/layers/layers_common.simd.hpp | 549 ++++++++++++++++++ 4 files changed, 582 insertions(+), 3 deletions(-) diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 4c8129cbda1c..70f9a5a73e5a 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -8,7 +8,7 @@ endif() set(the_description "Deep neural network module. It allows to load models from different frameworks and to make forward pass") -ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX) +ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX RVV) ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java objc js) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index fb57f265111d..68c543be2477 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -914,11 +914,12 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl bool useAVX; bool useAVX2; bool useAVX512; + bool useRVV; int blk_size_cn; ParallelConv() : input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0), - biasvec_(0), reluslope_(0), activ_(0), is1x1_(false), useAVX(false), useAVX2(false), useAVX512(false) + biasvec_(0), reluslope_(0), activ_(0), is1x1_(false), useAVX(false), useAVX2(false), useAVX512(false), useRVV(false) , blk_size_cn(0) {} @@ -976,6 +977,7 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl p.useAVX = checkHardwareSupport(CPU_AVX) && isConv2D; p.useAVX2 = checkHardwareSupport(CPU_AVX2) && isConv2D; p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX && isConv2D; + p.useRVV = checkHardwareSupport(CPU_RVV) && isConv2D; int kernel_d = isConv3D? kernel_size[0] : 1; int kernel_h = isConv1D? 1 : kernel_size[kernel_size.size() - 2]; @@ -1176,6 +1178,13 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl stride_h, stride_w, dilation_h, dilation_w, pad_t, pad_l, biasptr, relu, inptr_, height, width, outptr_, out_d, outH, outW); else + #endif + #if CV_TRY_RVV + if(useRVV) + opt_RVV::fastDepthwiseConv(wptr, kernel_h, kernel_w, + stride_h, stride_w, dilation_h, dilation_w, pad_t, pad_l, + biasptr, relu, inptr_, height, width, outptr_, out_d, outH, outW); + else #endif { const float w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2], @@ -1546,6 +1555,12 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl opt_AVX::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0, outShape, bsz, vsz, vsz_a, relu, cn0 == 0); else + #endif + #if CV_TRY_RVV + if(useRVV) + opt_RVV::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0, + outShape, bsz, vsz, vsz_a, relu, cn0 == 0); + else #endif for( int i = 0; i < outCn; i += 2 ) { @@ -2297,6 +2312,7 @@ class DeConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl useAVX = checkHardwareSupport(CPU_AVX); useAVX2 = checkHardwareSupport(CPU_AVX2); useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX; + useRVV = checkHardwareSupport(CPU_RVV); } void operator()(const Range& range_) const CV_OVERRIDE @@ -2328,6 +2344,12 @@ class DeConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl if( useAVX ) opt_AVX::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax ); else + #endif + #if CV_TRY_RVV + if( useRVV ) { + opt_RVV::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax ); + } + else #endif for( m = 0; m < mmax; m += 2 ) { @@ -2427,6 +2449,7 @@ class DeConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl bool useAVX; bool useAVX2; bool useAVX512; + bool useRVV; }; class Col2ImInvoker : public cv::ParallelLoopBody diff --git a/modules/dnn/src/layers/fully_connected_layer.cpp b/modules/dnn/src/layers/fully_connected_layer.cpp index d9c1fa65c143..529f3c04fdef 100644 --- a/modules/dnn/src/layers/fully_connected_layer.cpp +++ b/modules/dnn/src/layers/fully_connected_layer.cpp @@ -168,7 +168,7 @@ class FullyConnectedLayerImpl CV_FINAL : public InnerProductLayer class FullyConnected : public ParallelLoopBody { public: - FullyConnected() : srcMat(0), weights(0), biasMat(0), activ(0), dstMat(0), nstripes(0), useAVX(false), useAVX2(false), useAVX512(false) {} + FullyConnected() : srcMat(0), weights(0), biasMat(0), activ(0), dstMat(0), nstripes(0), useAVX(false), useAVX2(false), useAVX512(false), useRVV(false) {} static void run(const Mat& srcMat, const Mat& weights, const Mat& biasMat, Mat& dstMat, const ActivationLayer* activ, int nstripes) @@ -191,6 +191,7 @@ class FullyConnectedLayerImpl CV_FINAL : public InnerProductLayer p.useAVX = checkHardwareSupport(CPU_AVX); p.useAVX2 = checkHardwareSupport(CPU_AVX2); p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX; + p.useRVV = checkHardwareSupport(CPU_RVV); parallel_for_(Range(0, nstripes), p, nstripes); } @@ -239,6 +240,11 @@ class FullyConnectedLayerImpl CV_FINAL : public InnerProductLayer if( useAVX ) opt_AVX::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize); else + #endif + #if CV_TRY_RVV + if( useRVV ) + opt_RVV::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize); + else #endif { int i = 0; @@ -293,6 +299,7 @@ class FullyConnectedLayerImpl CV_FINAL : public InnerProductLayer bool useAVX; bool useAVX2; bool useAVX512; + bool useRVV; }; #ifdef HAVE_OPENCL diff --git a/modules/dnn/src/layers/layers_common.simd.hpp b/modules/dnn/src/layers/layers_common.simd.hpp index 706695a7b20f..762e22e54d2f 100644 --- a/modules/dnn/src/layers/layers_common.simd.hpp +++ b/modules/dnn/src/layers/layers_common.simd.hpp @@ -737,5 +737,554 @@ void fastGEMM( const float* aptr, size_t astep, const float* bptr, #endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY +#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_RVV + +void fastGEMM( const float* aptr, size_t astep, const float* bptr, + size_t bstep, float* cptr, size_t cstep, + int ma, int na, int nb ) +{ + int n = 0; + size_t vl = 8; + size_t mvl0 = 8; + size_t mvl1 = 8; + for( ; n < nb; n += 16 ) + { + if ( n + 16 > nb) { + mvl0 = nb - n; + mvl1 = (nb - n -8) > 0 ? (nb - n -8) : 0; + } + + for( int m = 0; m < ma; m += 4 ) + { + const float* aptr0 = aptr + astep*m; + const float* aptr1 = aptr + astep*std::min(m+1, ma-1); + const float* aptr2 = aptr + astep*std::min(m+2, ma-1); + const float* aptr3 = aptr + astep*std::min(m+3, ma-1); + + float* cptr0 = cptr + cstep*m; + float* cptr1 = cptr + cstep*std::min(m+1, ma-1); + float* cptr2 = cptr + cstep*std::min(m+2, ma-1); + float* cptr3 = cptr + cstep*std::min(m+3, ma-1); + + vfloat32m2_t d00 = vfmv_v_f_f32m2(0, vl), d01 = vfmv_v_f_f32m2(0, vl); + vfloat32m2_t d10 = vfmv_v_f_f32m2(0, vl), d11 = vfmv_v_f_f32m2(0, vl); + vfloat32m2_t d20 = vfmv_v_f_f32m2(0, vl), d21 = vfmv_v_f_f32m2(0, vl); + vfloat32m2_t d30 = vfmv_v_f_f32m2(0, vl), d31 = vfmv_v_f_f32m2(0, vl); + + for( int k = 0; k < na; k++ ) + { + vfloat32m2_t a0 = vfmv_v_f_f32m2(aptr0[k], vl); + vfloat32m2_t a1 = vfmv_v_f_f32m2(aptr1[k], vl); + vfloat32m2_t a2 = vfmv_v_f_f32m2(aptr2[k], vl); + vfloat32m2_t a3 = vfmv_v_f_f32m2(aptr3[k], vl); + vfloat32m2_t b0 = vle32_v_f32m2(bptr + k*bstep + n, mvl0); + vfloat32m2_t b1 = vle32_v_f32m2(bptr + k*bstep + n + 8, mvl1); + d00 = vfmacc_vv_f32m2(d00, a0, b0, mvl0); + d01 = vfmacc_vv_f32m2(d01, a0, b1, mvl1); + d10 = vfmacc_vv_f32m2(d10, a1, b0, mvl0); + d11 = vfmacc_vv_f32m2(d11, a1, b1, mvl1); + d20 = vfmacc_vv_f32m2(d20, a2, b0, mvl0); + d21 = vfmacc_vv_f32m2(d21, a2, b1, mvl1); + d30 = vfmacc_vv_f32m2(d30, a3, b0, mvl0); + d31 = vfmacc_vv_f32m2(d31, a3, b1, mvl1); + } + vse32_v_f32m2(cptr0 + n, d00, mvl0); + vse32_v_f32m2(cptr1 + n, d10, mvl0); + vse32_v_f32m2(cptr2 + n, d20, mvl0); + vse32_v_f32m2(cptr3 + n, d30, mvl0); + vse32_v_f32m2(cptr0 + n + 8, d01, mvl1); + vse32_v_f32m2(cptr1 + n + 8, d11, mvl1); + vse32_v_f32m2(cptr2 + n + 8, d21, mvl1); + vse32_v_f32m2(cptr3 + n + 8, d31, mvl1); + } + } +} + +void fastGEMM1T( const float* vec, const float* weights, + size_t wstep, const float* bias, + float* dst, int nvecs, int vecsize ) +{ + int i = 0; + size_t vl = 8; + for( ; i <= nvecs - 8; i += 8 ) + { + const float* wptr = weights + i*wstep; + vfloat32m2_t vs0 = vfmv_v_f_f32m2(0, vl), vs1 = vfmv_v_f_f32m2(0, vl), + vs2 = vfmv_v_f_f32m2(0, vl), vs3 = vfmv_v_f_f32m2(0, vl), + vs4 = vfmv_v_f_f32m2(0, vl), vs5 = vfmv_v_f_f32m2(0, vl), + vs6 = vfmv_v_f_f32m2(0, vl), vs7 = vfmv_v_f_f32m2(0, vl); + + for( int k = 0; k < vecsize; k += 8, wptr += 8 ) + { + vfloat32m2_t v = vle32_v_f32m2(vec + k, vl); + + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vl), v, vl); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep, vl), v, vl); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*2, vl), v, vl); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*3, vl), v, vl); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*4, vl), v, vl); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*5, vl), v, vl); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*6, vl), v, vl); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*7, vl), v, vl); + } + + // Calculate the sum of each vector + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vl); + vfloat32m1_t temp0 = vfredsum_vs_f32m2_f32m1(temp0, vs0, zero, vl); + vfloat32m1_t temp1 = vfredsum_vs_f32m2_f32m1(temp1, vs1, zero, vl); + vfloat32m1_t temp2 = vfredsum_vs_f32m2_f32m1(temp2, vs2, zero, vl); + vfloat32m1_t temp3 = vfredsum_vs_f32m2_f32m1(temp3, vs3, zero, vl); + vfloat32m1_t temp4 = vfredsum_vs_f32m2_f32m1(temp4, vs4, zero, vl); + vfloat32m1_t temp5 = vfredsum_vs_f32m2_f32m1(temp5, vs5, zero, vl); + vfloat32m1_t temp6 = vfredsum_vs_f32m2_f32m1(temp6, vs6, zero, vl); + vfloat32m1_t temp7 = vfredsum_vs_f32m2_f32m1(temp7, vs7, zero, vl); + float32_t sum[8]; + sum[0] = vfmv_f_s_f32m1_f32(temp0); + sum[1] = vfmv_f_s_f32m1_f32(temp1); + sum[2] = vfmv_f_s_f32m1_f32(temp2); + sum[3] = vfmv_f_s_f32m1_f32(temp3); + sum[4] = vfmv_f_s_f32m1_f32(temp4); + sum[5] = vfmv_f_s_f32m1_f32(temp5); + sum[6] = vfmv_f_s_f32m1_f32(temp6); + sum[7] = vfmv_f_s_f32m1_f32(temp7); + vfloat32m2_t s0 = vfadd_vv_f32m2(vle32_v_f32m2(sum, vl), vle32_v_f32m2(bias + i, vl), vl); + vse32_v_f32m2(dst + i, s0, vl); + } + int mvl = nvecs - i; + if (mvl > 0) + { + const float* wptr = weights + i*wstep; + vfloat32m2_t vs0 = vfmv_v_f_f32m2(0, vl), vs1 = vfmv_v_f_f32m2(0, vl), + vs2 = vfmv_v_f_f32m2(0, vl), vs3 = vfmv_v_f_f32m2(0, vl), + vs4 = vfmv_v_f_f32m2(0, vl), vs5 = vfmv_v_f_f32m2(0, vl), + vs6 = vfmv_v_f_f32m2(0, vl), vs7 = vfmv_v_f_f32m2(0, vl); + int k = 0; + for( ; k <= vecsize - 8; k += 8, wptr += 8 ) + { + vfloat32m2_t v = vle32_v_f32m2(vec + k, vl); + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vl), v, vl); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*std::min(1, mvl-1), vl), v, vl); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*std::min(2, mvl-1), vl), v, vl); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*std::min(3, mvl-1), vl), v, vl); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), vl), v, vl); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), vl), v, vl); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), vl), v, vl); + } + int kvl = vecsize - k; + if (kvl > 0) { + vfloat32m2_t v = vle32_v_f32m2(vec + k, kvl); + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, kvl), v, kvl); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*std::min(1, mvl-1), kvl), v, kvl); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*std::min(2, mvl-1), kvl), v, kvl); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*std::min(3, mvl-1), kvl), v, kvl); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), kvl), v, kvl); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), kvl), v, kvl); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), kvl), v, kvl); + } + // Calculate the sum of each vector + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vl); + vfloat32m1_t temp0 = vfmv_v_f_f32m1(0, 4), temp1 = vfmv_v_f_f32m1(0, 4), + temp2 = vfmv_v_f_f32m1(0, 4), temp3 = vfmv_v_f_f32m1(0, 4), + temp4 = vfmv_v_f_f32m1(0, 4), temp5 = vfmv_v_f_f32m1(0, 4), + temp6 = vfmv_v_f_f32m1(0, 4), temp7 = vfmv_v_f_f32m1(0, 4); + temp0 = vfredsum_vs_f32m2_f32m1(temp0, vs0, zero, vl); + temp1 = vfredsum_vs_f32m2_f32m1(temp1, vs1, zero, vl); + temp2 = vfredsum_vs_f32m2_f32m1(temp2, vs2, zero, vl); + temp3 = vfredsum_vs_f32m2_f32m1(temp3, vs3, zero, vl); + temp4 = vfredsum_vs_f32m2_f32m1(temp4, vs4, zero, vl); + temp5 = vfredsum_vs_f32m2_f32m1(temp5, vs5, zero, vl); + temp6 = vfredsum_vs_f32m2_f32m1(temp6, vs6, zero, vl); + temp7 = vfredsum_vs_f32m2_f32m1(temp7, vs7, zero, vl); + + float32_t sum[8]; + sum[0] = vfmv_f_s_f32m1_f32(temp0); + sum[1] = vfmv_f_s_f32m1_f32(temp1); + sum[2] = vfmv_f_s_f32m1_f32(temp2); + sum[3] = vfmv_f_s_f32m1_f32(temp3); + sum[4] = vfmv_f_s_f32m1_f32(temp4); + sum[5] = vfmv_f_s_f32m1_f32(temp5); + sum[6] = vfmv_f_s_f32m1_f32(temp6); + sum[7] = vfmv_f_s_f32m1_f32(temp7); + + vfloat32m2_t s0 = vfadd_vv_f32m2(vle32_v_f32m2(sum, mvl), vle32_v_f32m2(bias + i, mvl), mvl); + vse32_v_f32m2(dst + i, s0, mvl); + } +} + +enum { FASCONV_BASE_VECSZ = 4 }; // TODO: Large base size. +void fastConv( const float* weights, size_t wstep, const float* bias, + const float* rowbuf, float* output, const int* outShape, + int blockSize, int vecsize, int vecsize_aligned, + const float* relu, bool initOutput ) +{ + int vl = 4; + int outCn = outShape[1]; + size_t outPlaneSize = outShape[2]*outShape[3]; + float r0 = 1.f, r1 = 1.f, r2 = 1.f; + vfloat32m1_t vr0 = vfmv_v_f_f32m1(1, vl), vr1 = vfmv_v_f_f32m1(1, vl), vr2 = vfmv_v_f_f32m1(1, vl); + int maskbuf[FASCONV_BASE_VECSZ] = {0}; + int rsz = blockSize % FASCONV_BASE_VECSZ; + for( int i = 0; i < rsz; i++ ) + maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1; + vint32m1_t vmaskbuf = vle32_v_i32m1(maskbuf ,vl); + vbool32_t mask = vmslt_vx_i32m1_b32(vmaskbuf, 0, vl); // mask for tail + // now compute dot product of the weights + // and im2row-transformed part of the tensor + for( int i = 0; i < outCn; i += 3 ) + { + const float* wptr0 = weights + i*wstep; + const float* wptr1 = wptr0 + wstep; + const float* wptr2 = wptr1 + wstep; + float* outptr0 = output + i*outPlaneSize; + float* outptr1 = outptr0 + outPlaneSize; + float* outptr2 = outptr1 + outPlaneSize; + float bias0 = bias[i], bias1 = bias[i+1], bias2 = bias[i+2]; + + if( i+2 >= outCn ) + { + wptr2 = wptr1; + outptr2 = outptr1; + bias2 = bias1; + if( i+1 >= outCn ) + { + wptr2 = wptr1 = wptr0; + outptr2 = outptr1 = outptr0; + bias2 = bias1 = bias0; + } + } + + if( relu ) + { + r0 = relu[i]; r1 = relu[i+1]; r2 = relu[i+2]; + if( i+2 >= outCn ) + { + r2 = r1; + if( i+1 >= outCn ) + r2 = r1 = r0; + } + vr0 = vfmv_v_f_f32m1(r0, vl); + vr1 = vfmv_v_f_f32m1(r1, vl); + vr2 = vfmv_v_f_f32m1(r2, vl); + } + + int j = 0; + for( ; j < blockSize; j += FASCONV_BASE_VECSZ ) + { + bool tail = false; + if (j + FASCONV_BASE_VECSZ > blockSize) + { + if (j == 0) { + vl = blockSize; + } + else { + j = blockSize - FASCONV_BASE_VECSZ; + tail = true; + } + } + int k = 0; + const float* rptr = rowbuf + j*vecsize_aligned; + int vlm2 = 8; + vfloat32m2_t vs00 = vfmv_v_f_f32m2(0, vlm2), vs01 = vfmv_v_f_f32m2(0, vlm2), + vs02 = vfmv_v_f_f32m2(0, vlm2), vs03 = vfmv_v_f_f32m2(0, vlm2), + vs10 = vfmv_v_f_f32m2(0, vlm2), vs11 = vfmv_v_f_f32m2(0, vlm2), + vs12 = vfmv_v_f_f32m2(0, vlm2), vs13 = vfmv_v_f_f32m2(0, vlm2), + vs20 = vfmv_v_f_f32m2(0, vlm2), vs21 = vfmv_v_f_f32m2(0, vlm2), + vs22 = vfmv_v_f_f32m2(0, vlm2), vs23 = vfmv_v_f_f32m2(0, vlm2); + + for (; k < vecsize; k += 8, rptr += 8 ) + { + if (k+8 >= vecsize) { + vlm2 = vecsize - k; + } + vfloat32m2_t w0 = vle32_v_f32m2(wptr0 + k, vlm2); + vfloat32m2_t w1 = vle32_v_f32m2(wptr1 + k, vlm2); + vfloat32m2_t w2 = vle32_v_f32m2(wptr2 + k, vlm2); + vfloat32m2_t r0 = vle32_v_f32m2(rptr, vlm2); + + vs00 = vfmacc_vv_f32m2(vs00, w0, r0, vlm2); + vs10 = vfmacc_vv_f32m2(vs10, w1, r0, vlm2); + vs20 = vfmacc_vv_f32m2(vs20, w2, r0, vlm2); + + r0 = vle32_v_f32m2(rptr + vecsize_aligned, vlm2); + vs01 = vfmacc_vv_f32m2(vs01, w0, r0, vlm2); + vs11 = vfmacc_vv_f32m2(vs11, w1, r0, vlm2); + vs21 = vfmacc_vv_f32m2(vs21, w2, r0, vlm2); + + r0 = vle32_v_f32m2(rptr + vecsize_aligned*2, vlm2); + vs02 = vfmacc_vv_f32m2(vs02, w0, r0, vlm2); + vs12 = vfmacc_vv_f32m2(vs12, w1, r0, vlm2); + vs22 = vfmacc_vv_f32m2(vs22, w2, r0, vlm2); + + r0 = vle32_v_f32m2(rptr + vecsize_aligned*3, vlm2); + vs03 = vfmacc_vv_f32m2(vs03, w0, r0, vlm2); + vs13 = vfmacc_vv_f32m2(vs13, w1, r0, vlm2); + vs23 = vfmacc_vv_f32m2(vs23, w2, r0, vlm2); + } + vfloat32m1_t s0, s1, s2; + + if( initOutput ) + { + s0 = vfmv_v_f_f32m1(bias0, vl); + s1 = vfmv_v_f_f32m1(bias1, vl); + s2 = vfmv_v_f_f32m1(bias2, vl); + } + else + { + s0 = vle32_v_f32m1(outptr0 + j, vl); + s1 = vle32_v_f32m1(outptr1 + j, vl); + s2 = vle32_v_f32m1(outptr2 + j, vl); + } + // compute sum of each vs + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vl); + vfloat32m1_t temp00 = vfredsum_vs_f32m2_f32m1(temp00, vs00, zero, 8); + vfloat32m1_t temp01 = vfredsum_vs_f32m2_f32m1(temp01, vs01, zero, 8); + vfloat32m1_t temp02 = vfredsum_vs_f32m2_f32m1(temp02, vs02, zero, 8); + vfloat32m1_t temp03 = vfredsum_vs_f32m2_f32m1(temp03, vs03, zero, 8); + vfloat32m1_t temp10 = vfredsum_vs_f32m2_f32m1(temp10, vs10, zero, 8); + vfloat32m1_t temp11 = vfredsum_vs_f32m2_f32m1(temp11, vs11, zero, 8); + vfloat32m1_t temp12 = vfredsum_vs_f32m2_f32m1(temp12, vs12, zero, 8); + vfloat32m1_t temp13 = vfredsum_vs_f32m2_f32m1(temp13, vs13, zero, 8); + vfloat32m1_t temp20 = vfredsum_vs_f32m2_f32m1(temp20, vs20, zero, 8); + vfloat32m1_t temp21 = vfredsum_vs_f32m2_f32m1(temp21, vs21, zero, 8); + vfloat32m1_t temp22 = vfredsum_vs_f32m2_f32m1(temp22, vs22, zero, 8); + vfloat32m1_t temp23 = vfredsum_vs_f32m2_f32m1(temp23, vs23, zero, 8); + float32_t sum0[4], sum1[4], sum2[4]; + sum0[0] = vfmv_f_s_f32m1_f32(temp00); + sum0[1] = vfmv_f_s_f32m1_f32(temp01); + sum0[2] = vfmv_f_s_f32m1_f32(temp02); + sum0[3] = vfmv_f_s_f32m1_f32(temp03); + sum1[0] = vfmv_f_s_f32m1_f32(temp10); + sum1[1] = vfmv_f_s_f32m1_f32(temp11); + sum1[2] = vfmv_f_s_f32m1_f32(temp12); + sum1[3] = vfmv_f_s_f32m1_f32(temp13); + sum2[0] = vfmv_f_s_f32m1_f32(temp20); + sum2[1] = vfmv_f_s_f32m1_f32(temp21); + sum2[2] = vfmv_f_s_f32m1_f32(temp22); + sum2[3] = vfmv_f_s_f32m1_f32(temp23); + + s0 = vfadd_vv_f32m1(vle32_v_f32m1(sum0, vl), s0, vl); + s1 = vfadd_vv_f32m1(vle32_v_f32m1(sum1, vl), s1, vl); + s2 = vfadd_vv_f32m1(vle32_v_f32m1(sum2, vl), s2, vl); + + + if( relu ) + { + vbool32_t m0 = vmfgt_vf_f32m1_b32(s0, 0, vl); + vbool32_t m1 = vmfgt_vf_f32m1_b32(s1, 0, vl); + vbool32_t m2 = vmfgt_vf_f32m1_b32(s2, 0, vl); + s0 = vmerge_vvm_f32m1(m0, vfmul_vv_f32m1(s0, vr0, vl), s0, vl); + s1 = vmerge_vvm_f32m1(m1, vfmul_vv_f32m1(s1, vr1, vl), s1, vl); + s2 = vmerge_vvm_f32m1(m2, vfmul_vv_f32m1(s2, vr2, vl), s2, vl); + } + + if( tail ) + { + s0 = vmerge_vvm_f32m1(mask, vle32_v_f32m1(outptr0 + j, vl), s0, vl); + s1 = vmerge_vvm_f32m1(mask, vle32_v_f32m1(outptr1 + j, vl), s1, vl); + s2 = vmerge_vvm_f32m1(mask, vle32_v_f32m1(outptr2 + j, vl), s2, vl); + } + + vse32_v_f32m1(outptr0 + j, s0, vl); + vse32_v_f32m1(outptr1 + j, s1, vl); + vse32_v_f32m1(outptr2 + j, s2, vl); + } + } +} + +/* +Example for load_deinterleave: + input: ptr[16] = {1,2,3, ... ,14,15,16} + output: a = {1, 3, 5, 7, 9, 11, 13, 15} + output: b = {2, 4, 6, 8,10, 12, 14, 16} +*/ +static inline void vfloat32m2_load_deinterleave(const float* ptr, vfloat32m2_t& a, vfloat32m2_t& b) +{ + int vl = 8; + uint32_t masks[] = {1,1,1,1,0,0,0,0}; + vuint32m2_t vm = vle32_v_u32m2(masks,vl); + vbool16_t mask01 = vmseq_vx_u32m2_b16 (vm, 0, vl); + vbool16_t mask10 = vmseq_vx_u32m2_b16 (vm, 1, vl); + vfloat32m2_t ta = vle32_v_f32m2(ptr, vl), tb = vle32_v_f32m2(ptr+8, vl); + uint idx[] = {0,2,4,6,1,3,5,7}; + uint idxa[] = {0,0,0,0,0,1,2,3}, idxb[] = {4,5,6,7,0,0,0,0}; + vuint32m2_t vidxa = vle32_v_u32m2(idxa, 8), vidxb = vle32_v_u32m2(idxb, 8); + vuint32m2_t vidx = vle32_v_u32m2(idx, 8); + vfloat32m2_t high = vfmv_v_f_f32m2(0, 8), low = vfmv_v_f_f32m2(0, 8); + high = vrgather_vv_f32m2(ta, vidx, 8); + low = vrgather_vv_f32m2(tb, vidx, 8); + a = vrgather_vv_f32m2_m(mask01, high, low, vidxa, 8); + b = vrgather_vv_f32m2_m(mask10, low, high, vidxb, 8); +} + +void fastDepthwiseConv( const float* wptr, + int kernel_h, int kernel_w, + int stride_h, int stride_w, + int dilation_h, int dilation_w, + int pad_t, int pad_l, + const float* biasptr, const float* relu, + const float* inptr_, + int height, int width, + float* outptr_, + int out_d, int outH, int outW ) +{ + int vl = 8; + const float w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2], + w10 = wptr[3], w11 = wptr[4], w12 = wptr[5], + w20_ = wptr[6], w21_ = wptr[7], w22_ = wptr[8]; + int outW1 = std::min(outW, (width - dilation_w*(kernel_w - 1) + pad_l)/stride_w); + float relu_coeff = relu ? relu[out_d] : 1.f, bias = biasptr[out_d]; + + for (int out_i = 0; out_i < outH; out_i++) + { + int in_i = out_i * stride_h - pad_t, out_j = 0; + const float* imgptr0 = inptr_ + in_i*width; + const float* imgptr1 = imgptr0 + dilation_h*width; + const float* imgptr2 = imgptr0 + (dilation_h*2)*width; + float out, w00 = w00_, w01 = w01_, w02 = w02_; + float w20 = w20_, w21 = w21_, w22 = w22_; + if (in_i < 0) + { + w00 = w01 = w02 = 0.f; + imgptr0 = imgptr1; + } + else if (in_i + dilation_h*(kernel_h-1) >= height) + { + w20 = w21 = w22 = 0.f; + imgptr2 = imgptr1; + } + float* outptr = outptr_ + out_i*outW; + if (pad_l > 0) + { + out = imgptr0[0]*w01 + imgptr0[dilation_w]*w02 + + imgptr1[0]*w11 + imgptr1[dilation_w]*w12 + + imgptr2[0]*w21 + imgptr2[dilation_w]*w22 + bias; + if (relu) + out = out > 0.f ? out : out*relu_coeff; + outptr[0] = out; + out_j = 1; + } + + if (stride_w == 1 || (stride_w == 2 && dilation_w == 1)) + { + const int VECSZ = 8; + vfloat32m2_t vw00 = vfmv_v_f_f32m2(w00, vl), vw01 = vfmv_v_f_f32m2(w01, vl), vw02 = vfmv_v_f_f32m2(w02, vl), + vw10 = vfmv_v_f_f32m2(w10, vl), vw11 = vfmv_v_f_f32m2(w11, vl), vw12 = vfmv_v_f_f32m2(w12, vl), + vw20 = vfmv_v_f_f32m2(w20, vl), vw21 = vfmv_v_f_f32m2(w21, vl), vw22 = vfmv_v_f_f32m2(w22, vl); + vfloat32m2_t vbias = vfmv_v_f_f32m2(bias, vl), vrc = vfmv_v_f_f32m2(relu_coeff, vl); + + if( stride_w == 1 ) + for( ; out_j < outW1; out_j += VECSZ ) + { + if (out_j + VECSZ > outW1 && out_j > pad_l) + out_j = outW1 - VECSZ; + int in_j = out_j * stride_w - pad_l; + vfloat32m2_t v00 = vle32_v_f32m2(imgptr0 + in_j, vl), + v01 = vle32_v_f32m2(imgptr0 + in_j + dilation_w, vl), + v02 = vle32_v_f32m2(imgptr0 + in_j + dilation_w*2, vl), + v10 = vle32_v_f32m2(imgptr1 + in_j, vl), + v11 = vle32_v_f32m2(imgptr1 + in_j + dilation_w, vl), + v12 = vle32_v_f32m2(imgptr1 + in_j + dilation_w*2, vl), + v20 = vle32_v_f32m2(imgptr2 + in_j, vl), + v21 = vle32_v_f32m2(imgptr2 + in_j + dilation_w, vl), + v22 = vle32_v_f32m2(imgptr2 + in_j + dilation_w*2, vl); + + vfloat32m2_t vout0 = vfmacc_vv_f32m2(vbias, v00, vw00, vl); + vfloat32m2_t vout1 = vfmul_vv_f32m2(v01, vw01, vl); + vfloat32m2_t vout2 = vfmul_vv_f32m2(v02, vw02, vl); + + vout0 = vfmacc_vv_f32m2(vout0, v10, vw10, vl); + vout1 = vfmacc_vv_f32m2(vout1, v11, vw11, vl); + vout2 = vfmacc_vv_f32m2(vout2, v12, vw12, vl); + + vout0 = vfmacc_vv_f32m2(vout0, v20, vw20, vl); + vout1 = vfmacc_vv_f32m2(vout1, v21, vw21, vl); + vout2 = vfmacc_vv_f32m2(vout2, v22, vw22, vl); + + vout0 = vfadd_vv_f32m2(vfadd_vv_f32m2(vout0, vout1, vl), vout2, vl); + if (relu) + { + vbool16_t m = vmfgt_vf_f32m2_b16(vout0, 0, vl); + vout0 = vmerge_vvm_f32m2(m, vfmul_vv_f32m2(vout0, vrc, vl), vout0, vl); + } + vse32_v_f32m2(outptr + out_j, vout0, vl); + } + else + for( ; out_j < outW1; out_j += VECSZ ) + { + if (out_j + VECSZ > outW1 && out_j > pad_l) + out_j = outW1 - VECSZ; + int in_j = out_j * stride_w - pad_l; + vfloat32m2_t v00, v01, v02, v10, v11, v12, v20, v21, v22, unused; + vfloat32m2_load_deinterleave(imgptr0 + in_j, v00, v01); + vfloat32m2_load_deinterleave(imgptr0 + in_j + 2, v02, unused); + vfloat32m2_load_deinterleave(imgptr1 + in_j, v10, v11); + vfloat32m2_load_deinterleave(imgptr1 + in_j + 2, v12, unused); + vfloat32m2_load_deinterleave(imgptr2 + in_j, v20, v21); + vfloat32m2_load_deinterleave(imgptr2 + in_j + 2, v22, unused); + + vfloat32m2_t vout0 = vfmacc_vv_f32m2(vbias, v00, vw00, vl); + vfloat32m2_t vout1 = vfmul_vv_f32m2(v01, vw01, vl); + vfloat32m2_t vout2 = vfmul_vv_f32m2(v02, vw02, vl); + + vout0 = vfmacc_vv_f32m2(vout0, v10, vw10, vl); + vout1 = vfmacc_vv_f32m2(vout1, v11, vw11, vl); + vout2 = vfmacc_vv_f32m2(vout2, v12, vw12, vl); + + vout0 = vfmacc_vv_f32m2(vout0, v20, vw20, vl); + vout1 = vfmacc_vv_f32m2(vout1, v21, vw21, vl); + vout2 = vfmacc_vv_f32m2(vout2, v22, vw22, vl); + + vout0 = vfadd_vv_f32m2(vfadd_vv_f32m2(vout0, vout1, vl), vout2, vl); + if (relu) + { + vbool16_t m = vmfgt_vf_f32m2_b16(vout0, 0, vl); + vout0 = vmerge_vvm_f32m2(m, vfmul_vv_f32m2(vout0, vrc, vl), vout0, vl); + } + vse32_v_f32m2(outptr + out_j, vout0, vl); + } + } + + for (; out_j < outW1; out_j++) + { + int in_j = out_j * stride_w - pad_l; + out = imgptr0[in_j]*w00 + imgptr0[in_j + dilation_w]*w01 + imgptr0[in_j + dilation_w*2]*w02 + + imgptr1[in_j]*w10 + imgptr1[in_j + dilation_w]*w11 + imgptr1[in_j + dilation_w*2]*w12 + + imgptr2[in_j]*w20 + imgptr2[in_j + dilation_w]*w21 + imgptr2[in_j + dilation_w*2]*w22 + bias; + if (relu) + out = out > 0.f ? out : out*relu_coeff; + outptr[out_j] = out; + } + + for (; out_j < outW; out_j++ ) + { + int in_j0 = out_j * stride_w - pad_l, in_j1 = in_j0 + dilation_w, in_j2 = in_j0 + dilation_w*2; + float s0 = 1.f, s1 = 1.f, s2 = 1.f; + if (in_j0 >= width) + { + in_j0 = 0; + s0 = 0.f; + } + if (in_j1 >= width) + { + in_j1 = 0; + s1 = 0.f; + } + if (in_j2 >= width) + { + in_j2 = 0; + s2 = 0.f; + } + out = imgptr0[in_j0]*w00*s0 + imgptr0[in_j1]*w01*s1 + imgptr0[in_j2]*w02*s2 + + imgptr1[in_j0]*w10*s0 + imgptr1[in_j1]*w11*s1 + imgptr1[in_j2]*w12*s2 + + imgptr2[in_j0]*w20*s0 + imgptr2[in_j1]*w21*s1 + imgptr2[in_j2]*w22*s2 + bias; + if (relu) + out = out > 0.f ? out : out*relu_coeff; + outptr[out_j] = out; + } + } +} + +#endif // CV_RVV + CV_CPU_OPTIMIZATION_NAMESPACE_END }} // namespace From 7c73e28a6d5ceae9ffac5a3f6c771a8efb7bb35e Mon Sep 17 00:00:00 2001 From: Hamdi Sahloul Date: Mon, 26 Jul 2021 19:03:13 +0900 Subject: [PATCH 107/376] Improves FLANN's heap allocations by a memory pool --- modules/flann/include/opencv2/flann/heap.h | 167 +++++++++++++----- .../flann/hierarchical_clustering_index.h | 6 +- .../include/opencv2/flann/kdtree_index.h | 9 +- .../include/opencv2/flann/kmeans_index.h | 8 +- 4 files changed, 130 insertions(+), 60 deletions(-) diff --git a/modules/flann/include/opencv2/flann/heap.h b/modules/flann/include/opencv2/flann/heap.h index ee1c682cfe98..8cace2044973 100644 --- a/modules/flann/include/opencv2/flann/heap.h +++ b/modules/flann/include/opencv2/flann/heap.h @@ -36,9 +36,21 @@ #include #include +#include + namespace cvflann { +// TODO: Define x > y operator and use std::greater instead +template +struct greater +{ + bool operator()(const T& x, const T& y) const + { + return y < x; + } +}; + /** * Priority Queue Implementation * @@ -49,117 +61,180 @@ namespace cvflann template class Heap { - /** * Storage array for the heap. * Type T must be comparable. */ std::vector heap; - int length; - +public: /** - * Number of element in the heap + * \brief Constructs a heap with a pre-allocated capacity + * + * \param capacity heap maximum capacity */ - int count; - - + Heap(const int capacity) + { + reserve(capacity); + } -public: /** - * Constructor. + * \brief Move-constructs a heap from an external vector * - * Params: - * sz = heap size + * \param vec external vector */ + Heap(std::vector&& vec) + : heap(std::move(vec)) + { + std::make_heap(heap.begin(), heap.end(), greater()); + } - Heap(int sz) + /** + * + * \returns heap size + */ + int size() const { - length = sz; - heap.reserve(length); - count = 0; + return (int)heap.size(); } /** * - * Returns: heap size + * \returns heap capacity */ - int size() + int capacity() const { - return count; + return (int)heap.capacity(); } /** - * Tests if the heap is empty + * \brief Tests if the heap is empty * - * Returns: true is heap empty, false otherwise + * \returns true is heap empty, false otherwise */ bool empty() { - return size()==0; + return heap.empty(); } /** - * Clears the heap. + * \brief Clears the heap. */ void clear() { heap.clear(); - count = 0; } - struct CompareT + /** + * \brief Sets the heap maximum capacity. + * + * \param capacity heap maximum capacity + */ + void reserve(const int capacity) { - bool operator()(const T& t_1, const T& t_2) const - { - return t_2 < t_1; - } - }; + heap.reserve(capacity); + } /** - * Insert a new element in the heap. + * \brief Inserts a new element in the heap. * * We select the next empty leaf node, and then keep moving any larger * parents down until the right location is found to store this element. * - * Params: - * value = the new element to be inserted in the heap + * \param value the new element to be inserted in the heap */ void insert(T value) { /* If heap is full, then return without adding this element. */ - if (count == length) { + if (size() == capacity()) { return; } heap.push_back(value); - static CompareT compareT; - std::push_heap(heap.begin(), heap.end(), compareT); - ++count; + std::push_heap(heap.begin(), heap.end(), greater()); } - - /** - * Returns the node of minimum value from the heap (top of the heap). + * \brief Returns the node of minimum value from the heap (top of the heap). * - * Params: - * value = out parameter used to return the min element - * Returns: false if heap empty + * \param[out] value parameter used to return the min element + * \returns false if heap empty */ bool popMin(T& value) { - if (count == 0) { + if (empty()) { return false; } value = heap[0]; - static CompareT compareT; - std::pop_heap(heap.begin(), heap.end(), compareT); + std::pop_heap(heap.begin(), heap.end(), greater()); heap.pop_back(); - --count; return true; /* Return old last node. */ } + + /** + * \brief Returns a shared heap for the given memory pool ID. + * + * It constructs the heap if it does not already exists. + * + * \param poolId a user-chosen hashable ID for identifying the heap. + * For thread-safe operations, using current thread ID is a good choice. + * \param capacity heap maximum capacity + * \param iterThreshold remove heaps that were not reused for more than specified iterations count + * if iterThreshold value is less 2, it will be internally adjusted to twice the number of CPU threads + * \returns pointer to the heap + */ + template + static cv::Ptr> getPooledInstance( + const HashableT& poolId, const int capacity, int iterThreshold = 0) + { + static cv::Mutex mutex; + const cv::AutoLock lock(mutex); + + struct HeapMapValueType { + cv::Ptr> heapPtr; + int iterCounter; + }; + typedef std::unordered_map HeapMapType; + + static HeapMapType heapsPool; + typename HeapMapType::iterator heapIt = heapsPool.find(poolId); + + if (heapIt == heapsPool.end()) + { + // Construct the heap as it does not already exists + HeapMapValueType heapAndTimePair = {cv::makePtr>(capacity), 0}; + const std::pair& emplaceResult = heapsPool.emplace(poolId, std::move(heapAndTimePair)); + CV_CheckEQ(static_cast(emplaceResult.second), 1, "Failed to insert the heap into its memory pool"); + heapIt = emplaceResult.first; + } + else + { + CV_CheckEQ(heapIt->second.heapPtr.use_count(), 1, "Cannot modify a heap that is currently accessed by another caller"); + heapIt->second.heapPtr->clear(); + heapIt->second.heapPtr->reserve(capacity); + heapIt->second.iterCounter = 0; + } + + if (iterThreshold <= 1) { + iterThreshold = 2 * cv::getNumThreads(); + } + + // Remove heaps that were not reused for more than given iterThreshold + typename HeapMapType::iterator cleanupIt = heapsPool.begin(); + while (cleanupIt != heapsPool.end()) + { + if (cleanupIt->second.iterCounter++ > iterThreshold) + { + CV_Assert(cleanupIt != heapIt); + cleanupIt = heapsPool.erase(cleanupIt); + continue; + } + ++cleanupIt; + } + + return heapIt->second.heapPtr; + } }; } diff --git a/modules/flann/include/opencv2/flann/hierarchical_clustering_index.h b/modules/flann/include/opencv2/flann/hierarchical_clustering_index.h index 2d39d4f0f654..60662e7714b3 100644 --- a/modules/flann/include/opencv2/flann/hierarchical_clustering_index.h +++ b/modules/flann/include/opencv2/flann/hierarchical_clustering_index.h @@ -532,7 +532,7 @@ class HierarchicalClusteringIndex : public NNIndex const bool explore_all_trees = get_param(searchParams,"explore_all_trees",false); // Priority queue storing intermediate branches in the best-bin-first search - Heap* heap = new Heap((int)size_); + const cv::Ptr>& heap = Heap::getPooledInstance(cv::utils::getThreadID(), (int)size_); std::vector checked(size_,false); int checks = 0; @@ -548,8 +548,6 @@ class HierarchicalClusteringIndex : public NNIndex findNN(node, result, vec, checks, maxChecks, heap, checked, false); } - delete heap; - CV_Assert(result.full()); } @@ -742,7 +740,7 @@ class HierarchicalClusteringIndex : public NNIndex void findNN(NodePtr node, ResultSet& result, const ElementType* vec, int& checks, int maxChecks, - Heap* heap, std::vector& checked, bool explore_all_trees = false) + const cv::Ptr>& heap, std::vector& checked, bool explore_all_trees = false) { if (node->childs==NULL) { if (!explore_all_trees && (checks>=maxChecks) && result.full()) { diff --git a/modules/flann/include/opencv2/flann/kdtree_index.h b/modules/flann/include/opencv2/flann/kdtree_index.h index 603fdbd421a5..8245f7db796e 100644 --- a/modules/flann/include/opencv2/flann/kdtree_index.h +++ b/modules/flann/include/opencv2/flann/kdtree_index.h @@ -445,11 +445,12 @@ class KDTreeIndex : public NNIndex { int i; BranchSt branch; - int checkCount = 0; - Heap* heap = new Heap((int)size_); DynamicBitset checked(size_); + // Priority queue storing intermediate branches in the best-bin-first search + const cv::Ptr>& heap = Heap::getPooledInstance(cv::utils::getThreadID(), (int)size_); + /* Search once through each tree down to root. */ for (i = 0; i < trees_; ++i) { searchLevel(result, vec, tree_roots_[i], 0, checkCount, maxCheck, @@ -464,8 +465,6 @@ class KDTreeIndex : public NNIndex epsError, heap, checked, false); } - delete heap; - CV_Assert(result.full()); } @@ -476,7 +475,7 @@ class KDTreeIndex : public NNIndex * at least "mindistsq". */ void searchLevel(ResultSet& result_set, const ElementType* vec, NodePtr node, DistanceType mindist, int& checkCount, int maxCheck, - float epsError, Heap* heap, DynamicBitset& checked, bool explore_all_trees = false) + float epsError, const cv::Ptr>& heap, DynamicBitset& checked, bool explore_all_trees = false) { if (result_set.worstDist() } else { // Priority queue storing intermediate branches in the best-bin-first search - Heap* heap = new Heap((int)size_); + const cv::Ptr>& heap = Heap::getPooledInstance(cv::utils::getThreadID(), (int)size_); int checks = 0; for (int i=0; i KMeansNodePtr node = branch.node; findNN(node, result, vec, checks, maxChecks, heap); } - delete heap; - CV_Assert(result.full()); } } @@ -1529,7 +1527,7 @@ class KMeansIndex : public NNIndex void findNN(KMeansNodePtr node, ResultSet& result, const ElementType* vec, int& checks, int maxChecks, - Heap* heap) + const cv::Ptr>& heap) { // Ignore those clusters that are too far away { @@ -1577,7 +1575,7 @@ class KMeansIndex : public NNIndex * distances = array with the distances to each child node. * Returns: */ - int exploreNodeBranches(KMeansNodePtr node, const ElementType* q, DistanceType* domain_distances, Heap* heap) + int exploreNodeBranches(KMeansNodePtr node, const ElementType* q, DistanceType* domain_distances, const cv::Ptr>& heap) { int best_index = 0; From 9d61c181434a6903fa15e4915b9fffed65ebcae8 Mon Sep 17 00:00:00 2001 From: utibenkei Date: Sun, 8 Aug 2021 01:08:31 +0900 Subject: [PATCH 108/376] fix testSaveLoad --- modules/ml/misc/java/test/MLTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ml/misc/java/test/MLTest.java b/modules/ml/misc/java/test/MLTest.java index 2b08543a843b..504805dffa97 100644 --- a/modules/ml/misc/java/test/MLTest.java +++ b/modules/ml/misc/java/test/MLTest.java @@ -36,7 +36,7 @@ public void testSaveLoad() { String filename = OpenCVTestRunner.getTempFileName("yml"); saved.save(filename); SVM loaded = SVM.load(filename); - assertTrue(saved.isTrained()); + assertTrue(loaded.isTrained()); } } From 8199967b3189fb9aa711afc4e815cd13f312b7ae Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Wed, 11 Aug 2021 19:08:52 +0300 Subject: [PATCH 109/376] fix choose minimum angle in rotatingCalipers --- modules/imgproc/src/rotcalipers.cpp | 55 ++++++++++++++++++----------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/modules/imgproc/src/rotcalipers.cpp b/modules/imgproc/src/rotcalipers.cpp index 527f71a2477f..e3d81c7e0c0f 100644 --- a/modules/imgproc/src/rotcalipers.cpp +++ b/modules/imgproc/src/rotcalipers.cpp @@ -88,6 +88,32 @@ enum { CALIPERS_MAXHEIGHT=0, CALIPERS_MINAREARECT=1, CALIPERS_MAXDIST=2 }; // Notes: //F*/ +static void rotate90CCW(const cv::Point2f& in, cv::Point2f &out) +{ + out.x = -in.y; + out.y = in.x; +} + +static void rotate90CW(const cv::Point2f& in, cv::Point2f &out) +{ + out.x = in.y; + out.y = -in.x; +} + +static void rotate180(const cv::Point2f& in, cv::Point2f &out) +{ + out.x = -in.x; + out.y = -in.y; +} + +/* return true if first vector is to the right (clockwise) of the second */ +static bool firstVecIsRight(const cv::Point2f& vec1, const cv::Point2f &vec2) +{ + cv::Point2f tmp; + rotate90CW(vec1, tmp); + return tmp.x * vec2.x + tmp.y * vec2.y < 0; +} + /* we will use usual cartesian coordinates */ static void rotatingCalipers( const Point2f* points, int n, int mode, float* out ) { @@ -100,6 +126,7 @@ static void rotatingCalipers( const Point2f* points, int n, int mode, float* out Point2f* vect = (Point2f*)(inv_vect_length + n); int left = 0, bottom = 0, right = 0, top = 0; int seq[4] = { -1, -1, -1, -1 }; + Point2f rot_vect[4]; /* rotating calipers sides will always have coordinates (a,b) (-b,a) (-a,-b) (b, -a) @@ -179,32 +206,18 @@ static void rotatingCalipers( const Point2f* points, int n, int mode, float* out /* all of edges will be checked while rotating calipers by 90 degrees */ for( k = 0; k < n; k++ ) { - /* sinus of minimal angle */ - /*float sinus;*/ - - /* compute cosine of angle between calipers side and polygon edge */ - /* dp - dot product */ - float dp[4] = { - +base_a * vect[seq[0]].x + base_b * vect[seq[0]].y, - -base_b * vect[seq[1]].x + base_a * vect[seq[1]].y, - -base_a * vect[seq[2]].x - base_b * vect[seq[2]].y, - +base_b * vect[seq[3]].x - base_a * vect[seq[3]].y, - }; - - float maxcos = dp[0] * inv_vect_length[seq[0]]; - /* number of calipers edges, that has minimal angle with edge */ int main_element = 0; - /* choose minimal angle */ - for ( i = 1; i < 4; ++i ) + /* choose minimum angle between calipers side and polygon edge by dot product sign */ + rot_vect[0] = vect[seq[0]]; + rotate90CW(vect[seq[1]], rot_vect[1]); + rotate180(vect[seq[2]], rot_vect[2]); + rotate90CCW(vect[seq[3]], rot_vect[3]); + for (i = 1; i < 4; i++) { - float cosalpha = dp[i] * inv_vect_length[seq[i]]; - if (cosalpha > maxcos) - { + if (firstVecIsRight(rot_vect[i], rot_vect[main_element])) main_element = i; - maxcos = cosalpha; - } } /*rotate calipers*/ From 4d63a89fa6611a69c1999328375f39afde1b1696 Mon Sep 17 00:00:00 2001 From: Daniel Playfair Cal Date: Thu, 12 Aug 2021 03:58:08 +1000 Subject: [PATCH 110/376] Merge pull request #20536 from hedgepigdaniel:fix/ocl-context-create-ownership docs(core/ocl): clarify ownership of arguments passed into OpenCL related functions * docs(core/ocl): clarify ownership in OpenCLExecutionContext::create Although it is technically true that OpenCLExecutionContext::create calls `clRetainContext` on its context argument, it is misleading because it does not increase the reference count overall. Clarify that the ownership of one reference of the passed context and device is taken. * docs(core/ocl): document ownership transfer in ocl::Device::fromHandle --- modules/core/include/opencv2/core/ocl.hpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/modules/core/include/opencv2/core/ocl.hpp b/modules/core/include/opencv2/core/ocl.hpp index f9cc9e019a03..03666df5176b 100644 --- a/modules/core/include/opencv2/core/ocl.hpp +++ b/modules/core/include/opencv2/core/ocl.hpp @@ -235,7 +235,11 @@ class CV_EXPORTS_W_SIMPLE Device /** * @param d OpenCL handle (cl_device_id). clRetainDevice() is called on success. - */ + * + * @note Ownership of the passed device is passed to OpenCV on success. + * The caller should additionally call `clRetainDevice` on it if it intends + * to continue using the device. + */ static Device fromHandle(void* d); struct Impl; @@ -826,11 +830,13 @@ class CV_EXPORTS_W OpenCLExecutionContext OpenCLExecutionContext cloneWithNewQueue() const; /** @brief Creates OpenCL execution context - * OpenCV will check if available OpenCL platform has platformName name, then assign context to - * OpenCV and call `clRetainContext` function. The deviceID device will be used as target device and - * new command queue will be created. + * OpenCV will check if available OpenCL platform has platformName name, + * then assign context to OpenCV. + * The deviceID device will be used as target device and a new command queue will be created. * - * @note Lifetime of passed handles is transferred to OpenCV wrappers on success + * @note On success, ownership of one reference of the context and device is taken. + * The caller should additionally call `clRetainContext` and/or `clRetainDevice` + * to increase the reference count if it wishes to continue using them. * * @param platformName name of OpenCL platform to attach, this string is used to check if platform is available to OpenCV at runtime * @param platformID ID of platform attached context was created for (cl_platform_id) From 25cd7c7c509826e42214d00ca4e73ac096abd382 Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Thu, 12 Aug 2021 14:40:40 +0300 Subject: [PATCH 111/376] add note about Python's dsize to doc --- modules/imgproc/include/opencv2/imgproc.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index 5e66b14e3b2b..f7583c19267a 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -2223,7 +2223,7 @@ enlarge an image, it will generally look best with c#INTER_CUBIC (slow) or #INTE @param src input image. @param dst output image; it has the size dsize (when it is non-zero) or the size computed from src.size(), fx, and fy; the type of dst is the same as of src. -@param dsize output image size; if it equals zero, it is computed as: +@param dsize output image size; if it equals zero (`None` in Python), it is computed as: \f[\texttt{dsize = Size(round(fx*src.cols), round(fy*src.rows))}\f] Either dsize or both fx and fy must be non-zero. @param fx scale factor along the horizontal axis; when it equals 0, it is computed as From 955cf35d5f890b85baa12b254a325b98880813d0 Mon Sep 17 00:00:00 2001 From: JIANG Yichen Date: Mon, 9 Aug 2021 13:46:11 +0800 Subject: [PATCH 112/376] Implement ctc prefix beam search decode for TextRecognitionModel. The algorithm is based on Hannun's paper: First-Pass Large Vocabulary Continuous Speech Recognition using Bi-Directional Recurrent DNNs --- .../dnn_text_spotting.markdown | 5 + modules/dnn/include/opencv2/dnn/dnn.hpp | 13 +- modules/dnn/src/math_utils.hpp | 83 ++++++ modules/dnn/src/model.cpp | 248 +++++++++++++++--- modules/dnn/test/test_model.cpp | 19 ++ 5 files changed, 332 insertions(+), 36 deletions(-) create mode 100644 modules/dnn/src/math_utils.hpp diff --git a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown index 5f28b6ce7a16..b0be2627b291 100644 --- a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown +++ b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown @@ -26,6 +26,11 @@ Before recognition, you should `setVocabulary` and `setDecodeType`. - `T` is the sequence length - `B` is the batch size (only support `B=1` in inference) - and `Dim` is the length of vocabulary +1('Blank' of CTC is at the index=0 of Dim). +- "CTC-prefix-beam-search", the output of the text recognition model should be a probability matrix same with "CTC-greedy". + - The algorithm is proposed at Hannun's [paper](https://arxiv.org/abs/1408.2873). + - `setDecodeOptsCTCPrefixBeamSearch` could be used to control the beam size in search step. + - To futher optimize for big vocabulary, a new option `vocPruneSize` is introduced to avoid iterate the whole vocbulary + but only the number of `vocPruneSize` tokens with top probabilty. @ref cv::dnn::TextRecognitionModel::recognize() is the main function for text recognition. - The input image should be a cropped text image or an image with `roiRects` diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 255b41de88a5..a498039f6571 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -1373,7 +1373,9 @@ class CV_EXPORTS_W_SIMPLE TextRecognitionModel : public Model /** * @brief Set the decoding method of translating the network output into string - * @param[in] decodeType The decoding method of translating the network output into string: {'CTC-greedy': greedy decoding for the output of CTC-based methods} + * @param[in] decodeType The decoding method of translating the network output into string, currently supported type: + * - `"CTC-greedy"` greedy decoding for the output of CTC-based methods + * - `"CTC-prefix-beam-search"` Prefix beam search decoding for the output of CTC-based methods */ CV_WRAP TextRecognitionModel& setDecodeType(const std::string& decodeType); @@ -1385,6 +1387,15 @@ class CV_EXPORTS_W_SIMPLE TextRecognitionModel : public Model CV_WRAP const std::string& getDecodeType() const; + /** + * @brief Set the decoding method options for `"CTC-prefix-beam-search"` decode usage + * @param[in] beamSize Beam size for search + * @param[in] vocPruneSize Parameter to optimize big vocabulary search, + * only take top @p vocPruneSize tokens in each search step, @p vocPruneSize <= 0 stands for disable this prune. + */ + CV_WRAP + TextRecognitionModel& setDecodeOptsCTCPrefixBeamSearch(int beamSize, int vocPruneSize = 0); + /** * @brief Set the vocabulary for recognition. * @param[in] vocabulary the associated vocabulary of the network. diff --git a/modules/dnn/src/math_utils.hpp b/modules/dnn/src/math_utils.hpp new file mode 100644 index 000000000000..19ee474c7365 --- /dev/null +++ b/modules/dnn/src/math_utils.hpp @@ -0,0 +1,83 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// Code is borrowed from https://github.com/kaldi-asr/kaldi/blob/master/src/base/kaldi-math.h + +// base/kaldi-math.h + +// Copyright 2009-2011 Ondrej Glembek; Microsoft Corporation; Yanmin Qian; +// Jan Silovsky; Saarland University +// +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef __OPENCV_DNN_MATH_UTILS_HPP__ +#define __OPENCV_DNN_MATH_UTILS_HPP__ + +#ifdef OS_QNX +#include +#else +#include +#endif + +#include + +#ifndef FLT_EPSILON +#define FLT_EPSILON 1.19209290e-7f +#endif + +namespace cv { namespace dnn { + +const float kNegativeInfinity = -std::numeric_limits::infinity(); + +const float kMinLogDiffFloat = std::log(FLT_EPSILON); + +#if !defined(_MSC_VER) || (_MSC_VER >= 1700) +inline float Log1p(float x) { return log1pf(x); } +#else +inline float Log1p(float x) { + const float cutoff = 1.0e-07; + if (x < cutoff) + return x - 2 * x * x; + else + return Log(1.0 + x); +} +#endif + +inline float Exp(float x) { return expf(x); } + +inline float LogAdd(float x, float y) { + float diff; + if (x < y) { + diff = x - y; + x = y; + } else { + diff = y - x; + } + // diff is negative. x is now the larger one. + + if (diff >= kMinLogDiffFloat) { + float res; + res = x + Log1p(Exp(diff)); + return res; + } else { + return x; // return the larger one. + } +} + +}} // namespace + +#endif // __OPENCV_DNN_MATH_UTILS_HPP__ diff --git a/modules/dnn/src/model.cpp b/modules/dnn/src/model.cpp index 0af8223a7feb..bc8709d22edc 100644 --- a/modules/dnn/src/model.cpp +++ b/modules/dnn/src/model.cpp @@ -3,8 +3,10 @@ // of this distribution and at http://opencv.org/license.html. #include "precomp.hpp" +#include "math_utils.hpp" #include #include +#include #include #include @@ -552,6 +554,9 @@ struct TextRecognitionModel_Impl : public Model::Impl std::string decodeType; std::vector vocabulary; + int beamSize = 10; + int vocPruneSize = 0; + TextRecognitionModel_Impl() { CV_TRACE_FUNCTION(); @@ -575,6 +580,13 @@ struct TextRecognitionModel_Impl : public Model::Impl decodeType = type; } + inline + void setDecodeOptsCTCPrefixBeamSearch(int beam, int vocPrune) + { + beamSize = beam; + vocPruneSize = vocPrune; + } + virtual std::string decode(const Mat& prediction) { @@ -586,53 +598,213 @@ struct TextRecognitionModel_Impl : public Model::Impl CV_Error(Error::StsBadArg, "TextRecognitionModel: vocabulary is not specified"); std::string decodeSeq; - if (decodeType == "CTC-greedy") + if (decodeType == "CTC-greedy") { + decodeSeq = ctcGreedyDecode(prediction); + } else if (decodeType == "CTC-prefix-beam-search") { + decodeSeq = ctcPrefixBeamSearchDecode(prediction); + } else if (decodeType.length() == 0) { + CV_Error(Error::StsBadArg, "Please set decodeType"); + } else { + CV_Error_(Error::StsBadArg, ("Unsupported decodeType: %s", decodeType.c_str())); + } + + return decodeSeq; + } + + virtual + std::string ctcGreedyDecode(const Mat& prediction) + { + std::string decodeSeq; + CV_CheckEQ(prediction.dims, 3, ""); + CV_CheckType(prediction.type(), CV_32FC1, ""); + const int vocLength = (int)(vocabulary.size()); + CV_CheckLE(prediction.size[1], vocLength, ""); + bool ctcFlag = true; + int lastLoc = 0; + for (int i = 0; i < prediction.size[0]; i++) { - CV_CheckEQ(prediction.dims, 3, ""); - CV_CheckType(prediction.type(), CV_32FC1, ""); - const int vocLength = (int)(vocabulary.size()); - CV_CheckLE(prediction.size[1], vocLength, ""); - bool ctcFlag = true; - int lastLoc = 0; - for (int i = 0; i < prediction.size[0]; i++) + const float* pred = prediction.ptr(i); + int maxLoc = 0; + float maxScore = pred[0]; + for (int j = 1; j < vocLength + 1; j++) { - const float* pred = prediction.ptr(i); - int maxLoc = 0; - float maxScore = pred[0]; - for (int j = 1; j < vocLength + 1; j++) + float score = pred[j]; + if (maxScore < score) { - float score = pred[j]; - if (maxScore < score) - { - maxScore = score; - maxLoc = j; - } + maxScore = score; + maxLoc = j; } + } - if (maxLoc > 0) - { - std::string currentChar = vocabulary.at(maxLoc - 1); - if (maxLoc != lastLoc || ctcFlag) - { - lastLoc = maxLoc; - decodeSeq += currentChar; - ctcFlag = false; - } - } - else + if (maxLoc > 0) + { + std::string currentChar = vocabulary.at(maxLoc - 1); + if (maxLoc != lastLoc || ctcFlag) { - ctcFlag = true; + lastLoc = maxLoc; + decodeSeq += currentChar; + ctcFlag = false; } } - } else if (decodeType.length() == 0) { - CV_Error(Error::StsBadArg, "Please set decodeType"); - } else { - CV_Error_(Error::StsBadArg, ("Unsupported decodeType: %s", decodeType.c_str())); + else + { + ctcFlag = true; + } } - return decodeSeq; } + struct PrefixScore + { + // blank ending score + float pB; + // none blank ending score + float pNB; + + PrefixScore() : pB(kNegativeInfinity), pNB(kNegativeInfinity) + { + + } + PrefixScore(float pB, float pNB) : pB(pB), pNB(pNB) + { + + } + }; + + struct PrefixHash + { + size_t operator()(const std::vector& prefix) const + { + // BKDR hash + unsigned int seed = 131; + size_t hash = 0; + for (size_t i = 0; i < prefix.size(); i++) + { + hash = hash * seed + prefix[i]; + } + return hash; + } + }; + + static + std::vector> TopK( + const float* predictions, int length, int k) + { + std::vector> results; + // No prune. + if (k <= 0) + { + for (int i = 0; i < length; ++i) + { + results.emplace_back(predictions[i], i); + } + return results; + } + + for (int i = 0; i < k; ++i) + { + results.emplace_back(predictions[i], i); + } + std::make_heap(results.begin(), results.end(), std::greater>{}); + + for (int i = k; i < length; ++i) + { + if (predictions[i] > results.front().first) + { + std::pop_heap(results.begin(), results.end(), std::greater>{}); + results.pop_back(); + results.emplace_back(predictions[i], i); + std::push_heap(results.begin(), results.end(), std::greater>{}); + } + } + return results; + } + + static inline + bool PrefixScoreCompare( + const std::pair, PrefixScore>& a, + const std::pair, PrefixScore>& b) + { + float probA = LogAdd(a.second.pB, a.second.pNB); + float probB = LogAdd(b.second.pB, b.second.pNB); + return probA > probB; + } + + virtual + std::string ctcPrefixBeamSearchDecode(const Mat& prediction) { + // CTC prefix beam seach decode. + // For more detail, refer to: + // https://distill.pub/2017/ctc/#inference + // https://gist.github.com/awni/56369a90d03953e370f3964c826ed4b0i + using Beam = std::vector, PrefixScore>>; + using BeamInDict = std::unordered_map, PrefixScore, PrefixHash>; + + CV_CheckType(prediction.type(), CV_32FC1, ""); + CV_CheckEQ(prediction.dims, 3, ""); + CV_CheckEQ(prediction.size[1], 1, ""); + CV_CheckEQ(prediction.size[2], (int)vocabulary.size() + 1, ""); // Length add 1 for ctc blank + + std::string decodeSeq; + Beam beam = {std::make_pair(std::vector(), PrefixScore(0.0, kNegativeInfinity))}; + for (int i = 0; i < prediction.size[0]; i++) + { + // Loop over time + BeamInDict nextBeam; + const float* pred = prediction.ptr(i); + std::vector> topkPreds = + TopK(pred, vocabulary.size() + 1, vocPruneSize); + for (const auto& each : topkPreds) + { + // Loop over vocabulary + float prob = each.first; + int token = each.second; + for (const auto& it : beam) + { + const std::vector& prefix = it.first; + const PrefixScore& prefixScore = it.second; + if (token == 0) // 0 stands for ctc blank + { + PrefixScore& nextScore = nextBeam[prefix]; + nextScore.pB = LogAdd(nextScore.pB, + LogAdd(prefixScore.pB + prob, prefixScore.pNB + prob)); + continue; + } + + std::vector nPrefix(prefix); + nPrefix.push_back(token); + PrefixScore& nextScore = nextBeam[nPrefix]; + if (prefix.size() > 0 && token == prefix.back()) + { + nextScore.pNB = LogAdd(nextScore.pNB, prefixScore.pB + prob); + PrefixScore& mScore = nextBeam[prefix]; + mScore.pNB = LogAdd(mScore.pNB, prefixScore.pNB + prob); + } + else + { + nextScore.pNB = LogAdd(nextScore.pNB, + LogAdd(prefixScore.pB + prob, prefixScore.pNB + prob)); + } + } + } + // Beam prune + Beam newBeam(nextBeam.begin(), nextBeam.end()); + int newBeamSize = std::min(static_cast(newBeam.size()), beamSize); + std::nth_element(newBeam.begin(), newBeam.begin() + newBeamSize, + newBeam.end(), PrefixScoreCompare); + newBeam.resize(newBeamSize); + std::sort(newBeam.begin(), newBeam.end(), PrefixScoreCompare); + beam = std::move(newBeam); + } + + CV_Assert(!beam.empty()); + for (int token : beam[0].first) + { + CV_Check(token, token > 0 && token <= vocabulary.size(), ""); + decodeSeq += vocabulary.at(token - 1); + } + return decodeSeq; + } + virtual std::string recognize(InputArray frame) { @@ -698,6 +870,12 @@ const std::string& TextRecognitionModel::getDecodeType() const return TextRecognitionModel_Impl::from(impl).decodeType; } +TextRecognitionModel& TextRecognitionModel::setDecodeOptsCTCPrefixBeamSearch(int beamSize, int vocPruneSize) +{ + TextRecognitionModel_Impl::from(impl).setDecodeOptsCTCPrefixBeamSearch(beamSize, vocPruneSize); + return *this; +} + TextRecognitionModel& TextRecognitionModel::setVocabulary(const std::vector& inputVoc) { TextRecognitionModel_Impl::from(impl).setVocabulary(inputVoc); diff --git a/modules/dnn/test/test_model.cpp b/modules/dnn/test/test_model.cpp index f7befa9937ae..6ac9702c6993 100644 --- a/modules/dnn/test/test_model.cpp +++ b/modules/dnn/test/test_model.cpp @@ -615,6 +615,25 @@ TEST_P(Test_Model, TextRecognition) testTextRecognitionModel(weightPath, "", imgPath, seq, decodeType, vocabulary, size, mean, scale); } +TEST_P(Test_Model, TextRecognitionWithCTCPrefixBeamSearch) +{ + if (target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + + std::string imgPath = _tf("text_rec_test.png"); + std::string weightPath = _tf("onnx/models/crnn.onnx", false); + std::string seq = "welcome"; + + Size size{100, 32}; + double scale = 1.0 / 127.5; + Scalar mean = Scalar(127.5); + std::string decodeType = "CTC-prefix-beam-search"; + std::vector vocabulary = {"0","1","2","3","4","5","6","7","8","9", + "a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"}; + + testTextRecognitionModel(weightPath, "", imgPath, seq, decodeType, vocabulary, size, mean, scale); +} + TEST_P(Test_Model, TextDetectionByDB) { if (target == DNN_TARGET_OPENCL_FP16) From 4300bb2e1f5eadd9b6eb1244ab2ed0250c2418b2 Mon Sep 17 00:00:00 2001 From: Iyad Ahmed Date: Thu, 12 Aug 2021 16:51:02 +0000 Subject: [PATCH 113/376] Merge pull request #20541 from iyadahmed:video_capture_timeout_prop * VideoCapture timeout set/get * Common formatting for enum values * Fix enum values wrongly in videoio.hpp * Define timeout enum values in public api and align with master --- modules/videoio/include/opencv2/videoio.hpp | 2 ++ modules/videoio/src/cap_ffmpeg_api.hpp | 4 ++- modules/videoio/src/cap_ffmpeg_impl.hpp | 33 +++++++++++++++++---- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index ba9c18bd97e1..aa247dd84eac 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -179,6 +179,8 @@ enum VideoCaptureProperties { CAP_PROP_BITRATE =47, //!< (read-only) Video bitrate in kbits/s CAP_PROP_ORIENTATION_META=48, //!< (read-only) Frame rotation defined by stream meta (applicable for FFmpeg back-end only) CAP_PROP_ORIENTATION_AUTO=49, //!< if true - rotates output frames of CvCapture considering video file's metadata (applicable for FFmpeg back-end only) (https://github.com/opencv/opencv/issues/15499) + CAP_PROP_OPEN_TIMEOUT_MSEC=53, + CAP_PROP_READ_TIMEOUT_MSEC=54, #ifndef CV_DOXYGEN CV__CAP_PROP_LATEST #endif diff --git a/modules/videoio/src/cap_ffmpeg_api.hpp b/modules/videoio/src/cap_ffmpeg_api.hpp index 984d36f23cf1..e6187655394d 100644 --- a/modules/videoio/src/cap_ffmpeg_api.hpp +++ b/modules/videoio/src/cap_ffmpeg_api.hpp @@ -30,7 +30,9 @@ enum CV_FFMPEG_CAP_PROP_CODEC_PIXEL_FORMAT=46, CV_FFMPEG_CAP_PROP_BITRATE=47, CV_FFMPEG_CAP_PROP_ORIENTATION_META=48, - CV_FFMPEG_CAP_PROP_ORIENTATION_AUTO=49 + CV_FFMPEG_CAP_PROP_ORIENTATION_AUTO=49, + CV_FFMPEG_CAP_PROP_OPEN_TIMEOUT_MSEC=53, + CV_FFMPEG_CAP_PROP_READ_TIMEOUT_MSEC=54 }; typedef struct CvCapture_FFMPEG CvCapture_FFMPEG; diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index 4164ab941c1c..937d34821573 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -186,8 +186,8 @@ extern "C" { #endif #if USE_AV_INTERRUPT_CALLBACK -#define LIBAVFORMAT_INTERRUPT_OPEN_TIMEOUT_MS 30000 -#define LIBAVFORMAT_INTERRUPT_READ_TIMEOUT_MS 30000 +#define LIBAVFORMAT_INTERRUPT_OPEN_DEFAULT_TIMEOUT_MS 30000 +#define LIBAVFORMAT_INTERRUPT_READ_DEFAULT_TIMEOUT_MS 30000 #ifdef _WIN32 // http://stackoverflow.com/questions/5404277/porting-clock-gettime-to-windows @@ -534,6 +534,8 @@ struct CvCapture_FFMPEG AVDictionary *dict; #endif #if USE_AV_INTERRUPT_CALLBACK + int open_timeout_ms; + int read_timeout_ms; AVInterruptCallbackMetadata interrupt_metadata; #endif @@ -568,6 +570,11 @@ void CvCapture_FFMPEG::init() frame_number = 0; eps_zero = 0.000025; +#if USE_AV_INTERRUPT_CALLBACK + open_timeout_ms = LIBAVFORMAT_INTERRUPT_OPEN_DEFAULT_TIMEOUT_MS; + read_timeout_ms = LIBAVFORMAT_INTERRUPT_READ_DEFAULT_TIMEOUT_MS; +#endif + rotation_angle = 0; #if (LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(52, 111, 0)) @@ -923,7 +930,7 @@ bool CvCapture_FFMPEG::open( const char* _filename ) #if USE_AV_INTERRUPT_CALLBACK /* interrupt callback */ - interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_OPEN_TIMEOUT_MS; + interrupt_metadata.timeout_after_ms = open_timeout_ms; get_monotonic_time(&interrupt_metadata.value); ic = avformat_alloc_context(); @@ -1227,7 +1234,7 @@ bool CvCapture_FFMPEG::grabFrame() #if USE_AV_INTERRUPT_CALLBACK // activate interrupt callback get_monotonic_time(&interrupt_metadata.value); - interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_READ_TIMEOUT_MS; + interrupt_metadata.timeout_after_ms = read_timeout_ms; #endif // get the next frame @@ -1483,6 +1490,12 @@ double CvCapture_FFMPEG::getProperty( int property_id ) const #else return 0; #endif +#if USE_AV_INTERRUPT_CALLBACK + case CV_FFMPEG_CAP_PROP_OPEN_TIMEOUT_MSEC: + return static_cast(open_timeout_ms); + case CV_FFMPEG_CAP_PROP_READ_TIMEOUT_MSEC: + return static_cast(read_timeout_ms); +#endif // USE_AV_INTERRUPT_CALLBACK default: break; } @@ -1677,6 +1690,14 @@ bool CvCapture_FFMPEG::setProperty( int property_id, double value ) return false; #endif break; +#if USE_AV_INTERRUPT_CALLBACK + case CV_FFMPEG_CAP_PROP_OPEN_TIMEOUT_MSEC: + open_timeout_ms = (int)value; + break; + case CV_FFMPEG_CAP_PROP_READ_TIMEOUT_MSEC: + read_timeout_ms = (int)value; + break; +#endif // USE_AV_INTERRUPT_CALLBACK default: return false; } @@ -3114,7 +3135,7 @@ bool InputMediaStream_FFMPEG::open(const char* fileName, int* codec, int* chroma #if USE_AV_INTERRUPT_CALLBACK /* interrupt callback */ - interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_OPEN_TIMEOUT_MS; + interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_OPEN_DEFAULT_TIMEOUT_MS; get_monotonic_time(&interrupt_metadata.value); ctx_ = avformat_alloc_context(); @@ -3241,7 +3262,7 @@ bool InputMediaStream_FFMPEG::read(unsigned char** data, int* size, int* endOfFi #if USE_AV_INTERRUPT_CALLBACK // activate interrupt callback get_monotonic_time(&interrupt_metadata.value); - interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_READ_TIMEOUT_MS; + interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_READ_DEFAULT_TIMEOUT_MS; #endif // free last packet if exist From cfb36443fb02586aac34cb14f6f67b551e29cb68 Mon Sep 17 00:00:00 2001 From: Julia Bareeva <34717687+JulieBar@users.noreply.github.com> Date: Fri, 13 Aug 2021 15:41:00 +0300 Subject: [PATCH 114/376] Merge pull request #20506 from JulieBar:lstm_activations * Support activations(Sigmoid, Tanh) for LSTM * fix warning --- modules/dnn/src/layers/recurrent_layers.cpp | 49 ++++++++++++++++++--- modules/dnn/src/onnx/onnx_importer.cpp | 38 +++++++++++++--- modules/dnn/test/test_onnx_importer.cpp | 5 +++ 3 files changed, 79 insertions(+), 13 deletions(-) diff --git a/modules/dnn/src/layers/recurrent_layers.cpp b/modules/dnn/src/layers/recurrent_layers.cpp index a6715aefca92..9088c13390cf 100644 --- a/modules/dnn/src/layers/recurrent_layers.cpp +++ b/modules/dnn/src/layers/recurrent_layers.cpp @@ -80,12 +80,31 @@ static void sigmoid(const Mat &src, Mat &dst) cv::pow(1 + dst, -1, dst); } +typedef void (*ActivationFunction)(const Mat &src, Mat &dst); +static ActivationFunction get_activation_function(const String& activation) { + // most used activations for PyTorch and TF : Tanh, Sigmoid + // if you need to support more optional activations use std::map instead + if (activation == "Tanh") + { + return tanh; + } + else if (activation == "Sigmoid") + { + return sigmoid; + } + else + { + CV_Error(Error::StsNotImplemented, + cv::format("Activation function [%s] for layer LSTM is not supported", activation.c_str())); + } +} + class LSTMLayerImpl CV_FINAL : public LSTMLayer { int numTimeStamps, numSamples; bool allocated; - MatShape outTailShape; //shape of single output sample + MatShape outTailShape; //shape of single output sample MatShape outTsShape; //shape of N output samples bool useTimestampDim; @@ -95,6 +114,10 @@ class LSTMLayerImpl CV_FINAL : public LSTMLayer bool reverse; // If true, go in negative direction along the time axis bool bidirectional; // If true, produces both forward and reversed directions along time axis + ActivationFunction f_activation; + ActivationFunction g_activation; + ActivationFunction h_activation; + public: LSTMLayerImpl(const LayerParams& params) @@ -145,6 +168,20 @@ class LSTMLayerImpl CV_FINAL : public LSTMLayer reverse = params.get("reverse", false); CV_Assert(!reverse || !bidirectional); + // read activations + DictValue activations = params.get("activations", ""); + if (activations.size() == 1) // if activations wasn't specified use default + { + f_activation = sigmoid; + g_activation = tanh; + h_activation = tanh; + } else { + CV_Assert(activations.size() == 3); + f_activation = get_activation_function(activations.getStringValue(0)); + g_activation = get_activation_function(activations.getStringValue(1)); + h_activation = get_activation_function(activations.getStringValue(2)); + } + allocated = false; outTailShape.clear(); } @@ -339,15 +376,15 @@ class LSTMLayerImpl CV_FINAL : public LSTMLayer Mat gatesIF = gates.colRange(0, 2*numOut); gemm(cInternal, blobs[5], 1, gateI, 1, gateI); gemm(cInternal, blobs[6], 1, gateF, 1, gateF); - sigmoid(gatesIF, gatesIF); + f_activation(gatesIF, gatesIF); } else { Mat gatesIFO = gates.colRange(0, 3*numOut); - sigmoid(gatesIFO, gatesIFO); + f_activation(gatesIFO, gatesIFO); } - tanh(gateG, gateG); + g_activation(gateG, gateG); //compute c_t multiply(gateF, cInternal, gateF); // f_t (*) c_{t-1} @@ -362,11 +399,11 @@ class LSTMLayerImpl CV_FINAL : public LSTMLayer if (usePeephole) { gemm(cInternal, blobs[7], 1, gateO, 1, gateO); - sigmoid(gateO, gateO); + f_activation(gateO, gateO); } //compute h_t - tanh(cInternal, hInternal); + h_activation(cInternal, hInternal); multiply(gateO, hInternal, hInternal); //save results in output blobs diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index b833b2ea443f..32b56278bda7 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -244,6 +244,10 @@ static DictValue parse(const ::google::protobuf::RepeatedField< ::google::protob return DictValue::arrayInt(&dst[0], src.size()); } +static DictValue parseStr(const ::google::protobuf::RepeatedPtrField< ::std::string>& src) { + return DictValue::arrayString(src.begin(), static_cast(src.size())); +} + LayerParams ONNXImporter::getLayerParams(const opencv_onnx::NodeProto& node_proto) { LayerParams lp; @@ -301,6 +305,10 @@ LayerParams ONNXImporter::getLayerParams(const opencv_onnx::NodeProto& node_prot CV_Assert(attribute_proto.ints_size() == 1 || attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3); lp.set("dilation", parse(attribute_proto.ints())); } + else if(attribute_name == "activations" && node_proto.op_type() == "LSTM") + { + lp.set(attribute_name, parseStr(attribute_proto.strings())); + } else if (attribute_proto.has_i()) { ::google::protobuf::int64 src = attribute_proto.i(); @@ -997,18 +1005,32 @@ void ONNXImporter::parseLSTM(LayerParams& layerParams, const opencv_onnx::NodePr lstmParams.name += "/lstm"; // https://pytorch.org/docs/stable/nn.html#lstm - CV_Assert(node_proto.input_size() == 7); + CV_Assert(node_proto.input_size() >= 7); Mat Wx = getBlob(node_proto, 1); Mat Wh = getBlob(node_proto, 2); Mat b = getBlob(node_proto, 3); - Mat h0 = getBlob(node_proto, 5); - Mat c0 = getBlob(node_proto, 6); - - b = b.reshape(1, b.size[0]); const int numHidden = lstmParams.get("hidden_size"); const int numDirs = Wx.size[0]; // Is 1 for forward only and 2 for bidirectional LSTM. const int numFeatures = Wx.size[2]; + + Mat h0, c0; + if (!node_proto.input(5).empty()) { + h0 = getBlob(node_proto, 5); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + } else { + // initial_h attribute can be empty in case of keras2onnx producer. fill it with zeros + h0 = Mat::zeros(numDirs * numFeatures, numHidden, CV_32FC1); + } + if (!node_proto.input(6).empty()) { + c0 = getBlob(node_proto, 6); + c0 = c0.reshape(1, c0.size[0] * c0.size[1]); + } else { + // initial_c attribute can be empty in case of keras2onnx producer. fill it with zeros + c0 = Mat::zeros(numDirs * numFeatures, numHidden, CV_32FC1); + } + + b = b.reshape(1, b.size[0]); Mat bx = b.colRange(0, b.cols / 2); Mat bh = b.colRange(b.cols / 2, b.cols); b = bx + bh; @@ -1036,8 +1058,7 @@ void ONNXImporter::parseLSTM(LayerParams& layerParams, const opencv_onnx::NodePr } Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); - h0 = h0.reshape(1, h0.size[0] * h0.size[1]); - c0 = c0.reshape(1, c0.size[0] * c0.size[1]); + lstmParams.blobs.resize(5); lstmParams.blobs[0] = Wh; @@ -1045,6 +1066,9 @@ void ONNXImporter::parseLSTM(LayerParams& layerParams, const opencv_onnx::NodePr lstmParams.blobs[2] = b; lstmParams.blobs[3] = h0; lstmParams.blobs[4] = c0; + + // read direction attribute + lstmParams.set("reverse", lstmParams.get("direction", "") == "reverse"); lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 05f77730af07..a446a37c7944 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -665,6 +665,11 @@ TEST_P(Test_ONNX_layers, Split_EltwiseMax) testONNXModels("split_max"); } +TEST_P(Test_ONNX_layers, LSTM_Activations) +{ + testONNXModels("lstm_cntk_tanh", pb, 0, 0, false, false); +} + TEST_P(Test_ONNX_layers, LSTM) { testONNXModels("lstm", npy, 0, 0, false, false); From 917cd13ce24cd89770b78769d7bc9729277ff4c8 Mon Sep 17 00:00:00 2001 From: Iyad Ahmed Date: Fri, 13 Aug 2021 20:12:05 +0000 Subject: [PATCH 115/376] Merge pull request #20549 from iyadahmed:video_capture_timeout_set_get * VideoCapture add open/read timeout params to FFMPEG backend * Fix wrong enum name * Fix wrong enum name --- modules/videoio/include/opencv2/videoio.hpp | 2 ++ modules/videoio/src/cap_ffmpeg_impl.hpp | 25 +++++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 348448bda7a8..16016e4b8e9a 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -186,6 +186,8 @@ enum VideoCaptureProperties { CAP_PROP_HW_ACCELERATION=50, //!< (**open-only**) Hardware acceleration type (see #VideoAccelerationType). Setting supported only via `params` parameter in cv::VideoCapture constructor / .open() method. Default value is backend-specific. CAP_PROP_HW_DEVICE =51, //!< (**open-only**) Hardware device index (select GPU if multiple available). Device enumeration is acceleration type specific. CAP_PROP_HW_ACCELERATION_USE_OPENCL=52, //!< (**open-only**) If non-zero, create new OpenCL context and bind it to current thread. The OpenCL context created with Video Acceleration context attached it (if not attached yet) for optimized GPU data copy between HW accelerated decoder and cv::UMat. + CAP_PROP_OPEN_TIMEOUT_MSEC=53, //!< (**open-only**) timeout in milliseconds for opening a video capture (applicable for FFmpeg back-end only) + CAP_PROP_READ_TIMEOUT_MSEC=54, //!< (**open-only**) timeout in milliseconds for reading from a video capture (applicable for FFmpeg back-end only) #ifndef CV_DOXYGEN CV__CAP_PROP_LATEST #endif diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index 1e73cb8fc881..9ec75501d040 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -183,8 +183,8 @@ extern "C" { #endif #if USE_AV_INTERRUPT_CALLBACK -#define LIBAVFORMAT_INTERRUPT_OPEN_TIMEOUT_MS 30000 -#define LIBAVFORMAT_INTERRUPT_READ_TIMEOUT_MS 30000 +#define LIBAVFORMAT_INTERRUPT_OPEN_DEFAULT_TIMEOUT_MS 30000 +#define LIBAVFORMAT_INTERRUPT_READ_DEFAULT_TIMEOUT_MS 30000 #ifdef _WIN32 // http://stackoverflow.com/questions/5404277/porting-clock-gettime-to-windows @@ -523,6 +523,8 @@ struct CvCapture_FFMPEG AVDictionary *dict; #if USE_AV_INTERRUPT_CALLBACK + int open_timeout; + int read_timeout; AVInterruptCallbackMetadata interrupt_metadata; #endif @@ -569,6 +571,11 @@ void CvCapture_FFMPEG::init() #endif dict = NULL; +#if USE_AV_INTERRUPT_CALLBACK + open_timeout = LIBAVFORMAT_INTERRUPT_OPEN_DEFAULT_TIMEOUT_MS; + read_timeout = LIBAVFORMAT_INTERRUPT_READ_DEFAULT_TIMEOUT_MS; +#endif + rawMode = false; rawModeInitialized = false; memset(&packet_filtered, 0, sizeof(packet_filtered)); @@ -928,6 +935,16 @@ bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters& if (params.has(CAP_PROP_HW_ACCELERATION_USE_OPENCL)) { use_opencl = params.get(CAP_PROP_HW_ACCELERATION_USE_OPENCL); } +#if USE_AV_INTERRUPT_CALLBACK + if (params.has(CAP_PROP_OPEN_TIMEOUT_MSEC)) + { + open_timeout = params.get(CAP_PROP_OPEN_TIMEOUT_MSEC); + } + if (params.has(CAP_PROP_READ_TIMEOUT_MSEC)) + { + read_timeout = params.get(CAP_PROP_READ_TIMEOUT_MSEC); + } +#endif if (params.warnUnusedParameters()) { CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: unsupported parameters in .open(), see logger INFO channel for details. Bailout"); @@ -937,7 +954,7 @@ bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters& #if USE_AV_INTERRUPT_CALLBACK /* interrupt callback */ - interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_OPEN_TIMEOUT_MS; + interrupt_metadata.timeout_after_ms = open_timeout; get_monotonic_time(&interrupt_metadata.value); ic = avformat_alloc_context(); @@ -1282,7 +1299,7 @@ bool CvCapture_FFMPEG::grabFrame() #if USE_AV_INTERRUPT_CALLBACK // activate interrupt callback get_monotonic_time(&interrupt_metadata.value); - interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_READ_TIMEOUT_MS; + interrupt_metadata.timeout_after_ms = read_timeout; #endif #if USE_AV_SEND_FRAME_API From 9ef41f68fbec10b41948f10d86a2f8f8b8cff56a Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Mon, 16 Aug 2021 15:44:54 +0300 Subject: [PATCH 116/376] fix Split partial sum --- modules/dnn/src/onnx/onnx_importer.cpp | 2 +- modules/dnn/test/test_onnx_importer.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 32b56278bda7..6da2c5edf6b1 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -851,7 +851,7 @@ void ONNXImporter::parseSplit(LayerParams& layerParams, const opencv_onnx::NodeP std::vector slicePoints(numSplits - 1, splits.get(0)); for (int i = 1; i < splits.size() - 1; ++i) { - slicePoints[i] = slicePoints[i - 1] + splits.get(i - 1); + slicePoints[i] = slicePoints[i - 1] + splits.get(i); } layerParams.set("slice_point", DictValue::arrayInt(&slicePoints[0], slicePoints.size())); } diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index a446a37c7944..983f72d6d688 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -617,6 +617,7 @@ TEST_P(Test_ONNX_layers, Split) testONNXModels("split_2"); testONNXModels("split_3"); testONNXModels("split_4"); + testONNXModels("split_sizes"); } TEST_P(Test_ONNX_layers, Slice) From 8dcec034ed6b52b2074573a56573883698e446b0 Mon Sep 17 00:00:00 2001 From: zyp Date: Mon, 16 Aug 2021 18:20:10 +0200 Subject: [PATCH 117/376] Merge pull request #18694 from zyp:gstreamer_gray16 * videoio/gstreamer: Add support for GRAY16_LE. * videoio/gstreamer: added BGRA/BGRx support Co-authored-by: Maksim Shabunin --- modules/videoio/src/cap_gstreamer.cpp | 23 +++++++++++++++++++++-- modules/videoio/test/test_gstreamer.cpp | 12 ++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index 60ecf6611a87..e040a22cb0f1 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -475,8 +475,9 @@ bool GStreamerCapture::retrieveFrame(int, OutputArray dst) // video/x-raw, format=I420 -> 8bit, 1 channel (height is 1.5x larger than true height) // video/x-bayer -> 8bit, 1 channel // image/jpeg -> 8bit, mjpeg: buffer_size x 1 x 1 + // video/x-raw, format=GRAY16_LE (BE) -> 16 bit, 1 channel + // video/x-raw, format={BGRA, RGBA, BGRx, RGBx} -> 8bit, 4 channels // bayer data is never decoded, the user is responsible for that - // everything is 8 bit, so we just test the caps for bit depth Size sz = Size(frame_width, frame_height); guint n_planes = GST_VIDEO_INFO_N_PLANES(&info); if (name == "video/x-raw") @@ -507,6 +508,24 @@ bool GStreamerCapture::retrieveFrame(int, OutputArray dst) src.copyTo(dst); return true; } + else if (format == "GRAY16_LE" || format == "GRAY16_BE") + { + CV_CheckEQ((int)n_planes, 1, ""); + size_t step = GST_VIDEO_INFO_PLANE_STRIDE(&info, 0); + CV_CheckGE(step, (size_t)frame_width, ""); + Mat src(sz, CV_16UC1, map_info.data + GST_VIDEO_INFO_PLANE_OFFSET(&info, 0), step); + src.copyTo(dst); + return true; + } + else if (format == "BGRA" || format == "RGBA" || format == "BGRX" || format == "RGBX") + { + CV_CheckEQ((int)n_planes, 1, ""); + size_t step = GST_VIDEO_INFO_PLANE_STRIDE(&info, 0); + CV_CheckGE(step, (size_t)frame_width, ""); + Mat src(sz, CV_8UC4, map_info.data + GST_VIDEO_INFO_PLANE_OFFSET(&info, 0), step); + src.copyTo(dst); + return true; + } else if (format == "UYVY" || format == "YUY2" || format == "YVYU") { CV_CheckEQ((int)n_planes, 1, ""); @@ -1008,7 +1027,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam sink_pad.attach(gst_element_get_static_pad(sink, "sink")); peer_caps.attach(gst_pad_peer_query_caps(sink_pad, NULL)); if (!gst_caps_can_intersect(caps, peer_caps)) { - caps.attach(gst_caps_from_string("video/x-raw, format=(string){UYVY,YUY2,YVYU,NV12,NV21,YV12,I420}")); + caps.attach(gst_caps_from_string("video/x-raw, format=(string){UYVY,YUY2,YVYU,NV12,NV21,YV12,I420,BGRA,RGBA,BGRx,RGBx,GRAY16_LE,GRAY16_BE}")); CV_Assert(caps); } } diff --git a/modules/videoio/test/test_gstreamer.cpp b/modules/videoio/test/test_gstreamer.cpp index ca100367b126..207f6de50baa 100644 --- a/modules/videoio/test/test_gstreamer.cpp +++ b/modules/videoio/test/test_gstreamer.cpp @@ -35,6 +35,10 @@ TEST_P(videoio_gstreamer, read_check) cvtColor(decode_frame, rgb_frame, convertToRGB); cvtColor(rgb_frame, gray_frame, COLOR_RGB2GRAY); + if (gray_frame.depth() == CV_16U) + { + gray_frame.convertTo(gray_frame, CV_8U, 255.0/65535); + } vector circles; HoughCircles(gray_frame, circles, HOUGH_GRADIENT, 1, gray_frame.rows/16, 100, 30, 1, 30 ); @@ -58,6 +62,10 @@ TEST_P(videoio_gstreamer, read_check) static const Param test_data[] = { make_tuple("video/x-raw, format=BGR" , Size(640, 480), Size(640, 480), COLOR_BGR2RGB), + make_tuple("video/x-raw, format=BGRA" , Size(640, 480), Size(640, 480), COLOR_BGRA2RGB), + make_tuple("video/x-raw, format=RGBA" , Size(640, 480), Size(640, 480), COLOR_RGBA2RGB), + make_tuple("video/x-raw, format=BGRx" , Size(640, 480), Size(640, 480), COLOR_BGRA2RGB), + make_tuple("video/x-raw, format=RGBx" , Size(640, 480), Size(640, 480), COLOR_RGBA2RGB), make_tuple("video/x-raw, format=GRAY8", Size(640, 480), Size(640, 480), COLOR_GRAY2RGB), make_tuple("video/x-raw, format=UYVY" , Size(640, 480), Size(640, 480), COLOR_YUV2RGB_UYVY), make_tuple("video/x-raw, format=YUY2" , Size(640, 480), Size(640, 480), COLOR_YUV2RGB_YUY2), @@ -76,6 +84,10 @@ static const Param test_data[] = { make_tuple("video/x-raw, format=NV21" , Size(322, 242), Size(322, 363), COLOR_YUV2RGB_NV21), make_tuple("video/x-raw, format=YV12" , Size(322, 242), Size(322, 363), COLOR_YUV2RGB_YV12), make_tuple("video/x-raw, format=I420" , Size(322, 242), Size(322, 363), COLOR_YUV2RGB_I420), + + // 16 bit + make_tuple("video/x-raw, format=GRAY16_LE", Size(640, 480), Size(640, 480), COLOR_GRAY2RGB), + make_tuple("video/x-raw, format=GRAY16_BE", Size(640, 480), Size(640, 480), COLOR_GRAY2RGB), }; INSTANTIATE_TEST_CASE_P(videoio, videoio_gstreamer, testing::ValuesIn(test_data)); From a50dec88d5e39a24438f9215a2fa4ee4cc6ae3ab Mon Sep 17 00:00:00 2001 From: rogday Date: Tue, 17 Aug 2021 18:23:27 +0300 Subject: [PATCH 118/376] Merge pull request #20547 from rogday:gdb_pretty_printer * add gdb rpetty printer for cv::Mat * address review comments --- .../linux_gcc_cmake/linux_gcc_cmake.markdown | 2 +- .../images/example.png | Bin 0 -> 35727 bytes .../linux_gdb_pretty_printer.markdown | 38 ++++ .../linux_install/linux_install.markdown | 2 +- .../table_of_content_introduction.markdown | 1 + samples/gdb/gdbinit | 23 ++ samples/gdb/mat_pretty_printer.py | 212 ++++++++++++++++++ 7 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 doc/tutorials/introduction/linux_gdb_pretty_printer/images/example.png create mode 100644 doc/tutorials/introduction/linux_gdb_pretty_printer/linux_gdb_pretty_printer.markdown create mode 100644 samples/gdb/gdbinit create mode 100644 samples/gdb/mat_pretty_printer.py diff --git a/doc/tutorials/introduction/linux_gcc_cmake/linux_gcc_cmake.markdown b/doc/tutorials/introduction/linux_gcc_cmake/linux_gcc_cmake.markdown index eb59fea209d6..ee3f1eb7f9b9 100644 --- a/doc/tutorials/introduction/linux_gcc_cmake/linux_gcc_cmake.markdown +++ b/doc/tutorials/introduction/linux_gcc_cmake/linux_gcc_cmake.markdown @@ -1,7 +1,7 @@ Using OpenCV with gcc and CMake {#tutorial_linux_gcc_cmake} =============================== -@prev_tutorial{tutorial_linux_install} +@prev_tutorial{tutorial_linux_gdb_pretty_printer} @next_tutorial{tutorial_linux_eclipse} | | | diff --git a/doc/tutorials/introduction/linux_gdb_pretty_printer/images/example.png b/doc/tutorials/introduction/linux_gdb_pretty_printer/images/example.png new file mode 100644 index 0000000000000000000000000000000000000000..0ec673dcc21d74eb626b3f9b7a627f029c781873 GIT binary patch literal 35727 zcmbrlW0+-Iuq~LCwr$&XW~FUb+N`u~+qP}nwr$(4&N}Da+kN|~-`DYD`kp5u7SFF0RZp;#D#t;xnx{qIy-1Cdmi&y zA4v-j=OA?g0fX0s3L^`L){(2%8IdJnH=7a-N1+TmW$a(4g=x0RE1jVmkLa#J21nl^}d$n#8@U_4Y~&IuYU z&FwrjQd8%U)UZJx%_>wfDK{JV=m3}E(zWPg(3PSt-5M7(*wS!Lix?}Xf`Du|6>_sO zM1Rht1kHd0+a4+y3on~6;&EQA@@ug*fnn>OmQRDU;-fmA^QqGy4JRzLy4y-$0BwKw z)f;G(6efmaEx)dwQcAfvu&R|63;8LbT}`8{zZwrHtQLJ9spzUa(#~^S#m6K9kZ)r`I`(cn zT<}P^MNX8ej`;3ki#85SCv0BrID&+oN{H+=MEntxO~GV{=;FwtWP=PQcp{aOu|T1b-QMq-|)kGaV3}-4{ z!`Om~kQl4^1W1A~AtxwUWCo;|n7F*tQo!qpQ)sta|G+`UE-liZYJ;W2f)nF6a?;$K zT%>XAzbo&C)xb9-E$Za2ZzToJL`~*gwudl`YF4-$1#Qz_lADiIW>eWYSSF9zf?Da; zN2amEk;*sbk%w4+ACz0Gp@&Q!aO^+dw-&)aJ7c{%XZKc#n{LWw^zh! zA?I!p&t9dTb0xN?_ZH~wtnB?5!OE(GOF|hS;whQvbo?U%xCSs~GbL!jR=WXg%I%hgKMR zik_mTg*P(zleb%%fLZ@=Fm(K>vbk5N+&0$-Fl4krnqToJvPL*A8m1$NJacRyl zT{9k{`#S2cb@LhiSa3qy-1!h^>wxxpsSeW!8`Zif|6n8i0dNXkhM{cl1#$YLStKHn zGK`P`X;y#A`ww*>?*^KlVxl+?&Ut$Z zVu7V-nyiZu*F+5$Dj_4qNuNZKG3|4nD+qQ!tEE8kIrH6>l4FUo=-nv^*&`rD+Ea

Q(%%|WTqhQs~!=|nUv!kcA~{jVq#P>iWFDX^L^Gw>VMb4%l#d#%X-bh74JL( z4j+4>3;Zk0U_oq^|EX980&kuHrWgYqq8ezT62`}_ky5YSVHE+X1k^c)>yVW4DRZMM zX7xKUxP*%@^8BGo!7FeOjDsa?U9Q}ri&`#ZYBn?pu8m=wZwm7{_nd8o=uxjU!4^qx zAW@JcRyZXCJ*Kfxu6g85th>NLn{6OVsAQCFo@E+9^@ zh{;`r<`v-*A`(cBI509f{X`h2aM3mtT#^v^DLq|^9WXzGjXv>!by9#UJ~j!f@uB9+ z-ETz&==3rJX7a&T_a!8hlvO8y9zNXz0Y#sqlvpAndwnTFL?A;LBzURNS{XXqvc-wp z)#Y;bRD@g~p$7>q*WeB7REKbW zWJ+hUIa16TEa(3QzaJ1`KlYDc4z>U8M6#T(UheOt{J4BIHjqdd^w?&sLu(6p1C*D* z!5$hKf4P=!+K6j9G-|j|kqFrfC-`qN@&>0?<0ZH_3kJAjqBtP%Y-XxtS){IS;n&(# zqwO+4>2Q(XPPqlrI6{&VZ^VnV8R0|LLJ*OCAY9F3ursn+Q_3#xK1c%RaeDm9=>Cxp z9lO$Z$`?J~vzOW=hAg&34-^pLaa}Xijzze!jNMxi4<1NzA@e$C&SjDuh*t=@s*1X| znHI2aQE-py?f$)nzJq#}m^2<2?NnrE)xgSgOm^k|=*R0SmyO1C1^l1$FLLY7go)~^ zXYr|jce6l9YRY+3*3V`kH`|a-VR=jEh-VQZDve#7M+5~yH_)Uo2&8`&C#-Nt2^Rmm zD^)c+i_>qII+{ac4f{&D)h?D1nfi%v(FuppPME(VxM3BPI)?aXEbtXSc0&>WUe1rN z24p93d~DqW^H0Cvz|A;)NF@I)c>)ax`Okj|2>s`^1bM)0IGNgL$K*%p(F8QvclzrS zRae%|#5iphw=q@99N$gZK-p+me!iS`hW-|Kh_0?wM*8J49kuBpwLCfu_smc?qyWq$ zW{j1%c!+Fxg3jR?aYf?HxE$N;MRqRziOtO;bpX1S0v`0>F&6&e0;;wANuHyDF{(0o z6@&~Lm7T}uazEZ`TOilinF_erZzGGwQ+0!QTGx7g8eO~oiS7O6kAD50{~U9_)P29W zJZxRk*vyxQ2hxn!92nB}bwWStM$P#teE|;?X|&iO$u^|()MA~6q8gpGF~%to$L-Jr zs|QD&tRg^nZe4>mCgmkff}zOM5UKSkW;H>c+f)aTu1tz*!?ZCkoXsV$mtI|{VE-%< z76DZ)^N4%=!f%6OR+ZoBhSA@#P3C5ST6idYP)KGVL~!EFu(CN^!q?_;7(S++T5i@1 zYoS756xh0=FnO{F?n?pYx%s&Q1m>vq|yg7%06qh|oA*%KaN!6C_Kan|@ub)@KgQS;m3y zVO$@bSiEyEdA*VeKDP4>gR!FTEzC~adq3-BbR{^nKH=nA*m-4>$$y@Rx_B$&ZGYWS zL06UGf%s{JpBG^$C@Q+@UVnCcj8OptHor|znRR)eka$H5O}^7v7%GZ$z{$$qCG)UN zhFJIn9vJfce8=~n(m18(fe@dYVTLiNi#*VMo=_GSm|tqzu1HXt7=kHG@icgN1ZFr! zmfu_dtgsD@N$ggwd0bxMP@AnL-iBnYL_pJ9l zsb#$w8xtm;T5McTOB9uu=oj+_5YCp`Y9nN|oUy%gVY4@QyYoaOVm{Qj!H?X#pjM#P zeI72S5F!W8l>-l%eCH01s*t|k9}T9eeF-w-@dj?!>Cea#b(IDhEQYl&>5Nu$@yycB`jWJQKxZX`!X} z3Bpyij3C=)^OzShxD`)TdPtZEih{zt%f~I*%PyR^jk19G<-9{Ve|sw6nT$ZN3>d>B zDi_$0!oX5wsuaw_J`@FT9%)#a1-ba6-H{<*Ip3SCsFN83MJ>cs*7*PikjUqWlVwf= zb9gT*m~AGiYL4iVU=LGofF}h(yjU8RVfB$w&Lg&9m-rDjXTKNDGxJeQa&Es>chdBy z`C8@P1$&gecGIlS#MPUnokCmcgFv5WURg=~wWEihODCk_76pEgv&0~?<7;G`-Stsu zFy$;<$5gk+A9AGofFq0Qli6~3`{sG9^1QqzL_tX^s0LFv7{}Hf&W$M{f#0ep9$)DW zQ}4DmO9n^w(TnxGf0ChLeyq_4d)?_^<6I8&yzj_HjeqnaotsfF&u?*F{|@0iWp_;b zA~BaN@B}Cd^Yqxeog{v5H^ri#jcIHtIDD7KZ$W9bBBX=|zw5{Q@)^eS-YW}SSR51* zxBTrD<4x{x6Nn~be=_$h0h+zq93G54?E26U9_-9LaeSzP%e4~j&Z?tr#gbh?Wvmev zt2Q#8o2AkKRIK1YRUs=|q;4=m zJm}2-7XtH6%%AbpgLqCbgdyDp*eiLrbIcd zq+ps+AJp4@^s`(87xSm#-m#}Ca9AAiRwJe4gBYt>B$(G!Xbhk6KVX7eqaZB9en0!f zMnFG19H^4(KkTI&HYGbX41iIM!Uy$D#l(`2qLD7+6ES#iv8#i~ek3+6d&yi6Xo{q+ zWv-7HrzB@-kkcEVg-?Pv+Mrc-%9l}^mn3MIWj}}0RyGBBY;>%qr33)##yTrQh{~Zg z;1mVDp%vM--xBGuru!Hj?;gkMzWdbFXV1U8n{q5luWd~aAjpqD=JVTF*l7v+R29bj z;=(i;CQG3m)iVIaPJ?2TsGGHnvbEe0rgq&aU7T*FeW@`_zTF+VETFW$ zdCi4K%b-n|gFaK-?IHHb3Yt)w!77mr6x0SteZ&~u!Pt?~w}QBNxEtQA7oqf;!iam7 zs*Rk)u(6rS&dL(cr|lyMMvI}CoHEHD?XBPRj3{S3w>)=ozSOdm;F(k;6R{zi$)XzB z7*dqomUPl2(!zE*!g%7^+l-$%<|?*mgf80~^R9w8wdeHP@r3a( ziHQd-Yc<1jB^6Vmb`pb@4qS)y6A0n_J3Znd1!_>sLTlSweHC#0gMmuVu| ztx!6dHEOw8CP9PYa*(r;!;Bs0g&R8MQ4P@)W2Fu zm#z}GX&^c+dlD@3D+sYVKB1>e&w<~8l}Vb!e5gt3onp6Er?0x6T%DKX<*fpCS$}oj z+D#!8brzQ$e?t(t`x-Cnhwn^R5Vda zhz|{4${hrh7>dgfy6d2-weiWxtIYHCY4-OQHQ_Cyi8Va^dyY$2xnn!x!yg2qY45P{ z-61?}btfNZd4IX7x`Kpx`t8>eHXRn-DL*3vG-Y5@DuHb`4!6`{#kp?3>DPU3dOX;0 zX<^P-lQq11Wzo_RdzYj$6vwmPt?JhI7{V7wrTxw^S7BY5*n#mi%;!+d>mC;F%iZf0 ze~OOM`dX*&rD*iIY(3pcy9GdAVH{P+%*fC&?U{ORdow;eJ>6A(Y=1RY=D{1oJ}kc~ zNL;U9i7G|)y4^&G2(O-gAkP|QPbT46%J8UzhLqXPpl4E4QWsNj2|y}tyQQ6>ep6q^ zd3czvn!;+zPidAP)1;G6&Bz9j;Ac$MuhZ4#F8TkwbP@WqbPlxS(W=6rP|! z&SiFLr<*2=c za-yG+SB##(y%IV3Q1=GT?!f`iUQYpt7sq{BW6#OoP)pR)j578X6_oBYA9!5-w6$vG z>Ts)4om$v(Ra*qoXD>J3DBzT1FhMRGfB=toQ0%DyrYA-s4Mf1|IZSaV zc1nels8MM-G5`7`<93I5SxyyvA8gHrRQntnWBuiX?3xG-e`PqFkF&ABMA~G%&!8O& zLi2?{6fK-Zl1TQ$Yu`^2={t~5!P+VVIB zEo-b!?+Sx7!Q3E!YpOgy^{9*Xu*?PHO5?QnP{4ts@uFgy@XWM$okt^w zA-+N>^z^It#eV}OAoW2crzawiiVTGq!h;tNlOjc`CXJtUkZVXc+F!mp4Dlq7DsQh- z#@8mZvXOq@xVR5!%&_T3}$A2@=LQYzaXpjqIwv}BwXmunU2Zr1`s2TZ(4 zARuPWkzUG_+c>-#2Hw^ll`uG4OZd$_d4%WKyUiUdGrSqPP`$pixKdMvnKv2}oxhIW zclo};k*WfhBugDTp8cjtl3X-D%Z+oQ{clkjrndF}4IIu0+Iyy`NK;dG$Yo*)mBz`Gl4lq0C z?(zQNs%1hD#>gn?HqSt@sKLf|hIBo%9}!vlMM!3TmXHSK_7hw15X5hq9p8rVk8ry5 z%T>fo9GN7ocT`x$T!Grz6kT^zEp0)mgfomB#}%Q=HbTfsr#B!U%O3E6{?KaTaFP6+ z<>}W_`XV%W5>B{T{A-uK`_iJ7_BSD`k!xv z4ZW%y-AUImCDFCKcm|$44NV*`k+M8PTUv=3Z?4OSBg_^U(H z$V5Dyw?N~ni@d%&Q~Pb9B^D$(RVa!EMA#NiYPwptV2`nFrO? zA!rKdnV)u-m{=wiZ6TMeNUj`a4y&?EyK)?K%XK{SemrC#+d^C>NN-DvA(I2*`37Du zG{D1et{e^VVBw4}aH6=*r=A|xmLSKiy#dl-PrCHGPt>|J4Dtwk2AiLTM{N4z%yW1l z{KQh|^q4T3r3*>=+QF%JW|ov>X67MPtZY9(9#Tn|6Nj=*JM;p8q=xayI50wINWvB4 z;_#33-{#7V@SEE*B4Uu*jq@9-(BDga3wfA-`S8QXz)-ImgwF@`Gi~|Ds;mv6L4}d` zqQgt~X#4vC*82{Q^H4FJpPf@MOU-$e0S%)llgdr~$~xb3s(#v3=#Y{TY+4LMK-(3{ z&ZY3LFiZ@!-;AhqTaJFa{#KxJ)8Mf@$ zZiUows-Ob}aiV}tG6-FWd=@2e={f%-9qoyFYU7rZA0iD|#udN08m!WBhj!!%q=Dbs zI|Jms4XiO`Hx3B>FqKarFBoBj_dP+5pT|6GeJy}_B4#gSi!MM`aDX)M?2MRNES(nC z=&QpTCYDi+_?W=1gPBVGj5Wk^Wlc?4k2GnDhFMKqFc!t&-js>{)z>shvVg|L!9b6* zolj64n+05UJht;fGC$z);ujOE|MS#NyOxqh&z%T2gEw-l!*K1ZNqI=K?BO}xG;~B)f+XuFLIjiln|D?mm z3w4$eT55#dQHKBJGxP0E6|voS)@a%dx}Cj581_5I!IjpvX43Kbp1-v()x>*<;`vu| zk=pOSCs+76XmV#rwIcX26?EKh6IWp{%Q7eR^&Rf@Kl8MWL>4;QOy3W)E-;dFN&fCk2@-J z@JKgsVU_}=aT=SYtT0bqcpViC$4F&0Qoj`Yu+jM9)648l9kFB)VI$bH6A-sfN=hJ2 z>`^^?agJ95H1tSx_2#auhnVzIIpj=&(1{Li%RA;ypegm(M6on8Q1=UD=7d@!MZ|m= zyxfui%euU_cJzz|#QJ1>f31FGOX@tc@Yvm347gYVA`Ds@rRDhem zs2wL+Bzn;% z$d{cd=eyAD(!#?%_$r?;0*-jc!@(DA@1Wy0x<7ag*&LHQRi2g`0AKTT^R;L9F=4Ck zv6m^qpf&IM=y8?_ELJ&lJ9{&uTawqxT8cErI$%x)6pxPgb-`K>51fcu^g916fY4@T z=A6BgV-Rl$yg9hr`_MygT33+c^E5Ca(&1On5X^hl=D8Zo%vfXjKRpyTSdA8_EaAfw}B= zbo7HY4C&8H#IZ#nW_aei;WR)KlPx4`S?50|76(J`^~xmfN9bw^Ead&ZY&YH#?-4PI zAN@kl>bND!?P5t1OAz(e&Pik%l&??$PrdhiG362P^S!E}&&{1z#FDbmk^ zgyTDXe@5Y+_h*Z7j7dzh)H>0aZL?2c?A{Ic3+1+j4mP> z?>zoF$QEs`b+_PIOn25}F%9^{B@N=lxS@IDJ8AC;oq8v)dbB@M`g8Wjc?`0Pf=4-$ z%S?iSK=^|_c>2C?8rJS^&Ta|Xk`+S1eD(Rf5$e7j_CC@Y8~+vuMP>E-P=ys(bG-J{ zXRBWaM7mz9O_r^liJ1BiEr8I}%}Z&mYx$BPZl`|!OnPWy*l&$d=geKXXXOfs4v?B8 zOMR2rY{g*C*0RPQe(UKCkXNe-y*IV-4vaD+nV+!H9NJ6Toy3zjauir3a}y2|IdW zO&9A>gN=uj3eqHpJ#%@1-jJJja9}oYf;=nWAc3#FT~by$M;(lVdb+Ir0~EHnmU45R z?!>a)8<`tA(O<8(G~M}`VfD?CxUF${YvvCoQS+WunS#{t#1V;(jVuSl+z!o8wMQ%G z7NM=N&wqNkC^XKzUa8Hl=^JPebaRV*x)`C=H%PmrjiWWG z*&{HQi>f>p%%lsLd1Z8Htb`=U7|MJd{0u5 zr$_6`D4GuYpzmsJU5z>0^wJC3f}(bNu(hi6JsDlrPNNYIB4add9RW>!#CAEIfqPpZ zF-u=fb2eS)qOhbN*duXa^4;l>ol~I(y>LG?XZuS$5)d?DY^K)eyyM`77*l@q3kS=> zAfFbtkr>EKZLT>Jo<_2ScwDLyPe8wy44l>h)g_YDM0K|=5*Xe(^Zn}bXt2(8uI*1Q zL<*`_jh7;kFy{dhW^osSD9^{uOcG4U215{&x6mVkB@VTd3( z6?@aTQ4wi-+*1lv=wIm71}Rlp@Y`&L`i zi8wvpS3>bpuNrx@*m9MQVR?9w4}cUy6*Bw3DY4T-Qps#n-Dc=UX%Zg|J(4>{D=S3S=HqiCURmE|#^|CmXgD4A{P7uH062Oc&Mwj7uKCZAGs zmy+TI1J8EapU_pE1R1Y2bflWP)1z^%iY0;GE(4slr{@m?58xb`jvVGhc(E)|Q=t74 z#Oq;{CdWY2F**ohESXIt9-dol=vd83;^s;jO0X}jc83khd5G4=s@W105ij;nfoK}2 z(Z}OobOkX$e6@LPYZ6sZa-wxWo$fLp9X(o1BE0V=NMSf%s#3<*C6RMY#}EhZ)YPr@wk9fC2i&h5Qd5iU@F>gw;z?gZGQ>xucVs5^-i%qOgn%Vm1 zDWYe;f4u-Ozd?a_2$R}sVse{M?oZklf8EBo=CdJ9|GFj1XF`?v>agV9E-$zJFRWf9 z(cyDD^Hufi@Qah8_0A44sL1tOU1FL^=ppDjMUoFBgZe$wB2V91Tke$6DG3Q`X>R7f zG)_dT<3+`ID?>SOyd=dmsEIq5OlAc2_()y0udn#XB$gUl$>`hb^8-AXNVNcB7=KY$j9Pl5xnDxB7RH09 z;i-N9d7Ltp!T^KBkktUR&Y5Qr@A8+&eE1;nz;{XYd!0D#H6h3cIbfYW^9G@Y6Q0tg zSIUkKJk5ZqPVlj?_SF_eUQpEB2@q#bvcJB^k{pQ9hv4K~6MOSCyrgUEIV`y^van96 zIU=n8CcU$pJE_Yy?#s2)t}=2j0&$mt>kknRi!A&@_~(xZQlg1y z%eYANM|g2&X>KVDLy~f-jC11&G&7rQfOoR`)F{TJOZG+g7Ft_uxIX`L`HpdPE z8AOsWmA|!ucqN89W;~uv0HVK&eumyAwTF_8o!Jl^V6$WRvbYL2M>!&J6j*#Q+Nz zLEl;*F)Z)$y7Gb;uL7X>QR5jP+O%{V>8{|aXNl;pM7WwEwX)R*r)lqnM%Rt~bnkm6 zz*yY=+J=Td;LE_4ZLTJ&Z5ZNhUtzoHs(_GP*}!;m%T#(cI!RlI5V%}~mu-Gus?OtXlSRtY2Le|z#o=MY4CGbo5wRscI zKOCZ6Lh&o%!BBF$y`e9^+FdMlB2>r<4u#A-Py6D&HON_m>keqwUz6yszw=)RIq&X{ z$ECqUb*u|tm#_Cr|0MONMllo-A{HTI&2Fi3aN5i};iG)Zr!0nO=<=4FP_;dfMfvDn zBwu2@Lb7G4|Bzf^@bq6}Ki-F)32Z4V`Y0>ffO>Qs%`ti3m08$?RGo2B=jS_2YS%|c z%(d~xXYBrzsdT)??aaz}Al#BS9HJDym2!Vwd^Pa!B&-DlLOOPRGmVj5K)$tcq*amD zo8THS;|b9E?Qv%suw72Id)o&1FveW$E_44F4q=5vKW=la-j%R1AsTM+iVT^*MIcjg z8yVKR{<2{E;^9fqE#D2>i8+Y$VCymAZ^0@}`1e*TKb^11r9QEzGLtCE{fhcYhXC_x z`_MSUv(t7^szm0_;mRxFsLnfR#342dkhSZ?=}u>@6(ffE%um+81ZfAwh=F-a>Y$SS zK5)-%?Vi9R{d}8)sypZ9oApB5g25G}eaz=oaC`NRwQ9TV=3t)Uv;0$b7~GyIXdW$l zf4l4Y$X5%-a!o5(9%eV&VUE$%FjSoXug0q~#%a?%+ok~uco~@rWxXlxnzKLmilt(r zW2r#dX1WMEC_1x%?)G=)i>O*oQnh|i|s-D#xYowO^3+@1W(^_ewJQ4ndqb-a7 zU(jfh(9-DuRGkEIrd1%3N|iDXXa#D2WT~V~Us9}t;Mupv%x^tZ!^<(tYCFDMqPxjJ zpSr_j*owF4{9r%$9bH`zLUp=Qw;$W>$;t~%r@`UwJlM^4TS-G@@8~hzut{EBOXg=# zRO~$24a#LsFLzJxd_M$EW1SN;GV5i5$hrO>bv&c>bP72h=x2x7eMq|aX)&w&Pws}! zunq?(U^4cHPHTG?{nj>W%hw(gm*O`@@B7X95XaoV_zq*&6o5Zie90j8kxgEC3zBt$ zK#u`4#yaf0m)~I3?W=~6srP}ceEZ=*Wl25`Ij6cYbJr)pGRE4knY2ev1IyEwcx*{E zCSQlMP}2<$&#fU;Y1viV`;^4X^@3V3o+`0>Zg=pzqnoG0qv+*k;CYWfGm;NeO#snPdvNOCRIrCuD`Px-f3<8R@pkQ<4Q&WV4fKJ{Y3Brz! z1uxULczm2aJ}dgF-#)8KTNe>B`xPPgM_B)hB4QN;{a)fby(2X67b@l^X4wcN)(SL` zZU-3{|GvqWBhsgAIGi{G3mOO1D9XZ_jTF3$W~C}A4*Q6Je~E`Gi0|+Q*^otiW~)n0 zuGe|=P7>e=oxVw)i<+wS5r z-0z`nby(|f&%lr9+PdgbSC#UcLPVc{F4*nYt3-9MnXcqKZAg{&=lk{f;@0DVL-4#vEZhS;sG*mGaWr;5=h=+m>uc&F0-jJM8nJa#IvwltV!?(qJNCT@Mgx#@V`_6>%Yj~S`gVA5)j zi_4zF@dD;cUsLZ#T*RurroOhuKUkTFQ;O9*?Osf_R(Gb)1*hNPF@n)i{7^iFF}#Fw zoQ!HUqyX4P7J6>+7(b_qbkaN(MJ*8SX1n{PXQEIIYWi!D?xcEUJpO$tK32b13|{qG zUEA{zo=5SPsvg8Ma&!QW*__BQ+)+gL5Vl1mJ<&18vmXdP6ti!@@Ykg z7?(t5^PTARceDDGySv&7*KR%qZPHS>UaqYh893RxZN$Nm7&fM~5}czV^&CUBwq&*J z@_41KSTAo+`i81}+ig4jaau#cSk3We6VOxt8@bCw9UvvF9d7pKR*#Af-yg4~Kw0ae z5yYYR5g#uW*JA_eLQ}gC9S3LvlEWKQGa?=7xtAFYAodygj6~7;*(vV^=lvcrPdyM> z%dK8_e_Ah&jq!9noX0^F?j~C#qANC6C%2*%U1I)sQ@mPvE4~ES-B2zmi_;YvjI~yk6o?2pQU_S&Ww;l0=Od@aE>$B z0ur=)moD31d`L$4zPgd5Xr`t#yEMzJ3Qtx?$pW_X>3t`5VooTX}?(n8V7-PM}2q97vhWnXlv!(FS>YZ`Bai z?G#_j+>O<04iDsVIbW)Hh7vbpCf=9O;xDg{@`Kk;=X|`layOsv9b5@s1veu&G8w1krt-*=0CQ}hVd8!txZ6qa%hjQ#1+5_>TD-}L9rMF&NxL&Wx zj=1eRI}Wyc>mwIEMp#EwHS5j%V7=D3e6U^yZBjad(?&4N53AQYnVyFYkon|>uehP6 zpMZbNaDRHNGO3Q@o3un*D<9WR&J5ezN3*bxe$Cdp4o&dp`($1_K2M)d4^GSMPD0UO zue7IbTaS>zR~_;M*?v9rtt9#Z=#~ZQcsK(aY{1Ih3_x`xJ*e5=jWUw6dxkHSyMnRL zGd6Qtr_AyrIjbfrr18oO7o_HeIjRW|Ww38Xs)C^F5v~kGjoC>AXE;cc#F0ix|36hB zd6L}EKABTb;nXoqM8&SRaf)};pKHUf167oilVt3jE*#n-ydX#Do1)r>F2j4Abt_HH zp~%k&U$0!&^i_bxZgN>}X<$vPab92GR715;-+}q}!0%d4SrdA~;sONR)1Q&R5YaHY zBNO5s0E@kP!4l$DwC4c-3PCt}XI*-ki!7T8dU{UCf#P8EWrF3=_A;md%ubag`B$bG z=oXAu>46H?R!W-h&q@S5lVFheMNqL0;fF75gRc*7+u8A!Y`5PgJ?QBI6Bems+tNBD z^B8!UCpadU4S(&Eg}R)A^JgvEy{5mpCe_evN~FmcFy`eE1zcgm*3rHLUz+1;I1+Gnjz6qm z>gOp<161~tnlGcsn>NnlsSU-L282M4k|Lo)qx;yf+X<)HKr6rm4a^N(y8qwOh(-Ix zBukU+&Pk=CVuM8GB7X*lmwj0n%Fd3LY(AowDEHORf#DYCy6AZ8jV|@2z z#gEylL5xl{DSXzn@0`vV@jFpqELa!oer9M;Ll*a5we*xAZ3^aX(f=x~MTlbgZ7A!T zh(`EoIW)Q*P0i2(>d#cm;Ys#-o;$ex2AN;KRbEwVJ5(0FV#O4j950gm$&3V>cyC9u z&P20fuW%#XVDA*kWKOURuEWJR)~tB%;w=b-w_1mzb;Hut=J#yy+5tH)-=m9RouG6O5?xd4_1H8>-H4@3Cr`#%K_w%g5MLR8xo1|2q8)% z!6|~Ok5VEXw_)7V{GLK7usXPf+mL-4k@tF>*Vg=rV(`k-@h8j0sssbs1gn~NeJY)U z<-%nLXX8E#4qy5csai^EQYtm;UxMLS{67c=^#2vX@cv&2hB@l^{0lgxy6T23UJV~2 zn)#dom+{R3BqPTbMr2q=_A0yA&LC^^2yY%*#b+_5NLHwpWP8tw4Cnvk2~ur(+N0LO z5HTTcIz!7>&?q<}%{>EDIoDjaINYW2@P{%&!geA7RH$5gPB6>Fs7!b2K_IvJT*AEN z%U_qSD8Eb&(X&tPv?#{@c`8a+J}-uFyyoERfKwqHQjBaMhYA1zrC2tfku7?`7xxIY zTm9~Sgm@yy0+X;daXu#-p*+rE)t_{H-KT3e#l&+B2keJ000fM#@>wphnDc&a-^HsFT-|-2X`ixWrko;4p*Yy` za+U57Ib}Kjf{^N%CnSU*M~SA!ExF;E+U!-ByZ6T><7|u@z24wiKw6z(cDWkAcR4vW zHnugtbw8t|F@Cpm7UJXmj9n;fSw;pyU4eBZ+!KkrsXX!cYIMJ5Gn$DeCV!k@_hJ`h z`l6h@2k$#ku4_e9G>pLm1=#B_c)qg~1t}b5@VfZ5pW0_W=4?w1p6xB_J7aVSJEjgtTg_jukVra~)%YB4 zaoH}y3J>>x)Tmy5+H*y($lL@6&c-PD*q)c|*)G@BOiP;NwMh_XDd$bo<<997b1eF^ zq=4wJfRdG*R2Z(avKUm+t7oQ;-FeJSX^Q|V7|jV8P>|E)*AhBfRGC_5gVT|1x zI`WT3IKk{iLz7oZIdZffx_#5HX^aPRUg2Q0-0Ru$*g(=I#6SUYbl{8(o($ujt5-RWIEktV`pevBPe71wq~~GPNc@B#v4?!L(E;9mLOTGVVBX` z51>u9)j^#s@cY9f3EmBbD1!7p92nbyd_ym?d7s}AzDFt#@$t|Tmi97IcBPvEMjJz` zjZ^3OO8OKfd%2W2pahY0h&!@qvUh^P0f4IPtoxUTtBN<3O%lAbD!vL*`{9A?M$Zg9 zCo+Z$Ji^Vq(V`k_BZ`nh20x;7cn723aK_oQ5ye(s*5f&|HkAg8Sykhx3?az&Biwja zjNQtHzKimi88TgJouMY;3>fJ94Fgn-MY)-O3LI}Ud5@*uB1ZzLJUcWR{plKixtw>{ z`_I`45%RKm{f0ekd@M->Jas`&ZVc{~pVnknc!0A%(oZ{TV=OzT20tb5y5)Tfu9nfl z7arnTn{LVVjk7~CK=N&@UPNo{k)%7i8|l1r2!jqaMW0TAd5-Ak@c2-kLdGfZ z6a^w1>N`mMoogUT`zuc9L-FBbciV+Vw7u^?v;YRCZ(d=VUa5G0+LOVaoohb%sW2I2 zR<4VNi+<=^EGmeIh#WNut$1I5+RbQ*7%!rmu5^Yo%AyT}F4Mo+b&gu^WL*YEWOMr~ zn_E{$eA@(!?Yd~*mqAWXi9F5x3Sq3_Fa4_pweDJzZz`uPsGjhqhrD?nAdNmu4IBjK_Un^lj=)&I5IHf0yB3YsRWWA|d+M>Ol?3 zB))IOUo(aokxBSF({G!=jL9TVO^*^NRa-|S!HVGQj(9V_-=OruGX(|-d!(OpOSBgy z%#kBE{$i~{0RCy)m<4C4F+Q?^c@;A2s>4VA`!4;@^~f(z0Uo?ncb{&#hx@|k3aN_P zgvkA;CE1x&Gs~*t4!Pce*7)&>Zumak0BHqgAdDBh&lY&~A z;3p=rPezO$7Su(obPJOnDUXlhCFiqb#)R?q1%aSlQE6M)B^jQ*59!!Q10D&`TqZ*-Bnnogq#KMEyW$9`c_K*P>j zbljQ@lX7DM@sEbG>9vvz>+ABTO;BR(ZrSr%M9m%ypYc zLJ*QD1ps*b57q=dxe*jL%Z zs8vdyUn(Q>sz*09KO|C#iSHY7#=1|6xKjQFU21KemnRkytJQZqgEH4qly!y^mpeWh z$R#l2(?pwMlL2#^3%>sA=i;F7!C5b+PGvm%E&IT}9Rbtz6ZZfGUVGMPE3El~xvewq z(aFf-Py|P^?viGj>UcIh&k*qB3kA`Vn}-5Oma{(}Kh8{zz~5lOHtEkSb>tMb*7+F% z=aq@B>$w|@Q`Y1%U*&&sd6GM^-@GX6Cjf|f%c^N3q&rw`Mdn{X9JoiBZ$_&;NxN_W zdcw)ncUMz6C?}_-DQ!K}`vU#Z!qj7gF}Ov$u!N?6ZbxW!%1uf|D&z%+GgX9xNy*42 zzHR9{D~XN){Sr2n>dIZYlRb$h0gcL@b#CL5ROS~ARVxchNv~Zyy)keVZQm#kFVh{4 z)K3Ho;ab>$vwj1i`?E11^9G=^K!Y>95T^<4ZzQOU2Xn~J{)+s*VL!o;pWgk^3sHwckQZm3 zsXnESx0n6L?5s~RR0r5G7srCtirn5){RJNQhnnS*+;)x_*AsR{q?4XaP3fDhSd6y- z!@+d7G5tS@dZwykZaU4`e7v4%t|v-WR~>zIn*(dgO1$U2xVKSuyF1;}iK#o9?xdYO ztE!@TK;0t^;%|E(wmE+C$*K(Q-~XZhbvWpOO{uNs+Z`i#4Z}#v|8<@r9wbwvZcyka zMukS+6#mpo^*i&ZSR&lPiGoR-=u^JUBXgb$*4Xi?dZ<*9pDNsCYz(y`4G*&~Kw&LW;KF3pvvqQt03T@2`T1p?CDBeVSE{z^g9ElAb;rYvY<}QaL7Mdd+!U53u0#G zCfh1ZTUYa1_9gs9BBVy4MJe)p;V?j)4zQC>MX6Ed^`k$k>jlgm|&j}EyVUhG-)pu=lvZlFPqh~wicQkhE7Y|T8p)G7 zpF_k?P@z}c$8(JGKzGVdJi6)6b^vSjQ;yI{R@t zifBFq>vNsX&zsm3RkZI9rVPL`q3zv9^hJ z@U|Lqk7A@DQL*x#!90V6ifV=;pQfX{o`Iei*xR0ZeU<1L;uJ({$$O~JA~N$AEJhLn zWJt|zx48A@cx11ArYEvoy;^(tc-Dl%+7~)Q=KDo7OZ+XIf*hN843h~ z`Wdbyf&x~y1_wz>s4tR!V8p~?DD#ZwT$yQhTZMdy*X|* zGL;gxv>ynAyWMNJq($G)9Eu=TbzMk!XsB!!VzpkFDyF?oh8&odAzYzHOCgeK-^7v% z(BC(9QsqkYb<@&@-QD7h8+^`4S;T0O`)YFLi@Tjy|E?3c`rV2 zuTTl*K&2n82`7u2Y` z$IJQ8?Z^Tm{;wcoFpuYtY4RDGji1_tU4x7+prl}un8If$S2LA{#RcVcNQMUf^QPdC zTKf$X2Q5O{Fi{hcQ`j!?bC<QIBDino|g)H`(2FB*~$r zKE$uup-U$fos?J(kepiNwoUnSTJ8^UP&b$<6cB=fum7P5)+^1eVvRXaf^X*d5;NeS zURBlLm0E#>~q~e>%vp)XWjIlsbFYX^L{|ZJ9Srqq6k^qk=aZXN4 zU!QxAsLj$yj>tT&g`Ho zoUn(>l3%OVpTr320KCWM=DD=Voi0znVNY3l?xEELLto!DIX-o5iEFG z){noN>SZZy3)kyjb#{W-Wo~Mwvj9*|CRr>ypAhjn-YBylc!#Ik&?a@hclxbSt8(OH zdNU3uGsA$^`Z6ed=}(u|16Ia3KVWXgl_;N3A+=8IYEKL$3&_w*t$y~z9R+b29+dei zB_c=v&@HDo$@nocIkkto5C&@49tdE98GOE>{`m0r|m&uR?^i`ylx!PD!7&QruoLHW( zNCcP!X-+i0Xx>bY%5-1p7gYJLa_RyK)ewz1(zL0*_^MjwRVClqv}$~Xt%~kt0V-)M zxpJk6rvs(4)1gfK1u@r65Md@VVSl+_Wvb4}N(&)XPf)M#GUXBxa_i*ism73)Nl+|$ zm8(<0EYRc9xhfJ1sujZEB=JZB4rc-yoSJnz2YaB@elOn91NO;H6+;5jWk9Ai2@exZ zxtVDk3UXBPp_oGAcv8uhXmj3N1&?VrWz&*ENyFJ|j5Oxt$njQYx%F!>d60Bbt~MKf zZD4V!!((CFctc8mVA?q~emG`zUJ(m?Zz~`Ft_=U!=PpI$dif`4{yQ~%E5ld`yI$aXuE$%ST>eQvHc$3QY{QY>I4^JL4qAjYr( zN4VJmQuh)x&I&UFOafYdU8iuA+LTe{Iwt6DcTE z$BTCB9-;HiH_9`fG0>32m@<$s_EKi^qZs_qAp>8$D6CM4#-irfd>q8#ApQ$%77x*- zsB!3%KM^7Z#@`ze8`g<1g|dY}f4_Cp?9#+6)ys^RYvn(ltonqPCwo;z{-mb>s(rOp zJ-*%RQv(Y0d|xCsqS$VQ&9D_5V{bf&V#e%On`6W08n-CL?9PKtV`Y%_f&R{)hl$}! z!#;TVOzrJG-^S-5`J#v z(L@!Yrdp}q(Aedb)BIshW0calFeYjiGgcT=@2$>X9!qoBUtZE&;4^hXI}g$?7Equs zWbfkhcaOkK=nE1BL7*5J2W8+Rp{uh3P@+}I^_HJ z5_G>@UN03@{z)UJ>h~x>etufC3q`5P%G4l>WBOmHAu!MXLJi}eZ`f3BCq;l+buqe2 z3kzG4Ckxq;A{%VO1ch~ob<5^G4$)t5g-I?ME(Nk4n{XG+>1@r$cnXIo3tid>CUgzJ>Z6k0HBbv1sn}H&Ud6={ zDJv}+x)8+igDoNvNzx3lJTRj+A7T7m&X_EG8Jm!{C^P9IJ$z5s_N6S!j6jkGfq;lZ znkvG^GLlc7_);?-MkZMPAX!1Rn#>a0@YOkSjuoJfQ`d##!u3wPTu@Nz6^SBJc)?~% zqF`x|Yxu2o(nRAg4eGi49XD5Syst!fzRxSxHAa)2H@}?w$ z{3TD&C=Xd#k5!Qzps4D7^1UnOfI|1HU${*r?@J{17RL6~z`uA+Va$A@wBF2Mm$D;5 zmd}fJl#8mjkD?tkB>N!umI%D_) zBA2$f)sWxRW??8SL8VV&Ong(QeTb~ZmS0>_kYlnc=^IDloeRH;2s+I2QCc;8QiKK? zq-MC(mGQ+eq7HJog|X?lg&-@k#Y@qDpNUq1@VM>K%?+F}0jrm*|3o*&t&M}7_yg*e zbx5uu%S%~*3hz9nUB!r?+s~sJozeRaW{HJ_I_=TR&-7`{?X%rS&uBBLJ-a!^J45hw z*%XZ+`Fr}t^vora{|yH!5@nt?gce3J|NFF_jvA31K}=p%K~UQ^k}}4Jvrb*)C|Woy zuSdu%3e;*&a7Tn2DF{(g8I?%<-5I>sse6pPA%VAK2h+ zjzt$emRNe;FG6Rwrg2tb6J71<1S>@;S(%HhI)(SHo8cN@rYRfd8Yn^=W8w#-RJz&} zK54E`6y#We;o!XN#&9%W5y~*R@ce=CFfC#HO-D^bOx!RvvgyVK%r9E7Dt4vt)&L2{ zqi?h=_zzD!n&OZCMxOO{V4L>I{{Mm%eMpuJuK>Pz=--kCkMn;=B1^dd`Tbs(YCP%* zDH6li^~6)o<&RV3b*)5PDo|T-5l|?W`o+$br*Ol31@cl~kds%*8sZ@Q_slbD%14bg zH=(XTJT?&v?z_C@)`Vy+bPZ*vMIZ5Y=utjAPkE@3Eoh91hy8INPD7|V@uB+y4d<$RYbAcs2?;!H0S*P3T<2Ww9RRR;<>+=5oXu)0j?&2z9RknFi#KCms z{69jNI%ji}%Y|7r{RoIHaUeRSOti54u?WtrrTIKoUuJWTJVF ziA~H37&!UM-upK8()zF&SmO-XN=ErQL*I#&?g?T!uCy`9?Z4%VTAeLypHWIg3@c(+ z`Mi>-7Vf*G6N8D&&nqZsNB9N9)m8$mX2y(MgUVXa6C0>NfLYd?=M(XBpEj+@&@&Ky z#4%A{l%*}p84Z<8#e+M)JAq-Zd&x?-%_=|g&trV4bzy6#-;75mJ9<_euQVs@s$;qo ztDr`pqd9*DN%hUxbhIJ4@DGS|;bvM9naxe^#~rw{h(;0~m8thrmMvU3$(E@|X5LR2 zSvGaIPmdZyO^4R;q=LQ;cX<3Z7wV=U4Fy6juj}q!Y||34i3>KC>!Tj+OJ*gMT=Ae= z8Lgv}oani8aoXzU{7|oH&*+fyA-7Ymuc4RouG-`|)1tQj#6T;&p2ccWEi*d{$7`jq ztBvT(7!}Uu6zrBmTs>`&g@b7jhY3nz-ou_mjzxi~+#YIv_G@!2qnY{Kb9}x$53O6! zHfy*}1Ur4>)5F_Ya$f&lTk=X_gx&3!55JIYT%!k-3VXp{D>{nP1;v_+NAC9ai36Fe zmRAI~>n1PgfcyX>`J)U81=(G~Ao`bdnR=Bb>&Y5WOn8-o#OzEj-xGtB)s*>|Qoa(O z>mAf)%Inc*u-2&Z*anGLkS7+`dKqi6ZHy)+-c$T2Wl2>7W`w))StSr@P}C*!5&duX zKXZ&jXOkDdXh<&W=hoRG1!)TH&P~pavYzg&5(x1g^st=}RgYrZ52{ef@j_g=yLSE+ znToe(*qWqS-6MoQ-IbMG7jd%m%o2;XW|kgwmE@TCjNUmb9xd8G4f77qXB7+M-zki1 zm*EJG8+v{kt}{et?Idk?k1^p)7Mv2)3wVlaBOwnvgP zi-?Ur=bL9vMbLNgF}FR^fBl1UEg#{u+pnAj{vo*{OQS0Q56Ty7CPeI<0ut(f4)^bh+}015gi9>3)@H3O@LgP6bM!z3W(bAUq}%53oe z?@4Y^q+WhASj|a})*9l-70xGBJV}A`FNhj9+dj zt82l>wYkQ1Yeu~heE0hx{)sG*He90a2g%#hZ~MLFP8ziFSyt}86!=R5a&Ers(&Xs; zrlfXz4$d{_WNcjS-NisdV}oanDRA?!J#UIZ^c~%xN5}r0a6worNP}rIPZ~pC70s?YalY zHZZY3ZQ%U}KRrUdJg_-F=XxfXME?yqv)y;rp=r{3PyBz=WfvdRyC~7ed%|T(Jl^Bo z+0K|d+C|cAcFstbqn5dNldd(5$tx|`G*@3k!IbA=eSAc{<0a8&ou-WLpPtDsCJPmo zoIUiKeaj1q-KRgZZNhK1b$BM@P`8yN%&FOKn~tEGv66)H`KLq|f6%x5qA=UX=b|cI zu$W2cJHj~x-~7mop7iKVwmCjQUN7+pd40KOtJ#US-_S}Zn>l5X^t=o$G_X^saLuW( zA2+iuFJxB@VN4*|!!K$d=5P@B|A98|rXMAV{Tpo766y_^&nWI#k;>cYvoY7dPBswm zDq-hk!DUfMPhb7L82S5+Ic394u1p>h)8Nz?C6*|_cS=jpL7UQ~Al;HN(!w&7dcQXf zKHz_V&2RL0H@b=rWN5^Gz7kcL7) z+pR9lIYtTTVvJ5GsQU*1=m7o&dUvnlZ&}nW?hRZ1f|_yXRInf)f1w1A!-8Ii+Fh|M zkDZk&q3a)|WJ?ofbhwjfzUk*#EafJCZ{!N>YKi9E?g;5v(DbyRk@>wYSl%4l%Rf^X^{59 zvLP0hvsS&|Xn_MI<*1jjjxy9_2{NDN#n%))I;Dxzr`=1xCtow{n8J{ex~lurLH2-qIgxII4~`^ zW9}!W@r6X_W)%5Mc;HXrXlSk7ld#jAmxzmGtt*Mc9n+PwZLFc+1Ef+K z>OaHdnqyEK667AAj11gFk$(h8fc&osa{S^fiU2@hqSbU-OBK*mEc{{APDnR`?{GvK zwsggFV#eCP+3oi{N;74D?%~(*K1_s(O|3hLVs z+Zou#dIb@0FJ=yu?Tacp*4HT}M2@4l#7~{)l{EbIEfXpDY~!gNZVkD* zF6f+>(ezcMe=xl1M0#N2W@tGF<`kEvO_32tJCxX>T^k%HqwSTv+0w?Xx6%9yif;yX z<&0Yi)qowCX9owfyRt4Qs!NrsPnJl^$Oc?qq-QctR$>mS%TJ5O0<0JJY#w)nf2+ojuS5LPs%e5J%@5P!ugt!~JLTW7bMl5jnG#fGJ1U+Hos ziv8bUyc+YNIcZuVx+-5`YrbE455c1?RuF55Kmia+pJSo2!aBe$LSAC<9u)LGBKoW` zalm;%n1Xcb)a238v36+k4%E8Q$waidk(GGJUn6%b^Rmk9)6N zY2T*jZN-x!5JJPutCQP+YC z(xiVPkYvrVs4ZVm3vc{pWD0zh?Gp^W1gZ2cyo^@CPoY=qF<*n^Donm(Up~7-GmIb@ z)9uUH`%khHtQlH`-uI$*FBR4$9|nvkhUXCjuO@}q22WiPlPwdOw9R#z)ZRxN64i^J zOs?VgLf}c)NZ24!UgO}!M9}P~xCkgH79hbCs4NN=)GxH2Y+HLq09^{k0&FbuY*Ukr zxl!x{GB~ufSvc8P3L1k9s;P`A<6!%Ve5yiX_~6%}W1P}ML^%lcU{CKk6uqV1eB3Jw z4mwo54J@^b9a8!KAPP)C{*ppDgcK6FfW|2;65D|%$@6?Q&}MIuMIb`xrHjuBLbYXw>sK0;emO%v@x6i^w?+;70~ zZ%&P_f;||v%DjBiVi}|w_Zz$@@6pl}^qvukaVe`&pk%ZgdZRP^Wm8;sPnQl*Z#I_? zr9de@Z$7cgcy6fIGx^!9@<<69?Z~CF4c(!kauT57@NuqIt;ejWlRM<`0)sAKvL*>~ zciud&!#Z+1>IR=I2T%X85jLtMJgnXR+%U}z`}GPf<5BM$AMEj{$YGbM3tw|sVFEb5 z)bvvc?-?g2>&cGm=I(2pzL<4HCq8y5SmC0nJ~FWlDD(3}62L?+x1D2`_v~o$0=yJ%Bk8Wlx4*{%#d~AuZD!f`C#X( znTiY=d*s!QZKsI|4g>tos66H+4215ELt{=Bm9CZB`~o1Fj(!yxR|2FUFT>vFS=NABhYD=Et(}<43a&u9P<&2IoqH1(Y zE{rmuni*=TJX}RcF7Xg$Z>BX~98N_ADky52nK*LFAyi#>F zYoI2D{4NbzPJ+0^8ze>axw+MFP;zv7HOPr4eUjvcN^KT2a|u*-kE}w|YS6=pAS#L^ ziUF8>4Sy-Lfz*g`Q*Ib+n3F{Hcm~wPcFwZ!z>;gP(Pm~jb*tbRkQ9*)>}x*20fPa6 zDv^FqRke~u1}>nRrR0q9YIxqM;u+2N9k!Ai)7_Ig&d-Ym+4~t}7pCWpbyY>sIl_x2 z1m|;v6nAaUy;XAdmhQL)dRPK1u9e@y!q?aC;=h=G0W>K9fFzp3mOI8jn7ePKR9K)6 z(u6de|Dcc7@$w2LQ;gjt@DS%$z?P^>0l^G=9DAm_AM1v>YinA?6j^t#8dkoN*wZ+#{D$t{U0G^tT%_e4#D8a z$5;F^u4QO$8KVb_ZM@GhKdmkGEs9e!-D=@hN^03u>xL_f7UHLU?{VN#MuZp19#bFK zXBx%)1;kxTA)lw z?rYz+fc{U)iHJdJq(v8Cqp{_dMhpY{5MNEoFQ~*~tmY-Uty$N3F`+P{p7?|SK2Mo_ ztpY?wJR*mbr1bbBcc^{qOm(KFszvtcXgb`yQIW7AfTVO%P0PFqsVhL_l#?tC*UiE( zW8X#)uf_SWLe4DbvUJlK`Pn6rqBYs=RcbA^c>xUp4(b(U`CmhYa{R+T(T?JOZ*}J6 zZxuK;)K7p8U*HnNM+S>CW^L$}kMH6Bks;5JZ2Et-6G<6@_9>9fmV_A%4S)Zhi4Ph% z1qGNgK&=XBv>Kn*HFpj3M5(UX5TD}RdUHIniiL4hXdNmJY zjM>Vo)Y2=AzHM)AgW*JhcN1*hfNTO6CH+05zST zue2_?0Aj0di(P47I?B_g$Y^uXbXyu@@&8%S;rZVM9nz|0%H6#d-tm-*^Hmnx?D+`X z#01#}#?fF25C%Err+%1@q9=>FepIDNb!IBHXR578qF8Z#a1W)ii?5YiqAwW!O%>!|W0Ef~MNeb)0C@W)_ zmQ3fgi3qLV5#+2T9G*1**m%FSJ5gGA9fghEY@L97dO6_`4O)ZVc9zel72m$nEzh3F& zUYF-Lvya4YFLZ}P)VaVry8VEC-}KVZspqIGBIssMX-!p_kW23Ep`u+A3lAmE6y+#*A!b!{56~$k2IZ3et56nR| z2E$B!mL-itf0_P_?UqwjUa3R(*-YeZ?F>boiRINBu@ zCiAx%5qA^C<@$}C<`@!M0;;3|obLjV!AmWz`)y$#qTE&)T*|^4Dk&xwI_twp%^=_l zD{6#L?vqxN4>k5KWkeo4kBUuJn38ao7QUr#d0!Y}jwMBdKo|uOivv`aK_mrre{d5{ z)Vz};-*|A+Jmz>%SOep00bSM(?=rk*TC}H+;rW07-Soa&f11m4xO=P{pVz%VllDBI z=asN{<^DRy*BPW51&o)gMw$D_Y(?0cci|Z&AACB^mdK^IFx|FB?v3y@|JN z#l{}Q9Lr@*FI(>j5@`xCGz~7P*)7Fh_m2(ejRz3wKsIb4r6$R%hW!pEQb6&)v4mtq zvL>lIC#Bhhsp!U}xlTO;UUPE?ZNu>M*zmA+f0Voo4wm6mGWQ4Cw{w&_N(22G=UVr7 z_41(l-xxb`);u&46mJz_$En_(A7jw5r{Rs9`)~dAOg=wxFlwV1W;VjT?c@E@tfs=m z>6$WXBiGs6!>I{RFZV012VcgeS-2NAMBp$5MnO8^$O9+|%=T?$&s?1qQFGR`#H(XT zGb5SK{^(_#a$w^BoOZG2d@=Y#mQ6Znq!&4#)`c3gPQjQBD5EPiqogUJ9)+pQDow*3 zQht?sr%5M-3fn#>O&q;c7Sj7-U>r@nkVH8dC#taJqA?aKhy&r_=P^r7 z6B7G2ryfOvW_8MAlgns8N>Y+L-CvZ`R~NM^NOL0%aBKuv_4W55E-0)Wsl++q9Up&@ z-K+?%w_fYxLe?a?=(t9?l$hYrq#@w>_^ZuPbHwQ?w8o1uJg?C|JM9s?v273gtkELC zyJB#CT6{WM{^IPHP9j24nrn2!*K}xt82xc|8-eq=DL*Au*-&R=qAgh~|EPgavLYED z$qdDcyA!7k=3g!Ty7h`mKK;1gIz*^~06F-B?b<-m56}49zdIg{NIO#|Bhc@)xj?nTc{HBlP}OHt`jC2 zeem%$HvLbz0J7w)-=6q73I>!^imJL?X?KW?5eVvkSL}xK*OwTiMUFVXeck`4c1<+& zRO@GC-y)BsbG|j$nZ)POY}JLDxzE@Tn{H=MT1MG?Fu@+)m9f~O*kZ9sqIL;JhdObZ zkr68hFlvSd#rQKuhtZ%1aTwN`3Eqb#)6UKaypPm1(R%c#{xs~sgtjbC|JEvHcU1+q zR`a6t(Nt~9MwSZ`f+KT!)B#-+Z-(9sau;NQ=)YtW7%f3_AUQFO3}HIj97tJwrw;rr zY#D!2x%mE3_`ba^z#$hx-)7K%N+sZG*uQ2R#Hj=SM``cpjZC6ISCH(!l>i zS#q;rI9%AAG=#s(?GlrmGd!Hg(b?bJz1iq1I9E)8QoOf@+gOwl6b4LJ!nhtVj{ZEQ z=*+HwMwFN`VrKcwEIFryT{{1e71jt~jjl z3mb2DJj6K`886pGNm6tc%cy=I=WUXnoq@yOg>28sL}Kvk@z$AjnfkQd(F#LK^NT?C zYQjFx!`~|7_*#D6eR?yI)Z_V({*%uK>BK(w{fvCIyE_TF`--&5+NtQo=P>7FTF2`0 z{wMC&VHoS$_~*#Viuu&P8|c~UvqS%C-V;?j(RTUyDO0hJtuu+s2&WvvDigU2$MKGl zad(`frz=epvrvgZdgjM2vW>DfNW(8n<0;h%%!!S+F}SEoXDlb;`VP@|0axBK!_{5P z(ns!Hjb*_fK_d2nFuiJl{MQh??52hoBW#PeP0gH-C?^cz;zPj1p2yDsJkXsu{C^Gb z`Oq>Xs1XVA2bh#)D{VO{hX~i%*yLe+{7%@gq#PJdSPK$>448vl#)sjHf}Y-KgH_zF zKIaeDBNW#;So{(=B)fRK8|#7(nYbihJ8Wo9`Gq4O!y?r$yoY~~MEFYPfo3bX7Ff6+ zOf$kd-W0bYd0Kkv%Kn|{Z+es=(!rCjEUk%E&5ZLocb);&FJqA=i4hSIKvZzk|Etd& zm*TG_fGduxFKo<2I6groa{XFZQAvcAI35F^?7AA9q1k%)Ys+E|3M%II$t(I4q_TE= z@LY(jHErc|5QVX9IWo1Z;%F2V=~fhk?0&*LFPBxj*_&r&b4wzF!!;8{cTb+y6&ag&{D>afoGLAn|1;Ix)5%Xuu9T!jdK#eEx}}iasuDuzkL>tj>~;(K zQy`C)6!Wdz8(eU`s_QBuQDY)}Jk$?sSeiNqJCtPU3kzzZ%QM57cC2GEKi*j1JF^ht zw#;8k!#%zkA68WticmPG&Xu}di%1Y|RopLSA*DCn)Au!Swu?MUbGFmoROHe3=ewjh z$HsytS|x9=i!73*At~Nh14iZf`_K|Ej330qqw_5fEa5}2(7k}EmjCoqyBmN`uSt8a&ikj4H@Pu z+Ij@Kaf3N1@(7}Y02;8MzQoNFQc{@c+aA|AdC_P;yWf8P#CN)F-@^MeCYo?+p!+Xu zEcs7xHEEVifI^tPusCqiq6vhwt=T~PE*wfQC*Iowa-%W*Z6E!%foIp3!G+#78fs_v zIceT(F5veNf*@DUQaUF-SBJsCS25eupDr0{P{HNZLtO2DJVKgNxcfq7Utf<`EbP?{UHi{Z7pvNr!cU8nZ5pbQ z^+uS~G6w?#kSiEi#Y22~`vPY3d8EKu@iL~I6Az&C=#BjmjJQB#w7^EedSj#@wiKW^ z7X!aB$iwO5iO!qns;#LntDRNX-m>S>lMU5%)Keft>%Vd^r*X2z?=u^0-ZPHhr|%Cq zM>bs`c#?$VZMK@zlVbJ*M3+jCEl|JKW+zm>0Jhss^hxpzsV#S?*N?WrNfQG7v>Gr& z=~tzgna7xphg|8}17({VZ%)BEe+bFk{23$`wI~YSpenyM82F zX@jH4q2Uq|$M{P{jY@t*hN#TGrytCG#`QO@XX1Qa*}0>`_LMF5Bi&D=;uP!fZJ{O^ysvomtuyV&;rD-S^+)Mc|6Ql5AWcCWS5PaLWULqf zCK`?HInA*Mu}JmP?XZcNJN}ymk|Ul++_Kki^sy2!+1t%yx{Z2z!*%ay$CHlNcSt%} zGC=g@*Zs!D=9yoeA}%1c3v~r_0ndEA{jctiNx@bA65u4S?;9hTsvf zjLM}616Q6|wL>*@Aw`b&NbbjlpoNKg}52+TJ~A-ZMCKzLp<+J7Fqja(cj~J@7#gvUb7Z=O^nx z+jwbRv`<8+Pj=W?AoAk5N7Qtf>|SG2v(IVBB$1q;pGcxKiU0?mjY;6>kYNzvWMPQbqI9;~fM!8>O)Z&sSmB@dmD z-qGyRGsivLwWWw%9O;c=*>WH82rdwn-4*xQwQ`(R&;H8mLk4%gT>+;V@D$8^ovm>* z-W$5^lJUYs4)mw`Qw2YtoaBd;W4z{-mZIaC6j8t?Qm!(W;@2t%BHC|_V?TY5Jrq|z*{WOo+4Ic=jA{)CK=g@+x(Z%YN1cV}x zAc)OvYY9Kgjexv{iF3m2kgGd9QdCeD64{W{OEW9|ZX@(9EwEI1jL~#XL%ughO-d~Z zWh*uZIc4f@yCThE2X{Ud@4}=*-x_n5@iESLUz7l^ozs)irah@0Sw~e@t zDP#c%!a%ivhkI@)!oTfu#Uqe@*AR7JNcn$j!X-X|#J}|bmRKm#KNkZ-%7+8aZ>sML zeSa08AZFLa# z7^*I8bz_`Z&C5uf^HHW@X*1H%B3o~2;8mK?b;!NEv(qogepNv4PL(tc3;5skf|hngqpU587Y6=$xt`^*Ub2c zEnHF%sF8XvQ^t`(Q6_NCD+Sk{msZQ-EcUr8v-CN3S;fR1KROCbm?1_2+R5c9#%#tlK(Ere9()dWHg01O69 z%nW+TcvclTH#yJj*S%L^Q9H*+A3Qtn(EuB-f|Fg}2(PyRR}~8tkM?a#LG)QXk(b_AyTx^uv% z%|v|ScVdF>Bzk(q=}42o@?HzDOqY9eOhtrZw84M@3;ivW3^8I@yqY|t0;R>eWCCXo zwM|P{L9FJgl$9Ns^LIu-dG28UGg4d!<4*=aw)VT=CcAZ^U$(p);9!cUoer zYp%H#cfw3w?}_s8=VNkkE69V4jz0($5Gx{=|_lth$T52R~K2`@TpAfdOMXvqCWaddDaarNS25k=vf z7t717+^ZPuSi`!zojZ&N_EChn6^uzn?z%Ia^0@Mp{5wi? z8Yl)*1dK*9P=~CX0Dx6QtvvrIe$P*0)0-$D#RhZVumLY+2?IZY?T5=iO@)_t0IVj> z+cW?q`f+|}UzrWOzK%KL@p5PsI-}r42GkM)B#9$6Ew?bzKM6`w;(qc}b);`F0b(F$ z-aE!fzv0O1rq*6yNjq&3HMENeaGDrju(THI_oO~i5H63V<}($l9Ik{VzRmE(_;1c zm@(|A?GBhuF^-R`<`@heH0e)%%ptub$xZi3rE`HO%JsKjr@H=`2BIvi{V0Wfm;?tA zHFx+xt6ghdaTM0gF4V9!C>#BO_hb$R{&&=swpNT30!+2XJ-Ffu%$CS6WwX(o92>sv zC3fjz$0LV8QK#+Uf>vmQ%dP#fO#_EHJ3}#@iM`)?f%J2AuCE0Dqz8eccNskfr>cmI z>gTv?cfvbu75m*Jdw+!+-oLHacm+l9K+~`iQeH82%Z(y&)_Cnxoih^lw|Qi2+R70A zj`SWY|GLJaM$6c{h@3j6vRK)CBJ*jXzQ;HT^xyY<^N%Ygcb+?|^|zUoY;0N(0z45% z5+7m2Qum;c`e!9|q${*##B6ZjjxSlq+!^!10?8TEIda}r1hw(^PCb^3q2(=@hGS}A z?shwy-ro!{-#rKV#enRg8hoAL0RFqxjS_U#-_=dUi{}+98kjwvGyng`T$ z6N7(*$eOijaak3jWGQ-#8}+0#G~8eZ42%pR^w9TY)Us^#jL?+cbaO>=b-sIVn*XkC zhzgFCK1w1?fmh%$C-~*QbXQ6x+=_}*TC1lR|9lMeesyQzX=uM6wG^n``Jnb&zM&`` zjECRI8fg0tRFMx4E(AzD3g=yIzbd4|l->OS%p& z{sn(0Q)VMh5IG z13S58IYF~*zgadq#Hv-{D*aUGtb*TkWF1uM3oaLhO85JCCx$)L)H{D0l3cwZlqf7v z&{`Hs+MI5pL(i@ih%3*>T2O|e#Tuop$Y1y&{rh}~XhdIYb8~>|{M$h;c_PJ@y`Zq1 za-T>4NeXYW)#mbkSn$5A{-uHU_eZWMkqB8ys4y}q6J;Zc6bk}GI%W?AO^un#u*9iq z$e57`eK**JGMk$cw~Ips{r`;n#HOb7vX`%Y)jpK$JF@!RY;6m7wddEqxPU?=ZF8p6 zly%EHtnMqBN*{@d=-Ro2OIJ?O&d$l`#qoQe6uOG`PpObJkWg5v8!_EA-nSvDZP~)J z-#2dIF*;*lThhAv?k2k)U?}}Q56l`5*LAD>4W8k>ddBW5##rx#QJXvbcX|A)3n#)2`M1`TI=Uyqno? z@qOBQD)ZmdrPnP@=Zl_tE&ue!w#a>7E=kXs=dWkl+y61EtMWT$1kdSg*=l)d96bsol zHEiQ`v!fg9)Za~MtU0VB+;H>a8RP1#6I?rfdipm^S=-`V`c`J^!HM0;F*CnSx-rYW z*lf&#ZD&D#91gyF!6Z+m<`@Lygz<2pOCfikksM%dOT0< zk4*M|yV&5ZZMEgsH=F0o@hbN@n84s0t#v?R!*kII>7O2!?CN;swr1zikdWNvs)dc+ z>t37foM#^VP?a0pO0GWlj3e-* z%B*FH1~VBAQX(!)>P%hdrTpe$`(MAB{P!D*mn@xi>4U=j>aE>+ya^rm+vK19zCHWL zl>-*{=geC+TlY;tM4}p#mz}ljf|q7x7CHhQO^*`N_cawC=sq8J%53tnpZ`|>DXE#6 zxa2`--N!!~&$h4&|46^|CM+b!6?l`f>I^>D%X2n3VAiQE`E!K1AI^-La%GXC6Q^-& zs_TyD%Q)08+q8Qszv>OR@lTP@e$k?g%@*hXPMZIxE+bVr5|qHzwre^~Ihb;6##HB= z5{BlhZaFm)!F3m{^DP!na4!;Vt8#zFEYfgp#bwv$-^{*EOZj|Oj(OQKaY8~ybz1m=>Yl*3Z!C#s&8Ih3@#W;FJG_57eX^pHcj`(PhVKCh zoy{*LyQC_2>}&w;>;Jp&YKY^}8JtHJTxk-`Yg)cGct#{i{}33 t+w^~8e{sb#jr;FoHc7<-^ZJhTKlXCQ)k|&%Ov+^d0#8>zmvv4FO#q(g3$OqH literal 0 HcmV?d00001 diff --git a/doc/tutorials/introduction/linux_gdb_pretty_printer/linux_gdb_pretty_printer.markdown b/doc/tutorials/introduction/linux_gdb_pretty_printer/linux_gdb_pretty_printer.markdown new file mode 100644 index 000000000000..9d6446992000 --- /dev/null +++ b/doc/tutorials/introduction/linux_gdb_pretty_printer/linux_gdb_pretty_printer.markdown @@ -0,0 +1,38 @@ +Using OpenCV with gdb-powered IDEs {#tutorial_linux_gdb_pretty_printer} +===================== + +@prev_tutorial{tutorial_linux_install} +@next_tutorial{tutorial_linux_gcc_cmake} + +| | | +| -: | :- | +| Original author | Egor Smirnov | +| Compatibility | OpenCV >= 4.0 | + +@tableofcontents + +# Capabilities {#tutorial_linux_gdb_pretty_printer_capabilities} + +This pretty-printer can show element type, `is_continuous`, `is_submatrix` flags and (possibly truncated) matrix. It is known to work in Clion, VS Code and gdb. + +![Clion example](images/example.png) + + +# Installation {#tutorial_linux_gdb_pretty_printer_installation} + +Move into `opencv/samples/gdb/`. Place `mat_pretty_printer.py` in a convinient place, rename `gdbinit` to `.gdbinit` and move it into your home folder. Change 'source' line of `.gdbinit` to point to your `mat_pretty_printer.py` path. + +In order to check version of python bundled with your gdb, use the following commands from the gdb shell: + + python + import sys + print(sys.version_info) + end + +If the version of python 3 installed in your system doesn't match the version in gdb, create a new virtual environment with the exact same version, install `numpy` and change the path to python3 in `.gdbinit` accordingly. + + +# Usage {#tutorial_linux_gdb_pretty_printer_usage} + +The fields in a debugger prefixed with `view_` are pseudo-fields added for convinience, the rest are left as is. +If you feel that the number of elements in truncated view is too low, you can edit `mat_pretty_printer.py` - `np.set_printoptions` controlls everything matrix display-related. diff --git a/doc/tutorials/introduction/linux_install/linux_install.markdown b/doc/tutorials/introduction/linux_install/linux_install.markdown index 5083fac282f8..e69f6ea70749 100644 --- a/doc/tutorials/introduction/linux_install/linux_install.markdown +++ b/doc/tutorials/introduction/linux_install/linux_install.markdown @@ -1,7 +1,7 @@ Installation in Linux {#tutorial_linux_install} ===================== -@next_tutorial{tutorial_linux_gcc_cmake} +@next_tutorial{tutorial_linux_gdb_pretty_printer} | | | | -: | :- | diff --git a/doc/tutorials/introduction/table_of_content_introduction.markdown b/doc/tutorials/introduction/table_of_content_introduction.markdown index d1f2aa3ca319..8fa89d7d7f9b 100644 --- a/doc/tutorials/introduction/table_of_content_introduction.markdown +++ b/doc/tutorials/introduction/table_of_content_introduction.markdown @@ -6,6 +6,7 @@ Introduction to OpenCV {#tutorial_table_of_content_introduction} ##### Linux - @subpage tutorial_linux_install +- @subpage tutorial_linux_gdb_pretty_printer - @subpage tutorial_linux_gcc_cmake - @subpage tutorial_linux_eclipse diff --git a/samples/gdb/gdbinit b/samples/gdb/gdbinit new file mode 100644 index 000000000000..228e8f702367 --- /dev/null +++ b/samples/gdb/gdbinit @@ -0,0 +1,23 @@ +set auto-load local-gdbinit on +set print elements 0 +add-auto-load-safe-path / + +python +# Update GDB's Python paths with the `sys.path` values of the local +# Python installation, whether that is brew'ed Python, a virtualenv, +# or another system python. + +# Convert GDB to interpret in Python + +import os, subprocess, sys + +# Execute a Python using the user's shell and pull out the sys.path (for site-packages) +paths = subprocess.check_output('/usr/bin/python3 -c "import os,sys;print(os.linesep.join(sys.path).strip())"',shell=True).decode("utf-8").split() + +# Extend GDB's Python's search path +sys.path.extend(paths) + +end + + +source /your/path/to/mat_pretty_printer.py diff --git a/samples/gdb/mat_pretty_printer.py b/samples/gdb/mat_pretty_printer.py new file mode 100644 index 000000000000..e6ad2cbde212 --- /dev/null +++ b/samples/gdb/mat_pretty_printer.py @@ -0,0 +1,212 @@ +import gdb +import numpy as np +from enum import Enum + +np.set_printoptions(suppress=True) # prevent numpy exponential notation on print, default False +# np.set_printoptions(threshold=sys.maxsize) + + +def conv(obj, t): + return gdb.parse_and_eval(f'({t})({obj})') + + +def booli(obj): + return conv(str(obj).lower(), 'bool') + + +def stri(obj): + s = f'"{obj}"' + return conv(s.translate(s.maketrans('\n', ' ')), 'char*') + + +class MagicValues(Enum): + MAGIC_VAL = 0x42FF0000 + AUTO_STEP = 0 + CONTINUOUS_FLAG = 1 << 14 + SUBMATRIX_FLAG = 1 << 15 + + +class MagicMasks(Enum): + MAGIC_MASK = 0xFFFF0000 + TYPE_MASK = 0x00000FFF + DEPTH_MASK = 7 + + +class Depth(Enum): + CV_8U = 0 + CV_8S = 1 + CV_16U = 2 + CV_16S = 3 + CV_32S = 4 + CV_32F = 5 + CV_64F = 6 + CV_16F = 7 + + +def create_enum(n): + def make_type(depth, cn): + return depth.value + ((cn - 1) << 3) + defs = [(f'{depth.name}C{i}', make_type(depth, i)) for depth in Depth for i in range(1, n + 1)] + return Enum('Type', defs) + + +Type = create_enum(512) + + +class Flags: + def depth(self): + return Depth(self.flags & MagicMasks.DEPTH_MASK.value) + + def dtype(self): + depth = self.depth() + ret = None + + if depth == Depth.CV_8U: + ret = (np.uint8, 'uint8_t') + elif depth == Depth.CV_8S: + ret = (np.int8, 'int8_t') + elif depth == Depth.CV_16U: + ret = (np.uint16, 'uint16_t') + elif depth == Depth.CV_16S: + ret = (np.int16, 'int16_t') + elif depth == Depth.CV_32S: + ret = (np.int32, 'int32_t') + elif depth == Depth.CV_32F: + ret = (np.float32, 'float') + elif depth == Depth.CV_64F: + ret = (np.float64, 'double') + elif depth == Depth.CV_16F: + ret = (np.float16, 'float16') + + return ret + + def type(self): + return Type(self.flags & MagicMasks.TYPE_MASK.value) + + def channels(self): + return ((self.flags & (511 << 3)) >> 3) + 1 + + def is_continuous(self): + return (self.flags & MagicValues.CONTINUOUS_FLAG.value) != 0 + + def is_submatrix(self): + return (self.flags & MagicValues.SUBMATRIX_FLAG.value) != 0 + + def __init__(self, flags): + self.flags = flags + + def __iter__(self): + return iter({ + 'type': stri(self.type().name), + 'is_continuous': booli(self.is_continuous()), + 'is_submatrix': booli(self.is_submatrix()) + }.items()) + + +class Size: + def __init__(self, ptr): + self.ptr = ptr + + def dims(self): + return int((self.ptr - 1).dereference()) + + def to_numpy(self): + return np.array([int(self.ptr[i]) for i in range(self.dims())], dtype=np.int64) + + def __iter__(self): + return iter({'size': stri(self.to_numpy())}.items()) + + +class Mat: + def __init__(self, m, size, flags): + (dtype, ctype) = flags.dtype() + elsize = np.dtype(dtype).itemsize + + ptr = m['data'] + dataptr = int(ptr) + length = (int(m['dataend']) - dataptr) // elsize + start = (int(m['datastart']) - dataptr) // elsize + + if length == 0: + self.mat = np.array([]) + self.view = self.mat + return + + if dtype != np.float16: + ctype = gdb.lookup_type(ctype) + ptr = ptr.cast(ctype.array(length - 1).pointer()).dereference() + self.mat = np.array([ptr[i] for i in range(length)], dtype=dtype) + else: + u16 = gdb.lookup_type('uint16_t') + ptr = ptr.cast(u16.array(length - 1).pointer()).dereference() + self.mat = np.array([ptr[i] for i in range(length)], dtype=np.uint16) + self.mat = self.mat.view(np.float16) + + steps = np.asarray([int(m['step']['p'][i]) for i in range(size.dims())], dtype=np.int64) + self.view = np.lib.stride_tricks.as_strided(self.mat[start:], shape=size.to_numpy(), strides=steps) + + def __iter__(self): + return iter({'data': stri(self.view)}.items()) + + +class MatPrinter: + """Print a cv::Mat""" + + def __init__(self, mat): + self.mat = mat + + def views(self): + m = self.mat + + flags = Flags(int(m['flags'])) + size = Size(m['size']['p']) + data = Mat(m, size, flags) + + for x in [flags, size, data]: + for k, v in x: + yield 'view_' + k, v + + def real(self): + m = self.mat + + for field in m.type.fields(): + k = field.name + v = m[k] + yield k, v + + # TODO: add an enum in interface.h with all cv::Mat element types and use that instead + # yield 'test', gdb.parse_and_eval(f'(cv::MatTypes)0') + + def children(self): # TODO: hide real members under new child somehow + yield from self.views() + yield from self.real() + + +def get_type(val): + # Get the type. + vtype = val.type + + # If it points to a reference, get the reference. + if vtype.code == gdb.TYPE_CODE_REF: + vtype = vtype.target() + + # Get the unqualified type, stripped of typedefs. + vtype = vtype.unqualified().strip_typedefs() + + # Get the type name. + typename = vtype.tag + + return typename + + +def mat_printer(val): + typename = get_type(val) + + if typename is None: + return None + + if str(typename) == 'cv::Mat': + return MatPrinter(val) + + +gdb.pretty_printers.append(mat_printer) From 210bfaf8d68b556697a90aba54729418a1f29290 Mon Sep 17 00:00:00 2001 From: thezane <10068531+thezane@users.noreply.github.com> Date: Tue, 17 Aug 2021 13:09:25 -0400 Subject: [PATCH 119/376] Merge pull request #20483 from thezane:support-cumsum-layer-for-onnx * Support cumsum layer for onnx * Add unit tests * Address review comments --- .../dnn/include/opencv2/dnn/all_layers.hpp | 9 ++ modules/dnn/src/init.cpp | 1 + modules/dnn/src/layers/cumsum_layer.cpp | 131 ++++++++++++++++++ modules/dnn/src/onnx/onnx_importer.cpp | 22 ++- modules/dnn/test/test_onnx_importer.cpp | 9 ++ 5 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 modules/dnn/src/layers/cumsum_layer.cpp diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 9c96c5a5f187..794bfeedda33 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -723,6 +723,15 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS CumSumLayer : public Layer + { + public: + int exclusive; + int reverse; + + static Ptr create(const LayerParams& params); + }; + //! @} //! @} CV__DNN_INLINE_NS_END diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index ebd887999b83..1916aa0ec94f 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -140,6 +140,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(LSTM, LSTMLayer); CV_DNN_REGISTER_LAYER_CLASS(GRU, GRULayer); + CV_DNN_REGISTER_LAYER_CLASS(CumSum, CumSumLayer); } CV__DNN_INLINE_NS_END diff --git a/modules/dnn/src/layers/cumsum_layer.cpp b/modules/dnn/src/layers/cumsum_layer.cpp new file mode 100644 index 000000000000..9c70f306d486 --- /dev/null +++ b/modules/dnn/src/layers/cumsum_layer.cpp @@ -0,0 +1,131 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include + +namespace cv +{ +namespace dnn +{ + +class CumSumLayerImpl CV_FINAL : public CumSumLayer +{ +public: + CumSumLayerImpl(const LayerParams ¶ms) + { + axis_raw = params.get("axis", 0); + exclusive_raw = params.get("exclusive", 0); + reverse_raw = params.get("reverse", 0); + setParamsFrom(params); + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return true; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + if (inputs_arr.depth() == CV_16S) + { + forward_fallback(inputs_arr, outputs_arr, internals_arr); + return; + } + + std::vector inputs, outputs, internals; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + // Get x tensor. + const auto &src_mat = inputs[0]; + const auto *src_ptr = src_mat.ptr(); + + // Get axis. + const int axis = normalize_axis(axis_raw, src_mat.dims); + + // Get y tensor. + auto &dst_mat = outputs[0]; + src_mat.copyTo(dst_mat); + auto *dst_ptr = dst_mat.ptr(); + + // Get flags. + const auto exclusive = exclusive_raw == 1; + const auto reverse = reverse_raw == 1; + + // Get parameters to iterate outer dimension. + const size_t outer_size = src_mat.total(0, axis); + const size_t outer_step_length = src_mat.total(axis); + + // Get parameters to iterate inner dimension. + const size_t inner_size = src_mat.size[axis]; + + if (!inner_size) + return; + + const size_t inner_step_length = src_mat.total(axis + 1); + const int inner_step = (reverse ? -1 : 1) * inner_step_length; + const int inner_start = reverse ? inner_size - 1 : 0; + const int inner_stop = reverse ? -1 : inner_size; + const int inner_delta = reverse ? -1 : 1; + + // Get parameters to populate channels. + const size_t num_channels = src_mat.total(axis + 1); + + for (size_t outer_dim = 0; outer_dim < outer_size; outer_dim++) + { + const size_t outer_offset = outer_dim * outer_step_length; + size_t src_offset = outer_offset + inner_start * inner_step_length; + + // Populate first element of inner dimension. + for (size_t channel = 0; channel < num_channels; channel++) + { + if (exclusive) + { + dst_ptr[src_offset + channel] = 0.0f; + } + else + { + dst_ptr[src_offset + channel] = src_ptr[src_offset + channel]; + src_offset += inner_step; + } + } + + // Populate remaining elements of inner dimension. + for (int inner_dim = inner_start + inner_delta; inner_dim != inner_stop; inner_dim += inner_delta) + { + const size_t dst_offset = outer_offset + inner_dim * inner_step_length; + + for (size_t channel = 0; channel < num_channels; channel++) + { + const size_t previous_dst_offset = dst_offset - inner_step; + dst_ptr[dst_offset + channel] = dst_ptr[previous_dst_offset + channel] + + src_ptr[src_offset + channel]; + src_offset += inner_step; + } + } + } + } + + int axis_raw; + int exclusive_raw; + int reverse_raw; +}; + +Ptr CumSumLayer::create(const LayerParams& params) +{ + return Ptr(new CumSumLayerImpl(params)); +} + +} +} diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 946623fd4092..3379ea3a0bb6 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -177,6 +177,7 @@ class ONNXImporter void parseUpsample (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseSoftMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseCumSum (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseCustom (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); const DispatchMap dispatch; @@ -641,7 +642,8 @@ const std::set& ONNXImporter::getSupportedTypes() "Dropout", "Identity", "Crop", - "Normalize" + "Normalize", + "CumSum" }; return layerTypes; } @@ -2399,6 +2401,23 @@ void ONNXImporter::parseDetectionOutput(LayerParams& layerParams, const opencv_o addLayer(layerParams, node_proto); } +void ONNXImporter::parseCumSum(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "CumSum"; + + // Get axis. + const std::string& input1 = node_proto.input(1); + + if (constBlobs.find(input1) != constBlobs.end()) + { + Mat axis_blob = getBlob(input1); + CV_Assert(axis_blob.total() == 1u); + layerParams.set("axis", axis_blob.at(0)); + } + + addLayer(layerParams, node_proto); +} + void ONNXImporter::parseCustom(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { for (int j = 0; j < node_proto.input_size(); j++) { @@ -2456,6 +2475,7 @@ const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() dispatch["Upsample"] = &ONNXImporter::parseUpsample; dispatch["SoftMax"] = dispatch["LogSoftmax"] = &ONNXImporter::parseSoftMax; dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput; + dispatch["CumSum"] = &ONNXImporter::parseCumSum; dispatch["Custom"] = &ONNXImporter::parseCustom; return dispatch; diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 26e5fb632c93..737a184cbdd5 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -1362,6 +1362,15 @@ TEST_P(Test_ONNX_nets, Resnet34_kinetics) expectNoFallbacksFromIE(net); } +TEST_P(Test_ONNX_layers, CumSum) +{ + testONNXModels("cumsum_1d_exclusive_1"); + testONNXModels("cumsum_1d_reverse"); + testONNXModels("cumsum_1d_exclusive_1_reverse"); + testONNXModels("cumsum_2d_dim_1"); + testONNXModels("cumsum_3d_dim_2"); +} + INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_nets, dnnBackendsAndTargets()); }} // namespace From 46fb88c76fbe8d4b7ff3cc8eb53e5842753f6c3e Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Tue, 17 Aug 2021 20:11:22 +0300 Subject: [PATCH 120/376] Merge pull request #20546 from sivanov-work:initial_vpl_source G-API: oneVPL (simplification) source base commit * oneVPL source initial * Fix compilation * Fix compilation path * Fix NO VPL compile * Fix unused vars * Fix unused vars in example * Simplify oneVPL search: no custom path & download * Fix standalone GAPI * Apply comments --- modules/gapi/CMakeLists.txt | 15 ++ modules/gapi/cmake/init.cmake | 7 + modules/gapi/cmake/standalone.cmake | 7 + .../gapi/streaming/onevpl/onevpl_source.hpp | 44 +++ .../gapi/samples/onevpl_infer_single_roi.cpp | 254 ++++++++++++++++++ .../src/streaming/onevpl/onevpl_source.cpp | 48 ++++ .../streaming/onevpl/onevpl_source_priv.cpp | 63 +++++ .../streaming/onevpl/onevpl_source_priv.hpp | 62 +++++ 8 files changed, 500 insertions(+) create mode 100644 modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp create mode 100644 modules/gapi/samples/onevpl_infer_single_roi.cpp create mode 100644 modules/gapi/src/streaming/onevpl/onevpl_source.cpp create mode 100644 modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp create mode 100644 modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index c5046e8be6d8..69c0aaaae817 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -163,6 +163,10 @@ set(gapi_srcs src/backends/ie/bindings_ie.cpp src/backends/python/gpythonbackend.cpp + # Streaming source + src/streaming/onevpl/onevpl_source.cpp + src/streaming/onevpl/onevpl_source_priv.cpp + # Utils (ITT tracing) src/utils/itt.cpp ) @@ -234,6 +238,17 @@ if(HAVE_PLAIDML) ocv_target_include_directories(${the_module} SYSTEM PRIVATE ${PLAIDML_INCLUDE_DIRS}) endif() +if(HAVE_GAPI_ONEVPL) + if(TARGET opencv_test_gapi) + ocv_target_compile_definitions(opencv_test_gapi PRIVATE -DHAVE_ONEVPL) + ocv_target_link_libraries(opencv_test_gapi PRIVATE ${VPL_IMPORTED_TARGETS}) + endif() + ocv_target_compile_definitions(${the_module} PRIVATE -DHAVE_ONEVPL) + ocv_target_link_libraries(${the_module} PRIVATE ${VPL_IMPORTED_TARGETS}) + if(HAVE_D3D11 AND HAVE_OPENCL) + ocv_target_include_directories(${the_module} SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS}) + endif() +endif() if(WIN32) # Required for htonl/ntohl on Windows diff --git a/modules/gapi/cmake/init.cmake b/modules/gapi/cmake/init.cmake index 4c25c75f555c..1c464328ca1d 100644 --- a/modules/gapi/cmake/init.cmake +++ b/modules/gapi/cmake/init.cmake @@ -32,3 +32,10 @@ if(WITH_PLAIDML) set(HAVE_PLAIDML TRUE) endif() endif() + +if(WITH_GAPI_ONEVPL) + find_package(VPL) + if(VPL_FOUND) + set(HAVE_GAPI_ONEVPL TRUE) + endif() +endif() diff --git a/modules/gapi/cmake/standalone.cmake b/modules/gapi/cmake/standalone.cmake index d08eda1be5eb..f81c1c8a85de 100644 --- a/modules/gapi/cmake/standalone.cmake +++ b/modules/gapi/cmake/standalone.cmake @@ -6,6 +6,13 @@ if (NOT TARGET ade ) find_package(ade 0.1.0 REQUIRED) endif() +if (WITH_GAPI_ONEVPL) + find_package(VPL) + if(VPL_FOUND) + set(HAVE_GAPI_ONEVPL TRUE) + endif() +endif() + set(FLUID_TARGET fluid) set(FLUID_ROOT "${CMAKE_CURRENT_LIST_DIR}/../") diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp new file mode 100644 index 000000000000..fec8c73dffeb --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp @@ -0,0 +1,44 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP +#define OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP + +#include +#include +#include + +namespace cv { +namespace gapi { +namespace wip { + +class GAPI_EXPORTS OneVPLSource : public IStreamSource +{ +public: + struct Priv; + + explicit OneVPLSource(const std::string& filePath); + ~OneVPLSource() override; + + bool pull(cv::gapi::wip::Data& data) override; + GMetaArg descr_of() const override; + +private: + explicit OneVPLSource(std::unique_ptr&& impl); + std::unique_ptr m_priv; +}; + +template +GAPI_EXPORTS_W cv::Ptr inline make_vpl_src(const std::string& filePath, Args&&... args) +{ + return make_src(filePath, std::forward(args)...); +} + +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp new file mode 100644 index 000000000000..8a7efafabfd8 --- /dev/null +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -0,0 +1,254 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include // CommandLineParser + +const std::string about = + "This is an OpenCV-based version of oneVPLSource decoder example"; +const std::string keys = + "{ h help | | Print this help message }" + "{ input | | Path to the input demultiplexed video file }" + "{ output | | Path to the output RAW video file. Use .avi extension }" + "{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }" + "{ faced | CPU | Target device for face detection model (e.g. CPU, GPU, VPU, ...) }"; + +namespace { +std::string get_weights_path(const std::string &model_path) { + const auto EXT_LEN = 4u; + const auto sz = model_path.size(); + CV_Assert(sz > EXT_LEN); + + auto ext = model_path.substr(sz - EXT_LEN); + std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){ + return static_cast(std::tolower(c)); + }); + CV_Assert(ext == ".xml"); + return model_path.substr(0u, sz - EXT_LEN) + ".bin"; +} +} // anonymous namespace + +namespace custom { +G_API_NET(FaceDetector, , "face-detector"); + +using GDetections = cv::GArray; +using GRect = cv::GOpaque; +using GSize = cv::GOpaque; +using GPrims = cv::GArray; + +G_API_OP(LocateROI, , "sample.custom.locate-roi") { + static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &) { + return cv::empty_gopaque_desc(); + } +}; + +G_API_OP(ParseSSD, , "sample.custom.parse-ssd") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; + +G_API_OP(BBoxes, , "sample.custom.b-boxes") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; + +GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) { + // This is the place where we can run extra analytics + // on the input image frame and select the ROI (region + // of interest) where we want to detect our objects (or + // run any other inference). + // + // Currently it doesn't do anything intelligent, + // but only crops the input image to square (this is + // the most convenient aspect ratio for detectors to use) + + static void run(const cv::Size& in_size, cv::Rect &out_rect) { + + // Identify the central point & square size (- some padding) + const auto center = cv::Point{in_size.width/2, in_size.height/2}; + auto sqside = std::min(in_size.width, in_size.height); + + // Now build the central square ROI + out_rect = cv::Rect{ center.x - sqside/2 + , center.y - sqside/2 + , sqside + , sqside + }; + } +}; + +GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { + static void run(const cv::Mat &in_ssd_result, + const cv::Rect &in_roi, + const cv::Size &in_parent_size, + std::vector &out_objects) { + const auto &in_ssd_dims = in_ssd_result.size; + CV_Assert(in_ssd_dims.dims() == 4u); + + const int MAX_PROPOSALS = in_ssd_dims[2]; + const int OBJECT_SIZE = in_ssd_dims[3]; + CV_Assert(OBJECT_SIZE == 7); // fixed SSD object size + + const cv::Size up_roi = in_roi.size(); + const cv::Rect surface({0,0}, in_parent_size); + + out_objects.clear(); + + const float *data = in_ssd_result.ptr(); + for (int i = 0; i < MAX_PROPOSALS; i++) { + const float image_id = data[i * OBJECT_SIZE + 0]; + const float label = data[i * OBJECT_SIZE + 1]; + const float confidence = data[i * OBJECT_SIZE + 2]; + const float rc_left = data[i * OBJECT_SIZE + 3]; + const float rc_top = data[i * OBJECT_SIZE + 4]; + const float rc_right = data[i * OBJECT_SIZE + 5]; + const float rc_bottom = data[i * OBJECT_SIZE + 6]; + (void) label; // unused + + if (image_id < 0.f) { + break; // marks end-of-detections + } + if (confidence < 0.5f) { + continue; // skip objects with low confidence + } + + // map relative coordinates to the original image scale + // taking the ROI into account + cv::Rect rc; + rc.x = static_cast(rc_left * up_roi.width); + rc.y = static_cast(rc_top * up_roi.height); + rc.width = static_cast(rc_right * up_roi.width) - rc.x; + rc.height = static_cast(rc_bottom * up_roi.height) - rc.y; + rc.x += in_roi.x; + rc.y += in_roi.y; + out_objects.emplace_back(rc & surface); + } + } +}; + +GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { + // This kernel converts the rectangles into G-API's + // rendering primitives + static void run(const std::vector &in_face_rcs, + const cv::Rect &in_roi, + std::vector &out_prims) { + out_prims.clear(); + const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) { + return cv::gapi::wip::draw::Rect(rc, clr, 2); + }; + out_prims.emplace_back(cvt(in_roi, CV_RGB(0,255,255))); // cyan + for (auto &&rc : in_face_rcs) { + out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green + } + } +}; + +} // namespace custom + +int main(int argc, char *argv[]) { + + cv::CommandLineParser cmd(argc, argv, keys); + cmd.about(about); + if (cmd.has("help")) { + cmd.printMessage(); + return 0; + } + + // get file name + std::string file_path = cmd.get("input"); + const std::string output = cmd.get("output"); + const auto face_model_path = cmd.get("facem"); + + // check ouput file extension + if (!output.empty()) { + auto ext = output.find_last_of("."); + if (ext == std::string::npos || (output.substr(ext + 1) != "avi")) { + std::cerr << "Output file should have *.avi extension for output video" << std::endl; + return -1; + } + } + + auto face_net = cv::gapi::ie::Params { + face_model_path, // path to topology IR + get_weights_path(face_model_path), // path to weights + cmd.get("faced"), // device specifier + }; + auto kernels = cv::gapi::kernels + < custom::OCVLocateROI + , custom::OCVParseSSD + , custom::OCVBBoxes>(); + auto networks = cv::gapi::networks(face_net); + + // Create source + cv::Ptr cap; + try { + cap = cv::gapi::wip::make_vpl_src(file_path); + std::cout << "oneVPL source desription: " << cap->descr_of() << std::endl; + } catch (const std::exception& ex) { + std::cerr << "Cannot create source: " << ex.what() << std::endl; + return -1; + } + + cv::GMetaArg descr = cap->descr_of(); + auto frame_descr = cv::util::get(descr); + + // Now build the graph + cv::GFrame in; + auto size = cv::gapi::streaming::size(in); + auto roi = custom::LocateROI::on(size); + auto blob = cv::gapi::infer(roi, in); + auto rcs = custom::ParseSSD::on(blob, roi, size); + auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, roi)); + auto out = cv::gapi::streaming::BGR(out_frame); + + cv::GStreamingCompiled pipeline; + try { + pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) + .compileStreaming(cv::compile_args(kernels, networks)); + } catch (const std::exception& ex) { + std::cerr << "Exception occured during pipeline construction: " << ex.what() << std::endl; + return -1; + } + // The execution part + + // TODO USE may set pool size from outside and set queue_capacity size, + // compile arg: cv::gapi::streaming::queue_capacity + pipeline.setSource(std::move(cap)); + pipeline.start(); + + int framesCount = 0; + cv::TickMeter t; + cv::VideoWriter writer; + if (!output.empty() && !writer.isOpened()) { + const auto sz = cv::Size{frame_descr.size.width, frame_descr.size.height}; + writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz); + CV_Assert(writer.isOpened()); + } + + cv::Mat outMat; + t.start(); + while (pipeline.pull(cv::gout(outMat))) { + cv::imshow("Out", outMat); + cv::waitKey(1); + if (!output.empty()) { + writer << outMat; + } + framesCount++; + } + t.stop(); + std::cout << "Elapsed time: " << t.getTimeSec() << std::endl; + std::cout << "FPS: " << framesCount / t.getTimeSec() << std::endl; + std::cout << "framesCount: " << framesCount << std::endl; + + return 0; +} diff --git a/modules/gapi/src/streaming/onevpl/onevpl_source.cpp b/modules/gapi/src/streaming/onevpl/onevpl_source.cpp new file mode 100644 index 000000000000..988986f6d9d9 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/onevpl_source.cpp @@ -0,0 +1,48 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include + +#include "streaming/onevpl/onevpl_source_priv.hpp" + +namespace cv { +namespace gapi { +namespace wip { + +#ifdef HAVE_ONEVPL +OneVPLSource::OneVPLSource(const std::string& filePath) : + OneVPLSource(std::unique_ptr(new OneVPLSource::Priv(filePath))) { + + if (filePath.empty()) { + util::throw_error(std::logic_error("Cannot create 'OneVPLSource' on empty source file name")); + } +} +#else +OneVPLSource::OneVPLSource(const std::string&) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} +#endif + +OneVPLSource::OneVPLSource(std::unique_ptr&& impl) : + IStreamSource(), + m_priv(std::move(impl)) { +} + +OneVPLSource::~OneVPLSource() { +} + +bool OneVPLSource::pull(cv::gapi::wip::Data& data) +{ + return m_priv->pull(data); +} + +GMetaArg OneVPLSource::descr_of() const +{ + return m_priv->descr_of(); +} +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp b/modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp new file mode 100644 index 000000000000..5c4e8e694175 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp @@ -0,0 +1,63 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include +#include + +#include "streaming/onevpl/onevpl_source_priv.hpp" +#include "logger.hpp" + +#ifndef HAVE_ONEVPL +namespace cv { +namespace gapi { +namespace wip { +bool OneVPLSource::Priv::pull(cv::gapi::wip::Data&) { + return true; +} +GMetaArg OneVPLSource::Priv::descr_of() const { + return {}; +} +} // namespace wip +} // namespace gapi +} // namespace cv + +#else // HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { +OneVPLSource::Priv::Priv() : + mfx_handle(MFXLoad()) +{ + GAPI_LOG_INFO(nullptr, "Initialized MFX handle: " << mfx_handle); + description_is_valid = false; +} + +OneVPLSource::Priv::Priv(const std::string&) : + OneVPLSource::Priv() +{ +} + +OneVPLSource::Priv::~Priv() +{ + GAPI_LOG_INFO(nullptr, "Unload MFX handle: " << mfx_handle); + MFXUnload(mfx_handle); +} + +bool OneVPLSource::Priv::pull(cv::gapi::wip::Data&) +{ + return false; +} + +GMetaArg OneVPLSource::Priv::descr_of() const +{ + return {}; +} +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp b/modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp new file mode 100644 index 000000000000..b139add99372 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp @@ -0,0 +1,62 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_PRIV_HPP +#define OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_PRIV_HPP + +#include + +#include +#include + +#include +#include +#include + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif // MFX_VERSION + +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { + +struct OneVPLSource::Priv +{ + explicit Priv(const std::string& file_path); + ~Priv(); + + bool pull(cv::gapi::wip::Data& data); + GMetaArg descr_of() const; +private: + Priv(); + mfxLoader mfx_handle; + bool description_is_valid; +}; +} // namespace wip +} // namespace gapi +} // namespace cv + +#else // HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { +struct OneVPLSource::Priv final +{ + bool pull(cv::gapi::wip::Data&); + GMetaArg descr_of() const; +}; +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_PRIV_HPP From 95919051e0470237dd508a2086d8e519f5e091bf Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Wed, 18 Aug 2021 10:42:32 +0300 Subject: [PATCH 121/376] Merge pull request #20528 from TolyaTalamanov:at/fix-overwrite-blob-precision-bug [G-API] Prohibit setPrecision & preprocessing for importedNetworks * Prohibit preprocessing for imported networks * Fix typo input_names -> output_names * Move setBlob logic to separate function * Change comment --- modules/gapi/src/backends/ie/giebackend.cpp | 194 +++++++++++------- .../src/backends/ie/giebackend/giewrapper.cpp | 18 -- .../src/backends/ie/giebackend/giewrapper.hpp | 3 - 3 files changed, 121 insertions(+), 94 deletions(-) diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index fc9fc502ef6d..007f0db7afcc 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -243,10 +243,6 @@ struct IEUnit { this_plugin = cv::gimpl::ie::wrap::getPlugin(params); this_plugin.SetConfig(params.config); this_network = cv::gimpl::ie::wrap::importNetwork(this_plugin, params, rctx); - // FIXME: ICNNetwork returns InputsDataMap/OutputsDataMap, - // but ExecutableNetwork returns ConstInputsDataMap/ConstOutputsDataMap - inputs = cv::gimpl::ie::wrap::toInputsDataMap(this_network.GetInputsInfo()); - outputs = cv::gimpl::ie::wrap::toOutputsDataMap(this_network.GetOutputsInfo()); if (!params.reshape_table.empty() || !params.layer_names_to_reshape.empty()) { GAPI_LOG_WARNING(NULL, "Reshape isn't supported for imported network"); } @@ -270,10 +266,18 @@ struct IEUnit { + params.model_path)); } if (params.num_in == 1u && params.input_names.empty()) { - params.input_names = { inputs.begin()->first }; + if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + params.input_names = { inputs.begin()->first }; + } else { + params.input_names = { this_network.GetInputsInfo().begin()->first }; + } } if (params.num_out == 1u && params.output_names.empty()) { - params.output_names = { outputs.begin()->first }; + if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + params.output_names = { outputs.begin()->first }; + } else { + params.output_names = { this_network.GetOutputsInfo().begin()->first }; + } } if (!params.reshape_table.empty()) { GAPI_Assert((params.reshape_table.size() + params.layer_names_to_reshape.size()) <= @@ -533,6 +537,24 @@ inline IE::Blob::Ptr extractBlob(IECallContext& ctx, std::size_t i) { } GAPI_Assert(false); } + + +static void setBlob(InferenceEngine::InferRequest& req, + cv::gapi::ie::detail::ParamDesc::Kind kind, + const std::string& layer_name, + IE::Blob::Ptr blob) { + // NB: In case importNetwork preprocessing must be + // passed as SetBlob argument. + if (kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + req.SetBlob(layer_name, blob); + } else { + GAPI_Assert(kind == cv::gapi::ie::detail::ParamDesc::Kind::Import); + IE::PreProcessInfo info; + info.setResizeAlgorithm(IE::RESIZE_BILINEAR); + req.SetBlob(layer_name, blob, info); + } +} + } // anonymous namespace std::vector cv::gimpl::ie::IECompiled::createInferRequests() { @@ -891,25 +913,30 @@ struct Infer: public cv::detail::KernelTag { // meta order. GAPI_Assert(uu.params.input_names.size() == in_metas.size() && "Known input layers count doesn't match input meta count"); - for (auto &&it : ade::util::zip(ade::util::toRange(uu.params.input_names), - ade::util::toRange(in_metas))) { - const auto &input_name = std::get<0>(it); - auto &&ii = uu.inputs.at(input_name); - const auto & mm = std::get<1>(it); - configureInputInfo(ii, mm); - if (uu.params.layer_names_to_reshape.find(input_name) != - uu.params.layer_names_to_reshape.end()) { - configureInputReshapeByImage(ii, mm, input_reshape_table); + // NB: Configuring input precision and network reshape must be done + // only in the loadNetwork case. + if (uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + for (auto &&it : ade::util::zip(ade::util::toRange(uu.params.input_names), + ade::util::toRange(in_metas))) { + const auto &input_name = std::get<0>(it); + auto &&ii = uu.inputs.at(input_name); + const auto & mm = std::get<1>(it); + + configureInputInfo(ii, mm); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm, input_reshape_table); + } + ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); } - ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); - } - // FIXME: This isn't the best place to call reshape function. - // Сorrect solution would be to do this in compile() method of network, - // but now input meta isn't passed to compile() method. - if (!input_reshape_table.empty()) { - const_cast(&uu.net)->reshape(input_reshape_table); + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } } // FIXME: It would be nice here to have an exact number of network's @@ -941,7 +968,10 @@ struct Infer: public cv::detail::KernelTag { // and redirect our data producers to this memory // (A memory dialog comes to the picture again) IE::Blob::Ptr this_blob = extractBlob(*ctx, i); - req.SetBlob(ctx->uu.params.input_names[i], this_blob); + setBlob(req, + ctx->uu.params.kind, + ctx->uu.params.input_names[i], + this_blob); } // FIXME: Should it be done by kernel ? // What about to do that in RequestPool ? @@ -973,22 +1003,26 @@ struct InferROI: public cv::detail::KernelTag { GAPI_Assert(1u == uu.params.input_names.size()); GAPI_Assert(2u == in_metas.size()); - // 0th is ROI, 1st is input image - const auto &input_name = uu.params.input_names.at(0); - auto &&ii = uu.inputs.at(input_name); - auto &&mm = in_metas.at(1u); - configureInputInfo(ii, mm); - if (uu.params.layer_names_to_reshape.find(input_name) != - uu.params.layer_names_to_reshape.end()) { - configureInputReshapeByImage(ii, mm, input_reshape_table); - } - ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); + // NB: Configuring input precision and network reshape must be done + // only in the loadNetwork case. + if (uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + // 0th is ROI, 1st is input image + const auto &input_name = uu.params.input_names.at(0); + auto &&ii = uu.inputs.at(input_name); + auto &&mm = in_metas.at(1u); + configureInputInfo(ii, mm); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm, input_reshape_table); + } + ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); - // FIXME: This isn't the best place to call reshape function. - // Сorrect solution would be to do this in compile() method of network, - // but now input meta isn't passed to compile() method. - if (!input_reshape_table.empty()) { - const_cast(&uu.net)->reshape(input_reshape_table); + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } } // FIXME: It would be nice here to have an exact number of network's @@ -1017,10 +1051,11 @@ struct InferROI: public cv::detail::KernelTag { auto&& this_roi = ctx->inArg(0).rref(); IE::Blob::Ptr this_blob = extractBlob(*ctx, 1); - - req.SetBlob(*(ctx->uu.params.input_names.begin()), - IE::make_shared_blob(this_blob, toIE(this_roi))); - + setBlob(req, + ctx->uu.params.kind, + *(ctx->uu.params.input_names.begin()), + IE::make_shared_blob(this_blob, + toIE(this_roi))); // FIXME: Should it be done by kernel ? // What about to do that in RequestPool ? req.StartAsync(); @@ -1055,23 +1090,27 @@ struct InferList: public cv::detail::KernelTag { GAPI_Assert(uu.params.input_names.size() == (in_metas.size() - 1u) && "Known input layers count doesn't match input meta count"); - std::size_t idx = 1u; - for (auto &&input_name : uu.params.input_names) { - auto &&ii = uu.inputs.at(input_name); - const auto & mm = in_metas[idx++]; - configureInputInfo(ii, mm); - if (uu.params.layer_names_to_reshape.find(input_name) != - uu.params.layer_names_to_reshape.end()) { - configureInputReshapeByImage(ii, mm, input_reshape_table); + // NB: Configuring input precision and network reshape must be done + // only in the loadNetwork case. + if (uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + std::size_t idx = 1u; + for (auto &&input_name : uu.params.input_names) { + auto &&ii = uu.inputs.at(input_name); + const auto & mm = in_metas[idx++]; + configureInputInfo(ii, mm); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm, input_reshape_table); + } + ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); } - ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); - } - // FIXME: This isn't the best place to call reshape function. - // Сorrect solution would be to do this in compile() method of network, - // but now input meta isn't passed to compile() method. - if (!input_reshape_table.empty()) { - const_cast(&uu.net)->reshape(input_reshape_table); + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } } // roi-list version is much easier at the moment. @@ -1117,7 +1156,10 @@ struct InferList: public cv::detail::KernelTag { cv::gimpl::ie::RequestPool::Task { [ctx, rc, this_blob](InferenceEngine::InferRequest &req) { IE::Blob::Ptr roi_blob = IE::make_shared_blob(this_blob, toIE(rc)); - req.SetBlob(ctx->uu.params.input_names[0u], roi_blob); + setBlob(req, + ctx->uu.params.kind, + ctx->uu.params.input_names[0u], + roi_blob); req.StartAsync(); }, std::bind(callback, std::placeholders::_1, pos) @@ -1191,19 +1233,23 @@ struct InferList2: public cv::detail::KernelTag { && "Non-array inputs are not supported"); if (op.k.inKinds[idx] == cv::detail::OpaqueKind::CV_RECT) { - // This is a cv::Rect -- configure the IE preprocessing - configureInputInfo(ii, mm_0); - if (uu.params.layer_names_to_reshape.find(input_name) != - uu.params.layer_names_to_reshape.end()) { - configureInputReshapeByImage(ii, mm_0, input_reshape_table); - } - ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); - - // FIXME: This isn't the best place to call reshape function. - // Сorrect solution would be to do this in compile() method of network, - // but now input meta isn't passed to compile() method. - if (!input_reshape_table.empty()) { - const_cast(&uu.net)->reshape(input_reshape_table); + // NB: Configuring input precision and network reshape must be done + // only in the loadNetwork case. + if (uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + // This is a cv::Rect -- configure the IE preprocessing + configureInputInfo(ii, mm_0); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm_0, input_reshape_table); + } + ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); + + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } } } else { // This is a cv::GMat (equals to: cv::Mat) @@ -1268,8 +1314,10 @@ struct InferList2: public cv::detail::KernelTag { GAPI_Assert(false && "Only Rect and Mat types are supported for infer list 2!"); } - - req.SetBlob(ctx->uu.params.input_names[in_idx], this_blob); + setBlob(req, + ctx->uu.params.kind, + ctx->uu.params.input_names[in_idx], + this_blob); } req.StartAsync(); }, diff --git a/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp b/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp index d4ec806e4846..1f9721dbf4ef 100644 --- a/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp +++ b/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp @@ -22,24 +22,6 @@ namespace IE = InferenceEngine; namespace giewrap = cv::gimpl::ie::wrap; using GIEParam = cv::gapi::ie::detail::ParamDesc; -IE::InputsDataMap giewrap::toInputsDataMap (const IE::ConstInputsDataMap& inputs) { - IE::InputsDataMap transformed; - auto convert = [](const std::pair& p) { - return std::make_pair(p.first, std::const_pointer_cast(p.second)); - }; - std::transform(inputs.begin(), inputs.end(), std::inserter(transformed, transformed.end()), convert); - return transformed; -} - -IE::OutputsDataMap giewrap::toOutputsDataMap (const IE::ConstOutputsDataMap& outputs) { - IE::OutputsDataMap transformed; - auto convert = [](const std::pair& p) { - return std::make_pair(p.first, std::const_pointer_cast(p.second)); - }; - std::transform(outputs.begin(), outputs.end(), std::inserter(transformed, transformed.end()), convert); - return transformed; -} - #if INF_ENGINE_RELEASE < 2020000000 // < 2020.1 // Load extensions (taken from DNN module) std::vector giewrap::getExtensions(const GIEParam& params) { diff --git a/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp b/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp index 7e67cb8989d6..2e4bac12704a 100644 --- a/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp +++ b/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp @@ -29,9 +29,6 @@ namespace wrap { GAPI_EXPORTS std::vector getExtensions(const GIEParam& params); GAPI_EXPORTS IE::CNNNetwork readNetwork(const GIEParam& params); -IE::InputsDataMap toInputsDataMap (const IE::ConstInputsDataMap& inputs); -IE::OutputsDataMap toOutputsDataMap(const IE::ConstOutputsDataMap& outputs); - #if INF_ENGINE_RELEASE < 2019020000 // < 2019.R2 using Plugin = IE::InferencePlugin; GAPI_EXPORTS IE::InferencePlugin getPlugin(const GIEParam& params); From 03b989251d30efff3e90e0b5e02bae1e2c4d825f Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Wed, 18 Aug 2021 18:12:27 +0300 Subject: [PATCH 122/376] Check adapter in executor --- modules/gapi/src/executor/gexecutor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/gapi/src/executor/gexecutor.cpp b/modules/gapi/src/executor/gexecutor.cpp index 6f313197ba36..9b51e70d5dae 100644 --- a/modules/gapi/src/executor/gexecutor.cpp +++ b/modules/gapi/src/executor/gexecutor.cpp @@ -159,8 +159,8 @@ void writeBackExec(const Mag& mag, const RcDesc &rc, GRunArgP &g_arg) // a real copy (add a pass to StreamingBackend?) auto& out_mat = *util::get(g_arg); const auto& rmat = mag.template slot().at(rc.id); - auto mag_data = rmat.get()->data(); - if (out_mat.data != mag_data) { + auto* adapter = rmat.get(); + if (adapter != nullptr && out_mat.data != adapter->data()) { auto view = rmat.access(RMat::Access::R); asMat(view).copyTo(out_mat); } From fe625a558e0c9ab56f781c47bb0db25079368681 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Wed, 18 Aug 2021 18:37:35 +0300 Subject: [PATCH 123/376] fix hasDynamicShapes for batch_size and fix axis selection in Scale layer --- modules/dnn/src/onnx/onnx_importer.cpp | 13 ++++++++++++- modules/dnn/test/test_onnx_importer.cpp | 2 ++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 32b56278bda7..dd84e3b7cd5b 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -475,7 +475,8 @@ void ONNXImporter::populateNet() for (int j = 0; j < inpShape.size(); ++j) { inpShape[j] = tensorShape.dim(j).dim_value(); - if (!tensorShape.dim(j).dim_param().empty()) + // NHW, NCHW(NHWC), NCDHW(NDHWC); do not set this flag if only N is dynamic + if (!tensorShape.dim(j).dim_param().empty() && !(j == 0 && inpShape.size() >= 3)) hasDynamicShapes = true; } if (!inpShape.empty() && !hasDynamicShapes) @@ -1407,6 +1408,16 @@ void ONNXImporter::parseMul(LayerParams& layerParams, const opencv_onnx::NodePro //Replace input to Power node_proto.set_input(1, powerParams.name); } + + const MatShape& broadShape = outShapes[node_proto.input(1)]; + const size_t outShapeSize = outShapes[node_proto.input(0)].size(); + const size_t diff = outShapeSize - broadShape.size(); + + size_t axis; + for (axis = diff; axis < broadShape.size() && broadShape[axis - diff] == 1; ++axis) {} + + CV_Assert(axis != outShapeSize); + layerParams.set("axis", static_cast(axis)); layerParams.type = "Scale"; } addLayer(layerParams, node_proto); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index a446a37c7944..d13b39064ec8 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -284,6 +284,7 @@ TEST_P(Test_ONNX_layers, Scale) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); testONNXModels("scale"); + testONNXModels("scale_broadcast", npy, 0, 0, false, true, 3); } TEST_P(Test_ONNX_layers, ReduceMean3D) @@ -825,6 +826,7 @@ TEST_P(Test_ONNX_layers, DynamicAxes) testONNXModels("resize_opset11_torch1.6_dynamic_axes"); testONNXModels("average_pooling_dynamic_axes"); testONNXModels("maxpooling_sigmoid_dynamic_axes"); + testONNXModels("dynamic_batch"); } TEST_P(Test_ONNX_layers, MaxPool1d) From fa90e14b0606b76a4efc9739c4ea87e659c2aab8 Mon Sep 17 00:00:00 2001 From: SamFC10 Date: Thu, 19 Aug 2021 09:56:47 +0530 Subject: [PATCH 124/376] int8 layers and 8-bit quantization support --- modules/dnn/CMakeLists.txt | 1 + .../dnn/include/opencv2/dnn/all_layers.hpp | 87 +- modules/dnn/include/opencv2/dnn/dnn.hpp | 46 + modules/dnn/src/dnn.cpp | 383 +++++- modules/dnn/src/init.cpp | 38 + .../dnn/src/int8layers/batch_norm_layer.cpp | 178 +++ .../dnn/src/int8layers/convolution_layer.cpp | 1136 +++++++++++++++ .../dnn/src/int8layers/elementwise_layers.cpp | 190 +++ modules/dnn/src/int8layers/eltwise_layer.cpp | 577 ++++++++ .../src/int8layers/fully_connected_layer.cpp | 266 ++++ modules/dnn/src/int8layers/layers_common.hpp | 41 + .../dnn/src/int8layers/layers_common.simd.hpp | 637 +++++++++ modules/dnn/src/int8layers/pooling_layer.cpp | 595 ++++++++ .../int8layers/quantize_dequantize_layer.cpp | 157 +++ modules/dnn/src/int8layers/scale_layer.cpp | 211 +++ modules/dnn/src/int8layers/softmax_layer.cpp | 176 +++ modules/dnn/src/layers/batch_norm_layer.cpp | 12 + modules/dnn/src/layers/blank_layer.cpp | 5 + modules/dnn/src/layers/concat_layer.cpp | 32 +- modules/dnn/src/layers/const_layer.cpp | 9 + modules/dnn/src/layers/convolution_layer.cpp | 42 + modules/dnn/src/layers/elementwise_layers.cpp | 179 +++ modules/dnn/src/layers/eltwise_layer.cpp | 31 + modules/dnn/src/layers/flatten_layer.cpp | 5 + .../dnn/src/layers/fully_connected_layer.cpp | 39 + modules/dnn/src/layers/padding_layer.cpp | 12 + modules/dnn/src/layers/permute_layer.cpp | 63 +- modules/dnn/src/layers/pooling_layer.cpp | 17 + modules/dnn/src/layers/reorg_layer.cpp | 5 + modules/dnn/src/layers/reshape_layer.cpp | 5 + modules/dnn/src/layers/scale_layer.cpp | 8 + .../dnn/src/layers/shuffle_channel_layer.cpp | 6 + modules/dnn/src/layers/slice_layer.cpp | 30 +- modules/dnn/src/layers/softmax_layer.cpp | 16 + modules/dnn/src/layers/split_layer.cpp | 11 + modules/dnn/test/test_int8_layers.cpp | 1220 +++++++++++++++++ 36 files changed, 6400 insertions(+), 66 deletions(-) create mode 100644 modules/dnn/src/int8layers/batch_norm_layer.cpp create mode 100644 modules/dnn/src/int8layers/convolution_layer.cpp create mode 100644 modules/dnn/src/int8layers/elementwise_layers.cpp create mode 100644 modules/dnn/src/int8layers/eltwise_layer.cpp create mode 100644 modules/dnn/src/int8layers/fully_connected_layer.cpp create mode 100644 modules/dnn/src/int8layers/layers_common.hpp create mode 100644 modules/dnn/src/int8layers/layers_common.simd.hpp create mode 100644 modules/dnn/src/int8layers/pooling_layer.cpp create mode 100644 modules/dnn/src/int8layers/quantize_dequantize_layer.cpp create mode 100644 modules/dnn/src/int8layers/scale_layer.cpp create mode 100644 modules/dnn/src/int8layers/softmax_layer.cpp create mode 100644 modules/dnn/test/test_int8_layers.cpp diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 70f9a5a73e5a..c1236c4653f2 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -9,6 +9,7 @@ endif() set(the_description "Deep neural network module. It allows to load models from different frameworks and to make forward pass") ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX RVV) +ocv_add_dispatched_file_force_all("int8layers/layers_common" AVX2 AVX512_SKX) ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java objc js) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 794bfeedda33..fbe16850d4d5 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -258,6 +258,14 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS ConvolutionLayerInt8 : public BaseConvolutionLayer + { + public: + int input_zp, output_zp; + float output_sc; + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS DeconvolutionLayer : public BaseConvolutionLayer { public: @@ -300,6 +308,13 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS PoolingLayerInt8 : public PoolingLayer + { + public: + int input_zp, output_zp; + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS SoftmaxLayer : public Layer { public: @@ -308,6 +323,14 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS SoftmaxLayerInt8 : public SoftmaxLayer + { + public: + float output_sc; + int output_zp; + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS InnerProductLayer : public Layer { public: @@ -315,6 +338,13 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS InnerProductLayerInt8 : public InnerProductLayer + { + public: + int output_zp; + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS MVNLayer : public Layer { public: @@ -341,6 +371,22 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS QuantizeLayer : public Layer + { + public: + float scale; + int zeropoint; + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS DequantizeLayer : public Layer + { + public: + float scale; + int zeropoint; + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS ConcatLayer : public Layer { public: @@ -352,6 +398,7 @@ CV__DNN_INLINE_NS_BEGIN * Details: https://github.com/torch/nn/blob/master/doc/containers.md#depthconcat */ bool padding; + int paddingValue; static Ptr create(const LayerParams ¶ms); }; @@ -459,7 +506,11 @@ CV__DNN_INLINE_NS_BEGIN { public: virtual void forwardSlice(const float* src, float* dst, int len, - size_t outPlaneSize, int cn0, int cn1) const = 0; + size_t outPlaneSize, int cn0, int cn1) const {}; + virtual void forwardSlice(const int* src, const int* lut, int* dst, int len, + size_t outPlaneSize, int cn0, int cn1) const {}; + virtual void forwardSlice(const int8_t* src, const int8_t* lut, int8_t* dst, int len, + size_t outPlaneSize, int cn0, int cn1) const {}; }; class CV_EXPORTS ReLULayer : public ActivationLayer @@ -542,6 +593,12 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS ActivationLayerInt8 : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + /* Layers used in semantic segmentation */ class CV_EXPORTS CropLayer : public Layer @@ -563,6 +620,12 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS EltwiseLayerInt8 : public Layer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS BatchNormLayer : public ActivationLayer { public: @@ -572,6 +635,14 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS BatchNormLayerInt8 : public BatchNormLayer + { + public: + float input_sc, output_sc; + int input_zp, output_zp; + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS MaxUnpoolLayer : public Layer { public: @@ -591,12 +662,26 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS ScaleLayerInt8 : public ScaleLayer + { + public: + float output_sc; + int output_zp; + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS ShiftLayer : public Layer { public: static Ptr create(const LayerParams& params); }; + class CV_EXPORTS ShiftLayerInt8 : public Layer + { + public: + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS DataAugmentationLayer : public Layer { public: diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index a498039f6571..bf1670051ac0 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -235,6 +235,15 @@ CV__DNN_INLINE_NS_BEGIN */ virtual void forward(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals); + /** @brief Tries to quantize the given layer and compute the quantization parameters required for fixed point implementation. + * @param[in] scales input and output scales. + * @param[in] zeropoints input and output zeropoints. + * @param[out] params Quantized parameters required for fixed point implementation of that layer. + * @returns True if layer can be quantized. + */ + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params); + /** @brief Given the @p input blobs, computes the output @p blobs. * @param[in] inputs the input blobs. * @param[out] outputs allocated output blobs, which will store results of the computation. @@ -368,6 +377,16 @@ CV__DNN_INLINE_NS_BEGIN */ virtual void getScaleShift(Mat& scale, Mat& shift) const; + /** + * @brief Returns scale and zeropoint of layers + * @param[out] scale Output scale + * @param[out] zeropoint Output zeropoint + * + * By default, @p scale is 1 and @p zeropoint is 0. + */ + virtual void getScaleZeropoint(float& scale, int& zeropoint) const; + + /** * @brief "Deattaches" all the layers, attached to particular layer. */ @@ -453,13 +472,21 @@ CV__DNN_INLINE_NS_BEGIN /** @brief Adds new layer to the net. * @param name unique name of the adding layer. * @param type typename of the adding layer (type must be registered in LayerRegister). + * @param dtype datatype of output blobs. * @param params parameters which will be used to initialize the creating layer. * @returns unique identifier of created layer, or -1 if a failure will happen. */ + int addLayer(const String &name, const String &type, const int &dtype, LayerParams ¶ms); + + /** @overload Datatype of output blobs set to default CV_32F */ int addLayer(const String &name, const String &type, LayerParams ¶ms); + /** @brief Adds new layer and connects its first input to the first output of previously added layer. * @see addLayer() */ + int addLayerToPrev(const String &name, const String &type, const int &dtype, LayerParams ¶ms); + + /** @overload */ int addLayerToPrev(const String &name, const String &type, LayerParams ¶ms); /** @brief Converts string name of the layer to the integer identifier. @@ -551,6 +578,25 @@ CV__DNN_INLINE_NS_BEGIN CV_WRAP_AS(forwardAndRetrieve) void forward(CV_OUT std::vector >& outputBlobs, const std::vector& outBlobNames); + /** @brief Returns a quantized Net from a floating-point Net. + * @param calibData Calibration data to compute the quantization parameters. + * @param inputsDtype Datatype of quantized net's inputs. Can be CV_32F or CV_8S. + * @param outputsDtype Datatype of quantized net's outputs. Can be CV_32F or CV_8S. + */ + CV_WRAP Net quantize(InputArrayOfArrays calibData, int inputsDtype, int outputsDtype); + + /** @brief Returns input scale and zeropoint for a quantized Net. + * @param scales output parameter for returning input scales. + * @param zeropoints output parameter for returning input zeropoints. + */ + CV_WRAP void getInputDetails(CV_OUT std::vector& scales, CV_OUT std::vector& zeropoints) const; + + /** @brief Returns output scale and zeropoint for a quantized Net. + * @param scales output parameter for returning output scales. + * @param zeropoints output parameter for returning output zeropoints. + */ + CV_WRAP void getOutputDetails(CV_OUT std::vector& scales, CV_OUT std::vector& zeropoints) const; + /** * @brief Compile Halide layers. * @param[in] scheduler Path to YAML file with scheduling directives. diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 52a5fcba28ce..492ad166d038 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -574,9 +574,9 @@ struct LayerPin struct LayerData { - LayerData() : id(-1), skip(false), flag(0) {} - LayerData(int _id, const String &_name, const String &_type, LayerParams &_params) - : id(_id), name(_name), type(_type), params(_params), skip(false), flag(0) + LayerData() : id(-1), dtype(CV_32F), skip(false), flag(0) {} + LayerData(int _id, const String &_name, const String &_type, const int &_dtype, LayerParams &_params) + : id(_id), name(_name), type(_type), dtype(_dtype), params(_params), skip(false), flag(0) { CV_TRACE_FUNCTION(); @@ -588,6 +588,7 @@ struct LayerData int id; String name; String type; + int dtype; // Datatype of output blobs. LayerParams params; std::vector inputBlobsId; @@ -944,7 +945,7 @@ struct BlobManager } } - void reuseOrCreate(const MatShape& shape, const LayerPin& lp, Mat& dst, bool use_half) + void reuseOrCreate(const MatShape& shape, const LayerPin& lp, Mat& dst, const int& dtype) { if (!DNN_DISABLE_MEMORY_OPTIMIZATIONS) { @@ -966,7 +967,8 @@ struct BlobManager { Mat& unusedBlob = hostIt->second; if (unusedBlob.total() >= targetTotal && - unusedBlob.total() < bestBlobTotal) + unusedBlob.total() < bestBlobTotal && + unusedBlob.type() == dtype) { bestBlobPin = hostIt->first; bestBlob = unusedBlob; @@ -985,14 +987,13 @@ struct BlobManager { // if dst already has been allocated with total(shape) elements, // it won't be recreated and pointer of dst.data remains the same. - dst.create(shape, use_half ? CV_16S : CV_32F); + dst.create(shape, dtype); addHost(lp, dst); } } void allocateBlobsForLayer(LayerData &ld, const LayerShapes& layerShapes, - std::vector& pinsForInternalBlobs, - bool use_half = false) + std::vector& pinsForInternalBlobs) { CV_TRACE_FUNCTION(); @@ -1063,7 +1064,7 @@ struct BlobManager reuse(ld.inputBlobsId[0], blobPin); } else - reuseOrCreate(shapes[index], blobPin, *blobs[index], use_half); + reuseOrCreate(shapes[index], blobPin, *blobs[index], ld.dtype); } } } @@ -1193,6 +1194,7 @@ struct Net::Impl : public detail::NetImplBase lastLayerId = 0; netWasAllocated = false; + netWasQuantized = false; fusion = true; isAsync = false; preferableBackend = DNN_BACKEND_DEFAULT; @@ -1217,6 +1219,7 @@ struct Net::Impl : public detail::NetImplBase int lastLayerId; bool netWasAllocated; + bool netWasQuantized; bool fusion; bool isAsync; std::vector layersTimings; @@ -1372,7 +1375,7 @@ struct Net::Impl : public detail::NetImplBase currLayer->unsetAttached(); } - + netWasAllocated = false; layersTimings.clear(); } @@ -2541,10 +2544,11 @@ struct Net::Impl : public detail::NetImplBase CV_Assert(layerShapesIt != layersShapes.end()); + if (preferableBackend == DNN_BACKEND_OPENCV && preferableTarget == DNN_TARGET_OPENCL_FP16 && ld.dtype == CV_32F) + ld.dtype = CV_16S; + std::vector pinsForInternalBlobs; - blobManager.allocateBlobsForLayer(ld, layerShapesIt->second, pinsForInternalBlobs, - preferableBackend == DNN_BACKEND_OPENCV && - preferableTarget == DNN_TARGET_OPENCL_FP16); + blobManager.allocateBlobsForLayer(ld, layerShapesIt->second, pinsForInternalBlobs); ld.outputBlobsWrappers.resize(ld.outputBlobs.size()); for (int i = 0; i < ld.outputBlobs.size(); ++i) ld.outputBlobsWrappers[i] = wrap(ld.outputBlobs[i]); @@ -3148,7 +3152,8 @@ struct Net::Impl : public detail::NetImplBase Mat& inp = layers[0].outputBlobs[i]; CV_Assert(inp.total()); if (preferableBackend == DNN_BACKEND_OPENCV && - preferableTarget == DNN_TARGET_OPENCL_FP16) + preferableTarget == DNN_TARGET_OPENCL_FP16 && + layers[0].dtype == CV_32F) { layers[0].outputBlobs[i].create(inp.dims, inp.size, CV_16S); } @@ -3458,6 +3463,25 @@ struct Net::Impl : public detail::NetImplBase #endif } + void getQuantizationParams(const Mat& src, std::vector& scales, std::vector& zeropoints) + { + const int qmin = -128; // INT8_MIN + const int qmax = 127; // INT8_MAX + + double rmin, rmax, sc, zp; + cv::minMaxIdx(src, &rmin, &rmax); + + // 0 must be present in the range [rmin, rmax] + rmin = std::min(rmin, 0.0); + rmax = std::max(rmax, 0.0); + + sc = (rmax == rmin) ? 1.0 : (rmax - rmin)/(qmax - qmin); + zp = qmin - (rmin/sc); + + scales.push_back((float)sc); + zeropoints.push_back((int)std::round(zp)); + } + void getLayerShapesRecursively(int id, LayersShapesMap& inOutShapes) { std::vector& inputLayerIds = layers[id].inputBlobsId; @@ -3588,7 +3612,8 @@ struct Net::Impl : public detail::NetImplBase Mat& inp = layers[0].outputBlobs[i]; CV_Assert(inp.total()); if (preferableBackend == DNN_BACKEND_OPENCV && - preferableTarget == DNN_TARGET_OPENCL_FP16) + preferableTarget == DNN_TARGET_OPENCL_FP16 && + layers[0].dtype == CV_32F) { layers[0].outputBlobs[i].create(inp.dims, inp.size, CV_16S); } @@ -3614,7 +3639,7 @@ struct Net::Impl : public detail::NetImplBase const MatShape& shape = layersShapes[inputLayerId].out[inputLayerIds[i].oid]; layersShapes[layerId].in.push_back(shape); } - it->second.layerInstance->updateMemoryShapes(layersShapes[layerId].in); + it->second.getLayerInstance()->updateMemoryShapes(layersShapes[layerId].in); } } } @@ -4019,7 +4044,7 @@ Net::~Net() { } -int Net::addLayer(const String &name, const String &type, LayerParams ¶ms) +int Net::addLayer(const String &name, const String &type, const int &dtype, LayerParams ¶ms) { CV_TRACE_FUNCTION(); @@ -4042,23 +4067,35 @@ int Net::addLayer(const String &name, const String &type, LayerParams ¶ms) id = ++impl->lastLayerId; impl->layerNameToId.insert(std::make_pair(name, id)); - impl->layers.insert(std::make_pair(id, LayerData(id, name, type, params))); + impl->layers.insert(std::make_pair(id, LayerData(id, name, type, dtype, params))); if (params.get("has_dynamic_shapes", false)) impl->hasDynamicShapes = true; return id; } -int Net::addLayerToPrev(const String &name, const String &type, LayerParams ¶ms) +int Net::addLayer(const String &name, const String &type, LayerParams ¶ms) +{ + CV_TRACE_FUNCTION(); + return addLayer(name, type, CV_32F, params); +} + +int Net::addLayerToPrev(const String &name, const String &type, const int &dtype, LayerParams ¶ms) { CV_TRACE_FUNCTION(); int prvLid = impl->lastLayerId; - int newLid = this->addLayer(name, type, params); + int newLid = this->addLayer(name, type, dtype, params); this->connect(prvLid, 0, newLid, 0); return newLid; } +int Net::addLayerToPrev(const String &name, const String &type, LayerParams ¶ms) +{ + CV_TRACE_FUNCTION(); + return addLayerToPrev(name, type, CV_32F, params); +} + void Net::connect(int outLayerId, int outNum, int inpLayerId, int inpNum) { CV_TRACE_FUNCTION(); @@ -4169,16 +4206,19 @@ void Net::forward(OutputArrayOfArrays outputBlobs, const String& outputName) ld.outputBlobsWrappers[i]->copyToHost(); } } - if (ld.outputBlobs[0].depth() == CV_32F) + if (ld.outputBlobs[0].depth() == CV_16S) { - std::vector & outputvec = *(std::vector *)outputBlobs.getObj(); - outputvec = ld.outputBlobs; - } else { std::vector & outputvec = *(std::vector *)outputBlobs.getObj(); outputvec.resize(ld.outputBlobs.size()); for (int i = 0; i < outputvec.size(); i++) convertFp16(ld.outputBlobs[i], outputvec[i]); } + else + { + // Output depth can be CV_32F or CV_8S + std::vector & outputvec = *(std::vector *)outputBlobs.getObj(); + outputvec = ld.outputBlobs; + } } else if (outputBlobs.isUMatVector()) { @@ -4264,11 +4304,277 @@ void Net::forward(std::vector >& outputBlobs, } } +Net Net::quantize(InputArrayOfArrays calibData, int inputsDtype, int outputsDtype) +{ + CV_TRACE_FUNCTION(); + + // Net can be quantized only once. + if (impl->netWasQuantized) + CV_Error(Error::StsBadArg, "Cannot quantize a quantized net"); + + CV_CheckType(inputsDtype, inputsDtype == CV_32F || inputsDtype == CV_8S, "Input depth should be CV_32F or CV_8S"); + CV_CheckType(outputsDtype, outputsDtype == CV_32F || outputsDtype == CV_8S, "Output depth should be CV_32F or CV_8S"); + + bool originalFusion = impl->fusion; + int prefBackend = impl->preferableBackend; + int prefTarget = impl->preferableTarget; + + // Disable fusions and use CPU backend to quantize net + setPreferableBackend(DNN_BACKEND_OPENCV); + setPreferableTarget(DNN_TARGET_CPU); + enableFusion(false); + + if (calibData.isMat()) + { + setInput(calibData.getMat()); + } + else if (calibData.isMatVector()) + { + std::vector calibDataVec; + calibData.getMatVector(calibDataVec); + + std::vector inpNames = impl->netInputLayer->outNames; + CV_CheckEQ(calibDataVec.size(), inpNames.size(), "Calibration data size should be equal to number of inputs"); + for (int i = 0; i < calibDataVec.size(); i++) + setInput(calibDataVec[i], inpNames[i]); + } + + std::vector outNames = getUnconnectedOutLayersNames(); + std::vector pins; + for (int i = 0; i < outNames.size(); i++) + pins.push_back(impl->getPinByAlias(outNames[i])); + impl->setUpNet(pins); + + // Compute scales and zeropoints for all the layers + std::vector > scales; + std::vector > zeropoints; + for (Impl::MapIdToLayerData::iterator it = impl->layers.begin(); it != impl->layers.end(); it++) + { + LayerData& ld = it->second; + if (!ld.skip) + { + Ptr layer = ld.layerInstance; + std::vector inps(ld.inputBlobs.size()); + for (int i = 0; i < ld.inputBlobs.size(); ++i) + inps[i] = *ld.inputBlobs[i]; + layer->forward(inps, ld.outputBlobs, ld.internals); + } + + std::vector sc; + std::vector zp; + if (ld.type == "TanH") + { + sc.push_back(1.f/128); + zp.push_back(0); + } + else if (ld.type == "Sigmoid" || ld.type == "Softmax" || ld.type == "SoftMax") + { + if (ld.params.get("log_softmax", false)) + { + sc.push_back(16.f/256); + zp.push_back(127); + } + else + { + sc.push_back(1.f/256); + zp.push_back(-128); + } + } + else if (ld.type == "Split" || ld.type == "Slice" || ld.type == "Crop") + { + std::vector inp_sc; std::vector inp_zp; + impl->getQuantizationParams(*ld.inputBlobs[0], inp_sc, inp_zp); + sc.assign(ld.outputBlobs.size(), inp_sc[0]); + zp.assign(ld.outputBlobs.size(), inp_zp[0]); + } + else + { + for (int i = 0; i < ld.outputBlobs.size(); i++) + impl->getQuantizationParams(ld.outputBlobs[i], sc, zp); + } + scales.push_back(sc); + zeropoints.push_back(zp); + } + + // For some layers, the input and output scales/zeropoints must be equal so that rescaling of inputs + // is not needed during quantized inference. We start from the last layer and modify the layer's input scales/zeropoints + // TODO : Need a different approach. Current solution fails when 2 such layers have the same input layer + for (Impl::MapIdToLayerData::reverse_iterator it = impl->layers.rbegin(); it != impl->layers.rend(); ++it) + { + LayerData& ld = it->second; + // Layers with multiple outputs. Number of outputs is equal to number of inputs + if (ld.type == "Blank" || ld.type == "Dropout" || ld.type == "Identity" || ld.type == "Silence" || + ld.type == "Flatten" || ld.type == "Padding" || ld.type == "Permute" || ld.type == "Reshape" || + ld.type == "ReLU6" || ld.type == "Reorg" || ld.type == "ShuffleChannel" || + (ld.type == "ReLU" && !ld.params.get("negative_slope", 0.f)) /* ReLU with negative slope 0 */) + { + for (int i = 0; i < ld.outputBlobs.size(); i++) + { + LayerPin &pin = ld.inputBlobsId[i]; + scales[pin.lid][pin.oid] = scales[ld.id][i]; + zeropoints[pin.lid][pin.oid] = zeropoints[ld.id][i]; + } + } + // Layers with multiple inputs and single output. + else if ((ld.type == "Pooling" && toLowerCase(ld.params.get("pool", "max")) == "max") /* Max Pooling */ || + (ld.type == "Eltwise" && toLowerCase(ld.params.get("operation", "sum")) == "max") /* Elementwise max */ || + ld.type == "Concat") + { + for (int i = 0; i < ld.inputBlobsId.size(); i++) + { + LayerPin &pin = ld.inputBlobsId[i]; + scales[pin.lid][pin.oid] = scales[ld.id][0]; + zeropoints[pin.lid][pin.oid] = zeropoints[ld.id][0]; + } + } + } + + // Create a new Net and add quantized layers to it. + Net dstNet; + dstNet.impl->netWasQuantized = true; + dstNet.setInputsNames(impl->netInputLayer->outNames); + dstNet.setPreferableBackend(prefBackend); + dstNet.setPreferableTarget(prefTarget); + dstNet.enableFusion(originalFusion); + + for (Impl::MapIdToLayerData::iterator it = impl->layers.begin(); it != impl->layers.end(); it++) + { + LayerData ld = it->second; + if (ld.id == 0) + { + LayerData &quantInpLd = dstNet.impl->layers[0]; + quantInpLd.dtype = inputsDtype; + quantInpLd.params.set("scales", DictValue::arrayReal(scales[0].data(), scales[0].size())); + quantInpLd.params.set("zeropoints", DictValue::arrayInt(zeropoints[0].data(), zeropoints[0].size())); + continue; + } + + std::vector inpPins = ld.inputBlobsId; + // Fill input and output scales/zeropoints for the layer + std::vector > inp_out_sc(2); + std::vector > inp_out_zp(2); + for (int i = 0; i < inpPins.size(); i++) + { + LayerPin &pin = inpPins[i]; + inp_out_sc[0].push_back(scales[pin.lid][pin.oid]); + inp_out_zp[0].push_back(zeropoints[pin.lid][pin.oid]); + } + inp_out_sc[1] = scales[ld.id]; + inp_out_zp[1] = zeropoints[ld.id]; + + // Quantize layer + Ptr layer = ld.layerInstance; + if (layer->tryQuantize(inp_out_sc, inp_out_zp, ld.params)) + { + ld.type += "Int8"; + ld.dtype = CV_8S; + } + ld.params.set("scales", DictValue::arrayReal(inp_out_sc[1].data(), inp_out_sc[1].size())); + ld.params.set("zeropoints", DictValue::arrayInt(inp_out_zp[1].data(), inp_out_zp[1].size())); + + // Check and add quantize/dequantize node before layer + for (int i = 0; i < inpPins.size(); i++) + { + LayerPin &pin = inpPins[i]; + LayerData &inpLd = dstNet.impl->getLayerData(impl->getLayerName(pin.lid)); + pin.lid = inpLd.id; + if (inpLd.dtype != ld.dtype) + { + String layerName = (inpLd.dtype == CV_32F && ld.dtype == CV_8S) ? cv::format("quantize/%s/%d", inpLd.name.c_str(), pin.oid) + : cv::format("dequantize/%s/%d", inpLd.name.c_str(), pin.oid); + // Check if quantize/dequantize node for the input layer already exists + if (dstNet.impl->getLayerId(layerName) >= 0) + { + pin.lid = dstNet.impl->getLayerId(layerName); + pin.oid = 0; + } + else + { + LayerParams lp; + lp.set("scales", inp_out_sc[0][i]); + lp.set("zeropoints", inp_out_zp[0][i]); + lp.name = layerName; + lp.type = (inpLd.dtype == CV_32F && ld.dtype == CV_8S) ? "Quantize" : "Dequantize"; + int newLid = dstNet.addLayer(lp.name, lp.type, ld.dtype, lp); + dstNet.connect(pin.lid, pin.oid, newLid, 0); + pin.lid = newLid; pin.oid = 0; + } + } + } + + // Add quantized layer to Net and connect to its inputs. + int newLid = dstNet.addLayer(ld.name, ld.type, ld.dtype, ld.params); + for( int i = 0; i < inpPins.size(); i++ ) + dstNet.connect(inpPins[i].lid, inpPins[i].oid, newLid, i); + + // If the layer is a output layer, add quantize/dequantize node after it based on output's data type. + if (ld.requiredOutputs.size() == 0 && ld.dtype != outputsDtype) + { + LayerParams lp; + lp.set("scales", inp_out_sc[1][0]); + lp.set("zeropoints", inp_out_zp[1][0]); + lp.name = ((ld.dtype == CV_32F && outputsDtype == CV_8S) ? "quantize/" : "dequantize/") + ld.name; + lp.type = (ld.dtype == CV_32F && outputsDtype == CV_8S) ? "Quantize" : "Dequantize"; + dstNet.addLayerToPrev(lp.name, lp.type, outputsDtype, lp); + } + } + // Restore FP32 Net's backend, target and fusion + setPreferableBackend(prefBackend); + setPreferableTarget(prefTarget); + enableFusion(originalFusion); + return dstNet; +} + +void Net::getInputDetails(std::vector& scales, std::vector& zeropoints) const +{ + if (!impl->netWasQuantized) + CV_Error(Error::StsBadFunc, "Net isn't quantized"); + + LayerParams &lp = impl->layers[0].params; + DictValue sc = lp.get("scales"); + DictValue zp = lp.get("zeropoints"); + + for (int i = 0; i < sc.size(); i++) + { + scales.push_back(sc.get(i)); + zeropoints.push_back(zp.get(i)); + } +} + +void Net::getOutputDetails(std::vector& scales, std::vector& zeropoints) const +{ + if (!impl->netWasQuantized) + CV_Error(Error::StsBadFunc, "Net isn't quantized"); + + std::vector outLayerIds = getUnconnectedOutLayers(); + for (auto &lid : outLayerIds) + { + LayerParams &lp = impl->layers[lid].params; + DictValue sc = lp.get("scales"); + DictValue zp = lp.get("zeropoints"); + + for (int i = 0; i < sc.size(); i++) + { + scales.push_back(sc.get(i)); + zeropoints.push_back(zp.get(i)); + } + } +} + void Net::setPreferableBackend(int backendId) { CV_TRACE_FUNCTION(); CV_TRACE_ARG(backendId); + if (backendId == DNN_BACKEND_DEFAULT) + backendId = (Backend)PARAM_DNN_BACKEND_DEFAULT; + + if (impl->netWasQuantized && backendId != DNN_BACKEND_OPENCV) + { + CV_LOG_WARNING(NULL, "DNN: Only default backend supports quantized networks"); + backendId = DNN_BACKEND_OPENCV; + } + #ifdef HAVE_INF_ENGINE if (backendId == DNN_BACKEND_INFERENCE_ENGINE) backendId = getInferenceEngineBackendTypeParam(); @@ -4277,7 +4583,6 @@ void Net::setPreferableBackend(int backendId) if( impl->preferableBackend != backendId ) { impl->preferableBackend = backendId; - impl->netWasAllocated = false; impl->clear(); } } @@ -4287,6 +4592,13 @@ void Net::setPreferableTarget(int targetId) CV_TRACE_FUNCTION(); CV_TRACE_ARG(targetId); + if (impl->netWasQuantized && targetId != DNN_TARGET_CPU && + targetId != DNN_TARGET_OPENCL && targetId != DNN_TARGET_OPENCL_FP16) + { + CV_LOG_WARNING(NULL, "DNN: Only CPU and OpenCL/OpenCL FP16 target is supported by quantized networks"); + targetId = DNN_TARGET_CPU; + } + if( impl->preferableTarget != targetId ) { impl->preferableTarget = targetId; @@ -4306,7 +4618,6 @@ void Net::setPreferableTarget(int targetId) impl->preferableTarget = DNN_TARGET_OPENCL; #endif } - impl->netWasAllocated = false; impl->clear(); } } @@ -4935,9 +5246,10 @@ void Net::getMemoryConsumption(const int layerId, ShapesVec inLayerShapes, outLayerShapes; getLayerShapes(netInputShapes, layerId, inLayerShapes, outLayerShapes); + size_t elemSize = (impl->netWasQuantized) ? sizeof(char) : sizeof(float); for(int i = 0; i < outLayerShapes.size(); i++) { - blobs += total(outLayerShapes[i]) * sizeof(float); + blobs += total(outLayerShapes[i]) * elemSize; } } @@ -4986,7 +5298,7 @@ void Net::getMemoryConsumption(const std::vector& netInputShapes, std::vector > inLayerShapes, outLayerShapes; getLayersShapes(netInputShapes, layerIds, inLayerShapes, outLayerShapes); - + size_t elemSize = (impl->netWasQuantized) ? sizeof(char) : sizeof(float); for(int i = 0; i < layerIds.size(); i++) { int w = 0, b = 0; @@ -5001,7 +5313,7 @@ void Net::getMemoryConsumption(const std::vector& netInputShapes, for(int j = 0; j < outLayerShapes[i].size(); j++) { - b += total(outLayerShapes[i][j]) * sizeof(float); + b += total(outLayerShapes[i][j]) * elemSize; } weights.push_back(w); @@ -5021,7 +5333,6 @@ void Net::enableFusion(bool fusion) if( impl->fusion != fusion ) { impl->fusion = fusion; - impl->netWasAllocated = false; impl->clear(); } } @@ -5195,6 +5506,12 @@ void Layer::getScaleShift(Mat& scale, Mat& shift) const shift = Mat(); } +void Layer::getScaleZeropoint(float& scale, int& zeropoint) const +{ + scale = 1.f; + zeropoint = 0; +} + void Layer::unsetAttached() { setActivation(Ptr()); @@ -5321,6 +5638,12 @@ void Layer::run(const std::vector &inputs, std::vector &outputs, std:: this->forward(inputs, outputs, internals); } +bool Layer::tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) +{ + return false; +} + Layer::~Layer() {} bool Layer::getMemoryShapes(const std::vector &inputs, diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 1916aa0ec94f..9d8a3783a2e9 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -141,6 +141,44 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(LSTM, LSTMLayer); CV_DNN_REGISTER_LAYER_CLASS(GRU, GRULayer); CV_DNN_REGISTER_LAYER_CLASS(CumSum, CumSumLayer); + + CV_DNN_REGISTER_LAYER_CLASS(Quantize, QuantizeLayer); + CV_DNN_REGISTER_LAYER_CLASS(Dequantize, DequantizeLayer); + CV_DNN_REGISTER_LAYER_CLASS(ConvolutionInt8, ConvolutionLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(InnerProductInt8, InnerProductLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(PoolingInt8, PoolingLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(EltwiseInt8, EltwiseLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(BatchNormInt8, BatchNormLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(ScaleInt8, ScaleLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(ShiftInt8, ShiftLayerInt8); + + CV_DNN_REGISTER_LAYER_CLASS(ReLUInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(ReLU6Int8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(SigmoidInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(TanHInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(SwishInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(MishInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(ELUInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(BNLLInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(AbsValInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(SoftmaxInt8, SoftmaxLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(SoftMaxInt8, SoftmaxLayerInt8); + + CV_DNN_REGISTER_LAYER_CLASS(ConcatInt8, ConcatLayer); + CV_DNN_REGISTER_LAYER_CLASS(FlattenInt8, FlattenLayer); + CV_DNN_REGISTER_LAYER_CLASS(PaddingInt8, PaddingLayer); + CV_DNN_REGISTER_LAYER_CLASS(BlankInt8, BlankLayer); + CV_DNN_REGISTER_LAYER_CLASS(DropoutInt8, BlankLayer); + CV_DNN_REGISTER_LAYER_CLASS(IdentityInt8, BlankLayer); + CV_DNN_REGISTER_LAYER_CLASS(SilenceInt8, BlankLayer); + CV_DNN_REGISTER_LAYER_CLASS(ConstInt8, ConstLayer); + CV_DNN_REGISTER_LAYER_CLASS(ReshapeInt8, ReshapeLayer); + CV_DNN_REGISTER_LAYER_CLASS(SplitInt8, SplitLayer); + CV_DNN_REGISTER_LAYER_CLASS(SliceInt8, SliceLayer); + CV_DNN_REGISTER_LAYER_CLASS(CropInt8, CropLayer); + CV_DNN_REGISTER_LAYER_CLASS(PermuteInt8, PermuteLayer); + CV_DNN_REGISTER_LAYER_CLASS(ReorgInt8, ReorgLayer); + CV_DNN_REGISTER_LAYER_CLASS(ShuffleChannelInt8, ShuffleChannelLayer); } CV__DNN_INLINE_NS_END diff --git a/modules/dnn/src/int8layers/batch_norm_layer.cpp b/modules/dnn/src/int8layers/batch_norm_layer.cpp new file mode 100644 index 000000000000..c5b8c3d9e9b0 --- /dev/null +++ b/modules/dnn/src/int8layers/batch_norm_layer.cpp @@ -0,0 +1,178 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include + +namespace cv +{ +namespace dnn +{ + +class BatchNormLayerInt8Impl CV_FINAL : public BatchNormLayerInt8 +{ +public: + Mat origin_weights, origin_bias; + Mat weights_, bias_; + mutable int dims; + + BatchNormLayerInt8Impl(const LayerParams& params) + : dims(-1) + { + setParamsFrom(params); + useGlobalStats = params.get("use_global_stats", true); + input_sc = params.get("input_scale"); + input_zp = params.get("input_zeropoint"); + output_sc = params.get("scales"); + output_zp = params.get("zeropoints"); + + CV_Assert(blobs.size() == 2); + size_t n = blobs[0].total(); + CV_Assert(blobs[1].total() == n && + blobs[0].isContinuous() && blobs[1].isContinuous() && + blobs[0].type() == CV_32F && blobs[1].type() == CV_32F); + + origin_weights = blobs[0]; + origin_bias = blobs[1]; + } + + virtual void finalize(InputArrayOfArrays, OutputArrayOfArrays) CV_OVERRIDE + { + origin_weights.convertTo(weights_, CV_32F, input_sc/output_sc); + addWeighted(origin_bias, 1.0/output_sc, weights_, -input_zp, output_zp, bias_, CV_32F); + } + + void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE + { + scale = origin_weights; + shift = origin_bias; + } + + void getScaleZeropoint(float& scale, int& zeropoint) const CV_OVERRIDE + { + scale = output_sc; + zeropoint = output_zp; + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + Mat w_, b_; + top->getScaleShift(w_, b_); + if (w_.empty() && b_.empty()) + return false; + + const int numChannels = weights_.total(); + const int numFusedWeights = w_.total(); + const int numFusedBias = b_.total(); + + if ((numFusedWeights != numChannels && numFusedWeights != 1 && !w_.empty()) || + (numFusedBias != numChannels && numFusedBias != 1 && !b_.empty())) + return false; + + float new_sc; + int new_zp; + top->getScaleZeropoint(new_sc, new_zp); + + Mat w = numFusedWeights == 1 ? Mat(1, numChannels, CV_32F, Scalar(w_.at(0))) : + (w_.empty() ? Mat::ones(1, numChannels, CV_32F) : w_.reshape(1, 1)); + + Mat b = numFusedBias == 1 ? Mat(1, numChannels, CV_32F, Scalar(b_.at(0))) : + (b_.empty() ? Mat::zeros(1, numChannels, CV_32F) : b_.reshape(1, 1)); + + weights_ = Mat(); bias_ = Mat(); + multiply(origin_weights, w, weights_, input_sc/new_sc, CV_32F); + multiply(origin_bias, w, bias_); + add(bias_, b, bias_); + addWeighted(bias_, 1.0/new_sc, weights_, -input_zp, new_zp, bias_, CV_32F); + return true; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + dims = inputs[0].size(); + if (!useGlobalStats && inputs[0][0] != 1) + CV_Error(Error::StsNotImplemented, "Batch normalization in training mode with batch size > 1"); + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return true; + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + return activ_int8->blobs.empty(); + } + return false; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + CV_Assert(blobs.size() == 2); + CV_Assert(inputs.size() == 1); + + Mat &inpBlob = inputs[0]; + int planeSize = 1; + for (size_t i = 2; i < inpBlob.dims; i++) { + planeSize *= inpBlob.size[i]; + } + + for (size_t ii = 0; ii < outputs.size(); ii++) + { + Mat &outBlob = outputs[ii]; + + for(int num = 0; num < outBlob.size[0]; num++) + { + for (int n = 0; n < outBlob.size[1]; n++) + { + float w = weights_.at(n); + float b = bias_.at(n); + Mat inpBlobPlane(1, planeSize, CV_8S, inpBlob.ptr(num, n)); + Mat outBlobPlane(1, planeSize, CV_8S, outBlob.ptr(num, n)); + inpBlobPlane.convertTo(outBlobPlane, CV_8S, w, b); + } + } + } + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(outputs); // suppress unused variable warning + + int64 flops = 0; + for(int i = 0; i < inputs.size(); i++) + { + flops += 3*total(inputs[i]); + } + return flops; + } + +private: + bool useGlobalStats; +}; + +Ptr BatchNormLayerInt8::create(const LayerParams& params) +{ + return Ptr(new BatchNormLayerInt8Impl(params)); +} + +} // namespace dnn +} // namespace cv diff --git a/modules/dnn/src/int8layers/convolution_layer.cpp b/modules/dnn/src/int8layers/convolution_layer.cpp new file mode 100644 index 000000000000..05749885c05b --- /dev/null +++ b/modules/dnn/src/int8layers/convolution_layer.cpp @@ -0,0 +1,1136 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include + +#include "opencv2/core/hal/hal.hpp" +#include "opencv2/core/hal/intrin.hpp" +#include +#include + +namespace cv +{ +namespace dnn +{ + +#if CV_SIMD +static inline void v_expand_mul_add(const v_int8x16& a, const v_int8x16& b, + v_int32x4& out0, v_int32x4& out1, v_int32x4& out2, v_int32x4& out3) +{ + v_int16x8 a0, a1, b0, b1; + v_expand(a, a0, a1); + v_expand(b, b0, b1); + + v_int32x4 t0, t1; + v_mul_expand(a0, b0, t0, t1); + out0 += t0; out1 += t1; + + v_mul_expand(a1, b1, t0, t1); + out2 += t0; out3 += t1; +} +#endif + +class BaseConvolutionLayerInt8Impl : public ConvolutionLayerInt8 +{ +public: + BaseConvolutionLayerInt8Impl(const LayerParams ¶ms) + { + setParamsFrom(params); + getConvolutionKernelParams(params, kernel_size, pads_begin, pads_end, strides, dilations, padMode, adjust_pads); + + numOutput = params.get("num_output"); + int ngroups = params.get("group", 1); + CV_Assert(numOutput % ngroups == 0); + + input_zp = params.get("input_zeropoint"); + output_zp = params.get("zeropoints"); + output_sc = params.get("scales"); + + if (kernel_size.size() == 2) { + kernel = Size(kernel_size[1], kernel_size[0]); + stride = Size(strides[1], strides[0]); + for (int i = 0; i < pads_begin.size(); i++) { + if (pads_begin[i] != pads_end[i]) + CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding in convolution layer"); + } + pad = Size(pads_begin[1], pads_begin[0]); + dilation = Size(dilations[1], dilations[0]); + + adjustPad.height = adjust_pads[0]; + adjustPad.width = adjust_pads[1]; + } + + for (int i = 0; i < adjust_pads.size(); i++) { + CV_Assert(adjust_pads[i] < strides[i]); + } + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + // blobs[0] - Weights (INT8) + // blobs[1] - Biases (INT32) + // blobs[2] - Multipliers for convolution output stage (FP32) + CV_Assert(!inputs.empty() && blobs.size() == 3); + MatSize weightShape = blobs[0].size; + + CV_Assert(inputs[0].dims == outputs[0].dims); + if (weightShape.dims() == 3) + { + kernel_size.assign(1, kernel_size[0]); + strides.assign(1, strides[0]); + dilations.assign(1, dilations[0]); + pads_begin.assign(1, pads_begin[0]); + pads_end.assign(1, pads_end[0]); + } + CV_Assert(weightShape.dims() == kernel_size.size() + 2); + for (int i = 0; i < kernel_size.size(); i++) { + CV_Assert(weightShape[i + 2] == kernel_size[i]); + } + + const Mat &input = inputs[0]; + CV_Assert(((input.dims == 3 && kernel_size.size() == 1) || input.dims == 4 || input.dims == 5) && input.type() == CV_8S); + for (size_t i = 0; i < outputs.size(); i++) + { + CV_Assert(inputs[i].type() == input.type()); + CV_Assert(((input.dims == 3 && kernel_size.size() == 1) || inputs[i].dims == 4 || inputs[i].dims == 5) && inputs[i].size[1] == input.size[1]); + for (int j = 0; j < inputs[i].dims; j++) { + CV_Assert(inputs[i].size[j] == input.size[j]); + } + } + + std::vector inpShape; + std::vector outShape; + for (int i = 2; i < inputs[0].dims; i++) { + inpShape.push_back(inputs[0].size[i]); + outShape.push_back(outputs[0].size[i]); + } + getConvPoolPaddings(inpShape, kernel_size, strides, padMode, pads_begin, pads_end); + if (pads_begin.size() == 2) { + for (int i = 0; i < pads_begin.size(); i++) { + if (pads_begin[i] != pads_end[i]) + CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding in convolution layer"); + } + pad = Size(pads_begin[1], pads_begin[0]); + } + } + + virtual MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const = 0; + bool is1x1() const + { + return (kernel.height == 1 && kernel.width == 1) && + (stride.height == 1 && stride.width == 1) && + (dilation.height == 1 && dilation.width == 1); + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + Mat w, b; + top->getScaleShift(w, b); + if (w.empty() && b.empty()) + return false; + + CV_Assert((w.empty() || w.type() == CV_32F) && + (b.empty() || b.type() == CV_32F)); + + float new_sc; + int new_zp; + top->getScaleZeropoint(new_sc, new_zp); + fuseWeights(w, b, new_sc); + output_sc = new_sc; + output_zp = new_zp; + return true; + } + + virtual void fuseWeights(const Mat& w_, const Mat& b_, const float& new_sc) = 0; +}; + +//TODO: simultaneously convolution and bias addition for cache optimization +class ConvolutionLayerInt8Impl CV_FINAL : public BaseConvolutionLayerInt8Impl +{ +public: + enum { VEC_ALIGN = 32, DFT_TYPE = CV_8S }; + Mat weightsMat; + std::vector biasvec; + Mat outputMultiplier; + Mat activationLUT; + Ptr activ; + + ConvolutionLayerInt8Impl(const LayerParams ¶ms) : BaseConvolutionLayerInt8Impl(params){} + + MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const CV_OVERRIDE + { + CV_Assert(!blobs.empty()); + int dims = inpShape.size(); + int inpD = dims == 5 ? inpShape[2] : 1; + int inpH = inpShape[dims - 2]; + int inpW = inpShape.back(); + int inpGroupCn = blobs[0].size[1]; + int ksize = inpGroupCn * std::accumulate(kernel_size.begin(), kernel_size.end(), + 1, std::multiplies()); + return shape(inpD * inpH * inpW, ksize); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + size_t ksize = kernel_size.size(); + // Only default backend and Conv1D/Conv2D/Conv3D are supported + return backendId == DNN_BACKEND_OPENCV && ksize >= 1 && ksize <= 3; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(!blobs.empty()); + const int* weightShape = blobs[0].size.p; + CV_Assert(blobs[1].total() == (size_t)weightShape[0]); + + internals.clear(); + + CV_Assert(inputs.size() != 0); + std::vector inpShape(inputs[0].begin() + 2, inputs[0].end()); + + int outCn = weightShape[0]; + std::vector outShape; + outShape.push_back(inputs[0][0]); + outShape.push_back(outCn); + + int inpCn = inputs[0][1]; + if (padMode.empty()) + { + for (int i = 0; i < inpShape.size(); i++) + outShape.push_back((inpShape[i] + pads_begin[i] + pads_end[i] - dilations[i] * (kernel_size[i] - 1) - 1) / strides[i] + 1); + } + else + { + getConvPoolOutParams(inpShape, kernel_size, strides, padMode, dilations, outShape); + } + + int ngroups = inpCn / weightShape[1]; + if (ngroups == 0 || ngroups * weightShape[1] != inpCn) + CV_Error(Error::StsError, format("Number of input channels should " + "be multiple of %d but got %d", weightShape[1], inpCn)); + CV_Assert(ngroups > 0 && inpCn % ngroups == 0 && outCn % ngroups == 0); + + outputs.resize(1, outShape); + + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + BaseConvolutionLayerInt8Impl::finalize(inputs_arr, outputs_arr); + + std::vector inputs; + inputs_arr.getMatVector(inputs); + // prepare weightsMat where each row is aligned and has enough zero padding on the right to + // use vectorized (i.e. with intrinsics) loops without tail processing + Mat wm = blobs[0].reshape(1, numOutput); + if( wm.step1() % VEC_ALIGN != 0 ) + { + int newcols = (int)alignSize(wm.step1(), VEC_ALIGN); + Mat wm_buffer = Mat(numOutput, newcols, wm.type()); + Mat wm_padding = wm_buffer.colRange(wm.cols, newcols); + wm_padding.setTo(Scalar::all(0)); + Mat wm_aligned = wm_buffer.colRange(0, wm.cols); + wm.copyTo(wm_aligned); + wm = wm_aligned; + } + weightsMat = wm; + + Mat biasMat = blobs[1]; + biasvec.resize(numOutput+2); + for(int i = 0; i < numOutput; i++ ) + biasvec[i] = biasMat.at(i); + + outputMultiplier = blobs[2]; + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + activ = activ_int8; + if (!activ_int8->blobs.empty()) + activ_int8->blobs[0].convertTo(activationLUT, CV_32S); + return true; + } + return false; + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + return BaseConvolutionLayerInt8Impl::tryFuse(top); + } + + void fuseWeights(const Mat& w_, const Mat& b_, const float& new_sc) CV_OVERRIDE + { + const int outCn = weightsMat.size[0]; + Mat w = w_.total() == 1 ? Mat(1, outCn, CV_32F, Scalar(w_.at(0))) : w_; + Mat b = b_.total() == 1 ? Mat(1, outCn, CV_32F, Scalar(b_.at(0))) : b_; + CV_Assert_N(!weightsMat.empty(), biasvec.size() == outCn + 2, + w.empty() || outCn == w.total(), b.empty() || outCn == b.total()); + + for (int i = 0; i < outCn; ++i) + { + float off = outputMultiplier.at(i) * output_sc; + if (!w.empty()) + off *= w.at(i); + + if (!b.empty()) + biasvec[i] += (int)std::round(b.at(i)/off); + + outputMultiplier.at(i) = off/new_sc; + } + biasvec[outCn] = biasvec[outCn+1] = biasvec[outCn-1]; + } + + class ParallelConv : public cv::ParallelLoopBody + { + public: + enum { BLK_SIZE = 32, BLK_SIZE_CN = 64 }; + + const Mat* input_; + const Mat* weights_; + Mat* output_; + int outShape[4]; // used only for conv2d + std::vector kernel_size, pads_begin, pads_end, strides, dilations; + int ngroups_, nstripes_; + std::vector ofstab_; + const std::vector* biasvec_; + const Mat* activLUT_; + const ActivationLayerInt8* activ_; + bool is1x1_; + bool useAVX2; + bool useAVX512; + int blk_size_cn; + int inpZp, outZp; + const float* multiplier; + + ParallelConv() + : input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0), + biasvec_(0), activLUT_(0), activ_(0), is1x1_(false), useAVX2(false), useAVX512(false) + , blk_size_cn(0), inpZp(0), outZp(0), multiplier(0) + {} + + static void run( const Mat& input, Mat& output, const Mat& weights, const Mat& multipliers, + const std::vector& biasvec, const Mat& activLUT, + const std::vector& kernel_size, const std::vector& strides, + const std::vector& pads_begin, const std::vector& pads_end, + const std::vector& dilations, + const ActivationLayerInt8* activ, int ngroups, int nstripes, int inp_Zp, int out_Zp) + { + size_t karea = std::accumulate(kernel_size.begin(), kernel_size.end(), + 1, std::multiplies()); + bool isConv1D = input.dims == 3; + bool isConv2D = input.dims == 4; + bool isConv3D = input.dims == 5; + CV_CheckEQ(static_cast(kernel_size.size()), input.dims - 2, ""); + CV_Assert_N(input.dims == output.dims, + input.size[0] == output.size[0], + weights.rows == output.size[1], + weights.cols == (input.size[1]/ngroups)*karea, + input.type() == CV_8SC1, + output.type() == CV_32SC1, + input.type() == weights.type(), + input.isContinuous(), + output.isContinuous(), + biasvec.size() == (size_t)output.size[1]+2); + CV_Check(weights.step1(), weights.step1() % VEC_ALIGN == 0, ""); + ParallelConv p; + + p.input_ = &input; + p.weights_ = &weights; + p.output_ = &output; + int max_ind = isConv1D? 3: 4; + for( int i = 0; i < max_ind; i++ ) p.outShape[i] = output.size[i]; + p.outShape[1] /= ngroups; + + p.kernel_size = kernel_size; p.strides = strides; p.dilations = dilations; + p.pads_begin = pads_begin; p.pads_end = pads_end; + + p.ngroups_ = ngroups; + p.nstripes_ = nstripes; + + int inpCnAll = input.size[1]; + int depth = (input.dims == 5) ? input.size[2] : 1; + int width = input.size[input.dims - 1]; + int height = isConv1D? 1 : input.size[input.dims - 2]; + int inpCn = inpCnAll / ngroups; + + p.is1x1_ = (isConv2D && kernel_size[0] == 1 && kernel_size[1] == 1 && + pads_begin[0] == 0 && pads_begin[1] == 0) || + (isConv1D && pads_begin[0] == 0 && kernel_size[0] == 1); + + p.useAVX2 = checkHardwareSupport(CPU_AVX2) && isConv2D; + p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX && isConv2D; + + int kernel_d = isConv3D? kernel_size[0] : 1; + int kernel_h = isConv1D? 1 : kernel_size[kernel_size.size() - 2]; + int kernel_w = kernel_size.back(); + + int blk_size_cn0 = cvCeil(1600./(kernel_w*kernel_h)); + int ncn = 32; + while (ncn*2 < blk_size_cn0 && ncn < inpCn) + ncn *= 2; + ncn = std::min(ncn, inpCn); + p.blk_size_cn = ncn; + + int dil_d = isConv3D? dilations[0] : 1; + int dil_h = isConv1D? 1 : dilations[dilations.size() - 2]; + int dil_w = dilations.back(); + + p.inpZp = inp_Zp; + p.outZp = out_Zp; + p.multiplier = multipliers.ptr(0); + + p.ofstab_.resize(karea * ncn); + int* ofstab = &p.ofstab_[0]; + + if (isConv1D) + { + for( int k = 0; k < ncn; k++ ) + for( int k_c = 0; k_c < kernel_w; k_c++ ) + ofstab[k*kernel_w + k_c] = k*width + k_c*dil_w; + } + else if (isConv2D) + { + for( int k = 0; k < ncn; k++ ) + for( int k_r = 0; k_r < kernel_h; k_r++ ) + for( int k_c = 0; k_c < kernel_w; k_c++ ) + ofstab[(k*kernel_h + k_r)*kernel_w + k_c] = + (k*height + k_r*dil_h)*width + k_c*dil_w; + } + else + { + for( int k = 0; k < ncn; k++ ) + for (int k_d = 0; k_d < kernel_d; k_d++) + for( int k_r = 0; k_r < kernel_h; k_r++ ) + for( int k_c = 0; k_c < kernel_w; k_c++ ) + ofstab[(k*kernel_d*kernel_h + k_d*kernel_h + k_r)*kernel_w + k_c] = + (k*depth*height + k_d*dil_d*height + k_r*dil_h)*width + k_c*dil_w; + } + + p.biasvec_ = &biasvec; + p.activLUT_ = &activLUT; + p.activ_ = !activLUT.empty() ? activ : 0; + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + virtual void operator ()(const Range &r0) const CV_OVERRIDE + { + const int valign = ConvolutionLayerInt8Impl::VEC_ALIGN; + int ngroups = ngroups_, batchSize = input_->size[0]*ngroups; + bool isConv1D = input_->dims == 3; + bool isConv2D = input_->dims == 4; + bool isConv3D = input_->dims == 5; + + int outW = output_->size[output_->dims - 1]; + int outH = isConv1D? 1 : output_->size[output_->dims - 2]; + int outCn = output_->size[1]/ngroups; + + int depth = isConv3D? input_->size[2] : 1; + int height = isConv1D? 1 : input_->size[input_->dims - 2]; + int width = input_->size[input_->dims - 1]; + int inpCn = input_->size[1]/ngroups; + + const int nstripes = nstripes_; + + int kernel_d = isConv3D? kernel_size[0] : 1; + int kernel_h = isConv1D? 1 : kernel_size[kernel_size.size() - 2]; + int kernel_w = kernel_size.back(); + int karea = kernel_w*kernel_h*kernel_d; + + int pad_d = isConv3D? pads_begin[0] : 0; + int pad_t = isConv1D? 0 : pads_begin[pads_begin.size() - 2]; + int pad_l = pads_begin.back(); + + int stride_d = isConv3D? strides[0] : 0; + int stride_h = isConv1D? 0 : strides[strides.size() - 2]; + int stride_w = strides.back(); + + int dilation_d = isConv3D? dilations[0] : 1; + int dilation_h = isConv1D? 1 : dilations[dilations.size() - 2]; + int dilation_w = dilations.back(); + + int i, j, k, d; + int inpPlaneSize = (int)input_->total(2); + int outPlaneSize = (int)output_->total(2); + bool is1x1 = is1x1_; + + int stripesPerSample; + int stripeSize; + Range r = r0; + bool depthWiseConvolution = !is1x1 && isConv2D && ngroups > 1 && inpCn == 1 && + outCn == 1 && kernel_d == 1 && dilation_d == 1 && stride_d == 0 && pad_d == 0 && + width >= 16 + dilation_w*(kernel_w - 1); + // for now only 3x3 depth-wise convolutions are supported + depthWiseConvolution = depthWiseConvolution && kernel_w == 3 && kernel_h == 3 && + // computing at most 1 pixel from each side can involve padding + max(stride_w, dilation_w) >= pad_l && max(stride_h, dilation_h) >= pad_t && + pad_l <= 1 && pad_t <= 1; + + if( !depthWiseConvolution && nstripes >= batchSize*2 ) + { + stripesPerSample = nstripes/batchSize; + stripeSize = (int)alignSize((outPlaneSize + stripesPerSample - 1)/stripesPerSample, 8); + stripeSize = std::min(stripeSize, outPlaneSize); + } + else + { + stripesPerSample = 1; + int samplesPerStripe = std::max((batchSize + nstripes - 1)/nstripes, 1); + r.start *= samplesPerStripe; + r.end *= samplesPerStripe; + stripeSize = outPlaneSize; + } + + const int8_t* data_inp0_ = input_->ptr(); + const int* ofstab = &ofstab_[0]; + const int8_t* wptr_orig_ = weights_->ptr(); + size_t wstep = weights_->step1(); + const int* biasptr_ = &biasvec_->at(0); + const int* lutptr_ = !activLUT_->empty() ? activLUT_->ptr() : 0; + int* data_out0_ = output_->ptr(); + AutoBuffer rowbuf0_; + int8_t* rowbuf0 = 0; + bool use_rowbuf = !depthWiseConvolution; + int blk_size = depthWiseConvolution ? outPlaneSize : min((int)BLK_SIZE, stripeSize); + + // im2row buffer is not used for depth-wise convolution + if(use_rowbuf) + { + size_t rowbufsz = alignSize(karea*blk_size_cn, valign)*min((int)BLK_SIZE, blk_size); + //printf("karea=%d, blk_size_cn=%d, rowbufsz=%d, stripeSize=%d\n", karea, blk_size_cn, (int)rowbufsz, stripeSize); + rowbuf0_.allocate(rowbufsz + valign); + rowbuf0 = alignPtr(rowbuf0_.data(), (int)(valign*sizeof(int8_t))); + // we clear the buffer once; ultimately, it lets us to avoid + // tail processing after running the unrolled/vectorized loop. + // the main idea is to make sure that the tail (a.k.a. padding) of each row + // (i.e. the elements with indices between vsz=karea*ncn and vsz_a) + // does not contain NaNs or Infs. Because the padding in the weights + // matrix is explicitly initialized with 0's, we handle all other + // cases nicely, i.e. we can skip expliciting re-initialization + // of the padding - we just retain elements from the previous iteration + // of the loop over channels (cn0). + memset(rowbuf0, (int8_t)inpZp, rowbufsz*sizeof(rowbuf0[0]) ); + } + + for( int stripe = r.start; stripe < r.end; stripe++ ) + { + int subsampleIdx = stripe/stripesPerSample; + if( subsampleIdx >= batchSize ) + break; + int stripeStart = (int)((stripe - subsampleIdx*stripesPerSample)*stripeSize); + int stripeEnd = (int)std::min(stripeStart + stripeSize, outPlaneSize); + const int8_t* data_inp0 = data_inp0_ + subsampleIdx*inpPlaneSize*inpCn; + int* data_out0 = data_out0_ + subsampleIdx*outPlaneSize*outCn; + int startOutCn = (subsampleIdx % ngroups)*outCn; + const int8_t* wptr_orig = wptr_orig_ + wstep*startOutCn; + const int* biasptr = biasptr_ + startOutCn; + const float* multptr = multiplier + startOutCn; + + for( int cn0 = 0; cn0 < inpCn; cn0 += blk_size_cn ) + { + int cn1 = std::min(cn0 + blk_size_cn, inpCn); + int ncn = cn1 - cn0, vsz = karea*ncn; + int vsz_a = (int)alignSize(vsz, valign); + const int8_t* wptr = wptr_orig + cn0*karea; + + for( int ofs0 = stripeStart; ofs0 < stripeEnd; ofs0 += blk_size ) + { + int ofs, ofs1 = std::min(ofs0 + blk_size, stripeEnd); + int bsz = ofs1 - ofs0; + + int out_d = ofs0 / (outH * outW); + int out_i = (ofs0 - out_d * outH * outW) / outW; + int out_j = ofs0 % outW; + + if (depthWiseConvolution) + { + CV_Assert(out_i == 0 && out_j == 0); + int in_d = out_d * stride_d - pad_d; + const int8_t* inptr_ = data_inp0 + (cn0*depth*height + in_d*height)*width; + int* outptr_ = data_out0 + ofs0; + + #if CV_TRY_AVX2 + if(useAVX2) + opt_AVX2::fastDepthwiseConv(wptr, kernel_h, kernel_w, + stride_h, stride_w, dilation_h, dilation_w, pad_t, pad_l, + biasptr, multptr, inptr_, height, width, outptr_, out_d, outH, outW, inpZp, outZp); + else + #endif + { + const int8_t w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2], + w10 = wptr[3], w11 = wptr[4], w12 = wptr[5], + w20_ = wptr[6], w21_ = wptr[7], w22_ = wptr[8]; + int outW1 = min(outW, (width - dilation_w*(kernel_w - 1) + pad_l)/stride_w); + int bias = biasptr[out_d], biasCopy; + float mult = multptr[out_d]; + + for (int out_i = 0; out_i < outH; out_i++) + { + int in_i = out_i * stride_h - pad_t, out_j = 0; + const int8_t* imgptr0 = inptr_ + in_i*width; + const int8_t* imgptr1 = imgptr0 + dilation_h*width; + const int8_t* imgptr2 = imgptr0 + (dilation_h*2)*width; + int8_t w00 = w00_, w01 = w01_, w02 = w02_; + int8_t w20 = w20_, w21 = w21_, w22 = w22_; + int out, out1; + // Bias has a fused offset component. bias = bias_quantized - input_zeropoint*sum_of_weights. + // In some cases below, certain weights are not used for convolution or set to zero. + // So we create a copy of bias at the start and remove the weight's components as necessary. + biasCopy = bias; + + if (in_i < 0) + { + biasCopy += inpZp * (w00 + w01 + w02); + w00 = w01 = w02 = 0; + imgptr0 = imgptr1; + } + else if (in_i + dilation_h*(kernel_h-1) >= height) + { + biasCopy += inpZp * (w20 + w21 + w22); + w20 = w21 = w22 = 0; + imgptr2 = imgptr1; + } + int* outptr = outptr_ + out_i*outW; + if (pad_l > 0) + { + out = (int)imgptr0[0]*w01 + (int)imgptr0[dilation_w]*w02 + + (int)imgptr1[0]*w11 + (int)imgptr1[dilation_w]*w12 + + (int)imgptr2[0]*w21 + (int)imgptr2[dilation_w]*w22 + + biasCopy + inpZp*(w00 + w10 + w20); + out1 = outZp + (int)std::round(out*mult); + outptr[0] = std::min(std::max(out1, -128), 127); + out_j = 1; + } + #if CV_SIMD + if( stride_w == 1 ) + { + const int out_delta = 16; + v_int8x16 vw00 = v_setall_s8(w00), vw01 = v_setall_s8(w01), vw02 = v_setall_s8(w02), + vw10 = v_setall_s8(w10), vw11 = v_setall_s8(w11), vw12 = v_setall_s8(w12), + vw20 = v_setall_s8(w20), vw21 = v_setall_s8(w21), vw22 = v_setall_s8(w22); + v_int32x4 vout0, vout1, vout2, vout3, vbias = v_setall_s32(biasCopy), voutzp = v_setall_s32(outZp), + outmin = v_setall_s32(-128), outmax = v_setall_s32(127); + v_float32x4 vmult = v_setall_f32(mult); + for( ; out_j < outW1; out_j += out_delta ) + { + if (out_j + out_delta > outW1) + { + if (out_j <= pad_l) + break; + out_j = outW1 - out_delta; + } + int in_j = out_j * stride_w - pad_l; + v_int8x16 v00 = v_load(imgptr0 + in_j), + v01 = v_load(imgptr0 + in_j + dilation_w), + v02 = v_load(imgptr0 + in_j + dilation_w*2), + v10 = v_load(imgptr1 + in_j), + v11 = v_load(imgptr1 + in_j + dilation_w), + v12 = v_load(imgptr1 + in_j + dilation_w*2), + v20 = v_load(imgptr2 + in_j), + v21 = v_load(imgptr2 + in_j + dilation_w), + v22 = v_load(imgptr2 + in_j + dilation_w*2); + + vout0 = vout1 = vout2 = vout3 = vbias; + v_expand_mul_add(v00, vw00, vout0, vout1, vout2, vout3); + v_expand_mul_add(v01, vw01, vout0, vout1, vout2, vout3); + v_expand_mul_add(v02, vw02, vout0, vout1, vout2, vout3); + v_expand_mul_add(v10, vw10, vout0, vout1, vout2, vout3); + v_expand_mul_add(v11, vw11, vout0, vout1, vout2, vout3); + v_expand_mul_add(v12, vw12, vout0, vout1, vout2, vout3); + v_expand_mul_add(v20, vw20, vout0, vout1, vout2, vout3); + v_expand_mul_add(v21, vw21, vout0, vout1, vout2, vout3); + v_expand_mul_add(v22, vw22, vout0, vout1, vout2, vout3); + + vout0 = voutzp + v_round(v_cvt_f32(vout0)*vmult); + vout1 = voutzp + v_round(v_cvt_f32(vout1)*vmult); + vout2 = voutzp + v_round(v_cvt_f32(vout2)*vmult); + vout3 = voutzp + v_round(v_cvt_f32(vout3)*vmult); + + vout0 = v_min(v_max(vout0, outmin), outmax); + vout1 = v_min(v_max(vout1, outmin), outmax); + vout2 = v_min(v_max(vout2, outmin), outmax); + vout3 = v_min(v_max(vout3, outmin), outmax); + + v_store(outptr + out_j, vout0); + v_store(outptr + out_j + 4, vout1); + v_store(outptr + out_j + 8, vout2); + v_store(outptr + out_j + 12, vout3); + } + } + #endif + for (; out_j < outW1; out_j++) + { + int in_j = out_j * stride_w - pad_l; + out = (int)imgptr0[in_j]*w00 + (int)imgptr0[in_j + dilation_w]*w01 + (int)imgptr0[in_j + dilation_w*2]*w02 + + (int)imgptr1[in_j]*w10 + (int)imgptr1[in_j + dilation_w]*w11 + (int)imgptr1[in_j + dilation_w*2]*w12 + + (int)imgptr2[in_j]*w20 + (int)imgptr2[in_j + dilation_w]*w21 + (int)imgptr2[in_j + dilation_w*2]*w22 + biasCopy; + out1 = outZp + (int)std::round(out*mult); + outptr[out_j] = std::min(std::max(out1, -128), 127); + } + + for (; out_j < outW; out_j++ ) + { + int in_j0 = out_j * stride_w - pad_l, in_j1 = in_j0 + dilation_w, in_j2 = in_j0 + dilation_w*2; + int s0 = 1, s1 = 1, s2 = 1; + if (in_j0 >= width) + { + in_j0 = 0; + s0 = 0; + biasCopy += inpZp*(w00 + w10 + w20); + } + if (in_j1 >= width) + { + in_j1 = 0; + s1 = 0; + biasCopy += inpZp*(w01 + w11 + w21); + } + if (in_j2 >= width) + { + in_j2 = 0; + s2 = 0; + biasCopy += inpZp*(w02 + w12 + w22); + } + out = (int)imgptr0[in_j0]*w00*s0 + (int)imgptr0[in_j1]*w01*s1 + (int)imgptr0[in_j2]*w02*s2 + + (int)imgptr1[in_j0]*w10*s0 + (int)imgptr1[in_j1]*w11*s1 + (int)imgptr1[in_j2]*w12*s2 + + (int)imgptr2[in_j0]*w20*s0 + (int)imgptr2[in_j1]*w21*s1 + (int)imgptr2[in_j2]*w22*s2 + biasCopy; + out1 = outZp + (int)std::round(out*mult); + outptr[out_j] = std::min(std::max(out1, -128), 127); + } + } + } + continue; + } + // do im2row for a part of input tensor + int8_t* rowbuf = rowbuf0; + + if (isConv1D) + { + for( ofs = ofs0; ofs < ofs1; out_j = 0, ++out_i ) + { + int delta = std::min(ofs1 - ofs, outW - out_j); + int out_j1 = out_j + delta; + + int in_j = out_j * stride_w - pad_l; + const int8_t* imgptr = data_inp0 + cn0*width + in_j; + ofs += delta; + + // do im2row for a part of input tensor + if( is1x1 ) + { + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w ) + { + for( k = 0; k < vsz; k++ ) + rowbuf[k] = imgptr[k*inpPlaneSize]; + } + } + else + { + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w ) + { + // this condition should be true for most of the tensor elements, i.e. + // most of the time the kernel aperture is inside the tensor X-Y plane. + if( out_j + 2 <= out_j1 && 0 <= in_j && in_j + stride_w*2 <= width - (kernel_w-1)*dilation_w ) + { + for( k = 0; k < vsz; k++ ) + { + int k1 = ofstab[k]; + int8_t v0 = imgptr[k1]; + int8_t v1 = imgptr[k1 + stride_w]; + rowbuf[k] = v0; + rowbuf[k+vsz_a] = v1; + } + out_j++; + rowbuf += vsz_a; + imgptr += stride_w; + in_j += stride_w; + } + else + { + int i0 = std::max(0, (-in_j + dilation_w-1)/dilation_w); + int i1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w); + + // here some non-continuous sub-row of the row will not be + // filled from the tensor; we need to make sure that the uncovered + // elements are explicitly set to 0's. the easiest way is to + // set all the elements to 0's before the loop. + memset(rowbuf, (int8_t)inpZp, vsz*sizeof(rowbuf[0])); + for( k = 0; k < ncn; k++ ) + { + for( i = i0; i < i1; i++ ) + { + int imgofs = k*width + i*dilation_w; + rowbuf[k*kernel_w + i] = imgptr[imgofs]; + } + } + } + } + } + } + } + else if (isConv2D) + { + if( is1x1 && stride_w == 1 && stride_h == 1 ) + { + const int8_t* imgptr = data_inp0 + (cn0*height + out_i)*width + out_j; + for( int j = 0; j < bsz; j++, rowbuf += vsz_a ) + { + if( j + 4 <= bsz ) + { + k = 0; + for( ; k < vsz; k++ ) + { + const int8_t* inp = imgptr + j + k*inpPlaneSize; + int8_t v0 = inp[0], v1 = inp[1], v2 = inp[2], v3 = inp[3]; + rowbuf[k] = v0; + rowbuf[k + vsz_a] = v1; + rowbuf[k + vsz_a*2] = v2; + rowbuf[k + vsz_a*3] = v3; + } + j += 3; + rowbuf += vsz_a*3; + } + else + { + for( k = 0; k < vsz; k++ ) + { + rowbuf[k] = imgptr[j + k*inpPlaneSize]; + } + } + } + } + else + for( ofs = ofs0; ofs < ofs1; out_j = 0, ++out_i ) + { + int delta = std::min(ofs1 - ofs, outW - out_j); + int out_j1 = out_j + delta; + + int in_i = out_i * stride_h - pad_t; + int in_j = out_j * stride_w - pad_l; + const int8_t* imgptr = data_inp0 + (cn0*height + in_i)*width + in_j; + ofs += delta; + + // do im2row for a part of input tensor + if( is1x1 ) + { + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w ) + { + for( k = 0; k < vsz; k++ ) + rowbuf[k] = imgptr[k*inpPlaneSize]; + } + } + else + { + bool ok_i = 0 <= in_i && in_i < height - (kernel_h-1)*dilation_h; + int i0 = std::max(0, (-in_i + dilation_h-1)/dilation_h); + int i1 = std::min(kernel_h, (height - in_i + dilation_h-1)/dilation_h); + + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w ) + { + // this condition should be true for most of the tensor elements, i.e. + // most of the time the kernel aperture is inside the tensor X-Y plane. + if( ok_i && out_j + 2 <= out_j1 && 0 <= in_j && in_j + stride_w*2 <= width - (kernel_w-1)*dilation_w ) + { + for( k = 0; k < vsz; k++ ) + { + int k1 = ofstab[k]; + int8_t v0 = imgptr[k1]; + int8_t v1 = imgptr[k1 + stride_w]; + rowbuf[k] = v0; + rowbuf[k+vsz_a] = v1; + } + out_j++; + rowbuf += vsz_a; + imgptr += stride_w; + in_j += stride_w; + } + else + { + int j0 = std::max(0, (-in_j + dilation_w-1)/dilation_w); + int j1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w); + + // here some non-continuous sub-row of the row will not be + // filled from the tensor; we need to make sure that the uncovered + // elements are explicitly set to 0's. the easiest way is to + // set all the elements to 0's before the loop. + memset(rowbuf, (int8_t)inpZp, vsz*sizeof(rowbuf[0])); + for( k = 0; k < ncn; k++ ) + { + for( i = i0; i < i1; i++ ) + { + for( j = j0; j < j1; j++ ) + { + int imgofs = k*(width*height) + i*(dilation_h*width) + j*dilation_w; + rowbuf[(k*kernel_h + i)*kernel_w + j] = imgptr[imgofs]; + } + } + } + } + } + } + } + } + else + { + for( ofs = ofs0; ofs < ofs1; out_d += (out_i + 1) / outH, out_i = (out_i + 1) % outH, out_j = 0 ) + { + int delta = std::min(ofs1 - ofs, outW - out_j); + int out_j1 = out_j + delta; + + int in_d = out_d * stride_d - pad_d; + int in_i = out_i * stride_h - pad_t; + int in_j = out_j * stride_w - pad_l; + const int8_t* imgptr = data_inp0 + (cn0*depth*height + in_d*height + in_i)*width + in_j; + ofs += delta; + + int d0 = std::max(0, (-in_d + dilation_d - 1) / dilation_d); + int d1 = std::min(kernel_d, (depth - in_d + dilation_d - 1) / dilation_d); + + int i0 = std::max(0, (-in_i + dilation_h-1)/dilation_h); + int i1 = std::min(kernel_h, (height - in_i + dilation_h-1)/dilation_h); + + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w ) + { + int j0 = std::max(0, (-in_j + dilation_w-1)/dilation_w); + int j1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w); + + // here some non-continuous sub-row of the row will not be + // filled from the tensor; we need to make sure that the uncovered + // elements are explicitly set to 0's. the easiest way is to + // set all the elements to 0's before the loop. + memset(rowbuf, (int8_t)inpZp, vsz*sizeof(rowbuf[0])); + for( k = 0; k < ncn; k++ ) + { + for ( d = d0; d < d1; d++) + { + for( i = i0; i < i1; i++ ) + { + for( j = j0; j < j1; j++ ) + { + int imgofs = k*(depth*width*height) + d*dilation_d*width*height + i*(dilation_h*width) + j*dilation_w; + rowbuf[(k*kernel_d*kernel_h + d*kernel_h + i)*kernel_w + j] = imgptr[imgofs]; + } + } + } + } + } + } + } + // now compute dot product of the weights + // and im2row-transformed part of the tensor + #if CV_TRY_AVX512_SKX + if(useAVX512) + opt_AVX2::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0, + outShape, bsz, vsz, vsz_a, outZp, multptr, cn0 == 0, cn1 == inpCn); + else + #endif + #if CV_TRY_AVX2 + if(useAVX2) + opt_AVX2::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0, + outShape, bsz, vsz, vsz_a, outZp, multptr, cn0 == 0, cn1 == inpCn); + else + #endif + for( int i = 0; i < outCn; i += 2 ) + { + const int8_t* wptr0 = wptr + i*wstep; + const int8_t* wptr1 = wptr0 + wstep; + int* outptr0 = data_out0 + ofs0 + i*outPlaneSize; + int* outptr1 = outptr0 + outPlaneSize; + int bias0 = biasptr[i], bias1 = biasptr[i+1]; + float mult0 = multptr[i], mult1 = multptr[i+1]; + + if( i+1 >= outCn ) + { + wptr1 = wptr0; + outptr1 = outptr0; + bias1 = bias0; + mult1 = mult0; + } + int j = 0; + #if CV_SIMD128 + v_int32x4 voutzp = v_setall_s32(outZp), outmin = v_setall_s32(-128), outmax = v_setall_s32(127); + v_float32x4 vmult0 = v_setall_f32(mult0), vmult1 = v_setall_f32(mult1); + for( ; j <= bsz - 4; j += 4 ) + { + const int8_t* rptr = rowbuf0 + j*vsz_a; + v_int32x4 s0, s1; + + if( cn0 == 0 ) + { + s0 = v_setall_s32(bias0); + s1 = v_setall_s32(bias1); + } + else + { + s0 = v_load(outptr0 + j); + s1 = v_load(outptr1 + j); + } + + v_int32x4 vs00 = v_setzero_s32(), vs01 = v_setzero_s32(), + vs02 = v_setzero_s32(), vs03 = v_setzero_s32(), + vs10 = v_setzero_s32(), vs11 = v_setzero_s32(), + vs12 = v_setzero_s32(), vs13 = v_setzero_s32(); + for( k = 0; k < vsz; k += 16, rptr += 16 ) + { + v_int8x16 w0 = v_load_aligned(wptr0 + k); + v_int8x16 w1 = v_load_aligned(wptr1 + k); + v_int8x16 r0 = v_load_aligned(rptr); + v_int8x16 r1 = v_load_aligned(rptr + vsz_a); + v_int8x16 r2 = v_load_aligned(rptr + vsz_a*2); + v_int8x16 r3 = v_load_aligned(rptr + vsz_a*3); + + vs00 = v_dotprod_expand_fast(w0, r0, vs00); + vs01 = v_dotprod_expand_fast(w0, r1, vs01); + vs02 = v_dotprod_expand_fast(w0, r2, vs02); + vs03 = v_dotprod_expand_fast(w0, r3, vs03); + + vs10 = v_dotprod_expand_fast(w1, r0, vs10); + vs11 = v_dotprod_expand_fast(w1, r1, vs11); + vs12 = v_dotprod_expand_fast(w1, r2, vs12); + vs13 = v_dotprod_expand_fast(w1, r3, vs13); + } + s0 += v_int32x4(v_reduce_sum(vs00), v_reduce_sum(vs01), v_reduce_sum(vs02), v_reduce_sum(vs03)); + s1 += v_int32x4(v_reduce_sum(vs10), v_reduce_sum(vs11), v_reduce_sum(vs12), v_reduce_sum(vs13)); + if( cn1 == inpCn ) + { + s0 = voutzp + v_round(v_cvt_f32(s0)*vmult0); + s1 = voutzp + v_round(v_cvt_f32(s1)*vmult1); + + s0 = v_min(v_max(s0, outmin), outmax); + s1 = v_min(v_max(s1, outmin), outmax); + } + v_store(outptr0 + j, s0); + v_store(outptr1 + j, s1); + } + #endif + for( ; j < bsz; j++ ) + { + const int8_t* rptr = rowbuf0 + j*vsz_a; + int s00, s10; + + if( cn0 == 0 ) + { + s00 = bias0; + s10 = bias1; + } + else + { + s00 = outptr0[j]; + s10 = outptr1[j]; + } + + for( k = 0; k < vsz; k++ ) + { + int8_t r0 = rptr[k]; + s00 += (int)wptr0[k] * r0; + s10 += (int)wptr1[k] * r0; + } + if( cn1 == inpCn ) + { + int out0 = outZp + (int)std::round(s00*mult0); + int out1 = outZp + (int)std::round(s10*mult1); + + s00 = std::min(std::max(out0, -128), 127); + s10 = std::min(std::max(out1, -128), 127); + } + + outptr0[j] = s00; + outptr1[j] = s10; + } + } + } + } + if( activ_ ) + activ_->forwardSlice(data_out0 + stripeStart, lutptr_, + data_out0 + stripeStart, (int)(stripeEnd - stripeStart), + outPlaneSize, startOutCn, startOutCn + outCn); + } + } + }; + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + +#if CV_SSE3 + uint32_t ftzMode = _MM_GET_FLUSH_ZERO_MODE(); + uint32_t dazMode = _MM_GET_DENORMALS_ZERO_MODE(); + _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); + _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON); +#endif + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + /*if (inputs[0].dims > 3) { + printf("conv %s: input (%d x %d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n", + name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2], inputs[0].size[3], + kernel.width, kernel.height, pad.width, pad.height, + stride.width, stride.height, dilation.width, dilation.height); + } + else { + printf("conv %s: input (%d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n", + name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2], + kernel.width, kernel.height, pad.width, pad.height, + stride.width, stride.height, dilation.width, dilation.height); + }*/ + + int inpGroupCn = blobs[0].size[1]; + CV_Assert_N(inputs.size() == (size_t)1, inputs[0].size[1] % inpGroupCn == 0, + outputs.size() == 1, inputs[0].data != outputs[0].data); + + int ngroups = inputs[0].size[1] / inpGroupCn; + CV_Assert(outputs[0].size[1] % ngroups == 0); + + int nstripes = std::max(getNumThreads(), 1); + Mat outputInt32 = Mat(shape(outputs[0]), CV_32S); + + ParallelConv::run(inputs[0], outputInt32, weightsMat, outputMultiplier, biasvec, activationLUT, kernel_size, strides, + pads_begin, pads_end, dilations, activ.get(), ngroups, nstripes, input_zp, output_zp); + + outputInt32.convertTo(outputs[0], CV_8S); + +#if CV_SSE3 + _MM_SET_FLUSH_ZERO_MODE(ftzMode); + _MM_SET_DENORMALS_ZERO_MODE(dazMode); +#endif + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_Assert(inputs.size() == outputs.size()); + + int64 flops = 0; + int karea = std::accumulate(kernel_size.begin(), kernel_size.end(), 1, std::multiplies()); + for (int i = 0; i < outputs.size(); i++) + { + flops += total(outputs[i])*(CV_BIG_INT(2)*karea*inputs[i][1] + 1); + } + return flops; + } +}; + +Ptr ConvolutionLayerInt8::create(const LayerParams ¶ms) +{ + return Ptr(new ConvolutionLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/elementwise_layers.cpp b/modules/dnn/src/int8layers/elementwise_layers.cpp new file mode 100644 index 000000000000..75118b6bc123 --- /dev/null +++ b/modules/dnn/src/int8layers/elementwise_layers.cpp @@ -0,0 +1,190 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include +#include + +namespace cv +{ +namespace dnn +{ + +class ActivationLayerInt8Impl CV_FINAL : public ActivationLayerInt8 +{ +public: + ActivationLayerInt8Impl(const LayerParams ¶ms) + { + setParamsFrom(params); + activationLUT = !blobs.empty() ? blobs[0] : Mat(); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return true; + } + + class Activation : public cv::ParallelLoopBody + { + public: + const Mat* src; + const Mat* lut; + Mat* dst; + int nstripes; + + Activation() : src(0), lut(0), dst(0), nstripes(0){} + + static void run(const Mat& src, const Mat& lut, Mat& dst, int nstripes) + { + Activation p; + + p.src = &src; + p.lut = &lut; + p.dst = &dst; + p.nstripes = nstripes; + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + void operator()(const Range &r) const CV_OVERRIDE + { + const int8_t* table = lut->ptr(); + int nsamples = 1, outCn = 1; + size_t planeSize = 1; + + if (src->dims > 1) + { + nsamples = src->size[0]; + outCn = src->size[1]; + } + else + outCn = src->size[0]; + + for (int i = 2; i < src->dims; ++i) + planeSize *= src->size[i]; + + size_t stripeSize = (planeSize + nstripes - 1)/nstripes; + size_t stripeStart = r.start*stripeSize; + size_t stripeEnd = std::min(r.end*stripeSize, planeSize); + int len = (int)(stripeEnd - stripeStart); + + for( int i = 0; i < nsamples; i++ ) + { + const int8_t* srcptr = src->ptr(i) + stripeStart; + int8_t* dstptr = dst->ptr(i) + stripeStart; + for( int cn = 0; cn < outCn; cn++, srcptr += planeSize, dstptr += planeSize ) + { + int i = 0; +#if CV_SIMD128 + for( ; i <= len - 16; i += 16 ) + { + v_int8x16 out(table[srcptr[i] + 128], table[srcptr[i+1] + 128], table[srcptr[i+2] + 128], table[srcptr[i+3] + 128], + table[srcptr[i+4] + 128], table[srcptr[i+5] + 128], table[srcptr[i+6] + 128], table[srcptr[i+7] + 128], + table[srcptr[i+8] + 128], table[srcptr[i+9] + 128], table[srcptr[i+10] + 128], table[srcptr[i+11] + 128], + table[srcptr[i+12] + 128], table[srcptr[i+13] + 128], table[srcptr[i+14] + 128], table[srcptr[i+15] + 128]); + v_store(dstptr + i, out); + } +#endif + for( ; i < len; i++ ) + { + dstptr[i] = table[srcptr[i] + 128]; + } + } + } + } + }; + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + for (size_t i = 0; i < inputs.size(); i++) + { + const Mat &src = inputs[i]; + if (!activationLUT.empty()) + { + const int nstripes = getNumThreads(); + Mat &dst = outputs[i]; + CV_Assert(src.size == dst.size && src.type() == dst.type() && + src.isContinuous() && dst.isContinuous() && src.type() == CV_8S); + + Activation::run(src, activationLUT, dst, nstripes); + } + else + { + src.copyTo(outputs[i]); + } + } + } + + void forwardSlice(const int8_t* src, const int8_t* lut, int8_t* dst, int len, size_t planeSize, int cn0, int cn1) const CV_OVERRIDE + { + for( int cn = cn0; cn < cn1; cn++, src += planeSize, dst += planeSize ) + { + int i = 0; +#if CV_SIMD128 + for( ; i <= len - 16; i += 16 ) + { + v_int8x16 out(lut[src[i] + 128], lut[src[i+1] + 128], lut[src[i+2] + 128], lut[src[i+3] + 128], + lut[src[i+4] + 128], lut[src[i+5] + 128], lut[src[i+6] + 128], lut[src[i+7] + 128], + lut[src[i+8] + 128], lut[src[i+9] + 128], lut[src[i+10] + 128], lut[src[i+11] + 128], + lut[src[i+12] + 128], lut[src[i+13] + 128], lut[src[i+14] + 128], lut[src[i+15] + 128]); + v_store(dst + i, out); + } +#endif + for( ; i < len; i++ ) + dst[i] = lut[src[i] + 128]; + } + } + + void forwardSlice(const int* src, const int* lut, int* dst, int len, size_t planeSize, int cn0, int cn1) const CV_OVERRIDE + { + for( int cn = cn0; cn < cn1; cn++, src += planeSize, dst += planeSize ) + { + int i = 0; +#if CV_SIMD128 + for( ; i <= len - 16; i += 16 ) + { + v_int32x4 out0(lut[src[i] + 128], lut[src[i+1] + 128], lut[src[i+2] + 128], lut[src[i+3] + 128]); + v_int32x4 out1(lut[src[i+4] + 128], lut[src[i+5] + 128], lut[src[i+6] + 128], lut[src[i+7] + 128]); + v_int32x4 out2(lut[src[i+8] + 128], lut[src[i+9] + 128], lut[src[i+10] + 128], lut[src[i+11] + 128]); + v_int32x4 out3(lut[src[i+12] + 128], lut[src[i+13] + 128], lut[src[i+14] + 128], lut[src[i+15] + 128]); + + v_store(dst + i, out0); + v_store(dst + i + 4, out1); + v_store(dst + i + 8, out2); + v_store(dst + i + 12, out3); + } +#endif + for( ; i < len; i++ ) + dst[i] = lut[src[i] + 128]; + } + + } + + Mat activationLUT; +}; + +Ptr ActivationLayerInt8::create(const LayerParams& params) +{ + return Ptr(new ActivationLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/eltwise_layer.cpp b/modules/dnn/src/int8layers/eltwise_layer.cpp new file mode 100644 index 000000000000..be7a32b1efd9 --- /dev/null +++ b/modules/dnn/src/int8layers/eltwise_layer.cpp @@ -0,0 +1,577 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include + +namespace cv +{ +namespace dnn +{ + +class EltwiseLayerInt8Impl CV_FINAL : public EltwiseLayerInt8 +{ +public: + enum EltwiseOp + { + PROD = 0, + SUM = 1, + MAX = 2 + } op; + std::vector coeffs; + std::vector zeropoints; + + enum OutputChannelsMode + { + ELTWISE_CHANNNELS_SAME = 0, //!< number of channels from inputs must be the same and equal to output's number of channels + ELTWISE_CHANNNELS_INPUT_0, //!< number of channels from inputs may be different, + //!< output's number of channels is equal to number of channels of first input + //!< number of channels of other inputs should not be greater than number of channels of first input + ELTWISE_CHANNNELS_INPUT_0_TRUNCATE, //!< number of channels from inputs may be different, + //!< output's number of channels is equal to number of channels of first input + //!< there is restriction on number of channels of other inputs + //!< extra channels of other inputs is ignored + ELTWISE_CHANNNELS_USE_MAX, //!< number of channels from inputs may be different, + //!< output's number of channels is equal to maximal number of input channels + //!< @note supported operation: `SUM` + } channelsModeInput; + + + mutable OutputChannelsMode channelsMode; //!< "optimized" channels mode (switch to ELTWISE_CHANNNELS_SAME if number of input channels are equal) + mutable /*size_t*/int outputChannels; + + EltwiseLayerInt8Impl(const LayerParams& params) + : outputChannels(0) + { + setParamsFrom(params); + offset = params.get("offset", 0.f); + hasVecInput = false; + op = SUM; + if (params.has("operation")) + { + String operation = toLowerCase(params.get("operation")); + if (operation == "prod") + op = PROD; + else if (operation == "sum") + op = SUM; + else if (operation == "max") + op = MAX; + else + CV_Error(cv::Error::StsBadArg, "Unknown operation type \"" + operation + "\""); + } + + if (params.has("coeff")) + { + DictValue paramCoeff = params.get("coeff"); + int i, n = paramCoeff.size(); + coeffs.resize(n); + for (i = 0; i < n; i++) + { + coeffs[i] = paramCoeff.get(i); + } + } + + if (params.has("input_zeropoints")) + { + DictValue zp = params.get("input_zeropoints"); + int i, n = zp.size(); + zeropoints.resize(n); + for (i = 0; i < n; i++) + { + zeropoints[i] = zp.get(i); + } + } + + channelsModeInput = ELTWISE_CHANNNELS_SAME; + if (params.has("output_channels_mode")) + { + String v = toLowerCase(params.get("output_channels_mode")); + if (v == "same") + { + channelsModeInput = ELTWISE_CHANNNELS_SAME; + } + else if (v == "input_0") + { + channelsModeInput = ELTWISE_CHANNNELS_INPUT_0; + } + else if (v == "input_0_truncate") + { + channelsModeInput = ELTWISE_CHANNNELS_INPUT_0_TRUNCATE; + } + else if (v == "max_input_channels") + { + channelsModeInput = ELTWISE_CHANNNELS_USE_MAX; + if (op != SUM) + CV_Error(cv::Error::StsBadArg, "[" + type + "]:(" + name + ") 'max' channels mode is limited to SUM operation only"); + } + else + CV_Error(cv::Error::StsBadArg, "[" + type + "]:(" + name + ") unknown channels mode: \"" + v + "\""); + } + channelsMode = channelsModeInput; + + // TODO Must have checks for other unknown options + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() >= 2); + CV_Assert(inputs[0].size() >= 2); + CV_Assert(coeffs.size() == 0 || coeffs.size() == inputs.size()); + CV_Assert(op == SUM || op == PROD || coeffs.size() == 0); + + int dims = inputs[0].size(); + // Number of channels in output shape is determined by the first input tensor. + bool variableChannels = false; + int numChannels = inputs[0][1]; + for (size_t i = 1; i < inputs.size(); i++) + { + CV_Assert(inputs[0][0] == inputs[i][0]); // batch sizes are equal + + int input_channels = inputs[i][1]; + if (numChannels != input_channels) + variableChannels = true; + + if (channelsModeInput == ELTWISE_CHANNNELS_SAME) + { + CV_Assert(numChannels == input_channels); + } + else if (channelsModeInput == ELTWISE_CHANNNELS_INPUT_0) + { + CV_Assert(numChannels >= input_channels); + } + else if (channelsModeInput == ELTWISE_CHANNNELS_INPUT_0_TRUNCATE) + { + // nothing to check + } + else if (channelsModeInput == ELTWISE_CHANNNELS_USE_MAX) + { + numChannels = std::max(numChannels, input_channels); + } + else + { + CV_Assert(0 && "Internal error"); + } + } + + channelsMode = variableChannels ? channelsModeInput : ELTWISE_CHANNNELS_SAME; + outputChannels = numChannels; + + outputs.assign(1, inputs[0]); + outputs[0][1] = numChannels; + + if (dims > 2) + { + size_t vecIdx = 0; + bool isVecFound = false; + for (size_t i = 0; i < inputs.size(); i++) + { + bool allOnes = isAllOnes(inputs[i], 2, dims); + if (!allOnes && !isVecFound) + { + vecIdx = i; + isVecFound = true; + } + + if (!allOnes && i != vecIdx) + { + for (size_t j = 2; j < dims; j++) + { + CV_Assert(inputs[vecIdx][j] == inputs[i][j]); + } + } + } + + if (channelsModeInput == ELTWISE_CHANNNELS_SAME && isVecFound) + { + for (size_t j = 2; j < dims; j++) + { + outputs[0][j] = inputs[vecIdx][j]; + } + } + } + + return false; + } + + void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays) CV_OVERRIDE + { + std::vector inputs; + inputs_arr.getMatVector(inputs); + + for (size_t i = 0; i < inputs.size(); i++) + { + MatShape inpShape = shape(inputs[i].size); + if (isAllOnes(inpShape, 2, inputs[i].dims)) + { + hasVecInput = true; + return; + } + } + } + + class EltwiseInvoker : public ParallelLoopBody + { + EltwiseLayerInt8Impl& self; + std::vector srcs; + std::vector srcNumChannels; + int nsrcs; + Mat* dst; + Mat* buf; + std::vector coeffs; + std::vector zeropoints; + int nstripes; + const Mat* activLUT; + const ActivationLayerInt8* activ; + int channels; + size_t planeSize; + float offset; + + EltwiseInvoker(EltwiseLayerInt8Impl& self_) + : self(self_) + , nsrcs(0), dst(0), buf(0), nstripes(0), activ(0), channels(0) + , planeSize(0), offset(0) + {} + + public: + static void run(EltwiseLayerInt8Impl& self, + const Mat* srcs, int nsrcs, Mat& buf, Mat& dst, + int nstripes, float offset) + { + const EltwiseOp op = self.op; + CV_Check(dst.dims, 1 < dst.dims && dst.dims <= 5, ""); CV_CheckTypeEQ(dst.type(), CV_8SC1, ""); CV_Assert(dst.isContinuous()); + CV_Assert(self.coeffs.empty() || self.coeffs.size() == (size_t)nsrcs); + CV_CheckGE(nsrcs, 2, ""); + + CV_Assert(self.outputChannels == dst.size[1]); + + EltwiseInvoker p(self); + p.srcs.resize(nsrcs); + p.srcNumChannels.resize(nsrcs); + p.coeffs = self.coeffs; // can be sorted + p.zeropoints = self.zeropoints; + + bool sortInputs = false; + for( int i = 0; i < nsrcs; i++ ) + { + p.srcs[i] = &srcs[i]; + CV_CheckEQ(srcs[i].dims, dst.dims, ""); + CV_Assert(srcs[i].isContinuous()); + CV_Assert(srcs[i].type() == dst.type()); + p.srcNumChannels[i] = (srcs[i].dims >= 4) ? srcs[i].size[1] : 1; + + if (self.channelsMode == ELTWISE_CHANNNELS_SAME) + { + CV_Assert(srcs[i].size == dst.size); + } + else if (self.channelsMode == ELTWISE_CHANNNELS_INPUT_0) + { + if (i == 0) + CV_Assert(srcs[0].size == dst.size); + CV_Assert(self.outputChannels >= p.srcNumChannels[i]); + sortInputs = true; + } + else if (self.channelsMode == ELTWISE_CHANNNELS_INPUT_0_TRUNCATE) + { + if (i == 0) + CV_Assert(srcs[0].size == dst.size); + sortInputs = true; + } + else if (self.channelsMode == ELTWISE_CHANNNELS_USE_MAX) + { + CV_Assert(op == SUM); + CV_Assert(self.outputChannels >= p.srcNumChannels[i]); + sortInputs = true; + } + else + { + CV_Assert(0 && "Internal error"); + } + + if (sortInputs) + { + // Sort srcs and coefficients in the desc order by number of channels + for (int j = i; j >= 1; j--) + { + if (std::min(self.outputChannels, p.srcs[j - 1]->size[1]) < std::min(self.outputChannels, p.srcs[j]->size[1])) + { + std::swap(p.srcs[j - 1], p.srcs[j]); + std::swap(p.srcNumChannels[j - 1], p.srcNumChannels[j]); + if (!p.coeffs.empty()) + std::swap(p.coeffs[j - 1], p.coeffs[j]); + if (!p.zeropoints.empty()) + std::swap(p.zeropoints[j - 1], p.zeropoints[j]); + } + else + break; + } + } + } + + p.nsrcs = nsrcs; + p.dst = &dst; + p.buf = &buf; + p.nstripes = nstripes; + p.offset = offset; + p.channels = (dst.dims >= 4 ? dst.size[1] : 1); + + p.planeSize = dst.total(dst.dims >= 4 ? 2 : 1); + CV_CheckEQ(dst.total(), dst.size[0] * p.channels * p.planeSize, ""); + p.activLUT = &self.activationLUT; + p.activ = !self.activationLUT.empty() ? self.activ.get() : 0; + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + void operator()(const Range& r) const CV_OVERRIDE + { + const EltwiseOp op = self.op; + size_t total = dst->size[0]*planeSize; + size_t stripeSize = (total + nstripes - 1)/nstripes; + size_t stripeStart = r.start*stripeSize; + size_t stripeEnd = std::min(r.end*stripeSize, total); + const float* coeffsptr = !coeffs.empty() ? &coeffs[0] : 0; + const int* zeropointsptr = !zeropoints.empty() ? &zeropoints[0] : 0; + const int8_t* lutptr = !activLUT->empty() ? activLUT->ptr() : 0; + int8_t* dstptr0 = dst->ptr(); + float* bufptr0 = buf->ptr(); + int blockSize0 = 1 << 12; + + for (size_t ofs = stripeStart; ofs < stripeEnd; ) + { + int sampleIdx = (int)(ofs / planeSize); + int delta = (int)ofs - sampleIdx * planeSize; + int blockSize = std::min(blockSize0, std::min((int)(stripeEnd - ofs), (int)planeSize - delta)); + if( blockSize <= 0 ) + break; + ofs += blockSize; + + for (int c = 0; c < channels; c++) + { + size_t dstIdx = delta + (sampleIdx*channels + c)*planeSize; + int8_t* dstptr = dstptr0 + dstIdx; + float* bufptr = bufptr0 + dstIdx; + + // process first two inputs + { + const int8_t* srcptr0 = srcs[0]->ptr() + dstIdx; + + const int inputIdx = 1; + int src1_channels = srcNumChannels[inputIdx]; + if (c >= src1_channels) + { + // no data from second input + if (!coeffsptr) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = srcptr0[j]; + } + } + else + { + float c0 = coeffsptr[0]; + int z0 = op == PROD ? zeropointsptr[0] : 0; + for (int j = 0; j < blockSize; j++) + { + bufptr[j] = c0 * (srcptr0[j] - z0); + } + } + } + else + { + size_t srcIdx = delta + (sampleIdx * src1_channels + c) * planeSize; + const int8_t* srcptrI = srcs[inputIdx]->ptr() + srcIdx; + + if (op == PROD) + { + float c0 = coeffsptr[0]; + float c1 = coeffsptr[1]; + int z0 = zeropointsptr[0]; + int z1 = zeropointsptr[1]; + for (int j = 0; j < blockSize; j++) + { + bufptr[j] = (c0*(srcptr0[j] - z0)) * (c1*(srcptrI[j] - z1)); + } + } + else if (op == MAX) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = std::max(srcptr0[j], srcptrI[j]); + } + } + else if (op == SUM) + { + float c0 = coeffsptr[0]; + float c1 = coeffsptr[1]; + for (int j = 0; j < blockSize; j++) + { + bufptr[j] = c0*srcptr0[j] + c1*srcptrI[j]; + } + } + else + CV_Error(Error::StsInternal, ""); + } + } + + // aggregate other inputs (3+) + for (size_t inputIdx = 2; inputIdx < nsrcs; inputIdx++) + { + int srcI_channels = srcNumChannels[inputIdx]; + if (c >= srcI_channels) + continue; // no data from second input + size_t srcIdx = delta + (sampleIdx * srcI_channels + c) * planeSize; + const int8_t* srcptrI = srcs[inputIdx]->ptr() + srcIdx; + + if (op == PROD) + { + float cI = coeffsptr[inputIdx]; + int zI = zeropointsptr[inputIdx]; + for (int j = 0; j < blockSize; j++) + { + bufptr[j] *= cI*(srcptrI[j] - zI); + } + } + else if (op == MAX) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = std::max(dstptr[j], srcptrI[j]); + } + } + else if (op == SUM) + { + float cI = coeffsptr[inputIdx]; + for (int j = 0; j < blockSize; j++) + { + bufptr[j] += cI * srcptrI[j]; + } + } + else + CV_Error(Error::StsInternal, ""); + } + + // add offset and saturate cast to int8 + if (op == SUM || op == PROD) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = saturate_cast(std::round(bufptr[j] + offset)); + } + } + } + if( activ ) + { + int8_t* ptr = dstptr0 + delta + sampleIdx*channels*planeSize; + activ->forwardSlice(ptr, lutptr, ptr, blockSize, planeSize, 0, channels); + } + } + } + }; + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + CV_Assert(outputs.size() == 1); + const int nstripes = getNumThreads(); + + if (channelsModeInput == ELTWISE_CHANNNELS_SAME && inputs[0].dims > 2) + { + for (size_t i = 0; i < inputs.size(); i++) + { + MatShape inpShape = shape(inputs[i].size); + bool allOnes = isAllOnes(inpShape, 2, inputs[i].dims); + + if (allOnes) + { + Mat tmpInput = inputs[i]; + MatShape outShape = shape(outputs[0].size); + size_t xSize = outShape[2]; + for (size_t j = 3; j < outShape.size(); j++) + xSize *= outShape[j]; + + int dimVec[3] = {outShape[0], outShape[1], (int) xSize}; + std::vector matSizesVec(&dimVec[0], &dimVec[0] + 3); + inputs[i] = Mat(matSizesVec, tmpInput.type()); + + std::vector idx(outShape.size(), 0); + std::vector outIdx(inpShape.size(), 0); + + for (size_t j = 0; j < outShape[0]; j++) + { + outIdx[0] = idx[0] = j; + for(size_t k = 0; k < outShape[1]; k++) + { + outIdx[1] = idx[1] = k; + for (size_t x = 0; x < xSize; x++) + { + outIdx[2] = x; + inputs[i].at(outIdx.data()) = tmpInput.at(idx.data()); + } + } + } + inputs[i] = inputs[i].reshape(0, outShape); + } + } + } + + Mat buf = Mat(shape(outputs[0]), CV_32F); // to store intermediate results + EltwiseInvoker::run(*this, &inputs[0], (int)inputs.size(), buf, outputs[0], nstripes, offset); + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(outputs); // suppress unused variable warning + CV_Assert(inputs.size()); + + // FIXIT: handle inputs with different number of channels + long flops = inputs.size() * total(inputs[0]); + + return flops; + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + activ = activ_int8; + if (!activ_int8->blobs.empty()) + activationLUT = activ_int8->blobs[0]; + return true; + } + return false; + } + + Mat activationLUT; + Ptr activ; + +private: + bool hasVecInput; + float offset; +}; + +Ptr EltwiseLayerInt8::create(const LayerParams& params) +{ + return Ptr(new EltwiseLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/fully_connected_layer.cpp b/modules/dnn/src/int8layers/fully_connected_layer.cpp new file mode 100644 index 000000000000..83da677a47f6 --- /dev/null +++ b/modules/dnn/src/int8layers/fully_connected_layer.cpp @@ -0,0 +1,266 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include + +namespace cv +{ +namespace dnn +{ + +class FullyConnectedLayerInt8Impl CV_FINAL : public InnerProductLayerInt8 +{ +public: + enum { VEC_ALIGN = 32 }; + FullyConnectedLayerInt8Impl(const LayerParams& params) + { + setParamsFrom(params); + output_zp = params.get("zeropoints"); + axis = params.get("axis", 1); + if (blobs.size() == 3) + { + // blobs[0] - Weights + // blobs[1] - Bias fused with offset + // blobs[2] - Multipliers for output stage + int numOutput = params.get("num_output"); + int innerSize = (int)blobs[0].total() / numOutput; + + CV_Assert(blobs[0].dims >= 2 && (size_t)(innerSize * numOutput) == blobs[0].total()); + CV_Assert((size_t)numOutput == blobs[1].total()); + + weightsMat = blobs[0] = blobs[0].reshape(1, numOutput); + int vecsize = weightsMat.cols; + if (vecsize % VEC_ALIGN != 0) + { + int vecsize_aligned = (int)alignSize(vecsize, VEC_ALIGN); + Mat weightsBuf(weightsMat.rows, vecsize_aligned, weightsMat.type()); + Mat wpadding = weightsBuf.colRange(vecsize, vecsize_aligned); + wpadding.setTo(Scalar::all(0)); + weightsMat = weightsBuf.colRange(0, vecsize); + blobs[0].copyTo(weightsMat); + } + biasMat = blobs[1] = blobs[1].reshape(1, 1); + outputMultiplier = blobs[2]; + } + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &) const CV_OVERRIDE + { + int numOutput, cAxis; + CV_CheckEQ(inputs.size(), (size_t)1, ""); + CV_CheckEQ(blobs[0].dims, 2, ""); + numOutput = blobs[0].size[0]; + CV_Assert((size_t)numOutput == blobs[1].total()); + cAxis = normalize_axis(axis, inputs[0]); + + MatShape outShape(cAxis + 1); + for (int i = 0; i < cAxis; ++i) + outShape[i] = inputs[0][i]; + outShape.back() = numOutput; + + outputs.resize(1, outShape); + return false; + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + virtual bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + activ = activ_int8; + if (!activ_int8->blobs.empty()) + activ_int8->blobs[0].convertTo(activationLUT, CV_32S); + return true; + } + return false; + } + + class FullyConnected : public ParallelLoopBody + { + public: + FullyConnected() : srcMat(0), weights(0), biasMat(0), outputMultiplier(0), activationLUT(0), activ(0), + dstMat(0), nstripes(0), outZp(0), useAVX2(false), useAVX512(false) {} + + static void run(const Mat& srcMat, const Mat& weights, const Mat& biasMat, const Mat& outputMultiplier, + const Mat& activationLUT, Mat& dstMat, const ActivationLayerInt8* activ, int nstripes, int outZp) + { + CV_Assert( srcMat.dims == 2 && srcMat.cols == weights.cols && + dstMat.rows == srcMat.rows && dstMat.cols == weights.rows && + srcMat.type() == weights.type() && srcMat.type() == CV_8S && + dstMat.type() == CV_32S && biasMat.type() == CV_32S && + biasMat.isContinuous() && (int)biasMat.total() == dstMat.cols ); + + FullyConnected p; + + p.srcMat = &srcMat; + p.weights = &weights; + p.biasMat = &biasMat; + p.outputMultiplier = &outputMultiplier; + p.activationLUT = &activationLUT; + p.dstMat = &dstMat; + p.nstripes = nstripes; + p.outZp = outZp; + p.activ = !activationLUT.empty() ? activ : 0; + p.useAVX2 = checkHardwareSupport(CPU_AVX2); + p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX; + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + void operator()(const Range& r) const CV_OVERRIDE + { + int valign = FullyConnectedLayerInt8Impl::VEC_ALIGN; + int nsamples = srcMat->rows; + int nw0 = weights->rows; + int k, vecsize = srcMat->cols; + int vecsize_aligned = (int)alignSize(vecsize, VEC_ALIGN); + size_t total = (size_t)nsamples*nw0; + size_t stripeSize = (total + nstripes - 1)/nstripes; + size_t stripeStart = r.start*stripeSize; + size_t stripeEnd = r.end == nstripes ? total : std::min(r.end*stripeSize, total); + size_t wstep = weights->step1(); + AutoBuffer srcbuf(vecsize_aligned + valign); + int8_t* sptr = alignPtr(srcbuf.data(), (int)(valign*sizeof(int8_t))); + const int* lutptr = !activationLUT->empty() ? activationLUT->ptr() : 0; + + for( k = vecsize; k < vecsize_aligned; k++ ) + sptr[k] = 0; + + for( size_t ofs = stripeStart; ofs < stripeEnd; ) + { + int sampleIdx = (int)(ofs / nw0); + int delta = (int)(ofs - (size_t)sampleIdx*nw0); + const int8_t* sptr_ = srcMat->ptr(sampleIdx); + const int8_t* wptr = weights->ptr(delta); + int* dptr = dstMat->ptr(sampleIdx) + delta; + const int* biasptr = biasMat->ptr() + delta; + const float* multptr = outputMultiplier->ptr() + delta; + int nw = std::min(nw0 - delta, (int)(stripeEnd - ofs)); + + memcpy(sptr, sptr_, vecsize*sizeof(sptr[0])); + #if CV_TRY_AVX512_SKX + if( useAVX512 ) + opt_AVX512_SKX::fastGEMM1T( sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp ); + else + #endif + #if CV_TRY_AVX2 + if( useAVX2 ) + opt_AVX2::fastGEMM1T( sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp ); + else + #endif + { + int i = 0; + #if CV_SIMD + for( ; i <= nw - 4; i += 4, wptr += 4*wstep ) + { + v_int32x4 vs0 = v_setzero_s32(), vs1 = v_setzero_s32(), + vs2 = v_setzero_s32(), vs3 = v_setzero_s32(); + v_int32x4 outzp = v_setall_s32(outZp), outmin = v_setall_s32(-128), outmax = v_setall_s32(127); + v_int32x4 s = v_load(biasptr + i); + v_float32x4 mult = v_load(multptr + i); + + for( k = 0; k < vecsize; k += 16 ) + { + v_int8x16 v = v_load_aligned(sptr + k); + vs0 = v_dotprod_expand_fast(v, v_load_aligned(wptr + k), vs0); + vs1 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep + k), vs1); + vs2 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep*2 + k), vs2); + vs3 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep*3 + k), vs3); + } + + s += v_int32x4(v_reduce_sum(vs0), v_reduce_sum(vs1), v_reduce_sum(vs2), v_reduce_sum(vs3)); + v_int32x4 out = outzp + v_round(v_cvt_f32(s)*mult); + v_store(dptr + i, v_min(v_max(out, outmin), outmax)); + } + #endif + + for( ; i < nw; i++, wptr += wstep ) + { + int s0 = biasptr[i]; + float mult0 = multptr[i]; + + for( k = 0; k < vecsize; k++ ) + { + int8_t v = sptr[k]; + s0 += (int)v*wptr[k]; + } + int out0 = outZp + (int)std::round(s0*mult0); + dptr[i] = std::min(std::max(out0, -128), 127); + } + } + + if(activ) + activ->forwardSlice(dptr, lutptr, dptr, 1, 1, delta, delta + nw); + + ofs += nw; + } + } + + const Mat *srcMat, *weights, *biasMat, *outputMultiplier, *activationLUT; + const ActivationLayerInt8* activ; + Mat* dstMat; + int nstripes, outZp; + bool useAVX2; + bool useAVX512; + }; + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector input, output; + inputs_arr.getMatVector(input); + outputs_arr.getMatVector(output); + + int axisCan = normalize_axis(axis, input[0].dims); + int outerSize = input[0].total(0, axisCan); + Mat srcMat = input[0].reshape(1, outerSize); + + Mat dstMat = output[0].reshape(1, outerSize); + Mat dstMatInt32= Mat(shape(dstMat), CV_32S); + + const int nstripes = getNumThreads(); + FullyConnected::run(srcMat, weightsMat, biasMat, outputMultiplier, activationLUT, dstMatInt32, activ.get(), nstripes, output_zp); + dstMatInt32.convertTo(dstMat, CV_8S); + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(inputs); // suppress unused variable warning + long flops = 0; + + int innerSize = blobs[0].size[1]; + for(int i = 0; i < outputs.size(); i++) + { + flops += CV_BIG_INT(3)*innerSize*total(outputs[i]); + } + + return flops; + + } + + Mat weightsMat, biasMat, outputMultiplier, activationLUT; + Ptr activ; +}; + +Ptr InnerProductLayerInt8::create(const LayerParams& params) +{ + return Ptr(new FullyConnectedLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/layers_common.hpp b/modules/dnn/src/int8layers/layers_common.hpp new file mode 100644 index 000000000000..cb185a9edaa4 --- /dev/null +++ b/modules/dnn/src/int8layers/layers_common.hpp @@ -0,0 +1,41 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef __OPENCV_DNN_LAYERS_LAYERS_COMMON_HPP__ +#define __OPENCV_DNN_LAYERS_LAYERS_COMMON_HPP__ +#include +#include + +#define CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY +// dispatched AVX/AVX2 optimizations +#include "./layers_common.simd.hpp" +#include "int8layers/layers_common.simd_declarations.hpp" +#undef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY + +#ifdef HAVE_OPENCL +#include "../ocl4dnn/include/ocl4dnn.hpp" +#endif + +namespace cv +{ +namespace dnn +{ +void getConvolutionKernelParams(const LayerParams ¶ms, std::vector& kernel, std::vector& pads_begin, + std::vector& pads_end, std::vector& strides, std::vector& dilations, + cv::String &padMode, std::vector& adjust_pads); + +void getPoolingKernelParams(const LayerParams ¶ms, std::vector& kernel, std::vector& globalPooling, + std::vector& pads_begin, std::vector& pads_end, std::vector& strides, cv::String &padMode); + +void getConvPoolOutParams(const std::vector& inp, const std::vector& kernel, + const std::vector& stride, const String &padMode, + const std::vector& dilation, std::vector& out); + + void getConvPoolPaddings(const std::vector& inp, const std::vector& kernel, + const std::vector& strides, const String &padMode, + std::vector& pads_begin, std::vector& pads_end); +} +} + +#endif diff --git a/modules/dnn/src/int8layers/layers_common.simd.hpp b/modules/dnn/src/int8layers/layers_common.simd.hpp new file mode 100644 index 000000000000..bf6149e5c958 --- /dev/null +++ b/modules/dnn/src/int8layers/layers_common.simd.hpp @@ -0,0 +1,637 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "opencv2/core/hal/intrin.hpp" + +namespace cv { +namespace dnn { +CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN + +void fastConv( const int8_t* weights, size_t wstep, const int* bias, + const int8_t* rowbuf, int* output, const int* outShape, + int blockSize, int vecsize, int vecsize_aligned, int outZp, + const float* multiplier, bool initOutput, bool finalOutput ); +void fastDepthwiseConv( const int8_t* wptr, + int kernel_h, int kernel_w, + int stride_h, int stride_w, + int dilation_h, int dilation_w, + int pad_t, int pad_l, + const int* biasptr, const float* multptr, + const int8_t* inptr_, + int height, int width, + int* outptr_, + int out_d, int outH, int outW, + int inpZp, int outZp ); +void fastGEMM1T( const int8_t* vec, const int8_t* weights, + size_t wstep, const int* bias, const float* multiplier, + int* dst, int nvecs, int vecsize, int outZp ); + +#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_AVX2 +#define OPENCV_FMADD_EPI8(_Tpvec, func) \ + inline _Tpvec _##func##_fmaddepi8_epi32(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ + { \ + _Tpvec even_a = _##func##_srai_epi16(_##func##_bslli_epi128(a, 1), 8); \ + _Tpvec odd_a = _##func##_srai_epi16(a, 8); \ + \ + _Tpvec even_b = _##func##_srai_epi16(_##func##_bslli_epi128(b, 1), 8); \ + _Tpvec odd_b = _##func##_srai_epi16(b, 8); \ + \ + _Tpvec prod0 = _##func##_madd_epi16(even_a, even_b); \ + _Tpvec prod1 = _##func##_madd_epi16(odd_a, odd_b); \ + return _##func##_add_epi32(_##func##_add_epi32(prod0, prod1), c); \ + } +OPENCV_FMADD_EPI8(__m256i, mm256) +//OPENCV_FMADD_EPI8(__m512i, mm512) + +enum { FASCONV_BASE_VECSZ = 4 }; + +void fastConv( const int8_t* weights, size_t wstep, const int* bias, + const int8_t* rowbuf, int* output, const int* outShape, + int blockSize, int vecsize, int vecsize_aligned, int outZp, + const float* multiplier, bool initOutput, bool finalOutput ) +{ + int outCn = outShape[1]; + size_t outPlaneSize = outShape[2]*outShape[3]; + int CV_DECL_ALIGNED(16) maskbuf[FASCONV_BASE_VECSZ] = {0}; + int rsz = blockSize % FASCONV_BASE_VECSZ; + for( int i = 0; i < rsz; i++ ) + maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1; + __m128 mask = _mm_loadu_ps((const float*)maskbuf); + + // now compute dot product of the weights + // and im2row-transformed part of the tensor + for( int i = 0; i < outCn; i += 3 ) + { + const int8_t* wptr0 = weights + i*wstep; + const int8_t* wptr1 = wptr0 + wstep; + const int8_t* wptr2 = wptr1 + wstep; + int* outptr0 = output + i*outPlaneSize; + int* outptr1 = outptr0 + outPlaneSize; + int* outptr2 = outptr1 + outPlaneSize; + int bias0 = bias[i], bias1 = bias[i+1], bias2 = bias[i+2]; + float mult0 = multiplier[i], mult1 = multiplier[i+1], mult2 = multiplier[i+2]; + + if( i+2 >= outCn ) + { + wptr2 = wptr1; + outptr2 = outptr1; + bias2 = bias1; + mult2 = mult1; + + if( i+1 >= outCn ) + { + wptr2 = wptr1 = wptr0; + outptr2 = outptr1 = outptr0; + bias2 = bias1 = bias0; + mult2 = mult1 = mult0; + } + } + int j = 0; + for( ; j < blockSize; j += FASCONV_BASE_VECSZ ) + { + bool tail = false; + if (j + FASCONV_BASE_VECSZ > blockSize) + { + if (j == 0) + break; + j = blockSize - FASCONV_BASE_VECSZ; + tail = true; + } + int k = 0; + const int8_t* rptr = rowbuf + j*vecsize_aligned; + + __m256i vs00 = _mm256_setzero_si256(), vs01 = _mm256_setzero_si256(), + vs02 = _mm256_setzero_si256(), vs03 = _mm256_setzero_si256(), + vs10 = _mm256_setzero_si256(), vs11 = _mm256_setzero_si256(), + vs12 = _mm256_setzero_si256(), vs13 = _mm256_setzero_si256(), + vs20 = _mm256_setzero_si256(), vs21 = _mm256_setzero_si256(), + vs22 = _mm256_setzero_si256(), vs23 = _mm256_setzero_si256(); + + /* TODO : Fix AVX-512 path. Segmentation fault in Conv2D Tests. +#if CV_AVX512_SKX // AVX512VL is necessary to avoid register spilling + if (vecsize >= 64) + { + __m512i vs00_5 = _mm512_setzero_si512(), vs01_5 = _mm512_setzero_si512(), + vs02_5 = _mm512_setzero_si512(), vs03_5 = _mm512_setzero_si512(), + vs10_5 = _mm512_setzero_si512(), vs11_5 = _mm512_setzero_si512(), + vs12_5 = _mm512_setzero_si512(), vs13_5 = _mm512_setzero_si512(), + vs20_5 = _mm512_setzero_si512(), vs21_5 = _mm512_setzero_si512(), + vs22_5 = _mm512_setzero_si512(), vs23_5 = _mm512_setzero_si512(); + + for (; k <= vecsize - 64; k += 64, rptr += 64) + { + __m512i w0 = _mm512_load_si512(wptr0 + k); + __m512i w1 = _mm512_load_si512(wptr1 + k); + __m512i w2 = _mm512_load_si512(wptr2 + k); + __m512i r0 = _mm512_load_si512(rptr); + + vs00_5 = _mm512_fmaddepi8_epi32(w0, r0, vs00_5); + vs10_5 = _mm512_fmaddepi8_epi32(w1, r0, vs10_5); + vs20_5 = _mm512_fmaddepi8_epi32(w2, r0, vs20_5); + + r0 = _mm512_load_si512(rptr + vecsize_aligned); + vs01_5 = _mm512_fmaddepi8_epi32(w0, r0, vs01_5); + vs11_5 = _mm512_fmaddepi8_epi32(w1, r0, vs11_5); + vs21_5 = _mm512_fmaddepi8_epi32(w2, r0, vs21_5); + + r0 = _mm512_load_si512(rptr + vecsize_aligned*2); + vs02_5 = _mm512_fmaddepi8_epi32(w0, r0, vs02_5); + vs12_5 = _mm512_fmaddepi8_epi32(w1, r0, vs12_5); + vs22_5 = _mm512_fmaddepi8_epi32(w2, r0, vs22_5); + + r0 = _mm512_load_si512(rptr + vecsize_aligned*3); + vs03_5 = _mm512_fmaddepi8_epi32(w0, r0, vs03_5); + vs13_5 = _mm512_fmaddepi8_epi32(w1, r0, vs13_5); + vs23_5 = _mm512_fmaddepi8_epi32(w2, r0, vs23_5); + } + + // now fold the 512 bit accumulator vectors into 256 bit vectors so that the AVX2 code can finish + // the tail of the vector + + vs00 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs00_5, 0), _mm512_extracti32x8_epi32(vs00_5, 1)); + vs10 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs10_5, 0), _mm512_extracti32x8_epi32(vs10_5, 1)); + vs20 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs20_5, 0), _mm512_extracti32x8_epi32(vs20_5, 1)); + + vs01 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs01_5, 0), _mm512_extracti32x8_epi32(vs01_5, 1)); + vs11 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs11_5, 0), _mm512_extracti32x8_epi32(vs11_5, 1)); + vs21 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs21_5, 0), _mm512_extracti32x8_epi32(vs21_5, 1)); + + vs02 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs02_5, 0), _mm512_extracti32x8_epi32(vs02_5, 1)); + vs12 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs12_5, 0), _mm512_extracti32x8_epi32(vs12_5, 1)); + vs22 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs22_5, 0), _mm512_extracti32x8_epi32(vs22_5, 1)); + + vs03 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs03_5, 0), _mm512_extracti32x8_epi32(vs03_5, 1)); + vs13 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs13_5, 0), _mm512_extracti32x8_epi32(vs13_5, 1)); + vs23 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs23_5, 0), _mm512_extracti32x8_epi32(vs23_5, 1)); + } +#endif + */ + for (; k < vecsize; k += 32, rptr += 32 ) + { + __m256i w0 = _mm256_load_si256((const __m256i*)(wptr0 + k)); + __m256i w1 = _mm256_load_si256((const __m256i*)(wptr1 + k)); + __m256i w2 = _mm256_load_si256((const __m256i*)(wptr2 + k)); + __m256i r0 = _mm256_load_si256((const __m256i*)rptr); + + vs00 = _mm256_fmaddepi8_epi32(w0, r0, vs00); + vs10 = _mm256_fmaddepi8_epi32(w1, r0, vs10); + vs20 = _mm256_fmaddepi8_epi32(w2, r0, vs20); + + r0 = _mm256_load_si256((const __m256i*)(rptr + vecsize_aligned)); + vs01 = _mm256_fmaddepi8_epi32(w0, r0, vs01); + vs11 = _mm256_fmaddepi8_epi32(w1, r0, vs11); + vs21 = _mm256_fmaddepi8_epi32(w2, r0, vs21); + + r0 = _mm256_load_si256((const __m256i*)(rptr + vecsize_aligned*2)); + vs02 = _mm256_fmaddepi8_epi32(w0, r0, vs02); + vs12 = _mm256_fmaddepi8_epi32(w1, r0, vs12); + vs22 = _mm256_fmaddepi8_epi32(w2, r0, vs22); + + r0 = _mm256_load_si256((const __m256i*)(rptr + vecsize_aligned*3)); + vs03 = _mm256_fmaddepi8_epi32(w0, r0, vs03); + vs13 = _mm256_fmaddepi8_epi32(w1, r0, vs13); + vs23 = _mm256_fmaddepi8_epi32(w2, r0, vs23); + } + + __m256i t0 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs00, vs01), _mm256_hadd_epi32(vs02, vs03)); + __m256i t1 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs10, vs11), _mm256_hadd_epi32(vs12, vs13)); + __m256i t2 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs20, vs21), _mm256_hadd_epi32(vs22, vs23)); + + t0 = _mm256_add_epi32(t0, _mm256_permute2x128_si256(t0, t0, 1)); + t1 = _mm256_add_epi32(t1, _mm256_permute2x128_si256(t1, t1, 1)); + t2 = _mm256_add_epi32(t2, _mm256_permute2x128_si256(t2, t2, 1)); + + __m128i s0, s1, s2; + + if( initOutput ) + { + s0 = _mm_set1_epi32(bias0); + s1 = _mm_set1_epi32(bias1); + s2 = _mm_set1_epi32(bias2); + } + else + { + s0 = _mm_loadu_si128((__m128i*)(outptr0 + j)); + s1 = _mm_loadu_si128((__m128i*)(outptr1 + j)); + s2 = _mm_loadu_si128((__m128i*)(outptr2 + j)); + } + + s0 = _mm_add_epi32(s0, _mm256_castsi256_si128(t0)); + s1 = _mm_add_epi32(s1, _mm256_castsi256_si128(t1)); + s2 = _mm_add_epi32(s2, _mm256_castsi256_si128(t2)); + + if( finalOutput ) + { + __m128i voutzp = _mm_set1_epi32(outZp); + __m128i outmin = _mm_set1_epi32(-128), outmax = _mm_set1_epi32(127); + s0 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(s0), _mm_set1_ps(mult0)))); + s1 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(s1), _mm_set1_ps(mult1)))); + s2 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(s2), _mm_set1_ps(mult2)))); + + s0 = _mm_min_epi32(_mm_max_epi32(s0, outmin), outmax); + s1 = _mm_min_epi32(_mm_max_epi32(s1, outmin), outmax); + s2 = _mm_min_epi32(_mm_max_epi32(s2, outmin), outmax); + } + if( tail ) + { + s0 = _mm_castps_si128(_mm_blendv_ps(_mm_loadu_ps((const float*)outptr0 + j), _mm_castsi128_ps(s0), mask)); + s1 = _mm_castps_si128(_mm_blendv_ps(_mm_loadu_ps((const float*)outptr1 + j), _mm_castsi128_ps(s1), mask)); + s2 = _mm_castps_si128(_mm_blendv_ps(_mm_loadu_ps((const float*)outptr2 + j), _mm_castsi128_ps(s2), mask)); + } + _mm_storeu_si128((__m128i*)(outptr0 + j), s0); + _mm_storeu_si128((__m128i*)(outptr1 + j), s1); + _mm_storeu_si128((__m128i*)(outptr2 + j), s2); + } + + for( ; j <= blockSize - 2; j += 2 ) + { + const int8_t* rptr0 = rowbuf + j*vecsize_aligned; + const int8_t* rptr1 = rowbuf + (j+1)*vecsize_aligned; + int s00, s01, s10, s11, s20, s21; + + if( initOutput ) + { + s00 = s01 = bias0; + s10 = s11 = bias1; + s20 = s21 = bias2; + } + else + { + s00 = outptr0[j]; s01 = outptr0[j+1]; + s10 = outptr1[j]; s11 = outptr1[j+1]; + s20 = outptr2[j]; s21 = outptr2[j+1]; + } + + for( int k = 0; k < vecsize; k++ ) + { + int8_t w0 = wptr0[k], w1 = wptr1[k], w2 = wptr2[k]; + int8_t r = rptr0[k]; + s00 += (int)w0*r; s10 += (int)w1*r; s20 += (int)w2*r; + r = rptr1[k]; + s01 += (int)w0*r; s11 += (int)w1*r; s21 += (int)w2*r; + } + + if( finalOutput ) + { + s00 = std::min(std::max(outZp + (int)std::round(s00*mult0), -128), 127); + s01 = std::min(std::max(outZp + (int)std::round(s01*mult0), -128), 127); + s10 = std::min(std::max(outZp + (int)std::round(s10*mult1), -128), 127); + s11 = std::min(std::max(outZp + (int)std::round(s11*mult1), -128), 127); + s20 = std::min(std::max(outZp + (int)std::round(s20*mult2), -128), 127); + s21 = std::min(std::max(outZp + (int)std::round(s21*mult2), -128), 127); + } + outptr0[j] = s00; + outptr0[j+1] = s01; + outptr1[j] = s10; + outptr1[j+1] = s11; + outptr2[j] = s20; + outptr2[j+1] = s21; + } + + for( ; j < blockSize; j++ ) + { + const int8_t* rptr0 = rowbuf + j*vecsize_aligned; + int s00, s10, s20; + + if( initOutput ) + { + s00 = bias0; + s10 = bias1; + s20 = bias2; + } + else + { + s00 = outptr0[j]; + s10 = outptr1[j]; + s20 = outptr2[j]; + } + + for( int k = 0; k < vecsize; k++ ) + { + int8_t w0 = wptr0[k], w1 = wptr1[k], w2 = wptr2[k]; + int8_t r = rptr0[k]; + s00 += (int)w0*r; s10 += (int)w1*r; s20 += (int)w2*r; + } + + if( finalOutput ) + { + s00 = std::min(std::max(outZp + (int)std::round(s00*mult0), -128), 127); + s10 = std::min(std::max(outZp + (int)std::round(s10*mult1), -128), 127); + s20 = std::min(std::max(outZp + (int)std::round(s20*mult2), -128), 127); + } + outptr0[j] = s00; + outptr1[j] = s10; + outptr2[j] = s20; + } + } + _mm256_zeroupper(); +} + +static inline void _mm256_expand_mul_add(const __m256i& a, const __m256i& b, + __m256i& out0, __m256i& out1, __m256i& out2, __m256i& out3) +{ + __m256i a0 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(a)); + __m256i a1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a, 1)); + + __m256i b0 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(b)); + __m256i b1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b, 1)); + + __m256i a0b0 = _mm256_mullo_epi16(a0, b0); + __m256i a1b1 = _mm256_mullo_epi16(a1, b1); + + out0 = _mm256_add_epi32(out0, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(a0b0))); + out1 = _mm256_add_epi32(out1, _mm256_cvtepi16_epi32(_mm256_extracti128_si256(a0b0, 1))); + out2 = _mm256_add_epi32(out2, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(a1b1))); + out3 = _mm256_add_epi32(out3, _mm256_cvtepi16_epi32(_mm256_extracti128_si256(a1b1, 1))); +} + +static inline void _mm256_load_deinterleave(const int8_t* ptr, __m256i& a, __m256i& b) +{ + __m256i t0 = _mm256_loadu_si256((const __m256i*)ptr); + __m256i t1 = _mm256_loadu_si256((const __m256i*)(ptr + 32)); + + const __m256i sh = _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, + 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15); + __m256i p0 = _mm256_shuffle_epi8(t0, sh); + __m256i p1 = _mm256_shuffle_epi8(t1, sh); + __m256i lo = _mm256_permute2x128_si256(p0, p1, 0 + 2*16); + __m256i hi = _mm256_permute2x128_si256(p0, p1, 1 + 3*16); + a = _mm256_unpacklo_epi64(lo, hi); + b = _mm256_unpackhi_epi64(lo, hi); +} + +void fastDepthwiseConv( const int8_t* wptr, + int kernel_h, int kernel_w, + int stride_h, int stride_w, + int dilation_h, int dilation_w, + int pad_t, int pad_l, + const int* biasptr, const float* multptr, + const int8_t* inptr_, + int height, int width, + int* outptr_, + int out_d, int outH, int outW, + int inpZp, int outZp) +{ + const int8_t w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2], + w10 = wptr[3], w11 = wptr[4], w12 = wptr[5], + w20_ = wptr[6], w21_ = wptr[7], w22_ = wptr[8]; + int outW1 = min(outW, (width - dilation_w*(kernel_w - 1) + pad_l)/stride_w); + float mult = multptr[out_d]; + int bias = biasptr[out_d]; + int biasCopy; + + for (int out_i = 0; out_i < outH; out_i++) + { + int in_i = out_i * stride_h - pad_t, out_j = 0; + const int8_t* imgptr0 = inptr_ + in_i*width; + const int8_t* imgptr1 = imgptr0 + dilation_h*width; + const int8_t* imgptr2 = imgptr0 + (dilation_h*2)*width; + int8_t w00 = w00_, w01 = w01_, w02 = w02_; + int8_t w20 = w20_, w21 = w21_, w22 = w22_; + int out; + biasCopy = bias; + if (in_i < 0) + { + biasCopy += inpZp * (w00 + w01 + w02); + w00 = w01 = w02 = 0; + imgptr0 = imgptr1; + } + else if (in_i + dilation_h*(kernel_h-1) >= height) + { + biasCopy += inpZp * (w20 + w21 + w22); + w20 = w21 = w22 = 0; + imgptr2 = imgptr1; + } + int* outptr = outptr_ + out_i*outW; + if (pad_l > 0) + { + out = (int)imgptr0[0]*w01 + (int)imgptr0[dilation_w]*w02 + + (int)imgptr1[0]*w11 + (int)imgptr1[dilation_w]*w12 + + (int)imgptr2[0]*w21 + (int)imgptr2[dilation_w]*w22 + + biasCopy + inpZp*(w00 + w10 + w20); + outptr[0] = std::min(std::max(outZp + (int)std::round(out*mult), -128), 127); + out_j = 1; + } + + if (stride_w == 1 || (stride_w == 2 && dilation_w == 1)) + { + const int VECSZ = 32; + __m256i vw00 = _mm256_set1_epi8(w00), vw01 = _mm256_set1_epi8(w01), vw02 = _mm256_set1_epi8(w02), + vw10 = _mm256_set1_epi8(w10), vw11 = _mm256_set1_epi8(w11), vw12 = _mm256_set1_epi8(w12), + vw20 = _mm256_set1_epi8(w20), vw21 = _mm256_set1_epi8(w21), vw22 = _mm256_set1_epi8(w22); + __m256i vbias = _mm256_set1_epi32(biasCopy), voutzp = _mm256_set1_epi32(outZp), + outmin = _mm256_set1_epi32(-128), outmax = _mm256_set1_epi32(127); + __m256 vmult = _mm256_set1_ps(mult); + __m256i vout0, vout1, vout2, vout3; + + if( stride_w == 1 ) + { + for( ; out_j < outW1; out_j += VECSZ ) + { + if (out_j + VECSZ > outW1) + { + if (out_j <= pad_l) + break; + out_j = outW1 - VECSZ; + } + int in_j = out_j * stride_w - pad_l; + __m256i v00 = _mm256_loadu_si256((const __m256i*)(imgptr0 + in_j)), + v01 = _mm256_loadu_si256((const __m256i*)(imgptr0 + in_j + dilation_w)), + v02 = _mm256_loadu_si256((const __m256i*)(imgptr0 + in_j + dilation_w*2)), + v10 = _mm256_loadu_si256((const __m256i*)(imgptr1 + in_j)), + v11 = _mm256_loadu_si256((const __m256i*)(imgptr1 + in_j + dilation_w)), + v12 = _mm256_loadu_si256((const __m256i*)(imgptr1 + in_j + dilation_w*2)), + v20 = _mm256_loadu_si256((const __m256i*)(imgptr2 + in_j)), + v21 = _mm256_loadu_si256((const __m256i*)(imgptr2 + in_j + dilation_w)), + v22 = _mm256_loadu_si256((const __m256i*)(imgptr2 + in_j + dilation_w*2)); + + vout0 = vout1 = vout2 = vout3 = vbias; + _mm256_expand_mul_add(v00, vw00, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v01, vw01, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v02, vw02, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v10, vw10, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v11, vw11, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v12, vw12, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v20, vw20, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v21, vw21, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v22, vw22, vout0, vout1, vout2, vout3); + + vout0 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout0), vmult))); + vout1 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout1), vmult))); + vout2 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout2), vmult))); + vout3 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout3), vmult))); + + vout0 = _mm256_min_epi32(_mm256_max_epi32(vout0, outmin), outmax); + vout1 = _mm256_min_epi32(_mm256_max_epi32(vout1, outmin), outmax); + vout2 = _mm256_min_epi32(_mm256_max_epi32(vout2, outmin), outmax); + vout3 = _mm256_min_epi32(_mm256_max_epi32(vout3, outmin), outmax); + + _mm256_storeu_si256((__m256i*)(outptr + out_j), vout0); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 8), vout1); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 16), vout2); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 24), vout3); + } + } + else + { + for( ; out_j < outW1; out_j += VECSZ ) + { + if (out_j + VECSZ > outW1) + { + if (out_j <= pad_l) + break; + out_j = outW1 - VECSZ; + } + int in_j = out_j * stride_w - pad_l; + __m256i v00, v01, v02, v10, v11, v12, v20, v21, v22, unused; + _mm256_load_deinterleave(imgptr0 + in_j, v00, v01); + _mm256_load_deinterleave(imgptr0 + in_j + 2, v02, unused); + _mm256_load_deinterleave(imgptr1 + in_j, v10, v11); + _mm256_load_deinterleave(imgptr1 + in_j + 2, v12, unused); + _mm256_load_deinterleave(imgptr2 + in_j, v20, v21); + _mm256_load_deinterleave(imgptr2 + in_j + 2, v22, unused); + + vout0 = vout1 = vout2 = vout3 = vbias; + _mm256_expand_mul_add(v00, vw00, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v01, vw01, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v02, vw02, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v10, vw10, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v11, vw11, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v12, vw12, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v20, vw20, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v21, vw21, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v22, vw22, vout0, vout1, vout2, vout3); + + vout0 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout0), vmult))); + vout1 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout1), vmult))); + vout2 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout2), vmult))); + vout3 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout3), vmult))); + + vout0 = _mm256_min_epi32(_mm256_max_epi32(vout0, outmin), outmax); + vout1 = _mm256_min_epi32(_mm256_max_epi32(vout1, outmin), outmax); + vout2 = _mm256_min_epi32(_mm256_max_epi32(vout2, outmin), outmax); + vout3 = _mm256_min_epi32(_mm256_max_epi32(vout3, outmin), outmax); + + _mm256_storeu_si256((__m256i*)(outptr + out_j), vout0); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 8), vout1); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 16), vout2); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 24), vout3); + } + } + } + + for (; out_j < outW1; out_j++) + { + int in_j = out_j * stride_w - pad_l; + out = (int)imgptr0[in_j]*w00 + (int)imgptr0[in_j + dilation_w]*w01 + (int)imgptr0[in_j + dilation_w*2]*w02 + + (int)imgptr1[in_j]*w10 + (int)imgptr1[in_j + dilation_w]*w11 + (int)imgptr1[in_j + dilation_w*2]*w12 + + (int)imgptr2[in_j]*w20 + (int)imgptr2[in_j + dilation_w]*w21 + (int)imgptr2[in_j + dilation_w*2]*w22 + biasCopy; + outptr[out_j] = std::min(std::max(outZp + (int)std::round(out*mult), -128), 127); + } + + for (; out_j < outW; out_j++ ) + { + int in_j0 = out_j * stride_w - pad_l, in_j1 = in_j0 + dilation_w, in_j2 = in_j0 + dilation_w*2; + int s0 = 1, s1 = 1, s2 = 1; + if (in_j0 >= width) + { + in_j0 = 0; + s0 = 0; + biasCopy += inpZp*(w00 + w10 + w20); + } + if (in_j1 >= width) + { + in_j1 = 0; + s1 = 0; + biasCopy += inpZp*(w01 + w11 + w21); + } + if (in_j2 >= width) + { + in_j2 = 0; + s2 = 0; + biasCopy += inpZp*(w02 + w12 + w22); + } + out = (int)imgptr0[in_j0]*w00*s0 + (int)imgptr0[in_j1]*w01*s1 + (int)imgptr0[in_j2]*w02*s2 + + (int)imgptr1[in_j0]*w10*s0 + (int)imgptr1[in_j1]*w11*s1 + (int)imgptr1[in_j2]*w12*s2 + + (int)imgptr2[in_j0]*w20*s0 + (int)imgptr2[in_j1]*w21*s1 + (int)imgptr2[in_j2]*w22*s2 + biasCopy; + outptr[out_j] = std::min(std::max(outZp + (int)std::round(out*mult), -128), 127); + } + } + _mm256_zeroupper(); +} + +// dst = vec * weights^t + bias +void fastGEMM1T( const int8_t* vec, const int8_t* weights, + size_t wstep, const int* bias, const float* multiplier, + int* dst, int nvecs, int vecsize, int outZp ) +{ + int i = 0; + + for( ; i <= nvecs - 8; i += 8 ) + { + const int8_t* wptr = weights + i*wstep; + __m256i vs0 = _mm256_setzero_si256(), vs1 = _mm256_setzero_si256(), + vs2 = _mm256_setzero_si256(), vs3 = _mm256_setzero_si256(), + vs4 = _mm256_setzero_si256(), vs5 = _mm256_setzero_si256(), + vs6 = _mm256_setzero_si256(), vs7 = _mm256_setzero_si256(); + + __m128i voutzp = _mm_set1_epi32(outZp); + __m128i outmin = _mm_set1_epi32(-128), outmax = _mm_set1_epi32(127); + + for( int k = 0; k < vecsize; k += 32, wptr += 32 ) + { + __m256i v = _mm256_load_si256((const __m256i*)(vec + k)); + + vs0 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)wptr), v, vs0); + vs1 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep)), v, vs1); + vs2 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*2)), v, vs2); + vs3 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*3)), v, vs3); + vs4 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*4)), v, vs4); + vs5 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*5)), v, vs5); + vs6 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*6)), v, vs6); + vs7 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*7)), v, vs7); + } + + __m256i s0 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs0, vs1), _mm256_hadd_epi32(vs2, vs3)); + __m256i s1 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs4, vs5), _mm256_hadd_epi32(vs6, vs7)); + + s0 = _mm256_add_epi32(s0, _mm256_permute2x128_si256(s0, s0, 1)); + s1 = _mm256_add_epi32(s1, _mm256_permute2x128_si256(s1, s1, 1)); + + __m128i t0 = _mm_add_epi32(_mm256_castsi256_si128(s0), _mm_loadu_si128((__m128i*)(bias + i))); + __m128i t1 = _mm_add_epi32(_mm256_castsi256_si128(s1), _mm_loadu_si128((__m128i*)(bias + i + 4))); + + t0 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(t0), _mm_loadu_ps(multiplier + i)))); + t1 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(t1), _mm_loadu_ps(multiplier + i + 4)))); + + t0 = _mm_min_epi32(_mm_max_epi32(t0, outmin), outmax); + t1 = _mm_min_epi32(_mm_max_epi32(t1, outmin), outmax); + + _mm_storeu_si128((__m128i*)(dst + i), t0); + _mm_storeu_si128((__m128i*)(dst + i + 4), t1); + } + + for( ; i < nvecs; i++ ) + { + const int8_t* wptr = weights + i*wstep; + __m256i vs0 = _mm256_setzero_si256(); + + for( int k = 0; k < vecsize; k += 32, wptr += 32 ) + { + __m256i v = _mm256_load_si256((const __m256i*)(vec + k)); + vs0 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)wptr), v, vs0); + } + + __m256i s0 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs0, vs0), vs0); + s0 = _mm256_add_epi32(s0, _mm256_permute2x128_si256(s0, s0, 1)); + int temp = _mm_extract_epi32(_mm256_castsi256_si128(s0), 0); + dst[i] = outZp + (int)std::round((temp + bias[i]) * multiplier[i]); + } + + _mm256_zeroupper(); +} +#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY + +CV_CPU_OPTIMIZATION_NAMESPACE_END +}} // namespace diff --git a/modules/dnn/src/int8layers/pooling_layer.cpp b/modules/dnn/src/int8layers/pooling_layer.cpp new file mode 100644 index 000000000000..20a0486a4625 --- /dev/null +++ b/modules/dnn/src/int8layers/pooling_layer.cpp @@ -0,0 +1,595 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include "opencv2/core/hal/intrin.hpp" + +#include +#include +#include +using std::max; +using std::min; + +namespace cv +{ +namespace dnn +{ + +class PoolingLayerInt8Impl CV_FINAL : public PoolingLayerInt8 +{ +public: + PoolingLayerInt8Impl(const LayerParams& params) + { + computeMaxIdx = false; + globalPooling = false; + isGlobalPooling = std::vector(3, false); + output_zp = params.get("zeropoints"); + input_zp = params.get("input_zeropoint", 0); + multiplier = params.get("multiplier", 1.f); + + hasDynamicShapes = params.get("has_dynamic_shapes", false); + shapesInitialized = !hasDynamicShapes; + + if (params.has("pool") || params.has("kernel_size") || + params.has("kernel_w") || params.has("kernel_h")) + { + String pool = toLowerCase(params.get("pool", "max")); + if (pool == "max") + type = MAX; + else if (pool == "ave") + type = AVE; + else if (pool == "sum") + type = SUM; + else + CV_Error(Error::StsBadArg, "Unknown pooling type \"" + pool + "\""); + + getPoolingKernelParams(params, kernel_size, isGlobalPooling, pads_begin, pads_end, strides, padMode); + globalPooling = isGlobalPooling[0] || isGlobalPooling[1] || isGlobalPooling[2]; + } + else + CV_Error(Error::StsBadArg, "Cannot determine pooling type"); + setParamsFrom(params); + ceilMode = params.get("ceil_mode", true); + spatialScale = params.get("spatial_scale", 1); + avePoolPaddedArea = params.get("ave_pool_padded_area", true); + } + + void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + CV_Assert(!inputs.empty()); + CV_Assert(outputs.size() == 1); + + std::vector inp; + std::vector out; + for (int i = 2; i < inputs[0].dims; i++) { + inp.push_back(inputs[0].size[i]); + out.push_back(outputs[0].size[i]); + } + if (globalPooling) { + std::vector finalKernel; + for (int i = 0; i < inp.size(); i++) { + int idx = isGlobalPooling.size() - inp.size() + i; + finalKernel.push_back(isGlobalPooling[idx] ? inp[i] : kernel_size[idx]); + } + kernel_size = finalKernel; + } + + getConvPoolPaddings(inp, kernel_size, strides, padMode, pads_begin, pads_end); + + if (inputs[0].dims == 3) + { + // Pool1D + kernel_size.assign(1, kernel_size[0]); + strides.assign(1, strides[0]); + pads_begin.assign(1, pads_begin[0]); + pads_end.assign(1, pads_end[0]); + } + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + if (backendId == DNN_BACKEND_OPENCV) + { + if (kernel_size.size() == 3) + return preferableTarget == DNN_TARGET_CPU; + if (kernel_size.size() <= 2) + return true; + else + return false; + } + return false; + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + return activ_int8->blobs.empty(); + } + return false; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + switch (type) + { + case MAX: + { + CV_Assert_N(inputs.size() == 1, outputs.size() == 1); + maxPooling(inputs[0], outputs[0]); + break; + } + case AVE: case SUM: + CV_Assert_N(inputs.size() == 1, outputs.size() == 1); + avePooling(inputs[0], outputs[0]); + break; + default: + CV_Error(Error::StsNotImplemented, "Not implemented"); + break; + } + } + + class PoolingInvoker : public ParallelLoopBody + { + public: + const Mat* src, *rois; + Mat *dst; + int pad_l, pad_t, pad_r, pad_b; + bool avePoolPaddedArea; + int nstripes, inpZp, outZp; + std::vector ofsbuf; + int poolingType; + float multiplier; + float spatialScale; + + std::vector pads_begin, pads_end; + std::vector kernel_size; + std::vector strides; + + PoolingInvoker() : src(0), rois(0), dst(0), pad_l(0), pad_t(0), pad_r(0), pad_b(0), + avePoolPaddedArea(false), nstripes(0), inpZp(0), outZp(0), + poolingType(MAX), multiplier(1), spatialScale(0){} + + static void run(const Mat& src, const Mat& rois, Mat& dst, + std::vector kernel_size, std::vector strides, + std::vector pads_begin, std::vector pads_end, + bool avePoolPaddedArea, int poolingType, float spatialScale, + float multiplier, int inpZp, int outZp, int nstripes) + { + CV_Assert_N( + src.isContinuous(), dst.isContinuous(), + src.type() == CV_8S, src.type() == dst.type(), + src.dims == 3 || src.dims == 4 || src.dims == 5, dst.dims == 3 || dst.dims == 4 || dst.dims == 5, + src.size[0] == dst.size[0], src.size[1] == dst.size[1], rois.empty()); + + PoolingInvoker p; + + bool isPool1D = src.dims == 3; + bool isPool3D = src.dims == 5; + + p.src = &src; + p.rois = &rois; + p.dst = &dst; + + p.kernel_size = kernel_size; + p.strides = strides; + p.pads_begin = pads_begin; + p.pads_end = pads_end; + + p.pad_l = pads_begin.back(); + p.pad_t = isPool1D ? 0 : pads_begin[pads_begin.size() - 2]; + p.pad_r = pads_end.back(); + p.pad_b = isPool1D ? 0 : pads_end[pads_end.size() - 2]; + + p.avePoolPaddedArea = avePoolPaddedArea; + p.nstripes = nstripes; + p.inpZp = inpZp; + p.outZp = outZp; + p.poolingType = poolingType; + p.spatialScale = spatialScale; + p.multiplier = multiplier; + + int height = isPool1D ? 1 : src.size[src.dims - 2]; + int width = src.size[src.dims - 1]; + + int kernel_d = isPool3D ? kernel_size[0] : 1; + int kernel_h = isPool1D ? 1 : kernel_size[kernel_size.size() - 2]; + int kernel_w = kernel_size.back(); + + p.ofsbuf.resize(kernel_d * kernel_h * kernel_w); + for (int i = 0; i < kernel_d; ++i) { + for (int j = 0; j < kernel_h; ++j) { + for (int k = 0; k < kernel_w; ++k) { + p.ofsbuf[i * kernel_h * kernel_w + j * kernel_w + k] = width * height * i + width * j + k; + } + } + } + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + void operator()(const Range& r) const CV_OVERRIDE + { + int channels = dst->size[1]; + + bool isPool3D = src->dims == 5; + bool isPool2D = src->dims == 4; + bool isPool1D = src->dims == 3; + int depth = isPool3D? dst->size[2] : 1; + int height = isPool1D? 1 : dst->size[dst->dims - 2]; + int width = dst->size[dst->dims - 1]; + + int inp_depth = isPool3D? src->size[2] : 1; + int inp_height = isPool1D? 1 : src->size[src->dims - 2]; + int inp_width = src->size[src->dims - 1]; + + size_t total = dst->total(); + size_t stripeSize = (total + nstripes - 1)/nstripes; + size_t stripeStart = r.start*stripeSize; + size_t stripeEnd = std::min(r.end*stripeSize, total); + + int kernel_d = isPool3D? kernel_size[0] : 1; + int kernel_h = isPool1D? 1 : kernel_size[kernel_size.size() - 2]; + int kernel_w = kernel_size.back(); + + int stride_d = isPool3D? strides[0] : 0; + int stride_h = isPool1D? 1 :strides[strides.size() - 2]; + int stride_w = strides.back(); + +#if CV_SIMD128 + const int* ofsptr = (const int*)&ofsbuf[0]; + if (poolingType == MAX && !ofsptr) + CV_Error(Error::StsBadArg, "ofsbuf should be initialized in this mode"); +#endif + + for( size_t ofs0 = stripeStart; ofs0 < stripeEnd; ) + { + size_t ofs = ofs0; + int x0 = (int)(ofs % width); + ofs /= width; + int y0 = (int)(ofs % height); + ofs /= height; + + int d0 = (int)(ofs % depth); + ofs /= depth; + + int c = (int)(ofs % channels); + int n = (int)(ofs / channels); + int ystart, yend; + int dstart = 0, dend = 1; + + const int8_t *srcData = 0; + int pad_d_begin = (pads_begin.size() == 3) ? pads_begin[0] : 0; + dstart = d0 * stride_d - pad_d_begin; + dend = min(dstart + kernel_d, (int)(inp_depth + pads_end[0])); + + ystart = y0 * stride_h - pad_t; + yend = min(ystart + kernel_h, inp_height + pad_b); + srcData = src->ptr(n, c); + + int ddelta = dend - dstart; + dstart = max(dstart, 0); + dend = min(dend, inp_depth); + int ydelta = yend - ystart; + ystart = max(ystart, 0); + yend = min(yend, inp_height); + int8_t *dstData = &dst->ptr(n, c, d0)[y0 * width]; + + int delta = std::min((int)(stripeEnd - ofs0), width - x0); + ofs0 += delta; + int x1 = x0 + delta; + + if( poolingType == MAX ) + for( ; x0 < x1; x0++ ) + { + int xstart = x0 * stride_w - pad_l; + int xend = min(xstart + kernel_w, inp_width); + xstart = max(xstart, 0); + if (xstart >= xend || ystart >= yend) + { + dstData[x0] = (int8_t)outZp; + continue; + } +#if CV_SIMD128 + if( isPool2D && xstart > 0 && x0 + 15 < x1 && (x0 + 15) * stride_w - pad_l + kernel_w < inp_width ) + { + v_int8x16 max_val0 = v_setall_s8(-128); + if( yend - ystart == kernel_h ) + { + const int8_t* srcData1 = srcData + ystart*inp_width + xstart; + if( stride_w == 1 ) + for (int k = 0; k < kernel_w*kernel_h; k++) + { + int index = ofsptr[k]; + v_int8x16 v0 = v_load(srcData1 + index); + max_val0 = v_max(max_val0, v0); + } + else if( stride_w == 2 ) + for (int k = 0; k < kernel_w*kernel_h; k++) + { + int index = ofsptr[k]; + v_int8x16 v0, dummy; + v_load_deinterleave(srcData1 + index, v0, dummy); + max_val0 = v_max(max_val0, v0); + } + else + for (int k = 0; k < kernel_w*kernel_h; k++) + { + int index = ofsptr[k]; + v_int8x16 v0(srcData1[index], srcData1[index + stride_w], + srcData1[index + stride_w*2], srcData1[index + stride_w*3], + srcData1[index + stride_w*4], srcData1[index + stride_w*5], + srcData1[index + stride_w*6], srcData1[index + stride_w*7], + srcData1[index + stride_w*8], srcData1[index + stride_w*9], + srcData1[index + stride_w*10], srcData1[index + stride_w*11], + srcData1[index + stride_w*12], srcData1[index + stride_w*13], + srcData1[index + stride_w*14], srcData1[index + stride_w*15]); + max_val0 = v_max(max_val0, v0); + } + } + else + { + for (int y = ystart; y < yend; ++y) + { + for (int x = xstart; x < xend; ++x) + { + const int index = y * inp_width + x; + v_int8x16 v0(srcData[index], srcData[index + stride_w], + srcData[index + stride_w*2], srcData[index + stride_w*3], + srcData[index + stride_w*4], srcData[index + stride_w*5], + srcData[index + stride_w*6], srcData[index + stride_w*7], + srcData[index + stride_w*8], srcData[index + stride_w*9], + srcData[index + stride_w*10], srcData[index + stride_w*11], + srcData[index + stride_w*12], srcData[index + stride_w*13], + srcData[index + stride_w*14], srcData[index + stride_w*15]); + max_val0 = v_max(max_val0, v0); + } + } + } + v_store(dstData + x0, max_val0); + x0 += 15; + } + else +#else + CV_UNUSED(isPool2D); +#endif + if( isPool1D ) + { + const int8_t* first = srcData + xstart; + const int8_t* last = srcData + xend; + const int8_t* max_elem = std::max_element(first, last); + if (max_elem != last) + dstData[x0] = *max_elem; + } + else + { + int8_t max_val = -128; + for (int d = dstart; d < dend; ++d) { + for (int y = ystart; y < yend; ++y) { + for (int x = xstart; x < xend; ++x) { + const int index = d * inp_width * inp_height + y * inp_width + x; + int8_t val = srcData[index]; + max_val = std::max(max_val, val); + } + } + } + dstData[x0] = max_val; + } + } + else if (poolingType == AVE || poolingType == SUM) + { + for( ; x0 < x1; ++x0) + { + int xstart = x0 * stride_w - pad_l; + int xend = min(xstart + kernel_w, inp_width + pad_r); + int xdelta = xend - xstart; + xstart = max(xstart, 0); + xend = min(xend, inp_width); + + int real_kernel_area = (dend - dstart) * (yend - ystart) * (xend - xstart); + int padded_kernel_area = xdelta * ydelta * ddelta; + int kernel_area = avePoolPaddedArea ? padded_kernel_area : real_kernel_area; + + int bias = (avePoolPaddedArea ? (padded_kernel_area - real_kernel_area) * inpZp : 0) + - (inpZp * kernel_area); + float inv_kernel_area = poolingType == AVE ? multiplier / kernel_area : multiplier; +#if CV_SIMD128 + if( isPool2D && xstart > 0 && x0 + 15 < x1 && (x0 + 15) * stride_w - pad_l + kernel_w < inp_width ) + { + v_int32x4 sum_val0 = v_setall_s32(bias), sum_val1 = v_setall_s32(bias), + sum_val2 = v_setall_s32(bias), sum_val3 = v_setall_s32(bias), + voutzp = v_setall_s32(outZp); + v_float32x4 ikarea = v_setall_f32(inv_kernel_area); + + for (int y = ystart; y < yend; ++y) + { + for (int x = xstart; x < xend; ++x) + { + const int index = y * inp_width + x; + v_int32x4 v0((int)srcData[index], (int)srcData[index + stride_w], + (int)srcData[index + stride_w*2], (int)srcData[index + stride_w*3]); + v_int32x4 v1((int)srcData[index + stride_w*4], (int)srcData[index + stride_w*5], + (int)srcData[index + stride_w*6], (int)srcData[index + stride_w*7]); + v_int32x4 v2((int)srcData[index + stride_w*8], (int)srcData[index + stride_w*9], + (int)srcData[index + stride_w*10], (int)srcData[index + stride_w*11]); + v_int32x4 v3((int)srcData[index + stride_w*12], (int)srcData[index + stride_w*13], + (int)srcData[index + stride_w*14], (int)srcData[index + stride_w*15]); + sum_val0 += v0; + sum_val1 += v1; + sum_val2 += v2; + sum_val3 += v3; + } + } + + sum_val0 = v_round(v_cvt_f32(sum_val0)*ikarea) + voutzp; + sum_val1 = v_round(v_cvt_f32(sum_val1)*ikarea) + voutzp; + sum_val2 = v_round(v_cvt_f32(sum_val2)*ikarea) + voutzp; + sum_val3 = v_round(v_cvt_f32(sum_val3)*ikarea) + voutzp; + + v_store(dstData + x0, v_pack(v_pack(sum_val0, sum_val1), v_pack(sum_val2, sum_val3))); + x0 += 15; + } + else +#endif + if( isPool1D ) + { + const int8_t* first = srcData + xstart; + const int8_t* last = srcData + xend; + int sum_val = bias + std::accumulate(first, last, 0); + dstData[x0] = saturate_cast(outZp + std::round(sum_val*inv_kernel_area)); + } + else + { + int sum_val = bias; + for (int d = dstart; d < dend; ++d) { + for (int y = ystart; y < yend; ++y) { + for (int x = xstart; x < xend; ++x) { + const int index = d * inp_width * inp_height + y * inp_width + x; + int8_t val = srcData[index]; + sum_val += (int)val; + } + } + } + dstData[x0] = saturate_cast(outZp + std::round(sum_val*inv_kernel_area)); + } + } + } + } + } + }; + + void maxPooling(Mat &src, Mat &dst) + { + const int nstripes = getNumThreads(); + Mat rois; + PoolingInvoker::run(src, rois, dst, kernel_size, strides, pads_begin, pads_end, avePoolPaddedArea, type, + spatialScale, multiplier, input_zp, output_zp, nstripes); + } + + void avePooling(Mat &src, Mat &dst) + { + const int nstripes = getNumThreads(); + Mat rois; + PoolingInvoker::run(src, rois, dst, kernel_size, strides, pads_begin, pads_end, avePoolPaddedArea, type, + spatialScale, multiplier, input_zp, output_zp, nstripes); + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() != 0); + + bool isPool1D = inputs[0].size() == 3; + std::vector inpShape(inputs[0].begin() + 2, inputs[0].end()); + std::vector outShape(inputs[0].begin(), inputs[0].begin() + 2); + + std::vector local_kernel; + if (globalPooling) { + for (int i = 0; i < inpShape.size(); i++) { + int idx = isGlobalPooling.size() - inpShape.size() + i; + local_kernel.push_back(isGlobalPooling[idx] ? inpShape[i] : kernel_size[idx]); + } + } else { + local_kernel = kernel_size; + } + + if (hasDynamicShapes && !shapesInitialized) + { + //Just copy input shapes for width and height to prevent errors on loading stage + for (int i = 0; i < inpShape.size(); i++) + outShape.push_back(inpShape[i]); + } + else if (padMode.empty()) + { + int addedDims = isPool1D? inpShape.size() : local_kernel.size(); + for (int i = 0; i < addedDims; i++) { + float dst = (float) (inpShape[i] + pads_begin[i] + pads_end[i] - local_kernel[i]) / strides[i]; + outShape.push_back(1 + (ceilMode ? ceil(dst) : floor(dst))); + } + + // If we have padding, ensure that the last pooling starts strictly + // inside the image (instead of at the padding); otherwise clip the last. + for (int i = 0; i < addedDims; i++) { + if (pads_end[i] && (outShape[2 + i] - 1) * strides[i] >= inpShape[i] + pads_end[i]) { + --outShape[2 + i]; + CV_Assert((outShape[2 + i] - 1) * strides[i] < inpShape[i] + pads_end[i]); + } + } + } + else { + getConvPoolOutParams(inpShape, local_kernel, strides, padMode, + std::vector(local_kernel.size(), 1), outShape); + } + + outputs.assign(1, outShape); + return false; + } + + bool updateMemoryShapes(const std::vector &inputs) CV_OVERRIDE + { + int dims = inputs[0].size(); + CV_Assert(inputs[0][dims - 1] > 0 && inputs[0][dims - 2] > 0); + shapesInitialized = true; + return true; + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(inputs); // suppress unused variable warning + long flops = 0; + bool isPool1D = inputs[0].size() == 3; + size_t karea = std::accumulate(kernel_size.begin(), isPool1D? kernel_size.begin() + 1 : kernel_size.end(), + 1, std::multiplies()); + for(int i = 0; i < outputs.size(); i++) + { + if (type == MAX) + { + if (i%2 == 0) + flops += total(outputs[i])*karea; + } + else + { + flops += total(outputs[i])*(karea + 1); + } + } + return flops; + } +private: + enum Type + { + MAX, + AVE, + STOCHASTIC, + SUM, + ROI, // RoI pooling, https://arxiv.org/pdf/1504.08083.pdf + PSROI // Position-sensitive RoI pooling, https://arxiv.org/pdf/1605.06409.pdf + }; + bool hasDynamicShapes; + bool shapesInitialized; + float multiplier; +}; + +Ptr PoolingLayerInt8::create(const LayerParams& params) +{ + return Ptr(new PoolingLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/quantize_dequantize_layer.cpp b/modules/dnn/src/int8layers/quantize_dequantize_layer.cpp new file mode 100644 index 000000000000..2ddb76a0e80d --- /dev/null +++ b/modules/dnn/src/int8layers/quantize_dequantize_layer.cpp @@ -0,0 +1,157 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +namespace cv +{ +namespace dnn +{ + +class QuantizeLayerImpl CV_FINAL : public QuantizeLayer +{ +public: + QuantizeLayerImpl(const LayerParams& params) + { + scale = params.get("scales", 1.0f); + zeropoint = params.get("zeropoints", 0); + setParamsFrom(params); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() == 1); + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + } + +#ifdef HAVE_OPENCL + bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_) + { + std::vector inputs, outputs; + inputs_.getUMatVector(inputs); + outputs_.getUMatVector(outputs); + + if (inputs_.depth() == CV_16S) + { + UMat inputFp32(shape(inputs[0]), CV_32F); + convertFp16(inputs[0], inputFp32); + inputFp32.copyTo(inputs[0]); + } + + inputs[0].convertTo(outputs[0], CV_8S, 1.f/scale, zeropoint); + return true; + } +#endif + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget), + forward_ocl(inputs_arr, outputs_arr, internals_arr)) + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + inputs[0].convertTo(outputs[0], CV_8S, 1.f/scale, zeropoint); + } +}; + +class DequantizeLayerImpl CV_FINAL : public DequantizeLayer +{ +public: + DequantizeLayerImpl(const LayerParams& params) + { + scale = params.get("scales", 1.0f); + zeropoint = params.get("zeropoints", 0); + setParamsFrom(params); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() == 1); + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + } + +#ifdef HAVE_OPENCL + bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_) + { + std::vector inputs, outputs; + inputs_.getUMatVector(inputs); + outputs_.getUMatVector(outputs); + + UMat outputFp32(shape(outputs[0]), CV_32F); + inputs[0].convertTo(outputFp32, CV_32F, scale, -(scale*zeropoint)); + + if (outputs_.depth() == CV_16S) + convertFp16(outputFp32, outputs[0]); + else + outputFp32.copyTo(outputs[0]); + return true; + } +#endif + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget), + forward_ocl(inputs_arr, outputs_arr, internals_arr)) + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + inputs[0].convertTo(outputs[0], CV_32F, scale, -(scale*zeropoint)); + } +}; + +Ptr QuantizeLayer::create(const LayerParams& params) +{ + return Ptr(new QuantizeLayerImpl(params)); +} + +Ptr DequantizeLayer::create(const LayerParams& params) +{ + return Ptr(new DequantizeLayerImpl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/scale_layer.cpp b/modules/dnn/src/int8layers/scale_layer.cpp new file mode 100644 index 000000000000..d7f676d047ab --- /dev/null +++ b/modules/dnn/src/int8layers/scale_layer.cpp @@ -0,0 +1,211 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include +#include + +namespace cv +{ +namespace dnn +{ + +class ScaleLayerInt8Impl CV_FINAL : public ScaleLayerInt8 +{ +public: + Mat weights, bias; + ScaleLayerInt8Impl(const LayerParams& params) + { + setParamsFrom(params); + hasBias = params.get("bias_term", false); + axis = params.get("axis", 1); + hasWeights = false; + + output_sc = params.get("scales"); + output_zp = params.get("zeropoints"); + + DictValue inpSc = params.get("input_scales"); + DictValue inpZp = params.get("input_zeropoints"); + + for (int i = 0; i < inpSc.size(); i++) + { + inp_sc.push_back(inpSc.get(i)); + inp_zp.push_back(inpZp.get(i)); + } + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + outputs.assign(1, inputs[0]); + return true; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays) CV_OVERRIDE + { + std::vector inputs; + inputs_arr.getMatVector(inputs); + hasWeights = blobs.size() == 2 || (blobs.size() <= 1 && !hasBias); + CV_Assert((inputs.size() == 2 && blobs.empty()) || blobs.size() == (int)hasWeights + (int)hasBias); + + if (!blobs.empty()) + { + Mat w = hasWeights ? blobs[0] : Mat::ones(blobs[0].size(), CV_32F); + Mat b = hasBias ? blobs.back() : Mat::zeros(blobs.back().size(), CV_32F); + + w = w.reshape(1, 1); + b = b.reshape(1, 1); + + w.convertTo(weights, CV_32F, inp_sc[0]/output_sc); + addWeighted(b, 1.0/output_sc, weights, -inp_zp[0], output_zp, bias, CV_32F); + } + else + { + // initialized during forward() + weights = Mat(); bias = Mat(); + } + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + return activ_int8->blobs.empty(); + } + return false; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + Mat &inpBlob = inputs[0]; + Mat &outBlob = outputs[0]; + + if (blobs.empty()) + { + CV_Assert(inp_sc.size() == 2 && inp_zp.size() == 2); + Mat inp_dequantized, w, b; + inputs[1].reshape(1, 1).convertTo(inp_dequantized, CV_32F, inp_sc[1], -(inp_sc[1]*inp_zp[1])); + w = hasWeights ? inp_dequantized : Mat::ones(inp_dequantized.size(), CV_32F); + b = hasBias ? inp_dequantized : Mat::zeros(inp_dequantized.size(), CV_32F); + + w.convertTo(weights, CV_32F, inp_sc[0]/output_sc); + addWeighted(b, 1.0/output_sc, weights, -inp_zp[0], output_zp, bias, CV_32F); + } + + MatShape inpShape = shape(inpBlob); + const int numWeights = weights.total(); + CV_Assert(numWeights != 0); + CV_CheckEQ(weights.total(), bias.total(), "Incompatible weights/bias blobs"); + + int endAxis; + for (endAxis = axis + 1; endAxis <= inpBlob.dims; ++endAxis) + { + if (total(inpShape, axis, endAxis) == numWeights) + break; + } + CV_Assert(total(inpShape, axis, endAxis) == numWeights); + CV_CheckTypeEQ(inpBlob.type(), CV_8SC1, ""); CV_CheckTypeEQ(outBlob.type(), CV_8SC1, ""); + + int numSlices = total(inpShape, 0, axis); + int8_t* inpData = (int8_t*)inpBlob.data; + int8_t* outData = (int8_t*)outBlob.data; + + if (endAxis != inpBlob.dims) + { + float* weightsData = (float*)weights.data; + float* biasesData = (float*)bias.data; + int spatialSize = total(inpShape, endAxis); // spatialSize != 1 + for (int i = 0; i < numSlices; ++i) + { + for (int j = 0; j < numWeights; ++j) + { + float w = weightsData[j]; + float b = biasesData[j]; + Mat inpSlice(1, spatialSize, CV_8S, inpData); + Mat outSlice(1, spatialSize, CV_8S, outData); + inpSlice.convertTo(outSlice, CV_8S, w, b); + inpData += spatialSize; + outData += spatialSize; + } + } + } + else + { + for (int i = 0; i < numSlices; ++i) + { + Mat inpSlice(1, numWeights, CV_8S, inpData); + Mat outSlice(1, numWeights, CV_8S, outData); + + multiply(inpSlice, weights, outSlice, 1.0, CV_8S); + add(outSlice, bias, outSlice, Mat(), CV_8S); + + inpData += numWeights; + outData += numWeights; + } + } + } + + void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE + { + scale = (hasWeights && !blobs.empty()) ? blobs[0] : Mat(); + shift = (hasBias && !blobs.empty()) ? blobs.back() : Mat(); + } + + void getScaleZeropoint(float& scale, int& zeropoint) const CV_OVERRIDE + { + scale = output_sc; + zeropoint = output_zp; + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(outputs); // suppress unused variable warning + long flops = 0; + for(int i = 0; i < inputs.size(); i++) + { + flops += 2*total(inputs[i]); + } + return flops; + } + +private: + bool hasWeights; + std::vector inp_sc; + std::vector inp_zp; +}; + + +Ptr ScaleLayerInt8::create(const LayerParams& params) +{ + return Ptr(new ScaleLayerInt8Impl(params)); +} + +Ptr ShiftLayerInt8::create(const LayerParams& params) +{ + LayerParams scaleParams = params; + scaleParams.type = "ScaleInt8"; + scaleParams.set("bias_term", true); + scaleParams.set("axis", 0); + return Ptr(new ScaleLayerInt8Impl(scaleParams)); +} + +} // namespace dnn +} // namespace cv diff --git a/modules/dnn/src/int8layers/softmax_layer.cpp b/modules/dnn/src/int8layers/softmax_layer.cpp new file mode 100644 index 000000000000..7e3c82bc21ab --- /dev/null +++ b/modules/dnn/src/int8layers/softmax_layer.cpp @@ -0,0 +1,176 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include +#include + +namespace cv +{ +namespace dnn +{ + +class SoftMaxLayerInt8Impl CV_FINAL : public SoftmaxLayerInt8 +{ +public: + + SoftMaxLayerInt8Impl(const LayerParams& params) + { + axisRaw = params.get("axis", 1); + logSoftMax = params.get("log_softmax", false); + output_sc = params.get("scales"); + output_zp = params.get("zeropoints"); + setParamsFrom(params); + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + bool inplace = Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + MatShape shape = inputs[0]; + int cAxis = normalize_axis(axisRaw, shape.size()); + shape[cAxis] = 1; + internals.assign(1, shape); + return inplace; + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + Ptr dequantize_layer = top.dynamicCast(); + return !dequantize_layer.empty() && preferableTarget != DNN_TARGET_OPENCL_FP16; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs, internals; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + internals_arr.getMatVector(internals); + + const Mat &src = inputs[0]; + Mat &dst = outputs[0]; + + int axis = normalize_axis(axisRaw, src.dims); + size_t outerSize = src.total(0, axis), channels = src.size[axis], + innerSize = src.total(axis + 1); + + CV_Assert(src.type() == CV_8S && (dst.type() == CV_8S || dst.type() == CV_32F)); + CV_Assert(src.isContinuous() && dst.isContinuous()); + + size_t outerStep = src.total(axis); + size_t cnStep = src.total(axis + 1); + const int8_t *srcPtr = src.ptr(); + const float *expPtr = blobs[0].ptr(); + + if (dst.type() == CV_32F) + { + float *dstPtr = dst.ptr(); + for (size_t outerDim = 0; outerDim < outerSize; outerDim++) + { + size_t srcOffset = outerDim * outerStep; + std::vector expSum(innerSize, 0.f); + + // sum exp along axis + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + expSum[i] += expPtr[srcPtr[offset + i] + 128]; + } + + // divide by computed sum + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + dstPtr[offset + i] = expPtr[srcPtr[offset + i] + 128]/expSum[i]; + } + + if (logSoftMax) + { + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + dstPtr[offset + i] = log(dstPtr[offset + i]); + } + } + } + } + else + { + const float inv_scale = 1.f/output_sc; + int8_t *dstPtr = dst.ptr(); + for (size_t outerDim = 0; outerDim < outerSize; outerDim++) + { + size_t srcOffset = outerDim * outerStep; + std::vector expSum(innerSize, 0.f); + + // sum exp along axis + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + expSum[i] += expPtr[srcPtr[offset + i] + 128]; + } + + // divide by computed sum and quantize to int8 + if (logSoftMax) + { + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + dstPtr[offset + i] = saturate_cast(output_zp + std::round(inv_scale*log(expPtr[srcPtr[offset + i] + 128]/expSum[i]))); + } + } + else + { + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + dstPtr[offset + i] = saturate_cast(output_zp + std::round(inv_scale*(expPtr[srcPtr[offset + i] + 128]/expSum[i]))); + } + } + } + } + } + + int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(outputs); // suppress unused variable warning + int64 flops = 0; + + for (int i = 0; i < inputs.size(); i++) + { + flops += 4*total(inputs[i]); + } + + return flops; + } + + int axisRaw; +}; + +Ptr SoftmaxLayerInt8::create(const LayerParams& params) +{ + return Ptr(new SoftMaxLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index e28c964689b9..49804c5c13e9 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -409,6 +409,18 @@ class BatchNormLayerImpl CV_FINAL : public BatchNormLayer } #endif // HAVE_DNN_NGRAPH + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + params.set("input_scale", scales[0][0]); + params.set("input_zeropoint", zeropoints[0][0]); + + params.blobs.clear(); + params.blobs.push_back(origin_weights); + params.blobs.push_back(origin_bias); + return true; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/blank_layer.cpp b/modules/dnn/src/layers/blank_layer.cpp index 5f93b458869d..59548a9c0c51 100644 --- a/modules/dnn/src/layers/blank_layer.cpp +++ b/modules/dnn/src/layers/blank_layer.cpp @@ -166,6 +166,11 @@ class BlankLayerImpl CV_FINAL : public BlankLayer } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } }; Ptr BlankLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/layers/concat_layer.cpp b/modules/dnn/src/layers/concat_layer.cpp index a950c56167fd..536114fcd772 100644 --- a/modules/dnn/src/layers/concat_layer.cpp +++ b/modules/dnn/src/layers/concat_layer.cpp @@ -70,6 +70,7 @@ class ConcatLayerImpl CV_FINAL : public ConcatLayer setParamsFrom(params); axis = params.get("axis", 1); padding = params.get("padding", false); + paddingValue = params.get("padding_value", 0); } virtual bool getMemoryShapes(const std::vector &inputs, @@ -119,13 +120,14 @@ class ConcatLayerImpl CV_FINAL : public ConcatLayer (backendId == DNN_BACKEND_VKCOM && haveVulkan() && !padding); } + template class ChannelConcatInvoker : public ParallelLoopBody { public: std::vector* inputs; Mat* output; int nstripes; - std::vector chptrs; + std::vector chptrs; static void run(std::vector& inputs, Mat& output, int nstripes) { @@ -139,14 +141,14 @@ class ConcatLayerImpl CV_FINAL : public ConcatLayer for( i = 0; i < ninputs; i++ ) { Mat& inp = inputs[i]; - CV_Assert( inp.isContinuous() && (inp.type() == CV_32F || inp.type() == CV_16S) && + CV_Assert( inp.isContinuous() && (inp.type() == CV_32F || inp.type() == CV_16S || inp.type() == CV_8S) && inp.dims == 4 && inp.size[0] == output.size[0] && inp.size[2] == output.size[2] && inp.size[3] == output.size[3] ); nchannels += inp.size[1]; } CV_Assert( nchannels == output.size[1] ); - CV_Assert( output.isContinuous() && (output.type() == CV_32F || output.type() == CV_16S) ); + CV_Assert( output.isContinuous() && (output.type() == CV_32F || output.type() == CV_16S || output.type() == CV_8S) ); cc.chptrs.resize(nchannels*batchsz); @@ -157,7 +159,7 @@ class ConcatLayerImpl CV_FINAL : public ConcatLayer for( int j = 0; j < batchsz; j++ ) for( int k = 0; k < inp.size[1]; k++ ) { - const float* ptr = inp.ptr(j, k); + const T* ptr = inp.ptr(j, k); cc.chptrs[ofs + j*nchannels + k] = ptr; } ofs += inp.size[1]; @@ -176,8 +178,8 @@ class ConcatLayerImpl CV_FINAL : public ConcatLayer size_t stripeSize = (total + nstripes - 1)/nstripes; size_t stripeStart = r.start*stripeSize; size_t stripeEnd = std::min(total, r.end*stripeSize); - const float** ptrs = (const float**)&chptrs[0]; - float* outptr = output->ptr(); + const T** ptrs = (const T**)&chptrs[0]; + T* outptr = output->ptr(); size_t blockSize0 = 1 << 16; for( size_t ofs0 = stripeStart; ofs0 < stripeEnd; ) @@ -248,7 +250,8 @@ class ConcatLayerImpl CV_FINAL : public ConcatLayer CV_TRACE_FUNCTION(); CV_TRACE_ARG_VALUE(name, "name", name.c_str()); - CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget), + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget) && + inputs_arr.depth() != CV_8S, forward_ocl(inputs_arr, outputs_arr, internals_arr)) std::vector inputs, outputs; @@ -259,12 +262,15 @@ class ConcatLayerImpl CV_FINAL : public ConcatLayer Mat& outMat = outputs[0]; if (padding) - outMat.setTo(0); + outMat.setTo(paddingValue); if( cAxis == 1 && outMat.dims == 4 && !padding) { int nstripes = getNumThreads(); - ChannelConcatInvoker::run(inputs, outMat, nstripes); + if (outMat.type() == CV_8S) + ChannelConcatInvoker::run(inputs, outMat, nstripes); + else + ChannelConcatInvoker::run(inputs, outMat, nstripes); } else { @@ -394,6 +400,14 @@ class ConcatLayerImpl CV_FINAL : public ConcatLayer return Ptr(new InfEngineNgraphNode(concat)); } #endif // HAVE_DNN_NGRAPH + + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + if (padding) + params.set("padding_value", zeropoints[1][0]); + return true; + } }; Ptr ConcatLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/layers/const_layer.cpp b/modules/dnn/src/layers/const_layer.cpp index bbea3e3f2c72..18f190b36b9a 100644 --- a/modules/dnn/src/layers/const_layer.cpp +++ b/modules/dnn/src/layers/const_layer.cpp @@ -112,6 +112,15 @@ class ConstLayerImpl CV_FINAL : public ConstLayer } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + Mat quantizedBlob; + blobs[0].convertTo(quantizedBlob, CV_8S, 1.f/scales[1][0], zeropoints[1][0]); + params.blobs.clear(); + params.blobs.push_back(quantizedBlob); + return true; + } }; Ptr ConstLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index 68c543be2477..ec2904ee69be 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -2083,6 +2083,48 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + // References - https://arxiv.org/pdf/1712.05877.pdf + + // Quantized convolution with variable weights is not supported. + if (blobs.empty()) + return false; + + float inputScale = scales[0][0], outputScale = scales[1][0]; + int inputZp = zeropoints[0][0]; + params.set("input_zeropoint", inputZp); + + Mat weightsQuantized(weightsMat.rows, weightsMat.cols, CV_8S); + Mat biasQuantized(1, numOutput, CV_32S); + Mat outputMultiplier(1, numOutput, CV_32F); + double realMin, realMax, weightsScale; + + for( int i = 0; i < numOutput; i++ ) + { + // Quantize weights + cv::minMaxIdx(weightsMat.row(i), &realMin, &realMax); + realMin = std::min(realMin, 0.0); + realMax = std::max(realMax, 0.0); + weightsScale = (realMax == realMin) ? 1.0 : std::max(-realMin, realMax)/127; + weightsMat.row(i).convertTo(weightsQuantized.row(i), CV_8S, 1.f/weightsScale); + + // Quantize biases + float biasScale = inputScale * weightsScale; + biasQuantized.at(i) = (int)std::round(biasvec[i]/biasScale) - inputZp*(cv::sum(weightsQuantized.row(i))[0]); + + // Store multiplier + outputMultiplier.at(i) = biasScale / outputScale; + } + + params.blobs.clear(); + params.blobs.push_back(weightsQuantized.reshape(1, shape(blobs[0]))); + params.blobs.push_back(biasQuantized); + params.blobs.push_back(outputMultiplier); + return true; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 9bb5be342f76..6dc1813c8bee 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -255,6 +255,12 @@ class ElementWiseLayer : public Func::Layer } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return func.tryQuantize(scales, zeropoints, params); + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { @@ -288,6 +294,8 @@ struct BaseFunctor bool tryFuse(Ptr&) { return false; } void getScaleShift(Mat&, Mat&) const {} + + bool tryQuantize(const std::vector>&, const std::vector>&, LayerParams&) { return false; } }; struct ReLUFunctor : public BaseFunctor @@ -436,6 +444,29 @@ struct ReLUFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + if (slope != 0.f) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = x >= 0.f ? x : slope*x; + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + } + return true; + } + int64 getFLOPSPerElement() const { return 1; } }; @@ -559,6 +590,12 @@ struct ReLU6Functor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + return true; + } + int64 getFLOPSPerElement() const { return 2; } }; @@ -651,6 +688,26 @@ struct TanHFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = tanh(x); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 1; } }; @@ -743,6 +800,26 @@ struct SwishFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = x / (1.0f + exp(-x)); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 3; } }; @@ -848,6 +925,28 @@ struct MishFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float eX = exp(x); + float n = (eX + 2) * eX; + float y = (x * n) / (n + 2); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 3; } }; @@ -940,6 +1039,26 @@ struct SigmoidFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = 1.f/(1.f + exp(-x)); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 3; } }; @@ -1032,6 +1151,26 @@ struct ELUFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = x >= 0.f ? x : exp(x) - 1; + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 2; } }; @@ -1130,6 +1269,26 @@ struct AbsValFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = abs(x); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 1; } }; @@ -1223,6 +1382,26 @@ struct BNLLFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = x > 0 ? x + log(1. + exp(-x)) : log(1. + exp(x)); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 5; } }; diff --git a/modules/dnn/src/layers/eltwise_layer.cpp b/modules/dnn/src/layers/eltwise_layer.cpp index a337c48d9e69..860560213d92 100644 --- a/modules/dnn/src/layers/eltwise_layer.cpp +++ b/modules/dnn/src/layers/eltwise_layer.cpp @@ -864,6 +864,37 @@ class EltwiseLayerImpl CV_FINAL : public EltwiseLayer } #endif // HAVE_DNN_NGRAPH + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + if (op == SUM) + { + std::vector newCoeffs; + float offset = zeropoints[1][0]; + float out_sc = scales[1][0]; + for (int i = 0; i < scales[0].size(); i++) + { + float coeff = coeffs.empty() ? 1.f : coeffs[i]; + float newcoeff = (scales[0][i] * coeff) / out_sc; + newCoeffs.push_back(newcoeff); + offset -= (newcoeff * zeropoints[0][i]); + } + params.set("coeff", DictValue::arrayReal(newCoeffs.data(), newCoeffs.size())); + params.set("offset", offset); + return true; + } + else if (op == PROD) + { + std::vector newCoeffs = scales[0]; + newCoeffs[0] /= scales[1][0]; + params.set("coeff", DictValue::arrayReal(newCoeffs.data(), newCoeffs.size())); + params.set("offset", zeropoints[1][0]); + params.set("input_zeropoints", DictValue::arrayInt(zeropoints[0].data(), zeropoints[0].size())); + return true; + } + return op == MAX; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/flatten_layer.cpp b/modules/dnn/src/layers/flatten_layer.cpp index 7cf01a14fa33..8ff862fab030 100644 --- a/modules/dnn/src/layers/flatten_layer.cpp +++ b/modules/dnn/src/layers/flatten_layer.cpp @@ -227,6 +227,11 @@ virtual Ptr initNgraph(const std::vector >& inp } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } int _startAxis; int _endAxis; diff --git a/modules/dnn/src/layers/fully_connected_layer.cpp b/modules/dnn/src/layers/fully_connected_layer.cpp index 529f3c04fdef..28ea7f347fef 100644 --- a/modules/dnn/src/layers/fully_connected_layer.cpp +++ b/modules/dnn/src/layers/fully_connected_layer.cpp @@ -618,6 +618,45 @@ class FullyConnectedLayerImpl CV_FINAL : public InnerProductLayer } #endif // HAVE_DNN_NGRAPH + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + if (blobs.empty()) + return false; + + int numOutput = blobs[0].size[0]; + float inputScale = scales[0][0], outputScale = scales[1][0]; + int inputZp = zeropoints[0][0]; + + Mat weightsQuantized(weightsMat.rows, weightsMat.cols, CV_8S); + Mat biasQuantized(1, numOutput, CV_32S); + Mat outputMultiplier(1, numOutput, CV_32F); + + double realMin, realMax, weightsScale; + for( int i = 0; i < numOutput; i++ ) + { + // Quantize weights + cv::minMaxIdx(weightsMat.row(i), &realMin, &realMax); + realMin = std::min(realMin, 0.0); + realMax = std::max(realMax, 0.0); + weightsScale = (realMax == realMin) ? 1.0 : std::max(-realMin, realMax)/127; + weightsMat.row(i).convertTo(weightsQuantized.row(i), CV_8S, 1.f/weightsScale); + + // Quantize biases + float biasScale = inputScale * weightsScale; + biasQuantized.at(i) = (int)std::round(biasMat.at(i)/biasScale) - inputZp*(cv::sum(weightsQuantized.row(i))[0]); + + // Store multiplier + outputMultiplier.at(i) = biasScale / outputScale; + } + + params.blobs.clear(); + params.blobs.push_back(weightsQuantized.reshape(1, shape(blobs[0]))); + params.blobs.push_back(biasQuantized); + params.blobs.push_back(outputMultiplier); + return true; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/padding_layer.cpp b/modules/dnn/src/layers/padding_layer.cpp index d18256879580..c1979ce701ac 100644 --- a/modules/dnn/src/layers/padding_layer.cpp +++ b/modules/dnn/src/layers/padding_layer.cpp @@ -134,6 +134,8 @@ class PaddingLayerImpl CV_FINAL : public PaddingLayer cv::convertFp16(paddingValue_fp32, paddingValue_fp16); outputs[0].setTo(paddingValue_fp16[0]); } + else if (inputs_arr.depth() == CV_8S) + outputs[0].setTo(saturate_cast(paddingValue)); else outputs[0].setTo(paddingValue); inputs[0].copyTo(outputs[0](dstRanges)); @@ -264,6 +266,16 @@ class PaddingLayerImpl CV_FINAL : public PaddingLayer } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + float outputScale = scales[1][0]; + int outputZp = zeropoints[1][0]; + float padValue = outputZp + std::round(params.get("value", 0)/outputScale); + params.set("value", padValue); + return true; + } + private: std::vector > paddings; // Pairs pad before, pad after. std::vector dstRanges; diff --git a/modules/dnn/src/layers/permute_layer.cpp b/modules/dnn/src/layers/permute_layer.cpp index c525c3f82f78..77c2469c050f 100644 --- a/modules/dnn/src/layers/permute_layer.cpp +++ b/modules/dnn/src/layers/permute_layer.cpp @@ -194,6 +194,7 @@ class PermuteLayerImpl CV_FINAL : public PermuteLayer #endif } + template class PermuteInvoker : public ParallelLoopBody { public: @@ -229,7 +230,7 @@ class PermuteLayerImpl CV_FINAL : public PermuteLayer size_t stripeStart = r.start*stripeSize; size_t stripeEnd = std::min(r.end*stripeSize, orows); - const size_t esz = sizeof(float); + const size_t esz = sizeof(T); size_t ostep0 = out->step[0]/esz, ostep1 = out->step[1]/esz, ostep2 = out->step[2]/esz; const size_t* ord = &order->at(0); size_t istep0 = inp->step[ord[0]]/esz, istep1 = inp->step[ord[1]]/esz, @@ -241,13 +242,13 @@ class PermuteLayerImpl CV_FINAL : public PermuteLayer int i1 = (int)(val % n1); int i0 = (int)(val / n1); - const float* inptr_orig = inp->ptr(); - float* outptr_orig = out->ptr(); + const T* inptr_orig = inp->ptr(); + T* outptr_orig = out->ptr(); for( size_t ofs = stripeStart; ofs < stripeEnd; ofs++ ) { - const float* inptr = inptr_orig + i0*istep0 + i1*istep1 + i2*istep2; - float* outptr = outptr_orig + i0*ostep0 + i1*ostep1 + i2*ostep2; + const T* inptr = inptr_orig + i0*istep0 + i1*istep1 + i2*istep2; + T* outptr = outptr_orig + i0*ostep0 + i1*ostep1 + i2*ostep2; for( int i3 = 0; i3 < n3; i3++ ) outptr[i3] = inptr[i3*istep3]; @@ -321,7 +322,8 @@ class PermuteLayerImpl CV_FINAL : public PermuteLayer CV_TRACE_FUNCTION(); CV_TRACE_ARG_VALUE(name, "name", name.c_str()); - CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget), + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget) && + inputs_arr.depth() != CV_8S, forward_ocl(inputs_arr, outputs_arr, internals_arr)) if (inputs_arr.depth() == CV_16S) @@ -365,24 +367,48 @@ class PermuteLayerImpl CV_FINAL : public PermuteLayer if( numAxes == 4 ) { int nstripes = getNumThreads(); - PermuteInvoker::run(inp, out, _order, nstripes); + if (inp.type() == CV_8S) + PermuteInvoker::run(inp, out, _order, nstripes); + else + PermuteInvoker::run(inp, out, _order, nstripes); } else { - const float *srcData = inp.ptr(); - float *dstData = out.ptr(); + if (inp.type() == CV_8S) + { + const int8_t *srcData = inp.ptr(); + int8_t *dstData = out.ptr(); - for (i = 0; i < count; ++i) + for (i = 0; i < count; ++i) + { + size_t oldPosition = 0; + size_t newPosition = i; + + for (j = 0; j < numAxes; ++j) + { + oldPosition += (newPosition / newStride[j]) * oldStride[order[j]]; + newPosition %= newStride[j]; + } + dstData[i] = srcData[oldPosition]; + } + } + else { - size_t oldPosition = 0; - size_t newPosition = i; + const float *srcData = inp.ptr(); + float *dstData = out.ptr(); - for (j = 0; j < numAxes; ++j) + for (i = 0; i < count; ++i) { - oldPosition += (newPosition / newStride[j]) * oldStride[order[j]]; - newPosition %= newStride[j]; + size_t oldPosition = 0; + size_t newPosition = i; + + for (j = 0; j < numAxes; ++j) + { + oldPosition += (newPosition / newStride[j]) * oldStride[order[j]]; + newPosition %= newStride[j]; + } + dstData[i] = srcData[oldPosition]; } - dstData[i] = srcData[oldPosition]; } } } @@ -436,6 +462,11 @@ class PermuteLayerImpl CV_FINAL : public PermuteLayer } #endif // HAVE_VULKAN + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } size_t _count; std::vector _order; diff --git a/modules/dnn/src/layers/pooling_layer.cpp b/modules/dnn/src/layers/pooling_layer.cpp index b8e2cfdf8f84..7653e536680c 100644 --- a/modules/dnn/src/layers/pooling_layer.cpp +++ b/modules/dnn/src/layers/pooling_layer.cpp @@ -1327,6 +1327,23 @@ class PoolingLayerImpl CV_FINAL : public PoolingLayer return true; } + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + if (type == MAX && !computeMaxIdx) + { + return true; + } + else if (type == AVE || type == SUM) + { + float multiplier = scales[0][0] / scales[1][0]; + params.set("multiplier", multiplier); + params.set("input_zeropoint", zeropoints[0][0]); + return true; + } + return false; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/reorg_layer.cpp b/modules/dnn/src/layers/reorg_layer.cpp index da1c61adac0e..797df4819d9e 100644 --- a/modules/dnn/src/layers/reorg_layer.cpp +++ b/modules/dnn/src/layers/reorg_layer.cpp @@ -231,6 +231,11 @@ class ReorgLayerImpl CV_FINAL : public ReorgLayer } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE diff --git a/modules/dnn/src/layers/reshape_layer.cpp b/modules/dnn/src/layers/reshape_layer.cpp index ab8f41c7b6dd..4c10d155c8ae 100644 --- a/modules/dnn/src/layers/reshape_layer.cpp +++ b/modules/dnn/src/layers/reshape_layer.cpp @@ -343,6 +343,11 @@ class ReshapeLayerImpl CV_FINAL : public ReshapeLayer } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } private: std::vector outShapes; diff --git a/modules/dnn/src/layers/scale_layer.cpp b/modules/dnn/src/layers/scale_layer.cpp index a5c268214e86..001db24a2df8 100644 --- a/modules/dnn/src/layers/scale_layer.cpp +++ b/modules/dnn/src/layers/scale_layer.cpp @@ -344,6 +344,14 @@ class ScaleLayerImpl CV_FINAL : public ScaleLayer shift = (hasBias && !blobs.empty()) ? blobs.back() : Mat(); } + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + params.set("input_scales", DictValue::arrayReal(scales[0].data(), scales[0].size())); + params.set("input_zeropoints", DictValue::arrayInt(zeropoints[0].data(), zeropoints[0].size())); + return true; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/shuffle_channel_layer.cpp b/modules/dnn/src/layers/shuffle_channel_layer.cpp index 6db74d1abda7..2a698d270fa8 100644 --- a/modules/dnn/src/layers/shuffle_channel_layer.cpp +++ b/modules/dnn/src/layers/shuffle_channel_layer.cpp @@ -147,6 +147,12 @@ class ShuffleChannelLayerImpl CV_FINAL : public ShuffleChannelLayer } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } + private: Ptr permute; std::vector permuteInpShape, permuteOutShape; diff --git a/modules/dnn/src/layers/slice_layer.cpp b/modules/dnn/src/layers/slice_layer.cpp index 54e234038710..9efd95cf48df 100644 --- a/modules/dnn/src/layers/slice_layer.cpp +++ b/modules/dnn/src/layers/slice_layer.cpp @@ -531,7 +531,12 @@ class SliceLayerImpl : public SliceLayer { std::vector inpIdx(dimsNum, 0); std::vector outIdx(dimsNum, 0); - getSliceRecursive(inpMat, inpIdx, finalSliceRanges[i], sliceSteps[i], 0, dimsNum, outputs[i], outIdx); + if (inpMat.type() == CV_16S) + getSliceRecursive(inpMat, inpIdx, finalSliceRanges[i], sliceSteps[i], 0, dimsNum, outputs[i], outIdx); + else if (inpMat.type() == CV_8S) + getSliceRecursive(inpMat, inpIdx, finalSliceRanges[i], sliceSteps[i], 0, dimsNum, outputs[i], outIdx); + else + getSliceRecursive(inpMat, inpIdx, finalSliceRanges[i], sliceSteps[i], 0, dimsNum, outputs[i], outIdx); } } } @@ -647,8 +652,20 @@ class SliceLayerImpl : public SliceLayer } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + const int numOutputs = scales[1].size(); + for (int i = 0; i < numOutputs; i++) + { + if (scales[1][i] != scales[0][0]) + return false; + } + return true; + } private: + template void getSliceRecursive(const Mat &inpMat, std::vector &inpIdx, const std::vector &sliceRanges, const std::vector &sliceSteps, int dim, int dimsNum, @@ -658,8 +675,6 @@ class SliceLayerImpl : public SliceLayer int end = sliceRanges[dim].end; int step = !sliceSteps.empty() ? sliceSteps[dim] : 1; - const bool is32F = inpMat.depth() == CV_32F; - // TODO optimization is required (for 2D tail case at least) for (int k = begin, j = 0; k < end; k += step, j++) { @@ -667,14 +682,9 @@ class SliceLayerImpl : public SliceLayer outIdx[dim] = j; if (dim + 1 < dimsNum) - getSliceRecursive(inpMat, inpIdx, sliceRanges, sliceSteps, dim + 1, dimsNum, outputs, outIdx); + getSliceRecursive(inpMat, inpIdx, sliceRanges, sliceSteps, dim + 1, dimsNum, outputs, outIdx); else - { - if (is32F) - outputs.at(outIdx.data()) = inpMat.at(inpIdx.data()); - else - outputs.at(outIdx.data()) = inpMat.at(inpIdx.data()); // 16F emulation - } + outputs.at(outIdx.data()) = inpMat.at(inpIdx.data()); } } diff --git a/modules/dnn/src/layers/softmax_layer.cpp b/modules/dnn/src/layers/softmax_layer.cpp index 546c1017add8..e937e98f8c77 100644 --- a/modules/dnn/src/layers/softmax_layer.cpp +++ b/modules/dnn/src/layers/softmax_layer.cpp @@ -374,6 +374,22 @@ class SoftMaxLayerImpl CV_FINAL : public SoftmaxLayer } #endif // HAVE_DNN_NGRAPH + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + float inpScale = scales[0][0]; + Mat lookUpTable(1, 256, CV_32F); + float* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - 127); // ensures exp(x) is always between (0, 1) + table[i+128] = std::exp(x); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/split_layer.cpp b/modules/dnn/src/layers/split_layer.cpp index b025d5ff1e49..2a4417615264 100644 --- a/modules/dnn/src/layers/split_layer.cpp +++ b/modules/dnn/src/layers/split_layer.cpp @@ -117,6 +117,17 @@ class SplitLayerImpl CV_FINAL : public SplitLayer } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + const int numOutputs = scales[1].size(); + for (int i = 0; i < numOutputs; i++) + { + if (scales[1][i] != scales[0][0]) + return false; + } + return true; + } }; Ptr SplitLayer::create(const LayerParams& params) diff --git a/modules/dnn/test/test_int8_layers.cpp b/modules/dnn/test/test_int8_layers.cpp new file mode 100644 index 000000000000..1fcb1d0dba7c --- /dev/null +++ b/modules/dnn/test/test_int8_layers.cpp @@ -0,0 +1,1220 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" +#include "npy_blob.hpp" +#include +#include +namespace opencv_test { namespace { + +template +static std::string _tf(TString filename) +{ + return (getOpenCVExtraDir() + "dnn/") + filename; +} + +class Test_Int8_layers : public DNNTestLayer +{ +public: + void testLayer(const String& basename, const String& importer, double l1, double lInf, + int numInps = 1, int numOuts = 1, bool useCaffeModel = false, + bool useCommonInputBlob = true, bool hasText = false) + { + CV_Assert_N(numInps >= 1, numInps <= 10, numOuts >= 1, numOuts <= 10); + std::vector inps(numInps), inps_int8(numInps); + std::vector refs(numOuts), outs_int8(numOuts), outs_dequantized(numOuts); + std::vector inputScale, outputScale; + std::vector inputZp, outputZp; + String inpPath, outPath; + Net net, qnet; + + if (importer == "Caffe") + { + String prototxt = _tf("layers/" + basename + ".prototxt"); + String caffemodel = _tf("layers/" + basename + ".caffemodel"); + net = readNetFromCaffe(prototxt, useCaffeModel ? caffemodel : String()); + + inpPath = _tf("layers/" + (useCommonInputBlob ? "blob" : basename + ".input")); + outPath = _tf("layers/" + basename); + } + else if (importer == "TensorFlow") + { + String netPath = _tf("tensorflow/" + basename + "_net.pb"); + String netConfig = hasText ? _tf("tensorflow/" + basename + "_net.pbtxt") : ""; + net = readNetFromTensorflow(netPath, netConfig); + + inpPath = _tf("tensorflow/" + basename + "_in"); + outPath = _tf("tensorflow/" + basename + "_out"); + } + else if (importer == "ONNX") + { + String onnxmodel = _tf("onnx/models/" + basename + ".onnx"); + net = readNetFromONNX(onnxmodel); + + inpPath = _tf("onnx/data/input_" + basename); + outPath = _tf("onnx/data/output_" + basename); + } + ASSERT_FALSE(net.empty()); + net.setPreferableBackend(backend); + net.setPreferableTarget(target); + + for (int i = 0; i < numInps; i++) + inps[i] = blobFromNPY(inpPath + ((numInps > 1) ? cv::format("_%d.npy", i) : ".npy")); + + for (int i = 0; i < numOuts; i++) + refs[i] = blobFromNPY(outPath + ((numOuts > 1) ? cv::format("_%d.npy", i) : ".npy")); + + qnet = net.quantize(inps, CV_8S, CV_8S); + qnet.getInputDetails(inputScale, inputZp); + qnet.getOutputDetails(outputScale, outputZp); + + // Quantize inputs to int8 + // int8_value = float_value/scale + zero-point + for (int i = 0; i < numInps; i++) + { + inps[i].convertTo(inps_int8[i], CV_8S, 1.f/inputScale[i], inputZp[i]); + String inp_name = numInps > 1 ? (importer == "Caffe" ? cv::format("input_%d", i) : cv::format("%d", i)) : ""; + qnet.setInput(inps_int8[i], inp_name); + } + qnet.forward(outs_int8); + + // Dequantize outputs and compare with reference outputs + // float_value = scale*(int8_value - zero-point) + for (int i = 0; i < numOuts; i++) + { + outs_int8[i].convertTo(outs_dequantized[i], CV_32F, outputScale[i], -(outputScale[i] * outputZp[i])); + normAssert(refs[i], outs_dequantized[i], "", l1, lInf); + } + } +}; + +TEST_P(Test_Int8_layers, Convolution1D) +{ + testLayer("conv1d", "ONNX", 0.00302, 0.00909); + testLayer("conv1d_bias", "ONNX", 0.00306, 0.00948); +} + +TEST_P(Test_Int8_layers, Convolution2D) +{ + testLayer("layer_convolution", "Caffe", 0.0174, 0.0758, 1, 1, true); + testLayer("single_conv", "TensorFlow", 0.00413, 0.02201); + testLayer("depthwise_conv2d", "TensorFlow", 0.0388, 0.169); + testLayer("atrous_conv2d_valid", "TensorFlow", 0.0193, 0.0633); + testLayer("atrous_conv2d_same", "TensorFlow", 0.0185, 0.1322); + testLayer("keras_atrous_conv2d_same", "TensorFlow", 0.0056, 0.0244); + testLayer("convolution", "ONNX", 0.0052, 0.01516); + testLayer("two_convolution", "ONNX", 0.00295, 0.00840); +} + +TEST_P(Test_Int8_layers, Convolution3D) +{ + testLayer("conv3d", "TensorFlow", 0.00734, 0.02434); + testLayer("conv3d", "ONNX", 0.00353, 0.00941); + testLayer("conv3d_bias", "ONNX", 0.00129, 0.00249); +} + +TEST_P(Test_Int8_layers, Flatten) +{ + testLayer("flatten", "TensorFlow", 0.0036, 0.0069, 1, 1, false, true, true); + testLayer("unfused_flatten", "TensorFlow", 0.0014, 0.0028); + testLayer("unfused_flatten_unknown_batch", "TensorFlow", 0.0043, 0.0051); +} + +TEST_P(Test_Int8_layers, Padding) +{ + testLayer("padding_valid", "TensorFlow", 0.0026, 0.0064); + testLayer("padding_same", "TensorFlow", 0.0081, 0.032); + testLayer("spatial_padding", "TensorFlow", 0.0078, 0.028); + testLayer("mirror_pad", "TensorFlow", 0.0064, 0.013); + testLayer("pad_and_concat", "TensorFlow", 0.0021, 0.0098); + testLayer("padding", "ONNX", 0.0005, 0.0069); + testLayer("ReflectionPad2d", "ONNX", 0.00062, 0.0018); + testLayer("ZeroPad2d", "ONNX", 0.00037, 0.0018); +} + +TEST_P(Test_Int8_layers, AvePooling) +{ + testLayer("layer_pooling_ave", "Caffe", 0.0021, 0.0075); + testLayer("ave_pool_same", "TensorFlow", 0.00153, 0.0041); + testLayer("average_pooling_1d", "ONNX", 0.002, 0.0048); + testLayer("average_pooling", "ONNX", 0.0014, 0.0032); + testLayer("average_pooling_dynamic_axes", "ONNX", 0.0014, 0.006); + + if (target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); + testLayer("ave_pool3d", "TensorFlow", 0.00175, 0.0047); + testLayer("ave_pool3d", "ONNX", 0.00063, 0.0016); +} + +TEST_P(Test_Int8_layers, MaxPooling) +{ + testLayer("pool_conv_1d", "ONNX", 0.0006, 0.0015); + if (target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); + testLayer("pool_conv_3d", "ONNX", 0.0033, 0.0124); + + /* All the below tests have MaxPooling as last layer, so computeMaxIdx is set to true + which is not supported by int8 maxpooling + testLayer("layer_pooling_max", "Caffe", 0.0021, 0.004); + testLayer("max_pool_even", "TensorFlow", 0.0048, 0.0139); + testLayer("max_pool_odd_valid", "TensorFlow", 0.0043, 0.012); + testLayer("conv_pool_nchw", "TensorFlow", 0.007, 0.025); + testLayer("max_pool3d", "TensorFlow", 0.0025, 0.0058); + testLayer("maxpooling_1d", "ONNX", 0.0018, 0.0037); + testLayer("two_maxpooling_1d", "ONNX", 0.0037, 0.0052); + testLayer("maxpooling", "ONNX", 0.0034, 0.0065); + testLayer("two_maxpooling", "ONNX", 0.0025, 0.0052); + testLayer("max_pool3d", "ONNX", 0.0028, 0.0069);*/ +} + +TEST_P(Test_Int8_layers, Reduce) +{ + testLayer("reduce_mean", "TensorFlow", 0.0005, 0.0014); + testLayer("reduce_mean", "ONNX", 0.00062, 0.0014); + testLayer("reduce_mean_axis1", "ONNX", 0.00032, 0.0007); + testLayer("reduce_mean_axis2", "ONNX", 0.00033, 0.001); + + testLayer("reduce_sum", "TensorFlow", 0.015, 0.031); + testLayer("reduce_sum_channel", "TensorFlow", 0.008, 0.019); + testLayer("sum_pool_by_axis", "TensorFlow", 0.012, 0.032); + testLayer("reduce_sum", "ONNX", 0.0025, 0.0048); + + testLayer("reduce_max", "ONNX", 0, 0); + testLayer("reduce_max_axis_0", "ONNX", 0.0042, 0.007); + testLayer("reduce_max_axis_1", "ONNX", 0.0018, 0.0036); + + if (target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); + testLayer("reduce_mean3d", "ONNX", 0.00048, 0.0016); +} + +TEST_P(Test_Int8_layers, ReLU) +{ + testLayer("layer_relu", "Caffe", 0.0005, 0.002); + testLayer("ReLU", "ONNX", 0.0012, 0.0047); +} + +TEST_P(Test_Int8_layers, LeakyReLU) +{ + testLayer("leaky_relu", "TensorFlow", 0.0002, 0.0004); +} + +TEST_P(Test_Int8_layers, ReLU6) +{ + testLayer("keras_relu6", "TensorFlow", 0.0018, 0.0062); + testLayer("keras_relu6", "TensorFlow", 0.0018, 0.0062, 1, 1, false, true, true); + testLayer("clip_by_value", "TensorFlow", 0.0009, 0.002); + testLayer("clip", "ONNX", 0.00006, 0.00037); +} + +TEST_P(Test_Int8_layers, Sigmoid) +{ + testLayer("maxpooling_sigmoid", "ONNX", 0.0011, 0.0032); + testLayer("maxpooling_sigmoid_dynamic_axes", "ONNX", 0.0011, 0.0032); + testLayer("maxpooling_sigmoid_1d", "ONNX", 0.0011, 0.0037); +} + +TEST_P(Test_Int8_layers, Mish) +{ + testLayer("mish", "ONNX", 0.0015, 0.0025); +} + +TEST_P(Test_Int8_layers, Softmax) +{ + testLayer("layer_softmax", "Caffe", 0.0011, 0.0036); + testLayer("keras_softmax", "TensorFlow", 0.00093, 0.0027); + testLayer("slim_softmax", "TensorFlow", 0.0016, 0.0034); + testLayer("slim_softmax_v2", "TensorFlow", 0.0029, 0.017); + testLayer("softmax", "ONNX", 0.0016, 0.0028); + testLayer("log_softmax", "ONNX", 0.014, 0.025); + testLayer("softmax_unfused", "ONNX", 0.0009, 0.0021); +} + +TEST_P(Test_Int8_layers, Concat) +{ + testLayer("layer_concat_shared_input", "Caffe", 0.0076, 0.029, 1, 1, true, false); + testLayer("concat_axis_1", "TensorFlow", 0.0056, 0.017); + testLayer("keras_pad_concat", "TensorFlow", 0.0032, 0.0089); + testLayer("concat_3d", "TensorFlow", 0.005, 0.014); + testLayer("concatenation", "ONNX", 0.0032, 0.009); +} + +TEST_P(Test_Int8_layers, BatchNorm) +{ + testLayer("layer_batch_norm", "Caffe", 0.0061, 0.019, 1, 1, true); + testLayer("fused_batch_norm", "TensorFlow", 0.0063, 0.02); + testLayer("batch_norm_text", "TensorFlow", 0.0048, 0.013, 1, 1, false, true, true); + testLayer("unfused_batch_norm", "TensorFlow", 0.0076, 0.019); + testLayer("fused_batch_norm_no_gamma", "TensorFlow", 0.0067, 0.015); + testLayer("unfused_batch_norm_no_gamma", "TensorFlow", 0.0123, 0.044); + testLayer("switch_identity", "TensorFlow", 0.0035, 0.011); + testLayer("batch_norm3d", "TensorFlow", 0.0077, 0.02); + testLayer("batch_norm", "ONNX", 0.0012, 0.0049); + testLayer("batch_norm_3d", "ONNX", 0.0039, 0.012); + testLayer("frozenBatchNorm2d", "ONNX", 0.001, 0.0018); + testLayer("batch_norm_subgraph", "ONNX", 0.0049, 0.0098); +} + +TEST_P(Test_Int8_layers, Scale) +{ + testLayer("batch_norm", "TensorFlow", 0.0028, 0.0098); + testLayer("scale", "ONNX", 0.0025, 0.0071); + testLayer("expand_hw", "ONNX", 0.0012, 0.0012); + testLayer("flatten_const", "ONNX", 0.0024, 0.0048); +} + +TEST_P(Test_Int8_layers, InnerProduct) +{ + testLayer("layer_inner_product", "Caffe", 0.005, 0.02, 1, 1, true); + testLayer("matmul", "TensorFlow", 0.0061, 0.019); + testLayer("nhwc_transpose_reshape_matmul", "TensorFlow", 0.0009, 0.0091); + testLayer("nhwc_reshape_matmul", "TensorFlow", 0.03, 0.071); + testLayer("matmul_layout", "TensorFlow", 0.035, 0.06); + testLayer("tf2_dense", "TensorFlow", 0, 0); + testLayer("matmul_add", "ONNX", 0.041, 0.082); + testLayer("linear", "ONNX", 0.0018, 0.0029); + testLayer("constant", "ONNX", 0.00021, 0.0006); + testLayer("lin_with_constant", "ONNX", 0.0011, 0.0016); +} + +TEST_P(Test_Int8_layers, Reshape) +{ + testLayer("reshape_layer", "TensorFlow", 0.0032, 0.0082); + testLayer("reshape_nchw", "TensorFlow", 0.0089, 0.029); + testLayer("reshape_conv", "TensorFlow", 0.035, 0.054); + testLayer("reshape_reduce", "TensorFlow", 0.0042, 0.0078); + testLayer("reshape_as_shape", "TensorFlow", 0.0014, 0.0028); + testLayer("reshape_no_reorder", "TensorFlow", 0.0014, 0.0028); + testLayer("shift_reshape_no_reorder", "TensorFlow", 0.0063, 0.014); + testLayer("dynamic_reshape", "ONNX", 0.0047, 0.0079); + testLayer("dynamic_reshape_opset_11", "ONNX", 0.0048, 0.0081); + testLayer("flatten_by_prod", "ONNX", 0.0048, 0.0081); + testLayer("squeeze", "ONNX", 0.0048, 0.0081); + testLayer("unsqueeze", "ONNX", 0.0033, 0.0053); + testLayer("squeeze_and_conv_dynamic_axes", "ONNX", 0.0054, 0.0154); + testLayer("unsqueeze_and_conv_dynamic_axes", "ONNX", 0.0037, 0.0151); +} + +TEST_P(Test_Int8_layers, Permute) +{ + testLayer("tf2_permute_nhwc_ncwh", "TensorFlow", 0.0028, 0.006); + testLayer("transpose", "ONNX", 0.0015, 0.0046); +} + +TEST_P(Test_Int8_layers, Identity) +{ + testLayer("expand_batch", "ONNX", 0.0027, 0.0036); + testLayer("expand_channels", "ONNX", 0.0013, 0.0019); + testLayer("expand_neg_batch", "ONNX", 0.00071, 0.0019); +} + +TEST_P(Test_Int8_layers, Slice) +{ + testLayer("split", "TensorFlow", 0.0033, 0.0056); + testLayer("slice_4d", "TensorFlow", 0.003, 0.0073); + testLayer("strided_slice", "TensorFlow", 0.008, 0.0142); + testLayer("slice", "ONNX", 0.0046, 0.0077); + testLayer("slice_dynamic_axes", "ONNX", 0.0039, 0.0084); + testLayer("slice_opset_11_steps_2d", "ONNX", 0.0052, 0.0124); + testLayer("slice_opset_11_steps_3d", "ONNX", 0.0068, 0.014); + testLayer("slice_opset_11_steps_4d", "ONNX", 0.0041, 0.008); + testLayer("slice_opset_11_steps_5d", "ONNX", 0.0085, 0.021); +} + +TEST_P(Test_Int8_layers, Dropout) +{ + testLayer("layer_dropout", "Caffe", 0.0021, 0.004); + testLayer("dropout", "ONNX", 0.0029, 0.004); +} + +TEST_P(Test_Int8_layers, Eltwise) +{ + testLayer("layer_eltwise", "Caffe", 0.062, 0.15); + testLayer("conv_2_inps", "Caffe", 0.0086, 0.0232, 2, 1, true, false); + testLayer("eltwise_sub", "TensorFlow", 0.015, 0.047); + testLayer("eltwise_add_vec", "TensorFlow", 0.037, 0.21); // tflite 0.0095, 0.0365 + testLayer("eltwise_mul_vec", "TensorFlow", 0.173, 1.14); // tflite 0.0028, 0.017 + testLayer("channel_broadcast", "TensorFlow", 0.0025, 0.0063); + testLayer("split_equals", "TensorFlow", 0.02, 0.065); + testLayer("mul", "ONNX", 0.0039, 0.014); + testLayer("split_max", "ONNX", 0.004, 0.012); +} + +INSTANTIATE_TEST_CASE_P(/**/, Test_Int8_layers, dnnBackendsAndTargets()); + +class Test_Int8_nets : public DNNTestLayer +{ +public: + void testClassificationNet(Net baseNet, const Mat& blob, const Mat& ref, double l1, double lInf) + { + Net qnet = baseNet.quantize(blob, CV_32F, CV_32F); + qnet.setPreferableBackend(backend); + qnet.setPreferableTarget(target); + + qnet.setInput(blob); + Mat out = qnet.forward(); + normAssert(ref, out, "", l1, lInf); + } + + void testDetectionNet(Net baseNet, const Mat& blob, const Mat& ref, + double confThreshold, double scoreDiff, double iouDiff) + { + Net qnet = baseNet.quantize(blob, CV_32F, CV_32F); + qnet.setPreferableBackend(backend); + qnet.setPreferableTarget(target); + + qnet.setInput(blob); + Mat out = qnet.forward(); + normAssertDetections(ref, out, "", confThreshold, scoreDiff, iouDiff); + } + + void testFaster(Net baseNet, const Mat& ref, double confThreshold, double scoreDiff, double iouDiff) + { + Mat inp = imread(_tf("dog416.png")); + resize(inp, inp, Size(800, 600)); + Mat blob = blobFromImage(inp, 1.0, Size(), Scalar(102.9801, 115.9465, 122.7717), false, false); + Mat imInfo = (Mat_(1, 3) << inp.rows, inp.cols, 1.6f); + + Net qnet = baseNet.quantize(std::vector{blob, imInfo}, CV_32F, CV_32F); + qnet.setPreferableBackend(backend); + qnet.setPreferableTarget(target); + + qnet.setInput(blob, "data"); + qnet.setInput(imInfo, "im_info"); + Mat out = qnet.forward(); + normAssertDetections(ref, out, "", confThreshold, scoreDiff, iouDiff); + } + + void testONNXNet(const String& basename, double l1, double lInf, bool useSoftmax = false) + { + String onnxmodel = findDataFile("dnn/onnx/models/" + basename + ".onnx", false); + + Mat blob = readTensorFromONNX(findDataFile("dnn/onnx/data/input_" + basename + ".pb")); + Mat ref = readTensorFromONNX(findDataFile("dnn/onnx/data/output_" + basename + ".pb")); + Net baseNet = readNetFromONNX(onnxmodel); + baseNet.setPreferableBackend(backend); + baseNet.setPreferableTarget(target); + + Net qnet = baseNet.quantize(blob, CV_32F, CV_32F); + qnet.setInput(blob); + Mat out = qnet.forward(); + + if (useSoftmax) + { + LayerParams lp; + Net netSoftmax; + netSoftmax.addLayerToPrev("softmaxLayer", "Softmax", lp); + netSoftmax.setPreferableBackend(DNN_BACKEND_OPENCV); + + netSoftmax.setInput(out); + out = netSoftmax.forward(); + + netSoftmax.setInput(ref); + ref = netSoftmax.forward(); + } + + normAssert(ref, out, "", l1, lInf); + } + + void testDarknetModel(const std::string& cfg, const std::string& weights, + const cv::Mat& ref, double scoreDiff, double iouDiff, + float confThreshold = 0.24, float nmsThreshold = 0.4) + { + CV_Assert(ref.cols == 7); + std::vector > refClassIds; + std::vector > refScores; + std::vector > refBoxes; + for (int i = 0; i < ref.rows; ++i) + { + int batchId = static_cast(ref.at(i, 0)); + int classId = static_cast(ref.at(i, 1)); + float score = ref.at(i, 2); + float left = ref.at(i, 3); + float top = ref.at(i, 4); + float right = ref.at(i, 5); + float bottom = ref.at(i, 6); + Rect2d box(left, top, right - left, bottom - top); + if (batchId >= refClassIds.size()) + { + refClassIds.resize(batchId + 1); + refScores.resize(batchId + 1); + refBoxes.resize(batchId + 1); + } + refClassIds[batchId].push_back(classId); + refScores[batchId].push_back(score); + refBoxes[batchId].push_back(box); + } + + Mat img1 = imread(_tf("dog416.png")); + Mat img2 = imread(_tf("street.png")); + std::vector samples(2); + samples[0] = img1; samples[1] = img2; + + // determine test type, whether batch or single img + int batch_size = refClassIds.size(); + CV_Assert(batch_size == 1 || batch_size == 2); + samples.resize(batch_size); + + Mat inp = blobFromImages(samples, 1.0/255, Size(416, 416), Scalar(), true, false); + + Net baseNet = readNetFromDarknet(findDataFile("dnn/" + cfg), findDataFile("dnn/" + weights, false)); + Net qnet = baseNet.quantize(inp, CV_32F, CV_32F); + qnet.setPreferableBackend(backend); + qnet.setPreferableTarget(target); + qnet.setInput(inp); + std::vector outs; + qnet.forward(outs, qnet.getUnconnectedOutLayersNames()); + + for (int b = 0; b < batch_size; ++b) + { + std::vector classIds; + std::vector confidences; + std::vector boxes; + for (int i = 0; i < outs.size(); ++i) + { + Mat out; + if (batch_size > 1){ + // get the sample slice from 3D matrix (batch, box, classes+5) + Range ranges[3] = {Range(b, b+1), Range::all(), Range::all()}; + out = outs[i](ranges).reshape(1, outs[i].size[1]); + }else{ + out = outs[i]; + } + for (int j = 0; j < out.rows; ++j) + { + Mat scores = out.row(j).colRange(5, out.cols); + double confidence; + Point maxLoc; + minMaxLoc(scores, 0, &confidence, 0, &maxLoc); + + if (confidence > confThreshold) { + float* detection = out.ptr(j); + double centerX = detection[0]; + double centerY = detection[1]; + double width = detection[2]; + double height = detection[3]; + boxes.push_back(Rect2d(centerX - 0.5 * width, centerY - 0.5 * height, + width, height)); + confidences.push_back(confidence); + classIds.push_back(maxLoc.x); + } + } + } + + // here we need NMS of boxes + std::vector indices; + NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices); + + std::vector nms_classIds; + std::vector nms_confidences; + std::vector nms_boxes; + + for (size_t i = 0; i < indices.size(); ++i) + { + int idx = indices[i]; + Rect2d box = boxes[idx]; + float conf = confidences[idx]; + int class_id = classIds[idx]; + nms_boxes.push_back(box); + nms_confidences.push_back(conf); + nms_classIds.push_back(class_id); + } + + if (cvIsNaN(iouDiff)) + { + if (b == 0) + std::cout << "Skip accuracy checks" << std::endl; + continue; + } + + normAssertDetections(refClassIds[b], refScores[b], refBoxes[b], nms_classIds, nms_confidences, nms_boxes, + format("batch size %d, sample %d\n", batch_size, b).c_str(), confThreshold, scoreDiff, iouDiff); + } + } +}; + +TEST_P(Test_Int8_nets, AlexNet) +{ +#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) + applyTestTag(CV_TEST_TAG_MEMORY_2GB); +#else + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); +#endif + if (backend != DNN_BACKEND_OPENCV) + throw SkipTestException("Only OpenCV backend is supported"); + + Net net = readNetFromCaffe(findDataFile("dnn/bvlc_alexnet.prototxt"), + findDataFile("dnn/bvlc_alexnet.caffemodel", false)); + + Mat inp = imread(_tf("grace_hopper_227.png")); + Mat blob = blobFromImage(inp, 1.0, Size(227, 227), Scalar(), false); + Mat ref = blobFromNPY(_tf("caffe_alexnet_prob.npy")); + + float l1 = 1e-4, lInf = 0.003; + testClassificationNet(net, blob, ref, l1, lInf); +} + +TEST_P(Test_Int8_nets, GoogLeNet) +{ + Net net = readNetFromCaffe(findDataFile("dnn/bvlc_googlenet.prototxt"), + findDataFile("dnn/bvlc_googlenet.caffemodel", false)); + + std::vector inpMats; + inpMats.push_back( imread(_tf("googlenet_0.png")) ); + inpMats.push_back( imread(_tf("googlenet_1.png")) ); + Mat blob = blobFromImages(inpMats, 1.0, Size(224, 224), Scalar(), false); + Mat ref = blobFromNPY(_tf("googlenet_prob.npy")); + + float l1 = 2e-4, lInf = 0.06; + testClassificationNet(net, blob, ref, l1, lInf); +} + +TEST_P(Test_Int8_nets, ResNet50) +{ + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); + if (backend != DNN_BACKEND_OPENCV) + throw SkipTestException("Only OpenCV backend is supported"); + + Net net = readNetFromCaffe(findDataFile("dnn/ResNet-50-deploy.prototxt"), + findDataFile("dnn/ResNet-50-model.caffemodel", false)); + + Mat inp = imread(_tf("googlenet_0.png")); + Mat blob = blobFromImage(inp, 1.0, Size(224, 224), Scalar(), false); + Mat ref = blobFromNPY(_tf("resnet50_prob.npy")); + + float l1 = 3e-4, lInf = 0.035; + testClassificationNet(net, blob, ref, l1, lInf); +} + +TEST_P(Test_Int8_nets, DenseNet121) +{ + applyTestTag(CV_TEST_TAG_MEMORY_512MB); + + Net net = readNetFromCaffe(findDataFile("dnn/DenseNet_121.prototxt", false), + findDataFile("dnn/DenseNet_121.caffemodel", false)); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0 / 255.0, Size(224, 224), Scalar(), true, true); + Mat ref = blobFromNPY(_tf("densenet_121_output.npy")); + + float l1 = 0.76, lInf = 3.31; // seems wrong + testClassificationNet(net, blob, ref, l1, lInf); +} + +TEST_P(Test_Int8_nets, SqueezeNet_v1_1) +{ + if(target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + + Net net = readNetFromCaffe(findDataFile("dnn/squeezenet_v1.1.prototxt"), + findDataFile("dnn/squeezenet_v1.1.caffemodel", false)); + + Mat inp = imread(_tf("googlenet_0.png")); + Mat blob = blobFromImage(inp, 1.0, Size(227, 227), Scalar(), false, true); + Mat ref = blobFromNPY(_tf("squeezenet_v1.1_prob.npy")); + + float l1 = 3e-4, lInf = 0.056; + testClassificationNet(net, blob, ref, l1, lInf); +} + +TEST_P(Test_Int8_nets, CaffeNet) +{ +#if defined(OPENCV_32BIT_CONFIGURATION) && (defined(HAVE_OPENCL) || defined(_WIN32)) + applyTestTag(CV_TEST_TAG_MEMORY_2GB); +#else + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); +#endif + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019030000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD + && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + float l1 = 4e-5, lInf = 0.0025; + testONNXNet("caffenet", l1, lInf); +} + +TEST_P(Test_Int8_nets, RCNN_ILSVRC13) +{ +#if defined(OPENCV_32BIT_CONFIGURATION) && (defined(HAVE_OPENCL) || defined(_WIN32)) + applyTestTag(CV_TEST_TAG_MEMORY_2GB); +#else + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); +#endif + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019030000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD + && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + float l1 = 0.02, lInf = 0.042; + testONNXNet("rcnn_ilsvrc13", l1, lInf); +} + +TEST_P(Test_Int8_nets, Inception_v2) +{ + testONNXNet("inception_v2", default_l1, default_lInf, true); +} + +TEST_P(Test_Int8_nets, MobileNet_v2) +{ + testONNXNet("mobilenetv2", default_l1, default_lInf, true); +} + +TEST_P(Test_Int8_nets, Shufflenet) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + } + testONNXNet("shufflenet", default_l1, default_lInf); +} + +TEST_P(Test_Int8_nets, MobileNet_SSD) +{ + Net net = readNetFromCaffe(findDataFile("dnn/MobileNetSSD_deploy.prototxt", false), + findDataFile("dnn/MobileNetSSD_deploy.caffemodel", false)); + + Mat inp = imread(_tf("street.png")); + Mat blob = blobFromImage(inp, 1.0 / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false); + Mat ref = blobFromNPY(_tf("mobilenet_ssd_caffe_out.npy")); + + float confThreshold = FLT_MIN, scoreDiff = 0.059, iouDiff = 0.11; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, MobileNet_v1_SSD) +{ + Net net = readNetFromTensorflow(findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false), + findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt")); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0, Size(300, 300), Scalar(), true, false); + Mat ref = blobFromNPY(_tf("tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy")); + + float confThreshold = 0.5, scoreDiff = 0.034, iouDiff = 0.13; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, MobileNet_v1_SSD_PPN) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + Net net = readNetFromTensorflow(findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false), + findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt")); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0, Size(300, 300), Scalar(), true, false); + Mat ref = blobFromNPY(_tf("tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy")); + + float confThreshold = 0.51, scoreDiff = 0.04, iouDiff = 0.06; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, Inception_v2_SSD) +{ + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2019010000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD && + getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + Net net = readNetFromTensorflow(findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false), + findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt")); + + Mat inp = imread(_tf("street.png")); + Mat blob = blobFromImage(inp, 1.0, Size(300, 300), Scalar(), true, false); + Mat ref = (Mat_(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729, + 0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131, + 0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015, + 0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527, + 0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384); + + float confThreshold = 0.5, scoreDiff = 0.0114, iouDiff = 0.22; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, opencv_face_detector) +{ + Net net = readNetFromCaffe(findDataFile("dnn/opencv_face_detector.prototxt"), + findDataFile("dnn/opencv_face_detector.caffemodel", false)); + + Mat inp = imread(findDataFile("gpu/lbpcascade/er.png")); + Mat blob = blobFromImage(inp, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false); + Mat ref = (Mat_(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631, + 0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168, + 0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290, + 0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477, + 0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494, + 0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801); + + float confThreshold = 0.5, scoreDiff = 0.002, iouDiff = 0.21; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, EfficientDet) +{ + if (target != DNN_TARGET_CPU) + { + if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); + } + Net net = readNetFromTensorflow(findDataFile("dnn/efficientdet-d0.pb", false), + findDataFile("dnn/efficientdet-d0.pbtxt")); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0/255, Size(512, 512), Scalar(123.675, 116.28, 103.53)); + Mat ref = (Mat_(3, 7) << 0, 1, 0.8437444, 0.153996080160141, 0.20534580945968628, 0.7463544607162476, 0.7414066195487976, + 0, 17, 0.8245924, 0.16657517850399017, 0.3996818959712982, 0.4111558794975281, 0.9306337833404541, + 0, 7, 0.8039304, 0.6118435263633728, 0.13175517320632935, 0.9065558314323425, 0.2943994700908661); + + float confThreshold = 0.65, scoreDiff = 0.17, iouDiff = 0.18; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, FasterRCNN_resnet50) +{ + applyTestTag( + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB), + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG + ); + +#ifdef INF_ENGINE_RELEASE + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && + (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + + if (INF_ENGINE_VER_MAJOR_GT(2019030000) && + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + + if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + + if (backend == DNN_BACKEND_CUDA && target == DNN_TARGET_CUDA_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); + + Net net = readNetFromTensorflow(findDataFile("dnn/faster_rcnn_resnet50_coco_2018_01_28.pb", false), + findDataFile("dnn/faster_rcnn_resnet50_coco_2018_01_28.pbtxt")); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0, Size(800, 600), Scalar(), true, false); + Mat ref = blobFromNPY(_tf("tensorflow/faster_rcnn_resnet50_coco_2018_01_28.detection_out.npy")); + + float confThreshold = 0.5, scoreDiff = 0.025, iouDiff = 0.15; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, FasterRCNN_inceptionv2) +{ + applyTestTag( + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB), + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG + ); + +#ifdef INF_ENGINE_RELEASE + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && + (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + + if (INF_ENGINE_VER_MAJOR_GT(2019030000) && + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + + if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + + if (backend == DNN_BACKEND_CUDA && target == DNN_TARGET_CUDA_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); + + Net net = readNetFromTensorflow(findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pb", false), + findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pbtxt")); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0, Size(800, 600), Scalar(), true, false); + Mat ref = blobFromNPY(_tf("tensorflow/faster_rcnn_inception_v2_coco_2018_01_28.detection_out.npy")); + + float confThreshold = 0.5, scoreDiff = 0.21, iouDiff = 0.1; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, FasterRCNN_vgg16) +{ + applyTestTag( +#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) + CV_TEST_TAG_MEMORY_2GB, +#else + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB), +#endif + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG + ); + +#if defined(INF_ENGINE_RELEASE) + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); +#endif + + Net net = readNetFromCaffe(findDataFile("dnn/faster_rcnn_vgg16.prototxt"), + findDataFile("dnn/VGG16_faster_rcnn_final.caffemodel", false)); + + Mat ref = (Mat_(3, 7) << 0, 2, 0.949398, 99.2454, 210.141, 601.205, 462.849, + 0, 7, 0.997022, 481.841, 92.3218, 722.685, 175.953, + 0, 12, 0.993028, 133.221, 189.377, 350.994, 563.166); + + float confThreshold = 0.8, scoreDiff = 0.024, iouDiff = 0.35; + testFaster(net, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, FasterRCNN_zf) +{ + applyTestTag( +#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) + CV_TEST_TAG_MEMORY_2GB, +#else + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB), +#endif + CV_TEST_TAG_DEBUG_LONG + ); + + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); + + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); + + if (target == DNN_TARGET_CUDA_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); + + Net net = readNetFromCaffe(findDataFile("dnn/faster_rcnn_zf.prototxt"), + findDataFile("dnn/ZF_faster_rcnn_final.caffemodel", false)); + + Mat ref = (Mat_(3, 7) << 0, 2, 0.90121, 120.407, 115.83, 570.586, 528.395, + 0, 7, 0.988779, 469.849, 75.1756, 718.64, 186.762, + 0, 12, 0.967198, 138.588, 206.843, 329.766, 553.176); + + float confThreshold = 0.8, scoreDiff = 0.021, iouDiff = 0.1; + testFaster(net, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, RFCN) +{ + applyTestTag( + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_2GB), + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG + ); + + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); + + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); + + Net net = readNetFromCaffe(findDataFile("dnn/rfcn_pascal_voc_resnet50.prototxt"), + findDataFile("dnn/resnet50_rfcn_final.caffemodel", false)); + + Mat ref = (Mat_(2, 7) << 0, 7, 0.991359, 491.822, 81.1668, 702.573, 178.234, + 0, 12, 0.94786, 132.093, 223.903, 338.077, 566.16); + + float confThreshold = 0.8, scoreDiff = 0.017, iouDiff = 0.11; + testFaster(net, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, YoloVoc) +{ + applyTestTag( +#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) + CV_TEST_TAG_MEMORY_2GB, +#else + CV_TEST_TAG_MEMORY_1GB, +#endif + CV_TEST_TAG_LONG + ); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); +#endif +#if defined(INF_ENGINE_RELEASE) + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && + target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); +#endif + + Mat ref = (Mat_(6, 7) << 0, 6, 0.750469f, 0.577374f, 0.127391f, 0.902949f, 0.300809f, + 0, 1, 0.780879f, 0.270762f, 0.264102f, 0.732475f, 0.745412f, + 0, 11, 0.901615f, 0.1386f, 0.338509f, 0.421337f, 0.938789f, + 1, 14, 0.623813f, 0.183179f, 0.381921f, 0.247726f, 0.625847f, + 1, 6, 0.667770f, 0.446555f, 0.453578f, 0.499986f, 0.519167f, + 1, 6, 0.844947f, 0.637058f, 0.460398f, 0.828508f, 0.66427f); + + std::string config_file = "yolo-voc.cfg"; + std::string weights_file = "yolo-voc.weights"; + + double scoreDiff = 0.1, iouDiff = 0.3; + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, 3), scoreDiff, iouDiff); + } + + { + SCOPED_TRACE("batch size 2"); + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff); + } +} + +TEST_P(Test_Int8_nets, TinyYoloVoc) +{ + applyTestTag(CV_TEST_TAG_MEMORY_512MB); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +#if defined(INF_ENGINE_RELEASE) + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && + target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); +#endif + + Mat ref = (Mat_(4, 7) << 0, 6, 0.761967f, 0.579042f, 0.159161f, 0.894482f, 0.31994f, + 0, 11, 0.780595f, 0.129696f, 0.386467f, 0.445275f, 0.920994f, + 1, 6, 0.651450f, 0.460526f, 0.458019f, 0.522527f, 0.5341f, + 1, 6, 0.928758f, 0.651024f, 0.463539f, 0.823784f, 0.654998f); + + std::string config_file = "tiny-yolo-voc.cfg"; + std::string weights_file = "tiny-yolo-voc.weights"; + + double scoreDiff = 0.043, iouDiff = 0.12; + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, 2), scoreDiff, iouDiff); + } + + { + SCOPED_TRACE("batch size 2"); + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff); + } +} + +TEST_P(Test_Int8_nets, YOLOv3) +{ + applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + + const int N0 = 3; + const int N1 = 6; + static const float ref_[/* (N0 + N1) * 7 */] = { +0, 16, 0.998836f, 0.160024f, 0.389964f, 0.417885f, 0.943716f, +0, 1, 0.987908f, 0.150913f, 0.221933f, 0.742255f, 0.746261f, +0, 7, 0.952983f, 0.614621f, 0.150257f, 0.901368f, 0.289251f, + +1, 2, 0.997412f, 0.647584f, 0.459939f, 0.821037f, 0.663947f, +1, 2, 0.989633f, 0.450719f, 0.463353f, 0.496306f, 0.522258f, +1, 0, 0.980053f, 0.195856f, 0.378454f, 0.258626f, 0.629257f, +1, 9, 0.785341f, 0.665503f, 0.373543f, 0.688893f, 0.439244f, +1, 9, 0.733275f, 0.376029f, 0.315694f, 0.401776f, 0.395165f, +1, 9, 0.384815f, 0.659824f, 0.372389f, 0.673927f, 0.429412f, + }; + Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); + + std::string config_file = "yolov3.cfg"; + std::string weights_file = "yolov3.weights"; + + double scoreDiff = 0.08, iouDiff = 0.21, confThreshold = 0.25; + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold); + } + +#if defined(INF_ENGINE_RELEASE) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_MYRIAD && + getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); + } +#endif + + { + SCOPED_TRACE("batch size 2"); + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, confThreshold); + } +} + +TEST_P(Test_Int8_nets, YOLOv4) +{ + applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +#if defined(INF_ENGINE_RELEASE) + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + const int N0 = 3; + const int N1 = 7; + static const float ref_[/* (N0 + N1) * 7 */] = { +0, 16, 0.992194f, 0.172375f, 0.402458f, 0.403918f, 0.932801f, +0, 1, 0.988326f, 0.166708f, 0.228236f, 0.737208f, 0.735803f, +0, 7, 0.94639f, 0.602523f, 0.130399f, 0.901623f, 0.298452f, + +1, 2, 0.99761f, 0.646556f, 0.45985f, 0.816041f, 0.659067f, +1, 0, 0.988913f, 0.201726f, 0.360282f, 0.266181f, 0.631728f, +1, 2, 0.98233f, 0.452007f, 0.462217f, 0.495612f, 0.521687f, +1, 9, 0.919195f, 0.374642f, 0.316524f, 0.398126f, 0.393714f, +1, 9, 0.856303f, 0.666842f, 0.372215f, 0.685539f, 0.44141f, +1, 9, 0.313516f, 0.656791f, 0.374734f, 0.671959f, 0.438371f, +1, 9, 0.256625f, 0.940232f, 0.326931f, 0.967586f, 0.374002f, + }; + Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); + + std::string config_file = "yolov4.cfg"; + std::string weights_file = "yolov4.weights"; + double scoreDiff = 0.1, iouDiff = 0.17; + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff); + } + + { + SCOPED_TRACE("batch size 2"); + +#if defined(INF_ENGINE_RELEASE) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_MYRIAD && + getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); + } +#endif + + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff); + } +} + +TEST_P(Test_Int8_nets, YOLOv4_tiny) +{ + applyTestTag( + target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB + ); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2021010000) + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + const float confThreshold = 0.6; + + const int N0 = 2; + const int N1 = 3; + static const float ref_[/* (N0 + N1) * 7 */] = { +0, 7, 0.85935f, 0.593484f, 0.141211f, 0.920356f, 0.291593f, +0, 16, 0.795188f, 0.169207f, 0.386886f, 0.423753f, 0.933004f, + +1, 2, 0.996832f, 0.653802f, 0.464573f, 0.815193f, 0.653292f, +1, 2, 0.963325f, 0.451151f, 0.458915f, 0.496255f, 0.52241f, +1, 0, 0.926244f, 0.194851f, 0.361743f, 0.260277f, 0.632364f, + }; + Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); + + std::string config_file = "yolov4-tiny.cfg"; + std::string weights_file = "yolov4-tiny.weights"; + double scoreDiff = 0.12; + double iouDiff = target == DNN_TARGET_OPENCL_FP16 ? 0.2 : 0.082; + +#if defined(INF_ENGINE_RELEASE) + if (target == DNN_TARGET_MYRIAD) // bad accuracy + iouDiff = std::numeric_limits::quiet_NaN(); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL) + iouDiff = std::numeric_limits::quiet_NaN(); + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) + iouDiff = std::numeric_limits::quiet_NaN(); +#endif + + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold); + } + + /* bad accuracy on second image + { + SCOPED_TRACE("batch size 2"); + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, confThreshold); + } + */ + +#if defined(INF_ENGINE_RELEASE) + if (target == DNN_TARGET_MYRIAD) // bad accuracy + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +} + +INSTANTIATE_TEST_CASE_P(/**/, Test_Int8_nets, dnnBackendsAndTargets()); +}} // namespace From 9cfa84313c5833d7295fcf57be93d5d2aaadfd88 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Sat, 10 Jul 2021 00:21:52 +0200 Subject: [PATCH 125/376] Use the one argument version of SetTotalBytesLimit. The two argument versions has been deprecated, cf https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.io.coded_stream --- modules/dnn/src/caffe/caffe_io.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/dnn/src/caffe/caffe_io.cpp b/modules/dnn/src/caffe/caffe_io.cpp index 2fc4d84f4604..ebecf95eea3a 100644 --- a/modules/dnn/src/caffe/caffe_io.cpp +++ b/modules/dnn/src/caffe/caffe_io.cpp @@ -92,6 +92,7 @@ #ifdef HAVE_PROTOBUF #include #include +#include #include #include @@ -1111,7 +1112,11 @@ static const int kProtoReadBytesLimit = INT_MAX; // Max size of 2 GB minus 1 by bool ReadProtoFromBinary(ZeroCopyInputStream* input, Message *proto) { CodedInputStream coded_input(input); +#if GOOGLE_PROTOBUF_VERSION >= 3006000 + coded_input.SetTotalBytesLimit(kProtoReadBytesLimit); +#else coded_input.SetTotalBytesLimit(kProtoReadBytesLimit, 536870912); +#endif return proto->ParseFromCodedStream(&coded_input); } From c08897cd106dbeca97c4591a8af088c563fe1444 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 19 Aug 2021 20:06:41 +0000 Subject: [PATCH 126/376] cmake: handle empty CVPY_SUFFIX --- modules/python/common.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/python/common.cmake b/modules/python/common.cmake index 6a438fd1a267..ebbb2e2f655d 100644 --- a/modules/python/common.cmake +++ b/modules/python/common.cmake @@ -86,7 +86,7 @@ set_target_properties(${the_module} PROPERTIES ARCHIVE_OUTPUT_NAME ${the_module} # prevent name conflict for python2/3 outputs PREFIX "" OUTPUT_NAME cv2 - SUFFIX ${CVPY_SUFFIX}) + SUFFIX "${CVPY_SUFFIX}") if(ENABLE_SOLUTION_FOLDERS) set_target_properties(${the_module} PROPERTIES FOLDER "bindings") From d6306f8ccbcd214dd806a8fed739eff03978ae40 Mon Sep 17 00:00:00 2001 From: Alexander Panov Date: Fri, 20 Aug 2021 13:57:05 +0300 Subject: [PATCH 127/376] Merge pull request #20564 from AleksandrPanov:update_kalman_sample Update kalman sample * updated view and comments, fixed dims * updated view and comments, added statePost --- samples/cpp/kalman.cpp | 62 ++++++++++++++----------- samples/python/kalman.py | 97 ++++++++++++++++++++-------------------- 2 files changed, 85 insertions(+), 74 deletions(-) diff --git a/samples/cpp/kalman.cpp b/samples/cpp/kalman.cpp index 501a749124c6..daf0ba5a7150 100644 --- a/samples/cpp/kalman.cpp +++ b/samples/cpp/kalman.cpp @@ -1,6 +1,6 @@ #include "opencv2/video/tracking.hpp" #include "opencv2/highgui.hpp" - +#include "opencv2/core/cvdef.h" #include using namespace cv; @@ -14,15 +14,19 @@ static void help() { printf( "\nExample of c calls to OpenCV's Kalman filter.\n" " Tracking of rotating point.\n" -" Rotation speed is constant.\n" +" Point moves in a circle and is characterized by a 1D state.\n" +" state_k+1 = state_k + speed + process_noise N(0, 1e-5)\n" +" The speed is constant.\n" " Both state and measurements vectors are 1D (a point angle),\n" -" Measurement is the real point angle + gaussian noise.\n" -" The real and the estimated points are connected with yellow line segment,\n" -" the real and the measured points are connected with red line segment.\n" +" Measurement is the real state + gaussian noise N(0, 1e-1).\n" +" The real and the measured points are connected with red line segment,\n" +" the real and the estimated points are connected with yellow line segment,\n" +" the real and the corrected estimated points are connected with green line segment.\n" " (if Kalman filter works correctly,\n" -" the yellow segment should be shorter than the red one).\n" +" the yellow segment should be shorter than the red one and\n" +" the green segment should be shorter than the yellow one)." "\n" -" Pressing any key (except ESC) will reset the tracking with a different speed.\n" +" Pressing any key (except ESC) will reset the tracking.\n" " Pressing ESC will stop the program.\n" ); } @@ -39,7 +43,9 @@ int main(int, char**) for(;;) { - randn( state, Scalar::all(0), Scalar::all(0.1) ); + img = Scalar::all(0); + state.at(0) = 0.0f; + state.at(1) = 2.f * (float)CV_PI / 6; KF.transitionMatrix = (Mat_(2, 2) << 1, 1, 0, 1); setIdentity(KF.measurementMatrix); @@ -60,36 +66,40 @@ int main(int, char**) double predictAngle = prediction.at(0); Point predictPt = calcPoint(center, R, predictAngle); - randn( measurement, Scalar::all(0), Scalar::all(KF.measurementNoiseCov.at(0))); - // generate measurement + randn( measurement, Scalar::all(0), Scalar::all(KF.measurementNoiseCov.at(0))); measurement += KF.measurementMatrix*state; double measAngle = measurement.at(0); Point measPt = calcPoint(center, R, measAngle); + // correct the state estimates based on measurements + // updates statePost & errorCovPost + KF.correct(measurement); + double improvedAngle = KF.statePost.at(0); + Point improvedPt = calcPoint(center, R, improvedAngle); + // plot points - #define drawCross( center, color, d ) \ - line( img, Point( center.x - d, center.y - d ), \ - Point( center.x + d, center.y + d ), color, 1, LINE_AA, 0); \ - line( img, Point( center.x + d, center.y - d ), \ - Point( center.x - d, center.y + d ), color, 1, LINE_AA, 0 ) - - img = Scalar::all(0); - drawCross( statePt, Scalar(255,255,255), 3 ); - drawCross( measPt, Scalar(0,0,255), 3 ); - drawCross( predictPt, Scalar(0,255,0), 3 ); - line( img, statePt, measPt, Scalar(0,0,255), 3, LINE_AA, 0 ); - line( img, statePt, predictPt, Scalar(0,255,255), 3, LINE_AA, 0 ); - - if(theRNG().uniform(0,4) != 0) - KF.correct(measurement); + img = img * 0.2; + drawMarker(img, measPt, Scalar(0, 0, 255), cv::MARKER_SQUARE, 5, 2); + drawMarker(img, predictPt, Scalar(0, 255, 255), cv::MARKER_SQUARE, 5, 2); + drawMarker(img, improvedPt, Scalar(0, 255, 0), cv::MARKER_SQUARE, 5, 2); + drawMarker(img, statePt, Scalar(255, 255, 255), cv::MARKER_STAR, 10, 1); + // forecast one step + Mat test = Mat(KF.transitionMatrix*KF.statePost); + drawMarker(img, calcPoint(center, R, Mat(KF.transitionMatrix*KF.statePost).at(0)), + Scalar(255, 255, 0), cv::MARKER_SQUARE, 12, 1); + + line( img, statePt, measPt, Scalar(0,0,255), 1, LINE_AA, 0 ); + line( img, statePt, predictPt, Scalar(0,255,255), 1, LINE_AA, 0 ); + line( img, statePt, improvedPt, Scalar(0,255,0), 1, LINE_AA, 0 ); + randn( processNoise, Scalar(0), Scalar::all(sqrt(KF.processNoiseCov.at(0, 0)))); state = KF.transitionMatrix*state + processNoise; imshow( "Kalman", img ); - code = (char)waitKey(100); + code = (char)waitKey(1000); if( code > 0 ) break; diff --git a/samples/python/kalman.py b/samples/python/kalman.py index 654e3de3da0d..cf152a8700fd 100755 --- a/samples/python/kalman.py +++ b/samples/python/kalman.py @@ -1,14 +1,18 @@ #!/usr/bin/env python """ Tracking of rotating point. - Rotation speed is constant. + Point moves in a circle and is characterized by a 1D state. + state_k+1 = state_k + speed + process_noise N(0, 1e-5) + The speed is constant. Both state and measurements vectors are 1D (a point angle), - Measurement is the real point angle + gaussian noise. - The real and the estimated points are connected with yellow line segment, - the real and the measured points are connected with red line segment. + Measurement is the real state + gaussian noise N(0, 1e-1). + The real and the measured points are connected with red line segment, + the real and the estimated points are connected with yellow line segment, + the real and the corrected estimated points are connected with green line segment. (if Kalman filter works correctly, - the yellow segment should be shorter than the red one). - Pressing any key (except ESC) will reset the tracking with a different speed. + the yellow segment should be shorter than the red one and + the green segment should be shorter than the yellow one). + Pressing any key (except ESC) will reset the tracking. Pressing ESC will stop the program. """ # Python 2/3 compatibility @@ -21,8 +25,7 @@ import numpy as np import cv2 as cv -from math import cos, sin, sqrt -import numpy as np +from math import cos, sin, sqrt, pi def main(): img_height = 500 @@ -30,64 +33,62 @@ def main(): kalman = cv.KalmanFilter(2, 1, 0) code = long(-1) - - cv.namedWindow("Kalman") - + num_circle_steps = 12 while True: - state = 0.1 * np.random.randn(2, 1) - - kalman.transitionMatrix = np.array([[1., 1.], [0., 1.]]) - kalman.measurementMatrix = 1. * np.ones((1, 2)) - kalman.processNoiseCov = 1e-5 * np.eye(2) - kalman.measurementNoiseCov = 1e-1 * np.ones((1, 1)) - kalman.errorCovPost = 1. * np.ones((2, 2)) - kalman.statePost = 0.1 * np.random.randn(2, 1) + img = np.zeros((img_height, img_width, 3), np.uint8) + state = np.array([[0.0],[(2 * pi) / num_circle_steps]]) # start state + kalman.transitionMatrix = np.array([[1., 1.], [0., 1.]]) # F. input + kalman.measurementMatrix = 1. * np.eye(1, 2) # H. input + kalman.processNoiseCov = 1e-5 * np.eye(2) # Q. input + kalman.measurementNoiseCov = 1e-1 * np.ones((1, 1)) # R. input + kalman.errorCovPost = 1. * np.eye(2, 2) # P._k|k KF state var + kalman.statePost = 0.1 * np.random.randn(2, 1) # x^_k|k KF state var while True: def calc_point(angle): - return (np.around(img_width/2 + img_width/3*cos(angle), 0).astype(int), - np.around(img_height/2 - img_width/3*sin(angle), 1).astype(int)) - + return (np.around(img_width / 2. + img_width / 3.0 * cos(angle), 0).astype(int), + np.around(img_height / 2. - img_width / 3.0 * sin(angle), 1).astype(int)) + img = img * 1e-3 state_angle = state[0, 0] state_pt = calc_point(state_angle) - + # advance Kalman filter to next timestep + # updates statePre, statePost, errorCovPre, errorCovPost + # k-> k+1, x'(k) = A*x(k) + # P'(k) = temp1*At + Q prediction = kalman.predict() - predict_angle = prediction[0, 0] - predict_pt = calc_point(predict_angle) - - measurement = kalman.measurementNoiseCov * np.random.randn(1, 1) + predict_pt = calc_point(prediction[0, 0]) # equivalent to calc_point(kalman.statePre[0,0]) # generate measurement + measurement = kalman.measurementNoiseCov * np.random.randn(1, 1) measurement = np.dot(kalman.measurementMatrix, state) + measurement measurement_angle = measurement[0, 0] measurement_pt = calc_point(measurement_angle) - # plot points - def draw_cross(center, color, d): - cv.line(img, - (center[0] - d, center[1] - d), (center[0] + d, center[1] + d), - color, 1, cv.LINE_AA, 0) - cv.line(img, - (center[0] + d, center[1] - d), (center[0] - d, center[1] + d), - color, 1, cv.LINE_AA, 0) - - img = np.zeros((img_height, img_width, 3), np.uint8) - draw_cross(np.int32(state_pt), (255, 255, 255), 3) - draw_cross(np.int32(measurement_pt), (0, 0, 255), 3) - draw_cross(np.int32(predict_pt), (0, 255, 0), 3) - - cv.line(img, state_pt, measurement_pt, (0, 0, 255), 3, cv.LINE_AA, 0) - cv.line(img, state_pt, predict_pt, (0, 255, 255), 3, cv.LINE_AA, 0) - + # correct the state estimates based on measurements + # updates statePost & errorCovPost kalman.correct(measurement) + improved_pt = calc_point(kalman.statePost[0, 0]) - process_noise = sqrt(kalman.processNoiseCov[0,0]) * np.random.randn(2, 1) - state = np.dot(kalman.transitionMatrix, state) + process_noise + # plot points + cv.drawMarker(img, measurement_pt, (0, 0, 255), cv.MARKER_SQUARE, 5, 2) + cv.drawMarker(img, predict_pt, (0, 255, 255), cv.MARKER_SQUARE, 5, 2) + cv.drawMarker(img, improved_pt, (0, 255, 0), cv.MARKER_SQUARE, 5, 2) + cv.drawMarker(img, state_pt, (255, 255, 255), cv.MARKER_STAR, 10, 1) + # forecast one step + cv.drawMarker(img, calc_point(np.dot(kalman.transitionMatrix, kalman.statePost)[0, 0]), + (255, 255, 0), cv.MARKER_SQUARE, 12, 1) + + cv.line(img, state_pt, measurement_pt, (0, 0, 255), 1, cv.LINE_AA, 0) # red measurement error + cv.line(img, state_pt, predict_pt, (0, 255, 255), 1, cv.LINE_AA, 0) # yellow pre-meas error + cv.line(img, state_pt, improved_pt, (0, 255, 0), 1, cv.LINE_AA, 0) # green post-meas error + + # update the real process + process_noise = sqrt(kalman.processNoiseCov[0, 0]) * np.random.randn(2, 1) + state = np.dot(kalman.transitionMatrix, state) + process_noise # x_k+1 = F x_k + w_k cv.imshow("Kalman", img) - - code = cv.waitKey(100) + code = cv.waitKey(1000) if code != -1: break From 6801dd043de53030a6f8e36893580951a20dd842 Mon Sep 17 00:00:00 2001 From: rogday Date: Fri, 20 Aug 2021 17:43:47 +0300 Subject: [PATCH 128/376] Merge pull request #20494 from rogday:onnx_diagnostic_fix fix ONNXImporter diagnostic mode layer registration issue * fix layer registration, thread unsafe access and align the behavior of DNN_DIAGNOSTICS_RUN between onnx and tf importers * move skipModelInput * print all missing layers * address TF issue --- apps/model-diagnostics/model_diagnostics.cpp | 2 + .../include/opencv2/dnn/layer_reg.private.hpp | 8 +- .../include/opencv2/dnn/utils/debug_utils.hpp | 24 ++ modules/dnn/src/debug_utils.cpp | 91 ++++++ modules/dnn/src/dnn.cpp | 18 +- modules/dnn/src/dnn_common.hpp | 43 +++ modules/dnn/src/onnx/onnx_importer.cpp | 308 ++++++------------ modules/dnn/src/tensorflow/tf_importer.cpp | 97 +++--- modules/dnn/test/test_tf_importer.cpp | 3 + 9 files changed, 321 insertions(+), 273 deletions(-) create mode 100644 modules/dnn/include/opencv2/dnn/utils/debug_utils.hpp create mode 100644 modules/dnn/src/debug_utils.cpp diff --git a/apps/model-diagnostics/model_diagnostics.cpp b/apps/model-diagnostics/model_diagnostics.cpp index d3934577aec6..6970c8507108 100644 --- a/apps/model-diagnostics/model_diagnostics.cpp +++ b/apps/model-diagnostics/model_diagnostics.cpp @@ -4,6 +4,7 @@ **************************************************/ #include #include +#include #include @@ -57,6 +58,7 @@ int main( int argc, const char** argv ) CV_Assert(!model.empty()); enableModelDiagnostics(true); + skipModelImport(true); redirectError(diagnosticsErrorCallback, NULL); Net ocvNet = readNet(model, config, frameworkId); diff --git a/modules/dnn/include/opencv2/dnn/layer_reg.private.hpp b/modules/dnn/include/opencv2/dnn/layer_reg.private.hpp index 46a58f09bc8b..e944644f8f21 100644 --- a/modules/dnn/include/opencv2/dnn/layer_reg.private.hpp +++ b/modules/dnn/include/opencv2/dnn/layer_reg.private.hpp @@ -12,10 +12,16 @@ CV__DNN_INLINE_NS_BEGIN //! @addtogroup dnn //! @{ -//! Register layer types of DNN model. typedef std::map > LayerFactory_Impl; + +//! Register layer types of DNN model. +//! +//! @note In order to thread-safely access the factory, see getLayerFactoryMutex() function. LayerFactory_Impl& getLayerFactoryImpl(); +//! Get the mutex guarding @ref LayerFactory_Impl, see getLayerFactoryImpl() function. +Mutex& getLayerFactoryMutex(); + //! @} CV__DNN_INLINE_NS_END } diff --git a/modules/dnn/include/opencv2/dnn/utils/debug_utils.hpp b/modules/dnn/include/opencv2/dnn/utils/debug_utils.hpp new file mode 100644 index 000000000000..71dd3ab8d670 --- /dev/null +++ b/modules/dnn/include/opencv2/dnn/utils/debug_utils.hpp @@ -0,0 +1,24 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_DNN_UTILS_DEBUG_UTILS_HPP +#define OPENCV_DNN_UTILS_DEBUG_UTILS_HPP + +#include "../dnn.hpp" + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +/** + * @brief Skip model import after diagnostic run in readNet() functions. + * @param[in] skip Indicates whether to skip the import. + * + * This is an internal OpenCV function not intended for users. + */ +CV_EXPORTS void skipModelImport(bool skip); + +CV__DNN_INLINE_NS_END +}} // namespace + +#endif // OPENCV_DNN_UTILS_DEBUG_UTILS_HPP diff --git a/modules/dnn/src/debug_utils.cpp b/modules/dnn/src/debug_utils.cpp new file mode 100644 index 000000000000..d951205bd876 --- /dev/null +++ b/modules/dnn/src/debug_utils.cpp @@ -0,0 +1,91 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +#include + +#include +#include +#include + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +bool DNN_DIAGNOSTICS_RUN = false; +bool DNN_SKIP_REAL_IMPORT = false; + +void enableModelDiagnostics(bool isDiagnosticsMode) +{ + DNN_DIAGNOSTICS_RUN = isDiagnosticsMode; + + if (DNN_DIAGNOSTICS_RUN) + { + detail::NotImplemented::Register(); + } + else + { + detail::NotImplemented::unRegister(); + } +} + +void skipModelImport(bool skip) +{ + DNN_SKIP_REAL_IMPORT = skip; +} + +void detail::LayerHandler::addMissing(const std::string& name, const std::string& type) +{ + cv::AutoLock lock(getLayerFactoryMutex()); + auto& registeredLayers = getLayerFactoryImpl(); + + // If we didn't add it, but can create it, it's custom and not missing. + if (layers.find(type) == layers.end() && registeredLayers.find(type) != registeredLayers.end()) + { + return; + } + + layers[type].insert(name); +} + +bool detail::LayerHandler::contains(const std::string& type) const +{ + return layers.find(type) != layers.end(); +} + +void detail::LayerHandler::printMissing() +{ + if (layers.empty()) + { + return; + } + + std::stringstream ss; + ss << "DNN: Not supported types:\n"; + for (const auto& type_names : layers) + { + const auto& type = type_names.first; + ss << "Type='" << type << "', affected nodes:\n["; + for (const auto& name : type_names.second) + { + ss << "'" << name << "', "; + } + ss.seekp(-2, std::ios_base::end); + ss << "]\n"; + } + CV_LOG_ERROR(NULL, ss.str()); +} + +LayerParams detail::LayerHandler::getNotImplementedParams(const std::string& name, const std::string& op) +{ + LayerParams lp; + lp.name = name; + lp.type = "NotImplemented"; + lp.set("type", op); + + return lp; +} + +CV__DNN_INLINE_NS_END +}} // namespace diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 492ad166d038..4e38b0374f00 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -94,22 +94,6 @@ static bool DNN_CHECK_NAN_INF = utils::getConfigurationParameterBool("OPENCV_DNN static bool DNN_CHECK_NAN_INF_DUMP = utils::getConfigurationParameterBool("OPENCV_DNN_CHECK_NAN_INF_DUMP", false); static bool DNN_CHECK_NAN_INF_RAISE_ERROR = utils::getConfigurationParameterBool("OPENCV_DNN_CHECK_NAN_INF_RAISE_ERROR", false); -bool DNN_DIAGNOSTICS_RUN = false; - -void enableModelDiagnostics(bool isDiagnosticsMode) -{ - DNN_DIAGNOSTICS_RUN = isDiagnosticsMode; - - if (DNN_DIAGNOSTICS_RUN) - { - detail::NotImplemented::Register(); - } - else - { - detail::NotImplemented::unRegister(); - } -} - using std::vector; using std::map; using std::make_pair; @@ -5662,7 +5646,7 @@ bool Layer::updateMemoryShapes(const std::vector &inputs) } ////////////////////////////////////////////////////////////////////////// -static Mutex& getLayerFactoryMutex() +Mutex& getLayerFactoryMutex() { static Mutex* volatile instance = NULL; if (instance == NULL) diff --git a/modules/dnn/src/dnn_common.hpp b/modules/dnn/src/dnn_common.hpp index 591be88079f3..3c68322e098c 100644 --- a/modules/dnn/src/dnn_common.hpp +++ b/modules/dnn/src/dnn_common.hpp @@ -5,6 +5,9 @@ #ifndef __OPENCV_DNN_COMMON_HPP__ #define __OPENCV_DNN_COMMON_HPP__ +#include +#include + #include namespace cv { namespace dnn { @@ -13,6 +16,9 @@ CV__DNN_INLINE_NS_BEGIN Mutex& getInitializationMutex(); void initializeLayerFactory(); +extern bool DNN_DIAGNOSTICS_RUN; +extern bool DNN_SKIP_REAL_IMPORT; + namespace detail { #define CALL_MEMBER_FN(object, ptrToMemFn) ((object).*(ptrToMemFn)) @@ -25,6 +31,43 @@ class NotImplemented : public Layer static void unRegister(); }; +template +Net readNet(Args&& ... args) +{ + Net net; + Importer importer(net, std::forward(args)...); + return net; +} + +template +Net readNetDiagnostic(Args&& ... args) +{ + Net maybeDebugNet = readNet(std::forward(args)...); + if (DNN_DIAGNOSTICS_RUN && !DNN_SKIP_REAL_IMPORT) + { + // if we just imported the net in diagnostic mode, disable it and import again + enableModelDiagnostics(false); + Net releaseNet = readNet(std::forward(args)...); + enableModelDiagnostics(true); + return releaseNet; + } + return maybeDebugNet; +} + +class LayerHandler +{ +public: + void addMissing(const std::string& name, const std::string& type); + bool contains(const std::string& type) const; + void printMissing(); + +protected: + LayerParams getNotImplementedParams(const std::string& name, const std::string& op); + +private: + std::unordered_map> layers; +}; + struct NetImplBase { const int networkId; // network global identifier diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 3379ea3a0bb6..8fc179037cde 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -41,6 +41,8 @@ CV__DNN_INLINE_NS_BEGIN extern bool DNN_DIAGNOSTICS_RUN; +class ONNXLayerHandler; + class ONNXImporter { opencv_onnx::ModelProto model_proto; @@ -61,60 +63,16 @@ class ONNXImporter void addConstant(const std::string& name, const Mat& blob); void addLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - static const std::set& getSupportedTypes(); public: - - ONNXImporter(Net& net, const char *onnxFile) - : dstNet(net), utilNet(), dispatch(buildDispatchMap()) - { - hasDynamicShapes = false; - CV_Assert(onnxFile); - CV_LOG_DEBUG(NULL, "DNN/ONNX: processing ONNX model from file: " << onnxFile); - - std::fstream input(onnxFile, std::ios::in | std::ios::binary); - if (!input) - { - CV_Error(Error::StsBadArg, cv::format("Can't read ONNX file: %s", onnxFile)); - } - - if (!model_proto.ParseFromIstream(&input)) - { - CV_Error(Error::StsUnsupportedFormat, cv::format("Failed to parse ONNX model: %s", onnxFile)); - } - - populateNet(); - } - - ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer) - : dstNet(net), utilNet(), dispatch(buildDispatchMap()) - { - hasDynamicShapes = false; - CV_LOG_DEBUG(NULL, "DNN/ONNX: processing in-memory ONNX model (" << sizeBuffer << " bytes)"); - - struct _Buf : public std::streambuf - { - _Buf(const char* buffer, size_t sizeBuffer) - { - char* p = const_cast(buffer); - setg(p, p, p + sizeBuffer); - } - }; - - _Buf buf(buffer, sizeBuffer); - std::istream input(&buf); - - if (!model_proto.ParseFromIstream(&input)) - CV_Error(Error::StsUnsupportedFormat, "Failed to parse onnx model from in-memory byte array."); - - populateNet(); - } + ONNXImporter(Net& net, const char *onnxFile); + ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer); void populateNet(); protected: + std::unique_ptr layerHandler; Net& dstNet; - Net utilNet; opencv_onnx::GraphProto graph_proto; std::string framework_name; @@ -131,9 +89,13 @@ class ONNXImporter void handleNode(const opencv_onnx::NodeProto& node_proto); private: + friend class ONNXLayerHandler; typedef void (ONNXImporter::*ONNXImporterNodeParser)(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); typedef std::map DispatchMap; + const DispatchMap dispatch; + static const DispatchMap buildDispatchMap(); + void parseMaxPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseAveragePool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseReduce (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -178,12 +140,84 @@ class ONNXImporter void parseSoftMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseCumSum (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - void parseCustom (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - const DispatchMap dispatch; - static const DispatchMap buildDispatchMap(); + void parseCustomLayer (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); +}; + +class ONNXLayerHandler : public detail::LayerHandler +{ +public: + explicit ONNXLayerHandler(ONNXImporter* importer_); + + void fillRegistry(const opencv_onnx::GraphProto& net); + +protected: + ONNXImporter* importer; }; +ONNXLayerHandler::ONNXLayerHandler(ONNXImporter* importer_) : importer(importer_){} + +void ONNXLayerHandler::fillRegistry(const opencv_onnx::GraphProto &net) +{ + int layersSize = net.node_size(); + for (int li = 0; li < layersSize; li++) { + const opencv_onnx::NodeProto &node_proto = net.node(li); + const std::string& name = node_proto.output(0); + const std::string& type = node_proto.op_type(); + if (importer->dispatch.find(type) == importer->dispatch.end()) + { + addMissing(name, type); + } + } + printMissing(); +} + +ONNXImporter::ONNXImporter(Net& net, const char *onnxFile) + : layerHandler(DNN_DIAGNOSTICS_RUN ? new ONNXLayerHandler(this) : nullptr), + dstNet(net), dispatch(buildDispatchMap()) +{ + hasDynamicShapes = false; + CV_Assert(onnxFile); + CV_LOG_DEBUG(NULL, "DNN/ONNX: processing ONNX model from file: " << onnxFile); + + std::fstream input(onnxFile, std::ios::in | std::ios::binary); + if (!input) + { + CV_Error(Error::StsBadArg, cv::format("Can't read ONNX file: %s", onnxFile)); + } + + if (!model_proto.ParseFromIstream(&input)) + { + CV_Error(Error::StsUnsupportedFormat, cv::format("Failed to parse ONNX model: %s", onnxFile)); + } + + populateNet(); +} + +ONNXImporter::ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer) + : layerHandler(DNN_DIAGNOSTICS_RUN ? new ONNXLayerHandler(this) : nullptr), dstNet(net), dispatch(buildDispatchMap()) +{ + hasDynamicShapes = false; + CV_LOG_DEBUG(NULL, "DNN/ONNX: processing in-memory ONNX model (" << sizeBuffer << " bytes)"); + + struct _Buf : public std::streambuf + { + _Buf(const char* buffer, size_t sizeBuffer) + { + char* p = const_cast(buffer); + setg(p, p, p + sizeBuffer); + } + }; + + _Buf buf(buffer, sizeBuffer); + std::istream input(&buf); + + if (!model_proto.ParseFromIstream(&input)) + CV_Error(Error::StsUnsupportedFormat, "Failed to parse onnx model from in-memory byte array."); + + populateNet(); +} + inline void replaceLayerParam(LayerParams& layerParams, const String& oldKey, const String& newKey) { if (layerParams.has(oldKey)) { @@ -422,11 +456,7 @@ Mat ONNXImporter::getBlob(const std::string& input_name) void ONNXImporter::addLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - int id; - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(layerParams.name, layerParams.type, layerParams); - else - id = dstNet.addLayer(layerParams.name, layerParams.type, layerParams); + int id = dstNet.addLayer(layerParams.name, layerParams.type, layerParams); for (int i = 0; i < node_proto.output_size(); ++i) { layer_id.insert(std::make_pair(node_proto.output(i), LayerInfo(id, i))); @@ -439,10 +469,7 @@ void ONNXImporter::addLayer(LayerParams& layerParams, const std::string& input_name = node_proto.input(j); IterLayerId_t layerId = layer_id.find(input_name); if (layerId != layer_id.end()) { - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, inpNum); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, inpNum); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, inpNum); ++inpNum; // Collect input shapes. IterShape_t shapeIt = outShapes.find(input_name); @@ -451,11 +478,7 @@ void ONNXImporter::addLayer(LayerParams& layerParams, } } // Compute shape of output blob for this layer. - Ptr layer; - if (DNN_DIAGNOSTICS_RUN) - layer = utilNet.getLayer(id); - else - layer = dstNet.getLayer(id); // FIXIT: avoid instantiation of layers during the import stage + Ptr layer = dstNet.getLayer(id); // FIXIT: avoid instantiation of layers during the import stage layer->getMemoryShapes(layerInpShapes, 0, layerOutShapes, layerInternalShapes); for (int i = 0; i < node_proto.output_size() && i < (int)layerOutShapes.size(); ++i) { @@ -532,35 +555,11 @@ void ONNXImporter::populateNet() layer_id.insert(std::make_pair(name, LayerInfo(0, netInputs.size() - 1))); } } - utilNet.setInputsNames(netInputs); dstNet.setInputsNames(netInputs); if (DNN_DIAGNOSTICS_RUN) { - auto &supportedTypes = getSupportedTypes(); - for (int li = 0; li < layersSize; li++) { - const opencv_onnx::NodeProto &node_proto = graph_proto.node(li); - std::string name = node_proto.output(0); - std::string layer_type = node_proto.op_type(); - auto registered = supportedTypes.find(layer_type); - if (registered == supportedTypes.end()) { - CV_LOG_ERROR(NULL, "DNN/ONNX: NOTE: Potential problem with creating node " << name<< " with type " << layer_type << ".\n Type " - << layer_type << " IS NOT SUPPORTED!\n" - ); - } - } - auto oldConstBlobs = constBlobs; - auto oldOutShapes = outShapes; - auto oldLayerId = layer_id; CV_LOG_INFO(NULL, "DNN/ONNX: start diagnostic run!"); - for (int li = 0; li < layersSize; li++) { - const opencv_onnx::NodeProto &node_proto = graph_proto.node(li); - handleNode(node_proto); - } - CV_LOG_INFO(NULL, "DNN/ONNX: diagnostic run completed!"); - constBlobs = oldConstBlobs; - outShapes = oldOutShapes; - layer_id = oldLayerId; - enableModelDiagnostics(false); + layerHandler->fillRegistry(graph_proto); } for(int li = 0; li < layersSize; li++) @@ -569,83 +568,7 @@ void ONNXImporter::populateNet() handleNode(node_proto); } - CV_LOG_DEBUG(NULL, "DNN/ONNX: import completed!"); -} - -const std::set& ONNXImporter::getSupportedTypes() -{ - static const std::set layerTypes = { - "MaxPool", - "AveragePool", - "GlobalAveragePool", - "GlobalMaxPool", - "ReduceMean", - "ReduceSum", - "ReduceMax", - "Slice", - "Split", - "Add", - "Sum", - "Sub", - "Pow", - "Max", - "Neg", - "Constant", - "LSTM", - "GRU", - "ImageScaler", - "Clip", - "LeakyRelu", - "Relu", - "Elu", - "Tanh", - "PRelu", - "LRN", - "InstanceNormalization", - "BatchNormalization", - "Gemm", - "MatMul", - "Mul", - "Div", - "Conv", - "ConvTranspose", - "Transpose", - "Squeeze", - "Flatten", - "Unsqueeze", - "Expand", - "Reshape", - "Pad", - "Shape", - "Cast", - "ConstantOfShape", - "ConstantFill", - "Gather", - "Concat", - "Resize", - "Upsample", - "SoftMax", - "Softmax", - "LogSoftmax", - "DetectionOutput", - "Interp", - "CropAndResize", - "ROIPooling", - "PSROIPooling", - "ChannelsPReLU", - "Sigmoid", - "Swish", - "Mish", - "AbsVal", - "BNLL", - "MaxUnpool", - "Dropout", - "Identity", - "Crop", - "Normalize", - "CumSum" - }; - return layerTypes; + CV_LOG_DEBUG(NULL, (DNN_DIAGNOSTICS_RUN ? "DNN/ONNX: diagnostic run completed!" : "DNN/ONNX: import completed!")); } void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) @@ -673,7 +596,7 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) } else { - parseCustom(layerParams, node_proto); + parseCustomLayer(layerParams, node_proto); } } catch (const cv::Exception& e) @@ -683,6 +606,7 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) CV_LOG_ERROR(NULL, "DNN/ONNX: Potential problem during processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) << "\n" << e.msg ); + cv::AutoLock lock(getLayerFactoryMutex()); auto registeredLayers = getLayerFactoryImpl(); if (registeredLayers.find(layerParams.type) != registeredLayers.end()) { @@ -1068,11 +992,7 @@ void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodePr constParams.name = layerParams.name + "/const"; constParams.type = "Const"; constParams.blobs.push_back((isSub ? -1 : 1) * blob); - int id; - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(constParams.name, constParams.type, constParams); - else - id = dstNet.addLayer(constParams.name, constParams.type, constParams); + int id = dstNet.addLayer(constParams.name, constParams.type, constParams); layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); outShapes[constParams.name] = shape(blob); @@ -1117,19 +1037,12 @@ void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodePr powerParams.type = "Power"; powerParams.set("scale", -1); - int id; //Create Power layer - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(powerParams.name, powerParams.type, powerParams); - else - id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); //Connect to input IterLayerId_t layerId = layer_id.find(node_proto.input(1)); CV_Assert(layerId != layer_id.end()); - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); //Add shape layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); outShapes[powerParams.name] = outShapes[node_proto.input(1)]; @@ -1404,18 +1317,11 @@ void ONNXImporter::parseInstanceNormalization(LayerParams& layerParams, const op layerParams.erase("epsilon"); //Create MVN layer - int id; - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); - else - id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); + int id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); //Connect to input IterLayerId_t layerId = layer_id.find(node_proto.input(0)); CV_Assert(layerId != layer_id.end()); - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); //Add shape layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0))); outShapes[mvnParams.name] = outShapes[node_proto.input(0)]; @@ -1621,19 +1527,12 @@ void ONNXImporter::parseMul(LayerParams& layerParams, const opencv_onnx::NodePro powerParams.type = "Power"; powerParams.set("power", -1); - int id; //Create Power layer - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(powerParams.name, powerParams.type, powerParams); - else - id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); //Connect to input IterLayerId_t layerId = layer_id.find(node_proto.input(1)); CV_Assert(layerId != layer_id.end()); - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); //Add shape layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); outShapes[powerParams.name] = outShapes[node_proto.input(1)]; @@ -2418,7 +2317,7 @@ void ONNXImporter::parseCumSum(LayerParams& layerParams, const opencv_onnx::Node addLayer(layerParams, node_proto); } -void ONNXImporter::parseCustom(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +void ONNXImporter::parseCustomLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { for (int j = 0; j < node_proto.input_size(); j++) { if (layer_id.find(node_proto.input(j)) == layer_id.end()) @@ -2476,23 +2375,18 @@ const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() dispatch["SoftMax"] = dispatch["LogSoftmax"] = &ONNXImporter::parseSoftMax; dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput; dispatch["CumSum"] = &ONNXImporter::parseCumSum; - dispatch["Custom"] = &ONNXImporter::parseCustom; return dispatch; } Net readNetFromONNX(const String& onnxFile) { - Net net; - ONNXImporter onnxImporter(net, onnxFile.c_str()); - return net; + return detail::readNetDiagnostic(onnxFile.c_str()); } Net readNetFromONNX(const char* buffer, size_t sizeBuffer) { - Net net; - ONNXImporter onnxImporter(net, buffer, sizeBuffer); - return net; + return detail::readNetDiagnostic(buffer, sizeBuffer); } Net readNetFromONNX(const std::vector& buffer) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index de7ec3dfccb1..f87988d0a117 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -507,7 +507,7 @@ void ExcludeLayer(tensorflow::GraphDef& net, const int layer_index, const int in net.mutable_node()->DeleteSubrange(layer_index, 1); } -class LayerHandler; +class TFLayerHandler; class TFImporter { @@ -516,8 +516,7 @@ class TFImporter TFImporter(Net& net, const char *dataModel, size_t lenModel, const char *dataConfig = NULL, size_t lenConfig = 0); protected: - std::unique_ptr layerHandler; - std::unique_ptr utilNet; + std::unique_ptr layerHandler; Net& dstNet; void populateNet(); @@ -559,7 +558,7 @@ class TFImporter void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId); void setPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer, std::string& inputName, float value = 0.); - friend class LayerHandler; + friend class TFLayerHandler; typedef void (TFImporter::*TFImporterNodeParser)(tensorflow::GraphDef&, const tensorflow::NodeDef&, LayerParams&); typedef std::map DispatchMap; @@ -625,18 +624,17 @@ void TFImporter::setPadding(LayerParams &layerParams, const tensorflow::NodeDef layerParams.set("pad_mode", "VALID"); } -class LayerHandler +class TFLayerHandler : public detail::LayerHandler { public: - LayerHandler(TFImporter* importer_); - ~LayerHandler() = default; + explicit TFLayerHandler(TFImporter* importer_); - bool handleMissing(const opencv_tensorflow::NodeDef& layer); - void handleFailed(const opencv_tensorflow::NodeDef& layer); + void fillRegistry(const tensorflow::GraphDef& net); + bool handleMissing(const tensorflow::NodeDef& layer); + void handleFailed(const tensorflow::NodeDef& layer); -private: +protected: TFImporter* importer; - std::set layers; }; const TFImporter::DispatchMap TFImporter::buildDispatchMap() @@ -2471,9 +2469,8 @@ void TFImporter::parseCustomLayer(tensorflow::GraphDef& net, const tensorflow::N } TFImporter::TFImporter(Net& net, const char *model, const char *config) - : layerHandler(DNN_DIAGNOSTICS_RUN ? new LayerHandler(this) : nullptr), - utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), - dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap()) + : layerHandler(DNN_DIAGNOSTICS_RUN ? new TFLayerHandler(this) : nullptr), + dstNet(net), dispatch(buildDispatchMap()) { if (model && model[0]) { @@ -2494,9 +2491,8 @@ TFImporter::TFImporter( const char *dataModel, size_t lenModel, const char *dataConfig, size_t lenConfig ) - : layerHandler(DNN_DIAGNOSTICS_RUN ? new LayerHandler(this) : nullptr), - utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), - dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap()) + : layerHandler(DNN_DIAGNOSTICS_RUN ? new TFLayerHandler(this) : nullptr), + dstNet(net), dispatch(buildDispatchMap()) { if (dataModel != NULL && lenModel > 0) { @@ -2855,6 +2851,11 @@ void TFImporter::populateNet() addConstNodes(netBin, value_id, layers_to_ignore); addConstNodes(netTxt, value_id, layers_to_ignore); + if (DNN_DIAGNOSTICS_RUN) { + CV_LOG_INFO(NULL, "DNN/TF: start diagnostic run!"); + layerHandler->fillRegistry(net); + } + for (int li = 0; li < layersSize; li++) { const tensorflow::NodeDef& layer = net.node(li); @@ -2873,7 +2874,7 @@ void TFImporter::populateNet() CV_Assert(!netInputsNames[i].empty()); } dstNet.setInputsNames(netInputsNames); - CV_LOG_DEBUG(NULL, "DNN/TF: ===================== Import completed ====================="); + CV_LOG_DEBUG(NULL, (DNN_DIAGNOSTICS_RUN? "DNN/TF: diagnostic run completed!" : "DNN/TF: import completed!")); } void TFImporter::addPermuteLayer(const int* order, const std::string& permName, Pin& inpId) @@ -2933,41 +2934,45 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer) } } -LayerHandler::LayerHandler(TFImporter* importer_) : importer(importer_) {} +TFLayerHandler::TFLayerHandler(TFImporter* importer_) : importer(importer_) {} -void LayerHandler::handleFailed(const opencv_tensorflow::NodeDef& layer) +void TFLayerHandler::fillRegistry(const tensorflow::GraphDef& net) { - LayerParams lp; - lp.name = layer.name(); - lp.type = "NotImplemented"; - lp.set("type", layer.op()); + for (int li = 0; li < net.node_size(); li++) { + const tensorflow::NodeDef& layer = net.node(li); - // the layer will be created or its params and type will be replaced - int id = importer->dstNet.addLayer(lp.name, "NotImplemented", lp); - if (id != -1) // internal layer failure before the call to addLayer() - { - importer->layer_id[lp.name] = id; + const std::string& name = layer.name(); + const std::string& type = layer.op(); + if (importer->dispatch.find(type) == importer->dispatch.end()) + { + addMissing(name, type); + } } -} + printMissing(); +}; -bool LayerHandler::handleMissing(const opencv_tensorflow::NodeDef& layer) +bool TFLayerHandler::handleMissing(const tensorflow::NodeDef& layer) { - LayerParams lp; - // If we didn't add it, but can create it, it's custom and not missing. - if (layers.find(layer.op()) == layers.end() && LayerFactory::createLayerInstance(layer.op(), lp)) - { - return false; - } + bool unsupported = contains(layer.op()); - if (layers.insert(layer.op()).second) + if (unsupported) { - CV_LOG_ERROR(NULL, "DNN/TF: Node='" << layer.name() << "' of type='"<< layer.op() - << "' is not supported. This error won't be displayed again."); + handleFailed(layer); } - handleFailed(layer); + return unsupported; +} - return true; +void TFLayerHandler::handleFailed(const tensorflow::NodeDef& layer) +{ + LayerParams lp = getNotImplementedParams(layer.name(), layer.op()); + + // the layer will be created or its params and type will be replaced + int id = importer->dstNet.addLayer(lp.name, lp.type, lp); + if (id != -1) // internal layer failure before the call to addLayer() + { + importer->layer_id[lp.name] = id; + } } } // namespace @@ -2976,17 +2981,13 @@ bool LayerHandler::handleMissing(const opencv_tensorflow::NodeDef& layer) Net readNetFromTensorflow(const String &model, const String &config) { - Net net; - TFImporter importer(net, model.c_str(), config.c_str()); - return net; + return detail::readNetDiagnostic(model.c_str(), config.c_str()); } Net readNetFromTensorflow(const char* bufferModel, size_t lenModel, const char* bufferConfig, size_t lenConfig) { - Net net; - TFImporter importer(net, bufferModel, lenModel, bufferConfig, lenConfig); - return net; + return detail::readNetDiagnostic(bufferModel, lenModel, bufferConfig, lenConfig); } Net readNetFromTensorflow(const std::vector& bufferModel, const std::vector& bufferConfig) diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 3f33f16774b2..3d53ced0a450 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -13,6 +13,7 @@ Test for Tensorflow models loading #include "npy_blob.hpp" #include // CV_DNN_REGISTER_LAYER_CLASS +#include namespace opencv_test { @@ -605,11 +606,13 @@ class Test_TensorFlow_diagnostics : public DNNTestLayer { Test_TensorFlow_diagnostics() { enableModelDiagnostics(true); + skipModelImport(true); } ~Test_TensorFlow_diagnostics() { enableModelDiagnostics(false); + skipModelImport(false); } void runFailingTensorFlowNet(const std::string& prefix, bool hasText = false) From f28e4b86fbc6bf361c2e6c7a36fd96c552a2aab7 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 21 Aug 2021 16:04:13 +0000 Subject: [PATCH 129/376] dnn(ocl): fix top initialization in verifyResult --- modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index fd989193431d..ef7c380c1be5 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -1257,8 +1257,11 @@ bool OCL4DNNConvSpatial::verifyResult(const UMat &bottom, else if (config->tested) return false; - int32_t sz[4] = {numImages, num_output_, output_h_, output_w_}; - top.zeros(4, sz, (use_half_) ? CV_16SC1 : CV_32FC1); + //int32_t sz[4] = {numImages, num_output_, output_h_, output_w_}; + CV_CheckEQ(top.total(), (size_t)numImages * num_output_ * output_h_ * output_w_, ""); + CV_CheckTypeEQ(top.type(), (use_half_) ? CV_16SC1 : CV_32FC1, ""); + top.setTo(Scalar::all(0)); + bool saved_tuned = tuned_; tuned_ = false; convolve(bottom, top, weight, bias, numImages, config); From aa5c4945d602452737c32168021847ee1593d914 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Mon, 23 Aug 2021 16:52:49 +0300 Subject: [PATCH 130/376] Fix GExecutor WriteBackExec --- modules/gapi/src/executor/gexecutor.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/gapi/src/executor/gexecutor.cpp b/modules/gapi/src/executor/gexecutor.cpp index 9b51e70d5dae..2ca675c7df05 100644 --- a/modules/gapi/src/executor/gexecutor.cpp +++ b/modules/gapi/src/executor/gexecutor.cpp @@ -157,10 +157,14 @@ void writeBackExec(const Mag& mag, const RcDesc &rc, GRunArgP &g_arg) // FIXME: // Rework, find a better way to check if there should be // a real copy (add a pass to StreamingBackend?) + // NB: In case RMat adapter not equal to "RMatAdapter" need to + // copy data back to the host as well. + // FIXME: Rename "RMatAdapter" to "OpenCVAdapter". auto& out_mat = *util::get(g_arg); const auto& rmat = mag.template slot().at(rc.id); auto* adapter = rmat.get(); - if (adapter != nullptr && out_mat.data != adapter->data()) { + if ((adapter != nullptr && out_mat.data != adapter->data()) || + (adapter == nullptr)) { auto view = rmat.access(RMat::Access::R); asMat(view).copyTo(out_mat); } From 350562919cdda295fafad9d94a47479441736d28 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 23 Aug 2021 18:10:46 +0000 Subject: [PATCH 131/376] highgui(win32): avoid using of stalled iterator --- modules/highgui/src/window_w32.cpp | 42 ++++++++++++++++-------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/modules/highgui/src/window_w32.cpp b/modules/highgui/src/window_w32.cpp index 716af1094c29..43a19baf8933 100644 --- a/modules/highgui/src/window_w32.cpp +++ b/modules/highgui/src/window_w32.cpp @@ -278,6 +278,21 @@ std::shared_ptr icvFindWindowByName(const char* name) return icvFindWindowByName(std::string(name)); } +// Mutex must be locked +static +std::shared_ptr icvFindWindowByHandle(HWND hwnd) +{ + auto& g_windows = getWindowsList(); + for (auto it = g_windows.begin(); it != g_windows.end(); ++it) + { + auto window = *it; + if (!window) + continue; + if (window->hwnd == hwnd || window->frame == hwnd) + return window; + } + return std::shared_ptr(); +} static LRESULT CALLBACK HighGUIProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); @@ -2187,22 +2202,15 @@ static void showSaveDialog(CvWindow& window) */ static bool handleMessage(MSG& message, int& keyCode) { - // whether we have to call translate and dispatch yet - // otherwise the message was handled specifically - bool is_processed = false; - - AutoLock lock(getWindowMutex()); - auto& g_windows = getWindowsList(); - for (auto it = g_windows.begin(); it != g_windows.end() && !is_processed; ++it) + std::shared_ptr window_; + { + AutoLock lock(getWindowMutex()); + window_ = icvFindWindowByHandle(message.hwnd); + } + if (window_) { - auto window_ = *it; - if (!window_) - continue; CvWindow& window = *window_; - if (!(window.hwnd == message.hwnd || window.frame == message.hwnd)) - continue; - is_processed = true; switch (message.message) { case WM_DESTROY: @@ -2215,7 +2223,6 @@ static bool handleMessage(MSG& message, int& keyCode) case WM_SYSKEYDOWN: if (message.wParam == VK_F10) { - is_processed = true; keyCode = (int)(message.wParam << 16); return true; } @@ -2245,7 +2252,6 @@ static bool handleMessage(MSG& message, int& keyCode) message.wParam == VK_PRIOR || message.wParam == VK_NEXT) { DispatchMessage(&message); - is_processed = true; keyCode = (int)(message.wParam << 16); return true; } @@ -2254,18 +2260,16 @@ static bool handleMessage(MSG& message, int& keyCode) default: DispatchMessage(&message); - is_processed = true; break; } } - - if (!is_processed) + else { TranslateMessage(&message); DispatchMessage(&message); } - return false; // no value to return, keep processing + return false; // no keyCode to return, keep processing } /* From 2f31763335ee4e53cb1a66c9298976ec00b5c160 Mon Sep 17 00:00:00 2001 From: HAN Liutong Date: Tue, 24 Aug 2021 11:42:19 +0800 Subject: [PATCH 132/376] fix v_reduce_sum --- modules/core/include/opencv2/core/hal/intrin_rvv.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp index 5b3319378103..55e0d88f592a 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp @@ -1399,8 +1399,8 @@ OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint16x8, v_uint32x4, vuint32m1_t, unsigned, u1 OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int16x8, v_int32x4, vint32m1_t, int, i16, i32, 8, wredsum) OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint32x4, v_uint64x2, vuint64m1_t, unsigned, u32, u64, 4, wredsumu) OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int32x4, v_int64x2, vint64m1_t, int, i32, i64, 4, wredsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint64x2, v_uint64x2, vuint64m1_t, uint64, u64, u64, 4, redsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int64x2, v_int64x2, vint64m1_t, int64, i64, i64, 4, redsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint64x2, v_uint64x2, vuint64m1_t, uint64, u64, u64, 2, redsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int64x2, v_int64x2, vint64m1_t, int64, i64, i64, 2, redsum) #define OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(_Tpvec, _wTpvec, _nwTpvec, scalartype, suffix, wsuffix, vl, red) \ inline scalartype v_reduce_sum(const _Tpvec& a) \ @@ -1411,9 +1411,9 @@ inline scalartype v_reduce_sum(const _Tpvec& a) \ return (scalartype)(_wTpvec(res).get0()); \ } -OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float32x4, v_float32x4, vfloat32m1_t, float, f32, f32, 8, fredsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float32x4, v_float32x4, vfloat32m1_t, float, f32, f32, 4, fredsum) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float64x2, v_float64x2, vfloat64m1_t, double, f64, f64, 4, fredsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float64x2, v_float64x2, vfloat64m1_t, double, f64, f64, 2, fredsum) #endif From fdaa6ff9e35c5d035b208582db1cc78de5c46d8a Mon Sep 17 00:00:00 2001 From: Nicolai Behmann Date: Tue, 24 Aug 2021 11:31:54 +0200 Subject: [PATCH 133/376] Merge pull request #20475 from nibeh:patch-1 * Added exposure and gain props, maximized pixel clk * removed pixel clock maximization pixel clock maximization is not suitable for all use cases, so I removed it from PR. --- modules/videoio/src/cap_ueye.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/modules/videoio/src/cap_ueye.cpp b/modules/videoio/src/cap_ueye.cpp index eadba1554689..efc2cb990e9c 100644 --- a/modules/videoio/src/cap_ueye.cpp +++ b/modules/videoio/src/cap_ueye.cpp @@ -171,6 +171,13 @@ double VideoCapture_uEye::getProperty(int property_id) const case CAP_PROP_FPS: value = fps; break; + case CAP_PROP_EXPOSURE: + ASSERT_UEYE(is_Exposure(cam_id, IS_EXPOSURE_CMD_GET_EXPOSURE, (void*)&value, sizeof(value))); + break; + case CAP_PROP_GAIN: + auto gain = is_SetHWGainFactor(cam_id, IS_GET_MASTER_GAIN_FACTOR, 100); + value = static_cast(gain)/100.0; + break; } return value; } @@ -201,6 +208,12 @@ bool VideoCapture_uEye::setProperty(int property_id, double value) break; ASSERT_UEYE(is_SetFrameRate(cam_id, value, &fps)); break; + case CAP_PROP_EXPOSURE: + ASSERT_UEYE(is_Exposure(cam_id, IS_EXPOSURE_CMD_SET_EXPOSURE, (void*)&value, sizeof(value))); + break; + case CAP_PROP_GAIN: + is_SetHWGainFactor(cam_id, IS_SET_MASTER_GAIN_FACTOR, static_cast(value)); + break; } if(set_format) { From 5ad6ff239bed8d9bfbd4e176a49ea1213989d5a6 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Tue, 24 Aug 2021 12:37:50 +0300 Subject: [PATCH 134/376] Merge pull request #20555 from TolyaTalamanov:at/fix-compileStreaming-bug [G-API] Extend compileStreaming to support different overloads * Make different overloads * Order python compileStreaming overloads * Fix compileStreaming bug * Replace gin -> descr_of * Set error message * Fix review comments * Use macros for pyopencv_to GMetaArgs * Use GAPI_PROP_RW * Not split Prims python stuff --- .../include/opencv2/gapi/gcomputation.hpp | 10 +-- .../gapi/include/opencv2/gapi/gstreaming.hpp | 4 +- .../gapi/misc/python/package/gapi/__init__.py | 2 + modules/gapi/misc/python/pyopencv_gapi.hpp | 70 +++++++++++-------- modules/gapi/misc/python/shadow_gapi.hpp | 5 +- .../misc/python/test/test_gapi_streaming.py | 44 ++++++++++++ 6 files changed, 98 insertions(+), 37 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/gcomputation.hpp b/modules/gapi/include/opencv2/gapi/gcomputation.hpp index a3566fb49521..7e07a587e15f 100644 --- a/modules/gapi/include/opencv2/gapi/gcomputation.hpp +++ b/modules/gapi/include/opencv2/gapi/gcomputation.hpp @@ -437,11 +437,7 @@ class GAPI_EXPORTS_W GComputation * * @sa @ref gapi_compile_args */ - GStreamingCompiled compileStreaming(GMetaArgs &&in_metas, GCompileArgs &&args = {}); - - /// @private -- Exclude this function from OpenCV documentation - GAPI_WRAP GStreamingCompiled compileStreaming(const cv::detail::ExtractMetaCallback &callback, - GCompileArgs &&args = {}); + GAPI_WRAP GStreamingCompiled compileStreaming(GMetaArgs &&in_metas, GCompileArgs &&args = {}); /** * @brief Compile the computation for streaming mode. @@ -464,6 +460,10 @@ class GAPI_EXPORTS_W GComputation */ GAPI_WRAP GStreamingCompiled compileStreaming(GCompileArgs &&args = {}); + /// @private -- Exclude this function from OpenCV documentation + GAPI_WRAP GStreamingCompiled compileStreaming(const cv::detail::ExtractMetaCallback &callback, + GCompileArgs &&args = {}); + // 2. Direct metadata version /** * @overload diff --git a/modules/gapi/include/opencv2/gapi/gstreaming.hpp b/modules/gapi/include/opencv2/gapi/gstreaming.hpp index 50abe69f87b7..0e7d2a847f4e 100644 --- a/modules/gapi/include/opencv2/gapi/gstreaming.hpp +++ b/modules/gapi/include/opencv2/gapi/gstreaming.hpp @@ -396,9 +396,11 @@ namespace streaming { * In the streaming mode the pipeline steps are connected with queues * and this compile argument controls every queue's size. */ -struct GAPI_EXPORTS queue_capacity +struct GAPI_EXPORTS_W_SIMPLE queue_capacity { + GAPI_WRAP explicit queue_capacity(size_t cap = 1) : capacity(cap) { }; + GAPI_PROP_RW size_t capacity; }; /** @} */ diff --git a/modules/gapi/misc/python/package/gapi/__init__.py b/modules/gapi/misc/python/package/gapi/__init__.py index dc874f0b0ca5..b1326712fcb6 100644 --- a/modules/gapi/misc/python/package/gapi/__init__.py +++ b/modules/gapi/misc/python/package/gapi/__init__.py @@ -295,3 +295,5 @@ def kernel_with_params(cls): cv.gapi.wip.draw.Mosaic = cv.gapi_wip_draw_Mosaic cv.gapi.wip.draw.Image = cv.gapi_wip_draw_Image cv.gapi.wip.draw.Poly = cv.gapi_wip_draw_Poly + +cv.gapi.streaming.queue_capacity = cv.gapi_streaming_queue_capacity diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index d378a91b5fd6..64f8277740d9 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -11,13 +11,14 @@ #include // NB: Python wrapper replaces :: with _ for classes -using gapi_GKernelPackage = cv::gapi::GKernelPackage; -using gapi_GNetPackage = cv::gapi::GNetPackage; -using gapi_ie_PyParams = cv::gapi::ie::PyParams; -using gapi_wip_IStreamSource_Ptr = cv::Ptr; -using detail_ExtractArgsCallback = cv::detail::ExtractArgsCallback; -using detail_ExtractMetaCallback = cv::detail::ExtractMetaCallback; -using vector_GNetParam = std::vector; +using gapi_GKernelPackage = cv::gapi::GKernelPackage; +using gapi_GNetPackage = cv::gapi::GNetPackage; +using gapi_ie_PyParams = cv::gapi::ie::PyParams; +using gapi_wip_IStreamSource_Ptr = cv::Ptr; +using detail_ExtractArgsCallback = cv::detail::ExtractArgsCallback; +using detail_ExtractMetaCallback = cv::detail::ExtractMetaCallback; +using vector_GNetParam = std::vector; +using gapi_streaming_queue_capacity = cv::gapi::streaming::queue_capacity; // NB: Python wrapper generate T_U for T // This behavior is only observed for inputs @@ -159,7 +160,7 @@ PyObject* pyopencv_from(const cv::gapi::wip::draw::Prims& value) } template<> -bool pyopencv_to(PyObject* obj, cv::gapi::wip::draw::Prim& value, const ArgInfo& info) +bool pyopencv_to(PyObject* obj, cv::gapi::wip::draw::Prim& value, const ArgInfo&) { #define TRY_EXTRACT(Prim) \ if (PyObject_TypeCheck(obj, reinterpret_cast(pyopencv_gapi_wip_draw_##Prim##_TypePtr))) \ @@ -175,6 +176,7 @@ bool pyopencv_to(PyObject* obj, cv::gapi::wip::draw::Prim& value, const ArgInfo& TRY_EXTRACT(Mosaic) TRY_EXTRACT(Image) TRY_EXTRACT(Poly) +#undef TRY_EXTRACT failmsg("Unsupported primitive type"); return false; @@ -186,6 +188,34 @@ bool pyopencv_to(PyObject* obj, cv::gapi::wip::draw::Prims& value, const ArgInfo return pyopencv_to_generic_vec(obj, value, info); } +template <> +bool pyopencv_to(PyObject* obj, cv::GMetaArg& value, const ArgInfo&) +{ +#define TRY_EXTRACT(Meta) \ + if (PyObject_TypeCheck(obj, \ + reinterpret_cast(pyopencv_##Meta##_TypePtr))) \ + { \ + value = reinterpret_cast(obj)->v; \ + return true; \ + } \ + + TRY_EXTRACT(GMatDesc) + TRY_EXTRACT(GScalarDesc) + TRY_EXTRACT(GArrayDesc) + TRY_EXTRACT(GOpaqueDesc) +#undef TRY_EXTRACT + + failmsg("Unsupported cv::GMetaArg type"); + return false; +} + +template <> +bool pyopencv_to(PyObject* obj, cv::GMetaArgs& value, const ArgInfo& info) +{ + return pyopencv_to_generic_vec(obj, value, info); +} + + template<> PyObject* pyopencv_from(const cv::GArg& value) { @@ -707,30 +737,12 @@ static cv::GRunArgs run_py_kernel(cv::detail::PyObjectHolder kernel, static GMetaArg get_meta_arg(PyObject* obj) { - if (PyObject_TypeCheck(obj, - reinterpret_cast(pyopencv_GMatDesc_TypePtr))) - { - return cv::GMetaArg{reinterpret_cast(obj)->v}; - } - else if (PyObject_TypeCheck(obj, - reinterpret_cast(pyopencv_GScalarDesc_TypePtr))) - { - return cv::GMetaArg{reinterpret_cast(obj)->v}; - } - else if (PyObject_TypeCheck(obj, - reinterpret_cast(pyopencv_GArrayDesc_TypePtr))) - { - return cv::GMetaArg{reinterpret_cast(obj)->v}; - } - else if (PyObject_TypeCheck(obj, - reinterpret_cast(pyopencv_GOpaqueDesc_TypePtr))) - { - return cv::GMetaArg{reinterpret_cast(obj)->v}; - } - else + cv::GMetaArg arg; + if (!pyopencv_to(obj, arg, ArgInfo("arg", false))) { util::throw_error(std::logic_error("Unsupported output meta type")); } + return arg; } static cv::GMetaArgs get_meta_args(PyObject* tuple) diff --git a/modules/gapi/misc/python/shadow_gapi.hpp b/modules/gapi/misc/python/shadow_gapi.hpp index 0b489dde0f55..33a0e0fde1bf 100644 --- a/modules/gapi/misc/python/shadow_gapi.hpp +++ b/modules/gapi/misc/python/shadow_gapi.hpp @@ -5,8 +5,9 @@ namespace cv { struct GAPI_EXPORTS_W_SIMPLE GCompileArg { - GAPI_WRAP GCompileArg(gapi::GKernelPackage pkg); - GAPI_WRAP GCompileArg(gapi::GNetPackage pkg); + GAPI_WRAP GCompileArg(gapi::GKernelPackage arg); + GAPI_WRAP GCompileArg(gapi::GNetPackage arg); + GAPI_WRAP GCompileArg(gapi::streaming::queue_capacity arg); }; class GAPI_EXPORTS_W_SIMPLE GInferInputs diff --git a/modules/gapi/misc/python/test/test_gapi_streaming.py b/modules/gapi/misc/python/test/test_gapi_streaming.py index 7ede1b5cf38d..d7914c5157b5 100644 --- a/modules/gapi/misc/python/test/test_gapi_streaming.py +++ b/modules/gapi/misc/python/test/test_gapi_streaming.py @@ -261,6 +261,7 @@ def test_gapi_streaming_meta(self): if curr_frame_number == max_num_frames: break + def test_desync(self): path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) @@ -307,6 +308,49 @@ def test_desync(self): self.assertLess(0, none_counter) + def test_compile_streaming_empty(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + comp.compileStreaming() + + + def test_compile_streaming_args(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + comp.compileStreaming(cv.gapi.compile_args(cv.gapi.streaming.queue_capacity(1))) + + + def test_compile_streaming_descr_of(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + img = np.zeros((3,300,300), dtype=np.float32) + comp.compileStreaming(cv.gapi.descr_of(img)) + + + def test_compile_streaming_descr_of_and_args(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + img = np.zeros((3,300,300), dtype=np.float32) + comp.compileStreaming(cv.gapi.descr_of(img), + cv.gapi.compile_args(cv.gapi.streaming.queue_capacity(1))) + + + def test_compile_streaming_meta(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + img = np.zeros((3,300,300), dtype=np.float32) + comp.compileStreaming([cv.GMatDesc(cv.CV_8U, 3, (300, 300))]) + + + def test_compile_streaming_meta_and_args(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + img = np.zeros((3,300,300), dtype=np.float32) + comp.compileStreaming([cv.GMatDesc(cv.CV_8U, 3, (300, 300))], + cv.gapi.compile_args(cv.gapi.streaming.queue_capacity(1))) + + + except unittest.SkipTest as e: message = str(e) From 65ef82a94674950d3861a172d32a2cce53f015cc Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Tue, 24 Aug 2021 15:41:57 +0300 Subject: [PATCH 135/376] Merge pull request #20570 from sivanov-work:vpl_source_data_adapter G-API: oneVPL (simplification) Add data adapter & Cfg params * Add cfg_param & data_provider * Fix compilation after rebase * Apply some comments * Apply default ctor outside class definition comment * Apply cfg param in source * Fix compilation: add virtual dtor * Move cfg_params in regular gapi src list * Fix compilation: add export.hpp * Add errno.h * Add errno.h * Apply namespace comment * Add several Doxygen & rename cfg_param * Fix build * Update Doxygen docs for onevpl * Fix typo --- modules/gapi/CMakeLists.txt | 7 +- .../gapi/streaming/onevpl/cfg_params.hpp | 88 +++++++++++ .../onevpl/data_provider_interface.hpp | 74 ++++++++++ .../gapi/streaming/onevpl/onevpl_source.hpp | 44 ------ .../opencv2/gapi/streaming/onevpl/source.hpp | 64 ++++++++ .../gapi/samples/onevpl_infer_single_roi.cpp | 51 ++++++- .../gapi/src/streaming/onevpl/cfg_params.cpp | 137 ++++++++++++++++++ .../data_provider_interface_exception.cpp | 22 +++ .../streaming/onevpl/file_data_provider.cpp | 47 ++++++ .../streaming/onevpl/file_data_provider.hpp | 33 +++++ .../src/streaming/onevpl/onevpl_source.cpp | 48 ------ modules/gapi/src/streaming/onevpl/source.cpp | 58 ++++++++ ...onevpl_source_priv.cpp => source_priv.cpp} | 22 +-- ...onevpl_source_priv.hpp => source_priv.hpp} | 13 +- .../test/streaming/gapi_streaming_tests.cpp | 70 +++++++++ 15 files changed, 665 insertions(+), 113 deletions(-) create mode 100644 modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp create mode 100644 modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp delete mode 100644 modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp create mode 100644 modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp create mode 100644 modules/gapi/src/streaming/onevpl/cfg_params.cpp create mode 100644 modules/gapi/src/streaming/onevpl/data_provider_interface_exception.cpp create mode 100644 modules/gapi/src/streaming/onevpl/file_data_provider.cpp create mode 100644 modules/gapi/src/streaming/onevpl/file_data_provider.hpp delete mode 100644 modules/gapi/src/streaming/onevpl/onevpl_source.cpp create mode 100644 modules/gapi/src/streaming/onevpl/source.cpp rename modules/gapi/src/streaming/onevpl/{onevpl_source_priv.cpp => source_priv.cpp} (66%) rename modules/gapi/src/streaming/onevpl/{onevpl_source_priv.hpp => source_priv.hpp} (80%) diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 69c0aaaae817..08dee2f9af3c 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -164,8 +164,11 @@ set(gapi_srcs src/backends/python/gpythonbackend.cpp # Streaming source - src/streaming/onevpl/onevpl_source.cpp - src/streaming/onevpl/onevpl_source_priv.cpp + src/streaming/onevpl/source.cpp + src/streaming/onevpl/source_priv.cpp + src/streaming/onevpl/file_data_provider.cpp + src/streaming/onevpl/cfg_params.cpp + src/streaming/onevpl/data_provider_interface_exception.cpp # Utils (ITT tracing) src/utils/itt.cpp diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp new file mode 100644 index 000000000000..9dc5ead7d740 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp @@ -0,0 +1,88 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_ONEVPL_CFG_PARAMS_HPP +#define OPENCV_GAPI_STREAMING_ONEVPL_CFG_PARAMS_HPP + +#include +#include +#include + +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +/** + * @brief Public class is using for creation of onevpl::GSource instances. + * + * Class members availaible through methods @ref CfgParam::get_name() and @ref CfgParam::get_value() are used by + * onevpl::GSource inner logic to create or find oneVPL particular implementation + * (software/hardware, specific API version and etc.). + * + * @note Because oneVPL may provide several implementations which are satisfying with multiple (or single one) @ref CfgParam + * criteria therefore it is possible to configure `preferred` parameters. This kind of CfgParams are created + * using `is_major = false` argument in @ref CfgParam::create method and are not used by creating oneVPL particular implementations. + * Instead they fill out a "score table" to select preferrable implementation from available list. Implementation are satisfying + * with most of these optional params would be chosen. + * If no one optional CfgParam params were present then first of available oneVPL implementation would be applied. + * Please get on https://spec.oneapi.io/versions/latest/elements/oneVPL/source/API_ref/VPL_disp_api_func.html?highlight=mfxcreateconfig#mfxsetconfigfilterproperty + * for using OneVPL configuration. In this schema `mfxU8 *name` represents @ref CfgParam::get_name() and + * `mfxVariant value` is @ref CfgParam::get_value() + */ +struct GAPI_EXPORTS CfgParam { + using name_t = std::string; + using value_t = cv::util::variant; + + /** + * Create onevp::GSource configuration parameter. + * + *@param name name of parameter. + *@param value value of parameter. + *@param is_major TRUE if parameter MUST be provided by OneVPL inner implementation, FALSE for optional (for resolve multiple available implementations). + * + */ + template + static CfgParam create(const std::string& name, ValueType&& value, bool is_major = true) { + CfgParam param(name, CfgParam::value_t(std::forward(value)), is_major); + return param; + } + + struct Priv; + + const name_t& get_name() const; + const value_t& get_value() const; + bool is_major() const; + bool operator==(const CfgParam& rhs) const; + bool operator< (const CfgParam& rhs) const; + bool operator!=(const CfgParam& rhs) const; + + CfgParam& operator=(const CfgParam& src); + CfgParam& operator=(CfgParam&& src); + CfgParam(const CfgParam& src); + CfgParam(CfgParam&& src); + ~CfgParam(); +private: + CfgParam(const std::string& param_name, value_t&& param_value, bool is_major_param); + std::shared_ptr m_priv; +}; + +} //namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_STREAMING_ONEVPL_CFG_PARAMS_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp new file mode 100644 index 000000000000..2c299520f757 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp @@ -0,0 +1,74 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ONEVPL_DATA_PROVIDER_INTERFACE_HPP +#define GAPI_STREAMING_ONEVPL_ONEVPL_DATA_PROVIDER_INTERFACE_HPP +#include +#include + +#include // GAPI_EXPORTS + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +struct GAPI_EXPORTS DataProviderException : public std::exception { + virtual ~DataProviderException() {}; +}; + +struct GAPI_EXPORTS DataProviderSystemErrorException : public DataProviderException { + DataProviderSystemErrorException(int error_code, const std::string& desription = std::string()); + virtual ~DataProviderSystemErrorException(); + virtual const char* what() const noexcept override; + +private: + std::string reason; +}; + +/** + * @brief Public interface allows to customize extraction of video stream data + * used by onevpl::GSource instead of reading stream from file (by default). + * + * Interface implementation constructor MUST provide consistency and creates fully operable object. + * If error happened implementation MUST throw `DataProviderException` kind exceptions + * + * @note Interface implementation MUST manage stream and other constructed resources by itself to avoid any kind of leak. + * For simple interface implementation example please see `StreamDataProvider` in `tests/streaming/gapi_streaming_tests.cpp` + */ +struct GAPI_EXPORTS IDataProvider { + using Ptr = std::shared_ptr; + + virtual ~IDataProvider() {}; + + /** + * The function is used by onevpl::GSource to extract binary data stream from @ref IDataProvider + * implementation. + * + * It MUST throw `DataProviderException` kind exceptions in fail cases. + * It MUST return 0 in EOF which considered as not-fail case. + * + * @param out_data_bytes_size the available capacity of out_data buffer. + * @param out_data the output consumer buffer with capacity out_data_bytes_size. + * @return fetched bytes count. + */ + virtual size_t fetch_data(size_t out_data_bytes_size, void* out_data) = 0; + + /** + * The function is used by onevpl::GSource to check more binary data availability. + * + * It MUST return TRUE in case of EOF and NO_THROW exceptions. + * + * @return boolean value which detects end of stream + */ + virtual bool empty() const = 0; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // GAPI_STREAMING_ONEVPL_ONEVPL_DATA_PROVIDER_INTERFACE_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp deleted file mode 100644 index fec8c73dffeb..000000000000 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp +++ /dev/null @@ -1,44 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html. -// -// Copyright (C) 2021 Intel Corporation - -#ifndef OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP -#define OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP - -#include -#include -#include - -namespace cv { -namespace gapi { -namespace wip { - -class GAPI_EXPORTS OneVPLSource : public IStreamSource -{ -public: - struct Priv; - - explicit OneVPLSource(const std::string& filePath); - ~OneVPLSource() override; - - bool pull(cv::gapi::wip::Data& data) override; - GMetaArg descr_of() const override; - -private: - explicit OneVPLSource(std::unique_ptr&& impl); - std::unique_ptr m_priv; -}; - -template -GAPI_EXPORTS_W cv::Ptr inline make_vpl_src(const std::string& filePath, Args&&... args) -{ - return make_src(filePath, std::forward(args)...); -} - -} // namespace wip -} // namespace gapi -} // namespace cv - -#endif // OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp new file mode 100644 index 000000000000..b7bf4f2ffbcf --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp @@ -0,0 +1,64 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP +#define OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP + +#include +#include +#include +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +using CfgParams = std::vector; + +/** + * @brief G-API streaming source based on OneVPL implementation. + * + * This class implements IStreamSource interface. + * Its constructor takes source file path (in usual way) or @ref onevpl::IDataProvider + * interface implementation (for not file-based sources). It also allows to pass-through + * oneVPL configuration parameters by using several @ref onevpl::CfgParam. + * + * @note stream sources are passed to G-API via shared pointers, so + * please gapi::make_onevpl_src<> to create objects and ptr() to pass a + * GSource to cv::gin(). + */ +class GAPI_EXPORTS GSource : public IStreamSource +{ +public: + struct Priv; + + GSource(const std::string& filePath, + const CfgParams& cfg_params = CfgParams{}); + GSource(std::shared_ptr source, + const CfgParams& cfg_params = CfgParams{}); + ~GSource() override; + + bool pull(cv::gapi::wip::Data& data) override; + GMetaArg descr_of() const override; + +private: + explicit GSource(std::unique_ptr&& impl); + std::unique_ptr m_priv; +}; +} // namespace onevpl + +template +GAPI_EXPORTS_W cv::Ptr inline make_onevpl_src(Args&&... args) +{ + return make_src(std::forward(args)...); +} + +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index 8a7efafabfd8..9a3abdf39203 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include // CommandLineParser const std::string about = @@ -19,7 +19,8 @@ const std::string keys = "{ input | | Path to the input demultiplexed video file }" "{ output | | Path to the output RAW video file. Use .avi extension }" "{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }" - "{ faced | CPU | Target device for face detection model (e.g. CPU, GPU, VPU, ...) }"; + "{ cfg_params | :;: | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }"; + namespace { std::string get_weights_path(const std::string &model_path) { @@ -155,6 +156,10 @@ GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { } // namespace custom +namespace cfg { +typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line); +} + int main(int argc, char *argv[]) { cv::CommandLineParser cmd(argc, argv, keys); @@ -178,10 +183,22 @@ int main(int argc, char *argv[]) { } } + // get oneVPL cfg params from cmd + std::stringstream params_list(cmd.get("cfg_params")); + std::vector source_cfgs; + try { + std::string line; + while (std::getline(params_list, line, ';')) { + source_cfgs.push_back(cfg::create_from_string(line)); + } + } catch (const std::exception& ex) { + std::cerr << "Invalid cfg parameter: " << ex.what() << std::endl; + return -1; + } + auto face_net = cv::gapi::ie::Params { face_model_path, // path to topology IR - get_weights_path(face_model_path), // path to weights - cmd.get("faced"), // device specifier + get_weights_path(face_model_path) // path to weights }; auto kernels = cv::gapi::kernels < custom::OCVLocateROI @@ -192,7 +209,7 @@ int main(int argc, char *argv[]) { // Create source cv::Ptr cap; try { - cap = cv::gapi::wip::make_vpl_src(file_path); + cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs); std::cout << "oneVPL source desription: " << cap->descr_of() << std::endl; } catch (const std::exception& ex) { std::cerr << "Cannot create source: " << ex.what() << std::endl; @@ -213,7 +230,7 @@ int main(int argc, char *argv[]) { cv::GStreamingCompiled pipeline; try { - pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) + pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) .compileStreaming(cv::compile_args(kernels, networks)); } catch (const std::exception& ex) { std::cerr << "Exception occured during pipeline construction: " << ex.what() << std::endl; @@ -252,3 +269,25 @@ int main(int argc, char *argv[]) { return 0; } + + +namespace cfg { +typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line) { + using namespace cv::gapi::wip; + + if (line.empty()) { + throw std::runtime_error("Cannot parse CfgParam from emply line"); + } + + std::string::size_type name_endline_pos = line.find(':'); + if (name_endline_pos == std::string::npos) { + throw std::runtime_error("Cannot parse CfgParam from: " + line + + "\nExpected separator \":\""); + } + + std::string name = line.substr(0, name_endline_pos); + std::string value = line.substr(name_endline_pos + 1); + + return cv::gapi::wip::onevpl::CfgParam::create(name, value); +} +} diff --git a/modules/gapi/src/streaming/onevpl/cfg_params.cpp b/modules/gapi/src/streaming/onevpl/cfg_params.cpp new file mode 100644 index 000000000000..456725508f8b --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/cfg_params.cpp @@ -0,0 +1,137 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +namespace util { +struct variant_comparator : cv::util::static_visitor { + variant_comparator(const CfgParam::value_t& rhs_value) : + rhs(rhs_value) {} + + template + bool visit(const ValueType& lhs) const { + return lhs < cv::util::get(rhs); + } +private: + const CfgParam::value_t& rhs; +}; +} // namespace util + +struct CfgParam::Priv { + Priv(const std::string& param_name, CfgParam::value_t&& param_value, bool is_major_param) : + name(param_name), value(std::forward(param_value)), major_flag(is_major_param) { + } + + const CfgParam::name_t& get_name_impl() const { + return name; + } + + const CfgParam::value_t& get_value_impl() const { + return value; + } + + bool is_major_impl() const { + return major_flag; + } + + // comparison implementation + bool operator< (const Priv& rhs) const { + // implement default pair comparison + if (get_name_impl() < rhs.get_name_impl()) { + return true; + } else if (get_name_impl() > rhs.get_name_impl()) { + return false; + } + + //TODO implement operator < for cv::util::variant + const CfgParam::value_t& lvar = get_value_impl(); + const CfgParam::value_t& rvar = rhs.get_value_impl(); + if (lvar.index() < rvar.index()) { + return true; + } else if (lvar.index() > rvar.index()) { + return false; + } + + util::variant_comparator comp(rvar); + return cv::util::visit(comp, lvar); + } + + bool operator==(const Priv& rhs) const { + return (get_name_impl() == rhs.get_name_impl()) + && (get_value_impl() == rhs.get_value_impl()); + } + + bool operator!=(const Priv& rhs) const { + return !(*this == rhs); + } + + CfgParam::name_t name; + CfgParam::value_t value; + bool major_flag; +}; + +CfgParam::CfgParam (const std::string& param_name, value_t&& param_value, bool is_major_param) : + m_priv(new Priv(param_name, std::move(param_value), is_major_param)) { +} + +CfgParam::~CfgParam() = default; + +CfgParam& CfgParam::operator=(const CfgParam& src) { + if (this != &src) { + m_priv = src.m_priv; + } + return *this; +} + +CfgParam& CfgParam::operator=(CfgParam&& src) { + if (this != &src) { + m_priv = std::move(src.m_priv); + } + return *this; +} + +CfgParam::CfgParam(const CfgParam& src) : + m_priv(src.m_priv) { +} + +CfgParam::CfgParam(CfgParam&& src) : + m_priv(std::move(src.m_priv)) { +} + +const CfgParam::name_t& CfgParam::get_name() const { + return m_priv->get_name_impl(); +} + +const CfgParam::value_t& CfgParam::get_value() const { + return m_priv->get_value_impl(); +} + +bool CfgParam::is_major() const { + return m_priv->is_major_impl(); +} + +bool CfgParam::operator< (const CfgParam& rhs) const { + return *m_priv < *rhs.m_priv; +} + +bool CfgParam::operator==(const CfgParam& rhs) const { + return *m_priv == *rhs.m_priv; +} + +bool CfgParam::operator!=(const CfgParam& rhs) const { + return *m_priv != *rhs.m_priv; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/data_provider_interface_exception.cpp b/modules/gapi/src/streaming/onevpl/data_provider_interface_exception.cpp new file mode 100644 index 000000000000..feaae7a727e4 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/data_provider_interface_exception.cpp @@ -0,0 +1,22 @@ +#include +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +DataProviderSystemErrorException::DataProviderSystemErrorException(int error_code, const std::string& desription) { + reason = desription + ", error: " + std::to_string(error_code) + ", desctiption: " + strerror(error_code); +} + +DataProviderSystemErrorException::~DataProviderSystemErrorException() = default; + +const char* DataProviderSystemErrorException::what() const noexcept { + return reason.c_str(); +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/file_data_provider.cpp b/modules/gapi/src/streaming/onevpl/file_data_provider.cpp new file mode 100644 index 000000000000..2a3df07c512e --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/file_data_provider.cpp @@ -0,0 +1,47 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation +#include + +#include "streaming/onevpl/file_data_provider.hpp" + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +FileDataProvider::FileDataProvider(const std::string& file_path) : + source_handle(fopen(file_path.c_str(), "rb"), &fclose) { + if (!source_handle) { + throw DataProviderSystemErrorException(errno, + "FileDataProvider: cannot open source file: " + file_path); + } +} + +FileDataProvider::~FileDataProvider() = default; + +size_t FileDataProvider::fetch_data(size_t out_data_bytes_size, void* out_data) { + if (empty()) { + return 0; + } + + size_t ret = fread(out_data, 1, out_data_bytes_size, source_handle.get()); + if (ret == 0) { + if (feof(source_handle.get())) { + source_handle.reset(); + } else { + throw DataProviderSystemErrorException (errno, "FileDataProvider::fetch_data error read"); + } + } + return ret; +} + +bool FileDataProvider::empty() const { + return !source_handle; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/file_data_provider.hpp b/modules/gapi/src/streaming/onevpl/file_data_provider.hpp new file mode 100644 index 000000000000..22348290be37 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/file_data_provider.hpp @@ -0,0 +1,33 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ONEVPL_FILE_DATA_PROVIDER_HPP +#define GAPI_STREAMING_ONEVPL_ONEVPL_FILE_DATA_PROVIDER_HPP +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +struct FileDataProvider : public IDataProvider { + + using file_ptr = std::unique_ptr; + FileDataProvider(const std::string& file_path); + ~FileDataProvider(); + + size_t fetch_data(size_t out_data_bytes_size, void* out_data) override; + bool empty() const override; +private: + file_ptr source_handle; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // GAPI_STREAMING_ONEVPL_ONEVPL_FILE_DATA_PROVIDER_HPP diff --git a/modules/gapi/src/streaming/onevpl/onevpl_source.cpp b/modules/gapi/src/streaming/onevpl/onevpl_source.cpp deleted file mode 100644 index 988986f6d9d9..000000000000 --- a/modules/gapi/src/streaming/onevpl/onevpl_source.cpp +++ /dev/null @@ -1,48 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html. -// -// Copyright (C) 2021 Intel Corporation - -#include - -#include "streaming/onevpl/onevpl_source_priv.hpp" - -namespace cv { -namespace gapi { -namespace wip { - -#ifdef HAVE_ONEVPL -OneVPLSource::OneVPLSource(const std::string& filePath) : - OneVPLSource(std::unique_ptr(new OneVPLSource::Priv(filePath))) { - - if (filePath.empty()) { - util::throw_error(std::logic_error("Cannot create 'OneVPLSource' on empty source file name")); - } -} -#else -OneVPLSource::OneVPLSource(const std::string&) { - GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); -} -#endif - -OneVPLSource::OneVPLSource(std::unique_ptr&& impl) : - IStreamSource(), - m_priv(std::move(impl)) { -} - -OneVPLSource::~OneVPLSource() { -} - -bool OneVPLSource::pull(cv::gapi::wip::Data& data) -{ - return m_priv->pull(data); -} - -GMetaArg OneVPLSource::descr_of() const -{ - return m_priv->descr_of(); -} -} // namespace wip -} // namespace gapi -} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/source.cpp b/modules/gapi/src/streaming/onevpl/source.cpp new file mode 100644 index 000000000000..0decb1358b7b --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/source.cpp @@ -0,0 +1,58 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include + +#include "streaming/onevpl/source_priv.hpp" +#include "streaming/onevpl/file_data_provider.hpp" +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +#ifdef HAVE_ONEVPL +GSource::GSource(const std::string& filePath, const CfgParams& cfg_params) : + GSource(std::unique_ptr(new GSource::Priv(std::make_shared(filePath), + cfg_params))) { + + if (filePath.empty()) { + util::throw_error(std::logic_error("Cannot create 'GSource' on empty source file name")); + } +} + +GSource::GSource(std::shared_ptr source, const CfgParams& cfg_params) : + GSource(std::unique_ptr(new GSource::Priv(source, cfg_params))) { +} +#else +GSource::GSource(const std::string&, const CfgParams&) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} + +GSource::GSource(std::shared_ptr, const CfgParams&) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} +#endif + +GSource::GSource(std::unique_ptr&& impl) : + IStreamSource(), + m_priv(std::move(impl)) { +} + +GSource::~GSource() = default; + +bool GSource::pull(cv::gapi::wip::Data& data) +{ + return m_priv->pull(data); +} + +GMetaArg GSource::descr_of() const +{ + return m_priv->descr_of(); +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp b/modules/gapi/src/streaming/onevpl/source_priv.cpp similarity index 66% rename from modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp rename to modules/gapi/src/streaming/onevpl/source_priv.cpp index 5c4e8e694175..ed838d6adc36 100644 --- a/modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.cpp @@ -7,19 +7,21 @@ #include #include -#include "streaming/onevpl/onevpl_source_priv.hpp" +#include "streaming/onevpl/source_priv.hpp" #include "logger.hpp" #ifndef HAVE_ONEVPL namespace cv { namespace gapi { namespace wip { -bool OneVPLSource::Priv::pull(cv::gapi::wip::Data&) { +namespace onevpl { +bool GSource::Priv::pull(cv::gapi::wip::Data&) { return true; } -GMetaArg OneVPLSource::Priv::descr_of() const { +GMetaArg GSource::Priv::descr_of() const { return {}; } +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv @@ -29,33 +31,35 @@ GMetaArg OneVPLSource::Priv::descr_of() const { namespace cv { namespace gapi { namespace wip { -OneVPLSource::Priv::Priv() : +namespace onevpl { +GSource::Priv::Priv() : mfx_handle(MFXLoad()) { GAPI_LOG_INFO(nullptr, "Initialized MFX handle: " << mfx_handle); description_is_valid = false; } -OneVPLSource::Priv::Priv(const std::string&) : - OneVPLSource::Priv() +GSource::Priv::Priv(std::shared_ptr, const std::vector&) : + GSource::Priv() { } -OneVPLSource::Priv::~Priv() +GSource::Priv::~Priv() { GAPI_LOG_INFO(nullptr, "Unload MFX handle: " << mfx_handle); MFXUnload(mfx_handle); } -bool OneVPLSource::Priv::pull(cv::gapi::wip::Data&) +bool GSource::Priv::pull(cv::gapi::wip::Data&) { return false; } -GMetaArg OneVPLSource::Priv::descr_of() const +GMetaArg GSource::Priv::descr_of() const { return {}; } +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp b/modules/gapi/src/streaming/onevpl/source_priv.hpp similarity index 80% rename from modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp rename to modules/gapi/src/streaming/onevpl/source_priv.hpp index b139add99372..c2789342532c 100644 --- a/modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.hpp @@ -14,7 +14,7 @@ #include #include -#include +#include #ifdef HAVE_ONEVPL #if (MFX_VERSION >= 2000) @@ -28,10 +28,12 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { -struct OneVPLSource::Priv +struct GSource::Priv { - explicit Priv(const std::string& file_path); + explicit Priv(std::shared_ptr provider, + const std::vector& params); ~Priv(); bool pull(cv::gapi::wip::Data& data); @@ -41,6 +43,7 @@ struct OneVPLSource::Priv mfxLoader mfx_handle; bool description_is_valid; }; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv @@ -50,11 +53,13 @@ struct OneVPLSource::Priv namespace cv { namespace gapi { namespace wip { -struct OneVPLSource::Priv final +namespace onevpl { +struct GSource::Priv final { bool pull(cv::gapi::wip::Data&); GMetaArg descr_of() const; }; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp index 5386d1736f67..76027a56c682 100644 --- a/modules/gapi/test/streaming/gapi_streaming_tests.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_tests.cpp @@ -25,6 +25,17 @@ #include #include +#include + +#ifdef HAVE_ONEVPL + +#if (MFX_VERSION >= 2000) +#include +#endif + +#include +#endif // HAVE_ONEVPL + namespace opencv_test { namespace @@ -273,6 +284,22 @@ void checkPullOverload(const cv::Mat& ref, EXPECT_EQ(0., cv::norm(ref, out_mat, cv::NORM_INF)); } +struct StreamDataProvider : public cv::gapi::wip::onevpl::IDataProvider { + + StreamDataProvider(std::istream& in) : data_stream (in) { + EXPECT_TRUE(in); + } + + size_t fetch_data(size_t out_data_size, void* out_data_buf) override { + data_stream.read(reinterpret_cast(out_data_buf), out_data_size); + return data_stream.gcount(); + } + bool empty() const override { + return data_stream.eof() || data_stream.bad(); + } +private: + std::istream& data_stream; +}; } // anonymous namespace TEST_P(GAPI_Streaming, SmokeTest_ConstInput_GMat) @@ -2214,4 +2241,47 @@ TEST(GAPI_Streaming, TestPythonAPI) cc.stop(); } +#ifdef HAVE_ONEVPL +const unsigned char hevc_header[] = { + 0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0C, 0x06, 0xFF, 0xFF, 0x01, 0x40, 0x00, + 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x00, + 0x00, 0x04, 0x02, 0x10, 0x30, 0x00, 0x00, 0x03, 0x00, 0x10, 0x00, 0x00, 0x03, + 0x01, 0xE5, 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x06, 0x01, 0x40, 0x00, 0x00, + 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x00, 0x00, + 0xA0, 0x10, 0x20, 0x61, 0x63, 0x41, 0x00, 0x86, 0x49, 0x1B, 0x2B, 0x20, 0x00, + 0x00, 0x00, 0x01, 0x44, 0x01, 0xC0, 0x71, 0xC0, 0xD9, 0x20, 0x00, 0x00, 0x00, + 0x01, 0x26, 0x01, 0xAF, 0x0C +}; +TEST(OneVPL_Source, Init) +{ + using CfgParam = cv::gapi::wip::onevpl::CfgParam; + + std::vector src_params; + src_params.push_back(CfgParam::create("mfxImplDescription.Impl", + MFX_IMPL_TYPE_HARDWARE)); + src_params.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_VIA_D3D11, false)); + src_params.push_back(CfgParam::create("mfxImplDescription.mfxDecoderDescription.decoder.CodecID", + MFX_CODEC_HEVC)); + std::stringstream stream(std::ios_base::in | std::ios_base::out | std::ios_base::binary); + EXPECT_TRUE(stream.write(reinterpret_cast(const_cast(hevc_header)), + sizeof(hevc_header))); + std::shared_ptr stream_data_provider = std::make_shared(stream); + + cv::Ptr cap; + bool cap_created = false; + try { + cap = cv::gapi::wip::make_onevpl_src(stream_data_provider, src_params); + cap_created = true; + } catch (const std::exception&) { + } + ASSERT_TRUE(cap_created); + + cv::gapi::wip::Data out; + while (cap->pull(out)) { + (void)out; + } + EXPECT_TRUE(stream_data_provider->empty()); +} +#endif } // namespace opencv_test From 6f4160c0140c8ef88d777ba2ee5a605bf345de64 Mon Sep 17 00:00:00 2001 From: Alexander Panov Date: Tue, 24 Aug 2021 22:29:40 +0300 Subject: [PATCH 136/376] Merge pull request #20584 from AleksandrPanov:fix_HoughCircles_ALT_dimensions Fix hough circles alt dimensions * fix OutputArray _circles dimensions * add houghcircles_alt test * fix warnings * add shape assert * change asserts --- modules/imgproc/src/hough.cpp | 6 ++- modules/python/test/test_houghcircles.py | 47 ++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/modules/imgproc/src/hough.cpp b/modules/imgproc/src/hough.cpp index 79f34521e83a..5c78c94983f2 100644 --- a/modules/imgproc/src/hough.cpp +++ b/modules/imgproc/src/hough.cpp @@ -2301,14 +2301,16 @@ static void HoughCircles( InputArray _image, OutputArray _circles, std::vector cw(ncircles); for( i = 0; i < ncircles; i++ ) cw[i] = GetCircle4f(circles[i]); - Mat(cw).copyTo(_circles); + if (ncircles > 0) + Mat(1, (int)ncircles, cv::traits::Type::value, &cw[0]).copyTo(_circles); } else if( type == CV_32FC3 ) { std::vector cwow(ncircles); for( i = 0; i < ncircles; i++ ) cwow[i] = GetCircle(circles[i]); - Mat(cwow).copyTo(_circles); + if (ncircles > 0) + Mat(1, (int)ncircles, cv::traits::Type::value, &cwow[0]).copyTo(_circles); } else CV_Error(Error::StsError, "Internal error"); diff --git a/modules/python/test/test_houghcircles.py b/modules/python/test/test_houghcircles.py index 90cd1841394c..2cfc6c9d68e3 100644 --- a/modules/python/test/test_houghcircles.py +++ b/modules/python/test/test_houghcircles.py @@ -80,5 +80,52 @@ def test_houghcircles(self): self.assertLess(float(len(circles) - matches_counter) / len(circles), .75) + def test_houghcircles_alt(self): + + fn = "samples/data/board.jpg" + + src = self.get_sample(fn, 1) + img = cv.cvtColor(src, cv.COLOR_BGR2GRAY) + img = cv.medianBlur(img, 5) + + circles = cv.HoughCircles(img, cv.HOUGH_GRADIENT_ALT, 1, 10, np.array([]), 300, 0.9, 1, 30) + + self.assertEqual(circles.shape, (1, 18, 3)) + + circles = circles[0] + + testCircles = [[38, 181, 17.6], + [99.7, 166, 13.12], + [142.7, 160, 13.52], + [223.6, 110, 8.62], + [79.1, 206.7, 8.62], + [47.5, 351.6, 11.64], + [189.5, 354.4, 11.64], + [189.8, 298.9, 10.64], + [189.5, 252.4, 14.62], + [252.5, 393.4, 15.62], + [602.9, 467.5, 11.42], + [222, 210.4, 9.12], + [263.1, 216.7, 9.12], + [359.8, 222.6, 9.12], + [518.9, 120.9, 9.12], + [413.8, 113.4, 9.12], + [489, 127.2, 9.12], + [448.4, 121.3, 9.12], + [384.6, 128.9, 8.62]] + + matches_counter = 0 + + for i in range(len(testCircles)): + for j in range(len(circles)): + + tstCircle = circleApproximation(testCircles[i]) + circle = circleApproximation(circles[j]) + if convContoursIntersectiponRate(tstCircle, circle) > 0.6: + matches_counter += 1 + + self.assertGreater(float(matches_counter) / len(testCircles), .5) + self.assertLess(float(len(circles) - matches_counter) / len(circles), .75) + if __name__ == '__main__': NewOpenCVTests.bootstrap() From 9bda96d39e7d63da97ef38a0dcb04ff9d0670a01 Mon Sep 17 00:00:00 2001 From: Dale Phurrough Date: Tue, 24 Aug 2021 20:06:36 +0200 Subject: [PATCH 137/376] add test case --- modules/core/test/test_umat.cpp | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/modules/core/test/test_umat.cpp b/modules/core/test/test_umat.cpp index c2bd6eceba0f..c323d17c06d3 100644 --- a/modules/core/test/test_umat.cpp +++ b/modules/core/test/test_umat.cpp @@ -1419,4 +1419,37 @@ TEST(UMat, resize_Mat_issue_13577) cv::ocl::setUseOpenCL(useOCL); // restore state } +TEST(UMat, exceptions_refcounts_issue_20594) +{ + if (!cv::ocl::useOpenCL()) + { + // skip test, difficult to create exception scenario without OpenCL + std::cout << "OpenCL is not enabled. Skip test" << std::endl; + return; + } + + UMat umat1(10, 10, CV_8UC1); + EXPECT_EQ(0, umat1.u->refcount); + + // cause exception in underlying allocator + void* const original_handle = umat1.u->handle; + umat1.u->handle = NULL; + try + { + Mat mat1 = umat1.getMat(ACCESS_RW); + } + catch (...) + { + // nothing + } + + // check for correct refcount, and no change of intentional bad handle + EXPECT_EQ(0, umat1.u->refcount); + EXPECT_EQ(NULL, umat1.u->handle); + + // reset UMat to good state + umat1.u->refcount = 0; + umat1.u->handle = original_handle; +} + } } // namespace opencv_test::ocl From 54a9e009703567c03d8fccbee7bb43f945b396a1 Mon Sep 17 00:00:00 2001 From: Dale Phurrough Date: Tue, 24 Aug 2021 18:56:25 +0200 Subject: [PATCH 138/376] fix opencv/opencv#20594 - exception handling with refcounts --- modules/core/src/matrix.cpp | 5 ++- modules/core/src/ocl.cpp | 26 +++++++++++++-- modules/core/src/umatrix.cpp | 63 +++++++++++++++++++++++------------- 3 files changed, 66 insertions(+), 28 deletions(-) diff --git a/modules/core/src/matrix.cpp b/modules/core/src/matrix.cpp index 4db7c7a89e78..1deff6b450d1 100644 --- a/modules/core/src/matrix.cpp +++ b/modules/core/src/matrix.cpp @@ -749,18 +749,17 @@ Mat::Mat(const Mat& m, const Rect& roi) data += roi.x*esz; CV_Assert( 0 <= roi.x && 0 <= roi.width && roi.x + roi.width <= m.cols && 0 <= roi.y && 0 <= roi.height && roi.y + roi.height <= m.rows ); - if( u ) - CV_XADD(&u->refcount, 1); if( roi.width < m.cols || roi.height < m.rows ) flags |= SUBMATRIX_FLAG; step[0] = m.step[0]; step[1] = esz; updateContinuityFlag(); + addref(); if( rows <= 0 || cols <= 0 ) { - release(); rows = cols = 0; + release(); } } diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index 8b1377eec6b5..2d85a32366d9 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -2774,19 +2774,33 @@ struct Kernel::Impl void cleanupUMats() { + bool exceptionOccurred = false; for( int i = 0; i < MAX_ARRS; i++ ) + { if( u[i] ) { if( CV_XADD(&u[i]->urefcount, -1) == 1 ) { u[i]->flags |= UMatData::ASYNC_CLEANUP; - u[i]->currAllocator->deallocate(u[i]); + try + { + u[i]->currAllocator->deallocate(u[i]); + } + catch(const std::exception& exc) + { + // limited by legacy before C++11, therefore log and + // remember some exception occurred to throw below + CV_LOG_ERROR(NULL, "OCL: Unexpected C++ exception in OpenCL Kernel::Impl::cleanupUMats(): " << exc.what()); + exceptionOccurred = true; + } } u[i] = 0; } + } nu = 0; haveTempDstUMats = false; haveTempSrcUMats = false; + CV_Assert(!exceptionOccurred); } void addUMat(const UMat& m, bool dst) @@ -2817,8 +2831,16 @@ struct Kernel::Impl void finit(cl_event e) { CV_UNUSED(e); - cleanupUMats(); isInProgress = false; + try + { + cleanupUMats(); + } + catch(...) + { + release(); + throw; + } release(); } diff --git a/modules/core/src/umatrix.cpp b/modules/core/src/umatrix.cpp index 94f828ba601c..da01f3785f9a 100644 --- a/modules/core/src/umatrix.cpp +++ b/modules/core/src/umatrix.cpp @@ -540,13 +540,26 @@ UMat Mat::getUMat(int accessFlags, UMatUsageFlags usageFlags) const CV_XADD(&(u->refcount), 1); CV_XADD(&(u->urefcount), 1); } - hdr.flags = flags; - setSize(hdr, dims, size.p, step.p); - finalizeHdr(hdr); - hdr.u = new_u; - hdr.offset = 0; //data - datastart; - hdr.addref(); - return hdr; + try + { + hdr.flags = flags; + setSize(hdr, dims, size.p, step.p); + finalizeHdr(hdr); + hdr.u = new_u; + hdr.offset = 0; //data - datastart; + hdr.addref(); + return hdr; + } + catch(...) + { + if (u != NULL) + { + CV_XADD(&(u->refcount), -1); + CV_XADD(&(u->urefcount), -1); + } + new_u->currAllocator->deallocate(new_u); + throw; + } } void UMat::create(int d, const int* _sizes, int _type, UMatUsageFlags _usageFlags) @@ -692,18 +705,17 @@ UMat::UMat(const UMat& m, const Rect& roi) offset += roi.x*esz; CV_Assert( 0 <= roi.x && 0 <= roi.width && roi.x + roi.width <= m.cols && 0 <= roi.y && 0 <= roi.height && roi.y + roi.height <= m.rows ); - if( u ) - CV_XADD(&(u->urefcount), 1); if( roi.width < m.cols || roi.height < m.rows ) flags |= SUBMATRIX_FLAG; step[0] = m.step[0]; step[1] = esz; updateContinuityFlag(); + addref(); if( rows <= 0 || cols <= 0 ) { - release(); rows = cols = 0; + release(); } } @@ -969,24 +981,29 @@ Mat UMat::getMat(int accessFlags) const // TODO Support ACCESS_READ (ACCESS_WRITE) without unnecessary data transfers accessFlags |= ACCESS_RW; UMatDataAutoLock autolock(u); - if(CV_XADD(&u->refcount, 1) == 0) - u->currAllocator->map(u, accessFlags); - if (u->data != 0) + try { - Mat hdr(dims, size.p, type(), u->data + offset, step.p); - hdr.flags = flags; - hdr.u = u; - hdr.datastart = u->data; - hdr.data = u->data + offset; - hdr.datalimit = hdr.dataend = u->data + u->size; - return hdr; + if(CV_XADD(&u->refcount, 1) == 0) + u->currAllocator->map(u, accessFlags); + if (u->data != 0) + { + Mat hdr(dims, size.p, type(), u->data + offset, step.p); + hdr.flags = flags; + hdr.u = u; + hdr.datastart = u->data; + hdr.data = u->data + offset; + hdr.datalimit = hdr.dataend = u->data + u->size; + return hdr; + } } - else + catch(...) { CV_XADD(&u->refcount, -1); - CV_Assert(u->data != 0 && "Error mapping of UMat to host memory."); - return Mat(); + throw; } + CV_XADD(&u->refcount, -1); + CV_Assert(u->data != 0 && "Error mapping of UMat to host memory."); + return Mat(); } void* UMat::handle(int accessFlags) const From 5093fe2c2fc73b35d2f18f8791e740bcff61c27b Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 17 Jun 2021 20:32:20 +0800 Subject: [PATCH 139/376] Add WebNN backend for OpenCV DNN Module Update dnn.cpp Update dnn.cpp Update dnn.cpp Update dnn.cpp Add WebNN head files into OpenCV 3rd partiy files Create webnn.hpp update cmake Complete README and add OpenCVDetectWebNN.cmake file add webnn.cpp Modify webnn.cpp Can successfully compile the codes for creating a MLContext Update webnn.cpp Update README.md Update README.md Update README.md Update README.md Update cmake files and update README.md Update OpenCVDetectWebNN.cmake and README.md Update OpenCVDetectWebNN.cmake Fix OpenCVDetectWebNN.cmake and update README.md Add source webnn_cpp.cpp and libary libwebnn_proc.so Update dnn.cpp Update dnn.cpp Update dnn.cpp Update dnn.cpp update dnn.cpp update op_webnn update op_webnn Update op_webnn.hpp update op_webnn.cpp & hpp Update op_webnn.hpp Update op_webnn update the skeleton Update op_webnn.cpp Update op_webnn Update op_webnn.cpp Update op_webnn.cpp Update op_webnn.hpp update op_webnn update op_webnn Solved the problems of released variables. Fixed the bugs in op_webnn.cpp Implement op_webnn Implement Relu by WebNN API Update dnn.cpp for better test Update elementwise_layers.cpp Implement ReLU6 Update elementwise_layers.cpp Implement SoftMax using WebNN API Implement Reshape by WebNN API Implement PermuteLayer by WebNN API Implement PoolingLayer using WebNN API Update pooling_layer.cpp Update pooling_layer.cpp Update pooling_layer.cpp Update pooling_layer.cpp Update pooling_layer.cpp Update pooling_layer.cpp Implement poolingLayer by WebNN API and add more detailed logs Update dnn.cpp Update dnn.cpp Remove redundant codes and add more logs for poolingLayer Add more logs in the pooling layer implementation Fix the indent issue and resolve the compiling issue Fix the build problems Fix the build issue FIx the build issue Update dnn.cpp Update dnn.cpp --- CMakeLists.txt | 17 + cmake/OpenCVDetectWebNN.cmake | 23 ++ cmake/checks/webnn.cpp | 13 + cmake/templates/cvconfig.h.in | 3 + modules/dnn/CMakeLists.txt | 10 +- modules/dnn/include/opencv2/dnn/dnn.hpp | 3 + modules/dnn/src/dnn.cpp | 308 ++++++++++++++++++ modules/dnn/src/layers/elementwise_layers.cpp | 149 +++++++++ modules/dnn/src/layers/permute_layer.cpp | 16 + modules/dnn/src/layers/pooling_layer.cpp | 120 +++++++ modules/dnn/src/layers/reshape_layer.cpp | 13 + modules/dnn/src/layers/softmax_layer.cpp | 24 ++ modules/dnn/src/op_webnn.cpp | 237 ++++++++++++++ modules/dnn/src/op_webnn.hpp | 119 +++++++ modules/dnn/src/webnn/README.md | 11 + modules/dnn/test/test_common.hpp | 3 +- modules/dnn/test/test_common.impl.hpp | 15 +- samples/dnn/classification.cpp | 3 +- 18 files changed, 1083 insertions(+), 4 deletions(-) create mode 100644 cmake/OpenCVDetectWebNN.cmake create mode 100644 cmake/checks/webnn.cpp create mode 100644 modules/dnn/src/op_webnn.cpp create mode 100644 modules/dnn/src/op_webnn.hpp create mode 100644 modules/dnn/src/webnn/README.md diff --git a/CMakeLists.txt b/CMakeLists.txt index 41a77f04674b..b9d03ad8ea6d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -290,6 +290,9 @@ OCV_OPTION(WITH_INF_ENGINE "Include Intel Inference Engine support" OFF OCV_OPTION(WITH_NGRAPH "Include nGraph support" WITH_INF_ENGINE VISIBLE_IF TRUE VERIFY TARGET ngraph::ngraph) +OCV_OPTION(WITH_WEBNN "Include WebNN support" OFF + VISIBLE_IF TRUE + VERIFY HAVE_WEBNN) OCV_OPTION(WITH_JASPER "Include JPEG2K support (Jasper)" ON VISIBLE_IF NOT IOS VERIFY HAVE_JASPER) @@ -791,6 +794,11 @@ if(WITH_VULKAN) include(cmake/OpenCVDetectVulkan.cmake) endif() +# --- WebNN --- +if(WITH_WEBNN) + include(cmake/OpenCVDetectWebNN.cmake) +endif() + # --- Inference Engine --- if(WITH_INF_ENGINE) include(cmake/OpenCVDetectInferenceEngine.cmake) @@ -1616,6 +1624,15 @@ if(WITH_VULKAN OR HAVE_VULKAN) endif() endif() +if(WITH_WEBNN OR HAVE_WEBNN) + status("") + status(" WebNN:" HAVE_WEBNN THEN "YES" ELSE "NO") + if(HAVE_WEBNN) + status(" Include path:" WEBNN_HEADER_DIRS THEN "${WEBNN_HEADER_DIRS}" ELSE "NO") + status(" Link libraries:" WEBNN_LIBRARIES THEN "${WEBNN_LIBRARIES}" ELSE "NO") + endif() +endif() + if(WITH_OPENCL OR HAVE_OPENCL) ocv_build_features_string(opencl_features IF HAVE_OPENCL_SVM THEN "SVM" diff --git a/cmake/OpenCVDetectWebNN.cmake b/cmake/OpenCVDetectWebNN.cmake new file mode 100644 index 000000000000..90e69c46f7ab --- /dev/null +++ b/cmake/OpenCVDetectWebNN.cmake @@ -0,0 +1,23 @@ +ocv_clear_vars(HAVE_WEBNN) +ocv_clear_vars(WEBNN_EMSDK) +if(WITH_WEBNN) + set(WEBNN_HEADER_DIRS "$ENV{WEBNN_NATIVE_DIR}/gen/src/include") + set(WEBNN_INCLUDE_DIRS "$ENV{WEBNN_NATIVE_DIR}/../../src/include") + set(WEBNN_LIBRARIES "$ENV{WEBNN_NATIVE_DIR}/libwebnn_native.so;$ENV{WEBNN_NATIVE_DIR}/libwebnn_proc.so") +endif() + +try_compile(VALID_WEBNN + "${OpenCV_BINARY_DIR}" + SOURCES "${OpenCV_SOURCE_DIR}/cmake/checks/webnn.cpp" + "$ENV{WEBNN_NATIVE_DIR}/gen/src/webnn/webnn_cpp.cpp" + CMAKE_FLAGS "-DINCLUDE_DIRECTORIES:STRING=${WEBNN_INCLUDE_DIRS}\;${WEBNN_HEADER_DIRS}" + "-DLINK_LIBRARIES:STRING=${WEBNN_LIBRARIES}" + OUTPUT_VARIABLE TRY_OUT + ) +if(NOT ${VALID_WEBNN}) + message(WARNING "Can't use WebNN-native") + return() +endif() +message(AUTHOR_WARNING "Use WebNN-native") + +set(HAVE_WEBNN 1) diff --git a/cmake/checks/webnn.cpp b/cmake/checks/webnn.cpp new file mode 100644 index 000000000000..1a05f3569957 --- /dev/null +++ b/cmake/checks/webnn.cpp @@ -0,0 +1,13 @@ +#include +#include +#include +#include + + +int main(int /*argc*/, char** /*argv*/) +{ + WebnnProcTable backendProcs = webnn_native::GetProcs(); + webnnProcSetProcs(&backendProcs); + ml::Context ml_context = ml::Context(webnn_native::CreateContext()); + return 0; +} \ No newline at end of file diff --git a/cmake/templates/cvconfig.h.in b/cmake/templates/cvconfig.h.in index 6439d8b43f06..39708e14bdbf 100644 --- a/cmake/templates/cvconfig.h.in +++ b/cmake/templates/cvconfig.h.in @@ -59,6 +59,9 @@ /* Vulkan support */ #cmakedefine HAVE_VULKAN +/* Webnn support */ +#cmakedefine HAVE_WEBNN + /* Define to 1 if you have the header file. */ #cmakedefine HAVE_INTTYPES_H 1 diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index c1236c4653f2..54406c799010 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -134,6 +134,14 @@ if(HAVE_TENGINE) list(APPEND libs -Wl,--whole-archive ${TENGINE_LIBRARIES} -Wl,--no-whole-archive) endif() +set(webnn_srcs "") +if(HAVE_WEBNN) + list(APPEND include_dirs ${WEBNN_HEADER_DIRS}) + list(APPEND include_dirs ${WEBNN_INCLUDE_DIRS}) + list(APPEND libs -Wl,--whole-archive ${WEBNN_LIBRARIES} -Wl,--no-whole-archive) + list(APPEND webnn_srcs $ENV{WEBNN_NATIVE_DIR}/gen/src/webnn/webnn_cpp.cpp) +endif() + ocv_module_include_directories(${include_dirs}) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") ocv_append_source_files_cxx_compiler_options(fw_srcs "-Wno-suggest-override") # GCC @@ -163,7 +171,7 @@ if(HAVE_NGRAPH) list(APPEND dnn_runtime_libs ngraph::ngraph) endif() -ocv_glob_module_sources(${sources_options} SOURCES ${fw_srcs}) +ocv_glob_module_sources(${sources_options} SOURCES ${fw_srcs} ${webnn_srcs}) ocv_create_module(${libs} ${dnn_runtime_libs}) ocv_add_samples() ocv_add_accuracy_tests(${dnn_runtime_libs}) diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index bf1670051ac0..7624d43894a6 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -74,6 +74,7 @@ CV__DNN_INLINE_NS_BEGIN DNN_BACKEND_OPENCV, DNN_BACKEND_VKCOM, DNN_BACKEND_CUDA, + DNN_BACKEND_WEBNN, #ifdef __OPENCV_BUILD DNN_BACKEND_INFERENCE_ENGINE_NGRAPH = 1000000, // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType() DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019, // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType() @@ -307,6 +308,8 @@ CV__DNN_INLINE_NS_BEGIN virtual Ptr initVkCom(const std::vector > &inputs); + virtual Ptr initWebnn(const std::vector > &inputs, const std::vector >& nodes); + /** * @brief Returns a CUDA backend node * diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 4e38b0374f00..878a85a99ba2 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -45,6 +45,7 @@ #include "ie_ngraph.hpp" #include "op_vkcom.hpp" #include "op_cuda.hpp" +#include "op_webnn.hpp" #ifdef HAVE_CUDA #include "cuda4dnn/init.hpp" @@ -224,6 +225,13 @@ class BackendRegistry #endif #endif // HAVE_INF_ENGINE +#ifdef HAVE_WEBNN + if (haveWebnn()) + { + backends.push_back(std::make_pair(DNN_BACKEND_WEBNN, DNN_TARGET_CPU)); + } +#endif // HAVE_WEBNN + #ifdef HAVE_OPENCL if (cv::ocl::useOpenCL()) { @@ -1114,6 +1122,14 @@ static Ptr wrapMat(int backendId, int targetId, cv::Mat& m) return Ptr(new NgraphBackendWrapper(targetId, m)); #else CV_Error(Error::StsNotImplemented, "This OpenCV version is built without support of Inference Engine + nGraph"); +#endif + } + else if (backendId == DNN_BACKEND_WEBNN) + { +#ifdef HAVE_WEBNN + return Ptr(new WebnnBackendWrapper(targetId, m)); +#else + CV_Error(Error::StsNotImplemented, "This OpenCV version is built without support of WebNN"); #endif } else if (backendId == DNN_BACKEND_VKCOM) @@ -1259,6 +1275,12 @@ struct Net::Impl : public detail::NetImplBase { return wrapMat(preferableBackend, preferableTarget, host); } + else if (preferableBackend == DNN_BACKEND_WEBNN) + { +#ifdef HAVE_WEBNN + return wrapMat(preferableBackend, preferableTarget, host); +#endif + } else if (preferableBackend == DNN_BACKEND_VKCOM) { #ifdef HAVE_VULKAN @@ -1399,6 +1421,13 @@ struct Net::Impl : public detail::NetImplBase preferableTarget == DNN_TARGET_FPGA ); } +#endif +#ifdef HAVE_WEBNN + if (preferableBackend == DNN_BACKEND_WEBNN) + { + CV_Assert(preferableTarget == DNN_TARGET_CPU || + preferableTarget == DNN_TARGET_OPENCL); + } #endif CV_Assert(preferableBackend != DNN_BACKEND_VKCOM || preferableTarget == DNN_TARGET_VULKAN); @@ -1622,6 +1651,14 @@ struct Net::Impl : public detail::NetImplBase initNgraphBackend(blobsToKeep_); #else CV_Error(Error::StsNotImplemented, "This OpenCV version is built without support of Inference Engine + nGraph"); +#endif + } + else if (preferableBackend == DNN_BACKEND_WEBNN) + { +#ifdef HAVE_WEBNN + initWebnnBackend(blobsToKeep_); +#else + CV_Error(Error::StsNotImplemented, "This OpenCV version is built without support of WebNN"); #endif } else if (preferableBackend == DNN_BACKEND_VKCOM) @@ -2341,6 +2378,265 @@ struct Net::Impl : public detail::NetImplBase } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + void addWebnnOutputs(LayerData &ld) + { + CV_TRACE_FUNCTION(); + + Ptr layerNet; + auto it = ld.backendNodes.find(preferableBackend); + if (it != ld.backendNodes.end()) + { + Ptr node = it->second; + if (!node.empty()) + { + Ptr webnnNode = node.dynamicCast(); + CV_Assert(!webnnNode.empty()); CV_Assert(!webnnNode->net.empty()); + layerNet = webnnNode->net; + } + } + + for (int i = 0; i < ld.inputBlobsId.size(); ++i) + { + LayerData &inpLd = layers[ld.inputBlobsId[i].lid]; + Ptr inpNode = inpLd.backendNodes[preferableBackend]; + if (!inpNode.empty()) + { + Ptr webnnInpNode = inpNode.dynamicCast(); + CV_Assert(!webnnInpNode.empty()); CV_Assert(!webnnInpNode->net.empty()); + if (layerNet != webnnInpNode->net) + { + webnnInpNode->net->addOutput(webnnInpNode->name); + webnnInpNode->net->setUnconnectedNodes(webnnInpNode); + } + } + } + } + + void initWebnnBackend(const std::vector& blobsToKeep_) + { + CV_TRACE_FUNCTION(); + CV_Assert_N(preferableBackend == DNN_BACKEND_WEBNN, haveWebnn()); + + MapIdToLayerData::iterator it; + Ptr net; + + for (it = layers.begin(); it != layers.end(); ++it) + { + LayerData &ld = it->second; + if (ld.id == 0) + { + CV_Assert((netInputLayer->outNames.empty() && ld.outputBlobsWrappers.size() == 1) || + (netInputLayer->outNames.size() == ld.outputBlobsWrappers.size())); + for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i) + { + Ptr wrapper = ld.outputBlobsWrappers[i].dynamicCast(); + std::string outputName = netInputLayer->outNames.empty() ? ld.name : netInputLayer->outNames[i]; + outputName = ld.outputBlobsWrappers.size() > 1 ? (outputName + "." + std::to_string(i)) : outputName; + wrapper->name = outputName; + } + } + else + { + for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i) + { + Ptr wrapper = ld.outputBlobsWrappers[i].dynamicCast(); + std::string outputName = ld.outputBlobsWrappers.size() > 1 ? (ld.name + "." + std::to_string(i)) : ld.name; + wrapper->name = outputName; + } + } + } + + // Build WebNN networks from sets of layers that support this + // backend. Split a whole model on several WebNN networks if + // some of layers are not implemented. + for (it = layers.begin(); it != layers.end(); ++it) + { + LayerData &ld = it->second; + + if (ld.id == 0 && ld.skip) + continue; + + bool fused = ld.skip; + Ptr layer = ld.layerInstance; + if (!fused && !layer->supportBackend(preferableBackend)) + { + // For test use. when not using WebNN, the test case will fail + // with the following code. + + CV_LOG_WARNING(NULL, "Layer " + ld.type + " name " + ld.name + " is unsupported by WebNN backend."); + + addWebnnOutputs(ld); + net = Ptr(); + layer->preferableTarget = DNN_TARGET_CPU; + + for (int i = 0; i < ld.inputBlobsId.size(); ++i) + { + LayerData &inpLd = layers[ld.inputBlobsId[i].lid]; + Ptr inpNode = inpLd.backendNodes[preferableBackend]; + if (!inpNode.empty()) { + Ptr webnnNode = inpNode.dynamicCast(); + CV_Assert(!webnnNode.empty()); + webnnNode->net->setUnconnectedNodes(webnnNode); + } + } + continue; + } + ld.skip = true; // Initially skip all WebNN supported layers. + + // Create a new network if one of inputs from different WebNN graph. + std::vector> inputNodes; + for (int i = 0; i < ld.inputBlobsId.size(); ++i) + { + // Layer_Test_ROIPooling.Accuracy has 2 inputs inpLD = 0, 0 -> has 4 inputNodes (input, rois, input, rois) + if (inputNodes.size() == ld.inputBlobsId.size()) { + break; + } + LayerData &inpLd = layers[ld.inputBlobsId[i].lid]; + Ptr inpNode = inpLd.backendNodes[preferableBackend]; + if (!inpNode.empty()) + { + Ptr webnnInpNode = inpNode.dynamicCast(); + CV_Assert(!webnnInpNode.empty()); CV_Assert(!webnnInpNode->net.empty()); + if (webnnInpNode->net == net && !fused) { + inputNodes.push_back(inpNode); + continue; + } + } + + if (net.empty()) { + net = Ptr(new WebnnNet()); + } + + if (!fused) { + std::vector inputNames; + std::vector inputs; + + auto curr_pos = inpLd.consumers.begin(); + auto compare = [&ld] (const LayerPin& lp) { return lp.lid == ld.id; }; + auto cons = curr_pos; + while ((cons = std::find_if(curr_pos, inpLd.consumers.end(), compare)) != + inpLd.consumers.end()) { + int cons_inp = cons->oid; + Ptr inpWrapper = inpLd.outputBlobsWrappers[cons_inp]. + dynamicCast(); + CV_Assert(!inpWrapper.empty()); + auto iter = std::find(inputNames.begin(), inputNames.end(), + inpWrapper->name); + if (iter == inputNames.end()) { + inputNames.push_back(inpWrapper->name); + inputs.push_back(inpLd.outputBlobs[cons_inp]); + } + curr_pos = cons + 1; + } + + auto inps = net->setInputs(inputs, inputNames); + for (auto& inp : inps) { + WebnnBackendNode* node = new WebnnBackendNode(inp); + node->net = net; + inputNodes.emplace_back(Ptr(node)); + } + } + } + + Ptr node; + if (!net.empty()) + { + if (fused) + { + bool inPlace = ld.inputBlobsId.size() == 1 && ld.outputBlobs.size() == 1 && + ld.inputBlobs[0]->data == ld.outputBlobs[0].data; + CV_Assert(inPlace); + node = layers[ld.inputBlobsId[0].lid].backendNodes[preferableBackend]; + ld.inputBlobsWrappers = layers[ld.inputBlobsId[0].lid].inputBlobsWrappers; + } + } + else { + net = Ptr(new WebnnNet()); + } + + if (!fused) + { + CV_Assert(ld.inputBlobsId.size() == inputNodes.size()); + for (int i = 0; i < ld.inputBlobsId.size(); ++i) + { + int lid = ld.inputBlobsId[i].lid; + int oid = ld.inputBlobsId[i].oid; + if (oid == 0 || lid == 0) + continue; + + auto webnnInpNode = inputNodes[i].dynamicCast(); + inputNodes[i] = Ptr(new WebnnBackendNode(webnnInpNode->operand)); + } + + if (layer->supportBackend(preferableBackend)) + { + node = layer->initWebnn(ld.inputBlobsWrappers, inputNodes); + for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i) + { + Ptr wrapper = ld.outputBlobsWrappers[i].dynamicCast(); + node.dynamicCast()->name = wrapper->name; + } + } + else + { + continue; + } + } + else if (node.empty()) + continue; + + ld.backendNodes[preferableBackend] = node; + + Ptr webnnNode = node.dynamicCast(); + CV_Assert(!webnnNode.empty()); + webnnNode->net = net; + + if (ld.consumers.empty()) { + // TF EAST_text_detection + webnnNode->net->setUnconnectedNodes(webnnNode); + } + for (const auto& pin : blobsToKeep_) + { + if (pin.lid == ld.id) + { + webnnNode->net->addOutput(webnnNode->name); + break; + } + } + net->addBlobs(ld.inputBlobsWrappers); + net->addBlobs(ld.outputBlobsWrappers); + addWebnnOutputs(ld); + } + + // Initialize all networks. + for (MapIdToLayerData::reverse_iterator it = layers.rbegin(); it != layers.rend(); ++it) + { + LayerData &ld = it->second; + auto iter = ld.backendNodes.find(preferableBackend); + if (iter == ld.backendNodes.end()) + continue; + + Ptr& node = iter->second; + if (node.empty()) + continue; + + Ptr webnnNode = node.dynamicCast(); + if (webnnNode.empty()) + continue; + + CV_Assert(!webnnNode->net.empty()); + + if (!webnnNode->net->isInitialized()) + { + webnnNode->net->setUnconnectedNodes(webnnNode); + webnnNode->net->createNet((Target)preferableTarget); + ld.skip = false; + } + } + } +#endif + void initVkComBackend() { CV_TRACE_FUNCTION(); @@ -3381,6 +3677,10 @@ struct Net::Impl : public detail::NetImplBase else if (preferableBackend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) { forwardNgraph(ld.outputBlobsWrappers, node, isAsync); + } + else if (preferableBackend == DNN_BACKEND_WEBNN) + { + forwardWebnn(ld.outputBlobsWrappers, node, isAsync); } else if (preferableBackend == DNN_BACKEND_VKCOM) { @@ -4815,6 +5115,7 @@ string Net::Impl::dump() case DNN_BACKEND_OPENCV: backend = "OCV/"; break; case DNN_BACKEND_VKCOM: backend = "VULKAN/"; break; case DNN_BACKEND_CUDA: backend = "CUDA/"; break; + case DNN_BACKEND_WEBNN: backend = "WEBNN/"; break; // don't use default: } out << "digraph G {\n"; @@ -5406,6 +5707,13 @@ Ptr Layer::initNgraph(const std::vector > & inp return Ptr(); } +Ptr Layer::initWebnn(const std::vector > & inputs, const std::vector >& nodes) +{ + CV_Error(Error::StsNotImplemented, "WebNN pipeline of " + type + + " layers is not defined."); + return Ptr(); +} + void Layer::applyHalideScheduler(Ptr& node, const std::vector &inputs, const std::vector &outputs, int targetId) const { diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 6dc1813c8bee..33baa5c50d58 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -47,9 +47,11 @@ #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" #include "../op_vkcom.hpp" +#include "../op_webnn.hpp" #include #include +#include #ifdef HAVE_OPENCL #include "opencl_kernels_dnn.hpp" @@ -59,6 +61,7 @@ #include "../cuda4dnn/primitives/activation.hpp" using namespace cv::dnn::cuda4dnn; #endif +#include namespace cv { @@ -181,6 +184,17 @@ class ElementWiseLayer : public Func::Layer } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + auto operand = func.initWebnnAPI(webnnGraphBuilder, webnnInpOperand); + return Ptr(new WebnnBackendNode(operand)); + } +#endif + virtual Ptr initVkCom(const std::vector >& inputs) CV_OVERRIDE { #ifdef HAVE_VULKAN @@ -314,6 +328,16 @@ struct ReLUFunctor : public BaseFunctor #ifdef HAVE_DNN_NGRAPH if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) return true; +#endif +#ifdef HAVE_WEBNN + if (backendId == DNN_BACKEND_WEBNN) { + // TODO: support PRELU + if (slope != 0) + { + CV_LOG_WARNING(NULL, "PRELU is not supported now."); + } + return slope == 0; + } #endif return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || @@ -436,6 +460,13 @@ struct ReLUFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + return builder.Relu(input); + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { @@ -486,6 +517,7 @@ struct ReLU6Functor : public BaseFunctor return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || backendId == DNN_BACKEND_HALIDE || + backendId == DNN_BACKEND_WEBNN || backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } @@ -582,6 +614,33 @@ struct ReLU6Functor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH + + +#ifdef HAVE_WEBNN + ml::Operand BuildConstant(const ml::GraphBuilder& builder, + const std::vector& dimensions, + const void* value, + size_t size, + ml::OperandType type) { + ml::OperandDescriptor desc; + desc.type = type; + desc.dimensions = dimensions.data(); + desc.dimensionsCount = (uint32_t)dimensions.size(); + ml::ArrayBufferView resource; + resource.buffer = const_cast(value); + resource.byteLength = size; + return builder.Constant(&desc, &resource); + } + + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + ml::ClampOptions clampOptions; + clampOptions.minValue = BuildConstant(builder, {}, &minValue, 1 * sizeof(float), ml::OperandType::Float32); + clampOptions.maxValue = BuildConstant(builder, {}, &maxValue, 1 * sizeof(float), ml::OperandType::Float32); + return builder.Clamp(input, &clampOptions); + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { @@ -680,6 +739,15 @@ struct TanHFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { @@ -792,6 +860,15 @@ struct SwishFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { @@ -917,6 +994,15 @@ struct MishFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { @@ -1031,6 +1117,15 @@ struct SigmoidFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { @@ -1143,6 +1238,15 @@ struct ELUFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { @@ -1261,6 +1365,15 @@ struct AbsValFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { @@ -1374,6 +1487,15 @@ struct BNLLFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { @@ -1546,6 +1668,15 @@ struct PowerFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { @@ -1686,6 +1817,15 @@ struct ExpFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { @@ -1823,6 +1963,15 @@ struct ChannelsPReLUFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) + { + CV_Error(Error::StsNotImplemented, ""); + ml::Operand operand; + return operand; + } +#endif + #ifdef HAVE_VULKAN std::shared_ptr initVkCom() { diff --git a/modules/dnn/src/layers/permute_layer.cpp b/modules/dnn/src/layers/permute_layer.cpp index 77c2469c050f..9e66eb6a648f 100644 --- a/modules/dnn/src/layers/permute_layer.cpp +++ b/modules/dnn/src/layers/permute_layer.cpp @@ -46,6 +46,7 @@ #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" #include "../op_vkcom.hpp" +#include "../op_webnn.hpp" #include #include @@ -119,6 +120,7 @@ class PermuteLayerImpl CV_FINAL : public PermuteLayer #endif return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || + backendId == DNN_BACKEND_WEBNN || ((backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && haveInfEngine()) || (backendId == DNN_BACKEND_VKCOM && haveVulkan()); } @@ -439,6 +441,20 @@ class PermuteLayerImpl CV_FINAL : public PermuteLayer } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + std::vector permutation(_order.begin(), _order.end()); + ml::TransposeOptions options; + options.permutation = permutation.data(); + options.permutationCount = permutation.size(); + auto operand = webnnGraphBuilder.Transpose(webnnInpOperand, &options); + return Ptr(new WebnnBackendNode(operand)); + } +#endif #ifdef HAVE_CUDA Ptr initCUDA( diff --git a/modules/dnn/src/layers/pooling_layer.cpp b/modules/dnn/src/layers/pooling_layer.cpp index 7653e536680c..20fa3c33d9a9 100644 --- a/modules/dnn/src/layers/pooling_layer.cpp +++ b/modules/dnn/src/layers/pooling_layer.cpp @@ -46,6 +46,7 @@ #include "../op_cuda.hpp" #include "../op_halide.hpp" #include "../op_inf_engine.hpp" +#include "../op_webnn.hpp" #ifdef HAVE_DNN_NGRAPH #include "../ie_ngraph.hpp" @@ -85,6 +86,7 @@ typedef int HALIDE_DIFF_T; #include "../cuda4dnn/primitives/max_unpooling.hpp" using namespace cv::dnn::cuda4dnn; #endif +#include namespace cv @@ -246,6 +248,51 @@ class PoolingLayerImpl CV_FINAL : public PoolingLayer (type == MAX || type == AVE); return false; } + else if (backendId == DNN_BACKEND_WEBNN) + { + if (kernel_size.empty() || kernel_size.size() == 2) + { + if (!haveWebnn()) + { + return false; + } + else + { + if (!ceilMode) + { + CV_LOG_WARNING(NULL, "ceilMode is not supported by WebNN backend."); + return false; + } + if (computeMaxIdx) + { + CV_LOG_WARNING(NULL, "Mask is not supported by WebNN backend."); + return false; + } + if (type != MAX && type != AVE) + { + if (type == STOCHASTIC) + { + CV_LOG_WARNING(NULL, "Stochastic Pooling is not supported by WebNN backend."); + } + if (type == SUM) + { + CV_LOG_WARNING(NULL, "Sum Pooling is not supported by WebNN backend."); + } + if (type == ROI) + { + CV_LOG_WARNING(NULL, "ROI Pooling is not supported by WebNN backend."); + } + if (type == PSROI) + { + CV_LOG_WARNING(NULL, "Position-sensitive ROI Pooling is not supported by WebNN backend."); + } + CV_LOG_WARNING(NULL, "WebNN backend only supports MaxPooling and AveragePooling currently."); + return false; + } + } + return true; + } + } return false; } @@ -607,6 +654,79 @@ class PoolingLayerImpl CV_FINAL : public PoolingLayer } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + struct Pool2dOptions { + public: + std::vector windowDimensions; + std::vector padding; + std::vector strides; + std::vector dilations; + ml::AutoPad autoPad = ml::AutoPad::Explicit; + ml::InputOperandLayout layout = ml::InputOperandLayout::Nchw; + + const ml::Pool2dOptions* AsPtr() { + if (!windowDimensions.empty()) { + mOptions.windowDimensionsCount = windowDimensions.size(); + mOptions.windowDimensions = windowDimensions.data(); + } + if (!padding.empty()) { + mOptions.paddingCount = padding.size(); + mOptions.padding = padding.data(); + } + if (!strides.empty()) { + mOptions.stridesCount = strides.size(); + mOptions.strides = strides.data(); + } + if (!dilations.empty()) { + mOptions.dilationsCount = dilations.size(); + mOptions.dilations = dilations.data(); + } + mOptions.layout = layout; + mOptions.autoPad = autoPad; + return &mOptions; + } + + private: + ml::Pool2dOptions mOptions; + }; + + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + std::cout << "Use WebNN Pooling Layer's Implementation." << std::endl; + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + Pool2dOptions options; + std::vector kernelSize(kernel_size.begin(), kernel_size.end()); + std::vector Strides(strides.begin(), strides.end()); + std::vector Padding; + if (padMode.empty()) { + Padding = {static_cast(pads_begin[0]), + static_cast(pads_end[0]), + static_cast(pads_begin[1]), + static_cast(pads_end[1])}; + } else if (padMode == "VALID") { + Padding = {0, 0, 0, 0}; + } else if (padMode == "SAME") { + options.autoPad = ml::AutoPad::SameUpper; + } + options.windowDimensions = kernelSize; + options.strides = Strides; + options.padding = Padding; + if (type == MAX) + { + auto operand = webnnGraphBuilder.MaxPool2d(webnnInpOperand, options.AsPtr()); + return Ptr(new WebnnBackendNode(operand)); + } + else if (type == AVE) + { + auto operand = webnnGraphBuilder.AveragePool2d(webnnInpOperand, options.AsPtr()); + return Ptr(new WebnnBackendNode(operand)); + } else { + CV_Error(Error::StsNotImplemented, "Unsupported pooling type"); + } + } +#endif // HAVE_WEBNN class PoolingInvoker : public ParallelLoopBody { diff --git a/modules/dnn/src/layers/reshape_layer.cpp b/modules/dnn/src/layers/reshape_layer.cpp index 4c10d155c8ae..0ba3abf04758 100644 --- a/modules/dnn/src/layers/reshape_layer.cpp +++ b/modules/dnn/src/layers/reshape_layer.cpp @@ -45,6 +45,7 @@ #include "../op_cuda.hpp" #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" +#include "../op_webnn.hpp" #include @@ -203,6 +204,7 @@ class ReshapeLayerImpl CV_FINAL : public ReshapeLayer { return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || + backendId == DNN_BACKEND_WEBNN || ((backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && haveInfEngine()); } @@ -330,6 +332,17 @@ class ReshapeLayerImpl CV_FINAL : public ReshapeLayer } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + const std::vector out(outShapes[0].begin(), outShapes[0].end()); + auto operand = webnnGraphBuilder.Reshape(webnnInpOperand, out.data(), out.size()); + return Ptr(new WebnnBackendNode(operand)); + } +#endif #ifdef HAVE_CUDA Ptr initCUDA( diff --git a/modules/dnn/src/layers/softmax_layer.cpp b/modules/dnn/src/layers/softmax_layer.cpp index e937e98f8c77..db2951808ffd 100644 --- a/modules/dnn/src/layers/softmax_layer.cpp +++ b/modules/dnn/src/layers/softmax_layer.cpp @@ -47,9 +47,11 @@ #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" #include "../op_vkcom.hpp" +#include "../op_webnn.hpp" #include #include +#include using std::max; #ifdef HAVE_OPENCL @@ -97,6 +99,16 @@ class SoftMaxLayerImpl CV_FINAL : public SoftmaxLayer virtual bool supportBackend(int backendId) CV_OVERRIDE { +#ifdef HAVE_WEBNN + if (backendId == DNN_BACKEND_WEBNN) { + // TODO: support logSoftMax + if (logSoftMax) + { + CV_LOG_WARNING(NULL, "logSoftMax is not supported by WebNN backend.") + } + return !logSoftMax; + } +#endif return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || (backendId == DNN_BACKEND_HALIDE && haveHalide() && axisRaw == 1) || @@ -390,6 +402,18 @@ class SoftMaxLayerImpl CV_FINAL : public SoftmaxLayer return true; } +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + auto operand = webnnGraphBuilder.Softmax(webnnInpOperand); + return Ptr(new WebnnBackendNode(operand)); + } + +#endif + int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/op_webnn.cpp b/modules/dnn/src/op_webnn.cpp new file mode 100644 index 000000000000..7cc9c4289ece --- /dev/null +++ b/modules/dnn/src/op_webnn.cpp @@ -0,0 +1,237 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2018-2019, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include +#include "op_webnn.hpp" + +#include +#include + +#include "opencv2/core/utils/filesystem.hpp" +#include "opencv2/core/utils/filesystem.private.hpp" + +#include + +namespace cv { namespace dnn { + +#ifdef HAVE_WEBNN + +static std::string kDefaultInpLayerName = "opencv_webnn_empty_inp_layer_name"; + +template +static inline std::vector getShape(const Mat& mat) +{ + std::vector result(mat.dims); + for (int i = 0; i < mat.dims; i++) + result[i] = (T)mat.size[i]; + return result; +} + +static std::vector > +webnnWrappers(const std::vector >& ptrs) +{ + std::vector > wrappers(ptrs.size()); + for (int i = 0; i < ptrs.size(); ++i) + { + CV_Assert(!ptrs[i].empty()); + wrappers[i] = ptrs[i].dynamicCast(); + CV_Assert(!wrappers[i].empty()); + } + return wrappers; +} + +// WebnnNet +WebnnNet::WebnnNet() +{ + hasNetOwner = false; + device_name = "CPU"; + + WebnnProcTable backendProcs = webnn_native::GetProcs(); + webnnProcSetProcs(&backendProcs); + context = ml::Context(webnn_native::CreateContext()); + builder = ::ml::CreateGraphBuilder(context); + namedOperands = ::ml::CreateNamedOperands(); +} + +void WebnnNet::addOutput(const std::string& name) +{ + requestedOutputs.push_back(name); +} + +void WebnnNet::createNet(Target targetId) { + init(targetId); +} + +void WebnnNet::init(Target targetId) +{ + switch (targetId) + { + case DNN_TARGET_CPU: + device_name = "CPU"; + break; + case DNN_TARGET_OPENCL: + device_name = "GPU"; + break; + default: + CV_Error(Error::StsNotImplemented, "Unknown target"); + }; + + graph = builder.Build(namedOperands); + CV_Assert(graph!=nullptr); + isInit = true; +} + +std::vector WebnnNet::setInputs(const std::vector& inputs, + const std::vector& names) { + CV_Assert_N(inputs.size() == names.size()); + std::vector current_inp; + for (size_t i = 0; i < inputs.size(); i++) + { + auto& m = inputs[i]; + std::vector dimensions = getShape(m); + ml::OperandDescriptor descriptor; + descriptor.dimensions = dimensions.data(); + descriptor.dimensionsCount = dimensions.size(); + if (m.type() == CV_32F) + { + descriptor.type = ml::OperandType::Float32; + } + else + { + CV_Error(Error::StsNotImplemented, format("Unsupported data type %s", typeToString(m.type()).c_str())); + } + ml::Operand inputOperand = builder.Input(names[i].c_str(), &descriptor); + current_inp.push_back(std::move(inputOperand)); + } + inputNames = names; + return current_inp; +} + +void WebnnNet::setUnconnectedNodes(Ptr& node) { + outputNames.push_back(node->name); + namedOperands.Set(outputNames.back().c_str(), node->operand); +} + +bool WebnnNet::isInitialized() +{ + return isInit; +} + +void WebnnNet::reset() +{ + allBlobs.clear(); + isInit = false; +} + +void WebnnNet::addBlobs(const std::vector >& ptrs) +{ + auto wrappers = webnnWrappers(ptrs); + for (const auto& wrapper : wrappers) + { + std::string name = wrapper->name; + name = name.empty() ? kDefaultInpLayerName : name; + allBlobs.insert({name, wrapper}); + } +} + +void WebnnNet::forward(const std::vector >& outBlobsWrappers, bool isAsync) +{ + CV_LOG_DEBUG(NULL, "WebnnNet::forward(" << (isAsync ? "async" : "sync") << ")"); + ml::NamedInputs named_inputs = ::ml::CreateNamedInputs(); + std::vector inputs(inputNames.size()); + for (int i = 0; i < inputNames.size(); ++i) { + const std::string& name = inputNames[i]; + ml::Input& input = inputs[i]; + auto blobIt = allBlobs.find(name); + CV_Assert(blobIt != allBlobs.end()); + const Ptr wrapper = blobIt->second; + input.resource.buffer = wrapper->host->data; + input.resource.byteLength = wrapper->size; + named_inputs.Set(name.c_str(), &input); + } + std::vector > outs = webnnWrappers(outBlobsWrappers); + ml::NamedOutputs named_outputs = ::ml::CreateNamedOutputs(); + std::vector outputs(outs.size()); + for (int i = 0; i < outs.size(); ++i) { + const std::string& name = outs[i]->name; + ml::ArrayBufferView& output = outputs[i]; + output.buffer = outs[i]->host->data; + output.byteLength = outs[i]->size; + named_outputs.Set(name.c_str(), &output); + } + ml::ComputeGraphStatus status = graph.Compute(named_inputs, named_outputs); + if (status != ::ml::ComputeGraphStatus::Success) { + CV_Error(Error::StsAssert, format("Failed to compute: %d", int(status))); + } +} + +// WebnnBackendNode +WebnnBackendNode::WebnnBackendNode(ml::Operand&& _operand) + : BackendNode(DNN_BACKEND_WEBNN), operand(std::move(_operand)) {} + +WebnnBackendNode::WebnnBackendNode(ml::Operand& _operand) + : BackendNode(DNN_BACKEND_WEBNN), operand(_operand) {} + +// WebnnBackendWrapper +WebnnBackendWrapper::WebnnBackendWrapper(int targetId, cv::Mat& m) + : BackendWrapper(DNN_BACKEND_WEBNN, targetId) +{ + size = m.total() * m.elemSize(); + // buffer.reset(new char[size]); + // std::memcpy(buffer.get(), m.data, size); + // dimensions = getShape(m); + // descriptor.dimensions = dimensions.data(); + // descriptor.dimensionsCount = dimensions.size(); + if (m.type() == CV_32F) + { + descriptor.type = ml::OperandType::Float32; + } + else + { + CV_Error(Error::StsNotImplemented, format("Unsupported data type %s", typeToString(m.type()).c_str())); + } + host = &m; +} + +WebnnBackendWrapper::~WebnnBackendWrapper() +{ + // nothing +} + +void WebnnBackendWrapper::copyToHost() +{ + CV_LOG_DEBUG(NULL, "WebnnBackendWrapper::copyToHost()"); + //CV_Error(Error::StsNotImplemented, ""); +} + +void WebnnBackendWrapper::setHostDirty() +{ + CV_LOG_DEBUG(NULL, "WebnnBackendWrapper::setHostDirty()"); + //CV_Error(Error::StsNotImplemented, ""); +} + +void forwardWebnn(const std::vector >& outBlobsWrappers, + Ptr& node, bool isAsync) +{ + CV_Assert(!node.empty()); + Ptr webnnNode = node.dynamicCast(); + CV_Assert(!webnnNode.empty()); + webnnNode->net->forward(outBlobsWrappers, isAsync); +} + + +#else +void forwardWebnn(const std::vector >& outBlobsWrappers, + Ptr& operand, bool isAsync) +{ + CV_Assert(false && "WebNN is not enabled in this OpenCV build"); +} + +#endif + +} +} \ No newline at end of file diff --git a/modules/dnn/src/op_webnn.hpp b/modules/dnn/src/op_webnn.hpp new file mode 100644 index 000000000000..5a33b4847113 --- /dev/null +++ b/modules/dnn/src/op_webnn.hpp @@ -0,0 +1,119 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2018-2019, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. + +#ifndef __OPENCV_DNN_OP_WEBNN_HPP__ +#define __OPENCV_DNN_OP_WEBNN_HPP__ + +#include "opencv2/core/cvdef.h" +#include "opencv2/core/cvstd.hpp" +#include "opencv2/dnn.hpp" + +#ifdef HAVE_WEBNN + +#include +#include +#include +#include + +#include +#include + +#endif // HAVE_WEBNN + +namespace cv { namespace dnn { + +constexpr bool haveWebnn() { +#ifdef HAVE_WEBNN + return true; +#else + return false; +#endif +} + +#ifdef HAVE_WEBNN + +class WebnnBackendNode; +class WebnnBackendWrapper; + + + +class WebnnNet +{ +public: + WebnnNet(); + + void addOutput(const std::string& name); + + bool isInitialized(); + void init(Target targetId); + + void forward(const std::vector >& outBlobsWrappers, bool isAsync); + + std::vector setInputs(const std::vector& inputs, const std::vector& names); + + void setUnconnectedNodes(Ptr& node); + void addBlobs(const std::vector >& ptrs); + + void createNet(Target targetId); + // void setNodePtr(std::shared_ptr* ptr); + + void reset(); + + ml::GraphBuilder builder; + ml::Context context; + ml::Graph graph; + + std::unordered_map> allBlobs; + + bool hasNetOwner; + std::string device_name; + bool isInit = false; + + std::vector requestedOutputs; + + std::vector inputNames; + std::vector outputNames; + ml::NamedOperands namedOperands; +}; + +class WebnnBackendNode : public BackendNode +{ +public: + WebnnBackendNode(ml::Operand&& operand); + WebnnBackendNode(ml::Operand& operand); + + std::string name; + ml::Operand operand; + Ptr net; +}; + +class WebnnBackendWrapper : public BackendWrapper +{ +public: + WebnnBackendWrapper(int targetId, Mat& m); + ~WebnnBackendWrapper(); + + virtual void copyToHost() CV_OVERRIDE; + virtual void setHostDirty() CV_OVERRIDE; + + std::string name; + Mat* host; + std::unique_ptr buffer; + size_t size; + std::vector dimensions; + ml::OperandDescriptor descriptor; +}; + +#endif // HAVE_WebNN + +void forwardWebnn(const std::vector >& outBlobsWrappers, + Ptr& node, bool isAsync); + +}} // namespace cv::dnn + + +#endif // __OPENCV_DNN_OP_WEBNN_HPP__ diff --git a/modules/dnn/src/webnn/README.md b/modules/dnn/src/webnn/README.md new file mode 100644 index 000000000000..4d6fe20c0c5b --- /dev/null +++ b/modules/dnn/src/webnn/README.md @@ -0,0 +1,11 @@ +## Build Instructions + +### Build WebNN-native and set the environment variable + +Refer to [WebNN's build instructions](https://github.com/webmachinelearning/webnn-native) to complete the build of WebNN-native. + +Set environment variable `WEBNN_NATIVE_DIR` to enable native DNN_BACKEND_WEBNN build: `export WEBNN_NATIVE_DIR=${PATH_TO_WebNN}`. Please let `WEBNN_NATIVE_DIR` points the output directory of webnn-native build (e.g. webnn-native/out/Release). + +### Test native DNN_BACKEND_WEBNN backend +Add -DWITH_WEBNN=ON to the cmake command to build the WebNN module such as: +`cmake -DWITH_WEBNN=ON ../opencv` (according to the [Installation in Linux](https://docs.opencv.org/master/d7/d9f/tutorial_linux_install.html)) \ No newline at end of file diff --git a/modules/dnn/test/test_common.hpp b/modules/dnn/test/test_common.hpp index 139f3d1671b1..f20aa507c100 100644 --- a/modules/dnn/test/test_common.hpp +++ b/modules/dnn/test/test_common.hpp @@ -135,7 +135,8 @@ testing::internal::ParamGenerator< tuple > dnnBackendsAndTarget bool withCpuOCV = true, bool withVkCom = true, bool withCUDA = true, - bool withNgraph = true + bool withNgraph = true, + bool withWebnn = true ); testing::internal::ParamGenerator< tuple > dnnBackendsAndTargetsIE(); diff --git a/modules/dnn/test/test_common.impl.hpp b/modules/dnn/test/test_common.impl.hpp index 3d56e6f30875..c312474256f2 100644 --- a/modules/dnn/test/test_common.impl.hpp +++ b/modules/dnn/test/test_common.impl.hpp @@ -29,6 +29,7 @@ void PrintTo(const cv::dnn::Backend& v, std::ostream* os) case DNN_BACKEND_CUDA: *os << "CUDA"; return; case DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019: *os << "DLIE"; return; case DNN_BACKEND_INFERENCE_ENGINE_NGRAPH: *os << "NGRAPH"; return; + case DNN_BACKEND_WEBNN: *os << "WEBNN"; return; } // don't use "default:" to emit compiler warnings *os << "DNN_BACKEND_UNKNOWN(" << (int)v << ")"; } @@ -247,7 +248,8 @@ testing::internal::ParamGenerator< tuple > dnnBackendsAndTarget bool withCpuOCV /*= true*/, bool withVkCom /*= true*/, bool withCUDA /*= true*/, - bool withNgraph /*= true*/ + bool withNgraph /*= true*/, + bool withWebnn /*= false*/ ) { #ifdef HAVE_INF_ENGINE @@ -302,6 +304,17 @@ testing::internal::ParamGenerator< tuple > dnnBackendsAndTarget } #endif +#ifdef HAVE_WEBNN + if (withWebnn) + { + for (auto target : getAvailableTargets(DNN_BACKEND_WEBNN)) { + targets.push_back(make_tuple(DNN_BACKEND_WEBNN, target)); + } + } +#else + CV_UNUSED(withWebnn); +#endif + { available = getAvailableTargets(DNN_BACKEND_OPENCV); for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i) diff --git a/samples/dnn/classification.cpp b/samples/dnn/classification.cpp index 769d6874bed4..8af62e7e8b1f 100644 --- a/samples/dnn/classification.cpp +++ b/samples/dnn/classification.cpp @@ -24,7 +24,8 @@ std::string keys = "2: Intel's Deep Learning Inference Engine (https://software.intel.com/openvino-toolkit), " "3: OpenCV implementation, " "4: VKCOM, " - "5: CUDA }," + "5: CUDA, " + "6: WebNN }" "{ target | 0 | Choose one of target computation devices: " "0: CPU target (by default), " "1: OpenCL, " From fbd98954826fa64a3ff77bef1c5a5fe8c46ce3c5 Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Mon, 2 Aug 2021 14:52:02 +0800 Subject: [PATCH 140/376] Fix the build issue --- modules/dnn/src/dnn.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 878a85a99ba2..65b5261c3707 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -2463,7 +2463,7 @@ struct Net::Impl : public detail::NetImplBase { // For test use. when not using WebNN, the test case will fail // with the following code. - + CV_LOG_WARNING(NULL, "Layer " + ld.type + " name " + ld.name + " is unsupported by WebNN backend."); addWebnnOutputs(ld); From b509b3101c4cabdc2c3c860e72c9d100a2191b08 Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Wed, 4 Aug 2021 15:05:27 +0800 Subject: [PATCH 141/376] Implement BatchNorm Layer by WebNN API --- modules/dnn/src/layers/batch_norm_layer.cpp | 38 +++++++++++++++++++++ modules/dnn/src/op_webnn.cpp | 9 ----- modules/dnn/src/op_webnn.hpp | 8 +++++ 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index 49804c5c13e9..1c408dafe0d5 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -15,6 +15,7 @@ Implementation of Batch Normalization layer. #include "../op_halide.hpp" #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" +#include "../op_webnn.hpp" #include @@ -172,6 +173,7 @@ class BatchNormLayerImpl CV_FINAL : public BatchNormLayer return (backendId == DNN_BACKEND_OPENCV) || backendId == DNN_BACKEND_CUDA || (backendId == DNN_BACKEND_HALIDE && haveHalide()) || + backendId == DNN_BACKEND_WEBNN || ((backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && haveInfEngine() && (preferableTarget == DNN_TARGET_CPU || dims == 4)); } @@ -421,6 +423,42 @@ class BatchNormLayerImpl CV_FINAL : public BatchNormLayer return true; } +#ifdef HAVE_WEBNN + ml::Operand BuildConstant(const ml::GraphBuilder& builder, + const std::vector& dimensions, + const void* value, + size_t size, + ml::OperandType type) { + ml::OperandDescriptor desc; + desc.type = type; + desc.dimensions = dimensions.data(); + desc.dimensionsCount = (uint32_t)dimensions.size(); + ml::ArrayBufferView resource; + resource.buffer = const_cast(value); + resource.byteLength = size; + return builder.Constant(&desc, &resource); + } + + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + std::vector weights_shape = getShape(weights_); + ml::Operand weights = BuildConstant(webnnGraphBuilder, weights_shape, weights_.data, weights_.total()*weights_.elemSize(), ml::OperandType::Float32); + std::vector shape(dims, 1); + shape[1] = weights_shape[1]; + ml::Operand weights_reshaped = webnnGraphBuilder.Reshape(weights, shape.data(), shape.size()); + ml::Operand mul_res = webnnGraphBuilder.Mul(webnnInpOperand, weights_reshaped); + std::vector bias_shape = getShape(bias_); + ml::Operand bias = BuildConstant(webnnGraphBuilder, bias_shape, bias_.data, bias_.total()*bias_.elemSize(), ml::OperandType::Float32); + shape[1] = bias_shape[1]; + ml::Operand bias_reshaped = webnnGraphBuilder.Reshape(bias, shape.data(), shape.size()); + ml::Operand add_res = webnnGraphBuilder.Add(mul_res, bias_reshaped); + return Ptr(new WebnnBackendNode(add_res)); + } +#endif + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/op_webnn.cpp b/modules/dnn/src/op_webnn.cpp index 7cc9c4289ece..00156eaf0971 100644 --- a/modules/dnn/src/op_webnn.cpp +++ b/modules/dnn/src/op_webnn.cpp @@ -22,15 +22,6 @@ namespace cv { namespace dnn { static std::string kDefaultInpLayerName = "opencv_webnn_empty_inp_layer_name"; -template -static inline std::vector getShape(const Mat& mat) -{ - std::vector result(mat.dims); - for (int i = 0; i < mat.dims; i++) - result[i] = (T)mat.size[i]; - return result; -} - static std::vector > webnnWrappers(const std::vector >& ptrs) { diff --git a/modules/dnn/src/op_webnn.hpp b/modules/dnn/src/op_webnn.hpp index 5a33b4847113..726ba94f389f 100644 --- a/modules/dnn/src/op_webnn.hpp +++ b/modules/dnn/src/op_webnn.hpp @@ -40,6 +40,14 @@ class WebnnBackendNode; class WebnnBackendWrapper; +template +inline std::vector getShape(const Mat& mat) +{ + std::vector result(mat.dims); + for (int i = 0; i < mat.dims; i++) + result[i] = (T)mat.size[i]; + return result; +} class WebnnNet { From f501b0c16fb0264ade47261c44d2f233b3dce01c Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Wed, 4 Aug 2021 17:24:18 +0800 Subject: [PATCH 142/376] Update convolution_layer.cpp This is a temporary file for Conv2d layer implementation --- modules/dnn/src/layers/convolution_layer.cpp | 34 ++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index ec2904ee69be..e6c24ced25dd 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -346,6 +346,10 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl #ifdef HAVE_VULKAN if (backendId == DNN_BACKEND_VKCOM) return ksize == 2; +#endif +#ifdef HAVE_WEBNN + if (backendId == DNN_BACKEND_WEBNN) + return ksize == 2 && !blobs.empty(); #endif return false; } @@ -895,6 +899,36 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + ml::Operand BuildConstant(const ml::GraphBuilder& builder, + const std::vector& dimensions, + const void* value, + size_t size, + ml::OperandType type) { + ml::OperandDescriptor desc; + desc.type = type; + desc.dimensions = dimensions.data(); + desc.dimensionsCount = (uint32_t)dimensions.size(); + ml::ArrayBufferView resource; + resource.buffer = const_cast(value); + resource.byteLength = size; + return builder.Constant(&desc, &resource); + } + + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + CV_Assert_N(inputs.size() >= 1, nodes.size() >= 1); + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + std::vector dims = kernel_size; + + + + return Ptr(new WebnnBackendNode(operand)); + } +#endif // HAVE_WEBNN + class ParallelConv : public cv::ParallelLoopBody { public: From fb0edb268a6e49785d8ac0acf395ffa0382f6a7f Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 5 Aug 2021 14:33:19 +0800 Subject: [PATCH 143/376] Integrate some general functions into op_webnn.cpp&hpp --- modules/dnn/src/layers/batch_norm_layer.cpp | 23 ++------- modules/dnn/src/layers/elementwise_layers.cpp | 19 +------ modules/dnn/src/layers/pooling_layer.cpp | 37 +------------- modules/dnn/src/op_webnn.cpp | 19 ++++++- modules/dnn/src/op_webnn.hpp | 51 +++++++++++++++++-- 5 files changed, 71 insertions(+), 78 deletions(-) diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index 1c408dafe0d5..c6a016f7c281 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -424,34 +424,19 @@ class BatchNormLayerImpl CV_FINAL : public BatchNormLayer } #ifdef HAVE_WEBNN - ml::Operand BuildConstant(const ml::GraphBuilder& builder, - const std::vector& dimensions, - const void* value, - size_t size, - ml::OperandType type) { - ml::OperandDescriptor desc; - desc.type = type; - desc.dimensions = dimensions.data(); - desc.dimensionsCount = (uint32_t)dimensions.size(); - ml::ArrayBufferView resource; - resource.buffer = const_cast(value); - resource.byteLength = size; - return builder.Constant(&desc, &resource); - } - virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { Ptr node = nodes[0].dynamicCast(); auto& webnnInpOperand = node->operand; auto& webnnGraphBuilder = node->net->builder; - std::vector weights_shape = getShape(weights_); - ml::Operand weights = BuildConstant(webnnGraphBuilder, weights_shape, weights_.data, weights_.total()*weights_.elemSize(), ml::OperandType::Float32); + std::vector weights_shape = webnn::getShape(weights_); + ml::Operand weights = webnn::BuildConstant(webnnGraphBuilder, weights_shape, weights_.data, weights_.total()*weights_.elemSize(), ml::OperandType::Float32); std::vector shape(dims, 1); shape[1] = weights_shape[1]; ml::Operand weights_reshaped = webnnGraphBuilder.Reshape(weights, shape.data(), shape.size()); ml::Operand mul_res = webnnGraphBuilder.Mul(webnnInpOperand, weights_reshaped); - std::vector bias_shape = getShape(bias_); - ml::Operand bias = BuildConstant(webnnGraphBuilder, bias_shape, bias_.data, bias_.total()*bias_.elemSize(), ml::OperandType::Float32); + std::vector bias_shape = webnn::getShape(bias_); + ml::Operand bias = webnn::BuildConstant(webnnGraphBuilder, bias_shape, bias_.data, bias_.total()*bias_.elemSize(), ml::OperandType::Float32); shape[1] = bias_shape[1]; ml::Operand bias_reshaped = webnnGraphBuilder.Reshape(bias, shape.data(), shape.size()); ml::Operand add_res = webnnGraphBuilder.Add(mul_res, bias_reshaped); diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 33baa5c50d58..528d062c385c 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -617,26 +617,11 @@ struct ReLU6Functor : public BaseFunctor #ifdef HAVE_WEBNN - ml::Operand BuildConstant(const ml::GraphBuilder& builder, - const std::vector& dimensions, - const void* value, - size_t size, - ml::OperandType type) { - ml::OperandDescriptor desc; - desc.type = type; - desc.dimensions = dimensions.data(); - desc.dimensionsCount = (uint32_t)dimensions.size(); - ml::ArrayBufferView resource; - resource.buffer = const_cast(value); - resource.byteLength = size; - return builder.Constant(&desc, &resource); - } - ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) { ml::ClampOptions clampOptions; - clampOptions.minValue = BuildConstant(builder, {}, &minValue, 1 * sizeof(float), ml::OperandType::Float32); - clampOptions.maxValue = BuildConstant(builder, {}, &maxValue, 1 * sizeof(float), ml::OperandType::Float32); + clampOptions.minValue = webnn::BuildConstant(builder, {}, &minValue, 1 * sizeof(float), ml::OperandType::Float32); + clampOptions.maxValue = webnn::BuildConstant(builder, {}, &maxValue, 1 * sizeof(float), ml::OperandType::Float32); return builder.Clamp(input, &clampOptions); } #endif diff --git a/modules/dnn/src/layers/pooling_layer.cpp b/modules/dnn/src/layers/pooling_layer.cpp index 20fa3c33d9a9..5f1f32e25589 100644 --- a/modules/dnn/src/layers/pooling_layer.cpp +++ b/modules/dnn/src/layers/pooling_layer.cpp @@ -655,48 +655,13 @@ class PoolingLayerImpl CV_FINAL : public PoolingLayer #endif // HAVE_DNN_NGRAPH #ifdef HAVE_WEBNN - struct Pool2dOptions { - public: - std::vector windowDimensions; - std::vector padding; - std::vector strides; - std::vector dilations; - ml::AutoPad autoPad = ml::AutoPad::Explicit; - ml::InputOperandLayout layout = ml::InputOperandLayout::Nchw; - - const ml::Pool2dOptions* AsPtr() { - if (!windowDimensions.empty()) { - mOptions.windowDimensionsCount = windowDimensions.size(); - mOptions.windowDimensions = windowDimensions.data(); - } - if (!padding.empty()) { - mOptions.paddingCount = padding.size(); - mOptions.padding = padding.data(); - } - if (!strides.empty()) { - mOptions.stridesCount = strides.size(); - mOptions.strides = strides.data(); - } - if (!dilations.empty()) { - mOptions.dilationsCount = dilations.size(); - mOptions.dilations = dilations.data(); - } - mOptions.layout = layout; - mOptions.autoPad = autoPad; - return &mOptions; - } - - private: - ml::Pool2dOptions mOptions; - }; - virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { std::cout << "Use WebNN Pooling Layer's Implementation." << std::endl; Ptr node = nodes[0].dynamicCast(); auto& webnnInpOperand = node->operand; auto& webnnGraphBuilder = node->net->builder; - Pool2dOptions options; + webnn::Pool2dOptions options; std::vector kernelSize(kernel_size.begin(), kernel_size.end()); std::vector Strides(strides.begin(), strides.end()); std::vector Padding; diff --git a/modules/dnn/src/op_webnn.cpp b/modules/dnn/src/op_webnn.cpp index 00156eaf0971..6e7fa46ce45e 100644 --- a/modules/dnn/src/op_webnn.cpp +++ b/modules/dnn/src/op_webnn.cpp @@ -20,6 +20,23 @@ namespace cv { namespace dnn { #ifdef HAVE_WEBNN +namespace webnn { +ml::Operand BuildConstant(const ml::GraphBuilder& builder, + const std::vector& dimensions, + const void* value, + size_t size, + ml::OperandType type) { + ml::OperandDescriptor desc; + desc.type = type; + desc.dimensions = dimensions.data(); + desc.dimensionsCount = (uint32_t)dimensions.size(); + ml::ArrayBufferView resource; + resource.buffer = const_cast(value); + resource.byteLength = size; + return builder.Constant(&desc, &resource); + } +} + static std::string kDefaultInpLayerName = "opencv_webnn_empty_inp_layer_name"; static std::vector > @@ -83,7 +100,7 @@ std::vector WebnnNet::setInputs(const std::vector& inputs, for (size_t i = 0; i < inputs.size(); i++) { auto& m = inputs[i]; - std::vector dimensions = getShape(m); + std::vector dimensions = webnn::getShape(m); ml::OperandDescriptor descriptor; descriptor.dimensions = dimensions.data(); descriptor.dimensionsCount = dimensions.size(); diff --git a/modules/dnn/src/op_webnn.hpp b/modules/dnn/src/op_webnn.hpp index 726ba94f389f..ed8093c99a01 100644 --- a/modules/dnn/src/op_webnn.hpp +++ b/modules/dnn/src/op_webnn.hpp @@ -39,16 +39,57 @@ constexpr bool haveWebnn() { class WebnnBackendNode; class WebnnBackendWrapper; - -template -inline std::vector getShape(const Mat& mat) +namespace webnn { +inline std::vector getShape(const Mat& mat) { - std::vector result(mat.dims); + std::vector result(mat.dims); for (int i = 0; i < mat.dims; i++) - result[i] = (T)mat.size[i]; + result[i] = (int32_t)mat.size[i]; return result; } +ml::Operand BuildConstant(const ml::GraphBuilder& builder, + const std::vector& dimensions, + const void* value, + size_t size, + ml::OperandType type); + +struct Pool2dOptions { + public: + std::vector windowDimensions; + std::vector padding; + std::vector strides; + std::vector dilations; + ml::AutoPad autoPad = ml::AutoPad::Explicit; + ml::InputOperandLayout layout = ml::InputOperandLayout::Nchw; + + const ml::Pool2dOptions* AsPtr() { + if (!windowDimensions.empty()) { + mOptions.windowDimensionsCount = windowDimensions.size(); + mOptions.windowDimensions = windowDimensions.data(); + } + if (!padding.empty()) { + mOptions.paddingCount = padding.size(); + mOptions.padding = padding.data(); + } + if (!strides.empty()) { + mOptions.stridesCount = strides.size(); + mOptions.strides = strides.data(); + } + if (!dilations.empty()) { + mOptions.dilationsCount = dilations.size(); + mOptions.dilations = dilations.data(); + } + mOptions.layout = layout; + mOptions.autoPad = autoPad; + return &mOptions; + } + + private: + ml::Pool2dOptions mOptions; + }; +} + class WebnnNet { public: From d045047d315b43a04bb031a321dd1c17a3e9f05f Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 5 Aug 2021 15:33:36 +0800 Subject: [PATCH 144/376] Update const_layer.cpp --- modules/dnn/src/layers/const_layer.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/modules/dnn/src/layers/const_layer.cpp b/modules/dnn/src/layers/const_layer.cpp index 18f190b36b9a..c8a258600615 100644 --- a/modules/dnn/src/layers/const_layer.cpp +++ b/modules/dnn/src/layers/const_layer.cpp @@ -10,6 +10,7 @@ #include "../op_cuda.hpp" #include "layers_common.hpp" #include "../ie_ngraph.hpp" +#include "../op_webnn.hpp" #ifdef HAVE_OPENCL #include "opencl_kernels_dnn.hpp" @@ -36,6 +37,7 @@ class ConstLayerImpl CV_FINAL : public ConstLayer return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH || + backendId == DNN_BACKEND_WEBNN || backendId == DNN_BACKEND_CUDA; } @@ -97,6 +99,15 @@ class ConstLayerImpl CV_FINAL : public ConstLayer } #endif // HAVE_DNN_NGRAPH +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnGraphBuilder = node->net->builder; + ml::Operand operand = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); + return Ptr(new WebnnBackendNode(operand)); + } +#endif #ifdef HAVE_CUDA Ptr initCUDA( From f7d88af8d0bdf39d0409e9b2d0924069906721de Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 5 Aug 2021 19:29:03 +0800 Subject: [PATCH 145/376] Update convolution_layer.cpp Still have some bugs that should be fixed. --- modules/dnn/src/layers/convolution_layer.cpp | 126 ++++++++++++++++--- 1 file changed, 108 insertions(+), 18 deletions(-) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index e6c24ced25dd..c4d5a0661344 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -47,6 +47,7 @@ #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" #include "../op_vkcom.hpp" +#include "../op_webnn.hpp" #include @@ -349,7 +350,13 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl #endif #ifdef HAVE_WEBNN if (backendId == DNN_BACKEND_WEBNN) - return ksize == 2 && !blobs.empty(); + { + if (ksize != 2) + { + CV_LOG_WARNING(NULL, "WebNN only supports Conv2d."); + } + return ksize == 2; + } #endif return false; } @@ -900,32 +907,115 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl #endif // HAVE_DNN_NGRAPH #ifdef HAVE_WEBNN - ml::Operand BuildConstant(const ml::GraphBuilder& builder, - const std::vector& dimensions, - const void* value, - size_t size, - ml::OperandType type) { - ml::OperandDescriptor desc; - desc.type = type; - desc.dimensions = dimensions.data(); - desc.dimensionsCount = (uint32_t)dimensions.size(); - ml::ArrayBufferView resource; - resource.buffer = const_cast(value); - resource.byteLength = size; - return builder.Constant(&desc, &resource); - } - virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { CV_Assert_N(inputs.size() >= 1, nodes.size() >= 1); Ptr node = nodes[0].dynamicCast(); auto& webnnInpOperand = node->operand; auto& webnnGraphBuilder = node->net->builder; - std::vector dims = kernel_size; + ml::Operand webnnWeights = nodes.size() > 1 ? nodes[1].dynamicCast()->operand : nullptr; + if (nodes.size() > 1) + CV_Assert(webnnWeights); + // const int inpCn = weightsMat.total()/(kernel_size[0]*kernel_size[1]*numOutput); + const int group = blobs.size() - hasBias(); + const int inpGroupCn = blobs[0].size[1]; + // const int group = inpCn / inpGroupCn; + std::cout<<"Group: "< kernel_shape; + if (group != 1) + { + kernel_shape.push_back(group); + } + kernel_shape.push_back(numOutput / group); + kernel_shape.push_back(inpGroupCn); + std::copy(kernel_size.begin(), kernel_size.end(), back_inserter(kernel_shape)); + if (nodes.size() == 1) + { + std::cout<<"fusedWeights: "<(new WebnnBackendNode(operand)); + ml::Conv2dOptions options; + options.autoPad = pad_type; + // std::vector Strides(strides.begin(), strides.end()); + if (!strides.empty()) + { + options.stridesCount = strides.size(); + options.strides = reinterpret_cast(strides.data()); + } + std::vector Padding; + if (padMode.empty()) + { + Padding = {static_cast(pads_begin[0]), + static_cast(pads_end[0]), + static_cast(pads_begin[1]), + static_cast(pads_end[1])}; + } + else if (padMode == "VALID") + { + Padding = {0, 0, 0, 0}; + } + if (!Padding.empty()) + { + options.paddingCount = Padding.size(); + options.padding = Padding.data(); + } + std::cout<<"Padding: "< Dilations(dilations.begin(), dilations.end()); + if (!dilations.empty()) + { + options.dilationsCount = dilations.size(); + options.dilations = reinterpret_cast(dilations.data()); + } + const ml::Operand weights = webnnWeights; + const ml::Operand operand = webnnGraphBuilder.Conv2d(webnnInpOperand, weights, &options); + // const ml::Operand operand = webnnGraphBuilder.Conv2d(webnnInpOperand, *const_cast(&webnnWeights), &options); + // const ml::Operand operand = webnnGraphBuilder.Conv2d(webnnInpOperand, webnnWeights, &options); + + ml::Operand result = operand; + if (hasBias() || fusedBias || nodes.size() == 3) + { + ml::Operand webnnBias; + if (nodes.size() == 3) + { + webnnBias = webnnGraphBuilder.Reshape(nodes[2].dynamicCast()->operand, kernel_shape.data(), kernel_shape.size()); + } + else + { + webnnBias = webnn::BuildConstant(webnnGraphBuilder, kernel_shape, biasvec.data(), biasvec.size()*sizeof(float), ml::OperandType::Float32); + } + const ml::Operand bias = webnnBias; + // result = webnnGraphBuilder.Add(operand, *const_cast(&webnnBias)); + result = webnnGraphBuilder.Add(operand, bias); + // result = webnnGraphBuilder.Add(operand, webnnBias); + } + return Ptr(new WebnnBackendNode(result)); } #endif // HAVE_WEBNN From 875cfef9653e799fb962a396203b9b8de1384aff Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 5 Aug 2021 23:14:52 +0800 Subject: [PATCH 146/376] Update conv2d layer and fc layer still have some problems to be fixed. --- modules/dnn/src/layers/convolution_layer.cpp | 46 ++++++++----------- .../dnn/src/layers/fully_connected_layer.cpp | 33 +++++++++++++ 2 files changed, 53 insertions(+), 26 deletions(-) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index c4d5a0661344..33d51c265bf8 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -920,9 +920,9 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl const int group = blobs.size() - hasBias(); const int inpGroupCn = blobs[0].size[1]; // const int group = inpCn / inpGroupCn; - std::cout<<"Group: "< kernel_shape; if (group != 1) { @@ -934,9 +934,9 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl if (nodes.size() == 1) { - std::cout<<"fusedWeights: "< Strides(strides.begin(), strides.end()); - if (!strides.empty()) + std::vector Strides(strides.begin(), strides.end()); + if (!Strides.empty()) { - options.stridesCount = strides.size(); - options.strides = reinterpret_cast(strides.data()); + options.stridesCount = Strides.size(); + options.strides = Strides.data(); } std::vector Padding; if (padMode.empty()) @@ -986,22 +986,19 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl options.paddingCount = Padding.size(); options.padding = Padding.data(); } - std::cout<<"Padding: "< Dilations(dilations.begin(), dilations.end()); - if (!dilations.empty()) + // std::cout<<"Padding: "< Dilations(dilations.begin(), dilations.end()); + if (!Dilations.empty()) { - options.dilationsCount = dilations.size(); - options.dilations = reinterpret_cast(dilations.data()); + options.dilationsCount = Dilations.size(); + options.dilations = Dilations.data(); } - const ml::Operand weights = webnnWeights; - const ml::Operand operand = webnnGraphBuilder.Conv2d(webnnInpOperand, weights, &options); - // const ml::Operand operand = webnnGraphBuilder.Conv2d(webnnInpOperand, *const_cast(&webnnWeights), &options); - // const ml::Operand operand = webnnGraphBuilder.Conv2d(webnnInpOperand, webnnWeights, &options); + ml::Operand operand = webnnGraphBuilder.Conv2d(webnnInpOperand, webnnWeights, &options); - ml::Operand result = operand; + // ml::Operand result = operand; if (hasBias() || fusedBias || nodes.size() == 3) { - ml::Operand webnnBias; + ml::Operand webnnBias = nullptr; if (nodes.size() == 3) { webnnBias = webnnGraphBuilder.Reshape(nodes[2].dynamicCast()->operand, kernel_shape.data(), kernel_shape.size()); @@ -1010,12 +1007,9 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl { webnnBias = webnn::BuildConstant(webnnGraphBuilder, kernel_shape, biasvec.data(), biasvec.size()*sizeof(float), ml::OperandType::Float32); } - const ml::Operand bias = webnnBias; - // result = webnnGraphBuilder.Add(operand, *const_cast(&webnnBias)); - result = webnnGraphBuilder.Add(operand, bias); - // result = webnnGraphBuilder.Add(operand, webnnBias); + operand = webnnGraphBuilder.Add(operand, webnnBias); } - return Ptr(new WebnnBackendNode(result)); + return Ptr(new WebnnBackendNode(operand)); } #endif // HAVE_WEBNN diff --git a/modules/dnn/src/layers/fully_connected_layer.cpp b/modules/dnn/src/layers/fully_connected_layer.cpp index 28ea7f347fef..47d580570948 100644 --- a/modules/dnn/src/layers/fully_connected_layer.cpp +++ b/modules/dnn/src/layers/fully_connected_layer.cpp @@ -46,6 +46,7 @@ #include "../op_halide.hpp" #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" +#include "../op_webnn.hpp" #include @@ -150,6 +151,7 @@ class FullyConnectedLayerImpl CV_FINAL : public InnerProductLayer return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || (backendId == DNN_BACKEND_HALIDE && haveHalide() && axis == 1) || + (backendId == DNN_BACKEND_WEBNN && axis == 1) || (((backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && !blobs.empty()) || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && axis == 1); } @@ -657,6 +659,37 @@ class FullyConnectedLayerImpl CV_FINAL : public InnerProductLayer return true; } +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnInpOperand = node->operand; + auto& webnnGraphBuilder = node->net->builder; + ml::GemmOptions gemmOptions = {}; + if (bias) + { + std::vector biasDims = {(int32_t)blobs[1].size[1]}; + ml::Operand bias = webnn::BuildConstant(webnnGraphBuilder, biasDims, blobs[1].data, blobs[1].total()*blobs[1].elemSize(), ml::OperandType::Float32); + gemmOptions.c = bias; + } + ml::Operand result = nullptr; + if (nodes.size() == 2) + { + auto& inp2 = nodes[1].dynamicCast()->operand; + result = webnnGraphBuilder.Gemm(webnnInpOperand, inp2, &gemmOptions); + } + else + { + std::vector weight_shape = {(int32_t)blobs[0].size[0], (int32_t)blobs[0].size[1]}; + std::cout<<"weight_shape: " << weight_shape[0]<<" "<(new WebnnBackendNode(result)); + } +#endif // HAVE_WEBNN + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { From 7b2d6cdcd24d63095b3730ea6024184b6cc99a54 Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Fri, 6 Aug 2021 15:15:13 +0800 Subject: [PATCH 147/376] update constLayer, conv layer, fc layer There are still some bugs to be fixed. --- modules/dnn/src/layers/const_layer.cpp | 18 ++++++++++++++--- modules/dnn/src/layers/convolution_layer.cpp | 20 +++++++++---------- .../dnn/src/layers/fully_connected_layer.cpp | 1 - 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/modules/dnn/src/layers/const_layer.cpp b/modules/dnn/src/layers/const_layer.cpp index c8a258600615..d531c7c6486a 100644 --- a/modules/dnn/src/layers/const_layer.cpp +++ b/modules/dnn/src/layers/const_layer.cpp @@ -102,9 +102,21 @@ class ConstLayerImpl CV_FINAL : public ConstLayer #ifdef HAVE_WEBNN virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { - Ptr node = nodes[0].dynamicCast(); - auto& webnnGraphBuilder = node->net->builder; - ml::Operand operand = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); + ml::Operand operand = nullptr; + if (nodes.size() != 0) + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnGraphBuilder = node->net->builder; + operand = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); + } + else + { + WebnnProcTable backendProcs = webnn_native::GetProcs(); + webnnProcSetProcs(&backendProcs); + ml::Context context = ml::Context(webnn_native::CreateContext()); + ml::GraphBuilder builder = ml::CreateGraphBuilder(context); + operand = webnn::BuildConstant(builder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); + } return Ptr(new WebnnBackendNode(operand)); } #endif diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index 33d51c265bf8..a71ef8fbaa8a 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -348,16 +348,16 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl if (backendId == DNN_BACKEND_VKCOM) return ksize == 2; #endif -#ifdef HAVE_WEBNN - if (backendId == DNN_BACKEND_WEBNN) - { - if (ksize != 2) - { - CV_LOG_WARNING(NULL, "WebNN only supports Conv2d."); - } - return ksize == 2; - } -#endif +// #ifdef HAVE_WEBNN +// if (backendId == DNN_BACKEND_WEBNN) +// { +// if (ksize != 2) +// { +// CV_LOG_WARNING(NULL, "WebNN only supports Conv2d."); +// } +// return ksize == 2; +// } +// #endif return false; } diff --git a/modules/dnn/src/layers/fully_connected_layer.cpp b/modules/dnn/src/layers/fully_connected_layer.cpp index 47d580570948..b1df6adf3f8b 100644 --- a/modules/dnn/src/layers/fully_connected_layer.cpp +++ b/modules/dnn/src/layers/fully_connected_layer.cpp @@ -681,7 +681,6 @@ class FullyConnectedLayerImpl CV_FINAL : public InnerProductLayer else { std::vector weight_shape = {(int32_t)blobs[0].size[0], (int32_t)blobs[0].size[1]}; - std::cout<<"weight_shape: " << weight_shape[0]<<" "< Date: Tue, 10 Aug 2021 14:25:03 +0800 Subject: [PATCH 148/376] Fix the build issue --- modules/dnn/src/layers/convolution_layer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index a71ef8fbaa8a..4b0271470cbe 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -994,7 +994,7 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl options.dilations = Dilations.data(); } ml::Operand operand = webnnGraphBuilder.Conv2d(webnnInpOperand, webnnWeights, &options); - + // ml::Operand result = operand; if (hasBias() || fusedBias || nodes.size() == 3) { From 6d7b77089f548914c6e7908378ac87bd2b8ce943 Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Tue, 10 Aug 2021 17:44:31 +0800 Subject: [PATCH 149/376] Update concat_layer.cpp Still have some bugs to be fixed. --- modules/dnn/src/layers/concat_layer.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/modules/dnn/src/layers/concat_layer.cpp b/modules/dnn/src/layers/concat_layer.cpp index 536114fcd772..f620d66a39da 100644 --- a/modules/dnn/src/layers/concat_layer.cpp +++ b/modules/dnn/src/layers/concat_layer.cpp @@ -47,6 +47,7 @@ #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" #include "../op_vkcom.hpp" +#include "../op_webnn.hpp" #ifdef HAVE_OPENCL #include "opencl_kernels_dnn.hpp" @@ -117,6 +118,7 @@ class ConcatLayerImpl CV_FINAL : public ConcatLayer (backendId == DNN_BACKEND_HALIDE && haveHalide() && axis == 1 && !padding) || // By channels (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && haveInfEngine() && !padding) || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH || + (backendId == DNN_BACKEND_WEBNN && !padding) || (backendId == DNN_BACKEND_VKCOM && haveVulkan() && !padding); } @@ -408,6 +410,22 @@ class ConcatLayerImpl CV_FINAL : public ConcatLayer params.set("padding_value", zeropoints[1][0]); return true; } + +#ifdef HAVE_WEBNN + virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnGraphBuilder = node->net->builder; + std::vector inputsOperand; + for (int i = 0; i < nodes.size(); i++) + { + inputsOperand.push_back(nodes[i].dynamicCast()->operand); + } + auto operand = webnnGraphBuilder.Concat(inputsOperand.size(), inputsOperand.data(), axis); + return Ptr(new WebnnBackendNode(operand)); + } +#endif + }; Ptr ConcatLayer::create(const LayerParams& params) From 66bfb50c1299291ec3f210265aa9cd512b61017e Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 12 Aug 2021 17:03:48 +0800 Subject: [PATCH 150/376] Update conv2d layer, fully connected layer and const layer --- modules/dnn/src/dnn.cpp | 6 ++ modules/dnn/src/layers/const_layer.cpp | 17 +---- modules/dnn/src/layers/convolution_layer.cpp | 64 +++++++++---------- .../dnn/src/layers/fully_connected_layer.cpp | 7 +- 4 files changed, 47 insertions(+), 47 deletions(-) diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 65b5261c3707..30262831c6a6 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -2571,6 +2571,12 @@ struct Net::Impl : public detail::NetImplBase if (layer->supportBackend(preferableBackend)) { + if (ld.type == "Const") { + ml::Operand fake_operand; + Ptr fake_input_node = Ptr(new WebnnBackendNode(fake_operand)); + fake_input_node->net = net; + inputNodes.push_back(fake_input_node); + } node = layer->initWebnn(ld.inputBlobsWrappers, inputNodes); for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i) { diff --git a/modules/dnn/src/layers/const_layer.cpp b/modules/dnn/src/layers/const_layer.cpp index d531c7c6486a..1f307b8fa6aa 100644 --- a/modules/dnn/src/layers/const_layer.cpp +++ b/modules/dnn/src/layers/const_layer.cpp @@ -103,20 +103,9 @@ class ConstLayerImpl CV_FINAL : public ConstLayer virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { ml::Operand operand = nullptr; - if (nodes.size() != 0) - { - Ptr node = nodes[0].dynamicCast(); - auto& webnnGraphBuilder = node->net->builder; - operand = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); - } - else - { - WebnnProcTable backendProcs = webnn_native::GetProcs(); - webnnProcSetProcs(&backendProcs); - ml::Context context = ml::Context(webnn_native::CreateContext()); - ml::GraphBuilder builder = ml::CreateGraphBuilder(context); - operand = webnn::BuildConstant(builder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); - } + Ptr node = nodes[0].dynamicCast(); + auto& webnnGraphBuilder = node->net->builder; + operand = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); return Ptr(new WebnnBackendNode(operand)); } #endif diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index 4b0271470cbe..352a5d6a87a3 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -348,16 +348,16 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl if (backendId == DNN_BACKEND_VKCOM) return ksize == 2; #endif -// #ifdef HAVE_WEBNN -// if (backendId == DNN_BACKEND_WEBNN) -// { -// if (ksize != 2) -// { -// CV_LOG_WARNING(NULL, "WebNN only supports Conv2d."); -// } -// return ksize == 2; -// } -// #endif +#ifdef HAVE_WEBNN + if (backendId == DNN_BACKEND_WEBNN) + { + if (ksize != 2) + { + CV_LOG_WARNING(NULL, "WebNN only supports Conv2d."); + } + return ksize == 2; + } +#endif return false; } @@ -918,43 +918,42 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl CV_Assert(webnnWeights); // const int inpCn = weightsMat.total()/(kernel_size[0]*kernel_size[1]*numOutput); const int group = blobs.size() - hasBias(); - const int inpGroupCn = blobs[0].size[1]; - // const int group = inpCn / inpGroupCn; - // std::cout<<"Group: "< kernel_shape; - if (group != 1) - { - kernel_shape.push_back(group); - } - kernel_shape.push_back(numOutput / group); - kernel_shape.push_back(inpGroupCn); - std::copy(kernel_size.begin(), kernel_size.end(), back_inserter(kernel_shape)); + // const int inpGroupCn = blobs[0].size[1]; + // // const int group = inpCn / inpGroupCn; + // const int group = 1; + // // std::cout<<"Group: "< kernel_shape; + // if (group != 1) + // { + // kernel_shape.push_back(group); + // } + // kernel_shape.push_back(numOutput / group); + // kernel_shape.push_back(inpGroupCn); + // std::copy(kernel_size.begin(), kernel_size.end(), back_inserter(kernel_shape)); if (nodes.size() == 1) { - // std::cout<<"fusedWeights: "< Dilations(dilations.begin(), dilations.end()); if (!Dilations.empty()) { @@ -1001,11 +999,13 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl ml::Operand webnnBias = nullptr; if (nodes.size() == 3) { - webnnBias = webnnGraphBuilder.Reshape(nodes[2].dynamicCast()->operand, kernel_shape.data(), kernel_shape.size()); + std::vector bias_shape = {1, numOutput / group, 1, 1}; + webnnBias = webnnGraphBuilder.Reshape(nodes[2].dynamicCast()->operand, bias_shape.data(), bias_shape.size()); } else { - webnnBias = webnn::BuildConstant(webnnGraphBuilder, kernel_shape, biasvec.data(), biasvec.size()*sizeof(float), ml::OperandType::Float32); + + webnnBias = webnn::BuildConstant(webnnGraphBuilder, {1, numOutput / group, 1, 1}, biasvec.data(), biasvec.size()*sizeof(float), ml::OperandType::Float32); } operand = webnnGraphBuilder.Add(operand, webnnBias); } diff --git a/modules/dnn/src/layers/fully_connected_layer.cpp b/modules/dnn/src/layers/fully_connected_layer.cpp index b1df6adf3f8b..83d5d2bb34d5 100644 --- a/modules/dnn/src/layers/fully_connected_layer.cpp +++ b/modules/dnn/src/layers/fully_connected_layer.cpp @@ -680,10 +680,15 @@ class FullyConnectedLayerImpl CV_FINAL : public InnerProductLayer } else { + std::vector input_shape(2, -1); + input_shape[1] = blobs[0].size[1]; + std::cout<<"input size: "< weight_shape = {(int32_t)blobs[0].size[0], (int32_t)blobs[0].size[1]}; + std::cout<<"weight size: "<(new WebnnBackendNode(result)); } From d13bef5ae777f39a229ba5a19ff85eb0b7c9510c Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Sun, 15 Aug 2021 15:29:46 +0800 Subject: [PATCH 151/376] Update convolution_layer.cpp --- modules/dnn/src/layers/convolution_layer.cpp | 30 ++++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index 352a5d6a87a3..9696714acd06 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -917,21 +917,21 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl if (nodes.size() > 1) CV_Assert(webnnWeights); // const int inpCn = weightsMat.total()/(kernel_size[0]*kernel_size[1]*numOutput); - const int group = blobs.size() - hasBias(); - // const int inpGroupCn = blobs[0].size[1]; + // const int group = blobs.size() - hasBias(); + const int inpGroupCn = blobs[0].size[1]; // // const int group = inpCn / inpGroupCn; - // const int group = 1; - // // std::cout<<"Group: "< kernel_shape; - // if (group != 1) - // { - // kernel_shape.push_back(group); - // } - // kernel_shape.push_back(numOutput / group); - // kernel_shape.push_back(inpGroupCn); - // std::copy(kernel_size.begin(), kernel_size.end(), back_inserter(kernel_shape)); + const int group = 1; + // std::cout<<"Group: "< kernel_shape; + if (group != 1) + { + kernel_shape.push_back(group); + } + kernel_shape.push_back(numOutput / group); + kernel_shape.push_back(inpGroupCn); + std::copy(kernel_size.begin(), kernel_size.end(), back_inserter(kernel_shape)); if (nodes.size() == 1) { @@ -953,7 +953,7 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl } else { - // webnnWeights = webnnGraphBuilder.Reshape(webnnWeights, kernel_shape.data(), kernel_shape.size()); + webnnWeights = webnnGraphBuilder.Reshape(webnnWeights, kernel_shape.data(), kernel_shape.size()); } ml::AutoPad pad_type = ml::AutoPad::Explicit; From fd494a50c370a5755433aaf215a4bb1ed111aee1 Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Sun, 22 Aug 2021 18:39:06 +0800 Subject: [PATCH 152/376] Add OpenCV.js DNN module WebNN Backend (both using webnn-polyfill and electron) --- CMakeLists.txt | 2 +- build_js/doc/doxygen/bib19450.aux | 134 +++++++++ cmake/OpenCVDetectWebNN.cmake | 36 ++- cmake/checks/webnn.cpp | 10 + ...s_image_classification_webnn_electron.html | 264 +++++++++++++++++ ...s_image_classification_webnn_polyfill.html | 265 ++++++++++++++++++ doc/js_tutorials/js_assets/main.js | 55 ++++ doc/js_tutorials/js_assets/node_setup.js | 12 + doc/js_tutorials/js_assets/package.json | 14 + .../js_assets/utils_webnn_electron.js | 159 +++++++++++ modules/dnn/src/dnn.cpp | 4 +- modules/dnn/src/layers/convolution_layer.cpp | 5 +- .../dnn/src/layers/fully_connected_layer.cpp | 4 +- modules/dnn/src/layers/pooling_layer.cpp | 3 +- modules/dnn/src/op_webnn.cpp | 6 + modules/dnn/src/op_webnn.hpp | 6 + modules/js/CMakeLists.txt | 2 +- platforms/js/build_js.py | 12 +- platforms/js/opencv_js.config.py | 2 +- 19 files changed, 970 insertions(+), 25 deletions(-) create mode 100644 build_js/doc/doxygen/bib19450.aux create mode 100644 doc/js_tutorials/js_assets/js_image_classification_webnn_electron.html create mode 100644 doc/js_tutorials/js_assets/js_image_classification_webnn_polyfill.html create mode 100644 doc/js_tutorials/js_assets/main.js create mode 100644 doc/js_tutorials/js_assets/node_setup.js create mode 100644 doc/js_tutorials/js_assets/package.json create mode 100644 doc/js_tutorials/js_assets/utils_webnn_electron.js diff --git a/CMakeLists.txt b/CMakeLists.txt index b9d03ad8ea6d..e8cd8105cf51 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1627,7 +1627,7 @@ endif() if(WITH_WEBNN OR HAVE_WEBNN) status("") status(" WebNN:" HAVE_WEBNN THEN "YES" ELSE "NO") - if(HAVE_WEBNN) + if(HAVE_WEBNN AND NOT EMSCRIPTEN) status(" Include path:" WEBNN_HEADER_DIRS THEN "${WEBNN_HEADER_DIRS}" ELSE "NO") status(" Link libraries:" WEBNN_LIBRARIES THEN "${WEBNN_LIBRARIES}" ELSE "NO") endif() diff --git a/build_js/doc/doxygen/bib19450.aux b/build_js/doc/doxygen/bib19450.aux new file mode 100644 index 000000000000..2e533495bdcb --- /dev/null +++ b/build_js/doc/doxygen/bib19450.aux @@ -0,0 +1,134 @@ +\relax +\bibstyle{doxygen} +\citation{AAM} +\citation{ABD12} +\citation{AMVOT} +\citation{ANB13} +\citation{Andreff99} +\citation{Arandjelovic:2012:TTE:2354409.2355123} +\citation{BA83} +\citation{BL07} +\citation{BT98} +\citation{Ballard1981} +\citation{Bolelli2017} +\citation{Bolelli2019} +\citation{Borgefors86} +\citation{Bouguet00} +\citation{BouguetMCT} +\citation{Bradski98} +\citation{Breiman84} +\citation{Brox2004} +\citation{CL12} +\citation{Canny86} +\citation{ChambolleEtAl} +\citation{Chaumette06} +\citation{Collins14} +\citation{DM03} +\citation{DM97} +\citation{Dalal2005} +\citation{Daniilidis98} +\citation{EM11} +\citation{EP08} +\citation{Eade13} +\citation{Eade17} +\citation{FHT98} +\citation{FL02} +\citation{Farneback2003} +\citation{Felzenszwalb04} +\citation{Fitzgibbon1999} +\citation{Fitzgibbon95} +\citation{GOTURN} +\citation{GW03} +\citation{Gallego2014ACF} +\citation{Grana2010} +\citation{Guil1999} +\citation{HH08} +\citation{HTF01} +\citation{Hartley99} +\citation{HartleyZ00} +\citation{Horaud95} +\citation{Hu62} +\citation{Ke17} +\citation{Kirkpatrick83} +\citation{KleeLaskowski85} +\citation{Kroeger2016} +\citation{LCS11} +\citation{Li2010SimultaneousRA} +\citation{Liao2007} +\citation{LibSVM} +\citation{Lienhart02} +\citation{Louhichi07} +\citation{Lowe04} +\citation{MA13} +\citation{MIL} +\citation{MK07} +\citation{MM06} +\citation{Ma:2003:IVI} +\citation{Madsen04} +\citation{Malis} +\citation{Marchand16} +\citation{Matas00} +\citation{Meyer92} +\citation{Mortensen95intelligentscissors} +\citation{Muja2009} +\citation{Nister03} +\citation{ORourke86} +\citation{PM03} +\citation{Park94} +\citation{Puzicha1997} +\citation{RB99} +\citation{RD05} +\citation{RPROP93} +\citation{RRKB11} +\citation{RS10} +\citation{Rafael12} +\citation{Rosten06} +\citation{Rubner2000} +\citation{RubnerSept98} +\citation{Shah2013SolvingTR} +\citation{Shi94} +\citation{Sklansky82} +\citation{Slabaugh} +\citation{Sol2018AML} +\citation{SteweniusCFS} +\citation{Suzuki85} +\citation{Taubin1991} +\citation{TehChin89} +\citation{Telea04} +\citation{Terzakis20} +\citation{Tsai89} +\citation{UES01} +\citation{V03} +\citation{VandLec} +\citation{Viola01} +\citation{Viola04} +\citation{WJ10} +\citation{Welch95} +\citation{Wu2009} +\citation{YM11} +\citation{Yuen90} +\citation{Zhang2000} +\citation{Zivkovic2004} +\citation{Zivkovic2006} +\citation{bigun2006vision} +\citation{blanco2010tutorial} +\citation{bottou2010large} +\citation{duda2018} +\citation{forssen2007maximally} +\citation{forstner1987fast} +\citation{gao2003complete} +\citation{gonzalez} +\citation{gruzman} +\citation{hesch2011direct} +\citation{jahne2000computer} +\citation{lepetit2009epnp} +\citation{liao2020real} +\citation{mair2010_agast} +\citation{nister2008linear} +\citation{penate2013exhaustive} +\citation{strobl2011iccv} +\citation{umeyama1991least} +\citation{vacavant2013benchmark} +\citation{van1995estimators} +\citation{yang1996structure} +\bibdata{bibTmpDir/bibTmpFile_1,bibTmpDir/bibTmpFile_2,bibTmpDir/bibTmpFile_3} diff --git a/cmake/OpenCVDetectWebNN.cmake b/cmake/OpenCVDetectWebNN.cmake index 90e69c46f7ab..8e83558674f4 100644 --- a/cmake/OpenCVDetectWebNN.cmake +++ b/cmake/OpenCVDetectWebNN.cmake @@ -1,19 +1,31 @@ ocv_clear_vars(HAVE_WEBNN) ocv_clear_vars(WEBNN_EMSDK) -if(WITH_WEBNN) - set(WEBNN_HEADER_DIRS "$ENV{WEBNN_NATIVE_DIR}/gen/src/include") - set(WEBNN_INCLUDE_DIRS "$ENV{WEBNN_NATIVE_DIR}/../../src/include") - set(WEBNN_LIBRARIES "$ENV{WEBNN_NATIVE_DIR}/libwebnn_native.so;$ENV{WEBNN_NATIVE_DIR}/libwebnn_proc.so") +if(NOT EMSCRIPTEN) + if(WITH_WEBNN) + set(WEBNN_HEADER_DIRS "$ENV{WEBNN_NATIVE_DIR}/gen/src/include") + set(WEBNN_INCLUDE_DIRS "$ENV{WEBNN_NATIVE_DIR}/../../src/include") + set(WEBNN_LIBRARIES "$ENV{WEBNN_NATIVE_DIR}/libwebnn_native.so;$ENV{WEBNN_NATIVE_DIR}/libwebnn_proc.so") + endif() endif() -try_compile(VALID_WEBNN - "${OpenCV_BINARY_DIR}" - SOURCES "${OpenCV_SOURCE_DIR}/cmake/checks/webnn.cpp" - "$ENV{WEBNN_NATIVE_DIR}/gen/src/webnn/webnn_cpp.cpp" - CMAKE_FLAGS "-DINCLUDE_DIRECTORIES:STRING=${WEBNN_INCLUDE_DIRS}\;${WEBNN_HEADER_DIRS}" - "-DLINK_LIBRARIES:STRING=${WEBNN_LIBRARIES}" - OUTPUT_VARIABLE TRY_OUT - ) +if(NOT EMSCRIPTEN) + try_compile(VALID_WEBNN + "${OpenCV_BINARY_DIR}" + SOURCES "${OpenCV_SOURCE_DIR}/cmake/checks/webnn.cpp" + "$ENV{WEBNN_NATIVE_DIR}/gen/src/webnn/webnn_cpp.cpp" + CMAKE_FLAGS "-DINCLUDE_DIRECTORIES:STRING=${WEBNN_INCLUDE_DIRS}\;${WEBNN_HEADER_DIRS}" + "-DLINK_LIBRARIES:STRING=${WEBNN_LIBRARIES}" + OUTPUT_VARIABLE TRY_OUT + ) +else() + try_compile(VALID_WEBNN + "${OpenCV_BINARY_DIR}" + SOURCES "${OpenCV_SOURCE_DIR}/cmake/checks/webnn.cpp" + OUTPUT_VARIABLE TRY_OUT + ) +endif() + + if(NOT ${VALID_WEBNN}) message(WARNING "Can't use WebNN-native") return() diff --git a/cmake/checks/webnn.cpp b/cmake/checks/webnn.cpp index 1a05f3569957..f1e365272ec2 100644 --- a/cmake/checks/webnn.cpp +++ b/cmake/checks/webnn.cpp @@ -1,13 +1,23 @@ #include #include +#ifdef __EMSCRIPTEN__ +#include +#include +#include +#else #include #include +#endif int main(int /*argc*/, char** /*argv*/) { +#ifdef __EMSCRIPTEN__ + ml::Context ml_context = ml::Context(emscripten_webnn_create_context()); +#else WebnnProcTable backendProcs = webnn_native::GetProcs(); webnnProcSetProcs(&backendProcs); ml::Context ml_context = ml::Context(webnn_native::CreateContext()); +#endif return 0; } \ No newline at end of file diff --git a/doc/js_tutorials/js_assets/js_image_classification_webnn_electron.html b/doc/js_tutorials/js_assets/js_image_classification_webnn_electron.html new file mode 100644 index 000000000000..f3b075489d0a --- /dev/null +++ b/doc/js_tutorials/js_assets/js_image_classification_webnn_electron.html @@ -0,0 +1,264 @@ + + + + + + Image Classification Example + + + + +

Image Classification Example

+

+ This tutorial shows you how to write an image classification example with OpenCV.js.
+ To try the example you should click the modelFile button(and configFile button if needed) to upload inference model. + You can find the model URLs and parameters in the model info section. + Then You should change the parameters in the first code snippet according to the uploaded model. + Finally click Try it button to see the result. You can choose any other images.
+

+ +
+
+ + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+
+
+ canvasInput +
+
+
+ modelFile +
+
+
+ configFile +
+
+
+ +
+

+
+ +
+

Help function

+

1.The parameters for model inference which you can modify to investigate more models.

+ +

2.Main loop in which will read the image from canvas and do inference once.

+ +

3.Load labels from txt file and process it into an array.

+ +

4.Get blob from image as input for net, and standardize it with mean and std.

+ +

5.Fetch model file and save to emscripten file system once click the input button.

+ +

6.The post-processing, including softmax if needed and get the top classes from the output vector.

+ +
+ +
+

Model Info:

+
+ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/js_tutorials/js_assets/js_image_classification_webnn_polyfill.html b/doc/js_tutorials/js_assets/js_image_classification_webnn_polyfill.html new file mode 100644 index 000000000000..6ef5e1517be5 --- /dev/null +++ b/doc/js_tutorials/js_assets/js_image_classification_webnn_polyfill.html @@ -0,0 +1,265 @@ + + + + + + Image Classification Example + + + + + +

Image Classification Example

+

+ This tutorial shows you how to write an image classification example with OpenCV.js.
+ To try the example you should click the modelFile button(and configFile button if needed) to upload inference model. + You can find the model URLs and parameters in the model info section. + Then You should change the parameters in the first code snippet according to the uploaded model. + Finally click Try it button to see the result. You can choose any other images.
+

+ +
+
+ + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+
+
+ canvasInput +
+
+
+ modelFile +
+
+
+ configFile +
+
+
+ +
+

+
+ +
+

Help function

+

1.The parameters for model inference which you can modify to investigate more models.

+ +

2.Main loop in which will read the image from canvas and do inference once.

+ +

3.Load labels from txt file and process it into an array.

+ +

4.Get blob from image as input for net, and standardize it with mean and std.

+ +

5.Fetch model file and save to emscripten file system once click the input button.

+ +

6.The post-processing, including softmax if needed and get the top classes from the output vector.

+ +
+ +
+

Model Info:

+
+ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/js_tutorials/js_assets/main.js b/doc/js_tutorials/js_assets/main.js new file mode 100644 index 000000000000..70e84862a22e --- /dev/null +++ b/doc/js_tutorials/js_assets/main.js @@ -0,0 +1,55 @@ +// Modules to control application life and create native browser window +const {app, BrowserWindow} = require('electron') +const path = require('path') + +// Keep a global reference of the window object, if you don't, the window will +// be closed automatically when the JavaScript object is garbage collected. +let mainWindow = {} + +function createWindow() { + // Create the browser window. + mainWindow = new BrowserWindow({ + width: 1220, + height: 840, + webPreferences: { + nodeIntegration: true, + preload: app.getAppPath()+"/node_setup.js" + } + }) + + // Load the index.html with 'numRunsParm' to run inference multiple times. + let url = `file://${__dirname}/js_image_classification_webnn_electron.html` + const numRunsParm = '?' + process.argv[2] + mainWindow.loadURL(url + numRunsParm) + + // Emitted when the window is closed. + mainWindow.on('closed', function() { + // Dereference the window object, usually you would store windows + // in an array if your app supports multi windows, this is the time + // when you should delete the corresponding element. + mainWindow = null + }) +} + +// This method will be called when Electron has finished +// initialization and is ready to create browser windows. +// Some APIs can only be used after this event occurs. +app.on('ready', createWindow) + +// Quit when all windows are closed. +app.on('window-all-closed', function() { + // On macOS it is common for applications and their menu bar + // to stay active until the user quits explicitly with Cmd + Q + if (process.platform !== 'darwin') app.quit() +}) + +app.on( + 'activate', + function() { + // On macOS it's common to re-create a window in the app when the + // dock icon is clicked and there are no other windows open. + if (mainWindow === null) createWindow() + }) + + // In this file you can include the rest of your app's specific main process + // code. You can also put them in separate files and require them here. diff --git a/doc/js_tutorials/js_assets/node_setup.js b/doc/js_tutorials/js_assets/node_setup.js new file mode 100644 index 000000000000..c269b5ce9488 --- /dev/null +++ b/doc/js_tutorials/js_assets/node_setup.js @@ -0,0 +1,12 @@ +const cv = require('./opencv'); +const webnn = require(process.env.WEBNN_NATIVE_DIR+'/../../node/lib/webnn'); +// navigator is undefined in node.js, but defined in electron.js. +if (global.navigator === undefined) { + global.navigator = {}; +} +global.navigator.ml = webnn.ml; +global.MLContext = webnn.MLContext +global.MLGraphBuilder = webnn.MLGraphBuilder +global.MLGraph = webnn.MLGraph +global.MLOperand = webnn.MLOperand +global.cv = cv; \ No newline at end of file diff --git a/doc/js_tutorials/js_assets/package.json b/doc/js_tutorials/js_assets/package.json new file mode 100644 index 000000000000..87011d592844 --- /dev/null +++ b/doc/js_tutorials/js_assets/package.json @@ -0,0 +1,14 @@ +{ + "name": "image_classification", + "version": "0.0.1", + "description": "An Electon.js example of image_classification using webnn-native", + "main": "main.js", + "author": "WebNN-native Authors", + "license": "Apache-2.0", + "scripts": { + "start": "electron ." + }, + "dependencies": { + "electron": "^11.0.3" + } +} diff --git a/doc/js_tutorials/js_assets/utils_webnn_electron.js b/doc/js_tutorials/js_assets/utils_webnn_electron.js new file mode 100644 index 000000000000..2c4e981715ba --- /dev/null +++ b/doc/js_tutorials/js_assets/utils_webnn_electron.js @@ -0,0 +1,159 @@ +function Utils(errorOutputId) { // eslint-disable-line no-unused-vars + let self = this; + this.errorOutput = document.getElementById(errorOutputId); + + const OPENCV_URL = 'opencv.js'; + this.loadOpenCv = async function(onloadCallback) { + if (cv.getBuildInformation) + { + console.log(cv.getBuildInformation()); + onloadCallback(); + } + else + { + // WASM + if (cv instanceof Promise) { + cv = await cv; + console.log(cv.getBuildInformation()); + onloadCallback(); + } else { + cv['onRuntimeInitialized']=()=>{ + console.log(cv.getBuildInformation()); + onloadCallback(); + } + } + } + }; + + this.createFileFromUrl = function(path, url, callback) { + let request = new XMLHttpRequest(); + request.open('GET', url, true); + request.responseType = 'arraybuffer'; + request.onload = function(ev) { + if (request.readyState === 4) { + if (request.status === 200) { + let data = new Uint8Array(request.response); + cv.FS_createDataFile('/', path, data, true, false, false); + callback(); + } else { + self.printError('Failed to load ' + url + ' status: ' + request.status); + } + } + }; + request.send(); + }; + + this.loadImageToCanvas = function(url, cavansId) { + let canvas = document.getElementById(cavansId); + let ctx = canvas.getContext('2d'); + let img = new Image(); + img.crossOrigin = 'anonymous'; + img.onload = function() { + canvas.width = img.width; + canvas.height = img.height; + ctx.drawImage(img, 0, 0, img.width, img.height); + }; + img.src = url; + }; + + this.executeCode = function(textAreaId) { + try { + this.clearError(); + let code = document.getElementById(textAreaId).value; + eval(code); + } catch (err) { + this.printError(err); + } + }; + + this.clearError = function() { + this.errorOutput.innerHTML = ''; + }; + + this.printError = function(err) { + if (typeof err === 'undefined') { + err = ''; + } else if (typeof err === 'number') { + if (!isNaN(err)) { + if (typeof cv !== 'undefined') { + err = 'Exception: ' + cv.exceptionFromPtr(err).msg; + } + } + } else if (typeof err === 'string') { + let ptr = Number(err.split(' ')[0]); + if (!isNaN(ptr)) { + if (typeof cv !== 'undefined') { + err = 'Exception: ' + cv.exceptionFromPtr(ptr).msg; + } + } + } else if (err instanceof Error) { + err = err.stack.replace(/\n/g, '
'); + } + this.errorOutput.innerHTML = err; + }; + + this.loadCode = function(scriptId, textAreaId) { + let scriptNode = document.getElementById(scriptId); + let textArea = document.getElementById(textAreaId); + if (scriptNode.type !== 'text/code-snippet') { + throw Error('Unknown code snippet type'); + } + textArea.value = scriptNode.text.replace(/^\n/, ''); + }; + + this.addFileInputHandler = function(fileInputId, canvasId) { + let inputElement = document.getElementById(fileInputId); + inputElement.addEventListener('change', (e) => { + let files = e.target.files; + if (files.length > 0) { + let imgUrl = URL.createObjectURL(files[0]); + self.loadImageToCanvas(imgUrl, canvasId); + } + }, false); + }; + + function onVideoCanPlay() { + if (self.onCameraStartedCallback) { + self.onCameraStartedCallback(self.stream, self.video); + } + }; + + this.startCamera = function(resolution, callback, videoId) { + const constraints = { + 'qvga': {width: {exact: 320}, height: {exact: 240}}, + 'vga': {width: {exact: 640}, height: {exact: 480}}}; + let video = document.getElementById(videoId); + if (!video) { + video = document.createElement('video'); + } + + let videoConstraint = constraints[resolution]; + if (!videoConstraint) { + videoConstraint = true; + } + + navigator.mediaDevices.getUserMedia({video: videoConstraint, audio: false}) + .then(function(stream) { + video.srcObject = stream; + video.play(); + self.video = video; + self.stream = stream; + self.onCameraStartedCallback = callback; + video.addEventListener('canplay', onVideoCanPlay, false); + }) + .catch(function(err) { + self.printError('Camera Error: ' + err.name + ' ' + err.message); + }); + }; + + this.stopCamera = function() { + if (this.video) { + this.video.pause(); + this.video.srcObject = null; + this.video.removeEventListener('canplay', onVideoCanPlay); + } + if (this.stream) { + this.stream.getVideoTracks()[0].stop(); + } + }; +}; diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 30262831c6a6..7c4950eb9255 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -4976,7 +4976,9 @@ void Net::setInput(InputArray blob, const String& name, double scalefactor, cons MatShape prevShape = shape(impl->netInputLayer->inputsData[pin.oid]); bool oldShape = prevShape == blobShape; - blob_.copyTo(impl->netInputLayer->inputsData[pin.oid]); + // blob_.copyTo(impl->netInputLayer->inputsData[pin.oid]); + impl->netInputLayer->inputsData.emplace(impl->netInputLayer->inputsData.begin()+pin.oid, blob_); + impl->netInputLayer->inputsData.erase(impl->netInputLayer->inputsData.begin()+pin.oid+1); if (!oldShape) { ld.outputBlobs[pin.oid] = impl->netInputLayer->inputsData[pin.oid]; if (impl->hasDynamicShapes) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index 9696714acd06..0b550edbab38 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -1004,8 +1004,9 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl } else { - - webnnBias = webnn::BuildConstant(webnnGraphBuilder, {1, numOutput / group, 1, 1}, biasvec.data(), biasvec.size()*sizeof(float), ml::OperandType::Float32); + // std::cout<<"mumOutput size: "<< numOutput < input_shape(2, -1); input_shape[1] = blobs[0].size[1]; - std::cout<<"input size: "< weight_shape = {(int32_t)blobs[0].size[0], (int32_t)blobs[0].size[1]}; - std::cout<<"weight size: "< initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { - std::cout << "Use WebNN Pooling Layer's Implementation." << std::endl; + // std::cout << "Use WebNN Pooling Layer's Implementation." << std::endl; Ptr node = nodes[0].dynamicCast(); auto& webnnInpOperand = node->operand; auto& webnnGraphBuilder = node->net->builder; @@ -675,6 +675,7 @@ class PoolingLayerImpl CV_FINAL : public PoolingLayer } else if (padMode == "SAME") { options.autoPad = ml::AutoPad::SameUpper; } + // std::cout << "padMode: " << padMode << std::endl; options.windowDimensions = kernelSize; options.strides = Strides; options.padding = Padding; diff --git a/modules/dnn/src/op_webnn.cpp b/modules/dnn/src/op_webnn.cpp index 6e7fa46ce45e..3005bc255e5c 100644 --- a/modules/dnn/src/op_webnn.cpp +++ b/modules/dnn/src/op_webnn.cpp @@ -58,9 +58,13 @@ WebnnNet::WebnnNet() hasNetOwner = false; device_name = "CPU"; +#ifdef __EMSCRIPTEN__ + context = ml::Context(emscripten_webnn_create_context()); +#else WebnnProcTable backendProcs = webnn_native::GetProcs(); webnnProcSetProcs(&backendProcs); context = ml::Context(webnn_native::CreateContext()); +#endif builder = ::ml::CreateGraphBuilder(context); namedOperands = ::ml::CreateNamedOperands(); } @@ -168,7 +172,9 @@ void WebnnNet::forward(const std::vector >& outBlobsWrappers const std::string& name = outs[i]->name; ml::ArrayBufferView& output = outputs[i]; output.buffer = outs[i]->host->data; + // std::cout<<"host data size: "<host->total()*outs[i]->host->elemSize()<size; + // std::cout<<"outs[i]->size: "<< outs[i]->size << std::endl; named_outputs.Set(name.c_str(), &output); } ml::ComputeGraphStatus status = graph.Compute(named_inputs, named_outputs); diff --git a/modules/dnn/src/op_webnn.hpp b/modules/dnn/src/op_webnn.hpp index ed8093c99a01..f2284c6d63cd 100644 --- a/modules/dnn/src/op_webnn.hpp +++ b/modules/dnn/src/op_webnn.hpp @@ -16,8 +16,14 @@ #include #include +#ifdef __EMSCRIPTEN__ +#include +#include +#include +#else #include #include +#endif #include #include diff --git a/modules/js/CMakeLists.txt b/modules/js/CMakeLists.txt index 5996e419ddc9..5fb1d35694fb 100644 --- a/modules/js/CMakeLists.txt +++ b/modules/js/CMakeLists.txt @@ -69,7 +69,7 @@ if(COMPILE_FLAGS) set_target_properties(${the_module} PROPERTIES COMPILE_FLAGS ${COMPILE_FLAGS}) endif() -set(EMSCRIPTEN_LINK_FLAGS "${EMSCRIPTEN_LINK_FLAGS} --memory-init-file 0 -s TOTAL_MEMORY=128MB -s WASM_MEM_MAX=1GB -s ALLOW_MEMORY_GROWTH=1") +set(EMSCRIPTEN_LINK_FLAGS "${EMSCRIPTEN_LINK_FLAGS} -s USE_WEBNN=1 --memory-init-file 0 -s TOTAL_MEMORY=128MB -s WASM_MEM_MAX=1GB -s ALLOW_MEMORY_GROWTH=1") set(EMSCRIPTEN_LINK_FLAGS "${EMSCRIPTEN_LINK_FLAGS} -s MODULARIZE=1 -s SINGLE_FILE=1") set(EMSCRIPTEN_LINK_FLAGS "${EMSCRIPTEN_LINK_FLAGS} -s EXPORT_NAME=\"'cv'\" -s DEMANGLE_SUPPORT=1") set(EMSCRIPTEN_LINK_FLAGS "${EMSCRIPTEN_LINK_FLAGS} -s FORCE_FILESYSTEM=1 --use-preload-plugins --bind --post-js ${JS_HELPER} ${COMPILE_FLAGS}") diff --git a/platforms/js/build_js.py b/platforms/js/build_js.py index f490eb58d534..6b5d48371fde 100644 --- a/platforms/js/build_js.py +++ b/platforms/js/build_js.py @@ -67,6 +67,8 @@ def __init__(self, options): self.options = options self.build_dir = check_dir(options.build_dir, create=True) self.opencv_dir = check_dir(options.opencv_dir) + print('-----------------------------------------------------------') + print('options.opencv_dir:', options.opencv_dir) self.emscripten_dir = check_dir(options.emscripten_dir) def get_toolchain_file(self): @@ -84,6 +86,7 @@ def get_cmake_cmd(self): "-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_TOOLCHAIN_FILE='%s'" % self.get_toolchain_file(), "-DCPU_BASELINE=''", + "-DCMAKE_INSTALL_PREFIX=/usr/local", "-DCPU_DISPATCH=''", "-DCV_TRACE=OFF", "-DBUILD_SHARED_LIBS=OFF", @@ -136,10 +139,11 @@ def get_cmake_cmd(self): "-DBUILD_opencv_js=ON", "-DBUILD_opencv_python2=OFF", "-DBUILD_opencv_python3=OFF", - "-DBUILD_EXAMPLES=OFF", + "-DBUILD_EXAMPLES=ON", "-DBUILD_PACKAGE=OFF", - "-DBUILD_TESTS=OFF", - "-DBUILD_PERF_TESTS=OFF"] + "-DBUILD_TESTS=ON", + "-DBUILD_PERF_TESTS=ON", + "-DWITH_WEBNN=ON"] if self.options.cmake_option: cmd += self.options.cmake_option if self.options.build_doc: @@ -169,7 +173,7 @@ def get_cmake_cmd(self): return cmd def get_build_flags(self): - flags = "" + flags = "-s USE_WEBNN=1 " if self.options.build_wasm: flags += "-s WASM=1 " elif self.options.disable_wasm: diff --git a/platforms/js/opencv_js.config.py b/platforms/js/opencv_js.config.py index d63135907c1d..5240c755f3fe 100644 --- a/platforms/js/opencv_js.config.py +++ b/platforms/js/opencv_js.config.py @@ -50,7 +50,7 @@ 'BackgroundSubtractorMOG2': ['BackgroundSubtractorMOG2', 'apply'], 'BackgroundSubtractor': ['apply', 'getBackgroundImage']} -dnn = {'dnn_Net': ['setInput', 'forward'], +dnn = {'dnn_Net': ['setInput', 'forward', 'setPreferableBackend'], '': ['readNetFromCaffe', 'readNetFromTensorflow', 'readNetFromTorch', 'readNetFromDarknet', 'readNetFromONNX', 'readNet', 'blobFromImage']} From 349052f097e16d5c08363ce65b63b1e8c7844a3a Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Mon, 23 Aug 2021 16:29:22 +0800 Subject: [PATCH 153/376] Delete bib19450.aux --- build_js/doc/doxygen/bib19450.aux | 134 ------------------------------ 1 file changed, 134 deletions(-) delete mode 100644 build_js/doc/doxygen/bib19450.aux diff --git a/build_js/doc/doxygen/bib19450.aux b/build_js/doc/doxygen/bib19450.aux deleted file mode 100644 index 2e533495bdcb..000000000000 --- a/build_js/doc/doxygen/bib19450.aux +++ /dev/null @@ -1,134 +0,0 @@ -\relax -\bibstyle{doxygen} -\citation{AAM} -\citation{ABD12} -\citation{AMVOT} -\citation{ANB13} -\citation{Andreff99} -\citation{Arandjelovic:2012:TTE:2354409.2355123} -\citation{BA83} -\citation{BL07} -\citation{BT98} -\citation{Ballard1981} -\citation{Bolelli2017} -\citation{Bolelli2019} -\citation{Borgefors86} -\citation{Bouguet00} -\citation{BouguetMCT} -\citation{Bradski98} -\citation{Breiman84} -\citation{Brox2004} -\citation{CL12} -\citation{Canny86} -\citation{ChambolleEtAl} -\citation{Chaumette06} -\citation{Collins14} -\citation{DM03} -\citation{DM97} -\citation{Dalal2005} -\citation{Daniilidis98} -\citation{EM11} -\citation{EP08} -\citation{Eade13} -\citation{Eade17} -\citation{FHT98} -\citation{FL02} -\citation{Farneback2003} -\citation{Felzenszwalb04} -\citation{Fitzgibbon1999} -\citation{Fitzgibbon95} -\citation{GOTURN} -\citation{GW03} -\citation{Gallego2014ACF} -\citation{Grana2010} -\citation{Guil1999} -\citation{HH08} -\citation{HTF01} -\citation{Hartley99} -\citation{HartleyZ00} -\citation{Horaud95} -\citation{Hu62} -\citation{Ke17} -\citation{Kirkpatrick83} -\citation{KleeLaskowski85} -\citation{Kroeger2016} -\citation{LCS11} -\citation{Li2010SimultaneousRA} -\citation{Liao2007} -\citation{LibSVM} -\citation{Lienhart02} -\citation{Louhichi07} -\citation{Lowe04} -\citation{MA13} -\citation{MIL} -\citation{MK07} -\citation{MM06} -\citation{Ma:2003:IVI} -\citation{Madsen04} -\citation{Malis} -\citation{Marchand16} -\citation{Matas00} -\citation{Meyer92} -\citation{Mortensen95intelligentscissors} -\citation{Muja2009} -\citation{Nister03} -\citation{ORourke86} -\citation{PM03} -\citation{Park94} -\citation{Puzicha1997} -\citation{RB99} -\citation{RD05} -\citation{RPROP93} -\citation{RRKB11} -\citation{RS10} -\citation{Rafael12} -\citation{Rosten06} -\citation{Rubner2000} -\citation{RubnerSept98} -\citation{Shah2013SolvingTR} -\citation{Shi94} -\citation{Sklansky82} -\citation{Slabaugh} -\citation{Sol2018AML} -\citation{SteweniusCFS} -\citation{Suzuki85} -\citation{Taubin1991} -\citation{TehChin89} -\citation{Telea04} -\citation{Terzakis20} -\citation{Tsai89} -\citation{UES01} -\citation{V03} -\citation{VandLec} -\citation{Viola01} -\citation{Viola04} -\citation{WJ10} -\citation{Welch95} -\citation{Wu2009} -\citation{YM11} -\citation{Yuen90} -\citation{Zhang2000} -\citation{Zivkovic2004} -\citation{Zivkovic2006} -\citation{bigun2006vision} -\citation{blanco2010tutorial} -\citation{bottou2010large} -\citation{duda2018} -\citation{forssen2007maximally} -\citation{forstner1987fast} -\citation{gao2003complete} -\citation{gonzalez} -\citation{gruzman} -\citation{hesch2011direct} -\citation{jahne2000computer} -\citation{lepetit2009epnp} -\citation{liao2020real} -\citation{mair2010_agast} -\citation{nister2008linear} -\citation{penate2013exhaustive} -\citation{strobl2011iccv} -\citation{umeyama1991least} -\citation{vacavant2013benchmark} -\citation{van1995estimators} -\citation{yang1996structure} -\bibdata{bibTmpDir/bibTmpFile_1,bibTmpDir/bibTmpFile_2,bibTmpDir/bibTmpFile_3} From 3c8266362931f4900a77a9fba85dfd060e961c8a Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 17 Jun 2021 20:32:20 +0800 Subject: [PATCH 154/376] Add WebNN backend for OpenCV DNN Module Update dnn.cpp Update dnn.cpp Update dnn.cpp Update dnn.cpp Add WebNN head files into OpenCV 3rd partiy files Create webnn.hpp update cmake Complete README and add OpenCVDetectWebNN.cmake file add webnn.cpp Modify webnn.cpp Can successfully compile the codes for creating a MLContext Update webnn.cpp Update README.md Update README.md Update README.md Update README.md Update cmake files and update README.md Update OpenCVDetectWebNN.cmake and README.md Update OpenCVDetectWebNN.cmake Fix OpenCVDetectWebNN.cmake and update README.md Add source webnn_cpp.cpp and libary libwebnn_proc.so Update dnn.cpp Update dnn.cpp Update dnn.cpp Update dnn.cpp update dnn.cpp update op_webnn update op_webnn Update op_webnn.hpp update op_webnn.cpp & hpp Update op_webnn.hpp Update op_webnn update the skeleton Update op_webnn.cpp Update op_webnn Update op_webnn.cpp Update op_webnn.cpp Update op_webnn.hpp update op_webnn update op_webnn Solved the problems of released variables. Fixed the bugs in op_webnn.cpp Implement op_webnn Implement Relu by WebNN API Update dnn.cpp for better test Update elementwise_layers.cpp Implement ReLU6 Update elementwise_layers.cpp Implement SoftMax using WebNN API Implement Reshape by WebNN API Implement PermuteLayer by WebNN API Implement PoolingLayer using WebNN API Update pooling_layer.cpp Update pooling_layer.cpp Update pooling_layer.cpp Update pooling_layer.cpp Update pooling_layer.cpp Update pooling_layer.cpp Implement poolingLayer by WebNN API and add more detailed logs Update dnn.cpp Update dnn.cpp Remove redundant codes and add more logs for poolingLayer Add more logs in the pooling layer implementation Fix the indent issue and resolve the compiling issue Fix the build problems Fix the build issue FIx the build issue Update dnn.cpp Update dnn.cpp --- cmake/OpenCVDetectWebNN.cmake | 1 - modules/dnn/src/dnn.cpp | 1 - modules/dnn/src/layers/pooling_layer.cpp | 1 - modules/dnn/src/op_webnn.cpp | 26 ++++++++++++++++++++++-- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/cmake/OpenCVDetectWebNN.cmake b/cmake/OpenCVDetectWebNN.cmake index 8e83558674f4..8a405aa6731b 100644 --- a/cmake/OpenCVDetectWebNN.cmake +++ b/cmake/OpenCVDetectWebNN.cmake @@ -25,7 +25,6 @@ else() ) endif() - if(NOT ${VALID_WEBNN}) message(WARNING "Can't use WebNN-native") return() diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 7c4950eb9255..6e04beb0c579 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -2463,7 +2463,6 @@ struct Net::Impl : public detail::NetImplBase { // For test use. when not using WebNN, the test case will fail // with the following code. - CV_LOG_WARNING(NULL, "Layer " + ld.type + " name " + ld.name + " is unsupported by WebNN backend."); addWebnnOutputs(ld); diff --git a/modules/dnn/src/layers/pooling_layer.cpp b/modules/dnn/src/layers/pooling_layer.cpp index 0b9b94fa5772..7336aba0abaf 100644 --- a/modules/dnn/src/layers/pooling_layer.cpp +++ b/modules/dnn/src/layers/pooling_layer.cpp @@ -675,7 +675,6 @@ class PoolingLayerImpl CV_FINAL : public PoolingLayer } else if (padMode == "SAME") { options.autoPad = ml::AutoPad::SameUpper; } - // std::cout << "padMode: " << padMode << std::endl; options.windowDimensions = kernelSize; options.strides = Strides; options.padding = Padding; diff --git a/modules/dnn/src/op_webnn.cpp b/modules/dnn/src/op_webnn.cpp index 3005bc255e5c..8c6e1119794d 100644 --- a/modules/dnn/src/op_webnn.cpp +++ b/modules/dnn/src/op_webnn.cpp @@ -20,6 +20,7 @@ namespace cv { namespace dnn { #ifdef HAVE_WEBNN +<<<<<<< HEAD namespace webnn { ml::Operand BuildConstant(const ml::GraphBuilder& builder, const std::vector& dimensions, @@ -39,6 +40,19 @@ ml::Operand BuildConstant(const ml::GraphBuilder& builder, static std::string kDefaultInpLayerName = "opencv_webnn_empty_inp_layer_name"; +======= +static std::string kDefaultInpLayerName = "opencv_webnn_empty_inp_layer_name"; + +template +static inline std::vector getShape(const Mat& mat) +{ + std::vector result(mat.dims); + for (int i = 0; i < mat.dims; i++) + result[i] = (T)mat.size[i]; + return result; +} + +>>>>>>> Add WebNN backend for OpenCV DNN Module static std::vector > webnnWrappers(const std::vector >& ptrs) { @@ -58,6 +72,7 @@ WebnnNet::WebnnNet() hasNetOwner = false; device_name = "CPU"; +<<<<<<< HEAD #ifdef __EMSCRIPTEN__ context = ml::Context(emscripten_webnn_create_context()); #else @@ -65,6 +80,11 @@ WebnnNet::WebnnNet() webnnProcSetProcs(&backendProcs); context = ml::Context(webnn_native::CreateContext()); #endif +======= + WebnnProcTable backendProcs = webnn_native::GetProcs(); + webnnProcSetProcs(&backendProcs); + context = ml::Context(webnn_native::CreateContext()); +>>>>>>> Add WebNN backend for OpenCV DNN Module builder = ::ml::CreateGraphBuilder(context); namedOperands = ::ml::CreateNamedOperands(); } @@ -104,7 +124,11 @@ std::vector WebnnNet::setInputs(const std::vector& inputs, for (size_t i = 0; i < inputs.size(); i++) { auto& m = inputs[i]; +<<<<<<< HEAD std::vector dimensions = webnn::getShape(m); +======= + std::vector dimensions = getShape(m); +>>>>>>> Add WebNN backend for OpenCV DNN Module ml::OperandDescriptor descriptor; descriptor.dimensions = dimensions.data(); descriptor.dimensionsCount = dimensions.size(); @@ -172,9 +196,7 @@ void WebnnNet::forward(const std::vector >& outBlobsWrappers const std::string& name = outs[i]->name; ml::ArrayBufferView& output = outputs[i]; output.buffer = outs[i]->host->data; - // std::cout<<"host data size: "<host->total()*outs[i]->host->elemSize()<size; - // std::cout<<"outs[i]->size: "<< outs[i]->size << std::endl; named_outputs.Set(name.c_str(), &output); } ml::ComputeGraphStatus status = graph.Compute(named_inputs, named_outputs); From aeefed1e1c4637826cc112fce1028dcb77f5d917 Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Mon, 2 Aug 2021 14:52:02 +0800 Subject: [PATCH 155/376] Fix the build issue --- modules/dnn/src/dnn.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 6e04beb0c579..f06abcdf3ec4 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -2463,6 +2463,10 @@ struct Net::Impl : public detail::NetImplBase { // For test use. when not using WebNN, the test case will fail // with the following code. +<<<<<<< HEAD +======= + +>>>>>>> Fix the build issue CV_LOG_WARNING(NULL, "Layer " + ld.type + " name " + ld.name + " is unsupported by WebNN backend."); addWebnnOutputs(ld); From 88c30be533c83c9ab0696449d6d7fa234b00b558 Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Wed, 4 Aug 2021 15:05:27 +0800 Subject: [PATCH 156/376] Implement BatchNorm Layer by WebNN API --- modules/dnn/src/layers/batch_norm_layer.cpp | 28 +++++++++++++++++++++ modules/dnn/src/op_webnn.cpp | 3 +++ modules/dnn/src/op_webnn.hpp | 10 ++++++++ 3 files changed, 41 insertions(+) diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index c6a016f7c281..b02aca5a4da8 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -424,19 +424,47 @@ class BatchNormLayerImpl CV_FINAL : public BatchNormLayer } #ifdef HAVE_WEBNN +<<<<<<< HEAD +======= + ml::Operand BuildConstant(const ml::GraphBuilder& builder, + const std::vector& dimensions, + const void* value, + size_t size, + ml::OperandType type) { + ml::OperandDescriptor desc; + desc.type = type; + desc.dimensions = dimensions.data(); + desc.dimensionsCount = (uint32_t)dimensions.size(); + ml::ArrayBufferView resource; + resource.buffer = const_cast(value); + resource.byteLength = size; + return builder.Constant(&desc, &resource); + } + +>>>>>>> Implement BatchNorm Layer by WebNN API virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { Ptr node = nodes[0].dynamicCast(); auto& webnnInpOperand = node->operand; auto& webnnGraphBuilder = node->net->builder; +<<<<<<< HEAD std::vector weights_shape = webnn::getShape(weights_); ml::Operand weights = webnn::BuildConstant(webnnGraphBuilder, weights_shape, weights_.data, weights_.total()*weights_.elemSize(), ml::OperandType::Float32); +======= + std::vector weights_shape = getShape(weights_); + ml::Operand weights = BuildConstant(webnnGraphBuilder, weights_shape, weights_.data, weights_.total()*weights_.elemSize(), ml::OperandType::Float32); +>>>>>>> Implement BatchNorm Layer by WebNN API std::vector shape(dims, 1); shape[1] = weights_shape[1]; ml::Operand weights_reshaped = webnnGraphBuilder.Reshape(weights, shape.data(), shape.size()); ml::Operand mul_res = webnnGraphBuilder.Mul(webnnInpOperand, weights_reshaped); +<<<<<<< HEAD std::vector bias_shape = webnn::getShape(bias_); ml::Operand bias = webnn::BuildConstant(webnnGraphBuilder, bias_shape, bias_.data, bias_.total()*bias_.elemSize(), ml::OperandType::Float32); +======= + std::vector bias_shape = getShape(bias_); + ml::Operand bias = BuildConstant(webnnGraphBuilder, bias_shape, bias_.data, bias_.total()*bias_.elemSize(), ml::OperandType::Float32); +>>>>>>> Implement BatchNorm Layer by WebNN API shape[1] = bias_shape[1]; ml::Operand bias_reshaped = webnnGraphBuilder.Reshape(bias, shape.data(), shape.size()); ml::Operand add_res = webnnGraphBuilder.Add(mul_res, bias_reshaped); diff --git a/modules/dnn/src/op_webnn.cpp b/modules/dnn/src/op_webnn.cpp index 8c6e1119794d..91ede1b2e4a8 100644 --- a/modules/dnn/src/op_webnn.cpp +++ b/modules/dnn/src/op_webnn.cpp @@ -43,6 +43,7 @@ static std::string kDefaultInpLayerName = "opencv_webnn_empty_inp_layer_name"; ======= static std::string kDefaultInpLayerName = "opencv_webnn_empty_inp_layer_name"; +<<<<<<< HEAD template static inline std::vector getShape(const Mat& mat) { @@ -53,6 +54,8 @@ static inline std::vector getShape(const Mat& mat) } >>>>>>> Add WebNN backend for OpenCV DNN Module +======= +>>>>>>> Implement BatchNorm Layer by WebNN API static std::vector > webnnWrappers(const std::vector >& ptrs) { diff --git a/modules/dnn/src/op_webnn.hpp b/modules/dnn/src/op_webnn.hpp index f2284c6d63cd..81d29aec3ec2 100644 --- a/modules/dnn/src/op_webnn.hpp +++ b/modules/dnn/src/op_webnn.hpp @@ -54,6 +54,7 @@ inline std::vector getShape(const Mat& mat) return result; } +<<<<<<< HEAD ml::Operand BuildConstant(const ml::GraphBuilder& builder, const std::vector& dimensions, const void* value, @@ -94,6 +95,15 @@ struct Pool2dOptions { private: ml::Pool2dOptions mOptions; }; +======= +template +inline std::vector getShape(const Mat& mat) +{ + std::vector result(mat.dims); + for (int i = 0; i < mat.dims; i++) + result[i] = (T)mat.size[i]; + return result; +>>>>>>> Implement BatchNorm Layer by WebNN API } class WebnnNet From f8d4bb42c26cf3215c01ff7f51e5ed12830884be Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Wed, 4 Aug 2021 17:24:18 +0800 Subject: [PATCH 157/376] Update convolution_layer.cpp This is a temporary file for Conv2d layer implementation --- modules/dnn/src/op_webnn.hpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/modules/dnn/src/op_webnn.hpp b/modules/dnn/src/op_webnn.hpp index 81d29aec3ec2..f2284c6d63cd 100644 --- a/modules/dnn/src/op_webnn.hpp +++ b/modules/dnn/src/op_webnn.hpp @@ -54,7 +54,6 @@ inline std::vector getShape(const Mat& mat) return result; } -<<<<<<< HEAD ml::Operand BuildConstant(const ml::GraphBuilder& builder, const std::vector& dimensions, const void* value, @@ -95,15 +94,6 @@ struct Pool2dOptions { private: ml::Pool2dOptions mOptions; }; -======= -template -inline std::vector getShape(const Mat& mat) -{ - std::vector result(mat.dims); - for (int i = 0; i < mat.dims; i++) - result[i] = (T)mat.size[i]; - return result; ->>>>>>> Implement BatchNorm Layer by WebNN API } class WebnnNet From ebad50b7cbde8d26b10878339ea314af4efed754 Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 5 Aug 2021 14:33:19 +0800 Subject: [PATCH 158/376] Integrate some general functions into op_webnn.cpp&hpp --- modules/dnn/src/layers/batch_norm_layer.cpp | 28 --------------------- modules/dnn/src/op_webnn.cpp | 28 +-------------------- 2 files changed, 1 insertion(+), 55 deletions(-) diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index b02aca5a4da8..c6a016f7c281 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -424,47 +424,19 @@ class BatchNormLayerImpl CV_FINAL : public BatchNormLayer } #ifdef HAVE_WEBNN -<<<<<<< HEAD -======= - ml::Operand BuildConstant(const ml::GraphBuilder& builder, - const std::vector& dimensions, - const void* value, - size_t size, - ml::OperandType type) { - ml::OperandDescriptor desc; - desc.type = type; - desc.dimensions = dimensions.data(); - desc.dimensionsCount = (uint32_t)dimensions.size(); - ml::ArrayBufferView resource; - resource.buffer = const_cast(value); - resource.byteLength = size; - return builder.Constant(&desc, &resource); - } - ->>>>>>> Implement BatchNorm Layer by WebNN API virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { Ptr node = nodes[0].dynamicCast(); auto& webnnInpOperand = node->operand; auto& webnnGraphBuilder = node->net->builder; -<<<<<<< HEAD std::vector weights_shape = webnn::getShape(weights_); ml::Operand weights = webnn::BuildConstant(webnnGraphBuilder, weights_shape, weights_.data, weights_.total()*weights_.elemSize(), ml::OperandType::Float32); -======= - std::vector weights_shape = getShape(weights_); - ml::Operand weights = BuildConstant(webnnGraphBuilder, weights_shape, weights_.data, weights_.total()*weights_.elemSize(), ml::OperandType::Float32); ->>>>>>> Implement BatchNorm Layer by WebNN API std::vector shape(dims, 1); shape[1] = weights_shape[1]; ml::Operand weights_reshaped = webnnGraphBuilder.Reshape(weights, shape.data(), shape.size()); ml::Operand mul_res = webnnGraphBuilder.Mul(webnnInpOperand, weights_reshaped); -<<<<<<< HEAD std::vector bias_shape = webnn::getShape(bias_); ml::Operand bias = webnn::BuildConstant(webnnGraphBuilder, bias_shape, bias_.data, bias_.total()*bias_.elemSize(), ml::OperandType::Float32); -======= - std::vector bias_shape = getShape(bias_); - ml::Operand bias = BuildConstant(webnnGraphBuilder, bias_shape, bias_.data, bias_.total()*bias_.elemSize(), ml::OperandType::Float32); ->>>>>>> Implement BatchNorm Layer by WebNN API shape[1] = bias_shape[1]; ml::Operand bias_reshaped = webnnGraphBuilder.Reshape(bias, shape.data(), shape.size()); ml::Operand add_res = webnnGraphBuilder.Add(mul_res, bias_reshaped); diff --git a/modules/dnn/src/op_webnn.cpp b/modules/dnn/src/op_webnn.cpp index 91ede1b2e4a8..de4180e4a111 100644 --- a/modules/dnn/src/op_webnn.cpp +++ b/modules/dnn/src/op_webnn.cpp @@ -20,7 +20,6 @@ namespace cv { namespace dnn { #ifdef HAVE_WEBNN -<<<<<<< HEAD namespace webnn { ml::Operand BuildConstant(const ml::GraphBuilder& builder, const std::vector& dimensions, @@ -40,22 +39,6 @@ ml::Operand BuildConstant(const ml::GraphBuilder& builder, static std::string kDefaultInpLayerName = "opencv_webnn_empty_inp_layer_name"; -======= -static std::string kDefaultInpLayerName = "opencv_webnn_empty_inp_layer_name"; - -<<<<<<< HEAD -template -static inline std::vector getShape(const Mat& mat) -{ - std::vector result(mat.dims); - for (int i = 0; i < mat.dims; i++) - result[i] = (T)mat.size[i]; - return result; -} - ->>>>>>> Add WebNN backend for OpenCV DNN Module -======= ->>>>>>> Implement BatchNorm Layer by WebNN API static std::vector > webnnWrappers(const std::vector >& ptrs) { @@ -75,7 +58,6 @@ WebnnNet::WebnnNet() hasNetOwner = false; device_name = "CPU"; -<<<<<<< HEAD #ifdef __EMSCRIPTEN__ context = ml::Context(emscripten_webnn_create_context()); #else @@ -83,11 +65,6 @@ WebnnNet::WebnnNet() webnnProcSetProcs(&backendProcs); context = ml::Context(webnn_native::CreateContext()); #endif -======= - WebnnProcTable backendProcs = webnn_native::GetProcs(); - webnnProcSetProcs(&backendProcs); - context = ml::Context(webnn_native::CreateContext()); ->>>>>>> Add WebNN backend for OpenCV DNN Module builder = ::ml::CreateGraphBuilder(context); namedOperands = ::ml::CreateNamedOperands(); } @@ -127,11 +104,8 @@ std::vector WebnnNet::setInputs(const std::vector& inputs, for (size_t i = 0; i < inputs.size(); i++) { auto& m = inputs[i]; -<<<<<<< HEAD + std::vector dimensions = webnn::getShape(m); -======= - std::vector dimensions = getShape(m); ->>>>>>> Add WebNN backend for OpenCV DNN Module ml::OperandDescriptor descriptor; descriptor.dimensions = dimensions.data(); descriptor.dimensionsCount = dimensions.size(); From c4c9292d1d993ffdd137bc62ebc91c62863dda9a Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 5 Aug 2021 15:33:36 +0800 Subject: [PATCH 159/376] Update const_layer.cpp --- modules/dnn/src/layers/const_layer.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/dnn/src/layers/const_layer.cpp b/modules/dnn/src/layers/const_layer.cpp index 1f307b8fa6aa..c8a258600615 100644 --- a/modules/dnn/src/layers/const_layer.cpp +++ b/modules/dnn/src/layers/const_layer.cpp @@ -102,10 +102,9 @@ class ConstLayerImpl CV_FINAL : public ConstLayer #ifdef HAVE_WEBNN virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { - ml::Operand operand = nullptr; Ptr node = nodes[0].dynamicCast(); auto& webnnGraphBuilder = node->net->builder; - operand = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); + ml::Operand operand = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); return Ptr(new WebnnBackendNode(operand)); } #endif From 20f323eef134b2ed9d050a7b338d0d36b03310fd Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 5 Aug 2021 19:29:03 +0800 Subject: [PATCH 160/376] Update convolution_layer.cpp Still have some bugs that should be fixed. --- modules/dnn/src/layers/convolution_layer.cpp | 86 ++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index 0b550edbab38..eb593fe4a90d 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -917,6 +917,7 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl if (nodes.size() > 1) CV_Assert(webnnWeights); // const int inpCn = weightsMat.total()/(kernel_size[0]*kernel_size[1]*numOutput); +<<<<<<< HEAD // const int group = blobs.size() - hasBias(); const int inpGroupCn = blobs[0].size[1]; // // const int group = inpCn / inpGroupCn; @@ -924,6 +925,14 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl // std::cout<<"Group: "<>>>>>> Update convolution_layer.cpp std::vector kernel_shape; if (group != 1) { @@ -935,19 +944,33 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl if (nodes.size() == 1) { +<<<<<<< HEAD webnnWeights = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); +======= + std::cout<<"fusedWeights: "<>>>>>> Update convolution_layer.cpp if (fusedWeights) { if (weightsMat.isContinuous()) { +<<<<<<< HEAD webnnWeights = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(weightsMat), weightsMat.data, weightsMat.total()*weightsMat.elemSize(), ml::OperandType::Float32); +======= + webnnWeights = webnn::BuildConstant(webnnGraphBuilder, kernel_shape, weightsMat.data, weightsMat.total()*weightsMat.elemSize(), ml::OperandType::Float32); +>>>>>>> Update convolution_layer.cpp } else { Mat newWeights; Mat cvWeights = weightsMat.colRange(0, blobs[0].total() / numOutput); cvWeights.copyTo(newWeights); +<<<<<<< HEAD webnnWeights = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(newWeights), newWeights.data, newWeights.total()*newWeights.elemSize(), ml::OperandType::Float32); +======= + webnnWeights = webnn::BuildConstant(webnnGraphBuilder, kernel_shape, newWeights.data, newWeights.total()*newWeights.elemSize(), ml::OperandType::Float32); +>>>>>>> Update convolution_layer.cpp } } } @@ -955,6 +978,7 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl { webnnWeights = webnnGraphBuilder.Reshape(webnnWeights, kernel_shape.data(), kernel_shape.size()); } +<<<<<<< HEAD ml::AutoPad pad_type = ml::AutoPad::Explicit; if (!padMode.empty()) @@ -1011,6 +1035,68 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl operand = webnnGraphBuilder.Add(operand, webnnBias); } return Ptr(new WebnnBackendNode(operand)); +======= + + ml::AutoPad pad_type = ml::AutoPad::Explicit; + if (!padMode.empty()) + pad_type = padMode == "VALID" ? ml::AutoPad::Explicit : ml::AutoPad::SameUpper; + + ml::Conv2dOptions options; + options.autoPad = pad_type; + // std::vector Strides(strides.begin(), strides.end()); + if (!strides.empty()) + { + options.stridesCount = strides.size(); + options.strides = reinterpret_cast(strides.data()); + } + std::vector Padding; + if (padMode.empty()) + { + Padding = {static_cast(pads_begin[0]), + static_cast(pads_end[0]), + static_cast(pads_begin[1]), + static_cast(pads_end[1])}; + } + else if (padMode == "VALID") + { + Padding = {0, 0, 0, 0}; + } + if (!Padding.empty()) + { + options.paddingCount = Padding.size(); + options.padding = Padding.data(); + } + std::cout<<"Padding: "< Dilations(dilations.begin(), dilations.end()); + if (!dilations.empty()) + { + options.dilationsCount = dilations.size(); + options.dilations = reinterpret_cast(dilations.data()); + } + const ml::Operand weights = webnnWeights; + const ml::Operand operand = webnnGraphBuilder.Conv2d(webnnInpOperand, weights, &options); + // const ml::Operand operand = webnnGraphBuilder.Conv2d(webnnInpOperand, *const_cast(&webnnWeights), &options); + // const ml::Operand operand = webnnGraphBuilder.Conv2d(webnnInpOperand, webnnWeights, &options); + + ml::Operand result = operand; + if (hasBias() || fusedBias || nodes.size() == 3) + { + ml::Operand webnnBias; + if (nodes.size() == 3) + { + webnnBias = webnnGraphBuilder.Reshape(nodes[2].dynamicCast()->operand, kernel_shape.data(), kernel_shape.size()); + } + else + { + webnnBias = webnn::BuildConstant(webnnGraphBuilder, kernel_shape, biasvec.data(), biasvec.size()*sizeof(float), ml::OperandType::Float32); + } + const ml::Operand bias = webnnBias; + // result = webnnGraphBuilder.Add(operand, *const_cast(&webnnBias)); + result = webnnGraphBuilder.Add(operand, bias); + // result = webnnGraphBuilder.Add(operand, webnnBias); + } + return Ptr(new WebnnBackendNode(result)); +>>>>>>> Update convolution_layer.cpp } #endif // HAVE_WEBNN From e6d792da1c3c948e2a25fbce7b6b74ac3bf1f93b Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 5 Aug 2021 23:14:52 +0800 Subject: [PATCH 161/376] Update conv2d layer and fc layer still have some problems to be fixed. --- modules/dnn/src/layers/convolution_layer.cpp | 86 -------------------- 1 file changed, 86 deletions(-) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index eb593fe4a90d..0b550edbab38 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -917,7 +917,6 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl if (nodes.size() > 1) CV_Assert(webnnWeights); // const int inpCn = weightsMat.total()/(kernel_size[0]*kernel_size[1]*numOutput); -<<<<<<< HEAD // const int group = blobs.size() - hasBias(); const int inpGroupCn = blobs[0].size[1]; // // const int group = inpCn / inpGroupCn; @@ -925,14 +924,6 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl // std::cout<<"Group: "<>>>>>> Update convolution_layer.cpp std::vector kernel_shape; if (group != 1) { @@ -944,33 +935,19 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl if (nodes.size() == 1) { -<<<<<<< HEAD webnnWeights = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); -======= - std::cout<<"fusedWeights: "<>>>>>> Update convolution_layer.cpp if (fusedWeights) { if (weightsMat.isContinuous()) { -<<<<<<< HEAD webnnWeights = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(weightsMat), weightsMat.data, weightsMat.total()*weightsMat.elemSize(), ml::OperandType::Float32); -======= - webnnWeights = webnn::BuildConstant(webnnGraphBuilder, kernel_shape, weightsMat.data, weightsMat.total()*weightsMat.elemSize(), ml::OperandType::Float32); ->>>>>>> Update convolution_layer.cpp } else { Mat newWeights; Mat cvWeights = weightsMat.colRange(0, blobs[0].total() / numOutput); cvWeights.copyTo(newWeights); -<<<<<<< HEAD webnnWeights = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(newWeights), newWeights.data, newWeights.total()*newWeights.elemSize(), ml::OperandType::Float32); -======= - webnnWeights = webnn::BuildConstant(webnnGraphBuilder, kernel_shape, newWeights.data, newWeights.total()*newWeights.elemSize(), ml::OperandType::Float32); ->>>>>>> Update convolution_layer.cpp } } } @@ -978,7 +955,6 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl { webnnWeights = webnnGraphBuilder.Reshape(webnnWeights, kernel_shape.data(), kernel_shape.size()); } -<<<<<<< HEAD ml::AutoPad pad_type = ml::AutoPad::Explicit; if (!padMode.empty()) @@ -1035,68 +1011,6 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl operand = webnnGraphBuilder.Add(operand, webnnBias); } return Ptr(new WebnnBackendNode(operand)); -======= - - ml::AutoPad pad_type = ml::AutoPad::Explicit; - if (!padMode.empty()) - pad_type = padMode == "VALID" ? ml::AutoPad::Explicit : ml::AutoPad::SameUpper; - - ml::Conv2dOptions options; - options.autoPad = pad_type; - // std::vector Strides(strides.begin(), strides.end()); - if (!strides.empty()) - { - options.stridesCount = strides.size(); - options.strides = reinterpret_cast(strides.data()); - } - std::vector Padding; - if (padMode.empty()) - { - Padding = {static_cast(pads_begin[0]), - static_cast(pads_end[0]), - static_cast(pads_begin[1]), - static_cast(pads_end[1])}; - } - else if (padMode == "VALID") - { - Padding = {0, 0, 0, 0}; - } - if (!Padding.empty()) - { - options.paddingCount = Padding.size(); - options.padding = Padding.data(); - } - std::cout<<"Padding: "< Dilations(dilations.begin(), dilations.end()); - if (!dilations.empty()) - { - options.dilationsCount = dilations.size(); - options.dilations = reinterpret_cast(dilations.data()); - } - const ml::Operand weights = webnnWeights; - const ml::Operand operand = webnnGraphBuilder.Conv2d(webnnInpOperand, weights, &options); - // const ml::Operand operand = webnnGraphBuilder.Conv2d(webnnInpOperand, *const_cast(&webnnWeights), &options); - // const ml::Operand operand = webnnGraphBuilder.Conv2d(webnnInpOperand, webnnWeights, &options); - - ml::Operand result = operand; - if (hasBias() || fusedBias || nodes.size() == 3) - { - ml::Operand webnnBias; - if (nodes.size() == 3) - { - webnnBias = webnnGraphBuilder.Reshape(nodes[2].dynamicCast()->operand, kernel_shape.data(), kernel_shape.size()); - } - else - { - webnnBias = webnn::BuildConstant(webnnGraphBuilder, kernel_shape, biasvec.data(), biasvec.size()*sizeof(float), ml::OperandType::Float32); - } - const ml::Operand bias = webnnBias; - // result = webnnGraphBuilder.Add(operand, *const_cast(&webnnBias)); - result = webnnGraphBuilder.Add(operand, bias); - // result = webnnGraphBuilder.Add(operand, webnnBias); - } - return Ptr(new WebnnBackendNode(result)); ->>>>>>> Update convolution_layer.cpp } #endif // HAVE_WEBNN From bcb0de9c1d2deb254e6470891a66a3ff1101c89b Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Fri, 6 Aug 2021 15:15:13 +0800 Subject: [PATCH 162/376] update constLayer, conv layer, fc layer There are still some bugs to be fixed. --- modules/dnn/src/layers/const_layer.cpp | 18 ++++++++++++++--- modules/dnn/src/layers/convolution_layer.cpp | 20 +++++++++---------- .../dnn/src/layers/fully_connected_layer.cpp | 1 - 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/modules/dnn/src/layers/const_layer.cpp b/modules/dnn/src/layers/const_layer.cpp index c8a258600615..d531c7c6486a 100644 --- a/modules/dnn/src/layers/const_layer.cpp +++ b/modules/dnn/src/layers/const_layer.cpp @@ -102,9 +102,21 @@ class ConstLayerImpl CV_FINAL : public ConstLayer #ifdef HAVE_WEBNN virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { - Ptr node = nodes[0].dynamicCast(); - auto& webnnGraphBuilder = node->net->builder; - ml::Operand operand = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); + ml::Operand operand = nullptr; + if (nodes.size() != 0) + { + Ptr node = nodes[0].dynamicCast(); + auto& webnnGraphBuilder = node->net->builder; + operand = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); + } + else + { + WebnnProcTable backendProcs = webnn_native::GetProcs(); + webnnProcSetProcs(&backendProcs); + ml::Context context = ml::Context(webnn_native::CreateContext()); + ml::GraphBuilder builder = ml::CreateGraphBuilder(context); + operand = webnn::BuildConstant(builder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); + } return Ptr(new WebnnBackendNode(operand)); } #endif diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index 0b550edbab38..f776dff13148 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -348,16 +348,16 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl if (backendId == DNN_BACKEND_VKCOM) return ksize == 2; #endif -#ifdef HAVE_WEBNN - if (backendId == DNN_BACKEND_WEBNN) - { - if (ksize != 2) - { - CV_LOG_WARNING(NULL, "WebNN only supports Conv2d."); - } - return ksize == 2; - } -#endif +// #ifdef HAVE_WEBNN +// if (backendId == DNN_BACKEND_WEBNN) +// { +// if (ksize != 2) +// { +// CV_LOG_WARNING(NULL, "WebNN only supports Conv2d."); +// } +// return ksize == 2; +// } +// #endif return false; } diff --git a/modules/dnn/src/layers/fully_connected_layer.cpp b/modules/dnn/src/layers/fully_connected_layer.cpp index cf5f7135c22f..809fae39407a 100644 --- a/modules/dnn/src/layers/fully_connected_layer.cpp +++ b/modules/dnn/src/layers/fully_connected_layer.cpp @@ -685,7 +685,6 @@ class FullyConnectedLayerImpl CV_FINAL : public InnerProductLayer // std::cout<<"input size: "< weight_shape = {(int32_t)blobs[0].size[0], (int32_t)blobs[0].size[1]}; - // std::cout<<"weight size: "< Date: Thu, 12 Aug 2021 17:03:48 +0800 Subject: [PATCH 163/376] Update conv2d layer, fully connected layer and const layer --- modules/dnn/src/layers/const_layer.cpp | 17 ++------ modules/dnn/src/layers/convolution_layer.cpp | 42 +++++++++++++------ .../dnn/src/layers/fully_connected_layer.cpp | 2 +- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/modules/dnn/src/layers/const_layer.cpp b/modules/dnn/src/layers/const_layer.cpp index d531c7c6486a..1f307b8fa6aa 100644 --- a/modules/dnn/src/layers/const_layer.cpp +++ b/modules/dnn/src/layers/const_layer.cpp @@ -103,20 +103,9 @@ class ConstLayerImpl CV_FINAL : public ConstLayer virtual Ptr initWebnn(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { ml::Operand operand = nullptr; - if (nodes.size() != 0) - { - Ptr node = nodes[0].dynamicCast(); - auto& webnnGraphBuilder = node->net->builder; - operand = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); - } - else - { - WebnnProcTable backendProcs = webnn_native::GetProcs(); - webnnProcSetProcs(&backendProcs); - ml::Context context = ml::Context(webnn_native::CreateContext()); - ml::GraphBuilder builder = ml::CreateGraphBuilder(context); - operand = webnn::BuildConstant(builder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); - } + Ptr node = nodes[0].dynamicCast(); + auto& webnnGraphBuilder = node->net->builder; + operand = webnn::BuildConstant(webnnGraphBuilder, webnn::getShape(blobs[0]), blobs[0].data, blobs[0].total()*blobs[0].elemSize(), ml::OperandType::Float32); return Ptr(new WebnnBackendNode(operand)); } #endif diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index f776dff13148..a0b911051acb 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -348,16 +348,16 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl if (backendId == DNN_BACKEND_VKCOM) return ksize == 2; #endif -// #ifdef HAVE_WEBNN -// if (backendId == DNN_BACKEND_WEBNN) -// { -// if (ksize != 2) -// { -// CV_LOG_WARNING(NULL, "WebNN only supports Conv2d."); -// } -// return ksize == 2; -// } -// #endif +#ifdef HAVE_WEBNN + if (backendId == DNN_BACKEND_WEBNN) + { + if (ksize != 2) + { + CV_LOG_WARNING(NULL, "WebNN only supports Conv2d."); + } + return ksize == 2; + } +#endif return false; } @@ -917,6 +917,7 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl if (nodes.size() > 1) CV_Assert(webnnWeights); // const int inpCn = weightsMat.total()/(kernel_size[0]*kernel_size[1]*numOutput); +<<<<<<< HEAD // const int group = blobs.size() - hasBias(); const int inpGroupCn = blobs[0].size[1]; // // const int group = inpCn / inpGroupCn; @@ -932,6 +933,23 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl kernel_shape.push_back(numOutput / group); kernel_shape.push_back(inpGroupCn); std::copy(kernel_size.begin(), kernel_size.end(), back_inserter(kernel_shape)); +======= + const int group = blobs.size() - hasBias(); + // const int inpGroupCn = blobs[0].size[1]; + // // const int group = inpCn / inpGroupCn; + // const int group = 1; + // // std::cout<<"Group: "< kernel_shape; + // if (group != 1) + // { + // kernel_shape.push_back(group); + // } + // kernel_shape.push_back(numOutput / group); + // kernel_shape.push_back(inpGroupCn); + // std::copy(kernel_size.begin(), kernel_size.end(), back_inserter(kernel_shape)); +>>>>>>> Update conv2d layer, fully connected layer and const layer if (nodes.size() == 1) { @@ -953,7 +971,7 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl } else { - webnnWeights = webnnGraphBuilder.Reshape(webnnWeights, kernel_shape.data(), kernel_shape.size()); + // webnnWeights = webnnGraphBuilder.Reshape(webnnWeights, kernel_shape.data(), kernel_shape.size()); } ml::AutoPad pad_type = ml::AutoPad::Explicit; @@ -1004,8 +1022,6 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl } else { - // std::cout<<"mumOutput size: "<< numOutput < input_shape(2, -1); input_shape[1] = blobs[0].size[1]; - // std::cout<<"input size: "< weight_shape = {(int32_t)blobs[0].size[0], (int32_t)blobs[0].size[1]}; + std::cout<<"weight size: "< Date: Sun, 15 Aug 2021 15:29:46 +0800 Subject: [PATCH 164/376] Update convolution_layer.cpp --- modules/dnn/src/layers/convolution_layer.cpp | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index a0b911051acb..9d3c9a07988c 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -917,7 +917,6 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl if (nodes.size() > 1) CV_Assert(webnnWeights); // const int inpCn = weightsMat.total()/(kernel_size[0]*kernel_size[1]*numOutput); -<<<<<<< HEAD // const int group = blobs.size() - hasBias(); const int inpGroupCn = blobs[0].size[1]; // // const int group = inpCn / inpGroupCn; @@ -933,23 +932,6 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl kernel_shape.push_back(numOutput / group); kernel_shape.push_back(inpGroupCn); std::copy(kernel_size.begin(), kernel_size.end(), back_inserter(kernel_shape)); -======= - const int group = blobs.size() - hasBias(); - // const int inpGroupCn = blobs[0].size[1]; - // // const int group = inpCn / inpGroupCn; - // const int group = 1; - // // std::cout<<"Group: "< kernel_shape; - // if (group != 1) - // { - // kernel_shape.push_back(group); - // } - // kernel_shape.push_back(numOutput / group); - // kernel_shape.push_back(inpGroupCn); - // std::copy(kernel_size.begin(), kernel_size.end(), back_inserter(kernel_shape)); ->>>>>>> Update conv2d layer, fully connected layer and const layer if (nodes.size() == 1) { @@ -971,7 +953,7 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl } else { - // webnnWeights = webnnGraphBuilder.Reshape(webnnWeights, kernel_shape.data(), kernel_shape.size()); + webnnWeights = webnnGraphBuilder.Reshape(webnnWeights, kernel_shape.data(), kernel_shape.size()); } ml::AutoPad pad_type = ml::AutoPad::Explicit; From 3c72a3c1ca61b05b815301febd780f6b8c15e621 Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Sun, 22 Aug 2021 18:39:06 +0800 Subject: [PATCH 165/376] Add OpenCV.js DNN module WebNN Backend (both using webnn-polyfill and electron) --- build_js/doc/doxygen/bib19450.aux | 134 ++++++++++++++++++ .../dnn/src/layers/fully_connected_layer.cpp | 2 +- modules/dnn/src/layers/pooling_layer.cpp | 1 + modules/dnn/src/op_webnn.cpp | 2 + 4 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 build_js/doc/doxygen/bib19450.aux diff --git a/build_js/doc/doxygen/bib19450.aux b/build_js/doc/doxygen/bib19450.aux new file mode 100644 index 000000000000..2e533495bdcb --- /dev/null +++ b/build_js/doc/doxygen/bib19450.aux @@ -0,0 +1,134 @@ +\relax +\bibstyle{doxygen} +\citation{AAM} +\citation{ABD12} +\citation{AMVOT} +\citation{ANB13} +\citation{Andreff99} +\citation{Arandjelovic:2012:TTE:2354409.2355123} +\citation{BA83} +\citation{BL07} +\citation{BT98} +\citation{Ballard1981} +\citation{Bolelli2017} +\citation{Bolelli2019} +\citation{Borgefors86} +\citation{Bouguet00} +\citation{BouguetMCT} +\citation{Bradski98} +\citation{Breiman84} +\citation{Brox2004} +\citation{CL12} +\citation{Canny86} +\citation{ChambolleEtAl} +\citation{Chaumette06} +\citation{Collins14} +\citation{DM03} +\citation{DM97} +\citation{Dalal2005} +\citation{Daniilidis98} +\citation{EM11} +\citation{EP08} +\citation{Eade13} +\citation{Eade17} +\citation{FHT98} +\citation{FL02} +\citation{Farneback2003} +\citation{Felzenszwalb04} +\citation{Fitzgibbon1999} +\citation{Fitzgibbon95} +\citation{GOTURN} +\citation{GW03} +\citation{Gallego2014ACF} +\citation{Grana2010} +\citation{Guil1999} +\citation{HH08} +\citation{HTF01} +\citation{Hartley99} +\citation{HartleyZ00} +\citation{Horaud95} +\citation{Hu62} +\citation{Ke17} +\citation{Kirkpatrick83} +\citation{KleeLaskowski85} +\citation{Kroeger2016} +\citation{LCS11} +\citation{Li2010SimultaneousRA} +\citation{Liao2007} +\citation{LibSVM} +\citation{Lienhart02} +\citation{Louhichi07} +\citation{Lowe04} +\citation{MA13} +\citation{MIL} +\citation{MK07} +\citation{MM06} +\citation{Ma:2003:IVI} +\citation{Madsen04} +\citation{Malis} +\citation{Marchand16} +\citation{Matas00} +\citation{Meyer92} +\citation{Mortensen95intelligentscissors} +\citation{Muja2009} +\citation{Nister03} +\citation{ORourke86} +\citation{PM03} +\citation{Park94} +\citation{Puzicha1997} +\citation{RB99} +\citation{RD05} +\citation{RPROP93} +\citation{RRKB11} +\citation{RS10} +\citation{Rafael12} +\citation{Rosten06} +\citation{Rubner2000} +\citation{RubnerSept98} +\citation{Shah2013SolvingTR} +\citation{Shi94} +\citation{Sklansky82} +\citation{Slabaugh} +\citation{Sol2018AML} +\citation{SteweniusCFS} +\citation{Suzuki85} +\citation{Taubin1991} +\citation{TehChin89} +\citation{Telea04} +\citation{Terzakis20} +\citation{Tsai89} +\citation{UES01} +\citation{V03} +\citation{VandLec} +\citation{Viola01} +\citation{Viola04} +\citation{WJ10} +\citation{Welch95} +\citation{Wu2009} +\citation{YM11} +\citation{Yuen90} +\citation{Zhang2000} +\citation{Zivkovic2004} +\citation{Zivkovic2006} +\citation{bigun2006vision} +\citation{blanco2010tutorial} +\citation{bottou2010large} +\citation{duda2018} +\citation{forssen2007maximally} +\citation{forstner1987fast} +\citation{gao2003complete} +\citation{gonzalez} +\citation{gruzman} +\citation{hesch2011direct} +\citation{jahne2000computer} +\citation{lepetit2009epnp} +\citation{liao2020real} +\citation{mair2010_agast} +\citation{nister2008linear} +\citation{penate2013exhaustive} +\citation{strobl2011iccv} +\citation{umeyama1991least} +\citation{vacavant2013benchmark} +\citation{van1995estimators} +\citation{yang1996structure} +\bibdata{bibTmpDir/bibTmpFile_1,bibTmpDir/bibTmpFile_2,bibTmpDir/bibTmpFile_3} diff --git a/modules/dnn/src/layers/fully_connected_layer.cpp b/modules/dnn/src/layers/fully_connected_layer.cpp index 87483e060cb8..1e8c9f5489df 100644 --- a/modules/dnn/src/layers/fully_connected_layer.cpp +++ b/modules/dnn/src/layers/fully_connected_layer.cpp @@ -684,7 +684,7 @@ class FullyConnectedLayerImpl CV_FINAL : public InnerProductLayer input_shape[1] = blobs[0].size[1]; ml::Operand webnnInpOperand_reshaped = webnnGraphBuilder.Reshape(webnnInpOperand, input_shape.data(), input_shape.size()); std::vector weight_shape = {(int32_t)blobs[0].size[0], (int32_t)blobs[0].size[1]}; - std::cout<<"weight size: "< >& outBlobsWrappers const std::string& name = outs[i]->name; ml::ArrayBufferView& output = outputs[i]; output.buffer = outs[i]->host->data; + // std::cout<<"host data size: "<host->total()*outs[i]->host->elemSize()<size; + // std::cout<<"outs[i]->size: "<< outs[i]->size << std::endl; named_outputs.Set(name.c_str(), &output); } ml::ComputeGraphStatus status = graph.Compute(named_inputs, named_outputs); From 473fb06808d0c18f8384391ad6e2b161bbf69493 Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 26 Aug 2021 16:26:12 +0800 Subject: [PATCH 166/376] Update dnn.cpp --- modules/dnn/src/dnn.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index f06abcdf3ec4..dea67e4a589c 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -4979,9 +4979,7 @@ void Net::setInput(InputArray blob, const String& name, double scalefactor, cons MatShape prevShape = shape(impl->netInputLayer->inputsData[pin.oid]); bool oldShape = prevShape == blobShape; - // blob_.copyTo(impl->netInputLayer->inputsData[pin.oid]); - impl->netInputLayer->inputsData.emplace(impl->netInputLayer->inputsData.begin()+pin.oid, blob_); - impl->netInputLayer->inputsData.erase(impl->netInputLayer->inputsData.begin()+pin.oid+1); + blob_.copyTo(impl->nPetInputLayer->inputsData[pin.oid]); if (!oldShape) { ld.outputBlobs[pin.oid] = impl->netInputLayer->inputsData[pin.oid]; if (impl->hasDynamicShapes) From 580da22d1e129a7b37d4212d235904a76dd6264f Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 26 Aug 2021 17:48:18 +0800 Subject: [PATCH 167/376] Fix Error in dnn.cpp --- modules/dnn/src/dnn.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index dea67e4a589c..0d0835b30f1d 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -2463,10 +2463,6 @@ struct Net::Impl : public detail::NetImplBase { // For test use. when not using WebNN, the test case will fail // with the following code. -<<<<<<< HEAD -======= - ->>>>>>> Fix the build issue CV_LOG_WARNING(NULL, "Layer " + ld.type + " name " + ld.name + " is unsupported by WebNN backend."); addWebnnOutputs(ld); @@ -4979,7 +4975,7 @@ void Net::setInput(InputArray blob, const String& name, double scalefactor, cons MatShape prevShape = shape(impl->netInputLayer->inputsData[pin.oid]); bool oldShape = prevShape == blobShape; - blob_.copyTo(impl->nPetInputLayer->inputsData[pin.oid]); + blob_.copyTo(impl->netInputLayer->inputsData[pin.oid]); if (!oldShape) { ld.outputBlobs[pin.oid] = impl->netInputLayer->inputsData[pin.oid]; if (impl->hasDynamicShapes) From f485a9931f50845e3df0f867b7cb2989be5c5c70 Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 26 Aug 2021 20:43:53 +0800 Subject: [PATCH 168/376] Resolve duplication in conditions in convolution_layer.cpp --- modules/dnn/src/layers/convolution_layer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index 9d3c9a07988c..adcc56fc76d1 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -354,8 +354,9 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl if (ksize != 2) { CV_LOG_WARNING(NULL, "WebNN only supports Conv2d."); + return false; } - return ksize == 2; + return true; } #endif return false; From 38d0063c367b2697b57ab0809530889cc2e20fd7 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Thu, 26 Aug 2021 09:46:25 +0200 Subject: [PATCH 169/376] Do not use deprecated ReleaseCleared in protobuf library. This is to make code work with protobuf arenas for memory management (ReleaseCleared is incompatible). The cleaning of the memory is also simpler. --- modules/dnn/src/caffe/caffe_importer.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/modules/dnn/src/caffe/caffe_importer.cpp b/modules/dnn/src/caffe/caffe_importer.cpp index 8673be03fb75..2d615c448a10 100644 --- a/modules/dnn/src/caffe/caffe_importer.cpp +++ b/modules/dnn/src/caffe/caffe_importer.cpp @@ -306,16 +306,13 @@ class CaffeImporter caffe::LayerParameter* binLayer = netBinary.mutable_layer(li); const int numBlobs = binLayer->blobs_size(); + std::vector blobs(numBlobs); + binLayer->mutable_blobs()->ExtractSubrange(0, numBlobs, blobs.data()); layerParams.blobs.resize(numBlobs); for (int bi = 0; bi < numBlobs; bi++) { - blobFromProto(binLayer->blobs(bi), layerParams.blobs[bi]); - } - binLayer->clear_blobs(); - CV_Assert(numBlobs == binLayer->blobs().ClearedCount()); - for (int bi = 0; bi < numBlobs; bi++) - { - delete binLayer->mutable_blobs()->ReleaseCleared(); + blobFromProto(*blobs[bi], layerParams.blobs[bi]); + delete blobs[bi]; } } From 994336bb5e881a839c661125c7d2b30d5b1f4ec0 Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 26 Aug 2021 22:19:56 +0800 Subject: [PATCH 170/376] Fixed the issues in the comments --- cmake/OpenCVDetectWebNN.cmake | 38 +++++++++++++------ cmake/templates/cvconfig.h.in | 3 -- ...s_image_classification_webnn_electron.html | 0 .../js_assets/{ => webnn-electron}/main.js | 0 .../{ => webnn-electron}/node_setup.js | 0 .../{ => webnn-electron}/package.json | 0 .../utils_webnn_electron.js | 0 modules/dnn/CMakeLists.txt | 4 +- modules/dnn/src/op_webnn.cpp | 3 -- modules/dnn/src/op_webnn.hpp | 7 +--- modules/js/CMakeLists.txt | 2 +- platforms/js/build_js.py | 11 ++++-- 12 files changed, 39 insertions(+), 29 deletions(-) rename doc/js_tutorials/js_assets/{ => webnn-electron}/js_image_classification_webnn_electron.html (100%) rename doc/js_tutorials/js_assets/{ => webnn-electron}/main.js (100%) rename doc/js_tutorials/js_assets/{ => webnn-electron}/node_setup.js (100%) rename doc/js_tutorials/js_assets/{ => webnn-electron}/package.json (100%) rename doc/js_tutorials/js_assets/{ => webnn-electron}/utils_webnn_electron.js (100%) diff --git a/cmake/OpenCVDetectWebNN.cmake b/cmake/OpenCVDetectWebNN.cmake index 8a405aa6731b..e2dd8f258c95 100644 --- a/cmake/OpenCVDetectWebNN.cmake +++ b/cmake/OpenCVDetectWebNN.cmake @@ -1,14 +1,18 @@ -ocv_clear_vars(HAVE_WEBNN) -ocv_clear_vars(WEBNN_EMSDK) if(NOT EMSCRIPTEN) if(WITH_WEBNN) - set(WEBNN_HEADER_DIRS "$ENV{WEBNN_NATIVE_DIR}/gen/src/include") - set(WEBNN_INCLUDE_DIRS "$ENV{WEBNN_NATIVE_DIR}/../../src/include") - set(WEBNN_LIBRARIES "$ENV{WEBNN_NATIVE_DIR}/libwebnn_native.so;$ENV{WEBNN_NATIVE_DIR}/libwebnn_proc.so") + ocv_check_environment_variables(WEBNN_HEADER_DIRS) + ocv_check_environment_variables(WEBNN_INCLUDE_DIRS) + ocv_check_environment_variables(WEBNN_LIBRARIES) + if(NOT DEFINED WEBNN_HEADER_DIRS) + set(WEBNN_HEADER_DIRS "$ENV{WEBNN_NATIVE_DIR}/gen/src/include") + endif() + if(NOT DEFINED WEBNN_INCLUDE_DIRS) + set(WEBNN_INCLUDE_DIRS "$ENV{WEBNN_NATIVE_DIR}/../../src/include") + endif() + if(NOT DEFINED WEBNN_LIBRARIES) + set(WEBNN_LIBRARIES "$ENV{WEBNN_NATIVE_DIR}/libwebnn_native.so;$ENV{WEBNN_NATIVE_DIR}/libwebnn_proc.so") + endif() endif() -endif() - -if(NOT EMSCRIPTEN) try_compile(VALID_WEBNN "${OpenCV_BINARY_DIR}" SOURCES "${OpenCV_SOURCE_DIR}/cmake/checks/webnn.cpp" @@ -25,10 +29,20 @@ else() ) endif() -if(NOT ${VALID_WEBNN}) - message(WARNING "Can't use WebNN-native") - return() +if(NOT VALID_WEBNN) + if(NOT EMSCRIPTEN) + message(WARNING "Can't use WebNN-native") + return() + else() + message(WARNING "Can't use WebNN") + return() + endif() +endif() + +if(NOT EMSCRIPTEN) + message(AUTHOR_WARNING "Use WebNN-native") +else() +message(AUTHOR_WARNING "Use WebNN") endif() -message(AUTHOR_WARNING "Use WebNN-native") set(HAVE_WEBNN 1) diff --git a/cmake/templates/cvconfig.h.in b/cmake/templates/cvconfig.h.in index 39708e14bdbf..6439d8b43f06 100644 --- a/cmake/templates/cvconfig.h.in +++ b/cmake/templates/cvconfig.h.in @@ -59,9 +59,6 @@ /* Vulkan support */ #cmakedefine HAVE_VULKAN -/* Webnn support */ -#cmakedefine HAVE_WEBNN - /* Define to 1 if you have the header file. */ #cmakedefine HAVE_INTTYPES_H 1 diff --git a/doc/js_tutorials/js_assets/js_image_classification_webnn_electron.html b/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html similarity index 100% rename from doc/js_tutorials/js_assets/js_image_classification_webnn_electron.html rename to doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html diff --git a/doc/js_tutorials/js_assets/main.js b/doc/js_tutorials/js_assets/webnn-electron/main.js similarity index 100% rename from doc/js_tutorials/js_assets/main.js rename to doc/js_tutorials/js_assets/webnn-electron/main.js diff --git a/doc/js_tutorials/js_assets/node_setup.js b/doc/js_tutorials/js_assets/webnn-electron/node_setup.js similarity index 100% rename from doc/js_tutorials/js_assets/node_setup.js rename to doc/js_tutorials/js_assets/webnn-electron/node_setup.js diff --git a/doc/js_tutorials/js_assets/package.json b/doc/js_tutorials/js_assets/webnn-electron/package.json similarity index 100% rename from doc/js_tutorials/js_assets/package.json rename to doc/js_tutorials/js_assets/webnn-electron/package.json diff --git a/doc/js_tutorials/js_assets/utils_webnn_electron.js b/doc/js_tutorials/js_assets/webnn-electron/utils_webnn_electron.js similarity index 100% rename from doc/js_tutorials/js_assets/utils_webnn_electron.js rename to doc/js_tutorials/js_assets/webnn-electron/utils_webnn_electron.js diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 54406c799010..5b11dd4b2c24 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -136,9 +136,9 @@ endif() set(webnn_srcs "") if(HAVE_WEBNN) - list(APPEND include_dirs ${WEBNN_HEADER_DIRS}) + list(APPEND include_dirs ${WEBNN_HEADER_DIRS}) list(APPEND include_dirs ${WEBNN_INCLUDE_DIRS}) - list(APPEND libs -Wl,--whole-archive ${WEBNN_LIBRARIES} -Wl,--no-whole-archive) + list(APPEND libs -Wl,--whole-archive ${WEBNN_LIBRARIES} -Wl,--no-whole-archive) list(APPEND webnn_srcs $ENV{WEBNN_NATIVE_DIR}/gen/src/webnn/webnn_cpp.cpp) endif() diff --git a/modules/dnn/src/op_webnn.cpp b/modules/dnn/src/op_webnn.cpp index e774beea22bf..4dba55bcbe80 100644 --- a/modules/dnn/src/op_webnn.cpp +++ b/modules/dnn/src/op_webnn.cpp @@ -1,9 +1,6 @@ // This file is part of OpenCV project. // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. -// -// Copyright (C) 2018-2019, Intel Corporation, all rights reserved. -// Third party copyrights are property of their respective owners. #include #include "op_webnn.hpp" diff --git a/modules/dnn/src/op_webnn.hpp b/modules/dnn/src/op_webnn.hpp index f2284c6d63cd..5b77b1082766 100644 --- a/modules/dnn/src/op_webnn.hpp +++ b/modules/dnn/src/op_webnn.hpp @@ -1,9 +1,6 @@ // This file is part of OpenCV project. // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. -// -// Copyright (C) 2018-2019, Intel Corporation, all rights reserved. -// Third party copyrights are property of their respective owners. #ifndef __OPENCV_DNN_OP_WEBNN_HPP__ #define __OPENCV_DNN_OP_WEBNN_HPP__ @@ -61,7 +58,7 @@ ml::Operand BuildConstant(const ml::GraphBuilder& builder, ml::OperandType type); struct Pool2dOptions { - public: + public: std::vector windowDimensions; std::vector padding; std::vector strides; @@ -91,7 +88,7 @@ struct Pool2dOptions { return &mOptions; } - private: + private: ml::Pool2dOptions mOptions; }; } diff --git a/modules/js/CMakeLists.txt b/modules/js/CMakeLists.txt index 5fb1d35694fb..5996e419ddc9 100644 --- a/modules/js/CMakeLists.txt +++ b/modules/js/CMakeLists.txt @@ -69,7 +69,7 @@ if(COMPILE_FLAGS) set_target_properties(${the_module} PROPERTIES COMPILE_FLAGS ${COMPILE_FLAGS}) endif() -set(EMSCRIPTEN_LINK_FLAGS "${EMSCRIPTEN_LINK_FLAGS} -s USE_WEBNN=1 --memory-init-file 0 -s TOTAL_MEMORY=128MB -s WASM_MEM_MAX=1GB -s ALLOW_MEMORY_GROWTH=1") +set(EMSCRIPTEN_LINK_FLAGS "${EMSCRIPTEN_LINK_FLAGS} --memory-init-file 0 -s TOTAL_MEMORY=128MB -s WASM_MEM_MAX=1GB -s ALLOW_MEMORY_GROWTH=1") set(EMSCRIPTEN_LINK_FLAGS "${EMSCRIPTEN_LINK_FLAGS} -s MODULARIZE=1 -s SINGLE_FILE=1") set(EMSCRIPTEN_LINK_FLAGS "${EMSCRIPTEN_LINK_FLAGS} -s EXPORT_NAME=\"'cv'\" -s DEMANGLE_SUPPORT=1") set(EMSCRIPTEN_LINK_FLAGS "${EMSCRIPTEN_LINK_FLAGS} -s FORCE_FILESYSTEM=1 --use-preload-plugins --bind --post-js ${JS_HELPER} ${COMPILE_FLAGS}") diff --git a/platforms/js/build_js.py b/platforms/js/build_js.py index 6b5d48371fde..8d3cb96cc39d 100644 --- a/platforms/js/build_js.py +++ b/platforms/js/build_js.py @@ -142,8 +142,7 @@ def get_cmake_cmd(self): "-DBUILD_EXAMPLES=ON", "-DBUILD_PACKAGE=OFF", "-DBUILD_TESTS=ON", - "-DBUILD_PERF_TESTS=ON", - "-DWITH_WEBNN=ON"] + "-DBUILD_PERF_TESTS=ON"] if self.options.cmake_option: cmd += self.options.cmake_option if self.options.build_doc: @@ -165,6 +164,9 @@ def get_cmake_cmd(self): cmd.append("-DBUILD_WASM_INTRIN_TESTS=ON") else: cmd.append("-DBUILD_WASM_INTRIN_TESTS=OFF") + + if self.options.webnn: + cmd.append("-DWITH_WEBNN=ON") flags = self.get_build_flags() if flags: @@ -173,7 +175,7 @@ def get_cmake_cmd(self): return cmd def get_build_flags(self): - flags = "-s USE_WEBNN=1 " + flags = "" if self.options.build_wasm: flags += "-s WASM=1 " elif self.options.disable_wasm: @@ -188,6 +190,8 @@ def get_build_flags(self): flags += "-msimd128 " if self.options.build_flags: flags += self.options.build_flags + if self.options.webnn: + flags += "-s USE_WEBNN=1 " return flags def config(self): @@ -247,6 +251,7 @@ def build_loader(self): # Write a path to modify file like argument of this flag parser.add_argument('--config', default=os.path.join(os.path.dirname(os.path.abspath(__file__)), 'opencv_js.config.py'), help="Specify configuration file with own list of exported into JS functions") + parser.add_argument('--webnn', action="store_true", help="Enable WebNN Backend") args = parser.parse_args() From 8ee33ca5511c186c81f0e2e9b705e5b0c5cc1935 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 28 Aug 2021 01:28:34 +0000 Subject: [PATCH 171/376] java(test): avoid deprecation warning - 'new Byte' => 'Byte.valueOf' --- .../org/opencv/test/utils/ConvertersTest.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/modules/java/test/common_test/src/org/opencv/test/utils/ConvertersTest.java b/modules/java/test/common_test/src/org/opencv/test/utils/ConvertersTest.java index 54d2736c4615..88d75dcdfbac 100644 --- a/modules/java/test/common_test/src/org/opencv/test/utils/ConvertersTest.java +++ b/modules/java/test/common_test/src/org/opencv/test/utils/ConvertersTest.java @@ -28,9 +28,9 @@ public void testMat_to_vector_char() { byte value1 = 2; byte value2 = 4; byte value3 = 3; - truth.add(new Byte(value1)); - truth.add(new Byte(value2)); - truth.add(new Byte(value3)); + truth.add(Byte.valueOf(value1)); + truth.add(Byte.valueOf(value2)); + truth.add(Byte.valueOf(value3)); assertEquals(truth, bs); } @@ -248,9 +248,9 @@ public void testMat_to_vector_uchar() { byte value1 = 2; byte value2 = 4; byte value3 = 3; - truth.add(new Byte(value1)); - truth.add(new Byte(value2)); - truth.add(new Byte(value3)); + truth.add(Byte.valueOf(value1)); + truth.add(Byte.valueOf(value2)); + truth.add(Byte.valueOf(value3)); assertEquals(truth, bs); } @@ -276,10 +276,10 @@ public void testVector_char_to_Mat() { byte value2 = 2; byte value3 = 3; byte value4 = 4; - bytes.add(new Byte(value1)); - bytes.add(new Byte(value2)); - bytes.add(new Byte(value3)); - bytes.add(new Byte(value4)); + bytes.add(Byte.valueOf(value1)); + bytes.add(Byte.valueOf(value2)); + bytes.add(Byte.valueOf(value3)); + bytes.add(Byte.valueOf(value4)); dst = Converters.vector_char_to_Mat(bytes); truth = new Mat(4, 1, CvType.CV_8SC1); @@ -499,10 +499,10 @@ public void testVector_uchar_to_Mat() { byte value2 = 2; byte value3 = 3; byte value4 = 4; - bytes.add(new Byte(value1)); - bytes.add(new Byte(value2)); - bytes.add(new Byte(value3)); - bytes.add(new Byte(value4)); + bytes.add(Byte.valueOf(value1)); + bytes.add(Byte.valueOf(value2)); + bytes.add(Byte.valueOf(value3)); + bytes.add(Byte.valueOf(value4)); dst = Converters.vector_uchar_to_Mat(bytes); truth = new Mat(4, 1, CvType.CV_8UC1); From 076587425e66a1dd66d06e9f274acd6f2eda2a3f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 28 Aug 2021 17:11:26 +0000 Subject: [PATCH 172/376] build: eliminate build warnings --- modules/features2d/src/kaze/KAZEFeatures.cpp | 61 +++++++++++-------- modules/photo/src/contrast_preserve.hpp | 4 +- modules/superres/src/btv_l1.cpp | 2 +- .../TrackingMotion/cornerDetector_Demo.cpp | 2 +- 4 files changed, 38 insertions(+), 31 deletions(-) diff --git a/modules/features2d/src/kaze/KAZEFeatures.cpp b/modules/features2d/src/kaze/KAZEFeatures.cpp index 58f0937f0038..ab591d417c69 100644 --- a/modules/features2d/src/kaze/KAZEFeatures.cpp +++ b/modules/features2d/src/kaze/KAZEFeatures.cpp @@ -311,7 +311,7 @@ class FindExtremumKAZEInvoker : public ParallelLoopBody void KAZEFeatures::Determinant_Hessian(std::vector& kpts) { int level = 0; - float dist = 0.0, smax = 3.0; + float smax = 3.0; int npoints = 0, id_repeated = 0; int left_x = 0, right_x = 0, up_y = 0, down_y = 0; bool is_extremum = false, is_repeated = false, is_out = false; @@ -338,17 +338,24 @@ void KAZEFeatures::Determinant_Hessian(std::vector& kpts) for (int j = 0; j < (int)kpts_par_[i].size(); j++) { level = i + 1; + const TEvolution& evolution_level = evolution_[level]; + is_extremum = true; is_repeated = false; is_out = false; + const KeyPoint& kpts_par_ij = kpts_par_[i][j]; + // Check in case we have the same point as maxima in previous evolution levels - for (int ik = 0; ik < (int)kpts.size(); ik++) { - if (kpts[ik].class_id == level || kpts[ik].class_id == level + 1 || kpts[ik].class_id == level - 1) { - dist = pow(kpts_par_[i][j].pt.x - kpts[ik].pt.x, 2) + pow(kpts_par_[i][j].pt.y - kpts[ik].pt.y, 2); + for (int ik = 0; ik < (int)kpts.size(); ik++) + { + const KeyPoint& kpts_ik = kpts[ik]; + if (kpts_ik.class_id == level || kpts_ik.class_id == level + 1 || kpts_ik.class_id == level - 1) { + Point2f diff = kpts_par_ij.pt - kpts_ik.pt; + float dist = diff.dot(diff); - if (dist < evolution_[level].sigma_size*evolution_[level].sigma_size) { - if (kpts_par_[i][j].response > kpts[ik].response) { + if (dist < evolution_level.sigma_size*evolution_level.sigma_size) { + if (kpts_par_ij.response > kpts_ik.response) { id_repeated = ik; is_repeated = true; } @@ -363,23 +370,23 @@ void KAZEFeatures::Determinant_Hessian(std::vector& kpts) if (is_extremum == true) { // Check that the point is under the image limits for the descriptor computation - left_x = cvRound(kpts_par_[i][j].pt.x - smax*kpts_par_[i][j].size); - right_x = cvRound(kpts_par_[i][j].pt.x + smax*kpts_par_[i][j].size); - up_y = cvRound(kpts_par_[i][j].pt.y - smax*kpts_par_[i][j].size); - down_y = cvRound(kpts_par_[i][j].pt.y + smax*kpts_par_[i][j].size); + left_x = cvRound(kpts_par_ij.pt.x - smax*kpts_par_ij.size); + right_x = cvRound(kpts_par_ij.pt.x + smax*kpts_par_ij.size); + up_y = cvRound(kpts_par_ij.pt.y - smax*kpts_par_ij.size); + down_y = cvRound(kpts_par_ij.pt.y + smax*kpts_par_ij.size); - if (left_x < 0 || right_x >= evolution_[level].Ldet.cols || - up_y < 0 || down_y >= evolution_[level].Ldet.rows) { + if (left_x < 0 || right_x >= evolution_level.Ldet.cols || + up_y < 0 || down_y >= evolution_level.Ldet.rows) { is_out = true; } if (is_out == false) { if (is_repeated == false) { - kpts.push_back(kpts_par_[i][j]); + kpts.push_back(kpts_par_ij); npoints++; } else { - kpts[id_repeated] = kpts_par_[i][j]; + kpts[id_repeated] = kpts_par_ij; } } } @@ -513,16 +520,16 @@ class KAZE_Descriptor_Invoker : public ParallelLoopBody if (options_.upright) { kpts[i].angle = 0.0; - if (options_.extended) + if (options_.extended) Get_KAZE_Upright_Descriptor_128(kpts[i], desc.ptr((int)i)); else Get_KAZE_Upright_Descriptor_64(kpts[i], desc.ptr((int)i)); } else { - KAZEFeatures::Compute_Main_Orientation(kpts[i], evolution, options_); + KAZEFeatures::Compute_Main_Orientation(kpts[i], evolution, options_); - if (options_.extended) + if (options_.extended) Get_KAZE_Descriptor_128(kpts[i], desc.ptr((int)i)); else Get_KAZE_Descriptor_64(kpts[i], desc.ptr((int)i)); @@ -712,26 +719,26 @@ void KAZE_Descriptor_Invoker::Get_KAZE_Upright_Descriptor_64(const KeyPoint &kpt y1 = (int)(sample_y - 0.5f); x1 = (int)(sample_x - 0.5f); - checkDescriptorLimits(x1, y1, options_.img_width, options_.img_height); + checkDescriptorLimits(x1, y1, options_.img_width, options_.img_height); y2 = (int)(sample_y + 0.5f); x2 = (int)(sample_x + 0.5f); - checkDescriptorLimits(x2, y2, options_.img_width, options_.img_height); + checkDescriptorLimits(x2, y2, options_.img_width, options_.img_height); fx = sample_x - x1; fy = sample_y - y1; - res1 = *(evolution[level].Lx.ptr(y1)+x1); - res2 = *(evolution[level].Lx.ptr(y1)+x2); - res3 = *(evolution[level].Lx.ptr(y2)+x1); - res4 = *(evolution[level].Lx.ptr(y2)+x2); + res1 = *(evolution[level].Lx.ptr(y1)+x1); + res2 = *(evolution[level].Lx.ptr(y1)+x2); + res3 = *(evolution[level].Lx.ptr(y2)+x1); + res4 = *(evolution[level].Lx.ptr(y2)+x2); rx = (1.0f - fx)*(1.0f - fy)*res1 + fx*(1.0f - fy)*res2 + (1.0f - fx)*fy*res3 + fx*fy*res4; - res1 = *(evolution[level].Ly.ptr(y1)+x1); - res2 = *(evolution[level].Ly.ptr(y1)+x2); - res3 = *(evolution[level].Ly.ptr(y2)+x1); - res4 = *(evolution[level].Ly.ptr(y2)+x2); + res1 = *(evolution[level].Ly.ptr(y1)+x1); + res2 = *(evolution[level].Ly.ptr(y1)+x2); + res3 = *(evolution[level].Ly.ptr(y2)+x1); + res4 = *(evolution[level].Ly.ptr(y2)+x2); ry = (1.0f - fx)*(1.0f - fy)*res1 + fx*(1.0f - fy)*res2 + (1.0f - fx)*fy*res3 + fx*fy*res4; rx = gauss_s1*rx; diff --git a/modules/photo/src/contrast_preserve.hpp b/modules/photo/src/contrast_preserve.hpp index 5681779fc9d9..4079272e994d 100644 --- a/modules/photo/src/contrast_preserve.hpp +++ b/modules/photo/src/contrast_preserve.hpp @@ -358,9 +358,9 @@ void Decolor::grayImContruct(vector &wei, const Mat &img, Mat &Gray) co { for(int i = 0;i(i,j)=Gray.at(i,j) + + Gray.at(i,j)=static_cast(Gray.at(i,j) + static_cast(wei[kk])*pow(rgb_channel[2].at(i,j),r)*pow(rgb_channel[1].at(i,j),g)* - pow(rgb_channel[0].at(i,j),b); + pow(rgb_channel[0].at(i,j),b)); kk=kk+1; } diff --git a/modules/superres/src/btv_l1.cpp b/modules/superres/src/btv_l1.cpp index cf2ca58ceb57..26c75a94ce44 100644 --- a/modules/superres/src/btv_l1.cpp +++ b/modules/superres/src/btv_l1.cpp @@ -356,7 +356,7 @@ namespace for (int m = 0, ind = 0; m <= ksize; ++m) { for (int l = ksize; l + m >= 0; --l, ++ind) - btvWeights[ind] = pow(alpha_f, std::abs(m) + std::abs(l)); + btvWeights[ind] = static_cast(pow(alpha_f, std::abs(m) + std::abs(l))); } } diff --git a/samples/cpp/tutorial_code/TrackingMotion/cornerDetector_Demo.cpp b/samples/cpp/tutorial_code/TrackingMotion/cornerDetector_Demo.cpp index a43024e557d5..e2835531e116 100644 --- a/samples/cpp/tutorial_code/TrackingMotion/cornerDetector_Demo.cpp +++ b/samples/cpp/tutorial_code/TrackingMotion/cornerDetector_Demo.cpp @@ -62,7 +62,7 @@ int main( int argc, char** argv ) { float lambda_1 = myHarris_dst.at(i, j)[0]; float lambda_2 = myHarris_dst.at(i, j)[1]; - Mc.at(i, j) = lambda_1*lambda_2 - 0.04f*pow( ( lambda_1 + lambda_2 ), 2 ); + Mc.at(i, j) = lambda_1*lambda_2 - 0.04f*((lambda_1 + lambda_2) * (lambda_1 + lambda_2)); } } From 3995deaf76d0c988d315630ace46eb01dba37174 Mon Sep 17 00:00:00 2001 From: Dale Phurrough Date: Sat, 28 Aug 2021 01:12:44 +0200 Subject: [PATCH 173/376] fix opencv/opencv#20544 nodiscard for msvc/gcc - includes workaround for preprocessor non-compliance - enable attribute syntax checking in msvc --- cmake/OpenCVCompilerOptions.cmake | 3 ++ modules/core/include/opencv2/core/cvdef.h | 39 ++++++++++++-- modules/core/include/opencv2/core/mat.hpp | 62 +++++++++++----------- modules/core/include/opencv2/core/matx.hpp | 14 ++--- 4 files changed, 77 insertions(+), 41 deletions(-) diff --git a/cmake/OpenCVCompilerOptions.cmake b/cmake/OpenCVCompilerOptions.cmake index 303d4f451e64..d444085ea3e7 100644 --- a/cmake/OpenCVCompilerOptions.cmake +++ b/cmake/OpenCVCompilerOptions.cmake @@ -400,6 +400,9 @@ if(MSVC) endif() endif() + # Enable [[attribute]] syntax checking to prevent silent failure: "attribute is ignored in this syntactic position" + add_extra_compiler_option("/w15240") + if(NOT ENABLE_NOISY_WARNINGS) ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4127) # conditional expression is constant ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4251) # class 'std::XXX' needs to have dll-interface to be used by clients of YYY diff --git a/modules/core/include/opencv2/core/cvdef.h b/modules/core/include/opencv2/core/cvdef.h index a4b560b4e1f0..6011b2a9313d 100644 --- a/modules/core/include/opencv2/core/cvdef.h +++ b/modules/core/include/opencv2/core/cvdef.h @@ -575,14 +575,47 @@ Cv64suf; # endif #endif +/****************************************************************************************\ +* CV_NODISCARD_STD attribute (C++17) * +* encourages the compiler to issue a warning if the return value is discarded * +\****************************************************************************************/ +#ifndef CV_NODISCARD_STD +# ifndef __has_cpp_attribute +// workaround preprocessor non-compliance https://reviews.llvm.org/D57851 +# define __has_cpp_attribute(__x) 0 +# endif +# if __has_cpp_attribute(nodiscard) +# define CV_NODISCARD_STD [[nodiscard]] +# elif __cplusplus >= 201703L +// available when compiler is C++17 compliant +# define CV_NODISCARD_STD [[nodiscard]] +# elif defined(_MSC_VER) && _MSC_VER >= 1911 && _MSVC_LANG >= 201703L +// available with VS2017 v15.3+ with /std:c++17 or higher; works on functions and classes +# define CV_NODISCARD_STD [[nodiscard]] +# elif defined(__GNUC__) && (((__GNUC__ * 100) + __GNUC_MINOR__) >= 700) && (__cplusplus >= 201103L) +// available with GCC 7.0+; works on functions, works or silently fails on classes +# define CV_NODISCARD_STD [[nodiscard]] +# elif defined(__GNUC__) && (((__GNUC__ * 100) + __GNUC_MINOR__) >= 408) && (__cplusplus >= 201103L) +// available with GCC 4.8+ but it usually does nothing and can fail noisily -- therefore not used +// define CV_NODISCARD_STD [[gnu::warn_unused_result]] +# endif +#endif +#ifndef CV_NODISCARD_STD +# define CV_NODISCARD_STD /* nothing by default */ +#endif + /****************************************************************************************\ -* CV_NODISCARD attribute * -* encourages the compiler to issue a warning if the return value is discarded (C++17) * +* CV_NODISCARD attribute (deprecated, GCC only) * +* DONT USE: use instead the standard CV_NODISCARD_STD macro above * +* this legacy method silently fails to issue warning until some version * +* after gcc 6.3.0. Yet with gcc 7+ you can use the above standard method * +* which makes this method useless. Don't use it. * +* @deprecated use instead CV_NODISCARD_STD * \****************************************************************************************/ #ifndef CV_NODISCARD # if defined(__GNUC__) -# define CV_NODISCARD __attribute__((__warn_unused_result__)) // at least available with GCC 3.4 +# define CV_NODISCARD __attribute__((__warn_unused_result__)) # elif defined(__clang__) && defined(__has_attribute) # if __has_attribute(__warn_unused_result__) # define CV_NODISCARD __attribute__((__warn_unused_result__)) diff --git a/modules/core/include/opencv2/core/mat.hpp b/modules/core/include/opencv2/core/mat.hpp index d0ce61e1234c..fdcb8fc817fb 100644 --- a/modules/core/include/opencv2/core/mat.hpp +++ b/modules/core/include/opencv2/core/mat.hpp @@ -1204,14 +1204,14 @@ class CV_EXPORTS Mat The method creates a square diagonal matrix from specified main diagonal. @param d One-dimensional matrix that represents the main diagonal. */ - static Mat diag(const Mat& d); + CV_NODISCARD_STD static Mat diag(const Mat& d); /** @brief Creates a full copy of the array and the underlying data. The method creates a full copy of the array. The original step[] is not taken into account. So, the array copy is a continuous array occupying total()*elemSize() bytes. */ - Mat clone() const CV_NODISCARD; + CV_NODISCARD_STD Mat clone() const; /** @brief Copies the matrix to another one. @@ -1375,20 +1375,20 @@ class CV_EXPORTS Mat @param cols Number of columns. @param type Created matrix type. */ - static MatExpr zeros(int rows, int cols, int type); + CV_NODISCARD_STD static MatExpr zeros(int rows, int cols, int type); /** @overload @param size Alternative to the matrix size specification Size(cols, rows) . @param type Created matrix type. */ - static MatExpr zeros(Size size, int type); + CV_NODISCARD_STD static MatExpr zeros(Size size, int type); /** @overload @param ndims Array dimensionality. @param sz Array of integers specifying the array shape. @param type Created matrix type. */ - static MatExpr zeros(int ndims, const int* sz, int type); + CV_NODISCARD_STD static MatExpr zeros(int ndims, const int* sz, int type); /** @brief Returns an array of all 1's of the specified size and type. @@ -1406,20 +1406,20 @@ class CV_EXPORTS Mat @param cols Number of columns. @param type Created matrix type. */ - static MatExpr ones(int rows, int cols, int type); + CV_NODISCARD_STD static MatExpr ones(int rows, int cols, int type); /** @overload @param size Alternative to the matrix size specification Size(cols, rows) . @param type Created matrix type. */ - static MatExpr ones(Size size, int type); + CV_NODISCARD_STD static MatExpr ones(Size size, int type); /** @overload @param ndims Array dimensionality. @param sz Array of integers specifying the array shape. @param type Created matrix type. */ - static MatExpr ones(int ndims, const int* sz, int type); + CV_NODISCARD_STD static MatExpr ones(int ndims, const int* sz, int type); /** @brief Returns an identity matrix of the specified size and type. @@ -1435,13 +1435,13 @@ class CV_EXPORTS Mat @param cols Number of columns. @param type Created matrix type. */ - static MatExpr eye(int rows, int cols, int type); + CV_NODISCARD_STD static MatExpr eye(int rows, int cols, int type); /** @overload @param size Alternative matrix size specification as Size(cols, rows) . @param type Created matrix type. */ - static MatExpr eye(Size size, int type); + CV_NODISCARD_STD static MatExpr eye(Size size, int type); /** @brief Allocates new array data if needed. @@ -2302,7 +2302,7 @@ template class Mat_ : public Mat Mat_ row(int y) const; Mat_ col(int x) const; Mat_ diag(int d=0) const; - Mat_ clone() const CV_NODISCARD; + CV_NODISCARD_STD Mat_ clone() const; //! overridden forms of Mat::elemSize() etc. size_t elemSize() const; @@ -2315,14 +2315,14 @@ template class Mat_ : public Mat size_t stepT(int i=0) const; //! overridden forms of Mat::zeros() etc. Data type is omitted, of course - static MatExpr zeros(int rows, int cols); - static MatExpr zeros(Size size); - static MatExpr zeros(int _ndims, const int* _sizes); - static MatExpr ones(int rows, int cols); - static MatExpr ones(Size size); - static MatExpr ones(int _ndims, const int* _sizes); - static MatExpr eye(int rows, int cols); - static MatExpr eye(Size size); + CV_NODISCARD_STD static MatExpr zeros(int rows, int cols); + CV_NODISCARD_STD static MatExpr zeros(Size size); + CV_NODISCARD_STD static MatExpr zeros(int _ndims, const int* _sizes); + CV_NODISCARD_STD static MatExpr ones(int rows, int cols); + CV_NODISCARD_STD static MatExpr ones(Size size); + CV_NODISCARD_STD static MatExpr ones(int _ndims, const int* _sizes); + CV_NODISCARD_STD static MatExpr eye(int rows, int cols); + CV_NODISCARD_STD static MatExpr eye(Size size); //! some more overridden methods Mat_& adjustROI( int dtop, int dbottom, int dleft, int dright ); @@ -2469,10 +2469,10 @@ class CV_EXPORTS UMat //! <0 - a diagonal from the lower half) UMat diag(int d=0) const; //! constructs a square diagonal matrix which main diagonal is vector "d" - static UMat diag(const UMat& d); + CV_NODISCARD_STD static UMat diag(const UMat& d); //! returns deep copy of the matrix, i.e. the data is copied - UMat clone() const CV_NODISCARD; + CV_NODISCARD_STD UMat clone() const; //! copies the matrix content to "m". // It calls m.create(this->size(), this->type()). void copyTo( OutputArray m ) const; @@ -2503,14 +2503,14 @@ class CV_EXPORTS UMat double dot(InputArray m) const; //! Matlab-style matrix initialization - static UMat zeros(int rows, int cols, int type); - static UMat zeros(Size size, int type); - static UMat zeros(int ndims, const int* sz, int type); - static UMat ones(int rows, int cols, int type); - static UMat ones(Size size, int type); - static UMat ones(int ndims, const int* sz, int type); - static UMat eye(int rows, int cols, int type); - static UMat eye(Size size, int type); + CV_NODISCARD_STD static UMat zeros(int rows, int cols, int type); + CV_NODISCARD_STD static UMat zeros(Size size, int type); + CV_NODISCARD_STD static UMat zeros(int ndims, const int* sz, int type); + CV_NODISCARD_STD static UMat ones(int rows, int cols, int type); + CV_NODISCARD_STD static UMat ones(Size size, int type); + CV_NODISCARD_STD static UMat ones(int ndims, const int* sz, int type); + CV_NODISCARD_STD static UMat eye(int rows, int cols, int type); + CV_NODISCARD_STD static UMat eye(Size size, int type); //! allocates new matrix data unless the matrix already has specified size and type. // previous data is unreferenced if needed. @@ -2767,7 +2767,7 @@ class CV_EXPORTS SparseMat SparseMat& operator = (const Mat& m); //! creates full copy of the matrix - SparseMat clone() const CV_NODISCARD; + CV_NODISCARD_STD SparseMat clone() const; //! copies all the data to the destination matrix. All the previous content of m is erased void copyTo( SparseMat& m ) const; @@ -3004,7 +3004,7 @@ template class SparseMat_ : public SparseMat SparseMat_& operator = (const Mat& m); //! makes full copy of the matrix. All the elements are duplicated - SparseMat_ clone() const CV_NODISCARD; + CV_NODISCARD_STD SparseMat_ clone() const; //! equivalent to cv::SparseMat::create(dims, _sizes, DataType<_Tp>::type) void create(int dims, const int* _sizes); //! converts sparse matrix to the old-style CvSparseMat. All the elements are copied diff --git a/modules/core/include/opencv2/core/matx.hpp b/modules/core/include/opencv2/core/matx.hpp index 733f675192c1..f25c8bce57ed 100644 --- a/modules/core/include/opencv2/core/matx.hpp +++ b/modules/core/include/opencv2/core/matx.hpp @@ -146,22 +146,22 @@ template class Matx Matx(std::initializer_list<_Tp>); //!< initialize from an initializer list #endif - static Matx all(_Tp alpha); - static Matx zeros(); - static Matx ones(); - static Matx eye(); - static Matx diag(const diag_type& d); + CV_NODISCARD_STD static Matx all(_Tp alpha); + CV_NODISCARD_STD static Matx zeros(); + CV_NODISCARD_STD static Matx ones(); + CV_NODISCARD_STD static Matx eye(); + CV_NODISCARD_STD static Matx diag(const diag_type& d); /** @brief Generates uniformly distributed random numbers @param a Range boundary. @param b The other range boundary (boundaries don't have to be ordered, the lower boundary is inclusive, the upper one is exclusive). */ - static Matx randu(_Tp a, _Tp b); + CV_NODISCARD_STD static Matx randu(_Tp a, _Tp b); /** @brief Generates normally distributed random numbers @param a Mean value. @param b Standard deviation. */ - static Matx randn(_Tp a, _Tp b); + CV_NODISCARD_STD static Matx randn(_Tp a, _Tp b); //! dot product computed with the default precision _Tp dot(const Matx<_Tp, m, n>& v) const; From 2ed5cba110552f8cfdc4925fd627f5a63187b76f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sun, 29 Aug 2021 09:17:44 +0000 Subject: [PATCH 174/376] build: eliminate build warnings --- modules/calib3d/src/chessboard.cpp | 8 ++++---- modules/gapi/test/streaming/gapi_streaming_tests.cpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/calib3d/src/chessboard.cpp b/modules/calib3d/src/chessboard.cpp index 18e2605f53b5..1801a1cbf4c9 100644 --- a/modules/calib3d/src/chessboard.cpp +++ b/modules/calib3d/src/chessboard.cpp @@ -1707,10 +1707,10 @@ void Chessboard::Board::normalizeOrientation(bool bblack) iter_bottom_left.getCell()->empty() || iter_bottom_right.getCell()->empty()) return; - float d1 = pow(top_left->top_left->x,2)+pow(top_left->top_left->y,2); - float d2 = pow((*iter_top_right)->x,2)+pow((*iter_top_right)->y,2); - float d3 = pow((*iter_bottom_left)->x,2)+pow((*iter_bottom_left)->y,2); - float d4 = pow((*iter_bottom_right)->x,2)+pow((*iter_bottom_right)->y,2); + float d1 = top_left->top_left->dot(*top_left->top_left); + float d2 = (*iter_top_right)->dot(*(*iter_top_right)); + float d3 = (*iter_bottom_left)->dot(*(*iter_bottom_left)); + float d4 = (*iter_bottom_right)->dot(*(*iter_bottom_right)); if(d2 <= d1 && d2 <= d3 && d2 <= d4) // top left is top right rotateLeft(); else if(d3 <= d1 && d3 <= d2 && d3 <= d4) // top left is bottom left diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp index 76027a56c682..369a0c069f4f 100644 --- a/modules/gapi/test/streaming/gapi_streaming_tests.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_tests.cpp @@ -292,7 +292,7 @@ struct StreamDataProvider : public cv::gapi::wip::onevpl::IDataProvider { size_t fetch_data(size_t out_data_size, void* out_data_buf) override { data_stream.read(reinterpret_cast(out_data_buf), out_data_size); - return data_stream.gcount(); + return (size_t)data_stream.gcount(); } bool empty() const override { return data_stream.eof() || data_stream.bad(); From f25951c412dec1329292136a171bf876b9cb7fbd Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 30 Aug 2021 11:46:14 +0000 Subject: [PATCH 175/376] core(ocl): handle NULL in dumpValue() debug call - NULL is used for allocation of workgroup local variables --- modules/core/src/ocl.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index 2d85a32366d9..0c14e7f3e073 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -2981,6 +2981,8 @@ bool Kernel::empty() const static cv::String dumpValue(size_t sz, const void* p) { + if (!p) + return "NULL"; if (sz == 4) return cv::format("%d / %uu / 0x%08x / %g", *(int*)p, *(int*)p, *(int*)p, *(float*)p); if (sz == 8) From ae6fabc6fe7e372808d3b8aab08398c87074fb7d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 30 Aug 2021 20:40:14 +0000 Subject: [PATCH 176/376] dnn(ocl): drop CL_KERNEL_PREFERRED_WORK_GROUP_SIZE_MULTIPLE check - it is a hint and it should not block kernel execution --- .../src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 38 +++---------------- 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index ef7c380c1be5..517a663e46a2 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -1437,26 +1437,13 @@ bool OCL4DNNConvSpatial::createGEMMLikeConvKernel(int32_t blockM, ocl::Program program = compileKernel(); if (program.ptr()) { - size_t workgroupSize_used; ocl::Kernel kernel(kernel_name_.c_str(), program); if (kernel.empty()) return false; - workgroupSize_used = kernel.preferedWorkGroupSizeMultiple(); - if (workgroupSize_used != simd_size) - { - std::cerr << "OpenCV(ocl4dnn): The OpenCL compiler chose a simd size (" << workgroupSize_used << ") that " << std::endl; - std::cerr << " does not equal the size (" << simd_size << ") kernel source required." << std::endl; - std::cerr << " Skip this kernel " << kernel_name_ << std::endl; - unloadProgram(kernel_name_); - return false; - } - else - { - kernelQueue.push_back(makePtr(kernel_name_, &global_size[0], &local_size[0], &workItemOutput[0], - true, KERNEL_TYPE_GEMM_LIKE)); - return true; - } + kernelQueue.push_back(makePtr(kernel_name_, &global_size[0], &local_size[0], &workItemOutput[0], + true, KERNEL_TYPE_GEMM_LIKE)); + return true; } else return false; @@ -1502,26 +1489,13 @@ bool OCL4DNNConvSpatial::createIDLFKernel(int32_t blockWidth, ocl::Program program = compileKernel(); if (program.ptr()) { - size_t workgroupSize_used; ocl::Kernel kernel(kernel_name_.c_str(), program); if (kernel.empty()) return false; - workgroupSize_used = kernel.preferedWorkGroupSizeMultiple(); - if (workgroupSize_used != simd_size) - { - std::cerr << "OpenCV(ocl4dnn): The OpenCL compiler chose a simd size (" << workgroupSize_used << ") that " << std::endl; - std::cerr << " does not equal the size (" << simd_size << ") kernel source required." << std::endl; - std::cerr << " Skip this kernel " << kernel_name_ << std::endl; - unloadProgram(kernel_name_); - return false; - } - else - { - kernelQueue.push_back(makePtr(kernel_name_, &global_size[0], &local_size[0], &workItemOutput[0], - true, KERNEL_TYPE_INTEL_IDLF)); - return true; - } + kernelQueue.push_back(makePtr(kernel_name_, &global_size[0], &local_size[0], &workItemOutput[0], + true, KERNEL_TYPE_INTEL_IDLF)); + return true; } else return false; From 16b95145439c509b9adb8600e0ff6dd83845f208 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Fri, 27 Aug 2021 15:01:09 +0300 Subject: [PATCH 177/376] feat: update conversion logic for `std::vector` in Python bindings `PyObject*` to `std::vector` conversion logic: - If user passed Numpy Array - If array is planar and T is a primitive type (doesn't require constructor call) that matches with the element type of array, then copy element one by one with the respect of the step between array elements. If compiler is lucky (or brave enough) copy loop can be vectorized. For classes that require constructor calls this path is not possible, because we can't begin an object lifetime without hacks. - Otherwise fall-back to general case - Otherwise - execute the general case: If PyObject* corresponds to Sequence protocol - iterate over the sequence elements and invoke the appropriate `pyopencv_to` function. `std::vector` to `PyObject*` conversion logic: - If `std::vector` is empty - return empty tuple. - If `T` has a corresponding `Mat` `DataType` than return Numpy array instance of the matching `dtype` e.g. `std::vector` is returned as `np.ndarray` of shape `Nx4` and `dtype=int`. This branch helps to optimize further evaluations in user code. - Otherwise - execute the general case: Construct a tuple of length N = `std::vector::size` and insert elements one by one. Unnecessary functions were removed and code was rearranged to allow compiler select the appropriate conversion function specialization. --- .../include/opencv2/core/bindings_utils.hpp | 47 ++ modules/core/src/bindings_utils.cpp | 47 ++ modules/python/src2/cv2.cpp | 493 ++++++++---------- modules/python/test/test_legacy.py | 9 +- modules/python/test/test_misc.py | 103 ++++ 5 files changed, 432 insertions(+), 267 deletions(-) diff --git a/modules/core/include/opencv2/core/bindings_utils.hpp b/modules/core/include/opencv2/core/bindings_utils.hpp index a3f83d9c2cf5..c53511f88fc7 100644 --- a/modules/core/include/opencv2/core/bindings_utils.hpp +++ b/modules/core/include/opencv2/core/bindings_utils.hpp @@ -122,6 +122,53 @@ String testReservedKeywordConversion(int positional_argument, int lambda = 2, in return format("arg=%d, lambda=%d, from=%d", positional_argument, lambda, from); } +CV_EXPORTS_W String dumpVectorOfInt(const std::vector& vec); + +CV_EXPORTS_W String dumpVectorOfDouble(const std::vector& vec); + +CV_EXPORTS_W String dumpVectorOfRect(const std::vector& vec); + +CV_WRAP static inline +void generateVectorOfRect(size_t len, CV_OUT std::vector& vec) +{ + vec.resize(len); + if (len > 0) + { + RNG rng(12345); + Mat tmp(static_cast(len), 1, CV_32SC4); + rng.fill(tmp, RNG::UNIFORM, 10, 20); + tmp.copyTo(vec); + } +} + +CV_WRAP static inline +void generateVectorOfInt(size_t len, CV_OUT std::vector& vec) +{ + vec.resize(len); + if (len > 0) + { + RNG rng(554433); + Mat tmp(static_cast(len), 1, CV_32SC1); + rng.fill(tmp, RNG::UNIFORM, -10, 10); + tmp.copyTo(vec); + } +} + +CV_WRAP static inline +void generateVectorOfMat(size_t len, int rows, int cols, int dtype, CV_OUT std::vector& vec) +{ + vec.resize(len); + if (len > 0) + { + RNG rng(65431); + for (size_t i = 0; i < len; ++i) + { + vec[i].create(rows, cols, dtype); + rng.fill(vec[i], RNG::UNIFORM, 0, 10); + } + } +} + CV_WRAP static inline void testRaiseGeneralException() { diff --git a/modules/core/src/bindings_utils.cpp b/modules/core/src/bindings_utils.cpp index 050b7247f87f..ea5b82ac7d8c 100644 --- a/modules/core/src/bindings_utils.cpp +++ b/modules/core/src/bindings_utils.cpp @@ -5,6 +5,7 @@ #include "precomp.hpp" #include "opencv2/core/bindings_utils.hpp" #include +#include namespace cv { namespace utils { @@ -208,4 +209,50 @@ CV_EXPORTS_W String dumpInputOutputArrayOfArrays(InputOutputArrayOfArrays argume return ss.str(); } +static inline std::ostream& operator<<(std::ostream& os, const cv::Rect& rect) +{ + return os << "[x=" << rect.x << ", y=" << rect.y << ", w=" << rect.width << ", h=" << rect.height << ']'; +} + +template +static inline String dumpVector(const std::vector& vec, Formatter format) +{ + std::ostringstream oss("[", std::ios::ate); + if (!vec.empty()) + { + oss << format << vec[0]; + for (std::size_t i = 1; i < vec.size(); ++i) + { + oss << ", " << format << vec[i]; + } + } + oss << "]"; + return oss.str(); +} + +static inline std::ostream& noFormat(std::ostream& os) +{ + return os; +} + +static inline std::ostream& floatFormat(std::ostream& os) +{ + return os << std::fixed << std::setprecision(2); +} + +String dumpVectorOfInt(const std::vector& vec) +{ + return dumpVector(vec, &noFormat); +} + +String dumpVectorOfDouble(const std::vector& vec) +{ + return dumpVector(vec, &floatFormat); +} + +String dumpVectorOfRect(const std::vector& vec) +{ + return dumpVector(vec, &noFormat); +} + }} // namespace diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index 6c5e6463d2fb..ff57978dc52b 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -493,6 +493,33 @@ bool parseSequence(PyObject* obj, RefWrapper (&value)[N], const ArgInfo& info } } // namespace +namespace traits { +template +struct BooleanConstant +{ + static const bool value = Value; + typedef BooleanConstant type; +}; + +typedef BooleanConstant TrueType; +typedef BooleanConstant FalseType; + +template +struct VoidType { + typedef void type; +}; + +template +struct IsRepresentableAsMatDataType : FalseType +{ +}; + +template +struct IsRepresentableAsMatDataType::channel_type>::type> : TrueType +{ +}; +} // namespace traits + typedef std::vector vector_uchar; typedef std::vector vector_char; typedef std::vector vector_int; @@ -1042,6 +1069,30 @@ bool pyopencv_to(PyObject* obj, uchar& value, const ArgInfo& info) return ivalue != -1 || !PyErr_Occurred(); } +template<> +bool pyopencv_to(PyObject* obj, char& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (isBool(obj)) + { + failmsg("Argument '%s' must be an integer, not bool", info.name); + return false; + } + if (PyArray_IsIntegerScalar(obj)) + { + value = saturate_cast(PyArray_PyIntAsInt(obj)); + } + else + { + failmsg("Argument '%s' is required to be an integer", info.name); + return false; + } + return !CV_HAS_CONVERSION_ERROR(value); +} + template<> PyObject* pyopencv_from(const double& value) { @@ -1454,277 +1505,12 @@ PyObject* pyopencv_from(const Point3d& p) return Py_BuildValue("(ddd)", p.x, p.y, p.z); } -template struct pyopencvVecConverter -{ - typedef typename DataType<_Tp>::channel_type _Cp; - static inline bool copyOneItem(PyObject *obj, size_t start, int channels, _Cp * data) - { - for(size_t j = 0; (int)j < channels; j++ ) - { - SafeSeqItem sub_item_wrap(obj, start + j); - PyObject* item_ij = sub_item_wrap.item; - if( PyInt_Check(item_ij)) - { - int v = (int)PyInt_AsLong(item_ij); - if( v == -1 && PyErr_Occurred() ) - return false; - data[j] = saturate_cast<_Cp>(v); - } - else if( PyLong_Check(item_ij)) - { - int v = (int)PyLong_AsLong(item_ij); - if( v == -1 && PyErr_Occurred() ) - return false; - data[j] = saturate_cast<_Cp>(v); - } - else if( PyFloat_Check(item_ij)) - { - double v = PyFloat_AsDouble(item_ij); - if( PyErr_Occurred() ) - return false; - data[j] = saturate_cast<_Cp>(v); - } - else - return false; - } - return true; - } - static bool to(PyObject* obj, std::vector<_Tp>& value, const ArgInfo& info) - { - if(!obj || obj == Py_None) - return true; - if (PyArray_Check(obj)) - { - Mat m; - pyopencv_to(obj, m, info); - m.copyTo(value); - return true; - } - else if (PySequence_Check(obj)) - { - const int type = traits::Type<_Tp>::value; - const int depth = CV_MAT_DEPTH(type), channels = CV_MAT_CN(type); - size_t i, n = PySequence_Size(obj); - value.resize(n); - for (i = 0; i < n; i++ ) - { - SafeSeqItem item_wrap(obj, i); - PyObject* item = item_wrap.item; - _Cp* data = (_Cp*)&value[i]; - - if( channels == 2 && PyComplex_Check(item) ) - { - data[0] = saturate_cast<_Cp>(PyComplex_RealAsDouble(item)); - data[1] = saturate_cast<_Cp>(PyComplex_ImagAsDouble(item)); - } - else if( channels > 1 ) - { - if( PyArray_Check(item)) - { - Mat src; - pyopencv_to(item, src, info); - if( src.dims != 2 || src.channels() != 1 || - ((src.cols != 1 || src.rows != channels) && - (src.cols != channels || src.rows != 1))) - break; - Mat dst(src.rows, src.cols, depth, data); - src.convertTo(dst, type); - if( dst.data != (uchar*)data ) - break; - } - else if (PySequence_Check(item)) - { - if (!copyOneItem(item, 0, channels, data)) - break; - } - else - { - break; - } - } - else if (channels == 1) - { - if (!copyOneItem(obj, i, channels, data)) - break; - } - else - { - break; - } - } - return i == n; - } - return false; - } - - static PyObject* from(const std::vector<_Tp>& value) - { - if(value.empty()) - return PyTuple_New(0); - int type = traits::Type<_Tp>::value; - int depth = CV_MAT_DEPTH(type), channels = CV_MAT_CN(type); - Mat src((int)value.size(), channels, depth, (uchar*)&value[0]); - return pyopencv_from(src); - } -}; - -template -bool pyopencv_to(PyObject* obj, std::vector<_Tp>& value, const ArgInfo& info) -{ - return pyopencvVecConverter<_Tp>::to(obj, value, info); -} - -template -PyObject* pyopencv_from(const std::vector<_Tp>& value) -{ - return pyopencvVecConverter<_Tp>::from(value); -} - -template static inline bool pyopencv_to_generic_vec(PyObject* obj, std::vector<_Tp>& value, const ArgInfo& info) -{ - if(!obj || obj == Py_None) - return true; - if (!PySequence_Check(obj)) - return false; - size_t n = PySequence_Size(obj); - value.resize(n); - for(size_t i = 0; i < n; i++ ) - { - SafeSeqItem item_wrap(obj, i); - if(!pyopencv_to(item_wrap.item, value[i], info)) - return false; - } - return true; -} - -template static inline PyObject* pyopencv_from_generic_vec(const std::vector<_Tp>& value) -{ - int i, n = (int)value.size(); - PyObject* seq = PyList_New(n); - for( i = 0; i < n; i++ ) - { - PyObject* item = pyopencv_from(value[i]); - if(!item) - break; - PyList_SetItem(seq, i, item); - } - if( i < n ) - { - Py_DECREF(seq); - return 0; - } - return seq; -} - template<> PyObject* pyopencv_from(const std::pair& src) { return Py_BuildValue("(id)", src.first, src.second); } -template struct pyopencvVecConverter > -{ - static bool to(PyObject* obj, std::vector >& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector >& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template struct pyopencvVecConverter > -{ - static bool to(PyObject* obj, std::vector >& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector >& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - template<> bool pyopencv_to(PyObject* obj, TermCriteria& dst, const ArgInfo& info) { @@ -1852,6 +1638,183 @@ PyObject* pyopencv_from(const Moments& m) "nu30", m.nu30, "nu21", m.nu21, "nu12", m.nu12, "nu03", m.nu03); } +template +struct pyopencvVecConverter; + +template +bool pyopencv_to(PyObject* obj, std::vector& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + return pyopencvVecConverter::to(obj, value, info); +} + +template +PyObject* pyopencv_from(const std::vector& value) +{ + return pyopencvVecConverter::from(value); +} + +template +static bool pyopencv_to_generic_vec(PyObject* obj, std::vector& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (!PySequence_Check(obj)) + { + failmsg("Can't parse '%s'. Input argument doesn't provide sequence protocol", info.name); + return false; + } + const size_t n = static_cast(PySequence_Size(obj)); + value.resize(n); + for (size_t i = 0; i < n; i++) + { + SafeSeqItem item_wrap(obj, i); + if (!pyopencv_to(item_wrap.item, value[i], info)) + { + failmsg("Can't parse '%s'. Sequence item with index %lu has a wrong type", info.name, i); + return false; + } + } + return true; +} + +template +static PyObject* pyopencv_from_generic_vec(const std::vector& value) +{ + Py_ssize_t n = static_cast(value.size()); + PySafeObject seq(PyTuple_New(n)); + for (Py_ssize_t i = 0; i < n; i++) + { + PyObject* item = pyopencv_from(value[i]); + // If item can't be assigned - PyTuple_SetItem raises exception and returns -1. + if (!item || PyTuple_SetItem(seq, i, item) == -1) + { + return NULL; + } + } + return seq.release(); +} + +template +struct pyopencvVecConverter +{ + typedef typename std::vector::iterator VecIt; + + static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) + { + if (!PyArray_Check(obj)) + { + return pyopencv_to_generic_vec(obj, value, info); + } + // If user passed an array it is possible to make faster conversions in several cases + PyArrayObject* array_obj = reinterpret_cast(obj); + const NPY_TYPES target_type = asNumpyType(); + const NPY_TYPES source_type = static_cast(PyArray_TYPE(array_obj)); + if (target_type == NPY_OBJECT) + { + // Non-planar arrays representing objects (e.g. array of N Rect is an array of shape Nx4) have NPY_OBJECT + // as their target type. + return pyopencv_to_generic_vec(obj, value, info); + } + if (PyArray_NDIM(array_obj) > 1) + { + failmsg("Can't parse %dD array as '%s' vector argument", PyArray_NDIM(array_obj), info.name); + return false; + } + if (target_type != source_type) + { + // Source type requires conversion + // Allowed conversions for target type is handled in the corresponding pyopencv_to function + return pyopencv_to_generic_vec(obj, value, info); + } + // For all other cases, all array data can be directly copied to std::vector data + // Simple `memcpy` is not possible because NumPy array can reference a slice of the bigger array: + // ``` + // arr = np.ones((8, 4, 5), dtype=np.int32) + // convertible_to_vector_of_int = arr[:, 0, 1] + // ``` + value.resize(static_cast(PyArray_SIZE(array_obj))); + const npy_intp item_step = PyArray_STRIDE(array_obj, 0) / PyArray_ITEMSIZE(array_obj); + const Tp* data_ptr = static_cast(PyArray_DATA(array_obj)); + for (VecIt it = value.begin(); it != value.end(); ++it, data_ptr += item_step) { + *it = *data_ptr; + } + return true; + } + + static PyObject* from(const std::vector& value) + { + if (value.empty()) + { + return PyTuple_New(0); + } + return from(value, ::traits::IsRepresentableAsMatDataType()); + } + +private: + static PyObject* from(const std::vector& value, ::traits::FalseType) + { + // Underlying type is not representable as Mat Data Type + return pyopencv_from_generic_vec(value); + } + + static PyObject* from(const std::vector& value, ::traits::TrueType) + { + // Underlying type is representable as Mat Data Type, so faster return type is available + typedef DataType DType; + typedef typename DType::channel_type UnderlyingArrayType; + + // If Mat is always exposed as NumPy array this code path can be reduced to the following snipped: + // Mat src(value); + // PyObject* array = pyopencv_from(src); + // return PyArray_Squeeze(reinterpret_cast(array)); + // This puts unnecessary restrictions on Mat object those might be avoided without losing the performance. + // Moreover, this version is a bit faster, because it doesn't create temporary objects with reference counting. + + const NPY_TYPES target_type = asNumpyType(); + const int cols = DType::channels; + PyObject* array; + if (cols == 1) + { + npy_intp dims = static_cast(value.size()); + array = PyArray_SimpleNew(1, &dims, target_type); + } + else + { + npy_intp dims[2] = {static_cast(value.size()), cols}; + array = PyArray_SimpleNew(2, dims, target_type); + } + if(!array) + { + // NumPy arrays with shape (N, 1) and (N) are not equal, so correct error message should distinguish + // them too. + String shape; + if (cols > 1) + { + shape = cv::format("(%d x %d)", static_cast(value.size()), cols); + } + else + { + shape = cv::format("(%d)", static_cast(value.size())); + } + CV_Error_(Error::StsError, ("Can't allocate NumPy array for vector with dtype=%d shape=%s", + static_cast(target_type), static_cast(value.size()), shape.c_str())); + } + // Fill the array + PyArrayObject* array_obj = reinterpret_cast(array); + UnderlyingArrayType* array_data = static_cast(PyArray_DATA(array_obj)); + // if Tp is representable as Mat DataType, so the following cast is pretty safe... + const UnderlyingArrayType* value_data = reinterpret_cast(value.data()); + memcpy(array_data, value_data, sizeof(UnderlyingArrayType) * value.size() * static_cast(cols)); + return array; + } +}; + static int OnError(int status, const char *func_name, const char *err_msg, const char *file_name, int line, void *userdata) { PyGILState_STATE gstate; diff --git a/modules/python/test/test_legacy.py b/modules/python/test/test_legacy.py index ab0a8bdc357e..e550ab73c8b3 100644 --- a/modules/python/test/test_legacy.py +++ b/modules/python/test/test_legacy.py @@ -20,8 +20,13 @@ def test_imencode(self): flag, ajpg = cv.imencode("img_q90.jpg", a, [cv.IMWRITE_JPEG_QUALITY, 90]) self.assertEqual(flag, True) self.assertEqual(ajpg.dtype, np.uint8) - self.assertGreater(ajpg.shape[0], 1) - self.assertEqual(ajpg.shape[1], 1) + self.assertTrue(isinstance(ajpg, np.ndarray), "imencode returned buffer of wrong type: {}".format(type(ajpg))) + self.assertEqual(len(ajpg.shape), 1, "imencode returned buffer with wrong shape: {}".format(ajpg.shape)) + self.assertGreaterEqual(len(ajpg), 1, "imencode length of the returned buffer should be at least 1") + self.assertLessEqual( + len(ajpg), a.size, + "imencode length of the returned buffer shouldn't exceed number of elements in original image" + ) def test_projectPoints(self): objpt = np.float64([[1,2,3]]) diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index 4d435a46b668..c992c9450d88 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -480,6 +480,109 @@ def test_reserved_keywords_are_transformed(self): cv.utils.testReservedKeywordConversion(20, lambda_=-4, from_=12), format_str.format(20, -4, 12) ) + def test_parse_vector_int_convertible(self): + np.random.seed(123098765) + try_to_convert = partial(self._try_to_convert, cv.utils.dumpVectorOfInt) + arr = np.random.randint(-20, 20, 40).astype(np.int32).reshape(10, 2, 2) + int_min, int_max = get_limits(ctypes.c_int) + for convertible in ((int_min, 1, 2, 3, int_max), [40, 50], tuple(), + np.array([int_min, -10, 24, int_max], dtype=np.int32), + np.array([10, 230, 12], dtype=np.uint8), arr[:, 0, 1],): + expected = "[" + ", ".join(map(str, convertible)) + "]" + actual = try_to_convert(convertible) + self.assertEqual(expected, actual, + msg=get_conversion_error_msg(convertible, expected, actual)) + + def test_parse_vector_int_not_convertible(self): + np.random.seed(123098765) + arr = np.random.randint(-20, 20, 40).astype(np.float).reshape(10, 2, 2) + int_min, int_max = get_limits(ctypes.c_int) + test_dict = {1: 2, 3: 10, 10: 20} + for not_convertible in ((int_min, 1, 2.5, 3, int_max), [True, 50], 'test', test_dict, + reversed([1, 2, 3]), + np.array([int_min, -10, 24, [1, 2]], dtype=np.object), + np.array([[1, 2], [3, 4]]), arr[:, 0, 1],): + with self.assertRaises(TypeError, msg=get_no_exception_msg(not_convertible)): + _ = cv.utils.dumpVectorOfInt(not_convertible) + + def test_parse_vector_double_convertible(self): + np.random.seed(1230965) + try_to_convert = partial(self._try_to_convert, cv.utils.dumpVectorOfDouble) + arr = np.random.randint(-20, 20, 40).astype(np.int32).reshape(10, 2, 2) + for convertible in ((1, 2.12, 3.5), [40, 50], tuple(), + np.array([-10, 24], dtype=np.int32), + np.array([-12.5, 1.4], dtype=np.double), + np.array([10, 230, 12], dtype=np.float), arr[:, 0, 1], ): + expected = "[" + ", ".join(map(lambda v: "{:.2f}".format(v), convertible)) + "]" + actual = try_to_convert(convertible) + self.assertEqual(expected, actual, + msg=get_conversion_error_msg(convertible, expected, actual)) + + def test_parse_vector_double_not_convertible(self): + test_dict = {1: 2, 3: 10, 10: 20} + for not_convertible in (('t', 'e', 's', 't'), [True, 50.55], 'test', test_dict, + np.array([-10.1, 24.5, [1, 2]], dtype=np.object), + np.array([[1, 2], [3, 4]]),): + with self.assertRaises(TypeError, msg=get_no_exception_msg(not_convertible)): + _ = cv.utils.dumpVectorOfDouble(not_convertible) + + def test_parse_vector_rect_convertible(self): + np.random.seed(1238765) + try_to_convert = partial(self._try_to_convert, cv.utils.dumpVectorOfRect) + arr_of_rect_int32 = np.random.randint(5, 20, 4 * 3).astype(np.int32).reshape(3, 4) + arr_of_rect_cast = np.random.randint(10, 40, 4 * 5).astype(np.uint8).reshape(5, 4) + for convertible in (((1, 2, 3, 4), (10, -20, 30, 10)), arr_of_rect_int32, arr_of_rect_cast, + arr_of_rect_int32.astype(np.int8), [[5, 3, 1, 4]], + ((np.int8(4), np.uint8(10), np.int(32), np.int16(55)),)): + expected = "[" + ", ".join(map(lambda v: "[x={}, y={}, w={}, h={}]".format(*v), convertible)) + "]" + actual = try_to_convert(convertible) + self.assertEqual(expected, actual, + msg=get_conversion_error_msg(convertible, expected, actual)) + + def test_parse_vector_rect_not_convertible(self): + np.random.seed(1238765) + arr = np.random.randint(5, 20, 4 * 3).astype(np.float).reshape(3, 4) + for not_convertible in (((1, 2, 3, 4), (10.5, -20, 30.1, 10)), arr, + [[5, 3, 1, 4], []], + ((np.float(4), np.uint8(10), np.int(32), np.int16(55)),)): + with self.assertRaises(TypeError, msg=get_no_exception_msg(not_convertible)): + _ = cv.utils.dumpVectorOfRect(not_convertible) + + def test_vector_general_return(self): + expected_number_of_mats = 5 + expected_shape = (10, 10, 3) + expected_type = np.uint8 + mats = cv.utils.generateVectorOfMat(5, 10, 10, cv.CV_8UC3) + self.assertTrue(isinstance(mats, tuple), + "Vector of Mats objects should be returned as tuple. Got: {}".format(type(mats))) + self.assertEqual(len(mats), expected_number_of_mats, "Returned array has wrong length") + for mat in mats: + self.assertEqual(mat.shape, expected_shape, "Returned Mat has wrong shape") + self.assertEqual(mat.dtype, expected_type, "Returned Mat has wrong elements type") + empty_mats = cv.utils.generateVectorOfMat(0, 10, 10, cv.CV_32FC1) + self.assertTrue(isinstance(empty_mats, tuple), + "Empty vector should be returned as empty tuple. Got: {}".format(type(mats))) + self.assertEqual(len(empty_mats), 0, "Vector of size 0 should be returned as tuple of length 0") + + def test_vector_fast_return(self): + expected_shape = (5, 4) + rects = cv.utils.generateVectorOfRect(expected_shape[0]) + self.assertTrue(isinstance(rects, np.ndarray), + "Vector of rectangles should be returned as numpy array. Got: {}".format(type(rects))) + self.assertEqual(rects.dtype, np.int32, "Vector of rectangles has wrong elements type") + self.assertEqual(rects.shape, expected_shape, "Vector of rectangles has wrong shape") + empty_rects = cv.utils.generateVectorOfRect(0) + self.assertTrue(isinstance(empty_rects, tuple), + "Empty vector should be returned as empty tuple. Got: {}".format(type(empty_rects))) + self.assertEqual(len(empty_rects), 0, "Vector of size 0 should be returned as tuple of length 0") + + expected_shape = (10,) + ints = cv.utils.generateVectorOfInt(expected_shape[0]) + self.assertTrue(isinstance(ints, np.ndarray), + "Vector of integers should be returned as numpy array. Got: {}".format(type(ints))) + self.assertEqual(ints.dtype, np.int32, "Vector of integers has wrong elements type") + self.assertEqual(ints.shape, expected_shape, "Vector of integers has wrong shape.") + class SamplesFindFile(NewOpenCVTests): From edc442afdb559c7a8d731b86637839b5f224de35 Mon Sep 17 00:00:00 2001 From: WJJ1995 Date: Wed, 1 Sep 2021 18:10:05 +0800 Subject: [PATCH 178/376] Merge pull request #20511 from wjj19950828:add_humanseg_support_0806 * support PPSeg model for dnn module * fixed README for CI * add test case * fixed bug * deal with comments * rm dnn_model_runner * update test case * fixed bug for testcase * update testcase --- modules/dnn/src/onnx/onnx_importer.cpp | 2 +- modules/dnn/test/test_onnx_importer.cpp | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 6da2c5edf6b1..41ff6c9b1eb4 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -2054,7 +2054,7 @@ void ONNXImporter::parseResize(LayerParams& layerParams, const opencv_onnx::Node layerParams.set("align_corners", interp_mode == "align_corners"); if (layerParams.get("mode") == "linear") { - layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? + layerParams.set("mode", interp_mode == "pytorch_half_pixel" || interp_mode == "half_pixel" ? "opencv_linear" : "bilinear"); } } diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 983f72d6d688..07a0290b9bfb 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -552,6 +552,11 @@ TEST_P(Test_ONNX_layers, DynamicResize) testONNXModels("dynamic_resize_scale_11", npy, 0, 0, false, true, 2); } +TEST_P(Test_ONNX_layers, Resize_HumanSeg) +{ + testONNXModels("resize_humanseg"); +} + TEST_P(Test_ONNX_layers, Div) { const String model = _tf("models/div.onnx"); From 390957fec4601b58982c3d15130485f531d22ba5 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Thu, 2 Sep 2021 10:32:17 +0300 Subject: [PATCH 179/376] fix: NumPy array allocation error message in vector conversion --- modules/python/src2/cv2.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index ff57978dc52b..67c416679950 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -1778,7 +1778,7 @@ struct pyopencvVecConverter const NPY_TYPES target_type = asNumpyType(); const int cols = DType::channels; - PyObject* array; + PyObject* array = NULL; if (cols == 1) { npy_intp dims = static_cast(value.size()); @@ -1796,14 +1796,16 @@ struct pyopencvVecConverter String shape; if (cols > 1) { - shape = cv::format("(%d x %d)", static_cast(value.size()), cols); + shape = format("(%d x %d)", static_cast(value.size()), cols); } else { - shape = cv::format("(%d)", static_cast(value.size())); + shape = format("(%d)", static_cast(value.size())); } - CV_Error_(Error::StsError, ("Can't allocate NumPy array for vector with dtype=%d shape=%s", - static_cast(target_type), static_cast(value.size()), shape.c_str())); + const String error_message = format("Can't allocate NumPy array for vector with dtype=%d and shape=%s", + static_cast(target_type), shape.c_str()); + emit_failmsg(PyExc_MemoryError, error_message.c_str()); + return array; } // Fill the array PyArrayObject* array_obj = reinterpret_cast(array); From d0e612dc368d657ca56101879dbc58c46a4dbb9b Mon Sep 17 00:00:00 2001 From: rogday Date: Fri, 3 Sep 2021 15:32:29 +0300 Subject: [PATCH 180/376] Merge pull request #20647 from rogday:resize_concat_optimization Fix resize+concat optimization * fix resize+concat optimization * add comment and fix indentation --- modules/dnn/src/layers/resize_layer.cpp | 7 +++++++ modules/dnn/test/test_tf_importer.cpp | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/modules/dnn/src/layers/resize_layer.cpp b/modules/dnn/src/layers/resize_layer.cpp index 40c7351984e0..8b7d802ab2eb 100644 --- a/modules/dnn/src/layers/resize_layer.cpp +++ b/modules/dnn/src/layers/resize_layer.cpp @@ -111,7 +111,14 @@ class ResizeLayerImpl : public ResizeLayer internals_arr.getMatVector(internals); if (outHeight == inputs[0].size[2] && outWidth == inputs[0].size[3]) + { + // outputs[0] = inputs[0] doesn't work due to BlobManager optimizations + if (inputs[0].data != outputs[0].data) + { + inputs[0].copyTo(outputs[0]); + } return; + } Mat& inp = inputs[0]; Mat& out = outputs[0]; diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 68d6e88a6642..1a2b976eb877 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -1125,6 +1125,11 @@ TEST_P(Test_TensorFlow_layers, resize_bilinear_down) runTensorFlowNet("resize_bilinear_down"); } +TEST_P(Test_TensorFlow_layers, resize_concat_optimization) +{ + runTensorFlowNet("resize_concat_optimization"); +} + TEST_P(Test_TensorFlow_layers, tf2_dense) { runTensorFlowNet("tf2_dense"); From 407adc7061c9d2126a5d27c53ff76b56a705f3e2 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 4 Sep 2021 04:35:00 +0000 Subject: [PATCH 181/376] dnn(ocl): fix buffer offsets in IDLF kernel - drop CreateSubBuffer - fix FUSED_CONV_ELTWISE mode --- modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp | 2 +- .../src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 86 +++++-------------- modules/dnn/src/opencl/conv_layer_spatial.cl | 25 ++++-- 3 files changed, 41 insertions(+), 72 deletions(-) diff --git a/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp b/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp index 7bb277d1020c..d6fb83becb92 100644 --- a/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp +++ b/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp @@ -269,7 +269,7 @@ class OCL4DNNConvSpatial void generate_idlf_tuneritems(std::vector< cv::Ptr > &tunerItems, int blockM, int blockK, int simd_size); void setFusionDefine(ocl4dnnFusedActiv_t fused_activ, bool fused_eltwise); - void setFusionArg(ocl4dnnFusedActiv_t fused_activ, bool fused_eltwise, ocl::Kernel &kernel, cl_uint &argIdx); + void setFusionArg(ocl4dnnFusedActiv_t fused_activ, bool fused_eltwise, int fused_eltwise_offset, ocl::Kernel &kernel, cl_uint &argIdx); int32_t group_; bool bias_term_; diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index 517a663e46a2..b4477ebfc4f0 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -270,17 +270,21 @@ void OCL4DNNConvSpatial::setFusionDefine(ocl4dnnFusedActiv_t fused_activ, } template -void OCL4DNNConvSpatial::setFusionArg(ocl4dnnFusedActiv_t fused_activ, bool fused_eltwise, ocl::Kernel &kernel, cl_uint &argIdx) +void OCL4DNNConvSpatial::setFusionArg(ocl4dnnFusedActiv_t fused_activ, bool fused_eltwise, int fused_eltwise_offset, ocl::Kernel &kernel, cl_uint &argIdx) { if (fused_eltwise) - kernel.set(argIdx++, (cl_mem)bottom_data2_.handle(ACCESS_READ)); + { + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom_data2_)); + if (fused_eltwise_offset >= 0) + kernel.set(argIdx++, fused_eltwise_offset); + } switch (fused_activ) { case OCL4DNN_CONV_FUSED_ACTIV_RELU: kernel.set(argIdx++, (float)negative_slope_); break; case OCL4DNN_CONV_FUSED_ACTIV_PRELU: - kernel.set(argIdx++, (cl_mem)negative_slope_umat_.handle(ACCESS_READ)); + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(negative_slope_umat_)); break; case OCL4DNN_CONV_FUSED_ACTIV_POWER: kernel.set(argIdx++, (float)power_); @@ -895,10 +899,12 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, if (config->kernelType == KERNEL_TYPE_INTEL_IDLF) { if (!swizzleWeight(weight, config->workItem_output[2], false)) return false; +#if 0 size_t total_bottom_size = bottom_dim_ * numImages; size_t total_kernel_size = kernel_h_ * kernel_w_ * channels_ * M_; size_t total_bias_size = M_ * group_; size_t total_top_size = top_dim_ * numImages; +#endif for (int32_t g = 0; g < group_; ++g) { bias_offset = M_ * g; int32_t image_offset = width_ * height_ * (channels_ / group_) * g; @@ -910,72 +916,22 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, return false; cl_uint argIdx = 0; - setFusionArg(fused_activ_, fused_eltwise_, kernel, argIdx); + setFusionArg(fused_activ_, fused_eltwise_, output_image_offset, kernel, argIdx); - UMat img_buffer; - if (image_offset) - { - CreateSubBuffer(bottom, img_buffer, image_offset, - total_bottom_size - image_offset, false); - if (img_buffer.empty()) - return false; + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); + kernel.set(argIdx++, image_offset); - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(img_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); - } - - UMat kernel_buffer; - if (kernel_offset) - { - CreateSubBuffer(swizzled_weights_umat, kernel_buffer, kernel_offset, - total_kernel_size - kernel_offset, false); - if (kernel_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(kernel_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(swizzled_weights_umat)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(swizzled_weights_umat)); + kernel.set(argIdx++, kernel_offset); - UMat bias_buffer; if (bias_term_) { - if (bias_offset) - { - CreateSubBuffer(bias, bias_buffer, bias_offset, - total_bias_size - bias_offset, false); - if (bias_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias)); + kernel.set(argIdx++, bias_offset); } - UMat out_buffer; - if (output_image_offset) - { - CreateSubBuffer(top, out_buffer, output_image_offset, - total_top_size - output_image_offset, true); - if (out_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(out_buffer)); - kernel.set(argIdx++, (int)(out_buffer.offset / element_size)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(top)); - kernel.set(argIdx++, (int)(top.offset / element_size)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(top)); + kernel.set(argIdx++, (int)(top.offset / element_size) + output_image_offset); kernel.set(argIdx++, (uint16_t)width_); kernel.set(argIdx++, (uint16_t)height_); @@ -1005,7 +961,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, return false; cl_uint argIdx = 0; - setFusionArg(fused_activ_, fused_eltwise_, kernel, argIdx); + setFusionArg(fused_activ_, fused_eltwise_, -1, kernel, argIdx); UMat img_buffer; if (image_offset) @@ -1112,7 +1068,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, return false; cl_uint argIdx = 0; - setFusionArg(fused_activ_, fused_eltwise_, kernel, argIdx); + setFusionArg(fused_activ_, fused_eltwise_, -1, kernel, argIdx); kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(weight)); if (bias_term_) @@ -1152,7 +1108,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, return false; cl_uint argIdx = 0; - setFusionArg(fused_activ_, fused_eltwise_, kernel, argIdx); + setFusionArg(fused_activ_, fused_eltwise_, -1, kernel, argIdx); kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); kernel.set(argIdx++, image_offset); kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(weight)); diff --git a/modules/dnn/src/opencl/conv_layer_spatial.cl b/modules/dnn/src/opencl/conv_layer_spatial.cl index 236e8d029af7..55015557a0dc 100644 --- a/modules/dnn/src/opencl/conv_layer_spatial.cl +++ b/modules/dnn/src/opencl/conv_layer_spatial.cl @@ -74,18 +74,22 @@ (_dst_)[(_offset_)] = ACTIVATION_RELU_FUNCTION(_x_, _channel_); \ } while(0) #define ELTWISE_DATA_ARG __global Dtype* eltwise_data, +#define ELTWISE_DATA_ARG_WITH_OFFSET __global Dtype* eltwise_ptr, int eltwise_offset, #else #define ACTIVATION_FUNCTION(_dst_, _offset_, _data_, _channel_) do { \ const Dtype _x_ = (_data_); \ (_dst_)[(_offset_)] = ACTIVATION_RELU_FUNCTION(_x_, _channel_); \ } while(0) #define ELTWISE_DATA_ARG +#define ELTWISE_DATA_ARG_WITH_OFFSET #endif #if APPLY_BIAS #define BIAS_KERNEL_ARG __global Dtype * biases_base, +#define BIAS_KERNEL_ARG_WITH_OFFSET __global Dtype * biases_base_ptr, int biases_base_offset, #else #define BIAS_KERNEL_ARG +#define BIAS_KERNEL_ARG_WITH_OFFSET #endif #define __CAT(x, y) x##y @@ -223,19 +227,28 @@ __attribute__((reqd_work_group_size(1, 1, SIMD_SIZE))) __attribute__((intel_reqd_sub_group_size(SIMD_SIZE))) __kernel void convolve_simd( - ELTWISE_DATA_ARG + ELTWISE_DATA_ARG_WITH_OFFSET FUSED_ARG - __global Dtype* inputs, - __global Dtype* weights, - BIAS_KERNEL_ARG - __global Dtype* outputs_base, - const int outputs_offset, + __global Dtype* inputs_ptr, const int inputs_offset, + __global Dtype* weights_ptr, const int weights_offset, + BIAS_KERNEL_ARG_WITH_OFFSET + __global Dtype* outputs_base, const int outputs_offset, const ushort input_width, const ushort input_height, const ushort output_width, const ushort output_height) { + __global Dtype* inputs = inputs_ptr + inputs_offset; + __global Dtype* weights = weights_ptr + weights_offset; +#if APPLY_BIAS + __global Dtype* biases_base = biases_base_ptr + biases_base_offset; +#endif + __global Dtype* outputs = outputs_base + outputs_offset; +#ifdef FUSED_CONV_ELTWISE + __global Dtype* eltwise_data = eltwise_ptr + eltwise_offset; +#endif + unsigned int oc = get_global_id(0) * OUT_BLOCK_WIDTH; // oc = Output Column unsigned int or = get_global_id(1) * OUT_BLOCK_HEIGHT; // or = Output Row unsigned int fm = get_global_id(2); // fm = Feature Map = od = Output Depth From aaff1256088ce892f977c73ff01817c2e389ac93 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 4 Sep 2021 01:34:02 +0000 Subject: [PATCH 182/376] core(ocl): debug capabilities --- modules/core/src/ocl.cpp | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index 0c14e7f3e073..daf4fcd28015 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -76,8 +76,11 @@ #undef CV__ALLOCATOR_STATS_LOG #define CV_OPENCL_ALWAYS_SHOW_BUILD_LOG 0 +#define CV_OPENCL_SHOW_BUILD_OPTIONS 0 +#define CV_OPENCL_SHOW_BUILD_KERNELS 0 #define CV_OPENCL_SHOW_RUN_KERNELS 0 +#define CV_OPENCL_SYNC_RUN_KERNELS 0 #define CV_OPENCL_TRACE_CHECK 0 #define CV_OPENCL_VALIDATE_BINARY_PROGRAMS 1 @@ -2983,6 +2986,8 @@ static cv::String dumpValue(size_t sz, const void* p) { if (!p) return "NULL"; + if (sz == 2) + return cv::format("%d / %uu / 0x%04x", *(short*)p, *(unsigned short*)p, *(short*)p); if (sz == 4) return cv::format("%d / %uu / 0x%08x / %g", *(int*)p, *(int*)p, *(int*)p, *(float*)p); if (sz == 8) @@ -3195,6 +3200,10 @@ bool Kernel::Impl::run(int dims, size_t globalsize[], size_t localsize[], return false; // OpenCV 5.0: raise error } +#if CV_OPENCL_SYNC_RUN_KERNELS + sync = true; +#endif + cl_command_queue qq = getQueue(q); if (haveTempDstUMats) sync = true; @@ -3625,7 +3634,28 @@ struct Program::Impl if (!param_buildExtraOptions.empty()) buildflags = joinBuildOptions(buildflags, param_buildExtraOptions); } +#if CV_OPENCL_SHOW_BUILD_OPTIONS + CV_LOG_INFO(NULL, "OpenCL program '" << sourceModule_ << "/" << sourceName_ << "' options:" << buildflags); +#endif compile(ctx, src_, errmsg); +#if CV_OPENCL_SHOW_BUILD_KERNELS + if (handle) + { + size_t retsz = 0; + char kernels_buffer[4096] = {0}; + cl_int result = clGetProgramInfo(handle, CL_PROGRAM_KERNEL_NAMES, sizeof(kernels_buffer), &kernels_buffer[0], &retsz); + CV_OCL_DBG_CHECK_RESULT(result, cv::format("clGetProgramInfo(CL_PROGRAM_KERNEL_NAMES: %s/%s)", sourceModule_.c_str(), sourceName_.c_str()).c_str()); + if (result == CL_SUCCESS && retsz < sizeof(kernels_buffer)) + { + kernels_buffer[retsz] = 0; + CV_LOG_INFO(NULL, "OpenCL program '" << sourceModule_ << "/" << sourceName_ << "' kernels: '" << kernels_buffer << "'"); + } + else + { + CV_LOG_ERROR(NULL, "OpenCL program '" << sourceModule_ << "/" << sourceName_ << "' can't retrieve kernel names!"); + } + } +#endif } bool compile(const Context& ctx, const ProgramSource::Impl* src_, String& errmsg) @@ -3857,7 +3887,6 @@ struct Program::Impl CV_LOG_INFO(NULL, result << ": Kernels='" << kernels_buffer << "'"); } #endif - } return handle != NULL; } From 5b2c0168340a8b4e67def8f52885ab21b4bbcadc Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 3 Sep 2021 02:38:53 +0000 Subject: [PATCH 183/376] dnn(ocl): avoid out of buffer access in copyWeightsSwizzled --- modules/dnn/src/opencl/conv_spatial_helper.cl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/dnn/src/opencl/conv_spatial_helper.cl b/modules/dnn/src/opencl/conv_spatial_helper.cl index 33d9db57c8ae..660d085956ff 100644 --- a/modules/dnn/src/opencl/conv_spatial_helper.cl +++ b/modules/dnn/src/opencl/conv_spatial_helper.cl @@ -62,8 +62,8 @@ __kernel void TEMPLATE(copyWeightsSwizzled, Dtype) //Original location //Output location - int outputSublayer = channels / swizzleFactor; - int outputSublayerIndex = channels % swizzleFactor; + //int outputSublayer = channels / swizzleFactor; + //int outputSublayerIndex = channels % swizzleFactor; int filter = sX / (kernel_w*kernel_h*channels); int kernel_X = sX % kernel_w; @@ -73,6 +73,10 @@ __kernel void TEMPLATE(copyWeightsSwizzled, Dtype) int FP = filter / swizzleFactor; int F1 = filter % swizzleFactor; - weightOut[FP*(kernel_w*kernel_h*channels*swizzleFactor) + kernel_C*(kernel_w*kernel_h*swizzleFactor) + kernel_Y*(kernel_w*swizzleFactor) + kernel_X*swizzleFactor + F1] - = weightIn[filter*(kernel_w*kernel_h*channels) + kernel_C*(kernel_w*kernel_h) + kernel_Y*kernel_w + kernel_X]; + int idxOut = FP*(kernel_w*kernel_h*channels*swizzleFactor) + kernel_C*(kernel_w*kernel_h*swizzleFactor) + kernel_Y*(kernel_w*swizzleFactor) + kernel_X*swizzleFactor + F1; + int idxIn = filter*(kernel_w*kernel_h*channels) + kernel_C*(kernel_w*kernel_h) + kernel_Y*kernel_w + kernel_X; + + // idxIn is not valid if (filter >= outputs) - no data for these elements. Output alignment gaps are filled by zeros + Dtype v = (filter < outputs) ? weightIn[idxIn] : (Dtype)0; + weightOut[idxOut] = v; } From 5578ad5e14fe1fcf4d7171f74c665db2a578187f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 4 Sep 2021 16:27:51 +0000 Subject: [PATCH 184/376] dnn(ocl): fix automatic globalsize adjusting - if kernel code doesn't support that --- modules/core/include/opencv2/core/ocl.hpp | 20 +++++++++++++++++-- modules/core/src/ocl.cpp | 8 ++++++++ modules/dnn/src/layers/batch_norm_layer.cpp | 2 +- modules/dnn/src/layers/mvn_layer.cpp | 2 +- modules/dnn/src/layers/slice_layer.cpp | 2 +- .../dnn/src/ocl4dnn/src/math_functions.cpp | 1 + .../src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 19 +++++++++--------- .../dnn/src/ocl4dnn/src/ocl4dnn_softmax.cpp | 2 +- modules/dnn/src/opencl/gemm_image.cl | 10 ++++++++++ 9 files changed, 50 insertions(+), 16 deletions(-) diff --git a/modules/core/include/opencv2/core/ocl.hpp b/modules/core/include/opencv2/core/ocl.hpp index f03de180fc32..b51b39359d59 100644 --- a/modules/core/include/opencv2/core/ocl.hpp +++ b/modules/core/include/opencv2/core/ocl.hpp @@ -562,7 +562,9 @@ class CV_EXPORTS Kernel i = set(i, a6); i = set(i, a7); i = set(i, a8); i = set(i, a9); i = set(i, a10); i = set(i, a11); i = set(i, a12); i = set(i, a13); i = set(i, a14); set(i, a15); return *this; } - /** @brief Run the OpenCL kernel. + + /** @brief Run the OpenCL kernel (globalsize value may be adjusted) + @param dims the work problem dimensions. It is the length of globalsize and localsize. It can be either 1, 2 or 3. @param globalsize work items for each dimension. It is not the final globalsize passed to OpenCL. Each dimension will be adjusted to the nearest integer divisible by the corresponding @@ -571,12 +573,26 @@ class CV_EXPORTS Kernel @param localsize work-group size for each dimension. @param sync specify whether to wait for OpenCL computation to finish before return. @param q command queue + + @note Use run_() if your kernel code doesn't support adjusted globalsize. */ bool run(int dims, size_t globalsize[], size_t localsize[], bool sync, const Queue& q=Queue()); + + /** @brief Run the OpenCL kernel + * + * @param dims the work problem dimensions. It is the length of globalsize and localsize. It can be either 1, 2 or 3. + * @param globalsize work items for each dimension. This value is passed to OpenCL without changes. + * @param localsize work-group size for each dimension. + * @param sync specify whether to wait for OpenCL computation to finish before return. + * @param q command queue + */ + bool run_(int dims, size_t globalsize[], size_t localsize[], bool sync, const Queue& q=Queue()); + bool runTask(bool sync, const Queue& q=Queue()); - /** @brief Similar to synchronized run() call with returning of kernel execution time + /** @brief Similar to synchronized run_() call with returning of kernel execution time + * * Separate OpenCL command queue may be used (with CL_QUEUE_PROFILING_ENABLE) * @return Execution time in nanoseconds or negative number on error */ diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index daf4fcd28015..a550c1d91ae0 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -3160,6 +3160,14 @@ bool Kernel::run(int dims, size_t _globalsize[], size_t _localsize[], } +bool Kernel::run_(int dims, size_t _globalsize[], size_t _localsize[], + bool sync, const Queue& q) +{ + CV_Assert(p); + return p->run(dims, _globalsize, _localsize, sync, NULL, q); +} + + static bool isRaiseErrorOnReuseAsyncKernel() { static bool initialized = false; diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index 42676c79386f..dcb400597560 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -231,7 +231,7 @@ class BatchNormLayerImpl CV_FINAL : public BatchNormLayer kernel.set(4, ocl::KernelArg::PtrReadOnly(umat_weight)); kernel.set(5, ocl::KernelArg::PtrReadOnly(umat_bias)); kernel.set(6, ocl::KernelArg::PtrWriteOnly(dst)); - bool ret = kernel.run(2, global, NULL, false); + bool ret = kernel.run_(2, global, NULL, false); if (!ret) return false; } diff --git a/modules/dnn/src/layers/mvn_layer.cpp b/modules/dnn/src/layers/mvn_layer.cpp index 8f06216df1e4..de2b0d569041 100644 --- a/modules/dnn/src/layers/mvn_layer.cpp +++ b/modules/dnn/src/layers/mvn_layer.cpp @@ -191,7 +191,7 @@ class MVNLayerImpl CV_FINAL : public MVNLayer k1.set(argId++, ocl::KernelArg::PtrReadOnly(bnorm_weight)); k1.set(argId++, ocl::KernelArg::PtrReadOnly(bnorm_bias)); k1.set(argId++, ocl::KernelArg::PtrWriteOnly(outMat)); - ret = k1.run(1, globalsize, localsize, false); + ret = k1.run_(1, globalsize, localsize, false); if (!ret) return false; } diff --git a/modules/dnn/src/layers/slice_layer.cpp b/modules/dnn/src/layers/slice_layer.cpp index 507964edf980..16f1958879ee 100644 --- a/modules/dnn/src/layers/slice_layer.cpp +++ b/modules/dnn/src/layers/slice_layer.cpp @@ -482,7 +482,7 @@ class SliceLayerImpl : public SliceLayer ocl::KernelArg::PtrReadOnly(input), ocl::KernelArg::PtrWriteOnly(output) ) - .run(2, (size_t*)ocl.global_size, (size_t*)ocl.local_size, false); + .run_(2, (size_t*)ocl.global_size, (size_t*)ocl.local_size, false); if (!ret) return false; } // for outputs.size() diff --git a/modules/dnn/src/ocl4dnn/src/math_functions.cpp b/modules/dnn/src/ocl4dnn/src/math_functions.cpp index 855a21e08ffe..c924d66b12b4 100644 --- a/modules/dnn/src/ocl4dnn/src/math_functions.cpp +++ b/modules/dnn/src/ocl4dnn/src/math_functions.cpp @@ -116,6 +116,7 @@ ocl::Image2D ocl4dnnGEMMCopyBufferToImage(UMat buffer, int offset, .args( ocl::KernelArg::PtrReadOnly(buffer), image, offset, + padded_width, padded_height, width, height, ld) .run(2, global_copy, NULL, false); diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index b4477ebfc4f0..3b73da801c9f 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -769,12 +769,11 @@ bool OCL4DNNConvSpatial::swizzleWeight(const UMat &weight, swizzled_factor ); - size_t global_work_size_copy[3] = { - (size_t) (alignSize(num_output_, swizzled_factor) * channels * kernel_w_ * kernel_h_), 1, 1 }; + size_t global_work_size_copy[1] = { (size_t)(alignSize(num_output_, swizzled_factor) * channels * kernel_w_ * kernel_h_) }; - if (!oclk_copy_weight.run(3, global_work_size_copy, NULL, false)) + if (!oclk_copy_weight.run_(1, global_work_size_copy, NULL, false)) { - std::cout << "Swizzle kernel run failed." << std::endl; + CV_LOG_ERROR(NULL, "DNN/OpenCL: Swizzle kernel run failed"); return false; } } else { @@ -937,7 +936,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, kernel.set(argIdx++, (uint16_t)height_); kernel.set(argIdx++, (uint16_t)output_w_); kernel.set(argIdx++, (uint16_t)output_h_); - if (!kernel.run(3, config->global_work_size, config->local_work_size, false)) + if (!kernel.run_(3, config->global_work_size, config->local_work_size, false)) { std::cout << "IDLF kernel run failed." << std::endl; return false; @@ -1056,7 +1055,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, gy = alignSize(gy, blockK); size_t global_size[3] = { gx, gy, config->global_work_size[2] }; - if (!kernel.run(3, global_size, config->local_work_size, false)) + if (!kernel.run_(3, global_size, config->local_work_size, false)) { std::cout << "GEMM like kernel run failed." << std::endl; return false; @@ -1085,9 +1084,9 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, global_size[1] = output_h_; global_size[2] = num_output_ * num_; - if (!kernel.run(3, global_size, NULL, false)) + if (!kernel.run_(3, global_size, NULL, false)) { - std::cout << "DWCONV kernel run failed." << std::endl; + CV_LOG_ERROR(NULL, "DNN/OpenCL: DWCONV kernel run failed"); return false; } } else { @@ -1127,11 +1126,11 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, kernel.set(argIdx++, (uint16_t)output_h_); kernel.set(argIdx++, (uint16_t)pad_w_); kernel.set(argIdx++, (uint16_t)pad_h_); - if (!kernel.run(3, config->global_work_size, + if (!kernel.run_(3, config->global_work_size, (config->use_null_local) ? NULL : config->local_work_size, false)) { - std::cout << "Basic kernel run failed." << std::endl; + CV_LOG_ERROR(NULL, "DNN/OpenCL: Basic kernel run failed"); return false; } } diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_softmax.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_softmax.cpp index 78576711a713..7b32189fdc77 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_softmax.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_softmax.cpp @@ -127,7 +127,7 @@ bool OCL4DNNSoftmax::Forward(const UMat& bottom, UMat& top) oclk_softmax_forward_kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); oclk_softmax_forward_kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(top)); } - ret = oclk_softmax_forward_kernel.run(3, global_size, local_size, false); + ret = oclk_softmax_forward_kernel.run_(3, global_size, local_size, false); } return ret; } diff --git a/modules/dnn/src/opencl/gemm_image.cl b/modules/dnn/src/opencl/gemm_image.cl index 710637a09326..f6e0020d82c4 100644 --- a/modules/dnn/src/opencl/gemm_image.cl +++ b/modules/dnn/src/opencl/gemm_image.cl @@ -954,6 +954,10 @@ __kernel void TEMPLATE(gemm_buffer_copy_image_transpose, Dtype)( { const int gidx = get_global_id(0); const int gidy = get_global_id(1); + + if (gidx >= width || gidy >= height) + return; + int2 coord_dst = (int2)(gidx, gidy); __global Dtype* A_off = A + offA; Dtype srcA = A_off[gidy * ldA + gidx]; @@ -968,12 +972,18 @@ __kernel void TEMPLATE(gemm_buffer_copy_image_no_transpose, Dtype)( __global Dtype* A, __write_only image2d_t ImA, int offA, + int padded_width, + int padded_height, int width, int height, int ldA) { const int gidx = get_global_id(0); const int gidy = get_global_id(1); + + if (gidx >= padded_width || gidy >= padded_height) + return; + int2 coord_dst = (int2)(gidx, gidy); #if TYPE == TYPE_HALF if (gidx >= width || gidy >= height) { From 36cc43170d9e1e244d72c25d69cef8d716599a10 Mon Sep 17 00:00:00 2001 From: Zhuo Zhang Date: Mon, 6 Sep 2021 12:03:59 +0800 Subject: [PATCH 185/376] docs: fix image path for py_fast doc --- doc/py_tutorials/py_feature2d/py_fast/py_fast.markdown | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/py_tutorials/py_feature2d/py_fast/py_fast.markdown b/doc/py_tutorials/py_feature2d/py_fast/py_fast.markdown index 4ce61370bd79..b1b8a81ca83c 100644 --- a/doc/py_tutorials/py_feature2d/py_fast/py_fast.markdown +++ b/doc/py_tutorials/py_feature2d/py_fast/py_fast.markdown @@ -98,7 +98,7 @@ import numpy as np import cv2 as cv from matplotlib import pyplot as plt -img = cv.imread('simple.jpg',0) +img = cv.imread('blox.jpg',0) # `/samples/data/blox.jpg` # Initiate FAST object with default values fast = cv.FastFeatureDetector_create() @@ -113,17 +113,17 @@ print( "nonmaxSuppression:{}".format(fast.getNonmaxSuppression()) ) print( "neighborhood: {}".format(fast.getType()) ) print( "Total Keypoints with nonmaxSuppression: {}".format(len(kp)) ) -cv.imwrite('fast_true.png',img2) +cv.imwrite('fast_true.png', img2) # Disable nonmaxSuppression fast.setNonmaxSuppression(0) -kp = fast.detect(img,None) +kp = fast.detect(img, None) print( "Total Keypoints without nonmaxSuppression: {}".format(len(kp)) ) img3 = cv.drawKeypoints(img, kp, None, color=(255,0,0)) -cv.imwrite('fast_false.png',img3) +cv.imwrite('fast_false.png', img3) @endcode See the results. First image shows FAST with nonmaxSuppression and second one without nonmaxSuppression: From 35e824c28710ca13b2770dc9d547bfc352293dc5 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 6 Sep 2021 05:51:37 +0300 Subject: [PATCH 186/376] dnn(ocl): fix out of bound access in GEMM-like kernels - dropped usage of CreateSubBuffer() - buffers lifetime management issue - fixed elementwise offset - avoid out of bounds read access --- .../src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 73 ++++------------- modules/dnn/src/opencl/conv_layer_spatial.cl | 79 ++++++++++++++++--- 2 files changed, 80 insertions(+), 72 deletions(-) diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index 3b73da801c9f..5eee1da4a004 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -945,9 +945,11 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, } else if (config->kernelType == KERNEL_TYPE_GEMM_LIKE) { if (!swizzleWeight(weight, config->workItem_output[1], true)) return false; +#if 0 size_t total_bottom_size = bottom_dim_ * numImages; size_t total_kernel_size = kernel_h_ * kernel_w_ * channels_ * M_; size_t total_bias_size = M_ * group_; +#endif size_t total_top_size = top_dim_ * numImages; for (int32_t g = 0; g < group_; ++g) { bias_offset = M_ * g; @@ -960,72 +962,25 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, return false; cl_uint argIdx = 0; - setFusionArg(fused_activ_, fused_eltwise_, -1, kernel, argIdx); - - UMat img_buffer; - if (image_offset) - { - CreateSubBuffer(bottom, img_buffer, image_offset, - total_bottom_size - image_offset, false); - if (img_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(img_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); - } + setFusionArg(fused_activ_, fused_eltwise_, output_image_offset, kernel, argIdx); - UMat kernel_buffer; - if (kernel_offset) - { - CreateSubBuffer(swizzled_weights_umat, kernel_buffer, kernel_offset, - total_kernel_size - kernel_offset, false); - if (kernel_buffer.empty()) - return false; + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); + kernel.set(argIdx++, (int)image_offset); + kernel.set(argIdx++, (int)(bottom.total() - image_offset)); - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(kernel_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(swizzled_weights_umat)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(swizzled_weights_umat)); + kernel.set(argIdx++, (int)kernel_offset); + kernel.set(argIdx++, (int)(swizzled_weights_umat.total() - kernel_offset)); - UMat bias_buffer; if (bias_term_) { - if (bias_offset) - { - CreateSubBuffer(bias, bias_buffer, bias_offset, - total_bias_size - bias_offset, false); - if (bias_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias)); + kernel.set(argIdx++, (int)bias_offset); } - UMat out_buffer; - if (output_image_offset) - { - CreateSubBuffer(top, out_buffer, output_image_offset, - total_top_size - output_image_offset, true); - if (out_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(out_buffer)); - kernel.set(argIdx++, (int)(out_buffer.offset / element_size)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(top)); - kernel.set(argIdx++, (int)(top.offset / element_size)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(top)); + kernel.set(argIdx++, (int)(top.offset / element_size) + output_image_offset); + kernel.set(argIdx++, (int)total_top_size - (int)(top.offset / element_size)); kernel.set(argIdx++, (uint16_t)width_); kernel.set(argIdx++, (uint16_t)height_); diff --git a/modules/dnn/src/opencl/conv_layer_spatial.cl b/modules/dnn/src/opencl/conv_layer_spatial.cl index 55015557a0dc..e7bbacd4c4c0 100644 --- a/modules/dnn/src/opencl/conv_layer_spatial.cl +++ b/modules/dnn/src/opencl/conv_layer_spatial.cl @@ -401,13 +401,12 @@ typedef struct half0 { half s0; } half0; //never used but makes compiler happy. #define ROW_PITCH input_width #define GEMM_LIKE_KERNEL_ARGS \ - ELTWISE_DATA_ARG \ + ELTWISE_DATA_ARG_WITH_OFFSET \ FUSED_ARG \ - const __global Dtype *src0, \ - const __global Dtype *src1, \ - BIAS_KERNEL_ARG \ - __global Dtype *dst_base, \ - const int dst_offset, \ + const __global Dtype *src0_ptr, const unsigned int src0_offset, const unsigned int src0_limit, \ + const __global Dtype *src1_ptr, const unsigned int src1_offset, const unsigned int src1_limit, \ + BIAS_KERNEL_ARG_WITH_OFFSET \ + __global Dtype *dst_base, const unsigned int dst_offset, const unsigned int dst_limit, \ const ushort input_width, \ const ushort input_height, \ const ushort output_width, \ @@ -437,7 +436,17 @@ typedef struct half0 { half s0; } half0; //never used but makes compiler happy. __attribute__((intel_reqd_sub_group_size(8))) __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) { + const __global Dtype *src0 = src0_ptr + src0_offset; + const __global Dtype *src1 = src1_ptr + src1_offset; +#if APPLY_BIAS + __global Dtype* biases_base = biases_base_ptr + biases_base_offset; +#endif + __global Dtype *dst = dst_base + dst_offset; +#ifdef FUSED_CONV_ELTWISE + __global Dtype* eltwise_data = eltwise_ptr + eltwise_offset; +#endif + const int group_x = get_group_id(0); const int group_y = get_group_id(1); const int global_x = get_global_id(0); @@ -460,6 +469,14 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) } typedef CAT( Dtype, KERNEL_WIDTH ) Dtype_t; +// U_GEMM_LIKE_CONV_k11x11_cn3_g1_s4x4_d1x1_b1_in240x240_p0x0_num1_M96_activ1_eltwise0_FP32_5_1_8_32_SIMD8 doesn't run properly (src0_read out of bounds) +// Test: DNNTestNetwork.AlexNet/0 (to run all kernels use OPENCV_OCL4DNN_FORCE_AUTO_TUNING=1) +#if 0 // INPUT_PAD_W == 0 && INPUT_PAD_H == 0 && DILATION_X == 1 && DILATION_Y == 1 && INPUT_PAD_BOTTOM == 0 && INPUT_PAD_RIGHT == 0 + #define OPTIMIZE_READ 1 +#else + #define OPTIMIZE_READ 0 +#endif + // True for all threads if filter_width is multiple of TILE_N // else, true for all but right-most column of threads. if( TILE_N_LAST == 0 || global_x < WIDTH1 / TILE_N ) @@ -476,7 +493,7 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) // atile is M rows x K columns. int curr_x = ( global_y % output_width ) * STRIDE_X; int curr_y = ( global_y / output_width ) * STRIDE_Y; -#if INPUT_PAD_H != 0 || INPUT_PAD_W != 0 || DILATION_X != 1 || DILATION_Y != 1 || INPUT_PAD_BOTTOM != 0 || INPUT_PAD_RIGHT != 0 +#if !OPTIMIZE_READ int saved_y = curr_y; #endif const __global Dtype *src0_read = src0 @@ -496,7 +513,7 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) do { int patch_row = 0; -#if INPUT_PAD_H != 0 || INPUT_PAD_W != 0 || DILATION_X != 1 || DILATION_Y != 1 || INPUT_PAD_BOTTOM != 0 || INPUT_PAD_RIGHT != 0 +#if !OPTIMIZE_READ curr_y = saved_y; #endif @@ -514,11 +531,17 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) // ... const bool kernel_width_is_odd = KERNEL_WIDTH % 2 == 1; -#if INPUT_PAD_W == 0 && INPUT_PAD_H == 0 && DILATION_X == 1 && DILATION_Y == 1 && INPUT_PAD_BOTTOM == 0 && INPUT_PAD_RIGHT == 0 +#if OPTIMIZE_READ #if KERNEL_WIDTH == 3 Dtype_t blockA00 = vload3(0, src0_read); Dtype* pblockA00 = (Dtype*)(&blockA00); #else + #if 0 // debug + if ((int)(src0_read - src0) >= src0_limit - KERNEL_WIDTH) + { + printf("CATCH: src0_read-src0: %d limit=%d curr_y,curr_x=%d,%d\n", (int)(src0_read - src0), src0_limit, curr_y, curr_x); + } + #endif Dtype_t blockA00 = ( (const __global Dtype_t*)src0_read )[ 0 ]; Dtype* pblockA00 = (Dtype*)(&blockA00); #endif @@ -639,7 +662,7 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) // atile is M rows x K columns. int curr_x = ( global_y % output_width ) * STRIDE_X; int curr_y = ( global_y / output_width ) * STRIDE_Y; -#if INPUT_PAD_H != 0 || INPUT_PAD_W != 0 || DILATION_X != 1 || DILATION_Y != 1 || INPUT_PAD_BOTTOM != 0 || INPUT_PAD_RIGHT != 0 +#if !OPTIMIZE_READ int saved_y = curr_y; #endif const __global Dtype *src0_read = src0 @@ -659,14 +682,14 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) do { int patch_row = 0; -#if INPUT_PAD_H != 0 || INPUT_PAD_W != 0 || DILATION_X != 1 || DILATION_Y != 1 || INPUT_PAD_BOTTOM != 0 || INPUT_PAD_RIGHT != 0 +#if !OPTIMIZE_READ curr_y = saved_y; #endif do { // Load atile and interleaved btile. const bool kernel_width_is_odd = KERNEL_WIDTH % 2 == 1; -#if INPUT_PAD_W == 0 && INPUT_PAD_H == 0 && DILATION_X == 1 && DILATION_Y == 1 && INPUT_PAD_BOTTOM == 0 && INPUT_PAD_RIGHT == 0 +#if OPTIMIZE_READ Dtype_t blockA00 = ( (const __global Dtype_t*)src0_read )[ 0 ]; Dtype* pblockA00 = (Dtype*)(&blockA00); #else @@ -803,7 +826,7 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) } } } -#endif +#endif // TILE_N_LAST > 0 } #endif #ifdef GEMM_LIKE_CONV_32_2 @@ -826,7 +849,17 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) __attribute__((intel_reqd_sub_group_size(8))) __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) { + const __global Dtype *src0 = src0_ptr + src0_offset; + const __global Dtype *src1 = src1_ptr + src1_offset; +#if APPLY_BIAS + __global Dtype* biases_base = biases_base_ptr + biases_base_offset; +#endif + __global Dtype *dst = dst_base + dst_offset; +#ifdef FUSED_CONV_ELTWISE + __global Dtype* eltwise_data = eltwise_ptr + eltwise_offset; +#endif + const int group_x = get_group_id(0); const int group_y = get_group_id(1); const int global_x = get_global_id(0); @@ -1388,7 +1421,17 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) __attribute__((intel_reqd_sub_group_size(16))) __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) { + const __global Dtype *src0 = src0_ptr + src0_offset; + const __global Dtype *src1 = src1_ptr + src1_offset; +#if APPLY_BIAS + __global Dtype* biases_base = biases_base_ptr + biases_base_offset; +#endif + __global Dtype *dst = dst_base + dst_offset; +#ifdef FUSED_CONV_ELTWISE + __global Dtype* eltwise_data = eltwise_ptr + eltwise_offset; +#endif + const int group_x = get_group_id(0); const int group_y = get_group_id(1); const int global_x = get_global_id(0); @@ -1574,7 +1617,17 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) __attribute__((intel_reqd_sub_group_size(16))) __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) { + const __global Dtype *src0 = src0_ptr + src0_offset; + const __global Dtype *src1 = src1_ptr + src1_offset; +#if APPLY_BIAS + __global Dtype* biases_base = biases_base_ptr + biases_base_offset; +#endif + __global Dtype *dst = dst_base + dst_offset; +#ifdef FUSED_CONV_ELTWISE + __global Dtype* eltwise_data = eltwise_ptr + eltwise_offset; +#endif + const int group_x = get_group_id(0); const int group_y = get_group_id(1); const int global_x = get_global_id(0); From 9e7b911e442fb5dafec429f40823b99233431056 Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Tue, 7 Sep 2021 11:41:37 +0800 Subject: [PATCH 187/376] Fix building issue --- cmake/templates/cvconfig.h.in | 3 +++ modules/dnn/CMakeLists.txt | 12 +++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/cmake/templates/cvconfig.h.in b/cmake/templates/cvconfig.h.in index 6439d8b43f06..39708e14bdbf 100644 --- a/cmake/templates/cvconfig.h.in +++ b/cmake/templates/cvconfig.h.in @@ -59,6 +59,9 @@ /* Vulkan support */ #cmakedefine HAVE_VULKAN +/* Webnn support */ +#cmakedefine HAVE_WEBNN + /* Define to 1 if you have the header file. */ #cmakedefine HAVE_INTTYPES_H 1 diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 5b11dd4b2c24..dd19f1bf13fd 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -135,11 +135,13 @@ if(HAVE_TENGINE) endif() set(webnn_srcs "") -if(HAVE_WEBNN) - list(APPEND include_dirs ${WEBNN_HEADER_DIRS}) - list(APPEND include_dirs ${WEBNN_INCLUDE_DIRS}) - list(APPEND libs -Wl,--whole-archive ${WEBNN_LIBRARIES} -Wl,--no-whole-archive) - list(APPEND webnn_srcs $ENV{WEBNN_NATIVE_DIR}/gen/src/webnn/webnn_cpp.cpp) +if(NOT EMSCRIPTEN) + if(HAVE_WEBNN) + list(APPEND include_dirs ${WEBNN_HEADER_DIRS}) + list(APPEND include_dirs ${WEBNN_INCLUDE_DIRS}) + list(APPEND libs -Wl,--whole-archive ${WEBNN_LIBRARIES} -Wl,--no-whole-archive) + list(APPEND webnn_srcs $ENV{WEBNN_NATIVE_DIR}/gen/src/webnn/webnn_cpp.cpp) + endif() endif() ocv_module_include_directories(${include_dirs}) From 0c13d34ade785d1d84da1532634294ff4dbc5fd5 Mon Sep 17 00:00:00 2001 From: Ruben Gonzalez <56379722+rgonzalezfluendo@users.noreply.github.com> Date: Tue, 7 Sep 2021 12:02:25 +0200 Subject: [PATCH 188/376] imgcodecs(doc): Sync imread_ docuemtation with the implemetation. Documentation was desynchronized in commit 11eafca3e2a4cbc62f1309d25db0ea3ed9a6ea8e --- modules/imgcodecs/src/loadsave.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/imgcodecs/src/loadsave.cpp b/modules/imgcodecs/src/loadsave.cpp index 28d8ff285bd4..f5ebb8a7bc1e 100644 --- a/modules/imgcodecs/src/loadsave.cpp +++ b/modules/imgcodecs/src/loadsave.cpp @@ -377,10 +377,6 @@ static void ApplyExifOrientation(ExifEntry_t orientationTag, Mat& img) * * @param[in] filename File to load * @param[in] flags Flags - * @param[in] hdrtype { LOAD_CVMAT=0, - * LOAD_IMAGE=1, - * LOAD_MAT=2 - * } * @param[in] mat Reference to C++ Mat object (If LOAD_MAT) * */ From c703f1eed6c3da4f14b37cc151e03463bb4ff54f Mon Sep 17 00:00:00 2001 From: Ilya Lavrenov Date: Thu, 9 Sep 2021 10:56:34 +0300 Subject: [PATCH 189/376] Merge pull request #20673 from ilya-lavrenov:print-openvino-information * Support of OpenVINO interface libraries * cmake: rename and move ocv_get_imported_target to OpenCVUtils.cmake --- CMakeLists.txt | 3 ++- cmake/OpenCVUtils.cmake | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 41a77f04674b..e5d809fefc79 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1526,6 +1526,7 @@ if(WITH_INF_ENGINE OR INF_ENGINE_TARGET) if(INF_ENGINE_TARGET) list(GET INF_ENGINE_TARGET 0 ie_target) set(__msg "YES (${INF_ENGINE_RELEASE} / ${INF_ENGINE_VERSION})") + ocv_get_imported_target(ie_target "${ie_target}") get_target_property(_lib ${ie_target} IMPORTED_LOCATION) get_target_property(_lib_imp_rel ${ie_target} IMPORTED_IMPLIB_RELEASE) get_target_property(_lib_imp_dbg ${ie_target} IMPORTED_IMPLIB_DEBUG) @@ -1549,7 +1550,7 @@ if(WITH_INF_ENGINE OR INF_ENGINE_TARGET) endif() if(WITH_NGRAPH OR HAVE_NGRAPH) if(HAVE_NGRAPH) - set(__target ngraph::ngraph) + ocv_get_imported_target(__target ngraph::ngraph) set(__msg "YES (${ngraph_VERSION})") get_target_property(_lib ${__target} IMPORTED_LOCATION) get_target_property(_lib_imp_rel ${__target} IMPORTED_IMPLIB_RELEASE) diff --git a/cmake/OpenCVUtils.cmake b/cmake/OpenCVUtils.cmake index 932eb039b141..d7097fdc30b9 100644 --- a/cmake/OpenCVUtils.cmake +++ b/cmake/OpenCVUtils.cmake @@ -1619,6 +1619,18 @@ function(ocv_add_external_target name inc link def) endif() endfunction() +# Returns the first non-interface target +function(ocv_get_imported_target imported interface) + set(__result "${interface}") + get_target_property(__type "${__result}" TYPE) + if(__type STREQUAL "INTERFACE_LIBRARY") + get_target_property(__libs "${__result}" INTERFACE_LINK_LIBRARIES) + list(GET __libs 0 __interface) + ocv_get_imported_target(__result "${__interface}") + endif() + set(${imported} "${__result}" PARENT_SCOPE) +endfunction() + macro(ocv_get_libname var_name) get_filename_component(__libname "${ARGN}" NAME) From 4807cd8a6eeaa494a76df435679ccc3b9e6545e6 Mon Sep 17 00:00:00 2001 From: rogday Date: Thu, 9 Sep 2021 14:41:40 +0300 Subject: [PATCH 190/376] Merge pull request #20605 from rogday:split_slice_shenanigans Add Normalize subgraph, fix Slice, Mul and Expand * Add Normalize subgraph, support for starts<0 and axis<0 in Slice, Mul broadcasting in the middle and fix Expand's unsqueeze * remove todos * remove range-based for loop * address review comments * change >> to > > in template * fix indexation * fix expand that does nothing --- modules/dnn/src/layers/slice_layer.cpp | 59 ++++++-- .../dnn/src/onnx/onnx_graph_simplifier.cpp | 22 +++ modules/dnn/src/onnx/onnx_importer.cpp | 131 ++++++++++++++---- modules/dnn/test/test_onnx_importer.cpp | 6 + 4 files changed, 174 insertions(+), 44 deletions(-) diff --git a/modules/dnn/src/layers/slice_layer.cpp b/modules/dnn/src/layers/slice_layer.cpp index 16f1958879ee..a47077281321 100644 --- a/modules/dnn/src/layers/slice_layer.cpp +++ b/modules/dnn/src/layers/slice_layer.cpp @@ -58,6 +58,31 @@ namespace cv namespace dnn { +void sliceRangesFromShape(const MatShape& inpShape, int& axis, std::vector >& sliceRanges) +{ + CV_Assert(inpShape.size() > 0); + bool axisNeg = (axis < 0); + axis = (axis + static_cast(inpShape.size())) % inpShape.size(); + int n = inpShape[axis]; + + for (size_t i = 0; i < sliceRanges.size(); ++i){ + std::vector& ranges = sliceRanges[i]; + if (axisNeg) + { + ranges.insert(ranges.begin(), axis, Range::all()); + } + Range& range = ranges.back(); + + if (range.start >= 0) + { + continue; + } + + CV_Assert(n != 0); + range.start = (n + range.start) % n; + } +} + class SliceLayerImpl : public SliceLayer { public: @@ -69,20 +94,22 @@ class SliceLayerImpl : public SliceLayer num_split = params.get("num_split", 0); hasDynamicShapes = params.get("has_dynamic_shapes", false); shapesInitialized = !hasDynamicShapes; + if (params.has("slice_point")) { CV_Assert(!params.has("begin") && !params.has("size") && !params.has("end")); const DictValue &indicesValue = params.get("slice_point"); + int size = axis > 0 ? axis + 1 : 1; sliceRanges.resize(indicesValue.size() + 1, - std::vector(axis + 1, Range::all())); + std::vector(size, Range::all())); int prevSlice = 0; for (int i = 0; i < indicesValue.size(); ++i) { - sliceRanges[i][axis].start = prevSlice; - sliceRanges[i][axis].end = indicesValue.get(i); - prevSlice = sliceRanges[i][axis].end; + sliceRanges[i][size - 1].start = prevSlice; + sliceRanges[i][size - 1].end = indicesValue.get(i); + prevSlice = sliceRanges[i][size - 1].end; } - sliceRanges.back()[axis].start = prevSlice; + sliceRanges.back()[size - 1].start = prevSlice; } else if (params.has("begin")) { @@ -97,7 +124,6 @@ class SliceLayerImpl : public SliceLayer { int start = begins.get(i); int sizeOrEnd = sizesOrEnds.get(i); // It may be negative to reverse indexation. - CV_Assert(start >= 0); sliceRanges[0][i].start = start; if (params.has("size")) @@ -154,16 +180,20 @@ class SliceLayerImpl : public SliceLayer CV_Assert(inputs.size() == 1); MatShape inpShape = inputs[0]; - if (!sliceRanges.empty()) + int axis_rw = axis; + std::vector > sliceRanges_rw = sliceRanges; + sliceRangesFromShape(inpShape, axis_rw, sliceRanges_rw); + + if (!sliceRanges_rw.empty()) { - outputs.resize(sliceRanges.size(), inpShape); + outputs.resize(sliceRanges_rw.size(), inpShape); for (int i = 0; i < outputs.size(); ++i) { - CV_Assert(sliceRanges[i].size() <= inpShape.size()); - for (int j = 0; j < sliceRanges[i].size(); ++j) + CV_Assert(sliceRanges_rw[i].size() <= inpShape.size()); + for (int j = 0; j < sliceRanges_rw[i].size(); ++j) { if (shapesInitialized || inpShape[j] > 0) - outputs[i][j] = normalize_axis_range(sliceRanges[i][j], inpShape[j]).size(); + outputs[i][j] = normalize_axis_range(sliceRanges_rw[i][j], inpShape[j]).size(); if (!sliceSteps.empty() && (i < sliceSteps.size()) && (j < sliceSteps[i].size()) && (sliceSteps[i][j] > 1)) outputs[i][j] = (outputs[i][j] + sliceSteps[i][j] - 1) / sliceSteps[i][j]; @@ -172,10 +202,10 @@ class SliceLayerImpl : public SliceLayer } else // Divide input blob on equal parts by axis. { - CV_Assert(0 <= axis && axis < inpShape.size()); + CV_Assert(0 <= axis_rw && axis_rw < inpShape.size()); int splits = num_split ? num_split : requiredOutputs; - CV_Assert(splits > 0 && inpShape[axis] % splits == 0); - inpShape[axis] /= splits; + CV_Assert(splits > 0 && inpShape[axis_rw] % splits == 0); + inpShape[axis_rw] /= splits; outputs.resize(splits, inpShape); } return false; @@ -200,6 +230,7 @@ class SliceLayerImpl : public SliceLayer CV_Assert(inputs.size() == 1); const MatSize& inpShape = inputs[0].size; + sliceRangesFromShape(shape(inputs[0]), axis, sliceRanges); finalSliceRanges = sliceRanges; if (sliceRanges.empty()) diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index b81ccf106c4f..76937e08f3bd 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -231,6 +231,27 @@ class NormalizeSubgraph2 : public NormalizeSubgraphBase } }; +class NormalizeSubgraph2_2 : public NormalizeSubgraphBase +{ +public: + NormalizeSubgraph2_2() + { + int input = addNodeToMatch(""); + int norm = addNodeToMatch("ReduceL2", input); + + int min = addNodeToMatch(""); + int max = addNodeToMatch(""); + int clip = addNodeToMatch("Clip", norm, min, max); + + int shape = addNodeToMatch(""); + int expand = addNodeToMatch("Expand", clip, shape); + + addNodeToMatch("Div", input, expand); + + setFusedNode("Normalize", input); + } +}; + class NormalizeSubgraph3 : public NormalizeSubgraphBase { public: @@ -555,6 +576,7 @@ void simplifySubgraphs(opencv_onnx::GraphProto& net) subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); + subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 6583d6cf624b..955c79c0fa5c 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -59,6 +59,8 @@ class ONNXImporter void addLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void expandMid(const std::string& prefix, opencv_onnx::NodeProto& node_proto, + const std::string& input, size_t n); public: ONNXImporter(Net& net, const char *onnxFile) @@ -427,6 +429,37 @@ void ONNXImporter::addLayer(LayerParams& layerParams, } } +/** @brief Make N copies of input layer and set them as input to node_proto. + * @param prefix prefix of new layers' names + * @param node_proto node which will contain all copies as inputs + * @param input name of the node to copy + * @param n number of copies + */ +void ONNXImporter::expandMid(const std::string& prefix, opencv_onnx::NodeProto& node_proto, + const std::string& input, size_t n) +{ + std::vector input_names; + input_names.reserve(n); + for (size_t j = 0; j < n; j++) + { + LayerParams copyLP; + copyLP.name = format("%s/copy_%d", prefix.c_str(), j); + copyLP.type = "Identity"; + CV_Assert((layer_id.find(copyLP.name) == layer_id.end()) && + "Couldn't copy the node: generated name already exists in the graph."); + input_names.push_back(copyLP.name); + + node_proto.set_input(0, input); + node_proto.set_output(0, copyLP.name); + addLayer(copyLP, node_proto); + } + node_proto.clear_input(); + for (size_t i = 0; i < input_names.size(); i++) + { + node_proto.add_input(input_names[i]); + } +} + void ONNXImporter::addConstant(const std::string& name, const Mat& blob) { constBlobs.insert(std::make_pair(name, blob)); @@ -1288,6 +1321,37 @@ void ONNXImporter::parseMatMul(LayerParams& layerParams, const opencv_onnx::Node addLayer(layerParams, node_proto); } +void findBroadAxis(const MatShape& broadShape, const MatShape& outShape, size_t& axis, int& broadAxis) +{ + const size_t diff = outShape.size() - broadShape.size(); + + // find the first non-one element of the broadcasting shape + axis = 0; + for (; axis < broadShape.size() && broadShape[axis] == 1; ++axis) {} + + // find the last non-one element of the broadcasting shape + size_t endAxis = broadShape.size(); + for (; endAxis > axis && broadShape[endAxis - 1] == 1; --endAxis) {} + + // find one between axis and endAxis - as it needs to be broadcasted, + // dimensions from the left of axis and from the right of endAxis will be handled by Scale layer + broadAxis = -1; + for (size_t i = axis; i < endAxis; ++i) + { + size_t outAxis = i + diff; + if (outShape[outAxis] == broadShape[i]) + { + continue; + } + + // ensure we need to broadcast only 1 dimension in the middle + CV_Assert(broadShape[i] == 1 && broadAxis == -1); + broadAxis = static_cast(outAxis); + } + + axis += diff; +} + // "Mul" "Div" void ONNXImporter::parseMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) { @@ -1410,13 +1474,31 @@ void ONNXImporter::parseMul(LayerParams& layerParams, const opencv_onnx::NodePro } const MatShape& broadShape = outShapes[node_proto.input(1)]; - const size_t outShapeSize = outShapes[node_proto.input(0)].size(); - const size_t diff = outShapeSize - broadShape.size(); + const MatShape& outShape = outShapes[node_proto.input(0)]; + + size_t axis = 0; + int broadAxis = -1; + findBroadAxis(broadShape, outShape, axis, broadAxis); + + // if there is a one dimension in the middle that should be broadcasted, broadcast it + if (broadAxis != -1) + { + opencv_onnx::NodeProto concat_node_proto = node_proto; + const std::string& input1 = concat_node_proto.input(1); + + expandMid(layerParams.name, concat_node_proto, input1, outShape[broadAxis]); - size_t axis; - for (axis = diff; axis < broadShape.size() && broadShape[axis - diff] == 1; ++axis) {} + LayerParams concatLP; + concatLP.name = layerParams.name + "/concat"; + concatLP.set("axis", broadAxis); + concatLP.type = "Concat"; + concat_node_proto.set_output(0, concatLP.name); - CV_Assert(axis != outShapeSize); + addLayer(concatLP, concat_node_proto); + node_proto.set_input(1, concatLP.name); + } + + CV_Assert(axis != outShape.size()); layerParams.set("axis", static_cast(axis)); layerParams.type = "Scale"; } @@ -1685,12 +1767,11 @@ void ONNXImporter::parseExpand(LayerParams& layerParams, const opencv_onnx::Node // Unsqueeze and repeat along new axis if (targetShape.size() == inpShape.size() + 1) { + inpShape.insert(inpShape.begin(), targetShape.size() - inpShape.size(), 1); for (int i = 0; i < targetShape.size(); i++) { - if (targetShape[i] == -1 && i < inpShape.size()) + if (abs(targetShape[i]) == 1) targetShape[i] = inpShape[i]; - else if (i < inpShape.size() && targetShape[i] != inpShape[i]) - inpShape.insert(inpShape.begin() + i, 1); } if (haveVariables) { @@ -1710,14 +1791,19 @@ void ONNXImporter::parseExpand(LayerParams& layerParams, const opencv_onnx::Node CV_CheckEQ(inpShape.size(), targetShape.size(), "Unsupported Expand op with different dims"); std::vector broadcast_axes; + // shapes aren't right-aligned here because targetShape.size() == inpShape.size() for (int i = 0; i < targetShape.size(); i++) { if (targetShape[i] != inpShape[i]) { if (inpShape[i] == 1) + { broadcast_axes.push_back(i); - else + } + else if (targetShape[i] != 1) + { CV_Error(Error::StsError, format("Could not be broadcast by axis: %d", i)); + } } } @@ -1756,31 +1842,16 @@ void ONNXImporter::parseExpand(LayerParams& layerParams, const opencv_onnx::Node } else if (broadcast_axes.size() == 1 && broadcast_axes[0] <= 1) { - String base_name = layerParams.name + "/copy_"; - std::vector input_names; - for (int j = 0; j < targetShape[broadcast_axes[0]]; j++) - { - std::ostringstream ss; - ss << j; - LayerParams copyLP; - copyLP.name = base_name + ss.str(); - copyLP.type = "Identity"; - CV_Assert(layer_id.find(copyLP.name) == layer_id.end()); - input_names.push_back(copyLP.name); - - node_proto.set_input(0, srcName); - node_proto.set_output(0, copyLP.name); - addLayer(copyLP, node_proto); - } - node_proto.clear_input(); - for (int i = 0; i < input_names.size(); i++) - { - node_proto.add_input(input_names[i]); - } + expandMid(layerParams.name, node_proto, srcName, targetShape[broadcast_axes[0]]); + layerParams.set("axis", broadcast_axes[0]); layerParams.type = "Concat"; node_proto.set_output(0, layerParams.name); } + else if (broadcast_axes.empty()) + { + layerParams.type = "Identity"; + } else CV_Error(Error::StsNotImplemented, "Unsupported Expand op"); addLayer(layerParams, node_proto); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index bd96729be1d8..7b94e02d0a01 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -285,6 +285,7 @@ TEST_P(Test_ONNX_layers, Scale) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); testONNXModels("scale"); testONNXModels("scale_broadcast", npy, 0, 0, false, true, 3); + testONNXModels("scale_broadcast_mid", npy, 0, 0, false, true, 2); } TEST_P(Test_ONNX_layers, ReduceMean3D) @@ -471,6 +472,8 @@ TEST_P(Test_ONNX_layers, MatMulAdd) TEST_P(Test_ONNX_layers, Expand) { + testONNXModels("expand"); + testONNXModels("expand_identity"); testONNXModels("expand_batch"); testONNXModels("expand_channels"); testONNXModels("expand_neg_batch"); @@ -611,6 +614,7 @@ TEST_P(Test_ONNX_layers, ReduceL2) testONNXModels("reduceL2"); testONNXModels("reduceL2_subgraph"); testONNXModels("reduceL2_subgraph_2"); + testONNXModels("reduceL2_subgraph2_2"); } TEST_P(Test_ONNX_layers, Split) @@ -624,6 +628,7 @@ TEST_P(Test_ONNX_layers, Split) testONNXModels("split_3"); testONNXModels("split_4"); testONNXModels("split_sizes"); + testONNXModels("split_neg_axis"); } TEST_P(Test_ONNX_layers, Slice) @@ -632,6 +637,7 @@ TEST_P(Test_ONNX_layers, Slice) testONNXModels("slice", npy, 0, 0, false, false); #else testONNXModels("slice"); + testONNXModels("slice_neg_starts"); testONNXModels("slice_opset_11"); #endif } From 068f33cfdf57ae5009d3e564a37a83bfefda9b22 Mon Sep 17 00:00:00 2001 From: Dale Phurrough Date: Thu, 9 Sep 2021 15:20:45 +0200 Subject: [PATCH 191/376] add nodiscard to features2d clone funcs --- .../features2d/include/opencv2/features2d.hpp | 8 ++++---- modules/python/src2/hdr_parser.py | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/modules/features2d/include/opencv2/features2d.hpp b/modules/features2d/include/opencv2/features2d.hpp index cff09170c500..bf193599e121 100644 --- a/modules/features2d/include/opencv2/features2d.hpp +++ b/modules/features2d/include/opencv2/features2d.hpp @@ -1082,7 +1082,7 @@ class CV_EXPORTS_W DescriptorMatcher : public Algorithm that is, copies both parameters and train data. If emptyTrainData is true, the method creates an object copy with the current parameters but with empty train data. */ - CV_WRAP virtual Ptr clone( bool emptyTrainData=false ) const = 0; + CV_WRAP CV_NODISCARD_STD virtual Ptr clone( bool emptyTrainData=false ) const = 0; /** @brief Creates a descriptor matcher of a given type with the default parameters (using default constructor). @@ -1142,7 +1142,7 @@ class CV_EXPORTS_W DescriptorMatcher : public Algorithm static bool isPossibleMatch( InputArray mask, int queryIdx, int trainIdx ); static bool isMaskedOut( InputArrayOfArrays masks, int queryIdx ); - static Mat clone_op( Mat m ) { return m.clone(); } + CV_NODISCARD_STD static Mat clone_op( Mat m ) { return m.clone(); } void checkMasks( InputArrayOfArrays masks, int queryDescriptorsCount ) const; //! Collection of descriptors from train images. @@ -1183,7 +1183,7 @@ class CV_EXPORTS_W BFMatcher : public DescriptorMatcher */ CV_WRAP static Ptr create( int normType=NORM_L2, bool crossCheck=false ) ; - virtual Ptr clone( bool emptyTrainData=false ) const CV_OVERRIDE; + CV_NODISCARD_STD virtual Ptr clone( bool emptyTrainData=false ) const CV_OVERRIDE; protected: virtual void knnMatchImpl( InputArray queryDescriptors, std::vector >& matches, int k, InputArrayOfArrays masks=noArray(), bool compactResult=false ) CV_OVERRIDE; @@ -1222,7 +1222,7 @@ class CV_EXPORTS_W FlannBasedMatcher : public DescriptorMatcher CV_WRAP static Ptr create(); - virtual Ptr clone( bool emptyTrainData=false ) const CV_OVERRIDE; + CV_NODISCARD_STD virtual Ptr clone( bool emptyTrainData=false ) const CV_OVERRIDE; protected: static void convertToDMatches( const DescriptorCollection& descriptors, const Mat& indices, const Mat& distances, diff --git a/modules/python/src2/hdr_parser.py b/modules/python/src2/hdr_parser.py index 749a9033eed6..951dfe11c33a 100755 --- a/modules/python/src2/hdr_parser.py +++ b/modules/python/src2/hdr_parser.py @@ -432,11 +432,18 @@ def parse_func_decl(self, decl_str, mat="Mat", docstring=""): # filter off some common prefixes, which are meaningless for Python wrappers. # note that we do not strip "static" prefix, which does matter; # it means class methods, not instance methods - decl_str = self.batch_replace(decl_str, [("static inline", ""), ("inline", ""), ("explicit ", ""), - ("CV_EXPORTS_W", ""), ("CV_EXPORTS", ""), ("CV_CDECL", ""), - ("CV_WRAP ", " "), ("CV_INLINE", ""), - ("CV_DEPRECATED", ""), ("CV_DEPRECATED_EXTERNAL", "")]).strip() - + decl_str = self.batch_replace(decl_str, [("static inline", ""), + ("inline", ""), + ("explicit ", ""), + ("CV_EXPORTS_W", ""), + ("CV_EXPORTS", ""), + ("CV_CDECL", ""), + ("CV_WRAP ", " "), + ("CV_INLINE", ""), + ("CV_DEPRECATED", ""), + ("CV_DEPRECATED_EXTERNAL", ""), + ("CV_NODISCARD_STD", ""), + ("CV_NODISCARD", "")]).strip() if decl_str.strip().startswith('virtual'): virtual_method = True From c7e0888982a4d95b17ed46c7f8c4f1b3c3d81a95 Mon Sep 17 00:00:00 2001 From: fortemSteve <89345810+fortemSteve@users.noreply.github.com> Date: Thu, 9 Sep 2021 15:23:49 -0600 Subject: [PATCH 192/376] Merge pull request #20591 from fortemSteve:ffmpeg_get_stream_open_time Add CAP_PROP_STREAM_OPEN_TIME * Added CAP_PROP_STREAM_OPEN_TIME to videoio module - can be used to query the time at which the stream was opened, in seconds since Jan 1 1970 (midnight, UTC). Useful for RTSP and other live video where absolute timestamps are needed. Only applicable to ffmpeg backends * use nanoseconds instead of seconds to mark the stream open time, and change the cap prop name to CAP_PROP_STREAM_OPEN_TIME_NSEC * use microseconds for CAP_PROP_STREAM_OPEN_TIME (nanoseconds rolls over too soon, and milliseconds/seconds requires a division) * fix whitespace issue --- modules/videoio/include/opencv2/videoio.hpp | 1 + modules/videoio/src/cap_ffmpeg_impl.hpp | 3 +++ 2 files changed, 4 insertions(+) diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 16016e4b8e9a..3276d0d5e4ae 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -188,6 +188,7 @@ enum VideoCaptureProperties { CAP_PROP_HW_ACCELERATION_USE_OPENCL=52, //!< (**open-only**) If non-zero, create new OpenCL context and bind it to current thread. The OpenCL context created with Video Acceleration context attached it (if not attached yet) for optimized GPU data copy between HW accelerated decoder and cv::UMat. CAP_PROP_OPEN_TIMEOUT_MSEC=53, //!< (**open-only**) timeout in milliseconds for opening a video capture (applicable for FFmpeg back-end only) CAP_PROP_READ_TIMEOUT_MSEC=54, //!< (**open-only**) timeout in milliseconds for reading from a video capture (applicable for FFmpeg back-end only) + CAP_PROP_STREAM_OPEN_TIME_USEC =55, //(use_opencl); #endif // USE_AV_HW_CODECS + case CAP_PROP_STREAM_OPEN_TIME_USEC: + //ic->start_time_realtime is in microseconds + return ((double)ic->start_time_realtime); default: break; } From d31b93b513df40f1549127836168a0fb17ca3cec Mon Sep 17 00:00:00 2001 From: rogday Date: Fri, 10 Sep 2021 14:07:16 +0300 Subject: [PATCH 193/376] Merge pull request #20674 from rogday:prelu_slope Fix PReLU negative slope access pattern * fix prelu negative slope access pattern * change begin() to ptr() --- modules/dnn/src/layers/elementwise_layers.cpp | 2 +- modules/dnn/test/test_onnx_importer.cpp | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index e6cf714bff0e..6e7fa43a7074 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -1542,7 +1542,7 @@ Ptr ChannelsPReLULayer::create(const LayerParams& params) if (params.blobs[0].total() == 1) { LayerParams reluParams = params; - reluParams.set("negative_slope", params.blobs[0].at(0)); + reluParams.set("negative_slope", *params.blobs[0].ptr()); return ReLULayer::create(reluParams); } Ptr l(new ElementWiseLayer(ChannelsPReLUFunctor(params.blobs[0]))); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 7b94e02d0a01..f55510ec7bd9 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -250,6 +250,11 @@ TEST_P(Test_ONNX_layers, ReLU) testONNXModels("ReLU"); } +TEST_P(Test_ONNX_layers, PReLU) +{ + testONNXModels("PReLU_slope"); +} + TEST_P(Test_ONNX_layers, Clip) { testONNXModels("clip", npy); From e3f4f874c5214397a41befa6735f31382e7c39d2 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 10 Sep 2021 15:00:11 +0300 Subject: [PATCH 194/376] Merge pull request #20670 from alalek:core_ocl_fix_intel_gpu_gemm_requirements core(OpenCL): fix intel_gpu_gemm kernel requirements * core(ocl): fix intel_gpu_gemm integration - allow bailout to generic OpenCL kernel * core(ocl): avoid failures of generic OpenCL gemm kernel * core(ocl): define alignment requirements of intel_gpu_gemm kernels --- modules/core/src/intel_gpu_gemm.inl.hpp | 69 +++++++---- modules/core/src/matmul.dispatch.cpp | 153 +++++++++++++----------- modules/core/test/ocl/test_gemm.cpp | 34 +++--- 3 files changed, 152 insertions(+), 104 deletions(-) diff --git a/modules/core/src/intel_gpu_gemm.inl.hpp b/modules/core/src/intel_gpu_gemm.inl.hpp index fbd567b949e3..fa66856f5eb0 100644 --- a/modules/core/src/intel_gpu_gemm.inl.hpp +++ b/modules/core/src/intel_gpu_gemm.inl.hpp @@ -24,11 +24,6 @@ #ifdef HAVE_OPENCL -#include -#include "opencl_kernels_core.hpp" -#include "opencv2/core/opencl/runtime/opencl_clamdblas.hpp" -#include "opencv2/core/opencl/runtime/opencl_core.hpp" - namespace cv { @@ -37,52 +32,79 @@ static bool intel_gpu_gemm( UMat B, Size sizeB, UMat D, Size sizeD, double alpha, double beta, - bool atrans, bool btrans) + bool atrans, bool btrans, + bool& isPropagatedC2D +) { CV_UNUSED(sizeB); int M = sizeD.height, N = sizeD.width, K = ((atrans)? sizeA.height : sizeA.width); + if (M < 4 || N < 4 || K < 4) // vload4 + return false; + + CV_LOG_VERBOSE(NULL, 0, "M=" << M << " N=" << N << " K=" << K); + std::string kernelName; - bool ret = true; - size_t lx = 8, ly = 4; - size_t dx = 4, dy = 8; + unsigned int lx = 8, ly = 4; + unsigned int dx = 4, dy = 8; if(!atrans && !btrans) { - if (M % 32 == 0 && N % 32 == 0 && K % 16 == 0) { kernelName = "intelblas_gemm_buffer_NN_sp"; } else { + if (M % 2 != 0) + return false; + // vload4(0, dst_write0) - 4 cols + // multiply by lx: 8 + if (N % (4*8) != 0) + return false; kernelName = "intelblas_gemm_buffer_NN"; } } else if(atrans && !btrans) { + if (M % 32 != 0) + return false; + if (N % 32 != 0) + return false; kernelName = "intelblas_gemm_buffer_TN"; } else if(!atrans && btrans) { + if (M % 128 != 0) + return false; + if (N % 8 != 0) + return false; + if (K % 512 != 0) + return false; kernelName = "intelblas_gemm_buffer_NT"; ly = 16; dx = 1; } else { + if (M % 32 != 0) + return false; + if (N % 32 != 0) + return false; + if (K % 16 != 0) + return false; kernelName = "intelblas_gemm_buffer_TT"; } - const size_t gx = (size_t)(N + dx - 1) / dx; - const size_t gy = (size_t)(M + dy - 1) / dy; + CV_LOG_DEBUG(NULL, "kernel: " << kernelName << " (M=" << M << " N=" << N << " K=" << K << ")"); - size_t local[] = {lx, ly, 1}; - size_t global[] = {(gx + lx - 1) / lx * lx, (gy + ly - 1) / ly * ly, 1}; + const size_t gx = divUp((size_t)N, dx); + const size_t gy = divUp((size_t)M, dy); - int stride = (M * N < 1024 * 1024) ? 10000000 : 256; + size_t local[] = {lx, ly, 1}; + size_t global[] = {roundUp(gx, lx), roundUp(gy, ly), 1}; ocl::Queue q; String errmsg; @@ -110,10 +132,13 @@ static bool intel_gpu_gemm( (int)(D.step / sizeof(float)) ); - ret = k.run(2, global, local, false, q); + bool ret = k.run(2, global, local, false, q); + return ret; } else { + int stride = (M * N < 1024 * 1024) ? 10000000 : 256; + for(int start_index = 0; start_index < K; start_index += stride) { ocl::Kernel k(kernelName.c_str(), program); @@ -132,12 +157,16 @@ static bool intel_gpu_gemm( (int) start_index, // 14 start_index stride); - ret = k.run(2, global, local, false, q); - if (!ret) return ret; + bool ret = k.run(2, global, local, false, q); + if (!ret) + { + if (start_index != 0) + isPropagatedC2D = false; // D array content is changed, need to rewrite + return false; + } } + return true; } - - return ret; } } // namespace cv diff --git a/modules/core/src/matmul.dispatch.cpp b/modules/core/src/matmul.dispatch.cpp index e81064ec1622..a7447330fcee 100644 --- a/modules/core/src/matmul.dispatch.cpp +++ b/modules/core/src/matmul.dispatch.cpp @@ -42,6 +42,8 @@ //M*/ #include "precomp.hpp" +#include + #include "opencl_kernels_core.hpp" #include "opencv2/core/opencl/runtime/opencl_clamdblas.hpp" #include "opencv2/core/opencl/runtime/opencl_core.hpp" @@ -155,10 +157,12 @@ static bool ocl_gemm_amdblas( InputArray matA, InputArray matB, double alpha, static bool ocl_gemm( InputArray matA, InputArray matB, double alpha, InputArray matC, double beta, OutputArray matD, int flags ) { - int depth = matA.depth(), cn = matA.channels(); - int type = CV_MAKETYPE(depth, cn); + int type = matA.type(); + int depth = CV_MAT_DEPTH(type); + int cn = CV_MAT_CN(type); - CV_Assert_N( type == matB.type(), (type == CV_32FC1 || type == CV_64FC1 || type == CV_32FC2 || type == CV_64FC2) ); + CV_CheckTypeEQ(type, matB.type(), ""); + CV_CheckType(type, type == CV_32FC1 || type == CV_64FC1 || type == CV_32FC2 || type == CV_64FC2, ""); const ocl::Device & dev = ocl::Device::getDefault(); bool doubleSupport = dev.doubleFPConfig() > 0; @@ -170,88 +174,103 @@ static bool ocl_gemm( InputArray matA, InputArray matB, double alpha, Size sizeA = matA.size(), sizeB = matB.size(), sizeC = haveC ? matC.size() : Size(0, 0); bool atrans = (flags & GEMM_1_T) != 0, btrans = (flags & GEMM_2_T) != 0, ctrans = (flags & GEMM_3_T) != 0; - CV_Assert( !haveC || matC.type() == type ); - - Size sizeD(((btrans)? sizeB.height : sizeB.width), - ((atrans)? sizeA.width : sizeA.height)); - matD.create(sizeD, type); - - UMat A = matA.getUMat(), B = matB.getUMat(), D = matD.getUMat(); - - - if (!dev.intelSubgroupsSupport() || (depth == CV_64F) || cn != 1) - { - String opts; - - if (atrans) - sizeA = Size(sizeA.height, sizeA.width); - if (btrans) - sizeB = Size(sizeB.height, sizeB.width); - if (haveC && ctrans) - sizeC = Size(sizeC.height, sizeC.width); - - CV_Assert( sizeA.width == sizeB.height && (!haveC || sizeC == sizeD) ); - - int max_wg_size = (int)dev.maxWorkGroupSize(); - int block_size = (max_wg_size / (32*cn) < 32) ? (max_wg_size / (16*cn) < 16) ? (max_wg_size / (8*cn) < 8) ? 1 : 8 : 16 : 32; - - if (atrans) - A = A.t(); - - if (btrans) - B = B.t(); + if (haveC) + CV_CheckTypeEQ(type, matC.type(), ""); - if (haveC) - ctrans ? transpose(matC, D) : matC.copyTo(D); + Size sizeD(((btrans) ? sizeB.height : sizeB.width), + ((atrans) ? sizeA.width : sizeA.height)); - int vectorWidths[] = { 4, 4, 2, 2, 1, 4, cn, -1 }; - int kercn = ocl::checkOptimalVectorWidth(vectorWidths, B, D); + if (atrans) + sizeA = Size(sizeA.height, sizeA.width); + if (btrans) + sizeB = Size(sizeB.height, sizeB.width); + if (haveC && ctrans) + sizeC = Size(sizeC.height, sizeC.width); - opts += format(" -D T=%s -D T1=%s -D WT=%s -D cn=%d -D kercn=%d -D LOCAL_SIZE=%d%s%s%s", - ocl::typeToStr(type), ocl::typeToStr(depth), ocl::typeToStr(CV_MAKETYPE(depth, kercn)), - cn, kercn, block_size, - (sizeA.width % block_size !=0) ? " -D NO_MULT" : "", - haveC ? " -D HAVE_C" : "", - doubleSupport ? " -D DOUBLE_SUPPORT" : ""); + CV_CheckEQ(sizeA.width, sizeB.height, ""); + if (haveC) + CV_CheckEQ(sizeC, sizeD, ""); - ocl::Kernel k("gemm", cv::ocl::core::gemm_oclsrc, opts); - if (k.empty()) - return false; + UMat A = matA.getUMat(); + UMat B = matB.getUMat(); - if (depth == CV_64F) - k.args(ocl::KernelArg::ReadOnlyNoSize(A), - ocl::KernelArg::ReadOnlyNoSize(B, cn, kercn), - ocl::KernelArg::ReadWrite(D, cn, kercn), - sizeA.width, alpha, beta); - else - k.args(ocl::KernelArg::ReadOnlyNoSize(A), - ocl::KernelArg::ReadOnlyNoSize(B, cn, kercn), - ocl::KernelArg::ReadWrite(D, cn, kercn), - sizeA.width, (float)alpha, (float)beta); + matD.create(sizeD, type); + UMat D = matD.getUMat(); - size_t globalsize[2] = { (size_t)sizeD.width * cn / kercn, (size_t)sizeD.height}; - size_t localsize[2] = { (size_t)block_size, (size_t)block_size}; + bool isPropagatedC2D = false; // D content is updated with C / C.t() - return k.run(2, globalsize, block_size!=1 ? localsize : NULL, false); - } - else + if (dev.intelSubgroupsSupport() && (depth == CV_32F) && cn == 1) { if (haveC && beta != 0.0) { ctrans ? transpose(matC, D) : matC.copyTo(D); + isPropagatedC2D = true; } else { beta = 0.0; } - return intel_gpu_gemm(A, sizeA, - B, sizeB, - D, sizeD, - alpha, - beta, - atrans, btrans); + bool res = intel_gpu_gemm(A, matA.size(), + B, matB.size(), + D, sizeD, + alpha, + beta, + atrans, btrans, + isPropagatedC2D); + if (res) + return true; + // fallback on generic OpenCL code } + + if (sizeD.width < 8 || sizeD.height < 8) + return false; + + String opts; + + int wg_size = (int)dev.maxWorkGroupSize(); + int sizeDmin = std::min(sizeD.width, sizeD.height); + wg_size = std::min(wg_size, sizeDmin * sizeDmin); + int block_size = (wg_size / (32*cn) < 32) ? (wg_size / (16*cn) < 16) ? (wg_size / (8*cn) < 8) ? 1 : 8 : 16 : 32; + + if (atrans) + A = A.t(); + + if (btrans) + B = B.t(); + + if (haveC && !isPropagatedC2D) + ctrans ? transpose(matC, D) : matC.copyTo(D); + + int vectorWidths[] = { 4, 4, 2, 2, 1, 4, cn, -1 }; + int kercn = ocl::checkOptimalVectorWidth(vectorWidths, B, D); + + opts += format(" -D T=%s -D T1=%s -D WT=%s -D cn=%d -D kercn=%d -D LOCAL_SIZE=%d%s%s%s", + ocl::typeToStr(type), ocl::typeToStr(depth), ocl::typeToStr(CV_MAKETYPE(depth, kercn)), + cn, kercn, block_size, + (sizeA.width % block_size !=0) ? " -D NO_MULT" : "", + haveC ? " -D HAVE_C" : "", + doubleSupport ? " -D DOUBLE_SUPPORT" : ""); + + ocl::Kernel k("gemm", cv::ocl::core::gemm_oclsrc, opts); + if (k.empty()) + return false; + + if (depth == CV_64F) + k.args(ocl::KernelArg::ReadOnlyNoSize(A), + ocl::KernelArg::ReadOnlyNoSize(B, cn, kercn), + ocl::KernelArg::ReadWrite(D, cn, kercn), + sizeA.width, alpha, beta); + else + k.args(ocl::KernelArg::ReadOnlyNoSize(A), + ocl::KernelArg::ReadOnlyNoSize(B, cn, kercn), + ocl::KernelArg::ReadWrite(D, cn, kercn), + sizeA.width, (float)alpha, (float)beta); + + size_t globalsize[2] = { (size_t)sizeD.width * cn / kercn, (size_t)sizeD.height}; + size_t localsize[2] = { (size_t)block_size, (size_t)block_size}; + + return k.run(2, globalsize, block_size !=1 ? localsize : NULL, false); } #endif diff --git a/modules/core/test/ocl/test_gemm.cpp b/modules/core/test/ocl/test_gemm.cpp index 825b506780ed..cb7cb0be1a76 100644 --- a/modules/core/test/ocl/test_gemm.cpp +++ b/modules/core/test/ocl/test_gemm.cpp @@ -67,6 +67,8 @@ PARAM_TEST_CASE(Gemm, double alpha, beta; + int M, N, K; + TEST_DECLARE_INPUT_PARAMETER(A); TEST_DECLARE_INPUT_PARAMETER(B); TEST_DECLARE_INPUT_PARAMETER(C); @@ -90,30 +92,27 @@ PARAM_TEST_CASE(Gemm, void generateTestData() { - // set minimum size to 20, since testing less sizes doesn't make sense - Size ARoiSize = randomSize(20, MAX_VALUE); - Border ABorder = randomBorder(0, use_roi ? MAX_VALUE : 0); - randomSubMat(A, A_roi, ARoiSize, ABorder, type, -11, 11); + M = (int)randomDoubleLog(1, 100); + N = (int)randomDoubleLog(1, 100); + K = (int)randomDoubleLog(1, 1200); - if (atrans) - ARoiSize = Size(ARoiSize.height, ARoiSize.width); + M = roundUp(M, 1); + N = roundUp(N, 1); + K = roundUp(K, 1); - Size BRoiSize = randomSize(20, MAX_VALUE); - if (btrans) - BRoiSize.width = ARoiSize.width; - else - BRoiSize.height = ARoiSize.width; + Size ARoiSize = (atrans) ? Size(M, K) : Size(K, M); + Border ABorder = randomBorder(0, use_roi ? MAX_VALUE : 0); + randomSubMat(A, A_roi, ARoiSize, ABorder, type, -11, 11); + Size BRoiSize = (btrans) ? Size(K, N) : Size(N, K); Border BBorder = randomBorder(0, use_roi ? MAX_VALUE : 0); randomSubMat(B, B_roi, BRoiSize, BBorder, type, -11, 11); - if (btrans) - BRoiSize = Size(BRoiSize.height, BRoiSize.width); - - Size DRoiSize = Size(BRoiSize.width, ARoiSize.height), CRoiSizeT(DRoiSize.height, DRoiSize.width); + Size CRoiSize = (ctrans) ? Size(M, N) : Size(N, M); Border CBorder = randomBorder(0, use_roi ? MAX_VALUE : 0); - randomSubMat(C, C_roi, ctrans ? CRoiSizeT : DRoiSize, CBorder, type, -11, 11); + randomSubMat(C, C_roi, CRoiSize, CBorder, type, -11, 11); + Size DRoiSize = Size(N, M); Border DBorder = randomBorder(0, use_roi ? MAX_VALUE : 0); randomSubMat(D, D_roi, DRoiSize, DBorder, type, -11, 11); @@ -132,11 +131,12 @@ OCL_TEST_P(Gemm, Accuracy) for (int i = 0; i < test_loop_times; ++i) { generateTestData(); + SCOPED_TRACE(cv::format("i=%d: M=%d N=%d K=%d", i, M, N, K)); OCL_OFF(cv::gemm(A_roi, B_roi, alpha, C_roi, beta, D_roi, flags)); OCL_ON(cv::gemm(uA_roi, uB_roi, alpha, uC_roi, beta, uD_roi, flags)); - double eps = D_roi.size().area() * 1e-4; + double eps = D_roi.size().area() * (1e-5 * K); OCL_EXPECT_MATS_NEAR(D, eps); } } From 9b4ecc96f63d64a07ac043ad06fa44a1fd02b18b Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 7 Sep 2021 04:39:28 +0000 Subject: [PATCH 195/376] core(ocl): buffer bounds in intelblas_gemm_buffer_NT --- modules/core/src/intel_gpu_gemm.inl.hpp | 6 +- modules/core/src/opencl/intel_gemm.cl | 186 +++++++++++------------- 2 files changed, 86 insertions(+), 106 deletions(-) diff --git a/modules/core/src/intel_gpu_gemm.inl.hpp b/modules/core/src/intel_gpu_gemm.inl.hpp index fa66856f5eb0..28cc4ab9b9eb 100644 --- a/modules/core/src/intel_gpu_gemm.inl.hpp +++ b/modules/core/src/intel_gpu_gemm.inl.hpp @@ -77,11 +77,7 @@ static bool intel_gpu_gemm( } else if(!atrans && btrans) { - if (M % 128 != 0) - return false; - if (N % 8 != 0) - return false; - if (K % 512 != 0) + if (K % 4 != 0) return false; kernelName = "intelblas_gemm_buffer_NT"; ly = 16; diff --git a/modules/core/src/opencl/intel_gemm.cl b/modules/core/src/opencl/intel_gemm.cl index 6cea8d7efdc9..53ae790779c8 100644 --- a/modules/core/src/opencl/intel_gemm.cl +++ b/modules/core/src/opencl/intel_gemm.cl @@ -392,6 +392,15 @@ __kernel void intelblas_gemm_buffer_NN( #define TILE_N 8 #define SLM_BLOCK 512 +/* + A K B.t() K D N + ----------- ----------- ----------- + | | | | | | + M | | x N | | => M | | + | | | | | | + ----------- ----------- ----------- +*/ + __attribute__((reqd_work_group_size(8, LWG_HEIGHT, 1))) __kernel void intelblas_gemm_buffer_NT( const __global float *src0, int off0, @@ -422,59 +431,79 @@ __kernel void intelblas_gemm_buffer_NT( float8 dot06 = 0.f; float8 dot07 = 0.f; - float4 brow0; - float4 brow1; - float4 brow2; - float4 brow3; - float4 brow4; - float4 brow5; - float4 brow6; - float4 brow7; - - __global float *dst_write0 = dst + local_x * VEC_SIZE + ( group_x * TILE_N ) + ( group_y * LWG_HEIGHT * TILE_M + local_y * TILE_M) * ldC + offd; + const int dst_row = (global_y * TILE_M); + __global float *dst_write0 = dst + global_x + dst_row * ldC + offd; - const __global float *src0_read = src0 + local_x * ( TILE_K / 8 ) + ( group_y * LWG_HEIGHT * TILE_M + local_y * TILE_M ) * ldA + off0; + const __global float *src0_read00 = src0 + off0; + const int a_row_base = global_y * TILE_M; + const int a_col_base = local_x * (TILE_K / 8); // <= TILE_K - 4 - const __global float *src1_read0 = src1 + ( group_x * TILE_N ) * ldB + off1; + const __global float *src1_read00 = src1 + off1; + const int b_row_base = (group_x * TILE_N); + //const int b_col_base = 0; __local float slm_brow[8 * SLM_BLOCK]; - __local float* slm_brow0; int local_index = mad24(local_y, 8, local_x) * 4; - int w; - for(int b_tile = 0; b_tile < K; b_tile += SLM_BLOCK) { + int w = 0; + for (int b_tile = 0; b_tile < K; b_tile += SLM_BLOCK) + { +#define UPDATE_BROW(_row) \ + { \ + float4 brow; \ + int b_row = b_row_base + _row; \ + int b_col = b_tile + local_index; \ + if (b_row < N && b_col <= K - 4 /*vload4*/) \ + brow = vload4(0, src1_read00 + mad24(b_row, ldB, b_col)); \ + else \ + brow = (float4)0; \ + vstore4(brow, 0, slm_brow + mad24(_row, SLM_BLOCK, local_index)); \ + } + barrier(CLK_LOCAL_MEM_FENCE); - vstore4(vload4(0, src1_read0 + mad24(0, ldB, local_index)), 0, slm_brow + mad24(0, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(1, ldB, local_index)), 0, slm_brow + mad24(1, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(2, ldB, local_index)), 0, slm_brow + mad24(2, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(3, ldB, local_index)), 0, slm_brow + mad24(3, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(4, ldB, local_index)), 0, slm_brow + mad24(4, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(5, ldB, local_index)), 0, slm_brow + mad24(5, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(6, ldB, local_index)), 0, slm_brow + mad24(6, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(7, ldB, local_index)), 0, slm_brow + mad24(7, SLM_BLOCK, local_index)); + UPDATE_BROW(0); + UPDATE_BROW(1); + UPDATE_BROW(2); + UPDATE_BROW(3); + UPDATE_BROW(4); + UPDATE_BROW(5); + UPDATE_BROW(6); + UPDATE_BROW(7); barrier(CLK_LOCAL_MEM_FENCE); - - slm_brow0 = slm_brow + local_x * (TILE_K / 8); - w = b_tile; - int end_w = min(b_tile + SLM_BLOCK, K); - while( w + TILE_K <= end_w ) { - float4 arow; - - brow0 = vload4(0, slm_brow0 + 0 * SLM_BLOCK); - brow1 = vload4(0, slm_brow0 + 1 * SLM_BLOCK); - brow2 = vload4(0, slm_brow0 + 2 * SLM_BLOCK); - brow3 = vload4(0, slm_brow0 + 3 * SLM_BLOCK); - brow4 = vload4(0, slm_brow0 + 4 * SLM_BLOCK); - brow5 = vload4(0, slm_brow0 + 5 * SLM_BLOCK); - brow6 = vload4(0, slm_brow0 + 6 * SLM_BLOCK); - brow7 = vload4(0, slm_brow0 + 7 * SLM_BLOCK); - -#define MM_DOT_PRODUCT(_row,_dot) \ - arow = vload4(0, src0_read + _row * ldA); \ - _dot = mad( (float8)(arow.x), (float8)(brow0.x, brow1.x, brow2.x, brow3.x, brow4.x, brow5.x, brow6.x, brow7.x), _dot ); \ - _dot = mad( (float8)(arow.y), (float8)(brow0.y, brow1.y, brow2.y, brow3.y, brow4.y, brow5.y, brow6.y, brow7.y), _dot ); \ - _dot = mad( (float8)(arow.z), (float8)(brow0.z, brow1.z, brow2.z, brow3.z, brow4.z, brow5.z, brow6.z, brow7.z), _dot ); \ - _dot = mad( (float8)(arow.w), (float8)(brow0.w, brow1.w, brow2.w, brow3.w, brow4.w, brow5.w, brow6.w, brow7.w), _dot ); +#undef UPDATE_BROW + + for (int k_tile_offset = 0; k_tile_offset < SLM_BLOCK; k_tile_offset += TILE_K) + { + int a_col = a_col_base + b_tile + k_tile_offset; + + if (a_col > K - 4 /*vload4*/) + break; + + int slm_brow_col = a_col_base + k_tile_offset; // <= SLM_BLOCK - 4 +#define READ_SLM_BROW(_row) \ + float4 brow##_row = vload4(0, slm_brow + mad24(_row, SLM_BLOCK, slm_brow_col)); + + READ_SLM_BROW(0); + READ_SLM_BROW(1); + READ_SLM_BROW(2); + READ_SLM_BROW(3); + READ_SLM_BROW(4); + READ_SLM_BROW(5); + READ_SLM_BROW(6); + READ_SLM_BROW(7); +#undef READ_SLM_BROW + +#define MM_DOT_PRODUCT(_row,_dot) \ + { \ + int a_row = a_row_base + _row; \ + if (a_row < M) { \ + float4 arow = vload4(0, src0_read00 + mad24(a_row, ldA, a_col)); \ + _dot = mad( (float8)(arow.x), (float8)(brow0.x, brow1.x, brow2.x, brow3.x, brow4.x, brow5.x, brow6.x, brow7.x), _dot ); \ + _dot = mad( (float8)(arow.y), (float8)(brow0.y, brow1.y, brow2.y, brow3.y, brow4.y, brow5.y, brow6.y, brow7.y), _dot ); \ + _dot = mad( (float8)(arow.z), (float8)(brow0.z, brow1.z, brow2.z, brow3.z, brow4.z, brow5.z, brow6.z, brow7.z), _dot ); \ + _dot = mad( (float8)(arow.w), (float8)(brow0.w, brow1.w, brow2.w, brow3.w, brow4.w, brow5.w, brow6.w, brow7.w), _dot ); \ + } \ + } MM_DOT_PRODUCT(0,dot00); MM_DOT_PRODUCT(1,dot01); @@ -485,53 +514,7 @@ __kernel void intelblas_gemm_buffer_NT( MM_DOT_PRODUCT(6,dot06); MM_DOT_PRODUCT(7,dot07); #undef MM_DOT_PRODUCT - - src0_read += TILE_K; - slm_brow0 += TILE_K; - w += TILE_K; } - src1_read0 += SLM_BLOCK; - } - - if(w < K) { - float4 arow; - -#define READ_BROW(_brow,_row) \ - _brow = vload4(0, slm_brow0 + _row * SLM_BLOCK); \ - _brow.x = (mad24(local_x, 4, w) < K) ? _brow.x : 0.0f; \ - _brow.y = (mad24(local_x, 4, w + 1) < K) ? _brow.y : 0.0f; \ - _brow.z = (mad24(local_x, 4, w + 2) < K) ? _brow.z : 0.0f; \ - _brow.w = (mad24(local_x, 4, w + 3) < K) ? _brow.w : 0.0f; - - READ_BROW(brow0,0); - READ_BROW(brow1,1); - READ_BROW(brow2,2); - READ_BROW(brow3,3); - READ_BROW(brow4,4); - READ_BROW(brow5,5); - READ_BROW(brow6,6); - READ_BROW(brow7,7); - -#define MM_DOT_PRODUCT(_row,_dot) \ - arow = vload4(0, src0_read + _row * ldA); \ - arow.x = (mad24(local_x, 4, w) < K) ? arow.x : 0.0f; \ - arow.y = (mad24(local_x, 4, w + 1) < K) ? arow.y : 0.0f; \ - arow.z = (mad24(local_x, 4, w + 2) < K) ? arow.z : 0.0f; \ - arow.w = (mad24(local_x, 4, w + 3) < K) ? arow.w : 0.0f; \ - _dot = mad( (float8)(arow.x), (float8)(brow0.x, brow1.x, brow2.x, brow3.x, brow4.x, brow5.x, brow6.x, brow7.x), _dot ); \ - _dot = mad( (float8)(arow.y), (float8)(brow0.y, brow1.y, brow2.y, brow3.y, brow4.y, brow5.y, brow6.y, brow7.y), _dot ); \ - _dot = mad( (float8)(arow.z), (float8)(brow0.z, brow1.z, brow2.z, brow3.z, brow4.z, brow5.z, brow6.z, brow7.z), _dot ); \ - _dot = mad( (float8)(arow.w), (float8)(brow0.w, brow1.w, brow2.w, brow3.w, brow4.w, brow5.w, brow6.w, brow7.w), _dot ); - - MM_DOT_PRODUCT(0,dot00); - MM_DOT_PRODUCT(1,dot01); - MM_DOT_PRODUCT(2,dot02); - MM_DOT_PRODUCT(3,dot03); - MM_DOT_PRODUCT(4,dot04); - MM_DOT_PRODUCT(5,dot05); - MM_DOT_PRODUCT(6,dot06); - MM_DOT_PRODUCT(7,dot07); -#undef MM_DOT_PRODUCT } #define REDUCE(_dot) \ @@ -572,21 +555,22 @@ __kernel void intelblas_gemm_buffer_NT( output = (local_x == 5) ? _dot.s5 : output; \ output = (local_x == 6) ? _dot.s6 : output; \ output = (local_x == 7) ? _dot.s7 : output; \ - if (beta != 0.0) \ + if (beta != 0.0f) \ dst_write0[0] = mad(output, (float)alpha, ((float)beta * dst_write0[0])); \ else \ dst_write0[0] = output * (float)alpha; \ dst_write0 += ldC; - if(global_x < N && global_y * 8 < M) { - OUTPUT(dot00); - if(mad24(global_y, 8, 1) < M) { OUTPUT(dot01); } - if(mad24(global_y, 8, 2) < M) { OUTPUT(dot02); } - if(mad24(global_y, 8, 3) < M) { OUTPUT(dot03); } - if(mad24(global_y, 8, 4) < M) { OUTPUT(dot04); } - if(mad24(global_y, 8, 5) < M) { OUTPUT(dot05); } - if(mad24(global_y, 8, 6) < M) { OUTPUT(dot06); } - if(mad24(global_y, 8, 7) < M) { OUTPUT(dot07); } + if (global_x < N && dst_row < M) + { + /*if (dst_row + 0 < M)*/ { OUTPUT(dot00); } + if (dst_row + 1 < M) { OUTPUT(dot01); } + if (dst_row + 2 < M) { OUTPUT(dot02); } + if (dst_row + 3 < M) { OUTPUT(dot03); } + if (dst_row + 4 < M) { OUTPUT(dot04); } + if (dst_row + 5 < M) { OUTPUT(dot05); } + if (dst_row + 6 < M) { OUTPUT(dot06); } + if (dst_row + 7 < M) { OUTPUT(dot07); } } #undef OUTPUT } From 1aacb9bb1513315267b42ecdc1bb05eb3b38a38b Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 10 Sep 2021 12:42:28 +0000 Subject: [PATCH 196/376] dnn(perf): update convolution tests --- modules/dnn/perf/perf_convolution.cpp | 232 +++++++++++++++++++++++++- 1 file changed, 226 insertions(+), 6 deletions(-) diff --git a/modules/dnn/perf/perf_convolution.cpp b/modules/dnn/perf/perf_convolution.cpp index c2a3a66ab909..bb890c6a00ab 100644 --- a/modules/dnn/perf/perf_convolution.cpp +++ b/modules/dnn/perf/perf_convolution.cpp @@ -26,25 +26,90 @@ struct ConvParam_t { double declared_flops; }; // Details: #12142 +// Last update: 2021-09 static const ConvParam_t testConvolutionConfigs[] = { + /* GFLOPS 3.398 x 20 = 67.956 */ {{7, 7}, {{1, 128, 46, 46}}, 128, 1, {1, 1}, {1, 1}, {3, 3}, {0, 0}, "", true, 3397788160.}, + /* GFLOPS 16.987 x 3 = 50.962 */ {{5, 5}, {{1, 1152, 16, 16}}, 1152, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 16987226112.}, + /* GFLOPS 23.122 x 2 = 46.244 */ {{5, 5}, {{1, 672, 32, 32}}, 672, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 23121788928.}, + /* GFLOPS 9.987 x 3 = 29.960 */ {{3, 3}, {{1, 256, 92, 92}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 9986707456.}, + /* GFLOPS 1.595 x 16 = 25.524 */ {{3, 3}, {{1, 256, 26, 26}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595230208.}, + /* GFLOPS 4.566 x 5 = 22.828 */ {{7, 7}, {{1, 172, 46, 46}}, 128, 1, {1, 1}, {1, 1}, {3, 3}, {0, 0}, "", true, 4565684736.}, + /* GFLOPS 1.596 x 14 = 22.338 */ {{3, 3}, {{1, 128, 52, 52}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595576320.}, + /* GFLOPS 1.595 x 12 = 19.141 */ {{3, 3}, {{1, 512, 13, 13}}, 1024, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595057152.}, + /* GFLOPS 6.814 x 2 = 13.629 */ {{3, 3}, {{1, 512, 38, 38}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 6814386176.}, + /* GFLOPS 6.637 x 2 = 13.274 */ {{3, 3}, {{1, 256, 75, 75}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 6636960000.}, + /* GFLOPS 11.797 x 1 = 11.797 */ {{5, 5}, {{1, 240, 64, 64}}, 240, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 11797463040.}, + /* GFLOPS 11.797 x 1 = 11.797 */ {{5, 5}, {{1, 480, 32, 32}}, 480, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 11796971520.}, + /* GFLOPS 10.701 x 1 = 10.701 */ {{3, 3}, {{1, 512, 38, 38}}, 804, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 10700715792.}, /* GFLOPS 10.087 x 1 = 10.087 */ {{3, 3}, {{1, 576, 38, 50}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 10086963200.}, + /* GFLOPS 9.993 x 1 = 9.993 */ {{3, 3}, {{1, 64, 368, 368}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 9993207808.}, + /* GFLOPS 9.989 x 1 = 9.989 */ {{3, 3}, {{1, 128, 184, 184}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 9988874240.}, + /* GFLOPS 9.986 x 1 = 9.986 */ {{3, 3}, {{1, 512, 46, 46}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 9985624064.}, /* GFLOPS 1.704 x 5 = 8.518 */ {{3, 3}, {{1, 512, 19, 19}}, 512, 512, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1703596544.}, /* GFLOPS 1.704 x 5 = 8.518 */ {{3, 3}, {{1, 512, 19, 19}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1703596544.}, + /* GFLOPS 4.247 x 2 = 8.494 */ {{3, 3}, {{1, 480, 32, 32}}, 480, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 4247224320.}, + /* GFLOPS 8.025 x 1 = 8.025 */ {{3, 3}, {{1, 1024, 19, 19}}, 1206, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 8025101478.}, + /* GFLOPS 0.798 x 9 = 7.180 */ {{3, 3}, {{1, 128, 52, 52}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 797788160.}, + /* GFLOPS 0.798 x 9 = 7.179 */ {{3, 3}, {{1, 256, 26, 26}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 797615104.}, + /* GFLOPS 6.641 x 1 = 6.641 */ {{3, 3}, {{1, 64, 300, 300}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 6641280000.}, /* GFLOPS 6.641 x 1 = 6.641 */ {{3, 3}, {{1, 64, 150, 200}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 6641280000.}, + /* GFLOPS 6.638 x 1 = 6.638 */ {{3, 3}, {{1, 128, 150, 150}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 6638400000.}, + /* GFLOPS 6.118 x 1 = 6.118 */ {{3, 3}, {{1, 144, 128, 128}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 6117654528.}, + /* GFLOPS 6.116 x 1 = 6.116 */ {{3, 3}, {{1, 1152, 16, 16}}, 1152, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 6115590144.}, + /* GFLOPS 5.780 x 1 = 5.780 */ {{5, 5}, {{1, 672, 32, 32}}, 672, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 5780447232.}, + /* GFLOPS 1.704 x 3 = 5.111 */ {{3, 3}, {{1, 512, 19, 19}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1703596544.}, + /* GFLOPS 4.997 x 1 = 4.997 */ {{3, 3}, {{1, 64, 184, 184}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 4996603904.}, + /* GFLOPS 4.994 x 1 = 4.994 */ {{3, 3}, {{1, 128, 92, 92}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 4994437120.}, + /* GFLOPS 4.993 x 1 = 4.993 */ {{3, 3}, {{1, 256, 46, 46}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 4993353728.}, + /* GFLOPS 4.993 x 1 = 4.993 */ {{3, 3}, {{1, 512, 46, 46}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 4992812032.}, /* GFLOPS 1.659 x 3 = 4.977 */ {{3, 3}, {{1, 960, 10, 10}}, 960, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1658976000.}, /* GFLOPS 2.156 x 2 = 4.312 */ {{3, 3}, {{1, 576, 19, 19}}, 576, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 2156088384.}, + /* GFLOPS 4.247 x 1 = 4.247 */ {{5, 5}, {{1, 144, 128, 128}}, 144, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 4247322624.}, + /* GFLOPS 0.798 x 5 = 3.988 */ {{3, 3}, {{1, 512, 13, 13}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 797528576.}, /* GFLOPS 0.958 x 4 = 3.833 */ {{3, 3}, {{1, 384, 19, 19}}, 384, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 958307712.}, + /* GFLOPS 0.624 x 6 = 3.746 */ {{3, 3}, {{1, 128, 46, 46}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 624304640.}, + /* GFLOPS 3.408 x 1 = 3.408 */ {{3, 3}, {{1, 256, 38, 38}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 3407562752.}, + /* GFLOPS 3.407 x 1 = 3.407 */ {{3, 3}, {{1, 512, 19, 19}}, 1024, 1, {1, 1}, {6, 6}, {6, 6}, {0, 0}, "", true, 3407193088.}, + /* GFLOPS 0.177 x 19 = 3.370 */ {{1, 1}, {{1, 512, 26, 26}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 177382400.}, + /* GFLOPS 0.302 x 11 = 3.325 */ {{3, 3}, {{1, 64, 64, 64}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 302252032.}, + /* GFLOPS 3.321 x 1 = 3.321 */ {{3, 3}, {{1, 64, 150, 150}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 3320640000.}, /* GFLOPS 0.830 x 4 = 3.321 */ {{3, 3}, {{1, 64, 75, 100}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 830160000.}, + /* GFLOPS 3.319 x 1 = 3.319 */ {{3, 3}, {{1, 128, 75, 75}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 3319200000.}, + /* GFLOPS 1.598 x 2 = 3.195 */ {{3, 3}, {{1, 32, 416, 416}}, 64, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 1597652992.}, + /* GFLOPS 1.598 x 2 = 3.195 */ {{3, 3}, {{1, 32, 208, 208}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1597652992.}, + /* GFLOPS 1.596 x 2 = 3.193 */ {{3, 3}, {{1, 64, 208, 208}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 1596268544.}, + /* GFLOPS 1.596 x 2 = 3.193 */ {{3, 3}, {{1, 64, 104, 104}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1596268544.}, + /* GFLOPS 1.596 x 2 = 3.191 */ {{3, 3}, {{1, 128, 104, 104}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595576320.}, + /* GFLOPS 1.595 x 2 = 3.190 */ {{3, 3}, {{1, 256, 52, 52}}, 512, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595230208.}, + /* GFLOPS 1.595 x 2 = 3.190 */ {{3, 3}, {{1, 512, 26, 26}}, 1024, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595057152.}, + /* GFLOPS 0.178 x 16 = 2.841 */ {{1, 1}, {{1, 256, 52, 52}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 177555456.}, + /* GFLOPS 2.719 x 1 = 2.719 */ {{3, 3}, {{1, 96, 256, 256}}, 96, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 2719481856.}, + /* GFLOPS 0.177 x 15 = 2.659 */ {{1, 1}, {{1, 1024, 13, 13}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 177295872.}, /* GFLOPS 1.245 x 2 = 2.490 */ {{3, 3}, {{1, 96, 75, 100}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1244880000.}, + /* GFLOPS 0.798 x 3 = 2.394 */ {{3, 3}, {{1, 64, 104, 104}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 798134272.}, + /* GFLOPS 0.472 x 5 = 2.360 */ {{3, 3}, {{1, 256, 20, 20}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 471961600.}, + /* GFLOPS 2.255 x 1 = 2.255 */ {{3, 3}, {{1, 128, 80, 100}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2255285760.}, + /* GFLOPS 2.153 x 1 = 2.153 */ {{3, 3}, {{1, 128, 78, 98}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2152611840.}, /* GFLOPS 2.100 x 1 = 2.100 */ {{3, 3}, {{1, 144, 75, 75}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 2100330000.}, + /* GFLOPS 2.052 x 1 = 2.052 */ {{3, 3}, {{1, 128, 76, 96}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2052298240.}, /* GFLOPS 1.022 x 2 = 2.044 */ {{3, 3}, {{1, 576, 19, 19}}, 273, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1021896057.}, + /* GFLOPS 1.995 x 1 = 1.995 */ {{9, 9}, {{1, 3, 320, 400}}, 32, 1, {1, 1}, {1, 1}, {4, 4}, {0, 0}, "", true, 1994752000.}, + /* GFLOPS 1.954 x 1 = 1.954 */ {{3, 3}, {{1, 128, 74, 94}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1954344960.}, /* GFLOPS 0.958 x 2 = 1.917 */ {{3, 3}, {{1, 192, 38, 38}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 958446336.}, /* GFLOPS 1.888 x 1 = 1.888 */ {{3, 3}, {{1, 1024, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1887539200.}, /* GFLOPS 1.888 x 1 = 1.888 */ {{3, 3}, {{1, 1024, 10, 10}}, 1024, 1024, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1887539200.}, + /* GFLOPS 1.859 x 1 = 1.859 */ {{3, 3}, {{1, 128, 72, 92}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1858752000.}, + /* GFLOPS 1.766 x 1 = 1.766 */ {{3, 3}, {{1, 128, 70, 90}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1765519360.}, /* GFLOPS 1.704 x 1 = 1.704 */ {{3, 3}, {{1, 256, 38, 38}}, 256, 256, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1703781376.}, /* GFLOPS 1.704 x 1 = 1.704 */ {{3, 3}, {{1, 256, 38, 38}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1703781376.}, + /* GFLOPS 1.675 x 1 = 1.675 */ {{3, 3}, {{1, 128, 68, 88}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1674647040.}, /* GFLOPS 1.660 x 1 = 1.660 */ {{3, 3}, {{1, 128, 75, 75}}, 128, 128, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1659600000.}, /* GFLOPS 1.660 x 1 = 1.660 */ {{3, 3}, {{1, 128, 75, 75}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1659600000.}, + /* GFLOPS 1.586 x 1 = 1.586 */ {{3, 3}, {{1, 128, 66, 86}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1586135040.}, + /* GFLOPS 1.500 x 1 = 1.500 */ {{3, 3}, {{1, 128, 64, 84}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1499983360.}, + /* GFLOPS 1.416 x 1 = 1.416 */ {{3, 3}, {{1, 128, 62, 82}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1416192000.}, + /* GFLOPS 0.472 x 3 = 1.416 */ {{3, 3}, {{1, 128, 40, 40}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 472064000.}, + /* GFLOPS 0.472 x 3 = 1.416 */ {{3, 3}, {{1, 512, 10, 10}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 471910400.}, /* GFLOPS 0.280 x 5 = 1.402 */ {{1, 1}, {{1, 576, 38, 50}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 280409600.}, /* GFLOPS 0.701 x 2 = 1.401 */ {{3, 3}, {{1, 128, 38, 50}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 700720000.}, /* GFLOPS 0.231 x 6 = 1.388 */ {{3, 3}, {{1, 128, 56, 56}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 231311360.}, @@ -53,20 +118,39 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.420 x 3 = 1.261 */ {{3, 3}, {{1, 96, 38, 50}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 420492800.}, /* GFLOPS 1.261 x 1 = 1.261 */ {{3, 3}, {{1, 192, 38, 50}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1261113600.}, /* GFLOPS 1.258 x 1 = 1.258 */ {{3, 3}, {{1, 1280, 10, 10}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1258038600.}, + /* GFLOPS 1.248 x 1 = 1.248 */ {{3, 3}, {{1, 256, 46, 46}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1248338432.}, /* GFLOPS 1.245 x 1 = 1.245 */ {{3, 3}, {{1, 64, 75, 75}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1245240000.}, + /* GFLOPS 1.210 x 1 = 1.210 */ {{3, 3}, {{1, 32, 256, 256}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1210056704.}, + /* GFLOPS 1.196 x 1 = 1.196 */ {{3, 3}, {{1, 384, 26, 26}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1196336128.}, + /* GFLOPS 1.195 x 1 = 1.195 */ {{9, 9}, {{1, 32, 240, 320}}, 3, 1, {1, 1}, {1, 1}, {4, 4}, {0, 0}, "", true, 1194624000.}, + /* GFLOPS 1.182 x 1 = 1.182 */ {{3, 3}, {{1, 32, 320, 400}}, 64, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 1181696000.}, + /* GFLOPS 1.181 x 1 = 1.181 */ {{3, 3}, {{1, 64, 160, 200}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 1180672000.}, /* GFLOPS 0.561 x 2 = 1.121 */ {{3, 3}, {{1, 128, 38, 50}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 560576000.}, + /* GFLOPS 1.112 x 1 = 1.112 */ {{3, 3}, {{1, 512, 10, 10}}, 1206, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1111570200.}, + /* GFLOPS 0.357 x 3 = 1.072 */ {{1, 1}, {{1, 64, 208, 208}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 357187584.}, + /* GFLOPS 1.062 x 1 = 1.062 */ {{3, 3}, {{1, 240, 64, 64}}, 240, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1061928960.}, + /* GFLOPS 0.076 x 14 = 1.058 */ {{3, 3}, {{1, 64, 32, 32}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 75563008.}, /* GFLOPS 1.051 x 1 = 1.051 */ {{3, 3}, {{1, 160, 38, 50}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1050988800.}, + /* GFLOPS 0.210 x 5 = 1.051 */ {{1, 1}, {{1, 256, 20, 20}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 210124800.}, + /* GFLOPS 0.210 x 5 = 1.049 */ {{1, 1}, {{1, 1024, 20, 20}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 209817600.}, /* GFLOPS 1.006 x 1 = 1.006 */ {{3, 3}, {{1, 1024, 10, 10}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1006441800.}, /* GFLOPS 0.246 x 4 = 0.985 */ {{1, 1}, {{1, 256, 75, 100}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 246240000.}, /* GFLOPS 0.189 x 5 = 0.947 */ {{1, 1}, {{1, 512, 19, 19}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 189452800.}, /* GFLOPS 0.189 x 5 = 0.947 */ {{1, 1}, {{1, 512, 19, 19}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 189452800.}, + /* GFLOPS 0.472 x 2 = 0.945 */ {{3, 3}, {{1, 64, 80, 80}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 472268800.}, /* GFLOPS 0.934 x 1 = 0.934 */ {{3, 3}, {{1, 96, 150, 150}}, 96, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 933660000.}, /* GFLOPS 0.231 x 4 = 0.925 */ {{3, 3}, {{1, 128, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 231311360.}, /* GFLOPS 0.896 x 1 = 0.896 */ {{5, 5}, {{1, 96, 27, 27}}, 256, 2, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 895981824.}, + /* GFLOPS 0.089 x 10 = 0.890 */ {{1, 1}, {{1, 128, 52, 52}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 88950784.}, + /* GFLOPS 0.089 x 10 = 0.888 */ {{1, 1}, {{1, 256, 26, 26}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 88777728.}, /* GFLOPS 0.876 x 1 = 0.876 */ {{3, 3}, {{1, 160, 38, 50}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 875824000.}, /* GFLOPS 0.850 x 1 = 0.850 */ {{7, 7}, {{1, 3, 600, 800}}, 24, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 849600000.}, /* GFLOPS 0.841 x 1 = 0.841 */ {{3, 3}, {{1, 128, 38, 50}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 840864000.}, /* GFLOPS 0.415 x 2 = 0.831 */ {{3, 3}, {{1, 32, 150, 150}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 415440000.}, + /* GFLOPS 0.757 x 1 = 0.757 */ {{1, 1}, {{1, 1024, 19, 19}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 757441536.}, + /* GFLOPS 0.712 x 1 = 0.712 */ {{1, 1}, {{1, 128, 208, 208}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 711606272.}, + /* GFLOPS 0.178 x 4 = 0.712 */ {{1, 1}, {{1, 128, 104, 104}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 177901568.}, + /* GFLOPS 0.354 x 2 = 0.707 */ {{1, 1}, {{1, 256, 52, 52}}, 255, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 353723760.}, /* GFLOPS 0.351 x 2 = 0.701 */ {{1, 1}, {{1, 576, 38, 50}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 350512000.}, /* GFLOPS 0.701 x 1 = 0.701 */ {{3, 3}, {{1, 128, 75, 100}}, 160, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 700720000.}, /* GFLOPS 0.694 x 1 = 0.694 */ {{3, 3}, {{1, 64, 56, 56}}, 192, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 694235136.}, @@ -75,19 +159,31 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.058 x 12 = 0.694 */ {{3, 3}, {{1, 128, 28, 28}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 57827840.}, /* GFLOPS 0.231 x 3 = 0.694 */ {{3, 3}, {{1, 512, 7, 7}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 231236096.}, /* GFLOPS 0.160 x 4 = 0.639 */ {{3, 3}, {{1, 64, 38, 38}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 159833472.}, + /* GFLOPS 0.211 x 3 = 0.634 */ {{1, 1}, {{1, 64, 80, 80}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 211353600.}, + /* GFLOPS 0.211 x 3 = 0.632 */ {{1, 1}, {{1, 128, 40, 40}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 210534400.}, + /* GFLOPS 0.210 x 3 = 0.630 */ {{1, 1}, {{1, 512, 40, 40}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 209920000.}, + /* GFLOPS 0.210 x 3 = 0.630 */ {{1, 1}, {{1, 512, 10, 10}}, 2048, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 209920000.}, /* GFLOPS 0.103 x 6 = 0.618 */ {{1, 1}, {{1, 256, 14, 14}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 102961152.}, /* GFLOPS 0.615 x 1 = 0.615 */ {{1, 1}, {{1, 320, 75, 100}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 615360000.}, + /* GFLOPS 0.305 x 2 = 0.609 */ {{3, 3}, {{1, 3, 416, 416}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 304578560.}, /* GFLOPS 0.597 x 1 = 0.597 */ {{3, 3}, {{1, 576, 19, 19}}, 576, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 597254400.}, + /* GFLOPS 0.278 x 2 = 0.557 */ {{1, 1}, {{1, 128, 46, 46}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 278431744.}, /* GFLOPS 0.185 x 3 = 0.554 */ {{1, 1}, {{1, 192, 75, 100}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 184800000.}, /* GFLOPS 0.553 x 1 = 0.553 */ {{3, 3}, {{1, 64, 75, 100}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 553440000.}, /* GFLOPS 0.539 x 1 = 0.539 */ {{3, 3}, {{1, 144, 75, 75}}, 144, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 539178048.}, /* GFLOPS 0.103 x 5 = 0.514 */ {{1, 1}, {{1, 1024, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 102810624.}, /* GFLOPS 0.491 x 1 = 0.491 */ {{1, 1}, {{1, 576, 38, 50}}, 224, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 490716800.}, + /* GFLOPS 0.483 x 1 = 0.483 */ {{7, 7}, {{1, 3, 320, 320}}, 64, 1, {2, 2}, {1, 1}, {3, 3}, {0, 0}, "", false, 483328000.}, /* GFLOPS 0.240 x 2 = 0.479 */ {{3, 3}, {{1, 96, 38, 38}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 239680896.}, + /* GFLOPS 0.477 x 1 = 0.477 */ {{3, 3}, {{1, 3, 368, 368}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 476692480.}, /* GFLOPS 0.237 x 2 = 0.474 */ {{7, 7}, {{1, 3, 224, 224}}, 64, 1, {2, 2}, {1, 1}, {3, 3}, {0, 0}, "", true, 236830720.}, /* GFLOPS 0.472 x 1 = 0.472 */ {{3, 3}, {{1, 512, 19, 19}}, 512, 512, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 471910400.}, /* GFLOPS 0.472 x 1 = 0.472 */ {{3, 3}, {{1, 512, 19, 19}}, 512, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 471910400.}, + /* GFLOPS 0.155 x 3 = 0.464 */ {{1, 1}, {{1, 112, 32, 32}}, 672, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 154828800.}, + /* GFLOPS 0.114 x 4 = 0.454 */ {{1, 1}, {{1, 192, 16, 16}}, 1152, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 113541120.}, /* GFLOPS 0.449 x 1 = 0.449 */ {{3, 3}, {{1, 384, 13, 13}}, 384, 2, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 448626048.}, + /* GFLOPS 0.089 x 5 = 0.443 */ {{1, 1}, {{1, 512, 13, 13}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 88691200.}, + /* GFLOPS 0.428 x 1 = 0.428 */ {{1, 1}, {{1, 64, 64, 64}}, 810, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 427991040.}, /* GFLOPS 0.426 x 1 = 0.426 */ {{3, 3}, {{1, 128, 75, 75}}, 128, 128, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 426037760.}, /* GFLOPS 0.426 x 1 = 0.426 */ {{3, 3}, {{1, 128, 75, 75}}, 128, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 426037760.}, /* GFLOPS 0.426 x 1 = 0.426 */ {{3, 3}, {{1, 128, 38, 38}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 426037760.}, @@ -95,46 +191,81 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.426 x 1 = 0.426 */ {{3, 3}, {{1, 256, 38, 38}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 425945344.}, /* GFLOPS 0.426 x 1 = 0.426 */ {{3, 3}, {{1, 256, 19, 19}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 425945344.}, /* GFLOPS 0.421 x 1 = 0.421 */ {{1, 1}, {{1, 576, 38, 50}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 420614400.}, + /* GFLOPS 0.420 x 1 = 0.420 */ {{1, 1}, {{1, 256, 40, 40}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 420249600.}, + /* GFLOPS 0.210 x 2 = 0.420 */ {{1, 1}, {{1, 256, 80, 80}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 210124800.}, + /* GFLOPS 0.420 x 1 = 0.420 */ {{1, 1}, {{1, 512, 20, 20}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 419840000.}, + /* GFLOPS 0.420 x 1 = 0.420 */ {{1, 1}, {{1, 1024, 10, 10}}, 2048, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 419635200.}, + /* GFLOPS 0.210 x 2 = 0.420 */ {{1, 1}, {{1, 2048, 10, 10}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 209766400.}, /* GFLOPS 0.415 x 1 = 0.415 */ {{3, 3}, {{1, 32, 150, 150}}, 32, 32, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 415440000.}, /* GFLOPS 0.415 x 1 = 0.415 */ {{3, 3}, {{1, 64, 150, 150}}, 64, 64, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 415080000.}, /* GFLOPS 0.415 x 1 = 0.415 */ {{3, 3}, {{1, 64, 150, 150}}, 64, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 415080000.}, /* GFLOPS 0.104 x 4 = 0.414 */ {{1, 1}, {{1, 64, 56, 56}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 103563264.}, /* GFLOPS 0.103 x 4 = 0.413 */ {{1, 1}, {{1, 128, 28, 28}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 103161856.}, + /* GFLOPS 0.399 x 1 = 0.399 */ {{3, 3}, {{1, 32, 208, 208}}, 64, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 399413248.}, + /* GFLOPS 0.200 x 2 = 0.399 */ {{3, 3}, {{1, 32, 104, 104}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 199706624.}, + /* GFLOPS 0.200 x 2 = 0.399 */ {{3, 3}, {{1, 64, 52, 52}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 199533568.}, + /* GFLOPS 0.399 x 1 = 0.399 */ {{3, 3}, {{1, 128, 52, 52}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 398894080.}, + /* GFLOPS 0.199 x 2 = 0.399 */ {{3, 3}, {{1, 128, 26, 26}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 199447040.}, + /* GFLOPS 0.399 x 1 = 0.399 */ {{3, 3}, {{1, 256, 26, 26}}, 512, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 398807552.}, + /* GFLOPS 0.399 x 1 = 0.399 */ {{3, 3}, {{1, 256, 13, 13}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 398807552.}, /* GFLOPS 0.376 x 1 = 0.376 */ {{1, 1}, {{1, 24, 300, 400}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 376320000.}, + /* GFLOPS 0.179 x 2 = 0.357 */ {{1, 1}, {{1, 64, 208, 208}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 178593792.}, + /* GFLOPS 0.089 x 4 = 0.357 */ {{1, 1}, {{1, 64, 104, 104}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 89296896.}, + /* GFLOPS 0.356 x 1 = 0.356 */ {{1, 1}, {{1, 128, 104, 104}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 355803136.}, + /* GFLOPS 0.355 x 1 = 0.355 */ {{1, 1}, {{1, 256, 52, 52}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 355110912.}, + /* GFLOPS 0.355 x 1 = 0.355 */ {{1, 1}, {{1, 512, 26, 26}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 354764800.}, + /* GFLOPS 0.355 x 1 = 0.355 */ {{1, 1}, {{1, 1024, 13, 13}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 354591744.}, + /* GFLOPS 0.355 x 1 = 0.355 */ {{1, 1}, {{1, 2048, 13, 13}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 354505216.}, + /* GFLOPS 0.177 x 2 = 0.353 */ {{1, 1}, {{1, 512, 26, 26}}, 255, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 176689500.}, + /* GFLOPS 0.070 x 5 = 0.348 */ {{1, 1}, {{1, 128, 46, 46}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 69607936.}, /* GFLOPS 0.347 x 1 = 0.347 */ {{3, 3}, {{1, 128, 28, 28}}, 192, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 346967040.}, /* GFLOPS 0.347 x 1 = 0.347 */ {{3, 3}, {{1, 128, 28, 28}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 346967040.}, /* GFLOPS 0.014 x 24 = 0.347 */ {{3, 3}, {{1, 128, 14, 14}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 14456960.}, + /* GFLOPS 0.113 x 3 = 0.340 */ {{1, 1}, {{1, 1152, 16, 16}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 113295360.}, /* GFLOPS 0.053 x 6 = 0.320 */ {{1, 1}, {{1, 576, 19, 19}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 53277824.}, /* GFLOPS 0.319 x 1 = 0.319 */ {{3, 3}, {{1, 192, 19, 19}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 319482112.}, + /* GFLOPS 0.317 x 1 = 0.317 */ {{3, 3}, {{1, 3, 300, 300}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 316800000.}, /* GFLOPS 0.315 x 1 = 0.315 */ {{3, 3}, {{1, 96, 75, 100}}, 96, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 315369600.}, /* GFLOPS 0.103 x 3 = 0.309 */ {{1, 1}, {{1, 512, 7, 7}}, 2048, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 102860800.}, /* GFLOPS 0.103 x 3 = 0.309 */ {{1, 1}, {{1, 512, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 102860800.}, + /* GFLOPS 0.154 x 2 = 0.309 */ {{1, 1}, {{1, 672, 32, 32}}, 112, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 154255360.}, /* GFLOPS 0.308 x 1 = 0.308 */ {{1, 1}, {{1, 320, 75, 100}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 307680000.}, + /* GFLOPS 0.034 x 9 = 0.304 */ {{1, 1}, {{1, 64, 64, 64}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 33816576.}, /* GFLOPS 0.299 x 1 = 0.299 */ {{3, 3}, {{1, 256, 13, 13}}, 384, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 299105664.}, /* GFLOPS 0.299 x 1 = 0.299 */ {{3, 3}, {{1, 384, 13, 13}}, 256, 2, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 299084032.}, /* GFLOPS 0.017 x 17 = 0.290 */ {{1, 1}, {{1, 32, 32, 64}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 17039360.}, /* GFLOPS 0.017 x 16 = 0.269 */ {{1, 1}, {{1, 128, 32, 64}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 16842752.}, /* GFLOPS 0.133 x 2 = 0.266 */ {{3, 3}, {{1, 128, 19, 19}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 133136800.}, + /* GFLOPS 0.266 x 1 = 0.266 */ {{1, 1}, {{1, 384, 52, 52}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 266160128.}, + /* GFLOPS 0.266 x 1 = 0.266 */ {{1, 1}, {{1, 768, 26, 26}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 265987072.}, /* GFLOPS 0.038 x 7 = 0.265 */ {{3, 3}, {{1, 16, 64, 128}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 37879808.}, + /* GFLOPS 0.019 x 14 = 0.264 */ {{3, 3}, {{1, 64, 16, 16}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 18890752.}, + /* GFLOPS 0.262 x 1 = 0.262 */ {{1, 1}, {{1, 2560, 20, 20}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 262195200.}, /* GFLOPS 0.126 x 2 = 0.252 */ {{3, 3}, {{1, 512, 5, 5}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 125812050.}, /* GFLOPS 0.248 x 1 = 0.248 */ {{1, 1}, {{1, 64, 150, 200}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 247680000.}, /* GFLOPS 0.040 x 6 = 0.240 */ {{1, 1}, {{1, 576, 19, 19}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 39958368.}, /* GFLOPS 0.080 x 3 = 0.240 */ {{3, 3}, {{1, 96, 19, 19}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 79893632.}, /* GFLOPS 0.240 x 1 = 0.240 */ {{3, 3}, {{1, 192, 38, 38}}, 192, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 239611584.}, /* GFLOPS 0.240 x 1 = 0.240 */ {{3, 3}, {{1, 192, 19, 19}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 239611584.}, + /* GFLOPS 0.079 x 3 = 0.237 */ {{1, 1}, {{1, 80, 32, 32}}, 480, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 79134720.}, /* GFLOPS 0.237 x 1 = 0.237 */ {{7, 7}, {{1, 3, 224, 224}}, 64, 1, {2, 2}, {1, 1}, {3, 3}, {0, 0}, "", false, 236830720.}, /* GFLOPS 0.237 x 1 = 0.237 */ {{7, 7}, {{1, 3, 224, 224}}, 64, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 236830720.}, + /* GFLOPS 0.118 x 2 = 0.236 */ {{3, 3}, {{1, 32, 80, 80}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 118169600.}, + /* GFLOPS 0.236 x 1 = 0.236 */ {{3, 3}, {{1, 256, 19, 19}}, 512, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 235980800.}, + /* GFLOPS 0.116 x 2 = 0.231 */ {{1, 1}, {{1, 24, 128, 128}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 115605504.}, /* GFLOPS 0.111 x 2 = 0.221 */ {{3, 3}, {{1, 192, 10, 10}}, 320, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 110624000.}, /* GFLOPS 0.213 x 1 = 0.213 */ {{3, 3}, {{1, 128, 38, 38}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 213018880.}, /* GFLOPS 0.213 x 1 = 0.213 */ {{3, 3}, {{1, 128, 19, 19}}, 256, 1, {1, 1}, {2, 2}, {2, 2}, {0, 0}, "", false, 213018880.}, /* GFLOPS 0.107 x 2 = 0.213 */ {{3, 3}, {{1, 128, 19, 19}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 106509440.}, /* GFLOPS 0.213 x 1 = 0.213 */ {{3, 3}, {{1, 256, 19, 19}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 212972672.}, + /* GFLOPS 0.213 x 1 = 0.213 */ {{3, 3}, {{1, 512, 38, 38}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 212949568.}, /* GFLOPS 0.212 x 1 = 0.212 */ {{7, 7}, {{1, 3, 300, 300}}, 32, 1, {2, 2}, {1, 1}, {3, 3}, {0, 0}, "", true, 212400000.}, /* GFLOPS 0.211 x 1 = 0.211 */ {{11, 11}, {{1, 3, 227, 227}}, 96, 1, {4, 4}, {1, 1}, {0, 0}, {0, 0}, "", true, 211120800.}, /* GFLOPS 0.210 x 1 = 0.210 */ {{3, 3}, {{1, 64, 38, 50}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 210307200.}, /* GFLOPS 0.210 x 1 = 0.210 */ {{1, 1}, {{1, 1024, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 209817600.}, /* GFLOPS 0.210 x 1 = 0.210 */ {{1, 1}, {{1, 1024, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 209817600.}, /* GFLOPS 0.104 x 2 = 0.208 */ {{3, 3}, {{1, 32, 75, 75}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 103860000.}, + /* GFLOPS 0.208 x 1 = 0.208 */ {{1, 1}, {{1, 16, 256, 256}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 207618048.}, /* GFLOPS 0.206 x 1 = 0.206 */ {{1, 1}, {{1, 256, 56, 56}}, 512, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "", false, 205922304.}, /* GFLOPS 0.206 x 1 = 0.206 */ {{1, 1}, {{1, 256, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 205922304.}, /* GFLOPS 0.103 x 2 = 0.206 */ {{1, 1}, {{1, 256, 56, 56}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 102961152.}, @@ -148,27 +279,35 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.190 x 1 = 0.190 */ {{1, 1}, {{1, 256, 38, 38}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 189637632.}, /* GFLOPS 0.190 x 1 = 0.190 */ {{1, 1}, {{1, 256, 38, 38}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 189637632.}, /* GFLOPS 0.047 x 4 = 0.190 */ {{1, 1}, {{1, 256, 38, 38}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 47409408.}, + /* GFLOPS 0.189 x 1 = 0.189 */ {{1, 1}, {{1, 1024, 19, 19}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 189360384.}, /* GFLOPS 0.038 x 5 = 0.189 */ {{3, 3}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 37814272.}, + /* GFLOPS 0.189 x 1 = 0.189 */ {{1, 1}, {{1, 1152, 16, 16}}, 320, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 188825600.}, /* GFLOPS 0.185 x 1 = 0.185 */ {{1, 1}, {{1, 128, 75, 75}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 185040000.}, /* GFLOPS 0.185 x 1 = 0.185 */ {{1, 1}, {{1, 128, 75, 75}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 185040000.}, /* GFLOPS 0.181 x 1 = 0.181 */ {{3, 3}, {{1, 160, 14, 14}}, 320, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 180696320.}, /* GFLOPS 0.181 x 1 = 0.181 */ {{3, 3}, {{1, 160, 14, 14}}, 320, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 180696320.}, /* GFLOPS 0.090 x 2 = 0.181 */ {{3, 3}, {{1, 224, 10, 10}}, 224, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 90339200.}, /* GFLOPS 0.180 x 1 = 0.180 */ {{1, 1}, {{1, 224, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 180232192.}, + /* GFLOPS 0.088 x 2 = 0.177 */ {{1, 1}, {{1, 1024, 13, 13}}, 255, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 88301655.}, /* GFLOPS 0.174 x 1 = 0.174 */ {{3, 3}, {{1, 96, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 173508608.}, /* GFLOPS 0.174 x 1 = 0.174 */ {{3, 3}, {{1, 96, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 173508608.}, /* GFLOPS 0.166 x 1 = 0.166 */ {{3, 3}, {{1, 160, 19, 19}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 166406560.}, /* GFLOPS 0.080 x 2 = 0.160 */ {{1, 1}, {{1, 576, 19, 19}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 79916736.}, /* GFLOPS 0.160 x 1 = 0.160 */ {{3, 3}, {{1, 128, 19, 19}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 159764160.}, + /* GFLOPS 0.160 x 1 = 0.160 */ {{3, 3}, {{1, 1024, 19, 19}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 159703512.}, /* GFLOPS 0.159 x 1 = 0.159 */ {{7, 7}, {{1, 3, 300, 300}}, 24, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 159300000.}, + /* GFLOPS 0.080 x 2 = 0.159 */ {{1, 1}, {{1, 40, 64, 64}}, 240, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 79626240.}, + /* GFLOPS 0.079 x 2 = 0.157 */ {{1, 1}, {{1, 480, 32, 32}}, 80, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 78725120.}, /* GFLOPS 0.155 x 1 = 0.155 */ {{1, 1}, {{1, 192, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 154542080.}, /* GFLOPS 0.146 x 1 = 0.146 */ {{3, 3}, {{1, 144, 14, 14}}, 288, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 146369664.}, /* GFLOPS 0.146 x 1 = 0.146 */ {{3, 3}, {{1, 144, 14, 14}}, 288, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 146369664.}, /* GFLOPS 0.072 x 2 = 0.144 */ {{1, 1}, {{1, 1024, 10, 10}}, 352, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 72124800.}, /* GFLOPS 0.140 x 1 = 0.140 */ {{1, 1}, {{1, 576, 38, 50}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 140204800.}, + /* GFLOPS 0.139 x 1 = 0.139 */ {{3, 3}, {{1, 256, 5, 5}}, 1206, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 138961350.}, /* GFLOPS 0.017 x 8 = 0.138 */ {{1, 1}, {{1, 16, 64, 128}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 17301504.}, /* GFLOPS 0.067 x 2 = 0.133 */ {{1, 1}, {{1, 576, 19, 19}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 66597280.}, /* GFLOPS 0.133 x 1 = 0.133 */ {{3, 3}, {{1, 128, 38, 38}}, 160, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 133136800.}, + /* GFLOPS 0.044 x 3 = 0.133 */ {{1, 1}, {{1, 512, 13, 13}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 44345600.}, /* GFLOPS 0.129 x 1 = 0.129 */ {{1, 1}, {{1, 160, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 128851968.}, /* GFLOPS 0.128 x 1 = 0.128 */ {{3, 3}, {{1, 64, 24, 24}}, 192, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 127512576.}, /* GFLOPS 0.120 x 1 = 0.120 */ {{5, 5}, {{1, 32, 28, 28}}, 96, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 120497664.}, @@ -176,22 +315,35 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.040 x 3 = 0.120 */ {{1, 1}, {{1, 96, 19, 19}}, 576, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 40131648.}, /* GFLOPS 0.118 x 1 = 0.118 */ {{1, 1}, {{1, 320, 38, 38}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 118477312.}, /* GFLOPS 0.017 x 7 = 0.118 */ {{1, 1}, {{1, 64, 64, 128}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 16908288.}, + /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 64, 80, 80}}, 64, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 118067200.}, + /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 64, 40, 40}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 118067200.}, /* GFLOPS 0.039 x 3 = 0.118 */ {{1, 1}, {{1, 1024, 10, 10}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 39340800.}, + /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 128, 40, 40}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 118016000.}, + /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 128, 20, 20}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 118016000.}, + /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 256, 20, 20}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 117990400.}, /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 256, 19, 19}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 117990400.}, /* GFLOPS 0.058 x 2 = 0.116 */ {{3, 3}, {{1, 16, 56, 56}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 58003456.}, /* GFLOPS 0.058 x 2 = 0.116 */ {{3, 3}, {{1, 32, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 57903104.}, /* GFLOPS 0.058 x 2 = 0.116 */ {{3, 3}, {{1, 64, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 57852928.}, /* GFLOPS 0.116 x 1 = 0.116 */ {{3, 3}, {{1, 128, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 115655680.}, /* GFLOPS 0.116 x 1 = 0.116 */ {{3, 3}, {{1, 128, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 115655680.}, + /* GFLOPS 0.115 x 1 = 0.115 */ {{3, 3}, {{1, 3, 512, 512}}, 32, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 115343360.}, + /* GFLOPS 0.114 x 1 = 0.114 */ {{1, 1}, {{1, 144, 128, 128}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 113639424.}, /* GFLOPS 0.112 x 1 = 0.112 */ {{1, 1}, {{1, 1024, 10, 10}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 111875400.}, + /* GFLOPS 0.110 x 1 = 0.110 */ {{1, 1}, {{1, 480, 32, 32}}, 112, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 110215168.}, + /* GFLOPS 0.107 x 1 = 0.107 */ {{1, 1}, {{1, 64, 32, 32}}, 810, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 106997760.}, /* GFLOPS 0.036 x 3 = 0.107 */ {{1, 1}, {{1, 192, 38, 38}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 35580160.}, /* GFLOPS 0.107 x 1 = 0.107 */ {{3, 3}, {{1, 32, 75, 75}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 106648064.}, /* GFLOPS 0.107 x 1 = 0.107 */ {{3, 3}, {{1, 64, 38, 38}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 106555648.}, + /* GFLOPS 0.105 x 1 = 0.105 */ {{1, 1}, {{1, 256, 40, 40}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 105062400.}, + /* GFLOPS 0.105 x 1 = 0.105 */ {{1, 1}, {{1, 512, 20, 20}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 104960000.}, /* GFLOPS 0.105 x 1 = 0.105 */ {{1, 1}, {{1, 512, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 104960000.}, /* GFLOPS 0.105 x 1 = 0.105 */ {{1, 1}, {{1, 512, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 104960000.}, + /* GFLOPS 0.105 x 1 = 0.105 */ {{1, 1}, {{1, 1024, 10, 10}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 104908800.}, /* GFLOPS 0.103 x 1 = 0.103 */ {{1, 1}, {{1, 128, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 103161856.}, /* GFLOPS 0.051 x 2 = 0.103 */ {{1, 1}, {{1, 256, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 51480576.}, /* GFLOPS 0.051 x 2 = 0.103 */ {{1, 1}, {{1, 256, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 51480576.}, + /* GFLOPS 0.008 x 12 = 0.101 */ {{1, 1}, {{1, 64, 32, 32}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 8454144.}, /* GFLOPS 0.101 x 1 = 0.101 */ {{1, 1}, {{1, 512, 19, 19}}, 273, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 101016825.}, /* GFLOPS 0.096 x 1 = 0.096 */ {{1, 1}, {{1, 480, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 96438272.}, /* GFLOPS 0.095 x 1 = 0.095 */ {{1, 1}, {{1, 128, 38, 38}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 95003648.}, @@ -208,8 +360,10 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.092 x 1 = 0.092 */ {{1, 1}, {{1, 192, 75, 100}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 92400000.}, /* GFLOPS 0.090 x 1 = 0.090 */ {{1, 1}, {{1, 448, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 90015744.}, /* GFLOPS 0.045 x 2 = 0.090 */ {{3, 3}, {{1, 576, 19, 19}}, 12, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 44918508.}, + /* GFLOPS 0.044 x 2 = 0.089 */ {{1, 1}, {{1, 256, 26, 26}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 44388864.}, /* GFLOPS 0.089 x 1 = 0.089 */ {{3, 3}, {{1, 112, 14, 14}}, 224, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 88554368.}, /* GFLOPS 0.089 x 1 = 0.089 */ {{3, 3}, {{1, 112, 14, 14}}, 224, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 88554368.}, + /* GFLOPS 0.088 x 1 = 0.088 */ {{1, 1}, {{1, 256, 26, 26}}, 255, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 88430940.}, /* GFLOPS 0.021 x 4 = 0.084 */ {{5, 1}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {1, 1}, {2, 0}, {0, 0}, "", false, 21037056.}, /* GFLOPS 0.021 x 4 = 0.084 */ {{1, 5}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {1, 1}, {0, 2}, {0, 0}, "", true, 21037056.}, /* GFLOPS 0.084 x 1 = 0.084 */ {{1, 1}, {{1, 416, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 83593216.}, @@ -217,9 +371,13 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.040 x 2 = 0.080 */ {{1, 1}, {{1, 576, 19, 19}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 39958368.}, /* GFLOPS 0.040 x 2 = 0.079 */ {{1, 1}, {{1, 24, 75, 75}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 39690000.}, /* GFLOPS 0.040 x 2 = 0.079 */ {{3, 3}, {{1, 3, 300, 300}}, 32, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 39600000.}, + /* GFLOPS 0.079 x 1 = 0.079 */ {{1, 1}, {{1, 240, 64, 64}}, 40, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 78807040.}, + /* GFLOPS 0.079 x 1 = 0.079 */ {{1, 1}, {{1, 384, 40, 40}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 78745600.}, /* GFLOPS 0.077 x 1 = 0.077 */ {{1, 1}, {{1, 96, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 77471744.}, /* GFLOPS 0.077 x 1 = 0.077 */ {{3, 3}, {{1, 192, 10, 10}}, 224, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 77436800.}, /* GFLOPS 0.077 x 1 = 0.077 */ {{1, 1}, {{1, 384, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 77170688.}, + /* GFLOPS 0.076 x 1 = 0.076 */ {{3, 3}, {{1, 3, 416, 416}}, 32, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 76144640.}, + /* GFLOPS 0.076 x 1 = 0.076 */ {{1, 1}, {{1, 96, 128, 128}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 75890688.}, /* GFLOPS 0.038 x 2 = 0.076 */ {{3, 3}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {8, 8}, {8, 8}, {0, 0}, "", true, 37814272.}, /* GFLOPS 0.038 x 2 = 0.076 */ {{3, 3}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {4, 4}, {4, 4}, {0, 0}, "", true, 37814272.}, /* GFLOPS 0.038 x 2 = 0.076 */ {{3, 3}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {2, 2}, {2, 2}, {0, 0}, "", true, 37814272.}, @@ -230,6 +388,9 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.071 x 1 = 0.071 */ {{1, 1}, {{1, 24, 150, 150}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 70560000.}, /* GFLOPS 0.070 x 1 = 0.070 */ {{3, 3}, {{1, 96, 14, 14}}, 208, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 70487872.}, /* GFLOPS 0.069 x 1 = 0.069 */ {{3, 3}, {{1, 96, 14, 14}}, 204, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 69132336.}, + /* GFLOPS 0.068 x 1 = 0.068 */ {{1, 1}, {{1, 32, 256, 256}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 68157440.}, + /* GFLOPS 0.005 x 14 = 0.066 */ {{3, 3}, {{1, 64, 8, 8}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 4722688.}, + /* GFLOPS 0.066 x 1 = 0.066 */ {{1, 1}, {{1, 672, 16, 16}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 66109440.}, /* GFLOPS 0.066 x 1 = 0.066 */ {{1, 1}, {{1, 1280, 10, 10}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 65561600.}, /* GFLOPS 0.033 x 2 = 0.065 */ {{3, 3}, {{1, 48, 14, 14}}, 192, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 32551680.}, /* GFLOPS 0.065 x 1 = 0.065 */ {{3, 3}, {{1, 192, 7, 7}}, 384, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 65046912.}, @@ -239,6 +400,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.032 x 2 = 0.064 */ {{3, 3}, {{1, 96, 12, 12}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 31868928.}, /* GFLOPS 0.061 x 1 = 0.061 */ {{1, 1}, {{1, 960, 10, 10}}, 320, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 61472000.}, /* GFLOPS 0.031 x 2 = 0.061 */ {{1, 1}, {{1, 960, 10, 10}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 30736000.}, + /* GFLOPS 0.061 x 1 = 0.061 */ {{1, 1}, {{1, 512, 46, 46}}, 28, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 60729200.}, /* GFLOPS 0.060 x 1 = 0.060 */ {{3, 3}, {{1, 96, 38, 38}}, 96, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 59920224.}, /* GFLOPS 0.059 x 1 = 0.059 */ {{1, 1}, {{1, 320, 38, 38}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 59238656.}, /* GFLOPS 0.059 x 1 = 0.059 */ {{3, 3}, {{1, 128, 19, 19}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 59008000.}, @@ -253,6 +415,11 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.053 x 1 = 0.053 */ {{3, 3}, {{1, 128, 38, 38}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 53254720.}, /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 528, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 53036032.}, /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 528, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 53036032.}, + /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 64, 80, 80}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52838400.}, + /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 64, 40, 40}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52838400.}, + /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 128, 80, 80}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52633600.}, + /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 128, 20, 20}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52633600.}, + /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 256, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52531200.}, /* GFLOPS 0.052 x 1 = 0.052 */ {{1, 1}, {{1, 1024, 10, 10}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 52454400.}, /* GFLOPS 0.052 x 1 = 0.052 */ {{1, 1}, {{1, 1024, 10, 10}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 52454400.}, /* GFLOPS 0.052 x 1 = 0.052 */ {{1, 1}, {{1, 1024, 10, 10}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52454400.}, @@ -268,6 +435,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.050 x 1 = 0.050 */ {{1, 1}, {{1, 992, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 49799680.}, /* GFLOPS 0.048 x 1 = 0.048 */ {{1, 1}, {{1, 960, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 48194048.}, /* GFLOPS 0.047 x 1 = 0.047 */ {{1, 1}, {{1, 256, 19, 19}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 47409408.}, + /* GFLOPS 0.047 x 1 = 0.047 */ {{1, 1}, {{1, 144, 64, 64}}, 40, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 47349760.}, /* GFLOPS 0.047 x 1 = 0.047 */ {{1, 1}, {{1, 512, 38, 50}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 46740000.}, /* GFLOPS 0.047 x 1 = 0.047 */ {{1, 1}, {{1, 928, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 46588416.}, /* GFLOPS 0.046 x 1 = 0.046 */ {{1, 1}, {{1, 64, 75, 75}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 46440000.}, @@ -280,6 +448,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.045 x 1 = 0.045 */ {{3, 3}, {{1, 3, 227, 227}}, 64, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "", true, 44946880.}, /* GFLOPS 0.044 x 1 = 0.044 */ {{3, 3}, {{1, 128, 19, 19}}, 192, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 44256000.}, /* GFLOPS 0.044 x 1 = 0.044 */ {{3, 3}, {{1, 1024, 10, 10}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 44239200.}, + /* GFLOPS 0.044 x 1 = 0.044 */ {{1, 1}, {{1, 512, 13, 13}}, 255, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 44172375.}, /* GFLOPS 0.043 x 1 = 0.043 */ {{7, 7}, {{1, 3, 96, 96}}, 64, 1, {2, 2}, {1, 1}, {3, 3}, {0, 0}, "", true, 43499520.}, /* GFLOPS 0.043 x 1 = 0.043 */ {{1, 1}, {{1, 864, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 43377152.}, /* GFLOPS 0.042 x 1 = 0.042 */ {{1, 1}, {{1, 832, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 41771520.}, @@ -289,6 +458,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.040 x 1 = 0.040 */ {{3, 3}, {{1, 64, 19, 19}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 39958368.}, /* GFLOPS 0.040 x 1 = 0.040 */ {{3, 3}, {{1, 256, 19, 19}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 39932376.}, /* GFLOPS 0.040 x 1 = 0.040 */ {{3, 3}, {{1, 3, 300, 300}}, 32, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 39600000.}, + /* GFLOPS 0.039 x 1 = 0.039 */ {{1, 1}, {{1, 240, 32, 32}}, 80, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 39403520.}, /* GFLOPS 0.039 x 1 = 0.039 */ {{1, 1}, {{1, 144, 75, 75}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 39015000.}, /* GFLOPS 0.039 x 1 = 0.039 */ {{1, 1}, {{1, 192, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 38635520.}, /* GFLOPS 0.039 x 1 = 0.039 */ {{1, 1}, {{1, 768, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 38560256.}, @@ -297,9 +467,11 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.036 x 1 = 0.036 */ {{1, 1}, {{1, 480, 14, 14}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 36164352.}, /* GFLOPS 0.018 x 2 = 0.036 */ {{1, 1}, {{1, 192, 38, 38}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 17790080.}, /* GFLOPS 0.035 x 1 = 0.035 */ {{1, 1}, {{1, 704, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 35348992.}, + /* GFLOPS 0.035 x 1 = 0.035 */ {{1, 1}, {{1, 512, 46, 46}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 34702400.}, /* GFLOPS 0.034 x 1 = 0.034 */ {{1, 1}, {{1, 672, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 33743360.}, /* GFLOPS 0.034 x 1 = 0.034 */ {{1, 1}, {{1, 128, 32, 64}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 33685504.}, /* GFLOPS 0.034 x 1 = 0.034 */ {{2, 2}, {{1, 64, 64, 128}}, 32, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "", false, 33619968.}, + /* GFLOPS 0.033 x 1 = 0.033 */ {{3, 3}, {{1, 256, 3, 3}}, 804, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 33350724.}, /* GFLOPS 0.033 x 1 = 0.033 */ {{1, 1}, {{1, 528, 14, 14}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 33147520.}, /* GFLOPS 0.033 x 1 = 0.033 */ {{1, 1}, {{1, 528, 14, 14}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 33147520.}, /* GFLOPS 0.033 x 1 = 0.033 */ {{1, 1}, {{1, 1024, 10, 10}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 32784000.}, @@ -307,24 +479,29 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.032 x 1 = 0.032 */ {{1, 1}, {{1, 512, 14, 14}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 32144000.}, /* GFLOPS 0.032 x 1 = 0.032 */ {{1, 1}, {{1, 640, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 32137728.}, /* GFLOPS 0.032 x 1 = 0.032 */ {{1, 1}, {{1, 508, 14, 14}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 31893120.}, + /* GFLOPS 0.011 x 3 = 0.032 */ {{1, 1}, {{1, 320, 16, 16}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 10502144.}, /* GFLOPS 0.031 x 1 = 0.031 */ {{1, 1}, {{1, 832, 7, 7}}, 384, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 31328640.}, /* GFLOPS 0.031 x 1 = 0.031 */ {{1, 1}, {{1, 832, 7, 7}}, 384, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 31328640.}, /* GFLOPS 0.031 x 1 = 0.031 */ {{1, 1}, {{1, 608, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 30532096.}, + /* GFLOPS 0.015 x 2 = 0.030 */ {{1, 1}, {{1, 128, 46, 46}}, 28, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 15226736.}, /* GFLOPS 0.015 x 2 = 0.030 */ {{5, 5}, {{1, 24, 14, 14}}, 64, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 15065344.}, /* GFLOPS 0.015 x 2 = 0.030 */ {{5, 5}, {{1, 24, 14, 14}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 15065344.}, /* GFLOPS 0.015 x 2 = 0.030 */ {{5, 5}, {{1, 48, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 15059072.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{3, 3}, {{1, 256, 10, 10}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 29497600.}, + /* GFLOPS 0.015 x 2 = 0.029 */ {{1, 1}, {{1, 112, 32, 32}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 14745600.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{1, 1}, {{1, 192, 28, 28}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 28976640.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{1, 1}, {{1, 192, 28, 28}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 28976640.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{1, 1}, {{1, 512, 14, 14}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 28929600.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{1, 1}, {{1, 512, 14, 14}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 28929600.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{1, 1}, {{1, 576, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 28926464.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 544, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 27320832.}, + /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 64, 16, 16}}, 810, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 26749440.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 384, 19, 19}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 26650464.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 576, 19, 19}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 26638912.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{3, 3}, {{1, 128, 38, 38}}, 8, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 26627360.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 528, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 26518016.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 528, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 26518016.}, + /* GFLOPS 0.009 x 3 = 0.026 */ {{1, 1}, {{1, 128, 46, 46}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 8700992.}, /* GFLOPS 0.026 x 1 = 0.026 */ {{1, 1}, {{1, 96, 75, 75}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 26055000.}, /* GFLOPS 0.026 x 1 = 0.026 */ {{1, 1}, {{1, 64, 56, 56}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 25890816.}, /* GFLOPS 0.026 x 1 = 0.026 */ {{1, 1}, {{1, 64, 56, 56}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 25890816.}, @@ -336,6 +513,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.013 x 2 = 0.026 */ {{1, 1}, {{1, 256, 28, 28}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 12870144.}, /* GFLOPS 0.026 x 1 = 0.026 */ {{1, 1}, {{1, 512, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 25715200.}, /* GFLOPS 0.013 x 2 = 0.026 */ {{1, 1}, {{1, 512, 14, 14}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 12857600.}, + /* GFLOPS 0.002 x 12 = 0.025 */ {{1, 1}, {{1, 64, 16, 16}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 2113536.}, /* GFLOPS 0.024 x 1 = 0.024 */ {{1, 1}, {{1, 480, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 24109568.}, /* GFLOPS 0.024 x 1 = 0.024 */ {{1, 1}, {{1, 128, 38, 38}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "", false, 23750912.}, /* GFLOPS 0.024 x 1 = 0.024 */ {{1, 1}, {{1, 256, 19, 19}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 23704704.}, @@ -345,7 +523,9 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.023 x 1 = 0.023 */ {{1, 1}, {{1, 448, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 22503936.}, /* GFLOPS 0.023 x 1 = 0.023 */ {{1, 1}, {{1, 512, 14, 14}}, 112, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 22500800.}, /* GFLOPS 0.022 x 1 = 0.022 */ {{1, 1}, {{1, 508, 14, 14}}, 112, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 22325184.}, + /* GFLOPS 0.022 x 1 = 0.022 */ {{3, 3}, {{1, 512, 10, 10}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 22120800.}, /* GFLOPS 0.021 x 1 = 0.021 */ {{3, 3}, {{1, 128, 12, 12}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 21242880.}, + /* GFLOPS 0.021 x 1 = 0.021 */ {{1, 1}, {{1, 40, 64, 64}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 21233664.}, /* GFLOPS 0.021 x 1 = 0.021 */ {{1, 1}, {{1, 416, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 20898304.}, /* GFLOPS 0.021 x 1 = 0.021 */ {{1, 1}, {{1, 832, 7, 7}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 20885760.}, /* GFLOPS 0.021 x 1 = 0.021 */ {{1, 1}, {{1, 832, 7, 7}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 20885760.}, @@ -360,6 +540,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.019 x 1 = 0.019 */ {{1, 1}, {{1, 192, 28, 28}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 19317760.}, /* GFLOPS 0.019 x 1 = 0.019 */ {{1, 1}, {{1, 192, 28, 28}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 19317760.}, /* GFLOPS 0.019 x 1 = 0.019 */ {{1, 1}, {{1, 384, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 19292672.}, + /* GFLOPS 0.019 x 1 = 0.019 */ {{1, 1}, {{1, 64, 64, 64}}, 36, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 19021824.}, /* GFLOPS 0.018 x 1 = 0.018 */ {{1, 1}, {{1, 576, 10, 10}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 18448000.}, /* GFLOPS 0.018 x 1 = 0.018 */ {{1, 1}, {{1, 480, 14, 14}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 18082176.}, /* GFLOPS 0.018 x 1 = 0.018 */ {{1, 1}, {{1, 480, 14, 14}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 18082176.}, @@ -371,13 +552,16 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.016 x 1 = 0.016 */ {{1, 1}, {{1, 832, 7, 7}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 15664320.}, /* GFLOPS 0.015 x 1 = 0.015 */ {{5, 5}, {{1, 48, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 15059072.}, /* GFLOPS 0.015 x 1 = 0.015 */ {{5, 5}, {{1, 32, 12, 12}}, 64, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 14754816.}, + /* GFLOPS 0.015 x 1 = 0.015 */ {{3, 3}, {{1, 128, 10, 10}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 14752000.}, /* GFLOPS 0.014 x 1 = 0.014 */ {{1, 1}, {{1, 288, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 14475776.}, /* GFLOPS 0.014 x 1 = 0.014 */ {{1, 1}, {{1, 512, 5, 5}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 13991250.}, /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 144, 38, 38}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 13354112.}, /* GFLOPS 0.007 x 2 = 0.013 */ {{1, 1}, {{1, 16, 56, 56}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6623232.}, + /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 512, 10, 10}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 13120000.}, /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 832, 7, 7}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 13053600.}, /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 832, 7, 7}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 13053600.}, /* GFLOPS 0.007 x 2 = 0.013 */ {{1, 1}, {{1, 32, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6522880.}, + /* GFLOPS 0.001 x 11 = 0.013 */ {{3, 3}, {{1, 64, 4, 4}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1180672.}, /* GFLOPS 0.006 x 2 = 0.013 */ {{1, 1}, {{1, 64, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6472704.}, /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 128, 56, 56}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 12895232.}, /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 256, 28, 28}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 12870144.}, @@ -394,6 +578,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.012 x 1 = 0.012 */ {{1, 1}, {{1, 640, 6, 6}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 11805696.}, /* GFLOPS 0.012 x 1 = 0.012 */ {{1, 1}, {{1, 928, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 11647104.}, /* GFLOPS 0.011 x 1 = 0.011 */ {{1, 1}, {{1, 896, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 11245696.}, + /* GFLOPS 0.011 x 1 = 0.011 */ {{1, 1}, {{1, 256, 13, 13}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 11097216.}, /* GFLOPS 0.011 x 1 = 0.011 */ {{3, 3}, {{1, 256, 10, 10}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 11061600.}, /* GFLOPS 0.006 x 2 = 0.011 */ {{3, 3}, {{1, 512, 5, 5}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 5530200.}, /* GFLOPS 0.011 x 1 = 0.011 */ {{1, 1}, {{1, 864, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 10844288.}, @@ -417,13 +602,13 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.008 x 1 = 0.008 */ {{1, 1}, {{1, 608, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 7633024.}, /* GFLOPS 0.008 x 1 = 0.008 */ {{5, 5}, {{1, 16, 14, 14}}, 48, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 7535808.}, /* GFLOPS 0.008 x 1 = 0.008 */ {{5, 5}, {{1, 16, 14, 14}}, 48, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 7535808.}, - /* GFLOPS 0.004 x 2 = 0.007 */ {{3, 3}, {{1, 64, 5, 5}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 3689600.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 640, 6, 6}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 7378560.}, /* GFLOPS 0.004 x 2 = 0.007 */ {{1, 1}, {{1, 48, 14, 14}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 3650304.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 384, 14, 14}}, 48, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 7234752.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 576, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 7231616.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 256, 12, 12}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 7091712.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 544, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 6830208.}, + /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 64, 8, 8}}, 810, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 6687360.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{3, 3}, {{1, 160, 6, 6}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 6637824.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 528, 14, 14}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6629504.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 528, 14, 14}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 6629504.}, @@ -434,11 +619,13 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.006 x 1 = 0.006 */ {{1, 1}, {{1, 512, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 6428800.}, /* GFLOPS 0.006 x 1 = 0.006 */ {{1, 1}, {{1, 512, 14, 14}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6428800.}, /* GFLOPS 0.006 x 1 = 0.006 */ {{1, 1}, {{1, 512, 14, 14}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 6428800.}, + /* GFLOPS 0.001 x 12 = 0.006 */ {{1, 1}, {{1, 64, 8, 8}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 528384.}, /* GFLOPS 0.006 x 1 = 0.006 */ {{3, 3}, {{1, 256, 10, 10}}, 12, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 5530800.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 192, 12, 12}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 5322240.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{3, 3}, {{1, 128, 5, 5}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 5310720.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{3, 3}, {{1, 128, 5, 5}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 5310720.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{3, 3}, {{1, 128, 5, 5}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 5310720.}, + /* GFLOPS 0.005 x 1 = 0.005 */ {{3, 3}, {{1, 128, 5, 5}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 5310720.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 1024, 10, 10}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4917600.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 1024, 10, 10}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 4917600.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 192, 28, 28}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4829440.}, @@ -446,6 +633,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 256, 14, 14}}, 48, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4826304.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 512, 14, 14}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 4821600.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 508, 14, 14}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 4783968.}, + /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 64, 32, 32}}, 36, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 4755456.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 64, 24, 24}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4755456.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 256, 12, 12}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4727808.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 1024, 3, 3}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4720896.}, @@ -455,6 +643,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.004 x 1 = 0.004 */ {{1, 1}, {{1, 16, 128, 256}}, 4, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 4325376.}, /* GFLOPS 0.004 x 1 = 0.004 */ {{1, 1}, {{1, 64, 64, 128}}, 4, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 4227072.}, /* GFLOPS 0.004 x 1 = 0.004 */ {{1, 1}, {{1, 832, 7, 7}}, 48, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 3916080.}, + /* GFLOPS 0.004 x 1 = 0.004 */ {{3, 3}, {{1, 256, 1, 1}}, 804, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 3705636.}, /* GFLOPS 0.004 x 1 = 0.004 */ {{5, 5}, {{1, 16, 12, 12}}, 32, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 3691008.}, /* GFLOPS 0.004 x 1 = 0.004 */ {{3, 3}, {{1, 64, 10, 10}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 3689600.}, /* GFLOPS 0.004 x 1 = 0.004 */ {{5, 5}, {{1, 32, 6, 6}}, 64, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 3688704.}, @@ -470,6 +659,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.003 x 1 = 0.003 */ {{1, 1}, {{1, 480, 14, 14}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 3013696.}, /* GFLOPS 0.003 x 1 = 0.003 */ {{1, 1}, {{1, 320, 12, 12}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2953728.}, /* GFLOPS 0.003 x 1 = 0.003 */ {{1, 1}, {{1, 640, 6, 6}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2951424.}, + /* GFLOPS 0.003 x 1 = 0.003 */ {{3, 3}, {{1, 256, 5, 5}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 2765400.}, /* GFLOPS 0.003 x 1 = 0.003 */ {{3, 3}, {{1, 128, 5, 5}}, 128, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 2655360.}, /* GFLOPS 0.003 x 1 = 0.003 */ {{1, 1}, {{1, 832, 7, 7}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2610720.}, /* GFLOPS 0.003 x 1 = 0.003 */ {{1, 1}, {{1, 256, 3, 3}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 2520882.}, @@ -482,32 +672,46 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 508, 4, 4}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 2082816.}, /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 1024, 1, 1}}, 1000, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2049000.}, /* GFLOPS 0.001 x 2 = 0.002 */ {{3, 3}, {{1, 256, 3, 3}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 995544.}, - /* GFLOPS 0.001 x 2 = 0.002 */ {{3, 3}, {{1, 128, 5, 5}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 922000.}, /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 1024, 3, 3}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1770336.}, + /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 64, 4, 4}}, 810, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 1671840.}, + /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 32, 80, 80}}, 4, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1664000.}, + /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 256, 5, 5}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1641600.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 640, 6, 6}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1475712.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 128, 5, 5}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1383000.}, + /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 64, 5, 5}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1328256.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 736, 3, 3}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1272672.}, + /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 64, 16, 16}}, 36, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 1188864.}, + /* GFLOPS 0.000 x 9 = 0.001 */ {{1, 1}, {{1, 64, 4, 4}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 132096.}, + /* GFLOPS 0.001 x 2 = 0.001 */ {{1, 1}, {{1, 256, 3, 3}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 590976.}, /* GFLOPS 0.001 x 2 = 0.001 */ {{1, 1}, {{1, 256, 3, 3}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 590976.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 128, 3, 3}}, 128, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1180160.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 256, 2, 2}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1120392.}, - /* GFLOPS 0.000 x 2 = 0.001 */ {{3, 3}, {{1, 128, 5, 5}}, 8, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 461000.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 192, 12, 12}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 887040.}, /* GFLOPS 0.000 x 2 = 0.001 */ {{3, 3}, {{1, 256, 2, 2}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 442464.}, - /* GFLOPS 0.000 x 2 = 0.001 */ {{1, 1}, {{1, 128, 5, 5}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 411200.}, + /* GFLOPS 0.000 x 2 = 0.001 */ {{1, 1}, {{1, 32, 80, 80}}, 1, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 416000.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 128, 5, 5}}, 12, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 691500.}, + /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 256, 3, 3}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 663696.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 640, 2, 2}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 655872.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 512, 5, 5}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 615000.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 512, 5, 5}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 615000.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 128, 3, 3}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 592128.}, - /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 256, 3, 3}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 590976.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 256, 3, 3}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 590976.}, + /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 128, 3, 3}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 590080.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 256, 3, 3}}, 126, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 581742.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 256, 4, 4}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 525312.}, + /* GFLOPS 0.000 x 4 = 0.000 */ {{1, 1}, {{1, 48, 1, 1}}, 1152, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 111744.}, + /* GFLOPS 0.000 x 4 = 0.000 */ {{1, 1}, {{1, 1152, 1, 1}}, 48, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 110640.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 5, 5}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 411200.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 128, 3, 3}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 331920.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 192, 5, 5}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 308000.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 64, 8, 8}}, 36, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 297216.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 2, 2}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 263168.}, /* GFLOPS 0.000 x 2 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 131328.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 126, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 258552.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 1024, 1, 1}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 196704.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 128, 3, 3}}, 8, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 165960.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 3, 3}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 148032.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 64, 3, 3}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 147584.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 64, 2, 2}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 147584.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 64, 2, 2}}, 128, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 147584.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 64, 2, 2}}, 128, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 147584.}, @@ -515,16 +719,32 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 1, 1}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 140322.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 131328.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 131328.}, + /* GFLOPS 0.000 x 3 = 0.000 */ {{1, 1}, {{1, 28, 1, 1}}, 672, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 38304.}, + /* GFLOPS 0.000 x 3 = 0.000 */ {{1, 1}, {{1, 672, 1, 1}}, 28, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 37660.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 3, 3}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 110808.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 3, 3}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 110808.}, /* GFLOPS 0.000 x 2 = 0.000 */ {{3, 3}, {{1, 128, 1, 1}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 55320.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 64, 4, 4}}, 36, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 74304.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 64, 2, 2}}, 64, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 73792.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 256, 1, 1}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 73744.}, + /* GFLOPS 0.000 x 3 = 0.000 */ {{1, 1}, {{1, 20, 1, 1}}, 480, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 19680.}, + /* GFLOPS 0.000 x 3 = 0.000 */ {{1, 1}, {{1, 480, 1, 1}}, 20, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 19220.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 49248.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 49248.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 128, 1, 1}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 36880.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 1, 1}}, 126, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 32382.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 128, 1, 1}}, 8, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 18440.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 64, 1, 1}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 16512.}, + /* GFLOPS 0.000 x 2 = 0.000 */ {{1, 1}, {{1, 10, 1, 1}}, 240, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 5040.}, + /* GFLOPS 0.000 x 2 = 0.000 */ {{1, 1}, {{1, 240, 1, 1}}, 10, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 4810.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 1, 1}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6168.}, - /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 1, 1}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 6168.} + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 1, 1}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 6168.}, + /* GFLOPS 0.000 x 2 = 0.000 */ {{1, 1}, {{1, 6, 1, 1}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1872.}, + /* GFLOPS 0.000 x 2 = 0.000 */ {{1, 1}, {{1, 144, 1, 1}}, 6, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1734.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 4, 1, 1}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 864.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 96, 1, 1}}, 4, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 772.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 8, 1, 1}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 544.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 32, 1, 1}}, 8, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 520.} }; struct ConvParamID { From de1a45987964d466a13b3f9ba2e1e59b3be38cca Mon Sep 17 00:00:00 2001 From: Dale Phurrough Date: Fri, 10 Sep 2021 17:59:56 +0200 Subject: [PATCH 197/376] fix opencv/opencv#20613 * copy 4.x selectOpenCLDevice() -- it is compatible * filter platforms rather than trying only first matching * this works on 3.4 and 4.x master --- modules/core/src/ocl.cpp | 59 ++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index a550c1d91ae0..e93e3094be53 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -1670,7 +1670,7 @@ static bool parseOpenCLDeviceConfiguration(const std::string& configurationStr, split(configurationStr, ':', parts); if (parts.size() > 3) { - std::cerr << "ERROR: Invalid configuration string for OpenCL device" << std::endl; + CV_LOG_ERROR(NULL, "OpenCL: Invalid configuration string for OpenCL device: " << configurationStr); return false; } if (parts.size() > 2) @@ -1687,22 +1687,20 @@ static bool parseOpenCLDeviceConfiguration(const std::string& configurationStr, } #if defined WINRT || defined _WIN32_WCE -static cl_device_id selectOpenCLDevice() +static cl_device_id selectOpenCLDevice(const char* configuration = NULL) { + CV_UNUSED(configuration) return NULL; } #else -// std::tolower is int->int -static char char_tolower(char ch) -{ - return (char)std::tolower((int)ch); -} -static cl_device_id selectOpenCLDevice() +static cl_device_id selectOpenCLDevice(const char* configuration = NULL) { std::string platform, deviceName; std::vector deviceTypes; - const char* configuration = getenv("OPENCV_OPENCL_DEVICE"); + if (!configuration) + configuration = getenv("OPENCV_OPENCL_DEVICE"); + if (configuration && (strcmp(configuration, "disabled") == 0 || !parseOpenCLDeviceConfiguration(std::string(configuration), platform, deviceTypes, deviceName) @@ -1747,22 +1745,24 @@ static cl_device_id selectOpenCLDevice() platforms.resize(numPlatforms); } - int selectedPlatform = -1; if (platform.length() > 0) { - for (size_t i = 0; i < platforms.size(); i++) + for (std::vector::iterator currentPlatform = platforms.begin(); currentPlatform != platforms.end();) { std::string name; - CV_OCL_DBG_CHECK(getStringInfo(clGetPlatformInfo, platforms[i], CL_PLATFORM_NAME, name)); + CV_OCL_DBG_CHECK(getStringInfo(clGetPlatformInfo, *currentPlatform, CL_PLATFORM_NAME, name)); if (name.find(platform) != std::string::npos) { - selectedPlatform = (int)i; - break; + ++currentPlatform; + } + else + { + currentPlatform = platforms.erase(currentPlatform); } } - if (selectedPlatform == -1) + if (platforms.size() == 0) { - std::cerr << "ERROR: Can't find OpenCL platform by name: " << platform << std::endl; + CV_LOG_ERROR(NULL, "OpenCL: Can't find OpenCL platform by name: " << platform); goto not_found; } } @@ -1781,7 +1781,7 @@ static cl_device_id selectOpenCLDevice() { int deviceType = 0; std::string tempStrDeviceType = deviceTypes[t]; - std::transform(tempStrDeviceType.begin(), tempStrDeviceType.end(), tempStrDeviceType.begin(), char_tolower); + std::transform(tempStrDeviceType.begin(), tempStrDeviceType.end(), tempStrDeviceType.begin(), details::char_tolower); if (tempStrDeviceType == "gpu" || tempStrDeviceType == "dgpu" || tempStrDeviceType == "igpu") deviceType = Device::TYPE_GPU; @@ -1793,17 +1793,15 @@ static cl_device_id selectOpenCLDevice() deviceType = Device::TYPE_ALL; else { - std::cerr << "ERROR: Unsupported device type for OpenCL device (GPU, CPU, ACCELERATOR): " << deviceTypes[t] << std::endl; + CV_LOG_ERROR(NULL, "OpenCL: Unsupported device type for OpenCL device (GPU, CPU, ACCELERATOR): " << deviceTypes[t]); goto not_found; } - std::vector devices; // TODO Use clReleaseDevice to cleanup - for (int i = selectedPlatform >= 0 ? selectedPlatform : 0; - (selectedPlatform >= 0 ? i == selectedPlatform : true) && (i < (int)platforms.size()); - i++) + std::vector devices; + for (std::vector::iterator currentPlatform = platforms.begin(); currentPlatform != platforms.end(); ++currentPlatform) { cl_uint count = 0; - cl_int status = clGetDeviceIDs(platforms[i], deviceType, 0, NULL, &count); + cl_int status = clGetDeviceIDs(*currentPlatform, deviceType, 0, NULL, &count); if (!(status == CL_SUCCESS || status == CL_DEVICE_NOT_FOUND)) { CV_OCL_DBG_CHECK_RESULT(status, "clGetDeviceIDs get count"); @@ -1812,7 +1810,7 @@ static cl_device_id selectOpenCLDevice() continue; size_t base = devices.size(); devices.resize(base + count); - status = clGetDeviceIDs(platforms[i], deviceType, count, &devices[base], &count); + status = clGetDeviceIDs(*currentPlatform, deviceType, count, &devices[base], &count); if (!(status == CL_SUCCESS || status == CL_DEVICE_NOT_FOUND)) { CV_OCL_DBG_CHECK_RESULT(status, "clGetDeviceIDs get IDs"); @@ -1844,13 +1842,16 @@ static cl_device_id selectOpenCLDevice() if (!configuration) return NULL; // suppress messages on stderr - std::cerr << "ERROR: Requested OpenCL device not found, check configuration: " << configuration << std::endl - << " Platform: " << (platform.length() == 0 ? "any" : platform) << std::endl - << " Device types: "; + std::ostringstream msg; + msg << "ERROR: Requested OpenCL device not found, check configuration: '" << configuration << "'" << std::endl + << " Platform: " << (platform.length() == 0 ? "any" : platform) << std::endl + << " Device types:"; for (size_t t = 0; t < deviceTypes.size(); t++) - std::cerr << deviceTypes[t] << " "; + msg << ' ' << deviceTypes[t]; + + msg << std::endl << " Device name: " << (deviceName.length() == 0 ? "any" : deviceName); - std::cerr << std::endl << " Device name: " << (deviceName.length() == 0 ? "any" : deviceName) << std::endl; + CV_LOG_ERROR(NULL, msg.str()); return NULL; } #endif From aa7ba0bc1a99357beed7f4624de353f55ceccb20 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 10 Sep 2021 20:58:37 +0000 Subject: [PATCH 198/376] build: winpack_dldt with dldt 2021.4.1 --- cmake/OpenCVDetectInferenceEngine.cmake | 4 +- ...-dldt-disable-multidevice-autoplugin.patch | 16 ++ ...20210630-dldt-disable-unused-targets.patch | 219 ++++++++++++++++++ .../2021.4.1/20210630-dldt-pdb.patch | 15 ++ .../2021.4.1/20210630-dldt-vs-version.patch | 16 ++ .../winpack_dldt/2021.4.1/build.config.py | 1 + .../winpack_dldt/2021.4.1/patch.config.py | 4 + .../winpack_dldt/2021.4.1/sysroot.config.py | 56 +++++ platforms/winpack_dldt/build_package.py | 5 +- 9 files changed, 332 insertions(+), 4 deletions(-) create mode 100644 platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-multidevice-autoplugin.patch create mode 100644 platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-unused-targets.patch create mode 100644 platforms/winpack_dldt/2021.4.1/20210630-dldt-pdb.patch create mode 100644 platforms/winpack_dldt/2021.4.1/20210630-dldt-vs-version.patch create mode 100644 platforms/winpack_dldt/2021.4.1/build.config.py create mode 100644 platforms/winpack_dldt/2021.4.1/patch.config.py create mode 100644 platforms/winpack_dldt/2021.4.1/sysroot.config.py diff --git a/cmake/OpenCVDetectInferenceEngine.cmake b/cmake/OpenCVDetectInferenceEngine.cmake index 6308d1b42480..41951b710ab4 100644 --- a/cmake/OpenCVDetectInferenceEngine.cmake +++ b/cmake/OpenCVDetectInferenceEngine.cmake @@ -147,8 +147,8 @@ if(INF_ENGINE_TARGET) endif() endif() if(NOT INF_ENGINE_RELEASE AND NOT INF_ENGINE_RELEASE_INIT) - message(WARNING "InferenceEngine version has not been set, 2021.4 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") - set(INF_ENGINE_RELEASE_INIT "2021040000") + message(WARNING "InferenceEngine version has not been set, 2021.4.1 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") + set(INF_ENGINE_RELEASE_INIT "2021040100") elseif(DEFINED INF_ENGINE_RELEASE) set(INF_ENGINE_RELEASE_INIT "${INF_ENGINE_RELEASE}") endif() diff --git a/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-multidevice-autoplugin.patch b/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-multidevice-autoplugin.patch new file mode 100644 index 000000000000..f1e748744277 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-multidevice-autoplugin.patch @@ -0,0 +1,16 @@ +diff --git a/inference-engine/src/CMakeLists.txt b/inference-engine/src/CMakeLists.txt +index 0ba0dd78..7d34e7cb 100644 +--- a/inference-engine/src/CMakeLists.txt ++++ b/inference-engine/src/CMakeLists.txt +@@ -26,9 +26,9 @@ endif() + + add_subdirectory(hetero_plugin) + +-add_subdirectory(auto_plugin) ++#add_subdirectory(auto_plugin) + +-add_subdirectory(multi_device) ++#add_subdirectory(multi_device) + + add_subdirectory(transformations) + diff --git a/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-unused-targets.patch b/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-unused-targets.patch new file mode 100644 index 000000000000..9d44cdadc6cd --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-unused-targets.patch @@ -0,0 +1,219 @@ +diff --git a/cmake/developer_package/add_ie_target.cmake b/cmake/developer_package/add_ie_target.cmake +index d49f16a4d..2726ca787 100644 +--- a/cmake/developer_package/add_ie_target.cmake ++++ b/cmake/developer_package/add_ie_target.cmake +@@ -92,7 +92,7 @@ function(addIeTarget) + if (ARG_TYPE STREQUAL EXECUTABLE) + add_executable(${ARG_NAME} ${all_sources}) + elseif(ARG_TYPE STREQUAL STATIC OR ARG_TYPE STREQUAL SHARED) +- add_library(${ARG_NAME} ${ARG_TYPE} ${all_sources}) ++ add_library(${ARG_NAME} ${ARG_TYPE} EXCLUDE_FROM_ALL ${all_sources}) + else() + message(SEND_ERROR "Invalid target type ${ARG_TYPE} specified for target name ${ARG_NAME}") + endif() +diff --git a/inference-engine/CMakeLists.txt b/inference-engine/CMakeLists.txt +index 1ac7fd8bf..df7091e51 100644 +--- a/inference-engine/CMakeLists.txt ++++ b/inference-engine/CMakeLists.txt +@@ -39,7 +39,7 @@ if(ENABLE_TESTS) + add_subdirectory(tests) + endif() + +-add_subdirectory(tools) ++#add_subdirectory(tools) + + function(ie_build_samples) + # samples should be build with the same flags as from OpenVINO package, +@@ -58,7 +58,7 @@ endfunction() + + # gflags and format_reader targets are kept inside of samples directory and + # they must be built even if samples build is disabled (required for tests and tools). +-ie_build_samples() ++#ie_build_samples() + + if(ENABLE_PYTHON) + add_subdirectory(ie_bridges/python) +@@ -142,7 +142,7 @@ endif() + # Developer package + # + +-openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) ++#openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) + + # for Template plugin + if(NGRAPH_INTERPRETER_ENABLE) +@@ -166,7 +166,7 @@ function(ie_generate_dev_package_config) + @ONLY) + endfunction() + +-ie_generate_dev_package_config() ++#ie_generate_dev_package_config() + + # + # Coverage +diff --git a/inference-engine/src/inference_engine/CMakeLists.txt b/inference-engine/src/inference_engine/CMakeLists.txt +index e8ed1a5c4..1fc9fc3ff 100644 +--- a/inference-engine/src/inference_engine/CMakeLists.txt ++++ b/inference-engine/src/inference_engine/CMakeLists.txt +@@ -110,7 +110,7 @@ add_cpplint_target(${TARGET_NAME}_plugin_api_cpplint FOR_SOURCES ${plugin_api_sr + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS} + ${PUBLIC_HEADERS}) +@@ -181,7 +181,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $ + $ + ${IE_STATIC_DEPENDENT_FILES}) +diff --git a/inference-engine/src/legacy_api/CMakeLists.txt b/inference-engine/src/legacy_api/CMakeLists.txt +index 8eae82bd2..e0e6745b1 100644 +--- a/inference-engine/src/legacy_api/CMakeLists.txt ++++ b/inference-engine/src/legacy_api/CMakeLists.txt +@@ -26,7 +26,7 @@ endif() + + file(TOUCH ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp) + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${PUBLIC_HEADERS}) + +diff --git a/inference-engine/src/mkldnn_plugin/CMakeLists.txt b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +index fe57b29dd..07831e2fb 100644 +--- a/inference-engine/src/mkldnn_plugin/CMakeLists.txt ++++ b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +@@ -67,7 +67,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # add test object library + +-add_library(${TARGET_NAME}_obj OBJECT ${SOURCES} ${HEADERS}) ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL ${SOURCES} ${HEADERS}) + target_link_libraries(${TARGET_NAME}_obj PUBLIC mkldnn) + + target_include_directories(${TARGET_NAME}_obj PRIVATE $ +diff --git a/inference-engine/src/preprocessing/CMakeLists.txt b/inference-engine/src/preprocessing/CMakeLists.txt +index f9548339d..ef962145a 100644 +--- a/inference-engine/src/preprocessing/CMakeLists.txt ++++ b/inference-engine/src/preprocessing/CMakeLists.txt +@@ -101,7 +101,7 @@ endif() + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS}) + +@@ -153,7 +153,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $) + + set_ie_threading_interface_for(${TARGET_NAME}_s) +diff --git a/inference-engine/src/vpu/common/CMakeLists.txt b/inference-engine/src/vpu/common/CMakeLists.txt +index 249e47c28..4ddf63049 100644 +--- a/inference-engine/src/vpu/common/CMakeLists.txt ++++ b/inference-engine/src/vpu/common/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h) + + function(add_common_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + ie_faster_build(${TARGET_NAME} + UNITY +@@ -60,7 +60,7 @@ add_common_target("vpu_common_lib" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_common_target("vpu_common_lib_test_static" TRUE) ++ #add_common_target("vpu_common_lib_test_static" TRUE) + else() + add_library("vpu_common_lib_test_static" ALIAS "vpu_common_lib") + endif() +diff --git a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +index bc73ab5b1..b4c1547fc 100644 +--- a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt ++++ b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h *.inc) + + function(add_graph_transformer_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + set_ie_threading_interface_for(${TARGET_NAME}) + +@@ -70,7 +70,7 @@ add_graph_transformer_target("vpu_graph_transformer" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) ++ #add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) + else() + add_library("vpu_graph_transformer_test_static" ALIAS "vpu_graph_transformer") + endif() +diff --git a/inference-engine/thirdparty/pugixml/CMakeLists.txt b/inference-engine/thirdparty/pugixml/CMakeLists.txt +index 8bcb2801a..f7e031c01 100644 +--- a/inference-engine/thirdparty/pugixml/CMakeLists.txt ++++ b/inference-engine/thirdparty/pugixml/CMakeLists.txt +@@ -41,7 +41,7 @@ if(BUILD_SHARED_LIBS) + else() + add_library(pugixml STATIC ${SOURCES}) + if (MSVC) +- add_library(pugixml_mt STATIC ${SOURCES}) ++ #add_library(pugixml_mt STATIC ${SOURCES}) + #if (WIN32) + # set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") + # set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") +diff --git a/ngraph/core/builder/CMakeLists.txt b/ngraph/core/builder/CMakeLists.txt +index ff5c381e7..2797ec9ab 100644 +--- a/ngraph/core/builder/CMakeLists.txt ++++ b/ngraph/core/builder/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/ngraph/core/reference/CMakeLists.txt b/ngraph/core/reference/CMakeLists.txt +index ef4a764ab..f6d3172e2 100644 +--- a/ngraph/core/reference/CMakeLists.txt ++++ b/ngraph/core/reference/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/openvino/itt/CMakeLists.txt b/openvino/itt/CMakeLists.txt +index e9f880b8c..c63f4df63 100644 +--- a/openvino/itt/CMakeLists.txt ++++ b/openvino/itt/CMakeLists.txt +@@ -6,7 +6,7 @@ set(TARGET_NAME itt) + + file(GLOB_RECURSE SOURCES "src/*.cpp" "src/*.hpp") + +-add_library(${TARGET_NAME} STATIC ${SOURCES}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + add_library(openvino::itt ALIAS ${TARGET_NAME}) + diff --git a/platforms/winpack_dldt/2021.4.1/20210630-dldt-pdb.patch b/platforms/winpack_dldt/2021.4.1/20210630-dldt-pdb.patch new file mode 100644 index 000000000000..65e6f84dc80b --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/20210630-dldt-pdb.patch @@ -0,0 +1,15 @@ +iff --git a/CMakeLists.txt b/CMakeLists.txt +index e0706a72e..9a053b1e4 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -6,6 +6,10 @@ cmake_minimum_required(VERSION 3.13) + + project(OpenVINO) + ++set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi /FS") ++set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++ + set(OpenVINO_MAIN_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + set(IE_MAIN_SOURCE_DIR ${OpenVINO_MAIN_SOURCE_DIR}/inference-engine) + diff --git a/platforms/winpack_dldt/2021.4.1/20210630-dldt-vs-version.patch b/platforms/winpack_dldt/2021.4.1/20210630-dldt-vs-version.patch new file mode 100644 index 000000000000..36b0068775eb --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/20210630-dldt-vs-version.patch @@ -0,0 +1,16 @@ +diff --git a/cmake/developer_package/vs_version/vs_version.cmake b/cmake/developer_package/vs_version/vs_version.cmake +index 14d4c0e1e..6a44f73b9 100644 +--- a/cmake/developer_package/vs_version/vs_version.cmake ++++ b/cmake/developer_package/vs_version/vs_version.cmake +@@ -8,9 +8,9 @@ set(IE_VS_VER_FILEVERSION_STR "${IE_VERSION_MAJOR}.${IE_VERSION_MINOR}.${IE_VERS + + set(IE_VS_VER_COMPANY_NAME_STR "Intel Corporation") + set(IE_VS_VER_PRODUCTVERSION_STR "${CI_BUILD_NUMBER}") +-set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit") ++set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit (for OpenCV Windows package)") + set(IE_VS_VER_COPYRIGHT_STR "Copyright (C) 2018-2021, Intel Corporation") +-set(IE_VS_VER_COMMENTS_STR "https://docs.openvinotoolkit.org/") ++set(IE_VS_VER_COMMENTS_STR "https://github.com/opencv/opencv/wiki/Intel%27s-Deep-Learning-Inference-Engine-backend") + + # + # ie_add_vs_version_file(NAME diff --git a/platforms/winpack_dldt/2021.4.1/build.config.py b/platforms/winpack_dldt/2021.4.1/build.config.py new file mode 100644 index 000000000000..a643c1792805 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/build.config.py @@ -0,0 +1 @@ +os.environ['CI_BUILD_NUMBER'] = '2021.4.1-opencv_winpack_dldt' diff --git a/platforms/winpack_dldt/2021.4.1/patch.config.py b/platforms/winpack_dldt/2021.4.1/patch.config.py new file mode 100644 index 000000000000..7f8715aae2da --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/patch.config.py @@ -0,0 +1,4 @@ +applyPatch('20210630-dldt-disable-unused-targets.patch') +applyPatch('20210630-dldt-pdb.patch') +applyPatch('20210630-dldt-disable-multidevice-autoplugin.patch') +applyPatch('20210630-dldt-vs-version.patch') diff --git a/platforms/winpack_dldt/2021.4.1/sysroot.config.py b/platforms/winpack_dldt/2021.4.1/sysroot.config.py new file mode 100644 index 000000000000..fa4281107d23 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/sysroot.config.py @@ -0,0 +1,56 @@ +sysroot_bin_dir = prepare_dir(self.sysrootdir / 'bin') +copytree(self.build_dir / 'install', self.sysrootdir / 'ngraph') +#rm_one(self.sysrootdir / 'ngraph' / 'lib' / 'ngraph.dll') + +build_config = 'Release' if not self.config.build_debug else 'Debug' +build_bin_dir = self.build_dir / 'bin' / 'intel64' / build_config + +def copy_bin(name): + global build_bin_dir, sysroot_bin_dir + copytree(build_bin_dir / name, sysroot_bin_dir / name) + +dll_suffix = 'd' if self.config.build_debug else '' +def copy_dll(name): + global copy_bin, dll_suffix + copy_bin(name + dll_suffix + '.dll') + copy_bin(name + dll_suffix + '.pdb') + +copy_bin('cache.json') +copy_dll('clDNNPlugin') +copy_dll('HeteroPlugin') +copy_dll('inference_engine') +copy_dll('inference_engine_ir_reader') +#copy_dll('inference_engine_ir_v7_reader') +copy_dll('inference_engine_legacy') +copy_dll('inference_engine_transformations') # runtime +copy_dll('inference_engine_lp_transformations') # runtime +#copy_dll('inference_engine_preproc') # runtime +copy_dll('MKLDNNPlugin') # runtime +copy_dll('myriadPlugin') # runtime +#copy_dll('MultiDevicePlugin') # runtime, not used +copy_dll('ngraph') +copy_bin('plugins.xml') +copy_bin('pcie-ma2x8x.elf') +copy_bin('usb-ma2x8x.mvcmd') + +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb' / 'bin', sysroot_bin_dir) +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb', self.sysrootdir / 'tbb') + +sysroot_ie_dir = prepare_dir(self.sysrootdir / 'deployment_tools' / 'inference_engine') +sysroot_ie_lib_dir = prepare_dir(sysroot_ie_dir / 'lib' / 'intel64') + +copytree(self.srcdir / 'inference-engine' / 'include', sysroot_ie_dir / 'include') +if not self.config.build_debug: + copytree(build_bin_dir / 'ngraph.lib', sysroot_ie_lib_dir / 'ngraph.lib') + copytree(build_bin_dir / 'inference_engine.lib', sysroot_ie_lib_dir / 'inference_engine.lib') + copytree(build_bin_dir / 'inference_engine_ir_reader.lib', sysroot_ie_lib_dir / 'inference_engine_ir_reader.lib') + copytree(build_bin_dir / 'inference_engine_legacy.lib', sysroot_ie_lib_dir / 'inference_engine_legacy.lib') +else: + copytree(build_bin_dir / 'ngraphd.lib', sysroot_ie_lib_dir / 'ngraphd.lib') + copytree(build_bin_dir / 'inference_engined.lib', sysroot_ie_lib_dir / 'inference_engined.lib') + copytree(build_bin_dir / 'inference_engine_ir_readerd.lib', sysroot_ie_lib_dir / 'inference_engine_ir_readerd.lib') + copytree(build_bin_dir / 'inference_engine_legacyd.lib', sysroot_ie_lib_dir / 'inference_engine_legacyd.lib') + +sysroot_license_dir = prepare_dir(self.sysrootdir / 'etc' / 'licenses') +copytree(self.srcdir / 'LICENSE', sysroot_license_dir / 'dldt-LICENSE') +copytree(self.sysrootdir / 'tbb/LICENSE', sysroot_license_dir / 'tbb-LICENSE') diff --git a/platforms/winpack_dldt/build_package.py b/platforms/winpack_dldt/build_package.py index bd4355e1cdf4..0194323930e6 100644 --- a/platforms/winpack_dldt/build_package.py +++ b/platforms/winpack_dldt/build_package.py @@ -469,7 +469,8 @@ def package_sources(self): def main(): dldt_src_url = 'https://github.com/openvinotoolkit/openvino' - dldt_src_commit = '2021.4' + dldt_src_commit = '2021.4.1' + dldt_config = None dldt_release = None build_cache_dir_default = os.environ.get('BUILD_CACHE_DIR', '.build_cache') @@ -503,7 +504,7 @@ def main(): parser.add_argument('--dldt_reference_dir', help='DLDT reference git repository (optional)') parser.add_argument('--dldt_src_dir', help='DLDT custom source repository (skip git checkout and patching, use for TESTING only)') - parser.add_argument('--dldt_config', help='Specify DLDT build configuration (defaults to evaluate from DLDT commit/branch)') + parser.add_argument('--dldt_config', default=dldt_config, help='Specify DLDT build configuration (defaults to evaluate from DLDT commit/branch)') parser.add_argument('--override_patch_hashsum', default='', help='(script debug mode)') From 51b03b87e6f9827cbee3feb6dd6dc385c4b77297 Mon Sep 17 00:00:00 2001 From: Zihao Mu Date: Fri, 10 Sep 2021 18:15:22 +0800 Subject: [PATCH 199/376] BiasAdd could load Const from second place. --- modules/dnn/src/tensorflow/tf_importer.cpp | 7 ++++++- modules/dnn/test/test_tf_importer.cpp | 12 ++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index ca9d7c5e2174..521c8ce4c384 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -914,7 +914,12 @@ void TFImporter::parseBias(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer_id[name] = id; // one input only - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + Pin inp0 = parsePin(layer.input(0)); + if (layer_id.find(inp0.name) != layer_id.end()) + // First operand is a constant. + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + else + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); } else { diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 1a2b976eb877..cdf3794bf18c 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -559,6 +559,18 @@ TEST_P(Test_TensorFlow_layers, l2_normalize) runTensorFlowNet("l2_normalize"); } +TEST_P(Test_TensorFlow_layers, BiasAdd) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD + && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + runTensorFlowNet("bias_add_1"); +} + // TODO: fix it and add to l2_normalize TEST_P(Test_TensorFlow_layers, l2_normalize_3d) { From 9d1e8b1e1ddef6b591d9cce41b625a0c8fb992a5 Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Sat, 11 Sep 2021 22:58:45 +0300 Subject: [PATCH 200/376] Update convexhull.cpp --- samples/cpp/convexhull.cpp | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/samples/cpp/convexhull.cpp b/samples/cpp/convexhull.cpp index bb34be246932..d839b8061ff8 100644 --- a/samples/cpp/convexhull.cpp +++ b/samples/cpp/convexhull.cpp @@ -38,23 +38,14 @@ int main( int argc, char** argv ) points.push_back(pt); } - vector hull; - convexHull(Mat(points), hull, true); + vector hull; + convexHull(points, hull, true); img = Scalar::all(0); for( i = 0; i < count; i++ ) circle(img, points[i], 3, Scalar(0, 0, 255), FILLED, LINE_AA); - int hullcount = (int)hull.size(); - Point pt0 = points[hull[hullcount-1]]; - - for( i = 0; i < hullcount; i++ ) - { - Point pt = points[hull[i]]; - line(img, pt0, pt, Scalar(0, 255, 0), 1,LINE_AA); - pt0 = pt; - } - + polylines(img, hull, true, Scalar(0, 255, 0), 1, LINE_AA); imshow("hull", img); char key = (char)waitKey(); From 6e66a9222a071892ea9f1c9c7951fdcc85e08f7b Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 11 Sep 2021 22:26:52 +0000 Subject: [PATCH 201/376] dnn(onnx): fix format specifier --- modules/dnn/src/onnx/onnx_importer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 955c79c0fa5c..2cd36dc94db5 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -443,7 +443,7 @@ void ONNXImporter::expandMid(const std::string& prefix, opencv_onnx::NodeProto& for (size_t j = 0; j < n; j++) { LayerParams copyLP; - copyLP.name = format("%s/copy_%d", prefix.c_str(), j); + copyLP.name = format("%s/copy_%d", prefix.c_str(), (int)j); copyLP.type = "Identity"; CV_Assert((layer_id.find(copyLP.name) == layer_id.end()) && "Couldn't copy the node: generated name already exists in the graph."); From 50462dcdc6d0f1cccae49f0fa54ccddd2a0e4a6c Mon Sep 17 00:00:00 2001 From: YashasSamaga Date: Mon, 13 Sep 2021 20:44:33 +0530 Subject: [PATCH 202/376] fix effrank assert to allow input effrank <= output effrank --- modules/dnn/src/cuda4dnn/primitives/padding.hpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/dnn/src/cuda4dnn/primitives/padding.hpp b/modules/dnn/src/cuda4dnn/primitives/padding.hpp index cbf75131a187..ce2a5c3c4776 100644 --- a/modules/dnn/src/cuda4dnn/primitives/padding.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/padding.hpp @@ -52,14 +52,19 @@ namespace cv { namespace dnn { namespace cuda4dnn { auto output_wrapper = outputs[0].dynamicCast(); auto output = output_wrapper->getSpan(); - auto effective_rank = get_effective_rank(input); - CV_Assert(get_effective_rank(input) == get_effective_rank(output)); - /* suppose we require padding for the first spatial axis (H in NCHW or D in NCDHW) * * there could be a case where the batch axis, channel axis, and the first spatial axis are all one * this would result in effective rank being less than the number of axes requiring padding */ + /* the effective rank of the input may be smaller than the effective rank of the output but the converse is never true + * input: [1, 1, 1, 3]; effective rank = 1 + * output: [1, 1, 3, 3]; effective rank = 2 + * + * hence, we use the effective rank of the output tensor for the padding operation + */ + auto effective_rank = get_effective_rank(output); + CV_Assert(get_effective_rank(input) <= effective_rank); effective_rank = std::max(effective_rank, dstRanges.size()); for (int i = effective_rank - dstRanges.size(); i < effective_rank; i++) From 1618c963e4c89a816dfa6bfebb3e94764b357d61 Mon Sep 17 00:00:00 2001 From: Alexander Panov Date: Mon, 13 Sep 2021 19:27:00 +0300 Subject: [PATCH 203/376] Merge pull request #20676 from AleksandrPanov:delete_createConvexHull_convertTo * deleted dublicated createConvexHull and convertTo * replaced checkVector(2) with points.empty() --- .../imgproc/src/min_enclosing_triangle.cpp | 22 ++----------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/modules/imgproc/src/min_enclosing_triangle.cpp b/modules/imgproc/src/min_enclosing_triangle.cpp index 4853a755d95f..265187141256 100644 --- a/modules/imgproc/src/min_enclosing_triangle.cpp +++ b/modules/imgproc/src/min_enclosing_triangle.cpp @@ -129,7 +129,6 @@ static bool areOnTheSameSideOfLine(const cv::Point2f &p1, const cv::Point2f &p2, static double areaOfTriangle(const cv::Point2f &a, const cv::Point2f &b, const cv::Point2f &c); -static void createConvexHull(cv::InputArray points, std::vector &polygon); static double distanceBtwPoints(const cv::Point2f &a, const cv::Point2f &b); @@ -319,29 +318,12 @@ namespace minEnclosingTriangle { static void findMinEnclosingTriangle(cv::InputArray points, CV_OUT cv::OutputArray triangle, CV_OUT double &area) { std::vector resultingTriangle, polygon; - - createConvexHull(points, polygon); + CV_Assert(!points.empty()); + convexHull(points, polygon, true, true); findMinEnclosingTriangle(polygon, resultingTriangle, area); cv::Mat(resultingTriangle).copyTo(triangle); } -//! Create the convex hull of the given set of points -/*! -* @param points The provided set of points -* @param polygon The polygon representing the convex hull of the points -*/ -static void createConvexHull(cv::InputArray points, std::vector &polygon) { - cv::Mat pointsMat = points.getMat(); - std::vector pointsVector; - - CV_Assert((pointsMat.checkVector(2) > 0) && - ((pointsMat.depth() == CV_32F) || (pointsMat.depth() == CV_32S))); - - pointsMat.convertTo(pointsVector, CV_32F); - - convexHull(pointsVector, polygon, true, true); -} - //! Find the minimum enclosing triangle and its area /*! * The overall complexity of the algorithm is theta(n) where "n" represents the number From 3385d386481962fb4b0619780c87fbf1c78d5c81 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 14 Sep 2021 04:15:35 +0000 Subject: [PATCH 204/376] cmake: fix handling of INF_ENGINE_RELEASE - default value should be handled earlier --- cmake/OpenCVDetectInferenceEngine.cmake | 27 +++++++++++++------------ 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/cmake/OpenCVDetectInferenceEngine.cmake b/cmake/OpenCVDetectInferenceEngine.cmake index 41951b710ab4..35640fa719fb 100644 --- a/cmake/OpenCVDetectInferenceEngine.cmake +++ b/cmake/OpenCVDetectInferenceEngine.cmake @@ -105,6 +105,20 @@ if(InferenceEngine_FOUND) message(STATUS "Detected InferenceEngine: cmake package (${InferenceEngine_VERSION})") endif() +if(DEFINED InferenceEngine_VERSION) + message(STATUS "InferenceEngine: ${InferenceEngine_VERSION}") + if(NOT INF_ENGINE_RELEASE AND NOT (InferenceEngine_VERSION VERSION_LESS "2021.4")) + math(EXPR INF_ENGINE_RELEASE_INIT "${InferenceEngine_VERSION_MAJOR} * 1000000 + ${InferenceEngine_VERSION_MINOR} * 10000 + ${InferenceEngine_VERSION_PATCH} * 100") + endif() +endif() +if(NOT INF_ENGINE_RELEASE AND NOT INF_ENGINE_RELEASE_INIT) + message(STATUS "WARNING: InferenceEngine version has not been set, 2021.4.1 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") + set(INF_ENGINE_RELEASE_INIT "2021040100") +elseif(DEFINED INF_ENGINE_RELEASE) + set(INF_ENGINE_RELEASE_INIT "${INF_ENGINE_RELEASE}") +endif() +set(INF_ENGINE_RELEASE "${INF_ENGINE_RELEASE_INIT}" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") + if(NOT INF_ENGINE_TARGET AND INF_ENGINE_LIB_DIRS AND INF_ENGINE_INCLUDE_DIRS) find_path(ie_custom_inc "inference_engine.hpp" PATHS "${INF_ENGINE_INCLUDE_DIRS}" NO_DEFAULT_PATH) if(CMAKE_BUILD_TYPE STREQUAL "Debug") @@ -140,19 +154,6 @@ endif() # Add more features to the target if(INF_ENGINE_TARGET) - if(DEFINED InferenceEngine_VERSION) - message(STATUS "InferenceEngine: ${InferenceEngine_VERSION}") - if(NOT INF_ENGINE_RELEASE AND NOT (InferenceEngine_VERSION VERSION_LESS "2021.4")) - math(EXPR INF_ENGINE_RELEASE_INIT "${InferenceEngine_VERSION_MAJOR} * 1000000 + ${InferenceEngine_VERSION_MINOR} * 10000 + ${InferenceEngine_VERSION_PATCH} * 100") - endif() - endif() - if(NOT INF_ENGINE_RELEASE AND NOT INF_ENGINE_RELEASE_INIT) - message(WARNING "InferenceEngine version has not been set, 2021.4.1 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") - set(INF_ENGINE_RELEASE_INIT "2021040100") - elseif(DEFINED INF_ENGINE_RELEASE) - set(INF_ENGINE_RELEASE_INIT "${INF_ENGINE_RELEASE}") - endif() - set(INF_ENGINE_RELEASE "${INF_ENGINE_RELEASE_INIT}" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") set_target_properties(${INF_ENGINE_TARGET} PROPERTIES INTERFACE_COMPILE_DEFINITIONS "HAVE_INF_ENGINE=1;INF_ENGINE_RELEASE=${INF_ENGINE_RELEASE}" ) From c410d7a97d03f2749d737b9e9232cd738fe09ed2 Mon Sep 17 00:00:00 2001 From: rogday Date: Tue, 14 Sep 2021 20:49:49 +0300 Subject: [PATCH 205/376] Merge pull request #20671 from rogday:yolov4x-mish Add support for YOLOv4x-mish * backport to 3.4 for supporting yolov4x-mish * add YOLOv4x-mish test * address review comments Co-authored-by: Guo Xu --- modules/dnn/src/darknet/darknet_io.cpp | 6 +- modules/dnn/src/layers/region_layer.cpp | 90 ++++++++++++++-------- modules/dnn/test/test_darknet_importer.cpp | 72 +++++++++++++++++ 3 files changed, 136 insertions(+), 32 deletions(-) diff --git a/modules/dnn/src/darknet/darknet_io.cpp b/modules/dnn/src/darknet/darknet_io.cpp index 4915538ff715..99715df829f1 100644 --- a/modules/dnn/src/darknet/darknet_io.cpp +++ b/modules/dnn/src/darknet/darknet_io.cpp @@ -470,7 +470,7 @@ namespace cv { fused_layer_names.push_back(last_layer); } - void setYolo(int classes, const std::vector& mask, const std::vector& anchors, float thresh, float nms_threshold, float scale_x_y) + void setYolo(int classes, const std::vector& mask, const std::vector& anchors, float thresh, float nms_threshold, float scale_x_y, int new_coords) { cv::dnn::LayerParams region_param; region_param.name = "Region-name"; @@ -484,6 +484,7 @@ namespace cv { region_param.set("thresh", thresh); region_param.set("nms_threshold", nms_threshold); region_param.set("scale_x_y", scale_x_y); + region_param.set("new_coords", new_coords); std::vector usedAnchors(numAnchors * 2); for (int i = 0; i < numAnchors; ++i) @@ -882,6 +883,7 @@ namespace cv { float thresh = getParam(layer_params, "thresh", 0.2); float nms_threshold = getParam(layer_params, "nms_threshold", 0.0); float scale_x_y = getParam(layer_params, "scale_x_y", 1.0); + int new_coords = getParam(layer_params, "new_coords", 0); std::string anchors_values = getParam(layer_params, "anchors", std::string()); CV_Assert(!anchors_values.empty()); @@ -894,7 +896,7 @@ namespace cv { CV_Assert(classes > 0 && num_of_anchors > 0 && (num_of_anchors * 2) == anchors_vec.size()); setParams.setPermute(false); - setParams.setYolo(classes, mask_vec, anchors_vec, thresh, nms_threshold, scale_x_y); + setParams.setYolo(classes, mask_vec, anchors_vec, thresh, nms_threshold, scale_x_y, new_coords); } else { CV_Error(cv::Error::StsParseError, "Unknown layer type: " + layer_type); diff --git a/modules/dnn/src/layers/region_layer.cpp b/modules/dnn/src/layers/region_layer.cpp index 4a8cb724d6bf..578b0d7dec7b 100644 --- a/modules/dnn/src/layers/region_layer.cpp +++ b/modules/dnn/src/layers/region_layer.cpp @@ -64,6 +64,7 @@ class RegionLayerImpl CV_FINAL : public RegionLayer public: int coords, classes, anchors, classfix; float thresh, nmsThreshold, scale_x_y; + int new_coords; bool useSoftmax, useLogistic; #ifdef HAVE_OPENCL UMat blob_umat; @@ -83,6 +84,7 @@ class RegionLayerImpl CV_FINAL : public RegionLayer useLogistic = params.get("logistic", false); nmsThreshold = params.get("nms_threshold", 0.4); scale_x_y = params.get("scale_x_y", 1.0); // Yolov4 + new_coords = params.get("new_coords", 0); // Yolov4x-mish CV_Assert(nmsThreshold >= 0.); CV_Assert(coords == 4); @@ -113,7 +115,7 @@ class RegionLayerImpl CV_FINAL : public RegionLayer { #ifdef HAVE_DNN_NGRAPH if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - return INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_2) && preferableTarget != DNN_TARGET_MYRIAD; + return INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_2) && preferableTarget != DNN_TARGET_MYRIAD && new_coords == 0; #endif return backendId == DNN_BACKEND_OPENCV; } @@ -259,26 +261,28 @@ class RegionLayerImpl CV_FINAL : public RegionLayer const float *srcData = inpBlob.ptr(); float *dstData = outBlob.ptr(); - // logistic activation for t0, for each grid cell (X x Y x Anchor-index) - for (int i = 0; i < batch_size*rows*cols*anchors; ++i) { - int index = cell_size*i; - float x = srcData[index + 4]; - dstData[index + 4] = logistic_activate(x); // logistic activation - } - - if (useSoftmax) { // Yolo v2 + if (new_coords == 0) { + // logistic activation for t0, for each grid cell (X x Y x Anchor-index) for (int i = 0; i < batch_size*rows*cols*anchors; ++i) { int index = cell_size*i; - softmax_activate(srcData + index + 5, classes, 1, dstData + index + 5); + float x = srcData[index + 4]; + dstData[index + 4] = logistic_activate(x); // logistic activation } - } - else if (useLogistic) { // Yolo v3 - for (int i = 0; i < batch_size*rows*cols*anchors; ++i){ - int index = cell_size*i; - const float* input = srcData + index + 5; - float* output = dstData + index + 5; - for (int c = 0; c < classes; ++c) - output[c] = logistic_activate(input[c]); + + if (useSoftmax) { // Yolo v2 + for (int i = 0; i < batch_size*rows*cols*anchors; ++i) { + int index = cell_size*i; + softmax_activate(srcData + index + 5, classes, 1, dstData + index + 5); + } + } + else if (useLogistic) { // Yolo v3 + for (int i = 0; i < batch_size*rows*cols*anchors; ++i){ + int index = cell_size*i; + const float* input = srcData + index + 5; + float* output = dstData + index + 5; + for (int c = 0; c < classes; ++c) + output[c] = logistic_activate(input[c]); + } } } for (int b = 0; b < batch_size; ++b) @@ -290,20 +294,46 @@ class RegionLayerImpl CV_FINAL : public RegionLayer int index = (y*cols + x)*anchors + a; // index for each grid-cell & anchor int p_index = index_sample_offset + index * cell_size + 4; float scale = dstData[p_index]; - if (classfix == -1 && scale < .5) scale = 0; // if(t0 < 0.5) t0 = 0; + if (classfix == -1 && scale < .5) + { + scale = 0; // if(t0 < 0.5) t0 = 0; + } int box_index = index_sample_offset + index * cell_size; - float x_tmp = (logistic_activate(srcData[box_index + 0]) - 0.5f) * scale_x_y + 0.5f; - float y_tmp = (logistic_activate(srcData[box_index + 1]) - 0.5f) * scale_x_y + 0.5f; - dstData[box_index + 0] = (x + x_tmp) / cols; - dstData[box_index + 1] = (y + y_tmp) / rows; - dstData[box_index + 2] = exp(srcData[box_index + 2]) * biasData[2 * a] / wNorm; - dstData[box_index + 3] = exp(srcData[box_index + 3]) * biasData[2 * a + 1] / hNorm; - - int class_index = index_sample_offset + index * cell_size + 5; - for (int j = 0; j < classes; ++j) { - float prob = scale*dstData[class_index + j]; // prob = IoU(box, object) = t0 * class-probability - dstData[class_index + j] = (prob > thresh) ? prob : 0; // if (IoU < threshold) IoU = 0; + if (new_coords == 1) { + float x_tmp = (srcData[box_index + 0] - 0.5f) * scale_x_y + 0.5f; + float y_tmp = (srcData[box_index + 1] - 0.5f) * scale_x_y + 0.5f; + dstData[box_index + 0] = (x + x_tmp) / cols; + dstData[box_index + 1] = (y + y_tmp) / rows; + dstData[box_index + 2] = (srcData[box_index + 2]) * (srcData[box_index + 2]) * 4 * biasData[2 * a] / wNorm; + dstData[box_index + 3] = (srcData[box_index + 3]) * (srcData[box_index + 3]) * 4 * biasData[2 * a + 1] / hNorm; + + scale = srcData[p_index]; + if (classfix == -1 && scale < thresh) + { + scale = 0; // if(t0 < 0.5) t0 = 0; + } + + int class_index = index_sample_offset + index * cell_size + 5; + for (int j = 0; j < classes; ++j) { + float prob = scale*srcData[class_index + j]; // prob = IoU(box, object) = t0 * class-probability + dstData[class_index + j] = (prob > thresh) ? prob : 0; // if (IoU < threshold) IoU = 0; + } + } + else + { + float x_tmp = (logistic_activate(srcData[box_index + 0]) - 0.5f) * scale_x_y + 0.5f; + float y_tmp = (logistic_activate(srcData[box_index + 1]) - 0.5f) * scale_x_y + 0.5f; + dstData[box_index + 0] = (x + x_tmp) / cols; + dstData[box_index + 1] = (y + y_tmp) / rows; + dstData[box_index + 2] = exp(srcData[box_index + 2]) * biasData[2 * a] / wNorm; + dstData[box_index + 3] = exp(srcData[box_index + 3]) * biasData[2 * a + 1] / hNorm; + + int class_index = index_sample_offset + index * cell_size + 5; + for (int j = 0; j < classes; ++j) { + float prob = scale*dstData[class_index + j]; // prob = IoU(box, object) = t0 * class-probability + dstData[class_index + j] = (prob > thresh) ? prob : 0; // if (IoU < threshold) IoU = 0; + } } } if (nmsThreshold > 0) { diff --git a/modules/dnn/test/test_darknet_importer.cpp b/modules/dnn/test/test_darknet_importer.cpp index 9983d99ef500..ea700573e653 100644 --- a/modules/dnn/test/test_darknet_importer.cpp +++ b/modules/dnn/test/test_darknet_importer.cpp @@ -681,6 +681,78 @@ TEST_P(Test_Darknet_nets, YOLOv4_tiny) #endif } +TEST_P(Test_Darknet_nets, YOLOv4x_mish) +{ + applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +#if defined(INF_ENGINE_RELEASE) + if (target == DNN_TARGET_MYRIAD) // NC_OUT_OF_MEMORY + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + // batchId, classId, confidence, left, top, right, bottom + const int N0 = 3; + const int N1 = 5; + static const float ref_[/* (N0 + N1) * 7 */] = { +0, 16, 0.925536f, 0.17188f, 0.386832f, 0.406138f, 0.941696f, +0, 1, 0.912028f, 0.162125f, 0.208863f, 0.741316f, 0.729332f, +0, 7, 0.841018f, 0.608953f, 0.128653f, 0.900692f, 0.295657f, + +1, 2, 0.925697f, 0.650438f, 0.458118f, 0.813927f, 0.661775f, +1, 0, 0.882156f, 0.203644f, 0.365763f, 0.265473f, 0.632195f, +1, 2, 0.848857f, 0.451044f, 0.462997f, 0.496629f, 0.522719f, +1, 9, 0.736015f, 0.374503f, 0.316029f, 0.399358f, 0.392883f, +1, 9, 0.727129f, 0.662469f, 0.373687f, 0.687877f, 0.441335f, + }; + Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); + + double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.006 : 8e-5; + double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.042 : 3e-4; + + std::string config_file = "yolov4x-mish.cfg"; + std::string weights_file = "yolov4x-mish.weights"; + +#if defined(INF_ENGINE_RELEASE) + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD && + getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + { + scoreDiff = 0.04; + iouDiff = 0.2; + } +#endif + + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff); + } + + { + SCOPED_TRACE("batch size 2"); + +#if defined(INF_ENGINE_RELEASE) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_MYRIAD && + getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); + } +#endif + + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff); + } +} + INSTANTIATE_TEST_CASE_P(/**/, Test_Darknet_nets, dnnBackendsAndTargets()); From 41a2eb5245583403f4a26ca24ec7ba736d2cca5e Mon Sep 17 00:00:00 2001 From: Rohit Sutradhar <40035473+r0hit2005@users.noreply.github.com> Date: Wed, 15 Sep 2021 22:20:49 +0530 Subject: [PATCH 206/376] Merge pull request #20361 from r0hit2005:master Tutorial for parallel_for_ and Universal Intrinsic (GSoC '21) * New parallel_for tutorial * Universal Intrinsics Draft Tutorial * Added draft of universal intrinsic tutorial * * Added final markdown for parallel_for_new * Added first half of universal intrinsic tutorial * Fixed warnings in documentation and sample code for parallel_for_new tutorial * Restored original parallel_for_ tutorial and table_of_content_core * Minor changes * Added demonstration of 1-D vectorized convolution * * Added 2-D convolution implementation and tutorial * Minor changes in vectorized implementation of 1-D and 2-D convolution * Minor changes to univ_intrin tutorial. Added new tutorials to the table of contents * Minor changes * Removed variable sized array initializations * Fixed conversion warnings * Added doxygen references, minor fixes * Added jpg image for parallel_for_ doc --- .../file_input_output_with_xml_yml.markdown | 2 +- ...ow_to_use_OpenCV_parallel_for_new.markdown | 166 +++++++++ .../images/convolution-example-matrix.gif | Bin 0 -> 150600 bytes .../images/resimg.jpg | Bin 0 -> 65566 bytes .../core/table_of_content_core.markdown | 3 +- .../core/univ_intrin/univ_intrin.markdown | 334 ++++++++++++++++++ .../how_to_use_OpenCV_parallel_for_new.cpp | 332 +++++++++++++++++ .../core/univ_intrin/univ_intrin.cpp | 230 ++++++++++++ 8 files changed, 1065 insertions(+), 2 deletions(-) create mode 100644 doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/how_to_use_OpenCV_parallel_for_new.markdown create mode 100644 doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/convolution-example-matrix.gif create mode 100644 doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/resimg.jpg create mode 100644 doc/tutorials/core/univ_intrin/univ_intrin.markdown create mode 100644 samples/cpp/tutorial_code/core/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_new.cpp create mode 100644 samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp diff --git a/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown b/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown index da060cf27dd8..46838dd99b4a 100644 --- a/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown +++ b/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown @@ -4,7 +4,7 @@ File Input and Output using XML and YAML files {#tutorial_file_input_output_with @tableofcontents @prev_tutorial{tutorial_discrete_fourier_transform} -@next_tutorial{tutorial_how_to_use_OpenCV_parallel_for_} +@next_tutorial{tutorial_how_to_use_OpenCV_parallel_for_new} | | | | -: | :- | diff --git a/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/how_to_use_OpenCV_parallel_for_new.markdown b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/how_to_use_OpenCV_parallel_for_new.markdown new file mode 100644 index 000000000000..57334e2b9b87 --- /dev/null +++ b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/how_to_use_OpenCV_parallel_for_new.markdown @@ -0,0 +1,166 @@ +How to use the OpenCV parallel_for_ to parallelize your code {#tutorial_how_to_use_OpenCV_parallel_for_new} +================================================================== + +@tableofcontents + +@prev_tutorial{tutorial_file_input_output_with_xml_yml} +@next_tutorial{tutorial_univ_intrin} + +| | | +| -: | :- | +| Compatibility | OpenCV >= 3.0 | + +Goal +---- + +The goal of this tutorial is to demonstrate the use of the OpenCV `parallel_for_` framework to easily parallelize your code. To illustrate the concept, we will write a program to perform convolution operation over an image. +The full tutorial code is [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_new.cpp). + +Precondition +---- + +### Parallel Frameworks +The first precondition is to have OpenCV built with a parallel framework. +In OpenCV 4.5, the following parallel frameworks are available in that order: + +* Intel Threading Building Blocks (3rdparty library, should be explicitly enabled) +* OpenMP (integrated to compiler, should be explicitly enabled) +* APPLE GCD (system wide, used automatically (APPLE only)) +* Windows RT concurrency (system wide, used automatically (Windows RT only)) +* Windows concurrency (part of runtime, used automatically (Windows only - MSVC++ >= 10)) +* Pthreads + +As you can see, several parallel frameworks can be used in the OpenCV library. Some parallel libraries are third party libraries and have to be explicitly enabled in CMake before building, while others are automatically available with the platform (e.g. APPLE GCD). + +### Race Conditions +Race conditions occur when more than one thread try to write *or* read and write to a particular memory location simultaneously. +Based on that, we can broadly classify algorithms into two categories:- +1. Algorithms in which only a single thread writes data to a particular memory location. + * In *convolution*, for example, even though multiple threads may read from a pixel at a particular time, only a single thread *writes* to a particular pixel. + +2. Algorithms in which multiple threads may write to a single memory location. + * Finding contours, features, etc. Such algorithms may require each thread to add data to a global variable simultaneously. For example, when detecting features, each thread will add features of their respective parts of the image to a common vector, thus creating a race condition. + +Convolution +----------- + +We will use the example of performing a convolution to demonstrate the use of `parallel_for_` to parallelize the computation. This is an example of an algorithm which does not lead to a race condition. + +Theory +------ +Convolution is a simple mathematical operation widely used in image processing. Here, we slide a smaller matrix, called the *kernel*, over an image and a sum of the product of pixel values and corresponding values in the kernel gives us the value of the particular pixel in the output (called the anchor point of the kernel). Based on the values in the kernel, we get different results. +In the example below, we use a 3x3 kernel (anchored at its center) and convolve over a 5x5 matrix to produce a 3x3 matrix. The size of the output can be altered by padding the input with suitable values. +![Convolution Animation](images/convolution-example-matrix.gif) + +For more information about different kernels and what they do, look [here](https://en.wikipedia.org/wiki/Kernel_(image_processing)) + +For the purpose of this tutorial, we will implement the simplest form of the function which takes a grayscale image (1 channel) and an odd length square kernel and produces an output image. +The operation will not be performed in-place. +@note We can store a few of the relevant pixels temporarily to make sure we use the original values during the convolution and then do it in-place. However, the purpose of this tutorial is to introduce parallel_for_ function and an inplace implementation may be too complicated. + +Pseudocode +----------- + + InputImage src, OutputImage dst, kernel(size n) + makeborder(src, n/2) + for each pixel (i, j) strictly inside borders, do: + { + value := 0 + for k := -n/2 to n/2, do: + for l := -n/2 to n/2, do: + value += kernel[n/2 + k][n/2 + l]*src[i + k][j + l] + + dst[i][j] := value + } + +For an *n-sized kernel*, we will add a border of size *n/2* to handle edge cases. +We then run two loops to move along the kernel and add the products to sum + +Implementation +-------------- + +### Sequential implementation + +@snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-sequential + +We first make an output matrix(dst) with the same size as src and add borders to the src image(to handle edge cases). +@snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-make-borders + +We then sequentially iterate over the pixels in the src image and compute the value over the kernel and the neighbouring pixel values. +We then fill value to the corresponding pixel in the dst image. +@snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-kernel-loop + +### Parallel implementation + +When looking at the sequential implementation, we can notice that each pixel depends on multiple neighbouring pixels but only one pixel is edited at a time. Thus, to optimize the computation, we can split the image into stripes and parallely perform convolution on each, by exploiting the multi-core architecture of modern processor. The OpenCV @ref cv::parallel_for_ framework automatically decides how to split the computation efficiently and does most of the work for us. + +@note Although values of a pixel in a particular stripe may depend on pixel values outside the stripe, these are only read only operations and hence will not cause undefined behaviour. + + +We first declare a custom class that inherits from @ref cv::ParallelLoopBody and override the `virtual void operator ()(const cv::Range& range) const`. +@snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-parallel + +The range in the `operator ()` represents the subset of values that will be treated by an individual thread. Based on the requirement, there may be different ways of splitting the range which in turn changes the computation. + +For example, we can either +1. Split the entire traversal of the image and obtain the [row, col] coordinate in the following way (as shown in the above code): + + @snippet how_to_use_OpenCV_parallel_for_new.cpp overload-full + + We would then call the parallel_for_ function in the following way: + @snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-parallel-function +
+ +2. Split the rows and compute for each row: + + @snippet how_to_use_OpenCV_parallel_for_new.cpp overload-row-split + + In this case, we call the parallel_for_ function with a different range: + @snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-parallel-function-row + +@note In our case, both implementations perform similarly. Some cases may allow better memory access patterns or other performance benefits. + +To set the number of threads, you can use: @ref cv::setNumThreads. You can also specify the number of splitting using the nstripes parameter in @ref cv::parallel_for_. For instance, if your processor has 4 threads, setting `cv::setNumThreads(2)` or setting `nstripes=2` should be the same as by default it will use all the processor threads available but will split the workload only on two threads. + +@note C++ 11 standard allows to simplify the parallel implementation by get rid of the `parallelConvolution` class and replacing it with lambda expression: + +@snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-parallel-cxx11 + +Results +----------- + +The resulting time taken for execution of the two implementations on a +* *512x512 input* with a *5x5 kernel*: + + This program shows how to use the OpenCV parallel_for_ function and + compares the performance of the sequential and parallel implementations for a + convolution operation + Usage: + ./a.out [image_path -- default lena.jpg] + + Sequential Implementation: 0.0953564s + Parallel Implementation: 0.0246762s + Parallel Implementation(Row Split): 0.0248722s + +
+ +* *512x512 input with a 3x3 kernel* + + This program shows how to use the OpenCV parallel_for_ function and + compares the performance of the sequential and parallel implementations for a + convolution operation + Usage: + ./a.out [image_path -- default lena.jpg] + + Sequential Implementation: 0.0301325s + Parallel Implementation: 0.0117053s + Parallel Implementation(Row Split): 0.0117894s + +The performance of the parallel implementation depends on the type of CPU you have. For instance, on 4 cores - 8 threads CPU, runtime may be 6x to 7x faster than a sequential implementation. There are many factors to explain why we do not achieve a speed-up of 8x: + * the overhead to create and manage the threads, + * background processes running in parallel, + * the difference between 4 hardware cores with 2 logical threads for each core and 8 hardware cores. + +In the tutorial, we used a horizontal gradient filter(as shown in the animation above), which produces an image highlighting the vertical edges. + +![result image](images/resimg.jpg) \ No newline at end of file diff --git a/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/convolution-example-matrix.gif b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/convolution-example-matrix.gif new file mode 100644 index 0000000000000000000000000000000000000000..070208eb3da505fe6d69aabefdded607633b5153 GIT binary patch literal 150600 zcmX8aWl&pdqXpmucXuo9aB!!%YjJmXFHk7Ko#O6L+}(@2yBBvT?vR^1-}^hm%x10i zWV1RqM9r$B^@9Osc{vHel=LUpyPo3YNPt0w13NesJMpe&gbBW$f%^1ptkF zc=^gN$-~Gaz$z-yMFjl*3u5Xom#)Svi0&Y6%n1-+lq%InqHO=ol#Q0IN6e2IoLnF|9X17dO9_~y?=3W@%Z#Kwfg}D4ZU`D zv2k>EaQ~W9-vC5J_Kz-H*t>^9rdm5VJbHPR)v%IO)H!}Q1m6wIY3M*9lZ>oCj4xk| zt-LIq9xWZ60bsBJP(&lEJM)`&(>wRRVG(65Bb(=+LrZrf%WIFfs|{Vl%V*bwG`s`T zXM{9#Q@b6%{o$Hg-YI#KGX;;Kanl;KbVE;^OA!=EBDM`pDqh z)#S&&`NOlr*T-XgY*;81EJQRy1RQ*70z@LqLuTa)g8+O%{-r{a?7kik6UMArV8`C8z#PPE$u-juR5MA}_b9o}IR?j;_Ub zNY;?(4PD$I4L3LV2I+vcv!ks?xZQVeXB!QdA2I){gilxm1O(r(bU!~oNOgZECI%ZD zLrM}786BCL7MWTWm0A@Y9UT)F`^(EcF*_Slp5$=<UVZPS@TF)OI!J`=6yOinLF>FnCze0@1H({bP`fX2q2JAL$)wHeKEYe3Nd7O{Q=?$#DuY#!||m{ z$Yvp`K-vqDVPfSKVj9E*h>>L*$v-d4P;`YHt$4&dl-YY;)7x_8Wu<$b{$jAw3_5 zRDEIdc46-UA`*leNdFht9w5psO}8zrokE~nIy!}j2O)0d>>45-1R{t4tEY#MbwdnX zJ9>f`2oV6Xm-WMUh-?rFHg2vUBZ9!VdG-l`ar5>G;uwTSh3;_f}2L$1>)0MM_R|wAMclQt*|L69_!59Q0 z2+I(tA#_6w{2$#A%pqt)T!paq@bLNoLG64LBFWS9BLsN}?Emu|A|7PgZ>PuqTl&Z2 z6og!eqL7zEcm&^#{LgTR(h#2!zyL5JiELedUoZkTqrqrh!9X|$m3+2rec@0v0jJ&O zXnoPhFLH?}61j%ru_Su+3WKqR-xFzU=HuCNjU`iAeC}tPV~wRVd7{Bcr1DK=b49XA zjE3V)|A3&8$rRh_b_XJ`nT#ge>i0+DsT6V*+Zzri(mCz7C)*p3XYwVY$&@;p zP8Z75D~+Z)n$K72%qMb{I$JI`+T72#r#f5z{p}4#{-WH~cC$a4#AH0()qZz8TcnVu z+}#2B?KRr}o$l^@`nNL>4gR9i)Ae$9vQTL})6@O-biF^3r}C%gYw~yvjxrmX0R9Y9KW!N~gDw z7|viyUkDm5PMM0}TDIAb<@$s8Un)7DXnQemTgPvz8{qz=X!F~ zA>;5fe;xih8HNQ>`jaOR}BpfF%VUjErSPc9aaP zE-*V*9`Ro}$15WS!HnlsP0M!Y)eTUfF^ZOJB&M2PxbxxK&NIe~`qA5jaORn^afk9I zNv6xDMJ@a4CKy?Xv*t~&4M$*Qj~p}AekyTQ`##YR*7kGS=*W)C;b^BX0WIT`EVFx+ zwXJWDSqxAK*Gw)wKx}inmWz-Ll48*0gcId2QRbUL91#Q8np-1>`eBlR3D-eFX9M7TblK#v|Alpn7RO`cv+bO^c-Eo~i-i#kZongJvpK+s zOV!Daw1E5R`VSJ~eXbgJP`RJ_sn1kHd6K2_(J1;pVV%PY_d+=p1LNR)ogJNLwb;B@ zh2KfP@pe)LBg^&g3DPtM(TdaF0OUnxb;CsbSvFGA-1*ILf;8 zrrKP?sZY(gTQUVBFCGA-qg-Pwg{WS#`9h)Dm6u{mAg)S<}Z z%B!-Qsjhbt7-+gfgSJf~r zC==+)$qjl~6Cg`jf~@VR)vQ@-lm@H`oL>Y~s@sk%toZGZc_Q(ILX0TM8iurj@nUv= zbFjE2yC^_>$NY=CYz=<0TciNMn)zQzD``A=ei%U_plzq}(^Ec?@OhUFCsNrkH5R^^ zBuo}t6+|)W6!LbVBmx~7_ZTo8)C{f*S>960A^wKSORW!=CtVC%2iQnl63Ci|7c3*{MsTZ z<2{;ExHsUbuYu$ha;?smd%ID(n=2Q;U<|vROsMeT_3=zmp*(dQow?QDojhv%A_~dt z^!Xe6Op(`CCF#d!H8AzGVvzYtf3NC^Jm~RxTnhAs0Gj#*mK{N8~n zEheR+!0Sdvp}fYU68+2ZZh zdz7I>+sF=dBB%_;4}KD8tM8m8n7n9S(o!hyc+v?az{%3Ua&~7W-DO)-9ayj?pg}v> zaHO=`o;RSoxnlIgQHlmLjDBl3yJM&BuY*G86K&{+*+tt$38$#*g=QvzVuc;GzdH4t zuRASG_@!v}@l`$LDu|-uZ`v{%9(zqZ4*~V)g^7uJX3MzOMLlNQMVeEmxpAml(|5vi zzL~4ZXt+HW9|xy>U45Q@GB1zLsXpHT{V?J#UamYxtDlubuK(hcx|*wv|Y|_-uy<9pzF0VkH-*@5f2m7#%Tyh<=eNkkXhE~pp-gMvhRKQn!6(Timk@a1N z=>NUU%?i6YFDm-nMA1jDMUj_IUM2Z(L?`_%6w$!E$o<-u(L2bwHu=f~@#?n_la8)U5$E<-C2D z7Z9WZ<*d_5xE)`bN3yi*~k>Xgz>l>0#J&3pAHY1^OftM4;!+iyKY4Yl3Z0V?l% z?&iMaCcJ;oyJ7*H3ARA`7a;J;Q1u%9r?CV7q;Pl3TyiaP(If5S$GBlIv=XTf_l6q9Gl!?f z#|-u>2qg0kCPVm%91$#(9!RnpENTO~=k+b6bynpSL)nM_F6tt^8X|lgB17+7$?K#s zD#3E%^~VcekTTe0C6q!ZxU&`Aj4w=VG`LSD%tj~7sw~WwAVgvm+jiMnl_T6uHr(?> z-pf1O*9Pz-J>35!Jb+KjFCsh_CL-)PJfcii+l@A6HG&#^7!gkpY0^)tUf@a7M`=tD z!~hj0vKr>l7W|qYoOcpgZ1V$$E$VlA)VNFt4MJ#vh4&g>v@}9=gHANGv@D}eXa|C8 zXi{|7b95&G$X6`bAU%>}$p6oYnDqn8vD!qgS=)H&7EI}E>U zlcdi4R0p4=xMa80w9s!izjADnrEHV;@X^0Vra`O5G87~a(t0*W1T}NVed4F66Mk0> z1pakFJgiHJmqSbB&q&tINM%U9Hc4$rOKr?Ya(u~f%wS2gauBCWi$;w1UIYEeNP}ZQ zuhdPeI|a#>V}l4imD-Ji5S>WJ^dk{9)m@_u5z!KSG6r?CFWEA()-v+TvnSWGr^m8k zcIdx&;nuEYF4^YPN9I%!B(yPj?+xd0bB2C`y+Q#tDdH4AtbDQuY_l(XGRE4o%X^V# z81nA<^P1ZoOHO~4y~LdLqQm**fHQKyr)l)Ac{(BfH^KS4a&c6p-l%6uOl!GVb{YS4 z^YDoB2r>)q+6&;21Q*-$-U-pZ(C2?W$tx#JrWS~8%)lmn&1o;s+R3o%Q$hNzAdimKR>;Txd*Y$+?RyEw zdRE3ksbgvS>*;S}#scsvvUz5iMMbtZc-{RAQ5b>_k`@LOhAL2G7DZk@mVvR{mN|H>vZA7JCub|(eFn2molpW;_wuLzz z!LCtJUV2{I9q}P+IYr~FXzTf$cJ7t(Wz~#mMnokIzST7tNd*4ZRmi?o6x!(;gk=&l|gIO3KYqy+EC3hFp{cBJ+GC$L{+}J^=C+>SMpfbj72CJ*1yNA_2qT3(kkdZ zi>fM6PhZeRt{Vi28ZT_C;q~ic!GJsqyOP(D>Q=@qRG0E>(Ry;C_}q&6%cy!pohnec zqne8`REY~r8r;W&4C{~?{Hvb4i5-TngNB{SHy2p%7JX-#P)k6I8FDovQ}s1DoRtg8 zKNoaorbah~f?A^Dg;_X9lSba?Iz;-AJ~9@|3z5OKOt5k1&~dR3!#T!0tRlA0Q$rFGP@!Cc850eVqd5y~A-TN;|SQpMIj z+r|Wn0075|jOxCT%T(&ME4+~h7XfFi28}cepr}KM z*9(g$ZkJ^WNo2yJ#nGb#dujR*@$YO{=bX2bPY9$;NH1|`A}=|djzaY&>m(k2+jg_W zYU!vD=*^W z`X*@K+rL8H<7eI{E7tZ)8`^Okab6!9*1PFgpp-%}cEeV$0GVc5(xDXAnC~%F|3|aN zL#wHUii@gL{onv9I1G(%GN@&V<^uAxZ}T_*84Q7F_b<9$fd0^+T%*&wB`-1DLNttT zbe-viK#-L}%PRX+WzCdoxr1UoGD>~SL~)!#(_m(Kd3h7}oREJVoDLk#*RFvt1Dan% z<=I5P1u~Bk4Zt4bvQ4}@_LR|9bi$IW0_uLWG(+1;K6K1KPQ3}HF{}+Hby#I6Cibuc(WOj7>`-~(=Xc)DOw3L69 z38MD3aOSLH7kU_h+$8m^Di~vpt-qTr@Su=CXiNT{B92VckQGr{s|sIRo;(-?T|K7p*_X}IHIMdE49ce=<8CWwE;%R7$Dx5}$q zLAi>X4?5^ccq!`vV80InSVMc>O4Khp0W(s4TI&pQ%UaFFa3BL|%R6*^0FA*cDwZf< z&96lc1xG}GW^HAbH)F1WzlX(l4$iS(L$)OB1|(hCYzm0h%IK(DsIM%^i8#_W1;!=u*^rF zoc}}{xT+f_c8#oFFtXX=SwY)y%XN;EE?0Q@EY=ZKcewglXF1WTGfW>Q-X>sN!r5P0 z;fRJl|L?9}CX4<&Af~f_YpfrzN|#>^xq&x!SQVHuou=I)v%^h|4JNGFRWe-L+rm8P zZm+>vXZ7W^m~B$V(-WLc*2lIJ$ZsyWR>`q%4wp%>rCliWQ{nbvu@l{gJIYi1D*)JVyd;Nt(?G8PAD3c54^EZI`#~kaTo{t6osa9!9_ghA-Pb z!oUo-5n6|q4kvXR(_iP7wC~E$+hytR*D4LW?tKfcVTsvev3UIss}(WU*4~^QB%{V+ zuuHGOCW6j8tUdpNwBDhq#FkXwA&tPu-`H-QpKEYR{JH+^$c_?O$k0MG(4$dIKG`~r z&Eu9+5)x8sLaN$8(Tzg{K8h`g0Etuiu7*R^lQ890o=Dh*PN;yd094kQ6@j3{IPQ+0 zDI&2={_Uqg+f&`oU6MJK+^O}#E$%=sYQ~ZfqmQLlj$NLivzp8Q2vB!OtB>Qkq3h4> zgSY>H&&~(7DsoX!I$jmx;!f_~4}M3lPbhOw{mhlfI#p!3)Q`L7x49&ya4^iDoZDHs=E~nsWPU9(bh*_Nn_TMCJll(7k=ztgj?hx)(TKWG`2Lv)`0rls$BX3H zH4*#Ep3yUF>-}9vt$F8T^vv_EQJLq=zj*XhjP6Dh%@@DZhp)9Sp5rfsyD!1;ug09u zB?3heV@~wZ$SxwjCnR+me_oAa9&*bcHpd>k4X=^@dsI;Rkc)mJD0nMRa@GWAs&~yi zb|2kSJHLKJy_+E2{i-b(yZTor!aMursmb|eJN8A#=|cj%h-+Bc`S)W61_OX0Oq5SlDyCV^SNh5&bQn@swiN|d=ISP4X8A~SRM$N{tBA3h%oMADR*ejCGZaH17 zyKtnG&F?wyaLuyL=O(rR#6^m;`Bp5EM0;62&cIamO?@g;Z*obqQlnXZYyCk}P6it> zid5rqMmO9-7FCs-{A}eJS4|iG;5u7LCrKNwVI771{*wJ1EjA3M2Fpr z-~atREaN^rTY7dqsO|ZA3q5}CegfsBQp&1rBs3J$43>%!etY!a=nR0b?FI^ZEG^wl zTD)+1EEMg|{8G;R^q;J;{AXRA2M$;j^0gI)a-OJ8-HDS|WpE`3Da}zU49w4z2Kg}Z zqi}k#pdAzR|Ksog6kOIrDa1Gk-9F2)3*u;N%t?6~4c15znku?x*s+SvNIt0j_%||h z@P;89bwP|J`$bF*t9LEk1FHz=lZ;jBWYWKr36H;s#a9v%`MV8eptu4DSy!$%@hGB8 zrGvE8Gk7&H*lNa zUis}VV(2(%x#b|wr?J0ulml-X+svr@50(iXz58>lUv+O^cA}ubjGJb8Ii~ z+CIFWZM!%rG^^EC;H5TUdEUpihB%RBzA+C4o^}hc^01U;sw?7y8MeY!wWd~max49h zyCt7YAbI`ohMxmiFM2$3AsI5}6uAVuzgODx34gyuX6}_+DuOdlOHx=vb)tVwGnlJ{ zg{7|74Y9smIB!U7IM~Y>yyF=-FXMMwHom*5zxjK2>E_iyskheaz8u>3yp+N*c6~oSK-$^Q*bH6%d}z)y z90=rzHAeTWQXd&vAyDG|n`gy%?d1PJfne-s|H)4Y0J5$G1bn_dfdc?=s-O=hoKF;Q zQ5ZbcKtux*AkniZJO^74RxvpYx=tu#M(|I#Lb%1uW3Uj$a3%FZ5t*BBOZPVymC&u| ztzI7QF84BYp#vu4B@*cYA~GvaVgnF`R=PO(*%jdZc7TMopp%@40S&H~6l>&&(+uED zFknsyRmU#Hpyp~g;7x&IJ=}#J#2#WQIgaq3WIvXgrLmlc8Ew>x)YuMN_|JrWk{dM{ z!OseN!~1V7@98C9pl*K!!4N(QNJfZZsU<`j9O&4r$jZVcCd`z{@-%qMDOzFSx)f#M z@s`W!*z83)hwSTH&<-E7&|BLm?jy2O$mzUJg#H$qtG$h26WZoT^V-};u6`cpS+9ub zqec!-9~C#ln*-)u7;-Lg*Pw{9C+9k3;Bw0VP~7NTSb_Ur{Om5-3Tz-C(s+vTqgNuc5zw$;<-` zHHoC`*l9JNY33!|mU8pQ^E&u0SolR2RBg^`%3#jbZx@$||E}~2^3{kr;wEz+(i0Mi znkysP zyY|KpdfHuQ?B&DfRHia^s{{S16wr>$$#%$ER`PgF7xd=@?M;kL70rs{Z1A2ykcVd#LlCr+%U%6Cd07Ri}!b;nv03;j}3VEd#8_P zIcN0;{YJCZto}m>s#OZ%J>N)Q1S7Ve4{sVswc0slG}kXs%sXb?>18XIB&!i?Ch#L_ zopw=dJ~>Tl_a(eTNr5Y=M>oi*bh14QV1k+m2}R@F+>**SQn%25>wCinFHPPb&NAqm zgS|m;BS$b7ahGx}q?R&){PVoof3sYIfebMgy#Ztyuvsp7Sqpr9saC1)TDJG-ABqpeBPZbTn-dtO|zCU zRb8>Ypx#}y?^@%wqkvDRzQvO^IZU@5Zerh$b7O0lpKQtO-?*`3=VdKOpj^#91<@bW#`33(>S$a8k zIrhOeP*^cpx!RF_*jPZ9Y_lajf%6o)Ol+dLcHwy$G8cquxZz(GTEv1;&FbJyMb4(W z>cw};cD9dqo0gFB#|e?NVU_DPcBI7v5700G1~OTH7Hxq&ciqmGf7y@AK3YBCf3nLZ z0pY311JULIiY;Q?NE{e}$RQYV)M5$<^z^=%5$PT`xx6UM1bs-p0QgZjjRk&_K8)#e z^Q1{X0iARJkS&hM>B|5j0bd~D1QyXyo3A4K%4|ni1Z-LQGuq; ztfZG*@dBVZUnd4NB3V5E4I@~XKG|s&)>)zlE3(t4AWXrfr*XX#SQegJ58BslXS(0LO$1q2Ph9;DP8MnM3w!RlNx354PhtLMaIrzgL&?-Caaj*%eRLg4*2 zA%@Z`D5&1tdk%ZH0c$bZ`*oqbk0}Ih5J(sw$N+#TffAu=N|D${e3odTJK!ImK%O+O zUf3v3)#ob=OfA}n=GFnh`~60W;W6zBtYh{~YLKUfibK-Ge;fsc>Aj_3y2v;#()#p*>7GFVj6mI{;FW>o zD~?dgIw>%Mg@m^Q;x%!Qod+nUxMXu8j3Ytvf$PM?A$32T z#ZGnbN{d=2A%`;p+zE7tDeZF)h7HLkH9=7IleyVXAaLnQi|O+BAOexpm3a^a8sPbC zfF{Kx*7^i&BB90-f@(qpra%>r_*zxmf-u;Fu%cwdl~U28nifQZ|OQ>4t8jeMGikbh^%&FRQQ zW7ka8cz4IRK?1K*6KML5qBgsjlU(qfUv@=r6b%Q!qN*g2&Zr=VFZ=^Qpc-gCsNlgK z38p3o30{bjWBfL&qU|13WM&I;B1y@x$j$WouX{p{&b%|P7z$0b)g^}eABjAeSYjLn zbs{Dv(_OwbSpIqLcXASg#c0zfDBV&Sb{bSgCnj~q8m=qnL*{7DGDpwHJcl$yQgx~^1&5fU`y((U3i@*2cLz9; zW36{Y*0Wpn#27g0hUiG8Z!IDPV30isW}(`GU>=XXOPXooZ^eA0<^XwURK&P%u1gZl{1}i^1)Nq*PU~JU+2m;rB1pmj9 z#$6}#6#-_^0yu;tXYg(D;I6tChgZY?%lU<}e15Hnl$M(R+_w-d8_$`F>}9a51g%&R z34X+6I8DUj251~plgg3}w<9}NbJ=UFG!gg*nxpdvOPZx**w0wX%-@BG+-0%QT5#nH zG(R{_7YMH}be!7PGO(&OVCMakbtFQSdozR3$hB8DXDmE>VMT!nq?(D`Sfwn&cra@S z-ht^!lPmqjRH9Si4nd86CG``4{HEfjk9sf6)#)bn9L=a)L!D4dokP8E^^Or?_FCgm zN&(mgQ0jV=mc_|55;`$zIvm9omI{hVu~snCV77&a_>HrZSz;KimvQ~DSR~wsns>+> z(>Fa@l&L5~y^N-LOV5=NSXK8>gKIZFki7E7*00SXgH2e2sp^f+2X*ictYJ6}vqZGc zj6;1S_vV}VmV(0OQO?#ztii^-p`qh)cJ;>8$JP$F)^Vut{6j6-!(4!n;SG1ilZIjJ zzE(kZ%#g75_3di7JHzRa@OCUl$HP|gn?b&CJ=@)e{)^GV7RO?W);$Zsk>Ot!zrSjl zJ9Y~do7F56DP>GAOBl_}mmg~({*r!c8=p(i`s8|uU$^@mRSr`$9xittr`DegHIXIz z(lZSZDvbp$X^Cn~iN5~YpEtg`)c+K2{5KU2P~N%|-b8bffL+Fei`#A+?0(6SK7Yt} zq|{*u%QYL?L7B$*DZCtxM1V28b{#8%6}AZFl?H88gM&89aikA+Hr>6|+oN?gMX34f zE=xvSvrpe+LVA=TYHkPhZOgmt9%H~b8>ep=aGVZgY+w5A-w zhfQ#JCQF8_W+X$KvEwmQowi2Q z(9+{cgLAS_N!mv$IS`yO_gD-elin%#FpVBZV~IY&?PLN+rDO&doJ3CBM8;+M?ZA-E z*~D+3>^Gl~^le6?YA7{x!KFh}?H$|N9|k0#u^e&n)!1fDS2I;;4)s?Dm{>VLSx;{r zGz*})x8$%&(-lp%$9A9*s9Pl*MHjl+bp!VDw#|JXt5kWi+@|aj_)D_9b~1jpDc8{^ zkul}B>|k#h^{Jo0xZ9MZo(VAqnzo;m7|oRYJjuSZE%_+ky*wA8;`-yYnkp^ZYjo24 z$!UO1miuWpUR`W!<}eD!q%eQ}#?%muYUiePPU>}9?|R;@yl#rajT?z^T-kW_tSBy#&L(tX^+eaA^G5~`(j}6 z;=3^KX&s$DIP}djA%p>`>XVt{OUk2l;8b~~=(vuEH zJO$W1cF(jHZ*cn>M@7&9i7)}X0m_2V0c~hc-S=8f@aS3yPiqL^l0vJKqR&(FBMuGa zt9tqitA>|E%85B zV8Wqp$?cZ^p8U2!eCJqhGOo|_=CwL?a_kY9s>r}NwO&W|t|Ze#a5e-)tnopfG3J=mEk;6V6G?HB7|xe;5| z|4yj(WBGqxb7I}&JT2hiBO4K{as6jy{A#A|cU!FKS={GN1NV;x^B)kLqi~#TkT$D` z-|!V3O$~aE6@16U3QJW6giQyUY2ElS-1TjDRU6IG)H<9@7gPX# zb_UlweF=l*wIaUaL&ES1#zTU9A{*QCF+^*eeKH#%pD`@U3$E5nN=E87mX+_1lipvi z2xRX%TJOLouOl|^X3{4&W6us6D1rJh@EOX(f2NaU??E{lAY~(aaqVvJb9+BtEC0XW zJ371)MPBA&?r%y&19~1@{@%z3><8BxgrU1_vq9gT539g?o|Ck{G(waWg)#33bDj~ivp3neuM1l+zow)o?+pjky_-i^GLs+0B1$E! zrga`^t2I|rxLf=8sucW_j)2$zHDa{1*V>kqbz$$05ru9reGhLG|8n1d#!ht6l6eM! zO;(k?%QL~qFPb8jZ6jWwT0fQm6R_<%SL)wK%zqz$|J-$k`&MfADXr9b;?=(PzL^H zOCP9PesF*BJ4kk)NHeH{k8nJI3!)T% zmfFGyIc0m!9mpj!a`sT}oX}8-<(thX0C#$oD10|pIV?GQDbqP^(HQkk=qXizpX`oj z_Y8|=YNcw`cAktw-mfigmuC-5t98bW%h>ia%cxDW2)umt`n9Q0Un_@ z(3FheyHvzv)o7t{YIQ`mE4!Jyt_feER}oiAOz*xd*Wyujd?PI3n(MbX3v(JpJ>AT3 zUAr|L*uMr(aXhAE#o%~cv#(7k-hVA&^BCzo7Uljk_85%+;aQ@V=m+)5oRx zXn$ETn$JFl*dP%MN`)=3u%u=_)t-LXp@5jWmGBYHK^v~M(xMU16~+k`WiG3-6w`&F zd0v+n%^Q)$!z$A@3f>r(6n1VH$$8uMxc$_r*u3CJ#~UkU5SsjA7c7|`OOLmbJcHib zZ6{vw&GVbHKCFa|Z4ar{`UX+f*1FQb^dnJqlcYF%73ZnV5z$a4rNV(d8`=#~{2c3~ z0d{$cm?DpMOcU5`Y&h}J)qxh{`ddo=81qIG;v4f)^I9Sz?c7$FHfGkr5v|jYla4gH zf`|Fa0VVM{1#+dQ&#q>5n5rVpUKk04;7m`y{zy|c2_ZT#Ley3Kqw zt{GR<^Hik!gI{pt{KXi9VYHH<4=?`d#Oh-?3H5f5a-X_L6{7q#(h_+U8`dadNPyWK zkzn@3Fa~3jW6=e1K^K-{E~pM;VYXp;vCP_a25aMTQh+Ea|Jl!TM#J$&^!;ZP;cIX( z3xBv=HP{ueu@W5YTL%h-_m*@EW->hzf8Zfnu8XC@(gH}I$WZ)d$Eh?U*$qSZ&8arw zu_{kVjp!}9Dx84zyDbL?H+_gn+{GKq!XmW`uBFbOJqCvJkbu-uTD62KHc|pGqUZfb z4bv8kKoyw8O3w?LzN|O5jS;sCiI5>WQVH;*KG;�kLhmq;l_lVQTe|l!cnr@9ed> z$V^wFRdzGfBqD?6Uz7VGb*EMqViYWRVJ2N ze`9LF6ZYrfhB;K`nnXNYE>nBE{s+IWHGQ|F5|`#e=Egw@PiG{nQkZ&n);qOi>A*Bf zY;tiq^}g07;#`tvKT%5;Au?w9Ea;3ats;S8Xg-~&lYdbzVL=fPLOxTl-WV%mmcpro zrxq%ApMYIWFWD6~S8k_aNa9G&Cz!dE%aT$~Csn#8UqO)^^uWyMf4uu>0>lojDEI^y z|K48gEw^L=ln7o>jlM3ckEGY(_9m+-{a3osd-+gcs5Hx!C;yFGH>5(--@&i#c+S-+ zQIuS`s;5hKxzWG5@qCkcO$%{+ASI$28Uyk%D1i_hGAkH1;sf`uWr{?&EwxkYipfTT z3#7RDh@o3JR`{^!&KOVSs@1eoMCUpTUzM=J^BuI9i2HUJWs~zSqjVNdbx(7evzb{W zs^VQ>tR74dI#?KOTST`hujdQ=9fyd`EZe({RJsJ>Q~cMEH&$u6aSksH9Lw#i_lTa_ z0E$;q*pmjOx^NxyDOZ-T?*-qCcsnaLS;MFnw`}lk(?4=Dm@j3&8eZ}yTs^RXzx`#* zHs8Y;0w=QtkBgZ=<3AH47+nRGrbiYlwv5civc(jc?Y4VF_FMXI|B1@pMKx;~_&!xG zaxS-rweZyH-RR~JoV`Z4LLihl=^96TVe&;CZ&dGUT_~n&je?4w*el7Yr?!2Te?DK15syy1JQo$m{qM5=N}+}8zp`8ha;lJyKepNsj@e0l z-es&UKeiP39Y;J}??w;u4!5V*hfuxkygfdTKu_D+FlFx@COt+fT}ej=JMCUg@lEbB z+-0~6iHTJBv{LN+a%U49+f;?0l~UG&4B#Ej8|}dMeLT7cI#BdS+L~`+IM5K?klgqdNW?RUUg%^ z|9<`x+-a{#={+9kFu@nv?M{Cl0(TRA=*Y+(0P%;_du{?{y7nM{c7s``4fnqN)mYe? zb^}$sFMYQjIaROyP1D=?P@~Ozp^g2vGGp}ERNLL@XbkJ&(R@b-?)syGSnk>|!a`j7Rzwcv9=*AI>0 z^-cJIkFI|+U44K3)@%PffBW_MarXy2mi9;gWv&TD;{rnT#laH#gMDZ6 zIS)pe99GK^N;Dt9N(v=D0W>3rYFdZe(?qa}<90ED%_T=FA@@AR>Tce-?9N9XCQrPN zK$*@*-6ls_&PTg4L7OMTISEqF&O<_@z{I22@F0iznh)nh3}^wXVhfo5he@8QEAdHK zfNMm7XE%nu{`Yta6^mmBKZqiqWmmrslf#07sHA|Xs7p+|~R!XYE9lQ=z;^{Axw}gbt6gS*^%Lo*- zPJ5=plrysw@1dsjT7@)GnUrpnj3s7gK4t?6g~v*igTOD$kMWFW#w^=rTlJI_<9p1* zW{anL%sFP0js|y{#?3Z+>}Ov%_9!_iRj6}`{(Gck+{{b)W)AwcL%43>fnm<+W&Y1F zk0Hp+QO?|4XP>X5utpIYeS;Ax30TE!roo49g(|gqI8y z+a@MjjX|Bo#5fqj#sJ~dQ!COE^Enygx4=kzRaVN}&o3_`GUz884J7gIgKp~OkPpQ? znG-HF`ZiwV#RZZ!l2Yif&>-J`;fIFb-0eODh-?jg6FAhEHdpLFcsS67N-ZpJHkL>< zp+L)$d|cCuB6~Dl`)!(u=mOCBX^FHs3Ef4l(R84Vc4*`_s#cL=^z}E3-Jv=Hjd4RT zJ&f}5Sm4y^q?rIL27vTxLu(U z8VEo%i%KLJY79rAUMz~rMQhw}bjK3L+9c|HB#Po4NFqz4p+T#0(l3N}d?$7!uVslM zM2Arf5F|QQCqJ@`I!r28c}zYWZ(r5_+}@~9|L$Az8cL@Jx4*?r3%O=3I}0PMVz*BT z*XbjUmIK323Cm2VEt9|>M05y30E|~!e?2-&Ga94yL&vYZU%eziR_4KsM>1x|GFH|s z<61aeKpYDjoXIpFX&^enJVnc)_*^eu)9!UjUzl52S+RjjNE#n)ilx`aH=Z&fBTHE# zdUfhSiv}QWgqUR^P;fj@&X?xb=fLn>!cQV_DQ#{a2ZK58MIb(mjo>=n8H2cyGAQM& z9|nQ`T-R3J;=oQkFunz(fL0#Lvkwo%{q>wWKQ>G}I%;)g=0M#iOS0&-SFE3@<*6#+U1HDIu4#?+gUk8&PH%# zFPi2MoY1S=9Kz6_6*8JvuO-uy5BOaj$H5&1GL{wSRSL=5%THVXq-2Quq(exD=H(cw zCn}+oWHS3&S-H0dzlKBt14TAyA5#O?$zfGW;U6%pXRzx#94tl=89VRVKKnk;=!S6i zx@w3*e&V@}rO9J&Oh%gK214DjvRDVjmKLh~3#T>mcp6OT&h? zRCcNnKR4m)7HZe*;rYHY?BM|HADHU3CdHM;Xzv;7OTH17sZN!YHeoOpTQOm{^tM@5 znw}Zs$vU7k0hacx-irZZNQd(!y)sw82G&#IsK8P(n-HGfZ!OkhB^R#K^b;Ir{>@=^ z%az@6!@~|Lr=%vIAoScTyMStj(!pQTsfD8s(rDM!r{zWF>qk@X10>}~>WjxRT7B># z%!4HrhzAl1>l^!LCoe2Oco%R)$eD4P7E`eru%W^cBg$ffqX@OW+_fz%Z|x!xfcb?D z=eu6vXw#sr=kc^SY3*u$#A~Gq!eT`-xj@5ZMh0sFrMWzhji1xi!nRb2xV7U>c+YPhl}1)LsAi{5X}r~@d^7j7_=0ii^f zUQl&|PBv$XHeG>`@BUCG166ZQs)lRPU~R4+YfLqBwASg>>vE5_vl0bhiDN9(j6zp# z55KXBJ+fAo+u6#ohGme2m8iKO&>g{eQy*s5GOlC%>mP-GcGjC;^xX++Sic4fguUn5 zl+cJK6aCA(y@dA<>lp|Cr?Ny*@jj|oZzul1dMm%$s&xZ5v9Em;is4k-7)0GD9s`qh zjXZE&+44c|Lfrr@GQ932H)zm6BjTr>Q>eh>|6(U>yh08=Aq2pof~z*;h;k0eTGp9-)l zvk^a*+M3It@y{`M8VEQWDBY{@McgutX5i`Ldal%=2mYB?m65E)K$DBn@4J${u|>bu zqqZ^7-PzTL^eM+iu;RK8`mf;Co#++xqKEC^@KQ2waPT!!Y=V$BE7%!{St2g3X(SMB zu7aHYkG6a2&Mey21svO|*tTukwr$(CZQHi3N>XuBv2B05);Vpp{R{TEoHwJjc{BSQ z4G4jf-hebgUTZ0xiEPiUc|-Db zcGH=%w!fnU@rs-^{ase+pu|C2vGJWqTZJ$^&xE!h`W_r9Cd9FNgCqmWNgtA=D_ljj zbKb#b!x>WP}X#+5|LX(KwmP_9`&$2jpx!XJze6 zZX}y*)Rk^^g#2ZWj%4B+5|M)f)N^->yQriexYeY*JLmQRA z&;x2q4%UWeN4m}nb~+n(iPJXGk``TQHjAwVr3u|>1d(QhJ=IYAV|hlWkUC5nhGhMY z=V-@F`zvGBm%g+bt8fl<)>@|qBbUz`&x=c+6cgcJP689T#Hx*YR}>B~MAp?O(?6gl z0k%J7rNz@iHu_YK{v6B@p6cpGDcN@XSFiEpZ`0goaFws#YtM2RZI;3A^2{9>tvnxWMlke!^BoR^tBjJRNXL- z>CASY-dvlz=F9Q#`o&g>IQxl+TuHA7`Yc3|fAiGSFN)*`*FijE0To=H*Tx+XCI}VG z@$~2F>`<*;&+;>71m1-#rVyBf)~p?BGEuG`YBCt?d_^{AJD7#MT9#%jGm|KpYMMM`tib4KCb`4wsFvmCo$X zl_dswG#+gAAC@z(`33F=${{M5v_&&-wI&`;DGb%#J_#W`2?0#Ut#4qnB&h*j(Gq|j zba|iZg4QIHo;HcXc8d1ZpAXj%vJe31CD}<$Ocj@|tXVN#{q4(vJ=3JH-M)JV*9xUO z1@H5&38*A>+ze0NDNo)wM}9!(giU$7i31XS{VE34GafIr2vsS8jRS&*n!nd!@otxW z?hx$0QuEz30Mw?1;*i`vfcZEDqXgNMV}j>kawINqnMV!BK##Uh$E_vMs~&pv8P${z z<*KtqqWck|+@0`Se3%pc48*4UO%t`p)O`e3sN1fCaWs+ZVk7pAeQ?bLn3gnji@|wG zVGsGKkQ|gB0fv7cvVWhzf1iT?fPw$;62BEPqj@u*K9e4brfh&DgmEPGsTokE0pn{% zGIDHDb>iZ9G{VPI>DY!5s8?ui!{`6^_3wKT^%`mWLMg`@Dd?dv!8vEh^~g{B&5*VX z0}j{#IYRl=;o;q*pL2Q5!$7|r!2&JMfVzypv&O(P%QI$AKf8VazV$%x8F!+`q{ivY zTZ+I_N%3opz-xoRCxhAt4cm*u)Mp1ktN?K1p-8aX|1dX@elPINQSz@h|37P4KwKi= zEkZyWz^PsG{GTT?0)k&6Cu9*xY#$NK?7qR|#v(B=Vo_9+C&m-@_~^y= z!;Qp}?BapEzrmBuL`(%Gi!V7-%;j=8oK9A{QY;h-1cD&Yxl;j_N+l9WJ>B6Ky-#%Cu% zKzhaTqzuuQ;z_67Q$^rP|9+eXBB55q=pHz)7Do_;;FLxI#obXsK)h%uajgNepy#ab z+}(5h*{Y=tz-O|ycl=?3f{R3YXri!8(cvwN&4hHxv0~Tawo6kKMX}1#ebdp=B}?Rt zkW94pOLLMRqk|w!B*VH+L=C~LN=aL>mGu&lqQk>YSkq{Zf~dkSmqRV~%udq+!0@Vz zLKV1^(Qw<^P}FUuhjl@F#nGzDmyJL-62s&`Mm-6;yei6447iLlL$laz3gpmoYJb$U z?9dzSwb3NGYAc0`!Y~KB9XLi@yW(Xa_qkgd!*E61!qMJh_E{DaKYtJL5sI3Zy~-9g z5?P|kF=?66J+NqM48dz`J54=LMxx@O!0QF^hs9(AEa{Se4~(Pna%_>|G%gi$fV* z+TNn2SCmp2^us-^DwnG*_`ILvG_Pebg!`E$k-8yAL@@DV2R#`N$!;Vb-Wyse?k=md zG6zCKJu{n-Pv}@)MLNTJY>%_9b=5RQk5kpyyC|0QuaiuAD$6W;s_WfyRMu{^l;?GA z#<$$cOoE9e7=|)kbGUy7jy%i0an64GeuBtDY6o(!1(}bw&aVDk4l#bEdMokgU3!4b z{QN)x_64Q2%C)xAoQEVq?`2~`O*FNvDpbT2u!cdC@bWhZQU7f}iedNdFiAoG{W!~E z_x-deLI2~ts$tg~X`oO4^SbMH_w#lL(ctTTib24cD}R#6?s3E6uir+h$m8YfdBfkI z(rQYrfzS8bzknKk=fc;ogg|iL0!oP2&|x|x52CJLr*N1;(8{;Wl_?697bK8BKtfWe zPkWFvugJZAW1^_*cu-22!`D?ZQtQ+CFh-an8)l;NX2~j$kE*O;xtA==Py2{O8nwPs z)6|9gIJx*xqGa?;7I}pbw3MVUR#@cvh`pMiS(##V>2c2R$nfYLnBq()w(`noa>(PE z;%w}-!8Tgk*!!3hs!c|SSw2O$_n8uW&o@a4kXMHUz*v$(2+4_ItVcx1S(0M($w_gp zM20mYbWT|^dN;`#eNV>>L0Gaz2q~FktS3xKS+ZvIDOq!_CoDx+a#r#w*?(G3*qXBB z?Ceu=cAic+hOp$F;!|>uSx>osXUV(Or{rC`p7QKtDR|AN#o4Ep;60y72w^KH#iy2% zv7JjPWhe?T!WmvP5{^V59V_WnJQj_~ z@P90!=oEd84mWYV*8P)E;#T9VNNEYO`yXKNWijmpx_qG_R_~eZ_y0*KEdP~IDpZUA zNhn2r0x|e?n$73ShI(^abKPwCYYiqN`E)t%sasLbm*f9mLV*M)LH)0UqTkcyvbDf8 zcU}(L_I#;*dh}oxs*v|z3B}iwStPfTp*dga|4Aqr!}@$=Pmhn61pzVyIDW$`9R)sM z2%39AmwpE!9uBmYyCF!?YlsBbdqF#4nDS+K@LRu;_apGbGZ!LBgMWqj8kbfcMANpP z9mH@^f*i)Ou4al5F@IPb#`ET{6Gjq)V3m6CXlCyJlTgZ{5*ZlZ_Xs6RDfUu%m8~Mv zIDR@H({=SX$}$Z5JCETF(zTAGhrZU2vTeDc4{)W%sVo$6y19<>1oz&Bi7CU+E3yK@ zvktR@+6~W&!pb=Z;xs?Thk!2pU_N!Vl}Krty=Hk-8b?OIp}a-0riM z*4536)>Uzty1I~bO)Ih<;(yx8NV?69ceWz-SF$l<=rq+!=imUGEJ^ScDnzJ!+;Xh)%^qlvhCd@rjBhh zZjvPpQ!rwOE>jJBL9Js4mJg3}10cAzWuEx&EYmRa3~tMeC@LRQKlv?e{do6B?$x~F z?RLxZLAvhKABJB{tgBii-E2!M>9Wt`ToQKfT`CEAw2Qisbk{ZZ9Nw(0dL7r+bM$+1 zZW|V_fE}w%>%&?1+SbA4$Z6ND@3XaN4PK9(=;c}uxKsk=t_9!EIpq zLni|t&TV^|xz>>Uh|KqsMuYYDaW%wzPPh&AMG~|Qt8w5CJ^o|127kuZ_{k;4+>=sc=Zowdo?If_ozUXo-4@MR zz<3kRL8g%}g%2(pN|4|VFL9*1%FYMAQrISMi66Z$!Ibf&*wpqdVinRTD_GL7J*7|} zDigd*?L8)Osm3fRX0&51Fl-l={<(Qs9~&hxOn{*TM<`1i()hjog-tSvx(fsW zEehgxWR1c6}3sL~b=c?n;DRBtx>LBSoH(mowkcVE1lN0Skue zh&~u#MHtaZk=o`h_6DF4WX>XaXzLHwv6tm!Pn{BOOBPVYNa7T-NrEST9A@J}%R_E{FXTq9(DY2-AAIAn?S=luGiE zhrUrQB&^E&zE9(!^R8Do%|qetQ-+DFMFr<-KbJ>cUXV0$pR3CKd2TU~+ESH4UXue!#gkisAjAG#T7=erIa7AS5Zpnp90N(__vzez&ELumv6m$ulFy`HAKG( z%4v|(869Yz*_Z097X^8692i~)jJPjBX(QL?pfx2Hfc$0jeky$!!Dss#G-YJpaOA09 zv!hdr6=+3^1`4+(c(;MJPHkwEBy51*;*Pd>QvLU}{pAxeYo2$t3_vKmrn$bMmWPnw zvfs|R3($pIjz(^GH4<+oPtfgS$NA_J_bX2yu>u$DpiQN^#xDKd(xHg#vf{Zr^+udK zFG5;#z8sS%egERA3wp2d&}YpD$Fty?XN#0%B9M4+kReNpG{kqEGec3_>N1_(Ot;wMB%6=obmwBa%DW-~cQeVD!dG!fomc|By;sHvm4>Ff5l zoX+%xzDL*AwLJ+^?;fTD6y`x3QdmUJ{7?PU_3X7}az^ff5a8H-d|(M8zUS+h!2@pA z0zYEuEBJ(Sp=;i4&xuuA;mFF*vrd-&K}O?ihvrn8y>R5Ioag%-DA0SQLjJirMEF8& zg4<4)C@Fsn+7-nDv`#?J3+$Aq5kQ+bb07vn?E456LZd5`WI zBUC@c-n~Qr*LsSNE;z{U5gh)f$Ot_}O2pUGm3efa*1lh|lf5S#3-9xYyEg@hd?$Pg zU>%_PPo)ZcJ9G|jBQtH)2@<7?WAmOiShr_{WxVQ}j<3+Ug3mfY{&!&uUNfkGx5uUU}yAiT*305&stwfjiha9kRa}JAiFaCB)+t!26lPz(>Kt*Fyu~ z`z|2h^H2fs5h3(Jv_+fzy5RRO>9e9$;`hbQ3Bnm@i|Y855jfJ$D;|u;+<{WGk3!dh zjHC-w^lLImwlN6rEa*8R0M-cJb5ZnnQZRLAa3_%_jk+k~x(M4-Fl!|g zJ!S}VWC%-T2-leizh;PFX9zE5Xn-iL@FDBRlIuVn(l1b`B;5Ki!B9_Xe5SJ%#++<|yr0z0%A zq2A=$p5#`{8C_cGCgd1hnHgQ(87OSZh0W8YrHt2%Z35!Yyyy_XZo*B2<8Mhq?3)Bd#2n4mq z8Goc1f8rT`1_jlb>UlNETp$RlSA^H|78aHnbD@#&G6_4s7Ng8bw3Qx(WfD6v8Gb~b zh!Q~!$aK!ijBsRv_&W*u3n>YSFbORS2-Yy^S5^{sR1$7ilHpxqXJR}tGzf`QGA?2K z=1}C8nnUZEe|TohTV_HbM}qof!rWWT58#&%ms((Dc`Ukn)V!wU1{3hPATa8Al4Mq@ zv=uO>mDh@8G6i%R8C9BU7I2ne@?>gk9F%d|S~9a#0I;IR2d6jdxz_+viUG7|9b_UU zdn`9JFFzI)ZGMu1jYU{>;VrXQ_VdQfC$XJk6FRf_F9 zONyp-!fGV)qsysyRO>|S&3Iz|cpQ3FM&UV-FLY+9RyLktc7;}UXqBh2bEW}wdNh~+ z;6&zoWF&M^Wa4{zAA5%GNDfj)Vgi?!hgYgh7XqAPZqa!XVN~{PR`&dR?gF%PEtNz) zSNhXlGP$9@%$-lPR8AXK4%S)b5a3<4yUNgsD&rK|k%KB#>pl)9F?SNW0E;U-cq4al zDz}C!?{PS-*%IyHE&Kj7Zw<=>)H)9sCLhx>2k$lJ&#x@YvmB&nn-4B0x2b&1{rpj? z0(hhXFR6l;_oCORG=burFRoOuYJUYq?`^3_ijQRPiacv zh_%A#!o=c8tP};tO4Say3&36RA_pXtb-)31>0xlWRd)$@cF7%c1k6U28cW=*ROxtD zE`E1~mvy$Rw_jj(-n0fpjT5{wwwtA3`V*9gZY4r4BcwVtyqb4eZFF7>Ruze6Il8w) z29|SjRe3vE5!ZCpn6_>qb+yL_kW^PL*&YOGAn-aj2vK5PMRYc2b@lhVeyeq!-UsYH zCA|7(-f?wU@3il1BR2I#aVuq^8CR9ido4(X&@#AfsY^INw|-`}{6{p`@9rWTtZEn? zpylWa32z|p?z)$(YG7C@dFs;AjG8qd2nnEaqv;@~sha6U_rkNL0q$B8>2ez{+TA@U za`W20afHS+9l%-+93TU7%h60mr6p~umWCKXQ%t2xy=n8HTvE_hdk%sg==#h z)L;?@60GH5Bhp5rj8^rgbiBr5(<4xmurglv` zOfB_C{XQqeEq4utP2`JoO-B(X2lwWK2%z} zVqlHRuZEtZn%9penrH~MO$dcX=w|E=*BpO8nS_8#nlChk;2y_A75P3RXoL@DCQyA% zLX)4uw0?6N9zZJ2&xoQ$n0zj`THq*Q){7IQPq%Cr>3q zS+_6}{KH~hd39-(R8!vxfz2pIgbz(rU{4gfn_+kRQe(#xc6e0$J*04fWtn07Lup|)awZ;8m!CD_hUi_%j$ zeIe+tm+GH&-Q3SoSe&+k7>N8q?1sCns+pqI?lyZtpo%kiLL~SKPw)fG_ATqdi4Q1T zR-q1_`X8!c&c^oHyW;z4xJT*^2yOrR=;)a}_uJ_qBDiK!*%F!=8gfl>^X!!F5s_^4 zJ~wh<5kRQg4r*t-j2(k=r_vtOA6KJf$*v%Gs(W;tMaIc%n{K{6L=YgLaU{2=&wiEK z+;^Wk(ZV_vBrs5*rei`dA;Bp_-X0SgcmB+o_OPC@nRaHN@-JAJDV_;PX>Ye)Y)4!V zGT0J^u3ol{-ltO7+eY13SN#wj3s^PtqCl9$v`GdGCXYldpFe_rtJG|^2o`}|r>0eB zS_Vr+Fjup!obOW*cU>ygFd}8pWi^dewMlBdZQnSM5tqI-8h7y!XBCol}Q04X$!(b$Y5H#>2 z#x53JlCLxzef7+gciT5I9JkUDA3qB2LY;>v=&YW8N!rn$hp9r`HgP(jirAGxti`>v zm5WM!cCajw3YS1HSyFFPp?A{J@tloOT>K%OSNJ-zL1VU3QBNs zOE_xJO5?W7*!68=rDVnh@j$`snt+DYu4V6)On~3O zO7-;Fl-vDnq; z>Q%Dbfb<8rhI_cr7VMp`W@kH^*%Y4WuqiKd#129o5gfaZWJWq&Ig3H%o3DBIPW746 zJ-A&sx##6%zwUntwte{BH=0b)Yhdhu#l_u*Z{jYsxJew)Up|!I#MgOeFufDDvqJt` zUdh5O$b+y^d^SJ`f~JToTQGlC8F576xn*R~*oFCRdK?+SLgEbLY^Xx6arYG$w(Tm? z^xFRBb!+zk|HRZw3~rp#wf_;8)@9$0I5;!{Jwn5ssF&M2gPe^gnyaWWZB(tb>|!rT>>7;hu)MEYyT2LF;tKdnJn|M!R{%V@$HP7PiaO!S$_+@6U+)kh| z4K^2cVCRy)uC#l_&3`81?g4!2)<|w^bob41;gRt2@zSeAF`?7fAs_UzHv9b^3rt6DF z4Pg+d{H2&p#iLAYtoBElPKUYNM9;sGxgN%(M5%&Qg{7R2Wipwx#$`iCPofF9;M*pH zTB6cCLqi|7rB*8C3nU_*kFyJ@Z9gZNWG??J`6Gn<*SZK)I z@c(}6E<*I!J-a-606|-1-Lz#vDq=L`VQgPRb9EF71Gi4R=wSI?Y_mquEO`3g9rwGKX!Nj4*S}T8p7~UCHs19*Wk* z0imsfH$Fe~GV$>fEULhRvj-gnA{|}+mJkKD0?=}LG)uCKa%Ouf$xAEMJV>E3&Z#bw zm#nZ3$%%Zk4#`V^|5*v%JS=6U+i_J9AreY;o0WJiQB&7Vx@#8G&2eSQ zXcgqJED=jx-21w`Fsxor63&cJv9D28)uE3x%g_nuI*D0@78%}tG28e+t|9WH(#e7no?=6z}nrXbX}D|HnbAj z=J%WvRT!%-ha4<}FnpSK)bFL;bmgX(FP(JURt@}(r~l{#IK#D=6^u#;m8TKicb@&! z-;9L3l9dkgNT8}W#3dkj9m7KWc_{eE;>+J8$#kl0tUaQEKyZlaKWozIAb@g5JHD_k9F6F;lcJXjAjZU zcp{j;!|o)(RdH^Jj|Nt-7Ks%sbC777qj1~9&>x~fQ1c=oDgPb1sgx3TU z92x)WdlO>}cZyNAEk>X66lY1k$8lUF0^AS>p_Vs>QjTUH>uX}wZy-6E*qG#+n~w(F zAteUkl8OmtVw2nnLBpArLjE1BEQqZQN3&H$UUwpgQ)Xl}tu!U4c%K?FV?=``H6x*z z5K79eOz`(T#8twai28FM-!fZ*KQKO_7Tp-Z?;A}17{MHdBa>pjV1-G^&PAY$E|LqDUIgEnJ%xPyBk$3^ly;nZ#${2( z=EkpLg(SB`AL5nh>Caj!jBB2bvY|KgHVm& zbZt{1vyjWQVl`z;KkRIiwv$Y2G~kEos&Ke^*i2EMN(9_S`lw$LyvivAz2tkTL;A@5 zr&Y0zKJ;TtyLSGpRu^2;qzh%YpNP|N{r9(-=|&`C1MpkFmp%2v)ihc<0;_vS%mH-nsF2+~;>o{jXpf^5@N&V7c?A0^G}A z%V>pT5X%5hm)T`1n)wa1!1kW=k75JqfaiTE|3{&K_*UN4-y_vg7nd`;qAYC%S#k!Ae)9fweo#san2({I1okg9jq zRsyTWDNaoYs|(Vw>9Th>!X#}JF1*sammeC9l^YZ`U|>uN64AQ*Oc*bK>yLc})qCG* z1*s~0h^2SN(V_o43SD?^;|D2_a#6580+H)LlY(#Lk z#KsuF3^p}|bJX6V?#@BN*Qoz%hAA)#V~JwSeGl)q_5h?`74U!wtAn9UW5MUoAjAw0 zHcG*WRL!|WI9njI+}Ku-fWblJ6Lj%oD)T3ozd{K1*}>J2fDw0j9IWFxE1^rbMODIG z5FaUO2305bF$`CAVYm||h`?aq^e4p^42BvKxWXBj$(bPHm!JY?xa**RdND`)BaM$s zS2`@|aY#OOAhCU*052xlwQz2L4(Q7}C`;y`=p^VkXk5^J#AgwN-QJKLA+HJQKz(Pa zC};S;&QV8~aDwep$(-E#kzO&&0R_)Kv08l2W&9jf$8e;qq#ZT~k^TwD0IC4IcSJ;~f;bmK;11NOGgSIuND-`) zc)br#rmKp1$eU(VSf)_iTTIPc6!HpMx))p;DIDLFDtLpRMREX3dBR*#fEf$ZQ?tNc za7@`!*t>Rux)%v+dsx$U%ztQn35t8sr`YriB0`I^R}s`vs?WKZ{Uy4H&@&vcUp2yI z2m!As{xw()f945#FTNHH3_ScGt!oq*95xUb>2WEQrhl(qvB$dy5m+6v+cB-VuL2x@Km zYuLWr2rfs-IvVm-7s1Qrw;a!eyRRMYv)HRf;Inq3`6$t!*DJ$+&s&Ws%@P*{ZXA%WmM z)Qua)RXKy5+GW@!MjwlPNZbrfC}P|g1pLK1QaEWL74|6t=_C}9u)+u!Y8mY3%};vl z<4T-jWko#9p|{%}7tRgM^_`uBsa4&ZF`WRbiFdUvcEk)G zLlx7>icg8{B`*#y_f4Jg0cOb@8U*1}TNLoufi;Ad)zk!cQUYJs6tAcMQh%c{F0f}m7L)ig>^<21mjEZ z#0~+aY`Yg~i`S*k-9#kvts3Bk*B}Hpw}dc7;$~(8XOdH~RGbII7f+oYm2rF*oNs63 zcIOy_P49D3mC1%anJ6GJVV>s9PP4YaY^nr&aEcYqVmu>FgG4O25@YGfWgy8c?L}w{ zh7Z*SX%YT|Ho(c_7`9@m`xrrn}#bA~w4Z{jTXDM<7Wf~zp6n{TQ363{&k*ZF(fekeDECM3qKWn3O zFoPXCiN$tERF{DL1XrkiXtv%4k}!PwO^*sJA8Cyt6{_Bzl>3!u}%Z%zQneZ?(O7cA*gu5}8oOd?DO zLHYBBODY#!WB8>ag(a>O@MJoRb>xwg8=nqPRn3Te?FRRI>-^JF-0hZ!kN%u}jdv`< z2iqdsdlon3^7mcxe*~#}VhVgbk&^Qc$meR>Iw@4!g;~dHpKS!aT&gyZE=g@|bbD^( z?}%mPttq?Bki_tj*R)^^%OH>-G?UdKk}>J+s5!tXF@Qmi&&o@5@;KLW_0fV?lrhxr zL_0+eD^fzptcMuiK~DEH2pr261PUkYgxefzN{NhZ2!qR3$~4q1>ldl^$Zciq#AN%a zt)?mZ;jwG={1E`F4LGi8gKh2MV>0MtY8qSa;$sAml5Sie;kwRl)R<{)aI-Gwq5f8x zSt;A#blvK(+Yy}_anIeM1#U{8(V?8ui^*J;*iO#ESt?%ig zg{QsZr@zyyburg1R<`gqR=b(D{L{8mc)Z1+s*R~*@T^BY^GzQT!}YfFmml=csa=9% z%gkpTZr7Z_f+sy>ev;sk;>eSJ?7AKV0^%Xx-aO;qeK|u^bJZiUR`~oRS4aIecI^uZ{QHA`Z)1A z>52!(NqQ9Jc-M6Nr(cFaWaY1V>nQAuq^orR5R8<0@b=?A8adBkOm;Qy`s_b_2f|JI z`*mA02l@aIV@3!~1oS;%d`6O!MplT1r!nGd>UszuAAMQ-R{+{_TE%_9%MP^yo}uTNy!|H>hp z%KOj6#T41YpC}>hQuKr=ac{w67z=}J_SqeqBh1u=>FKN<$eEh@t5@FXu~XECU*6-F zk(>oX06j@rhUr@}S)UpI;Xvdu&knKxC>EXN5FL5f(`miT2MwRrN19uW4;Pym+S*fi zKAThFSu7BXr=6Yp(_2NNoH;f9%Kj&zh+F3NpXaVxc~240v$IlF&IiypSm1ZMx-%t) zvnJtNHJVyC`kI*Rnx)pWr_~$6_Mb9^oZE^YLvAoMtnQA;o;2WJt`sR7656z~(_tYM zwuEmKhji)0UpB&-(b=p2mbGTCQS#%D_jt=F8#n8E8E&WpuC%XCk!LXIH|BFc&%HcY z*S7Il7AJV2q)ZHI%(tE82i}ZojNPRU7c)2}3Mq*%j88w7NanZMt)6zakx98YtTy0n zPZ%?|g>$pTrX=ONccKE2j zeaF!HEZ6?tb@(YKc)xJ~LcF6KbO8J~01G&RS)6^-EcRo9f)IDjSU(2BO|N zHe1HjPrYDJt;#d4klPQYcMT3#`JG+)2|M{eULxP1yrMf{29RF6Uqe;2tk>(;w%@^a z*fEITA^`3WKyOg5><9^*iKFFl1&O`W#{wXri|m~+67CC9oK4G|Tc~XdHgB*s8gAn4 z?%1*67RX-corOsr?4Rra%o*_e_D(3~&dK-}452oh0jTH}$h4=Fbbp;)E*{SNj5NXR zp5Y(WmhRN@PlM`QB8(re7N^ijw8a`gxfbjx=3Q`qKNcOh$et!KyyD;!xRBhI2@@QV z6+BAWSw9#*sSw=%6@KO#c&y96piFm6C^CkLU=q3|5zaWB96#gGaGQ#L(x-6qX@8(8 zaDozu<-&ji>AkO`vX%vdc|haZdU0h#eBL{8Gh%qQ{Y#}D@o3-h=uj{KsjxOc@%f(Lyg#hplbgfor}P5ql1_LvV&EWwUp2;QCoVWCPax>*6SP|uAE656 z@Yj*&SKyy#z;)SILIr3qzz+lUCnex%pTj?O;jUFIL(gi`xyNGy>|YvoJ|#3UpNzXtY{W*WOH}!)tQk< zT5Q?58<(Ovc~B>9OJcxJM@jG~{XLq_2*^b}0b?jwa=3}*|Nf9UUGL~|&0o7%?{M2x zN*<-?-T5VEmO|F>T-*Po_B%(zSuNu0-ECk|?zcd=&s|&DZs2(n#h(DJyo$m=aBj&} z+*v8kBB=nr6HT*d=*3J!OnGyxNQBZAY>TEac5K3OsuC>Q`Zg#W&yk(GMZ)MSqj9o3 zQzOK?nCgBrHwW6QkH?A^E@T<$P(DIGs7oX9`-T}(wLOf z0OKqpf$Y&F$ybYp{>I^5WcWj)Qej%UZCwQUWr5@rZ0^DD={e6*#1y$Y-ccBt$k<(# zp&0sSMYJ(ROqgXv+TK}E)k&O#RqEAM(qmD!EZ=)n z1v#H`M5FCaFb~%uwp=s+LNE4NhTRvd+_(Di-kwE3IPRXeWd!k(#!=IO<-ky9f$4IU zKrzzVj#8SV2%|_N+M@)6@0?6xtxU``eRvv~rj6afxXcQJ$+jvUBKfU1-b10rfCT#5 zO!sh=b?hKDY;%7XNuEXZp_4o{>OWy@=8a2YSuUX}j|H?eNi6tPscBCxWx#PYA)Z#V%6WU-C zW;pm0NhAfds40}M)0YQv3_mr~=*IAnWCfJ)E+<2Qeq23{Tyt?&8r!<$)@RsIAO1yq zL($o1Vf;PXXUY5#7`kVU;^*q8kHO1sC(g50FTi*Xnh`VxB0v_eHST4$Iv))&IdgiI zY|sqb#8n|CvT<=fCbVzWVGIiGN6&&cZ->_g77rguEdl{YX&Ixt&9J+%fS8UXgyo8h zljOReNh-N^MwMT&rxcD;H}G0b;%MfiSvAVE2k_f3+BGS2^j)KaoiaVu&g*cNv8dgeF8A$i$&86%&% z37_*#hcV2e?E)Ef$R7P0day#I?8%vXAP52h+BNa^AwJ5vMC~Udi}iir(M-OYAZ@7% zjss0P&SYh;u*iu9uCXTAU_=}U_oW_u^n0qzYAMcYY84{JxmF<&8K4t;wT^pUDKn#} z5YBW|vLm^eC9rE3;fGp42-z{IgFW?ca9QyI>G`N6KT7WpYmJzhcZ7|o;` zJp>zk9Fmjh4`(KZbRltyrECdR5u1ENY*-LEKVl0_*5=WXzN`6*ew03G$+Nh;WMzWl$ zZEA_C)Htrbdy7NlN^0>V@rC8|n2NEPVzvpm!~DARrXM}?3#s<=ea_j zDQ%(v1U>N=jx{Q#^-34>be6oDvM5=#YW8{psr=&^?hhd%TQrKz25UqIqn9!;J5rFE zn3Zmdd3a7)OS$16aveJen_)p$m|<$Yy{HhlDoFx^?z|=?QHiw#x?f{ z=T@5Snpe^13C9k z4i3}ZIEf86yVzb^RuN{0>rtn%y4qn|eI#q(ZOawMJq_aKN3T{YvDV|sh3@xN{Hih< zgkBRGWa|G5K5VUmA=&cvAD*kUrW<@p@^Tk?wb!(?*@3DOv@&D7< zcb#F%;I!dp{*s*AcirTyv$cxB(5km{>lfl)x5S><%M=ak7_kM65a2mNLhr;R(!G7! z&uPV6Z=dPZiVSaQ#?p5_IK;S8LjPQ)vV+cg*A**9H&^GPmz*;=WSr5+ z@8i*TnKS7mX=%UE{)02a%Rn*ju={dk)OQp541Ta@=+QD&c)Ud9vr$#jKZyA2daV6@ z(+$7;d+wV)^4HrI0M`TMrtiUNrDy;8Ck9aHX#D8^3cdH->r7$(vyyb`1$mnPL9y0* zuc_qILC5g9@fQuQMBybDz2cKABp8bP?R$?R`g5cn6tidHpM=uzbN}UJFZwM-02;wiYt0n(4v;)vuyU^N# za5jXPjsXrYghGk>L$^Skio{Ndh_(m>uD-`6M*!Eo1Ie!gS-uIUtPcNEP!&gr_)>`I zjD00$z&QE`{$c|BM7I-d{{XoN1sfnV{Y{7_UWA6BgoJM-Kq-xCEQbzm*nSP96q$xz zoP?mdk9@O-mj6#eF{*BqrfD<~R$MFx(|DFa9T??C5tJa9Z;!jQ1++L^%FBflwgk+Q*Q1r(yv792&|2KhCM~`M0=jsT^ZKIlq*51{67CyP(gYIvre?nulCSux0QskvbFoj~?rW}>S)Puva-G)-N!E~XeQi&n7qad6@ zW%{Zkv}a;;s-Uc@pa9Ne5#}=G6H~@!)7JBnm#nhv=i;BB63Nv;TBtlqOp%HsG0rhz z{a&-8++n%!W3u2ynoO{hq!3Ec^z=$$HR?Fca1yO%JnffK$l~K$T5_o9(#0Lq(#&9u z=uzrfSxHgT(oSJIMZhp^Xt`o1h(&Oy$Z!w^94zaeyj4EAH90v2H+-8K0Yor(omPwt zHj!>B@xTzOsq_g@v?(p2xkRhETsIjr61j4)D4XW8xw*O2O0e!vxL&ZRjXxNz=#blz zApI=xy6mYT0y%9VIr)8=ZF!N2e~I`wq4+_$c0Gm9F1dm`0sX%LT*fE@a0y2Gd33qL z0m30nf@MxGWo${ol!Ye&`G)XU6jY3qteGIRy=Gp<6>LE|61`?@i-WYhmJ&z7RA7|M zfhs}9#_r`7?aDs{{wkIhQB6-sincW5kGPoKz4#LG-zv9=yG;tQc`G0m?qwx z8Zu~>DaZmynu2l4`dM7e?LQ~cNlGZ|l$|dI20^V&WIfUhjq9=+* zVepj08E`{4oaq@fnNSKKp`F2x!6-_;Q`dmk^;M)6sHvLfqfpiX;6r>?BPj9+{z$#V8C6jP}{F1OWkzTqstgyYT5(nAAMg z({6TI0rJCULPG`C0%1z4S00d};w87l0f-)IgKCLoo`xp$r&D4pNK&i2=;eo20|=dJ zhE^+Tl0%|q=xQ4OSgoRA@I>OCa6@jky>H@>XrmJ#KV0f@W|en>L~*78sq&A2 z?q^rt>lx$HP;n9iR9XR_`IOKuk3fam@N&+TY|B*=r(2CTwj#al5f6E4&dsxB!iXtuCZj5 zJUp#<(AwWRV~1hFCrAwf!J2z=m+)@IA~tLUH?36Y33a&+Dmbue&0aW(!Z%PVyE5ot zCZ()iWk`Cjc)pHHTgZ%Sb{qV8SyX zEo43aZ+jlB)wwMPwAb!_8bn)>J>@a@;7G0RsNmv<1 z;Tz9Oa_aK^TH}wA1IHaQea0=BxR=7q%n9xkRqvKi*~B6cORoL4Vy3^Um|^vZ@9iLN}R5LS8R5BH)WFd zKKgXRiuYI4<}wGiLL&GR#+eEeZ7Tz@*;=hrt^&o9svldpDsvU9QfvH%WSDmDboX&m zI_27?I38aj>l*lhEBJ#0Hai04Uk~MBzv~Gu=!XJm9Gp1woPvR(_PIL$w=8UB_HL+< zgZR}6c`m%diqimcCOK`7fsdcKY7+VC619x?7G>}4cq2F>;J93K;IqO(4=X+b?l*)OBuj~vLTZdoifX{hHyf=({`Ii$`m|vS&g7;;wagi3j44yu2X`Wll0pKp3B zSSPI}dgU#8L6W&j-Z-!KIzmEhH(*1rqAKQyE0;4*pJzH|5S^^AYtXqkyk4C#2za3b zh5sm>wTmvfZtt`Y`K)I<_9!;4%bS_kcs)Y8Z|AfrBKkjS@8#P6@wLyoe@?*0o~&1< z?0&;R$oj>$*T9@NB|rQINMYVhA}NDbVAfV(wbMIDAt$&&cDdu1uK#p!^7^@>`@gw6 zeub^u)+~}brJ;wUY4)>e#_-4I9i{_(g3@`AhNRFXyhk{3p(p5Gn!3wRe6EwUjk743 zXS`ddH*pGeM{Fhv%WG>|yMDGWovREv063;j_ja0Yi#w&nn&^dU32Zm0w^Kc{TfK2V zxxDLqJYxK>S3Ka3xrBG$c9tOG*tx{Ey?<^_>$+YQpLM{n>gah8I2IM=9BV8QPQ9%}l-J|a7G^*22Y)^5 zof;dCa(j#~G=JIf!Sc_dty;Kf)G&AuVM2qsX4xwE5YEAf3USTi zrx0Vtg&Gr9%r%i?NRcB+mNa=1WlEJRS+-mW(&b5*F=^J!IMb#`oH=>+^!c;rPLw!z z3I%!;X;P(2nKpI$6lzqdQ-KQ zwsYy$wVM_1UA=pC@b&vr?q9)!2@6J>w=mzsi5WM3$T6^E$&)EpwtN|LOQM-MclP`l zbm+&3J95QJ8fj$IqFJ|g{Tg=7iLjtqs|C15HCd}~l2fG$*3h`Jg7Y37QP%HQW=(VD zr|NiY>C>rK*E?&@oKv0vq+*F3dh>`AM$JKn%zWVEx;&9Gt`M4eQJOj}Mmc}9b^ZJK z_fKui6JH8ar4x%V+UgbhpgGW?gb0i!D_OLAkGEO4a}T525{hMxRmSVYlhdL@2$%Wh z@DIflRb0^|4!b!BmkGrK377ch3vWYDc9RJIzk^KCWGDhXX%97iG;t-N_?T02wYW&E z=E4*6W0A@#t(?w?8YXxw#%gLvhZJiTsu9Q0VDZG2E>TNxmV|mVsJuA%L1w{;OgUvl zS%}oclo2~j5`ZP=_~;Z24XTYf#5yr07xXB+64FQ|%?yqo8;q~G`P35e&-^&t@gP?^ zU1g4M4CyVsi@k%f|SZnAt7Y=nb%@{+!ls3Urq$MZTY-K!jj#y$8_og3N6_(z5 z?fr_MZm_|m-GgN1b4G2|9oQgp%!8%>n_iJzXh>vr3YW)$v%RPm5W9Kh6oEmkPds_= z{TSqs%X$+IAt%!1PE*e7Xk8j8hO#(<-UVqp4aa~byCvx)#T1)cCD9jq)ab-Re$WDO zOM|fKccEe@Ql-#Becskrk*&TOYuS`-u!i^s#fxBPzkTe}F+yd!;8M><+r(ns>~lSB z)r4_f36Jzq-Qw`hn(w~-W)PaissUg}oCy>q(VOt7bHlF>!g*C&)c#UCZXYG`)rHgc zQX+u0Tr}gU5A5~t(oM%2=j9FpFlm5uMGU878HZMsHtC*SRCH;;Pu7Rwc5)QG3*uA2 z&DnKm$l^%^^z`PPx7V0XV{ocv1<81c_C1?0#^fr%?^7U@x<)TmlaWE3mGya9^P~p zFCf_ugBetjK+^H3pfp5z9u&@$KuEqBX%K}eRN)F)*g_Y+5QZ_7;S6b5LmS=@hdI>Y z4tdx^AN~-CK@{Q;iC9DC!atPGx_z@Un+eZYM>|G^eG-eGLL~GGu`ThAffVFiCa@@w zkV=b#ROBM-lCf%h1ZMzx1SBIVSxM(qD_WU^#TXB@GjnC-N7>Plwk%mnQ>Fx4dUR68 zvZc3LSW9xuI^`^BIioGf%xEFh$n+2bxmx}bm?NZ!As-?NV~{bJ$yBEMzQG4D&2f^; zl;$AG=e0R}0*lks<~C=-j}medoZ*b7ILSFnWR{aL0RaFZ`2+=I0R8|h2mmbrpaLKP zhX4W+00J8T0}B8N3jhiM0s{mB3mXCpA_5E(1QHYn6bA_o8wvpc3Kba&6(RiPsJ6MPuC1=S)3dj@v%JN%!nL@@b}wP!O!u+&f~()>cq*(#LU^m(&fa{?8VgA#?|x2*7?TT+s51V z$Jp%0+WN_^sL8go$*7M5O`pVk*%hct|)!NI}?aSK8%i8VB+VsoZ-OJ(K&E4Y7 z;rz|w$j;#P&f??F<>=Ag{LD|}s(%0+k*t)sc>E+qw_Sxpt+3L>O>eJfD#M+Rz1^5XH>;`8<6 zyuIV+*yHltiG8S(#`Ah z_3Qin?8U|G%**WL$n5Rx?atBd)YI_B$MNO8^3cxn&e8Mc#`Mh0^wQGw>&x`?^!4l5 z_4WGp>csZ(%=Yrt_VU{H^y>Eh{`dIx_{+`u&(ZqR)cV=l`swHT?Zx`@+xzt5{O-&A z@6!I>;r{Z{{{Hs=^3DJA$N%}w|N7Mb{M-NioJq5$ z&6_xL>fFh*r_Y~2g9;r=w5ZXeNRujE%CxD|r%(!vDk;FZ^8nAFKZ_1cdbH`%r%|U?O`3v$*RTP|j!nC^?c2C>>)y?~x9{Bz z1Sr8oaBt@0$d?wtB&@ls=g^}|pH97c^ahrwWp8G^yLal(UXlM1FbR_R^PAA8SI@q^ z`#7iEoI0@nz5V<69pK+z|9!FegM9LZ1IZO=AXCd}r^|N?zS1Ch4mK!Zgxw(!$b}gC z5uk<~cIe@UAch#gh9Q<{;)xxONFs_Xwg|w1Fvd941I{f7%Y+o(nB$H*J~x0TJ(d^b zkUSP?q>VrF*W!}&$@fQpOGfEolTbcsWtCD^`647{{U3FwsoyqPH@n1(9qrgQ$O$El=_`YEb)5_;&Xu$KP>X{5B)YU`~@PAX|Ajk1}l zr-E8)>aV~O%jdAHu4-(ubwr|}tFcBa?MAcSsw=KUUTZC{K(47C8=!*A?6Rr~8!oxU zmOCo4=(Y>(wD88;P_^6ETQ9xZ+Usq)cJd`+_9e!r>yeC8B@%1%ld*WGP)$2?D3{Gmuz#Z zD);R0%P?nLA3!iV-376q@@BYqSCw!upYVz&Nhn_pnz-ONszGT1v!U-IF zkU|L`2ysa9M|SU;045}He76qH9!D50$OZurZ?w|6?s@P#!~wE-f$~;lIsT2R3M$Az z0IP6{*fGx!q|=@R<$^CNAb>PFcps&#lLFdMzQeHcN=Z8->HWe?{-K zazFuS0wg0j2vef+nOgx`7&tS9Y+=!wfq_mKj93C!auQQaP~ZaG$3}6{j1oJfojPyH zfmRB$Y2S2195}$o61Z>#eMq4*p~?SGs{oA!1NeaIYBjVFKJI+YP(ud?K!q#TGI=Ag zUo6YXOckQ>-i3|D4yv*)UL_{S0YXUN)WrG*p8~+J!?yI0L{sSK(d}gVngx$d~sh|x2AgJ*S+*9Y8m6y>o1vIT?UMCTpJ`&PK|IwJ< z;u=HhVbV!$OlO|HK+ZTc(Fm0~r{@IF0u0^KS#`|aKIzKYqP*-gNMLE}5?QOHnlBt8 zOawO5ltp0y^FPykD@EOv**gF9;E|vBp8=RK)X55TK{2I3T;uFIQXH~1+8TMuE*^N^F%YqK~YHmukyvX*~xWUD6my72`sjFUEI zT5zrvQx5Cr-kG8;ySe`-T_zeWm)v23YLb^_2AlzthXyS7Ysha7v`ONu*Dnh#%OW)p)G^WYwNzbGjSAVt zu7=v06|GJ{qgpY>NeioshSI^}j?KlIHLxvu=bPF3)`$5-WX!VbiZ$C&p)NJChfOSD z1J2UO&UVBek_?S3tCHa)(4jzyR!lX);Rp(41jpVCr!&3=y$i4FGT=e;ci8;qIM2Dxcg}O1GoTzb?|IOLPV{N$ zqu=|0M=*X~^rk!g=tjSQyGO2Hl*bRiB_}tdTkf@&Q#su>TX)QZ!gO#v{p><7yV1Qt zkG9JL?M`1iKO#U5xVzo%Z!bpy_ziG0iQDScDYvM-{$j8XOzdO#NjvB+cY8Sg@pq3r zrr%G@4olI2mSAZPkhLq-XW~tt1JGVb-~j*m$Kh_*J%Okf~yPhXCaJAw7-3G z(3>j5zlZ<$$4~yxm*4!34}bB|kB{@K|NQWezx<8=eS{d&hza=3YWB~6kN5xotpPv< zW&jgbfKo(&2WWr_NL~myfDV{|54eC5IDrvJffaax1UMZ2Rus=r8Ov6G6wz%UsC#wC ze(Ps?qc?vlxPtM}b`oHMC#ZtQM|H|qgQj4ED1T+Ag)lgOGsqChr)2vAIXrlSbz>o~ zS1$Uo2Bc<$n9_!@XMKDY2L>Q=QMeQhR~jGJhki%^fEb8^IEaK;h=%BY`d5HTz=TeL zc`5%GgY5TykQj-R7=vN>ZQ)U3XV?qafrGAd4nMetXCQ=ya{y_!hS=v546q8;(2B13 zim(`qvN(&EunLfZ3b>eyy10wH*o(a=H@~JN@iawR;fPr1g_ih;l_-45xQts!f5=C2 zn|O;m$b+GXilVqG&ZlQc2!6^i57iKk+AxmNAdcmDj_8<<>c|c3hz#zycr%v?zBrHc zIFAHSV$ugE-RMtKh>XeDjLw*O?;wc*IeC|;5MzinRYiuKriN>nDGo`8#aEh9=j~5Aa1?hX8_lyD=knsON zkN~-p>BorwCUDNCdP#PTD>rQqxo1TwDli!pC25Wz`IJxzm0^;Sqp*@zd6ih1l_$9z zDkqVfu#{6_jK?^Wo~M&zsd?gHlWIwcWNCs2nGgwSXPfA0k49T?7&4?NUs%zU=@6BE z*_VG=lA@rMRY{nJd6-;zlwTQ5GTD~Q$djB$3~0HQDQK3IX^DIXl>6n0bY@~niIiMt zaEh5Wd#RBf`Io2(l2cijtl65N@S2C|npa7b+{l{v*)0|*4npqZ;BN?F7S)GD; zpe%}@%ZYFeT6w|X1RWp&5x@hI>7Z^op~4A&P*9^CAOn|Rb`NTq!5{+_V52)Qq(z#U z^@&m(N|$v>X8Aa!qeG&}5}u6+peQ<^1o{h9iJ-BWm*ubO2 zXO_gkrk=3^{&1mjx|5d>0enhvY$u^R8jYG6a2|SSSSm?*#+)G9Jf;76qF(x_kQ$XM zDyCtYm1O#vS}KK>mj!@`qfQWUTY#ZGYN2N74}2N~a~g4|3aC2@kb4-auL`HkD5!&K zsL3fgRhpWje4408kk_(qMVQnnxK-#3Y)TUsY!XJTqdge0Hie+1gk2m zG%0y+dX_=J8AOVtj7Oen>UlfB03qOg@KB>caDp1D5sSlNUUsXn@S<}#qKoP|_ez@o z>8R>3ti-CUsZgnux|Kq*pPBj;;diR#kO@qxo8^gf^Vh8;fUbMkuv%!AoLT@wnhAUg z1x-q;))ECokYDjyD`ar8aPU$jD`)X_1V+$ePxfd$l>|p{1zrEiOuckdFBz=s0F_>_ z1sMsBkt(Uh%ALs|24cVogdhn^JFo#uus1ttI!kHFaUIYjN6ZK9U zMPL->oHkPh3WNf^T1q^74gKnod$6@w(6h%dwAt{Ks|gIu>IEB@wa41DPHPIz`YX2u zVFnkY>uIVJ2d*8En_$b9ytf6ay0Cp3s^ZF%o;z_Jzy)}kg>f4((u7rX^rUi`x1h1N zeY=@~GDkD^Rc~mQqEZGZ@EYU;N6_P|)8j>zahMewv^`&J5azpu&%D_8DvU`(G zAOi=Cqh58R4-BdwOM|v@OAK|lcuOl%wE$h<1tDBbJ;<`@vbYyQ1Ftc-i3K9QRJ@m! ztNf`>fcpZCYe(Q44UiiRTR^-F@C4*rzScRZ^4JNfPy~Qz1+y>%fS9xb`?T{rDq(QC zpA^08qPmZ_rv)Gb{dS~V{I=^Cei^F(53mCIunl4>1E2cCM0&BKm!y5Vrbi?T{l>l> z5W8v_yg!Le_s1E-QD-dELBGYpp2;dcB?Zbuw>tkM!*(KG*y{={BwUQVXf!l{0(4`J zyRX8^x*ia5P7uU9T)vxItZSOkBC5o|r(C=U`e4UsTYO>+ z#|$6?e|mEjE3Jos~8+9W&m2$M993BSKMR*6VL))0F*>&35pv~ z_=sbwvbbQd1a~#blnjvpPy?L|1Hq`5njE=0oDQG*16DA;{FkXs6SPz`)w zqd*MGp-i*@%g>;@m0QfTqbmxm94g-R0`d4x~;#-n>5-1BrxFVsnJ-d#}ZQ{3ng8g zK>^gA()I!e0o~HR)>VHEAkbZH{_{x109`Q4*y+LqMl}Ke#Xf>NI8zF(J0YE3dfAz6 z-k8hTE9uW*s@|(?H^q`&H-tY$NZ(tR&`L3`wvC<*j*RX(nM0S$pZDZV4w(>+FZf3q zE8D@1(o-lMTc>4)%snS#U<5YMV!s3dAdceSbh^+}R7w@KB3>*iu7KVZ<3MeldtlT! zp3lCxaT;I_H@M+H9=(h7J&gZEJ4VhVtL>Wx{;(1)qkh|D7q!wpKwcC-fFEvVfC9qS5{FZ^vI)r)0-41S8xM1u<7P?12*fcdESmT-sk_i z)TzM43Xrr&OaMw8)ubIyMM$-aUTuvY=@1H;mF|U94!f8Mve~g6KyC@KwFG2UvL`dhL=PiM>5HpoMjLt=FTGNt|gdp_tk?L;;IEX<=7kFFYIZj=0+k zOtKC^_pS|lZUI2+uh{>Y3&M`RT`h>oPVn7sEC*jO36B&?PU$*Ie+wV+?pO42I`Kym z!gOG~$7?H9Km>NX+=WbOI@SXxuDw<)Fi{T#EKIm9q6JL=WQr!^Izi(gxdm7-)G~j* z{r&?pPy|jq+RI)zQJ?iA6YY-KlSps$c^C9T&y!bPtF2crRb=s1&#SJk_#sZEUTHWj zpVT5btU9jjV_Mpns`n00p>MtUoCoo{sq{|oYiJ@`qc5c*&+dKmt4PE8c|YDlO!@j8 zzlE-tKYtXU|Brot;JZ(g+CGhiTKHvpGJax<#8RQ zWq7OL&*tP05Ss${F@&d}9)k!0Eo7Kb;lqXpAu6gvaF8DW{WNam*wN!hkRe5mBwG_O9v+XgSm}T&zB`{wg@`3=*$o^US-|db;py^ zs?Bt{t~UQ|+}E->J885iQQtp-|0asmbZOKSlQR_X+WGUVZsAT}E1kOB>+$f()0$O# zcJN^7+SOa0@nXm64Zl*4zWsZ_0FW!MT;Klk<)lyJ-``pPasK*?grL=+Gw?u;V#{s8 zU>LlMxZ!w-aKfOB6A-}8Qj-cn4mtELJP@q|5j+v&8gZ_=N;(Wa7S&s^MHt7g4%rHtR1MD%+Jon`DMk}!tG^BPUNU0l` zSW^G=Hbxh9G&d!e1f@wzpDe?&9r1&vkLJEfaZpf0I&(`d)im|gQAt&GRaRRqbxf~5 zboEqPO>D3NbjUl8&RE)Wlh+pQ%ydUThuyE(KTiuZR8XnW$WV4#YD@eDk$;UwYRqfM0+E&NqaA zqX_5Vk$wgCJvkSC6S5jHOx6%&$;GTq_<2T65lqO*CQE`DYX1xf$qMgK5hgUCm=}X{H@sT4AS~CJf?DGJgM= zGmNj6IJK;|_F5@(YaTLWX{kN(WJy!jDVbX=bopzoi*ci4v6;=P%rtY30B556UdBwm z{RVnG6?IYiaH$hlyjR2^Ui@*$5jT(Ox8v3tw5~1JTG;tC*PQOUu_W7UBp*~eWw)#D z+;h+Z0X=j8oEobL+F0_;uJh{NZ1KEf9ZveaVlYeDeuVKz;VxcmI9(<8MCz`VXidfcx{;e}4n+ z&%k~q)I%7b>Yk8zhPT@d4uRY7SpsX7I|qKncUUppu8w!S;$2X3iCZ4?F31110daF;Svx6e6h%imr z&2M6g64(Q0IlXy8Z<=#!#4KiX_^8XX*+dk+r02G75=?HYaAOx?XFFNLOfk_VirncN zb_gmKfOct;39Dv9H+e}62qJmbv|!YFVE`2RP@gS}${8RK&W!@;m2U{C)-Dx*Dh|<~ z?!+ZL7i79$HVL041=~syw8%)tYM?ny=!q5zJZi+Dq7NnBA$D;Gi^6A{9#kPYnF=7M zK9v@#vFUS0=h9rZ1a0+1*-UAwoPJi-E{@D8S95$2|rU#u; z6;LbjWVMUc)vo7Isbbq?ja`yVms=%lga{z65B@2!rUgl4)tc7q*cME1nQU#Jam~zX zR<6NC&t@lQ$~Wu3R~8_E^IUWklA{;9yo0*i(Aa#=@4M8 zzRjjZ&x_uCZWvJeJn?7WODq**#l9gf8BMCY2>%kI8A$)W2~X@vPs8@uKS3TVf~Bb7 z28&mMGdAxtC(N|;ewjTL=4gg7Y+=s+SVtW;@Kr~43~K@M$WLZllkL32DBCC$XV4Rs z3kPG|0otHy!ZL5UOvD>U7{YNqGn#?KV@HEh%~DD+cin8HXnDW_Td;*K=Aj$2u@=c< zm;w8W@LvG?61(O!0s_cvno76%po11Pd994hBo-QX4yJ5dALi>iX)n@ihV-yGEZWZc zSyUj_FQWL61MxXwHgG0gFPS`)TadtgKj;K7@N8m7mw^IVJjF;48*A3A0t{i`!@9}P zZg{g>4PL+_udP~+Tj#pg)ya3R4^6;C+t{IEeoX(4b^NtSKN{Ja3)KMg=&?&b8+_6p za;WE=6C$YZ2V&4R4l|MJY^|E4en^4?wg@Fhqx&kQR^z|Ga3vS>l!>&yvX-};G++lS z=f7U~)fydN)FhmFY<`UwTHpdW)Z1>%eu)#z_k=iC`{LRxwHGK}mS`rrYohRg0;qvzZuW--NU#OaV`18I z201VZKmsKbLVYtxa>xtciqRaP1}z}>?I-`z`PPsf^lFU5*=_eoT?g9j1$N=ImNJ>CYa0S!fjaw_6Zp8T12@mG0r-1? z-b*2svkDr}wh8b65!?U`AVCuhLEag_e#@KW962$IuyB3_aLEA>OfWf$vrWE)WXz(&mtSmAV z03t93M_@-_*(@r&g2t1X05HG!!w)|h1N>0}Px?9#xW{9Wm>v*-ajVBB+z=r#y6L+` z@G*e^3YAOaB?$;d)e^DR%ftU)NJeZcu{_9(7_grsyb~oPMGK?Fst|;D8^d>#w|Lvd zn8k;1FvoLLkCG!uDqw}3!>~_?LJpw3 ztkjS!M1d~4zsn4|8UX)Tyd1tVdO81>gpQg-sI@Dr;+!G@GzrCQ1ZKFV-HSdB@Fbs%oG^$^b+o>B zd>*F=qAW_q()5gc2j2yb_c%1i~wl0K{C*4r$F&Fw2T` zkOq8$wp0QQ_!rz14ju|T0pZO^>%gjT0lnk{#M1HBfgz7zdIEJ;TfIGBZ0hI7$Rcu~Oq z6s*=m2SQ-E^&x;$D^LU77Ct-(Bg7sCr3xi50TyrpE4|Vbz)~(Pff?Y-{5&F=Jc|n5 zGQq^TP8`CvY)=0e`ANq>2OI2yH03!T+Cmnc$IfIf4FLlBk*H!|0uohE$tVLrO(X8h z&e)oa(-a9|;DZm~zc6qCPx?{miU}Uzfjy{}`vNtYpu;-TmLkx$8L)zwdQyTo0%-i4 zMHH0bTvbphQ?nXV<2=&~l^QYlqQ?xXFM7o4EDXH+!|9YL_8b5?MTAoPnf~K9GVy)v-NkKQuCe%5$zkfy|8v%T^6j*et+dES&~i#?}HR1U0tx8?fV; zQpEyHj3xLYx4cSnG}h~D9?cYh#Dm6EMFV!#3=J94 z>C^%aupj@+eAcDVf-k5(Gnl><2sH)`B(VVNNGU@|Y-CP*jn$W>$(ThG zaNG!mu|X_?0e}5h#OPR8!=f?Vlw%FQw-Gn@J6aSFr~CMz@jP1m@qvRZmd0Z|tOQv~ zYgyPBOCnnoQ7gvU>#DfSEub=7kj*_RwwBI7TVr89fVvmRSi zv4T~#o7tmLTb!&YKi%2IBsfiKv_6!@g^=1(5?mMUIc165(Obi<<=S(-wyy=-r_vZ7 zA>04Xusv=(E z4P9*eOS%F+TVc|+r3>+)!qrwT(KQn`gK_e1zodc zJB1?O)RBl$EV$=)Z!+$PmSmXR_8kl^~7+3*El z14iEv6~ZrbV8?69=!M@G7LDm`C;QDR{51zc;9&hNjy;QElJZ`g0$twqYYdKqlQ?RLkR3lf6WMPh$&Dx#?nm65=8j-3LSB@H=D0SYTvZ z;|q)5wxt)UR63sHTV^^6ej%^zWna$S2=wDTEzLNt-alAx#JGzT_0xI|K;XxUTB7HXor4ih(>3~b-w>%c+ONt zR#XN*PQ2*Fq-K3qxPyjRCSFh}w&fd6SGe5P%@tvUK|{X`nw^U2z>yU)nKhYy>9cs1 znie2j!D*kqX`o()o+f1_wnPF z?cZx;X;AjA1A1q74xGO+2NO=iwL0K=Mq@>0Jh~d@SeELLzP$63%8;I!be6SB#wtuk zX;#?cKQ1tq{stZoxp5$oX{c!eUhMyO4r|=bt-)Ny zf;3d)K5pbrZsjKD85V$zHaQ!ZtdRg;~qDD{A82fS`E(RuBK{) z{$$?{?wQmn-tO)7esB05C<%yfnNw@#>`BRT=z$txnh229nucXR?-}Ct5ylaA~4e;LXOd{1)&yDR2*8FZ~X2125^UUT}+i?forL z*4xi??Ch!&Ba7L@=GCpwnRd7z#REs0%Qe|=Tc5zyE?6h_7CI8D4 z2Jwf&@f_#z4yE#<8gcdVajXS$zix1?>x3Mxz^6X)2v_p673KdKr!gqMB>wib0MBwA zS2Qbc3@vY2E{{V7Byv^Y2NGlCe0JE9EOQbb;3YH;oz_dxH^aCAp zuW?q~3v>jhFA4ukXC7Td2eV*roK*ksH*fSau5)a}Y8U7LA!WS@`2iVFfEj>-PZjgn zssg`FE|MSDL&w{`_ZC-I?5cXjV^Hf{GFUv)OT1gDdLEx@l6UkNOp z&GxZ&bS>!ZwRO|zI=}rWKEw1(-*rn%Z}p}%8E<%qk9hy_*lUm}_l&-AJcV}y*LZb5 z^^TYKM@bi8>}%@kW!ro}D;Q7(2y<+AG;MdkG)8jPW_XEbzMQFfi3e_+|3-@U_|sKn zc9-U3C$6Elcx4wvIBtn8AQ*o+OB5$YudTY3&xMnRIz2Z=2DSmP<7BZ@c@3;LxgM~=7(z@F2p33K24(=Wrp!h!Pu0tZ307 z19bg1a_m^|+{OaBT#<^^$fPG&yI8p*14fV~lDJCJtZDP+tDGYT=)?)_kR+F;hN<*v z)Tn@wGLOnU*ApdEoFZ`^GDZ}a%R4huhAP_iYt))NxrVK}6)f7Y1w68C>-H_&xN_%$ z?AK5lx4e3_A*-g0j|B-^q7hDK`0(Dui5vgUQiV!WV@{|D5MYrDQ?de;BRlR4*)!a0{U8`#135ZACu3XnY}ofdY>+zju&g z^XT}SPs`rKj+07?1Op0iAfmMvS@Pw_(`?|qyXFsOEOs&Y4JDOO?0St-+Bqe!JkjP2{$2y71nbXUxi(xPB$HT5!hgg1%sG~drdXU zEuTH60eg{AHbZDA#u%DNr0IsnYODqDnj5m^7)T+8+$dyfyP*bvg-I&u8+6h|*QArp zK^diS*3E@rfv#`|z3b0V#)+I?R};DbAtNEVA&(JTuK!%0=!oVN6f}yFLqytxWLRl!Rtt zjFg^!MC-&aeP;R~2$*)h>(e1fsMkd-=PZ#l!SX>J6ImPxU>VtA@5?gVS2Fsr1(C9| zj>C?g81XG)%&;c`KY+?{s2-c_YHohmz*>+Wo1Ajnjk`r0&cD+9a^%z{J9*{LsT>#2 zEUom+CtcJwpu1O>PR4plv}B(U;HfbCht2d*qNqh`!WaBE)#-y*B|G_hgqt75=%xVTnMFbn=vc>O@KQy1j8P5wD65d&5Qn^xB^&b*M1}xsnHU))Gzp2s z$LY@!G>}soIaxX%Xa+OWGNA9YxxlXtAe-WBSH3tnGz6v+gGZU-6zJqdSjGo=$UGP= zZ8@S9R_brMl;J-&Hp5^FvyXNt=E+cY&&DD0kjiW(5hK^oV&M-iuydq2p~w`Y)#L-3 z5e7&xNxVq9Vr(;En(bcsx{dzKgOm6i0FS~%QwUI{?TqC@4VF)RPLBbs-E~u3-=E;| zZrt4o?wZD(;K3nCaCf($fdqGVr*U_8cM0wgAh<(t>*f2KnVp^5+1mYMpTezM_ntcE zyxyO>5-O3i)NpWBsK^nAr6@K(LePGq*{5&&uX&ylCB~-d{aWNaOo;Frp$7r^9Wp&V z;P$k59JeTp>F?6Vh|&8p9O4k)QYj~QoETx}>}s1b#ot4!KU9=TBwkOHNyL>4DU%$7 zg{;ZW!j^P}xbm=)7){G@R>ZZFSTJy?8an>Wx~!UppFQ?MH4tgIvw~c$eOAv#80at5 zj2a9C!+o@^mOu<{wX&;6`V2H!J#Y=pUAWBDH`1B|Sk2P=83SZ?-*v{nB#~klTDruk z&rsXDD8r^|Q?D>4RCo=`>AxO(Y^C)&#jzXa>ai^8BCpY<_cjzSNZKXoP^_PGG*ib@ zIs7DCMnrONL13|T-V-p`SGjHNWd|iCHja3E_UcMJq@d3dtE#8~4LJ$Kcv+-%?$XCm zE>LlTC{DLf5>gTunV}$I3-nuSPkT{!X1VmfPdcx6$!ZW5Ta_5%EnMW{+9?S~e>%A@ zv_2Zz(B((a3*`chZc|0_U0d-Et}#at(66yCpxFWa;>%h-e)HSy!xdv%uhiHf94Y4GbiEd9&X26x4udYReT5>|pw zkZ*ifY)4AlPKT4;oHF-x53p1R8F|>?;`y>HoQiPb^OH}qmynN0Ph6)xJR2j7SZr87 zQVoT5-R9~X(o%g)pW$V#YMAJ-o|54Ywa;!S{4JXm%>5Z|uUo#R5`cRy*Ux<6`7vp8Gy0-wMM z-Jw$CF7yU>>acJ6{5l!O-N#3)pX2ufWHJT%ICD(#=L6Zv&BZ}A-60NERpUV&zlEt z_q}9qSAQql7V|vo>6p)M1C}n9Ss$KNqb^S8y5HoU3J`*(h2K}ah-W%dLV}`2g*knL zIWIC=H@7g?S&LA~Nv}eNx1wMeV?xLz$x4~{ zYeo*g7=}~@hH4{65feuq0?VXrB2AGY?Ss+Iw}@x}P(z0G6uziX6luQXm_<<-*2+*_ z?=>IDHsJDKIi4oe3kltkEJQr>T)g<-cy1K<`nmYQxdhSM`0+-B=|;d3#m@*EI8C|4 zT}C}z+r10q$87y5DjAsO8=B0y8sssc9tv`}JaX1Cbkc42jcug7Xo@m?=QJWRS(Ogd z?Oui*@~?R`iWHR5L?n6oSi+QFO!H_NDRg^((;`l&MHnzNj#@#JMcRn+ZSdF8^0*aZ)7- zd+N_<;QB8GV%$_xl+*%l?-T!ag{o4hXY)2zU^t5wWK|?(%L>G^3lvs&<@s}j*mvHP z{prPY8lsFFYjcVlck9=wRoqMyJbSUo{jfL-n=$sJq872{0gMZjH(G=}%6=5HTKIxA zYEp%2Qwi|)dzz$@GPH%-Dkceq@OW*X#h*gbOsW40%_MiI!`rLj)y*k)Ve#F{$jkoL zM%x!KUh~gH#KNKJV++!mEtFH#3NTE;ehR^ZGsjt3(A6*eB8+O}W^VK&mM#tgq?q<2 z=-D-%O0e#vTriq&jRJ>Rplip6%!n;Wc78I{rEIH?Xi z_DJjys>k9Q3>C^MU~0RPJ18D-icKfCnAx*`$YP6v7}6Nbt`K<2y8JMGW}p~KMTE0W zej*H=Z&1@vq4Peb{FNzg!X4^vMMvcn0@nj{Ax(3ZnuK|X@|9RIPYyIP&$Zn?I0GjV zSj{TInQIRf2Z~X&0wfU)MUWg7zET{RW=C5Y#R&3QD1b3()gsLn&HPeD5iOJOTWB;G z4?Wk%={x{Xmgau(TY>xZ=!|hH;9{%QB87rs6kjy7Z*u`q3<}Cv8mo|8C}yBzG96fX z_6}f_gIWTz?Qv~1W|1n1b~}QZC0vLN%$}a|fTGDnk#);9S5V8v>pIMN{L^Sw8oDoG z%`T0DP`Ke#nsRRS>Kd)?9&~Tm7udnlRbar6S@a| z#ZaVTX*D?|O+LvFOQ)%i6{QHkWVS9zvy$=Nqv109p?R#uPT?~$eg}#xouW_=1ma5c zV(c;|C;MUvBFJ&)W}7qCqGh|fY52a-$3IELVOi)_4kuO>(|h53SH?hUG1q*N_xG}p z!7Xo2HcYUS`60-V=${nlPn3U6%_+u231;%lJ61`>YR^72QG}$ed|(9oly|Kf1X+j3 zh0;PtFWUYHtK+sVcI~w#E$yX6=nF5(p*W`A@Cy-I^E)jcG$)J69`InP;G;bl>ay-+ zrIyiW9<9<>)Gy-c1E;r9QmNPuCNoc@Z;a;AkI#N?^FM_&G<{tc|{oo%K{wHH0$7kISpv`WQ=vY<}1rGqtas8AIWg$ zC3d@CK6cAhcH}ZF+LXJ8p0*271#3hf);8@_qwLlfD;3M~rt?@Bb}F}`^VdlYXBI0~ zuFrm>vi=E=-8??qY1K((*Dox)7gl2)j<=4WxvmQCAdmfF_vI{WA6Obtj3v5ENYv)JG6BPYv`Z5X|Qq z7$p#_Mh&b12+pDg&H)7PUIYIVgb-4L5CcL?sX@#EAr;pkRfCY5YmmD^D8n@mwUBPc8N*Hk{A3I7)1|8nw6v zY%Y+_~oE~w3Pg#IO{gRR{??On{AtW}-9N4yJa&)_hx zptN^E?GusrBR($J!5}y+EG#J{EV(#5sXQ_|D$3m{A5Q`e ztlGMN_;iL(aZ^i4d_-yEcxlr}`8%I3iL2_Et7{*sZ)|GpuWuUu(=;~KGBnaUw9@fk z1odgx>{a*7UhmXZ-{fxJ)X6}5+Q7u}za#3=(V>Oq|8Ug*j;Kd!ivNYE_eS3l_59g? zI_m!^sMr3*r?+k<-yQY#>pzJ4T~O~Hy$kA@osEAG^*f(lznb#`}=o6{m!Q!kN5vmP@fGx|2KU47IHoDzwznQ{|ld%&6KXq8;l^}u>SwXr?q4L z3!g6idIHaEw-mF#*_I{Od_L94bd|k^tiU?2_X9ct#TjdU{dPJe01KCB(Vl6^tl!K4|cZSUu^Y9K<>{Cp9|!shsyu4(jPU= z5o&Uja=th4rR*;8bQ=i4z8&5+8*pCqeim^>piP~3-O3yu3xE=%-wwpN?lbhmH80r? zCb-^-{P7`}ekX87mv<+O(*Hr$i@xb_D;%DY{!iqm(w3bl%she~y6TQps_5IZ2ddat z#Fgz>3HwkHN*V5>-2@^ykw1xQo2GdYnr;@R$+~sUW+^Q8h(!_H(VoUY^`g=uIfHbo zu?)u{M57G*F)KP|suEmhPfRdNyV$IT9(9wrsowEUx${(mrFezmDeajB+5$Ttd|&zns>^qtA1P`TUPYI zl}}eR93P)l=~HUi77a#wRpj(ZF|qIuaJL`TcN*(3)cs<&sVtuhMz(4EJ%wo2I%F)z z+BWJuZV%oWIyq|F+fS1#y*QTn$QsRga@P6=weHXrJZ4jq{;Vy>NC{0h=h(CQP4FjZ-(D za`^A=chpw~BKZstV^n_zPnj2af2NgZ(&dbFhadSHlvu}!izV-rO`G!oF zf*xX6MNn&V$xmyiHFUNNMoU!K0uoU*S3&Qf?t+JSGd!CALN)71NMqtn77gM>lFg+ zq%|wwd+tk;#~b3(KRY})W3xBWu9wP<(8wKs$k@WjvpH9?&Gr4RsgG#ab1?rIP6lV2 z1b{mHQ>-!8U*IKonZbeXJRNcb% z-8>~HhHyps^D;*i^eKxY$VMsKkSiiWw51tjWVWGq*GMkj@0wx!*s_XV!^WcPf$9*w z`uQ}Li`P$C%ILr4%&@M?IY8d}ws2%%Ht-H>P1wosoRnQs70|{OIg#{c-1C`y@y&ya zS)SsUBFeH+e|c01aH0dFO!Dfb2!a?qW>Dy<39%bNune4+5nnnIXrPt-DH3};>7VoI zYwJoVQZXWekY->^6QsG_90HMRHb<#f=vUUg$dJ!4u>`J64J@WAcSvKj0f>d@=k6<%?Yxv4v@Z zXB25QGckM_CV91v(JDB#ZsQ3dowi;VIex3EC_=nlP#~b*Pwfx1?wG$|7v-`r%)-H4 zh%)DP$_f3?>C4&Dp}MQJfuY~}SfMe3n$$lkUY)3qGBo}w3h$dCip3>jT7i0k5k6OjWtP`iUm~-o*1k3~ z<;G>1^%W>vqGAkyrg5|uI+Gb-^7VyAOU>8JAtLASh=*o)=+(8eUg&x<4LPSip`^kS z^XgL5L3YV9_8H6af4r^b+g`y!6c8IiYBDy|g=4xqRo?)@GgjJKQLgkuinzhWB(`SiO<%jF?wVOj?JS(W>nz|k*He7Uv3vE@8HT!V zrN=yX3^{2!(!OsSnPhR()7<#O-PEiRP-SR7KvXa7-d-tL?KQKuaod)Hb0%c(EZ)v> z6|EtfHZ7~$vmN_aF{ZKl(n0IDob*}pLKi~UM+1~R6j(Beo=-G2fsl2JlGd;f+@co?ABhYzQ)T}M=E>Px+?3e-fYMpc4Enz9m!cAYTUzj+uI zNcKhWN7Z#cRmy z_B#~k(Iz$2zY)fHJ(Sg1o!KL(&myBUL2^(3^?UMf@%Fr_G8t3XBy5A)A+yzY-PMKa zVAl#%J%IDX(_)MHO)^8)iN5&LQiDE7TzTUFuYI}mFH6(owCOQ2L}R5IyUAwk1k+kW zhpF-IT4<=&&VKx9Z40fr<&478JMww`P>QE|Zt~ndM`!MprMQ0S`@wh7*yZ7=s)SsH z0n@+jV+2$aGW;FZhRx4g=ZY=mOO=j+R2}PR4lM(Ns1Cu?)qYSTEkit0SJB^J{*a@$ zj(k?Uj?c*4&4cvws%G?GCtob@{Yq2##w&=`8A>one;3Q)ySNwj{BnS8!8;v-_AT`D z%i(&}Lyetz=sTZ2QjTexFGh2vP~SmuRSqOAnYyh~(gVaFmMje`-ib?}T24;!V5dzw zj6><_k4s0bI|(P+e$P0RhBU!kN8dXkXI+SEJXtiE-t`f`U4|-k>@XNTKw5BT#Aj z(;oUHU;5Ct_%nO>qagfXk@jcj4G00a{cUov|3(6nYBU7%bMZ1@4G<{Ciz=ZRDCH3- zZ5ep1Y;*Y;8Tb&`2nhOe_>(`?m+ZlxF*PWunH|H@hFID!?k;Gn$j?~Q8?lxeK{~*i zAi(C(KQBn#K95~u0}H_*nu3|a>)^C6C?1QPj4uP2782triO;`nrDDf zR3Ms2F}y+XykY8=A==Bqx`!d%&0#UL-d8aSr5=oO923u^G!yDe^ z?ngi$Wk?@vc<85L8CH52UNaK4`4HtwA911LFrpGJ%Vs`Y6LFe~cD@|_t0#1TH)hZy z@=iLY#4&P|K05Uv@{5uC%(8nyXw<1xFi<9HTO^#?+;w6E9_=v}!zyH(APz?>?u9oF zZ!`waDj@OJ)5SQV1tEs=G5YE-A|W*9J~f`vD&~1PmXI*u&5{Ij*?z<~7Rf3q17aCV zY#EHu8%wkjwQddCnl&HWE z&rgt~%9r{`5cDLP!a|Usl$*3&lw7&^-EcH%b|f{~E!p%aReCs~s3p>!(BGLa<^Aqu zC}D_anDWp4kdud$A1l$<^r?YbF-%9vS%h)no@PRHHY2exBw!OEWt3@;Z-$7l?JGcb zU&P#!gw%I{+~Xx^n3*|}3{8>2DwCMQmqDqSmG7A)6{cMfl1@>YeoBx^2YF22)k+Pq z%EHmgP@woWO8A{J7B$5)L4U<@a2ooP6qK`X8aj{7Wf*#DO=crS8hUeVF+sX$Sk}s@ zH$pntJS4c`L9DpdTCO#HD=nh(D19$2Ufn8bP9{Uk(@N|8LhN)#8v{^BIqI)x6hpeB z0S53r;&RK7C$g2HSDIIwj9ToT$M=*8(-sBi6_>e^`)L)7G8SCu`Q>;vWmzOw&Kl)p z8m+shVB0GGk5=JV*;L{(*SHk;`N!O3?o2a07>@7+I6@eHZp(deCZ=-o`IciMgT3H1 zfU?G*9WkG)%wCAGfI&J1S-ZeT7GArpP^8TQonJ`yDXWyQP=&ux1EP%({S*<9R`_A8 zFflyoE5*0*73hl^#Hq)O0gbf%;8=56C^I*&ALQUO^H>U^R8}Nd@RPmgYOFjtjN!X^ z(8|*7fpr1EwQH9r@Ov)vm8)&Rt&GM|{DDyc!wNizQ;5lsnrd1&mnf{F&99;v)WA0N zp#8lph_KF88$;{_RBaQXjDg@xdP-x7GV+Rxv!-DbYJ74_v$2BmAXY-+i-1AGoa;1u zkshx}kmc(tfX=J9`mTtpjDMgtv*0+$o~WGAiU-9}IrK5jaM}q25aQ2YkanNO4FC|~ zLwyD4ct25VIPwq%xsmRf>if$3-HOdxhpw!`w+F(WZWU}mjw_W~>gwa<$1h6=ty8}) z`=@O|!;6G~8LH)a9rZdm1&;1M}4SA?hX-(i3pT@67cIc$LWR!o*JfO_oOtKIsxcZazQF2Sas?e!Lq0c1QmD;61UOt z4;fBL60oCV&AH&XcT-J2{pAW9Fh*38^i&=!BE$uO;WC5Ko_Nyc0|_z@ScNj>Hd~^w zFuKGbzpvi2e_O+B3qZ?r!9_pRi=5;W62eK8J8gwD9PwHfm2)!&ql_^?$E2k-TiKU0 z<}rcI%)_gJ$W1Wctx{!)&(eSsVB=&RVD+&(+L|}XIKWy*vH2;S!WZzNu+K#xI^v*8 zft($70d`iP#3w;w3M}Hi%kmqiWtyMyqYVlTzlst@_lRNrhHUrF7$U++iM||6#&~Ns zQ;8=rAdkxJgKUp1q5Y$U(34@N_+7Tja}xRlDkk%go_Se`c5I4UHCJLO<5#r1QR`M4 zD-#_IGh}$&j1Gp=?wW@*Ph@VK9;p7YPJRfpry9A0?nKopgXE$K{AURO2;9{`J<9JZ zvBohFoZh1N2!jWJQ3soFW|UkDbpPgWP5F)%o!Nkm#zX+h*4`o-m}= zDPrC}+M_&fh7@8x+6u*a3}rFa#!X~n7Fpz;`F@MJW9st<0YJy+h31BJC!2`zZ%hHp zh$8+YPgLa3`$dcD`$jOc7&EsYV6M(HZL_{y@Yi?E&D=jV>Xsyj8P* z`5rS7mPa_d8QGyvaeNj~0mE2{1KS{FBSaD9R3OPP9Qe8CNmnLPMG)~MHJl1uE)Yhx zd>*RGTO8bxelIw7hMZddQX$4Ct?V}rqlBGIuYM-jy%e0hXWHYl} zl;E^#V#(rXEH{hrm;(r($qG-Vv2_Et12zIDEkG!%y8a6d$Qa*Y@|hBMCqDh99uX;} zdoDmc=ujT37eTv7@a{y$?OVV!%)a?Tg=fR&ZL|MQUhR>G7>z`kf>svn76p{T}HZ& z=RVo_FV~jmn?SR9&%tWPDML8sstn8NghuRfMA>cyGYY<5;E;Dwaretf9xE75TdiRq z&p&)vt!N$l6TZ+zFb%~r?!4B?hS4@^<54H%=B~LWX;cGq)aJaANlcm8T(CHHkD2JI+63h4_N04)Lql!Fp}{++hjxryTM`2!(CI4i5_FekgWvVD=) z>X=lhZ<%+0rnSQsnd@iAvMs5Iw!nP}B zWtOT))^xgum5V{hVHA3nzY!a7NWy6Np7OI(?vH8B6rDby6lwxi*UQo+^;6XWF!kl# zQADNnl}n!ur@1>lh{zLrJrXG zpJHG-gCAHolgqxh}HF5!=3G>sU9eS!vo>YM(ImLLS&Ly&Bicx1ipVMR4kic*@jO4@dtSQS z&UE~gW-LGEWHJ&*)}7A0!fpyx&+4$h?#afaS%bt$qqlk^ZjL2hWxl~rJ&;yw)0U~X zb_by!^t&ZroWw3&o9hjtBWeA5=2T@K_mQWn+Qa4Wy(jg9SVP%ivjg#BD*E;Eu5Ys) zfAVbQ<#8x&rutsq=i&PUERAkN2~oyL#Qont4j2;&r)y9+m&sc8*C%aPFygW8+|l6| zq6j%A_8*SuVdVN6XiXLQP}BYl3x$lyISz>TP&r9pE1gz#j4iud;X)fEVsc_oG{iML zzLYwABK>|-oAnLEXWk`-025mTjFDid=U~Qk7;xEnIFn*@!U-WRXPjpzV`Un6*P87p z*z%xK1{-#Ck=6PE`jf;rsbWr90~i=jot*Xsk~X*ijIX z6iA{xyNG+i?4{2ip9zXj{(T8o6+XD>xf6?v&Mq-2nLgM~fV9Rszcat%B3T;DoPU?|izb_9*mYfSVbkkpRS8Cn{&= z0msSyiC`V@IXmJoeukWSPPdfwn|_*&{r4G%b;ZAOt-XCP2c_7nX@+${G))fOG^6;J zKcnnntxB*pyi?5a-8+OD`4&XKA~QC>I8PW%>iPkBFJ4@x#qoW<&0>Q{0Q2b5dcF%f z?ys(ksFr$&x>N?l7AYsmRV?eZ-h>KGt`4jYda!gWZMAV4q?Wme@GYJPw_God#pOTO zy$UJ1emozCHNSQuur7MNMAozBqyZ^W32pBXQ%zEPA4%ITIPgkdpUbLkbsnmmo6~;& zw%dUM2#$AtU=IZJDEc9r8lHZj7g!Lj@x%5ehh_2Tg`-stAf^_n6nud6qWQ-LP*UeY zi+S{6MT3IqO}C+J)lpwIMyz-9w;v~pxvG7le8{nNuJelP2&Cxcx-(WVqNIB`FlWT! zP-e(>@jH8jFvG{7p%r@^0e@J2g(yn^D82o7LB=uP<-`G;1H3~gNo=9qM57C~9^w^) zPSjGf+h_{2;tcyUrkgq@j)%bmr7#XZWhdgv5nDgea+^BvhUJiwj7kf=oyEl#n9^J+ z%V;%-#i#6=(g{7v$c_EdwG=dAs9fSzedp5yTd-u$%ZY@2S5aIOMkID&!wgVYN&RfQ z6wmM<4ZILzd5A?;w-N^#(8^Ut1+(c;!Glh(5t;hNKUDvKyy$`YOeljJ!9B6B#Mz~Y4dB;h$;j#5!l*M z-C&mtqzEGOiNi_!R`zTPb~=78IAWbT>=ZC{IxC%a`ll(59GDNKLXTP{@*@7=nM`ak zttTa}RE0uTGltQk;pxz*qTFcWgID3Fi`IF!D{2gxV2EkMuYqCrbC7LnE}TNk+&r#uW4e8~MOTdAITc*&p|CZ5BK z)W}6Jd5AWs{?&kGp=uzFP6%0Gq5lrufDyLB$P#HWOK=VX)_tW_%;(0^6`7aBB4M^R zFV%>duPY&2JJe9n;qG-?WY~Kj%BjQj9>AI{^9nFGIl0#C*>Nl92`V>(E?fyM!%Oz) zOy(5+F5XpDFWs!k`X%B?V~tr{U<-9|QVE7pecejg+Gr87eC zER0}%>ndKOPX)TlflAzHd+7S}Q0T{ytZ~GB?&Fa+$f%_iF%C{e*Frf0-qnu`BK5M0 zFga0#q!P*?bQWlWaY&%A!Lc!7w)dO!spZ1w-0UBSq%@U3lWm!00E+ ze6{_QO0hb32ZdrbqTOy}28*f*49W&D^7*Dsu#!4rHQHn<69*1WNVxj1c2e>~nwqbw zIEpJ2*eHGupbK4sE72Q`Bxq8iWe-0+bY-*YchFx}AjPmoK$}w3y-tiN3o*9IMOyfPMtZheP^u@0~&?zrpuBVdO`6>~uWQ z!XPW>hL25YhM~O(+=c;+(W@H?reet4(@Gk8@vF-VTZ+v524Yx6H=sl?jD;tXRV=3L?9Xzg6z8twbfUv zE>S_Cn*}5ud0qeh{-7JsU>ym(dBS?cSZkr96O`G!0xE>8b*}wXYli<^1pYN6@C3+_(4N?>=QOfB;*3!}E5 zV&i74qSOgsLlMb`Zh%Bc3mNe?KqvMe9Csx>(LMO(Pz8#jWODZP)u}LuI+1sxNkc*S zg&PE#n;Ae~pv8=lJ2;lbQ;`V_`JmMBIBP`grAoIY@9+`{^HODu-f2$S`!g2 z$M`HzA3^94ZD>GCevd11ISNW08}h(98@x7yKpZym-buBSE~S(jkMKahKOr39g!l(+ zJo(H3W*pxhXqd#-E~&qM$1arkjDy79q_h=D5~{(PyS`=~l2aJ|P(8kr8G~k{L&3|f zlh_p4;4J-tdI;BzwEgj7Kfw_w&ws7ZM9e|}7r;HURr%O6fWCYIrc4oT>0VH$N3 zr4K^!r4E!WsU0GDz=3y!vh$MGwnbq9kuZP(v1*^Nu*H94!x_aUdqX1{B?62CsZ4aG zP34DxEq*DQJ<9wt3%!1@S~5v;O&cej@Qyh(RLI(Kz(s5RSV0MArZG`Uh63tP*8o7H zw_mZE*v3PKk)!{79F(ejV-hiU*~@1owMNaC%%up}zz!5_dS9XvIHC5*G`Wvh<&$B7 zz+d9x5R`H`W{e(S2WrhtE93IuOk1_GGGG6E!ho)Pb$?5GVzFaj2>WA8(2WA-C6s7| ze0@Th^Lp$=L3yI;XX!k@gcm?B6|qbn6cIgeab-*}Ocq~EOw3Rr?1a{4m(NeFyGWE= zV_lFWHE@VHF}xlZCLTZ{lG3^!s!HE!+bb8X1~+>;Ui!he8)kwpY)U#~YBS*z2PbMV zkt`l@7y8raDQCr%ciA;e_FsvR8x#~;$6S)FUq5D2W(wCjFWp9s%Pm;7;M*?-;+!TN zfuQN-tRXb@5r~Z%j}o$=zLV?h{y9OR4y;%|;VdB5pkiZBAhR?SUZ?zF=**t-{88$R z-j`ppBN9Li$*SEMqY}~MH5QKvB}6sZ`ePQ_J=$-nP&71hEwde1tFjK)eFd7(7_t9w zYZz43KMm-arvU~Sigu!NVjw7m3_nH!mLy-*F2gYbdZM;6)!B#EV8cb{S>EeY&inx+ zDOa}7LA0#OS!9WUOwMDnENP_Qlmsrn9G$2M0F(Jb$>*rME0D{R>xz^ml~sIl8P$l| zw4rdzaMjPSu_EOBFEzAL+pa0(V{@~0@yC1p<9fH3YeCS$uA(c8Q%Xk0e(73u`xqZae>%s)blv9!n(-c z8cAd}un2L*SypH8=A`d*y54^?PuE*HgDQ3^iJ=ay%N)hSw5}Iw ztX?E*aNC$XXr}uo3GpEDH=}@oovr@voPqJq?>Clipd!i{&4<2nNmcc&;Jo#)x*B+M zRQ;^2L`O0K7^b~KgSS;(G;V?MpfBk5t?n5srpQL(7{9N4QYrTph?CJNN*Caxx9)v* zmgVZ8`~=C-`luIiUpklZqx2I_wm&>?3lJJ%2pc1eSMcW3!o4v-@a(sjKp}TQ+ci!o zZ1!tRT}BZs99FN|__ld-C~Tm`XudbYKnjx%nw>`3KQZYss)jq;S%vt)JEc*(WoIS) zUz=I>cgDF&g@lJ~5`TY^id1r|IOj5=C)vZPGIjOvEwVGxj@Wf|HqZzb66MhTjIc#` zPN=px!5mzRy1Dz~*aR7Ej|OEMiq`A~w3{Wii>q%oV742Y3O}Jj7niyJ1>>uk;qsUK z_x~WZ*;f~ zrW>GO;S0p9(}rR&iqB?K)gtdBf6eQ(HOqSBBv?$ zbFk@3y?$tyq1u#Xl%jQ^zIBoPv3jkvaI|Hq{h>~_wMm>+bltIPy`}2Zk=VXz8d39n@ikH`k>ld`KK%2HHla?xYV3|#)y-lTS8WO^$RKXEhdL5&A zwy#^Zl~*~(8_KN+{NQG-6`_ru*&nd*iKEPLpAMj@B<_q zIAxlFIhGWSFKcq%7N_5PF1*Z2P|;+6R&{*3X@7x6(}GY;M{=>nZw6i%{OthT9*yy8G8@QbK~yaDzSK;{Ufb3Eg?gi>;Z`E&`ZaS8YH z63LnZF~l7(1BEqo15Ig~guC5jU zx#RpI*{+kymgsl>@syJk@y?(klPRr36|HYV)%vKt(f!xuhkd7RHv1OD5WJ-e`rigP zN!JYB&TI_9PgBeI9X)CKS}JlDU(3$CU{7Ft4j^dvmjSR&L6uR8!LoEpPDlFl*q^VQ zitKmn*Ya4C&uN{)s}9}X^w1o3bsML-;ZtPHvw89D&Q`BaRSsEy{S7^L`DAgu@ap9B z^w*2{8@I43_oo}Efzx=;sT_tYv+TcbQm)B&W{Im;b^I1E0k@SyH;B(`BHg!S3ASPy zf2W^r0=W=>AULgWiHG1aElwSq{I)`?c9}RcV_M2G*0y|adxRw4cGMi1E`B3lx>fwS zs+nv?gYWuCi=-M;{Jdydowb-Cx5$&@=C|qsq%~(Bb~T@I3laOvvy)E4s)_!2*?~;v zeFU-WC3WjrKP4IS=a}$7x*DgV5LdELA_W@=CZ3CD^SylWeb1iTlA|9Qy{foI0~-kH z-`uuAcj5LBTC{o~g^vgg^Zl*s>I(h{pW$%7KKqzeJmA^gk%J!v{*8{? z=-bX&7yod(r`yRm-BAG`|Gm5!-pZyt#o4;`@-j#X8iG=DN$vOe(+fLrU&2Cf4Vt!{hil; zIZ?0(4|%E6xX=rqo^}C}w*cBXpCEd$_AEEPoi`)=r%uw8l212YsntGd2Co7a##ZN!jPnAfmE zt$Iu$y(bC!;pReJ+I%w&gP@zEiAwD{3NF=}IrmN9GyhT-h=r+GVd+E(^vCCt3UrI5 z8q^@?!K2m$lZ_=ZYSh^tUDHgYvTJd-aI=tbyKO{Po{Y{ZK_lu!H?D=v9dZ=N#^C|J z+ThOXE7{-@fT)sL)zy`FqqN6PG-}F}h3wi6ron<%Gud2fA$FPam6k9FA|+V5qBgge zN63g6g#ZqCY*t%M^EvlBJP3#V=>yyDNQ}*F0^K9~{zM{&%lSl$s8F99^qKlVu;S)? znZ~mMsxKE|N6F*VqFg)Y`FfLqDilil*4kR1YjcyG<&E?fbqbT|)L4a!=B$vEJL?1J z?sQc{)=)}KdnR+SY4WeTtbZWgP7JHvudaPfrOcvPBIPZ{$|qRqbu78XPny%)Lz0-E zaXh3PT^f{AsOvD7R$bR*c#VuQJl(d<@QhowO_Q7+wk>i|gFjH&C3>VYyR^*0;=V82 zRs#lW2iF|Ca{b5&?6r(dB0*46RFRC_1R62Ic{C!CMccw4MR|r#ai9>Rlb20Jz;F=(wL9J>X~o zA`5LNOfOH%l&~o1oe(1Sw%q&}FSvBePSmkvba;ds1|+pzm5fNKKG=dtwGwbCfCynb zuS6)m4IesVvG9FMpksS2jH``NIo`ZS#McLL_cHY37j~XP2^79W5%nJa3=u*ZYX+LU z1=-~dcjMrLAXxLdVwM(J(a&URgr_{}8M{@jRXd!9w?8Wh#*U`GgZ$gt- zb*?*pPSlF)$-z8bkJF>i1{O(D&q|KaAGTFzeRk{1HxST#3qYl@<-e}<7uNTfP*Vij zqJ};o*)|Nt2UbX?LdXX&y0L1+r^Osfxz7z~QExeYI7!Us&(jtfvG0yRLle-PeXdK8 zxc8$*^PCNb^MxF7_l*d@JGVO$w-~}g&(G_RAYEm00JghTZZT>AHriinN!v2X00soZ z@`KnAYY49n{&z<}j&@UXebNvsj~oJKs^q6{%(;xn6D!jn`aURllLMIPzrOb)m`}8- zY2|dnFPOI)im%WVtg!}v?sEhaPTK-RmWloj?6AJBkSoFL;={X?lw3%{ec$+Nw9&_t zaG)nEZM2MYW}nLTW*3f-#%m0S52bwF4*qEe%!CV+;A50MmmPlu)~gh^?bbwFbnGNy z(JBKtF8lB;c>fR-1~h*tLe<%IQX+fVj>(1j6{spsL2gqk2K(OY{-~yba-l9NVz1Q?7@Tf2_HgpS>E>in+J8f1puT2_g-Xd6rYW3zMtQq&~I&mgP4D4E929XKu zl5hLf^G!Yfqg@8C4kM^jcr1a9!r01s1Q5~cTr7B2%TvL>2q_JtClAZbJ00lXB_AR11vhK`|XH=%U z52nUn3|Rb)e@UEx6((X*LRly&ShsUFyl11QGgLNH2kQ@I#|MA&`tLJ_Vjy@CmswD= zoc5~XVR;FxO}Ye;%570i=aRRCZit-HwbyMGh-UciJOdqlyXg!2tAiZidy0Eg4~jxk zu`P1w@}Z|+Z9yc9C}sq{LZebVBDbPwzfCv&xfrX)ZJtLhqc#QxIqaGq3B;$T);2I*GzH7S%$={mp4?Aaa-ZsKH6l`6SW>L5zv@mV!zeYYwof6&@ z@N!CP@XX_G{Q3&FihYmbk5naDz!9Lsc&$`ZN6Ac@X}jz-RX0{X3X(K!2IL6Ej`sheo{X+I>&npfkluVU(M z)x7fDFUe&?VDaw{zjX^8{J71s{glA`?}lz=t0iOl$6qCll`(QWEGEz##4)PcT=E4* za?kHP<$R6_o|0Sdg^~M-H{U1yOl}YyqYVpXTX>|1Zvva{okedQMjmxa@f|$#!Ya3H zhG(|MO3!;)DxG|ziVyOmSDOt`#B(DWb^_ZT^w_)p0?Mg&&(QKl^kkit<^tEH4P3i& zZJlWPboV)`r$TrH7NEa|?Jw&;&3LmO=R}lOh|@i-{kFeJ%0W2;xIEckk=_=|!R=mh zdoDON-sYokuIvB2-oAyms#SXSh5m^(S!#dp0xl4Em}%khMO?DYSdPNJ`Gpn1*2 zbnPezbuN6#@glLJ#E#GDkkz-F()^jZ>t4>)UHt~G8gI_(C>A&*P`aE!`gC0rGm1U` z;Hw2&_^9P3e3Sq2^+2`zX|Y@Q4p{kmW@CW3#Vc%lll|u^q5EYQo{#;q*5Ge__v?Of z=W~BH%wwkXXbbixe#n7z#cMC$d&s(&I?MG0@q&^b(Q>` zPa`~RJvkX=IVth^8sQ~Fv#a$GWE-LVFhWP7;GfXL!lA zWKmM)OFo7?(V@|f;$9vcMI}(p6)O^E#NDOZYv$12-iK&`P++a>EwD6pa3+p+CtgiuR~- zi-llzm}s~7aY{vq&F{ubeV>tx$2^ch7Fab>Vam!0V917b>;i+PMJ>r zfzGxWUA;LkaW1<$<>#?7JyA&E*)sjU1Ij{3$$LuA?xg8e;U7v+vyr^|}Ib8s?RW!bJJ+-YOwWGm^^lZ5^F-_O^a(lsXCq67E zk98Nm3iE7zXYvYn0~&iW3x{@dH`*f?pCeloVK?eY>uhR|d>S9>Be(2Awjdhcz9WVX z8b7X7kJ$ac+L$Xkl$`A+Ih~*?8^>IuqFiiAkBtoHAe1)SXb(9BfOnYT?&~X|RZL|e#)ID`H z+ff`$O3VdqLV8wUzES+tLV^Kd+;U}fHaXvIWik`kIuW@lE?yPUF^#rPXR6E?J8=)L5?E^=jrKsLiEe4}G|1t7@>rNoQt z+XR*HRSQ{Iox#QbexKlXp*3+>ZQ((GKfuO!0)5w`FApe1H$ygGq?fXyxl^HGk$`Af zovtdUZ@Q*&tF#tjk<)Nkui=A55#TfBqkXI|)%?yeqIMsCZ8Q`2!u zV`7sXv^9u#Yt6QP+5%0t-x~Ryp+(v0fT?vSwq>e3 z!RA482wv%{N11u4F?E2#qciN8s!1;{^i&wg>naD?91T9#C?YUU5L+6*t%bweW+GI# zuFk;YX}y`=>V8gz$+*T zQKyOs=g)m$ z8PsJ`H7%^c$g;LC`%8q3QHmafxe6X+JI3gU-U^#NZ^UxQBS@*qPe3WGnq_4=dn@&b zH;>2?R?9w_l%-y4o1e?9{Xso48bI|^l=uLs)~0GFB@3YqBaC$=dZ=i9Cg~`HV}bll zIOMnW{xzK&ADQ;^MOS?2hN@wi^@WHAb6;@C$*EZ~l!i(+ebx{#SGM5=$7me~0PUu& zyd}JRc;SX+QOP$5<^iB{WX=nor9}>c9m2GCFJ`Hwre$GOhH9S;#PQ@su;7BA3zXm) zo$99!SkU)h!krM~&uZ1@&lRtjw+1{<&Qc}6|88cPB2Ju%^e)ByLu z!cKlDZV$cmb|E2tKD}6*3Ib;hU0=4PTdeh54ec63dpGPpH)7xXJc}1eL95QwEGNty zM8j;dvKHyXVfk_>-C8Aobp&jH}7Oj$w98#XGQh~SE!5sI-d%*S*iGo`>7E&24CG1R& zi`v$8TBDnEw~GNc`Hfb&l2*kObDhImtTztj2QiC!6Q#>m6%tNW8cr7+clo6SWvw~w z!+A=Vd9|fhRl_zV(>6`VyZ(JDHEwryNltB7PMwT49aGo;XT{F>Ua?>Ot=I_w?*cf~ ze^u;=A0QDaAdxWsgMj~=u%i&XJK#tNF#lDt<70iG!uuZtoRa2W1pMEG{jUQ~L;H?^ zGrTA4!lKN7D|RJz4r+FOF>YSozX-U5G@qolz`qr{q=dMzVKNcjn)C{U7+ddgwp#dbUqRa!h4=$9uV6-8Nm@ zP+Q--)L5O;+%wYB-qAYT+CIMDIr^9X?wncZncjcrzyI3b?*jNc|GoHo;GO^O$a?3$ zhiCUkrdR%!>)*~s|CZ~2`S0=XYvZds|M1^;?*jNl^FQ(W^3CMR)70|We+2Nq<@)W{ zjjgX6=l{Up+yBYdcW(cd>w6~)n`eLd@6DUP<@*0->#g5b55KLi|FysOHvgCZK0N(? za`^vUt{<))y#8;z{w{!@9{i2h4{rYU>%aatUf;Za_@93LzvK1)vA_3j?)JyK_Ky$# z#_Oy9^51{{ldx}gAKe}Njo1Gz*N<2J^52jDC4ldB|3$$6D`EeC3*bk6e-ZGr#din% zuK@mQ^Zx3;5b*!siv8jr1pMY50spsRANyYf{N>kQ2mJqnfZw4}>%Al3Vm5amIzOHy z6Nwl+W}E4Wraeg~VzKBK>HiA?PTiHn?Yi#Pp;j@M&*%*C%KP64c+nn~2-z)a|t!^(^V!DBM1RNL$frujS)%bHL0-Z)v9uM*z z0mqffXUuXr_?k*YQkDAUpNf5nQCY@>PODO_Ky#+6?bk{rk@OJf2L9Pbht7}N16ib? z&4G}Xw~swMP6#!&G=>fzJX*G)#Q!4TeC~^L))nis|5og-;PHI@o~N6`s1n1uJ`d2B zu=TG(4FP@r+X*SFx z^1sZx`SFf`)9%XdYe*YK6_;8S`(e~i{G=wx#w(0D$7b1$<2kIPh~}t1-b-lxy-JZN zjv%w2Bu&jr94kiuRFb4XY+Vwsu4#SXrp#Amp8U~)*DS*zoUSa>%mv>(POH4?Fx$*r zr#!=g9!o2EpOkN%{xG(9@8Jk#UUIw?9m%BH$-9c+DC+nW4LU;Jg4pe9Sc-sY&@ zdzt>M(f_dPs3?ZAhLLgFr@H#%{5`>WtD_*Iamv^-9evZL2fs;sjI5j)kq0fCTuWDb zv_nZBHT^|*F3+#?p4&Iri>MVgIhZb3KE#c_ry+*&wrOkbF*cG|;(>36FISTS z4@l`rCIiYtuta{Kp?Q({+H-F%<$)DgX(L;Mc_mDhrsBCb77+l%04Z!RheebGLOd@3 z5(k~I_+~)x%Q=^tq-YSpmg@A-6v&4~{3@8jCYplNuK3+6)`KqUk#QX>2J|JFnC(@@@|$itQ-qjy@vJfc5P~02Wiu(69l@^z3-FK-BqPN4->fjF-rx@FtL#lcEm8Qs zaMCd2-)0-5UO4-0Kvs zs5J*0|H;8Kht=ATs-|wf8$*~8(vmNoGNBN7Bv1{s9V>%MHb>4tEL|3ER-;B;gb^TC z{*@r5i;AuvBLFeL2si+B6(+QAgpZpgzzq$-BD-Ob|2P!F&xFi9H4z5gU7qXS^jrU6 zAsch0gP7JE1SA5;2z}Ta4`IDdOLJ92R7n)q6Zq;4jTU^LZA zG7KR!hcBR%mgT_=DpjXste9ly?qo@>qF6+6JsGwNIOR;8F8y#Q`Bg+6-EhD(&pwo8 z2$@PXOqH+L0|9?}y;n{1+qC86h$eVk@r_xadpHk1TvP0azN-6>STJnIA=EHDV-}uQ zFzWO1s4othbR@-=cqg4ujBHK7GZI}>GY2172dOA3Yg1(OAgBMOu=qZNNrf=z#bA8tkXWpfCIixK zFT&PMaMtse5b8&Y;C7C?m$}@?M19B#p`1;-u#Ncq8A8Xir(JKR%6kJ-61Q=HM7akmnZ^2<<_vxo($ zQ9hhA_9yat#ZHRS6v@sQjFRv;LJjj99ZS!Esye*|IUw0*C8r7}q;)8e9>lXVxaXO= zH!h6QoDe3zPe%AOAsH|bpEa03PO&zr6z`IfBLGh&_B5qBcAfeY9m2iwu`9XsH~nq5 zD(g6p8c(}MLd*0a*U}z%63MAWV4v~GnuB-NExN7*X_edBdvz33XIG(nCB4Hi0ec7KSw)(9wXS}<3O z4u-l0-k#bTLwe60eE)1DEOUJsM7DEId9T>(Ih&Y+KN($Pv0ueM{J|OEhFJZx{W$v< z0l)P8)BOX0+A#qBbQMgfw+)9nZ6P?KSB*5n5nvGICy_Vllf}Qaci&dQK5Gl_^s@K6 zNe||oq>qLi|R8MG3#of zM0fz}(l&(lVi3*mZ;T7eJNBv1xoXL0N#zS)J%s_9scGbK;;Q#5nB~4XMgJ^H_&um? z0<6 zGx*SJQqLXxL1V?Iw1wDZ#&qijU(0939QUz=Q1~DZz)y0;67l^@{#|`S@7Yp5%zB$K z9w776nPILjY_C2+7L>KGP2wFDOjeRpZVC$1AAaJa9=1c_9&*xC8kETrd$&Fy(v2 zP8&Q%>hEO%-y|IH@iDj(5WVZW~d}jX#SxeZ&IjhW*9P0m|tQfnf_Ju*VGBBFQMyLiI90uP1sC{oI!Sgdpi8Xb2;(7PZn z1?9X^i}xd1m`Jz?^r@5|C)wD#G3K60Ejsz*M)}5(`Mn@`a$Uq`v|=0D8;-4`LT6Jjkh7zN%Q8KI7b)DEj0* zHp@!i$-Q4oTj+59Bn$j8iWF&C@lX{AQ;l$F1fS4Z_!^OZi~T_nEDUAguIyR`t$PKd zCtRf`>@n<_=@kJbga+OA#beVtY`Y37mk)aEAx}7q*#}GbEtN%2Hs>6z=6wGBDN@)C zpxl%%oKG^=2n1)9op1M?jhPK4!-B-*17|e=a?~YS*EA1SH$iI5(&|j>SuPRjLKR3f zzB@^iUr(T6$u5=w>O=)9bV(Ltah|Gy^a>r(07VnkRW7)Y0G0;e`--Jnk#iLkKFE>6 zsp>Jii1h&RGmBirLG@lI%1UBm$7$L3LRdr%XvCZvi_BCzp2m17x!J_>4*XgbJOM?x z5^{`+xlU+em{RPW1PD4*9nUQMG00t`%5|IEtuANEU&i`imbxIH>|Dt!j?nSP{J$$21t1pNY{l6;p=3IagC`wF&2T~gJujm#WnT|?`6f4b}E)Cf310Q-kRG^=j-RQV3E ze!uH#gqX$>mIA~(fIOhbzq7{fv5p|IoU=xbspI{-R82&;Dyo$^5QcsAAp&g_uoa?4 z7NSx~=F30yG{UHi{@G|~VHw)4@W*fb7_pzRUgG7AAOuLE`!~u=7=nj< z+m8~}ah{kN+ns7pa&4>;kvcn))*Dl9yry4j`VdFQ z>|>Q+8Ny2nVUZai47=Mfa>gMC-e-r~zve{lWkS(E0QJ>*abf?0ewGk^~5yd=_8HWK@i4Hx1WT6m2X#48=^c< zh1G`fGO7hh%`gfSO=L`DJcLYI*S&(D0B+gC8d{~q9!mkgOrN>o=jroivPL!@2_`8e zKYmwcgdBA|X{ZxclLUQL_4509qZ&Nj&qM^@kkjhI(ukZ0;mYt;Qv=o(30-k`jD&&$ z*FSYkuZH+F14}O1g=mn3cX3W0^q!vkdcVOljCa{7(;z8#wh2ZXyrPWpC}YKH2s2P6Ku$k1lm$<9HIOtkdfwr^YuD$-;>Zr z-3B@<(3O<{t50yULHlbRVj3@H8Qu_4_(b8pM$<%b$d3N`&gT=9)+=d7YSVrz4RxN^ zvJI+Vivy8YSzi4_!X0vQS1-VulRx|`Dl?t86cVpC8O+7-Y(CG~^FqrQeRr~52R{dx zUP1B{Lj8u7AIq)^Uriyd`GoKhpt4j>7qyY1Rm-%y;RUf7z`KdDv-}mUhbFpRilEpZ zX={>Ti^r)INc0`uXG`mKF?eQ6J8Ef@Z%vfXnyzLIxohMy4^q(RhXB-%GorL+WH|nV zxkbVlvV2%r<%VClUw}^Q+pCM{)liKLfyI;q)x2Sv2R*DzP?)U{=3$(+=P@V-o9G6q zL%uz|vOSQXLOkLi((Uy~fnC<9f}g#mJifcU*@u@sYpCA=ou%C&Y2S^FU~XXm9~;$} z0ZDJKdx(t+XdvivL@!vd0_al}wC@`*G@z)RP^-pyIkK(W(JvLnprCiP$nm3^Vr7v)w=CHvb; zj`0;u7pMDml@%Bte4yaGo}wF(H%!r>-xpwqh~qc1pvC4PAM4&NobUJBRg?M0jkqVM zKp6FTjqmOU#@f(mvyFO+?e?_L0)Wm*+j3KfjC-aTRsR{-r$}3ei!PX7Elep4#6Cwv zm+ebE#-GZzb*`Ym4*M!yzKVTkHX&9SvqesM`Cp!h#ePNAGF{W5=1C;%p4MNx^rg3d zI)k|csb#N23Eth{=6~b7?e7ip+#H;I9oN6SYc^WsOt{0hj-W9_p;o#+@V#g2yw<#A z;!K!V2j8+PePIFLuO3_pG}IVg-q`&)Hx;@#$=|hUaFBA`mg^sxr@mA|ZBSwE0>3;O z#y-r|Kllo@2cS*_)7`7c-uqhL(-hnn1wI}e-Zx3y6v>`dF-N)!Jwz)$4MCrKkez2F zY+|9_6Qbti7upKd90@8Eih!Zb9$RjiQ3sf~D}K7Pe4d->W;2VNA_G$C@xHrweWyM?4cr4!ar9QN zj7LJz|9rpK`n@-jNcC%vCf?a>I*U{5P-~!2B^1Fko|*`~M6#<&qkms)sqHig7G=~Q z*_R0%yx8gLd$_Uj;OH)`VV)!Fm-oARG=%|dE^k|?VO~~HW}vUO&T3~nm*!ospyX5# zlt9CLdu?`mazl47&zsNvaJn*AVfe%O#X4}~>z5xt`b{k8d0-pnwRNPqp`1+)INCZk z9Qc>5T(1Vx$V;z8vsk9+L{Nx!>jcom&S2km+9`Ynyqtdm$L@*GR*?9`ArYfg%M{k6x#q&J%}LFVDZ;e-eAm>i=*h#y(Ajx0jK z3!@T642|gA`grw~92y=pB;-uBNuSwGVGF| zkDSb0hpCs#Nca6@El>yR2Qh_rg?^Q60AK*F;N#CA?mn;m1f^|2%bLfV_i;%^fX``- z)W&mLlvaTEMYm7@yp0}Bx>5uQ*N>YULJ!T`g&Bl6!*4$WKiK>pXQuH?MjYrjTmPNp z-!hN|>k28Zce1G*9xL&;V&7@&ASLwSWN|ms751Q}5kaz04n`w2g2he-r$6gQ`<+l2 zAIV1+&-<16mkejNQA_uuomQ!VAvfPHSnJ;wSlGEC6@H>Th>@m~N6J!|RQp&UjOn8> z$>{CI~<&xHeG^H83?j1yiongTyaGNm7i43AC*>*N+nHDl871Ch1NcXl8M zbqEopcVi=KF&(m#q)3~7#}%ZEemIauux^_VOAR5T6-yNp3P(s}l0#yA7#A~rYP6Qv zUn5VVozz&bj&3E-j_;U|Ho3P!dS_vRz1CVex~{p4Ka>%sVUDYT4W=bW_bnL8WQaeH zEwcQ?I{G?+FD)c!lqll7!X$-B&GJ5lA+R{hdax$$3tXb`bUt$Edzvy!ZO9HCDyKu9 ziW)gd$tU}LC{*Fy?{esv-jYLW;Utslale!$%WpzMO(_FMkClRmU6r*j4r4+qT0)bS z@{v|G!irCwc}Qe73MM-j%fdC@9jV-Y{+n{v^+E3L2YMBXseX4LRSAPe;-0MRqR{?Q z#TOZ|y8Lwv3pRHgdVgv85QOZ~)`fCM#!5x~)cKnAgPH?68$O2hW@loJVjlidO=g$z zs(6`dge?X`vY(3<4y^?S`V1LE*|j2DtvWr*Ma6C+O}e48;99F*dSyh!7g>RAw${uW8N<2<%A5*zBU3)}67Fto z>AiKv>~c&`p~l)SYto6V;BN;P}y-!q)mK^#w|PV)$jLNeu> zcIafdCFtl4-zNgwl zyXbz?_3UU2`@kIpGr1PLVuXTo83IW_dwa3$+Ak4qN`km{5AVqzw&M_QVEO|k5V1Iu z?z<02nUX>ViG=fQ&jOwiNSRbqG1`Gpmpa>h@$F_H1y-ZYbUG9ekRc#2sNrLq{IF80#u2bdG2mm} z%t#n;Cy+=73nc(nFbu1?GUtfyH5>c1Qe7@s`L`_Z78??SPRWN07S1vkm_YS7c$&~v zN2MK3t}nc!PTRAmWA}3BnN`~9k3;x&{=Ys&lC5-ai#}e9^Ak6K=GjbvNjA&K5>jL7 zDWn*f_k*z}B)003=|stECsr&X&s}Qq%$lDWPvq3*r{oV>9sP(nvmJQpeOH5$BxlTT z4{m!aNr#rb05C$R12c{n_vknvE``M}Qn5^^V8ot>ZsrM=xg2A$z?CH^L5hw};BpI6 zs9hqHdKJ8%E39R*{p?*xc z=zB<}0T6xU>X)-T&yR|Aj`AR5Te-&24;lg~EU|^&uJ1L^`AZH4%g#=r8c)^BG=#3I zpT+w@3RFBYz0FEu{S}57is(E9VIjxPkdF^$(iIRbwh;FB2s&;cOK=ak5Lvko4L8lm zwbypgl5q}h+FcY9>#@+tjX51NAl@~hjK8UfdJx->*ljkB2j`p00Dy@&l741qq$Zwu z&1N3BBf+w$^$-W&K7029v|M}6gh8tYyE!{2rLb2kv-=C&IlqSbL8Yj{S*Td3Q55A1I6v}m=O-*%lC=PaiQ>kJVp$* zz=I<25`x^NtY8DSWc6|55n|IqLb*4uwg%@mscGlSdAZ!_G}4L|jlAX_VYl z(Cw;GlP9ygdm=V=qLTq$7!Q){su`tF`YD;s@Cl+XbsQlDo32zC?)WQ}ALjU8*%1YR z22ad#lf}eZEEE#PXf@YSoYA2b;7G|HOfNjK2q0mkU=x=jQjuNZ&+`(eHtk{#+30#k z9K4-CCFKwLlOzcy;*o1b>h^OQ@iXm`Nkrd$86n8cBD8DG9Ry{z2C4PTo-T2j+zytcu6dOCCa-Xchk-o6 zOIL|OhbE=ATcuqRwVv9Vy&1D9&=EQ;i=I`~ahB7J39-S>C8aL%bR{6T)b(SnBRKWz zk$81jEgWHEP^Q^r@-H+mlPEe+PpZH?2QPID6!GQlXx}jQsu&=LFo(csI>}^0OjRCv zzVvjuKeAK_0vY&cS{5`kY@-~eg4XuR*rIB~T(2GcZBMZ@L;*o&erbU>k;Ac(u+VmWO{4q?c+Rx+b;@m{&^+w|Pe z6kv35!NcUi-P~neS?Phu*?1lLvSE3$>uFN2P=PkMC~?vB+6i?5p~`0oT80&4ETKNv zNw$+9Q@;=zG)YqPAq&w6O<`awUB0&jrPZc79I`?e`ggvwx*}7oQ2kFDYTA+u+~-Iu zYzyCZla)K&7~;|xct?w(h>}QsgpdY+tP@>85MjT- z<r8l)GN6@7!~GlgP$dT|Q6(!6o2G~O!^HZs+8Ee! zp3-Pu-nOp(4+e?7dhG#eQymKfpXgNb>E>4qeL;kclH%<()~(&JFv`MNqR`DVv#o^H zEp*UUb=SsczZ405&}9hdK5sOKbCL~x`f_-P&Wy?&%YbJD^eDe5&QZIdzFlc#AWmz9 z`FRz1x0NtNAN6XuhoGO3C%35Ihj9?PlNM?FaX9sy5}JArc9P~!{KxJ0v|)+ORlqi~ z1$OkS-doM40M_dCCt6sJpLOm#dgPigH|q^lb#V2|cZlg&eJTA=@;PZ+{qDTZZ8Jij zR!oK(Gx>Q5EoiPWb}-3|Amfdp&6+Nfw}%!O_LltVV2{gV&)dz6 z=FgA@*S5--ECz%ueS0i-&#r)?MynN^RsgHAg1BgVC9B{e*6q)pT66Kk3C{3?5o|+B z3nk0;gU^Dnh>f8tjb^x#Nvy8%25$#3@Zr@ZEKCLr<`=7M`9?`<2e!*`n&F39ntPJj zmOg?%N#f0>z8>yGsy21$1u+#N%$vDrD%>ZdkADmu^WS%vIt+On(YZDOM=7?thcj3> z9QwywrYY?CS=1Hylij!-;W)lo1mwZidC`>!STzfW&)ix0hGVB*?8a>U zJeECFFIThMK+btPP7nEp%4LI8WG)ka3=7vAgmMgWurWfBs?N8m={uEk0$kngT*gWW z5g)`zGX09HY%Dh_6x@?4u_%`OCP8hTI$-JFY!QE7VP~OQJxS0rX4|y(YvT{W+0!q8 zRd93NcpkA?0m{Ay9YyPGdr@EGn8z>5U|V`Mm=qM|^71{tH%x(i#?%9wWW}SxW?M#l z`$Z=EB|-ad%N;QJOjFy(g__|?lBXpo))P48Nqr~l7ZWQU*}3Ue;QcyN`B}4o6#L3h zD-yblp$iIhb%(?Fysou^)!!W(Iv0oa7yAzQv&uB)JilJf9WI&3U7^jj)9uD=Z80@% z(rxHM@X`0)&gJTDe#T$oR-G7I(C)#QJ@VAgCXVdtGlwZOy_xwRgXl9C~gyr=NS5q5| zslp$ASR$rxaAD|oVdTzV9~Hd$UM@i1cj-Qgm?2baVxz8JTk>O!aLT^7gv@THLm_qOxR0r-CU z67D*kW?e(&QcCRnV_SCSkw+O9Y10@#KR!$O$hu{=400u_qSk`Xa!>M-TC~6UtAVv# zzneVnlvcxK>`%9dsT%>ItZe~zleuG!AG8t?huTtF23q|C(QerFcl5pPdgC@gqW5(8 zdq%gbk}CSyiB9)cYj{y*zRHm*N!4=d7JjhCd6kCtroH#c zx&HJ@>3s^Pyp#f5RtMy!d|$Z|1jTHLp*~Jl_BCbk0M4KiCOO%iUXu|SQh*dmU3d^V2hG&Z&eCaYmI)+hFV5O628`m;!j zS^XZL8%dKjj5)&r=BI&abx#%(HRpS9wG1QAl_4NXbUkeIq}lLm0{0gzkR}sDTNz9p zfqe`k?YyPN9{v0s=hPa{YIBu~w@+8Ll2VpGTL9-#T>IjXNEJFq`M zsP|CMrZ&lAT^EKvGukq=^no-qLB=D~G+sFz-x8gg%{ZK-S9MIz_*Q}}>Zig4d^CK3 zhYWj7RY##s{UA3*9KTFrEbH`E`vxtNG>aZT9|G^b(l#=SnAp0^H=>k+LMbUP2Rs-^ z*;rv!6O&&}S`-=j%X=gsV*+S!d8To)js@uRwfGLBd1qD(gL8B^X)J3xw7HR!QI)Zo zuYpcPS{COrSrM_bG+w?R{j*Al`|L@jo~{q?iErC^CSmzaB`_CC{w)1Nr5N#%9&Onv zMqV;Z*?MUeuDQ4|jwBKp18!ve0#GK1O|!T(^?7|e>*Y1VLgf*?+puw5k;+KRt`E+V z-oy{0GMA}X)3PLTjUc=gmubDC;9Ky6y=BUImIRhhJYjtlLtk2Y+b#~AwKm(E&n1W_ zIvc4%6a^0h&8~*=qOl2n7Ud<4u*p!=s;tdrL&vq^ zVDamPbjb*1eq`LG24O55B^jUhuw>`5u5#)`SH*vZ@|36qa6ja_*)xa(tED6(8f$*J zj5rMgUUqOt1Lp0VSA+IA*v3atlzAql(eoeG3^yX>tlY-DN5s;!Xe2+g&7FOeqDV{? znU#H%XOfm1=HOTcfLM?gRV}T2jw*3qEq_IG5on#J2SPU-I`c7kjd34#I6CFSRJA7p z=j_qozlqR{iYd%**@#s45#ZsjSsLQkuPx_OUH$ZJW8HgL&iA z5f9Pd7xt|wSbu~f{)1=9cpHV}e6F^%M=n)o;WY>W{f=uosYqbS2HP4yAp_rVipdh8 z5H&?1AgCnyhA8JnW3NQ1gkhT&u$x31N{CD+XaVf)EYnUK7cUG9ldACPNMsE=d(J+a6zM6Nm|oBQhPIV8%2QEk-gpg5Xf6b?)h ziYEqn+fX>>6{innzq4g>0yw;Y7zDxtU-44(2>mnbVHhH3f{!#|=Lw}(_&5zr)W|q^ zl(|kgb%wht%6&8lsea%Sh4^U#l17oNP_uqdL1S*dUz z!GPjOSbWXloU*jK-s?LbF?iWr|{L45e0gG6-3_Gf((^{3Z== zF4VT*3WOWdT9Mo1g_SI`r=N%2r#8zv_n4Yl#7%yfTG?aDT8Ph|y?d1J5+&p_)Rs(f zOGvYpw|%5W3|rH2*8-7Gb8;A~I}ny%b&}Y9f#OSNJrju3{Up^Xjx(blZTRs-PzHs` zvQ4CFOX5V90c#-&<^q}U_1Ihs6?+o^Tg{rH$W1<>M{531j%rr>Tn=d_;-f<}!r<=t5E@LA$VmWa_> z-uqg%BU#k@-D+hzi|vQ&5VWhKpOEd5s$$nFIsUCU>edbd%$d?K3Qy}e%qd>q0)1Po zXH-TA1h!-4jHf|^PlL|7OM}s50?N#cMMm$5pg=+szS`hT?!_g;*c@cM8kn%J_oge3G$C-jI03b?$4{2zrBAXJ`vi@hxE3V=XFXd89d%78eI40)1R!EolRqQ zJdWWuy2SU#w!3bp%sRf^psOOR@8x5ZpuX{H7f^qPw7SIbeEroR=MAmWHlyV!zwFuY z$ltv7((dTIGj8a8riQ~mLL&H9R|I|m`|a?)3f)5NB$ICXy(*NV6VI3}+(8%IuSr4j zk^YFA{4tvUlBIjs_M_`$9a=JFS;U}YRqJ~e(d*9vG?LWFu!#PA%%K2oHy zLZn$j}dY)DcZLc=FU6HdNr7QstuvYiAeQe;A(T}0o$OW#G#P`=9%38KTnG8x>Z+udbV zANY8`%kXT%2t&b0Zt}%mn^M*U(_`m}Rr0yc@DHac)hr&)X%VgF9)<3n-D=E95eBz| zDYwracfcOTxfstPEKj~EFH;^BnhD=K0)8LN5>G~bHB3A;&Pon8Wn3d!LYp>1WGWS9GZW*Z6qhU&7c_&4B_Q?5Qu}EtC`myjx+lbBaLoHt zN_LMLV@)KVQnn8U#lSS~5g^xBst5&f#Ul2JaS+O7^%H{GrHZjoI*QP}*}nd+^s||; zgUP3IN>u|hRi-juB=WQ*9m&(upi~lN3@R!?1Ahr%jPb$Ks5o8g1c&T_>;kr0`g_H0 zj@W4|bG@%jjr_TqM6bF)YMEHyLz>CUJTc5%(PT-b_I9a~OqMLShc>h{ zuA-LvXaMagV)N#3n)?F>2O3Y465Y2$R?Q(#gNmR?h{U8L%3@2OT}_$gWN#m1&#?+W z@&m8xip%dbn9pee;h0{)0=GlUeyR$8i+%IiiXdXz6W${bQxPOc+t&eTO?w>D7Y5X} z^p&-EWu&DEAPe@WjEE-f!G!V3w zkIPFZKf&HL%iudnv(ZdTugVIa%RD`f7p+PTr=ygv%weL-Ew>8FugcrD-jb+VX|;|# zIZ+=w@u{Z^*`X`KIdy%adm^(brmxO3vrgip&myQURkY~?oy3<{W#7|8Inak8R2$Bm zguc<1#oMHr*c1Y*OEPW9y{kW0o(d<^*A7+J5YTB)+pvup=vASW=0jHs(!^NNCp1@A zj?p&-RMwT#mo$$xnIP6YD>i4FHAylQb&=taLZE*LYDqKLe12 zAsVl%XEbcvb!*lJrP2K~&7YH7XMg2&fyoHWkf93=)EZ?A~lEl7EPSY=<3a zN9q|9yNcwZAPPZ2eHRQ?g{xu;)I}ONsNQyi@r3BP5%`l3_{Se+rbZL+ljA&28k-rH zo~gd842liZf*gL?;MKlpC)MfNi<;EQ1)Ndioh8UHrdLY_{2$itx~a`a-}`+Mf=h9C zcPnm%;!=u3aWC#}#oda#ySvlkUfkW?t$1#JYpuO!-+Rva@OPfFA^)HaWv-Oz^p?@WwwK_Yel^1G1J#5U~*## z^}9RMP6qGGhPVzX9?p)gOBrTMPhBpg`Ks8BOK`ByVQ8!SH6xBK7XJLQDe4KjNVB}B z^KCjme>5$)IFIwVrw4(%WM9()gpQ^?ssVq=eA2o;cQN@Y>C9`4FUi}C%UHQH)%c`d zR7&i*=U6=tOkjkjnO$AXN_+==sDV2mo%3c$6ZB_^t%h2-(3Yynd0le$w>S@8@#$to z{0)u0*cDQnu%1U#vV00p-hCU*iuRG}j2UB>5z-B9$#|qWimqLb!RWWa=Zl(Qw`6~A z;nm&7=jXx*gjQ$O7Ph{2*vE2bD)9hfR-AlG6t)hp%)n+NEZ!|f?wLmyCR@6~38=L& zuG&0i!$k%NZZX!~YvjB#%@ZrVsTD!e>_D=FYxVlZ--RBUK{n*<*ZU!QQ<- z%%ZmNev>s%ab1|XJn>ZZJiH+=2&LM(&=j^)Mo_2GGrTzZE> zI8KGYTLDP@0t!dIW~CxNhnOVp=ZKTfMptQk4T?n$*96QVYjiMh!2u zj*`V3US78}F%9)?w+l%ea<$s3jRIKPNADze-ZA~cAPev~n4)c9UZKvDA{*;zE-S;Dpv zy4*P|np1m5is-Ac)qJDQ2B+>pqwYnc&Wf=%q|xdnR}U%%la^``$z_pJ^b?N=A{keR zORf=Alks1$4L+CLFA0;cO{TV7-(6XojKAO0Hlccr7nns%n0azpXf;_Bb6Gau7v!*- zg>acC->W0YTeWl995mU8I{i1IhX5Gi09=3^(8YgJ@Q45?bO1c!dky}d6g(oqdkP*3 z2?HAaUnzJj+J8Fm$OQjJ^k{_a|04zej_9HAfB-yXFdh}{e(CEla3x zZmsX=ZR}}mes}Z(!>t3$o&Q4gom12QarD#gj{ds)-;Vy@5%}TZ|3u(tPyZ|bzA~_U zH#ocaFG~M3)KED*vHxCxA6|I+52Bwx8~q;z_|flGuwCniNG(9{8NBmzWc|~?@rI{AI&fSzasj@qrIh}zWFu2=V0R!^2!Pu^B<{+FXa-#)+IzFPc$aP)VN|K;eP{^RIZ{&Dn2 z5C7`G|4&N){Ce8oe>ygL`v1kz-~Hq0SN?x<^zVrN_2l1<9u5L{g(s9k)#(nnpC(fo zvdirao+FkCOtH%o5C0dUx6dDpCjAsnDE+??z15>t(VwVRn`5LSmc{XG;Q$08nST&H zj61!~NJHsNu|mE~rVJLoYJo<*#o9>Yi6SL*PdE|3uhmk$=}ewZd6KnOv+aywsBAN{ zcDvif{@f_{g*Ih!2LiEN%XG%n$h5LDl#jL6@9IscMd59> zIvLGOUW$=NwK{5Ba+DflY;MsI{ZA|(R!mchANOP zpNYq@H(G8fIqso6ezuwW0kZk!*E^y=BoZ2mWbW`h4KZz>K=~o$bar=Uo6M~D!|xUV zL%Zp#^*F!b_2^!_;RL_lOzOBMsG$>p{#j}>2-mcPOcq9scH|#KuN^|}p+U|-nN_kK z&KRDu?Z=dbo9jqAB(ct zua78mxc1Etb3^kQigRe|1`m@1+u#oJ8OCu-Q^HKkj*8%z=u3*TCFto2W5bVei!zGJ z%5rm<(GSuhTOi~NOp!f&N9D}}$tM9x=f{?DBbD!n-lQqKJjSJk@1(X!ZJMD}@{N9_ zVe5Us+_-3A#+dG?scLER-M5vg8RfH)WLm!f22Wo}T=2D!0fonxwm z*f110Mwg+#VAw=7*t^ie0`K4If)E^eNqe#B3AXLp(P5dK{1T-7e6<8FHElxeD{8Qz z=@{B;jI~l4uSU96-9>?&Csku1{7`4xoD1uh>n;qP^sKyPSuDOYTS%m|-iF9p4qTG6<$WEzNRl}fmeR3B- z9fE{#StQBQDP#7F7__XhfCM|i+o}7pX1sJ?(Dc$jhH;2HRPa7b*h>c)-{3L&@%DBB zqL{4TH^H;cl*hIJ-$X&^P;lv&Z-1@>l#7n(fc0NCiQy$R>jKMfHr9t(JKet{`qwEc z955*uZdWg~!oT`^qyhNpIrEUvnXuR!6={z^1tOo+&@T7^n>&GQiFmUS^X;=bFA{ky zgiETA+=9;GIzJTwC?X^v#tP)r&U;LOf~Iqe1+n&NM>df{cluqrxtDK#pv488zbb?} z(C6WQEGZ;$vu0zPH7M)>1&&g&DqQnvJ~b?o&$reBlv_w05FUVrfxRS<2&12@EdvrL zUu4m=yNRG_^Jme>Tqp_jcd~%&i#4cjNV0}=AOJK+m7O$@m6&FL`;%l_oRjADGTJy? z9|u}kWH|;QO#&Qm^%)w9Wc)65GH3txgD(JXGwT4R3#>dcB@X>S&Y%$6q276&W>r2- zTgyE>m>BAsDB2CLHrb_FR>{r^CyN@VAgttXubDsVi}BXtXZ$|3eSaiL(!WDwt)7$( zRi_EKH;(ukIphK-U$lV!3ZLn&EyYL0OfNk4<&MEDnMSqxhxD4IKRht?mlM)?wQ$H= z`@0Gp`AeAiR0uTxpt&I$C-WnGM#v+=>&u=+W}eKv%9p!T8I#nf-Pu2hMRVdN zQRPMBnq$qT$OiEBCnYZ8vfmyV)r8~A^}huzR4A!dSl5;5ii)WdIwbxKh&qkNZqBq|(A9Nl5|US`;H9*BoF&!<-(4mD)V;Qe?^mK{B(|k|N!B8sG*Qxc7)hpJc2Chc^#11^KQZRNR)0 za9{a`_~VRi!tg(IquQ`S4gA#i5o_te8D|Zosn|mCbR&7qkq{uSLAfXk=@{ayc7WGb z!7Lx{GVQk$G?m+N@M;<0iDnI9l8d(8eHb9ZW(##i+9iQX1vctjnLe(l0_rs?CW306 zY1&cVX+5#V_;q6AmYH9s_2~4S6^^A5cC%O@$;43fQfhMhrc=R6187IF~Qg zn5NDA+2c^n?2--}_G~yo^|Q80vFv<{ zd~=m!#ktGO(|oIoYk@1%Pubql-%e^M7&l!PJ|DLKh3H@35xr;+KOAi@3HXVBcOI9_@Ff-qJIiV+L#B-D{t$`gX)t_a=u1W;%?_7m3ak zZ$YZ_SbgGYzWwbkNtl)1^y~BT%G-U*gtx0xblCdM+uvWXLIX6LrZW?-=QU031q&IL z$shGD;&p}c26c2c4zHE|_V^rnkH4OKLY~WFUmyp~AKtD%k-xlGu^;|qJI~~=<6kaF zyd1DZ{gZ-!u$6d=>4wZlwCU+Ag*PSySQ7u;?2{pnl$w_)8G9{f!=Jpnb&hq(^^G8V(a zB|}0UJUleQJ%&QEFGF%`!YfL`D|vkV;5@^kFZ#D8gB`nXo>v86PgVdPDL9IRtn@d3tx2$DR~IrJcQpm3@-pg zeBg;PHw#_DHET^V?OBY78ZwDpjMn7}!?KTbRn<&2iyB*uSyPX)9`dgXjiwHbg360p zqYZ>FjeX)a*rkm|eT>~Riyonk0uTE)mjqod8kxXEUQGI3(4r6zTaXO$*N~)LCqebt#uX6@oi9=bV8h*t~44;RK)^W@3eD@~hS%5ii{sNzb&q6UB7OaZ3_b?_i}x~20srToFmzmYa-;mLoj zff^Roa;F0~yssj@c_+5yxIrV*Jh>wdI_B`?W=ZDG9i=MK$DSvh*&+>jIU{hq+!X(zEfK&rNO<4vkF8A}svKLCj zrZJHx){0yhTDh&%VJHV%8XAujfNEz&H3^*?mia&rV7)9qG%v&*$!*j}_Ur;;+d-4? zl{d)b$qqr|lVZmeAhouoRk&x-m+;nURB$e)-@EILJRvMt6!XbMVkbdgL4zIeGc**6 zC!V68yP)+T>d>KhRWQqhda)n(Wt0$(xs7Y9#vSD)i+Ra~mElmKw%E%uWnh9S0MesY zLi+admd@k~B3`lnd9JzZctx|6K$PbiRQ^0UTo`IQ7f#ZT?P*0jlF(z+0Gk{bqi?bF zBPP~eqDZ7A0%8ThBiZ0(82(A9z>_*28NI(ImE@@(j_->Ni@*W=Em5Qm9Ft(qSUnn9 zGjL74QlKK-q};Pn!%)1OwnCdMk1o0LvAe`3mG>56?TOIb!3Er2GFtbu}$)lq1VsO3~lE(ATcM|_7~*l#D)E`+$% zOvNT$4s%}3frn0zhgXmq(?CNiMO?Cgh_YJvq1w*0YTzU%l53|NY~2A{OsB8ju4SFM zEGY7%qBHE*#_KYnXo94{ z|EfVeia;?r&azYKOltLcItI6tx4*&lsVEgZv!*m@qPR={`c~2UQm)^$1o8^|eRcQi z-DFvCQP5ml8P=MyT-CpSm`o#ZxMQ~>QiUc51p02;gYr7HH?TGf6?d0R+1B9Ic4ExID~ z(X?ne0H?k$S$z23a5)=~JM~#P8|s9cSF=Luho(noe*L;vIa3a;(Vk%d3w?cchuIKo zKj6<~hAiQV@#iZHTmxKp)PP_nMQYKLK*-=t(p@}wq_otr* z>#rcqU$?2PnoT2$mmp$=)^crA#*Dtj5wkDCAYy$`L}tBeSV=S$C$pIRSCxTy4;dkl zoPv`E17A| z)lj5W`P4JC|73@6U|@nkJa=?iEbB(cRKBwNhN%4R-C4c}WPK-a{kNnK;bZ7UH8vNbCpGQ)aF>I}Tb3Hn{fZdlQ2#p-W_(?j^fcCy1h zX45C1)1Y4$=ZlBN6#Lm+sZ#9FVzJ1pS&T~ z^udj`<5_K1D90E-Zb@x3@qf2dxKf8i4w-!`r$PvAkIo~@CJ>0ZnJ`}lWFL~!UD~^7 z-m>3X>D*$hmgYY8n(NgBOP<%4o)e~BpJtu#ttGKk-O9V(wQSwVRNRHrCh~1eP7SLd zZCuhK_+)h6hsOLRz1XKAuDuev`!tL{<#oZmaT7IgM}rgQ^}*YJB2Lr&K?m>HilngI z=H7SpSb^luk>yd?{jX*9Uvk^ijIYm)F^?}O_X+Mt-^|r(^qw$d?%A`m!b!rTN&eQX zKRK~nmPrg z%n<(|5bAz=R8I8!pA>vSpyitObitn)7s%PlLXU-E+ikJ!X15B|nbDC4;)} zsO3+2pJXi0Kg|k!jOuz*YPS)AQRN%JT(X*$?pdWC+oyK*aF9S%_rN$8MWt$uiQGXK zL*+Gnn4~B4#CQ5BayMpaE73YqS}Sq6-m2I^b)tb})a5-}! zJ!<3UF!BW*Q6Kd{o`a18>ci^wC>WYhVRWp>^Ga@Bf3)T^;!+0_NZSq3e*6OgFMR?y z=lYtLU9})ungi-g1{OwuR>fZwO%;*=lpt`veW&S(vy;+PjknWcY;#UCQqr_J0NOVJ zX(9UDj_MH&ncxWsu)U`1#d7W5tGFH_Qv$uEW>nRdLOu0He97iGoz}?uB5E`_Z`tQnP91D zIJ>BMp|x`A`CP5xEcj4NbIN`!~EJEi;g`60b`9ca?fC(cN@VLnnFP-7z)!HzV*o)risk4bLnmi2BJ) zQW!gRIgvQy>?nBY)_mJU#0uhKl>Hbbe^iL9W{aDw9t=Bf+jllEQ9D5pv0F1W&LHix z2^FEMVb{dJ?s75Df1r9w)TY|?^X++~3lZW;(f3THZO8bXsl`&nwq({{oT2#rgJ7G; z06_En{dWrfcH#qI(`go3%G-Hfi<|TgNlvmSOP}=bvS7l>J@a6K^Fgu)=AMPWFhSU>aveP||J zkp+|Y6#T9}tl$GgWTsR;kd$%@B$fgWtS#~RzWk*|Tk4mgV35Tj;`m(6JWp?{EpH#x zECQ7C?^grFP?51gk6@|33(Ods(V3M}6#M#iGG7ed0z<4y4IcDHTi~}2L!8dgekjuD zh?R6zoc)7dlDgd>g@7%59sdfvf!g4ofIb}i5>Y=a8gsLTjionru_i*aFdxhaQdoDK zls}bNPTGbPMD7SGf=Y2e3l_qC6yd4m9pe62ZrofB(R{T)O8D??l~&YTMkPy0`1Hh> zLHTh+xs2RyeP_qHXS;zbshZBAU`x@;t@mqNOj4t^!sVM8jwpIu=&%5#CfuZGd`7r; zY6Afbf{3&QWLhq~yCd0r1OU{Co8s6E)Z^|HBL~36rp%kC5jgS+h4dd`bQB!W_zZvP zuf*`O5&in{s%Ojyn>}YwdbcF9Selkg$vq-g>GNJ;XT_E>${<~#n77(gsL!?ilhYUe z4ZL36TXb(cs3IQTo+%u>nQUv|q0AuHBxT-797-k9Rd_Be;Ca;@KTknl+eo>0G1j!K)6tp zxo-xuMk zYAcBO!cQrey8Gdx(vjBXQakLHS`g#cPb-==lvvyziSUH+&}g7a#teemoTJx{pfMC+7rp|0uPS*MNYYWn_&h1V-mv$_YJM;F2(<%&* zAfT28EIuZSgTCPtUCQ>SHCwCSh`PcqvK_m~$;$|yOs#kg&5u8e+Fc@W!5^%CDK=I} zw;CI2>?L=jDAAiI^KZWKEp%Z$$68>V#zI37n(2ZrUGTS{8QuJ}_PuL@Ciu4`1jvT= zAmKGX;agQLz^%yhJ(-JbBkKV^xSuXEtwL=HwK22zf7z&Bg^R?g<2D-+HA>y2Ka_csP;0OnM(rUdK zw{1N#s*E}@T!lW*9^27anEL*#91Pidw@ZKa-b&g)kf@93S>4RI6^BU2Dbi`n@evTd`t1 z&#V@`W9igb{11gH`1u^k@-K)d2SorU;S$FxBX@ar;V$1Y-2CDjZ6aAO10+vW9!NK` zkR1o5KoGx8lJOl^-&5Z5hB@OIe z1i#;Ks%$T$OtIToPl}G#vFir=kVT!DL&ffa%wgb`mcE}77=k>gSj&%*3AQJr4^1y;ji{yCsCGVAz77`*Xx)K z_hE%b_f>3{`e1^VZagl5`2j`|7*Y)&8t(CylJhbw6)F(Pzbhms{Mszj#UG*NQvh=) zq*y3Q1m25ud_Wx*;REe^?|yv4Wl{njo3;CeB=#2KpF1uZKF$U_dP=|#t>P!T+M+!IWrkUqKLXMeBsg z^MwHgR4XN>eFWFl0l}d;sCKk0_5O>}L|ArE?3x+mZkec>0QyZnj8a-dIM7TO@Bst! zdB^{2YhX^gz-9NRjKu(F3K4~=4%gnsqD?{^DPm|T;9(uNPq!b0Dgg)&j{o6^OrYNc z{{Rr!1t3p~3a$1c?$);J*R8tqOB?~wytqioIIgy_StEV1tm=$scwQBG;d6kiMk0t2 zyo~V$Xl6MSn~m?ovG2kqJ)RYZmN~~W-2e^$z&b{3w=U99zZq$e`f{Qs8R19t_kVLh zB(dG-zQ2LrNJv$PaEDs65itdKK|C!zvJY*Gma&dTA!evGf;qow;JOM|7`7fTId@TQ zBAK}u(UjAunS`iKRX7~(_&I>3Nm0M8$A?>FaAy zpc7-j{PhvmZc{L+zE7yPk72Zlk%EEF9sv8`C+jbIH(UfkF+!q&6Bm#|gNg(4!woyb zv$3(}8ukV9kPc+l^*8esFWCFjZ3E}Xdd^BqFi^?p7^@^F^Mq@AzA85IdkHeF_9%aZ z=J1uY!t5$?L;y*3X}|?*#usbkV-WC1Yh(Z`0*4vd2NQ1_;(NICJ);&3>qbiX1GHhk zvUNAMlTeordNRT$Df{|Uh7b&QN?An+Q?$~TH<_3FVt{4MwwtibNKM^_=(d_nz=`Ra za;G~rngwvW4gTg90KkK|kw7gJ-LPs#-YkaSC}aFLGE%&Xv0mlShv)FYbby57AJAvN zrCb4R54HlUW-H@1YpaonFzAm7A9b(m@T7YFu1G@+L%SSTE6iI11NnLO#CVu-N4hN{ zh!=rmvEvdc@{K1UII%4O1+r_1Vn+$%w1+%mmhyi|LtHq>KPmYiMvkkaf=Gw_>nVRF z1IHF(@s3uyFUm#e{eBG~fpVOLXzAtUeLn+p`qMDk_E`rtX8T_R6j*ZolW)bTXCr5f zqxuRyali#HU*R+FhFLTK09N@RBN=2j85Ln+4CF!ZmrTM-D7EboUunttA^8PQ00(aS z^<@Gkt5o(X5VfO{_)#(;ti{_3OhFYyvKNtbJgQp*8^e%*WGwQ* zgc5m}c_!LlA~59|B<))%`}w5>WbN!marFP31d4U>3x#%+o~#5Cv;gP8esp<71en3W zpa23mRZS~efw$P@sIm;lyc?!;X#0Dq)K$E3U7h%Li?1re+>#_4r0n&;*pBap{!J@Y`kZ~ zh~gZrwTML1kqaiYUvDyQze43R@;C0(6clI(jiI31TfL|jB(^# z$jag7bT{##;hlx|dHDtA{x;#kbrf81+6#KYnDR*fxP3whYH7^Ql*#8=F29la>?YoV zE^jx(Mk(xvV+SnF)GQYS;&(0MPpTuG)>0~id^07l7lYMl`RkeZDH>&ck)+ zRh{dC$0{XyR86FCFrn4bL|_$G{ReI&x%{ZsXVV1Y>P-?ao$@ zZ+k=Yqf**GP5eEfzKva`P-Ntgn+eChV#%W>BA;nbal!yaXWhb6R>6|TWi$(%wJ9d7 zE6#LlI{a##H61BhiL7JkAYG(If3}qQr70~I>m2?{|EU+PF6bz;L|{dvPOj$fw?z#- zXedgt>@Z6!)AVr$sM*WJ7pKMVJzY$b9aCN33cj ziAYx~Se3HJL{34DRG6s$=a|wU>lP6bKff=`VWK|s9rc1D(Eeo+vv(bxO%mH#ZHyzT z1@uXvMTvD>ndmtSlHt6ikEaM7X^`)2gb%|^Fkkhymd(F!KnlHi;UFUC0IjJ3VqgLw zkb;JvcZ@TUhDF4GcP_GVBGX*Gu;iDz5(xG~3Lt+Py>ePf%Y!qdeDdMENpRJ~$te;zgQ|Mk;0Yf$&CCQN+&P<2`3 zm#(#snf?3zfurhRHEmzbWrt_otgIs6nz+_RcdzBuz@GTK!#wu-T}Y6RkV}=}cljtI zAM-#BfsiV*2%jiL1-=Js+^Bss{IU_2{PF}fNp(T<6o$h-`Xf2Hqr~&dqM{1PDs%nF zL;L)r);4qO(St}7UY>SSOJ?Yt=%c&dW2>7jEwy7i2}?%<3sA5{F!78544Z4x@p-DH z@tc_sym_~OR72+h4h8n~-SKM1j>6l)Jc8K|KC@cRV>@ZHW}nkuC~2YAMU8n!2cLh_n2Ibnfv8ys^@?STF>!GU3rAnfgmX?z=Z5NpKez}E6Cf5d)gUZT4x z4e~qAobBM$Vnz1S!_hV@gwWoR=YV-er_QYSAL>C<3L(UJI+S`S%dY3nrEE_IM+5vx=Hx=_l7S4M%zcMj^ zaf`1mQ5u=98`U?m5<0HTxVR(d?WePxe5#zTnyyGMblg=e8+UITfW!p@*$2T6IJPd| zZSN!HzGDwwF#kBrqmO1wzZT}@=OP-5{;D!q8=8yB>_sMN(OS>-Esq@%eH#>Kax)!33 zb036drrG(>l6w<(PZBZs(C7^yOyX6MG&UrD92&WfiADh&-CH!@z@g8i#-@ep+S=(Fu6eft<)ZJNvCkw1dyw^Et=Fig_m}U z%TK&sE#qfR7f%Fn9;IBbvLsh_IDbuZ9;f4jeB9hNq%^ef2McG@kK_CkXXEo1O#f&~Iu9rv}8Y zbDzCKsDC&pzi|y8WtfH8r$`jeK0z5#PJj3`-*c9B^ZF9Z!wzCr_Si#h_G+hTVSEy-6X<>iI0G$eL7fnK`GYWSK;28#I|(J!F` zWPY+*Uke+{2S4;6Af*qHI%H*ekn|=T&cw&sslt&H_|WGsFfQg1dW_l z)eqwXb8~Yf5{!R=3(GA3GV6BW2FLvjOZatFT8erQN_yDHWx<=waQEEv0Zl>^{31I{@!VSD@cjJ^VKow_X1u@ zE`Q&sp+ssnHl{0YE+!fP#+9`$_O=S8T& z_%aP^~LA2UO4qPpg%x$Uoiz&;OzT{D4pxSIe`W;aFBg}L#|NGRMRZF)Sz0$ zU9t=!3crqT9C1C&@61(MBEq0G`L*t@WOWrK1A%>bZ zP`iFQ#5?6cl8Ajfl+~F$JvS$S`g#FBl=wamZ+hIgq-ia!fVOIj+Ju5@na-dakx$01 zv}ng#MvcX6I3z2MB!x-}2tFjWr7a%!L`YCnbMJw#l>mh{dN0#6_eOtd*{>;pFQ=}? zkgU+Gm8EMxt;gTKr_LgCtymc>FKeS|oR)YF9cI#tfLJ%eb2r=dBF7_^7OCIX3=*Sj zTxPPJ|5}kN&dze))g0bU2mO3o$GU6~hFAZLZr`WBN5&2jbNs`p+* zF~wtOtYsiSiQ{A%Y+@>2?PuM&ucz`%&u&_+W4t5z!<z&f{C@xFA`t_xy>iXfrTIZNCvP|Nh zDAz28Y?Ox!U-B#^JkQ^R0#r1Ad1+fpLrzNEmZP>EW6m37-QBNrtE#6m1FCQae-x9& zVM$=x```o`7&;$~%=9(hG*`)A)!Zo4dM?$DRzdDtS|_|m#RdQ{0vRA()NR8f8oQ*6yZ_4Et&DaZ7OhE$T?*BmHJqfEm?kO99o92(Mprl1 z1-=l=*L>p{2XqqK5jO?^z+UElSl)5LNs@QIQVzX(S>uB7P;^0E*cLi5cr7S zg|HnP{TcVt55+18Q*G0i0<9ax)lqGo){U^cV+xTO)kPxobZ6g#!1&x3t|V^YK}#Pl%t0}vH_ zwIm#S&QDI>e<$|xSQcicwfAOLT+3XQ2MM?C+s4hY=T{@GA;-n-0{En|(6V0QG(||! zBnL=x!zh9KSXZ0kl>N03NMVY0@}*wjhJ?56$&NpgZKfs5;M1}5CzNHW2R6}-?)8K7 z$P+znaQRvq3lQ%KcQ^EX-(ULg#_rt4ydE}}`yw>RnQ7*kKZIa5LC0Yv!}YL6x$`%D zAp;gZ_2K&P`nO=+q*SsR65Dk_k-Pw4p)q_h9^mDZ+*n!XD53Dgd?nEh<58WEKBG8{ zKzYOX!Uk&79lsq>9d20#9T*Rilks9ndmzB4B8iqDUde?hDM#*jPmH4NH$ z@3ik;H2@tmH%IYMG2sr0HBu1)1@a1F^==!DQ>BD|L=a`3B$FUGb}%M2!J>f~@`C|i z6?;ONqxqC{tFh+F&@mC-WI2YRcFdy1e!*04SCN;hjn!FDK0tiDYh|7F6?_EsQ2$Qu z&1#XM*3?P-^iejmF0GNcp^}L)lXkRmYWhd{;^&H8zA2Vy5g;@z1>wL06pUI=wS@}& z7qo!@fH5hG*H4NufkSCSr!ClzmF78FM(RMT?wX)kO)Hc1a!H{1hxh0WMWPG`u18Ot zawD{PR5@@qT&=MIUj;>s)c>$d#qD^mg11j|Wn@>a8F3v+lW1p!?pg)?4^frGC25cs z{0y{qLWyG{2CP2&R-4y*QP6VfLEG#tESmrHb7G*S2txQx*UVCd7q9rnliHo5Q1q7S z_sEg8Ww$0>?@K?Qpxt_ZXDc*jgcXE`CW4In9^832yJ`&p9;#h3iO$vaEj&mlg~!C( z^K7H392x-*0HR`cY}S4W{wACN0t;PSSZMF|Z}NB?fvwGc2qZttD=}Y`($8vG>29Cm z<-qH*2Ve=z1N(QH;HUsUancB;x9Xbh^DF^uu+X%TQ_v$C`k~5!gNnLp;K z)99rt#3f5J$(0FlXpX%UO)I0n>RDkhWnFN(M6#la(^9|gU;PuU;U|J;SRa~NenKCI zD!72Uc^=(HE#X|6wbDR#)Ro^~7PwdWVJEDdjm@W*NugG?ppxA*{i5swKIm-upI^;5 ztXYW(5*#Y{a1>at7}I=xHT=p9w=(p4rDXMUZ>tHexcf3vR)KdU+vRVlG4kio-op*L z)FRqs5z~`A!A&BSaSqkVoA)-s(McoBR7lq4EPDH8@TZrlgFCw>$r^yqcclJr=2x&r zgjOj@MJYP7C%Qs+c?(RO2IO0$0#`k%bknstU ziv*Jg+C_hEzHJDW*A~Nioh(V+IRxs?{6oogQng24l{{+Wvct{bokPNK)H#0H>dnv* zH(-72X?`I+WcJqe%U5cK9PWqNe;(ChJcMFS^+j3~MlB(@uhh+s6_H;{jU`Kxk4k{u#@HE7l`;oj9;Qv~ z>2(tkzCdekldbLHZd_euQZ!_ri|aC;kwBEpk02|rL?}&GI0}2Z*Jz2Gk6ACRqz{{`7+k~%^w|w2tH|M7D8XC3JO}_3(tY>T z5I}BX&?>4&Bsz8YO20nGw+NUt(mq84WP~*Y&~S}8*%ljNy>o7o*B0ZJem`9v-gP!) zZX#&t%L7kN2xc#NNAA#^4$(M4XQmQ#A}Q_p7U4EAISkQeW*$PGA$?UI@7W^hue>^B zL$Yx)WTABm7+H$_Z3tU-+O2+++#=OEN{fLZb)5ma!}c$GvaKQgjHqq0B=UsPJdW!u z68U`2JVR2h46e5A{aYNHF`ZOsGNiZU=~k_IQaPr!4IWl<7Q+FiZA1R>FqSt`bo?Fb z4oH;XNbbk&)C|0B^mB@85hD?fZm`BiH14hlx$)p+B2(B>__yvV84+UtNmi#Ow87z9XDlIhqN9J<`v<*2EsKDWtf*qznSg7}l4P`z! zK;D6p^HLNq4PH8lGQf#aF|9zN$oQz=pw3HCmOP$%nGDk-PcB$pl}}6lNnh>0zGtFY6B5F$;8tr{bRAEX{|6(@@VYj3MH+Ex@ ztYQx?N_0vpTjwGiZ&RV*B9-ay-f^ZR_U|cp8s&!XI{?2M4&V+?fS>>n*h7LU%!5LqJW}YLWy&5->-=j1wX+}~O)9wRJq1s@ zV^!kiQ{=>0;w2%gvTZ_Jgy*tNJB}quoMsxGwzGT`=+aj5lb9~<{%hoW3f^`YfsMWN zERa@CzEoOlL0>F)EQ*4cPqx{v6z*&aiMli;nl6>R->LAx0~DWDbd*+C>ZUo%{;eQ` zedck>JafQ2ef%i%yfky0F8lZ>yX=UBteeW$gfoFIkDC77?I(;64C0bS@s%;d85r0l zC~F=U@)i1<(igiNX9!yq2bYBu<>w`pm9A6f_mCwNS>#t~vtg+h9OvUnABYQ-I#J_% z%_^%pFSDpCJJ_PAwDEUdii#hgFC(&Cc{r|?FRMi%E4`*~kT}UI4BzB9k>CLgmC~Dm zc0$O@n_Voa^p`7qPL#1NYy4Ao3`!cR7~1(V>-f~mm9|^C`ie9mCy>aDPWJa;GQ;ot zlL!Zf%6ZG0UzSec@6_tV(ZA?7Gh?p*KnRnNI) zZ*aws=3wu*N#C~gu*d13MPh#$D(7^WcKaFovaDwL&L%%wPw0Z@qnVa0 zG#5YbQF5Ktp`R@~oGpzwhp)Y;mIo~QYlG4(7}0t~Gw^ZxY{p^(N||ZdLu`qOX(cVo+W1>3kYy^a z3U-4v&vb2Ce;gQ4{c=CAYp?ktV`aVB?ZKghpa;P#k(h(U1i&~EgDp>3c;^6%U8pzc zDBtb#%CI;ntCQSULJaIc{A`2`tjOP)C10kI6a2+RY7sn(#`ll#C2J*4Rhi8&<*%#9 zE!aq6js`G$;1#+Mtlk&1=>*)iCCGNn2}$oi3YflWP@7jjdx(`(u;Ea$oYUKjDX_A_ zV}r(O;eOjOFWQkLI|MsiEfVEQ1hJ9Ful~&fHK(S(H93f4ssp9`*mHVWySsOatk?E~ znMPA+A*}GQ05la*RwV2XUN-r0b#$L@d~9eD!10s|=i<9`tk#!|=~iMHbX;G742~ad zJJ>@#0{`}eU;nyV`}~n#-E4jhfWz8H9|K_>q}_$NsD2UxBiQN+R1<&}hOt)x$*C(I z8umLVxghm|i>&vvxQ_gvob=x8F_ZzBk;x*K4q_GOC4#_29st_y4cE=7#Ob+&XEh?K z1?L6}q1r6ej{~L|dw?_$6AZ+m?AFf&$f#c+5-^GV<>-cH6+2_)EN+lF3HIP%f4g45 z!)X*(NWzFg16i$!oLGzKHt_K`eErTTd9QY7Pa+giFOgD9rN*Fv^YJe3PQAULsm>Pb zBg?4^TH+8u^B_uX!C4-(h3o4m6=4se`#)seV{}}7!|(CQ#J17cHkvfHo5r?nHMVWr zw$m7mZQD-MIk~RqUiWj(I`eks?d-kQ{_pSSr-&TMZi8}%TjV0(6a)zN2YY7|H^YG> zqgGw#Fb3C2qTw_EZ&|m!qHL;N)^!LF2ep_Te<_@H(LlC7d9$ykyh@e3GOKNQPku7Y zr90BRU$=BMc}B9@ep2YPpDA!%9-h+mDzlcLw*k3WeB-qJ;zm2vqG`owip=Gp_0)yO z<*3klPtEmRkJAj=&AGC1RH@YsKH24`x%<7iYvHZ6(G&I0R!_1y_eOFjJ6o^LXX{!n zA2lw|k=E(0Rw59i_j2n`H$>muC$C&CyY1(nU)pGsTM1}i9MRpK@VNsdUJ4-EL>0O5 zG`LyTZR|2#U!>bYf5U{#kA*m$1YUFTBVUA;y8jG%p+0wQY~YS;f2{#D2viJcOek1H99Rsh zf4BimTm(2MWCFJTS_2pa|Fs6damlIhC~)v7xQG};iI`bGp#d^_PI7Eia!M*%N-SC~ z9$IcmW*$XWT2yv+O7^c}pVk1kC?~I&B(InnzpU{$rSGDA_@Wa3aszT|Qa{wCewfLq z*(yl0E33<^Xc_;@4Y+<<1KJjWy7rFxs@z7_uAkz7rK^XfYly9r?I$W| zw*#$d1OGq+XTL`O0}Tu}Kiw_tN>r^6BRC>A~{(+)79L%IVZU%E127>e0gL z+5GCKFL3zhA7bG4a(&^G7`R_QI$J;cyMFilKi0tJ?aJou{qE)Z-uZusfxY|3|0)9q zBOM1PhX)UDhbyCp7w`Y|1&&_d{!0v;&rO}5ET7*$o_|^c{|_-Ra&d8SIX!#1Fn=>V zayK)5KRtVYH2g3*`7k&4aMJy7KJjod_i%mp@bK`d3_PB%KR&+w3k|G1J)AxL{rmj# z_%bo|dUo*kd-iR9;ca#8?Q-b-e*OJn@6Xl5pR2jQTf2Yn=l=eE1l>)7?uS9IQ=qp6 z(4TeC|D_B7{{4Jf1FxUfKwVycAj~O+c3IpuV+bOdaF(P}!B8X)o9e=7ec?zf{#5Y) zZw+M8G*C)gu-lwXjyaVo1h)9XT}u694M-;bYYog9Nu@F%(sGB;|V4OA}I>W4-U z%CuCiHk!{@EK0dlueTcPk7vuY)@V~e-kojo`Ppsv`htTK(WluP42FUmNlv!07>>k} zNaWCe+R_wBFrJMQ?TyF3^TM(64kl76=gYoV>P~euTS6z;@<&{DwAd_7Xh-APceY;7 zPwhzJeRR6p!8Ik~bLDusogB{;%H;OMaa*WN)LSu4^SwV^Z}dlko^?Alp3nE^6Gwb~ zyWbwb%O-LadVAC$iig^NGU@u2eEcaV-LY=?*!$uK=Ige@1VibL=ZnZ|p67$1YQ7uH zq=9YVhv!MU>wD=#T@V0@Hs5n4u`SaD!)ZC$i)6+U-SDIGG~bWrE=ehf;IV8qiWP*S zDT?JlvM>%6r%We~WZ^tLNSvEeFG}E*mokZw)Geoo`KCFhm#Sq+14uGXu`tzE4k<68 z)2}Hv&a_2JFHV)1qd7`;Qj@l%5DwO?1aq_Ja|-))u4)HRaR`v8);p)po(wQP-v(%{3W8VL7HXL7oVY~;*K|` zWq0oGBD_cQ{HpUFv9qQl$Pga0`*fJ5(!Oeom#MLx<@qufx~kX#&BHY&pxxzJHu&&BJtO8c_Bh=zW_5VI zssP6{m`;Y*1NK(}a3t0ZfHRl}SElX;Oa!ysftZ5TPTjTX_a7RCQ?uw&;+^FEN2+7`uIM$NqpN zIeyBlK6uO0fqR{s5blG}E>r3NybuNnBoR$EP@jD(>|32Yi<9nRS6(L>5LO=b0vf&)xwOg_hYj z0HJt2q__>!yTMPvI2U!xuU@rnE*xZqh+}*-26UMeRc0&L?sKQskRx_x!6(MuCLcGv zvsR}nD)`Gv!$q287;h*`=-yG1OI<;1U(Ex2kOupz%a|wwY%65IDD4^1ar7USH5;{T zp5Pi#%HZzM(Nh%sUC@o5$YV(c4Cn(vpm3wWtz5)oy%{zOJN5-fUCizfw+9P%DCnI< znGZY_j{pytl=vK1J};AL89#n|R>LB_ORE9QgxO+Rf2nc7-eAk*+=udzl1gNUa60v1 zB%w{wG_Qp7m^{sK@Qe~1v8J|14zT6o^!34NdrC7RTENll_<@(4%-U_yV;h~2L8}0Q zhO-e0coENQ%|nFaGOW}-Ix4=WJI%NBKRXVr}#-Wo0~LtR_C%#E8FX<;dnq$m>V4i9{q-8*r&A z-)GN=BvWGnX-PYYqS{(7NFxe_#8%z>VPVDJxwe7b{_3K9rvA>~Z)!9nn>YM*% z>&OyS47U6O_t7OZOhR@KE%W6UHKPOg0O}q_=gSb2E=!b|?7sH}8QyhRpDDHSZsFT< zSI*0In5Q*A@=xjE&9d9CNm+*E`|e|krmWGqUC2~2uj98fp?s>c2E_0hqIxZ)j)T@B zWOPeIkCIMFi<3tjo$a70rItJHovcHy`PXUp6t4$8l>Qi}<^og%TX|7}50wPqNnglUN1;3T5Tf)G5 z>q44>xuZ`tuUD5^hdHX9qR(~TJEr9SHrILZTPunLL4%vGNzx2nm^xKkPPjh#=Q7y9 zs;#dr54$$A7=5|+6v{UQ=X~=z;DwJ8$}Sw%`(5}r=Y^j(#<_sZXCC9H zcg|1S>y=_&Ip1c?a-N_^Qn^i`CJno!Ip1iDq^*bF|FH(l_8$kse0Yt$!`1sfp4w7} zHk@kTx84OXh5^ALPlA8;z_(sT6?*Q}Rw>%RK5_5Quh{V9%-`}(x&y@>6h zCGK~k>a#%QBb|z}rDl0X;keQ25sK;CFyf2l?tjs0c;D(zu;SmwMdmy6)1NEgj|4Qi zdjPehH$eRha0Lmx%?vgS8H(GPO+66L%y|Xcf8EUy@x(8e!gIFXutv=nXT?{n&9yHz zh^Q^--i?Sf4B8DPn1nl6Q9W2SEkKz%fKKuY!>GG3b|9m<(F$`2`I9ZrQvei95bA6o ze34$rqTbM=hdqwI zkfhn?9dW}xFqYadzRf-9pY~lBEld=(%pjXu6GJknC@s`8Efnu_uA3XKN*#{)B`63c zGE_1+FfHtEHrN~{JmJW~*gP;*Jv>c4bgwvYamYsqDq?t)QkN^J)IDMeENliR`X_Z* z15V_kWaNr@aEl~iTUfYZnsMJuWbAWfLRxtGQgl~YxXy66;4tMFPQ)Rr>BLG@Nm;0O zTf_^s_lQ~aDtF{ZSpY*o%zYo6%nTT+J=iytPj1s;feI2z1B^coTFTYW6vwAZ5gc|2 zfZ85;)#3-!ACmJD_-oYQhFZT+GWIdd|94t!g=93yJ(|fPdYzjvs|7}+51RrcXcDa! zG6;q6%{~rS1Kc{#WbefGb;rFbE*{YpEKxn2$ODDgL(=Xf#Kt^|V$4yACSoeomD(b< z^TvywCi+WT>>rp!mhdpJv3RvOY}j@P_z+0=)8Lq8NN64ifiY#*_OLCnCwlp&AS2ON#Q|&oF7m_neHF9n(k`ydZD(B##{`#6=>~4x;UlY|D_MuVHeusCNmFbq7;?0X{|5XlqCB=ENl3aV(alUIuI97@rU)I3V+VpW4*R+y zCf^kQBM_iOD>dY;6nip@z#bugK}UmX#tf-WSf8ieAWJ@|zPl`aHMys?L)Sk?J)%5Iasyq_nfRS-U0I*lLz zH!mws8u0j9Do5SO)(IOg2!*FujPwhdoez3LGjcn^s?$D>Lor5ED-)qSMP(wzfZLFz zv=DQ|g2FxyO|+R)uu0%7=^pocPot1gX3Gu)(ASyM!dS3(LAV=^Er%WQFGQ6EE!Cw_ ziKmt^FJR-0_d!*-K?4x4uFCJ`3BLhw$hmIAA=kY zk8T#1&u5n4=?rC_$cPeiYF|{QQIphSMgG+BrO$BuCgVw_qBW9CdGSXD20~|~g8Cg- zOHsY?IeQfZGdx}LBrBAeXeC6YVwMQ(7@g&BzT{V^{1YlTv~*~+8Gr(Dn~p}Co>Vpc zBXHWIAV?oN49{Io5GsZlJT-HWg|`rAz#_%6NjVQ(v9l;DGzlIGT<0c{k)T=m+{_J^ zS6?yX8z8D!8f6)MAn8T{b}d(roL3>w`p;OWq)EL1WIr910y&3_as=5QMWxMq8J4^+ z77+tyaRc<$VE$TF5-U|3>m1!?{ak*3l45>Iwuef|A2i#3ti8$FTWP$`d>t zk}Ughr80|T93;aF>oR*)5CUPV>MnzRgcq@5{L!4-(Oze(tNk;KMOTj=$$@!ZM|LJ* zdhW5W1WhT*)#4{;cJAT5F>>%cf=N3%((WHpdbx*n`021m0m1n}Vt;}?^(F+19 zA5=J~;)dN#$d!Io!ma1d|Ki~kQ6Iod*zT{dGa7LssFeTC4mG7h`6qK0zkz{ZblMux zZ7nEl-IXJ2G{QQXJ+6^weTt42W(PztownLO`1CsiCWirQyp$mDFMRFy%<&vWPXAjD z`$%qCKipD>TI9uC@jh_GagN%I{(-llrFzI-(TQi+k*_0*Uif|oYE_fkbJ)~_g*W=R ziUqS-vCCcHWGKsIeh>>9^PD=_+}V*GCt;b^zq8ktE2I|W>>&gI5TbR8-UoSkBZ7w3 zux}47=-Ct9e9WDqlfj3>lL51_ic+}z>YP?9N;VQYVnQTxK_-{JW!c^9>k*7}1$3Ep zwwnxmvcazWV@VtKPW2hy-Rt`>D*|#uf|tK{PMe7qWZQ51&+n{R@)X9c7V)xR;3-5P zC`QcReRu~1GT5euWSG#qaHc@flf^#1KV|32`PZ-_Ee2f;lveH&f$hUMV0}>{UeAY~- zstC${c~$^E6btIRM_YWL?|kNy>2uz)n|ivpX^2(=HN;%vAT?2{oNy||`yf~tcQYsy zl`d!5Y6vy-S5#(Z))^GF0IR{P2D>`Gsxy;Dgo8m@4Iji1uW*g@h-D27WdVYak~4$p z=Rdz@E_)K@blKG8l&rU(Z1bw%wegse;f5mpokbSj9auf6_&7*dh2=)rBiR~Fuv&k( z#5I*aVs6{>Cp=1A-b)3!9dy0!d~M6LyUOgQIbQMF#o5~Wk<(C*Ql`Fjm?4uL{BoS# zzim=8AF$ONq-$ZGlgy(V6|{O3ufvoeaJ*XC?dp3xcQ@eSwv zcaCqR%_)P2jF;_Yj&4fcH11}MUlG#9s%>Uik7nFf<_?kOdykGc-#K;o(wsogg6?JJ z`(gLhIYifGqUZHD#M`p0Yn-R+sGbYHG^Px*tA?wn*_tbvveV3~E4bbp1D5L^!V5lz z+m+YLj`U0``Kwxx0K>HJl}$wcsqPIF6Ey6%U)MDS@#j-pntSxWZDqapB(afbf5wPb z(_#ILV)}hFMRYT+{Rp(1OGdhT8O0a)ddL4W`VIBk6|B!_YZ=%+mhbUo!c-n#VTAFU zjt=S`H};loTZZ80y}a*fr=rOw~95>3}@FMrb+aLNdP3!AESjIj9qD^`)%WZzTqvxO5 zh`m=dzm5qQBVRn^{=6!vee}43x{SGrB4#$!!zMVs*~l{zntChIeXxZxkHvCq%Y9u^ zXk<5hXL@{C10m0;#6A*kCrf{OBXXaDR`|0{`(PpUd;AO7#r7XMU6UC-vkdEt>YVqn zt&iG+m|Y~4G6*Dc^_eArRt-`+L@ExMT z)DQDSW%{%RSat(rNn~;m)Ey4P(m52mqCD=8Bit)vD0SBGhhd81D7=PwwX9ZZbw)}U zyjnmU>!WZGtXO9Yb~_CL$k_55FIDnma)^HDQOG3%UPb6=qTRejrz_27$F|4Q&r;r_ z^xUn;6&4St_pV-6ZZ+?X+VEw?$lnCK^-(Xjx9vEq+v5FD05m{&_8UP6bQSwPNPOoT zAt;h0d*MW=u$uumhDpUiM7{@L>a>F;IPW>wgxF6l|;{f;h(14lW|jpwI%SZ;ySe5bz}75K6v!Tp85W}YaDq91~+ zalO$d|0f=xVRo? zY`wT!qIU|OP}TOroLG#*h}`EP3ukyzH%?3z(f9&pe^n=$mh7r&0uQb)YgDeIx^5YG zihvcQ#4Rg`%;e!T&8N^|Pb`tMAN51%`H%C0oLNyU zK=QBl*LM*Nw$>x9isV2*1B5+>ce?zo((a zB%S;UQfMedw;{(|eH3x#4^G?fJ*jOMus6y}ow|c%%XDySg#ib|f)MpfAeI-N8*fu| z16Z+@9;l2PpP&pqc*OgGzpDx!P4)%|HaLQ?CDe9abU9$(nj`mVP&~LZ3}Ak&@S7c? z7|gB{f2CI|rj(Wsly{bZDyQzA2p3hSIUxNbY5t3G_$pGLoa}hVRD_-OAj%@h;KRO5 z^x;AgfBJ@G8$}&fS0#XJ6m}0qNQrR@QaSE6kgTgpy9>ZDj0hG&b5MX_s1xW>8qZ&hK1bf^Lxo(V*@wdGUngmUVGt%UC$o z2V;Z!<;O2(r1M~kAbnbfpa+=C z7u1iIi#4X3_74jd>njGy$0k^%8?)EhE16H9Wqd)P`WC(*XP1InM8dJxI_WXTj#VGh zDRTJj2f2htM*?AelnJjh>U<9vOKx-9vHWBE*ZX)oFJVLl@Q+jFM+g=AUy2hbyo&i~ zbpiNK`e(l|c(GNJh(##XMJa0GVj3m$fvE0&5W;bxoTV)qy;SHa1MR2DO$2l0|$#G@RXH@i z1FoxbZ5TeW%YGAP-rNSUAtm9aA8>_>;&nJZJMIyB7G*6@$LLQ z=8kIlEniIHNQZt5=?C#R9;qc6plRoU!HNrlGj4F!U|R_zaH0efw+RjI5U8VABU&4r zFBQd@hjgeKn4yOGea>eR+CPss;Fkel-p`pkPOR~$lQ`yQtd2j|m!c@t`ykY*&z%y_ zMb74eD!W<25p}<^3qt+S5FG1|GvKcyt{t5-5t9Oi-*p|Q>i+nmm18WWY;_iK)+#ej z4NYhKZW1NpQ4o3o)j2f|rIct=*usIOR9=?^v;foEJ77Qzx^qM}R!p5B8!4S*tl{vcgKDJ8wJ#Eg)FwA^NaW@0(afdoow!QY?Jg2l3kr0n5^ys zUen1Tqn2t4C$KvS6v5Eha1@V_u{yFlLdT&D(Q- zqThlj>!-zGcp4>T+BWO`+GPAeFY~f~X%yJ$F!gmkAdv$P0a?O{uMWaQ(}f`?d(6AN zK2|Jmsb7D}&SJZ8F`zTxO`mx*@hf9tm)dz+F){E0r&~L)GF}*%e*-)gjKD+zXPc?l z$;<}ywR4>cPJmOayaVA$dW=xwQsWyt`Ux>V&!gwN)jT?Y#8r@G^}rP%zZW)oB9)RP zteY2C)?LzRW2<^taP5RZE?UgkUEpFN7?m}U%e`+FzxOCU=8mnGIlSayJ^D>Dbn%Tn zB%!UkjP^|g7JD>;ai%x;)3Jf)05OU%G_W?yt%t5u_?*BGztf`Lp;5SJt#97{1TGWiu`io zPr^McCMh0PAo330OjST_eBn*)KgHq0WAO$f!vb*5i?H~eK03>lY%Ac+X>R;knh>>n6hdN}ppH#%I$!9ic51WuiXM#lj_$^JuP03kDE!h!(Qo&ZaoSpJH?01C$K zJkRy3=!0aB40@~KFSKe}LFO_p8w`=QGFp>>&nKi!VsRB=XX13_gF1Q8%Ie`fveayu z%t@VK66A%PMgggjKw59nqSG3Ws_}HEB&@FR5NFA~xhQmfNeg273J3w)ch0$+rd~2G z;;3?>EhUpmbij)F3dYYeW> zJx)EAM};$#k06q39LXx`k1W{LC(`$IW5k>Ug31^G2y3tQ&Wp+qu*Q#Dn(sp-NZXnp z=@}IiArCSs{Nm^B2jWWR?wcz#|ZG$!cz%~?wp)BT9?sYlZ}s( zNTin1dz;SI99WR$ESwdSt7ys)?aB-qW^ihQo9)PX=q);!1fmFX%9b%&Gyo8~XoW|% zSwesErBLs1)oC|!YKd~Yk8MG)$`$~D2Ec`b5#Ou;w4xx1x^5=@Odc5wA%Y+-6fxw< zV9U<@l0gAGl%dU;^wXf|q#52r(k36CP(ewSOl~0g1qBJ3!ju76-b}`0J5D+T7e<*s zI*u}FFc_wwGDR~OoHx@mP8o%JgGmFtsv;}hkA?yWy2-4s@|^+H6kQ+8axHTrf^~hH zO(R!qxmoYYgyraJ)sy#1; zJvwHygdqKF%cmppx7pA|YlPsr^79ta7%;Xx{O6z*f)EP|9O4X+;ilDfqXg!Fixo%l7rAOB0ou=dKU{lZ#Y4IE18GsI zQ3%Ne5DN-e?hVg3vC#+VrC=5RuOth$aejR1&wpU}P^xE}yZa?NnFTO1G&+|^A!qrO@B zMNdoK>QI{iUWZ+AnJ*=5lSgECL*_v=(0DotZ!wx$`?7e4vKqXa6&LSqn{3-9)Q zr~DO%mZzc4?Pkrg4<9fFGviWcGaF4MQ@_Jc@2^#}w~I!x*w|h2%7++XIpogNs3w?c z2Q(2ZFks=*2k{zHZdph8!#G zFaNIa(5^meaH7qw2Pq$pt{P$gmx2yva!12pp2hV?Lq&p$&Vr8rMlwX7YaPUhCFcJxH5MDJXP0Q{;DvH_efO8yhf2I?@`{d$~ z=&lDHuLthw`;Jp4Is#nX<$k0K*j{j^a!X{;)Ist!^A$xHT7O{#Z8@H7wwGuRsep!} z!*!POY-+xInoo6=(Ij)xYK$s~SVU&v@}U@nM`{o9L>leZ2FEZyW+yME?T`D2mZoM{ z#^8L1!loud>L$Jd-PW8Ls3nIQTc#eI-yqra9X1au=5?if4ZQCPFG$BghOWoUUvW)1 zV@+%`9+FK}9?gfrEoev&A~}tJnjTKTo!s~zlI!lF3z$y@Oe-aYr{$eygoNiv{?6J8 zhvk8T_Tc&c^-zHGhj)wFJG138vC%>IQLLqI2@yu}6+uoRLczKsnk#y4^(wSBDdPoKo(tbOI@xOk>pVexzeL7jAXQFNM~V)7EeL(tPBWQ-7_Mp-?6@ ztT`q!ki|2*1Z<*+%q5PjdarV4vbXzlwjKF)u%0ZPp?-foGLtmF0C~hH%(&wC2sZ3I zSx<%N?YC_DD4u;e#DZBkHxeRQl$SMwzi2VYKie`*YsW=P)F9y0;T6BwSh`FwZC|C- zx<@&!0Sr#K>cLZ1A0iz-%hMe#{WxA~Uy#@RsM-FLW1HuAxlFXF_o4^lI9Y|dUdOVl zlegYC*Fb`5yJtj}^)>Z0>17B$>c}$_O@Jdfu|R&dC}7q^0o|Z7IiQEer8)u7Zf`K{ z91xWpuwxx?V;v~Uz#GKv*SAK)Ozm;E>=N!S=aTIZzS!8Vb@YcB~TG&WMq^YhOYfR6Sd*3ZNCpufD1B%!87!y1s((3UZl zeld=Ra)(q*H|iz0fL_bAC3=EuM-ml#W<9exqI;Ow>rs(A&fFcb(rWIXXTBjtkr0euT?fP*4wC&^E72<4_@}REP@9Z|ADN&)NP$hp&b2C&=W44tVeWU-)={u(j zE%5n+)I!(HnhM)ps&`(a*YecjVSJ`uoO=2EL`!DPMe=Z`$Yn{@3MrfNnK#(;0ntUY?*jb^1$88g(Z$AagwQEeaH z>JCHpY#rigg8#zOYiTF&l49YOO1lnbt6YmT&K$0 ziM-aJ?>rpz78^kk-fHjOl%8Zg+il1mLCxRIK<7#c@9X(5rQ5vpGZnMiLsIfO`ixQ{ z^TmDI>;r#t2$%!Qo|P2HZYpdR`*}L=yda5r!4+K>K;e z?y4i+MRvCNCazms%pGp~@7s}wdZkCuVP8D%B>t7Og^5D#i!U*g9*A^?}cHi+24jV?_(}Ixo&pKwCk}1{wOhOP2l;n{?$r$ z+JS`nv<5zSzc&t|)pSxuy|a51;7M|?=#Y9on6g=RQO}9reM+~#N}46$S0fs> z&W4)D@kX-wTBMCExP%#%4=tHiNegeI#Lb&emiF_9ujh&1ZKV`F5NUpyq@wzSqe&30 zN}GC_vaVYiu|wGRH4C+oh!?RvsKAtg1Nm(0@Ipq%QLY)J?qYZx5o7#^+>86(!5M0 z7-Z?^4iEUku!PIDxY*yR{}mx2Q2`N^9VH;oajdgB|JF46amg!#aBo5d*T-%U{?F%_ zMQv7vy^(R;4RZ?$yO2X!3>baV9MW?WSQeEk0XK;vUn+2fU}C1hfxI_`^=M29|1QBs z?W3K?Q~6%Dfz!%^Xqrbs1$7oLy>IsSM;N(SQ*$P6WjAYlKjigB#qj)k1e0x2b{>8% z@oJ!}1LS=Z+siK$k~nDDRTDk_lQnZay^po%xru@s2Ufhi9F)v!iE{2-A@h4$@eU#eh6EvwOpw@`swwJ#%KMyecAn8h1jh#@ zDDV4v@QqR&NyC7)Imr+--c8^^eRTNhV9rz9vuKBVt>_NAJ}o6A7ywTk9%d%5{Gzmc z3L~FN(_IdnECrXvb65`AD}l%z!EjFprE5V>#;g-GjNe z5LD(PKMIEn8|5Q1LOG%nWJi}Ghr^SAMFNx_kyB5P$KQpr6{30G&BXeq9rGm|%r($@ zNI{|ygJ&8eQ$@PsTVEMKlPMa#G0zUV4p@7s>x*DR0i)2^1!2skLLc-|Na`;W3OlND zXueru|CS2>w%inCfCtlTkDBDpZ}|m4eIxcZM3Rr)NrK#t!;ytH!(E&ZR5Km!Z&TwG zAX*^;oY&|thH}h8sMaLA<{4lpo-v~-KOmv?kHmpe$)j&RmBMVDDFxCMX*v}P@Xmn2 z#gpntgNjTTcj3h1ZIM$WDh!$2EC==NcQe^<}8l~}YHH@%+l*|7heeI%E{noHp6EnaM zj!0*F43l18f~8=3DIK)pH=fk*=UGLNDRC^VMi#N4=*}-87iul0+WOQ;Mlac@0!8_#N+>$V4eh7y(t|D4O|fjbYe7#}@9F}Wvh)UM zmaNXX$`(vA;Cp!KQGd7+jd9oTV+qaHBIbupUY2Pm^9^-yV$+dFOW)umvUlu`rZc+C zf{=QG=j?J&yC6*H(GcpCyQfb&)_xg%XQ9)3&RJ?2AxW=cj}M+qY3XTsxd}i`()(a- z@2w5Kq-q*p1Yxy>Q1+-N+yx!x)YqZ!XY3-CPSq%Q+d8|o1)QDTjL#4VQU3%& zve^p9z#g5-EJbnSEyp^XT;38pBLs<%hK>k?*g3lsqdnrS25L+vLc|F*{%u)m25+&n zF4oqM%sUo%4ls#&(Pn#^ZWy}z|nBWL->|Cbpd*Ed^UhWKn){Iq~$$i4FIO3_yrPD%`c7?@SQk?7}0uy)40FoEq_WuPKzgEwJf*0WA@~oE1iO z`49V=*2Q`peA=d`yUMNtv2YD`3Q_J#p67Q*S6wteKYIN(jSgS-N<(aMdBVplM$n-j zRy0@_T*SxZ+&F&uJbkrow*6BYF0s!( zxux0LFaEig`FfBo+xh&NSD3ipUG(>G^9n@?n&XjRWAb2Ac3@+sVRg4aVEUluyiXS3 z1%nu7PUNu~|G9eNDOg7*i{NoH2O%N?Qg;W%G-A&s@dlOfwp;pR5u~q-5mH^02PaK} z8L69L2D%G496?4W2V??QMl{_CI^mk! zIs1&lX#s#Dp*=`Cl|CUxauL^T-+YUB1IXK9 z#g|{noYKjqS!bmR4t_KxNY{|d4wK6U{NlTik{&;h>p771-3Qk_kW(Vx0vy)v=8Hq+ zibEwxL=-8X?Q^moDzgToFc+xEQ~WTTCQ+qOH8oKc@aK*z5UVi!nqv6oQ>-9N#=o?$ zBtP@LhC+)d3@wTj1{REl2Mp$RfXCiMO%qikpjfd&LG;a-vzTJf;Xnh+^tPG8kk`bJ z(Nx!W1Ng@T#ZWE$@$hrAYXa=ZAg$QI_9)h*h@XMA_oRw6QooO zIb@++08nEpO;g$!>RV&s=lM`tTv6IR9U)HKQvF7yG@GUb;%M1WmTJCH zI*XS&mb&?ty5V8D$543;BaUZLd0LuyRvddan32+gQ7{K{ z%2W9)nfZ|VS>%-X&ZU`T_(xtItKm`kbzrUt7SUS?fl zVx3bKb9JKGa1zNX7}a8K7B(Cty)?^U&Q59`h6SRCJUr zO0=a(cFUZJlE(xl(I ziFpxF>(h-!XQm&NrjsB7b8A1(54Y;py@=u6&bXB$(AZ%e$V_7-GoQjAfTFCbGV@zh z%jmIUoJ=Q4rX3!pFW}vF5kNq@(5`Z^47aI@UCVO!h>v>hv+7ZI_7n0tq8#FsiVhij z+bru^OPv(A$qd0E$cn|jkdO4JdcoBG%}5t}LqA4axkmveaWjh(7dx*?AHUP*AwNAo z5V0_9liH(Y04unJXI^`l9rH`h_w9n9CTqibc?MW9P+twc0(8O&Y(g7}L(o`fq%qT| z+K^|o>%s4t{QYxOu=nV~G|LZA1`>pNtX}3F#?JbUy_)S2O|#dH0`-~RBWRBIsQqca zg7a6p*rPc-x(bI(iCMUS?-k1AOwBV@p644O29RJTa@_hpBxd z+|VJE?KnZz^Z*$;769xWbZu5`wj&I-MOpqlP>11U6b4C;YKIDeTsxg%Ha_WxfcHJZ zfhAjP{9Tp&@1EaL2ORt=ZJdKm7mGigz4@~|3t?o*%W%5e#}5cTC{%8G)Vw}$dH^bl z!&u4rWx3QB5wM+M08HqO0vt4u#Nis3vHFLd+3qz4wIe0fENmauUWGMbXhRUxF&;$y z9|%D#tJ}B+yR*F;CW`MQ61DbqH8DedU=ru?qZp7{j&}|z51P-;$9)`np^B=NcXF-HI+-lWgU+(g9EPcjdglx(e3iP}`35pZh6+!$<{LV0 z43=0Vrtnl+KkXeI10IdkI845~x^$FUv{I4ha9UQnTDE>dcZXJ;RJzi8=F2TE=KI!P zeq*o~-yS?26N7&f?AK;Yuv>3&__o1t_vFCD<;2zM^o{G}*D)!Dn@&Q4vl^GPS*xoZ zm)o-{jsBU7TdR94mq+Te3IqgzKgrsbCU zC%>orpZs3rGr#8%<=_>Q;uTZ4FYi}-?-`Q%m)}R^ z{mb5mCKZPzroP(y$kgwz{5~$y?N#6>q!*?ZmZSxHr4==26s2aC^<>r5{uhVe)L;I+ zyrO-kvT3}wETOKcrLMiNzNfx^aO@w7KiD$xZ;Jod;lFD9$-|DR%g*tCDE`=X&tHl^ z-1}>xcWURqQ~bqOivRG6@Q3E_hv$xmm+nVqw*QB~zaHIK9o=~O>+lkN&?n{8xnk_-};&2ZLI7usXNL@(+aW3vZl}SOgx6)vv*tt$qiRkKz9Z zgfCS`)f>w-z4N=Yu2@+2eE<@%^nXJ5b$&wM2%-)-ia!-={u|*Jbd>!?_>9xlX47S# zmt*MF8XdNL|04XAHs_c(3nNY6*MMLDBK&4n!=6xd-2Pb2*W1TXypEKnIhl_5;**vJ zHvL8Tq9croQby&<#VY0Eqh78@^HmxGIe`&3wF}M8J$iCXHs$OvF34*ii1vvkyCSgl zt1>}Xd&6qCZa3bL{iB6C>v8#XFI(01{17OqLdTc8)78;({qa@Yg4?nZDtm>WzW0Dr z*u^#duYyqMp;prh1dL;`u8jT#e%*fyaYEZ3rT>=H7gJr*(D$u$iJ><_XUazKGcoOE zC}napmKUi!LY@cyL&~NvZ3jYrBzF2=gujcoMZ;OUXB6`uj&9ph0L^kc4&?=5`<3Nd zngIF4>2?x-*U%cr$qS}Y#S0q^?W7ox9PA`F3rX#ySrjeKrW!U5nI@QE;nT!B^lR-U zo0XOBWq(2IE6RXyr;boNTWRgby2F*x(uXlyk^1_G(;pNtJ|9?QL@_(jr6w~k9u(>O zB9)~kCR-hrz76B2_bjWTHz}$jhf43~et&K6l{N2LS(VpxmswUgEwv6)ciaqGhkG8D z9oJfrw3XF0MtbE|_DyM?R4>>e(bfHS^QNfJG4s-8TueTqZEj==uWEtljY!pRcrl!{ z!&iBqhEJx)F$JEtA{BRNahIXE+>S|G*IrXsoSWa2F&umT8Dgt$fLuQ2wQX6oo%a(h z&sy=nY(7@?6+*t4I#NkzKOud@ zY1Nu5lm))F}C{Uaeb;*sU$q`fM@9m>?aij=z6iUFnu_d1d%=wwIPtlG z{KIkKH3lSX^)^3^=d4{Oiys4mQNgm-l)?$1W@8S4ul~@oQI!IN@S(F}9-xHqb$cEU z%=F>xFeSz3v7zs?@5F`RcjybH7E%v@syb^pxHz0W_Jnh>Cp9-+JPb;wsNP2hV{FCB z#TIsEp%W2spXpJT4S0)HOC{Q&j)8%Q|K(dNo_a6fF-9-@*15Jam8$2@iJs2vJ=v)S0o zc4Hk2e68V215#n4?LsYe^C>e1sNY$9507czjHM2403QReX079}zwvfY-zz0hz1gxw zF_gMCtc@`APk^WH1eTV>hGnJ~FtZF$Ri1cv-9 zYPJCTJ-jK@-$xzoO~Fx-5Z`V?(;qgEB}#I;FHLRP_l%(j*BM^dp%608cP_5SQ)__y zrp#xZ6Y2*;`j$`y3>Q0EnfBVuhTH%dy3 z_ys`nGEg~>O8khRQl_gk%PKZITZK-6Q|-UVQ~Y3E1?ijpEcL+Zj{2wcHT$g7l~Aki z%32+pHDiFBq9MNHTAL_#Yy?BFICg_r$IfIU4jt_L0f$@@y;EVXkwHBnGFzWYP-&!M z`Jn^a&8=E7dXlC)Qor1<-smA_I|xD6Sb6=c#%-Qt^dIQ(e{O8KR$|VI6qt}%-zOrE ztAY(9EP)SKX$}dpqb!xt%j5;q4f39Ss`vgjT#M_Uw_~?4T+VPT-$%bY`rVi7Qcu|L zwqgd$-hPKr<{^6+G0GC+kKMiwCvew=3tL`mUe1!_i;YW{c?dZa{E%Kyb4woEV00P*VOf zlA;>T#b&$b520ypPQ^TiMCh=i+=aQPE}EA|%ZR~?Q|bWYE<259-vb_dT9S5uoAnOm1z$(^aBcE!?>6t+{X}TdRp!-&*?XFY$!O{jo|*O- zF$sdHROY(utY2&R$qTg`z0;whkCtVy>3V~GXZ?&m2kLDPv(?d@<;HSH+D%?_jbpAA zJkKV&H+)>GMS<>8k4J%MgbmA^E<9`GDbGt9pPD;DstV;!taKiiCOaA%n<|fNoFA9h zlN-Fy;{$+{XdL_UP1iG{r*3VJE9cQo?M&DWFn;;e8kRD367hPb4EcqwXic4#CntV{ zPgAfT@47r!EcKxe3*3>-z0*NEUva!Qu|w`uU@KYQ2`sOXX=rvAuh_;jWUf;~6W#PaMRro^j=I33%z{^FfL^}~P8!$FfM*udJZB`A< zwW6uzDjT;cBp++tclN2bq$$eK5_!I%uH$u+?DND@bk^I)e1hh2@l$v7sMW`p+fH-w z^C;mb*JEPr+Pk*Pgz={xfuHviUwvtf)zQk60GbuO=^#Gq$BLO5Nq>k(Dx7;I=IZ4kSJ|2sSahS1P(kBF6b zVXb&!(IsDaG{aI-!@8Tpx)DOa$?>92cEYDDqV6mt9v6`6 z5C|Glqn12SVZuku-^^*t9&SNhpGgn1SZ`d(gz{N)oKzC3GN=qTR=0 zq=jLJc>)Izgw_oxFidI8(m(~A>dudvYzFc-smfUnL2IRT`@dZOghL99w4O%upMbUrAE zDW!^uor7TsbOe*Zair@;LJp}XlIV&}iO!AjZw)N&92_nHuKW{#rbQz>sZ<(7>oPnM zZbYNmP)aKvp=myvtVMjFRO~S%S=%E$8G-#3L}7n2c-DqAaV$CMbv1oh zbvXd6Hb80aK*R&AdxRM4ev8t13#%oBeC*&|xR}L?kUdBSXGE60Jrsf9k^QOELn;>B zqX>UsnV2=064#I54a|fkgSlvlRHw-azJbvlGWwHKKtyfXvxpjAn)G}P?6mym%I7d1 zhLP-~z#f=Jz9TNNCp+ztAU^d00t6XgSY1j8saM;z^{Cua66vnXbYQxz6ejczWaYlr zSJM&dTws0D^RztDT?t%%)}w3ITl&%c7IF1sqbcDN$P;-L9Hi?C5sYG8OptPgyA=gsK3#xWl(#ut zD0QMhqD9vqaPo>9_tBc?6^Q!%1DE^->B5YJK%zV_I(6xs^wBB+fDAmmIV)nccH{?s zSG!K!3P8L!5Pa$BBynhEmjm;j^?R;YA!=hv@nL>b(Rck|@D=^{NB|r@9eCirAfxQN zju(bhFK(SzT2~NdEP*7JwOivKdO$D5%9iB!6s^_(g%&cgD7A7k6*~!07{munk--Y) z<}xqD%1QoMD=qN#C)hexms@(mi|8wuS!{+`3?le`LY{u0kHUZh4a^JDGr(&V^&;Ff18 z7?NYH6})j5Yy1`v=Ip_b6eoe47tUSq@M@X06E7 zEgU&!9gCS`o{h@=6vnadC8nzhttbtRL2uSU=AWG3$Y|=53%{6{x;n`yvC6Pcw7or2 z#WpEMmuQA*0ppy*s6r=dy@#8N7DFpMTlZPBuU%jYyGl|7N-+SiZ^Y`ej#_rSYzt)+ zS|?2L?(0n$Tw{t78LH#9CO}BsXeYydv2KRO%SNyn-{TRJkJ*I_XAI|-i-?z9OowBX z{%{V47ZVAn`1tYL47W2n^YVTAVJ+}^fj2fR_G=hfN5o^+E<=7tI7-K1M|?Q^#|-ep zB}i1}r^9^kl)Et2twGnJ`Tco(dj!i)vM5i8 zLVX#zL4$z*jI77w=T<*Z9VPFHj$61XH%g%jT6NLSC3Kb~WKCx|<4jHa?jQO1J!=f< zAGCV%9R_`xI!d7ToP6bV?`0!5((}JV;DvrCS4;I~r+u$j&(M_uP#u)XN;bk08U|<; zC|mWd^v4%SgNlCJewJ;Tau$F43*To>eAQ8~yS!YYTXC48X)eDBmk|E>y74U%i#tH{ z#DsGeS=zr%&a9=Pm1dAidN9apF!T#DA5lRyBz?8H(ggy85ewWrhIwt;B{BAjd;k2t z3zjeMRU{lXVSr_-2C|dEf+LfVQ!Hi9frLUJ`JB=kfgCIw(JrfY1#F?7<#xaF5s4|; z^7^h!01ZAeX6UEfUNgebUJ$?Y1kTAo^K7>tBQQXA6wWhtuKj1ekOJkpJN@VEgndmu zJebwP%5CWTAwP$)nB(-+iY&Q`w)6M4x;PV=t-Xo6sHcN{FsLks*m#T5QAhb;r2yo* z_`CQq};bnNsV89Qfg2rimYBB#K!uA{gM6d`X`#Mh>7- zR2e1ovVMhHg|aItGMnxL&|JxelAG?Y`ze^R4`6LmI~y+TMZ^WifnsLHDgV|vYPoFg zasUf`9ayN&DQlYRAw8gT-m#_Us}Js|r0Za6q;_k@`Q5Nw9iWQyD>6Iv(Jtss!!HaU z?MrCB%1*DuC{eemIUIq(N3CmL%%m9b>+r$RbbBRt8A5o{m2RFOQ7SsOyrjS+}3ycm7}bQjHK;?VE<=}qO`ENRjtvaw)Au8zTFdHS{hq31vZ5yS{N1l0`3}bL{p7ZuNRo`96E=kR zd$NoNpAc3&U$zzw^d=EW(&-S>nfK2n_8T>u^S?c6>|DJ`x?%J!hIrA=xJ=JTdJ)s((JRK->+nJ04)(8j;7H>{uU{-#fB1tJXeJ zS~xIHJGnePnIJm7lRIQDgM^=^4+%!^3F00I&Qxb_o#-_p9tED0*fi;#P|cmskeut* zWaZlJ=dzy8|Bgu3J%ut~=&_yC7JrM4E+89UK*i0%V5zMdIA8HOxj;Saiaz@djl5W~ z;U+@gJEYq?(Yq8Gy{dUUoj#CExofL6ek(^5ldlQnFRuP-c5VK#$!z?jrt){U`5L3|uA@Q#H6J5l4xP)_p`h=? z%!2VBZ|`$RA3p5e=jlHTmZoIr#|)@?Nqjt0xjCSFbbEk5y{)<#>b#cwzB`zGk7Dpg z`&(f6^P?V}iJ-pO!Qr1Wg%ej!rihr!-*!jUq#?EF_xe?cWqkLK9TBbehrglrw>tz+ z-Djvl$(`LFUpAng-i;5C)dysSqoAal7*goWN3_k=YmOz?m$Bz=;uLpujOTw4{f!SC-z5{V3x6X@b* zJA6=aa2O|$W);2(2Z5-x-^?rKOX{HVrmN1Xl>9~bNW+I}-#%oh_btsb%$ET>qp4)n zE2b1IfAX7_&)LppIqr-V65dL{AbMPBuP@J^t^S0-!q3SF`5E?xy=7tUTyO*p0w@)V zbQdp8f59OU2{%=pnos1u?U>=I+qIlIN80YOT>4|}$RyQMl;y<*V z8gxdj*Ph_0(G|iXp{Fb4ygyl}@KrGD-aZfZll|Tcp9#C!M0dnmaMaWN<-x#r^Y4OM zK%v1;qaW4QUVNVJ|Ll>HnQMNNy;-N|6b8}73V~5&9faQ_4fVvleylL|8#}D27C_;6 zZ_zkoRaePruZdkFie(<&Df-6HE2ldv(1H;H)Jt>}C$O$(>nG&EHjp6kzP^wk3kXz_ zB#+>9lBA4Lb&?`-cDa-So7or;=ym1sQ$`Pp)NftqDHytj zdSzLbXNm+?G{}{l9LHO(nU!4wz!aY(v8$RIOHgO4m~mlIqB3SDtb(p6@-!Z^_w$rS zjWkcX7G>JXDMp=F9Ro9~=o+OFatz>%M_l5I%K3Rq?2@Rc89_zY?fo>Z?Zm{D&n0=B` z-!ld^q-q*PCDrSj3cyWh3E<;+_BhrS&S`tfx}lodemv&J)i+7P-}pv!hbt1}td64$ z5bR6S%KAEYKg+|7_ONdpCg@>SWFFLuuhV1bxE)8F4ZYI{Z9%y*s%q{`7u*ALk{uEoYn8^B(g~FGfEie@l@6S6v zdAA+!WN?LF!--h+`?puWZv~6107&QVg~nt(cA`-1=G-&RW9x6f;Tk$8F#^;POx~%l z7YYcNCIfA);t#HGWe_ zxd@LY-SqLHf%eX8$5!y4#D7eL8NV*_aodLnWjjU&%aHANHY4%&Pcn%lZYGgu#0iPX!oDHwOqAIe_M9Qklcp0W-_$}Vdrrug6M5cqHy z?Q3LZ3BO7i zm!k+QIV1($fJ^MN%>r)2qH>B`-B)@!Yay5}^#+`=q`XAB?gcH?R$dJM^$Q4Wts;u{ z40dr>;y8ORD@@Oqb@;DDCLa~b7|DqM(8?|fdOfioent;h7`46;^#%{sP^V-tDS!ci zAx$ZEVi(yfu1ilEPHI8qvg*Jfbj2*ZX{b@HsV~4<*Li<9<+3qqFR;**qhNmIK7d80 zB-jAHTt!&iU64}oml6Ks2cuIP5msqf#c6FVR6`JoC7HeFNNng;atg9)_uC`&a|q@{ zK{+l4L!(d|rue!k;*&)`R{L=1LRBQU)H53HwYUCEELV^jaiPMBbpPsuTP7B_sqE3i z@SDgYVlP$G3}b!XRVA&JM#y@yfiN5;R1*9Z^*4a#b^8TSkUfcQGdPH!k3~zUHoPh<)@Bu_ ziH+Kso`R8AEn0qb(}g6UD|O?(}0 zkRB8;aOjUfzqNA4{tK*VC7Xa6{_cj&zm>H55<#UN&3PZc)A-~%(Xv}#Ik35u=E;fH zn%$06t5~}mouYrhy3N- zpP0%I?QS>=f2b5RxE4i&pJSK_MwW)dN!a)Odi3WTYH#A>et}ip>Cb#4#F~oc<3bmZmCr%>kiE z@KtTaTJekDxgi@*gDkgg_T;G!ICsgqw%}`d$Ar%NvqlWCNd*%Efl2#Vs0M7unK>AS zHwkL-K9`?}KoS!W?>-2*mmjK8eUzkoO)SUNn!m5fP#qzue~*bMWrUDI#s4pbNpU|fI?4W(v{EQ?y`ZpY)AwyH$p z{b*gwyWheM?Not7SA|cz%0D}9h(5MigrkNH5o?l zC*79ljrLJw00Qek^JOA8CsHIKgvQ~6bUW+s@IGH}2qyr}AT{%ckuah`)dodJ5e1;9 zybC(dAf^&1!7kJ{)afVNDXQ_i@=F}2w2I{CPpH%tWw$T^{RVAcQpH2ILXR2naT5G_ z4Z!N>6bcWFSi(JlkMzcg-I#}MNNFD*5=5mKAq1Tt{Dj;2iQuR(dSOdC)s{Ll5d#vO z2gC!(-GyObh`{JJ?a;gTRS3YWjNlHd;e~+ApvC$Iijvhjs?hr%$m?|u#{3=}L4>sZ z{L3An1#uNgPN&75b^M_fdXdwKAA!eg`QWI=i|#6VabIKxYD(!gZ`9EII9iV8G?ZFj z4nH)_pEX{;T!f_GE(H>l_(c-(IcDO!Fs9SxaDQ?ULZIk>Qs*5Q$G410SUn2m zB48#}?2weiD|*xF9Ej8Z#h;`zq6o|@vH1yCw+ffCMr1gK+T(-U^+?GP(K%znad>Bs zu#n;a2H;YH6kyBP}4&OA#V=qLQ%lR9xq*H`Xx|jW{Brz1H@tVqY#_IVpXQj z^1I2Dug^ya67grzY}!OQ_I=Bt z9Ya}Z{iIBn64_qm@@X(W(j!e85Zg}ODhCPlrxeCQ2pTpA2J;%Q=j7L6t5Um+5NKET z(?{Wj&kPEI0?Bi#@G*Y%A1Wz4t9nv^YuP{~l*CKY3c+y%3|d7AVairc24pQ~DB&>sy2SIJ;||)L0AqHIg(K;tRCIB2Ky? z7xchK?@_V`6@ZP9rCPdimWrvAAMqj=38PxIOL@GErt3BY{AIwmAPuTr(I7xJ_-s95 z+ZQYDs;bvQkSRh=u%fifv6uj>mV`R%y3*<B(`@=>vTE4+WO`MC()iV)poUCSN zthVM*jsGB;K>f^)|9R@IeutLMO=XD0!;}XT;;|{$$*9(;n*MTW&lMNOZ?zwYR8rnE zI-8?wza>|18;TJ{7L$G$MCR(lRk0%6M9(7`%$j}%!Wc#hseqW*xuaHdTc%y*3|4(M z%%t?$@=MWtNdd@p0bIz^M2`UJ^Y`%OZZA#jv^!?n* zC5d8flDxUKCcOpY+q?duTmp_G?ya#(KZj~ZaY1>YypNt395^FPS8f&m$yA^BMv5+0lu{4YK z{rhWNB=x?f#@QRutH1Nb50M&Y3ahML1HjpKR+o+kwlC}Ru{fRLMqOw&@c{N%a+_Xx zn?CuTE<2k(UmF(WquaEVsH)?JP!;k1%?9=LZ}NMHetQ{>#|5$>#RnlxX3NddO5gl<9b}!{W zxvDm1#I5EUkHqGzpTleyiwX;bHLQYKENvCJn)&H~u1*U_}c-EboW&r~t3O!~Qx zcXbibwhoF;**tA$BkZ$T;o|^jDb2s0Dr}EXSy!TM+VCK4wpg(G4siAk@G)`VM8;gx zgCw|k#P1GBALEfz9MHxc&<7$3eH<`J9pP@<@1pJSP?jGW&)UYU^@$0dZ`w}PdmIFh zKkHc#j5}I!>alpGudZ5LiQBO++bDEw<$XTebkiEX+4|bPG@ZS7KeqA7&h|TSvo6{( zpx-el`vT~`Da{PINTg_+ZLgv)giXM3YP!@12{wxS@Ui)_%Iv?iOC!#3I>a{X!2)_$*UE-PrLUMwg{s^LV8^AfKws&VzJs6zM5brI5An0 z78`}DQ3#zA%dPQGSC@f+#^_Bm9%e~W6N|5|@r8d5?GA%~)Y$Af*-qX_Uf!k`89V+w z&eVH-1iujxay_!U_3*u><%x0B)Pv#|UhNj%IaA5XkGat9xk2_!70-73NRb#6+DsBuzNY$aCLrrC0>fX(@~A46b=JvT2N9G^Z*A2=4Uak=fmFp>a5bC_wN;D5du?4`GlnlBJI5j?G@C;2A=_2rU1(iaa(d=X$XH3Z!WnM&j~D z42;T&8N*4)Nd^r_)?imHO0!JHe)pM+`!2`NA$ulU5x7Z{B#!$(;Vx8!Q5pY$LqO@Jd={i%@Bt1QKsS=%YkjIjt|D~zM;b>)Yk zx*-^|+HJ#_+N_yOD9V2hzM$ZT^ir1lRO(h>;$(>wdUZPxlZaxpmm(l>f!KSuel}bp zBV#W~4Gu4fQd>Qrx<;D-!jZ<-Ur}-qW}frs*%3a*CzA~&_iPf8RO!%;O{}ASQ|F8< z*r}ejx%m@$D7#q@L)CXwc+V8JZ*Cr9dgGyYrJ=e0jpVGUxn*T^xswP}@@xQy&LP20 zLvi|)W;f{BPo;}P*3o9F2(zRauCO{Mp6R+{nC{-5an#ANe3OQai%b{$nBk<#?v7rz zUTgk51M!atxb3XJ>g97m!ypq3@Ce{BOsy?wQ=6-be9&a?8&BbilbTL@WG(7*RU5Nk zNVm7rg3V8EaX&PImz-%X2DFqQOyh-vU|oxccLI3c+S!BY?~8TGRz^rqTOhelty)Be{ze{TKtOH_oVo-1;_~>& z?k&X!Lv@U4Z z8j0jeoD|OtFd6A7!FIGO$omk&Gp%yNG_woEqvPY7hv_ zA~m>9U;-qF*91~C%OT1#wI7|awv5kXU)+Y=xb@@+R;Xv^;2>ZrAvAC4p#@4X2c@tH&u^u#6pRkFIs`Rwr%dZ zk4%amO-**cdKV(KzIC4;V zLkT+t?w6@GLCs2qszo%Oc0zF>8h#<6!cg~6FqWhg9O56i^JuZnE;x_(LX3TR7J&Sp zm}(7&i%#Dh0BtgZUB&a;?3M|Zft>wSBP+iyOvc-`Qjn;AhD?D>D`J;EOW%#4N7N=- z%Iqw&-wkWc?Hz1DNDn};td{Z#JUc* zR%7Z*pECt?kKD~ALaxSW8-3-XND}GMN+bv336Hu`9Dy~^FUR@ciMOF;-hvZuy^rc% zw$e09@1=-oke+Nz>Sxkcs=7GB=*@L?Zb%)F+b7rL`IVpu( z4CcD*y~IvnQnPJJ8m^`BteXMeyNzhQl=+{&&J>gFmMrcq6-Ug{862nclSiIFuTGPy z!pHT5q)i-ZlC=+U^9$>3jP+gAKTe4s4zC2HrV-aaL8C$#PMy9D+6>6w)TXpwYuL3d zvpjmf>&pzObbnsi`E^UWo30f5`qeZj*4q$7ydit;hpZ>WZB0YG?A$gaK)M`cAu)hB#w0O9NjDH z8?raT^MmHhxaiEZK`~W|L?3 zL!}DX=oeUdy{y>)u5|{^$zUzWpkkB&k=hWE92)pUHI7)cUbJZ7L8t$%Bm#rr+uLEX z?HaN?TQOfMl(*4_d-(4fc@1t04f{$q2pTtF1lQ4b6Uf5!4+aedfdnWbVwmZMI`4BE zI1N|B$lSzo`y6rckuqO=-aT$_#ISB|9%w5Ikr&3C17rgR& zrsM*h%m7g#6Vc&KzA57NAZ3`r-*Sg0VTC3ibJkJSH^lun6q${bcHevqH^FG3RCzE_ zp&#R~*KL<8lv6iluo{8~C#%gZse#ZLRE0ExcSbyk6}3&o_=n$nQN{ps$h+Zn8cqAC zOg}1bs>~IB7X0NAN~JeBp?5=t^+5HJbvxmFM;}yVNKzzXDE?lO%7~lVP;l2+5TcM= zsO6-s-CY>dPK9D+rlYKHinH}O7-}XHUi2LglTJ9FtBBgFjM}=Y$hvXYT6!xb7XE$5 zkh$BcsnI$Mh{CLu3b&g|lAekrj@l8YI2H`#-H*fIrE>P#!iY3;<}R-LRb-e|=&DWG z{byH@r0}<7yxU=j`RcAKd!%D>v1j()N97_9Z5k3BQ#Z2>0!M0l>fau7G~PKhzPrV~ zUYTAm>0e*W^L_=pI~9MKTK~dt$PE(2M+4|f zQ=>~QZuU~x5rSKhzsyNUi_!Ucm84gdW{j0)pD(6$8{;1%q@B`zJ1tIIwVaqdQ0yqK z+RsZeD^2#@_6^$)y(!HXv~tWI%L5&Hu|mH4(8xL#KL`QDr;J0END5!@3!>>u1$VxI z=yPwV^U6|+kdk!6O;VG~D#pqjvdZ53(uVF9*J`p^q1B9GBhifB&_B&keAOi&^PAkvQ*Nv z1edpUSf3~|KoGN9=Ac8`^wzpYhvaA^KS6lyFASY8!_D#KRpbnUbJp*O7`v+&@^PY@ z;23*5%G)^9Ntu)4lsSUNG+6}m3&}LssBDPas4vgv*`cBbr*g)Z z5lPz*3#AKFm&qDT69^D^J>75VL9d26h*u`hv;ty8 z!b(_25vIi}#}&a8;;U4;G{abzL*8VJ{$o`LJGQoD^Tpu^n-akH!FomBt~rlvd)GFF zywVr2nB$WhqXvzNJ+Ue}9L7mZx!xbe=s4LR!)Nw8Y*J!A7BAmH>%>g-!(U{c&!a-d z$%savf2hRWgA=+0+QrR=a9tp`Qf7}WA%FCmx9$}BnN2m)&-_PZKT8bQB)2i>zi~i* zio7(qKBm0Bad-=w?~3ZHX=L&(g)dS=y65FcbUxi zO?G?>>5TT$<^m3Z02Oewi*-og(1C*x|5h1flQljz^rXjH0K(*2w~fdKoRX8{0TbYR zfy9_1oF|O(2dZy`S*!{xZu7df4ssUOH&9_nA~SFvJ5kRNPFv|t9VAo8vXy=EH773KCu;VyW05~Lw(&QP+0IUpLG*7m zsaDl!G&u25tZN-1);24XYzZ)HN8Xq#xf}k`D5SNXj_#nXmK=q*gtVFn)xwvV8kY8A z{VcD0;H5xRaRiZwv<@$0VIFY`5rWhBLXM+zfw{zb_jmy8PD?>*>rC}tbE!DwHq2R5 zC%}8*3H~&S@!G&uEz}vIE+$Drcf!7Jw|R3Ia`=WBN1pQ*jdaHG@^z)wHWSf>QH@u{ zInOt2$umw%ky<>11Sy(}RS#j{RQ0BsFx+GX9gi5?JJ+d)3%AWuvc!5y=C=y^4WCJv zHcDCe6Ta_Tz3YAn8&-GyWz?X>QK-H}twFIgRi)4($i<>w{WF{EGmMG$-kNp~XJ3Qs z*5xrZc5RB8oz^^;;nl`x7N@RmSJ?y`)vpbLWz6~PPKF9@?a3Et)^?iMR2>}+q~w{?2-P$7bc+wmLVuKO617E?Q8a{V;2k z<8U(;a65FnHb3Kbny0s8;Z*&8)0fPxGx9pM#N%40X0LGw<#)H8cX3Q`rTQLXqS55B zwdABj>tfwho6YlnfZH;r$vb4i6}o7T%%lI2$^E0VC;E-|8BglYtrYTY3sw`f^wy$= z+v1YPzdYgVym>lxqd6vTd3v>Id5S+llZdmNzqv<2>aHbRX^8b!NC59sC~sIwGeb;s zSaEZB$X&Q{aq!POTfBmZ5XCn_<_M!6CzPTe-wX3^ctood$1U+jqTkEFwbT>vdAeY= z#Itw`v$rH9(8lk1@M9K`2zMvac*ZK;C)l^dI(w#g^QDGhWQ+0DV_Bwt<4Z4YNnaBe z0{pus@C;x?0B{3xKxcpu00;{JgA9Nn{NFVJMDl;t1YrMrO#uJ3CV)ZoziI+_1aF9_ z{+$wdZ3<9w<5SVR)&xiyM9D}PUYi1xxR{hQw3JL-^we+Zx%im)WZCI4IX;MTa#H^j z6yWCjr1shr_#{B|PfdV9OitsqCLpD%DWh)tPfb8h^|OhVsk@e$<$tXSc$jHPSiRN+ z+=K0%ZS6h6UV{R!5dp8@P_K~mFA@1ZvC%NvYwh$8Ke+32pRTC)63@VL|ENyK0TN7v*`*%vz|as_UP==(CpIE?9#tF0ta(T|IrcnCn9jL^V$)ZKRW(Dr34O_{&P)W=bx0o z)y?nS<=;n3EB`+`0)Osz76*4u9{*cM;Nb4&UnznA=m;D=KL59rz<=%tT<(rsjZI$d z_Wa*c0)PH5DS^Agt%vc+hv}Jz#ifUnzQ?QO$Lr0f(~+mssh9s_O5o`T`n&~wm2v;; zdaViEzt#lKb-MztC*G+JI^^~QPmxFm{!Hc9l7-F-MJ6{5Kg|l`Sa)pKJW~V zJ@fbQ!^^+?{gi9)^FL$H&p-eEU-Q~<5mJ~5hX*<+;)o=cXcB}VhN9w%EINl`iY>-?A&fHCs3DCu*7zchHWpywk3a?) zKOX=HO79tk6nN;WBDl1@e$Ba~8B>7$TXW~pU)OjhY+mtHpMVVF`<$>Wo&e0XM# zS8nO%n{b8&CYeQ!xh9(K(b=Dwcdp2%o@~Y`=%9oiH7B5W_SvJNi>k?Ji+|qPW}%c; zYH3M_BI;tEnjV=ckCAo?>7!3xYU-(|KJ=-mp>~qytBAJ$3aY4Ik~(FoxaMl=s;|(Z z2%x+sqG_;>78@n8vD#WJg1R>A?3>8y>7%jK-XW~3zryOJowOzz-?z8S+O3h3K5Oo| zB}PkUtIhCY;B8Eb(Wk7qQER2LEHaDkzWgevE{3auHmqjoXm?sNIPzL>igVtVaK8{o zOkS|wjm?AaS=!hGZ+?ix$1guh4Z|S&>Q#Tc^oL?_J(>OVs#AVku$D<&%V#MI znhgwOrXB>&7jW2K)#gXR3W8)#V31(1I;I*h+@%E5qh5HP00Q4J&3&&64*)^%8r)H* zgcD!^;8akYrdjZYICRLIWGI4@A<#z6_=4g8o)AL+$j5LQ_<|3&WuyrDEeBpJVx#QE zIlvWw3k=Mm7Pknd*hOK9iVGu>QkJ>_tPPAQ#(*rH+OGI^pPg#6CDIERn}*8yE!9!l2a#a%nV{7@lw+&=HYvI^rW`04T*e zk@A?s1Ct4(s5w~vF>j%)keQ-ZFejq2WMUMcy)Zeq-d!fyWBIN<{S~l(J)~i$n%K%xYO&|*D@K(i*}~5C6qps0Wk)-x&vey0)_j#g}PM+t|~_mZ~HJk~tpJTO$^CKD(6g5lF5M&Cad_#P+qc>qfF%~dw9k&?uL`)!R0DX`O98zvX*76 zk1K;Y%x6w>m#N$U;x>2xTt5afc!~TqBOh6@>bM4$>#=1%tJ%+gKC_v{4Cp``TFrwl zw4x8q=r;HGAaUMIen~@T2}OET_chLbaYfrszr@oJOZ1{gP3lq?TGXaawW`O_WK^&E z)q+m%LM}}iTHB`5MeG@_AG+&e>g&;i1WKsUY>#3a``E}%wyIsNY*??F+0Bl&wBxa8 zM^7Yrwj7MMw|zWrd;6r_{eoLU@7*{gI`_1uu``OwP z33$R2h~Z3I+JMpj#E~En@NnVy_rWlaxy->$^P1cI<~SEjOLxxmzDSrv8_#&P87*{0 zU%cq{UbsL;F0ur8?W-oY7dk}FF{np9P^!G>Ir*XV;51+q$Ot<%#!hyzm;LN$H#^!Z z;bC1EqZDwDyWHna_qy9%I)7lqB^t4Jz4P7ge+Rta2SND4_r32E07tq=+;OA_9qCYh zJmlxmb*ckTtf|fXtxGP|M^Rlw9`469MvslsYlHO9NWJP;&-&J%o*1xKN82j_3%lDs z7PqfE0ruC2K=l6iz~95+h3|Xf8~^x+|Fwf84|(P{pXigXJS9mrt*al1^VU~5YO`-} z#V=nOx!8vP>0dv7@|*wsA0z>5zWtSAiBte(UFc8n}TR$O#?DejjKGBIpV6 zw}13!f+(1R_-BIs#~}X)W{p>I1L$#k27NO~X!<673Mglu)q{v+UqFaLn$>l{;bE?K zgcq2Er$-zk7=ljtgiTm`v9W?wScQ3Tg;;2XC#ZidxF9!(abK7XVn}SG7KZ$$gKfoh z*(7An^?>=LhCQNg7l(v&sD$X3fl%mydKiU#_w?}fXlXsvZ!Ka2!p1Eh79P2Y=V86_<0dXfulHzrihAt2!(&hiiEg~g7}K)fp0XX zVq++YMn{db_=u-wjWw8b2dEHK$BW-JXPKBUZ?+~x_!~wzjHI}ZqR52C_>Q6wkDs85 z%&3g@=#1w9ZN-oT9ROfF01Vd%iPiXJ&9)D~&;0Xnb<*w}cF#|koV0shF4)M$zF zRgSa*hv?*s;5bI(Xbb1)8|kQy?AVUA2am^?k|Riu_1KbucnSDu9|1i|M&v{mIVz-lr%Vr$hHq&5CIJk1eJi6bIA%y>2HR{4KG&( z;_z}!P?V*niy&iDt2B-x*O3btWO9gh>u{BJ=#KJ830t{`l_`(O=#uiM1}!O;6XOkM z){h|&VnINc0Xdje=9W4sW#LXh_lF(ji>{mlV|-H1bNAEOgU?tD1;() zn7>Fcj`B(-HA!3R+a_8kTzxkfk}`S*Nt4a8cr|-7S&p>(m!`FHF2;y zhM8GmwMImhIrY^YbhJkJgkQpm91)sDpcjg(*A8Bw1>q-p)TxA8xq)E-2qp>$Z5@yZ1O;LTh`^c52%cWokc(gzu~`5zV4m9e4o7F3`H*F;$z${|q-q(T=jo%2MhPtt zH4@35y_ssi>0_$#Gzkz;dDJ4Y(^BeVb#k>L3bhs@@KWHxCsVLUw$nOBRU}+cLgTX> zn)93&*rpP21?(^k)exddIHGiVQmAyID9Mt__y@XU05A%cG+Knc$pr)dmZNx?p0$~3 z`j7mkVM6*hvHd038!%@zep;mc*OE9Xfs%;}A{YiHP;)adHP?xqupp?dC<%W@ z1p)v703ZivV68FgiYDS}Z4h91d7}})n^6jDKdNYLpaVScu9g~`Bmk-vxt2-buI{=B zU*L~FFav1`sRBuuGD1Kmw0)XaYZbZ*n8XDa`U#WurHT+VAQTD;MWzQkBi>W6hEp-j zB{J9s05pIwSyv`<`k3q(muCS5#cHf~Xq}T8k3w)#e(A3S5C$p#8mL}bt(VXRR;sOm zr?Z6^t~QFJ?*)-Mnq?WsZ1j1PE1+USDg$9Zu^p{kRVa-LGM=Esi#@Bx*&vZOeP zB6^Z9%cKMl0&n!GL%@}NxT4i61{eT1f0K8HKnLHtiiwJjpjmB4O9}h1wfDekNsEh1 z+W;~^i7h4t0LG&ZiHq|ZV4pez@HM0W`(*;_I%42Efb&ah`XqmQ0u-<{MKd3aWMh_?|tBxCm)Ox6wiwEip z01)s51Z#NqxVa*^mIWc3DR4CVpi0Ajc~fL~h> zT>`j05C#iS01_6QA{0z=Avc+}Lqg+9BMY&#fI4W=Q)as|eZxY4O9LdMx5fml6i5h! zPz{8@y@Kn#OX#vMOTNaS!_vBvzw(tz(+4jAIPj~C^4l9q%dabj%=)&gDt zlv0qSh{kA7`vp&1k^IR#iEAyUDxk4L19fCeaHXpQ`%h8zuooaE;r2FEwVko%HH>=?{uwP>-B zqN<3uyTC|knvSTNQo5uEoT>lI#iWWEpo2w{gr(0ry$%}*c~TY>oUjVo9ks(q7R*~2 zivTR32_l@ft#t>|YYPhG3y+`>amvSjOvB)Jtm+891TX_DNdPEd%qsfBn9ImG009`# z%|5WmasbJSthqvqi^YHi5xD~SiL{!Wo&YQbEzkinP{l2lh)N1;Ob`McU;<45z>=7| zykM*KWEK=q0&&Y%7DG2q1Vyh*oH=4k4tlYW^&HVN0ITB!VT{J^vPUdnGXy+7d0)t(%78MF+Bo&Jpx65xsHs}_NUXrH_45`e5+Zy zNNSXvI-B+^c|mQ(qz2GRVAVYU1Pawh6Y8olf=U1LKw_OQdAqOB)T*4VN6W-bt-Ml~ z-P(5)!4ovn=jYbVe3e>Bvn3j{HEXy%OoBQ6l6WB4UuoEkaf?%1i|09j#qh}m2+&{U zMR}Z7)zNJ`1z^t0#%+vKAN5RbB+;1vw#%|3LulKadW^Ax#SF!~*`g%dCo2Jfe9|pz zcVuk>cKHF6dDp)E(scmSb6|hM-I?VqWvaW_MK_bkZJWkzgI%1SdJWV6 zZNve7;DV;V`q`;E4v9xi6`8UHvt+lUF$F_#1VxV4WhAvjo#{#4C}GZ=RN-8mF{n+jO$Ptm0rc=;V6!Uo-d@1=z@&jZGP;??&^`Q z7|-tGmrlU7zLdAV>6H<1o*3%9p6y*8jNER+BWjt4`|ZE&=>5%|Q9`P(r?5e(%k<9Op4&Xd}?to_C9DmOPfAG=1@KjCmwcPL@ zsfqF)US3}B@$Cr}Z;aUgk?|OxzA~!u{8^(v-Oe9xiy<$iZf5e*?cGC9?L=?w5AW-z zUg}Km^7mfs$9VBH5488F?3Zrvf{FDFS!M`t^w#ds*!}gKep^RRS@iDmEB^8^zv^@z z?o^M3=kNyqju>A^;EV0?1!&YeZ}(n?mlx1-nlAK3U-sFJ@++?*N$-Wq5S?hhrxXwM zPUv*~{`PL4e*h2_z_G*$Idh!f`4Z{*pbuc1ANr&pV6hueEw}52pN1*F3W@J8HT>4H zPv&VKeoqhi94PgbpSfKS3=3fOKacKC9P$M}{ORuZ>-Oh@e{$^p-J;I=%|Gw{2X;#j z-xGhcxS#L$4GQ4@Uh^-R3~n&@GPC!x-q^wK4L5fFv6lRo82t$A{_q0-6aD;*+47CA zdbgkbKT$NT}nkKRLv5hYHfSkdA|j2SgjZ14_PGl0Z-09M0N1iWx1|@2e)xLcNHaaYDN)@W7PNhPn zI<>0Rt5~fz&8pSw*REOv*oq3aQ`xg=(XM6t6hPRvapkI|duuG*x^wl?Rl3ygU%-JK z-e_Qv6fb7n*zse>3J6G^T-owv%$YSO;4HwifzJR!k0xCjL1+x1kD6PDFbbKnXYk`=n_pksef;vR?a!}&egQy?;UpGQ3N-LQ1QS$H z!2#k3rjI=?;UmI=DzxxIg~(U{If&Nt@I!ip`)x!LaT9U35{WBLMHYFAfWwS>DB!dJ zW*nf#8*Ri$ZvBw{Q6mm!+i!@R+B$HG!$R(RJ%}FSuWKzlUDZ-B zHso{Gzq};#RxxqakE=D^^tH`7C7ke1VT&~ukA(2lvqM^&&FIh;2Zc6KM5m<|P~$cV z_EBy#@%CF%!X0nYZh47ht3T3J7eZv)b$4BO-<5aX?ueZ?-h1<{H+hHlRS7VOVAgEx3L!QXmYf~h-ny1P@kaBCrIT`U@^#kkAKmQe?F4h39C$6bZ* zy#v?#Tty1^wq93_HrtUmrha;H#fdfGSjZz+8bY;SUK>QLwf6k;u5|%B?1_Ud`|Q&% zw>-kkH+NPyw@aS8?viGwebHxg{#$U~d*7Y+ri3xv_uq-H`4?4NRegEpQ|DuH=bM*) z`lAJqgnI0!f4q@~TtB#T(8JHwYtqX%|7+8u&z{cKy`R0vsP>1?6APRl>Lm)oSV-^fz5hF;! z6B@A&O6#5q`2;=>f)I4%J0A%Xls+Xcu!&B5U)zR4Ke@d_j9|nTPRK|i`>{uNJIu=| zOjtZN#t|w#%wiJ#U_Cq95sySeT^@h9MJ{rVgHiM#6)i@IT~H!GhpYvG;#5TjC1PHF zM2G_07f27ekQ?~eWVL*=Node!jP{@#DNSjcxrNeWyp~=%5f?=lw};pLP;$` z4VUZNW977nNiIQ=9mC|BGo;u-5kk_67faj{ow)`%Ky#89x+F0B4Z?_4b6bQLu;7JjgwvC~DF!DO0fmcpbepB*W*BiZ&WpBT z85&*Xh|&l%NDXeC1q6>fi3iVzUhbaqq~uO>s#AV`NSO55#+QZ>P-0f2pi>l`G7-5< zhx!tA)eNd@tR+%z1fc_mA;D4ZfmLa-w50Z61qlw*02!#1q-FF?6D|;e!>E;7=!9ua zZ8}St))J>btsW$%DOL6X_Lo&vPka2?j!|%+0$Py8FaWyL&m|N8p<$T8WJ2(RL@ttq zQKb_RP7}@SQI)WOf!!BHiG{azzyya$SvkeZO|v$@1R-FpXl5+jb+ zmguDxq2*m`*iPfJ6I5u!C(x$n)79Y;y3wuNK1Umvp$aq&{ZI@Op7B^h9+Z(NxC~|+ zX+_RnNJBMJMbn5VTJ?E-W~>u3SoSlWRN=fB2??t>bi?+1#a(n3o6R@7fGY)TC;I z5Ii`HcM%I?WHyzN0YbuLeh`KaOh&VtmE^`eizC1C(Sq>*O^*wXG@~y(5t}8lC`RAm z)&^L)ej8Q^7Gz)oLNu7SSxhP5_4%uW*q0d2nK`aATwvQpdd_F9FgNhRHyo5+fYlEwML((Tpmz z&N{)90}y73W^&4aTEuEY@Mf2Cd9CVD$Acq0R~C(;vJi%~i(WZ0u#h1KD!@Pv#E@4U z&P53@5P=GOFbQPO(M~=XLA8OP2Yylaxk+B$5?o;IE|5*nf3(*NM)2(G4N=g8Zk7`s z0H9y@chLh)LIq5qg?HOH(vs#hBPC;kMb2mjJ1*h>iTB`{8o+?P;-)64P3@pmucx%L z-UmoT8HB=s+02BR-AXI*ZUK)~!n1=yT(w!PcvB4>#odAfcti$h=M}lbqd{k8B!W9D z_d-@kO=wc!H0CC8x<>6ODK7^jCC82tqc-x!l)V}sdFAOs)a``-QD_Xx|IIA1Od z4|;9VmV#B4>R<(2A7Eg2jMTM+vTy`s08Wt;CC;!6iwbUJgdyCB0x`8I%WvfDnioI` zIiD%tJbk%-ARGvD2fz^kJ6+L$fC@W+fB;7SK)H4EnF_rJfD7Ip^x)4*v0Q*dVWeRR z7CZp!@;+=Dybfu8qmKY8DErL`PBFF*Le$vSgp*(Y-s?(y)gwLYnJnHmcXxonGC8nN zQ0w@R>!a~AYJw=qIeDRot-^3KZ(uVExID?rJe5f~ruYiZ=mB#{y}Y?AzcHLBxRE>H znbQNH(#tcCVSuJn7ti=T#zDSZQk*XEj3uy?9S{wm%OgGmHPLXo+v5|eD}W$dfYpMs zMC+pJTd;kLqOgmFvFpA9K!LI&vVob9*^2=uT)~BaJGwi*=CQl;rT;% z!l@OrAeKTZk) z9WcfqSOjCVGJ0r%0_Xtx14ETiMjklECV+-Yh%GMACdHe>LrDT;)B#oqf*&BjU274# zA}&1~9-LXW2UMr3m@`WNf*|{VV@s~RB8a6E0NazQ>7u>abHsIlLCc|n(@+A%GCn1v z#6Ykv5NSReNdsSK4MqDw3D7tHLQ;ky)BpnbfN(Lp8-N9cok;3z% z!l`n&Crbk<%dLDcD+HuE9XPo&OFT2%!6tx%MPLOuv8~ox893AhdgwamBT8TEu;!ou z3Mc>@z#j^Pjw6_W8hHYE6r6mFrdYwmaKlBQF)=)x!IkvK<^nYu(1e6k#KBTN8hHR1 zu$mWp80)iuM>DEXWGY8O5W>)aMIePL;J=DdLg%{0zd9nAJd!A@nk|{Q5ve#V^Sc6D zJdTsK-D*D@>A#HwK-&;QXRJ&XNk=(r4jZrmtOS8esY(I@1H5}jT4=o;YRR+tN`4$j zhTJRj`?eaX0frMa=xV|Ln_=7*JOwatx_TxAeczj zln(Q|19sfQ=rKrvbW7)gt|d@G2Wde82uok`sT4$jYlBD^00RJBNS@M|?~JdTlZ|&n zhLFjw_#gzuV5}HRMNH^RKhY6BK*b0xSTlExS+Y5V);$Be?)jW*F0b z+ydF0O|fjbth|o@rwc5_*~d8egJ8JI1L#Mj(<#&$13HaM${+$y@v{w04_|--Fmnh( zD1oaBQH8;kY9xeQ5XG+>9~E^`ixGm=nt>`{C<2kuf+zwfoFJ9E!q?EGC1ui5JtLGL z)zM_Yl?w|@OMoc&$j|%84#bLMQ#g+V(B&KeGv&Z{`c3~)&=>Th;B>)sc>-a5&nf~a(WGey(oF+UJ^0Vn*_H(*G{LD1aOgf0L95OmP$!jLYA z*fM|vto*tEf_+e{sa7AHw~U0W&{4amf>Dv}6;35qOmfU3#Y}WPCzU;;bwxQYjm-f1 zgikPnAel?56wuxggAA}aC==6})mfxz#3o6BbUTA*Lb`ukk{#HyaVvzVRLEyVp+Fs1 z;#$;>4WA+G%kLXmueF$xHCS=AT8Vhs%Us!X#jUip&vuOqgmgIy7y?A2)j*t!)I)$1 zFw>x|)yqjx21pGE0Nj3XFFi^YI!L#p^?{zl06)Fc<*CsWfB@gjwy9mnswLZz;aKU+ zT8(_9knP$@{o0W=p~@|>P{mxoFx#CxRhKmH3AHaoDE~q8c66vz#aI(O5yc)`3#0@S3!R0^QHu&eAnO9~xWKh0HmeAGA$Z z)x{Q8l}A!&2B0-A^8Hfa5lcg)TvGu~!{w{T{at&xT+~e%=k!?U?S+ioTG0jBu+7c; zC9P=eUhh4;DgEB~Tw6sM-)Li_3Zyd-`^`yIvE%*S3%*_nvfulyOU{ifOVHGUOH6}R zH|@pT@Lga7ejC?i-IpLu&U@e+x?Ks*i>{>KhOA)u{TRp9V0IheXNj=<{a{Ju*8e?X zS4IGSZT4MP?~ zWk@}$u1ir*Xk_WVYNlFio+0C-J$+!+4hAm&rmSOP zmd15HXO;70EuLq5473vVV+sywYT9Rh?q`YERn3Hq#9--`erd+o44Iy3ntqZivFXtO zMTtgXD8A^Q@afeBYRz-t-SueNr3#KdY68kbFCOFut)QkRR=1mjKNjQ*5@Vj$=Byp) z7q#U8zFtZeRh1S`vMy`0K5Mj2Yqef$wr*?wduO9&=AMyRUp*d5S?F*z=?mFn3`Xh9 z-DX7oSa1eE_OMwyzHH0}4v@Hjw!p~s&4H)L+pqU(ZJ^F!4^q3&Sx8i?XuNLp)RHW$A)Z7cAJ$l z2k!0doY3e7%-KEm9U5M4ssK=?mS%2a0K=~CK4KaYw(jO~16k>8l$LEoW@IEDZ`&^H z*WT&vjp$&Gn^#+J7zz&KE{@ctTcy@3^UYm)uJ8T1?d&!eDwz@x4{;H9xe+gM6BiPkPVt)t zK^1TD6<3m3L_`;#ahsl3#7=Mp*V=+UXz~^typC$cL~ovs@88&P4X1FHc#-K#=Fb*o zx_)x~W<9>{@5CHG0xxi*xpFMuo&z5(9M5qOc4QHbq#rj~2v3;*$c`J4dl6w?;Wsn# zQ*s%&cJ4YWYC5M`NTJODzh|meZG?CW3^nPg(d{n>bFHE6F?aB*K6BaNZDY3ZG*@6; zL-Ld0ORE2YPMshj# z>?R+vC?6b74`Wo;U{CjTQD^1-oo!t9+^!z75jJfg4{{-|WDGxGN|*IYUvg$i>W|KK zyWVx_;x_5-t6pDpLf2eUH>e%w@w79CMppDlhPq~Vb|NnnX;0N>hau^k^E;O|Y(E}7 z@8e(h^=}7v*v9cg*6RE~^fhj9^L9xphoCM`zj%*#wVn6>Hh*EF#&kKd_G`!OCF6FD zXP$m9U965{aVL1NyJcfn<8)8>O=R?kfA=DHd7-fM(!O`*rg;Zc_PqZ0MeKHg2zh4* z?DuE^4=_=Z7i@z50pFT|Geq`dUl9IwZGjj*>V|4|Z+B5`d6;LVho^QczI7+p^qXJy z!VPq3{&=Yg_QQHGqVL~wr)7~O49mF2lUJsFkEclPy`e$cb>C}O-*ByWZ#P%xz#seu zm~V@}c$?pE13hg}=i$bmWVC;n^TbP_5A)C+%|}bvPIaC`l7;7bMmTl#&2O&MH`yDA5%-g-Vs8Rcb@AHG5VVuS99v z(sf(+uHCYk?4s?vckN%lc=--K3t*7Oi)b+aLJ}xx4jna;C2umsSn*|x0vacDlo9fo z$BSKIPF(tQpv|2#gZBKHGV6h{J98Xs`!?>}9#uZWrfpq1XY8D5;X#OygbG@60T2HB zIrQkxmFWc)i<7BRm|zhoz`|H1V+9DHxSrkm`R(b~N0RslwPihTMjVL)qv3ng@N4Vt z@Bja9D~$t|JLBkZK>JRS8AHO-%$bUKUY77?VsvNl4&@56F;3h&*Vx zAxj0&Ggftg6-EnTFUsg%jDoe*%rBkHCF6{Yz!#rmKhAcMYO=Xj+8CkOE z8 z{!2AzdiN_p4NN<|b!w;e?)&mxJa25*f@>d=wPk1f{WR2Y@okve_Cl^TWt#hTDH^cr z$#!eQJ%62w(OowXN8WG$b)Vb&KT^armJf?vY;qe4;DM^bg90L~bD9d^#xzAReK3e( z4m6obXm*w%jHiR%0mJQbM;HytB?F~1Vd`9vK6%oWhz5OL@1yMOZZ?sM|fllb!UsgOhI-^fCU-c7bjhbFpY}I0SpNl-!mhhrQ8@PtM~hByrfsZ1prcc3J#`H*XE{NEC{G`8lTPm1N56B6Wy zg6;r9J@boO-+K5OrvxJ?z>bAeoM4jR z6@#D-Z=lmE7%Yhf!7zaWsPGm79e^-1$(JzJC7J~YgDubzI}zk8gbLlAHo>3?4lLkq zGu-75c{m0l2ytnuSVF%{3Q~>C(u{f#K>%F9Kae8vn_f$%be7qUD4tJ>RKyN?!oWw< z5#R^xvnKu0Qq0fP06)V7m`&YCum~x|f}|?d6{pH6bvkaSTJL6qw7Jxb%@$ecG`Fn(OCO-gpx)m;)KbJuX-)moeWig1W)2(85ZE z+)lMMgK%9P%$Si_xuS6rkzifxLU)HKj0c6^$r{R2RG3T_lQ{kP{vRK?Xt4CtE@HR!rJpJT`#uT@3?R zot=WN9)y9*(3@TxYK>nYsG5LBtVYfp7yuA>WrDMOwCiz@0b&SnW}|iq2A3F1pS3U( zd+N;UXc)F>0&%>uDPp9NxXph$aXcrbl#$63QkK)P2U|eSgDG;W=pMlTsyt26D33Zw-vbn-8%FL+|&a&yw|Szb<1-j?2!9ZP0Hr*$x(it`bFDmeNKCS z0PSs=*ZkWshB>2kP9VKM4CjLPkkA46RzW1A2W)}^DpYq1RHu6KG0M{y#9L(lQzm)g zW>9$Zp1N|(G!~bvo@6&E-fIHmG6ibhxY;*?_baWP?PoK&ol7o?+{c3MvEM9I6F+<0 z5WeuB0&DS&AKiB(fBBChfbh9jZ_BHW0v_z35GdboQVQarAn@hyRPgD_pQa27Cqms3 za`1%L?IswPZR`nb!qETTn8W{1?QO3<-s`^q{r7$S4awczAJ_n%A9YUiIbT#H-~m2g z^npZ+Fj<*h-(Mw}S`)#Xq_Sb^%LSxbpQzZ5~S zDA?$&83Fo7{^_4|(3$`3;O<=$00v;R)L^g~9?vP@JTRaUB4GpiL<4~T4biy;7pT$@ zsFDrrKn`?Z=W*XzL_rV001ecR*V#5dxjx8KJo`;#NK2N03l5gqM_oni!(q zBvvB7^$cE-;2n-xG1cKGu2CF1R@0;(+8v@uRKUl991iXuAPQn2a-Ju$pepKNJSn0s z%7eA!Vi7vx1N|Z}UYd|;q6OxKF(%^}5~7xsqGDj(DDslipkdvm;w?sE__*RL`XTNG zBFVX(Db=DaUgK>j;S%cC8)|jYH@aOnDx`Tp5*hBJMBa%Z3Z$T@BQUaKI}RhHIN{>$Tgq4< zLmlIkF(W=k57?PxNU0>1SfqCm!z2~tLAIhy)+936Bu(nzLNcU7^5jl3aqGWm{Z{lWmey9J42Xqq;-t<}7pX6o7JbS5KohU99x5M)NnmTf2Lx#)ia-+&q@jqYZR zjzo*K`5DlIG`p0bh15Re%bpl(OlS#;KLcDV>I=+F9O+qUi`x zqGB>BT+yO`M(NeS-<#S=n{MZf)~TW{Dx-dlB=)JBMCwOQ-vAU`l3pq@T4kZ`r>64f zp+adK3P7WdDyf$0P5CLNLgY}=W2zb>LxJii8o;2!YBYYTR_Z~enyRkuDzDx~VTmP? z5>>DQkFXAFkID|Q4w1qsE3-DMvpy@dMys?6(X>{pvm$|9^=7YbE4S_{2BNB~W}>}; zD|&@vKhi3dnUSrcD@Cj;yV7R2#;d&E=!v%fYGFL8y-wdmJd$1&WufwknZ25m%4(;o zsln3FfnnynE-b^wX}yx`1ww2k>Kg(StdzRzKXNLfHY~?>tbu;3#Lmpo>FK@t>xu%{ zto|!SI*-aeRib(<%*O2fJuJl1Y|UCMjACr9wrmN?tk3@JE!r%&x(>FYtlxG z{;SR=?aKl!)J82hn=U5)P}9tQk$7lC(&MQb(*Zoa_xd4 zZO)D@+{SGZ32jxPZOM9V+w$yV>McXat>6AF*@i5SoUPq5CZW3RWFfAgDsBLRE#N*b z9&QrrtT4)EdHpzzOL^^ZS1P%>?+yru4M79nUlh1@BS|H%53mf?eNwvW2EdG z2&~pUk4v3v>JqHVY3=JRZ}WyP$4cT9C;<|juMsH0nbj?iC;<%|8HsM6M6w~6C2b8k z4HekS66kOD-mZ|eA#C0u_=+z9%c~@^0J6@G5sa(BO)Lm>2c!;er;^U;tzc#m7z(m2 z_dX9w0l*HOqS*~_0grI6N@4}ukx5_HFS-0bzRZ~%Ce{c469kyHRoK^cjn?IJIH zjGYH}v6{iF5SKBxl29iNKoaBWd5vyi6!GjluZqS71p`k);cZKe-OCV7#&xm9!O#b% zp&1Wy%=#OgJf78BqSARsoKF?S6nA4+DYq1+r@rvM7)5 z6S9Rw6{`q99;$-ujg%Q2Y@Q=0%>$cSUCr>ywNwUAoatWgGvbiznNSLR9ZiigG0!Ur zl|Tld&KS1qxX~IF#pC3PCFm#zm4HPIdMhxWQH z89y{k7pT9{02jc_J4+q(_DZoLvdz5kMc@FnzNHF6BQIZs6~w>`Xq3Kq87L<%&^+Bo zqn|L#;he^_RZ}SmeSl$YQWGP<4cs&N;l&XQ!Kfj@6%9d6gCss1^%DR=S>J>Uw3_wm za6SvsLf5lOjWcG@%UHKHN0q=EPV(ph@l_Z0AsX@_Q?wE{Fs;C{MPIaHJwm^DT@_og z%3U%K3xM!QCRbYj!Y4CyLmRee*C-KB;0N`z8_S4e3q$H;VV~66TsK_zo}k@u^R0IF z2Eeu;k2Y`1^rYen%dGEWo~RhG3~N93GX97akkJSVus)XGa&xiQz9tz-cl`A>c7LZF zML@m_v{w9 zf%j+s2C!@Pw}ziM-U)bEcevS_wD;bxgiB-bVmOM|IPF3B-QM>pfVfk~xbgZpjoY}8 z=N;8DHjX3HCXa9La|5}F54n@ul!{+8fv>kpf;T1m_zyq1mQQVw54d`Z_eq2KEjRg= zm$_$^GK&lEmt(n=A2^VkIh;FcmwWk#gLa9F`PQ`ge+GG+$9a^8>zpg2^p3bzCi#Wi zdB*m+q63NsPMWbk`c6Z-q)$4f$EBrTI;Lm(q-QBOEV`%bhBWYl6C6_&bh>Yb=cwDM zsB>u>`9i9HI;?~GF@?I9f@!SZIoJ7-mnY5)KL literal 0 HcmV?d00001 diff --git a/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/resimg.jpg b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/resimg.jpg new file mode 100644 index 0000000000000000000000000000000000000000..96885c3c51f4fdc55728acadfd99eb3a32654344 GIT binary patch literal 65566 zcmdqIbx>Se*EiUNAVGr%Pl5$aaEAcF-L-LdcMAjv9^73TCpa_~+})vZcXyjk?tSlb z-*;xds+p>9{+I(*r`hMMz4!99e!Y2~eO>{)mJksa0l>fj05H%W!1EJ;RLI84*51g$ z!N~BfzS~=KTO$*Cdn1Ft2cH)J-vBTFx!~d9;NcMwpcf*_tKSz2777{)^cM>Q`!)J& zYzz`S92`87_r!$6?`bF~Xjqt7zI@Sw`G@}H|GNpE+W{DeFlDd_urTidFEC(WF<_oM z0mM)yUc&z7<)7;%96SQd3s}Tg&+`Dpzn)y5{PXnx@dr544rS!^AHwQ*)Q71q!zl)z zXV+swyzS#WtI8GR-(}S;YiDZIieZ zb}ePJBnh*hE+S;bTnm_~;|Y9>IXfJoWG*R_Ro%w#n2g+GJpcd@BHT?^yWc5*ZGv3T z9+|AdZ*`|si5(fc0+AKc$rk`PecEG{#ViE65JBz^(`sib>D?`sL0D2C z3;R^A{RnrvrHij_bA8HI%FdT~Er2uCff(M#!A$svaNpVPt*_A+4hZ}19J-B~Ao?IgLh zBPj3bE6he~0*xYa{?zpCcG!U-;{+k`_&V_RVwngf1%H@x;UqTt1A?d5*Jh}4&)D9L zmz^T%t%@A&yE`rj6Yv&zxgC@-Ywno`H#-gqmYB7%g_KY7krhmD@`GA@ziyN#$eupb zOBZ`5PR=sdIvwUWb2c=I#Mq?OthS^i4d=68@Y-k}Ph`w;j{*Sb)B<|7$6`(-M|ld{ z4EL7{MAMwy%}r&hrToadUCsTj24Iq=ilplGt)*tg^+}Rv0BoNQ3kB=7a8>^dux56c zBC2`f*k5b6Wa69L0ugvW;1$@^@Ny=K?YLaF{2D5962a^Q$fF*M<5!s@B<1V9?q>Iv z2nBIMzvj5lJ7qkOlJK?W#i|Y`qNeJ$+5Qo2k1yy{p65p4)~nnlR}1m83z)qHIRqy? z+HM06<7E@xgq*h0W%JvbwH5liKQAwn&jA3~UrtKXEBkPH>3ExM=51LD;gHzkr~$49 zaLF;yOfw=47;KTh)~wO3k7@?kd`xbSjMsXtxR zpGO|SLY8d$IF#^k$dr6PT=vSHy4GqT;s$oWho0kPpk%yew~ma~QFb|M!m;?)7P46y zaV)n#rraTxR$sp`36k1m!+G@jvy4TOa#J>zcvI=3gMX)hizrpecTm;frS0wn?Zcs* zR#6t8L6kdnPh`x+?qJgL8M6K{R3!{`8{E?S%wA}zBU;0QfYbY2%b=!d*48Of)IWG-_skMh|-}nlGh!IPX+-;`Z;z zCg!)SCEV8aAzw_Xf(+4|mA4RL__Quo6R0n>ZTv(z3$DjxiWlNe=BTV)cL#K&+{%To z!lh?aB^!sEWrCF``8t+#0RT_%!KFRDzr5@bzUD_&?Rw6m; z4MVsb(Enb-(Wh=n7n^LnLV1ettKL-mm$bz(-8yCNB8A9TsgTKaNAxABb8n(%MjFil zO};DGzE+0nNv90uZYLdGkvA;I0R>%W&hb3XZ&XkCMIw3YQr4hpr0hCF zyNN0z&rVKUJ>s~}ZZS}V8gsr`#!MD1*bKLI`9bM~G1V*hcYwW(l_)j&z@I^iW0^3{ zaH*tIbw|PbL)0MV?l=s0?LzANltMf~#80s2x>o$u%6>U9YxT#UzjK4M{i>1jo89cq zGd7Xjeuaa}DJgx!x!PcYhVvPtXzvQz^p;Q}*4fFPS&!dgD_MLKw>b9Wh>gT~UWL`f zm`Nuu(VKU+Q)M)(&OX^BQ`<#c;MbS>w;C_f1}0)~h%Qz`*&d*N^07e|XsVP5(P`R@ zXIw-#snNnL)oRSOOKJr1jGV*CI(6vt3^}0~@avU-g1U*>iD$i*-jlnd3XSF-xaHM= z(d(UWZm?TNiX(#cIU5HP8x2#D6lZa+SrlJ)@cEMMXTlSFeuwv_OT2j;k~HQq#f$8^ z5rkm~={owbY|Q|Eg!%===eUra@EHpQ1AJJ*N5AIJP_co!tfr4&pU1S9&a7Pzi7zx6 zS7^02gb(*M<(IC_g!5!h*}FxvG$5tS5Bh9}v#c8FWV(Iayy~lL87b(%w?8yePOk6O z8z(X&jM8VyW1BZ~__D5YDKQM9o}^fizT|=?li$nar#VfZW1Pnf#7B5&GS-7kH+6fj z=kDrT4|dPKgcubo-GT<6($1Fl)i+u~qiJ_rI2|f&enBnmxKiFLNomi#E-y=mm9IBt z4ShdvN^!PtNR#hQW;-@*LDbex8l(hGIL&-*e;j{IWYwAf3(v*gYEDrC)yq%d#=vB5 zu7T#4745w9f)5*IS*Pfkrm2Vwv{7>(8FLYYJe4fQowqA~sG}^ta_fZb2BI8qKcit~NCr7Oe3_KAyc$CddXS=&~ z!8lxHt__!xc{Zz|)|i6}b$KRUbsg@8cMIMDzt_2zV5lmJ@GwilZV0}QDfQ~tndgFx z=n1CS6Exf*j9hP`Wdt^Y+LpCF3=UYgpjFA>6K}a_FDP{h_uM(P;uo2lrIFV3=3b)+ zt!z$2D@4(AR5_PVvq>OXbB9xv0T^R7)G~DO9qK}^S-2`tTW}t=ZG1NrF<*HpxmeuP z4|tj&=}}**SUjk=Gco+xvO$$ z$XAo1=pmA?DACvccs@--aAjpQ>UI%!%V8czX{3r;2@TPD)D}c^^N|(|SSA=-tnoO~ zJ)-#@xV6p_fhGu|u2U0l7T|^lR7|4MZqj7b?lIh;Mbg=03Q1Z_=flp;)4B|^+NIV! zP+5d|m+*MF@>W()N}z7^Q}Q0Mcnf67`iHyTEg#ETz|z1mTjl37*+JowTcF8gy=@v( z^&D&H#ZsQgM$vcyfxFYz-E4?qFxi8Ky7ld4IsS~5^FoB()%)LN!KS1}hJx5yi+zmP z+c8c>Co{77sZ6S;sx0myqiEnlc9=^J_6v&buBET%#A)6%vgO!brCo$FpcOaf&9mCJ z_K`x?qyiJSd6bJ?HjoZQpG6okyPZsk{$PX;igR_r1~F%fNf9&EZ0I0VkkPx?ciBSt z=V6bqo)W^Oe&x;$rlHgAr>@jD&GRE3@+&5NJe;2+a8u_U81EB0qJLq~W#Y*8cJ)A-u|xL3B} z!ZefVNQPk;tcfy?Lt41T@bNKOd8P3PgEi{Ur+Aczikx>^UXq*70Oi&k&TLJqLi^4# zOK1+)t*;2R;wt>oMHdLfFPrL3&e8sQK9TA{>A_!N)GjxAim~hNwyl`%Zk@t)2(=Oj zl(`t{Jd$9qNVmDg08%H1qE|z!%FBwARhmc=Hw_(XIXG$Ka9bwibsHqN`yJYdAgnt2 z>lo=@XC?zn!Eh=yvSjk)gT~K*90jm&KG$bp4y8ShWpGD$i~|38ljnNae(F&wlh^k@ z)z63Mj^>Ov4+}J*py0kuEsMrL1K`*ZQ|Jqk!?})f&ixuaF~zg)Mh#hxKeJX66(SHZ9j2O;<0Cf`ZB=-S>P|K$cHgeS{AC;K zr-HAAp%vTdB&tCEo#TNux8gQq=BFv`zHi_{5!{V_! z-t7iy7vy9~&9suGbjJXr88`CB&3%ysGfX7&GWj#%uQ}Wu~Wk43Jn?{jK;q|JZ`DL%lU8i_6f+ zXyW9&XMbf|shL&n{+wsG(R+39QPeYlVZFLMWjV9aVT_4VR-tf;uQIp*8kyY6>NPDp zYJRXZj zL)iF((z#Tr08K?6rl`_|u;?7G-@@Bw7N)q?ph7H1Vch*4x!;*X3dx$_)S!bry)H@u ztnzBZlE~Rk>>HP(Tdirr7cBBpE;h5Y>GzBL%1q_Os?dl|N}#wv;{W5*)FrpOcO4Op zQ=o>KIUnzR)9g6A>ZbLp0`~^qz+_1t6S-yuH@r;O_nA%Qdp?fPtee=!(jUruT1nVd z)~mO2li%>;i{q4_7mA@ek0iPz&6lIjk*vx|VyC#->Ykb6-_bAi(qFW2bH|K}bKe9Z zSdqN+F3U)cbj%50N_g6n!(G@qFdI~ zeR2|0be$r?5*Tl{8l1v9e6U`%zCTelf{r9oiYy!Sk{)Pz@7Hu{88Cc$^$Dsce{&VY zSj~NEJW*_1VY7ZcE}&SQO+OGpF0~sll1rnqgi3HYp(;X!F6rih#t*c43r&l%caa0$ zkx7+u*p-(uCF7#IFI#9OKZp(wMQF(kxHo{Ccv`rGT3nWKRmmZmt&IqS5M`u>^kD))! zokiwu9V}TE-KbSyFQ{3ReBb;Mjy5C3HtDud?4Q!Oa9$NTB1vPJMnK$5vX6a++1Xs~ zqYVRXR`I;Nunx_0zfv@q=kY8XxJcIE%%!1B1*9^DB_bbHMSviSZ1+pmveLJY0gcca z=BD8&l0_#*C4+g2He-G~(gjn+v<)o|qMTQx?J})r>{e(%tge91r?y2Sd47zi>!ezX_|<0LcJ}@EZAh)8cL_-yU3la{W2!hUHgUSIaw& z5bX_=$enVrc-LKYxTvTJDdXD%*RFOs?$xs|08s$$XleQrys=h}6NHqT6n7miP-N+h zbmDaC%D^U>rfkmMa_!i&)6Coaa9d4z8dNSraP4646V=S&)?Q}dSMQPZ3>b)`3oYti z@L2i`tu+_&3dOH$C3<(kMph0WiG^gVLxO75^Za^1+<*l;nl$EasF!_Cq-sV9-tUf# z;p3NAbXPufGYgyhMK}r@PIc79*T~YFGK+Om^bRgjVp{vpfR8SQBb5)kqP^PqTc1*n zAuDdQo9&I(N&C({7Mo&5^dNY^LYAs^b48orw(9nEldJlpsy4=Sk66=5@gv@` zE;KBy*W@T(wiaHwG_{H)r~Kggh52UKd9p{#Knf;9#BCs}YMw>Ls+rtwq`dCO@0jGp z=&3LrYte(K&1SFO=z`!{(qg;6+$!8~kmKEOw6Xin-S&*dS{7m$O68b1F+@^-^@Bv> z)?;`D1a?P@mz`JPyZ&xQQ#5*5*3D#KX`(3iQNyb|UztImk7p_><$xFpeEn~^e4wBr z_J6|@|19$dOMLPAf6J2ppHB+=zu*z}|3^G3yavF)zJPs!2#@ya1soJ6hQi%1U|(Wj zzGH+Fc#ZY;J(Iki9Xzw3!q+HlT(VEBijL7)6_r(E<2d9jLi+X&{%r<+6iTZnr=PzV zX50F<$NY^b!|=a&2AtdMYD6;e%Ri|8b9evqx7>UUiJ^CPtGMhI^V1`Zbd^HeXy~L5 z&S6CD6Ae1_{aU1v(4gM~esycBm!w@DdL=jYUZST$HY%Q4d^tDe6LCIxcm+?i7mKP% z(EEo&a5>Y*-wT0TLx0b7XI;Fu1|{@PDY~g|7CqI;-)rjqZ}*}WhQA2@i}3&MUhMe( z_e811|9v75&BLDwK{QXlCz4;N(RKa3lJvr*(e&DCo!x_?bC>wpZ(8C~#Vyf2{*cFy zf8Xilp?}j}G|MKD{^p@Z_zvSPa;;TTNiz#NK!bE^Xss7Kr2yl>DqdIo=9)bul%eA$ ztIquIEzkeWqjT{e;drP0y)}^epRK6w{`CSj+o3w!A1TPIg0sEJPrd0HQ_7k$5Nq>9 z`m#f^lcE}i6PeRr;Z|^$PBu4M>3q+-Eq@qTsGYnC(Mf&=4E}p;>KTxb3w_*|j(;d< z)v53><8A2qXxKn$O}Hz;tz3+!!?Et?#GKsVU> zZ$26omj2#VBXswU=Q_6ubymgDY$N{PKgKk%&inpfN$+1{x9pu#Tg(|J=Z>7op#fa?h75;irGsE_;$7TPe z)1bgc_3rlcgf0Ga`V_f6Fo!t_0nveKN507N9sD~VBaoioA+Cfx2PTF-!A1+tgkC-*l* z%sVO<9HKN2?5e5)It;LVG?*y^x$%b;lhkj>&CC=#bz`fue!WI2D_2|R-FvE8$!KTx zOj|a!_hh@F$=I~IN86}#AEyEm`&@802+16KWOJU#F!B)hv+EO- z=15T(GZsj*4FpzJUE9T#`ylxFAP(~19Ni1Wtk$*3d6u^7LHrv;$3?qHWm}y+Mm3xF zI8kiS%*#XPY>{s61l#5KS_s@-hTT(|ibCSyk=#9#S$RGAkUe3A9+#W=9=N6{%Hc!; z+%(>SkiFJxH11xKxs86*@0$(h85(&pXpVDBL5yuy^uR@1d?GKsLEO8AzJq)mskKmzpNefy0vEi+{0&rbk|M_WGN)N2 z@Vo7zPCz7`?K`2jQN97HGE^b`wca!*?76HOnRI3H+OIsx*!$A1f~Y!d{4#Gfy*uhZ zoqN<9OwxJ;UuXN0ze~?E(vR@Xv09XD^_4oNO>2dR+@R$y4ISNgbGA zTBqU=;CQqSX4g$oZs-0zSZB|1E0*cxeh5iLagC*Ym#`n zEb^sBeksm$6JZ0k{faYIc1dYS1fJO0$9kv2Jjd}C)xC8`3}7B|O!SFV&j9x! znrz@QoWNP_V-O`PZ~fa=9TWrB`{>9f1;M$rcTvBXK0wYFW^u_ZtVuVuwkHEEXv;c& zxuF!I+2AQbtRSA5`FRe6JtD5XO{-5T8=PKRnXlLg3aoyzS&sMm0E>+g=bB5oM_%d5cpWY^;oczq;q*$hC8~A3pkx)rV)mX4L_w)Dv~0sMs0VzD^_G$&wAn9mWt!^&U`*FcLvsb1Ik$ z>0J(*%RbTfTu)3UwW0J0TV+q_dB<(@37BryoH@quo{@_a>s6CmMd6TC9`hwsH$tvh zsdKspq1T;j#pdBs^}nj%rHO@n=^eg6@>7Xc$JN1`uCynp=-1E~dJxIPUFDaeG^s2T zU=)sq@rzaFZHi9dqKW@#rK(hIVf`l0l7;(qK4aL#bhRq@PWk?|^JcZPWUgC!@?w^j zf52Lo;$8{}PW4OX0VdXbVLlqZZ9uiTYr#!^y2B4D$zrmkNW@f=R&f25SBZZc=T^L)vBb-6@F(x5_(yE{BM2HwhKIr@yQW1=o-Q4IR z+={CB3GA|Zg)a-s=BO=WkI~$JAD18ovW5-HaW04RhRJ$5ULy6$m!9y;5p1gF5NwVo zRK$-s*K2cpL69}Ad@js7!)DzqL1S6=aL@kELLG>l)r)L3x-=ypxvf+NB?Ad&jMF4r zqE*n~GwvqS+fL%J_`QK#TPup~<~j^tz}2%HYtV3@48IJEe?3Ytnn+((Z1gnx4A^}J z{IUL`-`39^J)|1b4=Gx~uAU4h<}~QOZ2bY4>V?)o?!{iPelwK^>Nq9ft4Ca z-}l~g`sc6V&gba*>=LkfgLuE!Zl`H>kcy+vEkkNn5G&edZBs63}=StoM^7lkRCm*Uf%%~Syl>0v)y*+Mg@%50FxWFzH4(R z(qlRPg{ZyiI?j1m0V3pyoBMKIL~p%l@l+&kz?_tiJTTL$dt*lJpFJdHg2>-F(Mzt{ zj&4Grrl1nQa~_g8S2e?<_r_ppt*~$vWP<~1Em9`Py+C5o&ZR$_od(t(7V_J#5hcv~ z!2*XZ{WVb|acJA$@Q-XV%75oarDwqJ;sE~W&E*G2bQc=L&Z-{#ou#8AO{1H=AUR!t zvpry@)EZzO;BV6^v-y2~01az+uQ~pBRkbC_URLFaIP~rkcXW2+Jl8ZD`N4o6TfmH- zjIoVW>Rqht9xAXf%rS_bJ*|xpEwWUYdydtI!)Ik zhylIO47uj1+87?A#-)(LzDpy{c&_oOs!)>wwLHE1#BMv+Ak{tfHD0SdXQ^W4R#v=c z?Q8O! z+K#Zty5)@Lj|DYrQg}T|YC-Cp-)rsbPkb5kLiIh1I~(|TRytiZva4q6yPn1MUPg7) zTBypsves?W(Xi^1@Q<^|w6^&`^K=)*{yv0UrvP=V_k~dm#2(fG)syI?Kieo zg=&nV!|R7eMC}hqsu@-jME5@j?xhS1cLd#>=W4H0(!#h5I zZ;RSB^ktyXf!9{iTa*+^FLr7_QWZI}Bi~YKAVF}VHFcrY`T?0pb;}xVL1{*eUW?GM zWz0K5wa(-FwMrO=4PR82oN7THZd2+iEH@usDyQZa)LcL=sBUFD!rjq}34x)5*%TXl7*HkeZ4cyt?KKY&7MqXEV#|plS z4wRkg7fjnuWG+NzzR0|fs8-wRyj{0P5U#k=?6FQm8Qn|TBLv=$(~a!ebe0?J-vr=( zZEQkt8tM*#+V(X)c`sn;n~8ioy^uDp?LS}fybzj5$mS0!#btYQOcmiAelDu6iF|2n zg<9fs^>dzoPkadJCT)1hN#lK^N>c`0&osbb9>7eRZlzTSPcvl5nF@hxl5@0JQbm3>rI3v6R);3_*s{GLaSrU zmcro0HJk9&kyJ7#qpq{nB;}fSdUZEC8S}LNXU?sv&$PCHqI|1!p6PO zgjr3-jhi{QudK*8gpAFZ9}>yJOJ8Mo#8mp2xm=9Q{I<0MKBeWKn+v_h1?aZVfaaY& zH-wyBjXs}3YajD=6fJ`~6spmHy8POqb)CEe1a;M}B<@P-USqIx|B&dA$&+G|fAV@- zwR-4_MI?GtPb-JYDjRFaWJMy8y61QJM^ogbMTs6V%aGT-tIlC1z~uk{YTv}3;L-4v zS#O-(#wA^;*F&<}w@MT@5oWgo1$~bpp~E+0zDF>;LuWSgGR3Cz{ssq!sXUg;rBd8? zw55i2B&8f^Lg=*rv^akL$|R{cyqJ+vS?!yZX(7QmLv~16wQvJred|ge(?%OdPL5Pj zi9>T62|{lwqA4NgDg8cg4nhxdTtjm3y=keBS%w*5p>QCpa|m?QMRMUq&}SU+JDz**36(siV}RRH$l5VS*{C z$w|oHV|uFW%1M;vKE~b@nm!v$aR)686Z~rb>bv=(N#3UCI`QCp z5g}xIOfv+(I`xR)TUbP3pDn{}3~d%*%wc*YyZ~`DK{DwWxkoV?;|^`nS@xP_Ib3@j zeZJAchg#6qv~2v+UV*78V|0pjXsjIE$*(Z$+@A}tsg}m5uNY?=yJXK*<^0*XzajqC z`P+O@51CQe0&Jg!dszwG@~4^JF+4nT@GVC7EwLY40>|Y?1E!Dj6m=(I{a1<_?Y%ia z4;932fx$-&nl#~2trvl#)&!<0-B2GE8-)X^(|c*vC=fPmPt8_VT+k!96ScyP{B8aB z!brw{lp~Es2(&b_+|AsF=@!eds6X#oYbuAwM3^j+_E zs!rtvHC*SU7wetg9vnC53;C&MgRH4_zEN4R_Z~1hXihuH8ZfBP@5a^6E_bHr+J>93D?4Y2x7Qj<`yGv%m5>LF5gPzklJ1xX(l! zW!1%uh)f&a7*UALh0fFHlWG4@UE!CGpo|!|Uw7HjhIUTuNQxiw>)nOLyb|$q_?#Z5H9i7LJ$XBInxd4)d zbi>DY!{V#@oq_L*NG61;3TJtz5(fg$!ysOWdZ0ny+h)Di%P%R-BkXo7&j20(rX{?o z+^O4CW9U`_Yl~v6lOGnDo;70nf)8FP)-xa-2?;Uf)x3wuT2%g^>9mOxT9=Lv+gO3) znNdR+0c+0Z*Gs+)Fjye{#Q|ttN`QXkwR(LcW6p;Zj&T}*hLP>3D9uetlD@GX!ef^v zB9VU>`g+$i`BP-6_KMTp$XRJk*Ig{E#oEUqJ^ zk0@0d%yxrWM`iI+oT4#=)0+b5*uUBb73*Ujir##3NVh(3=I4XSoBm7_OiCohBU%?Z3hI1#BnVvk5??^3|$abCBLfb+_hN+hi#UAAo~CHA3cl#Ev(0!KY^#71uMv34~Tp38SdYO~h^r=PmQ*N0_1fFifDe3hkFsHBa-86l2|Txd;64)^q# zLJTen&W3S~ff;ZlBVI}v>uC^(iP|$D&Mx=+K$QmzuR)<04BXI!Jj4gCD9EHs8c;I- z#n)2{Hvj1_uA;@0`=MIHNnIDS4k7mMMA_L3nQGAsjdQEGbI=VGmRC|hv*@b-@bW1+h zQ7E$h#=);NT|D2JJ;rbe;a$B`k=jjt`E+4dz^w<_Rl+ks>e1jYd4h+!QVlF@Z%bcw^cm~YezoL|F}S8`En1eaMTrav%@^;P zopg_U{B~worNQ-LR+)QQHn>B4-P5~99v@W0NwhlLH7hs>$&iQ;MT7Gc%gB1tn60?JW-V4f*2bgTVGw2YXkkzj#-tPv`?ZXnPe#Hm zH|fZ;oWTYD9%LRe*8D(l1_f&V%Ir5+Mc&boewl_D>qs`pLgBsmsPdZlEBM*_L#hoU zMYaU&B?y*-{XwDiXa@0d&*_I{J> zFiNArKnJgJ&fyT5jE6akZ*0v8Ew`Oav4$yZmoB=OnS$WrCqtD9e@0IO3;mieD)X3hnEy@OPY(lrJu=cuo)?seJ+2l((6I z?UP3o{)y@Ec;K0Dr~Kf3kKgta++VP(GQYSm2u?(IBQi{S)kb6yOb4E1V3e9uZ%~eJ zaRKx0BO+zz_rN;HZ1;3y*-QIHijz63LQSi;$FXr*qv z;KLx#HM+CQ7pD(ugz@KvinW7w@~L#4hj(#i zTo)FL=G#UlE#!uj6S_NZXPkT;8O<}=1jyvRDT;~uteuHI<3JONdGGbtxu^%I4i zuQkf3a`s36LfE*K;`NR&EK=b>mODQDSm>=Rg3d>1*;VidSzg&KsD=U-AlJ?%++v^# z7Y$oOBuBbcj53Q~Y0EkcT&qO9agRqD)(De$L|L!9a+qVd-O@wMr!WX3wf1%+YdXTL zh2cKy>XHRt8V=D5QGz4~Q%;?J`+VP;%*JkR?cuKb!FLiG(|A0yR4Vl`rr&j?OM(O~ z6SwLeWC84m*f^f2cEY)Zi^2N_jf=LIHOqnpRz)QA$+)6SrNKky;q_z|io(9DJTGZ! z<0``UbRNn6)pvgELPbYrl^(_;E8G~L=u_uWs&GuVS3YG+C-@-L_5&da0I{<(Zab$>dsT(W)97wh_aLL%%Op9y~iWLOz8zr%DpCIh! z57iq`@QA-8=6D@UW7tMgY7@Jb7JSs#V7)2Q!*ed^F#BQd`+1^p>~PmR7meAW8;Mk+ zWm5K5d)uK~r!AEvY{^kpTCx9*3&?k-xfv0pV<#_wD~AYj z$sNkf29csz-oFa7tZ})~S(BLSYFO(vy>`b%B_`-V%$fTNRG()VZrzcGWHQzqNX6HRx6L2JV> za!!ykH-2@|ekg$ugkv@doyUvxpY{hBv?)HCMQf@tGZ{Kr5Bzc4Tqf(V z;b9nc4b}^-zBo=MRQ_ zWLjiHpu}aXMCCe(06P@W|MeuqbOE)#55HA$Iz97XsG%hT$HHOyV2)c{$>c}JqJ?*M{$xUAPx@``H(!O=|DP9?SJy%=a*sO$mA zq3U0^u0{*`K}1tEMVs+5lyr$q$28Fx>y_YWzhDRI(_PgGu#rc7sj zOL>#mCzi7%Gh!|U-Jp1h&RUAc*oRifzc7+Ned`fS77^oxUuai7Nv_k%6fP0HM-bbU zqIzt5aLIe){%t9AwO(|=hztlvJEB9r1-=)KgPuL-_7FFSyv7TYR6B&l*| zaQ~e6wZx=2TN|ecik2a_Q#wRtth$kyI~AZi2$Yjyb7&-XQbfmKo~y@}J}r%1u)@F? zA?70$*N;DP{&*37-^0pZTxhn7XzI%D@vjtmD3N~}Ld-!UBEHBn#Yh)C&d!8LQiDpi z2jeC1v-Ydto`eeWF6piEz5G-;#tKH)#hoIx5Xs9rDhCa$

WLJ-eb_7`wu{Jk4&m zCgo-tl{dfu*eL76{gR>&u5-w#yh9Cf(rCCZM0u~M8f_IMiCJ9J71Qth!6r5Nq5bAMbyf+C8cl~pJNtPBB z+7DV&?+JsDLK24+86$xOko(VK%z@URF1Kii^VDvuguWXg|+?;HPQQ`H>& z-D)fU?%aq1>!e3ddWuXwU)zo#Q@@{^=(cMyt?%%HX|Re~*M^O7kCP=rYZ)!`sB&Rr z9FJP|D=u>$_sCiEcQE}LzEh356elTrRGH+?$aZ|NcahH?W+im2-+R2A79)x!FJZI; zEJL_KLdTLz7V9QsHH-|;(L5I&&o>P}`+|CK=#fBhKxa1i>GiV2SkmR9AVrtDqPk2{(%qa;jgc^ z@3c~t8F|cuWRSv<6PJ*;q+xsUfNANR|r=x_7xHrr58#r$dG z8L$*usK_2(U~byz#}k^Z^eG9o-_pw1-_O6h@m7|qL&vZ1NoD!x5zLq!G-G7%G&T^+ z!lH{3pM<}l*z}9WRi9}D#|BieUL3GjylkxDO!|q=2%1UGx5hWmTW1u;jLCBZ;rSQqu*k^TQwW9Jn5EX{#O6a=v4ZFZKtkg_t&h0CCf$WZNaROqaFJ* z0LhHuwCko81*KIX6BH7iaZzI-0GLn5x}Ujx27FK)p_eL8HqrN}S*5DeMr;6K> z(NSEbsogk-c~3xgB*l5vpctXAZzkCDb(4^+C~N06XgBtf>Pyb}Whn*d8DKu|kHZ}f zX8T|UsWo7B=npe@qY&&MYs3=4uto;TBSU-V|2Ssw=Li>MCR=$&5{$_c_3DxEVN;J# zwG}*Yi@4Uc!+^EVHyjiW{|&c}*ZAozzp_Il0vkFnj^Jt-bnB6dNWfF@i_UUOgS~}agCU4rZl?3j-G>?*uO9B;(55r~U z4xE)JxD}O!jA&#}_d#W{ob~ceGUT<$5(BPV2~i4JGb$2S5gp{}ihjFjr{K0a#;M*n znXw;#yBF(rYVq7Bu7&9H-abJ_8)$r^~FciKMhy z*E)WFI8p;X2nb|KOH12#rk9l`XF8jgQ_GslD9d!g zNgr6I!rn#V!JWomzXn~;&!Sh)fQ2cH z^JlO z#ZIxFIglX%a%(4d=?*QLtq_-6Fwy()8CI8>LNO}Qp>#)H7AuKdx~q`mZgL_4=5@X4 zA$YEuq|dCt2bjzq`(DDENpuz8G0t$Qw=h~W#=8b^jnCZ<8l}uXX=Y{;7e&kJ4~DE1 zr;V7{yD6V}AAGn}G~Oye#}_AT^&3FgC(=2kWNHf+{h`dw*thr#0hKC@f8aO155>Qr z-zE8)?2H7YQDC4NluydNiKu;5x=zx~-k?0-o{}C)7?2*0HLr<tHkbQ{V8LiLX|MbSzAS zW#XbgGO<2;Oat?x>5?VBKjM=I^r=hRo$E%H@Xt}P zI$`;$437_Ih&18YEu11C4r5fV6l!u0->?4M8^PgW*n`f2N4!QNkUX~C*Mhd|8CXpa z`3!|H&PqF*^ItUJjM;JD4qHdHv;&8i`bTzeCOsI({nG+3s~~Fl(pc6n<}dTJwcqpk zqZ!%=*IOA$EVNM{%D?pkBZh*y9rmcJ!%=xO#gwW?jPZ#s85`IYnppF?#GV+B?54i( z<-M&N(&U`UB5mKqRx4hziZ2A8N#t6q3h@b3g}I!aK~Kvu{5dVBImSi>)g|uJue5O& z_w2_aLJQBv$+7tU8zRmzmg?(oc2l~b$hU7G)_bV0E~TRm-+1*!y&>Ekz_fmMVA6i< z6oz-LJ4h9;Dq5%hrk0PUmT{qVUx`SuNI%5h6;xd_ea9E4D7NR`X9oMVeP16^OtV0T z?O1VBpyqMPt(k?qjJ!3@mZEjcD_~iKq0T&5-X@Lg+Ltj2e=ITYAvAZ1XS%jxifRDA zf)z;gnGh5?mC0Ln=X2U`ai=<3#(Maz!`mR@qXb!EkGg=ZAvQ8d0EsuL@0#RqTFuBVO`q5br(~y9w z!o=>>vNI>;aW=|(;^g$`^ukG{2O;94Gm8g$=q85TXhLQUiL(UdR={BJl)#%cNFwJ~ zF}E%&{1U>~%jK|+&K|xBHk#B+tK42R@bjXr)!!D8c>D^xa$e;`iLayoOKreH2;DB6 zOa@eEbEi?N5(wcQU9I1oKn)YqVgzpEeL>SoM~3JHSs<5&AdJWnvIoj)qVm$iC!CKF zy|drb^ro#qc?wftYBGo8BP+&?4KRzR6qBkb6>Cl48=sK5MR5jZYSX^0jbpj?yFxus zK-0Am^#A>okIp76!SL%C=Y%#4LgV)3@m#L(y*ce$RT=WMK5o#gacjLHTn9q4kW~{r z3SJ+2(~{zoF26Dz8v!bEhVYm5g){TRI(hgan>D@4$qCH@bi`jDe=yvNPCQ7O-6qZA z?h24NU@0F_+cLe-b3ssrf5NkG`mb{2>NclAK5WtxhZr-=b=!`U#axkh>eldAk@OyE z1`GD~X&I)*Jdo|CM}>k86~0$Q9F-lN7?0WrE?R&UQec*X5nRf(Z(FoTOr)0B7PeE~ zRVmR|1)JkT-FL=&%+$2lxhQ6R=6)_c8X#dwx+9lvcdQvBQR!HTKqzV(Ap8*B{NpqS z+lkp3q^z{QQzBEf5nGtcEU}%oiPBGi{bc9i?l(upIX68;=pD;RC6`Gw)|Yo0bnC`P zuvwXTs|Y>Rh>q+o@Od8vm|S20i=&iI4|=zaQz?oM>=AQ+H+(TCM1_5&wD4hxZmG5` zWob1pRmqCS26uJbDyWzuiZg}(t-rKZR!#~7o&k4wV37uhFCsbW7Y06q+0w@b?~Cu& z{|8m?{8(4?ZhNPVZQD-69Vd;sW81bHHf(I$wrv|b)(#stW|O}8p7Wl2&s~4O`eDsE z*BtYCK4VOYTa~bjN)Pefjk4I=t{LNnqUJx!{~!j-k>$qDTTB+*$8esWawZe@dDryQ zUG;XenQ}m?t|bB20;D<+FsS=6yCR38qtbVyd^0V}ZC7GJL_DL+m{e>+nvM;KmZ&BJ zLQ7sJcjUcKc>{VKvxzU~r}7Zl_g~LwcBBvZ?_M!OPq+R6R0M6fKT02C(+KZ*i=7+} zp)P+I{Ez;B_Z=*qSW(U|YsGY_q#pK4HQTMxjWmjOi#6wjFHRYgVLPhuOJ6!6N6ZI3mW+mon33X>CODqsv}~W zq`0Ebly$f#PdYhoj(vaA?OUxUsdwA*n_7C|DsgA1k}pIz{DW{-OLZ4lWi?#8KkR!B zr!G(v8&mHAJu)X`J3Z)y$mx@E(3-%01RDQH@7dVJzv?WobDq=Iq^@5Wm3GX&n<14^ ztM?%TwgQUcFjmckcz3MSQ(QDaLWQbXZ4MAtUp=l4fpaTGiyAqX^1q#p9%H`)r0Ra! zV5z{FW9U3?6+|peN)^{s(2Wrj(QX-8LAH#kw0Yy`8R+b{6==cFbf;&W*Tc6xGAs3^ zk-jMNgh?-~ES*VtQaLJ~l4u6?L41O3o_|pLMA@XjM&t{M4G2ZbSHgOidpJ4?Qo)De zL)?xe3wL{}YRr<8OLRZ9bF}K&V)MqCbCrMTnp?J) z?fihAyJpmj@iNqM%SuYgI<;qQ_;eiuQp$JBH49erO>Q{l_ywKSfVURaaJwku4j z_Q@g9(mKHVuordpY8|44A9L_jI#aabY0sC_mV_r3KlV>-b?2}0OCsDJ86;Y_p*l0w z0M*#5RY$^ri$jbOiBYQ76GP|4=ABzsc9k=yU6uE2b}$^3r0Lk77x5+M3@P+`7Yv5PCmXcJUChApXZ zDf;~}=n~5gsriT5dB~msyW9vO-xI8ha|bxy0v%L?YqL1<1dFBwaxa~ogs6N$HHz%Hf) zTi+~xtdrvWRqDS{?$5to>XjIuh$>PKv=n}78Qfj1q7vb*>mDDr#v`uE z-Z-4N1F=L0lVO$QnAN)D38STMj2$XIpGEK~6m9+JZ&a?4-r~i5lEtlJqMfQx3Cfa8 zE$4bZSqCc2h4Fv|)d@lvw!D%Zm+Zgv<8lY1VxbH?rW<~ zy1eK8VcE7*1uHHGU6zvCv6#GCAL3hwS1OB5y;+y)Io+L5aqFb_?rDiM`jkjY7<)GI zADRO~71MB+N4d{LIuzH_7E5np1$|ThK`dap$Zikf{U!=}#J6y1=k%z9sgH?}BrNOv zBrq4JlU$tC{ycpz)ee0W^-T{avp8H6QKryKXE30&AXy2W#vp3#-FX4 z$8^=TC3~&Ignj!r`@P`*fIf$kK;38UMFbRp%7%^Pe#+)+ayC$aU0M09xY5`w>BnrJ zNZ(-VR*bvt+JG^nHpItTV@g~4kGdIS_94!q30DVxOrqm1N=rR7AJN<;DP`8<6SFuU z!?MMS2GAQpLgAXh?=%mZ-Y*(@qZ?mcIuC$I#RJ7PuEdxv5INZPM3S@`%7#(a$SJ?H zwgLu}5NGf{$TG#wEIqO(;2y}GO}cncf>&y#A*+(9Tn$$Lz%MPScX#?KcmE-vr!#lt zW?C1Ce zU`B0%7!yi)Gp5xq;#Nq^Gb;t$_jxY=PoKPTvE`L^Y0o7(flh53eSqV-Ei2(%zSNPx zEHB^j-l8mHN!s9QZw;GKKFz##?z1B(rn(^btw4U|t)kCJ?an7yd=+P+R#fWyDkk&o z@l$yiTL9xEFb>VM%TGdG_8g!!>udvawXBTS#elz%WRS1gZFNuoE$00MdAoAGN)KCc zk-jW`zGy-pTD^(Ba@2nC9|RA>Or9W8_Kk;w3kvk7494iKvUpzC2Td_?cSw|vMw=6f za1}RUtfRP=`rU(R`cs-)vrM0O<^+x0vR{CcwoKiG@2!R>q@P|A0z$h^*H7cFzB8Ge_bSQH)72x(u=xkjuvphwiqh> zI)c2DSPkdg`_p_SBW|5JJ(fFah`Rf@MaUJ%#C6eha;BqA>BC+B7|O+6ZyW6B_3GR; zYdnoDZ_k;x$cxn{B{|uSy$Ydi%f)fQF%e|tTxhyJ_l)t zUypd$gZ@=1?;ihwJj;joN*)lMyL2B4ihW~gtVYNA!p@}BGv?HXOwE)6b3wq6nO^aU zMhm$l%?7AAvrx5fRhel9?f(BeW$zHBUdb=C1>SZW(c+U>_(44$kFX}zwsH%Ixhv9qog+OoG{}noSt~O*o{ruqwv+>_ z5M22xH7_uH6?4N`n&5ETswr9Uo=;vV00p(@MmXKL?zCr{v*N{QxnJbzV#_vus=rdd z%>#iOZ2GXS_)7b~?JE56YBdMJHLHoqI#RJaF!!mmkqj|qBG?k`*}mte#Sbuk|R;T2-65^<^AM*k)9t3n>ytY^Ey<;{L z+dfCc#a|VzRT*byDnng19pIrBaL`QnE%;g889UzmzQ6th=7R{ngm;9pA3mqs0r<~k z&G`fGYnOZYvl9OUFOmKSUfRy1AH}PgR?nVnU01m$8~{XFr}lJPJo!O?$w|stu$rgi zT^iddos~^k{=hO5%vSBZLf|#Fy=Q(EML#4$4p-twH{~3!*d1By?zZ&XtZn`78y6Gu zTz25-tjYT3^7N?1@kv3|J#g{o$r0AfJ3GE8WHdU{D9^>g2@lVgLf5hJvyW+)HC3Q{ z2p%xA>~xhGeM>{pzLPRZon8ZhW8D>u4{Dj}{TRY(mg6sf-ejx~;h(!HK#;1$O|>3x z%TD95ui)hAM7vsi7~a;Exvb9)sC)+Xs6@%${FSTQsf;^~iQJfbU&hY(b|ckZcx8LL zT#cX*LOSW+i@Zhp0=@vR*E_dB&Jfg;DUX3Vs&DMZoluNAR)hZw@a*{Xuvz6?O?Nst zSN?<882rxyZPu0fdn~>H(#SbGw``m}*p0tc-MyM~nmuw8^L_w2DRbWy5Gy2r9>fg`wFNB{XdJT|)5>_I*T<&U{>1#SBB(O7CN=R!^ zj!4B(zKd{{mR?`33C8HXq1a_#mlN7(=?*G0&>SSx0K?fTIvFawTCvH8pLqVrH^zU0 znYh7yt>eP;|IfgU04m}e1#D!f}`gb4A z8m-zKY#9wft@>bd-`L6fmA?!}%!lJBragAq%#nPWcY=8|@Kn4yIOnzx4@@&s!g=q>S{VT1ZHW7hKZ+7^`&b=++8al_KVjkawkmaf9|ZQ9fyBySGagl?I7y zTkE(Lyz*|XD{z)XvmFU@O*`RF%y512C+F;gB5C4?M|R8({+!u!F+kX31<}bN54V{b z6yDmwlIr;5j@1AFr_IHG-fRKDTR>fV#y^i0+lNb;39Ge}qd>s!W0)Ev#aQWKFwSUD z@!Q3yC4LW%CV6S)7lZ-kKx8idXtiIDO|}l**ma1PUU}#-uJ}oJiXjW_-d1*92zgdg zp(xdd#zsBRS|EL zzastDAQizVK5n~ZcSJ97`ymLmEb6qOszjTQI+(y*us7?^tTtm~#UF*~Z)3FIVN2mC zNab%=$eER-o*kGj*>d4l`eZTs}wS#Ciol%FtGh~K>{4s1X&SZ=m&`pKf!WdI2`Ru9jmTat}DM6>y zT|wfdV2ITp(OL^WL(7g84%rp9URP7|sFdqHV#6b@qx7G&kS)Z%oI86GszOh1S$>b! z&>929q}|2kuELvSea|Ya!{7N5{DG06N)|JfaZjX_miyYQx(UGcs#^etxOTHNI&rk9znGAAeHlvC*HOGd^B+O^WwGq_qF zrAS|YtF$PLe}mqRvYxy3^!q$Jtp2Bo@X4?RMZeYS>iIXXUisQPfeFfMVey3B9w+>( z7!nN_4?Vh#{xj_}TYjF)<3-FdTI9^bY*Wb65-D?db#8~s9#BHH;kiU}R}QI5YF+Wg z$}kb{@3sqHyLb}HaI)({)inLmG)BwurY#=xm;HQ;Te&tOb(MXkfvYt=pc4~75b4kwEI`IfJ1 zF}&Dr{WDGz0>tENFrDm}0`?|ojVWRo*gqONH#x=ae4*)@*f#x~e_FGJezL2Bm&19Y zf>LT*Q;=Ofh=r=sxxB&NMo17Zo$=_%{xb<{O_4?`kH6emO5%n+~Wx7H{J3 zqz?XoxKN6#OUtci5;<$yfEW5@Dl7LSUMm5NcOve)6}){?P>*(Ktb~bd*o_tPjn2Pl zxK*mof)AOa3R!R^pO)F$x?%%)wo(Wklm#}CVORcdm8y5}xLsd`N0+*pBSQIJ3R=Ti zW~++SD`TLP1a#@Mf++qQvfYn<;VsJ5hbm03;_*^_#m<{hE>O2% zcfetU1V*c4z=vgP4c{e0{zX65)D@JzMKSKeJctIAr!Nb(85>n@|J0#L#EIm*X-b2H zjHouKrp8b?lTB7Vt7hKr`HKp)bO)bk#S=^v)IfL*baL%>j~8?5d~(3e_W$_Y|5Fmi zUqrOd8j@!&d4{Q+@fbw`9BZ(@&sytB3=w^{3>0xo)~j4)u3nY!EQTEBbOxP`c6}Ja zKc}$e?x=H~Sj`94%oE)aB*g+@UpfV9gQ;KhkI#}jHl-7ey7TuD4&pS2U!Za_bLm_p zjl9g0IsHb;Q`NqhvPdxOz?B*DfwOMHGK|40dCzH*6m&GOJN1-!V%BxSHD<2wQtYB9 zqtW(nY4eiI&Wun9&fSKxbJs6uK}LO8Qnn_^2Wkhr_H-9x`>y$O`WhDc(P^A30VD!W z+8@CZ;4yCFg@DhxVejw-;jJ#$$P2VKVSD#r@fMmo3X*B|B~}^E`?moQRwmnax?%zU zqm%E!1B2<6bMuSjBmi8&HqJNro?EbRwhikN9bU@3wU=g7?c^!HeIT~h+q{xSNOvmv z52A#f3?TSvB+uHA} zE=oeG&QckJB^6O;-mfJ7Rpe_YJm{vr8k5{S_T(IV_cEFK^%$hmRs8BPMQYzMWeJX- zlMt4aLC(m-5a)IG`H3R+Lg4mdABgp9t?Y#4V}3>9g&w zwfhf3s;N(BtxG3vG74EvcryXqiKtHX#}%LWrDiMGv%@VWBuesVC$!h^_jLM6Y@-TXal@{MBg+FAm~ z7DGcmNgcilB3Estog^3UDhOwXKAAQJ4;sMwCTj+jgDG2dn9Ti^St?4G%!K$9&2^kN zdK)=-x7%y(fQ5nW68^Bd*?O15UGU3HWimOd~kNV#( zR@na_Q(azM>*`;2u+6`gpW|Bd-6m*5jvs3aA3p}rp62yqDqkT!ZaZgT<%7uywxOj{`Z;i z{qr>s|B1n3M1%PH^$R2e13ypCiT+1c|t!P(Hn~ zG8P&)bD%+UxC;j5X{4(aB`K?M)_w|7_bScnHT zXZ1Uuueqag-}Ek@oZplDW{lB1|0aJ(-Anypv)jJcBCjeeAZ=zDRWC9&mwLx_=DeVt zK@bn}8#}D?kB%0=%HENl9fZC)iF)G45Fr!!{`Y{uWRE9FueM5c;BZz>=3R5K7J3m* z(jACTv|F(FzVHtM_-eg4S-Yz$6c*$562W=TG$jnhm=xOigz+lB%X7MGxg=t(Pk1C< z<4)RX#gv&8J4T71b2w0%)4~8w_#NO@#?mv+#LCSS1Oxh;t|+mVWu=2C_6BlL6OB3z zv+J01Ssr>wO!NRwh~&58SePcxG^{gyG~`th6*(53*FQg;oXJ_?noVN413hRKe>vAF zh&EnpR5p^`{gBc4^PndmnI_#ber4oBP_ap)a3mTXr?4w`d2eI$KEq3EpMOvqsmI%xXA`ksXU!RN}!xY=5txFR-AU4t4-%}Ujw&$gJMXI7NhH@1+%z63cFP>%>!#TDm*ylqGDxXg>6x2UP_W*>Qc^N_wTvLInIf=|KLsNo ztW-nR);NkWMjTOk9PTayD1fk9pIRFqKD``8vmH|4O%T?$=1q~j_75VkxLXjZ2?tP7 z%hz(Vh^%usW=BMxDgEdvEXi)aXe^|yF8ag}>TPS`QY8#I)Tg%Ek+QbKYGpn~5$7K} zNnlbvbrE@1yp#ryPo$iJo@Cbe@xm%AChtPKa0?VOEd(}+U$JDvBSo{oR^H6#Vj!s? zu)pu@TUy$WF|qM}s}T22Sw5HQzuAy1fSEYdiNEDRxs@*NhLYD|i|f0r;PYnui&lCx zg|*%j0Ta#t`tsMj^s&%+prGrZ18$U_Yr%=!Sh*lRCQQn zh??c`DLaFlZXnafQJ5wk^pys(N>hqEJX;z{$5$m;_|@%^P?qnaoOYFsOk z?fAM=+>qjMrG~3>g}`>vG+66JHNIU(yZrEGv$5>ga!yh>e2s$zW-b+Wo<3XLH3$N5 zGOT2^X9bspf^u^p zRb;gi&RUoytyf@h#3e6R1_e$SZFI7uP))F8J3l@O;t?#apn`(Ef^u-UToMqfKA{XJ1-CtxcFRVh_&qTrMn3>G36EYrt4#Nx;tp zVCGl7FOqET$_#sU(k-dD4?u86`KQ!OvNz+R}-K`xPgQ4SYD8DZWLpbi(ns(^MXWk^0{E zpvhb6X*r!jWv3{&7EBVG3YbUE7>d!<3Ma?z7@YF%fHcXP@qLYTvH@C$vQElQPm104 z$%|I@OngDO)pp|4I*SrCcQi=7l#!qwjoc6=`ah1A$2 z!leKKdx3J`wEWyBIX-y{oQ8pw?8?JMYM#KF{%Yb5C)$u3zWC!LO4zxe*wB4}Cs!Fi zchuXK?j60(nQ``@)xN_n=`+e;0i8weZxon_CkBHst)1_dY;v{R;f%luuP;A)s>8*gMwiU%gQtLYaUwPpu_T%~{|2MQoO|S?y;3fc^l9`B4!+TYRvoPp!X zk&!QkE^-#X#J&^@@~M~rGOu=09mQc1o)+Vph{%JZ8Gzg!y(p3Vb_ioRnV{KiLEYQU zv?&fh_+hE&kstzSC~m+=k>6o+bzbY-dCD-DDDD4Pwe)MvZZQR(P%h$DOhs2~qM4kU z_=Pa*$H^9UwcKC7eT^RSZ!Vt>2G$?B!P27iUaMOk_NXfoHq!L_Q+(0m8|8)+qBc+1 z^lKm$FuOqs@wX6?BUzEebYCNHc+N`-x+)W|XkX$j%=*{pxzudQ!`fd&QTwz5OtA&| zp>lwsOw|aG0~-PXi9rawPi#`4N*CGay!o17_D5o2rt~(;yCr9RXq8s#?s6Q3 zJx8rRD((AP0RE}W@X8@QM`#sw!=BzvYb;HuFa#13%3px%#^k|%LY7Ac#j*kLH-uSDN4OPDfbfQF? zAJ~JI3WrL&RJzt%YqI<*-IR8;ZT%0!N9Ehx;nKbMCK9`}iC)}aEjqXeKz|zqNyNcY zjbT$A>8@LA<}0x~csr)UjpCT2*Oath*#ETu@|16}g;x6nA!YzPWEUEbC>Nl$LImAy z;lQS|h>n+sSx1p?u4wW-Tj|ju+(Z$|xS-c9dMaTC-Ad zk?F6+WTHHgIs{}qiDWrvFR$p^*Zvd75UO1~5zcfpoM{6qUS8i7u3v1HzN9D_d%(vY z64rm(Ivs;Y-2_XiY1wrxneJ0R%wB@?mqJI>qgb{ES<-{uJ2dClnH;8A`;NA*-@YoL z8ycANnmK})pYkLZ_=NEwm7OqJ`9M7ED^f*)z$Gfq5b=ij&V}*8Kxp$YVcUb^gFZnu zC_Pi%iYd~@>;+c>$h4>Op^8-hlP_1Q3$j1VJuc5obZsC+2D-7s7lLR&V#~#gfPNKTp2S$q5O<=@uf?BQ{UR+-D8$ z>8bggHSIJXM-iP)`|bb^_?GgpT`HOmh9Z89hx9fAtkOf~N5UTd)YFuv+7oiQaXS-_ zje9h+$e6?qj$gR-Tvz~d&+S{z-lo87$Ll;a*zaw^a!jiP=TL1SUF6@+81jPIcNP3^ zxOjv|&eX6+(V5p;Z*8UlJ#pbV5giGlpOGyxIm0BnyfSm9)q5s86w86Gr}l6V;eSpj%aTt)7_WriFqo9u@f z^G$P4X<2WEK7$D43stb1i;XLOTdk##lX6}JRW43R7EZy?s*5$ln+@ zLvg!)NGvB^Ndjk(o_9F=>jG{iXm1m?c)TqO-7A9F4{-BEag5gh$CH#}^YAdZgCKUosV;5+05Z zX8ZvbFF<Qfo=U`SfE^clla8uC)Xxg%H5)=*yxUi%hqCP$3>^u~{3dpI%CR5=9C`NZ zS2uw}Z8G`QfDoB!OS=MS7TX4DfbC&w(tv}&Z+KOH=v#Xm5j3-Td_oV)@6{NuF;~gU zY+~>qkdjPZKF>6cgdLa~s|L~?0@du=0{ts0yomwT@~JW zh|eKsHk6!w$Ux0MLfa5`rXGz?t9tDrFV7M@qOFQfDxi0S@LDrdf7_946DL72QT{ss zpw|{}j+Z20 zm@W#qP2UA63tX7C{>A{^a#7be6U=R82kdSVuq(R3`Xar;~97j7fp!l*po;Q&ouCT&6Fwh8tQD1X-^ z>awliSc&S+oN|?GqHPz}2T1l@C}F+Rvaxbs=1hX2l3NDXH`p zcq5kuN)fUkvP4vE*W!Nux1bazJz4fsBJ?60#*mwTE0?dYzeyai*lNDv8kA?Uv+HBH zP?rfjOyjOpT~95_t2F90(*TMW;E}0pjK))1zHhDMiML#jUCfOsbx>Cktk^qD>yX!} z7mx-rfEC10!k)h3cND(e53j?D>Teor63(hO0M*s>q9$l+dxj z8}zOpT+E2bQS1*`X{{5w(9*d_YvT9RlK$dg*|qU7ZPl(caxe7+%!86?P17z|Qornc z%0he-bhfpsbmhWf?NH+Y3Z?p(SEcMnX?;cFSCmKPwL9Rw4z`v=GnLREp=JMJDRPUg z;tE8*rJ{j2(~!5Xd{+K^0}XV_iqAel*IKS;Iu}L)jS9kPjtMgDR7du`BZb%H@6mh5 zIyYiah-UTeqHG3F&m6QpOw^ei%~9=2om8dOyQ#cDZa{_J6GtH|2>2JLb*7BpNtT+j zB|Wq&5n3fWMs7n|v!^AD$;rcc(X5Ldc+iF@*Wp9Hwa1w%fQiL`6d@N#5Kfh-VP(6G z8BXA^4O+qLzw^YP;Z>v$)7#{Za}{( z>X7&Jkp0QzB^1hbClS^qgjq|yON2_+RQdfyR1VwB3`>+9;dE09t3WhN&|rDE5~M(- zVk^*CGHkHMaxbeN^Q7)qsFOhevf;ZNd>>9NF75h&et5}(gXYnXADuqo?1lO84Uc(Y zaM_s`w=H(sm#)8A)@&KpADWZ<<5jm5ggjK``T-9{Pq@*O9oRohM4Jn@)zWn4SFX6W{ zA*=Oe(1q~fSJS967yjXRy?<-_nz89@#l8U1k_$HXPZY>c-*D8A;0n%6%y|yiu>(}O zok?ZYf|olfpOnvtK($m3JXA0|kyGoGk(IV);k!Wlog>(iZ5~2JO~A&u zaEiG_F|#_xA1jp}$u;)Sm=#|PC-_5Z9k@8C3Ox8wdt2-V5#@7fc_V3JnwmkTB&Bz; zaI=;sn(urgu|*3MMPjiD1(2^J;u=f8Pgbr3W2m(A1X;T>bXBChA85Ew$^69}grMVj zZjr-pfsX_;P{-VCY-Pw16U?D0?8!^`cu_;34R|?E>Z?x;f{OoZ=v%NDyhx!5$q*hn zF5yEx$oj}GS@m{=p{OYVF@*(PL_{o-LKlq-lbO=Ot9wW3MVOrY1yB z{)t+Lf>}W38$sj3S^`^2d)y9X(?ap`A8;@6jy!%rRl+rmH0Jn93i~R}(cTk_aH(Xc ziY2P`T8vpMw0nvuX5n^lP}Qy@qZ3BspQL$6^Zdial1&@iHb3tBIJBCrGfP*p)9Vk^tt!W zdVP@zVWMQdsubo!nELXG4Ht%8qRO<*hF=e&Fw#nhh9fy*&D{hLh)o)oYxKI zT1i)(Ru{%~%@gCuasV^pM=~7G~f@FZS z$&&zTDibMJ%3^W2cBsCVA70y4fJ!=YWB|SoQX@Y|M~v!_;fLtHy}r;r_IAtE1gr;N ze3B!LbaU^Z4nll0UVXC2`NFjgf9SigZqQ^`rjYm2qPUK8T)N;J7w0MF8NOzh#$2T6 z(CuxeK&r)CK(HvqbkE8#zA z&HLI;#=oZOhc+H6&c0r$&O0}>pB(y=NX(5!t;I)G4G5?Er7$R*-;@3MSv{z+29J|u z^NmQDW)Z`rbXlNuEmC97OdX%gv39O3ukawbj;PI)`>Qb3QpnR@cN68H7f9BaeX30^ zcb$aU5qyjer!EvS-Npk78!jpK_io#s@^R^owPf2G*~X(Zh$5e}SQk9hk3nO=vWgGR zA}8J%Y)nXU4Xi0?nfShpv-pnQF>};yxweTD?h>qAP_B15NK9Wza3sp-q%%OFI1fN4 zsD?|ZprK&nPG!AZM z=5$sO7yZaT$vfn9eHcsndUM+FXIEja+HFR~so~o?7O|G4m40Bk_-tha%f(GfOhFKK zUD?njJqsafKseTN*|tBB2oUL(5K52A_!mx+Ju+j$FoUzv<^ld`-kX&ux;VwH(n|ZG ztXlWm1l2ap25JIRcBWfwQ~PeGvdy)OEO9h8ZdB6?aYV|7Ir7q0?#ml8b7RR`kj{j* zO;PO*Qwqmin`8mYuyC*3;C_UO@zNI-8?-yr7JvT?R7RQ4E#S>YMoW0}N zBQ%rFV5^HDTA>w3`*+6vL+sd?zJ-@uZTnehdmd8U5=adkwxvwZbpp--6$3Pbty{+C zZPS+5;YP2`1$i&Vhx@yHc6BDT-;gF^ezQ^!-whHeHF!0$y!Ygl1t8_h(E+CI>8x|# z+|7$R9Y36*$g2NAfMZe$8pn4%IHP}upO?WJE3IjxumBjJL>HOB9wjdX5&gM?RE1_) zco)_km*B(_#{Tlu>gpd9@+GJgP?TK^Ip_#aN8J2K6*)9o`e;EC%$)TlFbrJU9@u15 zU>@ziXoYDeW=Mfnn!#me@6m~iRN%2hjUqcI&Trp;nZOs!hOBMUY{n==Jo8}x-fLfC z9hKSeiM?%JuldSrmn7Me;27{VDD#=Rn@%!yTz{Oal(ii6GXiqd+fIfqH|kh=75YJQ zGLJNn8|#Cy21Luu_=w-?=!}Sb^cGQ1^QUJ#%Ya##2b0kvd6hXzkj`XT?1)1YE0($S z5SysX6uWJ3zB1J{B!7Q0V}ik3W}t*km{L?1%Z0id>H5?LzfCDRw%|062;R=Eb&baK zV!CdaX2JNAExRuI0)>@3{D$k%1$j4b1Jv+VxZx2Lh-dxji<%2`3EUd?3RXRROk~QL zv~7hpmnGtPLbqJlvh&HPFokNdc0ne(r3KhARv6NfV9!PUDfgB* z0~X(@g{lc*(?a;RW@Z_rDh|5rEAj$spO7Q6sU|p;@H^*+Ihe#Yj@N_s zO|^Vtb;5|lR%#}NWYXd&RPL?=_l=p_8ifrH!9Qk8DLClmls6^l9Q8)fwWGmvE@X=< z-Plglofw>Iy0A*Yz-3o2wjQMD8Sq!7=t z@>jYoQLgM<-?~a_Rg*--q?)wxm@BdA%&7KiWDd0QeQ0}5{}n?>09EvZ%jSH+t3Wa4 z(YTL<_1krJ6QIn}exq%?Bjl(vSlU@3Io?Cq?>3N;FaT#|+ zH{LiG=~iW&B{m^6+6lG{pr(s~=3*tzR2YH$=>=D14%`n{Q3elAL58k$GS_bnNgFU9 zFB~2mF_x!`S^`!a`$mhFjzw_ z4u4rg^#q>Ep$E#>3~zQvt|)!8AI#r6ZRH3WA025S-qoCEFk1VbkN@{_bxB2D*i1cl zx3(Rrc4-R^NSq1wJ-3Uy+isM)<`oobrIAL&pGa@h^^#Na{dy)tv#5Cj^W^jFIXj}g<<&6{2@Tv(M{ zi6ZD&g<412aju9r(`cMQX~SZ9)0CO9!&ZA@xQJ*G9Sad~>AfX0X?GAKCTS;bCdq)i zudEMc7hZ_ym+vFhIQK9a(BvHUG{d&a7TZKQywq4w}wEdK36G`bBC*&kggzxCK zHyJ8{1^X`=q1>Ei3JKzL-h#M`#&F0NwjaqxO|L2}a{SCQU9qxr{~(5;lK41fL*)FX zlmCFZ9?)_N4+CLQg}`o1`&=@67HFxC;x8n9h+xoNV2)g;Ps1)0wFuP>@Q3$1!k*o& z@~97%Tp?R(ksL>t{wq0;O11P(rMNvcosi+PosPlH3&s5o)KJM6PjbI7vy1kQ7o2L@ z9<3J6VE^$bA8h!Gz8j21)q2pO+mb6vMC}0P#o!{=iYmoJYiLl8sR&0Ii|XVw1Ldf7 zZG~@>IhXE7NPCj3z`+G4*A`Er-60=S($t^ka6we-kOuhU;URQY7CO)NHJAZHT7Nj=AjCv9wU8xS~;%1$OdFD(VDPiDA=e>F%L zsxF?c>sD{f0?(c=vgBi4V+SfzkO1wiFzo{yO9wTf2(o_K_^lxr{Rdz4v!o9gmkZ1B zBUOO9eLA(Wd}%(rNeV2tjp|o*QhqFSBd~-i3YZA^9vx?4!>8E{aFI#{cRK8g>nYiE z5T(Q;?*T34WBA)~1M|XK^O~q@;MS#ETs2D2?y(Uhzc459aM3sK+bv-)K4iw=da2fL zT1^3&NXSaSVr)h2dHLyA@YbUfnA$tHG^@V?f%IpI)rh4Ag-i7lk)gMc+IfWGFrat>DM zrX$nO@sQmvv-Z*cfX~ZRI2_{o6_8Y=%f?Ssf{mNWSX@v?F`3IO@F?XQCFrMPRTAxE zC-R_cPNE30g4BNe+UkyXyy5Z?+xO;Rdn475(n`dTVM(8#{#?zaQBss`UVcu@CwklC z`g6{F>^J(GP;wdkt-22cKbOkzPpA;%ADfL&wTaGw7izEJ4(t<}8#T|8G9?`edMxHwsfOdJ7|BN*f7wMNZU6Gc&{QO6Oblvs7126E zV+9=0oW?H}8h3KMr%D|w_R)BahwmT{^0tQU1q4xYxTk$Nan&*N`mJaYL}-AIB3R$!t$V{!QN zH$CLYcU!3q=9Y=<5SwkrHIGU>LvP{?;1El!W#C^+RXev3#9Fc^(v%0PtgYSg4Qljz zXnQ~_U9YKI++OsB{tLh9kHWJe<_KiV1o{We;p`1X=VE8x37?|^EA51UVUn4kQ6lLx z``*^ubt(qG0At-qu+_(S0;_An^(?`dKpGc0UkptCKL}ZBLant-v+I^4WL2iRbJ{odhDT`eLsg(cFsOK7=$@1JAc6HO?le{5P%3bX&3(V^O-ov2_j3q3 z+b8h&o}kO&wRqs1rgCKIVC5|zxw$6xI zc41b@p$T_LvAOjJD5ywzRv>C}%*vdqAYOuC$j1cvwHx){Cd>!VfQ0%K{&|#7f3B7JBu0OK z-;wnsX{Ir}-kuX(qD>iODxr3vXAPiL*g49mtTodwrNXlk!)}WzI>!Z@e{Rb8A4J|S zQJct;KP3Mk#6$i+0G&W$zsF7`Z8t-erVWJIQKpdxEzZ(gzQVp4M4u?sD%~6Pp`@~9 zZp;n$SMvGHRAUCUvtv{a&VT72LN_McIMCF^wz7!qE&9Yn{qkjLV_joWMXd&OCuCbD zGI88aJdJrmxRHor@R@<3Zz*{n>6#lsNflCCQnC9NoV}$QjX#?N&SYXx;JHZlV_(+L z!L&kRSt`p!#}fRCbo6;fb&RcryuHtKbex$Rry_OgBvvToa_=v0c^2>7oXk|jtPF-O zU&J;Zio66?nMh7vB2ASBa(=D-z(~!n&4oK`r#YAe{xV#me!|F z++-%CuU`F&dFBoc4dYb1q8F-uwj+lF^p^Xy|7NU>(i7rSLm+)b?nMbpHURSAgbRpOEw;3&f2sV%Mb98Na?wLqfa(CXKNn zDe=)QkHGRsY7mC$8W5XK1zuU5dJTzDD~d6jgko5QWYcp~LbagQhD7`FY9c#O-3{5Ade^W=Q>2Df50uf<%i@Rnp~#b>(9bGtu^ zZ{d(7U9?rQC0SwSinCw4McLIux)yZ>#;h3;DW25mTV*<559pe}*8)XMI8W9IgB!G4 z`7g+^B?jM+Yc`JoAYNRcyX08PB&p_7K*-VS{{RBH8i`K2A^!k^UG6)`M0j-3?na;C z3xL+5O3Ju>)r$ghhQjvuCPP>*6KbLD1-P+3#XHNv5N75SHRl3tc_$pW9vB(lg9bsr z$&8N(Jf5{z`LWJOT&{Js*>zUlf!DgNm%~?Ac$n~*eDQM#EHVD->HG<6!1L*3Pyre z$n>s&PSv)-TNK%n`B#||q#cD+v5bB8qf}CVf?mYUA?sFMP z4CNyX*)WuBkI=HVJ;OVTWZlI%6qbnX7Df#L2$qF7!V@_RqO>xr9=@h`^_BW{inJAw zoL*P^QH>QUcy)M%$xzuF6 zb+l-ZOdVoXDVE=WSaaqS{EYOjX358QMX?ZVwqQajd~6IlAMby zn2_#bG>G?sQ!-m3YkU!5D!m&awVRgmDKTg?ecZ37BG(MVIV~rST*$Vz!4E(pn5Lk+|5EsCsTji>24#yF&E^hW!YvBQ=XsUg&i0U?RD% z$rioH9Zs}E^pBQ$83hvMQbu-v0^33aNO&D9u_XEqF6v%{pEu;!>}qcyi*D5SW{W=o zj|rn9XhMM+>pCh+)H!x-1*nCmY>ozgt?0zGVKnD0#4pj%vSQ#;(xY6~y9U+d_~J-! zcrG02UZQ`XFf28jl&P1y9GX;)O*lW|=W00kusq;PzdY9@a5&r-Op@sPyy1tQc8;=*nFFPHGBUd_ipS^0#Yq-_544^iY zZ8Yev+#2Jgu{Y)klXCeG+Ax@%-b9F%ipCmmBLKV`*^NWoV^I@)&41EdZp3n8Ym#VW znKE3N#^Z)m`FKGn5ul~1LhjN<*nNooB&6mhCdc2S4&R$bPmbd>?-hN8 z5@{E`TX`c|pzolS3nSZDlWIx#{{TnoKikX*pO0GY@FnefQM4o%NoXriu(CH$ErW6w zUmgB=f;-_MZj(+5zxt4=o>hCmb#$&t`4-NJd2XQY7){!K%55~(P}=tRO4&{9CM6GG z{{V&Eqzk}{en*2Iz00-0c=Ah)hJz~VcnWH>(;J^i6*X2^WxH?hTJ@lz-@q$gKa<-R zIl-QzBAZk>V-3UsAP7yRwK_Q17grRg#GB-?=KlZ~z5ScFgv6G)v`mwqj zEMmkQw(M1ex(aPO(g*4Qu)at{j_@*ryY>Au*+R-sA1JfaAxTS3IYs)A-gs7{UD)6y z$C?U0w&+2dO4=z$tO+0Gq-6~5<%%W(KIygkNVmv@^ao;nGmpvkEReVpCI*8tk(t|lHsl@|B7-i^wQ)LYcK z<43^q#-Row5Ire7icRK7dBWdeE^8VJ(AAJ>`@N(+(OGaqO|#hgO2SX82kKzJgjaGs z>H`ak5gDW(EnEH(+!YcY91y!?kHqdV!0uf(ATG){P{NRjO=OnVQ2UMU-tv3_?KsGl z3q!KdK;V&7r*4Uvc*U5gMM8wF>_mMS^NuLCWAsXmQauC@YSU1Q~b5G1s3sHMmYplc{ za+s=dd(NoUvM++yc0$U#crwn{5v1#a0ty#sGD6a*owVqO+?6e$3!CJtG8aMr#@l`hLQF zCnz613=-7tvJ)?GC-PZL{{X?F;xt1-+;R9bc!AqycC}=rq=!l-XUJv0pX)|nR-F14 z{{XS{LgIH+9gi_k-aM!B$usvx2`Rr9u`5m^{6$ofL@xz#-;#J<&@+aRJAi)zaGfr> z2;#>dYGOiAlJBsz2MN64+_1&H{{U?|l4Qzg)A5E!Ig!N2R(N_yw7~th62_yJmR7A)_(}QT6PsV(VXmzqoK19x;?V@d^ zXd7(uOz=cWXJ6z}-`XxfN@Te#sZhGOm5I&W{AFg7Q(F7OkeM+hZO+@aAE^v#6sY)z z5UkCaML{arW`tyI?#FYLUWMO*YD^;^?Ybo-PvCZ*NP_5L`ZQU%ED_YAPXwMM%@w<2 zxZcz85n8_z*4{P@UBS$rhPjb6*O98(fsMtA57##yAFPtKnlHLtF%iL?#%?|Z7j%{T zVlW-hLMEzqK+@SKsy7sNU5ZPuk+Ks{$w8}APvFsV7M2z&<*f+hCbuZi{7bcS?BHL~C))2QRUjoTc`+NO*A(=)ekeWSF=^%(4rpBSb+?;4{ zggM%<`4Ci38;WRByL^UlG2KwpY;{_++-}54{nF?lkdleaFe^(z8#f^J*o4TA zQ34eq7>&utPmvDQ8rO3Rz{JKKuH;&D+{7cjNZb}NvTP8Wtwfi2*l3pqrM(h}5|<>k zIXH$X2H0ay)Xhj59mvz#`6YcsO8IbNDZxo=2|^-TJf|L_AppeZH!#^^GJB){071N-KURbr z{{R^J^weZT8*a*yMJ5E+_pp;%RM4VlJDuchJo=5ptJM?c&jN7@-aC$;-K%3bH&jKC zYe>6_cO{|nAqo^wBsDRv%wm!yp*U##;seO=&oT3GI->@K?uJqvU7E=&M32~duFR1( z>%Aup5G@ec#*D6`FfTfxEqX({k;9kR-HFOfQ6?`8R-}*7sFaLQYbIz2(n|FSl%fnt zV+GORL`v&XX`~+Tm?OX^PupSB;{@lRYp*AD{ZCQNw*sM0=9u96EkWr{8;Q!1eg*!G z-GzL&`e0)oYW$l$Q-;#UF)xW%vgufM{JuGOfn~IH395-b_;HCs>`A(!o3nC--wjAhG_8YZV6X?VT6P%1{hZ zC^19z?qs;E>;`To{{UetecjdmFzok>)E#aFyeg#EID8*p>0-EWrI^om^xN^s5Mgy^ zua&N|q*=C_jLM91uQ`$9*K+tWIrTuFd zB3_J6g8^~9V^kdKYi0PiBquAuJTI@(l>nP#_~!)Lt9doz?;B0K+W3y~v2IR{N_2j3 zAcL_$y!Op58>BW#HN_{aY;=haDf|X*2)wB6zGoIdlO}j8(o$Fn5qYbTZXL>4Y$YRte z7rXJvmX*{FR#8CrB`yYnU=6AJN+^vJ$FT$Yb+au<^tzxg{o@R&xmp?@Z5 zF0A;RX3df+c}tRXu@l|R>gs{>877St{S&aPU|YKGxDq@VWDD5Pa$n04K~zt?MAMBl z()iw4nQG!XJ}w1HC~Ejz15-p_jDe%AqWZ>yy3^x$Av+1IijNEdy}1?Y4qBx%gLuqK z{{U;?90Yumya1qVn$|O1=r!LH9utWE{C-0yZN#E#CdzB^jpF5Nx$}%ltu>3w#O=te zhZq`p=V3U9$=0yzZ?^G@Q`+&xLn{qv4NYk_=8%w%ja+wV?cM5A4qd=^uyogb;Oi2U zMYYw|HVr2|55D~ws49{lDa_5KL)h41q<|%F596%R7g}D!dhxB{1x-f1>l<_=hwl{u z-!S8-1K~+IJdV z;2P)rQ?@&`=U>5^53&0%=}3;Gua;c8TvYAewTY?d5y`^Z6J}mApieb(W23zJZ|N~; zdNJ0Y1CX~tG#I;@X0xikjEtu{M+!!*m_(Ux)2*5hK!{SgWp!SKbQ8xlDj=RyVJB8NHD8(94=~r=$C~^*wk@JYed$ zaxE-7{mfAIFU;5Jsi{}+L-eKqrl4)(4Mp8yz<&NQydq6}Ty}NKnp$-n0^xz8>rSg2 zTp|ud@>h)4{iACA{{UE1=DIM1<@U~Za^39=<;!k0t9AT2yMl|KZ}Ett9_{Dbi>d(K zakXH1;olA*cmtKZnl;*(gRZy&Yc{&rYgaW0n^^ul+ylzq-+O6@h;AL9AC+*ZzH2s{ z`@Lh#LvMGBtl5ds>6{J>?|Z?o4LMa#)qE}rW}b=%f_LWxiP;7`&mLwwB3%x+&cxHk zqrUf&@G2Yb6Am~6k^cbPa;32cY{04{KqWUr<3(HrzHl>xX}wA|;tCxm^_vUe5&^-8T8SX{fg8%x$N~C* z+0~wm+&!F5I|rP09~#NzF+9vFTOdC1<>w)_pTlq{07n5Iro6mfjxb)S9a=ZM*uenW z>+75e6c(lH1AIbl;c-v*OssC;Wz^JHoSA;Lg}~F-_Q?*71m~63B?p1Mb`AO7N0`&S zR%DHK!(M(dh?t7fm1ntE1H0?eqt6HEj*y=@D5@}*jrqjQib5|$K)CZ0&N>G)Q%e_( zFrKl9Y0k|r&P!CU2C)FQ)yktl(NFzh83LPHyqd!kork7%fGAMCMhTk}!KPx}8luVH ziSvoef_#)SgL1+%t}(-Wc6BkZ)zGT3(Vi{c#e^o!eDSvwhl=CT@{UkYruByEQ@!tf z^@+6WcPz(kv%`+YiDlkXwa4slCSDK7#ZJQ`VTm-pOtlzFzZ-7&z^J;O94~EQqTvyE zn$2o$$DHdr{{R`=8R@9ys|z1Hge$1kz^~Jq296YSwyCLrY zf1vDL<-GynjtD}ecFX`kp)e>2h>NUC677C4J1vjfbAV1@I^9PUPaYNaLjhH#qVOw| z?h>L&xF3z;LN+DC%HA(VF&#CsXCNH%e%xn`*Bd~jxHY^qVWa$c#u0^wTz0rv#He}K z>Y!db!h|lI-E)HlJ&!zTaUxaYPt`D1h^zTxEwhThJN+aDkx<2cFx9?1*eoLu-mt5Q z5GFae9aFLRZhVOLBN_hE{6MDV&E=6XfqcgfmA9MooG9ZELyP`0IFxj4(p)4F`IDC+ z=S_g`w7+?!gLq6qD`L+o%{}@t~`QI2YZNA?) ziKjGZ=yEYRIviDA_kL~yQ@u^)SJEGH;7)rMaZEw9)5C+e2OeU2Mi6o-hM<`Mr@R#H1=ZJu#{U5EIN)fVZ_$uR z>z}g>eh7%*kPTq_5@xLAZ{$Bt4nqA3gEs#F8311CGk~R1m>`jOZQ(IMo}w^YQ3 zv2hj&L}qGGalZcm-CS~8G{~E2)p8>aW%`&38%ZaP{{VPe+BSPQ%36ziJdP;e?PA(j z$9P2=X!DNawnd=~l08;9y^Y5ha6FmG7Dn~=B+AO;(9Zt=?rK#AuGx6}lM?tT#%i(@ zEl?kqj2bOa1F;8f8mrt$s zcZpX}rwC!45Rg|yCU0cC9(Hq)sb+-Q>%`58BgA>|^tf{H`9EOF4Baew^fF)@ft7PP z&7uq8(I|Y=0z$Sn^FZ+~%s$$7Cxkb_)^m}kH z8%HQnl~khZd>ZF?jD?2YJBQDVrlE23qhRe?PmXp)V z3Rj{NeBh|#$6IH0zsogQ2-5-DQ-^v9P*$oxK||25rx7D!esKs;Xm>h?%YtiZV8J20 z;e7B;cQfqD0VbE_!f0Jl_{M{r@e{43#AD~+?*oxRr zjjzyPq%y@|ztPGnC=5IU{$iFQmzT);ab1bb7mw!wzKF}@!o0ZS7I^?^ zIj3XOn!z7;XmUA1j?=FJCUPu@92h{5KepVK&4drv=qL_ePZ}{o1*GGQMlPv-oGUwI z<5|hiJ@cD-qJVo0j+#H5OC#}dpj~*x9uPTKH=nn@A9asAzum#_l}UioEh*@laH-Ki z?^_-9aOwgpASuX}<^KR{zX%ZAaSu=h7d0|TPk3`rVRUcK9cUynD|bJ(2bBu*ZuzFd4Ns;W`DKa2(8)x8+mYN!XBeE$IMI%uz% zv-kXCpc!-sD}2rS-b$|x>s}>!`Y<#H&qH1B2$~K9=c52eRhJvm`EcC;l{J_XpSJ`# z7pE=S&BG?xcWbP$4lbN(_{~vtv6f+rgY(8Mu=zIQa$W4>St$CP)Yr}(Non(OP?|Pt z-dT|3<@0bguFmjkKH~lE1t4iCQTvJ!rVAW8TwE zPmSQfn;rs5uR6dS!VB0szyXM@o(Lu_8sW0oh^+0_N^aL@z2bRix?BJ(Q{i!9K1pbp z1l9QB)Oj~B3uG~PiR8#9o!I)sYC}c+2)p7xv4s#w1?Ks|;d(Eo&#RTUg)g09MOxlW zHS+K9V-%VfI0a~3#uzTcy>37rJ9z;qJ{;JC(;#>cN15(9IF_riW~M$>*3 z$T2h>wCVose}rSXUo6m zj8#ji`7yEfV(5sx^3NLi$R|HjTX&j8>a_Fo(Q{)RSIxeRV;DK@`N=?y0(E_VSTr`_ zpg`r>*^wD+pqpwJ1!$on_+!Ub<>G{M@eBb7(A|F@_X&^KpSzpY9ZSRcI54lS9e2R? zdt6otDeK>!k0G3bE8U#21ciPs41IQRVlgnCb8!%R4It@SaicwFWHy&Orw^6`o_ zu$%e6jNIMSA2{gSxN8O=Q-=%7InfeYYXxqQNZ|pMrkpo@+^AO0W9t!0>3HqQS#sTQ zVg{)cAdFIOk#LCAIbZJ)oLc)F=P(~%miW&zGMipBd0bNhLA)FObBGVx7x=MVxi41G z#wc_9@@Y9_Dgoyx_~{6xiC_un&e1Tn~IS zFGg+bmA*9MVJ!;y5tsn5ICE3Z7P8W<$>%jzO$~FqkZzYNfBbKZv=r^(xF|nocwAJ} zBRW=SFt;cg3}7#JoT+E14NHkoz8_cxjZUBMG?5jRRfiGW-$cs&^ZVuY4#(X>zo8Nx z^7L$B;I{h4?mil|wSPcDIRY84D0T6>ig0lYh5^OaRGo2&ym?n4vJVIAfT7rNl*Tnu z+5Z4pHk#MH@%3>+h5&FA4crLI8kPL(DRnLUyLSHoxw0pqy@|60vtBT^(Qc=Z!X}#E zoQ=Cxs`fnOLMGf0wn|=I1&wEP0`-Jd|U~%LwZS$kPP4u=QtH< zxNBdG5zCA`V2QM=%Kre3so%sf1gabe$CPpU48y?eROawr08hzU{3q$iBh%z|W3B8> z8Ay||U{`F(;9~0?@465T54ReNL$8N}z!(hFHOEF2Qa5p++Gdox9)kCfiPrANAfSAoYM>shV?6Q?2k~ zj|476-)1@nxw4Nw7pOT7BUmg)kvTs?GSC%MoEj>MCsTJ2Py^FL&%R7V%3$&9NkV+G-ydU_mt?^Nrz5;etc&pJ}WO^N6yeK<{2$V&+?&1XA;qpb(;ddd^!6(V>qs_i-~W#*s60++gE1 z*;XXT zCls!`)-Km&cW*ZUM@bwXjU zy%hWg9k-BUN_rPx8uOLknhU1^pw%1?LXh^n7_XN|@H#VCczNeWHU(WCvaYr^^x62p zn#HyDrZ;P?7#JE!^NUxe@AZHq{8;N^&3%&@)dfGT7)%X3mqb?c>l&)_caxr`JVMF& zVT)Uw;%zPB`j~YCUAc4c8pgoIH9ecb9RN)BZg)*)__`=J^@K80*$-YEtvvK=X@@u} z@d3L=2|$fsCSs}i*M&@8;cyEgHd9I0bZ`Mdq#Yk8tQS*EO69}o1D`X2J}?vp9ugp! z05WZHJ8N3wTFsqZY;-=xVbDkOvvvN1Hc`tQ`aT~202nTtMd+C-UuF0`=F;756@?4;8(Y{qofIi$Sw@0_=*}eY& zwfM*^Ht*lwP)LA>>|-i+G#QAWL~oxN4=bO3aNS`i1(6=P*^T!jHM`bA5l!?|cZf0N zHYI9*SR@BSdF`z@pTLJ{<>w)-Xok-5S4eSh9N|ztR7A3Q4PY$_$nYK$o2)`|CoT9g z5Jy~)drcUQLbHSZV$y?3AoOrl?1BBRhe*?{>}{{XYbLKQ{a z{;>f-alPLdnGJFk*~H40%VH3C~=E(?;~*Lae(%-L1; zWR_GT!GJW{1bevjh(PP&xp2B5J%Kmb#ARC0uJv$Nx#AhQwMKyd0Pw)Z=$-E%sRj>N zMO9L7a^nO94-)~vVUh>a$%`>R9lrIg(~{Ngb17&Zop z&{bSyl>vmy=^yeL2w z0PokvC%6PWM;NH8_%9b0kfXPUNq{5R6!zS^bhqor`KC$J(g-gmB{q$iMd)(g% ztQ?0u8kbWOoaT?4amT#fS??M;k@qJUA)wF-*kCrDk>kdPIgNoWvf8yu+yQqAJZ}O3Duq{03=K$B*E@OY_{Ja!r7!Jg7*fl*t!Y2D zFG6h`F4kE(gWWTD0_mo@^oCFZ)agomAH$X@c#A0YJkBv73B>n*n=u|@zkWaFBXtA6 zke3Q5U{5k#%_vmn&j$aaz9fm5eTR;A6=Q8$#=hQxq4dH*yON z=;o8Q;q^gP$ zHPTbV;<7f9b`S{b1Fc2MF1RmV@syLdc23t3DTbr`{9<($Xy2g!069qnFe$a~?B#ar&U zq*Nno=3`M!?_6((Il9g0+;9v)l0(k^aT#C312o=;2~N7f5RwWntC0md>xzFl6im@6 z(catuPk88L=bF4x&8vl`R7K|4d(JxUzU9j#S_ZHkpo0b=YsRrU_ru;PdO$lxPIG=z z-~2*jUJDVratnAax#mapRoD^KwaW|m%0H0GB@djl=NQ1WxYy1kOLla|7K>kCM=v#y ze6khsn>Zsp1I#el#?AwPs7|Zej~;VVQ(|bpE;Ep!0tD%jFmED+hqu7QD8xLq!O>mj#IqlfXktgI zxvuWz5KVZgGWCSoZgtF}0!H?wCn2=)LqBV zjAjG!PlMToe?Xv1)W$zWp~BtlJD5F)4i<;^gFu9QkCE2#jYJ#l5>wSMtHh3zrnwX~ zt;HK&m?=Ub(1(q4F(Y8B5S$95>)?xst?(ck< zkfA_omkDKwPcCpjdWi2?&A9KIym{mDT#ou8Yv*nZPB*`YSpMR>{{S0o=4Rrc+>__O z)G$5Bvry=`wmY@)tVXMmIO*0E!g3#+Jdj48d}QS8aCq)ingn|WSWZfJ}r$z?;% z1_gn8HGW(Qe{@l|+V5g0I<;ClrGE|r>0v-A9gW~`pxCT}PaX;q|1A=mdmCM$Cmy0zyJ{Q{5WU-0K9+3+ZD3Z#qtn8 zjGzxA{Riv1`TOK?h-*xEoQLBUsuH=QW4~Ca(PL+sn`^tCHhg>r0JVATrlU^DYN?JgB2icdJ>nZVFO+zLG$(0zyM zYbi^nn1}mcz;OspnDf`Xbr1wJoF?bzAbj5*oa$DF{&6(ys%-LLO|5X9Y5JA`;NUkD* zEeeR-Wk!>QMHcm%s25<@JmpChy?^64@|HJmGtJ83VFE9eZzaT?uvy!Tl5MRXP&YE0 zM4aXzjD+I5-a{>Y_gr(dU>}^|2bBqSMrS77A3ttrN2#aFUj_)G`T@tN$xjM*uII)Q z6tVnAw~30>@|*j4%@9BuLeKA6!hnwRlwF4LNq4_|p0Lvf=UW4dmsSFRWs2rqI-16M zU**18gUR_~xj&(;*qbqeI7K@hWjX{Nw2t37d&u6rZ#A%a7(d6(ZGFA_$Y-JqIt)k_ zOu`Bta(oUbMKmGbPdm5Cm4WAq!anU@vAk;|i+RI|-`)tXh}z@L(mFuca1R*)*#2)o z+|^tJN>VFStKKlD6>>sH>f(Z0^cbElK{3-dgi>{r!@zP8@)$jmkEMfton<5{*R**{ z&pA<9_xBvV&fcQ1@-{G5J9xGgzL#0Bu90Zoyt=^dg9@wuLR4S%>Y$f`hIdAl`+(#b(vDo5iK9rl+*xGBnpk9EUz-$}#XQXDkj zS@FJBGGTSudFty<2qR|8;lDm|HSkd8o4_$e6&@AZ=Gxu11#%HCn>iU>`SRenLqJCy zyarQF!8AuMapAlrHbr8OkiWlK-Vg_^oG-k6yjl%9!mT7Vr?U>sz+G2|j9kY|A?8<4 zjN}v~0w24LLtq_U4p(}y4v1@oT__Ik2KcNHj)KA!5bp#IF?Dnyj|GL+W###8F^&zuxy*%QWO_3_Km>&_MoT77>j&aowmjSkP2cqr`n z*#`}tm^X~M8ZI)NdBh2$)(8%`*kJ?TBIBV6*`TOn%uZjlv+c$*_x}KuR;pc_r?;~e zV9h7#sQI`Mhn(SrChcA?#%|Si+km?C{M@MeL9SCbX`{b|z$-w5$>SEO2(bX=M#>O9{EtmI39 zl83x}b>I{mPbPBSNBSHXGnZEuQz2USIb|hwpK!s*0WJW~CcQuN>y+*&po6_~p8L@vPXZHdM}H zoH|9DCi%^w7h%-j&n_~?sI!50-<&Ed;hS`!dtwAaM*B7sG(EFq;4o&57rr1K*G4{< z(G&(w5SVqm7vnCfh8_GKu@OP+1Go3cASyeZH@6N5K;_E*u!j?6)TBOKB$U;&)s?J- zM@}PYtQnvs0X82Va=Nu(m2#c&tJXK$O}o9({NdKLEGF|?WML%a3oC9#A3&O%2b>9} ziMbtMhNa{$92=K8D$QQk_8Cdt=Q?BRonX0g(4f~aiW}pPOld(R{+@6@0)}AM+Oq=>c2hbkPhdK`^0jELw0}j6zMA) zd7Qa9_TK)VIPT3K<5%u*bfV~bq9!6QgZjN18qws=9`GH4Kf5QsR&NQYeBh(KP5ev{ z-Bc%od9>Oy(i(p_2@zrrXqD%b@iRof(p)c*7nR32l)SH_yUa4Mwwyp4dxi)MH(1NR ziHtIe=Yf#*gS>(VKqWeI>LYIAlsm30{8v#W{-!#Y7q3!gG)DIu$nZ?vYO32H`Gn!t z93ja^z)sgS6#NjLLSPJ5k&@BftVTHaTq+2So~{ghHr)Ap=F6-TdNih-4rYdAU>N5K9gt=Zr$mLzB-q zdRV#Xe;j0V7~Tbh@6HoJ!#t5Wn8^}IPWc$)F+}UXJMo>^FD}11&QQ>>c*h7WX~Of# zjkJUa`a`y5!f0$1b1_^6eXSbB6?0%}ZTno;B~dt#JU!rxD!Cv0n{v_$0Y8+D$91@y ziLV1_qa#zo#7k02VaOcV}(-#6c&I)#>RTT!?A}*7ThGxRjcyzo3XEif@b% zVUzRD@e#WhcT95g?Kx`BhZegd-Et7%CXTV7`??+e9ZS4WtDvU>V6zjWfV@5oRpsLj zQ&q$yu@(7zM|f0Iq*<12uVx*Kr=;YZU_GLpHa`~;j0!hAJwG~p<&u-INhj{dS;vb; zzHCnhCtT}m;&7SZTol1Kv?(53E;LF7c5UxH?eeZXBsXo5&L^`EA&&4lcnfEGx7p5g zVLR3Mr}x1?dlsYf(BKd$01CKRlZ{o+Y^Zt(HNp9uxQH@q)G+bH`^J<+k@>;GX&cFQ zWo6_sCWO-S>nYjX8vJ6#s1qFrksNm+xxDEQoz&gP{{XhC2I0POH9I5P1?C1acu(lY zyvTA-c)6BpIZBhj@vDR1w=;V9aVVub%LNgzyaf7Fj4N95A`a#fM|yA9o#FGNvr({d zgsv|D(*44Yb7*InI)`*(cZ^p)OJzN~e9r>)+hXHPh5AgAXd?6r$*72hN z01ej%IT&}Pt*%@;tBNtapBM*voZ5+5jL9I0$=v5$;xKI5q!MrjStDf8>CP$ujStkH zST~dwF9#T%O`N9OW@%AU)WR5olax;mO!1;E9U^j$2rv+uf7Ztbh|~fsAN*6nq!=n{ zCkLM{TBBZi>l34_0GYY_$c8%R!OgrVBD%%gkoa+N<`vooBj)R$Tx4qQadD*aaqxHi z`SWu3zvW&?sO8VSTtbEry2>m(}eUkUidt7}4y4ZiTnH9#Oc zdPg^RN~jbC4%==oGKltX!?!ixzWW7y6f*hJWedKO8n>l1ZuZ^hB2gVHSBb*7hfq60 za(lowr~w@BG&cs{2Sbd^on)0Y%frYpGQ2m?E`KJmw}a(s$@xR=D0qT>lz`(3vpX-UV8T0U@rx=ZjmE+bEk=8fP*Zr_+(D^XQ9 zJ!5ILhWK5Ya8e|LG6b%-xb3%mV2(6W0FN(uaqOI8uJ0U7aKVRm!m{{Ki=)ZJ$eWcs zH}RIX>uX=UZYcp7*>{D~?#FD~EmbwSw-*|Dtic1iNwQn~V4h5LL)`I!F$FHK1AuV0 za2*#LrLB~*y*F7sDurnHV^DxQaA4Rbnu)&gA8WUy4td2OX2VwpmtJg6uftY$X0gae3`@wpE#9UyYj350ME`DbdbN^b91!l=a|C(07JfU zAV@gg6ao0wb@u%|o-sjLf;M>RD9;u|DQ|v74DruU>(P zpo+SZ3>tu{YBHTJ0ZQgd60u2p3xsu$jrW&m-r0tb zZQNU}weDrq3{cmRb6&TrM{V$4aA-{tX3%o^!^Bq61m#|st*BGves%oi8;C0eui5J! zJl%tQw>b2wu+*8b*8Aj%x?_$Hb{xb<$uf(Og83i3g10Wu0eUjz3NE%3=PeL?u*Eel zC+8}qEmxQS05V8-P4hPA4IUP9Xx1}E>Jqy5lGO7Ibq@mQvl)leA#Mp)* z_kaN6vlInhA2=S};Q~5gUD(JOdmMN6DCYyD(2XCKYpp~YbY1z57QMqn+wY}v{*$-b zUbKE(3AVnBwelds*BCzEu63Hx3ugg(H-*SR_yLj-q`g(xm7GYmAEN&N2Ir`d7~j_( zZm?k?Q26EcxWn*jb51`5#YZLVU3|D3@DcbJewR}!Y~k1GSz&9(4kwT&A^_%yUTRIo z;kKCtP^03`KqKEFm{$sf!J$9{%3}7dm ztp*?_-#BZY2kmojj9^|>Ia}3wn8B0+ z*0>|qDol$MJQvI>j{pfI?0D7@FSv;|J-!zewvkgWw;qL%AtmE96cHfz9A@RT5&f(3}pjQCJ@4ey5+5SaTR-A|r4mb`7KVPeS zOV)7g9L`C%!IP_vQ|AkHnot9czaOBd4j+nP09LPxV0XPy>`;yP!z8q2N!d2eu;s0? zi~J@%F2O&PE*6n6q;vJ=g%ad?mW$4?arr&OML&|_6^6<@vU(<4J?<(K%0WFDQ%ZNb zyiXrm;*FH;Yf|)ekatxd#zVO-jjQl=SjE`(vz1=ls+uN|^^_F|Vm?vh0<`O&)09S# zYi!M`@T^UoK!^}^Om<*f*b5J41&ek~Pz&))2Zm%iUcH$D)6U{({(;5;;1c_cuza5%u_goCIc$pBh5>F(FYC@j1k%y?*ob zJM7c@xGhZ0|_S+l6b^mo3wm$E&$IKF;8ES$EDM=V9tK9cGtS_Z>)E5!^WtK z7VgAbbd1rU(^YM)>l*dl+BZIDIC}720b+Zi=77S9r4Dz#OgCU_Z!j_PX7R@21wj>w ze*=UlIRy=a6$C+q`A@r>m>`|ObboG)G*Y!uPkLW@QpGo)tg0PPZG^sX8{}9a^x`n~ z3EwV^mKJz=?+8c+%Y{+nl^+=~%UY`N&J0Eu5O*9`RKT9?k6#xHiA!oI=AHOV_6~ln zFsntVOJR&DtE`cHZyG_XNR_L`o!xQTGTO1q8sxzYZ<`N$VaRk`hF5PAY|Wz zSRIa^%l{0F9sOIl`U`ykTj?(Cf-p;49L(PLv|AfYXa;LzR(OUTp=KXt^wf3 z0#?W!4;H7+0b)}cV9y3P1!uDGjwyz=f83f&%xQpKB#{au^G`!2V8CKP{b=K(0tXY| zt)Dm*OYDnd!u6NPZ9oe|;1eE{R8Txa6|uGv;JMWo1Yp4h=UA(=RmcS8J)PX@1HwB( zuK_c5LXBEaA#QR}7}VTVGZ1s7e1*nqqMR27O%eY9h(Y*p+C@n4bg#pQ7fSfU%|Y4V z#cx)>M+Pp1!}}Or{*|3*{`i#-0~HM85H>`41IdX)SPSUqC;h0C7W%sxm8n62hWCu6 z#VgN?`(}c>gfLt%boM{S18M@saARyKIX65l=VntdEgRaL;x^m1Da!n~2NIajIt#2U zWl^9w6I%sH)eH@!n}Vn=!Hx(W!3~4Aw-r0;BOv?2<8!+W-?Ml_c@7{C=Mg#Kia3?* zlSOqqFsBArTRh@mNK=8Y{^HI=R5|f*x}fuEQw_q>8rN3^BQy_xsWECC^Uo}E%+Ot# zmr7UbdP6w z8gF|{aS?kOG=3i&Cc-u&LE+yf83e&p7?YQJG7>#vLV(5K_8#)JC^c2v`Nj2AG~^fb zd7e3?)yi~4`C6EYm|se9e5_O}Z&z0PV#p=Et^E{%E>w9Q)?D`vQwEZoaBmg?iGlTP zio8$dV9KUxUFpN>>kPNT^+Nn(dtgUfpPbx-T#R*f!<2@|_+BwSClCyWlWXCV^t-cw z`*3ERo-$dj%6u8R9G61J4#_P%X#Vh$5yrgb{{Y(?uFV8XgAGg8@vjvj>37~88f8&9 z<51TJp7MzBb&5Mklie_6pbf*uzT5{w5IEd6l9CI7= zHMNJ67f3oC(^}}l$x2M!yc>zSIC?@AqPJK)ExQhmCJFMmVFGZ{>|?Xi1qcbn>027o z=)-Vd1(#MC2_y=6XHWLie{S(rb4gH)M71ACay>-$yGcJ0IX*+ctjS}$b9_=a7mmhzwmQosh6yuI_dydRtbo+EQ&ZW0HC zb{|>Dmsc#?ux<;pi=#&Vas*U#@?XoGN(6+oA&;QCSk~w^-(%@?dNZq^*7_c?E{&D7 zb@(_81Y;6i0D8ED&~Jr_;N3$h5-DvOHOD#2fnkmMLo8s*?b=2CJ4d&pe;~*2+YqiV zVAlA+0u9sx(fPT*=X)b#m*NYjg04Ctg>$0{^UyN;axq$}yYsK_Fa9hwX6LK{$wPSj z1SG~cwUgs_c|f==LDz#Dr;x=*WE|ug&3t4H*i-8%K!vjbY&yU0Zt6g8_?_a!&cWqc ziND_%kuQMrvlX&V7JA2|U648L;^wf3#BW8=7IT2{v<~LD_l4>gCGYZN2P{>dTZ0#N zVLl(mx#g*~B~*TxfDc<5-IuSP}1%lR)=fYP6_qD%vFbl}zp3M*S@tO)+y<2waq z-xvp!knmgslWgSE(J>H^^)pZ>es32gVbISwhIg*f>Cj|Au^A_8{{S#od_J^@;#blu z$_Tl$U)MMus0-sqYhMI-l&y}~^UBeAGaz{8sWI6fA z(SHy1fi2(olwnhRZpC#>VMl6@&-8ynQt^_5!d7#w%t;2MDGOlb?`wPWyif}(y=7={ zR>Wa=Jh-sCY$>4f0nQ&s$AQ^P+Ec~I`9QOfCi^hN-}_KF2LOJC z+Mt2^t`tOo+m0aQ-m)O~guBlUQ!DVfw^ld9&2fp(cPZ;4=t5@E2_+L(e-jjIF?IVw zI3WQ`d=JtCn_1WRfsbC_FIE{r(@DZ@fvK?Gh%fHowrWc%yE!=%S`*yLH}c6I#5Ba< zt>X&Ri)6%GY&Q7Ct<4Xc4B`I(b8>K_t4%swNo3n9E1rfBKOdZhE+rbZ9-&@Oz@v@n{ z0lnkmbfpCti-N~EAhOUtae%}{=X;y)H)2OtKL$0~WBg|b)#X%KLrlGM*x6; zuU-xljISS1;<&46xR^eY@$fEMVJPH?A^WDXyQ-1ToZ16>%0+hLPPeCcTe^+89vXgd ze+l@wvHnDunK#xEa>D+u=APodu5O!t< z7%xe`k;IrN#u9?OI%A_PD`nGpT#bZ~h`5nYh*cO!f-q_r1iUvY;cR1WNQn=EYt zFw1hwOrcHp;KJ4rZ=Cg~7K}-5*Q|DxW-hxS;j508@v`y1*Asq|*-l-KJV*}V{B6wu za^<{C5vF|p_(i239O9yMZk4P&+<82~Iy_Bb8>>a8ApWV7fVV-~Xv<(oeq1}%C5ILH#avYF)t)}F^_n-& z&+Ko9HAr%=+GRTE2DBVvJnnvR#Am6<2xiUCf_Hz38A;WUD}F9;Jf^(reuWY7vj8c& zW8rc|5|q^1AKoT%7Pbalx*2d!bC$1V4+obPV!^Zrf&{`-w++lG)Vtj{WJX@vM{+*y zDGb%pgq|8Wo68*?AHV|uk#bLF0ymHe)=!Le+K}J0h+wEnk@LaM@y@nWZNmCCa6~~0 z1Oer599$EmNpf=z5&46^kMQ7bbeJfTDi4~!c(xD(JXnqz0m%h73lzkNcf6@E_ZRfa&`%(k~bFJ(d3BeAE)ONdw1O0twGz3>IiB@Z})4 zIP#KVDR{sKw)E>-&uq`mckRivB9>iPc+OFfK&n8TVIAHDhpaK^3FKNSTY+XWG zFh5i?Y#*Nhc|Zr+^0?g133JRm@@Bm-qId?_{5Gmy<{G-|0;D65Fzte-$Thana^diT z8%|&JEw3lNe6`@4Dr-_1DKS!3A`^PHqrEvsty7+ig3df72wLPhGQ@)XhLZ z8|cl!q2u&bpN#ktYVSJ>-Qrh;!r!PS0C0e0fYA5)DsTL)8-$}1OS!}+?YsbhrF@A= z@AZeFF;RxckWa$q>x!PG;PT)trY`F9F;e{p1yFU7NzR!hJX%KsM5KcFQs9@9q_jFU>8X_#;90WT)O}&Y5xGeGhucO2DJNf4SxjN-^qyOXEa)I zXGxMpZ!Q98_S&!5IV77ip)q*C*q#gsdTnuWwx?VG@oVjjhGhT->w;-}P)^Z~M|%&q z@xZ|lW(4|o-f0DcTWr7>4Hhob{dY4wbLczx42hoawj)YD$Cdhf<{E0I3(5I}4R62KYN1K_$}*u0qMhSVRYm*Fe0Rt2-m8Yer`oRPl`!frq!}-Y z*mtarhki#7Gj1&8YrvrQ$isTUiDZ5`%r8PAwF>-0oB+)YqV`Lc6zdgb9`R%z2yy%l z&x0twmyQi}mB`W)*li{x&-x%Vu;3cXRja#d5f^MjorI!o}zDPCGAlknQ_{3u<_<)jx3H}UQWL9uH?(dYjG+s&BpOA05<(NaT8$Dj)lenW&M?8xIRP9=NYm>@sFnq zNzNKBb#u4SbIgnKKnb#5XYp z%`$TvhyaC^Wu!0R_F>>;JQi7O00TWX87v~d&jno$3h6f9$+1j5P$t66> z?uc~a7n-0`)&dnu5Y9(~oGF}y?1#nY)*HrFO56B2FrFp>;BRa?Ieqt=gI!(+Oj#so z@I#DkRTsUldDsK;%=Tgr82B|!qjyg^s2Qa+vl#P(5OJ?YsCs>2^Obk2%MLIwz^rGW z%ZQkIECcAmoGP$*Ihds~OQvj}4nCH3t33Yz25EA$@(jKu*)_9Q1GZCxmc3l)$Whq% zlLZKF(syjxgM&>0CoWiNzm{Bd7xtI|A6)zI0ibg9;5T{v8#T^6Esrwn{{ZF40K4%` ze~9phGzJGZzB6uRtzO^Muq}3ZFcEZp4X>{_9oZMROS11JP&OE#-~oW2OICktiU12X z0)yGgvb@ZE28R4%W@QS2sjH01*p{xd zqp>qDQOo1=z6DVeq675~S4C~k+G5h)&Qts*IcfJ~PtpL!+I?*?Z40`X5QUD@!G$AH zi8yar3}8`TTgP|}xOWT#a|%*|0Aqom*9Xta4ib#xn~P;Rfd}51 zA{%R{-Z;mI0n4B`+RXT`>wLr4#n~DR-{{T6qdeuCX=NC|11+x!;DG08t zV_XuHk<~dmhp?TC<+u!}8DeH4E|Ik*55s`04}%naISXtHY9E@gq0lym7w{K2+B}ew z6ADP9qBMQRFtaASH2m?(P(#M}XD?8#y&>9C$D!P5ep6t;_#XxXx{(T?l+53x-*`?Q=3F&@%Bsx672Kw4~x-3D^YiQ~JX|3%%>k1V;dvEb}?H29A?Z zvbf3`XhL^auNa0e9t<3=w!FJAZ&+MHmT%_}4UdEIo(+E6aev{s{{R}_{PPMb&2I=I zXg=Au^f@VVqNFi+%`C0BKTN~6!%N&>Zz}*5^Gw<6Tt1Aw4V|T8o*{75``T&A?o`bX zKMa68BR`jz9W&OzPODgyFgmaT6%UGfUGa~flHXU5!T}Vw`1h|mGEE@g0HKtbCj1=BhsqJ5>~hJl>- zG4Q7ZDCzha$-g2G2NnooQNP|s0g)^pR`r3> zHC3RB`fyi14;SPFxcj#l<6Iv&a%!7CXN8@S9xmanZ+LBxWZZhnk=UE;G~v+TQ1c}? z#VFxu-F!IK@pxdNcl39IQKN%vCM4@uka)yurG8uM0e-SXoQjSL+oq{F+ld8{0R$O zEVM#+#R;K~0{;N|Zy6X<{vmzhMYRKxzgar-7x8cgKFK`UnnA7at^WYb-l#sj?=*K* zc;gFDtDgBnIqtS))wUZb*&Nr_q9?8w}$WAIPwDPGJA)URXn7P>=up^kJ@vmwT=LDAf{hZyT z!40`IyDkDfwNiR%kuEg}u}?oY9*EH|5=8k7-$iQbs{Yt3hZ<)q@MYA(M%&hFdDm^< z<;wYgwhe$zfZK$o_PWzq&tdRcymtjx)ltQ-9*m?2wcjoDy1@ipjCZ!Nid&Z-qzh7`Be59!)cq#H>6WH>~Go03q+Z3ir?-)&x?pUygD4(?5*Z9dnwV z^F$7DRHt|ffy@5p#F=+u9UqKybKw5~UUAm!;%`oSinuq&zIDbEv=Q=tSo%2xE}5eT zMImu@b;o8KLDWiMxf-Aq>Dq=9*6u1Xy!6V%RUx~R@`NOiM03!#$zoVY|#=?sxro4hWUH0-zolrRD=p_1%P zjkNQ|Dmh#Wvv^^#*sSxDp1ngX6UVE`bPE?XLGf0>Yhi#%DxxZr+MH z-Q>w-VclQu+`$V%J4O8B)}aT4ZG7i_eKOW@!Ui1#Y5Br?LuKn9OhyR7DayH7vb+3c z8YT=IG9@N`Nlao_86z$9VFRTTgL!Y9s?i2bDkg%0UyRLT218IwdimW z#EY5_!s`?o7M<@@!A?z;cCfA>jcnWf;j6uGA~o^HIiV1s$H#c>_*8W>-3uXfR4el9 z^Z@1H8vK}9r8q|#6xx)JGYp4i=6|(Pko)Iva{%GAVUHV$9$XgB2g!m^e6AEsq!q6Q zBrAzY=hNb06g-F<3GQN|3#k<3{9GdYwPUf4vXVG-->gzlAR$-g*^L>3D0I=A4}xt* zpHmtYrvTdDDqvJW46;e6_p;Yx87_K{H>4Z#0yG5G8r>jDJ0noF++TSiO% zUy%_$T6G_@0v|DcoI;aD&MjW1CyZ}zqiz)(cFmNXc@Bdbd>q}09yDHW6AMhxuKT!r zvKuxMugEgYh~gK&*uw%}B(+|))-#(sk)F`Cx;e*SJ4BZK%ScY5u z$Xpb4s&Tw+{w8o)>E0zyt4Vq;5HJ0-C=5kAcXC~-vs!&(wOsA<&MnnnZQ`g+(^uQG z5h_rN%li1lMMHPC*WQo3R0C1*pcpb^vaK|SGZYf(pewrR#t3S#f{KluLoCA6?*9Pt z=Aw%;jWuEA!d~}qqqF40vb9XMO@CPhO|X`r)-`MiY z5Wny_!;F4hyf{J$q!sT4sXev*Ik^u!?i2V77I5euHZ#wqY~Kb10d}uu9_Z-=3hXyK<$9IAkPj#nEv0xn*|dmLqv8=sR$CE!~kp&d+) zZokDFlH|3tM?-v~KF!0m51AphzV0Mn2L>a8n4+^;H;zt8SCOX7I9+6s?Na>W1~yir z-?hR$sLKjG^Ng+jrciyl3_*4BO?I5YuS1Y4w()4*DaJ7~jYn*6Xs0-YRYW7BvxX7H zd(~#0m>{LGMPWE8$|USvBld$i7M`-nPlmELtPA-^8%b|FMyJL)6JkN|;KDLn!l3;I zb#&qpQ+*jLuM3F8dmQWh3!QA-`8aTdxo}3E3zES*C!yTLE(?78->d>s)dzevo)-t| znPvWN3JZ$Qz<*iJi(jlz3aTD7kLMr~$Qt#_j)zNqI}8hN0Jvr-J{$^mxII9gBuqJE z$3a6q(IquL@j4Q}6ypp1p}R~pz3tt@E(%G~_`}$bEl;VNSHb*b-MAWCOC98E4jT|Y zcHiR#!wo};g;81|bj}H?1->~hBwdsRgw~0P6BGvGBNU#d`vcjC0^D$SaGp$3ag)!T zBMjhU^ALMQ$}MOyFSsJ&Ite0s<1`T@X2)z27hFMia4rBab{t`azD;d7CJ+r1VZbLD zOBV_n`TNc#ZPm~?$T>y$!ky@a<5;2x23&*4dYB3-(XMv%cHz+AYIbw1sa|yK@?^ZX za<3TUE?+43hIctH?9xhdGjR|I%hjai-gvfESas!ovdhcg{lMwX~ zlX*5gp~i7k;yPDdcNh;@UqUKu(sUlnKVm((lX%-7LBd^T80Z9wlqk6f(1 z=y$<#ev-w7i|iQM+1p)cd{rFKR02rS1E)MQn{@vG7vk!V^z*z<4)G}3M8}J>V`JkB zT^!DKgZE5!00H(^4&p)Qau3qnCYWf)vHt+f*;PC5AClmRd6^G1P~hpqYP-{hw#Fv0 zRHat0`1qJGVi+kXGhm;G1=PxW{dpH~PYKH-8Fd z%_JZ?QM;Al5q?H3c!|E2zoUm$0QS1c+)Q_AKPCv-OE=y|wTf>H)DE&XzwY$vV-ria zc+FdexRVg5&pVmCWN7(v%r=L=GZ33DUSO*j3B-iTW96R+SBTtIT64+ zAV_6cT2REAIvkech+}ajZkd6ZdJsBS1eJ z!iTim-t&nbOm2(8fG@8<iv6}?EGK^oO=oq=h8BReFz&=PS%^z<96g*gLI40yjSu3Cp-&s0* zf41vjoq~3qVs`3u*^AAKvEDYj!I%$sV!Ne%=KeEGD-syEHRXG0z;s{==o3?V;|M`V z5jN9}1`2cybQu#UtGXy;YO)p7bnC|Pnwr3cTt34OQ&^iFVl6zPvYf7qydL}-A*@nkhQY1}7muRvNx;(p92{g3L+Chn3^=JT!*AnxC?UU`?=?}; z{Ob`^rt;-->+!AraeR;=&$snlJ+7kT2bs=TnWA7kBcRJ<=cbp76sfM=>Sm(pW65s) z9rKhO&&#YwZH4o#J`5ecz8aYs9AQS%-mn^LI~Oa}RJwV=UBaH0J`}_yq8`@-&KD`o zaFI3npXXIIZWqTm?Zf%3sGQwkyjO60d}RedKtg(43t82OJ~Jiz#IIHzGgISm$jF2| z2P=SP?|onsAu_)VaCUb?6?F8lxeqkn*M5vDQX+1k{rvyNKA0`m#Fg1a@n+^TUnaB$as7C3W zuC}@i8TuJDf5BW58i0%Ba^j8}8lM5)R}^^gJN@j$18_9%+;~hH9N#~=gGsI))#l?2 zHNsNWKWv>%1`neR9!=r6YLn*SpC_Cxa;m)V4F}xx=Q&gCc)*g~zj-i4ss7Yj!0Xn- zc#%6PJiTByblu}guhET~m)9;1?J?a(bi%; z>A{PF3%VhR@Wkwnz>c}+CnTd%Uj{_d34RO)=@aqOKWtDZL7V1(SOD|}H>}luuNNNM zysODKyjuXSznp?J3*X};2!j0(XF+z(vyf5Lw-D9mRNH_ zH;mD0ALf-vk^=Y_R|>X4e8IR&yEC9vubI|ReBw&neVDsuI#%E9ae@&Kz-r*CFJrr< z=D5{HzWyi|lmHwHMUnb*f_flrfKl^uRRxnrk(MEhi8za-M|+MW)>DYrMCBl04Xcsv zF9?IKF(h-%)O7(bOuNC95SGEe$Qg5XCH7%=&Uv1U4PBLAG5vmkfK zBZx;31^D9y4=7?cP}V_s{p)OMH^Fq`K~bc>d^1D|1LwR>2~)zj5H_c%Z_XM1+Vau` zKqGm3!>pl^>S@oMLkFkrfmJ!kv^5`?CbCDD-)x7$gJccuP?~2VwK#k<-f7i9Pb0sq zcSO~qz0BJ^pM*N);N8L76z$U!;r=l5(ZD7$Tm1w&Hmb z#6!Wp@qj^{dGms4FKu`=hq8~({{R` zkmNsn#0E#RqyugPZF~Vy_?%p9TEusz3;<}VfKCU>;|~Vthve%lBzyO2&sCp{cLI$g z@ZrK9=3c&vh8p!$^ufGXOgii)F-kEq?j2aA6tNK;}1o6e0c9H)8MOiPN!1BKsW|6%b zgM^*WV5ZxVe}Wu{hWue>@{aFEj8fqj@Q5&t&qcXn-YyAgJb8O32s4EGaa)6!CPx|( zsTu)uS^!Twj>~NPm??6iunO*gzAO8%>X?wTwg?lT71-kI4PDyl^Q;ho5vA{aVF2bP z@nE?f-M@1+4&dQD7gG?jIQQoWrP9-)cH*0FfZrItG-~02D2Ux^rb@@ZmB|k5)}p2m zZ9`M*?a9M5J4uV}6mvuo%jisy>+|C-08sW!IfC-*irm<;e3v4v*bCdvA{?E!&M}i$ zqii|KIP3gm@~c%|Zu507B+A_S@M}Jw&_J5IF#0;_t{>&jMo66g(d0J>z7?zUjVi|X z3VU3x*HX?kB<{ z;@j38rM)2S=ayv2qP*>_AchC?Pw1`*A#v2IPK#d#7^}Auq2SZu!AcDj9lF*=0OtN( zew^0-0FWgQya7z)4Bu_A`Y?VQF+|rT#YMccEb!S}(|7@blXKPtExlvT6hM?_{7VicY*K+O;d`bqkC z6$Ji>$LDxt0CfGht!DA=r+x=DO5KI$og4~X1g)wwSp$~?!R|~6kP`JPsesSo9?#(& zV9AMAHQ;#uSzW~36K)-jY;FAyj2P%`bT=Gm5k3o(T%qEH2YJ9!*=qOWSz!v%N@N%q zXbA(0tl}ABdr9+;f`Y6{b^K(NH{%K`d)6o@(TQ03)^1-INee@BH`bi)JY5Xh-7wqN{{Tn%#0-VU8p&W$JYF+(a3DAG<%~IK54(u^ijB)C@FJY<4~$A^ zqO94@I#X570Z?u0=NNL9q|h63C~TGg03A%_5?TfC0T5t@GUsPR`u_g_&_S1DejEeV z8FZTG+k)$<=mZ~yaf2=?O`zyJ;)rNqZ4Vw~$c7M_Ou@AAu>hyz6AVyOUFEDFlZul~ z6PUc>Q(`gUj}NB|gZ{IzaxI<}PCrJBfDTphoyow7 zs`wToysBPRbAi*G6XK=wFfH>nf*^2%;uwPQNj@2MoG81s)*O@kwivh3kA%W4hsZ5| z7Go%J5d3cbb6m7r%@P<^QK9q8^aiAa?x@D6kBtvChU>HgNe{egphx*D^@>C-ydH)E zW4>2*Ocdz(_Ta(;3m+n1y#D}mWd8t9JM5Foi?sxXYMrt;z!p&6P1q(eWrq#bH`|YF z72&e1!%)u0z{7K?c-^@(ViMcaf|!MKy%+J0N~)=8H~7k18)z9H83_TLs(5*ZjyX%I zb7$bV#pVJPA#S5EcAAj$txTh}ABPDMv=v7NOv2HKUV6ZMfDQxStc2$EH8W|7GMeHv z;W)}A%Gh@_CB>~G2i^P4myk>lsqc~q^rJ({{VBD7QMf|Iz`;ZC}3fi zd~e?(U5HE3%-#;?G)!57?PEu7GQ>f!!Rw^+*^4Cbx?V#!fV@BMQ-C;PV-2v?!~naa z&GUmdYp&(lfK^+3d2R!a*WSO+`v_nY5g*gDQvRD?8y<4MhiSqpik)B_S4{HK{w7$) z$CY+Jdxck_I%-Tkq=4zr`&!PG#xKR8j$P@=*ZZsh0YRZR=L-v=F;R0TKe%v|aNZdt zJF!d%mevmS<;S?K>BkEg4ZxIVjENl+sQ2dZ;tth%tA@8c;85n*+Pt;J zAIj#`yb$8!)`lMDB??9BH-Xxpk{CnpXyg|L0|FiuFf7o6WO{hUv?y=^v!@SRgx>9* zsg0>hQH62O#1a#DWNAD>q}@6k8d7mOhm_flnIopaK3a*9P(n?*gXPv};doely5ld! zP+;nFidW)PxwSqiMVM+{Bu& zm%php7QBpD{{Ry;zZ3@9#`4HoH7&K{*ZITRsM2StupN+7%@4M}Ln{iL{{ZF|zyfWX z=^Dk{Qd|TciIBRso;Xje8?tYcczsMAMO^(y`oS?(dV162Tv;Mp0VD2_fp?%w@U94Z z!c6y;mv}kjN312GOZ%;~E9W06FvWz;Z@jE%JBU+${O!tSMX>z}*?8V+CE%yaE++xz z_oq1Eom_wq5dF;C^k4z(){XO;^T$ulNOIi%G92Aszwx{~xFQSL(Ek9B^pH`>KWT*7 z>wmsSBF%ZlV^Gl9-7uvR)Uej;C*=X)G618FK|^w=XnFFaZWN^h6S(O7;Pf@`z(yZ7 zU_k@l#9jPcbvyq6ZFE{R3hgjK1g*Ydyg|sJ)%(D;tt9kgWdJ48b+acYL*T$D5XsgH z&Z2m;U3Z2m0;+A=l>B8tLA9W}^>H=ghsb^jf>sMHy%SP*-HIBktJ8)!n#};U3-arsMj>| zk-k>^ykSmDlf{=f)ZC7N$TbdJ-~Vpx^mxs9`oC`!P`BSrPA_(0olI zAHH`GM+0B$C)gX;DT1&lu8coeCJ^J;gg+Z&+eHx`2fkRPKncc}JQlLYC)!&axLq!f zp@&&=#k4?sg3E@|U`?p;_y#0zHz3w`wftl>OYL?iq;4cp!$7;!mZq_>32QjlI1U@I z{p1IBUNJ?lIz2drUV?lR0$yyy1!=VTw>f-m@XD!RDZbowCk7a5Kx?#u}(4$9plaszDKd9Wt}u@*el~V8nRzz1dK|s>FoxYp)nWa zg-MH`u_m`!~hf#0RR91000000000000000p#Rzc2mt~C0Y3mk-~Yq_ k6cGUc000000000000000005x>+5iXv0|5a)07KvZ*>sBET>t<8 literal 0 HcmV?d00001 diff --git a/doc/tutorials/core/table_of_content_core.markdown b/doc/tutorials/core/table_of_content_core.markdown index 4cd77fcdfc0f..ff142b7f9c88 100644 --- a/doc/tutorials/core/table_of_content_core.markdown +++ b/doc/tutorials/core/table_of_content_core.markdown @@ -9,4 +9,5 @@ The Core Functionality (core module) {#tutorial_table_of_content_core} - @subpage tutorial_basic_linear_transform - @subpage tutorial_discrete_fourier_transform - @subpage tutorial_file_input_output_with_xml_yml -- @subpage tutorial_how_to_use_OpenCV_parallel_for_ +- @subpage tutorial_how_to_use_OpenCV_parallel_for_new +- @subpage tutorial_univ_intrin diff --git a/doc/tutorials/core/univ_intrin/univ_intrin.markdown b/doc/tutorials/core/univ_intrin/univ_intrin.markdown new file mode 100644 index 000000000000..7e6a8a7cc8b2 --- /dev/null +++ b/doc/tutorials/core/univ_intrin/univ_intrin.markdown @@ -0,0 +1,334 @@ +Vectorizing your code using Universal Intrinsics {#tutorial_univ_intrin} +================================================================== + +@tableofcontents + +@prev_tutorial{tutorial_how_to_use_OpenCV_parallel_for_new} + +| | | +| -: | :- | +| Compatibility | OpenCV >= 3.0 | + +Goal +---- + +The goal of this tutorial is to provide a guide to using the @ref core_hal_intrin feature to vectorize your C++ code for a faster runtime. +We'll briefly look into _SIMD intrinsics_ and how to work with wide _registers_, followed by a tutorial on the basic operations using wide registers. + +Theory +------ + +In this section, we will briefly look into a few concepts to better help understand the functionality. + +### Intrinsics +Intrinsics are functions which are separately handled by the compiler. These functions are often optimized to perform in the most efficient ways possible and hence run faster than normal implementations. However, since these functions depend on the compiler, it makes it difficult to write portable applications. + +### SIMD +SIMD stands for **Single Instruction, Multiple Data**. SIMD Intrinsics allow the processor to vectorize calculations. The data is stored in what are known as *registers*. A *register* may be *128-bits*, *256-bits* or *512-bits* wide. Each *register* stores **multiple values** of the **same data type**. The size of the register and the size of each value determines the number of values stored in total. + +Depending on what *Instruction Sets* your CPU supports, you may be able to use the different registers. To learn more, look [here](https://en.wikipedia.org/wiki/Instruction_set_architecture) + +Universal Intrinsics +-------------------- + +OpenCVs universal intrinsics provides an abstraction to SIMD vectorization methods and allows the user to use intrinsics without the need to write system specific code. + +OpenCV Universal Intrinsics support the following instruction sets: +* *128 bit* registers of various types support is implemented for a wide range of architectures including + * x86(SSE/SSE2/SSE4.2), + * ARM(NEON), + * PowerPC(VSX), + * MIPS(MSA). +* *256 bit* registers are supported on x86(AVX2) and +* *512 bit* registers are supported on x86(AVX512) + +**We will now introduce the available structures and functions:** +* Register structures +* Load and store +* Mathematical Operations +* Reduce and Mask + +### Register Structures + +The Universal Intrinsics set implements every register as a structure based on the particular SIMD register. +All types contain the `nlanes` enumeration which gives the exact number of values that the type can hold. This eliminates the need to hardcode the number of values during implementations. + +@note Each register structure is under the `cv` namespace. + +There are **two types** of registers: + +* **Variable sized registers**: These structures do not have a fixed size and their exact bit length is deduced during compilation, based on the available SIMD capabilities. Consequently, the value of the `nlanes` enum is determined in compile time. +
+ + Each structure follows the following convention: + + v_[type of value][size of each value in bits] + + For instance, **v_uint8 holds 8-bit unsigned integers** and **v_float32 holds 32-bit floating point values**. We then declare a register like we would declare any object in C++ + + Based on the available SIMD instruction set, a particular register will hold different number of values. + For example: If your computer supports a maximum of 256bit registers, + * *v_uint8* will hold 32 8-bit unsigned integers + * *v_float64* will hold 4 64-bit floats (doubles) + + v_uint8 a; // a is a register supporting uint8(char) data + int n = a.nlanes; // n holds 32 + + Available data type and sizes: + |Type|Size in bits| + |-:|:-| + |uint| 8, 16, 32, 64| + |int | 8, 16, 32, 64| + |float | 32, 64| + +* **Constant sized registers**: These structures have a fixed bit size and hold a constant number of values. We need to know what SIMD instruction set is supported by the system and select compatible registers. Use these only if exact bit length is necessary. +
+ + Each structure follows the convention: + + v_[type of value][size of each value in bits]x[number of values] + + Suppose we want to store + * 32-bit(*size in bits*) signed integers in a **128 bit register**. Since the register size is already known, we can find out the *number of data points in register* (*128/32 = 4*): + + v_int32x8 reg1 // holds 8 32-bit signed integers. + + * 64-bit floats in 512 bit register: + + v_float64x8 reg2 // reg2.nlanes = 8 + +### Load and Store operations + +Now that we know how registers work, let us look at the functions used for filling these registers with values. + +* **Load**: Load functions allow you to *load* values into a register. + * *Constructors* - When declaring a register structure, we can either provide a memory address from where the register will pick up contiguous values, or provide the values explicitly as multiple arguments (Explicit multiple arguments is available only for Constant Sized Registers): + + float ptr[32] = {1, 2, 3 ..., 32}; // ptr is a pointer to a contiguous memory block of 32 floats + + // Variable Sized Registers // + int x = v_float32().nlanes; // set x as the number of values the register can hold + + v_float32 reg1(ptr); // reg1 stores first x values according to the maximum register size available. + v_float32 reg2(ptr + x); // reg stores the next x values + + // Constant Sized Registers // + v_float32x4 reg1(ptr); // reg1 stores the first 4 floats (1, 2, 3, 4) + v_float32x4 reg2(ptr + 4); // reg2 stores the next 4 floats (5, 6, 7, 8) + + // Or we can explicitly write down the values. + v_float32x4(1, 2, 3, 4); + +
+ * *Load Function* - We can use the load method and provide the memory address of the data: + + float ptr[32] = {1, 2, 3, ..., 32}; + v_float32 reg_var; + reg_var = vx_load(ptr); // loads values from ptr[0] upto ptr[reg_var.nlanes - 1] + + v_float32x4 reg_128; + reg_128 = v_load(ptr); // loads values from ptr[0] upto ptr[3] + + v_float32x8 reg_256; + reg_256 = v256_load(ptr); // loads values from ptr[0] upto ptr[7] + + v_float32x16 reg_512; + reg_512 = v512_load(ptr); // loads values from ptr[0] upto ptr[15] + + @note The load function assumes data is unaligned. If your data is aligned, you may use the `vx_load_aligned()` function. +
+ +* **Store**: Store functions allow you to *store* the values from a register into a particular memory location. + * To store values from a register into a memory location, you may use the *v_store()* function: + + float ptr[4]; + v_store(ptr, reg); // store the first 128 bits(interpreted as 4x32-bit floats) of reg into ptr. +
+@note Ensure **ptr** has the same type as register. You can also cast the register into the proper type before carrying out operations. Simply typecasting the pointer to a particular type will lead wrong interpretation of data. + +### Binary and Unary Operators + +The universal intrinsics set provides element wise binary and unary operations. + +* **Arithmetics**: We can add, subtract, multiply and divide two registers element-wise. The registers must be of the same width and hold the same type. To multiply two registers, for example: + + v_float32 a, b; // {a1, ..., an}, {b1, ..., bn} + v_float32 c; + c = a + b // {a1 + b1, ..., an + bn} + c = a * b; // {a1 * b1, ..., an * bn} + +
+ +* **Bitwise Logic and Shifts**: We can left shift or right shift the bits of each element of the register. We can also apply bitwise &, |, ^ and ~ operators between two registers element-wise: + + v_int32 as; // {a1, ..., an} + v_int32 al = as << 2; // {a1 << 2, ..., an << 2} + v_int32 bl = as >> 2; // {a1 >> 2, ..., an >> 2} + + v_int32 a, b; + v_int32 a_and_b = a & b; // {a1 & b1, ..., an & bn} + +
+ +* **Comparison Operators**: We can compare values between two registers using the <, >, <= , >=, == and != operators. Since each register contains multiple values, we don't get a single bool for these operations. Instead, for true values, all bits are converted to one (0xff for 8 bits, 0xffff for 16 bits, etc), while false values return bits converted to zero. + + // let us consider the following code is run in a 128-bit register + v_uint8 a; // a = {0, 1, 2, ..., 15} + v_uint8 b; // b = {15, 14, 13, ..., 0} + + v_uint8 c = a < b; + + /* + let us look at the first 4 values in binary + + a = |00000000|00000001|00000010|00000011| + b = |00001111|00001110|00001101|00001100| + c = |11111111|11111111|11111111|11111111| + + If we store the values of c and print them as integers, we will get 255 for true values and 0 for false values. + */ + --- + // In a computer supporting 256-bit registers + v_int32 a; // a = {1, 2, 3, 4, 5, 6, 7, 8} + v_int32 b; // b = {8, 7, 6, 5, 4, 3, 2, 1} + + v_int32 c = (a < b); // c = {-1, -1, -1, -1, 0, 0, 0, 0} + + /* + The true values are 0xffffffff, which in signed 32-bit integer representation is equal to -1. + */ +
+ +* **Min/Max operations**: We can use the *v_min()* and *v_max()* functions to return registers containing element-wise min, or max, of the two registers: + + v_int32 a; // {a1, ..., an} + v_int32 b; // {b1, ..., bn} + + v_int32 mn = v_min(a, b); // {min(a1, b1), ..., min(an, bn)} + v_int32 mx = v_max(a, b); // {max(a1, b1), ..., max(an, bn)} +
+ +@note Comparison and Min/Max operators are not available for 64 bit integers. Bitwise shift and logic operators are available only for integer values. Bitwise shift is available only for 16, 32 and 64 bit registers. + +### Reduce and Mask + +* **Reduce Operations**: The *v_reduce_min()*, *v_reduce_max()* and *v_reduce_sum()* return a single value denoting the min, max or sum of the entire register: + + v_int32 a; // a = {a1, ..., a4} + int mn = v_reduce_min(a); // mn = min(a1, ..., an) + int sum = v_reduce_sum(a); // sum = a1 + ... + an +
+ +* **Mask Operations**: Mask operations allow us to replicate conditionals in wide registers. These include: + * *v_check_all()* - Returns a bool, which is true if all the values in the register are less than zero. + * *v_check_any()* - Returns a bool, which is true if any value in the register is less than zero. + * *v_select()* - Returns a register, which blends two registers, based on a mask. + + v_uint8 a; // {a1, .., an} + v_uint8 b; // {b1, ..., bn} + + v_int32x4 mask: // {0xff, 0, 0, 0xff, ..., 0xff, 0} + + v_uint8 Res = v_select(mask, a, b) // {a1, b2, b3, a4, ..., an-1, bn} + + /* + "Res" will contain the value from "a" if mask is true (all bits set to 1), + and value from "b" if mask is false (all bits set to 0) + + We can use comparison operators to generate mask and v_select to obtain results based on conditionals. + It is common to set all values of b to 0. Thus, v_select will give values of "a" or 0 based on the mask. + */ + +## Demonstration +In the following section, we will vectorize a simple convolution function for single channel and compare the results to a scalar implementation. +@note Not all algorithms are improved by manual vectorization. In fact, in certain cases, the compiler may *autovectorize* the code, thus producing faster results for scalar implementations. + +You may learn more about convolution from the previous tutorial. We use the same naive implementation from the previous tutorial and compare it to the vectorized version. + +The full tutorial code is [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/univ_intrin/univ_intrin.cpp). + +### Vectorizing Convolution + +We will first implement a 1-D convolution and then vectorize it. The 2-D vectorized convolution will perform 1-D convolution across the rows to produce the correct results. + +#### 1-D Convolution: Scalar +@snippet univ_intrin.cpp convolution-1D-scalar + +1. We first set up variables and make a border on both sides of the src matrix, to take care of edge cases. + @snippet univ_intrin.cpp convolution-1D-border + +2. For the main loop, we select an index *i* and offset it on both sides along with the kernel, using the k variable. We store the value in *value* and add it to the *dst* matrix. + @snippet univ_intrin.cpp convolution-1D-scalar-main + +#### 1-D Convolution: Vector + +We will now look at the vectorized version of 1-D convolution. +@snippet univ_intrin.cpp convolution-1D-vector + +1. In our case, the kernel is a float. Since the kernel's datatype is the largest, we convert src to float32, forming *src_32*. We also make a border like we did for the naive case. + @snippet univ_intrin.cpp convolution-1D-convert + +2. Now, for each column in the *kernel*, we calculate the scalar product of the value with all *window* vectors of length `step`. We add these values to the already stored values in ans + @snippet univ_intrin.cpp convolution-1D-main + + * We declare a pointer to the src_32 and kernel and run a loop for each kernel element + @snippet univ_intrin.cpp convolution-1D-main-h1 + + * We load a register with the current kernel element. A window is shifted from *0* to *len - step* and its product with the kernel_wide array is added to the values stored in *ans*. We store the values back into *ans* + @snippet univ_intrin.cpp convolution-1D-main-h2 + + * Since the length might not be divisible by steps, we take care of the remaining values directly. The number of *tail* values will always be less than *step* and will not affect the performance significantly. We store all the values to *ans* which is a float pointer. We can also directly store them in a `Mat` object + @snippet univ_intrin.cpp convolution-1D-main-h3 + + * Here is an iterative example: + + For example: + kernel: {k1, k2, k3} + src: ...|a1|a2|a3|a4|... + + + iter1: + for each idx i in (0, len), 'step' idx at a time + kernel_wide: |k1|k1|k1|k1| + window: |a0|a1|a2|a3| + ans: ...| 0| 0| 0| 0|... + sum = ans + window * kernel_wide + = |a0 * k1|a1 * k1|a2 * k1|a3 * k1| + + iter2: + kernel_wide: |k2|k2|k2|k2| + window: |a1|a2|a3|a4| + ans: ...|a0 * k1|a1 * k1|a2 * k1|a3 * k1|... + sum = ans + window * kernel_wide + = |a0 * k1 + a1 * k2|a1 * k1 + a2 * k2|a2 * k1 + a3 * k2|a3 * k1 + a4 * k2| + + iter3: + kernel_wide: |k3|k3|k3|k3| + window: |a2|a3|a4|a5| + ans: ...|a0 * k1 + a1 * k2|a1 * k1 + a2 * k2|a2 * k1 + a3 * k2|a3 * k1 + a4 * k2|... + sum = sum + window * kernel_wide + = |a0*k1 + a1*k2 + a2*k3|a1*k1 + a2*k2 + a3*k3|a2*k1 + a3*k2 + a4*k3|a3*k1 + a4*k2 + a5*k3| + + +@note The function parameters also include *row*, *rowk* and *len*. These values are used when using the function as an intermediate step of 2-D convolution + +#### 2-D Convolution + +Suppose our kernel has *ksize* rows. To compute the values for a particular row, we compute the 1-D convolution of the previous *ksize/2* and the next *ksize/2* rows, with the corresponding kernel row. The final values is simply the sum of the individual 1-D convolutions +@snippet univ_intrin.cpp convolution-2D + +1. We first initialize variables and make a border above and below the *src* matrix. The left and right sides are handled by the 1-D convolution function. + @snippet univ_intrin.cpp convolution-2D-init + +2. For each row, we calculate the 1-D convolution of the rows above and below it. we then add the values to the *dst* matrix. + @snippet univ_intrin.cpp convolution-2D-main + +3. We finally convert the *dst* matrix to a *8-bit* `unsigned char` matrix + @snippet univ_intrin.cpp convolution-2D-conv + +Results +------- + +In the tutorial, we used a horizontal gradient kernel. We obtain the same output image for both methods. + +Improvement in runtime varies and will depend on the SIMD capabilities available in your CPU. diff --git a/samples/cpp/tutorial_code/core/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_new.cpp b/samples/cpp/tutorial_code/core/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_new.cpp new file mode 100644 index 000000000000..cfa9d22b0d05 --- /dev/null +++ b/samples/cpp/tutorial_code/core/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_new.cpp @@ -0,0 +1,332 @@ +#include +#include +#include +#include +#include + +using namespace std; +using namespace cv; + +namespace +{ +//! [convolution-sequential] +void conv_seq(Mat src, Mat &dst, Mat kernel) +{ + //![convolution-make-borders] + int rows = src.rows, cols = src.cols; + dst = Mat(rows, cols, src.type()); + + // Taking care of edge values + // Make border = kernel.rows / 2; + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + //![convolution-make-borders] + + //! [convolution-kernel-loop] + for (int i = 0; i < rows; i++) + { + uchar *dptr = dst.ptr(i); + for (int j = 0; j < cols; j++) + { + double value = 0; + + for (int k = -sz; k <= sz; k++) + { + // slightly faster results when we create a ptr due to more efficient memory access. + uchar *sptr = src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + dptr[j] = saturate_cast(value); + } + } + //! [convolution-kernel-loop] +} +//! [convolution-sequential] + +#ifdef CV_CXX11 +void conv_parallel(Mat src, Mat &dst, Mat kernel) +{ + int rows = src.rows, cols = src.cols; + + dst = Mat(rows, cols, CV_8UC1, Scalar(0)); + + // Taking care of edge values + // Make border = kernel.rows / 2; + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + + //! [convolution-parallel-cxx11] + parallel_for_(Range(0, rows * cols), [&](const Range &range) + { + for (int r = range.start; r < range.end; r++) + { + int i = r / cols, j = r % cols; + + double value = 0; + for (int k = -sz; k <= sz; k++) + { + uchar *sptr = src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + dst.ptr(i)[j] = saturate_cast(value); + } + }); + //! [convolution-parallel-cxx11] +} + +void conv_parallel_row_split(Mat src, Mat &dst, Mat kernel) +{ + int rows = src.rows, cols = src.cols; + + dst = Mat(rows, cols, CV_8UC1, Scalar(0)); + + // Taking care of edge values + // Make border = kernel.rows / 2; + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + + //! [convolution-parallel-cxx11-row-split] + parallel_for_(Range(0, rows), [&](const Range &range) + { + for (int i = range.start; i < range.end; i++) + { + + uchar *dptr = dst.ptr(i); + for (int j = 0; j < cols; j++) + { + double value = 0; + for (int k = -sz; k <= sz; k++) + { + uchar *sptr = src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + dptr[j] = saturate_cast(value); + } + } + }); + //! [convolution-parallel-cxx11-row-split] +} +#else + +//! [convolution-parallel] +class parallelConvolution : public ParallelLoopBody +{ +private: + Mat m_src, &m_dst; + Mat m_kernel; + int sz; + +public: + parallelConvolution(Mat src, Mat &dst, Mat kernel) + : m_src(src), m_dst(dst), m_kernel(kernel) + { + sz = kernel.rows / 2; + } + + //! [overload-full] + virtual void operator()(const Range &range) const CV_OVERRIDE + { + for (int r = range.start; r < range.end; r++) + { + int i = r / m_src.cols, j = r % m_src.cols; + + double value = 0; + for (int k = -sz; k <= sz; k++) + { + uchar *sptr = m_src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += m_kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + m_dst.ptr(i)[j] = saturate_cast(value); + } + } + //! [overload-full] +}; +//! [convolution-parallel] + +void conv_parallel(Mat src, Mat &dst, Mat kernel) +{ + int rows = src.rows, cols = src.cols; + + dst = Mat(rows, cols, CV_8UC1, Scalar(0)); + + // Taking care of edge values + // Make border = kernel.rows / 2; + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + + //! [convolution-parallel-function] + parallelConvolution obj(src, dst, kernel); + parallel_for_(Range(0, rows * cols), obj); + //! [convolution-parallel-function] +} + +//! [conv-parallel-row-split] +class parallelConvolutionRowSplit : public ParallelLoopBody +{ +private: + Mat m_src, &m_dst; + Mat m_kernel; + int sz; + +public: + parallelConvolutionRowSplit(Mat src, Mat &dst, Mat kernel) + : m_src(src), m_dst(dst), m_kernel(kernel) + { + sz = kernel.rows / 2; + } + + //! [overload-row-split] + virtual void operator()(const Range &range) const CV_OVERRIDE + { + for (int i = range.start; i < range.end; i++) + { + + uchar *dptr = dst.ptr(i); + for (int j = 0; j < cols; j++) + { + double value = 0; + for (int k = -sz; k <= sz; k++) + { + uchar *sptr = src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + dptr[j] = saturate_cast(value); + } + } + } + //! [overload-row-split] +}; +//! [conv-parallel-row-split] + +void conv_parallel_row_split(Mat src, Mat &dst, Mat kernel) +{ + int rows = src.rows, cols = src.cols; + + dst = Mat(rows, cols, CV_8UC1, Scalar(0)); + + // Taking care of edge values + // Make border = kernel.rows / 2; + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + + //! [convolution-parallel-function-row] + parallelConvolutionRowSplit obj(src, dst, kernel); + parallel_for_(Range(0, rows), obj); + //! [convolution-parallel-function-row] +} + +#endif + +static void help(char *progName) +{ + cout << endl + << " This program shows how to use the OpenCV parallel_for_ function and \n" + << " compares the performance of the sequential and parallel implementations for a \n" + << " convolution operation\n" + << " Usage:\n " + << progName << " [image_path -- default lena.jpg] " << endl + << endl; +} +} + +int main(int argc, char *argv[]) +{ + + help(argv[0]); + const char *filepath = argc >= 2 ? argv[1] : "../../../../data/lena.jpg"; + + Mat src, dst, kernel; + src = imread(filepath, IMREAD_GRAYSCALE); + + if (src.empty()) + { + cerr << "Can't open [" << filepath << "]" << endl; + return EXIT_FAILURE; + } + namedWindow("Input", 1); + namedWindow("Output1", 1); + namedWindow("Output2", 1); + namedWindow("Output3", 1); + imshow("Input", src); + + kernel = (Mat_(3, 3) << 1, 0, -1, + 1, 0, -1, + 1, 0, -1); + + /* + Uncomment the kernels you want to use or write your own kernels to test out + performance. + */ + + /* + kernel = (Mat_(5, 5) << 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1); + kernel /= 100; + */ + + /* + kernel = (Mat_(3, 3) << 1, 1, 1, + 0, 0, 0, + -1, -1, -1); + + */ + + double t = (double)getTickCount(); + + conv_seq(src, dst, kernel); + + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Sequential implementation: " << t << "s" << endl; + + imshow("Output1", dst); + waitKey(0); + + t = (double)getTickCount(); + + conv_parallel(src, dst, kernel); + + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Parallel Implementation: " << t << "s" << endl; + + imshow("Output2", dst); + waitKey(0); + + t = (double)getTickCount(); + + conv_parallel_row_split(src, dst, kernel); + + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Parallel Implementation(Row Split): " << t << "s" << endl + << endl; + + imshow("Output3", dst); + waitKey(0); + + // imwrite("src.png", src); + // imwrite("dst.png", dst); + + return 0; +} \ No newline at end of file diff --git a/samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp b/samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp new file mode 100644 index 000000000000..9be4170d7b5f --- /dev/null +++ b/samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp @@ -0,0 +1,230 @@ +#include +#include +#include +#include +#include +#include + +using namespace std; +using namespace cv; + +const int N = 100005, K = 2000; + +namespace +{ + +void conv_seq(Mat src, Mat &dst, Mat kernel) +{ + int rows = src.rows, cols = src.cols; + dst = Mat(rows, cols, CV_8UC1); + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + for (int i = 0; i < rows; i++) + { + uchar *dptr = dst.ptr(i); + for (int j = 0; j < cols; j++) + { + float value = 0; + + for (int k = -sz; k <= sz; k++) + { + // slightly faster results when we create a ptr due to more efficient memory access. + uchar *sptr = src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + dptr[j] = saturate_cast(value); + } + } +} + +//! [convolution-1D-scalar] +void conv1d(Mat src, Mat &dst, Mat kernel) +{ + + //! [convolution-1D-border] + int len = src.cols; + dst = Mat(1, len, CV_8UC1); + + int sz = kernel.cols / 2; + copyMakeBorder(src, src, 0, 0, sz, sz, BORDER_REPLICATE); + //! [convolution-1D-border] + + //! [convolution-1D-scalar-main] + for (int i = 0; i < len; i++) + { + double value = 0; + for (int k = -sz; k <= sz; k++) + value += src.ptr(0)[i + k + sz] * kernel.ptr(0)[k + sz]; + + dst.ptr(0)[i] = saturate_cast(value); + } + //! [convolution-1D-scalar-main] +} +//! [convolution-1D-scalar] + +//! [convolution-1D-vector] +void conv1dsimd(Mat src, Mat kernel, float *ans, int row = 0, int rowk = 0, int len = -1) +{ + if (len == -1) + len = src.cols; + + //! [convolution-1D-convert] + Mat src_32, kernel_32; + + const int alpha = 1; + src.convertTo(src_32, CV_32FC1, alpha); + + int ksize = kernel.cols, sz = kernel.cols / 2; + copyMakeBorder(src_32, src_32, 0, 0, sz, sz, BORDER_REPLICATE); + //! [convolution-1D-convert] + + + //! [convolution-1D-main] + //! [convolution-1D-main-h1] + int step = v_float32().nlanes; + float *sptr = src_32.ptr(row), *kptr = kernel.ptr(rowk); + for (int k = 0; k < ksize; k++) + { + //! [convolution-1D-main-h1] + //! [convolution-1D-main-h2] + v_float32 kernel_wide = vx_setall_f32(kptr[k]); + int i; + for (i = 0; i + step < len; i += step) + { + v_float32 window = vx_load(sptr + i + k); + v_float32 sum = vx_load(ans + i) + kernel_wide * window; + v_store(ans + i, sum); + } + //! [convolution-1D-main-h2] + + //! [convolution-1D-main-h3] + for (; i < len; i++) + { + *(ans + i) += sptr[i + k]*kptr[k]; + } + //! [convolution-1D-main-h3] + } + //! [convolution-1D-main] +} +//! [convolution-1D-vector] + +//! [convolution-2D] +void convolute_simd(Mat src, Mat &dst, Mat kernel) +{ + //! [convolution-2D-init] + int rows = src.rows, cols = src.cols; + int ksize = kernel.rows, sz = ksize / 2; + dst = Mat(rows, cols, CV_32FC1); + + copyMakeBorder(src, src, sz, sz, 0, 0, BORDER_REPLICATE); + + int step = v_float32().nlanes; + //! [convolution-2D-init] + + //! [convolution-2D-main] + for (int i = 0; i < rows; i++) + { + for (int k = 0; k < ksize; k++) + { + float ans[N] = {0}; + conv1dsimd(src, kernel, ans, i + k, k, cols); + int j; + for (j = 0; j + step < cols; j += step) + { + v_float32 sum = vx_load(&dst.ptr(i)[j]) + vx_load(&ans[j]); + v_store(&dst.ptr(i)[j], sum); + } + + for (; j < cols; j++) + dst.ptr(i)[j] += ans[j]; + } + } + //! [convolution-2D-main] + + //! [convolution-2D-conv] + const int alpha = 1; + dst.convertTo(dst, CV_8UC1, alpha); + //! [convolution-2D-conv] +} +//! [convolution-2D] + +static void help(char *progName) +{ + cout << endl + << " This program shows how to use the OpenCV parallel_for_ function and \n" + << " compares the performance of the sequential and parallel implementations for a \n" + << " convolution operation\n" + << " Usage:\n " + << progName << " [image_path -- default lena.jpg] " << endl + << endl; +} +} + +int main(int argc, char *argv[]) +{ + + // 1-D Convolution // + Mat vsrc(1, N, CV_8UC1), k(1, K, CV_32FC1), vdst; + RNG rng(time(0)); + rng.RNG::fill(vsrc, RNG::UNIFORM, Scalar(0), Scalar(255)); + rng.RNG::fill(k, RNG::UNIFORM, Scalar(-50), Scalar(50)); + + double t = (double)getTickCount(); + conv1d(vsrc, vdst, k); + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Sequential 1-D convolution implementation: " << t << "s" << endl; + + t = (double)getTickCount(); + float ans[N] = {0}; + conv1dsimd(vsrc, k, ans); + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Vectorized 1-D convolution implementation: " << t << "s" << endl; + + // 2-D Convolution // + help(argv[0]); + + const char *filepath = argc >= 2 ? argv[1] : "../../../../data/lena.jpg"; + + Mat src, dst1, dst2, kernel; + src = imread(filepath, IMREAD_GRAYSCALE); + + if (src.empty()) + { + cerr << "Can't open [" << filepath << "]" << endl; + return EXIT_FAILURE; + } + namedWindow("Input", 1); + namedWindow("Output", 1); + imshow("Input", src); + + kernel = (Mat_(3, 3) << 1, 0, -1, + 2, 0, -2, + 1, 0, -1); + + t = (double)getTickCount(); + + conv_seq(src, dst1, kernel); + + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Sequential 2-D convolution implementation: " << t << "s" << endl; + + imshow("Output", dst1); + waitKey(0); + + t = (double)getTickCount(); + + convolute_simd(src, dst2, kernel); + + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Vectorized 2-D convolution implementation: " << t << "s" << endl + << endl; + + imshow("Output", dst2); + waitKey(0); + + return 0; +} \ No newline at end of file From 5c25d17f1ed0ae4ff5e07ae589b6f896172bcb67 Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Thu, 16 Sep 2021 13:09:56 +0800 Subject: [PATCH 207/376] Update tutorial --- .../js_image_classification_webnn_electron.html | 1 + doc/js_tutorials/js_setup/js_setup/js_setup.markdown | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html b/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html index f3b075489d0a..c8119247f94d 100644 --- a/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html +++ b/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html @@ -126,6 +126,7 @@

Model Info:

let net = cv.readNet(configPath, modelPath); net.setPreferableBackend(6); net.setInput(input); + net.forward(); const start = performance.now(); const result = net.forward(); const time = performance.now()-start; diff --git a/doc/js_tutorials/js_setup/js_setup/js_setup.markdown b/doc/js_tutorials/js_setup/js_setup/js_setup.markdown index ad14185a35c7..992747744327 100644 --- a/doc/js_tutorials/js_setup/js_setup/js_setup.markdown +++ b/doc/js_tutorials/js_setup/js_setup/js_setup.markdown @@ -145,6 +145,12 @@ Building OpenCV.js from Source python ./platforms/js/build_js.py build_js --cmake_option="-DOPENCV_EXTRA_MODULES_PATH=opencv_contrib/modules" @endcode +-# [optional] To enable WebNN backend, append `--webnn` option. + + For example: + @code{.bash} + emcmake python ./opencv/platforms/js/build_js.py build_js --webnn + @endcode Running OpenCV.js Tests --------------------------------------- From 9c5d7716e273284d4973eefdef079436c7caaa9d Mon Sep 17 00:00:00 2001 From: SamFC10 Date: Fri, 17 Sep 2021 17:40:57 +0530 Subject: [PATCH 208/376] fix for unsqueeze opset version 13 --- modules/dnn/src/onnx/onnx_importer.cpp | 12 ++++++++++-- modules/dnn/test/test_onnx_importer.cpp | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 2cd36dc94db5..5343c0536178 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -1693,8 +1693,16 @@ void ONNXImporter::parseFlatten(LayerParams& layerParams, const opencv_onnx::Nod void ONNXImporter::parseUnsqueeze(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - CV_Assert(node_proto.input_size() == 1); - DictValue axes = layerParams.get("axes"); + CV_Assert(node_proto.input_size() == 1 || node_proto.input_size() == 2); + DictValue axes; + if (node_proto.input_size() == 2) + { + Mat blob = getBlob(node_proto, 1); + axes = DictValue::arrayInt(blob.ptr(), blob.total()); + } + else + axes = layerParams.get("axes"); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) { // Constant input. diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index f55510ec7bd9..b4ecd4601c4f 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -605,6 +605,7 @@ TEST_P(Test_ONNX_layers, DynamicReshape) TEST_P(Test_ONNX_layers, Reshape) { testONNXModels("unsqueeze"); + testONNXModels("unsqueeze_opset_13"); } TEST_P(Test_ONNX_layers, Squeeze) From 3c89a28a0612c3f44d9b44fbe28e4bf0ded564d2 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Sat, 18 Sep 2021 10:02:55 +0300 Subject: [PATCH 209/376] Merge pull request #20611 from VadimLevin:dev/vlevin/pure-python-modules * feat: OpenCV extension with pure Python modules * feat: cv2 is now a Python package instead of extension module Python package cv2 now can handle both Python and C extension modules properly without additional "subfolders" like "_extra_py_code". * feat: can call native function from its reimplementation in Python --- .../include/opencv2/core/bindings_utils.hpp | 6 ++ .../misc/python/package/utils/__init__.py | 14 ++++ modules/python/common.cmake | 47 +++++++---- modules/python/package/cv2/__init__.py | 81 +++++++++++++++---- .../package/cv2/_extra_py_code/__init__.py | 53 ------------ .../package/extra_modules/misc/__init__.py | 1 + .../package/extra_modules/misc/version.py | 5 ++ modules/python/python3/CMakeLists.txt | 14 ++++ modules/python/python_loader.cmake | 1 - modules/python/test/test_misc.py | 31 +++++++ 10 files changed, 169 insertions(+), 84 deletions(-) create mode 100644 modules/core/misc/python/package/utils/__init__.py delete mode 100644 modules/python/package/cv2/_extra_py_code/__init__.py create mode 100644 modules/python/package/extra_modules/misc/__init__.py create mode 100644 modules/python/package/extra_modules/misc/version.py diff --git a/modules/core/include/opencv2/core/bindings_utils.hpp b/modules/core/include/opencv2/core/bindings_utils.hpp index a3f45eb0c771..f9c73dd4ff74 100644 --- a/modules/core/include/opencv2/core/bindings_utils.hpp +++ b/modules/core/include/opencv2/core/bindings_utils.hpp @@ -116,6 +116,12 @@ String dumpRange(const Range& argument) } } +CV_WRAP static inline +int testOverwriteNativeMethod(int argument) +{ + return argument; +} + CV_WRAP static inline String testReservedKeywordConversion(int positional_argument, int lambda = 2, int from = 3) { diff --git a/modules/core/misc/python/package/utils/__init__.py b/modules/core/misc/python/package/utils/__init__.py new file mode 100644 index 000000000000..49cd40ba2db2 --- /dev/null +++ b/modules/core/misc/python/package/utils/__init__.py @@ -0,0 +1,14 @@ +from collections import namedtuple + +import cv2 + + +NativeMethodPatchedResult = namedtuple("NativeMethodPatchedResult", + ("py", "native")) + + +def testOverwriteNativeMethod(arg): + return NativeMethodPatchedResult( + arg + 1, + cv2.utils._native.testOverwriteNativeMethod(arg) + ) diff --git a/modules/python/common.cmake b/modules/python/common.cmake index cedf07143488..251b78c6cb3b 100644 --- a/modules/python/common.cmake +++ b/modules/python/common.cmake @@ -1,6 +1,31 @@ # This file is included from a subdirectory set(PYTHON_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}") +function(ocv_add_python_files_from_path search_path) + file(GLOB_RECURSE extra_py_files + RELATIVE "${search_path}" + # Plain Python code + "${search_path}/*.py" + # Type annotations + "${search_path}/*.pyi" + ) + message(DEBUG "Extra Py files for ${search_path}: ${extra_py_files}") + if(extra_py_files) + list(SORT extra_py_files) + foreach(filename ${extra_py_files}) + get_filename_component(module "${filename}" DIRECTORY) + if(NOT ${module} IN_LIST extra_modules) + list(APPEND extra_modules ${module}) + endif() + configure_file("${search_path}/${filename}" "${__loader_path}/cv2/${filename}" COPYONLY) + install(FILES "${search_path}/${filename}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/${module}/" COMPONENT python) + endforeach() + message(STATUS "Found ${extra_modules} Python modules from ${search_path}") + else() + message(WARNING "Can't add Python files and modules from ${module_path}. There is no .py or .pyi files") + endif() +endfunction() + ocv_add_module(${MODULE_NAME} BINDINGS PRIVATE_REQUIRED opencv_python_bindings_generator) include_directories(SYSTEM @@ -224,23 +249,15 @@ if(NOT OPENCV_SKIP_PYTHON_LOADER) if (";${OPENCV_MODULE_${m}_WRAPPERS};" MATCHES ";python;" AND HAVE_${m} AND EXISTS "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package" ) - set(__base "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package") - file(GLOB_RECURSE extra_py_files - RELATIVE "${__base}" - "${__base}/**/*.py" - ) - if(extra_py_files) - list(SORT extra_py_files) - foreach(f ${extra_py_files}) - get_filename_component(__dir "${f}" DIRECTORY) - configure_file("${__base}/${f}" "${__loader_path}/cv2/_extra_py_code/${f}" COPYONLY) - install(FILES "${__base}/${f}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/_extra_py_code/${__dir}/" COMPONENT python) - endforeach() - else() - message(WARNING "Module ${m} has no .py files in misc/python/package") - endif() + ocv_add_python_files_from_path("${OPENCV_MODULE_${m}_LOCATION}/misc/python/package") endif() endforeach(m) + + if(NOT "${OCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "") + foreach(extra_ocv_py_modules_path ${OCV_PYTHON_EXTRA_MODULES_PATH}) + ocv_add_python_files_from_path(${extra_ocv_py_modules_path}) + endforeach() + endif() endif() # NOT OPENCV_SKIP_PYTHON_LOADER unset(PYTHON_SRC_DIR) diff --git a/modules/python/package/cv2/__init__.py b/modules/python/package/cv2/__init__.py index 27e7edd083d4..80e26122764f 100644 --- a/modules/python/package/cv2/__init__.py +++ b/modules/python/package/cv2/__init__.py @@ -2,6 +2,7 @@ OpenCV Python binary extension loader ''' import os +import importlib import sys __all__ = [] @@ -15,16 +16,53 @@ print(' pip install numpy') raise +# TODO +# is_x64 = sys.maxsize > 2**32 + -py_code_loader = None -if sys.version_info[:2] >= (3, 0): +def __load_extra_py_code_for_module(base, name, enable_debug_print=False): + module_name = "{}.{}".format(__name__, name) + export_module_name = "{}.{}".format(base, name) + native_module = sys.modules.pop(module_name, None) try: - from . import _extra_py_code as py_code_loader - except: - pass + py_module = importlib.import_module(module_name) + except ImportError as err: + if enable_debug_print: + print("Can't load Python code for module:", module_name, + ". Reason:", err) + # Extension doesn't contain extra py code + return False + + if not hasattr(base, name): + setattr(sys.modules[base], name, py_module) + sys.modules[export_module_name] = py_module + # If it is C extension module it is already loaded by cv2 package + if native_module: + setattr(py_module, "_native", native_module) + for k, v in filter(lambda kv: not hasattr(py_module, kv[0]), + native_module.__dict__.items()): + if enable_debug_print: print(' symbol: {} = {}'.format(k, v)) + setattr(py_module, k, v) + return True + + +def __collect_extra_submodules(enable_debug_print=False): + def modules_filter(module): + return all(( + # module is not internal + not module.startswith("_"), + # it is not a file + os.path.isdir(os.path.join(_extra_submodules_init_path, module)) + )) + if sys.version_info[0] < 3: + if enable_debug_print: + print("Extra submodules is loaded only for Python 3") + return [] + + __INIT_FILE_PATH = os.path.abspath(__file__) + _extra_submodules_init_path = os.path.dirname(__INIT_FILE_PATH) + return filter(modules_filter, os.listdir(_extra_submodules_init_path)) -# TODO -# is_x64 = sys.maxsize > 2**32 def bootstrap(): import sys @@ -107,23 +145,36 @@ def load_first_config(fnames, required=True): # amending of LD_LIBRARY_PATH works for sub-processes only os.environ['LD_LIBRARY_PATH'] = ':'.join(l_vars['BINARIES_PATHS']) + ':' + os.environ.get('LD_LIBRARY_PATH', '') - if DEBUG: print('OpenCV loader: replacing cv2 module') - del sys.modules['cv2'] - import cv2 + if DEBUG: print("Relink everything from native cv2 module to cv2 package") + + py_module = sys.modules.pop("cv2") + + native_module = importlib.import_module("cv2") + + sys.modules["cv2"] = py_module + setattr(py_module, "_native", native_module) + + for item_name, item in filter(lambda kv: kv[0] not in ("__file__", "__loader__", "__spec__", + "__name__", "__package__"), + native_module.__dict__.items()): + if item_name not in g_vars: + g_vars[item_name] = item sys.path = save_sys_path # multiprocessing should start from bootstrap code (https://github.com/opencv/opencv/issues/18502) try: - import sys del sys.OpenCV_LOADER - except: - pass + except Exception as e: + if DEBUG: + print("Exception during delete OpenCV_LOADER:", e) if DEBUG: print('OpenCV loader: binary extension... OK') - if py_code_loader: - py_code_loader.init('cv2') + for submodule in __collect_extra_submodules(DEBUG): + if __load_extra_py_code_for_module("cv2", submodule, DEBUG): + if DEBUG: print("Extra Python code for", submodule, "is loaded") if DEBUG: print('OpenCV loader: DONE') + bootstrap() diff --git a/modules/python/package/cv2/_extra_py_code/__init__.py b/modules/python/package/cv2/_extra_py_code/__init__.py deleted file mode 100644 index be84566825bf..000000000000 --- a/modules/python/package/cv2/_extra_py_code/__init__.py +++ /dev/null @@ -1,53 +0,0 @@ -import sys -import importlib - -__all__ = ['init'] - - -DEBUG = False -if hasattr(sys, 'OpenCV_LOADER_DEBUG'): - DEBUG = True - - -def _load_py_code(base, name): - try: - m = importlib.import_module(__name__ + name) - except ImportError: - return # extension doesn't exist? - - if DEBUG: print('OpenCV loader: added python code extension for: ' + name) - - if hasattr(m, '__all__'): - export_members = { k : getattr(m, k) for k in m.__all__ } - else: - export_members = m.__dict__ - - for k, v in export_members.items(): - if k.startswith('_'): # skip internals - continue - if isinstance(v, type(sys)): # don't bring modules - continue - if DEBUG: print(' symbol: {} = {}'.format(k, v)) - setattr(sys.modules[base + name ], k, v) - - del sys.modules[__name__ + name] - - -# TODO: listdir -def init(base): - _load_py_code(base, '.cv2') # special case - prefix = base - prefix_len = len(prefix) - - modules = [ m for m in sys.modules.keys() if m.startswith(prefix) ] - for m in modules: - m2 = m[prefix_len:] # strip prefix - if len(m2) == 0: - continue - if m2.startswith('._'): # skip internals - continue - if m2.startswith('.load_config_'): # skip helper files - continue - _load_py_code(base, m2) - - del sys.modules[__name__] diff --git a/modules/python/package/extra_modules/misc/__init__.py b/modules/python/package/extra_modules/misc/__init__.py new file mode 100644 index 000000000000..3e0559d3d471 --- /dev/null +++ b/modules/python/package/extra_modules/misc/__init__.py @@ -0,0 +1 @@ +from .version import get_ocv_version diff --git a/modules/python/package/extra_modules/misc/version.py b/modules/python/package/extra_modules/misc/version.py new file mode 100644 index 000000000000..d34705872eda --- /dev/null +++ b/modules/python/package/extra_modules/misc/version.py @@ -0,0 +1,5 @@ +import cv2 + + +def get_ocv_version(): + return getattr(cv2, "__version__", "unavailable") diff --git a/modules/python/python3/CMakeLists.txt b/modules/python/python3/CMakeLists.txt index d95af21e04b3..0bb401f2ec16 100644 --- a/modules/python/python3/CMakeLists.txt +++ b/modules/python/python3/CMakeLists.txt @@ -15,9 +15,23 @@ set(the_description "The python3 bindings") set(MODULE_NAME python3) set(MODULE_INSTALL_SUBDIR python3) +set(_ocv_extra_modules_path ${CMAKE_CURRENT_LIST_DIR}/../package/extra_modules) +set(_old_ocv_python_extra_modules_path ${OCV_PYTHON_EXTRA_MODULES_PATH}) + +if("${OCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "") + set(OCV_PYTHON_EXTRA_MODULES_PATH ${_ocv_extra_modules_path}) +else() + list(APPEND OCV_PYTHON_EXTRA_MODULES_PATH ${_ocv_extra_modules_path}) +endif() + +unset(_ocv_extra_modules_path) + set(PYTHON PYTHON3) include(../common.cmake) +set(OCV_PYTHON_EXTRA_MODULES_PATH ${_old_ocv_python_extra_modules_path}) + +unset(_old_ocv_python_extra_modules_path) unset(MODULE_NAME) unset(MODULE_INSTALL_SUBDIR) diff --git a/modules/python/python_loader.cmake b/modules/python/python_loader.cmake index 677111b0612a..a872ffd763de 100644 --- a/modules/python/python_loader.cmake +++ b/modules/python/python_loader.cmake @@ -25,7 +25,6 @@ endif() set(PYTHON_LOADER_FILES "setup.py" "cv2/__init__.py" "cv2/load_config_py2.py" "cv2/load_config_py3.py" - "cv2/_extra_py_code/__init__.py" ) foreach(fname ${PYTHON_LOADER_FILES}) get_filename_component(__dir "${fname}" DIRECTORY) diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index 111b1427b4e1..76803992dca9 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -4,6 +4,12 @@ import ctypes from functools import partial from collections import namedtuple +import sys + +if sys.version_info[0] < 3: + from collections import Sequence +else: + from collections.abc import Sequence import numpy as np import cv2 as cv @@ -585,6 +591,31 @@ def test_vector_fast_return(self): self.assertEqual(ints.shape, expected_shape, "Vector of integers has wrong shape.") +class CanUsePurePythonModuleFunction(NewOpenCVTests): + def test_can_get_ocv_version(self): + import sys + if sys.version_info[0] < 3: + raise unittest.SkipTest('Python 2.x is not supported') + + self.assertEqual(cv.misc.get_ocv_version(), cv.__version__, + "Can't get package version using Python misc module") + + def test_native_method_can_be_patched(self): + import sys + + if sys.version_info[0] < 3: + raise unittest.SkipTest('Python 2.x is not supported') + + res = cv.utils.testOverwriteNativeMethod(10) + self.assertTrue(isinstance(res, Sequence), + msg="Overwritten method should return sequence. " + "Got: {} of type {}".format(res, type(res))) + self.assertSequenceEqual(res, (11, 10), + msg="Failed to overwrite native method") + res = cv.utils._native.testOverwriteNativeMethod(123) + self.assertEqual(res, 123, msg="Failed to call native method implementation") + + class SamplesFindFile(NewOpenCVTests): def test_ExistedFile(self): From ba8f9d8620de9ff6cd0b81ab422a4d5d08f5b4e0 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Mon, 20 Sep 2021 16:28:32 +0300 Subject: [PATCH 210/376] Merge pull request #20601 from sivanov-work:surf_bk_test G-API: oneVPL (simplification) added surface & frame adapter * added surface & frame adapter * Add FrameAdapter unut tests * Fix compilation after rebase * Fix compilation tests * Apply some review comments * Fix compile warning * Revert back CV_Func usage * Apply some comments --- modules/gapi/CMakeLists.txt | 2 + .../accelerators/accel_policy_interface.hpp | 59 ++++++ .../surface/cpu_frame_adapter.cpp | 117 +++++++++++ .../surface/cpu_frame_adapter.hpp | 41 ++++ .../onevpl/accelerators/surface/surface.cpp | 76 +++++++ .../onevpl/accelerators/surface/surface.hpp | 102 ++++++++++ .../gapi_streaming_vpl_core_test.cpp | 185 ++++++++++++++++++ 7 files changed, 582 insertions(+) create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp create mode 100644 modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 08dee2f9af3c..56d051cf0fde 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -169,6 +169,8 @@ set(gapi_srcs src/streaming/onevpl/file_data_provider.cpp src/streaming/onevpl/cfg_params.cpp src/streaming/onevpl/data_provider_interface_exception.cpp + src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp + src/streaming/onevpl/accelerators/surface/surface.cpp # Utils (ITT tracing) src/utils/itt.cpp diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp new file mode 100644 index 000000000000..87b1246d257e --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp @@ -0,0 +1,59 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_INTERFACE_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_INTERFACE_HPP + +#include +#include +#include + +#include + +#ifdef HAVE_ONEVPL +#include + +namespace cv { +namespace gapi { +namespace wip { + +class Surface; +struct VPLAccelerationPolicy +{ + virtual ~VPLAccelerationPolicy() {} + + using pool_key_t = void*; + + using session_t = mfxSession; + using surface_t = Surface; + using surface_ptr_t = std::shared_ptr; + using surface_weak_ptr_t = std::weak_ptr; + using surface_ptr_ctr_t = std::function out_buf_ptr, + size_t out_buf_ptr_offset, + size_t out_buf_ptr_size)>; + + virtual void init(session_t session) = 0; + virtual void deinit(session_t session) = 0; + + // Limitation: cannot give guarantee in succesful memory realloccation + // for existing workspace in existing pool (see realloc) + // thus it is not implemented, + // PLEASE provide initial memory area large enough + virtual pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator) = 0; + + virtual surface_weak_ptr_t get_free_surface(pool_key_t key) = 0; + virtual size_t get_free_surface_count(pool_key_t key) const = 0; + virtual size_t get_surface_count(pool_key_t key) const = 0; + + virtual cv::MediaFrame::AdapterPtr create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) = 0; +}; +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_INTERFACE_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp new file mode 100644 index 000000000000..3fd959ca8b46 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp @@ -0,0 +1,117 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONEVPL + +#if (MFX_VERSION >= 2000) +#include +#endif + +#include + +namespace cv { +namespace gapi { +namespace wip { + +VPLMediaFrameCPUAdapter::VPLMediaFrameCPUAdapter(std::shared_ptr surface): + parent_surface_ptr(surface) { + + GAPI_Assert(parent_surface_ptr && "Surface is nullptr"); + parent_surface_ptr->obtain_lock(); + + GAPI_LOG_DEBUG(nullptr, "surface: " << parent_surface_ptr->get_handle() << + ", w: " << parent_surface_ptr->get_info().Width << + ", h: " << parent_surface_ptr->get_info().Height << + ", p: " << parent_surface_ptr->get_data().Pitch); +} + +VPLMediaFrameCPUAdapter::~VPLMediaFrameCPUAdapter() { + + // Each VPLMediaFrameCPUAdapter releases mfx surface counter + // The last VPLMediaFrameCPUAdapter releases shared Surface pointer + // The last surface pointer releases workspace memory + parent_surface_ptr->release_lock(); +} + +cv::GFrameDesc VPLMediaFrameCPUAdapter::meta() const { + GFrameDesc desc; + const Surface::info_t& info = parent_surface_ptr->get_info(); + switch(info.FourCC) + { + case MFX_FOURCC_I420: + throw std::runtime_error("MediaFrame doesn't support I420 type"); + break; + case MFX_FOURCC_NV12: + desc.fmt = MediaFormat::NV12; + break; + default: + throw std::runtime_error("MediaFrame unknown 'fmt' type: " + std::to_string(info.FourCC)); + } + + desc.size = cv::Size{info.Width, info.Height}; + return desc; +} + +MediaFrame::View VPLMediaFrameCPUAdapter::access(MediaFrame::Access) { + const Surface::data_t& data = parent_surface_ptr->get_data(); + const Surface::info_t& info = parent_surface_ptr->get_info(); + using stride_t = typename cv::MediaFrame::View::Strides::value_type; + + stride_t pitch = static_cast(data.Pitch); + switch(info.FourCC) { + case MFX_FOURCC_I420: + { + cv::MediaFrame::View::Ptrs pp = { + data.Y, + data.U, + data.V, + nullptr + }; + cv::MediaFrame::View::Strides ss = { + pitch, + pitch / 2, + pitch / 2, 0u + }; + return cv::MediaFrame::View(std::move(pp), std::move(ss)); + } + case MFX_FOURCC_NV12: + { + cv::MediaFrame::View::Ptrs pp = { + data.Y, + data.UV, nullptr, nullptr + }; + cv::MediaFrame::View::Strides ss = { + pitch, + pitch, 0u, 0u + }; + return cv::MediaFrame::View(std::move(pp), std::move(ss)); + } + break; + default: + throw std::runtime_error("MediaFrame unknown 'fmt' type: " + std::to_string(info.FourCC)); + } +} + +cv::util::any VPLMediaFrameCPUAdapter::blobParams() const { + GAPI_Assert("VPLMediaFrameCPUAdapter::blobParams() is not implemented"); + return {}; +} + +void VPLMediaFrameCPUAdapter::serialize(cv::gapi::s11n::IOStream&) { + GAPI_Assert("VPLMediaFrameCPUAdapter::serialize() is not implemented"); +} + +void VPLMediaFrameCPUAdapter::deserialize(cv::gapi::s11n::IIStream&) { + GAPI_Assert("VPLMediaFrameCPUAdapter::deserialize() is not implemented"); +} +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp new file mode 100644 index 000000000000..16111dadafe3 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp @@ -0,0 +1,41 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_CPU_FRAME_ADAPTER_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_CPU_FRAME_ADAPTER_HPP +#include + +#include +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { + +class Surface; +class VPLMediaFrameCPUAdapter : public cv::MediaFrame::IAdapter { +public: + // GAPI_EXPORTS for tests + GAPI_EXPORTS explicit VPLMediaFrameCPUAdapter(std::shared_ptr assoc_surface); + GAPI_EXPORTS ~VPLMediaFrameCPUAdapter(); + cv::GFrameDesc meta() const override; + MediaFrame::View access(MediaFrame::Access) override; + + // The default implementation does nothing + cv::util::any blobParams() const override; + void serialize(cv::gapi::s11n::IOStream&) override; + void deserialize(cv::gapi::s11n::IIStream&) override; +private: + std::shared_ptr parent_surface_ptr; +}; +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_CPU_FRAME_ADAPTER_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp new file mode 100644 index 000000000000..5d315412e1be --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp @@ -0,0 +1,76 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { + +Surface::Surface(std::unique_ptr&& surf, std::shared_ptr associated_memory) : + workspace_memory_ptr(associated_memory), + mfx_surface(std::move(surf)), + mirrored_locked_count() { + + GAPI_Assert(mfx_surface && "Surface is nullptr"); + mirrored_locked_count.store(mfx_surface->Data.Locked); + GAPI_LOG_DEBUG(nullptr, "create surface: " << mfx_surface << + ", locked count: " << mfx_surface->Data.Locked); +} + +Surface::~Surface() { + GAPI_LOG_DEBUG(nullptr, "destroy surface: " << mfx_surface << + ", worspace memory counter: " << workspace_memory_ptr.use_count()); +} + +std::shared_ptr Surface::create_surface(std::unique_ptr&& surf, + std::shared_ptr accociated_memory) { + surface_ptr_t ret {new Surface(std::move(surf), accociated_memory)}; + return ret; +} + +Surface::handle_t* Surface::get_handle() const { + return mfx_surface.get(); +} + +const Surface::info_t& Surface::get_info() const { + return mfx_surface->Info; +} + +const Surface::data_t& Surface::get_data() const { + return mfx_surface->Data; +} + +size_t Surface::get_locks_count() const { + return mirrored_locked_count.load(); +} + +size_t Surface::obtain_lock() { + size_t locked_count = mirrored_locked_count.fetch_add(1); + GAPI_Assert(locked_count < std::numeric_limits::max() && "Too many references "); + mfx_surface->Data.Locked = static_cast(locked_count + 1); + GAPI_LOG_DEBUG(nullptr, "surface: " << mfx_surface.get() << + ", locked times: " << locked_count + 1); + return locked_count; // return preceding value +} + +size_t Surface::release_lock() { + size_t locked_count = mirrored_locked_count.fetch_sub(1); + GAPI_Assert(locked_count < std::numeric_limits::max() && "Too many references "); + GAPI_Assert(locked_count && "Surface lock counter is invalid"); + mfx_surface->Data.Locked = static_cast(locked_count - 1); + GAPI_LOG_DEBUG(nullptr, "surface: " << mfx_surface.get() << + ", locked times: " << locked_count - 1); + return locked_count; // return preceding value +} +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp new file mode 100644 index 000000000000..3fc30099cf49 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp @@ -0,0 +1,102 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_HPP + +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif + +#include + + +namespace cv { +namespace gapi { +namespace wip { + +/** + * @brief Inner class for managing oneVPL surface through interface `mfxFrameSurface1`. + * + * Surface has no own memory and shares accelerator allocated memory using reference counter semantics. + * So it lives till a last memory consumer (surface/accelerator/media frame) lives. + * This approach allows to support different scenarious in releasing allocated memory + * + * VPL surface `mfxFrameSurface1` support Lock-Free semantics and application MUST NOT operate with a + * surface in locked state. But VPL inner counter is not threadsafe so it would be failed in any concurrent scenario. + * std::atomic counter introduced in a way to overcome this problem. + * But only few scenarious for concurrency are supported here because it is not assumed to implement entire Surface in + * for a fully multithread approach. + * Supported concurrency scenarios deal only with transaction pair: @ref Surface::get_locks_count() against + * @ref Surface::release_lock() - which may be called from different threads. On the other hand @ref Surface::get_locks_count() against + * @ref Surface::obtain_lock() happens in single thread only. Surface doesn't support shared ownership that + * because it doesn't require thread safe guarantee between transactions: + * - @ref Surface::obtain_lock() against @ref Surface::obtain_lock() + * - @ref Surface::obtain_lock() against @ref Surface::release_lock() + * - @ref Surface::release_lock() against @ref Surface::release_lock() + */ +class Surface { + using handle_t = mfxFrameSurface1; + + std::shared_ptr workspace_memory_ptr; + std::unique_ptr mfx_surface; + std::atomic mirrored_locked_count; +public: + using info_t = mfxFrameInfo; + using data_t = mfxFrameData; + + // GAPI_EXPORTS for tests + GAPI_EXPORTS static std::shared_ptr create_surface(std::unique_ptr&& surf, + std::shared_ptr accociated_memory); + GAPI_EXPORTS ~Surface(); + + GAPI_EXPORTS handle_t* get_handle() const; + GAPI_EXPORTS const info_t& get_info() const; + GAPI_EXPORTS const data_t& get_data() const; + + /** + * Extract value thread-safe lock counter (see @ref Surface description). + * It's usual situation that counter may be instantly decreased in other thread after this method called. + * We need instantaneous value. This method syncronized in inter-threading way with @ref Surface::release_lock() + * + * @return fetched locks count. + */ + GAPI_EXPORTS size_t get_locks_count() const; + + /** + * Atomically increase value of thread-safe lock counter (see @ref Surface description). + * This method is single-threaded happens-after @ref Surface::get_locks_count() and + * multi-threaded happens-before @ref Surface::release_lock() + * + * @return locks count just before its increasing. + */ + GAPI_EXPORTS size_t obtain_lock(); + + /** + * Atomically decrease value of thread-safe lock counter (see @ref Surface description). + * This method is synchronized with @ref Surface::get_locks_count() and + * multi-threaded happens-after @ref Surface::obtain_lock() + * + * @return locks count just before its decreasing. + */ + GAPI_EXPORTS size_t release_lock(); +private: + Surface(std::unique_ptr&& surf, std::shared_ptr accociated_memory); +}; + +using surface_ptr_t = std::shared_ptr; +using surface_weak_ptr_t = std::weak_ptr; +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_HPP diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp new file mode 100644 index 000000000000..be905029baf8 --- /dev/null +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp @@ -0,0 +1,185 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + + +#include "../test_precomp.hpp" + +#include "../common/gapi_tests_common.hpp" + +#include + +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include + +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" + +namespace opencv_test +{ +namespace +{ +TEST(OneVPL_Source_Surface, InitSurface) +{ + using namespace cv::gapi::wip; + + // create raw MFX handle + std::unique_ptr handle(new mfxFrameSurface1{}); + mfxFrameSurface1 *mfx_core_handle = handle.get(); + + // create preallocate surface memory: empty for test + std::shared_ptr associated_memory {}; + auto surf = Surface::create_surface(std::move(handle), associated_memory); + + // check self consistency + EXPECT_EQ(reinterpret_cast(surf->get_handle()), + reinterpret_cast(mfx_core_handle)); + EXPECT_EQ(surf->get_locks_count(), 0); + EXPECT_EQ(surf->obtain_lock(), 0); + EXPECT_EQ(surf->get_locks_count(), 1); + EXPECT_EQ(surf->release_lock(), 1); + EXPECT_EQ(surf->get_locks_count(), 0); +} + +TEST(OneVPL_Source_Surface, ConcurrentLock) +{ + using namespace cv::gapi::wip; + + // create raw MFX handle + std::unique_ptr handle(new mfxFrameSurface1{}); + + // create preallocate surface memory: empty for test + std::shared_ptr associated_memory {}; + auto surf = Surface::create_surface(std::move(handle), associated_memory); + + // check self consistency + EXPECT_EQ(surf->get_locks_count(), 0); + + // MFX internal limitation: do not exceede U16 range + // so I16 is using here + int16_t lock_counter = std::numeric_limits::max() - 1; + std::promise barrier; + std::future sync = barrier.get_future(); + + + std::thread worker_thread([&barrier, surf, lock_counter] () { + barrier.set_value(); + + // concurrent lock + for (int16_t i = 0; i < lock_counter; i ++) { + surf->obtain_lock(); + } + }); + sync.wait(); + + // concurrent lock + for (int16_t i = 0; i < lock_counter; i ++) { + surf->obtain_lock(); + } + + worker_thread.join(); + EXPECT_EQ(surf->get_locks_count(), lock_counter * 2); +} + +TEST(OneVPL_Source_Surface, MemoryLifeTime) +{ + using namespace cv::gapi::wip; + + // create preallocate surface memory + std::unique_ptr preallocated_memory_ptr(new char); + std::shared_ptr associated_memory (preallocated_memory_ptr.get(), + [&preallocated_memory_ptr] (void* ptr) { + EXPECT_TRUE(preallocated_memory_ptr); + EXPECT_EQ(preallocated_memory_ptr.get(), ptr); + preallocated_memory_ptr.reset(); + }); + + // generate surfaces + constexpr size_t surface_num = 10000; + std::vector> surfaces(surface_num); + std::generate(surfaces.begin(), surfaces.end(), [surface_num, associated_memory](){ + std::unique_ptr handle(new mfxFrameSurface1{}); + return Surface::create_surface(std::move(handle), associated_memory); + }); + + // destroy surfaces + { + std::thread deleter_thread([&surfaces]() { + surfaces.clear(); + }); + deleter_thread.join(); + } + + // workspace memory must be alive + EXPECT_EQ(surfaces.size(), 0); + EXPECT_TRUE(associated_memory != nullptr); + EXPECT_TRUE(preallocated_memory_ptr.get() != nullptr); + + // generate surfaces again + 1 + constexpr size_t surface_num_plus_one = 10001; + surfaces.resize(surface_num_plus_one); + std::generate(surfaces.begin(), surfaces.end(), [surface_num_plus_one, associated_memory](){ + std::unique_ptr handle(new mfxFrameSurface1{}); + return Surface::create_surface(std::move(handle), associated_memory); + }); + + // remember one surface + std::shared_ptr last_surface = surfaces.back(); + + // destroy another surfaces + surfaces.clear(); + + // destroy associated_memory + associated_memory.reset(); + + // workspace memory must be still alive + EXPECT_EQ(surfaces.size(), 0); + EXPECT_TRUE(associated_memory == nullptr); + EXPECT_TRUE(preallocated_memory_ptr.get() != nullptr); + + // destroy last surface + last_surface.reset(); + + // workspace memory must be freed + EXPECT_TRUE(preallocated_memory_ptr.get() == nullptr); +} + +TEST(OneVPL_Source_CPUFrameAdapter, InitFrameAdapter) +{ + using namespace cv::gapi::wip; + + // create raw MFX handle + std::unique_ptr handle(new mfxFrameSurface1{}); + + // create preallocate surface memory: empty for test + std::shared_ptr associated_memory {}; + auto surf = Surface::create_surface(std::move(handle), associated_memory); + + // check consistency + EXPECT_EQ(surf->get_locks_count(), 0); + + { + VPLMediaFrameCPUAdapter adapter(surf); + EXPECT_EQ(surf->get_locks_count(), 1); + } + EXPECT_EQ(surf->get_locks_count(), 0); +} +} +} // namespace opencv_test +#endif // HAVE_ONEVPL From b3cc8289956b70e36383d614bb1f98b5eca5be69 Mon Sep 17 00:00:00 2001 From: Giles Payne Date: Mon, 20 Sep 2021 22:44:13 +0900 Subject: [PATCH 211/376] Fix put/get functions for non-contiguous Mat on iOS/macOS --- modules/core/misc/objc/common/Mat.mm | 95 ++++++++++++++--------- modules/core/misc/objc/test/MatTest.swift | 22 +++++- 2 files changed, 78 insertions(+), 39 deletions(-) diff --git a/modules/core/misc/objc/common/Mat.mm b/modules/core/misc/objc/common/Mat.mm index 045bd8393ea3..dc2316c87bda 100644 --- a/modules/core/misc/objc/common/Mat.mm +++ b/modules/core/misc/objc/common/Mat.mm @@ -13,25 +13,28 @@ #import "CvType.h" #import "CVObjcUtil.h" -// return true if we have reached the final index -static bool incIdx(cv::Mat* mat, std::vector& indices) { - for (int dim = mat->dims-1; dim>=0; dim--) { - indices[dim] = (indices[dim] + 1) % mat->size[dim]; - if (indices[dim] != 0) { - return false; - } +static int idx2Offset(cv::Mat* mat, std::vector& indices) { + int offset = indices[0]; + for (int dim=1; dim < mat->dims; dim++) { + offset = offset*mat->size[dim] + indices[dim]; } - return true; + return offset; } -// returns true if final index was reached -static bool updateIdx(cv::Mat* mat, std::vector& indices, int inc) { - for (int index = 0; index < inc; index++) { - if (incIdx(mat, indices)) { - return true; - } +static void offset2Idx(cv::Mat* mat, size_t offset, std::vector& indices) { + for (int dim=mat->dims-1; dim>=0; dim--) { + indices[dim] = offset % mat->size[dim]; + offset = (offset - indices[dim]) / mat->size[dim]; } - return false; +} + +// returns true if final index was reached +static bool updateIdx(cv::Mat* mat, std::vector& indices, size_t inc) { + size_t currentOffset = idx2Offset(mat, indices); + size_t newOffset = currentOffset + inc; + bool reachedEnd = newOffset>=(size_t)mat->total(); + offset2Idx(mat, reachedEnd?0:newOffset, indices); + return reachedEnd; } @implementation Mat { @@ -724,22 +727,31 @@ int getMatAvailable(cv::Mat* mat, std::vector& indices) { } int arrayAvailable = count; + size_t countBytes = count * sizeof(T); + size_t remainingBytes = (size_t)(mat->total() - idx2Offset(mat, tempIndices))*mat->elemSize(); + countBytes = (countBytes>remainingBytes)?remainingBytes:countBytes; + int result = (int)countBytes; int matAvailable = getMatAvailable(mat, tempIndices); int available = MIN(arrayAvailable, matAvailable); - int result = (int)(available * mat->elemSize() / mat->channels()); if (mat->isContinuous()) { memcpy(tBuffer, mat->ptr(tempIndices.data()), available * sizeof(T)); } else { - int copyOffset = 0; - int copyCount = MIN((mat->size[mat->dims - 1] - tempIndices[mat->dims - 1]) * mat->channels(), available); - while (available > 0) { - memcpy(tBuffer + copyOffset, mat->ptr(tempIndices.data()), copyCount * sizeof(T)); - if (updateIdx(mat, tempIndices, copyCount / mat->channels())) { - break; - } - available -= copyCount; - copyOffset += copyCount * sizeof(T); - copyCount = MIN(mat->size[mat->dims-1] * mat->channels(), available); + char* buff = (char*)tBuffer; + size_t blockSize = mat->size[mat->dims-1] * mat->elemSize(); + size_t firstPartialBlockSize = (mat->size[mat->dims-1] - tempIndices[mat->dims-1]) * mat->step[mat->dims-1]; + for (int dim=mat->dims-2; dim>=0 && blockSize == mat->step[dim]; dim--) { + blockSize *= mat->size[dim]; + firstPartialBlockSize += (mat->size[dim] - (tempIndices[dim]+1)) * mat->step[dim]; + } + size_t copyCount = (countBytesptr(tempIndices.data()); + while(countBytes>0) { + memcpy(buff, data, copyCount); + updateIdx(mat, tempIndices, copyCount / mat->elemSize()); + countBytes -= copyCount; + buff += copyCount; + copyCount = countBytesptr(tempIndices.data()); } } return result; @@ -817,22 +829,31 @@ - (int)get:(NSArray*)indices count:(int)count shortBuffer:(short*)buf } int arrayAvailable = count; + size_t countBytes = count * sizeof(T); + size_t remainingBytes = (size_t)(mat->total() - idx2Offset(mat, tempIndices))*mat->elemSize(); + countBytes = (countBytes>remainingBytes)?remainingBytes:countBytes; + int result = (int)countBytes; int matAvailable = getMatAvailable(mat, tempIndices); int available = MIN(arrayAvailable, matAvailable); - int result = (int)(available * mat->elemSize() / mat->channels()); if (mat->isContinuous()) { memcpy(mat->ptr(tempIndices.data()), tBuffer, available * sizeof(T)); } else { - int copyOffset = 0; - int copyCount = MIN((mat->size[mat->dims - 1] - tempIndices[mat->dims - 1]) * mat->channels(), available); - while (available > 0) { - memcpy(mat->ptr(tempIndices.data()), tBuffer + copyOffset, copyCount * sizeof(T)); - if (updateIdx(mat, tempIndices, copyCount / mat->channels())) { - break; - } - available -= copyCount; - copyOffset += copyCount * sizeof(T); - copyCount = MIN(mat->size[mat->dims-1] * (int)mat->channels(), available); + char* buff = (char*)tBuffer; + size_t blockSize = mat->size[mat->dims-1] * mat->elemSize(); + size_t firstPartialBlockSize = (mat->size[mat->dims-1] - tempIndices[mat->dims-1]) * mat->step[mat->dims-1]; + for (int dim=mat->dims-2; dim>=0 && blockSize == mat->step[dim]; dim--) { + blockSize *= mat->size[dim]; + firstPartialBlockSize += (mat->size[dim] - (tempIndices[dim]+1)) * mat->step[dim]; + } + size_t copyCount = (countBytesptr(tempIndices.data()); + while(countBytes>0){ + memcpy(data, buff, copyCount); + updateIdx(mat, tempIndices, copyCount / mat->elemSize()); + countBytes -= copyCount; + buff += copyCount; + copyCount = countBytesptr(tempIndices.data()); } } return result; diff --git a/modules/core/misc/objc/test/MatTest.swift b/modules/core/misc/objc/test/MatTest.swift index 8a513505cc14..2dcfedf41fb0 100644 --- a/modules/core/misc/objc/test/MatTest.swift +++ b/modules/core/misc/objc/test/MatTest.swift @@ -431,7 +431,7 @@ class MatTests: OpenCVTestCase { // whole Mat var bytesNum = try m.get(row: 1, col: 1, data: &buff) - XCTAssertEqual(12, bytesNum); + XCTAssertEqual(12, bytesNum) XCTAssert(buff == [110, 111, 120, 121, 130, 131]) // sub-Mat @@ -442,8 +442,26 @@ class MatTests: OpenCVTestCase { XCTAssert(buff00 == [230, 231, 240, 241]) var buff11 = [Int16](repeating: 0, count: 4) bytesNum = try sm.get(row: 1, col: 1, data: &buff11) - XCTAssertEqual(4, bytesNum); + XCTAssertEqual(4, bytesNum) XCTAssert(buff11 == [340, 341, 0, 0]) + + let m2 = Mat(sizes: [5, 6, 8], type: CvType.CV_16S) + let data:[Int16] = (0.. Date: Sun, 15 Aug 2021 20:33:06 +0000 Subject: [PATCH 212/376] python: cv.Mat wrapper over numpy.ndarray --- .../py_bindings_basics.markdown | 8 ++ .../python/package/mat_wrapper/__init__.py | 33 +++++ modules/python/src2/cv2.cpp | 88 +++++++++--- modules/python/test/test_mat.py | 131 ++++++++++++++++++ modules/python/test/tests_common.py | 1 + 5 files changed, 244 insertions(+), 17 deletions(-) create mode 100644 modules/core/misc/python/package/mat_wrapper/__init__.py create mode 100644 modules/python/test/test_mat.py diff --git a/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown b/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown index 2c5eccd0d6f7..001952deca39 100644 --- a/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown +++ b/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown @@ -60,6 +60,14 @@ of C++. So this is the basic version of how OpenCV-Python bindings are generated. +@note There is no 1:1 mapping of numpy.ndarray on cv::Mat. For example, cv::Mat has channels field, +which is emulated as last dimension of numpy.ndarray and implicitly converted. +However, such implicit conversion has problem with passing of 3D numpy arrays into C++ code +(the last dimension is implicitly reinterpreted as number of channels). +Refer to the [issue](https://github.com/opencv/opencv/issues/19091) for workarounds if you need to process 3D arrays or ND-arrays with channels. +OpenCV 4.5.4+ has `cv.Mat` wrapper derived from `numpy.ndarray` to explicitly handle the channels behavior. + + How to extend new modules to Python? ------------------------------------ diff --git a/modules/core/misc/python/package/mat_wrapper/__init__.py b/modules/core/misc/python/package/mat_wrapper/__init__.py new file mode 100644 index 000000000000..7309c32b01c8 --- /dev/null +++ b/modules/core/misc/python/package/mat_wrapper/__init__.py @@ -0,0 +1,33 @@ +__all__ = [] + +import sys +import numpy as np +import cv2 as cv + +# NumPy documentation: https://numpy.org/doc/stable/user/basics.subclassing.html + +class Mat(np.ndarray): + ''' + cv.Mat wrapper for numpy array. + + Stores extra metadata information how to interpret and process of numpy array for underlying C++ code. + ''' + + def __new__(cls, arr, **kwargs): + obj = arr.view(Mat) + return obj + + def __init__(self, arr, **kwargs): + self.wrap_channels = kwargs.pop('wrap_channels', getattr(arr, 'wrap_channels', False)) + if len(kwargs) > 0: + raise TypeError('Unknown parameters: {}'.format(repr(kwargs))) + + def __array_finalize__(self, obj): + if obj is None: + return + self.wrap_channels = getattr(obj, 'wrap_channels', None) + + +Mat.__module__ = cv.__name__ +cv.Mat = Mat +cv._registerMatType(Mat) diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index e97f17470c97..6231fde67ff4 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -49,6 +49,8 @@ static PyObject* opencv_error = NULL; +static PyTypeObject* pyopencv_Mat_TypePtr = nullptr; + class ArgInfo { public: @@ -638,10 +640,20 @@ static bool isBool(PyObject* obj) CV_NOEXCEPT return PyArray_IsScalar(obj, Bool) || PyBool_Check(obj); } +template +static std::string pycv_dumpArray(const T* arr, int n) +{ + std::ostringstream out; + out << "["; + for (int i = 0; i < n; ++i) + out << " " << arr[i]; + out << " ]"; + return out.str(); +} + // special case, when the converter needs full ArgInfo structure static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) { - bool allowND = true; if(!o || o == Py_None) { if( !m.data ) @@ -727,12 +739,29 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) return false; } - int size[CV_MAX_DIM+1]; - size_t step[CV_MAX_DIM+1]; size_t elemsize = CV_ELEM_SIZE1(type); const npy_intp* _sizes = PyArray_DIMS(oarr); const npy_intp* _strides = PyArray_STRIDES(oarr); + + CV_LOG_DEBUG(NULL, "Incoming ndarray '" << info.name << "': ndims=" << ndims << " _sizes=" << pycv_dumpArray(_sizes, ndims) << " _strides=" << pycv_dumpArray(_strides, ndims)); + bool ismultichannel = ndims == 3 && _sizes[2] <= CV_CN_MAX; + if (pyopencv_Mat_TypePtr && PyObject_TypeCheck(o, pyopencv_Mat_TypePtr)) + { + bool wrapChannels = false; + PyObject* pyobj_wrap_channels = PyObject_GetAttrString(o, "wrap_channels"); + if (pyobj_wrap_channels) + { + if (!pyopencv_to_safe(pyobj_wrap_channels, wrapChannels, ArgInfo("cv.Mat.wrap_channels", 0))) + { + // TODO extra message + Py_DECREF(pyobj_wrap_channels); + return false; + } + Py_DECREF(pyobj_wrap_channels); + } + ismultichannel = wrapChannels && ndims >= 1; + } for( int i = ndims-1; i >= 0 && !needcopy; i-- ) { @@ -746,14 +775,26 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) needcopy = true; } - if( ismultichannel && _strides[1] != (npy_intp)elemsize*_sizes[2] ) - needcopy = true; + if (ismultichannel) + { + int channels = ndims >= 1 ? (int)_sizes[ndims - 1] : 1; + if (channels > CV_CN_MAX) + { + failmsg("%s unable to wrap channels, too high (%d > CV_CN_MAX=%d)", info.name, (int)channels, (int)CV_CN_MAX); + return false; + } + ndims--; + type |= CV_MAKETYPE(0, channels); + + if (ndims >= 1 && _strides[ndims - 1] != (npy_intp)elemsize*_sizes[ndims]) + needcopy = true; + } if (needcopy) { if (info.outputarg) { - failmsg("Layout of the output array %s is incompatible with cv::Mat (step[ndims-1] != elemsize or step[1] != elemsize*nchannels)", info.name); + failmsg("Layout of the output array %s is incompatible with cv::Mat", info.name); return false; } @@ -769,6 +810,9 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) _strides = PyArray_STRIDES(oarr); } + int size[CV_MAX_DIM+1] = {}; + size_t step[CV_MAX_DIM+1] = {}; + // Normalize strides in case NPY_RELAXED_STRIDES is set size_t default_step = elemsize; for ( int i = ndims - 1; i >= 0; --i ) @@ -787,23 +831,16 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) } // handle degenerate case + // FIXIT: Don't force 1D for Scalars if( ndims == 0) { size[ndims] = 1; step[ndims] = elemsize; ndims++; } - if( ismultichannel ) - { - ndims--; - type |= CV_MAKETYPE(0, size[2]); - } - - if( ndims > 2 && !allowND ) - { - failmsg("%s has more than 2 dimensions", info.name); - return false; - } +#if 1 + CV_LOG_DEBUG(NULL, "Construct Mat: ndims=" << ndims << " size=" << pycv_dumpArray(size, ndims) << " step=" << pycv_dumpArray(step, ndims) << " type=" << cv::typeToString(type)); +#endif m = Mat(ndims, size, type, PyArray_DATA(oarr), step); m.u = g_numpyAllocator.allocate(o, ndims, size, type, step); @@ -2183,7 +2220,24 @@ static int convert_to_char(PyObject *o, char *dst, const ArgInfo& info) #include "pyopencv_generated_types_content.h" #include "pyopencv_generated_funcs.h" +static PyObject* pycvRegisterMatType(PyObject *self, PyObject *value) +{ + CV_LOG_DEBUG(NULL, cv::format("pycvRegisterMatType %p %p\n", self, value)); + + if (0 == PyType_Check(value)) + { + PyErr_SetString(PyExc_TypeError, "Type argument is expected"); + return NULL; + } + + Py_INCREF(value); + pyopencv_Mat_TypePtr = (PyTypeObject*)value; + + Py_RETURN_NONE; +} + static PyMethodDef special_methods[] = { + {"_registerMatType", (PyCFunction)(pycvRegisterMatType), METH_O, "_registerMatType(cv.Mat) -> None (Internal)"}, {"redirectError", CV_PY_FN_WITH_KW(pycvRedirectError), "redirectError(onError) -> None"}, #ifdef HAVE_OPENCV_HIGHGUI {"createTrackbar", (PyCFunction)pycvCreateTrackbar, METH_VARARGS, "createTrackbar(trackbarName, windowName, value, count, onChange) -> None"}, diff --git a/modules/python/test/test_mat.py b/modules/python/test/test_mat.py new file mode 100644 index 000000000000..72614fda36f6 --- /dev/null +++ b/modules/python/test/test_mat.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +from __future__ import print_function + +import numpy as np +import cv2 as cv + +import os +import sys +import unittest + +from tests_common import NewOpenCVTests + +try: + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') + + + class MatTest(NewOpenCVTests): + + def test_mat_construct(self): + data = np.random.random([10, 10, 3]) + + #print(np.ndarray.__dictoffset__) # 0 + #print(cv.Mat.__dictoffset__) # 88 (> 0) + #print(cv.Mat) # + #print(cv.Mat.__base__) # + + mat_data0 = cv.Mat(data) + assert isinstance(mat_data0, cv.Mat) + assert isinstance(mat_data0, np.ndarray) + self.assertEqual(mat_data0.wrap_channels, False) + res0 = cv.utils.dumpInputArray(mat_data0) + self.assertEqual(res0, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=300 dims(-1)=3 size(-1)=[10 10 3] type(-1)=CV_64FC1") + + mat_data1 = cv.Mat(data, wrap_channels=True) + assert isinstance(mat_data1, cv.Mat) + assert isinstance(mat_data1, np.ndarray) + self.assertEqual(mat_data1.wrap_channels, True) + res1 = cv.utils.dumpInputArray(mat_data1) + self.assertEqual(res1, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=100 dims(-1)=2 size(-1)=10x10 type(-1)=CV_64FC3") + + mat_data2 = cv.Mat(mat_data1) + assert isinstance(mat_data2, cv.Mat) + assert isinstance(mat_data2, np.ndarray) + self.assertEqual(mat_data2.wrap_channels, True) # fail if __array_finalize__ doesn't work + res2 = cv.utils.dumpInputArray(mat_data2) + self.assertEqual(res2, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=100 dims(-1)=2 size(-1)=10x10 type(-1)=CV_64FC3") + + + def test_mat_construct_4d(self): + data = np.random.random([5, 10, 10, 3]) + + mat_data0 = cv.Mat(data) + assert isinstance(mat_data0, cv.Mat) + assert isinstance(mat_data0, np.ndarray) + self.assertEqual(mat_data0.wrap_channels, False) + res0 = cv.utils.dumpInputArray(mat_data0) + self.assertEqual(res0, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=1500 dims(-1)=4 size(-1)=[5 10 10 3] type(-1)=CV_64FC1") + + mat_data1 = cv.Mat(data, wrap_channels=True) + assert isinstance(mat_data1, cv.Mat) + assert isinstance(mat_data1, np.ndarray) + self.assertEqual(mat_data1.wrap_channels, True) + res1 = cv.utils.dumpInputArray(mat_data1) + self.assertEqual(res1, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=500 dims(-1)=3 size(-1)=[5 10 10] type(-1)=CV_64FC3") + + mat_data2 = cv.Mat(mat_data1) + assert isinstance(mat_data2, cv.Mat) + assert isinstance(mat_data2, np.ndarray) + self.assertEqual(mat_data2.wrap_channels, True) # __array_finalize__ doesn't work + res2 = cv.utils.dumpInputArray(mat_data2) + self.assertEqual(res2, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=500 dims(-1)=3 size(-1)=[5 10 10] type(-1)=CV_64FC3") + + + def test_mat_wrap_channels_fail(self): + data = np.random.random([2, 3, 4, 520]) + + mat_data0 = cv.Mat(data) + assert isinstance(mat_data0, cv.Mat) + assert isinstance(mat_data0, np.ndarray) + self.assertEqual(mat_data0.wrap_channels, False) + res0 = cv.utils.dumpInputArray(mat_data0) + self.assertEqual(res0, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=12480 dims(-1)=4 size(-1)=[2 3 4 520] type(-1)=CV_64FC1") + + with self.assertRaises(cv.error): + mat_data1 = cv.Mat(data, wrap_channels=True) # argument unable to wrap channels, too high (520 > CV_CN_MAX=512) + res1 = cv.utils.dumpInputArray(mat_data1) + print(mat_data1.__dict__) + print(res1) + + + def test_ufuncs(self): + data = np.arange(10) + mat_data = cv.Mat(data) + mat_data2 = 2 * mat_data + self.assertEqual(type(mat_data2), cv.Mat) + np.testing.assert_equal(2 * data, 2 * mat_data) + + + def test_comparison(self): + # Undefined behavior, do NOT use that. + # Behavior may be changed in the future + + data = np.ones((10, 10, 3)) + mat_wrapped = cv.Mat(data, wrap_channels=True) + mat_simple = cv.Mat(data) + np.testing.assert_equal(mat_wrapped, mat_simple) # ???: wrap_channels is not checked for now + np.testing.assert_equal(data, mat_simple) + np.testing.assert_equal(data, mat_wrapped) + + #self.assertEqual(mat_wrapped, mat_simple) # ??? + #self.assertTrue(mat_wrapped == mat_simple) # ??? + #self.assertTrue((mat_wrapped == mat_simple).all()) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass + + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/modules/python/test/tests_common.py b/modules/python/test/tests_common.py index a938a8e2cbb9..2245722939a2 100644 --- a/modules/python/test/tests_common.py +++ b/modules/python/test/tests_common.py @@ -10,6 +10,7 @@ import argparse import numpy as np +#sys.OpenCV_LOADER_DEBUG = True import cv2 as cv # Python 3 moved urlopen to urllib.requests From f7b4b750d8930b5bb6696cea6d609dc70a0597db Mon Sep 17 00:00:00 2001 From: mikael Date: Tue, 21 Sep 2021 19:46:33 +0200 Subject: [PATCH 213/376] Detect FP16 on FreeBSD aarch64 --- modules/core/src/system.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index d8b8f6755950..df9e8a0ce76f 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -533,7 +533,7 @@ struct HWFeatures } #endif // CV_CPUID_X86 - #if defined __ANDROID__ || defined __linux__ + #if defined __ANDROID__ || defined __linux__ || defined __FreeBSD__ #ifdef __aarch64__ have[CV_CPU_NEON] = true; have[CV_CPU_FP16] = true; @@ -559,7 +559,7 @@ struct HWFeatures CV_LOG_INFO(NULL, "- FP16 instructions is NOT enabled via build flags"); #endif #endif - #elif defined __arm__ + #elif defined __arm__ && !defined __FreeBSD__ int cpufile = open("/proc/self/auxv", O_RDONLY); if (cpufile >= 0) From 38b9ec7a18e95140a1f34d7b1c52f4fe74c14459 Mon Sep 17 00:00:00 2001 From: rogday Date: Wed, 22 Sep 2021 15:17:37 +0300 Subject: [PATCH 214/376] Merge pull request #20682 from rogday:min * Add Min layer to CPU, OpenCL, Halide, Inference Engine, NGraph and CUDA * fix indentation * add min to fusion and halide tests; fix doc --- .../dnn/include/opencv2/dnn/all_layers.hpp | 2 +- modules/dnn/src/cuda/eltwise_ops.cu | 7 ++++ modules/dnn/src/cuda/functors.hpp | 15 ++++++++ .../dnn/src/cuda4dnn/kernels/eltwise_ops.hpp | 3 ++ .../dnn/src/cuda4dnn/primitives/eltwise.hpp | 5 ++- modules/dnn/src/layers/eltwise_layer.cpp | 34 ++++++++++++++++++- modules/dnn/src/onnx/onnx_importer.cpp | 10 +++--- modules/dnn/test/test_halide_layers.cpp | 2 +- modules/dnn/test/test_layers.cpp | 2 +- modules/dnn/test/test_onnx_importer.cpp | 5 +++ 10 files changed, 76 insertions(+), 9 deletions(-) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index fbe16850d4d5..cfe6595d78c8 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -610,7 +610,7 @@ CV__DNN_INLINE_NS_BEGIN /** @brief Element wise operation on inputs Extra optional parameters: - - "operation" as string. Values are "sum" (default), "prod", "max", "div" + - "operation" as string. Values are "sum" (default), "prod", "max", "div", "min" - "coeff" as float array. Specify weights of inputs for SUM operation - "output_channels_mode" as string. Values are "same" (default, all input must have the same layout), "input_0", "input_0_truncate", "max_input_channels" */ diff --git a/modules/dnn/src/cuda/eltwise_ops.cu b/modules/dnn/src/cuda/eltwise_ops.cu index b24801531f67..109c3fbdc9a2 100644 --- a/modules/dnn/src/cuda/eltwise_ops.cu +++ b/modules/dnn/src/cuda/eltwise_ops.cu @@ -74,6 +74,11 @@ void eltwise_max_2(const Stream& stream, Span output, View x, View y) { eltwise_op>(stream, output, x, y); } +template +void eltwise_min_2(const Stream& stream, Span output, View x, View y) { + eltwise_op>(stream, output, x, y); +} + template void eltwise_sum_2(const Stream& stream, Span output, View x, View y) { eltwise_op>(stream, output, x, y); @@ -100,11 +105,13 @@ void eltwise_div_2(const Stream& stream, Span output, View x, View y) { template void eltwise_sum_coeff_2(const Stream&, Span<__half>, __half, View<__half>, __half, View<__half>); template void eltwise_sum_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); template void eltwise_max_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); + template void eltwise_min_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); #endif template void eltwise_div_2(const Stream& stream, Span output, View x, View y); template void eltwise_prod_2(const Stream& stream, Span output, View x, View y); template void eltwise_sum_coeff_2(const Stream&, Span, float, View, float, View); template void eltwise_sum_2(const Stream& stream, Span output, View x, View y); template void eltwise_max_2(const Stream& stream, Span output, View x, View y); + template void eltwise_min_2(const Stream& stream, Span output, View x, View y); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda/functors.hpp b/modules/dnn/src/cuda/functors.hpp index 1c29de04264b..f01a07c77ee4 100644 --- a/modules/dnn/src/cuda/functors.hpp +++ b/modules/dnn/src/cuda/functors.hpp @@ -262,6 +262,21 @@ struct MaxFunctor { } }; +template +struct MinFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE MinFunctor() { } + CUDA4DNN_DEVICE MinFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T x, T y) { + using csl::device::min; + return min(x, y); + } +}; + template struct SumFunctor { struct Params { diff --git a/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp b/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp index 092b1571af59..096ba04e487c 100644 --- a/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp +++ b/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp @@ -15,6 +15,9 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { template void eltwise_max_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + template + void eltwise_min_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + template void eltwise_sum_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); diff --git a/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp b/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp index adcd7ab3f87e..b46f0d870f4d 100644 --- a/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp @@ -25,7 +25,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { MAX, SUM, PRODUCT, - DIV + DIV, + MIN, }; class EltwiseOpBase : public CUDABackendNode { @@ -78,6 +79,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { switch (op) { case EltwiseOpType::MAX: kernels::eltwise_max_2(stream, output, input_x, input_y); break; + case EltwiseOpType::MIN: kernels::eltwise_min_2(stream, output, input_x, input_y); break; case EltwiseOpType::PRODUCT: kernels::eltwise_prod_2(stream, output, input_x, input_y); break; case EltwiseOpType::DIV: kernels::eltwise_div_2(stream, output, input_x, input_y); break; case EltwiseOpType::SUM: @@ -104,6 +106,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { switch (op) { case EltwiseOpType::MAX: kernels::eltwise_max_2(stream, output, output, input); break; + case EltwiseOpType::MIN: kernels::eltwise_min_2(stream, output, output, input); break; case EltwiseOpType::PRODUCT: kernels::eltwise_prod_2(stream, output, output, input); break; case EltwiseOpType::DIV: kernels::eltwise_div_2(stream, output, output, input); break; case EltwiseOpType::SUM: diff --git a/modules/dnn/src/layers/eltwise_layer.cpp b/modules/dnn/src/layers/eltwise_layer.cpp index 860560213d92..2c473ff4129e 100644 --- a/modules/dnn/src/layers/eltwise_layer.cpp +++ b/modules/dnn/src/layers/eltwise_layer.cpp @@ -71,7 +71,8 @@ class EltwiseLayerImpl CV_FINAL : public EltwiseLayer PROD = 0, SUM = 1, MAX = 2, - DIV = 3 + DIV = 3, + MIN = 4, } op; std::vector coeffs; @@ -109,6 +110,8 @@ class EltwiseLayerImpl CV_FINAL : public EltwiseLayer op = SUM; else if (operation == "max") op = MAX; + else if (operation == "min") + op = MIN; else if (operation == "div") op = DIV; else @@ -470,6 +473,13 @@ class EltwiseLayerImpl CV_FINAL : public EltwiseLayer dstptr[j] = std::max(srcptr0[j], srcptrI[j]); } } + else if (op == MIN) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = std::min(srcptr0[j], srcptrI[j]); + } + } else if (op == SUM) { if (!coeffsptr || (coeffsptr[0] == 1.0f && coeffsptr[1] == 1.0f)) @@ -524,6 +534,13 @@ class EltwiseLayerImpl CV_FINAL : public EltwiseLayer dstptr[j] = std::max(dstptr[j], srcptrI[j]); } } + else if (op == MIN) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = std::min(dstptr[j], srcptrI[j]); + } + } else if (op == SUM) { if (!coeffsptr || coeffsptr[inputIdx] == 1.0f) @@ -641,6 +658,11 @@ class EltwiseLayerImpl CV_FINAL : public EltwiseLayer for (int i = 2; i < inputs.size(); ++i) max(inputs[i], outputs[0], outputs[0]); break; + case MIN: + min(inputs[0], inputs[1], outputs[0]); + for (int i = 2; i < inputs.size(); ++i) + min(inputs[i], outputs[0], outputs[0]); + break; default: return false; } @@ -745,6 +767,7 @@ class EltwiseLayerImpl CV_FINAL : public EltwiseLayer auto op_ = [this] { switch (op) { case MAX: return cuda4dnn::EltwiseOpType::MAX; + case MIN: return cuda4dnn::EltwiseOpType::MIN; case SUM: return cuda4dnn::EltwiseOpType::SUM; case PROD: return cuda4dnn::EltwiseOpType::PRODUCT; case DIV: return cuda4dnn::EltwiseOpType::DIV; @@ -799,6 +822,12 @@ class EltwiseLayerImpl CV_FINAL : public EltwiseLayer for (int i = 2; i < inputBuffers.size(); ++i) topExpr = max(topExpr, inputBuffers[i](x, y, c, n)); break; + case MIN: + topExpr = min(inputBuffers[0](x, y, c, n), + inputBuffers[1](x, y, c, n)); + for (int i = 2; i < inputBuffers.size(); ++i) + topExpr = min(topExpr, inputBuffers[i](x, y, c, n)); + break; default: return Ptr(); } @@ -823,6 +852,8 @@ class EltwiseLayerImpl CV_FINAL : public EltwiseLayer ieLayer.setEltwiseType(InferenceEngine::Builder::EltwiseLayer::EltwiseType::DIV); else if (op == MAX) ieLayer.setEltwiseType(InferenceEngine::Builder::EltwiseLayer::EltwiseType::MAX); + else if (op == MIN) + ieLayer.setEltwiseType(InferenceEngine::Builder::EltwiseLayer::EltwiseType::MIN); else CV_Error(Error::StsNotImplemented, "Unsupported eltwise operation"); @@ -857,6 +888,7 @@ class EltwiseLayerImpl CV_FINAL : public EltwiseLayer case PROD: curr_node = std::make_shared(curr_node, next_node); break; case DIV: curr_node = std::make_shared(curr_node, next_node); break; case MAX: curr_node = std::make_shared(curr_node, next_node); break; + case MIN: curr_node = std::make_shared(curr_node, next_node); break; default: CV_Error(Error::StsNotImplemented, "Unsupported eltwise operation"); } } diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 82ab8062253f..9cc81768c3f4 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -105,7 +105,7 @@ class ONNXImporter void parseSplit (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseBias (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parsePow (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - void parseMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMinMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseNeg (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseConstant (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseLSTM (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -1105,10 +1105,12 @@ void ONNXImporter::parsePow(LayerParams& layerParams, const opencv_onnx::NodePro addLayer(layerParams, node_proto); } -void ONNXImporter::parseMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +// "Min" "Max" +void ONNXImporter::parseMinMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { + const std::string& layer_type = node_proto.op_type(); layerParams.type = "Eltwise"; - layerParams.set("operation", "max"); + layerParams.set("operation", layer_type == "Max" ? "max" : "min"); addLayer(layerParams, node_proto); } @@ -2421,7 +2423,7 @@ const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() dispatch["Split"] = &ONNXImporter::parseSplit; dispatch["Add"] = dispatch["Sum"] = dispatch["Sub"] = &ONNXImporter::parseBias; dispatch["Pow"] = &ONNXImporter::parsePow; - dispatch["Max"] = &ONNXImporter::parseMax; + dispatch["Min"] = dispatch["Max"] = &ONNXImporter::parseMinMax; dispatch["Neg"] = &ONNXImporter::parseNeg; dispatch["Constant"] = &ONNXImporter::parseConstant; dispatch["LSTM"] = &ONNXImporter::parseLSTM; diff --git a/modules/dnn/test/test_halide_layers.cpp b/modules/dnn/test/test_halide_layers.cpp index 165ee4d67b44..a6c1b7d0f8a8 100644 --- a/modules/dnn/test/test_halide_layers.cpp +++ b/modules/dnn/test/test_halide_layers.cpp @@ -893,7 +893,7 @@ TEST_P(Eltwise, Accuracy) INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, Eltwise, Combine( /*input size*/ Values(Vec3i(1, 4, 5), Vec3i(2, 8, 6)), -/*operation*/ Values("prod", "sum", "div", "max"), +/*operation*/ Values("prod", "sum", "div", "max", "min"), /*num convs*/ Values(1, 2, 3), /*weighted(for sum only)*/ Bool(), dnnBackendsAndTargetsWithHalide() diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp index 04d5fa63559e..ee0629b85380 100644 --- a/modules/dnn/test/test_layers.cpp +++ b/modules/dnn/test/test_layers.cpp @@ -2340,7 +2340,7 @@ class TestLayerFusion : public DNNTestLayer { static testing::internal::ParamGenerator eltwiseOpList() { // TODO: automate list generation - return Values("sum", "max", "prod", "div"); + return Values("sum", "max", "min", "prod", "div"); } static testing::internal::ParamGenerator activationLayersList() diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 103582fbdd3c..ca3c29fbafb3 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -301,6 +301,11 @@ TEST_P(Test_ONNX_layers, ReduceMax) testONNXModels("reduce_max_axis_1"); } +TEST_P(Test_ONNX_layers, Min) +{ + testONNXModels("min", npy, 0, 0, false, true, 2); +} + TEST_P(Test_ONNX_layers, Scale) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) From 86a51015b1b5e1d2c555aff6abdf0f50d4ed3f5a Mon Sep 17 00:00:00 2001 From: Amir Tulegenov Date: Wed, 22 Sep 2021 20:42:32 +0600 Subject: [PATCH 215/376] Merge pull request #19554 from amirtu:OCV-215_cvtColorTwoPlane_wrong_output_when_Y_Plane_Mat_has_step different paddings in cvtColorTwoPlane() for biplane YUV420 * Different paddings support in cvtColorTwoPlane() for biplane YUV420 * Build fix for dispatch case. * Resoted old behaviour for y.step==uv.step to exclude perf regressions. Co-authored-by: amir.tulegenov Co-authored-by: Alexander Smorkalov --- 3rdparty/carotene/hal/tegra_hal.hpp | 39 ++++++++------- 3rdparty/openvx/hal/openvx_hal.cpp | 14 ++++-- 3rdparty/openvx/hal/openvx_hal.hpp | 3 ++ .../include/opencv2/imgproc/hal/hal.hpp | 5 ++ modules/imgproc/src/color_yuv.dispatch.cpp | 40 +++++++++++---- modules/imgproc/src/color_yuv.simd.hpp | 49 ++++++++----------- modules/imgproc/src/hal_replacement.hpp | 35 +++++++++++++ modules/imgproc/test/test_color.cpp | 34 +++++++++---- 8 files changed, 149 insertions(+), 70 deletions(-) diff --git a/3rdparty/carotene/hal/tegra_hal.hpp b/3rdparty/carotene/hal/tegra_hal.hpp index 2dfa30b6190b..7fcca975fedf 100644 --- a/3rdparty/carotene/hal/tegra_hal.hpp +++ b/3rdparty/carotene/hal/tegra_hal.hpp @@ -1778,30 +1778,30 @@ TegraCvtColor_Invoker(bgrx2hsvf, bgrx2hsv, src_data + static_cast(range. : CV_HAL_ERROR_NOT_IMPLEMENTED \ ) -#define TEGRA_CVT2PYUVTOBGR(src_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx) \ +#define TEGRA_CVT2PYUVTOBGR_EX(y_data, y_step, uv_data, uv_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx) \ ( \ CAROTENE_NS::isSupportedConfiguration() ? \ dcn == 3 ? \ uIdx == 0 ? \ (swapBlue ? \ CAROTENE_NS::yuv420i2rgb(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step) : \ CAROTENE_NS::yuv420i2bgr(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step)), \ CV_HAL_ERROR_OK : \ uIdx == 1 ? \ (swapBlue ? \ CAROTENE_NS::yuv420sp2rgb(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step) : \ CAROTENE_NS::yuv420sp2bgr(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step)), \ CV_HAL_ERROR_OK : \ CV_HAL_ERROR_NOT_IMPLEMENTED : \ @@ -1809,29 +1809,32 @@ TegraCvtColor_Invoker(bgrx2hsvf, bgrx2hsv, src_data + static_cast(range. uIdx == 0 ? \ (swapBlue ? \ CAROTENE_NS::yuv420i2rgbx(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step) : \ CAROTENE_NS::yuv420i2bgrx(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step)), \ CV_HAL_ERROR_OK : \ uIdx == 1 ? \ (swapBlue ? \ CAROTENE_NS::yuv420sp2rgbx(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step) : \ CAROTENE_NS::yuv420sp2bgrx(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step)), \ CV_HAL_ERROR_OK : \ CV_HAL_ERROR_NOT_IMPLEMENTED : \ CV_HAL_ERROR_NOT_IMPLEMENTED \ : CV_HAL_ERROR_NOT_IMPLEMENTED \ ) +#define TEGRA_CVT2PYUVTOBGR(src_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx) \ + TEGRA_CVT2PYUVTOBGR_EX(src_data, src_step, src_data + src_step * dst_height, src_step, dst_data, dst_step, \ + dst_width, dst_height, dcn, swapBlue, uIdx); #undef cv_hal_cvtBGRtoBGR #define cv_hal_cvtBGRtoBGR TEGRA_CVTBGRTOBGR @@ -1847,6 +1850,8 @@ TegraCvtColor_Invoker(bgrx2hsvf, bgrx2hsv, src_data + static_cast(range. #define cv_hal_cvtBGRtoHSV TEGRA_CVTBGRTOHSV #undef cv_hal_cvtTwoPlaneYUVtoBGR #define cv_hal_cvtTwoPlaneYUVtoBGR TEGRA_CVT2PYUVTOBGR +#undef cv_hal_cvtTwoPlaneYUVtoBGREx +#define cv_hal_cvtTwoPlaneYUVtoBGREx TEGRA_CVT2PYUVTOBGR_EX #endif // OPENCV_IMGPROC_HAL_INTERFACE_H diff --git a/3rdparty/openvx/hal/openvx_hal.cpp b/3rdparty/openvx/hal/openvx_hal.cpp index 53a2711d527c..498748239220 100644 --- a/3rdparty/openvx/hal/openvx_hal.cpp +++ b/3rdparty/openvx/hal/openvx_hal.cpp @@ -923,6 +923,11 @@ int ovx_hal_cvtGraytoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, } int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx) +{ + return ovx_hal_cvtTwoPlaneYUVtoBGREx(a, astep, a + h * astep, astep, b, bstep, w, h, bcn, swapBlue, uIdx); +} + +int ovx_hal_cvtTwoPlaneYUVtoBGREx(const uchar * a, size_t astep, const uchar * b, size_t bstep, uchar * c, size_t cstep, int w, int h, int bcn, bool swapBlue, int uIdx) { if (skipSmallImages(w, h)) return CV_HAL_ERROR_NOT_IMPLEMENTED; @@ -933,8 +938,7 @@ int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t if (w & 1 || h & 1) // It's not described in spec but sample implementation unable to convert odd sized images return CV_HAL_ERROR_NOT_IMPLEMENTED; - refineStep(w, h, uIdx ? VX_DF_IMAGE_NV21 : VX_DF_IMAGE_NV12, astep); - refineStep(w, h, bcn == 3 ? VX_DF_IMAGE_RGB : VX_DF_IMAGE_RGBX, bstep); + try { ivx::Context ctx = getOpenVXHALContext(); @@ -943,8 +947,8 @@ int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t std::vector ptrs; addr.push_back(ivx::Image::createAddressing(w, h, 1, (vx_int32)astep)); ptrs.push_back((void*)a); - addr.push_back(ivx::Image::createAddressing(w / 2, h / 2, 2, (vx_int32)astep)); - ptrs.push_back((void*)(a + h * astep)); + addr.push_back(ivx::Image::createAddressing(w / 2, h / 2, 2, (vx_int32)bstep)); + ptrs.push_back((void*)b); vxImage ia = ivx::Image::createFromHandle(ctx, uIdx ? VX_DF_IMAGE_NV21 : VX_DF_IMAGE_NV12, addr, ptrs); @@ -952,7 +956,7 @@ int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t return CV_HAL_ERROR_NOT_IMPLEMENTED; // OpenCV store NV12/NV21 as RANGE_RESTRICTED while OpenVX expect RANGE_FULL vxImage ib = ivx::Image::createFromHandle(ctx, bcn == 3 ? VX_DF_IMAGE_RGB : VX_DF_IMAGE_RGBX, - ivx::Image::createAddressing(w, h, bcn, (vx_int32)bstep), b); + ivx::Image::createAddressing(w, h, bcn, (vx_int32)cstep), c); ivx::IVX_CHECK_STATUS(vxuColorConvert(ctx, ia, ib)); } catch (ivx::RuntimeError & e) diff --git a/3rdparty/openvx/hal/openvx_hal.hpp b/3rdparty/openvx/hal/openvx_hal.hpp index c94cde3158a9..8b5a22f7dec1 100644 --- a/3rdparty/openvx/hal/openvx_hal.hpp +++ b/3rdparty/openvx/hal/openvx_hal.hpp @@ -49,6 +49,7 @@ int ovx_hal_morph(cvhalFilter2D *filter_context, uchar *a, size_t astep, uchar * int ovx_hal_cvtBGRtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int depth, int acn, int bcn, bool swapBlue); int ovx_hal_cvtGraytoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int depth, int bcn); int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx); +int ovx_hal_cvtTwoPlaneYUVtoBGREx(const uchar * a, size_t astep, const uchar * b, size_t bstep, uchar * c, size_t cstep, int w, int h, int bcn, bool swapBlue, int uIdx); int ovx_hal_cvtThreePlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx); int ovx_hal_cvtBGRtoThreePlaneYUV(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int acn, bool swapBlue, int uIdx); int ovx_hal_cvtOnePlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx, int ycn); @@ -130,6 +131,8 @@ int ovx_hal_integral(int depth, int sdepth, int, const uchar * a, size_t astep, #define cv_hal_cvtGraytoBGR ovx_hal_cvtGraytoBGR #undef cv_hal_cvtTwoPlaneYUVtoBGR #define cv_hal_cvtTwoPlaneYUVtoBGR ovx_hal_cvtTwoPlaneYUVtoBGR +#undef cv_hal_cvtTwoPlaneYUVtoBGREx +#define cv_hal_cvtTwoPlaneYUVtoBGREx ovx_hal_cvtTwoPlaneYUVtoBGREx #undef cv_hal_cvtThreePlaneYUVtoBGR #define cv_hal_cvtThreePlaneYUVtoBGR ovx_hal_cvtThreePlaneYUVtoBGR #undef cv_hal_cvtBGRtoThreePlaneYUV diff --git a/modules/imgproc/include/opencv2/imgproc/hal/hal.hpp b/modules/imgproc/include/opencv2/imgproc/hal/hal.hpp index a435fd6b8558..9e5728d0c60e 100644 --- a/modules/imgproc/include/opencv2/imgproc/hal/hal.hpp +++ b/modules/imgproc/include/opencv2/imgproc/hal/hal.hpp @@ -198,6 +198,11 @@ CV_EXPORTS void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, int dst_width, int dst_height, int dcn, bool swapBlue, int uIdx); +CV_EXPORTS void cvtTwoPlaneYUVtoBGR(const uchar * y_data, size_t y_step, const uchar * uv_data, size_t uv_step, + uchar * dst_data, size_t dst_step, + int dst_width, int dst_height, + int dcn, bool swapBlue, int uIdx); + CV_EXPORTS void cvtThreePlaneYUVtoBGR(const uchar * src_data, size_t src_step, uchar * dst_data, size_t dst_step, int dst_width, int dst_height, diff --git a/modules/imgproc/src/color_yuv.dispatch.cpp b/modules/imgproc/src/color_yuv.dispatch.cpp index 18b2096680c6..cac4fa1b4133 100644 --- a/modules/imgproc/src/color_yuv.dispatch.cpp +++ b/modules/imgproc/src/color_yuv.dispatch.cpp @@ -124,8 +124,9 @@ void cvtTwoPlaneYUVtoBGR(const uchar * src_data, size_t src_step, CALL_HAL(cvtTwoPlaneYUVtoBGR, cv_hal_cvtTwoPlaneYUVtoBGR, src_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx); - CV_CPU_DISPATCH(cvtTwoPlaneYUVtoBGR, (src_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx), - CV_CPU_DISPATCH_MODES_ALL); + cvtTwoPlaneYUVtoBGR( + src_data, src_step, src_data + src_step * dst_height, src_step, dst_data, dst_step, + dst_width, dst_height, dcn, swapBlue, uIdx); } void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, size_t src_step, @@ -135,7 +136,20 @@ void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, size_t src { CV_INSTRUMENT_REGION(); - CV_CPU_DISPATCH(cvtTwoPlaneYUVtoBGR, (y_data, uv_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx), + cvtTwoPlaneYUVtoBGR(y_data, src_step, uv_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx); +} + +void cvtTwoPlaneYUVtoBGR(const uchar * y_data, size_t y_step, const uchar * uv_data, size_t uv_step, + uchar * dst_data, size_t dst_step, + int dst_width, int dst_height, + int dcn, bool swapBlue, int uIdx) +{ + CV_INSTRUMENT_REGION(); + + CALL_HAL(cvtTwoPlaneYUVtoBGREx, cv_hal_cvtTwoPlaneYUVtoBGREx, + y_data, y_step, uv_data, uv_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx); + + CV_CPU_DISPATCH(cvtTwoPlaneYUVtoBGR, (y_data, y_step, uv_data, uv_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx), CV_CPU_DISPATCH_MODES_ALL); } @@ -172,7 +186,8 @@ void cvtBGRtoTwoPlaneYUV(const uchar * src_data, size_t src_step, { CV_INSTRUMENT_REGION(); - // TODO: add hal replacement method + CALL_HAL(cvtBGRtoTwoPlaneYUV, cv_hal_cvtBGRtoTwoPlaneYUV, + src_data, src_step, y_data, dst_step, uv_data, dst_step, width, height, scn, swapBlue, uIdx); CV_CPU_DISPATCH(cvtBGRtoTwoPlaneYUV, (src_data, src_step, y_data, uv_data, dst_step, width, height, scn, swapBlue, uIdx), CV_CPU_DISPATCH_MODES_ALL); @@ -406,14 +421,21 @@ void cvtColorTwoPlaneYUV2BGRpair( InputArray _ysrc, InputArray _uvsrc, OutputArr Mat ysrc = _ysrc.getMat(), uvsrc = _uvsrc.getMat(); - CV_CheckEQ(ysrc.step, uvsrc.step, ""); - _dst.create( ysz, CV_MAKETYPE(depth, dcn)); Mat dst = _dst.getMat(); - hal::cvtTwoPlaneYUVtoBGR(ysrc.data, uvsrc.data, ysrc.step, - dst.data, dst.step, dst.cols, dst.rows, - dcn, swapb, uidx); + if(ysrc.step == uvsrc.step) + { + hal::cvtTwoPlaneYUVtoBGR(ysrc.data, uvsrc.data, ysrc.step, + dst.data, dst.step, dst.cols, dst.rows, + dcn, swapb, uidx); + } + else + { + hal::cvtTwoPlaneYUVtoBGR(ysrc.data, ysrc.step, uvsrc.data, uvsrc.step, + dst.data, dst.step, dst.cols, dst.rows, + dcn, swapb, uidx); + } } } // namespace cv diff --git a/modules/imgproc/src/color_yuv.simd.hpp b/modules/imgproc/src/color_yuv.simd.hpp index 076d1a4bd573..196a03b99558 100644 --- a/modules/imgproc/src/color_yuv.simd.hpp +++ b/modules/imgproc/src/color_yuv.simd.hpp @@ -17,11 +17,7 @@ void cvtYUVtoBGR(const uchar * src_data, size_t src_step, uchar * dst_data, size_t dst_step, int width, int height, int depth, int dcn, bool swapBlue, bool isCbCr); -void cvtTwoPlaneYUVtoBGR(const uchar * src_data, size_t src_step, - uchar * dst_data, size_t dst_step, - int dst_width, int dst_height, - int dcn, bool swapBlue, int uIdx); -void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, size_t src_step, +void cvtTwoPlaneYUVtoBGR(const uchar * y_data, size_t y_step, const uchar * uv_data, size_t uv_step, uchar * dst_data, size_t dst_step, int dst_width, int dst_height, int dcn, bool swapBlue, int uIdx); @@ -1177,24 +1173,28 @@ struct YUV420sp2RGB8Invoker : ParallelLoopBody uchar * dst_data; size_t dst_step; int width; - const uchar* my1, *muv; - size_t stride; + const uchar* my1; + size_t my1_step; + const uchar* muv; + size_t muv_step; - YUV420sp2RGB8Invoker(uchar * _dst_data, size_t _dst_step, int _dst_width, size_t _stride, const uchar* _y1, const uchar* _uv) - : dst_data(_dst_data), dst_step(_dst_step), width(_dst_width), my1(_y1), muv(_uv), stride(_stride) {} + YUV420sp2RGB8Invoker(uchar * _dst_data, size_t _dst_step, int _dst_width, + const uchar* _y1, size_t _y1_step, const uchar* _uv, size_t _uv_step) : + dst_data(_dst_data), dst_step(_dst_step), width(_dst_width), + my1(_y1), my1_step(_y1_step), muv(_uv), muv_step(_uv_step) {} void operator()(const Range& range) const CV_OVERRIDE { const int rangeBegin = range.start * 2; const int rangeEnd = range.end * 2; - const uchar* y1 = my1 + rangeBegin * stride, *uv = muv + rangeBegin * stride / 2; + const uchar* y1 = my1 + rangeBegin * my1_step, *uv = muv + rangeBegin * muv_step / 2; - for (int j = rangeBegin; j < rangeEnd; j += 2, y1 += stride * 2, uv += stride) + for (int j = rangeBegin; j < rangeEnd; j += 2, y1 += my1_step * 2, uv += muv_step) { uchar* row1 = dst_data + dst_step * j; uchar* row2 = dst_data + dst_step * (j + 1); - const uchar* y2 = y1 + stride; + const uchar* y2 = y1 + my1_step; int i = 0; #if CV_SIMD @@ -1395,9 +1395,10 @@ struct YUV420p2RGB8Invoker : ParallelLoopBody #define MIN_SIZE_FOR_PARALLEL_YUV420_CONVERSION (320*240) template -inline void cvtYUV420sp2RGB(uchar * dst_data, size_t dst_step, int dst_width, int dst_height, size_t _stride, const uchar* _y1, const uchar* _uv) +inline void cvtYUV420sp2RGB(uchar * dst_data, size_t dst_step, int dst_width, int dst_height, + const uchar* _y1, size_t _y1_step, const uchar* _uv, size_t _uv_step) { - YUV420sp2RGB8Invoker converter(dst_data, dst_step, dst_width, _stride, _y1, _uv); + YUV420sp2RGB8Invoker converter(dst_data, dst_step, dst_width, _y1, _y1_step, _uv, _uv_step); if (dst_width * dst_height >= MIN_SIZE_FOR_PARALLEL_YUV420_CONVERSION) parallel_for_(Range(0, dst_height/2), converter); else @@ -1817,26 +1818,16 @@ void cvtYUVtoBGR(const uchar * src_data, size_t src_step, CvtColorLoop(src_data, src_step, dst_data, dst_step, width, height, YCrCb2RGB_f(dcn, blueIdx, isCbCr)); } -void cvtTwoPlaneYUVtoBGR(const uchar * src_data, size_t src_step, - uchar * dst_data, size_t dst_step, - int dst_width, int dst_height, - int dcn, bool swapBlue, int uIdx) -{ - CV_INSTRUMENT_REGION(); - - const uchar* uv = src_data + src_step * static_cast(dst_height); - cvtTwoPlaneYUVtoBGR(src_data, uv, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx); -} - typedef void (*cvt_2plane_yuv_ptr_t)(uchar * /* dst_data*/, size_t /* dst_step */, int /* dst_width */, int /* dst_height */, - size_t /* _stride */, const uchar* /* _y1 */, - const uchar* /* _uv */); + size_t /* _y1_step */, + const uchar* /* _uv */, + size_t /* _uv_step */); -void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, size_t src_step, +void cvtTwoPlaneYUVtoBGR(const uchar * y_data, size_t y_step, const uchar * uv_data, size_t uv_step, uchar * dst_data, size_t dst_step, int dst_width, int dst_height, int dcn, bool swapBlue, int uIdx) @@ -1859,7 +1850,7 @@ void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, size_t src default: CV_Error( CV_StsBadFlag, "Unknown/unsupported color conversion code" ); break; }; - cvtPtr(dst_data, dst_step, dst_width, dst_height, src_step, y_data, uv_data); + cvtPtr(dst_data, dst_step, dst_width, dst_height, y_data, y_step, uv_data, uv_step); } typedef void (*cvt_3plane_yuv_ptr_t)(uchar * /* dst_data */, diff --git a/modules/imgproc/src/hal_replacement.hpp b/modules/imgproc/src/hal_replacement.hpp index d6b1d782736f..3368093c56f1 100644 --- a/modules/imgproc/src/hal_replacement.hpp +++ b/modules/imgproc/src/hal_replacement.hpp @@ -498,6 +498,39 @@ inline int hal_ni_cvtLabtoBGR(const uchar * src_data, size_t src_step, uchar * d */ inline int hal_ni_cvtTwoPlaneYUVtoBGR(const uchar * src_data, size_t src_step, uchar * dst_data, size_t dst_step, int dst_width, int dst_height, int dcn, bool swapBlue, int uIdx) { return CV_HAL_ERROR_NOT_IMPLEMENTED; } +/** + @brief Extended version of hal_cvtTwoPlaneYUVtoBGR. + @param y_data,y_step source image data and step (Y-plane) + @param uv_data,uv_step source image data and step (UV-plane) + @param dst_data,dst_step destination image data and step + @param dst_width,dst_height destination image size + @param dcn destination image channels (3 or 4) + @param swapBlue if set to true B and R destination channels will be swapped (write RGB) + @param uIdx U-channel index in the interleaved U/V plane (0 or 1) + Convert from YUV (YUV420sp (or NV12/NV21) - Y plane followed by interleaved U/V plane) to BGR, RGB, BGRA or RGBA. + Only for CV_8U. + */ +inline int hal_ni_cvtTwoPlaneYUVtoBGREx(const uchar * y_data, size_t y_step, const uchar * uv_data, size_t uv_step, + uchar * dst_data, size_t dst_step, int dst_width, int dst_height, + int dcn, bool swapBlue, int uIdx) { return CV_HAL_ERROR_NOT_IMPLEMENTED; } + +/** + @brief hal_cvtBGRtoTwoPlaneYUV + @param src_data,src_step source image data and step + @param y_data,y_step destination image data and step (Y-plane) + @param uv_data,uv_step destination image data and step (UV-plane) + @param width,height image size + @param scn source image channels (3 or 4) + @param swapBlue if set to true B and R source channels will be swapped (treat as RGB) + @param uIdx U-channel plane index (0 or 1) + Convert from BGR, RGB, BGRA or RGBA to YUV (YUV420sp (or NV12/NV21) - Y plane followed by interleaved U/V plane). + Only for CV_8U. + */ +inline int hal_ni_cvtBGRtoTwoPlaneYUV(const uchar * src_data, size_t src_step, + uchar * y_data, size_t y_step, uchar * uv_data, size_t uv_step, + int width, int height, + int scn, bool swapBlue, int uIdx) { return CV_HAL_ERROR_NOT_IMPLEMENTED; } + /** @brief hal_cvtThreePlaneYUVtoBGR @param src_data,src_step source image data and step @@ -576,6 +609,8 @@ inline int hal_ni_cvtMultipliedRGBAtoRGBA(const uchar * src_data, size_t src_ste #define cv_hal_cvtBGRtoLab hal_ni_cvtBGRtoLab #define cv_hal_cvtLabtoBGR hal_ni_cvtLabtoBGR #define cv_hal_cvtTwoPlaneYUVtoBGR hal_ni_cvtTwoPlaneYUVtoBGR +#define cv_hal_cvtTwoPlaneYUVtoBGREx hal_ni_cvtTwoPlaneYUVtoBGREx +#define cv_hal_cvtBGRtoTwoPlaneYUV hal_ni_cvtBGRtoTwoPlaneYUV #define cv_hal_cvtThreePlaneYUVtoBGR hal_ni_cvtThreePlaneYUVtoBGR #define cv_hal_cvtBGRtoThreePlaneYUV hal_ni_cvtBGRtoThreePlaneYUV #define cv_hal_cvtOnePlaneYUVtoBGR hal_ni_cvtOnePlaneYUVtoBGR diff --git a/modules/imgproc/test/test_color.cpp b/modules/imgproc/test/test_color.cpp index 450cf5792386..204203d053da 100644 --- a/modules/imgproc/test/test_color.cpp +++ b/modules/imgproc/test/test_color.cpp @@ -3072,20 +3072,34 @@ TEST(ImgProc_RGB2YUV, regression_13668) EXPECT_EQ(res, ref); } -TEST(ImgProc_cvtColorTwoPlane, missing_check_17036) // test can be removed if required feature is implemented +TEST(ImgProc_cvtColorTwoPlane, y_plane_padding_differs_from_uv_plane_padding_17036) { - std::vector y_data(700 * 480); - std::vector uv_data(640 * 240); + RNG &rng = theRNG(); - Mat y_plane_padding(480, 640, CV_8UC1, y_data.data(), 700); // with stride - Mat uv_plane(240, 320, CV_8UC2, uv_data.data()); + std::vector y_reference(640 * 480); + std::vector uv_reference(640 * 240); + std::vector y_padded(700 * 480); + std::vector uv_padded(700 * 240); - Mat result; + Mat y_reference_mat(480, 640, CV_8UC1, y_reference.data()); + Mat uv_reference_mat(240, 320, CV_8UC2, uv_reference.data()); + Mat y_padded_mat(480, 640, CV_8UC1, y_padded.data(), 700); + Mat uv_padded_mat(240, 320, CV_8UC2, uv_padded.data(), 700); + + rng.fill(y_reference_mat, RNG::UNIFORM, 16, 235 + 1); + rng.fill(uv_reference_mat, RNG::UNIFORM, 16, 240 + 1); + + y_reference_mat.copyTo(y_padded_mat(Rect(0, 0, y_reference_mat.cols, y_reference_mat.rows))); + uv_reference_mat.copyTo(uv_padded_mat(Rect(0, 0, uv_reference_mat.cols, uv_reference_mat.rows))); + + Mat rgb_reference_mat, rgb_y_padded_mat, rgb_uv_padded_mat; + + cvtColorTwoPlane(y_reference_mat, uv_reference_mat, rgb_reference_mat, COLOR_YUV2RGB_NV21); + cvtColorTwoPlane(y_padded_mat, uv_reference_mat, rgb_y_padded_mat, COLOR_YUV2RGB_NV21); + cvtColorTwoPlane(y_reference_mat, uv_padded_mat, rgb_uv_padded_mat, COLOR_YUV2RGB_NV21); - EXPECT_THROW( - cvtColorTwoPlane(y_plane_padding, uv_plane, result, COLOR_YUV2RGB_NV21); - , cv::Exception - ); + EXPECT_DOUBLE_EQ(cvtest::norm(rgb_reference_mat, rgb_y_padded_mat, NORM_INF), .0); + EXPECT_DOUBLE_EQ(cvtest::norm(rgb_reference_mat, rgb_uv_padded_mat, NORM_INF), .0); } From 54386c82fd5ca3079f60d1758bb1d25651a329c9 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Thu, 23 Sep 2021 14:34:30 +0300 Subject: [PATCH 216/376] Merge pull request #20727 from sivanov-work:merge_vpl_accel_impl G-API: oneVPL (simplification) added CPU, DX11(fallback CPU) accels & surface pool * Add CPU, DX11(fallback CPU) accels & surface pool * Fix build for surface_pool * Apply some comments * Fix indentation --- modules/gapi/CMakeLists.txt | 6 + .../onevpl/accelerators/accel_policy_cpu.cpp | 225 ++++++++++++++++++ .../onevpl/accelerators/accel_policy_cpu.hpp | 55 +++++ .../onevpl/accelerators/accel_policy_dx11.cpp | 114 +++++++++ .../onevpl/accelerators/accel_policy_dx11.hpp | 67 ++++++ .../accelerators/surface/surface_pool.cpp | 69 ++++++ .../accelerators/surface/surface_pool.hpp | 45 ++++ .../gapi_streaming_vpl_core_test.cpp | 161 ++++++++++++- 8 files changed, 741 insertions(+), 1 deletion(-) create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp create mode 100644 modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 56d051cf0fde..380f1d7d8456 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -171,6 +171,9 @@ set(gapi_srcs src/streaming/onevpl/data_provider_interface_exception.cpp src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp src/streaming/onevpl/accelerators/surface/surface.cpp + src/streaming/onevpl/accelerators/surface/surface_pool.cpp + src/streaming/onevpl/accelerators/accel_policy_cpu.cpp + src/streaming/onevpl/accelerators/accel_policy_dx11.cpp # Utils (ITT tracing) src/utils/itt.cpp @@ -247,6 +250,9 @@ if(HAVE_GAPI_ONEVPL) if(TARGET opencv_test_gapi) ocv_target_compile_definitions(opencv_test_gapi PRIVATE -DHAVE_ONEVPL) ocv_target_link_libraries(opencv_test_gapi PRIVATE ${VPL_IMPORTED_TARGETS}) + if(HAVE_D3D11 AND HAVE_OPENCL) + ocv_target_include_directories(opencv_test_gapi SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS}) + endif() endif() ocv_target_compile_definitions(${the_module} PRIVATE -DHAVE_ONEVPL) ocv_target_link_libraries(${the_module} PRIVATE ${VPL_IMPORTED_TARGETS}) diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp new file mode 100644 index 000000000000..0a5f68ae4e6a --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp @@ -0,0 +1,225 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL +#include +#include + +#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" +#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef _WIN32 + #include + #include +#endif +namespace cv { +namespace gapi { +namespace wip { + +VPLCPUAccelerationPolicy::VPLCPUAccelerationPolicy() { + GAPI_LOG_INFO(nullptr, "created"); +} + +VPLCPUAccelerationPolicy::~VPLCPUAccelerationPolicy() { + for (auto& pair : pool_table) { + pair.second.clear(); + // do not free key here: last surface will release it + } + GAPI_LOG_INFO(nullptr, "destroyed"); +} + +void VPLCPUAccelerationPolicy::init(session_t session) { + GAPI_LOG_INFO(nullptr, "initialize session: " << session); +} + +void VPLCPUAccelerationPolicy::deinit(session_t session) { + GAPI_LOG_INFO(nullptr, "deinitialize session: " << session); +} + +VPLCPUAccelerationPolicy::pool_key_t +VPLCPUAccelerationPolicy::create_surface_pool(size_t pool_size, size_t surface_size_bytes, + surface_ptr_ctr_t creator) { + GAPI_LOG_DEBUG(nullptr, "pool size: " << pool_size << ", surface size bytes: " << surface_size_bytes); + + // create empty pool + pool_t pool; + pool.reserve(pool_size); + + // allocate workplace memory area + size_t preallocated_raw_bytes = pool_size * surface_size_bytes; + size_t page_size_bytes = 4 * 1024; + void *preallocated_pool_memory_ptr = nullptr; + +#ifdef _WIN32 + SYSTEM_INFO sysInfo; + GetSystemInfo(&sysInfo); + page_size_bytes = sysInfo.dwPageSize; + + GAPI_LOG_DEBUG(nullptr, "page size: " << page_size_bytes << ", preallocated_raw_bytes: " << preallocated_raw_bytes); + preallocated_pool_memory_ptr = _aligned_malloc(preallocated_raw_bytes, page_size_bytes); +#else + GAPI_Assert(false && "Compatibility is not tested for systems differ than \"_WIN32\". " + "Please feel free to set it up under OPENCV contribution policy"); +#endif + + if (!preallocated_pool_memory_ptr) { + throw std::runtime_error("VPLCPUAccelerationPolicy::create_surface_pool - failed: not enough memory." + "Requested surface count: " + std::to_string(pool_size) + + ", surface bytes: " + std::to_string(surface_size_bytes)); + } + + // fill pool with surfaces + std::shared_ptr workspace_mem_owner (preallocated_pool_memory_ptr, [] (void *ptr){ + GAPI_LOG_INFO(nullptr, "Free workspace memory: " << ptr); +#ifdef _WIN32 + _aligned_free(ptr); + GAPI_LOG_INFO(nullptr, "Released workspace memory: " << ptr); + ptr = nullptr; +#else + GAPI_Assert(false && "Not implemented for systems differ than \"_WIN32\". " + "Please feel free to set it up under OPENCV contribution policy"); +#endif + + }); + size_t i = 0; + try { + for (; i < pool_size; i++) { + size_t preallocated_mem_offset = static_cast(i) * surface_size_bytes; + + surface_ptr_t surf_ptr = creator(workspace_mem_owner, + preallocated_mem_offset, + preallocated_raw_bytes); + pool.push_back(std::move(surf_ptr)); + } + } catch (const std::exception& ex) { + throw std::runtime_error(std::string("VPLCPUAccelerationPolicy::create_surface_pool - ") + + "cannot construct surface index: " + std::to_string(i) + ", error: " + + ex.what() + + "Requested surface count: " + std::to_string(pool_size) + + ", surface bytes: " + std::to_string(surface_size_bytes)); + } + + // remember pool by key + GAPI_LOG_INFO(nullptr, "New pool allocated, key: " << preallocated_pool_memory_ptr << + ", surface count: " << pool.size() << + ", surface size bytes: " << surface_size_bytes); + if (!pool_table.emplace(preallocated_pool_memory_ptr, std::move(pool)).second) { + GAPI_LOG_WARNING(nullptr, "Cannot insert pool, table size: " + std::to_string(pool_table.size()) << + ", key: " << preallocated_pool_memory_ptr << " exists"); + GAPI_Assert(false && "Cannot create pool in VPLCPUAccelerationPolicy"); + } + + return preallocated_pool_memory_ptr; +} + +VPLCPUAccelerationPolicy::surface_weak_ptr_t VPLCPUAccelerationPolicy::get_free_surface(pool_key_t key) { + auto pool_it = pool_table.find(key); + if (pool_it == pool_table.end()) { + throw std::runtime_error("VPLCPUAccelerationPolicy::get_free_surface - " + "key is not found, table size: " + + std::to_string(pool_table.size())); + } + + pool_t& requested_pool = pool_it->second; +#ifdef TEST_PERF + return requested_pool.find_free(); +#else // TEST_PERF + auto it = + std::find_if(requested_pool.begin(), requested_pool.end(), + [](const surface_ptr_t& val) { + GAPI_DbgAssert(val && "Pool contains empty surface"); + return !val->get_locks_count(); + }); + + // Limitation realloc pool might be a future extension + if (it == requested_pool.end()) { + std::stringstream ss; + ss << "cannot get free surface from pool, key: " << key << ", size: " << requested_pool.size(); + const std::string& str = ss.str(); + GAPI_LOG_WARNING(nullptr, str); + throw std::runtime_error(std::string(__FUNCTION__) + " - " + str); + } + + return *it; +#endif // TEST_PERF +} + +size_t VPLCPUAccelerationPolicy::get_free_surface_count(pool_key_t key) const { + auto pool_it = pool_table.find(key); + if (pool_it == pool_table.end()) { + GAPI_LOG_WARNING(nullptr, "key is not found: " << key << + ", table size: " << pool_table.size()); + return 0; + } +#ifdef TEST_PERF + return 0; +#else // TEST_PERF + const pool_t& requested_pool = pool_it->second; + size_t free_surf_count = + std::count_if(requested_pool.begin(), requested_pool.end(), + [](const surface_ptr_t& val) { + GAPI_Assert(val && "Pool contains empty surface"); + return !val->get_locks_count(); + }); + return free_surf_count; +#endif // TEST_PERF +} + +size_t VPLCPUAccelerationPolicy::get_surface_count(pool_key_t key) const { + auto pool_it = pool_table.find(key); + if (pool_it == pool_table.end()) { + GAPI_LOG_DEBUG(nullptr, "key is not found: " << key << + ", table size: " << pool_table.size()); + return 0; + } +#ifdef TEST_PERF + return 0; +#else // TEST_PERF + return pool_it->second.size(); +#endif // TEST_PERF +} + +cv::MediaFrame::AdapterPtr VPLCPUAccelerationPolicy::create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) { + auto pool_it = pool_table.find(key); + if (pool_it == pool_table.end()) { + std::stringstream ss; + ss << "key is not found: " << key << ", table size: " << pool_table.size(); + const std::string& str = ss.str(); + GAPI_LOG_WARNING(nullptr, str); + throw std::runtime_error(std::string(__FUNCTION__) + " - " + str); + } + + pool_t& requested_pool = pool_it->second; +#ifdef TEST_PERF + return cv::MediaFrame::AdapterPtr{new VPLMediaFrameCPUAdapter(requested_pool.find_by_handle(surface))}; +#else // TEST_PERF + auto it = + std::find_if(requested_pool.begin(), requested_pool.end(), + [surface](const surface_ptr_t& val) { + GAPI_DbgAssert(val && "Pool contains empty surface"); + return val->get_handle() == surface; + }); + + // Limitation realloc pool might be a future extension + if (it == requested_pool.end()) { + std::stringstream ss; + ss << "cannot get requested surface from pool, key: " << key << ", surf: " + << surface << ", pool size: " << requested_pool.size(); + const std::string& str = ss.str(); + GAPI_LOG_WARNING(nullptr, str); + throw std::runtime_error(std::string(__FUNCTION__) + " - " + str); + } + + return cv::MediaFrame::AdapterPtr{new VPLMediaFrameCPUAdapter(*it)}; +#endif // TEST_PERF +} +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp new file mode 100644 index 000000000000..cfe30573159c --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp @@ -0,0 +1,55 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_CPU_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_CPU_HPP + +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL +#include +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#ifdef TEST_PERF +#include "streaming/onevpl/accelerators/surface/surface_pool.hpp" +#endif // TEST_PERF + +namespace cv { +namespace gapi { +namespace wip { + +// GAPI_EXPORTS for tests +struct GAPI_EXPORTS VPLCPUAccelerationPolicy final : public VPLAccelerationPolicy +{ + VPLCPUAccelerationPolicy(); + ~VPLCPUAccelerationPolicy(); +#ifdef TEST_PERF + using pool_t = CachedPool; +#else // TEST_PERF + using pool_t = std::vector; +#endif // TEST_PERF + + void init(session_t session) override; + void deinit(session_t session) override; + pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator) override; + surface_weak_ptr_t get_free_surface(pool_key_t key) override; + size_t get_free_surface_count(pool_key_t key) const override; + size_t get_surface_count(pool_key_t key) const override; + + cv::MediaFrame::AdapterPtr create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) override; + +private: + std::map pool_table; +}; +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_CPU_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp new file mode 100644 index 000000000000..348b864a15a3 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp @@ -0,0 +1,114 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/accelerators/accel_policy_dx11.hpp" +#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#pragma comment(lib,"d3d11.lib") + +#define D3D11_NO_HELPERS +#include +#include +#include +#include "opencv2/core/directx.hpp" + +#ifdef HAVE_OPENCL +#include +#endif + +namespace cv { +namespace gapi { +namespace wip { +VPLDX11AccelerationPolicy::VPLDX11AccelerationPolicy() +{ +#ifdef CPU_ACCEL_ADAPTER + adapter.reset(new VPLCPUAccelerationPolicy); +#endif +} + +VPLDX11AccelerationPolicy::~VPLDX11AccelerationPolicy() +{ + if (hw_handle) + { + GAPI_LOG_INFO(nullptr, "VPLDX11AccelerationPolicy release ID3D11Device"); + hw_handle->Release(); + } +} + +void VPLDX11AccelerationPolicy::init(session_t session) { + mfxStatus sts = MFXVideoCORE_GetHandle(session, MFX_HANDLE_D3D11_DEVICE, reinterpret_cast(&hw_handle)); + if (sts != MFX_ERR_NONE) + { + throw std::logic_error("Cannot create VPLDX11AccelerationPolicy, MFXVideoCORE_GetHandle error"); + } + + GAPI_LOG_INFO(nullptr, "VPLDX11AccelerationPolicy initialized, session: " << session); +} + +void VPLDX11AccelerationPolicy::deinit(session_t session) { + GAPI_LOG_INFO(nullptr, "deinitialize session: " << session); +} + +VPLDX11AccelerationPolicy::pool_key_t +VPLDX11AccelerationPolicy::create_surface_pool(size_t pool_size, size_t surface_size_bytes, + surface_ptr_ctr_t creator) { + GAPI_LOG_DEBUG(nullptr, "pool size: " << pool_size << ", surface size bytes: " << surface_size_bytes); + +#ifdef CPU_ACCEL_ADAPTER + return adapter->create_surface_pool(pool_size, surface_size_bytes, creator); +#endif + (void)pool_size; + (void)surface_size_bytes; + (void)creator; + throw std::runtime_error("VPLDX11AccelerationPolicy::create_surface_pool() is not implemented"); +} + +VPLDX11AccelerationPolicy::surface_weak_ptr_t VPLDX11AccelerationPolicy::get_free_surface(pool_key_t key) +{ +#ifdef CPU_ACCEL_ADAPTER + return adapter->get_free_surface(key); +#endif + (void)key; + throw std::runtime_error("VPLDX11AccelerationPolicy::get_free_surface() is not implemented"); +} + +size_t VPLDX11AccelerationPolicy::get_free_surface_count(pool_key_t key) const { +#ifdef CPU_ACCEL_ADAPTER + return adapter->get_free_surface_count(key); +#endif + (void)key; + throw std::runtime_error("get_free_surface_count() is not implemented"); +} + +size_t VPLDX11AccelerationPolicy::get_surface_count(pool_key_t key) const { +#ifdef CPU_ACCEL_ADAPTER + return adapter->get_surface_count(key); +#endif + (void)key; + throw std::runtime_error("VPLDX11AccelerationPolicy::get_surface_count() is not implemented"); +} + +cv::MediaFrame::AdapterPtr VPLDX11AccelerationPolicy::create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) { + +#ifdef CPU_ACCEL_ADAPTER + return adapter->create_frame_adapter(key, surface); +#endif + (void)key; + (void)surface; + throw std::runtime_error("VPLDX11AccelerationPolicy::create_frame_adapter() is not implemented"); +} +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp new file mode 100644 index 000000000000..04970432c5a5 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp @@ -0,0 +1,67 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_DX11_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_DX11_HPP + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS +//TODO Remove the next MACRO right after DX11 implementation +#define CPU_ACCEL_ADAPTER + +#ifdef HAVE_ONEVPL +#include +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" + +#ifdef CPU_ACCEL_ADAPTER +#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" +#endif + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#define D3D11_NO_HELPERS +#include +#include +#include "opencv2/core/directx.hpp" +#ifdef HAVE_OPENCL +#include +#endif + +namespace cv { +namespace gapi { +namespace wip { + +// GAPI_EXPORTS for tests +struct GAPI_EXPORTS VPLDX11AccelerationPolicy final: public VPLAccelerationPolicy +{ + // GAPI_EXPORTS for tests + VPLDX11AccelerationPolicy(); + ~VPLDX11AccelerationPolicy(); + + void init(session_t session) override; + void deinit(session_t session) override; + pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator) override; + surface_weak_ptr_t get_free_surface(pool_key_t key) override; + size_t get_free_surface_count(pool_key_t key) const override; + size_t get_surface_count(pool_key_t key) const override; + + cv::MediaFrame::AdapterPtr create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) override; + +private: + ID3D11Device *hw_handle; + +#ifdef CPU_ACCEL_ADAPTER + std::unique_ptr adapter; +#endif +}; +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_DX11_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp new file mode 100644 index 000000000000..8ead7965b471 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp @@ -0,0 +1,69 @@ +#include "streaming/onevpl/accelerators/surface/surface_pool.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { + +void CachedPool::reserve(size_t size) { + surfaces.reserve(size); +} + +size_t CachedPool::size() const { + return surfaces.size(); +} + +void CachedPool::clear() { + surfaces.clear(); + next_free_it = surfaces.begin(); + cache.clear(); +} + +void CachedPool::push_back(surface_ptr_t &&surf) { + cache.insert(std::make_pair(surf->get_handle(), surf)); + surfaces.push_back(std::move(surf)); + next_free_it = surfaces.begin(); +} + +CachedPool::surface_ptr_t CachedPool::find_free() { + auto it = + std::find_if(next_free_it, surfaces.end(), + [](const surface_ptr_t& val) { + GAPI_DbgAssert(val && "Pool contains empty surface"); + return !val->get_locks_count(); + }); + + // Limitation realloc pool might be a future extension + if (it == surfaces.end()) { + it = std::find_if(surfaces.begin(), next_free_it, + [](const surface_ptr_t& val) { + GAPI_DbgAssert(val && "Pool contains empty surface"); + return !val->get_locks_count(); + }); + if (it == next_free_it) { + std::stringstream ss; + ss << "cannot get free surface from pool, size: " << surfaces.size(); + const std::string& str = ss.str(); + GAPI_LOG_WARNING(nullptr, str); + throw std::runtime_error(std::string(__FUNCTION__) + " - " + str); + } + } + + next_free_it = it; + ++next_free_it; + + return *it; +} + +CachedPool::surface_ptr_t CachedPool::find_by_handle(mfxFrameSurface1* handle) { + auto it = cache.find(handle); + GAPI_Assert(it != cache.end() && "Cannot find cached surface from pool. Data corruption is possible"); + return it->second; +} +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp new file mode 100644 index 000000000000..2059e9b27e8e --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp @@ -0,0 +1,45 @@ +#ifndef GAPI_STREAMING_ONEVPL_SURFACE_SURFACE_POOL_HPP +#define GAPI_STREAMING_ONEVPL_SURFACE_SURFACE_POOL_HPP + +#include +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif + +#include + +namespace cv { +namespace gapi { +namespace wip { + +class Surface; +// GAPI_EXPORTS for tests +class GAPI_EXPORTS CachedPool { +public: + using surface_ptr_t = std::shared_ptr; + using surface_container_t = std::vector; + using free_surface_iterator_t = typename surface_container_t::iterator; + using cached_surface_container_t = std::map; + + void push_back(surface_ptr_t &&surf); + void reserve(size_t size); + size_t size() const; + void clear(); + surface_ptr_t find_free(); + surface_ptr_t find_by_handle(mfxFrameSurface1* handle); +private: + surface_container_t surfaces; + free_surface_iterator_t next_free_it; + cached_surface_container_t cache; +}; +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_SURFACE_SURFACE_POOL_HPP diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp index be905029baf8..5a36a2befb8c 100644 --- a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp @@ -9,6 +9,7 @@ #include "../common/gapi_tests_common.hpp" +#include #include #include @@ -30,11 +31,18 @@ #ifdef HAVE_ONEVPL #include "streaming/onevpl/accelerators/surface/surface.hpp" #include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" +#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" namespace opencv_test { namespace { +cv::gapi::wip::surface_ptr_t create_test_surface(std::shared_ptr out_buf_ptr, + size_t, size_t) { + std::unique_ptr handle(new mfxFrameSurface1{}); + return cv::gapi::wip::Surface::create_surface(std::move(handle), out_buf_ptr); +} + TEST(OneVPL_Source_Surface, InitSurface) { using namespace cv::gapi::wip; @@ -160,7 +168,7 @@ TEST(OneVPL_Source_Surface, MemoryLifeTime) EXPECT_TRUE(preallocated_memory_ptr.get() == nullptr); } -TEST(OneVPL_Source_CPUFrameAdapter, InitFrameAdapter) +TEST(OneVPL_Source_CPU_FrameAdapter, InitFrameAdapter) { using namespace cv::gapi::wip; @@ -180,6 +188,157 @@ TEST(OneVPL_Source_CPUFrameAdapter, InitFrameAdapter) } EXPECT_EQ(surf->get_locks_count(), 0); } + +TEST(OneVPL_Source_CPU_Accelerator, InitDestroy) +{ + using cv::gapi::wip::VPLCPUAccelerationPolicy; + using cv::gapi::wip::VPLAccelerationPolicy; + + auto acceleration_policy = std::make_shared(); + + size_t surface_count = 10; + size_t surface_size_bytes = 1024; + size_t pool_count = 3; + std::vector pool_export_keys; + pool_export_keys.reserve(pool_count); + + // create several pools + for (size_t i = 0; i < pool_count; i++) + { + VPLAccelerationPolicy::pool_key_t key = + acceleration_policy->create_surface_pool(surface_count, + surface_size_bytes, + create_test_surface); + // check consistency + EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); + EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + + pool_export_keys.push_back(key); + } + + EXPECT_NO_THROW(acceleration_policy.reset()); +} + +TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConsume) +{ + using cv::gapi::wip::VPLCPUAccelerationPolicy; + using cv::gapi::wip::VPLAccelerationPolicy; + using cv::gapi::wip::Surface; + + auto acceleration_policy = std::make_shared(); + + size_t surface_count = 10; + size_t surface_size_bytes = 1024; + + VPLAccelerationPolicy::pool_key_t key = + acceleration_policy->create_surface_pool(surface_count, + surface_size_bytes, + create_test_surface); + // check consistency + EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); + EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + + // consume available surfaces + std::vector> surfaces; + surfaces.reserve(surface_count); + for (size_t i = 0; i < surface_count; i++) { + std::shared_ptr surf = acceleration_policy->get_free_surface(key).lock(); + EXPECT_TRUE(surf.get() != nullptr); + EXPECT_EQ(surf->obtain_lock(), 0); + surfaces.push_back(std::move(surf)); + } + + // check consistency (no free surfaces) + EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); + EXPECT_EQ(acceleration_policy->get_free_surface_count(key), 0); + + // fail consume non-free surfaces + for (size_t i = 0; i < surface_count; i++) { + EXPECT_THROW(acceleration_policy->get_free_surface(key), std::runtime_error); + } + + // release surfaces + for (auto& surf : surfaces) { + EXPECT_EQ(surf->release_lock(), 1); + } + surfaces.clear(); + + // check consistency + EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); + EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + + //check availability after release + for (size_t i = 0; i < surface_count; i++) { + std::shared_ptr surf = acceleration_policy->get_free_surface(key).lock(); + EXPECT_TRUE(surf.get() != nullptr); + EXPECT_EQ(surf->obtain_lock(), 0); + } +} + +TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConcurrentConsume) +{ + using cv::gapi::wip::VPLCPUAccelerationPolicy; + using cv::gapi::wip::VPLAccelerationPolicy; + using cv::gapi::wip::Surface; + + auto acceleration_policy = std::make_shared(); + + size_t surface_count = 10; + size_t surface_size_bytes = 1024; + + VPLAccelerationPolicy::pool_key_t key = + acceleration_policy->create_surface_pool(surface_count, + surface_size_bytes, + create_test_surface); + + // check consistency + EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); + EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + + // consume available surfaces + std::vector> surfaces; + surfaces.reserve(surface_count); + for (size_t i = 0; i < surface_count; i++) { + std::shared_ptr surf = acceleration_policy->get_free_surface(key).lock(); + EXPECT_TRUE(surf.get() != nullptr); + EXPECT_EQ(surf->obtain_lock(), 0); + surfaces.push_back(std::move(surf)); + } + + std::promise launch_promise; + std::future sync = launch_promise.get_future(); + std::promise surface_released_promise; + std::future released_result = surface_released_promise.get_future(); + std::thread worker_thread([&launch_promise, &surface_released_promise, &surfaces] () { + launch_promise.set_value(); + + // concurrent release surfaces + size_t surfaces_count = surfaces.size(); + for (auto& surf : surfaces) { + EXPECT_EQ(surf->release_lock(), 1); + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + surfaces.clear(); + + surface_released_promise.set_value(surfaces_count); + }); + sync.wait(); + + // check free surface concurrently + std::future_status status; + size_t free_surface_count = 0; + size_t free_surface_count_prev = 0; + do { + status = released_result.wait_for(std::chrono::seconds(1)); + free_surface_count = acceleration_policy->get_free_surface_count(key); + EXPECT_TRUE(free_surface_count >= free_surface_count_prev); + free_surface_count_prev = free_surface_count; + } while (status != std::future_status::ready); + std::cerr<< "Ready" << std::endl; + free_surface_count = acceleration_policy->get_free_surface_count(key); + worker_thread.join(); + EXPECT_TRUE(free_surface_count >= free_surface_count_prev); +} } } // namespace opencv_test #endif // HAVE_ONEVPL From 499d8adb75f18113c1fd5ec36cbfbe5fac9b993a Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Thu, 23 Sep 2021 22:59:40 +0300 Subject: [PATCH 217/376] Merge pull request #20705 from TolyaTalamanov:at/handle-reshape-in-gexecutor G-API: Handle reshape for generic case in GExecutor * Handle reshape for generic case for GExecutor * Add initResources * Add tests * Refactor reshape method --- modules/gapi/src/executor/gexecutor.cpp | 17 +- .../test/internal/gapi_int_executor_tests.cpp | 217 ++++++++++++++++++ 2 files changed, 230 insertions(+), 4 deletions(-) diff --git a/modules/gapi/src/executor/gexecutor.cpp b/modules/gapi/src/executor/gexecutor.cpp index 2ca675c7df05..ad9d380b4ec4 100644 --- a/modules/gapi/src/executor/gexecutor.cpp +++ b/modules/gapi/src/executor/gexecutor.cpp @@ -7,8 +7,6 @@ #include "precomp.hpp" -#include - #include #include @@ -411,7 +409,8 @@ bool cv::gimpl::GExecutor::canReshape() const { // FIXME: Introduce proper reshaping support on GExecutor level // for all cases! - return (m_ops.size() == 1) && m_ops[0].isl_exec->canReshape(); + return std::all_of(m_ops.begin(), m_ops.end(), + [](const OpDesc& op) { return op.isl_exec->canReshape(); }); } void cv::gimpl::GExecutor::reshape(const GMetaArgs& inMetas, const GCompileArgs& args) @@ -421,7 +420,17 @@ void cv::gimpl::GExecutor::reshape(const GMetaArgs& inMetas, const GCompileArgs& ade::passes::PassContext ctx{g}; passes::initMeta(ctx, inMetas); passes::inferMeta(ctx, true); - m_ops[0].isl_exec->reshape(g, args); + + // NB: Before reshape islands need to re-init resources for every slot. + for (auto slot : m_slots) + { + initResource(slot.slot_nh, slot.data_nh); + } + + for (auto& op : m_ops) + { + op.isl_exec->reshape(g, args); + } } void cv::gimpl::GExecutor::prepareForNewStream() diff --git a/modules/gapi/test/internal/gapi_int_executor_tests.cpp b/modules/gapi/test/internal/gapi_int_executor_tests.cpp index 04afa1fda8e5..90a338a3d04c 100644 --- a/modules/gapi/test/internal/gapi_int_executor_tests.cpp +++ b/modules/gapi/test/internal/gapi_int_executor_tests.cpp @@ -6,10 +6,158 @@ #include "../test_precomp.hpp" +#include "../gapi_mock_kernels.hpp" namespace opencv_test { +namespace +{ + +class GMockExecutable final: public cv::gimpl::GIslandExecutable +{ + virtual inline bool canReshape() const override { + return m_priv->m_can_reshape; + } + + virtual void reshape(ade::Graph&, const GCompileArgs&) override + { + m_priv->m_reshape_counter++; + } + virtual void handleNewStream() override { } + virtual void run(std::vector&&, std::vector&&) { } + virtual bool allocatesOutputs() const override + { + return true; + } + + virtual cv::RMat allocate(const cv::GMatDesc&) const override + { + m_priv->m_allocate_counter++; + return cv::RMat(); + } + + // NB: GMockBackendImpl creates new unique_ptr + // on every compile call. Need to share counters between instances in order + // to validate it in tests. + struct Priv + { + bool m_can_reshape; + int m_reshape_counter; + int m_allocate_counter; + }; + + std::shared_ptr m_priv; + +public: + GMockExecutable(bool can_reshape = true) + : m_priv(new Priv{can_reshape, 0, 0}) + { + }; + + void setReshape(bool can_reshape) { m_priv->m_can_reshape = can_reshape; } + + int getReshapeCounter() const { return m_priv->m_reshape_counter; } + int getAllocateCounter() const { return m_priv->m_allocate_counter; } +}; + +class GMockBackendImpl final: public cv::gapi::GBackend::Priv +{ + virtual void unpackKernel(ade::Graph &, + const ade::NodeHandle &, + const cv::GKernelImpl &) override { } + + virtual EPtr compile(const ade::Graph &, + const cv::GCompileArgs &, + const std::vector &) const override + { + ++m_compile_counter; + return EPtr{new GMockExecutable(m_exec)}; + } + + mutable int m_compile_counter = 0; + GMockExecutable m_exec; + + virtual bool controlsMerge() const override { + return true; + } + + virtual bool allowsMerge(const cv::gimpl::GIslandModel::Graph &, + const ade::NodeHandle &, + const ade::NodeHandle &, + const ade::NodeHandle &) const override { + return false; + } + +public: + GMockBackendImpl(const GMockExecutable& exec) : m_exec(exec) { }; + int getCompileCounter() const { return m_compile_counter; } +}; + +class GMockFunctor : public gapi::cpu::GOCVFunctor +{ +public: + GMockFunctor(cv::gapi::GBackend backend, + const char* id, + const Meta &meta, + const Impl& impl) + : gapi::cpu::GOCVFunctor(id, meta, impl), m_backend(backend) + { + } + + cv::gapi::GBackend backend() const override { return m_backend; } + +private: + cv::gapi::GBackend m_backend; +}; + +template +GMockFunctor mock_kernel(const cv::gapi::GBackend& backend, Callable c) +{ + using P = cv::detail::OCVCallHelper; + return GMockFunctor{ backend + , K::id() + , &K::getOutMeta + , std::bind(&P::callFunctor, std::placeholders::_1, c) + }; +} + +void dummyFooImpl(const cv::Mat&, cv::Mat&) { }; +void dummyBarImpl(const cv::Mat&, const cv::Mat&, cv::Mat&) { }; + +struct GExecutorReshapeTest: public ::testing::Test +{ + GExecutorReshapeTest() + : comp([](){ + cv::GMat in; + cv::GMat out = I::Bar::on(I::Foo::on(in), in); + return cv::GComputation(in, out); + }) + { + backend_impl1 = std::make_shared(island1); + backend1 = cv::gapi::GBackend{backend_impl1}; + backend_impl2 = std::make_shared(island2); + backend2 = cv::gapi::GBackend{backend_impl2}; + auto kernel1 = mock_kernel(backend1, dummyFooImpl); + auto kernel2 = mock_kernel(backend2, dummyBarImpl); + pkg = cv::gapi::kernels(kernel1, kernel2); + in_mat1 = cv::Mat::eye(32, 32, CV_8UC1); + in_mat2 = cv::Mat::eye(64, 64, CV_8UC1); + } + + cv::GComputation comp; + GMockExecutable island1; + std::shared_ptr backend_impl1; + cv::gapi::GBackend backend1; + GMockExecutable island2; + std::shared_ptr backend_impl2; + cv::gapi::GBackend backend2; + cv::gapi::GKernelPackage pkg; + cv::Mat in_mat1, in_mat2, out_mat;; +}; + +} // anonymous namespace + // FIXME: avoid code duplication // The below graph and config is taken from ComplexIslands test suite TEST(GExecutor, SmokeTest) @@ -77,6 +225,75 @@ TEST(GExecutor, SmokeTest) // with breakdown worked) } +TEST_F(GExecutorReshapeTest, ReshapeInsteadOfRecompile) +{ + // NB: Initial state + EXPECT_EQ(0, backend_impl1->getCompileCounter()); + EXPECT_EQ(0, backend_impl2->getCompileCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + EXPECT_EQ(0, island2.getReshapeCounter()); + + // NB: First compilation. + comp.apply(cv::gin(in_mat1), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(1, backend_impl1->getCompileCounter()); + EXPECT_EQ(1, backend_impl2->getCompileCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + EXPECT_EQ(0, island2.getReshapeCounter()); + + // NB: GMockBackendImpl implements "reshape" method, + // so it won't be recompiled if the meta is changed. + comp.apply(cv::gin(in_mat2), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(1, backend_impl1->getCompileCounter()); + EXPECT_EQ(1, backend_impl2->getCompileCounter()); + EXPECT_EQ(1, island1.getReshapeCounter()); + EXPECT_EQ(1, island2.getReshapeCounter()); +} + +TEST_F(GExecutorReshapeTest, OneBackendNotReshapable) +{ + // NB: Make first island not reshapable + island1.setReshape(false); + + // NB: Initial state + EXPECT_EQ(0, backend_impl1->getCompileCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + EXPECT_EQ(0, backend_impl2->getCompileCounter()); + EXPECT_EQ(0, island2.getReshapeCounter()); + + // NB: First compilation. + comp.apply(cv::gin(in_mat1), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(1, backend_impl1->getCompileCounter()); + EXPECT_EQ(1, backend_impl2->getCompileCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + EXPECT_EQ(0, island2.getReshapeCounter()); + + // NB: Since one of islands isn't reshapable + // the entire graph isn't reshapable as well. + comp.apply(cv::gin(in_mat2), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(2, backend_impl1->getCompileCounter()); + EXPECT_EQ(2, backend_impl2->getCompileCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + EXPECT_EQ(0, island2.getReshapeCounter()); +} + +TEST_F(GExecutorReshapeTest, ReshapeCallAllocate) +{ + // NB: Initial state + EXPECT_EQ(0, island1.getAllocateCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + + // NB: First compilation. + comp.apply(cv::gin(in_mat1), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(1, island1.getAllocateCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + + // NB: The entire graph is reshapable, so it won't be recompiled, but reshaped. + // Check that reshape call "allocate" to reallocate buffers. + comp.apply(cv::gin(in_mat2), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(2, island1.getAllocateCounter()); + EXPECT_EQ(1, island1.getReshapeCounter()); +} + // FIXME: Add explicit tests on GMat/GScalar/GArray being connectors // between executed islands From f9bd83c8547054357b0ecc55dadda787e5422dad Mon Sep 17 00:00:00 2001 From: Tsukasa Sugiura Date: Fri, 24 Sep 2021 15:35:42 +0900 Subject: [PATCH 218/376] fix cast in text detection sample --- samples/dnn/text_detection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/dnn/text_detection.py b/samples/dnn/text_detection.py index 7014a8014861..22c4f6b5ec4e 100644 --- a/samples/dnn/text_detection.py +++ b/samples/dnn/text_detection.py @@ -214,8 +214,8 @@ def main(): 0.5, (255, 0, 0)) for j in range(4): - p1 = (vertices[j][0], vertices[j][1]) - p2 = (vertices[(j + 1) % 4][0], vertices[(j + 1) % 4][1]) + p1 = (int(vertices[j][0]), int(vertices[j][1])) + p2 = (int(vertices[(j + 1) % 4][0]), int(vertices[(j + 1) % 4][1])) cv.line(frame, p1, p2, (0, 255, 0), 1) # Put efficiency information From 3673b45437e4c1e0490c0c48ba76439c4f135361 Mon Sep 17 00:00:00 2001 From: Ruslan Garnov Date: Fri, 24 Sep 2021 14:25:57 +0300 Subject: [PATCH 219/376] Added desync RMats and MediaFrames support --- .../gapi/include/opencv2/gapi/gstreaming.hpp | 12 +++- .../include/opencv2/gapi/streaming/desync.hpp | 5 +- modules/gapi/src/api/kernels_streaming.cpp | 7 +- .../gapi/src/executor/gstreamingexecutor.cpp | 5 +- .../test/streaming/gapi_streaming_tests.cpp | 68 ++++++++++++++++++- 5 files changed, 90 insertions(+), 7 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/gstreaming.hpp b/modules/gapi/include/opencv2/gapi/gstreaming.hpp index 0e7d2a847f4e..890eb584fb0c 100644 --- a/modules/gapi/include/opencv2/gapi/gstreaming.hpp +++ b/modules/gapi/include/opencv2/gapi/gstreaming.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2018 Intel Corporation +// Copyright (C) 2018-2021 Intel Corporation #ifndef OPENCV_GAPI_GSTREAMING_COMPILED_HPP @@ -65,6 +65,7 @@ using OptionalOpaqueRef = OptRef; using GOptRunArgP = util::variant< optional*, optional*, + optional*, optional*, cv::detail::OptionalVectorRef, cv::detail::OptionalOpaqueRef @@ -74,6 +75,7 @@ using GOptRunArgsP = std::vector; using GOptRunArg = util::variant< optional, optional, + optional, optional, optional, optional @@ -95,6 +97,14 @@ template<> inline GOptRunArgP wrap_opt_arg(optional &m) { return GOptRunArgP{&m}; } +template<> inline GOptRunArgP wrap_opt_arg(optional &m) { + return GOptRunArgP{&m}; +} + +template<> inline GOptRunArgP wrap_opt_arg(optional &f) { + return GOptRunArgP{&f}; +} + template<> inline GOptRunArgP wrap_opt_arg(optional &s) { return GOptRunArgP{&s}; } diff --git a/modules/gapi/include/opencv2/gapi/streaming/desync.hpp b/modules/gapi/include/opencv2/gapi/streaming/desync.hpp index 86de279fe941..1ed6e24b49a4 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/desync.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/desync.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation #ifndef OPENCV_GAPI_GSTREAMING_DESYNC_HPP @@ -73,9 +73,10 @@ G desync(const G &g) { * which produces an array of cv::util::optional<> objects. * * @note This feature is highly experimental now and is currently - * limited to a single GMat argument only. + * limited to a single GMat/GFrame argument only. */ GAPI_EXPORTS GMat desync(const GMat &g); +GAPI_EXPORTS GFrame desync(const GFrame &f); } // namespace streaming } // namespace gapi diff --git a/modules/gapi/src/api/kernels_streaming.cpp b/modules/gapi/src/api/kernels_streaming.cpp index c88fbede5bff..2c50551f4ed4 100644 --- a/modules/gapi/src/api/kernels_streaming.cpp +++ b/modules/gapi/src/api/kernels_streaming.cpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation #include "precomp.hpp" @@ -75,6 +75,11 @@ cv::GMat cv::gapi::streaming::desync(const cv::GMat &g) { // object will feed both branches of the streaming executable. } +// All notes from the above desync(GMat) are also applicable here +cv::GFrame cv::gapi::streaming::desync(const cv::GFrame &f) { + return cv::gapi::copy(detail::desync(f)); +} + cv::GMat cv::gapi::streaming::BGR(const cv::GFrame& in) { return cv::gapi::streaming::GBGR::on(in); } diff --git a/modules/gapi/src/executor/gstreamingexecutor.cpp b/modules/gapi/src/executor/gstreamingexecutor.cpp index 27049aef6327..2379e3e16499 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.cpp +++ b/modules/gapi/src/executor/gstreamingexecutor.cpp @@ -186,8 +186,9 @@ void sync_data(cv::gimpl::stream::Result &r, cv::GOptRunArgsP &outputs) // FIXME: this conversion should be unified switch (out_obj.index()) { - HANDLE_CASE(cv::Scalar); break; - HANDLE_CASE(cv::RMat); break; + HANDLE_CASE(cv::Scalar); break; + HANDLE_CASE(cv::RMat); break; + HANDLE_CASE(cv::MediaFrame); break; case T::index_of*>(): { // Mat: special handling. diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp index 369a0c069f4f..4f8b2563acc4 100644 --- a/modules/gapi/test/streaming/gapi_streaming_tests.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_tests.cpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2019-2020 Intel Corporation +// Copyright (C) 2019-2021 Intel Corporation #include "../test_precomp.hpp" @@ -2284,4 +2284,70 @@ TEST(OneVPL_Source, Init) EXPECT_TRUE(stream_data_provider->empty()); } #endif + +TEST(GAPI_Streaming, TestDesyncRMat) { + cv::GMat in; + auto blurred = cv::gapi::blur(in, cv::Size{3,3}); + auto desynced = cv::gapi::streaming::desync(blurred); + auto out = in - blurred; + auto pipe = cv::GComputation(cv::GIn(in), cv::GOut(desynced, out)).compileStreaming(); + + cv::Size sz(32,32); + cv::Mat in_mat(sz, CV_8UC3); + cv::randu(in_mat, cv::Scalar::all(0), cv::Scalar(255)); + pipe.setSource(cv::gin(in_mat)); + pipe.start(); + + cv::optional out_desync; + cv::optional out_rmat; + while (true) { + // Initially it throwed "bad variant access" since there was + // no RMat handling in wrap_opt_arg + EXPECT_NO_THROW(pipe.pull(cv::gout(out_desync, out_rmat))); + if (out_rmat) break; + } +} + +G_API_OP(GTestBlur, , "test.blur") { + static GFrameDesc outMeta(GFrameDesc d) { return d; } +}; +GAPI_OCV_KERNEL(GOcvTestBlur, GTestBlur) { + static void run(const cv::MediaFrame& in, cv::MediaFrame& out) { + auto d = in.desc(); + GAPI_Assert(d.fmt == cv::MediaFormat::BGR); + auto view = in.access(cv::MediaFrame::Access::R); + cv::Mat mat(d.size, CV_8UC3, view.ptr[0]); + cv::Mat blurred; + cv::blur(mat, blurred, cv::Size{3,3}); + out = cv::MediaFrame::Create(blurred); + } +}; + +TEST(GAPI_Streaming, TestDesyncMediaFrame) { + initTestDataPath(); + cv::GFrame in; + auto blurred = GTestBlur::on(in); + auto desynced = cv::gapi::streaming::desync(blurred); + auto out = GTestBlur::on(blurred); + auto pipe = cv::GComputation(cv::GIn(in), cv::GOut(desynced, out)) + .compileStreaming(cv::compile_args(cv::gapi::kernels())); + + std::string filepath = findDataFile("cv/video/768x576.avi"); + try { + pipe.setSource(filepath); + } catch(...) { + throw SkipTestException("Video file can not be opened"); + } + pipe.start(); + + cv::optional out_desync; + cv::optional out_frame; + while (true) { + // Initially it throwed "bad variant access" since there was + // no MediaFrame handling in wrap_opt_arg + EXPECT_NO_THROW(pipe.pull(cv::gout(out_desync, out_frame))); + if (out_frame) break; + } +} + } // namespace opencv_test From 91ff45fbde176ff749a7d4c1b265c73370e2ea7e Mon Sep 17 00:00:00 2001 From: easonycwang Date: Fri, 24 Sep 2021 17:44:58 +0800 Subject: [PATCH 220/376] Tile: This submission is used to improve the performance of the inpaint algorithm for 3 channels images(RGB or BGR). Reason: The original algorithm implementation did not consider the cache hits. The loop of channels is outside the core loop, so the perfmance is not very good. Moving the channel loop inside the core loop can significantly improve cache hits, thereby improving performance. Performance: 360P, about >= 30% improvement iphone8P: 5.52ms -> 3.75ms iphone6s: 14.04ms -> 9.15ms --- modules/photo/src/inpaint.cpp | 92 ++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 40 deletions(-) diff --git a/modules/photo/src/inpaint.cpp b/modules/photo/src/inpaint.cpp index 14c178e24853..74e006588b20 100644 --- a/modules/photo/src/inpaint.cpp +++ b/modules/photo/src/inpaint.cpp @@ -308,50 +308,58 @@ icvTeleaInpaintFMM(const CvMat *f, CvMat *t, CvMat *out, int range, CvPriorityQu FastMarching_solve(i+1,j,i,j+1,f,t)); CV_MAT_ELEM(*t,float,i,j) = dist; + cv::Point2f gradT[3]; for (color=0; color<=2; color++) { - cv::Point2f gradI,gradT,r; - float Ia=0,Jx=0,Jy=0,s=1.0e-20f,w,dst,lev,dir,sat; - if (CV_MAT_ELEM(*f,uchar,i,j+1)!=INSIDE) { if (CV_MAT_ELEM(*f,uchar,i,j-1)!=INSIDE) { - gradT.x=(float)((CV_MAT_ELEM(*t,float,i,j+1)-CV_MAT_ELEM(*t,float,i,j-1)))*0.5f; + gradT[color].x=(float)((CV_MAT_ELEM(*t,float,i,j+1)-CV_MAT_ELEM(*t,float,i,j-1)))*0.5f; } else { - gradT.x=(float)((CV_MAT_ELEM(*t,float,i,j+1)-CV_MAT_ELEM(*t,float,i,j))); + gradT[color].x=(float)((CV_MAT_ELEM(*t,float,i,j+1)-CV_MAT_ELEM(*t,float,i,j))); } } else { if (CV_MAT_ELEM(*f,uchar,i,j-1)!=INSIDE) { - gradT.x=(float)((CV_MAT_ELEM(*t,float,i,j)-CV_MAT_ELEM(*t,float,i,j-1))); + gradT[color].x=(float)((CV_MAT_ELEM(*t,float,i,j)-CV_MAT_ELEM(*t,float,i,j-1))); } else { - gradT.x=0; + gradT[color].x=0; } } if (CV_MAT_ELEM(*f,uchar,i+1,j)!=INSIDE) { if (CV_MAT_ELEM(*f,uchar,i-1,j)!=INSIDE) { - gradT.y=(float)((CV_MAT_ELEM(*t,float,i+1,j)-CV_MAT_ELEM(*t,float,i-1,j)))*0.5f; + gradT[color].y=(float)((CV_MAT_ELEM(*t,float,i+1,j)-CV_MAT_ELEM(*t,float,i-1,j)))*0.5f; } else { - gradT.y=(float)((CV_MAT_ELEM(*t,float,i+1,j)-CV_MAT_ELEM(*t,float,i,j))); + gradT[color].y=(float)((CV_MAT_ELEM(*t,float,i+1,j)-CV_MAT_ELEM(*t,float,i,j))); } } else { if (CV_MAT_ELEM(*f,uchar,i-1,j)!=INSIDE) { - gradT.y=(float)((CV_MAT_ELEM(*t,float,i,j)-CV_MAT_ELEM(*t,float,i-1,j))); + gradT[color].y=(float)((CV_MAT_ELEM(*t,float,i,j)-CV_MAT_ELEM(*t,float,i-1,j))); } else { - gradT.y=0; + gradT[color].y=0; } } - for (k=i-range; k<=i+range; k++) { - int km=k-1+(k==1),kp=k-1-(k==t->rows-2); - for (l=j-range; l<=j+range; l++) { - int lm=l-1+(l==1),lp=l-1-(l==t->cols-2); - if (k>0&&l>0&&krows-1&&lcols-1) { - if ((CV_MAT_ELEM(*f,uchar,k,l)!=INSIDE)&& - ((l-j)*(l-j)+(k-i)*(k-i)<=range*range)) { + } + + cv::Point2f gradI,r; + float Jx[3] = {0,0,0}; + float Jy[3] = {0,0,0}; + float Ia[3] = {0,0,0}; + float s[3] = {1.0e-20f,1.0e-20f,1.0e-20f}; + float w,dst,lev,dir,sat; + + for (k=i-range; k<=i+range; k++) { + int km=k-1+(k==1),kp=k-1-(k==t->rows-2); + for (l=j-range; l<=j+range; l++) { + int lm=l-1+(l==1),lp=l-1-(l==t->cols-2); + if (k>0&&l>0&&krows-1&&lcols-1) { + if ((CV_MAT_ELEM(*f,uchar,k,l)!=INSIDE)&& + ((l-j)*(l-j)+(k-i)*(k-i)<=range*range)) { + for (color=0; color<=2; color++) { r.y = (float)(i-k); r.x = (float)(j-l); dst = (float)(1./(VectorLength(r)*sqrt((double)VectorLength(r)))); lev = (float)(1./(1+fabs(CV_MAT_ELEM(*t,float,k,l)-CV_MAT_ELEM(*t,float,i,j)))); - dir=VectorScalMult(r,gradT); + dir=VectorScalMult(r,gradT[color]); if (fabs(dir)<=0.01) dir=0.000001f; w = (float)fabs(dst*lev*dir); @@ -381,18 +389,18 @@ icvTeleaInpaintFMM(const CvMat *f, CvMat *t, CvMat *out, int range, CvPriorityQu gradI.y=0; } } - Ia += (float)w * (float)(CV_MAT_3COLOR_ELEM(*out,uchar,km,lm,color)); - Jx -= (float)w * (float)(gradI.x*r.x); - Jy -= (float)w * (float)(gradI.y*r.y); - s += w; + Ia[color] += (float)w * (float)(CV_MAT_3COLOR_ELEM(*out,uchar,km,lm,color)); + Jx[color] -= (float)w * (float)(gradI.x*r.x); + Jy[color] -= (float)w * (float)(gradI.y*r.y); + s[color] += w; } } } } - sat = (float)((Ia/s+(Jx+Jy)/(sqrt(Jx*Jx+Jy*Jy)+1.0e-20f)+0.5f)); - { + } + for (color=0; color<=2; color++) { + sat = (float)((Ia[color]/s[color]+(Jx[color]+Jy[color])/(sqrt(Jx[color]*Jx[color]+Jy[color]*Jy[color])+1.0e-20f)+0.5f)); CV_MAT_3COLOR_ELEM(*out,uchar,i-1,j-1,color) = cv::saturate_cast(sat); - } } CV_MAT_ELEM(*f,uchar,i,j) = BAND; @@ -540,17 +548,19 @@ icvNSInpaintFMM(const CvMat *f, CvMat *t, CvMat *out, int range, CvPriorityQueue FastMarching_solve(i+1,j,i,j+1,f,t)); CV_MAT_ELEM(*t,float,i,j) = dist; - for (color=0; color<=2; color++) { - cv::Point2f gradI,r; - float Ia=0,s=1.0e-20f,w,dst,dir; - - for (k=i-range; k<=i+range; k++) { - int km=k-1+(k==1),kp=k-1-(k==f->rows-2); - for (l=j-range; l<=j+range; l++) { - int lm=l-1+(l==1),lp=l-1-(l==f->cols-2); - if (k>0&&l>0&&krows-1&&lcols-1) { - if ((CV_MAT_ELEM(*f,uchar,k,l)!=INSIDE)&& - ((l-j)*(l-j)+(k-i)*(k-i)<=range*range)) { + cv::Point2f gradI,r; + float Ia[3]={0,0,0}; + float s[3]={1.0e-20f,1.0e-20f,1.0e-20f}; + float w,dst,dir; + + for (k=i-range; k<=i+range; k++) { + int km=k-1+(k==1),kp=k-1-(k==f->rows-2); + for (l=j-range; l<=j+range; l++) { + int lm=l-1+(l==1),lp=l-1-(l==f->cols-2); + if (k>0&&l>0&&krows-1&&lcols-1) { + if ((CV_MAT_ELEM(*f,uchar,k,l)!=INSIDE)&& + ((l-j)*(l-j)+(k-i)*(k-i)<=range*range)) { + for (color=0; color<=2; color++) { r.y=(float)(k-i); r.x=(float)(l-j); @@ -594,13 +604,15 @@ icvNSInpaintFMM(const CvMat *f, CvMat *t, CvMat *out, int range, CvPriorityQueue dir = (float)fabs(VectorScalMult(r,gradI)/sqrt(VectorLength(r)*VectorLength(gradI))); } w = dst*dir; - Ia += (float)w * (float)(CV_MAT_3COLOR_ELEM(*out,uchar,km,lm,color)); - s += w; + Ia[color] += (float)w * (float)(CV_MAT_3COLOR_ELEM(*out,uchar,km,lm,color)); + s[color] += w; } } } } - CV_MAT_3COLOR_ELEM(*out,uchar,i-1,j-1,color) = cv::saturate_cast((double)Ia/s); + } + for (color=0; color<=2; color++) { + CV_MAT_3COLOR_ELEM(*out,uchar,i-1,j-1,color) = cv::saturate_cast((double)Ia[color]/s[color]); } CV_MAT_ELEM(*f,uchar,i,j) = BAND; From 9e835e8edb62b42ca53d887be4c124fed142dfcd Mon Sep 17 00:00:00 2001 From: thezane <10068531+thezane@users.noreply.github.com> Date: Sat, 25 Sep 2021 13:42:12 -0400 Subject: [PATCH 221/376] Merge pull request #20636 from thezane:recoverPoseFromDifferentCameras Recover pose from different cameras (version 2) * add recoverPose for two different cameras * Address review comments from original PR * Address new review comments * Rename private api Co-authored-by: tompollok Co-authored-by: Zane --- modules/calib3d/include/opencv2/calib3d.hpp | 70 +++++++++++++++++++ modules/calib3d/src/five-point.cpp | 70 ++++++++++++++----- .../calib3d/test/test_cameracalibration.cpp | 48 +++++++++++-- 3 files changed, 162 insertions(+), 26 deletions(-) diff --git a/modules/calib3d/include/opencv2/calib3d.hpp b/modules/calib3d/include/opencv2/calib3d.hpp index 4928df65d1f1..37c5e7089a34 100644 --- a/modules/calib3d/include/opencv2/calib3d.hpp +++ b/modules/calib3d/include/opencv2/calib3d.hpp @@ -2842,6 +2842,76 @@ unit length. */ CV_EXPORTS_W void decomposeEssentialMat( InputArray E, OutputArray R1, OutputArray R2, OutputArray t ); +/** @brief Recovers the relative camera rotation and the translation from corresponding points in two images from two different cameras, using cheirality check. Returns the number of +inliers that pass the check. + +@param points1 Array of N 2D points from the first image. The point coordinates should be +floating-point (single or double precision). +@param points2 Array of the second image points of the same size and format as points1 . +@param cameraMatrix1 Input/output camera matrix for the first camera, the same as in +@ref calibrateCamera. Furthermore, for the stereo case, additional flags may be used, see below. +@param distCoeffs1 Input/output vector of distortion coefficients, the same as in +@ref calibrateCamera. +@param cameraMatrix2 Input/output camera matrix for the first camera, the same as in +@ref calibrateCamera. Furthermore, for the stereo case, additional flags may be used, see below. +@param distCoeffs2 Input/output vector of distortion coefficients, the same as in +@ref calibrateCamera. +@param E The output essential matrix. +@param R Output rotation matrix. Together with the translation vector, this matrix makes up a tuple +that performs a change of basis from the first camera's coordinate system to the second camera's +coordinate system. Note that, in general, t can not be used for this tuple, see the parameter +described below. +@param t Output translation vector. This vector is obtained by @ref decomposeEssentialMat and +therefore is only known up to scale, i.e. t is the direction of the translation vector and has unit +length. +@param method Method for computing an essential matrix. +- @ref RANSAC for the RANSAC algorithm. +- @ref LMEDS for the LMedS algorithm. +@param prob Parameter used for the RANSAC or LMedS methods only. It specifies a desirable level of +confidence (probability) that the estimated matrix is correct. +@param threshold Parameter used for RANSAC. It is the maximum distance from a point to an epipolar +line in pixels, beyond which the point is considered an outlier and is not used for computing the +final fundamental matrix. It can be set to something like 1-3, depending on the accuracy of the +point localization, image resolution, and the image noise. +@param mask Input/output mask for inliers in points1 and points2. If it is not empty, then it marks +inliers in points1 and points2 for then given essential matrix E. Only these inliers will be used to +recover pose. In the output mask only inliers which pass the cheirality check. + +This function decomposes an essential matrix using @ref decomposeEssentialMat and then verifies +possible pose hypotheses by doing cheirality check. The cheirality check means that the +triangulated 3D points should have positive depth. Some details can be found in @cite Nister03. + +This function can be used to process the output E and mask from @ref findEssentialMat. In this +scenario, points1 and points2 are the same input for findEssentialMat.: +@code + // Example. Estimation of fundamental matrix using the RANSAC algorithm + int point_count = 100; + vector points1(point_count); + vector points2(point_count); + + // initialize the points here ... + for( int i = 0; i < point_count; i++ ) + { + points1[i] = ...; + points2[i] = ...; + } + + // Input: camera calibration of both cameras, for example using intrinsic chessboard calibration. + Mat cameraMatrix1, distCoeffs1, cameraMatrix2, distCoeffs2; + + // Output: Essential matrix, relative rotation and relative translation. + Mat E, R, t, mask; + + recoverPose(points1, points2, cameraMatrix1, distCoeffs1, cameraMatrix2, distCoeffs2, E, R, t, mask); +@endcode + */ +CV_EXPORTS_W int recoverPose( InputArray points1, InputArray points2, + InputArray cameraMatrix1, InputArray distCoeffs1, + InputArray cameraMatrix2, InputArray distCoeffs2, + OutputArray E, OutputArray R, OutputArray t, + int method = cv::RANSAC, double prob = 0.999, double threshold = 1.0, + InputOutputArray mask = noArray()); + /** @brief Recovers the relative camera rotation and the translation from an estimated essential matrix and the corresponding points in two images, using cheirality check. Returns the number of inliers that pass the check. diff --git a/modules/calib3d/src/five-point.cpp b/modules/calib3d/src/five-point.cpp index d005248535c6..d947334142e6 100644 --- a/modules/calib3d/src/five-point.cpp +++ b/modules/calib3d/src/five-point.cpp @@ -401,6 +401,29 @@ class EMEstimatorCallback CV_FINAL : public PointSetRegistrator::Callback } }; +// Find essential matrix given undistorted points and two cameras. +static Mat findEssentialMat_( InputArray _points1, InputArray _points2, + InputArray cameraMatrix1, InputArray cameraMatrix2, + int method, double prob, double threshold, OutputArray _mask) +{ + // Scale the points back. We use "arithmetic mean" between the supplied two camera matrices. + // Thanks to such 2-stage procedure RANSAC threshold still makes sense, because the undistorted + // and rescaled points have a similar value range to the original ones. + Mat _pointsTransformed1, _pointsTransformed2; + Mat cm1 = cameraMatrix1.getMat(), cm2 = cameraMatrix2.getMat(), cm0; + Mat(cm1 + cm2).convertTo(cm0, CV_64F, 0.5); + CV_Assert(cm0.rows == 3 && cm0.cols == 3); + CV_Assert(std::abs(cm0.at(2, 0)) < 1e-3 && + std::abs(cm0.at(2, 1)) < 1e-3 && + std::abs(cm0.at(2, 2) - 1.) < 1e-3); + Mat affine = cm0.rowRange(0, 2); + + transform(_points1, _pointsTransformed1, affine); + transform(_points2, _pointsTransformed2, affine); + + return findEssentialMat(_pointsTransformed1, _pointsTransformed2, cm0, method, prob, threshold, _mask); +} + } // Input should be a vector of n 2D points or a Nx2 matrix @@ -489,25 +512,10 @@ cv::Mat cv::findEssentialMat( InputArray _points1, InputArray _points2, CV_INSTRUMENT_REGION(); // Undistort image points, bring them to 3x3 identity "camera matrix" - Mat _pointsUntistorted1, _pointsUntistorted2; - undistortPoints(_points1, _pointsUntistorted1, cameraMatrix1, distCoeffs1); - undistortPoints(_points2, _pointsUntistorted2, cameraMatrix2, distCoeffs2); - - // Scale the points back. We use "arithmetic mean" between the supplied two camera matrices. - // Thanks to such 2-stage procedure RANSAC threshold still makes sense, because the undistorted - // and rescaled points have a similar value range to the original ones. - Mat cm1 = cameraMatrix1.getMat(), cm2 = cameraMatrix2.getMat(), cm0; - Mat(cm1 + cm2).convertTo(cm0, CV_64F, 0.5); - CV_Assert(cm0.rows == 3 && cm0.cols == 3); - CV_Assert(std::abs(cm0.at(2, 0)) < 1e-3 && - std::abs(cm0.at(2, 1)) < 1e-3 && - std::abs(cm0.at(2, 2) - 1.) < 1e-3); - Mat affine = cm0.rowRange(0, 2); - - transform(_pointsUntistorted1, _pointsUntistorted1, affine); - transform(_pointsUntistorted2, _pointsUntistorted2, affine); - - return findEssentialMat(_pointsUntistorted1, _pointsUntistorted2, cm0, method, prob, threshold, _mask); + Mat _pointsUndistorted1, _pointsUndistorted2; + undistortPoints(_points1, _pointsUndistorted1, cameraMatrix1, distCoeffs1); + undistortPoints(_points2, _pointsUndistorted2, cameraMatrix2, distCoeffs2); + return findEssentialMat_(_pointsUndistorted1, _pointsUndistorted2, cameraMatrix1, cameraMatrix2, method, prob, threshold, _mask); } cv::Mat cv::findEssentialMat( InputArray points1, InputArray points2, @@ -524,6 +532,30 @@ cv::Mat cv::findEssentialMat( InputArray points1, InputArray points2, } +int cv::recoverPose( InputArray _points1, InputArray _points2, + InputArray cameraMatrix1, InputArray distCoeffs1, + InputArray cameraMatrix2, InputArray distCoeffs2, + OutputArray E, OutputArray R, OutputArray t, + int method, double prob, double threshold, + InputOutputArray _mask) +{ + CV_INSTRUMENT_REGION(); + + // Undistort image points, bring them to 3x3 identity "camera matrix" + Mat _pointsUndistorted1, _pointsUndistorted2; + undistortPoints(_points1, _pointsUndistorted1, cameraMatrix1, distCoeffs1); + undistortPoints(_points2, _pointsUndistorted2, cameraMatrix2, distCoeffs2); + + // Get essential matrix. + Mat _E = findEssentialMat_(_pointsUndistorted1, _pointsUndistorted2, cameraMatrix1, cameraMatrix2, + method, prob, threshold, _mask); + CV_Assert(_E.cols == 3 && _E.rows == 3); + E.create(3, 3, _E.type()); + _E.copyTo(E); + + return recoverPose(_E, _pointsUndistorted1, _pointsUndistorted2, Mat::eye(3,3, CV_64F), R, t, _mask); +} + int cv::recoverPose( InputArray E, InputArray _points1, InputArray _points2, InputArray _cameraMatrix, OutputArray _R, OutputArray _t, double distanceThresh, InputOutputArray _mask, OutputArray triangulatedPoints) diff --git a/modules/calib3d/test/test_cameracalibration.cpp b/modules/calib3d/test/test_cameracalibration.cpp index 9f3663e8cce3..10350527fa78 100644 --- a/modules/calib3d/test/test_cameracalibration.cpp +++ b/modules/calib3d/test/test_cameracalibration.cpp @@ -2142,7 +2142,17 @@ TEST(CV_RecoverPoseTest, regression_15341) // camera matrix with both focal lengths = 1, and principal point = (0, 0) const Mat cameraMatrix = Mat::eye(3, 3, CV_64F); - const Mat zeroDistCoeffs = Mat::zeros(1, 5, CV_64F); + + // camera matrix with focal lengths 0.5 and 0.6 respectively and principal point = (100, 200) + double cameraMatrix2Data[] = { 0.5, 0, 100, + 0, 0.6, 200, + 0, 0, 1 }; + const Mat cameraMatrix2( 3, 3, CV_64F, cameraMatrix2Data ); + + // zero and nonzero distortion coefficients + double nonZeroDistCoeffsData[] = { 0.01, 0.0001, 0, 0, 1e-04, 0.2, 0.02, 0.0002 }; // k1, k2, p1, p2, k3, k4, k5, k6 + vector distCoeffsList = {Mat::zeros(1, 5, CV_64F), Mat{1, 8, CV_64F, nonZeroDistCoeffsData}}; + const auto &zeroDistCoeffs = distCoeffsList[0]; int Inliers = 0; @@ -2158,14 +2168,26 @@ TEST(CV_RecoverPoseTest, regression_15341) // Estimation of fundamental matrix using the RANSAC algorithm Mat E, E2, R, t; + + // Check pose when camera matrices are different. + for (const auto &distCoeffs: distCoeffsList) + { + E = findEssentialMat(points1, points2, cameraMatrix, distCoeffs, cameraMatrix2, distCoeffs, RANSAC, 0.999, 1.0, mask); + recoverPose(points1, points2, cameraMatrix, distCoeffs, cameraMatrix2, distCoeffs, E2, R, t, RANSAC, 0.999, 1.0, mask); + EXPECT_LT(cv::norm(E, E2, NORM_INF), 1e-4) << + "Two big difference between the same essential matrices computed using different functions with different cameras, testcase " << testcase; + EXPECT_EQ(0, (int)mask[13]) << "Detecting outliers in function failed with different cameras, testcase " << testcase; + } + + // Check pose when camera matrices are the same. E = findEssentialMat(points1, points2, cameraMatrix, RANSAC, 0.999, 1.0, mask); E2 = findEssentialMat(points1, points2, cameraMatrix, zeroDistCoeffs, cameraMatrix, zeroDistCoeffs, RANSAC, 0.999, 1.0, mask); EXPECT_LT(cv::norm(E, E2, NORM_INF), 1e-4) << - "Two big difference between the same essential matrices computed using different functions, testcase " << testcase; - EXPECT_EQ(0, (int)mask[13]) << "Detecting outliers in function findEssentialMat failed, testcase " << testcase; + "Two big difference between the same essential matrices computed using different functions with same cameras, testcase " << testcase; + EXPECT_EQ(0, (int)mask[13]) << "Detecting outliers in function findEssentialMat failed with same cameras, testcase " << testcase; points2[12] = Point2f(0.0f, 0.0f); // provoke another outlier detection for recover Pose Inliers = recoverPose(E, points1, points2, cameraMatrix, R, t, mask); - EXPECT_EQ(0, (int)mask[12]) << "Detecting outliers in function failed, testcase " << testcase; + EXPECT_EQ(0, (int)mask[12]) << "Detecting outliers in function failed with same cameras, testcase " << testcase; } else // testcase with mat input data { @@ -2185,14 +2207,26 @@ TEST(CV_RecoverPoseTest, regression_15341) // Estimation of fundamental matrix using the RANSAC algorithm Mat E, E2, R, t; + + // Check pose when camera matrices are different. + for (const auto &distCoeffs: distCoeffsList) + { + E = findEssentialMat(points1, points2, cameraMatrix, distCoeffs, cameraMatrix2, distCoeffs, RANSAC, 0.999, 1.0, mask); + recoverPose(points1, points2, cameraMatrix, distCoeffs, cameraMatrix2, distCoeffs, E2, R, t, RANSAC, 0.999, 1.0, mask); + EXPECT_LT(cv::norm(E, E2, NORM_INF), 1e-4) << + "Two big difference between the same essential matrices computed using different functions with different cameras, testcase " << testcase; + EXPECT_EQ(0, (int)mask.at(13)) << "Detecting outliers in function failed with different cameras, testcase " << testcase; + } + + // Check pose when camera matrices are the same. E = findEssentialMat(points1, points2, cameraMatrix, RANSAC, 0.999, 1.0, mask); E2 = findEssentialMat(points1, points2, cameraMatrix, zeroDistCoeffs, cameraMatrix, zeroDistCoeffs, RANSAC, 0.999, 1.0, mask); EXPECT_LT(cv::norm(E, E2, NORM_INF), 1e-4) << - "Two big difference between the same essential matrices computed using different functions, testcase " << testcase; - EXPECT_EQ(0, (int)mask.at(13)) << "Detecting outliers in function findEssentialMat failed, testcase " << testcase; + "Two big difference between the same essential matrices computed using different functions with same cameras, testcase " << testcase; + EXPECT_EQ(0, (int)mask.at(13)) << "Detecting outliers in function findEssentialMat failed with same cameras, testcase " << testcase; points2.at(12) = Point2f(0.0f, 0.0f); // provoke an outlier detection Inliers = recoverPose(E, points1, points2, cameraMatrix, R, t, mask); - EXPECT_EQ(0, (int)mask.at(12)) << "Detecting outliers in function failed, testcase " << testcase; + EXPECT_EQ(0, (int)mask.at(12)) << "Detecting outliers in function failed with same cameras, testcase " << testcase; } EXPECT_EQ(Inliers, point_count - invalid_point_count) << "Number of inliers differs from expected number of inliers, testcase " << testcase; From 236c64a17d4c9e1eac3fb25044bdfeceba1d7463 Mon Sep 17 00:00:00 2001 From: Nicholas Ho <88894303+Nicholas-Ho-arm@users.noreply.github.com> Date: Sat, 25 Sep 2021 18:43:33 +0100 Subject: [PATCH 222/376] Merge pull request #20712 from Nicholas-Ho-arm:3.4_RowVec_8u32f * Add RowVec_8u32f * Fix build errors in Linux x64 Debug and armeabi-v7a * Reformat code to make it more clean and conventional * Optimise with vx_load_expand_q() --- modules/imgproc/src/filter.simd.hpp | 47 ++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/modules/imgproc/src/filter.simd.hpp b/modules/imgproc/src/filter.simd.hpp index bed6f83071bb..94dbce060c45 100644 --- a/modules/imgproc/src/filter.simd.hpp +++ b/modules/imgproc/src/filter.simd.hpp @@ -465,6 +465,49 @@ struct RowVec_8u32s bool smallValues; }; +struct RowVec_8u32f +{ + RowVec_8u32f() {} + RowVec_8u32f( const Mat& _kernel ) : kernel(_kernel) {} + + int operator()(const uchar* _src, uchar* _dst, int width, int cn) const + { + CV_INSTRUMENT_REGION(); + + int i = 0, k, _ksize = kernel.rows + kernel.cols - 1; + float* dst = (float*)_dst; + const float* _kx = kernel.ptr(); + width *= cn; + for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + { + v_float32 s0 = vx_setzero_f32(); + v_float32 s1 = vx_setzero_f32(); + v_float32 s2 = vx_setzero_f32(); + v_float32 s3 = vx_setzero_f32(); + k = 0; + for( ; k < _ksize ; k++ ) + { + v_float32 f = vx_setall_f32(_kx[k]); + const uchar* src = (const uchar*)_src + i + k * cn; + v_float32 vs_ll = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src))); + v_float32 vs_lh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + v_float32::nlanes))); + v_float32 vs_hl = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 2*v_float32::nlanes))); + v_float32 vs_hh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 3*v_float32::nlanes))); + s0 = v_muladd(vs_ll, f, s0); + s1 = v_muladd(vs_lh, f, s1); + s2 = v_muladd(vs_hl, f, s2); + s3 = v_muladd(vs_hh, f, s3); + } + v_store(dst + i, s0); + v_store(dst + i + v_float32::nlanes, s1); + v_store(dst + i + 2*v_float32::nlanes, s2); + v_store(dst + i + 3*v_float32::nlanes, s3); + } + return i; + } + + Mat kernel; +}; struct SymmRowSmallVec_8u32s { @@ -2292,6 +2335,7 @@ struct FilterVec_32f #else typedef RowNoVec RowVec_8u32s; +typedef RowNoVec RowVec_8u32f; typedef RowNoVec RowVec_16s32f; typedef RowNoVec RowVec_32f; typedef SymmRowSmallNoVec SymmRowSmallVec_8u32s; @@ -2899,7 +2943,8 @@ Ptr getLinearRowFilter( return makePtr > (kernel, anchor, RowVec_8u32s(kernel)); if( sdepth == CV_8U && ddepth == CV_32F ) - return makePtr >(kernel, anchor); + return makePtr > + (kernel, anchor, RowVec_8u32f(kernel)); if( sdepth == CV_8U && ddepth == CV_64F ) return makePtr >(kernel, anchor); if( sdepth == CV_16U && ddepth == CV_32F ) From fdc8ed8d05d6f073dab21a6f0fdb3f20953738be Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Sat, 25 Sep 2021 09:31:44 +0300 Subject: [PATCH 223/376] Update perf_bgfg_mog2.cpp, perf_bgfg_knn.cpp --- modules/video/perf/opencl/perf_bgfg_knn.cpp | 6 +++--- modules/video/perf/opencl/perf_bgfg_mog2.cpp | 6 +++--- modules/video/perf/perf_bgfg_knn.cpp | 6 +++--- modules/video/perf/perf_bgfg_mog2.cpp | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/video/perf/opencl/perf_bgfg_knn.cpp b/modules/video/perf/opencl/perf_bgfg_knn.cpp index 30419af42274..c96ba6881ad2 100644 --- a/modules/video/perf/opencl/perf_bgfg_knn.cpp +++ b/modules/video/perf/opencl/perf_bgfg_knn.cpp @@ -20,7 +20,7 @@ typedef TestBaseWithParam KNN_GetBackgroundImage; using namespace opencv_test; -OCL_PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +OCL_PERF_TEST_P(KNN_Apply, KNN, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoKNNParamType params = GetParam(); @@ -51,8 +51,8 @@ OCL_PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/vid } OCL_PERF_TEST_P(KNN_GetBackgroundImage, KNN, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoKNNParamType params = GetParam(); diff --git a/modules/video/perf/opencl/perf_bgfg_mog2.cpp b/modules/video/perf/opencl/perf_bgfg_mog2.cpp index 9952be79f6dc..b127a50d8df2 100644 --- a/modules/video/perf/opencl/perf_bgfg_mog2.cpp +++ b/modules/video/perf/opencl/perf_bgfg_mog2.cpp @@ -20,7 +20,7 @@ typedef TestBaseWithParam MOG2_GetBackgroundImage; using namespace opencv_test; -OCL_PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +OCL_PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoMOG2ParamType params = GetParam(); @@ -51,8 +51,8 @@ OCL_PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/v } OCL_PERF_TEST_P(MOG2_GetBackgroundImage, Mog2, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoMOG2ParamType params = GetParam(); diff --git a/modules/video/perf/perf_bgfg_knn.cpp b/modules/video/perf/perf_bgfg_knn.cpp index d9ead09fd93d..5d4f3bc400bd 100644 --- a/modules/video/perf/perf_bgfg_knn.cpp +++ b/modules/video/perf/perf_bgfg_knn.cpp @@ -15,7 +15,7 @@ typedef tuple VideoKNNParamType; typedef TestBaseWithParam KNN_Apply; typedef TestBaseWithParam KNN_GetBackgroundImage; -PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +PERF_TEST_P(KNN_Apply, KNN, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoKNNParamType params = GetParam(); @@ -46,8 +46,8 @@ PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/video/1 } PERF_TEST_P(KNN_GetBackgroundImage, KNN, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoKNNParamType params = GetParam(); diff --git a/modules/video/perf/perf_bgfg_mog2.cpp b/modules/video/perf/perf_bgfg_mog2.cpp index 92e5d0283fad..b932d6a22cd9 100644 --- a/modules/video/perf/perf_bgfg_mog2.cpp +++ b/modules/video/perf/perf_bgfg_mog2.cpp @@ -15,7 +15,7 @@ typedef tuple VideoMOG2ParamType; typedef TestBaseWithParam MOG2_Apply; typedef TestBaseWithParam MOG2_GetBackgroundImage; -PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoMOG2ParamType params = GetParam(); @@ -46,8 +46,8 @@ PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/video } PERF_TEST_P(MOG2_GetBackgroundImage, Mog2, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoMOG2ParamType params = GetParam(); From 2cc14bd0fb2b898d7667c01bd24754dc9df0d5bd Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Mon, 27 Sep 2021 10:53:23 +0300 Subject: [PATCH 224/376] Verbose output for errors found by header parser. --- modules/python/src2/hdr_parser.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/python/src2/hdr_parser.py b/modules/python/src2/hdr_parser.py index 951dfe11c33a..80cdea339fd5 100755 --- a/modules/python/src2/hdr_parser.py +++ b/modules/python/src2/hdr_parser.py @@ -53,13 +53,13 @@ def batch_replace(self, s, pairs): def get_macro_arg(self, arg_str, npos): npos2 = npos3 = arg_str.find("(", npos) if npos2 < 0: - print("Error: no arguments for the macro at %d" % (self.lineno,)) + print("Error: no arguments for the macro at %s:%d" % (self.hname, self.lineno)) sys.exit(-1) balance = 1 while 1: t, npos3 = self.find_next_token(arg_str, ['(', ')'], npos3+1) if npos3 < 0: - print("Error: no matching ')' in the macro call at %d" % (self.lineno,)) + print("Error: no matching ')' in the macro call at %s:%d" % (self.hname, self.lineno)) sys.exit(-1) if t == '(': balance += 1 @@ -161,7 +161,7 @@ def parse_arg(self, arg_str, argno): angle_stack.append(0) elif w == "," or w == '>': if not angle_stack: - print("Error at %d: argument contains ',' or '>' not within template arguments" % (self.lineno,)) + print("Error at %s:%d: argument contains ',' or '>' not within template arguments" % (self.hname, self.lineno)) sys.exit(-1) if w == ",": arg_type += "_and_" @@ -191,7 +191,7 @@ def parse_arg(self, arg_str, argno): p1 = arg_name.find("[") p2 = arg_name.find("]",p1+1) if p2 < 0: - print("Error at %d: no closing ]" % (self.lineno,)) + print("Error at %s:%d: no closing ]" % (self.hname, self.lineno)) sys.exit(-1) counter_str = arg_name[p1+1:p2].strip() if counter_str == "": From 8fa8d471af43f62c60ea13c2ae1274390e34768b Mon Sep 17 00:00:00 2001 From: WJJ1995 Date: Tue, 28 Sep 2021 05:59:09 +0800 Subject: [PATCH 225/376] Merge pull request #20290 from wjj19950828:add_paddle_humanseg_demo Add paddle humanseg demo * fixed onnx resize op bug * add humanseg demo for PaddlePaddle sample * update README.md and flake8 format * update func name * update README.md for enviroment setup * update README.md in the way install paddle2onnx * update README.md * update README.md * add paddleseg in requirements.txt * deal with comments * replace picture --- .../dnn_conversion/paddlepaddle/README.md | 69 +++++++++-- .../paddlepaddle/data/result_test_human.jpg | Bin 0 -> 62199 bytes .../paddlepaddle/paddle_humanseg.py | 112 ++++++++++++++++++ .../paddlepaddle/paddle_resnet50.py | 18 +-- .../dnn_conversion/requirements.txt | 1 + 5 files changed, 183 insertions(+), 17 deletions(-) create mode 100644 samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/data/result_test_human.jpg create mode 100644 samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_humanseg.py diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/README.md b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/README.md index 7aba491c9d98..1b1a28767c80 100644 --- a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/README.md +++ b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/README.md @@ -1,6 +1,6 @@ -# Run PaddlePaddle model by OpenCV +# Run PaddlePaddle model using OpenCV -This tutorial shows how to run PaddlePaddle model by opencv. +These two demonstrations show how to inference PaddlePaddle model using OpenCV. ## Environment Setup @@ -10,16 +10,69 @@ pip install paddlehub pip install paddle2onnx ``` -## Run PaddlePaddle model demo +## 1. Run PaddlePaddle ResNet50 using OpenCV -Run the example code as below, +### Run PaddlePaddle model demo + +Run the code sample as follows: ```shell python paddle_resnet50.py ``` -there are 3 part of this execution +There are three parts to the process: + +1. Export PaddlePaddle ResNet50 model to onnx format. +2. Use `cv2.dnn.readNetFromONNX` to load the model file. +3. Preprocess image file and do the inference. + +## 2. Run PaddleSeg Portrait Segmentation using OpenCV + +### Convert to ONNX Model + +#### 1. Get Paddle Inference model + +For more details, please refer to [PaddleSeg](https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.1/contrib/HumanSeg/README.md). + +```shell +wget https://x2paddle.bj.bcebos.com/inference/models/humanseg_hrnet18_small_v1.zip +unzip humanseg_hrnet18_small_v1.zip +``` + +Notes: + +* The exported model must have a fixed input shape, as dynamic is not supported at this moment. + +#### 2. Convert to ONNX model using paddle2onnx + +To convert the model, use the following command: + +``` +paddle2onnx --model_dir humanseg_hrnet18_small_v1 \ + --model_filename model.pdmodel \ + --params_filename model.pdiparams \ + --opset_version 11 \ + --save_file humanseg_hrnet18_tiny.onnx +``` + +The converted model can be found in the current directory by the name `humanseg_hrnet18_tiny.onnx` . + +### Run PaddleSeg Portrait Segmentation demo + +Run the code sample as follows: + +```shell +python paddle_humanseg.py +``` + +There are three parts to the process: + +1. Use `cv2.dnn.readNetFromONNX` to load the model file. +2. Preprocess image file and do inference. +3. Postprocess image file and visualize. + +The resulting file can be found at `data/result_test_human.jpg` . + +### Portrait segmentation visualization -- 1. Export PaddlePaddle ResNet50 model to onnx format; -- 2. Use `cv2.dnn.readNetFromONNX` load model file; -- 3. Preprocess image file and do inference. + diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/data/result_test_human.jpg b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/data/result_test_human.jpg new file mode 100644 index 0000000000000000000000000000000000000000..652b03f3b464ab432133061e780e7519c90c4ff7 GIT binary patch literal 62199 zcmbTdWl$VX)IPefc#uFKxCIL=5+JxHB)Ej&PLN>1-Q5=t5Nrv-Z8s1$xH}0D++7w} zY=MP^0QvKK->Q4R+z+inx3xnbl047`sru>&Hq~mJXTdyQ3PON0RUJJ2jJfl zKmmY_^nub+QFWK?uaY+QUoddBC>tn4p2x!;OQO3TVCDy!;$G{Ap0HZ`~O^!D`+ z3=R#COioSD%+Ad(EN*OWZSU+N_Vy1@XXh8_%PY+F&40MC0Kor+^-%vm;v#>*h5g?g z;QxmU3)}bM1d`+6vI^o+ywb-1;Qr*9P&fhQ>$IY}ZbCNU_oq}=9uq`Q*@8RL&tn7!|NbJMCf zSR8ece+;LnTljKx^R~A*er*78OJQ_L1grsjkUocWEsew`x-S*WD%pR|j)9rb+lZEc{mZLA!uH{|8irA>=r zNui7{ zQBpDKD!-dH4@~(yZa4VT`~wgU?wC`- z1oEo1nh>N~R5x;jle+sVY!kWBA|KrXerAP-|dfSO04)i@Cah z4tA|Rp$=0!5n_K%Fzfll7zLQO3Q<^i|zReD2av_NW5b>$*h;F%7k%SFi+IMvifdW#P--{R51{ z4VqGgAk9oOLW=S6o=~$F!x=1|gBIuaRAiS8S3oBQh$JNTcDGuS7SA2*OnnaWpymy>UG zr+wn#j5bgwM+gNx&`O={IBg2J`n~pes`m$_{!((2OpAeCXw8yHti14=oa%WBtqRBX zq1FC@rue?0b|t^FCRjs!Ng)8asoLwfZ3-N$bsw$nfLAqu`|DT6jekj+P2i`ykp|rO zR9=|)M66>+J${8gF?*k3$6yT4S>0=~&sx!~W|B-e3VKv2oAGQZi$^$o@Y9^s25t7? zx(bsrLxg{AC~wJXkg7ZmdCK|t^}gXiO{N;_FA{uGr-K0Yd-f$qUzPk6<1@0vR@X8y zQ)>w}*}KpJFN54_m$U6J$+G(*gP6;$BhgwSSw(-@h*l0+`vcDbiuV#AlZrzGo)8|H zICQRem0y`f&w%#B7y(SCEoPUvK07Kotggv7f@Z zTPCVmD~4xMOzyhr)%1yZ3q#8ts84f`qBA7Kky;bZE&_SKGl~wK@){zvy=s$7NDJh~ zF>}Qjyh-XF%}Cl~VTF_*##Gct9L)uleoTR1)S`~2Ni{r-8>4m`KNm%16l%T={4u{! zV~LBi1u5J%;MxqNr`hKCM$0w@>C|-G8=hXyy|~NSp1q+=TgD@LHLh-*aG@0&U@g+4 z)PR-NyH$qV-6iu_*TV}h1!sO94HR&38E31P~ zg3C+Zgnf0E=wj>~1ARe>{MPQX&Y&6dybm2mn5?o*m%+)nb-))4h=lV_8ceYlakt{D zE2J5BIWSkFvcMop`W2-|T;?Dbdi)>2yqyrP*EALJoR+O(tI_y+yf=h17c{}1EH9S~ znr`FX9ET4SEjI$8PRX@mT%PqwntFEAQ!_!b-pqn4q$+5rg~{pYO8IC*TI65_OnG9) zHJ^bv@LU5W8x!#?=xeZ|loh;KTj`bF*X{fi-pM1)orJe6$7n>Ym!vs3DX2M)Aw|pe z9CwnW0;$ zW;OrM%-RD|b7jk`OQHp5TT)7mG|&L3s**Vqxw~_FGFyMZ?JSSA{yHfdVSHK1S}DYF3hijNEHeQ5h+XQ;)h0> ztTLc@v^;G2y4WIb43hkWbt3#`Y^9UKAcXs7!9S}mlUle2q0~T5WO7{;iV5J;yQe(~3N38FL z>U|l_){Gn8g+SLHxcwv058UO)so{b+qj z<$-dl8`5@LjE>97ee9#kf|wIIM`F(VjbmCb{SLo8cl(+FKF9n6q?wPde+cA1+7fIysh~(y+A_Vb z`US8y0I$g3fv=K?n$>>*^gGD4vUsRR4Ct_rycZc)1y_#jV!*E}pM0LSG2FC|M?$FO zOdtH1W1q19o2?88B(pS>waLnJj9~Ekq9$*M(PKmgS#^AquVSIy`)+$DBI6G!E^D(s z5Gb2JBWi%}tLY97b3oOxKOGAeUb{_jSPm3&SmTj6B7J)>g)a3=fEmSCy*P;$2=7-*=>uD>e3?&0(W&MW}>2biraZsryk8BAV_=Blbw zSy*98Nb$`s!_}B1L;{-F8=#W)=W+3Im?o%llGiypgEs*v zU_J2mEzgLzgptnIEv+xxMW_2wJpJT45P}nZuB6&$3tTvXp2pcN?Jn|1jyI|;cLnRGt85{!xs&Dop&+Ytf{o|yn_N$-l6?cCmdzl&KeW$zBkQ>XEcNHt~SmyTuN9J1ec&L{0~oIdR2UStSUsFpVO+;9Ay-#W@nJm4M; z2EBI_lkoqkxS%&Cyp(1p5G&GLs~1!v(Z$z&QAKASH~AY_#Kw@Z+A=EElU72XOQFK( ze%tO;@67@!quk(+b;9Vhsh3H-Y_nnPE7@*uhTI(pif;1B<3vGBkPEJT@8Z;(4>szn2};Pv#9S3-f*7IfMGAsLK-rnP+5_&Fes(lK*!8}tONz6n zL){i}ahUs!yD|06kRjNm>w}wrjYGd?l@y0TRtji&aIV-0+Nk*TOw7{jA3$zH{ZC)H zwt@3Jv*~V;WIlztP(vimmR#&ZcgBlv-VKXhuwVyZX4Mv&<5EG#JbFx?_f@!v7iA!uh4&~i;!J_mv^|KOq zFYbW*5z^pgCs|oSoOL31_?S!>G-6OW_&qNUEnApDIo#Xog~#X2)k z3}OrDd?j|Yb#rt!R+3hx4>0UD)?DG{Uv6yD7ou3eA@jL5NCZEnMpB9#_r2-W5t=}x)TyUbyn2dCM44wWz?d@ZeE6H7=(<_-i4gDPF_-Bw>ReNMyK zs>aMBkRb#KSxY0eLzEUt%) zK_m(k5HdN>#g=>A#^!B}!;_9XG{`>p(f!Nj{s#j#U&-?5C^jG6hEqo5BDZCLFIDuW z1=jbZVtYk09i%y&w62^3qPdJU+3ZS*iPW$h4pYHzf+hY!HJ1jxEm?tntkel$8{+F= zG*SrRe0qB_Bz^_``+8z>PK$^VfU8Y?I_@@>2F9zkI5K3NQeJ;dD)Tm2-!`_^gp8WY zvxT};^Or&%D*5TEyd)q5zTHxs(xB=g*tGQ%G})$B8C)hI&yq?OaF%4{PjT>ZK-M~` zw>HpJ__eNEtqKE`o&WNNw3D0c{DRAxCVJRh6HiN=eT)74-oLh!)^unw60w{d;AuLu zl;zy^ciUisO){f}nPloMi{G{cQ%vhW0A7$n3`jR((+Z47h-Pc|ANj76)OWr=e*F(X z_oIC9s?lPxFe zKCg4=tN>BjJ_RWssIle!<_;6*sytGEx3~0Na{Y-(Vlim2)0O5#_vGa{UB-_Ne{}it z26A7XL1Cp5!K9(<_Ff#M zP{Sn<^_V;W!IU{{>KKp{w4%hyRwOYF4|o$os}`@(J3y+%MSJU*65qx_{th*O1qrnuFwv3&%qyrf&nN8jA182qkN}Q532dE>_a&_Xip80cjc3aW$sy|5P2e%T= zW?OSdzZ6?DmnwSQiTR2pQ8*aUuz-?DM=OSJ(%B+)kqv(=kEEX`a% zh;Z&%DE`WUm@lB3;VwAPphThEd}W?yk#|Rhi{X6%&6@>-{4JYz?IBY#zxX)IG*Z0xTg;DH(kRf_3WY_>NUx?=vSx-#6s`kYVfjVWK9! zs`SZ*6g1T)R&KPs1Oy*5UT?wu@S}oWy}ZgZ`YuLV4iD_?{Z+?T6>`M|tkq|A@Hnk^ z(GZfO!)VS1iY*b0f%?%WBx-gcOVgiu1tF1>9hox?^DZ@N*o2_`P@XfgIMJsl_$X1vHK3rI_ zvGIP!hz6M5H7373d#w6dTgbMX2WIUZV{r#-VU?%VG*Ibm-d2hLFMn`-zqSYYLGgYJVQ_n2h(Vp*+PS!$(D z(D0Iir}kv6GwT{5vm1mA?tc{zYs>rjd)b2kQR-#97Qfy2lhaDEU)ow^=bi86@wH^2kR(S#J9}DcvelvVjN)?9Rt_P|5%xIH<}pI? zd+%jGw(gN1gFks=Sk6GrR*oa2gnMpPf>EDyrx$+CxTHU>X_AeaFw?qaQ6`seN)`p9 z-g?}bnl?}FBE#p;lpnt2(f}`%V0!B{gGcX}Kot2rn*L5oy-|$_<7Zw*OQgeP-;!7 z_OYg!iXz*(=*@8k64mV^36^~3!Pz=K0kP$H8|ie3@piqLm)D>8RY%9PjGfzxlzW+B zrS|En10?MWM{D>Z&!*Td1FYG^ciSxt+I7mE0d}scD2s>(`q0k9?FH690MSKWfuc!; z4|Uhz>%ch@TmOO!TSvR>pm)Ab1LXYvTwh_btp$@B^-l(&e#{7wqle%c`jgYY;1L#t zOkq&a!lBK;q<`U{+W?s zM)>eQfGm7{QS@t8qg>k-KI9=RGH$78{h`~&PwB9?QnJ9|60TH9N6h4O9e^5^O>bd#G- zwy6T_*qW00+6?8un5%Da@jdPl`{RZctBq$QtP%r(H@Z`p(l^A=+vH+7)nA0(*c)>= zq<&DTe!8RnlEe7@WccR|qKq9Ug%!7U68*B>(oK#m-*}TsiLKk+=A>Xsw%>`f&wW2oLL6=_B{{Y`%OgHa|S^*=W zJhKUHlfng)c8y%I>8r~M+ko~s{AX=)m+_})XhzlB3X`w;L3mvQ9%LAUcDX(clB1Z% zCreW_gH-2hN5D;iqjoQXr5*feM)3z0)y(g$PVjHbHNk$vRUu;FwEL(AA96V3Bs`&* z(!+T@gVKTeGx_9S zZsFRgQkPrhNW+h|Dlf{*C4f}Lr3Wl8288^nGJLrmnv40qgYouPMfw z9!yshE<~Cd5Az{i*i$93%nTyd(?Us8xLc_^u0h)gjE?>7H3 zc-&qnpp)5Q1ogHMS(6iW3kFSuFl*LPC3STTh(9>XZG%VFwxUI$?efTBe7NsD^+b6z z)o1Hc$@u37W1oZ_tE2+&6qgR^6(ZbILNd2QDR&d}(!y{U5X_~%bhU@Gf~Zarbxip3hpB{CQfhS~UhD5Y6NkfF@S7HS#Z6#9y@pkZXwR`yUctnuH0 zDDxAP$mGpR$C!}v{%TtbvfKCU{%V9P-2>nBCSkAqxT~kE-bYAUC@2wb<)*?g_cKU( z7K{rN?`D&LF-mHTMtvP>k}I$E>UfG`Gg6P+UbhCyA!akR0z`>)EGSpz;!+kFo&|K{Y^_U>retJDJPK!7=y)q0%4PHwiEnQUcI1)dtqILNTu z=pyuIdq#LUgb@Q>1>qQR0RfgDYcPX-Ph*A=Mz7W%b|X1Xcgrh`-tLSTDdNfT!$S?4 zXVH&5QnP3oh%cc^Zzawd)A~gyYwQkgmR6TL7F2LjNj>v7bBf4>)-Byz*`!gkKLjoc zRQriDipKo8q&>AJyhW4sKCMRxH8XfYqmOF|t92GRJePx>@U^!`Gz^g{ir?thqT8sI zZHUCjY5t24-5Ae)KHRY0A zw%DXe+vRR%w$2rJCF_PI)*=gQ17fPTKu&pf3M?F-qY)QS>`lt%X>jG;A-Z<|{Ht}or6E+6pg66;Bolg0>VA$h)ICihV0Ieu8V(bcQMRo_QGC%wooMo<=Tcs@i*pDXs3&^_=488yp| z-R)HsB$g;KTaFmPWOTIchS7z6WZ$+ka_RP}cIAmS z(6y$~aa2CjitE7Y0K@y-V*m(1e##WwL((_4Jbm9f^>NAV z%!Ov_Ms`R1=jUVM>mh!-ZF733x4D|ttoEc6T*LroM6LI$a>j94hx2Pjt*{!tiz7ZZ z0JZ@;f?-bnO3;PgFu)tiGu8LlcnQ}?nZ=T1B|z_v|K?n;iOD42Wq42y(QbVRU9wZR zvo8sdO>y}6sh*O;01Ar26m2v##b|N+ON>bw*nX?+^NI8%4_~`IHwxi<`{KJB8RNJ} z!OQEbKYK7cfLS9^58UUjTgjEZ6!asvB%`{3!CyDJ^#3Y=81`R8mP~#&u}^1bbDlW)Y#J@bqP{ zPM>=#%^z^OJXFlMA?to4*?*V~QuvSu%CfhQZq}rqX_EM^GJ_AEH{yCW+3h*yd|(zC z8%#FMl1P&4{h<^@J8bSb1Wjug+5+Qe;4{X;O=m9(>u{U2Bg**(;o3hlPYj<20o+Ie z9$&G!4CaZSkL*oOdL=4$y~WA6{yUpoyCG#$F9gw&sAy>Ckw4hL;5>G`Ea&~k%&(#- z>EQ<@f-_Xvk1HmL&>5s{bW%uj9(hbIv5HH`@#&p%6vkgHTrN5IlID!*Kb6?uzXQ$j z-^w{un3ueN{vp-W4kiQoJ{crXHgEhqVS<_s8hV3X)70daEzi?mK4Ke$7g8TcniV~n zAEpGqkRFOV8nozX68`F!HxO|6fzS4t{|rtUwXM9S(CY{{nQ+-K2-&p2X`v=6xfnOB zPCqbS|K+MUuO?oop!*x;>}rd?3$vt z>l$*br7501Isd8{PMnV+^Kwj*O$ zGKOlH>2BZDq14X}myoqgd&+?ym3yf^6%>62C~Cb^VFQY<`^P50)q*J~on@);o>6TL z@izg94H@o8J%WQniAKe6Sm5w@R1fUPb$;fjy?PNG?C)ZB- zLUS&5V=IIBQLZuHCT^9Aj8{g1xb`lGsV!n7R6in2xWYVM5tB-hiQav^Co}cS-iRr+ zEnfJPrRHBYrIkgy!g{uAL-XmmO>`4ri?$1}HR44EzFEmCW$bFVBnp?Vr}$k(z~w^F zZTXc#FjWi2l(x>isO*>v_DNr&ndQQsX9!R=x|yL@)*-nM)MLZeI8J=frNbc<4%k5 zEoum>tw%6UhsL&tYHl|}o=E@NNcegYELk4-^4>`ksk67YsTbg=y--LoTI^UeZ}Rbq zUW>WZVp3YBieJn`a7>Es?~B`)f0iR=-SY#!{R2d+c9JyUY7|;00z%y%WKPH`tz42!|+c!NfKE0AR zo6Hlh%SVjR+{}rn8W$>MZ+*X5|D08xvoE^#2{6pMKh9j<_zwWhFh0HYDTN0 zp3`9;?t*y+b2K2AnW7n2$R`TI6V(|6=7)&vzt7$IA)Q-SE}8urBRO-sp3X1Pjbl)1 zFKN==^8GzoP^j^`S92EVyAA*WGc_pjGrxGrlS6AM8_{;?yxG7L<#29q?Kh~N!|JcX z_8>V)Azz9FRlfJXxq^Lly>JSCOJw?7s#9_o?i#(yl`(B`uxB;-u=C7 z;7n}-DJN5%FgN;yv&4H)3FJRJ*_+$!vy4O_Vv^s_!T;(jv8sh;;?S=Mc#xKo_1+a&4|Wf1}pxD@pNcVvm{(cuqV7v z2uzaT(;S)9o8u(%r#;y><7s}D)3OwD=lr*7QUXWM-kS^*WJr%5JO zCiR@cKOc2gX?}gUTa8PXr6lKe@)2VFQwb_pV2_dPI#`T&`IS6BY(})PyoYxrS^@x( z+RW}2dt~PUCSX{!$aL*WE0C$aR7p#Vk1~0%#eVAfbbKuj0`l9Nj|vP|I0oxP)Pjdgs#dO9IWAW!xa&mRcbvB zY((lExz>EOJqK_c74#L)nZ;-kRKdT5;QO49W{4vmGTN_D$m#GTI1!; z7k@>j$0?pEF{F(n^`wo0IlFm3^@6tzLQ8H*+Z(3tCtb1F?OZeF(3{PjPcf>2UVi+n z=x$(2m)$MOjEA8Hp&_16Fnc0Ka+a|_Zv1CTAX2^NH`#Zj`aeLFLwJ|M#h`JcL(xlH zmDdl$-59_@A7!G6Fe-OUOMIy$KDad?INRbH6IX6xl71_Q9-q4XYryx_k4tyvGkxf1 zOAp=iYl9-cM;p71A3R_n?p{)P896##2l&fhwFbGOG6_A{m+s%$jwe*gIi{aFJgMJN zB*RO%7y~hk#oFPdAM4`yZq<69+MkZvQ}Uz)oUIXKE`LAU)E<`@3q(uLeW30GX#*CX z@82D4_CgIod7B*L@ArOV!oQl}*ePO9ZYx!HC}?HvPg z4i0VxI3hNS&#U|rp~nn*#T;rD6UI_ zQtU3`rk#nFb*K7l3rfX(vZgt)@DE}`A$qjHk~NS=n84F#kC29T7I3GQq>|@pUQ)f5 z!>!@b7V7E6oE=JWO})6i=wd~8&x{pNVBM5K$j<=&pEKW3;x&Avy69}%iBOXe`)i6N zjk8)6?ZRxR#J*{0GDj`}v!e_04;nTpM;=6X2eCD2*XsKRAZq{x^2y}5cw>S30D#AY z`dC9;IH!07gCax7&>!Q6g%8QKC&shaW1%73hAwDPK3Vi<1`Y`{!M7|lx)BJ@PbD~e zvC8Ild*Ja4BI^tap|Ft-dD`GyW$twUNKSC^*d&M1nBgoS->%)x+-$jL$(sJv!2`!8 zi$;bvP#^ZA1TI}rQv2p&lebIUh4&nbJ0&j8u6V*{bwFZCD(Oo63R>A$FtaL(A#s_I zcDeD{j=v;{rQlM!2Afh^P+3bsRT9~Uaq7SVSZ}Syq@J)!{`%#CMj&UBOhXn+Di)9M zpz(EY4-@BOQ=Y6Le4GsP>LScBJKS(3R@2d)((YX@)dqc;IVRb;v?295F!IDg3R9~t znOMXBI_qLEBoi6>ILOK65?nf{&?Nz*pFJ5U@-t?6K{z1Wd0(|gruirzl<^1Yr4Ae zG}Cq*W$*ykdVWWFm-fzQCcKu7EcG;h+Rzj@n{3CmdyK!+?}B7vOI9ACz-N@&tZ#^w zTYPQ*LelHxKD5-_VEC3wa&;-6oBi z-r)D#bN#44)vVo2r1xXGRDRDV;Ju~r4A?rFX82eL@MxUPorQtiTx9bfAl@J#rku3A z%FhF&T*2?%qy76?i%R{HUfFy^*X)!q;(%`r*SuDrpJq?hk#5xGptCh*g{Wsz@pF zt+3Y={Dd!dEOq9fn#-Q)q23?CVU6hKkdeT^QN5F***U>V%OqhJtAPA1`>k|8n)T#w zm3}|31^TS??zUO@Z;A8I@H=6zx*7erqcR9myUf^5m;;h$kzE4 zbc{K|;_=Qb{hK^h!`&nb1;)Nax+Bjpc{XtVDVaq8Y6)y!b3Sd$2baE4Q~MHU{ixC# z*Q04j=03A%W5Y+}dDbHK8STSRSrKeANqWOz{{VJc^T_@VNz~iy-W29n4tTjO#Ep$t zb;dE7KvjJXBO4qT(ILSAwkW=iacD#Np~GqQ26fCykjaJy-p8q`xV6xmPQcaFb@FW zOkNrbyruH@^{ote<5uZYsitZ}7sH*fnOsgoG&$`QE~d7?wflYd z_oBZ3Byi>O8Zw~4zFdi3JF}>~&}D9r>%BH~Mrq^>%JUoCIvxc1ozeq-ZK}zWsjoc0 zho4j^T;$oa(F0-?H*h*gd=XF4#wz@@3eVMg#z3@1`Y^G42 zleK-%l5qV@o2RuiL$>xB>$M7?Q5KE};X?g29~;JbkY%QN2;VuCOm&b9`!k~xjz_qR4H0yQ-5NjKS<*LH4PP&N@xqxC>zHk**s=H6}*>2h@4&A zKkHz*7OMw@wrrD4*6ho2&ZJ%b1K69|b)GCy=@!#Dz2;m|G5fJ{t5&%``XuCnffyWm zgG_Y&(U!dAH~wC7PNv++H=9OGjEqx4YPk?${N93`nRZOJ`DR8!bSzge8alY5MpU#_ z+06yFa`cURMI_I1e&}!m`Q|TeU8aSZ<(v7{`jo4Ae}GJS`kYp`K0`8;z+7tOS;e>( z!n~OHN>ubgtTs=`JoTVN&k5pBUOm=f`_5!^9Li>%Dbn}e=%wu-k^K1isGMb$<8ytl zOFPD`P71}L?tI|>&2ICGm|icg5k6O0fSw$)qtnx`v)Wj_>B@2&x-Xp~rYbCzhaNV0 zBy_!=A$AA8wyb(g+s5m#cN4tK`$9yI?Rn*6}Oa%CM$-m{QpW zgYgs84~rI8Art#XFF7e~y(4gN&VZrc+08VrNn~muWM;zxU%w!%hIRO4*rw9xm+?p#Fd zKvd z{ZbI$sa@=$H2-0TUjJ(LZg3gujb&>?Vb5>fEQxUd$yb0Z@99Mn-|4x;bt~PKKNeP6H8_B~UOyOmIEyg<$wFFkP#6&Rl3ZNU&VRsl?1*`Op;hxi%wAct|#)DX=qz zE+)FBzt&6{wYsRBV&`c`v&zn>vfSmvST~n&n?k)Y<{2AR{;I0Z#P~pK`{HlOxD;*p z!1|6Koa0Ka^`7u14Taym%802hTDE86pJbVK=C}4vUffsXGz>n(mY{i0aEwP1Wc~MT zXg5eiDscfW7j8WFSrVFAvy4R<3rM3Z$ujrDIv+<7?cOtmto;K#mDVBFcI#pb4XRS~XyV`icz%9h zAj0zv^1?P{*)}BWd*3?HLsiPA%}z>louFf5S3d%B-i;Pe?uU?-qpz5AZ&JDi;+e_3 zyUYBLJ=wXRqU-2?(b#VfzE9_VzHSpd88D~kupyOMPtx3>ck$o}@AT(xnKvxr0^TgX zy?DH0yi!4(RS$jtnoJ=L8F^&xTzlWQhA+S>$Q8v^$4|;x*O5jj!GP<51P2D>{ypSM zSn%4+;)(ita_P+BEaJKne4%oCDOG8GI*G|JID%a>XLm?VLY*JErsgv#KCr#gb5wNX zqLXA?vz^Kzoc{pbf^DpuU-X%Y{1CcRr5@ee)6T`sNy;o(aea0&nkvVVbIDHs0H;bR z;MrOcMC?5XV|*sQ`H_5-WLv0?AsXw0o8KS{daIlZ>1OKLOVgsA-uJw5(pcuG&ZV8c zEa$90$~~ICy(QSmS`CGB69~&v8I*Ub#x(zBzMm+>K9u{}=6!mP_{-)qRWnZ}yP$G^ z4fHY^WU_(=eyk6oT2~v@Wv`ca^OrcW&{88;Y?IhmQSp~55S)%d42b=RR)0aufOS1* zwmy8!^$4{6WR3Cf=C|3PM~tN%695{fLWE2wPj)vox044ieNt_5fs^u>kiw2zR(Ys1 z=&jwZ@1YE)th=V4H-_7$8z+JPQE@c^SgpZ?Y0% zD0~M3@!{P%*o6S4U(Y>=j11#(bK`G8{?@_qOt?^}RQ>D#?X}<4nR8B(^r_ zJ&L4o{!ng%Io+PGe5%Mc<}HhA83#u=M~rYfo(j_zUQ>0u`R;D-sBOzxHdaiR0hjo7 zyB{=9=QJh$qz%h@!`V~0wPDsvF^2Zgb=@Aynl#8WvM!W**vXAZ_rq@Kj1C0^>7c}V zo>@Fv7<`_f!qP#ZUiBs{9gj8)Fo1<$hqKjmZ~WWT!P>F&MD1LbY~dHVxoOL><^5Ql zV{U9>dE3)$>ZgzFdv2s`hZd$4`{Rg8!c@3Y3AWQx%I|4!$+P<*qH^TYYNbmR`ftBa z&clwa->~>6D-uW5Iai5-!_j?uD$zz*$f)nx5VE0_qfv&gXBAG%b zq0++CZn`WSW3mqap3|nZ8LUsAS2>cUkSba!^@=1Ndnciio!iR)APTy{IXQKjSrlCK} z_TQPOMMjK-Meh;fqzi)~@V!+ahH-15Eif2`zv&xy@$0Xe$rfh{#Zq0?GG5O>>6d7P z1BX0&80lN_y(8D0R@HYd<2qYTcc7kChK}aesl%eCnVsBF(X+=B!I{ZaKG+1;ZnBo1 zshWz45_vGfg4a7xm*Ff$Hm`aLl`Te%0I_6gD&WDK@m$O<#nQO+d4-S*1?QQ3qNP-8 zaydjwPlLIaiL>LU|4|z#l--@xzwD`Ny$BArQCv@$GETbW+{rEI9`>h8>42hb2jwZ} z-#Ot^KV#I|uI0-a10;)emXd8-lp7cnF2)f30q8#_=w3I_vtG~hbW!Z?Izm_QdM9Fq zyc{ZvM-1Kl_5d}mV@b&PAx*LtS6Y_x)25nXf4(2M(A|zy>qqV0tZP96MUe_OeM!B8T{-k%dU$=^^ zj0ct!B<#rd2n0A#zJT zaTWahmN6hiruwI;87Fmqz;CEFg?MQx%E-(U(tds0UO_un9t~$%+KvmK!(rk%iiveg z$7+b|9+NXQ_`0;TWNIb&ctEB_2QS5u(Zqfr;BRQ!yVbJk{pKIk>J#jyF5vvIfZS<1 z;m?1+FgD$LcRG{}eivQUDmQM)5@ZsMOpCmk_#L2on#2~`0~*Gy`2Gmu^rZWt-cxEefXiMN3@)cq4i&)_2=qYIc#+hgQCGs7C4q~mp?;EaQ`*Jq*G-rJ&viMw%woPYZ3 z=(X?{hu9D-5a-neF}x3<$MA~$wiBz2Drhh#C zRqZl(Cq&(YYKI*T(TcBg;Efj?TCb-;%||1#+4Y#7DX7mWgn|g;7|(B|aysNPO*sNH z)1VdfP1k^Q7bHETeEO(0QN9{@UQT|)89Z~a4ASH|s?}3Iayn$B{K`S1zKL&zei_Nz zr^E-Q+9~pw-X%ZT^Y(wWnh5W%?8K5j2_$Fviuxx=_$6l*nuN6g0iFd^{h##f71LoX z6dj=QN2Pm(j8=vb5LL!2+oM7X9;XAF=Fsr%ci_T>#k?dEeJe-99|+doASH_=^~kSb zpGCH~P(p%1sjW09q5+ivtp1}-ROU$A>oer#Z$r(lJ`q^U3Ega@k?GBO@5G;i7LwV@ zKE?>zdZvG+eJL%JY1IZXgP%&~Ja=tkH1`rJ{nMVDR`a0;8(79O_pW|({88{E-V)U% z0yE^XJ9-Qn^POHrLcvcy=b*2oKWYeUpjn+&a0#!PbwvcR+;P0M@Snx7`KElmo%)E7g1h;+Sl#lo#g&40=`- zV&xNDJk>gvXK&-JZrVF$D0$jQ72$p;)~CL|WZrNI2OM$!b6wBG4-`dl1DK`cjAJL( zyvF6-=4nXkbDvsNY4RcBD*IVny`<`zj+-X^I{H{xup~z0ZpTcHhv;k3{vCW|@kWJv z5!U24(83SduogKj*pPCp7bQjy%1$y#1xY0NZIp#)!yE(Jx~&~ch1z`xj3x}_Mc&GXO`=6mdvv2KAb*LiKX4j3$M%c1USQfw>h9D9- z9Y#lb`k%so6g2DQj@MkYK3t#$kc_Y3M;-ba{B`i}h;<(h-bH<5bkR*2%yK7GIL1f5 z3Ff|n_(l6}c(cRCJv+qnCDcWZ$H-4DcITbVf^rDY9k{Q>xQD~cc2?bd73$Kp?e>?d zS~YH}-L2oquY#+GSBJeH#ebRlT-r61Ln&bH%1v()19b!$k9vEy)GLOOj7c`uDW;GRDUyk&lG zyd!ZBg>Kw5l1&A-nQtxt;KjXGGDjtlh53H(B%fOzDDd=B28&c!qihtx1bdO6TGPAM z{5x}Cvfb)SBn3)O&mL0(vT#4xX8dto5zVuVs|`8j_2mgAb!Vm5c>FJ}jWbGEsUq;7M2)qp?DHw7`?+RL~hz2BmxxV5PimRTuryaaisXsWz>EscuLv= zawIyOlPEa#hBxEikf`B|g)pUCV1SkOm$%Do#BP0R(Y^J!=n8 z(KO5W+5XId$jZLa&!;1&<6o@cJXpZIJY9SnlWTa+%GO;RW#;$mwwnHj9v{rAQBzf+ zCiPZLD?ghtV)$#JU3nH(zaIV@$t}aGA|Df#JC986h;8@IF^%OH>A0+_E>9oQz3=0+F9k z1`oK!XJ{TA(&e1XELo075=4+OCE_Xx1RUUlw3F0__av&`C{Tr1&kKjqZl?LK^N%Ow z&pNJ4pSp^l?&6B!gx#Je}(PsL&2t_6Cp8_k)vh*0A!HGi0{eo+W=L)TflFn z-kn~y*ty=5b?mJy$P}RIUtX!FHo>!7U?j*rr!~(>0##ij?_!6jP*1a}0FQ1P$C6nHfWd>^Q-)vU-GCP_gaa|4hF;~WE= zo^y`Eyaz+@7scIsQ=d}M{9*9=_e+@^tdnVht{^0ks>>$evIBxZB=rCsfPI6*nmcHo z7x6cYd{wB)JbGn?)R7yAQZ{K!Mk{#ss8s~9&HxxJymLlvjgDJ^=XU08CFa|*{LaiZ zI+bh2EpPaz|`a{7YeZ;|nGGb z#L5;ar6xWG0opJL86Ytk$9{20FRgyV99L5!Tfi{#6N4cG7#Zw4pTn+drSFa9dnrzb zWX}w7kY*!nNq|mSjsPB{n)-~N89W|iMz?0Jyq^C6f%8r9H$pbjR_>D@mr(&Ev=gFozLf7k0-@f!q&t>a*axM zakcF&?!T=%8rt;Qo{uCic?+ypF^O#?kZm%=%C3*MB&aCpz#hyi8hiOwC1LZO&_$lPb-QC>N$_&)1M z)Lt2E=2(_zSR|G*42lji4gfr3r>%Nf2N&V7c(!3i_OD3=GxZS_AX8iA^tbN8& zhldj9}nI62711!!C8f-nhQfM&CNU2N@aM?YLw*r&}K zoz5;Tqr|==TsUSYJxRxd_ygSfo1!udLGoTXwnlJsVDqtpYV(7lYGl++>W3cHy=~1 zu=s;-<%CG^$BgvH6|JW0Z+$AZ;e+k%n)BGS>)k$87ciW0$r!GeN4XK;atG!9d;WDY zM@AlPO&#U6^6UVgetT8hO-&haNaLxlK{ZW;65YpRR&DjHU;;Sp>CgWFUa?!A^lHUN zQ?ZWzCCS_mx$ZqGwyl2|w=54noyBwi0NaFs1f56YP~B?#;b2=Fae>qPKl;>J+T>2O z8nw*bPWdhGjM2Bu4^006^{c!1U-2^6O<^6xY(YE$)p64q>;C}PuPW82cbS-a{&hcw zbq0d$D`W%E=luTwpRH*bxaxURglb9IpHpeS7rZ{O2V&dt!ue+){c%uD@kc|I%#xl@ zAQUJ1SB&Y}*^NLl1{`{E{(r)+F0!+})FUU<{*}6vWMS<2C8_FCd{EI4B!`BS=!~1x`BXElfk6v>vYW%rE4>^e~rEn4Ug?H z*Sfc_(y!WlVDP-C3hEip*B{Ef4^h$XB9y#G803s~sqAj1g#t*Y^Q>xfMe1onojJ>M z=oo%1cx;^gyaWFC1Kibx_?_Xhq>IG=0KW14Yr$>xB~mipz3STPfDOU$IPcf#?NfJS zrdWDi&vB39hlci&Z&N)-&yFfsekFKrVY}_hSHJH+`t{*2tSMY&eQFnlCdSvxJ;F`jVR2&i)1ab3m{{ZTxQf)}AJqS)WZ2AZO7C(oWg11qSLF6#O{b-lR z+5>VflNsc1ALCvZZ>j(W<{h!xrfpU=0Ouneu~E*=-X4i-Gq5KQ=4ZwBL)md5mW0X9Jwq z$UYUb)vbV7u1HWuIXrt;T9-QgtYL-Ya0fzbt~rXPl4cJCN^{dK zT~>qQ`+L~H5`&(&HSrXG5d2M}&K2Fr-PrnnI`mJ6e;sF7Om`BnVbnKX4R%t;#%)~2 zrV3Gw&!=_&02JF<+8IQgW1efpJ}>-j)3nPuQSVi74szqtxc>l%zqB+uew8$m-T=E! zM&Vx~{7Lw|ulV;@kejz#P~;?P#BS%QtFMZTW1-0!QoW9c<3Gh2JaMWurG$l|4ofg3 z=Le7Rt|C7V-Ccam8=s)-?_Dp$j|h!oMUr(5o^#31QrYf5#R^HvVps2%sIw7Q-V!p7S301;l+DLhQ5YWtvh`kmh2!I8SD>F@UEiE z!$Mni1vx!RjMHzG8PlTX&K4~mec_@`eT8mmHdl}VnA~81cs&0ApU%786TnF#hA;vT zU@N1K;rwZ80^4(p^#1_sR#LARvqgoRp2wW6lT3qp^N#+%j~%Kzg}Ap&FJ8wdr&{hc z9|<+OGYIj4$II9xg+te+d}xi;;lN`J3D*Vf;ClH&7H)N8B}9A3fnSq*Kb<$x#dVDJ4Od#*FRrM zl{E)gw;rRmde@te$ugYN{pyOJowe<{)a0v#!`A*JQ$B<6&+Vb(%@0o+Md-WKVNgEJ z=4zf)kz+h>5jICo3jK4^y)VPRwhfnwtPYib;w^hjyhxa}#+__0EuAt3Ss!CBQII>>;m;1@DO(JkDJ$xm zl76a9?d0RWy$)~eN)^?qw;#vz{{Vn`kH>F_SDqpk-xz#7uC3FfMPuR%xL{--X+#SZ z!^XswS)))HPC$}2!6d1r@DIe#5BTOE?2E9L+K5cOscfu+AATZ!@IIYKzB5>!2=Lai zV{s*pqv78xLGfEjVRLdV!GYdZP^FoE0&Z*)@zApHVHjt{0amfG!fB*yk0Iys6o~1Ue zsf|fBC?U7|5}mzlLyK32Y~dGn_mc zy}iQc{sa`@{uTL!Y2$G?nbfON@om{zw{F_M(EI9md`=FMrA83tR^98e*GqQabLmfq zzqBvFuM&7NKZoBLB>R2MfOOM!Ed{>PGQ}z!k^l=EZXolxg(MtffKLH?@VnvuuLN^j-`HC@&P}EAI_I_l zWp2leX0$Z#4EWPRw^Xq4MB0sFj1k zB;;;oT&T`L$S1xlw9&t4{{RvG&en8*&S7BD|~O{{Y255%`)bZ9VRrQORaZ@x}Ir(!7)ie{}Y!LmUzSAOblZ$E{*(zB2fAlW(e7K&0n}BRzd9 zjM4PZk9w{6dwn_d`*HrVOUcLG9Gs%CBoW6-?5ynk3*ZZLsOtX!*>K&SS>yR|an3SJ zAE9DBvTO97Huk+b9>Ebn%B?tG`v7DWtM8?hKC1cT{UW58Z964`3^mYO>V z9Yy{04JtgPNMjsvM$Ut91Ovx$fsA;5t@}XuPR%#PcXsGo0z{2~J$M9>$Kh49zm8rP z)Nhq7CceG7lj8}ot|fjgd;9Qsw-N1OHH4xwDVh@m-aUcEW%iY zjJCJsjk#BS@An>)sr)LrOASj_@aKrM?M~ZEfo8gB1hC2FmQ~&sSl4h2z#I~KR|Dcp zWov$Ux%x3uAD9+5ubEH>s?7iu8T|Y6Z;+DP=Uv?EeCX!2he{4umxUDn$e2Olpr<~oIm((hW;M@GQy)Bgb1 zUrl(<(%MK_Q9;N!;=Ci`CV=r?wDAsi9!ICWX+p2Hj;0uje#&|ryI)~;$m1s;#oC3x zXolg)9N-?c3Td*tZgGtC2DY@%1nSz1Alg~GWE^35HN0VS309ijlvh<^I5{VSNcx!N zer%o&2V?xJWxgADngu3JDewb1(Djhe$TFw3N7Iyg@<0Av#o|&j|Dm5C^+OgDe z&UpilY0a)Y7a;-U`}OCK(ydwe1H=&VC85Y1VSyh*gG|(X3*tdIku>H#e~4qZy%B&q z+LvtnP*eqT*YN!+@Ambr%tR6n?ttSpf_xp~RT(;apQ8`)p?o#sbsTB35&$Ie&+F-1 zDLCk4=XZ2pxm3B%Kse}6HB#eDgoCpHjDiUEsqHk+5X8Y|)1)IfB!ixQl{;ygz0LyZ z(&KyaQHpM9gdW_>WMtuV6 zB2-4%9g5UVqF<0o+d0VOs3-YUsM}&)No>m+$BHGphv#xnZhL=|RUp*aww>e>F~O=j zhLx#8;Z3BxdaX9r%TQtof${DMsFYiJl^U)qLtQnEqYTHV=kNULL#zTgWB3}WCX;Z; zujLRvovDLNxKV){WOXC8K2Z9^ueoY%S!6;tbsn8R$E9Ul-KPF~_2ac#8f@TgDtdRQ zF7&xdZYX`vy&R~YYTn`{@KJzAxw#MD2aD|A*Pd{43(L618e=3!aspkT>lSpk-Q&riJPr2{Y{QFm{ zd_C|jlE%B>8RI$i#ct{)8@73#%2Jh(&xJl2HZer9!a>OE{(bA&EOh&Gr;MoM&{kH3 z;j2g_MT#&wbgrc7rWgt;k~Z=W)3tFsvwEI+<0v3iV5&LDuQu^lmTk{H z2dTw(zY{e$>_mv?ZZp)^l4{-|TaCzHoRN;FHPJ~m9Z_$t!nrEpx94iVJOrY4N$FZX9EKFs zqggl&-2VWa*2?`)K53qd@Y_Q2(I#Z%@;J?U_JM5)v=O=u*zN`maNZ1oB8onp0T>|m z@925wxUT;IPl7^YXqXUv_^v!eQ{3;tR8dVcOG)u|zu>odG&`(D=Q3MKmWj>?3{>&B zXNDY(#8(UB4-0%H@sYUIFWLM#sc>YL$8?cgG0x&*$IH+km4{yDyIoSzlF!VPbF_BP z{{X9{c_+m9ZevqCFS8lx-`D*A0Q#%I%y>&G&QqOexXbyX?DxN#eRofD&&=rI@$Z(R z`L3U*;Bj6e(jd^T#kH2Km+>@#BWq}6&nE*QsK;J3s09*@m=(FaZk3s2K!sUK9P3{snwt@fMk+{3`Khh_s8zE~Xldo%N(5NbIA9r8iNS zBUNHb`F>L|ZOlGmGHYgU+NyUVo&J7ewz964ja4qC9%`Dd9+2UYiIf%pxfsn%_ z?#KjGvH10^_-6LN_$%S}hqR40+DnNpG~`+4OaA~qBNU4}%Ogn1<&vRQlrC~Ia7XC; zFOhL_$$9DOerR*t=q`_KZ@}b>F2c}h<@_7H{7DEQ6skHvA^ zXt3VL43foc+Qyv?#@UFL^>YfkyU*X*@4G&ft zjB#5@Bgt_v+mAANWN8Xu3?h)DB#=o2LEuk`{weUEid)9IN?NtVk14o@Sqn!pNJ(^H zr#L4BV1ft-ITg*sxepLg%ibdyH_X>8Zq}al?Cs>Yzf;uUn5@SgT80(cT#c_^cm5dq zRQ|-jv;P2%J|JA;4l#`NhN}k z%*Q0Xr+xKx4;Ag!Buj8T(;sjb>M(8)Hm$BNrcABFsDkK+NU{29`)sCF3a-OFEoBp zlT)LLs~U3A$=d7I-8Vc>QvI2=JwL?LMXUJY!fieo&z#oq%#*g$f}t5U2IC)ijPuV* z^>2i~8a@g9B-U+o7<@RM9BBuTFJy>G<__jdj0}^>IUsfb007I@FX3fm4Z-L-^Xc2) zpU`&TJWb)-ZEfU^@CS|fD#L<*&N_NmwI2>Lj9X5%YL{MdO8xE1et|9?!ctd+rTBl5 z^sk6MHh7=I5?lC_!+#PqKN4u>;IcNCa2o0u_Z+LHK43a!eL7&*v-~&xpgcM86G(-j zzKxuq-T?!gdVfm%@X=q!9}KR3-Qdl4VtHmaw7i99iPJgScLGP$9xJ2Lzi2y6H&RVf z-$uF6wFCvPJc$$RQxM<4S+W}h91+;`>t7YbTmhM7N!7ys@|F60Nq>25<#oI0k3PO% zTL~vP>mSGFeckah;a|gFjeifXJ{)-e078N6CjS7BJKK0sZY(6~B#H5rJ&5Q>1ad+A z#r>wgWxt6(voDO=!^IX4E{}I}FZLd@8!3+RB|H<1?R*eQkO=L?eXa0|_N?&6v1cBU zt!oPv#{d|vP@z6bH%g>SCtw~9RLaVhyoA%RCjlpu_T z3<&oIxp-5J^DZ96a$;MlMZbABcD&J;pB`AE% zFg${JKj-{w*t|2ZTMIDDD99eQ(tL6M0D@(B-@!M{;%|Uga%);CW@)W1rvCs-w8*(^lb_+6~t6}D)7u6gUB^kv^ z4(Rm#cj3;39-0_is3mipf1KBbd|U8Uv>Tq?04Ojq*k}I$u4~$L&x(2!+E@v63EPfH zuOac5iS2c}vuO)wE&XfJtm7CqXGuBDBa6{I1h=F+rg-4jQ>S<;EifxX4@?dRO6WW< z;p8^z+lF?EyQnOhdK@5bIp-XESC1T5HA^16@O-qnmgE{QhvK~fOlzL}S6`xhG1D)Q z3F8_11JC1JCYR$WEHBIqgeG&GjN|gJOYo15tYwXS$#H?yu&-8(6IVQ%ji&BqYySWY zw0juKMGBG!KOn9y_eQX_xiGvcc{sv~?|f%=!;w6R-2*>!>ig>5v9kd#&s@q6H zVF9I+lyZ?sKFWU>0Fn9zA0(f z8x*-xaezl(YV>KmZKvF6M{KK|xxpUaO5Q0rlg-NYJZHuK01YfGFFfes7zYe7$N9xB zi{V{6ZsBT#bR&_+<@Bu|h<-7%gLH;P;GO}j9}oOP(xX6Tmuc#LUY^yAtrlw+&W$Zg z!CGVF{gpTzasL3;s&;-5(;57~vo3RhK>q;iRm+cvdK`#%IVagq^2JuN@z#a*)jnK-_XbKHo~Sd+LpkTe)s{8UD5D+GX==+HaY-g5x=_A)CXJ*#U4Zm=(bw174T# zzg3F*$XYYu)C_~c{{TPdwQ(rIqw_jBr%fDh#d{0QG5q4>jO3o!?OYUEt2~=av6_{cGC1 z4|8s|$~Xi8o^#K7@J|n3n+t_G$Oi_!FTwsVf*qzwc;k)L&slRu$n2p}DQbIeiFs-y zS8NPox=Ai=#gk_k#xY(+2ac@OGaTP{1oQs@)~nW^6s=~3pptQs#tG-GJPcPoq=OkG zqjYoM9c_{0mgWLT!8rUqE9b`4rSq9(Cp(V>V!f;4Uy0$ridYfDZU=4%@5Ok<#+M>Y zOM&weRYnduBl(YS)K{TdK{aE~grxT?T-_3-gLds%SCUTevu>{l`uf)?bE5>Dl!#+IV~q1t%%!l>nr2j2UoSDY zeLB=y%DWOQ~fevwub*G;UxgZ!pq2Q3`I^}6ejWJg3z(vY zN5k|WkHZz~T0XxVR_iEmcm}?3k4nGOW)~J@p;M3sde?KH_}cOdHHzXw8An3Lt|}^1 zRShW4RJTX6>wYV>n@!8eG6ziiSCD*7@r|3>yfI0*o&e7^nc|;_7mWwm?cXKF0R-1C z`wsyxqmNQ+mcxXFMNtTb!!cOxkkQYjyK1=syrd5l07% zt>d+4J7d3(BHmo|$vH5$vH4Ftj@^Z^iO}QbQZw`iq55NxdLG8P4GZkkS

+*(@=Z zBtoAtU`9B|0FQ3I+>F`N{5N%EM7{*^OmM@<%nyYo0LHbQ`$f*6+0Gre!1Snrt#W?~nF+u6<5GrorO>01?QB^7F#R%Goek z?{9!|Gr$U|kWZ#_U#{?mI}WlsP}b`G`?K+GIi)I`OIm4JJ6rBYSZ1^q?>zI~`9Ngm z_Cb^A03+J1==x30rD(I^Z6Y}<n&SIf@ubyY%Z=5_O#6)k(Ip~OzmF1jz?S? z$=3EmRrU~Ur%9&I21P^|_>dbn5zO${Xhya4wNL-FS zzs|Vdjea4v)BG8IJ-Y>oZsSDf0GxyPACP~MU#;SDsNS2K=KbX-_f9|Nei@Tuo6Tq_ zt9#4yDc*RVJDEbn*eBno6@N(6;?%a2AUVc4$K_r*VetwhI7Xav-~2tR(mVy?l$`?o zpy7x(&lwrV9=w|RiuBYYc%$~LHBNc^$3@~V30uW#%-G3Zn6Df0{nVNj(URIm$j;zK z06g)>QT=~2?K;M+w>HNzgM~eF$F+Fx#EEULZp;dsnT`|=NhgeR(y_xgdo!mHq04iY zw)oYl**@F57_QX0W_cuGAUMVqyK*zg^{+_yN&8~z{uO5O^$)SEWqi198KvGbNdb{Y zF~?pqMR`E*<)rHGBqtUkGOKahaMLfm9AKm`|>w4FPns7%AX57-L zOO|i#MYQ+rtbDbL)8u^hY`+asPEl5ub^TlLKSIA}@7njm{x83@veoq&E##Uo@J}>* zcVuK_`Vo)In)*)DR=c(k-dSGD51s}ljc^zrR~>!5`;6D*SB2)f@aglkso5lX7tM{~ zQl(d?Cy~h`jAOB{sQ&O(|T9bU~Nq4vAkGB%rQ@C}X`%dAVTZW!PAq1Yp zog~T4}fE?K)+h zwxE{_6f6clb!Pr8-z#B&#(P)M-a7C-rkA8#UTRu%g9!pi#6U1P0~k2L$*-FHSMb+W z@mGqS?xvCLmUsIsZ5o2ANWeJjz#Y#@{e!~npNK2s-X@Nib#*)ScF`>!skpbconFQ` zOiZU&-CpW?`RTXcW6aNtwemK$uQ3GmE!!2b;XP&EVI|GUE>BWx*EAp5b56a1UTZA( zB!kUqTYk&hWvsj7wAs!=larsWIsB{kikM38-DviyR;L+U%(Wa!zxMX1YlJ1-_qbT_HU<9l!e3$au5D8jYpz+3g?^sa>qAoDw+CB;cO-HO=tkKs(0wxyag zxrfYfO>!F7!F!u+MENC`amO9H_O6*ytwGtIW1MEjT==(BzK_bD_^>@eKHVz<{@M$Y zU9d^&bDHY({{V%z$aXn%+me4u(}&>PmjGp04EpA!%bRgDk>q!}TZV;DSswW)XEi_LQ2z{*iqjia2Na&Ryy@ca|J zx#x2bPZ*^6lDZw^)a12a6Y9FGo^9>fC(|dIj?-J392RnM(BywA?qA>zrW7(r_`%@y z2BH4|g#)TE+OFrGa6ixaH8zvn6yl6cKVC^VcJ{~MfBk=jUlT(q+>Qn?LGM>v;C`*A z$L2JWbjZhg&WFK%AddI!sIC_{2M0e&mYUrc3Z7=IkBW7jHX^o{SpJy)RZm*+zl*M| z+7`SePhVVmb*+mZh29`wyTf~N+t#fA0K%%)T;-!~zo(^A=SsoNO`cV#t?ZK-r)FYK zNj<8Qzg&a$uTjbq~g}2()twiy*e4~1hR5V zAK{{`TX=g~wu{P)F&W6@(NUmW_IWg0Yn>m?FfQ%6$QE94z{>xY!pOgd5aQ-OqG?Hs^y`$p|lY`gRxN!968R~c9XHHPlbJ{fzF8fU_E95et zdhqWVSS6OXA;9F42=wN?7SF}6E|Q4M2gnCrhaXDwpNLo9b@E9b4o(gaLtToF!%+8h%8&4q- zq^*B`9_MmPZb4|=5yvfRi<&Q3Ff)0)dII!;LdfO#UeG&XrI6lWZOK_}NVr6yb& zPUgOq;oF;xY==DhR_FX8bEE=RhmJVxdHgGP!j?heKow6PEeit6Fg66V#Rc)%cj zbkuv7RA5R#C=FqM_lA zUOm^XUD^nnmgAgpSza5r1O{M8&p76_w=4spgnAC+58 zIy_*<(DTJ;O=skg>W2rDpY!SXEXjBTz>p9dLVhubR!WyhFu2ZXa7F3M1Dqa}lcMXZtZBBI^mgOU2W0RS1gr9_K7i)6-qJ4+!Xit{NF-mH>gC8XM;JK? z80(Y4Bz7Qr=kabWp-z@&szYhTGt}TLMOwK-r+v%f6 zmQLc9(tkaVkK_6_&s40H;A#z z894-idmRHng3C>dLeXbKE1~;M)J#Z;PB&qXC6DnrJDA`M3<1pPs|BsJIzEYQ9%oXh z@RF*?4?9m)g>p$_gCRNYG2Ljg+G#onSuZzCgsIt=s!pgqn>(hVBe?9dyH0>_Z-)@nn#j+>>R_sacPCu1PuUUDvUuTYOii8;g z@tp7lOQ0l{I#=3mvFA;hBrJaJc^N-SgIU#*%Sw`H(7Up+On^xsXE^DB#sTU`9A>^B zC&kMHno+Kv(qHg@27bq%U@2x91|tn6q@eBUmDl2tgR5LMf-i^klZ&gBCRqpkvO&RP z@1K+pT>k)evTyte;qMk{Z{gi_K!Ia2`NR|DxoxQA9vO12kVbgmsphsceKO=}@2lTT zsXf>YB+TC|NCyZBx1}T#X}VUU=6$K+jzAQGk*O>}$mqaGN8?}6 z*^D8G!AVKmnQ8ib?_>E72QOiZPkTLnTc0QVMbq^U1$h4eRPf#WDHOJM@-Enq$X9VH z%6q9?5$r{F{{Rd1`C3GhNls21aly|$Pbc-R@Ak>~cy;dq>Aw>ooBc0Jw^Y1A7$#Vq zkwX#>RdRh!8_*1Rhr`biMQwQ;Qob>P{!Mn|xYe0u>zQ)O4Z=$IvyHmhUh2>1Yuz8X zQk_3%8kC>9(rNh~`G2WRC5b2rz$B0|G1DjUHRT>W)!>T!85m@?NdBMC^RBL67~30A zBXP?0Bxk3+d6&dJOiy|xOpG7BfKEDa4;AV|4xz1$Wm(Iqx29|McPwO zl6w7XVk^5A0HY~591i~g=N0DqHm0$VW+bRnfUE&0@dv*?{Wz~k_z`$ zsi{cPn~k+ET9^Y5GXNf>uOMcujY8T7Qdk~y^yjT?O`+So1je9Xbj4%q3j(*A@r-4T2!r`f5 z94W;m%rfzV`(o`7OkL30hgN=7#uSoD+*BiB9&B)`gv90h}XsVCUnv-oJ_Lbn< zF#<7&V`%MMhl~6zX1by(#yM_A1}o64ZxCtgh97Wu_O1iPy1JcpQ+6=@KhOE%zIQi* zjNXS{9p1&O9|KuPs>U&c+ZDa0cnZ*{{o3Q)RGI@rb8fB=Bns%P5mx{&PPO4Eqp`uW z9#4nX%^uMR@6I~c5Ag%Qkii?`5y(6huVqOi36?!8i1?9l34oJ>+Zm!t$j_CgcqBd$ zjde1poaA)vU6+RZ5fP9(rx+badYeYkzTK(rMF*3|<6W#8Q`xG5xE{dqT*|V!Zp9rx zz`Awe^4c&z3e&atQKz;6+dyGGw%w#Q3M- zO{_Mdk`d|Bxf@@F*6*pNARND3b+2UcR;gyzSZBdKc@@fOS5P*q8Qd|yj+ms}Z>UBx zNNhg|ti`f7kOmYUm0IuNwX^_(5cC_Rd$gBa%|S23?$+1-+m$~YJUH7>}w z^E1aE;f!%HL{owO$fCUlSS1c|f<19XE@W$De7_fhyj^W?wc$D2?t9fm_R zIbg^xAm;06*u_zNOK=AKD1Sj~K=|;~AvZv@1(E$W&2}Zq(%#E_rTzss8{9 zU&WD~tr$JW9Mjj}--`BU$j>0GqF47Ihn*wF*e z2OLnNbzyTo&zj}3B*my@-({%+&F_KRl@OxL&nzn^y9ib6Kqp0Vt zdC!P+XraH)l)RpxQnj=+i<8uc!5SvH9>!>L1~b8~>d#ERP#$Oq$QzC;Tj6Gj4eV&s zs3hYgjOR7z*Io^>j@!&gmykPks1-p|s$t<}6GrbhwHE1Y$%CsgpRoju7#g-<<2 zd2W^7jPIeCP5>Q2;=Y=*JSraz>#dAi4_xHttRD?{gr^q3fH=YHUhQM>_Swcn0M@my z!<$_H0LQ2#j^di5HPqGSZS0SfyhGs)eV=)>01TXSRCM1DYsn~4ZNTn3SJt=R2<&dU zzy}_m{c5Xk@HWjQ-Xa+P0CtBB4t0$8`K)~37l(EERBg25-2VXes@z@@@e+;Z%Yn~R zUrt$k9kDS4Qojf%p47Rs+F;v@o$=VcKp}m%sBT4E7HVUUxmNLey2s@s4byMuVwkC zc!u&$SyXgTIswm0(a|S;Yg279RwCBXmTR1J2dby$GJgu?-(O8C+ThOMu25hDxwtGq z^*ufL6|vzfE34bc^c$CW{?Y{cgT~lrkc^1I>HttlJa!%PUtPs8jw2wPBCg}_dRd=6 zz_p$miYQ6FCft7Y{&qU4b)&7^n|MeT(4XEAN6vc>rBKx_TJl$n#9T1JMLhXn;3z!- zW+No<0LkE1jMiFRjiZ#6ZEc`CTt+f8(*T^3PfmxESy0af&E(4xks7jz*eU(m0rv;1 zFeG4P@=53`@g&-5pVKw&qsu3`ld0%FCclBM{7tFDZ47J3NwFMC21|Va=ngZ1)SBdi zM}K-H@Wzch*ukIlsm|A(i2%9JIbsz+86W@v8Lrn>isQu>6KZ}TaR!qFhwS#szzLa7 z3pZi`1Od~G$89wy^3mFllF5uH}#sBgv7u z2=2pzK*$PLu@%4KPl5I~T7?L8I2nW;judieHF0UX~w?T~XfBN-$+u$v%;54@Dfzyin2P>o7yIUU+ zMd1B*2L$QTfsx4v^Y;G$>({3IHG@)))IlY?ft+NHqP~*8_y=T>m0K`4?4qgJd>Mut zovUsE&Ik+F>rRu6%Y{aoJhJ#)+gfEHc0Wv${&}u5P`cD&yb@f&w`+zQ)DgkR&Orom z$UJ)HzMr4r9kSXmFpQD?+Tc8I@V#x=l!pVc#}%aHr1d=~nMqLOS{_9Mi5aGf#X_o( z(V-_K@y2t5&PH>Lci>isgXFrg)*zlhcyJG6U0tVx{>>b)n_P!na-`!2IUkljgwjdj zd95Z{BZq;5!rbVK$j7+m>Q+o(9}tU~$);a&yO8;5>g5 zTHUOQPZ`cVtJpuYzjjk^L0);47@82zoTDwOPwe@@p- zmO?ddLi7T-4;RIq#jS*s4q<)LWd)IxWYjMnq>cHl_3qhLpEl68L$7nsen)aUnY4S;GjWct_eD}1` zGrlrcvD2H<(U3FKpf$mKNVuJ$WnIUf)#$6IM-q;l`kKntJT+;5{<1&QwGybDNs~`w z&Abe=CEzkbLEY5-YtWb^w&7WShw-Z#M}q8a$Z-%D=yp0#V->WS$q>B;u2F$ziOdN@f^<@b+b_-W$dW>PnkBpfj4M>Xr(Z;2q(EtVqVBv;9v8}a3{ z>Y-BSZsrG;>t5mT+g%rT5q|WLae#TJN>b4?n-HL?Hyw{#({v=gA+~4mt)%d^yh*vE z9Xeu<3Yh-PrGV%wV)pGJ7|7srUYseX6N$Y}UsklXvCC{;J-Svs$BXQA(nCt49kW(^ zZ?8cVE<zo0QqBllY~!a;YlF77JG4p9b}Db8QG#0|0P)*R8(Ru?)kIwdqC*QMx?I zUM|LM)%-Ag#DE8>rY^T?i}%;ulU&!0Z{E~5mnJcsR}*{U+vrCV5$G$Dqq8)P+C7@f zTeFe5k$*n59=oqvEwcpLPSc!Mj%hwWx{hG7&%rqBS{ME_62wZ))Oyv$u3MEG%2qS{ zdGSGXT|>!YF}oS*-nIM>@nY8aB1qs9j;6fh#db@t+%&Qb;G9-2krnozb>+h$03I`3 zs}UZ4R(I9HIMP!`zJA#g+DY)2!k$0e8XY>}EbsPpTYHF?_lMVHkjH`O?T)qcW~T&N zCy67~Zj(0mA0)&afZbKI(1LJt=t-}1@ZHt##Qy+?dcVVe6Kc>h+*?ByogK#2#Hx2k z=vW6-&s>fx!L){(SI{Q#j;#?}%K_w(Aj1>~5x3MI-JeV^t$r(v!d0h=#AR5Qt8P$w z+U;{`bi4iOpPJ$!Un|U3A#WLL+RtD3bGp-D()>rKM;X-+NgOx{9DuMqjzHu9XuGhr zwhas11}wRG21Z99{c(ejt$Fr|{h^^>!Ko`WYi}s~Qt~heBXRWvXCu@Tz#_YCVP}@g zSuE7G#HS-^;4gf7ag&<)9E!j8W_Jfuqj7&d572XI$3Ee@oUyc$QuVgK=2o+}g6C03 zo;G-+Q*=Qm9fE>-9s(YD&o$NQH=@VI7FPh3-yD)jjPaG)dwS##tz%mKjw``rj2KHB zB9d|dSk4FL3H-e&^IwbjlJZD{Jn%&t$GDBS1&IW9I0O;xU$Qzh>tQEKTJx<>#_-sh zxJ+d_@X^7hzFM6Is4g`NYilHOs+X-KtTTWC&PnO?_adQ5kJu-kz{p^XpWc}p_mVNd zRLS6ya99DxKqt0doqhJ3J=fPFXN!j$Nl!_798$xl>EGD z_@l&@wi;}cY8T0E0+#aGn-20f9COJ7ILA3S&o#n}-xK(X(nvH|8%MKQnkeLa8Rj|p znTK?& zn_8AzslMB(v;{;2w6Y{*fI~M_PT3?5fSe3hp!jRzUZcu<0$HGv*HH@4 z7EnkkRmRVgI8r$$*1WI6f7tqDCf3eY5nN(PB@>WtPColKcQ84~1dcP1NxuoYd zV-%trc;hQJ00qg&DxedPKp8!GBi^`iykU)9y_!o;>$&P$2BT-LG?Lp}JZg@B37G+Wu6HN|<#-^R4nWQ_GuUX_eXA%|L7MQ5 zLRocfV$SB)-r^`U{759Wo#K;#Mn=v;jGTjv=bG&N8}Vthdw8t%`{{3_zy?!jv6b~C zt2Xao6nmUk>HI6l*=9Ef?DI;hooMxOcArbDT_@Gwm20P$qnl(EC{(@`b1kiHwBGmf z^71`4_eQi-`>IAO0{2d~x0J}Sf!l#vIxmjADdLEb={mekB!ek$a7s*hCt>Dz8 z5UhF6UMfv%RJlv0jacCO)LLLOT%?j5ob)2Fzr6J)rNq&`9$H)|GC&(6JdTyK{{RZ3 zPLO2VFXB4aN8$CfSka|$GuzU-7+X(bBiu4aJu9-EH6?R5Bbd=VA8n;t7-Eb72NAdU!ltJR9bg#5!`U?2XTcZ5~4I z=Q*!bvV=9dagb}rJ{;dc9@ba%?~3)E4^)mwMo8`6s;)K|x}4G2r9xDerh`Jb8D7=G zd`*cy#VBFa8t$jNL7Z0w@e$rzM*#E4#w()-D8{mFm{s^$=AIdhoA->cz(3Ztd|PZ6 zM{q`RIL{SVK(;pegBD(W>r2HmB(_@zKQBOfROeOc)OrsO8ATp(plQ)TsCkZnWwVki z)VwpP+Cgl)WCIn!MQFjxD&XfJVvCJp=F&xq8A#_G5;I>ZDOnwe^|azsHB@* zvw^cV2+zH9z6yd{`&D-&k~ul+S{@|RStAGsW82!fy}!L;t)g5s|^gavnR@Z*5*jS2+WYeXF*N?nI5BiGLAdTW5~m zesFo+UL&jc>gqX3M?AS8=Z~dD;x826+(_}wA?kWpEqyyq2`3%+Bi6bc6BJ~(Xx^L0 zlCVBxgpaR(&ZV;Pq6Q4d8Sk3nTXS`OI}W%R{{RYV%+N3;h{t`^lY=%>is*XAo#N=` z0nzYHU43reNb@v6#Gs9ZUR`&-O89a8b8&>h`w>U-10o3OLnWlKIReXjk&qMU-L?g?% zs?}3xwngHP-H)I3G*_8hc+iN3S^ogwCW_q8+C=&Q{@dw6n+J=p08O}LA)tqkSrpGP19QTj>Qnqrmh~wqYe_U5RrhHU} z*~6QXHuWB#j((ryS2yC%6e@7_ zMD4T@k>B;HRO1v?S<0&1`zKEE1Q#p2a6Ed|KNRb5M{2}4`=nQqcu(TWeT^iNT=d8% zf^pK9#$Oa|#gTtF5=TOR`qi~JW^&0kaNim=hLZL1H*!YS2LSZwcs+PE=1r|?QMnP_ zHwV_YJXx;KtlhCKo@)W&%VduA6Zf6X;qIej!u~GQbo=$0?ii>%f^)}1*QZ+gOWt8Iz3j>jn$I`l=hrTSj@V2`Z*pY%OVb?T8YOYMzHmB*I zfpwT}wCLSr2jv*89d`c!De}qNui;-md?EPJX+E0{FBFZS{p@}f-fDjnt!^#LM?P>p zYtW5o#x^+|UCQ4Obu|z^WXK$Kubh4>>uVIQt0~DO5s_Yz@f+e4eoTbSIvfIe*UrBZ zykGX|-xHkVkLV& zxLVFswcPV@Fj1?f$FunI=_Q1;asmiF>&`VfK;75BQ(BiEFHKI|DU&$^y>t52=fKc8<$tK;{-L!*@_UZc9 zHSy!Z*8UXnjGqxL?Yz5d+jPBXQ~1Q}1d)P2+Gz;%03I=1$AW$(MRv9OR1LH<3#d=d#-S2RvU5`jF8H>9OH4Xh2yNh5ju1+)FF9NdbIY7wY0vv z_D%ec10O@{39ei7CcQg;Cmu9=O;R7V+uR8QE<>Q=NZ10bJ#qj)nDwZ?w{=ehS<7dC z;kVSS{H>PP$tev4XBkqY5O4_q5DE z8gtbr7XJMo$^QV6z<76G(6zg8`Z~&u1c0BlM9M+N2;2b-Fi1S)1JHG=aoF9q+i^O& z#=B!rw1fa>rb6x`+dVs0&x(92EIOPwP-^kp#~fv2fWdG34{cJJJ$ zsrXM>)pextM~Zw!B&}^K$!lteR7Z|><2V3gk~5qRYpC({&)F_#wuB$wwg6F|l#IK5 z2qAO+*{OU>t69RMOtY{zS4>82pA=N&yr$;YG4=7mQo=%|EIMt=S15Pu7~? znA$fEgXUIKkUEot(z0o-&$Gj3bZbRBn=#p3YYe(vHqp)JHr0z@A&*{j`RBGXT_1ux zQ>@tBwx8lJ65QR|+#E)3nPfJ^p3DaFy|I?S&ZBiG{8<%z}0Y#?WWDG^mkk7 z=jd&0HaeV?(sA1E-!_h)#Be-Uzi7!3NbDml%aQ@$kQm?)K?EEScqDUB+FmW76CQAD z)xI45%zqJepAW9BuK11(FVBINEjIGm=XQ;UNjAC|^Zn3rGuVz3UHyrECS2IIwWaIY zRlHHl<^7-c#uLEg$Pz~xJr`&lL1WUtW5_&4!D67P=3l#-`$;}m`MAHsM((sn=NXsw zej60&(x{sG_uBse(mc#~%`DDM&HyBJ&MN+e@dhiq#fff)cp2STpAkQ2e~aD|K>DVE zEXwliS*!z0nZY3LQMGs-2?TMFE6?m1d@Fk-*SdtcjrK^sM3_d-HkM!p;(90~p1{}N z;5dse$0YFeViHV#HpHb4N zr;Jwa12&m-mr<$`OfQxI4oOpnP&<+U9qXFB@k(6k#zQH_dgi*bI@KXfCl;<4S~8`H zbm}WfZjQr2)T7dMHH9VlbAg_juT$`jtg_&lI^(^3_cw?ZJIL+?gMvA)O7ItqVwN>l zlN~dYS>=hH)A8oFFE5$UqPJ1UKGn`@bNM%D z&Hdqmc&St7jqY>Rha}oPZ{d_c+oGO@MmX(VhO3~%cN(_Pa6LNLoP0U)U8E4l6S&Cg zGhU5r;+rXA7;t|OzW7qKFz1PIr)uX`WNH|*X zKZ+N~^G@n`AoV|;dKg=&Yq6YS_iHWcSAy_peQ|742fzl#T}=;F3A#@&5olYL2Yy1bELY?|>@aEZdn4BmXS0Idb{{SApm2LR*!QuQlJgFZdGDjWw=DDxg z?@~*RN?S)fw4VN%uV(QMjF);8(QF?w9;4f}d@gO6)WYU*k4IzCrAbbe`kyCFaA1+6 zU~;77XT3CEoaI4MM+XD)tG+qYA4~BS)uS&P@&rTRvNmqyh##eLkn~72zH{@pPBB zGAEpKxD0ScIn8&!6ZKZ|q}wKX;CKH3J*&jEo5q&;i#C2^N;XSFIHJlAG!|pddB>(H zh6OwjNIfunzwDwn*xII@O3HUoRuD z^sPIsA_YU`zsx(HhwD^W1(B2>8$ca<@$j5hODP&XtK7_aCqY#Mmp#6tH0SkYXpQ6`040qYdJLQjJX9# z#z3xv!l`L3(2>9=wrMujr6sM4egW6@&kfA6H<+>HVC3VD->rJKo$(3ewfW{Ck%N=P zanf7B(Qb+q2OU5hpKN+o9K=t&%{p|`KrZbI3V+$mE~HE z>*^CTJ78q+d;V2t#Cqh{SuE!nKmBuAHu9Dmv&}VOB#EuAXR)8pw|%4@zO~qBe;2hI zI0O<)_jtu}7|QM4$vrVpNQWeX0QJXeQi^+`l$>@t-xB!x>t7yH%G*d8&m8yZo->|% z_2W3G{43&{y$bClSx(Y<0Q9b2CS*NoG;D z9wqT`dlUQNjz}5c9AiJH>0TM(9}(PIW-m1GI*jK6x9@yYJoctC`8esH)}l#Vl^sma zA9%8PE*?1MCzHtjb;!>&*JCou$ByLHty+IJM`;MjKD_5OLeR05QB$>-l_N^9ND2-Q1a?1KZD=)Pqwvp( zZ?vnW#KbR3_HTy%DNSy8G|Q`;B=Rh7;{>QA0zf=uXFU2>&k|~U5~=|{qrc%@XTy&g z-07E)TwBc1%M^jt!3X9CAo4%Pv9C&1s7{*QroQw|O;(59o-4i=_OjSpMkNjANb3ZM z7?KGJ11<&>5C$=lI{{vDJ|kVolSSaa2x=Z6y@o&DMWk69fe-`$(@LPEV}J?CAoX5r z&b%S;)5Sj&G_5mL@h8L$O8FXhOaA~m<)k+t&moHdouSNORIUIUOCAFe_rHg4ycO^V z_G!~T5BQ(M!pa#Dr@gSXjFQ0Id5B$sUzmjes3plp85lME#m8B`BA#HW(xJ~dKW6pO za<_SOZ*39g@YOuiDZx&SIi#mIs#k67`BUxae7oYG*~j*ey3w?s7<^mSbZ-af`r9eB zvx87F-z+YLgyckzfS|76rAZ_Wxjb>VAG0@wJT2lR)I25Qokv*mn=N&yX+LEw{c?e4 zX5)exh$kc!$6rp?d`GPKlJsgCuBR;Qu&it6F=K!L$0U$F26z|%@mTtWhP0{Y=$ep6 zQJtJ_V7EJtKob7K4}Gn`bEuztQa!OAUo9LkglhKGEO)= zXB-kcN7c1sY|!|J!s*exdwFF(X1QC7UErnU%l@c?!p;EP zm^~x_VEy1nHG7+4u=Mb=RbeN~*{}F#^ERw>4KGHBrkwzdNhI|pk8(*lC*0M0)i3-x z>zotF{{TPElTOuI{@`iPFoJYl&u=RN+kd>IaLdj^55xj$<=uqI7-!{ik+E_J$6x15 zRP{UJ>DQtAq^xh|0gx1S+ME0g+<^V+(8O7E_Q;3F}<-*M}vDmWBKF;C9CqpQ-7y z=+XVM?(tJU!?B*5rM{gH#UB{N8tz~u(d^{u3LcX%Jy?!Z ze+a?qMm|CPpYQDT-y2xk>-vh@*_JOYyO0*;R|Fs($s}a;!8otDbT5fEFKHV^I)nfz zQNUg}1JGi)eVq&(6&h;E9rg3yX@Q0wh1Y-U%=L)wFDzkOt%5`!d)RlafA~nW<+Ckr zU-yyiaIL`y83&$!8sk>e%3d+i^rAo*!aC(gJaCqZ5_OCbi#rr?_BjZ+|D&FbQ*y}!C%Y8;@ z_eCJ;Bban_`tmx1*!mIH^xZ<+WyESh7#IVtDRi6Fw3+S0B#{t82kN;UJ;$|l)1^`k zRD`V2wHlQ$l@&@&rna&0UZdeSG>K=>uC(!Cd2MiFGg~4FZKjR+0f_04M>*q>PM&l09zIMp9Jw{XG;?t6>0lQQoH4syp*lF zzQ1#^>v2TqTOx*QLR__DYrg#&efy&`T=;3@3l$oMj$zg8eBFZL!TxYU=a^d$ouHlr z0gkn`;F#{AMO-i=0EQSN*!mu|M@{iwui@*s@AN%JOL?N&Rg?l4=O8kk7=Ud=?jrtIZ{hinsGG^8WxYxV|o~BO!d#&q{*&;^Ih%EKeOhc&%+4LeqRlu1N>499Kzo`#Ahmj@Ef?bT}`T1}iVvFJh2@G0zRiIpZ9k zO6A7m@f_aDlDoHOujT${wTI2HSk?QQiTP~)=c(d8B9c^c=m^JJr={wI%jK8ko_XfC z{BhykM@74q%TckG-gvSkf_RED{XqbZ{Y7z>w&-<)`!mV_MJ{AeK7OPC3BVb^B;?~H z4B&I%aZX99x;4Gee+P~BFtm?G(rpdRu#jMqpyNK(o#K5R<5I|kkT@c?{1qIM#-c;U z2h05{V^q@Z<^*j7k(|`WGLo&zwm9oz9ZHuxs>5B>bjzKOxM1;)_15UW5N^Q3lj+V) zH&yV&26H0E#(gP~X%@EYv_?1`tLf;*HDIJ`7j$WAJ};W;+^A#7C)c%kmyhpcc%*;F zCz|bV6;d)4M?+qJ@djPC(g1J|sIIz6DQFDw{{RqQH`<(*#v46z`2PSZ=dW)c``tPa zdC%6gwfVo&WD4gyk6&!o1^l3?A-^NRu9-2#5M1h#DEXY6bnDZlW?$dC5~a^@593l_ zKH^Cyj(Xy=E~U>0wPM?5k+JAsv(})t{wvzA~}U5fa(0fyoaLH_{iucmZwiB=kpg%pt7sU-k7KGpF2Pk^spyeiXIJFwLw6w^HK z;#b3m)O=_z7A&KZ008yR{{UXPPYnDMGutcq1ZSO~bgsWbv$NN4(&B6$aya*|Uhu|& zrlq2wP2=N0uO&x9bm2hBi!wNbtu*fvj)I`zQKe2@11 zCD6rV5=YIp-T>8AKty75&@W2U)4Ug|K`1V#1A;qaAO5QP1O64=j(CI)c=>bqREyvX zB7f8lI(iB})+%eEFB@~jB=B9_v2GdUej>6iJPD}_qaz6)qUjk}-u&O`+a7J^-)7GAR0jRLSUtxvLdGB9D z{{X_jvj-*tf_eZc$>19SoJIP3(D;s$=ug)5*!j{whq|s#*h>M|3(#b9?an#(_pR+S z!1v8)5CRAUdUmgF{{V%5WsLbF@)YR&4{Z-I#0R!2UlGHi8pc`P$AfDB01x#Bh^Pdd zRXsn!TIHOAL=DGIwe&Oi7TR89h!c(uC_VwS>*Wr8s9Z%3!+6Jij~upbVmnVEp<;I6 z99Bj5g|%%(-Q7+bw%?)e(DB>(SGx>%|}I9X`PEvTDc8 zm){KaIYVM2kV)y*s@wcD)|K)jZoP3|Pd9+<+m9r4>$FsV@UQH<+o!qOI#PT|4#N8D z?0n$A4)s!?fJq~YQSjGOVZt0R=Yxv+Ja`J#NBW_T{8PW-1GVTO1M5QKN^hXPxo(He zQhYSkHxJz$o;rSiN_0OB^;IAgJmc$MN`JxyYDNSj9P|{c;0r)8_H_fU>{0%*Q@+9V z#@X}kpYZEZfaDCE@=aQQ0P2dTEApK2j-tKoJQHZA9!SS=#XR^b0=s;mG3dSj0If{8 zl0Am8j_1v{UkMQ9J;9EnI(2hoPww%5OwBUII z{`u+6CB(2=SWg(=L+51pd8$S*6^|G{;Ze!(<5GM%Mk5_@*1nG(2(-W+Ko~u7_|i9l ztj`7|IsX7=n&Mb?53RfGeAy?%T~G`K;j_z23M&d`^aq|*`E9J(avVU~M z1Gr+Ywegoj@z$*OI={pXHHat5i0UzfT%3{yK*=YP22E(_U$FPbpW24;d};AFUH;9n zwna;A8%&2b+GIEZViGahtOha^NXHiSjguQYfs_RUSgk~!o? zBLsU9ivE1eaqlq2&kvKl45^S65y! zp4WUxJbz$<9ur4)1!h9=zYa~o#Kz!li{|hYSQ?A=KDvuAL{(MOz{GH zm5`P{K;ph_`1kNj#$N{y+q_c@M@|xLk649=*}*3OGaMM$@Bm;kaul{RO{TgY?JgwA z^I9&SH!FQ5r~CuuUli)TA^3TywugBLywj2QS}{_F1HsHqlF5VEk~;Ctbsik}0D=hTJ-Tt$qWFSV^G>+_(r{8WEJO=R;YXK(e=3NobE(M(cYAE!zb%P;aXQiE zl@#UOUw^pfJZJF+H<7gO56&^jlHo@f?g#k(RnJ*o=@CojT|+g&Jh7Rw3H87~&o$}3 z1^t&{)b9*FF!9`0Xb`bWSaKhbHt&$G)6e%$Ls9sr_EzyOi7b=&N5uLzw+T`fIWFV3 zm~qsHZJc{HPo;EKr#v-(VNxwi(f)R^^7y>N6JHeNS4Y~~97KA?mvMTxV&lqK#v@Rt zj(Us?1O651UIP82XYl+ZP1W^dG(ZJOnU3H)2FMu$*R6Qor=#j#9@OX2wYwRuuI;2L zJ-o_8BMzX0{z&$&ui=NnJzK}Ok!re}658l-N$@b~t!)AY;5w$wEOk+=p6m10qVJq|O2`hGlDvc>yPcoSam zwZ*&mo+zaelM}HA2N~*5wRrxi;_nJxXrJ(oL-79qPZlsErHm3m8zcbd1yG?1d+>SZ z*0|X`IpL!(n;(a)kdvR5+$itH00Z={oF@!sxR|+43YV0ncGvy7rEi^?+d{TI$KFYL zzw){1x-aeDCXXDJTIYnWuI=D`u<90a0Ilheaf9?H=}W5q-BRh6&*Cea%{n*_%>{;J zGqK3y7yzEUjy-W+aSfflvUzrzMx3Y)cK-lnz;bm0#D%qh=kLqA4S>CaT?h68K4b5tiCL zcT1nBTT7&tFmwL^O$Ot~(xV>$HArHP`@=sJ^yv}vn{918hfHuk)wAct#q5!yQ|*z^geQ~Us9brIJNzJ%XXUm)vK+<7cQ*gx0(X;f-wDnIv1AnDr}p;*2T7GNS}8szD#?HxjuDLCt*i;_nef4uJL$ z&AI1aDMGj22RR4SWcTzH()=&@>#k`YE4aV#uZZN+Axqg(+RENPJ!c_IV6;pbSdiQj z2~xl)QO$UKRh;8;yl49Zgp`%sX0K;^yGbtjQpEF&;ME(gNZQKQ*ReEGqki%!P^&5QAdG|4t#_Xf z{{U=lHuf7xe0$*8Rw-_bZF{8wBQfNlFicc{aHQdhAdnX$C$ECz;1BH~rmml5;<%;N zWsTV)xwf7wtvwa;QGCFp{pMkf%0jkx6~Xx|KOV}SEi85s(}ne%WzM}?qw?wVxxZq*%io$a5)&Ld_@P)rHejgi{VzR zoa{Ks!RcKuggh(aO*w_L%Hl9V0B7m>SGo@cSdu}Abl?F}N8qa{n<#*KQl5QEO3hpr zHLisphVi|GXl^r}2|oV-=QZlOM~Q9Xja0DU_Tsq99X8(31@iHaZk2f^jd)PQ-#DtJ z#4v`)79~B-@jO{-tAi(AXs!?KZoCGfxBmcG>67i^G(rf-U$54jw&9Ne55u)caJ&)F z^c0|sWG@4$uh0FnvBEKJ)HX=#*NRC%QWaP0=}$uB?hD7|O7U;bip_`n^PMP=R*C`qzrTY zKf;Gr$V$a=Y61CIB7ttCkW;6i=B6WQ0FJ!kfEfIk10M91MPawlXtY6$4%}j#Em3lF zk(^|7sY8K`^x~3Egq&j?bJm4bTHu->XaY9UKQ5H3XghwDVIvNwdY@eT`cvTvxRIQ8 zC2{^VDeH7LnN6&~U_$<4g|rNC2Y!`qLZOLO`i{A!SmPsc0CUbw4ySQlHdp&Hg&-;F zIQ1hx{d!Sogl$2SR<@}HhreoQWGqu}9N^>Lg*9yknNZsN-2epV2a1qdFIHc3kye;T zoQ^)c)3zZSbII@2W|y>$^#txv7RXP^58+Sy5|ViOW~~VE!2td=frGZy9sPRJRS>Vx zs%(Mi0Ozk13k)6?BeAQ(HB>L0;Qmx8Ry{xi=~SeOy^5b>034y?t}EzY_$T+ne*^fV z#QNWeynAbJJ)V@7;?myfLnB-{murwl7^15X2;qvb!B9to6;YC-+b6YqSL|h|K{m6i zcyjwv`z@5Q#c@2~;j)3$Anlee06F6%*O8viAK5iImY(namlF!gGNjY;*XEC=wf_Je z$Esa_Vt8uVEi|iPvfADh3NSO4KBSYLIrSvu)p4fUt+L;V(WL5A*0Algh_)lyu{*iR zW6<^{wQR3dV=zjeD&PU@{&QcPxw*;RdIuz6v@F4WXzC0syT7>4(EU2rjkU^KMrE3O zDaS+8{N}v3;-~Eu@RQ+n(bKgn`-|;OKr(6iQY?=KK_JLkl~;_M0$2hG01!_UKW)#4 znuKx->s?;nVglR6Bry4G2?Lh%I6RVaFaYbGD@ektUhG?bM{o9aE#cj6N6X#leGJ|& zwJ{Mi+1uAAsTII{U-7qvl|<+yFRWQkqW z`CxEdaCZPW&Swux6N;_up&PTzPnu~*cd_DH#r?*!ajAG~!xGJNY}(J0rNt7NBxt{Q zHlP5O2j$>%o_m__d46&&1`RtMgdWda!4Qky84{Ihw(nz z__U)4{hQs()u&{=F5bI!QlnnIMeX*Eo8JEb@IS2xE-rNmBQ~o$MQy;2&K6~cB?J{! zKz6E(3=V*BI#p3YSpTz97I=Osrc*PnX&>Qv}q6sbPR4PeMiO$0NwX-~XFo5v@7z;OkOMd))70}?#T1jc4t+EJ zdr~|Lu$`cNYq7@qT-!q}Z2`^)=|ftoB&U#Xq4&B%u^jn~EqbaPnw+F8o z{{TPdig>aFXB|gc&@if>nLBr<6kPBD7~`L66nlp%GXBmwh40(zL9)g-4D}~J&a^~h z`@Zx`2`W`oenj@`Pm~%(J6r8e#xo6;r#uUt#bT9^XV(DKcRCM%{6Qd(#eM|Sq>ab# z?=3BicTT`?{$jApXBj>Ds|#qshWPlWq$;4A;KgNFELwijD(?EN9=&d@sfAo63d`O| z;k;GwKf@+fzVME(cw>f8=EqcGT0xRZtEnR$xe9UIn&Zxg;|~LP>f6R%9Mgr2iFPHj zx41?~%YtyKcm+c)0R&@?MlwCAt;k|_`j4mQPlYYmJQvn6+*?ecE?|*H9RXG#9;$Mq z3@||ef(Wl08N*G|ZiPuX_LI75+g{10`?mc%amy`?VR~)f%=P~O0sL0b{x*0@El1+7 zhqO3#``IIgx`cF;m^jC8eLh>E2QI4jf65tK~k;BAb>$242%q8pgFI~Gb|QumDHRa;-!Ce>-TEc z%`=@f2_;ckqwH^t-w$-(jNc6P&kfnfZxzJ1;cX=`aD`%76r2vFl~2A%&nG_t>N-8f znW#;tTmj}=%+X7?`@yhr?V9@A!G9lgIW*bii&T;egqkhUJAe<$yz|ryl5x%n>Ugi2 zd~G~eo-@_&Aw#xZQsKuK;K;;(I{M!ZF}R5;*hy&>?|I$XKfxo=!_-ofihh5g&i#mt z40q{A*;$VO=C2r(V1@mAP`ge@QGv#5^)#F6b#mOK_Js0!)NO7Qa!EC!#xgJzlk4kH zvXV|0z%i)JX>C&VSlDGgLe_9-6Vz)&DZpXHN3PyyCoxhN)p%oizC7q z!0mya)YMky+ zdUvTJor%a+$m&g5mrmTowgCsRr50XNjGj+ZLTx)RvuuQ2u^UQqJAeABKerLo9xCE# zdzfU8oMdrB=v?8OB%fZ?r8fRSt1B(oE4*+q-kU7jeq|%n=Bz%LsNPyO54rx7qe^YS zu>+p}0EJ4NknC3%MqiW?J-cF-IeFw0!5km+#a&$-D-W1-%cV&pker_}0c|XqIfS;Pf=HXx?$ilRWg@(?3d0RExHwYD+<_ zcZ0NZ&tF4%u*QocY8%}0znE-*%4AbVAt!7kfmmq~>&#!JOH7?D# z1Hd=}ueP4p+8dGVI{yHZLg~#J%M9b!ucc2s+5(u8Q3h$h)vQ*MX$S*p;QG{Y%^(4ZJQ6Wi#+4+C=57EJ#yflB zn3`J_*arit^rM=3c>}vDwcW!E4Dsk`Bc5%?V_=_py(XOzB}06}gW8roD+B;}Jn=>5 z#jjGFlTLGhKs;Ap@SDa~-VeIEn(jyQCh~+)tYwR^VoLB291gy{E1xzG6o4=tM<=CQ ziqcD2&OT|MrDI^bY9>z+n)#~442Q~jX7X-PgF_;_f38JZ)lYH*i<)q(pQB#>nzkXVpH zgTWw^kT85|&Myb+b2E5{OpeC*#?0Ezo;h`V2Q}rN7Zb<65%`IB zAB!)I(>1JS!ueFHPY?w`0Ote|f&n-@^dO(l{Yn@2F z^tUpgb|1V41-kL{9Wji03eE6@k$9WLaM;T_2pZ*KR)4wJ7Ho5cP^9_+YcG`0<373Q zKj9>{p81WGWI$MmM8IS&a5r`Yj!rlPoM2b${3C?G;qc0HOP*T%wmyH0bIev@jC5(X zsl!B;#l0yPN-G`wacVYYw@s&BGYDniBswkE_n^m z=hqeTxQbGtiIo>CxZ3?5yE}R7-8P+@vG({(ULPAzo|L{_SE)g!fA4!>oa4SKNEj8% zWR6D%fm27M{_o9@l08TA%`jPpaune7$QbtRS#!xg=Ud!@ONIdL!NENG)1tm9AI!Kt zzO`j^;SmNwpQ-Eqf2BF}v~!Vw0-R&|)bmRIVzT*_o_5A@_4cUR?sXmb&w99;atU@T z0!N{z{{UvApDjx8dguC8UZkFZ%HG8>a99FI@bF3LR${fbhfQw~c%#HNnl6TFWIdjX{YjIF3bSoKpc`z0meuGoN-t`75@Nd z{{R|%Uj3qVFARJiu)ox_%?QC~9;#30y3!aG-1??@#;vXDHp1R}!Wuh!c(NEn6j?YK87xW38N%d)>)tT< zk>C%8no!m}b>fRXF4&S5OS#nyj0SM3PUk+#2|kta^z%BHekGg9>bmr=X!FAS-}dk7 ztbHfdF!7~nK_kHQ@7UkP{v`0x)b(hz-wkTz0gp|GRhIhIjyYC#iCPj4In2Y{3i00; z_(R2B0`UaC8u4zEEtTXSF^)_S>Pf?^0z)am!vaCU=OBUVzZyT_r`{0oUC_~f82mTX zb?eK3e$97y!rloo1`4PRBY~U{LL7oKkZWhbemGCtgWyi19<|`nqxj>)l5USqkr`P_ z0?1Q-7)3g(C=5srnaS7*`zpC+eHuSl%rO4NN$B?9WaDg2~0(_q#ZUr_O`&t0u}dPyzg@;>I>y1I9XI*AzmfM#PgvF@DSd=Od@4Dbd}QJbM0>XZt!W zq_lu>j=$2G_H4h(zz)Ei{{WuVZlu!OKFL;Vjj#Chrs~YTAc!`!WVj$~wG0-{P-`G*~sg6T%O1ZFLZMC0nRKWj*j*5#NF{>M2HrOjNDO z%-5B~sryH@&IZxfO`0p6EK6+h#If#%=1i~y8=G?+vEwGCH!@2*OEiq`>`ObGmM5tv z*1FFVd=L1O;H@<3ekAaH)xEm)ol3~5lg8qNh5T>{KJ|m5_-9epbt`=zN`Omsbuz_o z5MaC5oB#_D21x^x=}}JxuyW*$K1Tek{{YMtB~L`GXUxh@M&9wWY{M?xkWPN56v1R-mu!O^j0}!X^{oA@ ztn?!7q9v69!#6|J(ml*$B;CmPE(PgVTZQPnHucRUPqtM-yw3EYHj@0u!61ZSp4sN#x_bRQydGRE2JxcRs|RfT&w zcC>N=KqbH(KV2D^CQ$zS~$-;8|QrEZU;^? zlkM+X#yuH&*p;QX&c!%$o;L7BJ)YJ}Q9O~4l##_OS`sb1a!;??tkOsnK>MryjpRCR1)s+Q;~ z<=2KFvx45@(_XWba;}Q`!}h=&o}=@q>~zC?@WXqmT0kT(^GHYo2abcTanyUz@a;>N zK(a+Eh~NQ)l4m@wF-*3VC1JRp2nRgl{A%6BhuOTyq0}tiHGD@aNJu@w$m$3jd(mqg zO6E6sLzHdcunIBIC{$kf=wnaR6g-Y+RDq7TyT4I&%GweRGAj+6XoRyIp@-> zS!nu(p&H01v^WQHo0QLPPv?w!(`C^$2x4+(v`C3o1kAk%BoWU;gZPRa4KzctDcD<< zjLCZ^&7Pq2{{ZXpPH5qBVCQommC4|Je@b+=H#&M5E_|K430z1Ro-#NDl1_Vi3Ur!2 zt8*F~JJ^~t61?Lm3}ELt$?b#rk5Nn5xoKe}n9nu?BETiXjtI%YsS+sFN&862I4hik zPaYWYEH?rPH1%lInLf}ZjDoq~j(X#_UTAG6RDuI@4Y*}_3M=dk2?HPkGC3GH?U700 zVhpc)=qakK}5-jNET$_>WhqoUH`PCEP<3z{h$XM~#HOZaB%# zf5M*}TA!NAEu1pQQ(=gb81w|=85H!g)x6BfYRTF6LUWH$JN4;Va$j^#x72|eNs+Ev zLIU7}&peOUnz0lT2)AUmSAai1^Y~Ps*?P3F#~r=1x?DRfWJYjE=b^?sV09pjQ<^<$ zH&m9+U>|^5)qw=^GB7^Btz2lh>eDLLWC|{f1SDiJ_M}m8_QpcEBd&NH9DY21I+9_j z!7_P~vhm9#XAQ=3KQGdqWpAqAN|tufyN8!*lMy2w&jXMKPta$zN{7F*`-N*Vw3jzg zL$E?mIo!QJ&nAVwl@LhqFa&Ty_5T1ls~0xfrHs#MbO>B&RSgzR=sm>kG1dL>3uNfbmCr#u6|>a!}yCADWH0?QuNG31=!HV0q@ap4wclrMd%u}NM@a+bz;!(C0Gi(>UTDl^Icr5b|K`EPF0kggOP)g&;x;=TI}V~yh*3&u*c$Ct6O(#Sl-z# z2#onfkIqV|FTpq@5^y=duIJ%L?EPvIwCv9$3_(&#M3vat&V_lO5g%W2f)g>&n3<=9?A{E zPWI+jR$FU!ll69QXLriJvkz8sidF2T_EHZm1m#k8i^@^^uL&HSXEf z6&J0{nPg}s2^b^;g2emMHS`fmix_{KjN>4j@-z8^j@$K*J!zaS;6nK@i9}ZmjbYJ*d%Er`EoXHu z(y$Hjt4wzSFp;#5NdydJiuu+ZBlbPin%`RZh4GJ9@g}vc+lRVu5L%n7t7i&9fg+7t z8DBXTF;5ZHU6U@{F$1>UcMSzhSH2hyM3KoF$8M+`I3WQgm=S=) z@C#rJn)WEQ1dco`r1PBTl6`ou5%~T4cziSX9pg#98rw5j9(YjegQ8h_z`-ofKYJTY^n0TF}j zv&Bz>UbXCMrA!O5*Tq}Gd5n%CTXbCt@OfFXA%W5N7C&(@kQj_v5fr&4q4Ni~h!URpiA zSb?~P1YnGh4u?$CJj)p{auXrrY@VlSQ=eSd+<0a? zLj6sA=lgGX7G>3Rj~!|7{Ij96ydYo(Q@IH}GC^VVQC~-&VckqJs~zI*cH-g9PnmQ* z-@u+Mwein`bl(+Ok>%gnq@_n87>}8Lp@9e7S2_Da_!KzSSq0k`pzSnWl+eBOsB6?7HA>RXTwvZ_k1u7amHgU5BP6t+8=Z-RUuZh~Hi6y_f@gIhCn}ol-j^*vT=oI+cQREy^`%jatCHm6qAy2uaS(6hdAKVto(JY z==Txa+X#@y6bl$+Ibjl&QGj#EBoaB~;~i-e#+To|mg`Hkxj`&*2reg{K^vyxtQn=< z(TF2BBOGLAv-X%}qf8rygP-C4r6#Q0%0Uu{KXBJf;VAd)vSl##$uxpyjq z*N_JQ@tmAs6JiU^QaS$5apbxH%p_z5&&n8p2nVTLk(0^b6dV1!p$eKjy!)kVxtS9N-KT zp<$#@ist?~BoLsERFISkVglM=Leec+-PdTN4U9`T&#g!Rlp-~!5GH? zWMtqBf;xi3xfi}#2hMyk1XRO3ZUcoh~$itK*=PENZJTwHxY=GmpVqoS^{W zatH?`0hP}jB`Eg+WREoG4%)m36>$ufmv2vSE3NdWXcK)@BJvnyeWHBwg6TG+c`YskVpe37(9+8G0Krj zlPdYKsq$uEeozPmf(}kd7~R-lfs=^Z%IZi-YnY^PTYH_%9YEv}!N55H3=$Lq1_o*{ zj9uEfQL;(BNu|{F&|az&PNVml-bTmwtrOY8s)HBl}}AvklC#lBI|^JRFAU>T(FjPz!tg zei%}Ftxb?&9ZQD<& zV_-s@90mbM13AwpXyEZd*R{oxPqui1@I+XLLvtqd0B}I+bAkpi4{i-dEOr)=A(98! zA|$i6caea~7>r;8jQ7CCc@-G9S#~IzGbA~bI1ROdIW7nQj(QS6KEMoGjFY-=@fka7 zdgOj$wyQKTsxUGO)^bz;N`P=dKA->zBRIhtvUsCkHum~NdbQoW@FS#66u914=jOv? zmdgM}a0U)WF@gnzcTmNp-4iGQ!=WXW%8lMx2LmLIhoC1MaVo)WB(Xx5a!0plArPr> zGC?H>1Lna1o&y7tKs9lji*2`eur^KZ%ZQ?$Mhl&-BW0Hg0SAy|1b{~% z61eoKR?^1mNv

1Y>k#dlm{*l?3MmjFK=x2Ot6oB0qTb=6^O#u1^n&d{_)t5^9SK zOw2ZzLjuGw`G`A!BXQspfshYMLE=9Ys;k?0ox;X87MRX{#oSUGYTXdGGZ(CTGBX$hbTL#D&j76cNuY+aXL=dasJEH7}o7i|k@m z5}k*7fIt8eN|03az~}%Y83o)n)>f+0OomoXmmoF(45fj=#^MQVj0_L}!NzGpsMybb zwvfdmLmMQd<0Ue~=Fd481P}nmKm#C}rOgT6<4={}v8f)v;w4gsWh}@^Edd~7CAk~` z6;4ScV+Xk)pT@o*yMeD+teIm4SQ|bD2Lm924gl;22Q`{rI|eeFNtOwZlN6D#1fc|i zcW^LR0&$GbLw^Ody0XXi%bS_mHry#V++@Z<4&O5Tqa2@@SDBq-Yhdu2Q*-Cj;k;6IyOUOH zOWF6a%Z{w@^c#$Ce=DWy`gxxB;r{@G9vCu9_P-YBk=~fWmcz`J>Q=_$q?QS^WBaNX zIp-%eHP)Z-!^48o$HksHxsLX9KV`RzNo$vv8@B+fA^MXIlxBtV0;&=Zsg83%$u<2=@1i2fveCe)(-*u2z7+N~N!f_1hfw;(Fv zMh4-GspJ9)1mM@@Pb|sQT4{a%0Ps(jh9T4VPgl|Y8fjW(oErV5%vbh>$XTP(OVf22H^fUUQCY^tuvirb=3{_YIRp&jt}*6568O*X zp6kT1c%M=Dp|03lNnsK|(Adn<7R&xKlpX~Lh4a`Q}}RitRGk6^2*{g1;PS`b_xa^j?D6RFeGG> zYlOVE0^uz*Ulv-(9sHnfVq|^gK)@I{%Wwh42>_5eIG+N1QQ;2;SsxR8M)4KDoXW{< zrrW{2pi-p%@nQrIr$L^1;8M8pr-8067WU;HW;=}WEUkwGU}pm$4azbI2OMA%U!?H0 zd~PCzT3D)1&~fFHQde50li8(ZveNpV=36?_griY>HnyME)<;ukrd{0K6teNAn5w{G z(l!B(TXp~&umA%%9r1yJbYt**Sxbq0f2B03<_RH{790q2~I z05VQV&PUaIJThOyy}a4zw0YOV$V()#|^BRioF>s#^9ve7hA2IzETi+W!E?p91&?!!P1b9e9q? z%EBTDmf`tk9Dsmjo%qNK3vrM*uLSsm{{RIy@z;!!=STPh;jyRLTeuJ6O<*Bl#1H~2 zA1w*wjm60T@DCO8_r_bV9sEo2UDu0m^)%ODXK5_##mR}Sq2;`>8D#|E4D8D}$tQ8H zW5N1Qj(#9`h8-#?g@iW5ubFzL8+%0~kf<|~M>qhIa87wOzYK5>4rP#zFENs;)~!E= zKQ6D=Pb1T-j;T{fG?Ds^`y2dp@khk(f_57B#7_|}p{!eK7f?xlh>@k(SqWB9ilBgw zqk-w3PHX3{_$Y^j{AuE^k5NhC9}a5QV)sV1l6iF7V=BidnDGGkBV!R9e3O6x$?67& zz@HuXGxmGwz<@v z-ZtlGI6X?>l14saymjX!j1gZmlU2mz{ApVsmr}JEb0u}kxn*samIb#K5Yv~(uhQI?SImsuJf=DESGFC4bn@4!lEiKV0HB;??*tsdXA*jxuvVv`F7KHeM@&BO$%yr#?KlemSsaP zmV^)FNE{9W4wkWmy z)SADS{1J6CEc}@#CA9^O+dEm|A&JgOIUtUwpl`ZK0~|>`-j8*gd;Ls#lHp>J;mF1T z3>T;z^(UU02D&Rhi~1`k+3!3vrOJi8q>2_@vMFGtM-8|!0|NvQ039<{f8k<$E}AB= z)jSyjvRy3Ftakyxku!|A`IummNN<=nGC=~hhFyYss;lOhtdYrBTG(1Vs|KZM2$Y;W zF|wnKs054xFirr*K_}@~VY_`U?mM`qcDfEEf>@QXvVaIVI3xpsk-#|_Jx<-vi9QFk zh2KECj@SkY!d4`p0~rOdI+92u1Oc2k7ObZD6RFw7_B*@lLmY9b5!^W>f^bRdKsX$N zPZd*XhL@ms;rYqnQAZqWj2Hf+0Of>1fio7mn(;lkX5sg%11&71Rrlq*UKIQPYGy7UlM#f_=l+2U92RgsIXm^DUru1!T&STq|-&0D+D$ zNgmm!Us~#RF=|?6>{Hv%7?LTXaLVdNK?HRNt$f4rf5yKU{A;L5;Y+U&S$&Sh@adnm!D#L~$jAa)6eXrCJ9JK2iY7MoSH+1mnJN z3FM%sgsSP&r1?C%{{WFO#8amxlF;{mj6VUqDe;oXKCy82caW~wuPxLTDoOcR9vOKg z4CEe!b6#Jme$Af^{tM|c`0wIhj3m{dum($gHal?&-N*;Zj5jf0dXbPY2|NNSEob(^ z_?=}3*k$k?rLES(zi^Ss!621f_9{+E#sEDIN00c+;!lhGd8wZec+d;l1K^x$vsF5jDSI>!*K=a7SYRU!Gi5`!I6UC;{ftVBo3!ICm>hPPJ-sU zTbx~eN6QJGMOfJbd0}N41&IKhfN((|a6llI2d*$kw&v+A=-6!3yPr00}&dfWUyn zo`;+j!OtRH6y1R70%R2(MtJ9fHV(t!}rn+F1uxY#w%C{W$B_@%#Z-?(VD|$CGP! zv;q{{L%m4J!Q7yR;~b1}jx&-OC=!!W)3Z z#PVRPTltv4yRtdS^b|t^Kri z7e8h)mL>&}aF_>jf-}!N{v34eMHQzkTpohzRhIVp7Hu{;z(!r&k* zRPA?^w$exiutqXK$G1_=a#34GF)1WTXFj2IEJ^YwwhN7dug1>;`{BPj80l0rz^fJo!1$?x2rwH3Q-5-Qu7 zlgn7&YI%)_-H4qKciYh9uTH;hlDBJ5>t}qEpu-kO%_<fAMSrMVSl_;Hrgagn`~akYrvr|^f2sVrAhhatO)@c< zs}GbC1MpP!-TuhN20b&1D5{>474;uBr+eoFOMpafOu>U`3D#z3=DhKZ8rLI zcP5plv$LdOm8X=bCwK!pMo-JQWak^W^rDKN`FK)F!zR~t31?Tjw~3}#-tJtI*#&;~ z0m$kJ&OaU!8=X>EW4%kDzHDcDF94kN#xd6&y=bDgNjYp!QI-?xV%)1GxepUDB?lWo z91LTjAD81(t+mW{Q%^m>X;BP&3guraa>u7)qZsw))S{}(nP@MxI(v;yNM37qfnpA$ z%GgFX9&mDcj0OJ7k_UtI4u-!4eg%)x&4LjVqEk3{t zk^mSy&t9B=lbNofzqMaH-Zz-U%M^k}B}%?CjN|Vq_2BjA0HU7vPq2THdJFFqUBH(x z_@ebzY`H^l-9rqF_83fe&vDZg?k{{vVcU++9^zkOxE8 zu17$428t`w=9QVL9CvZf%OMV@7>MU_QGj;kvOpOe1KXU`v5O5snPfL4NR1&^ln_A# zArjSYe^NM3Yd8?a%6f>(A1bAUQ$uO#&+E{jUC)ZLclnUrOt zMnJ)vB;|=Y7#spQ7#^mIDaLY7ag6@R&@~9uG%_(oY?npA1mQ>=@&_2tVb`^k@J5EM z9f#QCf8Pjl=)W=Dg2dxIo=ETQMHQl)q@CDZvZd9Ih^W$ASYWKXGr&1PyWfvN_;3KH zUwD5{{SkfVlbeS#zqMok5AJiqPB0{F354x?5?6)NP$@te6x@N zBL&7sAY-V&2exsHH~taZS#9OHj^aQSj?mnkfyh3ir|ZQORJPh6Ww(bT3fi0ANZ=1K zg8(-Gc91~po!-25$3qdeHc}yiQ?qKShDG2Ua5&(8C#HIy3MxRIF7Ba-ELSY0R3vhD=n=WnRMJwfgcXri`ts?%$g+h9b| zc$>vh$0g)?+}|XHSi9~l%2m1%jAcQ`sK-j&i{n>}^!rGjb)&6OtC7cG}91)CT z9AJ=nG*Mj?vDms(e3N|!_cbiOIWDLqk5lm66G9bE!k`u`z~r3w4TIEVX9RIudM>x% z4;WlM?Ee4{?M2LYhEVSslE(mUEsT@GoMRarV*t@beKrRXR}j*w@9LNN0yMQxhW-rk z4wE#W*ik^T%qN;yu(B5_z?^~49WqWbaf-v#{ss79$nxQ6(IaNww;wS)0iTx~V1bMR zF_A?T>(;>F>T4(tLBN#o6aayN=?lj0}ie$B$ z%DZ%HhG*VNgSY@e03F%RbHE<-QC~ToVDQ*@Cr&mgC1VaasUJl0HTWdm8Hn&G~e3{Eb`qFqW}g3 WC>&sJ&Iho-=cN=;TP=#Rx&PTnl3ZZ` literal 0 HcmV?d00001 diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_humanseg.py b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_humanseg.py new file mode 100644 index 000000000000..e2ef62eade66 --- /dev/null +++ b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_humanseg.py @@ -0,0 +1,112 @@ +import os +import paddlehub.vision.transforms as T +import numpy as np +import cv2 as cv + + +def get_color_map_list(num_classes): + """ + Returns the color map for visualizing the segmentation mask, + which can support arbitrary number of classes. + + Args: + num_classes (int): Number of classes. + + Returns: + (list). The color map. + """ + + num_classes += 1 + color_map = num_classes * [0, 0, 0] + for i in range(0, num_classes): + j = 0 + lab = i + while lab: + color_map[i * 3] |= (((lab >> 0) & 1) << (7 - j)) + color_map[i * 3 + 1] |= (((lab >> 1) & 1) << (7 - j)) + color_map[i * 3 + 2] |= (((lab >> 2) & 1) << (7 - j)) + j += 1 + lab >>= 3 + color_map = color_map[3:] + return color_map + + +def visualize(image, result, save_dir=None, weight=0.6): + """ + Convert predict result to color image, and save added image. + + Args: + image (str): The path of origin image. + result (np.ndarray): The predict result of image. + save_dir (str): The directory for saving visual image. Default: None. + weight (float): The image weight of visual image, and the result weight is (1 - weight). Default: 0.6 + + Returns: + vis_result (np.ndarray): If `save_dir` is None, return the visualized result. + """ + + color_map = get_color_map_list(256) + color_map = [color_map[i:i + 3] for i in range(0, len(color_map), 3)] + color_map = np.array(color_map).astype("uint8") + # Use OpenCV LUT for color mapping + c1 = cv.LUT(result, color_map[:, 0]) + c2 = cv.LUT(result, color_map[:, 1]) + c3 = cv.LUT(result, color_map[:, 2]) + pseudo_img = np.dstack((c1, c2, c3)) + + im = cv.imread(image) + vis_result = cv.addWeighted(im, weight, pseudo_img, 1 - weight, 0) + + if save_dir is not None: + if not os.path.exists(save_dir): + os.makedirs(save_dir) + image_name = os.path.split(image)[-1] + out_path = os.path.join(save_dir, image_name) + cv.imwrite(out_path, vis_result) + else: + return vis_result + + +def preprocess(image_path): + ''' preprocess input image file to np.ndarray + + Args: + image_path(str): Path of input image file + + Returns: + ProcessedImage(numpy.ndarray): A numpy.ndarray + variable which shape is (1, 3, 192, 192) + ''' + transforms = T.Compose([ + T.Resize((192, 192)), + T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + ], + to_rgb=True) + return np.expand_dims(transforms(image_path), axis=0) + + +if __name__ == '__main__': + img_path = "../../../../data/messi5.jpg" + # load PPSeg Model use cv.dnn + net = cv.dnn.readNetFromONNX('humanseg_hrnet18_tiny.onnx') + # read and preprocess image file + im = preprocess(img_path) + # inference + net.setInput(im) + result = net.forward(['save_infer_model/scale_0.tmp_1']) + # post process + image = cv.imread(img_path) + r, c, _ = image.shape + result = np.argmax(result[0], axis=1).astype(np.uint8) + result = cv.resize(result[0, :, :], + dsize=(c, r), + interpolation=cv.INTER_NEAREST) + + print("grid_image.shape is: ", result.shape) + folder_path = "data" + if not os.path.exists(folder_path): + os.makedirs(folder_path) + file_path = os.path.join(folder_path, '%s.jpg' % "result_test_human") + result_color = visualize(img_path, result) + cv.imwrite(file_path, result_color) + print('%s saved' % file_path) diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_resnet50.py b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_resnet50.py index b95ce917e67c..90c0d26e157d 100644 --- a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_resnet50.py +++ b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_resnet50.py @@ -16,15 +16,15 @@ def preprocess(image_path): variable which shape is (1, 3, 224, 224) ''' transforms = T.Compose([ - T.Resize((256, 256)), - T.CenterCrop(224), - T.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225])], - to_rgb=True) + T.Resize((256, 256)), + T.CenterCrop(224), + T.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225])], + to_rgb=True) return np.expand_dims(transforms(image_path), axis=0) -def export_onnx_mobilenetv2(save_path): +def export_onnx_resnet50(save_path): ''' export PaddlePaddle model to ONNX format Args: @@ -35,7 +35,7 @@ def export_onnx_mobilenetv2(save_path): ''' model = hub.Module(name="resnet50_vd_imagenet_ssld") input_spec = paddle.static.InputSpec( - [1, 3, 224, 224], "float32", "image") + [1, 3, 224, 224], "float32", "image") paddle.onnx.export(model, save_path, input_spec=[input_spec], opset_version=10) @@ -45,9 +45,9 @@ def export_onnx_mobilenetv2(save_path): save_path = './resnet50' image_file = './data/cat.jpg' labels = open('./data/labels.txt').read().strip().split('\n') - model = export_onnx_mobilenetv2(save_path) + model = export_onnx_resnet50(save_path) - # load mobilenetv2 use cv.dnn + # load resnet50 use cv.dnn net = cv.dnn.readNetFromONNX(save_path + '.onnx') # read and preprocess image file im = preprocess(image_file) diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt b/samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt index eb217e27dfd1..6887c2ab2ce2 100644 --- a/samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt +++ b/samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt @@ -12,3 +12,4 @@ paddlepaddle>=2.0.0 paddlepaddle-gpu>=2.0.0 paddlehub>=2.1.0 paddle2onnx>=0.5.1 +paddleseg>=2.0.0 \ No newline at end of file From 982745fb839ee27dc0e3b9ab45b9b3e9111593bf Mon Sep 17 00:00:00 2001 From: Alexander Panov Date: Tue, 28 Sep 2021 10:30:07 +0300 Subject: [PATCH 226/376] Merge pull request #20735 from AleksandrPanov:radon_checkerboard generate radon checkerboard * added _make_round_rect * added round rect to make_checkerboard_pattern * added markers * update docs * removed links to findChessboardCornersSB() and added checks to markers --- doc/pattern_tools/gen_pattern.py | 97 +++++++++++++++++-- .../camera_calibration_pattern.markdown | 4 + modules/calib3d/include/opencv2/calib3d.hpp | 4 + 3 files changed, 98 insertions(+), 7 deletions(-) diff --git a/doc/pattern_tools/gen_pattern.py b/doc/pattern_tools/gen_pattern.py index a6ffc7ca7ed7..83ed115d361c 100755 --- a/doc/pattern_tools/gen_pattern.py +++ b/doc/pattern_tools/gen_pattern.py @@ -6,13 +6,14 @@ -o, --output - output file (default out.svg) -r, --rows - pattern rows (default 11) -c, --columns - pattern columns (default 8) --T, --type - type of pattern, circles, acircles, checkerboard (default circles) +-T, --type - type of pattern, circles, acircles, checkerboard, radon_checkerboard (default circles) -s, --square_size - size of squares in pattern (default 20.0) -R, --radius_rate - circles_radius = square_size/radius_rate (default 5.0) -u, --units - mm, inches, px, m (default mm) -w, --page_width - page width in units (default 216) -h, --page_height - page height in units (default 279) -a, --page_size - page size (default A4), supersedes -h -w arguments +-m, --markers - list of cells with markers for the radon checkerboard -H, --help - show help """ @@ -22,7 +23,7 @@ class PatternMaker: - def __init__(self, cols, rows, output, units, square_size, radius_rate, page_width, page_height): + def __init__(self, cols, rows, output, units, square_size, radius_rate, page_width, page_height, markers): self.cols = cols self.rows = rows self.output = output @@ -31,6 +32,7 @@ def __init__(self, cols, rows, output, units, square_size, radius_rate, page_wid self.radius_rate = radius_rate self.width = page_width self.height = page_height + self.markers = markers self.g = SVG("g") # the svg group container def make_circles_pattern(self): @@ -70,6 +72,74 @@ def make_checkerboard_pattern(self): height=spacing, fill="black", stroke="none") self.g.append(square) + @staticmethod + def _make_round_rect(x, y, diam, corners=("right", "right", "right", "right")): + rad = diam / 2 + cw_point = ((0, 0), (diam, 0), (diam, diam), (0, diam)) + mid_cw_point = ((0, rad), (rad, 0), (diam, rad), (rad, diam)) + res_str = "M{},{} ".format(x + mid_cw_point[0][0], y + mid_cw_point[0][1]) + n = len(cw_point) + for i in range(n): + if corners[i] == "right": + res_str += "L{},{} L{},{} ".format(x + cw_point[i][0], y + cw_point[i][1], + x + mid_cw_point[(i + 1) % n][0], y + mid_cw_point[(i + 1) % n][1]) + elif corners[i] == "round": + res_str += "A{},{} 0,0,1 {},{} ".format(rad, rad, x + mid_cw_point[(i + 1) % n][0], + y + mid_cw_point[(i + 1) % n][1]) + else: + raise TypeError("unknown corner type") + return res_str + + def _get_type(self, x, y): + corners = ["right", "right", "right", "right"] + is_inside = True + if x == 0: + corners[0] = "round" + corners[3] = "round" + is_inside = False + if y == 0: + corners[0] = "round" + corners[1] = "round" + is_inside = False + if x == self.cols - 1: + corners[1] = "round" + corners[2] = "round" + is_inside = False + if y == self.rows - 1: + corners[2] = "round" + corners[3] = "round" + is_inside = False + return corners, is_inside + + def make_radon_checkerboard_pattern(self): + spacing = self.square_size + xspacing = (self.width - self.cols * self.square_size) / 2.0 + yspacing = (self.height - self.rows * self.square_size) / 2.0 + for x in range(0, self.cols): + for y in range(0, self.rows): + if x % 2 == y % 2: + corner_types, is_inside = self._get_type(x, y) + if is_inside: + square = SVG("rect", x=x * spacing + xspacing, y=y * spacing + yspacing, width=spacing, + height=spacing, fill="black", stroke="none") + else: + square = SVG("path", d=self._make_round_rect(x * spacing + xspacing, y * spacing + yspacing, + spacing, corner_types), fill="black", stroke="none") + self.g.append(square) + if self.markers is not None: + r = self.square_size * 0.17 + pattern_width = ((self.cols - 1.0) * spacing) + (2.0 * r) + pattern_height = ((self.rows - 1.0) * spacing) + (2.0 * r) + x_spacing = (self.width - pattern_width) / 2.0 + y_spacing = (self.height - pattern_height) / 2.0 + for x, y in self.markers: + color = "black" + if x % 2 == y % 2: + color = "white" + dot = SVG("circle", cx=(x * spacing) + x_spacing + r, + cy=(y * spacing) + y_spacing + r, r=r, fill=color, stroke="none") + self.g.append(dot) + def save(self): c = canvas(self.g, width="%d%s" % (self.width, self.units), height="%d%s" % (self.height, self.units), viewBox="0 0 %d %d" % (self.width, self.height)) @@ -85,7 +155,7 @@ def main(): type=int) parser.add_argument("-r", "--rows", help="pattern rows", default="11", action="store", dest="rows", type=int) parser.add_argument("-T", "--type", help="type of pattern", default="circles", action="store", dest="p_type", - choices=["circles", "acircles", "checkerboard"]) + choices=["circles", "acircles", "checkerboard", "radon_checkerboard"]) parser.add_argument("-u", "--units", help="length unit", default="mm", action="store", dest="units", choices=["mm", "inches", "px", "m"]) parser.add_argument("-s", "--square_size", help="size of squares in pattern", default="20.0", action="store", @@ -96,8 +166,12 @@ def main(): dest="page_width", type=float) parser.add_argument("-h", "--page_height", help="page height in units", default=argparse.SUPPRESS, action="store", dest="page_height", type=float) - parser.add_argument("-a", "--page_size", help="page size, superseded if -h and -w are set", default="A4", action="store", - dest="page_size", choices=["A0", "A1", "A2", "A3", "A4", "A5"]) + parser.add_argument("-a", "--page_size", help="page size, superseded if -h and -w are set", default="A4", + action="store", dest="page_size", choices=["A0", "A1", "A2", "A3", "A4", "A5"]) + parser.add_argument("-m", "--markers", help="list of cells with markers for the radon checkerboard. Marker " + "coordinates as list of numbers: -m 1 2 3 4 means markers in cells " + "[1, 2] and [3, 4]", + action="store", dest="markers", nargs="+", type=int) args = parser.parse_args() show_help = args.show_help @@ -121,10 +195,19 @@ def main(): "A5": [148, 210]} page_width = page_sizes[page_size][0] page_height = page_sizes[page_size][1] - pm = PatternMaker(columns, rows, output, units, square_size, radius_rate, page_width, page_height) + if len(args.markers) % 2 == 1: + raise ValueError("The length of the markers array={} must be even".format(len(args.markers))) + markers = set() + for x, y in zip(args.markers[::2], args.markers[1::2]): + if x in range(0, columns) and y in range(0, rows): + markers.add((x, y)) + else: + raise ValueError("The marker {},{} is outside the checkerboard".format(x, y)) + + pm = PatternMaker(columns, rows, output, units, square_size, radius_rate, page_width, page_height, markers) # dict for easy lookup of pattern type mp = {"circles": pm.make_circles_pattern, "acircles": pm.make_acircles_pattern, - "checkerboard": pm.make_checkerboard_pattern} + "checkerboard": pm.make_checkerboard_pattern, "radon_checkerboard": pm.make_radon_checkerboard_pattern} mp[p_type]() # this should save pattern to output pm.save() diff --git a/doc/tutorials/calib3d/camera_calibration_pattern/camera_calibration_pattern.markdown b/doc/tutorials/calib3d/camera_calibration_pattern/camera_calibration_pattern.markdown index c87f9f95f847..b13f09756b15 100644 --- a/doc/tutorials/calib3d/camera_calibration_pattern/camera_calibration_pattern.markdown +++ b/doc/tutorials/calib3d/camera_calibration_pattern/camera_calibration_pattern.markdown @@ -36,6 +36,10 @@ create a circle board pattern in file acircleboard.svg with 7 rows, 5 columns an python gen_pattern.py -o acircleboard.svg --rows 7 --columns 5 --type acircles --square_size 10 --radius_rate 2 +create a radon checkerboard for findChessboardCornersSB() with markers in (7 4), (7 5), (8 5) cells: + + python gen_pattern.py -o radon_checkerboard.svg --rows 10 --columns 15 --type radon_checkerboard -s 12.1 -m 7 4 7 5 8 5 + If you want to change unit use -u option (mm inches, px, m) If you want to change page size use -w and -h options diff --git a/modules/calib3d/include/opencv2/calib3d.hpp b/modules/calib3d/include/opencv2/calib3d.hpp index 37c5e7089a34..198e405f6d2b 100644 --- a/modules/calib3d/include/opencv2/calib3d.hpp +++ b/modules/calib3d/include/opencv2/calib3d.hpp @@ -1489,6 +1489,8 @@ Sample usage of detecting and drawing chessboard corners: : the board to make the detection more robust in various environments. Otherwise, if there is no border and the background is dark, the outer black squares cannot be segmented properly and so the square grouping and ordering algorithm fails. + +Use gen_pattern.py (@ref tutorial_camera_calibration_pattern) to create checkerboard. */ CV_EXPORTS_W bool findChessboardCorners( InputArray image, Size patternSize, OutputArray corners, int flags = CALIB_CB_ADAPTIVE_THRESH + CALIB_CB_NORMALIZE_IMAGE ); @@ -1545,6 +1547,8 @@ transformation it is beneficial to use round corners for the field corners which are located on the outside of the board. The following figure illustrates a sample checkerboard optimized for the detection. However, any other checkerboard can be used as well. + +Use gen_pattern.py (@ref tutorial_camera_calibration_pattern) to create checkerboard. ![Checkerboard](pics/checkerboard_radon.png) */ CV_EXPORTS_AS(findChessboardCornersSBWithMeta) From 27df98721137959ac91436e3a59b06c915d5ee82 Mon Sep 17 00:00:00 2001 From: Giles Payne Date: Tue, 28 Sep 2021 20:31:07 +0900 Subject: [PATCH 227/376] Fix bug in initializers Rect2f(Point2f,Point2f) and Rect2d(Point2d,Point2d) --- modules/core/misc/objc/common/Rect2d.mm | 8 ++++---- modules/core/misc/objc/common/Rect2f.mm | 8 ++++---- modules/core/misc/objc/test/RectTest.swift | 24 ++++++++++++++++++++++ 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/modules/core/misc/objc/common/Rect2d.mm b/modules/core/misc/objc/common/Rect2d.mm index e0aa8b28447f..977294d66c10 100644 --- a/modules/core/misc/objc/common/Rect2d.mm +++ b/modules/core/misc/objc/common/Rect2d.mm @@ -64,10 +64,10 @@ - (instancetype)init { } - (instancetype)initWithPoint:(Point2d*)point1 point:(Point2d*)point2 { - int x = (point1.x < point2.x ? point1.x : point2.x); - int y = (point1.y < point2.y ? point1.y : point2.y); - int width = (point1.x > point2.x ? point1.x : point2.x) - x; - int height = (point1.y > point2.y ? point1.y : point2.y) - y; + double x = (point1.x < point2.x ? point1.x : point2.x); + double y = (point1.y < point2.y ? point1.y : point2.y); + double width = (point1.x > point2.x ? point1.x : point2.x) - x; + double height = (point1.y > point2.y ? point1.y : point2.y) - y; return [self initWithX:x y:y width:width height:height]; } diff --git a/modules/core/misc/objc/common/Rect2f.mm b/modules/core/misc/objc/common/Rect2f.mm index 243154fe1fff..2d6c326cc62b 100644 --- a/modules/core/misc/objc/common/Rect2f.mm +++ b/modules/core/misc/objc/common/Rect2f.mm @@ -64,10 +64,10 @@ - (instancetype)init { } - (instancetype)initWithPoint:(Point2f*)point1 point:(Point2f*)point2 { - int x = (point1.x < point2.x ? point1.x : point2.x); - int y = (point1.y < point2.y ? point1.y : point2.y); - int width = (point1.x > point2.x ? point1.x : point2.x) - x; - int height = (point1.y > point2.y ? point1.y : point2.y) - y; + float x = (point1.x < point2.x ? point1.x : point2.x); + float y = (point1.y < point2.y ? point1.y : point2.y); + float width = (point1.x > point2.x ? point1.x : point2.x) - x; + float height = (point1.y > point2.y ? point1.y : point2.y) - y; return [self initWithX:x y:y width:width height:height]; } diff --git a/modules/core/misc/objc/test/RectTest.swift b/modules/core/misc/objc/test/RectTest.swift index d8a4b4d98701..4b152e094609 100644 --- a/modules/core/misc/objc/test/RectTest.swift +++ b/modules/core/misc/objc/test/RectTest.swift @@ -101,6 +101,30 @@ class RectTest: OpenCVTestCase { XCTAssertEqual(1, r.height); } + func testRect2fPointPoint() { + let p1 = Point2f(x:4.3, y:4.1) + let p2 = Point2f(x:2.7, y:3.9) + + let r = Rect2f(point: p1, point: p2) + XCTAssertNotNil(r); + XCTAssertEqual(2.7, r.x); + XCTAssertEqual(3.9, r.y); + XCTAssertEqual(1.6, r.width, accuracy: OpenCVTestCase.FEPS); + XCTAssertEqual(0.2, r.height, accuracy: OpenCVTestCase.FEPS); + } + + func testRect2dPointPoint() { + let p1 = Point2d(x:4.7879839, y:4.9922311) + let p2 = Point2d(x:2.1213123, y:3.1122129) + + let r = Rect2d(point: p1, point: p2) + XCTAssertNotNil(r); + XCTAssertEqual(2.1213123, r.x); + XCTAssertEqual(3.1122129, r.y); + XCTAssertEqual(2.6666716, r.width, accuracy: OpenCVTestCase.EPS); + XCTAssertEqual(1.8800182, r.height, accuracy: OpenCVTestCase.EPS); + } + func testRectPointSize() { let p1 = Point(x: 4, y: 4) let sz = Size(width: 3, height: 1) From b25ad12f1aab0c37afa41a729db85bf97730cedc Mon Sep 17 00:00:00 2001 From: Dmitriy Fishman Date: Tue, 28 Sep 2021 15:29:47 +0300 Subject: [PATCH 228/376] Update video_input_psnr_ssim.markdown --- .../video-input-psnr-ssim/video_input_psnr_ssim.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown b/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown index ffd4d0213e6a..7bc856134794 100644 --- a/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown +++ b/doc/tutorials/videoio/video-input-psnr-ssim/video_input_psnr_ssim.markdown @@ -180,7 +180,7 @@ implementation below. This will return a similarity index for each channel of the image. This value is between zero and one, where one corresponds to perfect fit. Unfortunately, the many Gaussian blurring is quite -costly, so while the PSNR may work in a real time like environment (24 frame per second) this will +costly, so while the PSNR may work in a real time like environment (24 frames per second) this will take significantly more than to accomplish similar performance results. Therefore, the source code presented at the start of the tutorial will perform the PSNR measurement From 2f83c3b689423c74a296473057333cef20673109 Mon Sep 17 00:00:00 2001 From: Cavendish-Koo <1020057909@qq.com> Date: Tue, 28 Sep 2021 21:18:07 +0800 Subject: [PATCH 229/376] fix the bug of HoughlinesSDIV --- modules/imgproc/src/hough.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/imgproc/src/hough.cpp b/modules/imgproc/src/hough.cpp index bbb6dfcbdb44..e66be8b4b700 100644 --- a/modules/imgproc/src/hough.cpp +++ b/modules/imgproc/src/hough.cpp @@ -435,12 +435,14 @@ HoughLinesSDiv( InputArray image, OutputArray lines, int type, } } + int pos = (int)(lst.size() - 1); + if( pos >= 0 && lst[pos].rho < 0 ) + lst.pop_back(); + lines.create((int)lst.size(), 1, type); Mat _lines = lines.getMat(); for( size_t idx = 0; idx < lst.size(); idx++ ) { - if( lst[idx].rho < 0 ) - continue; if (type == CV_32FC2) { _lines.at((int)idx) = Vec2f(lst[idx].rho, lst[idx].theta); From c1148c4ea6c800730e1bb31fa3032f3fbe4c8c4a Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Tue, 28 Sep 2021 18:02:21 +0300 Subject: [PATCH 230/376] Merge pull request #20739 from sivanov-work:merge_base_decode G-API: oneVPL (simplification) Add simple decode pipeline * Add simple decode pipeline & add onevpl namespace * Address some review comments * Add compilation guard --- modules/gapi/CMakeLists.txt | 4 + .../onevpl/accelerators/accel_policy_cpu.cpp | 2 + .../onevpl/accelerators/accel_policy_cpu.hpp | 2 + .../onevpl/accelerators/accel_policy_dx11.cpp | 7 +- .../onevpl/accelerators/accel_policy_dx11.hpp | 2 + .../accelerators/accel_policy_interface.hpp | 2 + .../surface/cpu_frame_adapter.cpp | 2 + .../surface/cpu_frame_adapter.hpp | 2 + .../onevpl/accelerators/surface/surface.cpp | 2 + .../onevpl/accelerators/surface/surface.hpp | 2 + .../accelerators/surface/surface_pool.cpp | 2 + .../accelerators/surface/surface_pool.hpp | 2 + .../engine/decode/decode_engine_legacy.cpp | 310 ++++++++++++++++++ .../engine/decode/decode_engine_legacy.hpp | 48 +++ .../onevpl/engine/decode/decode_session.cpp | 78 +++++ .../onevpl/engine/decode/decode_session.hpp | 60 ++++ .../onevpl/engine/engine_session.cpp | 33 ++ .../onevpl/engine/engine_session.hpp | 49 +++ .../onevpl/engine/processing_engine_base.cpp | 134 ++++++++ .../onevpl/engine/processing_engine_base.hpp | 96 ++++++ modules/gapi/src/streaming/onevpl/utils.hpp | 38 +++ .../gapi_streaming_vpl_core_test.cpp | 130 +++++++- 22 files changed, 992 insertions(+), 15 deletions(-) create mode 100644 modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp create mode 100644 modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp create mode 100644 modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp create mode 100644 modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp create mode 100644 modules/gapi/src/streaming/onevpl/engine/engine_session.cpp create mode 100644 modules/gapi/src/streaming/onevpl/engine/engine_session.hpp create mode 100644 modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp create mode 100644 modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp create mode 100644 modules/gapi/src/streaming/onevpl/utils.hpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 380f1d7d8456..5fd108c0ec01 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -174,6 +174,10 @@ set(gapi_srcs src/streaming/onevpl/accelerators/surface/surface_pool.cpp src/streaming/onevpl/accelerators/accel_policy_cpu.cpp src/streaming/onevpl/accelerators/accel_policy_dx11.cpp + src/streaming/onevpl/engine/engine_session.cpp + src/streaming/onevpl/engine/processing_engine_base.cpp + src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp + src/streaming/onevpl/engine/decode/decode_session.cpp # Utils (ITT tracing) src/utils/itt.cpp diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp index 0a5f68ae4e6a..75cb2fdb3870 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp @@ -20,6 +20,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { VPLCPUAccelerationPolicy::VPLCPUAccelerationPolicy() { GAPI_LOG_INFO(nullptr, "created"); @@ -219,6 +220,7 @@ cv::MediaFrame::AdapterPtr VPLCPUAccelerationPolicy::create_frame_adapter(pool_k return cv::MediaFrame::AdapterPtr{new VPLMediaFrameCPUAdapter(*it)}; #endif // TEST_PERF } +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp index cfe30573159c..dc5a9347b5f8 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp @@ -22,6 +22,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { // GAPI_EXPORTS for tests struct GAPI_EXPORTS VPLCPUAccelerationPolicy final : public VPLAccelerationPolicy @@ -47,6 +48,7 @@ struct GAPI_EXPORTS VPLCPUAccelerationPolicy final : public VPLAccelerationPolic private: std::map pool_table; }; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp index 348b864a15a3..cb27df86619b 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp @@ -8,6 +8,7 @@ #include "streaming/onevpl/accelerators/accel_policy_dx11.hpp" #include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" #include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/utils.hpp" #include "logger.hpp" #ifdef HAVE_DIRECTX @@ -27,6 +28,8 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { + VPLDX11AccelerationPolicy::VPLDX11AccelerationPolicy() { #ifdef CPU_ACCEL_ADAPTER @@ -47,7 +50,8 @@ void VPLDX11AccelerationPolicy::init(session_t session) { mfxStatus sts = MFXVideoCORE_GetHandle(session, MFX_HANDLE_D3D11_DEVICE, reinterpret_cast(&hw_handle)); if (sts != MFX_ERR_NONE) { - throw std::logic_error("Cannot create VPLDX11AccelerationPolicy, MFXVideoCORE_GetHandle error"); + throw std::logic_error("Cannot create VPLDX11AccelerationPolicy, MFXVideoCORE_GetHandle error: " + + mfxstatus_to_string(sts)); } GAPI_LOG_INFO(nullptr, "VPLDX11AccelerationPolicy initialized, session: " << session); @@ -106,6 +110,7 @@ cv::MediaFrame::AdapterPtr VPLDX11AccelerationPolicy::create_frame_adapter(pool_ (void)surface; throw std::runtime_error("VPLDX11AccelerationPolicy::create_frame_adapter() is not implemented"); } +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp index 04970432c5a5..a875f57085f5 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp @@ -32,6 +32,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { // GAPI_EXPORTS for tests struct GAPI_EXPORTS VPLDX11AccelerationPolicy final: public VPLAccelerationPolicy @@ -57,6 +58,7 @@ struct GAPI_EXPORTS VPLDX11AccelerationPolicy final: public VPLAccelerationPolic std::unique_ptr adapter; #endif }; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp index 87b1246d257e..31ee91535c9a 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp @@ -19,6 +19,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { class Surface; struct VPLAccelerationPolicy @@ -51,6 +52,7 @@ struct VPLAccelerationPolicy virtual cv::MediaFrame::AdapterPtr create_frame_adapter(pool_key_t key, mfxFrameSurface1* surface) = 0; }; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp index 3fd959ca8b46..d3020ab16888 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp @@ -19,6 +19,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { VPLMediaFrameCPUAdapter::VPLMediaFrameCPUAdapter(std::shared_ptr surface): parent_surface_ptr(surface) { @@ -111,6 +112,7 @@ void VPLMediaFrameCPUAdapter::serialize(cv::gapi::s11n::IOStream&) { void VPLMediaFrameCPUAdapter::deserialize(cv::gapi::s11n::IIStream&) { GAPI_Assert("VPLMediaFrameCPUAdapter::deserialize() is not implemented"); } +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp index 16111dadafe3..04a9bdc2751d 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp @@ -16,6 +16,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { class Surface; class VPLMediaFrameCPUAdapter : public cv::MediaFrame::IAdapter { @@ -33,6 +34,7 @@ class VPLMediaFrameCPUAdapter : public cv::MediaFrame::IAdapter { private: std::shared_ptr parent_surface_ptr; }; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp index 5d315412e1be..3f5fd00305af 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp @@ -12,6 +12,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { Surface::Surface(std::unique_ptr&& surf, std::shared_ptr associated_memory) : workspace_memory_ptr(associated_memory), @@ -69,6 +70,7 @@ size_t Surface::release_lock() { ", locked times: " << locked_count - 1); return locked_count; // return preceding value } +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp index 3fc30099cf49..828e5cb1c706 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp @@ -23,6 +23,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { /** * @brief Inner class for managing oneVPL surface through interface `mfxFrameSurface1`. @@ -95,6 +96,7 @@ class Surface { using surface_ptr_t = std::shared_ptr; using surface_weak_ptr_t = std::weak_ptr; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp index 8ead7965b471..729b37f7ea51 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp @@ -7,6 +7,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { void CachedPool::reserve(size_t size) { surfaces.reserve(size); @@ -63,6 +64,7 @@ CachedPool::surface_ptr_t CachedPool::find_by_handle(mfxFrameSurface1* handle) { GAPI_Assert(it != cache.end() && "Cannot find cached surface from pool. Data corruption is possible"); return it->second; } +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp index 2059e9b27e8e..029fa350db6e 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp @@ -17,6 +17,7 @@ namespace cv { namespace gapi { namespace wip { +namespace onevpl { class Surface; // GAPI_EXPORTS for tests @@ -38,6 +39,7 @@ class GAPI_EXPORTS CachedPool { free_surface_iterator_t next_free_it; cached_surface_container_t cache; }; +} // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp new file mode 100644 index 000000000000..ad4a4eca8ec7 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp @@ -0,0 +1,310 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL + +#include +#include + +#include + +#include "streaming/onevpl/engine/decode/decode_engine_legacy.hpp" +#include "streaming/onevpl/engine/decode/decode_session.hpp" +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/utils.hpp" +#include "logger.hpp" + + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +/* UTILS */ +mfxU32 GetSurfaceSize(mfxU32 FourCC, mfxU32 width, mfxU32 height) { + mfxU32 nbytes = 0; + + mfxU32 half_width = width / 2; + mfxU32 half_height = height / 2; + switch (FourCC) { + case MFX_FOURCC_I420: + case MFX_FOURCC_NV12: + nbytes = width * height + 2 * half_width * half_height; + break; + case MFX_FOURCC_I010: + case MFX_FOURCC_P010: + nbytes = width * height + 2 * half_width * half_height; + nbytes *= 2; + break; + case MFX_FOURCC_RGB4: + nbytes = width * height * 4; + break; + default: + GAPI_LOG_WARNING(nullptr, "Unsupported FourCC requested: " << FourCC); + GAPI_Assert(false && "Unsupported FourCC requested"); + break; + } + return nbytes; +} + +surface_ptr_t create_surface_RGB4(mfxFrameInfo frameInfo, + std::shared_ptr out_buf_ptr, + size_t out_buf_ptr_offset, + size_t out_buf_size) +{ + mfxU8* buf = reinterpret_cast(out_buf_ptr.get()); + mfxU16 surfW = frameInfo.Width * 4; + mfxU16 surfH = frameInfo.Height; + (void)surfH; + + // TODO more intelligent check + if (out_buf_size <= out_buf_ptr_offset) { + throw std::runtime_error(std::string("Insufficient buffer size: ") + + std::to_string(out_buf_size) + ", buffer offset: " + + std::to_string(out_buf_ptr_offset) + + ", expected surface width: " + std::to_string(surfW) + + ", height: " + std::to_string(surfH)); + } + + std::unique_ptr handle(new mfxFrameSurface1); + memset(handle.get(), 0, sizeof(mfxFrameSurface1)); + + handle->Info = frameInfo; + handle->Data.B = buf + out_buf_ptr_offset; + handle->Data.G = handle->Data.B + 1; + handle->Data.R = handle->Data.B + 2; + handle->Data.A = handle->Data.B + 3; + handle->Data.Pitch = surfW; + + return Surface::create_surface(std::move(handle), out_buf_ptr); +} + +surface_ptr_t create_surface_other(mfxFrameInfo frameInfo, + std::shared_ptr out_buf_ptr, + size_t out_buf_ptr_offset, + size_t out_buf_size) +{ + mfxU8* buf = reinterpret_cast(out_buf_ptr.get()); + mfxU16 surfH = frameInfo.Height; + mfxU16 surfW = (frameInfo.FourCC == MFX_FOURCC_P010) ? frameInfo.Width * 2 : frameInfo.Width; + + // TODO more intelligent check + if (out_buf_size <= + out_buf_ptr_offset + (surfW * surfH) + ((surfW / 2) * (surfH / 2))) { + throw std::runtime_error(std::string("Insufficient buffer size: ") + + std::to_string(out_buf_size) + ", buffer offset: " + + std::to_string(out_buf_ptr_offset) + + ", expected surface width: " + std::to_string(surfW) + + ", height: " + std::to_string(surfH)); + } + + std::unique_ptr handle(new mfxFrameSurface1); + memset(handle.get(), 0, sizeof(mfxFrameSurface1)); + + handle->Info = frameInfo; + handle->Data.Y = buf + out_buf_ptr_offset; + handle->Data.U = buf + out_buf_ptr_offset + (surfW * surfH); + handle->Data.V = handle->Data.U + ((surfW / 2) * (surfH / 2)); + handle->Data.Pitch = surfW; + + return Surface::create_surface(std::move(handle), out_buf_ptr); +} + +VPLLegacyDecodeEngine::VPLLegacyDecodeEngine(std::unique_ptr&& accel) + : ProcessingEngineBase(std::move(accel)) { + + GAPI_LOG_INFO(nullptr, "Create Legacy Decode Engine"); + create_pipeline( + // 1) Read File + [this] (EngineSession& sess) -> ExecutionStatus + { + LegacyDecodeSession &my_sess = static_cast(sess); + my_sess.last_status = ReadEncodedStream(my_sess.stream, my_sess.data_provider); + if (my_sess.last_status != MFX_ERR_NONE) { + my_sess.data_provider.reset(); //close source + } + return ExecutionStatus::Continue; + }, + // 2) enqueue ASYNC decode + [this] (EngineSession& sess) -> ExecutionStatus + { + LegacyDecodeSession &my_sess = static_cast(sess); + + my_sess.last_status = + MFXVideoDECODE_DecodeFrameAsync(my_sess.session, + my_sess.last_status == MFX_ERR_NONE + ? &my_sess.stream + : nullptr, /* No more data to read, start decode draining mode*/ + my_sess.procesing_surface_ptr.lock()->get_handle(), + &my_sess.output_surface_ptr, + &my_sess.sync); + return ExecutionStatus::Continue; + }, + // 3) Wait for ASYNC decode result + [this] (EngineSession& sess) -> ExecutionStatus + { + if (sess.last_status == MFX_ERR_NONE) // Got 1 decoded frame + { + do { + //TODO try to extract TIMESTAMP + sess.last_status = MFXVideoCORE_SyncOperation(sess.session, sess.sync, 100); + if (MFX_ERR_NONE == sess.last_status) { + + LegacyDecodeSession& my_sess = static_cast(sess); + on_frame_ready(my_sess); + } + } while (sess.last_status == MFX_WRN_IN_EXECUTION); + } + return ExecutionStatus::Continue; + }, + // 4) Falls back on generic status procesing + [this] (EngineSession& sess) -> ExecutionStatus + { + return this->process_error(sess.last_status, static_cast(sess)); + } + ); +} + +void VPLLegacyDecodeEngine::initialize_session(mfxSession mfx_session, + DecoderParams&& decoder_param, + std::shared_ptr provider) +{ + mfxFrameAllocRequest decRequest = {}; + // Query number required surfaces for decoder + MFXVideoDECODE_QueryIOSurf(mfx_session, &decoder_param.param, &decRequest); + + // External (application) allocation of decode surfaces + GAPI_LOG_DEBUG(nullptr, "Query IOSurf for session: " << mfx_session << + ", mfxFrameAllocRequest.NumFrameSuggested: " << decRequest.NumFrameSuggested << + ", mfxFrameAllocRequest.Type: " << decRequest.Type); + + mfxU32 singleSurfaceSize = GetSurfaceSize(decoder_param.param.mfx.FrameInfo.FourCC, + decoder_param.param.mfx.FrameInfo.Width, + decoder_param.param.mfx.FrameInfo.Height); + if (!singleSurfaceSize) { + throw std::runtime_error("Cannot determine surface size for: fourCC" + + std::to_string(decoder_param.param.mfx.FrameInfo.FourCC) + + ", width: " + std::to_string(decoder_param.param.mfx.FrameInfo.Width) + + ", height: " + std::to_string(decoder_param.param.mfx.FrameInfo.Height)); + } + + const auto &frameInfo = decoder_param.param.mfx.FrameInfo; + auto surface_creator = + [&frameInfo] (std::shared_ptr out_buf_ptr, size_t out_buf_ptr_offset, + size_t out_buf_size) -> surface_ptr_t { + return (frameInfo.FourCC == MFX_FOURCC_RGB4) ? + create_surface_RGB4(frameInfo, out_buf_ptr, out_buf_ptr_offset, + out_buf_size) : + create_surface_other(frameInfo, out_buf_ptr, out_buf_ptr_offset, + out_buf_size);}; + + //TODO Configure preallocation size (how many frames we can hold) + const size_t preallocated_frames_count = 30; + VPLAccelerationPolicy::pool_key_t decode_pool_key = + acceleration_policy->create_surface_pool(decRequest.NumFrameSuggested * preallocated_frames_count, + singleSurfaceSize, + surface_creator); + + // create session + std::shared_ptr sess_ptr = + register_session(mfx_session, + std::move(decoder_param), + provider); + + sess_ptr->init_surface_pool(decode_pool_key); + // prepare working decode surface + sess_ptr->swap_surface(*this); +} + +ProcessingEngineBase::ExecutionStatus VPLLegacyDecodeEngine::execute_op(operation_t& op, EngineSession& sess) { + return op(sess); +} + +void VPLLegacyDecodeEngine::on_frame_ready(LegacyDecodeSession& sess) +{ + GAPI_LOG_DEBUG(nullptr, "[" << sess.session << "], frame ready"); + + // manage memory ownership rely on acceleration policy + auto frame_adapter = acceleration_policy->create_frame_adapter(sess.decoder_pool_id, + sess.output_surface_ptr); + ready_frames.emplace(cv::MediaFrame(std::move(frame_adapter)), sess.generate_frame_meta()); +} + +ProcessingEngineBase::ExecutionStatus VPLLegacyDecodeEngine::process_error(mfxStatus status, LegacyDecodeSession& sess) +{ + GAPI_LOG_DEBUG(nullptr, "status: " << mfxstatus_to_string(status)); + + switch (status) { + case MFX_ERR_NONE: + return ExecutionStatus::Continue; + case MFX_ERR_MORE_DATA: // The function requires more bitstream at input before decoding can proceed + if (!sess.data_provider || sess.data_provider->empty()) { + // No more data to drain from decoder, start encode draining mode + return ExecutionStatus::Processed; + } + else + return ExecutionStatus::Continue; // read more data + break; + case MFX_ERR_MORE_SURFACE: + { + // The function requires more frame surface at output before decoding can proceed. + // This applies to external memory allocations and should not be expected for + // a simple internal allocation case like this + try { + sess.swap_surface(*this); + return ExecutionStatus::Continue; + } catch (const std::exception& ex) { + GAPI_LOG_WARNING(nullptr, "[" << sess.session << "] error: " << ex.what()); + } + break; + } + case MFX_ERR_DEVICE_LOST: + // For non-CPU implementations, + // Cleanup if device is lost + GAPI_DbgAssert(false && "VPLLegacyDecodeEngine::process_error - " + "MFX_ERR_DEVICE_LOST is not processed"); + break; + case MFX_WRN_DEVICE_BUSY: + // For non-CPU implementations, + // Wait a few milliseconds then try again + GAPI_DbgAssert(false && "VPLLegacyDecodeEngine::process_error - " + "MFX_WRN_DEVICE_BUSY is not processed"); + break; + case MFX_WRN_VIDEO_PARAM_CHANGED: + // The decoder detected a new sequence header in the bitstream. + // Video parameters may have changed. + // In external memory allocation case, might need to reallocate the output surface + GAPI_DbgAssert(false && "VPLLegacyDecodeEngine::process_error - " + "MFX_WRN_VIDEO_PARAM_CHANGED is not processed"); + break; + case MFX_ERR_INCOMPATIBLE_VIDEO_PARAM: + // The function detected that video parameters provided by the application + // are incompatible with initialization parameters. + // The application should close the component and then reinitialize it + GAPI_DbgAssert(false && "VPLLegacyDecodeEngine::process_error - " + "MFX_ERR_INCOMPATIBLE_VIDEO_PARAM is not processed"); + break; + case MFX_ERR_REALLOC_SURFACE: + // Bigger surface_work required. May be returned only if + // mfxInfoMFX::EnableReallocRequest was set to ON during initialization. + // This applies to external memory allocations and should not be expected for + // a simple internal allocation case like this + GAPI_DbgAssert(false && "VPLLegacyDecodeEngine::process_error - " + "MFX_ERR_REALLOC_SURFACE is not processed"); + break; + default: + GAPI_LOG_WARNING(nullptr, "Unknown status code: " << mfxstatus_to_string(status)); + break; + } + + return ExecutionStatus::Failed; +} + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp new file mode 100644 index 000000000000..5db54c31992c --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp @@ -0,0 +1,48 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONVPL_DECODE_ENGINE_LEGACY_HPP +#define GAPI_STREAMING_ONVPL_DECODE_ENGINE_LEGACY_HPP +#include +#include + +#include "streaming/onevpl/engine/processing_engine_base.hpp" + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) + #include +#endif +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +class LegacyDecodeSession; +struct DecoderParams; +struct IDataProvider; +struct VPLAccelerationPolicy; + +class VPLLegacyDecodeEngine : public ProcessingEngineBase { +public: + + VPLLegacyDecodeEngine(std::unique_ptr&& accel); + void initialize_session(mfxSession mfx_session, DecoderParams&& decoder_param, + std::shared_ptr provider) override; + +private: + ExecutionStatus execute_op(operation_t& op, EngineSession& sess) override; + ExecutionStatus process_error(mfxStatus status, LegacyDecodeSession& sess); + + void on_frame_ready(LegacyDecodeSession& sess); +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONVPL_DECODE_ENGINE_LEGACY_HPP diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp new file mode 100644 index 000000000000..3468869e1567 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp @@ -0,0 +1,78 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL + +#include +#include + +#include "streaming/onevpl/engine/decode/decode_session.hpp" +#include "streaming/onevpl/engine/decode/decode_engine_legacy.hpp" +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/utils.hpp" + +#include "logger.hpp" +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +LegacyDecodeSession::LegacyDecodeSession(mfxSession sess, + DecoderParams&& decoder_param, + std::shared_ptr provider) : + EngineSession(sess, std::move(decoder_param.stream)), + mfx_decoder_param(std::move(decoder_param.param)), + data_provider(std::move(provider)), + procesing_surface_ptr(), + output_surface_ptr(), + decoded_frames_count() +{ +} + +LegacyDecodeSession::~LegacyDecodeSession() +{ + GAPI_LOG_INFO(nullptr, "Close Decode for session: " << session); + MFXVideoDECODE_Close(session); +} + +void LegacyDecodeSession::swap_surface(VPLLegacyDecodeEngine& engine) { + VPLAccelerationPolicy* acceleration_policy = engine.get_accel(); + GAPI_Assert(acceleration_policy && "Empty acceleration_policy"); + auto old_locked = procesing_surface_ptr.lock(); + try { + auto cand = acceleration_policy->get_free_surface(decoder_pool_id).lock(); + + GAPI_LOG_DEBUG(nullptr, "[" << session << "] swap surface" + ", old: " << (old_locked ? old_locked->get_handle() : nullptr) << + ", new: "<< cand->get_handle()); + + procesing_surface_ptr = cand; + } catch (const std::exception& ex) { + GAPI_LOG_WARNING(nullptr, "[" << session << "] error: " << ex.what() << + "Abort"); + } +} + +void LegacyDecodeSession::init_surface_pool(VPLAccelerationPolicy::pool_key_t key) { + GAPI_Assert(key && "Init decode pull with empty key"); + decoder_pool_id = key; +} + +Data::Meta LegacyDecodeSession::generate_frame_meta() { + const auto now = std::chrono::system_clock::now(); + const auto dur = std::chrono::duration_cast + (now.time_since_epoch()); + Data::Meta meta { + {cv::gapi::streaming::meta_tag::timestamp, int64_t{dur.count()} }, + {cv::gapi::streaming::meta_tag::seq_id, int64_t{decoded_frames_count++}} + }; + return meta; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp new file mode 100644 index 000000000000..46b1decc81cd --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp @@ -0,0 +1,60 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONVPL_ENGINE_DECODE_DECODE_SESSION_HPP +#define GAPI_STREAMING_ONVPL_ENGINE_DECODE_DECODE_SESSION_HPP +#include +#include + +#include + +#include "streaming/onevpl/engine/engine_session.hpp" +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) + #include +#endif +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +struct IDataProvider; +class Surface; +struct VPLAccelerationPolicy; + +class LegacyDecodeSession : public EngineSession { +public: + friend class VPLLegacyDecodeEngine; + + LegacyDecodeSession(mfxSession sess, DecoderParams&& decoder_param, std::shared_ptr provider); + ~LegacyDecodeSession(); + using EngineSession::EngineSession; + + void swap_surface(VPLLegacyDecodeEngine& engine); + void init_surface_pool(VPLAccelerationPolicy::pool_key_t key); + + mfxVideoParam mfx_decoder_param; + std::shared_ptr data_provider; + + Data::Meta generate_frame_meta(); +private: + VPLAccelerationPolicy::pool_key_t decoder_pool_id; + mfxFrameAllocRequest request; + + std::weak_ptr procesing_surface_ptr; + mfxFrameSurface1* output_surface_ptr; + + int64_t decoded_frames_count; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONVPL_ENGINE_DECODE_DECODE_SESSION_HPP diff --git a/modules/gapi/src/streaming/onevpl/engine/engine_session.cpp b/modules/gapi/src/streaming/onevpl/engine/engine_session.cpp new file mode 100644 index 000000000000..9f8028361a90 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/engine_session.cpp @@ -0,0 +1,33 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation +#ifdef HAVE_ONEVPL + +#include "streaming/onevpl/engine/engine_session.hpp" +#include "streaming/onevpl/utils.hpp" +#include "logger.hpp" + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +EngineSession::EngineSession(mfxSession sess, mfxBitstream&& str) : + session(sess), stream(std::move(str)) {} +EngineSession::~EngineSession() +{ + GAPI_LOG_INFO(nullptr, "Close session: " << session); + MFXClose(session); +} + +std::string EngineSession::error_code_to_str() const +{ + return mfxstatus_to_string(last_status); +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp b/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp new file mode 100644 index 000000000000..d8f2f6b31246 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp @@ -0,0 +1,49 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ENGINE_ENGINE_SESSION_HPP +#define GAPI_STREAMING_ONEVPL_ENGINE_ENGINE_SESSION_HPP + +#include +#include +#include +#include +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +// GAPI_EXPORTS for tests +struct GAPI_EXPORTS DecoderParams { + mfxBitstream stream; + mfxVideoParam param; +}; + +struct GAPI_EXPORTS EngineSession { + mfxSession session; + mfxBitstream stream; + mfxSyncPoint sync; + mfxStatus last_status; + + EngineSession(mfxSession sess, mfxBitstream&& str); + std::string error_code_to_str() const; + virtual ~EngineSession(); +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ENGINE_ENGINE_SESSION_HPP diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp new file mode 100644 index 000000000000..3161b1627e8a --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp @@ -0,0 +1,134 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL + +#include + +#include +#include "streaming/onevpl/engine/processing_engine_base.hpp" +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#include "logger.hpp" + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +ProcessingEngineBase::ProcessingEngineBase(std::unique_ptr&& accel) : + acceleration_policy(std::move(accel)) { +} + +ProcessingEngineBase::~ProcessingEngineBase() { + GAPI_LOG_INFO(nullptr, "destroyed"); +} + +ProcessingEngineBase::ExecutionStatus ProcessingEngineBase::process(mfxSession session) { + auto sess_it = sessions.find(session); + if (sess_it == sessions.end()) { + return ExecutionStatus::SessionNotFound; + } + + session_ptr processing_session = sess_it->second; + ExecutionData& exec_data = execution_table[session]; + + GAPI_LOG_DEBUG(nullptr, "[" << session <<"] start op id: " << exec_data.op_id); + ExecutionStatus status = execute_op(pipeline.at(exec_data.op_id), *processing_session); + size_t old_op_id = exec_data.op_id++; + if (exec_data.op_id == pipeline.size()) + { + exec_data.op_id = 0; + } + GAPI_LOG_DEBUG(nullptr, "[" << session <<"] finish op id: " << old_op_id << + ", " << processing_session->error_code_to_str() << + ", " << ProcessingEngineBase::status_to_string(status) << + ", next op id: " << exec_data.op_id); + + if (status == ExecutionStatus::Failed) { + + GAPI_LOG_WARNING(nullptr, "Operation for session: " << session << + ", " << ProcessingEngineBase::status_to_string(status) << + " - remove it"); + sessions.erase(sess_it); + execution_table.erase(session); + } + + if (status == ExecutionStatus::Processed) { + sessions.erase(sess_it); + execution_table.erase(session); + } + + return status; +} + +const char* ProcessingEngineBase::status_to_string(ExecutionStatus status) +{ + switch(status) { + case ExecutionStatus::Continue: return "CONTINUE"; + case ExecutionStatus::Processed: return "PROCESSED"; + case ExecutionStatus::SessionNotFound: return "NOT_FOUND_SESSION"; + case ExecutionStatus::Failed: return "FAILED"; + default: + return "UNKNOWN"; + } +} + +ProcessingEngineBase::ExecutionStatus ProcessingEngineBase::execute_op(operation_t& op, EngineSession& sess) +{ + return op(sess); +} + +size_t ProcessingEngineBase::get_ready_frames_count() const +{ + return ready_frames.size(); +} + +void ProcessingEngineBase::get_frame(Data &data) +{ + data = ready_frames.front(); + ready_frames.pop(); +} + +const VPLAccelerationPolicy* ProcessingEngineBase::get_accel() const { + return acceleration_policy.get(); +} + +VPLAccelerationPolicy* ProcessingEngineBase::get_accel() { + return const_cast(static_cast(this)->get_accel()); +} + + +// Read encoded stream from file +mfxStatus ReadEncodedStream(mfxBitstream &bs, std::shared_ptr& data_provider) { + + if (!data_provider) { + return MFX_ERR_MORE_DATA; + } + + mfxU8 *p0 = bs.Data; + mfxU8 *p1 = bs.Data + bs.DataOffset; + if (bs.DataOffset > bs.MaxLength - 1) { + return MFX_ERR_NOT_ENOUGH_BUFFER; + } + if (bs.DataLength + bs.DataOffset > bs.MaxLength) { + return MFX_ERR_NOT_ENOUGH_BUFFER; + } + + std::copy_n(p0, bs.DataLength, p1); + + bs.DataOffset = 0; + bs.DataLength += static_cast(data_provider->fetch_data(bs.MaxLength - bs.DataLength, + bs.Data + bs.DataLength)); + if (bs.DataLength == 0) + return MFX_ERR_MORE_DATA; + + return MFX_ERR_NONE; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp new file mode 100644 index 000000000000..7d4869ac66f6 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp @@ -0,0 +1,96 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ENGINE_PROCESSING_ENGINE_BASE_HPP +#define GAPI_STREAMING_ONEVPL_ENGINE_PROCESSING_ENGINE_BASE_HPP + +#include +#include "streaming/onevpl/engine/engine_session.hpp" +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +struct VPLAccelerationPolicy; +struct IDataProvider; + +// GAPI_EXPORTS for tests +class GAPI_EXPORTS ProcessingEngineBase { +public: + enum class ExecutionStatus { + Continue, + Processed, + SessionNotFound, + Failed + }; + struct ExecutionData { + size_t op_id = 0; + }; + + using file_ptr = std::unique_ptr; + + using session_ptr = std::shared_ptr; + using SessionsTable = std::map; + using ExecutionDataTable = std::map; + + using frame_t = cv::gapi::wip::Data; + using frames_container_t = std::queue; + using operation_t = std::function; + + static const char * status_to_string(ExecutionStatus); + + ProcessingEngineBase(std::unique_ptr&& accel); + virtual ~ProcessingEngineBase(); + + virtual void initialize_session(mfxSession mfx_session, + DecoderParams&& decoder_param, + std::shared_ptr provider) = 0; + + ExecutionStatus process(mfxSession session); + size_t get_ready_frames_count() const; + void get_frame(Data &data); + + const VPLAccelerationPolicy* get_accel() const; + VPLAccelerationPolicy* get_accel(); +protected: + SessionsTable sessions; + frames_container_t ready_frames; + ExecutionDataTable execution_table; + + std::vector pipeline; + std::unique_ptr acceleration_policy; + + virtual ExecutionStatus execute_op(operation_t& op, EngineSession& sess); + + template + void create_pipeline(Ops&&...ops) + { + GAPI_DbgAssert(pipeline.empty() && "Pipeline must be empty"); + std::vector({std::forward(ops)...}).swap(pipeline); + } + + template + std::shared_ptr register_session(mfxSession key, + SessionArgs&& ...args) + { + auto sess_impl = std::make_shared(key, + std::forward(args)...); + sessions.emplace(key, sess_impl); + execution_table.emplace(key, ExecutionData{}); + return sess_impl; + } +}; + + +mfxStatus ReadEncodedStream(mfxBitstream &bs, std::shared_ptr& data_provider); +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // GAPI_STREAMING_ONEVPL_ENGINE_PROCESSING_ENGINE_BASE_HPP diff --git a/modules/gapi/src/streaming/onevpl/utils.hpp b/modules/gapi/src/streaming/onevpl/utils.hpp new file mode 100644 index 000000000000..0512c4f68734 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/utils.hpp @@ -0,0 +1,38 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ONEVPL_UTILS_HPP +#define GAPI_STREAMING_ONEVPL_ONEVPL_UTILS_HPP + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif // MFX_VERSION + +#include +#include + +#include +#include + +#include + + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +inline std::string mfxstatus_to_string(mfxStatus) { + return "UNKNOWN"; +} + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ONEVPL_UTILS_HPP diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp index 5a36a2befb8c..0b8822b366a0 100644 --- a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -29,23 +30,88 @@ #include #ifdef HAVE_ONEVPL +#include + #include "streaming/onevpl/accelerators/surface/surface.hpp" #include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" #include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" +#include "streaming/onevpl/engine/processing_engine_base.hpp" +#include "streaming/onevpl/engine/engine_session.hpp" namespace opencv_test { namespace { -cv::gapi::wip::surface_ptr_t create_test_surface(std::shared_ptr out_buf_ptr, + +struct EmptyDataProvider : public cv::gapi::wip::onevpl::IDataProvider { + + size_t fetch_data(size_t, void*) override { + return 0; + } + bool empty() const override { + return true; + } +}; + +struct TestProcessingSession : public cv::gapi::wip::onevpl::EngineSession { + TestProcessingSession(mfxSession mfx_session) : + EngineSession(mfx_session, {}) { + } +}; + +struct TestProcessingEngine: public cv::gapi::wip::onevpl::ProcessingEngineBase { + + size_t pipeline_stage_num = 0; + + TestProcessingEngine(std::unique_ptr&& accel) : + cv::gapi::wip::onevpl::ProcessingEngineBase(std::move(accel)) { + using cv::gapi::wip::onevpl::EngineSession; + create_pipeline( + // 0) + [this] (EngineSession&) -> ExecutionStatus + { + pipeline_stage_num = 0; + return ExecutionStatus::Continue; + }, + // 1) + [this] (EngineSession&) -> ExecutionStatus + { + pipeline_stage_num = 1; + return ExecutionStatus::Continue; + }, + // 2) + [this] (EngineSession&) -> ExecutionStatus + { + pipeline_stage_num = 2; + return ExecutionStatus::Continue; + }, + // 3) + [this] (EngineSession&) -> ExecutionStatus + { + pipeline_stage_num = 3; + ready_frames.emplace(cv::MediaFrame()); + return ExecutionStatus::Processed; + } + ); + } + + void initialize_session(mfxSession mfx_session, + cv::gapi::wip::onevpl::DecoderParams&&, + std::shared_ptr) override { + + register_session(mfx_session); + } +}; + +cv::gapi::wip::onevpl::surface_ptr_t create_test_surface(std::shared_ptr out_buf_ptr, size_t, size_t) { std::unique_ptr handle(new mfxFrameSurface1{}); - return cv::gapi::wip::Surface::create_surface(std::move(handle), out_buf_ptr); + return cv::gapi::wip::onevpl::Surface::create_surface(std::move(handle), out_buf_ptr); } TEST(OneVPL_Source_Surface, InitSurface) { - using namespace cv::gapi::wip; + using namespace cv::gapi::wip::onevpl; // create raw MFX handle std::unique_ptr handle(new mfxFrameSurface1{}); @@ -67,7 +133,7 @@ TEST(OneVPL_Source_Surface, InitSurface) TEST(OneVPL_Source_Surface, ConcurrentLock) { - using namespace cv::gapi::wip; + using namespace cv::gapi::wip::onevpl; // create raw MFX handle std::unique_ptr handle(new mfxFrameSurface1{}); @@ -107,7 +173,7 @@ TEST(OneVPL_Source_Surface, ConcurrentLock) TEST(OneVPL_Source_Surface, MemoryLifeTime) { - using namespace cv::gapi::wip; + using namespace cv::gapi::wip::onevpl; // create preallocate surface memory std::unique_ptr preallocated_memory_ptr(new char); @@ -170,7 +236,7 @@ TEST(OneVPL_Source_Surface, MemoryLifeTime) TEST(OneVPL_Source_CPU_FrameAdapter, InitFrameAdapter) { - using namespace cv::gapi::wip; + using namespace cv::gapi::wip::onevpl; // create raw MFX handle std::unique_ptr handle(new mfxFrameSurface1{}); @@ -191,8 +257,8 @@ TEST(OneVPL_Source_CPU_FrameAdapter, InitFrameAdapter) TEST(OneVPL_Source_CPU_Accelerator, InitDestroy) { - using cv::gapi::wip::VPLCPUAccelerationPolicy; - using cv::gapi::wip::VPLAccelerationPolicy; + using cv::gapi::wip::onevpl::VPLCPUAccelerationPolicy; + using cv::gapi::wip::onevpl::VPLAccelerationPolicy; auto acceleration_policy = std::make_shared(); @@ -221,9 +287,9 @@ TEST(OneVPL_Source_CPU_Accelerator, InitDestroy) TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConsume) { - using cv::gapi::wip::VPLCPUAccelerationPolicy; - using cv::gapi::wip::VPLAccelerationPolicy; - using cv::gapi::wip::Surface; + using cv::gapi::wip::onevpl::VPLCPUAccelerationPolicy; + using cv::gapi::wip::onevpl::VPLAccelerationPolicy; + using cv::gapi::wip::onevpl::Surface; auto acceleration_policy = std::make_shared(); @@ -277,9 +343,9 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConsume) TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConcurrentConsume) { - using cv::gapi::wip::VPLCPUAccelerationPolicy; - using cv::gapi::wip::VPLAccelerationPolicy; - using cv::gapi::wip::Surface; + using cv::gapi::wip::onevpl::VPLCPUAccelerationPolicy; + using cv::gapi::wip::onevpl::VPLAccelerationPolicy; + using cv::gapi::wip::onevpl::Surface; auto acceleration_policy = std::make_shared(); @@ -339,6 +405,42 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConcurrentConsume) worker_thread.join(); EXPECT_TRUE(free_surface_count >= free_surface_count_prev); } + +TEST(OneVPL_Source_ProcessingEngine, Init) +{ + using namespace cv::gapi::wip::onevpl; + std::unique_ptr accel; + TestProcessingEngine engine(std::move(accel)); + + mfxSession mfx_session{}; + engine.initialize_session(mfx_session, DecoderParams{}, std::shared_ptr{}); + + EXPECT_EQ(engine.get_ready_frames_count(), 0); + ProcessingEngineBase::ExecutionStatus ret = engine.process(mfx_session); + EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Continue); + EXPECT_EQ(engine.pipeline_stage_num, 0); + + ret = engine.process(mfx_session); + EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Continue); + EXPECT_EQ(engine.pipeline_stage_num, 1); + + ret = engine.process(mfx_session); + EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Continue); + EXPECT_EQ(engine.pipeline_stage_num, 2); + + ret = engine.process(mfx_session); + EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Processed); + EXPECT_EQ(engine.pipeline_stage_num, 3); + EXPECT_EQ(engine.get_ready_frames_count(), 1); + + ret = engine.process(mfx_session); + EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::SessionNotFound); + EXPECT_EQ(engine.pipeline_stage_num, 3); + EXPECT_EQ(engine.get_ready_frames_count(), 1); + + cv::gapi::wip::Data frame; + engine.get_frame(frame); +} } } // namespace opencv_test #endif // HAVE_ONEVPL From 5865af7f6e76c501b3c016b83686c49036deb726 Mon Sep 17 00:00:00 2001 From: Smirnov Alexey Date: Wed, 29 Sep 2021 12:13:13 +0300 Subject: [PATCH 231/376] Add more parameters to render fixture --- .../gapi/test/common/gapi_tests_helpers.hpp | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/modules/gapi/test/common/gapi_tests_helpers.hpp b/modules/gapi/test/common/gapi_tests_helpers.hpp index ce0c90442d72..3f4e1ee0b380 100644 --- a/modules/gapi/test/common/gapi_tests_helpers.hpp +++ b/modules/gapi/test/common/gapi_tests_helpers.hpp @@ -50,6 +50,18 @@ namespace opencv_test __TUPLE_PARAM_TYPE(index) param_name = getSpecificParam(); \ __WRAP_VAARGS(__DEFINE_PARAMS_IMPL7(index+1, __VA_ARGS__)) +#define __DEFINE_PARAMS_IMPL9(index, param_name, ...) \ + __TUPLE_PARAM_TYPE(index) param_name = getSpecificParam(); \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL8(index+1, __VA_ARGS__)) + +#define __DEFINE_PARAMS_IMPL10(index, param_name, ...) \ + __TUPLE_PARAM_TYPE(index) param_name = getSpecificParam(); \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL9(index+1, __VA_ARGS__)) + +#define __DEFINE_PARAMS_IMPL11(index, param_name, ...) \ + __TUPLE_PARAM_TYPE(index) param_name = getSpecificParam(); \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL10(index+1, __VA_ARGS__)) + // user interface to define member variables of specified names #define DEFINE_SPECIFIC_PARAMS_0() @@ -76,6 +88,15 @@ namespace opencv_test #define DEFINE_SPECIFIC_PARAMS_8(...) \ __WRAP_VAARGS(__DEFINE_PARAMS_IMPL8(0, __VA_ARGS__)) + +#define DEFINE_SPECIFIC_PARAMS_9(...) \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL9(0, __VA_ARGS__)) + +#define DEFINE_SPECIFIC_PARAMS_10(...) \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL10(0, __VA_ARGS__)) + +#define DEFINE_SPECIFIC_PARAMS_11(...) \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL11(0, __VA_ARGS__)) } // namespace opencv_test #endif //OPENCV_GAPI_TESTS_HELPERS_HPP From 846317ef372e37332fca8804c76cfa612cdf66e8 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 29 Sep 2021 13:16:46 +0000 Subject: [PATCH 232/376] dnn(ocl): fix conv BASIC workgroup --- .../dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 13 ++++++++++--- modules/dnn/src/opencl/conv_layer_spatial.cl | 12 ++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index 5eee1da4a004..a1164273accf 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -1081,9 +1081,16 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, kernel.set(argIdx++, (uint16_t)output_h_); kernel.set(argIdx++, (uint16_t)pad_w_); kernel.set(argIdx++, (uint16_t)pad_h_); - if (!kernel.run_(3, config->global_work_size, - (config->use_null_local) ? NULL : config->local_work_size, - false)) + + size_t wgs = kernel.workGroupSize(); + if (!wgs) + { + CV_LOG_ERROR(NULL, "DNN/OpenCL: Can't query workGroupSize of Basic kernel"); + return false; + } + size_t lws[1] = { wgs }; + size_t gws[1] = { roundUp((size_t)output_w_ * output_h_ * M_, (unsigned)lws[0]) }; + if (!kernel.run_(1, gws, lws, false)) { CV_LOG_ERROR(NULL, "DNN/OpenCL: Basic kernel run failed"); return false; diff --git a/modules/dnn/src/opencl/conv_layer_spatial.cl b/modules/dnn/src/opencl/conv_layer_spatial.cl index e7bbacd4c4c0..455f0ed7ea06 100644 --- a/modules/dnn/src/opencl/conv_layer_spatial.cl +++ b/modules/dnn/src/opencl/conv_layer_spatial.cl @@ -158,10 +158,14 @@ __kernel void ConvolveBasic( ) { __global Dtype* convolved_image = convolved_image_base + convolved_image_base_offset; - const int outputX = get_global_id(0); - const int outputY = get_global_id(1); - const int kernelNum = get_global_id(2) * ZPAR; - if (outputX < output_width && outputY < output_height) + const int out_idx = get_global_id(0); // 1D task layout: [output_width * output_height * OUTPUT_Z] + const int plane_size = output_width * output_height; + const int out_plane_idx = out_idx % plane_size; + const int outputZ = out_idx / plane_size; + const int outputY = out_plane_idx / output_width; + const int outputX = out_plane_idx % output_width; + const int kernelNum = outputZ * ZPAR; + if (kernelNum < OUTPUT_Z) { Dtype sum[ZPAR]; for (int kern = 0; kern < ZPAR; kern++) From 9a8552e8aec3b314cc947b16db8ea182d691b322 Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Sat, 25 Sep 2021 09:31:44 +0300 Subject: [PATCH 233/376] Update perf_bgfg_mog2.cpp, perf_bgfg_knn.cpp --- modules/video/perf/opencl/perf_bgfg_knn.cpp | 6 +++--- modules/video/perf/opencl/perf_bgfg_mog2.cpp | 6 +++--- modules/video/perf/perf_bgfg_knn.cpp | 6 +++--- modules/video/perf/perf_bgfg_mog2.cpp | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/video/perf/opencl/perf_bgfg_knn.cpp b/modules/video/perf/opencl/perf_bgfg_knn.cpp index 833db66d86e1..6e95e52b2fd0 100644 --- a/modules/video/perf/opencl/perf_bgfg_knn.cpp +++ b/modules/video/perf/opencl/perf_bgfg_knn.cpp @@ -19,7 +19,7 @@ typedef TestBaseWithParam KNN_GetBackgroundImage; using namespace opencv_test; -OCL_PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +OCL_PERF_TEST_P(KNN_Apply, KNN, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoKNNParamType params = GetParam(); @@ -51,8 +51,8 @@ OCL_PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/vid } OCL_PERF_TEST_P(KNN_GetBackgroundImage, KNN, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoKNNParamType params = GetParam(); diff --git a/modules/video/perf/opencl/perf_bgfg_mog2.cpp b/modules/video/perf/opencl/perf_bgfg_mog2.cpp index 8e5f09525782..3de03fa1ee99 100644 --- a/modules/video/perf/opencl/perf_bgfg_mog2.cpp +++ b/modules/video/perf/opencl/perf_bgfg_mog2.cpp @@ -19,7 +19,7 @@ typedef TestBaseWithParam MOG2_GetBackgroundImage; using namespace opencv_test; -OCL_PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +OCL_PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoMOG2ParamType params = GetParam(); @@ -51,8 +51,8 @@ OCL_PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/v } OCL_PERF_TEST_P(MOG2_GetBackgroundImage, Mog2, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoMOG2ParamType params = GetParam(); diff --git a/modules/video/perf/perf_bgfg_knn.cpp b/modules/video/perf/perf_bgfg_knn.cpp index 23bb1fe0a507..42d6368fd30c 100644 --- a/modules/video/perf/perf_bgfg_knn.cpp +++ b/modules/video/perf/perf_bgfg_knn.cpp @@ -14,7 +14,7 @@ typedef tuple VideoKNNParamType; typedef TestBaseWithParam KNN_Apply; typedef TestBaseWithParam KNN_GetBackgroundImage; -PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +PERF_TEST_P(KNN_Apply, KNN, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoKNNParamType params = GetParam(); @@ -46,8 +46,8 @@ PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/video/1 } PERF_TEST_P(KNN_GetBackgroundImage, KNN, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoKNNParamType params = GetParam(); diff --git a/modules/video/perf/perf_bgfg_mog2.cpp b/modules/video/perf/perf_bgfg_mog2.cpp index f911a9c74e4f..610138f85c60 100644 --- a/modules/video/perf/perf_bgfg_mog2.cpp +++ b/modules/video/perf/perf_bgfg_mog2.cpp @@ -14,7 +14,7 @@ typedef tuple VideoMOG2ParamType; typedef TestBaseWithParam MOG2_Apply; typedef TestBaseWithParam MOG2_GetBackgroundImage; -PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoMOG2ParamType params = GetParam(); @@ -46,8 +46,8 @@ PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/video } PERF_TEST_P(MOG2_GetBackgroundImage, Mog2, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoMOG2ParamType params = GetParam(); From f8f6cd6ef5d2b06845e19ed44f3865a51f1a1162 Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Tue, 28 Sep 2021 21:23:26 +0300 Subject: [PATCH 234/376] Update OpenCVDetectVTK.cmake --- cmake/OpenCVDetectVTK.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/OpenCVDetectVTK.cmake b/cmake/OpenCVDetectVTK.cmake index 57c154475c67..b1a58ca085bc 100644 --- a/cmake/OpenCVDetectVTK.cmake +++ b/cmake/OpenCVDetectVTK.cmake @@ -1,7 +1,7 @@ if(NOT VTK_FOUND) find_package(VTK QUIET NAMES vtk VTK) if(VTK_FOUND) - if(VTK_VERSION VERSION_EQUAL "9") # VTK 9.0 + if(NOT (VTK_VERSION VERSION_LESS "9.0.0") AND (VTK_VERSION VERSION_LESS "10.0.0")) # VTK 9.x find_package(VTK 9 QUIET NAMES vtk COMPONENTS FiltersExtraction FiltersSources From 9b768727080b3279c244ad595115b1d5126d32ed Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Fri, 1 Oct 2021 16:23:16 +0300 Subject: [PATCH 235/376] restore LSD --- modules/imgproc/include/opencv2/imgproc.hpp | 10 +- modules/imgproc/src/lsd.cpp | 983 +++++++++++++++++++- modules/imgproc/test/test_lsd.cpp | 4 - samples/cpp/lsd_lines.cpp | 73 ++ 4 files changed, 1055 insertions(+), 15 deletions(-) create mode 100644 samples/cpp/lsd_lines.cpp diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index f7583c19267a..855cbaf15983 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -1220,12 +1220,14 @@ class CV_EXPORTS_W Subdiv2D //! @addtogroup imgproc_feature //! @{ +/** @example samples/cpp/lsd_lines.cpp +An example using the LineSegmentDetector +\image html building_lsd.png "Sample output image" width=434 height=300 +*/ + /** @brief Line segment detector class following the algorithm described at @cite Rafael12 . - -@note Implementation has been removed due original code license conflict - */ class CV_EXPORTS_W LineSegmentDetector : public Algorithm { @@ -1288,8 +1290,6 @@ to edit those, as to tailor it for their own application. @param log_eps Detection threshold: -log10(NFA) \> log_eps. Used only when advance refinement is chosen. @param density_th Minimal density of aligned region points in the enclosing rectangle. @param n_bins Number of bins in pseudo-ordering of gradient modulus. - -@note Implementation has been removed due original code license conflict */ CV_EXPORTS_W Ptr createLineSegmentDetector( int refine = LSD_REFINE_STD, double scale = 0.8, diff --git a/modules/imgproc/src/lsd.cpp b/modules/imgproc/src/lsd.cpp index 1ec984d29099..d06759c2bb3b 100644 --- a/modules/imgproc/src/lsd.cpp +++ b/modules/imgproc/src/lsd.cpp @@ -42,7 +42,123 @@ #include "precomp.hpp" #include -namespace cv { +///////////////////////////////////////////////////////////////////////////////////////// +// Default LSD parameters +// SIGMA_SCALE 0.6 - Sigma for Gaussian filter is computed as sigma = sigma_scale/scale. +// QUANT 2.0 - Bound to the quantization error on the gradient norm. +// ANG_TH 22.5 - Gradient angle tolerance in degrees. +// LOG_EPS 0.0 - Detection threshold: -log10(NFA) > log_eps +// DENSITY_TH 0.7 - Minimal density of region points in rectangle. +// N_BINS 1024 - Number of bins in pseudo-ordering of gradient modulus. + +#define M_3_2_PI (3 * CV_PI) / 2 // 3/2 pi +#define M_2__PI (2 * CV_PI) // 2 pi + +#ifndef M_LN10 +#define M_LN10 2.30258509299404568402 +#endif + +#define NOTDEF double(-1024.0) // Label for pixels with undefined gradient. + +#define NOTUSED 0 // Label for pixels not used in yet. +#define USED 1 // Label for pixels already used in detection. + +#define RELATIVE_ERROR_FACTOR 100.0 + +const double DEG_TO_RADS = CV_PI / 180; + +#define log_gamma(x) ((x)>15.0?log_gamma_windschitl(x):log_gamma_lanczos(x)) + +struct edge +{ + cv::Point p; + bool taken; +}; + +///////////////////////////////////////////////////////////////////////////////////////// + +inline double distSq(const double x1, const double y1, + const double x2, const double y2) +{ + return (x2 - x1)*(x2 - x1) + (y2 - y1)*(y2 - y1); +} + +inline double dist(const double x1, const double y1, + const double x2, const double y2) +{ + return sqrt(distSq(x1, y1, x2, y2)); +} + +// Signed angle difference +inline double angle_diff_signed(const double& a, const double& b) +{ + double diff = a - b; + while(diff <= -CV_PI) diff += M_2__PI; + while(diff > CV_PI) diff -= M_2__PI; + return diff; +} + +// Absolute value angle difference +inline double angle_diff(const double& a, const double& b) +{ + return std::fabs(angle_diff_signed(a, b)); +} + +// Compare doubles by relative error. +inline bool double_equal(const double& a, const double& b) +{ + // trivial case + if(a == b) return true; + + double abs_diff = fabs(a - b); + double aa = fabs(a); + double bb = fabs(b); + double abs_max = (aa > bb)? aa : bb; + + if(abs_max < DBL_MIN) abs_max = DBL_MIN; + + return (abs_diff / abs_max) <= (RELATIVE_ERROR_FACTOR * DBL_EPSILON); +} + +inline bool AsmallerB_XoverY(const edge& a, const edge& b) +{ + if (a.p.x == b.p.x) return a.p.y < b.p.y; + else return a.p.x < b.p.x; +} + +/** + * Computes the natural logarithm of the absolute value of + * the gamma function of x using Windschitl method. + * See http://www.rskey.org/gamma.htm + */ +inline double log_gamma_windschitl(const double& x) +{ + return 0.918938533204673 + (x-0.5)*log(x) - x + + 0.5*x*log(x*sinh(1/x) + 1/(810.0*pow(x, 6.0))); +} + +/** + * Computes the natural logarithm of the absolute value of + * the gamma function of x using the Lanczos approximation. + * See http://www.rskey.org/gamma.htm + */ +inline double log_gamma_lanczos(const double& x) +{ + static double q[7] = { 75122.6331530, 80916.6278952, 36308.2951477, + 8687.24529705, 1168.92649479, 83.8676043424, + 2.50662827511 }; + double a = (x + 0.5) * log(x + 5.5) - (x + 5.5); + double b = 0; + for(int n = 0; n < 7; ++n) + { + a -= log(x + double(n)); + b += q[n] * pow(x, double(n)); + } + return a + log(b); +} +/////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cv{ class LineSegmentDetectorImpl CV_FINAL : public LineSegmentDetector { @@ -113,7 +229,164 @@ class LineSegmentDetectorImpl CV_FINAL : public LineSegmentDetector int compareSegments(const Size& size, InputArray lines1, InputArray lines2, InputOutputArray _image = noArray()) CV_OVERRIDE; private: + Mat image; + Mat scaled_image; + Mat_ angles; // in rads + Mat_ modgrad; + Mat_ used; + + int img_width; + int img_height; + double LOG_NT; + + bool w_needed; + bool p_needed; + bool n_needed; + + const double SCALE; + const int doRefine; + const double SIGMA_SCALE; + const double QUANT; + const double ANG_TH; + const double LOG_EPS; + const double DENSITY_TH; + const int N_BINS; + + struct RegionPoint { + int x; + int y; + uchar* used; + double angle; + double modgrad; + }; + + struct normPoint + { + Point2i p; + int norm; + }; + + std::vector ordered_points; + + struct rect + { + double x1, y1, x2, y2; // first and second point of the line segment + double width; // rectangle width + double x, y; // center of the rectangle + double theta; // angle + double dx,dy; // (dx,dy) is vector oriented as the line segment + double prec; // tolerance angle + double p; // probability of a point with angle within 'prec' + }; + LineSegmentDetectorImpl& operator= (const LineSegmentDetectorImpl&); // to quiet MSVC + +/** + * Detect lines in the whole input image. + * + * @param lines Return: A vector of Vec4f elements specifying the beginning and ending point of a line. + * Where Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 - end. + * Returned lines are strictly oriented depending on the gradient. + * @param widths Return: Vector of widths of the regions, where the lines are found. E.g. Width of line. + * @param precisions Return: Vector of precisions with which the lines are found. + * @param nfas Return: Vector containing number of false alarms in the line region, with precision of 10%. + * The bigger the value, logarithmically better the detection. + * * -1 corresponds to 10 mean false alarms + * * 0 corresponds to 1 mean false alarm + * * 1 corresponds to 0.1 mean false alarms + */ + void flsd(std::vector& lines, + std::vector& widths, std::vector& precisions, + std::vector& nfas); + +/** + * Finds the angles and the gradients of the image. Generates a list of pseudo ordered points. + * + * @param threshold The minimum value of the angle that is considered defined, otherwise NOTDEF + * @param n_bins The number of bins with which gradients are ordered by, using bucket sort. + * @param ordered_points Return: Vector of coordinate points that are pseudo ordered by magnitude. + * Pixels would be ordered by norm value, up to a precision given by max_grad/n_bins. + */ + void ll_angle(const double& threshold, const unsigned int& n_bins); + +/** + * Grow a region starting from point s with a defined precision, + * returning the containing points size and the angle of the gradients. + * + * @param s Starting point for the region. + * @param reg Return: Vector of points, that are part of the region + * @param reg_angle Return: The mean angle of the region. + * @param prec The precision by which each region angle should be aligned to the mean. + */ + void region_grow(const Point2i& s, std::vector& reg, + double& reg_angle, const double& prec); + +/** + * Finds the bounding rotated rectangle of a region. + * + * @param reg The region of points, from which the rectangle to be constructed from. + * @param reg_angle The mean angle of the region. + * @param prec The precision by which points were found. + * @param p Probability of a point with angle within 'prec'. + * @param rec Return: The generated rectangle. + */ + void region2rect(const std::vector& reg, const double reg_angle, + const double prec, const double p, rect& rec) const; + +/** + * Compute region's angle as the principal inertia axis of the region. + * @return Regions angle. + */ + double get_theta(const std::vector& reg, const double& x, + const double& y, const double& reg_angle, const double& prec) const; + +/** + * An estimation of the angle tolerance is performed by the standard deviation of the angle at points + * near the region's starting point. Then, a new region is grown starting from the same point, but using the + * estimated angle tolerance. If this fails to produce a rectangle with the right density of region points, + * 'reduce_region_radius' is called to try to satisfy this condition. + */ + bool refine(std::vector& reg, double reg_angle, + const double prec, double p, rect& rec, const double& density_th); + +/** + * Reduce the region size, by elimination the points far from the starting point, until that leads to + * rectangle with the right density of region points or to discard the region if too small. + */ + bool reduce_region_radius(std::vector& reg, double reg_angle, + const double prec, double p, rect& rec, double density, const double& density_th); + +/** + * Try some rectangles variations to improve NFA value. Only if the rectangle is not meaningful (i.e., log_nfa <= log_eps). + * @return The new NFA value. + */ + double rect_improve(rect& rec) const; + +/** + * Calculates the number of correctly aligned points within the rectangle. + * @return The new NFA value. + */ + double rect_nfa(const rect& rec) const; + +/** + * Computes the NFA values based on the total number of points, points that agree. + * n, k, p are the binomial parameters. + * @return The new NFA value. + */ + double nfa(const int& n, const int& k, const double& p) const; + +/** + * Is the point at place 'address' aligned to angle theta, up to precision 'prec'? + * @return Whether the point is aligned. + */ + bool isAligned(int x, int y, const double& theta, const double& prec) const; + +public: + // Compare norm + static inline bool compare_norm( const normPoint& n1, const normPoint& n2 ) + { + return (n1.norm > n2.norm); + } }; ///////////////////////////////////////////////////////////////////////////////////////// @@ -131,12 +404,13 @@ CV_EXPORTS Ptr createLineSegmentDetector( LineSegmentDetectorImpl::LineSegmentDetectorImpl(int _refine, double _scale, double _sigma_scale, double _quant, double _ang_th, double _log_eps, double _density_th, int _n_bins) + : img_width(0), img_height(0), LOG_NT(0), w_needed(false), p_needed(false), n_needed(false), + SCALE(_scale), doRefine(_refine), SIGMA_SCALE(_sigma_scale), QUANT(_quant), + ANG_TH(_ang_th), LOG_EPS(_log_eps), DENSITY_TH(_density_th), N_BINS(_n_bins) { CV_Assert(_scale > 0 && _sigma_scale > 0 && _quant >= 0 && _ang_th > 0 && _ang_th < 180 && _density_th >= 0 && _density_th < 1 && _n_bins > 0); - CV_UNUSED(_refine); CV_UNUSED(_log_eps); - CV_Error(Error::StsNotImplemented, "Implementation has been removed due original code license issues"); } void LineSegmentDetectorImpl::detect(InputArray _image, OutputArray _lines, @@ -144,11 +418,708 @@ void LineSegmentDetectorImpl::detect(InputArray _image, OutputArray _lines, { CV_INSTRUMENT_REGION(); - CV_UNUSED(_image); CV_UNUSED(_lines); - CV_UNUSED(_width); CV_UNUSED(_prec); CV_UNUSED(_nfa); - CV_Error(Error::StsNotImplemented, "Implementation has been removed due original code license issues"); + image = _image.getMat(); + CV_Assert(!image.empty() && image.type() == CV_8UC1); + + std::vector lines; + std::vector w, p, n; + w_needed = _width.needed(); + p_needed = _prec.needed(); + if (doRefine < LSD_REFINE_ADV) + n_needed = false; + else + n_needed = _nfa.needed(); + + flsd(lines, w, p, n); + + Mat(lines).copyTo(_lines); + if(w_needed) Mat(w).copyTo(_width); + if(p_needed) Mat(p).copyTo(_prec); + if(n_needed) Mat(n).copyTo(_nfa); + + // Clear used structures + ordered_points.clear(); } +void LineSegmentDetectorImpl::flsd(std::vector& lines, + std::vector& widths, std::vector& precisions, + std::vector& nfas) +{ + // Angle tolerance + const double prec = CV_PI * ANG_TH / 180; + const double p = ANG_TH / 180; + const double rho = QUANT / sin(prec); // gradient magnitude threshold + + if(SCALE != 1) + { + Mat gaussian_img; + const double sigma = (SCALE < 1)?(SIGMA_SCALE / SCALE):(SIGMA_SCALE); + const double sprec = 3; + const unsigned int h = (unsigned int)(ceil(sigma * sqrt(2 * sprec * log(10.0)))); + Size ksize(1 + 2 * h, 1 + 2 * h); // kernel size + GaussianBlur(image, gaussian_img, ksize, sigma); + // Scale image to needed size + resize(gaussian_img, scaled_image, Size(), SCALE, SCALE, INTER_LINEAR_EXACT); + ll_angle(rho, N_BINS); + } + else + { + scaled_image = image; + ll_angle(rho, N_BINS); + } + + LOG_NT = 5 * (log10(double(img_width)) + log10(double(img_height))) / 2 + log10(11.0); + const size_t min_reg_size = size_t(-LOG_NT/log10(p)); // minimal number of points in region that can give a meaningful event + + // // Initialize region only when needed + // Mat region = Mat::zeros(scaled_image.size(), CV_8UC1); + used = Mat_::zeros(scaled_image.size()); // zeros = NOTUSED + std::vector reg; + + // Search for line segments + for(size_t i = 0, points_size = ordered_points.size(); i < points_size; ++i) + { + const Point2i& point = ordered_points[i].p; + if((used.at(point) == NOTUSED) && (angles.at(point) != NOTDEF)) + { + double reg_angle; + region_grow(ordered_points[i].p, reg, reg_angle, prec); + + // Ignore small regions + if(reg.size() < min_reg_size) { continue; } + + // Construct rectangular approximation for the region + rect rec; + region2rect(reg, reg_angle, prec, p, rec); + + double log_nfa = -1; + if(doRefine > LSD_REFINE_NONE) + { + // At least REFINE_STANDARD lvl. + if(!refine(reg, reg_angle, prec, p, rec, DENSITY_TH)) { continue; } + + if(doRefine >= LSD_REFINE_ADV) + { + // Compute NFA + log_nfa = rect_improve(rec); + if(log_nfa <= LOG_EPS) { continue; } + } + } + // Found new line + + // Add the offset + rec.x1 += 0.5; rec.y1 += 0.5; + rec.x2 += 0.5; rec.y2 += 0.5; + + // scale the result values if a sub-sampling was performed + if(SCALE != 1) + { + rec.x1 /= SCALE; rec.y1 /= SCALE; + rec.x2 /= SCALE; rec.y2 /= SCALE; + rec.width /= SCALE; + } + + //Store the relevant data + lines.push_back(Vec4f(float(rec.x1), float(rec.y1), float(rec.x2), float(rec.y2))); + if(w_needed) widths.push_back(rec.width); + if(p_needed) precisions.push_back(rec.p); + if(n_needed && doRefine >= LSD_REFINE_ADV) nfas.push_back(log_nfa); + } + } +} + +void LineSegmentDetectorImpl::ll_angle(const double& threshold, + const unsigned int& n_bins) +{ + //Initialize data + angles = Mat_(scaled_image.size()); + modgrad = Mat_(scaled_image.size()); + + img_width = scaled_image.cols; + img_height = scaled_image.rows; + + // Undefined the down and right boundaries + angles.row(img_height - 1).setTo(NOTDEF); + angles.col(img_width - 1).setTo(NOTDEF); + + // Computing gradient for remaining pixels + double max_grad = -1; + for(int y = 0; y < img_height - 1; ++y) + { + const uchar* scaled_image_row = scaled_image.ptr(y); + const uchar* next_scaled_image_row = scaled_image.ptr(y+1); + double* angles_row = angles.ptr(y); + double* modgrad_row = modgrad.ptr(y); + for(int x = 0; x < img_width-1; ++x) + { + int DA = next_scaled_image_row[x + 1] - scaled_image_row[x]; + int BC = scaled_image_row[x + 1] - next_scaled_image_row[x]; + int gx = DA + BC; // gradient x component + int gy = DA - BC; // gradient y component + double norm = std::sqrt((gx * gx + gy * gy) / 4.0); // gradient norm + + modgrad_row[x] = norm; // store gradient + + if (norm <= threshold) // norm too small, gradient no defined + { + angles_row[x] = NOTDEF; + } + else + { + angles_row[x] = fastAtan2(float(gx), float(-gy)) * DEG_TO_RADS; // gradient angle computation + if (norm > max_grad) { max_grad = norm; } + } + + } + } + + // Compute histogram of gradient values + double bin_coef = (max_grad > 0) ? double(n_bins - 1) / max_grad : 0; // If all image is smooth, max_grad <= 0 + for(int y = 0; y < img_height - 1; ++y) + { + const double* modgrad_row = modgrad.ptr(y); + for(int x = 0; x < img_width - 1; ++x) + { + normPoint _point; + int i = int(modgrad_row[x] * bin_coef); + _point.p = Point(x, y); + _point.norm = i; + ordered_points.push_back(_point); + } + } + + // Sort + std::sort(ordered_points.begin(), ordered_points.end(), compare_norm); +} + +void LineSegmentDetectorImpl::region_grow(const Point2i& s, std::vector& reg, + double& reg_angle, const double& prec) +{ + reg.clear(); + + // Point to this region + RegionPoint seed; + seed.x = s.x; + seed.y = s.y; + seed.used = &used.at(s); + reg_angle = angles.at(s); + seed.angle = reg_angle; + seed.modgrad = modgrad.at(s); + reg.push_back(seed); + + float sumdx = float(std::cos(reg_angle)); + float sumdy = float(std::sin(reg_angle)); + *seed.used = USED; + + //Try neighboring regions + for (size_t i = 0;i(yy); + const double* angles_row = angles.ptr(yy); + const double* modgrad_row = modgrad.ptr(yy); + for(int xx = xx_min; xx <= xx_max; ++xx) + { + uchar& is_used = used_row[xx]; + if(is_used != USED && + (isAligned(xx, yy, reg_angle, prec))) + { + const double& angle = angles_row[xx]; + // Add point + is_used = USED; + RegionPoint region_point; + region_point.x = xx; + region_point.y = yy; + region_point.used = &is_used; + region_point.modgrad = modgrad_row[xx]; + region_point.angle = angle; + reg.push_back(region_point); + + // Update region's angle + sumdx += cos(float(angle)); + sumdy += sin(float(angle)); + // reg_angle is used in the isAligned, so it needs to be updates? + reg_angle = fastAtan2(sumdy, sumdx) * DEG_TO_RADS; + } + } + } + } +} + +void LineSegmentDetectorImpl::region2rect(const std::vector& reg, + const double reg_angle, const double prec, const double p, rect& rec) const +{ + double x = 0, y = 0, sum = 0; + for(size_t i = 0; i < reg.size(); ++i) + { + const RegionPoint& pnt = reg[i]; + const double& weight = pnt.modgrad; + x += double(pnt.x) * weight; + y += double(pnt.y) * weight; + sum += weight; + } + + // Weighted sum must differ from 0 + CV_Assert(sum > 0); + + x /= sum; + y /= sum; + + double theta = get_theta(reg, x, y, reg_angle, prec); + + // Find length and width + double dx = cos(theta); + double dy = sin(theta); + double l_min = 0, l_max = 0, w_min = 0, w_max = 0; + + for(size_t i = 0; i < reg.size(); ++i) + { + double regdx = double(reg[i].x) - x; + double regdy = double(reg[i].y) - y; + + double l = regdx * dx + regdy * dy; + double w = -regdx * dy + regdy * dx; + + if(l > l_max) l_max = l; + else if(l < l_min) l_min = l; + if(w > w_max) w_max = w; + else if(w < w_min) w_min = w; + } + + // Store values + rec.x1 = x + l_min * dx; + rec.y1 = y + l_min * dy; + rec.x2 = x + l_max * dx; + rec.y2 = y + l_max * dy; + rec.width = w_max - w_min; + rec.x = x; + rec.y = y; + rec.theta = theta; + rec.dx = dx; + rec.dy = dy; + rec.prec = prec; + rec.p = p; + + // Min width of 1 pixel + if(rec.width < 1.0) rec.width = 1.0; +} + +double LineSegmentDetectorImpl::get_theta(const std::vector& reg, const double& x, + const double& y, const double& reg_angle, const double& prec) const +{ + double Ixx = 0.0; + double Iyy = 0.0; + double Ixy = 0.0; + + // Compute inertia matrix + for(size_t i = 0; i < reg.size(); ++i) + { + const double& regx = reg[i].x; + const double& regy = reg[i].y; + const double& weight = reg[i].modgrad; + double dx = regx - x; + double dy = regy - y; + Ixx += dy * dy * weight; + Iyy += dx * dx * weight; + Ixy -= dx * dy * weight; + } + + // Check if inertia matrix is null + CV_Assert(!(double_equal(Ixx, 0) && double_equal(Iyy, 0) && double_equal(Ixy, 0))); + + // Compute smallest eigenvalue + double lambda = 0.5 * (Ixx + Iyy - sqrt((Ixx - Iyy) * (Ixx - Iyy) + 4.0 * Ixy * Ixy)); + + // Compute angle + double theta = (fabs(Ixx)>fabs(Iyy))? + double(fastAtan2(float(lambda - Ixx), float(Ixy))): + double(fastAtan2(float(Ixy), float(lambda - Iyy))); // in degs + theta *= DEG_TO_RADS; + + // Correct angle by 180 deg if necessary + if(angle_diff(theta, reg_angle) > prec) { theta += CV_PI; } + + return theta; +} + +bool LineSegmentDetectorImpl::refine(std::vector& reg, double reg_angle, + const double prec, double p, rect& rec, const double& density_th) +{ + double density = double(reg.size()) / (dist(rec.x1, rec.y1, rec.x2, rec.y2) * rec.width); + + if (density >= density_th) { return true; } + + // Try to reduce angle tolerance + double xc = double(reg[0].x); + double yc = double(reg[0].y); + const double& ang_c = reg[0].angle; + double sum = 0, s_sum = 0; + int n = 0; + + for (size_t i = 0; i < reg.size(); ++i) + { + *(reg[i].used) = NOTUSED; + if (dist(xc, yc, reg[i].x, reg[i].y) < rec.width) + { + const double& angle = reg[i].angle; + double ang_d = angle_diff_signed(angle, ang_c); + sum += ang_d; + s_sum += ang_d * ang_d; + ++n; + } + } + CV_Assert(n > 0); + double mean_angle = sum / double(n); + // 2 * standard deviation + double tau = 2.0 * sqrt((s_sum - 2.0 * mean_angle * sum) / double(n) + mean_angle * mean_angle); + + // Try new region + region_grow(Point(reg[0].x, reg[0].y), reg, reg_angle, tau); + + if (reg.size() < 2) { return false; } + + region2rect(reg, reg_angle, prec, p, rec); + density = double(reg.size()) / (dist(rec.x1, rec.y1, rec.x2, rec.y2) * rec.width); + + if (density < density_th) + { + return reduce_region_radius(reg, reg_angle, prec, p, rec, density, density_th); + } + else + { + return true; + } +} + +bool LineSegmentDetectorImpl::reduce_region_radius(std::vector& reg, double reg_angle, + const double prec, double p, rect& rec, double density, const double& density_th) +{ + // Compute region's radius + double xc = double(reg[0].x); + double yc = double(reg[0].y); + double radSq1 = distSq(xc, yc, rec.x1, rec.y1); + double radSq2 = distSq(xc, yc, rec.x2, rec.y2); + double radSq = radSq1 > radSq2 ? radSq1 : radSq2; + + while(density < density_th) + { + radSq *= 0.75*0.75; // Reduce region's radius to 75% of its value + // Remove points from the region and update 'used' map + for (size_t i = 0; i < reg.size(); ++i) + { + if(distSq(xc, yc, double(reg[i].x), double(reg[i].y)) > radSq) + { + // Remove point from the region + *(reg[i].used) = NOTUSED; + std::swap(reg[i], reg[reg.size() - 1]); + reg.pop_back(); + --i; // To avoid skipping one point + } + } + + if(reg.size() < 2) { return false; } + + // Re-compute rectangle + region2rect(reg ,reg_angle, prec, p, rec); + + // Re-compute region points density + density = double(reg.size()) / + (dist(rec.x1, rec.y1, rec.x2, rec.y2) * rec.width); + } + + return true; +} + +double LineSegmentDetectorImpl::rect_improve(rect& rec) const +{ + double delta = 0.5; + double delta_2 = delta / 2.0; + + double log_nfa = rect_nfa(rec); + + if(log_nfa > LOG_EPS) return log_nfa; // Good rectangle + + // Try to improve + // Finer precision + rect r = rect(rec); // Copy + for(int n = 0; n < 5; ++n) + { + r.p /= 2; + r.prec = r.p * CV_PI; + double log_nfa_new = rect_nfa(r); + if(log_nfa_new > log_nfa) + { + log_nfa = log_nfa_new; + rec = rect(r); + } + } + if(log_nfa > LOG_EPS) return log_nfa; + + // Try to reduce width + r = rect(rec); + for(unsigned int n = 0; n < 5; ++n) + { + if((r.width - delta) >= 0.5) + { + r.width -= delta; + double log_nfa_new = rect_nfa(r); + if(log_nfa_new > log_nfa) + { + rec = rect(r); + log_nfa = log_nfa_new; + } + } + } + if(log_nfa > LOG_EPS) return log_nfa; + + // Try to reduce one side of rectangle + r = rect(rec); + for(unsigned int n = 0; n < 5; ++n) + { + if((r.width - delta) >= 0.5) + { + r.x1 += -r.dy * delta_2; + r.y1 += r.dx * delta_2; + r.x2 += -r.dy * delta_2; + r.y2 += r.dx * delta_2; + r.width -= delta; + double log_nfa_new = rect_nfa(r); + if(log_nfa_new > log_nfa) + { + rec = rect(r); + log_nfa = log_nfa_new; + } + } + } + if(log_nfa > LOG_EPS) return log_nfa; + + // Try to reduce other side of rectangle + r = rect(rec); + for(unsigned int n = 0; n < 5; ++n) + { + if((r.width - delta) >= 0.5) + { + r.x1 -= -r.dy * delta_2; + r.y1 -= r.dx * delta_2; + r.x2 -= -r.dy * delta_2; + r.y2 -= r.dx * delta_2; + r.width -= delta; + double log_nfa_new = rect_nfa(r); + if(log_nfa_new > log_nfa) + { + rec = rect(r); + log_nfa = log_nfa_new; + } + } + } + if(log_nfa > LOG_EPS) return log_nfa; + + // Try finer precision + r = rect(rec); + for(unsigned int n = 0; n < 5; ++n) + { + if((r.width - delta) >= 0.5) + { + r.p /= 2; + r.prec = r.p * CV_PI; + double log_nfa_new = rect_nfa(r); + if(log_nfa_new > log_nfa) + { + rec = rect(r); + log_nfa = log_nfa_new; + } + } + } + + return log_nfa; +} + +double LineSegmentDetectorImpl::rect_nfa(const rect& rec) const +{ + int total_pts = 0, alg_pts = 0; + double half_width = rec.width / 2.0; + double dyhw = rec.dy * half_width; + double dxhw = rec.dx * half_width; + + edge ordered_x[4]; + edge* min_y = &ordered_x[0]; + edge* max_y = &ordered_x[0]; // Will be used for loop range + + ordered_x[0].p.x = int(rec.x1 - dyhw); ordered_x[0].p.y = int(rec.y1 + dxhw); ordered_x[0].taken = false; + ordered_x[1].p.x = int(rec.x2 - dyhw); ordered_x[1].p.y = int(rec.y2 + dxhw); ordered_x[1].taken = false; + ordered_x[2].p.x = int(rec.x2 + dyhw); ordered_x[2].p.y = int(rec.y2 - dxhw); ordered_x[2].taken = false; + ordered_x[3].p.x = int(rec.x1 + dyhw); ordered_x[3].p.y = int(rec.y1 - dxhw); ordered_x[3].taken = false; + + std::sort(ordered_x, ordered_x + 4, AsmallerB_XoverY); + + // Find min y. And mark as taken. find max y. + for(unsigned int i = 1; i < 4; ++i) + { + if(min_y->p.y > ordered_x[i].p.y) {min_y = &ordered_x[i]; } + if(max_y->p.y < ordered_x[i].p.y) {max_y = &ordered_x[i]; } + } + min_y->taken = true; + + // Find leftmost untaken point; + edge* leftmost = 0; + for(unsigned int i = 0; i < 4; ++i) + { + if(!ordered_x[i].taken) + { + if(!leftmost) // if uninitialized + { + leftmost = &ordered_x[i]; + } + else if (leftmost->p.x > ordered_x[i].p.x) + { + leftmost = &ordered_x[i]; + } + } + } + CV_Assert(leftmost != NULL); + leftmost->taken = true; + + // Find rightmost untaken point; + edge* rightmost = 0; + for(unsigned int i = 0; i < 4; ++i) + { + if(!ordered_x[i].taken) + { + if(!rightmost) // if uninitialized + { + rightmost = &ordered_x[i]; + } + else if (rightmost->p.x < ordered_x[i].p.x) + { + rightmost = &ordered_x[i]; + } + } + } + CV_Assert(rightmost != NULL); + rightmost->taken = true; + + // Find last untaken point; + edge* tailp = 0; + for(unsigned int i = 0; i < 4; ++i) + { + if(!ordered_x[i].taken) + { + if(!tailp) // if uninitialized + { + tailp = &ordered_x[i]; + } + else if (tailp->p.x > ordered_x[i].p.x) + { + tailp = &ordered_x[i]; + } + } + } + CV_Assert(tailp != NULL); + tailp->taken = true; + + double flstep = (min_y->p.y != leftmost->p.y) ? + (min_y->p.x - leftmost->p.x) / (min_y->p.y - leftmost->p.y) : 0; //first left step + double slstep = (leftmost->p.y != tailp->p.x) ? + (leftmost->p.x - tailp->p.x) / (leftmost->p.y - tailp->p.x) : 0; //second left step + + double frstep = (min_y->p.y != rightmost->p.y) ? + (min_y->p.x - rightmost->p.x) / (min_y->p.y - rightmost->p.y) : 0; //first right step + double srstep = (rightmost->p.y != tailp->p.x) ? + (rightmost->p.x - tailp->p.x) / (rightmost->p.y - tailp->p.x) : 0; //second right step + + double lstep = flstep, rstep = frstep; + + double left_x = min_y->p.x, right_x = min_y->p.x; + + // Loop around all points in the region and count those that are aligned. + int min_iter = min_y->p.y; + int max_iter = max_y->p.y; + for(int y = min_iter; y <= max_iter; ++y) + { + if (y < 0 || y >= img_height) continue; + + for(int x = int(left_x); x <= int(right_x); ++x) + { + if (x < 0 || x >= img_width) continue; + + ++total_pts; + if(isAligned(x, y, rec.theta, rec.prec)) + { + ++alg_pts; + } + } + + if(y >= leftmost->p.y) { lstep = slstep; } + if(y >= rightmost->p.y) { rstep = srstep; } + + left_x += lstep; + right_x += rstep; + } + + return nfa(total_pts, alg_pts, rec.p); +} + +double LineSegmentDetectorImpl::nfa(const int& n, const int& k, const double& p) const +{ + // Trivial cases + if(n == 0 || k == 0) { return -LOG_NT; } + if(n == k) { return -LOG_NT - double(n) * log10(p); } + + double p_term = p / (1 - p); + + double log1term = (double(n) + 1) - log_gamma(double(k) + 1) + - log_gamma(double(n-k) + 1) + + double(k) * log(p) + double(n-k) * log(1.0 - p); + double term = exp(log1term); + + if(double_equal(term, 0)) + { + if(k > n * p) return -log1term / M_LN10 - LOG_NT; + else return -LOG_NT; + } + + // Compute more terms if needed + double bin_tail = term; + double tolerance = 0.1; // an error of 10% in the result is accepted + for(int i = k + 1; i <= n; ++i) + { + double bin_term = double(n - i + 1) / double(i); + double mult_term = bin_term * p_term; + term *= mult_term; + bin_tail += term; + if(bin_term < 1) + { + double err = term * ((1 - pow(mult_term, double(n-i+1))) / (1 - mult_term) - 1); + if(err < tolerance * fabs(-log10(bin_tail) - LOG_NT) * bin_tail) break; + } + + } + return -log10(bin_tail) - LOG_NT; +} + +inline bool LineSegmentDetectorImpl::isAligned(int x, int y, const double& theta, const double& prec) const +{ + if(x < 0 || y < 0 || x >= angles.cols || y >= angles.rows) { return false; } + const double& a = angles.at(y, x); + if(a == NOTDEF) { return false; } + + // It is assumed that 'theta' and 'a' are in the range [-pi,pi] + double n_theta = theta - a; + if(n_theta < 0) { n_theta = -n_theta; } + if(n_theta > M_3_2_PI) + { + n_theta -= M_2__PI; + if(n_theta < 0) n_theta = -n_theta; + } + + return n_theta <= prec; +} + + void LineSegmentDetectorImpl::drawSegments(InputOutputArray _image, InputArray lines) { CV_INSTRUMENT_REGION(); diff --git a/modules/imgproc/test/test_lsd.cpp b/modules/imgproc/test/test_lsd.cpp index e162a3c6f42d..43d00b4928a2 100644 --- a/modules/imgproc/test/test_lsd.cpp +++ b/modules/imgproc/test/test_lsd.cpp @@ -5,8 +5,6 @@ namespace opencv_test { namespace { -#if 0 // LSD implementation has been removed due original code license issues - const Size img_size(640, 480); const int LSD_TEST_SEED = 0x134679; const int EPOCHS = 20; @@ -404,6 +402,4 @@ TEST_F(Imgproc_LSD_Common, compareSegmentsVec4i) ASSERT_EQ(result2, 11); } -#endif - }} // namespace diff --git a/samples/cpp/lsd_lines.cpp b/samples/cpp/lsd_lines.cpp new file mode 100644 index 000000000000..3feed9cbc2c6 --- /dev/null +++ b/samples/cpp/lsd_lines.cpp @@ -0,0 +1,73 @@ +#include "opencv2/imgproc.hpp" +#include "opencv2/imgcodecs.hpp" +#include "opencv2/highgui.hpp" +#include + +using namespace std; +using namespace cv; + +int main(int argc, char** argv) +{ + cv::CommandLineParser parser(argc, argv, + "{input i|building.jpg|input image}" + "{refine r|false|if true use LSD_REFINE_STD method, if false use LSD_REFINE_NONE method}" + "{canny c|false|use Canny edge detector}" + "{overlay o|false|show result on input image}" + "{help h|false|show help message}"); + + if (parser.get("help")) + { + parser.printMessage(); + return 0; + } + + parser.printMessage(); + + String filename = samples::findFile(parser.get("input")); + bool useRefine = parser.get("refine"); + bool useCanny = parser.get("canny"); + bool overlay = parser.get("overlay"); + + Mat image = imread(filename, IMREAD_GRAYSCALE); + + if( image.empty() ) + { + cout << "Unable to load " << filename; + return 1; + } + + imshow("Source Image", image); + + if (useCanny) + { + Canny(image, image, 50, 200, 3); // Apply Canny edge detector + } + + // Create and LSD detector with standard or no refinement. + Ptr ls = useRefine ? createLineSegmentDetector(LSD_REFINE_STD) : createLineSegmentDetector(LSD_REFINE_NONE); + + double start = double(getTickCount()); + vector lines_std; + + // Detect the lines + ls->detect(image, lines_std); + + double duration_ms = (double(getTickCount()) - start) * 1000 / getTickFrequency(); + std::cout << "It took " << duration_ms << " ms." << std::endl; + + // Show found lines + if (!overlay || useCanny) + { + image = Scalar(0, 0, 0); + } + + ls->drawSegments(image, lines_std); + + String window_name = useRefine ? "Result - standard refinement" : "Result - no refinement"; + window_name += useCanny ? " - Canny edge detector used" : ""; + + imshow(window_name, image); + + waitKey(); + return 0; +} From 6d83a73858b2b1103e103ad0d4c51b147cab41db Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Fri, 1 Oct 2021 13:07:55 +0300 Subject: [PATCH 236/376] Fix bugs with hanging frames --- modules/gapi/src/backends/ie/giebackend.cpp | 6 +- .../gapi/src/executor/gstreamingexecutor.cpp | 4 + .../gapi/test/infer/gapi_infer_ie_test.cpp | 135 ++++++++++++++++++ 3 files changed, 144 insertions(+), 1 deletion(-) diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 007f0db7afcc..03584c9561c9 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -630,6 +630,11 @@ void cv::gimpl::ie::RequestPool::callback(cv::gimpl::ie::RequestPool::Task task, InferenceEngine::InferRequest& request, size_t id) { task.callback(request); + // NB: IE::InferRequest keeps the callback until the new one is set. + // Since user's callback might keep resources that should be released, + // need to destroy its after execution. + // Let's set the empty one to cause the destruction of a callback. + request.SetCompletionCallback([](){}); m_idle_ids.push(id); } @@ -831,7 +836,6 @@ static void PostOutputs(InferenceEngine::InferRequest &request, auto output = ctx->output(i); ctx->out.meta(output, ctx->input(0).meta); ctx->out.post(std::move(output)); - } } diff --git a/modules/gapi/src/executor/gstreamingexecutor.cpp b/modules/gapi/src/executor/gstreamingexecutor.cpp index 2379e3e16499..bc43db8a3779 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.cpp +++ b/modules/gapi/src/executor/gstreamingexecutor.cpp @@ -347,6 +347,10 @@ bool QueueReader::getInputVector(std::vector &in_queues, cv::GRunArgs &in_constants, cv::GRunArgs &isl_inputs) { + // NB: Need to release resources from the previous step, to fetch new ones. + // On some systems it might be impossible to allocate new memory + // until the old one is released. + m_cmd.clear(); // NOTE: in order to maintain the GRunArg's underlying object // lifetime, keep the whole cmd vector (of size == # of inputs) // in memory. diff --git a/modules/gapi/test/infer/gapi_infer_ie_test.cpp b/modules/gapi/test/infer/gapi_infer_ie_test.cpp index e33f165ae165..b7ea891b812f 100644 --- a/modules/gapi/test/infer/gapi_infer_ie_test.cpp +++ b/modules/gapi/test/infer/gapi_infer_ie_test.cpp @@ -9,6 +9,8 @@ #ifdef HAVE_INF_ENGINE #include +#include +#include #include @@ -2052,6 +2054,139 @@ TEST(IEFrameAdapter, blobParams) EXPECT_EQ(expected, actual); } +namespace +{ + +struct Sync { + std::mutex m; + std::condition_variable cv; + int counter = 0; +}; + +class GMockMediaAdapter final: public cv::MediaFrame::IAdapter { +public: + explicit GMockMediaAdapter(cv::Mat m, Sync& sync) + : m_mat(m), m_sync(sync) { + } + + cv::GFrameDesc meta() const override { + return cv::GFrameDesc{cv::MediaFormat::BGR, m_mat.size()}; + } + + cv::MediaFrame::View access(cv::MediaFrame::Access) override { + cv::MediaFrame::View::Ptrs pp = { m_mat.ptr(), nullptr, nullptr, nullptr }; + cv::MediaFrame::View::Strides ss = { m_mat.step, 0u, 0u, 0u }; + return cv::MediaFrame::View(std::move(pp), std::move(ss)); + } + + ~GMockMediaAdapter() { + { + std::lock_guard lk{m_sync.m}; + m_sync.counter--; + } + m_sync.cv.notify_one(); + } + +private: + cv::Mat m_mat; + Sync& m_sync; +}; + +// NB: This source is needed to simulate real +// cases where the memory resources are limited. +// GMockSource(int limit) - accept the number of MediaFrames that +// the source can produce until resources are over. +class GMockSource : public cv::gapi::wip::IStreamSource { +public: + explicit GMockSource(int limit) + : m_limit(limit), m_mat(cv::Size(1920, 1080), CV_8UC3) { + cv::randu(m_mat, cv::Scalar::all(0), cv::Scalar::all(255)); + } + + bool pull(cv::gapi::wip::Data& data) { + std::unique_lock lk(m_sync.m); + m_sync.counter++; + // NB: Can't produce new frames until old ones are released. + m_sync.cv.wait(lk, [this]{return m_sync.counter <= m_limit;}); + + data = cv::MediaFrame::Create(m_mat, m_sync); + return true; + } + + GMetaArg descr_of() const override { + return GMetaArg{cv::GFrameDesc{cv::MediaFormat::BGR, m_mat.size()}}; + } + +private: + int m_limit; + cv::Mat m_mat; + Sync m_sync; +}; + +struct LimitedSourceInfer: public ::testing::Test { + using AGInfo = std::tuple; + G_API_NET(AgeGender, , "test-age-gender"); + + LimitedSourceInfer() + : comp([](){ + cv::GFrame in; + cv::GMat age, gender; + std::tie(age, gender) = cv::gapi::infer(in); + return cv::GComputation(cv::GIn(in), cv::GOut(age, gender)); + }) { + initDLDTDataPath(); + } + + GStreamingCompiled compileStreaming(int nireq) { + cv::gapi::ie::detail::ParamDesc params; + params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); + params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); + params.device_id = "CPU"; + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id } + .cfgOutputLayers({ "age_conv3", "prob" }) + .cfgNumRequests(nireq); + + return comp.compileStreaming(cv::compile_args(cv::gapi::networks(pp))); + } + + void run(const int max_frames, const int limit, const int nireq) { + auto pipeline = compileStreaming(nireq); + pipeline.setSource(limit); + pipeline.start(); + + int num_frames = 0; + while (num_frames != max_frames && + pipeline.pull(cv::gout(out_age, out_gender))) { + ++num_frames; + } + } + + cv::GComputation comp; + cv::Mat out_age, out_gender; +}; + +} // anonymous namespace + +TEST_F(LimitedSourceInfer, ReleaseFrame) +{ + constexpr int max_frames = 50; + constexpr int resources_limit = 1; + constexpr int nireq = 1; + + run(max_frames, resources_limit, nireq); +} + +TEST_F(LimitedSourceInfer, ReleaseFrameAsync) +{ + constexpr int max_frames = 50; + constexpr int resources_limit = 4; + constexpr int nireq = 8; + + run(max_frames, resources_limit, nireq); +} + } // namespace opencv_test #endif // HAVE_INF_ENGINE From 1e74f5850bec99979584e0e37eeeab8c133be3a9 Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Fri, 1 Oct 2021 23:17:02 +0900 Subject: [PATCH 237/376] suppress GaussianBlur to generate empty images * sharp Gaussian kernel causes over flow and ends up in blank image --- modules/cudafilters/test/test_filters.cpp | 4 ++-- modules/imgproc/src/smooth.simd.hpp | 5 ++++- modules/imgproc/test/test_smooth_bitexact.cpp | 11 +++++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/modules/cudafilters/test/test_filters.cpp b/modules/cudafilters/test/test_filters.cpp index a86d8c9fff44..bb235dad0937 100644 --- a/modules/cudafilters/test/test_filters.cpp +++ b/modules/cudafilters/test/test_filters.cpp @@ -445,8 +445,8 @@ PARAM_TEST_CASE(GaussianBlur, cv::cuda::DeviceInfo, cv::Size, MatDepth, Channels CUDA_TEST_P(GaussianBlur, Accuracy) { cv::Mat src = randomMat(size, type); - double sigma1 = randomDouble(0.1, 1.0); - double sigma2 = randomDouble(0.1, 1.0); + double sigma1 = randomDouble(0.0, 1.0); + double sigma2 = randomDouble(0.0, 1.0); cv::Ptr gauss = cv::cuda::createGaussianFilter(src.type(), -1, ksize, sigma1, sigma2, borderType); diff --git a/modules/imgproc/src/smooth.simd.hpp b/modules/imgproc/src/smooth.simd.hpp index 3a39765c71f2..62ff31ac940c 100644 --- a/modules/imgproc/src/smooth.simd.hpp +++ b/modules/imgproc/src/smooth.simd.hpp @@ -1958,7 +1958,10 @@ class fixedSmoothInvoker : public ParallelLoopBody } else if (kxlen % 2 == 1) { - hlineSmoothFunc = hlineSmoothONa_yzy_a; + if (kx[(kxlen - 1)/ 2] == FT::one()) + hlineSmoothFunc = hlineSmooth1N1; + else + hlineSmoothFunc = hlineSmoothONa_yzy_a; for (int i = 0; i < kxlen / 2; i++) if (!(kx[i] == kx[kxlen - 1 - i])) { diff --git a/modules/imgproc/test/test_smooth_bitexact.cpp b/modules/imgproc/test/test_smooth_bitexact.cpp index 246f1df7980b..d4ae2af833b1 100644 --- a/modules/imgproc/test/test_smooth_bitexact.cpp +++ b/modules/imgproc/test/test_smooth_bitexact.cpp @@ -249,4 +249,15 @@ TEST(GaussianBlur_Bitexact, regression_9863) checkGaussianBlur_8Uvs32F(src8u, src32f, 151, 30); } +TEST(GaussianBlur_Bitexact, overflow_20792) +{ + Mat src(128, 128, CV_16UC1, Scalar(255)); + Mat dst; + double sigma = theRNG().uniform(0.0, 0.2); // a peaky kernel + GaussianBlur(src, dst, Size(7, 7), sigma, 0.9); + int count = (int)countNonZero(dst); + int nintyPercent = (int)(src.rows*src.cols * 0.9); + EXPECT_GT(count, nintyPercent); +} + }} // namespace From f977d10a19e1da13532ea038ac8d7e2cf882040f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 1 Oct 2021 18:51:01 +0000 Subject: [PATCH 238/376] dnn(ocl): fix conv DWCONV workgroup --- .../dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 15 +++++++++------ modules/dnn/src/opencl/conv_layer_spatial.cl | 11 +++++++---- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index a1164273accf..45bd249e5d62 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -1034,12 +1034,15 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, kernel.set(argIdx++, (uint16_t)output_w_); kernel.set(argIdx++, (uint16_t)output_h_); - size_t global_size[3]; - global_size[0] = output_w_; - global_size[1] = output_h_; - global_size[2] = num_output_ * num_; - - if (!kernel.run_(3, global_size, NULL, false)) + size_t wgs = kernel.workGroupSize(); + if (!wgs) + { + CV_LOG_ERROR(NULL, "DNN/OpenCL: Can't query workGroupSize of DWCONV kernel"); + return false; + } + size_t lws[1] = { wgs }; + size_t gws[1] = { roundUp((size_t)output_w_ * output_h_ * num_output_ * num_, (unsigned)lws[0]) }; + if (!kernel.run_(1, gws, lws, false)) { CV_LOG_ERROR(NULL, "DNN/OpenCL: DWCONV kernel run failed"); return false; diff --git a/modules/dnn/src/opencl/conv_layer_spatial.cl b/modules/dnn/src/opencl/conv_layer_spatial.cl index 455f0ed7ea06..eb5d354020bd 100644 --- a/modules/dnn/src/opencl/conv_layer_spatial.cl +++ b/modules/dnn/src/opencl/conv_layer_spatial.cl @@ -1850,10 +1850,13 @@ __kernel void DWCONV( const ushort output_width, const ushort output_height) { __global Dtype* convolved_image = convolved_image_base + convolved_image_offset; - const int outputX = get_global_id(0); - const int outputY = get_global_id(1); - const int outputZ = get_global_id(2); - if(outputX < output_width && outputY < output_height) + const int out_idx = get_global_id(0); // 1D task layout: [output_width * output_height * OUTPUT_Z] + const int plane_size = output_width * output_height; + const int out_plane_idx = out_idx % plane_size; + const int outputZ = out_idx / plane_size; + const int outputY = out_plane_idx / output_width; + const int outputX = out_plane_idx % output_width; + if (outputZ < OUTPUT_Z) { Dtype sum = 0.; From 4d87f6025eb4be3b2f2a27fb542b5c46fce60193 Mon Sep 17 00:00:00 2001 From: shengyu Date: Wed, 29 Sep 2021 21:02:40 +0800 Subject: [PATCH 239/376] remove redundant semicolons --- modules/gapi/include/opencv2/gapi/own/types.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/own/types.hpp b/modules/gapi/include/opencv2/gapi/own/types.hpp index 53bb867e32f9..6bd688098269 100644 --- a/modules/gapi/include/opencv2/gapi/own/types.hpp +++ b/modules/gapi/include/opencv2/gapi/own/types.hpp @@ -27,7 +27,7 @@ class Point { public: Point() = default; - Point(int _x, int _y) : x(_x), y(_y) {}; + Point(int _x, int _y) : x(_x), y(_y) {} int x = 0; int y = 0; @@ -37,7 +37,7 @@ class Point2f { public: Point2f() = default; - Point2f(float _x, float _y) : x(_x), y(_y) {}; + Point2f(float _x, float _y) : x(_x), y(_y) {} float x = 0.f; float y = 0.f; @@ -47,9 +47,9 @@ class Rect { public: Rect() = default; - Rect(int _x, int _y, int _width, int _height) : x(_x), y(_y), width(_width), height(_height) {}; + Rect(int _x, int _y, int _width, int _height) : x(_x), y(_y), width(_width), height(_height) {} #if !defined(GAPI_STANDALONE) - Rect(const cv::Rect& other) : x(other.x), y(other.y), width(other.width), height(other.height) {}; + Rect(const cv::Rect& other) : x(other.x), y(other.y), width(other.width), height(other.height) {} inline Rect& operator=(const cv::Rect& other) { x = other.x; @@ -104,9 +104,9 @@ class Size { public: Size() = default; - Size(int _width, int _height) : width(_width), height(_height) {}; + Size(int _width, int _height) : width(_width), height(_height) {} #if !defined(GAPI_STANDALONE) - Size(const cv::Size& other) : width(other.width), height(other.height) {}; + Size(const cv::Size& other) : width(other.width), height(other.height) {} inline Size& operator=(const cv::Size& rhs) { width = rhs.width; From 259c39a63aa0be0d1bcda6bc6741548b24ecd13d Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Sat, 2 Oct 2021 13:32:11 +0300 Subject: [PATCH 240/376] additional changes --- modules/imgproc/include/opencv2/imgproc.hpp | 7 +++++-- modules/imgproc/src/lsd.cpp | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index 855cbaf15983..576bebc21b4c 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -1228,6 +1228,9 @@ An example using the LineSegmentDetector /** @brief Line segment detector class following the algorithm described at @cite Rafael12 . + +@note Implementation has been removed from OpenCV version 3.4.6 to 3.4.15 and version 4.1.0 to 4.5.3 due original code license conflict. +restored again after [Computation of a NFA](https://github.com/rafael-grompone-von-gioi/binomial_nfa) code published under the MIT license. */ class CV_EXPORTS_W LineSegmentDetector : public Algorithm { @@ -1241,8 +1244,8 @@ class CV_EXPORTS_W LineSegmentDetector : public Algorithm @param image A grayscale (CV_8UC1) input image. If only a roi needs to be selected, use: `lsd_ptr-\>detect(image(roi), lines, ...); lines += Scalar(roi.x, roi.y, roi.x, roi.y);` - @param lines A vector of Vec4i or Vec4f elements specifying the beginning and ending point of a line. Where - Vec4i/Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 - end. Returned lines are strictly + @param lines A vector of Vec4f elements specifying the beginning and ending point of a line. Where + Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 - end. Returned lines are strictly oriented depending on the gradient. @param width Vector of widths of the regions, where the lines are found. E.g. Width of line. @param prec Vector of precisions with which the lines are found. diff --git a/modules/imgproc/src/lsd.cpp b/modules/imgproc/src/lsd.cpp index d06759c2bb3b..c4a77ee265b0 100644 --- a/modules/imgproc/src/lsd.cpp +++ b/modules/imgproc/src/lsd.cpp @@ -191,8 +191,8 @@ class LineSegmentDetectorImpl CV_FINAL : public LineSegmentDetector * If only a roi needs to be selected, use * lsd_ptr->detect(image(roi), ..., lines); * lines += Scalar(roi.x, roi.y, roi.x, roi.y); - * @param _lines Return: A vector of Vec4i or Vec4f elements specifying the beginning and ending point of a line. - * Where Vec4i/Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 - end. + * @param _lines Return: A vector of Vec4f elements specifying the beginning and ending point of a line. + * Where Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 - end. * Returned lines are strictly oriented depending on the gradient. * @param width Return: Vector of widths of the regions, where the lines are found. E.g. Width of line. * @param prec Return: Vector of precisions with which the lines are found. From ef53a9229f4f666459480b2366a4680921d45e5a Mon Sep 17 00:00:00 2001 From: Jannik Bamberger Date: Sat, 2 Oct 2021 00:04:23 +0200 Subject: [PATCH 241/376] Automatically set the correct OpenCV version in build.gradle Automatically sets the correct OpenCV version in the CMAKE example contained in the build.gradle file of the Android SDK. --- modules/java/android_sdk/build.gradle.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/java/android_sdk/build.gradle.in b/modules/java/android_sdk/build.gradle.in index 6d317fcd3961..14a7b2c6fe21 100644 --- a/modules/java/android_sdk/build.gradle.in +++ b/modules/java/android_sdk/build.gradle.in @@ -58,7 +58,7 @@ // // - Use find_package() in app/CMakeLists.txt: // -// find_package(OpenCV 3.4 REQUIRED java) +// find_package(OpenCV @OPENCV_VERSION_MAJOR@.@OPENCV_VERSION_MINOR@ REQUIRED java) // ... // target_link_libraries(native-lib ${OpenCV_LIBRARIES}) // From de5b6386e01d2e6ea622913627b8776c73d6ada3 Mon Sep 17 00:00:00 2001 From: Jonas Vautherin Date: Sat, 2 Oct 2021 03:07:25 +0200 Subject: [PATCH 242/376] Fix gst_initializer Use the return value of gst_init_check instead of testing the error pointer --- modules/videoio/src/cap_gstreamer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index 6bad5289098e..d8390691ffda 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -216,10 +216,10 @@ class gst_initializer call_deinit = utils::getConfigurationParameterBool("OPENCV_VIDEOIO_GSTREAMER_CALL_DEINIT", false); GSafePtr err; - gst_init_check(NULL, NULL, err.getRef()); - if (err) + gboolean gst_init_res = gst_init_check(NULL, NULL, err.getRef()); + if (!gst_init_res) { - CV_WARN("Can't initialize GStreamer: " << err->message); + CV_WARN("Can't initialize GStreamer: " << (err ? err->message : "")); isFailed = true; return; } From 87ebf2e50bc468356493dcc6860c482c902ad69e Mon Sep 17 00:00:00 2001 From: SamFC10 Date: Sun, 3 Oct 2021 15:16:01 +0530 Subject: [PATCH 243/376] fix illegal memory access in int8 convolution --- .../dnn/src/int8layers/convolution_layer.cpp | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/modules/dnn/src/int8layers/convolution_layer.cpp b/modules/dnn/src/int8layers/convolution_layer.cpp index 05749885c05b..ea29610222e7 100644 --- a/modules/dnn/src/int8layers/convolution_layer.cpp +++ b/modules/dnn/src/int8layers/convolution_layer.cpp @@ -159,7 +159,7 @@ class ConvolutionLayerInt8Impl CV_FINAL : public BaseConvolutionLayerInt8Impl enum { VEC_ALIGN = 32, DFT_TYPE = CV_8S }; Mat weightsMat; std::vector biasvec; - Mat outputMultiplier; + std::vector outputMultiplier; Mat activationLUT; Ptr activ; @@ -249,10 +249,14 @@ class ConvolutionLayerInt8Impl CV_FINAL : public BaseConvolutionLayerInt8Impl Mat biasMat = blobs[1]; biasvec.resize(numOutput+2); + + Mat outMult = blobs[2]; + outputMultiplier.resize(numOutput+2); for(int i = 0; i < numOutput; i++ ) + { biasvec[i] = biasMat.at(i); - - outputMultiplier = blobs[2]; + outputMultiplier[i] = outMult.at(i); + } } bool setActivation(const Ptr& layer) CV_OVERRIDE @@ -283,16 +287,17 @@ class ConvolutionLayerInt8Impl CV_FINAL : public BaseConvolutionLayerInt8Impl for (int i = 0; i < outCn; ++i) { - float off = outputMultiplier.at(i) * output_sc; + float off = outputMultiplier[i] * output_sc; if (!w.empty()) off *= w.at(i); if (!b.empty()) biasvec[i] += (int)std::round(b.at(i)/off); - outputMultiplier.at(i) = off/new_sc; + outputMultiplier[i] = off/new_sc; } biasvec[outCn] = biasvec[outCn+1] = biasvec[outCn-1]; + outputMultiplier[outCn] = outputMultiplier[outCn+1] = outputMultiplier[outCn-1]; } class ParallelConv : public cv::ParallelLoopBody @@ -315,7 +320,7 @@ class ConvolutionLayerInt8Impl CV_FINAL : public BaseConvolutionLayerInt8Impl bool useAVX512; int blk_size_cn; int inpZp, outZp; - const float* multiplier; + const std::vector* multiplier; ParallelConv() : input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0), @@ -323,7 +328,7 @@ class ConvolutionLayerInt8Impl CV_FINAL : public BaseConvolutionLayerInt8Impl , blk_size_cn(0), inpZp(0), outZp(0), multiplier(0) {} - static void run( const Mat& input, Mat& output, const Mat& weights, const Mat& multipliers, + static void run( const Mat& input, Mat& output, const Mat& weights, const std::vector& multipliers, const std::vector& biasvec, const Mat& activLUT, const std::vector& kernel_size, const std::vector& strides, const std::vector& pads_begin, const std::vector& pads_end, @@ -392,7 +397,7 @@ class ConvolutionLayerInt8Impl CV_FINAL : public BaseConvolutionLayerInt8Impl p.inpZp = inp_Zp; p.outZp = out_Zp; - p.multiplier = multipliers.ptr(0); + p.multiplier = &multipliers; p.ofstab_.resize(karea * ncn); int* ofstab = &p.ofstab_[0]; @@ -501,6 +506,7 @@ class ConvolutionLayerInt8Impl CV_FINAL : public BaseConvolutionLayerInt8Impl const int8_t* wptr_orig_ = weights_->ptr(); size_t wstep = weights_->step1(); const int* biasptr_ = &biasvec_->at(0); + const float* multptr_ = &multiplier->at(0); const int* lutptr_ = !activLUT_->empty() ? activLUT_->ptr() : 0; int* data_out0_ = output_->ptr(); AutoBuffer rowbuf0_; @@ -539,7 +545,7 @@ class ConvolutionLayerInt8Impl CV_FINAL : public BaseConvolutionLayerInt8Impl int startOutCn = (subsampleIdx % ngroups)*outCn; const int8_t* wptr_orig = wptr_orig_ + wstep*startOutCn; const int* biasptr = biasptr_ + startOutCn; - const float* multptr = multiplier + startOutCn; + const float* multptr = multptr_ + startOutCn; for( int cn0 = 0; cn0 < inpCn; cn0 += blk_size_cn ) { From 505dde09de0b245bb9958d42673671761a690be1 Mon Sep 17 00:00:00 2001 From: YashasSamaga Date: Mon, 4 Oct 2021 12:38:45 +0530 Subject: [PATCH 244/376] support broadcasting in eltwise ops --- modules/dnn/src/cuda/eltwise_ops.cu | 271 ++++++++++++++++-- modules/dnn/src/cuda/kernel_dispatcher.hpp | 18 ++ .../dnn/src/cuda4dnn/kernels/eltwise_ops.hpp | 14 +- modules/dnn/src/dnn.cpp | 14 +- 4 files changed, 282 insertions(+), 35 deletions(-) diff --git a/modules/dnn/src/cuda/eltwise_ops.cu b/modules/dnn/src/cuda/eltwise_ops.cu index 109c3fbdc9a2..db4daa47188a 100644 --- a/modules/dnn/src/cuda/eltwise_ops.cu +++ b/modules/dnn/src/cuda/eltwise_ops.cu @@ -5,13 +5,16 @@ #include #include +#include "array.hpp" #include "functors.hpp" #include "grid_stride_range.hpp" #include "execution.hpp" #include "vector_traits.hpp" +#include "kernel_dispatcher.hpp" #include "../cuda4dnn/csl/stream.hpp" #include "../cuda4dnn/csl/span.hpp" +#include "../cuda4dnn/csl/tensor.hpp" #include @@ -40,6 +43,32 @@ namespace raw { v_store(output_vPtr[i], vec_x); } } + + template + __global__ void eltwise_op_bcast( + Span output, array out_strides, + View x, array x_strides, array x_bcast, + View y, array y_strides, array y_bcast, + const typename EltwiseOp::Params params) { + EltwiseOp eltwise_op(params); + + for (auto i : grid_stride_range(output.size())) { + index_type out_index = i / out_strides[0]; + index_type x_index = x_bcast[0] ? 0 : out_index * x_strides[0]; + index_type y_index = y_bcast[0] ? 0 : out_index * y_strides[0]; + + for (int j = 1; j < Rank; j++) + { + out_index = (i % out_strides[j - 1]) / out_strides[j]; + if (!x_bcast[j]) + x_index += out_index * x_strides[j]; + if (!y_bcast[j]) + y_index += out_index * y_strides[j]; + } + + output[i] = eltwise_op(x[x_index], y[y_index]); + } + } } template static @@ -55,63 +84,251 @@ void launch_vectorized_eltwise_op(const Stream& stream, Span output, View launch_kernel(kernel, policy, output, x, y, params); } +template static +void launch_eltwise_op_bcast( + const Stream& stream, + Span output, const std::vector& outStride, + View x, const std::vector& inStride1, const std::vector& inBcast1, + View y, const std::vector& inStride2, const std::vector& inBcast2, + const typename EltwiseOp::Params& params) +{ + CV_Assert(outStride.size() == Rank); + CV_Assert(inStride1.size() == Rank); + CV_Assert(inStride2.size() == Rank); + CV_Assert(inBcast1.size() == Rank); + CV_Assert(inBcast2.size() == Rank); + + array outStride_k, inStride1_k, inStride2_k; + outStride_k.assign(std::begin(outStride), std::end(outStride)); + inStride1_k.assign(std::begin(inStride1), std::end(inStride1)); + inStride2_k.assign(std::begin(inStride2), std::end(inStride2)); + + array inBcast1_k, inBcast2_k; + inBcast1_k.assign(std::begin(inBcast1), std::end(inBcast1)); + inBcast2_k.assign(std::begin(inBcast2), std::end(inBcast2)); + + auto kernel = raw::eltwise_op_bcast; + auto policy = make_policy(kernel, output.size(), 0, stream); + launch_kernel(kernel, policy, output, outStride_k, x, inStride1_k, inBcast1_k, y, inStride2_k, inBcast2_k, params); +} + +GENERATE_KERNEL_DISPATCHER_2TP(eltwise_op_bcast_dispatcher, launch_eltwise_op_bcast); + template static -void eltwise_op(const Stream& stream, Span output, View x, View y, const typename EltwiseOp::Params& params = {}) { - CV_Assert(x.size() == y.size()); - CV_Assert(x.size() == output.size()); +void eltwise_op(const Stream& stream, TensorSpan output, TensorView x, TensorView y, const typename EltwiseOp::Params& params = {}) { + if (is_shape_same(output, x) && is_shape_same(output, y)) + { + /* no broadcasting; use fast path */ + CV_Assert(x.size() == y.size()); + CV_Assert(x.size() == output.size()); + + if (is_fully_aligned(output, 4) && is_fully_aligned(x, 4) && is_fully_aligned(y, 4)) { + launch_vectorized_eltwise_op(stream, output, x, y, params); + } else if (is_fully_aligned(output, 2) && is_fully_aligned(x, 2) && is_fully_aligned(y, 2)) { + launch_vectorized_eltwise_op(stream, output, x, y, params); + } else { + launch_vectorized_eltwise_op(stream, output, x, y, params); + } + } + else + { + CV_Assert(is_shape_compatible(output, x)); + CV_Assert(is_shape_compatible(output, y)); + + /* matching singleton axes in both input tensors can be eliminated + * + * Reasoning: + * ---------- + * Singleton axes do not contribute towards address calculation. They are redundant + * unless there is broadcasting. If both input tensors have singleton axis at a + * specified position, there is no broadcasting on that axis. + * + * Example: + * --------- + * x: [1, 256, 32, 32] -> [256, 32, 32] + * y: [1, 256, 1, 1] -> [256, 1, 1] + */ + for (int r = 0; r < output.rank(); r++) + { + while (x.get_axis_size(r) == 1 && y.get_axis_size(r) == 1) { + CV_Assert(output.get_axis_size(r) == 1); + + x.squeeze(r); + y.squeeze(r); + output.squeeze(r); + } + } + + auto inShape1 = x.shape_as_vector(); + auto inShape2 = y.shape_as_vector(); + auto outShape = output.shape_as_vector(); + + /* contiguous axes that do not broadcast can be merged into one axis + * + * Example: + * --------- + * x: [32, 8, 8] -> [32, 64] + * y: [1, 8, 8] -> [1, 64] + */ + for (int i = 0; i < inShape1.size(); i++) { + /* check if axis `i` requires any broadcasting */ + if (inShape1[i] == inShape2[i]) { + /* loop invariant: `i` is the first axis in the contiguous axis sequence */ + + int j = i + 1; /* `j` is the axis which we will attempt to merge */ + while (j < inShape1.size() && inShape1[j] == inShape2[j]) { + CV_Assert(outShape[j] == inShape1[j]); + + /* `j` axis is also used fully; merge `i` and `j` */ + auto new_size = inShape1[i] * inShape1[j]; + inShape1[i] = new_size; + inShape2[i] = new_size; + + /* delete axis `j` */ + inShape1.erase(std::begin(inShape1) + j); + inShape2.erase(std::begin(inShape2) + j); + outShape.erase(std::begin(outShape) + j); + + /* optimizations should not break the invariants */ + CV_Assert(inShape1.size() == outShape.size()); + CV_Assert(inShape2.size() == outShape.size()); + CV_Assert(inShape1[i] == outShape[i]); + CV_Assert(inShape2[i] == outShape[i]); + } + } + } + + /* contiguous broadcasting axes on the same tensor can be merged into one axis + * + * Example: + * --------- + * x: [256, 8, 8] -> [256, 64] + * y: [256, 1, 1] -> [256, 1] + */ + for (int i = 0; i < inShape1.size(); i++) { + /* check if axis `i` requires any broadcasting in tensor 1 */ + if (inShape1[i] == 1 && inShape2[i] != 1) { + /* loop invariant: `i` is the first axis in the contiguous axis sequence */ + + int j = i + 1; /* `j` is the axis which we will attempt to merge */ + while (j < inShape1.size() && inShape1[j] == 1 && inShape2[j] != 1) { + CV_Assert(outShape[j] == inShape2[j]); + + /* `j` axis is also used fully; merge `i` and `j` */ + inShape1[i] = 1; + inShape2[i] = inShape2[i] * inShape2[j]; + outShape[i] = inShape2[i]; + + /* delete axis `j` */ + inShape1.erase(std::begin(inShape1) + j); + inShape2.erase(std::begin(inShape2) + j); + outShape.erase(std::begin(outShape) + j); + + /* optimizations should not break the invariants */ + CV_Assert(inShape1.size() == outShape.size()); + CV_Assert(inShape2.size() == outShape.size()); + CV_Assert(inShape1[i] == 1); + CV_Assert(inShape2[i] == outShape[i]); + } + } + + /* check if axis `i` requires any broadcasting in tensor 2 */ + if (inShape1[i] != 1 && inShape2[i] == 1) { + /* loop invariant: `i` is the first axis in the contiguous axis sequence */ + + int j = i + 1; /* `j` is the axis which we will attempt to merge */ + while (j < inShape1.size() && inShape1[j] != 1 && inShape2[j] == 1) { + CV_Assert(outShape[j] == inShape1[j]); + + /* `j` axis is also used fully; merge `i` and `j` */ + inShape1[i] = inShape1[i] * inShape1[j]; + inShape2[i] = 1; + outShape[i] = inShape1[i]; + + /* delete axis `j` */ + inShape1.erase(std::begin(inShape1) + j); + inShape2.erase(std::begin(inShape2) + j); + outShape.erase(std::begin(outShape) + j); + + /* optimizations should not break the invariants */ + CV_Assert(inShape1.size() == outShape.size()); + CV_Assert(inShape2.size() == outShape.size()); + CV_Assert(inShape1[i] == outShape[i]); + CV_Assert(inShape2[i] == 1); + } + } + } + + auto rank = outShape.size(); + + std::vector inStride1(rank), inStride2(rank), outStride(rank); + inStride1.back() = 1; + inStride2.back() = 1; + outStride.back() = 1; + /* garbage, ..., garbage, 1 */ + + std::copy(std::begin(inShape1) + 1, std::end(inShape1), std::begin(inStride1)); + std::copy(std::begin(inShape2) + 1, std::end(inShape2), std::begin(inStride2)); + std::copy(std::begin(outShape) + 1, std::end(outShape), std::begin(outStride)); + /* dim[0], dim[1], ..., dim[-1], 1 */ + + std::partial_sum(inStride1.rbegin(), inStride1.rend(), inStride1.rbegin(), std::multiplies()); + std::partial_sum(inStride2.rbegin(), inStride2.rend(), inStride2.rbegin(), std::multiplies()); + std::partial_sum(outStride.rbegin(), outStride.rend(), outStride.rbegin(), std::multiplies()); + /* stride[0], stride[1], ..., stride[-2], 1 */ + + std::vector inBcast1(rank), inBcast2(rank); + std::transform(std::begin(inShape1), std::end(inShape1), std::begin(inBcast1), [](std::size_t sz) { return sz == 1; }); + std::transform(std::begin(inShape2), std::end(inShape2), std::begin(inBcast2), [](std::size_t sz) { return sz == 1; }); - if (is_fully_aligned(output, 4) && is_fully_aligned(x, 4) && is_fully_aligned(y, 4)) { - launch_vectorized_eltwise_op(stream, output, x, y, params); - } else if (is_fully_aligned(output, 2) && is_fully_aligned(x, 2) && is_fully_aligned(y, 2)) { - launch_vectorized_eltwise_op(stream, output, x, y, params); - } else { - launch_vectorized_eltwise_op(stream, output, x, y, params); + CV_Assert(1 <= rank && rank <= CSL_MAX_TENSOR_RANK); + eltwise_op_bcast_dispatcher(rank, stream, output, outStride, x, inStride1, inBcast1, y, inStride2, inBcast2, params); } } template -void eltwise_max_2(const Stream& stream, Span output, View x, View y) { +void eltwise_max_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y) { eltwise_op>(stream, output, x, y); } template -void eltwise_min_2(const Stream& stream, Span output, View x, View y) { +void eltwise_min_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y) { eltwise_op>(stream, output, x, y); } template -void eltwise_sum_2(const Stream& stream, Span output, View x, View y) { +void eltwise_sum_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y) { eltwise_op>(stream, output, x, y); } template -void eltwise_sum_coeff_2(const Stream& stream, Span output, T coeff_x, View x, T coeff_y, View y) { +void eltwise_sum_coeff_2(const Stream& stream, TensorSpan output, T coeff_x, TensorView x, T coeff_y, TensorView y) { eltwise_op>(stream, output, x, y, {coeff_x, coeff_y}); } template -void eltwise_prod_2(const Stream& stream, Span output, View x, View y) { +void eltwise_prod_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y) { eltwise_op>(stream, output, x, y); } template -void eltwise_div_2(const Stream& stream, Span output, View x, View y) { +void eltwise_div_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y) { eltwise_op>(stream, output, x, y); } #if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) - template void eltwise_div_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); - template void eltwise_prod_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); - template void eltwise_sum_coeff_2(const Stream&, Span<__half>, __half, View<__half>, __half, View<__half>); - template void eltwise_sum_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); - template void eltwise_max_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); - template void eltwise_min_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); + template void eltwise_div_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y); + template void eltwise_prod_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y); + template void eltwise_sum_coeff_2(const Stream&, TensorSpan<__half>, __half, TensorView<__half>, __half, TensorView<__half>); + template void eltwise_sum_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y); + template void eltwise_max_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y); + template void eltwise_min_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y); #endif - template void eltwise_div_2(const Stream& stream, Span output, View x, View y); - template void eltwise_prod_2(const Stream& stream, Span output, View x, View y); - template void eltwise_sum_coeff_2(const Stream&, Span, float, View, float, View); - template void eltwise_sum_2(const Stream& stream, Span output, View x, View y); - template void eltwise_max_2(const Stream& stream, Span output, View x, View y); - template void eltwise_min_2(const Stream& stream, Span output, View x, View y); + template void eltwise_div_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y); + template void eltwise_prod_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y); + template void eltwise_sum_coeff_2(const Stream&, TensorSpan, float, TensorView, float, TensorView); + template void eltwise_sum_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y); + template void eltwise_max_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y); + template void eltwise_min_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda/kernel_dispatcher.hpp b/modules/dnn/src/cuda/kernel_dispatcher.hpp index 6eff8344d3a2..b0fc65885073 100644 --- a/modules/dnn/src/cuda/kernel_dispatcher.hpp +++ b/modules/dnn/src/cuda/kernel_dispatcher.hpp @@ -73,4 +73,22 @@ name(selector, std::forward(args)...); \ } +// Same as GENERATE_KERNEL_DISPATCHER but takes two class template parameters T and TP1 instead of just T +#define GENERATE_KERNEL_DISPATCHER_2TP(name,func); \ + template static \ + typename std::enable_if \ + ::type name(int selector, Args&& ...args) { \ + if(selector == start) \ + func(std::forward(args)...); \ + } \ + \ + template static \ + typename std::enable_if \ + ::type name(int selector, Args&& ...args) { \ + if(selector == start) \ + func(std::forward(args)...); \ + else \ + name(selector, std::forward(args)...); \ + } + #endif /* OPENCV_DNN_SRC_CUDA_KERNEL_DISPATCHER_HPP */ diff --git a/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp b/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp index 096ba04e487c..0e44372fee3d 100644 --- a/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp +++ b/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp @@ -6,29 +6,29 @@ #define OPENCV_DNN_SRC_CUDA4DNN_KERNELS_ELTWISE_OPS_HPP #include "../csl/stream.hpp" -#include "../csl/span.hpp" +#include "../csl/tensor.hpp" #include namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { template - void eltwise_max_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + void eltwise_max_2(const csl::Stream& stream, csl::TensorSpan output, csl::TensorView x, csl::TensorView y); template - void eltwise_min_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + void eltwise_min_2(const csl::Stream& stream, csl::TensorSpan output, csl::TensorView x, csl::TensorView y); template - void eltwise_sum_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + void eltwise_sum_2(const csl::Stream& stream, csl::TensorSpan output, csl::TensorView x, csl::TensorView y); template - void eltwise_sum_coeff_2(const csl::Stream& stream, csl::Span output, T coeff_x, csl::View x, T coeff_y, csl::View y); + void eltwise_sum_coeff_2(const csl::Stream& stream, csl::TensorSpan output, T coeff_x, csl::TensorView x, T coeff_y, csl::TensorView y); template - void eltwise_prod_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + void eltwise_prod_2(const csl::Stream& stream, csl::TensorSpan output, csl::TensorView x, csl::TensorView y); template - void eltwise_div_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + void eltwise_div_2(const csl::Stream& stream, csl::TensorSpan output, csl::TensorView x, csl::TensorView y); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 4e38b0374f00..c8fc7721d42f 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -2710,7 +2710,19 @@ struct Net::Impl : public detail::NetImplBase // we create a temporary backend node for eltwise layer to obtain the eltwise configuration cuda4dnn::csl::CSLContext context; // assume that initCUDA and EltwiseOp do not use the context during init const auto node = nextData->layerInstance->initCUDA(&context, nextData->inputBlobsWrappers, nextData->outputBlobsWrappers); - const auto eltwiseNode = node.dynamicCast(); + auto eltwiseNode = node.dynamicCast(); + + // broadcasting not supported in fused ops + auto required_shape = shape(nextData->outputBlobs[0]); + for (int i = 0; i < nextData->inputBlobs.size(); i++) + { + if (shape(*nextData->inputBlobs[i]) != required_shape) + { + eltwiseNode.reset(); + break; + } + } + // CUDA backend uses EltwiseOp when all operands have the same number of channels; otherwise, ShortcutOp is used. // Hence, a successful cast to EltwiseOp implies that the number of channels is same in all operand tensors. if (eltwiseNode.empty() || eltwiseNode->op != cuda4dnn::EltwiseOpType::SUM || !eltwiseNode->coeffs.empty()) From 62414e3073eeec874305e858b297ed8dba5a1b89 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 4 Oct 2021 10:46:32 +0000 Subject: [PATCH 245/376] core(parallel): suppress TSAN warning --- modules/core/src/parallel_impl.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/core/src/parallel_impl.cpp b/modules/core/src/parallel_impl.cpp index 7f30c9d5f3e1..1d9690b04c1f 100644 --- a/modules/core/src/parallel_impl.cpp +++ b/modules/core/src/parallel_impl.cpp @@ -615,6 +615,11 @@ void ThreadPool::run(const Range& range, const ParallelLoopBody& body, double ns { WorkerThread& thread = *(threads[i].get()); if ( +#if defined(__clang__) && defined(__has_feature) +#if __has_feature(thread_sanitizer) + 1 || // Robust workaround to avoid data race warning of `thread.job` +#endif +#endif #if !defined(CV_USE_GLOBAL_WORKERS_COND_VAR) thread.isActive || #endif From d20c9bde7e427962edd824deae7d63f629d8d871 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 4 Oct 2021 09:40:16 +0000 Subject: [PATCH 246/376] core(TLS): force TlsAbstraction initialization before main() --- modules/core/src/system.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index df9e8a0ce76f..e2284b041523 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -1834,6 +1834,12 @@ static void WINAPI opencv_fls_destructor(void* pData) #endif // CV_USE_FLS #endif // _WIN32 +static TlsAbstraction* const g_force_initialization_of_TlsAbstraction +#if defined __GNUC__ + __attribute__((unused)) +#endif + = getTlsAbstraction(); + } // namespace details using namespace details; From d94d469c867b9d5c595b43b67243bfbd6d311d8c Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 4 Oct 2021 16:27:05 +0000 Subject: [PATCH 247/376] valgrind: update suppressions --- platforms/scripts/valgrind.supp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/scripts/valgrind.supp b/platforms/scripts/valgrind.supp index 600c61fad0cd..922982844681 100644 --- a/platforms/scripts/valgrind.supp +++ b/platforms/scripts/valgrind.supp @@ -51,7 +51,7 @@ OpenCV-TLS-getTlsStorage Memcheck:Leak ... - fun:_ZN2cvL13getTlsStorageEv + fun:_ZN2cv*L13getTlsStorageEv } { @@ -159,7 +159,7 @@ OpenCV-DNN-getLayerFactoryMutex Memcheck:Leak ... - fun:_ZN2cv3dnn*L20getLayerFactoryMutexEv + fun:_ZN2cv3dnn*20getLayerFactoryMutexEv } { From 9085b933d81ffd12fbdabffe1ffc69a33580e631 Mon Sep 17 00:00:00 2001 From: Zihao Mu Date: Tue, 5 Oct 2021 00:37:38 +0800 Subject: [PATCH 248/376] Merge pull request #20702 from zihaomu:tf_expand_dim_layer Add ExpandDims layer of tf_importer.cpp * Add ExpandDims to tf_importer. * add -1 expand test case. * Support different dimensions of input. * Compatible with 5-dimensional NDHWC data * Code align * support 3-dim input. * 3-dim bug fixed. * fixing error of code format. --- modules/dnn/src/tensorflow/tf_importer.cpp | 134 ++++++++++++++++++++- modules/dnn/test/test_tf_importer.cpp | 13 ++ 2 files changed, 144 insertions(+), 3 deletions(-) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 813407dbd84a..9a9cd4692f08 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -555,7 +555,7 @@ class TFImporter std::map layer_id; private: - void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId); + void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId, int orderSize = 4); void setPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer, std::string& inputName, float value = 0.); friend class TFLayerHandler; @@ -595,6 +595,7 @@ class TFImporter void parseClipByValue (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseLeakyRelu (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseActivation (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseExpandDims (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseCustomLayer (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); }; @@ -672,6 +673,7 @@ const TFImporter::DispatchMap TFImporter::buildDispatchMap() dispatch["LeakyRelu"] = &TFImporter::parseLeakyRelu; dispatch["Abs"] = dispatch["Tanh"] = dispatch["Sigmoid"] = dispatch["Relu"] = dispatch["Elu"] = dispatch["Exp"] = dispatch["Identity"] = dispatch["Relu6"] = &TFImporter::parseActivation; + dispatch["ExpandDims"] = &TFImporter::parseExpandDims; return dispatch; } @@ -1113,6 +1115,123 @@ void TFImporter::parseReshape(tensorflow::GraphDef& net, const tensorflow::NodeD } } +void TFImporter::parseExpandDims(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_Assert(!netInputShapes.empty()); + + CV_CheckGT(num_inputs, 0, ""); + Pin inpId = parsePin(layer.input(0)); + DataLayout inpLayout = getDataLayout(layer.input(0), data_layouts); + + // Get input shape + std::vector inShape_, outShape_; + int inpIdindex = layer_id.find(inpId.name)->second; + + dstNet.getLayerShapes(netInputShapes, inpIdindex, inShape_, outShape_); + MatShape inpShape = outShape_[0]; + std::vector outShape = inpShape; + + int outShapeSize = outShape.size(); + + CV_Assert(inpShape.size() >= 1); + // 2nd blob is dims tensor + int axis = getConstBlob(layer, value_id, 1).int_val().Get(0); + + // Convert negative numbers to positive numbers, axis can be in range [-(D+1), D]. + if(axis < 0) + { + axis = inpShape.size() + axis + 1; + } + + CV_Assert(0 <= axis && axis <= inpShape.size()); + + // After ExpendDims, 3-dim data will become 4-dim data, and OpenCV retains 4-dim data as NCHW data layout. + // Convert OpenCV's NHC to NCH first. + if(outShapeSize == 3) + { + // If axis equal to outShapeSize, that mean we expand in Channel dimmension, and do not add permuteLayer. + if(axis != outShapeSize) + { + int order[] = {0, 2, 1}; // From OpenCV's NHC to NCH. + addPermuteLayer(order, name + "/nch", inpId, 3); + + std::swap(outShape[1], outShape[2]); + } + axis = (axis != 0)?(axis % outShapeSize + 1):2; + } + + if(inpShape.size() == 4) + { + if(axis == inpShape.size()) + { + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + addPermuteLayer(order, name + "/nhwc", inpId); + + // Convert shape From OpenCV's NCHW to NHWC. + if(inpLayout == DATA_LAYOUT_NHWC) + { + std::swap(outShape[1], outShape[2]); + std::swap(outShape[2], outShape[3]); + } + } + if(inpLayout == DATA_LAYOUT_NHWC || inpLayout == DATA_LAYOUT_NCHW) + { + // toNCHW + axis = (axis != 0)?(axis % outShapeSize + 1):0; + } + } + + // After ExpendDims, 5-dim data will become 6-dim data, and OpenCV retains 6-dim data as original data layout. + // Convert OpenCV's NCDHW to NDHWC first. + if (inpShape.size() == 5 && (inpLayout == DATA_LAYOUT_NDHWC || inpLayout == DATA_LAYOUT_UNKNOWN)) + { + int order[] = {0, 2, 3, 4, 1}; // From OpenCV's NCDHW to NDHWC. + addPermuteLayer(order, name + "/ndhwc", inpId, 5); + + // Convert shape From OpenCV's NCDHW to NDHWC. + if(inpLayout == DATA_LAYOUT_NDHWC) + { + std::swap(outShape[1], outShape[2]); + std::swap(outShape[2], outShape[3]); + std::swap(outShape[3], outShape[4]); + } + } + + outShape.insert(outShape.begin() + axis, 1); + outShapeSize += 1; + + // From OpenCV's NCDHW to NDHWC. + if((inpLayout != DATA_LAYOUT_NHWC && inpLayout != DATA_LAYOUT_NCHW) && outShapeSize == 5) + { + for(int i = 1; i < outShapeSize - 1; i++) + { + std::swap(outShape[outShapeSize - i - 1], outShape[outShapeSize - i]); + } + } + + layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + int id = dstNet.addLayer(name, "Reshape", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, inpId, id, 0); + + if(outShapeSize == 5) + { + data_layouts[name] = DATA_LAYOUT_NDHWC; + } + else if(outShapeSize == 4) + { + data_layouts[name] = DATA_LAYOUT_NCHW; + } + else + { + data_layouts[name] = inpLayout; + } +} + // "Flatten" "Squeeze" void TFImporter::parseFlatten(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { @@ -1419,6 +1538,15 @@ void TFImporter::parsePlaceholder(tensorflow::GraphDef& net, const tensorflow::N if (dims[0] == -1) // It's OK to have undetermined batch size dims[0] = 1; } + + if (dims.size() == 5 && predictedLayout == DATA_LAYOUT_NDHWC) + { + std::swap(dims[3], dims[4]); // NDHWC->NDHCW + std::swap(dims[2], dims[3]); // NDHCW->NDCHW + std::swap(dims[1], dims[2]); // NDCHW->NCDHW + if (dims[0] == -1) // It's OK to have undetermined batch size + dims[0] = 1; + } bool hasNeg = false; for (int i = 0; i < dims.size() && !hasNeg; ++i) { @@ -2882,10 +3010,10 @@ void TFImporter::populateNet() CV_LOG_DEBUG(NULL, (DNN_DIAGNOSTICS_RUN? "DNN/TF: diagnostic run completed!" : "DNN/TF: import completed!")); } -void TFImporter::addPermuteLayer(const int* order, const std::string& permName, Pin& inpId) +void TFImporter::addPermuteLayer(const int* order, const std::string& permName, Pin& inpId, int orderSize) { LayerParams permLP; - permLP.set("order", DictValue::arrayInt(order, 4)); + permLP.set("order", DictValue::arrayInt(order, orderSize)); CV_Assert(layer_id.find(permName) == layer_id.end()); int permId = dstNet.addLayer(permName, "Permute", permLP); layer_id[permName] = permId; diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index eec1a628f1af..8e7f96948066 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -593,6 +593,19 @@ TEST_P(Test_TensorFlow_layers, BiasAdd) runTensorFlowNet("bias_add_1"); } +TEST_P(Test_TensorFlow_layers, ExpandDims) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD + && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + runTensorFlowNet("expand_dims_1"); + runTensorFlowNet("expand_dims_2"); +} + // TODO: fix it and add to l2_normalize TEST_P(Test_TensorFlow_layers, l2_normalize_3d) { From 24f43e7ae91feda546f884dbba1ff858e1c1ed25 Mon Sep 17 00:00:00 2001 From: xhawk18 Date: Tue, 5 Oct 2021 00:44:57 +0800 Subject: [PATCH 249/376] Merge pull request #20183 from xhawk18:3.4 * improve compatibility for qt 6. * cmake(highgui): rework Qt dependency support * cmake(highgui): workaround Qt5Config.cmake "components" bug Co-authored-by: Alexander Alekhin --- CMakeLists.txt | 15 ++++-- cmake/OpenCVDetectVTK.cmake | 29 +++-------- cmake/OpenCVFindLibsGUI.cmake | 55 +++++++++++++++------ modules/highgui/CMakeLists.txt | 81 +++++++++++++++++-------------- modules/highgui/src/window_QT.cpp | 52 +++++++++++++++++--- 5 files changed, 146 insertions(+), 86 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 94ef43fcb46f..d9f98bf23ec0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1232,12 +1232,17 @@ status("") status(" GUI: ") if(WITH_QT OR HAVE_QT) - if(HAVE_QT5) - status(" QT:" "YES (ver ${Qt5Core_VERSION_STRING})") - status(" QT OpenGL support:" HAVE_QT_OPENGL THEN "YES (${Qt5OpenGL_LIBRARIES} ${Qt5OpenGL_VERSION_STRING})" ELSE NO) - elseif(HAVE_QT) + if(HAVE_QT) status(" QT:" "YES (ver ${QT_VERSION_MAJOR}.${QT_VERSION_MINOR}.${QT_VERSION_PATCH} ${QT_EDITION})") - status(" QT OpenGL support:" HAVE_QT_OPENGL THEN "YES (${QT_QTOPENGL_LIBRARY})" ELSE NO) + if(HAVE_QT_OPENGL) + if(Qt${QT_VERSION_MAJOR}OpenGL_LIBRARIES) + status(" QT OpenGL support:" HAVE_QT_OPENGL THEN "YES (${Qt${QT_VERSION_MAJOR}OpenGL_LIBRARIES} ${Qt${QT_VERSION_MAJOR}OpenGL_VERSION_STRING})" ELSE NO) + else() + status(" QT OpenGL support:" HAVE_QT_OPENGL THEN "YES (${QT_QTOPENGL_LIBRARY})" ELSE NO) + endif() + else() + status(" QT OpenGL support:" "NO") + endif() else() status(" QT:" "NO") endif() diff --git a/cmake/OpenCVDetectVTK.cmake b/cmake/OpenCVDetectVTK.cmake index 14956886e366..94ab7355922f 100644 --- a/cmake/OpenCVDetectVTK.cmake +++ b/cmake/OpenCVDetectVTK.cmake @@ -26,31 +26,14 @@ if(VTK_VERSION VERSION_LESS "5.8.0") endif() # Different Qt versions can't be linked together -if(HAVE_QT5 AND VTK_VERSION VERSION_LESS "6.0.0") - if(VTK_USE_QT) - message(STATUS "VTK support is disabled. Incompatible combination: OpenCV + Qt5 and VTK ver.${VTK_VERSION} + Qt4") - endif() -endif() - -# Different Qt versions can't be linked together. VTK 6.0.0 doesn't provide a way to get Qt version it was linked with -if(HAVE_QT5 AND VTK_VERSION VERSION_EQUAL "6.0.0" AND NOT DEFINED FORCE_VTK) - message(STATUS "VTK support is disabled. Possible incompatible combination: OpenCV+Qt5, and VTK ver.${VTK_VERSION} with Qt4") - message(STATUS "If it is known that VTK was compiled without Qt4, please define '-DFORCE_VTK=TRUE' flag in CMake") +if((HAVE_QT AND VTK_USE_QT) + AND NOT DEFINED FORCE_VTK # deprecated + AND NOT DEFINED OPENCV_FORCE_VTK +) + message(STATUS "VTK support is disabled. Possible incompatible combination: OpenCV+Qt, and VTK ver.${VTK_VERSION} with Qt") + message(STATUS "If it is known that VTK was compiled without Qt4, please define '-DOPENCV_FORCE_VTK=TRUE' flag in CMake") return() endif() -# Different Qt versions can't be linked together -if(HAVE_QT AND VTK_VERSION VERSION_GREATER "6.0.0" AND NOT ${VTK_QT_VERSION} STREQUAL "") - if(HAVE_QT5 AND ${VTK_QT_VERSION} EQUAL "4") - message(STATUS "VTK support is disabled. Incompatible combination: OpenCV + Qt5 and VTK ver.${VTK_VERSION} + Qt4") - return() - endif() - - if(NOT HAVE_QT5 AND ${VTK_QT_VERSION} EQUAL "5") - message(STATUS "VTK support is disabled. Incompatible combination: OpenCV + Qt4 and VTK ver.${VTK_VERSION} + Qt5") - return() - endif() -endif() - set(HAVE_VTK ON) message(STATUS "Found VTK ${VTK_VERSION} (${VTK_USE_FILE})") diff --git a/cmake/OpenCVFindLibsGUI.cmake b/cmake/OpenCVFindLibsGUI.cmake index 82f7e0ff1230..5e7db6c199fe 100644 --- a/cmake/OpenCVFindLibsGUI.cmake +++ b/cmake/OpenCVFindLibsGUI.cmake @@ -11,25 +11,50 @@ if(WITH_WIN32UI) CMAKE_FLAGS "-DLINK_LIBRARIES:STRING=user32;gdi32") endif() +macro(ocv_find_package_Qt4) + find_package(Qt4 COMPONENTS QtCore QtGui QtTest ${ARGN}) + if(QT4_FOUND) + set(QT_FOUND 1) + ocv_assert(QT_VERSION_MAJOR EQUAL 4) + endif() +endmacro() + +macro(ocv_find_package_Qt OCV_QT_VER) + find_package(Qt${OCV_QT_VER} COMPONENTS Core Gui Widgets Test Concurrent ${ARGN} NO_MODULE) + if(Qt${OCV_QT_VER}_FOUND) + set(QT_FOUND 1) + set(QT_VERSION "${Qt${OCV_QT_VER}_VERSION}") + set(QT_VERSION_MAJOR "${Qt${OCV_QT_VER}_VERSION_MAJOR}") + set(QT_VERSION_MINOR "${Qt${OCV_QT_VER}_VERSION_MINOR}") + set(QT_VERSION_PATCH "${Qt${OCV_QT_VER}_VERSION_PATCH}") + set(QT_VERSION_TWEAK "${Qt${OCV_QT_VER}_VERSION_TWEAK}") + set(QT_VERSION_COUNT "${Qt${OCV_QT_VER}_VERSION_COUNT}") + endif() +endmacro() + # --- QT4 --- -ocv_clear_vars(HAVE_QT HAVE_QT5) if(WITH_QT) - if(NOT WITH_QT EQUAL 4) - find_package(Qt5 COMPONENTS Core Gui Widgets Test Concurrent REQUIRED NO_MODULE) - if(Qt5_FOUND) - set(HAVE_QT5 ON) - set(HAVE_QT ON) - find_package(Qt5 COMPONENTS OpenGL QUIET) - if(Qt5OpenGL_FOUND) - set(QT_QTOPENGL_FOUND ON) - endif() + if(NOT WITH_QT GREATER 0) + # BUG: Qt5Config.cmake script can't handle components properly: find_package(QT NAMES Qt6 Qt5 REQUIRED NO_MODULE COMPONENTS Core Gui Widgets Test Concurrent) + ocv_find_package_Qt(6 QUIET) + if(NOT QT_FOUND) + ocv_find_package_Qt(5 QUIET) + endif() + if(NOT QT_FOUND) + ocv_find_package_Qt4(QUIET) endif() + elseif(WITH_QT EQUAL 4) + ocv_find_package_Qt4(REQUIRED) + else() # WITH_QT= + ocv_find_package_Qt("${WITH_QT}" REQUIRED) endif() - - if(NOT HAVE_QT) - find_package(Qt4 REQUIRED QtCore QtGui QtTest) - if(QT4_FOUND) - set(HAVE_QT TRUE) + if(QT_FOUND) + set(HAVE_QT ON) + if(QT_VERSION_MAJOR GREATER 4) + find_package(Qt${QT_VERSION_MAJOR} COMPONENTS OpenGL QUIET) + if(Qt${QT_VERSION_MAJOR}OpenGL_FOUND) + set(QT_QTOPENGL_FOUND ON) # HAVE_QT_OPENGL is defined below + endif() endif() endif() endif() diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index 296f4cab2bbb..6e6996109936 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -41,46 +41,55 @@ file(GLOB highgui_ext_hdrs # Removing WinRT API headers by default list(REMOVE_ITEM highgui_ext_hdrs "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/highgui_winrt.hpp") -if(HAVE_QT5) - # "Automoc" doesn't work properly with opencv_world build, use QT5_WRAP_CPP() directly - #set(CMAKE_AUTOMOC ON) - - set(CMAKE_INCLUDE_CURRENT_DIR ON) - - QT5_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) - QT5_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) - list(APPEND highgui_srcs - ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.cpp - ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h - ${_MOC_OUTFILES} - ${_RCC_OUTFILES}) - - foreach(dt5_dep Core Gui Widgets Test Concurrent) - add_definitions(${Qt5${dt5_dep}_DEFINITIONS}) - include_directories(${Qt5${dt5_dep}_INCLUDE_DIRS}) - list(APPEND HIGHGUI_LIBRARIES ${Qt5${dt5_dep}_LIBRARIES}) - endforeach() +if(HAVE_QT) + if(QT_VERSION_MAJOR GREATER 4) + # "Automoc" doesn't work properly with opencv_world build, use QT_WRAP_CPP() directly + #set(CMAKE_AUTOMOC ON) + + set(CMAKE_INCLUDE_CURRENT_DIR ON) + + if(QT_VERSION_MAJOR EQUAL 6) + QT6_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) + QT6_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) + elseif(QT_VERSION_MAJOR EQUAL 5) + QT5_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) + QT5_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) + else() + message(FATAL_ERROR "Unsuported QT version: ${QT_VERSION_MAJOR}") + endif() - if(HAVE_QT_OPENGL) - add_definitions(${Qt5OpenGL_DEFINITIONS}) - include_directories(${Qt5OpenGL_INCLUDE_DIRS}) - list(APPEND HIGHGUI_LIBRARIES ${Qt5OpenGL_LIBRARIES}) - endif() + list(APPEND highgui_srcs + ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h + ${_MOC_OUTFILES} + ${_RCC_OUTFILES}) + + set(qt_deps Core Gui Widgets Test Concurrent) + if(HAVE_QT_OPENGL) + list(APPEND qt_deps OpenGL) + endif() -elseif(HAVE_QT) - if (HAVE_QT_OPENGL) - set(QT_USE_QTOPENGL TRUE) - endif() - include(${QT_USE_FILE}) + foreach(dt_dep ${qt_deps}) + add_definitions(${Qt${QT_VERSION_MAJOR}${dt_dep}_DEFINITIONS}) + include_directories(${Qt${QT_VERSION_MAJOR}${dt_dep}_INCLUDE_DIRS}) + list(APPEND HIGHGUI_LIBRARIES ${Qt${QT_VERSION_MAJOR}${dt_dep}_LIBRARIES}) + endforeach() + else() + ocv_assert(QT_VERSION_MAJOR EQUAL 4) + if (HAVE_QT_OPENGL) + set(QT_USE_QTOPENGL TRUE) + endif() + include(${QT_USE_FILE}) - QT4_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) - QT4_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) + QT4_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) + QT4_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) - list(APPEND HIGHGUI_LIBRARIES ${QT_LIBRARIES}) - list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.cpp ${_MOC_OUTFILES} ${_RCC_OUTFILES}) - ocv_check_flag_support(CXX -Wno-missing-declarations _have_flag "") - if(${_have_flag}) - set_source_files_properties(${_RCC_OUTFILES} PROPERTIES COMPILE_FLAGS -Wno-missing-declarations) + list(APPEND HIGHGUI_LIBRARIES ${QT_LIBRARIES}) + list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.cpp ${_MOC_OUTFILES} ${_RCC_OUTFILES}) + ocv_check_flag_support(CXX -Wno-missing-declarations _have_flag "") + if(${_have_flag}) + set_source_files_properties(${_RCC_OUTFILES} PROPERTIES COMPILE_FLAGS -Wno-missing-declarations) + endif() endif() elseif(WINRT) if(NOT WINRT_8_0) diff --git a/modules/highgui/src/window_QT.cpp b/modules/highgui/src/window_QT.cpp index 8dff03117e1b..e7ee43cf5e43 100644 --- a/modules/highgui/src/window_QT.cpp +++ b/modules/highgui/src/window_QT.cpp @@ -64,6 +64,38 @@ #endif +#if QT_VERSION >= QT_VERSION_CHECK(5, 15, 0) +#define Qt_MiddleButton Qt::MiddleButton +inline Qt::Orientation wheelEventOrientation(QWheelEvent *we) { + if (std::abs(we->angleDelta().x()) < std::abs(we->angleDelta().y())) + return Qt::Vertical; + else + return Qt::Horizontal; +} +inline int wheelEventDelta(QWheelEvent *we) { + if(wheelEventOrientation(we) == Qt::Vertical) + return we->angleDelta().y(); + else + return we->angleDelta().x(); +} +inline QPoint wheelEventPos(QWheelEvent *we) { + return we->position().toPoint(); +} +#else +#define Qt_MiddleButton Qt::MidButton +inline Qt::Orientation wheelEventOrientation(QWheelEvent *we) { + return we->orientation(); +} +inline int wheelEventDelta(QWheelEvent *we) { + return we->delta(); +} +inline QPoint wheelEventPos(QWheelEvent *we) { + return we->pos(); +} + +#endif + + //Static and global first static GuiReceiver *guiMainThread = NULL; static int parameterSystemC = 1; @@ -1579,7 +1611,9 @@ CvWinProperties::CvWinProperties(QString name_paraWindow, QObject* /*parent*/) myLayout->setObjectName(QString::fromUtf8("boxLayout")); myLayout->setContentsMargins(0, 0, 0, 0); myLayout->setSpacing(0); +#if QT_VERSION < QT_VERSION_CHECK(5, 13, 0) myLayout->setMargin(0); +#endif myLayout->setSizeConstraint(QLayout::SetFixedSize); setLayout(myLayout); @@ -1957,7 +1991,9 @@ void CvWindow::createBarLayout() myBarLayout->setObjectName(QString::fromUtf8("barLayout")); myBarLayout->setContentsMargins(0, 0, 0, 0); myBarLayout->setSpacing(0); +#if QT_VERSION < QT_VERSION_CHECK(5, 13, 0) myBarLayout->setMargin(0); +#endif } @@ -1967,7 +2003,9 @@ void CvWindow::createGlobalLayout() myGlobalLayout->setObjectName(QString::fromUtf8("boxLayout")); myGlobalLayout->setContentsMargins(0, 0, 0, 0); myGlobalLayout->setSpacing(0); +#if QT_VERSION < QT_VERSION_CHECK(5, 13, 0) myGlobalLayout->setMargin(0); +#endif setMinimumSize(1, 1); if (param_flags == CV_WINDOW_AUTOSIZE) @@ -2205,7 +2243,7 @@ void CvWindow::icvLoadControlPanel() } if (t->type == type_CvButtonbar) { - int subsize = settings.beginReadArray(QString("buttonbar")+i); + int subsize = settings.beginReadArray(QString("buttonbar%1").arg(i)); if ( subsize == ((CvButtonbar*)t)->layout()->count() ) icvLoadButtonbar((CvButtonbar*)t,&settings); @@ -2236,7 +2274,7 @@ void CvWindow::icvSaveControlPanel() } if (t->type == type_CvButtonbar) { - settings.beginWriteArray(QString("buttonbar")+i); + settings.beginWriteArray(QString("buttonbar%1").arg(i)); icvSaveButtonbar((CvButtonbar*)t,&settings); settings.endArray(); } @@ -2396,14 +2434,14 @@ void OCVViewPort::icvmouseHandler(QMouseEvent* evnt, type_mouse_event category, flags |= CV_EVENT_FLAG_LBUTTON; if(buttons & Qt::RightButton) flags |= CV_EVENT_FLAG_RBUTTON; - if(buttons & Qt::MidButton) + if(buttons & Qt_MiddleButton) flags |= CV_EVENT_FLAG_MBUTTON; if (cv_event == -1) { if (category == mouse_wheel) { QWheelEvent *we = (QWheelEvent *) evnt; - cv_event = ((we->orientation() == Qt::Vertical) ? CV_EVENT_MOUSEWHEEL : CV_EVENT_MOUSEHWHEEL); - flags |= (we->delta() & 0xffff)<<16; + cv_event = ((wheelEventOrientation(we) == Qt::Vertical) ? CV_EVENT_MOUSEWHEEL : CV_EVENT_MOUSEHWHEEL); + flags |= (wheelEventDelta(we) & 0xffff)<<16; return; } switch(evnt->button()) @@ -2416,7 +2454,7 @@ void OCVViewPort::icvmouseHandler(QMouseEvent* evnt, type_mouse_event category, cv_event = tableMouseButtons[category][1]; flags |= CV_EVENT_FLAG_RBUTTON; break; - case Qt::MidButton: + case Qt_MiddleButton: cv_event = tableMouseButtons[category][2]; flags |= CV_EVENT_FLAG_MBUTTON; break; @@ -2772,7 +2810,7 @@ void DefaultViewPort::wheelEvent(QWheelEvent* evnt) { icvmouseEvent((QMouseEvent *)evnt, mouse_wheel); - scaleView(evnt->delta() / 240.0, evnt->pos()); + scaleView(wheelEventDelta(evnt) / 240.0, wheelEventPos(evnt)); viewport()->update(); QWidget::wheelEvent(evnt); From f11f2bfb56e4bbeb6a05ec4112adc375ea8fc515 Mon Sep 17 00:00:00 2001 From: keroiber <39693294+keroiber@users.noreply.github.com> Date: Mon, 4 Oct 2021 18:51:49 +0200 Subject: [PATCH 250/376] Merge pull request #20743 from keroiber:prefix_js_function_bindings_with_namespace * Prefix global javascript functions with sub-namespaces * js: handle 'namespace_prefix_override', update filtering - avoid functions override with same name but different namespace Co-authored-by: Alexander Alekhin --- modules/js/generator/CMakeLists.txt | 1 + modules/js/generator/embindgen.py | 51 +++++++++++++++++++++++------ platforms/js/opencv_js.config.py | 24 ++++++++++++-- 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/modules/js/generator/CMakeLists.txt b/modules/js/generator/CMakeLists.txt index 7a534296511f..3d66df154f7f 100644 --- a/modules/js/generator/CMakeLists.txt +++ b/modules/js/generator/CMakeLists.txt @@ -60,6 +60,7 @@ add_custom_command( ${JS_SOURCE_DIR}/src/core_bindings.cpp ${CMAKE_CURRENT_SOURCE_DIR}/embindgen.py ${CMAKE_CURRENT_SOURCE_DIR}/templates.py + "${OPENCV_JS_WHITELIST_FILE}" ${scripts_hdr_parser} #(not needed - generated by CMake) ${CMAKE_CURRENT_BINARY_DIR}/headers.txt ${opencv_hdrs} diff --git a/modules/js/generator/embindgen.py b/modules/js/generator/embindgen.py index dc7a001df1f0..7eb20c98615c 100644 --- a/modules/js/generator/embindgen.py +++ b/modules/js/generator/embindgen.py @@ -104,6 +104,9 @@ def makeWhiteList(module_list): return wl white_list = None +namespace_prefix_override = { + 'dnn' : '' +} # Features to be exported export_enums = False @@ -271,6 +274,8 @@ def __init__(self, class_name, name, decl, is_constructor, is_class_method, is_c class FuncInfo(object): def __init__(self, class_name, name, cname, namespace, isconstructor): + self.name_id = '_'.join([namespace] + ([class_name] if class_name else []) + [name]) # unique id for dict key + self.class_name = class_name self.name = name self.cname = cname @@ -295,9 +300,9 @@ def __init__(self): self.bindings = [] self.wrapper_funcs = [] - self.classes = {} + self.classes = {} # FIXIT 'classes' should belong to 'namespaces' self.namespaces = {} - self.enums = {} + self.enums = {} # FIXIT 'enums' should belong to 'namespaces' self.parser = hdr_parser.CppHeaderParser() self.class_idx = 0 @@ -419,7 +424,8 @@ def add_func(self, decl): else: func_map = self.namespaces.setdefault(namespace, Namespace()).funcs - func = func_map.setdefault(name, FuncInfo(class_name, name, cpp_name, namespace, is_constructor)) + fi = FuncInfo(class_name, name, cpp_name, namespace, is_constructor) + func = func_map.setdefault(fi.name_id, fi) variant = FuncVariant(class_name, name, decl, is_constructor, is_class_method, is_const_method, is_virtual_method, is_pure_virtual_method, ref_return, const_return) @@ -430,7 +436,7 @@ def save(self, path, name, buf): f.write(buf.getvalue()) f.close() - def gen_function_binding_with_wrapper(self, func, class_info): + def gen_function_binding_with_wrapper(self, func, ns_name, class_info): binding_text = None wrapper_func_text = None @@ -488,8 +494,23 @@ def gen_function_binding_with_wrapper(self, func, class_info): # Wrapper function - wrap_func_name = (func.class_name+"_" if class_info != None else "") + func.name.split("::")[-1] + "_wrapper" - js_func_name = func.name + if ns_name != None and ns_name != "cv": + ns_parts = ns_name.split(".") + if ns_parts[0] == "cv": + ns_parts = ns_parts[1:] + ns_part = "_".join(ns_parts) + "_" + ns_id = '_'.join(ns_parts) + ns_prefix = namespace_prefix_override.get(ns_id, ns_id) + if ns_prefix: + ns_prefix = ns_prefix + '_' + else: + ns_prefix = '' + if class_info == None: + js_func_name = ns_prefix + func.name + wrap_func_name = js_func_name + "_wrapper" + else: + wrap_func_name = ns_prefix + func.class_name + "_" + func.name + "_wrapper" + js_func_name = func.name # TODO: Name functions based wrap directives or based on arguments list if index > 0: @@ -740,12 +761,22 @@ def gen(self, dst_file, src_files, core_bindings): # step 2: generate bindings # Global functions for ns_name, ns in sorted(self.namespaces.items()): - if ns_name.split('.')[0] != 'cv': + ns_parts = ns_name.split('.') + if ns_parts[0] != 'cv': + print('Ignore namespace: {}'.format(ns_name)) continue - for name, func in sorted(ns.funcs.items()): + else: + ns_parts = ns_parts[1:] + ns_id = '_'.join(ns_parts) + ns_prefix = namespace_prefix_override.get(ns_id, ns_id) + for name_id, func in sorted(ns.funcs.items()): + name = func.name + if ns_prefix: + name = ns_prefix + '_' + name if name in ignore_list: continue if not name in white_list['']: + #print('Not in whitelist: "{}" from ns={}'.format(name, ns_name)) continue ext_cnst = False @@ -769,7 +800,7 @@ def gen(self, dst_file, src_files, core_bindings): continue if with_wrapped_functions: - binding, wrapper = self.gen_function_binding_with_wrapper(func, class_info=None) + binding, wrapper = self.gen_function_binding_with_wrapper(func, ns_name, class_info=None) self.bindings += binding self.wrapper_funcs += wrapper else: @@ -802,7 +833,7 @@ def gen(self, dst_file, src_files, core_bindings): class_bindings.append(constructor_template.substitute(signature=', '.join(args))) else: if with_wrapped_functions and (len(method.variants) > 1 or len(method.variants[0].args)>0 or "String" in method.variants[0].rettype): - binding, wrapper = self.gen_function_binding_with_wrapper(method, class_info=class_info) + binding, wrapper = self.gen_function_binding_with_wrapper(method, None, class_info=class_info) self.wrapper_funcs = self.wrapper_funcs + wrapper class_bindings = class_bindings + binding else: diff --git a/platforms/js/opencv_js.config.py b/platforms/js/opencv_js.config.py index 3e09805cd33c..05d2883b78b6 100644 --- a/platforms/js/opencv_js.config.py +++ b/platforms/js/opencv_js.config.py @@ -55,8 +55,26 @@ 'BFMatcher': ['isMaskSupported', 'create'], '': ['drawKeypoints', 'drawMatches', 'drawMatchesKnn']} -calib3d = {'': ['findHomography', 'calibrateCameraExtended', 'drawFrameAxes', 'estimateAffine2D', \ - 'getDefaultNewCameraMatrix', 'initUndistortRectifyMap', 'Rodrigues', \ - 'solvePnP', 'solvePnPRansac', 'solvePnPRefineLM']} +calib3d = { + '': [ + 'findHomography', + 'calibrateCameraExtended', + 'drawFrameAxes', + 'estimateAffine2D', + 'getDefaultNewCameraMatrix', + 'initUndistortRectifyMap', + 'Rodrigues', + 'solvePnP', + 'solvePnPRansac', + 'solvePnPRefineLM', + 'projectPoints', + + # cv::fisheye namespace + 'fisheye_initUndistortRectifyMap', + 'fisheye_projectPoints', + ], +} white_list = makeWhiteList([core, imgproc, objdetect, video, dnn, features2d, calib3d]) + +# namespace_prefix_override['dnn'] = '' # compatibility stuff (enabled by default) From cce78cc5e21d9e72d04504864dfb06afdebe56cb Mon Sep 17 00:00:00 2001 From: Jebastin Nadar Date: Mon, 4 Oct 2021 23:37:38 +0530 Subject: [PATCH 251/376] Merge pull request #20535 from SamFC10:onnx-q dnn : int8 quantized layers support in onnx importer * added quantized layers support in onnx importer * added more cases in eltwise node, some more checks * added tests for quantized nodes * relax thresholds for failed tests, address review comments * refactoring based on review comments * added support for unsupported cases and pre-quantized resnet50 test * relax thresholds due to int8 resize layer --- .../dnn/include/opencv2/dnn/all_layers.hpp | 7 + modules/dnn/src/dnn.cpp | 5 +- modules/dnn/src/init.cpp | 2 + ...ntize_layer.cpp => quantization_utils.cpp} | 53 ++ modules/dnn/src/layers/resize_layer.cpp | 154 ++++-- .../dnn/src/onnx/onnx_graph_simplifier.cpp | 21 +- modules/dnn/src/onnx/onnx_importer.cpp | 486 +++++++++++++++++- modules/dnn/test/test_int8_layers.cpp | 8 +- modules/dnn/test/test_onnx_importer.cpp | 111 ++++ 9 files changed, 794 insertions(+), 53 deletions(-) rename modules/dnn/src/int8layers/{quantize_dequantize_layer.cpp => quantization_utils.cpp} (73%) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index cfe6595d78c8..9fde7ad4c46d 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -387,6 +387,13 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS RequantizeLayer : public Layer + { + public: + float scale, shift; + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS ConcatLayer : public Layer { public: diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 4e38b0374f00..24daeb249f1f 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -4055,6 +4055,9 @@ int Net::addLayer(const String &name, const String &type, const int &dtype, Laye if (params.get("has_dynamic_shapes", false)) impl->hasDynamicShapes = true; + if (dtype == CV_8S) + impl->netWasQuantized = true; + return id; } @@ -4389,7 +4392,7 @@ Net Net::quantize(InputArrayOfArrays calibData, int inputsDtype, int outputsDtyp // Layers with multiple outputs. Number of outputs is equal to number of inputs if (ld.type == "Blank" || ld.type == "Dropout" || ld.type == "Identity" || ld.type == "Silence" || ld.type == "Flatten" || ld.type == "Padding" || ld.type == "Permute" || ld.type == "Reshape" || - ld.type == "ReLU6" || ld.type == "Reorg" || ld.type == "ShuffleChannel" || + ld.type == "ReLU6" || ld.type == "Reorg" || ld.type == "ShuffleChannel" || ld.type == "Resize" || (ld.type == "ReLU" && !ld.params.get("negative_slope", 0.f)) /* ReLU with negative slope 0 */) { for (int i = 0; i < ld.outputBlobs.size(); i++) diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 9d8a3783a2e9..123cb170b72e 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -144,6 +144,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(Quantize, QuantizeLayer); CV_DNN_REGISTER_LAYER_CLASS(Dequantize, DequantizeLayer); + CV_DNN_REGISTER_LAYER_CLASS(Requantize, RequantizeLayer); CV_DNN_REGISTER_LAYER_CLASS(ConvolutionInt8, ConvolutionLayerInt8); CV_DNN_REGISTER_LAYER_CLASS(InnerProductInt8, InnerProductLayerInt8); CV_DNN_REGISTER_LAYER_CLASS(PoolingInt8, PoolingLayerInt8); @@ -173,6 +174,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(SilenceInt8, BlankLayer); CV_DNN_REGISTER_LAYER_CLASS(ConstInt8, ConstLayer); CV_DNN_REGISTER_LAYER_CLASS(ReshapeInt8, ReshapeLayer); + CV_DNN_REGISTER_LAYER_CLASS(ResizeInt8, ResizeLayer); CV_DNN_REGISTER_LAYER_CLASS(SplitInt8, SplitLayer); CV_DNN_REGISTER_LAYER_CLASS(SliceInt8, SliceLayer); CV_DNN_REGISTER_LAYER_CLASS(CropInt8, CropLayer); diff --git a/modules/dnn/src/int8layers/quantize_dequantize_layer.cpp b/modules/dnn/src/int8layers/quantization_utils.cpp similarity index 73% rename from modules/dnn/src/int8layers/quantize_dequantize_layer.cpp rename to modules/dnn/src/int8layers/quantization_utils.cpp index 2ddb76a0e80d..0346f147bab6 100644 --- a/modules/dnn/src/int8layers/quantize_dequantize_layer.cpp +++ b/modules/dnn/src/int8layers/quantization_utils.cpp @@ -10,6 +10,7 @@ namespace cv namespace dnn { +// Quantize FP32/FP16 Inputs to INT8 class QuantizeLayerImpl CV_FINAL : public QuantizeLayer { public: @@ -77,6 +78,7 @@ class QuantizeLayerImpl CV_FINAL : public QuantizeLayer } }; +// Dequantize INT8 Inputs to FP32/FP16 class DequantizeLayerImpl CV_FINAL : public DequantizeLayer { public: @@ -143,6 +145,52 @@ class DequantizeLayerImpl CV_FINAL : public DequantizeLayer } }; +// Rescale/Requantize INT8 Inputs from (scale1, zeropoint1) to (scale2, zeropoint2) +class RequantizeLayerImpl CV_FINAL : public RequantizeLayer +{ +public: + RequantizeLayerImpl(const LayerParams& params) + { + scale = params.get("scale", 1.f); + shift = params.get("shift", 0.f); + setParamsFrom(params); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() == 1); + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + inputs[0].convertTo(outputs[0], CV_8S, scale, shift); + } +}; + Ptr QuantizeLayer::create(const LayerParams& params) { return Ptr(new QuantizeLayerImpl(params)); @@ -153,5 +201,10 @@ Ptr DequantizeLayer::create(const LayerParams& params) return Ptr(new DequantizeLayerImpl(params)); } +Ptr RequantizeLayer::create(const LayerParams& params) +{ + return Ptr(new RequantizeLayerImpl(params)); +} + } } diff --git a/modules/dnn/src/layers/resize_layer.cpp b/modules/dnn/src/layers/resize_layer.cpp index f24c823807f2..47ff0719fe8e 100644 --- a/modules/dnn/src/layers/resize_layer.cpp +++ b/modules/dnn/src/layers/resize_layer.cpp @@ -49,6 +49,8 @@ class ResizeLayerImpl : public ResizeLayer alignCorners = params.get("align_corners", false); halfPixelCenters = params.get("half_pixel_centers", false); + if (interpolation == "opencv_linear") + halfPixelCenters = true; } bool getMemoryShapes(const std::vector &inputs, @@ -131,8 +133,11 @@ class ResizeLayerImpl : public ResizeLayer Mat& inp = inputs[0]; Mat& out = outputs[0]; - if ((interpolation == "nearest" && !alignCorners && !halfPixelCenters) || interpolation == "opencv_linear" || (interpolation == "bilinear" && halfPixelCenters)) + int depth = inp.depth(); + if ((interpolation == "nearest" && !alignCorners && !halfPixelCenters) || (interpolation == "opencv_linear" && depth != CV_8S) || + (interpolation == "bilinear" && halfPixelCenters && depth != CV_8S)) { + // INTER_LINEAR Resize mode does not support INT8 inputs InterpolationFlags mode = interpolation == "nearest" ? INTER_NEAREST : INTER_LINEAR; for (size_t n = 0; n < inputs[0].size[0]; ++n) { @@ -164,34 +169,66 @@ class ResizeLayerImpl : public ResizeLayer widthOffset = 0.5f * scaleWidth; } - for (int y = 0; y < outHeight; ++y) + if (depth == CV_8S) { - float input_y = y * scaleHeight + heightOffset; - int y0 = halfPixelCenters ? std::floor(input_y) : lroundf(input_y); - y0 = std::min(y0, inpHeight - 1); + for (int y = 0; y < outHeight; ++y) + { + float input_y = y * scaleHeight + heightOffset; + int y0 = halfPixelCenters ? std::floor(input_y) : lroundf(input_y); + y0 = std::min(y0, inpHeight - 1); + + const int8_t* inpData_row = inpPlanes.ptr(y0); + + for (int x = 0; x < outWidth; ++x) + { + float input_x = x * scaleWidth + widthOffset; + int x0 = halfPixelCenters ? std::floor(input_x) : lroundf(input_x); + x0 = std::min(x0, inpWidth - 1); + + int8_t* outData = outPlanes.ptr(y, x); + const int8_t* inpData_row_c = inpData_row; - const float* inpData_row = inpPlanes.ptr(y0); + for (int c = 0; c < numPlanes; ++c) + { + *outData = inpData_row_c[x0]; - for (int x = 0; x < outWidth; ++x) + inpData_row_c += inpSpatialSize; + outData += outSpatialSize; + } + } + } + } + else + { + for (int y = 0; y < outHeight; ++y) { - float input_x = x * scaleWidth + widthOffset; - int x0 = halfPixelCenters ? std::floor(input_x) : lroundf(input_x); - x0 = std::min(x0, inpWidth - 1); + float input_y = y * scaleHeight + heightOffset; + int y0 = halfPixelCenters ? std::floor(input_y) : lroundf(input_y); + y0 = std::min(y0, inpHeight - 1); - float* outData = outPlanes.ptr(y, x); - const float* inpData_row_c = inpData_row; + const float* inpData_row = inpPlanes.ptr(y0); - for (int c = 0; c < numPlanes; ++c) + for (int x = 0; x < outWidth; ++x) { - *outData = inpData_row_c[x0]; + float input_x = x * scaleWidth + widthOffset; + int x0 = halfPixelCenters ? std::floor(input_x) : lroundf(input_x); + x0 = std::min(x0, inpWidth - 1); + + float* outData = outPlanes.ptr(y, x); + const float* inpData_row_c = inpData_row; - inpData_row_c += inpSpatialSize; - outData += outSpatialSize; + for (int c = 0; c < numPlanes; ++c) + { + *outData = inpData_row_c[x0]; + + inpData_row_c += inpSpatialSize; + outData += outSpatialSize; + } } } } } - else if (interpolation == "bilinear") + else if (interpolation == "bilinear" || interpolation == "opencv_linear") { const int inpHeight = inp.size[2]; const int inpWidth = inp.size[3]; @@ -202,31 +239,65 @@ class ResizeLayerImpl : public ResizeLayer Mat inpPlanes = inp.reshape(1, numPlanes * inpHeight); Mat outPlanes = out.reshape(1, numPlanes * outHeight); - for (int y = 0; y < outHeight; ++y) + if (depth == CV_8S) + { + for (int y = 0; y < outHeight; ++y) + { + float input_y = halfPixelCenters ? std::max((y + 0.5f) * scaleHeight - 0.5f, 0.0f) : y * scaleHeight; + int y0 = static_cast(input_y); + const int8_t* inpData_row0 = inpPlanes.ptr(y0); + const int8_t* inpData_row1 = inpPlanes.ptr(std::min(y0 + 1, inpHeight - 1)); + for (int x = 0; x < outWidth; ++x) + { + float input_x = halfPixelCenters ? std::max((x + 0.5f) * scaleWidth - 0.5f, 0.0f) : x * scaleWidth; + int x0 = static_cast(input_x); + int x1 = std::min(x0 + 1, inpWidth - 1); + + int8_t* outData = outPlanes.ptr(y, x); + const int8_t* inpData_row0_c = inpData_row0; + const int8_t* inpData_row1_c = inpData_row1; + for (int c = 0; c < numPlanes; ++c) + { + *outData = static_cast(inpData_row0_c[x0] + + (input_y - y0) * (inpData_row1_c[x0] - inpData_row0_c[x0]) + + (input_x - x0) * (inpData_row0_c[x1] - inpData_row0_c[x0] + + (input_y - y0) * (inpData_row1_c[x1] - inpData_row0_c[x1] - inpData_row1_c[x0] + inpData_row0_c[x0]))); + + inpData_row0_c += inpSpatialSize; + inpData_row1_c += inpSpatialSize; + outData += outSpatialSize; + } + } + } + } + else { - float input_y = y * scaleHeight; - int y0 = static_cast(input_y); - const float* inpData_row0 = inpPlanes.ptr(y0); - const float* inpData_row1 = inpPlanes.ptr(std::min(y0 + 1, inpHeight - 1)); - for (int x = 0; x < outWidth; ++x) + for (int y = 0; y < outHeight; ++y) { - float input_x = x * scaleWidth; - int x0 = static_cast(input_x); - int x1 = std::min(x0 + 1, inpWidth - 1); - - float* outData = outPlanes.ptr(y, x); - const float* inpData_row0_c = inpData_row0; - const float* inpData_row1_c = inpData_row1; - for (int c = 0; c < numPlanes; ++c) + float input_y = y * scaleHeight; + int y0 = static_cast(input_y); + const float* inpData_row0 = inpPlanes.ptr(y0); + const float* inpData_row1 = inpPlanes.ptr(std::min(y0 + 1, inpHeight - 1)); + for (int x = 0; x < outWidth; ++x) { - *outData = inpData_row0_c[x0] + - (input_y - y0) * (inpData_row1_c[x0] - inpData_row0_c[x0]) + - (input_x - x0) * (inpData_row0_c[x1] - inpData_row0_c[x0] + - (input_y - y0) * (inpData_row1_c[x1] - inpData_row0_c[x1] - inpData_row1_c[x0] + inpData_row0_c[x0])); - - inpData_row0_c += inpSpatialSize; - inpData_row1_c += inpSpatialSize; - outData += outSpatialSize; + float input_x = x * scaleWidth; + int x0 = static_cast(input_x); + int x1 = std::min(x0 + 1, inpWidth - 1); + + float* outData = outPlanes.ptr(y, x); + const float* inpData_row0_c = inpData_row0; + const float* inpData_row1_c = inpData_row1; + for (int c = 0; c < numPlanes; ++c) + { + *outData = inpData_row0_c[x0] + + (input_y - y0) * (inpData_row1_c[x0] - inpData_row0_c[x0]) + + (input_x - x0) * (inpData_row0_c[x1] - inpData_row0_c[x0] + + (input_y - y0) * (inpData_row1_c[x1] - inpData_row0_c[x1] - inpData_row1_c[x0] + inpData_row0_c[x0])); + + inpData_row0_c += inpSpatialSize; + inpData_row1_c += inpSpatialSize; + outData += outSpatialSize; + } } } } @@ -363,6 +434,11 @@ class ResizeLayerImpl : public ResizeLayer } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } protected: int outWidth, outHeight; diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index 7caa5ed54d0a..c4ebcd1c63ca 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -594,7 +594,8 @@ void simplifySubgraphs(opencv_onnx::GraphProto& net) Mat getMatFromTensor(opencv_onnx::TensorProto& tensor_proto) { if (tensor_proto.raw_data().empty() && tensor_proto.float_data().empty() && - tensor_proto.double_data().empty() && tensor_proto.int64_data().empty()) + tensor_proto.double_data().empty() && tensor_proto.int64_data().empty() && + tensor_proto.int32_data().empty()) return Mat(); opencv_onnx::TensorProto_DataType datatype = tensor_proto.data_type(); @@ -663,6 +664,24 @@ Mat getMatFromTensor(opencv_onnx::TensorProto& tensor_proto) convertInt64ToInt32(src, dst, blob.total()); } } + else if (datatype == opencv_onnx::TensorProto_DataType_INT8 || + datatype == opencv_onnx::TensorProto_DataType_UINT8) + { + // TODO : Add support for uint8 weights and acitvations. For now, converting uint8 tensors to int8. + int offset = datatype == opencv_onnx::TensorProto_DataType_INT8 ? 0 : -128; + int depth = datatype == opencv_onnx::TensorProto_DataType_INT8 ? CV_8S : CV_8U; + + if (!tensor_proto.int32_data().empty()) + { + const ::google::protobuf::RepeatedField field = tensor_proto.int32_data(); + Mat(sizes, CV_32SC1, (void*)field.data()).convertTo(blob, CV_8S, 1.0, offset); + } + else + { + char* val = const_cast(tensor_proto.raw_data().c_str()); + Mat(sizes, depth, val).convertTo(blob, CV_8S, 1.0, offset); + } + } else { std::string errorMsg = "Unsupported data type: " + diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 9ef947c64516..7c230f28c8d8 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -63,6 +63,8 @@ class ONNXImporter void addConstant(const std::string& name, const Mat& blob); void addLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void handleQuantizedNode(LayerParams& layerParams, + const opencv_onnx::NodeProto& node_proto); void expandMid(const std::string& prefix, opencv_onnx::NodeProto& node_proto, const std::string& input, size_t n); @@ -142,6 +144,14 @@ class ONNXImporter void parseSoftMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseCumSum (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQuantDequant (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQConv (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQMatMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQEltwise (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQLeakyRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQSigmoid (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQAvgPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseCustomLayer (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); }; @@ -242,7 +252,7 @@ void runLayer(LayerParams& params, const std::vector& inputs, CV_Assert((bool)layer); std::vector inpShapes(inputs.size()); - int ddepth = CV_32F; + int ddepth = params.get("depth", CV_32F); for (size_t i = 0; i < inputs.size(); ++i) { inpShapes[i] = shape(inputs[i]); @@ -458,7 +468,8 @@ Mat ONNXImporter::getBlob(const std::string& input_name) void ONNXImporter::addLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - int id = dstNet.addLayer(layerParams.name, layerParams.type, layerParams); + int depth = layerParams.get("depth", CV_32F); + int id = dstNet.addLayer(layerParams.name, layerParams.type, depth, layerParams); for (int i = 0; i < node_proto.output_size(); ++i) { layer_id.insert(std::make_pair(node_proto.output(i), LayerInfo(id, i))); @@ -525,6 +536,51 @@ void ONNXImporter::addConstant(const std::string& name, const Mat& blob) outShapes.insert(std::make_pair(name, shape(blob))); } +void ONNXImporter::handleQuantizedNode(LayerParams& layerParams, + const opencv_onnx::NodeProto& node_proto) +{ + // Quantized nodes have output names ending with 'quantized' + std::string outName = node_proto.output(0); + int len = outName.length(); + if (len <= 9) + return; + + if (outName.substr(len - 9) == "quantized") + { + outName = outName.substr(0, len - 9); + Mat scale, zeropoint; + + if (constBlobs.find(outName + "scale") != constBlobs.end() && + constBlobs.find(outName + "zero_point") != constBlobs.end()) + { + scale = getBlob(outName + "scale"); + zeropoint = getBlob(outName + "zero_point"); + } + else + { + std::string inpName = node_proto.input(0); + inpName = inpName.substr(0, inpName.length() - 9); + scale = getBlob(inpName + "scale"); + zeropoint = getBlob(inpName + "zero_point"); + + for (int i = 0; i < node_proto.output_size(); i++) + { + std::string out = node_proto.output(i); + out = out.substr(0, out.length() - 9); + addConstant(out + "scale", scale); + addConstant(out + "zero_point", zeropoint); + } + } + + if (scale.total() != 1 || zeropoint.total() != 1) + CV_Error(Error::StsNotImplemented, "Per-channel scales/zeropoints are not supported"); + + layerParams.set("depth", CV_8S); + layerParams.set("scales", DictValue::arrayReal(scale.ptr(), 1)); + layerParams.set("zeropoints", DictValue::arrayInt(zeropoint.ptr(), 1)); + } +} + void ONNXImporter::populateNet() { CV_Assert(model_proto.has_graph()); @@ -623,6 +679,8 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) layerParams.type = layer_type; layerParams.set("has_dynamic_shapes", hasDynamicShapes); + handleQuantizedNode(layerParams, node_proto); + DispatchMap::const_iterator iter = dispatch.find(layer_type); if (iter != dispatch.end()) { @@ -684,7 +742,8 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) void ONNXImporter::parseMaxPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - layerParams.type = "Pooling"; + int depth = layerParams.get("depth", CV_32F); + layerParams.type = (depth == CV_8S) ? "PoolingInt8" : "Pooling"; layerParams.set("pool", "MAX"); layerParams.set("ceil_mode", layerParams.has("pad_mode")); addLayer(layerParams, node_proto); @@ -988,7 +1047,8 @@ void ONNXImporter::parseSplit(LayerParams& layerParams, const opencv_onnx::NodeP { layerParams.set("num_split", node_proto.output_size()); } - layerParams.type = "Slice"; + int depth = layerParams.get("depth", CV_32F); + layerParams.type = (depth == CV_8S) ? "SliceInt8" : "Slice"; addLayer(layerParams, node_proto); } @@ -1743,7 +1803,8 @@ void ONNXImporter::parseConvTranspose(LayerParams& layerParams, const opencv_onn void ONNXImporter::parseTranspose(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - layerParams.type = "Permute"; + int depth = layerParams.get("depth", CV_32F); + layerParams.type = (depth == CV_8S) ? "PermuteInt8" : "Permute"; replaceLayerParam(layerParams, "perm", "order"); CV_Assert(node_proto.input_size() == 1); @@ -1807,6 +1868,8 @@ void ONNXImporter::parseSqueeze(LayerParams& layerParams, const opencv_onnx::Nod addConstant(layerParams.name, out); return; } + int depth = layerParams.get("depth", CV_32F); + layerParams.type += (depth == CV_8S) ? "Int8" : ""; addLayer(layerParams, node_proto); } @@ -1862,12 +1925,14 @@ void ONNXImporter::parseUnsqueeze(LayerParams& layerParams, const opencv_onnx::N if (axes.size() != 1) CV_Error(Error::StsNotImplemented, "Multidimensional unsqueeze"); + int depth = layerParams.get("depth", CV_32F); + MatShape inpShape = outShapes[node_proto.input(0)]; int axis = axes.getIntValue(0); CV_Assert(0 <= axis && axis <= inpShape.size()); std::vector outShape = inpShape; outShape.insert(outShape.begin() + axis, 1); - layerParams.type = "Reshape"; + layerParams.type = (depth == CV_8S) ? "ReshapeInt8" : "Reshape"; layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); if (hasDynamicShapes) { @@ -2004,6 +2069,8 @@ void ONNXImporter::parseExpand(LayerParams& layerParams, const opencv_onnx::Node void ONNXImporter::parseReshape(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape")); + int depth = layerParams.get("depth", CV_32F); + layerParams.type += (depth == CV_8S) ? "Int8" : ""; if (node_proto.input_size() == 2) { Mat blob = getBlob(node_proto, 1); @@ -2038,7 +2105,8 @@ void ONNXImporter::parseReshape(LayerParams& layerParams, const opencv_onnx::Nod void ONNXImporter::parsePad(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - layerParams.type = "Padding"; + int depth = layerParams.get("depth", CV_32F); + layerParams.type = (depth == CV_8S) ? "PaddingInt8" : "Padding"; replaceLayerParam(layerParams, "mode", "type"); if (node_proto.input_size() == 3 || node_proto.input_size() == 2) { @@ -2051,7 +2119,8 @@ void ONNXImporter::parsePad(LayerParams& layerParams, const opencv_onnx::NodePro if (node_proto.input_size() == 3) { Mat value = getBlob(node_proto, 2); - layerParams.set("value", value.ptr()[0]); + float padValue = (depth == CV_8S) ? (float)value.ptr()[0] : value.ptr()[0]; + layerParams.set("value", padValue); } } addLayer(layerParams, node_proto); @@ -2270,6 +2339,9 @@ void ONNXImporter::parseResize(LayerParams& layerParams, const opencv_onnx::Node for (int i = 1; i < node_proto.input_size(); i++) CV_Assert(layer_id.find(node_proto.input(i)) == layer_id.end()); + int depth = layerParams.get("depth", CV_32F); + layerParams.type += (depth == CV_8S) ? "Int8" : ""; + if (layerParams.has("coordinate_transformation_mode")) { String interp_mode = layerParams.get("coordinate_transformation_mode"); @@ -2419,6 +2491,396 @@ void ONNXImporter::parseCustomLayer(LayerParams& layerParams, const opencv_onnx: addLayer(layerParams, node_proto); } +void ONNXImporter::parseQuantDequant(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 3); + layerParams.type = (node_proto.op_type() == "QuantizeLinear") ? "Quantize" : "Dequantize"; + + if (node_proto.op_type() == "DequantizeLinear") + { + Mat scale = getBlob(node_proto, 1); + Mat zeropoint = getBlob(node_proto, 2); + + layerParams.set("scales", DictValue::arrayReal(scale.ptr(), 1)); + layerParams.set("zeropoints", DictValue::arrayInt(zeropoint.ptr(), 1)); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQConv(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int ninputs = node_proto.input_size(); + CV_Assert(ninputs == 8 || ninputs == 9); + + Mat inp_sc = getBlob(node_proto, 1); + Mat inp_zp = getBlob(node_proto, 2); + + Mat weights = getBlob(node_proto, 3); + int outCn = weights.size[0]; + Mat w_scale = getBlob(node_proto, 4); + CV_Assert(w_scale.total() == 1 || w_scale.total() == outCn); + Mat wt_sc = (w_scale.total() == outCn) ? w_scale : Mat(1, outCn, CV_32F, Scalar(w_scale.at(0))); + + Mat out_sc = getBlob(node_proto, 6); + Mat bias = (ninputs == 9) ? getBlob(node_proto, 8) : Mat::zeros(1, outCn, CV_32S); + + Mat weights_2d = weights.reshape(1, outCn); + Mat biasFused(1, outCn, CV_32S); + Mat outputMultiplier(1, outCn, CV_32F); + for (int i = 0; i < outCn; i++) + { + biasFused.at(i) = bias.at(i) - inp_zp.at(0)*(cv::sum(weights_2d.row(i))[0]); + outputMultiplier.at(i) = (inp_sc.at(0) * wt_sc.at(i)) / out_sc.at(0); + } + + layerParams.type = "ConvolutionInt8"; + layerParams.set("num_output", outCn); + layerParams.set("input_zeropoint", inp_zp.at(0)); + layerParams.blobs.push_back(weights); + layerParams.blobs.push_back(biasFused); + layerParams.blobs.push_back(outputMultiplier); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQMatMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int ninputs = node_proto.input_size(); + CV_Assert(ninputs == 8); + + if (constBlobs.find(node_proto.input(3)) == constBlobs.end()) + CV_Error(Error::StsNotImplemented, "Variable weights is not supported"); + + int firstInpDims = outShapes[node_proto.input(0)].size(); + + Mat inp_sc = getBlob(node_proto, 1); + Mat inp_zp = getBlob(node_proto, 2); + + Mat weights = getBlob(node_proto, 3).t(); + int outCn = weights.size[0]; + int secondInpDims = weights.dims; + + Mat w_scale = getBlob(node_proto, 4); + CV_Assert(w_scale.total() == 1 || w_scale.total() == outCn); + Mat wt_sc = (w_scale.total() == outCn) ? w_scale : Mat(1, outCn, CV_32F, Scalar(w_scale.at(0))); + Mat out_sc = getBlob(node_proto, 6); + + Mat bias(1, outCn, CV_32S); + Mat outputMultiplier(1, outCn, CV_32F); + for (int i = 0; i < outCn; i++) + { + bias.at(i) = -inp_zp.at(0)*(cv::sum(weights.row(i))[0]); + outputMultiplier.at(i) = (inp_sc.at(0) * wt_sc.at(i)) / out_sc.at(0); + } + + layerParams.type = "InnerProductInt8"; + layerParams.set("num_output", outCn); + layerParams.set("axis", firstInpDims - secondInpDims + 1); + layerParams.blobs.push_back(weights); + layerParams.blobs.push_back(bias); + layerParams.blobs.push_back(outputMultiplier); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQEltwise(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_Assert(node_proto.input_size() == 8); + std::string op = (node_proto.op_type() == "QLinearAdd") ? "sum" : "prod"; + int constId = -1; + for (int i = 0; i < 4; i += 3) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + constId = i; + } + + Mat inp_0_sc = getBlob(node_proto, 1); + Mat inp_0_zp = getBlob(node_proto, 2); + + Mat inp_1_sc = getBlob(node_proto, 4); + Mat inp_1_zp = getBlob(node_proto, 5); + + // Set 2nd input as the const input + if (constId == 0) + { + cv::swap(inp_0_sc, inp_1_sc); + cv::swap(inp_0_zp, inp_1_zp); + } + + float out_sc = getBlob(node_proto, 6).at(0); + int8_t out_zp = getBlob(node_proto, 7).at(0); + + std::vector inp_scales = {inp_0_sc.at(0), inp_1_sc.at(0)}; + std::vector inp_zps = {inp_0_zp.at(0), inp_1_zp.at(0)}; + + std::vector coeffs; + float offset; + if (op == "sum") + { + coeffs = {inp_scales[0]/out_sc, inp_scales[1]/out_sc}; + offset = out_zp - coeffs[0]*inp_zps[0] - coeffs[1]*inp_zps[1]; + } + else + { + coeffs = {inp_scales[0]/out_sc, inp_scales[1]}; + offset = out_zp; + } + + if (constId != -1) + { + Mat blob = getBlob(node_proto, constId); + if (blob.total() == 1) + { + float val = inp_scales[1] * (blob.at(0) - inp_zps[1]); + float scale = inp_scales[0] / out_sc; + if (op == "prod") + scale *= val; + + float shift = out_zp - scale*inp_zps[0]; + if (op == "sum") + shift += (val/out_sc); + + LayerParams rescaleParams; + rescaleParams.name = layerParams.name; + rescaleParams.type = "Requantize"; + rescaleParams.set("depth", CV_8S); + rescaleParams.set("scale", scale); + rescaleParams.set("shift", shift); + addLayer(rescaleParams, node_proto); + return; + } + else + { + MatShape inpShape = outShapes[node_proto.input(3 - constId)]; + if (blob.dims == 2) + blob = blob.t(); + + if (shape(blob) == inpShape) + { + LayerParams constParams; + constParams.name = layerParams.name + "/const"; + constParams.type = "ConstInt8"; + constParams.set("depth", CV_8S); + constParams.set("scales", DictValue::arrayReal(inp_1_sc.ptr(), 1)); + constParams.set("zeropoints", DictValue::arrayInt(inp_1_zp.ptr(), 1)); + constParams.blobs.push_back(blob); + + int id = dstNet.addLayer(constParams.name, constParams.type, CV_8S, constParams); + layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); + outShapes[constParams.name] = shape(blob); + node_proto.set_input(constId, constParams.name); + + layerParams.type = "EltwiseInt8"; + layerParams.set("operation", op); + layerParams.set("coeff", DictValue::arrayReal(coeffs.data(), coeffs.size())); + layerParams.set("offset", offset); + } + else + { + layerParams.type = "ScaleInt8"; + layerParams.set("bias_term", op == "sum"); + int axis = 1; + for (int i = 0; i < graph_proto.initializer_size(); i++) + { + opencv_onnx::TensorProto tensor_proto = graph_proto.initializer(i); + if (tensor_proto.name() == node_proto.input(constId)) + { + axis = inpShape.size() - tensor_proto.dims_size(); + break; + } + } + layerParams.set("axis", axis); + blob = blob.reshape(1, 1); + Mat blob_dequantized; + blob.convertTo(blob_dequantized, CV_32F, inp_scales[1], -(inp_scales[1] * inp_zps[1])); + layerParams.blobs.push_back(blob_dequantized); + layerParams.set("input_scales", DictValue::arrayReal(inp_scales.data(), inp_scales.size())); + } + } + } + else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(3)]) + { + layerParams.type = "EltwiseInt8"; + layerParams.set("operation", op); + layerParams.set("coeff", DictValue::arrayReal(coeffs.data(), coeffs.size())); + layerParams.set("offset", offset); + } + else + { + layerParams.type = "ScaleInt8"; + layerParams.set("bias_term", op == "sum"); + layerParams.set("input_scales", DictValue::arrayReal(inp_scales.data(), inp_scales.size())); + } + + layerParams.set("input_zeropoints", DictValue::arrayInt(inp_zps.data(), inp_zps.size())); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQLeakyRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 5); + + float slope = layerParams.get("alpha"); + float inp_sc = getBlob(node_proto, 1).at(0); + int8_t inp_zp = getBlob(node_proto, 2).at(0); + float out_sc = getBlob(node_proto, 3).at(0); + int8_t out_zp = getBlob(node_proto, 4).at(0); + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inp_sc*(i - inp_zp); + float y = x >= 0.f ? x : slope*x; + int quantized = out_zp + cvRound(y/out_sc); + table[i+128] = saturate_cast(quantized); + } + + layerParams.type = "ReLUInt8"; + layerParams.blobs.push_back(lookUpTable); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQSigmoid(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 5); + + float inp_sc = getBlob(node_proto, 1).at(0); + int8_t inp_zp = getBlob(node_proto, 2).at(0); + float out_sc = getBlob(node_proto, 3).at(0); + int8_t out_zp = getBlob(node_proto, 4).at(0); + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inp_sc*(i - inp_zp); + float y = 1.f/(1.f + std::exp(-x)); + int quantized = out_zp + cvRound(y/out_sc); + table[i+128] = saturate_cast(quantized); + } + + layerParams.type = "SigmoidInt8"; + layerParams.blobs.push_back(lookUpTable); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQAvgPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 5); + float inp_sc = getBlob(node_proto, 1).at(0); + int8_t inp_zp = getBlob(node_proto, 2).at(0); + float out_sc = getBlob(node_proto, 3).at(0); + + layerParams.type = "PoolingInt8"; + layerParams.set("pool", "ave"); + layerParams.set("global_pooling", node_proto.op_type() == "QLinearGlobalAveragePool"); + layerParams.set("multiplier", inp_sc/out_sc); + layerParams.set("input_zeropoint", inp_zp); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQConcat(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + layerParams.type = "ConcatInt8"; + int num_inputs = node_proto.input_size(); + + float out_scale = getBlob(node_proto, 0).at(0); + int out_zp = getBlob(node_proto, 1).at(0); + + for (int i = 2; i < num_inputs; i += 3) + { + float inp_scale = getBlob(node_proto, i + 1).at(0); + int inp_zp = getBlob(node_proto, i + 2).at(0); + + if (inp_scale != out_scale || inp_zp != out_zp) + { + float scale = inp_scale/out_scale; + float shift = out_zp - scale*inp_zp; + + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + { + Mat blob = getBlob(node_proto, i); + Mat blob_rescaled; + blob.convertTo(blob_rescaled, CV_8S, scale, shift); + constBlobs[node_proto.input(i)] = blob_rescaled; + } + else + { + LayerParams rescaleParams; + rescaleParams.name = node_proto.input(i) + "/rescale"; + rescaleParams.type = "Requantize"; + rescaleParams.set("depth", CV_8S); + rescaleParams.set("scale", scale); + rescaleParams.set("shift", shift); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(i)); + proto.add_output(rescaleParams.name); + addLayer(rescaleParams, proto); + node_proto.set_input(i, rescaleParams.name); + } + } + } + + bool hasVariableInps = false; + for (int i = 2; i < num_inputs; i += 3) + { + if (layer_id.find(node_proto.input(i)) != layer_id.end()) + { + hasVariableInps = true; + break; + } + } + + if (!hasVariableInps) + { + std::vector inputs, concatenated; + MatShape inputShape; + for (size_t i = 2; i < num_inputs; i += 3) + { + Mat blob = getBlob(node_proto, i); + if (blob.size.dims() > inputShape.size()) + { + inputShape = shape(blob); + } + inputs.push_back(blob); + } + + int axis = layerParams.get("axis", 1); + for (size_t i = 0; i < inputs.size(); ++i) + { + MatShape targetShape = inputShape; + targetShape[axis] = shape(inputs[i])[axis]; + CV_CheckEQ(total(targetShape), total(shape(inputs[i])), ""); + inputs[i] = inputs[i].reshape(0, targetShape); + } + runLayer(layerParams, inputs, concatenated); + CV_Assert(concatenated.size() == 1); + addConstant(layerParams.name, concatenated[0]); + return; + } + else + { + for (int i = 2; i < num_inputs; i += 3) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + { + LayerParams constParams; + constParams.name = node_proto.input(i); + constParams.type = "ConstInt8"; + constParams.blobs.push_back(getBlob(node_proto, i)); + constParams.set("depth", CV_8S); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + } + } + } + addLayer(layerParams, node_proto); +} + const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() { DispatchMap dispatch; @@ -2468,6 +2930,14 @@ const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() dispatch["SoftMax"] = dispatch["LogSoftmax"] = &ONNXImporter::parseSoftMax; dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput; dispatch["CumSum"] = &ONNXImporter::parseCumSum; + dispatch["QuantizeLinear"] = dispatch["DequantizeLinear"] = &ONNXImporter::parseQuantDequant; + dispatch["QLinearConv"] = &ONNXImporter::parseQConv; + dispatch["QLinearMatMul"] = &ONNXImporter::parseQMatMul; + dispatch["QLinearAdd"] = dispatch["QLinearMul"] = &ONNXImporter::parseQEltwise; + dispatch["QLinearLeakyRelu"] = &ONNXImporter::parseQLeakyRelu; + dispatch["QLinearSigmoid"] = &ONNXImporter::parseQSigmoid; + dispatch["QLinearAveragePool"] = dispatch["QLinearGlobalAveragePool"] = &ONNXImporter::parseQAvgPool; + dispatch["QLinearConcat"] = &ONNXImporter::parseQConcat; return dispatch; } diff --git a/modules/dnn/test/test_int8_layers.cpp b/modules/dnn/test/test_int8_layers.cpp index 1fcb1d0dba7c..5e6c05c7162f 100644 --- a/modules/dnn/test/test_int8_layers.cpp +++ b/modules/dnn/test/test_int8_layers.cpp @@ -583,7 +583,7 @@ TEST_P(Test_Int8_nets, ResNet50) Mat blob = blobFromImage(inp, 1.0, Size(224, 224), Scalar(), false); Mat ref = blobFromNPY(_tf("resnet50_prob.npy")); - float l1 = 3e-4, lInf = 0.035; + float l1 = 3e-4, lInf = 0.04; testClassificationNet(net, blob, ref, l1, lInf); } @@ -714,7 +714,7 @@ TEST_P(Test_Int8_nets, MobileNet_v1_SSD_PPN) Mat blob = blobFromImage(inp, 1.0, Size(300, 300), Scalar(), true, false); Mat ref = blobFromNPY(_tf("tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy")); - float confThreshold = 0.51, scoreDiff = 0.04, iouDiff = 0.06; + float confThreshold = 0.51, scoreDiff = 0.05, iouDiff = 0.06; testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); } @@ -815,7 +815,7 @@ TEST_P(Test_Int8_nets, FasterRCNN_resnet50) Mat blob = blobFromImage(inp, 1.0, Size(800, 600), Scalar(), true, false); Mat ref = blobFromNPY(_tf("tensorflow/faster_rcnn_resnet50_coco_2018_01_28.detection_out.npy")); - float confThreshold = 0.5, scoreDiff = 0.025, iouDiff = 0.15; + float confThreshold = 0.5, scoreDiff = 0.05, iouDiff = 0.15; testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); } @@ -1127,7 +1127,7 @@ TEST_P(Test_Int8_nets, YOLOv4) std::string config_file = "yolov4.cfg"; std::string weights_file = "yolov4.weights"; - double scoreDiff = 0.1, iouDiff = 0.17; + double scoreDiff = 0.15, iouDiff = 0.2; { SCOPED_TRACE("batch size 1"); testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 4334da2ad63d..5d324b8aac08 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -991,6 +991,112 @@ TEST_P(Test_ONNX_layers, ConvResizePool1d) testONNXModels("conv_resize_pool_1d"); } +TEST_P(Test_ONNX_layers, Quantized_Convolution) +{ + testONNXModels("quantized_conv_uint8_weights", npy, 0.004, 0.02); + testONNXModels("quantized_conv_int8_weights", npy, 0.03, 0.5); + testONNXModels("quantized_conv_per_channel_weights", npy, 0.06, 0.4); +} + +TEST_P(Test_ONNX_layers, Quantized_MatMul) +{ + testONNXModels("quantized_matmul_uint8_weights", npy, 0.005, 0.007); + testONNXModels("quantized_matmul_int8_weights", npy, 0.06, 0.2); + testONNXModels("quantized_matmul_per_channel_weights", npy, 0.06, 0.22); +} + +TEST_P(Test_ONNX_layers, Quantized_MatMul_Variable_Weights) +{ + // Unsupported + EXPECT_THROW( + { + testONNXModels("quantized_matmul_variable_inputs"); + }, cv::Exception); +} + +TEST_P(Test_ONNX_layers, Quantized_Eltwise) +{ + testONNXModels("quantized_eltwise"); +} + +TEST_P(Test_ONNX_layers, Quantized_Eltwise_Scalar) +{ + testONNXModels("quantized_eltwise_scalar"); +} + +TEST_P(Test_ONNX_layers, Quantized_Eltwise_Broadcast) +{ + testONNXModels("quantized_eltwise_broadcast"); +} + +TEST_P(Test_ONNX_layers, Quantized_LeakyReLU) +{ + testONNXModels("quantized_leaky_relu"); +} + +TEST_P(Test_ONNX_layers, Quantized_Sigmoid) +{ + testONNXModels("quantized_sigmoid"); +} + +TEST_P(Test_ONNX_layers, Quantized_MaxPool) +{ + testONNXModels("quantized_maxpool"); +} + +TEST_P(Test_ONNX_layers, Quantized_AvgPool) +{ + testONNXModels("quantized_avgpool"); +} + +TEST_P(Test_ONNX_layers, Quantized_Split) +{ + testONNXModels("quantized_split"); +} + +TEST_P(Test_ONNX_layers, Quantized_Pad) +{ + testONNXModels("quantized_padding"); +} + +TEST_P(Test_ONNX_layers, Quantized_Reshape) +{ + testONNXModels("quantized_reshape"); +} + +TEST_P(Test_ONNX_layers, Quantized_Transpose) +{ + testONNXModels("quantized_transpose"); +} + +TEST_P(Test_ONNX_layers, Quantized_Squeeze) +{ + testONNXModels("quantized_squeeze"); +} + +TEST_P(Test_ONNX_layers, Quantized_Unsqueeze) +{ + testONNXModels("quantized_unsqueeze"); +} + +TEST_P(Test_ONNX_layers, Quantized_Resize) +{ + testONNXModels("quantized_resize_nearest"); + testONNXModels("quantized_resize_bilinear", npy, 2e-4, 0.003); + testONNXModels("quantized_resize_bilinear_align", npy, 3e-4, 0.003); +} + +TEST_P(Test_ONNX_layers, Quantized_Concat) +{ + testONNXModels("quantized_concat"); + testONNXModels("quantized_concat_const_blob"); +} + +TEST_P(Test_ONNX_layers, Quantized_Constant) +{ + testONNXModels("quantized_constant", npy, 0.002, 0.008); +} + INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_ONNX_layers, dnnBackendsAndTargets()); class Test_ONNX_nets : public Test_ONNX_layers @@ -1127,6 +1233,11 @@ TEST_P(Test_ONNX_nets, ResNet50v1) testONNXModels("resnet50v1", pb, default_l1, default_lInf, true, target != DNN_TARGET_MYRIAD); } +TEST_P(Test_ONNX_nets, ResNet50_Int8) +{ + testONNXModels("resnet50_int8", pb, default_l1, default_lInf, true); +} + TEST_P(Test_ONNX_nets, ResNet101_DUC_HDC) { applyTestTag(CV_TEST_TAG_VERYLONG); From 4938765eb3f9808ae4e08a006bb4e88eedc55a86 Mon Sep 17 00:00:00 2001 From: Shivanshu Tyagi Date: Mon, 4 Oct 2021 23:48:02 +0530 Subject: [PATCH 252/376] Merge pull request #20291 from spazewalker:master speech recognition sample * speech recognition sample added.(initial commit) * fixed typos, removed plt * trailing whitespaces removed * masking removed and using opencv for displaying spectrogram * description added * requested changes and add opencl fp16 target * parenthesis and halide removed * workaround 3d matrix issue * handle multi channel audio support for multiple files at once * suggested changes fix whitespaces --- samples/dnn/speech_recognition.py | 506 ++++++++++++++++++++++++++++++ 1 file changed, 506 insertions(+) create mode 100644 samples/dnn/speech_recognition.py diff --git a/samples/dnn/speech_recognition.py b/samples/dnn/speech_recognition.py new file mode 100644 index 000000000000..025607edab3a --- /dev/null +++ b/samples/dnn/speech_recognition.py @@ -0,0 +1,506 @@ +import numpy as np +import cv2 as cv +import argparse +import os +import soundfile as sf # Temporary import to load audio files + +''' + You can download the converted onnx model from https://drive.google.com/drive/folders/1wLtxyao4ItAg8tt4Sb63zt6qXzhcQoR6?usp=sharing + or convert the model yourself. + + You can get the original pre-trained Jasper model from NVIDIA : https://ngc.nvidia.com/catalog/models/nvidia:jasper_pyt_onnx_fp16_amp/files + Download and unzip : `$ wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/jasper_pyt_onnx_fp16_amp/versions/20.10.0/zip -O jasper_pyt_onnx_fp16_amp_20.10.0.zip && unzip -o ./jasper_pyt_onnx_fp16_amp_20.10.0.zip && unzip -o ./jasper_pyt_onnx_fp16_amp.zip` + + you can get the script to convert the model here : https://gist.github.com/spazewalker/507f1529e19aea7e8417f6e935851a01 + + You can convert the model using the following steps: + 1. Import onnx and load the original model + ``` + import onnx + model = onnx.load("./jasper-onnx/1/model.onnx") + ``` + + 3. Change data type of input layer + ``` + inp = model.graph.input[0] + model.graph.input.remove(inp) + inp.type.tensor_type.elem_type = 1 + model.graph.input.insert(0,inp) + ``` + + 4. Change the data type of output layer + ``` + out = model.graph.output[0] + model.graph.output.remove(out) + out.type.tensor_type.elem_type = 1 + model.graph.output.insert(0,out) + ``` + + 5. Change the data type of every initializer and cast it's values from FP16 to FP32 + ``` + for i,init in enumerate(model.graph.initializer): + model.graph.initializer.remove(init) + init.data_type = 1 + init.raw_data = np.frombuffer(init.raw_data, count=np.product(init.dims), dtype=np.float16).astype(np.float32).tobytes() + model.graph.initializer.insert(i,init) + ``` + + 6. Add an additional reshape node to handle the inconsistant input from python and c++ of openCV. + see https://github.com/opencv/opencv/issues/19091 + Make & insert a new node with 'Reshape' operation & required initializer + ``` + tensor = numpy_helper.from_array(np.array([0,64,-1]),name='shape_reshape') + model.graph.initializer.insert(0,tensor) + node = onnx.helper.make_node(op_type='Reshape',inputs=['input__0','shape_reshape'], outputs=['input_reshaped'], name='reshape__0') + model.graph.node.insert(0,node) + model.graph.node[1].input[0] = 'input_reshaped' + ``` + + 7. Finally save the model + ``` + with open('jasper_dynamic_input_float.onnx','wb') as f: + onnx.save_model(model,f) + ``` + + Original Repo : https://github.com/NVIDIA/DeepLearningExamples/tree/master/PyTorch/SpeechRecognition/Jasper + ''' + +class FilterbankFeatures: + def __init__(self, + sample_rate=16000, window_size=0.02, window_stride=0.01, + n_fft=512, preemph=0.97, n_filt=64, lowfreq=0, + highfreq=None, log=True, dither=1e-5): + ''' + Initializes pre-processing class. Default values are the values used by the Jasper + architecture for pre-processing. For more details, refer to the paper here: + https://arxiv.org/abs/1904.03288 + ''' + self.win_length = int(sample_rate * window_size) # frame size + self.hop_length = int(sample_rate * window_stride) # stride + self.n_fft = n_fft or 2 ** np.ceil(np.log2(self.win_length)) + self.log = log + self.dither = dither + self.n_filt = n_filt + self.preemph = preemph + highfreq = highfreq or sample_rate / 2 + self.window_tensor = np.hanning(self.win_length) + + self.filterbanks = self.mel(sample_rate, self.n_fft, n_mels=n_filt, fmin=lowfreq, fmax=highfreq) + self.filterbanks.dtype=np.float32 + self.filterbanks = np.expand_dims(self.filterbanks,0) + + def normalize_batch(self, x, seq_len): + ''' + Normalizes the features. + ''' + x_mean = np.zeros((seq_len.shape[0], x.shape[1]), dtype=x.dtype) + x_std = np.zeros((seq_len.shape[0], x.shape[1]), dtype=x.dtype) + for i in range(x.shape[0]): + x_mean[i, :] = np.mean(x[i, :, :seq_len[i]],axis=1) + x_std[i, :] = np.std(x[i, :, :seq_len[i]],axis=1) + # make sure x_std is not zero + x_std += 1e-10 + return (x - np.expand_dims(x_mean,2)) / np.expand_dims(x_std,2) + + def calculate_features(self, x, seq_len): + ''' + Calculates filterbank features. + args: + x : mono channel audio + seq_len : length of the audio sample + returns: + x : filterbank features + ''' + dtype = x.dtype + + seq_len = np.ceil(seq_len / self.hop_length) + seq_len = np.array(seq_len,dtype=np.int32) + + # dither + if self.dither > 0: + x += self.dither * np.random.randn(*x.shape) + + # do preemphasis + if self.preemph is not None: + x = np.concatenate( + (np.expand_dims(x[0],-1), x[1:] - self.preemph * x[:-1]), axis=0) + + # Short Time Fourier Transform + x = self.stft(x, n_fft=self.n_fft, hop_length=self.hop_length, + win_length=self.win_length, + fft_window=self.window_tensor) + + # get power spectrum + x = (x**2).sum(-1) + + # dot with filterbank energies + x = np.matmul(np.array(self.filterbanks,dtype=x.dtype), x) + + # log features if required + if self.log: + x = np.log(x + 1e-20) + + # normalize if required + x = self.normalize_batch(x, seq_len).astype(dtype) + return x + + # Mel Frequency calculation + def hz_to_mel(self, frequencies): + ''' + Converts frequencies from hz to mel scale. Input can be a number or a vector. + ''' + frequencies = np.asanyarray(frequencies) + + f_min = 0.0 + f_sp = 200.0 / 3 + + mels = (frequencies - f_min) / f_sp + + # Fill in the log-scale part + min_log_hz = 1000.0 # beginning of log region (Hz) + min_log_mel = (min_log_hz - f_min) / f_sp # same (Mels) + logstep = np.log(6.4) / 27.0 # step size for log region + + if frequencies.ndim: + # If we have array data, vectorize + log_t = frequencies >= min_log_hz + mels[log_t] = min_log_mel + np.log(frequencies[log_t] / min_log_hz) / logstep + elif frequencies >= min_log_hz: + # If we have scalar data, directly + mels = min_log_mel + np.log(frequencies / min_log_hz) / logstep + return mels + + def mel_to_hz(self, mels): + ''' + Converts frequencies from mel to hz scale. Input can be a number or a vector. + ''' + mels = np.asanyarray(mels) + + # Fill in the linear scale + f_min = 0.0 + f_sp = 200.0 / 3 + freqs = f_min + f_sp * mels + + # And now the nonlinear scale + min_log_hz = 1000.0 # beginning of log region (Hz) + min_log_mel = (min_log_hz - f_min) / f_sp # same (Mels) + logstep = np.log(6.4) / 27.0 # step size for log region + + if mels.ndim: + # If we have vector data, vectorize + log_t = mels >= min_log_mel + freqs[log_t] = min_log_hz * np.exp(logstep * (mels[log_t] - min_log_mel)) + elif mels >= min_log_mel: + # If we have scalar data, check directly + freqs = min_log_hz * np.exp(logstep * (mels - min_log_mel)) + + return freqs + + def mel_frequencies(self, n_mels=128, fmin=0.0, fmax=11025.0): + ''' + Calculates n mel frequencies between 2 frequencies + args: + n_mels : number of bands + fmin : min frequency + fmax : max frequency + returns: + mels : vector of mel frequencies + ''' + # 'Center freqs' of mel bands - uniformly spaced between limits + min_mel = self.hz_to_mel(fmin) + max_mel = self.hz_to_mel(fmax) + + mels = np.linspace(min_mel, max_mel, n_mels) + + return self.mel_to_hz(mels) + + def mel(self, sr, n_fft, n_mels=128, fmin=0.0, fmax=None, dtype=np.float32): + ''' + Generates mel filterbank + args: + sr : Sampling rate + n_fft : number of FFT components + n_mels : number of Mel bands to generate + fmin : lowest frequency (in Hz) + fmax : highest frequency (in Hz). sr/2.0 if None + dtype : the data type of the output basis. + returns: + mels : Mel transform matrix + ''' + # default Max freq = half of sampling rate + if fmax is None: + fmax = float(sr) / 2 + + # Initialize the weights + n_mels = int(n_mels) + weights = np.zeros((n_mels, int(1 + n_fft // 2)), dtype=dtype) + + # Center freqs of each FFT bin + fftfreqs = np.linspace(0, float(sr) / 2, int(1 + n_fft // 2), endpoint=True) + + # 'Center freqs' of mel bands - uniformly spaced between limits + mel_f = self.mel_frequencies(n_mels + 2, fmin=fmin, fmax=fmax) + + fdiff = np.diff(mel_f) + ramps = np.subtract.outer(mel_f, fftfreqs) + + for i in range(n_mels): + # lower and upper slopes for all bins + lower = -ramps[i] / fdiff[i] + upper = ramps[i + 2] / fdiff[i + 1] + + # .. then intersect them with each other and zero + weights[i] = np.maximum(0, np.minimum(lower, upper)) + + # Using Slaney-style mel which is scaled to be approx constant energy per channel + enorm = 2.0 / (mel_f[2 : n_mels + 2] - mel_f[:n_mels]) + weights *= enorm[:, np.newaxis] + return weights + + # STFT preperation + def pad_window_center(self, data, size, axis=-1, **kwargs): + ''' + Centers the data and pads. + args: + data : Vector to be padded and centered + size : Length to pad data + axis : Axis along which to pad and center the data + kwargs : arguments passed to np.pad + return : centered and padded data + ''' + kwargs.setdefault("mode", "constant") + n = data.shape[axis] + lpad = int((size - n) // 2) + lengths = [(0, 0)] * data.ndim + lengths[axis] = (lpad, int(size - n - lpad)) + if lpad < 0: + raise Exception( + ("Target size ({:d}) must be at least input size ({:d})").format(size, n) + ) + return np.pad(data, lengths, **kwargs) + + def frame(self, x, frame_length, hop_length): + ''' + Slices a data array into (overlapping) frames. + args: + x : array to frame + frame_length : length of frame + hop_length : Number of steps to advance between frames + return : A framed view of `x` + ''' + if x.shape[-1] < frame_length: + raise Exception( + "Input is too short (n={:d})" + " for frame_length={:d}".format(x.shape[-1], frame_length) + ) + x = np.asfortranarray(x) + n_frames = 1 + (x.shape[-1] - frame_length) // hop_length + strides = np.asarray(x.strides) + new_stride = np.prod(strides[strides > 0] // x.itemsize) * x.itemsize + shape = list(x.shape)[:-1] + [frame_length, n_frames] + strides = list(strides) + [hop_length * new_stride] + return np.lib.stride_tricks.as_strided(x, shape=shape, strides=strides) + + def dtype_r2c(self, d, default=np.complex64): + ''' + Find the complex numpy dtype corresponding to a real dtype. + args: + d : The real-valued dtype to convert to complex. + default : The default complex target type, if `d` does not match a known dtype + return : The complex dtype + ''' + mapping = { + np.dtype(np.float32): np.complex64, + np.dtype(np.float64): np.complex128, + } + dt = np.dtype(d) + if dt.kind == "c": + return dt + return np.dtype(mapping.get(dt, default)) + + def stft(self, y, n_fft, hop_length=None, win_length=None, fft_window=None, pad_mode='reflect', return_complex=False): + ''' + Short Time Fourier Transform. The STFT represents a signal in the time-frequency + domain by computing discrete Fourier transforms (DFT) over short overlapping windows. + args: + y : input signal + n_fft : length of the windowed signal after padding with zeros. + hop_length : number of audio samples between adjacent STFT columns. + win_length : Each frame of audio is windowed by window of length win_length and + then padded with zeros to match n_fft + fft_window : a vector or array of length `n_fft` having values computed by a + window function + pad_mode : mode while padding the singnal + return_complex : returns array with complex data type if `True` + return : Matrix of short-term Fourier transform coefficients. + ''' + if win_length is None: + win_length = n_fft + if hop_length is None: + hop_length = int(win_length // 4) + if y.ndim!=1: + raise Exception(f'Invalid input shape. Only Mono Channeled audio supported. Input must have shape (Audio,). Got {y.shape}') + + # Pad the window out to n_fft size + fft_window = self.pad_window_center(fft_window, n_fft) + + # Reshape so that the window can be broadcast + fft_window = fft_window.reshape((-1, 1)) + + # Pad the time series so that frames are centered + y = np.pad(y, int(n_fft // 2), mode=pad_mode) + + # Window the time series. + y_frames = self.frame(y, frame_length=n_fft, hop_length=hop_length) + + # Convert data type to complex + dtype = self.dtype_r2c(y.dtype) + + # Pre-allocate the STFT matrix + stft_matrix = np.empty( (int(1 + n_fft // 2), y_frames.shape[-1]), dtype=dtype, order="F") + + stft_matrix = np.fft.rfft( fft_window * y_frames, axis=0) + return stft_matrix if return_complex==True else np.stack((stft_matrix.real,stft_matrix.imag),axis=-1) + +class Decoder: + ''' + Used for decoding the output of jasper model. + ''' + def __init__(self): + labels=[' ','a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z',"'"] + self.labels_map = {i: label for i,label in enumerate(labels)} + self.blank_id = 28 + + def decode(self,x): + """ + Takes output of Jasper model and performs ctc decoding algorithm to + remove duplicates and special symbol. Returns prediction + """ + x = np.argmax(x,axis=-1) + hypotheses = [] + prediction = x.tolist() + # CTC decoding procedure + decoded_prediction = [] + previous = self.blank_id + for p in prediction: + if (p != previous or previous == self.blank_id) and p != self.blank_id: + decoded_prediction.append(p) + previous = p + hypothesis = ''.join([self.labels_map[c] for c in decoded_prediction]) + hypotheses.append(hypothesis) + return hypotheses + +def predict(features, net, decoder): + ''' + Passes the features through the Jasper model and decodes the output to english transcripts. + args: + features : input features, calculated using FilterbankFeatures class + net : Jasper model dnn.net object + decoder : Decoder object + return : Predicted text + ''' + # This is a workaround https://github.com/opencv/opencv/issues/19091 + # expanding 1 dimentions allows us to pass it to the network + # from python. This should be resolved in the future. + features = np.expand_dims(features,axis=3) + + # make prediction + net.setInput(features) + output = net.forward() + + # decode output to transcript + prediction = decoder.decode(output.squeeze(0)) + return prediction[0] + +if __name__ == '__main__': + + # Computation backends supported by layers + backends = (cv.dnn.DNN_BACKEND_DEFAULT, cv.dnn.DNN_BACKEND_INFERENCE_ENGINE, cv.dnn.DNN_BACKEND_OPENCV) + # Target Devices for computation + targets = (cv.dnn.DNN_TARGET_CPU, cv.dnn.DNN_TARGET_OPENCL, cv.dnn.DNN_TARGET_OPENCL_FP16) + + parser = argparse.ArgumentParser(description='This script runs Jasper Speech recognition model', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--input_audio', type=str, required=True, help='Path to input audio file. OR Path to a txt file with relative path to multiple audio files in different lines') + parser.add_argument('--show_spectrogram', action='store_true', help='Whether to show a spectrogram of the input audio.') + parser.add_argument('--model', type=str, default='jasper.onnx', help='Path to the onnx file of Jasper. default="jasper.onnx"') + parser.add_argument('--output', type=str, help='Path to file where recognized audio transcript must be saved. Leave this to print on console.') + parser.add_argument('--backend', choices=backends, default=cv.dnn.DNN_BACKEND_DEFAULT, type=int, + help='Select a computation backend: ' + "%d: automatically (by default) " + "%d: OpenVINO Inference Engine " + "%d: OpenCV Implementation " % backends) + parser.add_argument('--target', choices=targets, default=cv.dnn.DNN_TARGET_CPU, type=int, + help='Select a target device: ' + "%d: CPU target (by default) " + "%d: OpenCL " + "%d: OpenCL FP16 " % targets) + + args, _ = parser.parse_known_args() + + if args.input_audio and not os.path.isfile(args.input_audio): + raise OSError("Input audio file does not exist") + if not os.path.isfile(args.model): + raise OSError("Jasper model file does not exist") + if args.input_audio.endswith('.txt'): + with open(args.input_audio) as f: + content = f.readlines() + content = [x.strip() for x in content] + audio_file_paths = content + for audio_file_path in audio_file_paths: + if not os.path.isfile(audio_file_path): + raise OSError("Audio file({audio_file_path}) does not exist") + else: + audio_file_paths = [args.input_audio] + audio_file_paths = [os.path.abspath(x) for x in audio_file_paths] + + # Read audio Files + features = [] + try: + for audio_file_path in audio_file_paths: + audio = sf.read(audio_file_path) + # If audio is stereo, just take one channel. + X = audio[0] if audio[0].ndim==1 else audio[0][:,0] + features.append(X) + except: + raise Exception(f"Soundfile cannot read {args.input_audio}. Try a different format") + + # Get Filterbank Features + feature_extractor = FilterbankFeatures() + for i in range(len(features)): + X = features[i] + seq_len = np.array([X.shape[0]], dtype=np.int32) + features[i] = feature_extractor.calculate_features(x=X, seq_len=seq_len) + + # Load Network + net = cv.dnn.readNetFromONNX(args.model) + net.setPreferableBackend(args.backend) + net.setPreferableTarget(args.target) + + # Show spectogram if required + if args.show_spectrogram and not args.input_audio.endswith('.txt'): + img = cv.normalize(src=features[0][0], dst=None, alpha=0, beta=255, norm_type=cv.NORM_MINMAX, dtype=cv.CV_8U) + img = cv.applyColorMap(img, cv.COLORMAP_JET) + cv.imshow('spectogram', img) + cv.waitKey(0) + + # Initialize decoder + decoder = Decoder() + + # Make prediction + prediction = [] + print("Predicting...") + for feature in features: + print(f"\rAudio file {len(prediction)+1}/{len(features)}", end='') + prediction.append(predict(feature, net, decoder)) + print("") + + # save transcript if required + if args.output: + with open(args.output,'w') as f: + for pred in prediction: + f.write(pred+'\n') + print("Transcript was written to {}".format(args.output)) + else: + print(prediction) + cv.destroyAllWindows() From ebef84e9ea8492d15e02ef622e652af29a632eff Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 4 Oct 2021 20:47:07 +0000 Subject: [PATCH 253/376] pre: OpenCV 3.4.16 (version++) --- .../cross_referencing/tutorial_cross_referencing.markdown | 4 ++-- modules/core/include/opencv2/core/version.hpp | 4 ++-- modules/dnn/include/opencv2/dnn/dnn.hpp | 4 ++-- modules/python/package/setup.py | 2 +- platforms/android/build_sdk.py | 2 +- platforms/android/service/readme.txt | 2 +- platforms/maven/opencv-it/pom.xml | 2 +- platforms/maven/opencv/pom.xml | 2 +- platforms/maven/pom.xml | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown index ac347cbca975..27dfc5588a5c 100644 --- a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown +++ b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown @@ -39,14 +39,14 @@ Open your Doxyfile using your favorite text editor and search for the key `TAGFILES`. Change it as follows: @code -TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.15 +TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.16 @endcode If you had other definitions already, you can append the line using a `\`: @code TAGFILES = ./docs/doxygen-tags/libstdc++.tag=https://gcc.gnu.org/onlinedocs/libstdc++/latest-doxygen \ - ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.15 + ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.16 @endcode Doxygen can now use the information from the tag file to link to the OpenCV diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index b80af3e713da..dc8a3e603cb3 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -7,8 +7,8 @@ #define CV_VERSION_MAJOR 3 #define CV_VERSION_MINOR 4 -#define CV_VERSION_REVISION 15 -#define CV_VERSION_STATUS "-dev" +#define CV_VERSION_REVISION 16 +#define CV_VERSION_STATUS "-pre" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 524dc587e6c3..a2a48b7f3e31 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -47,9 +47,9 @@ #include "opencv2/core/async.hpp" #if !defined CV_DOXYGEN && !defined CV_STATIC_ANALYSIS && !defined CV_DNN_DONT_ADD_EXPERIMENTAL_NS -#define CV__DNN_EXPERIMENTAL_NS_BEGIN namespace experimental_dnn_34_v22 { +#define CV__DNN_EXPERIMENTAL_NS_BEGIN namespace experimental_dnn_34_v23 { #define CV__DNN_EXPERIMENTAL_NS_END } -namespace cv { namespace dnn { namespace experimental_dnn_34_v22 { } using namespace experimental_dnn_34_v22; }} +namespace cv { namespace dnn { namespace experimental_dnn_34_v23 { } using namespace experimental_dnn_34_v23; }} #else #define CV__DNN_EXPERIMENTAL_NS_BEGIN #define CV__DNN_EXPERIMENTAL_NS_END diff --git a/modules/python/package/setup.py b/modules/python/package/setup.py index 7c015770e6c7..47a0629dc786 100644 --- a/modules/python/package/setup.py +++ b/modules/python/package/setup.py @@ -9,7 +9,7 @@ def main(): os.chdir(SCRIPT_DIR) package_name = 'opencv' - package_version = os.environ.get('OPENCV_VERSION', '3.4.15') # TODO + package_version = os.environ.get('OPENCV_VERSION', '3.4.16') # TODO long_description = 'Open Source Computer Vision Library Python bindings' # TODO diff --git a/platforms/android/build_sdk.py b/platforms/android/build_sdk.py index cbde665611a4..54d6f8fb4a7a 100755 --- a/platforms/android/build_sdk.py +++ b/platforms/android/build_sdk.py @@ -269,7 +269,7 @@ def build_engine(self, abi, engdest): # Add extra data apkxmldest = check_dir(os.path.join(apkdest, "res", "xml"), create=True) apklibdest = check_dir(os.path.join(apkdest, "libs", abi.name), create=True) - for ver, d in self.extra_packs + [("3.4.15", os.path.join(self.libdest, "lib"))]: + for ver, d in self.extra_packs + [("3.4.16", os.path.join(self.libdest, "lib"))]: r = ET.Element("library", attrib={"version": ver}) log.info("Adding libraries from %s", d) diff --git a/platforms/android/service/readme.txt b/platforms/android/service/readme.txt index a10138c1a6b7..c3d3244a18ee 100644 --- a/platforms/android/service/readme.txt +++ b/platforms/android/service/readme.txt @@ -12,7 +12,7 @@ manually using adb tool: adb install /apk/OpenCV__Manager__.apk -Example: OpenCV_3.4.15-dev_Manager_3.49_armeabi-v7a.apk +Example: OpenCV_3.4.16-dev_Manager_3.49_armeabi-v7a.apk Use the list of platforms below to determine proper OpenCV Manager package for your device: diff --git a/platforms/maven/opencv-it/pom.xml b/platforms/maven/opencv-it/pom.xml index bef5fe28aa5d..039f95116092 100644 --- a/platforms/maven/opencv-it/pom.xml +++ b/platforms/maven/opencv-it/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 3.4.15 + 3.4.16 org.opencv opencv-it diff --git a/platforms/maven/opencv/pom.xml b/platforms/maven/opencv/pom.xml index 6b11c7a0face..64f3f5656fe5 100644 --- a/platforms/maven/opencv/pom.xml +++ b/platforms/maven/opencv/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 3.4.15 + 3.4.16 org.opencv opencv diff --git a/platforms/maven/pom.xml b/platforms/maven/pom.xml index 69e57724895f..f3fadc106f45 100644 --- a/platforms/maven/pom.xml +++ b/platforms/maven/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.opencv opencv-parent - 3.4.15 + 3.4.16 pom OpenCV Parent POM From 3e6f27522b1c5a57a680cb553a2ef42945268a5f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 4 Oct 2021 22:35:47 +0000 Subject: [PATCH 254/376] pre: OpenCV 4.5.4 (version++) --- .../cross_referencing/tutorial_cross_referencing.markdown | 4 ++-- modules/core/include/opencv2/core/version.hpp | 4 ++-- modules/dnn/include/opencv2/dnn/version.hpp | 2 +- modules/python/package/setup.py | 2 +- platforms/maven/opencv-it/pom.xml | 2 +- platforms/maven/opencv/pom.xml | 2 +- platforms/maven/pom.xml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown index 749356063547..da3047aa8796 100644 --- a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown +++ b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown @@ -46,14 +46,14 @@ Open your Doxyfile using your favorite text editor and search for the key `TAGFILES`. Change it as follows: @code -TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.5.3 +TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.5.4 @endcode If you had other definitions already, you can append the line using a `\`: @code TAGFILES = ./docs/doxygen-tags/libstdc++.tag=https://gcc.gnu.org/onlinedocs/libstdc++/latest-doxygen \ - ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.5.3 + ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.5.4 @endcode Doxygen can now use the information from the tag file to link to the OpenCV diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index f627d7147265..4aaf30ce5ed4 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -7,8 +7,8 @@ #define CV_VERSION_MAJOR 4 #define CV_VERSION_MINOR 5 -#define CV_VERSION_REVISION 3 -#define CV_VERSION_STATUS "-dev" +#define CV_VERSION_REVISION 4 +#define CV_VERSION_STATUS "-pre" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) diff --git a/modules/dnn/include/opencv2/dnn/version.hpp b/modules/dnn/include/opencv2/dnn/version.hpp index 0efc9c023765..63e70d2d6e1c 100644 --- a/modules/dnn/include/opencv2/dnn/version.hpp +++ b/modules/dnn/include/opencv2/dnn/version.hpp @@ -6,7 +6,7 @@ #define OPENCV_DNN_VERSION_HPP /// Use with major OpenCV version only. -#define OPENCV_DNN_API_VERSION 20210608 +#define OPENCV_DNN_API_VERSION 20211004 #if !defined CV_DOXYGEN && !defined CV_STATIC_ANALYSIS && !defined CV_DNN_DONT_ADD_INLINE_NS #define CV__DNN_INLINE_NS __CV_CAT(dnn4_v, OPENCV_DNN_API_VERSION) diff --git a/modules/python/package/setup.py b/modules/python/package/setup.py index cb585aa80a2d..030523182aa6 100644 --- a/modules/python/package/setup.py +++ b/modules/python/package/setup.py @@ -9,7 +9,7 @@ def main(): os.chdir(SCRIPT_DIR) package_name = 'opencv' - package_version = os.environ.get('OPENCV_VERSION', '4.5.3') # TODO + package_version = os.environ.get('OPENCV_VERSION', '4.5.4') # TODO long_description = 'Open Source Computer Vision Library Python bindings' # TODO diff --git a/platforms/maven/opencv-it/pom.xml b/platforms/maven/opencv-it/pom.xml index 1166eee2fb16..0209abc3a140 100644 --- a/platforms/maven/opencv-it/pom.xml +++ b/platforms/maven/opencv-it/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 4.5.3 + 4.5.4 org.opencv opencv-it diff --git a/platforms/maven/opencv/pom.xml b/platforms/maven/opencv/pom.xml index 4ac2c2b7f5c4..3551ffc09f3b 100644 --- a/platforms/maven/opencv/pom.xml +++ b/platforms/maven/opencv/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 4.5.3 + 4.5.4 org.opencv opencv diff --git a/platforms/maven/pom.xml b/platforms/maven/pom.xml index 72f4f82b1ce1..66aa22c8ecf0 100644 --- a/platforms/maven/pom.xml +++ b/platforms/maven/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.opencv opencv-parent - 4.5.3 + 4.5.4 pom OpenCV Parent POM From c54abde1bd775788dd2a49c4f6cbb44afd9924f3 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 5 Oct 2021 10:09:27 +0300 Subject: [PATCH 255/376] ffmpeg/3.4: update FFmpeg wrapper 2021.10 - FFmpeg 3.4.8 (no changes) --- 3rdparty/ffmpeg/ffmpeg.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/3rdparty/ffmpeg/ffmpeg.cmake b/3rdparty/ffmpeg/ffmpeg.cmake index a1ada4eeaa78..336bf0d22758 100644 --- a/3rdparty/ffmpeg/ffmpeg.cmake +++ b/3rdparty/ffmpeg/ffmpeg.cmake @@ -1,8 +1,8 @@ -# Binaries branch name: ffmpeg/3.4_20210302 -# Binaries were created for OpenCV: 2ab1f3f166fccc3a01497209cc01c5cea44ff201 -ocv_update(FFMPEG_BINARIES_COMMIT "e99214251d9f3cde7c48abd46b2259bddc9885b6") -ocv_update(FFMPEG_FILE_HASH_BIN32 "fad5ada9be36120bba8966709e7953a8") -ocv_update(FFMPEG_FILE_HASH_BIN64 "650e2272728491923e566f784f79cfef") +# Binaries branch name: ffmpeg/3.4_20211005 +# Binaries were created for OpenCV: 95c1d2a8872b222f32bd88db9f1efcbd9f70a9cf +ocv_update(FFMPEG_BINARIES_COMMIT "0bf6c0753d435d2c82c03c48db0c6e18ac79976c") +ocv_update(FFMPEG_FILE_HASH_BIN32 "55c25bbc13e4a12d4339b70d3b76987f") +ocv_update(FFMPEG_FILE_HASH_BIN64 "67caee9231c6843483b4de9815d6526e") ocv_update(FFMPEG_FILE_HASH_CMAKE "3b90f67f4b429e77d3da36698cef700c") function(download_win_ffmpeg script_var) From 19a880bb918122168f656804d43993b81e7b18ae Mon Sep 17 00:00:00 2001 From: Giles Payne Date: Tue, 5 Oct 2021 20:16:06 +0900 Subject: [PATCH 256/376] Simple matrix multiplication for Mat in iOS/Android --- modules/core/misc/java/src/java/core+Mat.java | 22 ++++++++++++++-- .../core/misc/java/src/java/core+MatMatMul.kt | 3 +++ modules/core/misc/java/test/MatTest.java | 10 +++++++ modules/core/misc/objc/common/Mat.h | 10 +++++++ modules/core/misc/objc/common/Mat.mm | 5 ++++ modules/core/misc/objc/common/MatExt.swift | 6 +++++ modules/core/misc/objc/test/MatTest.swift | 10 +++++++ modules/java/generator/src/cpp/Mat.cpp | 26 +++++++++++++++++++ 8 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 modules/core/misc/java/src/java/core+MatMatMul.kt diff --git a/modules/core/misc/java/src/java/core+Mat.java b/modules/core/misc/java/src/java/core+Mat.java index 5fcc72773873..2d9ee314a367 100644 --- a/modules/core/misc/java/src/java/core+Mat.java +++ b/modules/core/misc/java/src/java/core+Mat.java @@ -466,16 +466,32 @@ public void locateROI(Size wholeSize, Point ofs) { // C++: Mat Mat::mul(Mat m, double scale = 1) // - // javadoc: Mat::mul(m, scale) + /** + * Element-wise multiplication with scale factor + * @param m operand with with which to perform element-wise multiplication + * @param scale scale factor + */ public Mat mul(Mat m, double scale) { return new Mat(n_mul(nativeObj, m.nativeObj, scale)); } - // javadoc: Mat::mul(m) + /** + * Element-wise multiplication + * @param m operand with with which to perform element-wise multiplication + */ public Mat mul(Mat m) { return new Mat(n_mul(nativeObj, m.nativeObj)); } + /** + * Matrix multiplication + * @param m operand with with which to perform matrix multiplication + * @see Core#gemm(Mat, Mat, double, Mat, double, Mat, int) + */ + public Mat matMul(Mat m) { + return new Mat(n_matMul(nativeObj, m.nativeObj)); + } + // // C++: static Mat Mat::ones(int rows, int cols, int type) // @@ -1732,6 +1748,8 @@ public long getNativeObjAddr() { private static native long n_mul(long nativeObj, long m_nativeObj); + private static native long n_matMul(long nativeObj, long m_nativeObj); + // C++: static Mat Mat::ones(int rows, int cols, int type) private static native long n_ones(int rows, int cols, int type); diff --git a/modules/core/misc/java/src/java/core+MatMatMul.kt b/modules/core/misc/java/src/java/core+MatMatMul.kt new file mode 100644 index 000000000000..3dcb6bde79b5 --- /dev/null +++ b/modules/core/misc/java/src/java/core+MatMatMul.kt @@ -0,0 +1,3 @@ +package org.opencv.core + +operator fun Mat.times(other: Mat): Mat = this.matMul(other) diff --git a/modules/core/misc/java/test/MatTest.java b/modules/core/misc/java/test/MatTest.java index 3075dba16b35..323440b2eb96 100644 --- a/modules/core/misc/java/test/MatTest.java +++ b/modules/core/misc/java/test/MatTest.java @@ -686,6 +686,16 @@ public void testMulMatDouble() { assertMatEqual(truth, dst, EPS); } + public void testMatMulMat() { + Mat m1 = new Mat(2, 2, CvType.CV_32F, new Scalar(2)); + Mat m2 = new Mat(2, 2, CvType.CV_32F, new Scalar(3)); + + dst = m1.matMul(m2); + + truth = new Mat(2, 2, CvType.CV_32F, new Scalar(12)); + assertMatEqual(truth, dst, EPS); + } + public void testOnesIntIntInt() { dst = Mat.ones(matSize, matSize, CvType.CV_32F); diff --git a/modules/core/misc/objc/common/Mat.h b/modules/core/misc/objc/common/Mat.h index fd1dce27ba4c..b5e868306c2d 100644 --- a/modules/core/misc/objc/common/Mat.h +++ b/modules/core/misc/objc/common/Mat.h @@ -114,7 +114,17 @@ CV_EXPORTS @interface Mat : NSObject - (BOOL)isSubmatrix; - (void)locateROI:(Size2i*)wholeSize ofs:(Point2i*)offset NS_SWIFT_NAME(locateROI(wholeSize:offset:)); - (Mat*)mul:(Mat*)mat scale:(double)scale; +/** + Performs element-wise multiplication + @param mat operand with with which to perform element-wise multiplication +*/ - (Mat*)mul:(Mat*)mat; +/** + Performs matrix multiplication + @param mat operand with with which to perform matrix multiplication + @see `Core.gemm(...)` +*/ +- (Mat*)matMul:(Mat*)mat; + (Mat*)ones:(int)rows cols:(int)cols type:(int)type NS_SWIFT_NAME(ones(rows:cols:type:)); + (Mat*)ones:(Size2i*)size type:(int)type NS_SWIFT_NAME(ones(size:type:)); + (Mat*)onesEx:(NSArray*)sizes type:(int)type NS_SWIFT_NAME(ones(sizes:type:)); diff --git a/modules/core/misc/objc/common/Mat.mm b/modules/core/misc/objc/common/Mat.mm index dc2316c87bda..ab5e1de6f9a2 100644 --- a/modules/core/misc/objc/common/Mat.mm +++ b/modules/core/misc/objc/common/Mat.mm @@ -372,6 +372,11 @@ - (Mat*)mul:(Mat*)mat { return [[Mat alloc] initWithNativeMat:new cv::Mat(_nativePtr->mul(*(cv::Mat*)mat.nativePtr))]; } +- (Mat*)matMul:(Mat*)mat { + cv::Mat temp = self.nativeRef * mat.nativeRef; + return [Mat fromNative:temp]; +} + + (Mat*)ones:(int)rows cols:(int)cols type:(int)type { return [[Mat alloc] initWithNativeMat:new cv::Mat(cv::Mat::ones(rows, cols, type))]; } diff --git a/modules/core/misc/objc/common/MatExt.swift b/modules/core/misc/objc/common/MatExt.swift index a6ba548599d8..f0f4446682fe 100644 --- a/modules/core/misc/objc/common/MatExt.swift +++ b/modules/core/misc/objc/common/MatExt.swift @@ -715,3 +715,9 @@ public extension Mat { return MatAt(mat: self, indices: indices) } } + +public extension Mat { + static func *(lhs:Mat, rhs: Mat) -> Mat { + return lhs.matMul(rhs) + } +} diff --git a/modules/core/misc/objc/test/MatTest.swift b/modules/core/misc/objc/test/MatTest.swift index 2dcfedf41fb0..87f99fb84a01 100644 --- a/modules/core/misc/objc/test/MatTest.swift +++ b/modules/core/misc/objc/test/MatTest.swift @@ -683,6 +683,16 @@ class MatTests: OpenCVTestCase { try assertMatEqual(truth!, dst, OpenCVTestCase.EPS) } + func testMatMulMat() throws { + let m1 = Mat(rows: 2, cols: 2, type: CvType.CV_32F, scalar: Scalar(2)) + let m2 = Mat(rows: 2, cols: 2, type: CvType.CV_32F, scalar: Scalar(3)) + + dst = m1.matMul(m2) + + truth = Mat(rows: 2, cols: 2, type: CvType.CV_32F, scalar: Scalar(12)) + try assertMatEqual(truth!, dst, OpenCVTestCase.EPS) + } + func testOnesIntIntInt() throws { dst = Mat.ones(rows: OpenCVTestCase.matSize, cols: OpenCVTestCase.matSize, type: CvType.CV_32F) diff --git a/modules/java/generator/src/cpp/Mat.cpp b/modules/java/generator/src/cpp/Mat.cpp index d59fe4a5065a..f43a4c6ed5ca 100644 --- a/modules/java/generator/src/cpp/Mat.cpp +++ b/modules/java/generator/src/cpp/Mat.cpp @@ -1372,6 +1372,32 @@ JNIEXPORT jlong JNICALL Java_org_opencv_core_Mat_n_1mul__JJ } +// +// Mat Mat Mat::matMul(Mat m) +// + +JNIEXPORT jlong JNICALL Java_org_opencv_core_Mat_n_1matMul__JJ + (JNIEnv* env, jclass, jlong self, jlong m_nativeObj); + +JNIEXPORT jlong JNICALL Java_org_opencv_core_Mat_n_1matMul__JJ + (JNIEnv* env, jclass, jlong self, jlong m_nativeObj) +{ + static const char method_name[] = "Mat::n_1matMul__JJ()"; + try { + LOGD("%s", method_name); + Mat* me = (Mat*) self; //TODO: check for NULL + Mat& m = *((Mat*)m_nativeObj); + Mat _retval_ = (*me) * m; + return (jlong) new Mat(_retval_); + } catch(const std::exception &e) { + throwJavaException(env, &e, method_name); + } catch (...) { + throwJavaException(env, 0, method_name); + } + + return 0; +} + // // static Mat Mat::ones(int rows, int cols, int type) From b7a7119b1fc44a64d22b59410482b4351cd6dcdb Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 5 Oct 2021 14:32:39 +0300 Subject: [PATCH 257/376] ffmpeg/4.x: update FFmpeg wrapper 2021.10 - FFmpeg 4.4 (no changes) --- 3rdparty/ffmpeg/ffmpeg.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/3rdparty/ffmpeg/ffmpeg.cmake b/3rdparty/ffmpeg/ffmpeg.cmake index 764e4b19d830..2eea6581780e 100644 --- a/3rdparty/ffmpeg/ffmpeg.cmake +++ b/3rdparty/ffmpeg/ffmpeg.cmake @@ -1,8 +1,8 @@ -# Binaries branch name: ffmpeg/master_20210608 -# Binaries were created for OpenCV: eaa9228a4fdfb9c2465aea65a50ce2d16b55dce0 -ocv_update(FFMPEG_BINARIES_COMMIT "213fcd5d4897319a83207406036c4a5957fba010") -ocv_update(FFMPEG_FILE_HASH_BIN32 "bab661341c30862fa88627130219c0a5") -ocv_update(FFMPEG_FILE_HASH_BIN64 "ac99f9767a83103c31709628af685924") +# Binaries branch name: ffmpeg/master_20211005 +# Binaries were created for OpenCV: 672399c751c431bbe52818b33fd3ca17b51e0e16 +ocv_update(FFMPEG_BINARIES_COMMIT "40b4666d1aa374205fd61373496e15d92ecd5313") +ocv_update(FFMPEG_FILE_HASH_BIN32 "c2f9a897d464a2dce2286f8067ad9d90") +ocv_update(FFMPEG_FILE_HASH_BIN64 "878a4e8fe5a4d68f18c9cdde543b9ead") ocv_update(FFMPEG_FILE_HASH_CMAKE "8862c87496e2e8c375965e1277dee1c7") function(download_win_ffmpeg script_var) From e5fb50476c14d0f096ae2b93b2b4be578710a43f Mon Sep 17 00:00:00 2001 From: HAN Liutong Date: Tue, 5 Oct 2021 23:35:00 +0800 Subject: [PATCH 258/376] Merge pull request #20521 from hanliutong:dev-rvv-multiVLEN Make the implementation of optimization in DNN adjustable to different vector sizes with RVV intrinsics. * Update fastGEMM for multi VLEN. * Update fastGEMM1T for multi VLEN. * Update fastDepthwiseConv for multi VLEN. * Update fastConv for multi VLEN. * Replace malloc with cv::AutoBuffer. --- modules/dnn/src/layers/layers_common.simd.hpp | 626 ++++++++++-------- 1 file changed, 346 insertions(+), 280 deletions(-) diff --git a/modules/dnn/src/layers/layers_common.simd.hpp b/modules/dnn/src/layers/layers_common.simd.hpp index 762e22e54d2f..0a077e463158 100644 --- a/modules/dnn/src/layers/layers_common.simd.hpp +++ b/modules/dnn/src/layers/layers_common.simd.hpp @@ -744,58 +744,66 @@ void fastGEMM( const float* aptr, size_t astep, const float* bptr, int ma, int na, int nb ) { int n = 0; - size_t vl = 8; - size_t mvl0 = 8; - size_t mvl1 = 8; - for( ; n < nb; n += 16 ) + int vl = vsetvlmax_e32m4(); + int mvl = vl; + for( ; n < nb; n += vl ) { - if ( n + 16 > nb) { - mvl0 = nb - n; - mvl1 = (nb - n -8) > 0 ? (nb - n -8) : 0; + if ( n + vl > nb) { + mvl = nb - n; } - for( int m = 0; m < ma; m += 4 ) + for( int m = 0; m < ma; m += 7 ) { const float* aptr0 = aptr + astep*m; const float* aptr1 = aptr + astep*std::min(m+1, ma-1); const float* aptr2 = aptr + astep*std::min(m+2, ma-1); const float* aptr3 = aptr + astep*std::min(m+3, ma-1); + const float* aptr4 = aptr + astep*std::min(m+4, ma-1); + const float* aptr5 = aptr + astep*std::min(m+5, ma-1); + const float* aptr6 = aptr + astep*std::min(m+6, ma-1); float* cptr0 = cptr + cstep*m; float* cptr1 = cptr + cstep*std::min(m+1, ma-1); float* cptr2 = cptr + cstep*std::min(m+2, ma-1); float* cptr3 = cptr + cstep*std::min(m+3, ma-1); - - vfloat32m2_t d00 = vfmv_v_f_f32m2(0, vl), d01 = vfmv_v_f_f32m2(0, vl); - vfloat32m2_t d10 = vfmv_v_f_f32m2(0, vl), d11 = vfmv_v_f_f32m2(0, vl); - vfloat32m2_t d20 = vfmv_v_f_f32m2(0, vl), d21 = vfmv_v_f_f32m2(0, vl); - vfloat32m2_t d30 = vfmv_v_f_f32m2(0, vl), d31 = vfmv_v_f_f32m2(0, vl); + float* cptr4 = cptr + cstep*std::min(m+4, ma-1); + float* cptr5 = cptr + cstep*std::min(m+5, ma-1); + float* cptr6 = cptr + cstep*std::min(m+6, ma-1); + + vfloat32m4_t d0 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d1 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d2 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d3 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d4 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d5 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d6 = vfmv_v_f_f32m4(0, vl); for( int k = 0; k < na; k++ ) { - vfloat32m2_t a0 = vfmv_v_f_f32m2(aptr0[k], vl); - vfloat32m2_t a1 = vfmv_v_f_f32m2(aptr1[k], vl); - vfloat32m2_t a2 = vfmv_v_f_f32m2(aptr2[k], vl); - vfloat32m2_t a3 = vfmv_v_f_f32m2(aptr3[k], vl); - vfloat32m2_t b0 = vle32_v_f32m2(bptr + k*bstep + n, mvl0); - vfloat32m2_t b1 = vle32_v_f32m2(bptr + k*bstep + n + 8, mvl1); - d00 = vfmacc_vv_f32m2(d00, a0, b0, mvl0); - d01 = vfmacc_vv_f32m2(d01, a0, b1, mvl1); - d10 = vfmacc_vv_f32m2(d10, a1, b0, mvl0); - d11 = vfmacc_vv_f32m2(d11, a1, b1, mvl1); - d20 = vfmacc_vv_f32m2(d20, a2, b0, mvl0); - d21 = vfmacc_vv_f32m2(d21, a2, b1, mvl1); - d30 = vfmacc_vv_f32m2(d30, a3, b0, mvl0); - d31 = vfmacc_vv_f32m2(d31, a3, b1, mvl1); + float32_t a0 = aptr0[k]; + float32_t a1 = aptr1[k]; + float32_t a2 = aptr2[k]; + float32_t a3 = aptr3[k]; + float32_t a4 = aptr4[k]; + float32_t a5 = aptr5[k]; + float32_t a6 = aptr6[k]; + + vfloat32m4_t b = vle32_v_f32m4(bptr + k*bstep + n, mvl); + d0 = vfmacc_vf_f32m4(d0, a0, b, mvl); + d1 = vfmacc_vf_f32m4(d1, a1, b, mvl); + d2 = vfmacc_vf_f32m4(d2, a2, b, mvl); + d3 = vfmacc_vf_f32m4(d3, a3, b, mvl); + d4 = vfmacc_vf_f32m4(d4, a4, b, mvl); + d5 = vfmacc_vf_f32m4(d5, a5, b, mvl); + d6 = vfmacc_vf_f32m4(d6, a6, b, mvl); } - vse32_v_f32m2(cptr0 + n, d00, mvl0); - vse32_v_f32m2(cptr1 + n, d10, mvl0); - vse32_v_f32m2(cptr2 + n, d20, mvl0); - vse32_v_f32m2(cptr3 + n, d30, mvl0); - vse32_v_f32m2(cptr0 + n + 8, d01, mvl1); - vse32_v_f32m2(cptr1 + n + 8, d11, mvl1); - vse32_v_f32m2(cptr2 + n + 8, d21, mvl1); - vse32_v_f32m2(cptr3 + n + 8, d31, mvl1); + vse32_v_f32m4(cptr0 + n, d0, mvl); + vse32_v_f32m4(cptr1 + n, d1, mvl); + vse32_v_f32m4(cptr2 + n, d2, mvl); + vse32_v_f32m4(cptr3 + n, d3, mvl); + vse32_v_f32m4(cptr4 + n, d4, mvl); + vse32_v_f32m4(cptr5 + n, d5, mvl); + vse32_v_f32m4(cptr6 + n, d6, mvl); } } } @@ -804,71 +812,108 @@ void fastGEMM1T( const float* vec, const float* weights, size_t wstep, const float* bias, float* dst, int nvecs, int vecsize ) { + int vlm2 = vsetvlmax_e32m2(); int i = 0; - size_t vl = 8; - for( ; i <= nvecs - 8; i += 8 ) + for( ; i <= nvecs - 15; i += 15 ) { const float* wptr = weights + i*wstep; - vfloat32m2_t vs0 = vfmv_v_f_f32m2(0, vl), vs1 = vfmv_v_f_f32m2(0, vl), - vs2 = vfmv_v_f_f32m2(0, vl), vs3 = vfmv_v_f_f32m2(0, vl), - vs4 = vfmv_v_f_f32m2(0, vl), vs5 = vfmv_v_f_f32m2(0, vl), - vs6 = vfmv_v_f_f32m2(0, vl), vs7 = vfmv_v_f_f32m2(0, vl); - - for( int k = 0; k < vecsize; k += 8, wptr += 8 ) + vfloat32m2_t + vs0 = vfmv_v_f_f32m2(0, vlm2), vs1 = vfmv_v_f_f32m2(0, vlm2), vs2 = vfmv_v_f_f32m2(0, vlm2), + vs3 = vfmv_v_f_f32m2(0, vlm2), vs4 = vfmv_v_f_f32m2(0, vlm2), vs5 = vfmv_v_f_f32m2(0, vlm2), + vs6 = vfmv_v_f_f32m2(0, vlm2), vs7 = vfmv_v_f_f32m2(0, vlm2), vs8 = vfmv_v_f_f32m2(0, vlm2), + vs9 = vfmv_v_f_f32m2(0, vlm2), vs10 = vfmv_v_f_f32m2(0, vlm2), vs11 = vfmv_v_f_f32m2(0, vlm2), + vs12 = vfmv_v_f_f32m2(0, vlm2), vs13 = vfmv_v_f_f32m2(0, vlm2), vs14 = vfmv_v_f_f32m2(0, vlm2); + int k = 0; + for( ; k < vecsize - vlm2; k += vlm2, wptr += vlm2 ) { - vfloat32m2_t v = vle32_v_f32m2(vec + k, vl); - - vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vl), v, vl); - vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep, vl), v, vl); - vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*2, vl), v, vl); - vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*3, vl), v, vl); - vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*4, vl), v, vl); - vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*5, vl), v, vl); - vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*6, vl), v, vl); - vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*7, vl), v, vl); + vfloat32m2_t v = vle32_v_f32m2(vec + k, vlm2); + + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vlm2), v, vlm2); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep, vlm2), v, vlm2); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*2, vlm2), v, vlm2); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*3, vlm2), v, vlm2); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*4, vlm2), v, vlm2); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*5, vlm2), v, vlm2); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*6, vlm2), v, vlm2); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*7, vlm2), v, vlm2); + vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*8, vlm2), v, vlm2); + vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*9, vlm2), v, vlm2); + vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*10, vlm2), v, vlm2); + vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*11, vlm2), v, vlm2); + vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*12, vlm2), v, vlm2); + vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*13, vlm2), v, vlm2); + vs14 = vfmacc_vv_f32m2(vs14, vle32_v_f32m2(wptr + wstep*14, vlm2), v, vlm2); + } + int kvl = vecsize - k; + if (kvl > 0) { + vfloat32m2_t v = vle32_v_f32m2(vec + k, kvl); + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, kvl), v, kvl); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*1, kvl), v, kvl); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*2, kvl), v, kvl); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*3, kvl), v, kvl); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*4, kvl), v, kvl); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*5, kvl), v, kvl); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*6, kvl), v, kvl); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*7, kvl), v, kvl); + vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*8, kvl), v, kvl); + vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*9, kvl), v, kvl); + vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*10, kvl), v, kvl); + vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*11, kvl), v, kvl); + vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*12, kvl), v, kvl); + vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*13, kvl), v, kvl); + vs14 = vfmacc_vv_f32m2(vs14, vle32_v_f32m2(wptr + wstep*14, kvl), v, kvl); } // Calculate the sum of each vector - vfloat32m1_t zero = vfmv_v_f_f32m1(0, vl); - vfloat32m1_t temp0 = vfredsum_vs_f32m2_f32m1(temp0, vs0, zero, vl); - vfloat32m1_t temp1 = vfredsum_vs_f32m2_f32m1(temp1, vs1, zero, vl); - vfloat32m1_t temp2 = vfredsum_vs_f32m2_f32m1(temp2, vs2, zero, vl); - vfloat32m1_t temp3 = vfredsum_vs_f32m2_f32m1(temp3, vs3, zero, vl); - vfloat32m1_t temp4 = vfredsum_vs_f32m2_f32m1(temp4, vs4, zero, vl); - vfloat32m1_t temp5 = vfredsum_vs_f32m2_f32m1(temp5, vs5, zero, vl); - vfloat32m1_t temp6 = vfredsum_vs_f32m2_f32m1(temp6, vs6, zero, vl); - vfloat32m1_t temp7 = vfredsum_vs_f32m2_f32m1(temp7, vs7, zero, vl); - float32_t sum[8]; - sum[0] = vfmv_f_s_f32m1_f32(temp0); - sum[1] = vfmv_f_s_f32m1_f32(temp1); - sum[2] = vfmv_f_s_f32m1_f32(temp2); - sum[3] = vfmv_f_s_f32m1_f32(temp3); - sum[4] = vfmv_f_s_f32m1_f32(temp4); - sum[5] = vfmv_f_s_f32m1_f32(temp5); - sum[6] = vfmv_f_s_f32m1_f32(temp6); - sum[7] = vfmv_f_s_f32m1_f32(temp7); - vfloat32m2_t s0 = vfadd_vv_f32m2(vle32_v_f32m2(sum, vl), vle32_v_f32m2(bias + i, vl), vl); - vse32_v_f32m2(dst + i, s0, vl); + float32_t sum[15]; + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vlm2); + sum[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs0, zero, vlm2)); + sum[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs1, zero, vlm2)); + sum[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs2, zero, vlm2)); + sum[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs3, zero, vlm2)); + sum[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs4, zero, vlm2)); + sum[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs5, zero, vlm2)); + sum[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs6, zero, vlm2)); + sum[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs7, zero, vlm2)); + sum[8] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs8, zero, vlm2)); + sum[9] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs9, zero, vlm2)); + sum[10] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs10, zero, vlm2)); + sum[11] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs11, zero, vlm2)); + sum[12] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs12, zero, vlm2)); + sum[13] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs13, zero, vlm2)); + sum[14] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs14, zero, vlm2)); + + vfloat32m4_t s0 = vfadd_vv_f32m4(vle32_v_f32m4(sum, 15), vle32_v_f32m4(bias + i, 15), 15); + vse32_v_f32m4(dst + i, s0, 15); } int mvl = nvecs - i; if (mvl > 0) { const float* wptr = weights + i*wstep; - vfloat32m2_t vs0 = vfmv_v_f_f32m2(0, vl), vs1 = vfmv_v_f_f32m2(0, vl), - vs2 = vfmv_v_f_f32m2(0, vl), vs3 = vfmv_v_f_f32m2(0, vl), - vs4 = vfmv_v_f_f32m2(0, vl), vs5 = vfmv_v_f_f32m2(0, vl), - vs6 = vfmv_v_f_f32m2(0, vl), vs7 = vfmv_v_f_f32m2(0, vl); + vfloat32m2_t + vs0 = vfmv_v_f_f32m2(0, vlm2), vs1 = vfmv_v_f_f32m2(0, vlm2), vs2 = vfmv_v_f_f32m2(0, vlm2), + vs3 = vfmv_v_f_f32m2(0, vlm2), vs4 = vfmv_v_f_f32m2(0, vlm2), vs5 = vfmv_v_f_f32m2(0, vlm2), + vs6 = vfmv_v_f_f32m2(0, vlm2), vs7 = vfmv_v_f_f32m2(0, vlm2), vs8 = vfmv_v_f_f32m2(0, vlm2), + vs9 = vfmv_v_f_f32m2(0, vlm2), vs10 = vfmv_v_f_f32m2(0, vlm2), vs11 = vfmv_v_f_f32m2(0, vlm2), + vs12 = vfmv_v_f_f32m2(0, vlm2), vs13 = vfmv_v_f_f32m2(0, vlm2); int k = 0; - for( ; k <= vecsize - 8; k += 8, wptr += 8 ) + for( ; k <= vecsize - vlm2; k += vlm2, wptr += vlm2 ) { - vfloat32m2_t v = vle32_v_f32m2(vec + k, vl); - vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vl), v, vl); - vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*std::min(1, mvl-1), vl), v, vl); - vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*std::min(2, mvl-1), vl), v, vl); - vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*std::min(3, mvl-1), vl), v, vl); - vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), vl), v, vl); - vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), vl), v, vl); - vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), vl), v, vl); + vfloat32m2_t v = vle32_v_f32m2(vec + k, vlm2); + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vlm2), v, vlm2); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*std::min(1, mvl-1), vlm2), v, vlm2); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*std::min(2, mvl-1), vlm2), v, vlm2); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*std::min(3, mvl-1), vlm2), v, vlm2); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), vlm2), v, vlm2); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), vlm2), v, vlm2); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), vlm2), v, vlm2); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*std::min(7, mvl-1), vlm2), v, vlm2); + vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*std::min(8, mvl-1), vlm2), v, vlm2); + vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*std::min(9, mvl-1), vlm2), v, vlm2); + vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*std::min(10, mvl-1), vlm2), v, vlm2); + vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*std::min(11, mvl-1), vlm2), v, vlm2); + vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*std::min(12, mvl-1), vlm2), v, vlm2); + vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*std::min(13, mvl-1), vlm2), v, vlm2); } int kvl = vecsize - k; if (kvl > 0) { @@ -880,54 +925,47 @@ void fastGEMM1T( const float* vec, const float* weights, vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), kvl), v, kvl); vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), kvl), v, kvl); vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), kvl), v, kvl); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*std::min(7, mvl-1), kvl), v, kvl); + vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*std::min(8, mvl-1), kvl), v, kvl); + vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*std::min(9, mvl-1), kvl), v, kvl); + vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*std::min(10, mvl-1), kvl), v, kvl); + vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*std::min(11, mvl-1), kvl), v, kvl); + vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*std::min(12, mvl-1), kvl), v, kvl); + vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*std::min(13, mvl-1), kvl), v, kvl); } // Calculate the sum of each vector - vfloat32m1_t zero = vfmv_v_f_f32m1(0, vl); - vfloat32m1_t temp0 = vfmv_v_f_f32m1(0, 4), temp1 = vfmv_v_f_f32m1(0, 4), - temp2 = vfmv_v_f_f32m1(0, 4), temp3 = vfmv_v_f_f32m1(0, 4), - temp4 = vfmv_v_f_f32m1(0, 4), temp5 = vfmv_v_f_f32m1(0, 4), - temp6 = vfmv_v_f_f32m1(0, 4), temp7 = vfmv_v_f_f32m1(0, 4); - temp0 = vfredsum_vs_f32m2_f32m1(temp0, vs0, zero, vl); - temp1 = vfredsum_vs_f32m2_f32m1(temp1, vs1, zero, vl); - temp2 = vfredsum_vs_f32m2_f32m1(temp2, vs2, zero, vl); - temp3 = vfredsum_vs_f32m2_f32m1(temp3, vs3, zero, vl); - temp4 = vfredsum_vs_f32m2_f32m1(temp4, vs4, zero, vl); - temp5 = vfredsum_vs_f32m2_f32m1(temp5, vs5, zero, vl); - temp6 = vfredsum_vs_f32m2_f32m1(temp6, vs6, zero, vl); - temp7 = vfredsum_vs_f32m2_f32m1(temp7, vs7, zero, vl); - - float32_t sum[8]; - sum[0] = vfmv_f_s_f32m1_f32(temp0); - sum[1] = vfmv_f_s_f32m1_f32(temp1); - sum[2] = vfmv_f_s_f32m1_f32(temp2); - sum[3] = vfmv_f_s_f32m1_f32(temp3); - sum[4] = vfmv_f_s_f32m1_f32(temp4); - sum[5] = vfmv_f_s_f32m1_f32(temp5); - sum[6] = vfmv_f_s_f32m1_f32(temp6); - sum[7] = vfmv_f_s_f32m1_f32(temp7); - - vfloat32m2_t s0 = vfadd_vv_f32m2(vle32_v_f32m2(sum, mvl), vle32_v_f32m2(bias + i, mvl), mvl); - vse32_v_f32m2(dst + i, s0, mvl); + float32_t sum[14]; + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vlm2); + sum[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs0, zero, vlm2)); + sum[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs1, zero, vlm2)); + sum[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs2, zero, vlm2)); + sum[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs3, zero, vlm2)); + sum[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs4, zero, vlm2)); + sum[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs5, zero, vlm2)); + sum[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs6, zero, vlm2)); + sum[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs7, zero, vlm2)); + sum[8] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs8, zero, vlm2)); + sum[9] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs9, zero, vlm2)); + sum[10] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs10, zero, vlm2)); + sum[11] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs11, zero, vlm2)); + sum[12] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs12, zero, vlm2)); + sum[13] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs13, zero, vlm2)); + + vfloat32m4_t s0 = vfadd_vv_f32m4(vle32_v_f32m4(sum, mvl), vle32_v_f32m4(bias + i, mvl), mvl); + vse32_v_f32m4(dst + i, s0, mvl); } } -enum { FASCONV_BASE_VECSZ = 4 }; // TODO: Large base size. +enum { FASCONV_BASE_VECSZ = 8 }; void fastConv( const float* weights, size_t wstep, const float* bias, const float* rowbuf, float* output, const int* outShape, int blockSize, int vecsize, int vecsize_aligned, const float* relu, bool initOutput ) { - int vl = 4; + int vl = FASCONV_BASE_VECSZ; + int vlm1Max = vsetvlmax_e32m1(); int outCn = outShape[1]; size_t outPlaneSize = outShape[2]*outShape[3]; - float r0 = 1.f, r1 = 1.f, r2 = 1.f; - vfloat32m1_t vr0 = vfmv_v_f_f32m1(1, vl), vr1 = vfmv_v_f_f32m1(1, vl), vr2 = vfmv_v_f_f32m1(1, vl); - int maskbuf[FASCONV_BASE_VECSZ] = {0}; - int rsz = blockSize % FASCONV_BASE_VECSZ; - for( int i = 0; i < rsz; i++ ) - maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1; - vint32m1_t vmaskbuf = vle32_v_i32m1(maskbuf ,vl); - vbool32_t mask = vmslt_vx_i32m1_b32(vmaskbuf, 0, vl); // mask for tail // now compute dot product of the weights // and im2row-transformed part of the tensor for( int i = 0; i < outCn; i += 3 ) @@ -953,20 +991,6 @@ void fastConv( const float* weights, size_t wstep, const float* bias, } } - if( relu ) - { - r0 = relu[i]; r1 = relu[i+1]; r2 = relu[i+2]; - if( i+2 >= outCn ) - { - r2 = r1; - if( i+1 >= outCn ) - r2 = r1 = r0; - } - vr0 = vfmv_v_f_f32m1(r0, vl); - vr1 = vfmv_v_f_f32m1(r1, vl); - vr2 = vfmv_v_f_f32m1(r2, vl); - } - int j = 0; for( ; j < blockSize; j += FASCONV_BASE_VECSZ ) { @@ -983,110 +1007,152 @@ void fastConv( const float* weights, size_t wstep, const float* bias, } int k = 0; const float* rptr = rowbuf + j*vecsize_aligned; - int vlm2 = 8; - vfloat32m2_t vs00 = vfmv_v_f_f32m2(0, vlm2), vs01 = vfmv_v_f_f32m2(0, vlm2), - vs02 = vfmv_v_f_f32m2(0, vlm2), vs03 = vfmv_v_f_f32m2(0, vlm2), - vs10 = vfmv_v_f_f32m2(0, vlm2), vs11 = vfmv_v_f_f32m2(0, vlm2), - vs12 = vfmv_v_f_f32m2(0, vlm2), vs13 = vfmv_v_f_f32m2(0, vlm2), - vs20 = vfmv_v_f_f32m2(0, vlm2), vs21 = vfmv_v_f_f32m2(0, vlm2), - vs22 = vfmv_v_f_f32m2(0, vlm2), vs23 = vfmv_v_f_f32m2(0, vlm2); - - for (; k < vecsize; k += 8, rptr += 8 ) + int vlm1 = vsetvlmax_e32m1(); + vfloat32m1_t + vs00 = vfmv_v_f_f32m1(0, vlm1), vs10 = vfmv_v_f_f32m1(0, vlm1), vs20 = vfmv_v_f_f32m1(0, vlm1), + vs01 = vfmv_v_f_f32m1(0, vlm1), vs11 = vfmv_v_f_f32m1(0, vlm1), vs21 = vfmv_v_f_f32m1(0, vlm1), + vs02 = vfmv_v_f_f32m1(0, vlm1), vs12 = vfmv_v_f_f32m1(0, vlm1), vs22 = vfmv_v_f_f32m1(0, vlm1), + vs03 = vfmv_v_f_f32m1(0, vlm1), vs13 = vfmv_v_f_f32m1(0, vlm1), vs23 = vfmv_v_f_f32m1(0, vlm1), + vs04 = vfmv_v_f_f32m1(0, vlm1), vs14 = vfmv_v_f_f32m1(0, vlm1), vs24 = vfmv_v_f_f32m1(0, vlm1), + vs05 = vfmv_v_f_f32m1(0, vlm1), vs15 = vfmv_v_f_f32m1(0, vlm1), vs25 = vfmv_v_f_f32m1(0, vlm1), + vs06 = vfmv_v_f_f32m1(0, vlm1), vs16 = vfmv_v_f_f32m1(0, vlm1), vs26 = vfmv_v_f_f32m1(0, vlm1), + vs07 = vfmv_v_f_f32m1(0, vlm1), vs17 = vfmv_v_f_f32m1(0, vlm1), vs27 = vfmv_v_f_f32m1(0, vlm1); + + for (; k < vecsize; k += vlm1, rptr += vlm1 ) { - if (k+8 >= vecsize) { - vlm2 = vecsize - k; + if (k + vlm1 >= vecsize) { + vlm1 = vecsize - k; } - vfloat32m2_t w0 = vle32_v_f32m2(wptr0 + k, vlm2); - vfloat32m2_t w1 = vle32_v_f32m2(wptr1 + k, vlm2); - vfloat32m2_t w2 = vle32_v_f32m2(wptr2 + k, vlm2); - vfloat32m2_t r0 = vle32_v_f32m2(rptr, vlm2); - - vs00 = vfmacc_vv_f32m2(vs00, w0, r0, vlm2); - vs10 = vfmacc_vv_f32m2(vs10, w1, r0, vlm2); - vs20 = vfmacc_vv_f32m2(vs20, w2, r0, vlm2); - - r0 = vle32_v_f32m2(rptr + vecsize_aligned, vlm2); - vs01 = vfmacc_vv_f32m2(vs01, w0, r0, vlm2); - vs11 = vfmacc_vv_f32m2(vs11, w1, r0, vlm2); - vs21 = vfmacc_vv_f32m2(vs21, w2, r0, vlm2); - - r0 = vle32_v_f32m2(rptr + vecsize_aligned*2, vlm2); - vs02 = vfmacc_vv_f32m2(vs02, w0, r0, vlm2); - vs12 = vfmacc_vv_f32m2(vs12, w1, r0, vlm2); - vs22 = vfmacc_vv_f32m2(vs22, w2, r0, vlm2); - - r0 = vle32_v_f32m2(rptr + vecsize_aligned*3, vlm2); - vs03 = vfmacc_vv_f32m2(vs03, w0, r0, vlm2); - vs13 = vfmacc_vv_f32m2(vs13, w1, r0, vlm2); - vs23 = vfmacc_vv_f32m2(vs23, w2, r0, vlm2); + vfloat32m1_t w0 = vle32_v_f32m1(wptr0 + k, vlm1); + vfloat32m1_t w1 = vle32_v_f32m1(wptr1 + k, vlm1); + vfloat32m1_t w2 = vle32_v_f32m1(wptr2 + k, vlm1); + vfloat32m1_t r0 = vle32_v_f32m1(rptr, vlm1); + + vs00 = vfmacc_vv_f32m1(vs00, w0, r0, vlm1); + vs10 = vfmacc_vv_f32m1(vs10, w1, r0, vlm1); + vs20 = vfmacc_vv_f32m1(vs20, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned, vlm1); + vs01 = vfmacc_vv_f32m1(vs01, w0, r0, vlm1); + vs11 = vfmacc_vv_f32m1(vs11, w1, r0, vlm1); + vs21 = vfmacc_vv_f32m1(vs21, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*2, vlm1); + vs02 = vfmacc_vv_f32m1(vs02, w0, r0, vlm1); + vs12 = vfmacc_vv_f32m1(vs12, w1, r0, vlm1); + vs22 = vfmacc_vv_f32m1(vs22, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*3, vlm1); + vs03 = vfmacc_vv_f32m1(vs03, w0, r0, vlm1); + vs13 = vfmacc_vv_f32m1(vs13, w1, r0, vlm1); + vs23 = vfmacc_vv_f32m1(vs23, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*4, vlm1); + vs04 = vfmacc_vv_f32m1(vs04, w0, r0, vlm1); + vs14 = vfmacc_vv_f32m1(vs14, w1, r0, vlm1); + vs24 = vfmacc_vv_f32m1(vs24, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*5, vlm1); + vs05 = vfmacc_vv_f32m1(vs05, w0, r0, vlm1); + vs15 = vfmacc_vv_f32m1(vs15, w1, r0, vlm1); + vs25 = vfmacc_vv_f32m1(vs25, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*6, vlm1); + vs06 = vfmacc_vv_f32m1(vs06, w0, r0, vlm1); + vs16 = vfmacc_vv_f32m1(vs16, w1, r0, vlm1); + vs26 = vfmacc_vv_f32m1(vs26, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*7, vlm1); + vs07 = vfmacc_vv_f32m1(vs07, w0, r0, vlm1); + vs17 = vfmacc_vv_f32m1(vs17, w1, r0, vlm1); + vs27 = vfmacc_vv_f32m1(vs27, w2, r0, vlm1); } - vfloat32m1_t s0, s1, s2; + // compute sum of each vs + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vlm1Max); + // vl is required here to be at least FASCONV_BASE_VECSZ, aka 8. + float32_t sum0[FASCONV_BASE_VECSZ], sum1[FASCONV_BASE_VECSZ], sum2[FASCONV_BASE_VECSZ]; + sum0[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs00, zero, vlm1Max)); + sum0[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs01, zero, vlm1Max)); + sum0[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs02, zero, vlm1Max)); + sum0[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs03, zero, vlm1Max)); + sum0[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs04, zero, vlm1Max)); + sum0[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs05, zero, vlm1Max)); + sum0[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs06, zero, vlm1Max)); + sum0[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs07, zero, vlm1Max)); + sum1[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs10, zero, vlm1Max)); + sum1[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs11, zero, vlm1Max)); + sum1[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs12, zero, vlm1Max)); + sum1[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs13, zero, vlm1Max)); + sum1[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs14, zero, vlm1Max)); + sum1[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs15, zero, vlm1Max)); + sum1[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs16, zero, vlm1Max)); + sum1[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs17, zero, vlm1Max)); + sum2[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs20, zero, vlm1Max)); + sum2[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs21, zero, vlm1Max)); + sum2[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs22, zero, vlm1Max)); + sum2[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs23, zero, vlm1Max)); + sum2[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs24, zero, vlm1Max)); + sum2[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs25, zero, vlm1Max)); + sum2[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs26, zero, vlm1Max)); + sum2[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs27, zero, vlm1Max)); + + // if VLEN = 128, so LMUL = 2 for vl = 8. + // otherwise, VLEN >=256, we only use fist 8 element of the vReg. + vfloat32m2_t s0, s1, s2; if( initOutput ) { - s0 = vfmv_v_f_f32m1(bias0, vl); - s1 = vfmv_v_f_f32m1(bias1, vl); - s2 = vfmv_v_f_f32m1(bias2, vl); + s0 = vfmv_v_f_f32m2(bias0, vl); + s1 = vfmv_v_f_f32m2(bias1, vl); + s2 = vfmv_v_f_f32m2(bias2, vl); } else { - s0 = vle32_v_f32m1(outptr0 + j, vl); - s1 = vle32_v_f32m1(outptr1 + j, vl); - s2 = vle32_v_f32m1(outptr2 + j, vl); + s0 = vle32_v_f32m2(outptr0 + j, vl); + s1 = vle32_v_f32m2(outptr1 + j, vl); + s2 = vle32_v_f32m2(outptr2 + j, vl); } - // compute sum of each vs - vfloat32m1_t zero = vfmv_v_f_f32m1(0, vl); - vfloat32m1_t temp00 = vfredsum_vs_f32m2_f32m1(temp00, vs00, zero, 8); - vfloat32m1_t temp01 = vfredsum_vs_f32m2_f32m1(temp01, vs01, zero, 8); - vfloat32m1_t temp02 = vfredsum_vs_f32m2_f32m1(temp02, vs02, zero, 8); - vfloat32m1_t temp03 = vfredsum_vs_f32m2_f32m1(temp03, vs03, zero, 8); - vfloat32m1_t temp10 = vfredsum_vs_f32m2_f32m1(temp10, vs10, zero, 8); - vfloat32m1_t temp11 = vfredsum_vs_f32m2_f32m1(temp11, vs11, zero, 8); - vfloat32m1_t temp12 = vfredsum_vs_f32m2_f32m1(temp12, vs12, zero, 8); - vfloat32m1_t temp13 = vfredsum_vs_f32m2_f32m1(temp13, vs13, zero, 8); - vfloat32m1_t temp20 = vfredsum_vs_f32m2_f32m1(temp20, vs20, zero, 8); - vfloat32m1_t temp21 = vfredsum_vs_f32m2_f32m1(temp21, vs21, zero, 8); - vfloat32m1_t temp22 = vfredsum_vs_f32m2_f32m1(temp22, vs22, zero, 8); - vfloat32m1_t temp23 = vfredsum_vs_f32m2_f32m1(temp23, vs23, zero, 8); - float32_t sum0[4], sum1[4], sum2[4]; - sum0[0] = vfmv_f_s_f32m1_f32(temp00); - sum0[1] = vfmv_f_s_f32m1_f32(temp01); - sum0[2] = vfmv_f_s_f32m1_f32(temp02); - sum0[3] = vfmv_f_s_f32m1_f32(temp03); - sum1[0] = vfmv_f_s_f32m1_f32(temp10); - sum1[1] = vfmv_f_s_f32m1_f32(temp11); - sum1[2] = vfmv_f_s_f32m1_f32(temp12); - sum1[3] = vfmv_f_s_f32m1_f32(temp13); - sum2[0] = vfmv_f_s_f32m1_f32(temp20); - sum2[1] = vfmv_f_s_f32m1_f32(temp21); - sum2[2] = vfmv_f_s_f32m1_f32(temp22); - sum2[3] = vfmv_f_s_f32m1_f32(temp23); - - s0 = vfadd_vv_f32m1(vle32_v_f32m1(sum0, vl), s0, vl); - s1 = vfadd_vv_f32m1(vle32_v_f32m1(sum1, vl), s1, vl); - s2 = vfadd_vv_f32m1(vle32_v_f32m1(sum2, vl), s2, vl); - + s0 = vfadd_vv_f32m2(vle32_v_f32m2(sum0, vl), s0, vl); + s1 = vfadd_vv_f32m2(vle32_v_f32m2(sum1, vl), s1, vl); + s2 = vfadd_vv_f32m2(vle32_v_f32m2(sum2, vl), s2, vl); if( relu ) { - vbool32_t m0 = vmfgt_vf_f32m1_b32(s0, 0, vl); - vbool32_t m1 = vmfgt_vf_f32m1_b32(s1, 0, vl); - vbool32_t m2 = vmfgt_vf_f32m1_b32(s2, 0, vl); - s0 = vmerge_vvm_f32m1(m0, vfmul_vv_f32m1(s0, vr0, vl), s0, vl); - s1 = vmerge_vvm_f32m1(m1, vfmul_vv_f32m1(s1, vr1, vl), s1, vl); - s2 = vmerge_vvm_f32m1(m2, vfmul_vv_f32m1(s2, vr2, vl), s2, vl); + vfloat32m2_t vr0 = vfmv_v_f_f32m2(1, vl), vr1 = vfmv_v_f_f32m2(1, vl), vr2 = vfmv_v_f_f32m2(1, vl); + float r0 = relu[i], r1 = relu[i+1], r2 = relu[i+2]; + if( i+2 >= outCn ) + { + r2 = r1; + if( i+1 >= outCn ) + r2 = r1 = r0; + } + vr0 = vfmv_v_f_f32m2(r0, vl); + vr1 = vfmv_v_f_f32m2(r1, vl); + vr2 = vfmv_v_f_f32m2(r2, vl); + vbool16_t m0 = vmfgt_vf_f32m2_b16(s0, 0, vl); + vbool16_t m1 = vmfgt_vf_f32m2_b16(s1, 0, vl); + vbool16_t m2 = vmfgt_vf_f32m2_b16(s2, 0, vl); + s0 = vmerge_vvm_f32m2(m0, vfmul_vv_f32m2(s0, vr0, vl), s0, vl); + s1 = vmerge_vvm_f32m2(m1, vfmul_vv_f32m2(s1, vr1, vl), s1, vl); + s2 = vmerge_vvm_f32m2(m2, vfmul_vv_f32m2(s2, vr2, vl), s2, vl); } if( tail ) { - s0 = vmerge_vvm_f32m1(mask, vle32_v_f32m1(outptr0 + j, vl), s0, vl); - s1 = vmerge_vvm_f32m1(mask, vle32_v_f32m1(outptr1 + j, vl), s1, vl); - s2 = vmerge_vvm_f32m1(mask, vle32_v_f32m1(outptr2 + j, vl), s2, vl); + int maskbuf[FASCONV_BASE_VECSZ] = {0}; + int rsz = blockSize % FASCONV_BASE_VECSZ; + for( int i = 0; i < rsz; i++ ) + maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1; + vint32m2_t vmaskbuf = vle32_v_i32m2(maskbuf ,vl); + vbool16_t mask = vmslt_vx_i32m2_b16(vmaskbuf, 0, vl); // mask for tail + s0 = vmerge_vvm_f32m2(mask, vle32_v_f32m2(outptr0 + j, vl), s0, vl); + s1 = vmerge_vvm_f32m2(mask, vle32_v_f32m2(outptr1 + j, vl), s1, vl); + s2 = vmerge_vvm_f32m2(mask, vle32_v_f32m2(outptr2 + j, vl), s2, vl); } - vse32_v_f32m1(outptr0 + j, s0, vl); - vse32_v_f32m1(outptr1 + j, s1, vl); - vse32_v_f32m1(outptr2 + j, s2, vl); + vse32_v_f32m2(outptr0 + j, s0, vl); + vse32_v_f32m2(outptr1 + j, s1, vl); + vse32_v_f32m2(outptr2 + j, s2, vl); } } } @@ -1097,23 +1163,27 @@ Example for load_deinterleave: output: a = {1, 3, 5, 7, 9, 11, 13, 15} output: b = {2, 4, 6, 8,10, 12, 14, 16} */ -static inline void vfloat32m2_load_deinterleave(const float* ptr, vfloat32m2_t& a, vfloat32m2_t& b) +static inline void vfloat32m2_load_deinterleave(const float* ptr, vfloat32m2_t& a, vfloat32m2_t& b, int vl) { - int vl = 8; - uint32_t masks[] = {1,1,1,1,0,0,0,0}; - vuint32m2_t vm = vle32_v_u32m2(masks,vl); - vbool16_t mask01 = vmseq_vx_u32m2_b16 (vm, 0, vl); - vbool16_t mask10 = vmseq_vx_u32m2_b16 (vm, 1, vl); - vfloat32m2_t ta = vle32_v_f32m2(ptr, vl), tb = vle32_v_f32m2(ptr+8, vl); - uint idx[] = {0,2,4,6,1,3,5,7}; - uint idxa[] = {0,0,0,0,0,1,2,3}, idxb[] = {4,5,6,7,0,0,0,0}; - vuint32m2_t vidxa = vle32_v_u32m2(idxa, 8), vidxb = vle32_v_u32m2(idxb, 8); - vuint32m2_t vidx = vle32_v_u32m2(idx, 8); - vfloat32m2_t high = vfmv_v_f_f32m2(0, 8), low = vfmv_v_f_f32m2(0, 8); - high = vrgather_vv_f32m2(ta, vidx, 8); - low = vrgather_vv_f32m2(tb, vidx, 8); - a = vrgather_vv_f32m2_m(mask01, high, low, vidxa, 8); - b = vrgather_vv_f32m2_m(mask10, low, high, vidxb, 8); + vuint64m4_t mask = vmv_v_x_u64m4(1,vl*2); + vuint32m4_t mask_re = vreinterpret_v_u64m4_u32m4(mask); + vbool8_t mask0 = vmseq_vx_u32m4_b8 (mask_re, 1, vl*2); + vbool8_t mask1 = vmseq_vx_u32m4_b8 (mask_re, 0, vl*2); + vfloat32m4_t tempa = vundefined_f32m4(), tempb = vundefined_f32m4(); + vfloat32m4_t vw = vle32_v_f32m4(ptr, vl*2); + tempa = vcompress_vm_f32m4(mask0, tempa, vw, vl*2); + tempb = vcompress_vm_f32m4(mask1, tempb, vw, vl*2); + /* The following instructions have not to be supported by the GNU toolchain. + So we temporarily use store and load instead. + // a = vlmul_trunc_v_f32m4_f32m2(tempa); + // b = vlmul_trunc_v_f32m4_f32m2(tempb); + */ + cv::AutoBuffer cvBuffer(sizeof(float32_t)*vl*2); + float* buffer = (float*)cvBuffer.data(); + vse32_v_f32m4(buffer, tempa, vl); + a = vle32_v_f32m2(buffer, vl); + vse32_v_f32m4(buffer, tempb, vl); + b = vle32_v_f32m2(buffer, vl); } void fastDepthwiseConv( const float* wptr, @@ -1127,7 +1197,7 @@ void fastDepthwiseConv( const float* wptr, float* outptr_, int out_d, int outH, int outW ) { - int vl = 8; + int vl = vsetvlmax_e32m2(); const float w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2], w10 = wptr[3], w11 = wptr[4], w12 = wptr[5], w20_ = wptr[6], w21_ = wptr[7], w22_ = wptr[8]; @@ -1166,17 +1236,11 @@ void fastDepthwiseConv( const float* wptr, if (stride_w == 1 || (stride_w == 2 && dilation_w == 1)) { - const int VECSZ = 8; - vfloat32m2_t vw00 = vfmv_v_f_f32m2(w00, vl), vw01 = vfmv_v_f_f32m2(w01, vl), vw02 = vfmv_v_f_f32m2(w02, vl), - vw10 = vfmv_v_f_f32m2(w10, vl), vw11 = vfmv_v_f_f32m2(w11, vl), vw12 = vfmv_v_f_f32m2(w12, vl), - vw20 = vfmv_v_f_f32m2(w20, vl), vw21 = vfmv_v_f_f32m2(w21, vl), vw22 = vfmv_v_f_f32m2(w22, vl); - vfloat32m2_t vbias = vfmv_v_f_f32m2(bias, vl), vrc = vfmv_v_f_f32m2(relu_coeff, vl); - if( stride_w == 1 ) - for( ; out_j < outW1; out_j += VECSZ ) + for( ; out_j < outW1; out_j += vl ) { - if (out_j + VECSZ > outW1 && out_j > pad_l) - out_j = outW1 - VECSZ; + if (out_j + vl > outW1) + vl = outW1 - out_j; int in_j = out_j * stride_w - pad_l; vfloat32m2_t v00 = vle32_v_f32m2(imgptr0 + in_j, vl), v01 = vle32_v_f32m2(imgptr0 + in_j + dilation_w, vl), @@ -1188,57 +1252,59 @@ void fastDepthwiseConv( const float* wptr, v21 = vle32_v_f32m2(imgptr2 + in_j + dilation_w, vl), v22 = vle32_v_f32m2(imgptr2 + in_j + dilation_w*2, vl); - vfloat32m2_t vout0 = vfmacc_vv_f32m2(vbias, v00, vw00, vl); - vfloat32m2_t vout1 = vfmul_vv_f32m2(v01, vw01, vl); - vfloat32m2_t vout2 = vfmul_vv_f32m2(v02, vw02, vl); + vfloat32m2_t vout0 = vfmul_vf_f32m2(v00, w00, vl); + vfloat32m2_t vout1 = vfmul_vf_f32m2(v01, w01, vl); + vfloat32m2_t vout2 = vfmul_vf_f32m2(v02, w02, vl); + vout0 = vfadd_vf_f32m2(vout0, bias, vl); - vout0 = vfmacc_vv_f32m2(vout0, v10, vw10, vl); - vout1 = vfmacc_vv_f32m2(vout1, v11, vw11, vl); - vout2 = vfmacc_vv_f32m2(vout2, v12, vw12, vl); + vout0 = vfmacc_vf_f32m2(vout0, w10, v10, vl); + vout1 = vfmacc_vf_f32m2(vout1, w11, v11, vl); + vout2 = vfmacc_vf_f32m2(vout2, w12, v12, vl); - vout0 = vfmacc_vv_f32m2(vout0, v20, vw20, vl); - vout1 = vfmacc_vv_f32m2(vout1, v21, vw21, vl); - vout2 = vfmacc_vv_f32m2(vout2, v22, vw22, vl); + vout0 = vfmacc_vf_f32m2(vout0, w20, v20, vl); + vout1 = vfmacc_vf_f32m2(vout1, w21, v21, vl); + vout2 = vfmacc_vf_f32m2(vout2, w22, v22, vl); vout0 = vfadd_vv_f32m2(vfadd_vv_f32m2(vout0, vout1, vl), vout2, vl); if (relu) { vbool16_t m = vmfgt_vf_f32m2_b16(vout0, 0, vl); - vout0 = vmerge_vvm_f32m2(m, vfmul_vv_f32m2(vout0, vrc, vl), vout0, vl); + vout0 = vmerge_vvm_f32m2(m, vfmul_vf_f32m2(vout0, relu_coeff, vl), vout0, vl); } vse32_v_f32m2(outptr + out_j, vout0, vl); } - else - for( ; out_j < outW1; out_j += VECSZ ) + else //stride_w == 2 && dilation_w == 1 + for( ; out_j < outW1; out_j += vl ) { - if (out_j + VECSZ > outW1 && out_j > pad_l) - out_j = outW1 - VECSZ; + if (out_j + vl > outW1) + vl = outW1 - out_j; int in_j = out_j * stride_w - pad_l; vfloat32m2_t v00, v01, v02, v10, v11, v12, v20, v21, v22, unused; - vfloat32m2_load_deinterleave(imgptr0 + in_j, v00, v01); - vfloat32m2_load_deinterleave(imgptr0 + in_j + 2, v02, unused); - vfloat32m2_load_deinterleave(imgptr1 + in_j, v10, v11); - vfloat32m2_load_deinterleave(imgptr1 + in_j + 2, v12, unused); - vfloat32m2_load_deinterleave(imgptr2 + in_j, v20, v21); - vfloat32m2_load_deinterleave(imgptr2 + in_j + 2, v22, unused); - - vfloat32m2_t vout0 = vfmacc_vv_f32m2(vbias, v00, vw00, vl); - vfloat32m2_t vout1 = vfmul_vv_f32m2(v01, vw01, vl); - vfloat32m2_t vout2 = vfmul_vv_f32m2(v02, vw02, vl); - - vout0 = vfmacc_vv_f32m2(vout0, v10, vw10, vl); - vout1 = vfmacc_vv_f32m2(vout1, v11, vw11, vl); - vout2 = vfmacc_vv_f32m2(vout2, v12, vw12, vl); - - vout0 = vfmacc_vv_f32m2(vout0, v20, vw20, vl); - vout1 = vfmacc_vv_f32m2(vout1, v21, vw21, vl); - vout2 = vfmacc_vv_f32m2(vout2, v22, vw22, vl); + vfloat32m2_load_deinterleave(imgptr0 + in_j, v00, v01, vl); + vfloat32m2_load_deinterleave(imgptr0 + in_j + 2, v02, unused, vl); + vfloat32m2_load_deinterleave(imgptr1 + in_j, v10, v11, vl); + vfloat32m2_load_deinterleave(imgptr1 + in_j + 2, v12, unused, vl); + vfloat32m2_load_deinterleave(imgptr2 + in_j, v20, v21, vl); + vfloat32m2_load_deinterleave(imgptr2 + in_j + 2, v22, unused, vl); + + vfloat32m2_t vout0 = vfmul_vf_f32m2(v00, w00, vl); + vfloat32m2_t vout1 = vfmul_vf_f32m2(v01, w01, vl); + vfloat32m2_t vout2 = vfmul_vf_f32m2(v02, w02, vl); + vout0 = vfadd_vf_f32m2(vout0, bias, vl); + + vout0 = vfmacc_vf_f32m2(vout0, w10, v10, vl); + vout1 = vfmacc_vf_f32m2(vout1, w11, v11, vl); + vout2 = vfmacc_vf_f32m2(vout2, w12, v12, vl); + + vout0 = vfmacc_vf_f32m2(vout0, w20, v20, vl); + vout1 = vfmacc_vf_f32m2(vout1, w21, v21, vl); + vout2 = vfmacc_vf_f32m2(vout2, w22, v22, vl); vout0 = vfadd_vv_f32m2(vfadd_vv_f32m2(vout0, vout1, vl), vout2, vl); if (relu) { vbool16_t m = vmfgt_vf_f32m2_b16(vout0, 0, vl); - vout0 = vmerge_vvm_f32m2(m, vfmul_vv_f32m2(vout0, vrc, vl), vout0, vl); + vout0 = vmerge_vvm_f32m2(m, vfmul_vf_f32m2(vout0, relu_coeff, vl), vout0, vl); } vse32_v_f32m2(outptr + out_j, vout0, vl); } From 646924fce8422647b42bfaa5e7b2a83cda7ecf6d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 5 Oct 2021 23:21:27 +0000 Subject: [PATCH 259/376] dnn(pytest/test_input_3d): reload model between switching targets --- modules/dnn/misc/python/test/test_dnn.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/dnn/misc/python/test/test_dnn.py b/modules/dnn/misc/python/test/test_dnn.py index f7bfc0111940..9ea93d3fe1d4 100644 --- a/modules/dnn/misc/python/test/test_dnn.py +++ b/modules/dnn/misc/python/test/test_dnn.py @@ -323,20 +323,22 @@ def test_input_3d(self): raise unittest.SkipTest("Missing DNN test files (dnn/onnx/data/{input/output}_hidden_lstm.npy). " "Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.") - net = cv.dnn.readNet(model) input = np.load(input_file) # we have to expand the shape of input tensor because Python bindings cut 3D tensors to 2D # it should be fixed in future. see : https://github.com/opencv/opencv/issues/19091 # please remove `expand_dims` after that input = np.expand_dims(input, axis=3) gold_output = np.load(output_file) - net.setInput(input) for backend, target in self.dnnBackendsAndTargets: printParams(backend, target) + net = cv.dnn.readNet(model) + net.setPreferableBackend(backend) net.setPreferableTarget(target) + + net.setInput(input) real_output = net.forward() normAssert(self, real_output, gold_output, "", getDefaultThreshold(target)) From 755e0143fb8c40fee082b8343a2b3ce383c47a0b Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 6 Oct 2021 15:05:45 +0300 Subject: [PATCH 260/376] Merge pull request #20815 from alalek:fix_20649_revert_19859 features2d: repair SimpleBlobDetector * features2d: revert code change by PR #19859 Reverted commit 76860933f0a31c0abd1b26c1f11b25972cda031e * features2d: check SimpleBlobDetector parameters consistency --- modules/features2d/src/blobdetector.cpp | 23 +++++++++++++------ modules/features2d/test/test_blobdetector.cpp | 1 + 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/modules/features2d/src/blobdetector.cpp b/modules/features2d/src/blobdetector.cpp index c2215cd57c70..fb79831807be 100644 --- a/modules/features2d/src/blobdetector.cpp +++ b/modules/features2d/src/blobdetector.cpp @@ -44,6 +44,8 @@ #include #include +#include + // Requires CMake flag: DEBUG_opencv_features2d=ON //#define DEBUG_BLOB_DETECTOR @@ -317,6 +319,19 @@ void SimpleBlobDetectorImpl::detect(InputArray image, std::vector& CV_Error(Error::StsUnsupportedFormat, "Blob detector only supports 8-bit images!"); } + CV_CheckGT(params.thresholdStep, 0.0f, ""); + if (params.minThreshold + params.thresholdStep >= params.maxThreshold) + { + // https://github.com/opencv/opencv/issues/6667 + CV_LOG_ONCE_INFO(NULL, "SimpleBlobDetector: params.minDistBetweenBlobs is ignored for case with single threshold"); +#if 0 // OpenCV 5.0 + CV_CheckEQ(params.minRepeatability, 1u, "Incompatible parameters for case with single threshold"); +#else + if (params.minRepeatability != 1) + CV_LOG_WARNING(NULL, "SimpleBlobDetector: params.minRepeatability=" << params.minRepeatability << " is incompatible for case with single threshold. Empty result is expected."); +#endif + } + std::vector < std::vector

> centers; for (double thresh = params.minThreshold; thresh < params.maxThreshold; thresh += params.thresholdStep) { @@ -325,19 +340,13 @@ void SimpleBlobDetectorImpl::detect(InputArray image, std::vector& std::vector < Center > curCenters; findBlobs(grayscaleImage, binarizedImage, curCenters); - if(params.maxThreshold - params.minThreshold <= params.thresholdStep) { - // if the difference between min and max threshold is less than the threshold step - // we're only going to enter the loop once, so we need to add curCenters - // to ensure we still use minDistBetweenBlobs - centers.push_back(curCenters); - } std::vector < std::vector
> newCenters; for (size_t i = 0; i < curCenters.size(); i++) { bool isNew = true; for (size_t j = 0; j < centers.size(); j++) { - double dist = norm(centers[j][centers[j].size() / 2 ].location - curCenters[i].location); + double dist = norm(centers[j][ centers[j].size() / 2 ].location - curCenters[i].location); isNew = dist >= params.minDistBetweenBlobs && dist >= centers[j][ centers[j].size() / 2 ].radius && dist >= curCenters[i].radius; if (!isNew) { diff --git a/modules/features2d/test/test_blobdetector.cpp b/modules/features2d/test/test_blobdetector.cpp index 56b714586205..c6ed09e6a279 100644 --- a/modules/features2d/test/test_blobdetector.cpp +++ b/modules/features2d/test/test_blobdetector.cpp @@ -12,6 +12,7 @@ TEST(Features2d_BlobDetector, bug_6667) SimpleBlobDetector::Params params; params.minThreshold = 250; params.maxThreshold = 260; + params.minRepeatability = 1; // https://github.com/opencv/opencv/issues/6667 std::vector keypoints; Ptr detector = SimpleBlobDetector::create(params); From eb56ca3b0d7dec26603f45b9d9e63fc186e27eaa Mon Sep 17 00:00:00 2001 From: Giles Payne Date: Wed, 6 Oct 2021 21:06:30 +0900 Subject: [PATCH 261/376] Fix build on older Xcode versions --- modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm | 4 ++++ modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm index 7bfee07eb131..84cf25879bfd 100644 --- a/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm +++ b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm @@ -22,11 +22,15 @@ } static UIFont* getAnySerif() { +#if defined(__IPHONE_OS_VERSION_MAX_ALLOWED) && __IPHONE_OS_VERSION_MAX_ALLOWED >= 130000 if (@available(iOS 13.0, *)) { return [UIFont fontWithDescriptor:[[UIFontDescriptor preferredFontDescriptorWithTextStyle:UIFontTextStyleBody] fontDescriptorWithDesign:UIFontDescriptorSystemDesignSerif] size:SIZE]; } else { return nil; } +#else + return nil; +#endif } static UIFont* getSystemFont() { diff --git a/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm index 6775f817806c..bdc5eda81158 100644 --- a/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm +++ b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm @@ -22,11 +22,15 @@ } static NSFont* getAnySerif() { +#if defined(__MAC_OS_X_VERSION_MAX_ALLOWED) && __MAC_OS_X_VERSION_MAX_ALLOWED >= 110000 if (@available(macOS 11.0, *)) { return [NSFont fontWithDescriptor:[[NSFontDescriptor preferredFontDescriptorForTextStyle:NSFontTextStyleBody options:@{}] fontDescriptorWithDesign:NSFontDescriptorSystemDesignSerif] size:SIZE]; } else { return nil; } +#else + return nil; +#endif } static NSFont* getSystemFont() { From a3d7811f240c19c2c07640be45dcf16a2b947248 Mon Sep 17 00:00:00 2001 From: Oliver Kuckertz Date: Wed, 6 Oct 2021 18:41:05 +0200 Subject: [PATCH 262/376] Merge pull request #20725 from mologie:fix-dnn-tf-on-arm * dnn: fix unaligned memory access crash on armv7 The getTensorContent function would return a Mat pointing to some member of a Protobuf-encoded message. Protobuf does not make any alignment guarantees, which results in a crash on armv7 when loading models while bit 2 is set in /proc/cpu/alignment (or the relevant kernel feature for alignment compatibility is disabled). Any read attempt from the previously unaligned data member would send SIGBUS. As workaround, this commit makes an aligned copy via existing clone functionality in getTensorContent. The unsafe copy=false option is removed. Unfortunately, a rather crude hack in PReLUSubgraph in fact writes(!) to the Protobuf message. We limit ourselves to fixing the alignment issues in this commit, and add getTensorContentRefUnaligned to cover the write case with a safe memcpy. A FIXME marks the issue. * dnn: reduce amount of .clone() calls * dnn: update FIXME comment Co-authored-by: Alexander Alekhin --- .../src/tensorflow/tf_graph_simplifier.cpp | 41 +++++++++++++++++-- .../src/tensorflow/tf_graph_simplifier.hpp | 2 +- modules/dnn/src/tensorflow/tf_importer.cpp | 4 ++ 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp b/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp index 354fef029762..54395504c79a 100644 --- a/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp +++ b/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp @@ -19,6 +19,16 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN using ::google::protobuf::RepeatedField; using ::google::protobuf::MapPair; +static Mat getTensorContentRef_(const tensorflow::TensorProto& tensor); +static inline +bool isAlignedMat(const Mat& m) +{ + int depth = m.depth(); + int alignment = CV_ELEM_SIZE1(depth); + return (((size_t)m.data) & (alignment - 1)) == 0; +} + + class TFNodeWrapper : public ImportNodeWrapper { public: @@ -719,8 +729,19 @@ class PReLUSubgraph : public TFSubgraph { if (!negativeScales) { - Mat scales = getTensorContent(inputNodes[1]->attr().at("value").tensor(), /*copy*/false); - scales *= -1; + Mat scalesRef = getTensorContentRef_(inputNodes[1]->attr().at("value").tensor()); + // FIXME: This breaks the const guarantees of tensor() by writing to scalesRef + if (isAlignedMat(scalesRef)) + { + scalesRef *= -1; + } + else + { + Mat scales = scalesRef.clone() * -1; + CV_Assert(scalesRef.isContinuous()); + CV_Assert(scales.isContinuous()); + memcpy(scalesRef.data, scales.data, scales.total() * scales.elemSize()); + } } } @@ -832,7 +853,8 @@ void RemoveIdentityOps(tensorflow::GraphDef& net) } } -Mat getTensorContent(const tensorflow::TensorProto &tensor, bool copy) +// NB: returned Mat::data pointer may be unaligned +Mat getTensorContentRef_(const tensorflow::TensorProto& tensor) { const std::string& content = tensor.tensor_content(); Mat m; @@ -904,7 +926,18 @@ Mat getTensorContent(const tensorflow::TensorProto &tensor, bool copy) CV_Error(Error::StsError, "Tensor's data type is not supported"); break; } - return copy ? m.clone() : m; + + return m; +} + +Mat getTensorContent(const tensorflow::TensorProto& tensor, bool forceCopy) +{ + // If necessary clone m to have aligned data pointer + Mat m = getTensorContentRef_(tensor); + if (forceCopy || !isAlignedMat(m)) + return m.clone(); + else + return m; } void releaseTensor(tensorflow::TensorProto* tensor) diff --git a/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp b/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp index 55f36cdb44cb..4f3dfa870d7f 100644 --- a/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp +++ b/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp @@ -21,7 +21,7 @@ void RemoveIdentityOps(tensorflow::GraphDef& net); void simplifySubgraphs(tensorflow::GraphDef& net); -Mat getTensorContent(const tensorflow::TensorProto &tensor, bool copy = true); +Mat getTensorContent(const tensorflow::TensorProto& tensor, bool forceCopy = true); void releaseTensor(tensorflow::TensorProto* tensor); diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 521c8ce4c384..16c5c163089f 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -122,8 +122,10 @@ void parseTensor(const tensorflow::TensorProto &tensor, Mat &dstBlob) } dstBlob.create(shape, CV_32F); + CV_Assert(dstBlob.isContinuous()); Mat tensorContent = getTensorContent(tensor, /*no copy*/false); + CV_Assert(tensorContent.isContinuous()); int size = tensorContent.total(); CV_Assert(size == (int)dstBlob.total()); @@ -2522,8 +2524,10 @@ void TFImporter::kernelFromTensor(const tensorflow::TensorProto &tensor, Mat &ds out_c = shape[0]; input_c = shape[1]; dstBlob.create(shape, CV_32F); + CV_Assert(dstBlob.isContinuous()); Mat tensorContent = getTensorContent(tensor, /*no copy*/false); + CV_Assert(tensorContent.isContinuous()); int size = tensorContent.total(); CV_Assert(size == (int)dstBlob.total()); From c6a6f39d29aec1a8f355abe8435b38d15b7f43ba Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 6 Oct 2021 16:49:02 +0000 Subject: [PATCH 263/376] android: drop sourceCompatibility/targetCompatibility options --- modules/java/android_sdk/android_gradle_lib/build.gradle | 4 ---- modules/java/android_sdk/build.gradle.in | 4 ---- 2 files changed, 8 deletions(-) diff --git a/modules/java/android_sdk/android_gradle_lib/build.gradle b/modules/java/android_sdk/android_gradle_lib/build.gradle index 3966c9def6cd..41cdb9e5e10b 100644 --- a/modules/java/android_sdk/android_gradle_lib/build.gradle +++ b/modules/java/android_sdk/android_gradle_lib/build.gradle @@ -36,10 +36,6 @@ android { proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.txt' } } - compileOptions { - sourceCompatibility JavaVersion.VERSION_1_6 - targetCompatibility JavaVersion.VERSION_1_6 - } sourceSets { main { diff --git a/modules/java/android_sdk/build.gradle.in b/modules/java/android_sdk/build.gradle.in index 426823187343..90f0e88fd736 100644 --- a/modules/java/android_sdk/build.gradle.in +++ b/modules/java/android_sdk/build.gradle.in @@ -128,10 +128,6 @@ android { proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.txt' } } - compileOptions { - sourceCompatibility JavaVersion.VERSION_1_6 - targetCompatibility JavaVersion.VERSION_1_6 - } sourceSets { main { From c49cfefe88633d73aed53061bb7b345445c74546 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 6 Oct 2021 17:25:32 +0000 Subject: [PATCH 264/376] videoio: fix plugins handling if no filesystem available --- modules/videoio/CMakeLists.txt | 7 ++- modules/videoio/src/cap.cpp | 12 ++++- modules/videoio/src/videoio_registry.cpp | 62 +++++++++++++----------- 3 files changed, 50 insertions(+), 31 deletions(-) diff --git a/modules/videoio/CMakeLists.txt b/modules/videoio/CMakeLists.txt index 534fcf0e370b..98127d1c9871 100644 --- a/modules/videoio/CMakeLists.txt +++ b/modules/videoio/CMakeLists.txt @@ -1,5 +1,10 @@ +set(VIDEOIO_ENABLE_PLUGINS_DEFAULT ON) +if(EMSCRIPTEN OR IOS OR WINRT) + set(VIDEOIO_ENABLE_PLUGINS_DEFAULT OFF) +endif() + set(VIDEOIO_PLUGIN_LIST "" CACHE STRING "List of videoio backends to be compiled as plugins (ffmpeg, gstreamer, mfx, msmf or special value 'all')") -set(VIDEOIO_ENABLE_PLUGINS "ON" CACHE BOOL "Allow building and using of videoio plugins") +set(VIDEOIO_ENABLE_PLUGINS "${VIDEOIO_ENABLE_PLUGINS_DEFAULT}" CACHE BOOL "Allow building and using of videoio plugins") mark_as_advanced(VIDEOIO_PLUGIN_LIST VIDEOIO_ENABLE_PLUGINS) string(REPLACE "," ";" VIDEOIO_PLUGIN_LIST "${VIDEOIO_PLUGIN_LIST}") # support comma-separated list (,) too diff --git a/modules/videoio/src/cap.cpp b/modules/videoio/src/cap.cpp index f8967514edb3..fa958e2c8f38 100644 --- a/modules/videoio/src/cap.cpp +++ b/modules/videoio/src/cap.cpp @@ -122,7 +122,11 @@ bool VideoCapture::open(const String& filename, int apiPreference, const std::ve const VideoBackendInfo& info = backends[i]; if (apiPreference == CAP_ANY || apiPreference == info.id) { - + if (!info.backendFactory) + { + CV_LOG_DEBUG(NULL, "VIDEOIO(" << info.name << "): factory is not available (plugins require filesystem support)"); + continue; + } CV_CAPTURE_LOG_DEBUG(NULL, cv::format("VIDEOIO(%s): trying capture filename='%s' ...", info.name, filename.c_str())); @@ -232,10 +236,14 @@ bool VideoCapture::open(int cameraNum, int apiPreference, const std::vector const VideoBackendInfo& info = backends[i]; if (apiPreference == CAP_ANY || apiPreference == info.id) { + if (!info.backendFactory) + { + CV_LOG_DEBUG(NULL, "VIDEOIO(" << info.name << "): factory is not available (plugins require filesystem support)"); + continue; + } CV_CAPTURE_LOG_DEBUG(NULL, cv::format("VIDEOIO(%s): trying capture cameraNum=%d ...", info.name, cameraNum)); - CV_Assert(!info.backendFactory.empty()); const Ptr backend = info.backendFactory->getBackend(); if (!backend.empty()) diff --git a/modules/videoio/src/videoio_registry.cpp b/modules/videoio/src/videoio_registry.cpp index 61e3ac0724c8..6cd32edc273c 100644 --- a/modules/videoio/src/videoio_registry.cpp +++ b/modules/videoio/src/videoio_registry.cpp @@ -8,6 +8,8 @@ #include "opencv2/videoio/registry.hpp" +#include "opencv2/core/utils/filesystem.private.hpp" // OPENCV_HAVE_FILESYSTEM_SUPPORT + #include "cap_librealsense.hpp" #include "cap_dshow.hpp" @@ -34,15 +36,19 @@ namespace cv { namespace { +#if OPENCV_HAVE_FILESYSTEM_SUPPORT && defined(ENABLE_PLUGINS) #define DECLARE_DYNAMIC_BACKEND(cap, name, mode) \ { \ cap, (BackendMode)(mode), 1000, name, createPluginBackendFactory(cap, name) \ -} +}, +#else +#define DECLARE_DYNAMIC_BACKEND(cap, name, mode) /* nothing */ +#endif #define DECLARE_STATIC_BACKEND(cap, name, mode, createCaptureFile, createCaptureCamera, createWriter) \ { \ cap, (BackendMode)(mode), 1000, name, createBackendFactory(createCaptureFile, createCaptureCamera, createWriter) \ -} +}, /** Ordering guidelines: - modern optimized, multi-platform libraries: ffmpeg, gstreamer, Media SDK @@ -55,90 +61,90 @@ namespace { static const struct VideoBackendInfo builtin_backends[] = { #ifdef HAVE_FFMPEG - DECLARE_STATIC_BACKEND(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, cvCreateFileCapture_FFMPEG_proxy, 0, cvCreateVideoWriter_FFMPEG_proxy), + DECLARE_STATIC_BACKEND(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, cvCreateFileCapture_FFMPEG_proxy, 0, cvCreateVideoWriter_FFMPEG_proxy) #elif defined(ENABLE_PLUGINS) || defined(HAVE_FFMPEG_WRAPPER) - DECLARE_DYNAMIC_BACKEND(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER), + DECLARE_DYNAMIC_BACKEND(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER) #endif #ifdef HAVE_GSTREAMER - DECLARE_STATIC_BACKEND(CAP_GSTREAMER, "GSTREAMER", MODE_CAPTURE_ALL | MODE_WRITER, createGStreamerCapture_file, createGStreamerCapture_cam, create_GStreamer_writer), + DECLARE_STATIC_BACKEND(CAP_GSTREAMER, "GSTREAMER", MODE_CAPTURE_ALL | MODE_WRITER, createGStreamerCapture_file, createGStreamerCapture_cam, create_GStreamer_writer) #elif defined(ENABLE_PLUGINS) - DECLARE_DYNAMIC_BACKEND(CAP_GSTREAMER, "GSTREAMER", MODE_CAPTURE_ALL | MODE_WRITER), + DECLARE_DYNAMIC_BACKEND(CAP_GSTREAMER, "GSTREAMER", MODE_CAPTURE_ALL | MODE_WRITER) #endif #ifdef HAVE_MFX // Media SDK - DECLARE_STATIC_BACKEND(CAP_INTEL_MFX, "INTEL_MFX", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, create_MFX_capture, 0, create_MFX_writer), + DECLARE_STATIC_BACKEND(CAP_INTEL_MFX, "INTEL_MFX", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, create_MFX_capture, 0, create_MFX_writer) #elif defined(ENABLE_PLUGINS) - DECLARE_DYNAMIC_BACKEND(CAP_INTEL_MFX, "INTEL_MFX", MODE_CAPTURE_BY_FILENAME | MODE_WRITER), + DECLARE_DYNAMIC_BACKEND(CAP_INTEL_MFX, "INTEL_MFX", MODE_CAPTURE_BY_FILENAME | MODE_WRITER) #endif // Apple platform #ifdef HAVE_AVFOUNDATION - DECLARE_STATIC_BACKEND(CAP_AVFOUNDATION, "AVFOUNDATION", MODE_CAPTURE_ALL | MODE_WRITER, create_AVFoundation_capture_file, create_AVFoundation_capture_cam, create_AVFoundation_writer), + DECLARE_STATIC_BACKEND(CAP_AVFOUNDATION, "AVFOUNDATION", MODE_CAPTURE_ALL | MODE_WRITER, create_AVFoundation_capture_file, create_AVFoundation_capture_cam, create_AVFoundation_writer) #endif // Windows #ifdef WINRT_VIDEO - DECLARE_STATIC_BACKEND(CAP_WINRT, "WINRT", MODE_CAPTURE_BY_INDEX, 0, create_WRT_capture, 0), + DECLARE_STATIC_BACKEND(CAP_WINRT, "WINRT", MODE_CAPTURE_BY_INDEX, 0, create_WRT_capture, 0) #endif #ifdef HAVE_MSMF - DECLARE_STATIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER, cvCreateCapture_MSMF, cvCreateCapture_MSMF, cvCreateVideoWriter_MSMF), + DECLARE_STATIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER, cvCreateCapture_MSMF, cvCreateCapture_MSMF, cvCreateVideoWriter_MSMF) #elif defined(ENABLE_PLUGINS) && defined(_WIN32) - DECLARE_DYNAMIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER), + DECLARE_DYNAMIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER) #endif #ifdef HAVE_DSHOW - DECLARE_STATIC_BACKEND(CAP_DSHOW, "DSHOW", MODE_CAPTURE_BY_INDEX, 0, create_DShow_capture, 0), + DECLARE_STATIC_BACKEND(CAP_DSHOW, "DSHOW", MODE_CAPTURE_BY_INDEX, 0, create_DShow_capture, 0) #endif // Linux, some Unix #if defined HAVE_CAMV4L2 - DECLARE_STATIC_BACKEND(CAP_V4L2, "V4L2", MODE_CAPTURE_ALL, create_V4L_capture_file, create_V4L_capture_cam, 0), + DECLARE_STATIC_BACKEND(CAP_V4L2, "V4L2", MODE_CAPTURE_ALL, create_V4L_capture_file, create_V4L_capture_cam, 0) #elif defined HAVE_VIDEOIO - DECLARE_STATIC_BACKEND(CAP_V4L, "V4L_BSD", MODE_CAPTURE_ALL, create_V4L_capture_file, create_V4L_capture_cam, 0), + DECLARE_STATIC_BACKEND(CAP_V4L, "V4L_BSD", MODE_CAPTURE_ALL, create_V4L_capture_file, create_V4L_capture_cam, 0) #endif // RGB-D universal #ifdef HAVE_OPENNI2 - DECLARE_STATIC_BACKEND(CAP_OPENNI2, "OPENNI2", MODE_CAPTURE_ALL, create_OpenNI2_capture_file, create_OpenNI2_capture_cam, 0), + DECLARE_STATIC_BACKEND(CAP_OPENNI2, "OPENNI2", MODE_CAPTURE_ALL, create_OpenNI2_capture_file, create_OpenNI2_capture_cam, 0) #endif #ifdef HAVE_LIBREALSENSE - DECLARE_STATIC_BACKEND(CAP_REALSENSE, "INTEL_REALSENSE", MODE_CAPTURE_BY_INDEX, 0, create_RealSense_capture, 0), + DECLARE_STATIC_BACKEND(CAP_REALSENSE, "INTEL_REALSENSE", MODE_CAPTURE_BY_INDEX, 0, create_RealSense_capture, 0) #endif // OpenCV file-based only - DECLARE_STATIC_BACKEND(CAP_IMAGES, "CV_IMAGES", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, create_Images_capture, 0, create_Images_writer), - DECLARE_STATIC_BACKEND(CAP_OPENCV_MJPEG, "CV_MJPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, createMotionJpegCapture, 0, createMotionJpegWriter), + DECLARE_STATIC_BACKEND(CAP_IMAGES, "CV_IMAGES", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, create_Images_capture, 0, create_Images_writer) + DECLARE_STATIC_BACKEND(CAP_OPENCV_MJPEG, "CV_MJPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, createMotionJpegCapture, 0, createMotionJpegWriter) // special interfaces / stereo cameras / other SDKs #if defined(HAVE_DC1394_2) - DECLARE_STATIC_BACKEND(CAP_FIREWIRE, "FIREWIRE", MODE_CAPTURE_BY_INDEX, 0, create_DC1394_capture, 0), + DECLARE_STATIC_BACKEND(CAP_FIREWIRE, "FIREWIRE", MODE_CAPTURE_BY_INDEX, 0, create_DC1394_capture, 0) #endif // GigE #ifdef HAVE_PVAPI - DECLARE_STATIC_BACKEND(CAP_PVAPI, "PVAPI", MODE_CAPTURE_BY_INDEX, 0, create_PvAPI_capture, 0), + DECLARE_STATIC_BACKEND(CAP_PVAPI, "PVAPI", MODE_CAPTURE_BY_INDEX, 0, create_PvAPI_capture, 0) #endif #ifdef HAVE_XIMEA - DECLARE_STATIC_BACKEND(CAP_XIAPI, "XIMEA", MODE_CAPTURE_ALL, create_XIMEA_capture_file, create_XIMEA_capture_cam, 0), + DECLARE_STATIC_BACKEND(CAP_XIAPI, "XIMEA", MODE_CAPTURE_ALL, create_XIMEA_capture_file, create_XIMEA_capture_cam, 0) #endif #ifdef HAVE_ARAVIS_API - DECLARE_STATIC_BACKEND(CAP_ARAVIS, "ARAVIS", MODE_CAPTURE_BY_INDEX, 0, create_Aravis_capture, 0), + DECLARE_STATIC_BACKEND(CAP_ARAVIS, "ARAVIS", MODE_CAPTURE_BY_INDEX, 0, create_Aravis_capture, 0) #endif #ifdef HAVE_UEYE // uEye - DECLARE_STATIC_BACKEND(CAP_UEYE, "UEYE", MODE_CAPTURE_BY_INDEX, 0, create_ueye_camera, 0), + DECLARE_STATIC_BACKEND(CAP_UEYE, "UEYE", MODE_CAPTURE_BY_INDEX, 0, create_ueye_camera, 0) #elif defined(ENABLE_PLUGINS) - DECLARE_DYNAMIC_BACKEND(CAP_UEYE, "UEYE", MODE_CAPTURE_BY_INDEX), + DECLARE_DYNAMIC_BACKEND(CAP_UEYE, "UEYE", MODE_CAPTURE_BY_INDEX) #endif #ifdef HAVE_GPHOTO2 - DECLARE_STATIC_BACKEND(CAP_GPHOTO2, "GPHOTO2", MODE_CAPTURE_ALL, createGPhoto2Capture, createGPhoto2Capture, 0), + DECLARE_STATIC_BACKEND(CAP_GPHOTO2, "GPHOTO2", MODE_CAPTURE_ALL, createGPhoto2Capture, createGPhoto2Capture, 0) #endif #ifdef HAVE_XINE - DECLARE_STATIC_BACKEND(CAP_XINE, "XINE", MODE_CAPTURE_BY_FILENAME, createXINECapture, 0, 0), + DECLARE_STATIC_BACKEND(CAP_XINE, "XINE", MODE_CAPTURE_BY_FILENAME, createXINECapture, 0, 0) #endif #if defined(HAVE_ANDROID_MEDIANDK) || defined(HAVE_ANDROID_NATIVE_CAMERA) DECLARE_STATIC_BACKEND(CAP_ANDROID, "ANDROID_NATIVE", @@ -163,7 +169,7 @@ static const struct VideoBackendInfo builtin_backends[] = #else 0, #endif - 0), + 0) #endif // dropped backends: MIL, TYZX }; From 2221dcc9f297072a0772d752c91397fcc7a6cdc3 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Tue, 5 Oct 2021 16:46:37 +0300 Subject: [PATCH 265/376] add SoftNMS implementation --- modules/dnn/include/opencv2/dnn/dnn.hpp | 33 +++++++++++ modules/dnn/src/nms.cpp | 77 +++++++++++++++++++++++++ modules/dnn/test/test_nms.cpp | 37 ++++++++++++ 3 files changed, 147 insertions(+) diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index bf1670051ac0..8b4e5e21d42a 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -1130,6 +1130,39 @@ CV__DNN_INLINE_NS_BEGIN CV_OUT std::vector& indices, const float eta = 1.f, const int top_k = 0); + /** + * @brief Enum of Soft NMS methods. + * @see softNMSBoxes + */ + enum class SoftNMSMethod + { + SOFTNMS_LINEAR = 1, + SOFTNMS_GAUSSIAN = 2 + }; + + /** @brief Performs soft non maximum suppression given boxes and corresponding scores. + * Reference: https://arxiv.org/abs/1704.04503 + * @param bboxes a set of bounding boxes to apply Soft NMS. + * @param scores a set of corresponding confidences. + * @param updated_scores a set of corresponding updated confidences. + * @param score_threshold a threshold used to filter boxes by score. + * @param nms_threshold a threshold used in non maximum suppression. + * @param indices the kept indices of bboxes after NMS. + * @param top_k keep at most @p top_k picked indices. + * @param sigma parameter of Gaussian weighting. + * @param method Gaussian or linear. + * @see SoftNMSMethod + */ + CV_EXPORTS_W void softNMSBoxes(const std::vector& bboxes, + const std::vector& scores, + CV_OUT std::vector& updated_scores, + const float score_threshold, + const float nms_threshold, + CV_OUT std::vector& indices, + size_t top_k = 0, + const float sigma = 0.5, + SoftNMSMethod method = SoftNMSMethod::SOFTNMS_GAUSSIAN); + /** @brief This class is presented high-level API for neural networks. * diff --git a/modules/dnn/src/nms.cpp b/modules/dnn/src/nms.cpp index 26489dc7e92a..d321cfed8853 100644 --- a/modules/dnn/src/nms.cpp +++ b/modules/dnn/src/nms.cpp @@ -58,6 +58,83 @@ void NMSBoxes(const std::vector& bboxes, const std::vector& NMSFast_(bboxes, scores, score_threshold, nms_threshold, eta, top_k, indices, rotatedRectIOU); } +void softNMSBoxes(const std::vector& bboxes, + const std::vector& scores, + std::vector& updated_scores, + const float score_threshold, + const float nms_threshold, + std::vector& indices, + size_t top_k, + const float sigma, + SoftNMSMethod method) +{ + CV_Assert_N(bboxes.size() == scores.size(), score_threshold >= 0, + nms_threshold >= 0, sigma >= 0); + + indices.clear(); + updated_scores.clear(); + + std::vector > score_index_vec(scores.size()); + for (size_t i = 0; i < scores.size(); i++) + { + score_index_vec[i].first = scores[i]; + score_index_vec[i].second = i; + } + + const auto score_cmp = [](const std::pair& a, const std::pair& b) + { + return a.first == b.first ? a.second > b.second : a.first < b.first; + }; + + top_k = top_k == 0 ? scores.size() : std::min(top_k, scores.size()); + ptrdiff_t start = 0; + while (indices.size() < top_k) + { + auto it = std::max_element(score_index_vec.begin() + start, score_index_vec.end(), score_cmp); + + float bscore = it->first; + size_t bidx = it->second; + + if (bscore < score_threshold) + { + break; + } + + indices.push_back(static_cast(bidx)); + updated_scores.push_back(bscore); + std::swap(score_index_vec[start], *it); // first start elements are chosen + + for (size_t i = start + 1; i < scores.size(); ++i) + { + float& bscore_i = score_index_vec[i].first; + const size_t bidx_i = score_index_vec[i].second; + + if (bscore_i < score_threshold) + { + continue; + } + + float overlap = rectOverlap(bboxes[bidx], bboxes[bidx_i]); + + switch (method) + { + case SoftNMSMethod::SOFTNMS_LINEAR: + if (overlap > nms_threshold) + { + bscore_i *= 1.f - overlap; + } + break; + case SoftNMSMethod::SOFTNMS_GAUSSIAN: + bscore_i *= exp(-(overlap * overlap) / sigma); + break; + default: + CV_Error(Error::StsBadArg, "Not supported SoftNMS method."); + } + } + ++start; + } +} + CV__DNN_INLINE_NS_END }// dnn }// cv diff --git a/modules/dnn/test/test_nms.cpp b/modules/dnn/test/test_nms.cpp index 53cb3500ec6e..5ba240287c21 100644 --- a/modules/dnn/test/test_nms.cpp +++ b/modules/dnn/test/test_nms.cpp @@ -37,4 +37,41 @@ TEST(NMS, Accuracy) ASSERT_EQ(indices[i], ref_indices[i]); } +TEST(SoftNMS, Accuracy) +{ + //reference results are obtained using TF v2.7 tf.image.non_max_suppression_with_scores + std::string dataPath = findDataFile("dnn/soft_nms_reference.yml"); + FileStorage fs(dataPath, FileStorage::READ); + + std::vector bboxes; + std::vector scores; + std::vector ref_indices; + std::vector ref_updated_scores; + + fs["boxes"] >> bboxes; + fs["probs"] >> scores; + fs["indices"] >> ref_indices; + fs["updated_scores"] >> ref_updated_scores; + + std::vector updated_scores; + const float score_thresh = .01f; + const float nms_thresh = .5f; + std::vector indices; + const size_t top_k = 0; + const float sigma = 1.; // sigma in TF is being multiplied by 2, so 0.5 should be passed there + cv::dnn::softNMSBoxes(bboxes, scores, updated_scores, score_thresh, nms_thresh, indices, top_k, sigma); + + ASSERT_EQ(ref_indices.size(), indices.size()); + for(size_t i = 0; i < indices.size(); i++) + { + ASSERT_EQ(indices[i], ref_indices[i]); + } + + ASSERT_EQ(ref_updated_scores.size(), updated_scores.size()); + for(size_t i = 0; i < updated_scores.size(); i++) + { + EXPECT_NEAR(updated_scores[i], ref_updated_scores[i], 1e-7); + } +} + }} // namespace From bdaa6a191005498f429142e3ddac039e589b3c83 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 6 Oct 2021 19:09:22 +0000 Subject: [PATCH 266/376] highgui: repair Qt backend --- modules/highgui/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index 3ef1535d2dc7..c5541241a4e9 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -51,6 +51,8 @@ set(OPENCV_HIGHGUI_BUILTIN_BACKEND "") if(HAVE_QT) set(OPENCV_HIGHGUI_BUILTIN_BACKEND "QT${QT_VERSION_MAJOR}") + add_definitions(-DHAVE_QT) + if(QT_VERSION_MAJOR GREATER 4) # "Automoc" doesn't work properly with opencv_world build, use QT_WRAP_CPP() directly #set(CMAKE_AUTOMOC ON) @@ -75,6 +77,7 @@ if(HAVE_QT) set(qt_deps Core Gui Widgets Test Concurrent) if(HAVE_QT_OPENGL) + add_definitions(-DHAVE_QT_OPENGL) list(APPEND qt_deps OpenGL) endif() From 94e92cd6c0250bd091ffea99f7ab036e6292d7a3 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 6 Oct 2021 21:27:18 +0000 Subject: [PATCH 267/376] dnn(ocl): skip int8 tests due to memory access issues --- modules/dnn/test/test_int8_layers.cpp | 131 ++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/modules/dnn/test/test_int8_layers.cpp b/modules/dnn/test/test_int8_layers.cpp index 5e6c05c7162f..85c30a4271e5 100644 --- a/modules/dnn/test/test_int8_layers.cpp +++ b/modules/dnn/test/test_int8_layers.cpp @@ -544,6 +544,12 @@ TEST_P(Test_Int8_nets, AlexNet) if (backend != DNN_BACKEND_OPENCV) throw SkipTestException("Only OpenCV backend is supported"); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + Net net = readNetFromCaffe(findDataFile("dnn/bvlc_alexnet.prototxt"), findDataFile("dnn/bvlc_alexnet.caffemodel", false)); @@ -557,6 +563,11 @@ TEST_P(Test_Int8_nets, AlexNet) TEST_P(Test_Int8_nets, GoogLeNet) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + Net net = readNetFromCaffe(findDataFile("dnn/bvlc_googlenet.prototxt"), findDataFile("dnn/bvlc_googlenet.caffemodel", false)); @@ -576,6 +587,11 @@ TEST_P(Test_Int8_nets, ResNet50) if (backend != DNN_BACKEND_OPENCV) throw SkipTestException("Only OpenCV backend is supported"); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + Net net = readNetFromCaffe(findDataFile("dnn/ResNet-50-deploy.prototxt"), findDataFile("dnn/ResNet-50-model.caffemodel", false)); @@ -591,6 +607,11 @@ TEST_P(Test_Int8_nets, DenseNet121) { applyTestTag(CV_TEST_TAG_MEMORY_512MB); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + Net net = readNetFromCaffe(findDataFile("dnn/DenseNet_121.prototxt", false), findDataFile("dnn/DenseNet_121.caffemodel", false)); @@ -607,6 +628,11 @@ TEST_P(Test_Int8_nets, SqueezeNet_v1_1) if(target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + Net net = readNetFromCaffe(findDataFile("dnn/squeezenet_v1.1.prototxt"), findDataFile("dnn/squeezenet_v1.1.caffemodel", false)); @@ -626,6 +652,11 @@ TEST_P(Test_Int8_nets, CaffeNet) applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); #endif + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019030000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) @@ -643,6 +674,11 @@ TEST_P(Test_Int8_nets, RCNN_ILSVRC13) applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); #endif + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019030000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) @@ -654,16 +690,31 @@ TEST_P(Test_Int8_nets, RCNN_ILSVRC13) TEST_P(Test_Int8_nets, Inception_v2) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + testONNXNet("inception_v2", default_l1, default_lInf, true); } TEST_P(Test_Int8_nets, MobileNet_v2) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + testONNXNet("mobilenetv2", default_l1, default_lInf, true); } TEST_P(Test_Int8_nets, Shufflenet) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) { if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); @@ -675,6 +726,11 @@ TEST_P(Test_Int8_nets, Shufflenet) TEST_P(Test_Int8_nets, MobileNet_SSD) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + Net net = readNetFromCaffe(findDataFile("dnn/MobileNetSSD_deploy.prototxt", false), findDataFile("dnn/MobileNetSSD_deploy.caffemodel", false)); @@ -688,6 +744,11 @@ TEST_P(Test_Int8_nets, MobileNet_SSD) TEST_P(Test_Int8_nets, MobileNet_v1_SSD) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + Net net = readNetFromTensorflow(findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false), findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt")); @@ -701,6 +762,11 @@ TEST_P(Test_Int8_nets, MobileNet_v1_SSD) TEST_P(Test_Int8_nets, MobileNet_v1_SSD_PPN) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, @@ -720,6 +786,11 @@ TEST_P(Test_Int8_nets, MobileNet_v1_SSD_PPN) TEST_P(Test_Int8_nets, Inception_v2_SSD) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2019010000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD && @@ -744,6 +815,11 @@ TEST_P(Test_Int8_nets, Inception_v2_SSD) TEST_P(Test_Int8_nets, opencv_face_detector) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + Net net = readNetFromCaffe(findDataFile("dnn/opencv_face_detector.prototxt"), findDataFile("dnn/opencv_face_detector.caffemodel", false)); @@ -762,6 +838,11 @@ TEST_P(Test_Int8_nets, opencv_face_detector) TEST_P(Test_Int8_nets, EfficientDet) { + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + if (target != DNN_TARGET_CPU) { if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); @@ -789,6 +870,11 @@ TEST_P(Test_Int8_nets, FasterRCNN_resnet50) CV_TEST_TAG_DEBUG_VERYLONG ); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #ifdef INF_ENGINE_RELEASE if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU)) @@ -827,6 +913,11 @@ TEST_P(Test_Int8_nets, FasterRCNN_inceptionv2) CV_TEST_TAG_DEBUG_VERYLONG ); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #ifdef INF_ENGINE_RELEASE if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU)) @@ -869,6 +960,11 @@ TEST_P(Test_Int8_nets, FasterRCNN_vgg16) CV_TEST_TAG_DEBUG_VERYLONG ); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); @@ -902,6 +998,11 @@ TEST_P(Test_Int8_nets, FasterRCNN_zf) CV_TEST_TAG_DEBUG_LONG ); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); @@ -932,6 +1033,11 @@ TEST_P(Test_Int8_nets, RFCN) CV_TEST_TAG_DEBUG_VERYLONG ); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); @@ -961,6 +1067,11 @@ TEST_P(Test_Int8_nets, YoloVoc) CV_TEST_TAG_LONG ); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); @@ -1003,6 +1114,11 @@ TEST_P(Test_Int8_nets, TinyYoloVoc) { applyTestTag(CV_TEST_TAG_MEMORY_512MB); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); @@ -1039,6 +1155,11 @@ TEST_P(Test_Int8_nets, YOLOv3) { applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); @@ -1097,6 +1218,11 @@ TEST_P(Test_Int8_nets, YOLOv4) { applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); @@ -1159,6 +1285,11 @@ TEST_P(Test_Int8_nets, YOLOv4_tiny) target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB ); + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2021010000) if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); From eab2b9dc09570b141adeba3814f196a4efd8adb2 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 6 Oct 2021 18:43:12 +0000 Subject: [PATCH 268/376] core: ensure is_trivially_copyable for simple types --- modules/core/include/opencv2/core/types.hpp | 126 +++----------------- modules/core/test/test_misc.cpp | 23 ++++ 2 files changed, 39 insertions(+), 110 deletions(-) diff --git a/modules/core/include/opencv2/core/types.hpp b/modules/core/include/opencv2/core/types.hpp index 3f0131da8c26..5cdddab35d55 100644 --- a/modules/core/include/opencv2/core/types.hpp +++ b/modules/core/include/opencv2/core/types.hpp @@ -162,13 +162,13 @@ template class Point_ //! default constructor Point_(); Point_(_Tp _x, _Tp _y); - Point_(const Point_& pt); - Point_(Point_&& pt) CV_NOEXCEPT; + Point_(const Point_& pt) = default; + Point_(Point_&& pt) CV_NOEXCEPT = default; Point_(const Size_<_Tp>& sz); Point_(const Vec<_Tp, 2>& v); - Point_& operator = (const Point_& pt); - Point_& operator = (Point_&& pt) CV_NOEXCEPT; + Point_& operator = (const Point_& pt) = default; + Point_& operator = (Point_&& pt) CV_NOEXCEPT = default; //! conversion to another data type template operator Point_<_Tp2>() const; @@ -244,13 +244,13 @@ template class Point3_ //! default constructor Point3_(); Point3_(_Tp _x, _Tp _y, _Tp _z); - Point3_(const Point3_& pt); - Point3_(Point3_&& pt) CV_NOEXCEPT; + Point3_(const Point3_& pt) = default; + Point3_(Point3_&& pt) CV_NOEXCEPT = default; explicit Point3_(const Point_<_Tp>& pt); Point3_(const Vec<_Tp, 3>& v); - Point3_& operator = (const Point3_& pt); - Point3_& operator = (Point3_&& pt) CV_NOEXCEPT; + Point3_& operator = (const Point3_& pt) = default; + Point3_& operator = (Point3_&& pt) CV_NOEXCEPT = default; //! conversion to another data type template operator Point3_<_Tp2>() const; //! conversion to cv::Vec<> @@ -320,12 +320,12 @@ template class Size_ //! default constructor Size_(); Size_(_Tp _width, _Tp _height); - Size_(const Size_& sz); - Size_(Size_&& sz) CV_NOEXCEPT; + Size_(const Size_& sz) = default; + Size_(Size_&& sz) CV_NOEXCEPT = default; Size_(const Point_<_Tp>& pt); - Size_& operator = (const Size_& sz); - Size_& operator = (Size_&& sz) CV_NOEXCEPT; + Size_& operator = (const Size_& sz) = default; + Size_& operator = (Size_&& sz) CV_NOEXCEPT = default; //! the area (width*height) _Tp area() const; //! aspect ratio (width/height) @@ -425,13 +425,13 @@ template class Rect_ //! default constructor Rect_(); Rect_(_Tp _x, _Tp _y, _Tp _width, _Tp _height); - Rect_(const Rect_& r); - Rect_(Rect_&& r) CV_NOEXCEPT; + Rect_(const Rect_& r) = default; + Rect_(Rect_&& r) CV_NOEXCEPT = default; Rect_(const Point_<_Tp>& org, const Size_<_Tp>& sz); Rect_(const Point_<_Tp>& pt1, const Point_<_Tp>& pt2); - Rect_& operator = ( const Rect_& r ); - Rect_& operator = ( Rect_&& r ) CV_NOEXCEPT; + Rect_& operator = (const Rect_& r) = default; + Rect_& operator = (Rect_&& r) CV_NOEXCEPT = default; //! the top-left corner Point_<_Tp> tl() const; //! the bottom-right corner @@ -1164,14 +1164,6 @@ template inline Point_<_Tp>::Point_(_Tp _x, _Tp _y) : x(_x), y(_y) {} -template inline -Point_<_Tp>::Point_(const Point_& pt) - : x(pt.x), y(pt.y) {} - -template inline -Point_<_Tp>::Point_(Point_&& pt) CV_NOEXCEPT - : x(std::move(pt.x)), y(std::move(pt.y)) {} - template inline Point_<_Tp>::Point_(const Size_<_Tp>& sz) : x(sz.width), y(sz.height) {} @@ -1180,20 +1172,6 @@ template inline Point_<_Tp>::Point_(const Vec<_Tp,2>& v) : x(v[0]), y(v[1]) {} -template inline -Point_<_Tp>& Point_<_Tp>::operator = (const Point_& pt) -{ - x = pt.x; y = pt.y; - return *this; -} - -template inline -Point_<_Tp>& Point_<_Tp>::operator = (Point_&& pt) CV_NOEXCEPT -{ - x = std::move(pt.x); y = std::move(pt.y); - return *this; -} - template template inline Point_<_Tp>::operator Point_<_Tp2>() const { @@ -1431,14 +1409,6 @@ template inline Point3_<_Tp>::Point3_(_Tp _x, _Tp _y, _Tp _z) : x(_x), y(_y), z(_z) {} -template inline -Point3_<_Tp>::Point3_(const Point3_& pt) - : x(pt.x), y(pt.y), z(pt.z) {} - -template inline -Point3_<_Tp>::Point3_(Point3_&& pt) CV_NOEXCEPT - : x(std::move(pt.x)), y(std::move(pt.y)), z(std::move(pt.z)) {} - template inline Point3_<_Tp>::Point3_(const Point_<_Tp>& pt) : x(pt.x), y(pt.y), z(_Tp()) {} @@ -1459,20 +1429,6 @@ Point3_<_Tp>::operator Vec<_Tp, 3>() const return Vec<_Tp, 3>(x, y, z); } -template inline -Point3_<_Tp>& Point3_<_Tp>::operator = (const Point3_& pt) -{ - x = pt.x; y = pt.y; z = pt.z; - return *this; -} - -template inline -Point3_<_Tp>& Point3_<_Tp>::operator = (Point3_&& pt) CV_NOEXCEPT -{ - x = std::move(pt.x); y = std::move(pt.y); z = std::move(pt.z); - return *this; -} - template inline _Tp Point3_<_Tp>::dot(const Point3_& pt) const { @@ -1685,14 +1641,6 @@ template inline Size_<_Tp>::Size_(_Tp _width, _Tp _height) : width(_width), height(_height) {} -template inline -Size_<_Tp>::Size_(const Size_& sz) - : width(sz.width), height(sz.height) {} - -template inline -Size_<_Tp>::Size_(Size_&& sz) CV_NOEXCEPT - : width(std::move(sz.width)), height(std::move(sz.height)) {} - template inline Size_<_Tp>::Size_(const Point_<_Tp>& pt) : width(pt.x), height(pt.y) {} @@ -1703,20 +1651,6 @@ Size_<_Tp>::operator Size_<_Tp2>() const return Size_<_Tp2>(saturate_cast<_Tp2>(width), saturate_cast<_Tp2>(height)); } -template inline -Size_<_Tp>& Size_<_Tp>::operator = (const Size_<_Tp>& sz) -{ - width = sz.width; height = sz.height; - return *this; -} - -template inline -Size_<_Tp>& Size_<_Tp>::operator = (Size_<_Tp>&& sz) CV_NOEXCEPT -{ - width = std::move(sz.width); height = std::move(sz.height); - return *this; -} - template inline _Tp Size_<_Tp>::area() const { @@ -1827,14 +1761,6 @@ template inline Rect_<_Tp>::Rect_(_Tp _x, _Tp _y, _Tp _width, _Tp _height) : x(_x), y(_y), width(_width), height(_height) {} -template inline -Rect_<_Tp>::Rect_(const Rect_<_Tp>& r) - : x(r.x), y(r.y), width(r.width), height(r.height) {} - -template inline -Rect_<_Tp>::Rect_(Rect_<_Tp>&& r) CV_NOEXCEPT - : x(std::move(r.x)), y(std::move(r.y)), width(std::move(r.width)), height(std::move(r.height)) {} - template inline Rect_<_Tp>::Rect_(const Point_<_Tp>& org, const Size_<_Tp>& sz) : x(org.x), y(org.y), width(sz.width), height(sz.height) {} @@ -1848,26 +1774,6 @@ Rect_<_Tp>::Rect_(const Point_<_Tp>& pt1, const Point_<_Tp>& pt2) height = std::max(pt1.y, pt2.y) - y; } -template inline -Rect_<_Tp>& Rect_<_Tp>::operator = ( const Rect_<_Tp>& r ) -{ - x = r.x; - y = r.y; - width = r.width; - height = r.height; - return *this; -} - -template inline -Rect_<_Tp>& Rect_<_Tp>::operator = ( Rect_<_Tp>&& r ) CV_NOEXCEPT -{ - x = std::move(r.x); - y = std::move(r.y); - width = std::move(r.width); - height = std::move(r.height); - return *this; -} - template inline Point_<_Tp> Rect_<_Tp>::tl() const { diff --git a/modules/core/test/test_misc.cpp b/modules/core/test/test_misc.cpp index 67d0a5399580..55615b0d5f5c 100644 --- a/modules/core/test/test_misc.cpp +++ b/modules/core/test/test_misc.cpp @@ -798,4 +798,27 @@ TEST(Core_Allocation, alignedAllocation) } } + +#if !(defined(__GNUC__) && __GNUC__ < 5) // GCC 4.8 emits: 'is_trivially_copyable' is not a member of 'std' +TEST(Core_Types, trivially_copyable) +{ + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + //EXPECT_TRUE(std::is_trivially_copyable::value); // derived from Vec (Matx) +} + +TEST(Core_Types, trivially_copyable_extra) +{ + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); +} +#endif + }} // namespace From 04b40ff2213ad4810e65feac41e798724dde0d83 Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Mon, 4 Oct 2021 14:42:32 +0200 Subject: [PATCH 269/376] add new supported MSVC version --- cmake/OpenCVDetectCXXCompiler.cmake | 2 ++ cmake/templates/OpenCVConfig.root-WIN32.cmake.in | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/cmake/OpenCVDetectCXXCompiler.cmake b/cmake/OpenCVDetectCXXCompiler.cmake index f9fc1189ced5..24eda01f1868 100644 --- a/cmake/OpenCVDetectCXXCompiler.cmake +++ b/cmake/OpenCVDetectCXXCompiler.cmake @@ -169,6 +169,8 @@ elseif(MSVC) set(OpenCV_RUNTIME vc15) elseif(MSVC_VERSION MATCHES "^192[0-9]$") set(OpenCV_RUNTIME vc16) + elseif(MSVC_VERSION MATCHES "^193[0-9]$") + set(OpenCV_RUNTIME vc17) else() message(WARNING "OpenCV does not recognize MSVC_VERSION \"${MSVC_VERSION}\". Cannot set OpenCV_RUNTIME") endif() diff --git a/cmake/templates/OpenCVConfig.root-WIN32.cmake.in b/cmake/templates/OpenCVConfig.root-WIN32.cmake.in index 5da438a9ee9c..b0f254ebe80f 100644 --- a/cmake/templates/OpenCVConfig.root-WIN32.cmake.in +++ b/cmake/templates/OpenCVConfig.root-WIN32.cmake.in @@ -137,6 +137,20 @@ elseif(MSVC) set(OpenCV_RUNTIME vc14) # selecting previous compatible runtime version endif() endif() + elseif(MSVC_VERSION MATCHES "^193[0-9]$") + set(OpenCV_RUNTIME vc17) + check_one_config(has_VS2022) + if(NOT has_VS2022) + set(OpenCV_RUNTIME vc16) + check_one_config(has_VS2019) + if(NOT has_VS2019) + set(OpenCV_RUNTIME vc15) # selecting previous compatible runtime version + check_one_config(has_VS2017) + if(NOT has_VS2017) + set(OpenCV_RUNTIME vc14) # selecting previous compatible runtime version + endif() + endif() + endif() endif() elseif(MINGW) set(OpenCV_RUNTIME mingw) From d023f316ac281d287a230816a4bac7bc13d1a9de Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Mon, 4 Oct 2021 14:42:08 +0200 Subject: [PATCH 270/376] fix OPENCV_DISABLE_THREAD_SUPPORT Message was, CMake Error at CMakeLists.txt:1475 (message): Not all parallel frameworks have been disabled (using Concurrency). --- cmake/OpenCVFindFrameworks.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/OpenCVFindFrameworks.cmake b/cmake/OpenCVFindFrameworks.cmake index 9d673f2697ff..19f6d66340c1 100644 --- a/cmake/OpenCVFindFrameworks.cmake +++ b/cmake/OpenCVFindFrameworks.cmake @@ -17,7 +17,7 @@ else() endif() # --- Concurrency --- -if(MSVC AND NOT HAVE_TBB) +if(MSVC AND NOT HAVE_TBB AND NOT OPENCV_DISABLE_THREAD_SUPPORT) set(_fname "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/concurrencytest.cpp") file(WRITE "${_fname}" "#if _MSC_VER < 1600\n#error\n#endif\nint main() { return 0; }\n") try_compile(HAVE_CONCURRENCY "${CMAKE_BINARY_DIR}" "${_fname}") From dfc94c58f0567446c14aa217d81efb5ca42b5aa8 Mon Sep 17 00:00:00 2001 From: Alexander Panov Date: Fri, 8 Oct 2021 01:46:25 +0300 Subject: [PATCH 271/376] Merge pull request #20823 from AleksandrPanov:fix_orb_integer_overflow Fix ORB integer overflow * set size_t step to fix integer overflow in ptr0 offset * added issue_537 test * minor fix tags, points * added size_t_step and offset to remove mixed unsigned and signed operations * features2d: update ORB checks Co-authored-by: Alexander Alekhin --- modules/features2d/src/orb.cpp | 11 ++++++++--- modules/features2d/test/test_orb.cpp | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/modules/features2d/src/orb.cpp b/modules/features2d/src/orb.cpp index 39ebec7f20f1..85045557b658 100644 --- a/modules/features2d/src/orb.cpp +++ b/modules/features2d/src/orb.cpp @@ -131,12 +131,17 @@ static void HarrisResponses(const Mat& img, const std::vector& layerinfo, std::vector& pts, int blockSize, float harris_k) { - CV_Assert( img.type() == CV_8UC1 && blockSize*blockSize <= 2048 ); + CV_CheckTypeEQ(img.type(), CV_8UC1, ""); + CV_CheckGT(blockSize, 0, ""); + CV_CheckLE(blockSize*blockSize, 2048, ""); size_t ptidx, ptsize = pts.size(); const uchar* ptr00 = img.ptr(); - int step = (int)(img.step/img.elemSize1()); + size_t size_t_step = img.step; + CV_CheckLE(size_t_step * blockSize + blockSize + 1, (size_t)INT_MAX, ""); // ofs computation, step+1 + int step = static_cast(size_t_step); + int r = blockSize/2; float scale = 1.f/((1 << 2) * blockSize * 255.f); @@ -154,7 +159,7 @@ HarrisResponses(const Mat& img, const std::vector& layerinfo, int y0 = cvRound(pts[ptidx].pt.y); int z = pts[ptidx].octave; - const uchar* ptr0 = ptr00 + (y0 - r + layerinfo[z].y)*step + x0 - r + layerinfo[z].x; + const uchar* ptr0 = ptr00 + (y0 - r + layerinfo[z].y)*size_t_step + (x0 - r + layerinfo[z].x); int a = 0, b = 0, c = 0; for( int k = 0; k < blockSize*blockSize; k++ ) diff --git a/modules/features2d/test/test_orb.cpp b/modules/features2d/test/test_orb.cpp index 0ffaa626d706..b37d4708d639 100644 --- a/modules/features2d/test/test_orb.cpp +++ b/modules/features2d/test/test_orb.cpp @@ -141,5 +141,31 @@ TEST(Features2D_ORB, regression_16197) ASSERT_NO_THROW(orbPtr->detectAndCompute(img, noArray(), kps, fv)); } +// https://github.com/opencv/opencv-python/issues/537 +BIGDATA_TEST(Features2D_ORB, regression_opencv_python_537) // memory usage: ~3 Gb +{ + applyTestTag( + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG, + CV_TEST_TAG_MEMORY_6GB + ); + + const int width = 25000; + const int height = 25000; + Mat img(Size(width, height), CV_8UC1, Scalar::all(0)); + + const int border = 23, num_lines = 23; + for (int i = 0; i < num_lines; i++) + { + cv::Point2i point1(border + i * 100, border + i * 100); + cv::Point2i point2(width - border - i * 100, height - border * i * 100); + cv::line(img, point1, point2, 255, 1, LINE_AA); + } + + Ptr orbPtr = ORB::create(31); + std::vector kps; + Mat fv; + ASSERT_NO_THROW(orbPtr->detectAndCompute(img, noArray(), kps, fv)); +} }} // namespace From 724e04e979f6706680ad359037ea536dc057d7e2 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 7 Oct 2021 17:58:19 +0000 Subject: [PATCH 272/376] dnn(ocl4dnn): add extra checks to convolution layer - prevent running code over unsupported/non-tested configurations - prevent integer div by zero --- modules/dnn/src/layers/convolution_layer.cpp | 20 +++++++++++++++++++ .../src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 1 + 2 files changed, 21 insertions(+) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index edbd2baefbce..1236ff578326 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -46,6 +46,7 @@ #include "../op_inf_engine.hpp" #include "../ie_ngraph.hpp" +#include #include #include "opencv2/core/hal/hal.hpp" @@ -1494,7 +1495,26 @@ class ConvolutionLayerImpl CV_FINAL : public BaseConvolutionLayerImpl config.pad = pad; config.stride = stride; config.dilation = dilation; + if (inputs[0].dims != 4 && inputs[0].dims != umat_blobs[0].dims) + { + static bool bypassCheck = utils::getConfigurationParameterBool("OPENCV_OCL4DNN_CONVOLUTION_IGNORE_INPUT_DIMS_4_CHECK", false); + if (!bypassCheck) + { + CV_LOG_ERROR(NULL, "DNN/OpenCL: Unsupported configuration: inputs[0].dims=" << inputs[0].dims << " umat_blobs[0].dims=" << umat_blobs[0].dims + << ". Consider reporting complete reproducer to https://github.com/opencv/opencv/issues/20833." + << " You can skip this check temporary through OPENCV_OCL4DNN_CONVOLUTION_IGNORE_INPUT_DIMS_4_CHECK=1" + ); + return false; + } + } config.group = inputs[0].size[1] / umat_blobs[0].size[1]; + if (config.group < 1) // config.group == 0 causes div by zero in ocl4dnn code + { + CV_LOG_WARNING(NULL, "DNN/OpenCL: Unsupported config.group=" << config.group + << ". Consider reporting complete reproducer to https://github.com/opencv/opencv/issues/20833" + ); + return false; + } config.bias_term = umat_blobs.size() == 2; config.use_half = use_half; diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index 45bd249e5d62..022312aa1fa6 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -167,6 +167,7 @@ OCL4DNNConvSpatial::OCL4DNNConvSpatial(OCL4DNNConvConfig config) channels_ = config.in_shape[dims - spatial_dims - 1]; num_output_ = config.out_shape[dims - spatial_dims - 1]; group_ = config.group; + CV_CheckGT(group_, 0, ""); // avoid div by zero below fused_activ_ = OCL4DNN_CONV_FUSED_ACTIV_NONE; fused_eltwise_ = false; From 27545dcc86c11aca4a5c8fff0ef14f839d9d8869 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 7 Oct 2021 23:27:10 +0000 Subject: [PATCH 273/376] core: add __NetBSD__ build fix in parallel.cpp --- modules/core/src/parallel.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/parallel.cpp b/modules/core/src/parallel.cpp index cfff4cea4bce..d1454457b7b8 100644 --- a/modules/core/src/parallel.cpp +++ b/modules/core/src/parallel.cpp @@ -54,8 +54,8 @@ #endif #if defined __unix__ || defined __APPLE__ || defined __GLIBC__ \ - || defined __HAIKU__ || defined __EMSCRIPTEN__ || defined __FreeBSD__ \ - || defined __OpenBSD__ + || defined __HAIKU__ || defined __EMSCRIPTEN__ \ + || defined __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__ #include #include #include From 8c2dd5fb9a4e568466f92db5d954d5ddc351bc9d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 8 Oct 2021 00:12:14 +0000 Subject: [PATCH 274/376] dnn(ocl4dnn): cleanup dead code, improve logging --- modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp | 2 - .../src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp | 100 ++++++------------ modules/dnn/src/opencl/conv_layer_spatial.cl | 39 ++----- 3 files changed, 42 insertions(+), 99 deletions(-) diff --git a/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp b/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp index d6fb83becb92..bf5fba71a1ef 100644 --- a/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp +++ b/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp @@ -222,8 +222,6 @@ class OCL4DNNConvSpatial bool createDWConvKernel(int32_t blockWidth, int32_t blockHeight, int32_t blockDepth); - void CreateSubBuffer(const UMat& buffer, UMat& sub_buffer, - int32_t offset, int32_t size, bool write_only); bool convolve(const UMat &bottom, UMat &top, const UMat &weight, const UMat &bias, int32_t numImages, diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index 45bd249e5d62..6c468fc7da31 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -218,14 +218,7 @@ OCL4DNNConvSpatial::OCL4DNNConvSpatial(OCL4DNNConvConfig config) #endif if (!use_cache_path_) { - static int warn_ = 0; - if (!warn_) - { - std::cerr - << "OpenCV(ocl4dnn): Kernel configuration cache directory doesn't exist: " << cache_path_ << std::endl - << std::endl; - warn_ = true; - } + CV_LOG_ONCE_ERROR(NULL, "OpenCV(ocl4dnn): Kernel configuration cache directory doesn't exist: " << cache_path_); } } @@ -418,7 +411,6 @@ void OCL4DNNConvSpatial::setupKernelDetails(int32_t kernelType, addDef("CHANNELS", channels_ / group_); addDef("APPLY_BIAS", bias_term_); addDef("OUTPUT_Z", M_); - addDef("ZPAR", 1); setFusionDefine(fused_activ_, fused_eltwise_); src_ = cv::ocl::dnn::conv_layer_spatial_oclsrc; @@ -672,8 +664,7 @@ void interleaveMatrix(Dtype* mem_dst, const Dtype *mem, int r, int c, int interleavedRows, int nonInterleavedRows, int blockWidth, int rowAlignment ) { - CHECK_EQ(interleavedRows % 2, 0) << - "interleaveMatrix only supports even values for interleavedRows."; + CV_Check(interleavedRows, interleavedRows % 2 == 0, "interleaveMatrix only supports even values for interleavedRows."); size_t memSize = r * c * sizeof(float); size_t dstSize = memSize * @@ -685,9 +676,12 @@ void interleaveMatrix(Dtype* mem_dst, const Dtype *mem, const int yStride = c * 2; const Dtype *pSrc = mem; Dtype* pDst = mem_dst; - for (int y = 0; y < r;) { - for (int rows = 0; rows < interleavedRows; rows += 2) { - if ( y >= r ) break; + for (int y = 0; y < r;) + { + for (int rows = 0; rows < interleavedRows; rows += 2) + { + if (y >= r) + break; if ((c % xStride) == 0) { for (int x = 0; x < c / xStride; x++) { memcpy(pDst + x * xStride * 2, // NOLINT @@ -712,11 +706,14 @@ void interleaveMatrix(Dtype* mem_dst, const Dtype *mem, y += 2; } - for (int rows = 0; rows < nonInterleavedRows; rows++) { - if (y >= r) break; + for (int rows = 0; rows < nonInterleavedRows; rows++) + { + if (y >= r) + break; const int stride = rowAlignment; int remaining = c; - for (int x = 0; x < c; x += stride) { + for (int x = 0; x < c; x += stride) + { if (remaining >= stride) { memcpy(pDst + x * 2, pSrc + x, stride * sizeof(Dtype)); // NOLINT remaining -=stride; @@ -852,34 +849,6 @@ bool OCL4DNNConvSpatial::createBasicKernel(int32_t blockWidth, return false; } -template<> -void OCL4DNNConvSpatial::CreateSubBuffer(const UMat& buffer, UMat& sub_buffer, - int32_t offset, int32_t size, bool write_only) -{ - cl_mem sub_mem; - cl_buffer_region region; - cl_int err; - size_t element_size = (use_half_) ? sizeof(short) : sizeof(float); - - region.origin = offset * element_size + buffer.offset; - region.size = size * element_size; - sub_mem = clCreateSubBuffer((cl_mem)buffer.handle(ACCESS_READ), - write_only ? CL_MEM_WRITE_ONLY : CL_MEM_READ_ONLY, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - if (err) - { - std::cout << "Failed to create sub buffer." << std::endl; - return; - } - - int step = element_size, rows = size, cols = 1; - ocl::convertFromBuffer(sub_mem, step, rows, cols, - (use_half_) ? CV_16SC1 : CV_32FC1, sub_buffer); - - //decrease ocl mem refcount - clReleaseMemObject(sub_mem); -} - template<> bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, const UMat &weight, const UMat &bias, @@ -938,7 +907,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, kernel.set(argIdx++, (uint16_t)output_h_); if (!kernel.run_(3, config->global_work_size, config->local_work_size, false)) { - std::cout << "IDLF kernel run failed." << std::endl; + CV_LOG_ERROR(NULL, "DNN/OpenCL: IDLF kernel run failed"); return false; } } @@ -1012,7 +981,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, if (!kernel.run_(3, global_size, config->local_work_size, false)) { - std::cout << "GEMM like kernel run failed." << std::endl; + CV_LOG_ERROR(NULL, "DNN/OpenCL: GEMM like kernel run failed"); return false; } } @@ -1115,14 +1084,9 @@ float OCL4DNNConvSpatial::timedConvolve(const UMat &bottom, UMat &top, { queue = cv::ocl::Queue::getDefault(); } - catch (const cv::Exception&) + catch (const std::exception& e) { - static int warn_ = 0; - if (!warn_) - { - std::cout << "OpenCV(ocl4dnn): Can't get OpenCL default queue for auto-tuning." << std::endl; - warn_ = true; - } + CV_LOG_ONCE_ERROR(NULL, "OpenCV(ocl4dnn): Can't get OpenCL default queue for auto-tuning: " << e.what()); return 1e6; } @@ -1326,9 +1290,9 @@ ocl::Program OCL4DNNConvSpatial::compileKernel() phash.insert(std::pair(kernel_name_, program)); if (!program.ptr()) { - std::cout << "Failed to compile kernel: " << kernel_name_ - << ", buildflags: " << options - << ", errmsg: " << errmsg << std::endl; + CV_LOG_WARNING(NULL, "DNN/OpenCL: Failed to compile kernel: " << kernel_name_ + << ", buildflags: '" << options << "', errmsg: '" << errmsg << "'" + ); } return program; } @@ -1754,7 +1718,8 @@ void OCL4DNNConvSpatial::setupConvolution(const UMat &bottom, fastestTime = kernelQueue[x]->executionTime; } } - if (fastestKernel < 0) break; + if (fastestKernel < 0) + break; // Test fastest kernel bool verified = verifyResult(bottom, top, weight, bias, numImages, kernelQueue[fastestKernel], verifyTop); if (verified == true) { @@ -1913,17 +1878,18 @@ bool OCL4DNNConvSpatial::setupKernelByConfig(int x, int y, int z, int typ { if (z == 1) z = 16; - CHECK_EQ(z == 16 || z == 8, true) << "invalid SIMD size" << std::endl; + CV_Check(z, z == 16 || z == 8, "DNN/OpenCL: IDLF - invalid SIMD size"); } kernelQueue.clear(); createConvolutionKernel(type, x, y, z); - if (kernelQueue.size() != 1) { - std::cerr << "Failed setup kernel by config:" + if (kernelQueue.size() != 1) + { + CV_LOG_ERROR(NULL, "DNN/OpenCL: Failed setup kernel by config: " << " x = " << x << " y = " << y << " z = " << z << " type = " << type - << std::endl; + ); return false; } bestKernelConfig = kernelQueue[0]; @@ -1955,13 +1921,9 @@ bool OCL4DNNConvSpatial::loadTunedConfig() { if (cache_path_.empty()) { - static int warn_ = 0; - if (!warn_) - { - std::cout << "OpenCV(ocl4dnn): consider to specify kernel configuration cache directory " << std::endl - << " via OPENCV_OCL4DNN_CONFIG_PATH parameter." << std::endl; - warn_ = true; - } + CV_LOG_ONCE_WARNING(NULL, "OpenCV(ocl4dnn): consider to specify kernel configuration cache directory " + "through OPENCV_OCL4DNN_CONFIG_PATH parameter." + ); } return false; } diff --git a/modules/dnn/src/opencl/conv_layer_spatial.cl b/modules/dnn/src/opencl/conv_layer_spatial.cl index eb5d354020bd..c9ddacfb8e50 100644 --- a/modules/dnn/src/opencl/conv_layer_spatial.cl +++ b/modules/dnn/src/opencl/conv_layer_spatial.cl @@ -161,23 +161,15 @@ __kernel void ConvolveBasic( const int out_idx = get_global_id(0); // 1D task layout: [output_width * output_height * OUTPUT_Z] const int plane_size = output_width * output_height; const int out_plane_idx = out_idx % plane_size; - const int outputZ = out_idx / plane_size; + const int outputZ = out_idx / plane_size; // kernelNum const int outputY = out_plane_idx / output_width; const int outputX = out_plane_idx % output_width; - const int kernelNum = outputZ * ZPAR; - if (kernelNum < OUTPUT_Z) + if (outputZ < OUTPUT_Z) { - Dtype sum[ZPAR]; - for (int kern = 0; kern < ZPAR; kern++) - { - sum[kern] = 0.0f; - } + Dtype sum = 0.0f; const int org_y = outputY * STRIDE_Y - pad_h; const int org_x = outputX * STRIDE_X - pad_w; - const int currentKernelOffset = kernel_offset + kernelNum*KERNEL_HEIGHT*KERNEL_WIDTH*CHANNELS; -#if APPLY_BIAS - const int biasIndex = bias_offset + kernelNum; -#endif + const int currentKernelOffset = kernel_offset + outputZ*KERNEL_HEIGHT*KERNEL_WIDTH*CHANNELS; const int local_image_offset = org_y * input_width + org_x; const int imageSize = input_width * input_height; __global Dtype* image_dataPtr = (image_data + (image_offset + local_image_offset)); @@ -186,17 +178,13 @@ __kernel void ConvolveBasic( { for (int y = 0; y < KERNEL_HEIGHT; y++) { + int y_ = org_y + y * DILATION_Y; for (int x = 0; x < KERNEL_WIDTH; x++) { - int y_ = org_y + y * DILATION_Y; int x_ = org_x + x * DILATION_X; - if (!(y_ >= 0 && y_ < input_height && x_ >= 0 && x_ < input_width)) - { - continue; - } - for (int kern = 0; kern < ZPAR; kern++) + if (y_ >= 0 && y_ < input_height && x_ >= 0 && x_ < input_width) { - sum[kern] += image_dataPtr[x * DILATION_X] * kernel_dataPtr[kern*KERNEL_HEIGHT*KERNEL_WIDTH*CHANNELS + x]; + sum = mad(image_dataPtr[x * DILATION_X], kernel_dataPtr[x], sum); } } image_dataPtr += input_width * DILATION_Y; @@ -205,18 +193,13 @@ __kernel void ConvolveBasic( image_dataPtr += imageSize - input_width*KERNEL_HEIGHT*DILATION_Y; } - for (int kern = 0; kern < ZPAR; kern++) - { - if (kernelNum + kern < OUTPUT_Z) - { - int offset = convolved_image_offset + (kernelNum+kern)*output_height*output_width + outputY*output_width + outputX; + int offset = convolved_image_offset + out_idx; #if APPLY_BIAS - ACTIVATION_FUNCTION(convolved_image, offset, sum[kern] + bias[biasIndex + kern], biasIndex + kern); + int biasIndex = bias_offset + outputZ; + ACTIVATION_FUNCTION(convolved_image, offset, sum + bias[biasIndex], biasIndex); #else - ACTIVATION_FUNCTION(convolved_image, offset, sum[kern], kernelNum + kern); + ACTIVATION_FUNCTION(convolved_image, offset, sum, outputZ); #endif - } - } } } From e75387f029fe6dbfe3b67837046c1983dfe19971 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 7 Oct 2021 15:32:18 +0000 Subject: [PATCH 275/376] core: fix compilation of copy ctors/assignment operators with GCC 4.x --- modules/core/include/opencv2/core/types.hpp | 37 +++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/modules/core/include/opencv2/core/types.hpp b/modules/core/include/opencv2/core/types.hpp index 5cdddab35d55..1e9a8b629c1d 100644 --- a/modules/core/include/opencv2/core/types.hpp +++ b/modules/core/include/opencv2/core/types.hpp @@ -162,13 +162,23 @@ template class Point_ //! default constructor Point_(); Point_(_Tp _x, _Tp _y); +#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 + Point_(const Point_& pt); + Point_(Point_&& pt) CV_NOEXCEPT = default; +#elif OPENCV_ABI_COMPATIBILITY < 500 Point_(const Point_& pt) = default; Point_(Point_&& pt) CV_NOEXCEPT = default; +#endif Point_(const Size_<_Tp>& sz); Point_(const Vec<_Tp, 2>& v); +#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 + Point_& operator = (const Point_& pt); + Point_& operator = (Point_&& pt) CV_NOEXCEPT = default; +#elif OPENCV_ABI_COMPATIBILITY < 500 Point_& operator = (const Point_& pt) = default; Point_& operator = (Point_&& pt) CV_NOEXCEPT = default; +#endif //! conversion to another data type template operator Point_<_Tp2>() const; @@ -244,13 +254,17 @@ template class Point3_ //! default constructor Point3_(); Point3_(_Tp _x, _Tp _y, _Tp _z); +#if OPENCV_ABI_COMPATIBILITY < 500 Point3_(const Point3_& pt) = default; Point3_(Point3_&& pt) CV_NOEXCEPT = default; +#endif explicit Point3_(const Point_<_Tp>& pt); Point3_(const Vec<_Tp, 3>& v); +#if OPENCV_ABI_COMPATIBILITY < 500 Point3_& operator = (const Point3_& pt) = default; Point3_& operator = (Point3_&& pt) CV_NOEXCEPT = default; +#endif //! conversion to another data type template operator Point3_<_Tp2>() const; //! conversion to cv::Vec<> @@ -320,12 +334,16 @@ template class Size_ //! default constructor Size_(); Size_(_Tp _width, _Tp _height); +#if OPENCV_ABI_COMPATIBILITY < 500 Size_(const Size_& sz) = default; Size_(Size_&& sz) CV_NOEXCEPT = default; +#endif Size_(const Point_<_Tp>& pt); +#if OPENCV_ABI_COMPATIBILITY < 500 Size_& operator = (const Size_& sz) = default; Size_& operator = (Size_&& sz) CV_NOEXCEPT = default; +#endif //! the area (width*height) _Tp area() const; //! aspect ratio (width/height) @@ -425,13 +443,17 @@ template class Rect_ //! default constructor Rect_(); Rect_(_Tp _x, _Tp _y, _Tp _width, _Tp _height); +#if OPENCV_ABI_COMPATIBILITY < 500 Rect_(const Rect_& r) = default; Rect_(Rect_&& r) CV_NOEXCEPT = default; +#endif Rect_(const Point_<_Tp>& org, const Size_<_Tp>& sz); Rect_(const Point_<_Tp>& pt1, const Point_<_Tp>& pt2); +#if OPENCV_ABI_COMPATIBILITY < 500 Rect_& operator = (const Rect_& r) = default; Rect_& operator = (Rect_&& r) CV_NOEXCEPT = default; +#endif //! the top-left corner Point_<_Tp> tl() const; //! the bottom-right corner @@ -1164,6 +1186,12 @@ template inline Point_<_Tp>::Point_(_Tp _x, _Tp _y) : x(_x), y(_y) {} +#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 +template inline +Point_<_Tp>::Point_(const Point_& pt) + : x(pt.x), y(pt.y) {} +#endif + template inline Point_<_Tp>::Point_(const Size_<_Tp>& sz) : x(sz.width), y(sz.height) {} @@ -1172,6 +1200,15 @@ template inline Point_<_Tp>::Point_(const Vec<_Tp,2>& v) : x(v[0]), y(v[1]) {} +#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 +template inline +Point_<_Tp>& Point_<_Tp>::operator = (const Point_& pt) +{ + x = pt.x; y = pt.y; + return *this; +} +#endif + template template inline Point_<_Tp>::operator Point_<_Tp2>() const { From 9537a909f7fbcca5e9596846e29cae5e7eb1c86a Mon Sep 17 00:00:00 2001 From: Jonas Vautherin Date: Fri, 8 Oct 2021 04:07:04 +0200 Subject: [PATCH 276/376] Merge pull request #20801 from JonasVautherin:fix-gst-error-handling * Fix gst error handling * Use the return value instead of the error, which gives no guarantee of being NULL in case of error * Test err pointer before accessing it * Remove unreachable code * videoio(gstreamer): restore check in writer code Co-authored-by: Alexander Alekhin --- modules/videoio/src/cap_gstreamer.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index d8390691ffda..dd6387d89278 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -761,7 +761,7 @@ bool GStreamerCapture::open(const String &filename_) } else { - CV_WARN("Error opening file: " << filename << " (" << err->message << ")"); + CV_WARN("Error opening file: " << filename << " (" << (err ? err->message : "") << ")"); return false; } } @@ -769,9 +769,9 @@ bool GStreamerCapture::open(const String &filename_) { GSafePtr err; uridecodebin.attach(gst_parse_launch(filename, err.getRef())); - if (err) + if (!uridecodebin) { - CV_WARN("Error opening bin: " << err->message); + CV_WARN("Error opening bin: " << (err ? err->message : "")); return false; } manualpipeline = true; @@ -1979,7 +1979,7 @@ void handleMessage(GstElement * pipeline) gst_message_parse_error(msg, err.getRef(), debug.getRef()); GSafePtr name; name.attach(gst_element_get_name(GST_MESSAGE_SRC (msg))); CV_WARN("Embedded video playback halted; module " << name.get() << - " reported: " << err->message); + " reported: " << (err ? err->message : "")); CV_LOG_DEBUG(NULL, "GStreamer debug: " << debug.get()); gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_NULL); From 4985311d46f8e073646de3d2abff9ff9a8c7dd05 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 8 Oct 2021 04:36:58 +0300 Subject: [PATCH 277/376] core(tls): avoid destruction of TlsAbstraction singleton --- modules/core/src/system.cpp | 135 ++++++++++++++++++++++-------------- 1 file changed, 83 insertions(+), 52 deletions(-) diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index e2284b041523..27142a403446 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -53,6 +53,18 @@ #include #include +#ifndef OPENCV_WITH_THREAD_SANITIZER + #if defined(__clang__) && defined(__has_feature) + #if __has_feature(thread_sanitizer) + #define OPENCV_WITH_THREAD_SANITIZER 1 + #include // assume C++11 + #endif + #endif +#endif +#ifndef OPENCV_WITH_THREAD_SANITIZER + #define OPENCV_WITH_THREAD_SANITIZER 0 +#endif + namespace cv { static void _initSystem() @@ -1426,74 +1438,75 @@ namespace details { #endif #endif -template -class DisposedSingletonMark -{ -private: - static bool mark; -protected: - DisposedSingletonMark() {} - ~DisposedSingletonMark() - { - mark = true; - } -public: - static bool isDisposed() { return mark; } -}; - // TLS platform abstraction layer -class TlsAbstraction : public DisposedSingletonMark +class TlsAbstraction { public: TlsAbstraction(); - ~TlsAbstraction(); - void* getData() const + ~TlsAbstraction() { - if (isDisposed()) // guard: static initialization order fiasco - return NULL; - return getData_(); - } - void setData(void *pData) - { - if (isDisposed()) // guard: static initialization order fiasco - return; - return setData_(pData); + // TlsAbstraction singleton should not be released + // There is no reliable way to avoid problems caused by static initialization order fiasco + // NB: Do NOT use logging here + fprintf(stderr, "OpenCV FATAL: TlsAbstraction::~TlsAbstraction() call is not expected\n"); + fflush(stderr); } + void* getData() const; + void setData(void *pData); + + void releaseSystemResources(); + private: - void* getData_() const; - void setData_(void *pData); #ifdef _WIN32 #ifndef WINRT DWORD tlsKey; + bool disposed; #endif #else // _WIN32 pthread_key_t tlsKey; +#if OPENCV_WITH_THREAD_SANITIZER + std::atomic disposed; +#else + bool disposed; +#endif #endif }; -template<> bool DisposedSingletonMark::mark = false; - -static TlsAbstraction& getTlsAbstraction_() +class TlsAbstractionReleaseGuard { - static TlsAbstraction g_tls; // disposed in atexit() handlers (required for unregistering our callbacks) - return g_tls; -} + TlsAbstraction& tls_; +public: + TlsAbstractionReleaseGuard(TlsAbstraction& tls) : tls_(tls) + { + /* nothing */ + } + ~TlsAbstractionReleaseGuard() + { + tls_.releaseSystemResources(); + } +}; + +// TODO use reference static TlsAbstraction* getTlsAbstraction() { #ifdef CV_CXX11 - static TlsAbstraction* instance = &getTlsAbstraction_(); + static TlsAbstraction *g_tls = new TlsAbstraction(); // memory leak is intended here to avoid disposing of TLS container + static TlsAbstractionReleaseGuard g_tlsReleaseGuard(*g_tls); #else - static TlsAbstraction* volatile instance = NULL; - if (instance == NULL) + static TlsAbstraction* volatile g_tls = NULL; + if (g_tls == NULL) { cv::AutoLock lock(cv::getInitializationMutex()); - if (instance == NULL) - instance = &getTlsAbstraction_(); + if (g_tls == NULL) + { + g_tls = new TlsAbstraction(); + static TlsAbstractionReleaseGuard g_tlsReleaseGuard(*g_tls); + } } #endif - return DisposedSingletonMark::isDisposed() ? NULL : instance; + return g_tls; } @@ -1501,12 +1514,15 @@ static TlsAbstraction* getTlsAbstraction() #ifdef WINRT static __declspec( thread ) void* tlsData = NULL; // using C++11 thread attribute for local thread data TlsAbstraction::TlsAbstraction() {} -TlsAbstraction::~TlsAbstraction() {} -void* TlsAbstraction::getData_() const +void TlsAbstraction::releaseSystemResources() +{ + cv::__termination = true; // DllMain is missing in static builds +} +void* TlsAbstraction::getData() const { return tlsData; } -void TlsAbstraction::setData_(void *pData) +void TlsAbstraction::setData(void *pData) { tlsData = pData; } @@ -1515,6 +1531,7 @@ void TlsAbstraction::setData_(void *pData) static void NTAPI opencv_fls_destructor(void* pData); #endif // CV_USE_FLS TlsAbstraction::TlsAbstraction() + : disposed(false) { #ifndef CV_USE_FLS tlsKey = TlsAlloc(); @@ -1523,8 +1540,10 @@ TlsAbstraction::TlsAbstraction() #endif // CV_USE_FLS CV_Assert(tlsKey != TLS_OUT_OF_INDEXES); } -TlsAbstraction::~TlsAbstraction() +void TlsAbstraction::releaseSystemResources() { + cv::__termination = true; // DllMain is missing in static builds + disposed = true; #ifndef CV_USE_FLS TlsFree(tlsKey); #else // CV_USE_FLS @@ -1532,16 +1551,20 @@ TlsAbstraction::~TlsAbstraction() #endif // CV_USE_FLS tlsKey = TLS_OUT_OF_INDEXES; } -void* TlsAbstraction::getData_() const +void* TlsAbstraction::getData() const { + if (disposed) + return NULL; #ifndef CV_USE_FLS return TlsGetValue(tlsKey); #else // CV_USE_FLS return FlsGetValue(tlsKey); #endif // CV_USE_FLS } -void TlsAbstraction::setData_(void *pData) +void TlsAbstraction::setData(void *pData) { + if (disposed) + return; // no-op #ifndef CV_USE_FLS CV_Assert(TlsSetValue(tlsKey, pData) == TRUE); #else // CV_USE_FLS @@ -1552,11 +1575,14 @@ void TlsAbstraction::setData_(void *pData) #else // _WIN32 static void opencv_tls_destructor(void* pData); TlsAbstraction::TlsAbstraction() + : disposed(false) { CV_Assert(pthread_key_create(&tlsKey, opencv_tls_destructor) == 0); } -TlsAbstraction::~TlsAbstraction() +void TlsAbstraction::releaseSystemResources() { + cv::__termination = true; // DllMain is missing in static builds + disposed = true; if (pthread_key_delete(tlsKey) != 0) { // Don't use logging here @@ -1564,12 +1590,16 @@ TlsAbstraction::~TlsAbstraction() fflush(stderr); } } -void* TlsAbstraction::getData_() const +void* TlsAbstraction::getData() const { + if (disposed) + return NULL; return pthread_getspecific(tlsKey); } -void TlsAbstraction::setData_(void *pData) +void TlsAbstraction::setData(void *pData) { + if (disposed) + return; // no-op CV_Assert(pthread_setspecific(tlsKey, pData) == 0); } #endif @@ -1597,6 +1627,7 @@ class TlsStorage TlsStorage() : tlsSlotsSize(0) { + (void)getTlsAbstraction(); // ensure singeton initialization (for correct order of atexit calls) tlsSlots.reserve(32); threads.reserve(32); g_isTlsStorageInitialized = true; @@ -1834,11 +1865,11 @@ static void WINAPI opencv_fls_destructor(void* pData) #endif // CV_USE_FLS #endif // _WIN32 -static TlsAbstraction* const g_force_initialization_of_TlsAbstraction +static TlsStorage* const g_force_initialization_of_TlsStorage #if defined __GNUC__ __attribute__((unused)) #endif - = getTlsAbstraction(); + = &getTlsStorage(); } // namespace details using namespace details; From 9c84749e2c28d0744376e515a159c3cf30bb43c4 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Wed, 6 Oct 2021 16:09:20 +0300 Subject: [PATCH 278/376] backport YOLOv4x-mish new_coords CUDA implementation --- modules/dnn/src/cuda/region.cu | 78 +++++++++++++------ modules/dnn/src/cuda4dnn/kernels/region.hpp | 2 +- .../dnn/src/cuda4dnn/primitives/region.hpp | 6 +- modules/dnn/src/layers/region_layer.cpp | 5 +- modules/dnn/test/test_darknet_importer.cpp | 10 ++- 5 files changed, 73 insertions(+), 28 deletions(-) diff --git a/modules/dnn/src/cuda/region.cu b/modules/dnn/src/cuda/region.cu index 06b44abe9c86..3700a93d997c 100644 --- a/modules/dnn/src/cuda/region.cu +++ b/modules/dnn/src/cuda/region.cu @@ -31,7 +31,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { size_type boxes_per_cell, size_type box_size, size_type rows, size_type cols, T scale_x_y, size_type height_norm, size_type width_norm, - T object_prob_cutoff) + T object_prob_cutoff, bool new_coords) { using vector2_type = get_vector_type_t; auto bias_vPtr = vector2_type::get_pointer(bias.data()); @@ -47,22 +47,43 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { const auto y = (box_index % batch_inner_size) / row_inner_size; const auto x = (box_index % row_inner_size) / col_inner_size; - using device::fast_sigmoid; - const auto tmp_x = (fast_sigmoid(input[box_offset + 0]) - static_cast(0.5)) * scale_x_y + static_cast(0.5); - const auto tmp_y = (fast_sigmoid(input[box_offset + 1]) - static_cast(0.5)) * scale_x_y + static_cast(0.5); - output[box_offset + 0] = (T(x) + tmp_x) / T(cols); - output[box_offset + 1] = (T(y) + tmp_y) / T(rows); + /* When new_coords is true, we shouldn't use logistic activation again */ + T objectness_prob; + if (new_coords) + { + const auto tmp_x = (input[box_offset + 0] - static_cast(0.5)) * scale_x_y + static_cast(0.5); + const auto tmp_y = (input[box_offset + 1] - static_cast(0.5)) * scale_x_y + static_cast(0.5); - vector2_type bias_xy; - v_load(bias_xy, bias_vPtr[box_of_the_cell]); + output[box_offset + 0] = fast_divide_ftz(static_cast(x) + tmp_x, static_cast(cols)); + output[box_offset + 1] = fast_divide_ftz(static_cast(y) + tmp_y, static_cast(rows)); - using device::fast_exp; - output[box_offset + 2] = fast_exp(input[box_offset + 2]) * bias_xy.data[0] / T(width_norm); - output[box_offset + 3] = fast_exp(input[box_offset + 3]) * bias_xy.data[1] / T(height_norm); + vector2_type bias_xy; + v_load(bias_xy, bias_vPtr[box_of_the_cell]); - /* squash objectness score into a probability */ - using device::fast_sigmoid; - T objectness_prob = fast_sigmoid(input[box_offset + 4]); + output[box_offset + 2] = input[box_offset + 2] * input[box_offset + 2] * + static_cast(4) * bias_xy.data[0] / static_cast(width_norm); + output[box_offset + 3] = input[box_offset + 3] * input[box_offset + 3] * + static_cast(4) * bias_xy.data[1] / static_cast(height_norm); + + objectness_prob = input[box_offset + 4]; + } + else + { + const auto tmp_x = (fast_sigmoid(input[box_offset + 0]) - static_cast(0.5)) * scale_x_y + static_cast(0.5); + const auto tmp_y = (fast_sigmoid(input[box_offset + 1]) - static_cast(0.5)) * scale_x_y + static_cast(0.5); + + output[box_offset + 0] = fast_divide_ftz(static_cast(x) + tmp_x, static_cast(cols)); + output[box_offset + 1] = fast_divide_ftz(static_cast(y) + tmp_y, static_cast(rows)); + + vector2_type bias_xy; + v_load(bias_xy, bias_vPtr[box_of_the_cell]); + + output[box_offset + 2] = fast_exp(input[box_offset + 2]) * bias_xy.data[0] / static_cast(width_norm); + output[box_offset + 3] = fast_exp(input[box_offset + 3]) * bias_xy.data[1] / static_cast(height_norm); + + /* squash objectness score into a probability */ + objectness_prob = fast_sigmoid(input[box_offset + 4]); + } /* ignore prediction if the objectness probability is less than the cutoff */ if (objectness_prob < object_prob_cutoff) @@ -73,7 +94,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { } template - __global__ void region_sigmoid_class_score(Span output, View input, T class_prob_cutoff, size_type box_size) + __global__ void region_sigmoid_class_score(Span output, View input, T class_prob_cutoff, + size_type box_size, bool new_coords) { for (auto idx : grid_stride_range(output.size())) { const index_type box_no = idx / box_size; @@ -92,9 +114,20 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { * * to obtain the actual class probability, we multiply the conditional probability * with the object probability + * + * when new_coords is true, we shouldn't use logistic activation again. */ - using device::fast_sigmoid; - auto actual_class_prob = objectness_prob * fast_sigmoid(input[idx]); + + T actual_class_prob; + if (new_coords) + { + actual_class_prob = objectness_prob * input[idx]; + } + else + { + actual_class_prob = objectness_prob * fast_sigmoid(input[idx]); + } + if (actual_class_prob <= class_prob_cutoff) actual_class_prob = T(0); output[idx] = actual_class_prob; @@ -147,7 +180,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { std::size_t boxes_per_cell, std::size_t box_size, std::size_t rows, std::size_t cols, T scale_x_y, std::size_t height_norm, std::size_t width_norm, - bool if_true_sigmoid_else_softmax /* true = sigmoid, false = softmax */) + bool if_true_sigmoid_else_softmax, /* true = sigmoid, false = softmax */ + bool new_coords) { CV_Assert(output.size() == input.size()); CV_Assert(output.size() % box_size == 0); @@ -158,12 +192,12 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { launch_kernel(box_kernel, box_policy, output, input, bias, boxes_per_cell, box_size, rows, cols, scale_x_y, height_norm, width_norm, - object_prob_cutoff); + object_prob_cutoff, new_coords); if (if_true_sigmoid_else_softmax) { auto kernel_score = raw::region_sigmoid_class_score; auto policy_score = make_policy(kernel_score, output.size(), 0, stream); - launch_kernel(kernel_score, policy_score, output, input, class_prob_cutoff, box_size); + launch_kernel(kernel_score, policy_score, output, input, class_prob_cutoff, box_size, new_coords); } else { auto kernel_score = raw::region_softmax_class_score; auto policy_score = make_policy(kernel_score, output.size(), 0, stream); @@ -173,10 +207,10 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { #if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) template void region(const Stream&, Span<__half>, View<__half>, View<__half>, - __half, __half, std::size_t, std::size_t, std::size_t, std::size_t, __half, std::size_t, std::size_t, bool); + __half, __half, std::size_t, std::size_t, std::size_t, std::size_t, __half, std::size_t, std::size_t, bool, bool); #endif template void region(const Stream&, Span, View, View, - float, float, std::size_t, std::size_t, std::size_t, std::size_t, float, std::size_t, std::size_t, bool); + float, float, std::size_t, std::size_t, std::size_t, std::size_t, float, std::size_t, std::size_t, bool, bool); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda4dnn/kernels/region.hpp b/modules/dnn/src/cuda4dnn/kernels/region.hpp index 87742d2f81fa..b815fb11c9c3 100644 --- a/modules/dnn/src/cuda4dnn/kernels/region.hpp +++ b/modules/dnn/src/cuda4dnn/kernels/region.hpp @@ -18,7 +18,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { std::size_t boxes_per_cell, std::size_t box_size, std::size_t rows, std::size_t cols, T scale_x_y, std::size_t height_norm, std::size_t width_norm, - bool if_true_sigmoid_else_softmax); + bool if_true_sigmoid_else_softmax, bool new_coords); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda4dnn/primitives/region.hpp b/modules/dnn/src/cuda4dnn/primitives/region.hpp index 7813a47bc759..d22d44214e7b 100644 --- a/modules/dnn/src/cuda4dnn/primitives/region.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/region.hpp @@ -60,6 +60,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { T class_prob_cutoff; T nms_iou_threshold; + bool new_coords; }; template @@ -87,6 +88,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { class_prob_cutoff = config.class_prob_cutoff; nms_iou_threshold = config.nms_iou_threshold; + new_coords = config.new_coords; } void forward( @@ -115,7 +117,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { boxes_per_cell, cell_box_size, rows, cols, scale_x_y, height_norm, width_norm, - if_true_sigmoid_else_softmax + if_true_sigmoid_else_softmax, + new_coords ); if (nms_iou_threshold > 0) { @@ -176,6 +179,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { T object_prob_cutoff, class_prob_cutoff; T nms_iou_threshold; + bool new_coords; }; }}} /* namespace cv::dnn::cuda4dnn */ diff --git a/modules/dnn/src/layers/region_layer.cpp b/modules/dnn/src/layers/region_layer.cpp index 242e5e6f881b..73ed53974fd6 100644 --- a/modules/dnn/src/layers/region_layer.cpp +++ b/modules/dnn/src/layers/region_layer.cpp @@ -125,7 +125,7 @@ class RegionLayerImpl CV_FINAL : public RegionLayer #endif #ifdef HAVE_CUDA if (backendId == DNN_BACKEND_CUDA) - return new_coords == 0; + return true; #endif return backendId == DNN_BACKEND_OPENCV; } @@ -437,11 +437,12 @@ class RegionLayerImpl CV_FINAL : public RegionLayer config.scale_x_y = scale_x_y; - config.object_prob_cutoff = (classfix == -1) ? 0.5 : 0.0; + config.object_prob_cutoff = (classfix == -1) ? thresh : 0.f; config.class_prob_cutoff = thresh; config.nms_iou_threshold = nmsThreshold; + config.new_coords = (new_coords == 1); return make_cuda_node(preferableTarget, std::move(context->stream), blobs[0], config); } #endif diff --git a/modules/dnn/test/test_darknet_importer.cpp b/modules/dnn/test/test_darknet_importer.cpp index c85981a8bc97..d822efa88939 100644 --- a/modules/dnn/test/test_darknet_importer.cpp +++ b/modules/dnn/test/test_darknet_importer.cpp @@ -745,8 +745,14 @@ TEST_P(Test_Darknet_nets, YOLOv4x_mish) }; Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); - double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.006 : 8e-5; - double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.042 : 3e-4; + double scoreDiff = 8e-5; + double iouDiff = 3e-4; + + if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD || target == DNN_TARGET_CUDA_FP16) + { + scoreDiff = 0.006; + iouDiff = 0.042; + } std::string config_file = "yolov4x-mish.cfg"; std::string weights_file = "yolov4x-mish.weights"; From b1cf5501233405de3ea5926d1d688e421b337458 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 8 Oct 2021 18:31:56 +0000 Subject: [PATCH 279/376] release: OpenCV 3.4.16 --- modules/core/include/opencv2/core/version.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index dc8a3e603cb3..d05f5219f026 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -8,7 +8,7 @@ #define CV_VERSION_MAJOR 3 #define CV_VERSION_MINOR 4 #define CV_VERSION_REVISION 16 -#define CV_VERSION_STATUS "-pre" +#define CV_VERSION_STATUS "" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) From 34d359fe035a92d48a399c6e6975c77513bd5139 Mon Sep 17 00:00:00 2001 From: Yuantao Feng Date: Sat, 9 Oct 2021 03:13:49 +0800 Subject: [PATCH 280/376] Merge pull request #20422 from fengyuentau:dnn_face Add DNN-based face detection and face recognition into modules/objdetect * Add DNN-based face detector impl and interface * Add a sample for DNN-based face detector * add recog * add notes * move samples from samples/cpp to samples/dnn * add documentation for dnn_face * add set/get methods for input size, nms & score threshold and topk * remove the DNN prefix from the face detector and face recognizer * remove default values in the constructor of impl * regenerate priors after setting input size * two filenames for readnet * Update face.hpp * Update face_recognize.cpp * Update face_match.cpp * Update face.hpp * Update face_recognize.cpp * Update face_match.cpp * Update face_recognize.cpp * Update dnn_face.markdown * Update dnn_face.markdown * Update face.hpp * Update dnn_face.markdown * add regression test for face detection * remove underscore prefix; fix warnings * add reference & acknowledgement for face detection * Update dnn_face.markdown * Update dnn_face.markdown * Update ts.hpp * Update test_face.cpp * Update face_match.cpp * fix a compile error for python interface; add python examples for face detection and recognition * Major changes for Vadim's comments: * Replace class name FaceDetector with FaceDetectorYN in related failes * Declare local mat before loop in modules/objdetect/src/face_detect.cpp * Make input image and save flag optional in samples/dnn/face_detect(.cpp, .py) * Add camera support in samples/dnn/face_detect(.cpp, .py) * correct file paths for regression test * fix convertion warnings; remove extra spaces * update face_recog * Update dnn_face.markdown * Fix warnings and errors for the default CI reports: * Remove trailing white spaces and extra new lines. * Fix convertion warnings for windows and iOS. * Add braces around initialization of subobjects. * Fix warnings and errors for the default CI systems: * Add prefix 'FR_' for each value name in enum DisType to solve the redefinition error for iOS compilation; Modify other code accordingly * Add bookmark '#tutorial_dnn_face' to solve warnings from doxygen * Correct documentations to solve warnings from doxygen * update FaceRecognizerSF * Fix the error for CI to find ONNX models correctly * add suffix f to float assignments * add backend & target options for initializing face recognizer * add checkeq for checking input size and preset size * update test and threshold * changes in response to alalek's comments: * fix typos in samples/dnn/face_match.py * import numpy before importing cv2 * add documentation to .setInputSize() * remove extra include in face_recognize.cpp * fix some bugs * Update dnn_face.markdown * update thresholds; remove useless code * add time suffix to YuNet filename in test * objdetect: update test code --- doc/tutorials/dnn/dnn_face/dnn_face.markdown | 95 ++++++ .../dnn_text_spotting.markdown | 2 +- .../dnn/table_of_content_dnn.markdown | 1 + modules/objdetect/CMakeLists.txt | 2 +- .../objdetect/include/opencv2/objdetect.hpp | 1 + .../include/opencv2/objdetect/face.hpp | 125 ++++++++ modules/objdetect/src/face_detect.cpp | 288 ++++++++++++++++++ modules/objdetect/src/face_recognize.cpp | 182 +++++++++++ modules/objdetect/test/test_face.cpp | 219 +++++++++++++ modules/objdetect/test/test_main.cpp | 17 +- modules/ts/include/opencv2/ts.hpp | 1 + samples/dnn/CMakeLists.txt | 1 + samples/dnn/face_detect.cpp | 132 ++++++++ samples/dnn/face_detect.py | 101 ++++++ samples/dnn/face_match.cpp | 103 +++++++ samples/dnn/face_match.py | 57 ++++ samples/dnn/results/audrybt1.jpg | Bin 0 -> 47680 bytes 17 files changed, 1324 insertions(+), 3 deletions(-) create mode 100644 doc/tutorials/dnn/dnn_face/dnn_face.markdown create mode 100644 modules/objdetect/include/opencv2/objdetect/face.hpp create mode 100644 modules/objdetect/src/face_detect.cpp create mode 100644 modules/objdetect/src/face_recognize.cpp create mode 100644 modules/objdetect/test/test_face.cpp create mode 100644 samples/dnn/face_detect.cpp create mode 100644 samples/dnn/face_detect.py create mode 100644 samples/dnn/face_match.cpp create mode 100644 samples/dnn/face_match.py create mode 100644 samples/dnn/results/audrybt1.jpg diff --git a/doc/tutorials/dnn/dnn_face/dnn_face.markdown b/doc/tutorials/dnn/dnn_face/dnn_face.markdown new file mode 100644 index 000000000000..e5092b8b92e8 --- /dev/null +++ b/doc/tutorials/dnn/dnn_face/dnn_face.markdown @@ -0,0 +1,95 @@ +# DNN-based Face Detection And Recognition {#tutorial_dnn_face} + +@tableofcontents + +@prev_tutorial{tutorial_dnn_text_spotting} +@next_tutorial{pytorch_cls_tutorial_dnn_conversion} + +| | | +| -: | :- | +| Original Author | Chengrui Wang, Yuantao Feng | +| Compatibility | OpenCV >= 4.5.1 | + +## Introduction + +In this section, we introduce the DNN-based module for face detection and face recognition. Models can be obtained in [Models](#Models). The usage of `FaceDetectorYN` and `FaceRecognizer` are presented in [Usage](#Usage). + +## Models + +There are two models (ONNX format) pre-trained and required for this module: +- [Face Detection](https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx): + - Size: 337KB + - Results on WIDER Face Val set: 0.830(easy), 0.824(medium), 0.708(hard) +- [Face Recognition](https://drive.google.com/file/d/1ClK9WiB492c5OZFKveF3XiHCejoOxINW/view?usp=sharing) + - Size: 36.9MB + - Results: + + | Database | Accuracy | Threshold (normL2) | Threshold (cosine) | + | -------- | -------- | ------------------ | ------------------ | + | LFW | 99.60% | 1.128 | 0.363 | + | CALFW | 93.95% | 1.149 | 0.340 | + | CPLFW | 91.05% | 1.204 | 0.275 | + | AgeDB-30 | 94.90% | 1.202 | 0.277 | + | CFP-FP | 94.80% | 1.253 | 0.212 | + +## Usage + +### DNNFaceDetector + +```cpp +// Initialize FaceDetectorYN +Ptr faceDetector = FaceDetectorYN::create(onnx_path, "", image.size(), score_thresh, nms_thresh, top_k); + +// Forward +Mat faces; +faceDetector->detect(image, faces); +``` + +The detection output `faces` is a two-dimension array of type CV_32F, whose rows are the detected face instances, columns are the location of a face and 5 facial landmarks. The format of each row is as follows: + +``` +x1, y1, w, h, x_re, y_re, x_le, y_le, x_nt, y_nt, x_rcm, y_rcm, x_lcm, y_lcm +``` +, where `x1, y1, w, h` are the top-left coordinates, width and height of the face bounding box, `{x, y}_{re, le, nt, rcm, lcm}` stands for the coordinates of right eye, left eye, nose tip, the right corner and left corner of the mouth respectively. + + +### Face Recognition + +Following Face Detection, run codes below to extract face feature from facial image. + +```cpp +// Initialize FaceRecognizer with model path (cv::String) +Ptr faceRecognizer = FaceRecognizer::create(model_path, ""); + +// Aligning and cropping facial image through the first face of faces detected by dnn_face::DNNFaceDetector +Mat aligned_face; +faceRecognizer->alignCrop(image, faces.row(0), aligned_face); + +// Run feature extraction with given aligned_face (cv::Mat) +Mat feature; +faceRecognizer->feature(aligned_face, feature); +feature = feature.clone(); +``` + +After obtaining face features *feature1* and *feature2* of two facial images, run codes below to calculate the identity discrepancy between the two faces. + +```cpp +// Calculating the discrepancy between two face features by using cosine distance. +double cos_score = faceRecognizer->match(feature1, feature2, FaceRecognizer::DisType::COSINE); +// Calculating the discrepancy between two face features by using normL2 distance. +double L2_score = faceRecognizer->match(feature1, feature2, FaceRecognizer::DisType::NORM_L2); +``` + +For example, two faces have same identity if the cosine distance is greater than or equal to 0.363, or the normL2 distance is less than or equal to 1.128. + +## Reference: + +- https://github.com/ShiqiYu/libfacedetection +- https://github.com/ShiqiYu/libfacedetection.train +- https://github.com/zhongyy/SFace + +## Acknowledgement + +Thanks [Professor Shiqi Yu](https://github.com/ShiqiYu/) and [Yuantao Feng](https://github.com/fengyuentau) for training and providing the face detection model. + +Thanks [Professor Deng](http://www.whdeng.cn/), [PhD Candidate Zhong](https://github.com/zhongyy/) and [Master Candidate Wang](https://github.com/crywang/) for training and providing the face recognition model. diff --git a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown index b0be2627b291..5c465941ca42 100644 --- a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown +++ b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown @@ -3,7 +3,7 @@ @tableofcontents @prev_tutorial{tutorial_dnn_OCR} -@next_tutorial{pytorch_cls_tutorial_dnn_conversion} +@next_tutorial{tutorial_dnn_face} | | | | -: | :- | diff --git a/doc/tutorials/dnn/table_of_content_dnn.markdown b/doc/tutorials/dnn/table_of_content_dnn.markdown index 0d5e43ee11a7..3f74826dacc7 100644 --- a/doc/tutorials/dnn/table_of_content_dnn.markdown +++ b/doc/tutorials/dnn/table_of_content_dnn.markdown @@ -10,6 +10,7 @@ Deep Neural Networks (dnn module) {#tutorial_table_of_content_dnn} - @subpage tutorial_dnn_custom_layers - @subpage tutorial_dnn_OCR - @subpage tutorial_dnn_text_spotting +- @subpage tutorial_dnn_face #### PyTorch models with OpenCV In this section you will find the guides, which describe how to run classification, segmentation and detection PyTorch DNN models with OpenCV. diff --git a/modules/objdetect/CMakeLists.txt b/modules/objdetect/CMakeLists.txt index 3fa0c5d33b1a..f4d5b22b74f8 100644 --- a/modules/objdetect/CMakeLists.txt +++ b/modules/objdetect/CMakeLists.txt @@ -1,5 +1,5 @@ set(the_description "Object Detection") -ocv_define_module(objdetect opencv_core opencv_imgproc opencv_calib3d WRAP java objc python js) +ocv_define_module(objdetect opencv_core opencv_imgproc opencv_calib3d opencv_dnn WRAP java objc python js) if(HAVE_QUIRC) get_property(QUIRC_INCLUDE GLOBAL PROPERTY QUIRC_INCLUDE_DIR) diff --git a/modules/objdetect/include/opencv2/objdetect.hpp b/modules/objdetect/include/opencv2/objdetect.hpp index eaee1290ce8d..59dca7399b6a 100644 --- a/modules/objdetect/include/opencv2/objdetect.hpp +++ b/modules/objdetect/include/opencv2/objdetect.hpp @@ -768,5 +768,6 @@ class CV_EXPORTS_W QRCodeDetector } #include "opencv2/objdetect/detection_based_tracker.hpp" +#include "opencv2/objdetect/face.hpp" #endif diff --git a/modules/objdetect/include/opencv2/objdetect/face.hpp b/modules/objdetect/include/opencv2/objdetect/face.hpp new file mode 100644 index 000000000000..f2429c5f31fa --- /dev/null +++ b/modules/objdetect/include/opencv2/objdetect/face.hpp @@ -0,0 +1,125 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_OBJDETECT_FACE_HPP +#define OPENCV_OBJDETECT_FACE_HPP + +#include + +/** @defgroup dnn_face DNN-based face detection and recognition + */ + +namespace cv +{ + +/** @brief DNN-based face detector, model download link: https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx. + */ +class CV_EXPORTS_W FaceDetectorYN +{ +public: + virtual ~FaceDetectorYN() {}; + + /** @brief Set the size for the network input, which overwrites the input size of creating model. Call this method when the size of input image does not match the input size when creating model + * + * @param input_size the size of the input image + */ + CV_WRAP virtual void setInputSize(const Size& input_size) = 0; + + CV_WRAP virtual Size getInputSize() = 0; + + /** @brief Set the score threshold to filter out bounding boxes of score less than the given value + * + * @param score_threshold threshold for filtering out bounding boxes + */ + CV_WRAP virtual void setScoreThreshold(float score_threshold) = 0; + + CV_WRAP virtual float getScoreThreshold() = 0; + + /** @brief Set the Non-maximum-suppression threshold to suppress bounding boxes that have IoU greater than the given value + * + * @param nms_threshold threshold for NMS operation + */ + CV_WRAP virtual void setNMSThreshold(float nms_threshold) = 0; + + CV_WRAP virtual float getNMSThreshold() = 0; + + /** @brief Set the number of bounding boxes preserved before NMS + * + * @param top_k the number of bounding boxes to preserve from top rank based on score + */ + CV_WRAP virtual void setTopK(int top_k) = 0; + + CV_WRAP virtual int getTopK() = 0; + + /** @brief A simple interface to detect face from given image + * + * @param image an image to detect + * @param faces detection results stored in a cv::Mat + */ + CV_WRAP virtual int detect(InputArray image, OutputArray faces) = 0; + + /** @brief Creates an instance of this class with given parameters + * + * @param model the path to the requested model + * @param config the path to the config file for compability, which is not requested for ONNX models + * @param input_size the size of the input image + * @param score_threshold the threshold to filter out bounding boxes of score smaller than the given value + * @param nms_threshold the threshold to suppress bounding boxes of IoU bigger than the given value + * @param top_k keep top K bboxes before NMS + * @param backend_id the id of backend + * @param target_id the id of target device + */ + CV_WRAP static Ptr create(const String& model, + const String& config, + const Size& input_size, + float score_threshold = 0.9f, + float nms_threshold = 0.3f, + int top_k = 5000, + int backend_id = 0, + int target_id = 0); +}; + +/** @brief DNN-based face recognizer, model download link: https://drive.google.com/file/d/1ClK9WiB492c5OZFKveF3XiHCejoOxINW/view. + */ +class CV_EXPORTS_W FaceRecognizerSF +{ +public: + virtual ~FaceRecognizerSF() {}; + + /** @brief Definition of distance used for calculating the distance between two face features + */ + enum DisType { FR_COSINE=0, FR_NORM_L2=1 }; + + /** @brief Aligning image to put face on the standard position + * @param src_img input image + * @param face_box the detection result used for indicate face in input image + * @param aligned_img output aligned image + */ + CV_WRAP virtual void alignCrop(InputArray src_img, InputArray face_box, OutputArray aligned_img) const = 0; + + /** @brief Extracting face feature from aligned image + * @param aligned_img input aligned image + * @param face_feature output face feature + */ + CV_WRAP virtual void feature(InputArray aligned_img, OutputArray face_feature) = 0; + + /** @brief Calculating the distance between two face features + * @param _face_feature1 the first input feature + * @param _face_feature2 the second input feature of the same size and the same type as _face_feature1 + * @param dis_type defining the similarity with optional values "FR_OSINE" or "FR_NORM_L2" + */ + CV_WRAP virtual double match(InputArray _face_feature1, InputArray _face_feature2, int dis_type = FaceRecognizerSF::FR_COSINE) const = 0; + + /** @brief Creates an instance of this class with given parameters + * @param model the path of the onnx model used for face recognition + * @param config the path to the config file for compability, which is not requested for ONNX models + * @param backend_id the id of backend + * @param target_id the id of target device + */ + CV_WRAP static Ptr create(const String& model, const String& config, int backend_id = 0, int target_id = 0); +}; + +} // namespace cv + +#endif diff --git a/modules/objdetect/src/face_detect.cpp b/modules/objdetect/src/face_detect.cpp new file mode 100644 index 000000000000..4095745b7ecd --- /dev/null +++ b/modules/objdetect/src/face_detect.cpp @@ -0,0 +1,288 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +#include "opencv2/imgproc.hpp" +#include "opencv2/core.hpp" +#include "opencv2/dnn.hpp" + +#include + +namespace cv +{ + +class FaceDetectorYNImpl : public FaceDetectorYN +{ +public: + FaceDetectorYNImpl(const String& model, + const String& config, + const Size& input_size, + float score_threshold, + float nms_threshold, + int top_k, + int backend_id, + int target_id) + { + net = dnn::readNet(model, config); + CV_Assert(!net.empty()); + + net.setPreferableBackend(backend_id); + net.setPreferableTarget(target_id); + + inputW = input_size.width; + inputH = input_size.height; + + scoreThreshold = score_threshold; + nmsThreshold = nms_threshold; + topK = top_k; + + generatePriors(); + } + + void setInputSize(const Size& input_size) override + { + inputW = input_size.width; + inputH = input_size.height; + generatePriors(); + } + + Size getInputSize() override + { + Size input_size; + input_size.width = inputW; + input_size.height = inputH; + return input_size; + } + + void setScoreThreshold(float score_threshold) override + { + scoreThreshold = score_threshold; + } + + float getScoreThreshold() override + { + return scoreThreshold; + } + + void setNMSThreshold(float nms_threshold) override + { + nmsThreshold = nms_threshold; + } + + float getNMSThreshold() override + { + return nmsThreshold; + } + + void setTopK(int top_k) override + { + topK = top_k; + } + + int getTopK() override + { + return topK; + } + + int detect(InputArray input_image, OutputArray faces) override + { + // TODO: more checkings should be done? + if (input_image.empty()) + { + return 0; + } + CV_CheckEQ(input_image.size(), Size(inputW, inputH), "Size does not match. Call setInputSize(size) if input size does not match the preset size"); + + // Build blob from input image + Mat input_blob = dnn::blobFromImage(input_image); + + // Forward + std::vector output_names = { "loc", "conf", "iou" }; + std::vector output_blobs; + net.setInput(input_blob); + net.forward(output_blobs, output_names); + + // Post process + Mat results = postProcess(output_blobs); + results.convertTo(faces, CV_32FC1); + return 1; + } +private: + void generatePriors() + { + // Calculate shapes of different scales according to the shape of input image + Size feature_map_2nd = { + int(int((inputW+1)/2)/2), int(int((inputH+1)/2)/2) + }; + Size feature_map_3rd = { + int(feature_map_2nd.width/2), int(feature_map_2nd.height/2) + }; + Size feature_map_4th = { + int(feature_map_3rd.width/2), int(feature_map_3rd.height/2) + }; + Size feature_map_5th = { + int(feature_map_4th.width/2), int(feature_map_4th.height/2) + }; + Size feature_map_6th = { + int(feature_map_5th.width/2), int(feature_map_5th.height/2) + }; + + std::vector feature_map_sizes; + feature_map_sizes.push_back(feature_map_3rd); + feature_map_sizes.push_back(feature_map_4th); + feature_map_sizes.push_back(feature_map_5th); + feature_map_sizes.push_back(feature_map_6th); + + // Fixed params for generating priors + const std::vector> min_sizes = { + {10.0f, 16.0f, 24.0f}, + {32.0f, 48.0f}, + {64.0f, 96.0f}, + {128.0f, 192.0f, 256.0f} + }; + const std::vector steps = { 8, 16, 32, 64 }; + + // Generate priors + priors.clear(); + for (size_t i = 0; i < feature_map_sizes.size(); ++i) + { + Size feature_map_size = feature_map_sizes[i]; + std::vector min_size = min_sizes[i]; + + for (int _h = 0; _h < feature_map_size.height; ++_h) + { + for (int _w = 0; _w < feature_map_size.width; ++_w) + { + for (size_t j = 0; j < min_size.size(); ++j) + { + float s_kx = min_size[j] / inputW; + float s_ky = min_size[j] / inputH; + + float cx = (_w + 0.5f) * steps[i] / inputW; + float cy = (_h + 0.5f) * steps[i] / inputH; + + Rect2f prior = { cx, cy, s_kx, s_ky }; + priors.push_back(prior); + } + } + } + } + } + + Mat postProcess(const std::vector& output_blobs) + { + // Extract from output_blobs + Mat loc = output_blobs[0]; + Mat conf = output_blobs[1]; + Mat iou = output_blobs[2]; + + // Decode from deltas and priors + const std::vector variance = {0.1f, 0.2f}; + float* loc_v = (float*)(loc.data); + float* conf_v = (float*)(conf.data); + float* iou_v = (float*)(iou.data); + Mat faces; + // (tl_x, tl_y, w, h, re_x, re_y, le_x, le_y, nt_x, nt_y, rcm_x, rcm_y, lcm_x, lcm_y, score) + // 'tl': top left point of the bounding box + // 're': right eye, 'le': left eye + // 'nt': nose tip + // 'rcm': right corner of mouth, 'lcm': left corner of mouth + Mat face(1, 15, CV_32FC1); + for (size_t i = 0; i < priors.size(); ++i) { + // Get score + float clsScore = conf_v[i*2+1]; + float iouScore = iou_v[i]; + // Clamp + if (iouScore < 0.f) { + iouScore = 0.f; + } + else if (iouScore > 1.f) { + iouScore = 1.f; + } + float score = std::sqrt(clsScore * iouScore); + face.at(0, 14) = score; + + // Get bounding box + float cx = (priors[i].x + loc_v[i*14+0] * variance[0] * priors[i].width) * inputW; + float cy = (priors[i].y + loc_v[i*14+1] * variance[0] * priors[i].height) * inputH; + float w = priors[i].width * exp(loc_v[i*14+2] * variance[0]) * inputW; + float h = priors[i].height * exp(loc_v[i*14+3] * variance[1]) * inputH; + float x1 = cx - w / 2; + float y1 = cy - h / 2; + face.at(0, 0) = x1; + face.at(0, 1) = y1; + face.at(0, 2) = w; + face.at(0, 3) = h; + + // Get landmarks + face.at(0, 4) = (priors[i].x + loc_v[i*14+ 4] * variance[0] * priors[i].width) * inputW; // right eye, x + face.at(0, 5) = (priors[i].y + loc_v[i*14+ 5] * variance[0] * priors[i].height) * inputH; // right eye, y + face.at(0, 6) = (priors[i].x + loc_v[i*14+ 6] * variance[0] * priors[i].width) * inputW; // left eye, x + face.at(0, 7) = (priors[i].y + loc_v[i*14+ 7] * variance[0] * priors[i].height) * inputH; // left eye, y + face.at(0, 8) = (priors[i].x + loc_v[i*14+ 8] * variance[0] * priors[i].width) * inputW; // nose tip, x + face.at(0, 9) = (priors[i].y + loc_v[i*14+ 9] * variance[0] * priors[i].height) * inputH; // nose tip, y + face.at(0, 10) = (priors[i].x + loc_v[i*14+10] * variance[0] * priors[i].width) * inputW; // right corner of mouth, x + face.at(0, 11) = (priors[i].y + loc_v[i*14+11] * variance[0] * priors[i].height) * inputH; // right corner of mouth, y + face.at(0, 12) = (priors[i].x + loc_v[i*14+12] * variance[0] * priors[i].width) * inputW; // left corner of mouth, x + face.at(0, 13) = (priors[i].y + loc_v[i*14+13] * variance[0] * priors[i].height) * inputH; // left corner of mouth, y + + faces.push_back(face); + } + + if (faces.rows > 1) + { + // Retrieve boxes and scores + std::vector faceBoxes; + std::vector faceScores; + for (int rIdx = 0; rIdx < faces.rows; rIdx++) + { + faceBoxes.push_back(Rect2i(int(faces.at(rIdx, 0)), + int(faces.at(rIdx, 1)), + int(faces.at(rIdx, 2)), + int(faces.at(rIdx, 3)))); + faceScores.push_back(faces.at(rIdx, 14)); + } + + std::vector keepIdx; + dnn::NMSBoxes(faceBoxes, faceScores, scoreThreshold, nmsThreshold, keepIdx, 1.f, topK); + + // Get NMS results + Mat nms_faces; + for (int idx: keepIdx) + { + nms_faces.push_back(faces.row(idx)); + } + return nms_faces; + } + else + { + return faces; + } + } +private: + dnn::Net net; + + int inputW; + int inputH; + float scoreThreshold; + float nmsThreshold; + int topK; + + std::vector priors; +}; + +Ptr FaceDetectorYN::create(const String& model, + const String& config, + const Size& input_size, + const float score_threshold, + const float nms_threshold, + const int top_k, + const int backend_id, + const int target_id) +{ + return makePtr(model, config, input_size, score_threshold, nms_threshold, top_k, backend_id, target_id); +} + +} // namespace cv diff --git a/modules/objdetect/src/face_recognize.cpp b/modules/objdetect/src/face_recognize.cpp new file mode 100644 index 000000000000..6550a13b4b3a --- /dev/null +++ b/modules/objdetect/src/face_recognize.cpp @@ -0,0 +1,182 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +#include "opencv2/dnn.hpp" + +#include + +namespace cv +{ + +class FaceRecognizerSFImpl : public FaceRecognizerSF +{ +public: + FaceRecognizerSFImpl(const String& model, const String& config, int backend_id, int target_id) + { + net = dnn::readNet(model, config); + CV_Assert(!net.empty()); + + net.setPreferableBackend(backend_id); + net.setPreferableTarget(target_id); + }; + void alignCrop(InputArray _src_img, InputArray _face_mat, OutputArray _aligned_img) const override + { + Mat face_mat = _face_mat.getMat(); + float src_point[5][2]; + for (int row = 0; row < 5; ++row) + { + for(int col = 0; col < 2; ++col) + { + src_point[row][col] = face_mat.at(0, row*2+col+4); + } + } + Mat warp_mat = getSimilarityTransformMatrix(src_point); + warpAffine(_src_img, _aligned_img, warp_mat, Size(112, 112), INTER_LINEAR); + }; + void feature(InputArray _aligned_img, OutputArray _face_feature) override + { + Mat inputBolb = dnn::blobFromImage(_aligned_img, 1, Size(112, 112), Scalar(0, 0, 0), true, false); + net.setInput(inputBolb); + net.forward(_face_feature); + }; + double match(InputArray _face_feature1, InputArray _face_feature2, int dis_type) const override + { + Mat face_feature1 = _face_feature1.getMat(), face_feature2 = _face_feature2.getMat(); + face_feature1 /= norm(face_feature1); + face_feature2 /= norm(face_feature2); + + if(dis_type == DisType::FR_COSINE){ + return sum(face_feature1.mul(face_feature2))[0]; + }else if(dis_type == DisType::FR_NORM_L2){ + return norm(face_feature1, face_feature2); + }else{ + throw std::invalid_argument("invalid parameter " + std::to_string(dis_type)); + } + + }; + +private: + Mat getSimilarityTransformMatrix(float src[5][2]) const { + float dst[5][2] = { {38.2946f, 51.6963f}, {73.5318f, 51.5014f}, {56.0252f, 71.7366f}, {41.5493f, 92.3655f}, {70.7299f, 92.2041f} }; + float avg0 = (src[0][0] + src[1][0] + src[2][0] + src[3][0] + src[4][0]) / 5; + float avg1 = (src[0][1] + src[1][1] + src[2][1] + src[3][1] + src[4][1]) / 5; + //Compute mean of src and dst. + float src_mean[2] = { avg0, avg1 }; + float dst_mean[2] = { 56.0262f, 71.9008f }; + //Subtract mean from src and dst. + float src_demean[5][2]; + for (int i = 0; i < 2; i++) + { + for (int j = 0; j < 5; j++) + { + src_demean[j][i] = src[j][i] - src_mean[i]; + } + } + float dst_demean[5][2]; + for (int i = 0; i < 2; i++) + { + for (int j = 0; j < 5; j++) + { + dst_demean[j][i] = dst[j][i] - dst_mean[i]; + } + } + double A00 = 0.0, A01 = 0.0, A10 = 0.0, A11 = 0.0; + for (int i = 0; i < 5; i++) + A00 += dst_demean[i][0] * src_demean[i][0]; + A00 = A00 / 5; + for (int i = 0; i < 5; i++) + A01 += dst_demean[i][0] * src_demean[i][1]; + A01 = A01 / 5; + for (int i = 0; i < 5; i++) + A10 += dst_demean[i][1] * src_demean[i][0]; + A10 = A10 / 5; + for (int i = 0; i < 5; i++) + A11 += dst_demean[i][1] * src_demean[i][1]; + A11 = A11 / 5; + Mat A = (Mat_(2, 2) << A00, A01, A10, A11); + double d[2] = { 1.0, 1.0 }; + double detA = A00 * A11 - A01 * A10; + if (detA < 0) + d[1] = -1; + double T[3][3] = { {1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0} }; + Mat s, u, vt, v; + SVD::compute(A, s, u, vt); + double smax = s.ptr(0)[0]>s.ptr(1)[0] ? s.ptr(0)[0] : s.ptr(1)[0]; + double tol = smax * 2 * FLT_MIN; + int rank = 0; + if (s.ptr(0)[0]>tol) + rank += 1; + if (s.ptr(1)[0]>tol) + rank += 1; + double arr_u[2][2] = { {u.ptr(0)[0], u.ptr(0)[1]}, {u.ptr(1)[0], u.ptr(1)[1]} }; + double arr_vt[2][2] = { {vt.ptr(0)[0], vt.ptr(0)[1]}, {vt.ptr(1)[0], vt.ptr(1)[1]} }; + double det_u = arr_u[0][0] * arr_u[1][1] - arr_u[0][1] * arr_u[1][0]; + double det_vt = arr_vt[0][0] * arr_vt[1][1] - arr_vt[0][1] * arr_vt[1][0]; + if (rank == 1) + { + if ((det_u*det_vt) > 0) + { + Mat uvt = u*vt; + T[0][0] = uvt.ptr(0)[0]; + T[0][1] = uvt.ptr(0)[1]; + T[1][0] = uvt.ptr(1)[0]; + T[1][1] = uvt.ptr(1)[1]; + } + else + { + double temp = d[1]; + d[1] = -1; + Mat D = (Mat_(2, 2) << d[0], 0.0, 0.0, d[1]); + Mat Dvt = D*vt; + Mat uDvt = u*Dvt; + T[0][0] = uDvt.ptr(0)[0]; + T[0][1] = uDvt.ptr(0)[1]; + T[1][0] = uDvt.ptr(1)[0]; + T[1][1] = uDvt.ptr(1)[1]; + d[1] = temp; + } + } + else + { + Mat D = (Mat_(2, 2) << d[0], 0.0, 0.0, d[1]); + Mat Dvt = D*vt; + Mat uDvt = u*Dvt; + T[0][0] = uDvt.ptr(0)[0]; + T[0][1] = uDvt.ptr(0)[1]; + T[1][0] = uDvt.ptr(1)[0]; + T[1][1] = uDvt.ptr(1)[1]; + } + double var1 = 0.0; + for (int i = 0; i < 5; i++) + var1 += src_demean[i][0] * src_demean[i][0]; + var1 = var1 / 5; + double var2 = 0.0; + for (int i = 0; i < 5; i++) + var2 += src_demean[i][1] * src_demean[i][1]; + var2 = var2 / 5; + double scale = 1.0 / (var1 + var2)* (s.ptr(0)[0] * d[0] + s.ptr(1)[0] * d[1]); + double TS[2]; + TS[0] = T[0][0] * src_mean[0] + T[0][1] * src_mean[1]; + TS[1] = T[1][0] * src_mean[0] + T[1][1] * src_mean[1]; + T[0][2] = dst_mean[0] - scale*TS[0]; + T[1][2] = dst_mean[1] - scale*TS[1]; + T[0][0] *= scale; + T[0][1] *= scale; + T[1][0] *= scale; + T[1][1] *= scale; + Mat transform_mat = (Mat_(2, 3) << T[0][0], T[0][1], T[0][2], T[1][0], T[1][1], T[1][2]); + return transform_mat; + } +private: + dnn::Net net; +}; + +Ptr FaceRecognizerSF::create(const String& model, const String& config, int backend_id, int target_id) +{ + return makePtr(model, config, backend_id, target_id); +} + +} // namespace cv diff --git a/modules/objdetect/test/test_face.cpp b/modules/objdetect/test/test_face.cpp new file mode 100644 index 000000000000..2e944c50df0a --- /dev/null +++ b/modules/objdetect/test/test_face.cpp @@ -0,0 +1,219 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + +// label format: +// image_name +// num_face +// face_1 +// face_.. +// face_num +std::map blobFromTXT(const std::string& path, int numCoords) +{ + std::ifstream ifs(path.c_str()); + CV_Assert(ifs.is_open()); + + std::map gt; + + Mat faces; + int faceNum = -1; + int faceCount = 0; + for (std::string line, key; getline(ifs, line); ) + { + std::istringstream iss(line); + if (line.find(".png") != std::string::npos) + { + // Get filename + iss >> key; + } + else if (line.find(" ") == std::string::npos) + { + // Get the number of faces + iss >> faceNum; + } + else + { + // Get faces + Mat face(1, numCoords, CV_32FC1); + for (int j = 0; j < numCoords; j++) + { + iss >> face.at(0, j); + } + faces.push_back(face); + faceCount++; + } + + if (faceCount == faceNum) + { + // Store faces + gt[key] = faces; + + faces.release(); + faceNum = -1; + faceCount = 0; + } + } + + return gt; +} + +TEST(Objdetect_face_detection, regression) +{ + // Pre-set params + float scoreThreshold = 0.7f; + float matchThreshold = 0.9f; + float l2disThreshold = 5.0f; + int numLM = 5; + int numCoords = 4 + 2 * numLM; + + // Load ground truth labels + std::map gt = blobFromTXT(findDataFile("dnn_face/detection/cascades_labels.txt"), numCoords); + // for (auto item: gt) + // { + // std::cout << item.first << " " << item.second.size() << std::endl; + // } + + // Initialize detector + std::string model = findDataFile("dnn/onnx/models/yunet-202109.onnx", false); + Ptr faceDetector = FaceDetectorYN::create(model, "", Size(300, 300)); + faceDetector->setScoreThreshold(0.7f); + + // Detect and match + for (auto item: gt) + { + std::string imagePath = findDataFile("cascadeandhog/images/" + item.first); + Mat image = imread(imagePath); + + // Set input size + faceDetector->setInputSize(image.size()); + + // Run detection + Mat faces; + faceDetector->detect(image, faces); + // std::cout << item.first << " " << item.second.rows << " " << faces.rows << std::endl; + + // Match bboxes and landmarks + std::vector matchedItem(item.second.rows, false); + for (int i = 0; i < faces.rows; i++) + { + if (faces.at(i, numCoords) < scoreThreshold) + continue; + + bool boxMatched = false; + std::vector lmMatched(numLM, false); + cv::Rect2f resBox(faces.at(i, 0), faces.at(i, 1), faces.at(i, 2), faces.at(i, 3)); + for (int j = 0; j < item.second.rows && !boxMatched; j++) + { + if (matchedItem[j]) + continue; + + // Retrieve bbox and compare IoU + cv::Rect2f gtBox(item.second.at(j, 0), item.second.at(j, 1), item.second.at(j, 2), item.second.at(j, 3)); + double interArea = (resBox & gtBox).area(); + double iou = interArea / (resBox.area() + gtBox.area() - interArea); + if (iou >= matchThreshold) + { + boxMatched = true; + matchedItem[j] = true; + } + + // Match landmarks if bbox is matched + if (!boxMatched) + continue; + for (int lmIdx = 0; lmIdx < numLM; lmIdx++) + { + float gtX = item.second.at(j, 4 + 2 * lmIdx); + float gtY = item.second.at(j, 4 + 2 * lmIdx + 1); + float resX = faces.at(i, 4 + 2 * lmIdx); + float resY = faces.at(i, 4 + 2 * lmIdx + 1); + float l2dis = cv::sqrt((gtX - resX) * (gtX - resX) + (gtY - resY) * (gtY - resY)); + + if (l2dis <= l2disThreshold) + { + lmMatched[lmIdx] = true; + } + } + } + EXPECT_TRUE(boxMatched) << "In image " << item.first << ", cannot match resBox " << resBox << " with any ground truth."; + if (boxMatched) + { + EXPECT_TRUE(std::all_of(lmMatched.begin(), lmMatched.end(), [](bool v) { return v; })) << "In image " << item.first << ", resBox " << resBox << " matched but its landmarks failed to match."; + } + } + } +} + +TEST(Objdetect_face_recognition, regression) +{ + // Pre-set params + float score_thresh = 0.9f; + float nms_thresh = 0.3f; + double cosine_similar_thresh = 0.363; + double l2norm_similar_thresh = 1.128; + + // Load ground truth labels + std::ifstream ifs(findDataFile("dnn_face/recognition/cascades_label.txt").c_str()); + CV_Assert(ifs.is_open()); + + std::set fSet; + std::map featureMap; + std::map, int> gtMap; + + + for (std::string line, key; getline(ifs, line);) + { + std::string fname1, fname2; + int label; + std::istringstream iss(line); + iss>>fname1>>fname2>>label; + // std::cout< faceDetector = FaceDetectorYN::create(detect_model, "", Size(150, 150), score_thresh, nms_thresh); + + std::string recog_model = findDataFile("dnn/onnx/models/face_recognizer_fast.onnx", false); + Ptr faceRecognizer = FaceRecognizerSF::create(recog_model, ""); + + // Detect and match + for (auto fname: fSet) + { + std::string imagePath = findDataFile("dnn_face/recognition/" + fname); + Mat image = imread(imagePath); + + Mat faces; + faceDetector->detect(image, faces); + + Mat aligned_face; + faceRecognizer->alignCrop(image, faces.row(0), aligned_face); + + Mat feature; + faceRecognizer->feature(aligned_face, feature); + + featureMap[fname] = feature.clone(); + } + + for (auto item: gtMap) + { + Mat feature1 = featureMap[item.first.first]; + Mat feature2 = featureMap[item.first.second]; + int label = item.second; + + double cos_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_COSINE); + double L2_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_NORM_L2); + + EXPECT_TRUE(label == 0 ? cos_score <= cosine_similar_thresh : cos_score > cosine_similar_thresh) << "Cosine match result of images " << item.first.first << " and " << item.first.second << " is different from ground truth (score: "<< cos_score <<";Thresh: "<< cosine_similar_thresh <<")."; + EXPECT_TRUE(label == 0 ? L2_score > l2norm_similar_thresh : L2_score <= l2norm_similar_thresh) << "L2norm match result of images " << item.first.first << " and " << item.first.second << " is different from ground truth (score: "<< L2_score <<";Thresh: "<< l2norm_similar_thresh <<")."; + } +} + +}} // namespace diff --git a/modules/objdetect/test/test_main.cpp b/modules/objdetect/test/test_main.cpp index 93e4d2860eb7..4031f0522b4b 100644 --- a/modules/objdetect/test/test_main.cpp +++ b/modules/objdetect/test/test_main.cpp @@ -7,4 +7,19 @@ #include #endif -CV_TEST_MAIN("cv") +static +void initTests() +{ +#ifdef HAVE_OPENCV_DNN + const char* extraTestDataPath = +#ifdef WINRT + NULL; +#else + getenv("OPENCV_DNN_TEST_DATA_PATH"); +#endif + if (extraTestDataPath) + cvtest::addDataSearchPath(extraTestDataPath); +#endif // HAVE_OPENCV_DNN +} + +CV_TEST_MAIN("cv", initTests()) diff --git a/modules/ts/include/opencv2/ts.hpp b/modules/ts/include/opencv2/ts.hpp index 394bc6e0faf9..2e7a241d8e19 100644 --- a/modules/ts/include/opencv2/ts.hpp +++ b/modules/ts/include/opencv2/ts.hpp @@ -37,6 +37,7 @@ #include #include #include +#include #ifndef OPENCV_32BIT_CONFIGURATION diff --git a/samples/dnn/CMakeLists.txt b/samples/dnn/CMakeLists.txt index 209fbb586c43..9a1aeed33980 100644 --- a/samples/dnn/CMakeLists.txt +++ b/samples/dnn/CMakeLists.txt @@ -4,6 +4,7 @@ set(OPENCV_DNN_SAMPLES_REQUIRED_DEPS opencv_core opencv_imgproc opencv_dnn + opencv_objdetect opencv_video opencv_imgcodecs opencv_videoio diff --git a/samples/dnn/face_detect.cpp b/samples/dnn/face_detect.cpp new file mode 100644 index 000000000000..8d91a109682f --- /dev/null +++ b/samples/dnn/face_detect.cpp @@ -0,0 +1,132 @@ +#include +#include +#include +#include + +#include + +using namespace cv; +using namespace std; + +static Mat visualize(Mat input, Mat faces, int thickness=2) +{ + Mat output = input.clone(); + for (int i = 0; i < faces.rows; i++) + { + // Print results + cout << "Face " << i + << ", top-left coordinates: (" << faces.at(i, 0) << ", " << faces.at(i, 1) << "), " + << "box width: " << faces.at(i, 2) << ", box height: " << faces.at(i, 3) << ", " + << "score: " << faces.at(i, 14) << "\n"; + + // Draw bounding box + rectangle(output, Rect2i(int(faces.at(i, 0)), int(faces.at(i, 1)), int(faces.at(i, 2)), int(faces.at(i, 3))), Scalar(0, 255, 0), thickness); + // Draw landmarks + circle(output, Point2i(int(faces.at(i, 4)), int(faces.at(i, 5))), 2, Scalar(255, 0, 0), thickness); + circle(output, Point2i(int(faces.at(i, 6)), int(faces.at(i, 7))), 2, Scalar( 0, 0, 255), thickness); + circle(output, Point2i(int(faces.at(i, 8)), int(faces.at(i, 9))), 2, Scalar( 0, 255, 0), thickness); + circle(output, Point2i(int(faces.at(i, 10)), int(faces.at(i, 11))), 2, Scalar(255, 0, 255), thickness); + circle(output, Point2i(int(faces.at(i, 12)), int(faces.at(i, 13))), 2, Scalar( 0, 255, 255), thickness); + } + return output; +} + +int main(int argc, char ** argv) +{ + CommandLineParser parser(argc, argv, + "{help h | | Print this message.}" + "{input i | | Path to the input image. Omit for detecting on default camera.}" + "{model m | yunet.onnx | Path to the model. Download yunet.onnx in https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx.}" + "{score_threshold | 0.9 | Filter out faces of score < score_threshold.}" + "{nms_threshold | 0.3 | Suppress bounding boxes of iou >= nms_threshold.}" + "{top_k | 5000 | Keep top_k bounding boxes before NMS.}" + "{save s | false | Set true to save results. This flag is invalid when using camera.}" + "{vis v | true | Set true to open a window for result visualization. This flag is invalid when using camera.}" + ); + if (argc == 1 || parser.has("help")) + { + parser.printMessage(); + return -1; + } + + String modelPath = parser.get("model"); + + float scoreThreshold = parser.get("score_threshold"); + float nmsThreshold = parser.get("nms_threshold"); + int topK = parser.get("top_k"); + + bool save = parser.get("save"); + bool vis = parser.get("vis"); + + // Initialize FaceDetectorYN + Ptr detector = FaceDetectorYN::create(modelPath, "", Size(320, 320), scoreThreshold, nmsThreshold, topK); + + // If input is an image + if (parser.has("input")) + { + String input = parser.get("input"); + Mat image = imread(input); + + // Set input size before inference + detector->setInputSize(image.size()); + + // Inference + Mat faces; + detector->detect(image, faces); + + // Draw results on the input image + Mat result = visualize(image, faces); + + // Save results if save is true + if(save) + { + cout << "Results saved to result.jpg\n"; + imwrite("result.jpg", result); + } + + // Visualize results + if (vis) + { + namedWindow(input, WINDOW_AUTOSIZE); + imshow(input, result); + waitKey(0); + } + } + else + { + int deviceId = 0; + VideoCapture cap; + cap.open(deviceId, CAP_ANY); + int frameWidth = int(cap.get(CAP_PROP_FRAME_WIDTH)); + int frameHeight = int(cap.get(CAP_PROP_FRAME_HEIGHT)); + detector->setInputSize(Size(frameWidth, frameHeight)); + + Mat frame; + TickMeter tm; + String msg = "FPS: "; + while(waitKey(1) < 0) // Press any key to exit + { + // Get frame + if (!cap.read(frame)) + { + cerr << "No frames grabbed!\n"; + break; + } + + // Inference + Mat faces; + tm.start(); + detector->detect(frame, faces); + tm.stop(); + + // Draw results on the input image + Mat result = visualize(frame, faces); + putText(result, msg + to_string(tm.getFPS()), Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + + // Visualize results + imshow("Live", result); + + tm.reset(); + } + } +} \ No newline at end of file diff --git a/samples/dnn/face_detect.py b/samples/dnn/face_detect.py new file mode 100644 index 000000000000..65069d6590da --- /dev/null +++ b/samples/dnn/face_detect.py @@ -0,0 +1,101 @@ +import argparse + +import numpy as np +import cv2 as cv + +def str2bool(v): + if v.lower() in ['on', 'yes', 'true', 'y', 't']: + return True + elif v.lower() in ['off', 'no', 'false', 'n', 'f']: + return False + else: + raise NotImplementedError + +parser = argparse.ArgumentParser() +parser.add_argument('--input', '-i', type=str, help='Path to the input image.') +parser.add_argument('--model', '-m', type=str, default='yunet.onnx', help='Path to the model. Download the model at https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx.') +parser.add_argument('--score_threshold', type=float, default=0.9, help='Filtering out faces of score < score_threshold.') +parser.add_argument('--nms_threshold', type=float, default=0.3, help='Suppress bounding boxes of iou >= nms_threshold.') +parser.add_argument('--top_k', type=int, default=5000, help='Keep top_k bounding boxes before NMS.') +parser.add_argument('--save', '-s', type=str2bool, default=False, help='Set true to save results. This flag is invalid when using camera.') +parser.add_argument('--vis', '-v', type=str2bool, default=True, help='Set true to open a window for result visualization. This flag is invalid when using camera.') +args = parser.parse_args() + +def visualize(input, faces, thickness=2): + output = input.copy() + if faces[1] is not None: + for idx, face in enumerate(faces[1]): + print('Face {}, top-left coordinates: ({:.0f}, {:.0f}), box width: {:.0f}, box height {:.0f}, score: {:.2f}'.format(idx, face[0], face[1], face[2], face[3], face[-1])) + + coords = face[:-1].astype(np.int32) + cv.rectangle(output, (coords[0], coords[1]), (coords[0]+coords[2], coords[1]+coords[3]), (0, 255, 0), 2) + cv.circle(output, (coords[4], coords[5]), 2, (255, 0, 0), 2) + cv.circle(output, (coords[6], coords[7]), 2, (0, 0, 255), 2) + cv.circle(output, (coords[8], coords[9]), 2, (0, 255, 0), 2) + cv.circle(output, (coords[10], coords[11]), 2, (255, 0, 255), 2) + cv.circle(output, (coords[12], coords[13]), 2, (0, 255, 255), 2) + return output + +if __name__ == '__main__': + + # Instantiate FaceDetectorYN + detector = cv.FaceDetectorYN.create( + args.model, + "", + (320, 320), + args.score_threshold, + args.nms_threshold, + args.top_k + ) + + # If input is an image + if args.input is not None: + image = cv.imread(args.input) + + # Set input size before inference + detector.setInputSize((image.shape[1], image.shape[0])) + + # Inference + faces = detector.detect(image) + + # Draw results on the input image + result = visualize(image, faces) + + # Save results if save is true + if args.save: + print('Resutls saved to result.jpg\n') + cv.imwrite('result.jpg', result) + + # Visualize results in a new window + if args.vis: + cv.namedWindow(args.input, cv.WINDOW_AUTOSIZE) + cv.imshow(args.input, result) + cv.waitKey(0) + else: # Omit input to call default camera + deviceId = 0 + cap = cv.VideoCapture(deviceId) + frameWidth = int(cap.get(cv.CAP_PROP_FRAME_WIDTH)) + frameHeight = int(cap.get(cv.CAP_PROP_FRAME_HEIGHT)) + detector.setInputSize([frameWidth, frameHeight]) + + tm = cv.TickMeter() + while cv.waitKey(1) < 0: + hasFrame, frame = cap.read() + if not hasFrame: + print('No frames grabbed!') + break + + # Inference + tm.start() + faces = detector.detect(frame) # faces is a tuple + tm.stop() + + # Draw results on the input image + frame = visualize(frame, faces) + + cv.putText(frame, 'FPS: {}'.format(tm.getFPS()), (0, 15), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0)) + + # Visualize results in a new Window + cv.imshow('Live', frame) + + tm.reset() \ No newline at end of file diff --git a/samples/dnn/face_match.cpp b/samples/dnn/face_match.cpp new file mode 100644 index 000000000000..f24134b8906d --- /dev/null +++ b/samples/dnn/face_match.cpp @@ -0,0 +1,103 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "opencv2/dnn.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/highgui.hpp" + +#include + +#include "opencv2/objdetect.hpp" + + +using namespace cv; +using namespace std; + + +int main(int argc, char ** argv) +{ + if (argc != 5) + { + std::cerr << "Usage " << argv[0] << ": " + << " " + << " " + << "" + << "\n"; + return -1; + } + + String det_onnx_path = argv[1]; + String reg_onnx_path = argv[2]; + String image1_path = argv[3]; + String image2_path = argv[4]; + std::cout< faceDetector; + + faceDetector = FaceDetectorYN::create(det_onnx_path, "", image1.size(), score_thresh, nms_thresh, top_k); + Mat faces_1; + faceDetector->detect(image1, faces_1); + if (faces_1.rows < 1) + { + std::cerr << "Cannot find a face in " << image1_path << "\n"; + return -1; + } + + faceDetector = FaceDetectorYN::create(det_onnx_path, "", image2.size(), score_thresh, nms_thresh, top_k); + Mat faces_2; + faceDetector->detect(image2, faces_2); + if (faces_2.rows < 1) + { + std::cerr << "Cannot find a face in " << image2_path << "\n"; + return -1; + } + + // Initialize FaceRecognizerSF + Ptr faceRecognizer = FaceRecognizerSF::create(reg_onnx_path, ""); + + + Mat aligned_face1, aligned_face2; + faceRecognizer->alignCrop(image1, faces_1.row(0), aligned_face1); + faceRecognizer->alignCrop(image2, faces_2.row(0), aligned_face2); + + Mat feature1, feature2; + faceRecognizer->feature(aligned_face1, feature1); + feature1 = feature1.clone(); + faceRecognizer->feature(aligned_face2, feature2); + feature2 = feature2.clone(); + + double cos_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_COSINE); + double L2_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_NORM_L2); + + if(cos_score >= cosine_similar_thresh) + { + std::cout << "They have the same identity;"; + } + else + { + std::cout << "They have different identities;"; + } + std::cout << " Cosine Similarity: " << cos_score << ", threshold: " << cosine_similar_thresh << ". (higher value means higher similarity, max 1.0)\n"; + + if(L2_score <= l2norm_similar_thresh) + { + std::cout << "They have the same identity;"; + } + else + { + std::cout << "They have different identities."; + } + std::cout << " NormL2 Distance: " << L2_score << ", threshold: " << l2norm_similar_thresh << ". (lower value means higher similarity, min 0.0)\n"; + + return 0; +} diff --git a/samples/dnn/face_match.py b/samples/dnn/face_match.py new file mode 100644 index 000000000000..b36c9f6367af --- /dev/null +++ b/samples/dnn/face_match.py @@ -0,0 +1,57 @@ +import argparse + +import numpy as np +import cv2 as cv + +parser = argparse.ArgumentParser() +parser.add_argument('--input1', '-i1', type=str, help='Path to the input image1.') +parser.add_argument('--input2', '-i2', type=str, help='Path to the input image2.') +parser.add_argument('--face_detection_model', '-fd', type=str, help='Path to the face detection model. Download the model at https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx.') +parser.add_argument('--face_recognition_model', '-fr', type=str, help='Path to the face recognition model. Download the model at https://drive.google.com/file/d/1ClK9WiB492c5OZFKveF3XiHCejoOxINW/view.') +args = parser.parse_args() + +# Read the input image +img1 = cv.imread(args.input1) +img2 = cv.imread(args.input2) + +# Instantiate face detector and recognizer +detector = cv.FaceDetectorYN.create( + args.face_detection_model, + "", + (img1.shape[1], img1.shape[0]) +) +recognizer = cv.FaceRecognizerSF.create( + args.face_recognition_model, + "" +) + +# Detect face +detector.setInputSize((img1.shape[1], img1.shape[0])) +face1 = detector.detect(img1) +detector.setInputSize((img2.shape[1], img2.shape[0])) +face2 = detector.detect(img2) +assert face1[1].shape[0] > 0, 'Cannot find a face in {}'.format(args.input1) +assert face2[1].shape[0] > 0, 'Cannot find a face in {}'.format(args.input2) + +# Align faces +face1_align = recognizer.alignCrop(img1, face1[1][0]) +face2_align = recognizer.alignCrop(img2, face2[1][0]) + +# Extract features +face1_feature = recognizer.faceFeature(face1_align) +face2_feature = recognizer.faceFeature(face2_align) + +# Calculate distance (0: cosine, 1: L2) +cosine_similarity_threshold = 0.363 +cosine_score = recognizer.faceMatch(face1_feature, face2_feature, 0) +msg = 'different identities' +if cosine_score >= cosine_similarity_threshold: + msg = 'the same identity' +print('They have {}. Cosine Similarity: {}, threshold: {} (higher value means higher similarity, max 1.0).'.format(msg, cosine_score, cosine_similarity_threshold)) + +l2_similarity_threshold = 1.128 +l2_score = recognizer.faceMatch(face1_feature, face2_feature, 1) +msg = 'different identities' +if l2_score <= l2_similarity_threshold: + msg = 'the same identity' +print('They have {}. NormL2 Distance: {}, threshold: {} (lower value means higher similarity, min 0.0).'.format(msg, l2_score, l2_similarity_threshold)) \ No newline at end of file diff --git a/samples/dnn/results/audrybt1.jpg b/samples/dnn/results/audrybt1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5829f82309cf904f7eddbe42ae555798ab3acff6 GIT binary patch literal 47680 zcmbTdbx<8m^foxSTae&x!GgO5f+x7U`^DWQxI4iK?(QVGh2VaX;4T-xL6`UUeYN|~ z*4FO!RCmqP^h}+n`qVl7oOAkh`E?V(mY0^324G+S0OsuiURME002b!I_TLEmUxSDH zZ$yBHhl58%Kt%kXj)a1Yh=hWKh=`1ajDq@Kd%HtNLq-4Z;lH2!uTNnS;NTEYkr0vo zXUhMl;k6gQL5A6dA%ugW24Hbu;Ba7G2LXyVIT8Lxyf9X$i%2PSSFUOs*SLCMcj(lWAg@@ncDnp)aAx@P7UmR8m_wytjO9-dy_KB3>j z!XqN1qLWin)6z3Czi0g@DlRE4E3c?*Y-(<4ZENr7>>n5$8Xg%P`#n21zp%KpyaL|Z z-r3#TKR7%(zP!4=xxKrG{(JZj7YqRRzhJ$M|F^hs-f+SGw+2Z6;evtneCu#H@CZ~K zi0?kBAelJhQga3)<4GhHHuRy;aH&G@Ou=q~?*_CLt}-vJBx|Ap-T1onU7 z0t4u9FmHqnHN1r7kChUYRr zJC59oi{?Z6B)?HsVp8OyF(Bu9s^gfs?w@J(K&$yhZyd>O3TI$yaX=xhxiUatG>OfaTrvTpU1c>$?bM6(GV_ z;&@k${MdVt=sfSUKp1En7>R4kmsrdT!Y|ZL3+sT@(T@Li8Jxe^-6nrI860CE_;CIE zS9_?R<^<*nk&JOsDa?_&RhJh&nMS~Ty&z9HMwpYL&gIA@tY|aLg|5{Z?)GD^XGyy~ z->AtG%(ImCD-d?EWAxzp3iL>HXZNcXs8oyzeYw&DSFHDp9zkxpkgs% zAqI&szr62>^9Q{G7o){uWZjO#7;Qc0Qr>eX4#-zqgd;+o2Tx2rcX8v2DWm@weK0wN zrjg15`|mI7o`qh4zlAbS9|K8zR^M)u486d+zA&mH6S311-#3zNbMwge3K(`DJeuxu7*~xFkzU@E%9XFRdW;1r%EI*+9f*@T4%V$A zykL7VD?=kLDTV|MQ;Lo>Cq$la*Ys2gM>K_R>L$d%xG$K0AzGnN7JOkZa8P(?9{4>+ zuIjDm2yPo3bB6*5Nf;Ze$Djv2uy3FTqOSDLPZK&&{xV{O>0Utt$@u1|DTXal{hhc5 zsth$13!z8P_WHLFb;tqnJl%qT$F~y_!g?ah@>V5+Na7^CbZGKV`{v2I8Vl>?fP-JM zyFpkRRTNNx@tuorPwxw!bfE)}#CE!Yq3$;-CUwh9<75Np#u6cwTF?5KYXZZFit^7I zj~@g3T*NHvz~CF}zi7YMj>ap|q;i{U6qI~zQ1eiXxY;0OK+?z`p(I0$O*F@EtuF|W z`_t8(R{%p#q3B4?#<6(WR+DLFZJ*MBCK02t9Q7hNd{mgM1yc6(KKl-T_7{ml>B~xf zQ(%Mx$Rv52(NEvat3JJy`*T|^KD(LzkI>+8!IW53;Q2%ej5M6GDdlOid>&IFqL!VY&Pj#z@FC=O# zEj$V-$a(4wjc<8SB53stcwrM;ep8chQ9lIikc~kzgo$lB%yNj2Ex92)h z;$1yy6Z`vv=-}I!44R+<(zs@rBcnIvv-{u5rvcL2=}C}E$=2)g(xdbaS(h`E5e^ZP z-z->MrLv0j82|8WTNib!@PXTt%>p^pH*@3<$LPo}XK|9wgfoRp(M4|p42g};@!Qz2 zI&3>6i72c037W!zp4TZF&t(Dh+D5$<6%sQ_A7oHWYscncVPkV}QqEGQiLcR~H7_)} zyj3ky!I>{3d*T{}psp(@5lJ6d=~Hvwt;CZ0{KodiA#KJ6XwYOkvid6yN6;*@71G{@ zw)0#K;4@kAsQDybAO4BH1w<78$7jb0dchA9gWjUl*qM+!}6z};bw^(zp?^{2vOTiaYRwt4)wux^SyO6~jHo zf#Z&yC|{*q({dVB_FDTy>ATXSH0v!=Sw#2;JykX9_ZpG`pP1<5{H;X)@inJlV^43HOh zUh!ie_X61H8qe|p$5d6K;JyL3lZVTPxuMynTH3FrBFj@JFM;wvR&=#H;WQjGpp6N0 zG*&rYp@_0h3Fqn}Eu$FpsEn9q0o^dr#X>l~wr14*$K4g)sIs9#Vo!0u$oz>7w5#0j zKvdtclP|9(3{N`RJ|ZA&8YGrTfeUS9?!kk0DML%_$_>53?=D+^j^lAmP7Yh8;^f3< zJFngJ4kaMj1>(?mbY3O|H6PX$&h3c$2gPOT6-re8scLgF>}rU8b~apj1rRzRlfj!) z&<{&PtP}&Hq!DsW*&1yfcG@&BhH@ZNZ~Bp{(OH3`ai67(fIk$trO*#MHC)?MA>Z0M z!@>)a6bcpWNeyW*)x#(_^FHa{!}S0Zp902>AnJ}##(IE|(OUh5j?{psb7QN&-8 z=2rmS&lQr?Hn-c<7IJ*I$Ud|BS;uc|eB|l?CTdKK4~eYD_7&J%&(+d+1;Uw^{D?dI z=DWqVE+RW?@ zXMQiv)wg;1x5lvg0#CoCHi2fufzB_a0r$TC)5uc7pJ-tF%%OQ4Bz-O0^{YMEtEOm$ z7dEb16f{1(*MB3x4}U9ANt|AM--*yIO0jg$?qRb;XbbMKl000t$z_FQU7Y=OV8jsYiM?9NMc(c zH{)Lo|Cn2BhQ3i6M8;cYg=;e++maHy;&uRP#nk4e-P+)j!yxOunQ(I~UH zv9T}IWv#kSi$WTl`yFO9H8DW4iwLQKRFid^sCb^*Kv#P&`T-x#WM!X*rs(&;FZo(6 z;8bABnj6DRS?AYZva~eM10?G5Xyv0#3LJzk0|@8jp$7kGZ+YdN9B$0kSD;~+(<<7AvUEyDLDT7L5cdJ zo4@7ate$7Oo>k8(Vfb`al~j9UP|eJIo47cY+=H?x>x`Q}_3?;utDUz=dltJntpd00 z!EWDHD;0cz88^(duC0sBofg480l7ij3f|y7R!pPvs84!e zcb&J?&6$C{KF%G5$6L&uSUr)J%z4(<^V@s#MnKrD03%yOe&H# z1ot%8o)h>EiLoYsF8S=wkFIuUDY}sFG-qh{p6f(ZBW$w4)8Vd3Z~XCy#zG@CoT9g~ z&mTtMzWOi5VW&k#&}7EPD2H2xXJR#H*Sp3syz>Zje6O_*B>01*Jnt&rj15vq@k8Qe zc#{8Jw!a{kdtDc{C0f(#JL)|dEZU~y-+bg9cUOMkw$cp;!=R* z2(B+|6xMxZnV@z0lKf9xXgJb?8E(4DZ~2xd*fQDBEB#_9Yj;@eqkB)-b|Qh-7=QXE z8#{_wGu$MM$RXUUAF^ra&_m483z;vW14vbe{L9CzU6{n8L}2rP^zOpIVUv)}n}%ls zDx)3luhVSf7`8CaTPjBzr-)A!zLpo1gByK>>d_ci9^j>&}A`UOe_*P6(K^!%i z+Z*h2dw(D{o4~0U;5yoOP(z0YsO9O<0zy^*OkS^=bpad$;RI6`1?)^&&@mtvcIr>9`Ud{71LBA+2+SC3Y*@h1%IWN!sVF$ihw<5KKlV8eY-pbZ6- z#d9za=gG3UHKYGQSXbpyWFV1^f%t1EPQrQRJGs$gjjvXO7W11-u(8JeYvytHR&<8U&U16d_ zUh+Kq{&|>1PdJ~~_~6;fFN2R<6>}_@*+BTKjiUO`WpAi+dSITMJ>SO$@OUvuE%8{T z(=`ezR(K=q#<#0$|3Re2^tgwk#GQMyz;BcDC>q{Jm^os5t!{3V6g7Lo(D~a9fvZbJ zii4?EJt=R1HMII{0F)>~V6yJz|GZEIAC6Z`&XH$mpE|i*bk{ z(SD~k@+2nQJLCfqJPke;Wq|yC9@}U zmi?&o()S8raWjj(@gRx-e6JXznb|(0FYU8;os&^H)CT^w0t(Kd9pRsdXtmY?Au?|) z?FkEl)Jq8Ju}XlSc(V$qURjePgNuUx&ueo^6sxRBQAs6Ps-)5lYzr<0p&>pCVyiSS zC`XSU0~Yfe?;~Fby!Zyd40mh1oL>u%@2b&T@5f1yi!D{rVC9e);BmG8LgYy%jQW4U zKn);|TC;3hIetV9c73#;>ADxEj~X&}UrzQldxf784(*1jQZ%+|*?&_-3~w_@G@nR2 zR~eSbwh-ec-Y}FjNgVM#!rnU>;Qk@nUC_F$_6|{4I zCC?K_dtJL2;y*0l08CBNPBr}r`kU?#v%Na6fR9eS3{1qy)5&ehJ+=IwOOy7mA4;2J zxhsALC|hNB62w!Z@6@4=DZ`M3j*4caKSjeldhhQ<#!pR&-m z*7sc34DYKae)44hyU%&1MlOf?C`ne9&?#zYCpL(CEJz<}Q@(t3Bs^j)XK8T6`HiVs zz44zf_63u#6++|?U~ObGW2aEDJOJChxJj$3KWV0B*9T8t!sneSyjk8?{md}9%xfW4Z-H@LsE zkyx{+jl=y>A!#`5`dnflLn1?TGSx?rAGlr4L(;_`SF(PZ-g+h+zby_Y?=;?@W?ydX zz4M%;S$#X$GDkIMq?o#EfMb)ELh+c=54i%Fm0Xn&a48U%zM^h`bs~$ks(7eL(ep`odWvp=v8=k4!DgF}! z5TX$QZ&o;D@E!D#`>-~TC~FTSudh?NV^4|CR*gJLnOppD#mNY#(L8Q#Pt2;M#!h=& z<0$OJE7xZl>^zsVLpFNISZ(y~nA0u^{Hg4#FIBj8SXCqPdL%y+VMqyd(#N}U%ZKBV_fo^6;0xn>3F}ox$MEfR6NYrdLoF&dHdf`z zssCvPkb2NFcK7psSHhr;A>W#39HaECygGiNm@iC)3VqI+b&KbJl5=g(_UDBAULxKx zAVBf)H;Tb#9wM{WpeXg?MKhU@9^^h$-BwK($Fbt?Fb&5!4C4ImzgXnW8=`|2z`@IG zFk{iv$xCx!V8UfanQVA#rN9$HBB>3qHbH`gcPC@ z#8+c|--|8Ov&6FUb|>a<0YE%izaoXo9mqjJu#I zxSU||cQSZiI0W_kSE9F= zRb}82`cwENUw;qollpUg0NoKePQBiEwj;icXUb8NZGsK{&4)spm9Sm?#NylDQ|A7c zk8eJ?S!^IM#;5F?u+5`@pBSWNKcRxu+R26s!uGH&D#6sz%R-f79$p*I`2N*US?*~HMr}v~GDROx?$`KGOJelV z4C@3!^oRm|1vK7xSw8X!I6Rm)!>k6XyQDD-awxqfXS9eFEZrV#O#D5bl7lU{6cpH} z`HS&M5o%U**a+3@WtTB=`(u8oSyVFQjnjUden^j*R4-+A&8BVTxMnJvdie?roqg=> z!7l6IeFg0Ez%!9mP+VVx~mLncr#5L+*v5?}&|Yeff)2ohGnMpVO8v@JjtDwVgr$vgOEtii=&*Xr+80oKP$wtOEy9as9qN17}i zt@n+N+GNWAT+CgRFPL0&%<5=RFw=aJ#tlAIn z_W_~W98u|EE7(7-7>q}f#o>`G*-ADu!LCLdOo5Ja&1Y`D89Y*s=qtI+Vd9UG#KG1W zf?_()Dr+5VHMkl~66&C=L`yI-9`D8t5*_~gwWBdX-7;BWQ7;mev-g;GnQk+;E*&H1 zIyfiK_T{^Gl>*47d_#t3XnZ{*a{T(7a3yo86$M=_p{w^ogSGkK&z4Au&>e|Ms@FejY!ys zA1K!#_t8Azi`+S@-&hb;bb738P3Ce=Tygn`WXklV+JIx(2*aHZvs@rL`t(G1z&Y6n zV_>%cNrYiUEe7x1SYeD}S8JACBy}X0Ue80DG?U`od}T7fz)lYyOvVnHuL1~vuXB#g zGUDt*E53h&9Jb3~#jq#(W_Le3EGJb;;}Fvem_~el zp1H)#ejl}6|A?5zZN72rBCcsuWb_@jmJ>Tan{|||V>Cj~-&bdv@=~$R${7UDWRalI z;zZY;7(~OFG?$DyGrdE*gO3#}tni%VTrNpaLbvul-C4dh3h@vGN;OGv%#AcI5>6I^ zi_W9E6pIW$d=e^ouDnq1Xp3I`@+iI*>g8?av8RHMDPyE42OA`qANC&7+ESrCne8x8 z(H<$TV(KEGlK*p*wX5CP0L&@QA=! z^bsTQBmU*y354_iK7HqlJP+Og7?q~{N9 zvz`3ri2wkunna!oxPdX$=p{6apwOj@7hiEs$l$*#NflUqPqCQGRC`wc8>MeOL`iAj zCm>aShTxiW#YlNT%cpoNOfXIfo+Vp8S|tsjeN@|@1CyJ7_XLT!P9n`6iQ^-#J@vle z>us}c_w+w|p_ji5bnBZ5h$;j|Rapn;Ftv&PsCbeLri7Yp<9^%#VV;I?O`T)?oA`fl z=l>kHJx^~vw_kyk9|zm?!P{rjC?<2~4B2v8L)|Gd&R3IOTWQq##tGY#UPD-D0gJi^ zy++>z{qJLI-y6(35=xaU>y6h7QcBVYXKPa`55&IH34$YHzR5U!G_xu9!<(%>@AcYO zNKNO}55X8Li1pn07yXaqE)~Mp=lD&GU}^Jy@MOHppWuOep8KmjpY~o;+%D5)6~3`J za|kjGdP%)H`ynL28-XR8Hr*3N0kvHDE*f*zZbf>P`^?u>r^4lLTA5i@uWk;9Oa#=$ zPYPuUzJ58-j!;Oh&VdTPZ}WIWUCh-z#AC>|8P{%(ac;CSs2I$G#ril^7H%)eCp)3@k3^PSqV7 zmSrL`M0NlU_G8czasFx9<@Lx0^TLy6U5TI6B|0;GE-50J=$vxf+_ptK4iXi89=t`e zvh*v^BvFlt>p14smuN|txlfc4fp;`LGEqKTq08Z_o;v5;omw>UyE&a zIAUOE8+v(eGM$4Vnz1as=gJ!)B9Vr2Y)LRj&{;a#CZK`5BEnD4p_iQ_)vAFGh zQ?V6uAqX-QN*My29^70_@31G}Q3x{g79%UJ(AxupC zSsYQyBeZM{Wn6^96Y<6Kw*BVwUvoc{(phvTauYhq6~zVZDWN+LC)QJIYJg@oZ;mms zRFbhsmA}_Lsgz2YVP-Z?!B*C!@W(CivL}WggpaP~Z=b|k2C)onf7KiwJa+AKE&WbRVsJC{3wcUGoq9ny(rU-gl;6k;q>4%V&G|r^`vTp; z$4k*)`IBsWe~$(#sb<&xj)#Oih-f(l2e8xBk@3SH+j#O`+nRz|>3uyT&zfaR(eJj3 z8D-5@gXG-~R)Y{(mIws>uqQwkj?Q-x15!>j*9|{WOam8g6GQk%$pke5vFYPqflJPp z{=9uOl28{lI;(g?tW4DFFuHk68AA@`j<9NX?ULDFi&?352-&^ILzZ-*1xN_K^<(GW zeLH628ee%-kEZ`kPIC$|e}xOb8ZTrxC+O`Hb{`unK(f|jt0Quk1EYO|mtaV76`fdi zH+G6UE<=JV`+5QyRW=L7NYj%gtP409GtS}I&Cmd3P;Jdf<4gzJ?&nX@%3 z8;I>#hvx$;=)xqAnv8pAKP3 z7&UF!4+#6IKeT@BY z&%B*wGJcNi3KLh#(n-#XhB3P51fU**0v}6x zeAfDW%d_C=+Ea5YqDfAsalPHqGX}{K;Eud`&WR?<+`A8OzsP}Eb3f71DpbIgP`9lb zmMiTlt-f0_Lsuk@=fV)gPut?&UaWj8L~yTNZxN0e$VYw42G${4Cwh6l#ZR%DnSGf} zH$IDMK9^)R?A1g){!wE1>Gwt(5ilSxDO>G+9Ca^?IIGa8IyQ^?ncZ2GsLeIF)~jDK z<5%%6gT7ke&ip7X#kTZ+ls5XGzi7@}P0E-UVZ3bgG_?&+t?CVJkY^nXI&S%k^)D`| zN?N%WDN&EpHjWUwTw1Y=?5QIKH;qoUFFArl8YLQvfA^Vbq8Gc=^oo_~9*skBnR_i$T~SOq@GDjKo$gwW)noO4 zY=Tum216~^f>(9uSN2VUg;`4Y(RG>;TxDmMJhsDZPEIK**|z7Lb_a*@d-Dz&lQ26S zB}BtA@VAKFMRy&~&7$xxgMp|^+ZQ9R04^D`g1-baW5{RobUA~de(I2^;B=u+z3@5c zvjOAztp}$QkLFOFZi7dWr>`wbX9fQp(UnKz>|cTF$*;e8McF;_WGQ#>??w*Gsu%oq zxih}zF~En%#uFyYyHzB6nG2bH*M4AzQ?XTHd2oYt>n;2+R+~2~0lfc4H$UE{Un$sC zMU`0Na9`0#C9Y9Y3O==Sx~vGG#X1R8v@rjheL%EM$XY@pgJ!$1BMJi(`57Uy{KrD( z3vJ;CL-rt(t|Tyj%4wcU$P9X-erV7jc0tlSh0j!|ba!KG>-?F*njo!;ciZg0L`5`; zl)Phc0WyIHWrVJ0#kxruW#PcTL@y5L#p%oB3S&GeeFyHf^wa19Ab*y66EzAp=jGR|d}U zI~E`Pck3d($*jg55&?cJ4u~#A-umPtvF6H%Pe^hbj+sUf7NyxTVnhqNs zE^Pyln9d@fjC6sxuNt^MMaaEcMZ!#GH2E@pwJ4y_d{{&4-8=Y6uDAb}i;!9UChTH1 z*{>Q$7_MQ4D5WHv*tG9Dt1JVA1|vz&vq<)#+;lhDdNzZX)vA*8_hB5gCco`zuRDGn zr|+asGR22{Y$4OwkNV>}0295J&OwM2L|8#&&Ybc$(HmYPPLZ*u`H+XsTdH-@G`e@> zy7J}BC%0pVEOJU72;aVXdGtuJGk;84>q8%_S42C%=1NEbI8%}_)o0X1^PTRT`^sfL z<`YJoA>@qx;?$$@5-w*=iO&zigV__Tn0NS*GO$5E|JxeuH`A*#f6~t8!P#3Ia*fd1s|}y?wDg>wNe=o%L~;xH$~{FDcx^Tsliw1-4wEZw zsnP#Ubw|mA0>s1--!5**2>eBiWp`Syc4n4`e|^CTLxzD%7qAR)oh>!Ou+?SDX9x$G zyi7*XYc%xOrFt_YOZbg>A9$cJ(9|+5c7c4^5*&Sa6ns~+HTB#Zwdj#+X&qU0>Rlc^ z&ERj~A-4WT12bb~wRI&YcAmau?M-~H*!^oxEAR-#}zWEUiF#hi@Uf?$M#l^?BK)lje@jE&$q7r*6?TLU_()|>dCUayh6cCr znIo`B%N?K+WgOu({>=3-(3hLGa8fcF?#UN}3bOUF^lbikko!b21({P^Q^)utwQ#`D z^Cdt!NK9OmB(SePu)%1MadUk~PYA8z;x}tvW3qO)Q{-+^mQ3zvI0R*^ji9eXuZ=J5RT>lxU%^U9C>l>P4Ea7EbdIY2+zlNVj)RA|L3*32}`y zWiI{$BizHMn7d^t+_nL}u)F7_qg29xvsWW7!erq#v)2#bgmj4r^FMy03mPHy%Hr@C z9XxYhW_Rb$3BM=_QihG*Drx+(3CDs`KG774p#C8w|5V5R zT+_4pGrzv>bZ+ZR@Ir3hy^aVPstLJmEi1LY6Z9Rdv+;JY3Y|(fE=-JteeWy*9lR%h-<7eui!RIRNERXdt(&4h2Y5@;BA~qj zAu)dLZ)4PxiDP6wJ`a~&v9@DG{|jMj;&X<48?93jwAdc@id;KL^tdu%z6Z)fGkEt9 zak_FT)|>dH#?@>m)3NxQ|7Oa{2yxqVwg*L60I}Aw)f4ge%iwP2M`P&0=E9$(lAahp zg0`ghegy4}IrdUym}VWB4^R2}K)|KW5oZ$$gJ~vZkl-tDTY0ut1G1v6RNyRRo}<0H ztFR%1t5Ah`m_{PPh|A%$^fI)eiGIJIMqgydId9eqCwx)&@n@oBim0{7P}@V zxGl!svw`-6AdZpLhr}RI^}^9@`74?Fk9lj%q;`XLSZ+hQ!Gn+j&Jl?)E3*mR{{O2SzC4k1fqj7WAZ`mRn2L?%a z&-XIpUa<+u?Z=3Vw1FtAr%ccBdA%eciDe>9IUKmL$nHs`OrKOXG_#Yur$t|qR{%%; zV;$B*cg;|PX>CH%RS`DIzl=h-@Y}-kS~q^vg@dp%uIB9XUE0#k0%p(-aCYdNKC&1@ zH`m;LgU1JU-xjA%D&peXKsWD?HFi)fW0*I?QzrnI{JgC`amGbT1Tm^%ezHKWN|{3s zFxsNbnGq)k{m_HvZ)=6H3TuLSer4ljW$T02Hv0D0q--a~hl9iaa`69)7Z+GfX?7zf zh^X+27XqotI-8ZPa$F@!-YVQVgVVl&$98f~|D}681Apiltl1LhoUpfK zWej+Bq@0l;xhN{JDm44M? z%x8c9UF%)zXo-d_JZaNACq+3`MFe+4gauYVIb$AQtvoRnLLm2=B^scXnWD_ugVY2#X2(A>y-3_>PbbdC;Vn6?*HRenAB zk-|INVcU}3oQB(u@G*`r3qAU3-Crw5))3u>&~2fe4i~Bg$psP+4fK!P@3Cs?-kB1SGuXC+0q#FeE2N*gTRc1 zYP?NV7D=q)>7)i)S29)Q$8m|70Sia_!1`Mnc~X0D`6X@452-cU5oviC1=TpgQ`mp) z709mvJUI*vxC{GT)|H58;?thqj#>0glG8RXy;ZrCQkb@73U_J^oH5NFVr?akxw2E- z4~_;Bzr_3+hnM^#+o*1i<@2K7v)NAm;pe626TaB?U7;`iMbnALqBG2lDhab!xPXYq z!5r<)kvIk8jFjwBhNE>lmOE9*2V&VRHV>M5y<4Jo~ z6wA-BVd=@BRJ3rbMw+NO|NT+uN&CabKrOiCJU%({qm^tTieJFqU?~2iXjIk)s+fM;9Q3Sb&LjTLrF>95q z!+kVhl~32bs)|gacG_$1cEyUO(K+3cZD2dBDObbxPJ0~_wJ|3|1X_S~I?E@gEJtFY z4U1J?Qpfq66u_q@FPp zryK92>)V5C#adV$sC%W`X(nNDKEUmhgfAFlvgeRuiwicn)R5hMcdd~kjn0lc9giz! z__O+nMAjs?+nl^PighS^3ZB3dc9-*87&a#-Q}C_oZ=ZKg{?3M=t)E&lbKL%TE6{o_ zyG{kU@__)p(V-d?|3%8M!Ie{&d5ZEX~gc-B#g^S_`LjwD7& z65b((-0A@V@^5ZKfPhu}Hh+r^9lds8lo<(NT>A?6o*(2Oyk)bp>;(tayfZvq)l*iMTloa_ukwO>L2bC? zX~7WgLu`Ers3_w_qz_jh#A_dU=8jjnRWq7v3BU{NTYj5K7Z>5L8kh9njKQM z{>Jl4ob=|g#XX7bFuUvpPM*s3=ori6Ao&Wr0z_~K|%D5-o*c|&6$iW0Pj(^@E7{w73j~8c*{9Q zoYFudUQo**H2tl-SK8JDFZj^xG8;yexpPWH{@hejcsWU#t_%AtxLB-IxbrC}pV2Um zSJ4Z~?Ty-NhK2A9q_M*6wh>|bQyuVu~z`f6S_?cZtML>;>VWC zZ-nJlQX2INgxa4ZSB;o{dl3tIwoHaKrDd>>4@93v%6a95qTTwe#F{M zvvo4@=m08i?e1=6qF07ytqDOI8$Ykg(h!aI=>pLYtNr&=HAYG4@1f3 z-o`TI=H_sYk&8U;JDwTI{0FX|mRcq><#U8voIvAQl2F*l((c^gT~W(2N-vR z!qoB~?F@b`HGdEmxT}#l#g@LK)*txbjyXn~&vpynh|~+lyQkDFjpFuL%J2*{jkja0 zIjcGFa%enZu~*pB-mN?*vQN2s+m-w(EHO%6nsN+vdIbH^e^;$^zUCU62i@tBR|cJ$ zcgz;Y^Rr7930-W!8iSn2n0iyzRLZ8@d45h*#F?d>!I?-CAvsa?^`d{P7vWl}^>Xk3 zRJQhWz+|Gkiq=xde*hzBSwGnYhSt}r1>iX&(dmenidv+jTG=ML(jhd-v|?yq6BrTs za6#0ySkyJIUEiKm#Cu6UONPgO(7-=RQ%`!ubg=4S%RXKTnB==Pm2{wWS20e}5RxHJ z1~{bAjpglw$+&*&6Rk)N!v_ETqv}1%0Pjma4+C4%-rBCC+&^pnhXO`a6;a>QF%x^B zmP3knP14=AGiYvynm*C~I>x)*2~nJsw1WRMkeCtfkNZ#ZKvZb$i00g>MaGwzKOTuI zI`}vji9h~bA?`(&0S7_aiHd8EC*cNWnqR$d!ev1PNt;^EEe0fLA5JG$WD0rGHhFcCrm@2F=ZNMA>bG`J)xLEETQ zY}3$%T!f5nDFeLR&lh1Yuf&wup zv=ToUdZ~!rrr;A|TOPcK@vI^W+ zYSMTV2Y)CJU0++OJyK2>M(i8=od8tg()pAlz3k2T^nDi^IKSvb0%r^ulTQ4vfbhH) zyQ%me1yaP(quZw=0O5|;OKml0=gAuQi4bDE8@U7!U!F2bBc?Qf>U+sPYTR$x~np`Gsv znglf7v1unFZi!RqV6H~Y!x?4wK-|Ros`4X^AW9?@QJMs+#Sn%#`-@sqOwElg-I*RN zPHVXbt^Rz{z|F%~U^QVxOY6=E$q#o+4B@B=DZX!_Xk$Y>Hd~TA89R=W!HJr^C`hC0 zY4tMV1&#I13A~3sLhg;AUoSm(9_<2%FL@$XcnpXSJ~iM7I(<^D50{2Vgy;QqP~VF* zKCdDL;HfA^+ZdZdevith(di+GhFP@mrpHilpXfW?sfVE6_dmzD0i zFA-|W`W>S;C_3!EU*uHk*fL{I{U22QWmKE(6ZVY;cP(0iyBBvW#i2-{xDCT^y5kBNZL?? z(YCP_@(_eE-Y^cDsT@{T=)01`1q`;e-A6gRa%o&+U-iK={c)<59Vu!^;-)|r^rM0_ zbzR~5hPpO)uiqrNab;t*&@OfFYleEogKyzPl5Y|Y)wyoKu~DY>z)J1;;q2$T?B6RZ zW4twS=9mdMr60#edq2B9nE41~FROg2GuoixF7Amu{L0a?cBTB*`w;T9W(16kc%ym? zSXAAGWx7$|MKX1+#9{A_#_C@hL}L$c%O6qQl}rwIPkOsl!u}d9k2>z*j`gkok|GtK z)~r=A)8n>Wb;i<%v59@DJj131lJ_ae4yGiQ&6Uo^IbHD9bPcBC!YBvp7IJ-0j<)?(T_5}P`^b#XhN`8+a4pON#>L1z$PJ}+Sg-l-UoxMqkW%= zI;akQqHOPfim3q@z$o2-nM&swsEVY*0pFriQ6rPbsHtu85QWXy)_2OQmYHT1R-Jk2 z5+9!EIfjYMLRs>OL}%C#qCFH9gjr=KiY!b7>24%+mSd64H23}cd->}{?3pu}`zIE( zxwJH@loY;tYG~%`!|1=lwSO?tPQ72>$3Lp&5~?_s1v0upFSYD(OgOGb?Iz5}5-bm; z^YN*2dqXhHI=bK+buspF&&9223&I3+nQN;bZ4Rjx5>UjC#P=i^UIU;brt>vZDW%RS z%_S2}@h%Pr-)&wekfGdxKins0*43Drn>eM5W!f}p$t*yHdD-UKH&&xm&&jS z?sVMxS1qWbSm(Ji$Uly)Xz4p#9e^rp_&#Vpum4+z?N-&x$f#bdM9mp=HaCUf=hPOTZdglmPU_)7vhCgx^?E@x)jxs59=e|G1p*u0 zg;QD4dpgsqhnhl3)`J{yD;}=;;W2BP%oazwD`@`?yF2yYK~UjyqNWG2Dc6X%{UgRx zI%|K5CwbApt1mud%v5Mpe>hMFOr40$H%cN|1XRjcJ2~j@T%x`!e`1HOb)KG|cl{*L z-bqW?4Eo{0kU({DIyp4hzoFg)8N?Bbkhj;Wf$SfZdKo{$)qQTu^n#V>n_DM`3OYgxOI5LhDd`19}SKxdr2L?1_+wTei z)|chcsLb*cUm(f7?lOJofw$##ew!N=?ShyjSeM0idN%u1Q-q+IBbaP!eaRT<@ty9* zuz9*O>jY5sWE^|FYXucWgnYqo8J`r7+%z?gll@n}YRQ<*PXV-p$C2b?`PZUzYkwrT ziHWsD7q7hjQt0!Ay^+hS;oWl@trP5Ddzotu%i<_v1@im6*d)pe)s(eG>js1oGD8hg z>`F~ksnZ>$v~}L^gs1f0?A}Z+rVeK#CQ_5#y~z~R1Vt}W{d`Z*2-k1(vC*|&mz7}=dMK6JSIvt)y{HSa=e!0dS}l zni5ApimUT>Dm(U?-0fmFjS!+JKir>=7DG{pH&EOEIXr`9MvNtF(R8&Dl0GOV5U@Jb z9&6JS9$8^_!K1+TrXmVGSA)I;;Cezyf%e`sM|6D}1afC|Eczax8HsCa&rD9p;DL;`$ zOYVtH*ObeTD2qC~mg=DOI+^`M`E&KH;tuu3&#IJRd1{8wWGG@$HFO~X9%s3R5!^~D zJie7VMb4FXNa(|)4+E(uhPMUnJkPZy@dO^jXXgeX4MCqHZ`CLSO z2*$+}ntp#@GELQ6j2@ZfhcP4wmp>ZiGs2;PQYa(8(#J$mO5eztA!Vj`3;qa^0Cna%*n$ zxElpXg0gdIN3G)v)vWuI$}X)sjT9c9Lu*$@YZ8y=V4QJI>^1;}yr~uJoFoP2$TJgCH(5Hv376sxD@ku~4%5;Gdz5-5jsDEw(1mg#aCbNTCW z&sR6#5*F11H@JXg5+o$Se3!c!V)Ua(M60qoRNYxX32kX=f+60PEHhZAcTca~0thpT zo~$!95G?7Gfkw^q8NKVX#gBR7?B`<-oO_X$HIlVu0Ti)e9$v-=ka0eTI6M0l91@i# zh`1f2q7T0Jg5m!1)9a1xcnyPlR>TL())iRY?xc+H*fJiAp^j-@@Z_Z%BTHk z*OT6?*&bTYeqGCZLqg54zn={;v~dZ*80t=bH8sXHTy4(S`?Y}*kJ3HTolxbMY-*lU zxuV?Hm{=2A-XN>ewaKVdQtgbb&no8VjS=NXva+2i>z~T4s?@tR$e9f>n3D7P*-)oZuWP#-Hf1Q z#sU7Y((iDiMI@-oKF#x9Fun7q8#1}Xz7w9x6W-#OnoUsljliPi(r7#7xnK|#bhU5M z&65H-_KAB6Ee7V<$P;zN9r8+K4ps>yzsLIb%L}#b;>n+xw*AT=b3s8S(-$4dI2f`ThN|L7yT zDmqYffmGIThCq@rS_Rn!9jx3ZIXKdE*C|r8?yo+gm`;%*+_NSY-*)a!u9)#7`I7il zo_ZSqK#NP87<}yzDSXVhDEuwwtwzVIa7;_+vAA}*+*p0N*+F1xNF8emuDj61fuC>53W|A2l~#dGOE4r0p8oqskQT zx1i^)`;Jz-JX0FX2xl}UVjDHJq%mfFuCIvTU0V^isP!5NKbr$fDeQ6(eQlbgd)b=k zX^ZnSTFzDm+rt{WR$&dG=iLiKTx zM)vTn1Z}+l=}ZrGj(jZD(|z~TeB6KZFF1yej&i5ukDmTwx8QESm%->`q$`4qC8%N5 zZJ5*^MSH~MUY=g?yEdl_egTRbnPP9GgYUS@bCWVItx9GpYKmUU$ksWZ`m~`%9@c}u-m$% zG*jXVdP-DsRsSMaV95p`;ceGi$XL7`%-al?dwM0Nv6w%9*s{=RS$Fbpad5r@cQzo_ zF)HYdU6aCzQ0|@_=)|v(Tf}eHtq`_;>z5`l*3^)?4jNwcF+W=&S>HEIMWzaYP<&@3 zQ|8mwmpuIDNw&lOtM_o;D(9uUD3x>&kHCaCiRHgP`@RwuH)mq6j0og(a%$%Bq5Qwc zEQM3eqy~QvZ}9`^K**_%+^hHLgGndE7VA8uPUV-=G95g#?=-}n&;aj@N0}>=M~6Y_ zWr}Y2Y4-CPt!+(d6Q`>N>)v1NwA;-`AB=zS*r}MeG8Zs)jJIpmw|De(zHYpSVk<;b ztO=5tfdSS`_IiTKXh2)E*3a#9Ij+*|MdKC@USD}x)5sMCxsnRyaj->Gt_YC>0GQw@ zrX)-8MBU8ato|RK%@Xnn{xa-U@qftGHkoTTI#BR&3+b|taZ$ov{X~uOV}JYSI+mP? zX;qL^R@404J6KCbk%~SLV}Q7zqXJ6;DJiKnnfkP~wWc{iCZ^P%cPmfz_dS(m&3w+} ziJkb;4|q$voNQj@tDP6=6M}weV!id$7Qe3H?l5wQ*kNwdzoe%4=!-^jvkknUX|cMB z;H>d*{ip5`PU}~hmvyHH7y%2UWM9jE(f6!ATw3~f+9#HaUIVlxsHIlW(RL@oYNIy+ z2l0h$(NlmN$9n-1I{_T0lM#_HIuP8HG+Zr7cM(XUmc%3%OzVf0oIDGy%c% z7va&RUDXIJ#*uYuPn*6C_%?UzhWd-s}CAHRY%Cnq+#7ac=iilgG+|R=ASXILY^@O7PQy3 zfO7!3{oRMW51G7yyXf_^)-7VB7H-c61CX;(;1@LZTs7v+QUveDlXepTE|CV&fvQC{E z%s?vmCggwPU~&KV)M-9Lx^XODX8!{;hYd`JFTMV-nMmpn8(%tKmGGr#zEU)!-5U7% zQ89O=PN;6d@?J!HyU2g~b`B9X9Q2o8aJsJ>@6EdlkG7BJmpkaT0`%P2J6TjmH%MX8 z@|DxHLy6;2|En>MdE`h|qA~0EhQd#7KJV=KKJc^@hNOlfZADM#LUopQYl1OfZT{Dn z<4^gWrpRJDEmwg4UAN=>?M&R)qLp6VK_<8kLj3>#G)_-aXWI5@_$Bg?v*wTke z*@Gp|gfyGm<|YTm;k$F!8f?=;n4Ami2jGMZDsgklt^Ts4QJgCw zM#5aqLCG$zM<@pd%a6!NAmYc4z46fWM9@cXpH+(g^Eoa)C5<#8OBmbQC#xn~#;;=Mhi4 zVN3Dpm#gC+Yk&rp^*=q$tiCBu2j=F1;q`x(nU)v^7 z1ZBb8_S`QzO-pgmvX9~bJ-fUgU8rmCEVHsSz7{&2Wq{-Qw>-AQsi$!?sVD;*ulAcBky1d=9;bz=^CA9$Z`Kf;FUafIl|J*Dn=5JT3TLmMX}W=or)g24Y$R%(ssbQzap_FJ*_j(#BceeD76E&E zM(Xmnh=JZI#eJqg#S=^7X19Iy;IoXqs*foV(Z-3~DrG!n_9^9IgiMy7#2*~j#9*Lb zZ6z4@e~p6w?~&C(UIw?HbYphJo=E3+&zr8uLO#fwQ;-(#H=oX~CJCHzKB3edwD*({ z-AmAG^elvJfA8kc9W$!Q`zmaF{uc6baaLdUXQ~Wdci0WaHDEP*GlJDRi6kWbKXFA# zlH`?omFH=F8qIG?5Lz1`Ipr)F`NNovkc{KV$9NZDrt+G?Ufe(z`TD}t>ERwRPp_7D zM@|(US)AMRhP${n2L?msmUwUM=TTGJezq7HFhzJn@lefg6FUxxy}T1s!nW5F@0pj9 zcQt6j)<$k~;Uzh@c|q>8PqOgyB9Bgz&Z4bR$Qv&dqmet~y!O4@vsJmqx-=x;7grX7 z0q_#6nZE4cP-(bOF2;uc|gtl7SW3J8aIIj_vnWif&*+ z%H`Gi;~%q}WJXH(nu9m#I+~YC8jZ9Drc%t+IGC`waO-8>!hWGa_7k{?J}%al#XV}y zCFaoXY0j+CcxT0S8(?MIxTIQ#9#hLC0C0*sDJFKY=1H2?q*iL&G+V^Er~zWsHs4bf zB}tZ9YMdQkyFqea*SB@eC8^4;1K~bteD*aaa8{SO6sQoEu(YBsc}H zIXg?uEj%M#N9&w86rDr3um9NTzLvj|8c)2TpxQIzK_T%i93C$Zh{&9pTUaH)UucvWd*XmO)B z>06c%j|L-4;}Oprah82g!oV5KU=zF2L`V0UeGMIG)JJ_L)hrg$%}AO+#FVy)Z3~^P zHt&KM#c?^8bRF^3{R9T%SBZ^?5gG6`xy9Uhh{e?JwDqY;=71z_PN)Gp!;y?AT+tI7 zwPxLP%*oykdVanuBPh+eipk8XDCD+JOdc(HB~K~0^Qt+|R0*bvZEKjudmo?TwSR=P zYLV%u(J@AAxtz}Q5%zvjq{8~-YDKqxMsW3cx(gYIOtijuIzzQ~X(>)lmf*_mpwz{p z`UDX^;5IQ8%&281P}jRC>w-wO8RX_&?kv_QwPx8bxCFua9w9L6S?%RNr6w|cUuatO zxS`S~+4GgX9)9`0LTbd5xiw>pG=Sz(y7es>qVj9g9p%H=N=r9_Kkf;QH>b9PA_XMU z*4&K)>7GzwsoITr-(Pcs0EfsMZa0m=z>h6^1WNV@be$i+ ztX|F;Cc2;`x$E3=AJ?XxuZ5RlxgypHM-KA|aOF>a>ChRLSLUty9yfLMxgIeZmN6B6 z(>H8&!so$xIX+txMk!L#)A&tK1Kd&IO~r!WyfnXZwgI91C?6Kc)ea5(H-qu*>X1Fn zj>}*mxH!U70kycPRS1;|prVkCFy?%-KarBzo>J4?n?$T&hK~Op<>-0{5d0U}jD^At zdJ@C7f6%{f`t}t)V!4k}63R(EU_#!81s8CD*Bw~*)F(nhQRhPWgQ*eP1GHYO(CNWk zn$CImJ?2OxYmSC!?C(8d`LnsoK;XtvjL$c3@{Op4v}Iko3MPS%mc0=is5SquCIYgl z5OneMO68Ub;el*qJeoSvqXbI{`z|qvmhGCqwdF}`cg62^A;xB@w~dvT3)Bfjv%s)y zv4lIToEB6#h7usQ&QfZ-1~qEOrN#11se0A&qtZ-cuf`#oO<;nw_!on^-oBm zAV0X~RB3S5S?+g-JCKbnjv0KOe@4Add37Yx5+ZYOu_`PQ#rL%#(>@JQXsn)8`2&kF zTK}kEggmC)SW+euTbof8?pB{8 zdx;S-AEH-(e)_L29pXJI$*QCX+a4}CX~0%gs0^<#gMZziTiup{Fn_w$+*FzCX$1I) zBuF*4Xm7M8(eu8#f8VRVjSm%)r=_iJ#!N3aKVBC6*`lO?UXS{5gNln|?I|eQbPYis zWb(0EG0tN3cKCbbxWAW+|W z>OOjof6f}ht0kEX{>E;LC22;?5Bte1It9IBQfgyp6@64w9HHJlgcN?o%}8((bgQ%X0%=PxhvSjBJ;OvImVLP_h~>c?`oU0!diX}uKi77sVxd;W z^Z{O$|1D>`e4J@{^0Jf$5FkJBu{6~tth&!H|NdPcL$h7vhow>&7oTGMydv&7mFpR6 z;<%U%ln}sc*4?-}jm=h5^{^But>8hU3{LhD#<@~vu~IPGCXFnAP?L`HEwQWH@@Lz! zKxk!GVPvx%S`w|8x%>7*lclpghE=ElNitk@Uzasn77ZBETulFV(5}Y!_jRhEvxzwN z@&Nos(IR`D#}RUa>}~K#-=@vKm=^M3iGoq`+B|qT*xvE$+ql;zea-cYZchfKo8~eNg?-Ub zvh~YW#Q|jBikb!JeKvERv+nl4VMT!WUOqeBb$ElgyG%S;F~FYzu?ohp*P~WL|I7qM z-tyW!1=KQ9t!nLjGo2Ybb1dHG{zd z=i)+P*awFE|8Jy)-vJ7&;NGWW4pdWa?k@FPTFL&6lH6T`1ga#voRw`3y5elZo9Rd5 zVAQ7P^f24Hm@Sl={8(1_R@51<2ZSLC`bQc6694K--MsCl?kMa9JNyy#uVoASSyHK{b&I6wVHX~ls%j>aCS(9z7OdA4Yq-vI}vLUUTz&opu*l3VDRi<{?Y~mCp~L5 zLFpnU`d#`?T0)^o>py|E2$!Y*0M^Z|kC+8sr!&UCKACEOZ3V}!YyxEuq*N4hJ9RLZ z?wXc)>fkj@yT%M{+=t`!X?klh_{xGr1;^ofMivU}aRevf?ojS&Hz2Z*Hr|AJ?589?f6r5SG%igl+YE|D9Ig#^PQw!H_`+My& zsZF_StBkV5DF>P^-)TagpA$$*1!irI`vk6pq%k~SzV{Mm4RKDWwAJGMeBAV?OPI&g z!sa$^}c!>yqg{2ZJUUbJI(ov+rDBbPO3m^ZG)W8)kXz5u_{Trt8?B9{FQV z<5Za-oY5RV0oYmLjNQ7CxqzKb4!pwSTkq8qKi}2Kn!#Ai9P4gZZ#AFs3{P1Df@nSv zI3@|F>`yW>E&I@uYk|sYbKW@J{LLU^{|y!YI>G`+1!L`PrVRMfF{JN2?{8AhGq$!Z z*tX@~JAWx^^O04x4lux}50NKBX)gYIOdAzl6Xu`ZLb6tDnEr)R113;mZ(am&n2)>K zP`-De<3UiA?S78sb7#m?YjAN%P2iR6#jww!nzHq!$zlL>`BzF}_LQmY21ya&7n%AS zYQ`O|>9^0-01|F*NX19r4&ak?n?GmD6jgoPh>qED^PTem7@ttj9nBY+Dcq;eno2XI%r+WKx0 zD@Hn~#9!sZGU@rriZS zXWwC)*B-9P5Kfsc8M)zVg3ixJ_xFA9 zl-+C5>Q9TG2~^Z`t;(FcI%n0vxfx75h||&@r@vQ?FN6X2rXU2?3(iqluu_yldhe!q zBeD%#s{Q-i;0$}(V^fRd2?G!&tN(u-g8y>~{_p4+|K4z>Og1u>pPQ8a1{BC2^R)qE zwXuQag5|as6T!sy%(n^%zmdl?IlH@rPQ%+EmxN2Bm5O*txPWxJO+rspXsKaBXdiv! zX5u4f=;#aJ9;awkiE1WT%!B;8oq2BgPF`|$-Blezbv6AH~NKK__Pq5`$cWM*?}2EN39KS-W6Sm=S@u-GJ~Z#KIX zHpcDx##7HRx9}CWOO{IIU90e^**TV+4)v;SvAt3fu zX7jBOFpUxLT>K~@hYQwoc#;{C1IutX)FW2ge~5kjI`1L$>d9)4xnKf6GkwtOP211r zv{z1!=F+z?3Hgo|`V1Augae5Yn=_4)iPDyD?)kIsV#nhU=-T=bvL!oAb0}ShA~A*~ z(>{Y~kxrtTHWflrP~V`ew&8B_R(%xvJE~;FktcmWXa)C#ZIP$Ot_yl{Y546l=rD7M zw$|`)wL=PDZanZQL&in$#Lpb_a!bo7(>CdwkAW@|q{2=iFH<^|ri(~@xGhgTL`pmP zLWH{6K7az5l*B(9C~vB`;?rS|m?1sp>lEorfjKkjTC%puRfNeTZ};Z}b@-~3Pz40GkXctcmyZv)hor{}$&eG)40FEc@G#qzQ&HLW?h zX||KSk(q4Y0^_%^k{D$$Un=(&#f)ud$7e`?a4T1=^rzWXZuW5qsKkrq8I0er`dzUG z*ykuRzdfrSNtf6uuL6Z>CfD| zG_}0D&2~tz|K9Fk52}OIJt;@c9q0!MNZ< zQ2y%95sh=5T;(SqNN-u%*uiIe^6mVGOc@(X_)5aXhZ$*@23sd1OFShP+83`@8NoJj z=6HSz6K9<#X*XwcL+vGzcu^T-Ud^Moi9Us)eVGE$(4k25=f-3L@^;8xUe9&r{7xsa zR2{5U?5q*s;^*Py5)>lXt_6Vnbc_I#8$}FyDY(AsMVp75T4M)BE>lkeL^iSyve+f2 zmy&60Fnh&i2w0g!5AaoVbKCOqlAJw&7G{;yAaLZtvy$lPAabh~X_|wMrEKf*OQ3<3 zw>k)n@(&>oU;!pbR8joPo?+BP8x=A1I$vvbpdNNIs|31Fi=+GH6PU{AZR+DNuJnGE z`8zJRoo|WWbg?RT%YL5)cfXyJ$TEV*|8BUVWaKN@_! zpAJ$h-2}`|L27c#q)8r-4w)mRK=w0lxU=Q&quYY}aJbSgH+{6UX~-w1*2|@1p#a|h z0I1(=LPd@~|3A{D;rtd{`QrD#t=lAIA)F8_47aGVy*kdcZ+vd6;^Io)|IOK|)7QE( zSF9S6+x@9-71;pEy490ZiRyHFwDwO|T@$U|8+f72l^1j?`oy~a?R4&IO3RxT6TowP z{pIUL<%^8>@Ap=ch<4)ITZhSxfIihO+~2>z4)Ce0uI8`*n!ra(F+v_j(Zr>5zXSdm z{Rdd$ml+ZkOab@42>;2IlhJ9N+DXoJM9V^u*BoTO$>K!QI#j&_4d-=bGIglYeuFUH z;VMb?&z+@le`{qTNj`l0;_)w=5#wsDe7D+i5RM*)y{i9B*0ohCB}rh-^LEO={JfdQ zO?F%gGgpqwhxq`Fr2F)0CVQXAmB<93Z)(G%g}>>ntmb4P_X8iT1wDA3yzea0xj(u2 zlh1QF`JB{bSB?Am$}ny~+e#T0PuFt6YS|L(I-r(LN~v9qSzuUG_t)EwB$WQ`=lN`_ z{{X&?qz|rjOhJFYxfs{fn!HWty!>f34zN-qn9?qlzs<)y{wdNh+>#qC#M<~^zo8l@ zzS5DFlHbR?oFD*Mpwys``5Y8t_)Bm!t&yT34#@kbT zxMB20l2GyaK9M@d9$yPYV!ZUg%YQY+R5NT z{izGH>}pMo1BU7x)NGBqN4QK;(>dsUFK+!v<ivDHB%izga`FSN>A&Ns>JH#EHy1pS6U5IqJ6qP8@Uc^qtIN(=- zO7Fe!qMT}ZjcudZ+r0#J*Z97QZs(7=>)YPr87=r+5dM|rgonkrB{X}KQ;G0?{KsZ= zTI`=PZ3`fhvp!?tL$U59Nub@;e&rvijob#8?r)=xQ#wEf^VlPCepheu_Q)eG@+i2K ztE+KJA|I!l+vNUEfAPY!zne-dw)ppe$Yu(DKYfci%rH1UFM~ov%J6aU6||Cj@`dew ze&mH8QFu?&8_F?X!(UtbsS|;}EcrR@wp}*ag+KdF3?*AgO~MvZ!I?}3Mv1n&Hs>Eo zaeD0=@4Lx>QL~*pckw~2r!$oag(dB+vNY0936(#R5wLKTW_H-t#=VcDC%%t5YF?~8 z-$uyAHoF+ww>QLbtV-Y)9mE>2B<-k7{TfMdMEgU8ZviW+`qQ487@JT~Dy;jQD*{&D zRit^7Iwd$dG3)t8*~@z*^#25Ae%rx!R z4MoWsQOR)yRHGK(XTI93(;U(d!~y2a-}8%ok{M<9Ydc;|y-}T^aGVn8!>T(nc04I2 z1@F~*PwzXuI5}pZheQ(!C7J;89(79Ae%0@jL}9e+PlIl=^5gOezobvexJa=}HBJT2 zP=mRswr}zY5}VgdD+|7pi{nM6+%Npu-dR8_06*fYk7JMZjVOM+UKsu(@T$7X0({K$RQ! z9@63bA0Tc<{$1wQ7x(M6SX6lEm=}wVb}ipzsL{;-&zpl{Ipgp|`M_9y2b&pB|PkRGw%AwxxRTLi_$rfAg?Gdm;6p zx7aWd4ZRjn@bLH#P{f#yV^<&`|ELKyND>Dv{RjkG&KXlwwgk_?VuJ!2 zSIh$4Mhh+=T1!bQeYM(Z!^L*MMve0Z{pY#pQ!k-Fvi_M>^o7=|bu;xZLY=dcXxx}y zkKt1SO*ZhoG77$G=z*n{njLtf=fWltRS5u!JnlOxAHA;rj6nCbu#fk)Co?ts3B`Kr z)vsA~eu83#vPONKPoQNlr67_&Hz#42UF|sqELMfA18qt5-~3G|Bb&e@r(;f+U)nVY z@_~w<7r5tIQF%cmywk?0cZQv%%J&e0yo0wM-emrM?Gf6XV}_fRJ(~aw^suN8nYZR7 zMF{u)UB6_0kJUZzLUgW_%DncD*5fB*q$PWIe0W;B`@2$v}OJ7zy|kG+4~YE3IpY0I>BC;meOuR1cTH7?XaY`cLJxc za`LHd_pm%RH@3=v`pN#jgJW$=v!t6sLL16P?;3W}f9Xz7wq?6X;|J+r?$c@F*!kb4 z?D~I{NlsE@M7J5E>fOr?3gM{zppK~RsnC#8bmBDU5{cA{vc=HG3-n~NKyx!E$#}@j{fBGVwz^0|6V*&EPvU)IxHlQ+pfPuun^0(!y9wkG5|6rrP>*KEq-PZ%7kd z^LnR0TAxd>0Bk|+Dbmg6FQy0|&Z#t?bbU*c^GHH+{n(Ufr^h}nU?xD za1(tC)CO-O-en#PwzyTSvlua}H(ut7-(@2X)u}l4&RejYL{6kG!*Rs1#3%jz`e{uL zq(iX6pJcW>%CEafp4vn-e(|g@De-gin0$m9{T{JQT}%*)yIoo7bUv!kLCUH?-wLMh zgJt`V8#v^&O#9^<%fK15uQUpnNu-jmIgnTz$_Sd0FZF$vS!gAKcZQcUio|-#7duZ0 zL5sZV2GE=F1hTz6XD}3$G6RyQod4+KI$oBIk_-u2%;G-c6H^rr0W8*dIjrRNGJ3WvEwEwtvB=5CwfnIGi+2eyS-kc|Yr;U8!slX=>&u=gmxm1Gjy;b5 zdwL4^?-_7u!tU8rA~w8AZrrRj{|8MK$O*eOc?oj)hujDk!Fc@PPY+znGKQjjM{{X8 zUHlXHUHW`a*`q$n?dR9km~z>`Bdk(2*{vKCHJhX)2o9|Z$cP~cxtw;M@2m(_jLHkt zitM_578OINxHH*XKlTFn)Q~>;o5dW_XNSe@Sfm5SxNl1QrSl`uk605#Ha;TWE8;zn z!jgH=ULCg*e4xDq<~g}q+oFv4>x|_MleSAqj^3Nk4pDY?*&)m*Q1WjmXoqfepMfzFBVU!)|1!n$LfnM7DQtn|?8)H4 zZN#zehMAh38@?wMNoPOX*U8I9!r0t`#-^kEy&B#2#`bojbLjCIpPkO2A{#O4ijIF@ zFEewGFkpGl!nl4?h5dH6B;+520WHzeUjb|2^7*^P%h)` ziK(rN_~DRxEyn1!W$M{gAhh3j_{15kq{eu_MdnZF`yVciR`OtE}zy^TJC7Hih}^-%yV+f%*UV zeH5h(gr3Z=)g`;4nr^JS+DZ3(FHT9q+{{MW?=!gwq>lxL$|et081!}cax@cITw??| zsTB@M(#BW>XpuC8tQ!y1CmrqB$-U{A7hc#xja#Q(i)rhE;Z%rd|EkE!PXX6=Wwwa! z#dQnU`hM$_K{`f+VS-P_(3!sg^6!hC>B!GSG3jAwoi@9?9;OI_+5*a&En{RDi+7z6Ptn zhY?NFWCK>hV>i5dq*q*0ooWJx(mB6D^PXmbHW>&Huq>PZ)vnS@&yK`t)NQf@e&B2K zUnZ0GskWxy8mXjgQ)dcM19Ze6M`ShjJicZgeUp_PgL}(EY1y^mzKxWz-8}aLrYX~L z@8TGi(oCb#PQ51ULE)siQL%tq?cd6lukQ_qWtrd(kYI+)V0w5Gi{$$f+W9FTBPi)# z +#fdH%U%X7nC-OI%MqV8B;l0==zh=JV5f6K|o7Ie|tkX+!>+92g>>X$bYbm^`u z3EVhOFhK!a6t6$Me5YrC@TD%3&0(ys;txiUb9NYJNS7nZon$+Y&FF`e_1iKd^Yj)2 z{}perLEx2sEkiU5t0_AGLejSvX(H)d-uSkte)`Z6)s82C#YaZANdkrN)P{CgiC4e9 z$8q^i-Ale#{3aj&@8~Z|465+R$&YC|wQ+VURCKEU0b1li+#j(hV1_+( zdgKR3@8jNaCSR5VQG*Dd9nsKkn`Xjx1fP-9*Y|I!^=VCa4jb+37C52f23R+B2hlxM z{B#ydM95nBmB_Tf$j;-4r?T=a-qCg4f$0C^+`@hNF1MZakH(+omg-(>C#%C3)~1c- zbZMr*v;#F}RHIhCSwHN_A^Osdo4%_(BuDTh9W6oKyb(Az;A$CkH_zfg-TP|R) z@W*!L45$2K_y~DdYJ-vsnKNo32Z}z*FmmhbpQX$yuG6e+EB^O6C#QZoszt8j?=j_( zD_@6)%FR@$q?T#g{ciq9eT*tCY<#b&i3#8e3A~|?0{D=OTD$e44P!GkOOsxP^=mm7 zv_;yb$vtC@L?rLQ#XtIlCGH_}mD?y*;jlQU6XxE=O0*sM zxKc$>vH8c)+mTZBK!+kzE5cHY-rp+}` z?tWoApE0?5JiMZ;uo~tks@i`U!vIWq(CM5#6%Hd(+zp~Xo%c*u1G+c>@ZX;XKmu69 zKTKP1HuJ?+{OwK0QiYdue`#H#Zd`Mw3B&HiCxxmk)n*x)qaK05Uo2};#w|HR zIJ(*R?sDoJXak|t_jyCPQuhwK^f>ooLwPGneHA6e?MWF99SeoHi6GXFFie{;UX}s+ zbUzvA<+xy%fW5@uo5!Cgg;)JGPFs3hW-~N1OEz;Ul)jDXC=alFyFAM2^pWFd+?@vg zcJV9waBs66)rt_!owT1-@{L&2Bz;9=_Ff;X!$hGE=)eb1@)&fB-ABT;2STx6or@yI z%^IKQ58VWLIgKq&O_HVhA!Pt=c9x$z+V4G4jzRuwdN3`vE%UZW?ZpFFwn-m~k*G1I zRwcmoOt;z3)9e4!(^*F~`G@^~bTc}nOOP&U7>Kk|N)8DD>FzEG=@3v_Qb0f?MoBne zG!oJ=MuWgc4jFuZd(Qbi&;L93+3tJi{#@7ldS9=14HHw-lg{7%rE&iONJOg^ts_7K z1&y^KJ_wO#4+47~^A2ILzwvo)&b2uW>HJA@V$ITRw#?GQqrPs!?$kFaSRQDKHNvqQ zI(bHAF6Xtmre?me!sdwO+i;{ell)Bkd4 z1dKR1&-hc#T2kWSBvFap`(;;6ILMo{vZxgvD2P&tzxK^>-gT*4-$H<1G`hKX_`O%4f1+CK`>Jheyi(^yo9`>4Vx*IFKy#Y*qGa$A;EUb~)=+cMkP zr=0v<**sf6!tL#>V;;R@z+1o+k-@KM(fjH>4RS{;kfviwA@F}@>ykdgH9j6|JE+UE zjdJ|jH6V9Y?AdrG<83EP0?sxj8Stfk0`+O%$`iPD7bRGjpXxQX-y|o!No6%47!pf` z$Lh|{);E1<_%g(~Jw;l}heOb3N(*g>pAV7v=0K9*b6$6mb9S3Cl|B1Zhe<&|evHHF z3brF8Jt_N%KXX4wC8kfT%D}BI^4|LOoARK`=Ltp89pV(%+geuS&cml3hO_${p($=Bbs_v`=O z^(tA|dRs4CwgsJa7lamN)3R49*OO^P$>EP&qw-S8QCUyYdXqx~nl>ic&m9sJvl+cF z*}@{}(LUIM-*`5_ZF4mNW-?b1CG-+`g1Pam;4;d6i!HTdwA=39RBZrKd{}Q)`z7;E z)h^Fe*&>T0c};13VpY7F&i9e^C@X@1ECq)M=ps8-AoR^!>0+XH@P4G8U^Lfc{yj8zG^;a*FK5yLflXs?sRp$(3KINbi5t*F55~oOWQ#Hpz=Y< ztVj(Z4N;{Yr3@jJ!z*$ESstqpA7=!h2nG7z5iYz{FX>1<54kR*IC#X8vK;Ga{x-J+ zpH!)1K0BzM(hmYE5!_Ew0`m>$TpP6F*)b9FX+YuafB4z7tcyEOEJ*6tobi$Fl-{%XqaIIdLJ zDp&r{zitM<{4@BT`BQU)0o(%0FJB4j*Qj>3I_*93dh~48VJE0eK;f0^}_#9Sy-db(@;8Q;=Xl>;pC?fZb+G2lc@930hnwF4|!q|S) zvOBX;H@A9tYJH+q=3EpVpL8LtrThL2l2t9dDbYf>)-=bH*}o6ME^DSV(XVd?`y#*s zdBUgW=?gseKi>22kCEF0!uy?40R(f4gO74|WXvvBKlS$S{n7Ypl1DL<-AR-Q%?#b4 z#5peOs+5_5I=NJ<)0rNBklrZ_-Gywa$EtWdc1%W@119feAPf@2jmn)1h?4w|J%U~@TJH}%(kKFo$22On2q`5$i#(;bO}-_FWo0FI zMXv{l5d@Z-2aJv8|W*T0{7S3wbb>VY4X`E!9s{9^7jKIDnYhkV(TnoGF>~vNZ7PC&;`V zdiHd_l42w|nLb^Wvx!Zxbnwk1ZN~)+-_|s{Yi&bt@~|SbxTKJ zKWu1ATd;Q}M)M|^XkY~vBdyHs1)QYsn8hd}nAh2^NOsenVIQ|-3*&K#SzySJ)q>M( z>vX~f5At!wxADs3g$L#^n-kR|NW29p-!JX`K)pOt^H>gL;z>ZL*56HpNRS2kPdVC6 z!$XCQ{~2kH1^)d>^|1xf@=d*XE`(w?eGRuPb!V~N9DTi@2WgX)cV5o)lt0@%PQIDwmb@v3Ncr`v9ZvuT7lDra|)?VcHk)Q|9&i zV60SN=H-FlEvB|K8(B^Dm0G34S#E6b#}Qs08934L37@NdTfq-tEm2q$yWVqrvY!*b zL#YpH`VUaQlINidueSV~Ho17ZdGLC$VIhxgwSipgaqVhv`e^hr(XWh^sZFRt5ke8f&uy+BdOl)^Ge079t@jKCxP2|N z=g1Y!grAP+E45kPI;%k6!$YKp?Yw)%=3ej_aP|IS7rIG?8XM9udsC8#>JtEh7DB(K z3$IP-;q@C$No;(>ayfb8!Lgm3&DKR6#-J+iEVminF!e7ZEa|d1xBmf3T%V!{=c$N1 z;#=lYoiHd>_B7cfqOt0Wgtud%4*}~BQ;+J`!_8u!X70K>f&4F5M=|Jim;%OdbNj~suMZi|E{{9KA z)%v%nC(m{0nuW8zD>ipX$nU2Uh4vwuK3Fr(`Tq;dy^Va91hzWG$l(L2;-IDtLV4mko90XDKLg#fsXq@It+LyMk-xl|6ZeKy%E(ZFp==dFHCmk(5c>CZjnz#iDbu z$^qgw`eK&%6&=H}F)1pBb6|vjcPuB`$E5MBN1Vplo_bW#VCSXp^!J}rBv67=dK@++ zk*71B_1()w=@cf6N0a3nnbI))FCS8$u%^&BULMM^Q9wQS=Am2eObLn!|RZtVciMUCzzi;I=Yr5_s#0t{M9Cmm{qq z?;vW@=c2H#H1GB!Hi)CII9>MVt8F%C+|8I?7XIV_vTkL`r=4x>>LML^HX>%$`egm)u1ZZwhCxuEzj z$f4fc@`HS1jW0GrT(Lq9N$wrKXu+Y+$=g0WMdzv$$fhA13Jgof8+*jd-Ws2ktxfrC9CYb0#34F>+&Dkb>n6owOR zzWu!nsDsXjzcfG)B_YspSY@47w3Ks}>yh!~qmh5^ag#JwS}Zj7zTe7JaQX$ql2LCA z21D})@72waC-(?wWw0v(>&1kX@U`UW#YJ*+WzTZo{irq>c&(_pW=jYN0+Ff6tpo=R z2lY~*JeIl^Mcf^J{(R(l*JeJEc*7D6&x)o6ggW7;J7Tb!FaBkBvD=U}Y3EEU8TvFp z=qCfofDTSJEq#F~;mt~2OiTYOm@eR7-*qT*BctKr_XZm}B?v*bt{ME-nu)96_$F}m zi?I1~0IsHS8Mfxatq)EaG}5SB7YzR<$`lP#jJ6*J_C+BZ8L!oUqM|mThOkwJB%XzUUHL>?%jt5A+HoBkdA&4rfb#w^c8pYElm9e^NptZ&YEi4&|Wsj zIhHp0n~*h<2w4WC$950{BRNq63Q8faXkIIqMjUIoKFCwFlMQ4OX&NybHK2@%EOGIz zui2qYNkNB({@d%La{k{{V7vn_^kXmRI%pWw+E$Zv6&73tLH?=Otn~eA>olUBKDX%d z7Je;Q-DmO_V@5I9tR?)z{k1SD62|Ha8t!9`a*Rs80THRlJ6j8bE7&UY-CSz-RO>u; zxy+lD?0h+n;%Y4K!Lwox5``IxTpea611sZ;%r_6OaSmtDCBL*GBK>^5d)jRM9P=|C zO;C|3(`SYP)}2^eFB=e;d)<#JyQlGPK{dn#1CNTtm zfA3JUTPD3g$i?VL$AuN*!k){O68y3(F_$hUTZfp07k95P3hPgO`F3G=?p3KcjK%fF zPIGnG{({FT?^czF$eH<601r55v$VjFvL!^JW@7)js##LAEPk9zP~xJQlhwXjwb5X( z;QI69PNkO)CyusFY!|~iLGX7f!NaG~p&7b8D+$GQ%WZ?EM&LIv-{0!wB11j$P%oZc zeHEj0iEITkP(v_84Q&6-rHgKGwRPb;l!rj4GF39w52K+ox=q=04Pef<7HH4)zwmP5 zZ7~RXv~onuLe`2|<_zx>^2t^?VDm8pzt&SWDVSE5WyGC%#2hFPFg-Rw!by9cb75b* z-za&-d}gNAb|>@kXigs@`|ho&DsJ9ueMxf(JpAprXF`)jI`5ydp%VPcawP*Z0c*Jg*%?|c!IK&ktRpZcu^WrjEDc?tXAK* zc4tpVv>xF0hzrJDP)2(PvuP@pknUo{+z=?vtmM zu`jJ2c^eXqDS$p){Rc?>*O{o7Fb)OT^%ihYyAI39C6kXzo7y&d4*=Fv9|y0g!Q}5W zf?Krad)XT6k$qeD3yr9Lu*RP@un%isV>TdF-FH7Eb+?$?+$=i4`AR*PRy>u1p-ZRP>Wq~{OvPcu&~ zdO(aHe+=u0j0UB6v+3{e<2eHwkSWlvL}=36!sjTK={l)Ww87P=$o4r`M`}d5nsByl zF1uGAuHQxAs&#ky)5N6mE)--a7~#B}EkddFag=d6xzhFg3*hk~b8~7Yvu_wBi^)Ph z7VFsu6!~a<*j00ScE*)TCgeOIwyy}F%g>@V_l5EE;C>GPODAAOb+psg8I|j!#)<~b zBilExLS25}rViHc3sl9JSnW{Z!Ug(?S|djy1zQYOX`xu!QaD>gSncv*LzqwfrcUYO znY8YthL7n|Hy`j$w@b9AAjwxvQiHOS9$Z)$L9NX90iu1S%gaVQtm7Ji4g9cqSO_cu z%H-8MRVWV{^A)ieuii7GC{LfKDl{{#W)e@+1Cs-4Puq_BMD2I0<~tNyGCq}-Z5RC} z3afE7b-^WZBv6$nTZ~J;{~wliRdAk^Kkuu%)>O%EykUb^>BYwr2`%BgTIsuSbz4@n zb4=88GRG3(B1~^Of&MU&@Ur-IbXK8SJU! z3=aF)FE>uxf25|exn<>94}U3Zs(mJXV*NO0Vo$ka0JdEs4rJ>SjsR;eTsjP`CwG$K zc*{fwSx5=J%Y0&W__2J=D(-NM|yI!O7eXZ9tyy=6(Nafgb+i8%e~FCrkxjsC~x zfB%Y-_JEVT2b@+#HO#J|2n46gUpMnF?jS)z0p@>du^onSI*e|}LCfki>0faoXEPY}Q* zF@$OHF2#)V2cOR9?eP+MtShJ6CL#Ck;?4{0J7;vC{P5&<;ncwfaFm@IAe=WV_wT%H z5&rc5nj%0h*GG8zd$Ye@F*SEi@t5b1FRZ}aQcWyVYfgPcP3&#xU7IQpQr{*e4F$Z8 zY?;~VqWFq->1#RFVtH>HxwDNe8cgY&I*Cfk01K*7v)b*1t)qAM;O3n8cH@=~xxJQBbq2=N%qWvvLIV^9tN` z=Pe?3$Osu3AaeB}WdyKp=S&8&^COp#HgPA2eB%%flU(~$=_g&p;#PxIs%sNC_*eTI z9h;=q>pEXuOSpf|+N!gj@vZk209UR(e|_h_31+tMF6p2*121A;eO=LObTRA7onEaX zq3Kypm(*dY7`fkAkYghmz$O++@W>{%3gxxVzW4HpY=~k#2n2KIv;kRjN>mSHOfh6% zFrdY&yK2<)Uvv5b-D>wsS?z#So7nE&cSnkpf6Rt0)9(A?@fk@c1K1GCm$lf0Cs2+_ls`SB(X8^ovl~zC(ru|=HQcb>Y4~+#`s>SMU zn8miy-H*bK1C`GzZS=^6*5d`2-RvXpVFS@1yb(>#zi^q*?RG*Uxjp(%F6~~3`jn#Q zj_DG3TznP@zw04cJ#{<&Q&Hsg5s_~9Z$m`eYV(*H9e@P$VDjvCVv~0V(ak(cX=QlW zeX!H3s7JxZi~U_rF+0dL({t4gzcmfX_rg8g5=0;=S)LhUd>RU0*@W5i;GOxVR9DzN z<$~3Rs(jvzPuDBpT_sHMG3l``@;S`Xqm7 zl5Vhv-dN_w$E`yh15J55#dxxreFq4`A*O2;sny~VsfI)baA1mp0Tf7!y<6ce>#dYi z4S{6wWy=;7EGf zl(@)sG_hVpaS4RQ(^RA+eH;kg& zi*+aGHzk10?JIg*W9}>xXnpvh5}Pfys^?awT)a}RFC!68uMqnc&r=TPv)1fh6@o%Y z4_VTUY#f-=^7<}PY24q>w4FQX9q&o*%P=ymWpyuqMiVZG(%3%2)<`HqGH_u-e&%%W za`pnBYPTfi0oA9ezusC(Sy?cb5{cPJKD_{ha+n!I2|Y6K2Y<)Uc1^#p>(w;MUwkcO zImGcw-e9>;(ZI?+cwrObj3RQrc59kZyW&MFHor|K#htDzqQo@o1Ac0%phNMXzySca zy(PfozA1$F741VV4~>?pd1e%Q*z3L@Ckwe^Cth1oOcqB#o^hujqNU0x^bN@Aok&ZW zeKgQm;;J0Ws{0LoRXX-F-egx~4{u6)lu7PPl^O3OpHhYaAVrwvGmS;%leTyXCk^56 z%5%^?6h-5M4u8nP$PQxbDo?0DXlB zgDq7$=@YPZvO|H0wevOWsc7(ilk$0#VeVZj?u&F1Q62bQ=&RPSmtvE8 z@=t@sXK9!^rv7H7MwKM?(f2tr6q0k<{fvnV$g<&T8DLW>kCyOg9mgJ9k)4!@yP2AV zoBBYThrVsIFSCszXF*EaL|hJzB8$8}OZ%*Miv|_uE0alL^_tULVf^*W8` z9nlx%yG1hO=2qmF9;;@J#9gwO{$V-T;ls-?Z25dp)}rB(@>mY`3hN87p$#U}Mq%e9 zSEYbsi583FP?Mpw2?t<-e7WHRXmhcpzg$26DtjAq~Y?%nz@Hyc`ikaaJwQi72i=L0_!> zn`*)C5UJSwAAkuf4Z`z4R8`1**mx=KTK0l9_Z*XwD!lbK<}hwPyjpG8fM)v@XmM`)pUf22AL-%>`7Z%43{{=a8}bxbr&JXqVL1@ZjxDbu_}>%19Gs%pE~wi`6URAWWxl0j-N!FMU(<~ zT%2xkA3xTSC&-OlV%w;dC|ihlxMMd-uDL&1%%jv;l{0{U|8`B2=R<(g7N6Fp=!VX( zlPRg$pWti3-!*fQ5+lbl+cH_+wez-^n#a)!UQ{mPN7gLhZVzrS)yA9QX1*SRy$urL zy1qe7yV^E~ZljV~gojU!+gZ)}NKV~QHJF{C+p|AJ3#Tx057u;0upQi|*Z0VsbI6fz zOE$E~HqX8@@slC;g{h6V$-io&+TJ1oN)irB_ zSC)7~zmae6L2xxy+r$)To!@N44C>)KUn1I5iOC2@Sv@X#i zhAZ3zi6L&`ACGQgXDNh6#-vBt;n}g`lONqZw*h_^rcKFZDbNy;4&-%%qd@lTx_!T? zd*_AhZsK?gX;03nPr(+(r51eD#cKKdA1F(5tyN-Ai*zuP#(4)2cb>V;8tWZuNH|rv z4FbCTpx`&+mM6CNpB)_A@8%8rJ|phU;$_HWUqiL#>Q;x2Y0avBiRVgln`8MBB;xt) zS*{~K6>gp*Kju`5)jth6q{na=vw%W@WMoyYSwTp!7L6@{T(WDljm56qa6| z_`XN2{%FBx(^q)Tkm|!6tzE5YhB0F-~;kBaFU3ej~;}=U!!xkI1(AG zDX|uV2yPMkH!W>*mm+qj_Fci2PU*am&pcKkb?M*Ci#yUXJOL`Ub1ILADuaI=4?slx ze7?HK^f~`|n+$6G@Vl4XeM?h+7ROLrHd87izjVF_hX@Lyc>D#iD%(6$)|QRfn;QL~ zH^X~6j>`p#B9t`WPFlQc$aszd{koUhQfWpgJ8eNy&Q2N{ntEXf5cMWO$&)q#F7{x7 z*P~Ce4a9i_h{?0bamz#}YZ6+2(!T0M@eeg{KU;h>ySzFSz-xPWU`Hsw|Lld&XPK#= zn&Q++AOQg6j#jVHCUQZAWl`++l7*Iiu!pl$54l>9J8IPZf=7g zd$bQbs9r@<%3syaF20I&mrNjE@{2kC{g2$~vJ9v?6K)n!Qx*pP5GaoOR5EZ{{r0Ni zo^iNWF#S;UOYi3!x!V^Vr)mc{95z)Udzp*ff8gP^Nd4f$2l_t;viTAtNvGA~ntK8y z!&@+(Ri!9>fJzqH=}r||W2xcQ$TR##bxjklea)eJja^?kGguUvbz}474>D@QyYAD;jQp`5*l9tIzUutsy6@%mG#x2}{7=N5PqHR1b_??Ko(jL_$-?)AKZlM&~*Xk7A z*4E)14@vc3455R+zk40l^&g-mOO>OYv0%8~jn$>jxf?I@M}{n^UlC?qfgDlPEBgIA zSD&A7V1~0bh5C1vE=bg_@@iAL~{&YDSuz{xAd>*9H+&Pw|XsxB?7qf(` zu`@-1{qlhQGC4Y&;+o*8*@MEM8K)0WTn3YAetUYL;+5W*V^v?W$juSMHQy97u{akQ@qD@61}pQw*| z`pgUbb7DK@utdiLq4JA^3ixI~AHH6mb8z4<@k$9L26y1uRQ7v`ym?Rl16^S{D-3UM*O(KzqOf};lZ%`6#L%yJ zLefA9Z&Mm4ktns`creK9fvc-@y3?ywFN9X8b;1GEm6MU#2zpN-sHHDNW00i6oHq9| z4BTNqZ>^1hr2Pl*q3^1`kRdis^fFP3C1w-KLC2%``+bAoT)qS$(keMzKQ-3g7kh{d z#s}eb$u5uHYWk&~9uI4kxa9lb&_3*5|5QixGU|O$V=cKM7O0UecO9*&V)2#Qf|Xw^7FT3!l6h_SIcr*zR-S;T3- zzPKk6B+1kL!2jLnDiVR=M@fc7QCgtkg_C+F-MC{5U^zMs&o3DT=R$&z4jQ@fX*g5Eq6g=ABE1B8V4d<I`@n4y zLuKRtN`r`k@m~MQ+a7MV#ymq29GJC2?xbTiTjMLnMt^<)L@TguN(=#lIH<6wPrA0^ zCm0swX#{RWB;0MKx9yum^3>%rq$mrT*B|uJOL5Snm4SJm6?>)&8N`Hc#_S_)27pA9 zb>p?J3@bNj1?Pv!>MrzyIen5;yfpIpr;ghEa!gp^(6y8?Tn`l(k2s5uqB@iQ+|akh z7aeQ^{pW8dI$ZLD$$CuhPm+ z&s$`gCn@F!b*gY{_8ik*OU4ZVGJi%3hXb(xQ71&k2@B&OkmcAk`%r}gMbgresNdnz zU^wnDxvQ#=TTXz}aW^3qTix4)spe0(g_(!U<}(T`g!v5;;X*+rqaO@k*qM%yg=QpPjFplGvGYYw)JRYdM{2 z*m^%buHC>epx3wV>=h3eDeyU4Mk#(}MeBp(MwERHLJngl_@=tjk?j@2_>|$J=~}$k z`-|t&*svUSx_Kubat3)*_jXbw&^q_G1=_$O@yU)*CT^^_xB~nl{c2lgQK2VkkpiLC znZPbn>N!_*d5Kk+T|29SD+WWukA4Sdj1OyjNnZ~POv}U5e|Ag>;H|ZKh*`}!87~}+ z=X88f$hepc-o((wVPlm+&Nj<=4eEnO z!`Gd52=nwQD{q(_MJ&(&n%)M*fbTo!;6vRo#2NGQw+$nS6)mQ03rYomR7n}fMKo}1 z!PHc+KbDeMxmL&tA@)J~j0t^Xj`GZ{W)X@oFfKS!G_?)ylNhJ&$E#JMB+|V;Th@CZ zxkqrUSWhXN+HhDLT%Xs6IC~UW_)^NV;;dbb^l#@x14{&&X3ntr=>w$UV#@O`5x{iF zIke|9L3J^!J5(tCHpN43Jv(&ATig$^$=FL2IsPA@S@wjs3B!wC-l6~x24&Q)}jkEYI=BwX*u(;tbRx;uEs)C=WiL!YwJx%8<`%6z* zcUY**BN z*P;Q`l?4I-M6(dP2xHIi;T7VY#s~r2@5q;N_3Azqj z)q_Y@HHUt58{(8`-C`8rW^;aM8Cj#Wnm>LEwKlxHfXUz#{Yh zb6`_(o^i4`K+Xul8kerGFLi9i4!IL>9T#d#!`Kq?o@uf=TwWe1{43RuT+9DeF1;c$ z6TM?wR8a2+Px4IhYyo7&<*`2S7XUDrg+3>*q;{!q*Osr=zhv2uVRJE6d6Z-NmBl6X z0h4Aw(G)1bjU8eo@mcQZpl{KhyzFaE%$N(CsP%9q$bk7bO-Y=QsR#uV5*0R=Y?b@y zEeID+uQT3Om{<&m!P3aP_Yuw~+j{Mu9fwq>8oW6S>0KQtamBJ7Q4xO|%zqR*gT<6R za^EC)V2qrw=0(eHoe@FZ5bK^p?m2ohXC(;5ljqOy->tla5oj*6FmaypzUwJiL;wdl zmi_}kyHs-OHnf`1zIR3u8+6I2%HH(#FDgMr-MX;$o$?RF1jXmtY|8dl_r>8pNU3Y> zKbJb;9@UV<%R8R1tv)IXja_TKx|d3kR$~SeB<}*`z0}erX@{cldpUzswr&z~w!EO~ zULloWmuu*d7>$8wTa^xV_L0Rnxs=pS(iDOX!s-+a1Qu?U4)oE4YoTN^P^*Zk`tr5b zz<-T}LDd@xX@41%vtS*yM7C);cmRhT_P{+1=e!d8N)`5;v|+P3mi-Sv*29GeaYRPs zelikmsg*FFoQToQR!OE6)zSx=A#4EXoa;1nb3-r~{t&yt6SozW_~rY&*vUynjMu)3 zeXAY^pi>BfPoB#Q-)3HEif)~LA4PddcCR(Suo7nuy{0gG4w>gmKAVz#HIbnG@K;Tb zaKnt?W895*8GDsV8oVCM)fk8rM~9UEOlz+R2ym)(c_p*k(v_eLAH-Q@3b5;AmCY-1 z489{Bn+@XPA^0vgT@`-!6>Tuom48A8=b**BMA26bEADGa4-;&2PAfR@9^vmmcNMIT#=K;`DOKVN2JYSl=Gsb=Nfw?9M9kLAs}nO3YP#m}Y-PpQ54*Ll|7-F^PG-ZcouyxiU%%8pGAfk5r6IcuhY; zyoL>ywN%liPyX~g4PDW!GiS^ex8P&m7bG5lJ(HM>!1)hA9Ql!v*kLNP1@Hbz%`b-9 z3-v%>EyWzU2b-PW*bd~$ecFSZGp&?{`yz4}?9$e=r*Qz3lh35{M?vt8IUV8&O;=u4g!jf;Z6acu5b2+IGCOP$Jh3f-qpGpmB z2l50BuFm7QOKiV*LC8VnEW;ovozE!vw-NZsH}X_6EG9LoXd7XZWX*@-PwsgWRu^1n z@p7|d%q}W6)tq?c= zwpR$fiZvdYpxAy6cGW;RSX&$a8Y~CsMO5zOjK|HB`&%2-*aD}I{@K7m7&;nIbT4DV zv^G+>E4>Pz%UGw;R3OMU03b!(zn{9@9*o+=HVJ@8zK7{vK`+v9t>3(@A5&MS2hsx` z&a*IGi}b1-UTCievN%+R&Yv$Ge|o8%g94vD^AsuX&EpT)Kka1_ir5Iug8GcC!u#5y zPJ|E!tNGXO+feOauwB`>ciunEthY@aV6{EAYAsL%WNfDhyP^|g{nWbnjs?TJ`0zAx z_JI&(6;K9vX{*LODMR~ENe=EqE0j70PoCZY3_l9YzzBr-&mqyh_;>3;%f%& z?I%fJJs(J0*Wu zag@G~GfeEqD!i`D5VltiDZ_VgP&D^DH1%-D{4q8<*C&AiUW;od?TlxWX8J5NtZu9f z%Ze39Os!)c0Yop++@xbn4n-K~QzN>T>L=9>}EhOTP+5JokM@qZ!P5k(_d@N>@G0 zpk*BY+5?(39Dy=aiR0eU68sT!PKao=M(`?{XNFPKl-4vnZQ?d6ZrVfxu{>5g(d@bh z@AgqaU%<$4=#1q_{qq_ZNRHeHzL6?N9!E=lqW>oA!{p+IGRip*6U;K0+QlC4qbO5k zxq(kTD$G3CC<)_5>T41+*`#x@_C7x3z;Cl5J&5EDBLhzZG-@-NHJf{AbOLsNX!eIU zpS)w*xmb*eO$IceAW-PF&cfXD`_i_;Gs@K|cc%`pMQ+cbDWR9d%@fMO!OCn(9sb%T zz5Ak`r{%$g+fk`nN&W|3Ac)8b=O6OlHUCt>&JN^Dczf$Stk=^_*u?O^ks??CdVWlw zOGbPzC9*v;FHcgV-c?F7@v&asbK}1T)+GM{IL&6SLp8p{zk#qzv_i04VxWRXS%5)d zrhZU_;7Lib7YPX!Cq3`Q{8rT9j0QmEbc(*!iLKJe=hr#!w-Fw2|VuX_(^+#0L{Gju-+aD*i6Q9W8x%kiDM|*4kvn`KS z`C=2l!DU9*-apEhB@%%j7GMRKZ+^HujaT|NRDv!ITX!2*|1$lM5A6Z*b zKCsrel_}TmNOI>ouPYgt@f04zVXx7nOBN-@5r{{hJNv7>;% uGnxMYjZusI*su8Dhy5VzzA%!#+l%&y)F<`Z;atWwXe#o5qT{0eTmFCU9RAw? literal 0 HcmV?d00001 From af5615123168363ea08c70a80b9b2e9aa006ce27 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 8 Oct 2021 19:47:07 +0000 Subject: [PATCH 281/376] highgui(docs): we don't support 32-bit integer images in imshow() --- modules/highgui/include/opencv2/highgui.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/highgui/include/opencv2/highgui.hpp b/modules/highgui/include/opencv2/highgui.hpp index cdab0ada7888..7ae239573759 100644 --- a/modules/highgui/include/opencv2/highgui.hpp +++ b/modules/highgui/include/opencv2/highgui.hpp @@ -383,10 +383,12 @@ cv::WINDOW_AUTOSIZE flag, the image is shown with its original size, however it Otherwise, the image is scaled to fit the window. The function may scale the image, depending on its depth: - If the image is 8-bit unsigned, it is displayed as is. -- If the image is 16-bit unsigned or 32-bit integer, the pixels are divided by 256. That is, the +- If the image is 16-bit unsigned, the pixels are divided by 256. That is, the value range [0,255\*256] is mapped to [0,255]. - If the image is 32-bit or 64-bit floating-point, the pixel values are multiplied by 255. That is, the value range [0,1] is mapped to [0,255]. +- 32-bit integer images are not processed anymore due to ambiguouty of required transform. + Convert to 8-bit unsigned matrix using a custom preprocessing specific to image's context. If window was created with OpenGL support, cv::imshow also support ogl::Buffer , ogl::Texture2D and cuda::GpuMat as input. From 788f330d07fe3dd229b3f2648a8090b61e555c70 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 8 Oct 2021 20:09:43 +0000 Subject: [PATCH 282/376] js: fix build of SIMD tests --- modules/ts/src/precomp.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/ts/src/precomp.hpp b/modules/ts/src/precomp.hpp index a9d4404e3b47..2725f6342904 100644 --- a/modules/ts/src/precomp.hpp +++ b/modules/ts/src/precomp.hpp @@ -1,7 +1,9 @@ #include "opencv2/ts.hpp" #include #include "opencv2/core/utility.hpp" +#if !defined(__EMSCRIPTEN__) #include "opencv2/core/private.hpp" +#endif #ifdef GTEST_LINKED_AS_SHARED_LIBRARY #error ts module should not have GTEST_LINKED_AS_SHARED_LIBRARY defined From 39ee5c5a461760b528e157960d36c567eba236f6 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 8 Oct 2021 20:34:07 +0000 Subject: [PATCH 283/376] plugins: reverse candidates order to fetch higher versions first --- modules/core/src/parallel/plugin_parallel_wrapper.impl.hpp | 3 +++ modules/highgui/src/plugin_wrapper.impl.hpp | 3 +++ modules/videoio/src/backend_plugin.cpp | 3 +++ 3 files changed, 9 insertions(+) diff --git a/modules/core/src/parallel/plugin_parallel_wrapper.impl.hpp b/modules/core/src/parallel/plugin_parallel_wrapper.impl.hpp index a5649b60c083..b9b3105d20e0 100644 --- a/modules/core/src/parallel/plugin_parallel_wrapper.impl.hpp +++ b/modules/core/src/parallel/plugin_parallel_wrapper.impl.hpp @@ -223,6 +223,9 @@ std::vector getPluginCandidates(const std::string& baseName) continue; std::vector candidates; cv::glob(utils::fs::join(path, plugin_expr), candidates); + // Prefer candisates with higher versions + // TODO: implemented accurate versions-based comparator + std::sort(candidates.begin(), candidates.end(), std::greater()); CV_LOG_DEBUG(NULL, " - " << path << ": " << candidates.size()); copy(candidates.begin(), candidates.end(), back_inserter(results)); } diff --git a/modules/highgui/src/plugin_wrapper.impl.hpp b/modules/highgui/src/plugin_wrapper.impl.hpp index 97aea69098cb..23f0ecf6bf84 100644 --- a/modules/highgui/src/plugin_wrapper.impl.hpp +++ b/modules/highgui/src/plugin_wrapper.impl.hpp @@ -224,6 +224,9 @@ std::vector getPluginCandidates(const std::string& baseName) continue; std::vector candidates; cv::glob(utils::fs::join(path, plugin_expr), candidates); + // Prefer candisates with higher versions + // TODO: implemented accurate versions-based comparator + std::sort(candidates.begin(), candidates.end(), std::greater()); CV_LOG_DEBUG(NULL, " - " << path << ": " << candidates.size()); copy(candidates.begin(), candidates.end(), back_inserter(results)); } diff --git a/modules/videoio/src/backend_plugin.cpp b/modules/videoio/src/backend_plugin.cpp index e42c98649cfc..039083c6ad47 100644 --- a/modules/videoio/src/backend_plugin.cpp +++ b/modules/videoio/src/backend_plugin.cpp @@ -373,6 +373,9 @@ std::vector getPluginCandidates(const std::string& baseName) continue; std::vector candidates; cv::glob(utils::fs::join(path, plugin_expr), candidates); + // Prefer candisates with higher versions + // TODO: implemented accurate versions-based comparator + std::sort(candidates.begin(), candidates.end(), std::greater()); CV_LOG_INFO(NULL, " - " << path << ": " << candidates.size()); copy(candidates.begin(), candidates.end(), back_inserter(results)); } From ac57be91e12cd4021c813a1e3978746f8aa47cb8 Mon Sep 17 00:00:00 2001 From: Yaniv Hollander Date: Sat, 9 Oct 2021 11:43:50 -0400 Subject: [PATCH 284/376] Merge pull request #20721 from YanivHollander:DocFixes Doc fixes * Update imgproc.hpp * Update imgproc.hpp --- modules/imgproc/include/opencv2/imgproc.hpp | 70 ++++++++++----------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index 24686228fd1a..9e5375ef4b21 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -2745,13 +2745,6 @@ CV_EXPORTS_W void warpPolar(InputArray src, OutputArray dst, Size dsize, //! @addtogroup imgproc_misc //! @{ -/** @overload */ -CV_EXPORTS_W void integral( InputArray src, OutputArray sum, int sdepth = -1 ); - -/** @overload */ -CV_EXPORTS_AS(integral2) void integral( InputArray src, OutputArray sum, - OutputArray sqsum, int sdepth = -1, int sqdepth = -1 ); - /** @brief Calculates the integral of an image. The function calculates one or more integral images for the source image as follows: @@ -2790,6 +2783,13 @@ CV_EXPORTS_AS(integral3) void integral( InputArray src, OutputArray sum, OutputArray sqsum, OutputArray tilted, int sdepth = -1, int sqdepth = -1 ); +/** @overload */ +CV_EXPORTS_W void integral( InputArray src, OutputArray sum, int sdepth = -1 ); + +/** @overload */ +CV_EXPORTS_AS(integral2) void integral( InputArray src, OutputArray sum, + OutputArray sqsum, int sdepth = -1, int sqdepth = -1 ); + //! @} imgproc_misc //! @addtogroup imgproc_motion @@ -3472,19 +3472,6 @@ the first variant of the function and distanceType == #DIST_L1. CV_EXPORTS_W void distanceTransform( InputArray src, OutputArray dst, int distanceType, int maskSize, int dstType=CV_32F); -/** @example samples/cpp/ffilldemo.cpp -An example using the FloodFill technique -*/ - -/** @overload - -variant without `mask` parameter -*/ -CV_EXPORTS int floodFill( InputOutputArray image, - Point seedPoint, Scalar newVal, CV_OUT Rect* rect = 0, - Scalar loDiff = Scalar(), Scalar upDiff = Scalar(), - int flags = 4 ); - /** @brief Fills a connected component with the given color. The function cv::floodFill fills a connected component starting from the seed point with the specified @@ -3561,6 +3548,19 @@ CV_EXPORTS_W int floodFill( InputOutputArray image, InputOutputArray mask, Scalar loDiff = Scalar(), Scalar upDiff = Scalar(), int flags = 4 ); +/** @example samples/cpp/ffilldemo.cpp +An example using the FloodFill technique +*/ + +/** @overload + +variant without `mask` parameter +*/ +CV_EXPORTS int floodFill( InputOutputArray image, + Point seedPoint, Scalar newVal, CV_OUT Rect* rect = 0, + Scalar loDiff = Scalar(), Scalar upDiff = Scalar(), + int flags = 4 ); + //! Performs linear blending of two images: //! \f[ \texttt{dst}(i,j) = \texttt{weights1}(i,j)*\texttt{src1}(i,j) + \texttt{weights2}(i,j)*\texttt{src2}(i,j) \f] //! @param src1 It has a type of CV_8UC(n) or CV_32FC(n), where n is a positive integer. @@ -4399,7 +4399,7 @@ lines are drawn using Gaussian filtering. CV_EXPORTS_W void line(InputOutputArray img, Point pt1, Point pt2, const Scalar& color, int thickness = 1, int lineType = LINE_8, int shift = 0); -/** @brief Draws a arrow segment pointing from the first point to the second one. +/** @brief Draws an arrow segment pointing from the first point to the second one. The function cv::arrowedLine draws an arrow between pt1 and pt2 points in the image. See also #line. @@ -4529,11 +4529,6 @@ CV_EXPORTS_W void drawMarker(InputOutputArray img, Point position, const Scalar& /* END OF MARKER SECTION */ /* ----------------------------------------------------------------------------------------- */ -/** @overload */ -CV_EXPORTS void fillConvexPoly(InputOutputArray img, const Point* pts, int npts, - const Scalar& color, int lineType = LINE_8, - int shift = 0); - /** @brief Fills a convex polygon. The function cv::fillConvexPoly draws a filled convex polygon. This function is much faster than the @@ -4552,10 +4547,9 @@ CV_EXPORTS_W void fillConvexPoly(InputOutputArray img, InputArray points, int shift = 0); /** @overload */ -CV_EXPORTS void fillPoly(InputOutputArray img, const Point** pts, - const int* npts, int ncontours, - const Scalar& color, int lineType = LINE_8, int shift = 0, - Point offset = Point() ); +CV_EXPORTS void fillConvexPoly(InputOutputArray img, const Point* pts, int npts, + const Scalar& color, int lineType = LINE_8, + int shift = 0); /** @example samples/cpp/tutorial_code/ImgProc/basic_drawing/Drawing_1.cpp An example using drawing functions @@ -4580,9 +4574,10 @@ CV_EXPORTS_W void fillPoly(InputOutputArray img, InputArrayOfArrays pts, Point offset = Point() ); /** @overload */ -CV_EXPORTS void polylines(InputOutputArray img, const Point* const* pts, const int* npts, - int ncontours, bool isClosed, const Scalar& color, - int thickness = 1, int lineType = LINE_8, int shift = 0 ); +CV_EXPORTS void fillPoly(InputOutputArray img, const Point** pts, + const int* npts, int ncontours, + const Scalar& color, int lineType = LINE_8, int shift = 0, + Point offset = Point() ); /** @brief Draws several polygonal curves. @@ -4601,6 +4596,11 @@ CV_EXPORTS_W void polylines(InputOutputArray img, InputArrayOfArrays pts, bool isClosed, const Scalar& color, int thickness = 1, int lineType = LINE_8, int shift = 0 ); +/** @overload */ +CV_EXPORTS void polylines(InputOutputArray img, const Point* const* pts, const int* npts, + int ncontours, bool isClosed, const Scalar& color, + int thickness = 1, int lineType = LINE_8, int shift = 0 ); + /** @example samples/cpp/contours2.cpp An example program illustrates the use of cv::findContours and cv::drawContours \image html WindowsQtContoursOutput.png "Screenshot of the program" @@ -4633,7 +4633,7 @@ parameter is only taken into account when there is hierarchy available. @param offset Optional contour shift parameter. Shift all the drawn contours by the specified \f$\texttt{offset}=(dx,dy)\f$ . @note When thickness=#FILLED, the function is designed to handle connected components with holes correctly -even when no hierarchy date is provided. This is done by analyzing all the outlines together +even when no hierarchy data is provided. This is done by analyzing all the outlines together using even-odd rule. This may give incorrect results if you have a joint collection of separately retrieved contours. In order to solve this problem, you need to call #drawContours separately for each sub-group of contours, or iterate over the collection using contourIdx parameter. @@ -4647,7 +4647,7 @@ CV_EXPORTS_W void drawContours( InputOutputArray image, InputArrayOfArrays conto /** @brief Clips the line against the image rectangle. The function cv::clipLine calculates a part of the line segment that is entirely within the specified -rectangle. it returns false if the line segment is completely outside the rectangle. Otherwise, +rectangle. It returns false if the line segment is completely outside the rectangle. Otherwise, it returns true . @param imgSize Image size. The image rectangle is Rect(0, 0, imgSize.width, imgSize.height) . @param pt1 First line point. From 4223495e6cd67011f86b8ecd9be1fa105018f3b1 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 9 Oct 2021 15:48:26 +0000 Subject: [PATCH 285/376] release: OpenCV 4.5.4 --- modules/core/include/opencv2/core/version.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index 4aaf30ce5ed4..2c9d0952e5d2 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -8,7 +8,7 @@ #define CV_VERSION_MAJOR 4 #define CV_VERSION_MINOR 5 #define CV_VERSION_REVISION 4 -#define CV_VERSION_STATUS "-pre" +#define CV_VERSION_STATUS "" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) From a9d7b6eab7bf030b9e90e8467df00f8551915c84 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Mon, 11 Oct 2021 18:58:10 +0300 Subject: [PATCH 286/376] fix const - input and remove unimplemented function --- modules/dnn/src/onnx/onnx_importer.cpp | 67 +++++++++++++++++-------- modules/dnn/test/test_onnx_importer.cpp | 7 +++ 2 files changed, 52 insertions(+), 22 deletions(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 5343c0536178..f497a17a7a07 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -53,7 +53,6 @@ class ONNXImporter Mat getBlob(const std::string& input_name); LayerParams getLayerParams(const opencv_onnx::NodeProto& node_proto); - bool isCeilMode(const LayerParams& layerParams); void addConstant(const std::string& name, const Mat& blob); void addLayer(LayerParams& layerParams, @@ -61,6 +60,7 @@ class ONNXImporter void expandMid(const std::string& prefix, opencv_onnx::NodeProto& node_proto, const std::string& input, size_t n); + void addNegation(const LayerParams& layerParams, opencv_onnx::NodeProto& node_proto, int input_id); public: ONNXImporter(Net& net, const char *onnxFile) @@ -460,6 +460,32 @@ void ONNXImporter::expandMid(const std::string& prefix, opencv_onnx::NodeProto& } } +/** @brief Multiply one of node_proto inputs by -1 + * @param layerParams parameters of the node + * @param node_proto node which input will be replaced + * @param input_id id of input to be multiplied by -1 + */ +void ONNXImporter::addNegation(const LayerParams& layerParams, opencv_onnx::NodeProto& node_proto, int input_id) +{ + LayerParams powerParams; + powerParams.name = layerParams.name + "/neg"; + powerParams.type = "Power"; + powerParams.set("scale", -1.f); + + //Create Power layer + int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + //Connect to input + IterLayerId_t layerId = layer_id.find(node_proto.input(input_id)); + CV_Assert(layerId != layer_id.end()); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); + outShapes[powerParams.name] = outShapes[node_proto.input(input_id)]; + + //Replace input to Power + node_proto.set_input(input_id, powerParams.name); +} + void ONNXImporter::addConstant(const std::string& name, const Mat& blob) { constBlobs.insert(std::make_pair(name, blob)); @@ -918,29 +944,42 @@ void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodePr else if (is_const_0 || is_const_1) { int const_blob_id = is_const_0 ? 0 : 1; + int input_id = 1 - const_blob_id; Mat blob = getBlob(node_proto, const_blob_id); int blob_total = blob.total(); + + const float inputScale = isSub && is_const_0 ? -1.f : 1.f; + const float constScale = isSub && is_const_1 ? -1.f : 1.f; + if (blob_total == 1) { layerParams.type = "Power"; - layerParams.set("shift", (isSub ? -1 : 1) * blob.ptr()[0]); + layerParams.set("scale", inputScale); + layerParams.set("shift", constScale * blob.ptr()[0]); } else { - MatShape inpShape = outShapes[node_proto.input(1 - const_blob_id)]; + MatShape inpShape = outShapes[node_proto.input(input_id)]; if (shape(blob) == inpShape) { LayerParams constParams; constParams.name = layerParams.name + "/const"; constParams.type = "Const"; - constParams.blobs.push_back((isSub ? -1 : 1) * blob); + constParams.blobs.push_back(blob); int id = dstNet.addLayer(constParams.name, constParams.type, constParams); layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); outShapes[constParams.name] = shape(blob); layerParams.type = "Eltwise"; + float coeffs[] = {1., isSub ? -1.f : 1.f}; + layerParams.set("coeff", DictValue::arrayReal(coeffs, 2)); node_proto.set_input(const_blob_id, constParams.name); } else { + if (inputScale < 0.f) + { + addNegation(layerParams, node_proto, input_id); + } + layerParams.type = "Scale"; layerParams.set("bias_term", true); int axis = 1; @@ -955,7 +994,7 @@ void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodePr } layerParams.set("axis", axis); blob = blob.reshape(1, 1); - layerParams.blobs.push_back((isSub ? -1 : 1) * blob); + layerParams.blobs.push_back(constScale * blob); } } } @@ -972,23 +1011,7 @@ void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodePr { if (isSub) { - LayerParams powerParams; - powerParams.name = layerParams.name + "/neg"; - powerParams.type = "Power"; - powerParams.set("scale", -1); - - //Create Power layer - int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(1)); - CV_Assert(layerId != layer_id.end()); - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); - outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - - //Replace input to Power - node_proto.set_input(1, powerParams.name); + addNegation(layerParams, node_proto, 1); } layerParams.type = "Scale"; layerParams.set("bias_term", true); diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index b4ecd4601c4f..daba29e5471c 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -931,6 +931,13 @@ TEST_P(Test_ONNX_layers, ConvResizePool1d) testONNXModels("conv_resize_pool_1d"); } +TEST_P(Test_ONNX_layers, SubFromConst) +{ + testONNXModels("sub_from_const1"); + testONNXModels("sub_from_const_eltwise"); + testONNXModels("sub_from_const_broadcast"); +} + INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_ONNX_layers, dnnBackendsAndTargets()); class Test_ONNX_nets : public Test_ONNX_layers From 238dbffb48e80901a92938e8d80d0bff7de38dcc Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Mon, 11 Oct 2021 20:59:44 +0300 Subject: [PATCH 287/376] change asserts for Sum --- modules/dnn/src/onnx/onnx_importer.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 5343c0536178..a4484570b123 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -903,7 +903,19 @@ void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodePr opencv_onnx::NodeProto node_proto = node_proto_; const std::string& layer_type = node_proto.op_type(); bool isSub = layer_type == "Sub"; - CV_CheckEQ(node_proto.input_size(), 2, ""); + CV_Assert((node_proto.input_size() == 2) || (layer_type == "Sum" && node_proto.input_size() > 2)); + + if (layer_type == "Sum" && node_proto.input_size() > 2) + { + for (int i = 0; i < node_proto.input_size(); ++i) + { + if (layer_id.find(node_proto.input(i)) == layer_id.end()) + { + CV_Error(Error::StsNotImplemented, "Sum of constants is not implemented for inputs > 2"); + } + } + } + bool is_const_0 = layer_id.find(node_proto.input(0)) == layer_id.end(); bool is_const_1 = layer_id.find(node_proto.input(1)) == layer_id.end(); if (is_const_0 && is_const_1) From 9e845eab7fb54a3cb814411fb8390c8c305c71ce Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Tue, 12 Oct 2021 13:30:59 +0800 Subject: [PATCH 288/376] Fixed comments --- build_js/doc/doxygen/bib19450.aux | 134 ------------------------------ cmake/OpenCVDetectWebNN.cmake | 2 +- 2 files changed, 1 insertion(+), 135 deletions(-) delete mode 100644 build_js/doc/doxygen/bib19450.aux diff --git a/build_js/doc/doxygen/bib19450.aux b/build_js/doc/doxygen/bib19450.aux deleted file mode 100644 index 2e533495bdcb..000000000000 --- a/build_js/doc/doxygen/bib19450.aux +++ /dev/null @@ -1,134 +0,0 @@ -\relax -\bibstyle{doxygen} -\citation{AAM} -\citation{ABD12} -\citation{AMVOT} -\citation{ANB13} -\citation{Andreff99} -\citation{Arandjelovic:2012:TTE:2354409.2355123} -\citation{BA83} -\citation{BL07} -\citation{BT98} -\citation{Ballard1981} -\citation{Bolelli2017} -\citation{Bolelli2019} -\citation{Borgefors86} -\citation{Bouguet00} -\citation{BouguetMCT} -\citation{Bradski98} -\citation{Breiman84} -\citation{Brox2004} -\citation{CL12} -\citation{Canny86} -\citation{ChambolleEtAl} -\citation{Chaumette06} -\citation{Collins14} -\citation{DM03} -\citation{DM97} -\citation{Dalal2005} -\citation{Daniilidis98} -\citation{EM11} -\citation{EP08} -\citation{Eade13} -\citation{Eade17} -\citation{FHT98} -\citation{FL02} -\citation{Farneback2003} -\citation{Felzenszwalb04} -\citation{Fitzgibbon1999} -\citation{Fitzgibbon95} -\citation{GOTURN} -\citation{GW03} -\citation{Gallego2014ACF} -\citation{Grana2010} -\citation{Guil1999} -\citation{HH08} -\citation{HTF01} -\citation{Hartley99} -\citation{HartleyZ00} -\citation{Horaud95} -\citation{Hu62} -\citation{Ke17} -\citation{Kirkpatrick83} -\citation{KleeLaskowski85} -\citation{Kroeger2016} -\citation{LCS11} -\citation{Li2010SimultaneousRA} -\citation{Liao2007} -\citation{LibSVM} -\citation{Lienhart02} -\citation{Louhichi07} -\citation{Lowe04} -\citation{MA13} -\citation{MIL} -\citation{MK07} -\citation{MM06} -\citation{Ma:2003:IVI} -\citation{Madsen04} -\citation{Malis} -\citation{Marchand16} -\citation{Matas00} -\citation{Meyer92} -\citation{Mortensen95intelligentscissors} -\citation{Muja2009} -\citation{Nister03} -\citation{ORourke86} -\citation{PM03} -\citation{Park94} -\citation{Puzicha1997} -\citation{RB99} -\citation{RD05} -\citation{RPROP93} -\citation{RRKB11} -\citation{RS10} -\citation{Rafael12} -\citation{Rosten06} -\citation{Rubner2000} -\citation{RubnerSept98} -\citation{Shah2013SolvingTR} -\citation{Shi94} -\citation{Sklansky82} -\citation{Slabaugh} -\citation{Sol2018AML} -\citation{SteweniusCFS} -\citation{Suzuki85} -\citation{Taubin1991} -\citation{TehChin89} -\citation{Telea04} -\citation{Terzakis20} -\citation{Tsai89} -\citation{UES01} -\citation{V03} -\citation{VandLec} -\citation{Viola01} -\citation{Viola04} -\citation{WJ10} -\citation{Welch95} -\citation{Wu2009} -\citation{YM11} -\citation{Yuen90} -\citation{Zhang2000} -\citation{Zivkovic2004} -\citation{Zivkovic2006} -\citation{bigun2006vision} -\citation{blanco2010tutorial} -\citation{bottou2010large} -\citation{duda2018} -\citation{forssen2007maximally} -\citation{forstner1987fast} -\citation{gao2003complete} -\citation{gonzalez} -\citation{gruzman} -\citation{hesch2011direct} -\citation{jahne2000computer} -\citation{lepetit2009epnp} -\citation{liao2020real} -\citation{mair2010_agast} -\citation{nister2008linear} -\citation{penate2013exhaustive} -\citation{strobl2011iccv} -\citation{umeyama1991least} -\citation{vacavant2013benchmark} -\citation{van1995estimators} -\citation{yang1996structure} -\bibdata{bibTmpDir/bibTmpFile_1,bibTmpDir/bibTmpFile_2,bibTmpDir/bibTmpFile_3} diff --git a/cmake/OpenCVDetectWebNN.cmake b/cmake/OpenCVDetectWebNN.cmake index e2dd8f258c95..809a30772ff8 100644 --- a/cmake/OpenCVDetectWebNN.cmake +++ b/cmake/OpenCVDetectWebNN.cmake @@ -42,7 +42,7 @@ endif() if(NOT EMSCRIPTEN) message(AUTHOR_WARNING "Use WebNN-native") else() -message(AUTHOR_WARNING "Use WebNN") + message(AUTHOR_WARNING "Use WebNN") endif() set(HAVE_WEBNN 1) From e48b1b3d4a11c849379c5f4599df5f1dec1726fa Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Tue, 12 Oct 2021 13:37:01 +0800 Subject: [PATCH 289/376] Address the comments --- cmake/OpenCVDetectWebNN.cmake | 4 +++- cmake/templates/cvconfig.h.in | 3 --- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmake/OpenCVDetectWebNN.cmake b/cmake/OpenCVDetectWebNN.cmake index 809a30772ff8..7a099f0e84e0 100644 --- a/cmake/OpenCVDetectWebNN.cmake +++ b/cmake/OpenCVDetectWebNN.cmake @@ -45,4 +45,6 @@ else() message(AUTHOR_WARNING "Use WebNN") endif() -set(HAVE_WEBNN 1) +if(VALID_WEBNN) + set(HAVE_WEBNN ON) +endif() diff --git a/cmake/templates/cvconfig.h.in b/cmake/templates/cvconfig.h.in index 39708e14bdbf..6439d8b43f06 100644 --- a/cmake/templates/cvconfig.h.in +++ b/cmake/templates/cvconfig.h.in @@ -59,9 +59,6 @@ /* Vulkan support */ #cmakedefine HAVE_VULKAN -/* Webnn support */ -#cmakedefine HAVE_WEBNN - /* Define to 1 if you have the header file. */ #cmakedefine HAVE_INTTYPES_H 1 From 4f3a426401bf39f7ee6d5e10d723076471db9e0e Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Tue, 12 Oct 2021 13:54:44 +0800 Subject: [PATCH 290/376] Update CMakeLists.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e8cd8105cf51..17ad595838d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -290,7 +290,7 @@ OCV_OPTION(WITH_INF_ENGINE "Include Intel Inference Engine support" OFF OCV_OPTION(WITH_NGRAPH "Include nGraph support" WITH_INF_ENGINE VISIBLE_IF TRUE VERIFY TARGET ngraph::ngraph) -OCV_OPTION(WITH_WEBNN "Include WebNN support" OFF +OCV_OPTION(WITH_WEBNN "Include WebNN support" WITH_WEBNN VISIBLE_IF TRUE VERIFY HAVE_WEBNN) OCV_OPTION(WITH_JASPER "Include JPEG2K support (Jasper)" ON From 3d93675ff9463fd8247b482c07e19ad8a59c61ff Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Mon, 4 Oct 2021 14:42:51 +0200 Subject: [PATCH 291/376] fix link error on shared libs with -MT --- modules/highgui/CMakeLists.txt | 2 +- modules/imgcodecs/CMakeLists.txt | 2 +- samples/CMakeLists.txt | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index 6e6996109936..e48e2470ceab 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -163,7 +163,7 @@ if(APPLE) add_apple_compiler_options(${the_module}) endif() -if(MSVC) +if(MSVC AND NOT BUILD_SHARED_LIBS AND BUILD_WITH_STATIC_CRT) set_target_properties(${the_module} PROPERTIES LINK_FLAGS "/NODEFAULTLIB:atlthunk.lib /NODEFAULTLIB:atlsd.lib /NODEFAULTLIB:libcmt.lib /DEBUG") endif() diff --git a/modules/imgcodecs/CMakeLists.txt b/modules/imgcodecs/CMakeLists.txt index 63edd45e789a..10fb3278fbe9 100644 --- a/modules/imgcodecs/CMakeLists.txt +++ b/modules/imgcodecs/CMakeLists.txt @@ -134,7 +134,7 @@ if(APPLE) add_apple_compiler_options(${the_module}) endif() -if(MSVC) +if(MSVC AND NOT BUILD_SHARED_LIBS AND BUILD_WITH_STATIC_CRT) set_target_properties(${the_module} PROPERTIES LINK_FLAGS "/NODEFAULTLIB:atlthunk.lib /NODEFAULTLIB:atlsd.lib /NODEFAULTLIB:libcmt.lib /DEBUG") endif() diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 4cc4c07d7b8a..87f5cfcf88ad 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -97,8 +97,10 @@ if(MSVC) endforeach(flag_var) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /NODEFAULTLIB:atlthunk.lib /NODEFAULTLIB:msvcrt.lib /NODEFAULTLIB:msvcrtd.lib") - set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} /NODEFAULTLIB:libcmt.lib") - set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /NODEFAULTLIB:libcmtd.lib") + if(NOT BUILD_WITH_STATIC_CRT) + set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} /NODEFAULTLIB:libcmt.lib") + set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /NODEFAULTLIB:libcmtd.lib") + endif() endif() endif() From 982503e9a8724465570ccab65e8e5c9656940b5f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 13 Oct 2021 01:14:37 +0000 Subject: [PATCH 292/376] core: ensure 'int' result from CV_POPCNT_U64(x) --- modules/core/include/opencv2/core/cv_cpu_dispatch.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/cv_cpu_dispatch.h b/modules/core/include/opencv2/core/cv_cpu_dispatch.h index 540fbb605cb0..e92798d6c952 100644 --- a/modules/core/include/opencv2/core/cv_cpu_dispatch.h +++ b/modules/core/include/opencv2/core/cv_cpu_dispatch.h @@ -55,7 +55,7 @@ # ifdef _MSC_VER # include # if defined(_M_X64) -# define CV_POPCNT_U64 _mm_popcnt_u64 +# define CV_POPCNT_U64 (int)_mm_popcnt_u64 # endif # define CV_POPCNT_U32 _mm_popcnt_u32 # else From e4a87f2f4f9276ed17551236eb4e08285e10824a Mon Sep 17 00:00:00 2001 From: icestraw Date: Wed, 13 Oct 2021 22:35:29 +0800 Subject: [PATCH 293/376] fix type error of slice indices --- doc/py_tutorials/py_imgproc/py_pyramids/py_pyramids.markdown | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/py_tutorials/py_imgproc/py_pyramids/py_pyramids.markdown b/doc/py_tutorials/py_imgproc/py_pyramids/py_pyramids.markdown index 63fde0a130bf..602ab29ad670 100644 --- a/doc/py_tutorials/py_imgproc/py_pyramids/py_pyramids.markdown +++ b/doc/py_tutorials/py_imgproc/py_pyramids/py_pyramids.markdown @@ -117,7 +117,7 @@ for i in range(5,0,-1): LS = [] for la,lb in zip(lpA,lpB): rows,cols,dpt = la.shape - ls = np.hstack((la[:,0:cols/2], lb[:,cols/2:])) + ls = np.hstack((la[:,0:cols//2], lb[:,cols//2:])) LS.append(ls) # now reconstruct @@ -127,7 +127,7 @@ for i in range(1,6): ls_ = cv.add(ls_, LS[i]) # image with direct connecting each half -real = np.hstack((A[:,:cols/2],B[:,cols/2:])) +real = np.hstack((A[:,:cols//2],B[:,cols//2:])) cv.imwrite('Pyramid_blending2.jpg',ls_) cv.imwrite('Direct_blending.jpg',real) From bd0732b1d03bcbc15d919d6dbe3dc45eac1b9514 Mon Sep 17 00:00:00 2001 From: Nicholas Ho <88894303+Nicholas-Ho-arm@users.noreply.github.com> Date: Fri, 15 Oct 2021 12:47:53 +0100 Subject: [PATCH 294/376] Merge pull request #20740 from Nicholas-Ho-arm:3.4_SymmColumnVec_32f8u * Add SymmColumnVec_32f8u * Fix double to float warnings --- modules/imgproc/src/filter.simd.hpp | 81 ++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/modules/imgproc/src/filter.simd.hpp b/modules/imgproc/src/filter.simd.hpp index 94dbce060c45..966de94c0b0d 100644 --- a/modules/imgproc/src/filter.simd.hpp +++ b/modules/imgproc/src/filter.simd.hpp @@ -1160,6 +1160,81 @@ struct SymmColumnVec_32s8u Mat kernel; }; +struct SymmColumnVec_32f8u +{ + SymmColumnVec_32f8u() { symmetryType = 0; delta = 0; } + SymmColumnVec_32f8u(const Mat& _kernel, int _symmetryType, int, double _delta) + { + symmetryType = _symmetryType; + kernel = _kernel; + delta = (float)_delta; + CV_Assert( (symmetryType & (KERNEL_SYMMETRICAL | KERNEL_ASYMMETRICAL)) != 0 ); + } + + int operator()(const uchar** _src, uchar* _dst, int width) const + { + CV_INSTRUMENT_REGION(); + + int _ksize = kernel.rows + kernel.cols - 1; + if( _ksize == 1 ) return 0; + const int ksize2 = _ksize / 2; + const float* ky = kernel.ptr() + ksize2; + int i = 0, k; + bool symmetrical = (symmetryType & KERNEL_SYMMETRICAL) != 0; + const float** src = (const float**)_src; + + if( symmetrical ) + { + for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + { + v_float32 v_ky0 = vx_setall_f32(ky[0]); + v_float32 v32_delta = vx_setall_f32(delta); + const float* S = src[0] + i; + v_float32 s0 = v_muladd(v_ky0, vx_load(S), v32_delta); + v_float32 s1 = v_muladd(v_ky0, vx_load(S + v_float32::nlanes), v32_delta); + v_float32 s2 = v_muladd(v_ky0, vx_load(S + 2*v_float32::nlanes), v32_delta); + v_float32 s3 = v_muladd(v_ky0, vx_load(S + 3*v_float32::nlanes), v32_delta); + for( k = 1; k <= ksize2; k++ ) + { + v_float32 v_kyk = vx_setall_f32(ky[k]); + const float* S0 = src[k] + i; + const float* S1 = src[-k] + i; + s0 = v_muladd(v_kyk, vx_load(S0) + vx_load(S1), s0); + s1 = v_muladd(v_kyk, vx_load(S0 + v_float32::nlanes) + vx_load(S1 + v_float32::nlanes), s1); + s2 = v_muladd(v_kyk, vx_load(S0 + 2*v_float32::nlanes) + vx_load(S1 + 2*v_float32::nlanes), s2); + s3 = v_muladd(v_kyk, vx_load(S0 + 3*v_float32::nlanes) + vx_load(S1 + 3*v_float32::nlanes), s3); + } + v_store(_dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); + } + } + else + { + for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + { + v_float32 s0 = vx_setall_f32(delta); + v_float32 s1 = vx_setall_f32(delta); + v_float32 s2 = vx_setall_f32(delta); + v_float32 s3 = vx_setall_f32(delta); + for( k = 1; k <= ksize2; k++ ) + { + v_float32 v_kyk = vx_setall_f32(ky[k]); + const float* S0 = src[k] + i; + const float* S1 = src[-k] + i; + s0 = v_muladd(v_kyk, vx_load(S0) - vx_load(S1), s0); + s1 = v_muladd(v_kyk, vx_load(S0 + v_float32::nlanes) - vx_load(S1 + v_float32::nlanes), s1); + s2 = v_muladd(v_kyk, vx_load(S0 + 2*v_float32::nlanes) - vx_load(S1 + 2*v_float32::nlanes), s2); + s3 = v_muladd(v_kyk, vx_load(S0 + 3*v_float32::nlanes) - vx_load(S1 + 3*v_float32::nlanes), s3); + } + v_store(_dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); + } + } + return i; + } + int symmetryType; + float delta; + Mat kernel; +}; + struct SymmColumnSmallVec_32s16s { @@ -2341,6 +2416,7 @@ typedef RowNoVec RowVec_32f; typedef SymmRowSmallNoVec SymmRowSmallVec_8u32s; typedef SymmRowSmallNoVec SymmRowSmallVec_32f; typedef ColumnNoVec SymmColumnVec_32s8u; +typedef ColumnNoVec SymmColumnVec_32f8u; typedef ColumnNoVec SymmColumnVec_32f16s; typedef ColumnNoVec SymmColumnVec_32f; typedef SymmColumnSmallNoVec SymmColumnSmallVec_32s16s; @@ -3031,8 +3107,9 @@ Ptr getLinearColumnFilter( (kernel, anchor, delta, symmetryType, FixedPtCastEx(bits), SymmColumnVec_32s8u(kernel, symmetryType, bits, delta)); if( ddepth == CV_8U && sdepth == CV_32F ) - return makePtr, ColumnNoVec> > - (kernel, anchor, delta, symmetryType); + return makePtr, SymmColumnVec_32f8u> > + (kernel, anchor, delta, symmetryType, Cast(), + SymmColumnVec_32f8u(kernel, symmetryType, 0, delta)); if( ddepth == CV_8U && sdepth == CV_64F ) return makePtr, ColumnNoVec> > (kernel, anchor, delta, symmetryType); From 1feb3838b51c0b27b0f3e64f02358ff5ef3111b8 Mon Sep 17 00:00:00 2001 From: Smirnov Egor Date: Thu, 14 Oct 2021 17:38:57 +0300 Subject: [PATCH 295/376] add Ceil, Floor, Log, Round, Sqrt, Not, Equal, Less, Greater --- .../dnn/include/opencv2/dnn/all_layers.hpp | 43 + modules/dnn/src/cuda/activations.cu | 42 + modules/dnn/src/cuda/functors.hpp | 90 ++ modules/dnn/src/cuda/math.hpp | 21 + .../dnn/src/cuda4dnn/kernels/activations.hpp | 18 + .../src/cuda4dnn/primitives/activation.hpp | 320 +++---- modules/dnn/src/init.cpp | 7 + modules/dnn/src/layers/elementwise_layers.cpp | 897 ++++++++---------- modules/dnn/src/layers/scale_layer.cpp | 53 +- modules/dnn/src/onnx/onnx_importer.cpp | 36 + modules/dnn/src/opencl/activations.cl | 36 + modules/dnn/test/test_onnx_importer.cpp | 76 ++ 12 files changed, 936 insertions(+), 703 deletions(-) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 9fde7ad4c46d..c70ce2b50abb 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -600,6 +600,42 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS CeilLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS FloorLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS LogLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS RoundLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS SqrtLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS NotLayer : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS ActivationLayerInt8 : public ActivationLayer { public: @@ -665,6 +701,7 @@ CV__DNN_INLINE_NS_BEGIN public: bool hasBias; int axis; + String mode; static Ptr create(const LayerParams& params); }; @@ -689,6 +726,12 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS CompareLayer : public Layer + { + public: + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS DataAugmentationLayer : public Layer { public: diff --git a/modules/dnn/src/cuda/activations.cu b/modules/dnn/src/cuda/activations.cu index 599d58852eac..c38fa0346f2f 100644 --- a/modules/dnn/src/cuda/activations.cu +++ b/modules/dnn/src/cuda/activations.cu @@ -128,6 +128,36 @@ void bnll(const Stream& stream, Span output, View input) { generic_op>(stream, output, input); } +template +void ceil(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void floor(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void log(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void rint(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void sqrt(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + +template +void not_k(const Stream& stream, Span output, View input) { + generic_op>(stream, output, input); +} + template void abs(const Stream& stream, Span output, View input) { generic_op>(stream, output, input); @@ -160,6 +190,12 @@ template void sigmoid<__half>(const Stream&, Span<__half>, View<__half>); template void elu<__half>(const Stream&, Span<__half>, View<__half>); template void abs<__half>(const Stream& stream, Span<__half> output, View<__half> input); template void bnll<__half>(const Stream&, Span<__half>, View<__half>); +template void ceil<__half>(const Stream&, Span<__half>, View<__half>); +template void floor<__half>(const Stream&, Span<__half>, View<__half>); +template void log<__half>(const Stream&, Span<__half>, View<__half>); +template void rint<__half>(const Stream&, Span<__half>, View<__half>); +template void sqrt<__half>(const Stream&, Span<__half>, View<__half>); +template void not_k<__half>(const Stream&, Span<__half>, View<__half>); template void power<__half>(const Stream&, Span<__half>, View<__half>, __half, __half, __half); template void exp<__half>(const Stream&, Span<__half>, View<__half>, __half, __half); #endif @@ -174,6 +210,12 @@ template void sigmoid(const Stream&, Span, View); template void elu(const Stream&, Span, View); template void abs(const Stream& stream, Span output, View input); template void bnll(const Stream&, Span, View); +template void ceil(const Stream&, Span, View); +template void floor(const Stream&, Span, View); +template void log(const Stream&, Span, View); +template void rint(const Stream&, Span, View); +template void sqrt(const Stream&, Span, View); +template void not_k(const Stream&, Span, View); template void power(const Stream&, Span, View, float, float, float); template void exp(const Stream&, Span, View, float, float); diff --git a/modules/dnn/src/cuda/functors.hpp b/modules/dnn/src/cuda/functors.hpp index f01a07c77ee4..04b545acaf85 100644 --- a/modules/dnn/src/cuda/functors.hpp +++ b/modules/dnn/src/cuda/functors.hpp @@ -209,6 +209,96 @@ struct BNLLFunctor { } }; +template +struct CeilFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE CeilFunctor() { } + CUDA4DNN_DEVICE CeilFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::ceil; + return ceil(value); + } +}; + +template +struct FloorFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE FloorFunctor() { } + CUDA4DNN_DEVICE FloorFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::floor; + return floor(value); + } +}; + +template +struct LogFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE LogFunctor() { } + CUDA4DNN_DEVICE LogFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::log; + return log(value); + } +}; + +template +struct RintFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE RintFunctor() { } + CUDA4DNN_DEVICE RintFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::rint; + return rint(value); + } +}; + +template +struct SqrtFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE SqrtFunctor() { } + CUDA4DNN_DEVICE SqrtFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::sqrt; + return sqrt(value); + } +}; + +template +struct NotFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE NotFunctor() { } + CUDA4DNN_DEVICE NotFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T value) { + using csl::device::floor; + return floor(static_cast(1.) - value); + } +}; + template struct PowerFunctor { struct Params { diff --git a/modules/dnn/src/cuda/math.hpp b/modules/dnn/src/cuda/math.hpp index 273f3fe98e0c..0da584197d81 100644 --- a/modules/dnn/src/cuda/math.hpp +++ b/modules/dnn/src/cuda/math.hpp @@ -119,6 +119,27 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace csl { namespace de template <> inline __device__ __half round(__half value) { return hrint(value); } #endif + template __device__ T floor(T value); + template <> inline __device__ double floor(double value) { return ::floor(value); } + template <> inline __device__ float floor(float value) { return floorf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half floor(__half value) { return hfloor(value); } +#endif + + template __device__ T log(T value); + template <> inline __device__ double log(double value) { return ::log(value); } + template <> inline __device__ float log(float value) { return logf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half log(__half value) { return hlog(value); } +#endif + + template __device__ T rint(T value); + template <> inline __device__ double rint(double value) { return ::rint(value); } + template <> inline __device__ float rint(float value) { return rintf(value); } +#if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) + template <> inline __device__ __half rint(__half value) { return hrint(value); } +#endif + template __device__ T ceil(T value); template <> inline __device__ double ceil(double value) { return ::ceil(value); } template <> inline __device__ float ceil(float value) { return ceilf(value); } diff --git a/modules/dnn/src/cuda4dnn/kernels/activations.hpp b/modules/dnn/src/cuda4dnn/kernels/activations.hpp index 0a7c9878fbad..0fcf7dab8aa6 100644 --- a/modules/dnn/src/cuda4dnn/kernels/activations.hpp +++ b/modules/dnn/src/cuda4dnn/kernels/activations.hpp @@ -42,6 +42,24 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { template void bnll(const csl::Stream& stream, csl::Span output, csl::View input); + template + void ceil(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void floor(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void log(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void rint(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void sqrt(const csl::Stream& stream, csl::Span output, csl::View input); + + template + void not_k(const csl::Stream& stream, csl::Span output, csl::View input); + template void power(const csl::Stream& stream, csl::Span output, csl::View input, T exp, T scale, T shift); diff --git a/modules/dnn/src/cuda4dnn/primitives/activation.hpp b/modules/dnn/src/cuda4dnn/primitives/activation.hpp index 84b95927a317..a179db2da51f 100644 --- a/modules/dnn/src/cuda4dnn/primitives/activation.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/activation.hpp @@ -18,14 +18,12 @@ namespace cv { namespace dnn { namespace cuda4dnn { - template - class ReLUOp final : public CUDABackendNode { - public: + template class Op, class T> + struct BaseOp : public CUDABackendNode + { + protected: using wrapper_type = GetCUDABackendWrapperType; - ReLUOp(csl::Stream stream_, T slope_) - : stream(std::move(stream_)), slope{ slope_ } { } - void forward( const std::vector>& inputs, const std::vector>& outputs, @@ -39,9 +37,21 @@ namespace cv { namespace dnn { namespace cuda4dnn { auto output_wrapper = outputs[i].dynamicCast(); auto output = output_wrapper->getSpan(); - kernels::relu(stream, output, input, slope); + static_cast*>(this)->calculate(output, input); } } + }; + + template + class ReLUOp final : public BaseOp { + public: + ReLUOp(csl::Stream stream_, T slope_) + : stream(std::move(stream_)), slope{ slope_ } { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::relu(stream, output, input, slope); + } private: csl::Stream stream; @@ -49,28 +59,14 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class ClippedReLUOp final : public CUDABackendNode { + class ClippedReLUOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - ClippedReLUOp(csl::Stream stream_, T min_, T max_) : stream(std::move(stream_)), min{ min_ }, max{ max_ } { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::clipped_relu(stream, output, input, min, max); - } + kernels::clipped_relu(stream, output, input, min, max); } private: @@ -79,35 +75,21 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class ChannelwiseReLUOp final : public CUDABackendNode { + class ChannelwiseReLUOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - ChannelwiseReLUOp(csl::Stream stream_, const Mat& slope) - : stream(std::move(stream_)) + : stream(std::move(stream_)) { CV_Assert(!slope.empty()); slopeTensor = csl::makeTensorHeader(slope); csl::copyMatToTensor(slope, slopeTensor, stream); } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - CV_Assert(input.get_axis_size(1) == slopeTensor.size()); - std::size_t inner_size = input.size_range(2, input.rank()); - kernels::axiswise_relu(stream, output, input, inner_size, slopeTensor); - } + CV_Assert(input.get_axis_size(1) == slopeTensor.size()); + std::size_t inner_size = input.size_range(2, input.rank()); + kernels::axiswise_relu(stream, output, input, inner_size, slopeTensor); } private: @@ -116,27 +98,13 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class TanHOp final : public CUDABackendNode { + class TanHOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - TanHOp(csl::Stream stream_) : stream(std::move(stream_)) { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::tanh(stream, output, input); - } + kernels::tanh(stream, output, input); } private: @@ -144,27 +112,13 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class SwishOp final : public CUDABackendNode { + class SwishOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - SwishOp(csl::Stream stream_) : stream(std::move(stream_)) { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::swish(stream, output, input); - } + kernels::swish(stream, output, input); } private: @@ -172,27 +126,13 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class MishOp final : public CUDABackendNode { + class MishOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - MishOp(csl::Stream stream_) : stream(std::move(stream_)) { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::mish(stream, output, input); - } + kernels::mish(stream, output, input); } private: @@ -200,27 +140,13 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class SigmoidOp final : public CUDABackendNode { + class SigmoidOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - SigmoidOp(csl::Stream stream_) : stream(std::move(stream_)) { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::sigmoid(stream, output, input); - } + kernels::sigmoid(stream, output, input); } private: @@ -228,27 +154,27 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class ELUOp final : public CUDABackendNode { + class ELUOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - ELUOp(csl::Stream stream_) : stream(std::move(stream_)) { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); + kernels::elu(stream, output, input); + } - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); + private: + csl::Stream stream; + }; - kernels::elu(stream, output, input); - } + template + class AbsValOp final : public BaseOp { + public: + AbsValOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::abs(stream, output, input); } private: @@ -256,27 +182,41 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class AbsValOp final : public CUDABackendNode { + class BNLLOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; + BNLLOp(csl::Stream stream_) : stream(std::move(stream_)) { } - AbsValOp(csl::Stream stream_) : stream(std::move(stream_)) { } + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::bnll(stream, output, input); + } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + private: + csl::Stream stream; + }; + + template + class CeilOp final : public BaseOp { + public: + CeilOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); + kernels::ceil(stream, output, input); + } - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); + private: + csl::Stream stream; + }; - kernels::abs(stream, output, input); - } + template + class FloorOp final : public BaseOp { + public: + FloorOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::floor(stream, output, input); } private: @@ -284,27 +224,41 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class BNLLOp final : public CUDABackendNode { + class LogOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; + LogOp(csl::Stream stream_) : stream(std::move(stream_)) { } - BNLLOp(csl::Stream stream_) : stream(std::move(stream_)) { } + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::log(stream, output, input); + } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + private: + csl::Stream stream; + }; + + template + class RoundOp final : public BaseOp { + public: + RoundOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); + kernels::rint(stream, output, input); + } - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); + private: + csl::Stream stream; + }; - kernels::bnll(stream, output, input); - } + template + class SqrtOp final : public BaseOp { + public: + SqrtOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::sqrt(stream, output, input); } private: @@ -312,28 +266,28 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class PowerOp final : public CUDABackendNode { + class NotOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; + NotOp(csl::Stream stream_) : stream(std::move(stream_)) { } + + void calculate(csl::TensorSpan output, csl::TensorView input) const + { + kernels::not_k(stream, output, input); + } + private: + csl::Stream stream; + }; + + template + class PowerOp final : public BaseOp { + public: PowerOp(csl::Stream stream_, T exp_, T scale_, T shift_) : stream(std::move(stream_)), exp{ exp_ }, scale{ scale_ }, shift{ shift_ } { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::power(stream, output, input, exp, scale, shift); - } + kernels::power(stream, output, input, exp, scale, shift); } private: @@ -342,28 +296,14 @@ namespace cv { namespace dnn { namespace cuda4dnn { }; template - class ExpOp final : public CUDABackendNode { + class ExpOp final : public BaseOp { public: - using wrapper_type = GetCUDABackendWrapperType; - ExpOp(csl::Stream stream_, T nScale_, T nShift_) : stream(std::move(stream_)), normScale{ nScale_ }, normShift{ nShift_ } { } - void forward( - const std::vector>& inputs, - const std::vector>& outputs, - csl::Workspace& workspace) override + void calculate(csl::TensorSpan output, csl::TensorView input) const { - for (int i = 0; i < inputs.size(); i++) - { - auto input_wrapper = inputs[i].dynamicCast(); - auto input = input_wrapper->getView(); - - auto output_wrapper = outputs[i].dynamicCast(); - auto output = output_wrapper->getSpan(); - - kernels::exp(stream, output, input, normScale, normShift); - } + kernels::exp(stream, output, input, normScale, normShift); } private: diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 123cb170b72e..affaa1a7e14b 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -111,6 +111,12 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(AbsVal, AbsLayer); CV_DNN_REGISTER_LAYER_CLASS(Power, PowerLayer); CV_DNN_REGISTER_LAYER_CLASS(Exp, ExpLayer); + CV_DNN_REGISTER_LAYER_CLASS(Ceil, CeilLayer); + CV_DNN_REGISTER_LAYER_CLASS(Floor, FloorLayer); + CV_DNN_REGISTER_LAYER_CLASS(Log, LogLayer); + CV_DNN_REGISTER_LAYER_CLASS(Round, RoundLayer); + CV_DNN_REGISTER_LAYER_CLASS(Sqrt, SqrtLayer); + CV_DNN_REGISTER_LAYER_CLASS(Not, NotLayer); CV_DNN_REGISTER_LAYER_CLASS(BatchNorm, BatchNormLayer); CV_DNN_REGISTER_LAYER_CLASS(MaxUnpool, MaxUnpoolLayer); CV_DNN_REGISTER_LAYER_CLASS(Dropout, BlankLayer); @@ -133,6 +139,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(Padding, PaddingLayer); CV_DNN_REGISTER_LAYER_CLASS(Proposal, ProposalLayer); CV_DNN_REGISTER_LAYER_CLASS(Scale, ScaleLayer); + CV_DNN_REGISTER_LAYER_CLASS(Compare, CompareLayer); CV_DNN_REGISTER_LAYER_CLASS(DataAugmentation, DataAugmentationLayer); CV_DNN_REGISTER_LAYER_CLASS(Correlation, CorrelationLayer); CV_DNN_REGISTER_LAYER_CLASS(Accum, AccumLayer); diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 1b6483a5a0a6..c95dbbc9337f 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -49,7 +49,7 @@ #include "../op_vkcom.hpp" #include -#include +#include #ifdef HAVE_OPENCL #include "opencl_kernels_dnn.hpp" @@ -69,6 +69,11 @@ using std::abs; using std::exp; using std::tanh; using std::pow; +using std::ceil; +using std::floor; +using std::log; +using std::sqrt; +using std::round; template class ElementWiseLayer : public Func::Layer @@ -599,18 +604,9 @@ struct ReLU6Functor : public BaseFunctor int64 getFLOPSPerElement() const { return 2; } }; -struct TanHFunctor : public BaseFunctor +template +struct BaseDefaultFunctor : public BaseFunctor { - typedef TanHLayer Layer; - - bool supportBackend(int backendId, int) - { - return backendId == DNN_BACKEND_OPENCV || - backendId == DNN_BACKEND_CUDA || - backendId == DNN_BACKEND_HALIDE || - backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; - } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const { for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) @@ -618,7 +614,7 @@ struct TanHFunctor : public BaseFunctor for( int i = 0; i < len; i++ ) { float x = srcptr[i]; - dstptr[i] = tanh(x); + dstptr[i] = static_cast(this)->calculate(x); } } } @@ -638,45 +634,53 @@ struct TanHFunctor : public BaseFunctor UMat& src = inputs[i]; UMat& dst = outputs[i]; - ocl::Kernel kernel("TanHForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); + ocl::Kernel kernel(ocl_kernel_name, ocl::dnn::activations_oclsrc, buildopt); + kernel.set(0, static_cast(src.total())); kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); + static_cast(this)->setKernelParams(kernel); size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); + CV_Assert(kernel.run(1, &gSize, nullptr, false)); } return true; } #endif -#ifdef HAVE_CUDA - Ptr initCUDA(int target, csl::Stream stream) - { - return make_cuda_node(target, stream); - } -#endif + inline void setKernelParams(ocl::Kernel& kernel) const {} -#ifdef HAVE_HALIDE - void attachHalide(const Halide::Expr& input, Halide::Func& top) + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) { - Halide::Var x("x"), y("y"), c("c"), n("n"); - top(x, y, c, n) = tanh(input); + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale * static_cast(i - inpZp); + float y = static_cast(this)->calculate(x); + int quantized = outZp + static_cast(std::round(y/outScale)); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; } -#endif // HAVE_HALIDE #ifdef HAVE_DNN_IE_NN_BUILDER_2019 InferenceEngine::Builder::Layer initInfEngineBuilderAPI() { - return InferenceEngine::Builder::TanHLayer(""); + CV_Error(Error::StsNotImplemented, ""); } #endif // HAVE_DNN_IE_NN_BUILDER_2019 #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { - return std::make_shared(node); + CV_Error(Error::StsNotImplemented, ""); } #endif // HAVE_DNN_NGRAPH @@ -688,84 +692,31 @@ struct TanHFunctor : public BaseFunctor } #endif // HAVE_VULKAN - bool tryQuantize(const std::vector > &scales, - const std::vector > &zeropoints, LayerParams& params) - { - float inpScale = scales[0][0], outScale = scales[1][0]; - int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; - - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inpScale*(i - inpZp); - float y = tanh(x); - int quantized = outZp + (int)std::round(y/outScale); - table[i+128] = saturate_cast(quantized); - } - params.blobs.clear(); - params.blobs.push_back(lookUpTable); - return true; - } - - int64 getFLOPSPerElement() const { return 1; } +private: + static const char* const ocl_kernel_name; }; -struct SwishFunctor : public BaseFunctor +struct TanHFunctor : public BaseDefaultFunctor { - typedef SwishLayer Layer; + typedef TanHLayer Layer; bool supportBackend(int backendId, int) { return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || - backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH;; + backendId == DNN_BACKEND_HALIDE || + backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = x / (1.0f + exp(-x)); - } - } + return tanh(x); } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("SwishForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; - } -#endif - #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) { - return make_cuda_node(target, stream); + return make_cuda_node(target, stream); } #endif @@ -773,57 +724,76 @@ struct SwishFunctor : public BaseFunctor void attachHalide(const Halide::Expr& input, Halide::Func& top) { Halide::Var x("x"), y("y"), c("c"), n("n"); - top(x, y, c, n) = input / (1.0f + exp(-input)); + top(x, y, c, n) = tanh(input); } #endif // HAVE_HALIDE #ifdef HAVE_DNN_IE_NN_BUILDER_2019 InferenceEngine::Builder::Layer initInfEngineBuilderAPI() { - CV_Error(Error::StsNotImplemented, ""); + return InferenceEngine::Builder::TanHLayer(""); } #endif // HAVE_DNN_IE_NN_BUILDER_2019 #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { - auto sigmoid = std::make_shared(node); - return std::make_shared(node, sigmoid); + return std::make_shared(node); } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_VULKAN - std::shared_ptr initVkCom() + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const TanHFunctor::BaseDefaultFunctor::ocl_kernel_name = "TanHForward"; + +struct SwishFunctor : public BaseDefaultFunctor +{ + typedef SwishLayer Layer; + + bool supportBackend(int backendId, int) { - // TODO: add vkcom implementation - return std::shared_ptr(); + return backendId == DNN_BACKEND_OPENCV || + backendId == DNN_BACKEND_CUDA || + backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } -#endif // HAVE_VULKAN - bool tryQuantize(const std::vector > &scales, - const std::vector > &zeropoints, LayerParams& params) + inline float calculate(float x) const { - float inpScale = scales[0][0], outScale = scales[1][0]; - int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + return x / (1.f + exp(-x)); + } - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inpScale*(i - inpZp); - float y = x / (1.0f + exp(-x)); - int quantized = outZp + (int)std::round(y/outScale); - table[i+128] = saturate_cast(quantized); - } - params.blobs.clear(); - params.blobs.push_back(lookUpTable); - return true; +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); + } +#endif + +#ifdef HAVE_HALIDE + void attachHalide(const Halide::Expr& input, Halide::Func& top) + { + Halide::Var x("x"), y("y"), c("c"), n("n"); + top(x, y, c, n) = input / (1.0f + exp(-input)); + } +#endif // HAVE_HALIDE + +#ifdef HAVE_DNN_NGRAPH + std::shared_ptr initNgraphAPI(const std::shared_ptr& node) + { + auto sigmoid = std::make_shared(node); + return std::make_shared(node, sigmoid); } +#endif // HAVE_DNN_NGRAPH int64 getFLOPSPerElement() const { return 3; } }; -struct MishFunctor : public BaseFunctor +template<> +const char* const SwishFunctor::BaseDefaultFunctor::ocl_kernel_name = "SwishForward"; + +struct MishFunctor : public BaseDefaultFunctor { typedef MishLayer Layer; @@ -834,53 +804,18 @@ struct MishFunctor : public BaseFunctor backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) + // Use fast approximation introduced in https://github.com/opencv/opencv/pull/17200 + if (x >= 8.f) { - for( int i = 0; i < len; i++ ) - { - // Use fast approximation introduced in https://github.com/opencv/opencv/pull/17200 - float x = srcptr[i]; - if (x >= 8.f) - dstptr[i] = x; - else - { - float eX = exp(x); - float n = (eX + 2) * eX; - dstptr[i] = (x * n) / (n + 2); - } - } - } - } - -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("MishForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); + return x; } - return true; + float eX = exp(x); + float n = (eX + 2.f) * eX; + return (x * n) / (n + 2.f); } -#endif #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) @@ -897,13 +832,6 @@ struct MishFunctor : public BaseFunctor } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 - #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { @@ -917,40 +845,13 @@ struct MishFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_VULKAN - std::shared_ptr initVkCom() - { - // TODO: add vkcom implementation - return std::shared_ptr(); - } -#endif // HAVE_VULKAN - - bool tryQuantize(const std::vector > &scales, - const std::vector > &zeropoints, LayerParams& params) - { - float inpScale = scales[0][0], outScale = scales[1][0]; - int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; - - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inpScale*(i - inpZp); - float eX = exp(x); - float n = (eX + 2) * eX; - float y = (x * n) / (n + 2); - int quantized = outZp + (int)std::round(y/outScale); - table[i+128] = saturate_cast(quantized); - } - params.blobs.clear(); - params.blobs.push_back(lookUpTable); - return true; - } - int64 getFLOPSPerElement() const { return 3; } }; -struct SigmoidFunctor : public BaseFunctor +template<> +const char* const MishFunctor::BaseDefaultFunctor::ocl_kernel_name = "MishForward"; + +struct SigmoidFunctor : public BaseDefaultFunctor { typedef SigmoidLayer Layer; @@ -962,45 +863,10 @@ struct SigmoidFunctor : public BaseFunctor backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const - { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = 1.f/(1.f + exp(-x)); - } - } - } - -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) + inline float calculate(float x) const { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("SigmoidForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; + return 1.f / (1.f + exp(-x)); } -#endif #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) @@ -1031,38 +897,13 @@ struct SigmoidFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_VULKAN - std::shared_ptr initVkCom() - { - // TODO: add vkcom implementation - return std::shared_ptr(); - } -#endif // HAVE_VULKAN - - bool tryQuantize(const std::vector > &scales, - const std::vector > &zeropoints, LayerParams& params) - { - float inpScale = scales[0][0], outScale = scales[1][0]; - int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; - - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inpScale*(i - inpZp); - float y = 1.f/(1.f + exp(-x)); - int quantized = outZp + (int)std::round(y/outScale); - table[i+128] = saturate_cast(quantized); - } - params.blobs.clear(); - params.blobs.push_back(lookUpTable); - return true; - } - int64 getFLOPSPerElement() const { return 3; } }; -struct ELUFunctor : public BaseFunctor +template<> +const char* const SigmoidFunctor::BaseDefaultFunctor::ocl_kernel_name = "SigmoidForward"; + +struct ELUFunctor : public BaseDefaultFunctor { typedef ELULayer Layer; @@ -1074,50 +915,70 @@ struct ELUFunctor : public BaseFunctor backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for(int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = x >= 0.f ? x : exp(x) - 1; - } - } + return x >= 0.f ? x : exp(x) - 1.f; } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) { - std::vector inputs; - std::vector outputs; + return make_cuda_node(target, stream); + } +#endif - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); +#ifdef HAVE_HALIDE + void attachHalide(const Halide::Expr& input, Halide::Func& top) + { + Halide::Var x("x"), y("y"), c("c"), n("n"); + top(x, y, c, n) = select(input >= 0.0f, input, exp(input) - 1); + } +#endif // HAVE_HALIDE - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; +#ifdef HAVE_DNN_IE_NN_BUILDER_2019 + InferenceEngine::Builder::Layer initInfEngineBuilderAPI() + { + return InferenceEngine::Builder::ELULayer(""); + } +#endif // HAVE_DNN_IE_NN_BUILDER_2019 - ocl::Kernel kernel("ELUForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); +#ifdef HAVE_DNN_NGRAPH + std::shared_ptr initNgraphAPI(const std::shared_ptr& node) + { + return std::make_shared(node, 1.0); + } +#endif // HAVE_DNN_NGRAPH - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } + int64 getFLOPSPerElement() const { return 2; } +}; - return true; - } +template<> +const char* const ELUFunctor::BaseDefaultFunctor::ocl_kernel_name = "ELUForward"; + +struct AbsValFunctor : public BaseDefaultFunctor +{ + typedef AbsLayer Layer; + + bool supportBackend(int backendId, int) + { +#ifdef HAVE_INF_ENGINE + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + return !INF_ENGINE_VER_MAJOR_EQ(INF_ENGINE_RELEASE_2019R1); #endif + return backendId == DNN_BACKEND_OPENCV || + backendId == DNN_BACKEND_CUDA || + backendId == DNN_BACKEND_HALIDE; + } + + inline float calculate(float x) const + { + return abs(x); + } #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) { - return make_cuda_node(target, stream); + return make_cuda_node(target, stream); } #endif @@ -1125,114 +986,125 @@ struct ELUFunctor : public BaseFunctor void attachHalide(const Halide::Expr& input, Halide::Func& top) { Halide::Var x("x"), y("y"), c("c"), n("n"); - top(x, y, c, n) = select(input >= 0.0f, input, exp(input) - 1); + top(x, y, c, n) = abs(input); } #endif // HAVE_HALIDE #ifdef HAVE_DNN_IE_NN_BUILDER_2019 InferenceEngine::Builder::Layer initInfEngineBuilderAPI() { - return InferenceEngine::Builder::ELULayer(""); + return InferenceEngine::Builder::ReLULayer("").setNegativeSlope(-0.999999f); } #endif // HAVE_DNN_IE_NN_BUILDER_2019 #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { - return std::make_shared(node, 1.0); + float coeff = -0.999999f; + // float coeff = preferableTarget == DNN_TARGET_MYRIAD ? -0.999f : -0.999999f; + auto slope = std::make_shared(ngraph::element::f32, ngraph::Shape{1}, &coeff); + return std::make_shared(node, slope); } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_VULKAN - std::shared_ptr initVkCom() + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const AbsValFunctor::BaseDefaultFunctor::ocl_kernel_name = "AbsValForward"; + +struct BNLLFunctor : public BaseDefaultFunctor +{ + typedef BNLLLayer Layer; + + bool supportBackend(int backendId, int) { - // TODO: add vkcom implementation - return std::shared_ptr(); + return backendId == DNN_BACKEND_OPENCV || + backendId == DNN_BACKEND_CUDA || + backendId == DNN_BACKEND_HALIDE; } -#endif // HAVE_VULKAN - bool tryQuantize(const std::vector > &scales, - const std::vector > &zeropoints, LayerParams& params) + inline float calculate(float x) const { - float inpScale = scales[0][0], outScale = scales[1][0]; - int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + // https://github.com/BVLC/caffe/blame/1.0/src/caffe/layers/bnll_layer.cpp#L17 + return x > 0 ? x + log(1.f + exp(-x)) : log(1.f + exp(x)); + } - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inpScale*(i - inpZp); - float y = x >= 0.f ? x : exp(x) - 1; - int quantized = outZp + (int)std::round(y/outScale); - table[i+128] = saturate_cast(quantized); - } - params.blobs.clear(); - params.blobs.push_back(lookUpTable); - return true; +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); } +#endif - int64 getFLOPSPerElement() const { return 2; } +#ifdef HAVE_HALIDE + void attachHalide(const Halide::Expr& input, Halide::Func& top) + { + Halide::Var x("x"), y("y"), c("c"), n("n"); + // https://github.com/BVLC/caffe/blame/1.0/src/caffe/layers/bnll_layer.cpp#L17 + top(x, y, c, n) = max(input, 0) + log(1.0f + exp(-abs(input))); + } +#endif // HAVE_HALIDE + + int64 getFLOPSPerElement() const { return 5; } }; -struct AbsValFunctor : public BaseFunctor +template<> +const char* const BNLLFunctor::BaseDefaultFunctor::ocl_kernel_name = "BNLLForward"; + +struct CeilFunctor : public BaseDefaultFunctor { - typedef AbsLayer Layer; + typedef CeilLayer Layer; bool supportBackend(int backendId, int) { -#ifdef HAVE_INF_ENGINE - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - return !INF_ENGINE_VER_MAJOR_EQ(INF_ENGINE_RELEASE_2019R1); -#endif - return backendId == DNN_BACKEND_OPENCV || - backendId == DNN_BACKEND_CUDA || - backendId == DNN_BACKEND_HALIDE; + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = abs(x); - } - } + return ceil(x); } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) { - std::vector inputs; - std::vector outputs; + return make_cuda_node(target, stream); + } +#endif + +#ifdef HAVE_HALIDE + void attachHalide(const Halide::Expr& input, Halide::Func& top) + { + Halide::Var x("x"), y("y"), c("c"), n("n"); + top(x, y, c, n) = ceil(input); + } +#endif // HAVE_HALIDE - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); + int64 getFLOPSPerElement() const { return 1; } +}; - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "CeilForward"; - ocl::Kernel kernel("AbsValForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); +struct FloorFunctor : public BaseDefaultFunctor +{ + typedef FloorLayer Layer; - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; + } - return true; + inline float calculate(float x) const + { + return floor(x); } -#endif #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) { - return make_cuda_node(target, stream); + return make_cuda_node(target, stream); } #endif @@ -1240,114 +1112,109 @@ struct AbsValFunctor : public BaseFunctor void attachHalide(const Halide::Expr& input, Halide::Func& top) { Halide::Var x("x"), y("y"), c("c"), n("n"); - top(x, y, c, n) = abs(input); + top(x, y, c, n) = floor(input); } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "FloorForward"; + +struct LogFunctor : public BaseDefaultFunctor +{ + typedef LogLayer Layer; + + bool supportBackend(int backendId, int) { - return InferenceEngine::Builder::ReLULayer("").setNegativeSlope(-0.999999f); + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 -#ifdef HAVE_DNN_NGRAPH - std::shared_ptr initNgraphAPI(const std::shared_ptr& node) + inline float calculate(float x) const { - float coeff = -0.999999f; - // float coeff = preferableTarget == DNN_TARGET_MYRIAD ? -0.999f : -0.999999f; - auto slope = std::make_shared(ngraph::element::f32, ngraph::Shape{1}, &coeff); - return std::make_shared(node, slope); + return log(x); } -#endif // HAVE_DNN_NGRAPH -#ifdef HAVE_VULKAN - std::shared_ptr initVkCom() +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) { - // TODO: add vkcom implementation - return std::shared_ptr(); + return make_cuda_node(target, stream); } -#endif // HAVE_VULKAN +#endif - bool tryQuantize(const std::vector > &scales, - const std::vector > &zeropoints, LayerParams& params) +#ifdef HAVE_HALIDE + void attachHalide(const Halide::Expr& input, Halide::Func& top) { - float inpScale = scales[0][0], outScale = scales[1][0]; - int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; - - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inpScale*(i - inpZp); - float y = abs(x); - int quantized = outZp + (int)std::round(y/outScale); - table[i+128] = saturate_cast(quantized); - } - params.blobs.clear(); - params.blobs.push_back(lookUpTable); - return true; + Halide::Var x("x"), y("y"), c("c"), n("n"); + top(x, y, c, n) = log(input); } +#endif // HAVE_HALIDE int64 getFLOPSPerElement() const { return 1; } }; -struct BNLLFunctor : public BaseFunctor +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "LogForward"; + +struct RoundFunctor : public BaseDefaultFunctor { - typedef BNLLLayer Layer; + typedef RoundLayer Layer; bool supportBackend(int backendId, int) { - return backendId == DNN_BACKEND_OPENCV || - backendId == DNN_BACKEND_CUDA || - backendId == DNN_BACKEND_HALIDE; + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - // https://github.com/BVLC/caffe/blame/1.0/src/caffe/layers/bnll_layer.cpp#L17 - dstptr[i] = x > 0 ? x + log(1. + exp(-x)) : log(1. + exp(x)); - } - } + // Rounds to even numbers in halfway cases, so 2.5 -> 2, -2.5 -> -2 + int old_rounding_direction = std::fegetround(); + std::fesetround(FE_TONEAREST); + float y = std::nearbyint(x); + std::fesetround(old_rounding_direction); + return y; } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) { - std::vector inputs; - std::vector outputs; + return make_cuda_node(target, stream); + } +#endif - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); +#ifdef HAVE_HALIDE + void attachHalide(const Halide::Expr& input, Halide::Func& top) + { + Halide::Var x("x"), y("y"), c("c"), n("n"); + top(x, y, c, n) = round(input); + } +#endif // HAVE_HALIDE - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; + int64 getFLOPSPerElement() const { return 2; } +}; - ocl::Kernel kernel("BNLLForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "RoundForward"; - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } +struct SqrtFunctor : public BaseDefaultFunctor +{ + typedef SqrtLayer Layer; - return true; + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; + } + + inline float calculate(float x) const + { + return sqrt(x); } -#endif #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) { - return make_cuda_node(target, stream); + return make_cuda_node(target, stream); } #endif @@ -1355,56 +1222,58 @@ struct BNLLFunctor : public BaseFunctor void attachHalide(const Halide::Expr& input, Halide::Func& top) { Halide::Var x("x"), y("y"), c("c"), n("n"); - // https://github.com/BVLC/caffe/blame/1.0/src/caffe/layers/bnll_layer.cpp#L17 - top(x, y, c, n) = max(input, 0) + log(1.0f + exp(-abs(input))); + top(x, y, c, n) = sqrt(input); } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 - #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { - CV_Error(Error::StsNotImplemented, ""); + return std::make_shared(node); } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_VULKAN - std::shared_ptr initVkCom() + int64 getFLOPSPerElement() const { return 1; } +}; + +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "SqrtForward"; + +struct NotFunctor : public BaseDefaultFunctor +{ + typedef NotLayer Layer; + + bool supportBackend(int backendId, int) { - // TODO: add vkcom implementation - return std::shared_ptr(); + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; } -#endif // HAVE_VULKAN - bool tryQuantize(const std::vector > &scales, - const std::vector > &zeropoints, LayerParams& params) + inline float calculate(float x) const { - float inpScale = scales[0][0], outScale = scales[1][0]; - int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + return floor(1.f - x); + } - Mat lookUpTable(1, 256, CV_8S); - int8_t* table = lookUpTable.ptr(); - for (int i = -128; i < 128; i++) - { - float x = inpScale*(i - inpZp); - float y = x > 0 ? x + log(1. + exp(-x)) : log(1. + exp(x)); - int quantized = outZp + (int)std::round(y/outScale); - table[i+128] = saturate_cast(quantized); - } - params.blobs.clear(); - params.blobs.push_back(lookUpTable); - return true; +#ifdef HAVE_CUDA + Ptr initCUDA(int target, csl::Stream stream) + { + return make_cuda_node(target, stream); } +#endif - int64 getFLOPSPerElement() const { return 5; } +#ifdef HAVE_HALIDE + void attachHalide(const Halide::Expr& input, Halide::Func& top) + { + Halide::Var x("x"), y("y"), c("c"), n("n"); + top(x, y, c, n) = floor(1.0f - input); + } +#endif // HAVE_HALIDE + + int64 getFLOPSPerElement() const { return 2; } }; +template<> +const char* const BaseDefaultFunctor::ocl_kernel_name = "NotForward"; + struct PowerFunctor : public BaseFunctor { typedef PowerLayer Layer; @@ -1583,7 +1452,7 @@ struct PowerFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return power == 1 ? 2 : 10; } }; -struct ExpFunctor : public BaseFunctor +struct ExpFunctor : public BaseDefaultFunctor { typedef ExpLayer Layer; float base, scale, shift; @@ -1609,47 +1478,16 @@ struct ExpFunctor : public BaseFunctor backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - float a = normScale, b = normShift; - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = exp(a*x + b); - } - } + return exp(normScale * x + normShift); } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) + inline void setKernelParams(ocl::Kernel& kernel) const { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("ExpForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - kernel.set(3, (float)normScale); - kernel.set(4, (float)normShift); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - return true; + kernel.set(3, normScale); + kernel.set(4, normShift); } -#endif #ifdef HAVE_CUDA Ptr initCUDA(int target, csl::Stream stream) @@ -1666,13 +1504,6 @@ struct ExpFunctor : public BaseFunctor } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 - #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { @@ -1686,17 +1517,12 @@ struct ExpFunctor : public BaseFunctor } #endif // HAVE_DNN_NGRAPH -#ifdef HAVE_VULKAN - std::shared_ptr initVkCom() - { - // TODO: add vkcom implementation - return std::shared_ptr(); - } -#endif // HAVE_VULKAN - int64 getFLOPSPerElement() const { return 3; } }; +template<> +const char* const ExpFunctor::BaseDefaultFunctor::ocl_kernel_name = "ExpForward"; + struct ChannelsPReLUFunctor : public BaseFunctor { typedef ChannelsPReLULayer Layer; @@ -1917,6 +1743,55 @@ Ptr BNLLLayer::create(const LayerParams& params) return l; } + +Ptr CeilLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr FloorLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr LogLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr RoundLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr SqrtLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + +Ptr NotLayer::create(const LayerParams& params) +{ + Ptr l(new ElementWiseLayer()); + l->setParamsFrom(params); + + return l; +} + Ptr PowerLayer::create(const LayerParams& params) { float power = params.get("power", 1.0f); diff --git a/modules/dnn/src/layers/scale_layer.cpp b/modules/dnn/src/layers/scale_layer.cpp index 001db24a2df8..003f78dc1de9 100644 --- a/modules/dnn/src/layers/scale_layer.cpp +++ b/modules/dnn/src/layers/scale_layer.cpp @@ -38,6 +38,7 @@ class ScaleLayerImpl CV_FINAL : public ScaleLayer hasBias = params.get("bias_term", false); axis = params.get("axis", 1); hasWeights = false; + mode = params.get("mode", "scale"); } bool getMemoryShapes(const std::vector &inputs, @@ -59,6 +60,10 @@ class ScaleLayerImpl CV_FINAL : public ScaleLayer virtual bool supportBackend(int backendId) CV_OVERRIDE { + if (mode != "scale") + { + return backendId == DNN_BACKEND_OPENCV; + } return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CUDA || backendId == DNN_BACKEND_HALIDE || @@ -66,6 +71,20 @@ class ScaleLayerImpl CV_FINAL : public ScaleLayer (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && axis > 0); } + template + void handleCompare(const Mat& a, const T& b, Mat& dst, const int spatialSize) + { + Mat out(1, spatialSize, CV_8U); + if (mode == "equal") + compare(a, b, out, CMP_EQ); + else if (mode == "greater") + compare(a, b, out, CMP_GT); + else + compare(a, b, out, CMP_LT); + + out.convertTo(dst, CV_32F, 1. / 255.); + } + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE { CV_TRACE_FUNCTION(); @@ -123,7 +142,16 @@ class ScaleLayerImpl CV_FINAL : public ScaleLayer float b = biasesData ? biasesData[j] : 0; Mat inpSlice(1, spatialSize, CV_32F, inpData); Mat outSlice(1, spatialSize, CV_32F, outData); - inpSlice.convertTo(outSlice, CV_32F, w, b); + + if (mode == "scale") + { + inpSlice.convertTo(outSlice, CV_32F, w, b); + } + else + { + handleCompare(inpSlice, b, outSlice, spatialSize); + } + inpData += spatialSize; outData += spatialSize; } @@ -142,7 +170,16 @@ class ScaleLayerImpl CV_FINAL : public ScaleLayer add(outSlice, bias, outSlice); } else if (hasBias) - add(inpSlice, bias, outSlice); + { + if (mode == "scale") + { + add(inpSlice, bias, outSlice); + } + else + { + handleCompare(inpSlice, bias, outSlice, numWeights); + } + } inpData += numWeights; outData += numWeights; } @@ -385,6 +422,18 @@ Ptr ShiftLayer::create(const LayerParams& params) return Ptr(new ScaleLayerImpl(scaleParams)); } +Ptr CompareLayer::create(const LayerParams& params) +{ + LayerParams compareParams; + compareParams.name = params.name; + compareParams.type = "Scale"; + compareParams.blobs = params.blobs; + compareParams.set("bias_term", true); + compareParams.set("axis", 0); + compareParams.set("mode", params.get("mode")); + return Ptr(new ScaleLayerImpl(compareParams)); +} + class DataAugmentationLayerImpl CV_FINAL : public DataAugmentationLayer { public: diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 7c230f28c8d8..4914810df8dc 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -118,6 +118,8 @@ class ONNXImporter void parseRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseElu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseTanh (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseAbs (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseCompare (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parsePRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseLRN (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -1375,6 +1377,38 @@ void ONNXImporter::parseTanh(LayerParams& layerParams, const opencv_onnx::NodePr addLayer(layerParams, node_proto); } +void ONNXImporter::parseAbs(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "AbsVal"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseCompare(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 2); + const std::string& layer_type = node_proto.op_type(); + + bool is_const_0 = layer_id.find(node_proto.input(0)) == layer_id.end(); + bool is_const_1 = layer_id.find(node_proto.input(1)) == layer_id.end(); + + if (is_const_0 || is_const_1) + { + Mat blob = getBlob(node_proto, static_cast(is_const_1)); + blob = blob.reshape(1, 1); + layerParams.blobs.push_back(blob); + } + + layerParams.type = "Compare"; + + if (layer_type == "Equal") + layerParams.set("mode", "equal"); + else if (layer_type == "Greater") + layerParams.set("mode", "greater"); + else + layerParams.set("mode", "less"); + addLayer(layerParams, node_proto); +} + void ONNXImporter::parsePRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { layerParams.type = "PReLU"; @@ -2904,6 +2938,8 @@ const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() dispatch["Relu"] = &ONNXImporter::parseRelu; dispatch["Elu"] = &ONNXImporter::parseElu; dispatch["Tanh"] = &ONNXImporter::parseTanh; + dispatch["Abs"] = &ONNXImporter::parseAbs; + dispatch["Equal"] = dispatch["Greater"] = dispatch["Less"] = &ONNXImporter::parseCompare; dispatch["PRelu"] = &ONNXImporter::parsePRelu; dispatch["LRN"] = &ONNXImporter::parseLRN; dispatch["InstanceNormalization"] = &ONNXImporter::parseInstanceNormalization; diff --git a/modules/dnn/src/opencl/activations.cl b/modules/dnn/src/opencl/activations.cl index 68f0dd7268c5..bc2a105aba7e 100644 --- a/modules/dnn/src/opencl/activations.cl +++ b/modules/dnn/src/opencl/activations.cl @@ -151,3 +151,39 @@ __kernel void ExpForward(const int n, __global const T* in, __global T* out, out[index] = exp(normShift + normScale * in[index]); } } + +__kernel void CeilForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = ceil(in[index]); +} + +__kernel void FloorForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = floor(in[index]); +} + +__kernel void LogForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = log(in[index]); +} + +__kernel void RoundForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = rint(in[index]); +} + +__kernel void SqrtForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = sqrt(in[index]); +} + +__kernel void NotForward(const int n, __global T* in, __global T* out) { + int index = get_global_id(0); + if(index < n) + out[index] = floor(1.0f - in[index]); +} diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 5d324b8aac08..b5b277a49266 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -353,6 +353,82 @@ TEST_P(Test_ONNX_layers, Exp) testONNXModels("exp"); } +TEST_P(Test_ONNX_layers, Elementwise_Ceil) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("ceil"); +} + +TEST_P(Test_ONNX_layers, Elementwise_Floor) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("floor"); +} + +TEST_P(Test_ONNX_layers, Elementwise_Log) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("log"); +} + +TEST_P(Test_ONNX_layers, Elementwise_Round) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("round"); +} + +TEST_P(Test_ONNX_layers, Elementwise_Sqrt) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("sqrt"); +} + +TEST_P(Test_ONNX_layers, Elementwise_not) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("not"); +} + +TEST_P(Test_ONNX_layers, Compare) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("equal"); + testONNXModels("greater"); + testONNXModels("less"); +} + +TEST_P(Test_ONNX_layers, CompareSameDims) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + testONNXModels("equal_same_dims", npy, 0, 0, false, true, 2); + testONNXModels("greater_same_dims", npy, 0, 0, false, true, 2); + testONNXModels("less_same_dims", npy, 0, 0, false, true, 2); +} + TEST_P(Test_ONNX_layers, Concatenation) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) From a5906827645e183a731ac6b7eeda0e7c8ebb0563 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 15 Oct 2021 16:58:37 +0300 Subject: [PATCH 296/376] cmake: update installation of python extra submodules - support Python standalone builds - loader installs submodules unconditionally --- modules/python/CMakeLists.txt | 5 -- modules/python/bindings/CMakeLists.txt | 4 + modules/python/common.cmake | 40 ---------- modules/python/package/cv2/__init__.py | 3 +- modules/python/python3/CMakeLists.txt | 14 ---- modules/python/python_loader.cmake | 104 +++++++++++++++++++------ 6 files changed, 86 insertions(+), 84 deletions(-) diff --git a/modules/python/CMakeLists.txt b/modules/python/CMakeLists.txt index a51acf386e19..c6a90752246f 100644 --- a/modules/python/CMakeLists.txt +++ b/modules/python/CMakeLists.txt @@ -20,11 +20,6 @@ add_subdirectory(bindings) add_subdirectory(test) -if(NOT OPENCV_SKIP_PYTHON_LOADER) - include("./python_loader.cmake") - message(STATUS "OpenCV Python: during development append to PYTHONPATH: ${CMAKE_BINARY_DIR}/python_loader") -endif() - if(__disable_python2) ocv_module_disable_(python2) endif() diff --git a/modules/python/bindings/CMakeLists.txt b/modules/python/bindings/CMakeLists.txt index 442107b13512..b39c67961563 100644 --- a/modules/python/bindings/CMakeLists.txt +++ b/modules/python/bindings/CMakeLists.txt @@ -8,6 +8,10 @@ set(OPENCV_PYTHON_BINDINGS_DIR "${CMAKE_CURRENT_BINARY_DIR}" CACHE INTERNAL "") # This file is included from a subdirectory set(PYTHON_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../") +if(NOT OPENCV_SKIP_PYTHON_LOADER) + include("${PYTHON_SOURCE_DIR}/python_loader.cmake") +endif() + # get list of modules to wrap set(OPENCV_PYTHON_MODULES) foreach(m ${OPENCV_MODULES_BUILD}) diff --git a/modules/python/common.cmake b/modules/python/common.cmake index 251b78c6cb3b..ebbb2e2f655d 100644 --- a/modules/python/common.cmake +++ b/modules/python/common.cmake @@ -1,31 +1,6 @@ # This file is included from a subdirectory set(PYTHON_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}") -function(ocv_add_python_files_from_path search_path) - file(GLOB_RECURSE extra_py_files - RELATIVE "${search_path}" - # Plain Python code - "${search_path}/*.py" - # Type annotations - "${search_path}/*.pyi" - ) - message(DEBUG "Extra Py files for ${search_path}: ${extra_py_files}") - if(extra_py_files) - list(SORT extra_py_files) - foreach(filename ${extra_py_files}) - get_filename_component(module "${filename}" DIRECTORY) - if(NOT ${module} IN_LIST extra_modules) - list(APPEND extra_modules ${module}) - endif() - configure_file("${search_path}/${filename}" "${__loader_path}/cv2/${filename}" COPYONLY) - install(FILES "${search_path}/${filename}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/${module}/" COMPONENT python) - endforeach() - message(STATUS "Found ${extra_modules} Python modules from ${search_path}") - else() - message(WARNING "Can't add Python files and modules from ${module_path}. There is no .py or .pyi files") - endif() -endfunction() - ocv_add_module(${MODULE_NAME} BINDINGS PRIVATE_REQUIRED opencv_python_bindings_generator) include_directories(SYSTEM @@ -243,21 +218,6 @@ if(NOT OPENCV_SKIP_PYTHON_LOADER) endif() configure_file("${PYTHON_SOURCE_DIR}/package/template/config-x.y.py.in" "${__python_loader_install_tmp_path}/cv2/${__target_config}" @ONLY) install(FILES "${__python_loader_install_tmp_path}/cv2/${__target_config}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/" COMPONENT python) - - # handle Python extra code - foreach(m ${OPENCV_MODULES_BUILD}) - if (";${OPENCV_MODULE_${m}_WRAPPERS};" MATCHES ";python;" AND HAVE_${m} - AND EXISTS "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package" - ) - ocv_add_python_files_from_path("${OPENCV_MODULE_${m}_LOCATION}/misc/python/package") - endif() - endforeach(m) - - if(NOT "${OCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "") - foreach(extra_ocv_py_modules_path ${OCV_PYTHON_EXTRA_MODULES_PATH}) - ocv_add_python_files_from_path(${extra_ocv_py_modules_path}) - endforeach() - endif() endif() # NOT OPENCV_SKIP_PYTHON_LOADER unset(PYTHON_SRC_DIR) diff --git a/modules/python/package/cv2/__init__.py b/modules/python/package/cv2/__init__.py index 80e26122764f..07d1e0d21e9a 100644 --- a/modules/python/package/cv2/__init__.py +++ b/modules/python/package/cv2/__init__.py @@ -41,7 +41,7 @@ def __load_extra_py_code_for_module(base, name, enable_debug_print=False): setattr(py_module, "_native", native_module) for k, v in filter(lambda kv: not hasattr(py_module, kv[0]), native_module.__dict__.items()): - if enable_debug_print: print(' symbol: {} = {}'.format(k, v)) + if enable_debug_print: print(' symbol({}): {} = {}'.format(name, k, v)) setattr(py_module, k, v) return True @@ -51,6 +51,7 @@ def modules_filter(module): return all(( # module is not internal not module.startswith("_"), + not module.startswith("python-"), # it is not a file os.path.isdir(os.path.join(_extra_submodules_init_path, module)) )) diff --git a/modules/python/python3/CMakeLists.txt b/modules/python/python3/CMakeLists.txt index 0bb401f2ec16..d95af21e04b3 100644 --- a/modules/python/python3/CMakeLists.txt +++ b/modules/python/python3/CMakeLists.txt @@ -15,23 +15,9 @@ set(the_description "The python3 bindings") set(MODULE_NAME python3) set(MODULE_INSTALL_SUBDIR python3) -set(_ocv_extra_modules_path ${CMAKE_CURRENT_LIST_DIR}/../package/extra_modules) -set(_old_ocv_python_extra_modules_path ${OCV_PYTHON_EXTRA_MODULES_PATH}) - -if("${OCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "") - set(OCV_PYTHON_EXTRA_MODULES_PATH ${_ocv_extra_modules_path}) -else() - list(APPEND OCV_PYTHON_EXTRA_MODULES_PATH ${_ocv_extra_modules_path}) -endif() - -unset(_ocv_extra_modules_path) - set(PYTHON PYTHON3) include(../common.cmake) -set(OCV_PYTHON_EXTRA_MODULES_PATH ${_old_ocv_python_extra_modules_path}) - -unset(_old_ocv_python_extra_modules_path) unset(MODULE_NAME) unset(MODULE_INSTALL_SUBDIR) diff --git a/modules/python/python_loader.cmake b/modules/python/python_loader.cmake index a872ffd763de..670d85154cf8 100644 --- a/modules/python/python_loader.cmake +++ b/modules/python/python_loader.cmake @@ -22,6 +22,12 @@ else() set(CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE "os.path.join(LOADER_DIR, 'not_installed')") endif() +if(OpenCV_FOUND) + return() # Ignore "standalone" builds of Python bindings +endif() + + + set(PYTHON_LOADER_FILES "setup.py" "cv2/__init__.py" "cv2/load_config_py2.py" "cv2/load_config_py3.py" @@ -39,34 +45,84 @@ foreach(fname ${PYTHON_LOADER_FILES}) endif() endforeach() -if(NOT OpenCV_FOUND) # Ignore "standalone" builds of Python bindings + + +if(WIN32) + if(CMAKE_GENERATOR MATCHES "Visual Studio") + list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}/Release'") # TODO: CMAKE_BUILD_TYPE is not defined + else() + list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}'") + endif() +else() + list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${LIBRARY_OUTPUT_PATH}'") +endif() +string(REPLACE ";" ",\n " CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_PATH}") +configure_file("${PYTHON_SOURCE_DIR}/package/template/config.py.in" "${__loader_path}/cv2/config.py" @ONLY) + + + +# install +if(DEFINED OPENCV_PYTHON_INSTALL_PATH) if(WIN32) - if(CMAKE_GENERATOR MATCHES "Visual Studio") - list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}/Release'") # TODO: CMAKE_BUILD_TYPE is not defined - else() - list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}'") - endif() + list(APPEND CMAKE_PYTHON_BINARIES_INSTALL_PATH "os.path.join(${CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE}, '${OPENCV_BIN_INSTALL_PATH}')") else() - list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${LIBRARY_OUTPUT_PATH}'") + list(APPEND CMAKE_PYTHON_BINARIES_INSTALL_PATH "os.path.join(${CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE}, '${OPENCV_LIB_INSTALL_PATH}')") endif() - string(REPLACE ";" ",\n " CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_PATH}") - configure_file("${PYTHON_SOURCE_DIR}/package/template/config.py.in" "${__loader_path}/cv2/config.py" @ONLY) - - # install - if(DEFINED OPENCV_PYTHON_INSTALL_PATH) - if(WIN32) - list(APPEND CMAKE_PYTHON_BINARIES_INSTALL_PATH "os.path.join(${CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE}, '${OPENCV_BIN_INSTALL_PATH}')") - else() - list(APPEND CMAKE_PYTHON_BINARIES_INSTALL_PATH "os.path.join(${CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE}, '${OPENCV_LIB_INSTALL_PATH}')") + set(CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_INSTALL_PATH}") + if (WIN32 AND HAVE_CUDA) + if (DEFINED CUDA_TOOLKIT_ROOT_DIR) + list(APPEND CMAKE_PYTHON_BINARIES_PATH "os.path.join(os.getenv('CUDA_PATH', '${CUDA_TOOLKIT_ROOT_DIR}'), 'bin')") endif() - set(CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_INSTALL_PATH}") - if (WIN32 AND HAVE_CUDA) - if (DEFINED CUDA_TOOLKIT_ROOT_DIR) - list(APPEND CMAKE_PYTHON_BINARIES_PATH "os.path.join(os.getenv('CUDA_PATH', '${CUDA_TOOLKIT_ROOT_DIR}'), 'bin')") + endif() + string(REPLACE ";" ",\n " CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_PATH}") + configure_file("${PYTHON_SOURCE_DIR}/package/template/config.py.in" "${__python_loader_install_tmp_path}/cv2/config.py" @ONLY) + install(FILES "${__python_loader_install_tmp_path}/cv2/config.py" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/" COMPONENT python) +endif() + + + +# +# Handle Python extra code (submodules) +# +function(ocv_add_python_files_from_path search_path) + file(GLOB_RECURSE extra_py_files + RELATIVE "${search_path}" + # Plain Python code + "${search_path}/*.py" + # Type annotations + "${search_path}/*.pyi" + ) + ocv_debug_message("Extra Py files for ${search_path}: ${extra_py_files}") + if(extra_py_files) + list(SORT extra_py_files) + foreach(filename ${extra_py_files}) + get_filename_component(module "${filename}" DIRECTORY) + if(NOT ${module} IN_LIST extra_modules) + list(APPEND extra_modules ${module}) endif() - endif() - string(REPLACE ";" ",\n " CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_PATH}") - configure_file("${PYTHON_SOURCE_DIR}/package/template/config.py.in" "${__python_loader_install_tmp_path}/cv2/config.py" @ONLY) - install(FILES "${__python_loader_install_tmp_path}/cv2/config.py" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/" COMPONENT python) + configure_file("${search_path}/${filename}" "${__loader_path}/cv2/${filename}" COPYONLY) + if(DEFINED OPENCV_PYTHON_INSTALL_PATH) + install(FILES "${search_path}/${filename}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/${module}/" COMPONENT python) + endif() + endforeach() + message(STATUS "Found '${extra_modules}' Python modules from ${search_path}") + else() + message(WARNING "Can't add Python files and modules from '${module_path}'. There is no .py or .pyi files") endif() +endfunction() + +ocv_add_python_files_from_path("${PYTHON_SOURCE_DIR}/package/extra_modules") + +foreach(m ${OPENCV_MODULES_BUILD}) + if (";${OPENCV_MODULE_${m}_WRAPPERS};" MATCHES ";python;" AND HAVE_${m} + AND EXISTS "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package" + ) + ocv_add_python_files_from_path("${OPENCV_MODULE_${m}_LOCATION}/misc/python/package") + endif() +endforeach(m) + +if(NOT "${OPENCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "") + foreach(extra_ocv_py_modules_path ${OPENCV_PYTHON_EXTRA_MODULES_PATH}) + ocv_add_python_files_from_path(${extra_ocv_py_modules_path}) + endforeach() endif() From f9e747dbc62277330cdade648ef53f5e5e6c69f9 Mon Sep 17 00:00:00 2001 From: Wehzie <39304339+Wehzie@users.noreply.github.com> Date: Thu, 14 Oct 2021 22:39:49 +0300 Subject: [PATCH 297/376] Fixed typo in CV_Error message Error was "Input parameters must be a matrices!", but "matrices" is plural and doesn't allow the unspecific article "a". --- modules/calib3d/src/calibration.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/calib3d/src/calibration.cpp b/modules/calib3d/src/calibration.cpp index 710b1d7659a4..650735f0354b 100644 --- a/modules/calib3d/src/calibration.cpp +++ b/modules/calib3d/src/calibration.cpp @@ -1737,7 +1737,7 @@ void cvCalibrationMatrixValues( const CvMat *calibMatr, CvSize imgSize, CV_Error(CV_StsNullPtr, "Some of parameters is a NULL pointer!"); if(!CV_IS_MAT(calibMatr)) - CV_Error(CV_StsUnsupportedFormat, "Input parameters must be a matrices!"); + CV_Error(CV_StsUnsupportedFormat, "Input parameters must be matrices!"); double dummy = .0; Point2d pp; @@ -3078,7 +3078,7 @@ cvDecomposeProjectionMatrix( const CvMat *projMatr, CvMat *calibMatr, CV_Error(CV_StsNullPtr, "Some of parameters is a NULL pointer!"); if(!CV_IS_MAT(projMatr) || !CV_IS_MAT(calibMatr) || !CV_IS_MAT(rotMatr) || !CV_IS_MAT(posVect)) - CV_Error(CV_StsUnsupportedFormat, "Input parameters must be a matrices!"); + CV_Error(CV_StsUnsupportedFormat, "Input parameters must be matrices!"); if(projMatr->cols != 4 || projMatr->rows != 3) CV_Error(CV_StsUnmatchedSizes, "Size of projection matrix must be 3x4!"); From 1926e919bead35b3428482e30d50a830c3253d89 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 18 Oct 2021 04:46:00 +0000 Subject: [PATCH 298/376] dnn(int8): fix using of incorrect UMat constructor --- modules/dnn/src/int8layers/quantization_utils.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/dnn/src/int8layers/quantization_utils.cpp b/modules/dnn/src/int8layers/quantization_utils.cpp index 0346f147bab6..d72487639e9e 100644 --- a/modules/dnn/src/int8layers/quantization_utils.cpp +++ b/modules/dnn/src/int8layers/quantization_utils.cpp @@ -52,9 +52,9 @@ class QuantizeLayerImpl CV_FINAL : public QuantizeLayer if (inputs_.depth() == CV_16S) { - UMat inputFp32(shape(inputs[0]), CV_32F); + UMat inputFp32; convertFp16(inputs[0], inputFp32); - inputFp32.copyTo(inputs[0]); + inputs[0] = inputFp32; // replace } inputs[0].convertTo(outputs[0], CV_8S, 1.f/scale, zeropoint); @@ -118,7 +118,7 @@ class DequantizeLayerImpl CV_FINAL : public DequantizeLayer inputs_.getUMatVector(inputs); outputs_.getUMatVector(outputs); - UMat outputFp32(shape(outputs[0]), CV_32F); + UMat outputFp32; inputs[0].convertTo(outputFp32, CV_32F, scale, -(scale*zeropoint)); if (outputs_.depth() == CV_16S) From f8f9f3c43851695726bf32d07bbd176a082755fd Mon Sep 17 00:00:00 2001 From: Sergiu Deitsch Date: Mon, 18 Oct 2021 14:56:15 +0200 Subject: [PATCH 299/376] fixed AVX compile error Some older compilers do not allow to pass a `const int` as an immediate. Use an unnamed enum instead. --- modules/core/include/opencv2/core/hal/intrin_avx.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_avx.hpp b/modules/core/include/opencv2/core/hal/intrin_avx.hpp index 54e8927192c4..09ff56647398 100644 --- a/modules/core/include/opencv2/core/hal/intrin_avx.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_avx.hpp @@ -2379,7 +2379,7 @@ inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& __m256i ab0 = _mm256_loadu_si256((const __m256i*)ptr); __m256i ab1 = _mm256_loadu_si256((const __m256i*)(ptr + 8)); - const int sh = 0+2*4+1*16+3*64; + enum { sh = 0+2*4+1*16+3*64 }; __m256i p0 = _mm256_shuffle_epi32(ab0, sh); __m256i p1 = _mm256_shuffle_epi32(ab1, sh); __m256i pl = _mm256_permute2x128_si256(p0, p1, 0 + 2*16); From 0cf79155d4e06cfcc64cf5eb2acc2fccc2af3db2 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Mon, 18 Oct 2021 19:20:55 +0300 Subject: [PATCH 300/376] Merge pull request #20773 from sivanov-work:merge_vpl_source_unite G-API: oneVPL (simplification) unite components in entire VPL source * Unify components in VPLSource * Revert back decode WRN & Add compile guard * Address come comments * Add source alias * Apply comment for exception handling --- modules/gapi/CMakeLists.txt | 2 + .../opencv2/gapi/streaming/onevpl/source.hpp | 2 + .../gapi/samples/onevpl_infer_single_roi.cpp | 4 +- .../onevpl/accelerators/accel_policy_dx11.cpp | 3 +- .../streaming/onevpl/cfg_params_parser.cpp | 123 +++++ .../streaming/onevpl/cfg_params_parser.hpp | 43 ++ .../onevpl/engine/processing_engine_base.cpp | 2 +- .../onevpl/engine/processing_engine_base.hpp | 3 + .../gapi/src/streaming/onevpl/source_priv.cpp | 332 ++++++++++++- .../gapi/src/streaming/onevpl/source_priv.hpp | 21 + modules/gapi/src/streaming/onevpl/utils.cpp | 454 ++++++++++++++++++ modules/gapi/src/streaming/onevpl/utils.hpp | 24 +- 12 files changed, 1001 insertions(+), 12 deletions(-) create mode 100644 modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp create mode 100644 modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp create mode 100644 modules/gapi/src/streaming/onevpl/utils.cpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 5fd108c0ec01..917d1e081494 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -168,6 +168,8 @@ set(gapi_srcs src/streaming/onevpl/source_priv.cpp src/streaming/onevpl/file_data_provider.cpp src/streaming/onevpl/cfg_params.cpp + src/streaming/onevpl/cfg_params_parser.cpp + src/streaming/onevpl/utils.cpp src/streaming/onevpl/data_provider_interface_exception.cpp src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp src/streaming/onevpl/accelerators/surface/surface.cpp diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp index b7bf4f2ffbcf..a8dbefdf5095 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp @@ -51,6 +51,8 @@ class GAPI_EXPORTS GSource : public IStreamSource }; } // namespace onevpl +using GVPLSource = onevpl::GSource; + template GAPI_EXPORTS_W cv::Ptr inline make_onevpl_src(Args&&... args) { diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index 9a3abdf39203..f3aee09d4299 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -19,6 +19,7 @@ const std::string keys = "{ input | | Path to the input demultiplexed video file }" "{ output | | Path to the output RAW video file. Use .avi extension }" "{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }" + "{ faced | CPU | Target device for face detection model (e.g. CPU, GPU, VPU, ...) }" "{ cfg_params | :;: | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }"; @@ -198,7 +199,8 @@ int main(int argc, char *argv[]) { auto face_net = cv::gapi::ie::Params { face_model_path, // path to topology IR - get_weights_path(face_model_path) // path to weights + get_weights_path(face_model_path), // path to weights + cmd.get("faced"), // device specifier }; auto kernels = cv::gapi::kernels < custom::OCVLocateROI diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp index cb27df86619b..8365dd20e3dc 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp @@ -30,7 +30,8 @@ namespace gapi { namespace wip { namespace onevpl { -VPLDX11AccelerationPolicy::VPLDX11AccelerationPolicy() +VPLDX11AccelerationPolicy::VPLDX11AccelerationPolicy() : + hw_handle(nullptr) { #ifdef CPU_ACCEL_ADAPTER adapter.reset(new VPLCPUAccelerationPolicy); diff --git a/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp b/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp new file mode 100644 index 000000000000..a683c7478eb1 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp @@ -0,0 +1,123 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include + +#include +#include + +#include "streaming/onevpl/cfg_params_parser.hpp" +#include "streaming/onevpl/utils.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONEVPL +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +template <> +struct ParamCreator { + template + CfgParam create (const std::string& name, ValueType&& value) { + return CfgParam::create(name, std::forward(value), is_major_flag); + } + bool is_major_flag = false; +}; + +template <> +struct ParamCreator { + template + mfxVariant create (const std::string& name, ValueType&& value) { + static_assert(std::is_same::type, mfxU32>::value, + "ParamCreator supports mfxU32 at the moment. " + "Feel free to extend for more types"); + return create_impl(name, value); + } +private: + mfxVariant create_impl(const std::string&, mfxU32 value) { + mfxVariant ret; + ret.Type = MFX_VARIANT_TYPE_U32; + ret.Data.U32 = value; + return ret; + } +}; + +template +std::vector get_params_from_string(const std::string& str) { + std::vector ret; + std::string::size_type pos = 0; + std::string::size_type endline_pos = std::string::npos; + do + { + endline_pos = str.find_first_of("\r\n", pos); + std::string line = str.substr(pos, endline_pos == std::string::npos ? std::string::npos : endline_pos - pos); + if (line.empty()) break; + + std::string::size_type name_endline_pos = line.find(':'); + if (name_endline_pos == std::string::npos) { + throw std::runtime_error("Cannot parse param from string: " + line + + ". Name and value should be separated by \":\"" ); + } + + std::string name = line.substr(0, name_endline_pos); + std::string value = line.substr(name_endline_pos + 2); + + ParamCreator creator; + if (name == "mfxImplDescription.Impl") { + ret.push_back(creator.create(name, cstr_to_mfx_impl(value.c_str()))); + } else if (name == "mfxImplDescription.mfxDecoderDescription.decoder.CodecID") { + ret.push_back(creator.create(name, cstr_to_mfx_codec_id(value.c_str()))); + } else if (name == "mfxImplDescription.AccelerationMode") { + ret.push_back(creator.create(name, cstr_to_mfx_accel_mode(value.c_str()))); + } else if (name == "mfxImplDescription.ApiVersion.Version") { + ret.push_back(creator.create(name, cstr_to_mfx_version(value.c_str()))); + } else { + GAPI_LOG_DEBUG(nullptr, "Cannot parse configuration param, name: " << name << + ", value: " << value); + } + + pos = endline_pos + 1; + } + while (endline_pos != std::string::npos); + + return ret; +} + +template +std::vector get_params_from_string(const std::string& str); +template +std::vector get_params_from_string(const std::string& str); + +mfxVariant cfg_param_to_mfx_variant(const CfgParam& cfg_val) { + const CfgParam::name_t& name = cfg_val.get_name(); + mfxVariant ret; + cv::util::visit(cv::util::overload_lambdas( + [&ret](uint8_t value) { ret.Type = MFX_VARIANT_TYPE_U8; ret.Data.U8 = value; }, + [&ret](int8_t value) { ret.Type = MFX_VARIANT_TYPE_I8; ret.Data.I8 = value; }, + [&ret](uint16_t value) { ret.Type = MFX_VARIANT_TYPE_U16; ret.Data.U16 = value; }, + [&ret](int16_t value) { ret.Type = MFX_VARIANT_TYPE_I16; ret.Data.I16 = value; }, + [&ret](uint32_t value) { ret.Type = MFX_VARIANT_TYPE_U32; ret.Data.U32 = value; }, + [&ret](int32_t value) { ret.Type = MFX_VARIANT_TYPE_I32; ret.Data.I32 = value; }, + [&ret](uint64_t value) { ret.Type = MFX_VARIANT_TYPE_U64; ret.Data.U64 = value; }, + [&ret](int64_t value) { ret.Type = MFX_VARIANT_TYPE_I64; ret.Data.I64 = value; }, + [&ret](float_t value) { ret.Type = MFX_VARIANT_TYPE_F32; ret.Data.F32 = value; }, + [&ret](double_t value) { ret.Type = MFX_VARIANT_TYPE_F64; ret.Data.F64 = value; }, + [&ret](void* value) { ret.Type = MFX_VARIANT_TYPE_PTR; ret.Data.Ptr = value; }, + [&ret, &name] (const std::string& value) { + auto parsed = get_params_from_string(name + ": " + value + "\n"); + if (parsed.empty()) { + throw std::logic_error("Unsupported parameter, name: " + name + ", value: " + value); + } + ret = *parsed.begin(); + }), cfg_val.get_value()); + return ret; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp b/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp new file mode 100644 index 000000000000..6247eef9161e --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp @@ -0,0 +1,43 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_CFG_PARAM_PARSER_HPP +#define GAPI_STREAMING_ONEVPL_CFG_PARAM_PARSER_HPP + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif // MFX_VERSION + +#include +#include + +#include +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +template +std::vector get_params_from_string(const std::string& str); + +template +struct ParamCreator { + template + ReturnType create(const std::string& name, ValueType&& value); +}; + +mfxVariant cfg_param_to_mfx_variant(const CfgParam& value); +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_CFG_PARAM_PARSER_HPP diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp index 3161b1627e8a..382c3ae88bad 100644 --- a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp @@ -117,7 +117,7 @@ mfxStatus ReadEncodedStream(mfxBitstream &bs, std::shared_ptr& da return MFX_ERR_NOT_ENOUGH_BUFFER; } - std::copy_n(p0, bs.DataLength, p1); + std::copy_n(p1, bs.DataLength, p0); bs.DataOffset = 0; bs.DataLength += static_cast(data_provider->fetch_data(bs.MaxLength - bs.DataLength, diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp index 7d4869ac66f6..b307c18112e3 100644 --- a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp @@ -11,6 +11,8 @@ #include "streaming/onevpl/engine/engine_session.hpp" #include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS +#ifdef HAVE_ONEVPL + namespace cv { namespace gapi { namespace wip { @@ -93,4 +95,5 @@ mfxStatus ReadEncodedStream(mfxBitstream &bs, std::shared_ptr& da } // namespace gapi } // namespace cv +#endif // HAVE_ONEVPL #endif // GAPI_STREAMING_ONEVPL_ENGINE_PROCESSING_ENGINE_BASE_HPP diff --git a/modules/gapi/src/streaming/onevpl/source_priv.cpp b/modules/gapi/src/streaming/onevpl/source_priv.cpp index ed838d6adc36..28d438a947c4 100644 --- a/modules/gapi/src/streaming/onevpl/source_priv.cpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.cpp @@ -7,6 +7,12 @@ #include #include +#include "streaming/onevpl/engine/decode/decode_engine_legacy.hpp" +#include "streaming/onevpl/accelerators/accel_policy_dx11.hpp" +#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" +#include "streaming/onevpl/utils.hpp" +#include "streaming/onevpl/cfg_params_parser.hpp" + #include "streaming/onevpl/source_priv.hpp" #include "logger.hpp" @@ -32,32 +38,346 @@ namespace cv { namespace gapi { namespace wip { namespace onevpl { + +enum { + VPL_NEW_API_MAJOR_VERSION = 2, + VPL_NEW_API_MINOR_VERSION = 2 +}; + + GSource::Priv::Priv() : - mfx_handle(MFXLoad()) + mfx_handle(MFXLoad()), + mfx_impl_description(), + mfx_handle_configs(), + cfg_params(), + mfx_session(), + description(), + description_is_valid(false), + engine() { GAPI_LOG_INFO(nullptr, "Initialized MFX handle: " << mfx_handle); - description_is_valid = false; } -GSource::Priv::Priv(std::shared_ptr, const std::vector&) : +GSource::Priv::Priv(std::shared_ptr provider, const std::vector& params) : GSource::Priv() { + // Enable Config + if (params.empty()) + { + GAPI_LOG_INFO(nullptr, "No special cfg params requested - use default"); + this->cfg_params = getDefaultCfgParams(); + } + else + { + this->cfg_params = params; + } + + GAPI_LOG_DEBUG(nullptr, "Requested cfg params count: " << cfg_params.size()); + this->mfx_handle_configs.resize(cfg_params.size()); + + // Build VPL handle config from major input params + // VPL dispatcher then uses this config handle to look up for all existing VPL impl + // satisfying major input params and available in the system + GAPI_LOG_INFO(nullptr, "Creating VPL config from input params"); + auto cfg_param_it = cfg_params.begin(); + for (mfxConfig& cfg_inst : mfx_handle_configs) { + cfg_inst = MFXCreateConfig(mfx_handle); + GAPI_Assert(cfg_inst && "MFXCreateConfig failed"); + + if (!cfg_param_it->is_major()) { + GAPI_LOG_DEBUG(nullptr, "Skip not major param: " << cfg_param_it->get_name()); + ++cfg_param_it; + continue; + } + + GAPI_LOG_DEBUG(nullptr, "Apply major param: " << cfg_param_it->get_name()); + mfxVariant mfx_param = cfg_param_to_mfx_variant(*cfg_param_it); + mfxStatus sts = MFXSetConfigFilterProperty(cfg_inst, + (mfxU8 *)cfg_param_it->get_name().c_str(), + mfx_param); + if (sts != MFX_ERR_NONE ) + { + GAPI_LOG_WARNING(nullptr, "MFXSetConfigFilterProperty failed, error: " << + mfxstatus_to_string(sts) << + " - for \"" << cfg_param_it->get_name() << "\""); + GAPI_Assert(false && "MFXSetConfigFilterProperty failed"); + } + + ++cfg_param_it; + } + + // collect optional-preferred input parameters from input params + // which may (optionally) or may not be used to choose the most preferrable + // VPL implementation (for example, specific API version or Debug/Release VPL build) + std::vector preferred_params; + std::copy_if(cfg_params.begin(), cfg_params.end(), std::back_inserter(preferred_params), + [] (const CfgParam& param) { return !param.is_major(); }); + std::sort(preferred_params.begin(), preferred_params.end()); + + GAPI_LOG_DEBUG(nullptr, "Find MFX better implementation from handle: " << mfx_handle << + " is satisfying preferrable params count: " << preferred_params.size()); + int i = 0; + mfxImplDescription *idesc = nullptr; + std::vector available_impl_descriptions; + std::map matches_count; + while (MFX_ERR_NONE == MFXEnumImplementations(mfx_handle, + i, + MFX_IMPLCAPS_IMPLDESCSTRUCTURE, + reinterpret_cast(&idesc))) { + + available_impl_descriptions.push_back(idesc); + + std::stringstream ss; + mfxHDL hImplPath = nullptr; + if (MFX_ERR_NONE == MFXEnumImplementations(mfx_handle, i, MFX_IMPLCAPS_IMPLPATH, &hImplPath)) { + if (hImplPath) { + ss << "Implementation path: " << reinterpret_cast(hImplPath) << std::endl; + MFXDispReleaseImplDescription(mfx_handle, hImplPath); + } + } + ss << *idesc << std::endl; + + GAPI_LOG_INFO(nullptr, "Implementation index: " << i << "\n" << ss.str()); + + // Only one VPL implementation is required for GSource here. + // Let's find intersection params from available impl with preferrable input params + // to find best match. + // An available VPL implementation with max matching count + std::vector impl_params = get_params_from_string(ss.str()); + std::sort(impl_params.begin(), impl_params.end()); + GAPI_LOG_DEBUG(nullptr, "Find implementation cfg params count" << impl_params.size()); + + std::vector matched_params; + std::set_intersection(impl_params.begin(), impl_params.end(), + preferred_params.begin(), preferred_params.end(), + std::back_inserter(matched_params)); + + if (preferred_params.empty()) { + // in case of no input preferrance we consider all params are matched + // for the first available VPL implementation. It will be a chosen one + matches_count.emplace(impl_params.size(), i++); + GAPI_LOG_DEBUG(nullptr, "No preferrable params, use the first one implementation"); + break; + } else { + GAPI_LOG_DEBUG(nullptr, "Equal param intersection count: " << matched_params.size()); + matches_count.emplace(matches_count.size(), i++); + } + } + + // Extract the most suitable VPL implementation by max score + auto max_match_it = matches_count.rbegin(); + GAPI_Assert(max_match_it != matches_count.rend() && + "Cannot find matched MFX implementation for requested configuration"); + + int impl_number = max_match_it->second; + GAPI_LOG_INFO(nullptr, "Chosen implementation index: " << impl_number); + + // release unusable impl available_impl_descriptions + std::swap(mfx_impl_description, available_impl_descriptions[impl_number]); + for (mfxImplDescription* unusable_impl_descr : available_impl_descriptions) { + if (unusable_impl_descr) { + MFXDispReleaseImplDescription(mfx_handle, unusable_impl_descr); + } + } + available_impl_descriptions.clear(); + + // create session for implementation + mfxStatus sts = MFXCreateSession(mfx_handle, impl_number, &mfx_session); + if (MFX_ERR_NONE != sts) { + GAPI_LOG_WARNING(nullptr, "Cannot create MFX Session for implementation index:" << + std::to_string(impl_number) << + ", error: " << mfxstatus_to_string(sts)); + } + + GAPI_LOG_INFO(nullptr, "Initialized MFX session: " << mfx_session); + + // initialize decoder + // Find codec ID from config + auto dec_it = std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { + return value.get_name() == "mfxImplDescription.mfxDecoderDescription.decoder.CodecID"; + }); + GAPI_Assert (dec_it != cfg_params.end() && "Cannot determine DecoderID from oneVPL config. Abort"); + + // create session driving engine if required + if (!engine) { + std::unique_ptr acceleration = initializeHWAccel(); + + // TODO Add factory static method in ProcessingEngineBase + if (mfx_impl_description->ApiVersion.Major >= VPL_NEW_API_MAJOR_VERSION) { + GAPI_Assert(false && + "GSource mfx_impl_description->ApiVersion.Major >= VPL_NEW_API_MAJOR_VERSION" + " - is not implemented"); + } else { + engine.reset(new VPLLegacyDecodeEngine(std::move(acceleration))); + } + } + + //create decoder for session accoring to header recovered from source file + DecoderParams decoder_param = create_decoder_from_file(*dec_it, provider); + + // create engine session for processing mfx session pipeline + engine->initialize_session(mfx_session, std::move(decoder_param), + provider); + + //prepare session for processing + engine->process(mfx_session); } GSource::Priv::~Priv() { + GAPI_LOG_INFO(nullptr, "Unload MFX implementation description: " << mfx_impl_description); + MFXDispReleaseImplDescription(mfx_handle, mfx_impl_description); GAPI_LOG_INFO(nullptr, "Unload MFX handle: " << mfx_handle); MFXUnload(mfx_handle); } -bool GSource::Priv::pull(cv::gapi::wip::Data&) +DecoderParams GSource::Priv::create_decoder_from_file(const CfgParam& decoder_cfg, + std::shared_ptr provider) { - return false; + GAPI_DbgAssert(provider && "Cannot create decoder, data provider is nullptr"); + + mfxBitstream bitstream{}; + const int BITSTREAM_BUFFER_SIZE = 2000000; + bitstream.MaxLength = BITSTREAM_BUFFER_SIZE; + bitstream.Data = (mfxU8 *)calloc(bitstream.MaxLength, sizeof(mfxU8)); + if(!bitstream.Data) { + throw std::runtime_error("Cannot allocate bitstream.Data bytes: " + + std::to_string(bitstream.MaxLength * sizeof(mfxU8))); + } + + mfxVariant decoder = cfg_param_to_mfx_variant(decoder_cfg); + // according to oneVPL documentation references + // https://spec.oneapi.io/versions/latest/elements/oneVPL/source/API_ref/VPL_disp_api_struct.html + // mfxVariant is an `union` type and considered different meaning for different param ids + // So CodecId has U32 data type + bitstream.CodecId = decoder.Data.U32; + + mfxStatus sts = ReadEncodedStream(bitstream, provider); + if(MFX_ERR_NONE != sts) { + throw std::runtime_error("Error reading bitstream, error: " + + mfxstatus_to_string(sts)); + } + + // Retrieve the frame information from input stream + mfxVideoParam mfxDecParams {}; + mfxDecParams.mfx.CodecId = decoder.Data.U32; + mfxDecParams.IOPattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;//MFX_IOPATTERN_OUT_VIDEO_MEMORY; + sts = MFXVideoDECODE_DecodeHeader(mfx_session, &bitstream, &mfxDecParams); + if(MFX_ERR_NONE != sts) { + throw std::runtime_error("Error decoding header, error: " + + mfxstatus_to_string(sts)); + } + + // Input parameters finished, now initialize decode + sts = MFXVideoDECODE_Init(mfx_session, &mfxDecParams); + if (MFX_ERR_NONE != sts) { + throw std::runtime_error("Error initializing Decode, error: " + + mfxstatus_to_string(sts)); + } + + // set valid description + description.size = cv::Size { + mfxDecParams.mfx.FrameInfo.Width, + mfxDecParams.mfx.FrameInfo.Height}; + switch(mfxDecParams.mfx.FrameInfo.FourCC) { + case MFX_FOURCC_I420: + GAPI_Assert(false && "Cannot create GMetaArg description: " + "MediaFrame doesn't support I420 type"); + case MFX_FOURCC_NV12: + description.fmt = cv::MediaFormat::NV12; + break; + default: + { + GAPI_LOG_WARNING(nullptr, "Cannot create GMetaArg description: " + "MediaFrame unknown 'fmt' type: " << + std::to_string(mfxDecParams.mfx.FrameInfo.FourCC)); + GAPI_Assert(false && "Cannot create GMetaArg description: invalid value"); + } + } + description_is_valid = true; + + return {bitstream, mfxDecParams}; +} + +std::unique_ptr GSource::Priv::initializeHWAccel() +{ + std::unique_ptr ret; + + auto accel_mode_it = std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { + return value.get_name() == "mfxImplDescription.AccelerationMode"; + }); + if (accel_mode_it == cfg_params.end()) + { + GAPI_LOG_DEBUG(nullptr, "No HW Accel requested. Use CPU"); + + ret.reset(new VPLCPUAccelerationPolicy); + return ret; + } + + GAPI_LOG_DEBUG(nullptr, "Add HW acceleration support"); + mfxVariant accel_mode = cfg_param_to_mfx_variant(*accel_mode_it); + + switch(accel_mode.Data.U32) { + case MFX_ACCEL_MODE_VIA_D3D11: + { + std::unique_ptr cand(new VPLDX11AccelerationPolicy); + ret = std::move(cand); + break; + } + case MFX_ACCEL_MODE_NA: + { + std::unique_ptr cand(new VPLCPUAccelerationPolicy); + ret = std::move(cand); + break; + } + default: + { + GAPI_LOG_WARNING(nullptr, "Cannot initialize HW Accel: " + "invalid accelerator type: " << + std::to_string(accel_mode.Data.U32)); + GAPI_Assert(false && "Cannot initialize HW Accel"); + } + } + + return ret; +} + +const std::vector& GSource::Priv::getDefaultCfgParams() +{ + static const std::vector def_params = + get_params_from_string( + "mfxImplDescription.Impl: MFX_IMPL_TYPE_HARDWARE\n" + "mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\n"); + + return def_params; +} + +const std::vector& GSource::Priv::getCfgParams() const +{ + return cfg_params; +} + +bool GSource::Priv::pull(cv::gapi::wip::Data& data) +{ + ProcessingEngineBase::ExecutionStatus status = ProcessingEngineBase::ExecutionStatus::Continue; + while (0 == engine->get_ready_frames_count() && + status == ProcessingEngineBase::ExecutionStatus::Continue) { + status = engine->process(mfx_session); + } + + if (engine->get_ready_frames_count()) { + engine->get_frame(data); + return true; + } else { + return false; + } } GMetaArg GSource::Priv::descr_of() const { - return {}; + GAPI_Assert(description_is_valid); + GMetaArg arg(description); + return arg; } } // namespace onevpl } // namespace wip diff --git a/modules/gapi/src/streaming/onevpl/source_priv.hpp b/modules/gapi/src/streaming/onevpl/source_priv.hpp index c2789342532c..cdaab4eb6a45 100644 --- a/modules/gapi/src/streaming/onevpl/source_priv.hpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.hpp @@ -25,23 +25,44 @@ #include +#include "streaming/onevpl/engine/processing_engine_base.hpp" + namespace cv { namespace gapi { namespace wip { namespace onevpl { +struct VPLAccelerationPolicy; +class ProcessingEngineBase; + struct GSource::Priv { explicit Priv(std::shared_ptr provider, const std::vector& params); ~Priv(); + static const std::vector& getDefaultCfgParams(); + const std::vector& getCfgParams() const; + bool pull(cv::gapi::wip::Data& data); GMetaArg descr_of() const; private: Priv(); + DecoderParams create_decoder_from_file(const CfgParam& decoder, + std::shared_ptr provider); + std::unique_ptr initializeHWAccel(); + mfxLoader mfx_handle; + mfxImplDescription *mfx_impl_description; + std::vector mfx_handle_configs; + std::vector cfg_params; + + mfxSession mfx_session; + + cv::GFrameDesc description; bool description_is_valid; + + std::unique_ptr engine; }; } // namespace onevpl } // namespace wip diff --git a/modules/gapi/src/streaming/onevpl/utils.cpp b/modules/gapi/src/streaming/onevpl/utils.cpp new file mode 100644 index 000000000000..72b5c75303a2 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/utils.cpp @@ -0,0 +1,454 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include + +#include +#include + +#ifdef HAVE_ONEVPL + +#include "streaming/onevpl/utils.hpp" +#include "logger.hpp" + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +const char* mfx_impl_to_cstr(const mfxIMPL impl) { + switch (impl) { + case MFX_IMPL_TYPE_SOFTWARE: + return "MFX_IMPL_TYPE_SOFTWARE"; + case MFX_IMPL_TYPE_HARDWARE: + return "MFX_IMPL_TYPE_HARDWARE"; + default: + return "unknown mfxIMPL"; + } +} + +mfxIMPL cstr_to_mfx_impl(const char* cstr) { + if (!strcmp(cstr, "MFX_IMPL_TYPE_SOFTWARE")) { + return MFX_IMPL_TYPE_SOFTWARE; + } else if (!strcmp(cstr, "MFX_IMPL_TYPE_HARDWARE")) { + return MFX_IMPL_TYPE_HARDWARE; + } + + throw std::logic_error(std::string("Invalid \"mfxImplDescription.Impl\":") + cstr); +} + +const char* mfx_accel_mode_to_cstr (const mfxAccelerationMode mode) { + switch (mode) { + case MFX_ACCEL_MODE_NA: + return "MFX_ACCEL_MODE_NA"; + case MFX_ACCEL_MODE_VIA_D3D9: + return "MFX_ACCEL_MODE_VIA_D3D9"; + case MFX_ACCEL_MODE_VIA_D3D11: + return "MFX_ACCEL_MODE_VIA_D3D11"; + case MFX_ACCEL_MODE_VIA_VAAPI: + return "MFX_ACCEL_MODE_VIA_VAAPI"; + case MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET: + return "MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET"; + case MFX_ACCEL_MODE_VIA_VAAPI_GLX: + return "MFX_ACCEL_MODE_VIA_VAAPI_GLX"; + case MFX_ACCEL_MODE_VIA_VAAPI_X11: + return "MFX_ACCEL_MODE_VIA_VAAPI_X11"; + case MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND: + return "MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND"; + case MFX_ACCEL_MODE_VIA_HDDLUNITE: + return "MFX_ACCEL_MODE_VIA_HDDLUNITE"; + default: + return "unknown mfxAccelerationMode"; + } + return "unknown mfxAccelerationMode"; +} + +mfxAccelerationMode cstr_to_mfx_accel_mode(const char* cstr) { + if (!strcmp(cstr, "MFX_ACCEL_MODE_NA")) { + return MFX_ACCEL_MODE_NA; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_D3D9")) { + return MFX_ACCEL_MODE_VIA_D3D9; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_D3D11")) { + return MFX_ACCEL_MODE_VIA_D3D11; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI")) { + return MFX_ACCEL_MODE_VIA_VAAPI; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET")) { + return MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_GLX")) { + return MFX_ACCEL_MODE_VIA_VAAPI_GLX; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_X11")) { + return MFX_ACCEL_MODE_VIA_VAAPI_X11; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND")) { + return MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_HDDLUNITE")) { + return MFX_ACCEL_MODE_VIA_HDDLUNITE; + } + throw std::logic_error(std::string("Invalid \"mfxImplDescription.AccelerationMode\":") + cstr); +} + +const char* mfx_resource_type_to_cstr (const mfxResourceType type) { + switch (type) { + case MFX_RESOURCE_SYSTEM_SURFACE: + return "MFX_RESOURCE_SYSTEM_SURFACE"; + case MFX_RESOURCE_VA_SURFACE: + return "MFX_RESOURCE_VA_SURFACE"; + case MFX_RESOURCE_VA_BUFFER: + return "MFX_RESOURCE_VA_BUFFER"; + case MFX_RESOURCE_DX9_SURFACE: + return "MFX_RESOURCE_DX9_SURFACE"; + case MFX_RESOURCE_DX11_TEXTURE: + return "MFX_RESOURCE_DX11_TEXTURE"; + case MFX_RESOURCE_DX12_RESOURCE: + return "MFX_RESOURCE_DX12_RESOURCE"; + case MFX_RESOURCE_DMA_RESOURCE: + return "MFX_RESOURCE_DMA_RESOURCE"; + case MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY: + return "MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY"; + default: + return "unknown mfxResourceType"; + } +} + +mfxResourceType cstr_to_mfx_resource_type(const char* cstr) { + if (!strcmp(cstr, "MFX_RESOURCE_SYSTEM_SURFACE")) { + return MFX_RESOURCE_SYSTEM_SURFACE; + } else if (!strcmp(cstr, "MFX_RESOURCE_VA_SURFACE")) { + return MFX_RESOURCE_VA_SURFACE; + } else if (!strcmp(cstr, "MFX_RESOURCE_VA_BUFFER")) { + return MFX_RESOURCE_VA_BUFFER; + } else if (!strcmp(cstr, "MFX_RESOURCE_DX9_SURFACE")) { + return MFX_RESOURCE_DX9_SURFACE; + } else if (!strcmp(cstr, "MFX_RESOURCE_DX11_TEXTURE")) { + return MFX_RESOURCE_DX11_TEXTURE; + } else if (!strcmp(cstr, "MFX_RESOURCE_DX12_RESOURCE")) { + return MFX_RESOURCE_DX12_RESOURCE; + } else if (!strcmp(cstr, "MFX_RESOURCE_DMA_RESOURCE")) { + return MFX_RESOURCE_DMA_RESOURCE; + } else if (!strcmp(cstr, "MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY")) { + return MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY; + } + throw std::logic_error(std::string("Invalid \"decoder.Profiles.MemDesc.MemHandleType\":") + cstr); +} + +mfxU32 cstr_to_mfx_codec_id(const char* cstr) { + if (!strcmp(cstr, "MFX_CODEC_AVC")) { + return MFX_CODEC_AVC; + } else if (!strcmp(cstr, "MFX_CODEC_HEVC")) { + return MFX_CODEC_HEVC; + } else if (!strcmp(cstr, "MFX_CODEC_MPEG2")) { + return MFX_CODEC_MPEG2; + } else if (!strcmp(cstr, "MFX_CODEC_VC1")) { + return MFX_CODEC_VC1; + } else if (!strcmp(cstr, "MFX_CODEC_CAPTURE")) { + return MFX_CODEC_CAPTURE; + } else if (!strcmp(cstr, "MFX_CODEC_VP9")) { + return MFX_CODEC_VP9; + } else if (!strcmp(cstr, "MFX_CODEC_AV1")) { + return MFX_CODEC_AV1; + } + throw std::logic_error(std::string("Cannot parse \"mfxImplDescription.mfxDecoderDescription.decoder.CodecID\" value: ") + cstr); +} + +const char* mfx_codec_type_to_cstr(const mfxU32 fourcc, const mfxU32 type) { + switch (fourcc) { + case MFX_CODEC_JPEG: { + switch (type) { + case MFX_PROFILE_UNKNOWN: + return "MFX_PROFILE_UNKNOWN"; + case MFX_PROFILE_JPEG_BASELINE: + return "MFX_PROFILE_JPEG_BASELINE"; + default: + return "::max(); + } + + const char* delim = strchr(cstr, '.'); + if (!delim) { + // in digital form - return as is + return std::stoul(cstr, nullptr, 10); + } + std::string major (cstr, delim - cstr); + std::string minor (delim + 1); + mfxU32 major_val = std::stoul(major, nullptr, 10); + mfxU32 minor_val = std::stoul(minor, nullptr, 10); + + // pack to digital form + return {major_val << 16 | minor_val}; +} + +std::ostream& operator<< (std::ostream& out, const mfxImplDescription& idesc) +{ + out << "mfxImplDescription.Version: " << static_cast(idesc.Version.Major) + << "." << static_cast(idesc.Version.Minor) << std::endl; + out << "mfxImplDescription.Impl: " << mfx_impl_to_cstr(idesc.Impl) << std::endl; + out << "(*)mfxImplDescription.AccelerationMode: " << mfx_accel_mode_to_cstr(idesc.AccelerationMode) << std::endl; + out << "mfxImplDescription.ApiVersion: " << idesc.ApiVersion.Major << "." << idesc.ApiVersion.Minor << std::endl; + out << "(*)mfxImplDescription.ApiVersion.Version: " << idesc.ApiVersion.Version << std::endl; + out << "mfxImplDescription.ImplName: " << idesc.ImplName << std::endl; + out << "mfxImplDescription.License: " << idesc.License << std::endl; + out << "mfxImplDescription.Keywords: " << idesc.Keywords << std::endl; + out << "mfxImplDescription.VendorID: " << idesc.VendorID << std::endl; + out << "mfxImplDescription.VendorImplID: " << idesc.VendorImplID << std::endl; + + const mfxAccelerationModeDescription &accel = idesc.AccelerationModeDescription; + out << "mfxImplDescription.mfxAccelerationMode.Version: " << static_cast(accel.Version.Major) + << "." << static_cast(accel.Version.Minor) << std::endl; + for (int mode = 0; mode < accel.NumAccelerationModes; mode++) { + out << "mfxImplDescription.mfxAccelerationMode.Mode: " << mfx_accel_mode_to_cstr(accel.Mode[mode]) << std::endl; + } + + const mfxDeviceDescription &dev = idesc.Dev; + out << "mfxImplDescription.mfxDeviceDescription.Version: " << static_cast(dev.Version.Major) + << "." << static_cast(dev.Version.Minor) << std::endl; + out << "mfxImplDescription.mfxDeviceDescription.DeviceID: " << dev.DeviceID << std::endl; + for (int subdevice = 0; subdevice < dev.NumSubDevices; subdevice++) { + out << "mfxImplDescription.mfxDeviceDescription.subdevices.Index: " << dev.SubDevices[subdevice].Index << std::endl; + out << "mfxImplDescription.mfxDeviceDescription.subdevices.SubDeviceID: " << dev.SubDevices[subdevice].SubDeviceID << std::endl; + } + + /* mfxDecoderDescription */ + const mfxDecoderDescription &dec = idesc.Dec; + out << "mfxImplDescription.mfxDecoderDescription.Version: " << static_cast(dec.Version.Major) + << "." << static_cast(dec.Version.Minor) << std::endl; + for (int codec = 0; codec < dec.NumCodecs; codec++) { + auto cid = dec.Codecs[codec].CodecID; + out << "(*)mfxImplDescription.mfxDecoderDescription.decoder.CodecID: " << cid;//(cid & 0xff) << "." << (cid >> 8 & 0xff) << "." << (cid >> 16 & 0xff) << "." << (cid >> 24 & 0xff) << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.MaxcodecLevel: " << dec.Codecs[codec].MaxcodecLevel << std::endl; + for (int profile = 0; profile < dec.Codecs[codec].NumProfiles; profile++) { + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles: " + << mfx_codec_type_to_cstr(dec.Codecs[codec].CodecID, + dec.Codecs[codec].Profiles[profile].Profile) << std::endl; + for (int memtype = 0; memtype < dec.Codecs[codec].Profiles[profile].NumMemTypes; memtype++) { + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.MemHandleType: " + << mfx_resource_type_to_cstr(dec.Codecs[codec].Profiles[profile].MemDesc[memtype].MemHandleType) << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Width.Min: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Width.Min << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Width.Max: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Width.Max << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Width.Step: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Width.Step << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Height.Min: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Height.Min << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Height.Max: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Height.Max << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Height.Step: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Height.Step << std::endl; + } + } + } + + out << "mfxImplDescription.NumExtParam: " << idesc.NumExtParam << std::endl; + + out << "\n(*) - configurable params" << std::endl; + return out; +} + +std::string mfxstatus_to_string(mfxStatus err) { + switch(err) + { + case MFX_ERR_NONE: + return "MFX_ERR_NONE"; + case MFX_ERR_UNKNOWN: + return "MFX_ERR_UNKNOWN"; + case MFX_ERR_NULL_PTR: + return "MFX_ERR_NULL_PTR"; + case MFX_ERR_UNSUPPORTED: + return "MFX_ERR_UNSUPPORTED"; + case MFX_ERR_MEMORY_ALLOC: + return "MFX_ERR_MEMORY_ALLOC"; + case MFX_ERR_NOT_ENOUGH_BUFFER: + return "MFX_ERR_NOT_ENOUGH_BUFFER"; + case MFX_ERR_INVALID_HANDLE: + return "MFX_ERR_INVALID_HANDLE"; + case MFX_ERR_LOCK_MEMORY: + return "MFX_ERR_LOCK_MEMORY"; + case MFX_ERR_NOT_INITIALIZED: + return "MFX_ERR_NOT_INITIALIZED"; + case MFX_ERR_NOT_FOUND: + return "MFX_ERR_NOT_FOUND"; + case MFX_ERR_MORE_DATA: + return "MFX_ERR_MORE_DATA"; + case MFX_ERR_MORE_SURFACE: + return "MFX_ERR_MORE_SURFACE"; + case MFX_ERR_ABORTED: + return "MFX_ERR_ABORTED"; + case MFX_ERR_DEVICE_LOST: + return "MFX_ERR_DEVICE_LOST"; + case MFX_ERR_INCOMPATIBLE_VIDEO_PARAM: + return "MFX_ERR_INCOMPATIBLE_VIDEO_PARAM"; + case MFX_ERR_INVALID_VIDEO_PARAM: + return "MFX_ERR_INVALID_VIDEO_PARAM"; + case MFX_ERR_UNDEFINED_BEHAVIOR: + return "MFX_ERR_UNDEFINED_BEHAVIOR"; + case MFX_ERR_DEVICE_FAILED: + return "MFX_ERR_DEVICE_FAILED"; + case MFX_ERR_MORE_BITSTREAM: + return "MFX_ERR_MORE_BITSTREAM"; + case MFX_ERR_GPU_HANG: + return "MFX_ERR_GPU_HANG"; + case MFX_ERR_REALLOC_SURFACE: + return "MFX_ERR_REALLOC_SURFACE"; + case MFX_ERR_RESOURCE_MAPPED: + return "MFX_ERR_RESOURCE_MAPPED"; + case MFX_ERR_NOT_IMPLEMENTED: + return "MFX_ERR_NOT_IMPLEMENTED"; + case MFX_WRN_DEVICE_BUSY: + return "MFX_WRN_DEVICE_BUSY"; + case MFX_WRN_VIDEO_PARAM_CHANGED: + return "MFX_WRN_VIDEO_PARAM_CHANGED"; + + + default: + break; + } + + std::string ret(""; + return ret; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/utils.hpp b/modules/gapi/src/streaming/onevpl/utils.hpp index 0512c4f68734..94f5a249e81b 100644 --- a/modules/gapi/src/streaming/onevpl/utils.hpp +++ b/modules/gapi/src/streaming/onevpl/utils.hpp @@ -26,9 +26,27 @@ namespace gapi { namespace wip { namespace onevpl { -inline std::string mfxstatus_to_string(mfxStatus) { - return "UNKNOWN"; -} +const char* mfx_impl_to_cstr(const mfxIMPL impl); + +mfxIMPL cstr_to_mfx_impl(const char* cstr); + +const char* mfx_accel_mode_to_cstr (const mfxAccelerationMode mode); + +mfxAccelerationMode cstr_to_mfx_accel_mode(const char* cstr); + +const char* mfx_resource_type_to_cstr (const mfxResourceType type); + +mfxResourceType cstr_to_mfx_resource_type(const char* cstr); + +mfxU32 cstr_to_mfx_codec_id(const char* cstr); + +const char* mfx_codec_type_to_cstr(const mfxU32 fourcc, const mfxU32 type); + +mfxU32 cstr_to_mfx_version(const char* cstr); + +std::string mfxstatus_to_string(mfxStatus err); + +std::ostream& operator<< (std::ostream& out, const mfxImplDescription& idesc); } // namespace onevpl } // namespace wip From b5a9a6793b3622b01fe6c7b025f85074fec99491 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Mon, 18 Oct 2021 19:31:48 +0300 Subject: [PATCH 301/376] Merge pull request #20856 from TolyaTalamanov:at/cfg-batch-size G-API: Extend ie::Params to specify batch size * Add cfgBatchSize to ie::Params * Fix comments to review --- .../opencv2/gapi/infer/bindings_ie.hpp | 3 ++ .../gapi/include/opencv2/gapi/infer/ie.hpp | 31 +++++++++-- modules/gapi/src/backends/ie/bindings_ie.cpp | 6 +++ modules/gapi/src/backends/ie/giebackend.cpp | 5 +- modules/gapi/src/backends/ie/util.hpp | 4 +- .../gapi/test/infer/gapi_infer_ie_test.cpp | 54 ++++++++++++++++++- 6 files changed, 94 insertions(+), 9 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp b/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp index 92ef2101a179..94272dea55b0 100644 --- a/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp @@ -44,6 +44,9 @@ class GAPI_EXPORTS_W_SIMPLE PyParams { GAPI_WRAP PyParams& cfgNumRequests(size_t nireq); + GAPI_WRAP + PyParams& cfgBatchSize(const size_t size); + GBackend backend() const; std::string tag() const; cv::util::any params() const; diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp index 2be739e51840..0e9c127fa15b 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ie.hpp @@ -79,6 +79,8 @@ struct ParamDesc { // NB: An optional config to setup RemoteContext for IE cv::util::any context_config; + + size_t batch_size; }; } // namespace detail @@ -120,7 +122,8 @@ template class Params { , {} , {} , 1u - , {}} { + , {} + , 1u} { }; /** @overload @@ -141,7 +144,8 @@ template class Params { , {} , {} , 1u - , {}} { + , {} + , 1u} { }; /** @brief Specifies sequence of network input layers names for inference. @@ -316,6 +320,19 @@ template class Params { return *this; } + /** @brief Specifies the inference batch size. + + The function is used to specify inference batch size. + Follow https://docs.openvinotoolkit.org/latest/classInferenceEngine_1_1CNNNetwork.html#a8e9d19270a48aab50cb5b1c43eecb8e9 for additional information + + @param size batch size which will be used. + @return reference to this parameter structure. + */ + Params& cfgBatchSize(const size_t size) { + desc.batch_size = size; + return *this; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::ie::backend(); } std::string tag() const { return Net::tag(); } @@ -350,7 +367,7 @@ class Params { const std::string &device) : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u, - {}}, + {}, 1u}, m_tag(tag) { }; @@ -368,7 +385,7 @@ class Params { const std::string &device) : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u, - {}}, + {}, 1u}, m_tag(tag) { }; @@ -435,6 +452,12 @@ class Params { return *this; } + /** @see ie::Params::cfgBatchSize */ + Params& cfgBatchSize(const size_t size) { + desc.batch_size = size; + return *this; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::ie::backend(); } std::string tag() const { return m_tag; } diff --git a/modules/gapi/src/backends/ie/bindings_ie.cpp b/modules/gapi/src/backends/ie/bindings_ie.cpp index 5874fe137864..39f07d28e5ee 100644 --- a/modules/gapi/src/backends/ie/bindings_ie.cpp +++ b/modules/gapi/src/backends/ie/bindings_ie.cpp @@ -49,3 +49,9 @@ cv::gapi::ie::PyParams& cv::gapi::ie::PyParams::cfgNumRequests(size_t nireq) { m_priv->cfgNumRequests(nireq); return *this; } + +cv::gapi::ie::PyParams& +cv::gapi::ie::PyParams::cfgBatchSize(const size_t size) { + m_priv->cfgBatchSize(size); + return *this; +} diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 03584c9561c9..c69302baca02 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -237,6 +237,7 @@ struct IEUnit { if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { net = cv::gimpl::ie::wrap::readNetwork(params); + net.setBatchSize(params.batch_size); inputs = net.getInputsInfo(); outputs = net.getOutputsInfo(); } else if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import) { @@ -1412,11 +1413,11 @@ std::vector cv::gapi::ie::util::to_ocv(const IE::SizeVector &dims) { return toCV(dims); } -IE::Blob::Ptr cv::gapi::ie::util::to_ie(cv::Mat &blob) { +IE::Blob::Ptr cv::gapi::ie::util::to_ie(const cv::Mat &blob) { return wrapIE(blob, cv::gapi::ie::TraitAs::IMAGE); } -IE::Blob::Ptr cv::gapi::ie::util::to_ie(cv::Mat &y_plane, cv::Mat &uv_plane) { +IE::Blob::Ptr cv::gapi::ie::util::to_ie(const cv::Mat &y_plane, const cv::Mat &uv_plane) { auto y_blob = wrapIE(y_plane, cv::gapi::ie::TraitAs::IMAGE); auto uv_blob = wrapIE(uv_plane, cv::gapi::ie::TraitAs::IMAGE); #if INF_ENGINE_RELEASE >= 2021010000 diff --git a/modules/gapi/src/backends/ie/util.hpp b/modules/gapi/src/backends/ie/util.hpp index 080c88498feb..e2f1f4414034 100644 --- a/modules/gapi/src/backends/ie/util.hpp +++ b/modules/gapi/src/backends/ie/util.hpp @@ -27,8 +27,8 @@ namespace util { // test suite only. GAPI_EXPORTS std::vector to_ocv(const InferenceEngine::SizeVector &dims); GAPI_EXPORTS cv::Mat to_ocv(InferenceEngine::Blob::Ptr blob); -GAPI_EXPORTS InferenceEngine::Blob::Ptr to_ie(cv::Mat &blob); -GAPI_EXPORTS InferenceEngine::Blob::Ptr to_ie(cv::Mat &y_plane, cv::Mat &uv_plane); +GAPI_EXPORTS InferenceEngine::Blob::Ptr to_ie(const cv::Mat &blob); +GAPI_EXPORTS InferenceEngine::Blob::Ptr to_ie(const cv::Mat &y_plane, const cv::Mat &uv_plane); }}}} diff --git a/modules/gapi/test/infer/gapi_infer_ie_test.cpp b/modules/gapi/test/infer/gapi_infer_ie_test.cpp index b7ea891b812f..49cf47048ee6 100644 --- a/modules/gapi/test/infer/gapi_infer_ie_test.cpp +++ b/modules/gapi/test/infer/gapi_infer_ie_test.cpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2019-2020 Intel Corporation +// Copyright (C) 2019-2021 Intel Corporation #include "../test_precomp.hpp" @@ -2187,6 +2187,58 @@ TEST_F(LimitedSourceInfer, ReleaseFrameAsync) run(max_frames, resources_limit, nireq); } +TEST(TestAgeGenderIE, InferWithBatch) +{ + initDLDTDataPath(); + + constexpr int batch_size = 4; + cv::gapi::ie::detail::ParamDesc params; + params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); + params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); + params.device_id = "CPU"; + + cv::Mat in_mat({batch_size, 3, 320, 240}, CV_8U); + cv::randu(in_mat, 0, 255); + + cv::Mat gapi_age, gapi_gender; + + // Load & run IE network + IE::Blob::Ptr ie_age, ie_gender; + { + auto plugin = cv::gimpl::ie::wrap::getPlugin(params); + auto net = cv::gimpl::ie::wrap::readNetwork(params); + setNetParameters(net); + net.setBatchSize(batch_size); + auto this_network = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params); + auto infer_request = this_network.CreateInferRequest(); + infer_request.SetBlob("data", cv::gapi::ie::util::to_ie(in_mat)); + infer_request.Infer(); + ie_age = infer_request.GetBlob("age_conv3"); + ie_gender = infer_request.GetBlob("prob"); + } + + // Configure & run G-API + using AGInfo = std::tuple; + G_API_NET(AgeGender, , "test-age-gender"); + + cv::GMat in; + cv::GMat age, gender; + std::tie(age, gender) = cv::gapi::infer(in); + cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender)); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }) + .cfgBatchSize(batch_size); + + comp.apply(cv::gin(in_mat), cv::gout(gapi_age, gapi_gender), + cv::compile_args(cv::gapi::networks(pp))); + + // Validate with IE itself (avoid DNN module dependency here) + normAssert(cv::gapi::ie::util::to_ocv(ie_age), gapi_age, "Test age output" ); + normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output"); +} + } // namespace opencv_test #endif // HAVE_INF_ENGINE From b5fcb06a7612e2f0a1f9c0ea2820a34ef0da4206 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 18 Oct 2021 07:15:15 +0000 Subject: [PATCH 302/376] core(SIMD): update int64 SSE constructor --- .../include/opencv2/core/hal/intrin_sse.hpp | 12 ++++++++++++ modules/core/test/test_intrin_utils.hpp | 17 +++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/modules/core/include/opencv2/core/hal/intrin_sse.hpp b/modules/core/include/opencv2/core/hal/intrin_sse.hpp index f4b43a2d7abb..2244717e1986 100644 --- a/modules/core/include/opencv2/core/hal/intrin_sse.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_sse.hpp @@ -244,7 +244,13 @@ struct v_uint64x2 explicit v_uint64x2(__m128i v) : val(v) {} v_uint64x2(uint64 v0, uint64 v1) { +#if defined(_MSC_VER) && _MSC_VER >= 1920/*MSVS 2019*/ && defined(_M_X64) + val = _mm_setr_epi64x((int64_t)v0, (int64_t)v1); +#elif defined(__GNUC__) + val = _mm_setr_epi64((__m64)v0, (__m64)v1); +#else val = _mm_setr_epi32((int)v0, (int)(v0 >> 32), (int)v1, (int)(v1 >> 32)); +#endif } uint64 get0() const @@ -272,7 +278,13 @@ struct v_int64x2 explicit v_int64x2(__m128i v) : val(v) {} v_int64x2(int64 v0, int64 v1) { +#if defined(_MSC_VER) && _MSC_VER >= 1920/*MSVS 2019*/ && defined(_M_X64) + val = _mm_setr_epi64x((int64_t)v0, (int64_t)v1); +#elif defined(__GNUC__) + val = _mm_setr_epi64((__m64)v0, (__m64)v1); +#else val = _mm_setr_epi32((int)v0, (int)(v0 >> 32), (int)v1, (int)(v1 >> 32)); +#endif } int64 get0() const diff --git a/modules/core/test/test_intrin_utils.hpp b/modules/core/test/test_intrin_utils.hpp index 5c22caaf12e5..3f196f134238 100644 --- a/modules/core/test/test_intrin_utils.hpp +++ b/modules/core/test/test_intrin_utils.hpp @@ -373,6 +373,23 @@ template struct TheTest EXPECT_EQ((LaneType)12, vx_setall_res2_[i]); } +#if CV_SIMD_WIDTH == 16 + { + uint64 a = CV_BIG_INT(0x7fffffffffffffff); + uint64 b = (uint64)CV_BIG_INT(0xcfffffffffffffff); + v_uint64x2 uint64_vec(a, b); + EXPECT_EQ(a, uint64_vec.get0()); + EXPECT_EQ(b, v_extract_n<1>(uint64_vec)); + } + { + int64 a = CV_BIG_INT(0x7fffffffffffffff); + int64 b = CV_BIG_INT(-1); + v_int64x2 int64_vec(a, b); + EXPECT_EQ(a, int64_vec.get0()); + EXPECT_EQ(b, v_extract_n<1>(int64_vec)); + } +#endif + return *this; } From 9a9e457dd6b1a1ad1a3d6dc293eba57b1907810e Mon Sep 17 00:00:00 2001 From: Michel Promonet Date: Mon, 18 Oct 2021 18:30:13 +0200 Subject: [PATCH 303/376] Allow to set av_log_set_level to reduce ffmpeg level below AV_LOG_ERROR --- modules/videoio/src/cap_ffmpeg_impl.hpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index 937d34821573..6877a963ef34 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -850,6 +850,7 @@ static void ffmpeg_log_callback(void *ptr, int level, const char *fmt, va_list v static bool skip_header = false; static int prev_level = -1; CV_UNUSED(ptr); + if (level>av_log_get_level()) return; if (!skip_header || level != prev_level) printf("[OPENCV:FFMPEG:%02d] ", level); vprintf(fmt, vargs); size_t fmt_len = strlen(fmt); @@ -866,15 +867,21 @@ class InternalFFMpegRegister static void initLogger_() { - #ifndef NO_GETENV +#ifndef NO_GETENV char* debug_option = getenv("OPENCV_FFMPEG_DEBUG"); - if (debug_option != NULL) + char* level_option = getenv("OPENCV_FFMPEG_LOGLEVEL"); + int level = AV_LOG_VERBOSE; + if (level_option != NULL) + { + level = atoi(level_option); + } + if ( (debug_option != NULL) || (level_option != NULL) ) { - av_log_set_level(AV_LOG_VERBOSE); + av_log_set_level(level); av_log_set_callback(ffmpeg_log_callback); } else - #endif +#endif { av_log_set_level(AV_LOG_ERROR); } From 6d5fdfbf73070e79bb6b061fd96827e8d43b037f Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 19 Oct 2021 09:28:12 +0000 Subject: [PATCH 304/376] samples: fix build without threading support --- samples/CMakeLists.txt | 1 + .../videoio/orbbec_astra/orbbec_astra.cpp | 13 +++++++++++++ samples/dnn/object_detection.cpp | 16 ++++++++++------ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 9d8f58808301..910454ca7194 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -120,6 +120,7 @@ else() find_package(Threads) endif() if((TARGET Threads::Threads OR HAVE_THREADS) AND NOT OPENCV_EXAMPLES_DISABLE_THREADS) + set(HAVE_THREADS 1) add_definitions(-DHAVE_THREADS=1) endif() diff --git a/samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp b/samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp index 581b1768f1f6..6bb9f904af32 100644 --- a/samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp +++ b/samples/cpp/tutorial_code/videoio/orbbec_astra/orbbec_astra.cpp @@ -4,6 +4,17 @@ #include #include + + +#if !defined(HAVE_THREADS) +int main() +{ + std::cout << "This sample is built without threading support. Sample code is disabled." << std::endl; + return 0; +} +#else + + #include #include #include @@ -200,3 +211,5 @@ int main() return 0; } + +#endif diff --git a/samples/dnn/object_detection.cpp b/samples/dnn/object_detection.cpp index 5ff112fe5d13..6fc8b2ab6164 100644 --- a/samples/dnn/object_detection.cpp +++ b/samples/dnn/object_detection.cpp @@ -5,7 +5,11 @@ #include #include -#ifdef CV_CXX11 +#if defined(CV_CXX11) && defined(HAVE_THREADS) +#define USE_THREADS 1 +#endif + +#ifdef USE_THREADS #include #include #include @@ -56,7 +60,7 @@ void drawPred(int classId, float conf, int left, int top, int right, int bottom, void callback(int pos, void* userdata); -#ifdef CV_CXX11 +#ifdef USE_THREADS template class QueueFPS : public std::queue { @@ -106,7 +110,7 @@ class QueueFPS : public std::queue TickMeter tm; std::mutex mutex; }; -#endif // CV_CXX11 +#endif // USE_THREADS int main(int argc, char** argv) { @@ -171,7 +175,7 @@ int main(int argc, char** argv) else cap.open(parser.get("device")); -#ifdef CV_CXX11 +#ifdef USE_THREADS bool process = true; // Frames capturing thread @@ -271,7 +275,7 @@ int main(int argc, char** argv) framesThread.join(); processingThread.join(); -#else // CV_CXX11 +#else // USE_THREADS if (asyncNumReq) CV_Error(Error::StsNotImplemented, "Asynchronous forward is supported only with Inference Engine backend."); @@ -302,7 +306,7 @@ int main(int argc, char** argv) imshow(kWinName, frame); } -#endif // CV_CXX11 +#endif // USE_THREADS return 0; } From b3f966e2ca82475837bbcfeccae72d9623a295c2 Mon Sep 17 00:00:00 2001 From: rogday Date: Tue, 19 Oct 2021 16:29:22 +0300 Subject: [PATCH 305/376] Merge pull request #20883 from rogday:eltwise_refactoring * backport elementwise_layers refactor * keep NULL --- modules/dnn/src/layers/elementwise_layers.cpp | 415 ++++-------------- 1 file changed, 96 insertions(+), 319 deletions(-) diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 6e7fa43a7074..d8f0b654d3ef 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -500,16 +500,9 @@ struct ReLU6Functor : public BaseFunctor int64 getFLOPSPerElement() const { return 2; } }; -struct TanHFunctor : public BaseFunctor +template +struct BaseDefaultFunctor : public BaseFunctor { - typedef TanHLayer Layer; - - bool supportBackend(int backendId, int) - { - return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE || - backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; - } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const { for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) @@ -517,7 +510,7 @@ struct TanHFunctor : public BaseFunctor for( int i = 0; i < len; i++ ) { float x = srcptr[i]; - dstptr[i] = tanh(x); + dstptr[i] = static_cast(this)->calculate(x); } } } @@ -537,10 +530,11 @@ struct TanHFunctor : public BaseFunctor UMat& src = inputs[i]; UMat& dst = outputs[i]; - ocl::Kernel kernel("TanHForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); + ocl::Kernel kernel(ocl_kernel_name, ocl::dnn::activations_oclsrc, buildopt); + kernel.set(0, static_cast(src.total())); kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); + static_cast(this)->setKernelParams(kernel); size_t gSize = src.total(); CV_Assert(kernel.run(1, &gSize, NULL, false)); @@ -550,6 +544,41 @@ struct TanHFunctor : public BaseFunctor } #endif + inline void setKernelParams(ocl::Kernel& kernel) const {} + +#ifdef HAVE_DNN_IE_NN_BUILDER_2019 + InferenceEngine::Builder::Layer initInfEngineBuilderAPI() + { + CV_Error(Error::StsNotImplemented, ""); + } +#endif // HAVE_DNN_IE_NN_BUILDER_2019 + +#ifdef HAVE_DNN_NGRAPH + std::shared_ptr initNgraphAPI(const std::shared_ptr& node) + { + CV_Error(Error::StsNotImplemented, ""); + } +#endif // HAVE_DNN_NGRAPH + +private: + static const char* const ocl_kernel_name; +}; + +struct TanHFunctor : public BaseDefaultFunctor +{ + typedef TanHLayer Layer; + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE || + backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; + } + + inline float calculate(float x) const + { + return tanh(x); + } + #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) { @@ -575,55 +604,23 @@ struct TanHFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return 1; } }; -struct SwishFunctor : public BaseFunctor +template<> +const char* const TanHFunctor::BaseDefaultFunctor::ocl_kernel_name = "TanHForward"; + +struct SwishFunctor : public BaseDefaultFunctor { typedef SwishLayer Layer; bool supportBackend(int backendId, int) { return backendId == DNN_BACKEND_OPENCV || - backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH;; - } - - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const - { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = x / (1.0f + exp(-x)); - } - } + backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) + inline float calculate(float x) const { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("SwishForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; + return x / (1.f + exp(-x)); } -#endif #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) @@ -633,13 +630,6 @@ struct SwishFunctor : public BaseFunctor } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 - #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { @@ -651,7 +641,10 @@ struct SwishFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return 3; } }; -struct MishFunctor : public BaseFunctor +template<> +const char* const SwishFunctor::BaseDefaultFunctor::ocl_kernel_name = "SwishForward"; + +struct MishFunctor : public BaseDefaultFunctor { typedef MishLayer Layer; @@ -661,53 +654,18 @@ struct MishFunctor : public BaseFunctor backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) + // Use fast approximation introduced in https://github.com/opencv/opencv/pull/17200 + if (x >= 8.f) { - for( int i = 0; i < len; i++ ) - { - // Use fast approximation introduced in https://github.com/opencv/opencv/pull/17200 - float x = srcptr[i]; - if (x >= 8.f) - dstptr[i] = x; - else - { - float eX = exp(x); - float n = (eX + 2) * eX; - dstptr[i] = (x * n) / (n + 2); - } - } + return x; } - } - -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("MishForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; + float eX = exp(x); + float n = (eX + 2.f) * eX; + return (x * n) / (n + 2.f); } -#endif #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) @@ -717,13 +675,6 @@ struct MishFunctor : public BaseFunctor } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 - #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { @@ -740,7 +691,10 @@ struct MishFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return 3; } }; -struct SigmoidFunctor : public BaseFunctor +template<> +const char* const MishFunctor::BaseDefaultFunctor::ocl_kernel_name = "MishForward"; + +struct SigmoidFunctor : public BaseDefaultFunctor { typedef SigmoidLayer Layer; @@ -750,45 +704,10 @@ struct SigmoidFunctor : public BaseFunctor backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const - { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = 1.f/(1.f + exp(-x)); - } - } - } - -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) + inline float calculate(float x) const { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("SigmoidForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; + return 1.f / (1.f + exp(-x)); } -#endif #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) @@ -815,7 +734,10 @@ struct SigmoidFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return 3; } }; -struct ELUFunctor : public BaseFunctor +template<> +const char* const SigmoidFunctor::BaseDefaultFunctor::ocl_kernel_name = "SigmoidForward"; + +struct ELUFunctor : public BaseDefaultFunctor { typedef ELULayer Layer; @@ -825,46 +747,11 @@ struct ELUFunctor : public BaseFunctor backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for(int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = x >= 0.f ? x : exp(x) - 1; - } - } + return x >= 0.f ? x : exp(x) - 1.f; } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("ELUForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; - } -#endif - #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) { @@ -890,7 +777,10 @@ struct ELUFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return 2; } }; -struct AbsValFunctor : public BaseFunctor +template<> +const char* const ELUFunctor::BaseDefaultFunctor::ocl_kernel_name = "ELUForward"; + +struct AbsValFunctor : public BaseDefaultFunctor { typedef AbsLayer Layer; @@ -903,45 +793,10 @@ struct AbsValFunctor : public BaseFunctor return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = abs(x); - } - } - } - -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("AbsValForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; + return abs(x); } -#endif #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) @@ -971,7 +826,10 @@ struct AbsValFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return 1; } }; -struct BNLLFunctor : public BaseFunctor +template<> +const char* const AbsValFunctor::BaseDefaultFunctor::ocl_kernel_name = "AbsValForward"; + +struct BNLLFunctor : public BaseDefaultFunctor { typedef BNLLLayer Layer; @@ -980,46 +838,11 @@ struct BNLLFunctor : public BaseFunctor return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - // https://github.com/BVLC/caffe/blame/1.0/src/caffe/layers/bnll_layer.cpp#L17 - dstptr[i] = x > 0 ? x + log(1. + exp(-x)) : log(1. + exp(x)); - } - } - } - -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) - { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("BNLLForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - - return true; + // https://github.com/BVLC/caffe/blame/1.0/src/caffe/layers/bnll_layer.cpp#L17 + return x > 0 ? x + log(1.f + exp(-x)) : log(1.f + exp(x)); } -#endif #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) @@ -1030,23 +853,12 @@ struct BNLLFunctor : public BaseFunctor } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 - -#ifdef HAVE_DNN_NGRAPH - std::shared_ptr initNgraphAPI(const std::shared_ptr& node) - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_NGRAPH - int64 getFLOPSPerElement() const { return 5; } }; +template<> +const char* const BNLLFunctor::BaseDefaultFunctor::ocl_kernel_name = "BNLLForward"; + struct PowerFunctor : public BaseFunctor { typedef PowerLayer Layer; @@ -1206,7 +1018,7 @@ struct PowerFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return power == 1 ? 2 : 10; } }; -struct ExpFunctor : public BaseFunctor +struct ExpFunctor : public BaseDefaultFunctor { typedef ExpLayer Layer; float base, scale, shift; @@ -1232,47 +1044,16 @@ struct ExpFunctor : public BaseFunctor backendId == DNN_BACKEND_HALIDE || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + inline float calculate(float x) const { - float a = normScale, b = normShift; - for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) - { - for( int i = 0; i < len; i++ ) - { - float x = srcptr[i]; - dstptr[i] = exp(a*x + b); - } - } + return exp(normScale * x + normShift); } -#ifdef HAVE_OPENCL - bool applyOCL(InputArrayOfArrays inps, OutputArrayOfArrays outs, OutputArrayOfArrays internals) + inline void setKernelParams(ocl::Kernel& kernel) const { - std::vector inputs; - std::vector outputs; - - inps.getUMatVector(inputs); - outs.getUMatVector(outputs); - String buildopt = oclGetTMacro(inputs[0]); - - for (size_t i = 0; i < inputs.size(); i++) - { - UMat& src = inputs[i]; - UMat& dst = outputs[i]; - - ocl::Kernel kernel("ExpForward", ocl::dnn::activations_oclsrc, buildopt); - kernel.set(0, (int)src.total()); - kernel.set(1, ocl::KernelArg::PtrReadOnly(src)); - kernel.set(2, ocl::KernelArg::PtrWriteOnly(dst)); - kernel.set(3, (float)normScale); - kernel.set(4, (float)normShift); - - size_t gSize = src.total(); - CV_Assert(kernel.run(1, &gSize, NULL, false)); - } - return true; + kernel.set(3, normScale); + kernel.set(4, normShift); } -#endif #ifdef HAVE_HALIDE void attachHalide(const Halide::Expr& input, Halide::Func& top) @@ -1282,13 +1063,6 @@ struct ExpFunctor : public BaseFunctor } #endif // HAVE_HALIDE -#ifdef HAVE_DNN_IE_NN_BUILDER_2019 - InferenceEngine::Builder::Layer initInfEngineBuilderAPI() - { - CV_Error(Error::StsNotImplemented, ""); - } -#endif // HAVE_DNN_IE_NN_BUILDER_2019 - #ifdef HAVE_DNN_NGRAPH std::shared_ptr initNgraphAPI(const std::shared_ptr& node) { @@ -1305,6 +1079,9 @@ struct ExpFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return 3; } }; +template<> +const char* const ExpFunctor::BaseDefaultFunctor::ocl_kernel_name = "ExpForward"; + struct ChannelsPReLUFunctor : public BaseFunctor { typedef ChannelsPReLULayer Layer; From 7da51787b9b27495ea754ad441781f6a9e472d3e Mon Sep 17 00:00:00 2001 From: Zhuo Zhang Date: Tue, 19 Oct 2021 21:30:27 +0800 Subject: [PATCH 306/376] Merge pull request #20900 from zchrissirhcz:3.4-hwfeatures-support-qnx * fix: correctly check neon flags for QNX platform * refactor: change __QNXNTO__ to __QNX__ --- modules/core/src/system.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index 27142a403446..49e146372ee4 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -129,7 +129,7 @@ void* allocSingletonNewBuffer(size_t size) { return malloc(size); } #if defined __ANDROID__ || defined __unix__ || defined __FreeBSD__ || defined __OpenBSD__ || defined __HAIKU__ # include # include -#if defined __QNXNTO__ +#if defined __QNX__ # include #else # include @@ -545,7 +545,7 @@ struct HWFeatures } #endif // CV_CPUID_X86 - #if defined __ANDROID__ || defined __linux__ || defined __FreeBSD__ + #if defined __ANDROID__ || defined __linux__ || defined __FreeBSD__ || defined __QNX__ #ifdef __aarch64__ have[CV_CPU_NEON] = true; have[CV_CPU_FP16] = true; From f77fdc0ce86a668f8acdfd94bf4bcbd65126c6f3 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 19 Oct 2021 09:28:12 +0000 Subject: [PATCH 307/376] samples: fix build without threading support --- samples/CMakeLists.txt | 1 + samples/dnn/object_detection.cpp | 16 ++++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 87f5cfcf88ad..68afc487a298 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -114,6 +114,7 @@ else() find_package(Threads) endif() if((TARGET Threads::Threads OR HAVE_THREADS) AND NOT OPENCV_EXAMPLES_DISABLE_THREADS) + set(HAVE_THREADS 1) add_definitions(-DHAVE_THREADS=1) endif() diff --git a/samples/dnn/object_detection.cpp b/samples/dnn/object_detection.cpp index c7e42430fe4f..5d201de3b54c 100644 --- a/samples/dnn/object_detection.cpp +++ b/samples/dnn/object_detection.cpp @@ -5,7 +5,11 @@ #include #include -#ifdef CV_CXX11 +#if defined(CV_CXX11) && defined(HAVE_THREADS) +#define USE_THREADS 1 +#endif + +#ifdef USE_THREADS #include #include #include @@ -51,7 +55,7 @@ void drawPred(int classId, float conf, int left, int top, int right, int bottom, void callback(int pos, void* userdata); -#ifdef CV_CXX11 +#ifdef USE_THREADS template class QueueFPS : public std::queue { @@ -101,7 +105,7 @@ class QueueFPS : public std::queue TickMeter tm; std::mutex mutex; }; -#endif // CV_CXX11 +#endif // USE_THREADS int main(int argc, char** argv) { @@ -166,7 +170,7 @@ int main(int argc, char** argv) else cap.open(parser.get("device")); -#ifdef CV_CXX11 +#ifdef USE_THREADS bool process = true; // Frames capturing thread @@ -266,7 +270,7 @@ int main(int argc, char** argv) framesThread.join(); processingThread.join(); -#else // CV_CXX11 +#else // USE_THREADS if (async) CV_Error(Error::StsNotImplemented, "Asynchronous forward is supported only with Inference Engine backend."); @@ -297,7 +301,7 @@ int main(int argc, char** argv) imshow(kWinName, frame); } -#endif // CV_CXX11 +#endif // USE_THREADS return 0; } From 805c2832c19b4d8d281561338739ae96e014a328 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 20 Oct 2021 05:45:40 +0000 Subject: [PATCH 308/376] 4.x: drop DISABLE_OPENCV_24_COMPATIBILITY macro not used in 4.x code --- samples/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 9d8f58808301..8b1907f38054 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -110,8 +110,6 @@ if(MSVC) endif() endif() -add_definitions(-DDISABLE_OPENCV_24_COMPATIBILITY=1) # Avoid C-like legacy API - if(OPENCV_EXAMPLES_DISABLE_THREADS) # nothing elseif(MSVC OR APPLE) From 9379e85e23b2feda1317d5fec2117f1ebe0b9193 Mon Sep 17 00:00:00 2001 From: Lukas Weber Date: Wed, 20 Oct 2021 09:07:48 +0200 Subject: [PATCH 309/376] changed no longer patented SIFT --- samples/python/stitching_detailed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/python/stitching_detailed.py b/samples/python/stitching_detailed.py index 4ee29048d118..dfa88beba154 100644 --- a/samples/python/stitching_detailed.py +++ b/samples/python/stitching_detailed.py @@ -36,7 +36,7 @@ # if SURF not available, ORB is default FEATURES_FIND_CHOICES['orb'] = cv.ORB.create try: - FEATURES_FIND_CHOICES['sift'] = cv.xfeatures2d_SIFT.create + FEATURES_FIND_CHOICES['sift'] = cv.SIFT_create except AttributeError: print("SIFT not available") try: From 1f9a7b8fd30f19e42fd29d1b8102d8ac466c0414 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Wed, 20 Oct 2021 12:43:32 +0300 Subject: [PATCH 310/376] Merge pull request #20738 from sivanov-work:merge_master_vpl_dev_select G-API: oneVPL - Implement IDeviceSelector & default cfg_param-based selector * Initial commit * Add MACRO undef * Change IDeviceSelector, Change Inf sample for choose external device * Fix compilation * Address some comments * Fix compilation * Add missing header * Add EXPORT to dev selector * Add guard * Remove enum type attr * Fix compilation without VPL * Add HAVE_INFER guard in sample * Remove unusable include from tests * Remove unusable include from sample * Remove cl_d3d11 header from unit test --- modules/gapi/CMakeLists.txt | 15 + .../onevpl/device_selector_interface.hpp | 102 ++++++ .../opencv2/gapi/streaming/onevpl/source.hpp | 24 ++ .../gapi/samples/onevpl_infer_single_roi.cpp | 138 +++++++- .../onevpl/accelerators/accel_policy_dx11.hpp | 1 + .../onevpl/cfg_param_device_selector.cpp | 314 ++++++++++++++++++ .../onevpl/cfg_param_device_selector.hpp | 44 +++ .../onevpl/device_selector_interface.cpp | 87 +++++ modules/gapi/src/streaming/onevpl/source.cpp | 63 +++- .../gapi/src/streaming/onevpl/source_priv.cpp | 6 +- .../gapi/src/streaming/onevpl/source_priv.hpp | 3 +- .../gapi_streaming_vpl_device_selector.cpp | 229 +++++++++++++ 12 files changed, 1018 insertions(+), 8 deletions(-) create mode 100644 modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp create mode 100644 modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp create mode 100644 modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp create mode 100644 modules/gapi/src/streaming/onevpl/device_selector_interface.cpp create mode 100644 modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 917d1e081494..61ab5397d7ab 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -181,6 +181,9 @@ set(gapi_srcs src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp src/streaming/onevpl/engine/decode/decode_session.cpp + src/streaming/onevpl/cfg_param_device_selector.cpp + src/streaming/onevpl/device_selector_interface.cpp + # Utils (ITT tracing) src/utils/itt.cpp ) @@ -283,3 +286,15 @@ endif() ocv_add_perf_tests() ocv_add_samples() + + +# Required for sample with inference on host +if (TARGET example_gapi_onevpl_infer_single_roi) + if(OPENCV_GAPI_INF_ENGINE) + ocv_target_link_libraries(example_gapi_onevpl_infer_single_roi PRIVATE ${INF_ENGINE_TARGET}) + ocv_target_compile_definitions(example_gapi_onevpl_infer_single_roi PRIVATE -DHAVE_INF_ENGINE) + endif() + if(HAVE_D3D11 AND HAVE_OPENCL) + ocv_target_include_directories(example_gapi_onevpl_infer_single_roi SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS}) + endif() +endif() diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp new file mode 100644 index 000000000000..ca19849d7216 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp @@ -0,0 +1,102 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_DEVICE_SELECTOR_INTERFACE_HPP +#define GAPI_STREAMING_ONEVPL_DEVICE_SELECTOR_INTERFACE_HPP + +#include +#include +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +enum class AccelType : uint8_t { + HOST, + DX11, + + LAST_VALUE = std::numeric_limits::max() +}; + +GAPI_EXPORTS const char* to_cstring(AccelType type); + +struct IDeviceSelector; +struct GAPI_EXPORTS Device { + friend struct IDeviceSelector; + using Ptr = void*; + + ~Device(); + const std::string& get_name() const; + Ptr get_ptr() const; + AccelType get_type() const; +private: + Device(Ptr device_ptr, const std::string& device_name, + AccelType device_type); + + std::string name; + Ptr ptr; + AccelType type; +}; + +struct GAPI_EXPORTS Context { + friend struct IDeviceSelector; + using Ptr = void*; + + ~Context(); + Ptr get_ptr() const; + AccelType get_type() const; +private: + Context(Ptr ctx_ptr, AccelType ctx_type); + Ptr ptr; + AccelType type; +}; + +struct GAPI_EXPORTS IDeviceSelector { + using Ptr = std::shared_ptr; + + struct GAPI_EXPORTS Score { + friend struct IDeviceSelector; + using Type = int16_t; + static constexpr Type MaxActivePriority = std::numeric_limits::max(); + static constexpr Type MinActivePriority = 0; + static constexpr Type MaxPassivePriority = MinActivePriority - 1; + static constexpr Type MinPassivePriority = std::numeric_limits::min(); + + Score(Type val); + ~Score(); + + operator Type () const; + Type get() const; + friend bool operator< (Score lhs, Score rhs) { + return lhs.get() < rhs.get(); + } + private: + Type value; + }; + + using DeviceScoreTable = std::map; + using DeviceContexts = std::vector; + + virtual ~IDeviceSelector(); + virtual DeviceScoreTable select_devices() const = 0; + virtual DeviceContexts select_context() = 0; +protected: + template + static Entity create(Args &&...args) { + return Entity(std::forward(args)...); + } +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // GAPI_STREAMING_ONEVPL_DEVICE_SELECTOR_INTERFACE_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp index a8dbefdf5095..6334480c1bb7 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace cv { namespace gapi { @@ -38,8 +39,31 @@ class GAPI_EXPORTS GSource : public IStreamSource GSource(const std::string& filePath, const CfgParams& cfg_params = CfgParams{}); + + GSource(const std::string& filePath, + const CfgParams& cfg_params, + const std::string& device_id, + void* accel_device_ptr, + void* accel_ctx_ptr); + + GSource(const std::string& filePath, + const CfgParams& cfg_params, + std::shared_ptr selector); + + GSource(std::shared_ptr source, const CfgParams& cfg_params = CfgParams{}); + + GSource(std::shared_ptr source, + const CfgParams& cfg_params, + const std::string& device_id, + void* accel_device_ptr, + void* accel_ctx_ptr); + + GSource(std::shared_ptr source, + const CfgParams& cfg_params, + std::shared_ptr selector); + ~GSource() override; bool pull(cv::gapi::wip::Data& data) override; diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index f3aee09d4299..deca86f1cac8 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -10,8 +11,30 @@ #include #include #include +#include #include // CommandLineParser +#ifdef HAVE_INF_ENGINE +#include // ParamMap + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#pragma comment(lib,"d3d11.lib") + +// get rid of generate macro max/min/etc from DX side +#define D3D11_NO_HELPERS +#define NOMINMAX +#include +#include +#include +#pragma comment(lib, "dxgi") +#undef NOMINMAX +#undef D3D11_NO_HELPERS + +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#endif // HAVE_INF_ENGINE + const std::string about = "This is an OpenCV-based version of oneVPLSource decoder example"; const std::string keys = @@ -36,6 +59,37 @@ std::string get_weights_path(const std::string &model_path) { CV_Assert(ext == ".xml"); return model_path.substr(0u, sz - EXT_LEN) + ".bin"; } + +#ifdef HAVE_INF_ENGINE +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + +using AccelParamsType = std::tuple, CComPtr>; + +AccelParamsType create_device_with_ctx(CComPtr adapter) { + UINT flags = 0; + D3D_FEATURE_LEVEL feature_levels[] = { D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + D3D_FEATURE_LEVEL featureLevel; + ID3D11Device* ret_device_ptr = nullptr; + ID3D11DeviceContext* ret_ctx_ptr = nullptr; + HRESULT err = D3D11CreateDevice(adapter, D3D_DRIVER_TYPE_UNKNOWN, + nullptr, flags, + feature_levels, + ARRAYSIZE(feature_levels), + D3D11_SDK_VERSION, &ret_device_ptr, + &featureLevel, &ret_ctx_ptr); + if (FAILED(err)) { + throw std::runtime_error("Cannot create D3D11CreateDevice, error: " + + std::to_string(HRESULT_CODE(err))); + } + + return std::make_tuple(ret_device_ptr, ret_ctx_ptr); +} +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#endif // HAVE_INF_ENGINE } // anonymous namespace namespace custom { @@ -197,11 +251,84 @@ int main(int argc, char *argv[]) { return -1; } + const std::string& device_id = cmd.get("faced"); auto face_net = cv::gapi::ie::Params { face_model_path, // path to topology IR get_weights_path(face_model_path), // path to weights - cmd.get("faced"), // device specifier + device_id }; + + // Create device_ptr & context_ptr using graphic API + // InferenceEngine requires such device & context to create its own + // remote shared context through InferenceEngine::ParamMap in + // GAPI InferenceEngine backend to provide interoperability with onevpl::GSource + // So GAPI InferenceEngine backend and onevpl::GSource MUST share the same + // device and context + void* accel_device_ptr = nullptr; + void* accel_ctx_ptr = nullptr; + +#ifdef HAVE_INF_ENGINE +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + CComPtr dx11_dev; + CComPtr dx11_ctx; + + if (device_id.find("GPU") != std::string::npos) { + CComPtr adapter_factory; + CComPtr intel_adapters; + { + IDXGIFactory* out_factory = nullptr; + HRESULT err = CreateDXGIFactory(__uuidof(IDXGIFactory), + reinterpret_cast(&out_factory)); + if (FAILED(err)) { + std::cerr << "Cannot create CreateDXGIFactory, error: " << HRESULT_CODE(err) << std::endl; + return -1; + } + adapter_factory.Attach(out_factory); + } + + CComPtr intel_adapter; + UINT adapter_index = 0; + const unsigned int refIntelVendorID = 0x8086; + IDXGIAdapter* out_adapter = nullptr; + + while (adapter_factory->EnumAdapters(adapter_index, &out_adapter) != DXGI_ERROR_NOT_FOUND) { + DXGI_ADAPTER_DESC desc{}; + out_adapter->GetDesc(&desc); + if (desc.VendorId == refIntelVendorID) { + intel_adapter.Attach(out_adapter); + break; + } + ++adapter_index; + } + + if (!intel_adapter) { + std::cerr << "No Intel GPU adapter on aboard. Exit" << std::endl; + return -1; + } + + std::tie(dx11_dev, dx11_ctx) = create_device_with_ctx(intel_adapter); + accel_device_ptr = reinterpret_cast(dx11_dev.p); + accel_ctx_ptr = reinterpret_cast(dx11_ctx.p); + + // put accel type description for VPL source + source_cfgs.push_back(cfg::create_from_string( + "mfxImplDescription.AccelerationMode" + ":" + "MFX_ACCEL_MODE_VIA_D3D11")); + } + +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + // set ctx_config for GPU device only - no need in case of CPU device type + if (device_id.find("GPU") != std::string::npos) { + InferenceEngine::ParamMap ctx_config({{"CONTEXT_TYPE", "VA_SHARED"}, + {"VA_DEVICE", accel_device_ptr} }); + + face_net.cfgContextParams(ctx_config); + } +#endif // HAVE_INF_ENGINE + auto kernels = cv::gapi::kernels < custom::OCVLocateROI , custom::OCVParseSSD @@ -211,7 +338,14 @@ int main(int argc, char *argv[]) { // Create source cv::Ptr cap; try { - cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs); + if (device_id.find("GPU") != std::string::npos) { + cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs, + device_id, + accel_device_ptr, + accel_ctx_ptr); + } else { + cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs); + } std::cout << "oneVPL source desription: " << cap->descr_of() << std::endl; } catch (const std::exception& ex) { std::cerr << "Cannot create source: " << ex.what() << std::endl; diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp index a875f57085f5..946640d886d1 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp @@ -22,6 +22,7 @@ #ifdef HAVE_DIRECTX #ifdef HAVE_D3D11 #define D3D11_NO_HELPERS +#define NOMINMAX #include #include #include "opencv2/core/directx.hpp" diff --git a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp new file mode 100644 index 000000000000..c8fd49c1adcc --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp @@ -0,0 +1,314 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL +#include +#include +#include + +#include "streaming/onevpl/cfg_param_device_selector.hpp" +#include "logger.hpp" + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#pragma comment(lib,"d3d11.lib") + +// get rid of generate macro max/min/etc from DX side +#define D3D11_NO_HELPERS +#define NOMINMAX +#include +#include +#include +#pragma comment(lib, "dxgi") +#undef D3D11_NO_HELPERS +#undef NOMINMAX + +#include +#include "opencv2/core/directx.hpp" +#ifdef HAVE_OPENCL +#include +#endif + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +// TODO Will be changed on generic function from `onevpl_param_parser` as soons as feature merges +static mfxVariant cfg_param_to_mfx_variant(const CfgParam& accel_param) { + mfxVariant ret; + const CfgParam::value_t& accel_val = accel_param.get_value(); + if (!cv::util::holds_alternative(accel_val)) { + // expected string or uint32_t as value + if (!cv::util::holds_alternative(accel_val)) { + throw std::logic_error("Incorrect value type of \"mfxImplDescription.AccelerationMode\" " + " std::string is expected" ); + } + ret.Type = MFX_VARIANT_TYPE_U32; + ret.Data.U32 = cv::util::get(accel_val); + return ret; + } + + const std::string& accel_val_str = cv::util::get(accel_val); + ret.Type = MFX_VARIANT_TYPE_U32; + if (accel_val_str == "MFX_ACCEL_MODE_NA") { + ret.Data.U32 = MFX_ACCEL_MODE_NA; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_D3D9") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_D3D9; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_D3D11") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_D3D11; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_VAAPI") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_VAAPI_GLX") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI_GLX; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_VAAPI_X11") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI_X11; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_HDDLUNITE") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_HDDLUNITE; + } + return ret; +} + +CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) : + suggested_device(IDeviceSelector::create(nullptr, "CPU", AccelType::HOST)), + suggested_context(IDeviceSelector::create(nullptr, AccelType::HOST)) { + + auto accel_mode_it = + std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { + return value.get_name() == "mfxImplDescription.AccelerationMode"; + }); + if (accel_mode_it == cfg_params.end()) + { + GAPI_LOG_DEBUG(nullptr, "No HW Accel requested. Use default CPU"); + return; + } + + GAPI_LOG_DEBUG(nullptr, "Add HW acceleration support"); + mfxVariant accel_mode = cfg_param_to_mfx_variant(*accel_mode_it); + + switch(accel_mode.Data.U32) { + case MFX_ACCEL_MODE_VIA_D3D11: { +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + ID3D11Device *hw_handle = nullptr; + ID3D11DeviceContext* device_context = nullptr; + + //Create device + UINT creationFlags = 0;//D3D11_CREATE_DEVICE_BGRA_SUPPORT; + +#if defined _DEBUG || defined CV_STATIC_ANALYSIS + // If the project is in a debug build, enable debugging via SDK Layers with this flag. + creationFlags |= D3D11_CREATE_DEVICE_DEBUG; +#endif + + D3D_FEATURE_LEVEL featureLevels[] = { D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + D3D_FEATURE_LEVEL featureLevel; + + CComPtr adapter_factory; + CComPtr intel_adapters; + { + IDXGIFactory* out_factory = nullptr; + HRESULT err = CreateDXGIFactory(__uuidof(IDXGIFactory), + reinterpret_cast(&out_factory)); + if (FAILED(err)) { + throw std::runtime_error("Cannot create CreateDXGIFactory, error: " + std::to_string(HRESULT_CODE(err))); + } + adapter_factory.Attach(out_factory); + } + + CComPtr intel_adapter; + UINT adapter_index = 0; + const unsigned int refIntelVendorID = 0x8086; + IDXGIAdapter* out_adapter = nullptr; + + while (adapter_factory->EnumAdapters(adapter_index, &out_adapter) != DXGI_ERROR_NOT_FOUND) { + DXGI_ADAPTER_DESC desc{}; + out_adapter->GetDesc(&desc); + if (desc.VendorId == refIntelVendorID) { + intel_adapter.Attach(out_adapter); + break; + } + ++adapter_index; + } + + if (!intel_adapter) { + throw std::runtime_error("No Intel GPU adapter on aboard"); + } + + // Create the Direct3D 11 API device object and a corresponding context. + HRESULT err = D3D11CreateDevice(intel_adapter, + D3D_DRIVER_TYPE_UNKNOWN, + nullptr, creationFlags, + featureLevels, ARRAYSIZE(featureLevels), + D3D11_SDK_VERSION, + &hw_handle, &featureLevel, + &device_context); + if(FAILED(err)) { + throw std::logic_error("Cannot create D3D11CreateDevice, error: " + std::to_string(HRESULT_CODE(err))); + } + + // oneVPL recommendation + { + ID3D11Multithread *pD11Multithread = nullptr; + device_context->QueryInterface(IID_PPV_ARGS(&pD11Multithread)); + pD11Multithread->SetMultithreadProtected(true); + pD11Multithread->Release(); + } + + suggested_device = IDeviceSelector::create(hw_handle, "GPU", AccelType::DX11); + suggested_context = IDeviceSelector::create(device_context, AccelType::DX11); +#else + GAPI_LOG_WARNING(nullptr, "Unavailable \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\"" + "was chosen for current project configuration"); + throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\""); +#endif // HAVE_DIRECTX +#endif // HAVE_D3D11 + break; + } + case MFX_ACCEL_MODE_NA: { + // nothing to do + break; + } + default: + throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode\" requested: " + + std::to_string(accel_mode.Data.U32)); + break; + } +} + +CfgParamDeviceSelector::CfgParamDeviceSelector(Device::Ptr device_ptr, + const std::string& device_id, + Context::Ptr ctx_ptr, + const CfgParams& cfg_params) : + suggested_device(IDeviceSelector::create(nullptr, "CPU", AccelType::HOST)), + suggested_context(IDeviceSelector::create(nullptr, AccelType::HOST)) { + auto accel_mode_it = + std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { + return value.get_name() == "mfxImplDescription.AccelerationMode"; + }); + if (accel_mode_it == cfg_params.end()) { + GAPI_LOG_WARNING(nullptr, "Cannot deternime \"device_ptr\" type. " + "Make sure a param \"mfxImplDescription.AccelerationMode\" " + "presents in configurations and has correct value according to " + "\"device_ptr\" type"); + throw std::logic_error("Missing \"mfxImplDescription.AccelerationMode\" param"); + } + + GAPI_LOG_DEBUG(nullptr, "Turn on HW acceleration support for device: " << + device_ptr << + ", context: " << ctx_ptr); + if (!device_ptr) { + GAPI_LOG_WARNING(nullptr, "Empty \"device_ptr\" is not allowed when " + "param \"mfxImplDescription.AccelerationMode\" existed"); + throw std::logic_error("Invalid param: \"device_ptr\""); + } + + if (!ctx_ptr) { + GAPI_LOG_WARNING(nullptr, "Empty \"ctx_ptr\" is not allowed"); + throw std::logic_error("Invalid param: \"ctx_ptr\""); + } + mfxVariant accel_mode = cfg_param_to_mfx_variant(*accel_mode_it); + + switch(accel_mode.Data.U32) { + case MFX_ACCEL_MODE_VIA_D3D11: { +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + suggested_device = IDeviceSelector::create(device_ptr, device_id, AccelType::DX11); + ID3D11Device* dx_device_ptr = + reinterpret_cast(suggested_device.get_ptr()); + dx_device_ptr->AddRef(); + + suggested_context = IDeviceSelector::create(ctx_ptr, AccelType::DX11); + ID3D11DeviceContext* dx_ctx_ptr = + reinterpret_cast(suggested_context.get_ptr()); + dx_ctx_ptr->AddRef(); +#else + GAPI_LOG_WARNING(nullptr, "Unavailable \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\"" + "was chosen for current project configuration"); + throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\""); +#endif // HAVE_DIRECTX +#endif // HAVE_D3D11 + break; + } + case MFX_ACCEL_MODE_NA: { + GAPI_LOG_WARNING(nullptr, "Incompatible \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_NA\" with " + "\"device_ptr\" and \"ctx_ptr\" arguments. " + "You should not clarify these arguments with \"MFX_ACCEL_MODE_NA\" mode"); + throw std::logic_error("Incompatible param: MFX_ACCEL_MODE_NA"); + } + default: + throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode\" requested: " + + std::to_string(accel_mode.Data.U32)); + break; + } +} + +CfgParamDeviceSelector::~CfgParamDeviceSelector() { + GAPI_LOG_INFO(nullptr, "release context: " << suggested_context.get_ptr()); + AccelType ctype = suggested_context.get_type(); + switch(ctype) { + case AccelType::HOST: + //nothing to do + break; + case AccelType::DX11: { +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + ID3D11DeviceContext* device_ctx_ptr = + reinterpret_cast(suggested_context.get_ptr()); + device_ctx_ptr->Release(); + device_ctx_ptr = nullptr; +#endif // HAVE_DIRECTX +#endif // HAVE_D3D11 + break; + } + default: + break; + } + + GAPI_LOG_INFO(nullptr, "release device by name: " << + suggested_device.get_name() << + ", ptr: " << suggested_device.get_ptr()); + AccelType dtype = suggested_device.get_type(); + switch(dtype) { + case AccelType::HOST: + //nothing to do + break; + case AccelType::DX11: { +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + ID3D11Device* device_ptr = reinterpret_cast(suggested_device.get_ptr()); + device_ptr->Release(); + device_ptr = nullptr; +#endif // HAVE_DIRECTX +#endif // HAVE_D3D11 + break; + } + default: + break; + } +} + +CfgParamDeviceSelector::DeviceScoreTable CfgParamDeviceSelector::select_devices() const { + return {std::make_pair(Score::MaxActivePriority, suggested_device)}; +} + +CfgParamDeviceSelector::DeviceContexts CfgParamDeviceSelector::select_context() { + return {suggested_context}; +} + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp new file mode 100644 index 000000000000..2a55fb09cf53 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp @@ -0,0 +1,44 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_CFG_PARAM_DEVICE_SELECTOR_HPP +#define GAPI_STREAMING_ONEVPL_CFG_PARAM_DEVICE_SELECTOR_HPP + +#ifdef HAVE_ONEVPL + +#include +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +struct GAPI_EXPORTS CfgParamDeviceSelector final: public IDeviceSelector { + CfgParamDeviceSelector(const CfgParams& params = {}); + CfgParamDeviceSelector(Device::Ptr device_ptr, + const std::string& device_id, + Context::Ptr ctx_ptr, + const CfgParams& params); + ~CfgParamDeviceSelector(); + + DeviceScoreTable select_devices() const override; + DeviceContexts select_context() override; + +private: + Device suggested_device; + Context suggested_context; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif //HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_CFG_PARAM_DEVICE_SELECTOR_HPP diff --git a/modules/gapi/src/streaming/onevpl/device_selector_interface.cpp b/modules/gapi/src/streaming/onevpl/device_selector_interface.cpp new file mode 100644 index 000000000000..1ac88bd80772 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/device_selector_interface.cpp @@ -0,0 +1,87 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +const char* to_cstring(AccelType type) { + + switch(type) { + case AccelType::HOST: + return "HOST"; + case AccelType::DX11: + return "DX11"; + default: + GAPI_DbgAssert(false && "Unexpected AccelType"); + break; + } + return "UNKNOWN"; +} + +Device::Device(Ptr device_ptr, const std::string& device_name, AccelType device_type) : + name(device_name), + ptr(device_ptr), + type(device_type) { +} + +Device::~Device() { +} + +const std::string& Device::get_name() const { + return name; +} + +Device::Ptr Device::get_ptr() const { + return ptr; +} + +AccelType Device::get_type() const { + return type; +} + +Context::Context(Ptr ctx_ptr, AccelType ctx_type) : + ptr(ctx_ptr), + type(ctx_type) { +} + +Context::~Context() { +} + +Context::Ptr Context::get_ptr() const { + return ptr; +} + +AccelType Context::get_type() const { + return type; +} + +IDeviceSelector::Score::Score(Type val) : + value(val) { +} + +IDeviceSelector::Score::~Score() { +} + +IDeviceSelector::Score::operator Type () const { + return value; +} +IDeviceSelector::Score::Type IDeviceSelector::Score::get() const { + return value; +} + +IDeviceSelector::~IDeviceSelector() { +} + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/source.cpp b/modules/gapi/src/streaming/onevpl/source.cpp index 0decb1358b7b..806017a90dd2 100644 --- a/modules/gapi/src/streaming/onevpl/source.cpp +++ b/modules/gapi/src/streaming/onevpl/source.cpp @@ -8,6 +8,8 @@ #include "streaming/onevpl/source_priv.hpp" #include "streaming/onevpl/file_data_provider.hpp" +#include "streaming/onevpl/cfg_param_device_selector.hpp" + namespace cv { namespace gapi { namespace wip { @@ -15,27 +17,82 @@ namespace onevpl { #ifdef HAVE_ONEVPL GSource::GSource(const std::string& filePath, const CfgParams& cfg_params) : - GSource(std::unique_ptr(new GSource::Priv(std::make_shared(filePath), - cfg_params))) { + GSource(filePath, cfg_params, std::make_shared(cfg_params)) { + if (filePath.empty()) { + util::throw_error(std::logic_error("Cannot create 'GSource' on empty source file name")); + } +} + +GSource::GSource(const std::string& filePath, + const CfgParams& cfg_params, + const std::string& device_id, + void* accel_device_ptr, + void* accel_ctx_ptr) : + GSource(filePath, cfg_params, + std::make_shared(accel_device_ptr, device_id, + accel_ctx_ptr, cfg_params)) { +} +GSource::GSource(const std::string& filePath, + const CfgParams& cfg_params, + std::shared_ptr selector) : + GSource(std::make_shared(filePath), cfg_params, selector) { if (filePath.empty()) { util::throw_error(std::logic_error("Cannot create 'GSource' on empty source file name")); } } GSource::GSource(std::shared_ptr source, const CfgParams& cfg_params) : - GSource(std::unique_ptr(new GSource::Priv(source, cfg_params))) { + GSource(source, cfg_params, + std::make_shared(cfg_params)) { +} + +GSource::GSource(std::shared_ptr source, + const CfgParams& cfg_params, + const std::string& device_id, + void* accel_device_ptr, + void* accel_ctx_ptr) : + GSource(source, cfg_params, + std::make_shared(accel_device_ptr, device_id, + accel_ctx_ptr, cfg_params)) { } + +// common delegating parameters c-tor +GSource::GSource(std::shared_ptr source, + const CfgParams& cfg_params, + std::shared_ptr selector) : + GSource(std::unique_ptr(new GSource::Priv(source, cfg_params, selector))) { +} + #else GSource::GSource(const std::string&, const CfgParams&) { GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); } +GSource::GSource(const std::string&, const CfgParams&, const std::string&, + void*, void*) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} + +GSource::GSource(const std::string&, const CfgParams&, std::shared_ptr) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} + GSource::GSource(std::shared_ptr, const CfgParams&) { GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); } + +GSource::GSource(std::shared_ptr, const CfgParams&, + const std::string&, void*, void*) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} + +GSource::GSource(std::shared_ptr, const CfgParams&, std::shared_ptr) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} #endif +// final delegating c-tor GSource::GSource(std::unique_ptr&& impl) : IStreamSource(), m_priv(std::move(impl)) { diff --git a/modules/gapi/src/streaming/onevpl/source_priv.cpp b/modules/gapi/src/streaming/onevpl/source_priv.cpp index 28d438a947c4..d00074925d42 100644 --- a/modules/gapi/src/streaming/onevpl/source_priv.cpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.cpp @@ -58,8 +58,10 @@ GSource::Priv::Priv() : GAPI_LOG_INFO(nullptr, "Initialized MFX handle: " << mfx_handle); } -GSource::Priv::Priv(std::shared_ptr provider, const std::vector& params) : - GSource::Priv() +GSource::Priv::Priv(std::shared_ptr provider, + const std::vector& params, + std::shared_ptr) : + GSource::Priv() { // Enable Config if (params.empty()) diff --git a/modules/gapi/src/streaming/onevpl/source_priv.hpp b/modules/gapi/src/streaming/onevpl/source_priv.hpp index cdaab4eb6a45..955184c05c88 100644 --- a/modules/gapi/src/streaming/onevpl/source_priv.hpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.hpp @@ -38,7 +38,8 @@ class ProcessingEngineBase; struct GSource::Priv { explicit Priv(std::shared_ptr provider, - const std::vector& params); + const std::vector& params, + std::shared_ptr selector); ~Priv(); static const std::vector& getDefaultCfgParams(); diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp new file mode 100644 index 000000000000..2f42742b88af --- /dev/null +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp @@ -0,0 +1,229 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + + +#include "../test_precomp.hpp" + +#include "../common/gapi_tests_common.hpp" + +#include +#include + +#include + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#pragma comment(lib,"d3d11.lib") + +// get rid of generate macro max/min/etc from DX side +#define D3D11_NO_HELPERS +#define NOMINMAX +#include +#include +#include "opencv2/core/directx.hpp" +#undef D3D11_NO_HELPERS +#undef NOMINMAX +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + +#ifdef HAVE_ONEVPL +#include +#include "streaming/onevpl/cfg_param_device_selector.hpp" + +namespace opencv_test +{ +namespace +{ + +void test_dev_eq(const typename cv::gapi::wip::onevpl::IDeviceSelector::DeviceScoreTable::value_type &scored_device, + cv::gapi::wip::onevpl::IDeviceSelector::Score expected_score, + cv::gapi::wip::onevpl::AccelType expected_type, + cv::gapi::wip::onevpl::Device::Ptr expected_ptr) { + EXPECT_EQ(std::get<0>(scored_device), expected_score); + EXPECT_EQ(std::get<1>(scored_device).get_type(), expected_type); + EXPECT_EQ(std::get<1>(scored_device).get_ptr(), expected_ptr); +} + +void test_ctx_eq(const typename cv::gapi::wip::onevpl::IDeviceSelector::DeviceContexts::value_type &ctx, + cv::gapi::wip::onevpl::AccelType expected_type, + cv::gapi::wip::onevpl::Context::Ptr expected_ptr) { + EXPECT_EQ(ctx.get_type(), expected_type); + EXPECT_EQ(ctx.get_ptr(), expected_ptr); +} + +void test_host_dev_eq(const typename cv::gapi::wip::onevpl::IDeviceSelector::DeviceScoreTable::value_type &scored_device, + cv::gapi::wip::onevpl::IDeviceSelector::Score expected_score) { + test_dev_eq(scored_device, expected_score, + cv::gapi::wip::onevpl::AccelType::HOST, nullptr); +} + +void test_host_ctx_eq(const typename cv::gapi::wip::onevpl::IDeviceSelector::DeviceContexts::value_type &ctx) { + test_ctx_eq(ctx, cv::gapi::wip::onevpl::AccelType::HOST, nullptr); +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDevice) +{ + using namespace cv::gapi::wip::onevpl; + CfgParamDeviceSelector selector; + IDeviceSelector::DeviceScoreTable devs = selector.select_devices(); + EXPECT_EQ(devs.size(), 1); + test_host_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority); + + IDeviceSelector::DeviceContexts ctxs = selector.select_context(); + EXPECT_EQ(ctxs.size(), 1); + test_host_ctx_eq(*ctxs.begin()); +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDeviceWithEmptyCfgParam) +{ + using namespace cv::gapi::wip::onevpl; + std::vector empty_params; + CfgParamDeviceSelector selector(empty_params); + IDeviceSelector::DeviceScoreTable devs = selector.select_devices(); + EXPECT_EQ(devs.size(), 1); + test_host_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority); + IDeviceSelector::DeviceContexts ctxs = selector.select_context(); + EXPECT_EQ(ctxs.size(), 1); + test_host_ctx_eq(*ctxs.begin()); +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDeviceWithAccelNACfgParam) +{ + using namespace cv::gapi::wip::onevpl; + std::vector cfg_params_w_no_accel; + cfg_params_w_no_accel.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_NA)); + CfgParamDeviceSelector selector(cfg_params_w_no_accel); + IDeviceSelector::DeviceScoreTable devs = selector.select_devices(); + EXPECT_EQ(devs.size(), 1); + test_host_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority); + + IDeviceSelector::DeviceContexts ctxs = selector.select_context(); + EXPECT_EQ(ctxs.size(), 1); + test_host_ctx_eq(*ctxs.begin()); +} + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDeviceWithEmptyCfgParam_DX11_ENABLED) +{ + using namespace cv::gapi::wip::onevpl; + std::vector empty_params; + CfgParamDeviceSelector selector(empty_params); + IDeviceSelector::DeviceScoreTable devs = selector.select_devices(); + EXPECT_EQ(devs.size(), 1); + test_host_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority); + + IDeviceSelector::DeviceContexts ctxs = selector.select_context(); + EXPECT_EQ(ctxs.size(), 1); + test_host_ctx_eq(*ctxs.begin()); +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDeviceWithDX11AccelCfgParam_DX11_ENABLED) +{ + using namespace cv::gapi::wip::onevpl; + std::vector cfg_params_w_dx11; + cfg_params_w_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_VIA_D3D11)); + std::unique_ptr selector_ptr; + EXPECT_NO_THROW(selector_ptr.reset(new CfgParamDeviceSelector(cfg_params_w_dx11))); + IDeviceSelector::DeviceScoreTable devs = selector_ptr->select_devices(); + + EXPECT_EQ(devs.size(), 1); + test_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority, + AccelType::DX11, + std::get<1>(*devs.begin()).get_ptr() /* compare just type */); + + IDeviceSelector::DeviceContexts ctxs = selector_ptr->select_context(); + EXPECT_EQ(ctxs.size(), 1); + EXPECT_TRUE(ctxs.begin()->get_ptr()); +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, NULLDeviceWithDX11AccelCfgParam_DX11_ENABLED) +{ + using namespace cv::gapi::wip::onevpl; + std::vector cfg_params_w_dx11; + cfg_params_w_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_VIA_D3D11)); + Device::Ptr empty_device_ptr = nullptr; + Context::Ptr empty_ctx_ptr = nullptr; + EXPECT_THROW(CfgParamDeviceSelector sel(empty_device_ptr, "GPU", + empty_ctx_ptr, + cfg_params_w_dx11), + std::logic_error); // empty_device_ptr must be invalid +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, ExternalDeviceWithDX11AccelCfgParam_DX11_ENABLED) +{ + using namespace cv::gapi::wip::onevpl; + ID3D11Device *device = nullptr; + ID3D11DeviceContext* device_context = nullptr; + { + UINT flags = 0; + D3D_FEATURE_LEVEL features[] = { D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + D3D_FEATURE_LEVEL feature_level; + + // Create the Direct3D 11 API device object and a corresponding context. + HRESULT err = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, + nullptr, flags, + features, + ARRAYSIZE(features), D3D11_SDK_VERSION, + &device, &feature_level, &device_context); + EXPECT_FALSE(FAILED(err)); + } + + std::unique_ptr selector_ptr; + std::vector cfg_params_w_dx11; + cfg_params_w_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_VIA_D3D11)); + EXPECT_NO_THROW(selector_ptr.reset(new CfgParamDeviceSelector(device, "GPU", + device_context, + cfg_params_w_dx11))); + IDeviceSelector::DeviceScoreTable devs = selector_ptr->select_devices(); + + EXPECT_EQ(devs.size(), 1); + test_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority, + AccelType::DX11, device); + + IDeviceSelector::DeviceContexts ctxs = selector_ptr->select_context(); + EXPECT_EQ(ctxs.size(), 1); + EXPECT_EQ(reinterpret_cast(ctxs.begin()->get_ptr()), + device_context); +} + +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + +#ifndef HAVE_DIRECTX +#ifndef HAVE_D3D11 +TEST(OneVPL_Source_Device_Selector_CfgParam, DX11DeviceFromCfgParamWithDX11Disabled) +{ + using namespace cv::gapi::wip::onevpl; + std::vector cfg_params_w_non_existed_dx11; + cfg_params_w_not_existed_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_VIA_D3D11)); + EXPECT_THROW(CfgParamDeviceSelector{cfg_params_w_non_existed_dx11}, + std::logic_error); +} +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + +TEST(OneVPL_Source_Device_Selector_CfgParam, UnknownPtrDeviceFromCfgParam) +{ + using namespace cv::gapi::wip::onevpl; + std::vector empty_params; + Device::Ptr empty_device_ptr = nullptr; + Context::Ptr empty_ctx_ptr = nullptr; + EXPECT_THROW(CfgParamDeviceSelector sel(empty_device_ptr, "", + empty_ctx_ptr, + empty_params), + std::logic_error); // params must describe device_ptr explicitly +} +} +} // namespace opencv_test +#endif // HAVE_ONEVPL From f36c268b9ed70e4a7051084350c35a98de5ff1de Mon Sep 17 00:00:00 2001 From: MaximMilashchenko <67949029+MaximMilashchenko@users.noreply.github.com> Date: Wed, 20 Oct 2021 16:18:24 +0300 Subject: [PATCH 311/376] Merge pull request #19721 from MaximMilashchenko:Audio add audio support in cap_msmf * audio msmf * fixed warnings * minor fix * fixed SampleTime MSMF * minor fix, fixed audio test, retrieveAudioFrame * fixed warnings * impelemented sync audio and video stream with start offset * fixed error * fixed docs * fixed audio sample * CAP_PROP_AUDIO_POS, minor fixed * fixed warnings * videoio(MSMF): update audio test checks, add debug logging * fixed * fixed desynchronization of time positions, warnings * fixed warnings * videoio(audio): tune tests checks * videoio(audio): update properties description * build warnings Co-authored-by: Alexander Alekhin --- modules/videoio/include/opencv2/videoio.hpp | 11 + modules/videoio/src/cap_msmf.cpp | 986 +++++++++++++++--- modules/videoio/test/test_audio.cpp | 273 +++++ modules/videoio/test/test_microphone.cpp | 41 + samples/cpp/videocapture_audio.cpp | 59 ++ .../cpp/videocapture_audio_combination.cpp | 69 ++ samples/cpp/videocapture_microphone.cpp | 57 + 7 files changed, 1360 insertions(+), 136 deletions(-) create mode 100644 modules/videoio/test/test_audio.cpp create mode 100644 modules/videoio/test/test_microphone.cpp create mode 100644 samples/cpp/videocapture_audio.cpp create mode 100644 samples/cpp/videocapture_audio_combination.cpp create mode 100644 samples/cpp/videocapture_microphone.cpp diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 3276d0d5e4ae..4b5bc135bc56 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -189,6 +189,17 @@ enum VideoCaptureProperties { CAP_PROP_OPEN_TIMEOUT_MSEC=53, //!< (**open-only**) timeout in milliseconds for opening a video capture (applicable for FFmpeg back-end only) CAP_PROP_READ_TIMEOUT_MSEC=54, //!< (**open-only**) timeout in milliseconds for reading from a video capture (applicable for FFmpeg back-end only) CAP_PROP_STREAM_OPEN_TIME_USEC =55, // #include #include +#include #include #include #include @@ -69,7 +70,6 @@ static void init_MFCreateDXGIDeviceManager() #endif #include - #include #include // QISearch @@ -108,6 +108,13 @@ class ComPtr { } + void swap(_In_ ComPtr& lp) + { + ComPtr tmp(p); + p = lp.p; + lp.p = tmp.p; + tmp = NULL; + } T** operator&() { CV_Assert(p == NULL); @@ -155,6 +162,7 @@ template inline T absDiff(T a, T b) { return a >= b ? a - b : b - a // Structure for collecting info about types of video which are supported by current video device struct MediaType { + //video param UINT32 width; UINT32 height; INT32 stride; // stride is negative if image is bottom-up @@ -165,9 +173,17 @@ struct MediaType UINT32 aspectRatioDenom; UINT32 sampleSize; UINT32 interlaceMode; + //audio param + UINT32 bit_per_sample; + UINT32 nChannels; + UINT32 nAvgBytesPerSec; + UINT32 nSamplesPerSec; + GUID majorType; // video or audio GUID subType; // fourCC + _ComPtr Type; MediaType(IMFMediaType *pType = 0) : + Type(pType), width(0), height(0), stride(0), isFixedSize(true), @@ -175,23 +191,38 @@ struct MediaType aspectRatioNum(1), aspectRatioDenom(1), sampleSize(0), interlaceMode(0), - majorType(MFMediaType_Video), + bit_per_sample(0), + nChannels(0), + nAvgBytesPerSec(0), + nSamplesPerSec(0), + majorType({ 0 }),//MFMediaType_Video subType({ 0 }) { if (pType) { - MFGetAttributeSize(pType, MF_MT_FRAME_SIZE, &width, &height); - pType->GetUINT32(MF_MT_DEFAULT_STRIDE, (UINT32*)&stride); // value is stored as UINT32 but should be casted to INT3) - pType->GetUINT32(MF_MT_FIXED_SIZE_SAMPLES, &isFixedSize); - MFGetAttributeRatio(pType, MF_MT_FRAME_RATE, &frameRateNum, &frameRateDenom); - MFGetAttributeRatio(pType, MF_MT_PIXEL_ASPECT_RATIO, &aspectRatioNum, &aspectRatioDenom); - pType->GetUINT32(MF_MT_SAMPLE_SIZE, &sampleSize); - pType->GetUINT32(MF_MT_INTERLACE_MODE, &interlaceMode); pType->GetGUID(MF_MT_MAJOR_TYPE, &majorType); pType->GetGUID(MF_MT_SUBTYPE, &subType); + if (majorType == MFMediaType_Audio) + { + pType->GetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, &bit_per_sample); + pType->GetUINT32(MF_MT_AUDIO_NUM_CHANNELS, &nChannels); + pType->GetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, &nAvgBytesPerSec); + pType->GetUINT32(MF_MT_AUDIO_FLOAT_SAMPLES_PER_SECOND, &nSamplesPerSec); + } + else if (majorType == MFMediaType_Video) + { + MFGetAttributeSize(pType, MF_MT_FRAME_SIZE, &width, &height); + pType->GetUINT32(MF_MT_DEFAULT_STRIDE, (UINT32*)&stride); // value is stored as UINT32 but should be casted to INT3) + pType->GetUINT32(MF_MT_FIXED_SIZE_SAMPLES, &isFixedSize); + MFGetAttributeRatio(pType, MF_MT_FRAME_RATE, &frameRateNum, &frameRateDenom); + MFGetAttributeRatio(pType, MF_MT_PIXEL_ASPECT_RATIO, &aspectRatioNum, &aspectRatioDenom); + pType->GetUINT32(MF_MT_SAMPLE_SIZE, &sampleSize); + pType->GetUINT32(MF_MT_INTERLACE_MODE, &interlaceMode); + pType->GetUINT32(MF_MT_INTERLACE_MODE, &interlaceMode); + } } } - static MediaType createDefault() + static MediaType createDefault_Video() { MediaType res; res.width = 640; @@ -199,11 +230,24 @@ struct MediaType res.setFramerate(30.0); return res; } - inline bool isEmpty() const + static MediaType createDefault_Audio() + { + MediaType res; + res.majorType = MFMediaType_Audio; + res.subType = MFAudioFormat_PCM; + res.bit_per_sample = 16; + res.nChannels = 1; + res.nSamplesPerSec = 44100; + return res; + } + inline bool isEmpty(bool flag = false) const { - return width == 0 && height == 0; + if (!flag) + return width == 0 && height == 0; + else + return nChannels == 0; } - _ComPtr createMediaType() const + _ComPtr createMediaType_Video() const { _ComPtr res; MFCreateMediaType(&res); @@ -225,6 +269,22 @@ struct MediaType res->SetGUID(MF_MT_SUBTYPE, subType); return res; } + _ComPtr createMediaType_Audio() const + { + _ComPtr res; + MFCreateMediaType(&res); + if (majorType != GUID()) + res->SetGUID(MF_MT_MAJOR_TYPE, majorType); + if (subType != GUID()) + res->SetGUID(MF_MT_SUBTYPE, subType); + if (bit_per_sample != 0) + res->SetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, bit_per_sample); + if (nChannels != 0) + res->SetUINT32(MF_MT_AUDIO_NUM_CHANNELS, nChannels); + if (nSamplesPerSec != 0) + res->SetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, nSamplesPerSec); + return res; + } void setFramerate(double fps) { frameRateNum = (UINT32)cvRound(fps * 1000.0); @@ -246,7 +306,7 @@ struct MediaType return wdiff + hdiff; } // check if 'this' is better than 'other' comparing to reference - bool isBetterThan(const MediaType& other, const MediaType& ref) const + bool VideoIsBetterThan(const MediaType& other, const MediaType& ref) const { const unsigned long thisDiff = resolutionDiff(ref); const unsigned long otherDiff = other.resolutionDiff(ref); @@ -268,6 +328,24 @@ struct MediaType } return false; } + bool AudioIsBetterThan(const MediaType& other, const MediaType& ref) const + { + double thisDiff = absDiff(nChannels, ref.nChannels); + double otherDiff = absDiff(other.nChannels, ref.nChannels); + if (otherDiff < thisDiff) + { + thisDiff = absDiff(bit_per_sample, ref.bit_per_sample); + otherDiff = absDiff(bit_per_sample, ref.bit_per_sample); + if (otherDiff < thisDiff) + { + thisDiff = absDiff(nSamplesPerSec, ref.nSamplesPerSec); + otherDiff = absDiff(nSamplesPerSec, ref.nSamplesPerSec); + if (otherDiff < thisDiff) + return true; + } + } + return false; + } }; void printFormat(std::ostream& out, const GUID& fmt) @@ -405,7 +483,7 @@ class SourceReaderCB : public IMFSourceReaderCallback return S_OK; } - HRESULT Wait(DWORD dwMilliseconds, _ComPtr& videoSample, BOOL& pbEOS) + HRESULT Wait(DWORD dwMilliseconds, _ComPtr& mediaSample, BOOL& pbEOS) { pbEOS = FALSE; @@ -423,14 +501,14 @@ class SourceReaderCB : public IMFSourceReaderCallback if (!pbEOS) { cv::AutoLock lock(m_mutex); - videoSample = m_lastSample; - CV_Assert(videoSample); + mediaSample = m_lastSample; + CV_Assert(mediaSample); m_lastSample.Release(); ResetEvent(m_hEvent); // event is auto-reset, but we need this forced reset due time gap between wait() and mutex hold. } - return m_hrStatus; } + private: // Destructor is private. Caller should call Release. virtual ~SourceReaderCB() @@ -496,22 +574,67 @@ class FormatStorage } } } + void countNumberOfAudioStreams(DWORD &numberOfAudioStreams) + { + std::pair best; + std::map::const_iterator i = formats.begin(); + for (; i != formats.end(); ++i) + { + if(i->second.majorType == MFMediaType_Audio) + { + if(best.second.isEmpty() || i->first.stream != best.first.stream) + { + numberOfAudioStreams++; + best = *i; + } + } + } + } std::pair findBestVideoFormat(const MediaType& newType) { std::pair best; std::map::const_iterator i = formats.begin(); for (; i != formats.end(); ++i) { - if (i->second.majorType != MFMediaType_Video) - continue; - if (newType.isEmpty()) // file input - choose first returned media type + if (i->second.majorType == MFMediaType_Video) { - best = *i; - break; + if (best.second.isEmpty() || i->second.VideoIsBetterThan(best.second, newType)) + { + best = *i; + } } - if (best.second.isEmpty() || i->second.isBetterThan(best.second, newType)) + } + return best; + } + std::pair findBestAudioFormat(const MediaType& newType) + { + std::pair best; + std::map::const_iterator i = formats.begin(); + best = *i; + for (; i != formats.end(); ++i) + { + if (i->second.majorType == MFMediaType_Audio) + { + if ( i->second.AudioIsBetterThan(best.second, newType)) + { + best = *i; + } + } + } + return best; + } + std::pair findAudioFormatByStream(const DWORD StreamIndex) + { + std::pair best; + std::map::const_iterator i = formats.begin(); + for (; i != formats.end(); ++i) + { + if (i->second.majorType == MFMediaType_Audio) { - best = *i; + if ((*i).first.stream == StreamIndex) + { + best = *i; + } } } return best; @@ -586,21 +709,30 @@ class CvCapture_MSMF : public cv::IVideoCapture virtual void close(); virtual double getProperty(int) const CV_OVERRIDE; virtual bool setProperty(int, double) CV_OVERRIDE; + bool grabAudioFrame(); + bool grabVideoFrame(); virtual bool grabFrame() CV_OVERRIDE; + bool retrieveAudioFrame(int, OutputArray); + bool retrieveVideoFrame(OutputArray); virtual bool retrieveFrame(int, cv::OutputArray) CV_OVERRIDE; virtual bool isOpened() const CV_OVERRIDE { return isOpen; } virtual int getCaptureDomain() CV_OVERRIDE { return CV_CAP_MSMF; } protected: - bool configureOutput(MediaType newType, cv::uint32_t outFormat); + bool configureOutput(); + bool configureAudioOutput(MediaType newType); + bool configureVideoOutput(MediaType newType, cv::uint32_t outFormat); bool setTime(double time, bool rough); + bool setTime(int numberFrame); bool configureHW(bool enable); + bool configureStreams(const cv::VideoCaptureParameters&); + bool setAudioProperties(const cv::VideoCaptureParameters&); template bool readComplexPropery(long prop, long& val) const; template bool writeComplexProperty(long prop, double val, long flags); _ComPtr getDefaultSourceConfig(UINT32 num = 10); - bool initStream(DWORD streamID, const MediaType& mt); + bool initStream(DWORD streamID, const MediaType mt); bool openFinalize_(const VideoCaptureParameters* params); @@ -615,17 +747,49 @@ class CvCapture_MSMF : public cv::IVideoCapture _ComPtr D3DMgr; #endif _ComPtr videoFileSource; - _ComPtr videoSample; _ComPtr readCallback; // non-NULL for "live" streams (camera capture) - DWORD dwStreamIndex; + std::vector dwStreamIndices; + std::vector<_ComPtr> audioSamples; + _ComPtr impendingVideoSample; + _ComPtr usedVideoSample; + DWORD dwVideoStreamIndex; + DWORD dwAudioStreamIndex; MediaType nativeFormat; - MediaType captureFormat; - int outputFormat; + MediaType captureVideoFormat; + MediaType captureAudioFormat; + bool device_status; //on or off + int videoStream; // look at CAP_PROP_VIDEO_STREAM + int audioStream; // look at CAP_PROP_AUDIO_STREAM + bool vEOS; + bool aEOS; + unsigned int audioBaseIndex; + int outputVideoFormat; + int outputAudioFormat; bool convertFormat; MFTIME duration; LONGLONG frameStep; - LONGLONG sampleTime; + LONGLONG nFrame; + LONGLONG impendingVideoSampleTime; + LONGLONG usedVideoSampleTime; + LONGLONG videoStartOffset; + LONGLONG videoSampleDuration; + LONGLONG requiredAudioTime; + LONGLONG audioSampleTime; + LONGLONG audioStartOffset; + LONGLONG audioSampleDuration; + LONGLONG audioTime; + LONGLONG chunkLengthOfBytes; + LONGLONG givenAudioTime; + LONGLONG numberOfAdditionalAudioBytes; // the number of additional bytes required to align the audio chunk + double bufferedAudioDuration; + LONGLONG audioSamplePos; + DWORD numberOfAudioStreams; + Mat audioFrame; + std::deque bufferAudioData; bool isOpen; + bool grabIsDone; + bool syncLastFrame; + bool lastFrame; }; CvCapture_MSMF::CvCapture_MSMF(): @@ -640,15 +804,42 @@ CvCapture_MSMF::CvCapture_MSMF(): D3DMgr(NULL), #endif videoFileSource(NULL), - videoSample(NULL), readCallback(NULL), - dwStreamIndex(0), - outputFormat(CV_CAP_MODE_BGR), + impendingVideoSample(NULL), + usedVideoSample(NULL), + dwVideoStreamIndex(0), + dwAudioStreamIndex(0), + device_status(false), + videoStream(0), + audioStream(-1), + vEOS(false), + aEOS(false), + audioBaseIndex(1), + outputVideoFormat(CV_CAP_MODE_BGR), + outputAudioFormat(CV_16S), convertFormat(true), duration(0), frameStep(0), - sampleTime(0), - isOpen(false) + nFrame(0), + impendingVideoSampleTime(0), + usedVideoSampleTime(0), + videoStartOffset(-1), + videoSampleDuration(0), + requiredAudioTime(0), + audioSampleTime(0), + audioStartOffset(-1), + audioSampleDuration(0), + audioTime(0), + chunkLengthOfBytes(0), + givenAudioTime(0), + numberOfAdditionalAudioBytes(0), + bufferedAudioDuration(0), + audioSamplePos(0), + numberOfAudioStreams(0), + isOpen(false), + grabIsDone(false), + syncLastFrame(true), + lastFrame(false) { } @@ -663,29 +854,37 @@ void CvCapture_MSMF::close() if (isOpen) { isOpen = false; - videoSample.Release(); + usedVideoSample.Release(); + for (auto item : audioSamples) + item.Release(); videoFileSource.Release(); + device_status = false; camid = -1; filename.clear(); } readCallback.Release(); } -bool CvCapture_MSMF::initStream(DWORD streamID, const MediaType& mt) +bool CvCapture_MSMF::initStream(DWORD streamID, const MediaType mt) { CV_LOG_DEBUG(NULL, "Init stream " << streamID << " with MediaType " << mt); - _ComPtr mediaTypeOut = mt.createMediaType(); - if (FAILED(videoFileSource->SetStreamSelection((DWORD)MF_SOURCE_READER_ALL_STREAMS, false))) + _ComPtr mediaTypesOut; + if (mt.majorType == MFMediaType_Audio) { - CV_LOG_WARNING(NULL, "Failed to reset streams"); - return false; + captureAudioFormat = mt; + mediaTypesOut = mt.createMediaType_Audio(); + } + if (mt.majorType == MFMediaType_Video) + { + captureVideoFormat = mt; + mediaTypesOut = mt.createMediaType_Video(); } if (FAILED(videoFileSource->SetStreamSelection(streamID, true))) { CV_LOG_WARNING(NULL, "Failed to select stream " << streamID); return false; } - HRESULT hr = videoFileSource->SetCurrentMediaType(streamID, NULL, mediaTypeOut.Get()); + HRESULT hr = videoFileSource->SetCurrentMediaType(streamID, NULL, mediaTypesOut.Get()); if (hr == MF_E_TOPO_CODEC_NOT_FOUND) { CV_LOG_WARNING(NULL, "Failed to set mediaType (stream " << streamID << ", " << mt << "(codec not found)"); @@ -701,7 +900,7 @@ bool CvCapture_MSMF::initStream(DWORD streamID, const MediaType& mt) CV_LOG_WARNING(NULL, "Failed to set mediaType (stream " << streamID << ", " << mt << "(HRESULT " << hr << ")"); return false; } - captureFormat = mt; + return true; } @@ -826,7 +1025,52 @@ bool CvCapture_MSMF::configureHW(const VideoCaptureParameters& params) return configureHW(va_type == VIDEO_ACCELERATION_D3D11 || va_type == VIDEO_ACCELERATION_ANY); } -bool CvCapture_MSMF::configureOutput(MediaType newType, cv::uint32_t outFormat) +bool CvCapture_MSMF::configureAudioOutput(MediaType newType) +{ + FormatStorage formats; + formats.read(videoFileSource.Get()); + std::pair bestMatch; + formats.countNumberOfAudioStreams(numberOfAudioStreams); + if (device_status) + bestMatch = formats.findBestAudioFormat(newType); + else + bestMatch = formats.findAudioFormatByStream(audioStream); + if (bestMatch.second.isEmpty(true)) + { + CV_LOG_DEBUG(NULL, "Can not find audio stream with requested parameters"); + return false; + } + dwAudioStreamIndex = bestMatch.first.stream; + dwStreamIndices.push_back(dwAudioStreamIndex); + MediaType newFormat = bestMatch.second; + + newFormat.majorType = MFMediaType_Audio; + newFormat.nSamplesPerSec = 44100; + switch (outputAudioFormat) + { + case CV_8S: + newFormat.subType = MFAudioFormat_PCM; + newFormat.bit_per_sample = 8; + break; + case CV_16S: + newFormat.subType = MFAudioFormat_PCM; + newFormat.bit_per_sample = 16; + break; + case CV_32S: + newFormat.subType = MFAudioFormat_PCM; + newFormat.bit_per_sample = 32; + case CV_32F: + newFormat.subType = MFAudioFormat_Float; + newFormat.bit_per_sample = 32; + break; + default: + break; + } + + return initStream(dwAudioStreamIndex, newFormat); +} + +bool CvCapture_MSMF::configureVideoOutput(MediaType newType, cv::uint32_t outFormat) { FormatStorage formats; formats.read(videoFileSource.Get()); @@ -836,9 +1080,11 @@ bool CvCapture_MSMF::configureOutput(MediaType newType, cv::uint32_t outFormat) CV_LOG_DEBUG(NULL, "Can not find video stream with requested parameters"); return false; } - dwStreamIndex = bestMatch.first.stream; + dwVideoStreamIndex = bestMatch.first.stream; + dwStreamIndices.push_back(dwVideoStreamIndex); nativeFormat = bestMatch.second; MediaType newFormat = nativeFormat; + if (convertFormat) { switch (outFormat) @@ -869,8 +1115,25 @@ bool CvCapture_MSMF::configureOutput(MediaType newType, cv::uint32_t outFormat) } // we select native format first and then our requested format (related issue #12822) if (!newType.isEmpty()) // camera input - initStream(dwStreamIndex, nativeFormat); - return initStream(dwStreamIndex, newFormat); + { + initStream(dwVideoStreamIndex, nativeFormat); + } + return initStream(dwVideoStreamIndex, newFormat); +} + +bool CvCapture_MSMF::configureOutput() +{ + if (FAILED(videoFileSource->SetStreamSelection((DWORD)MF_SOURCE_READER_ALL_STREAMS, false))) + { + CV_LOG_WARNING(NULL, "Failed to reset streams"); + return false; + } + bool tmp = true; + if (videoStream != -1) + tmp = (!device_status)? configureVideoOutput(MediaType(), outputVideoFormat) : configureVideoOutput(MediaType::createDefault_Video(), outputVideoFormat); + if (audioStream != -1) + tmp &= (!device_status)? configureAudioOutput(MediaType()) : configureAudioOutput(MediaType::createDefault_Audio()); + return tmp; } bool CvCapture_MSMF::open(int index, const cv::VideoCaptureParameters* params) @@ -882,10 +1145,19 @@ bool CvCapture_MSMF::open(int index, const cv::VideoCaptureParameters* params) if (params) { configureHW(*params); + configureStreams(*params); + } + if (videoStream != -1 && audioStream != -1 || videoStream == -1 && audioStream == -1) + { + CV_LOG_DEBUG(NULL, "Only one of the properties CAP_PROP_AUDIO_STREAM " << audioStream << " and " << CAP_PROP_VIDEO_STREAM << " must be different from -1"); + return false; } - DeviceList devices; - UINT32 count = devices.read(); + UINT32 count = 0; + if (audioStream != -1) + count = devices.read(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_AUDCAP_GUID); + if (videoStream != -1) + count = devices.read(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID); if (count == 0 || static_cast(index) > count) { CV_LOG_DEBUG(NULL, "Device " << index << " not found (total " << count << " devices)"); @@ -902,14 +1174,14 @@ bool CvCapture_MSMF::open(int index, const cv::VideoCaptureParameters* params) } isOpen = true; + device_status = true; camid = index; readCallback = cb; duration = 0; - if (configureOutput(MediaType::createDefault(), outputFormat)) + if (configureOutput()) { - frameStep = captureFormat.getFrameStep(); + frameStep = captureVideoFormat.getFrameStep(); } - if (isOpen && !openFinalize_(params)) { close(); @@ -928,8 +1200,9 @@ bool CvCapture_MSMF::open(const cv::String& _filename, const cv::VideoCapturePar if (params) { configureHW(*params); + configureStreams(*params); + setAudioProperties(*params); } - // Set source reader parameters _ComPtr attr = getDefaultSourceConfig(); cv::AutoBuffer unicodeFileName(_filename.length() + 1); @@ -937,11 +1210,11 @@ bool CvCapture_MSMF::open(const cv::String& _filename, const cv::VideoCapturePar if (SUCCEEDED(MFCreateSourceReaderFromURL(unicodeFileName.data(), attr.Get(), &videoFileSource))) { isOpen = true; - sampleTime = 0; - if (configureOutput(MediaType(), outputFormat)) + usedVideoSampleTime = 0; + if (configureOutput()) { - frameStep = captureFormat.getFrameStep(); filename = _filename; + frameStep = captureVideoFormat.getFrameStep(); PROPVARIANT var; HRESULT hr; if (SUCCEEDED(hr = videoFileSource->GetPresentationAttribute((DWORD)MF_SOURCE_READER_MEDIASOURCE, MF_PD_DURATION, &var)) && @@ -954,13 +1227,18 @@ bool CvCapture_MSMF::open(const cv::String& _filename, const cv::VideoCapturePar duration = 0; } } - if (isOpen && !openFinalize_(params)) { close(); return false; } - + if (isOpen) + if (audioStream != -1 && videoStream != -1) + { + isOpen = grabFrame(); + if (isOpen) + grabIsDone = true; + } return isOpen; } @@ -997,71 +1275,212 @@ bool CvCapture_MSMF::openFinalize_(const VideoCaptureParameters* params) return true; } -bool CvCapture_MSMF::grabFrame() +bool CvCapture_MSMF::configureStreams(const cv::VideoCaptureParameters& params) { - CV_TRACE_FUNCTION(); - if (readCallback) // async "live" capture mode + if (params.has(CAP_PROP_VIDEO_STREAM)) { - HRESULT hr = 0; - SourceReaderCB* reader = ((SourceReaderCB*)readCallback.Get()); - if (!reader->m_reader) + double value = params.get(CAP_PROP_VIDEO_STREAM); + if (value == -1 || value == 0) + videoStream = static_cast(value); + else { - // Initiate capturing with async callback - reader->m_reader = videoFileSource.Get(); - reader->m_dwStreamIndex = dwStreamIndex; - if (FAILED(hr = videoFileSource->ReadSample(dwStreamIndex, 0, NULL, NULL, NULL, NULL))) - { - CV_LOG_ERROR(NULL, "videoio(MSMF): can't grab frame - initial async ReadSample() call failed: " << hr); - reader->m_reader = NULL; - return false; - } + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_VIDEO_STREAM parameter value is invalid/unsupported: " << value); + return false; } - BOOL bEOS = false; - if (FAILED(hr = reader->Wait(10000, videoSample, bEOS))) // 10 sec + } + if (params.has(CAP_PROP_AUDIO_STREAM)) + { + double value = params.get(CAP_PROP_AUDIO_STREAM); + if (value == -1 || value > -1) + audioStream = static_cast(value); + else { - CV_LOG_WARNING(NULL, "videoio(MSMF): can't grab frame. Error: " << hr); + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_STREAM parameter value is invalid/unsupported: " << value); return false; } - if (bEOS) + } + return true; +} +bool CvCapture_MSMF::setAudioProperties(const cv::VideoCaptureParameters& params) +{ + if (params.has(CAP_PROP_AUDIO_DATA_DEPTH)) + { + int value = static_cast(params.get(CAP_PROP_AUDIO_DATA_DEPTH)); + if (value != CV_8S && value != CV_16S && value != CV_32S && value != CV_32F) { - CV_LOG_WARNING(NULL, "videoio(MSMF): EOS signal. Capture stream is lost"); + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_DATA_DEPTH parameter value is invalid/unsupported: " << value); return false; } - sampleTime = reader->m_lastSampleTimestamp; - return true; + else + { + outputAudioFormat = value; + } } - else if (isOpen) + if (params.has(CAP_PROP_AUDIO_SYNCHRONIZE)) + { + int value = static_cast(params.get(CAP_PROP_AUDIO_SYNCHRONIZE)); + syncLastFrame = (value != 0) ? true : false; + } + return true; +} + +bool CvCapture_MSMF::grabVideoFrame() +{ + DWORD streamIndex, flags; + HRESULT hr; + usedVideoSample.Release(); + + bool returnFlag = false; + bool stopFlag = false; + if (audioStream != -1) { - DWORD streamIndex, flags; - videoSample.Release(); - HRESULT hr; - for(;;) + usedVideoSample.swap(impendingVideoSample); + std::swap(usedVideoSampleTime, impendingVideoSampleTime); + } + while (!stopFlag) + { + for (;;) { CV_TRACE_REGION("ReadSample"); if (!SUCCEEDED(hr = videoFileSource->ReadSample( - dwStreamIndex, // Stream index. + dwVideoStreamIndex, // Stream index. 0, // Flags. &streamIndex, // Receives the actual stream index. &flags, // Receives status flags. - &sampleTime, // Receives the time stamp. - &videoSample // Receives the sample or NULL. + &impendingVideoSampleTime, // Receives the time stamp. + &impendingVideoSample // Receives the sample or NULL. ))) break; - if (streamIndex != dwStreamIndex) + if (streamIndex != dwVideoStreamIndex) break; if (flags & (MF_SOURCE_READERF_ERROR | MF_SOURCE_READERF_ALLEFFECTSREMOVED | MF_SOURCE_READERF_ENDOFSTREAM)) break; - if (videoSample) + if (impendingVideoSample) break; if (flags & MF_SOURCE_READERF_STREAMTICK) { CV_LOG_DEBUG(NULL, "videoio(MSMF): Stream tick detected. Retrying to grab the frame"); } } + if (SUCCEEDED(hr)) + { + if (streamIndex != dwVideoStreamIndex) + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): Wrong stream read. Abort capturing"); + close(); + } + else if (flags & MF_SOURCE_READERF_ERROR) + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): Stream reading error. Abort capturing"); + close(); + } + else if (flags & MF_SOURCE_READERF_ALLEFFECTSREMOVED) + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): Stream decoding error. Abort capturing"); + close(); + } + else if (flags & MF_SOURCE_READERF_ENDOFSTREAM) + { + vEOS = true; + lastFrame = true; + stopFlag = true; + if (audioStream == -1) + returnFlag = false; + else if (usedVideoSample) + returnFlag = true; + CV_LOG_DEBUG(NULL, "videoio(MSMF): End of video stream detected"); + } + else + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): got video frame with timestamp=" << impendingVideoSampleTime); + if (audioStream != -1) + { + if (!usedVideoSample) + { + usedVideoSample.swap(impendingVideoSample); + std::swap(usedVideoSampleTime, impendingVideoSampleTime); + videoStartOffset = usedVideoSampleTime; + } + else + { + stopFlag = true; + } + if (impendingVideoSample) + { + nFrame++; + videoSampleDuration = impendingVideoSampleTime - usedVideoSampleTime; + requiredAudioTime = impendingVideoSampleTime - givenAudioTime; + givenAudioTime += requiredAudioTime; + } + } + else + { + usedVideoSample.swap(impendingVideoSample); + std::swap(usedVideoSampleTime, impendingVideoSampleTime); + stopFlag = true; + nFrame++; + } + if (flags & MF_SOURCE_READERF_NEWSTREAM) + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): New stream detected"); + } + if (flags & MF_SOURCE_READERF_NATIVEMEDIATYPECHANGED) + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): Stream native media type changed"); + } + if (flags & MF_SOURCE_READERF_CURRENTMEDIATYPECHANGED) + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): Stream current media type changed"); + } + returnFlag = true; + } + } + } + return returnFlag; +} +bool CvCapture_MSMF::grabAudioFrame() +{ + DWORD streamIndex, flags; + HRESULT hr; + _ComPtr audioSample = NULL; + audioSamples.clear(); + + bool returnFlag = false; + audioTime = 0; + int numberOfSamples = -1; + if (bufferedAudioDuration*1e7 > requiredAudioTime) + return true; + while ((!vEOS) ? audioTime <= requiredAudioTime : !aEOS) + { + if (audioStartOffset - usedVideoSampleTime > videoSampleDuration) + return true; + for (;;) + { + CV_TRACE_REGION("ReadSample"); + if (!SUCCEEDED(hr = videoFileSource->ReadSample( + dwAudioStreamIndex, // Stream index. + 0, // Flags. + &streamIndex, // Receives the actual stream index. + &flags, // Receives status flags. + &audioSampleTime, // Receives the time stamp. + &audioSample // Receives the sample or NULL. + ))) + break; + if (streamIndex != dwAudioStreamIndex) + break; + if (flags & (MF_SOURCE_READERF_ERROR | MF_SOURCE_READERF_ALLEFFECTSREMOVED | MF_SOURCE_READERF_ENDOFSTREAM)) + break; + if (audioSample) + break; + if (flags & MF_SOURCE_READERF_STREAMTICK) + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): Stream tick detected. Retrying to grab the frame"); + } + } if (SUCCEEDED(hr)) { - if (streamIndex != dwStreamIndex) + if (streamIndex != dwAudioStreamIndex) { CV_LOG_DEBUG(NULL, "videoio(MSMF): Wrong stream read. Abort capturing"); close(); @@ -1078,12 +1497,25 @@ bool CvCapture_MSMF::grabFrame() } else if (flags & MF_SOURCE_READERF_ENDOFSTREAM) { - sampleTime += frameStep; - CV_LOG_DEBUG(NULL, "videoio(MSMF): End of stream detected"); + aEOS = true; + if (videoStream != -1 && !vEOS) + returnFlag = true; + CV_LOG_DEBUG(NULL, "videoio(MSMF): End of audio stream detected"); + break; } else { - sampleTime += frameStep; + audioSamples.push_back(audioSample); + audioSample = NULL; + numberOfSamples++; + audioSamples[numberOfSamples]->GetSampleDuration(&audioSampleDuration); + CV_LOG_DEBUG(NULL, "videoio(MSMF): got audio frame with timestamp=" << audioSampleTime << " duration=" << audioSampleDuration); + audioTime += (LONGLONG)(audioSampleDuration + bufferedAudioDuration*1e7); + if (nFrame == 1 && audioStartOffset == -1) + { + audioStartOffset = audioSampleTime - audioSampleDuration; + requiredAudioTime -= audioStartOffset; + } if (flags & MF_SOURCE_READERF_NEWSTREAM) { CV_LOG_DEBUG(NULL, "videoio(MSMF): New stream detected"); @@ -1096,33 +1528,189 @@ bool CvCapture_MSMF::grabFrame() { CV_LOG_DEBUG(NULL, "videoio(MSMF): Stream current media type changed"); } - return true; + returnFlag = true; + } + } + else + { + CV_LOG_DEBUG(NULL, "videoio(MSMF): ReadSample() method is not succeeded"); + return false; + } + } + + if (!audioSamples.empty() || !bufferAudioData.empty() && aEOS) + { + _ComPtr buf = NULL; + std::vector audioDataInUse; + BYTE* ptr = NULL; + DWORD maxsize = 0, cursize = 0; + CV_TRACE_REGION("get_contiguous_buffer"); + for (auto item : audioSamples) + { + if (!SUCCEEDED(item->ConvertToContiguousBuffer(&buf))) + { + CV_TRACE_REGION("get_buffer"); + DWORD bcnt = 0; + if (!SUCCEEDED(item->GetBufferCount(&bcnt))) + break; + if (bcnt == 0) + break; + if (!SUCCEEDED(item->GetBufferByIndex(0, &buf))) + break; + } + if (!SUCCEEDED(buf->Lock(&ptr, &maxsize, &cursize))) + break; + size_t lastSize = bufferAudioData.size(); + bufferAudioData.resize(lastSize+cursize); + for (unsigned int i = 0; i < cursize; i++) + { + bufferAudioData[lastSize+i]=*(ptr+i); + } + CV_TRACE_REGION_NEXT("unlock"); + buf->Unlock(); + buf = NULL; + } + audioSamples.clear(); + + audioSamplePos += chunkLengthOfBytes/((captureAudioFormat.bit_per_sample/8)*captureAudioFormat.nChannels); + chunkLengthOfBytes = (videoStream != -1) ? (LONGLONG)((requiredAudioTime*captureAudioFormat.nSamplesPerSec*captureAudioFormat.nChannels*(captureAudioFormat.bit_per_sample)/8)/1e7) : cursize; + if ((videoStream != -1) && (chunkLengthOfBytes % ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels) != 0)) + { + if ( (double)audioSamplePos/captureAudioFormat.nSamplesPerSec + audioStartOffset * 1e-7 - usedVideoSampleTime * 1e-7 >= 0 ) + chunkLengthOfBytes -= numberOfAdditionalAudioBytes; + numberOfAdditionalAudioBytes = ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels) + - chunkLengthOfBytes % ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels); + chunkLengthOfBytes += numberOfAdditionalAudioBytes; + } + if (lastFrame && !syncLastFrame|| aEOS && !vEOS) + { + chunkLengthOfBytes = bufferAudioData.size(); + } + CV_Check((double)chunkLengthOfBytes, chunkLengthOfBytes >= INT_MIN || chunkLengthOfBytes <= INT_MAX, "MSMF: The chunkLengthOfBytes is out of the allowed range"); + copy(bufferAudioData.begin(), bufferAudioData.begin() + (int)chunkLengthOfBytes, std::back_inserter(audioDataInUse)); + bufferAudioData.erase(bufferAudioData.begin(), bufferAudioData.begin() + (int)chunkLengthOfBytes); + if (audioFrame.empty()) + { + switch (outputAudioFormat) + { + case CV_8S: + cv::Mat((int)chunkLengthOfBytes/(captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_8S, audioDataInUse.data()).copyTo(audioFrame); + break; + case CV_16S: + cv::Mat((int)chunkLengthOfBytes/(2*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_16S, audioDataInUse.data()).copyTo(audioFrame); + break; + case CV_32S: + cv::Mat((int)chunkLengthOfBytes/(4*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_32S, audioDataInUse.data()).copyTo(audioFrame); + break; + case CV_32F: + cv::Mat((int)chunkLengthOfBytes/(4*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_32F, audioDataInUse.data()).copyTo(audioFrame); + break; + default: + break; + } + } + audioDataInUse.clear(); + audioDataInUse.shrink_to_fit(); + } + + return returnFlag; +} + +bool CvCapture_MSMF::grabFrame() +{ + CV_TRACE_FUNCTION(); + + if (grabIsDone) + { + grabIsDone = false; + return true; + } + + audioFrame = Mat(); + if (readCallback) // async "live" capture mode + { + audioSamples.push_back(NULL); + HRESULT hr = 0; + SourceReaderCB* reader = ((SourceReaderCB*)readCallback.Get()); + DWORD dwStreamIndex = 0; + if (videoStream != -1) + dwStreamIndex = dwVideoStreamIndex; + if (audioStream != -1) + dwStreamIndex = dwAudioStreamIndex; + if (!reader->m_reader) + { + // Initiate capturing with async callback + reader->m_reader = videoFileSource.Get(); + reader->m_dwStreamIndex = dwStreamIndex; + if (FAILED(hr = videoFileSource->ReadSample(dwStreamIndex, 0, NULL, NULL, NULL, NULL))) + { + CV_LOG_ERROR(NULL, "videoio(MSMF): can't grab frame - initial async ReadSample() call failed: " << hr); + reader->m_reader = NULL; + return false; } } + BOOL bEOS = false; + if (FAILED(hr = reader->Wait( videoStream == -1 ? INFINITE : 10000, (videoStream != -1) ? usedVideoSample : audioSamples[0], bEOS))) // 10 sec + { + CV_LOG_WARNING(NULL, "videoio(MSMF): can't grab frame. Error: " << hr); + return false; + } + if (bEOS) + { + CV_LOG_WARNING(NULL, "videoio(MSMF): EOS signal. Capture stream is lost"); + return false; + } + if (videoStream != -1) + usedVideoSampleTime = reader->m_lastSampleTimestamp; + return true; + } + else if (isOpen) + { + if (vEOS) + return false; + + bool returnFlag = true; + + if (videoStream != -1) + { + if (!vEOS) + returnFlag &= grabVideoFrame(); + if (!returnFlag) + return false; + } + + if (audioStream != -1) + { + bufferedAudioDuration = (double)(bufferAudioData.size()/((captureAudioFormat.bit_per_sample/8)*captureAudioFormat.nChannels))/captureAudioFormat.nSamplesPerSec; + audioFrame.release(); + if (!aEOS) + returnFlag &= grabAudioFrame(); + } + + return returnFlag; } return false; } -bool CvCapture_MSMF::retrieveFrame(int, cv::OutputArray frame) +bool CvCapture_MSMF::retrieveVideoFrame(cv::OutputArray frame) { CV_TRACE_FUNCTION(); do { - if (!videoSample) + if (!usedVideoSample) break; _ComPtr buf = NULL; - CV_TRACE_REGION("get_contiguous_buffer"); - if (!SUCCEEDED(videoSample->ConvertToContiguousBuffer(&buf))) + if (!SUCCEEDED(usedVideoSample->ConvertToContiguousBuffer(&buf))) { CV_TRACE_REGION("get_buffer"); DWORD bcnt = 0; - if (!SUCCEEDED(videoSample->GetBufferCount(&bcnt))) + if (!SUCCEEDED(usedVideoSample->GetBufferCount(&bcnt))) break; if (bcnt == 0) break; - if (!SUCCEEDED(videoSample->GetBufferByIndex(0, &buf))) + if (!SUCCEEDED(usedVideoSample->GetBufferByIndex(0, &buf))) break; } @@ -1158,27 +1746,27 @@ bool CvCapture_MSMF::retrieveFrame(int, cv::OutputArray frame) break; if (convertFormat) { - if (lock2d || (unsigned int)cursize == captureFormat.sampleSize) + if (lock2d || (unsigned int)cursize == captureVideoFormat.sampleSize) { - switch (outputFormat) + switch (outputVideoFormat) { case CV_CAP_MODE_YUYV: - cv::Mat(captureFormat.height, captureFormat.width, CV_8UC2, ptr, pitch).copyTo(frame); + cv::Mat(captureVideoFormat.height, captureVideoFormat.width, CV_8UC2, ptr, pitch).copyTo(frame); break; case CV_CAP_MODE_BGR: if (captureMode == MODE_HW) - cv::cvtColor(cv::Mat(captureFormat.height, captureFormat.width, CV_8UC4, ptr, pitch), frame, cv::COLOR_BGRA2BGR); + cv::cvtColor(cv::Mat(captureVideoFormat.height, captureVideoFormat.width, CV_8UC4, ptr, pitch), frame, cv::COLOR_BGRA2BGR); else - cv::Mat(captureFormat.height, captureFormat.width, CV_8UC3, ptr, pitch).copyTo(frame); + cv::Mat(captureVideoFormat.height, captureVideoFormat.width, CV_8UC3, ptr, pitch).copyTo(frame); break; case CV_CAP_MODE_RGB: if (captureMode == MODE_HW) - cv::cvtColor(cv::Mat(captureFormat.height, captureFormat.width, CV_8UC4, ptr, pitch), frame, cv::COLOR_BGRA2BGR); + cv::cvtColor(cv::Mat(captureVideoFormat.height, captureVideoFormat.width, CV_8UC4, ptr, pitch), frame, cv::COLOR_BGRA2BGR); else - cv::cvtColor(cv::Mat(captureFormat.height, captureFormat.width, CV_8UC3, ptr, pitch), frame, cv::COLOR_BGR2RGB); + cv::cvtColor(cv::Mat(captureVideoFormat.height, captureVideoFormat.width, CV_8UC3, ptr, pitch), frame, cv::COLOR_BGR2RGB); break; case CV_CAP_MODE_GRAY: - cv::Mat(captureFormat.height, captureFormat.width, CV_8UC1, ptr, pitch).copyTo(frame); + cv::Mat(captureVideoFormat.height, captureVideoFormat.width, CV_8UC1, ptr, pitch).copyTo(frame); break; default: frame.release(); @@ -1204,30 +1792,142 @@ bool CvCapture_MSMF::retrieveFrame(int, cv::OutputArray frame) return false; } +bool CvCapture_MSMF::retrieveAudioFrame(int index, cv::OutputArray frame) +{ + CV_TRACE_FUNCTION(); + if (audioStartOffset - usedVideoSampleTime > videoSampleDuration) + { + frame.release(); + return true; + } + do + { + if (audioFrame.empty()) + { + frame.release(); + if (aEOS) + return true; + } + cv::Mat data; + switch (outputAudioFormat) + { + case CV_8S: + data = cv::Mat(1, audioFrame.rows, CV_8S); + for (int i = 0; i < audioFrame.rows; i++) + data.at(0,i) = audioFrame.at(i,index-audioBaseIndex); + break; + case CV_16S: + data = cv::Mat(1, audioFrame.rows, CV_16S); + for (int i = 0; i < audioFrame.rows; i++) + data.at(0,i) = audioFrame.at(i,index-audioBaseIndex); + break; + case CV_32S: + data = cv::Mat(1, audioFrame.rows, CV_32S); + for (int i = 0; i < audioFrame.rows; i++) + data.at(0,i) = audioFrame.at(i,index-audioBaseIndex); + break; + case CV_32F: + data = cv::Mat(1, audioFrame.rows, CV_32F); + for (int i = 0; i < audioFrame.rows; i++) + data.at(0,i) = audioFrame.at(i,index-audioBaseIndex); + break; + default: + frame.release(); + break; + } + if (!data.empty()) + data.copyTo(frame); + + return !frame.empty(); + } while (0); + + return false; +} + +bool CvCapture_MSMF::retrieveFrame(int index, cv::OutputArray frame) +{ + CV_TRACE_FUNCTION(); + if (index < 0) + return false; + if ((unsigned int)index < audioBaseIndex) + { + if (videoStream == -1) + { + frame.release(); + return false; + } + else + return retrieveVideoFrame(frame); + } + else + { + if (audioStream == -1) + { + frame.release(); + return false; + } + else + return retrieveAudioFrame(index, frame); + } +} + bool CvCapture_MSMF::setTime(double time, bool rough) { + if (videoStream == -1) + return false; + if (videoStream != -1 && audioStream != -1) + if (time != 0) + return false; PROPVARIANT var; if (SUCCEEDED(videoFileSource->GetPresentationAttribute((DWORD)MF_SOURCE_READER_MEDIASOURCE, MF_SOURCE_READER_MEDIASOURCE_CHARACTERISTICS, &var)) && var.vt == VT_UI4 && var.ulVal & MFMEDIASOURCE_CAN_SEEK) { - videoSample.Release(); + usedVideoSample.Release(); bool useGrabbing = time > 0 && !rough && !(var.ulVal & MFMEDIASOURCE_HAS_SLOW_SEEK); PropVariantClear(&var); - sampleTime = (useGrabbing && time >= frameStep) ? (LONGLONG)floor(time + 0.5) - frameStep : (LONGLONG)floor(time + 0.5); + usedVideoSampleTime = (useGrabbing) ? 0 : (LONGLONG)floor(time + 0.5); + nFrame = (useGrabbing) ? 0 : usedVideoSampleTime/frameStep; + givenAudioTime = (useGrabbing) ? 0 : nFrame*frameStep; var.vt = VT_I8; - var.hVal.QuadPart = sampleTime; + var.hVal.QuadPart = usedVideoSampleTime; bool resOK = SUCCEEDED(videoFileSource->SetCurrentPosition(GUID_NULL, var)); PropVariantClear(&var); if (resOK && useGrabbing) { LONGLONG timeborder = (LONGLONG)floor(time + 0.5) - frameStep / 2; - do { resOK = grabFrame(); videoSample.Release(); } while (resOK && sampleTime < timeborder); + do { resOK = grabFrame(); usedVideoSample.Release(); } while (resOK && usedVideoSampleTime < timeborder); } return resOK; } return false; } +bool CvCapture_MSMF::setTime(int numberFrame) +{ + if (videoStream == -1) + return false; + if (videoStream != -1 && audioStream != -1) + if (numberFrame != 0) + return false; + PROPVARIANT var; + if (SUCCEEDED(videoFileSource->GetPresentationAttribute((DWORD)MF_SOURCE_READER_MEDIASOURCE, MF_SOURCE_READER_MEDIASOURCE_CHARACTERISTICS, &var)) && + var.vt == VT_UI4 && var.ulVal & MFMEDIASOURCE_CAN_SEEK) + { + usedVideoSample.Release(); + PropVariantClear(&var); + usedVideoSampleTime = 0; + nFrame = 0; + givenAudioTime = 0; + var.vt = VT_I8; + var.hVal.QuadPart = usedVideoSampleTime; + bool resOK = SUCCEEDED(videoFileSource->SetCurrentPosition(GUID_NULL, var)); + PropVariantClear(&var); + while (resOK && nFrame < numberFrame) { resOK = grabFrame(); usedVideoSample.Release(); }; + return resOK; + } + return false; +} + template bool CvCapture_MSMF::readComplexPropery(long prop, long & val) const { @@ -1269,29 +1969,31 @@ double CvCapture_MSMF::getProperty( int property_id ) const case CV_CAP_PROP_CONVERT_RGB: return convertFormat ? 1 : 0; case CV_CAP_PROP_SAR_NUM: - return captureFormat.aspectRatioNum; + return captureVideoFormat.aspectRatioNum; case CV_CAP_PROP_SAR_DEN: - return captureFormat.aspectRatioDenom; + return captureVideoFormat.aspectRatioDenom; case CV_CAP_PROP_FRAME_WIDTH: - return captureFormat.width; + return captureVideoFormat.width; case CV_CAP_PROP_FRAME_HEIGHT: - return captureFormat.height; + return captureVideoFormat.height; case CV_CAP_PROP_FOURCC: - return captureFormat.subType.Data1; + return captureVideoFormat.subType.Data1; case CV_CAP_PROP_FPS: - return captureFormat.getFramerate(); + return captureVideoFormat.getFramerate(); case CV_CAP_PROP_FRAME_COUNT: if (duration != 0) - return floor(((double)duration / 1e7)* captureFormat.getFramerate() + 0.5); + return floor(((double)duration / 1e7)* captureVideoFormat.getFramerate() + 0.5); else break; case CV_CAP_PROP_POS_FRAMES: - return floor(((double)sampleTime / 1e7)* captureFormat.getFramerate() + 0.5); + return (double)nFrame; case CV_CAP_PROP_POS_MSEC: - return (double)sampleTime / 1e4; + return (double)usedVideoSampleTime / 1e4; + case CAP_PROP_AUDIO_POS: + return (double)audioSamplePos; case CV_CAP_PROP_POS_AVI_RATIO: if (duration != 0) - return (double)sampleTime / duration; + return (double)usedVideoSampleTime / duration; else break; case CV_CAP_PROP_BRIGHTNESS: @@ -1383,6 +2085,18 @@ double CvCapture_MSMF::getProperty( int property_id ) const case CV_CAP_PROP_ISO_SPEED: case CV_CAP_PROP_SETTINGS: case CV_CAP_PROP_BUFFERSIZE: + case CAP_PROP_AUDIO_BASE_INDEX: + return audioBaseIndex; + case CAP_PROP_AUDIO_TOTAL_STREAMS: + return numberOfAudioStreams; + case CAP_PROP_AUDIO_TOTAL_CHANNELS: + return captureAudioFormat.nChannels; + case CAP_PROP_AUDIO_SAMPLES_PER_SECOND: + return captureAudioFormat.nSamplesPerSec; + case CAP_PROP_AUDIO_DATA_DEPTH: + return outputAudioFormat; + case CAP_PROP_AUDIO_SHIFT_NSEC: + return (double)(audioStartOffset - videoStartOffset)*1e2; default: break; } @@ -1408,7 +2122,7 @@ bool CvCapture_MSMF::writeComplexProperty(long prop, double val, long flags) bool CvCapture_MSMF::setProperty( int property_id, double value ) { - MediaType newFormat = captureFormat; + MediaType newFormat = captureVideoFormat; if (isOpen) switch (property_id) { @@ -1423,45 +2137,45 @@ bool CvCapture_MSMF::setProperty( int property_id, double value ) return false; } case CV_CAP_PROP_FOURCC: - return configureOutput(newFormat, (int)cvRound(value)); + return configureVideoOutput(newFormat, (int)cvRound(value)); case CV_CAP_PROP_FORMAT: - return configureOutput(newFormat, (int)cvRound(value)); + return configureVideoOutput(newFormat, (int)cvRound(value)); case CV_CAP_PROP_CONVERT_RGB: convertFormat = (value != 0); - return configureOutput(newFormat, outputFormat); + return configureVideoOutput(newFormat, outputVideoFormat); case CV_CAP_PROP_SAR_NUM: if (value > 0) { newFormat.aspectRatioNum = (UINT32)cvRound(value); - return configureOutput(newFormat, outputFormat); + return configureVideoOutput(newFormat, outputVideoFormat); } break; case CV_CAP_PROP_SAR_DEN: if (value > 0) { newFormat.aspectRatioDenom = (UINT32)cvRound(value); - return configureOutput(newFormat, outputFormat); + return configureVideoOutput(newFormat, outputVideoFormat); } break; case CV_CAP_PROP_FRAME_WIDTH: if (value >= 0) { newFormat.width = (UINT32)cvRound(value); - return configureOutput(newFormat, outputFormat); + return configureVideoOutput(newFormat, outputVideoFormat); } break; case CV_CAP_PROP_FRAME_HEIGHT: if (value >= 0) { newFormat.height = (UINT32)cvRound(value); - return configureOutput(newFormat, outputFormat); + return configureVideoOutput(newFormat, outputVideoFormat); } break; case CV_CAP_PROP_FPS: if (value >= 0) { newFormat.setFramerate(value); - return configureOutput(newFormat, outputFormat); + return configureVideoOutput(newFormat, outputVideoFormat); } break; case CV_CAP_PROP_FRAME_COUNT: @@ -1471,8 +2185,8 @@ bool CvCapture_MSMF::setProperty( int property_id, double value ) return setTime(duration * value, true); break; case CV_CAP_PROP_POS_FRAMES: - if (std::fabs(captureFormat.getFramerate()) > 0) - return setTime(value * 1e7 / captureFormat.getFramerate(), false); + if (std::fabs(captureVideoFormat.getFramerate()) > 0) + return setTime((int)value); break; case CV_CAP_PROP_POS_MSEC: return setTime(value * 1e4, false); diff --git a/modules/videoio/test/test_audio.cpp b/modules/videoio/test/test_audio.cpp new file mode 100644 index 000000000000..3ff51e26134d --- /dev/null +++ b/modules/videoio/test/test_audio.cpp @@ -0,0 +1,273 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + +//file name, number of audio channels, epsilon, video type, weight, height, number of frame, number of audio samples, fps, psnr Threshold, backend +typedef std::tuple paramCombination; +//file name, number of audio channels, number of audio samples, epsilon, backend +typedef std::tuple param; + +class AudioBaseTest +{ +protected: + AudioBaseTest(){}; + void getValidAudioData() + { + const double step = 3.14/22050; + double value = 0; + validAudioData.resize(expectedNumAudioCh); + for (int nCh = 0; nCh < expectedNumAudioCh; nCh++) + { + value = 0; + for(unsigned int i = 0; i < numberOfSamples; i++) + { + if (i != 0 && i % 44100 == 0) + value = 0; + validAudioData[nCh].push_back(sin(value)); + value += step; + } + } + } + void checkAudio() + { + getValidAudioData(); + + ASSERT_EQ(expectedNumAudioCh, (int)audioData.size()); + for (unsigned int nCh = 0; nCh < audioData.size(); nCh++) + { + ASSERT_EQ(numberOfSamples, audioData[nCh].size()) << "nCh=" << nCh; + for (unsigned int i = 0; i < numberOfSamples; i++) + { + EXPECT_NEAR(validAudioData[nCh][i], audioData[nCh][i], epsilon) << "sample index=" << i << " nCh=" << nCh; + } + } + } +protected: + int expectedNumAudioCh; + unsigned int numberOfSamples; + double epsilon; + VideoCaptureAPIs backend; + std::string root; + std::string fileName; + + std::vector> validAudioData; + std::vector> audioData; + std::vector params; + + Mat audioFrame; + VideoCapture cap; +}; + +class AudioTestFixture : public AudioBaseTest, public testing::TestWithParam +{ +public: + AudioTestFixture() + { + fileName = get<0>(GetParam()); + expectedNumAudioCh = get<1>(GetParam()); + numberOfSamples = get<2>(GetParam()); + epsilon = get<3>(GetParam()); + backend = get<4>(GetParam()); + root = "audio/"; + params = { CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, -1, + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S }; + } + + void doTest() + { + ASSERT_TRUE(cap.open(findDataFile(root + fileName), backend, params)); + const int audioBaseIndex = static_cast(cap.get(cv::CAP_PROP_AUDIO_BASE_INDEX)); + const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS); + ASSERT_EQ(expectedNumAudioCh, numberOfChannels); + double f = 0; + audioData.resize(numberOfChannels); + for (;;) + { + if (cap.grab()) + { + for (int nCh = 0; nCh < numberOfChannels; nCh++) + { + ASSERT_TRUE(cap.retrieve(audioFrame, audioBaseIndex)); + ASSERT_EQ(CV_16SC1, audioFrame.type()) << audioData[nCh].size(); + for (int i = 0; i < audioFrame.cols; i++) + { + f = ((double) audioFrame.at(0,i)) / (double) 32768; + audioData[nCh].push_back(f); + } + } + } + else { break; } + } + ASSERT_FALSE(audioData.empty()); + + checkAudio(); + } +}; + +const param audioParams[] = +{ + param("test_audio.wav", 1, 132300, 0.0001, cv::CAP_MSMF), + param("test_mono_audio.mp3", 1, 133104, 0.12, cv::CAP_MSMF), + param("test_stereo_audio.mp3", 2, 133104, 0.12, cv::CAP_MSMF), + param("test_audio.mp4", 1, 133104, 0.15, cv::CAP_MSMF) +}; + +class Audio : public AudioTestFixture{}; + +TEST_P(Audio, audio) +{ + if (!videoio_registry::hasBackend(cv::VideoCaptureAPIs(backend))) + throw SkipTestException(cv::videoio_registry::getBackendName(backend) + " backend was not found"); + + doTest(); +} + +INSTANTIATE_TEST_CASE_P(/**/, Audio, testing::ValuesIn(audioParams)); + +class MediaTestFixture : public AudioBaseTest, public testing::TestWithParam +{ +public: + MediaTestFixture(): + videoType(get<3>(GetParam())), + height(get<4>(GetParam())), + width(get<5>(GetParam())), + numberOfFrames(get<6>(GetParam())), + fps(get<8>(GetParam())), + psnrThreshold(get<9>(GetParam())) + { + fileName = get<0>(GetParam()); + expectedNumAudioCh = get<1>(GetParam()); + numberOfSamples = get<7>(GetParam()); + epsilon = get<2>(GetParam()); + backend = get<10>(GetParam()); + root = "audio/"; + params = { CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, 0, + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S }; + }; + + void doTest() + { + ASSERT_TRUE(cap.open(findDataFile(root + fileName), backend, params)); + + const int audioBaseIndex = static_cast(cap.get(cv::CAP_PROP_AUDIO_BASE_INDEX)); + const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS); + ASSERT_EQ(expectedNumAudioCh, numberOfChannels); + + const int samplePerSecond = (int)cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND); + ASSERT_EQ(44100, samplePerSecond); + int samplesPerFrame = (int)(1./fps*samplePerSecond); + int audioSamplesTolerance = samplesPerFrame / 2; + + double audio0_timestamp = 0; + + Mat videoFrame; + Mat img(height, width, videoType); + audioData.resize(numberOfChannels); + for (int frame = 0; frame < numberOfFrames; frame++) + { + SCOPED_TRACE(cv::format("frame=%d", frame)); + + ASSERT_TRUE(cap.grab()); + + if (frame == 0) + { + double audio_shift = cap.get(CAP_PROP_AUDIO_SHIFT_NSEC); + double video0_timestamp = cap.get(CAP_PROP_POS_MSEC) * 1e-3; + audio0_timestamp = video0_timestamp + audio_shift * 1e-9; + std::cout << "video0 timestamp: " << video0_timestamp << " audio0 timestamp: " << audio0_timestamp << " (audio shift nanoseconds: " << audio_shift << " , seconds: " << audio_shift * 1e-9 << ")" << std::endl; + } + + ASSERT_TRUE(cap.retrieve(videoFrame)); + if (epsilon >= 0) + { + generateFrame(frame, numberOfFrames, img); + ASSERT_EQ(img.size, videoFrame.size); + double psnr = cvtest::PSNR(img, videoFrame); + EXPECT_GE(psnr, psnrThreshold); + } + + int audioFrameCols = 0; + for (int nCh = 0; nCh < numberOfChannels; nCh++) + { + ASSERT_TRUE(cap.retrieve(audioFrame, audioBaseIndex+nCh)); + if (audioFrame.empty()) + continue; + ASSERT_EQ(CV_16SC1, audioFrame.type()); + if (nCh == 0) + audioFrameCols = audioFrame.cols; + else + ASSERT_EQ(audioFrameCols, audioFrame.cols) << "channel "<< nCh; + for (int i = 0; i < audioFrame.cols; i++) + { + double f = audioFrame.at(0,i) / 32768.0; + audioData[nCh].push_back(f); + } + } + + if (frame < 5 || frame >= numberOfFrames-5) + std::cout << "frame=" << frame << ": audioFrameSize=" << audioFrameCols << " videoTimestamp=" << cap.get(CAP_PROP_POS_MSEC) << " ms" << std::endl; + else if (frame == 6) + std::cout << "frame..." << std::endl; + + if (audioFrameCols == 0) + continue; + if (frame != 0 && frame != numberOfFrames-1) + { + // validate audio position + EXPECT_NEAR( + cap.get(CAP_PROP_AUDIO_POS) / samplePerSecond + audio0_timestamp, + cap.get(CAP_PROP_POS_MSEC) * 1e-3, + (1.0 / fps) * 0.3) + << "CAP_PROP_AUDIO_POS=" << cap.get(CAP_PROP_AUDIO_POS) << " CAP_PROP_POS_MSEC=" << cap.get(CAP_PROP_POS_MSEC); + } + if (frame != 0 && frame != numberOfFrames-1 && audioData[0].size() != (size_t)numberOfSamples) + { + // validate audio frame size + EXPECT_NEAR(audioFrame.cols, samplesPerFrame, audioSamplesTolerance); + } + } + ASSERT_FALSE(cap.grab()); + ASSERT_FALSE(audioData.empty()); + + std::cout << "Total audio samples=" << audioData[0].size() << std::endl; + + if (epsilon >= 0) + checkAudio(); + } +protected: + const int videoType; + const int height; + const int width; + const int numberOfFrames; + const int fps; + const double psnrThreshold; +}; + +const paramCombination mediaParams[] = +{ + paramCombination("test_audio.mp4", 1, 0.15, CV_8UC3, 240, 320, 90, 131819, 30, 30., cv::CAP_MSMF) +#if 0 + // https://filesamples.com/samples/video/mp4/sample_960x400_ocean_with_audio.mp4 + , paramCombination("sample_960x400_ocean_with_audio.mp4", 2, -1/*eplsilon*/, CV_8UC3, 400, 960, 1116, 2056588, 30, 30., cv::CAP_MSMF) +#endif +}; + +class Media : public MediaTestFixture{}; + +TEST_P(Media, audio) +{ + if (!videoio_registry::hasBackend(cv::VideoCaptureAPIs(backend))) + throw SkipTestException(cv::videoio_registry::getBackendName(backend) + " backend was not found"); + + doTest(); +} + +INSTANTIATE_TEST_CASE_P(/**/, Media, testing::ValuesIn(mediaParams)); + +}} //namespace diff --git a/modules/videoio/test/test_microphone.cpp b/modules/videoio/test/test_microphone.cpp new file mode 100644 index 000000000000..c82a7c4edac7 --- /dev/null +++ b/modules/videoio/test/test_microphone.cpp @@ -0,0 +1,41 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Usage: opencv_test_videoio --gtest_also_run_disabled_tests + +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + +TEST(DISABLED_videoio_micro, basic) +{ + int cursize = 0; + int validSize = 0; + Mat frame; + + std::vector params { CAP_PROP_AUDIO_STREAM, 0, CAP_PROP_VIDEO_STREAM, -1 }; + VideoCapture cap(0, cv::CAP_MSMF, params); + ASSERT_TRUE(cap.isOpened()); + + int samplesPerSecond = (int)cap.get(cv::CAP_PROP_AUDIO_SAMPLES_PER_SECOND); + const int audio_base_index = (int)cap.get(cv::CAP_PROP_AUDIO_BASE_INDEX); + + const double cvTickFreq = cv::getTickFrequency(); + int64 sysTimePrev = cv::getTickCount(); + int64 sysTimeCurr = cv::getTickCount(); + + cout << "Audio would be captured for the next 10 seconds" << endl; + while ((sysTimeCurr-sysTimePrev)/cvTickFreq < 10) + { + if (cap.grab()) + { + ASSERT_TRUE(cap.retrieve(frame, audio_base_index)); + sysTimeCurr = cv::getTickCount(); + } + } + validSize = samplesPerSecond*(int)((sysTimeCurr-sysTimePrev)/cvTickFreq); + cursize = (int)cap.get(cv::CAP_PROP_AUDIO_POS); + ASSERT_LT(validSize - cursize, cursize*0.05); +} + +}} // namespace diff --git a/samples/cpp/videocapture_audio.cpp b/samples/cpp/videocapture_audio.cpp new file mode 100644 index 000000000000..c9f1ec94ce66 --- /dev/null +++ b/samples/cpp/videocapture_audio.cpp @@ -0,0 +1,59 @@ +#include +#include +#include +#include + +using namespace cv; +using namespace std; + +int main(int argc, char** argv) +{ + CommandLineParser parser(argc, argv, "{@audio||}"); + string file = parser.get("@audio"); + + if (file.empty()) + { + return 1; + } + + Mat frame; + vector> audioData; + VideoCapture cap; + vector params { CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, -1, + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S }; + + cap.open(file, CAP_MSMF, params); + if (!cap.isOpened()) + { + cerr << "ERROR! Can't to open file: " + file << endl; + return -1; + } + + const int audioBaseIndex = (int)cap.get(CAP_PROP_AUDIO_BASE_INDEX); + const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS); + cout << "CAP_PROP_AUDIO_DATA_DEPTH: " << depthToString((int)cap.get(CAP_PROP_AUDIO_DATA_DEPTH)) << endl; + cout << "CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND) << endl; + cout << "CAP_PROP_AUDIO_TOTAL_CHANNELS: " << numberOfChannels << endl; + cout << "CAP_PROP_AUDIO_TOTAL_STREAMS: " << cap.get(CAP_PROP_AUDIO_TOTAL_STREAMS) << endl; + + int numberOfSamples = 0; + audioData.resize(numberOfChannels); + for (;;) + { + if (cap.grab()) + { + for (int nCh = 0; nCh < numberOfChannels; nCh++) + { + cap.retrieve(frame, audioBaseIndex+nCh); + audioData[nCh].push_back(frame); + numberOfSamples+=frame.cols; + } + } + else { break; } + } + + cout << "Number of samples: " << numberOfSamples << endl; + + return 0; +} diff --git a/samples/cpp/videocapture_audio_combination.cpp b/samples/cpp/videocapture_audio_combination.cpp new file mode 100644 index 000000000000..7f0deecf16c8 --- /dev/null +++ b/samples/cpp/videocapture_audio_combination.cpp @@ -0,0 +1,69 @@ +#include +#include +#include +#include + +using namespace cv; +using namespace std; + +int main(int argc, char** argv) +{ + cv::CommandLineParser parser(argc, argv, "{@audio||}"); + string file = parser.get("@audio"); + + if (file.empty()) + { + return 1; + } + + Mat videoFrame; + Mat audioFrame; + vector> audioData; + VideoCapture cap; + vector params { CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, 0, + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S }; + + cap.open(file, CAP_MSMF, params); + if (!cap.isOpened()) + { + cerr << "ERROR! Can't to open file: " + file << endl; + return -1; + } + + const int audioBaseIndex = (int)cap.get(CAP_PROP_AUDIO_BASE_INDEX); + const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS); + cout << "CAP_PROP_AUDIO_DATA_DEPTH: " << depthToString((int)cap.get(CAP_PROP_AUDIO_DATA_DEPTH)) << endl; + cout << "CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND) << endl; + cout << "CAP_PROP_AUDIO_TOTAL_CHANNELS: " << cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS) << endl; + cout << "CAP_PROP_AUDIO_TOTAL_STREAMS: " << cap.get(CAP_PROP_AUDIO_TOTAL_STREAMS) << endl; + + int numberOfSamples = 0; + int numberOfFrames = 0; + audioData.resize(numberOfChannels); + for (;;) + { + if (cap.grab()) + { + cap.retrieve(videoFrame); + for (int nCh = 0; nCh < numberOfChannels; nCh++) + { + cap.retrieve(audioFrame, audioBaseIndex+nCh); + if (!audioFrame.empty()) + audioData[nCh].push_back(audioFrame); + numberOfSamples+=audioFrame.cols; + } + if (!videoFrame.empty()) + { + numberOfFrames++; + imshow("Live", videoFrame); + if (waitKey(5) >= 0) + break; + } + } else { break; } + } + + cout << "Number of audio samples: " << numberOfSamples << endl + << "Number of video frames: " << numberOfFrames << endl; + return 0; +} diff --git a/samples/cpp/videocapture_microphone.cpp b/samples/cpp/videocapture_microphone.cpp new file mode 100644 index 000000000000..0c69ec929d4c --- /dev/null +++ b/samples/cpp/videocapture_microphone.cpp @@ -0,0 +1,57 @@ +#include +#include +#include +#include + +using namespace cv; +using namespace std; + +int main(int, char**) +{ + Mat frame; + vector audioData; + VideoCapture cap; + vector params { CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, -1 }; + + cap.open(0, CAP_MSMF, params); + if (!cap.isOpened()) + { + cerr << "ERROR! Can't to open microphone" << endl; + return -1; + } + + const int audioBaseIndex = (int)cap.get(CAP_PROP_AUDIO_BASE_INDEX); + const int numberOfChannels = (int)cap.get(CAP_PROP_AUDIO_TOTAL_CHANNELS); + cout << "CAP_PROP_AUDIO_DATA_DEPTH: " << depthToString((int)cap.get(CAP_PROP_AUDIO_DATA_DEPTH)) << endl; + cout << "CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << cap.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND) << endl; + cout << "CAP_PROP_AUDIO_TOTAL_CHANNELS: " << numberOfChannels << endl; + cout << "CAP_PROP_AUDIO_TOTAL_STREAMS: " << cap.get(CAP_PROP_AUDIO_TOTAL_STREAMS) << endl; + + const double cvTickFreq = getTickFrequency(); + int64 sysTimeCurr = getTickCount(); + int64 sysTimePrev = sysTimeCurr; + while ((sysTimeCurr-sysTimePrev)/cvTickFreq < 10) + { + if (cap.grab()) + { + for (int nCh = 0; nCh < numberOfChannels; nCh++) + { + cap.retrieve(frame, audioBaseIndex+nCh); + audioData.push_back(frame); + sysTimeCurr = getTickCount(); + } + } + else + { + cerr << "Grab error" << endl; + break; + } + } + int numberOfSamles = 0; + for (auto item : audioData) + numberOfSamles+=item.cols; + cout << "Number of samples: " << numberOfSamles << endl; + + return 0; +} From ce68291d83674d7cf4410cd640e723fb403bc406 Mon Sep 17 00:00:00 2001 From: Harvey Date: Thu, 21 Oct 2021 16:47:27 +0800 Subject: [PATCH 312/376] 32bit rgb bmp file should not copy data as rgba --- modules/imgcodecs/src/grfmt_bmp.cpp | 2 +- modules/imgcodecs/test/test_grfmt.cpp | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/imgcodecs/src/grfmt_bmp.cpp b/modules/imgcodecs/src/grfmt_bmp.cpp index 6a56d3ab2d75..aca9c7b2ba82 100644 --- a/modules/imgcodecs/src/grfmt_bmp.cpp +++ b/modules/imgcodecs/src/grfmt_bmp.cpp @@ -180,7 +180,7 @@ bool BmpDecoder::readHeader() throw; } // in 32 bit case alpha channel is used - so require CV_8UC4 type - m_type = iscolor ? (m_bpp == 32 ? CV_8UC4 : CV_8UC3 ) : CV_8UC1; + m_type = iscolor ? ((m_bpp == 32 && m_rle_code != BMP_RGB) ? CV_8UC4 : CV_8UC3 ) : CV_8UC1; m_origin = m_height > 0 ? IPL_ORIGIN_BL : IPL_ORIGIN_TL; m_height = std::abs(m_height); diff --git a/modules/imgcodecs/test/test_grfmt.cpp b/modules/imgcodecs/test/test_grfmt.cpp index fa03ef433306..6866c8d092d6 100644 --- a/modules/imgcodecs/test/test_grfmt.cpp +++ b/modules/imgcodecs/test/test_grfmt.cpp @@ -280,6 +280,16 @@ TEST(Imgcodecs_Bmp, read_rle8) EXPECT_PRED_FORMAT2(cvtest::MatComparator(0, 0), rle, ord); } +TEST(Imgcodecs_Bmp, read_32bit_rgb) +{ + const string root = cvtest::TS::ptr()->get_data_path(); + const string filenameInput = root + "readwrite/test_32bit_rgb.bmp"; + + const Mat img = cv::imread(filenameInput, IMREAD_UNCHANGED); + ASSERT_FALSE(img.empty()); + ASSERT_EQ(CV_8UC3, img.type()); +} + #ifdef HAVE_IMGCODEC_HDR TEST(Imgcodecs_Hdr, regression) { From bac1c6d12f7dde6617050c6cc6567729299b1425 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 21 Oct 2021 09:36:25 +0000 Subject: [PATCH 313/376] hotfix: repair Clang ABI --- modules/core/include/opencv2/core/types.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/core/include/opencv2/core/types.hpp b/modules/core/include/opencv2/core/types.hpp index 1e9a8b629c1d..7dfadb252295 100644 --- a/modules/core/include/opencv2/core/types.hpp +++ b/modules/core/include/opencv2/core/types.hpp @@ -162,7 +162,7 @@ template class Point_ //! default constructor Point_(); Point_(_Tp _x, _Tp _y); -#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 +#if (defined(__GNUC__) && __GNUC__ < 5) && !defined(__clang__) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 Point_(const Point_& pt); Point_(Point_&& pt) CV_NOEXCEPT = default; #elif OPENCV_ABI_COMPATIBILITY < 500 @@ -172,7 +172,7 @@ template class Point_ Point_(const Size_<_Tp>& sz); Point_(const Vec<_Tp, 2>& v); -#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 +#if (defined(__GNUC__) && __GNUC__ < 5) && !defined(__clang__) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 Point_& operator = (const Point_& pt); Point_& operator = (Point_&& pt) CV_NOEXCEPT = default; #elif OPENCV_ABI_COMPATIBILITY < 500 @@ -1186,7 +1186,7 @@ template inline Point_<_Tp>::Point_(_Tp _x, _Tp _y) : x(_x), y(_y) {} -#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 +#if (defined(__GNUC__) && __GNUC__ < 5) && !defined(__clang__) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 template inline Point_<_Tp>::Point_(const Point_& pt) : x(pt.x), y(pt.y) {} @@ -1200,7 +1200,7 @@ template inline Point_<_Tp>::Point_(const Vec<_Tp,2>& v) : x(v[0]), y(v[1]) {} -#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 +#if (defined(__GNUC__) && __GNUC__ < 5) && !defined(__clang__) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 template inline Point_<_Tp>& Point_<_Tp>::operator = (const Point_& pt) { From 7febec49b2e87677d2f5b4d4ad392a4aebda430d Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Thu, 21 Oct 2021 16:56:11 +0300 Subject: [PATCH 314/376] Merge pull request #20614 from mshabunin:use-onevpl-load videoio: use oneVPL load mechanism, encoder bitrate estimation * videoio: updated oneVPL support - use mfxLoad * videoio: advanced bitrate estimation for MFX encoder * videoio: improved MediaSDK/oneVPL/libva detection * videoio(ffmpeg): don't try oneVPL * videoio(test): tune checks of videoio_mfx.read_write_raw tests Co-authored-by: Alexander Alekhin --- modules/videoio/cmake/detect_msdk.cmake | 25 +++++++++----- modules/videoio/src/cap_ffmpeg_hw.hpp | 4 +++ modules/videoio/src/cap_mfx_common.cpp | 35 +++++++++++++++++-- modules/videoio/src/cap_mfx_common.hpp | 45 ++++++++++++++++++++----- modules/videoio/src/cap_mfx_reader.cpp | 16 +++++---- modules/videoio/src/cap_mfx_reader.hpp | 4 +-- modules/videoio/src/cap_mfx_writer.cpp | 31 +++++++++++++++-- modules/videoio/src/cap_mfx_writer.hpp | 4 +-- modules/videoio/test/test_mfx.cpp | 22 ++++++++---- 9 files changed, 148 insertions(+), 38 deletions(-) diff --git a/modules/videoio/cmake/detect_msdk.cmake b/modules/videoio/cmake/detect_msdk.cmake index 83701425e1f8..406c05824bae 100644 --- a/modules/videoio/cmake/detect_msdk.cmake +++ b/modules/videoio/cmake/detect_msdk.cmake @@ -3,16 +3,23 @@ set(MFX_DEFS "") if(NOT HAVE_MFX) find_package(VPL QUIET) if(VPL_FOUND) - set(MFX_INCLUDE_DIRS "") - set(MFX_LIBRARIES "${VPL_IMPORTED_TARGETS}") - set(HAVE_MFX TRUE) - list(APPEND MFX_DEFS "HAVE_ONEVPL") + message(STATUS "VPL_VERSION: ${VPL_VERSION}") + # NOTE: oneVPL since 2021.4 have version 2.4, version 2021.1 does not work + if (VPL_VERSION VERSION_LESS "2021.2" AND VPL_VERSION VERSION_GREATER "2021.0") + message(STATUS "VPL version is too old (${VPL_DIR} : ${VPL_VERSION}) - skipping") + else() + set(MFX_INCLUDE_DIRS "") + set(MFX_LIBRARIES "${VPL_IMPORTED_TARGETS}") + set(HAVE_MFX TRUE) + list(APPEND MFX_DEFS "HAVE_ONEVPL") + endif() endif() endif() +set(paths) if(NOT HAVE_MFX) - set(paths "${MFX_HOME}" ENV "MFX_HOME" ENV "INTELMEDIASDKROOT") + set(paths "${MFX_HOME}" ENV "MFX_HOME" ENV "INTELMEDIASDKROOT" ${paths}) if(MSVC) if(MSVC_VERSION LESS 1900) set(vs_suffix) @@ -31,7 +38,7 @@ if(NOT HAVE_MFX) NO_DEFAULT_PATH) find_library(MFX_LIBRARY NAMES mfx libmfx${vs_suffix} PATHS ${paths} - PATH_SUFFIXES "lib64" "lib/lin_x64" "lib/${vs_arch}" + PATH_SUFFIXES "lib64" "lib/lin_x64" "lib/${vs_arch}" "lib" NO_DEFAULT_PATH) if(MFX_INCLUDE AND MFX_LIBRARY) set(HAVE_MFX TRUE) @@ -46,10 +53,11 @@ if(NOT HAVE_MFX AND PKG_CONFIG_FOUND) endif() if(HAVE_MFX AND UNIX) + set(paths "${VA_ROOT_DIR}" ENV "VA_ROOT_DIR" ${paths}) foreach(mode NO_DEFAULT_PATH "") find_path(MFX_va_INCLUDE va/va.h PATHS ${paths} PATH_SUFFIXES "include" ${mode}) - find_library(MFX_va_LIBRARY va PATHS ${paths} PATH_SUFFIXES "lib64" "lib/lin_x64" ${mode}) - find_library(MFX_va_drm_LIBRARY va-drm PATHS ${paths} PATH_SUFFIXES "lib64" "lib/lin_x64" ${mode}) + find_library(MFX_va_LIBRARY va PATHS ${paths} PATH_SUFFIXES "lib64" "lib/lin_x64" "lib" ${mode}) + find_library(MFX_va_drm_LIBRARY va-drm PATHS ${paths} PATH_SUFFIXES "lib64" "lib/lin_x64" "lib" ${mode}) if(MFX_va_INCLUDE AND MFX_va_LIBRARY AND MFX_va_drm_LIBRARY) list(APPEND MFX_INCLUDE_DIRS "${MFX_va_INCLUDE}") list(APPEND MFX_LIBRARIES "${MFX_va_LIBRARY}" "${MFX_va_drm_LIBRARY}") @@ -61,6 +69,7 @@ if(HAVE_MFX AND UNIX) unset(MFX_va_drm_LIBRARY CACHE) endforeach() if(NOT(MFX_va_INCLUDE AND MFX_va_LIBRARY AND MFX_va_drm_LIBRARY)) + message(STATUS "libva not found - turning MFX OFF") set(HAVE_MFX FALSE) endif() diff --git a/modules/videoio/src/cap_ffmpeg_hw.hpp b/modules/videoio/src/cap_ffmpeg_hw.hpp index 2a7b2497395d..31db5fdd5d65 100644 --- a/modules/videoio/src/cap_ffmpeg_hw.hpp +++ b/modules/videoio/src/cap_ffmpeg_hw.hpp @@ -13,6 +13,10 @@ #endif #include +#if defined(HAVE_MFX) && defined(HAVE_ONEVPL) +#undef HAVE_MFX // libav's hwcontext_qsv.h doesn't expect oneVPL headers +#endif + #ifdef HAVE_D3D11 #define D3D11_NO_HELPERS #include diff --git a/modules/videoio/src/cap_mfx_common.cpp b/modules/videoio/src/cap_mfx_common.cpp index aea5c5856143..d0f94931f37c 100644 --- a/modules/videoio/src/cap_mfx_common.cpp +++ b/modules/videoio/src/cap_mfx_common.cpp @@ -14,11 +14,13 @@ using namespace std; using namespace cv; +#ifndef HAVE_ONEVPL static mfxIMPL getImpl() { static const size_t res = utils::getConfigurationParameterSizeT("OPENCV_VIDEOIO_MFX_IMPL", MFX_IMPL_AUTO_ANY); return (mfxIMPL)res; } +#endif static size_t getExtraSurfaceNum() { @@ -32,19 +34,46 @@ static size_t getPoolTimeoutSec() return res; } +#ifdef HAVE_ONEVPL +// oneVPL loader singleton (HW implementation only) +static mfxLoader setupVPLLoader() +{ + mfxLoader instance = MFXLoad(); + mfxConfig cfg = MFXCreateConfig(instance); + mfxVariant impl; + impl.Type = MFX_VARIANT_TYPE_U32; + impl.Data.U32 = MFX_IMPL_TYPE_HARDWARE; + MFXSetConfigFilterProperty(cfg, (const mfxU8*)"mfxImplDescription.Impl", impl); + DBG(cerr << "MFX Load: " << instance << endl); + return instance; +} + +mfxLoader getVPLLoaderInstance() +{ + static mfxLoader instance = setupVPLLoader(); + return instance; +} +#endif + //================================================================================================== -bool DeviceHandler::init(MFXVideoSession &session) +bool DeviceHandler::init(MFXVideoSession_WRAP &session) { mfxStatus res = MFX_ERR_NONE; - mfxIMPL impl = getImpl(); mfxVersion ver = { {19, 1} }; +#ifdef HAVE_ONEVPL + res = session.CreateSession(); + DBG(cout << "MFX CreateSession: " << res << endl); +#else + mfxIMPL impl = getImpl(); + res = session.Init(impl, &ver); DBG(cout << "MFX SessionInit: " << res << endl); res = session.QueryIMPL(&impl); DBG(cout << "MFX QueryIMPL: " << res << " => " << asHex(impl) << endl); +#endif res = session.QueryVersion(&ver); DBG(cout << "MFX QueryVersion: " << res << " => " << ver.Major << "." << ver.Minor << endl); @@ -77,7 +106,7 @@ VAHandle::~VAHandle() { } } -bool VAHandle::initDeviceSession(MFXVideoSession &session) { +bool VAHandle::initDeviceSession(MFXVideoSession_WRAP &session) { int majorVer = 0, minorVer = 0; VAStatus va_res = vaInitialize(display, &majorVer, &minorVer); DBG(cout << "vaInitialize: " << va_res << endl << majorVer << '.' << minorVer << endl); diff --git a/modules/videoio/src/cap_mfx_common.hpp b/modules/videoio/src/cap_mfx_common.hpp index 2830592163f4..9824e89dc5fa 100644 --- a/modules/videoio/src/cap_mfx_common.hpp +++ b/modules/videoio/src/cap_mfx_common.hpp @@ -12,12 +12,18 @@ #include #include +CV_SUPPRESS_DEPRECATED_START +# if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable:4201) // nonstandard extension used: nameless struct/union +# endif #ifdef HAVE_ONEVPL # include # include # include # include # include +# include #else # include # include @@ -28,6 +34,10 @@ # include # endif #endif +# if defined(_MSC_VER) +# pragma warning(pop) +# endif +CV_SUPPRESS_DEPRECATED_END // // // Debug helpers // @@ -176,10 +186,29 @@ inline void cleanup(T * &ptr) //================================================================================================== +#ifdef HAVE_ONEVPL +mfxLoader getVPLLoaderInstance(); +#endif + +//================================================================================================== + +class MFXVideoSession_WRAP : public MFXVideoSession +{ +#ifdef HAVE_ONEVPL +public: + mfxStatus CreateSession() + { + return MFXCreateSession(getVPLLoaderInstance(), 0, &m_session); + } +#endif +}; + +//================================================================================================== + class Plugin { public: - static Plugin * loadEncoderPlugin(MFXVideoSession &session, mfxU32 codecId) + static Plugin * loadEncoderPlugin(MFXVideoSession_WRAP &session, mfxU32 codecId) { #ifdef HAVE_MFX_PLUGIN static const mfxPluginUID hevc_enc_uid = { 0x6f, 0xad, 0xc7, 0x91, 0xa0, 0xc2, 0xeb, 0x47, 0x9a, 0xb6, 0xdc, 0xd5, 0xea, 0x9d, 0xa3, 0x47 }; @@ -190,7 +219,7 @@ class Plugin #endif return 0; } - static Plugin * loadDecoderPlugin(MFXVideoSession &session, mfxU32 codecId) + static Plugin * loadDecoderPlugin(MFXVideoSession_WRAP &session, mfxU32 codecId) { #ifdef HAVE_MFX_PLUGIN static const mfxPluginUID hevc_dec_uid = { 0x33, 0xa6, 0x1c, 0x0b, 0x4c, 0x27, 0x45, 0x4c, 0xa8, 0xd8, 0x5d, 0xde, 0x75, 0x7c, 0x6f, 0x8e }; @@ -213,9 +242,9 @@ class Plugin mfxStatus res; private: #ifdef HAVE_MFX_PLUGIN - MFXVideoSession &session; + MFXVideoSession_WRAP &session; mfxPluginUID uid; - Plugin(MFXVideoSession &_session, mfxPluginUID _uid) : session(_session), uid(_uid) + Plugin(MFXVideoSession_WRAP &_session, mfxPluginUID _uid) : session(_session), uid(_uid) { res = MFXVideoUSER_Load(session, &uid, 1); } @@ -298,9 +327,9 @@ class SurfacePool class DeviceHandler { public: virtual ~DeviceHandler() {} - bool init(MFXVideoSession &session); + bool init(MFXVideoSession_WRAP &session); protected: - virtual bool initDeviceSession(MFXVideoSession &session) = 0; + virtual bool initDeviceSession(MFXVideoSession_WRAP &session) = 0; }; @@ -340,7 +369,7 @@ class VAHandle : public DeviceHandler { private: VAHandle(const VAHandle &); VAHandle &operator=(const VAHandle &); - bool initDeviceSession(MFXVideoSession &session) CV_OVERRIDE; + bool initDeviceSession(MFXVideoSession_WRAP &session) CV_OVERRIDE; private: VADisplay display; int file; @@ -360,7 +389,7 @@ class DXHandle : public DeviceHandler { private: DXHandle(const DXHandle &); DXHandle &operator=(const DXHandle &); - bool initDeviceSession(MFXVideoSession &) CV_OVERRIDE { return true; } + bool initDeviceSession(MFXVideoSession_WRAP &) CV_OVERRIDE { return true; } }; #endif // _WIN32 diff --git a/modules/videoio/src/cap_mfx_reader.cpp b/modules/videoio/src/cap_mfx_reader.cpp index 330f9bd34081..3c9d6327f88c 100644 --- a/modules/videoio/src/cap_mfx_reader.cpp +++ b/modules/videoio/src/cap_mfx_reader.cpp @@ -41,7 +41,7 @@ VideoCapture_IntelMFX::VideoCapture_IntelMFX(const cv::String &filename) // Init device and session deviceHandler = createDeviceHandler(); - session = new MFXVideoSession(); + session = new MFXVideoSession_WRAP(); if (!deviceHandler->init(*session)) { MSG(cerr << "MFX: Can't initialize session" << endl); @@ -87,11 +87,11 @@ VideoCapture_IntelMFX::VideoCapture_IntelMFX(const cv::String &filename) return; } - // Adjust parameters + // Adjust parameters - COMMENTED: h265 decoder resets crop size to 0 (oneVPL/Win) - res = decoder->Query(¶ms, ¶ms); - DBG(cout << "MFX Query: " << res << endl << params.mfx << params.mfx.FrameInfo); - CV_Assert(res >= MFX_ERR_NONE); + //res = decoder->Query(¶ms, ¶ms); + //DBG(cout << "MFX Query: " << res << endl << params.mfx << params.mfx.FrameInfo); + //CV_Assert(res >= MFX_ERR_NONE); // Init surface pool @@ -105,7 +105,7 @@ VideoCapture_IntelMFX::VideoCapture_IntelMFX(const cv::String &filename) // Init decoder res = decoder->Init(¶ms); - DBG(cout << "MFX Init: " << res << endl << params.mfx.FrameInfo); + DBG(cout << "MFX decoder Init: " << res << endl << params.mfx.FrameInfo); if (res < MFX_ERR_NONE) { MSG(cerr << "MFX: Failed to init decoder: " << res << endl); @@ -113,6 +113,10 @@ VideoCapture_IntelMFX::VideoCapture_IntelMFX(const cv::String &filename) } frameSize = Size(params.mfx.FrameInfo.CropW, params.mfx.FrameInfo.CropH); + if (frameSize == Size(0, 0)) // sometimes Crop size is 0 + { + frameSize = Size(params.mfx.FrameInfo.Width, params.mfx.FrameInfo.Height); + } good = true; } diff --git a/modules/videoio/src/cap_mfx_reader.hpp b/modules/videoio/src/cap_mfx_reader.hpp index 122bc5076323..6350c690ac1d 100644 --- a/modules/videoio/src/cap_mfx_reader.hpp +++ b/modules/videoio/src/cap_mfx_reader.hpp @@ -8,7 +8,7 @@ #include "precomp.hpp" -class MFXVideoSession; +class MFXVideoSession_WRAP; class Plugin; class DeviceHandler; class ReadBitstream; @@ -27,7 +27,7 @@ class VideoCapture_IntelMFX : public cv::IVideoCapture bool isOpened() const CV_OVERRIDE; int getCaptureDomain() CV_OVERRIDE; private: - MFXVideoSession *session; + MFXVideoSession_WRAP *session; Plugin *plugin; DeviceHandler *deviceHandler; ReadBitstream *bs; diff --git a/modules/videoio/src/cap_mfx_writer.cpp b/modules/videoio/src/cap_mfx_writer.cpp index 3bcca33d0961..51157e9ba1cf 100644 --- a/modules/videoio/src/cap_mfx_writer.cpp +++ b/modules/videoio/src/cap_mfx_writer.cpp @@ -11,6 +11,31 @@ using namespace std; using namespace cv; +static float estimateBitrate(int codecId, size_t pixelNum, float fps) +{ + float bitrate = 0.f; + const float mp = pixelNum / 1000000.f; + if (codecId == MFX_CODEC_MPEG2) + { + bitrate = (mp * 43) * fps + 360; + } + else if (codecId == MFX_CODEC_AVC) + { + bitrate = (mp * 140 + 19) * pow(fps, 0.60f); + } + else if (codecId == MFX_CODEC_HEVC) + { + bitrate = (mp * 63 + 45) * pow(fps, 0.60f); + } + else + { + MSG(cerr << "MFX encoder Bitrate estimation FAILED" << endl); + } + DBG(cout << "MFX encoder Bitrate estimation (" << mp << " MP x " << fps << " fps): " << bitrate << endl); + return bitrate; + +} + static size_t getBitrateDivisor() { static const size_t res = utils::getConfigurationParameterSizeT("OPENCV_VIDEOIO_MFX_BITRATE_DIVISOR", 300); @@ -61,7 +86,7 @@ VideoWriter_IntelMFX::VideoWriter_IntelMFX(const String &filename, int _fourcc, // Init device and session deviceHandler = createDeviceHandler(); - session = new MFXVideoSession(); + session = new MFXVideoSession_WRAP(); if (!deviceHandler->init(*session)) { MSG(cerr << "MFX: Can't initialize session" << endl); @@ -90,7 +115,7 @@ VideoWriter_IntelMFX::VideoWriter_IntelMFX(const String &filename, int _fourcc, memset(¶ms, 0, sizeof(params)); params.mfx.CodecId = codecId; params.mfx.TargetUsage = MFX_TARGETUSAGE_BALANCED; - params.mfx.TargetKbps = saturate_cast((frameSize.area() * fps) / (42.6666 * getBitrateDivisor())); // TODO: set in options + params.mfx.TargetKbps = saturate_cast(estimateBitrate(codecId, frameSize.area(), (float)fps) * 300 / getBitrateDivisor()); // TODO: set in options params.mfx.RateControlMethod = MFX_RATECONTROL_VBR; params.mfx.FrameInfo.FrameRateExtN = cvRound(fps * 1000); params.mfx.FrameInfo.FrameRateExtD = 1000; @@ -122,7 +147,7 @@ VideoWriter_IntelMFX::VideoWriter_IntelMFX(const String &filename, int _fourcc, // Init encoder res = encoder->Init(¶ms); - DBG(cout << "MFX Init: " << res << endl << params.mfx.FrameInfo); + DBG(cout << "MFX encoder Init: " << res << endl << params.mfx.FrameInfo); if (res < MFX_ERR_NONE) { MSG(cerr << "MFX: Failed to init encoder: " << res << endl); diff --git a/modules/videoio/src/cap_mfx_writer.hpp b/modules/videoio/src/cap_mfx_writer.hpp index f4913a8f311e..fff2a6387f05 100644 --- a/modules/videoio/src/cap_mfx_writer.hpp +++ b/modules/videoio/src/cap_mfx_writer.hpp @@ -7,7 +7,7 @@ #include "precomp.hpp" -class MFXVideoSession; +class MFXVideoSession_WRAP; class Plugin; class DeviceHandler; class WriteBitstream; @@ -33,7 +33,7 @@ class VideoWriter_IntelMFX : public cv::IVideoWriter VideoWriter_IntelMFX & operator=(const VideoWriter_IntelMFX &); private: - MFXVideoSession *session; + MFXVideoSession_WRAP *session; Plugin *plugin; DeviceHandler *deviceHandler; WriteBitstream *bs; diff --git a/modules/videoio/test/test_mfx.cpp b/modules/videoio/test/test_mfx.cpp index 2048fe5af9da..032d5a96e025 100644 --- a/modules/videoio/test/test_mfx.cpp +++ b/modules/videoio/test/test_mfx.cpp @@ -97,6 +97,11 @@ TEST_P(videoio_mfx, read_write_raw) const String filename = cv::tempfile(ext); const int fourcc = fourccByExt(ext); + // For some reason MPEG2 codec does not work well with this particular videostream at 1 FPS + // even with large bitrate values. Thus skipping this case. + if (FPS == 1. && fourcc == VideoWriter::fourcc('M', 'P', 'G', '2')) + throw SkipTestException("This configuration is not supported"); + bool isColor = true; std::queue goodFrames; @@ -120,24 +125,29 @@ TEST_P(videoio_mfx, read_write_raw) ASSERT_TRUE(cap.isOpened()); EXPECT_EQ(FRAME_SIZE.width, cap.get(CAP_PROP_FRAME_WIDTH)); EXPECT_EQ(FRAME_SIZE.height, cap.get(CAP_PROP_FRAME_HEIGHT)); + double psnrThreshold = (fourcc == VideoWriter::fourcc('M', 'P', 'G', '2')) ? 27.0 : 29.5; // experimentally chosen value for (int i = 0; i < FRAME_COUNT; ++i) { + SCOPED_TRACE(i); ASSERT_TRUE(cap.read(frame)); ASSERT_FALSE(frame.empty()); ASSERT_EQ(FRAME_SIZE.width, frame.cols); ASSERT_EQ(FRAME_SIZE.height, frame.rows); // verify ASSERT_NE(goodFrames.size(), 0u); - const Mat &goodFrame = goodFrames.front(); + const Mat goodFrame = goodFrames.front(); goodFrames.pop(); EXPECT_EQ(goodFrame.depth(), frame.depth()); EXPECT_EQ(goodFrame.channels(), frame.channels()); EXPECT_EQ(goodFrame.type(), frame.type()); double psnr = cvtest::PSNR(goodFrame, frame); - if (fourcc == VideoWriter::fourcc('M', 'P', 'G', '2')) - EXPECT_GT(psnr, 31); // experimentally chosen value - else - EXPECT_GT(psnr, 33); // experimentally chosen value - goodFrames.pop(); + if ((i == 1 || i == 4) && fourcc == VideoWriter::fourcc('H', '2', '6', '5')) + { + // ignore bugs of some HW/SW configurations: + // - (added 2021-10) i7-11700K, Win10, oneVPL 2021.4.0 / 2021.6.0 + std::cout << "SKIP: bypass frame content check: i=" << i << " psnr=" << psnr << ", expected to be >= " << psnrThreshold << std::endl; + continue; + } + EXPECT_GE(psnr, psnrThreshold); } EXPECT_FALSE(cap.read(frame)); EXPECT_TRUE(frame.empty()); From d21622bef4bff56f322f56f7da3af28a8a2d7104 Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Thu, 21 Oct 2021 18:12:51 +0300 Subject: [PATCH 315/376] fix findMinEnclosingTriangle and add tests --- .../imgproc/src/min_enclosing_triangle.cpp | 3 +- modules/imgproc/test/test_convhull.cpp | 33 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/modules/imgproc/src/min_enclosing_triangle.cpp b/modules/imgproc/src/min_enclosing_triangle.cpp index 265187141256..7bd15fced0cc 100644 --- a/modules/imgproc/src/min_enclosing_triangle.cpp +++ b/modules/imgproc/src/min_enclosing_triangle.cpp @@ -317,8 +317,9 @@ namespace minEnclosingTriangle { */ static void findMinEnclosingTriangle(cv::InputArray points, CV_OUT cv::OutputArray triangle, CV_OUT double &area) { - std::vector resultingTriangle, polygon; CV_Assert(!points.empty()); + std::vector resultingTriangle; + cv::Mat polygon; convexHull(points, polygon, true, true); findMinEnclosingTriangle(polygon, resultingTriangle, area); cv::Mat(resultingTriangle).copyTo(triangle); diff --git a/modules/imgproc/test/test_convhull.cpp b/modules/imgproc/test/test_convhull.cpp index dee3769762c9..bc5c940827ca 100644 --- a/modules/imgproc/test/test_convhull.cpp +++ b/modules/imgproc/test/test_convhull.cpp @@ -2457,5 +2457,38 @@ TEST(Imgproc_minAreaRect, reproducer_19769) EXPECT_TRUE(checkMinAreaRect(rr, contour)) << rr.center << " " << rr.size << " " << rr.angle; } +TEST(Imgproc_minEnclosingTriangle, regression_17585) +{ + const int N = 3; + float pts_[N][2] = { {0, 0}, {0, 1}, {1, 1} }; + cv::Mat points(N, 2, CV_32FC1, static_cast(pts_)); + vector triangle; + + EXPECT_NO_THROW(minEnclosingTriangle(points, triangle)); +} + +TEST(Imgproc_minEnclosingTriangle, regression_20890) +{ + vector points; + points.push_back(Point(0, 0)); + points.push_back(Point(0, 1)); + points.push_back(Point(1, 1)); + vector triangle; + + EXPECT_NO_THROW(minEnclosingTriangle(points, triangle)); +} + +TEST(Imgproc_minEnclosingTriangle, regression_mat_with_diff_channels) +{ + const int N = 3; + float pts_[N][2] = { {0, 0}, {0, 1}, {1, 1} }; + cv::Mat points1xN(1, N, CV_32FC2, static_cast(pts_)); + cv::Mat pointsNx1(N, 1, CV_32FC2, static_cast(pts_)); + vector triangle; + + EXPECT_NO_THROW(minEnclosingTriangle(points1xN, triangle)); + EXPECT_NO_THROW(minEnclosingTriangle(pointsNx1, triangle)); +} + }} // namespace /* End of file. */ From d376fe9e173379630d869faa8e21c07bf78cd199 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Thu, 21 Oct 2021 19:12:03 +0300 Subject: [PATCH 316/376] Merge pull request #20921 from sivanov-work:atl_patch G-API: FIX OpenVINO build - Add CComPtr RAII replacement into G-API sample * Add CComPtr RAII replacement into G-API sample * Remove completely `atlbase.h' --- .../gapi/samples/onevpl_infer_single_roi.cpp | 47 +++++++++++++------ .../onevpl/cfg_param_device_selector.cpp | 13 +++-- modules/gapi/src/streaming/onevpl/utils.hpp | 21 +++++++++ 3 files changed, 60 insertions(+), 21 deletions(-) diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index deca86f1cac8..9da64c99d37a 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -25,7 +25,6 @@ #define D3D11_NO_HELPERS #define NOMINMAX #include -#include #include #pragma comment(lib, "dxgi") #undef NOMINMAX @@ -64,9 +63,29 @@ std::string get_weights_path(const std::string &model_path) { #ifdef HAVE_DIRECTX #ifdef HAVE_D3D11 -using AccelParamsType = std::tuple, CComPtr>; +// Since ATL headers might not be available on specific MSVS Build Tools +// we use simple `CComPtr` implementation like as `ComPtrGuard` +// which is not supposed to be the full functional replacement of `CComPtr` +// and it uses as RAII to make sure utilization is correct +template +void release(COMNonManageableType *ptr) { + if (ptr) { + ptr->Release(); + } +} + +template +using ComPtrGuard = std::unique_ptr)>; + +template +ComPtrGuard createCOMPtrGuard(COMNonManageableType *ptr = nullptr) { + return ComPtrGuard {ptr, &release}; +} + + +using AccelParamsType = std::tuple, ComPtrGuard>; -AccelParamsType create_device_with_ctx(CComPtr adapter) { +AccelParamsType create_device_with_ctx(IDXGIAdapter* adapter) { UINT flags = 0; D3D_FEATURE_LEVEL feature_levels[] = { D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0, @@ -85,7 +104,8 @@ AccelParamsType create_device_with_ctx(CComPtr adapter) { std::to_string(HRESULT_CODE(err))); } - return std::make_tuple(ret_device_ptr, ret_ctx_ptr); + return std::make_tuple(createCOMPtrGuard(ret_device_ptr), + createCOMPtrGuard(ret_ctx_ptr)); } #endif // HAVE_D3D11 #endif // HAVE_DIRECTX @@ -270,12 +290,11 @@ int main(int argc, char *argv[]) { #ifdef HAVE_INF_ENGINE #ifdef HAVE_DIRECTX #ifdef HAVE_D3D11 - CComPtr dx11_dev; - CComPtr dx11_ctx; + auto dx11_dev = createCOMPtrGuard(); + auto dx11_ctx = createCOMPtrGuard(); if (device_id.find("GPU") != std::string::npos) { - CComPtr adapter_factory; - CComPtr intel_adapters; + auto adapter_factory = createCOMPtrGuard(); { IDXGIFactory* out_factory = nullptr; HRESULT err = CreateDXGIFactory(__uuidof(IDXGIFactory), @@ -284,10 +303,10 @@ int main(int argc, char *argv[]) { std::cerr << "Cannot create CreateDXGIFactory, error: " << HRESULT_CODE(err) << std::endl; return -1; } - adapter_factory.Attach(out_factory); + adapter_factory = createCOMPtrGuard(out_factory); } - CComPtr intel_adapter; + auto intel_adapter = createCOMPtrGuard(); UINT adapter_index = 0; const unsigned int refIntelVendorID = 0x8086; IDXGIAdapter* out_adapter = nullptr; @@ -296,7 +315,7 @@ int main(int argc, char *argv[]) { DXGI_ADAPTER_DESC desc{}; out_adapter->GetDesc(&desc); if (desc.VendorId == refIntelVendorID) { - intel_adapter.Attach(out_adapter); + intel_adapter = createCOMPtrGuard(out_adapter); break; } ++adapter_index; @@ -307,9 +326,9 @@ int main(int argc, char *argv[]) { return -1; } - std::tie(dx11_dev, dx11_ctx) = create_device_with_ctx(intel_adapter); - accel_device_ptr = reinterpret_cast(dx11_dev.p); - accel_ctx_ptr = reinterpret_cast(dx11_ctx.p); + std::tie(dx11_dev, dx11_ctx) = create_device_with_ctx(intel_adapter.get()); + accel_device_ptr = reinterpret_cast(dx11_dev.get()); + accel_ctx_ptr = reinterpret_cast(dx11_ctx.get()); // put accel type description for VPL source source_cfgs.push_back(cfg::create_from_string( diff --git a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp index c8fd49c1adcc..0bdec7098603 100644 --- a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp +++ b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp @@ -10,6 +10,7 @@ #include #include "streaming/onevpl/cfg_param_device_selector.hpp" +#include "streaming/onevpl/utils.hpp" #include "logger.hpp" #ifdef HAVE_DIRECTX @@ -19,7 +20,6 @@ // get rid of generate macro max/min/etc from DX side #define D3D11_NO_HELPERS #define NOMINMAX -#include #include #include #pragma comment(lib, "dxgi") @@ -113,8 +113,7 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) : }; D3D_FEATURE_LEVEL featureLevel; - CComPtr adapter_factory; - CComPtr intel_adapters; + auto adapter_factory = createCOMPtrGuard(); { IDXGIFactory* out_factory = nullptr; HRESULT err = CreateDXGIFactory(__uuidof(IDXGIFactory), @@ -122,10 +121,10 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) : if (FAILED(err)) { throw std::runtime_error("Cannot create CreateDXGIFactory, error: " + std::to_string(HRESULT_CODE(err))); } - adapter_factory.Attach(out_factory); + adapter_factory = createCOMPtrGuard(out_factory); } - CComPtr intel_adapter; + auto intel_adapter = createCOMPtrGuard(); UINT adapter_index = 0; const unsigned int refIntelVendorID = 0x8086; IDXGIAdapter* out_adapter = nullptr; @@ -134,7 +133,7 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) : DXGI_ADAPTER_DESC desc{}; out_adapter->GetDesc(&desc); if (desc.VendorId == refIntelVendorID) { - intel_adapter.Attach(out_adapter); + intel_adapter = createCOMPtrGuard(out_adapter); break; } ++adapter_index; @@ -145,7 +144,7 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) : } // Create the Direct3D 11 API device object and a corresponding context. - HRESULT err = D3D11CreateDevice(intel_adapter, + HRESULT err = D3D11CreateDevice(intel_adapter.get(), D3D_DRIVER_TYPE_UNKNOWN, nullptr, creationFlags, featureLevels, ARRAYSIZE(featureLevels), diff --git a/modules/gapi/src/streaming/onevpl/utils.hpp b/modules/gapi/src/streaming/onevpl/utils.hpp index 94f5a249e81b..19c1b135dc50 100644 --- a/modules/gapi/src/streaming/onevpl/utils.hpp +++ b/modules/gapi/src/streaming/onevpl/utils.hpp @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -26,6 +27,26 @@ namespace gapi { namespace wip { namespace onevpl { +// Since ATL headers might not be available on specific MSVS Build Tools +// we use simple `CComPtr` implementation like as `ComPtrGuard` +// which is not supposed to be the full functional replacement of `CComPtr` +// and it uses as RAII to make sure utilization is correct +template +void release(COMNonManageableType *ptr) { + if (ptr) { + ptr->Release(); + } +} + +template +using ComPtrGuard = std::unique_ptr)>; + +template +ComPtrGuard createCOMPtrGuard(COMNonManageableType *ptr = nullptr) { + return ComPtrGuard {ptr, &release}; +} + + const char* mfx_impl_to_cstr(const mfxIMPL impl); mfxIMPL cstr_to_mfx_impl(const char* cstr); From 0a58d6812ac053d5f52142f6f6f3684d2beec800 Mon Sep 17 00:00:00 2001 From: atrutnev Date: Fri, 22 Oct 2021 11:06:09 +0300 Subject: [PATCH 317/376] fix gkernel Doxygen documentation --- modules/gapi/include/opencv2/gapi/gkernel.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gapi/include/opencv2/gapi/gkernel.hpp b/modules/gapi/include/opencv2/gapi/gkernel.hpp index cfac552bfee0..ca942495ab31 100644 --- a/modules/gapi/include/opencv2/gapi/gkernel.hpp +++ b/modules/gapi/include/opencv2/gapi/gkernel.hpp @@ -719,7 +719,7 @@ namespace gapi { * @{ */ /** - * @brief cv::use_only() is a special combinator which hints G-API to use only + * @brief cv::gapi::use_only() is a special combinator which hints G-API to use only * kernels specified in cv::GComputation::compile() (and not to extend kernels available by * default with that package). */ From 4214fc76f17c1fa58df9a90f6c8146852af4f82b Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Fri, 22 Oct 2021 18:22:53 +0800 Subject: [PATCH 318/376] Offer more accurate perf test on native --- samples/dnn/classification.cpp | 38 +++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/samples/dnn/classification.cpp b/samples/dnn/classification.cpp index 8af62e7e8b1f..12451e4336ef 100644 --- a/samples/dnn/classification.cpp +++ b/samples/dnn/classification.cpp @@ -4,6 +4,7 @@ #include #include #include +#include "sys/time.h" #include "common.hpp" @@ -142,28 +143,37 @@ int main(int argc, char** argv) net.setInput(blob); //! [Set input blob] //! [Make forward pass] - Mat prob = net.forward(); - //! [Make forward pass] - - //! [Get a class with a highest score] - Point classIdPoint; + double t_sum = 0.0; + double t; + int classId; double confidence; - minMaxLoc(prob.reshape(1, 1), 0, &confidence, 0, &classIdPoint); - int classId = classIdPoint.x; - //! [Get a class with a highest score] - - // Put efficiency information. - std::vector layersTimes; - double freq = getTickFrequency() / 1000; - double t = net.getPerfProfile(layersTimes) / freq; + for(int i = 0; i < 100; i++) { + Mat prob = net.forward(); + + //! [Make forward pass] + + //! [Get a class with a highest score] + Point classIdPoint; + minMaxLoc(prob.reshape(1, 1), 0, &confidence, 0, &classIdPoint); + classId = classIdPoint.x; + //! [Get a class with a highest score] + + // Put efficiency information. + std::vector layersTimes; + double freq = getTickFrequency() / 1000; + t = net.getPerfProfile(layersTimes) / freq; + t_sum += t; + } std::string label = format("Inference time: %.2f ms", t); + std::string label2 = format("Average time of 100 rounds: %.2f ms", t_sum/100); putText(frame, label, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + putText(frame, label2, Point(0, 35), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); // Print predicted class. label = format("%s: %.4f", (classes.empty() ? format("Class #%d", classId).c_str() : classes[classId].c_str()), confidence); - putText(frame, label, Point(0, 40), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + putText(frame, label, Point(0, 55), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); imshow(kWinName, frame); } From 5c15ea78d6322f71d000032cedb17853e85f012a Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Fri, 22 Oct 2021 18:30:52 +0800 Subject: [PATCH 319/376] Add better perf tests for both native and web --- .../js_image_classification_webnn_polyfill.html | 8 ++++++-- .../js_image_classification_webnn_electron.html | 9 ++++++--- samples/dnn/classification.cpp | 7 ++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/doc/js_tutorials/js_assets/js_image_classification_webnn_polyfill.html b/doc/js_tutorials/js_assets/js_image_classification_webnn_polyfill.html index 6ef5e1517be5..271fc64ae51f 100644 --- a/doc/js_tutorials/js_assets/js_image_classification_webnn_polyfill.html +++ b/doc/js_tutorials/js_assets/js_image_classification_webnn_polyfill.html @@ -127,13 +127,17 @@

Model Info:

let net = cv.readNet(configPath, modelPath); net.setPreferableBackend(6); net.setInput(input); + let result = net.forward(); const start = performance.now(); - const result = net.forward(); + for (i=0;i<1000;i++) + { + result = net.forward(); + } const time = performance.now()-start; const probs = softmax(result); const classes = getTopClasses(probs, labels); - updateResult(classes, time); + updateResult(classes, time/1000); input.delete(); net.delete(); result.delete(); diff --git a/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html b/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html index c8119247f94d..ca715fdfee03 100644 --- a/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html +++ b/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html @@ -126,14 +126,17 @@

Model Info:

let net = cv.readNet(configPath, modelPath); net.setPreferableBackend(6); net.setInput(input); - net.forward(); + let result = net.forward(); const start = performance.now(); - const result = net.forward(); + for (i=0;i<1000;i++) + { + result = net.forward(); + } const time = performance.now()-start; const probs = softmax(result); const classes = getTopClasses(probs, labels); - updateResult(classes, time); + updateResult(classes, time/1000); input.delete(); net.delete(); result.delete(); diff --git a/samples/dnn/classification.cpp b/samples/dnn/classification.cpp index 12451e4336ef..76dd6601e217 100644 --- a/samples/dnn/classification.cpp +++ b/samples/dnn/classification.cpp @@ -147,8 +147,9 @@ int main(int argc, char** argv) double t; int classId; double confidence; - for(int i = 0; i < 100; i++) { - Mat prob = net.forward(); + Mat prob = net.forward(); + for(int i = 0; i < 1000; i++) { + prob = net.forward(); //! [Make forward pass] @@ -165,7 +166,7 @@ int main(int argc, char** argv) t_sum += t; } std::string label = format("Inference time: %.2f ms", t); - std::string label2 = format("Average time of 100 rounds: %.2f ms", t_sum/100); + std::string label2 = format("Average time of 100 rounds: %.2f ms", t_sum/1000); putText(frame, label, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); putText(frame, label2, Point(0, 35), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); From d5ea2c276be5cdbb425821f1e34d4103f9cc4b4a Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Fri, 22 Oct 2021 18:42:52 +0800 Subject: [PATCH 320/376] Modify per tests for better results --- .../js_assets/js_image_classification_webnn_polyfill.html | 4 ++-- .../js_image_classification_webnn_electron.html | 4 ++-- samples/dnn/classification.cpp | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/js_tutorials/js_assets/js_image_classification_webnn_polyfill.html b/doc/js_tutorials/js_assets/js_image_classification_webnn_polyfill.html index 271fc64ae51f..a8910c3fd38a 100644 --- a/doc/js_tutorials/js_assets/js_image_classification_webnn_polyfill.html +++ b/doc/js_tutorials/js_assets/js_image_classification_webnn_polyfill.html @@ -129,7 +129,7 @@

Model Info:

net.setInput(input); let result = net.forward(); const start = performance.now(); - for (i=0;i<1000;i++) + for (i=0;i<200;i++) { result = net.forward(); } @@ -137,7 +137,7 @@

Model Info:

const probs = softmax(result); const classes = getTopClasses(probs, labels); - updateResult(classes, time/1000); + updateResult(classes, time/200); input.delete(); net.delete(); result.delete(); diff --git a/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html b/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html index ca715fdfee03..0da44330c942 100644 --- a/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html +++ b/doc/js_tutorials/js_assets/webnn-electron/js_image_classification_webnn_electron.html @@ -128,7 +128,7 @@

Model Info:

net.setInput(input); let result = net.forward(); const start = performance.now(); - for (i=0;i<1000;i++) + for (i=0;i<200;i++) { result = net.forward(); } @@ -136,7 +136,7 @@

Model Info:

const probs = softmax(result); const classes = getTopClasses(probs, labels); - updateResult(classes, time/1000); + updateResult(classes, time/200); input.delete(); net.delete(); result.delete(); diff --git a/samples/dnn/classification.cpp b/samples/dnn/classification.cpp index 76dd6601e217..7c05410d768d 100644 --- a/samples/dnn/classification.cpp +++ b/samples/dnn/classification.cpp @@ -148,7 +148,7 @@ int main(int argc, char** argv) int classId; double confidence; Mat prob = net.forward(); - for(int i = 0; i < 1000; i++) { + for(int i = 0; i < 200; i++) { prob = net.forward(); //! [Make forward pass] @@ -166,7 +166,7 @@ int main(int argc, char** argv) t_sum += t; } std::string label = format("Inference time: %.2f ms", t); - std::string label2 = format("Average time of 100 rounds: %.2f ms", t_sum/1000); + std::string label2 = format("Average time of 200 rounds: %.2f ms", t_sum/200); putText(frame, label, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); putText(frame, label2, Point(0, 35), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); From a6f57175679f230fb9e83ee8703594210e2e5017 Mon Sep 17 00:00:00 2001 From: berak Date: Thu, 21 Oct 2021 11:38:17 +0200 Subject: [PATCH 321/376] resolves #20913 imgproc: remove asserts for circles_ in HoughCircles --- modules/imgproc/src/hough.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/imgproc/src/hough.cpp b/modules/imgproc/src/hough.cpp index e66be8b4b700..b48b7ea13723 100644 --- a/modules/imgproc/src/hough.cpp +++ b/modules/imgproc/src/hough.cpp @@ -1728,7 +1728,6 @@ static void HoughCircles( InputArray _image, OutputArray _circles, } CV_Assert(!_image.empty() && _image.type() == CV_8UC1 && (_image.isMat() || _image.isUMat())); - CV_Assert(_circles.isMat() || _circles.isVector()); if( dp <= 0 || minDist <= 0 || param1 <= 0 || param2 <= 0) CV_Error( Error::StsOutOfRange, "dp, min_dist, canny_threshold and acc_threshold must be all positive numbers" ); From 9267536feefa0e0c5ca6f1113b6e22debe438ddf Mon Sep 17 00:00:00 2001 From: Harvey <619328684@qq.com> Date: Fri, 22 Oct 2021 22:04:19 +0800 Subject: [PATCH 322/376] Merge pull request #20875 from Harvey-Huang:master * bmp specified BI_BITFIELDS should take care RGBA bit mask * change the name * support xrgb bmp file * support xrgb bmp file(add test case) * update testing code --- modules/imgcodecs/src/grfmt_bmp.cpp | 56 +++++++++++++++++++++++++-- modules/imgcodecs/src/grfmt_bmp.hpp | 5 +++ modules/imgcodecs/test/test_grfmt.cpp | 26 +++++++++++++ 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/modules/imgcodecs/src/grfmt_bmp.cpp b/modules/imgcodecs/src/grfmt_bmp.cpp index 880a8cd105bd..c838e819f6f5 100644 --- a/modules/imgcodecs/src/grfmt_bmp.cpp +++ b/modules/imgcodecs/src/grfmt_bmp.cpp @@ -58,6 +58,7 @@ BmpDecoder::BmpDecoder() m_origin = ORIGIN_TL; m_bpp = 0; m_rle_code = BMP_RGB; + initMask(); } @@ -97,6 +98,7 @@ bool BmpDecoder::readHeader() int size = m_strm.getDWord(); CV_Assert(size > 0); // overflow, 2Gb limit + initMask(); if( size >= 36 ) { m_width = m_strm.getDWord(); @@ -107,7 +109,30 @@ bool BmpDecoder::readHeader() m_rle_code = (BmpCompression)m_rle_code_; m_strm.skip(12); int clrused = m_strm.getDWord(); - m_strm.skip( size - 36 ); + + if( m_bpp == 32 && m_rle_code == BMP_BITFIELDS && size >= 56 ) + { + m_strm.skip(4); //important colors + //0 is Red channel bit mask, 1 is Green channel bit mask, 2 is Blue channel bit mask, 3 is Alpha channel bit mask + for( int index_rgba = 0; index_rgba < 4; ++index_rgba ) + { + uint mask = m_strm.getDWord(); + m_rgba_mask[index_rgba] = mask; + if(mask != 0) + { + int bit_count = 0; + while(!(mask & 1)) + { + mask >>= 1; + ++bit_count; + } + m_rgba_bit_offset[index_rgba] = bit_count; + } + } + m_strm.skip( size - 56 ); + } + else + m_strm.skip( size - 36 ); if( m_width > 0 && m_height != 0 && (((m_bpp == 1 || m_bpp == 4 || m_bpp == 8 || @@ -486,8 +511,14 @@ decode_rle8_bad: ; icvCvt_BGRA2Gray_8u_C4C1R( src, 0, data, 0, Size(m_width,1) ); else if( img.channels() == 3 ) icvCvt_BGRA2BGR_8u_C4C3R(src, 0, data, 0, Size(m_width, 1)); - else if( img.channels() == 4 ) - memcpy(data, src, m_width * 4); + else if ( img.channels() == 4 ) + { + bool has_bit_mask = (m_rgba_bit_offset[0] >= 0) && (m_rgba_bit_offset[1] >= 0) && (m_rgba_bit_offset[2] >= 0); + if ( has_bit_mask ) + maskBGRA(data, src, m_width); + else + memcpy(data, src, m_width * 4); + } } result = true; break; @@ -503,7 +534,26 @@ decode_rle8_bad: ; return result; } +void BmpDecoder::initMask() +{ + memset(m_rgba_mask, 0, sizeof(m_rgba_mask)); + memset(m_rgba_bit_offset, -1, sizeof(m_rgba_bit_offset)); +} +void BmpDecoder::maskBGRA(uchar* des, uchar* src, int num) +{ + for( int i = 0; i < num; i++, des += 4, src += 4 ) + { + uint data = *((uint*)src); + des[0] = (uchar)((m_rgba_mask[2] & data) >> m_rgba_bit_offset[2]); + des[1] = (uchar)((m_rgba_mask[1] & data) >> m_rgba_bit_offset[1]); + des[2] = (uchar)((m_rgba_mask[0] & data) >> m_rgba_bit_offset[0]); + if (m_rgba_bit_offset[3] >= 0) + des[3] = (uchar)((m_rgba_mask[3] & data) >> m_rgba_bit_offset[3]); + else + des[3] = 255; + } +} ////////////////////////////////////////////////////////////////////////////////////////// BmpEncoder::BmpEncoder() diff --git a/modules/imgcodecs/src/grfmt_bmp.hpp b/modules/imgcodecs/src/grfmt_bmp.hpp index e94b8ff8b4ca..4b34124bf7c8 100644 --- a/modules/imgcodecs/src/grfmt_bmp.hpp +++ b/modules/imgcodecs/src/grfmt_bmp.hpp @@ -73,6 +73,9 @@ class BmpDecoder CV_FINAL : public BaseImageDecoder protected: + void initMask(); + void maskBGRA(uchar* des, uchar* src, int num); + enum Origin { ORIGIN_TL = 0, @@ -85,6 +88,8 @@ class BmpDecoder CV_FINAL : public BaseImageDecoder int m_bpp; int m_offset; BmpCompression m_rle_code; + uint m_rgba_mask[4]; + int m_rgba_bit_offset[4]; }; diff --git a/modules/imgcodecs/test/test_grfmt.cpp b/modules/imgcodecs/test/test_grfmt.cpp index 0d2868800487..03a8d408d6f0 100644 --- a/modules/imgcodecs/test/test_grfmt.cpp +++ b/modules/imgcodecs/test/test_grfmt.cpp @@ -295,6 +295,32 @@ TEST(Imgcodecs_Bmp, read_rle8) EXPECT_PRED_FORMAT2(cvtest::MatComparator(0, 0), rle, ord); } +TEST(Imgcodecs_Bmp, rgba_bit_mask) +{ + const string root = cvtest::TS::ptr()->get_data_path(); + const string filenameInput = root + "readwrite/test_rgba_mask.bmp"; + + const Mat img = cv::imread(filenameInput, IMREAD_UNCHANGED); + ASSERT_FALSE(img.empty()); + ASSERT_EQ(CV_8UC4, img.type()); + + const uchar* data = img.ptr(); + ASSERT_EQ(data[3], 255); +} + +TEST(Imgcodecs_Bmp, read_32bit_xrgb) +{ + const string root = cvtest::TS::ptr()->get_data_path(); + const string filenameInput = root + "readwrite/test_32bit_xrgb.bmp"; + + const Mat img = cv::imread(filenameInput, IMREAD_UNCHANGED); + ASSERT_FALSE(img.empty()); + ASSERT_EQ(CV_8UC4, img.type()); + + const uchar* data = img.ptr(); + ASSERT_EQ(data[3], 255); +} + #ifdef HAVE_IMGCODEC_HDR TEST(Imgcodecs_Hdr, regression) { From 75000a7f7f94ccdab9ddff3fa36c793d7baf8b3b Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Mon, 25 Oct 2021 17:40:11 +0800 Subject: [PATCH 323/376] Use more latest version of Electron --- doc/js_tutorials/js_assets/webnn-electron/main.js | 1 + doc/js_tutorials/js_assets/webnn-electron/package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/js_tutorials/js_assets/webnn-electron/main.js b/doc/js_tutorials/js_assets/webnn-electron/main.js index 70e84862a22e..b73878b07eb7 100644 --- a/doc/js_tutorials/js_assets/webnn-electron/main.js +++ b/doc/js_tutorials/js_assets/webnn-electron/main.js @@ -13,6 +13,7 @@ function createWindow() { height: 840, webPreferences: { nodeIntegration: true, + contextIsolation: false, preload: app.getAppPath()+"/node_setup.js" } }) diff --git a/doc/js_tutorials/js_assets/webnn-electron/package.json b/doc/js_tutorials/js_assets/webnn-electron/package.json index 87011d592844..e6a258ee4053 100644 --- a/doc/js_tutorials/js_assets/webnn-electron/package.json +++ b/doc/js_tutorials/js_assets/webnn-electron/package.json @@ -9,6 +9,6 @@ "start": "electron ." }, "dependencies": { - "electron": "^11.0.3" + "electron": "^15.1.2" } } From 3891f18e7115c9090706825d5bad67ea7db4cde5 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 18 Oct 2021 20:48:24 +0000 Subject: [PATCH 324/376] samples: update parallel_backend examples - use find_package(TBB) --- samples/cpp/CMakeLists.txt | 4 +- .../core/parallel_backend/CMakeLists.txt | 49 ++++++++++++------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/samples/cpp/CMakeLists.txt b/samples/cpp/CMakeLists.txt index b3789980226e..050397ee914e 100644 --- a/samples/cpp/CMakeLists.txt +++ b/samples/cpp/CMakeLists.txt @@ -72,6 +72,8 @@ include("tutorial_code/calib3d/real_time_pose_estimation/CMakeLists.txt" OPTIONA if(OpenCV_FOUND AND NOT CMAKE_VERSION VERSION_LESS "3.1") add_subdirectory("example_cmake") endif() -if(OpenCV_FOUND AND NOT CMAKE_VERSION VERSION_LESS "3.9") +if(OpenCV_FOUND AND NOT CMAKE_VERSION VERSION_LESS "3.9" + AND NOT OPENCV_EXAMPLES_SKIP_PARALLEL_BACKEND +) add_subdirectory("tutorial_code/core/parallel_backend") endif() diff --git a/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt b/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt index 0e67dc29e22a..9d4f91d4a552 100644 --- a/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt +++ b/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt @@ -2,25 +2,36 @@ cmake_minimum_required(VERSION 3.9) find_package(OpenCV REQUIRED COMPONENTS opencv_core) -find_package(OpenMP) -if(OpenMP_FOUND) - project(opencv_example_openmp_backend) - add_executable(opencv_example_openmp_backend example-openmp.cpp) - target_link_libraries(opencv_example_openmp_backend PRIVATE - opencv_core - OpenMP::OpenMP_CXX - ) +if(NOT OPENCV_EXAMPLES_SKIP_PARALLEL_BACKEND_OPENMP + AND NOT OPENCV_EXAMPLES_SKIP_OPENMP +) + find_package(OpenMP) + if(OpenMP_FOUND) + project(opencv_example_openmp_backend) + add_executable(opencv_example_openmp_backend example-openmp.cpp) + target_link_libraries(opencv_example_openmp_backend PRIVATE + opencv_core + OpenMP::OpenMP_CXX + ) + endif() endif() -# TODO: find_package(TBB) -find_path(TBB_INCLUDE_DIR NAMES "tbb/tbb.h") -find_library(TBB_LIBRARY NAMES "tbb") -if(TBB_INCLUDE_DIR AND TBB_LIBRARY AND NOT OPENCV_EXAMPLE_SKIP_TBB) - project(opencv_example_tbb_backend) - add_executable(opencv_example_tbb_backend example-tbb.cpp) - target_include_directories(opencv_example_tbb_backend SYSTEM PRIVATE ${TBB_INCLUDE_DIR}) - target_link_libraries(opencv_example_tbb_backend PRIVATE - opencv_core - ${TBB_LIBRARY} - ) +if(NOT OPENCV_EXAMPLES_SKIP_PARALLEL_BACKEND_TBB + AND NOT OPENCV_EXAMPLES_SKIP_TBB + AND NOT OPENCV_EXAMPLE_SKIP_TBB # deprecated (to be removed in OpenCV 5.0) +) + find_package(TBB) + if(NOT TBB_FOUND) + find_path(TBB_INCLUDE_DIR NAMES "tbb/tbb.h") + find_library(TBB_LIBRARY NAMES "tbb") + endif() + if(TBB_INCLUDE_DIR AND TBB_LIBRARY) + project(opencv_example_tbb_backend) + add_executable(opencv_example_tbb_backend example-tbb.cpp) + target_include_directories(opencv_example_tbb_backend SYSTEM PRIVATE ${TBB_INCLUDE_DIR}) + target_link_libraries(opencv_example_tbb_backend PRIVATE + opencv_core + ${TBB_LIBRARY} + ) + endif() endif() From 0b1ae11498022c290950043b38a2f707ff676be7 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 26 Oct 2021 14:29:57 +0000 Subject: [PATCH 325/376] cmake: find_package with QUIET --- samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt b/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt index 9d4f91d4a552..857d9b76ef5c 100644 --- a/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt +++ b/samples/cpp/tutorial_code/core/parallel_backend/CMakeLists.txt @@ -20,7 +20,7 @@ if(NOT OPENCV_EXAMPLES_SKIP_PARALLEL_BACKEND_TBB AND NOT OPENCV_EXAMPLES_SKIP_TBB AND NOT OPENCV_EXAMPLE_SKIP_TBB # deprecated (to be removed in OpenCV 5.0) ) - find_package(TBB) + find_package(TBB QUIET) if(NOT TBB_FOUND) find_path(TBB_INCLUDE_DIR NAMES "tbb/tbb.h") find_library(TBB_LIBRARY NAMES "tbb") From 0a71063530c215e8ad90da0c35b2e5cc829d64b5 Mon Sep 17 00:00:00 2001 From: MaximMilashchenko <67949029+MaximMilashchenko@users.noreply.github.com> Date: Tue, 26 Oct 2021 17:33:53 +0300 Subject: [PATCH 326/376] Merge pull request #20942 from MaximMilashchenko:AudioPatch Audio patch * fixed microphone, audio position * fixed docs * changed AudioOpenCheck --- modules/videoio/src/cap_msmf.cpp | 168 +++++++++++++++------------- modules/videoio/test/test_audio.cpp | 11 ++ 2 files changed, 104 insertions(+), 75 deletions(-) diff --git a/modules/videoio/src/cap_msmf.cpp b/modules/videoio/src/cap_msmf.cpp index 0fa064dfb87c..39f191e642fc 100644 --- a/modules/videoio/src/cap_msmf.cpp +++ b/modules/videoio/src/cap_msmf.cpp @@ -709,6 +709,7 @@ class CvCapture_MSMF : public cv::IVideoCapture virtual void close(); virtual double getProperty(int) const CV_OVERRIDE; virtual bool setProperty(int, double) CV_OVERRIDE; + bool configureAudioFrame(); bool grabAudioFrame(); bool grabVideoFrame(); virtual bool grabFrame() CV_OVERRIDE; @@ -1038,6 +1039,7 @@ bool CvCapture_MSMF::configureAudioOutput(MediaType newType) if (bestMatch.second.isEmpty(true)) { CV_LOG_DEBUG(NULL, "Can not find audio stream with requested parameters"); + isOpen = false; return false; } dwAudioStreamIndex = bestMatch.first.stream; @@ -1439,6 +1441,91 @@ bool CvCapture_MSMF::grabVideoFrame() return returnFlag; } +bool CvCapture_MSMF::configureAudioFrame() +{ + if (!audioSamples.empty() || !bufferAudioData.empty() && aEOS) + { + _ComPtr buf = NULL; + std::vector audioDataInUse; + BYTE* ptr = NULL; + DWORD maxsize = 0, cursize = 0; + CV_TRACE_REGION("get_contiguous_buffer"); + for (auto item : audioSamples) + { + if (!SUCCEEDED(item->ConvertToContiguousBuffer(&buf))) + { + CV_TRACE_REGION("get_buffer"); + DWORD bcnt = 0; + if (!SUCCEEDED(item->GetBufferCount(&bcnt))) + break; + if (bcnt == 0) + break; + if (!SUCCEEDED(item->GetBufferByIndex(0, &buf))) + break; + } + if (!SUCCEEDED(buf->Lock(&ptr, &maxsize, &cursize))) + break; + size_t lastSize = bufferAudioData.size(); + bufferAudioData.resize(lastSize+cursize); + for (unsigned int i = 0; i < cursize; i++) + { + bufferAudioData[lastSize+i]=*(ptr+i); + } + CV_TRACE_REGION_NEXT("unlock"); + buf->Unlock(); + buf = NULL; + } + audioSamples.clear(); + + audioSamplePos += chunkLengthOfBytes/((captureAudioFormat.bit_per_sample/8)*captureAudioFormat.nChannels); + chunkLengthOfBytes = (videoStream != -1) ? (LONGLONG)((requiredAudioTime*captureAudioFormat.nSamplesPerSec*captureAudioFormat.nChannels*(captureAudioFormat.bit_per_sample)/8)/1e7) : cursize; + if ((videoStream != -1) && (chunkLengthOfBytes % ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels) != 0)) + { + if ( (double)audioSamplePos/captureAudioFormat.nSamplesPerSec + audioStartOffset * 1e-7 - usedVideoSampleTime * 1e-7 >= 0 ) + chunkLengthOfBytes -= numberOfAdditionalAudioBytes; + numberOfAdditionalAudioBytes = ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels) + - chunkLengthOfBytes % ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels); + chunkLengthOfBytes += numberOfAdditionalAudioBytes; + } + if (lastFrame && !syncLastFrame || aEOS && !vEOS) + { + chunkLengthOfBytes = bufferAudioData.size(); + audioSamplePos += chunkLengthOfBytes/((captureAudioFormat.bit_per_sample/8)*captureAudioFormat.nChannels); + } + CV_Check((double)chunkLengthOfBytes, chunkLengthOfBytes >= INT_MIN || chunkLengthOfBytes <= INT_MAX, "MSMF: The chunkLengthOfBytes is out of the allowed range"); + copy(bufferAudioData.begin(), bufferAudioData.begin() + (int)chunkLengthOfBytes, std::back_inserter(audioDataInUse)); + bufferAudioData.erase(bufferAudioData.begin(), bufferAudioData.begin() + (int)chunkLengthOfBytes); + if (audioFrame.empty()) + { + switch (outputAudioFormat) + { + case CV_8S: + cv::Mat((int)chunkLengthOfBytes/(captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_8S, audioDataInUse.data()).copyTo(audioFrame); + break; + case CV_16S: + cv::Mat((int)chunkLengthOfBytes/(2*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_16S, audioDataInUse.data()).copyTo(audioFrame); + break; + case CV_32S: + cv::Mat((int)chunkLengthOfBytes/(4*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_32S, audioDataInUse.data()).copyTo(audioFrame); + break; + case CV_32F: + cv::Mat((int)chunkLengthOfBytes/(4*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_32F, audioDataInUse.data()).copyTo(audioFrame); + break; + default: + break; + } + } + audioDataInUse.clear(); + audioDataInUse.shrink_to_fit(); + + return true; + } + else + { + return false; + } +} + bool CvCapture_MSMF::grabAudioFrame() { DWORD streamIndex, flags; @@ -1500,6 +1587,8 @@ bool CvCapture_MSMF::grabAudioFrame() aEOS = true; if (videoStream != -1 && !vEOS) returnFlag = true; + if (videoStream == -1) + audioSamplePos += chunkLengthOfBytes/((captureAudioFormat.bit_per_sample/8)*captureAudioFormat.nChannels); CV_LOG_DEBUG(NULL, "videoio(MSMF): End of audio stream detected"); break; } @@ -1538,81 +1627,7 @@ bool CvCapture_MSMF::grabAudioFrame() } } - if (!audioSamples.empty() || !bufferAudioData.empty() && aEOS) - { - _ComPtr buf = NULL; - std::vector audioDataInUse; - BYTE* ptr = NULL; - DWORD maxsize = 0, cursize = 0; - CV_TRACE_REGION("get_contiguous_buffer"); - for (auto item : audioSamples) - { - if (!SUCCEEDED(item->ConvertToContiguousBuffer(&buf))) - { - CV_TRACE_REGION("get_buffer"); - DWORD bcnt = 0; - if (!SUCCEEDED(item->GetBufferCount(&bcnt))) - break; - if (bcnt == 0) - break; - if (!SUCCEEDED(item->GetBufferByIndex(0, &buf))) - break; - } - if (!SUCCEEDED(buf->Lock(&ptr, &maxsize, &cursize))) - break; - size_t lastSize = bufferAudioData.size(); - bufferAudioData.resize(lastSize+cursize); - for (unsigned int i = 0; i < cursize; i++) - { - bufferAudioData[lastSize+i]=*(ptr+i); - } - CV_TRACE_REGION_NEXT("unlock"); - buf->Unlock(); - buf = NULL; - } - audioSamples.clear(); - - audioSamplePos += chunkLengthOfBytes/((captureAudioFormat.bit_per_sample/8)*captureAudioFormat.nChannels); - chunkLengthOfBytes = (videoStream != -1) ? (LONGLONG)((requiredAudioTime*captureAudioFormat.nSamplesPerSec*captureAudioFormat.nChannels*(captureAudioFormat.bit_per_sample)/8)/1e7) : cursize; - if ((videoStream != -1) && (chunkLengthOfBytes % ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels) != 0)) - { - if ( (double)audioSamplePos/captureAudioFormat.nSamplesPerSec + audioStartOffset * 1e-7 - usedVideoSampleTime * 1e-7 >= 0 ) - chunkLengthOfBytes -= numberOfAdditionalAudioBytes; - numberOfAdditionalAudioBytes = ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels) - - chunkLengthOfBytes % ((int)(captureAudioFormat.bit_per_sample)/8* (int)captureAudioFormat.nChannels); - chunkLengthOfBytes += numberOfAdditionalAudioBytes; - } - if (lastFrame && !syncLastFrame|| aEOS && !vEOS) - { - chunkLengthOfBytes = bufferAudioData.size(); - } - CV_Check((double)chunkLengthOfBytes, chunkLengthOfBytes >= INT_MIN || chunkLengthOfBytes <= INT_MAX, "MSMF: The chunkLengthOfBytes is out of the allowed range"); - copy(bufferAudioData.begin(), bufferAudioData.begin() + (int)chunkLengthOfBytes, std::back_inserter(audioDataInUse)); - bufferAudioData.erase(bufferAudioData.begin(), bufferAudioData.begin() + (int)chunkLengthOfBytes); - if (audioFrame.empty()) - { - switch (outputAudioFormat) - { - case CV_8S: - cv::Mat((int)chunkLengthOfBytes/(captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_8S, audioDataInUse.data()).copyTo(audioFrame); - break; - case CV_16S: - cv::Mat((int)chunkLengthOfBytes/(2*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_16S, audioDataInUse.data()).copyTo(audioFrame); - break; - case CV_32S: - cv::Mat((int)chunkLengthOfBytes/(4*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_32S, audioDataInUse.data()).copyTo(audioFrame); - break; - case CV_32F: - cv::Mat((int)chunkLengthOfBytes/(4*captureAudioFormat.nChannels), captureAudioFormat.nChannels, CV_32F, audioDataInUse.data()).copyTo(audioFrame); - break; - default: - break; - } - } - audioDataInUse.clear(); - audioDataInUse.shrink_to_fit(); - } - + returnFlag &= configureAudioFrame(); return returnFlag; } @@ -1662,6 +1677,9 @@ bool CvCapture_MSMF::grabFrame() } if (videoStream != -1) usedVideoSampleTime = reader->m_lastSampleTimestamp; + if (audioStream != -1) + return configureAudioFrame(); + return true; } else if (isOpen) diff --git a/modules/videoio/test/test_audio.cpp b/modules/videoio/test/test_audio.cpp index 3ff51e26134d..0b637aeabdde 100644 --- a/modules/videoio/test/test_audio.cpp +++ b/modules/videoio/test/test_audio.cpp @@ -270,4 +270,15 @@ TEST_P(Media, audio) INSTANTIATE_TEST_CASE_P(/**/, Media, testing::ValuesIn(mediaParams)); +TEST(AudioOpenCheck, bad_arg_invalid_audio_stream) +{ + std::string fileName = "audio/test_audio.mp4"; + std::vector params { CAP_PROP_AUDIO_STREAM, 1, + CAP_PROP_VIDEO_STREAM, 0, + CAP_PROP_AUDIO_DATA_DEPTH, CV_16S }; + VideoCapture cap; + cap.open(findDataFile(fileName), cv::CAP_MSMF, params); + ASSERT_FALSE(cap.isOpened()); +} + }} //namespace From af154e30539988bf658a31b555a48669db1e0890 Mon Sep 17 00:00:00 2001 From: Harvey Date: Wed, 27 Oct 2021 13:52:54 +0800 Subject: [PATCH 327/376] fixed bug: opencv read tif file convert Palette color image as grayscale image --- modules/imgcodecs/src/grfmt_tiff.cpp | 7 ++++++- modules/imgcodecs/test/test_tiff.cpp | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/modules/imgcodecs/src/grfmt_tiff.cpp b/modules/imgcodecs/src/grfmt_tiff.cpp index a46ed3ac90fb..5e7523b2039d 100644 --- a/modules/imgcodecs/src/grfmt_tiff.cpp +++ b/modules/imgcodecs/src/grfmt_tiff.cpp @@ -303,7 +303,12 @@ bool TiffDecoder::readHeader() result = true; break; case 8: - m_type = CV_MAKETYPE(CV_8U, !isGrayScale ? wanted_channels : 1); + //Palette color, the value of the component is used as an index into the red, + //green and blue curves in the ColorMap field to retrieve an RGB triplet that defines the color. + if(photometric == PHOTOMETRIC_PALETTE) + m_type = CV_MAKETYPE(CV_8U, 3); + else + m_type = CV_MAKETYPE(CV_8U, !isGrayScale ? wanted_channels : 1); result = true; break; case 16: diff --git a/modules/imgcodecs/test/test_tiff.cpp b/modules/imgcodecs/test/test_tiff.cpp index dec38014aa28..a2f9655c73be 100644 --- a/modules/imgcodecs/test/test_tiff.cpp +++ b/modules/imgcodecs/test/test_tiff.cpp @@ -219,6 +219,16 @@ TEST(Imgcodecs_Tiff, readWrite_32FC3_RAW) EXPECT_EQ(0, remove(filenameOutput.c_str())); } +TEST(Imgcodecs_Tiff, read_palette_color_image) +{ + const string root = cvtest::TS::ptr()->get_data_path(); + const string filenameInput = root + "readwrite/test_palette_color_image.tif"; + + const Mat img = cv::imread(filenameInput, IMREAD_UNCHANGED); + ASSERT_FALSE(img.empty()); + ASSERT_EQ(CV_8UC3, img.type()); +} + //================================================================================================== From 9dadc06e64ef5e08fd3b403cf4c55e36345f9315 Mon Sep 17 00:00:00 2001 From: shengyu Date: Wed, 27 Oct 2021 20:19:05 +0800 Subject: [PATCH 328/376] remove redundant semicolons --- modules/gapi/include/opencv2/gapi/fluid/gfluidbuffer.hpp | 2 +- modules/gapi/include/opencv2/gapi/gtransform.hpp | 2 +- modules/gapi/include/opencv2/gapi/plaidml/gplaidmlkernel.hpp | 2 +- .../opencv2/gapi/streaming/onevpl/data_provider_interface.hpp | 4 ++-- modules/gapi/include/opencv2/gapi/util/optional.hpp | 2 +- modules/gapi/src/compiler/gislandmodel.hpp | 2 +- modules/gapi/src/compiler/transactions.hpp | 4 ++-- modules/gapi/src/executor/gtbbexecutor.hpp | 4 ++-- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/fluid/gfluidbuffer.hpp b/modules/gapi/include/opencv2/gapi/fluid/gfluidbuffer.hpp index daa9d4153ed6..551f0a398feb 100644 --- a/modules/gapi/include/opencv2/gapi/fluid/gfluidbuffer.hpp +++ b/modules/gapi/include/opencv2/gapi/fluid/gfluidbuffer.hpp @@ -25,7 +25,7 @@ namespace fluid { struct Border { // This constructor is required to support existing kernels which are part of G-API - Border(int _type, cv::Scalar _val) : type(_type), value(_val) {}; + Border(int _type, cv::Scalar _val) : type(_type), value(_val) {} int type; cv::Scalar value; diff --git a/modules/gapi/include/opencv2/gapi/gtransform.hpp b/modules/gapi/include/opencv2/gapi/gtransform.hpp index 5d1b91b51739..109bc87b7f04 100644 --- a/modules/gapi/include/opencv2/gapi/gtransform.hpp +++ b/modules/gapi/include/opencv2/gapi/gtransform.hpp @@ -31,7 +31,7 @@ struct GAPI_EXPORTS GTransform F pattern; F substitute; - GTransform(const std::string& d, const F &p, const F &s) : description(d), pattern(p), substitute(s){}; + GTransform(const std::string& d, const F &p, const F &s) : description(d), pattern(p), substitute(s) {} }; namespace detail diff --git a/modules/gapi/include/opencv2/gapi/plaidml/gplaidmlkernel.hpp b/modules/gapi/include/opencv2/gapi/plaidml/gplaidmlkernel.hpp index 7ce00cfa359b..e22ecc7211f2 100644 --- a/modules/gapi/include/opencv2/gapi/plaidml/gplaidmlkernel.hpp +++ b/modules/gapi/include/opencv2/gapi/plaidml/gplaidmlkernel.hpp @@ -59,7 +59,7 @@ class GAPI_EXPORTS GPlaidMLKernel using F = std::function; GPlaidMLKernel() = default; - explicit GPlaidMLKernel(const F& f) : m_f(f) {}; + explicit GPlaidMLKernel(const F& f) : m_f(f) {} void apply(GPlaidMLContext &ctx) const { diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp index 2c299520f757..ac3444757d38 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp @@ -17,7 +17,7 @@ namespace wip { namespace onevpl { struct GAPI_EXPORTS DataProviderException : public std::exception { - virtual ~DataProviderException() {}; + virtual ~DataProviderException() {} }; struct GAPI_EXPORTS DataProviderSystemErrorException : public DataProviderException { @@ -42,7 +42,7 @@ struct GAPI_EXPORTS DataProviderSystemErrorException : public DataProviderExcept struct GAPI_EXPORTS IDataProvider { using Ptr = std::shared_ptr; - virtual ~IDataProvider() {}; + virtual ~IDataProvider() {} /** * The function is used by onevpl::GSource to extract binary data stream from @ref IDataProvider diff --git a/modules/gapi/include/opencv2/gapi/util/optional.hpp b/modules/gapi/include/opencv2/gapi/util/optional.hpp index 6c8ceebbda23..dca03cadad86 100644 --- a/modules/gapi/include/opencv2/gapi/util/optional.hpp +++ b/modules/gapi/include/opencv2/gapi/util/optional.hpp @@ -33,7 +33,7 @@ namespace util // Constructors // NB.: there were issues with Clang 3.8 when =default() was used // instead {} - optional() {}; + optional() {} optional(const optional&) = default; explicit optional(T&&) noexcept; explicit optional(const T&) noexcept; diff --git a/modules/gapi/src/compiler/gislandmodel.hpp b/modules/gapi/src/compiler/gislandmodel.hpp index 2cdd10346c71..063504a92227 100644 --- a/modules/gapi/src/compiler/gislandmodel.hpp +++ b/modules/gapi/src/compiler/gislandmodel.hpp @@ -140,7 +140,7 @@ class GAPI_EXPORTS GIslandExecutable // FIXME: This thing will likely break stuff once we introduce // "multi-source streaming", a better design needs to be proposed // at that stage. - virtual void handleNewStream() {}; // do nothing here by default + virtual void handleNewStream() {} // do nothing here by default // This method is called for every IslandExecutable when // the stream-based execution is stopped. diff --git a/modules/gapi/src/compiler/transactions.hpp b/modules/gapi/src/compiler/transactions.hpp index bdc1723e197d..9092c6629119 100644 --- a/modules/gapi/src/compiler/transactions.hpp +++ b/modules/gapi/src/compiler/transactions.hpp @@ -69,8 +69,8 @@ struct ChangeT { struct Base { - virtual void commit (ade::Graph & ) {}; - virtual void rollback(ade::Graph & ) {}; + virtual void commit (ade::Graph & ) {} + virtual void rollback(ade::Graph & ) {} virtual ~Base() = default; }; diff --git a/modules/gapi/src/executor/gtbbexecutor.hpp b/modules/gapi/src/executor/gtbbexecutor.hpp index db3ee5c13307..3c2bf1ff9804 100644 --- a/modules/gapi/src/executor/gtbbexecutor.hpp +++ b/modules/gapi/src/executor/gtbbexecutor.hpp @@ -83,8 +83,8 @@ struct tile_node { std::vector dependants; - tile_node(decltype(sync_task_body::body)&& f) : task_body(sync_task_body{std::move(f)}) {}; - tile_node(async_tag, decltype(async_task_body::body)&& f) : task_body(async_task_body{std::move(f)}) {}; + tile_node(decltype(sync_task_body::body)&& f) : task_body(sync_task_body{std::move(f)}) {} + tile_node(async_tag, decltype(async_task_body::body)&& f) : task_body(async_task_body{std::move(f)}) {} }; std::ostream& operator<<(std::ostream& o, tile_node const& n); From 244ba1a61a8a514dbd6014766dc17dfde4560487 Mon Sep 17 00:00:00 2001 From: Chengrui Wang <80876977+crywang@users.noreply.github.com> Date: Wed, 27 Oct 2021 20:23:42 +0800 Subject: [PATCH 329/376] Merge pull request #20935 from crywang:dnn_face Fix problems in tutorial and python sample of dnn_face. * Update dnn_face.markdown * Update face_match.py --- doc/tutorials/dnn/dnn_face/dnn_face.markdown | 6 +++--- samples/dnn/face_match.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/tutorials/dnn/dnn_face/dnn_face.markdown b/doc/tutorials/dnn/dnn_face/dnn_face.markdown index e5092b8b92e8..202be3e0e302 100644 --- a/doc/tutorials/dnn/dnn_face/dnn_face.markdown +++ b/doc/tutorials/dnn/dnn_face/dnn_face.markdown @@ -12,7 +12,7 @@ ## Introduction -In this section, we introduce the DNN-based module for face detection and face recognition. Models can be obtained in [Models](#Models). The usage of `FaceDetectorYN` and `FaceRecognizer` are presented in [Usage](#Usage). +In this section, we introduce the DNN-based module for face detection and face recognition. Models can be obtained in [Models](#Models). The usage of `FaceDetectorYN` and `FaceRecognizerSF` are presented in [Usage](#Usage). ## Models @@ -58,8 +58,8 @@ x1, y1, w, h, x_re, y_re, x_le, y_le, x_nt, y_nt, x_rcm, y_rcm, x_lcm, y_lcm Following Face Detection, run codes below to extract face feature from facial image. ```cpp -// Initialize FaceRecognizer with model path (cv::String) -Ptr faceRecognizer = FaceRecognizer::create(model_path, ""); +// Initialize FaceRecognizerSF with model path (cv::String) +Ptr faceRecognizer = FaceRecognizerSF::create(model_path, ""); // Aligning and cropping facial image through the first face of faces detected by dnn_face::DNNFaceDetector Mat aligned_face; diff --git a/samples/dnn/face_match.py b/samples/dnn/face_match.py index b36c9f6367af..916c76abf1e8 100644 --- a/samples/dnn/face_match.py +++ b/samples/dnn/face_match.py @@ -38,20 +38,20 @@ face2_align = recognizer.alignCrop(img2, face2[1][0]) # Extract features -face1_feature = recognizer.faceFeature(face1_align) -face2_feature = recognizer.faceFeature(face2_align) +face1_feature = recognizer.feature(face1_align) +face2_feature = recognizer.feature(face2_align) # Calculate distance (0: cosine, 1: L2) cosine_similarity_threshold = 0.363 -cosine_score = recognizer.faceMatch(face1_feature, face2_feature, 0) +cosine_score = recognizer.match(face1_feature, face2_feature, 0) msg = 'different identities' if cosine_score >= cosine_similarity_threshold: msg = 'the same identity' print('They have {}. Cosine Similarity: {}, threshold: {} (higher value means higher similarity, max 1.0).'.format(msg, cosine_score, cosine_similarity_threshold)) l2_similarity_threshold = 1.128 -l2_score = recognizer.faceMatch(face1_feature, face2_feature, 1) +l2_score = recognizer.match(face1_feature, face2_feature, 1) msg = 'different identities' if l2_score <= l2_similarity_threshold: msg = 'the same identity' -print('They have {}. NormL2 Distance: {}, threshold: {} (lower value means higher similarity, min 0.0).'.format(msg, l2_score, l2_similarity_threshold)) \ No newline at end of file +print('They have {}. NormL2 Distance: {}, threshold: {} (lower value means higher similarity, min 0.0).'.format(msg, l2_score, l2_similarity_threshold)) From 84a81579ba839c4224d3b1fba806e1d64a475597 Mon Sep 17 00:00:00 2001 From: Noah Stier Date: Wed, 27 Oct 2021 12:01:53 -0700 Subject: [PATCH 330/376] tvl1 cuda optflow optimization --- modules/cudaoptflow/src/tvl1flow.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/cudaoptflow/src/tvl1flow.cpp b/modules/cudaoptflow/src/tvl1flow.cpp index 5f28d4c6174a..3ea2d365a62a 100644 --- a/modules/cudaoptflow/src/tvl1flow.cpp +++ b/modules/cudaoptflow/src/tvl1flow.cpp @@ -162,7 +162,9 @@ namespace GpuMat p32_buf; GpuMat diff_buf; - GpuMat norm_buf; + + GpuMat diff_sum_dev; + Mat diff_sum_host; }; void OpticalFlowDual_TVL1_Impl::calc(InputArray _frame0, InputArray _frame1, InputOutputArray _flow, Stream& stream) @@ -361,8 +363,11 @@ namespace estimateU(I1wx, I1wy, grad, rho_c, p11, p12, p21, p22, p31, p32, u1, u2, u3, diff, l_t, static_cast(theta_), gamma_, calcError, stream); if (calcError) { + cuda::calcSum(diff, diff_sum_dev, cv::noArray(), _stream); + diff_sum_dev.download(diff_sum_host, _stream); _stream.waitForCompletion(); - error = cuda::sum(diff, norm_buf)[0]; + + error = diff_sum_host.at(0,0); prevError = error; } else From 5dfe65d53ab3d1ac19a23f04b1fa274306615b65 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 28 Oct 2021 05:20:23 +0000 Subject: [PATCH 331/376] cmake: fix popcnt detection with Intel Compiler --- cmake/OpenCVCompilerOptimizations.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/OpenCVCompilerOptimizations.cmake b/cmake/OpenCVCompilerOptimizations.cmake index 970dd2890336..058443821a1b 100644 --- a/cmake/OpenCVCompilerOptimizations.cmake +++ b/cmake/OpenCVCompilerOptimizations.cmake @@ -238,7 +238,7 @@ if(X86 OR X86_64) ocv_intel_compiler_optimization_option(FP16 "-mavx" "/arch:AVX") ocv_intel_compiler_optimization_option(AVX "-mavx" "/arch:AVX") ocv_intel_compiler_optimization_option(FMA3 "" "") - ocv_intel_compiler_optimization_option(POPCNT "" "") + ocv_intel_compiler_optimization_option(POPCNT "-mpopcnt" "") # -mpopcnt is available since ICC 19.0.0 ocv_intel_compiler_optimization_option(SSE4_2 "-msse4.2" "/arch:SSE4.2") ocv_intel_compiler_optimization_option(SSE4_1 "-msse4.1" "/arch:SSE4.1") ocv_intel_compiler_optimization_option(SSE3 "-msse3" "/arch:SSE3") From 1726bb6c0decddb0fd6f6e2fc70777fcd2c59e68 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 28 Oct 2021 05:49:05 +0000 Subject: [PATCH 332/376] build(icc): fix nodiscard attribute handling --- modules/core/include/opencv2/core/cvdef.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/include/opencv2/core/cvdef.h b/modules/core/include/opencv2/core/cvdef.h index 6011b2a9313d..c2cdcad07591 100644 --- a/modules/core/include/opencv2/core/cvdef.h +++ b/modules/core/include/opencv2/core/cvdef.h @@ -589,6 +589,8 @@ Cv64suf; # elif __cplusplus >= 201703L // available when compiler is C++17 compliant # define CV_NODISCARD_STD [[nodiscard]] +# elif defined(__INTEL_COMPILER) + // see above, available when C++17 is enabled # elif defined(_MSC_VER) && _MSC_VER >= 1911 && _MSVC_LANG >= 201703L // available with VS2017 v15.3+ with /std:c++17 or higher; works on functions and classes # define CV_NODISCARD_STD [[nodiscard]] From 75e2ba5af3adc26b08c60a81fbd6931ced0977b3 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 28 Oct 2021 11:25:00 +0000 Subject: [PATCH 333/376] core(simd): fix compilation with MSVC-Clang --- modules/core/include/opencv2/core/hal/intrin_sse.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_sse.hpp b/modules/core/include/opencv2/core/hal/intrin_sse.hpp index 2244717e1986..443ee16097f3 100644 --- a/modules/core/include/opencv2/core/hal/intrin_sse.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_sse.hpp @@ -244,7 +244,7 @@ struct v_uint64x2 explicit v_uint64x2(__m128i v) : val(v) {} v_uint64x2(uint64 v0, uint64 v1) { -#if defined(_MSC_VER) && _MSC_VER >= 1920/*MSVS 2019*/ && defined(_M_X64) +#if defined(_MSC_VER) && _MSC_VER >= 1920/*MSVS 2019*/ && defined(_M_X64) && !defined(__clang__) val = _mm_setr_epi64x((int64_t)v0, (int64_t)v1); #elif defined(__GNUC__) val = _mm_setr_epi64((__m64)v0, (__m64)v1); @@ -278,7 +278,7 @@ struct v_int64x2 explicit v_int64x2(__m128i v) : val(v) {} v_int64x2(int64 v0, int64 v1) { -#if defined(_MSC_VER) && _MSC_VER >= 1920/*MSVS 2019*/ && defined(_M_X64) +#if defined(_MSC_VER) && _MSC_VER >= 1920/*MSVS 2019*/ && defined(_M_X64) && !defined(__clang__) val = _mm_setr_epi64x((int64_t)v0, (int64_t)v1); #elif defined(__GNUC__) val = _mm_setr_epi64((__m64)v0, (__m64)v1); From eb152d7431462579efd6873e94eae7c20453611f Mon Sep 17 00:00:00 2001 From: Maxim Pashchenkov Date: Thu, 28 Oct 2021 21:19:46 +0300 Subject: [PATCH 334/376] Merge pull request #20937 from mpashchenkov:mp/ocv-gapi-warnings G-API: Disable Windows warnings with 4996 code * Windows warnings 4503 and 4996 are disabled with dnn style * Applying comments to review * Reproducing * Added check MSVC_VERSION for both warnings --- modules/gapi/CMakeLists.txt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 61ab5397d7ab..185f5c66399e 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -29,12 +29,11 @@ ocv_add_module(gapi ) if(MSVC) - # Disable obsollete warning C4503 popping up on MSVC <<2017 - # https://docs.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-1-c4503?view=vs-2019 - ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4503) - if (OPENCV_GAPI_INF_ENGINE AND NOT INF_ENGINE_RELEASE VERSION_GREATER "2021000000") - # Disable IE deprecated code warning C4996 for releases < 2021.1 - ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4996) + if(MSVC_VERSION LESS 1910) + # Disable obsolete warning C4503 popping up on MSVC << 15 2017 + # https://docs.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-1-c4503?view=vs-2019 + # and IE deprecated code warning C4996 + ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4503 /wd4996) endif() endif() From d612c72405c69becc0b0b4af3a0942ea9ee86279 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 28 Oct 2021 21:08:36 +0000 Subject: [PATCH 335/376] build: fix MSVC-Clang warnings about unused parameters in stubs --- modules/core/src/hal_replacement.hpp | 25 +++++++++++++--------- modules/features2d/src/hal_replacement.hpp | 25 +++++++++++++--------- modules/imgproc/src/hal_replacement.hpp | 25 +++++++++++++--------- 3 files changed, 45 insertions(+), 30 deletions(-) diff --git a/modules/core/src/hal_replacement.hpp b/modules/core/src/hal_replacement.hpp index 1a558f253278..6ed795b5e1c0 100644 --- a/modules/core/src/hal_replacement.hpp +++ b/modules/core/src/hal_replacement.hpp @@ -47,12 +47,15 @@ #include "opencv2/core/hal/interface.h" -#if defined __GNUC__ -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunused-parameter" -#elif defined _MSC_VER -# pragma warning( push ) -# pragma warning( disable: 4100 ) +#if defined(__clang__) // clang or MSVC clang +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-parameter" +#elif defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4100) +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" #endif //! @addtogroup core_hal_interface @@ -731,10 +734,12 @@ inline int hal_ni_minMaxIdx(const uchar* src_data, size_t src_step, int width, i //! @} -#if defined __GNUC__ -# pragma GCC diagnostic pop -#elif defined _MSC_VER -# pragma warning( pop ) +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(_MSC_VER) +#pragma warning(pop) +#elif defined(__GNUC__) +#pragma GCC diagnostic pop #endif #include "hal_internal.hpp" diff --git a/modules/features2d/src/hal_replacement.hpp b/modules/features2d/src/hal_replacement.hpp index 7780aa4a2411..977cef1e3207 100644 --- a/modules/features2d/src/hal_replacement.hpp +++ b/modules/features2d/src/hal_replacement.hpp @@ -44,12 +44,15 @@ #include "opencv2/core/hal/interface.h" -#if defined __GNUC__ -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunused-parameter" -#elif defined _MSC_VER -# pragma warning( push ) -# pragma warning( disable: 4100 ) +#if defined(__clang__) // clang or MSVC clang +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-parameter" +#elif defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4100) +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" #endif //! @addtogroup features2d_hal_interface @@ -103,10 +106,12 @@ inline int hal_ni_FAST(const uchar* src_data, size_t src_step, int width, int he //! @} -#if defined __GNUC__ -# pragma GCC diagnostic pop -#elif defined _MSC_VER -# pragma warning( pop ) +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(_MSC_VER) +#pragma warning(pop) +#elif defined(__GNUC__) +#pragma GCC diagnostic pop #endif #include "custom_hal.hpp" diff --git a/modules/imgproc/src/hal_replacement.hpp b/modules/imgproc/src/hal_replacement.hpp index 3368093c56f1..d9ef7530f915 100644 --- a/modules/imgproc/src/hal_replacement.hpp +++ b/modules/imgproc/src/hal_replacement.hpp @@ -48,12 +48,15 @@ #include "opencv2/core/hal/interface.h" #include "opencv2/imgproc/hal/interface.h" -#if defined __GNUC__ -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunused-parameter" -#elif defined _MSC_VER -# pragma warning( push ) -# pragma warning( disable: 4100 ) +#if defined(__clang__) // clang or MSVC clang +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-parameter" +#elif defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4100) +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" #endif //! @addtogroup imgproc_hal_interface @@ -809,10 +812,12 @@ inline int hal_ni_canny(const uchar* src_data, size_t src_step, uchar* dst_data, //! @} -#if defined __GNUC__ -# pragma GCC diagnostic pop -#elif defined _MSC_VER -# pragma warning( pop ) +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(_MSC_VER) +#pragma warning(pop) +#elif defined(__GNUC__) +#pragma GCC diagnostic pop #endif #include "custom_hal.hpp" From a49cda65230c0094148fc7dddd70eac2c3ed418e Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 28 Oct 2021 21:32:47 +0000 Subject: [PATCH 336/376] core: eliminate Winvalid-noreturn in base.hpp --- modules/core/include/opencv2/core/base.hpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/core/include/opencv2/core/base.hpp b/modules/core/include/opencv2/core/base.hpp index 12504974d974..19d496080c4b 100644 --- a/modules/core/include/opencv2/core/base.hpp +++ b/modules/core/include/opencv2/core/base.hpp @@ -297,7 +297,10 @@ It is possible to alternate error processing by using redirectError(). */ CV_EXPORTS void error(int _code, const String& _err, const char* _func, const char* _file, int _line); -#ifdef __GNUC__ +#if defined(__clang__) && defined(_MSC_VER) // MSVC-Clang +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Winvalid-noreturn" +#elif defined(__GNUC__) # if defined __clang__ || defined __APPLE__ # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Winvalid-noreturn" @@ -316,7 +319,10 @@ CV_INLINE CV_NORETURN void errorNoReturn(int _code, const String& _err, const ch # endif #endif } -#ifdef __GNUC__ + +#if defined(__clang__) && defined(_MSC_VER) // MSVC-Clang +# pragma clang diagnostic pop +#elif defined(__GNUC__) # if defined __clang__ || defined __APPLE__ # pragma GCC diagnostic pop # endif From cb0ee1f8e1bdc92c4a8a0bd88590b45a972164eb Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Fri, 29 Oct 2021 17:21:13 +0800 Subject: [PATCH 337/376] Support latest WebNN Clamp op --- modules/dnn/src/layers/elementwise_layers.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 528d062c385c..93ec661403a1 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -620,8 +620,8 @@ struct ReLU6Functor : public BaseFunctor ml::Operand initWebnnAPI(const ml::GraphBuilder& builder, const ml::Operand& input) { ml::ClampOptions clampOptions; - clampOptions.minValue = webnn::BuildConstant(builder, {}, &minValue, 1 * sizeof(float), ml::OperandType::Float32); - clampOptions.maxValue = webnn::BuildConstant(builder, {}, &maxValue, 1 * sizeof(float), ml::OperandType::Float32); + clampOptions.minValue = minValue; + clampOptions.maxValue = maxValue; return builder.Clamp(input, &clampOptions); } #endif From 9637cf05741e022f83549f5d7284d45c6a68cf71 Mon Sep 17 00:00:00 2001 From: Ihsan Soydemir Date: Fri, 29 Oct 2021 13:30:51 +0300 Subject: [PATCH 338/376] Correct drive links for DB_IC15 and DB_TD500 --- .../dnn/dnn_text_spotting/dnn_text_spotting.markdown | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown index 5c465941ca42..0a22f272b573 100644 --- a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown +++ b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown @@ -102,7 +102,7 @@ recommended parameter setting: -inputHeight=736, -inputWidth=1280; description: This model is trained on ICDAR2015, so it can only detect English text instances. - DB_IC15_resnet18.onnx: -url: https://drive.google.com/uc?export=dowload&id=1sZszH3pEt8hliyBlTmB-iulxHP1dCQWV +url: https://drive.google.com/uc?export=dowload&id=1vY_KsDZZZb_svd5RT6pjyI8BS1nPbBSX sha: 19543ce09b2efd35f49705c235cc46d0e22df30b recommended parameter setting: -inputHeight=736, -inputWidth=1280; description: This model is trained on ICDAR2015, so it can only detect English text instances. @@ -114,7 +114,7 @@ recommended parameter setting: -inputHeight=736, -inputWidth=736; description: This model is trained on MSRA-TD500, so it can detect both English and Chinese text instances. - DB_TD500_resnet18.onnx: -url: https://drive.google.com/uc?export=dowload&id=1vY_KsDZZZb_svd5RT6pjyI8BS1nPbBSX +url: https://drive.google.com/uc?export=dowload&id=1sZszH3pEt8hliyBlTmB-iulxHP1dCQWV sha: 8a3700bdc13e00336a815fc7afff5dcc1ce08546 recommended parameter setting: -inputHeight=736, -inputWidth=736; description: This model is trained on MSRA-TD500, so it can detect both English and Chinese text instances. From e5647cf70d5d207b2147163a5ba842dfab7f3641 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 29 Oct 2021 02:02:32 +0300 Subject: [PATCH 339/376] cmake: use CMAKE_BUILD_TYPE=Release by default --- CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d9f98bf23ec0..b660cdd33b74 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,6 +98,12 @@ ocv_cmake_hook(CMAKE_INIT) # must go before the project()/enable_language() commands ocv_update(CMAKE_CONFIGURATION_TYPES "Debug;Release" CACHE STRING "Configs" FORCE) +if(NOT DEFINED CMAKE_BUILD_TYPE + AND NOT OPENCV_SKIP_DEFAULT_BUILD_TYPE +) + message(STATUS "'Release' build type is used by default. Use CMAKE_BUILD_TYPE to specify build type (Release or Debug)") + set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build") +endif() if(DEFINED CMAKE_BUILD_TYPE) set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "${CMAKE_CONFIGURATION_TYPES}") endif() @@ -599,10 +605,6 @@ endif() # ---------------------------------------------------------------------------- # OpenCV compiler and linker options # ---------------------------------------------------------------------------- -# In case of Makefiles if the user does not setup CMAKE_BUILD_TYPE, assume it's Release: -if(CMAKE_GENERATOR MATCHES "Makefiles|Ninja" AND "${CMAKE_BUILD_TYPE}" STREQUAL "") - set(CMAKE_BUILD_TYPE Release) -endif() ocv_cmake_hook(POST_CMAKE_BUILD_OPTIONS) From 6a73e5a7207393adfc93c0084de3ace15d772df2 Mon Sep 17 00:00:00 2001 From: Trutnev Aleksei Date: Fri, 29 Oct 2021 19:30:35 +0300 Subject: [PATCH 340/376] Merge pull request #20922 from alexgiving:atrutnev/align_expect_assert_macros GAPI: Align EXPECT/ASSERT macros * Align TEST macros * restart CI * Fix ASSERT_GT in gapi_async_test --- .../gapi/test/common/gapi_core_tests_inl.hpp | 56 ++++++------- .../test/common/gapi_imgproc_tests_inl.hpp | 80 +++++++++---------- .../test/common/gapi_operators_tests_inl.hpp | 6 +- modules/gapi/test/gapi_opaque_tests.cpp | 16 ++-- .../gapi/test/infer/gapi_infer_onnx_test.cpp | 2 +- .../test/internal/gapi_int_gmetaarg_test.cpp | 2 +- modules/gapi/test/own/conc_queue_tests.cpp | 2 +- .../test/own/last_written_value_tests.cpp | 2 +- modules/gapi/test/own/mat_tests.cpp | 78 +++++++++--------- modules/gapi/test/own/scalar_tests.cpp | 18 ++--- .../test/streaming/gapi_streaming_tests.cpp | 4 +- .../gapi_streaming_vpl_core_test.cpp | 70 ++++++++-------- modules/gapi/test/util/any_tests.cpp | 30 +++---- modules/gapi/test/util/variant_tests.cpp | 42 +++++----- 14 files changed, 204 insertions(+), 204 deletions(-) diff --git a/modules/gapi/test/common/gapi_core_tests_inl.hpp b/modules/gapi/test/common/gapi_core_tests_inl.hpp index d9287a176c46..09b61e28764a 100644 --- a/modules/gapi/test/common/gapi_core_tests_inl.hpp +++ b/modules/gapi/test/common/gapi_core_tests_inl.hpp @@ -138,7 +138,7 @@ TEST_P(MathOpTest, MatricesAccuracyTest) #else EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); #endif - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -173,7 +173,7 @@ TEST_P(MulDoubleTest, AccuracyTest) #else EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); #endif - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } TEST_P(DivTest, DISABLED_DivByZeroTest) // https://github.com/opencv/opencv/pull/12826 @@ -195,7 +195,7 @@ TEST_P(DivTest, DISABLED_DivByZeroTest) // https://github.com/opencv/opencv/pul // Comparison ////////////////////////////////////////////////////////////// { EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -304,7 +304,7 @@ TEST_P(Polar2CartTest, AccuracyTest) // // TODO: Make threshold a configurable parameter of this test (ADE-221) - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); cv::Mat &outx = out_mat_gapi, &outy = out_mat2; @@ -347,7 +347,7 @@ TEST_P(Cart2PolarTest, AccuracyTest) // // TODO: Make threshold a configurable parameter of this test (ADE-221) - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); cv::Mat &outm = out_mat_gapi, &outa = out_mat2; @@ -406,7 +406,7 @@ TEST_P(CmpTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); } } @@ -464,7 +464,7 @@ TEST_P(BitwiseTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); } } @@ -484,7 +484,7 @@ TEST_P(NotTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); } } @@ -508,7 +508,7 @@ TEST_P(SelectTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); } } @@ -528,7 +528,7 @@ TEST_P(MinTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); } } @@ -548,7 +548,7 @@ TEST_P(MaxTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); } } @@ -568,7 +568,7 @@ TEST_P(AbsDiffTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); } } @@ -589,7 +589,7 @@ TEST_P(AbsDiffCTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); } } @@ -659,7 +659,7 @@ TEST_P(AddWeightedTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } TEST_P(NormTest, AccuracyTest) @@ -738,7 +738,7 @@ TEST_P(ThresholdTest, AccuracyTestBinary) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cv::norm(out_mat_ocv, out_mat_gapi, NORM_L1)); } } @@ -764,7 +764,7 @@ TEST_P(ThresholdOTTest, AccuracyTestOtsu) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(ocv_res, out_gapi_scalar.val[0]); } } @@ -788,7 +788,7 @@ TEST_P(InRangeTest, AccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); } } @@ -992,7 +992,7 @@ TEST_P(RemapTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1011,7 +1011,7 @@ TEST_P(FlipTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1037,7 +1037,7 @@ TEST_P(CropTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz_out); + EXPECT_EQ(sz_out, out_mat_gapi.size()); } } @@ -1063,7 +1063,7 @@ TEST_P(CopyTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz_out); + EXPECT_EQ(sz_out, out_mat_gapi.size()); } } @@ -1245,7 +1245,7 @@ TEST_P(LUTTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1269,7 +1269,7 @@ TEST_P(ConvertToTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1378,7 +1378,7 @@ TEST_P(NormalizeTest, Test) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1654,7 +1654,7 @@ TEST_P(ReInitOutTest, TestWithAdd) // Comparison ////////////////////////////////////////////////////////////// EXPECT_EQ(0, cvtest::norm(out_mat_gapi, out_mat_ocv, NORM_INF)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); }; // run for uninitialized output @@ -1738,7 +1738,7 @@ TEST_P(SizeTest, ParseTest) cv::GComputation c(cv::GIn(in), cv::GOut(out)); c.apply(cv::gin(in_mat1), cv::gout(out_sz), getCompileArgs()); - EXPECT_EQ(out_sz, sz); + EXPECT_EQ(sz, out_sz); } TEST_P(SizeRTest, ParseTest) @@ -1751,7 +1751,7 @@ TEST_P(SizeRTest, ParseTest) cv::GComputation c(cv::GIn(op_rect), cv::GOut(out)); c.apply(cv::gin(rect), cv::gout(out_sz), getCompileArgs()); - EXPECT_EQ(out_sz, sz); + EXPECT_EQ(sz, out_sz); } namespace { @@ -1784,7 +1784,7 @@ TEST_P(SizeMFTest, ParseTest) cv::GComputation c(cv::GIn(in), cv::GOut(out)); c.apply(cv::gin(frame), cv::gout(out_sz), getCompileArgs()); - EXPECT_EQ(out_sz, sz); + EXPECT_EQ(sz, out_sz); } } // opencv_test diff --git a/modules/gapi/test/common/gapi_imgproc_tests_inl.hpp b/modules/gapi/test/common/gapi_imgproc_tests_inl.hpp index 755d09a6e4d2..6500c7853d82 100644 --- a/modules/gapi/test/common/gapi_imgproc_tests_inl.hpp +++ b/modules/gapi/test/common/gapi_imgproc_tests_inl.hpp @@ -92,7 +92,7 @@ TEST_P(Filter2DTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -116,7 +116,7 @@ TEST_P(BoxFilterTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -142,7 +142,7 @@ TEST_P(SepFilterTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -163,7 +163,7 @@ TEST_P(BlurTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -185,7 +185,7 @@ TEST_P(GaussianBlurTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -204,7 +204,7 @@ TEST_P(MedianBlurTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -225,7 +225,7 @@ TEST_P(ErodeTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -246,7 +246,7 @@ TEST_P(Erode3x3Test, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -267,7 +267,7 @@ TEST_P(DilateTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -288,7 +288,7 @@ TEST_P(Dilate3x3Test, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -311,7 +311,7 @@ TEST_P(MorphologyExTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -330,7 +330,7 @@ TEST_P(SobelTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -362,8 +362,8 @@ TEST_P(SobelXYTest, AccuracyTest) { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); EXPECT_TRUE(cmpF(out_mat_gapi2, out_mat_ocv2)); - EXPECT_EQ(out_mat_gapi.size(), sz); - EXPECT_EQ(out_mat_gapi2.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); + EXPECT_EQ(sz, out_mat_gapi2.size()); } } @@ -383,7 +383,7 @@ TEST_P(LaplacianTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -402,7 +402,7 @@ TEST_P(BilateralFilterTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -421,7 +421,7 @@ TEST_P(EqHistTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -440,7 +440,7 @@ TEST_P(CannyTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -611,7 +611,7 @@ TEST_P(BGR2RGBTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -630,7 +630,7 @@ TEST_P(RGB2GrayTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -649,7 +649,7 @@ TEST_P(BGR2GrayTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -668,7 +668,7 @@ TEST_P(RGB2YUVTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -687,7 +687,7 @@ TEST_P(YUV2RGBTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -706,7 +706,7 @@ TEST_P(BGR2I420Test, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), Size(sz.width, sz.height * 3 / 2)); + EXPECT_EQ(Size(sz.width, sz.height * 3 / 2), out_mat_gapi.size()); } } @@ -725,7 +725,7 @@ TEST_P(RGB2I420Test, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), Size(sz.width, sz.height * 3 / 2)); + EXPECT_EQ(Size(sz.width, sz.height * 3 / 2), out_mat_gapi.size()); } } @@ -744,7 +744,7 @@ TEST_P(I4202BGRTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), Size(sz.width, sz.height * 2 / 3)); + EXPECT_EQ(Size(sz.width, sz.height * 2 / 3), out_mat_gapi.size()); } } @@ -763,7 +763,7 @@ TEST_P(I4202RGBTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), Size(sz.width, sz.height * 2 / 3)); + EXPECT_EQ(Size(sz.width, sz.height * 2 / 3), out_mat_gapi.size()); } } @@ -787,7 +787,7 @@ TEST_P(NV12toRGBTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -811,7 +811,7 @@ TEST_P(NV12toBGRTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -839,7 +839,7 @@ TEST_P(NV12toGrayTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -882,7 +882,7 @@ TEST_P(NV12toRGBpTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi_planar, out_mat_ocv_planar)); - EXPECT_EQ(out_mat_gapi_planar.size(), sz_p); + EXPECT_EQ(sz_p, out_mat_gapi_planar.size()); } } @@ -912,7 +912,7 @@ TEST_P(NV12toBGRpTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi_planar, out_mat_ocv_planar)); - EXPECT_EQ(out_mat_gapi_planar.size(), sz_p); + EXPECT_EQ(sz_p, out_mat_gapi_planar.size()); } } @@ -931,7 +931,7 @@ TEST_P(RGB2LabTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -950,7 +950,7 @@ TEST_P(BGR2LUVTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -969,7 +969,7 @@ TEST_P(LUV2BGRTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -988,7 +988,7 @@ TEST_P(BGR2YUVTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1007,7 +1007,7 @@ TEST_P(YUV2BGRTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1026,7 +1026,7 @@ TEST_P(RGB2HSVTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1045,7 +1045,7 @@ TEST_P(BayerGR2RGBTest, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } @@ -1064,7 +1064,7 @@ TEST_P(RGB2YUV422Test, AccuracyTest) // Comparison ////////////////////////////////////////////////////////////// { EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); - EXPECT_EQ(out_mat_gapi.size(), sz); + EXPECT_EQ(sz, out_mat_gapi.size()); } } } // opencv_test diff --git a/modules/gapi/test/common/gapi_operators_tests_inl.hpp b/modules/gapi/test/common/gapi_operators_tests_inl.hpp index ad579290d53b..9f739428e003 100644 --- a/modules/gapi/test/common/gapi_operators_tests_inl.hpp +++ b/modules/gapi/test/common/gapi_operators_tests_inl.hpp @@ -36,7 +36,7 @@ TEST_P(MathOperatorMatScalarTest, OperatorAccuracyTest ) // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); } } @@ -63,7 +63,7 @@ TEST_P(MathOperatorMatMatTest, OperatorAccuracyTest ) // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_TRUE(cmpF(out_mat_gapi, out_mat_ocv)); } } @@ -83,7 +83,7 @@ TEST_P(NotOperatorTest, OperatorAccuracyTest) } // Comparison ////////////////////////////////////////////////////////////// { - ASSERT_EQ(out_mat_gapi.size(), sz); + ASSERT_EQ(sz, out_mat_gapi.size()); EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF)); } } diff --git a/modules/gapi/test/gapi_opaque_tests.cpp b/modules/gapi/test/gapi_opaque_tests.cpp index de3572c4bdb2..3af0a68909a2 100644 --- a/modules/gapi/test/gapi_opaque_tests.cpp +++ b/modules/gapi/test/gapi_opaque_tests.cpp @@ -161,7 +161,7 @@ TEST(GOpaque, TestOpaqueBetween) c.apply(cv::gin(mat_in), cv::gout(mat_out), cv::compile_args(cv::gapi::kernels())); int painted = mat_out.at(42, 42); - EXPECT_EQ(painted, 77); + EXPECT_EQ(77, painted); } TEST(GOpaque, TestOpaqueBetweenIslands) @@ -181,7 +181,7 @@ TEST(GOpaque, TestOpaqueBetweenIslands) c.apply(cv::gin(mat_in), cv::gout(mat_out), cv::compile_args(cv::gapi::kernels())); int painted = mat_out.at(42, 42); - EXPECT_EQ(painted, 77); + EXPECT_EQ(77, painted); } TEST(GOpaque, TestOpaqueCustomOut2) @@ -200,11 +200,11 @@ TEST(GOpaque, TestOpaqueCustomOut2) cv::GComputation c(cv::GIn(in1, in2), cv::GOut(std::get<0>(out), std::get<1>(out))); c.apply(cv::gin(input1, input2), cv::gout(out1, out2), cv::compile_args(cv::gapi::kernels())); - EXPECT_EQ(out1.num, input1.size().width * input1.size().height); - EXPECT_EQ(out1.s, str); + EXPECT_EQ(input1.size().width * input1.size().height, out1.num); + EXPECT_EQ(str, out1.s); - EXPECT_EQ(out2.num, input2.size().width * input2.size().height); - EXPECT_EQ(out2.s, str2); + EXPECT_EQ(input2.size().width * input2.size().height, out2.num); + EXPECT_EQ(str2, out2.s); } TEST(GOpaque, TestOpaqueOCLBackendIn) @@ -220,7 +220,7 @@ TEST(GOpaque, TestOpaqueOCLBackendIn) cv::compile_args(cv::gapi::kernels())); int painted = mat_out.at(42, 42); - EXPECT_EQ(painted, 77); + EXPECT_EQ(77, painted); } TEST(GOpaque, TestOpaqueOCLBackendBetween) @@ -240,7 +240,7 @@ TEST(GOpaque, TestOpaqueOCLBackendBetween) cv::compile_args(cv::gapi::kernels())); int painted = mat_out.at(42, 42); - EXPECT_EQ(painted, 77); + EXPECT_EQ(77, painted); } TEST(GOpaque, TestOpaqueOCLBackendOut) diff --git a/modules/gapi/test/infer/gapi_infer_onnx_test.cpp b/modules/gapi/test/infer/gapi_infer_onnx_test.cpp index b1bf9c935694..1f21f95ce89a 100644 --- a/modules/gapi/test/infer/gapi_infer_onnx_test.cpp +++ b/modules/gapi/test/infer/gapi_infer_onnx_test.cpp @@ -231,7 +231,7 @@ void remapToIESSDOut(const std::vector &detections, } // SSD-MobilenetV1 structure check - ASSERT_EQ(detections[0].total(), 1u); + ASSERT_EQ(1u, detections[0].total()); ASSERT_EQ(detections[2].total(), detections[0].total() * 100); ASSERT_EQ(detections[2].total(), detections[3].total()); ASSERT_EQ((detections[2].total() * 4), detections[1].total()); diff --git a/modules/gapi/test/internal/gapi_int_gmetaarg_test.cpp b/modules/gapi/test/internal/gapi_int_gmetaarg_test.cpp index 81f1ad4e2dd0..78048cfa3fee 100644 --- a/modules/gapi/test/internal/gapi_int_gmetaarg_test.cpp +++ b/modules/gapi/test/internal/gapi_int_gmetaarg_test.cpp @@ -102,7 +102,7 @@ TEST(GMetaArg, Can_Get_Metas_From_Input_Run_Args) GMatDesc m_desc; GMetaArgs meta_args = descr_of(cv::gin(m, s, v)); - EXPECT_EQ(meta_args.size(), 3u); + EXPECT_EQ(3u, meta_args.size()); EXPECT_NO_THROW(m_desc = util::get(meta_args[0])); EXPECT_NO_THROW(util::get(meta_args[1])); EXPECT_NO_THROW(util::get(meta_args[2])); diff --git a/modules/gapi/test/own/conc_queue_tests.cpp b/modules/gapi/test/own/conc_queue_tests.cpp index 6e268f318cb8..6234a42719b4 100644 --- a/modules/gapi/test/own/conc_queue_tests.cpp +++ b/modules/gapi/test/own/conc_queue_tests.cpp @@ -27,7 +27,7 @@ TEST(ConcQueue, PushPop) { int x; q.pop(x); - EXPECT_EQ(x, i); + EXPECT_EQ(i, x); } } diff --git a/modules/gapi/test/own/last_written_value_tests.cpp b/modules/gapi/test/own/last_written_value_tests.cpp index 4bfb27f15f3c..be9f13e47bc4 100644 --- a/modules/gapi/test/own/last_written_value_tests.cpp +++ b/modules/gapi/test/own/last_written_value_tests.cpp @@ -21,7 +21,7 @@ TEST(LastValue, PushPop) { int x = 1; v.pop(x); - EXPECT_EQ(x, i); + EXPECT_EQ(i, x); } } diff --git a/modules/gapi/test/own/mat_tests.cpp b/modules/gapi/test/own/mat_tests.cpp index 0ddc201f9fa1..22da4ef7186d 100644 --- a/modules/gapi/test/own/mat_tests.cpp +++ b/modules/gapi/test/own/mat_tests.cpp @@ -24,12 +24,12 @@ inline std::size_t multiply_dims(Dims const& dims){ TEST(OwnMat, DefaultConstruction) { Mat m; - ASSERT_EQ(m.data, nullptr); - ASSERT_EQ(m.cols, 0); - ASSERT_EQ(m.rows, 0); - ASSERT_EQ(m.cols, 0); - ASSERT_EQ(m.type(), 0); - ASSERT_EQ(m.depth(), 0); + ASSERT_EQ(nullptr, m.data); + ASSERT_EQ(0, m.cols); + ASSERT_EQ(0, m.rows); + ASSERT_EQ(0, m.cols); + ASSERT_EQ(0, m.type()); + ASSERT_EQ(0, m.depth()); ASSERT_TRUE(m.dims.empty()); ASSERT_TRUE(m.empty()); } @@ -40,15 +40,15 @@ TEST(OwnMat, Create) Mat m; m.create(size, CV_8UC1); - ASSERT_NE(m.data, nullptr); - ASSERT_EQ((cv::gapi::own::Size{m.cols, m.rows}), size); + ASSERT_NE(nullptr, m.data); + ASSERT_EQ(size, (cv::gapi::own::Size{m.cols, m.rows})); - ASSERT_EQ(m.total(), static_cast(size.height) * size.width); - ASSERT_EQ(m.type(), CV_8UC1); - ASSERT_EQ(m.depth(), CV_8U); - ASSERT_EQ(m.channels(), 1); - ASSERT_EQ(m.elemSize(), sizeof(uint8_t)); - ASSERT_EQ(m.step, sizeof(uint8_t) * m.cols); + ASSERT_EQ(static_cast(size.height) * size.width, m.total()); + ASSERT_EQ(CV_8UC1, m.type()); + ASSERT_EQ(CV_8U, m.depth()); + ASSERT_EQ(1, m.channels()); + ASSERT_EQ(sizeof(uint8_t), m.elemSize()); + ASSERT_EQ(sizeof(uint8_t) * m.cols, m.step); ASSERT_TRUE(m.dims.empty()); ASSERT_FALSE(m.empty()); } @@ -59,16 +59,16 @@ TEST(OwnMat, CreateND) Mat m; m.create(dims, CV_32F); - ASSERT_NE(nullptr , m.data ); + ASSERT_NE(nullptr , m.data); ASSERT_EQ((cv::gapi::own::Size{0,0}), (cv::gapi::own::Size{m.cols, m.rows})); ASSERT_EQ(multiply_dims(dims), m.total()); - ASSERT_EQ(CV_32F , m.type() ); - ASSERT_EQ(CV_32F , m.depth() ); - ASSERT_EQ(-1 , m.channels()); - ASSERT_EQ(sizeof(float) , m.elemSize()); - ASSERT_EQ(0u , m.step ); - ASSERT_EQ(dims , m.dims ); + ASSERT_EQ(CV_32F, m.type()); + ASSERT_EQ(CV_32F, m.depth()); + ASSERT_EQ(-1, m.channels()); + ASSERT_EQ(sizeof(float), m.elemSize()); + ASSERT_EQ(0u, m.step); + ASSERT_EQ(dims, m.dims); ASSERT_FALSE(m.empty()); } @@ -78,15 +78,15 @@ TEST(OwnMat, CreateOverload) Mat m; m.create(size.height,size.width, CV_8UC1); - ASSERT_NE(m.data, nullptr); - ASSERT_EQ((cv::Size{m.cols, m.rows}), size); + ASSERT_NE(nullptr, m.data); + ASSERT_EQ(size, (cv::Size{m.cols, m.rows})); - ASSERT_EQ(m.total(), static_cast(size.height) * size.width); - ASSERT_EQ(m.type(), CV_8UC1); - ASSERT_EQ(m.depth(), CV_8U); - ASSERT_EQ(m.channels(), 1); - ASSERT_EQ(m.elemSize(), sizeof(uint8_t)); - ASSERT_EQ(m.step, sizeof(uint8_t) * m.cols); + ASSERT_EQ(static_cast(size.height) * size.width, m.total()); + ASSERT_EQ(CV_8UC1, m.type()); + ASSERT_EQ(CV_8U, m.depth()); + ASSERT_EQ(1, m.channels()); + ASSERT_EQ(sizeof(uint8_t), m.elemSize()); + ASSERT_EQ(sizeof(uint8_t) * m.cols, m.step); ASSERT_TRUE(m.dims.empty()); ASSERT_FALSE(m.empty()); } @@ -97,14 +97,14 @@ TEST(OwnMat, Create3chan) Mat m; m.create(size, CV_8UC3); - ASSERT_NE(m.data, nullptr); - ASSERT_EQ((cv::Size{m.cols, m.rows}), size); + ASSERT_NE(nullptr, m.data); + ASSERT_EQ(size, (cv::Size{m.cols, m.rows})); - ASSERT_EQ(m.type(), CV_8UC3); - ASSERT_EQ(m.depth(), CV_8U); - ASSERT_EQ(m.channels(), 3); - ASSERT_EQ(m.elemSize(), 3 * sizeof(uint8_t)); - ASSERT_EQ(m.step, 3* sizeof(uint8_t) * m.cols); + ASSERT_EQ(CV_8UC3, m.type()); + ASSERT_EQ(CV_8U, m.depth()); + ASSERT_EQ(3, m.channels()); + ASSERT_EQ(3 * sizeof(uint8_t), m.elemSize()); + ASSERT_EQ(3 * sizeof(uint8_t) * m.cols, m.step); ASSERT_TRUE(m.dims.empty()); ASSERT_FALSE(m.empty()); } @@ -134,7 +134,7 @@ namespace { }; void ensure_mats_are_same(Mat const& copy, Mat const& m){ - EXPECT_NE(copy.data, nullptr); + EXPECT_NE(nullptr, copy.data); EXPECT_EQ(state_of(copy), state_of(m)); } } @@ -157,8 +157,8 @@ struct OwnMatMoveSemantics : NonEmptyMat, ::testing::Test { void ensure_state_moved_to(Mat const& moved_to) { - EXPECT_EQ(state_of(moved_to), initial_state); - EXPECT_EQ(state_of(moved_from), state_of(Mat{})); + EXPECT_EQ(state_of(moved_to), initial_state); + EXPECT_EQ(state_of(moved_from), state_of(Mat{})); } }; diff --git a/modules/gapi/test/own/scalar_tests.cpp b/modules/gapi/test/own/scalar_tests.cpp index 09fec67ab994..05f97be8dafe 100644 --- a/modules/gapi/test/own/scalar_tests.cpp +++ b/modules/gapi/test/own/scalar_tests.cpp @@ -17,7 +17,7 @@ TEST(Scalar, CreateEmpty) for (int i = 0; i < 4; ++i) { - EXPECT_EQ(s[i], 0.0); + EXPECT_EQ(0.0, s[i]); } } @@ -25,20 +25,20 @@ TEST(Scalar, CreateFromVal) { cv::gapi::own::Scalar s(5.0); - EXPECT_EQ(s[0], 5.0); - EXPECT_EQ(s[1], 0.0); - EXPECT_EQ(s[2], 0.0); - EXPECT_EQ(s[3], 0.0); + EXPECT_EQ(5.0, s[0]); + EXPECT_EQ(0.0, s[1]); + EXPECT_EQ(0.0, s[2]); + EXPECT_EQ(0.0, s[3]); } TEST(Scalar, CreateFromVals) { cv::gapi::own::Scalar s(5.3, 3.3, 4.1, -2.0); - EXPECT_EQ(s[0], 5.3); - EXPECT_EQ(s[1], 3.3); - EXPECT_EQ(s[2], 4.1); - EXPECT_EQ(s[3], -2.0); + EXPECT_EQ(5.3, s[0]); + EXPECT_EQ(3.3, s[1]); + EXPECT_EQ(4.1, s[2]); + EXPECT_EQ(-2.0, s[3]); } } // namespace opencv_test diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp index 4f8b2563acc4..57e061861c32 100644 --- a/modules/gapi/test/streaming/gapi_streaming_tests.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_tests.cpp @@ -1130,7 +1130,7 @@ TEST_F(GAPI_Streaming_TemplateTypes, UnusedVectorIsOK) } GAPI_Assert(out_mat || out_int); if (out_int) { - EXPECT_EQ( 3, out_int.value()); + EXPECT_EQ(3, out_int.value()); } } } @@ -1748,7 +1748,7 @@ TEST(GAPI_Streaming_Desync, MultipleDesyncOutputs_1) { if (out_vec || out_int) { EXPECT_EQ(320, out_vec.value()[0]); EXPECT_EQ(240, out_vec.value()[1]); - EXPECT_EQ( 3, out_int.value()); + EXPECT_EQ(3, out_int.value()); } } } diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp index 0b8822b366a0..3ffdaf0fc9bf 100644 --- a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp @@ -124,11 +124,11 @@ TEST(OneVPL_Source_Surface, InitSurface) // check self consistency EXPECT_EQ(reinterpret_cast(surf->get_handle()), reinterpret_cast(mfx_core_handle)); - EXPECT_EQ(surf->get_locks_count(), 0); - EXPECT_EQ(surf->obtain_lock(), 0); - EXPECT_EQ(surf->get_locks_count(), 1); - EXPECT_EQ(surf->release_lock(), 1); - EXPECT_EQ(surf->get_locks_count(), 0); + EXPECT_EQ(0, surf->get_locks_count()); + EXPECT_EQ(0, surf->obtain_lock()); + EXPECT_EQ(1, surf->get_locks_count()); + EXPECT_EQ(1, surf->release_lock()); + EXPECT_EQ(0, surf->get_locks_count()); } TEST(OneVPL_Source_Surface, ConcurrentLock) @@ -143,7 +143,7 @@ TEST(OneVPL_Source_Surface, ConcurrentLock) auto surf = Surface::create_surface(std::move(handle), associated_memory); // check self consistency - EXPECT_EQ(surf->get_locks_count(), 0); + EXPECT_EQ(0, surf->get_locks_count()); // MFX internal limitation: do not exceede U16 range // so I16 is using here @@ -168,7 +168,7 @@ TEST(OneVPL_Source_Surface, ConcurrentLock) } worker_thread.join(); - EXPECT_EQ(surf->get_locks_count(), lock_counter * 2); + EXPECT_EQ(lock_counter * 2, surf->get_locks_count()); } TEST(OneVPL_Source_Surface, MemoryLifeTime) @@ -180,7 +180,7 @@ TEST(OneVPL_Source_Surface, MemoryLifeTime) std::shared_ptr associated_memory (preallocated_memory_ptr.get(), [&preallocated_memory_ptr] (void* ptr) { EXPECT_TRUE(preallocated_memory_ptr); - EXPECT_EQ(preallocated_memory_ptr.get(), ptr); + EXPECT_EQ(ptr, preallocated_memory_ptr.get()); preallocated_memory_ptr.reset(); }); @@ -201,7 +201,7 @@ TEST(OneVPL_Source_Surface, MemoryLifeTime) } // workspace memory must be alive - EXPECT_EQ(surfaces.size(), 0); + EXPECT_EQ(0, surfaces.size()); EXPECT_TRUE(associated_memory != nullptr); EXPECT_TRUE(preallocated_memory_ptr.get() != nullptr); @@ -223,7 +223,7 @@ TEST(OneVPL_Source_Surface, MemoryLifeTime) associated_memory.reset(); // workspace memory must be still alive - EXPECT_EQ(surfaces.size(), 0); + EXPECT_EQ(0, surfaces.size()); EXPECT_TRUE(associated_memory == nullptr); EXPECT_TRUE(preallocated_memory_ptr.get() != nullptr); @@ -246,13 +246,13 @@ TEST(OneVPL_Source_CPU_FrameAdapter, InitFrameAdapter) auto surf = Surface::create_surface(std::move(handle), associated_memory); // check consistency - EXPECT_EQ(surf->get_locks_count(), 0); + EXPECT_EQ(0, surf->get_locks_count()); { VPLMediaFrameCPUAdapter adapter(surf); - EXPECT_EQ(surf->get_locks_count(), 1); + EXPECT_EQ(1, surf->get_locks_count()); } - EXPECT_EQ(surf->get_locks_count(), 0); + EXPECT_EQ(0, surf->get_locks_count()); } TEST(OneVPL_Source_CPU_Accelerator, InitDestroy) @@ -276,8 +276,8 @@ TEST(OneVPL_Source_CPU_Accelerator, InitDestroy) surface_size_bytes, create_test_surface); // check consistency - EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); - EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + EXPECT_EQ(surface_count, acceleration_policy->get_surface_count(key)); + EXPECT_EQ(surface_count, acceleration_policy->get_free_surface_count(key)); pool_export_keys.push_back(key); } @@ -301,8 +301,8 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConsume) surface_size_bytes, create_test_surface); // check consistency - EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); - EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + EXPECT_EQ(surface_count, acceleration_policy->get_surface_count(key)); + EXPECT_EQ(surface_count, acceleration_policy->get_free_surface_count(key)); // consume available surfaces std::vector> surfaces; @@ -310,13 +310,13 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConsume) for (size_t i = 0; i < surface_count; i++) { std::shared_ptr surf = acceleration_policy->get_free_surface(key).lock(); EXPECT_TRUE(surf.get() != nullptr); - EXPECT_EQ(surf->obtain_lock(), 0); + EXPECT_EQ(0, surf->obtain_lock()); surfaces.push_back(std::move(surf)); } // check consistency (no free surfaces) EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); - EXPECT_EQ(acceleration_policy->get_free_surface_count(key), 0); + EXPECT_EQ(0, acceleration_policy->get_free_surface_count(key)); // fail consume non-free surfaces for (size_t i = 0; i < surface_count; i++) { @@ -325,19 +325,19 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConsume) // release surfaces for (auto& surf : surfaces) { - EXPECT_EQ(surf->release_lock(), 1); + EXPECT_EQ(1, surf->release_lock()); } surfaces.clear(); // check consistency - EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); - EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + EXPECT_EQ(surface_count, acceleration_policy->get_surface_count(key)); + EXPECT_EQ(surface_count, acceleration_policy->get_free_surface_count(key)); //check availability after release for (size_t i = 0; i < surface_count; i++) { std::shared_ptr surf = acceleration_policy->get_free_surface(key).lock(); EXPECT_TRUE(surf.get() != nullptr); - EXPECT_EQ(surf->obtain_lock(), 0); + EXPECT_EQ(0, surf->obtain_lock()); } } @@ -358,8 +358,8 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConcurrentConsume) create_test_surface); // check consistency - EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); - EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + EXPECT_EQ(surface_count, acceleration_policy->get_surface_count(key)); + EXPECT_EQ(surface_count, acceleration_policy->get_free_surface_count(key)); // consume available surfaces std::vector> surfaces; @@ -367,7 +367,7 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConcurrentConsume) for (size_t i = 0; i < surface_count; i++) { std::shared_ptr surf = acceleration_policy->get_free_surface(key).lock(); EXPECT_TRUE(surf.get() != nullptr); - EXPECT_EQ(surf->obtain_lock(), 0); + EXPECT_EQ(0, surf->obtain_lock()); surfaces.push_back(std::move(surf)); } @@ -381,7 +381,7 @@ TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConcurrentConsume) // concurrent release surfaces size_t surfaces_count = surfaces.size(); for (auto& surf : surfaces) { - EXPECT_EQ(surf->release_lock(), 1); + EXPECT_EQ(1, surf->release_lock()); std::this_thread::sleep_for(std::chrono::seconds(1)); } surfaces.clear(); @@ -415,28 +415,28 @@ TEST(OneVPL_Source_ProcessingEngine, Init) mfxSession mfx_session{}; engine.initialize_session(mfx_session, DecoderParams{}, std::shared_ptr{}); - EXPECT_EQ(engine.get_ready_frames_count(), 0); + EXPECT_EQ(0, engine.get_ready_frames_count()); ProcessingEngineBase::ExecutionStatus ret = engine.process(mfx_session); EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Continue); - EXPECT_EQ(engine.pipeline_stage_num, 0); + EXPECT_EQ(0, engine.pipeline_stage_num); ret = engine.process(mfx_session); EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Continue); - EXPECT_EQ(engine.pipeline_stage_num, 1); + EXPECT_EQ(1, engine.pipeline_stage_num); ret = engine.process(mfx_session); EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Continue); - EXPECT_EQ(engine.pipeline_stage_num, 2); + EXPECT_EQ(2, engine.pipeline_stage_num); ret = engine.process(mfx_session); EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Processed); - EXPECT_EQ(engine.pipeline_stage_num, 3); - EXPECT_EQ(engine.get_ready_frames_count(), 1); + EXPECT_EQ(3, engine.pipeline_stage_num); + EXPECT_EQ(1, engine.get_ready_frames_count()); ret = engine.process(mfx_session); EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::SessionNotFound); - EXPECT_EQ(engine.pipeline_stage_num, 3); - EXPECT_EQ(engine.get_ready_frames_count(), 1); + EXPECT_EQ(3, engine.pipeline_stage_num); + EXPECT_EQ(1, engine.get_ready_frames_count()); cv::gapi::wip::Data frame; engine.get_frame(frame); diff --git a/modules/gapi/test/util/any_tests.cpp b/modules/gapi/test/util/any_tests.cpp index 238c6dbd7bff..890aabc72e36 100644 --- a/modules/gapi/test/util/any_tests.cpp +++ b/modules/gapi/test/util/any_tests.cpp @@ -17,10 +17,10 @@ TEST(Any, basic) any a(8); auto casted_pointer = any_cast(&a); ASSERT_NE(nullptr, casted_pointer); - ASSERT_EQ(*casted_pointer, 8); + ASSERT_EQ(8, *casted_pointer); *casted_pointer = 7; - ASSERT_EQ(any_cast(a), 7); + ASSERT_EQ(7, any_cast(a)); } TEST(Any, any_cast_ref_throws_on_empty) @@ -36,13 +36,13 @@ TEST(Any, copy) using namespace util; any a(8); - ASSERT_EQ(any_cast(a), 8); + ASSERT_EQ(8, any_cast(a)); any b (a); ASSERT_NE(nullptr, any_cast(&b)); - ASSERT_EQ(8 , any_cast(b)); - ASSERT_EQ(8 , any_cast(a)); + ASSERT_EQ(8, any_cast(b)); + ASSERT_EQ(8, any_cast(a)); } TEST(Any, copy_empty) @@ -63,12 +63,12 @@ TEST(Any, move) using namespace util; any a(8); - ASSERT_EQ(any_cast(a), 8); + ASSERT_EQ(8, any_cast(a)); any b (std::move(a)); ASSERT_NE(nullptr, any_cast(&b)); - ASSERT_EQ(8 , any_cast(b)); + ASSERT_EQ(8, any_cast(b)); ASSERT_EQ(nullptr, any_cast(&a)); } @@ -93,12 +93,12 @@ TEST(Any, move_assign) any a(8); any b; - ASSERT_EQ(any_cast(a), 8); + ASSERT_EQ(8, any_cast(a)); b = (std::move(a)); ASSERT_NE(nullptr, any_cast(&b)); - ASSERT_EQ(8 , any_cast(b)); + ASSERT_EQ(8, any_cast(b)); ASSERT_EQ(nullptr, any_cast(&a)); } @@ -108,14 +108,14 @@ TEST(Any, copy_assign) any a(8); any b; - ASSERT_EQ(any_cast(a), 8); + ASSERT_EQ(8, any_cast(a)); ASSERT_EQ(nullptr, any_cast(&b)); b = a; ASSERT_NE(nullptr, any_cast(&b)); - ASSERT_EQ(8 , any_cast(b)); - ASSERT_EQ(8 , any_cast(a)); + ASSERT_EQ(8, any_cast(b)); + ASSERT_EQ(8, any_cast(a)); } TEST(Any, get_ref_to_val_from_any) @@ -125,7 +125,7 @@ TEST(Any, get_ref_to_val_from_any) any a(x); int& casted_ref = any_cast(a); - ASSERT_EQ(casted_ref, 8); + ASSERT_EQ(8, casted_ref); } TEST(Any, update_val_via_ref) @@ -134,9 +134,9 @@ TEST(Any, update_val_via_ref) int x = 8; any a(x); int& casted_ref = any_cast(a); - ASSERT_EQ(casted_ref, 8); + ASSERT_EQ(8, casted_ref); casted_ref = 7; - ASSERT_EQ(any_cast(a), 7); + ASSERT_EQ(7, any_cast(a)); } } // namespace opencv_test diff --git a/modules/gapi/test/util/variant_tests.cpp b/modules/gapi/test/util/variant_tests.cpp index 7725f9a70211..aa096e711c75 100644 --- a/modules/gapi/test/util/variant_tests.cpp +++ b/modules/gapi/test/util/variant_tests.cpp @@ -587,34 +587,34 @@ TEST(Variant, DynamicVisitor) test_validation::MyBoolParamIndexedVisitor visitor(ss); EXPECT_TRUE(cv::util::visit(visitor, var, int{42})); - EXPECT_EQ(ss.str(), std::string("0:42,")); + EXPECT_EQ(std::string("0:42,"), ss.str()); } std::stringstream ss; test_validation::MyBoolNoParamNonIndexedVisitor visitor(ss); cv::util::visit(visitor, var); - EXPECT_EQ(ss.str(), std::string("0:42,")); + EXPECT_EQ(std::string("0:42,"), ss.str()); var = double{1.0}; EXPECT_TRUE(cv::util::visit(visitor, var)); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,")); + EXPECT_EQ(std::string("0:42,1:1,"), ss.str()); var = char{'a'}; EXPECT_TRUE(cv::util::visit(visitor, var)); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,"), ss.str()); var = float{6.0}; EXPECT_TRUE(cv::util::visit(visitor, var)); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,3:6,"), ss.str()); var = test_validation::MyType{}; EXPECT_TRUE(cv::util::visit(visitor, var)); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,3:6,4:MyType,"), ss.str()); var = test_validation::MyClass{}; EXPECT_TRUE(cv::util::visit(visitor, var)); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,5:MyClass,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,3:6,4:MyType,5:MyClass,"), ss.str()); } TEST(Variant, StaticVisitor) @@ -625,27 +625,27 @@ TEST(Variant, StaticVisitor) test_validation::MyVoidNoParamNonIndexedVisitor visitor(ss); cv::util::visit(visitor, var); - EXPECT_EQ(ss.str(), std::string("42,")); + EXPECT_EQ(std::string("42,"), ss.str()); var = double{1.0}; cv::util::visit(visitor, var); - EXPECT_EQ(ss.str(), std::string("42,1,")); + EXPECT_EQ(std::string("42,1,"), ss.str()); var = char{'a'}; cv::util::visit(visitor, var); - EXPECT_EQ(ss.str(), std::string("42,1,a,")); + EXPECT_EQ(std::string("42,1,a,"), ss.str()); var = float{6.0}; cv::util::visit(visitor, var); - EXPECT_EQ(ss.str(), std::string("42,1,a,6,")); + EXPECT_EQ(std::string("42,1,a,6,"), ss.str()); var = test_validation::MyType{}; cv::util::visit(visitor, var); - EXPECT_EQ(ss.str(), std::string("42,1,a,6,MyType,")); + EXPECT_EQ(std::string("42,1,a,6,MyType,"), ss.str()); var = test_validation::MyClass{}; cv::util::visit(visitor, var); - EXPECT_EQ(ss.str(), std::string("42,1,a,6,MyType,MyClass,")); + EXPECT_EQ(std::string("42,1,a,6,MyType,MyClass,"), ss.str()); } TEST(Variant, StaticIndexedVisitor) @@ -655,27 +655,27 @@ TEST(Variant, StaticIndexedVisitor) std::stringstream ss; cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor {ss}, var); - EXPECT_EQ(ss.str(), std::string("0:42,")); + EXPECT_EQ(std::string("0:42,"), ss.str()); var = double{1.0}; cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,")); + EXPECT_EQ(std::string("0:42,1:1,"), ss.str()); var = char{'a'}; cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,"), ss.str()); var = float{6.0}; cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,3:6,"), ss.str()); var = test_validation::MyType{}; cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,3:6,4:MyType,"), ss.str()); var = test_validation::MyClass{}; cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); - EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,5:MyClass,")); + EXPECT_EQ(std::string("0:42,1:1,2:a,3:6,4:MyType,5:MyClass,"), ss.str()); } @@ -686,7 +686,7 @@ TEST(Variant, LambdaVisitor) { cv::util::visit(cv::util::overload_lambdas( [](int value) { - EXPECT_EQ(value, 42); + EXPECT_EQ(42, value); }, [](double) { ADD_FAILURE() << "can't be called for `double`"; @@ -719,7 +719,7 @@ TEST(Variant, LambdaVisitor) ADD_FAILURE() << "can't be called for `double`"; }, [](char value) { - EXPECT_EQ(value, 'c'); + EXPECT_EQ('c', value); }, [](float) { ADD_FAILURE() << "can't be called for `float`"; From 69d0bc8fd56d6287e7851b2b9ee16e5ec4e61e69 Mon Sep 17 00:00:00 2001 From: Julie Bareeva Date: Fri, 29 Oct 2021 19:46:11 +0300 Subject: [PATCH 341/376] Added overflow handling during conversion from float to int for LinearFilter --- modules/core/include/opencv2/core/cuda/filters.hpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/core/include/opencv2/core/cuda/filters.hpp b/modules/core/include/opencv2/core/cuda/filters.hpp index bb94212299a7..bf3147edb95f 100644 --- a/modules/core/include/opencv2/core/cuda/filters.hpp +++ b/modules/core/include/opencv2/core/cuda/filters.hpp @@ -47,6 +47,7 @@ #include "vec_traits.hpp" #include "vec_math.hpp" #include "type_traits.hpp" +#include "nppdefs.h" /** @file * @deprecated Use @ref cudev instead. @@ -95,6 +96,12 @@ namespace cv { namespace cuda { namespace device const int x1 = __float2int_rd(x); const int y1 = __float2int_rd(y); + if (x1 <= NPP_MIN_32S || x1 >= NPP_MAX_32S || y1 <= NPP_MIN_32S || y1 >= NPP_MAX_32S) + { + elem_type src_reg = src(y1, x1); + out = out + src_reg * 1.0f; + return saturate_cast(out); + } const int x2 = x1 + 1; const int y2 = y1 + 1; From 40c748a2aeca327c11c5daac2fa0f387e24b7676 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 29 Oct 2021 16:08:57 +0000 Subject: [PATCH 342/376] python: properly handle step for multichannel case --- modules/core/src/matrix.cpp | 2 +- modules/python/src2/cv2.cpp | 2 ++ modules/python/test/test_misc.py | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/core/src/matrix.cpp b/modules/core/src/matrix.cpp index 4642d3b62320..1729862cb78d 100644 --- a/modules/core/src/matrix.cpp +++ b/modules/core/src/matrix.cpp @@ -463,7 +463,7 @@ Mat::Mat(Size _sz, int _type, void* _data, size_t _step) } else { - CV_Assert(_step >= minstep); + CV_CheckGE(_step, minstep, ""); if (_step % esz1 != 0) { diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index 6231fde67ff4..58b1357eff79 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -788,6 +788,8 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) if (ndims >= 1 && _strides[ndims - 1] != (npy_intp)elemsize*_sizes[ndims]) needcopy = true; + + elemsize = CV_ELEM_SIZE(type); } if (needcopy) diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index 76803992dca9..fe484089f6cc 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -187,6 +187,10 @@ def test_InputArrayOfArrays(self): #res6 = cv.utils.dumpInputArray([a, b]) #self.assertEqual(res6, "InputArrayOfArrays: empty()=false kind=0x00050000 flags=0x01050000 total(-1)=2 dims(-1)=1 size(-1)=2x1 type(0)=CV_32FC1 dims(0)=4 size(0)=[2 3 4 5]") + def test_20968(self): + pixel = np.uint8([[[40, 50, 200]]]) + _ = cv.cvtColor(pixel, cv.COLOR_RGB2BGR) # should not raise exception + def test_parse_to_bool_convertible(self): try_to_convert = partial(self._try_to_convert, cv.utils.dumpBool) for convertible_true in (True, 1, 64, np.bool(1), np.int8(123), np.int16(11), np.int32(2), From 0ee61d178f79d9d679af6b34eda1009cdf1415c1 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 30 Oct 2021 10:51:52 +0000 Subject: [PATCH 343/376] highgui: drop invalid cvGetWindowImageRect - return type is C++ template - removal from 'extern "C"' scope broke ABI anyway, so this symbols is removed completelly --- modules/highgui/include/opencv2/highgui/highgui_c.h | 5 ----- modules/highgui/src/window.cpp | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/modules/highgui/include/opencv2/highgui/highgui_c.h b/modules/highgui/include/opencv2/highgui/highgui_c.h index 35413139c79a..d8323d0d937f 100644 --- a/modules/highgui/include/opencv2/highgui/highgui_c.h +++ b/modules/highgui/include/opencv2/highgui/highgui_c.h @@ -135,11 +135,6 @@ CVAPI(int) cvNamedWindow( const char* name, int flags CV_DEFAULT(CV_WINDOW_AUTOS CVAPI(void) cvSetWindowProperty(const char* name, int prop_id, double prop_value); CVAPI(double) cvGetWindowProperty(const char* name, int prop_id); -#ifdef __cplusplus // FIXIT remove in OpenCV 4.0 -/* Get window image rectangle coordinates, width and height */ -CVAPI(cv::Rect)cvGetWindowImageRect(const char* name); -#endif - /* display image within window (highgui windows remember their content) */ CVAPI(void) cvShowImage( const char* name, const CvArr* image ); diff --git a/modules/highgui/src/window.cpp b/modules/highgui/src/window.cpp index 877d6751c93c..cfe58a0277a6 100644 --- a/modules/highgui/src/window.cpp +++ b/modules/highgui/src/window.cpp @@ -191,6 +191,7 @@ CV_IMPL double cvGetWindowProperty(const char* name, int prop_id) } } +static cv::Rect cvGetWindowImageRect(const char* name) { if (!name) From 66f3e9745724d0a4b984d764ccf3298a13e69302 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 30 Oct 2021 10:57:47 +0000 Subject: [PATCH 344/376] highgui: drop invalid cvGetWindowImageRect --- .../include/opencv2/highgui/highgui_c.h | 5 ----- modules/highgui/src/window.cpp | 22 +++++++------------ 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/modules/highgui/include/opencv2/highgui/highgui_c.h b/modules/highgui/include/opencv2/highgui/highgui_c.h index 5d20b9501e75..e508e1497521 100644 --- a/modules/highgui/include/opencv2/highgui/highgui_c.h +++ b/modules/highgui/include/opencv2/highgui/highgui_c.h @@ -129,11 +129,6 @@ CVAPI(int) cvNamedWindow( const char* name, int flags CV_DEFAULT(CV_WINDOW_AUTOS CVAPI(void) cvSetWindowProperty(const char* name, int prop_id, double prop_value); CVAPI(double) cvGetWindowProperty(const char* name, int prop_id); -#ifdef __cplusplus // FIXIT remove in OpenCV 4.0 -/* Get window image rectangle coordinates, width and height */ -CVAPI(cv::Rect)cvGetWindowImageRect(const char* name); -#endif - /* display image within window (highgui windows remember their content) */ CVAPI(void) cvShowImage( const char* name, const CvArr* image ); diff --git a/modules/highgui/src/window.cpp b/modules/highgui/src/window.cpp index d9481de6da24..481fee9fbda8 100644 --- a/modules/highgui/src/window.cpp +++ b/modules/highgui/src/window.cpp @@ -399,13 +399,13 @@ CV_IMPL double cvGetWindowProperty(const char* name, int prop_id) #endif } -cv::Rect cvGetWindowImageRect(const char* name) +cv::Rect cv::getWindowImageRect(const String& winname) { CV_TRACE_FUNCTION(); - CV_Assert(name); + CV_Assert(!winname.empty()); { - auto window = findWindow_(name); + auto window = findWindow_(winname); if (window) { return window->getImageRect(); @@ -416,7 +416,7 @@ cv::Rect cvGetWindowImageRect(const char* name) auto backend = getCurrentUIBackend(); if (backend) { - CV_LOG_WARNING(NULL, "Can't find window with name: '" << name << "'. Do nothing"); + CV_LOG_WARNING(NULL, "Can't find window with name: '" << winname << "'. Do nothing"); CV_NOT_FOUND_DEPRECATION; } else @@ -427,13 +427,13 @@ cv::Rect cvGetWindowImageRect(const char* name) #else #if defined (HAVE_QT) - return cvGetWindowRect_QT(name); + return cvGetWindowRect_QT(winname.c_str()); #elif defined(HAVE_WIN32UI) - return cvGetWindowRect_W32(name); + return cvGetWindowRect_W32(winname.c_str()); #elif defined (HAVE_GTK) - return cvGetWindowRect_GTK(name); + return cvGetWindowRect_GTK(winname.c_str()); #elif defined (HAVE_COCOA) - return cvGetWindowRect_COCOA(name); + return cvGetWindowRect_COCOA(winname.c_str()); #else return Rect(-1, -1, -1, -1); #endif @@ -441,12 +441,6 @@ cv::Rect cvGetWindowImageRect(const char* name) #endif } -cv::Rect cv::getWindowImageRect(const String& winname) -{ - CV_TRACE_FUNCTION(); - return cvGetWindowImageRect(winname.c_str()); -} - void cv::namedWindow( const String& winname, int flags ) { CV_TRACE_FUNCTION(); From 17234f82d025e3bbfbf611089637e5aa2038e7b8 Mon Sep 17 00:00:00 2001 From: Trutnev Aleksei Date: Mon, 1 Nov 2021 15:43:27 +0300 Subject: [PATCH 345/376] Merge pull request #20836 from alexgiving:atrutnev/rename_Adapter_to_IAdapter * Rename RMat::Adapter to RMat::IAdapter * Add comments --- modules/gapi/include/opencv2/gapi/rmat.hpp | 18 ++++++++++-------- modules/gapi/include/opencv2/gapi/s11n.hpp | 2 +- modules/gapi/src/backends/common/gbackend.hpp | 4 ++-- modules/gapi/test/rmat/rmat_test_common.hpp | 4 ++-- modules/gapi/test/rmat/rmat_tests.cpp | 4 ++-- modules/gapi/test/s11n/gapi_s11n_tests.cpp | 2 +- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/rmat.hpp b/modules/gapi/include/opencv2/gapi/rmat.hpp index 6b289001e7f3..38668d67a589 100644 --- a/modules/gapi/include/opencv2/gapi/rmat.hpp +++ b/modules/gapi/include/opencv2/gapi/rmat.hpp @@ -25,11 +25,11 @@ namespace cv { // "Remote Mat", a general class which provides an abstraction layer over the data // storage and placement (host, remote device etc) and allows to access this data. // -// The device specific implementation is hidden in the RMat::Adapter class +// The device specific implementation is hidden in the RMat::IAdapter class // // The basic flow is the following: // * Backend which is aware of the remote device: -// - Implements own AdapterT class which is derived from RMat::Adapter +// - Implements own AdapterT class which is derived from RMat::IAdapter // - Wraps device memory into RMat via make_rmat utility function: // cv::RMat rmat = cv::make_rmat(args); // @@ -101,25 +101,27 @@ class GAPI_EXPORTS RMat }; enum class Access { R, W }; - class Adapter + class IAdapter + // Adapter class is going to be deleted and renamed as IAdapter { public: - virtual ~Adapter() = default; + virtual ~IAdapter() = default; virtual GMatDesc desc() const = 0; // Implementation is responsible for setting the appropriate callback to // the view when accessed for writing, to ensure that the data from the view // is transferred to the device when the view is destroyed virtual View access(Access) = 0; virtual void serialize(cv::gapi::s11n::IOStream&) { - GAPI_Assert(false && "Generic serialize method of RMat::Adapter does nothing by default. " + GAPI_Assert(false && "Generic serialize method of RMat::IAdapter does nothing by default. " "Please, implement it in derived class to properly serialize the object."); } virtual void deserialize(cv::gapi::s11n::IIStream&) { - GAPI_Assert(false && "Generic deserialize method of RMat::Adapter does nothing by default. " + GAPI_Assert(false && "Generic deserialize method of RMat::IAdapter does nothing by default. " "Please, implement it in derived class to properly deserialize the object."); } }; - using AdapterP = std::shared_ptr; + using Adapter = IAdapter; // Keep backward compatibility + using AdapterP = std::shared_ptr; RMat() = default; RMat(AdapterP&& a) : m_adapter(std::move(a)) {} @@ -136,7 +138,7 @@ class GAPI_EXPORTS RMat // return nullptr if underlying type is different template T* get() const { - static_assert(std::is_base_of::value, "T is not derived from Adapter!"); + static_assert(std::is_base_of::value, "T is not derived from IAdapter!"); GAPI_Assert(m_adapter != nullptr); return dynamic_cast(m_adapter.get()); } diff --git a/modules/gapi/include/opencv2/gapi/s11n.hpp b/modules/gapi/include/opencv2/gapi/s11n.hpp index 53800970d1cb..6863a5ecabb9 100644 --- a/modules/gapi/include/opencv2/gapi/s11n.hpp +++ b/modules/gapi/include/opencv2/gapi/s11n.hpp @@ -431,7 +431,7 @@ struct deserialize_runarg { static GRunArg exec(cv::gapi::s11n::IIStream& is, uint32_t idx) { if (idx == GRunArg::index_of()) { // Type or void (if not found) - using TA = typename cv::util::find_adapter_impl::type; + using TA = typename cv::util::find_adapter_impl::type; return deserialize_arg_with_adapter::exec(is); } else if (idx == GRunArg::index_of()) { // Type or void (if not found) diff --git a/modules/gapi/src/backends/common/gbackend.hpp b/modules/gapi/src/backends/common/gbackend.hpp index b22ec5e17720..f005135d874a 100644 --- a/modules/gapi/src/backends/common/gbackend.hpp +++ b/modules/gapi/src/backends/common/gbackend.hpp @@ -45,7 +45,7 @@ namespace gimpl { #endif } - class RMatAdapter : public RMat::Adapter { + class RMatAdapter : public RMat::IAdapter { cv::Mat m_mat; public: const void* data() const { return m_mat.data; } @@ -58,7 +58,7 @@ namespace gimpl { struct Data; struct RcDesc; - struct GAPI_EXPORTS RMatMediaFrameAdapter final: public cv::RMat::Adapter + struct GAPI_EXPORTS RMatMediaFrameAdapter final: public cv::RMat::IAdapter { using MapDescF = std::function; using MapDataF = std::function; diff --git a/modules/gapi/test/rmat/rmat_test_common.hpp b/modules/gapi/test/rmat/rmat_test_common.hpp index 5685d0625354..218b21b63278 100644 --- a/modules/gapi/test/rmat/rmat_test_common.hpp +++ b/modules/gapi/test/rmat/rmat_test_common.hpp @@ -11,7 +11,7 @@ #include namespace opencv_test { -class RMatAdapterRef : public RMat::Adapter { +class RMatAdapterRef : public RMat::IAdapter { cv::Mat& m_mat; bool& m_callbackCalled; public: @@ -36,7 +36,7 @@ class RMatAdapterRef : public RMat::Adapter { virtual cv::GMatDesc desc() const override { return cv::descr_of(m_mat); } }; -class RMatAdapterCopy : public RMat::Adapter { +class RMatAdapterCopy : public RMat::IAdapter { cv::Mat& m_deviceMat; cv::Mat m_hostMat; bool& m_callbackCalled; diff --git a/modules/gapi/test/rmat/rmat_tests.cpp b/modules/gapi/test/rmat/rmat_tests.cpp index 52c3806c5b04..ea2f9040e582 100644 --- a/modules/gapi/test/rmat/rmat_tests.cpp +++ b/modules/gapi/test/rmat/rmat_tests.cpp @@ -94,7 +94,7 @@ TYPED_TEST(RMatTypedTest, CorrectAdapterCast) { EXPECT_NE(nullptr, this->rmat().template get()); } -class DummyAdapter : public RMat::Adapter { +class DummyAdapter : public RMat::IAdapter { virtual RMat::View access(RMat::Access) override { return {}; } virtual cv::GMatDesc desc() const override { return {}; } }; @@ -103,7 +103,7 @@ TYPED_TEST(RMatTypedTest, IncorrectAdapterCast) { EXPECT_EQ(nullptr, this->rmat().template get()); } -class RMatAdapterForBackend : public RMat::Adapter { +class RMatAdapterForBackend : public RMat::IAdapter { int m_i; public: RMatAdapterForBackend(int i) : m_i(i) {} diff --git a/modules/gapi/test/s11n/gapi_s11n_tests.cpp b/modules/gapi/test/s11n/gapi_s11n_tests.cpp index 4c6e63b55204..94b99f877a3e 100644 --- a/modules/gapi/test/s11n/gapi_s11n_tests.cpp +++ b/modules/gapi/test/s11n/gapi_s11n_tests.cpp @@ -127,7 +127,7 @@ template<> struct CompileArgTag { } // namespace cv namespace { -class MyRMatAdapter : public cv::RMat::Adapter { +class MyRMatAdapter : public cv::RMat::IAdapter { cv::Mat m_mat; int m_value; std::string m_str; From 30d6766db48a7b8265e2c0e08e4dab0e3b778ee8 Mon Sep 17 00:00:00 2001 From: Souriya Trinh Date: Thu, 28 Oct 2021 23:46:13 +0200 Subject: [PATCH 346/376] Add conventional Bayer naming. --- modules/imgproc/doc/colors.markdown | 15 ++- modules/imgproc/doc/pics/Bayer_patterns.png | Bin 0 -> 4298 bytes modules/imgproc/include/opencv2/imgproc.hpp | 129 +++++++++++++------- 3 files changed, 98 insertions(+), 46 deletions(-) create mode 100755 modules/imgproc/doc/pics/Bayer_patterns.png diff --git a/modules/imgproc/doc/colors.markdown b/modules/imgproc/doc/colors.markdown index 47d97a72631d..c1c2abfce735 100644 --- a/modules/imgproc/doc/colors.markdown +++ b/modules/imgproc/doc/colors.markdown @@ -151,16 +151,23 @@ sources on the web, primarily from the Charles Poynton site _0e!SSq;fs+m5LFu}wcb`I}|v zg&*ZbDjcl)p29kiz4j># zd`W}1^&=a9mEe*^{^k8Bn4xxNVz*%I$+3}*Md4Xfr1e}Ng2bJr3IYW_;{q9@c(L#m zwj`BHeb6|gosSE+d*AjXFP9SexHB*a0*Uy8s{2L#f$KmKkUdpn5M-aFGkDKjctjmk zK?wQH2Pkr7W)Zko-D@M>^3gpNBynm~p8IA&r}>O&(;T zG@kJN-a;;A37jwICW-EKND8z>C4R=t@IS^k?eX%-72~ECxG81ac;%x!{X@sr0|wD4 z5pt_=zQs_7WgTV>L(6R&6X42Jxy}u)jmC4uXz+~T8pFsoB9j-LmD=B=phFt>^JT5b z+hI2rQUVx$c!PF1WkVtkPL>{)^G@lA_qAbtwTj%+>9sMaU`65-_<9y~4(IDFaTkYz zQO{wkP7<}~DbZ#4Wi*cbo{pDDp2d&=AMFRaZc*lX3mN>`GDfk@`w7L)vh;kJkmH$C z%3{}da)y&Ha>Eo@F|-I|WBdgw)dD`;VDm-;Z#L%DT|^-u4%V$VYrrn}j$3C27HIyo z3He51N!{DkX-K|D@&5W=-wf)@DgCKztAYRr|m z+lq0Y-EytwX(^Q^Ku(Exb~`OX5sP~nG+X8irGGFFNi)Z^^)|J~7w#HuFCE;Kqc^VU zUm+|zLdHas)!37$(z!5K7w~YXGrz36mZ0TVLg~wNcP9~lAgOw%0>W$YMKiaFhD%41IkdX$cg^+;|hY!`M_yzuhB{1N!^KeJhdin*i<

hpZM`_scsj= z!@f>XjFh|g;+FzdgkUSB>k{1<7RiC9U#~sst75J>Fr8Jy)1pWXG5T(!0Rhn(#P$dI zizZa$*SOsO`Vnrsf57LCuxhl+tD_yj$DbmQq7O3%SxMoF!sXhB-K`x)nEev+k^a(7 zgW)!)wN|gJU}HBIY-C3*nWQ5UO;$2cyPuI!X;Un@Mtvl&;@ZL8N^wQZFzf)0hhbXt z8OcVUVJ=IlK%(Zx<%cI-lXURsR2xQv&kV4K*y7rbZ?!$SQi7b=&k zrD3E6qs;P_lj>;FQ7l!2dUuNlrk8O_PS~D5(1F0x@F{@HvP(ikutmL1c1eN3D6B7p zdffwk*(%lWss0wxPe^!r&sBA2?~Z5Yr0k{6f2q8_A}sc0-cYqxDr3$=(V5>K{gy0b znOb#t$U2K|vnii?D}ztExY8z-P3zeY>?bcC&wECiIs6&fRea4qGImgWHmgHp+xj8! z9Q5a!Ice@sDGzEI(TI7^=Ha}vT4@g6OVjRMAA+*jHbW~C>$pZZGGxU+Yl5A>7m9Cl zcT)5+iEPo$<};Gjta3=+_(Fh4>^X+XO;zTW&aD%X`kUBVX=v4;JB=&&W)g2@;`7I8 zd|BGH{36RoY?cvXAY~GhH1Vpd)Pq{)c4)bpkYXKUnEOPG`sSB+(kAmOf%oH$hazx0 z%+1EjGdluqQ`+@eemx0=@dn#Vg`R!Kjt%gRpInaQPfOKfq%+LQaaY*PLqh8(iP*NU zvn{G_-K)ekW${aG(omZ!@%M2E@OdYFNuOP$z`EV$_&Dpd#rAWhtcZzXSZXQ~-94EN zpZw`(ibB3D+-S7$wq;X&sNT|K6zzTqH3e!(J@Z}!&jKpyz;4r5bOjcHfzodG@tu*JdM1ga^P6`?zo_!H-M>=Fm4_G8!AtAi7030?(KK~ z0NvYhy?sJ`@Td8XM9VoW+{OCzXAP~T1?YQ|57IRpC&U*HbbOu{K0^kMsUx7r+ z1H&L-;0q(Pd8CLAJAPSfe0mcMa>d}~%qSITgLZrW{9 z9Xu}V4+6=fe%A&6(Iej^2WMqY;Qx*6n{@cI2}hp%uWk6Ht*OoSS*p7A@TxyYzW_#V2dU3(GfW%rFHJJMHj@ZH?zQc3abe& zh)kqM*_}uGg6)-#{6$~hKc|dJ$aFIFHj>Y`fge(3s9uN?rX*RG+UJl|4N3Z?(aiB+ zGYn5wzBzZaV9*C_eY_1F89BVlp5G1W3sA6Zv)kLtLC(Jj&2S$(rAjl26pMxKJiHyT zYG~SnuT*Yp_sL@}P3LKAwirgivp4@&GhmME>JOB6YT&hUm0=;@Z&$Eo7h)MkZ&*!~ zvN@zsc^)&uOLQV_b|P)P)FC-sk^K(2Oq%0teML7lS8VbZ`{tCy9V=m_=0E8IVO1Df zZMy>A6)C(eX5nRb`0E(qU>#$CS)0PYP@`Z=gn;&-x&2%(YjZtmIV1XG5v6DXPkSfK zjlq6p_Vdg7@gRBn5jhVVh<&9k<3_y)N>6(fFv8WH47kj|!=+Gz~_K_AFN-Zm5mH&>3yXSyrZ z@T|4&bFb~E48_6D4wz4*vAYWy@0t*lTS^ygUof-B(j}p-DDjr5^NnyQ-KdLoL;_M7 zbR(&NWFGgsCfLZJfprTA76?y3YP9P*^3SdndJf>kMb;14c#gFPU|l2hy8`$->nl}M>4K>kVqB<3Xe z_S0Pu`}(Dk9*tI;--VgM(9+u#V#&->fs~YK`)&9T`v%CMtA`Q zv}>Qad!;)kO`*8QX=el)FkBB7vQF~MrK}^8+Fo|J@xstD+Z9BPNWcRkAwnE>Ln7Eo zVx`@oEY3~p;-n5q>+;1zJE51_?mAH6{7XrE0_RO2a5>jnJR59y(j~*t9PupjD+$K%mmda`6ysXOqMaLyK)!v>UuZ zyI%QR5U}`Y6O(G%QwslSh|~LMrb2X6xXAMTM|hXZHtWeR)ZQU{-DhGhVPs0Jq!`wl zrl!)Iy4`B4)gl4sBA1kUF+VBJYadS=d#kDaFcYEIa1FORia9#(4=+R5<%P}(=R6i@ z7jfc`H~p^|IKvSGO$eZ)wSDVoDes;t4BKfc+Ql%e#FLYE?7X$=tR^p8UiNYX2c9hP zZeFVs=n8Y17i~-G-|4zoM$YV*$crBdwNkd$3@toR932UdSm^g{y8mXf1#QCVl#?94 zTQZjB7P28~Kc(Ev z{l~zL?ni*z{|_YnwFw9Gf5z8e+y3X={eNHsJ_Q7@yal6`^MdprSa7}!Sen^hZTR8# GgMS08YI(;1 literal 0 HcmV?d00001 diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index 576bebc21b4c..8db6cc5d8d4c 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -744,54 +744,99 @@ enum ColorConversionCodes { COLOR_RGBA2YUV_YV12 = 133, COLOR_BGRA2YUV_YV12 = 134, - //! Demosaicing - COLOR_BayerBG2BGR = 46, - COLOR_BayerGB2BGR = 47, - COLOR_BayerRG2BGR = 48, - COLOR_BayerGR2BGR = 49, - - COLOR_BayerBG2RGB = COLOR_BayerRG2BGR, - COLOR_BayerGB2RGB = COLOR_BayerGR2BGR, - COLOR_BayerRG2RGB = COLOR_BayerBG2BGR, - COLOR_BayerGR2RGB = COLOR_BayerGB2BGR, - - COLOR_BayerBG2GRAY = 86, - COLOR_BayerGB2GRAY = 87, - COLOR_BayerRG2GRAY = 88, - COLOR_BayerGR2GRAY = 89, + //! Demosaicing, see @ref color_convert_bayer "color conversions" for additional information + COLOR_BayerBG2BGR = 46, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2BGR = 47, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2BGR = 48, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2BGR = 49, //!< equivalent to GBRG Bayer pattern + + COLOR_BayerRGGB2BGR = COLOR_BayerBG2BGR, + COLOR_BayerGRBG2BGR = COLOR_BayerGB2BGR, + COLOR_BayerBGGR2BGR = COLOR_BayerRG2BGR, + COLOR_BayerGBRG2BGR = COLOR_BayerGR2BGR, + + COLOR_BayerRGGB2RGB = COLOR_BayerBGGR2BGR, + COLOR_BayerGRBG2RGB = COLOR_BayerGBRG2BGR, + COLOR_BayerBGGR2RGB = COLOR_BayerRGGB2BGR, + COLOR_BayerGBRG2RGB = COLOR_BayerGRBG2BGR, + + COLOR_BayerBG2RGB = COLOR_BayerRG2BGR, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2RGB = COLOR_BayerGR2BGR, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2RGB = COLOR_BayerBG2BGR, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2RGB = COLOR_BayerGB2BGR, //!< equivalent to GBRG Bayer pattern + + COLOR_BayerBG2GRAY = 86, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2GRAY = 87, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2GRAY = 88, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2GRAY = 89, //!< equivalent to GBRG Bayer pattern + + COLOR_BayerRGGB2GRAY = COLOR_BayerBG2GRAY, + COLOR_BayerGRBG2GRAY = COLOR_BayerGB2GRAY, + COLOR_BayerBGGR2GRAY = COLOR_BayerRG2GRAY, + COLOR_BayerGBRG2GRAY = COLOR_BayerGR2GRAY, //! Demosaicing using Variable Number of Gradients - COLOR_BayerBG2BGR_VNG = 62, - COLOR_BayerGB2BGR_VNG = 63, - COLOR_BayerRG2BGR_VNG = 64, - COLOR_BayerGR2BGR_VNG = 65, - - COLOR_BayerBG2RGB_VNG = COLOR_BayerRG2BGR_VNG, - COLOR_BayerGB2RGB_VNG = COLOR_BayerGR2BGR_VNG, - COLOR_BayerRG2RGB_VNG = COLOR_BayerBG2BGR_VNG, - COLOR_BayerGR2RGB_VNG = COLOR_BayerGB2BGR_VNG, + COLOR_BayerBG2BGR_VNG = 62, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2BGR_VNG = 63, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2BGR_VNG = 64, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2BGR_VNG = 65, //!< equivalent to GBRG Bayer pattern + + COLOR_BayerRGGB2BGR_VNG = COLOR_BayerBG2BGR_VNG, + COLOR_BayerGRBG2BGR_VNG = COLOR_BayerGB2BGR_VNG, + COLOR_BayerBGGR2BGR_VNG = COLOR_BayerRG2BGR_VNG, + COLOR_BayerGBRG2BGR_VNG = COLOR_BayerGR2BGR_VNG, + + COLOR_BayerRGGB2RGB_VNG = COLOR_BayerBGGR2BGR_VNG, + COLOR_BayerGRBG2RGB_VNG = COLOR_BayerGBRG2BGR_VNG, + COLOR_BayerBGGR2RGB_VNG = COLOR_BayerRGGB2BGR_VNG, + COLOR_BayerGBRG2RGB_VNG = COLOR_BayerGRBG2BGR_VNG, + + COLOR_BayerBG2RGB_VNG = COLOR_BayerRG2BGR_VNG, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2RGB_VNG = COLOR_BayerGR2BGR_VNG, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2RGB_VNG = COLOR_BayerBG2BGR_VNG, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2RGB_VNG = COLOR_BayerGB2BGR_VNG, //!< equivalent to GBRG Bayer pattern //! Edge-Aware Demosaicing - COLOR_BayerBG2BGR_EA = 135, - COLOR_BayerGB2BGR_EA = 136, - COLOR_BayerRG2BGR_EA = 137, - COLOR_BayerGR2BGR_EA = 138, - - COLOR_BayerBG2RGB_EA = COLOR_BayerRG2BGR_EA, - COLOR_BayerGB2RGB_EA = COLOR_BayerGR2BGR_EA, - COLOR_BayerRG2RGB_EA = COLOR_BayerBG2BGR_EA, - COLOR_BayerGR2RGB_EA = COLOR_BayerGB2BGR_EA, + COLOR_BayerBG2BGR_EA = 135, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2BGR_EA = 136, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2BGR_EA = 137, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2BGR_EA = 138, //!< equivalent to GBRG Bayer pattern + + COLOR_BayerRGGB2BGR_EA = COLOR_BayerBG2BGR_EA, + COLOR_BayerGRBG2BGR_EA = COLOR_BayerGB2BGR_EA, + COLOR_BayerBGGR2BGR_EA = COLOR_BayerRG2BGR_EA, + COLOR_BayerGBRG2BGR_EA = COLOR_BayerGR2BGR_EA, + + COLOR_BayerRGGB2RGB_EA = COLOR_BayerBGGR2BGR_EA, + COLOR_BayerGRBG2RGB_EA = COLOR_BayerGBRG2BGR_EA, + COLOR_BayerBGGR2RGB_EA = COLOR_BayerRGGB2BGR_EA, + COLOR_BayerGBRG2RGB_EA = COLOR_BayerGRBG2BGR_EA, + + COLOR_BayerBG2RGB_EA = COLOR_BayerRG2BGR_EA, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2RGB_EA = COLOR_BayerGR2BGR_EA, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2RGB_EA = COLOR_BayerBG2BGR_EA, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2RGB_EA = COLOR_BayerGB2BGR_EA, //!< equivalent to GBRG Bayer pattern //! Demosaicing with alpha channel - COLOR_BayerBG2BGRA = 139, - COLOR_BayerGB2BGRA = 140, - COLOR_BayerRG2BGRA = 141, - COLOR_BayerGR2BGRA = 142, - - COLOR_BayerBG2RGBA = COLOR_BayerRG2BGRA, - COLOR_BayerGB2RGBA = COLOR_BayerGR2BGRA, - COLOR_BayerRG2RGBA = COLOR_BayerBG2BGRA, - COLOR_BayerGR2RGBA = COLOR_BayerGB2BGRA, + COLOR_BayerBG2BGRA = 139, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2BGRA = 140, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2BGRA = 141, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2BGRA = 142, //!< equivalent to GBRG Bayer pattern + + COLOR_BayerRGGB2BGRA = COLOR_BayerBG2BGRA, + COLOR_BayerGRBG2BGRA = COLOR_BayerGB2BGRA, + COLOR_BayerBGGR2BGRA = COLOR_BayerRG2BGRA, + COLOR_BayerGBRG2BGRA = COLOR_BayerGR2BGRA, + + COLOR_BayerRGGB2RGBA = COLOR_BayerBGGR2BGRA, + COLOR_BayerGRBG2RGBA = COLOR_BayerGBRG2BGRA, + COLOR_BayerBGGR2RGBA = COLOR_BayerRGGB2BGRA, + COLOR_BayerGBRG2RGBA = COLOR_BayerGRBG2BGRA, + + COLOR_BayerBG2RGBA = COLOR_BayerRG2BGRA, //!< equivalent to RGGB Bayer pattern + COLOR_BayerGB2RGBA = COLOR_BayerGR2BGRA, //!< equivalent to GRBG Bayer pattern + COLOR_BayerRG2RGBA = COLOR_BayerBG2BGRA, //!< equivalent to BGGR Bayer pattern + COLOR_BayerGR2RGBA = COLOR_BayerGB2BGRA, //!< equivalent to GBRG Bayer pattern COLOR_COLORCVT_MAX = 143 }; From b3e16c6423d34a39a2c1d311ba344f0ceeef6253 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 2 Nov 2021 19:22:47 +0000 Subject: [PATCH 347/376] videoio(dshow): eliminate build warnings from MSVC-Clang --- modules/videoio/src/cap_dshow.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/videoio/src/cap_dshow.cpp b/modules/videoio/src/cap_dshow.cpp index afabe0a194cd..1dc18e0f3c47 100644 --- a/modules/videoio/src/cap_dshow.cpp +++ b/modules/videoio/src/cap_dshow.cpp @@ -93,8 +93,9 @@ Thanks to: #pragma warning(disable: 4995) #endif -#ifdef __MINGW32__ -// MinGW does not understand COM interfaces +#if defined(__clang__) // clang or MSVC clang +#pragma clang diagnostic ignored "-Wnon-virtual-dtor" +#elif defined(__GNUC__) // MinGW #pragma GCC diagnostic ignored "-Wnon-virtual-dtor" #endif From 9e758e2f920133ac3e644346dd52dc16135f602a Mon Sep 17 00:00:00 2001 From: Hanxi Guo Date: Wed, 3 Nov 2021 15:47:51 +0800 Subject: [PATCH 348/376] Add definition of HAVE_WEBNN macro --- CMakeLists.txt | 2 +- cmake/OpenCVDetectWebNN.cmake | 9 ++++----- modules/dnn/CMakeLists.txt | 4 ++++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 17ad595838d8..e8cd8105cf51 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -290,7 +290,7 @@ OCV_OPTION(WITH_INF_ENGINE "Include Intel Inference Engine support" OFF OCV_OPTION(WITH_NGRAPH "Include nGraph support" WITH_INF_ENGINE VISIBLE_IF TRUE VERIFY TARGET ngraph::ngraph) -OCV_OPTION(WITH_WEBNN "Include WebNN support" WITH_WEBNN +OCV_OPTION(WITH_WEBNN "Include WebNN support" OFF VISIBLE_IF TRUE VERIFY HAVE_WEBNN) OCV_OPTION(WITH_JASPER "Include JPEG2K support (Jasper)" ON diff --git a/cmake/OpenCVDetectWebNN.cmake b/cmake/OpenCVDetectWebNN.cmake index 7a099f0e84e0..e7ee687f5f44 100644 --- a/cmake/OpenCVDetectWebNN.cmake +++ b/cmake/OpenCVDetectWebNN.cmake @@ -37,14 +37,13 @@ if(NOT VALID_WEBNN) message(WARNING "Can't use WebNN") return() endif() +else() + set(HAVE_WEBNN ON) + message(STATUS "Set HAVE_WEBNN = ${HAVE_WEBNN}") endif() if(NOT EMSCRIPTEN) message(AUTHOR_WARNING "Use WebNN-native") else() message(AUTHOR_WARNING "Use WebNN") -endif() - -if(VALID_WEBNN) - set(HAVE_WEBNN ON) -endif() +endif() \ No newline at end of file diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index dd19f1bf13fd..d208ca14883c 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -22,6 +22,10 @@ if(OPENCV_DNN_OPENCL AND HAVE_OPENCL) add_definitions(-DCV_OCL4DNN=1) endif() +if(WITH_WEBNN AND HAVE_WEBNN) + add_definitions(-DHAVE_WEBNN=1) +endif() + ocv_option(OPENCV_DNN_CUDA "Build with CUDA support" HAVE_CUDA AND HAVE_CUBLAS From d484939c02eb5247e803365aa91c8312dd4db93d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 3 Nov 2021 18:59:36 +0300 Subject: [PATCH 349/376] Merge pull request #20999 from alalek:dnn_replace_deprecated_calls dnn(protobuf): replace deprecated calls * dnn: replace deprecated ByteSize() => ByteSizeLong() * dnn: replace deprecated calls, use GetRepeatedFieldRef --- modules/dnn/src/caffe/caffe_importer.cpp | 8 ++++---- modules/dnn/src/tensorflow/tf_importer.cpp | 20 ++++++++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/modules/dnn/src/caffe/caffe_importer.cpp b/modules/dnn/src/caffe/caffe_importer.cpp index 2d615c448a10..0b6c0a6e3814 100644 --- a/modules/dnn/src/caffe/caffe_importer.cpp +++ b/modules/dnn/src/caffe/caffe_importer.cpp @@ -49,6 +49,7 @@ #include #include #include +#include #include "caffe_io.hpp" #endif @@ -57,8 +58,7 @@ namespace dnn { CV__DNN_EXPERIMENTAL_NS_BEGIN #ifdef HAVE_PROTOBUF -using ::google::protobuf::RepeatedField; -using ::google::protobuf::RepeatedPtrField; +using ::google::protobuf::RepeatedFieldRef; using ::google::protobuf::Message; using ::google::protobuf::Descriptor; using ::google::protobuf::FieldDescriptor; @@ -136,7 +136,7 @@ class CaffeImporter #define SET_UP_FILED(getter, arrayConstr, gtype) \ if (isRepeated) { \ - const RepeatedField &v = refl->GetRepeatedField(msg, field); \ + const RepeatedFieldRef v = refl->GetRepeatedFieldRef(msg, field); \ params.set(name, DictValue::arrayConstr(v.begin(), (int)v.size())); \ } \ else { \ @@ -168,7 +168,7 @@ class CaffeImporter break; case FieldDescriptor::CPPTYPE_STRING: if (isRepeated) { - const RepeatedPtrField &v = refl->GetRepeatedPtrField(msg, field); + const RepeatedFieldRef v = refl->GetRepeatedFieldRef(msg, field); params.set(name, DictValue::arrayString(v.begin(), (int)v.size())); } else { diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 16c5c163089f..1a01ac87d677 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -2739,14 +2739,21 @@ DataLayout TFImporter::predictOutputDataLayout(const tensorflow::NodeDef& layer) void TFImporter::populateNet() { - CV_Assert(netBin.ByteSize() || netTxt.ByteSize()); +#if GOOGLE_PROTOBUF_VERSION < 3005000 + size_t netBinSize = saturate_cast(netBin.ByteSize()); + size_t netTxtSize = saturate_cast(netTxt.ByteSize()); +#else + size_t netBinSize = netBin.ByteSizeLong(); + size_t netTxtSize = netTxt.ByteSizeLong(); +#endif + CV_Assert(netBinSize || netTxtSize); CV_LOG_INFO(NULL, "DNN/TF: parsing model" << (netBin.has_versions() ? cv::format(" produced by TF v%d (min_consumer=%d)", (int)netBin.versions().producer(), (int)netBin.versions().min_consumer()) : cv::String(" (N/A version info)")) << ". Number of nodes = " << netBin.node_size() ); - if (netTxt.ByteSize()) + if (netTxtSize) { CV_LOG_INFO(NULL, "DNN/TF: parsing config" << (netTxt.has_versions() ? cv::format(" produced by TF v%d (min_consumer=%d)", (int)netTxt.versions().producer(), (int)netTxt.versions().min_consumer()) : cv::String(" (N/A version info)")) @@ -2775,7 +2782,7 @@ void TFImporter::populateNet() CV_LOG_DEBUG(NULL, "DNN/TF: sortByExecutionOrder(model) => " << netBin.node_size() << " nodes"); } - tensorflow::GraphDef& net = netTxt.ByteSize() != 0 ? netTxt : netBin; + tensorflow::GraphDef& net = netTxtSize != 0 ? netTxt : netBin; int layersSize = net.node_size(); @@ -2873,7 +2880,12 @@ void TFImporter::addPermuteLayer(const int* order, const std::string& permName, void TFImporter::parseNode(const tensorflow::NodeDef& layer) { - tensorflow::GraphDef& net = netTxt.ByteSize() != 0 ? netTxt : netBin; +#if GOOGLE_PROTOBUF_VERSION < 3005000 + size_t netTxtSize = saturate_cast(netTxt.ByteSize()); +#else + size_t netTxtSize = netTxt.ByteSizeLong(); +#endif + tensorflow::GraphDef& net = netTxtSize != 0 ? netTxt : netBin; const std::string& name = layer.name(); const std::string& type = layer.op(); From 66dd8712886e7e531bc02fcf8ae53427b71e4194 Mon Sep 17 00:00:00 2001 From: cpengu Date: Fri, 15 Oct 2021 11:11:35 +0800 Subject: [PATCH 350/376] Update qrcode.cpp Fixed issue #20880, QRDetect::searchHorizontalLines() boundary condition will skip the matched qrcode near the end --- modules/objdetect/src/qrcode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/objdetect/src/qrcode.cpp b/modules/objdetect/src/qrcode.cpp index 3cc130ac8916..1479aabf085c 100644 --- a/modules/objdetect/src/qrcode.cpp +++ b/modules/objdetect/src/qrcode.cpp @@ -200,7 +200,7 @@ vector QRDetect::searchHorizontalLines() } } pixels_position.push_back(width_bin_barcode - 1); - for (size_t i = 2; i < pixels_position.size() - 4; i+=2) + for (size_t i = 2; i < pixels_position.size() - 3; i+=2) { test_lines[0] = static_cast(pixels_position[i - 1] - pixels_position[i - 2]); test_lines[1] = static_cast(pixels_position[i ] - pixels_position[i - 1]); From 8e72e1ed88215b91157393e959e1de3d559fc785 Mon Sep 17 00:00:00 2001 From: APrigarina Date: Wed, 3 Nov 2021 20:04:27 +0300 Subject: [PATCH 351/376] add test case for QR detect fix --- modules/objdetect/test/test_qrcode.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/modules/objdetect/test/test_qrcode.cpp b/modules/objdetect/test/test_qrcode.cpp index fc554987400f..19a9f762600f 100644 --- a/modules/objdetect/test/test_qrcode.cpp +++ b/modules/objdetect/test/test_qrcode.cpp @@ -649,6 +649,26 @@ TEST(Objdetect_QRCode_decodeMulti, check_output_parameters_type_19363) #endif } +TEST(Objdetect_QRCode_detect, detect_regression_20882) +{ + const std::string name_current_image = "qrcode_near_the_end.jpg"; + const std::string root = "qrcode/"; + + std::string image_path = findDataFile(root + name_current_image); + Mat src = imread(image_path); + ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; + + QRCodeDetector qrcode; + std::vector corners; + Mat straight_barcode; + cv::String decoded_info; + EXPECT_TRUE(qrcode.detect(src, corners)); + EXPECT_TRUE(!corners.empty()); +#ifdef HAVE_QUIRC + EXPECT_NO_THROW(qrcode.decode(src, corners, straight_barcode)); +#endif +} + TEST(Objdetect_QRCode_basic, not_found_qrcode) { std::vector corners; From c1d61c88e9636bb0e7eb8d2ae1af78160af03f83 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 4 Nov 2021 09:59:19 +0000 Subject: [PATCH 352/376] dnn(cmake): don't hijack OpenCL options with Tengine --- modules/dnn/CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 2e97b8866351..9dc048f5b680 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -13,9 +13,6 @@ ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX) ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java js) ocv_option(OPENCV_DNN_OPENCL "Build with OpenCL support" HAVE_OPENCL AND NOT APPLE) -if(HAVE_TENGINE) - add_definitions(-DHAVE_TENGINE=1) -endif() if(OPENCV_DNN_OPENCL AND HAVE_OPENCL) add_definitions(-DCV_OCL4DNN=1) @@ -23,6 +20,10 @@ else() ocv_cmake_hook_append(INIT_MODULE_SOURCES_opencv_dnn "${CMAKE_CURRENT_LIST_DIR}/cmake/hooks/INIT_MODULE_SOURCES_opencv_dnn.cmake") endif() +if(HAVE_TENGINE) + add_definitions(-DHAVE_TENGINE=1) +endif() + if(MSVC) add_definitions( -D_CRT_SECURE_NO_WARNINGS=1 ) ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4244 /wd4267 /wd4018 /wd4355 /wd4800 /wd4251 /wd4996 /wd4146 From 562f2375c5d6ce9c3366fbdec774088fbf5f5759 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 4 Nov 2021 13:26:33 +0000 Subject: [PATCH 353/376] dnn(test): skip tests with high memory usage - 32-bit configuration may fail due to memory fragmentation --- modules/dnn/test/test_model.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/modules/dnn/test/test_model.cpp b/modules/dnn/test/test_model.cpp index 6ac9702c6993..4c08cc30b948 100644 --- a/modules/dnn/test/test_model.cpp +++ b/modules/dnn/test/test_model.cpp @@ -282,7 +282,10 @@ TEST_P(Test_Model, Classify) TEST_P(Test_Model, DetectRegion) { - applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_MEMORY_1GB); + applyTestTag( + CV_TEST_TAG_LONG, + CV_TEST_TAG_MEMORY_2GB + ); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) @@ -332,7 +335,10 @@ TEST_P(Test_Model, DetectRegion) TEST_P(Test_Model, DetectRegionWithNmsAcrossClasses) { - applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_MEMORY_1GB); + applyTestTag( + CV_TEST_TAG_LONG, + CV_TEST_TAG_MEMORY_2GB + ); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) @@ -582,6 +588,10 @@ TEST_P(Test_Model, Detection_normalized) TEST_P(Test_Model, Segmentation) { + applyTestTag( + CV_TEST_TAG_MEMORY_2GB + ); + std::string inp = _tf("dog416.png"); std::string weights_file = _tf("fcn8s-heavy-pascal.prototxt"); std::string config_file = _tf("fcn8s-heavy-pascal.caffemodel", false); From ffd010767fe401f87a747c0e7f935ddb7da4c63b Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Thu, 4 Nov 2021 22:48:10 +0100 Subject: [PATCH 354/376] Only use fma functions when CV_FMA3 is set. In practice, processors offering AVX2/AVX512 also FMA, that is why it got unnoticed. --- .../core/include/opencv2/core/hal/intrin_avx.hpp | 14 +++++++++++++- .../include/opencv2/core/hal/intrin_avx512.hpp | 14 +++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/modules/core/include/opencv2/core/hal/intrin_avx.hpp b/modules/core/include/opencv2/core/hal/intrin_avx.hpp index 54e8927192c4..7fbf6f131473 100644 --- a/modules/core/include/opencv2/core/hal/intrin_avx.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_avx.hpp @@ -1390,11 +1390,21 @@ OPENCV_HAL_IMPL_AVX_CHECK_SHORT(v_int16x16) ////////// Other math ///////// /** Some frequent operations **/ +#if CV_FMA3 #define OPENCV_HAL_IMPL_AVX_MULADD(_Tpvec, suffix) \ inline _Tpvec v_fma(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ { return _Tpvec(_mm256_fmadd_##suffix(a.val, b.val, c.val)); } \ inline _Tpvec v_muladd(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ - { return _Tpvec(_mm256_fmadd_##suffix(a.val, b.val, c.val)); } \ + { return _Tpvec(_mm256_fmadd_##suffix(a.val, b.val, c.val)); } +#else +#define OPENCV_HAL_IMPL_AVX_MULADD(_Tpvec, suffix) \ + inline _Tpvec v_fma(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ + { return _Tpvec(_mm256_add_##suffix(_mm256_mul_##suffix(a.val, b.val), c.val)); } \ + inline _Tpvec v_muladd(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ + { return _Tpvec(_mm256_add_##suffix(_mm256_mul_##suffix(a.val, b.val), c.val)); } +#endif + +#define OPENCV_HAL_IMPL_AVX_MISC(_Tpvec, suffix) \ inline _Tpvec v_sqrt(const _Tpvec& x) \ { return _Tpvec(_mm256_sqrt_##suffix(x.val)); } \ inline _Tpvec v_sqr_magnitude(const _Tpvec& a, const _Tpvec& b) \ @@ -1404,6 +1414,8 @@ OPENCV_HAL_IMPL_AVX_CHECK_SHORT(v_int16x16) OPENCV_HAL_IMPL_AVX_MULADD(v_float32x8, ps) OPENCV_HAL_IMPL_AVX_MULADD(v_float64x4, pd) +OPENCV_HAL_IMPL_AVX_MISC(v_float32x8, ps) +OPENCV_HAL_IMPL_AVX_MISC(v_float64x4, pd) inline v_int32x8 v_fma(const v_int32x8& a, const v_int32x8& b, const v_int32x8& c) { diff --git a/modules/core/include/opencv2/core/hal/intrin_avx512.hpp b/modules/core/include/opencv2/core/hal/intrin_avx512.hpp index 75a3bd4b85fe..d20d6dd1ffd3 100644 --- a/modules/core/include/opencv2/core/hal/intrin_avx512.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_avx512.hpp @@ -1385,11 +1385,21 @@ inline v_uint64x8 v_popcount(const v_uint64x8& a) { return v_popcount(v_reinte ////////// Other math ///////// /** Some frequent operations **/ +#if CV_FMA3 #define OPENCV_HAL_IMPL_AVX512_MULADD(_Tpvec, suffix) \ inline _Tpvec v_fma(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ { return _Tpvec(_mm512_fmadd_##suffix(a.val, b.val, c.val)); } \ inline _Tpvec v_muladd(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ - { return _Tpvec(_mm512_fmadd_##suffix(a.val, b.val, c.val)); } \ + { return _Tpvec(_mm512_fmadd_##suffix(a.val, b.val, c.val)); } +#else +#define OPENCV_HAL_IMPL_AVX512_MULADD(_Tpvec, suffix) \ + inline _Tpvec v_fma(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ + { return _Tpvec(_mm512_add_##suffix(_mm512_mul_##suffix(a.val, b.val), c.val)); } \ + inline _Tpvec v_muladd(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ + { return _Tpvec(_mm512_add_##suffix(_mm512_mul_##suffix(a.val, b.val), c.val)); } +#endif + +#define OPENCV_HAL_IMPL_AVX512_MISC(_Tpvec, suffix) \ inline _Tpvec v_sqrt(const _Tpvec& x) \ { return _Tpvec(_mm512_sqrt_##suffix(x.val)); } \ inline _Tpvec v_sqr_magnitude(const _Tpvec& a, const _Tpvec& b) \ @@ -1399,6 +1409,8 @@ inline v_uint64x8 v_popcount(const v_uint64x8& a) { return v_popcount(v_reinte OPENCV_HAL_IMPL_AVX512_MULADD(v_float32x16, ps) OPENCV_HAL_IMPL_AVX512_MULADD(v_float64x8, pd) +OPENCV_HAL_IMPL_AVX512_MISC(v_float32x16, ps) +OPENCV_HAL_IMPL_AVX512_MISC(v_float64x8, pd) inline v_int32x16 v_fma(const v_int32x16& a, const v_int32x16& b, const v_int32x16& c) { return a * b + c; } From 968d94d4174c75dd8debfa62eff050dc0b9b42a8 Mon Sep 17 00:00:00 2001 From: Nikolaos Pappas Date: Wed, 3 Nov 2021 22:13:30 +0000 Subject: [PATCH 355/376] Fix trackbar in falsecolor cpp sample --- samples/cpp/falsecolor.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/samples/cpp/falsecolor.cpp b/samples/cpp/falsecolor.cpp index f73ffad4cee6..bfe43a72ca8a 100644 --- a/samples/cpp/falsecolor.cpp +++ b/samples/cpp/falsecolor.cpp @@ -16,14 +16,14 @@ struct ParamColorMap { String winName="False color"; static const String ColorMaps[] = { "Autumn", "Bone", "Jet", "Winter", "Rainbow", "Ocean", "Summer", "Spring", "Cool", "HSV", "Pink", "Hot", "Parula", "Magma", "Inferno", "Plasma", "Viridis", - "Cividis", "Twilight", "Twilight Shifted", "Turbo", "User defined (random)" }; + "Cividis", "Twilight", "Twilight Shifted", "Turbo", "Deep Green", "User defined (random)" }; static void TrackColorMap(int x, void *r) { ParamColorMap *p = (ParamColorMap*)r; Mat dst; p->iColormap= x; - if (x == COLORMAP_TURBO + 1) + if (x == COLORMAP_DEEPGREEN + 1) { Mat lutRND(256, 1, CV_8UC3); randu(lutRND, Scalar(0, 0, 0), Scalar(255, 255, 255)); @@ -97,10 +97,10 @@ int main(int argc, char** argv) imshow("Gray image",img); namedWindow(winName); - createTrackbar("colormap", winName,&p.iColormap,1,TrackColorMap,(void*)&p); + createTrackbar("colormap", winName, NULL, COLORMAP_DEEPGREEN + 1, TrackColorMap, (void*)&p); setTrackbarMin("colormap", winName, COLORMAP_AUTUMN); - setTrackbarMax("colormap", winName, COLORMAP_TURBO+1); - setTrackbarPos("colormap", winName, -1); + setTrackbarMax("colormap", winName, COLORMAP_DEEPGREEN + 1); + setTrackbarPos("colormap", winName, COLORMAP_AUTUMN); TrackColorMap(0, (void*)&p); From a2716712ab9efcefe7dc0f3c83f5f3d5f1cd402d Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 6 Nov 2021 18:16:42 +0300 Subject: [PATCH 356/376] highgui(win32): fix trackbar setRange --- modules/highgui/src/window_w32.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/highgui/src/window_w32.cpp b/modules/highgui/src/window_w32.cpp index 43a19baf8933..76f320f19fe5 100644 --- a/modules/highgui/src/window_w32.cpp +++ b/modules/highgui/src/window_w32.cpp @@ -2984,7 +2984,7 @@ class Win32UITrackbar : public UITrackbar CvTrackbar& trackbar = *trackbar_ptr; CV_CheckLE(range.start, range.end, "Invalid trackbar range"); trackbar.minval = range.start; - trackbar.maxval = range.start; + trackbar.maxval = range.end; SendMessage(trackbar.hwnd, TBM_SETRANGEMIN, (WPARAM)TRUE, (LPARAM)trackbar.minval); SendMessage(trackbar.hwnd, TBM_SETRANGEMAX, (WPARAM)TRUE, (LPARAM)trackbar.maxval); } From fa5c7a9e75ed5b87dbcce17a2a5164acec65f5c7 Mon Sep 17 00:00:00 2001 From: Lukas-Alexander Weber <32765578+lukasalexanderweber@users.noreply.github.com> Date: Mon, 8 Nov 2021 12:54:06 +0100 Subject: [PATCH 357/376] Merge pull request #21020 from lukasalexanderweber:squash Created Stitching Tool based on stitching_detailed.py --- apps/opencv_stitching_tool/README.md | 3 + .../opencv_stitching/.gitignore | 4 + .../opencv_stitching/__init__.py | 0 .../opencv_stitching/blender.py | 48 +++ .../opencv_stitching/camera_adjuster.py | 49 +++ .../opencv_stitching/camera_estimator.py | 27 ++ .../opencv_stitching/camera_wave_corrector.py | 28 ++ .../exposure_error_compensator.py | 40 ++ .../opencv_stitching/feature_detector.py | 44 ++ .../opencv_stitching/feature_matcher.py | 98 +++++ .../opencv_stitching/image_handler.py | 94 ++++ .../opencv_stitching/megapix_downscaler.py | 12 + .../opencv_stitching/megapix_scaler.py | 27 ++ .../opencv_stitching/panorama_estimation.py | 27 ++ .../opencv_stitching/seam_finder.py | 127 ++++++ .../opencv_stitching/stitcher.py | 207 +++++++++ .../opencv_stitching/stitching_error.py | 2 + .../opencv_stitching/subsetter.py | 95 ++++ .../opencv_stitching/test/.gitignore | 13 + .../test/SAMPLE_IMAGES_TO_DOWNLOAD.txt | 5 + .../test/stitching_detailed.py | 406 ++++++++++++++++++ .../opencv_stitching/test/test_composition.py | 67 +++ .../opencv_stitching/test/test_matcher.py | 47 ++ .../test/test_megapix_scaler.py | 59 +++ .../opencv_stitching/test/test_performance.py | 65 +++ .../test/test_registration.py | 100 +++++ .../opencv_stitching/test/test_stitcher.py | 108 +++++ .../opencv_stitching/timelapser.py | 50 +++ .../opencv_stitching/warper.py | 71 +++ .../opencv_stitching_tool.py | 232 ++++++++++ 30 files changed, 2155 insertions(+) create mode 100644 apps/opencv_stitching_tool/README.md create mode 100644 apps/opencv_stitching_tool/opencv_stitching/.gitignore create mode 100644 apps/opencv_stitching_tool/opencv_stitching/__init__.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/blender.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/camera_adjuster.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/camera_estimator.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/camera_wave_corrector.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/exposure_error_compensator.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/feature_detector.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/feature_matcher.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/image_handler.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/megapix_downscaler.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/megapix_scaler.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/panorama_estimation.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/seam_finder.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/stitcher.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/stitching_error.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/subsetter.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/.gitignore create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/SAMPLE_IMAGES_TO_DOWNLOAD.txt create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/stitching_detailed.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/test_composition.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/test_matcher.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/test_megapix_scaler.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/test_performance.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/test_registration.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/test/test_stitcher.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/timelapser.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching/warper.py create mode 100644 apps/opencv_stitching_tool/opencv_stitching_tool.py diff --git a/apps/opencv_stitching_tool/README.md b/apps/opencv_stitching_tool/README.md new file mode 100644 index 000000000000..1cf3f019d030 --- /dev/null +++ b/apps/opencv_stitching_tool/README.md @@ -0,0 +1,3 @@ +## In-Depth Stitching Tool for experiments and research + +Visit [opencv_stitching_tutorial](https://github.com/lukasalexanderweber/opencv_stitching_tutorial) for a detailed Tutorial diff --git a/apps/opencv_stitching_tool/opencv_stitching/.gitignore b/apps/opencv_stitching_tool/opencv_stitching/.gitignore new file mode 100644 index 000000000000..1f4d07f716a5 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/.gitignore @@ -0,0 +1,4 @@ +# python binary files +*.pyc +__pycache__ +.pylint* diff --git a/apps/opencv_stitching_tool/opencv_stitching/__init__.py b/apps/opencv_stitching_tool/opencv_stitching/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/apps/opencv_stitching_tool/opencv_stitching/blender.py b/apps/opencv_stitching_tool/opencv_stitching/blender.py new file mode 100644 index 000000000000..04e6efe528a6 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/blender.py @@ -0,0 +1,48 @@ +import cv2 as cv +import numpy as np + + +class Blender: + + BLENDER_CHOICES = ('multiband', 'feather', 'no',) + DEFAULT_BLENDER = 'multiband' + DEFAULT_BLEND_STRENGTH = 5 + + def __init__(self, blender_type=DEFAULT_BLENDER, + blend_strength=DEFAULT_BLEND_STRENGTH): + self.blender_type = blender_type + self.blend_strength = blend_strength + self.blender = None + + def prepare(self, corners, sizes): + dst_sz = cv.detail.resultRoi(corners=corners, sizes=sizes) + blend_width = (np.sqrt(dst_sz[2] * dst_sz[3]) * + self.blend_strength / 100) + + if self.blender_type == 'no' or blend_width < 1: + self.blender = cv.detail.Blender_createDefault( + cv.detail.Blender_NO + ) + + elif self.blender_type == "multiband": + self.blender = cv.detail_MultiBandBlender() + self.blender.setNumBands((np.log(blend_width) / + np.log(2.) - 1.).astype(np.int)) + + elif self.blender_type == "feather": + self.blender = cv.detail_FeatherBlender() + self.blender.setSharpness(1. / blend_width) + + self.blender.prepare(dst_sz) + + def feed(self, img, mask, corner): + """https://docs.opencv.org/master/d6/d4a/classcv_1_1detail_1_1Blender.html#a64837308bcf4e414a6219beff6cbe37a""" # noqa + self.blender.feed(cv.UMat(img.astype(np.int16)), mask, corner) + + def blend(self): + """https://docs.opencv.org/master/d6/d4a/classcv_1_1detail_1_1Blender.html#aa0a91ce0d6046d3a63e0123cbb1b5c00""" # noqa + result = None + result_mask = None + result, result_mask = self.blender.blend(result, result_mask) + result = cv.convertScaleAbs(result) + return result diff --git a/apps/opencv_stitching_tool/opencv_stitching/camera_adjuster.py b/apps/opencv_stitching_tool/opencv_stitching/camera_adjuster.py new file mode 100644 index 000000000000..03aa834d3177 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/camera_adjuster.py @@ -0,0 +1,49 @@ +from collections import OrderedDict +import cv2 as cv +import numpy as np + +from .stitching_error import StitchingError + + +class CameraAdjuster: + """https://docs.opencv.org/master/d5/d56/classcv_1_1detail_1_1BundleAdjusterBase.html""" # noqa + + CAMERA_ADJUSTER_CHOICES = OrderedDict() + CAMERA_ADJUSTER_CHOICES['ray'] = cv.detail_BundleAdjusterRay + CAMERA_ADJUSTER_CHOICES['reproj'] = cv.detail_BundleAdjusterReproj + CAMERA_ADJUSTER_CHOICES['affine'] = cv.detail_BundleAdjusterAffinePartial + CAMERA_ADJUSTER_CHOICES['no'] = cv.detail_NoBundleAdjuster + + DEFAULT_CAMERA_ADJUSTER = list(CAMERA_ADJUSTER_CHOICES.keys())[0] + DEFAULT_REFINEMENT_MASK = "xxxxx" + + def __init__(self, + adjuster=DEFAULT_CAMERA_ADJUSTER, + refinement_mask=DEFAULT_REFINEMENT_MASK): + + self.adjuster = CameraAdjuster.CAMERA_ADJUSTER_CHOICES[adjuster]() + self.set_refinement_mask(refinement_mask) + self.adjuster.setConfThresh(1) + + def set_refinement_mask(self, refinement_mask): + mask_matrix = np.zeros((3, 3), np.uint8) + if refinement_mask[0] == 'x': + mask_matrix[0, 0] = 1 + if refinement_mask[1] == 'x': + mask_matrix[0, 1] = 1 + if refinement_mask[2] == 'x': + mask_matrix[0, 2] = 1 + if refinement_mask[3] == 'x': + mask_matrix[1, 1] = 1 + if refinement_mask[4] == 'x': + mask_matrix[1, 2] = 1 + self.adjuster.setRefinementMask(mask_matrix) + + def adjust(self, features, pairwise_matches, estimated_cameras): + b, cameras = self.adjuster.apply(features, + pairwise_matches, + estimated_cameras) + if not b: + raise StitchingError("Camera parameters adjusting failed.") + + return cameras diff --git a/apps/opencv_stitching_tool/opencv_stitching/camera_estimator.py b/apps/opencv_stitching_tool/opencv_stitching/camera_estimator.py new file mode 100644 index 000000000000..8520eb0ddf2c --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/camera_estimator.py @@ -0,0 +1,27 @@ +from collections import OrderedDict +import cv2 as cv +import numpy as np + +from .stitching_error import StitchingError + + +class CameraEstimator: + + CAMERA_ESTIMATOR_CHOICES = OrderedDict() + CAMERA_ESTIMATOR_CHOICES['homography'] = cv.detail_HomographyBasedEstimator + CAMERA_ESTIMATOR_CHOICES['affine'] = cv.detail_AffineBasedEstimator + + DEFAULT_CAMERA_ESTIMATOR = list(CAMERA_ESTIMATOR_CHOICES.keys())[0] + + def __init__(self, estimator=DEFAULT_CAMERA_ESTIMATOR, **kwargs): + self.estimator = CameraEstimator.CAMERA_ESTIMATOR_CHOICES[estimator]( + **kwargs + ) + + def estimate(self, features, pairwise_matches): + b, cameras = self.estimator.apply(features, pairwise_matches, None) + if not b: + raise StitchingError("Homography estimation failed.") + for cam in cameras: + cam.R = cam.R.astype(np.float32) + return cameras diff --git a/apps/opencv_stitching_tool/opencv_stitching/camera_wave_corrector.py b/apps/opencv_stitching_tool/opencv_stitching/camera_wave_corrector.py new file mode 100644 index 000000000000..6a9142d7f543 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/camera_wave_corrector.py @@ -0,0 +1,28 @@ +from collections import OrderedDict +import cv2 as cv +import numpy as np + + +class WaveCorrector: + """https://docs.opencv.org/master/d7/d74/group__stitching__rotation.html#ga83b24d4c3e93584986a56d9e43b9cf7f""" # noqa + WAVE_CORRECT_CHOICES = OrderedDict() + WAVE_CORRECT_CHOICES['horiz'] = cv.detail.WAVE_CORRECT_HORIZ + WAVE_CORRECT_CHOICES['vert'] = cv.detail.WAVE_CORRECT_VERT + WAVE_CORRECT_CHOICES['auto'] = cv.detail.WAVE_CORRECT_AUTO + WAVE_CORRECT_CHOICES['no'] = None + + DEFAULT_WAVE_CORRECTION = list(WAVE_CORRECT_CHOICES.keys())[0] + + def __init__(self, wave_correct_kind=DEFAULT_WAVE_CORRECTION): + self.wave_correct_kind = WaveCorrector.WAVE_CORRECT_CHOICES[ + wave_correct_kind + ] + + def correct(self, cameras): + if self.wave_correct_kind is not None: + rmats = [np.copy(cam.R) for cam in cameras] + rmats = cv.detail.waveCorrect(rmats, self.wave_correct_kind) + for idx, cam in enumerate(cameras): + cam.R = rmats[idx] + return cameras + return cameras diff --git a/apps/opencv_stitching_tool/opencv_stitching/exposure_error_compensator.py b/apps/opencv_stitching_tool/opencv_stitching/exposure_error_compensator.py new file mode 100644 index 000000000000..36e02920914d --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/exposure_error_compensator.py @@ -0,0 +1,40 @@ +from collections import OrderedDict +import cv2 as cv + + +class ExposureErrorCompensator: + + COMPENSATOR_CHOICES = OrderedDict() + COMPENSATOR_CHOICES['gain_blocks'] = cv.detail.ExposureCompensator_GAIN_BLOCKS # noqa + COMPENSATOR_CHOICES['gain'] = cv.detail.ExposureCompensator_GAIN + COMPENSATOR_CHOICES['channel'] = cv.detail.ExposureCompensator_CHANNELS + COMPENSATOR_CHOICES['channel_blocks'] = cv.detail.ExposureCompensator_CHANNELS_BLOCKS # noqa + COMPENSATOR_CHOICES['no'] = cv.detail.ExposureCompensator_NO + + DEFAULT_COMPENSATOR = list(COMPENSATOR_CHOICES.keys())[0] + DEFAULT_NR_FEEDS = 1 + DEFAULT_BLOCK_SIZE = 32 + + def __init__(self, + compensator=DEFAULT_COMPENSATOR, + nr_feeds=DEFAULT_NR_FEEDS, + block_size=DEFAULT_BLOCK_SIZE): + + if compensator == 'channel': + self.compensator = cv.detail_ChannelsCompensator(nr_feeds) + elif compensator == 'channel_blocks': + self.compensator = cv.detail_BlocksChannelsCompensator( + block_size, block_size, nr_feeds + ) + else: + self.compensator = cv.detail.ExposureCompensator_createDefault( + ExposureErrorCompensator.COMPENSATOR_CHOICES[compensator] + ) + + def feed(self, *args): + """https://docs.opencv.org/master/d2/d37/classcv_1_1detail_1_1ExposureCompensator.html#ae6b0cc69a7bc53818ddea53eddb6bdba""" # noqa + self.compensator.feed(*args) + + def apply(self, *args): + """https://docs.opencv.org/master/d2/d37/classcv_1_1detail_1_1ExposureCompensator.html#a473eaf1e585804c08d77c91e004f93aa""" # noqa + return self.compensator.apply(*args) diff --git a/apps/opencv_stitching_tool/opencv_stitching/feature_detector.py b/apps/opencv_stitching_tool/opencv_stitching/feature_detector.py new file mode 100644 index 000000000000..995517b01bf5 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/feature_detector.py @@ -0,0 +1,44 @@ +from collections import OrderedDict +import cv2 as cv + + +class FeatureDetector: + DETECTOR_CHOICES = OrderedDict() + try: + cv.xfeatures2d_SURF.create() # check if the function can be called + DETECTOR_CHOICES['surf'] = cv.xfeatures2d_SURF.create + except (AttributeError, cv.error): + print("SURF not available") + + # if SURF not available, ORB is default + DETECTOR_CHOICES['orb'] = cv.ORB.create + + try: + DETECTOR_CHOICES['sift'] = cv.SIFT_create + except AttributeError: + print("SIFT not available") + + try: + DETECTOR_CHOICES['brisk'] = cv.BRISK_create + except AttributeError: + print("BRISK not available") + + try: + DETECTOR_CHOICES['akaze'] = cv.AKAZE_create + except AttributeError: + print("AKAZE not available") + + DEFAULT_DETECTOR = list(DETECTOR_CHOICES.keys())[0] + + def __init__(self, detector=DEFAULT_DETECTOR, **kwargs): + self.detector = FeatureDetector.DETECTOR_CHOICES[detector](**kwargs) + + def detect_features(self, img, *args, **kwargs): + return cv.detail.computeImageFeatures2(self.detector, img, + *args, **kwargs) + + @staticmethod + def draw_keypoints(img, features, **kwargs): + kwargs.setdefault('color', (0, 255, 0)) + keypoints = features.getKeypoints() + return cv.drawKeypoints(img, keypoints, None, **kwargs) diff --git a/apps/opencv_stitching_tool/opencv_stitching/feature_matcher.py b/apps/opencv_stitching_tool/opencv_stitching/feature_matcher.py new file mode 100644 index 000000000000..8c1d384b7bae --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/feature_matcher.py @@ -0,0 +1,98 @@ +import math +import cv2 as cv +import numpy as np + + +class FeatureMatcher: + + MATCHER_CHOICES = ('homography', 'affine') + DEFAULT_MATCHER = 'homography' + DEFAULT_RANGE_WIDTH = -1 + + def __init__(self, + matcher_type=DEFAULT_MATCHER, + range_width=DEFAULT_RANGE_WIDTH, + **kwargs): + + if matcher_type == "affine": + """https://docs.opencv.org/master/d3/dda/classcv_1_1detail_1_1AffineBestOf2NearestMatcher.html""" # noqa + self.matcher = cv.detail_AffineBestOf2NearestMatcher(**kwargs) + elif range_width == -1: + """https://docs.opencv.org/master/d4/d26/classcv_1_1detail_1_1BestOf2NearestMatcher.html""" # noqa + self.matcher = cv.detail.BestOf2NearestMatcher_create(**kwargs) + else: + """https://docs.opencv.org/master/d8/d72/classcv_1_1detail_1_1BestOf2NearestRangeMatcher.html""" # noqa + self.matcher = cv.detail.BestOf2NearestRangeMatcher_create( + range_width, **kwargs + ) + + def match_features(self, features, *args, **kwargs): + pairwise_matches = self.matcher.apply2(features, *args, **kwargs) + self.matcher.collectGarbage() + return pairwise_matches + + @staticmethod + def draw_matches_matrix(imgs, features, matches, conf_thresh=1, + inliers=False, **kwargs): + matches_matrix = FeatureMatcher.get_matches_matrix(matches) + for idx1, idx2 in FeatureMatcher.get_all_img_combinations(len(imgs)): + match = matches_matrix[idx1, idx2] + if match.confidence < conf_thresh: + continue + if inliers: + kwargs['matchesMask'] = match.getInliers() + yield idx1, idx2, FeatureMatcher.draw_matches( + imgs[idx1], features[idx1], + imgs[idx2], features[idx2], + match, + **kwargs + ) + + @staticmethod + def draw_matches(img1, features1, img2, features2, match1to2, **kwargs): + kwargs.setdefault('flags', cv.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS) + + keypoints1 = features1.getKeypoints() + keypoints2 = features2.getKeypoints() + matches = match1to2.getMatches() + + return cv.drawMatches( + img1, keypoints1, img2, keypoints2, matches, None, **kwargs + ) + + @staticmethod + def get_matches_matrix(pairwise_matches): + return FeatureMatcher.array_in_sqare_matrix(pairwise_matches) + + @staticmethod + def get_confidence_matrix(pairwise_matches): + matches_matrix = FeatureMatcher.get_matches_matrix(pairwise_matches) + match_confs = [[m.confidence for m in row] for row in matches_matrix] + match_conf_matrix = np.array(match_confs) + return match_conf_matrix + + @staticmethod + def array_in_sqare_matrix(array): + matrix_dimension = int(math.sqrt(len(array))) + rows = [] + for i in range(0, len(array), matrix_dimension): + rows.append(array[i:i+matrix_dimension]) + return np.array(rows) + + def get_all_img_combinations(number_imgs): + ii, jj = np.triu_indices(number_imgs, k=1) + for i, j in zip(ii, jj): + yield i, j + + @staticmethod + def get_match_conf(match_conf, feature_detector_type): + if match_conf is None: + match_conf = \ + FeatureMatcher.get_default_match_conf(feature_detector_type) + return match_conf + + @staticmethod + def get_default_match_conf(feature_detector_type): + if feature_detector_type == 'orb': + return 0.3 + return 0.65 diff --git a/apps/opencv_stitching_tool/opencv_stitching/image_handler.py b/apps/opencv_stitching_tool/opencv_stitching/image_handler.py new file mode 100644 index 000000000000..a3b76b288ab5 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/image_handler.py @@ -0,0 +1,94 @@ +import cv2 as cv + +from .megapix_downscaler import MegapixDownscaler +from .stitching_error import StitchingError + +class ImageHandler: + + DEFAULT_MEDIUM_MEGAPIX = 0.6 + DEFAULT_LOW_MEGAPIX = 0.1 + DEFAULT_FINAL_MEGAPIX = -1 + + def __init__(self, + medium_megapix=DEFAULT_MEDIUM_MEGAPIX, + low_megapix=DEFAULT_LOW_MEGAPIX, + final_megapix=DEFAULT_FINAL_MEGAPIX): + + if medium_megapix < low_megapix: + raise StitchingError("Medium resolution megapix need to be " + "greater or equal than low resolution " + "megapix") + + self.medium_scaler = MegapixDownscaler(medium_megapix) + self.low_scaler = MegapixDownscaler(low_megapix) + self.final_scaler = MegapixDownscaler(final_megapix) + + self.scales_set = False + self.img_names = [] + self.img_sizes = [] + + def set_img_names(self, img_names): + self.img_names = img_names + + def resize_to_medium_resolution(self): + return self.read_and_resize_imgs(self.medium_scaler) + + def resize_to_low_resolution(self, medium_imgs=None): + if medium_imgs and self.scales_set: + return self.resize_medium_to_low(medium_imgs) + return self.read_and_resize_imgs(self.low_scaler) + + def resize_to_final_resolution(self): + return self.read_and_resize_imgs(self.final_scaler) + + def read_and_resize_imgs(self, scaler): + for img, size in self.input_images(): + yield self.resize_img_by_scaler(scaler, size, img) + + def resize_medium_to_low(self, medium_imgs): + for img, size in zip(medium_imgs, self.img_sizes): + yield self.resize_img_by_scaler(self.low_scaler, size, img) + + @staticmethod + def resize_img_by_scaler(scaler, size, img): + desired_size = scaler.get_scaled_img_size(size) + return cv.resize(img, desired_size, + interpolation=cv.INTER_LINEAR_EXACT) + + def input_images(self): + self.img_sizes = [] + for name in self.img_names: + img = self.read_image(name) + size = self.get_image_size(img) + self.img_sizes.append(size) + self.set_scaler_scales() + yield img, size + + @staticmethod + def get_image_size(img): + """(width, height)""" + return (img.shape[1], img.shape[0]) + + @staticmethod + def read_image(img_name): + img = cv.imread(img_name) + if img is None: + raise StitchingError("Cannot read image " + img_name) + return img + + def set_scaler_scales(self): + if not self.scales_set: + first_img_size = self.img_sizes[0] + self.medium_scaler.set_scale_by_img_size(first_img_size) + self.low_scaler.set_scale_by_img_size(first_img_size) + self.final_scaler.set_scale_by_img_size(first_img_size) + self.scales_set = True + + def get_medium_to_final_ratio(self): + return self.final_scaler.scale / self.medium_scaler.scale + + def get_medium_to_low_ratio(self): + return self.low_scaler.scale / self.medium_scaler.scale + + def get_final_to_low_ratio(self): + return self.low_scaler.scale / self.final_scaler.scale diff --git a/apps/opencv_stitching_tool/opencv_stitching/megapix_downscaler.py b/apps/opencv_stitching_tool/opencv_stitching/megapix_downscaler.py new file mode 100644 index 000000000000..f7553acc2eea --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/megapix_downscaler.py @@ -0,0 +1,12 @@ +from .megapix_scaler import MegapixScaler + + +class MegapixDownscaler(MegapixScaler): + + @staticmethod + def force_downscale(scale): + return min(1.0, scale) + + def set_scale(self, scale): + scale = self.force_downscale(scale) + super().set_scale(scale) diff --git a/apps/opencv_stitching_tool/opencv_stitching/megapix_scaler.py b/apps/opencv_stitching_tool/opencv_stitching/megapix_scaler.py new file mode 100644 index 000000000000..96d47536f951 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/megapix_scaler.py @@ -0,0 +1,27 @@ +import numpy as np + + +class MegapixScaler: + def __init__(self, megapix): + self.megapix = megapix + self.is_scale_set = False + self.scale = None + + def set_scale_by_img_size(self, img_size): + self.set_scale( + self.get_scale_by_resolution(img_size[0] * img_size[1]) + ) + + def set_scale(self, scale): + self.scale = scale + self.is_scale_set = True + + def get_scale_by_resolution(self, resolution): + if self.megapix > 0: + return np.sqrt(self.megapix * 1e6 / resolution) + return 1.0 + + def get_scaled_img_size(self, img_size): + width = int(round(img_size[0] * self.scale)) + height = int(round(img_size[1] * self.scale)) + return (width, height) diff --git a/apps/opencv_stitching_tool/opencv_stitching/panorama_estimation.py b/apps/opencv_stitching_tool/opencv_stitching/panorama_estimation.py new file mode 100644 index 000000000000..e3a45773ea2c --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/panorama_estimation.py @@ -0,0 +1,27 @@ +import statistics + + +def estimate_final_panorama_dimensions(cameras, warper, img_handler): + medium_to_final_ratio = img_handler.get_medium_to_final_ratio() + + panorama_scale_determined_on_medium_img = \ + estimate_panorama_scale(cameras) + + panorama_scale = (panorama_scale_determined_on_medium_img * + medium_to_final_ratio) + panorama_corners = [] + panorama_sizes = [] + + for size, camera in zip(img_handler.img_sizes, cameras): + width, height = img_handler.final_scaler.get_scaled_img_size(size) + roi = warper.warp_roi(width, height, camera, panorama_scale, medium_to_final_ratio) + panorama_corners.append(roi[0:2]) + panorama_sizes.append(roi[2:4]) + + return panorama_scale, panorama_corners, panorama_sizes + + +def estimate_panorama_scale(cameras): + focals = [cam.focal for cam in cameras] + panorama_scale = statistics.median(focals) + return panorama_scale diff --git a/apps/opencv_stitching_tool/opencv_stitching/seam_finder.py b/apps/opencv_stitching_tool/opencv_stitching/seam_finder.py new file mode 100644 index 000000000000..675f266d0230 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/seam_finder.py @@ -0,0 +1,127 @@ +from collections import OrderedDict +import cv2 as cv +import numpy as np + +from .blender import Blender + + +class SeamFinder: + """https://docs.opencv.org/master/d7/d09/classcv_1_1detail_1_1SeamFinder.html""" # noqa + SEAM_FINDER_CHOICES = OrderedDict() + SEAM_FINDER_CHOICES['dp_color'] = cv.detail_DpSeamFinder('COLOR') + SEAM_FINDER_CHOICES['dp_colorgrad'] = cv.detail_DpSeamFinder('COLOR_GRAD') + SEAM_FINDER_CHOICES['voronoi'] = cv.detail.SeamFinder_createDefault(cv.detail.SeamFinder_VORONOI_SEAM) # noqa + SEAM_FINDER_CHOICES['no'] = cv.detail.SeamFinder_createDefault(cv.detail.SeamFinder_NO) # noqa + + DEFAULT_SEAM_FINDER = list(SEAM_FINDER_CHOICES.keys())[0] + + def __init__(self, finder=DEFAULT_SEAM_FINDER): + self.finder = SeamFinder.SEAM_FINDER_CHOICES[finder] + + def find(self, imgs, corners, masks): + """https://docs.opencv.org/master/d0/dd5/classcv_1_1detail_1_1DpSeamFinder.html#a7914624907986f7a94dd424209a8a609""" # noqa + imgs_float = [img.astype(np.float32) for img in imgs] + return self.finder.find(imgs_float, corners, masks) + + @staticmethod + def resize(seam_mask, mask): + dilated_mask = cv.dilate(seam_mask, None) + resized_seam_mask = cv.resize(dilated_mask, (mask.shape[1], + mask.shape[0]), + 0, 0, cv.INTER_LINEAR_EXACT) + return cv.bitwise_and(resized_seam_mask, mask) + + @staticmethod + def draw_seam_mask(img, seam_mask, color=(0, 0, 0)): + seam_mask = cv.UMat.get(seam_mask) + overlayed_img = np.copy(img) + overlayed_img[seam_mask == 0] = color + return overlayed_img + + @staticmethod + def draw_seam_polygons(panorama, blended_seam_masks, alpha=0.5): + return add_weighted_image(panorama, blended_seam_masks, alpha) + + @staticmethod + def draw_seam_lines(panorama, blended_seam_masks, + linesize=1, color=(0, 0, 255)): + seam_lines = \ + SeamFinder.exctract_seam_lines(blended_seam_masks, linesize) + panorama_with_seam_lines = panorama.copy() + panorama_with_seam_lines[seam_lines == 255] = color + return panorama_with_seam_lines + + @staticmethod + def exctract_seam_lines(blended_seam_masks, linesize=1): + seam_lines = cv.Canny(np.uint8(blended_seam_masks), 100, 200) + seam_indices = (seam_lines == 255).nonzero() + seam_lines = remove_invalid_line_pixels( + seam_indices, seam_lines, blended_seam_masks + ) + kernelsize = linesize + linesize - 1 + kernel = np.ones((kernelsize, kernelsize), np.uint8) + return cv.dilate(seam_lines, kernel) + + @staticmethod + def blend_seam_masks(seam_masks, corners, sizes, colors=[ + (255, 000, 000), # Blue + (000, 000, 255), # Red + (000, 255, 000), # Green + (000, 255, 255), # Yellow + (255, 000, 255), # Magenta + (128, 128, 255), # Pink + (128, 128, 128), # Gray + (000, 000, 128), # Brown + (000, 128, 255)] # Orange + ): + + blender = Blender("no") + blender.prepare(corners, sizes) + + for idx, (seam_mask, size, corner) in enumerate( + zip(seam_masks, sizes, corners)): + if idx+1 > len(colors): + raise ValueError("Not enough default colors! Pass additional " + "colors to \"colors\" parameter") + one_color_img = create_img_by_size(size, colors[idx]) + blender.feed(one_color_img, seam_mask, corner) + + return blender.blend() + + +def create_img_by_size(size, color=(0, 0, 0)): + width, height = size + img = np.zeros((height, width, 3), np.uint8) + img[:] = color + return img + + +def add_weighted_image(img1, img2, alpha): + return cv.addWeighted( + img1, alpha, img2, (1.0 - alpha), 0.0 + ) + + +def remove_invalid_line_pixels(indices, lines, mask): + for x, y in zip(*indices): + if check_if_pixel_or_neighbor_is_black(mask, x, y): + lines[x, y] = 0 + return lines + + +def check_if_pixel_or_neighbor_is_black(img, x, y): + check = [is_pixel_black(img, x, y), + is_pixel_black(img, x+1, y), is_pixel_black(img, x-1, y), + is_pixel_black(img, x, y+1), is_pixel_black(img, x, y-1)] + return any(check) + + +def is_pixel_black(img, x, y): + return np.all(get_pixel_value(img, x, y) == 0) + + +def get_pixel_value(img, x, y): + try: + return img[x, y] + except IndexError: + pass diff --git a/apps/opencv_stitching_tool/opencv_stitching/stitcher.py b/apps/opencv_stitching_tool/opencv_stitching/stitcher.py new file mode 100644 index 000000000000..c08112664f02 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/stitcher.py @@ -0,0 +1,207 @@ +from types import SimpleNamespace + +from .image_handler import ImageHandler +from .feature_detector import FeatureDetector +from .feature_matcher import FeatureMatcher +from .subsetter import Subsetter +from .camera_estimator import CameraEstimator +from .camera_adjuster import CameraAdjuster +from .camera_wave_corrector import WaveCorrector +from .warper import Warper +from .panorama_estimation import estimate_final_panorama_dimensions +from .exposure_error_compensator import ExposureErrorCompensator +from .seam_finder import SeamFinder +from .blender import Blender +from .timelapser import Timelapser +from .stitching_error import StitchingError + + +class Stitcher: + DEFAULT_SETTINGS = { + "medium_megapix": ImageHandler.DEFAULT_MEDIUM_MEGAPIX, + "detector": FeatureDetector.DEFAULT_DETECTOR, + "nfeatures": 500, + "matcher_type": FeatureMatcher.DEFAULT_MATCHER, + "range_width": FeatureMatcher.DEFAULT_RANGE_WIDTH, + "try_use_gpu": False, + "match_conf": None, + "confidence_threshold": Subsetter.DEFAULT_CONFIDENCE_THRESHOLD, + "matches_graph_dot_file": Subsetter.DEFAULT_MATCHES_GRAPH_DOT_FILE, + "estimator": CameraEstimator.DEFAULT_CAMERA_ESTIMATOR, + "adjuster": CameraAdjuster.DEFAULT_CAMERA_ADJUSTER, + "refinement_mask": CameraAdjuster.DEFAULT_REFINEMENT_MASK, + "wave_correct_kind": WaveCorrector.DEFAULT_WAVE_CORRECTION, + "warper_type": Warper.DEFAULT_WARP_TYPE, + "low_megapix": ImageHandler.DEFAULT_LOW_MEGAPIX, + "compensator": ExposureErrorCompensator.DEFAULT_COMPENSATOR, + "nr_feeds": ExposureErrorCompensator.DEFAULT_NR_FEEDS, + "block_size": ExposureErrorCompensator.DEFAULT_BLOCK_SIZE, + "finder": SeamFinder.DEFAULT_SEAM_FINDER, + "final_megapix": ImageHandler.DEFAULT_FINAL_MEGAPIX, + "blender_type": Blender.DEFAULT_BLENDER, + "blend_strength": Blender.DEFAULT_BLEND_STRENGTH, + "timelapse": Timelapser.DEFAULT_TIMELAPSE} + + def __init__(self, **kwargs): + self.initialize_stitcher(**kwargs) + + def initialize_stitcher(self, **kwargs): + self.settings = Stitcher.DEFAULT_SETTINGS.copy() + self.validate_kwargs(kwargs) + self.settings.update(kwargs) + + args = SimpleNamespace(**self.settings) + self.img_handler = ImageHandler(args.medium_megapix, + args.low_megapix, + args.final_megapix) + self.detector = \ + FeatureDetector(args.detector, nfeatures=args.nfeatures) + match_conf = \ + FeatureMatcher.get_match_conf(args.match_conf, args.detector) + self.matcher = FeatureMatcher(args.matcher_type, args.range_width, + try_use_gpu=args.try_use_gpu, + match_conf=match_conf) + self.subsetter = \ + Subsetter(args.confidence_threshold, args.matches_graph_dot_file) + self.camera_estimator = CameraEstimator(args.estimator) + self.camera_adjuster = \ + CameraAdjuster(args.adjuster, args.refinement_mask) + self.wave_corrector = WaveCorrector(args.wave_correct_kind) + self.warper = Warper(args.warper_type) + self.compensator = \ + ExposureErrorCompensator(args.compensator, args.nr_feeds, + args.block_size) + self.seam_finder = SeamFinder(args.finder) + self.blender = Blender(args.blender_type, args.blend_strength) + self.timelapser = Timelapser(args.timelapse) + + def stitch(self, img_names): + self.initialize_registration(img_names) + + imgs = self.resize_medium_resolution() + features = self.find_features(imgs) + matches = self.match_features(features) + imgs, features, matches = self.subset(imgs, features, matches) + cameras = self.estimate_camera_parameters(features, matches) + cameras = self.refine_camera_parameters(features, matches, cameras) + cameras = self.perform_wave_correction(cameras) + panorama_scale, panorama_corners, panorama_sizes = \ + self.estimate_final_panorama_dimensions(cameras) + + self.initialize_composition(panorama_corners, panorama_sizes) + + imgs = self.resize_low_resolution(imgs) + imgs = self.warp_low_resolution_images(imgs, cameras, panorama_scale) + self.estimate_exposure_errors(imgs) + seam_masks = self.find_seam_masks(imgs) + + imgs = self.resize_final_resolution() + imgs = self.warp_final_resolution_images(imgs, cameras, panorama_scale) + imgs = self.compensate_exposure_errors(imgs) + seam_masks = self.resize_seam_masks(seam_masks) + self.blend_images(imgs, seam_masks) + + return self.create_final_panorama() + + def initialize_registration(self, img_names): + self.img_handler.set_img_names(img_names) + + def resize_medium_resolution(self): + return list(self.img_handler.resize_to_medium_resolution()) + + def find_features(self, imgs): + return [self.detector.detect_features(img) for img in imgs] + + def match_features(self, features): + return self.matcher.match_features(features) + + def subset(self, imgs, features, matches): + names, sizes, imgs, features, matches = \ + self.subsetter.subset(self.img_handler.img_names, + self.img_handler.img_sizes, + imgs, features, matches) + self.img_handler.img_names, self.img_handler.img_sizes = names, sizes + return imgs, features, matches + + def estimate_camera_parameters(self, features, matches): + return self.camera_estimator.estimate(features, matches) + + def refine_camera_parameters(self, features, matches, cameras): + return self.camera_adjuster.adjust(features, matches, cameras) + + def perform_wave_correction(self, cameras): + return self.wave_corrector.correct(cameras) + + def estimate_final_panorama_dimensions(self, cameras): + return estimate_final_panorama_dimensions(cameras, self.warper, + self.img_handler) + + def initialize_composition(self, corners, sizes): + if self.timelapser.do_timelapse: + self.timelapser.initialize(corners, sizes) + else: + self.blender.prepare(corners, sizes) + + def resize_low_resolution(self, imgs=None): + return list(self.img_handler.resize_to_low_resolution(imgs)) + + def warp_low_resolution_images(self, imgs, cameras, final_scale): + camera_aspect = self.img_handler.get_medium_to_low_ratio() + scale = final_scale * self.img_handler.get_final_to_low_ratio() + return list(self.warp_images(imgs, cameras, scale, camera_aspect)) + + def warp_final_resolution_images(self, imgs, cameras, scale): + camera_aspect = self.img_handler.get_medium_to_final_ratio() + return self.warp_images(imgs, cameras, scale, camera_aspect) + + def warp_images(self, imgs, cameras, scale, aspect=1): + self._masks = [] + self._corners = [] + for img_warped, mask_warped, corner in \ + self.warper.warp_images_and_image_masks( + imgs, cameras, scale, aspect + ): + self._masks.append(mask_warped) + self._corners.append(corner) + yield img_warped + + def estimate_exposure_errors(self, imgs): + self.compensator.feed(self._corners, imgs, self._masks) + + def find_seam_masks(self, imgs): + return self.seam_finder.find(imgs, self._corners, self._masks) + + def resize_final_resolution(self): + return self.img_handler.resize_to_final_resolution() + + def compensate_exposure_errors(self, imgs): + for idx, img in enumerate(imgs): + yield self.compensator.apply(idx, self._corners[idx], + img, self._masks[idx]) + + def resize_seam_masks(self, seam_masks): + for idx, seam_mask in enumerate(seam_masks): + yield SeamFinder.resize(seam_mask, self._masks[idx]) + + def blend_images(self, imgs, masks): + for idx, (img, mask) in enumerate(zip(imgs, masks)): + if self.timelapser.do_timelapse: + self.timelapser.process_and_save_frame( + self.img_handler.img_names[idx], img, self._corners[idx] + ) + else: + self.blender.feed(img, mask, self._corners[idx]) + + def create_final_panorama(self): + if not self.timelapser.do_timelapse: + return self.blender.blend() + + @staticmethod + def validate_kwargs(kwargs): + for arg in kwargs: + if arg not in Stitcher.DEFAULT_SETTINGS: + raise StitchingError("Invalid Argument: " + arg) + + def collect_garbage(self): + del self.img_handler.img_names, self.img_handler.img_sizes, + del self._corners, self._masks diff --git a/apps/opencv_stitching_tool/opencv_stitching/stitching_error.py b/apps/opencv_stitching_tool/opencv_stitching/stitching_error.py new file mode 100644 index 000000000000..6d42e95437b3 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/stitching_error.py @@ -0,0 +1,2 @@ +class StitchingError(Exception): + pass diff --git a/apps/opencv_stitching_tool/opencv_stitching/subsetter.py b/apps/opencv_stitching_tool/opencv_stitching/subsetter.py new file mode 100644 index 000000000000..4ea6acc60de2 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/subsetter.py @@ -0,0 +1,95 @@ +from itertools import chain +import math +import cv2 as cv +import numpy as np + +from .feature_matcher import FeatureMatcher +from .stitching_error import StitchingError + + +class Subsetter: + + DEFAULT_CONFIDENCE_THRESHOLD = 1 + DEFAULT_MATCHES_GRAPH_DOT_FILE = None + + def __init__(self, + confidence_threshold=DEFAULT_CONFIDENCE_THRESHOLD, + matches_graph_dot_file=DEFAULT_MATCHES_GRAPH_DOT_FILE): + self.confidence_threshold = confidence_threshold + self.save_file = matches_graph_dot_file + + def subset(self, img_names, img_sizes, imgs, features, matches): + self.save_matches_graph_dot_file(img_names, matches) + indices = self.get_indices_to_keep(features, matches) + + img_names = Subsetter.subset_list(img_names, indices) + img_sizes = Subsetter.subset_list(img_sizes, indices) + imgs = Subsetter.subset_list(imgs, indices) + features = Subsetter.subset_list(features, indices) + matches = Subsetter.subset_matches(matches, indices) + return img_names, img_sizes, imgs, features, matches + + def save_matches_graph_dot_file(self, img_names, pairwise_matches): + if self.save_file: + with open(self.save_file, 'w') as filehandler: + filehandler.write(self.get_matches_graph(img_names, + pairwise_matches) + ) + + def get_matches_graph(self, img_names, pairwise_matches): + return cv.detail.matchesGraphAsString(img_names, pairwise_matches, + self.confidence_threshold) + + def get_indices_to_keep(self, features, pairwise_matches): + indices = cv.detail.leaveBiggestComponent(features, + pairwise_matches, + self.confidence_threshold) + indices_as_list = [int(idx) for idx in list(indices[:, 0])] + + if len(indices_as_list) < 2: + raise StitchingError("No match exceeds the " + "given confidence theshold.") + + return indices_as_list + + @staticmethod + def subset_list(list_to_subset, indices): + return [list_to_subset[i] for i in indices] + + @staticmethod + def subset_matches(pairwise_matches, indices): + indices_to_delete = Subsetter.get_indices_to_delete( + math.sqrt(len(pairwise_matches)), + indices + ) + + matches_matrix = FeatureMatcher.get_matches_matrix(pairwise_matches) + matches_matrix_subset = Subsetter.subset_matrix(matches_matrix, + indices_to_delete) + matches_subset = Subsetter.matrix_rows_to_list(matches_matrix_subset) + + return matches_subset + + @staticmethod + def get_indices_to_delete(nr_elements, indices_to_keep): + return list(set(range(int(nr_elements))) - set(indices_to_keep)) + + @staticmethod + def subset_matrix(matrix_to_subset, indices_to_delete): + for idx, idx_to_delete in enumerate(indices_to_delete): + matrix_to_subset = Subsetter.delete_index_from_matrix( + matrix_to_subset, + idx_to_delete-idx # matrix shape reduced by one at each step + ) + + return matrix_to_subset + + @staticmethod + def delete_index_from_matrix(matrix, idx): + mask = np.ones(matrix.shape[0], bool) + mask[idx] = 0 + return matrix[mask, :][:, mask] + + @staticmethod + def matrix_rows_to_list(matrix): + return list(chain.from_iterable(matrix.tolist())) diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/.gitignore b/apps/opencv_stitching_tool/opencv_stitching/test/.gitignore new file mode 100644 index 000000000000..93426ff2b0f9 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/.gitignore @@ -0,0 +1,13 @@ +# Ignore everything +* + +# But not these files... +!.gitignore +!test_matcher.py +!test_stitcher.py +!test_megapix_scaler.py +!test_registration.py +!test_composition.py +!test_performance.py +!stitching_detailed.py +!SAMPLE_IMAGES_TO_DOWNLOAD.txt \ No newline at end of file diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/SAMPLE_IMAGES_TO_DOWNLOAD.txt b/apps/opencv_stitching_tool/opencv_stitching/test/SAMPLE_IMAGES_TO_DOWNLOAD.txt new file mode 100644 index 000000000000..cecf3b8ba76a --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/SAMPLE_IMAGES_TO_DOWNLOAD.txt @@ -0,0 +1,5 @@ +https://github.com/opencv/opencv_extra/tree/master/testdata/stitching + +s1.jpg s2.jpg +boat1.jpg boat2.jpg boat3.jpg boat4.jpg boat5.jpg boat6.jpg +budapest1.jpg budapest2.jpg budapest3.jpg budapest4.jpg budapest5.jpg budapest6.jpg \ No newline at end of file diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/stitching_detailed.py b/apps/opencv_stitching_tool/opencv_stitching/test/stitching_detailed.py new file mode 100644 index 000000000000..b12210de669c --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/stitching_detailed.py @@ -0,0 +1,406 @@ +""" +Stitching sample (advanced) +=========================== +Show how to use Stitcher API from python. +""" + +# Python 2/3 compatibility +from __future__ import print_function + +from types import SimpleNamespace +from collections import OrderedDict + +import cv2 as cv +import numpy as np + +EXPOS_COMP_CHOICES = OrderedDict() +EXPOS_COMP_CHOICES['gain_blocks'] = cv.detail.ExposureCompensator_GAIN_BLOCKS +EXPOS_COMP_CHOICES['gain'] = cv.detail.ExposureCompensator_GAIN +EXPOS_COMP_CHOICES['channel'] = cv.detail.ExposureCompensator_CHANNELS +EXPOS_COMP_CHOICES['channel_blocks'] = cv.detail.ExposureCompensator_CHANNELS_BLOCKS +EXPOS_COMP_CHOICES['no'] = cv.detail.ExposureCompensator_NO + +BA_COST_CHOICES = OrderedDict() +BA_COST_CHOICES['ray'] = cv.detail_BundleAdjusterRay +BA_COST_CHOICES['reproj'] = cv.detail_BundleAdjusterReproj +BA_COST_CHOICES['affine'] = cv.detail_BundleAdjusterAffinePartial +BA_COST_CHOICES['no'] = cv.detail_NoBundleAdjuster + +FEATURES_FIND_CHOICES = OrderedDict() +try: + cv.xfeatures2d_SURF.create() # check if the function can be called + FEATURES_FIND_CHOICES['surf'] = cv.xfeatures2d_SURF.create +except (AttributeError, cv.error) as e: + print("SURF not available") +# if SURF not available, ORB is default +FEATURES_FIND_CHOICES['orb'] = cv.ORB.create +try: + FEATURES_FIND_CHOICES['sift'] = cv.xfeatures2d_SIFT.create +except AttributeError: + print("SIFT not available") +try: + FEATURES_FIND_CHOICES['brisk'] = cv.BRISK_create +except AttributeError: + print("BRISK not available") +try: + FEATURES_FIND_CHOICES['akaze'] = cv.AKAZE_create +except AttributeError: + print("AKAZE not available") + +SEAM_FIND_CHOICES = OrderedDict() +SEAM_FIND_CHOICES['dp_color'] = cv.detail_DpSeamFinder('COLOR') +SEAM_FIND_CHOICES['dp_colorgrad'] = cv.detail_DpSeamFinder('COLOR_GRAD') +SEAM_FIND_CHOICES['voronoi'] = cv.detail.SeamFinder_createDefault(cv.detail.SeamFinder_VORONOI_SEAM) +SEAM_FIND_CHOICES['no'] = cv.detail.SeamFinder_createDefault(cv.detail.SeamFinder_NO) + +ESTIMATOR_CHOICES = OrderedDict() +ESTIMATOR_CHOICES['homography'] = cv.detail_HomographyBasedEstimator +ESTIMATOR_CHOICES['affine'] = cv.detail_AffineBasedEstimator + +WARP_CHOICES = ( + 'spherical', + 'plane', + 'affine', + 'cylindrical', + 'fisheye', + 'stereographic', + 'compressedPlaneA2B1', + 'compressedPlaneA1.5B1', + 'compressedPlanePortraitA2B1', + 'compressedPlanePortraitA1.5B1', + 'paniniA2B1', + 'paniniA1.5B1', + 'paniniPortraitA2B1', + 'paniniPortraitA1.5B1', + 'mercator', + 'transverseMercator', +) + +WAVE_CORRECT_CHOICES = OrderedDict() +WAVE_CORRECT_CHOICES['horiz'] = cv.detail.WAVE_CORRECT_HORIZ +WAVE_CORRECT_CHOICES['no'] = None +WAVE_CORRECT_CHOICES['vert'] = cv.detail.WAVE_CORRECT_VERT + +BLEND_CHOICES = ('multiband', 'feather', 'no',) + +def get_matcher(args): + try_cuda = args.try_cuda + matcher_type = args.matcher + if args.match_conf is None: + if args.features == 'orb': + match_conf = 0.3 + else: + match_conf = 0.65 + else: + match_conf = args.match_conf + range_width = args.rangewidth + if matcher_type == "affine": + matcher = cv.detail_AffineBestOf2NearestMatcher(False, try_cuda, match_conf) + elif range_width == -1: + matcher = cv.detail.BestOf2NearestMatcher_create(try_cuda, match_conf) + else: + matcher = cv.detail.BestOf2NearestRangeMatcher_create(range_width, try_cuda, match_conf) + return matcher + + +def get_compensator(args): + expos_comp_type = EXPOS_COMP_CHOICES[args.expos_comp] + expos_comp_nr_feeds = args.expos_comp_nr_feeds + expos_comp_block_size = args.expos_comp_block_size + # expos_comp_nr_filtering = args.expos_comp_nr_filtering + if expos_comp_type == cv.detail.ExposureCompensator_CHANNELS: + compensator = cv.detail_ChannelsCompensator(expos_comp_nr_feeds) + # compensator.setNrGainsFilteringIterations(expos_comp_nr_filtering) + elif expos_comp_type == cv.detail.ExposureCompensator_CHANNELS_BLOCKS: + compensator = cv.detail_BlocksChannelsCompensator( + expos_comp_block_size, expos_comp_block_size, + expos_comp_nr_feeds + ) + # compensator.setNrGainsFilteringIterations(expos_comp_nr_filtering) + else: + compensator = cv.detail.ExposureCompensator_createDefault(expos_comp_type) + return compensator + + +def main(): + + args = { + "img_names":["boat5.jpg", "boat2.jpg", + "boat3.jpg", "boat4.jpg", + "boat1.jpg", "boat6.jpg"], + "try_cuda": False, + "work_megapix": 0.6, + "features": "orb", + "matcher": "homography", + "estimator": "homography", + "match_conf": None, + "conf_thresh": 1.0, + "ba": "ray", + "ba_refine_mask": "xxxxx", + "wave_correct": "horiz", + "save_graph": None, + "warp": "spherical", + "seam_megapix": 0.1, + "seam": "dp_color", + "compose_megapix": 3, + "expos_comp": "gain_blocks", + "expos_comp_nr_feeds": 1, + "expos_comp_nr_filtering": 2, + "expos_comp_block_size": 32, + "blend": "multiband", + "blend_strength": 5, + "output": "time_test.jpg", + "timelapse": None, + "rangewidth": -1 + } + + args = SimpleNamespace(**args) + img_names = args.img_names + work_megapix = args.work_megapix + seam_megapix = args.seam_megapix + compose_megapix = args.compose_megapix + conf_thresh = args.conf_thresh + ba_refine_mask = args.ba_refine_mask + wave_correct = WAVE_CORRECT_CHOICES[args.wave_correct] + if args.save_graph is None: + save_graph = False + else: + save_graph = True + warp_type = args.warp + blend_type = args.blend + blend_strength = args.blend_strength + result_name = args.output + if args.timelapse is not None: + timelapse = True + if args.timelapse == "as_is": + timelapse_type = cv.detail.Timelapser_AS_IS + elif args.timelapse == "crop": + timelapse_type = cv.detail.Timelapser_CROP + else: + print("Bad timelapse method") + exit() + else: + timelapse = False + finder = FEATURES_FIND_CHOICES[args.features]() + seam_work_aspect = 1 + full_img_sizes = [] + features = [] + images = [] + is_work_scale_set = False + is_seam_scale_set = False + is_compose_scale_set = False + for name in img_names: + full_img = cv.imread(cv.samples.findFile(name)) + if full_img is None: + print("Cannot read image ", name) + exit() + full_img_sizes.append((full_img.shape[1], full_img.shape[0])) + if work_megapix < 0: + img = full_img + work_scale = 1 + is_work_scale_set = True + else: + if is_work_scale_set is False: + work_scale = min(1.0, np.sqrt(work_megapix * 1e6 / (full_img.shape[0] * full_img.shape[1]))) + is_work_scale_set = True + img = cv.resize(src=full_img, dsize=None, fx=work_scale, fy=work_scale, interpolation=cv.INTER_LINEAR_EXACT) + if is_seam_scale_set is False: + seam_scale = min(1.0, np.sqrt(seam_megapix * 1e6 / (full_img.shape[0] * full_img.shape[1]))) + seam_work_aspect = seam_scale / work_scale + is_seam_scale_set = True + img_feat = cv.detail.computeImageFeatures2(finder, img) + features.append(img_feat) + img = cv.resize(src=full_img, dsize=None, fx=seam_scale, fy=seam_scale, interpolation=cv.INTER_LINEAR_EXACT) + images.append(img) + + matcher = get_matcher(args) + p = matcher.apply2(features) + matcher.collectGarbage() + + if save_graph: + with open(args.save_graph, 'w') as fh: + fh.write(cv.detail.matchesGraphAsString(img_names, p, conf_thresh)) + + indices = cv.detail.leaveBiggestComponent(features, p, conf_thresh) + img_subset = [] + img_names_subset = [] + full_img_sizes_subset = [] + for i in range(len(indices)): + img_names_subset.append(img_names[indices[i, 0]]) + img_subset.append(images[indices[i, 0]]) + full_img_sizes_subset.append(full_img_sizes[indices[i, 0]]) + images = img_subset + img_names = img_names_subset + full_img_sizes = full_img_sizes_subset + num_images = len(img_names) + if num_images < 2: + print("Need more images") + exit() + + estimator = ESTIMATOR_CHOICES[args.estimator]() + b, cameras = estimator.apply(features, p, None) + if not b: + print("Homography estimation failed.") + exit() + for cam in cameras: + cam.R = cam.R.astype(np.float32) + + adjuster = BA_COST_CHOICES[args.ba]() + adjuster.setConfThresh(1) + refine_mask = np.zeros((3, 3), np.uint8) + if ba_refine_mask[0] == 'x': + refine_mask[0, 0] = 1 + if ba_refine_mask[1] == 'x': + refine_mask[0, 1] = 1 + if ba_refine_mask[2] == 'x': + refine_mask[0, 2] = 1 + if ba_refine_mask[3] == 'x': + refine_mask[1, 1] = 1 + if ba_refine_mask[4] == 'x': + refine_mask[1, 2] = 1 + adjuster.setRefinementMask(refine_mask) + b, cameras = adjuster.apply(features, p, cameras) + if not b: + print("Camera parameters adjusting failed.") + exit() + focals = [] + for cam in cameras: + focals.append(cam.focal) + focals.sort() + if len(focals) % 2 == 1: + warped_image_scale = focals[len(focals) // 2] + else: + warped_image_scale = (focals[len(focals) // 2] + focals[len(focals) // 2 - 1]) / 2 + if wave_correct is not None: + rmats = [] + for cam in cameras: + rmats.append(np.copy(cam.R)) + rmats = cv.detail.waveCorrect(rmats, wave_correct) + for idx, cam in enumerate(cameras): + cam.R = rmats[idx] + corners = [] + masks_warped = [] + images_warped = [] + sizes = [] + masks = [] + for i in range(0, num_images): + um = cv.UMat(255 * np.ones((images[i].shape[0], images[i].shape[1]), np.uint8)) + masks.append(um) + + warper = cv.PyRotationWarper(warp_type, warped_image_scale * seam_work_aspect) # warper could be nullptr? + for idx in range(0, num_images): + K = cameras[idx].K().astype(np.float32) + swa = seam_work_aspect + K[0, 0] *= swa + K[0, 2] *= swa + K[1, 1] *= swa + K[1, 2] *= swa + corner, image_wp = warper.warp(images[idx], K, cameras[idx].R, cv.INTER_LINEAR, cv.BORDER_REFLECT) + corners.append(corner) + sizes.append((image_wp.shape[1], image_wp.shape[0])) + images_warped.append(image_wp) + p, mask_wp = warper.warp(masks[idx], K, cameras[idx].R, cv.INTER_NEAREST, cv.BORDER_CONSTANT) + masks_warped.append(mask_wp.get()) + + images_warped_f = [] + for img in images_warped: + imgf = img.astype(np.float32) + images_warped_f.append(imgf) + + compensator = get_compensator(args) + compensator.feed(corners=corners, images=images_warped, masks=masks_warped) + + seam_finder = SEAM_FIND_CHOICES[args.seam] + masks_warped = seam_finder.find(images_warped_f, corners, masks_warped) + compose_scale = 1 + corners = [] + sizes = [] + blender = None + timelapser = None + # https://github.com/opencv/opencv/blob/master/samples/cpp/stitching_detailed.cpp#L725 ? + for idx, name in enumerate(img_names): + full_img = cv.imread(name) + if not is_compose_scale_set: + if compose_megapix > 0: + compose_scale = min(1.0, np.sqrt(compose_megapix * 1e6 / (full_img.shape[0] * full_img.shape[1]))) + is_compose_scale_set = True + compose_work_aspect = compose_scale / work_scale + warped_image_scale *= compose_work_aspect + warper = cv.PyRotationWarper(warp_type, warped_image_scale) + for i in range(0, len(img_names)): + cameras[i].focal *= compose_work_aspect + cameras[i].ppx *= compose_work_aspect + cameras[i].ppy *= compose_work_aspect + sz = (int(round(full_img_sizes[i][0] * compose_scale)), + int(round(full_img_sizes[i][1] * compose_scale))) + K = cameras[i].K().astype(np.float32) + roi = warper.warpRoi(sz, K, cameras[i].R) + corners.append(roi[0:2]) + sizes.append(roi[2:4]) + if abs(compose_scale - 1) > 1e-1: + img = cv.resize(src=full_img, dsize=None, fx=compose_scale, fy=compose_scale, + interpolation=cv.INTER_LINEAR_EXACT) + else: + img = full_img + _img_size = (img.shape[1], img.shape[0]) + K = cameras[idx].K().astype(np.float32) + corner, image_warped = warper.warp(img, K, cameras[idx].R, cv.INTER_LINEAR, cv.BORDER_REFLECT) + mask = 255 * np.ones((img.shape[0], img.shape[1]), np.uint8) + p, mask_warped = warper.warp(mask, K, cameras[idx].R, cv.INTER_NEAREST, cv.BORDER_CONSTANT) + compensator.apply(idx, corners[idx], image_warped, mask_warped) + image_warped_s = image_warped.astype(np.int16) + dilated_mask = cv.dilate(masks_warped[idx], None) + seam_mask = cv.resize(dilated_mask, (mask_warped.shape[1], mask_warped.shape[0]), 0, 0, cv.INTER_LINEAR_EXACT) + mask_warped = cv.bitwise_and(seam_mask, mask_warped) + if blender is None and not timelapse: + blender = cv.detail.Blender_createDefault(cv.detail.Blender_NO) + dst_sz = cv.detail.resultRoi(corners=corners, sizes=sizes) + blend_width = np.sqrt(dst_sz[2] * dst_sz[3]) * blend_strength / 100 + if blend_width < 1: + blender = cv.detail.Blender_createDefault(cv.detail.Blender_NO) + elif blend_type == "multiband": + blender = cv.detail_MultiBandBlender() + blender.setNumBands((np.log(blend_width) / np.log(2.) - 1.).astype(np.int64)) + elif blend_type == "feather": + blender = cv.detail_FeatherBlender() + blender.setSharpness(1. / blend_width) + blender.prepare(dst_sz) + elif timelapser is None and timelapse: + timelapser = cv.detail.Timelapser_createDefault(timelapse_type) + timelapser.initialize(corners, sizes) + if timelapse: + ma_tones = np.ones((image_warped_s.shape[0], image_warped_s.shape[1]), np.uint8) + timelapser.process(image_warped_s, ma_tones, corners[idx]) + pos_s = img_names[idx].rfind("/") + if pos_s == -1: + fixed_file_name = "fixed_" + img_names[idx] + else: + fixed_file_name = img_names[idx][:pos_s + 1] + "fixed_" + img_names[idx][pos_s + 1:] + cv.imwrite(fixed_file_name, timelapser.getDst()) + else: + blender.feed(cv.UMat(image_warped_s), mask_warped, corners[idx]) + if not timelapse: + result = None + result_mask = None + result, result_mask = blender.blend(result, result_mask) + # cv.imwrite(result_name, result) + return result + # zoom_x = 600.0 / result.shape[1] + # dst = cv.normalize(src=result, dst=None, alpha=255., norm_type=cv.NORM_MINMAX, dtype=cv.CV_8U) + # dst = cv.resize(dst, dsize=None, fx=zoom_x, fy=zoom_x) + # cv.imshow(result_name, dst) + # cv.waitKey() + + + +if __name__ == '__main__': + import tracemalloc + import time + tracemalloc.start() + start = time.time() + result = main() + current, peak = tracemalloc.get_traced_memory() + print(f"Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB") + tracemalloc.stop() + end = time.time() + print(end - start) diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/test_composition.py b/apps/opencv_stitching_tool/opencv_stitching/test/test_composition.py new file mode 100644 index 000000000000..b0b4d76c87ac --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/test_composition.py @@ -0,0 +1,67 @@ +import unittest +import os +import sys + +import numpy as np +import cv2 as cv + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..'))) + +from opencv_stitching.stitcher import Stitcher + + +class TestImageComposition(unittest.TestCase): + + # visual test: look especially in the sky + def test_exposure_compensation(self): + img = cv.imread("s1.jpg") + img = increase_brightness(img, value=25) + cv.imwrite("s1_bright.jpg", img) + + stitcher = Stitcher(compensator="no", blender_type="no") + result = stitcher.stitch(["s1_bright.jpg", "s2.jpg"]) + + cv.imwrite("without_exposure_comp.jpg", result) + + stitcher = Stitcher(blender_type="no") + result = stitcher.stitch(["s1_bright.jpg", "s2.jpg"]) + + cv.imwrite("with_exposure_comp.jpg", result) + + def test_timelapse(self): + stitcher = Stitcher(timelapse='as_is') + _ = stitcher.stitch(["s1.jpg", "s2.jpg"]) + frame1 = cv.imread("fixed_s1.jpg") + + max_image_shape_derivation = 3 + np.testing.assert_allclose(frame1.shape[:2], + (700, 1811), + atol=max_image_shape_derivation) + + left = cv.cvtColor(frame1[:, :1300, ], cv.COLOR_BGR2GRAY) + right = cv.cvtColor(frame1[:, 1300:, ], cv.COLOR_BGR2GRAY) + + self.assertGreater(cv.countNonZero(left), 800000) + self.assertEqual(cv.countNonZero(right), 0) + + +def increase_brightness(img, value=30): + hsv = cv.cvtColor(img, cv.COLOR_BGR2HSV) + h, s, v = cv.split(hsv) + + lim = 255 - value + v[v > lim] = 255 + v[v <= lim] += value + + final_hsv = cv.merge((h, s, v)) + img = cv.cvtColor(final_hsv, cv.COLOR_HSV2BGR) + return img + + +def starttest(): + unittest.main() + + +if __name__ == "__main__": + starttest() diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/test_matcher.py b/apps/opencv_stitching_tool/opencv_stitching/test/test_matcher.py new file mode 100644 index 000000000000..a2424ec9ce81 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/test_matcher.py @@ -0,0 +1,47 @@ +import unittest +import os +import sys + +import numpy as np + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..'))) + +from opencv_stitching.feature_matcher import FeatureMatcher +# %% + + +class TestMatcher(unittest.TestCase): + + def test_array_in_sqare_matrix(self): + array = np.zeros(9) + + matrix = FeatureMatcher.array_in_sqare_matrix(array) + + np.testing.assert_array_equal(matrix, np.array([[0., 0., 0.], + [0., 0., 0.], + [0., 0., 0.]])) + + def test_get_all_img_combinations(self): + nimgs = 3 + + combinations = list(FeatureMatcher.get_all_img_combinations(nimgs)) + + self.assertEqual(combinations, [(0, 1), (0, 2), (1, 2)]) + + def test_get_match_conf(self): + explicit_match_conf = FeatureMatcher.get_match_conf(1, 'orb') + implicit_match_conf_orb = FeatureMatcher.get_match_conf(None, 'orb') + implicit_match_conf_other = FeatureMatcher.get_match_conf(None, 'surf') + + self.assertEqual(explicit_match_conf, 1) + self.assertEqual(implicit_match_conf_orb, 0.3) + self.assertEqual(implicit_match_conf_other, 0.65) + + +def starttest(): + unittest.main() + + +if __name__ == "__main__": + starttest() diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/test_megapix_scaler.py b/apps/opencv_stitching_tool/opencv_stitching/test/test_megapix_scaler.py new file mode 100644 index 000000000000..0afdad2628b7 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/test_megapix_scaler.py @@ -0,0 +1,59 @@ +import unittest +import os +import sys + +import cv2 as cv + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..'))) + +from opencv_stitching.megapix_scaler import MegapixScaler +from opencv_stitching.megapix_downscaler import MegapixDownscaler +#%% + + +class TestScaler(unittest.TestCase): + + def setUp(self): + self.img = cv.imread("s1.jpg") + self.size = (self.img.shape[1], self.img.shape[0]) + + def test_get_scale_by_resolution(self): + scaler = MegapixScaler(0.6) + + scale = scaler.get_scale_by_resolution(1_200_000) + + self.assertEqual(scale, 0.7071067811865476) + + def test_get_scale_by_image(self): + scaler = MegapixScaler(0.6) + + scaler.set_scale_by_img_size(self.size) + + self.assertEqual(scaler.scale, 0.8294067854101966) + + def test_get_scaled_img_size(self): + scaler = MegapixScaler(0.6) + scaler.set_scale_by_img_size(self.size) + + size = scaler.get_scaled_img_size(self.size) + self.assertEqual(size, (1033, 581)) + # 581*1033 = 600173 px = ~0.6 MP + + def test_force_of_downscaling(self): + normal_scaler = MegapixScaler(2) + downscaler = MegapixDownscaler(2) + + normal_scaler.set_scale_by_img_size(self.size) + downscaler.set_scale_by_img_size(self.size) + + self.assertEqual(normal_scaler.scale, 1.5142826857233715) + self.assertEqual(downscaler.scale, 1.0) + + +def starttest(): + unittest.main() + + +if __name__ == "__main__": + starttest() diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/test_performance.py b/apps/opencv_stitching_tool/opencv_stitching/test/test_performance.py new file mode 100644 index 000000000000..60b03a8bfe2b --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/test_performance.py @@ -0,0 +1,65 @@ +import unittest +import os +import sys +import time +import tracemalloc + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..'))) + +from opencv_stitching.stitcher import Stitcher +from stitching_detailed import main +# %% + + +class TestStitcher(unittest.TestCase): + + def test_performance(self): + + print("Run new Stitcher class:") + + start = time.time() + tracemalloc.start() + + stitcher = Stitcher(final_megapix=3) + stitcher.stitch(["boat5.jpg", "boat2.jpg", + "boat3.jpg", "boat4.jpg", + "boat1.jpg", "boat6.jpg"]) + stitcher.collect_garbage() + + _, peak_memory = tracemalloc.get_traced_memory() + tracemalloc.stop() + end = time.time() + time_needed = end - start + + print(f"Peak was {peak_memory / 10**6} MB") + print(f"Time was {time_needed} s") + + print("Run original stitching_detailed.py:") + + start = time.time() + tracemalloc.start() + + main() + + _, peak_memory_detailed = tracemalloc.get_traced_memory() + tracemalloc.stop() + end = time.time() + time_needed_detailed = end - start + + print(f"Peak was {peak_memory_detailed / 10**6} MB") + print(f"Time was {time_needed_detailed} s") + + self.assertLessEqual(peak_memory / 10**6, + peak_memory_detailed / 10**6) + uncertainty_based_on_run = 0.25 + self.assertLessEqual(time_needed - uncertainty_based_on_run, + time_needed_detailed) + + +def starttest(): + unittest.main() + + +if __name__ == "__main__": + starttest() diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/test_registration.py b/apps/opencv_stitching_tool/opencv_stitching/test/test_registration.py new file mode 100644 index 000000000000..98e792fd01fe --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/test_registration.py @@ -0,0 +1,100 @@ +import unittest +import os +import sys + +import numpy as np +import cv2 as cv + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..'))) + +from opencv_stitching.feature_detector import FeatureDetector +from opencv_stitching.feature_matcher import FeatureMatcher +from opencv_stitching.subsetter import Subsetter + + +class TestImageRegistration(unittest.TestCase): + + def test_feature_detector(self): + img1 = cv.imread("s1.jpg") + + default_number_of_keypoints = 500 + detector = FeatureDetector("orb") + features = detector.detect_features(img1) + self.assertEqual(len(features.getKeypoints()), + default_number_of_keypoints) + + other_keypoints = 1000 + detector = FeatureDetector("orb", nfeatures=other_keypoints) + features = detector.detect_features(img1) + self.assertEqual(len(features.getKeypoints()), other_keypoints) + + def test_feature_matcher(self): + img1, img2 = cv.imread("s1.jpg"), cv.imread("s2.jpg") + + detector = FeatureDetector("orb") + features = [detector.detect_features(img1), + detector.detect_features(img2)] + + matcher = FeatureMatcher() + pairwise_matches = matcher.match_features(features) + self.assertEqual(len(pairwise_matches), len(features)**2) + self.assertGreater(pairwise_matches[1].confidence, 2) + + matches_matrix = FeatureMatcher.get_matches_matrix(pairwise_matches) + self.assertEqual(matches_matrix.shape, (2, 2)) + conf_matrix = FeatureMatcher.get_confidence_matrix(pairwise_matches) + self.assertTrue(np.array_equal( + conf_matrix > 2, + np.array([[False, True], [True, False]]) + )) + + def test_subsetting(self): + img1, img2 = cv.imread("s1.jpg"), cv.imread("s2.jpg") + img3, img4 = cv.imread("boat1.jpg"), cv.imread("boat2.jpg") + img5 = cv.imread("boat3.jpg") + img_names = ["s1.jpg", "s2.jpg", "boat1.jpg", "boat2.jpg", "boat3.jpg"] + + detector = FeatureDetector("orb") + features = [detector.detect_features(img1), + detector.detect_features(img2), + detector.detect_features(img3), + detector.detect_features(img4), + detector.detect_features(img5)] + matcher = FeatureMatcher() + pairwise_matches = matcher.match_features(features) + subsetter = Subsetter(confidence_threshold=1, + matches_graph_dot_file="dot_graph.txt") # view in https://dreampuf.github.io # noqa + + indices = subsetter.get_indices_to_keep(features, pairwise_matches) + indices_to_delete = subsetter.get_indices_to_delete(len(img_names), + indices) + + self.assertEqual(indices, [2, 3, 4]) + self.assertEqual(indices_to_delete, [0, 1]) + + subsetted_image_names = subsetter.subset_list(img_names, indices) + self.assertEqual(subsetted_image_names, + ['boat1.jpg', 'boat2.jpg', 'boat3.jpg']) + + matches_subset = subsetter.subset_matches(pairwise_matches, indices) + # FeatureMatcher.get_confidence_matrix(pairwise_matches) + # FeatureMatcher.get_confidence_matrix(subsetted_matches) + self.assertEqual(pairwise_matches[13].confidence, + matches_subset[1].confidence) + + graph = subsetter.get_matches_graph(img_names, pairwise_matches) + self.assertTrue(graph.startswith("graph matches_graph{")) + + subsetter.save_matches_graph_dot_file(img_names, pairwise_matches) + with open('dot_graph.txt', 'r') as file: + graph = file.read() + self.assertTrue(graph.startswith("graph matches_graph{")) + + +def starttest(): + unittest.main() + + +if __name__ == "__main__": + starttest() diff --git a/apps/opencv_stitching_tool/opencv_stitching/test/test_stitcher.py b/apps/opencv_stitching_tool/opencv_stitching/test/test_stitcher.py new file mode 100644 index 000000000000..5a24f752c0b3 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/test/test_stitcher.py @@ -0,0 +1,108 @@ +import unittest +import os +import sys + +import numpy as np +import cv2 as cv + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..'))) + +from opencv_stitching.stitcher import Stitcher +# %% + + +class TestStitcher(unittest.TestCase): + + def test_stitcher_aquaduct(self): + stitcher = Stitcher(n_features=250) + result = stitcher.stitch(["s1.jpg", "s2.jpg"]) + cv.imwrite("result.jpg", result) + + max_image_shape_derivation = 3 + np.testing.assert_allclose(result.shape[:2], + (700, 1811), + atol=max_image_shape_derivation) + + @unittest.skip("skip boat test (high resuolution ran >30s)") + def test_stitcher_boat1(self): + settings = {"warper_type": "fisheye", + "wave_correct_kind": "no", + "finder": "dp_colorgrad", + "compensator": "no", + "conf_thresh": 0.3} + + stitcher = Stitcher(**settings) + result = stitcher.stitch(["boat5.jpg", "boat2.jpg", + "boat3.jpg", "boat4.jpg", + "boat1.jpg", "boat6.jpg"]) + + cv.imwrite("boat_fisheye.jpg", result) + + max_image_shape_derivation = 600 + np.testing.assert_allclose(result.shape[:2], + (14488, 7556), + atol=max_image_shape_derivation) + + @unittest.skip("skip boat test (high resuolution ran >30s)") + def test_stitcher_boat2(self): + settings = {"warper_type": "compressedPlaneA2B1", + "finder": "dp_colorgrad", + "compensator": "channel_blocks", + "conf_thresh": 0.3} + + stitcher = Stitcher(**settings) + result = stitcher.stitch(["boat5.jpg", "boat2.jpg", + "boat3.jpg", "boat4.jpg", + "boat1.jpg", "boat6.jpg"]) + + cv.imwrite("boat_plane.jpg", result) + + max_image_shape_derivation = 600 + np.testing.assert_allclose(result.shape[:2], + (7400, 12340), + atol=max_image_shape_derivation) + + def test_stitcher_boat_aquaduct_subset(self): + settings = {"final_megapix": 1} + + stitcher = Stitcher(**settings) + result = stitcher.stitch(["boat5.jpg", + "s1.jpg", "s2.jpg", + "boat2.jpg", + "boat3.jpg", "boat4.jpg", + "boat1.jpg", "boat6.jpg"]) + cv.imwrite("subset_low_res.jpg", result) + + max_image_shape_derivation = 100 + np.testing.assert_allclose(result.shape[:2], + (839, 3384), + atol=max_image_shape_derivation) + + def test_stitcher_budapest(self): + settings = {"matcher_type": "affine", + "estimator": "affine", + "adjuster": "affine", + "warper_type": "affine", + "wave_correct_kind": "no", + "confidence_threshold": 0.3} + + stitcher = Stitcher(**settings) + result = stitcher.stitch(["budapest1.jpg", "budapest2.jpg", + "budapest3.jpg", "budapest4.jpg", + "budapest5.jpg", "budapest6.jpg"]) + + cv.imwrite("budapest.jpg", result) + + max_image_shape_derivation = 50 + np.testing.assert_allclose(result.shape[:2], + (1155, 2310), + atol=max_image_shape_derivation) + + +def starttest(): + unittest.main() + + +if __name__ == "__main__": + starttest() diff --git a/apps/opencv_stitching_tool/opencv_stitching/timelapser.py b/apps/opencv_stitching_tool/opencv_stitching/timelapser.py new file mode 100644 index 000000000000..4085f473fa4e --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/timelapser.py @@ -0,0 +1,50 @@ +import os +import cv2 as cv +import numpy as np + + +class Timelapser: + + TIMELAPSE_CHOICES = ('no', 'as_is', 'crop',) + DEFAULT_TIMELAPSE = 'no' + + def __init__(self, timelapse=DEFAULT_TIMELAPSE): + self.do_timelapse = True + self.timelapse_type = None + self.timelapser = None + + if timelapse == "as_is": + self.timelapse_type = cv.detail.Timelapser_AS_IS + elif timelapse == "crop": + self.timelapse_type = cv.detail.Timelapser_CROP + else: + self.do_timelapse = False + + if self.do_timelapse: + self.timelapser = cv.detail.Timelapser_createDefault( + self.timelapse_type + ) + + def initialize(self, *args): + """https://docs.opencv.org/master/dd/dac/classcv_1_1detail_1_1Timelapser.html#aaf0f7c4128009f02473332a0c41f6345""" # noqa + self.timelapser.initialize(*args) + + def process_and_save_frame(self, img_name, img, corner): + self.process_frame(img, corner) + cv.imwrite(self.get_fixed_filename(img_name), self.get_frame()) + + def process_frame(self, img, corner): + mask = np.ones((img.shape[0], img.shape[1]), np.uint8) + img = img.astype(np.int16) + self.timelapser.process(img, mask, corner) + + def get_frame(self): + frame = self.timelapser.getDst() + frame = np.float32(cv.UMat.get(frame)) + frame = cv.convertScaleAbs(frame) + return frame + + @staticmethod + def get_fixed_filename(img_name): + dirname, filename = os.path.split(img_name) + return os.path.join(dirname, "fixed_" + filename) diff --git a/apps/opencv_stitching_tool/opencv_stitching/warper.py b/apps/opencv_stitching_tool/opencv_stitching/warper.py new file mode 100644 index 000000000000..c31a8648c0a9 --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching/warper.py @@ -0,0 +1,71 @@ +import cv2 as cv +import numpy as np + + +class Warper: + + WARP_TYPE_CHOICES = ('spherical', 'plane', 'affine', 'cylindrical', + 'fisheye', 'stereographic', 'compressedPlaneA2B1', + 'compressedPlaneA1.5B1', + 'compressedPlanePortraitA2B1', + 'compressedPlanePortraitA1.5B1', + 'paniniA2B1', 'paniniA1.5B1', 'paniniPortraitA2B1', + 'paniniPortraitA1.5B1', 'mercator', + 'transverseMercator') + + DEFAULT_WARP_TYPE = 'spherical' + + def __init__(self, warper_type=DEFAULT_WARP_TYPE, scale=1): + self.warper_type = warper_type + self.warper = cv.PyRotationWarper(warper_type, scale) + self.scale = scale + + def warp_images_and_image_masks(self, imgs, cameras, scale=None, aspect=1): + self.update_scale(scale) + for img, camera in zip(imgs, cameras): + yield self.warp_image_and_image_mask(img, camera, scale, aspect) + + def warp_image_and_image_mask(self, img, camera, scale=None, aspect=1): + self.update_scale(scale) + corner, img_warped = self.warp_image(img, camera, aspect) + mask = 255 * np.ones((img.shape[0], img.shape[1]), np.uint8) + _, mask_warped = self.warp_image(mask, camera, aspect, mask=True) + return img_warped, mask_warped, corner + + def warp_image(self, image, camera, aspect=1, mask=False): + if mask: + interp_mode = cv.INTER_NEAREST + border_mode = cv.BORDER_CONSTANT + else: + interp_mode = cv.INTER_LINEAR + border_mode = cv.BORDER_REFLECT + + corner, warped_image = self.warper.warp(image, + Warper.get_K(camera, aspect), + camera.R, + interp_mode, + border_mode) + return corner, warped_image + + def warp_roi(self, width, height, camera, scale=None, aspect=1): + self.update_scale(scale) + roi = (width, height) + K = Warper.get_K(camera, aspect) + return self.warper.warpRoi(roi, K, camera.R) + + def update_scale(self, scale): + if scale is not None and scale != self.scale: + self.warper = cv.PyRotationWarper(self.warper_type, scale) # setScale not working: https://docs.opencv.org/master/d5/d76/classcv_1_1PyRotationWarper.html#a90b000bb75f95294f9b0b6ec9859eb55 + self.scale = scale + + @staticmethod + def get_K(camera, aspect=1): + K = camera.K().astype(np.float32) + """ Modification of intrinsic parameters needed if cameras were + obtained on different scale than the scale of the Images which should + be warped """ + K[0, 0] *= aspect + K[0, 2] *= aspect + K[1, 1] *= aspect + K[1, 2] *= aspect + return K diff --git a/apps/opencv_stitching_tool/opencv_stitching_tool.py b/apps/opencv_stitching_tool/opencv_stitching_tool.py new file mode 100644 index 000000000000..1ee96aa8cb4a --- /dev/null +++ b/apps/opencv_stitching_tool/opencv_stitching_tool.py @@ -0,0 +1,232 @@ +""" +Stitching sample (advanced) +=========================== + +Show how to use Stitcher API from python. +""" + +# Python 2/3 compatibility +from __future__ import print_function + +import argparse + +import cv2 as cv +import numpy as np + +from opencv_stitching.stitcher import Stitcher + +from opencv_stitching.image_handler import ImageHandler +from opencv_stitching.feature_detector import FeatureDetector +from opencv_stitching.feature_matcher import FeatureMatcher +from opencv_stitching.subsetter import Subsetter +from opencv_stitching.camera_estimator import CameraEstimator +from opencv_stitching.camera_adjuster import CameraAdjuster +from opencv_stitching.camera_wave_corrector import WaveCorrector +from opencv_stitching.warper import Warper +from opencv_stitching.exposure_error_compensator import ExposureErrorCompensator # noqa +from opencv_stitching.seam_finder import SeamFinder +from opencv_stitching.blender import Blender +from opencv_stitching.timelapser import Timelapser + +parser = argparse.ArgumentParser( + prog="opencv_stitching_tool.py", + description="Rotation model images stitcher" +) +parser.add_argument( + 'img_names', nargs='+', + help="Files to stitch", type=str +) +parser.add_argument( + '--medium_megapix', action='store', + default=ImageHandler.DEFAULT_MEDIUM_MEGAPIX, + help="Resolution for image registration step. " + "The default is %s Mpx" % ImageHandler.DEFAULT_MEDIUM_MEGAPIX, + type=float, dest='medium_megapix' +) +parser.add_argument( + '--detector', action='store', + default=FeatureDetector.DEFAULT_DETECTOR, + help="Type of features used for images matching. " + "The default is '%s'." % FeatureDetector.DEFAULT_DETECTOR, + choices=FeatureDetector.DETECTOR_CHOICES.keys(), + type=str, dest='detector' +) +parser.add_argument( + '--nfeatures', action='store', + default=500, + help="Type of features used for images matching. " + "The default is 500.", + type=int, dest='nfeatures' +) +parser.add_argument( + '--matcher_type', action='store', default=FeatureMatcher.DEFAULT_MATCHER, + help="Matcher used for pairwise image matching. " + "The default is '%s'." % FeatureMatcher.DEFAULT_MATCHER, + choices=FeatureMatcher.MATCHER_CHOICES, + type=str, dest='matcher_type' +) +parser.add_argument( + '--range_width', action='store', + default=FeatureMatcher.DEFAULT_RANGE_WIDTH, + help="uses range_width to limit number of images to match with.", + type=int, dest='range_width' +) +parser.add_argument( + '--try_use_gpu', + action='store', + default=False, + help="Try to use CUDA. The default value is no. " + "All default values are for CPU mode.", + type=bool, dest='try_use_gpu' +) +parser.add_argument( + '--match_conf', action='store', + help="Confidence for feature matching step. " + "The default is 0.3 for ORB and 0.65 for other feature types.", + type=float, dest='match_conf' +) +parser.add_argument( + '--confidence_threshold', action='store', + default=Subsetter.DEFAULT_CONFIDENCE_THRESHOLD, + help="Threshold for two images are from the same panorama confidence. " + "The default is '%s'." % Subsetter.DEFAULT_CONFIDENCE_THRESHOLD, + type=float, dest='confidence_threshold' +) +parser.add_argument( + '--matches_graph_dot_file', action='store', + default=Subsetter.DEFAULT_MATCHES_GRAPH_DOT_FILE, + help="Save matches graph represented in DOT language to file.", + type=str, dest='matches_graph_dot_file' +) +parser.add_argument( + '--estimator', action='store', + default=CameraEstimator.DEFAULT_CAMERA_ESTIMATOR, + help="Type of estimator used for transformation estimation. " + "The default is '%s'." % CameraEstimator.DEFAULT_CAMERA_ESTIMATOR, + choices=CameraEstimator.CAMERA_ESTIMATOR_CHOICES.keys(), + type=str, dest='estimator' +) +parser.add_argument( + '--adjuster', action='store', + default=CameraAdjuster.DEFAULT_CAMERA_ADJUSTER, + help="Bundle adjustment cost function. " + "The default is '%s'." % CameraAdjuster.DEFAULT_CAMERA_ADJUSTER, + choices=CameraAdjuster.CAMERA_ADJUSTER_CHOICES.keys(), + type=str, dest='adjuster' +) +parser.add_argument( + '--refinement_mask', action='store', + default=CameraAdjuster.DEFAULT_REFINEMENT_MASK, + help="Set refinement mask for bundle adjustment. It looks like 'x_xxx', " + "where 'x' means refine respective parameter and '_' means don't " + "refine, and has the following format:. " + "The default mask is '%s'. " + "If bundle adjustment doesn't support estimation of selected " + "parameter then the respective flag is ignored." + "" % CameraAdjuster.DEFAULT_REFINEMENT_MASK, + type=str, dest='refinement_mask' +) +parser.add_argument( + '--wave_correct_kind', action='store', + default=WaveCorrector.DEFAULT_WAVE_CORRECTION, + help="Perform wave effect correction. " + "The default is '%s'" % WaveCorrector.DEFAULT_WAVE_CORRECTION, + choices=WaveCorrector.WAVE_CORRECT_CHOICES.keys(), + type=str, dest='wave_correct_kind' +) +parser.add_argument( + '--warper_type', action='store', default=Warper.DEFAULT_WARP_TYPE, + help="Warp surface type. The default is '%s'." % Warper.DEFAULT_WARP_TYPE, + choices=Warper.WARP_TYPE_CHOICES, + type=str, dest='warper_type' +) +parser.add_argument( + '--low_megapix', action='store', default=ImageHandler.DEFAULT_LOW_MEGAPIX, + help="Resolution for seam estimation and exposure estimation step. " + "The default is %s Mpx." % ImageHandler.DEFAULT_LOW_MEGAPIX, + type=float, dest='low_megapix' +) +parser.add_argument( + '--compensator', action='store', + default=ExposureErrorCompensator.DEFAULT_COMPENSATOR, + help="Exposure compensation method. " + "The default is '%s'." % ExposureErrorCompensator.DEFAULT_COMPENSATOR, + choices=ExposureErrorCompensator.COMPENSATOR_CHOICES.keys(), + type=str, dest='compensator' +) +parser.add_argument( + '--nr_feeds', action='store', + default=ExposureErrorCompensator.DEFAULT_NR_FEEDS, + help="Number of exposure compensation feed.", + type=np.int32, dest='nr_feeds' +) +parser.add_argument( + '--block_size', action='store', + default=ExposureErrorCompensator.DEFAULT_BLOCK_SIZE, + help="BLock size in pixels used by the exposure compensator. " + "The default is '%s'." % ExposureErrorCompensator.DEFAULT_BLOCK_SIZE, + type=np.int32, dest='block_size' +) +parser.add_argument( + '--finder', action='store', default=SeamFinder.DEFAULT_SEAM_FINDER, + help="Seam estimation method. " + "The default is '%s'." % SeamFinder.DEFAULT_SEAM_FINDER, + choices=SeamFinder.SEAM_FINDER_CHOICES.keys(), + type=str, dest='finder' +) +parser.add_argument( + '--final_megapix', action='store', + default=ImageHandler.DEFAULT_FINAL_MEGAPIX, + help="Resolution for compositing step. Use -1 for original resolution. " + "The default is %s" % ImageHandler.DEFAULT_FINAL_MEGAPIX, + type=float, dest='final_megapix' +) +parser.add_argument( + '--blender_type', action='store', default=Blender.DEFAULT_BLENDER, + help="Blending method. The default is '%s'." % Blender.DEFAULT_BLENDER, + choices=Blender.BLENDER_CHOICES, + type=str, dest='blender_type' +) +parser.add_argument( + '--blend_strength', action='store', default=Blender.DEFAULT_BLEND_STRENGTH, + help="Blending strength from [0,100] range. " + "The default is '%s'." % Blender.DEFAULT_BLEND_STRENGTH, + type=np.int32, dest='blend_strength' +) +parser.add_argument( + '--timelapse', action='store', default=Timelapser.DEFAULT_TIMELAPSE, + help="Output warped images separately as frames of a time lapse movie, " + "with 'fixed_' prepended to input file names. " + "The default is '%s'." % Timelapser.DEFAULT_TIMELAPSE, + choices=Timelapser.TIMELAPSE_CHOICES, + type=str, dest='timelapse' +) +parser.add_argument( + '--output', action='store', default='result.jpg', + help="The default is 'result.jpg'", + type=str, dest='output' +) + +__doc__ += '\n' + parser.format_help() + +if __name__ == '__main__': + print(__doc__) + args = parser.parse_args() + args_dict = vars(args) + + # Extract In- and Output + img_names = args_dict.pop("img_names") + img_names = [cv.samples.findFile(img_name) for img_name in img_names] + output = args_dict.pop("output") + + stitcher = Stitcher(**args_dict) + panorama = stitcher.stitch(img_names) + + cv.imwrite(output, panorama) + + zoom_x = 600.0 / panorama.shape[1] + preview = cv.resize(panorama, dsize=None, fx=zoom_x, fy=zoom_x) + + cv.imshow(output, preview) + cv.waitKey() + cv.destroyAllWindows() From d2c1f1131bdc4fadd1a4ed3e9511b4ae4e752cf1 Mon Sep 17 00:00:00 2001 From: jcong Date: Tue, 9 Nov 2021 17:23:34 +0800 Subject: [PATCH 358/376] videoio: drop unnecessary offset for accessing video output buffer Fix: #21021 NDK API AMediaCodec_getOutputBuffer() returns MediaCodecBuffer::data() which is actually ABuffer::data(). The returned buffer address is already adjusted by offset. More info: ABuffer::base() returns base address without offset ABuffer::data() returns base + offset Change-Id: I2936339ce4fa9acf657a5a7d92adc1275d7b28a1 --- modules/videoio/src/cap_android_mediandk.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/videoio/src/cap_android_mediandk.cpp b/modules/videoio/src/cap_android_mediandk.cpp index 822024d53fc0..4fb4a82c2fae 100644 --- a/modules/videoio/src/cap_android_mediandk.cpp +++ b/modules/videoio/src/cap_android_mediandk.cpp @@ -92,7 +92,7 @@ class AndroidMediaNdkCapture : public IVideoCapture AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_HEIGHT, &frameHeight); AMediaFormat_getInt32(mediaFormat.get(), AMEDIAFORMAT_KEY_COLOR_FORMAT, &colorFormat); uint8_t* codecBuffer = AMediaCodec_getOutputBuffer(mediaCodec.get(), bufferIndex, &bufferSize); - buffer = std::vector(codecBuffer + info.offset, codecBuffer + bufferSize); + buffer = std::vector(codecBuffer, codecBuffer + bufferSize); LOGV("colorFormat: %d", colorFormat); LOGV("buffer size: %zu", bufferSize); LOGV("width (frame): %d", frameWidth); From 98b6ce353cc9342bee472e594eaf4b23136b0430 Mon Sep 17 00:00:00 2001 From: ZaKiiiiiiiii <56301098+Crayon-new@users.noreply.github.com> Date: Wed, 10 Nov 2021 00:24:04 +0800 Subject: [PATCH 359/376] Merge pull request #20904 from Crayon-new:fix_bug_in_maxLayer fix bug: wrong output dimension when "keep_dims" is false in pooling layer. * fix bug in max layer * code align * delete permute layer and add test case * add name assert * check other cases * remove c++11 features * style:add "const" remove assert * style:sanitize file names --- modules/dnn/src/tensorflow/tf_importer.cpp | 77 +++++++++++++++------- modules/dnn/test/test_tf_importer.cpp | 13 ++++ 2 files changed, 66 insertions(+), 24 deletions(-) diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 1a01ac87d677..8cbe1c4b2393 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -2142,6 +2142,7 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& const std::string& type = layer.op(); const int num_inputs = layer.input_size(); std::string pool_type = cv::toLowerCase(type); + DataLayout layout = getDataLayout(name, data_layouts); if (pool_type == "mean") { @@ -2205,6 +2206,16 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& if (!keepDims) { + if (layout == DATA_LAYOUT_NHWC) + { + LayerParams permLP; + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + std::string permName = name + "/nhwc"; + Pin inpId = Pin(layerShapeName); + addPermuteLayer(order, permName, inpId); + layerShapeName = permName; + } + LayerParams squeezeLp; std::string squeezeName = name + "/squeeze"; CV_Assert(layer_id.find(squeezeName) == layer_id.end()); @@ -2227,22 +2238,30 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& layerParams.set("pool", pool_type); layerParams.set(axis == 2 ? "kernel_w" : "kernel_h", 1); layerParams.set(axis == 2 ? "global_pooling_h" : "global_pooling_w", true); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - if (!keepDims) + if (keepDims) + { + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + } + else { // To keep correct order after squeeze dims we first need to change layout from NCHW to NHWC + std::string poolingName = name + "/Pooling"; + CV_Assert(layer_id.find(poolingName) == layer_id.end()); + int id = dstNet.addLayer(poolingName, "Pooling", layerParams); + layer_id[poolingName] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + LayerParams permLP; int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - std::string permName = name + "/nchw"; - Pin inpId = Pin(name); + std::string permName = name + "/nhwc"; + Pin inpId = Pin(poolingName); addPermuteLayer(order, permName, inpId); LayerParams squeezeLp; - std::string squeezeName = name + "/squeeze"; - CV_Assert(layer_id.find(squeezeName) == layer_id.end()); + const std::string& squeezeName = name; squeezeLp.set("axis", indices.at(0)); squeezeLp.set("end_axis", indices.at(0) + 1); int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); @@ -2254,32 +2273,34 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& { int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. Pin inpId = parsePin(layer.input(0)); - addPermuteLayer(order, name + "/nhwc", inpId); + std::string permName = name + "/nhwc"; + addPermuteLayer(order, permName, inpId); layerParams.set("pool", pool_type); layerParams.set("kernel_h", 1); layerParams.set("global_pooling_w", true); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, inpId, id, 0); + std::string poolingName = name + "/Pooling"; + CV_Assert(layer_id.find(poolingName) == layer_id.end()); + int id = dstNet.addLayer(poolingName, "Pooling", layerParams); + layer_id[poolingName] = id; + connect(layer_id, dstNet, Pin(permName), id, 0); if (!keepDims) { LayerParams squeezeLp; - std::string squeezeName = name + "/squeeze"; - CV_Assert(layer_id.find(squeezeName) == layer_id.end()); + const std::string& squeezeName = name; int channel_id = 3; // TF NHWC layout squeezeLp.set("axis", channel_id - 1); squeezeLp.set("end_axis", channel_id); int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); layer_id[squeezeName] = squeezeId; - connect(layer_id, dstNet, Pin(name), squeezeId, 0); + connect(layer_id, dstNet, Pin(poolingName), squeezeId, 0); } else { int order[] = {0, 3, 1, 2}; // From NHWC to OpenCV's NCHW. - Pin inpId = parsePin(name); - addPermuteLayer(order, name + "/nchw", inpId); + Pin inpId = parsePin(poolingName); + addPermuteLayer(order, name, inpId); } } } else { @@ -2288,18 +2309,26 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& layerParams.set("pool", pool_type); layerParams.set("global_pooling", true); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - if (!keepDims) + if (keepDims) { + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + } + else + { + std::string poolingName = name + "/Pooling"; + CV_Assert(layer_id.find(poolingName) == layer_id.end()); + int id = dstNet.addLayer(poolingName, "Pooling", layerParams); + layer_id[poolingName] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); LayerParams flattenLp; - std::string flattenName = name + "/flatten"; - CV_Assert(layer_id.find(flattenName) == layer_id.end()); + const std::string& flattenName = name; int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); layer_id[flattenName] = flattenId; - connect(layer_id, dstNet, Pin(name), flattenId, 0); + connect(layer_id, dstNet, Pin(poolingName), flattenId, 0); + data_layouts[name] = DATA_LAYOUT_PLANAR; } } } diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index cdf3794bf18c..b688c313837a 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -413,6 +413,19 @@ TEST_P(Test_TensorFlow_layers, pooling_reduce_sum) runTensorFlowNet("reduce_sum"); // a SUM pooling over all spatial dimensions. } +TEST_P(Test_TensorFlow_layers, pooling_reduce_sum2) +{ + int axises[] = {0, 1, 2, 3}; + for (int keepdims = 0; keepdims <= 1; ++keepdims) + { + for (int i = 0; i < sizeof(axises)/sizeof(axises[0]); ++i) + { + runTensorFlowNet(cv::format("reduce_sum_%d_%s", axises[i], (keepdims ? "True" : "False"))); + } + runTensorFlowNet(cv::format("reduce_sum_1_2_%s", keepdims ? "True" : "False")); + } +} + TEST_P(Test_TensorFlow_layers, max_pool_grad) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) From 0c0a8392e5f97224676c312207d181a03e77c083 Mon Sep 17 00:00:00 2001 From: AleksandrPanov Date: Wed, 10 Nov 2021 11:14:06 +0300 Subject: [PATCH 360/376] fix markers parse in gen_pattern.py --- doc/pattern_tools/gen_pattern.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/doc/pattern_tools/gen_pattern.py b/doc/pattern_tools/gen_pattern.py index 83ed115d361c..4618bc318b54 100755 --- a/doc/pattern_tools/gen_pattern.py +++ b/doc/pattern_tools/gen_pattern.py @@ -171,7 +171,7 @@ def main(): parser.add_argument("-m", "--markers", help="list of cells with markers for the radon checkerboard. Marker " "coordinates as list of numbers: -m 1 2 3 4 means markers in cells " "[1, 2] and [3, 4]", - action="store", dest="markers", nargs="+", type=int) + default=argparse.SUPPRESS, action="store", dest="markers", nargs="+", type=int) args = parser.parse_args() show_help = args.show_help @@ -195,14 +195,16 @@ def main(): "A5": [148, 210]} page_width = page_sizes[page_size][0] page_height = page_sizes[page_size][1] - if len(args.markers) % 2 == 1: - raise ValueError("The length of the markers array={} must be even".format(len(args.markers))) - markers = set() - for x, y in zip(args.markers[::2], args.markers[1::2]): - if x in range(0, columns) and y in range(0, rows): - markers.add((x, y)) - else: - raise ValueError("The marker {},{} is outside the checkerboard".format(x, y)) + markers = None + if p_type == "radon_checkerboard" and "markers" in args: + if len(args.markers) % 2 == 1: + raise ValueError("The length of the markers array={} must be even".format(len(args.markers))) + markers = set() + for x, y in zip(args.markers[::2], args.markers[1::2]): + if x in range(0, columns) and y in range(0, rows): + markers.add((x, y)) + else: + raise ValueError("The marker {},{} is outside the checkerboard".format(x, y)) pm = PatternMaker(columns, rows, output, units, square_size, radius_rate, page_width, page_height, markers) # dict for easy lookup of pattern type From d934bb15b0c44b0ac50f2b2556393d9f4f28a620 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 10 Nov 2021 15:03:45 +0300 Subject: [PATCH 361/376] Merge pull request #20998 from alalek:update_protobuf_3.19.1 3rdparty(protobuf): upgrade 3.5.2 => 3.19.1 * 3rdparty(protobuf): upgrade 3.5.2 => 3.19.1 * dnn: update protobuf files (3.19.1) * 3rdparty(protobuf): re-apply OpenCV patch for custom fields (3.19.1) * protobuf: suppress new build warnings * protobuf: remove unused files --- 3rdparty/protobuf/CMakeLists.txt | 106 +- 3rdparty/protobuf/LICENSE | 12 +- 3rdparty/protobuf/README.md | 2 +- 3rdparty/protobuf/src/google/protobuf/any.cc | 87 +- 3rdparty/protobuf/src/google/protobuf/any.h | 83 +- .../protobuf/src/google/protobuf/any.pb.cc | 440 - .../protobuf/src/google/protobuf/any.pb.h | 323 - .../protobuf/src/google/protobuf/any_lite.cc | 99 + .../protobuf/src/google/protobuf/api.pb.cc | 1608 - .../protobuf/src/google/protobuf/api.pb.h | 1165 - .../protobuf/src/google/protobuf/arena.cc | 648 +- 3rdparty/protobuf/src/google/protobuf/arena.h | 1047 +- .../protobuf/src/google/protobuf/arena_impl.h | 594 +- .../src/google/protobuf/arenastring.cc | 249 +- .../src/google/protobuf/arenastring.h | 582 +- .../src/google/protobuf/descriptor.cc | 5168 +- .../protobuf/src/google/protobuf/descriptor.h | 1168 +- .../src/google/protobuf/descriptor.pb.cc | 16520 ++--- .../src/google/protobuf/descriptor.pb.h | 16753 +++-- .../google/protobuf/descriptor_database.cc | 792 +- .../src/google/protobuf/descriptor_database.h | 190 +- .../src/google/protobuf/duration.pb.cc | 431 - .../src/google/protobuf/duration.pb.h | 232 - .../src/google/protobuf/dynamic_message.cc | 773 +- .../src/google/protobuf/dynamic_message.h | 95 +- .../protobuf/src/google/protobuf/empty.pb.cc | 342 - .../protobuf/src/google/protobuf/empty.pb.h | 190 - .../time.h => explicitly_constructed.h} | 78 +- .../src/google/protobuf/extension_set.cc | 2130 +- .../src/google/protobuf/extension_set.h | 1347 +- .../google/protobuf/extension_set_heavy.cc | 656 +- .../src/google/protobuf/extension_set_inl.h | 276 + .../google/protobuf/field_access_listener.h | 172 + .../src/google/protobuf/field_mask.pb.cc | 373 - .../src/google/protobuf/field_mask.pb.h | 287 +- .../protobuf/generated_enum_reflection.h | 36 +- .../src/google/protobuf/generated_enum_util.h | 43 +- .../google/protobuf/generated_message_bases.h | 87 + .../protobuf/generated_message_reflection.cc | 2826 +- .../protobuf/generated_message_reflection.h | 712 +- .../generated_message_table_driven.cc | 103 - .../protobuf/generated_message_table_driven.h | 240 +- .../generated_message_table_driven_lite.cc | 109 - .../generated_message_table_driven_lite.h | 823 - .../protobuf/generated_message_tctable_decl.h | 139 + .../protobuf/generated_message_tctable_impl.h | 302 + .../generated_message_tctable_impl.inc | 92 + .../google/protobuf/generated_message_util.cc | 327 +- .../google/protobuf/generated_message_util.h | 331 +- .../protobuf/src/google/protobuf/has_bits.h | 29 +- .../statusor.cc => implicit_weak_message.cc} | 33 +- .../google/protobuf/implicit_weak_message.h | 186 + .../google/protobuf/inlined_string_field.h | 384 + .../src/google/protobuf/io/coded_stream.cc | 679 +- .../src/google/protobuf/io/coded_stream.h | 1341 +- .../src/google/protobuf/io/coded_stream_inl.h | 90 - .../src/google/protobuf/io/gzip_stream.cc | 330 - .../src/google/protobuf/io/gzip_stream.h | 209 - .../google/protobuf/{stubs => io}/io_win32.cc | 107 +- .../google/protobuf/{stubs => io}/io_win32.h | 78 +- .../src/google/protobuf/io/package_info.h | 1 - .../src/google/protobuf/io/printer.cc | 372 - .../protobuf/src/google/protobuf/io/printer.h | 363 - .../protobuf/src/google/protobuf/io/strtod.cc | 91 +- .../protobuf/src/google/protobuf/io/strtod.h | 2 +- .../src/google/protobuf/io/tokenizer.cc | 374 +- .../src/google/protobuf/io/tokenizer.h | 113 +- .../src/google/protobuf/io/zero_copy_stream.h | 17 +- .../protobuf/io/zero_copy_stream_impl.cc | 206 +- .../protobuf/io/zero_copy_stream_impl.h | 130 +- .../protobuf/io/zero_copy_stream_impl_lite.cc | 228 +- .../protobuf/io/zero_copy_stream_impl_lite.h | 148 +- .../internal/error_listener.cc => map.cc} | 9 +- 3rdparty/protobuf/src/google/protobuf/map.h | 1012 +- .../protobuf/src/google/protobuf/map_entry.h | 86 +- .../src/google/protobuf/map_entry_lite.h | 532 +- .../protobuf/src/google/protobuf/map_field.cc | 523 +- .../protobuf/src/google/protobuf/map_field.h | 981 +- .../src/google/protobuf/map_field_inl.h | 234 +- .../src/google/protobuf/map_field_lite.h | 107 +- .../src/google/protobuf/map_type_handler.h | 750 +- .../protobuf/src/google/protobuf/message.cc | 421 +- .../protobuf/src/google/protobuf/message.h | 1279 +- .../src/google/protobuf/message_lite.cc | 552 +- .../src/google/protobuf/message_lite.h | 431 +- .../protobuf/src/google/protobuf/metadata.h | 44 +- .../src/google/protobuf/metadata_lite.h | 242 +- .../src/google/protobuf/package_info.h | 64 - .../src/google/protobuf/parse_context.cc | 559 + .../src/google/protobuf/parse_context.h | 938 + .../google/protobuf/{service.cc => port.h} | 20 +- .../protobuf/src/google/protobuf/port_def.inc | 824 + .../src/google/protobuf/port_undef.inc | 145 + .../protobuf/src/google/protobuf/reflection.h | 317 +- .../src/google/protobuf/reflection_internal.h | 182 +- .../src/google/protobuf/reflection_ops.cc | 337 +- .../src/google/protobuf/reflection_ops.h | 18 +- .../src/google/protobuf/repeated_field.cc | 78 +- .../src/google/protobuf/repeated_field.h | 2872 +- .../src/google/protobuf/repeated_ptr_field.cc | 157 + .../src/google/protobuf/repeated_ptr_field.h | 2014 + .../protobuf/src/google/protobuf/service.h | 292 - .../src/google/protobuf/source_context.pb.cc | 381 - .../src/google/protobuf/source_context.pb.h | 243 - .../protobuf/src/google/protobuf/struct.pb.cc | 1475 - .../protobuf/src/google/protobuf/struct.pb.h | 1030 - .../protobuf/stubs/atomic_sequence_num.h | 54 - .../src/google/protobuf/stubs/atomicops.h | 237 - .../stubs/atomicops_internals_arm64_gcc.h | 325 - .../stubs/atomicops_internals_arm_gcc.h | 151 - .../stubs/atomicops_internals_arm_qnx.h | 146 - .../atomicops_internals_generic_c11_atomic.h | 231 - .../stubs/atomicops_internals_generic_gcc.h | 163 - .../stubs/atomicops_internals_mips_gcc.h | 313 - .../stubs/atomicops_internals_power.h | 440 - .../stubs/atomicops_internals_ppc_gcc.h | 155 - .../stubs/atomicops_internals_solaris.h | 188 - .../protobuf/stubs/atomicops_internals_tsan.h | 219 - .../stubs/atomicops_internals_x86_gcc.cc | 137 - .../stubs/atomicops_internals_x86_gcc.h | 293 - .../stubs/atomicops_internals_x86_msvc.cc | 113 - .../stubs/atomicops_internals_x86_msvc.h | 150 - .../src/google/protobuf/stubs/bytestream.cc | 12 +- .../src/google/protobuf/stubs/bytestream.h | 53 +- .../src/google/protobuf/stubs/callback.h | 99 +- .../src/google/protobuf/stubs/casts.h | 23 +- .../src/google/protobuf/stubs/common.cc | 261 +- .../src/google/protobuf/stubs/common.h | 101 +- .../src/google/protobuf/stubs/fastmem.h | 153 - .../protobuf/src/google/protobuf/stubs/hash.h | 350 +- .../src/google/protobuf/stubs/int128.cc | 75 +- .../src/google/protobuf/stubs/int128.h | 16 +- .../src/google/protobuf/stubs/logging.h | 38 +- .../src/google/protobuf/stubs/macros.h | 87 +- .../src/google/protobuf/stubs/map_util.h | 47 +- .../src/google/protobuf/stubs/mathlimits.cc | 144 - .../src/google/protobuf/stubs/mathlimits.h | 303 - .../src/google/protobuf/stubs/mathutil.h | 141 - .../src/google/protobuf/stubs/mutex.h | 134 +- .../src/google/protobuf/stubs/once.cc | 99 - .../protobuf/src/google/protobuf/stubs/once.h | 136 +- .../google/protobuf/stubs/platform_macros.h | 12 +- .../protobuf/src/google/protobuf/stubs/port.h | 283 +- .../src/google/protobuf/stubs/scoped_ptr.h | 236 - .../src/google/protobuf/stubs/shared_ptr.h | 471 - .../src/google/protobuf/stubs/singleton.h | 68 - .../src/google/protobuf/stubs/status.cc | 194 +- .../src/google/protobuf/stubs/status.h | 164 +- .../src/google/protobuf/stubs/status_macros.h | 89 - .../src/google/protobuf/stubs/statusor.h | 259 - .../src/google/protobuf/stubs/stl_util.h | 70 +- .../src/google/protobuf/stubs/stringpiece.cc | 102 +- .../src/google/protobuf/stubs/stringpiece.h | 285 +- .../src/google/protobuf/stubs/stringprintf.cc | 27 +- .../src/google/protobuf/stubs/stringprintf.h | 21 +- .../protobuf/stubs/structurally_valid.cc | 45 +- .../src/google/protobuf/stubs/strutil.cc | 665 +- .../src/google/protobuf/stubs/strutil.h | 414 +- .../src/google/protobuf/stubs/substitute.cc | 38 +- .../src/google/protobuf/stubs/substitute.h | 66 +- .../src/google/protobuf/stubs/template_util.h | 138 - .../src/google/protobuf/stubs/time.cc | 365 - .../src/google/protobuf/stubs/type_traits.h | 364 - .../src/google/protobuf/text_format.cc | 1716 +- .../src/google/protobuf/text_format.h | 408 +- .../src/google/protobuf/timestamp.pb.cc | 431 - .../src/google/protobuf/timestamp.pb.h | 232 - .../protobuf/src/google/protobuf/type.pb.cc | 2844 - .../protobuf/src/google/protobuf/type.pb.h | 2383 - .../src/google/protobuf/unknown_field_set.cc | 238 +- .../src/google/protobuf/unknown_field_set.h | 216 +- .../protobuf/util/delimited_message_util.cc | 79 - .../protobuf/util/delimited_message_util.h | 66 - .../google/protobuf/util/field_comparator.cc | 208 - .../google/protobuf/util/field_comparator.h | 259 - .../google/protobuf/util/field_mask_util.cc | 690 - .../google/protobuf/util/field_mask_util.h | 236 - .../google/protobuf/util/internal/constants.h | 103 - .../protobuf/util/internal/datapiece.cc | 406 - .../google/protobuf/util/internal/datapiece.h | 219 - .../internal/default_value_objectwriter.cc | 647 - .../internal/default_value_objectwriter.h | 324 - .../protobuf/util/internal/error_listener.h | 103 - .../util/internal/expecting_objectwriter.h | 238 - .../util/internal/field_mask_utility.cc | 220 - .../util/internal/field_mask_utility.h | 72 - .../protobuf/util/internal/json_escaping.cc | 356 - .../protobuf/util/internal/json_escaping.h | 91 - .../util/internal/json_objectwriter.cc | 196 - .../util/internal/json_objectwriter.h | 233 - .../util/internal/json_stream_parser.cc | 864 - .../util/internal/json_stream_parser.h | 272 - .../protobuf/util/internal/location_tracker.h | 65 - .../util/internal/mock_error_listener.h | 63 - .../util/internal/object_location_tracker.h | 64 - .../protobuf/util/internal/object_source.h | 79 - .../protobuf/util/internal/object_writer.cc | 92 - .../protobuf/util/internal/object_writer.h | 146 - .../protobuf/util/internal/proto_writer.cc | 799 - .../protobuf/util/internal/proto_writer.h | 353 - .../util/internal/protostream_objectsource.cc | 1136 - .../util/internal/protostream_objectsource.h | 326 - .../util/internal/protostream_objectwriter.cc | 1274 - .../util/internal/protostream_objectwriter.h | 408 - .../util/internal/structured_objectwriter.h | 118 - .../protobuf/util/internal/type_info.cc | 179 - .../google/protobuf/util/internal/type_info.h | 92 - .../google/protobuf/util/internal/utility.cc | 409 - .../google/protobuf/util/internal/utility.h | 208 - .../src/google/protobuf/util/json_util.cc | 252 - .../src/google/protobuf/util/json_util.h | 200 - .../protobuf/util/message_differencer.cc | 1756 - .../protobuf/util/message_differencer.h | 879 - .../src/google/protobuf/util/package_info.h | 46 - .../src/google/protobuf/util/time_util.cc | 504 - .../src/google/protobuf/util/time_util.h | 296 - .../src/google/protobuf/util/type_resolver.h | 77 - .../protobuf/util/type_resolver_util.cc | 247 - .../google/protobuf/util/type_resolver_util.h | 54 - .../src/google/protobuf/wire_format.cc | 1283 +- .../src/google/protobuf/wire_format.h | 235 +- .../src/google/protobuf/wire_format_lite.cc | 590 +- .../src/google/protobuf/wire_format_lite.h | 1867 +- .../google/protobuf/wire_format_lite_inl.h | 1077 - .../src/google/protobuf/wrappers.pb.cc | 2796 - .../src/google/protobuf/wrappers.pb.h | 1487 - modules/dnn/CMakeLists.txt | 7 + modules/dnn/misc/caffe/opencv-caffe.pb.cc | 54110 +++++++--------- modules/dnn/misc/caffe/opencv-caffe.pb.h | 43244 +++++++----- modules/dnn/misc/onnx/opencv-onnx.pb.cc | 8359 ++- modules/dnn/misc/onnx/opencv-onnx.pb.h | 7839 ++- modules/dnn/misc/tensorflow/attr_value.pb.cc | 2217 +- modules/dnn/misc/tensorflow/attr_value.pb.h | 1767 +- modules/dnn/misc/tensorflow/function.pb.cc | 2151 +- modules/dnn/misc/tensorflow/function.pb.h | 1527 +- modules/dnn/misc/tensorflow/graph.pb.cc | 1503 +- modules/dnn/misc/tensorflow/graph.pb.h | 1088 +- modules/dnn/misc/tensorflow/op_def.pb.cc | 3412 +- modules/dnn/misc/tensorflow/op_def.pb.h | 2764 +- modules/dnn/misc/tensorflow/tensor.pb.cc | 1202 +- modules/dnn/misc/tensorflow/tensor.pb.h | 956 +- .../dnn/misc/tensorflow/tensor_shape.pb.cc | 861 +- modules/dnn/misc/tensorflow/tensor_shape.pb.h | 550 +- modules/dnn/misc/tensorflow/types.pb.cc | 120 +- modules/dnn/misc/tensorflow/types.pb.h | 95 +- modules/dnn/misc/tensorflow/versions.pb.cc | 530 +- modules/dnn/misc/tensorflow/versions.pb.h | 309 +- 247 files changed, 117540 insertions(+), 146689 deletions(-) delete mode 100644 3rdparty/protobuf/src/google/protobuf/any.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/any.pb.h create mode 100644 3rdparty/protobuf/src/google/protobuf/any_lite.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/api.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/api.pb.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/duration.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/duration.pb.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/empty.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/empty.pb.h rename 3rdparty/protobuf/src/google/protobuf/{stubs/time.h => explicitly_constructed.h} (52%) create mode 100644 3rdparty/protobuf/src/google/protobuf/extension_set_inl.h create mode 100644 3rdparty/protobuf/src/google/protobuf/field_access_listener.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/field_mask.pb.cc create mode 100644 3rdparty/protobuf/src/google/protobuf/generated_message_bases.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/generated_message_table_driven.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/generated_message_table_driven_lite.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/generated_message_table_driven_lite.h create mode 100644 3rdparty/protobuf/src/google/protobuf/generated_message_tctable_decl.h create mode 100644 3rdparty/protobuf/src/google/protobuf/generated_message_tctable_impl.h create mode 100644 3rdparty/protobuf/src/google/protobuf/generated_message_tctable_impl.inc rename 3rdparty/protobuf/src/google/protobuf/{stubs/statusor.cc => implicit_weak_message.cc} (64%) create mode 100644 3rdparty/protobuf/src/google/protobuf/implicit_weak_message.h create mode 100644 3rdparty/protobuf/src/google/protobuf/inlined_string_field.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/io/coded_stream_inl.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/io/gzip_stream.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/io/gzip_stream.h rename 3rdparty/protobuf/src/google/protobuf/{stubs => io}/io_win32.cc (79%) rename 3rdparty/protobuf/src/google/protobuf/{stubs => io}/io_win32.h (58%) delete mode 100644 3rdparty/protobuf/src/google/protobuf/io/printer.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/io/printer.h rename 3rdparty/protobuf/src/google/protobuf/{util/internal/error_listener.cc => map.cc} (92%) delete mode 100644 3rdparty/protobuf/src/google/protobuf/package_info.h create mode 100644 3rdparty/protobuf/src/google/protobuf/parse_context.cc create mode 100644 3rdparty/protobuf/src/google/protobuf/parse_context.h rename 3rdparty/protobuf/src/google/protobuf/{service.cc => port.h} (82%) create mode 100644 3rdparty/protobuf/src/google/protobuf/port_def.inc create mode 100644 3rdparty/protobuf/src/google/protobuf/port_undef.inc create mode 100644 3rdparty/protobuf/src/google/protobuf/repeated_ptr_field.cc create mode 100644 3rdparty/protobuf/src/google/protobuf/repeated_ptr_field.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/service.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/source_context.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/source_context.pb.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/struct.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/struct.pb.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomic_sequence_num.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_arm64_gcc.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_arm_gcc.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_arm_qnx.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_generic_c11_atomic.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_generic_gcc.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_mips_gcc.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_power.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_ppc_gcc.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_solaris.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_tsan.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_x86_gcc.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_x86_gcc.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_x86_msvc.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/atomicops_internals_x86_msvc.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/fastmem.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/mathlimits.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/mathlimits.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/mathutil.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/once.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/scoped_ptr.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/shared_ptr.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/singleton.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/status_macros.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/statusor.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/template_util.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/time.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/stubs/type_traits.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/timestamp.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/timestamp.pb.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/type.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/type.pb.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/delimited_message_util.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/delimited_message_util.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/field_comparator.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/field_comparator.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/field_mask_util.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/field_mask_util.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/constants.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/datapiece.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/datapiece.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/default_value_objectwriter.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/default_value_objectwriter.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/error_listener.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/expecting_objectwriter.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/field_mask_utility.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/field_mask_utility.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/json_escaping.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/json_escaping.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/json_objectwriter.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/json_objectwriter.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/json_stream_parser.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/json_stream_parser.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/location_tracker.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/mock_error_listener.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/object_location_tracker.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/object_source.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/object_writer.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/object_writer.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/proto_writer.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/proto_writer.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/protostream_objectsource.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/protostream_objectsource.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/protostream_objectwriter.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/protostream_objectwriter.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/structured_objectwriter.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/type_info.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/type_info.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/utility.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/internal/utility.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/json_util.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/json_util.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/message_differencer.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/message_differencer.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/package_info.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/time_util.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/time_util.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/type_resolver.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/type_resolver_util.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/util/type_resolver_util.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/wire_format_lite_inl.h delete mode 100644 3rdparty/protobuf/src/google/protobuf/wrappers.pb.cc delete mode 100644 3rdparty/protobuf/src/google/protobuf/wrappers.pb.h diff --git a/3rdparty/protobuf/CMakeLists.txt b/3rdparty/protobuf/CMakeLists.txt index f249d2dcc39f..6de8148e58c1 100644 --- a/3rdparty/protobuf/CMakeLists.txt +++ b/3rdparty/protobuf/CMakeLists.txt @@ -13,6 +13,9 @@ if(MSVC) /wd4701 /wd4703 # potentially uninitialized local/pointer variable 'value' used /wd4505 # unreferenced local function has been removed ) + if(MSVC_VERSION LESS 1910) # MSVS 2015 + ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4309) # 'static_cast': truncation of constant value + endif() else() #NOTE: -Wno-invalid-offsetof was used as solution for invalid offset warning on protobuf #3450 ocv_warnings_disable(CMAKE_CXX_FLAGS -Wno-deprecated -Wmissing-prototypes -Wmissing-declarations -Wshadow @@ -49,98 +52,101 @@ endfunction() set(PROTOBUF_ROOT "${CMAKE_CURRENT_LIST_DIR}") -if(MSVC) - set(ATOMICOPS_INTERNALS ${PROTOBUF_ROOT}/src/google/protobuf/stubs/atomicops_internals_x86_msvc.cc) -else() - set(ATOMICOPS_INTERNALS ${PROTOBUF_ROOT}/src/google/protobuf/stubs/atomicops_internals_x86_gcc.cc) -endif() - append_if_exist(Protobuf_SRCS # libprotobuf-lite + ${PROTOBUF_ROOT}/src/google/protobuf/any_lite.cc ${PROTOBUF_ROOT}/src/google/protobuf/arena.cc ${PROTOBUF_ROOT}/src/google/protobuf/arenastring.cc ${PROTOBUF_ROOT}/src/google/protobuf/extension_set.cc - ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_table_driven_lite.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/generated_enum_util.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_table_driven_lite.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_tctable_lite.cc ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_util.cc + ${PROTOBUF_ROOT}/src/google/protobuf/implicit_weak_message.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/inlined_string_field.cc ${PROTOBUF_ROOT}/src/google/protobuf/io/coded_stream.cc + ${PROTOBUF_ROOT}/src/google/protobuf/io/io_win32.cc + ${PROTOBUF_ROOT}/src/google/protobuf/io/strtod.cc ${PROTOBUF_ROOT}/src/google/protobuf/io/zero_copy_stream.cc + ${PROTOBUF_ROOT}/src/google/protobuf/io/zero_copy_stream_impl.cc ${PROTOBUF_ROOT}/src/google/protobuf/io/zero_copy_stream_impl_lite.cc + ${PROTOBUF_ROOT}/src/google/protobuf/map.cc ${PROTOBUF_ROOT}/src/google/protobuf/message_lite.cc + ${PROTOBUF_ROOT}/src/google/protobuf/parse_context.cc ${PROTOBUF_ROOT}/src/google/protobuf/repeated_field.cc - ${ATOMICOPS_INTERNALS} + ${PROTOBUF_ROOT}/src/google/protobuf/repeated_ptr_field.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/bytestream.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/common.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/int128.cc - ${PROTOBUF_ROOT}/src/google/protobuf/stubs/io_win32.cc - ${PROTOBUF_ROOT}/src/google/protobuf/stubs/once.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/status.cc - ${PROTOBUF_ROOT}/src/google/protobuf/stubs/statusor.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/stubs/statusor.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/stringpiece.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/stringprintf.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/structurally_valid.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/strutil.cc - ${PROTOBUF_ROOT}/src/google/protobuf/stubs/time.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/stubs/time.cc ${PROTOBUF_ROOT}/src/google/protobuf/wire_format_lite.cc + # libprotobuf ${PROTOBUF_ROOT}/src/google/protobuf/any.cc - ${PROTOBUF_ROOT}/src/google/protobuf/any.pb.cc - ${PROTOBUF_ROOT}/src/google/protobuf/api.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/any.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/api.pb.cc # ${PROTOBUF_ROOT}/src/google/protobuf/compiler/importer.cc # ${PROTOBUF_ROOT}/src/google/protobuf/compiler/parser.cc ${PROTOBUF_ROOT}/src/google/protobuf/descriptor.cc ${PROTOBUF_ROOT}/src/google/protobuf/descriptor.pb.cc ${PROTOBUF_ROOT}/src/google/protobuf/descriptor_database.cc - ${PROTOBUF_ROOT}/src/google/protobuf/duration.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/duration.pb.cc ${PROTOBUF_ROOT}/src/google/protobuf/dynamic_message.cc - ${PROTOBUF_ROOT}/src/google/protobuf/empty.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/empty.pb.cc ${PROTOBUF_ROOT}/src/google/protobuf/extension_set_heavy.cc - ${PROTOBUF_ROOT}/src/google/protobuf/field_mask.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/field_mask.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_bases.cc ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_reflection.cc - ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_table_driven.cc - ${PROTOBUF_ROOT}/src/google/protobuf/io/gzip_stream.cc - ${PROTOBUF_ROOT}/src/google/protobuf/io/printer.cc - ${PROTOBUF_ROOT}/src/google/protobuf/io/strtod.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_table_driven.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/generated_message_tctable_full.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/io/gzip_stream.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/io/printer.cc ${PROTOBUF_ROOT}/src/google/protobuf/io/tokenizer.cc - ${PROTOBUF_ROOT}/src/google/protobuf/io/zero_copy_stream_impl.cc ${PROTOBUF_ROOT}/src/google/protobuf/map_field.cc ${PROTOBUF_ROOT}/src/google/protobuf/message.cc ${PROTOBUF_ROOT}/src/google/protobuf/reflection_ops.cc - ${PROTOBUF_ROOT}/src/google/protobuf/service.cc - ${PROTOBUF_ROOT}/src/google/protobuf/source_context.pb.cc - ${PROTOBUF_ROOT}/src/google/protobuf/struct.pb.cc - ${PROTOBUF_ROOT}/src/google/protobuf/stubs/mathlimits.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/service.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/source_context.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/struct.pb.cc ${PROTOBUF_ROOT}/src/google/protobuf/stubs/substitute.cc ${PROTOBUF_ROOT}/src/google/protobuf/text_format.cc - ${PROTOBUF_ROOT}/src/google/protobuf/timestamp.pb.cc - ${PROTOBUF_ROOT}/src/google/protobuf/type.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/timestamp.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/type.pb.cc ${PROTOBUF_ROOT}/src/google/protobuf/unknown_field_set.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/delimited_message_util.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/field_comparator.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/field_mask_util.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/datapiece.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/default_value_objectwriter.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/delimited_message_util.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/field_comparator.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/field_mask_util.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/datapiece.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/default_value_objectwriter.cc # ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/error_listener.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/field_mask_utility.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_escaping.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_objectwriter.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_stream_parser.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/object_writer.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/proto_writer.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/protostream_objectsource.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/protostream_objectwriter.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/type_info.cc -# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/type_info_test_helper.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/utility.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/json_util.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/message_differencer.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/time_util.cc - ${PROTOBUF_ROOT}/src/google/protobuf/util/type_resolver_util.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/field_mask_utility.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_escaping.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_objectwriter.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/json_stream_parser.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/object_writer.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/proto_writer.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/protostream_objectsource.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/protostream_objectwriter.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/type_info.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/internal/utility.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/json_util.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/message_differencer.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/time_util.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/util/type_resolver_util.cc ${PROTOBUF_ROOT}/src/google/protobuf/wire_format.cc - ${PROTOBUF_ROOT}/src/google/protobuf/wrappers.pb.cc +# ${PROTOBUF_ROOT}/src/google/protobuf/wrappers.pb.cc ) -include_directories(BEFORE "${PROTOBUF_ROOT}/src") # ensure using if own headers: https://github.com/opencv/opencv/issues/13328 + +include_directories(BEFORE "${PROTOBUF_ROOT}/src") # ensure using of own headers: https://github.com/opencv/opencv/issues/13328 + add_library(libprotobuf STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${Protobuf_SRCS}) target_include_directories(libprotobuf SYSTEM PUBLIC $) set_target_properties(libprotobuf diff --git a/3rdparty/protobuf/LICENSE b/3rdparty/protobuf/LICENSE index f028c8232420..19b305b00060 100644 --- a/3rdparty/protobuf/LICENSE +++ b/3rdparty/protobuf/LICENSE @@ -1,14 +1,4 @@ -This license applies to all parts of Protocol Buffers except the following: - - - Atomicops support for generic gcc, located in - src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. - This file is copyrighted by Red Hat Inc. - - - Atomicops support for AIX/POWER, located in - src/google/protobuf/stubs/atomicops_internals_power.h. - This file is copyrighted by Bloomberg Finance LP. - -Copyright 2014, Google Inc. All rights reserved. +Copyright 2008 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/3rdparty/protobuf/README.md b/3rdparty/protobuf/README.md index 30404036a8ec..32aa03b64c98 100644 --- a/3rdparty/protobuf/README.md +++ b/3rdparty/protobuf/README.md @@ -1,3 +1,3 @@ Project: Protocol Buffers - Google's data interchange format Source code: https://github.com/protocolbuffers/protobuf -Version: 3.5.2 +Version: 3.19.1 diff --git a/3rdparty/protobuf/src/google/protobuf/any.cc b/3rdparty/protobuf/src/google/protobuf/any.cc index 83edba5788ad..73c002f60409 100644 --- a/3rdparty/protobuf/src/google/protobuf/any.cc +++ b/3rdparty/protobuf/src/google/protobuf/any.cc @@ -30,85 +30,54 @@ #include +#include +#include #include +#include + +#include namespace google { namespace protobuf { namespace internal { -namespace { -string GetTypeUrl(const Descriptor* message, - const string& type_url_prefix) { - if (!type_url_prefix.empty() && - type_url_prefix[type_url_prefix.size() - 1] == '/') { - return type_url_prefix + message->full_name(); - } else { - return type_url_prefix + "/" + message->full_name(); - } -} -} // namespace - -const char kAnyFullTypeName[] = "google.protobuf.Any"; -const char kTypeGoogleApisComPrefix[] = "type.googleapis.com/"; -const char kTypeGoogleProdComPrefix[] = "type.googleprod.com/"; - -AnyMetadata::AnyMetadata(UrlType* type_url, ValueType* value) - : type_url_(type_url), value_(value) { +bool AnyMetadata::PackFrom(Arena* arena, const Message& message) { + return PackFrom(arena, message, kTypeGoogleApisComPrefix); } -void AnyMetadata::PackFrom(const Message& message) { - PackFrom(message, kTypeGoogleApisComPrefix); -} - -void AnyMetadata::PackFrom(const Message& message, - const string& type_url_prefix) { - type_url_->SetNoArena(&::google::protobuf::internal::GetEmptyString(), - GetTypeUrl(message.GetDescriptor(), type_url_prefix)); - message.SerializeToString(value_->MutableNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited())); +bool AnyMetadata::PackFrom(Arena* arena, const Message& message, + StringPiece type_url_prefix) { + type_url_->Set( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString(), + GetTypeUrl(message.GetDescriptor()->full_name(), type_url_prefix), arena); + return message.SerializeToString( + value_->Mutable(ArenaStringPtr::EmptyDefault{}, arena)); } bool AnyMetadata::UnpackTo(Message* message) const { - if (!InternalIs(message->GetDescriptor())) { + if (!InternalIs(message->GetDescriptor()->full_name())) { return false; } - return message->ParseFromString(value_->GetNoArena()); + return message->ParseFromString(value_->Get()); } -bool AnyMetadata::InternalIs(const Descriptor* descriptor) const { - const string type_url = type_url_->GetNoArena(); - string full_name; - if (!ParseAnyTypeUrl(type_url, &full_name)) { - return false; - } - return full_name == descriptor->full_name(); -} - -bool ParseAnyTypeUrl(const string& type_url, string* full_type_name) { - size_t pos = type_url.find_last_of("/"); - if (pos == string::npos || pos + 1 == type_url.size()) { - return false; - } - *full_type_name = type_url.substr(pos + 1); - return true; -} - - bool GetAnyFieldDescriptors(const Message& message, const FieldDescriptor** type_url_field, const FieldDescriptor** value_field) { - const Descriptor* descriptor = message.GetDescriptor(); - if (descriptor->full_name() != kAnyFullTypeName) { - return false; - } - *type_url_field = descriptor->FindFieldByNumber(1); - *value_field = descriptor->FindFieldByNumber(2); - return (*type_url_field != NULL && - (*type_url_field)->type() == FieldDescriptor::TYPE_STRING && - *value_field != NULL && - (*value_field)->type() == FieldDescriptor::TYPE_BYTES); + const Descriptor* descriptor = message.GetDescriptor(); + if (descriptor->full_name() != kAnyFullTypeName) { + return false; + } + *type_url_field = descriptor->FindFieldByNumber(1); + *value_field = descriptor->FindFieldByNumber(2); + return (*type_url_field != nullptr && + (*type_url_field)->type() == FieldDescriptor::TYPE_STRING && + *value_field != nullptr && + (*value_field)->type() == FieldDescriptor::TYPE_BYTES); } } // namespace internal } // namespace protobuf } // namespace google + +#include diff --git a/3rdparty/protobuf/src/google/protobuf/any.h b/3rdparty/protobuf/src/google/protobuf/any.h index c2c27ec33381..e8336fa14a37 100644 --- a/3rdparty/protobuf/src/google/protobuf/any.h +++ b/3rdparty/protobuf/src/google/protobuf/any.h @@ -34,49 +34,89 @@ #include #include -#include -#include #include +#include + +#include namespace google { namespace protobuf { + +class FieldDescriptor; +class Message; + namespace internal { +extern const char kAnyFullTypeName[]; // "google.protobuf.Any". +extern const char kTypeGoogleApisComPrefix[]; // "type.googleapis.com/". +extern const char kTypeGoogleProdComPrefix[]; // "type.googleprod.com/". + +std::string GetTypeUrl(StringPiece message_name, + StringPiece type_url_prefix); + // Helper class used to implement google::protobuf::Any. -class LIBPROTOBUF_EXPORT AnyMetadata { +class PROTOBUF_EXPORT AnyMetadata { typedef ArenaStringPtr UrlType; typedef ArenaStringPtr ValueType; public: // AnyMetadata does not take ownership of "type_url" and "value". - AnyMetadata(UrlType* type_url, ValueType* value); + constexpr AnyMetadata(UrlType* type_url, ValueType* value) + : type_url_(type_url), value_(value) {} // Packs a message using the default type URL prefix: "type.googleapis.com". // The resulted type URL will be "type.googleapis.com/". - void PackFrom(const Message& message); + // Returns false if serializing the message failed. + template + bool PackFrom(Arena* arena, const T& message) { + return InternalPackFrom(arena, message, kTypeGoogleApisComPrefix, + T::FullMessageName()); + } + + bool PackFrom(Arena* arena, const Message& message); + // Packs a message using the given type URL prefix. The type URL will be // constructed by concatenating the message type's full name to the prefix - // with an optional "/" separator if the prefix doesn't already end up "/". + // with an optional "/" separator if the prefix doesn't already end with "/". // For example, both PackFrom(message, "type.googleapis.com") and // PackFrom(message, "type.googleapis.com/") yield the same result type // URL: "type.googleapis.com/". - void PackFrom(const Message& message, const string& type_url_prefix); + // Returns false if serializing the message failed. + template + bool PackFrom(Arena* arena, const T& message, + StringPiece type_url_prefix) { + return InternalPackFrom(arena, message, type_url_prefix, + T::FullMessageName()); + } + + bool PackFrom(Arena* arena, const Message& message, + StringPiece type_url_prefix); // Unpacks the payload into the given message. Returns false if the message's // type doesn't match the type specified in the type URL (i.e., the full // name after the last "/" of the type URL doesn't match the message's actual // full name) or parsing the payload has failed. + template + bool UnpackTo(T* message) const { + return InternalUnpackTo(T::FullMessageName(), message); + } + bool UnpackTo(Message* message) const; // Checks whether the type specified in the type URL matches the given type. - // A type is consdiered matching if its full name matches the full name after + // A type is considered matching if its full name matches the full name after // the last "/" in the type URL. - template + template bool Is() const { - return InternalIs(T::default_instance().GetDescriptor()); + return InternalIs(T::FullMessageName()); } private: - bool InternalIs(const Descriptor* message) const; + bool InternalPackFrom(Arena* arena, const MessageLite& message, + StringPiece type_url_prefix, + StringPiece type_name); + bool InternalUnpackTo(StringPiece type_name, + MessageLite* message) const; + bool InternalIs(StringPiece type_name) const; UrlType* type_url_; ValueType* value_; @@ -84,15 +124,22 @@ class LIBPROTOBUF_EXPORT AnyMetadata { GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(AnyMetadata); }; -extern const char kAnyFullTypeName[]; // "google.protobuf.Any". -extern const char kTypeGoogleApisComPrefix[]; // "type.googleapis.com/". -extern const char kTypeGoogleProdComPrefix[]; // "type.googleprod.com/". - // Get the proto type name from Any::type_url value. For example, passing // "type.googleapis.com/rpc.QueryOrigin" will return "rpc.QueryOrigin" in // *full_type_name. Returns false if the type_url does not have a "/" // in the type url separating the full type name. -bool ParseAnyTypeUrl(const string& type_url, string* full_type_name); +// +// NOTE: this function is available publicly as: +// google::protobuf::Any() // static method on the generated message type. +bool ParseAnyTypeUrl(StringPiece type_url, std::string* full_type_name); + +// Get the proto type name and prefix from Any::type_url value. For example, +// passing "type.googleapis.com/rpc.QueryOrigin" will return +// "type.googleapis.com/" in *url_prefix and "rpc.QueryOrigin" in +// *full_type_name. Returns false if the type_url does not have a "/" in the +// type url separating the full type name. +bool ParseAnyTypeUrl(StringPiece type_url, std::string* url_prefix, + std::string* full_type_name); // See if message is of type google.protobuf.Any, if so, return the descriptors // for "type_url" and "value" fields. @@ -102,6 +149,8 @@ bool GetAnyFieldDescriptors(const Message& message, } // namespace internal } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_ANY_H__ diff --git a/3rdparty/protobuf/src/google/protobuf/any.pb.cc b/3rdparty/protobuf/src/google/protobuf/any.pb.cc deleted file mode 100644 index 296874770e30..000000000000 --- a/3rdparty/protobuf/src/google/protobuf/any.pb.cc +++ /dev/null @@ -1,440 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: google/protobuf/any.proto - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -// This is a temporary google only hack -#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS -#include "third_party/protobuf/version.h" -#endif -// @@protoc_insertion_point(includes) -namespace google { -namespace protobuf { -class AnyDefaultTypeInternal { - public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; -} _Any_default_instance_; -} // namespace protobuf -} // namespace google -namespace protobuf_google_2fprotobuf_2fany_2eproto { -void InitDefaultsAnyImpl() { - GOOGLE_PROTOBUF_VERIFY_VERSION; - -#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - ::google::protobuf::internal::InitProtobufDefaultsForceUnique(); -#else - ::google::protobuf::internal::InitProtobufDefaults(); -#endif // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - { - void* ptr = &::google::protobuf::_Any_default_instance_; - new (ptr) ::google::protobuf::Any(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); - } - ::google::protobuf::Any::InitAsDefaultInstance(); -} - -void InitDefaultsAny() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsAnyImpl); -} - -::google::protobuf::Metadata file_level_metadata[1]; - -const ::google::protobuf::uint32 TableStruct::offsets[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { - ~0u, // no _has_bits_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Any, _internal_metadata_), - ~0u, // no _extensions_ - ~0u, // no _oneof_case_ - ~0u, // no _weak_field_map_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Any, type_url_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Any, value_), -}; -static const ::google::protobuf::internal::MigrationSchema schemas[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { - { 0, -1, sizeof(::google::protobuf::Any)}, -}; - -static ::google::protobuf::Message const * const file_default_instances[] = { - reinterpret_cast(&::google::protobuf::_Any_default_instance_), -}; - -void protobuf_AssignDescriptors() { - AddDescriptors(); - ::google::protobuf::MessageFactory* factory = NULL; - AssignDescriptors( - "google/protobuf/any.proto", schemas, file_default_instances, TableStruct::offsets, factory, - file_level_metadata, NULL, NULL); -} - -void protobuf_AssignDescriptorsOnce() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &protobuf_AssignDescriptors); -} - -void protobuf_RegisterTypes(const ::std::string&) GOOGLE_PROTOBUF_ATTRIBUTE_COLD; -void protobuf_RegisterTypes(const ::std::string&) { - protobuf_AssignDescriptorsOnce(); - ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 1); -} - -void AddDescriptorsImpl() { - InitDefaults(); - static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { - "\n\031google/protobuf/any.proto\022\017google.prot" - "obuf\"&\n\003Any\022\020\n\010type_url\030\001 \001(\t\022\r\n\005value\030\002" - " \001(\014Bo\n\023com.google.protobufB\010AnyProtoP\001Z" - "%github.com/golang/protobuf/ptypes/any\242\002" - "\003GPB\252\002\036Google.Protobuf.WellKnownTypesb\006p" - "roto3" - }; - ::google::protobuf::DescriptorPool::InternalAddGeneratedFile( - descriptor, 205); - ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( - "google/protobuf/any.proto", &protobuf_RegisterTypes); -} - -void AddDescriptors() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &AddDescriptorsImpl); -} -// Force AddDescriptors() to be called at dynamic initialization time. -struct StaticDescriptorInitializer { - StaticDescriptorInitializer() { - AddDescriptors(); - } -} static_descriptor_initializer; -} // namespace protobuf_google_2fprotobuf_2fany_2eproto -namespace google { -namespace protobuf { - -// =================================================================== - -void Any::InitAsDefaultInstance() { -} -void Any::PackFrom(const ::google::protobuf::Message& message) { - _any_metadata_.PackFrom(message); -} - -void Any::PackFrom(const ::google::protobuf::Message& message, - const ::std::string& type_url_prefix) { - _any_metadata_.PackFrom(message, type_url_prefix); -} - -bool Any::UnpackTo(::google::protobuf::Message* message) const { - return _any_metadata_.UnpackTo(message); -} - -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int Any::kTypeUrlFieldNumber; -const int Any::kValueFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 - -Any::Any() - : ::google::protobuf::Message(), _internal_metadata_(NULL), _any_metadata_(&type_url_, &value_) { - if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) { - ::protobuf_google_2fprotobuf_2fany_2eproto::InitDefaultsAny(); - } - SharedCtor(); - // @@protoc_insertion_point(constructor:google.protobuf.Any) -} -Any::Any(const Any& from) - : ::google::protobuf::Message(), - _internal_metadata_(NULL), - _cached_size_(0), - _any_metadata_(&type_url_, &value_) { - _internal_metadata_.MergeFrom(from._internal_metadata_); - type_url_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.type_url().size() > 0) { - type_url_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.type_url_); - } - value_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.value().size() > 0) { - value_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.value_); - } - // @@protoc_insertion_point(copy_constructor:google.protobuf.Any) -} - -void Any::SharedCtor() { - type_url_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - value_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - _cached_size_ = 0; -} - -Any::~Any() { - // @@protoc_insertion_point(destructor:google.protobuf.Any) - SharedDtor(); -} - -void Any::SharedDtor() { - type_url_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - value_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} - -void Any::SetCachedSize(int size) const { - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); -} -const ::google::protobuf::Descriptor* Any::descriptor() { - ::protobuf_google_2fprotobuf_2fany_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fany_2eproto::file_level_metadata[kIndexInFileMessages].descriptor; -} - -const Any& Any::default_instance() { - ::protobuf_google_2fprotobuf_2fany_2eproto::InitDefaultsAny(); - return *internal_default_instance(); -} - -Any* Any::New(::google::protobuf::Arena* arena) const { - Any* n = new Any; - if (arena != NULL) { - arena->Own(n); - } - return n; -} - -void Any::Clear() { -// @@protoc_insertion_point(message_clear_start:google.protobuf.Any) - ::google::protobuf::uint32 cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - _internal_metadata_.Clear(); -} - -bool Any::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - // @@protoc_insertion_point(parse_start:google.protobuf.Any) - for (;;) { - ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // string type_url = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_type_url())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->type_url().data(), static_cast(this->type_url().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Any.type_url")); - } else { - goto handle_unusual; - } - break; - } - - // bytes value = 2; - case 2: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadBytes( - input, this->mutable_value())); - } else { - goto handle_unusual; - } - break; - } - - default: { - handle_unusual: - if (tag == 0) { - goto success; - } - DO_(::google::protobuf::internal::WireFormat::SkipField( - input, tag, _internal_metadata_.mutable_unknown_fields())); - break; - } - } - } -success: - // @@protoc_insertion_point(parse_success:google.protobuf.Any) - return true; -failure: - // @@protoc_insertion_point(parse_failure:google.protobuf.Any) - return false; -#undef DO_ -} - -void Any::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:google.protobuf.Any) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string type_url = 1; - if (this->type_url().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->type_url().data(), static_cast(this->type_url().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Any.type_url"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->type_url(), output); - } - - // bytes value = 2; - if (this->value().size() > 0) { - ::google::protobuf::internal::WireFormatLite::WriteBytesMaybeAliased( - 2, this->value(), output); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - ::google::protobuf::internal::WireFormat::SerializeUnknownFields( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), output); - } - // @@protoc_insertion_point(serialize_end:google.protobuf.Any) -} - -::google::protobuf::uint8* Any::InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const { - (void)deterministic; // Unused - // @@protoc_insertion_point(serialize_to_array_start:google.protobuf.Any) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string type_url = 1; - if (this->type_url().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->type_url().data(), static_cast(this->type_url().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Any.type_url"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 1, this->type_url(), target); - } - - // bytes value = 2; - if (this->value().size() > 0) { - target = - ::google::protobuf::internal::WireFormatLite::WriteBytesToArray( - 2, this->value(), target); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), target); - } - // @@protoc_insertion_point(serialize_to_array_end:google.protobuf.Any) - return target; -} - -size_t Any::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Any) - size_t total_size = 0; - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - total_size += - ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance())); - } - // string type_url = 1; - if (this->type_url().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->type_url()); - } - - // bytes value = 2; - if (this->value().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::BytesSize( - this->value()); - } - - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = cached_size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); - return total_size; -} - -void Any::MergeFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_merge_from_start:google.protobuf.Any) - GOOGLE_DCHECK_NE(&from, this); - const Any* source = - ::google::protobuf::internal::DynamicCastToGenerated( - &from); - if (source == NULL) { - // @@protoc_insertion_point(generalized_merge_from_cast_fail:google.protobuf.Any) - ::google::protobuf::internal::ReflectionOps::Merge(from, this); - } else { - // @@protoc_insertion_point(generalized_merge_from_cast_success:google.protobuf.Any) - MergeFrom(*source); - } -} - -void Any::MergeFrom(const Any& from) { -// @@protoc_insertion_point(class_specific_merge_from_start:google.protobuf.Any) - GOOGLE_DCHECK_NE(&from, this); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - if (from.type_url().size() > 0) { - - type_url_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.type_url_); - } - if (from.value().size() > 0) { - - value_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.value_); - } -} - -void Any::CopyFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_copy_from_start:google.protobuf.Any) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -void Any::CopyFrom(const Any& from) { -// @@protoc_insertion_point(class_specific_copy_from_start:google.protobuf.Any) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool Any::IsInitialized() const { - return true; -} - -void Any::Swap(Any* other) { - if (other == this) return; - InternalSwap(other); -} -void Any::InternalSwap(Any* other) { - using std::swap; - type_url_.Swap(&other->type_url_); - value_.Swap(&other->value_); - _internal_metadata_.Swap(&other->_internal_metadata_); - swap(_cached_size_, other->_cached_size_); -} - -::google::protobuf::Metadata Any::GetMetadata() const { - protobuf_google_2fprotobuf_2fany_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fany_2eproto::file_level_metadata[kIndexInFileMessages]; -} - - -// @@protoc_insertion_point(namespace_scope) -} // namespace protobuf -} // namespace google - -// @@protoc_insertion_point(global_scope) diff --git a/3rdparty/protobuf/src/google/protobuf/any.pb.h b/3rdparty/protobuf/src/google/protobuf/any.pb.h deleted file mode 100644 index 32847239bef1..000000000000 --- a/3rdparty/protobuf/src/google/protobuf/any.pb.h +++ /dev/null @@ -1,323 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: google/protobuf/any.proto - -#ifndef PROTOBUF_google_2fprotobuf_2fany_2eproto__INCLUDED -#define PROTOBUF_google_2fprotobuf_2fany_2eproto__INCLUDED - -#include - -#include - -#if GOOGLE_PROTOBUF_VERSION < 3005000 -#error This file was generated by a newer version of protoc which is -#error incompatible with your Protocol Buffer headers. Please update -#error your headers. -#endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION -#error This file was generated by an older version of protoc which is -#error incompatible with your Protocol Buffer headers. Please -#error regenerate this file with a newer version of protoc. -#endif - -#include -#include -#include -#include -#include -#include -#include -#include // IWYU pragma: export -#include // IWYU pragma: export -#include -#include -// @@protoc_insertion_point(includes) - -namespace protobuf_google_2fprotobuf_2fany_2eproto { -// Internal implementation detail -- do not use these members. -struct LIBPROTOBUF_EXPORT TableStruct { - static const ::google::protobuf::internal::ParseTableField entries[]; - static const ::google::protobuf::internal::AuxillaryParseTableField aux[]; - static const ::google::protobuf::internal::ParseTable schema[1]; - static const ::google::protobuf::internal::FieldMetadata field_metadata[]; - static const ::google::protobuf::internal::SerializationTable serialization_table[]; - static const ::google::protobuf::uint32 offsets[]; -}; -void LIBPROTOBUF_EXPORT AddDescriptors(); -void LIBPROTOBUF_EXPORT InitDefaultsAnyImpl(); -void LIBPROTOBUF_EXPORT InitDefaultsAny(); -inline void LIBPROTOBUF_EXPORT InitDefaults() { - InitDefaultsAny(); -} -} // namespace protobuf_google_2fprotobuf_2fany_2eproto -namespace google { -namespace protobuf { -class Any; -class AnyDefaultTypeInternal; -LIBPROTOBUF_EXPORT extern AnyDefaultTypeInternal _Any_default_instance_; -} // namespace protobuf -} // namespace google -namespace google { -namespace protobuf { - -// =================================================================== - -class LIBPROTOBUF_EXPORT Any : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Any) */ { - public: - Any(); - virtual ~Any(); - - Any(const Any& from); - - inline Any& operator=(const Any& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 - Any(Any&& from) noexcept - : Any() { - *this = ::std::move(from); - } - - inline Any& operator=(Any&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { - if (this != &from) InternalSwap(&from); - } else { - CopyFrom(from); - } - return *this; - } - #endif - static const ::google::protobuf::Descriptor* descriptor(); - static const Any& default_instance(); - - static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY - static inline const Any* internal_default_instance() { - return reinterpret_cast( - &_Any_default_instance_); - } - static PROTOBUF_CONSTEXPR int const kIndexInFileMessages = - 0; - - // implements Any ----------------------------------------------- - - void PackFrom(const ::google::protobuf::Message& message); - void PackFrom(const ::google::protobuf::Message& message, - const ::std::string& type_url_prefix); - bool UnpackTo(::google::protobuf::Message* message) const; - template bool Is() const { - return _any_metadata_.Is(); - } - - void Swap(Any* other); - friend void swap(Any& a, Any& b) { - a.Swap(&b); - } - - // implements Message ---------------------------------------------- - - inline Any* New() const PROTOBUF_FINAL { return New(NULL); } - - Any* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL; - void CopyFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void MergeFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void CopyFrom(const Any& from); - void MergeFrom(const Any& from); - void Clear() PROTOBUF_FINAL; - bool IsInitialized() const PROTOBUF_FINAL; - - size_t ByteSizeLong() const PROTOBUF_FINAL; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PROTOBUF_FINAL; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PROTOBUF_FINAL; - ::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const PROTOBUF_FINAL; - int GetCachedSize() const PROTOBUF_FINAL { return _cached_size_; } - private: - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PROTOBUF_FINAL; - void InternalSwap(Any* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; - } - public: - - ::google::protobuf::Metadata GetMetadata() const PROTOBUF_FINAL; - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - // string type_url = 1; - void clear_type_url(); - static const int kTypeUrlFieldNumber = 1; - const ::std::string& type_url() const; - void set_type_url(const ::std::string& value); - #if LANG_CXX11 - void set_type_url(::std::string&& value); - #endif - void set_type_url(const char* value); - void set_type_url(const char* value, size_t size); - ::std::string* mutable_type_url(); - ::std::string* release_type_url(); - void set_allocated_type_url(::std::string* type_url); - - // bytes value = 2; - void clear_value(); - static const int kValueFieldNumber = 2; - const ::std::string& value() const; - void set_value(const ::std::string& value); - #if LANG_CXX11 - void set_value(::std::string&& value); - #endif - void set_value(const char* value); - void set_value(const void* value, size_t size); - ::std::string* mutable_value(); - ::std::string* release_value(); - void set_allocated_value(::std::string* value); - - // @@protoc_insertion_point(class_scope:google.protobuf.Any) - private: - - ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; - ::google::protobuf::internal::ArenaStringPtr type_url_; - ::google::protobuf::internal::ArenaStringPtr value_; - mutable int _cached_size_; - ::google::protobuf::internal::AnyMetadata _any_metadata_; - friend struct ::protobuf_google_2fprotobuf_2fany_2eproto::TableStruct; - friend void ::protobuf_google_2fprotobuf_2fany_2eproto::InitDefaultsAnyImpl(); -}; -// =================================================================== - - -// =================================================================== - -#ifdef __GNUC__ - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wstrict-aliasing" -#endif // __GNUC__ -// Any - -// string type_url = 1; -inline void Any::clear_type_url() { - type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Any::type_url() const { - // @@protoc_insertion_point(field_get:google.protobuf.Any.type_url) - return type_url_.GetNoArena(); -} -inline void Any::set_type_url(const ::std::string& value) { - - type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Any.type_url) -} -#if LANG_CXX11 -inline void Any::set_type_url(::std::string&& value) { - - type_url_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Any.type_url) -} -#endif -inline void Any::set_type_url(const char* value) { - GOOGLE_DCHECK(value != NULL); - - type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Any.type_url) -} -inline void Any::set_type_url(const char* value, size_t size) { - - type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Any.type_url) -} -inline ::std::string* Any::mutable_type_url() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Any.type_url) - return type_url_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Any::release_type_url() { - // @@protoc_insertion_point(field_release:google.protobuf.Any.type_url) - - return type_url_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Any::set_allocated_type_url(::std::string* type_url) { - if (type_url != NULL) { - - } else { - - } - type_url_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), type_url); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Any.type_url) -} - -// bytes value = 2; -inline void Any::clear_value() { - value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Any::value() const { - // @@protoc_insertion_point(field_get:google.protobuf.Any.value) - return value_.GetNoArena(); -} -inline void Any::set_value(const ::std::string& value) { - - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Any.value) -} -#if LANG_CXX11 -inline void Any::set_value(::std::string&& value) { - - value_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Any.value) -} -#endif -inline void Any::set_value(const char* value) { - GOOGLE_DCHECK(value != NULL); - - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Any.value) -} -inline void Any::set_value(const void* value, size_t size) { - - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Any.value) -} -inline ::std::string* Any::mutable_value() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Any.value) - return value_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Any::release_value() { - // @@protoc_insertion_point(field_release:google.protobuf.Any.value) - - return value_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Any::set_allocated_value(::std::string* value) { - if (value != NULL) { - - } else { - - } - value_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Any.value) -} - -#ifdef __GNUC__ - #pragma GCC diagnostic pop -#endif // __GNUC__ - -// @@protoc_insertion_point(namespace_scope) - -} // namespace protobuf -} // namespace google - -// @@protoc_insertion_point(global_scope) - -#endif // PROTOBUF_google_2fprotobuf_2fany_2eproto__INCLUDED diff --git a/3rdparty/protobuf/src/google/protobuf/any_lite.cc b/3rdparty/protobuf/src/google/protobuf/any_lite.cc new file mode 100644 index 000000000000..a98559da140c --- /dev/null +++ b/3rdparty/protobuf/src/google/protobuf/any_lite.cc @@ -0,0 +1,99 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +#include +#include +#include +#include + +namespace google { +namespace protobuf { +namespace internal { + +std::string GetTypeUrl(StringPiece message_name, + StringPiece type_url_prefix) { + if (!type_url_prefix.empty() && + type_url_prefix[type_url_prefix.size() - 1] == '/') { + return StrCat(type_url_prefix, message_name); + } else { + return StrCat(type_url_prefix, "/", message_name); + } +} + +const char kAnyFullTypeName[] = "google.protobuf.Any"; +const char kTypeGoogleApisComPrefix[] = "type.googleapis.com/"; +const char kTypeGoogleProdComPrefix[] = "type.googleprod.com/"; + +bool AnyMetadata::InternalPackFrom(Arena* arena, const MessageLite& message, + StringPiece type_url_prefix, + StringPiece type_name) { + type_url_->Set(&::google::protobuf::internal::GetEmptyString(), + GetTypeUrl(type_name, type_url_prefix), arena); + return message.SerializeToString( + value_->Mutable(ArenaStringPtr::EmptyDefault{}, arena)); +} + +bool AnyMetadata::InternalUnpackTo(StringPiece type_name, + MessageLite* message) const { + if (!InternalIs(type_name)) { + return false; + } + return message->ParseFromString(value_->Get()); +} + +bool AnyMetadata::InternalIs(StringPiece type_name) const { + StringPiece type_url = type_url_->Get(); + return type_url.size() >= type_name.size() + 1 && + type_url[type_url.size() - type_name.size() - 1] == '/' && + HasSuffixString(type_url, type_name); +} + +bool ParseAnyTypeUrl(StringPiece type_url, std::string* url_prefix, + std::string* full_type_name) { + size_t pos = type_url.find_last_of('/'); + if (pos == std::string::npos || pos + 1 == type_url.size()) { + return false; + } + if (url_prefix) { + *url_prefix = std::string(type_url.substr(0, pos + 1)); + } + *full_type_name = std::string(type_url.substr(pos + 1)); + return true; +} + +bool ParseAnyTypeUrl(StringPiece type_url, std::string* full_type_name) { + return ParseAnyTypeUrl(type_url, nullptr, full_type_name); +} + +} // namespace internal +} // namespace protobuf +} // namespace google diff --git a/3rdparty/protobuf/src/google/protobuf/api.pb.cc b/3rdparty/protobuf/src/google/protobuf/api.pb.cc deleted file mode 100644 index 439a47c62dc0..000000000000 --- a/3rdparty/protobuf/src/google/protobuf/api.pb.cc +++ /dev/null @@ -1,1608 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: google/protobuf/api.proto - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -// This is a temporary google only hack -#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS -#include "third_party/protobuf/version.h" -#endif -// @@protoc_insertion_point(includes) -namespace google { -namespace protobuf { -class ApiDefaultTypeInternal { - public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; -} _Api_default_instance_; -class MethodDefaultTypeInternal { - public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; -} _Method_default_instance_; -class MixinDefaultTypeInternal { - public: - ::google::protobuf::internal::ExplicitlyConstructed - _instance; -} _Mixin_default_instance_; -} // namespace protobuf -} // namespace google -namespace protobuf_google_2fprotobuf_2fapi_2eproto { -void InitDefaultsApiImpl() { - GOOGLE_PROTOBUF_VERIFY_VERSION; - -#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - ::google::protobuf::internal::InitProtobufDefaultsForceUnique(); -#else - ::google::protobuf::internal::InitProtobufDefaults(); -#endif // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMethod(); - protobuf_google_2fprotobuf_2ftype_2eproto::InitDefaultsOption(); - protobuf_google_2fprotobuf_2fsource_5fcontext_2eproto::InitDefaultsSourceContext(); - protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMixin(); - { - void* ptr = &::google::protobuf::_Api_default_instance_; - new (ptr) ::google::protobuf::Api(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); - } - ::google::protobuf::Api::InitAsDefaultInstance(); -} - -void InitDefaultsApi() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsApiImpl); -} - -void InitDefaultsMethodImpl() { - GOOGLE_PROTOBUF_VERIFY_VERSION; - -#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - ::google::protobuf::internal::InitProtobufDefaultsForceUnique(); -#else - ::google::protobuf::internal::InitProtobufDefaults(); -#endif // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - protobuf_google_2fprotobuf_2ftype_2eproto::InitDefaultsOption(); - { - void* ptr = &::google::protobuf::_Method_default_instance_; - new (ptr) ::google::protobuf::Method(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); - } - ::google::protobuf::Method::InitAsDefaultInstance(); -} - -void InitDefaultsMethod() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsMethodImpl); -} - -void InitDefaultsMixinImpl() { - GOOGLE_PROTOBUF_VERIFY_VERSION; - -#ifdef GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - ::google::protobuf::internal::InitProtobufDefaultsForceUnique(); -#else - ::google::protobuf::internal::InitProtobufDefaults(); -#endif // GOOGLE_PROTOBUF_ENFORCE_UNIQUENESS - { - void* ptr = &::google::protobuf::_Mixin_default_instance_; - new (ptr) ::google::protobuf::Mixin(); - ::google::protobuf::internal::OnShutdownDestroyMessage(ptr); - } - ::google::protobuf::Mixin::InitAsDefaultInstance(); -} - -void InitDefaultsMixin() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &InitDefaultsMixinImpl); -} - -::google::protobuf::Metadata file_level_metadata[3]; - -const ::google::protobuf::uint32 TableStruct::offsets[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { - ~0u, // no _has_bits_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, _internal_metadata_), - ~0u, // no _extensions_ - ~0u, // no _oneof_case_ - ~0u, // no _weak_field_map_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, name_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, methods_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, options_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, version_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, source_context_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, mixins_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Api, syntax_), - ~0u, // no _has_bits_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, _internal_metadata_), - ~0u, // no _extensions_ - ~0u, // no _oneof_case_ - ~0u, // no _weak_field_map_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, name_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, request_type_url_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, request_streaming_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, response_type_url_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, response_streaming_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, options_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Method, syntax_), - ~0u, // no _has_bits_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Mixin, _internal_metadata_), - ~0u, // no _extensions_ - ~0u, // no _oneof_case_ - ~0u, // no _weak_field_map_ - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Mixin, name_), - GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(::google::protobuf::Mixin, root_), -}; -static const ::google::protobuf::internal::MigrationSchema schemas[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { - { 0, -1, sizeof(::google::protobuf::Api)}, - { 12, -1, sizeof(::google::protobuf::Method)}, - { 24, -1, sizeof(::google::protobuf::Mixin)}, -}; - -static ::google::protobuf::Message const * const file_default_instances[] = { - reinterpret_cast(&::google::protobuf::_Api_default_instance_), - reinterpret_cast(&::google::protobuf::_Method_default_instance_), - reinterpret_cast(&::google::protobuf::_Mixin_default_instance_), -}; - -void protobuf_AssignDescriptors() { - AddDescriptors(); - ::google::protobuf::MessageFactory* factory = NULL; - AssignDescriptors( - "google/protobuf/api.proto", schemas, file_default_instances, TableStruct::offsets, factory, - file_level_metadata, NULL, NULL); -} - -void protobuf_AssignDescriptorsOnce() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &protobuf_AssignDescriptors); -} - -void protobuf_RegisterTypes(const ::std::string&) GOOGLE_PROTOBUF_ATTRIBUTE_COLD; -void protobuf_RegisterTypes(const ::std::string&) { - protobuf_AssignDescriptorsOnce(); - ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 3); -} - -void AddDescriptorsImpl() { - InitDefaults(); - static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { - "\n\031google/protobuf/api.proto\022\017google.prot" - "obuf\032$google/protobuf/source_context.pro" - "to\032\032google/protobuf/type.proto\"\201\002\n\003Api\022\014" - "\n\004name\030\001 \001(\t\022(\n\007methods\030\002 \003(\0132\027.google.p" - "rotobuf.Method\022(\n\007options\030\003 \003(\0132\027.google" - ".protobuf.Option\022\017\n\007version\030\004 \001(\t\0226\n\016sou" - "rce_context\030\005 \001(\0132\036.google.protobuf.Sour" - "ceContext\022&\n\006mixins\030\006 \003(\0132\026.google.proto" - "buf.Mixin\022\'\n\006syntax\030\007 \001(\0162\027.google.proto" - "buf.Syntax\"\325\001\n\006Method\022\014\n\004name\030\001 \001(\t\022\030\n\020r" - "equest_type_url\030\002 \001(\t\022\031\n\021request_streami" - "ng\030\003 \001(\010\022\031\n\021response_type_url\030\004 \001(\t\022\032\n\022r" - "esponse_streaming\030\005 \001(\010\022(\n\007options\030\006 \003(\013" - "2\027.google.protobuf.Option\022\'\n\006syntax\030\007 \001(" - "\0162\027.google.protobuf.Syntax\"#\n\005Mixin\022\014\n\004n" - "ame\030\001 \001(\t\022\014\n\004root\030\002 \001(\tBu\n\023com.google.pr" - "otobufB\010ApiProtoP\001Z+google.golang.org/ge" - "nproto/protobuf/api;api\242\002\003GPB\252\002\036Google.P" - "rotobuf.WellKnownTypesb\006proto3" - }; - ::google::protobuf::DescriptorPool::InternalAddGeneratedFile( - descriptor, 750); - ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( - "google/protobuf/api.proto", &protobuf_RegisterTypes); - ::protobuf_google_2fprotobuf_2fsource_5fcontext_2eproto::AddDescriptors(); - ::protobuf_google_2fprotobuf_2ftype_2eproto::AddDescriptors(); -} - -void AddDescriptors() { - static GOOGLE_PROTOBUF_DECLARE_ONCE(once); - ::google::protobuf::GoogleOnceInit(&once, &AddDescriptorsImpl); -} -// Force AddDescriptors() to be called at dynamic initialization time. -struct StaticDescriptorInitializer { - StaticDescriptorInitializer() { - AddDescriptors(); - } -} static_descriptor_initializer; -} // namespace protobuf_google_2fprotobuf_2fapi_2eproto -namespace google { -namespace protobuf { - -// =================================================================== - -void Api::InitAsDefaultInstance() { - ::google::protobuf::_Api_default_instance_._instance.get_mutable()->source_context_ = const_cast< ::google::protobuf::SourceContext*>( - ::google::protobuf::SourceContext::internal_default_instance()); -} -void Api::clear_options() { - options_.Clear(); -} -void Api::clear_source_context() { - if (GetArenaNoVirtual() == NULL && source_context_ != NULL) { - delete source_context_; - } - source_context_ = NULL; -} -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int Api::kNameFieldNumber; -const int Api::kMethodsFieldNumber; -const int Api::kOptionsFieldNumber; -const int Api::kVersionFieldNumber; -const int Api::kSourceContextFieldNumber; -const int Api::kMixinsFieldNumber; -const int Api::kSyntaxFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 - -Api::Api() - : ::google::protobuf::Message(), _internal_metadata_(NULL) { - if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) { - ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsApi(); - } - SharedCtor(); - // @@protoc_insertion_point(constructor:google.protobuf.Api) -} -Api::Api(const Api& from) - : ::google::protobuf::Message(), - _internal_metadata_(NULL), - methods_(from.methods_), - options_(from.options_), - mixins_(from.mixins_), - _cached_size_(0) { - _internal_metadata_.MergeFrom(from._internal_metadata_); - name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.name().size() > 0) { - name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.name_); - } - version_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.version().size() > 0) { - version_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.version_); - } - if (from.has_source_context()) { - source_context_ = new ::google::protobuf::SourceContext(*from.source_context_); - } else { - source_context_ = NULL; - } - syntax_ = from.syntax_; - // @@protoc_insertion_point(copy_constructor:google.protobuf.Api) -} - -void Api::SharedCtor() { - name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - version_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - ::memset(&source_context_, 0, static_cast( - reinterpret_cast(&syntax_) - - reinterpret_cast(&source_context_)) + sizeof(syntax_)); - _cached_size_ = 0; -} - -Api::~Api() { - // @@protoc_insertion_point(destructor:google.protobuf.Api) - SharedDtor(); -} - -void Api::SharedDtor() { - name_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - version_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (this != internal_default_instance()) delete source_context_; -} - -void Api::SetCachedSize(int size) const { - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); -} -const ::google::protobuf::Descriptor* Api::descriptor() { - ::protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages].descriptor; -} - -const Api& Api::default_instance() { - ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsApi(); - return *internal_default_instance(); -} - -Api* Api::New(::google::protobuf::Arena* arena) const { - Api* n = new Api; - if (arena != NULL) { - arena->Own(n); - } - return n; -} - -void Api::Clear() { -// @@protoc_insertion_point(message_clear_start:google.protobuf.Api) - ::google::protobuf::uint32 cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - methods_.Clear(); - options_.Clear(); - mixins_.Clear(); - name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - version_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (GetArenaNoVirtual() == NULL && source_context_ != NULL) { - delete source_context_; - } - source_context_ = NULL; - syntax_ = 0; - _internal_metadata_.Clear(); -} - -bool Api::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - // @@protoc_insertion_point(parse_start:google.protobuf.Api) - for (;;) { - ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // string name = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_name())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Api.name")); - } else { - goto handle_unusual; - } - break; - } - - // repeated .google.protobuf.Method methods = 2; - case 2: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_methods())); - } else { - goto handle_unusual; - } - break; - } - - // repeated .google.protobuf.Option options = 3; - case 3: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(26u /* 26 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_options())); - } else { - goto handle_unusual; - } - break; - } - - // string version = 4; - case 4: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(34u /* 34 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_version())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->version().data(), static_cast(this->version().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Api.version")); - } else { - goto handle_unusual; - } - break; - } - - // .google.protobuf.SourceContext source_context = 5; - case 5: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(42u /* 42 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage( - input, mutable_source_context())); - } else { - goto handle_unusual; - } - break; - } - - // repeated .google.protobuf.Mixin mixins = 6; - case 6: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(50u /* 50 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_mixins())); - } else { - goto handle_unusual; - } - break; - } - - // .google.protobuf.Syntax syntax = 7; - case 7: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(56u /* 56 & 0xFF */)) { - int value; - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( - input, &value))); - set_syntax(static_cast< ::google::protobuf::Syntax >(value)); - } else { - goto handle_unusual; - } - break; - } - - default: { - handle_unusual: - if (tag == 0) { - goto success; - } - DO_(::google::protobuf::internal::WireFormat::SkipField( - input, tag, _internal_metadata_.mutable_unknown_fields())); - break; - } - } - } -success: - // @@protoc_insertion_point(parse_success:google.protobuf.Api) - return true; -failure: - // @@protoc_insertion_point(parse_failure:google.protobuf.Api) - return false; -#undef DO_ -} - -void Api::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:google.protobuf.Api) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string name = 1; - if (this->name().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Api.name"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->name(), output); - } - - // repeated .google.protobuf.Method methods = 2; - for (unsigned int i = 0, - n = static_cast(this->methods_size()); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( - 2, this->methods(static_cast(i)), output); - } - - // repeated .google.protobuf.Option options = 3; - for (unsigned int i = 0, - n = static_cast(this->options_size()); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( - 3, this->options(static_cast(i)), output); - } - - // string version = 4; - if (this->version().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->version().data(), static_cast(this->version().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Api.version"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 4, this->version(), output); - } - - // .google.protobuf.SourceContext source_context = 5; - if (this->has_source_context()) { - ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( - 5, *this->source_context_, output); - } - - // repeated .google.protobuf.Mixin mixins = 6; - for (unsigned int i = 0, - n = static_cast(this->mixins_size()); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( - 6, this->mixins(static_cast(i)), output); - } - - // .google.protobuf.Syntax syntax = 7; - if (this->syntax() != 0) { - ::google::protobuf::internal::WireFormatLite::WriteEnum( - 7, this->syntax(), output); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - ::google::protobuf::internal::WireFormat::SerializeUnknownFields( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), output); - } - // @@protoc_insertion_point(serialize_end:google.protobuf.Api) -} - -::google::protobuf::uint8* Api::InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const { - (void)deterministic; // Unused - // @@protoc_insertion_point(serialize_to_array_start:google.protobuf.Api) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string name = 1; - if (this->name().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Api.name"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 1, this->name(), target); - } - - // repeated .google.protobuf.Method methods = 2; - for (unsigned int i = 0, - n = static_cast(this->methods_size()); i < n; i++) { - target = ::google::protobuf::internal::WireFormatLite:: - InternalWriteMessageToArray( - 2, this->methods(static_cast(i)), deterministic, target); - } - - // repeated .google.protobuf.Option options = 3; - for (unsigned int i = 0, - n = static_cast(this->options_size()); i < n; i++) { - target = ::google::protobuf::internal::WireFormatLite:: - InternalWriteMessageToArray( - 3, this->options(static_cast(i)), deterministic, target); - } - - // string version = 4; - if (this->version().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->version().data(), static_cast(this->version().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Api.version"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 4, this->version(), target); - } - - // .google.protobuf.SourceContext source_context = 5; - if (this->has_source_context()) { - target = ::google::protobuf::internal::WireFormatLite:: - InternalWriteMessageToArray( - 5, *this->source_context_, deterministic, target); - } - - // repeated .google.protobuf.Mixin mixins = 6; - for (unsigned int i = 0, - n = static_cast(this->mixins_size()); i < n; i++) { - target = ::google::protobuf::internal::WireFormatLite:: - InternalWriteMessageToArray( - 6, this->mixins(static_cast(i)), deterministic, target); - } - - // .google.protobuf.Syntax syntax = 7; - if (this->syntax() != 0) { - target = ::google::protobuf::internal::WireFormatLite::WriteEnumToArray( - 7, this->syntax(), target); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), target); - } - // @@protoc_insertion_point(serialize_to_array_end:google.protobuf.Api) - return target; -} - -size_t Api::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Api) - size_t total_size = 0; - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - total_size += - ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance())); - } - // repeated .google.protobuf.Method methods = 2; - { - unsigned int count = static_cast(this->methods_size()); - total_size += 1UL * count; - for (unsigned int i = 0; i < count; i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSize( - this->methods(static_cast(i))); - } - } - - // repeated .google.protobuf.Option options = 3; - { - unsigned int count = static_cast(this->options_size()); - total_size += 1UL * count; - for (unsigned int i = 0; i < count; i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSize( - this->options(static_cast(i))); - } - } - - // repeated .google.protobuf.Mixin mixins = 6; - { - unsigned int count = static_cast(this->mixins_size()); - total_size += 1UL * count; - for (unsigned int i = 0; i < count; i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSize( - this->mixins(static_cast(i))); - } - } - - // string name = 1; - if (this->name().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->name()); - } - - // string version = 4; - if (this->version().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->version()); - } - - // .google.protobuf.SourceContext source_context = 5; - if (this->has_source_context()) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::MessageSize( - *this->source_context_); - } - - // .google.protobuf.Syntax syntax = 7; - if (this->syntax() != 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::EnumSize(this->syntax()); - } - - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = cached_size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); - return total_size; -} - -void Api::MergeFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_merge_from_start:google.protobuf.Api) - GOOGLE_DCHECK_NE(&from, this); - const Api* source = - ::google::protobuf::internal::DynamicCastToGenerated( - &from); - if (source == NULL) { - // @@protoc_insertion_point(generalized_merge_from_cast_fail:google.protobuf.Api) - ::google::protobuf::internal::ReflectionOps::Merge(from, this); - } else { - // @@protoc_insertion_point(generalized_merge_from_cast_success:google.protobuf.Api) - MergeFrom(*source); - } -} - -void Api::MergeFrom(const Api& from) { -// @@protoc_insertion_point(class_specific_merge_from_start:google.protobuf.Api) - GOOGLE_DCHECK_NE(&from, this); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - methods_.MergeFrom(from.methods_); - options_.MergeFrom(from.options_); - mixins_.MergeFrom(from.mixins_); - if (from.name().size() > 0) { - - name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.name_); - } - if (from.version().size() > 0) { - - version_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.version_); - } - if (from.has_source_context()) { - mutable_source_context()->::google::protobuf::SourceContext::MergeFrom(from.source_context()); - } - if (from.syntax() != 0) { - set_syntax(from.syntax()); - } -} - -void Api::CopyFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_copy_from_start:google.protobuf.Api) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -void Api::CopyFrom(const Api& from) { -// @@protoc_insertion_point(class_specific_copy_from_start:google.protobuf.Api) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool Api::IsInitialized() const { - return true; -} - -void Api::Swap(Api* other) { - if (other == this) return; - InternalSwap(other); -} -void Api::InternalSwap(Api* other) { - using std::swap; - methods_.InternalSwap(&other->methods_); - options_.InternalSwap(&other->options_); - mixins_.InternalSwap(&other->mixins_); - name_.Swap(&other->name_); - version_.Swap(&other->version_); - swap(source_context_, other->source_context_); - swap(syntax_, other->syntax_); - _internal_metadata_.Swap(&other->_internal_metadata_); - swap(_cached_size_, other->_cached_size_); -} - -::google::protobuf::Metadata Api::GetMetadata() const { - protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages]; -} - - -// =================================================================== - -void Method::InitAsDefaultInstance() { -} -void Method::clear_options() { - options_.Clear(); -} -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int Method::kNameFieldNumber; -const int Method::kRequestTypeUrlFieldNumber; -const int Method::kRequestStreamingFieldNumber; -const int Method::kResponseTypeUrlFieldNumber; -const int Method::kResponseStreamingFieldNumber; -const int Method::kOptionsFieldNumber; -const int Method::kSyntaxFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 - -Method::Method() - : ::google::protobuf::Message(), _internal_metadata_(NULL) { - if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) { - ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMethod(); - } - SharedCtor(); - // @@protoc_insertion_point(constructor:google.protobuf.Method) -} -Method::Method(const Method& from) - : ::google::protobuf::Message(), - _internal_metadata_(NULL), - options_(from.options_), - _cached_size_(0) { - _internal_metadata_.MergeFrom(from._internal_metadata_); - name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.name().size() > 0) { - name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.name_); - } - request_type_url_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.request_type_url().size() > 0) { - request_type_url_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.request_type_url_); - } - response_type_url_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.response_type_url().size() > 0) { - response_type_url_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.response_type_url_); - } - ::memcpy(&request_streaming_, &from.request_streaming_, - static_cast(reinterpret_cast(&syntax_) - - reinterpret_cast(&request_streaming_)) + sizeof(syntax_)); - // @@protoc_insertion_point(copy_constructor:google.protobuf.Method) -} - -void Method::SharedCtor() { - name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - request_type_url_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - response_type_url_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - ::memset(&request_streaming_, 0, static_cast( - reinterpret_cast(&syntax_) - - reinterpret_cast(&request_streaming_)) + sizeof(syntax_)); - _cached_size_ = 0; -} - -Method::~Method() { - // @@protoc_insertion_point(destructor:google.protobuf.Method) - SharedDtor(); -} - -void Method::SharedDtor() { - name_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - request_type_url_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - response_type_url_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} - -void Method::SetCachedSize(int size) const { - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); -} -const ::google::protobuf::Descriptor* Method::descriptor() { - ::protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages].descriptor; -} - -const Method& Method::default_instance() { - ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMethod(); - return *internal_default_instance(); -} - -Method* Method::New(::google::protobuf::Arena* arena) const { - Method* n = new Method; - if (arena != NULL) { - arena->Own(n); - } - return n; -} - -void Method::Clear() { -// @@protoc_insertion_point(message_clear_start:google.protobuf.Method) - ::google::protobuf::uint32 cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - options_.Clear(); - name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - request_type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - response_type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - ::memset(&request_streaming_, 0, static_cast( - reinterpret_cast(&syntax_) - - reinterpret_cast(&request_streaming_)) + sizeof(syntax_)); - _internal_metadata_.Clear(); -} - -bool Method::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - // @@protoc_insertion_point(parse_start:google.protobuf.Method) - for (;;) { - ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // string name = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_name())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Method.name")); - } else { - goto handle_unusual; - } - break; - } - - // string request_type_url = 2; - case 2: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_request_type_url())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->request_type_url().data(), static_cast(this->request_type_url().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Method.request_type_url")); - } else { - goto handle_unusual; - } - break; - } - - // bool request_streaming = 3; - case 3: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(24u /* 24 & 0xFF */)) { - - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &request_streaming_))); - } else { - goto handle_unusual; - } - break; - } - - // string response_type_url = 4; - case 4: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(34u /* 34 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_response_type_url())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->response_type_url().data(), static_cast(this->response_type_url().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Method.response_type_url")); - } else { - goto handle_unusual; - } - break; - } - - // bool response_streaming = 5; - case 5: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(40u /* 40 & 0xFF */)) { - - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - bool, ::google::protobuf::internal::WireFormatLite::TYPE_BOOL>( - input, &response_streaming_))); - } else { - goto handle_unusual; - } - break; - } - - // repeated .google.protobuf.Option options = 6; - case 6: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(50u /* 50 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_options())); - } else { - goto handle_unusual; - } - break; - } - - // .google.protobuf.Syntax syntax = 7; - case 7: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(56u /* 56 & 0xFF */)) { - int value; - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( - input, &value))); - set_syntax(static_cast< ::google::protobuf::Syntax >(value)); - } else { - goto handle_unusual; - } - break; - } - - default: { - handle_unusual: - if (tag == 0) { - goto success; - } - DO_(::google::protobuf::internal::WireFormat::SkipField( - input, tag, _internal_metadata_.mutable_unknown_fields())); - break; - } - } - } -success: - // @@protoc_insertion_point(parse_success:google.protobuf.Method) - return true; -failure: - // @@protoc_insertion_point(parse_failure:google.protobuf.Method) - return false; -#undef DO_ -} - -void Method::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:google.protobuf.Method) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string name = 1; - if (this->name().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Method.name"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->name(), output); - } - - // string request_type_url = 2; - if (this->request_type_url().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->request_type_url().data(), static_cast(this->request_type_url().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Method.request_type_url"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 2, this->request_type_url(), output); - } - - // bool request_streaming = 3; - if (this->request_streaming() != 0) { - ::google::protobuf::internal::WireFormatLite::WriteBool(3, this->request_streaming(), output); - } - - // string response_type_url = 4; - if (this->response_type_url().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->response_type_url().data(), static_cast(this->response_type_url().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Method.response_type_url"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 4, this->response_type_url(), output); - } - - // bool response_streaming = 5; - if (this->response_streaming() != 0) { - ::google::protobuf::internal::WireFormatLite::WriteBool(5, this->response_streaming(), output); - } - - // repeated .google.protobuf.Option options = 6; - for (unsigned int i = 0, - n = static_cast(this->options_size()); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( - 6, this->options(static_cast(i)), output); - } - - // .google.protobuf.Syntax syntax = 7; - if (this->syntax() != 0) { - ::google::protobuf::internal::WireFormatLite::WriteEnum( - 7, this->syntax(), output); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - ::google::protobuf::internal::WireFormat::SerializeUnknownFields( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), output); - } - // @@protoc_insertion_point(serialize_end:google.protobuf.Method) -} - -::google::protobuf::uint8* Method::InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const { - (void)deterministic; // Unused - // @@protoc_insertion_point(serialize_to_array_start:google.protobuf.Method) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string name = 1; - if (this->name().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Method.name"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 1, this->name(), target); - } - - // string request_type_url = 2; - if (this->request_type_url().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->request_type_url().data(), static_cast(this->request_type_url().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Method.request_type_url"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 2, this->request_type_url(), target); - } - - // bool request_streaming = 3; - if (this->request_streaming() != 0) { - target = ::google::protobuf::internal::WireFormatLite::WriteBoolToArray(3, this->request_streaming(), target); - } - - // string response_type_url = 4; - if (this->response_type_url().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->response_type_url().data(), static_cast(this->response_type_url().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Method.response_type_url"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 4, this->response_type_url(), target); - } - - // bool response_streaming = 5; - if (this->response_streaming() != 0) { - target = ::google::protobuf::internal::WireFormatLite::WriteBoolToArray(5, this->response_streaming(), target); - } - - // repeated .google.protobuf.Option options = 6; - for (unsigned int i = 0, - n = static_cast(this->options_size()); i < n; i++) { - target = ::google::protobuf::internal::WireFormatLite:: - InternalWriteMessageToArray( - 6, this->options(static_cast(i)), deterministic, target); - } - - // .google.protobuf.Syntax syntax = 7; - if (this->syntax() != 0) { - target = ::google::protobuf::internal::WireFormatLite::WriteEnumToArray( - 7, this->syntax(), target); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), target); - } - // @@protoc_insertion_point(serialize_to_array_end:google.protobuf.Method) - return target; -} - -size_t Method::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Method) - size_t total_size = 0; - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - total_size += - ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance())); - } - // repeated .google.protobuf.Option options = 6; - { - unsigned int count = static_cast(this->options_size()); - total_size += 1UL * count; - for (unsigned int i = 0; i < count; i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSize( - this->options(static_cast(i))); - } - } - - // string name = 1; - if (this->name().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->name()); - } - - // string request_type_url = 2; - if (this->request_type_url().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->request_type_url()); - } - - // string response_type_url = 4; - if (this->response_type_url().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->response_type_url()); - } - - // bool request_streaming = 3; - if (this->request_streaming() != 0) { - total_size += 1 + 1; - } - - // bool response_streaming = 5; - if (this->response_streaming() != 0) { - total_size += 1 + 1; - } - - // .google.protobuf.Syntax syntax = 7; - if (this->syntax() != 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::EnumSize(this->syntax()); - } - - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = cached_size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); - return total_size; -} - -void Method::MergeFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_merge_from_start:google.protobuf.Method) - GOOGLE_DCHECK_NE(&from, this); - const Method* source = - ::google::protobuf::internal::DynamicCastToGenerated( - &from); - if (source == NULL) { - // @@protoc_insertion_point(generalized_merge_from_cast_fail:google.protobuf.Method) - ::google::protobuf::internal::ReflectionOps::Merge(from, this); - } else { - // @@protoc_insertion_point(generalized_merge_from_cast_success:google.protobuf.Method) - MergeFrom(*source); - } -} - -void Method::MergeFrom(const Method& from) { -// @@protoc_insertion_point(class_specific_merge_from_start:google.protobuf.Method) - GOOGLE_DCHECK_NE(&from, this); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - options_.MergeFrom(from.options_); - if (from.name().size() > 0) { - - name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.name_); - } - if (from.request_type_url().size() > 0) { - - request_type_url_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.request_type_url_); - } - if (from.response_type_url().size() > 0) { - - response_type_url_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.response_type_url_); - } - if (from.request_streaming() != 0) { - set_request_streaming(from.request_streaming()); - } - if (from.response_streaming() != 0) { - set_response_streaming(from.response_streaming()); - } - if (from.syntax() != 0) { - set_syntax(from.syntax()); - } -} - -void Method::CopyFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_copy_from_start:google.protobuf.Method) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -void Method::CopyFrom(const Method& from) { -// @@protoc_insertion_point(class_specific_copy_from_start:google.protobuf.Method) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool Method::IsInitialized() const { - return true; -} - -void Method::Swap(Method* other) { - if (other == this) return; - InternalSwap(other); -} -void Method::InternalSwap(Method* other) { - using std::swap; - options_.InternalSwap(&other->options_); - name_.Swap(&other->name_); - request_type_url_.Swap(&other->request_type_url_); - response_type_url_.Swap(&other->response_type_url_); - swap(request_streaming_, other->request_streaming_); - swap(response_streaming_, other->response_streaming_); - swap(syntax_, other->syntax_); - _internal_metadata_.Swap(&other->_internal_metadata_); - swap(_cached_size_, other->_cached_size_); -} - -::google::protobuf::Metadata Method::GetMetadata() const { - protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages]; -} - - -// =================================================================== - -void Mixin::InitAsDefaultInstance() { -} -#if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int Mixin::kNameFieldNumber; -const int Mixin::kRootFieldNumber; -#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 - -Mixin::Mixin() - : ::google::protobuf::Message(), _internal_metadata_(NULL) { - if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) { - ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMixin(); - } - SharedCtor(); - // @@protoc_insertion_point(constructor:google.protobuf.Mixin) -} -Mixin::Mixin(const Mixin& from) - : ::google::protobuf::Message(), - _internal_metadata_(NULL), - _cached_size_(0) { - _internal_metadata_.MergeFrom(from._internal_metadata_); - name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.name().size() > 0) { - name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.name_); - } - root_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.root().size() > 0) { - root_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.root_); - } - // @@protoc_insertion_point(copy_constructor:google.protobuf.Mixin) -} - -void Mixin::SharedCtor() { - name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - root_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - _cached_size_ = 0; -} - -Mixin::~Mixin() { - // @@protoc_insertion_point(destructor:google.protobuf.Mixin) - SharedDtor(); -} - -void Mixin::SharedDtor() { - name_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - root_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} - -void Mixin::SetCachedSize(int size) const { - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); -} -const ::google::protobuf::Descriptor* Mixin::descriptor() { - ::protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages].descriptor; -} - -const Mixin& Mixin::default_instance() { - ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMixin(); - return *internal_default_instance(); -} - -Mixin* Mixin::New(::google::protobuf::Arena* arena) const { - Mixin* n = new Mixin; - if (arena != NULL) { - arena->Own(n); - } - return n; -} - -void Mixin::Clear() { -// @@protoc_insertion_point(message_clear_start:google.protobuf.Mixin) - ::google::protobuf::uint32 cached_has_bits = 0; - // Prevent compiler warnings about cached_has_bits being unused - (void) cached_has_bits; - - name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - root_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - _internal_metadata_.Clear(); -} - -bool Mixin::MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) { -#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure - ::google::protobuf::uint32 tag; - // @@protoc_insertion_point(parse_start:google.protobuf.Mixin) - for (;;) { - ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); - tag = p.first; - if (!p.second) goto handle_unusual; - switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // string name = 1; - case 1: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_name())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Mixin.name")); - } else { - goto handle_unusual; - } - break; - } - - // string root = 2; - case 2: { - if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_root())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->root().data(), static_cast(this->root().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "google.protobuf.Mixin.root")); - } else { - goto handle_unusual; - } - break; - } - - default: { - handle_unusual: - if (tag == 0) { - goto success; - } - DO_(::google::protobuf::internal::WireFormat::SkipField( - input, tag, _internal_metadata_.mutable_unknown_fields())); - break; - } - } - } -success: - // @@protoc_insertion_point(parse_success:google.protobuf.Mixin) - return true; -failure: - // @@protoc_insertion_point(parse_failure:google.protobuf.Mixin) - return false; -#undef DO_ -} - -void Mixin::SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:google.protobuf.Mixin) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string name = 1; - if (this->name().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Mixin.name"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->name(), output); - } - - // string root = 2; - if (this->root().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->root().data(), static_cast(this->root().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Mixin.root"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 2, this->root(), output); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - ::google::protobuf::internal::WireFormat::SerializeUnknownFields( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), output); - } - // @@protoc_insertion_point(serialize_end:google.protobuf.Mixin) -} - -::google::protobuf::uint8* Mixin::InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const { - (void)deterministic; // Unused - // @@protoc_insertion_point(serialize_to_array_start:google.protobuf.Mixin) - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - // string name = 1; - if (this->name().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->name().data(), static_cast(this->name().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Mixin.name"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 1, this->name(), target); - } - - // string root = 2; - if (this->root().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->root().data(), static_cast(this->root().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "google.protobuf.Mixin.root"); - target = - ::google::protobuf::internal::WireFormatLite::WriteStringToArray( - 2, this->root(), target); - } - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()), target); - } - // @@protoc_insertion_point(serialize_to_array_end:google.protobuf.Mixin) - return target; -} - -size_t Mixin::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:google.protobuf.Mixin) - size_t total_size = 0; - - if ((_internal_metadata_.have_unknown_fields() && ::google::protobuf::internal::GetProto3PreserveUnknownsDefault())) { - total_size += - ::google::protobuf::internal::WireFormat::ComputeUnknownFieldsSize( - (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance())); - } - // string name = 1; - if (this->name().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->name()); - } - - // string root = 2; - if (this->root().size() > 0) { - total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->root()); - } - - int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); - GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); - _cached_size_ = cached_size; - GOOGLE_SAFE_CONCURRENT_WRITES_END(); - return total_size; -} - -void Mixin::MergeFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_merge_from_start:google.protobuf.Mixin) - GOOGLE_DCHECK_NE(&from, this); - const Mixin* source = - ::google::protobuf::internal::DynamicCastToGenerated( - &from); - if (source == NULL) { - // @@protoc_insertion_point(generalized_merge_from_cast_fail:google.protobuf.Mixin) - ::google::protobuf::internal::ReflectionOps::Merge(from, this); - } else { - // @@protoc_insertion_point(generalized_merge_from_cast_success:google.protobuf.Mixin) - MergeFrom(*source); - } -} - -void Mixin::MergeFrom(const Mixin& from) { -// @@protoc_insertion_point(class_specific_merge_from_start:google.protobuf.Mixin) - GOOGLE_DCHECK_NE(&from, this); - _internal_metadata_.MergeFrom(from._internal_metadata_); - ::google::protobuf::uint32 cached_has_bits = 0; - (void) cached_has_bits; - - if (from.name().size() > 0) { - - name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.name_); - } - if (from.root().size() > 0) { - - root_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.root_); - } -} - -void Mixin::CopyFrom(const ::google::protobuf::Message& from) { -// @@protoc_insertion_point(generalized_copy_from_start:google.protobuf.Mixin) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -void Mixin::CopyFrom(const Mixin& from) { -// @@protoc_insertion_point(class_specific_copy_from_start:google.protobuf.Mixin) - if (&from == this) return; - Clear(); - MergeFrom(from); -} - -bool Mixin::IsInitialized() const { - return true; -} - -void Mixin::Swap(Mixin* other) { - if (other == this) return; - InternalSwap(other); -} -void Mixin::InternalSwap(Mixin* other) { - using std::swap; - name_.Swap(&other->name_); - root_.Swap(&other->root_); - _internal_metadata_.Swap(&other->_internal_metadata_); - swap(_cached_size_, other->_cached_size_); -} - -::google::protobuf::Metadata Mixin::GetMetadata() const { - protobuf_google_2fprotobuf_2fapi_2eproto::protobuf_AssignDescriptorsOnce(); - return ::protobuf_google_2fprotobuf_2fapi_2eproto::file_level_metadata[kIndexInFileMessages]; -} - - -// @@protoc_insertion_point(namespace_scope) -} // namespace protobuf -} // namespace google - -// @@protoc_insertion_point(global_scope) diff --git a/3rdparty/protobuf/src/google/protobuf/api.pb.h b/3rdparty/protobuf/src/google/protobuf/api.pb.h deleted file mode 100644 index 96ff3f159b07..000000000000 --- a/3rdparty/protobuf/src/google/protobuf/api.pb.h +++ /dev/null @@ -1,1165 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: google/protobuf/api.proto - -#ifndef PROTOBUF_google_2fprotobuf_2fapi_2eproto__INCLUDED -#define PROTOBUF_google_2fprotobuf_2fapi_2eproto__INCLUDED - -#include - -#include - -#if GOOGLE_PROTOBUF_VERSION < 3005000 -#error This file was generated by a newer version of protoc which is -#error incompatible with your Protocol Buffer headers. Please update -#error your headers. -#endif -#if 3005001 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION -#error This file was generated by an older version of protoc which is -#error incompatible with your Protocol Buffer headers. Please -#error regenerate this file with a newer version of protoc. -#endif - -#include -#include -#include -#include -#include -#include -#include -#include // IWYU pragma: export -#include // IWYU pragma: export -#include -#include -#include -// @@protoc_insertion_point(includes) - -namespace protobuf_google_2fprotobuf_2fapi_2eproto { -// Internal implementation detail -- do not use these members. -struct LIBPROTOBUF_EXPORT TableStruct { - static const ::google::protobuf::internal::ParseTableField entries[]; - static const ::google::protobuf::internal::AuxillaryParseTableField aux[]; - static const ::google::protobuf::internal::ParseTable schema[3]; - static const ::google::protobuf::internal::FieldMetadata field_metadata[]; - static const ::google::protobuf::internal::SerializationTable serialization_table[]; - static const ::google::protobuf::uint32 offsets[]; -}; -void LIBPROTOBUF_EXPORT AddDescriptors(); -void LIBPROTOBUF_EXPORT InitDefaultsApiImpl(); -void LIBPROTOBUF_EXPORT InitDefaultsApi(); -void LIBPROTOBUF_EXPORT InitDefaultsMethodImpl(); -void LIBPROTOBUF_EXPORT InitDefaultsMethod(); -void LIBPROTOBUF_EXPORT InitDefaultsMixinImpl(); -void LIBPROTOBUF_EXPORT InitDefaultsMixin(); -inline void LIBPROTOBUF_EXPORT InitDefaults() { - InitDefaultsApi(); - InitDefaultsMethod(); - InitDefaultsMixin(); -} -} // namespace protobuf_google_2fprotobuf_2fapi_2eproto -namespace google { -namespace protobuf { -class Api; -class ApiDefaultTypeInternal; -LIBPROTOBUF_EXPORT extern ApiDefaultTypeInternal _Api_default_instance_; -class Method; -class MethodDefaultTypeInternal; -LIBPROTOBUF_EXPORT extern MethodDefaultTypeInternal _Method_default_instance_; -class Mixin; -class MixinDefaultTypeInternal; -LIBPROTOBUF_EXPORT extern MixinDefaultTypeInternal _Mixin_default_instance_; -} // namespace protobuf -} // namespace google -namespace google { -namespace protobuf { - -// =================================================================== - -class LIBPROTOBUF_EXPORT Api : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Api) */ { - public: - Api(); - virtual ~Api(); - - Api(const Api& from); - - inline Api& operator=(const Api& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 - Api(Api&& from) noexcept - : Api() { - *this = ::std::move(from); - } - - inline Api& operator=(Api&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { - if (this != &from) InternalSwap(&from); - } else { - CopyFrom(from); - } - return *this; - } - #endif - static const ::google::protobuf::Descriptor* descriptor(); - static const Api& default_instance(); - - static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY - static inline const Api* internal_default_instance() { - return reinterpret_cast( - &_Api_default_instance_); - } - static PROTOBUF_CONSTEXPR int const kIndexInFileMessages = - 0; - - void Swap(Api* other); - friend void swap(Api& a, Api& b) { - a.Swap(&b); - } - - // implements Message ---------------------------------------------- - - inline Api* New() const PROTOBUF_FINAL { return New(NULL); } - - Api* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL; - void CopyFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void MergeFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void CopyFrom(const Api& from); - void MergeFrom(const Api& from); - void Clear() PROTOBUF_FINAL; - bool IsInitialized() const PROTOBUF_FINAL; - - size_t ByteSizeLong() const PROTOBUF_FINAL; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PROTOBUF_FINAL; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PROTOBUF_FINAL; - ::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const PROTOBUF_FINAL; - int GetCachedSize() const PROTOBUF_FINAL { return _cached_size_; } - private: - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PROTOBUF_FINAL; - void InternalSwap(Api* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; - } - public: - - ::google::protobuf::Metadata GetMetadata() const PROTOBUF_FINAL; - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - // repeated .google.protobuf.Method methods = 2; - int methods_size() const; - void clear_methods(); - static const int kMethodsFieldNumber = 2; - const ::google::protobuf::Method& methods(int index) const; - ::google::protobuf::Method* mutable_methods(int index); - ::google::protobuf::Method* add_methods(); - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Method >* - mutable_methods(); - const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Method >& - methods() const; - - // repeated .google.protobuf.Option options = 3; - int options_size() const; - void clear_options(); - static const int kOptionsFieldNumber = 3; - const ::google::protobuf::Option& options(int index) const; - ::google::protobuf::Option* mutable_options(int index); - ::google::protobuf::Option* add_options(); - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >* - mutable_options(); - const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >& - options() const; - - // repeated .google.protobuf.Mixin mixins = 6; - int mixins_size() const; - void clear_mixins(); - static const int kMixinsFieldNumber = 6; - const ::google::protobuf::Mixin& mixins(int index) const; - ::google::protobuf::Mixin* mutable_mixins(int index); - ::google::protobuf::Mixin* add_mixins(); - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Mixin >* - mutable_mixins(); - const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Mixin >& - mixins() const; - - // string name = 1; - void clear_name(); - static const int kNameFieldNumber = 1; - const ::std::string& name() const; - void set_name(const ::std::string& value); - #if LANG_CXX11 - void set_name(::std::string&& value); - #endif - void set_name(const char* value); - void set_name(const char* value, size_t size); - ::std::string* mutable_name(); - ::std::string* release_name(); - void set_allocated_name(::std::string* name); - - // string version = 4; - void clear_version(); - static const int kVersionFieldNumber = 4; - const ::std::string& version() const; - void set_version(const ::std::string& value); - #if LANG_CXX11 - void set_version(::std::string&& value); - #endif - void set_version(const char* value); - void set_version(const char* value, size_t size); - ::std::string* mutable_version(); - ::std::string* release_version(); - void set_allocated_version(::std::string* version); - - // .google.protobuf.SourceContext source_context = 5; - bool has_source_context() const; - void clear_source_context(); - static const int kSourceContextFieldNumber = 5; - const ::google::protobuf::SourceContext& source_context() const; - ::google::protobuf::SourceContext* release_source_context(); - ::google::protobuf::SourceContext* mutable_source_context(); - void set_allocated_source_context(::google::protobuf::SourceContext* source_context); - - // .google.protobuf.Syntax syntax = 7; - void clear_syntax(); - static const int kSyntaxFieldNumber = 7; - ::google::protobuf::Syntax syntax() const; - void set_syntax(::google::protobuf::Syntax value); - - // @@protoc_insertion_point(class_scope:google.protobuf.Api) - private: - - ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Method > methods_; - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option > options_; - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Mixin > mixins_; - ::google::protobuf::internal::ArenaStringPtr name_; - ::google::protobuf::internal::ArenaStringPtr version_; - ::google::protobuf::SourceContext* source_context_; - int syntax_; - mutable int _cached_size_; - friend struct ::protobuf_google_2fprotobuf_2fapi_2eproto::TableStruct; - friend void ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsApiImpl(); -}; -// ------------------------------------------------------------------- - -class LIBPROTOBUF_EXPORT Method : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Method) */ { - public: - Method(); - virtual ~Method(); - - Method(const Method& from); - - inline Method& operator=(const Method& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 - Method(Method&& from) noexcept - : Method() { - *this = ::std::move(from); - } - - inline Method& operator=(Method&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { - if (this != &from) InternalSwap(&from); - } else { - CopyFrom(from); - } - return *this; - } - #endif - static const ::google::protobuf::Descriptor* descriptor(); - static const Method& default_instance(); - - static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY - static inline const Method* internal_default_instance() { - return reinterpret_cast( - &_Method_default_instance_); - } - static PROTOBUF_CONSTEXPR int const kIndexInFileMessages = - 1; - - void Swap(Method* other); - friend void swap(Method& a, Method& b) { - a.Swap(&b); - } - - // implements Message ---------------------------------------------- - - inline Method* New() const PROTOBUF_FINAL { return New(NULL); } - - Method* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL; - void CopyFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void MergeFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void CopyFrom(const Method& from); - void MergeFrom(const Method& from); - void Clear() PROTOBUF_FINAL; - bool IsInitialized() const PROTOBUF_FINAL; - - size_t ByteSizeLong() const PROTOBUF_FINAL; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PROTOBUF_FINAL; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PROTOBUF_FINAL; - ::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const PROTOBUF_FINAL; - int GetCachedSize() const PROTOBUF_FINAL { return _cached_size_; } - private: - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PROTOBUF_FINAL; - void InternalSwap(Method* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; - } - public: - - ::google::protobuf::Metadata GetMetadata() const PROTOBUF_FINAL; - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - // repeated .google.protobuf.Option options = 6; - int options_size() const; - void clear_options(); - static const int kOptionsFieldNumber = 6; - const ::google::protobuf::Option& options(int index) const; - ::google::protobuf::Option* mutable_options(int index); - ::google::protobuf::Option* add_options(); - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >* - mutable_options(); - const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >& - options() const; - - // string name = 1; - void clear_name(); - static const int kNameFieldNumber = 1; - const ::std::string& name() const; - void set_name(const ::std::string& value); - #if LANG_CXX11 - void set_name(::std::string&& value); - #endif - void set_name(const char* value); - void set_name(const char* value, size_t size); - ::std::string* mutable_name(); - ::std::string* release_name(); - void set_allocated_name(::std::string* name); - - // string request_type_url = 2; - void clear_request_type_url(); - static const int kRequestTypeUrlFieldNumber = 2; - const ::std::string& request_type_url() const; - void set_request_type_url(const ::std::string& value); - #if LANG_CXX11 - void set_request_type_url(::std::string&& value); - #endif - void set_request_type_url(const char* value); - void set_request_type_url(const char* value, size_t size); - ::std::string* mutable_request_type_url(); - ::std::string* release_request_type_url(); - void set_allocated_request_type_url(::std::string* request_type_url); - - // string response_type_url = 4; - void clear_response_type_url(); - static const int kResponseTypeUrlFieldNumber = 4; - const ::std::string& response_type_url() const; - void set_response_type_url(const ::std::string& value); - #if LANG_CXX11 - void set_response_type_url(::std::string&& value); - #endif - void set_response_type_url(const char* value); - void set_response_type_url(const char* value, size_t size); - ::std::string* mutable_response_type_url(); - ::std::string* release_response_type_url(); - void set_allocated_response_type_url(::std::string* response_type_url); - - // bool request_streaming = 3; - void clear_request_streaming(); - static const int kRequestStreamingFieldNumber = 3; - bool request_streaming() const; - void set_request_streaming(bool value); - - // bool response_streaming = 5; - void clear_response_streaming(); - static const int kResponseStreamingFieldNumber = 5; - bool response_streaming() const; - void set_response_streaming(bool value); - - // .google.protobuf.Syntax syntax = 7; - void clear_syntax(); - static const int kSyntaxFieldNumber = 7; - ::google::protobuf::Syntax syntax() const; - void set_syntax(::google::protobuf::Syntax value); - - // @@protoc_insertion_point(class_scope:google.protobuf.Method) - private: - - ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; - ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option > options_; - ::google::protobuf::internal::ArenaStringPtr name_; - ::google::protobuf::internal::ArenaStringPtr request_type_url_; - ::google::protobuf::internal::ArenaStringPtr response_type_url_; - bool request_streaming_; - bool response_streaming_; - int syntax_; - mutable int _cached_size_; - friend struct ::protobuf_google_2fprotobuf_2fapi_2eproto::TableStruct; - friend void ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMethodImpl(); -}; -// ------------------------------------------------------------------- - -class LIBPROTOBUF_EXPORT Mixin : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:google.protobuf.Mixin) */ { - public: - Mixin(); - virtual ~Mixin(); - - Mixin(const Mixin& from); - - inline Mixin& operator=(const Mixin& from) { - CopyFrom(from); - return *this; - } - #if LANG_CXX11 - Mixin(Mixin&& from) noexcept - : Mixin() { - *this = ::std::move(from); - } - - inline Mixin& operator=(Mixin&& from) noexcept { - if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { - if (this != &from) InternalSwap(&from); - } else { - CopyFrom(from); - } - return *this; - } - #endif - static const ::google::protobuf::Descriptor* descriptor(); - static const Mixin& default_instance(); - - static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY - static inline const Mixin* internal_default_instance() { - return reinterpret_cast( - &_Mixin_default_instance_); - } - static PROTOBUF_CONSTEXPR int const kIndexInFileMessages = - 2; - - void Swap(Mixin* other); - friend void swap(Mixin& a, Mixin& b) { - a.Swap(&b); - } - - // implements Message ---------------------------------------------- - - inline Mixin* New() const PROTOBUF_FINAL { return New(NULL); } - - Mixin* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL; - void CopyFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void MergeFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL; - void CopyFrom(const Mixin& from); - void MergeFrom(const Mixin& from); - void Clear() PROTOBUF_FINAL; - bool IsInitialized() const PROTOBUF_FINAL; - - size_t ByteSizeLong() const PROTOBUF_FINAL; - bool MergePartialFromCodedStream( - ::google::protobuf::io::CodedInputStream* input) PROTOBUF_FINAL; - void SerializeWithCachedSizes( - ::google::protobuf::io::CodedOutputStream* output) const PROTOBUF_FINAL; - ::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray( - bool deterministic, ::google::protobuf::uint8* target) const PROTOBUF_FINAL; - int GetCachedSize() const PROTOBUF_FINAL { return _cached_size_; } - private: - void SharedCtor(); - void SharedDtor(); - void SetCachedSize(int size) const PROTOBUF_FINAL; - void InternalSwap(Mixin* other); - private: - inline ::google::protobuf::Arena* GetArenaNoVirtual() const { - return NULL; - } - inline void* MaybeArenaPtr() const { - return NULL; - } - public: - - ::google::protobuf::Metadata GetMetadata() const PROTOBUF_FINAL; - - // nested types ---------------------------------------------------- - - // accessors ------------------------------------------------------- - - // string name = 1; - void clear_name(); - static const int kNameFieldNumber = 1; - const ::std::string& name() const; - void set_name(const ::std::string& value); - #if LANG_CXX11 - void set_name(::std::string&& value); - #endif - void set_name(const char* value); - void set_name(const char* value, size_t size); - ::std::string* mutable_name(); - ::std::string* release_name(); - void set_allocated_name(::std::string* name); - - // string root = 2; - void clear_root(); - static const int kRootFieldNumber = 2; - const ::std::string& root() const; - void set_root(const ::std::string& value); - #if LANG_CXX11 - void set_root(::std::string&& value); - #endif - void set_root(const char* value); - void set_root(const char* value, size_t size); - ::std::string* mutable_root(); - ::std::string* release_root(); - void set_allocated_root(::std::string* root); - - // @@protoc_insertion_point(class_scope:google.protobuf.Mixin) - private: - - ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; - ::google::protobuf::internal::ArenaStringPtr name_; - ::google::protobuf::internal::ArenaStringPtr root_; - mutable int _cached_size_; - friend struct ::protobuf_google_2fprotobuf_2fapi_2eproto::TableStruct; - friend void ::protobuf_google_2fprotobuf_2fapi_2eproto::InitDefaultsMixinImpl(); -}; -// =================================================================== - - -// =================================================================== - -#ifdef __GNUC__ - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wstrict-aliasing" -#endif // __GNUC__ -// Api - -// string name = 1; -inline void Api::clear_name() { - name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Api::name() const { - // @@protoc_insertion_point(field_get:google.protobuf.Api.name) - return name_.GetNoArena(); -} -inline void Api::set_name(const ::std::string& value) { - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Api.name) -} -#if LANG_CXX11 -inline void Api::set_name(::std::string&& value) { - - name_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Api.name) -} -#endif -inline void Api::set_name(const char* value) { - GOOGLE_DCHECK(value != NULL); - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Api.name) -} -inline void Api::set_name(const char* value, size_t size) { - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Api.name) -} -inline ::std::string* Api::mutable_name() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Api.name) - return name_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Api::release_name() { - // @@protoc_insertion_point(field_release:google.protobuf.Api.name) - - return name_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Api::set_allocated_name(::std::string* name) { - if (name != NULL) { - - } else { - - } - name_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), name); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Api.name) -} - -// repeated .google.protobuf.Method methods = 2; -inline int Api::methods_size() const { - return methods_.size(); -} -inline void Api::clear_methods() { - methods_.Clear(); -} -inline const ::google::protobuf::Method& Api::methods(int index) const { - // @@protoc_insertion_point(field_get:google.protobuf.Api.methods) - return methods_.Get(index); -} -inline ::google::protobuf::Method* Api::mutable_methods(int index) { - // @@protoc_insertion_point(field_mutable:google.protobuf.Api.methods) - return methods_.Mutable(index); -} -inline ::google::protobuf::Method* Api::add_methods() { - // @@protoc_insertion_point(field_add:google.protobuf.Api.methods) - return methods_.Add(); -} -inline ::google::protobuf::RepeatedPtrField< ::google::protobuf::Method >* -Api::mutable_methods() { - // @@protoc_insertion_point(field_mutable_list:google.protobuf.Api.methods) - return &methods_; -} -inline const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Method >& -Api::methods() const { - // @@protoc_insertion_point(field_list:google.protobuf.Api.methods) - return methods_; -} - -// repeated .google.protobuf.Option options = 3; -inline int Api::options_size() const { - return options_.size(); -} -inline const ::google::protobuf::Option& Api::options(int index) const { - // @@protoc_insertion_point(field_get:google.protobuf.Api.options) - return options_.Get(index); -} -inline ::google::protobuf::Option* Api::mutable_options(int index) { - // @@protoc_insertion_point(field_mutable:google.protobuf.Api.options) - return options_.Mutable(index); -} -inline ::google::protobuf::Option* Api::add_options() { - // @@protoc_insertion_point(field_add:google.protobuf.Api.options) - return options_.Add(); -} -inline ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >* -Api::mutable_options() { - // @@protoc_insertion_point(field_mutable_list:google.protobuf.Api.options) - return &options_; -} -inline const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >& -Api::options() const { - // @@protoc_insertion_point(field_list:google.protobuf.Api.options) - return options_; -} - -// string version = 4; -inline void Api::clear_version() { - version_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Api::version() const { - // @@protoc_insertion_point(field_get:google.protobuf.Api.version) - return version_.GetNoArena(); -} -inline void Api::set_version(const ::std::string& value) { - - version_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Api.version) -} -#if LANG_CXX11 -inline void Api::set_version(::std::string&& value) { - - version_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Api.version) -} -#endif -inline void Api::set_version(const char* value) { - GOOGLE_DCHECK(value != NULL); - - version_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Api.version) -} -inline void Api::set_version(const char* value, size_t size) { - - version_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Api.version) -} -inline ::std::string* Api::mutable_version() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Api.version) - return version_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Api::release_version() { - // @@protoc_insertion_point(field_release:google.protobuf.Api.version) - - return version_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Api::set_allocated_version(::std::string* version) { - if (version != NULL) { - - } else { - - } - version_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), version); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Api.version) -} - -// .google.protobuf.SourceContext source_context = 5; -inline bool Api::has_source_context() const { - return this != internal_default_instance() && source_context_ != NULL; -} -inline const ::google::protobuf::SourceContext& Api::source_context() const { - const ::google::protobuf::SourceContext* p = source_context_; - // @@protoc_insertion_point(field_get:google.protobuf.Api.source_context) - return p != NULL ? *p : *reinterpret_cast( - &::google::protobuf::_SourceContext_default_instance_); -} -inline ::google::protobuf::SourceContext* Api::release_source_context() { - // @@protoc_insertion_point(field_release:google.protobuf.Api.source_context) - - ::google::protobuf::SourceContext* temp = source_context_; - source_context_ = NULL; - return temp; -} -inline ::google::protobuf::SourceContext* Api::mutable_source_context() { - - if (source_context_ == NULL) { - source_context_ = new ::google::protobuf::SourceContext; - } - // @@protoc_insertion_point(field_mutable:google.protobuf.Api.source_context) - return source_context_; -} -inline void Api::set_allocated_source_context(::google::protobuf::SourceContext* source_context) { - ::google::protobuf::Arena* message_arena = GetArenaNoVirtual(); - if (message_arena == NULL) { - delete reinterpret_cast< ::google::protobuf::MessageLite*>(source_context_); - } - if (source_context) { - ::google::protobuf::Arena* submessage_arena = NULL; - if (message_arena != submessage_arena) { - source_context = ::google::protobuf::internal::GetOwnedMessage( - message_arena, source_context, submessage_arena); - } - - } else { - - } - source_context_ = source_context; - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Api.source_context) -} - -// repeated .google.protobuf.Mixin mixins = 6; -inline int Api::mixins_size() const { - return mixins_.size(); -} -inline void Api::clear_mixins() { - mixins_.Clear(); -} -inline const ::google::protobuf::Mixin& Api::mixins(int index) const { - // @@protoc_insertion_point(field_get:google.protobuf.Api.mixins) - return mixins_.Get(index); -} -inline ::google::protobuf::Mixin* Api::mutable_mixins(int index) { - // @@protoc_insertion_point(field_mutable:google.protobuf.Api.mixins) - return mixins_.Mutable(index); -} -inline ::google::protobuf::Mixin* Api::add_mixins() { - // @@protoc_insertion_point(field_add:google.protobuf.Api.mixins) - return mixins_.Add(); -} -inline ::google::protobuf::RepeatedPtrField< ::google::protobuf::Mixin >* -Api::mutable_mixins() { - // @@protoc_insertion_point(field_mutable_list:google.protobuf.Api.mixins) - return &mixins_; -} -inline const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Mixin >& -Api::mixins() const { - // @@protoc_insertion_point(field_list:google.protobuf.Api.mixins) - return mixins_; -} - -// .google.protobuf.Syntax syntax = 7; -inline void Api::clear_syntax() { - syntax_ = 0; -} -inline ::google::protobuf::Syntax Api::syntax() const { - // @@protoc_insertion_point(field_get:google.protobuf.Api.syntax) - return static_cast< ::google::protobuf::Syntax >(syntax_); -} -inline void Api::set_syntax(::google::protobuf::Syntax value) { - - syntax_ = value; - // @@protoc_insertion_point(field_set:google.protobuf.Api.syntax) -} - -// ------------------------------------------------------------------- - -// Method - -// string name = 1; -inline void Method::clear_name() { - name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Method::name() const { - // @@protoc_insertion_point(field_get:google.protobuf.Method.name) - return name_.GetNoArena(); -} -inline void Method::set_name(const ::std::string& value) { - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Method.name) -} -#if LANG_CXX11 -inline void Method::set_name(::std::string&& value) { - - name_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.name) -} -#endif -inline void Method::set_name(const char* value) { - GOOGLE_DCHECK(value != NULL); - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Method.name) -} -inline void Method::set_name(const char* value, size_t size) { - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.name) -} -inline ::std::string* Method::mutable_name() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Method.name) - return name_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Method::release_name() { - // @@protoc_insertion_point(field_release:google.protobuf.Method.name) - - return name_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Method::set_allocated_name(::std::string* name) { - if (name != NULL) { - - } else { - - } - name_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), name); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Method.name) -} - -// string request_type_url = 2; -inline void Method::clear_request_type_url() { - request_type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Method::request_type_url() const { - // @@protoc_insertion_point(field_get:google.protobuf.Method.request_type_url) - return request_type_url_.GetNoArena(); -} -inline void Method::set_request_type_url(const ::std::string& value) { - - request_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Method.request_type_url) -} -#if LANG_CXX11 -inline void Method::set_request_type_url(::std::string&& value) { - - request_type_url_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.request_type_url) -} -#endif -inline void Method::set_request_type_url(const char* value) { - GOOGLE_DCHECK(value != NULL); - - request_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Method.request_type_url) -} -inline void Method::set_request_type_url(const char* value, size_t size) { - - request_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.request_type_url) -} -inline ::std::string* Method::mutable_request_type_url() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Method.request_type_url) - return request_type_url_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Method::release_request_type_url() { - // @@protoc_insertion_point(field_release:google.protobuf.Method.request_type_url) - - return request_type_url_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Method::set_allocated_request_type_url(::std::string* request_type_url) { - if (request_type_url != NULL) { - - } else { - - } - request_type_url_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), request_type_url); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Method.request_type_url) -} - -// bool request_streaming = 3; -inline void Method::clear_request_streaming() { - request_streaming_ = false; -} -inline bool Method::request_streaming() const { - // @@protoc_insertion_point(field_get:google.protobuf.Method.request_streaming) - return request_streaming_; -} -inline void Method::set_request_streaming(bool value) { - - request_streaming_ = value; - // @@protoc_insertion_point(field_set:google.protobuf.Method.request_streaming) -} - -// string response_type_url = 4; -inline void Method::clear_response_type_url() { - response_type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Method::response_type_url() const { - // @@protoc_insertion_point(field_get:google.protobuf.Method.response_type_url) - return response_type_url_.GetNoArena(); -} -inline void Method::set_response_type_url(const ::std::string& value) { - - response_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Method.response_type_url) -} -#if LANG_CXX11 -inline void Method::set_response_type_url(::std::string&& value) { - - response_type_url_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Method.response_type_url) -} -#endif -inline void Method::set_response_type_url(const char* value) { - GOOGLE_DCHECK(value != NULL); - - response_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Method.response_type_url) -} -inline void Method::set_response_type_url(const char* value, size_t size) { - - response_type_url_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Method.response_type_url) -} -inline ::std::string* Method::mutable_response_type_url() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Method.response_type_url) - return response_type_url_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Method::release_response_type_url() { - // @@protoc_insertion_point(field_release:google.protobuf.Method.response_type_url) - - return response_type_url_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Method::set_allocated_response_type_url(::std::string* response_type_url) { - if (response_type_url != NULL) { - - } else { - - } - response_type_url_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), response_type_url); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Method.response_type_url) -} - -// bool response_streaming = 5; -inline void Method::clear_response_streaming() { - response_streaming_ = false; -} -inline bool Method::response_streaming() const { - // @@protoc_insertion_point(field_get:google.protobuf.Method.response_streaming) - return response_streaming_; -} -inline void Method::set_response_streaming(bool value) { - - response_streaming_ = value; - // @@protoc_insertion_point(field_set:google.protobuf.Method.response_streaming) -} - -// repeated .google.protobuf.Option options = 6; -inline int Method::options_size() const { - return options_.size(); -} -inline const ::google::protobuf::Option& Method::options(int index) const { - // @@protoc_insertion_point(field_get:google.protobuf.Method.options) - return options_.Get(index); -} -inline ::google::protobuf::Option* Method::mutable_options(int index) { - // @@protoc_insertion_point(field_mutable:google.protobuf.Method.options) - return options_.Mutable(index); -} -inline ::google::protobuf::Option* Method::add_options() { - // @@protoc_insertion_point(field_add:google.protobuf.Method.options) - return options_.Add(); -} -inline ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >* -Method::mutable_options() { - // @@protoc_insertion_point(field_mutable_list:google.protobuf.Method.options) - return &options_; -} -inline const ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option >& -Method::options() const { - // @@protoc_insertion_point(field_list:google.protobuf.Method.options) - return options_; -} - -// .google.protobuf.Syntax syntax = 7; -inline void Method::clear_syntax() { - syntax_ = 0; -} -inline ::google::protobuf::Syntax Method::syntax() const { - // @@protoc_insertion_point(field_get:google.protobuf.Method.syntax) - return static_cast< ::google::protobuf::Syntax >(syntax_); -} -inline void Method::set_syntax(::google::protobuf::Syntax value) { - - syntax_ = value; - // @@protoc_insertion_point(field_set:google.protobuf.Method.syntax) -} - -// ------------------------------------------------------------------- - -// Mixin - -// string name = 1; -inline void Mixin::clear_name() { - name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Mixin::name() const { - // @@protoc_insertion_point(field_get:google.protobuf.Mixin.name) - return name_.GetNoArena(); -} -inline void Mixin::set_name(const ::std::string& value) { - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Mixin.name) -} -#if LANG_CXX11 -inline void Mixin::set_name(::std::string&& value) { - - name_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Mixin.name) -} -#endif -inline void Mixin::set_name(const char* value) { - GOOGLE_DCHECK(value != NULL); - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Mixin.name) -} -inline void Mixin::set_name(const char* value, size_t size) { - - name_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Mixin.name) -} -inline ::std::string* Mixin::mutable_name() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Mixin.name) - return name_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Mixin::release_name() { - // @@protoc_insertion_point(field_release:google.protobuf.Mixin.name) - - return name_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Mixin::set_allocated_name(::std::string* name) { - if (name != NULL) { - - } else { - - } - name_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), name); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Mixin.name) -} - -// string root = 2; -inline void Mixin::clear_root() { - root_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline const ::std::string& Mixin::root() const { - // @@protoc_insertion_point(field_get:google.protobuf.Mixin.root) - return root_.GetNoArena(); -} -inline void Mixin::set_root(const ::std::string& value) { - - root_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:google.protobuf.Mixin.root) -} -#if LANG_CXX11 -inline void Mixin::set_root(::std::string&& value) { - - root_.SetNoArena( - &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:google.protobuf.Mixin.root) -} -#endif -inline void Mixin::set_root(const char* value) { - GOOGLE_DCHECK(value != NULL); - - root_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:google.protobuf.Mixin.root) -} -inline void Mixin::set_root(const char* value, size_t size) { - - root_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:google.protobuf.Mixin.root) -} -inline ::std::string* Mixin::mutable_root() { - - // @@protoc_insertion_point(field_mutable:google.protobuf.Mixin.root) - return root_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline ::std::string* Mixin::release_root() { - // @@protoc_insertion_point(field_release:google.protobuf.Mixin.root) - - return root_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); -} -inline void Mixin::set_allocated_root(::std::string* root) { - if (root != NULL) { - - } else { - - } - root_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), root); - // @@protoc_insertion_point(field_set_allocated:google.protobuf.Mixin.root) -} - -#ifdef __GNUC__ - #pragma GCC diagnostic pop -#endif // __GNUC__ -// ------------------------------------------------------------------- - -// ------------------------------------------------------------------- - - -// @@protoc_insertion_point(namespace_scope) - -} // namespace protobuf -} // namespace google - -// @@protoc_insertion_point(global_scope) - -#endif // PROTOBUF_google_2fprotobuf_2fapi_2eproto__INCLUDED diff --git a/3rdparty/protobuf/src/google/protobuf/arena.cc b/3rdparty/protobuf/src/google/protobuf/arena.cc index e92899888d18..7624e0b2f3fa 100644 --- a/3rdparty/protobuf/src/google/protobuf/arena.cc +++ b/3rdparty/protobuf/src/google/protobuf/arena.cc @@ -31,355 +31,481 @@ #include #include +#include +#include +#include #include +#include +#include +#include #ifdef ADDRESS_SANITIZER #include #endif // ADDRESS_SANITIZER -#include +#include namespace google { -static const size_t kMinCleanupListElements = 8; -static const size_t kMaxCleanupListElements = 64; // 1kB on 64-bit. - namespace protobuf { namespace internal { - -google::protobuf::internal::SequenceNumber ArenaImpl::lifecycle_id_generator_; -#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) -ArenaImpl::ThreadCache& ArenaImpl::thread_cache() { - static internal::ThreadLocalStorage* thread_cache_ = - new internal::ThreadLocalStorage(); - return *thread_cache_->Get(); -} -#elif defined(PROTOBUF_USE_DLLS) -ArenaImpl::ThreadCache& ArenaImpl::thread_cache() { - static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_ = { -1, NULL }; - return thread_cache_; +static SerialArena::Memory AllocateMemory(const AllocationPolicy* policy_ptr, + size_t last_size, size_t min_bytes) { + AllocationPolicy policy; // default policy + if (policy_ptr) policy = *policy_ptr; + size_t size; + if (last_size != 0) { + // Double the current block size, up to a limit. + auto max_size = policy.max_block_size; + size = std::min(2 * last_size, max_size); + } else { + size = policy.start_block_size; + } + // Verify that min_bytes + kBlockHeaderSize won't overflow. + GOOGLE_CHECK_LE(min_bytes, + std::numeric_limits::max() - SerialArena::kBlockHeaderSize); + size = std::max(size, SerialArena::kBlockHeaderSize + min_bytes); + + void* mem; + if (policy.block_alloc == nullptr) { + mem = ::operator new(size); + } else { + mem = policy.block_alloc(size); + } + return {mem, size}; } + +class GetDeallocator { + public: + GetDeallocator(const AllocationPolicy* policy, size_t* space_allocated) + : dealloc_(policy ? policy->block_dealloc : nullptr), + space_allocated_(space_allocated) {} + + void operator()(SerialArena::Memory mem) const { +#ifdef ADDRESS_SANITIZER + // This memory was provided by the underlying allocator as unpoisoned, + // so return it in an unpoisoned state. + ASAN_UNPOISON_MEMORY_REGION(mem.ptr, mem.size); +#endif // ADDRESS_SANITIZER + if (dealloc_) { + dealloc_(mem.ptr, mem.size); + } else { +#if defined(__GXX_DELETE_WITH_SIZE__) || defined(__cpp_sized_deallocation) + ::operator delete(mem.ptr, mem.size); #else -GOOGLE_THREAD_LOCAL ArenaImpl::ThreadCache ArenaImpl::thread_cache_ = {-1, NULL}; + ::operator delete(mem.ptr); #endif - -void ArenaImpl::Init() { - lifecycle_id_ = lifecycle_id_generator_.GetNext(); - google::protobuf::internal::NoBarrier_Store(&hint_, 0); - google::protobuf::internal::NoBarrier_Store(&threads_, 0); - - if (initial_block_) { - // Thread which calls Init() owns the first block. This allows the - // single-threaded case to allocate on the first block without having to - // perform atomic operations. - InitBlock(initial_block_, &thread_cache(), options_.initial_block_size); - ThreadInfo* info = NewThreadInfo(initial_block_); - info->next = NULL; - google::protobuf::internal::NoBarrier_Store(&threads_, - reinterpret_cast(info)); - google::protobuf::internal::NoBarrier_Store(&space_allocated_, - options_.initial_block_size); - CacheBlock(initial_block_); - } else { - google::protobuf::internal::NoBarrier_Store(&space_allocated_, 0); + } + *space_allocated_ += mem.size; } -} -ArenaImpl::~ArenaImpl() { - // Have to do this in a first pass, because some of the destructors might - // refer to memory in other blocks. - CleanupList(); - FreeBlocks(); + private: + void (*dealloc_)(void*, size_t); + size_t* space_allocated_; +}; + +SerialArena::SerialArena(Block* b, void* owner) : space_allocated_(b->size) { + owner_ = owner; + head_ = b; + ptr_ = b->Pointer(kBlockHeaderSize + ThreadSafeArena::kSerialArenaSize); + limit_ = b->Pointer(b->size & static_cast(-8)); } -uint64 ArenaImpl::Reset() { - // Have to do this in a first pass, because some of the destructors might - // refer to memory in other blocks. - CleanupList(); - uint64 space_allocated = FreeBlocks(); - Init(); +SerialArena* SerialArena::New(Memory mem, void* owner) { + GOOGLE_DCHECK_LE(kBlockHeaderSize + ThreadSafeArena::kSerialArenaSize, mem.size); - return space_allocated; + auto b = new (mem.ptr) Block{nullptr, mem.size}; + return new (b->Pointer(kBlockHeaderSize)) SerialArena(b, owner); } -ArenaImpl::Block* ArenaImpl::NewBlock(void* me, Block* my_last_block, - size_t min_bytes) { - size_t size; - if (my_last_block != NULL) { - // Double the current block size, up to a limit. - size = std::min(2 * my_last_block->size, options_.max_block_size); - } else { - size = options_.start_block_size; +template +SerialArena::Memory SerialArena::Free(Deallocator deallocator) { + Block* b = head_; + Memory mem = {b, b->size}; + while (b->next) { + b = b->next; // We must first advance before deleting this block + deallocator(mem); + mem = {b, b->size}; } - // Verify that min_bytes + kHeaderSize won't overflow. - GOOGLE_CHECK_LE(min_bytes, std::numeric_limits::max() - kHeaderSize); - size = std::max(size, kHeaderSize + min_bytes); - - Block* b = reinterpret_cast(options_.block_alloc(size)); - InitBlock(b, me, size); - google::protobuf::internal::NoBarrier_AtomicIncrement(&space_allocated_, size); - return b; + return mem; } -void ArenaImpl::InitBlock(Block* b, void *me, size_t size) { - b->pos = kHeaderSize; - b->size = size; - b->owner = me; - b->next = NULL; -#ifdef ADDRESS_SANITIZER - // Poison the rest of the block for ASAN. It was unpoisoned by the underlying - // malloc but it's not yet usable until we return it as part of an allocation. - ASAN_POISON_MEMORY_REGION( - reinterpret_cast(b) + b->pos, b->size - b->pos); -#endif // ADDRESS_SANITIZER +PROTOBUF_NOINLINE +std::pair +SerialArena::AllocateAlignedWithCleanupFallback( + size_t n, const AllocationPolicy* policy) { + AllocateNewBlock(n + kCleanupSize, policy); + return AllocateFromExistingWithCleanupFallback(n); } -ArenaImpl::CleanupChunk* ArenaImpl::ExpandCleanupList(CleanupChunk* cleanup, - Block* b) { - size_t size = cleanup ? cleanup->size * 2 : kMinCleanupListElements; - size = std::min(size, kMaxCleanupListElements); - size_t bytes = internal::AlignUpTo8(CleanupChunk::SizeOf(size)); - if (b->avail() < bytes) { - b = GetBlock(bytes); - } - CleanupChunk* list = - reinterpret_cast(AllocFromBlock(b, bytes)); - list->next = b->thread_info->cleanup; - list->size = size; - list->len = 0; - b->thread_info->cleanup = list; - return list; +PROTOBUF_NOINLINE +void* SerialArena::AllocateAlignedFallback(size_t n, + const AllocationPolicy* policy) { + AllocateNewBlock(n, policy); + return AllocateFromExisting(n); } -inline GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE -void ArenaImpl::AddCleanupInBlock( - Block* b, void* elem, void (*func)(void*)) { - CleanupChunk* cleanup = b->thread_info->cleanup; - if (cleanup == NULL || cleanup->len == cleanup->size) { - cleanup = ExpandCleanupList(cleanup, b); - } +void SerialArena::AllocateNewBlock(size_t n, const AllocationPolicy* policy) { + // Sync limit to block + head_->start = reinterpret_cast(limit_); + + // Record how much used in this block. + space_used_ += ptr_ - head_->Pointer(kBlockHeaderSize); - CleanupNode* node = &cleanup->nodes[cleanup->len++]; + auto mem = AllocateMemory(policy, head_->size, n); + // We don't want to emit an expensive RMW instruction that requires + // exclusive access to a cacheline. Hence we write it in terms of a + // regular add. + auto relaxed = std::memory_order_relaxed; + space_allocated_.store(space_allocated_.load(relaxed) + mem.size, relaxed); + head_ = new (mem.ptr) Block{head_, mem.size}; + ptr_ = head_->Pointer(kBlockHeaderSize); + limit_ = head_->Pointer(head_->size); - node->elem = elem; - node->cleanup = func; +#ifdef ADDRESS_SANITIZER + ASAN_POISON_MEMORY_REGION(ptr_, limit_ - ptr_); +#endif // ADDRESS_SANITIZER } -void ArenaImpl::AddCleanup(void* elem, void (*cleanup)(void*)) { - return AddCleanupInBlock(GetBlock(0), elem, cleanup); +uint64_t SerialArena::SpaceUsed() const { + uint64_t space_used = ptr_ - head_->Pointer(kBlockHeaderSize); + space_used += space_used_; + // Remove the overhead of the SerialArena itself. + space_used -= ThreadSafeArena::kSerialArenaSize; + return space_used; +} + +void SerialArena::CleanupList() { + Block* b = head_; + b->start = reinterpret_cast(limit_); + do { + auto* limit = reinterpret_cast( + b->Pointer(b->size & static_cast(-8))); + auto it = b->start; + auto num = limit - it; + if (num > 0) { + for (; it < limit; it++) { + it->cleanup(it->elem); + } + } + b = b->next; + } while (b); } -void* ArenaImpl::AllocateAligned(size_t n) { - GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. - return AllocFromBlock(GetBlock(n), n); +ThreadSafeArena::CacheAlignedLifecycleIdGenerator + ThreadSafeArena::lifecycle_id_generator_; +#if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) +ThreadSafeArena::ThreadCache& ThreadSafeArena::thread_cache() { + static internal::ThreadLocalStorage* thread_cache_ = + new internal::ThreadLocalStorage(); + return *thread_cache_->Get(); +} +#elif defined(PROTOBUF_USE_DLLS) +ThreadSafeArena::ThreadCache& ThreadSafeArena::thread_cache() { + static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache_ = { + 0, static_cast(-1), nullptr}; + return thread_cache_; } +#else +PROTOBUF_THREAD_LOCAL ThreadSafeArena::ThreadCache + ThreadSafeArena::thread_cache_ = {0, static_cast(-1), + nullptr}; +#endif -void* ArenaImpl::AllocateAlignedAndAddCleanup(size_t n, - void (*cleanup)(void*)) { - GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. +void ThreadSafeArena::InitializeFrom(void* mem, size_t size) { + GOOGLE_DCHECK_EQ(reinterpret_cast(mem) & 7, 0u); + GOOGLE_DCHECK(!AllocPolicy()); // Reset should call InitializeWithPolicy instead. + Init(); - Block* b = GetBlock(n); - void* mem = AllocFromBlock(b, n); - AddCleanupInBlock(b, mem, cleanup); - return mem; + // Ignore initial block if it is too small. + if (mem != nullptr && size >= kBlockHeaderSize + kSerialArenaSize) { + alloc_policy_.set_is_user_owned_initial_block(true); + SetInitialBlock(mem, size); + } } -inline GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE -ArenaImpl::Block* ArenaImpl::GetBlock(size_t n) { - Block* my_block = NULL; - - // If this thread already owns a block in this arena then try to use that. - // This fast path optimizes the case where multiple threads allocate from the - // same arena. - ThreadCache* tc = &thread_cache(); - if (tc->last_lifecycle_id_seen == lifecycle_id_) { - my_block = tc->last_block_used_; - if (my_block->avail() >= n) { - return my_block; - } +void ThreadSafeArena::InitializeWithPolicy(void* mem, size_t size, + AllocationPolicy policy) { +#ifndef NDEBUG + const uint64_t old_alloc_policy = alloc_policy_.get_raw(); + // If there was a policy (e.g., in Reset()), make sure flags were preserved. +#define GOOGLE_DCHECK_POLICY_FLAGS_() \ + if (old_alloc_policy > 3) \ + GOOGLE_CHECK_EQ(old_alloc_policy & 3, alloc_policy_.get_raw() & 3) +#else +#define GOOGLE_DCHECK_POLICY_FLAGS_() +#endif // NDEBUG + + if (policy.IsDefault()) { + // Legacy code doesn't use the API above, but provides the initial block + // through ArenaOptions. I suspect most do not touch the allocation + // policy parameters. + InitializeFrom(mem, size); + GOOGLE_DCHECK_POLICY_FLAGS_(); + return; } + GOOGLE_DCHECK_EQ(reinterpret_cast(mem) & 7, 0u); + Init(); - // Check whether we own the last accessed block on this arena. - // This fast path optimizes the case where a single thread uses multiple - // arenas. - Block* b = reinterpret_cast(google::protobuf::internal::Acquire_Load(&hint_)); - if (b != NULL && b->owner == tc) { - my_block = b; - if (my_block->avail() >= n) { - return my_block; - } + // Ignore initial block if it is too small. We include an optional + // AllocationPolicy in this check, so that this can be allocated on the + // first block. + constexpr size_t kAPSize = internal::AlignUpTo8(sizeof(AllocationPolicy)); + constexpr size_t kMinimumSize = kBlockHeaderSize + kSerialArenaSize + kAPSize; + + // The value for alloc_policy_ stores whether or not allocations should be + // recorded. + alloc_policy_.set_should_record_allocs( + policy.metrics_collector != nullptr && + policy.metrics_collector->RecordAllocs()); + // Make sure we have an initial block to store the AllocationPolicy. + if (mem != nullptr && size >= kMinimumSize) { + alloc_policy_.set_is_user_owned_initial_block(true); + } else { + auto tmp = AllocateMemory(&policy, 0, kMinimumSize); + mem = tmp.ptr; + size = tmp.size; } - return GetBlockSlow(tc, my_block, n); -} + SetInitialBlock(mem, size); + + auto sa = threads_.load(std::memory_order_relaxed); + // We ensured enough space so this cannot fail. + void* p; + if (!sa || !sa->MaybeAllocateAligned(kAPSize, &p)) { + GOOGLE_LOG(FATAL) << "MaybeAllocateAligned cannot fail here."; + return; + } + new (p) AllocationPolicy{policy}; + // Low bits store flags, so they mustn't be overwritten. + GOOGLE_DCHECK_EQ(0, reinterpret_cast(p) & 3); + alloc_policy_.set_policy(reinterpret_cast(p)); + GOOGLE_DCHECK_POLICY_FLAGS_(); -inline GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE -void* ArenaImpl::AllocFromBlock(Block* b, size_t n) { - GOOGLE_DCHECK_EQ(internal::AlignUpTo8(b->pos), b->pos); // Must be already aligned. - GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. - GOOGLE_DCHECK_GE(b->avail(), n); - size_t p = b->pos; - b->pos = p + n; -#ifdef ADDRESS_SANITIZER - ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast(b) + p, n); -#endif // ADDRESS_SANITIZER - return reinterpret_cast(b) + p; +#undef GOOGLE_DCHECK_POLICY_FLAGS_ } -ArenaImpl::Block* ArenaImpl::GetBlockSlow(void* me, Block* my_full_block, - size_t n) { - ThreadInfo* info = - my_full_block ? my_full_block->thread_info : GetThreadInfo(me, n); - GOOGLE_DCHECK(info != NULL); - Block* b = info->head; - if (b->avail() < n) { - Block* new_b = NewBlock(me, b, n); - new_b->thread_info = info; - new_b->next = b; - info->head = new_b; - b = new_b; +void ThreadSafeArena::Init() { +#ifndef NDEBUG + const bool was_message_owned = IsMessageOwned(); +#endif // NDEBUG + ThreadCache& tc = thread_cache(); + auto id = tc.next_lifecycle_id; + // We increment lifecycle_id's by multiples of two so we can use bit 0 as + // a tag. + constexpr uint64_t kDelta = 2; + constexpr uint64_t kInc = ThreadCache::kPerThreadIds * kDelta; + if (PROTOBUF_PREDICT_FALSE((id & (kInc - 1)) == 0)) { + constexpr auto relaxed = std::memory_order_relaxed; + // On platforms that don't support uint64_t atomics we can certainly not + // afford to increment by large intervals and expect uniqueness due to + // wrapping, hence we only add by 1. + id = lifecycle_id_generator_.id.fetch_add(1, relaxed) * kInc; } - CacheBlock(b); - return b; + tc.next_lifecycle_id = id + kDelta; + // Message ownership is stored in tag_and_id_, and is set in the constructor. + // This flag bit must be preserved, even across calls to Reset(). + tag_and_id_ = id | (tag_and_id_ & kMessageOwnedArena); + hint_.store(nullptr, std::memory_order_relaxed); + threads_.store(nullptr, std::memory_order_relaxed); +#ifndef NDEBUG + GOOGLE_CHECK_EQ(was_message_owned, IsMessageOwned()); +#endif // NDEBUG } -uint64 ArenaImpl::SpaceAllocated() const { - return google::protobuf::internal::NoBarrier_Load(&space_allocated_); +void ThreadSafeArena::SetInitialBlock(void* mem, size_t size) { + SerialArena* serial = SerialArena::New({mem, size}, &thread_cache()); + serial->set_next(NULL); + threads_.store(serial, std::memory_order_relaxed); + CacheSerialArena(serial); } -uint64 ArenaImpl::SpaceUsed() const { - ThreadInfo* info = - reinterpret_cast(google::protobuf::internal::Acquire_Load(&threads_)); - uint64 space_used = 0; +ThreadSafeArena::~ThreadSafeArena() { + // Have to do this in a first pass, because some of the destructors might + // refer to memory in other blocks. + CleanupList(); - for ( ; info; info = info->next) { - // Remove the overhead of the ThreadInfo itself. - space_used -= sizeof(ThreadInfo); - for (Block* b = info->head; b; b = b->next) { - space_used += (b->pos - kHeaderSize); - } + size_t space_allocated = 0; + auto mem = Free(&space_allocated); + + // Policy is about to get deleted. + auto* p = alloc_policy_.get(); + ArenaMetricsCollector* collector = p ? p->metrics_collector : nullptr; + + if (alloc_policy_.is_user_owned_initial_block()) { + space_allocated += mem.size; + } else { + GetDeallocator(alloc_policy_.get(), &space_allocated)(mem); } - return space_used; + if (collector) collector->OnDestroy(space_allocated); } -uint64 ArenaImpl::FreeBlocks() { - uint64 space_allocated = 0; - // By omitting an Acquire barrier we ensure that any user code that doesn't - // properly synchronize Reset() or the destructor will throw a TSAN warning. - ThreadInfo* info = - reinterpret_cast(google::protobuf::internal::NoBarrier_Load(&threads_)); - - while (info) { - // This is inside the block we are freeing, so we need to read it now. - ThreadInfo* next_info = info->next; - for (Block* b = info->head; b; ) { - // This is inside the block we are freeing, so we need to read it now. - Block* next_block = b->next; - space_allocated += (b->size); - -#ifdef ADDRESS_SANITIZER - // This memory was provided by the underlying allocator as unpoisoned, so - // return it in an unpoisoned state. - ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast(b), b->size); -#endif // ADDRESS_SANITIZER +SerialArena::Memory ThreadSafeArena::Free(size_t* space_allocated) { + SerialArena::Memory mem = {nullptr, 0}; + auto deallocator = GetDeallocator(alloc_policy_.get(), space_allocated); + PerSerialArena([deallocator, &mem](SerialArena* a) { + if (mem.ptr) deallocator(mem); + mem = a->Free(deallocator); + }); + return mem; +} - if (b != initial_block_) { - options_.block_dealloc(b, b->size); - } +uint64_t ThreadSafeArena::Reset() { + // Have to do this in a first pass, because some of the destructors might + // refer to memory in other blocks. + CleanupList(); - b = next_block; + // Discard all blocks except the special block (if present). + size_t space_allocated = 0; + auto mem = Free(&space_allocated); + + AllocationPolicy* policy = alloc_policy_.get(); + if (policy) { + auto saved_policy = *policy; + if (alloc_policy_.is_user_owned_initial_block()) { + space_allocated += mem.size; + } else { + GetDeallocator(alloc_policy_.get(), &space_allocated)(mem); + mem.ptr = nullptr; + mem.size = 0; + } + ArenaMetricsCollector* collector = saved_policy.metrics_collector; + if (collector) collector->OnReset(space_allocated); + InitializeWithPolicy(mem.ptr, mem.size, saved_policy); + } else { + GOOGLE_DCHECK(!alloc_policy_.should_record_allocs()); + // Nullptr policy + if (alloc_policy_.is_user_owned_initial_block()) { + space_allocated += mem.size; + InitializeFrom(mem.ptr, mem.size); + } else { + GetDeallocator(alloc_policy_.get(), &space_allocated)(mem); + Init(); } - info = next_info; } return space_allocated; } -void ArenaImpl::CleanupList() { - // By omitting an Acquire barrier we ensure that any user code that doesn't - // properly synchronize Reset() or the destructor will throw a TSAN warning. - ThreadInfo* info = - reinterpret_cast(google::protobuf::internal::NoBarrier_Load(&threads_)); - - for ( ; info; info = info->next) { - CleanupChunk* list = info->cleanup; - while (list) { - size_t n = list->len; - CleanupNode* node = &list->nodes[list->len - 1]; - for (size_t i = 0; i < n; i++, node--) { - node->cleanup(node->elem); - } - list = list->next; - } +std::pair +ThreadSafeArena::AllocateAlignedWithCleanup(size_t n, + const std::type_info* type) { + SerialArena* arena; + if (PROTOBUF_PREDICT_TRUE(!alloc_policy_.should_record_allocs() && + GetSerialArenaFast(&arena))) { + return arena->AllocateAlignedWithCleanup(n, alloc_policy_.get()); + } else { + return AllocateAlignedWithCleanupFallback(n, type); } } -ArenaImpl::ThreadInfo* ArenaImpl::NewThreadInfo(Block* b) { - GOOGLE_DCHECK(FindThreadInfo(b->owner) == NULL); - ThreadInfo* info = - reinterpret_cast(AllocFromBlock(b, sizeof(ThreadInfo))); - b->thread_info = info; - info->owner = b->owner; - info->head = b; - info->cleanup = NULL; - return info; +void ThreadSafeArena::AddCleanup(void* elem, void (*cleanup)(void*)) { + SerialArena* arena; + if (PROTOBUF_PREDICT_FALSE(!GetSerialArenaFast(&arena))) { + arena = GetSerialArenaFallback(&thread_cache()); + } + arena->AddCleanup(elem, cleanup, AllocPolicy()); } -ArenaImpl::ThreadInfo* ArenaImpl::FindThreadInfo(void* me) { - ThreadInfo* info = - reinterpret_cast(google::protobuf::internal::Acquire_Load(&threads_)); - for ( ; info; info = info->next) { - if (info->owner == me) { - return info; +PROTOBUF_NOINLINE +void* ThreadSafeArena::AllocateAlignedFallback(size_t n, + const std::type_info* type) { + if (alloc_policy_.should_record_allocs()) { + alloc_policy_.RecordAlloc(type, n); + SerialArena* arena; + if (PROTOBUF_PREDICT_TRUE(GetSerialArenaFast(&arena))) { + return arena->AllocateAligned(n, alloc_policy_.get()); } } + return GetSerialArenaFallback(&thread_cache()) + ->AllocateAligned(n, alloc_policy_.get()); +} - return NULL; +PROTOBUF_NOINLINE +std::pair +ThreadSafeArena::AllocateAlignedWithCleanupFallback( + size_t n, const std::type_info* type) { + if (alloc_policy_.should_record_allocs()) { + alloc_policy_.RecordAlloc(type, n); + SerialArena* arena; + if (GetSerialArenaFast(&arena)) { + return arena->AllocateAlignedWithCleanup(n, alloc_policy_.get()); + } + } + return GetSerialArenaFallback(&thread_cache()) + ->AllocateAlignedWithCleanup(n, alloc_policy_.get()); } -ArenaImpl::ThreadInfo* ArenaImpl::GetThreadInfo(void* me, size_t n) { - ThreadInfo* info = FindThreadInfo(me); +uint64_t ThreadSafeArena::SpaceAllocated() const { + SerialArena* serial = threads_.load(std::memory_order_acquire); + uint64_t res = 0; + for (; serial; serial = serial->next()) { + res += serial->SpaceAllocated(); + } + return res; +} - if (!info) { - // This thread doesn't have any ThreadInfo, which also means it doesn't have - // any blocks yet. So we'll allocate its first block now. - Block* b = NewBlock(me, NULL, sizeof(ThreadInfo) + n); - info = NewThreadInfo(b); +uint64_t ThreadSafeArena::SpaceUsed() const { + SerialArena* serial = threads_.load(std::memory_order_acquire); + uint64_t space_used = 0; + for (; serial; serial = serial->next()) { + space_used += serial->SpaceUsed(); + } + return space_used - (alloc_policy_.get() ? sizeof(AllocationPolicy) : 0); +} + +void ThreadSafeArena::CleanupList() { + PerSerialArena([](SerialArena* a) { a->CleanupList(); }); +} + +PROTOBUF_NOINLINE +SerialArena* ThreadSafeArena::GetSerialArenaFallback(void* me) { + // Look for this SerialArena in our linked list. + SerialArena* serial = threads_.load(std::memory_order_acquire); + for (; serial; serial = serial->next()) { + if (serial->owner() == me) { + break; + } + } - google::protobuf::internal::AtomicWord head; + if (!serial) { + // This thread doesn't have any SerialArena, which also means it doesn't + // have any blocks yet. So we'll allocate its first block now. + serial = SerialArena::New( + AllocateMemory(alloc_policy_.get(), 0, kSerialArenaSize), me); + + SerialArena* head = threads_.load(std::memory_order_relaxed); do { - head = google::protobuf::internal::NoBarrier_Load(&threads_); - info->next = reinterpret_cast(head); - } while (google::protobuf::internal::Release_CompareAndSwap( - &threads_, head, reinterpret_cast(info)) != head); + serial->set_next(head); + } while (!threads_.compare_exchange_weak( + head, serial, std::memory_order_release, std::memory_order_relaxed)); } - return info; + CacheSerialArena(serial); + return serial; } } // namespace internal -void Arena::CallDestructorHooks() { - uint64 space_allocated = impl_.SpaceAllocated(); - // Call the reset hook - if (on_arena_reset_ != NULL) { - on_arena_reset_(this, hooks_cookie_, space_allocated); - } +PROTOBUF_FUNC_ALIGN(32) +void* Arena::AllocateAlignedNoHook(size_t n) { + return impl_.AllocateAligned(n, nullptr); +} - // Call the destruction hook - if (on_arena_destruction_ != NULL) { - on_arena_destruction_(this, hooks_cookie_, space_allocated); - } +PROTOBUF_FUNC_ALIGN(32) +void* Arena::AllocateAlignedWithHook(size_t n, const std::type_info* type) { + return impl_.AllocateAligned(n, type); } -void Arena::OnArenaAllocation(const std::type_info* allocated_type, - size_t n) const { - if (on_arena_allocation_ != NULL) { - on_arena_allocation_(allocated_type, n, hooks_cookie_); - } +PROTOBUF_FUNC_ALIGN(32) +std::pair +Arena::AllocateAlignedWithCleanup(size_t n, const std::type_info* type) { + return impl_.AllocateAlignedWithCleanup(n, type); } } // namespace protobuf } // namespace google + +#include diff --git a/3rdparty/protobuf/src/google/protobuf/arena.h b/3rdparty/protobuf/src/google/protobuf/arena.h index 32be9a172630..6dd6467f58bb 100644 --- a/3rdparty/protobuf/src/google/protobuf/arena.h +++ b/3rdparty/protobuf/src/google/protobuf/arena.h @@ -33,14 +33,14 @@ #ifndef GOOGLE_PROTOBUF_ARENA_H__ #define GOOGLE_PROTOBUF_ARENA_H__ + #include +#include +#include #ifdef max #undef max // Visual Studio defines this macro #endif -#if LANG_CXX11 -#include -#endif -#if defined(_MSC_VER) && !_HAS_EXCEPTIONS +#if defined(_MSC_VER) && !defined(_LIBCPP_STD_VER) && !_HAS_EXCEPTIONS // Work around bugs in MSVC header when _HAS_EXCEPTIONS=0. #include #include @@ -51,38 +51,72 @@ using type_info = ::type_info; #include #endif +#include #include -#include +#include + +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif namespace google { namespace protobuf { -class Arena; // defined below -class Message; // message.h +struct ArenaOptions; // defined below +class Arena; // defined below +class Message; // defined in message.h +class MessageLite; +template +class Map; + +namespace arena_metrics { + +void EnableArenaMetrics(ArenaOptions* options); + +} // namespace arena_metrics + +namespace TestUtil { +class ReflectionTester; // defined in test_util.h +} // namespace TestUtil namespace internal { -struct ArenaStringPtr; // arenastring.h -class LazyField; // lazy_field.h -template -class GenericTypeHandler; // repeated_field.h +struct ArenaStringPtr; // defined in arenastring.h +class InlinedStringField; // defined in inlined_string_field.h +class LazyField; // defined in lazy_field.h +class EpsCopyInputStream; // defined in parse_context.h + +template +class GenericTypeHandler; // defined in repeated_field.h + +inline PROTOBUF_ALWAYS_INLINE +void* AlignTo(void* ptr, size_t align) { + return reinterpret_cast( + (reinterpret_cast(ptr) + align - 1) & (~align + 1)); +} // Templated cleanup methods. -template void arena_destruct_object(void* object) { +template +void arena_destruct_object(void* object) { reinterpret_cast(object)->~T(); } -template void arena_delete_object(void* object) { + +template +struct ObjectDestructor { + constexpr static void (*destructor)(void*) = &arena_destruct_object; +}; + +template +struct ObjectDestructor { + constexpr static void (*destructor)(void*) = nullptr; +}; + +template +void arena_delete_object(void* object) { delete reinterpret_cast(object); } -inline void arena_free(void* object, size_t size) { -#if defined(__GXX_DELETE_WITH_SIZE__) || defined(__cpp_sized_deallocation) - ::operator delete(object, size); -#else - (void)size; - ::operator delete(object); -#endif -} - } // namespace internal // ArenaOptions provides optional additional parameters to arena construction @@ -117,49 +151,45 @@ struct ArenaOptions { // from the arena. By default, it contains a ptr to a wrapper function that // calls free. void (*block_dealloc)(void*, size_t); - // Hooks for adding external functionality such as user-specific metrics - // collection, specific debugging abilities, etc. - // Init hook may return a pointer to a cookie to be stored in the arena. - // reset and destruction hooks will then be called with the same cookie - // pointer. This allows us to save an external object per arena instance and - // use it on the other hooks (Note: It is just as legal for init to return - // NULL and not use the cookie feature). - // on_arena_reset and on_arena_destruction also receive the space used in - // the arena just before the reset. - void* (*on_arena_init)(Arena* arena); - void (*on_arena_reset)(Arena* arena, void* cookie, uint64 space_used); - void (*on_arena_destruction)(Arena* arena, void* cookie, uint64 space_used); - - // type_info is promised to be static - its lifetime extends to - // match program's lifetime (It is given by typeid operator). - // Note: typeid(void) will be passed as allocated_type every time we - // intentionally want to avoid monitoring an allocation. (i.e. internal - // allocations for managing the arena) - void (*on_arena_allocation)(const std::type_info* allocated_type, - uint64 alloc_size, void* cookie); ArenaOptions() - : start_block_size(kDefaultStartBlockSize), - max_block_size(kDefaultMaxBlockSize), + : start_block_size(internal::AllocationPolicy::kDefaultStartBlockSize), + max_block_size(internal::AllocationPolicy::kDefaultMaxBlockSize), initial_block(NULL), initial_block_size(0), - block_alloc(&::operator new), - block_dealloc(&internal::arena_free), - on_arena_init(NULL), - on_arena_reset(NULL), - on_arena_destruction(NULL), - on_arena_allocation(NULL) {} + block_alloc(nullptr), + block_dealloc(nullptr), + make_metrics_collector(nullptr) {} private: - // Constants define default starting block size and max block size for - // arena allocator behavior -- see descriptions above. - static const size_t kDefaultStartBlockSize = 256; - static const size_t kDefaultMaxBlockSize = 8192; + // If make_metrics_collector is not nullptr, it will be called at Arena init + // time. It may return a pointer to a collector instance that will be notified + // of interesting events related to the arena. + internal::ArenaMetricsCollector* (*make_metrics_collector)(); + + internal::ArenaMetricsCollector* MetricsCollector() const { + return make_metrics_collector ? (*make_metrics_collector)() : nullptr; + } + + internal::AllocationPolicy AllocationPolicy() const { + internal::AllocationPolicy res; + res.start_block_size = start_block_size; + res.max_block_size = max_block_size; + res.block_alloc = block_alloc; + res.block_dealloc = block_dealloc; + res.metrics_collector = MetricsCollector(); + return res; + } + + friend void arena_metrics::EnableArenaMetrics(ArenaOptions*); + + friend class Arena; + friend class ArenaOptionsTestFriend; }; // Support for non-RTTI environments. (The metrics hooks API uses type // information.) -#ifndef GOOGLE_PROTOBUF_NO_RTTI +#if PROTOBUF_RTTI #define RTTI_TYPE_ID(type) (&typeid(type)) #else #define RTTI_TYPE_ID(type) (NULL) @@ -185,14 +215,15 @@ struct ArenaOptions { // any special requirements on the type T, and will invoke the object's // destructor when the arena is destroyed. // -// The arena message allocation protocol, required by CreateMessage, is as -// follows: +// The arena message allocation protocol, required by +// CreateMessage(Arena* arena, Args&&... args), is as follows: // -// - The type T must have (at least) two constructors: a constructor with no -// arguments, called when a T is allocated on the heap; and a constructor with -// a google::protobuf::Arena* argument, called when a T is allocated on an arena. If the -// second constructor is called with a NULL arena pointer, it must be -// equivalent to invoking the first (no-argument) constructor. +// - The type T must have (at least) two constructors: a constructor callable +// with `args` (without `arena`), called when a T is allocated on the heap; +// and a constructor callable with `Arena* arena, Args&&... args`, called when +// a T is allocated on an arena. If the second constructor is called with a +// NULL arena pointer, it must be equivalent to invoking the first +// (`args`-only) constructor. // // - The type T must have a particular type trait: a nested type // |InternalArenaConstructable_|. This is usually a typedef to |void|. If no @@ -205,23 +236,28 @@ struct ArenaOptions { // present on the type, then its destructor is always called when the // containing arena is destroyed. // -// - One- and two-user-argument forms of CreateMessage() also exist that -// forward these constructor arguments to T's constructor: for example, -// CreateMessage(Arena*, arg1, arg2) forwards to a constructor T(Arena*, -// arg1, arg2). -// // This protocol is implemented by all arena-enabled proto2 message classes as -// well as RepeatedPtrField. -// -// Do NOT subclass Arena. This class will be marked as final when C++11 is -// enabled. -class LIBPROTOBUF_EXPORT Arena { +// well as protobuf container types like RepeatedPtrField and Map. The protocol +// is internal to protobuf and is not guaranteed to be stable. Non-proto types +// should not rely on this protocol. +class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { public: - // Arena constructor taking custom options. See ArenaOptions below for + // Default constructor with sensible default options, tuned for average + // use-cases. + inline Arena() : impl_() {} + + // Construct an arena with default options, except for the supplied + // initial block. It is more efficient to use this constructor + // instead of passing ArenaOptions if the only configuration needed + // by the caller is supplying an initial block. + inline Arena(char* initial_block, size_t initial_block_size) + : impl_(initial_block, initial_block_size) {} + + // Arena constructor taking custom options. See ArenaOptions above for // descriptions of the options available. - explicit Arena(const ArenaOptions& options) : impl_(options) { - Init(options); - } + explicit Arena(const ArenaOptions& options) + : impl_(options.initial_block, options.initial_block_size, + options.AllocationPolicy()) {} // Block overhead. Use this as a guide for how much to over-allocate the // initial block if you want an allocation of size N to fit inside it. @@ -229,29 +265,14 @@ class LIBPROTOBUF_EXPORT Arena { // WARNING: if you allocate multiple objects, it is difficult to guarantee // that a series of allocations will fit in the initial block, especially if // Arena changes its alignment guarantees in the future! - static const size_t kBlockOverhead = internal::ArenaImpl::kHeaderSize; + static const size_t kBlockOverhead = + internal::ThreadSafeArena::kBlockHeaderSize + + internal::ThreadSafeArena::kSerialArenaSize; - // Default constructor with sensible default options, tuned for average - // use-cases. - Arena() : impl_(ArenaOptions()) { Init(ArenaOptions()); } + inline ~Arena() {} - ~Arena() { - if (on_arena_reset_ != NULL || on_arena_destruction_ != NULL) { - CallDestructorHooks(); - } - } - - void Init(const ArenaOptions& options) { - on_arena_allocation_ = options.on_arena_allocation; - on_arena_reset_ = options.on_arena_reset; - on_arena_destruction_ = options.on_arena_destruction; - // Call the initialization hook - if (options.on_arena_init != NULL) { - hooks_cookie_ = options.on_arena_init(this); - } else { - hooks_cookie_ = NULL; - } - } + // TODO(protobuf-team): Fix callers to use constructor and delete this method. + void Init(const ArenaOptions&) {} // API to create proto2 message objects on the arena. If the arena passed in // is NULL, then a heap allocated object is returned. Type T must be a message @@ -263,69 +284,15 @@ class LIBPROTOBUF_EXPORT Arena { // // This function also accepts any type T that satisfies the arena message // allocation protocol, documented above. -#if LANG_CXX11 template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE static T* CreateMessage( - ::google::protobuf::Arena* arena, Args&&... args) { - static_assert( - InternalHelper::is_arena_constructable::value, - "CreateMessage can only construct types that are ArenaConstructable"); - if (arena == NULL) { - return new T(NULL, std::forward(args)...); - } else { - return arena->CreateMessageInternal(std::forward(args)...); - } - } -#endif - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* CreateMessage(::google::protobuf::Arena* arena) { -#if LANG_CXX11 - static_assert( - InternalHelper::is_arena_constructable::value, - "CreateMessage can only construct types that are ArenaConstructable"); -#endif - if (arena == NULL) { - return new T; - } else { - return arena->CreateMessageInternal(); - } - } - - // One-argument form of CreateMessage. This is useful for constructing objects - // that implement the arena message construction protocol described above but - // take additional constructor arguments. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* CreateMessage(::google::protobuf::Arena* arena, const Arg& arg) { -#if LANG_CXX11 + PROTOBUF_ALWAYS_INLINE static T* CreateMessage(Arena* arena, Args&&... args) { static_assert( InternalHelper::is_arena_constructable::value, "CreateMessage can only construct types that are ArenaConstructable"); -#endif - if (arena == NULL) { - return new T(NULL, arg); - } else { - return arena->CreateMessageInternal(arg); - } - } - - // Two-argument form of CreateMessage. This is useful for constructing objects - // that implement the arena message construction protocol described above but - // take additional constructor arguments. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* CreateMessage(::google::protobuf::Arena* arena, - const Arg1& arg1, - const Arg2& arg2) { -#if LANG_CXX11 - static_assert( - InternalHelper::is_arena_constructable::value, - "CreateMessage can only construct types that are ArenaConstructable"); -#endif - if (arena == NULL) { - return new T(NULL, arg1, arg2); - } else { - return arena->CreateMessageInternal(arg1, arg2); - } + // We must delegate to CreateMaybeMessage() and NOT CreateMessageInternal() + // because protobuf generated classes specialize CreateMaybeMessage() and we + // need to use that specialization for code size reasons. + return Arena::CreateMaybeMessage(arena, static_cast(args)...); } // API to create any objects on the arena. Note that only the object will @@ -343,152 +310,10 @@ class LIBPROTOBUF_EXPORT Arena { // (unless the destructor is trivial). Hence, from T's point of view, it is as // if the object were allocated on the heap (except that the underlying memory // is obtained from the arena). -#if LANG_CXX11 template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, Args&&... args) { - if (arena == NULL) { - return new T(std::forward(args)...); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - std::forward(args)...); - } - } -#endif - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena) { - if (arena == NULL) { - return new T(); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value); - } - } - - // Version of the above with one constructor argument for the created object. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, const Arg& arg) { - if (arena == NULL) { - return new T(arg); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - arg); - } - } - - // Version of the above with two constructor arguments for the created object. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, const Arg1& arg1, const Arg2& arg2) { - if (arena == NULL) { - return new T(arg1, arg2); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - arg1, arg2); - } - } - - // Version of the above with three constructor arguments for the created - // object. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, - const Arg1& arg1, - const Arg2& arg2, - const Arg3& arg3) { - if (arena == NULL) { - return new T(arg1, arg2, arg3); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - arg1, arg2, arg3); - } - } - - // Version of the above with four constructor arguments for the created - // object. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, - const Arg1& arg1, const Arg2& arg2, - const Arg3& arg3, const Arg4& arg4) { - if (arena == NULL) { - return new T(arg1, arg2, arg3, arg4); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - arg1, arg2, arg3, arg4); - } - } - - // Version of the above with five constructor arguments for the created - // object. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, - const Arg1& arg1, const Arg2& arg2, - const Arg3& arg3, const Arg4& arg4, - const Arg5& arg5) { - if (arena == NULL) { - return new T(arg1, arg2, arg3, arg4, arg5); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - arg1, arg2, arg3, arg4, arg5); - } - } - - // Version of the above with six constructor arguments for the created - // object. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, - const Arg1& arg1, const Arg2& arg2, - const Arg3& arg3, const Arg4& arg4, - const Arg5& arg5, const Arg6& arg6) { - if (arena == NULL) { - return new T(arg1, arg2, arg3, arg4, arg5, arg6); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - arg1, arg2, arg3, arg4, arg5, arg6); - } - } - - // Version of the above with seven constructor arguments for the created - // object. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, - const Arg1& arg1, const Arg2& arg2, - const Arg3& arg3, const Arg4& arg4, - const Arg5& arg5, const Arg6& arg6, - const Arg7& arg7) { - if (arena == NULL) { - return new T(arg1, arg2, arg3, arg4, arg5, arg6, arg7); - } else { - return arena->CreateInternal(google::protobuf::internal::has_trivial_destructor::value, - arg1, arg2, arg3, arg4, arg5, arg6, arg7); - } - } - - // Version of the above with eight constructor arguments for the created - // object. - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* Create(::google::protobuf::Arena* arena, - const Arg1& arg1, const Arg2& arg2, - const Arg3& arg3, const Arg4& arg4, - const Arg5& arg5, const Arg6& arg6, - const Arg7& arg7, const Arg8& arg8) { - if (arena == NULL) { - return new T(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); - } else { - return arena->CreateInternal( - google::protobuf::internal::has_trivial_destructor::value, - arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); - } + PROTOBUF_NDEBUG_INLINE static T* Create(Arena* arena, Args&&... args) { + return CreateInternal(arena, std::is_convertible(), + static_cast(args)...); } // Create an array of object type T on the arena *without* invoking the @@ -497,10 +322,14 @@ class LIBPROTOBUF_EXPORT Arena { // To ensure safe uses, this function checks at compile time // (when compiled as C++11) that T is trivially default-constructible and // trivially destructible. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* CreateArray(::google::protobuf::Arena* arena, size_t num_elements) { - GOOGLE_CHECK_LE(num_elements, - std::numeric_limits::max() / sizeof(T)) + template + PROTOBUF_NDEBUG_INLINE static T* CreateArray(Arena* arena, + size_t num_elements) { + static_assert(std::is_trivial::value, + "CreateArray requires a trivially constructible type"); + static_assert(std::is_trivially_destructible::value, + "CreateArray requires a trivially destructible type"); + GOOGLE_CHECK_LE(num_elements, std::numeric_limits::max() / sizeof(T)) << "Requested size is too large to fit into size_t."; if (arena == NULL) { return static_cast(::operator new[](num_elements * sizeof(T))); @@ -509,41 +338,31 @@ class LIBPROTOBUF_EXPORT Arena { } } + // The following are routines are for monitoring. They will approximate the + // total sum allocated and used memory, but the exact value is an + // implementation deal. For instance allocated space depends on growth + // policies. Do not use these in unit tests. // Returns the total space allocated by the arena, which is the sum of the - // sizes of the underlying blocks. This method is relatively fast; a counter - // is kept as blocks are allocated. - uint64 SpaceAllocated() const { return impl_.SpaceAllocated(); } + // sizes of the underlying blocks. + uint64_t SpaceAllocated() const { return impl_.SpaceAllocated(); } // Returns the total space used by the arena. Similar to SpaceAllocated but // does not include free space and block overhead. The total space returned // may not include space used by other threads executing concurrently with // the call to this method. - uint64 SpaceUsed() const { return impl_.SpaceUsed(); } - // DEPRECATED. Please use SpaceAllocated() and SpaceUsed(). - // - // Combines SpaceAllocated and SpaceUsed. Returns a pair of - // . - std::pair SpaceAllocatedAndUsed() const { - return std::make_pair(SpaceAllocated(), SpaceUsed()); - } + uint64_t SpaceUsed() const { return impl_.SpaceUsed(); } // Frees all storage allocated by this arena after calling destructors // registered with OwnDestructor() and freeing objects registered with Own(). // Any objects allocated on this arena are unusable after this call. It also // returns the total space used by the arena which is the sums of the sizes // of the allocated blocks. This method is not thread-safe. - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE uint64 Reset() { - // Call the reset hook - if (on_arena_reset_ != NULL) { - on_arena_reset_(this, hooks_cookie_, impl_.SpaceAllocated()); - } - return impl_.Reset(); - } + uint64_t Reset() { return impl_.Reset(); } // Adds |object| to a list of heap-allocated objects to be freed with |delete| // when the arena is destroyed or reset. - template GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE - void Own(T* object) { - OwnInternal(object, google::protobuf::internal::is_convertible()); + template + PROTOBUF_ALWAYS_INLINE void Own(T* object) { + OwnInternal(object, std::is_convertible()); } // Adds |object| to a list of objects whose destructors will be manually @@ -551,8 +370,8 @@ class LIBPROTOBUF_EXPORT Arena { // that it does not free the underlying memory with |delete|; hence, it is // normally only used for objects that are placement-newed into // arena-allocated memory. - template GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE - void OwnDestructor(T* object) { + template + PROTOBUF_ALWAYS_INLINE void OwnDestructor(T* object) { if (object != NULL) { impl_.AddCleanup(object, &internal::arena_destruct_object); } @@ -562,102 +381,208 @@ class LIBPROTOBUF_EXPORT Arena { // will be manually called when the arena is destroyed or reset. This differs // from OwnDestructor() in that any member function may be specified, not only // the class destructor. - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE void OwnCustomDestructor( - void* object, void (*destruct)(void*)) { + PROTOBUF_ALWAYS_INLINE void OwnCustomDestructor(void* object, + void (*destruct)(void*)) { impl_.AddCleanup(object, destruct); } // Retrieves the arena associated with |value| if |value| is an arena-capable - // message, or NULL otherwise. This differs from value->GetArena() in that the - // latter is a virtual call, while this method is a templated call that - // resolves at compile-time. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static ::google::protobuf::Arena* GetArena(const T* value) { - return GetArenaInternal(value, is_arena_constructable()); + // message, or NULL otherwise. If possible, the call resolves at compile time. + // Note that we can often devirtualize calls to `value->GetArena()` so usually + // calling this method is unnecessary. + template + PROTOBUF_ALWAYS_INLINE static Arena* GetArena(const T* value) { + return GetArenaInternal(value); } template class InternalHelper { + public: + // Provides access to protected GetOwningArena to generated messages. + static Arena* GetOwningArena(const T* p) { return p->GetOwningArena(); } + + // Provides access to protected GetArenaForAllocation to generated messages. + static Arena* GetArenaForAllocation(const T* p) { + return GetArenaForAllocationInternal( + p, std::is_convertible()); + } + + // Creates message-owned arena. + static Arena* CreateMessageOwnedArena() { + return new Arena(internal::MessageOwned{}); + } + + // Checks whether the given arena is message-owned. + static bool IsMessageOwnedArena(Arena* arena) { + return arena->IsMessageOwned(); + } + + private: + static Arena* GetArenaForAllocationInternal( + const T* p, std::true_type /*is_derived_from*/) { + return p->GetArenaForAllocation(); + } + + static Arena* GetArenaForAllocationInternal( + const T* p, std::false_type /*is_derived_from*/) { + return GetArenaForAllocationForNonMessage( + p, typename is_arena_constructable::type()); + } + + static Arena* GetArenaForAllocationForNonMessage( + const T* p, std::true_type /*is_arena_constructible*/) { + return p->GetArena(); + } + + static Arena* GetArenaForAllocationForNonMessage( + const T* p, std::false_type /*is_arena_constructible*/) { + return GetArenaForAllocationForNonMessageNonArenaConstructible( + p, typename has_get_arena::type()); + } + + static Arena* GetArenaForAllocationForNonMessageNonArenaConstructible( + const T* p, std::true_type /*has_get_arena*/) { + return p->GetArena(); + } + + static Arena* GetArenaForAllocationForNonMessageNonArenaConstructible( + const T* /* p */, std::false_type /*has_get_arena*/) { + return nullptr; + } + template static char DestructorSkippable(const typename U::DestructorSkippable_*); template static double DestructorSkippable(...); - typedef google::protobuf::internal::integral_constant< + typedef std::integral_constant< bool, sizeof(DestructorSkippable(static_cast(0))) == sizeof(char) || - google::protobuf::internal::has_trivial_destructor::value> + std::is_trivially_destructible::value> is_destructor_skippable; - template + template static char ArenaConstructable( const typename U::InternalArenaConstructable_*); - template + template static double ArenaConstructable(...); - typedef google::protobuf::internal::integral_constant( - static_cast(0))) == - sizeof(char)> + typedef std::integral_constant( + static_cast(0))) == + sizeof(char)> is_arena_constructable; -#if LANG_CXX11 + template () + .GetArena())>::value, + int>::type = 0> + static char HasGetArena(decltype(&U::GetArena)); + template + static double HasGetArena(...); + + typedef std::integral_constant(nullptr)) == + sizeof(char)> + has_get_arena; + template static T* Construct(void* ptr, Args&&... args) { - return new (ptr) T(std::forward(args)...); + return new (ptr) T(static_cast(args)...); } -#else - template - static T* Construct(void* ptr, const Arg1& arg1) { - return new (ptr) T(arg1); - } - template - static T* Construct(void* ptr, const Arg1& arg1, const Arg2& arg2) { - return new (ptr) T(arg1, arg2); - } - template - static T* Construct(void* ptr, const Arg1& arg1, - const Arg2& arg2, const Arg3& arg3) { - return new (ptr) T(arg1, arg2, arg3); + + static inline PROTOBUF_ALWAYS_INLINE T* New() { + return new T(nullptr); } -#endif // LANG_CXX11 - static Arena* GetArena(const T* p) { return p->GetArenaNoVirtual(); } + static Arena* GetArena(const T* p) { return p->GetArena(); } friend class Arena; + friend class TestUtil::ReflectionTester; }; - // Helper typetrait that indicates support for arenas in a type T at compile + // Helper typetraits that indicates support for arenas in a type T at compile // time. This is public only to allow construction of higher-level templated - // utilities. is_arena_constructable::value is true if the message type T - // has arena support enabled, and false otherwise. + // utilities. + // + // is_arena_constructable::value is true if the message type T has arena + // support enabled, and false otherwise. + // + // is_destructor_skippable::value is true if the message type T has told + // the arena that it is safe to skip the destructor, and false otherwise. // // This is inside Arena because only Arena has the friend relationships // necessary to see the underlying generated code traits. template struct is_arena_constructable : InternalHelper::is_arena_constructable {}; + template + struct is_destructor_skippable : InternalHelper::is_destructor_skippable { + }; private: - void CallDestructorHooks(); - void OnArenaAllocation(const std::type_info* allocated_type, size_t n) const; - inline void AllocHook(const std::type_info* allocated_type, size_t n) const { - if (GOOGLE_PREDICT_FALSE(hooks_cookie_ != NULL)) { - OnArenaAllocation(allocated_type, n); + internal::ThreadSafeArena impl_; + + template + struct has_get_arena : InternalHelper::has_get_arena {}; + + // Constructor solely used by message-owned arena. + inline Arena(internal::MessageOwned) : impl_(internal::MessageOwned{}) {} + + // Checks whether this arena is message-owned. + PROTOBUF_ALWAYS_INLINE bool IsMessageOwned() const { + return impl_.IsMessageOwned(); + } + + template + PROTOBUF_NDEBUG_INLINE static T* CreateMessageInternal(Arena* arena, + Args&&... args) { + static_assert( + InternalHelper::is_arena_constructable::value, + "CreateMessage can only construct types that are ArenaConstructable"); + if (arena == NULL) { + return new T(nullptr, static_cast(args)...); + } else { + return arena->DoCreateMessage(static_cast(args)...); + } + } + + // This specialization for no arguments is necessary, because its behavior is + // slightly different. When the arena pointer is nullptr, it calls T() + // instead of T(nullptr). + template + PROTOBUF_NDEBUG_INLINE static T* CreateMessageInternal(Arena* arena) { + static_assert( + InternalHelper::is_arena_constructable::value, + "CreateMessage can only construct types that are ArenaConstructable"); + if (arena == NULL) { + // Generated arena constructor T(Arena*) is protected. Call via + // InternalHelper. + return InternalHelper::New(); + } else { + return arena->DoCreateMessage(); } } - // Allocate and also optionally call on_arena_allocation callback with the - // allocated type info when the hooks are in place in ArenaOptions and - // the cookie is not null. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - void* AllocateInternal(bool skip_explicit_ownership) { - const size_t n = internal::AlignUpTo8(sizeof(T)); - AllocHook(RTTI_TYPE_ID(T), n); + // Allocate and also optionally call collector with the allocated type info + // when allocation recording is enabled. + PROTOBUF_NDEBUG_INLINE void* AllocateInternal(size_t size, size_t align, + void (*destructor)(void*), + const std::type_info* type) { // Monitor allocation if needed. - if (skip_explicit_ownership) { - return impl_.AllocateAligned(n); + if (destructor == nullptr) { + return AllocateAlignedWithHook(size, align, type); } else { - return impl_.AllocateAlignedAndAddCleanup( - n, &internal::arena_destruct_object); + if (align <= 8) { + auto res = AllocateAlignedWithCleanup(internal::AlignUpTo8(size), type); + res.second->elem = res.first; + res.second->cleanup = destructor; + return res.first; + } else { + auto res = AllocateAlignedWithCleanup(size + align - 8, type); + auto ptr = internal::AlignTo(res.first, align); + res.second->elem = ptr; + res.second->cleanup = destructor; + return ptr; + } } } @@ -666,218 +591,132 @@ class LIBPROTOBUF_EXPORT Arena { // as it can cause confusing API usages, and end up having double free in // user code. These are used only internally from LazyField and Repeated // fields, since they are designed to work in all mode combinations. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static Msg* CreateMaybeMessage(Arena* arena, google::protobuf::internal::true_type) { - return CreateMessage(arena); + template + PROTOBUF_ALWAYS_INLINE static Msg* DoCreateMaybeMessage(Arena* arena, + std::true_type, + Args&&... args) { + return CreateMessageInternal(arena, std::forward(args)...); } - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* CreateMaybeMessage(Arena* arena, google::protobuf::internal::false_type) { - return Create(arena); + template + PROTOBUF_ALWAYS_INLINE static T* DoCreateMaybeMessage(Arena* arena, + std::false_type, + Args&&... args) { + return Create(arena, std::forward(args)...); } - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static T* CreateMaybeMessage(Arena* arena) { - return CreateMaybeMessage(arena, is_arena_constructable()); + template + PROTOBUF_ALWAYS_INLINE static T* CreateMaybeMessage(Arena* arena, + Args&&... args) { + return DoCreateMaybeMessage(arena, is_arena_constructable(), + std::forward(args)...); } // Just allocate the required size for the given type assuming the // type has a trivial constructor. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternalRawArray(size_t num_elements) { - GOOGLE_CHECK_LE(num_elements, - std::numeric_limits::max() / sizeof(T)) + template + PROTOBUF_NDEBUG_INLINE T* CreateInternalRawArray(size_t num_elements) { + GOOGLE_CHECK_LE(num_elements, std::numeric_limits::max() / sizeof(T)) << "Requested size is too large to fit into size_t."; - const size_t n = internal::AlignUpTo8(sizeof(T) * num_elements); - // Monitor allocation if needed. - AllocHook(RTTI_TYPE_ID(T), n); - return static_cast(impl_.AllocateAligned(n)); + // We count on compiler to realize that if sizeof(T) is a multiple of + // 8 AlignUpTo can be elided. + const size_t n = sizeof(T) * num_elements; + return static_cast( + AllocateAlignedWithHook(n, alignof(T), RTTI_TYPE_ID(T))); } -#if LANG_CXX11 template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, Args&&... args) { - return new (AllocateInternal(skip_explicit_ownership)) - T(std::forward(args)...); - } -#else - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership) { - return new (AllocateInternal(skip_explicit_ownership)) T(); - } - - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, const Arg& arg) { - return new (AllocateInternal(skip_explicit_ownership)) T(arg); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, - const Arg1& arg1, - const Arg2& arg2) { - return new (AllocateInternal(skip_explicit_ownership)) T(arg1, arg2); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, - const Arg1& arg1, - const Arg2& arg2, - const Arg3& arg3) { - return new (AllocateInternal(skip_explicit_ownership)) - T(arg1, arg2, arg3); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, - const Arg1& arg1, - const Arg2& arg2, - const Arg3& arg3, - const Arg4& arg4) { - return new (AllocateInternal(skip_explicit_ownership)) - T(arg1, arg2, arg3, arg4); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, - const Arg1& arg1, - const Arg2& arg2, - const Arg3& arg3, - const Arg4& arg4, - const Arg5& arg5) { - return new (AllocateInternal(skip_explicit_ownership)) - T(arg1, arg2, arg3, arg4, arg5); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, - const Arg1& arg1, - const Arg2& arg2, - const Arg3& arg3, - const Arg4& arg4, - const Arg5& arg5, - const Arg6& arg6) { - return new (AllocateInternal(skip_explicit_ownership)) - T(arg1, arg2, arg3, arg4, arg5, arg6); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, - const Arg1& arg1, - const Arg2& arg2, - const Arg3& arg3, - const Arg4& arg4, - const Arg5& arg5, - const Arg6& arg6, - const Arg7& arg7) { - return new (AllocateInternal(skip_explicit_ownership)) - T(arg1, arg2, arg3, arg4, arg5, arg6, arg7); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateInternal(bool skip_explicit_ownership, - const Arg1& arg1, - const Arg2& arg2, - const Arg3& arg3, - const Arg4& arg4, - const Arg5& arg5, - const Arg6& arg6, - const Arg7& arg7, - const Arg8& arg8) { - return new (AllocateInternal(skip_explicit_ownership)) - T(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); - } -#endif -#if LANG_CXX11 - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE T* CreateMessageInternal( - Args&&... args) { + PROTOBUF_NDEBUG_INLINE T* DoCreateMessage(Args&&... args) { return InternalHelper::Construct( - AllocateInternal(InternalHelper::is_destructor_skippable::value), + AllocateInternal(sizeof(T), alignof(T), + internal::ObjectDestructor< + InternalHelper::is_destructor_skippable::value, + T>::destructor, + RTTI_TYPE_ID(T)), this, std::forward(args)...); } -#endif - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE T* CreateMessageInternal() { - return InternalHelper::Construct( - AllocateInternal(InternalHelper::is_destructor_skippable::value), - this); - } - - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateMessageInternal(const Arg& arg) { - return InternalHelper::Construct( - AllocateInternal(InternalHelper::is_destructor_skippable::value), - this, arg); - } - - template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - T* CreateMessageInternal(const Arg1& arg1, const Arg2& arg2) { - return InternalHelper::Construct( - AllocateInternal(InternalHelper::is_destructor_skippable::value), - this, arg1, arg2); - } // CreateInArenaStorage is used to implement map field. Without it, - // google::protobuf::Map need to call generated message's protected arena constructor, - // which needs to declare google::protobuf::Map as friend of generated message. - template - static void CreateInArenaStorage(T* ptr, Arena* arena) { + // Map need to call generated message's protected arena constructor, + // which needs to declare Map as friend of generated message. + template + static void CreateInArenaStorage(T* ptr, Arena* arena, Args&&... args) { CreateInArenaStorageInternal(ptr, arena, - typename is_arena_constructable::type()); - RegisterDestructorInternal( - ptr, arena, - typename InternalHelper::is_destructor_skippable::type()); + typename is_arena_constructable::type(), + std::forward(args)...); + if (arena != nullptr) { + RegisterDestructorInternal( + ptr, arena, + typename InternalHelper::is_destructor_skippable::type()); + } } - template - static void CreateInArenaStorageInternal( - T* ptr, Arena* arena, google::protobuf::internal::true_type) { - InternalHelper::Construct(ptr, arena); + template + static void CreateInArenaStorageInternal(T* ptr, Arena* arena, + std::true_type, Args&&... args) { + InternalHelper::Construct(ptr, arena, std::forward(args)...); } - template - static void CreateInArenaStorageInternal( - T* ptr, Arena* /* arena */, google::protobuf::internal::false_type) { - new (ptr) T(); + template + static void CreateInArenaStorageInternal(T* ptr, Arena* /* arena */, + std::false_type, Args&&... args) { + new (ptr) T(std::forward(args)...); } template - static void RegisterDestructorInternal( - T* /* ptr */, Arena* /* arena */, google::protobuf::internal::true_type) {} + static void RegisterDestructorInternal(T* /* ptr */, Arena* /* arena */, + std::true_type) {} template - static void RegisterDestructorInternal( - T* ptr, Arena* arena, google::protobuf::internal::false_type) { + static void RegisterDestructorInternal(T* ptr, Arena* arena, + std::false_type) { arena->OwnDestructor(ptr); } + // These implement Create(). The second parameter has type 'true_type' if T is + // a subtype of Message and 'false_type' otherwise. + template + PROTOBUF_ALWAYS_INLINE static T* CreateInternal(Arena* arena, std::true_type, + Args&&... args) { + if (arena == nullptr) { + return new T(std::forward(args)...); + } else { + auto destructor = + internal::ObjectDestructor::value, + T>::destructor; + T* result = + new (arena->AllocateInternal(sizeof(T), alignof(T), destructor, + RTTI_TYPE_ID(T))) + T(std::forward(args)...); + return result; + } + } + template + PROTOBUF_ALWAYS_INLINE static T* CreateInternal(Arena* arena, std::false_type, + Args&&... args) { + if (arena == nullptr) { + return new T(std::forward(args)...); + } else { + auto destructor = + internal::ObjectDestructor::value, + T>::destructor; + return new (arena->AllocateInternal(sizeof(T), alignof(T), destructor, + RTTI_TYPE_ID(T))) + T(std::forward(args)...); + } + } + // These implement Own(), which registers an object for deletion (destructor // call and operator delete()). The second parameter has type 'true_type' if T - // is a subtype of ::google::protobuf::Message and 'false_type' otherwise. Collapsing + // is a subtype of Message and 'false_type' otherwise. Collapsing // all template instantiations to one for generic Message reduces code size, // using the virtual destructor instead. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - void OwnInternal(T* object, google::protobuf::internal::true_type) { + template + PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::true_type) { if (object != NULL) { - impl_.AddCleanup(object, - &internal::arena_delete_object< ::google::protobuf::Message>); + impl_.AddCleanup(object, &internal::arena_delete_object); } } - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - void OwnInternal(T* object, google::protobuf::internal::false_type) { + template + PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::false_type) { if (object != NULL) { impl_.AddCleanup(object, &internal::arena_delete_object); } @@ -885,42 +724,88 @@ class LIBPROTOBUF_EXPORT Arena { // Implementation for GetArena(). Only message objects with // InternalArenaConstructable_ tags can be associated with an arena, and such - // objects must implement a GetArenaNoVirtual() method. - template GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static ::google::protobuf::Arena* GetArenaInternal( - const T* value, google::protobuf::internal::true_type) { + // objects must implement a GetArena() method. + template ::value, int>::type = 0> + PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) { return InternalHelper::GetArena(value); } + template ::value && + has_get_arena::value, + int>::type = 0> + PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) { + return value->GetArena(); + } + template ::value && + !has_get_arena::value, + int>::type = 0> + PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) { + (void)value; + return nullptr; + } template - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - static ::google::protobuf::Arena* GetArenaInternal( - const T* /* value */, google::protobuf::internal::false_type) { - return NULL; + PROTOBUF_ALWAYS_INLINE static Arena* GetOwningArena(const T* value) { + return GetOwningArenaInternal( + value, std::is_convertible()); } - // For friends of arena. - void* AllocateAligned(size_t n) { - AllocHook(NULL, n); - return impl_.AllocateAligned(internal::AlignUpTo8(n)); + // Implementation for GetOwningArena(). All and only message objects have + // GetOwningArena() method. + template + PROTOBUF_ALWAYS_INLINE static Arena* GetOwningArenaInternal( + const T* value, std::true_type) { + return InternalHelper::GetOwningArena(value); + } + template + PROTOBUF_ALWAYS_INLINE static Arena* GetOwningArenaInternal( + const T* /* value */, std::false_type) { + return nullptr; } - internal::ArenaImpl impl_; + // For friends of arena. + void* AllocateAligned(size_t n, size_t align = 8) { + if (align <= 8) { + return AllocateAlignedNoHook(internal::AlignUpTo8(n)); + } else { + // We are wasting space by over allocating align - 8 bytes. Compared + // to a dedicated function that takes current alignment in consideration. + // Such a scheme would only waste (align - 8)/2 bytes on average, but + // requires a dedicated function in the outline arena allocation + // functions. Possibly re-evaluate tradeoffs later. + return internal::AlignTo(AllocateAlignedNoHook(n + align - 8), align); + } + } - void* (*on_arena_init_)(Arena* arena); - void (*on_arena_allocation_)(const std::type_info* allocated_type, - uint64 alloc_size, void* cookie); - void (*on_arena_reset_)(Arena* arena, void* cookie, uint64 space_used); - void (*on_arena_destruction_)(Arena* arena, void* cookie, uint64 space_used); + void* AllocateAlignedWithHook(size_t n, size_t align, + const std::type_info* type) { + if (align <= 8) { + return AllocateAlignedWithHook(internal::AlignUpTo8(n), type); + } else { + // We are wasting space by over allocating align - 8 bytes. Compared + // to a dedicated function that takes current alignment in consideration. + // Such a schemee would only waste (align - 8)/2 bytes on average, but + // requires a dedicated function in the outline arena allocation + // functions. Possibly re-evaluate tradeoffs later. + return internal::AlignTo(AllocateAlignedWithHook(n + align - 8, type), + align); + } + } - // The arena may save a cookie it receives from the external on_init hook - // and then use it when calling the on_reset and on_destruction hooks. - void* hooks_cookie_; + void* AllocateAlignedNoHook(size_t n); + void* AllocateAlignedWithHook(size_t n, const std::type_info* type); + std::pair + AllocateAlignedWithCleanup(size_t n, const std::type_info* type); template - friend class ::google::protobuf::internal::GenericTypeHandler; + friend class internal::GenericTypeHandler; friend struct internal::ArenaStringPtr; // For AllocateAligned. - friend class internal::LazyField; // For CreateMaybeMessage. + friend class internal::InlinedStringField; // For AllocateAligned. + friend class internal::LazyField; // For CreateMaybeMessage. + friend class internal::EpsCopyInputStream; // For parser performance + friend class MessageLite; template friend class Map; }; @@ -929,6 +814,8 @@ class LIBPROTOBUF_EXPORT Arena { #undef RTTI_TYPE_ID } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_ARENA_H__ diff --git a/3rdparty/protobuf/src/google/protobuf/arena_impl.h b/3rdparty/protobuf/src/google/protobuf/arena_impl.h index 6cc7096bfbe0..2ffac319b411 100644 --- a/3rdparty/protobuf/src/google/protobuf/arena_impl.h +++ b/3rdparty/protobuf/src/google/protobuf/arena_impl.h @@ -33,210 +33,530 @@ #ifndef GOOGLE_PROTOBUF_ARENA_IMPL_H__ #define GOOGLE_PROTOBUF_ARENA_IMPL_H__ +#include #include +#include -#include -#include #include #include -#include -#include -#include +#ifdef ADDRESS_SANITIZER +#include +#endif // ADDRESS_SANITIZER + +#include -namespace google { +namespace google { namespace protobuf { namespace internal { -inline size_t AlignUpTo8(size_t n) { +inline constexpr size_t AlignUpTo8(size_t n) { // Align n to next multiple of 8 (from Hacker's Delight, Chapter 3.) - return (n + 7) & -8; + return (n + 7) & static_cast(-8); } +using LifecycleIdAtomic = uint64_t; + +// MetricsCollector collects stats for a particular arena. +class PROTOBUF_EXPORT ArenaMetricsCollector { + public: + ArenaMetricsCollector(bool record_allocs) : record_allocs_(record_allocs) {} + + // Invoked when the arena is about to be destroyed. This method will + // typically finalize any metric collection and delete the collector. + // space_allocated is the space used by the arena. + virtual void OnDestroy(uint64_t space_allocated) = 0; + + // OnReset() is called when the associated arena is reset. + // space_allocated is the space used by the arena just before the reset. + virtual void OnReset(uint64_t space_allocated) = 0; + + // OnAlloc is called when an allocation happens. + // type_info is promised to be static - its lifetime extends to + // match program's lifetime (It is given by typeid operator). + // Note: typeid(void) will be passed as allocated_type every time we + // intentionally want to avoid monitoring an allocation. (i.e. internal + // allocations for managing the arena) + virtual void OnAlloc(const std::type_info* allocated_type, + uint64_t alloc_size) = 0; + + // Does OnAlloc() need to be called? If false, metric collection overhead + // will be reduced since we will not do extra work per allocation. + bool RecordAllocs() { return record_allocs_; } + + protected: + // This class is destructed by the call to OnDestroy(). + ~ArenaMetricsCollector() = default; + const bool record_allocs_; +}; + +struct AllocationPolicy { + static constexpr size_t kDefaultStartBlockSize = 256; + static constexpr size_t kDefaultMaxBlockSize = 8192; + + size_t start_block_size = kDefaultStartBlockSize; + size_t max_block_size = kDefaultMaxBlockSize; + void* (*block_alloc)(size_t) = nullptr; + void (*block_dealloc)(void*, size_t) = nullptr; + ArenaMetricsCollector* metrics_collector = nullptr; + + bool IsDefault() const { + return start_block_size == kDefaultMaxBlockSize && + max_block_size == kDefaultMaxBlockSize && block_alloc == nullptr && + block_dealloc == nullptr && metrics_collector == nullptr; + } +}; + +// Tagged pointer to an AllocationPolicy. +class TaggedAllocationPolicyPtr { + public: + constexpr TaggedAllocationPolicyPtr() : policy_(0) {} + + explicit TaggedAllocationPolicyPtr(AllocationPolicy* policy) + : policy_(reinterpret_cast(policy)) {} + + void set_policy(AllocationPolicy* policy) { + auto bits = policy_ & kTagsMask; + policy_ = reinterpret_cast(policy) | bits; + } + + AllocationPolicy* get() { + return reinterpret_cast(policy_ & kPtrMask); + } + const AllocationPolicy* get() const { + return reinterpret_cast(policy_ & kPtrMask); + } + + AllocationPolicy& operator*() { return *get(); } + const AllocationPolicy& operator*() const { return *get(); } + + AllocationPolicy* operator->() { return get(); } + const AllocationPolicy* operator->() const { return get(); } + + bool is_user_owned_initial_block() const { + return static_cast(get_mask()); + } + void set_is_user_owned_initial_block(bool v) { + set_mask(v); + } + + bool should_record_allocs() const { + return static_cast(get_mask()); + } + void set_should_record_allocs(bool v) { set_mask(v); } + + uintptr_t get_raw() const { return policy_; } + + inline void RecordAlloc(const std::type_info* allocated_type, + size_t n) const { + get()->metrics_collector->OnAlloc(allocated_type, n); + } + + private: + enum : uintptr_t { + kUserOwnedInitialBlock = 1, + kRecordAllocs = 2, + }; + + static constexpr uintptr_t kTagsMask = 7; + static constexpr uintptr_t kPtrMask = ~kTagsMask; + + template + uintptr_t get_mask() const { + return policy_ & kMask; + } + template + void set_mask(bool v) { + if (v) { + policy_ |= kMask; + } else { + policy_ &= ~kMask; + } + } + uintptr_t policy_; +}; + +// A simple arena allocator. Calls to allocate functions must be properly +// serialized by the caller, hence this class cannot be used as a general +// purpose allocator in a multi-threaded program. It serves as a building block +// for ThreadSafeArena, which provides a thread-safe arena allocator. +// +// This class manages +// 1) Arena bump allocation + owning memory blocks. +// 2) Maintaining a cleanup list. +// It delagetes the actual memory allocation back to ThreadSafeArena, which +// contains the information on block growth policy and backing memory allocation +// used. +class PROTOBUF_EXPORT SerialArena { + public: + struct Memory { + void* ptr; + size_t size; + }; + + // Node contains the ptr of the object to be cleaned up and the associated + // cleanup function ptr. + struct CleanupNode { + void* elem; // Pointer to the object to be cleaned up. + void (*cleanup)(void*); // Function pointer to the destructor or deleter. + }; + + void CleanupList(); + uint64_t SpaceAllocated() const { + return space_allocated_.load(std::memory_order_relaxed); + } + uint64_t SpaceUsed() const; + + bool HasSpace(size_t n) { return n <= static_cast(limit_ - ptr_); } + + void* AllocateAligned(size_t n, const AllocationPolicy* policy) { + GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. + GOOGLE_DCHECK_GE(limit_, ptr_); + if (PROTOBUF_PREDICT_FALSE(!HasSpace(n))) { + return AllocateAlignedFallback(n, policy); + } + return AllocateFromExisting(n); + } + + private: + void* AllocateFromExisting(size_t n) { + void* ret = ptr_; + ptr_ += n; +#ifdef ADDRESS_SANITIZER + ASAN_UNPOISON_MEMORY_REGION(ret, n); +#endif // ADDRESS_SANITIZER + return ret; + } + + public: + // Allocate space if the current region provides enough space. + bool MaybeAllocateAligned(size_t n, void** out) { + GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. + GOOGLE_DCHECK_GE(limit_, ptr_); + if (PROTOBUF_PREDICT_FALSE(!HasSpace(n))) return false; + *out = AllocateFromExisting(n); + return true; + } + + std::pair AllocateAlignedWithCleanup( + size_t n, const AllocationPolicy* policy) { + GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. + if (PROTOBUF_PREDICT_FALSE(!HasSpace(n + kCleanupSize))) { + return AllocateAlignedWithCleanupFallback(n, policy); + } + return AllocateFromExistingWithCleanupFallback(n); + } + + private: + std::pair AllocateFromExistingWithCleanupFallback( + size_t n) { + void* ret = ptr_; + ptr_ += n; + limit_ -= kCleanupSize; +#ifdef ADDRESS_SANITIZER + ASAN_UNPOISON_MEMORY_REGION(ret, n); + ASAN_UNPOISON_MEMORY_REGION(limit_, kCleanupSize); +#endif // ADDRESS_SANITIZER + return CreatePair(ret, reinterpret_cast(limit_)); + } + + public: + void AddCleanup(void* elem, void (*cleanup)(void*), + const AllocationPolicy* policy) { + auto res = AllocateAlignedWithCleanup(0, policy); + res.second->elem = elem; + res.second->cleanup = cleanup; + } + + void* owner() const { return owner_; } + SerialArena* next() const { return next_; } + void set_next(SerialArena* next) { next_ = next; } + + private: + friend class ThreadSafeArena; + friend class ArenaBenchmark; + + // Creates a new SerialArena inside mem using the remaining memory as for + // future allocations. + static SerialArena* New(SerialArena::Memory mem, void* owner); + // Free SerialArena returning the memory passed in to New + template + Memory Free(Deallocator deallocator); + + // Blocks are variable length malloc-ed objects. The following structure + // describes the common header for all blocks. + struct Block { + Block(Block* next, size_t size) : next(next), size(size), start(nullptr) {} + + char* Pointer(size_t n) { + GOOGLE_DCHECK(n <= size); + return reinterpret_cast(this) + n; + } + + Block* const next; + const size_t size; + CleanupNode* start; + // data follows + }; + + void* owner_; // &ThreadCache of this thread; + Block* head_; // Head of linked list of blocks. + SerialArena* next_; // Next SerialArena in this linked list. + size_t space_used_ = 0; // Necessary for metrics. + std::atomic space_allocated_; + + // Next pointer to allocate from. Always 8-byte aligned. Points inside + // head_ (and head_->pos will always be non-canonical). We keep these + // here to reduce indirection. + char* ptr_; + char* limit_; + + // Constructor is private as only New() should be used. + inline SerialArena(Block* b, void* owner); + void* AllocateAlignedFallback(size_t n, const AllocationPolicy* policy); + std::pair AllocateAlignedWithCleanupFallback( + size_t n, const AllocationPolicy* policy); + void AllocateNewBlock(size_t n, const AllocationPolicy* policy); + + std::pair CreatePair(void* ptr, CleanupNode* node) { + return {ptr, node}; + } + + public: + static constexpr size_t kBlockHeaderSize = AlignUpTo8(sizeof(Block)); + static constexpr size_t kCleanupSize = AlignUpTo8(sizeof(CleanupNode)); +}; + +// Tag type used to invoke the constructor of message-owned arena. +// Only message-owned arenas use this constructor for creation. +// Such constructors are internal implementation details of the library. +struct MessageOwned { + explicit MessageOwned() = default; +}; + // This class provides the core Arena memory allocation library. Different // implementations only need to implement the public interface below. // Arena is not a template type as that would only be useful if all protos // in turn would be templates, which will/cannot happen. However separating // the memory allocation part from the cruft of the API users expect we can // use #ifdef the select the best implementation based on hardware / OS. -class LIBPROTOBUF_EXPORT ArenaImpl { +class PROTOBUF_EXPORT ThreadSafeArena { public: - struct Options { - size_t start_block_size; - size_t max_block_size; - char* initial_block; - size_t initial_block_size; - void* (*block_alloc)(size_t); - void (*block_dealloc)(void*, size_t); - - template - explicit Options(const O& options) - : start_block_size(options.start_block_size), - max_block_size(options.max_block_size), - initial_block(options.initial_block), - initial_block_size(options.initial_block_size), - block_alloc(options.block_alloc), - block_dealloc(options.block_dealloc) {} - }; - - template - explicit ArenaImpl(const O& options) : options_(options) { - if (options_.initial_block != NULL && options_.initial_block_size > 0) { - GOOGLE_CHECK_GE(options_.initial_block_size, sizeof(Block)) - << ": Initial block size too small for header."; - initial_block_ = reinterpret_cast(options_.initial_block); - } else { - initial_block_ = NULL; - } + ThreadSafeArena() { Init(); } + // Constructor solely used by message-owned arena. + ThreadSafeArena(internal::MessageOwned) : tag_and_id_(kMessageOwnedArena) { Init(); } + ThreadSafeArena(char* mem, size_t size) { InitializeFrom(mem, size); } + + explicit ThreadSafeArena(void* mem, size_t size, + const AllocationPolicy& policy) { + InitializeWithPolicy(mem, size, policy); + } + // Destructor deletes all owned heap allocated objects, and destructs objects // that have non-trivial destructors, except for proto2 message objects whose // destructors can be skipped. Also, frees all blocks except the initial block // if it was passed in. - ~ArenaImpl(); + ~ThreadSafeArena(); - uint64 Reset(); + uint64_t Reset(); - uint64 SpaceAllocated() const; - uint64 SpaceUsed() const; + uint64_t SpaceAllocated() const; + uint64_t SpaceUsed() const; - void* AllocateAligned(size_t n); + void* AllocateAligned(size_t n, const std::type_info* type) { + SerialArena* arena; + if (PROTOBUF_PREDICT_TRUE(!alloc_policy_.should_record_allocs() && + GetSerialArenaFast(&arena))) { + return arena->AllocateAligned(n, AllocPolicy()); + } else { + return AllocateAlignedFallback(n, type); + } + } - void* AllocateAlignedAndAddCleanup(size_t n, void (*cleanup)(void*)); + // This function allocates n bytes if the common happy case is true and + // returns true. Otherwise does nothing and returns false. This strange + // semantics is necessary to allow callers to program functions that only + // have fallback function calls in tail position. This substantially improves + // code for the happy path. + PROTOBUF_NDEBUG_INLINE bool MaybeAllocateAligned(size_t n, void** out) { + SerialArena* a; + if (PROTOBUF_PREDICT_TRUE(!alloc_policy_.should_record_allocs() && + GetSerialArenaFromThreadCache(&a))) { + return a->MaybeAllocateAligned(n, out); + } + return false; + } + + std::pair AllocateAlignedWithCleanup( + size_t n, const std::type_info* type); // Add object pointer and cleanup function pointer to the list. void AddCleanup(void* elem, void (*cleanup)(void*)); + // Checks whether this arena is message-owned. + PROTOBUF_ALWAYS_INLINE bool IsMessageOwned() const { + return tag_and_id_ & kMessageOwnedArena; + } + private: - // Node contains the ptr of the object to be cleaned up and the associated - // cleanup function ptr. - struct CleanupNode { - void* elem; // Pointer to the object to be cleaned up. - void (*cleanup)(void*); // Function pointer to the destructor or deleter. - }; + // Unique for each arena. Changes on Reset(). + uint64_t tag_and_id_ = 0; + // The LSB of tag_and_id_ indicates if the arena is message-owned. + enum : uint64_t { kMessageOwnedArena = 1 }; + + TaggedAllocationPolicyPtr alloc_policy_; // Tagged pointer to AllocPolicy. + + // Pointer to a linked list of SerialArena. + std::atomic threads_; + std::atomic hint_; // Fast thread-local block access + + const AllocationPolicy* AllocPolicy() const { return alloc_policy_.get(); } + void InitializeFrom(void* mem, size_t size); + void InitializeWithPolicy(void* mem, size_t size, AllocationPolicy policy); + void* AllocateAlignedFallback(size_t n, const std::type_info* type); + std::pair + AllocateAlignedWithCleanupFallback(size_t n, const std::type_info* type); + + void Init(); + void SetInitialBlock(void* mem, size_t size); - // Cleanup uses a chunked linked list, to reduce pointer chasing. - struct CleanupChunk { - static size_t SizeOf(size_t i) { - return sizeof(CleanupChunk) + (sizeof(CleanupNode) * (i - 1)); + // Delete or Destruct all objects owned by the arena. + void CleanupList(); + + inline uint64_t LifeCycleId() const { + return tag_and_id_ & ~kMessageOwnedArena; + } + + inline void CacheSerialArena(SerialArena* serial) { + thread_cache().last_serial_arena = serial; + thread_cache().last_lifecycle_id_seen = tag_and_id_; + // TODO(haberman): evaluate whether we would gain efficiency by getting rid + // of hint_. It's the only write we do to ThreadSafeArena in the allocation + // path, which will dirty the cache line. + + hint_.store(serial, std::memory_order_release); + } + + PROTOBUF_NDEBUG_INLINE bool GetSerialArenaFast(SerialArena** arena) { + if (GetSerialArenaFromThreadCache(arena)) return true; + + // Check whether we own the last accessed SerialArena on this arena. This + // fast path optimizes the case where a single thread uses multiple arenas. + ThreadCache* tc = &thread_cache(); + SerialArena* serial = hint_.load(std::memory_order_acquire); + if (PROTOBUF_PREDICT_TRUE(serial != NULL && serial->owner() == tc)) { + *arena = serial; + return true; } - size_t len; // Number of elements currently present. - size_t size; // Total elements in the list. - CleanupChunk* next; // Next node in the list. - CleanupNode nodes[1]; // True length is |size|. - }; + return false; + } + + PROTOBUF_NDEBUG_INLINE bool GetSerialArenaFromThreadCache( + SerialArena** arena) { + // If this thread already owns a block in this arena then try to use that. + // This fast path optimizes the case where multiple threads allocate from + // the same arena. + ThreadCache* tc = &thread_cache(); + if (PROTOBUF_PREDICT_TRUE(tc->last_lifecycle_id_seen == tag_and_id_)) { + *arena = tc->last_serial_arena; + return true; + } + return false; + } + SerialArena* GetSerialArenaFallback(void* me); - struct Block; + template + void PerSerialArena(Functor fn) { + // By omitting an Acquire barrier we ensure that any user code that doesn't + // properly synchronize Reset() or the destructor will throw a TSAN warning. + SerialArena* serial = threads_.load(std::memory_order_relaxed); - // Tracks per-thread info. ThreadInfos are kept in a linked list. - struct ThreadInfo { - void *owner; // &ThreadCache of this thread; - Block* head; // Head of linked list of blocks. - CleanupChunk* cleanup; // Head of cleanup list. - ThreadInfo* next; // Next ThreadInfo in this linked list. - }; + for (; serial; serial = serial->next()) fn(serial); + } - // Blocks are variable length malloc-ed objects. The following structure - // describes the common header for all blocks. - struct Block { - void* owner; // &ThreadCache of thread that owns this block. - ThreadInfo* thread_info; // ThreadInfo of thread that owns this block. - Block* next; // Next block in arena (may have different owner) - // ((char*) &block) + pos is next available byte. It is always - // aligned at a multiple of 8 bytes. - size_t pos; - size_t size; // total size of the block. - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - size_t avail() const { return size - pos; } - // data follows - }; + // Releases all memory except the first block which it returns. The first + // block might be owned by the user and thus need some extra checks before + // deleting. + SerialArena::Memory Free(size_t* space_allocated); - struct ThreadCache { +#ifdef _MSC_VER +#pragma warning(disable : 4324) +#endif + struct alignas(64) ThreadCache { #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) // If we are using the ThreadLocalStorage class to store the ThreadCache, // then the ThreadCache's default constructor has to be responsible for // initializing it. - ThreadCache() : last_lifecycle_id_seen(-1), last_block_used_(NULL) {} + ThreadCache() + : next_lifecycle_id(0), + last_lifecycle_id_seen(-1), + last_serial_arena(NULL) {} #endif + // Number of per-thread lifecycle IDs to reserve. Must be power of two. + // To reduce contention on a global atomic, each thread reserves a batch of + // IDs. The following number is calculated based on a stress test with + // ~6500 threads all frequently allocating a new arena. + static constexpr size_t kPerThreadIds = 256; + // Next lifecycle ID available to this thread. We need to reserve a new + // batch, if `next_lifecycle_id & (kPerThreadIds - 1) == 0`. + uint64_t next_lifecycle_id; // The ThreadCache is considered valid as long as this matches the // lifecycle_id of the arena being used. - int64 last_lifecycle_id_seen; - Block* last_block_used_; + uint64_t last_lifecycle_id_seen; + SerialArena* last_serial_arena; + }; + + // Lifecycle_id can be highly contended variable in a situation of lots of + // arena creation. Make sure that other global variables are not sharing the + // cacheline. +#ifdef _MSC_VER +#pragma warning(disable : 4324) +#endif + struct alignas(64) CacheAlignedLifecycleIdGenerator { + std::atomic id; }; - static google::protobuf::internal::SequenceNumber lifecycle_id_generator_; + static CacheAlignedLifecycleIdGenerator lifecycle_id_generator_; #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) - // Android ndk does not support GOOGLE_THREAD_LOCAL keyword so we use a custom thread - // local storage class we implemented. - // iOS also does not support the GOOGLE_THREAD_LOCAL keyword. + // iOS does not support __thread keyword so we use a custom thread local + // storage class we implemented. static ThreadCache& thread_cache(); #elif defined(PROTOBUF_USE_DLLS) // Thread local variables cannot be exposed through DLL interface but we can // wrap them in static functions. static ThreadCache& thread_cache(); #else - static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_; + static PROTOBUF_THREAD_LOCAL ThreadCache thread_cache_; static ThreadCache& thread_cache() { return thread_cache_; } #endif - void Init(); - - // Free all blocks and return the total space used which is the sums of sizes - // of the all the allocated blocks. - uint64 FreeBlocks(); - - void AddCleanupInBlock(Block* b, void* elem, void (*func)(void*)); - CleanupChunk* ExpandCleanupList(CleanupChunk* cleanup, Block* b); - // Delete or Destruct all objects owned by the arena. - void CleanupList(); - - inline void CacheBlock(Block* block) { - thread_cache().last_block_used_ = block; - thread_cache().last_lifecycle_id_seen = lifecycle_id_; - // TODO(haberman): evaluate whether we would gain efficiency by getting rid - // of hint_. It's the only write we do to ArenaImpl in the allocation path, - // which will dirty the cache line. - google::protobuf::internal::Release_Store(&hint_, reinterpret_cast(block)); - } - - google::protobuf::internal::AtomicWord threads_; // Pointer to a linked list of ThreadInfo. - google::protobuf::internal::AtomicWord hint_; // Fast thread-local block access - google::protobuf::internal::AtomicWord space_allocated_; // Sum of sizes of all allocated blocks. - - Block *initial_block_; // If non-NULL, points to the block that came from - // user data. - - // Returns a block owned by this thread. - Block* GetBlock(size_t n); - Block* GetBlockSlow(void* me, Block* my_full_block, size_t n); - Block* NewBlock(void* me, Block* my_last_block, size_t min_bytes); - void InitBlock(Block* b, void *me, size_t size); - static void* AllocFromBlock(Block* b, size_t n); - ThreadInfo* NewThreadInfo(Block* b); - ThreadInfo* FindThreadInfo(void* me); - ThreadInfo* GetThreadInfo(void* me, size_t n); - - int64 lifecycle_id_; // Unique for each arena. Changes on Reset(). - - Options options_; - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArenaImpl); + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ThreadSafeArena); + // All protos have pointers back to the arena hence Arena must have + // pointer stability. + ThreadSafeArena(ThreadSafeArena&&) = delete; + ThreadSafeArena& operator=(ThreadSafeArena&&) = delete; public: - // kHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8 to - // protect the invariant that pos is always at a multiple of 8. - static const size_t kHeaderSize = (sizeof(Block) + 7) & -8; -#if LANG_CXX11 - static_assert(kHeaderSize % 8 == 0, "kHeaderSize must be a multiple of 8."); -#endif + // kBlockHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8 + // to protect the invariant that pos is always at a multiple of 8. + static constexpr size_t kBlockHeaderSize = SerialArena::kBlockHeaderSize; + static constexpr size_t kSerialArenaSize = + (sizeof(SerialArena) + 7) & static_cast(-8); + static_assert(kBlockHeaderSize % 8 == 0, + "kBlockHeaderSize must be a multiple of 8."); + static_assert(kSerialArenaSize % 8 == 0, + "kSerialArenaSize must be a multiple of 8."); }; } // namespace internal } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_ARENA_IMPL_H__ diff --git a/3rdparty/protobuf/src/google/protobuf/arenastring.cc b/3rdparty/protobuf/src/google/protobuf/arenastring.cc index 7f33a0c8658d..169f52729d61 100644 --- a/3rdparty/protobuf/src/google/protobuf/arenastring.cc +++ b/3rdparty/protobuf/src/google/protobuf/arenastring.cc @@ -28,16 +28,259 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// The ArenaString implementation is not included in the open-source release. Do -// not include this file in the distribution. - #include +#include +#include +#include +#include +#include +#include +#include +#include + +// clang-format off +#include +// clang-format on + namespace google { namespace protobuf { namespace internal { +const std::string& LazyString::Init() const { + static WrappedMutex mu{GOOGLE_PROTOBUF_LINKER_INITIALIZED}; + mu.Lock(); + const std::string* res = inited_.load(std::memory_order_acquire); + if (res == nullptr) { + auto init_value = init_value_; + res = ::new (static_cast(string_buf_)) + std::string(init_value.ptr, init_value.size); + inited_.store(res, std::memory_order_release); + } + mu.Unlock(); + return *res; +} + + +std::string* ArenaStringPtr::SetAndReturnNewString() { + std::string* new_string = new std::string(); + tagged_ptr_.Set(new_string); + return new_string; +} + +void ArenaStringPtr::DestroyNoArenaSlowPath() { delete UnsafeMutablePointer(); } + +void ArenaStringPtr::Set(const std::string* default_value, + ConstStringParam value, ::google::protobuf::Arena* arena) { + if (IsDefault(default_value)) { + tagged_ptr_.Set(Arena::Create(arena, value)); + } else { + UnsafeMutablePointer()->assign(value.data(), value.length()); + } +} + +void ArenaStringPtr::Set(const std::string* default_value, std::string&& value, + ::google::protobuf::Arena* arena) { + if (IsDefault(default_value)) { + if (arena == nullptr) { + tagged_ptr_.Set(new std::string(std::move(value))); + } else { + tagged_ptr_.Set(Arena::Create(arena, std::move(value))); + } + } else if (IsDonatedString()) { + std::string* current = tagged_ptr_.Get(); + auto* s = new (current) std::string(std::move(value)); + arena->OwnDestructor(s); + tagged_ptr_.Set(s); + } else /* !IsDonatedString() */ { + *UnsafeMutablePointer() = std::move(value); + } +} + +void ArenaStringPtr::Set(EmptyDefault, ConstStringParam value, + ::google::protobuf::Arena* arena) { + Set(&GetEmptyStringAlreadyInited(), value, arena); +} + +void ArenaStringPtr::Set(EmptyDefault, std::string&& value, + ::google::protobuf::Arena* arena) { + Set(&GetEmptyStringAlreadyInited(), std::move(value), arena); +} + +void ArenaStringPtr::Set(NonEmptyDefault, ConstStringParam value, + ::google::protobuf::Arena* arena) { + Set(nullptr, value, arena); +} + +void ArenaStringPtr::Set(NonEmptyDefault, std::string&& value, + ::google::protobuf::Arena* arena) { + Set(nullptr, std::move(value), arena); +} + +std::string* ArenaStringPtr::Mutable(EmptyDefault, ::google::protobuf::Arena* arena) { + if (!IsDonatedString() && !IsDefault(&GetEmptyStringAlreadyInited())) { + return UnsafeMutablePointer(); + } else { + return MutableSlow(arena); + } +} + +std::string* ArenaStringPtr::Mutable(const LazyString& default_value, + ::google::protobuf::Arena* arena) { + if (!IsDonatedString() && !IsDefault(nullptr)) { + return UnsafeMutablePointer(); + } else { + return MutableSlow(arena, default_value); + } +} + +std::string* ArenaStringPtr::MutableNoCopy(const std::string* default_value, + ::google::protobuf::Arena* arena) { + if (!IsDonatedString() && !IsDefault(default_value)) { + return UnsafeMutablePointer(); + } else { + GOOGLE_DCHECK(IsDefault(default_value)); + // Allocate empty. The contents are not relevant. + std::string* new_string = Arena::Create(arena); + tagged_ptr_.Set(new_string); + return new_string; + } +} + +template +std::string* ArenaStringPtr::MutableSlow(::google::protobuf::Arena* arena, + const Lazy&... lazy_default) { + const std::string* const default_value = + sizeof...(Lazy) == 0 ? &GetEmptyStringAlreadyInited() : nullptr; + GOOGLE_DCHECK(IsDefault(default_value)); + std::string* new_string = + Arena::Create(arena, lazy_default.get()...); + tagged_ptr_.Set(new_string); + return new_string; +} + +std::string* ArenaStringPtr::Release(const std::string* default_value, + ::google::protobuf::Arena* arena) { + if (IsDefault(default_value)) { + return nullptr; + } else { + return ReleaseNonDefault(default_value, arena); + } +} + +std::string* ArenaStringPtr::ReleaseNonDefault(const std::string* default_value, + ::google::protobuf::Arena* arena) { + GOOGLE_DCHECK(!IsDefault(default_value)); + + if (!IsDonatedString()) { + std::string* released; + if (arena != nullptr) { + released = new std::string; + released->swap(*UnsafeMutablePointer()); + } else { + released = UnsafeMutablePointer(); + } + tagged_ptr_.Set(const_cast(default_value)); + return released; + } else /* IsDonatedString() */ { + GOOGLE_DCHECK(arena != nullptr); + std::string* released = new std::string(Get()); + tagged_ptr_.Set(const_cast(default_value)); + return released; + } +} + +void ArenaStringPtr::SetAllocated(const std::string* default_value, + std::string* value, ::google::protobuf::Arena* arena) { + // Release what we have first. + if (arena == nullptr && !IsDefault(default_value)) { + delete UnsafeMutablePointer(); + } + if (value == nullptr) { + tagged_ptr_.Set(const_cast(default_value)); + } else { +#ifdef NDEBUG + tagged_ptr_.Set(value); + if (arena != nullptr) { + arena->Own(value); + } +#else + // On debug builds, copy the string so the address differs. delete will + // fail if value was a stack-allocated temporary/etc., which would have + // failed when arena ran its cleanup list. + std::string* new_value = Arena::Create(arena, *value); + delete value; + tagged_ptr_.Set(new_value); +#endif + } +} + +void ArenaStringPtr::Destroy(const std::string* default_value, + ::google::protobuf::Arena* arena) { + if (arena == nullptr) { + GOOGLE_DCHECK(!IsDonatedString()); + if (!IsDefault(default_value)) { + delete UnsafeMutablePointer(); + } + } +} + +void ArenaStringPtr::Destroy(EmptyDefault, ::google::protobuf::Arena* arena) { + Destroy(&GetEmptyStringAlreadyInited(), arena); +} + +void ArenaStringPtr::Destroy(NonEmptyDefault, ::google::protobuf::Arena* arena) { + Destroy(nullptr, arena); +} + +void ArenaStringPtr::ClearToEmpty() { + if (IsDefault(&GetEmptyStringAlreadyInited())) { + // Already set to default -- do nothing. + } else { + // Unconditionally mask away the tag. + // + // UpdateDonatedString uses assign when capacity is larger than the new + // value, which is trivially true in the donated string case. + // const_cast(PtrValue())->clear(); + tagged_ptr_.Get()->clear(); + } +} + +void ArenaStringPtr::ClearToDefault(const LazyString& default_value, + ::google::protobuf::Arena* arena) { + (void)arena; + if (IsDefault(nullptr)) { + // Already set to default -- do nothing. + } else if (!IsDonatedString()) { + UnsafeMutablePointer()->assign(default_value.get()); + } +} + +inline void SetStrWithHeapBuffer(std::string* str, ArenaStringPtr* s) { + TaggedPtr res; + res.Set(str); + s->UnsafeSetTaggedPointer(res); +} + +const char* EpsCopyInputStream::ReadArenaString(const char* ptr, + ArenaStringPtr* s, + Arena* arena) { + GOOGLE_DCHECK(arena != nullptr); + + int size = ReadSize(&ptr); + if (!ptr) return nullptr; + + auto* str = Arena::Create(arena); + ptr = ReadString(ptr, size, str); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + + SetStrWithHeapBuffer(str, s); + + return ptr; +} } // namespace internal } // namespace protobuf } // namespace google + +#include diff --git a/3rdparty/protobuf/src/google/protobuf/arenastring.h b/3rdparty/protobuf/src/google/protobuf/arenastring.h index c9d045a1592d..38c36378cd30 100644 --- a/3rdparty/protobuf/src/google/protobuf/arenastring.h +++ b/3rdparty/protobuf/src/google/protobuf/arenastring.h @@ -32,307 +32,389 @@ #define GOOGLE_PROTOBUF_ARENASTRING_H__ #include +#include +#include -#include -#include -#include #include -#include +#include +#include +#include + +#include + +#ifdef SWIG +#error "You cannot SWIG proto headers" +#endif -// This is the implementation of arena string fields written for the open-source -// release. The ArenaStringPtr struct below is an internal implementation class -// and *should not be used* by user code. It is used to collect string -// operations together into one place and abstract away the underlying -// string-field pointer representation, so that (for example) an alternate -// implementation that knew more about ::std::string's internals could integrate more -// closely with the arena allocator. namespace google { namespace protobuf { namespace internal { -struct LIBPROTOBUF_EXPORT ArenaStringPtr { - inline void Set(const ::std::string* default_value, - const ::std::string& value, ::google::protobuf::Arena* arena) { - if (ptr_ == default_value) { - CreateInstance(arena, &value); - } else { - *ptr_ = value; - } +template +class ExplicitlyConstructed; + +class SwapFieldHelper; + +// Lazy string instance to support string fields with non-empty default. +// These are initialized on the first call to .get(). +class PROTOBUF_EXPORT LazyString { + public: + // We explicitly make LazyString an aggregate so that MSVC can do constant + // initialization on it without marking it `constexpr`. + // We do not want to use `constexpr` because it makes it harder to have extern + // storage for it and causes library bloat. + struct InitValue { + const char* ptr; + size_t size; + }; + // We keep a union of the initialization value and the std::string to save on + // space. We don't need the string array after Init() is done. + union { + mutable InitValue init_value_; + alignas(std::string) mutable char string_buf_[sizeof(std::string)]; + }; + mutable std::atomic inited_; + + const std::string& get() const { + // This check generates less code than a call-once invocation. + auto* res = inited_.load(std::memory_order_acquire); + if (PROTOBUF_PREDICT_FALSE(res == nullptr)) return Init(); + return *res; } - inline void SetLite(const ::std::string* default_value, - const ::std::string& value, - ::google::protobuf::Arena* arena) { - Set(default_value, value, arena); - } + private: + // Initialize the string in `string_buf_`, update `inited_` and return it. + // We return it here to avoid having to read it again in the inlined code. + const std::string& Init() const; +}; - // Basic accessors. - inline const ::std::string& Get() const { return *ptr_; } - - inline ::std::string* Mutable(const ::std::string* default_value, - ::google::protobuf::Arena* arena) { - if (ptr_ == default_value) { - CreateInstance(arena, default_value); - } - return ptr_; - } +template +class TaggedPtr { + public: + TaggedPtr() = default; + explicit constexpr TaggedPtr(const ExplicitlyConstructed* ptr) + : ptr_(const_cast*>(ptr)) {} - // Release returns a ::std::string* instance that is heap-allocated and is not - // Own()'d by any arena. If the field was not set, it returns NULL. The caller - // retains ownership. Clears this field back to NULL state. Used to implement - // release_() methods on generated classes. - inline ::std::string* Release(const ::std::string* default_value, - ::google::protobuf::Arena* arena) { - if (ptr_ == default_value) { - return NULL; - } - ::std::string* released = NULL; - if (arena != NULL) { - // ptr_ is owned by the arena. - released = new ::std::string; - released->swap(*ptr_); - } else { - released = ptr_; - } - ptr_ = const_cast< ::std::string* >(default_value); - return released; + void SetTagged(T* p) { + Set(p); + ptr_ = reinterpret_cast(as_int() | 1); } + void Set(T* p) { ptr_ = p; } + T* Get() const { return reinterpret_cast(as_int() & -2); } + bool IsTagged() const { return as_int() & 1; } + + // Returned value is only safe to dereference if IsTagged() == false. + // It is safe to compare. + T* UnsafeGet() const { return static_cast(ptr_); } - // UnsafeArenaRelease returns a ::std::string*, but it may be arena-owned (i.e. - // have its destructor already registered) if arena != NULL. If the field was - // not set, this returns NULL. This method clears this field back to NULL - // state. Used to implement unsafe_arena_release_() methods on - // generated classes. - inline ::std::string* UnsafeArenaRelease(const ::std::string* default_value, - ::google::protobuf::Arena* /* arena */) { - if (ptr_ == default_value) { - return NULL; - } - ::std::string* released = ptr_; - ptr_ = const_cast< ::std::string* >(default_value); - return released; + bool IsNull() { return ptr_ == nullptr; } + + private: + uintptr_t as_int() const { return reinterpret_cast(ptr_); } + void* ptr_; +}; + +static_assert(std::is_trivial>::value, + "TaggedPtr must be trivial"); + +// This class encapsulates a pointer to a std::string with or without a donated +// buffer, tagged by bottom bit. It is a high-level wrapper that almost directly +// corresponds to the interface required by string fields in generated +// code. It replaces the old std::string* pointer in such cases. +// +// The object has different but similar code paths for when the default value is +// the empty string and when it is a non-empty string. +// The empty string is handled different throughout the library and there is a +// single global instance of it we can share. +// +// For fields with an empty string default value, there are three distinct +// states: +// +// - Pointer set to 'String' tag (LSB is 0), equal to +// &GetEmptyStringAlreadyInited(): field is set to its default value. Points +// to a true std::string*, but we do not own that std::string* (it's a +// globally shared instance). +// +// - Pointer set to 'String' tag (LSB is 0), but not equal to the global empty +// string: field points to a true std::string* instance that we own. This +// instance is either on the heap or on the arena (i.e. registered on +// free()/destructor-call list) as appropriate. +// +// - Pointer set to 'DonatedString' tag (LSB is 1): points to a std::string +// instance with a buffer on the arena (arena is never nullptr in this case). +// +// For fields with a non-empty string default value, there are three distinct +// states: +// +// - Pointer set to 'String' tag (LSB is 0), equal to `nullptr`: +// Field is in "default" mode and does not point to any actual instance. +// Methods that might need to create an instance of the object will pass a +// `const LazyString&` for it. +// +// - Pointer set to 'String' tag (LSB is 0), but not equal to `nullptr`: +// field points to a true std::string* instance that we own. This instance is +// either on the heap or on the arena (i.e. registered on +// free()/destructor-call list) as appropriate. +// +// - Pointer set to 'DonatedString' tag (LSB is 1): points to a std::string +// instance with a buffer on the arena (arena is never nullptr in this case). +// +// Generated code and reflection code both ensure that ptr_ is never null for +// fields with an empty default. +// Because ArenaStringPtr is used in oneof unions, its constructor is a NOP and +// so the field is always manually initialized via method calls. +// +// Side-note: why pass information about the default on every API call? Because +// we don't want to hold it in a member variable, or else this would go into +// every proto message instance. This would be a huge waste of space, since the +// default instance pointer is typically a global (static class field). We want +// the generated code to be as efficient as possible, and if we take +// the default value information as a parameter that's in practice taken from a +// static class field, and compare ptr_ to the default value, we end up with a +// single "cmp %reg, GLOBAL" in the resulting machine code. (Note that this also +// requires the String tag to be 0 so we can avoid the mask before comparing.) +struct PROTOBUF_EXPORT ArenaStringPtr { + ArenaStringPtr() = default; + explicit constexpr ArenaStringPtr( + const ExplicitlyConstructed* default_value) + : tagged_ptr_(default_value) {} + + // Some methods below are overloaded on a `default_value` and on tags. + // The tagged overloads help reduce code size in the callers in generated + // code, while the `default_value` overloads are useful from reflection. + // By-value empty struct arguments are elided in the ABI. + struct EmptyDefault {}; + struct NonEmptyDefault {}; + + void Set(const std::string* default_value, ConstStringParam value, + ::google::protobuf::Arena* arena); + void Set(const std::string* default_value, std::string&& value, + ::google::protobuf::Arena* arena); + void Set(EmptyDefault, ConstStringParam value, ::google::protobuf::Arena* arena); + void Set(EmptyDefault, std::string&& value, ::google::protobuf::Arena* arena); + void Set(NonEmptyDefault, ConstStringParam value, ::google::protobuf::Arena* arena); + void Set(NonEmptyDefault, std::string&& value, ::google::protobuf::Arena* arena); + template + void Set(FirstParam p1, const char* str, ::google::protobuf::Arena* arena) { + Set(p1, ConstStringParam(str), arena); + } + template + void Set(FirstParam p1, const char* str, size_t size, + ::google::protobuf::Arena* arena) { + ConstStringParam sp{str, size}; // for string_view and `const string &` + Set(p1, sp, arena); + } + template + void Set(FirstParam p1, + std::reference_wrapper const_string_ref, + ::google::protobuf::Arena* arena) { + Set(p1, const_string_ref.get(), arena); } - // Takes a string that is heap-allocated, and takes ownership. The string's - // destructor is registered with the arena. Used to implement - // set_allocated_ in generated classes. - inline void SetAllocated(const ::std::string* default_value, - ::std::string* value, ::google::protobuf::Arena* arena) { - if (arena == NULL && ptr_ != default_value) { - Destroy(default_value, arena); - } - if (value != NULL) { - ptr_ = value; - if (arena != NULL) { - arena->Own(value); - } - } else { - ptr_ = const_cast< ::std::string* >(default_value); - } + template + void SetBytes(FirstParam p1, SecondParam&& p2, ::google::protobuf::Arena* arena) { + Set(p1, static_cast(p2), arena); + } + template + void SetBytes(FirstParam p1, const void* str, size_t size, + ::google::protobuf::Arena* arena) { + // must work whether ConstStringParam is string_view or `const string &` + ConstStringParam sp{static_cast(str), size}; + Set(p1, sp, arena); } - // Takes a string that has lifetime equal to the arena's lifetime. The arena - // must be non-null. It is safe only to pass this method a value returned by - // UnsafeArenaRelease() on another field of a message in the same arena. Used - // to implement unsafe_arena_set_allocated_ in generated classes. - inline void UnsafeArenaSetAllocated(const ::std::string* default_value, - ::std::string* value, - ::google::protobuf::Arena* /* arena */) { - if (value != NULL) { - ptr_ = value; - } else { - ptr_ = const_cast< ::std::string* >(default_value); - } + // Basic accessors. + PROTOBUF_NDEBUG_INLINE const std::string& Get() const { + // Unconditionally mask away the tag. + return *tagged_ptr_.Get(); + } + PROTOBUF_NDEBUG_INLINE const std::string* GetPointer() const { + // Unconditionally mask away the tag. + return tagged_ptr_.Get(); } + // For fields with an empty default value. + std::string* Mutable(EmptyDefault, ::google::protobuf::Arena* arena); + // For fields with a non-empty default value. + std::string* Mutable(const LazyString& default_value, ::google::protobuf::Arena* arena); + + // Release returns a std::string* instance that is heap-allocated and is not + // Own()'d by any arena. If the field is not set, this returns nullptr. The + // caller retains ownership. Clears this field back to nullptr state. Used to + // implement release_() methods on generated classes. + PROTOBUF_NODISCARD std::string* Release(const std::string* default_value, + ::google::protobuf::Arena* arena); + PROTOBUF_NODISCARD std::string* ReleaseNonDefault( + const std::string* default_value, ::google::protobuf::Arena* arena); + + // Takes a std::string that is heap-allocated, and takes ownership. The + // std::string's destructor is registered with the arena. Used to implement + // set_allocated_ in generated classes. + void SetAllocated(const std::string* default_value, std::string* value, + ::google::protobuf::Arena* arena); + // Swaps internal pointers. Arena-safety semantics: this is guarded by the // logic in Swap()/UnsafeArenaSwap() at the message level, so this method is // 'unsafe' if called directly. - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE void Swap(ArenaStringPtr* other) { - std::swap(ptr_, other->ptr_); - } + inline PROTOBUF_NDEBUG_INLINE static void InternalSwap( + const std::string* default_value, ArenaStringPtr* rhs, Arena* rhs_arena, + ArenaStringPtr* lhs, Arena* lhs_arena); // Frees storage (if not on an arena). - inline void Destroy(const ::std::string* default_value, - ::google::protobuf::Arena* arena) { - if (arena == NULL && ptr_ != default_value) { - delete ptr_; - } - } + void Destroy(const std::string* default_value, ::google::protobuf::Arena* arena); + void Destroy(EmptyDefault, ::google::protobuf::Arena* arena); + void Destroy(NonEmptyDefault, ::google::protobuf::Arena* arena); - // Clears content, but keeps allocated string if arena != NULL, to avoid the - // overhead of heap operations. After this returns, the content (as seen by - // the user) will always be the empty string. Assumes that |default_value| - // is an empty string. - inline void ClearToEmpty(const ::std::string* default_value, - ::google::protobuf::Arena* /* arena */) { - if (ptr_ == default_value) { - // Already set to default (which is empty) -- do nothing. - } else { - ptr_->clear(); - } - } + // Clears content, but keeps allocated std::string, to avoid the overhead of + // heap operations. After this returns, the content (as seen by the user) will + // always be the empty std::string. Assumes that |default_value| is an empty + // std::string. + void ClearToEmpty(); - // Clears content, but keeps allocated string if arena != NULL, to avoid the - // overhead of heap operations. After this returns, the content (as seen by - // the user) will always be equal to |default_value|. - inline void ClearToDefault(const ::std::string* default_value, - ::google::protobuf::Arena* /* arena */) { - if (ptr_ == default_value) { - // Already set to default -- do nothing. - } else { - // Have another allocated string -- rather than throwing this away and - // resetting ptr_ to the canonical default string instance, we just reuse - // this instance. - *ptr_ = *default_value; - } - } + // Clears content, assuming that the current value is not the empty + // string default. + void ClearNonDefaultToEmpty(); - // Called from generated code / reflection runtime only. Resets value to point - // to a default string pointer, with the semantics that this ArenaStringPtr - // does not own the pointed-to memory. Disregards initial value of ptr_ (so - // this is the *ONLY* safe method to call after construction or when - // reinitializing after becoming the active field in a oneof union). - inline void UnsafeSetDefault(const ::std::string* default_value) { - // Casting away 'const' is safe here: accessors ensure that ptr_ is only - // returned as a const if it is equal to default_value. - ptr_ = const_cast< ::std::string* >(default_value); - } + // Clears content, but keeps allocated std::string if arena != nullptr, to + // avoid the overhead of heap operations. After this returns, the content + // (as seen by the user) will always be equal to |default_value|. + void ClearToDefault(const LazyString& default_value, ::google::protobuf::Arena* arena); - // The 'NoArena' variants of methods below assume arena == NULL and are - // optimized to provide very little overhead relative to a raw string pointer - // (while still being in-memory compatible with other code that assumes - // ArenaStringPtr). Note the invariant that a class instance that has only - // ever been mutated by NoArena methods must *only* be in the String state - // (i.e., tag bits are not used), *NEVER* ArenaString. This allows all - // tagged-pointer manipulations to be avoided. - inline void SetNoArena(const ::std::string* default_value, - const ::std::string& value) { - if (ptr_ == default_value) { - CreateInstanceNoArena(&value); - } else { - *ptr_ = value; - } + // Called from generated code / reflection runtime only. Resets value to point + // to a default string pointer, with the semantics that this + // ArenaStringPtr does not own the pointed-to memory. Disregards initial value + // of ptr_ (so this is the *ONLY* safe method to call after construction or + // when reinitializing after becoming the active field in a oneof union). + inline void UnsafeSetDefault(const std::string* default_value); + + // Returns a mutable pointer, but doesn't initialize the string to the + // default value. + std::string* MutableNoArenaNoDefault(const std::string* default_value); + + // Get a mutable pointer with unspecified contents. + // Similar to `MutableNoArenaNoDefault`, but also handles the arena case. + // If the value was donated, the contents are discarded. + std::string* MutableNoCopy(const std::string* default_value, + ::google::protobuf::Arena* arena); + + // Destroy the string. Assumes `arena == nullptr`. + void DestroyNoArena(const std::string* default_value); + + // Internal setter used only at parse time to directly set a donated string + // value. + void UnsafeSetTaggedPointer(TaggedPtr value) { + tagged_ptr_ = value; } - -#if LANG_CXX11 - void SetNoArena(const ::std::string* default_value, ::std::string&& value) { - if (IsDefault(default_value)) { - ptr_ = new ::std::string(std::move(value)); - } else { - *ptr_ = std::move(value); - } + // Generated code only! An optimization, in certain cases the generated + // code is certain we can obtain a std::string with no default checks and + // tag tests. + std::string* UnsafeMutablePointer() PROTOBUF_RETURNS_NONNULL; + + inline bool IsDefault(const std::string* default_value) const { + // Relies on the fact that kPtrTagString == 0, so if IsString(), ptr_ is the + // actual std::string pointer (and if !IsString(), ptr_ will never be equal + // to any aligned |default_value| pointer). The key is that we want to avoid + // masking in the fastpath const-pointer Get() case for non-arena code. + return tagged_ptr_.UnsafeGet() == default_value; } -#endif - - void AssignWithDefault(const ::std::string* default_value, ArenaStringPtr value); - inline const ::std::string& GetNoArena() const { return *ptr_; } + private: + TaggedPtr tagged_ptr_; - inline ::std::string* MutableNoArena(const ::std::string* default_value) { - if (ptr_ == default_value) { - CreateInstanceNoArena(default_value); - } - return ptr_; - } + bool IsDonatedString() const { return false; } - inline ::std::string* ReleaseNoArena(const ::std::string* default_value) { - if (ptr_ == default_value) { - return NULL; - } else { - ::std::string* released = ptr_; - ptr_ = const_cast< ::std::string* >(default_value); - return released; - } + // Swaps tagged pointer without debug hardening. This is to allow python + // protobuf to maintain pointer stability even in DEBUG builds. + inline PROTOBUF_NDEBUG_INLINE static void UnsafeShallowSwap( + ArenaStringPtr* rhs, ArenaStringPtr* lhs) { + std::swap(lhs->tagged_ptr_, rhs->tagged_ptr_); } - inline void SetAllocatedNoArena(const ::std::string* default_value, - ::std::string* value) { - if (ptr_ != default_value) { - delete ptr_; - } - if (value != NULL) { - ptr_ = value; - } else { - ptr_ = const_cast< ::std::string* >(default_value); - } - } + friend class ::google::protobuf::internal::SwapFieldHelper; - inline void DestroyNoArena(const ::std::string* default_value) { - if (ptr_ != default_value) { - delete ptr_; - } - } + // Slow paths. - inline void ClearToEmptyNoArena(const ::std::string* default_value) { - if (ptr_ == default_value) { - // Nothing: already equal to default (which is the empty string). - } else { - ptr_->clear(); - } - } + // MutableSlow requires that !IsString() || IsDefault + // Variadic to support 0 args for EmptyDefault and 1 arg for LazyString. + template + std::string* MutableSlow(::google::protobuf::Arena* arena, const Lazy&... lazy_default); - inline void ClearToDefaultNoArena(const ::std::string* default_value) { - if (ptr_ == default_value) { - // Nothing: already set to default. - } else { - // Reuse existing allocated instance. - *ptr_ = *default_value; - } - } - - // Internal accessor used only at parse time to provide direct access to the - // raw pointer from the shared parse routine (in the non-arenas case). The - // parse routine does the string allocation in order to save code size in the - // generated parsing code. - inline ::std::string** UnsafeRawStringPointer() { - return &ptr_; - } + // Sets value to a newly allocated string and returns it + std::string* SetAndReturnNewString(); - inline bool IsDefault(const ::std::string* default_value) const { - return ptr_ == default_value; - } + // Destroys the non-default string value out-of-line + void DestroyNoArenaSlowPath(); - private: - ::std::string* ptr_; - - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE - void CreateInstance(::google::protobuf::Arena* arena, - const ::std::string* initial_value) { - GOOGLE_DCHECK(initial_value != NULL); - ptr_ = new ::std::string(*initial_value); - if (arena != NULL) { - arena->Own(ptr_); - } - } - GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE - void CreateInstanceNoArena(const ::std::string* initial_value) { - GOOGLE_DCHECK(initial_value != NULL); - ptr_ = new ::std::string(*initial_value); - } }; -} // namespace internal -} // namespace protobuf +inline void ArenaStringPtr::UnsafeSetDefault(const std::string* value) { + tagged_ptr_.Set(const_cast(value)); +} +// Make sure rhs_arena allocated rhs, and lhs_arena allocated lhs. +inline PROTOBUF_NDEBUG_INLINE void ArenaStringPtr::InternalSwap( // + const std::string* default_value, // + ArenaStringPtr* rhs, Arena* rhs_arena, // + ArenaStringPtr* lhs, Arena* lhs_arena) { + // Silence unused variable warnings in release buildls. + (void)default_value; + (void)rhs_arena; + (void)lhs_arena; + std::swap(lhs->tagged_ptr_, rhs->tagged_ptr_); +#ifdef PROTOBUF_FORCE_COPY_IN_SWAP + auto force_realloc = [default_value](ArenaStringPtr* p, Arena* arena) { + if (p->IsDefault(default_value)) return; + std::string* old_value = p->tagged_ptr_.Get(); + std::string* new_value = + p->IsDonatedString() + ? Arena::Create(arena, *old_value) + : Arena::Create(arena, std::move(*old_value)); + if (arena == nullptr) delete old_value; + p->tagged_ptr_.Set(new_value); + }; + // Because, at this point, tagged_ptr_ has been swapped, arena should also be + // swapped. + force_realloc(lhs, rhs_arena); + force_realloc(rhs, lhs_arena); +#endif // PROTOBUF_FORCE_COPY_IN_SWAP +} +inline void ArenaStringPtr::ClearNonDefaultToEmpty() { + // Unconditionally mask away the tag. + tagged_ptr_.Get()->clear(); +} -namespace protobuf { -namespace internal { +inline std::string* ArenaStringPtr::MutableNoArenaNoDefault( + const std::string* default_value) { + // VERY IMPORTANT for performance and code size: this will reduce to a member + // variable load, a pointer check (against |default_value|, in practice a + // static global) and a branch to the slowpath (which calls operator new and + // the ctor). DO NOT add any tagged-pointer operations here. + if (IsDefault(default_value)) { + return SetAndReturnNewString(); + } else { + return UnsafeMutablePointer(); + } +} -inline void ArenaStringPtr::AssignWithDefault(const ::std::string* default_value, - ArenaStringPtr value) { - const ::std::string* me = *UnsafeRawStringPointer(); - const ::std::string* other = *value.UnsafeRawStringPointer(); - // If the pointers are the same then do nothing. - if (me != other) { - SetNoArena(default_value, value.GetNoArena()); +inline void ArenaStringPtr::DestroyNoArena(const std::string* default_value) { + if (!IsDefault(default_value)) { + DestroyNoArenaSlowPath(); } } +inline std::string* ArenaStringPtr::UnsafeMutablePointer() { + GOOGLE_DCHECK(!tagged_ptr_.IsTagged()); + GOOGLE_DCHECK(tagged_ptr_.UnsafeGet() != nullptr); + return tagged_ptr_.UnsafeGet(); +} + + } // namespace internal } // namespace protobuf - } // namespace google + +#include + #endif // GOOGLE_PROTOBUF_ARENASTRING_H__ diff --git a/3rdparty/protobuf/src/google/protobuf/descriptor.cc b/3rdparty/protobuf/src/google/protobuf/descriptor.cc index 3f54b84841d3..c8ce218a981d 100644 --- a/3rdparty/protobuf/src/google/protobuf/descriptor.cc +++ b/3rdparty/protobuf/src/google/protobuf/descriptor.cc @@ -32,48 +32,53 @@ // Based on original Protocol Buffers design by // Sanjay Ghemawat, Jeff Dean, and others. -#include +#include + +#include +#include +#include +#include #include #include -#ifndef _SHARED_PTR_H -#include -#endif #include #include +#include +#include #include -#include -#include #include #include -#include -#include #include #include -#include +#include +#include +#include #include #include #include -#include -#include #include #include #include #include #include #include +#include #include - +#include #include #include +#include #undef PACKAGE // autoheader #defines this. :( -namespace google { +#include + +namespace google { namespace protobuf { -struct Symbol { +class Symbol { + public: enum Type { NULL_SYMBOL, MESSAGE, @@ -81,139 +86,264 @@ struct Symbol { ONEOF, ENUM, ENUM_VALUE, + ENUM_VALUE_OTHER_PARENT, SERVICE, METHOD, - PACKAGE + PACKAGE, + QUERY_KEY }; - Type type; - union { - const Descriptor* descriptor; - const FieldDescriptor* field_descriptor; - const OneofDescriptor* oneof_descriptor; - const EnumDescriptor* enum_descriptor; - const EnumValueDescriptor* enum_value_descriptor; - const ServiceDescriptor* service_descriptor; - const MethodDescriptor* method_descriptor; - const FileDescriptor* package_file_descriptor; + + Symbol() : ptr_(nullptr) {} + + // Every object we store derives from internal::SymbolBase, where we store the + // symbol type enum. + // Storing in the object can be done without using more space in most cases, + // while storing it in the Symbol type would require 8 bytes. +#define DEFINE_MEMBERS(TYPE, TYPE_CONSTANT, FIELD) \ + explicit Symbol(TYPE* value) : ptr_(value) { \ + value->symbol_type_ = TYPE_CONSTANT; \ + } \ + const TYPE* FIELD() const { \ + return type() == TYPE_CONSTANT ? static_cast(ptr_) : nullptr; \ + } + + DEFINE_MEMBERS(Descriptor, MESSAGE, descriptor) + DEFINE_MEMBERS(FieldDescriptor, FIELD, field_descriptor) + DEFINE_MEMBERS(OneofDescriptor, ONEOF, oneof_descriptor) + DEFINE_MEMBERS(EnumDescriptor, ENUM, enum_descriptor) + DEFINE_MEMBERS(ServiceDescriptor, SERVICE, service_descriptor) + DEFINE_MEMBERS(MethodDescriptor, METHOD, method_descriptor) + + // We use a special node for FileDescriptor. + // It is potentially added to the table with multiple different names, so we + // need a separate place to put the name. + struct Package : internal::SymbolBase { + const std::string* name; + const FileDescriptor* file; + }; + DEFINE_MEMBERS(Package, PACKAGE, package_file_descriptor) + + // Enum values have two different parents. + // We use two different identitied for the same object to determine the two + // different insertions in the map. + static Symbol EnumValue(EnumValueDescriptor* value, int n) { + Symbol s; + internal::SymbolBase* ptr; + if (n == 0) { + ptr = static_cast*>(value); + ptr->symbol_type_ = ENUM_VALUE; + } else { + ptr = static_cast*>(value); + ptr->symbol_type_ = ENUM_VALUE_OTHER_PARENT; + } + s.ptr_ = ptr; + return s; + } + + const EnumValueDescriptor* enum_value_descriptor() const { + return type() == ENUM_VALUE + ? static_cast( + static_cast*>(ptr_)) + : type() == ENUM_VALUE_OTHER_PARENT + ? static_cast( + static_cast*>(ptr_)) + : nullptr; + } + + // Not a real symbol. + // Only used for heterogeneous lookups and never actually inserted in the + // tables. + struct QueryKey : internal::SymbolBase { + StringPiece name; + const void* parent; + int field_number; }; + DEFINE_MEMBERS(QueryKey, QUERY_KEY, query_key); +#undef DEFINE_MEMBERS - inline Symbol() : type(NULL_SYMBOL) { descriptor = NULL; } - inline bool IsNull() const { return type == NULL_SYMBOL; } - inline bool IsType() const { return type == MESSAGE || type == ENUM; } - inline bool IsAggregate() const { - return type == MESSAGE || type == PACKAGE || type == ENUM || - type == SERVICE; + Type type() const { + return ptr_ == nullptr ? NULL_SYMBOL + : static_cast(ptr_->symbol_type_); } - -#define CONSTRUCTOR(TYPE, TYPE_CONSTANT, FIELD) \ - inline explicit Symbol(const TYPE* value) { \ - type = TYPE_CONSTANT; \ - this->FIELD = value; \ + bool IsNull() const { return type() == NULL_SYMBOL; } + bool IsType() const { return type() == MESSAGE || type() == ENUM; } + bool IsAggregate() const { + return type() == MESSAGE || type() == PACKAGE || type() == ENUM || + type() == SERVICE; } - CONSTRUCTOR(Descriptor, MESSAGE, descriptor) - CONSTRUCTOR(FieldDescriptor, FIELD, field_descriptor) - CONSTRUCTOR(OneofDescriptor, ONEOF, oneof_descriptor) - CONSTRUCTOR(EnumDescriptor, ENUM, enum_descriptor) - CONSTRUCTOR(EnumValueDescriptor, ENUM_VALUE, enum_value_descriptor) - CONSTRUCTOR(ServiceDescriptor, SERVICE, service_descriptor) - CONSTRUCTOR(MethodDescriptor, METHOD, method_descriptor) - CONSTRUCTOR(FileDescriptor, PACKAGE, package_file_descriptor) -#undef CONSTRUCTOR - const FileDescriptor* GetFile() const { - switch (type) { - case NULL_SYMBOL: - return NULL; + switch (type()) { case MESSAGE: - return descriptor->file(); + return descriptor()->file(); case FIELD: - return field_descriptor->file(); + return field_descriptor()->file(); case ONEOF: - return oneof_descriptor->containing_type()->file(); + return oneof_descriptor()->containing_type()->file(); case ENUM: - return enum_descriptor->file(); + return enum_descriptor()->file(); case ENUM_VALUE: - return enum_value_descriptor->type()->file(); + return enum_value_descriptor()->type()->file(); case SERVICE: - return service_descriptor->file(); + return service_descriptor()->file(); case METHOD: - return method_descriptor->service()->file(); + return method_descriptor()->service()->file(); case PACKAGE: - return package_file_descriptor; + return package_file_descriptor()->file; + default: + return nullptr; + } + } + + StringPiece full_name() const { + switch (type()) { + case MESSAGE: + return descriptor()->full_name(); + case FIELD: + return field_descriptor()->full_name(); + case ONEOF: + return oneof_descriptor()->full_name(); + case ENUM: + return enum_descriptor()->full_name(); + case ENUM_VALUE: + return enum_value_descriptor()->full_name(); + case SERVICE: + return service_descriptor()->full_name(); + case METHOD: + return method_descriptor()->full_name(); + case PACKAGE: + return *package_file_descriptor()->name; + case QUERY_KEY: + return query_key()->name; + default: + GOOGLE_CHECK(false); + } + return ""; + } + + std::pair parent_name_key() const { + const auto or_file = [&](const void* p) { return p ? p : GetFile(); }; + switch (type()) { + case MESSAGE: + return {or_file(descriptor()->containing_type()), descriptor()->name()}; + case FIELD: { + auto* field = field_descriptor(); + return {or_file(field->is_extension() ? field->extension_scope() + : field->containing_type()), + field->name()}; + } + case ONEOF: + return {oneof_descriptor()->containing_type(), + oneof_descriptor()->name()}; + case ENUM: + return {or_file(enum_descriptor()->containing_type()), + enum_descriptor()->name()}; + case ENUM_VALUE: + return {or_file(enum_value_descriptor()->type()->containing_type()), + enum_value_descriptor()->name()}; + case ENUM_VALUE_OTHER_PARENT: + return {enum_value_descriptor()->type(), + enum_value_descriptor()->name()}; + case SERVICE: + return {GetFile(), service_descriptor()->name()}; + case METHOD: + return {method_descriptor()->service(), method_descriptor()->name()}; + case QUERY_KEY: + return {query_key()->parent, query_key()->name}; + default: + GOOGLE_CHECK(false); + } + return {}; + } + + std::pair parent_number_key() const { + switch (type()) { + case FIELD: + return {field_descriptor()->containing_type(), + field_descriptor()->number()}; + case ENUM_VALUE: + return {enum_value_descriptor()->type(), + enum_value_descriptor()->number()}; + case QUERY_KEY: + return {query_key()->parent, query_key()->field_number}; + default: + GOOGLE_CHECK(false); } - return NULL; + return {}; } + + private: + const internal::SymbolBase* ptr_; }; const FieldDescriptor::CppType -FieldDescriptor::kTypeToCppTypeMap[MAX_TYPE + 1] = { - static_cast(0), // 0 is reserved for errors - - CPPTYPE_DOUBLE, // TYPE_DOUBLE - CPPTYPE_FLOAT, // TYPE_FLOAT - CPPTYPE_INT64, // TYPE_INT64 - CPPTYPE_UINT64, // TYPE_UINT64 - CPPTYPE_INT32, // TYPE_INT32 - CPPTYPE_UINT64, // TYPE_FIXED64 - CPPTYPE_UINT32, // TYPE_FIXED32 - CPPTYPE_BOOL, // TYPE_BOOL - CPPTYPE_STRING, // TYPE_STRING - CPPTYPE_MESSAGE, // TYPE_GROUP - CPPTYPE_MESSAGE, // TYPE_MESSAGE - CPPTYPE_STRING, // TYPE_BYTES - CPPTYPE_UINT32, // TYPE_UINT32 - CPPTYPE_ENUM, // TYPE_ENUM - CPPTYPE_INT32, // TYPE_SFIXED32 - CPPTYPE_INT64, // TYPE_SFIXED64 - CPPTYPE_INT32, // TYPE_SINT32 - CPPTYPE_INT64, // TYPE_SINT64 + FieldDescriptor::kTypeToCppTypeMap[MAX_TYPE + 1] = { + static_cast(0), // 0 is reserved for errors + + CPPTYPE_DOUBLE, // TYPE_DOUBLE + CPPTYPE_FLOAT, // TYPE_FLOAT + CPPTYPE_INT64, // TYPE_INT64 + CPPTYPE_UINT64, // TYPE_UINT64 + CPPTYPE_INT32, // TYPE_INT32 + CPPTYPE_UINT64, // TYPE_FIXED64 + CPPTYPE_UINT32, // TYPE_FIXED32 + CPPTYPE_BOOL, // TYPE_BOOL + CPPTYPE_STRING, // TYPE_STRING + CPPTYPE_MESSAGE, // TYPE_GROUP + CPPTYPE_MESSAGE, // TYPE_MESSAGE + CPPTYPE_STRING, // TYPE_BYTES + CPPTYPE_UINT32, // TYPE_UINT32 + CPPTYPE_ENUM, // TYPE_ENUM + CPPTYPE_INT32, // TYPE_SFIXED32 + CPPTYPE_INT64, // TYPE_SFIXED64 + CPPTYPE_INT32, // TYPE_SINT32 + CPPTYPE_INT64, // TYPE_SINT64 }; -const char * const FieldDescriptor::kTypeToName[MAX_TYPE + 1] = { - "ERROR", // 0 is reserved for errors - - "double", // TYPE_DOUBLE - "float", // TYPE_FLOAT - "int64", // TYPE_INT64 - "uint64", // TYPE_UINT64 - "int32", // TYPE_INT32 - "fixed64", // TYPE_FIXED64 - "fixed32", // TYPE_FIXED32 - "bool", // TYPE_BOOL - "string", // TYPE_STRING - "group", // TYPE_GROUP - "message", // TYPE_MESSAGE - "bytes", // TYPE_BYTES - "uint32", // TYPE_UINT32 - "enum", // TYPE_ENUM - "sfixed32", // TYPE_SFIXED32 - "sfixed64", // TYPE_SFIXED64 - "sint32", // TYPE_SINT32 - "sint64", // TYPE_SINT64 +const char* const FieldDescriptor::kTypeToName[MAX_TYPE + 1] = { + "ERROR", // 0 is reserved for errors + + "double", // TYPE_DOUBLE + "float", // TYPE_FLOAT + "int64", // TYPE_INT64 + "uint64", // TYPE_UINT64 + "int32", // TYPE_INT32 + "fixed64", // TYPE_FIXED64 + "fixed32", // TYPE_FIXED32 + "bool", // TYPE_BOOL + "string", // TYPE_STRING + "group", // TYPE_GROUP + "message", // TYPE_MESSAGE + "bytes", // TYPE_BYTES + "uint32", // TYPE_UINT32 + "enum", // TYPE_ENUM + "sfixed32", // TYPE_SFIXED32 + "sfixed64", // TYPE_SFIXED64 + "sint32", // TYPE_SINT32 + "sint64", // TYPE_SINT64 }; -const char * const FieldDescriptor::kCppTypeToName[MAX_CPPTYPE + 1] = { - "ERROR", // 0 is reserved for errors - - "int32", // CPPTYPE_INT32 - "int64", // CPPTYPE_INT64 - "uint32", // CPPTYPE_UINT32 - "uint64", // CPPTYPE_UINT64 - "double", // CPPTYPE_DOUBLE - "float", // CPPTYPE_FLOAT - "bool", // CPPTYPE_BOOL - "enum", // CPPTYPE_ENUM - "string", // CPPTYPE_STRING - "message", // CPPTYPE_MESSAGE +const char* const FieldDescriptor::kCppTypeToName[MAX_CPPTYPE + 1] = { + "ERROR", // 0 is reserved for errors + + "int32", // CPPTYPE_INT32 + "int64", // CPPTYPE_INT64 + "uint32", // CPPTYPE_UINT32 + "uint64", // CPPTYPE_UINT64 + "double", // CPPTYPE_DOUBLE + "float", // CPPTYPE_FLOAT + "bool", // CPPTYPE_BOOL + "enum", // CPPTYPE_ENUM + "string", // CPPTYPE_STRING + "message", // CPPTYPE_MESSAGE }; -const char * const FieldDescriptor::kLabelToName[MAX_LABEL + 1] = { - "ERROR", // 0 is reserved for errors +const char* const FieldDescriptor::kLabelToName[MAX_LABEL + 1] = { + "ERROR", // 0 is reserved for errors - "optional", // LABEL_OPTIONAL - "required", // LABEL_REQUIRED - "repeated", // LABEL_REPEATED + "optional", // LABEL_OPTIONAL + "required", // LABEL_REQUIRED + "repeated", // LABEL_REPEATED }; const char* FileDescriptor::SyntaxName(FileDescriptor::Syntax syntax) { @@ -226,12 +356,12 @@ const char* FileDescriptor::SyntaxName(FileDescriptor::Syntax syntax) { return "unknown"; } GOOGLE_LOG(FATAL) << "can't reach here."; - return NULL; + return nullptr; } -static const char * const kNonLinkedWeakMessageReplacementName = "google.protobuf.Empty"; +static const char* const kNonLinkedWeakMessageReplacementName = "google.protobuf.Empty"; -#if !defined(_MSC_VER) || _MSC_VER >= 1900 +#if !defined(_MSC_VER) || (_MSC_VER >= 1900 && _MSC_VER < 1912) const int FieldDescriptor::kMaxNumber; const int FieldDescriptor::kFirstReservedNumber; const int FieldDescriptor::kLastReservedNumber; @@ -248,19 +378,19 @@ char ToLower(char ch) { return (ch >= 'A' && ch <= 'Z') ? (ch - 'A' + 'a') : ch; } -string ToCamelCase(const string& input, bool lower_first) { +std::string ToCamelCase(const std::string& input, bool lower_first) { bool capitalize_next = !lower_first; - string result; + std::string result; result.reserve(input.size()); - for (int i = 0; i < input.size(); i++) { - if (input[i] == '_') { + for (char character : input) { + if (character == '_') { capitalize_next = true; } else if (capitalize_next) { - result.push_back(ToUpper(input[i])); + result.push_back(ToUpper(character)); capitalize_next = false; } else { - result.push_back(input[i]); + result.push_back(character); } } @@ -272,38 +402,38 @@ string ToCamelCase(const string& input, bool lower_first) { return result; } -string ToJsonName(const string& input) { +std::string ToJsonName(const std::string& input) { bool capitalize_next = false; - string result; + std::string result; result.reserve(input.size()); - for (int i = 0; i < input.size(); i++) { - if (input[i] == '_') { + for (char character : input) { + if (character == '_') { capitalize_next = true; } else if (capitalize_next) { - result.push_back(ToUpper(input[i])); + result.push_back(ToUpper(character)); capitalize_next = false; } else { - result.push_back(input[i]); + result.push_back(character); } } return result; } -string EnumValueToPascalCase(const string& input) { +std::string EnumValueToPascalCase(const std::string& input) { bool next_upper = true; - string result; + std::string result; result.reserve(input.size()); - for (int i = 0; i < input.size(); i++) { - if (input[i] == '_') { + for (char character : input) { + if (character == '_') { next_upper = true; } else { if (next_upper) { - result.push_back(ToUpper(input[i])); + result.push_back(ToUpper(character)); } else { - result.push_back(ToLower(input[i])); + result.push_back(ToLower(character)); } next_upper = false; } @@ -317,16 +447,16 @@ class PrefixRemover { public: PrefixRemover(StringPiece prefix) { // Strip underscores and lower-case the prefix. - for (int i = 0; i < prefix.size(); i++) { - if (prefix[i] != '_') { - prefix_ += ascii_tolower(prefix[i]); + for (char character : prefix) { + if (character != '_') { + prefix_ += ascii_tolower(character); } } } // Tries to remove the enum prefix from this enum value. // If this is not possible, returns the input verbatim. - string MaybeRemove(StringPiece str) { + std::string MaybeRemove(StringPiece str) { // We can't just lowercase and strip str and look for a prefix. // We need to properly recognize the difference between: // @@ -346,14 +476,14 @@ class PrefixRemover { } if (ascii_tolower(str[i]) != prefix_[j++]) { - return str.as_string(); + return std::string(str); } } // If we didn't make it through the prefix, we've failed to strip the // prefix. if (j < prefix_.size()) { - return str.as_string(); + return std::string(str); } // Skip underscores between prefix and further characters. @@ -363,47 +493,41 @@ class PrefixRemover { // Enum label can't be the empty string. if (i == str.size()) { - return str.as_string(); + return std::string(str); } // We successfully stripped the prefix. str.remove_prefix(i); - return str.as_string(); + return std::string(str); } private: - string prefix_; + std::string prefix_; }; -// A DescriptorPool contains a bunch of hash_maps to implement the +// A DescriptorPool contains a bunch of hash-maps to implement the // various Find*By*() methods. Since hashtable lookups are O(1), it's -// most efficient to construct a fixed set of large hash_maps used by +// most efficient to construct a fixed set of large hash-maps used by // all objects in the pool rather than construct one or more small -// hash_maps for each object. -// -// The keys to these hash_maps are (parent, name) or (parent, number) -// pairs. Unfortunately STL doesn't provide hash functions for pair<>, -// so we must invent our own. +// hash-maps for each object. // -// TODO(kenton): Use StringPiece rather than const char* in keys? It would -// be a lot cleaner but we'd just have to convert it back to const char* -// for the open source release. +// The keys to these hash-maps are (parent, name) or (parent, number) pairs. -typedef std::pair PointerStringPair; +typedef std::pair PointerStringPair; -struct PointerStringPairEqual { - inline bool operator()(const PointerStringPair& a, - const PointerStringPair& b) const { - return a.first == b.first && strcmp(a.second, b.second) == 0; - } -}; +typedef std::pair DescriptorIntPair; -template +#define HASH_MAP std::unordered_map +#define HASH_SET std::unordered_set +#define HASH_FXN hash + +template struct PointerIntegerPairHash { size_t operator()(const PairType& p) const { - // FIXME(kenton): What is the best way to compute this hash? I have - // no idea! This seems a bit better than an XOR. - return reinterpret_cast(p.first) * ((1 << 16) - 1) + p.second; + static const size_t prime1 = 16777499; + static const size_t prime2 = 16777619; + return reinterpret_cast(p.first) * prime1 ^ + static_cast(p.second) * prime2; } #ifdef _MSC_VER @@ -412,21 +536,16 @@ struct PointerIntegerPairHash { static const size_t min_buckets = 8; #endif inline bool operator()(const PairType& a, const PairType& b) const { - return a.first < b.first || - (a.first == b.first && a.second < b.second); + return a < b; } }; -typedef std::pair DescriptorIntPair; -typedef std::pair EnumIntPair; - struct PointerStringPairHash { size_t operator()(const PointerStringPair& p) const { - // FIXME(kenton): What is the best way to compute this hash? I have - // no idea! This seems a bit better than an XOR. - hash cstring_hash; - return reinterpret_cast(p.first) * ((1 << 16) - 1) + - cstring_hash(p.second); + static const size_t prime = 16777619; + hash string_hash; + return reinterpret_cast(p.first) * prime ^ + static_cast(string_hash(p.second)); } #ifdef _MSC_VER @@ -436,77 +555,497 @@ struct PointerStringPairHash { #endif inline bool operator()(const PointerStringPair& a, const PointerStringPair& b) const { - if (a.first < b.first) return true; - if (a.first > b.first) return false; - return strcmp(a.second, b.second) < 0; + return a < b; } }; const Symbol kNullSymbol; -typedef hash_map, streq> - SymbolsByNameMap; -typedef hash_map - SymbolsByParentMap; -typedef hash_map, streq> - FilesByNameMap; -typedef hash_map - FieldsByNameMap; -typedef hash_map > - FieldsByNumberMap; -typedef hash_map > - EnumValuesByNumberMap; -// This is a map rather than a hash_map, since we use it to iterate +struct SymbolByFullNameHash { + size_t operator()(Symbol s) const { + return HASH_FXN{}(s.full_name()); + } +}; +struct SymbolByFullNameEq { + bool operator()(Symbol a, Symbol b) const { + return a.full_name() == b.full_name(); + } +}; +using SymbolsByNameSet = + HASH_SET; + +struct SymbolByParentHash { + size_t operator()(Symbol s) const { + return PointerStringPairHash{}(s.parent_name_key()); + } +}; +struct SymbolByParentEq { + bool operator()(Symbol a, Symbol b) const { + return a.parent_name_key() == b.parent_name_key(); + } +}; +using SymbolsByParentSet = + HASH_SET; + +typedef HASH_MAP> + FilesByNameMap; + +typedef HASH_MAP + FieldsByNameMap; + +struct FieldsByNumberHash { + size_t operator()(Symbol s) const { + return PointerIntegerPairHash>{}( + s.parent_number_key()); + } +}; +struct FieldsByNumberEq { + bool operator()(Symbol a, Symbol b) const { + return a.parent_number_key() == b.parent_number_key(); + } +}; +using FieldsByNumberSet = + HASH_SET; +using EnumValuesByNumberSet = FieldsByNumberSet; + +// This is a map rather than a hash-map, since we use it to iterate // through all the extensions that extend a given Descriptor, and an // ordered data structure that implements lower_bound is convenient // for that. typedef std::map - ExtensionsGroupedByDescriptorMap; -typedef hash_map LocationsByPathMap; - -std::set* allowed_proto3_extendees_ = NULL; -GOOGLE_PROTOBUF_DECLARE_ONCE(allowed_proto3_extendees_init_); - -void DeleteAllowedProto3Extendee() { - delete allowed_proto3_extendees_; -} + ExtensionsGroupedByDescriptorMap; +typedef HASH_MAP + LocationsByPathMap; -void InitAllowedProto3Extendee() { - allowed_proto3_extendees_ = new std::set; +std::set* NewAllowedProto3Extendee() { + auto allowed_proto3_extendees = new std::set; const char* kOptionNames[] = { - "FileOptions", "MessageOptions", "FieldOptions", "EnumOptions", + "FileOptions", "MessageOptions", "FieldOptions", "EnumOptions", "EnumValueOptions", "ServiceOptions", "MethodOptions", "OneofOptions"}; - for (int i = 0; i < GOOGLE_ARRAYSIZE(kOptionNames); ++i) { + for (const char* option_name : kOptionNames) { // descriptor.proto has a different package name in opensource. We allow // both so the opensource protocol compiler can also compile internal // proto3 files with custom options. See: b/27567912 - allowed_proto3_extendees_->insert(string("google.protobuf.") + - kOptionNames[i]); + allowed_proto3_extendees->insert(std::string("google.protobuf.") + + option_name); // Split the word to trick the opensource processing scripts so they - // will keep the origial package name. - allowed_proto3_extendees_->insert(string("proto") + "2." + kOptionNames[i]); + // will keep the original package name. + allowed_proto3_extendees->insert(std::string("proto") + "2." + option_name); } - - google::protobuf::internal::OnShutdown(&DeleteAllowedProto3Extendee); + return allowed_proto3_extendees; } // Checks whether the extendee type is allowed in proto3. // Only extensions to descriptor options are allowed. We use name comparison // instead of comparing the descriptor directly because the extensions may be // defined in a different pool. -bool AllowedExtendeeInProto3(const string& name) { - ::google::protobuf::GoogleOnceInit(&allowed_proto3_extendees_init_, &InitAllowedProto3Extendee); - return allowed_proto3_extendees_->find(name) != - allowed_proto3_extendees_->end(); +bool AllowedExtendeeInProto3(const std::string& name) { + static auto allowed_proto3_extendees = + internal::OnShutdownDelete(NewAllowedProto3Extendee()); + return allowed_proto3_extendees->find(name) != + allowed_proto3_extendees->end(); } +// This bump allocator arena is optimized for the use case of this file. It is +// mostly optimized for memory usage, since these objects are expected to live +// for the entirety of the program. +// +// Some differences from other arenas: +// - It has a fixed number of non-trivial types it can hold. This allows +// tracking the allocations with a single byte. In contrast, google::protobuf::Arena +// uses 16 bytes per non-trivial object created. +// - It has some extra metadata for rollbacks. This is necessary for +// implementing the API below. This metadata is flushed at the end and would +// not cause persistent memory usage. +// - It tries to squeeze every byte of out the blocks. If an allocation is too +// large for the current block we move the block to a secondary area where we +// can still use it for smaller objects. This complicates rollback logic but +// makes it much more memory efficient. +// +// The allocation strategy is as follows: +// - Memory is allocated from the front, with a forced 8 byte alignment. +// - Metadata is allocated from the back, one byte per element. +// - The metadata encodes one of two things: +// * For types we want to track, the index into KnownTypes. +// * For raw memory blocks, the size of the block (in 8 byte increments +// to allow for a larger limit). +// - When the raw data is too large to represent in the metadata byte, we +// allocate this memory separately in the heap and store an OutOfLineAlloc +// object instead. These come from large array allocations and alike. +// +// Blocks are kept in 3 areas: +// - `current_` is the one we are currently allocating from. When we need to +// allocate a block that doesn't fit there, we make a new block and move the +// old `current_` to one of the areas below. +// - Blocks that have no more usable space left (ie less than 9 bytes) are +// stored in `full_blocks_`. +// - Blocks that have some usable space are categorized in +// `small_size_blocks_` depending on how much space they have left. +// See `kSmallSizes` to see which sizes we track. +// +class TableArena { + public: + // Allocate a block on `n` bytes, with no destructor information saved. + void* AllocateMemory(uint32_t n) { + uint32_t tag = SizeToRawTag(n) + kFirstRawTag; + if (tag > 255) { + // We can't fit the size, use an OutOfLineAlloc. + return Create(OutOfLineAlloc{::operator new(n), n})->ptr; + } + + return AllocRawInternal(n, static_cast(tag)); + } + + // Allocate and construct an element of type `T` as if by + // `T(std::forward(args...))`. + // The object is registered for destruction, if its destructor is not trivial. + template + T* Create(Args&&... args) { + static_assert(alignof(T) <= 8, ""); + return ::new (AllocRawInternal(sizeof(T), TypeTag(KnownTypes{}))) + T(std::forward(args)...); + } + + TableArena() {} + + TableArena(const TableArena&) = delete; + TableArena& operator=(const TableArena&) = delete; + + ~TableArena() { + // Uncomment this to debug usage statistics of the arena blocks. + // PrintUsageInfo(); + + for (Block* list : GetLists()) { + while (list != nullptr) { + Block* b = list; + list = list->next; + b->VisitBlock(DestroyVisitor{}); + b->Destroy(); + } + } + } + + + // This function exists for debugging only. + // It can be called from the destructor to dump some info in the tests to + // inspect the usage of the arena. + void PrintUsageInfo() const { + const auto print_histogram = [](Block* b, int size) { + std::map unused_space_count; + int count = 0; + for (; b != nullptr; b = b->next) { + ++unused_space_count[b->space_left()]; + ++count; + } + if (size > 0) { + fprintf(stderr, " Blocks `At least %d`", size); + } else { + fprintf(stderr, " Blocks `full`"); + } + fprintf(stderr, ": %d blocks.\n", count); + for (auto p : unused_space_count) { + fprintf(stderr, " space=%4u, count=%3u\n", p.first, p.second); + } + }; + + fprintf(stderr, "TableArena unused space histogram:\n"); + fprintf(stderr, " Current: %u\n", + current_ != nullptr ? current_->space_left() : 0); + print_histogram(full_blocks_, 0); + for (size_t i = 0; i < kSmallSizes.size(); ++i) { + print_histogram(small_size_blocks_[i], kSmallSizes[i]); + } + } + + // Current allocation count. + // This can be used for checkpointing. + size_t num_allocations() const { return num_allocations_; } + + // Rollback the latest allocations until we reach back to `checkpoint` + // num_allocations. + void RollbackTo(size_t checkpoint) { + while (num_allocations_ > checkpoint) { + GOOGLE_DCHECK(!rollback_info_.empty()); + auto& info = rollback_info_.back(); + Block* b = info.block; + + VisitAlloc(b->data(), &b->start_offset, &b->end_offset, DestroyVisitor{}, + KnownTypes{}); + if (--info.count == 0) { + rollback_info_.pop_back(); + } + --num_allocations_; + } + + // Reconstruct the lists and destroy empty blocks. + auto lists = GetLists(); + current_ = full_blocks_ = nullptr; + small_size_blocks_.fill(nullptr); + + for (Block* list : lists) { + while (list != nullptr) { + Block* b = list; + list = list->next; + + if (b->start_offset == 0) { + // This is empty, free it. + b->Destroy(); + } else { + RelocateToUsedList(b); + } + } + } + } + + // Clear all rollback information. Reduces memory usage. + // Trying to rollback past num_allocations() is now impossible. + void ClearRollbackData() { + rollback_info_.clear(); + rollback_info_.shrink_to_fit(); + } + + private: + static constexpr size_t RoundUp(size_t n) { return (n + 7) & ~7; } + + using Tag = unsigned char; + + void* AllocRawInternal(uint32_t size, Tag tag) { + GOOGLE_DCHECK_GT(size, 0); + size = RoundUp(size); + + Block* to_relocate = nullptr; + Block* to_use = nullptr; + + for (size_t i = 0; i < kSmallSizes.size(); ++i) { + if (small_size_blocks_[i] != nullptr && size <= kSmallSizes[i]) { + to_use = to_relocate = PopBlock(small_size_blocks_[i]); + break; + } + } + + if (to_relocate != nullptr) { + // We found one in the loop. + } else if (current_ != nullptr && size + 1 <= current_->space_left()) { + to_use = current_; + } else { + // No space left anywhere, make a new block. + to_relocate = current_; + // For now we hardcode the size to one page. Note that the maximum we can + // allocate in the block according to the limits of Tag is less than 2k, + // so this can fit anything that Tag can represent. + constexpr size_t kBlockSize = 4096; + to_use = current_ = ::new (::operator new(kBlockSize)) Block(kBlockSize); + GOOGLE_DCHECK_GE(current_->space_left(), size + 1); + } + + ++num_allocations_; + if (!rollback_info_.empty() && rollback_info_.back().block == to_use) { + ++rollback_info_.back().count; + } else { + rollback_info_.push_back({to_use, 1}); + } + + void* p = to_use->Allocate(size, tag); + if (to_relocate != nullptr) { + RelocateToUsedList(to_relocate); + } + return p; + } + + static void OperatorDelete(void* p, size_t s) { +#if defined(__GXX_DELETE_WITH_SIZE__) || defined(__cpp_sized_deallocation) + ::operator delete(p, s); +#else + ::operator delete(p); +#endif + } + + struct OutOfLineAlloc { + void* ptr; + uint32_t size; + }; + + template + struct TypeList { + static constexpr Tag kSize = static_cast(sizeof...(T)); + }; + + template + static void RunVisitor(char* p, uint16_t* start, Visitor visit) { + *start -= RoundUp(sizeof(T)); + visit(reinterpret_cast(p + *start)); + } + + // Visit the allocation at the passed location. + // It updates start/end to be after the visited object. + // This allows visiting a whole block by calling the function in a loop. + template + static void VisitAlloc(char* p, uint16_t* start, uint16_t* end, Visitor visit, + TypeList) { + const Tag tag = static_cast(p[*end]); + if (tag >= kFirstRawTag) { + // Raw memory. Skip it. + *start -= TagToSize(tag); + } else { + using F = void (*)(char*, uint16_t*, Visitor); + static constexpr F kFuncs[] = {&RunVisitor...}; + kFuncs[tag](p, start, visit); + } + ++*end; + } + + template + static constexpr Tag TypeTag(TypeList) { + return 0; + } + + template < + typename U, typename T, typename... Ts, + typename = typename std::enable_if::value>::type> + static constexpr Tag TypeTag(TypeList) { + return 1 + TypeTag(TypeList{}); + } + + template + static constexpr Tag TypeTag(TypeList<>) { + static_assert(std::is_trivially_destructible::value, ""); + return SizeToRawTag(sizeof(U)); + } + + using KnownTypes = + TypeList, std::array, + std::array, std::array, + FileDescriptorTables, SourceCodeInfo, FileOptions, + MessageOptions, FieldOptions, ExtensionRangeOptions, + OneofOptions, EnumOptions, EnumValueOptions, ServiceOptions, + MethodOptions>; + static constexpr Tag kFirstRawTag = KnownTypes::kSize; + + + struct DestroyVisitor { + template + void operator()(T* p) { + p->~T(); + } + void operator()(OutOfLineAlloc* p) { OperatorDelete(p->ptr, p->size); } + }; + + static uint32_t SizeToRawTag(size_t n) { return (RoundUp(n) / 8) - 1; } + + static uint32_t TagToSize(Tag tag) { + GOOGLE_DCHECK_GE(tag, kFirstRawTag); + return static_cast(tag - kFirstRawTag + 1) * 8; + } + + struct Block { + uint16_t start_offset; + uint16_t end_offset; + uint16_t capacity; + Block* next; + + // `allocated_size` is the total size of the memory block allocated. + // The `Block` structure is constructed at the start and the rest of the + // memory is used as the payload of the `Block`. + explicit Block(uint32_t allocated_size) { + start_offset = 0; + end_offset = capacity = + reinterpret_cast(this) + allocated_size - data(); + next = nullptr; + } + + char* data() { + return reinterpret_cast(this) + RoundUp(sizeof(Block)); + } + + uint32_t memory_used() { + return data() + capacity - reinterpret_cast(this); + } + uint32_t space_left() const { return end_offset - start_offset; } + + void* Allocate(uint32_t n, Tag tag) { + GOOGLE_DCHECK_LE(n + 1, space_left()); + void* p = data() + start_offset; + start_offset += n; + data()[--end_offset] = tag; + return p; + } + + void Destroy() { OperatorDelete(this, memory_used()); } + + void PrependTo(Block*& list) { + next = list; + list = this; + } + + template + void VisitBlock(Visitor visit) { + for (uint16_t s = start_offset, e = end_offset; s != 0;) { + VisitAlloc(data(), &s, &e, visit, KnownTypes{}); + } + } + }; + + Block* PopBlock(Block*& list) { + Block* res = list; + list = list->next; + return res; + } + + void RelocateToUsedList(Block* to_relocate) { + if (current_ == nullptr) { + current_ = to_relocate; + current_->next = nullptr; + return; + } else if (current_->space_left() < to_relocate->space_left()) { + std::swap(current_, to_relocate); + current_->next = nullptr; + } + + for (int i = kSmallSizes.size(); --i >= 0;) { + if (to_relocate->space_left() >= 1 + kSmallSizes[i]) { + to_relocate->PrependTo(small_size_blocks_[i]); + return; + } + } + + to_relocate->PrependTo(full_blocks_); + } + + static constexpr std::array kSmallSizes = { + {// Sizes for pointer arrays. + 8, 16, 24, 32, + // Sizes for string arrays (for descriptor names). + // The most common array sizes are 2 and 3. + 2 * sizeof(std::string), 3 * sizeof(std::string)}}; + + // Helper function to iterate all lists. + std::array GetLists() const { + std::array res; + res[0] = current_; + res[1] = full_blocks_; + std::copy(small_size_blocks_.begin(), small_size_blocks_.end(), &res[2]); + return res; + } + + Block* current_ = nullptr; + std::array small_size_blocks_ = {{}}; + Block* full_blocks_ = nullptr; + + size_t num_allocations_ = 0; + struct RollbackInfo { + Block* block; + size_t count; + }; + std::vector rollback_info_; +}; + +constexpr std::array TableArena::kSmallSizes; + } // anonymous namespace // =================================================================== @@ -556,42 +1095,47 @@ class DescriptorPool::Tables { // The stack of files which are currently being built. Used to detect // cyclic dependencies when loading files from a DescriptorDatabase. Not - // used when fallback_database_ == NULL. - std::vector pending_files_; + // used when fallback_database_ == nullptr. + std::vector pending_files_; // A set of files which we have tried to load from the fallback database // and encountered errors. We will not attempt to load them again during // execution of the current public API call, but for compatibility with // legacy clients, this is cleared at the beginning of each public API call. - // Not used when fallback_database_ == NULL. - hash_set known_bad_files_; + // Not used when fallback_database_ == nullptr. + HASH_SET known_bad_files_; // A set of symbols which we have tried to load from the fallback database // and encountered errors. We will not attempt to load them again during // execution of the current public API call, but for compatibility with // legacy clients, this is cleared at the beginning of each public API call. - hash_set known_bad_symbols_; + HASH_SET known_bad_symbols_; // The set of descriptors for which we've already loaded the full // set of extensions numbers from fallback_database_. - hash_set extensions_loaded_from_db_; + HASH_SET extensions_loaded_from_db_; + + // Maps type name to Descriptor::WellKnownType. This is logically global + // and const, but we make it a member here to simplify its construction and + // destruction. This only has 20-ish entries and is one per DescriptorPool, + // so the overhead is small. + HASH_MAP well_known_types_; // ----------------------------------------------------------------- // Finding items. // Find symbols. This returns a null Symbol (symbol.IsNull() is true) // if not found. - inline Symbol FindSymbol(const string& key) const; + inline Symbol FindSymbol(StringPiece key) const; // This implements the body of DescriptorPool::Find*ByName(). It should // really be a private method of DescriptorPool, but that would require // declaring Symbol in descriptor.h, which would drag all kinds of other // stuff into the header. Yay C++. - Symbol FindByNameHelper( - const DescriptorPool* pool, const string& name); + Symbol FindByNameHelper(const DescriptorPool* pool, StringPiece name); - // These return NULL if not found. - inline const FileDescriptor* FindFile(const string& key) const; + // These return nullptr if not found. + inline const FileDescriptor* FindFile(StringPiece key) const; inline const FieldDescriptor* FindExtension(const Descriptor* extendee, int number) const; inline void FindAllExtensions(const Descriptor* extendee, @@ -604,7 +1148,7 @@ class DescriptorPool::Tables { // the key already exists in the table. For AddSymbol(), the string passed // in must be one that was constructed using AllocateString(), as it will // be used as a key in the symbols_by_name_ map without copying. - bool AddSymbol(const string& full_name, Symbol symbol); + bool AddSymbol(const std::string& full_name, Symbol symbol); bool AddFile(const FileDescriptor* file); bool AddExtension(const FieldDescriptor* field); @@ -615,67 +1159,84 @@ class DescriptorPool::Tables { // destroyed. Note that the object's destructor will never be called, // so its fields must be plain old data (primitive data types and // pointers). All of the descriptor types are such objects. - template Type* Allocate(); + template + Type* Allocate(); // Allocate an array of objects which will be reclaimed when the // pool in destroyed. Again, destructors are never called. - template Type* AllocateArray(int count); + template + Type* AllocateArray(int count); // Allocate a string which will be destroyed when the pool is destroyed. // The string is initialized to the given value for convenience. - string* AllocateString(const string& value); - - // Allocate a GoogleOnceDynamic which will be destroyed when the pool is - // destroyed. - GoogleOnceDynamic* AllocateOnceDynamic(); + const std::string* AllocateString(StringPiece value); + + // Copy the input into a NUL terminated string whose lifetime is managed by + // the pool. + const char* Strdup(StringPiece value); + + // Allocates an array of strings which will be destroyed when the pool is + // destroyed. The array is initialized with the input values. + template + const std::string* AllocateStringArray(In&&... values); + + struct FieldNamesResult { + std::string* array; + int lowercase_index; + int camelcase_index; + int json_index; + }; + // Allocate all 5 names of the field: + // name, full name, lowercase, camelcase and json. + // This function will dedup the strings when possible. + // The resulting array contains `name` at index 0, `full_name` at index 1 and + // the other 3 indices are specified in the result. + FieldNamesResult AllocateFieldNames(const std::string& name, + const std::string& scope, + const std::string* opt_json_name); + + // Create an object that will be deleted when the pool is destroyed. + // The object is value initialized, and its destructor will be called if + // non-trivial. + template + Type* Create(); // Allocate a protocol message object. Some older versions of GCC have // trouble understanding explicit template instantiations in some cases, so // in those cases we have to pass a dummy pointer of the right type as the // parameter instead of specifying the type explicitly. - template Type* AllocateMessage(Type* dummy = NULL); + template + Type* AllocateMessage(Type* dummy = nullptr); // Allocate a FileDescriptorTables object. FileDescriptorTables* AllocateFileTables(); private: - std::vector strings_; // All strings in the pool. - std::vector messages_; // All messages in the pool. - std::vector - once_dynamics_; // All GoogleOnceDynamics in the pool. - std::vector - file_tables_; // All file tables in the pool. - std::vector allocations_; // All other memory allocated in the pool. - - SymbolsByNameMap symbols_by_name_; - FilesByNameMap files_by_name_; + // All other memory allocated in the pool. Must be first as other objects can + // point into these. + TableArena arena_; + + SymbolsByNameSet symbols_by_name_; + FilesByNameMap files_by_name_; ExtensionsGroupedByDescriptorMap extensions_; struct CheckPoint { explicit CheckPoint(const Tables* tables) - : strings_before_checkpoint(tables->strings_.size()), - messages_before_checkpoint(tables->messages_.size()), - once_dynamics_before_checkpoint(tables->once_dynamics_.size()), - file_tables_before_checkpoint(tables->file_tables_.size()), - allocations_before_checkpoint(tables->allocations_.size()), + : arena_before_checkpoint(tables->arena_.num_allocations()), pending_symbols_before_checkpoint( tables->symbols_after_checkpoint_.size()), pending_files_before_checkpoint( tables->files_after_checkpoint_.size()), pending_extensions_before_checkpoint( tables->extensions_after_checkpoint_.size()) {} - int strings_before_checkpoint; - int messages_before_checkpoint; - int once_dynamics_before_checkpoint; - int file_tables_before_checkpoint; - int allocations_before_checkpoint; + int arena_before_checkpoint; int pending_symbols_before_checkpoint; int pending_files_before_checkpoint; int pending_extensions_before_checkpoint; }; std::vector checkpoints_; - std::vector symbols_after_checkpoint_; - std::vector files_after_checkpoint_; + std::vector symbols_after_checkpoint_; + std::vector files_after_checkpoint_; std::vector extensions_after_checkpoint_; // Allocate some bytes which will be reclaimed when the pool is @@ -703,23 +1264,19 @@ class FileDescriptorTables { // ----------------------------------------------------------------- // Finding items. - // Find symbols. These return a null Symbol (symbol.IsNull() is true) - // if not found. + // Returns a null Symbol (symbol.IsNull() is true) if not found. inline Symbol FindNestedSymbol(const void* parent, - const string& name) const; - inline Symbol FindNestedSymbolOfType(const void* parent, - const string& name, - const Symbol::Type type) const; - - // These return NULL if not found. - inline const FieldDescriptor* FindFieldByNumber( - const Descriptor* parent, int number) const; + StringPiece name) const; + + // These return nullptr if not found. + inline const FieldDescriptor* FindFieldByNumber(const Descriptor* parent, + int number) const; inline const FieldDescriptor* FindFieldByLowercaseName( - const void* parent, const string& lowercase_name) const; + const void* parent, StringPiece lowercase_name) const; inline const FieldDescriptor* FindFieldByCamelcaseName( - const void* parent, const string& camelcase_name) const; + const void* parent, StringPiece camelcase_name) const; inline const EnumValueDescriptor* FindEnumValueByNumber( - const EnumDescriptor* parent, int number) const; + const EnumDescriptor* parent, int number) const; // This creates a new EnumValueDescriptor if not found, in a thread-safe way. inline const EnumValueDescriptor* FindEnumValueByNumberCreatingIfUnknown( const EnumDescriptor* parent, int number) const; @@ -731,22 +1288,22 @@ class FileDescriptorTables { // the key already exists in the table. For AddAliasUnderParent(), the // string passed in must be one that was constructed using AllocateString(), // as it will be used as a key in the symbols_by_parent_ map without copying. - bool AddAliasUnderParent(const void* parent, const string& name, + bool AddAliasUnderParent(const void* parent, const std::string& name, Symbol symbol); - bool AddFieldByNumber(const FieldDescriptor* field); - bool AddEnumValueByNumber(const EnumValueDescriptor* value); + bool AddFieldByNumber(FieldDescriptor* field); + bool AddEnumValueByNumber(EnumValueDescriptor* value); // Adds the field to the lowercase_name and camelcase_name maps. Never // fails because we allow duplicates; the first field by the name wins. void AddFieldByStylizedNames(const FieldDescriptor* field); // Populates p->first->locations_by_path_ from p->second. - // Unusual signature dictated by GoogleOnceDynamic. + // Unusual signature dictated by internal::call_once. static void BuildLocationsByPath( std::pair* p); // Returns the location denoted by the specified path through info, - // or NULL if not found. + // or nullptr if not found. // The value of info must be that of the corresponding FileDescriptor. // (Conceptually a pure function, but stateful as an optimisation.) const SourceCodeInfo_Location* GetSourceLocation( @@ -765,88 +1322,60 @@ class FileDescriptorTables { const FileDescriptorTables* tables); void FieldsByCamelcaseNamesLazyInitInternal() const; - SymbolsByParentMap symbols_by_parent_; + SymbolsByParentSet symbols_by_parent_; mutable FieldsByNameMap fields_by_lowercase_name_; - mutable FieldsByNameMap* fields_by_lowercase_name_tmp_; - mutable GoogleOnceDynamic fields_by_lowercase_name_once_; + std::unique_ptr fields_by_lowercase_name_tmp_; + mutable internal::once_flag fields_by_lowercase_name_once_; mutable FieldsByNameMap fields_by_camelcase_name_; - mutable FieldsByNameMap* fields_by_camelcase_name_tmp_; - mutable GoogleOnceDynamic fields_by_camelcase_name_once_; - FieldsByNumberMap fields_by_number_; // Not including extensions. - EnumValuesByNumberMap enum_values_by_number_; - mutable EnumValuesByNumberMap unknown_enum_values_by_number_ - GOOGLE_GUARDED_BY(unknown_enum_values_mu_); + std::unique_ptr fields_by_camelcase_name_tmp_; + mutable internal::once_flag fields_by_camelcase_name_once_; + FieldsByNumberSet fields_by_number_; // Not including extensions. + EnumValuesByNumberSet enum_values_by_number_; + mutable EnumValuesByNumberSet unknown_enum_values_by_number_ + PROTOBUF_GUARDED_BY(unknown_enum_values_mu_); // Populated on first request to save space, hence constness games. - mutable GoogleOnceDynamic locations_by_path_once_; + mutable internal::once_flag locations_by_path_once_; mutable LocationsByPathMap locations_by_path_; // Mutex to protect the unknown-enum-value map due to dynamic // EnumValueDescriptor creation on unknown values. - mutable Mutex unknown_enum_values_mu_; + mutable internal::WrappedMutex unknown_enum_values_mu_; }; -DescriptorPool::Tables::Tables() - // Start some hash_map and hash_set objects with a small # of buckets - : known_bad_files_(3), - known_bad_symbols_(3), - extensions_loaded_from_db_(3), - symbols_by_name_(3), - files_by_name_(3) {} - - -DescriptorPool::Tables::~Tables() { - GOOGLE_DCHECK(checkpoints_.empty()); - // Note that the deletion order is important, since the destructors of some - // messages may refer to objects in allocations_. - STLDeleteElements(&messages_); - for (int i = 0; i < allocations_.size(); i++) { - operator delete(allocations_[i]); - } - STLDeleteElements(&strings_); - STLDeleteElements(&file_tables_); - STLDeleteElements(&once_dynamics_); -} +DescriptorPool::Tables::Tables() { + well_known_types_.insert({ + {"google.protobuf.DoubleValue", Descriptor::WELLKNOWNTYPE_DOUBLEVALUE}, + {"google.protobuf.FloatValue", Descriptor::WELLKNOWNTYPE_FLOATVALUE}, + {"google.protobuf.Int64Value", Descriptor::WELLKNOWNTYPE_INT64VALUE}, + {"google.protobuf.UInt64Value", Descriptor::WELLKNOWNTYPE_UINT64VALUE}, + {"google.protobuf.Int32Value", Descriptor::WELLKNOWNTYPE_INT32VALUE}, + {"google.protobuf.UInt32Value", Descriptor::WELLKNOWNTYPE_UINT32VALUE}, + {"google.protobuf.StringValue", Descriptor::WELLKNOWNTYPE_STRINGVALUE}, + {"google.protobuf.BytesValue", Descriptor::WELLKNOWNTYPE_BYTESVALUE}, + {"google.protobuf.BoolValue", Descriptor::WELLKNOWNTYPE_BOOLVALUE}, + {"google.protobuf.Any", Descriptor::WELLKNOWNTYPE_ANY}, + {"google.protobuf.FieldMask", Descriptor::WELLKNOWNTYPE_FIELDMASK}, + {"google.protobuf.Duration", Descriptor::WELLKNOWNTYPE_DURATION}, + {"google.protobuf.Timestamp", Descriptor::WELLKNOWNTYPE_TIMESTAMP}, + {"google.protobuf.Value", Descriptor::WELLKNOWNTYPE_VALUE}, + {"google.protobuf.ListValue", Descriptor::WELLKNOWNTYPE_LISTVALUE}, + {"google.protobuf.Struct", Descriptor::WELLKNOWNTYPE_STRUCT}, + }); +} + +DescriptorPool::Tables::~Tables() { GOOGLE_DCHECK(checkpoints_.empty()); } FileDescriptorTables::FileDescriptorTables() - // Initialize all the hash tables to start out with a small # of buckets. - : symbols_by_parent_(3), - fields_by_lowercase_name_(3), - fields_by_lowercase_name_tmp_(new FieldsByNameMap()), - fields_by_camelcase_name_(3), - fields_by_camelcase_name_tmp_(new FieldsByNameMap()), - fields_by_number_(3), - enum_values_by_number_(3), - unknown_enum_values_by_number_(3), - locations_by_path_(3) {} + : fields_by_lowercase_name_tmp_(new FieldsByNameMap()), + fields_by_camelcase_name_tmp_(new FieldsByNameMap()) {} FileDescriptorTables::~FileDescriptorTables() {} -namespace { - -FileDescriptorTables* file_descriptor_tables_ = NULL; -GOOGLE_PROTOBUF_DECLARE_ONCE(file_descriptor_tables_once_init_); - -void DeleteFileDescriptorTables() { - delete file_descriptor_tables_; - file_descriptor_tables_ = NULL; -} - -void InitFileDescriptorTables() { - file_descriptor_tables_ = new FileDescriptorTables(); - internal::OnShutdown(&DeleteFileDescriptorTables); -} - -inline void InitFileDescriptorTablesOnce() { - ::google::protobuf::GoogleOnceInit( - &file_descriptor_tables_once_init_, &InitFileDescriptorTables); -} - -} // anonymous namespace - inline const FileDescriptorTables& FileDescriptorTables::GetEmptyInstance() { - InitFileDescriptorTablesOnce(); - return *file_descriptor_tables_; + static auto file_descriptor_tables = + internal::OnShutdownDelete(new FileDescriptorTables()); + return *file_descriptor_tables; } void DescriptorPool::Tables::AddCheckpoint() { @@ -862,6 +1391,7 @@ void DescriptorPool::Tables::ClearLastCheckpoint() { symbols_after_checkpoint_.clear(); files_after_checkpoint_.clear(); extensions_after_checkpoint_.clear(); + arena_.ClearRollbackData(); } } @@ -869,19 +1399,18 @@ void DescriptorPool::Tables::RollbackToLastCheckpoint() { GOOGLE_DCHECK(!checkpoints_.empty()); const CheckPoint& checkpoint = checkpoints_.back(); - for (int i = checkpoint.pending_symbols_before_checkpoint; - i < symbols_after_checkpoint_.size(); - i++) { - symbols_by_name_.erase(symbols_after_checkpoint_[i]); + for (size_t i = checkpoint.pending_symbols_before_checkpoint; + i < symbols_after_checkpoint_.size(); i++) { + Symbol::QueryKey name; + name.name = symbols_after_checkpoint_[i]; + symbols_by_name_.erase(Symbol(&name)); } - for (int i = checkpoint.pending_files_before_checkpoint; - i < files_after_checkpoint_.size(); - i++) { + for (size_t i = checkpoint.pending_files_before_checkpoint; + i < files_after_checkpoint_.size(); i++) { files_by_name_.erase(files_after_checkpoint_[i]); } - for (int i = checkpoint.pending_extensions_before_checkpoint; - i < extensions_after_checkpoint_.size(); - i++) { + for (size_t i = checkpoint.pending_extensions_before_checkpoint; + i < extensions_after_checkpoint_.size(); i++) { extensions_.erase(extensions_after_checkpoint_[i]); } @@ -891,71 +1420,48 @@ void DescriptorPool::Tables::RollbackToLastCheckpoint() { extensions_after_checkpoint_.resize( checkpoint.pending_extensions_before_checkpoint); - STLDeleteContainerPointers( - strings_.begin() + checkpoint.strings_before_checkpoint, strings_.end()); - STLDeleteContainerPointers( - messages_.begin() + checkpoint.messages_before_checkpoint, - messages_.end()); - STLDeleteContainerPointers( - once_dynamics_.begin() + checkpoint.once_dynamics_before_checkpoint, - once_dynamics_.end()); - STLDeleteContainerPointers( - file_tables_.begin() + checkpoint.file_tables_before_checkpoint, - file_tables_.end()); - for (int i = checkpoint.allocations_before_checkpoint; - i < allocations_.size(); - i++) { - operator delete(allocations_[i]); - } - - strings_.resize(checkpoint.strings_before_checkpoint); - messages_.resize(checkpoint.messages_before_checkpoint); - once_dynamics_.resize(checkpoint.once_dynamics_before_checkpoint); - file_tables_.resize(checkpoint.file_tables_before_checkpoint); - allocations_.resize(checkpoint.allocations_before_checkpoint); + arena_.RollbackTo(checkpoint.arena_before_checkpoint); checkpoints_.pop_back(); } // ------------------------------------------------------------------- -inline Symbol DescriptorPool::Tables::FindSymbol(const string& key) const { - const Symbol* result = FindOrNull(symbols_by_name_, key.c_str()); - if (result == NULL) { - return kNullSymbol; - } else { - return *result; - } +inline Symbol DescriptorPool::Tables::FindSymbol(StringPiece key) const { + Symbol::QueryKey name; + name.name = key; + auto it = symbols_by_name_.find(Symbol(&name)); + return it == symbols_by_name_.end() ? kNullSymbol : *it; } inline Symbol FileDescriptorTables::FindNestedSymbol( - const void* parent, const string& name) const { - const Symbol* result = - FindOrNull(symbols_by_parent_, PointerStringPair(parent, name.c_str())); - if (result == NULL) { - return kNullSymbol; - } else { - return *result; - } -} - -inline Symbol FileDescriptorTables::FindNestedSymbolOfType( - const void* parent, const string& name, const Symbol::Type type) const { - Symbol result = FindNestedSymbol(parent, name); - if (result.type != type) return kNullSymbol; - return result; + const void* parent, StringPiece name) const { + Symbol::QueryKey query; + query.name = name; + query.parent = parent; + auto it = symbols_by_parent_.find(Symbol(&query)); + return it == symbols_by_parent_.end() ? kNullSymbol : *it; } -Symbol DescriptorPool::Tables::FindByNameHelper( - const DescriptorPool* pool, const string& name) { +Symbol DescriptorPool::Tables::FindByNameHelper(const DescriptorPool* pool, + StringPiece name) { + if (pool->mutex_ != nullptr) { + // Fast path: the Symbol is already cached. This is just a hash lookup. + ReaderMutexLock lock(pool->mutex_); + if (known_bad_symbols_.empty() && known_bad_files_.empty()) { + Symbol result = FindSymbol(name); + if (!result.IsNull()) return result; + } + } MutexLockMaybe lock(pool->mutex_); - known_bad_symbols_.clear(); - known_bad_files_.clear(); + if (pool->fallback_database_ != nullptr) { + known_bad_symbols_.clear(); + known_bad_files_.clear(); + } Symbol result = FindSymbol(name); - if (result.IsNull() && pool->underlay_ != NULL) { + if (result.IsNull() && pool->underlay_ != nullptr) { // Symbol not found; check the underlay. - result = - pool->underlay_->tables_->FindByNameHelper(pool->underlay_, name); + result = pool->underlay_->tables_->FindByNameHelper(pool->underlay_, name); } if (result.IsNull()) { @@ -969,19 +1475,31 @@ Symbol DescriptorPool::Tables::FindByNameHelper( } inline const FileDescriptor* DescriptorPool::Tables::FindFile( - const string& key) const { - return FindPtrOrNull(files_by_name_, key.c_str()); + StringPiece key) const { + return FindPtrOrNull(files_by_name_, key); } inline const FieldDescriptor* FileDescriptorTables::FindFieldByNumber( const Descriptor* parent, int number) const { - return FindPtrOrNull(fields_by_number_, std::make_pair(parent, number)); + // If `number` is within the sequential range, just index into the parent + // without doing a table lookup. + if (parent != nullptr && // + 1 <= number && number <= parent->sequential_field_limit_) { + return parent->field(number - 1); + } + + Symbol::QueryKey query; + query.parent = parent; + query.field_number = number; + + auto it = fields_by_number_.find(Symbol(&query)); + return it == fields_by_number_.end() ? nullptr : it->field_descriptor(); } const void* FileDescriptorTables::FindParentForFieldsByMap( const FieldDescriptor* field) const { if (field->is_extension()) { - if (field->extension_scope() == NULL) { + if (field->extension_scope() == nullptr) { return field->file(); } else { return field->extension_scope(); @@ -997,20 +1515,22 @@ void FileDescriptorTables::FieldsByLowercaseNamesLazyInitStatic( } void FileDescriptorTables::FieldsByLowercaseNamesLazyInitInternal() const { - for (FieldsByNumberMap::const_iterator it = fields_by_number_.begin(); - it != fields_by_number_.end(); it++) { - PointerStringPair lowercase_key(FindParentForFieldsByMap(it->second), - it->second->lowercase_name().c_str()); - InsertIfNotPresent(&fields_by_lowercase_name_, lowercase_key, it->second); + for (Symbol symbol : symbols_by_parent_) { + const FieldDescriptor* field = symbol.field_descriptor(); + if (!field) continue; + PointerStringPair lowercase_key(FindParentForFieldsByMap(field), + field->lowercase_name().c_str()); + InsertIfNotPresent(&fields_by_lowercase_name_, lowercase_key, field); } } inline const FieldDescriptor* FileDescriptorTables::FindFieldByLowercaseName( - const void* parent, const string& lowercase_name) const { - fields_by_lowercase_name_once_.Init( + const void* parent, StringPiece lowercase_name) const { + internal::call_once( + fields_by_lowercase_name_once_, &FileDescriptorTables::FieldsByLowercaseNamesLazyInitStatic, this); return FindPtrOrNull(fields_by_lowercase_name_, - PointerStringPair(parent, lowercase_name.c_str())); + PointerStringPair(parent, lowercase_name)); } void FileDescriptorTables::FieldsByCamelcaseNamesLazyInitStatic( @@ -1019,25 +1539,41 @@ void FileDescriptorTables::FieldsByCamelcaseNamesLazyInitStatic( } void FileDescriptorTables::FieldsByCamelcaseNamesLazyInitInternal() const { - for (FieldsByNumberMap::const_iterator it = fields_by_number_.begin(); - it != fields_by_number_.end(); it++) { - PointerStringPair camelcase_key(FindParentForFieldsByMap(it->second), - it->second->camelcase_name().c_str()); - InsertIfNotPresent(&fields_by_camelcase_name_, camelcase_key, it->second); + for (Symbol symbol : symbols_by_parent_) { + const FieldDescriptor* field = symbol.field_descriptor(); + if (!field) continue; + PointerStringPair camelcase_key(FindParentForFieldsByMap(field), + field->camelcase_name().c_str()); + InsertIfNotPresent(&fields_by_camelcase_name_, camelcase_key, field); } } inline const FieldDescriptor* FileDescriptorTables::FindFieldByCamelcaseName( - const void* parent, const string& camelcase_name) const { - fields_by_camelcase_name_once_.Init( - &FileDescriptorTables::FieldsByCamelcaseNamesLazyInitStatic, this); + const void* parent, StringPiece camelcase_name) const { + internal::call_once( + fields_by_camelcase_name_once_, + FileDescriptorTables::FieldsByCamelcaseNamesLazyInitStatic, this); return FindPtrOrNull(fields_by_camelcase_name_, - PointerStringPair(parent, camelcase_name.c_str())); + PointerStringPair(parent, camelcase_name)); } inline const EnumValueDescriptor* FileDescriptorTables::FindEnumValueByNumber( const EnumDescriptor* parent, int number) const { - return FindPtrOrNull(enum_values_by_number_, std::make_pair(parent, number)); + // If `number` is within the sequential range, just index into the parent + // without doing a table lookup. + const int base = parent->value(0)->number(); + if (base <= number && + number <= static_cast(base) + parent->sequential_value_limit_) { + return parent->value(number - base); + } + + Symbol::QueryKey query; + query.parent = parent; + query.field_number = number; + + auto it = enum_values_by_number_.find(Symbol(&query)); + return it == enum_values_by_number_.end() ? nullptr + : it->enum_value_descriptor(); } inline const EnumValueDescriptor* @@ -1045,54 +1581,60 @@ FileDescriptorTables::FindEnumValueByNumberCreatingIfUnknown( const EnumDescriptor* parent, int number) const { // First try, with map of compiled-in values. { - const EnumValueDescriptor* desc = - FindPtrOrNull(enum_values_by_number_, std::make_pair(parent, number)); - if (desc != NULL) { - return desc; + const auto* value = FindEnumValueByNumber(parent, number); + if (value != nullptr) { + return value; } } + + Symbol::QueryKey query; + query.parent = parent; + query.field_number = number; + // Second try, with reader lock held on unknown enum values: common case. { ReaderMutexLock l(&unknown_enum_values_mu_); - const EnumValueDescriptor* desc = FindPtrOrNull( - unknown_enum_values_by_number_, std::make_pair(parent, number)); - if (desc != NULL) { - return desc; + auto it = unknown_enum_values_by_number_.find(Symbol(&query)); + if (it != unknown_enum_values_by_number_.end() && + it->enum_value_descriptor() != nullptr) { + return it->enum_value_descriptor(); } } // If not found, try again with writer lock held, and create new descriptor if // necessary. { WriterMutexLock l(&unknown_enum_values_mu_); - const EnumValueDescriptor* desc = FindPtrOrNull( - unknown_enum_values_by_number_, std::make_pair(parent, number)); - if (desc != NULL) { - return desc; + auto it = unknown_enum_values_by_number_.find(Symbol(&query)); + if (it != unknown_enum_values_by_number_.end() && + it->enum_value_descriptor() != nullptr) { + return it->enum_value_descriptor(); } // Create an EnumValueDescriptor dynamically. We don't insert it into the // EnumDescriptor (it's not a part of the enum as originally defined), but // we do insert it into the table so that we can return the same pointer // later. - string enum_value_name = StringPrintf( - "UNKNOWN_ENUM_VALUE_%s_%d", parent->name().c_str(), number); - DescriptorPool::Tables* tables = - const_cast(DescriptorPool::generated_pool()-> - tables_.get()); - EnumValueDescriptor* result = tables->Allocate(); - result->name_ = tables->AllocateString(enum_value_name); - result->full_name_ = tables->AllocateString(parent->full_name() + - "." + enum_value_name); + std::string enum_value_name = StringPrintf("UNKNOWN_ENUM_VALUE_%s_%d", + parent->name().c_str(), number); + auto* pool = DescriptorPool::generated_pool(); + auto* tables = const_cast(pool->tables_.get()); + EnumValueDescriptor* result; + { + // Must lock the pool because we will do allocations in the shared arena. + MutexLockMaybe l2(pool->mutex_); + result = tables->Allocate(); + result->all_names_ = tables->AllocateStringArray( + enum_value_name, + StrCat(parent->full_name(), ".", enum_value_name)); + } result->number_ = number; result->type_ = parent; result->options_ = &EnumValueOptions::default_instance(); - InsertIfNotPresent(&unknown_enum_values_by_number_, - std::make_pair(parent, number), result); + unknown_enum_values_by_number_.insert(Symbol::EnumValue(result, 0)); return result; } } - inline const FieldDescriptor* DescriptorPool::Tables::FindExtension( const Descriptor* extendee, int number) const { return FindPtrOrNull(extensions_, std::make_pair(extendee, number)); @@ -1110,9 +1652,10 @@ inline void DescriptorPool::Tables::FindAllExtensions( // ------------------------------------------------------------------- -bool DescriptorPool::Tables::AddSymbol( - const string& full_name, Symbol symbol) { - if (InsertIfNotPresent(&symbols_by_name_, full_name.c_str(), symbol)) { +bool DescriptorPool::Tables::AddSymbol(const std::string& full_name, + Symbol symbol) { + GOOGLE_DCHECK_EQ(full_name, symbol.full_name()); + if (symbols_by_name_.insert(symbol).second) { symbols_after_checkpoint_.push_back(full_name.c_str()); return true; } else { @@ -1120,14 +1663,16 @@ bool DescriptorPool::Tables::AddSymbol( } } -bool FileDescriptorTables::AddAliasUnderParent( - const void* parent, const string& name, Symbol symbol) { - PointerStringPair by_parent_key(parent, name.c_str()); - return InsertIfNotPresent(&symbols_by_parent_, by_parent_key, symbol); +bool FileDescriptorTables::AddAliasUnderParent(const void* parent, + const std::string& name, + Symbol symbol) { + GOOGLE_DCHECK_EQ(name, symbol.parent_name_key().second); + GOOGLE_DCHECK_EQ(parent, symbol.parent_name_key().first); + return symbols_by_parent_.insert(symbol).second; } bool DescriptorPool::Tables::AddFile(const FileDescriptor* file) { - if (InsertIfNotPresent(&files_by_name_, file->name().c_str(), file)) { + if (InsertIfNotPresent(&files_by_name_, file->name(), file)) { files_after_checkpoint_.push_back(file->name().c_str()); return true; } else { @@ -1137,10 +1682,8 @@ bool DescriptorPool::Tables::AddFile(const FileDescriptor* file) { void FileDescriptorTables::FinalizeTables() { // Clean up the temporary maps used by AddFieldByStylizedNames(). - delete fields_by_lowercase_name_tmp_; - fields_by_lowercase_name_tmp_ = NULL; - delete fields_by_camelcase_name_tmp_; - fields_by_camelcase_name_tmp_ = NULL; + fields_by_lowercase_name_tmp_ = nullptr; + fields_by_camelcase_name_tmp_ = nullptr; } void FileDescriptorTables::AddFieldByStylizedNames( @@ -1155,31 +1698,46 @@ void FileDescriptorTables::AddFieldByStylizedNames( // entries from fields_by_number_. PointerStringPair lowercase_key(parent, field->lowercase_name().c_str()); - if (!InsertIfNotPresent(fields_by_lowercase_name_tmp_, lowercase_key, - field)) { + if (!InsertIfNotPresent(fields_by_lowercase_name_tmp_.get(), + lowercase_key, field)) { InsertIfNotPresent( &fields_by_lowercase_name_, lowercase_key, FindPtrOrNull(*fields_by_lowercase_name_tmp_, lowercase_key)); } PointerStringPair camelcase_key(parent, field->camelcase_name().c_str()); - if (!InsertIfNotPresent(fields_by_camelcase_name_tmp_, camelcase_key, - field)) { + if (!InsertIfNotPresent(fields_by_camelcase_name_tmp_.get(), + camelcase_key, field)) { InsertIfNotPresent( &fields_by_camelcase_name_, camelcase_key, FindPtrOrNull(*fields_by_camelcase_name_tmp_, camelcase_key)); } } -bool FileDescriptorTables::AddFieldByNumber(const FieldDescriptor* field) { - DescriptorIntPair key(field->containing_type(), field->number()); - return InsertIfNotPresent(&fields_by_number_, key, field); +bool FileDescriptorTables::AddFieldByNumber(FieldDescriptor* field) { + // Skip fields that are at the start of the sequence. + if (field->containing_type() != nullptr && field->number() >= 1 && + field->number() <= field->containing_type()->sequential_field_limit_) { + if (field->is_extension()) { + // Conflicts with the field that already exists in the sequential range. + return false; + } + // Only return true if the field at that index matches. Otherwise it + // conflicts with the existing field in the sequential range. + return field->containing_type()->field(field->number() - 1) == field; + } + + return fields_by_number_.insert(Symbol(field)).second; } -bool FileDescriptorTables::AddEnumValueByNumber( - const EnumValueDescriptor* value) { - EnumIntPair key(value->type(), value->number()); - return InsertIfNotPresent(&enum_values_by_number_, key, value); +bool FileDescriptorTables::AddEnumValueByNumber(EnumValueDescriptor* value) { + // Skip values that are at the start of the sequence. + const int base = value->type()->value(0)->number(); + if (base <= value->number() && + value->number() <= + static_cast(base) + value->type()->sequential_value_limit_) + return true; + return enum_values_by_number_.insert(Symbol::EnumValue(value, 0)).second; } bool DescriptorPool::Tables::AddExtension(const FieldDescriptor* field) { @@ -1194,51 +1752,125 @@ bool DescriptorPool::Tables::AddExtension(const FieldDescriptor* field) { // ------------------------------------------------------------------- -template +template Type* DescriptorPool::Tables::Allocate() { return reinterpret_cast(AllocateBytes(sizeof(Type))); } -template +template Type* DescriptorPool::Tables::AllocateArray(int count) { return reinterpret_cast(AllocateBytes(sizeof(Type) * count)); } -string* DescriptorPool::Tables::AllocateString(const string& value) { - string* result = new string(value); - strings_.push_back(result); - return result; +const std::string* DescriptorPool::Tables::AllocateString( + StringPiece value) { + return arena_.Create(value); } -GoogleOnceDynamic* DescriptorPool::Tables::AllocateOnceDynamic() { - GoogleOnceDynamic* result = new GoogleOnceDynamic(); - once_dynamics_.push_back(result); +const char* DescriptorPool::Tables::Strdup(StringPiece value) { + char* p = AllocateArray(static_cast(value.size() + 1)); + memcpy(p, value.data(), value.size()); + p[value.size()] = 0; + return p; +} + +template +const std::string* DescriptorPool::Tables::AllocateStringArray(In&&... values) { + auto& array = *arena_.Create>(); + array = {{std::string(std::forward(values))...}}; + return array.data(); +} + +DescriptorPool::Tables::FieldNamesResult +DescriptorPool::Tables::AllocateFieldNames(const std::string& name, + const std::string& scope, + const std::string* opt_json_name) { + std::string lowercase_name = name; + LowerString(&lowercase_name); + + std::string camelcase_name = ToCamelCase(name, /* lower_first = */ true); + std::string json_name; + if (opt_json_name != nullptr) { + json_name = *opt_json_name; + } else { + json_name = ToJsonName(name); + } + + const bool lower_eq_name = lowercase_name == name; + const bool camel_eq_name = camelcase_name == name; + const bool json_eq_name = json_name == name; + const bool json_eq_camel = json_name == camelcase_name; + + const int total_count = 2 + (lower_eq_name ? 0 : 1) + + (camel_eq_name ? 0 : 1) + + (json_eq_name || json_eq_camel ? 0 : 1); + FieldNamesResult result{nullptr, 0, 0, 0}; + // We use std::array to allow handling of the destruction of the strings. + switch (total_count) { + case 2: + result.array = arena_.Create>()->data(); + break; + case 3: + result.array = arena_.Create>()->data(); + break; + case 4: + result.array = arena_.Create>()->data(); + break; + case 5: + result.array = arena_.Create>()->data(); + break; + } + + result.array[0] = name; + if (scope.empty()) { + result.array[1] = name; + } else { + result.array[1] = StrCat(scope, ".", name); + } + int index = 2; + if (lower_eq_name) { + result.lowercase_index = 0; + } else { + result.lowercase_index = index; + result.array[index++] = std::move(lowercase_name); + } + + if (camel_eq_name) { + result.camelcase_index = 0; + } else { + result.camelcase_index = index; + result.array[index++] = std::move(camelcase_name); + } + + if (json_eq_name) { + result.json_index = 0; + } else if (json_eq_camel) { + result.json_index = result.camelcase_index; + } else { + result.json_index = index; + result.array[index] = std::move(json_name); + } + return result; } -template +template +Type* DescriptorPool::Tables::Create() { + return arena_.Create(); +} + +template Type* DescriptorPool::Tables::AllocateMessage(Type* /* dummy */) { - Type* result = new Type; - messages_.push_back(result); - return result; + return arena_.Create(); } FileDescriptorTables* DescriptorPool::Tables::AllocateFileTables() { - FileDescriptorTables* result = new FileDescriptorTables; - file_tables_.push_back(result); - return result; + return arena_.Create(); } void* DescriptorPool::Tables::AllocateBytes(int size) { - // TODO(kenton): Would it be worthwhile to implement this in some more - // sophisticated way? Probably not for the open source release, but for - // internal use we could easily plug in one of our existing memory pool - // allocators... - if (size == 0) return NULL; - - void* result = operator new(size); - allocations_.push_back(result); - return result; + if (size == 0) return nullptr; + return arena_.AllocateMemory(size); } void FileDescriptorTables::BuildLocationsByPath( @@ -1253,7 +1885,8 @@ const SourceCodeInfo_Location* FileDescriptorTables::GetSourceLocation( const std::vector& path, const SourceCodeInfo* info) const { std::pair p( std::make_pair(this, info)); - locations_by_path_once_.Init(&FileDescriptorTables::BuildLocationsByPath, &p); + internal::call_once(locations_by_path_once_, + FileDescriptorTables::BuildLocationsByPath, &p); return FindPtrOrNull(locations_by_path_, Join(path, ",")); } @@ -1263,45 +1896,44 @@ const SourceCodeInfo_Location* FileDescriptorTables::GetSourceLocation( DescriptorPool::ErrorCollector::~ErrorCollector() {} DescriptorPool::DescriptorPool() - : mutex_(NULL), - fallback_database_(NULL), - default_error_collector_(NULL), - underlay_(NULL), - tables_(new Tables), - enforce_dependencies_(true), - lazily_build_dependencies_(false), - allow_unknown_(false), - enforce_weak_(false), - disallow_enforce_utf8_(false) {} + : mutex_(nullptr), + fallback_database_(nullptr), + default_error_collector_(nullptr), + underlay_(nullptr), + tables_(new Tables), + enforce_dependencies_(true), + lazily_build_dependencies_(false), + allow_unknown_(false), + enforce_weak_(false), + disallow_enforce_utf8_(false) {} DescriptorPool::DescriptorPool(DescriptorDatabase* fallback_database, ErrorCollector* error_collector) - : mutex_(new Mutex), - fallback_database_(fallback_database), - default_error_collector_(error_collector), - underlay_(NULL), - tables_(new Tables), - enforce_dependencies_(true), - lazily_build_dependencies_(false), - allow_unknown_(false), - enforce_weak_(false), - disallow_enforce_utf8_(false) { -} + : mutex_(new internal::WrappedMutex), + fallback_database_(fallback_database), + default_error_collector_(error_collector), + underlay_(nullptr), + tables_(new Tables), + enforce_dependencies_(true), + lazily_build_dependencies_(false), + allow_unknown_(false), + enforce_weak_(false), + disallow_enforce_utf8_(false) {} DescriptorPool::DescriptorPool(const DescriptorPool* underlay) - : mutex_(NULL), - fallback_database_(NULL), - default_error_collector_(NULL), - underlay_(underlay), - tables_(new Tables), - enforce_dependencies_(true), - lazily_build_dependencies_(false), - allow_unknown_(false), - enforce_weak_(false), - disallow_enforce_utf8_(false) {} + : mutex_(nullptr), + fallback_database_(nullptr), + default_error_collector_(nullptr), + underlay_(underlay), + tables_(new Tables), + enforce_dependencies_(true), + lazily_build_dependencies_(false), + allow_unknown_(false), + enforce_weak_(false), + disallow_enforce_utf8_(false) {} DescriptorPool::~DescriptorPool() { - if (mutex_ != NULL) delete mutex_; + if (mutex_ != nullptr) delete mutex_; } // DescriptorPool::BuildFile() defined later. @@ -1311,17 +1943,18 @@ void DescriptorPool::InternalDontEnforceDependencies() { enforce_dependencies_ = false; } -void DescriptorPool::AddUnusedImportTrackFile(const string& file_name) { - unused_import_track_files_.insert(file_name); +void DescriptorPool::AddUnusedImportTrackFile(ConstStringParam file_name, + bool is_error) { + unused_import_track_files_[std::string(file_name)] = is_error; } void DescriptorPool::ClearUnusedImportTrackFiles() { unused_import_track_files_.clear(); } -bool DescriptorPool::InternalIsFileLoaded(const string& filename) const { +bool DescriptorPool::InternalIsFileLoaded(ConstStringParam filename) const { MutexLockMaybe lock(mutex_); - return tables_->FindFile(filename) != NULL; + return tables_->FindFile(filename) != nullptr; } // generated_pool ==================================================== @@ -1329,43 +1962,38 @@ bool DescriptorPool::InternalIsFileLoaded(const string& filename) const { namespace { -EncodedDescriptorDatabase* generated_database_ = NULL; -DescriptorPool* generated_pool_ = NULL; -GOOGLE_PROTOBUF_DECLARE_ONCE(generated_pool_init_); +EncodedDescriptorDatabase* GeneratedDatabase() { + static auto generated_database = + internal::OnShutdownDelete(new EncodedDescriptorDatabase()); + return generated_database; +} -void DeleteGeneratedPool() { - delete generated_database_; - generated_database_ = NULL; - delete generated_pool_; - generated_pool_ = NULL; +DescriptorPool* NewGeneratedPool() { + auto generated_pool = new DescriptorPool(GeneratedDatabase()); + generated_pool->InternalSetLazilyBuildDependencies(); + return generated_pool; } -static void InitGeneratedPool() { - generated_database_ = new EncodedDescriptorDatabase; - generated_pool_ = new DescriptorPool(generated_database_); - generated_pool_->InternalSetLazilyBuildDependencies(); +} // anonymous namespace - internal::OnShutdown(&DeleteGeneratedPool); +DescriptorDatabase* DescriptorPool::internal_generated_database() { + return GeneratedDatabase(); } -inline void InitGeneratedPoolOnce() { - ::google::protobuf::GoogleOnceInit(&generated_pool_init_, &InitGeneratedPool); +DescriptorPool* DescriptorPool::internal_generated_pool() { + static DescriptorPool* generated_pool = + internal::OnShutdownDelete(NewGeneratedPool()); + return generated_pool; } -} // anonymous namespace - const DescriptorPool* DescriptorPool::generated_pool() { - InitGeneratedPoolOnce(); - return generated_pool_; + const DescriptorPool* pool = internal_generated_pool(); + // Ensure that descriptor.proto has been registered in the generated pool. + DescriptorProto::descriptor(); + return pool; } - -DescriptorPool* DescriptorPool::internal_generated_pool() { - InitGeneratedPoolOnce(); - return generated_pool_; -} - void DescriptorPool::InternalAddGeneratedFile( const void* encoded_file_descriptor, int size) { // So, this function is called in the process of initializing the @@ -1390,8 +2018,7 @@ void DescriptorPool::InternalAddGeneratedFile( // Therefore, when we parse one, we have to be very careful to avoid using // any descriptor-based operations, since this might cause infinite recursion // or deadlock. - InitGeneratedPoolOnce(); - GOOGLE_CHECK(generated_database_->Add(encoded_file_descriptor, size)); + GOOGLE_CHECK(GeneratedDatabase()->Add(encoded_file_descriptor, size)); } @@ -1401,149 +2028,195 @@ void DescriptorPool::InternalAddGeneratedFile( // there's any good way to factor it out. Think about this some time when // there's nothing more important to do (read: never). -const FileDescriptor* DescriptorPool::FindFileByName(const string& name) const { +const FileDescriptor* DescriptorPool::FindFileByName( + ConstStringParam name) const { MutexLockMaybe lock(mutex_); - tables_->known_bad_symbols_.clear(); - tables_->known_bad_files_.clear(); + if (fallback_database_ != nullptr) { + tables_->known_bad_symbols_.clear(); + tables_->known_bad_files_.clear(); + } const FileDescriptor* result = tables_->FindFile(name); - if (result != NULL) return result; - if (underlay_ != NULL) { + if (result != nullptr) return result; + if (underlay_ != nullptr) { result = underlay_->FindFileByName(name); - if (result != NULL) return result; + if (result != nullptr) return result; } if (TryFindFileInFallbackDatabase(name)) { result = tables_->FindFile(name); - if (result != NULL) return result; + if (result != nullptr) return result; } - return NULL; + return nullptr; } const FileDescriptor* DescriptorPool::FindFileContainingSymbol( - const string& symbol_name) const { + ConstStringParam symbol_name) const { MutexLockMaybe lock(mutex_); - tables_->known_bad_symbols_.clear(); - tables_->known_bad_files_.clear(); + if (fallback_database_ != nullptr) { + tables_->known_bad_symbols_.clear(); + tables_->known_bad_files_.clear(); + } Symbol result = tables_->FindSymbol(symbol_name); if (!result.IsNull()) return result.GetFile(); - if (underlay_ != NULL) { + if (underlay_ != nullptr) { const FileDescriptor* file_result = - underlay_->FindFileContainingSymbol(symbol_name); - if (file_result != NULL) return file_result; + underlay_->FindFileContainingSymbol(symbol_name); + if (file_result != nullptr) return file_result; } if (TryFindSymbolInFallbackDatabase(symbol_name)) { result = tables_->FindSymbol(symbol_name); if (!result.IsNull()) return result.GetFile(); } - return NULL; + return nullptr; } const Descriptor* DescriptorPool::FindMessageTypeByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - return (result.type == Symbol::MESSAGE) ? result.descriptor : NULL; + ConstStringParam name) const { + return tables_->FindByNameHelper(this, name).descriptor(); } const FieldDescriptor* DescriptorPool::FindFieldByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - if (result.type == Symbol::FIELD && - !result.field_descriptor->is_extension()) { - return result.field_descriptor; - } else { - return NULL; + ConstStringParam name) const { + if (const FieldDescriptor* field = + tables_->FindByNameHelper(this, name).field_descriptor()) { + if (!field->is_extension()) { + return field; + } } + return nullptr; } const FieldDescriptor* DescriptorPool::FindExtensionByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - if (result.type == Symbol::FIELD && - result.field_descriptor->is_extension()) { - return result.field_descriptor; - } else { - return NULL; + ConstStringParam name) const { + if (const FieldDescriptor* field = + tables_->FindByNameHelper(this, name).field_descriptor()) { + if (field->is_extension()) { + return field; + } } + return nullptr; } const OneofDescriptor* DescriptorPool::FindOneofByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - return (result.type == Symbol::ONEOF) ? result.oneof_descriptor : NULL; + ConstStringParam name) const { + return tables_->FindByNameHelper(this, name).oneof_descriptor(); } const EnumDescriptor* DescriptorPool::FindEnumTypeByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - return (result.type == Symbol::ENUM) ? result.enum_descriptor : NULL; + ConstStringParam name) const { + return tables_->FindByNameHelper(this, name).enum_descriptor(); } const EnumValueDescriptor* DescriptorPool::FindEnumValueByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - return (result.type == Symbol::ENUM_VALUE) ? - result.enum_value_descriptor : NULL; + ConstStringParam name) const { + return tables_->FindByNameHelper(this, name).enum_value_descriptor(); } const ServiceDescriptor* DescriptorPool::FindServiceByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - return (result.type == Symbol::SERVICE) ? result.service_descriptor : NULL; + ConstStringParam name) const { + return tables_->FindByNameHelper(this, name).service_descriptor(); } const MethodDescriptor* DescriptorPool::FindMethodByName( - const string& name) const { - Symbol result = tables_->FindByNameHelper(this, name); - return (result.type == Symbol::METHOD) ? result.method_descriptor : NULL; + ConstStringParam name) const { + return tables_->FindByNameHelper(this, name).method_descriptor(); } const FieldDescriptor* DescriptorPool::FindExtensionByNumber( const Descriptor* extendee, int number) const { + if (extendee->extension_range_count() == 0) return nullptr; // A faster path to reduce lock contention in finding extensions, assuming // most extensions will be cache hit. - if (mutex_ != NULL) { + if (mutex_ != nullptr) { ReaderMutexLock lock(mutex_); const FieldDescriptor* result = tables_->FindExtension(extendee, number); - if (result != NULL) { + if (result != nullptr) { return result; } } MutexLockMaybe lock(mutex_); - tables_->known_bad_symbols_.clear(); - tables_->known_bad_files_.clear(); + if (fallback_database_ != nullptr) { + tables_->known_bad_symbols_.clear(); + tables_->known_bad_files_.clear(); + } const FieldDescriptor* result = tables_->FindExtension(extendee, number); - if (result != NULL) { + if (result != nullptr) { return result; } - if (underlay_ != NULL) { + if (underlay_ != nullptr) { result = underlay_->FindExtensionByNumber(extendee, number); - if (result != NULL) return result; + if (result != nullptr) return result; } if (TryFindExtensionInFallbackDatabase(extendee, number)) { result = tables_->FindExtension(extendee, number); - if (result != NULL) { + if (result != nullptr) { return result; } } - return NULL; + return nullptr; +} + +const FieldDescriptor* DescriptorPool::InternalFindExtensionByNumberNoLock( + const Descriptor* extendee, int number) const { + if (extendee->extension_range_count() == 0) return nullptr; + + const FieldDescriptor* result = tables_->FindExtension(extendee, number); + if (result != nullptr) { + return result; + } + + if (underlay_ != nullptr) { + result = underlay_->InternalFindExtensionByNumberNoLock(extendee, number); + if (result != nullptr) return result; + } + + return nullptr; +} + +const FieldDescriptor* DescriptorPool::FindExtensionByPrintableName( + const Descriptor* extendee, ConstStringParam printable_name) const { + if (extendee->extension_range_count() == 0) return nullptr; + const FieldDescriptor* result = FindExtensionByName(printable_name); + if (result != nullptr && result->containing_type() == extendee) { + return result; + } + if (extendee->options().message_set_wire_format()) { + // MessageSet extensions may be identified by type name. + const Descriptor* type = FindMessageTypeByName(printable_name); + if (type != nullptr) { + // Look for a matching extension in the foreign type's scope. + const int type_extension_count = type->extension_count(); + for (int i = 0; i < type_extension_count; i++) { + const FieldDescriptor* extension = type->extension(i); + if (extension->containing_type() == extendee && + extension->type() == FieldDescriptor::TYPE_MESSAGE && + extension->is_optional() && extension->message_type() == type) { + // Found it. + return extension; + } + } + } + } + return nullptr; } void DescriptorPool::FindAllExtensions( const Descriptor* extendee, std::vector* out) const { MutexLockMaybe lock(mutex_); - tables_->known_bad_symbols_.clear(); - tables_->known_bad_files_.clear(); + if (fallback_database_ != nullptr) { + tables_->known_bad_symbols_.clear(); + tables_->known_bad_files_.clear(); + } // Initialize tables_->extensions_ from the fallback database first // (but do this only once per descriptor). - if (fallback_database_ != NULL && + if (fallback_database_ != nullptr && tables_->extensions_loaded_from_db_.count(extendee) == 0) { std::vector numbers; if (fallback_database_->FindAllExtensionNumbers(extendee->full_name(), &numbers)) { - for (int i = 0; i < numbers.size(); ++i) { - int number = numbers[i]; - if (tables_->FindExtension(extendee, number) == NULL) { + for (int number : numbers) { + if (tables_->FindExtension(extendee, number) == nullptr) { TryFindExtensionInFallbackDatabase(extendee, number); } } @@ -1552,7 +2225,7 @@ void DescriptorPool::FindAllExtensions( } tables_->FindAllExtensions(extendee, out); - if (underlay_ != NULL) { + if (underlay_ != nullptr) { underlay_->FindAllExtensions(extendee, out); } } @@ -1560,253 +2233,199 @@ void DescriptorPool::FindAllExtensions( // ------------------------------------------------------------------- -const FieldDescriptor* -Descriptor::FindFieldByNumber(int key) const { - const FieldDescriptor* result = - file()->tables_->FindFieldByNumber(this, key); - if (result == NULL || result->is_extension()) { - return NULL; +const FieldDescriptor* Descriptor::FindFieldByNumber(int key) const { + const FieldDescriptor* result = file()->tables_->FindFieldByNumber(this, key); + if (result == nullptr || result->is_extension()) { + return nullptr; } else { return result; } } -const FieldDescriptor* -Descriptor::FindFieldByLowercaseName(const string& key) const { +const FieldDescriptor* Descriptor::FindFieldByLowercaseName( + ConstStringParam key) const { const FieldDescriptor* result = - file()->tables_->FindFieldByLowercaseName(this, key); - if (result == NULL || result->is_extension()) { - return NULL; + file()->tables_->FindFieldByLowercaseName(this, key); + if (result == nullptr || result->is_extension()) { + return nullptr; } else { return result; } } -const FieldDescriptor* -Descriptor::FindFieldByCamelcaseName(const string& key) const { +const FieldDescriptor* Descriptor::FindFieldByCamelcaseName( + ConstStringParam key) const { const FieldDescriptor* result = - file()->tables_->FindFieldByCamelcaseName(this, key); - if (result == NULL || result->is_extension()) { - return NULL; + file()->tables_->FindFieldByCamelcaseName(this, key); + if (result == nullptr || result->is_extension()) { + return nullptr; } else { return result; } } -const FieldDescriptor* -Descriptor::FindFieldByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::FIELD); - if (!result.IsNull() && !result.field_descriptor->is_extension()) { - return result.field_descriptor; - } else { - return NULL; - } +const FieldDescriptor* Descriptor::FindFieldByName(ConstStringParam key) const { + const FieldDescriptor* field = + file()->tables_->FindNestedSymbol(this, key).field_descriptor(); + return field != nullptr && !field->is_extension() ? field : nullptr; } -const OneofDescriptor* -Descriptor::FindOneofByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::ONEOF); - if (!result.IsNull()) { - return result.oneof_descriptor; - } else { - return NULL; - } +const OneofDescriptor* Descriptor::FindOneofByName(ConstStringParam key) const { + return file()->tables_->FindNestedSymbol(this, key).oneof_descriptor(); } -const FieldDescriptor* -Descriptor::FindExtensionByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::FIELD); - if (!result.IsNull() && result.field_descriptor->is_extension()) { - return result.field_descriptor; - } else { - return NULL; - } +const FieldDescriptor* Descriptor::FindExtensionByName( + ConstStringParam key) const { + const FieldDescriptor* field = + file()->tables_->FindNestedSymbol(this, key).field_descriptor(); + return field != nullptr && field->is_extension() ? field : nullptr; } -const FieldDescriptor* -Descriptor::FindExtensionByLowercaseName(const string& key) const { +const FieldDescriptor* Descriptor::FindExtensionByLowercaseName( + ConstStringParam key) const { const FieldDescriptor* result = - file()->tables_->FindFieldByLowercaseName(this, key); - if (result == NULL || !result->is_extension()) { - return NULL; + file()->tables_->FindFieldByLowercaseName(this, key); + if (result == nullptr || !result->is_extension()) { + return nullptr; } else { return result; } } -const FieldDescriptor* -Descriptor::FindExtensionByCamelcaseName(const string& key) const { +const FieldDescriptor* Descriptor::FindExtensionByCamelcaseName( + ConstStringParam key) const { const FieldDescriptor* result = - file()->tables_->FindFieldByCamelcaseName(this, key); - if (result == NULL || !result->is_extension()) { - return NULL; + file()->tables_->FindFieldByCamelcaseName(this, key); + if (result == nullptr || !result->is_extension()) { + return nullptr; } else { return result; } } -const Descriptor* -Descriptor::FindNestedTypeByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::MESSAGE); - if (!result.IsNull()) { - return result.descriptor; - } else { - return NULL; - } +const Descriptor* Descriptor::FindNestedTypeByName(ConstStringParam key) const { + return file()->tables_->FindNestedSymbol(this, key).descriptor(); } -const EnumDescriptor* -Descriptor::FindEnumTypeByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::ENUM); - if (!result.IsNull()) { - return result.enum_descriptor; - } else { - return NULL; - } +const EnumDescriptor* Descriptor::FindEnumTypeByName( + ConstStringParam key) const { + return file()->tables_->FindNestedSymbol(this, key).enum_descriptor(); } -const EnumValueDescriptor* -Descriptor::FindEnumValueByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::ENUM_VALUE); - if (!result.IsNull()) { - return result.enum_value_descriptor; - } else { - return NULL; - } +const EnumValueDescriptor* Descriptor::FindEnumValueByName( + ConstStringParam key) const { + return file()->tables_->FindNestedSymbol(this, key).enum_value_descriptor(); } -const EnumValueDescriptor* -EnumDescriptor::FindValueByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::ENUM_VALUE); - if (!result.IsNull()) { - return result.enum_value_descriptor; - } else { - return NULL; - } +const FieldDescriptor* Descriptor::map_key() const { + if (!options().map_entry()) return nullptr; + GOOGLE_DCHECK_EQ(field_count(), 2); + return field(0); +} + +const FieldDescriptor* Descriptor::map_value() const { + if (!options().map_entry()) return nullptr; + GOOGLE_DCHECK_EQ(field_count(), 2); + return field(1); } -const EnumValueDescriptor* -EnumDescriptor::FindValueByNumber(int key) const { +const EnumValueDescriptor* EnumDescriptor::FindValueByName( + ConstStringParam key) const { + return file()->tables_->FindNestedSymbol(this, key).enum_value_descriptor(); +} + +const EnumValueDescriptor* EnumDescriptor::FindValueByNumber(int key) const { return file()->tables_->FindEnumValueByNumber(this, key); } -const EnumValueDescriptor* -EnumDescriptor::FindValueByNumberCreatingIfUnknown(int key) const { +const EnumValueDescriptor* EnumDescriptor::FindValueByNumberCreatingIfUnknown( + int key) const { return file()->tables_->FindEnumValueByNumberCreatingIfUnknown(this, key); } -const MethodDescriptor* -ServiceDescriptor::FindMethodByName(const string& key) const { - Symbol result = - file()->tables_->FindNestedSymbolOfType(this, key, Symbol::METHOD); - if (!result.IsNull()) { - return result.method_descriptor; - } else { - return NULL; - } +const MethodDescriptor* ServiceDescriptor::FindMethodByName( + ConstStringParam key) const { + return file()->tables_->FindNestedSymbol(this, key).method_descriptor(); } -const Descriptor* -FileDescriptor::FindMessageTypeByName(const string& key) const { - Symbol result = tables_->FindNestedSymbolOfType(this, key, Symbol::MESSAGE); - if (!result.IsNull()) { - return result.descriptor; - } else { - return NULL; - } +const Descriptor* FileDescriptor::FindMessageTypeByName( + ConstStringParam key) const { + return tables_->FindNestedSymbol(this, key).descriptor(); } -const EnumDescriptor* -FileDescriptor::FindEnumTypeByName(const string& key) const { - Symbol result = tables_->FindNestedSymbolOfType(this, key, Symbol::ENUM); - if (!result.IsNull()) { - return result.enum_descriptor; - } else { - return NULL; - } +const EnumDescriptor* FileDescriptor::FindEnumTypeByName( + ConstStringParam key) const { + return tables_->FindNestedSymbol(this, key).enum_descriptor(); } -const EnumValueDescriptor* -FileDescriptor::FindEnumValueByName(const string& key) const { - Symbol result = - tables_->FindNestedSymbolOfType(this, key, Symbol::ENUM_VALUE); - if (!result.IsNull()) { - return result.enum_value_descriptor; - } else { - return NULL; - } +const EnumValueDescriptor* FileDescriptor::FindEnumValueByName( + ConstStringParam key) const { + return tables_->FindNestedSymbol(this, key).enum_value_descriptor(); } -const ServiceDescriptor* -FileDescriptor::FindServiceByName(const string& key) const { - Symbol result = tables_->FindNestedSymbolOfType(this, key, Symbol::SERVICE); - if (!result.IsNull()) { - return result.service_descriptor; - } else { - return NULL; - } +const ServiceDescriptor* FileDescriptor::FindServiceByName( + ConstStringParam key) const { + return tables_->FindNestedSymbol(this, key).service_descriptor(); } -const FieldDescriptor* -FileDescriptor::FindExtensionByName(const string& key) const { - Symbol result = tables_->FindNestedSymbolOfType(this, key, Symbol::FIELD); - if (!result.IsNull() && result.field_descriptor->is_extension()) { - return result.field_descriptor; - } else { - return NULL; - } +const FieldDescriptor* FileDescriptor::FindExtensionByName( + ConstStringParam key) const { + const FieldDescriptor* field = + tables_->FindNestedSymbol(this, key).field_descriptor(); + return field != nullptr && field->is_extension() ? field : nullptr; } -const FieldDescriptor* -FileDescriptor::FindExtensionByLowercaseName(const string& key) const { +const FieldDescriptor* FileDescriptor::FindExtensionByLowercaseName( + ConstStringParam key) const { const FieldDescriptor* result = tables_->FindFieldByLowercaseName(this, key); - if (result == NULL || !result->is_extension()) { - return NULL; + if (result == nullptr || !result->is_extension()) { + return nullptr; } else { return result; } } -const FieldDescriptor* -FileDescriptor::FindExtensionByCamelcaseName(const string& key) const { +const FieldDescriptor* FileDescriptor::FindExtensionByCamelcaseName( + ConstStringParam key) const { const FieldDescriptor* result = tables_->FindFieldByCamelcaseName(this, key); - if (result == NULL || !result->is_extension()) { - return NULL; + if (result == nullptr || !result->is_extension()) { + return nullptr; } else { return result; } } +void Descriptor::ExtensionRange::CopyTo( + DescriptorProto_ExtensionRange* proto) const { + proto->set_start(this->start); + proto->set_end(this->end); + if (options_ != &ExtensionRangeOptions::default_instance()) { + *proto->mutable_options() = *options_; + } +} + const Descriptor::ExtensionRange* Descriptor::FindExtensionRangeContainingNumber(int number) const { // Linear search should be fine because we don't expect a message to have // more than a couple extension ranges. for (int i = 0; i < extension_range_count(); i++) { if (number >= extension_range(i)->start && - number < extension_range(i)->end) { + number < extension_range(i)->end) { return extension_range(i); } } - return NULL; + return nullptr; } -const Descriptor::ReservedRange* -Descriptor::FindReservedRangeContainingNumber(int number) const { +const Descriptor::ReservedRange* Descriptor::FindReservedRangeContainingNumber( + int number) const { // TODO(chrisn): Consider a non-linear search. for (int i = 0; i < reserved_range_count(); i++) { - if (number >= reserved_range(i)->start && - number < reserved_range(i)->end) { + if (number >= reserved_range(i)->start && number < reserved_range(i)->end) { return reserved_range(i); } } - return NULL; + return nullptr; } const EnumDescriptor::ReservedRange* @@ -1818,82 +2437,86 @@ EnumDescriptor::FindReservedRangeContainingNumber(int number) const { return reserved_range(i); } } - return NULL; + return nullptr; } // ------------------------------------------------------------------- -bool DescriptorPool::TryFindFileInFallbackDatabase(const string& name) const { - if (fallback_database_ == NULL) return false; +bool DescriptorPool::TryFindFileInFallbackDatabase( + StringPiece name) const { + if (fallback_database_ == nullptr) return false; - if (tables_->known_bad_files_.count(name) > 0) return false; + auto name_string = std::string(name); + if (tables_->known_bad_files_.count(name_string) > 0) return false; FileDescriptorProto file_proto; - if (!fallback_database_->FindFileByName(name, &file_proto) || - BuildFileFromDatabase(file_proto) == NULL) { - tables_->known_bad_files_.insert(name); + if (!fallback_database_->FindFileByName(name_string, &file_proto) || + BuildFileFromDatabase(file_proto) == nullptr) { + tables_->known_bad_files_.insert(std::move(name_string)); return false; } return true; } -bool DescriptorPool::IsSubSymbolOfBuiltType(const string& name) const { - string prefix = name; +bool DescriptorPool::IsSubSymbolOfBuiltType(StringPiece name) const { + auto prefix = std::string(name); for (;;) { - string::size_type dot_pos = prefix.find_last_of('.'); - if (dot_pos == string::npos) { + std::string::size_type dot_pos = prefix.find_last_of('.'); + if (dot_pos == std::string::npos) { break; } prefix = prefix.substr(0, dot_pos); Symbol symbol = tables_->FindSymbol(prefix); // If the symbol type is anything other than PACKAGE, then its complete // definition is already known. - if (!symbol.IsNull() && symbol.type != Symbol::PACKAGE) { + if (!symbol.IsNull() && symbol.type() != Symbol::PACKAGE) { return true; } } - if (underlay_ != NULL) { + if (underlay_ != nullptr) { // Check to see if any prefix of this symbol exists in the underlay. return underlay_->IsSubSymbolOfBuiltType(name); } return false; } -bool DescriptorPool::TryFindSymbolInFallbackDatabase(const string& name) const { - if (fallback_database_ == NULL) return false; +bool DescriptorPool::TryFindSymbolInFallbackDatabase( + StringPiece name) const { + if (fallback_database_ == nullptr) return false; - if (tables_->known_bad_symbols_.count(name) > 0) return false; + auto name_string = std::string(name); + if (tables_->known_bad_symbols_.count(name_string) > 0) return false; FileDescriptorProto file_proto; - if (// We skip looking in the fallback database if the name is a sub-symbol - // of any descriptor that already exists in the descriptor pool (except - // for package descriptors). This is valid because all symbols except - // for packages are defined in a single file, so if the symbol exists - // then we should already have its definition. - // - // The other reason to do this is to support "overriding" type - // definitions by merging two databases that define the same type. (Yes, - // people do this.) The main difficulty with making this work is that - // FindFileContainingSymbol() is allowed to return both false positives - // (e.g., SimpleDescriptorDatabase, UpgradedDescriptorDatabase) and false - // negatives (e.g. ProtoFileParser, SourceTreeDescriptorDatabase). - // When two such databases are merged, looking up a non-existent - // sub-symbol of a type that already exists in the descriptor pool can - // result in an attempt to load multiple definitions of the same type. - // The check below avoids this. + if ( // We skip looking in the fallback database if the name is a sub-symbol + // of any descriptor that already exists in the descriptor pool (except + // for package descriptors). This is valid because all symbols except + // for packages are defined in a single file, so if the symbol exists + // then we should already have its definition. + // + // The other reason to do this is to support "overriding" type + // definitions by merging two databases that define the same type. (Yes, + // people do this.) The main difficulty with making this work is that + // FindFileContainingSymbol() is allowed to return both false positives + // (e.g., SimpleDescriptorDatabase, UpgradedDescriptorDatabase) and + // false negatives (e.g. ProtoFileParser, SourceTreeDescriptorDatabase). + // When two such databases are merged, looking up a non-existent + // sub-symbol of a type that already exists in the descriptor pool can + // result in an attempt to load multiple definitions of the same type. + // The check below avoids this. IsSubSymbolOfBuiltType(name) // Look up file containing this symbol in fallback database. - || !fallback_database_->FindFileContainingSymbol(name, &file_proto) + || !fallback_database_->FindFileContainingSymbol(name_string, &file_proto) // Check if we've already built this file. If so, it apparently doesn't // contain the symbol we're looking for. Some DescriptorDatabases // return false positives. - || tables_->FindFile(file_proto.name()) != NULL + || tables_->FindFile(file_proto.name()) != nullptr // Build the file. - || BuildFileFromDatabase(file_proto) == NULL) { - tables_->known_bad_symbols_.insert(name); + || BuildFileFromDatabase(file_proto) == nullptr) { + tables_->known_bad_symbols_.insert(std::move(name_string)); return false; } @@ -1902,22 +2525,22 @@ bool DescriptorPool::TryFindSymbolInFallbackDatabase(const string& name) const { bool DescriptorPool::TryFindExtensionInFallbackDatabase( const Descriptor* containing_type, int field_number) const { - if (fallback_database_ == NULL) return false; + if (fallback_database_ == nullptr) return false; FileDescriptorProto file_proto; if (!fallback_database_->FindFileContainingExtension( - containing_type->full_name(), field_number, &file_proto)) { + containing_type->full_name(), field_number, &file_proto)) { return false; } - if (tables_->FindFile(file_proto.name()) != NULL) { + if (tables_->FindFile(file_proto.name()) != nullptr) { // We've already loaded this file, and it apparently doesn't contain the // extension we're looking for. Some DescriptorDatabases return false // positives. return false; } - if (BuildFileFromDatabase(file_proto) == NULL) { + if (BuildFileFromDatabase(file_proto) == nullptr) { return false; } @@ -1927,33 +2550,27 @@ bool DescriptorPool::TryFindExtensionInFallbackDatabase( // =================================================================== bool FieldDescriptor::is_map_message_type() const { - return message_type_->options().map_entry(); + return type_descriptor_.message_type->options().map_entry(); } -string FieldDescriptor::DefaultValueAsString(bool quote_string_type) const { +std::string FieldDescriptor::DefaultValueAsString( + bool quote_string_type) const { GOOGLE_CHECK(has_default_value()) << "No default value"; switch (cpp_type()) { case CPPTYPE_INT32: - return SimpleItoa(default_value_int32()); - break; + return StrCat(default_value_int32_t()); case CPPTYPE_INT64: - return SimpleItoa(default_value_int64()); - break; + return StrCat(default_value_int64_t()); case CPPTYPE_UINT32: - return SimpleItoa(default_value_uint32()); - break; + return StrCat(default_value_uint32_t()); case CPPTYPE_UINT64: - return SimpleItoa(default_value_uint64()); - break; + return StrCat(default_value_uint64_t()); case CPPTYPE_FLOAT: return SimpleFtoa(default_value_float()); - break; case CPPTYPE_DOUBLE: return SimpleDtoa(default_value_double()); - break; case CPPTYPE_BOOL: return default_value_bool() ? "true" : "false"; - break; case CPPTYPE_STRING: if (quote_string_type) { return "\"" + CEscape(default_value_string()) + "\""; @@ -1964,10 +2581,8 @@ string FieldDescriptor::DefaultValueAsString(bool quote_string_type) const { return default_value_string(); } } - break; case CPPTYPE_ENUM: return default_value_enum()->name(); - break; case CPPTYPE_MESSAGE: GOOGLE_LOG(DFATAL) << "Messages can't have default values!"; break; @@ -2051,13 +2666,7 @@ void Descriptor::CopyTo(DescriptorProto* proto) const { enum_type(i)->CopyTo(proto->add_enum_type()); } for (int i = 0; i < extension_range_count(); i++) { - DescriptorProto::ExtensionRange* range = proto->add_extension_range(); - range->set_start(extension_range(i)->start); - range->set_end(extension_range(i)->end); - const ExtensionRangeOptions* options = extension_range(i)->options_; - if (options != &ExtensionRangeOptions::default_instance()) { - range->mutable_options()->CopyFrom(*options); - } + extension_range(i)->CopyTo(proto->add_extension_range()); } for (int i = 0; i < extension_count(); i++) { extension(i)->CopyTo(proto->add_extension()); @@ -2100,13 +2709,15 @@ void FieldDescriptor::CopyTo(FieldDescriptorProto* proto) const { if (has_json_name_) { proto->set_json_name(json_name()); } - + if (proto3_optional_) { + proto->set_proto3_optional(true); + } // Some compilers do not allow static_cast directly between two enum types, // so we must cast to int first. proto->set_label(static_cast( - implicit_cast(label()))); + implicit_cast(label()))); proto->set_type(static_cast( - implicit_cast(type()))); + implicit_cast(type()))); if (is_extension()) { if (!containing_type()->is_unqualified_placeholder_) { @@ -2137,7 +2748,7 @@ void FieldDescriptor::CopyTo(FieldDescriptorProto* proto) const { proto->set_default_value(DefaultValueAsString(false)); } - if (containing_oneof() != NULL && !is_extension()) { + if (containing_oneof() != nullptr && !is_extension()) { proto->set_oneof_index(containing_oneof()->index()); } @@ -2227,40 +2838,42 @@ void MethodDescriptor::CopyTo(MethodDescriptorProto* proto) const { namespace { -bool RetrieveOptionsAssumingRightPool(int depth, const Message& options, - std::vector* option_entries) { +bool RetrieveOptionsAssumingRightPool( + int depth, const Message& options, + std::vector* option_entries) { option_entries->clear(); const Reflection* reflection = options.GetReflection(); std::vector fields; reflection->ListFields(options, &fields); - for (int i = 0; i < fields.size(); i++) { + for (const FieldDescriptor* field : fields) { int count = 1; bool repeated = false; - if (fields[i]->is_repeated()) { - count = reflection->FieldSize(options, fields[i]); + if (field->is_repeated()) { + count = reflection->FieldSize(options, field); repeated = true; } for (int j = 0; j < count; j++) { - string fieldval; - if (fields[i]->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { - string tmp; + std::string fieldval; + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + std::string tmp; TextFormat::Printer printer; + printer.SetExpandAny(true); printer.SetInitialIndentLevel(depth + 1); - printer.PrintFieldValueToString(options, fields[i], - repeated ? j : -1, &tmp); + printer.PrintFieldValueToString(options, field, repeated ? j : -1, + &tmp); fieldval.append("{\n"); fieldval.append(tmp); fieldval.append(depth * 2, ' '); fieldval.append("}"); } else { - TextFormat::PrintFieldValueToString(options, fields[i], - repeated ? j : -1, &fieldval); + TextFormat::PrintFieldValueToString(options, field, repeated ? j : -1, + &fieldval); } - string name; - if (fields[i]->is_extension()) { - name = "(." + fields[i]->full_name() + ")"; + std::string name; + if (field->is_extension()) { + name = "(." + field->full_name() + ")"; } else { - name = fields[i]->name(); + name = field->name(); } option_entries->push_back(name + " = " + fieldval); } @@ -2271,7 +2884,7 @@ bool RetrieveOptionsAssumingRightPool(int depth, const Message& options, // Used by each of the option formatters. bool RetrieveOptions(int depth, const Message& options, const DescriptorPool* pool, - std::vector* option_entries) { + std::vector* option_entries) { // When printing custom options for a descriptor, we must use an options // message built on top of the same DescriptorPool where the descriptor // is coming from. This is to ensure we are interpreting custom options @@ -2281,16 +2894,20 @@ bool RetrieveOptions(int depth, const Message& options, } else { const Descriptor* option_descriptor = pool->FindMessageTypeByName(options.GetDescriptor()->full_name()); - if (option_descriptor == NULL) { - // google/protobuf/descriptor.proto is not in the pool. This means no - // custom options are used so we are safe to proceed with the compiled - // options message type. + if (option_descriptor == nullptr) { + // descriptor.proto is not in the pool. This means no custom options are + // used so we are safe to proceed with the compiled options message type. return RetrieveOptionsAssumingRightPool(depth, options, option_entries); } DynamicMessageFactory factory; - google::protobuf::scoped_ptr dynamic_options( + std::unique_ptr dynamic_options( factory.GetPrototype(option_descriptor)->New()); - if (dynamic_options->ParseFromString(options.SerializeAsString())) { + std::string serialized = options.SerializeAsString(); + io::CodedInputStream input( + reinterpret_cast(serialized.c_str()), + serialized.size()); + input.SetExtensionRegistry(pool, &factory); + if (dynamic_options->ParseFromCodedStream(&input)) { return RetrieveOptionsAssumingRightPool(depth, *dynamic_options, option_entries); } else { @@ -2304,8 +2921,8 @@ bool RetrieveOptions(int depth, const Message& options, // Formats options that all appear together in brackets. Does not include // brackets. bool FormatBracketedOptions(int depth, const Message& options, - const DescriptorPool* pool, string* output) { - std::vector all_options; + const DescriptorPool* pool, std::string* output) { + std::vector all_options; if (RetrieveOptions(depth, options, pool, &all_options)) { output->append(Join(all_options, ", ")); } @@ -2314,13 +2931,12 @@ bool FormatBracketedOptions(int depth, const Message& options, // Formats options one per line bool FormatLineOptions(int depth, const Message& options, - const DescriptorPool* pool, string* output) { - string prefix(depth * 2, ' '); - std::vector all_options; + const DescriptorPool* pool, std::string* output) { + std::string prefix(depth * 2, ' '); + std::vector all_options; if (RetrieveOptions(depth, options, pool, &all_options)) { - for (int i = 0; i < all_options.size(); i++) { - strings::SubstituteAndAppend(output, "$0option $1;\n", - prefix, all_options[i]); + for (const std::string& option : all_options) { + strings::SubstituteAndAppend(output, "$0option $1;\n", prefix, option); } } return !all_options.empty(); @@ -2328,31 +2944,31 @@ bool FormatLineOptions(int depth, const Message& options, class SourceLocationCommentPrinter { public: - template - SourceLocationCommentPrinter(const DescType* desc, - const string& prefix, + template + SourceLocationCommentPrinter(const DescType* desc, const std::string& prefix, const DebugStringOptions& options) : options_(options), prefix_(prefix) { // Perform the SourceLocation lookup only if we're including user comments, // because the lookup is fairly expensive. - have_source_loc_ = options.include_comments && - desc->GetSourceLocation(&source_loc_); + have_source_loc_ = + options.include_comments && desc->GetSourceLocation(&source_loc_); } SourceLocationCommentPrinter(const FileDescriptor* file, const std::vector& path, - const string& prefix, + const std::string& prefix, const DebugStringOptions& options) : options_(options), prefix_(prefix) { // Perform the SourceLocation lookup only if we're including user comments, // because the lookup is fairly expensive. - have_source_loc_ = options.include_comments && - file->GetSourceLocation(path, &source_loc_); + have_source_loc_ = + options.include_comments && file->GetSourceLocation(path, &source_loc_); } - void AddPreComment(string* output) { + void AddPreComment(std::string* output) { if (have_source_loc_) { // Detached leading comments. - for (int i = 0 ; i < source_loc_.leading_detached_comments.size(); ++i) { - *output += FormatComment(source_loc_.leading_detached_comments[i]); + for (const std::string& leading_detached_comment : + source_loc_.leading_detached_comments) { + *output += FormatComment(leading_detached_comment); *output += "\n"; } // Attached leading comments. @@ -2361,7 +2977,7 @@ class SourceLocationCommentPrinter { } } } - void AddPostComment(string* output) { + void AddPostComment(std::string* output) { if (have_source_loc_ && source_loc_.trailing_comments.size() > 0) { *output += FormatComment(source_loc_.trailing_comments); } @@ -2369,13 +2985,12 @@ class SourceLocationCommentPrinter { // Format comment such that each line becomes a full-line C++-style comment in // the DebugString() output. - string FormatComment(const string& comment_text) { - string stripped_comment = comment_text; + std::string FormatComment(const std::string& comment_text) { + std::string stripped_comment = comment_text; StripWhitespace(&stripped_comment); - std::vector lines = Split(stripped_comment, "\n"); - string output; - for (int i = 0; i < lines.size(); ++i) { - const string& line = lines[i]; + std::vector lines = Split(stripped_comment, "\n"); + std::string output; + for (const std::string& line : lines) { strings::SubstituteAndAppend(&output, "$0// $1\n", prefix_, line); } return output; @@ -2386,32 +3001,31 @@ class SourceLocationCommentPrinter { bool have_source_loc_; SourceLocation source_loc_; DebugStringOptions options_; - string prefix_; + std::string prefix_; }; } // anonymous namespace -string FileDescriptor::DebugString() const { +std::string FileDescriptor::DebugString() const { DebugStringOptions options; // default options return DebugStringWithOptions(options); } -string FileDescriptor::DebugStringWithOptions( +std::string FileDescriptor::DebugStringWithOptions( const DebugStringOptions& debug_string_options) const { - string contents; + std::string contents; { std::vector path; path.push_back(FileDescriptorProto::kSyntaxFieldNumber); - SourceLocationCommentPrinter syntax_comment( - this, path, "", debug_string_options); + SourceLocationCommentPrinter syntax_comment(this, path, "", + debug_string_options); syntax_comment.AddPreComment(&contents); strings::SubstituteAndAppend(&contents, "syntax = \"$0\";\n\n", - SyntaxName(syntax())); + SyntaxName(syntax())); syntax_comment.AddPostComment(&contents); } - SourceLocationCommentPrinter - comment_printer(this, "", debug_string_options); + SourceLocationCommentPrinter comment_printer(this, "", debug_string_options); comment_printer.AddPreComment(&contents); std::set public_dependencies; @@ -2424,21 +3038,21 @@ string FileDescriptor::DebugStringWithOptions( for (int i = 0; i < dependency_count(); i++) { if (public_dependencies.count(i) > 0) { strings::SubstituteAndAppend(&contents, "import public \"$0\";\n", - dependency(i)->name()); + dependency(i)->name()); } else if (weak_dependencies.count(i) > 0) { strings::SubstituteAndAppend(&contents, "import weak \"$0\";\n", - dependency(i)->name()); + dependency(i)->name()); } else { strings::SubstituteAndAppend(&contents, "import \"$0\";\n", - dependency(i)->name()); + dependency(i)->name()); } } if (!package().empty()) { std::vector path; path.push_back(FileDescriptorProto::kPackageFieldNumber); - SourceLocationCommentPrinter package_comment( - this, path, "", debug_string_options); + SourceLocationCommentPrinter package_comment(this, path, "", + debug_string_options); package_comment.AddPreComment(&contents); strings::SubstituteAndAppend(&contents, "package $0;\n\n", package()); package_comment.AddPostComment(&contents); @@ -2475,16 +3089,15 @@ string FileDescriptor::DebugStringWithOptions( contents.append("\n"); } - const Descriptor* containing_type = NULL; + const Descriptor* containing_type = nullptr; for (int i = 0; i < extension_count(); i++) { if (extension(i)->containing_type() != containing_type) { if (i > 0) contents.append("}\n\n"); containing_type = extension(i)->containing_type(); strings::SubstituteAndAppend(&contents, "extend .$0 {\n", - containing_type->full_name()); + containing_type->full_name()); } - extension(i)->DebugString(1, FieldDescriptor::PRINT_LABEL, &contents, - debug_string_options); + extension(i)->DebugString(1, &contents, debug_string_options); } if (extension_count() > 0) contents.append("}\n\n"); @@ -2493,31 +3106,30 @@ string FileDescriptor::DebugStringWithOptions( return contents; } -string Descriptor::DebugString() const { +std::string Descriptor::DebugString() const { DebugStringOptions options; // default options return DebugStringWithOptions(options); } -string Descriptor::DebugStringWithOptions( +std::string Descriptor::DebugStringWithOptions( const DebugStringOptions& options) const { - string contents; + std::string contents; DebugString(0, &contents, options, /* include_opening_clause */ true); return contents; } -void Descriptor::DebugString(int depth, string *contents, - const DebugStringOptions& - debug_string_options, +void Descriptor::DebugString(int depth, std::string* contents, + const DebugStringOptions& debug_string_options, bool include_opening_clause) const { if (options().map_entry()) { // Do not generate debug string for auto-generated map-entry type. return; } - string prefix(depth * 2, ' '); + std::string prefix(depth * 2, ' '); ++depth; - SourceLocationCommentPrinter - comment_printer(this, prefix, debug_string_options); + SourceLocationCommentPrinter comment_printer(this, prefix, + debug_string_options); comment_printer.AddPreComment(contents); if (include_opening_clause) { @@ -2552,9 +3164,8 @@ void Descriptor::DebugString(int depth, string *contents, enum_type(i)->DebugString(depth, contents, debug_string_options); } for (int i = 0; i < field_count(); i++) { - if (field(i)->containing_oneof() == NULL) { - field(i)->DebugString(depth, FieldDescriptor::PRINT_LABEL, contents, - debug_string_options); + if (field(i)->real_containing_oneof() == nullptr) { + field(i)->DebugString(depth, contents, debug_string_options); } else if (field(i)->containing_oneof()->field(0) == field(i)) { // This is the first field in this oneof, so print the whole oneof. field(i)->containing_oneof()->DebugString(depth, contents, @@ -2563,24 +3174,21 @@ void Descriptor::DebugString(int depth, string *contents, } for (int i = 0; i < extension_range_count(); i++) { - strings::SubstituteAndAppend(contents, "$0 extensions $1 to $2;\n", - prefix, - extension_range(i)->start, - extension_range(i)->end - 1); + strings::SubstituteAndAppend(contents, "$0 extensions $1 to $2;\n", prefix, + extension_range(i)->start, + extension_range(i)->end - 1); } // Group extensions by what they extend, so they can be printed out together. - const Descriptor* containing_type = NULL; + const Descriptor* containing_type = nullptr; for (int i = 0; i < extension_count(); i++) { if (extension(i)->containing_type() != containing_type) { if (i > 0) strings::SubstituteAndAppend(contents, "$0 }\n", prefix); containing_type = extension(i)->containing_type(); - strings::SubstituteAndAppend(contents, "$0 extend .$1 {\n", - prefix, containing_type->full_name()); + strings::SubstituteAndAppend(contents, "$0 extend .$1 {\n", prefix, + containing_type->full_name()); } - extension(i)->DebugString( - depth + 1, FieldDescriptor::PRINT_LABEL, contents, - debug_string_options); + extension(i)->DebugString(depth + 1, contents, debug_string_options); } if (extension_count() > 0) strings::SubstituteAndAppend(contents, "$0 }\n", prefix); @@ -2591,9 +3199,11 @@ void Descriptor::DebugString(int depth, string *contents, const Descriptor::ReservedRange* range = reserved_range(i); if (range->end == range->start + 1) { strings::SubstituteAndAppend(contents, "$0, ", range->start); + } else if (range->end > FieldDescriptor::kMaxNumber) { + strings::SubstituteAndAppend(contents, "$0 to max, ", range->start); } else { - strings::SubstituteAndAppend(contents, "$0 to $1, ", - range->start, range->end - 1); + strings::SubstituteAndAppend(contents, "$0 to $1, ", range->start, + range->end - 1); } } contents->replace(contents->size() - 2, 2, ";\n"); @@ -2603,7 +3213,7 @@ void Descriptor::DebugString(int depth, string *contents, strings::SubstituteAndAppend(contents, "$0 reserved ", prefix); for (int i = 0; i < reserved_name_count(); i++) { strings::SubstituteAndAppend(contents, "\"$0\", ", - CEscape(reserved_name(i))); + CEscape(reserved_name(i))); } contents->replace(contents->size() - 2, 2, ";\n"); } @@ -2612,21 +3222,21 @@ void Descriptor::DebugString(int depth, string *contents, comment_printer.AddPostComment(contents); } -string FieldDescriptor::DebugString() const { +std::string FieldDescriptor::DebugString() const { DebugStringOptions options; // default options return DebugStringWithOptions(options); } -string FieldDescriptor::DebugStringWithOptions( +std::string FieldDescriptor::DebugStringWithOptions( const DebugStringOptions& debug_string_options) const { - string contents; + std::string contents; int depth = 0; if (is_extension()) { strings::SubstituteAndAppend(&contents, "extend .$0 {\n", - containing_type()->full_name()); + containing_type()->full_name()); depth = 1; } - DebugString(depth, PRINT_LABEL, &contents, debug_string_options); + DebugString(depth, &contents, debug_string_options); if (is_extension()) { contents.append("}\n"); } @@ -2634,8 +3244,8 @@ string FieldDescriptor::DebugStringWithOptions( } // The field type string used in FieldDescriptor::DebugString() -string FieldDescriptor::FieldTypeNameDebugString() const { - switch(type()) { +std::string FieldDescriptor::FieldTypeNameDebugString() const { + switch (type()) { case TYPE_MESSAGE: return "." + message_type()->full_name(); case TYPE_ENUM: @@ -2645,13 +3255,11 @@ string FieldDescriptor::FieldTypeNameDebugString() const { } } -void FieldDescriptor::DebugString(int depth, - PrintLabelFlag print_label_flag, - string *contents, - const DebugStringOptions& - debug_string_options) const { - string prefix(depth * 2, ' '); - string field_type; +void FieldDescriptor::DebugString( + int depth, std::string* contents, + const DebugStringOptions& debug_string_options) const { + std::string prefix(depth * 2, ' '); + std::string field_type; // Special case map fields. if (is_map()) { @@ -2663,44 +3271,32 @@ void FieldDescriptor::DebugString(int depth, field_type = FieldTypeNameDebugString(); } - bool print_label = true; - // Determine whether to omit label: - // 1. For an optional field, omit label if it's in oneof or in proto3. - // 2. For a repeated field, omit label if it's a map. - if (is_optional() && (print_label_flag == OMIT_LABEL || - file()->syntax() == FileDescriptor::SYNTAX_PROTO3)) { - print_label = false; - } else if (is_map()) { - print_label = false; - } - string label; - if (print_label) { - label = kLabelToName[this->label()]; - label.push_back(' '); + std::string label = StrCat(kLabelToName[this->label()], " "); + + // Label is omitted for maps, oneof, and plain proto3 fields. + if (is_map() || real_containing_oneof() || + (is_optional() && !has_optional_keyword())) { + label.clear(); } - SourceLocationCommentPrinter - comment_printer(this, prefix, debug_string_options); + SourceLocationCommentPrinter comment_printer(this, prefix, + debug_string_options); comment_printer.AddPreComment(contents); - strings::SubstituteAndAppend(contents, "$0$1$2 $3 = $4", - prefix, - label, - field_type, - type() == TYPE_GROUP ? message_type()->name() : - name(), - number()); + strings::SubstituteAndAppend( + contents, "$0$1$2 $3 = $4", prefix, label, field_type, + type() == TYPE_GROUP ? message_type()->name() : name(), number()); bool bracketed = false; if (has_default_value()) { bracketed = true; strings::SubstituteAndAppend(contents, " [default = $0", - DefaultValueAsString(true)); + DefaultValueAsString(true)); } if (has_json_name_) { if (!bracketed) { bracketed = true; - contents->append("["); + contents->append(" ["); } else { contents->append(", "); } @@ -2709,7 +3305,7 @@ void FieldDescriptor::DebugString(int depth, contents->append("\""); } - string formatted_options; + std::string formatted_options; if (FormatBracketedOptions(depth, options(), file()->pool(), &formatted_options)) { contents->append(bracketed ? ", " : " ["); @@ -2735,25 +3331,25 @@ void FieldDescriptor::DebugString(int depth, comment_printer.AddPostComment(contents); } -string OneofDescriptor::DebugString() const { +std::string OneofDescriptor::DebugString() const { DebugStringOptions options; // default values return DebugStringWithOptions(options); } -string OneofDescriptor::DebugStringWithOptions( +std::string OneofDescriptor::DebugStringWithOptions( const DebugStringOptions& options) const { - string contents; + std::string contents; DebugString(0, &contents, options); return contents; } -void OneofDescriptor::DebugString(int depth, string* contents, - const DebugStringOptions& - debug_string_options) const { - string prefix(depth * 2, ' '); +void OneofDescriptor::DebugString( + int depth, std::string* contents, + const DebugStringOptions& debug_string_options) const { + std::string prefix(depth * 2, ' '); ++depth; - SourceLocationCommentPrinter - comment_printer(this, prefix, debug_string_options); + SourceLocationCommentPrinter comment_printer(this, prefix, + debug_string_options); comment_printer.AddPreComment(contents); strings::SubstituteAndAppend(contents, "$0oneof $1 {", prefix, name()); @@ -2765,38 +3361,36 @@ void OneofDescriptor::DebugString(int depth, string* contents, } else { contents->append("\n"); for (int i = 0; i < field_count(); i++) { - field(i)->DebugString(depth, FieldDescriptor::OMIT_LABEL, contents, - debug_string_options); + field(i)->DebugString(depth, contents, debug_string_options); } strings::SubstituteAndAppend(contents, "$0}\n", prefix); } comment_printer.AddPostComment(contents); } -string EnumDescriptor::DebugString() const { +std::string EnumDescriptor::DebugString() const { DebugStringOptions options; // default values return DebugStringWithOptions(options); } -string EnumDescriptor::DebugStringWithOptions( +std::string EnumDescriptor::DebugStringWithOptions( const DebugStringOptions& options) const { - string contents; + std::string contents; DebugString(0, &contents, options); return contents; } -void EnumDescriptor::DebugString(int depth, string *contents, - const DebugStringOptions& - debug_string_options) const { - string prefix(depth * 2, ' '); +void EnumDescriptor::DebugString( + int depth, std::string* contents, + const DebugStringOptions& debug_string_options) const { + std::string prefix(depth * 2, ' '); ++depth; - SourceLocationCommentPrinter - comment_printer(this, prefix, debug_string_options); + SourceLocationCommentPrinter comment_printer(this, prefix, + debug_string_options); comment_printer.AddPreComment(contents); - strings::SubstituteAndAppend(contents, "$0enum $1 {\n", - prefix, name()); + strings::SubstituteAndAppend(contents, "$0enum $1 {\n", prefix, name()); FormatLineOptions(depth, options(), file()->pool(), contents); @@ -2810,9 +3404,11 @@ void EnumDescriptor::DebugString(int depth, string *contents, const EnumDescriptor::ReservedRange* range = reserved_range(i); if (range->end == range->start) { strings::SubstituteAndAppend(contents, "$0, ", range->start); + } else if (range->end == INT_MAX) { + strings::SubstituteAndAppend(contents, "$0 to max, ", range->start); } else { - strings::SubstituteAndAppend(contents, "$0 to $1, ", - range->start, range->end); + strings::SubstituteAndAppend(contents, "$0 to $1, ", range->start, + range->end); } } contents->replace(contents->size() - 2, 2, ";\n"); @@ -2822,7 +3418,7 @@ void EnumDescriptor::DebugString(int depth, string *contents, strings::SubstituteAndAppend(contents, "$0 reserved ", prefix); for (int i = 0; i < reserved_name_count(); i++) { strings::SubstituteAndAppend(contents, "\"$0\", ", - CEscape(reserved_name(i))); + CEscape(reserved_name(i))); } contents->replace(contents->size() - 2, 2, ";\n"); } @@ -2832,31 +3428,30 @@ void EnumDescriptor::DebugString(int depth, string *contents, comment_printer.AddPostComment(contents); } -string EnumValueDescriptor::DebugString() const { +std::string EnumValueDescriptor::DebugString() const { DebugStringOptions options; // default values return DebugStringWithOptions(options); } -string EnumValueDescriptor::DebugStringWithOptions( +std::string EnumValueDescriptor::DebugStringWithOptions( const DebugStringOptions& options) const { - string contents; + std::string contents; DebugString(0, &contents, options); return contents; } -void EnumValueDescriptor::DebugString(int depth, string *contents, - const DebugStringOptions& - debug_string_options) const { - string prefix(depth * 2, ' '); +void EnumValueDescriptor::DebugString( + int depth, std::string* contents, + const DebugStringOptions& debug_string_options) const { + std::string prefix(depth * 2, ' '); - SourceLocationCommentPrinter - comment_printer(this, prefix, debug_string_options); + SourceLocationCommentPrinter comment_printer(this, prefix, + debug_string_options); comment_printer.AddPreComment(contents); - strings::SubstituteAndAppend(contents, "$0$1 = $2", - prefix, name(), number()); + strings::SubstituteAndAppend(contents, "$0$1 = $2", prefix, name(), number()); - string formatted_options; + std::string formatted_options; if (FormatBracketedOptions(depth, options(), type()->file()->pool(), &formatted_options)) { strings::SubstituteAndAppend(contents, " [$0]", formatted_options); @@ -2866,23 +3461,23 @@ void EnumValueDescriptor::DebugString(int depth, string *contents, comment_printer.AddPostComment(contents); } -string ServiceDescriptor::DebugString() const { +std::string ServiceDescriptor::DebugString() const { DebugStringOptions options; // default values return DebugStringWithOptions(options); } -string ServiceDescriptor::DebugStringWithOptions( +std::string ServiceDescriptor::DebugStringWithOptions( const DebugStringOptions& options) const { - string contents; + std::string contents; DebugString(&contents, options); return contents; } -void ServiceDescriptor::DebugString(string *contents, - const DebugStringOptions& - debug_string_options) const { - SourceLocationCommentPrinter - comment_printer(this, /* prefix */ "", debug_string_options); +void ServiceDescriptor::DebugString( + std::string* contents, + const DebugStringOptions& debug_string_options) const { + SourceLocationCommentPrinter comment_printer(this, /* prefix */ "", + debug_string_options); comment_printer.AddPreComment(contents); strings::SubstituteAndAppend(contents, "service $0 {\n", name()); @@ -2898,40 +3493,38 @@ void ServiceDescriptor::DebugString(string *contents, comment_printer.AddPostComment(contents); } -string MethodDescriptor::DebugString() const { +std::string MethodDescriptor::DebugString() const { DebugStringOptions options; // default values return DebugStringWithOptions(options); } -string MethodDescriptor::DebugStringWithOptions( +std::string MethodDescriptor::DebugStringWithOptions( const DebugStringOptions& options) const { - string contents; + std::string contents; DebugString(0, &contents, options); return contents; } -void MethodDescriptor::DebugString(int depth, string *contents, - const DebugStringOptions& - debug_string_options) const { - string prefix(depth * 2, ' '); +void MethodDescriptor::DebugString( + int depth, std::string* contents, + const DebugStringOptions& debug_string_options) const { + std::string prefix(depth * 2, ' '); ++depth; - SourceLocationCommentPrinter - comment_printer(this, prefix, debug_string_options); + SourceLocationCommentPrinter comment_printer(this, prefix, + debug_string_options); comment_printer.AddPreComment(contents); - strings::SubstituteAndAppend(contents, "$0rpc $1($4.$2) returns ($5.$3)", - prefix, name(), - input_type()->full_name(), - output_type()->full_name(), - client_streaming() ? "stream " : "", - server_streaming() ? "stream " : ""); + strings::SubstituteAndAppend( + contents, "$0rpc $1($4.$2) returns ($5.$3)", prefix, name(), + input_type()->full_name(), output_type()->full_name(), + client_streaming() ? "stream " : "", server_streaming() ? "stream " : ""); - string formatted_options; + std::string formatted_options; if (FormatLineOptions(depth, options(), service()->file()->pool(), &formatted_options)) { - strings::SubstituteAndAppend(contents, " {\n$0$1}\n", - formatted_options, prefix); + strings::SubstituteAndAppend(contents, " {\n$0$1}\n", formatted_options, + prefix); } else { contents->append(";\n"); } @@ -2944,16 +3537,16 @@ void MethodDescriptor::DebugString(int depth, string *contents, bool FileDescriptor::GetSourceLocation(const std::vector& path, SourceLocation* out_location) const { - GOOGLE_CHECK_NOTNULL(out_location); + GOOGLE_CHECK(out_location != nullptr); if (source_code_info_) { if (const SourceCodeInfo_Location* loc = - tables_->GetSourceLocation(path, source_code_info_)) { - const RepeatedField& span = loc->span(); + tables_->GetSourceLocation(path, source_code_info_)) { + const RepeatedField& span = loc->span(); if (span.size() == 3 || span.size() == 4) { - out_location->start_line = span.Get(0); + out_location->start_line = span.Get(0); out_location->start_column = span.Get(1); - out_location->end_line = span.Get(span.size() == 3 ? 0 : 2); - out_location->end_column = span.Get(span.size() - 1); + out_location->end_line = span.Get(span.size() == 3 ? 0 : 2); + out_location->end_column = span.Get(span.size() - 1); out_location->leading_comments = loc->leading_comments(); out_location->trailing_comments = loc->trailing_comments(); @@ -2975,9 +3568,9 @@ bool FileDescriptor::GetSourceLocation(SourceLocation* out_location) const { bool FieldDescriptor::is_packed() const { if (!is_packable()) return false; if (file_->syntax() == FileDescriptor::SYNTAX_PROTO2) { - return (options_ != NULL) && options_->packed(); + return (options_ != nullptr) && options_->packed(); } else { - return options_ == NULL || !options_->has_packed() || options_->packed(); + return options_ == nullptr || !options_->has_packed() || options_->packed(); } } @@ -3037,7 +3630,7 @@ void Descriptor::GetLocationPath(std::vector* output) const { void FieldDescriptor::GetLocationPath(std::vector* output) const { if (is_extension()) { - if (extension_scope() == NULL) { + if (extension_scope() == nullptr) { output->push_back(FileDescriptorProto::kExtensionFieldNumber); output->push_back(index()); } else { @@ -3096,17 +3689,17 @@ namespace { // pointers in the original options, not the mutable copy). The Message must be // one of the Options messages in descriptor.proto. struct OptionsToInterpret { - OptionsToInterpret(const string& ns, - const string& el, - const Message* orig_opt, + OptionsToInterpret(const std::string& ns, const std::string& el, + const std::vector& path, const Message* orig_opt, Message* opt) : name_scope(ns), element_name(el), + element_path(path), original_options(orig_opt), - options(opt) { - } - string name_scope; - string element_name; + options(opt) {} + std::string name_scope; + std::string element_name; + std::vector element_path; const Message* original_options; Message* options; }; @@ -3115,8 +3708,7 @@ struct OptionsToInterpret { class DescriptorBuilder { public: - DescriptorBuilder(const DescriptorPool* pool, - DescriptorPool::Tables* tables, + DescriptorBuilder(const DescriptorPool* pool, DescriptorPool::Tables* tables, DescriptorPool::ErrorCollector* error_collector); ~DescriptorBuilder(); @@ -3138,7 +3730,7 @@ class DescriptorBuilder { std::vector options_to_interpret_; bool had_errors_; - string filename_; + std::string filename_; FileDescriptor* file_; FileDescriptorTables* file_tables_; std::set dependencies_; @@ -3155,19 +3747,17 @@ class DescriptorBuilder { // actually found in possible_undeclared_dependency_, which may be a parent // of the symbol actually looked for. const FileDescriptor* possible_undeclared_dependency_; - string possible_undeclared_dependency_name_; + std::string possible_undeclared_dependency_name_; // If LookupSymbol() could resolve a symbol which is not defined, // record the resolved name. This is only used by AddNotDefinedError() // to report a more useful error message. - string undefine_resolved_name_; + std::string undefine_resolved_name_; - void AddError(const string& element_name, - const Message& descriptor, + void AddError(const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, - const string& error); - void AddError(const string& element_name, - const Message& descriptor, + const std::string& error); + void AddError(const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, const char* error); void AddRecursiveImportError(const FileDescriptorProto& proto, int from_here); @@ -3177,19 +3767,18 @@ class DescriptorBuilder { // Adds an error indicating that undefined_symbol was not defined. Must // only be called after LookupSymbol() fails. void AddNotDefinedError( - const string& element_name, - const Message& descriptor, - DescriptorPool::ErrorCollector::ErrorLocation location, - const string& undefined_symbol); + const std::string& element_name, const Message& descriptor, + DescriptorPool::ErrorCollector::ErrorLocation location, + const std::string& undefined_symbol); - void AddWarning(const string& element_name, const Message& descriptor, + void AddWarning(const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, - const string& error); + const std::string& error); // Silly helper which determines if the given file is in the given package. // I.e., either file->package() == package_name or file->package() is a // nested package within package_name. - bool IsInPackage(const FileDescriptor* file, const string& package_name); + bool IsInPackage(const FileDescriptor* file, const std::string& package_name); // Helper function which finds all public dependencies of the given file, and // stores the them in the dependencies_ set in the builder. @@ -3199,15 +3788,16 @@ class DescriptorBuilder { // - Search the pool's underlay if not found in tables_. // - Insure that the resulting Symbol is from one of the file's declared // dependencies. - Symbol FindSymbol(const string& name, bool build_it = true); + Symbol FindSymbol(const std::string& name, bool build_it = true); // Like FindSymbol() but does not require that the symbol is in one of the // file's declared dependencies. - Symbol FindSymbolNotEnforcingDeps(const string& name, bool build_it = true); + Symbol FindSymbolNotEnforcingDeps(const std::string& name, + bool build_it = true); // This implements the body of FindSymbolNotEnforcingDeps(). Symbol FindSymbolNotEnforcingDepsHelper(const DescriptorPool* pool, - const string& name, + const std::string& name, bool build_it = true); // Like FindSymbol(), but looks up the name relative to some other symbol @@ -3224,10 +3814,8 @@ class DescriptorBuilder { // that LookupSymbol may still return a non-type symbol in LOOKUP_TYPES mode, // if it believes that's all it could refer to. The caller should always // check that it receives the type of symbol it was expecting. - enum ResolveMode { - LOOKUP_ALL, LOOKUP_TYPES - }; - Symbol LookupSymbol(const string& name, const string& relative_to, + enum ResolveMode { LOOKUP_ALL, LOOKUP_TYPES }; + Symbol LookupSymbol(const std::string& name, const std::string& relative_to, DescriptorPool::PlaceholderType placeholder_type = DescriptorPool::PLACEHOLDER_MESSAGE, ResolveMode resolve_mode = LOOKUP_ALL, @@ -3235,29 +3823,28 @@ class DescriptorBuilder { // Like LookupSymbol() but will not return a placeholder even if // AllowUnknownDependencies() has been used. - Symbol LookupSymbolNoPlaceholder(const string& name, - const string& relative_to, + Symbol LookupSymbolNoPlaceholder(const std::string& name, + const std::string& relative_to, ResolveMode resolve_mode = LOOKUP_ALL, bool build_it = true); // Calls tables_->AddSymbol() and records an error if it fails. Returns // true if successful or false if failed, though most callers can ignore // the return value since an error has already been recorded. - bool AddSymbol(const string& full_name, - const void* parent, const string& name, - const Message& proto, Symbol symbol); + bool AddSymbol(const std::string& full_name, const void* parent, + const std::string& name, const Message& proto, Symbol symbol); // Like AddSymbol(), but succeeds if the symbol is already defined as long // as the existing definition is also a package (because it's OK to define // the same package in two different files). Also adds all parents of the - // packgae to the symbol table (e.g. AddPackage("foo.bar", ...) will add + // package to the symbol table (e.g. AddPackage("foo.bar", ...) will add // "foo.bar" and "foo" to the table). - void AddPackage(const string& name, const Message& proto, - const FileDescriptor* file); + void AddPackage(const std::string& name, const Message& proto, + FileDescriptor* file); // Checks that the symbol name contains only alphanumeric characters and // underscores. Records an error otherwise. - void ValidateSymbolName(const string& name, const string& full_name, + void ValidateSymbolName(const std::string& name, const std::string& full_name, const Message& proto); // Used by BUILD_ARRAY macro (below) to avoid having to have the type @@ -3271,36 +3858,41 @@ class DescriptorBuilder { // descriptor. Remembers its uninterpreted options, to be interpreted // later. DescriptorT must be one of the Descriptor messages from // descriptor.proto. - template void AllocateOptions( - const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor); + template + void AllocateOptions(const typename DescriptorT::OptionsType& orig_options, + DescriptorT* descriptor, int options_field_tag, + const std::string& option_name); // Specialization for FileOptions. void AllocateOptions(const FileOptions& orig_options, FileDescriptor* descriptor); // Implementation for AllocateOptions(). Don't call this directly. - template void AllocateOptionsImpl( - const string& name_scope, - const string& element_name, + template + void AllocateOptionsImpl( + const std::string& name_scope, const std::string& element_name, const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor); + DescriptorT* descriptor, const std::vector& options_path, + const std::string& option_name); + + // Allocates an array of two strings, the first one is a copy of `proto_name`, + // and the second one is the full name. + // Full proto name is "scope.proto_name" if scope is non-empty and + // "proto_name" otherwise. + const std::string* AllocateNameStrings(const std::string& scope, + const std::string& proto_name); // These methods all have the same signature for the sake of the BUILD_ARRAY // macro, below. - void BuildMessage(const DescriptorProto& proto, - const Descriptor* parent, + void BuildMessage(const DescriptorProto& proto, const Descriptor* parent, Descriptor* result); void BuildFieldOrExtension(const FieldDescriptorProto& proto, - const Descriptor* parent, - FieldDescriptor* result, + Descriptor* parent, FieldDescriptor* result, bool is_extension); - void BuildField(const FieldDescriptorProto& proto, - const Descriptor* parent, + void BuildField(const FieldDescriptorProto& proto, Descriptor* parent, FieldDescriptor* result) { BuildFieldOrExtension(proto, parent, result, false); } - void BuildExtension(const FieldDescriptorProto& proto, - const Descriptor* parent, + void BuildExtension(const FieldDescriptorProto& proto, Descriptor* parent, FieldDescriptor* result) { BuildFieldOrExtension(proto, parent, result, true); } @@ -3308,28 +3900,24 @@ class DescriptorBuilder { const Descriptor* parent, Descriptor::ExtensionRange* result); void BuildReservedRange(const DescriptorProto::ReservedRange& proto, - const Descriptor* parent, - Descriptor::ReservedRange* result); + const Descriptor* parent, + Descriptor::ReservedRange* result); void BuildReservedRange(const EnumDescriptorProto::EnumReservedRange& proto, const EnumDescriptor* parent, EnumDescriptor::ReservedRange* result); - void BuildOneof(const OneofDescriptorProto& proto, - Descriptor* parent, + void BuildOneof(const OneofDescriptorProto& proto, Descriptor* parent, OneofDescriptor* result); void CheckEnumValueUniqueness(const EnumDescriptorProto& proto, const EnumDescriptor* result); - void BuildEnum(const EnumDescriptorProto& proto, - const Descriptor* parent, + void BuildEnum(const EnumDescriptorProto& proto, const Descriptor* parent, EnumDescriptor* result); void BuildEnumValue(const EnumValueDescriptorProto& proto, const EnumDescriptor* parent, EnumValueDescriptor* result); - void BuildService(const ServiceDescriptorProto& proto, - const void* dummy, + void BuildService(const ServiceDescriptorProto& proto, const void* dummy, ServiceDescriptor* result); void BuildMethod(const MethodDescriptorProto& proto, - const ServiceDescriptor* parent, - MethodDescriptor* result); + const ServiceDescriptor* parent, MethodDescriptor* result); void LogUnusedDependency(const FileDescriptorProto& proto, const FileDescriptor* result); @@ -3361,8 +3949,8 @@ class DescriptorBuilder { class OptionInterpreter { public: // Creates an interpreter that operates in the context of the pool of the - // specified builder, which must not be NULL. We don't take ownership of the - // builder. + // specified builder, which must not be nullptr. We don't take ownership of + // the builder. explicit OptionInterpreter(DescriptorBuilder* builder); ~OptionInterpreter(); @@ -3372,13 +3960,22 @@ class DescriptorBuilder { // Otherwise returns true. bool InterpretOptions(OptionsToInterpret* options_to_interpret); + // Updates the given source code info by re-writing uninterpreted option + // locations to refer to the corresponding interpreted option. + void UpdateSourceCodeInfo(SourceCodeInfo* info); + class AggregateOptionFinder; private: // Interprets uninterpreted_option_ on the specified message, which // must be the mutable copy of the original options message to which - // uninterpreted_option_ belongs. - bool InterpretSingleOption(Message* options); + // uninterpreted_option_ belongs. The given src_path is the source + // location path to the uninterpreted option, and options_path is the + // source location path to the options message. The location paths are + // recorded and then used in UpdateSourceCodeInfo. + bool InterpretSingleOption(Message* options, + const std::vector& src_path, + const std::vector& options_path); // Adds the uninterpreted_option to the given options message verbatim. // Used when AllowUnknownDependencies() is in effect and we can't find @@ -3394,7 +3991,8 @@ class DescriptorBuilder { intermediate_fields_iter, std::vector::const_iterator intermediate_fields_end, - const FieldDescriptor* innermost_field, const string& debug_msg_name, + const FieldDescriptor* innermost_field, + const std::string& debug_msg_name, const UnknownFieldSet& unknown_fields); // Validates the value for the option field of the currently interpreted @@ -3409,19 +4007,19 @@ class DescriptorBuilder { // Convenience functions to set an int field the right way, depending on // its wire type (a single int CppType can represent multiple wire types). - void SetInt32(int number, int32 value, FieldDescriptor::Type type, + void SetInt32(int number, int32_t value, FieldDescriptor::Type type, UnknownFieldSet* unknown_fields); - void SetInt64(int number, int64 value, FieldDescriptor::Type type, + void SetInt64(int number, int64_t value, FieldDescriptor::Type type, UnknownFieldSet* unknown_fields); - void SetUInt32(int number, uint32 value, FieldDescriptor::Type type, + void SetUInt32(int number, uint32_t value, FieldDescriptor::Type type, UnknownFieldSet* unknown_fields); - void SetUInt64(int number, uint64 value, FieldDescriptor::Type type, + void SetUInt64(int number, uint64_t value, FieldDescriptor::Type type, UnknownFieldSet* unknown_fields); // A helper function that adds an error at the specified location of the // option we're currently interpreting, and returns false. bool AddOptionError(DescriptorPool::ErrorCollector::ErrorLocation location, - const string& msg) { + const std::string& msg) { builder_->AddError(options_to_interpret_->element_name, *uninterpreted_option_, location, msg); return false; @@ -3429,30 +4027,44 @@ class DescriptorBuilder { // A helper function that adds an error at the location of the option name // and returns false. - bool AddNameError(const string& msg) { + bool AddNameError(const std::string& msg) { +#ifdef PROTOBUF_INTERNAL_IGNORE_FIELD_NAME_ERRORS_ + return true; +#else // PROTOBUF_INTERNAL_IGNORE_FIELD_NAME_ERRORS_ return AddOptionError(DescriptorPool::ErrorCollector::OPTION_NAME, msg); +#endif // PROTOBUF_INTERNAL_IGNORE_FIELD_NAME_ERRORS_ } // A helper function that adds an error at the location of the option name // and returns false. - bool AddValueError(const string& msg) { + bool AddValueError(const std::string& msg) { return AddOptionError(DescriptorPool::ErrorCollector::OPTION_VALUE, msg); } - // We interpret against this builder's pool. Is never NULL. We don't own + // We interpret against this builder's pool. Is never nullptr. We don't own // this pointer. DescriptorBuilder* builder_; - // The options we're currently interpreting, or NULL if we're not in a call - // to InterpretOptions. + // The options we're currently interpreting, or nullptr if we're not in a + // call to InterpretOptions. const OptionsToInterpret* options_to_interpret_; // The option we're currently interpreting within options_to_interpret_, or - // NULL if we're not in a call to InterpretOptions(). This points to a + // nullptr if we're not in a call to InterpretOptions(). This points to a // submessage of the original option, not the mutable copy. Therefore we // can use it to find locations recorded by the parser. const UninterpretedOption* uninterpreted_option_; + // This maps the element path of uninterpreted options to the element path + // of the resulting interpreted option. This is used to modify a file's + // source code info to account for option interpretation. + std::map, std::vector> interpreted_paths_; + + // This maps the path to a repeated option field to the known number of + // elements the field contains. This is used to track the compute the + // index portion of the element path when interpreting a single option. + std::map, int> repeated_option_counts_; + // Factory used to create the dynamic messages we need to parse // any aggregate option values we encounter. DynamicMessageFactory dynamic_factory_; @@ -3477,10 +4089,10 @@ class DescriptorBuilder { return pool->enforce_weak_; } static inline bool get_is_placeholder(const Descriptor* descriptor) { - return descriptor->is_placeholder_; + return descriptor != nullptr && descriptor->is_placeholder_; } static inline void assert_mutex_held(const DescriptorPool* pool) { - if (pool->mutex_ != NULL) { + if (pool->mutex_ != nullptr) { pool->mutex_->AssertHeld(); } } @@ -3501,14 +4113,15 @@ class DescriptorBuilder { const EnumDescriptorProto& proto); void ValidateEnumValueOptions(EnumValueDescriptor* enum_value, const EnumValueDescriptorProto& proto); + void ValidateExtensionRangeOptions( + const std::string& full_name, Descriptor::ExtensionRange* extension_range, + const DescriptorProto_ExtensionRange& proto); void ValidateServiceOptions(ServiceDescriptor* service, const ServiceDescriptorProto& proto); void ValidateMethodOptions(MethodDescriptor* method, const MethodDescriptorProto& proto); - void ValidateProto3(FileDescriptor* file, - const FileDescriptorProto& proto); - void ValidateProto3Message(Descriptor* message, - const DescriptorProto& proto); + void ValidateProto3(FileDescriptor* file, const FileDescriptorProto& proto); + void ValidateProto3Message(Descriptor* message, const DescriptorProto& proto); void ValidateProto3Field(FieldDescriptor* field, const FieldDescriptorProto& proto); void ValidateProto3Enum(EnumDescriptor* enm, @@ -3530,119 +4143,116 @@ class DescriptorBuilder { const FileDescriptor* DescriptorPool::BuildFile( const FileDescriptorProto& proto) { - GOOGLE_CHECK(fallback_database_ == NULL) - << "Cannot call BuildFile on a DescriptorPool that uses a " - "DescriptorDatabase. You must instead find a way to get your file " - "into the underlying database."; - GOOGLE_CHECK(mutex_ == NULL); // Implied by the above GOOGLE_CHECK. + GOOGLE_CHECK(fallback_database_ == nullptr) + << "Cannot call BuildFile on a DescriptorPool that uses a " + "DescriptorDatabase. You must instead find a way to get your file " + "into the underlying database."; + GOOGLE_CHECK(mutex_ == nullptr); // Implied by the above GOOGLE_CHECK. tables_->known_bad_symbols_.clear(); tables_->known_bad_files_.clear(); - return DescriptorBuilder(this, tables_.get(), NULL).BuildFile(proto); + return DescriptorBuilder(this, tables_.get(), nullptr).BuildFile(proto); } const FileDescriptor* DescriptorPool::BuildFileCollectingErrors( - const FileDescriptorProto& proto, - ErrorCollector* error_collector) { - GOOGLE_CHECK(fallback_database_ == NULL) - << "Cannot call BuildFile on a DescriptorPool that uses a " - "DescriptorDatabase. You must instead find a way to get your file " - "into the underlying database."; - GOOGLE_CHECK(mutex_ == NULL); // Implied by the above GOOGLE_CHECK. + const FileDescriptorProto& proto, ErrorCollector* error_collector) { + GOOGLE_CHECK(fallback_database_ == nullptr) + << "Cannot call BuildFile on a DescriptorPool that uses a " + "DescriptorDatabase. You must instead find a way to get your file " + "into the underlying database."; + GOOGLE_CHECK(mutex_ == nullptr); // Implied by the above GOOGLE_CHECK. tables_->known_bad_symbols_.clear(); tables_->known_bad_files_.clear(); - return DescriptorBuilder(this, tables_.get(), - error_collector).BuildFile(proto); + return DescriptorBuilder(this, tables_.get(), error_collector) + .BuildFile(proto); } const FileDescriptor* DescriptorPool::BuildFileFromDatabase( const FileDescriptorProto& proto) const { mutex_->AssertHeld(); if (tables_->known_bad_files_.count(proto.name()) > 0) { - return NULL; + return nullptr; } const FileDescriptor* result = - DescriptorBuilder(this, tables_.get(), - default_error_collector_).BuildFile(proto); - if (result == NULL) { + DescriptorBuilder(this, tables_.get(), default_error_collector_) + .BuildFile(proto); + if (result == nullptr) { tables_->known_bad_files_.insert(proto.name()); } return result; } DescriptorBuilder::DescriptorBuilder( - const DescriptorPool* pool, - DescriptorPool::Tables* tables, + const DescriptorPool* pool, DescriptorPool::Tables* tables, DescriptorPool::ErrorCollector* error_collector) - : pool_(pool), - tables_(tables), - error_collector_(error_collector), - had_errors_(false), - possible_undeclared_dependency_(NULL), - undefine_resolved_name_("") {} + : pool_(pool), + tables_(tables), + error_collector_(error_collector), + had_errors_(false), + possible_undeclared_dependency_(nullptr), + undefine_resolved_name_("") {} DescriptorBuilder::~DescriptorBuilder() {} void DescriptorBuilder::AddError( - const string& element_name, - const Message& descriptor, + const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, - const string& error) { - if (error_collector_ == NULL) { + const std::string& error) { + if (error_collector_ == nullptr) { if (!had_errors_) { GOOGLE_LOG(ERROR) << "Invalid proto descriptor for file \"" << filename_ << "\":"; } GOOGLE_LOG(ERROR) << " " << element_name << ": " << error; } else { - error_collector_->AddError(filename_, element_name, - &descriptor, location, error); + error_collector_->AddError(filename_, element_name, &descriptor, location, + error); } had_errors_ = true; } void DescriptorBuilder::AddError( - const string& element_name, - const Message& descriptor, - DescriptorPool::ErrorCollector::ErrorLocation location, - const char* error) { - AddError(element_name, descriptor, location, string(error)); + const std::string& element_name, const Message& descriptor, + DescriptorPool::ErrorCollector::ErrorLocation location, const char* error) { + AddError(element_name, descriptor, location, std::string(error)); } void DescriptorBuilder::AddNotDefinedError( - const string& element_name, - const Message& descriptor, + const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, - const string& undefined_symbol) { - if (possible_undeclared_dependency_ == NULL && + const std::string& undefined_symbol) { + if (possible_undeclared_dependency_ == nullptr && undefine_resolved_name_.empty()) { AddError(element_name, descriptor, location, "\"" + undefined_symbol + "\" is not defined."); } else { - if (possible_undeclared_dependency_ != NULL) { + if (possible_undeclared_dependency_ != nullptr) { AddError(element_name, descriptor, location, "\"" + possible_undeclared_dependency_name_ + - "\" seems to be defined in \"" + - possible_undeclared_dependency_->name() + "\", which is not " - "imported by \"" + filename_ + "\". To use it here, please " - "add the necessary import."); + "\" seems to be defined in \"" + + possible_undeclared_dependency_->name() + + "\", which is not " + "imported by \"" + + filename_ + + "\". To use it here, please " + "add the necessary import."); } if (!undefine_resolved_name_.empty()) { AddError(element_name, descriptor, location, "\"" + undefined_symbol + "\" is resolved to \"" + - undefine_resolved_name_ + "\", which is not defined. " - "The innermost scope is searched first in name resolution. " - "Consider using a leading '.'(i.e., \"." - + undefined_symbol + - "\") to start from the outermost scope."); + undefine_resolved_name_ + + "\", which is not defined. " + "The innermost scope is searched first in name resolution. " + "Consider using a leading '.'(i.e., \"." + + undefined_symbol + "\") to start from the outermost scope."); } } } void DescriptorBuilder::AddWarning( - const string& element_name, const Message& descriptor, + const std::string& element_name, const Message& descriptor, DescriptorPool::ErrorCollector::ErrorLocation location, - const string& error) { - if (error_collector_ == NULL) { + const std::string& error) { + if (error_collector_ == nullptr) { GOOGLE_LOG(WARNING) << filename_ << " " << element_name << ": " << error; } else { error_collector_->AddWarning(filename_, element_name, &descriptor, location, @@ -3651,27 +4261,27 @@ void DescriptorBuilder::AddWarning( } bool DescriptorBuilder::IsInPackage(const FileDescriptor* file, - const string& package_name) { + const std::string& package_name) { return HasPrefixString(file->package(), package_name) && (file->package().size() == package_name.size() || file->package()[package_name.size()] == '.'); } void DescriptorBuilder::RecordPublicDependencies(const FileDescriptor* file) { - if (file == NULL || !dependencies_.insert(file).second) return; - for (int i = 0; file != NULL && i < file->public_dependency_count(); i++) { + if (file == nullptr || !dependencies_.insert(file).second) return; + for (int i = 0; file != nullptr && i < file->public_dependency_count(); i++) { RecordPublicDependencies(file->public_dependency(i)); } } Symbol DescriptorBuilder::FindSymbolNotEnforcingDepsHelper( - const DescriptorPool* pool, const string& name, bool build_it) { + const DescriptorPool* pool, const std::string& name, bool build_it) { // If we are looking at an underlay, we must lock its mutex_, since we are // accessing the underlay's tables_ directly. - MutexLockMaybe lock((pool == pool_) ? NULL : pool->mutex_); + MutexLockMaybe lock((pool == pool_) ? nullptr : pool->mutex_); Symbol result = pool->tables_->FindSymbol(name); - if (result.IsNull() && pool->underlay_ != NULL) { + if (result.IsNull() && pool->underlay_ != nullptr) { // Symbol not found; check the underlay. result = FindSymbolNotEnforcingDepsHelper(pool->underlay_, name); } @@ -3692,12 +4302,19 @@ Symbol DescriptorBuilder::FindSymbolNotEnforcingDepsHelper( return result; } -Symbol DescriptorBuilder::FindSymbolNotEnforcingDeps(const string& name, +Symbol DescriptorBuilder::FindSymbolNotEnforcingDeps(const std::string& name, bool build_it) { - return FindSymbolNotEnforcingDepsHelper(pool_, name, build_it); + Symbol result = FindSymbolNotEnforcingDepsHelper(pool_, name, build_it); + // Only find symbols which were defined in this file or one of its + // dependencies. + const FileDescriptor* file = result.GetFile(); + if (file == file_ || dependencies_.count(file) > 0) { + unused_dependency_.erase(file); + } + return result; } -Symbol DescriptorBuilder::FindSymbol(const string& name, bool build_it) { +Symbol DescriptorBuilder::FindSymbol(const std::string& name, bool build_it) { Symbol result = FindSymbolNotEnforcingDeps(name, build_it); if (result.IsNull()) return result; @@ -3711,11 +4328,10 @@ Symbol DescriptorBuilder::FindSymbol(const string& name, bool build_it) { // dependencies. const FileDescriptor* file = result.GetFile(); if (file == file_ || dependencies_.count(file) > 0) { - unused_dependency_.erase(file); return result; } - if (result.type == Symbol::PACKAGE) { + if (result.type() == Symbol::PACKAGE) { // Arg, this is overcomplicated. The symbol is a package name. It could // be that the package was defined in multiple files. result.GetFile() // returns the first file we saw that used this package. We've determined @@ -3727,8 +4343,8 @@ Symbol DescriptorBuilder::FindSymbol(const string& name, bool build_it) { for (std::set::const_iterator it = dependencies_.begin(); it != dependencies_.end(); ++it) { - // Note: A dependency may be NULL if it was not found or had errors. - if (*it != NULL && IsInPackage(*it, name)) return result; + // Note: A dependency may be nullptr if it was not found or had errors. + if (*it != nullptr && IsInPackage(*it, name)) return result; } } @@ -3737,11 +4353,10 @@ Symbol DescriptorBuilder::FindSymbol(const string& name, bool build_it) { return kNullSymbol; } -Symbol DescriptorBuilder::LookupSymbolNoPlaceholder(const string& name, - const string& relative_to, - ResolveMode resolve_mode, - bool build_it) { - possible_undeclared_dependency_ = NULL; +Symbol DescriptorBuilder::LookupSymbolNoPlaceholder( + const std::string& name, const std::string& relative_to, + ResolveMode resolve_mode, bool build_it) { + possible_undeclared_dependency_ = nullptr; undefine_resolved_name_.clear(); if (!name.empty() && name[0] == '.') { @@ -3760,27 +4375,27 @@ Symbol DescriptorBuilder::LookupSymbolNoPlaceholder(const string& name, // } // So, we look for just "Foo" first, then look for "Bar.baz" within it if // found. - string::size_type name_dot_pos = name.find_first_of('.'); - string first_part_of_name; - if (name_dot_pos == string::npos) { + std::string::size_type name_dot_pos = name.find_first_of('.'); + std::string first_part_of_name; + if (name_dot_pos == std::string::npos) { first_part_of_name = name; } else { first_part_of_name = name.substr(0, name_dot_pos); } - string scope_to_try(relative_to); + std::string scope_to_try(relative_to); while (true) { // Chop off the last component of the scope. - string::size_type dot_pos = scope_to_try.find_last_of('.'); - if (dot_pos == string::npos) { + std::string::size_type dot_pos = scope_to_try.find_last_of('.'); + if (dot_pos == std::string::npos) { return FindSymbol(name, build_it); } else { scope_to_try.erase(dot_pos); } // Append ".first_part_of_name" and try to find. - string::size_type old_size = scope_to_try.size(); + std::string::size_type old_size = scope_to_try.size(); scope_to_try.append(1, '.'); scope_to_try.append(first_part_of_name); Symbol result = FindSymbol(scope_to_try, build_it); @@ -3814,7 +4429,7 @@ Symbol DescriptorBuilder::LookupSymbolNoPlaceholder(const string& name, } Symbol DescriptorBuilder::LookupSymbol( - const string& name, const string& relative_to, + const std::string& name, const std::string& relative_to, DescriptorPool::PlaceholderType placeholder_type, ResolveMode resolve_mode, bool build_it) { Symbol result = @@ -3827,16 +4442,16 @@ Symbol DescriptorBuilder::LookupSymbol( return result; } -static bool ValidateQualifiedName(const string& name) { +static bool ValidateQualifiedName(StringPiece name) { bool last_was_period = false; - for (int i = 0; i < name.size(); i++) { + for (char character : name) { // I don't trust isalnum() due to locales. :( - if (('a' <= name[i] && name[i] <= 'z') || - ('A' <= name[i] && name[i] <= 'Z') || - ('0' <= name[i] && name[i] <= '9') || (name[i] == '_')) { + if (('a' <= character && character <= 'z') || + ('A' <= character && character <= 'Z') || + ('0' <= character && character <= '9') || (character == '_')) { last_was_period = false; - } else if (name[i] == '.') { + } else if (character == '.') { if (last_was_period) return false; last_was_period = true; } else { @@ -3847,36 +4462,35 @@ static bool ValidateQualifiedName(const string& name) { return !name.empty() && !last_was_period; } -Symbol DescriptorPool::NewPlaceholder(const string& name, +Symbol DescriptorPool::NewPlaceholder(StringPiece name, PlaceholderType placeholder_type) const { MutexLockMaybe lock(mutex_); return NewPlaceholderWithMutexHeld(name, placeholder_type); } Symbol DescriptorPool::NewPlaceholderWithMutexHeld( - const string& name, PlaceholderType placeholder_type) const { + StringPiece name, PlaceholderType placeholder_type) const { if (mutex_) { mutex_->AssertHeld(); } // Compute names. - const string* placeholder_full_name; - const string* placeholder_name; - const string* placeholder_package; + StringPiece placeholder_full_name; + StringPiece placeholder_name; + const std::string* placeholder_package; if (!ValidateQualifiedName(name)) return kNullSymbol; if (name[0] == '.') { // Fully-qualified. - placeholder_full_name = tables_->AllocateString(name.substr(1)); + placeholder_full_name = name.substr(1); } else { - placeholder_full_name = tables_->AllocateString(name); + placeholder_full_name = name; } - string::size_type dotpos = placeholder_full_name->find_last_of('.'); - if (dotpos != string::npos) { - placeholder_package = tables_->AllocateString( - placeholder_full_name->substr(0, dotpos)); - placeholder_name = tables_->AllocateString( - placeholder_full_name->substr(dotpos + 1)); + std::string::size_type dotpos = placeholder_full_name.find_last_of('.'); + if (dotpos != std::string::npos) { + placeholder_package = + tables_->AllocateString(placeholder_full_name.substr(0, dotpos)); + placeholder_name = placeholder_full_name.substr(dotpos + 1); } else { placeholder_package = &internal::GetEmptyString(); placeholder_name = placeholder_full_name; @@ -3884,19 +4498,18 @@ Symbol DescriptorPool::NewPlaceholderWithMutexHeld( // Create the placeholders. FileDescriptor* placeholder_file = NewPlaceholderFileWithMutexHeld( - *placeholder_full_name + ".placeholder.proto"); + StrCat(placeholder_full_name, ".placeholder.proto")); placeholder_file->package_ = placeholder_package; if (placeholder_type == PLACEHOLDER_ENUM) { placeholder_file->enum_type_count_ = 1; - placeholder_file->enum_types_ = - tables_->AllocateArray(1); + placeholder_file->enum_types_ = tables_->AllocateArray(1); EnumDescriptor* placeholder_enum = &placeholder_file->enum_types_[0]; - memset(placeholder_enum, 0, sizeof(*placeholder_enum)); + memset(static_cast(placeholder_enum), 0, sizeof(*placeholder_enum)); - placeholder_enum->full_name_ = placeholder_full_name; - placeholder_enum->name_ = placeholder_name; + placeholder_enum->all_names_ = + tables_->AllocateStringArray(placeholder_name, placeholder_full_name); placeholder_enum->file_ = placeholder_file; placeholder_enum->options_ = &EnumOptions::default_instance(); placeholder_enum->is_placeholder_ = true; @@ -3905,15 +4518,18 @@ Symbol DescriptorPool::NewPlaceholderWithMutexHeld( // Enums must have at least one value. placeholder_enum->value_count_ = 1; placeholder_enum->values_ = tables_->AllocateArray(1); + // Disable fast-path lookup for this enum. + placeholder_enum->sequential_value_limit_ = -1; EnumValueDescriptor* placeholder_value = &placeholder_enum->values_[0]; - memset(placeholder_value, 0, sizeof(*placeholder_value)); + memset(static_cast(placeholder_value), 0, + sizeof(*placeholder_value)); - placeholder_value->name_ = tables_->AllocateString("PLACEHOLDER_VALUE"); // Note that enum value names are siblings of their type, not children. - placeholder_value->full_name_ = - placeholder_package->empty() ? placeholder_value->name_ : - tables_->AllocateString(*placeholder_package + ".PLACEHOLDER_VALUE"); + placeholder_value->all_names_ = tables_->AllocateStringArray( + "PLACEHOLDER_VALUE", placeholder_package->empty() + ? "PLACEHOLDER_VALUE" + : *placeholder_package + ".PLACEHOLDER_VALUE"); placeholder_value->number_ = 0; placeholder_value->type_ = placeholder_enum; @@ -3922,14 +4538,14 @@ Symbol DescriptorPool::NewPlaceholderWithMutexHeld( return Symbol(placeholder_enum); } else { placeholder_file->message_type_count_ = 1; - placeholder_file->message_types_ = - tables_->AllocateArray(1); + placeholder_file->message_types_ = tables_->AllocateArray(1); Descriptor* placeholder_message = &placeholder_file->message_types_[0]; - memset(placeholder_message, 0, sizeof(*placeholder_message)); + memset(static_cast(placeholder_message), 0, + sizeof(*placeholder_message)); - placeholder_message->full_name_ = placeholder_full_name; - placeholder_message->name_ = placeholder_name; + placeholder_message->all_names_ = + tables_->AllocateStringArray(placeholder_name, placeholder_full_name); placeholder_message->file_ = placeholder_file; placeholder_message->options_ = &MessageOptions::default_instance(); placeholder_message->is_placeholder_ = true; @@ -3938,29 +4554,31 @@ Symbol DescriptorPool::NewPlaceholderWithMutexHeld( if (placeholder_type == PLACEHOLDER_EXTENDABLE_MESSAGE) { placeholder_message->extension_range_count_ = 1; placeholder_message->extension_ranges_ = - tables_->AllocateArray(1); + tables_->AllocateArray(1); placeholder_message->extension_ranges_->start = 1; // kMaxNumber + 1 because ExtensionRange::end is exclusive. placeholder_message->extension_ranges_->end = - FieldDescriptor::kMaxNumber + 1; + FieldDescriptor::kMaxNumber + 1; + placeholder_message->extension_ranges_->options_ = nullptr; } return Symbol(placeholder_message); } } -FileDescriptor* DescriptorPool::NewPlaceholderFile(const string& name) const { +FileDescriptor* DescriptorPool::NewPlaceholderFile( + StringPiece name) const { MutexLockMaybe lock(mutex_); return NewPlaceholderFileWithMutexHeld(name); } FileDescriptor* DescriptorPool::NewPlaceholderFileWithMutexHeld( - const string& name) const { + StringPiece name) const { if (mutex_) { mutex_->AssertHeld(); } FileDescriptor* placeholder = tables_->Allocate(); - memset(placeholder, 0, sizeof(*placeholder)); + memset(static_cast(placeholder), 0, sizeof(*placeholder)); placeholder->name_ = tables_->AllocateString(name); placeholder->package_ = &internal::GetEmptyString(); @@ -3969,26 +4587,32 @@ FileDescriptor* DescriptorPool::NewPlaceholderFileWithMutexHeld( placeholder->tables_ = &FileDescriptorTables::GetEmptyInstance(); placeholder->source_code_info_ = &SourceCodeInfo::default_instance(); placeholder->is_placeholder_ = true; - placeholder->syntax_ = FileDescriptor::SYNTAX_PROTO2; + placeholder->syntax_ = FileDescriptor::SYNTAX_UNKNOWN; placeholder->finished_building_ = true; - // All other fields are zero or NULL. + // All other fields are zero or nullptr. return placeholder; } -bool DescriptorBuilder::AddSymbol( - const string& full_name, const void* parent, const string& name, - const Message& proto, Symbol symbol) { - // If the caller passed NULL for the parent, the symbol is at file scope. +bool DescriptorBuilder::AddSymbol(const std::string& full_name, + const void* parent, const std::string& name, + const Message& proto, Symbol symbol) { + // If the caller passed nullptr for the parent, the symbol is at file scope. // Use its file as the parent instead. - if (parent == NULL) parent = file_; + if (parent == nullptr) parent = file_; + if (full_name.find('\0') != std::string::npos) { + AddError(full_name, proto, DescriptorPool::ErrorCollector::NAME, + "\"" + full_name + "\" contains null character."); + return false; + } if (tables_->AddSymbol(full_name, symbol)) { if (!file_tables_->AddAliasUnderParent(parent, name, symbol)) { // This is only possible if there was already an error adding something of // the same name. if (!had_errors_) { - GOOGLE_LOG(DFATAL) << "\"" << full_name << "\" not previously defined in " + GOOGLE_LOG(DFATAL) << "\"" << full_name + << "\" not previously defined in " "symbols_by_name_, but was defined in " "symbols_by_parent_; this shouldn't be possible."; } @@ -3998,65 +4622,77 @@ bool DescriptorBuilder::AddSymbol( } else { const FileDescriptor* other_file = tables_->FindSymbol(full_name).GetFile(); if (other_file == file_) { - string::size_type dot_pos = full_name.find_last_of('.'); - if (dot_pos == string::npos) { + std::string::size_type dot_pos = full_name.find_last_of('.'); + if (dot_pos == std::string::npos) { AddError(full_name, proto, DescriptorPool::ErrorCollector::NAME, "\"" + full_name + "\" is already defined."); } else { AddError(full_name, proto, DescriptorPool::ErrorCollector::NAME, "\"" + full_name.substr(dot_pos + 1) + - "\" is already defined in \"" + - full_name.substr(0, dot_pos) + "\"."); + "\" is already defined in \"" + + full_name.substr(0, dot_pos) + "\"."); } } else { // Symbol seems to have been defined in a different file. AddError(full_name, proto, DescriptorPool::ErrorCollector::NAME, "\"" + full_name + "\" is already defined in file \"" + - other_file->name() + "\"."); + (other_file == nullptr ? "null" : other_file->name()) + + "\"."); } return false; } } -void DescriptorBuilder::AddPackage( - const string& name, const Message& proto, const FileDescriptor* file) { - if (tables_->AddSymbol(name, Symbol(file))) { - // Success. Also add parent package, if any. - string::size_type dot_pos = name.find_last_of('.'); - if (dot_pos == string::npos) { +void DescriptorBuilder::AddPackage(const std::string& name, + const Message& proto, FileDescriptor* file) { + if (name.find('\0') != std::string::npos) { + AddError(name, proto, DescriptorPool::ErrorCollector::NAME, + "\"" + name + "\" contains null character."); + return; + } + + Symbol existing_symbol = tables_->FindSymbol(name); + // It's OK to redefine a package. + if (existing_symbol.IsNull()) { + auto* package = tables_->AllocateArray(1); + // If the name is the package name, then it is already in the arena. + // If not, copy it there. It came from the call to AddPackage below. + package->name = + &name == &file->package() ? &name : tables_->AllocateString(name); + package->file = file; + tables_->AddSymbol(*package->name, Symbol(package)); + // Also add parent package, if any. + std::string::size_type dot_pos = name.find_last_of('.'); + if (dot_pos == std::string::npos) { // No parents. ValidateSymbolName(name, name, proto); } else { - // Has parent. - string* parent_name = tables_->AllocateString(name.substr(0, dot_pos)); - AddPackage(*parent_name, proto, file); - ValidateSymbolName(name.substr(dot_pos + 1), name, proto); - } - } else { - Symbol existing_symbol = tables_->FindSymbol(name); - // It's OK to redefine a package. - if (existing_symbol.type != Symbol::PACKAGE) { - // Symbol seems to have been defined in a different file. - AddError(name, proto, DescriptorPool::ErrorCollector::NAME, - "\"" + name + "\" is already defined (as something other than " - "a package) in file \"" + existing_symbol.GetFile()->name() + - "\"."); + // Has parent. + AddPackage(name.substr(0, dot_pos), proto, file); + ValidateSymbolName(name.substr(dot_pos + 1), name, proto); } + } else if (existing_symbol.type() != Symbol::PACKAGE) { + // Symbol seems to have been defined in a different file. + AddError(name, proto, DescriptorPool::ErrorCollector::NAME, + "\"" + name + + "\" is already defined (as something other than " + "a package) in file \"" + + existing_symbol.GetFile()->name() + "\"."); } } -void DescriptorBuilder::ValidateSymbolName( - const string& name, const string& full_name, const Message& proto) { +void DescriptorBuilder::ValidateSymbolName(const std::string& name, + const std::string& full_name, + const Message& proto) { if (name.empty()) { AddError(full_name, proto, DescriptorPool::ErrorCollector::NAME, "Missing name."); } else { - for (int i = 0; i < name.size(); i++) { + for (char character : name) { // I don't trust isalnum() due to locales. :( - if ((name[i] < 'a' || 'z' < name[i]) && - (name[i] < 'A' || 'Z' < name[i]) && - (name[i] < '0' || '9' < name[i]) && - (name[i] != '_')) { + if ((character < 'a' || 'z' < character) && + (character < 'A' || 'Z' < character) && + (character < '0' || '9' < character) && (character != '_')) { AddError(full_name, proto, DescriptorPool::ErrorCollector::NAME, "\"" + name + "\" is not a valid identifier."); } @@ -4068,32 +4704,49 @@ void DescriptorBuilder::ValidateSymbolName( // This generic implementation is good for all descriptors except // FileDescriptor. -template void DescriptorBuilder::AllocateOptions( +template +void DescriptorBuilder::AllocateOptions( const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor) { + DescriptorT* descriptor, int options_field_tag, + const std::string& option_name) { + std::vector options_path; + descriptor->GetLocationPath(&options_path); + options_path.push_back(options_field_tag); AllocateOptionsImpl(descriptor->full_name(), descriptor->full_name(), - orig_options, descriptor); + orig_options, descriptor, options_path, option_name); } // We specialize for FileDescriptor. void DescriptorBuilder::AllocateOptions(const FileOptions& orig_options, FileDescriptor* descriptor) { + std::vector options_path; + options_path.push_back(FileDescriptorProto::kOptionsFieldNumber); // We add the dummy token so that LookupSymbol does the right thing. AllocateOptionsImpl(descriptor->package() + ".dummy", descriptor->name(), - orig_options, descriptor); + orig_options, descriptor, options_path, + "google.protobuf.FileOptions"); } -template void DescriptorBuilder::AllocateOptionsImpl( - const string& name_scope, - const string& element_name, +template +void DescriptorBuilder::AllocateOptionsImpl( + const std::string& name_scope, const std::string& element_name, const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor) { + DescriptorT* descriptor, const std::vector& options_path, + const std::string& option_name) { // We need to use a dummy pointer to work around a bug in older versions of // GCC. Otherwise, the following two lines could be replaced with: // typename DescriptorT::OptionsType* options = // tables_->AllocateMessage(); - typename DescriptorT::OptionsType* const dummy = NULL; + typename DescriptorT::OptionsType* const dummy = nullptr; typename DescriptorT::OptionsType* options = tables_->AllocateMessage(dummy); + + if (!orig_options.IsInitialized()) { + AddError(name_scope + "." + element_name, orig_options, + DescriptorPool::ErrorCollector::OPTION_NAME, + "Uninterpreted option is missing name or value."); + return; + } + // Avoid using MergeFrom()/CopyFrom() in this class to make it -fno-rtti // friendly. Without RTTI, MergeFrom() and CopyFrom() will fallback to the // reflection based method, which requires the Descriptor. However, we are in @@ -4109,51 +4762,75 @@ template void DescriptorBuilder::AllocateOptionsImpl( // OptionsType::GetDescriptor() to be called which may then deadlock since // we're still trying to build it. if (options->uninterpreted_option_size() > 0) { - options_to_interpret_.push_back( - OptionsToInterpret(name_scope, element_name, &orig_options, options)); + options_to_interpret_.push_back(OptionsToInterpret( + name_scope, element_name, options_path, &orig_options, options)); + } + + // If the custom option is in unknown fields, no need to interpret it. + // Remove the dependency file from unused_dependency. + const UnknownFieldSet& unknown_fields = orig_options.unknown_fields(); + if (!unknown_fields.empty()) { + // Can not use options->GetDescriptor() which may case deadlock. + Symbol msg_symbol = tables_->FindSymbol(option_name); + if (msg_symbol.type() == Symbol::MESSAGE) { + for (int i = 0; i < unknown_fields.field_count(); ++i) { + assert_mutex_held(pool_); + const FieldDescriptor* field = + pool_->InternalFindExtensionByNumberNoLock( + msg_symbol.descriptor(), unknown_fields.field(i).number()); + if (field) { + unused_dependency_.erase(field->file()); + } + } + } } } - // A common pattern: We want to convert a repeated field in the descriptor // to an array of values, calling some method to build each value. -#define BUILD_ARRAY(INPUT, OUTPUT, NAME, METHOD, PARENT) \ - OUTPUT->NAME##_count_ = INPUT.NAME##_size(); \ - AllocateArray(INPUT.NAME##_size(), &OUTPUT->NAME##s_); \ - for (int i = 0; i < INPUT.NAME##_size(); i++) { \ - METHOD(INPUT.NAME(i), PARENT, OUTPUT->NAME##s_ + i); \ +#define BUILD_ARRAY(INPUT, OUTPUT, NAME, METHOD, PARENT) \ + OUTPUT->NAME##_count_ = INPUT.NAME##_size(); \ + AllocateArray(INPUT.NAME##_size(), &OUTPUT->NAME##s_); \ + for (int i = 0; i < INPUT.NAME##_size(); i++) { \ + METHOD(INPUT.NAME(i), PARENT, OUTPUT->NAME##s_ + i); \ } void DescriptorBuilder::AddRecursiveImportError( const FileDescriptorProto& proto, int from_here) { - string error_message("File recursively imports itself: "); - for (int i = from_here; i < tables_->pending_files_.size(); i++) { + std::string error_message("File recursively imports itself: "); + for (size_t i = from_here; i < tables_->pending_files_.size(); i++) { error_message.append(tables_->pending_files_[i]); error_message.append(" -> "); } error_message.append(proto.name()); - AddError(proto.name(), proto, DescriptorPool::ErrorCollector::OTHER, - error_message); + if (static_cast(from_here) < tables_->pending_files_.size() - 1) { + AddError(tables_->pending_files_[from_here + 1], proto, + DescriptorPool::ErrorCollector::IMPORT, error_message); + } else { + AddError(proto.name(), proto, DescriptorPool::ErrorCollector::IMPORT, + error_message); + } } void DescriptorBuilder::AddTwiceListedError(const FileDescriptorProto& proto, int index) { - AddError(proto.name(), proto, DescriptorPool::ErrorCollector::OTHER, + AddError(proto.dependency(index), proto, + DescriptorPool::ErrorCollector::IMPORT, "Import \"" + proto.dependency(index) + "\" was listed twice."); } void DescriptorBuilder::AddImportError(const FileDescriptorProto& proto, int index) { - string message; - if (pool_->fallback_database_ == NULL) { - message = "Import \"" + proto.dependency(index) + - "\" has not been loaded."; + std::string message; + if (pool_->fallback_database_ == nullptr) { + message = "Import \"" + proto.dependency(index) + "\" has not been loaded."; } else { message = "Import \"" + proto.dependency(index) + "\" was not found or had errors."; } - AddError(proto.name(), proto, DescriptorPool::ErrorCollector::OTHER, message); + AddError(proto.dependency(index), proto, + DescriptorPool::ErrorCollector::IMPORT, message); } static bool ExistingFileMatchesProto(const FileDescriptor* existing_file, @@ -4181,7 +4858,7 @@ const FileDescriptor* DescriptorBuilder::BuildFile( // This is fine, because this idempotency "feature" really only exists to // accommodate one hack in the proto1->proto2 migration layer. const FileDescriptor* existing_file = tables_->FindFile(filename_); - if (existing_file != NULL) { + if (existing_file != nullptr) { // File already in pool. Compare the existing one to the input. if (ExistingFileMatchesProto(existing_file, proto)) { // They're identical. Return the existing descriptor. @@ -4200,10 +4877,10 @@ const FileDescriptor* DescriptorBuilder::BuildFile( // mid-file, but that's pretty ugly, and I'm pretty sure there are // some languages out there that do not allow recursive dependencies // at all. - for (int i = 0; i < tables_->pending_files_.size(); i++) { + for (size_t i = 0; i < tables_->pending_files_.size(); i++) { if (tables_->pending_files_[i] == proto.name()) { AddRecursiveImportError(proto, i); - return NULL; + return nullptr; } } @@ -4211,12 +4888,13 @@ const FileDescriptor* DescriptorBuilder::BuildFile( // attempt to load all dependencies now, before checkpointing tables_. This // avoids confusion with recursive checkpoints. if (!pool_->lazily_build_dependencies_) { - if (pool_->fallback_database_ != NULL) { + if (pool_->fallback_database_ != nullptr) { tables_->pending_files_.push_back(proto.name()); for (int i = 0; i < proto.dependency_size(); i++) { - if (tables_->FindFile(proto.dependency(i)) == NULL && - (pool_->underlay_ == NULL || - pool_->underlay_->FindFileByName(proto.dependency(i)) == NULL)) { + if (tables_->FindFile(proto.dependency(i)) == nullptr && + (pool_->underlay_ == nullptr || + pool_->underlay_->FindFileByName(proto.dependency(i)) == + nullptr)) { // We don't care what this returns since we'll find out below anyway. pool_->TryFindFileInFallbackDatabase(proto.dependency(i)); } @@ -4248,8 +4926,9 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( result->is_placeholder_ = false; result->finished_building_ = false; + SourceCodeInfo* info = nullptr; if (proto.has_source_code_info()) { - SourceCodeInfo *info = tables_->AllocateMessage(); + info = tables_->AllocateMessage(); info->CopyFrom(proto.source_code_info()); result->source_code_info_ = info; } else { @@ -4288,35 +4967,30 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( } result->pool_ = pool_; + if (result->name().find('\0') != std::string::npos) { + AddError(result->name(), proto, DescriptorPool::ErrorCollector::NAME, + "\"" + result->name() + "\" contains null character."); + return nullptr; + } + // Add to tables. if (!tables_->AddFile(result)) { AddError(proto.name(), proto, DescriptorPool::ErrorCollector::OTHER, "A file with this name is already in the pool."); // Bail out early so that if this is actually the exact same file, we // don't end up reporting that every single symbol is already defined. - return NULL; + return nullptr; } if (!result->package().empty()) { AddPackage(result->package(), proto, result); } // Make sure all dependencies are loaded. - std::set seen_dependencies; + std::set seen_dependencies; result->dependency_count_ = proto.dependency_size(); result->dependencies_ = tables_->AllocateArray(proto.dependency_size()); - if (pool_->lazily_build_dependencies_) { - result->dependencies_once_ = tables_->AllocateOnceDynamic(); - result->dependencies_names_ = - tables_->AllocateArray(proto.dependency_size()); - if (proto.dependency_size() > 0) { - memset(result->dependencies_names_, 0, - sizeof(*result->dependencies_names_) * proto.dependency_size()); - } - } else { - result->dependencies_once_ = NULL; - result->dependencies_names_ = NULL; - } + result->dependencies_once_ = nullptr; unused_dependency_.clear(); std::set weak_deps; for (int i = 0; i < proto.weak_dependency_size(); ++i) { @@ -4328,7 +5002,7 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( } const FileDescriptor* dependency = tables_->FindFile(proto.dependency(i)); - if (dependency == NULL && pool_->underlay_ != NULL) { + if (dependency == nullptr && pool_->underlay_ != nullptr) { dependency = pool_->underlay_->FindFileByName(proto.dependency(i)); } @@ -4336,10 +5010,10 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( // Recursive import. dependency/result is not fully initialized, and it's // dangerous to try to do anything with it. The recursive import error // will be detected and reported in DescriptorBuilder::BuildFile(). - return NULL; + return nullptr; } - if (dependency == NULL) { + if (dependency == nullptr) { if (!pool_->lazily_build_dependencies_) { if (pool_->allow_unknown_ || (!pool_->enforce_weak_ && weak_deps.find(i) != weak_deps.end())) { @@ -4362,15 +5036,26 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( result->dependencies_[i] = dependency; if (pool_->lazily_build_dependencies_ && !dependency) { - result->dependencies_names_[i] = - tables_->AllocateString(proto.dependency(i)); + if (result->dependencies_once_ == nullptr) { + result->dependencies_once_ = + tables_->Create(); + result->dependencies_once_->dependencies_names = + tables_->AllocateArray(proto.dependency_size()); + if (proto.dependency_size() > 0) { + std::fill_n(result->dependencies_once_->dependencies_names, + proto.dependency_size(), nullptr); + } + } + + result->dependencies_once_->dependencies_names[i] = + tables_->Strdup(proto.dependency(i)); } } // Check public dependencies. int public_dependency_count = 0; - result->public_dependencies_ = tables_->AllocateArray( - proto.public_dependency_size()); + result->public_dependencies_ = + tables_->AllocateArray(proto.public_dependency_size()); for (int i = 0; i < proto.public_dependency_size(); i++) { // Only put valid public dependency indexes. int index = proto.public_dependency(i); @@ -4384,8 +5069,7 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( unused_dependency_.erase(result->dependency(index)); } } else { - AddError(proto.name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(proto.name(), proto, DescriptorPool::ErrorCollector::OTHER, "Invalid public dependency index."); } } @@ -4404,30 +5088,28 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( // Check weak dependencies. int weak_dependency_count = 0; - result->weak_dependencies_ = tables_->AllocateArray( - proto.weak_dependency_size()); + result->weak_dependencies_ = + tables_->AllocateArray(proto.weak_dependency_size()); for (int i = 0; i < proto.weak_dependency_size(); i++) { int index = proto.weak_dependency(i); if (index >= 0 && index < proto.dependency_size()) { result->weak_dependencies_[weak_dependency_count++] = index; } else { - AddError(proto.name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(proto.name(), proto, DescriptorPool::ErrorCollector::OTHER, "Invalid weak dependency index."); } } result->weak_dependency_count_ = weak_dependency_count; // Convert children. - BUILD_ARRAY(proto, result, message_type, BuildMessage , NULL); - BUILD_ARRAY(proto, result, enum_type , BuildEnum , NULL); - BUILD_ARRAY(proto, result, service , BuildService , NULL); - BUILD_ARRAY(proto, result, extension , BuildExtension, NULL); + BUILD_ARRAY(proto, result, message_type, BuildMessage, nullptr); + BUILD_ARRAY(proto, result, enum_type, BuildEnum, nullptr); + BUILD_ARRAY(proto, result, service, BuildService, nullptr); + BUILD_ARRAY(proto, result, extension, BuildExtension, nullptr); // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { AllocateOptions(proto.options(), result); } @@ -4447,6 +5129,9 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( option_interpreter.InterpretOptions(&(*iter)); } options_to_interpret_.clear(); + if (info != nullptr) { + option_interpreter.UpdateSourceCodeInfo(info); + } } // Validate options. See comments at InternalSetLazilyBuildDependencies about @@ -4465,64 +5150,91 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( // Again, see comments at InternalSetLazilyBuildDependencies about error - // checking. - if (!unused_dependency_.empty() && !pool_->lazily_build_dependencies_) { + // checking. Also, don't log unused dependencies if there were previous + // errors, since the results might be inaccurate. + if (!had_errors_ && !unused_dependency_.empty() && + !pool_->lazily_build_dependencies_) { LogUnusedDependency(proto, result); } if (had_errors_) { - return NULL; + return nullptr; } else { return result; } } + +const std::string* DescriptorBuilder::AllocateNameStrings( + const std::string& scope, const std::string& proto_name) { + if (scope.empty()) { + return tables_->AllocateStringArray(proto_name, proto_name); + } else { + return tables_->AllocateStringArray(proto_name, + StrCat(scope, ".", proto_name)); + } +} + void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, const Descriptor* parent, Descriptor* result) { - const string& scope = (parent == NULL) ? - file_->package() : parent->full_name(); - string* full_name = tables_->AllocateString(scope); - if (!full_name->empty()) full_name->append(1, '.'); - full_name->append(proto.name()); + const std::string& scope = + (parent == nullptr) ? file_->package() : parent->full_name(); + result->all_names_ = AllocateNameStrings(scope, proto.name()); + ValidateSymbolName(proto.name(), result->full_name(), proto); - ValidateSymbolName(proto.name(), *full_name, proto); - - result->name_ = tables_->AllocateString(proto.name()); - result->full_name_ = full_name; - result->file_ = file_; + result->file_ = file_; result->containing_type_ = parent; - result->is_placeholder_ = false; + result->is_placeholder_ = false; result->is_unqualified_placeholder_ = false; + result->well_known_type_ = Descriptor::WELLKNOWNTYPE_UNSPECIFIED; + + auto it = pool_->tables_->well_known_types_.find(result->full_name()); + if (it != pool_->tables_->well_known_types_.end()) { + result->well_known_type_ = it->second; + } + + // Calculate the continuous sequence of fields. + // These can be fast-path'd during lookup and don't need to be added to the + // tables. + // We use uint16_t to save space for sequential_field_limit_, so stop before + // overflowing it. Worst case, we are not taking full advantage on huge + // messages, but it is unlikely. + result->sequential_field_limit_ = 0; + for (int i = 0; i < std::numeric_limits::max() && + i < proto.field_size() && proto.field(i).number() == i + 1; + ++i) { + result->sequential_field_limit_ = i + 1; + } // Build oneofs first so that fields and extension ranges can refer to them. - BUILD_ARRAY(proto, result, oneof_decl , BuildOneof , result); - BUILD_ARRAY(proto, result, field , BuildField , result); - BUILD_ARRAY(proto, result, nested_type , BuildMessage , result); - BUILD_ARRAY(proto, result, enum_type , BuildEnum , result); + BUILD_ARRAY(proto, result, oneof_decl, BuildOneof, result); + BUILD_ARRAY(proto, result, field, BuildField, result); + BUILD_ARRAY(proto, result, nested_type, BuildMessage, result); + BUILD_ARRAY(proto, result, enum_type, BuildEnum, result); BUILD_ARRAY(proto, result, extension_range, BuildExtensionRange, result); - BUILD_ARRAY(proto, result, extension , BuildExtension , result); - BUILD_ARRAY(proto, result, reserved_range , BuildReservedRange , result); + BUILD_ARRAY(proto, result, extension, BuildExtension, result); + BUILD_ARRAY(proto, result, reserved_range, BuildReservedRange, result); // Copy reserved names. int reserved_name_count = proto.reserved_name_size(); result->reserved_name_count_ = reserved_name_count; result->reserved_names_ = - tables_->AllocateArray(reserved_name_count); + tables_->AllocateArray(reserved_name_count); for (int i = 0; i < reserved_name_count; ++i) { result->reserved_names_[i] = tables_->AllocateString(proto.reserved_name(i)); } // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { - AllocateOptions(proto.options(), result); + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { + AllocateOptions(proto.options(), result, + DescriptorProto::kOptionsFieldNumber, + "google.protobuf.MessageOptions"); } - AddSymbol(result->full_name(), parent, result->name(), - proto, Symbol(result)); + AddSymbol(result->full_name(), parent, result->name(), proto, Symbol(result)); for (int i = 0; i < proto.reserved_range_size(); i++) { const DescriptorProto_ReservedRange& range1 = proto.reserved_range(i); @@ -4532,37 +5244,37 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, AddError(result->full_name(), proto.reserved_range(i), DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Reserved range $0 to $1 overlaps with " - "already-defined range $2 to $3.", - range2.start(), range2.end() - 1, - range1.start(), range1.end() - 1)); + "already-defined range $2 to $3.", + range2.start(), range2.end() - 1, + range1.start(), range1.end() - 1)); } } } - hash_set reserved_name_set; + HASH_SET reserved_name_set; for (int i = 0; i < proto.reserved_name_size(); i++) { - const string& name = proto.reserved_name(i); + const std::string& name = proto.reserved_name(i); if (reserved_name_set.find(name) == reserved_name_set.end()) { reserved_name_set.insert(name); } else { AddError(name, proto, DescriptorPool::ErrorCollector::NAME, - strings::Substitute( - "Field name \"$0\" is reserved multiple times.", - name)); + strings::Substitute("Field name \"$0\" is reserved multiple times.", + name)); } } + for (int i = 0; i < result->field_count(); i++) { const FieldDescriptor* field = result->field(i); for (int j = 0; j < result->extension_range_count(); j++) { const Descriptor::ExtensionRange* range = result->extension_range(j); if (range->start <= field->number() && field->number() < range->end) { - AddError(field->full_name(), proto.extension_range(j), - DescriptorPool::ErrorCollector::NUMBER, - strings::Substitute( - "Extension range $0 to $1 includes field \"$2\" ($3).", - range->start, range->end - 1, - field->name(), field->number())); + AddError( + field->full_name(), proto.extension_range(j), + DescriptorPool::ErrorCollector::NUMBER, + strings::Substitute( + "Extension range $0 to $1 includes field \"$2\" ($3).", + range->start, range->end - 1, field->name(), field->number())); } } for (int j = 0; j < result->reserved_range_count(); j++) { @@ -4570,21 +5282,21 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, if (range->start <= field->number() && field->number() < range->end) { AddError(field->full_name(), proto.reserved_range(j), DescriptorPool::ErrorCollector::NUMBER, - strings::Substitute( - "Field \"$0\" uses reserved number $1.", - field->name(), field->number())); + strings::Substitute("Field \"$0\" uses reserved number $1.", + field->name(), field->number())); } } if (reserved_name_set.find(field->name()) != reserved_name_set.end()) { - AddError(field->full_name(), proto.field(i), - DescriptorPool::ErrorCollector::NAME, - strings::Substitute( - "Field name \"$0\" is reserved.", field->name())); + AddError( + field->full_name(), proto.field(i), + DescriptorPool::ErrorCollector::NAME, + strings::Substitute("Field name \"$0\" is reserved.", field->name())); } + } // Check that extension ranges don't overlap and don't include - // reserved field numbers. + // reserved field numbers or names. for (int i = 0; i < result->extension_range_count(); i++) { const Descriptor::ExtensionRange* range1 = result->extension_range(i); for (int j = 0; j < result->reserved_range_count(); j++) { @@ -4593,9 +5305,9 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, AddError(result->full_name(), proto.extension_range(i), DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Extension range $0 to $1 overlaps with " - "reserved range $2 to $3.", - range1->start, range1->end - 1, - range2->start, range2->end - 1)); + "reserved range $2 to $3.", + range1->start, range1->end - 1, range2->start, + range2->end - 1)); } } for (int j = i + 1; j < result->extension_range_count(); j++) { @@ -4604,88 +5316,74 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, AddError(result->full_name(), proto.extension_range(i), DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Extension range $0 to $1 overlaps with " - "already-defined range $2 to $3.", - range2->start, range2->end - 1, - range1->start, range1->end - 1)); + "already-defined range $2 to $3.", + range2->start, range2->end - 1, range1->start, + range1->end - 1)); } } } } - void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, - const Descriptor* parent, + Descriptor* parent, FieldDescriptor* result, bool is_extension) { - const string& scope = (parent == NULL) ? - file_->package() : parent->full_name(); - string* full_name = tables_->AllocateString(scope); - if (!full_name->empty()) full_name->append(1, '.'); - full_name->append(proto.name()); - - ValidateSymbolName(proto.name(), *full_name, proto); - - result->name_ = tables_->AllocateString(proto.name()); - result->full_name_ = full_name; - result->file_ = file_; - result->number_ = proto.number(); + const std::string& scope = + (parent == nullptr) ? file_->package() : parent->full_name(); + + // We allocate all names in a single array, and dedup them. + // We remember the indices for the potentially deduped values. + auto all_names = tables_->AllocateFieldNames( + proto.name(), scope, + proto.has_json_name() ? &proto.json_name() : nullptr); + result->all_names_ = all_names.array; + result->lowercase_name_index_ = all_names.lowercase_index; + result->camelcase_name_index_ = all_names.camelcase_index; + result->json_name_index_ = all_names.json_index; + + ValidateSymbolName(proto.name(), result->full_name(), proto); + + result->file_ = file_; + result->number_ = proto.number(); result->is_extension_ = is_extension; + result->is_oneof_ = false; + result->proto3_optional_ = proto.proto3_optional(); - // If .proto files follow the style guide then the name should already be - // lower-cased. If that's the case we can just reuse the string we already - // allocated rather than allocate a new one. - string lowercase_name(proto.name()); - LowerString(&lowercase_name); - if (lowercase_name == proto.name()) { - result->lowercase_name_ = result->name_; - } else { - result->lowercase_name_ = tables_->AllocateString(lowercase_name); + if (proto.proto3_optional() && + file_->syntax() != FileDescriptor::SYNTAX_PROTO3) { + AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, + "The [proto3_optional=true] option may only be set on proto3" + "fields, not " + + result->full_name()); } - // Don't bother with the above optimization for camel-case names since - // .proto files that follow the guide shouldn't be using names in this - // format, so the optimization wouldn't help much. - result->camelcase_name_ = - tables_->AllocateString(ToCamelCase(proto.name(), - /* lower_first = */ true)); - - if (proto.has_json_name()) { - result->has_json_name_ = true; - result->json_name_ = tables_->AllocateString(proto.json_name()); - } else { - result->has_json_name_ = false; - result->json_name_ = tables_->AllocateString(ToJsonName(proto.name())); - } + result->has_json_name_ = proto.has_json_name(); // Some compilers do not allow static_cast directly between two enum types, // so we must cast to int first. - result->type_ = static_cast( - implicit_cast(proto.type())); + result->type_ = static_cast( + implicit_cast(proto.type())); result->label_ = static_cast( - implicit_cast(proto.label())); + implicit_cast(proto.label())); - // An extension cannot have a required field (b/13365836). - if (result->is_extension_ && - result->label_ == FieldDescriptor::LABEL_REQUIRED) { - AddError(result->full_name(), proto, - // Error location `TYPE`: we would really like to indicate - // `LABEL`, but the `ErrorLocation` enum has no entry for this, and - // we don't necessarily know about all implementations of the - // `ErrorCollector` interface to extend them to handle the new - // error location type properly. - DescriptorPool::ErrorCollector::TYPE, - "Message extensions cannot have required fields."); + if (result->label_ == FieldDescriptor::LABEL_REQUIRED) { + // An extension cannot have a required field (b/13365836). + if (result->is_extension_) { + AddError(result->full_name(), proto, + // Error location `TYPE`: we would really like to indicate + // `LABEL`, but the `ErrorLocation` enum has no entry for this, + // and we don't necessarily know about all implementations of the + // `ErrorCollector` interface to extend them to handle the new + // error location type properly. + DescriptorPool::ErrorCollector::TYPE, + "The extension " + result->full_name() + " cannot be required."); + } } // Some of these may be filled in when cross-linking. - result->containing_type_ = NULL; - result->extension_scope_ = NULL; - result->message_type_ = NULL; - result->enum_type_ = NULL; - result->type_name_ = NULL; - result->type_once_ = NULL; - result->default_value_enum_ = NULL; - result->default_value_enum_name_ = NULL; + result->containing_type_ = nullptr; + result->type_once_ = nullptr; + result->default_value_enum_ = nullptr; result->has_default_value_ = proto.has_default_value(); if (proto.has_default_value() && result->is_repeated()) { @@ -4696,23 +5394,23 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, if (proto.has_type()) { if (proto.has_default_value()) { - char* end_pos = NULL; + char* end_pos = nullptr; switch (result->cpp_type()) { case FieldDescriptor::CPPTYPE_INT32: - result->default_value_int32_ = - strtol(proto.default_value().c_str(), &end_pos, 0); + result->default_value_int32_t_ = + strtol(proto.default_value().c_str(), &end_pos, 0); break; case FieldDescriptor::CPPTYPE_INT64: - result->default_value_int64_ = - strto64(proto.default_value().c_str(), &end_pos, 0); + result->default_value_int64_t_ = + strto64(proto.default_value().c_str(), &end_pos, 0); break; case FieldDescriptor::CPPTYPE_UINT32: - result->default_value_uint32_ = - strtoul(proto.default_value().c_str(), &end_pos, 0); + result->default_value_uint32_t_ = + strtoul(proto.default_value().c_str(), &end_pos, 0); break; case FieldDescriptor::CPPTYPE_UINT64: - result->default_value_uint64_ = - strtou64(proto.default_value().c_str(), &end_pos, 0); + result->default_value_uint64_t_ = + strtou64(proto.default_value().c_str(), &end_pos, 0); break; case FieldDescriptor::CPPTYPE_FLOAT: if (proto.default_value() == "inf") { @@ -4724,7 +5422,7 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, } else if (proto.default_value() == "nan") { result->default_value_float_ = std::numeric_limits::quiet_NaN(); - } else { + } else { result->default_value_float_ = io::SafeDoubleToFloat( io::NoLocaleStrtod(proto.default_value().c_str(), &end_pos)); } @@ -4739,7 +5437,7 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, } else if (proto.default_value() == "nan") { result->default_value_double_ = std::numeric_limits::quiet_NaN(); - } else { + } else { result->default_value_double_ = io::NoLocaleStrtod(proto.default_value().c_str(), &end_pos); } @@ -4757,12 +5455,12 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, break; case FieldDescriptor::CPPTYPE_ENUM: // This will be filled in when cross-linking. - result->default_value_enum_ = NULL; + result->default_value_enum_ = nullptr; break; case FieldDescriptor::CPPTYPE_STRING: if (result->type() == FieldDescriptor::TYPE_BYTES) { result->default_value_string_ = tables_->AllocateString( - UnescapeCEscapeString(proto.default_value())); + UnescapeCEscapeString(proto.default_value())); } else { result->default_value_string_ = tables_->AllocateString(proto.default_value()); @@ -4773,34 +5471,35 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, DescriptorPool::ErrorCollector::DEFAULT_VALUE, "Messages can't have default values."); result->has_default_value_ = false; + result->default_generated_instance_ = nullptr; break; } - if (end_pos != NULL) { - // end_pos is only set non-NULL by the parsers for numeric types, above. - // This checks that the default was non-empty and had no extra junk - // after the end of the number. + if (end_pos != nullptr) { + // end_pos is only set non-null by the parsers for numeric types, + // above. This checks that the default was non-empty and had no extra + // junk after the end of the number. if (proto.default_value().empty() || *end_pos != '\0') { AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::DEFAULT_VALUE, "Couldn't parse default value \"" + proto.default_value() + - "\"."); + "\"."); } } } else { // No explicit default value switch (result->cpp_type()) { case FieldDescriptor::CPPTYPE_INT32: - result->default_value_int32_ = 0; + result->default_value_int32_t_ = 0; break; case FieldDescriptor::CPPTYPE_INT64: - result->default_value_int64_ = 0; + result->default_value_int64_t_ = 0; break; case FieldDescriptor::CPPTYPE_UINT32: - result->default_value_uint32_ = 0; + result->default_value_uint32_t_ = 0; break; case FieldDescriptor::CPPTYPE_UINT64: - result->default_value_uint64_ = 0; + result->default_value_uint64_t_ = 0; break; case FieldDescriptor::CPPTYPE_FLOAT: result->default_value_float_ = 0.0f; @@ -4813,12 +5512,13 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, break; case FieldDescriptor::CPPTYPE_ENUM: // This will be filled in when cross-linking. - result->default_value_enum_ = NULL; + result->default_value_enum_ = nullptr; break; case FieldDescriptor::CPPTYPE_STRING: result->default_value_string_ = &internal::GetEmptyString(); break; case FieldDescriptor::CPPTYPE_MESSAGE: + result->default_generated_instance_ = nullptr; break; } } @@ -4838,15 +5538,15 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, // on extension numbers. AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Field numbers cannot be greater than $0.", - FieldDescriptor::kMaxNumber)); + FieldDescriptor::kMaxNumber)); } else if (result->number() >= FieldDescriptor::kFirstReservedNumber && result->number() <= FieldDescriptor::kLastReservedNumber) { AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, strings::Substitute( - "Field numbers $0 through $1 are reserved for the protocol " - "buffer library implementation.", - FieldDescriptor::kFirstReservedNumber, - FieldDescriptor::kLastReservedNumber)); + "Field numbers $0 through $1 are reserved for the protocol " + "buffer library implementation.", + FieldDescriptor::kFirstReservedNumber, + FieldDescriptor::kLastReservedNumber)); } if (is_extension) { @@ -4856,17 +5556,13 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, "FieldDescriptorProto.extendee not set for extension field."); } - result->extension_scope_ = parent; + result->scope_.extension_scope = parent; if (proto.has_oneof_index()) { - AddError(result->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "FieldDescriptorProto.oneof_index should not be set for " "extensions."); } - - // Fill in later (maybe). - result->containing_oneof_ = NULL; } else { if (proto.has_extendee()) { AddError(result->full_name(), proto, @@ -4880,41 +5576,37 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, if (proto.oneof_index() < 0 || proto.oneof_index() >= parent->oneof_decl_count()) { AddError(result->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + DescriptorPool::ErrorCollector::TYPE, strings::Substitute("FieldDescriptorProto.oneof_index $0 is " - "out of range for type \"$1\".", - proto.oneof_index(), - parent->name())); - result->containing_oneof_ = NULL; + "out of range for type \"$1\".", + proto.oneof_index(), parent->name())); } else { - result->containing_oneof_ = parent->oneof_decl(proto.oneof_index()); + result->is_oneof_ = true; + result->scope_.containing_oneof = + parent->oneof_decl(proto.oneof_index()); } - } else { - result->containing_oneof_ = NULL; } } // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { - AllocateOptions(proto.options(), result); + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { + AllocateOptions(proto.options(), result, + FieldDescriptorProto::kOptionsFieldNumber, + "google.protobuf.FieldOptions"); } - AddSymbol(result->full_name(), parent, result->name(), - proto, Symbol(result)); + AddSymbol(result->full_name(), parent, result->name(), proto, Symbol(result)); } void DescriptorBuilder::BuildExtensionRange( - const DescriptorProto::ExtensionRange& proto, - const Descriptor* parent, + const DescriptorProto::ExtensionRange& proto, const Descriptor* parent, Descriptor::ExtensionRange* result) { result->start = proto.start(); result->end = proto.end(); if (result->start <= 0) { - AddError(parent->full_name(), proto, - DescriptorPool::ErrorCollector::NUMBER, + AddError(parent->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, "Extension numbers must be positive integers."); } @@ -4924,28 +5616,34 @@ void DescriptorBuilder::BuildExtensionRange( // numbers are actually used as int32s in the message_set_wire_format. if (result->start >= result->end) { - AddError(parent->full_name(), proto, - DescriptorPool::ErrorCollector::NUMBER, + AddError(parent->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, "Extension range end number must be greater than start number."); } - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { + std::vector options_path; + parent->GetLocationPath(&options_path); + options_path.push_back(DescriptorProto::kExtensionRangeFieldNumber); + // find index of this extension range in order to compute path + int index; + for (index = 0; parent->extension_ranges_ + index != result; index++) { + } + options_path.push_back(index); + options_path.push_back(DescriptorProto_ExtensionRange::kOptionsFieldNumber); AllocateOptionsImpl(parent->full_name(), parent->full_name(), - proto.options(), result); + proto.options(), result, options_path, + "google.protobuf.ExtensionRangeOptions"); } } void DescriptorBuilder::BuildReservedRange( - const DescriptorProto::ReservedRange& proto, - const Descriptor* parent, + const DescriptorProto::ReservedRange& proto, const Descriptor* parent, Descriptor::ReservedRange* result) { result->start = proto.start(); result->end = proto.end(); if (result->start <= 0) { - AddError(parent->full_name(), proto, - DescriptorPool::ErrorCollector::NUMBER, + AddError(parent->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, "Reserved numbers must be positive integers."); } } @@ -4957,8 +5655,7 @@ void DescriptorBuilder::BuildReservedRange( result->end = proto.end(); if (result->start > result->end) { - AddError(parent->full_name(), proto, - DescriptorPool::ErrorCollector::NUMBER, + AddError(parent->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, "Reserved range end number must be greater than start number."); } } @@ -4966,30 +5663,24 @@ void DescriptorBuilder::BuildReservedRange( void DescriptorBuilder::BuildOneof(const OneofDescriptorProto& proto, Descriptor* parent, OneofDescriptor* result) { - string* full_name = tables_->AllocateString(parent->full_name()); - full_name->append(1, '.'); - full_name->append(proto.name()); - - ValidateSymbolName(proto.name(), *full_name, proto); - - result->name_ = tables_->AllocateString(proto.name()); - result->full_name_ = full_name; + result->all_names_ = AllocateNameStrings(parent->full_name(), proto.name()); + ValidateSymbolName(proto.name(), result->full_name(), proto); result->containing_type_ = parent; // We need to fill these in later. result->field_count_ = 0; - result->fields_ = NULL; + result->fields_ = nullptr; + result->options_ = nullptr; // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { - AllocateOptions(proto.options(), result); + if (proto.has_options()) { + AllocateOptions(proto.options(), result, + OneofDescriptorProto::kOptionsFieldNumber, + "google.protobuf.OneofOptions"); } - AddSymbol(result->full_name(), parent, result->name(), - proto, Symbol(result)); + AddSymbol(result->full_name(), parent, result->name(), proto, Symbol(result)); } void DescriptorBuilder::CheckEnumValueUniqueness( @@ -5021,13 +5712,12 @@ void DescriptorBuilder::CheckEnumValueUniqueness( // NAME_TYPE_LAST_NAME = 2, // } PrefixRemover remover(result->name()); - std::map values; + std::map values; for (int i = 0; i < result->value_count(); i++) { - const google::protobuf::EnumValueDescriptor* value = result->value(i); - string stripped = + const EnumValueDescriptor* value = result->value(i); + std::string stripped = EnumValueToPascalCase(remover.MaybeRemove(value->name())); - std::pair::iterator, - bool> + std::pair::iterator, bool> insert_result = values.insert(std::make_pair(stripped, value)); bool inserted = insert_result.second; @@ -5039,11 +5729,13 @@ void DescriptorBuilder::CheckEnumValueUniqueness( // stripping should de-dup the labels in this case). if (!inserted && insert_result.first->second->name() != value->name() && insert_result.first->second->number() != value->number()) { - string error_message = - "When enum name is stripped and label is PascalCased (" + stripped + - "), this value label conflicts with " + values[stripped]->name() + - ". This will make the proto fail to compile for some languages, such " - "as C#."; + std::string error_message = + "Enum name " + value->name() + " has the same name as " + + values[stripped]->name() + + " if you ignore case and strip out the enum name prefix (if any). " + "This is error-prone and can lead to undefined behavior. " + "Please avoid doing this. If you are using allow_alias, please " + "assign the same numeric value to both enums."; // There are proto2 enums out there with conflicting names, so to preserve // compatibility we issue only a warning for proto2. if (result->file()->syntax() == FileDescriptor::SYNTAX_PROTO2) { @@ -5060,29 +5752,38 @@ void DescriptorBuilder::CheckEnumValueUniqueness( void DescriptorBuilder::BuildEnum(const EnumDescriptorProto& proto, const Descriptor* parent, EnumDescriptor* result) { - const string& scope = (parent == NULL) ? - file_->package() : parent->full_name(); - string* full_name = tables_->AllocateString(scope); - if (!full_name->empty()) full_name->append(1, '.'); - full_name->append(proto.name()); - - ValidateSymbolName(proto.name(), *full_name, proto); + const std::string& scope = + (parent == nullptr) ? file_->package() : parent->full_name(); - result->name_ = tables_->AllocateString(proto.name()); - result->full_name_ = full_name; - result->file_ = file_; + result->all_names_ = AllocateNameStrings(scope, proto.name()); + ValidateSymbolName(proto.name(), result->full_name(), proto); + result->file_ = file_; result->containing_type_ = parent; - result->is_placeholder_ = false; + result->is_placeholder_ = false; result->is_unqualified_placeholder_ = false; if (proto.value_size() == 0) { // We cannot allow enums with no values because this would mean there // would be no valid default value for fields of this type. - AddError(result->full_name(), proto, - DescriptorPool::ErrorCollector::NAME, + AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Enums must contain at least one value."); } + // Calculate the continuous sequence of the labels. + // These can be fast-path'd during lookup and don't need to be added to the + // tables. + // We use uint16_t to save space for sequential_value_limit_, so stop before + // overflowing it. Worst case, we are not taking full advantage on huge + // enums, but it is unlikely. + for (int i = 0; + i < std::numeric_limits::max() && i < proto.value_size() && + // We do the math in int64_t to avoid overflows. + proto.value(i).number() == + static_cast(i) + proto.value(0).number(); + ++i) { + result->sequential_value_limit_ = i; + } + BUILD_ARRAY(proto, result, value, BuildEnumValue, result); BUILD_ARRAY(proto, result, reserved_range, BuildReservedRange, result); @@ -5090,7 +5791,7 @@ void DescriptorBuilder::BuildEnum(const EnumDescriptorProto& proto, int reserved_name_count = proto.reserved_name_size(); result->reserved_name_count_ = reserved_name_count; result->reserved_names_ = - tables_->AllocateArray(reserved_name_count); + tables_->AllocateArray(reserved_name_count); for (int i = 0; i < reserved_name_count; ++i) { result->reserved_names_[i] = tables_->AllocateString(proto.reserved_name(i)); @@ -5099,14 +5800,14 @@ void DescriptorBuilder::BuildEnum(const EnumDescriptorProto& proto, CheckEnumValueUniqueness(proto, result); // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { - AllocateOptions(proto.options(), result); + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { + AllocateOptions(proto.options(), result, + EnumDescriptorProto::kOptionsFieldNumber, + "google.protobuf.EnumOptions"); } - AddSymbol(result->full_name(), parent, result->name(), - proto, Symbol(result)); + AddSymbol(result->full_name(), parent, result->name(), proto, Symbol(result)); for (int i = 0; i < proto.reserved_range_size(); i++) { const EnumDescriptorProto_EnumReservedRange& range1 = @@ -5114,27 +5815,26 @@ void DescriptorBuilder::BuildEnum(const EnumDescriptorProto& proto, for (int j = i + 1; j < proto.reserved_range_size(); j++) { const EnumDescriptorProto_EnumReservedRange& range2 = proto.reserved_range(j); - if (range1.end() > range2.start() && range2.end() > range1.start()) { + if (range1.end() >= range2.start() && range2.end() >= range1.start()) { AddError(result->full_name(), proto.reserved_range(i), DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Reserved range $0 to $1 overlaps with " - "already-defined range $2 to $3.", - range2.start(), range2.end() - 1, - range1.start(), range1.end() - 1)); + "already-defined range $2 to $3.", + range2.start(), range2.end(), range1.start(), + range1.end())); } } } - hash_set reserved_name_set; + HASH_SET reserved_name_set; for (int i = 0; i < proto.reserved_name_size(); i++) { - const string& name = proto.reserved_name(i); + const std::string& name = proto.reserved_name(i); if (reserved_name_set.find(name) == reserved_name_set.end()) { reserved_name_set.insert(name); } else { AddError(name, proto, DescriptorPool::ErrorCollector::NAME, - strings::Substitute( - "Enum value \"$0\" is reserved multiple times.", - name)); + strings::Substitute("Enum value \"$0\" is reserved multiple times.", + name)); } } @@ -5145,16 +5845,15 @@ void DescriptorBuilder::BuildEnum(const EnumDescriptorProto& proto, if (range->start <= value->number() && value->number() <= range->end) { AddError(value->full_name(), proto.reserved_range(j), DescriptorPool::ErrorCollector::NUMBER, - strings::Substitute( - "Enum value \"$0\" uses reserved number $1.", - value->name(), value->number())); + strings::Substitute("Enum value \"$0\" uses reserved number $1.", + value->name(), value->number())); } } if (reserved_name_set.find(value->name()) != reserved_name_set.end()) { - AddError(value->full_name(), proto.value(i), - DescriptorPool::ErrorCollector::NAME, - strings::Substitute( - "Enum value \"$0\" is reserved.", value->name())); + AddError( + value->full_name(), proto.value(i), + DescriptorPool::ErrorCollector::NAME, + strings::Substitute("Enum value \"$0\" is reserved.", value->name())); } } } @@ -5162,46 +5861,49 @@ void DescriptorBuilder::BuildEnum(const EnumDescriptorProto& proto, void DescriptorBuilder::BuildEnumValue(const EnumValueDescriptorProto& proto, const EnumDescriptor* parent, EnumValueDescriptor* result) { - result->name_ = tables_->AllocateString(proto.name()); - result->number_ = proto.number(); - result->type_ = parent; - // Note: full_name for enum values is a sibling to the parent's name, not a // child of it. - string* full_name = tables_->AllocateString(*parent->full_name_); - full_name->resize(full_name->size() - parent->name_->size()); - full_name->append(*result->name_); - result->full_name_ = full_name; + std::string full_name; + size_t scope_len = parent->full_name().size() - parent->name().size(); + full_name.reserve(scope_len + proto.name().size()); + full_name.append(parent->full_name().data(), scope_len); + full_name.append(proto.name()); + + result->all_names_ = + tables_->AllocateStringArray(proto.name(), std::move(full_name)); + result->number_ = proto.number(); + result->type_ = parent; - ValidateSymbolName(proto.name(), *full_name, proto); + ValidateSymbolName(proto.name(), result->full_name(), proto); // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { - AllocateOptions(proto.options(), result); + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { + AllocateOptions(proto.options(), result, + EnumValueDescriptorProto::kOptionsFieldNumber, + "google.protobuf.EnumValueOptions"); } // Again, enum values are weird because we makes them appear as siblings // of the enum type instead of children of it. So, we use // parent->containing_type() as the value's parent. bool added_to_outer_scope = - AddSymbol(result->full_name(), parent->containing_type(), result->name(), - proto, Symbol(result)); + AddSymbol(result->full_name(), parent->containing_type(), result->name(), + proto, Symbol::EnumValue(result, 0)); // However, we also want to be able to search for values within a single // enum type, so we add it as a child of the enum type itself, too. // Note: This could fail, but if it does, the error has already been // reported by the above AddSymbol() call, so we ignore the return code. - bool added_to_inner_scope = - file_tables_->AddAliasUnderParent(parent, result->name(), Symbol(result)); + bool added_to_inner_scope = file_tables_->AddAliasUnderParent( + parent, result->name(), Symbol::EnumValue(result, 1)); if (added_to_inner_scope && !added_to_outer_scope) { // This value did not conflict with any values defined in the same enum, // but it did conflict with some other symbol defined in the enum type's // scope. Let's print an additional error to explain this. - string outer_scope; - if (parent->containing_type() == NULL) { + std::string outer_scope; + if (parent->containing_type() == nullptr) { outer_scope = file_->package(); } else { outer_scope = parent->containing_type()->full_name(); @@ -5213,12 +5915,12 @@ void DescriptorBuilder::BuildEnumValue(const EnumValueDescriptorProto& proto, outer_scope = "\"" + outer_scope + "\""; } - AddError(result->full_name(), proto, - DescriptorPool::ErrorCollector::NAME, + AddError(result->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Note that enum values use C++ scoping rules, meaning that " "enum values are siblings of their type, not children of it. " - "Therefore, \"" + result->name() + "\" must be unique within " - + outer_scope + ", not just within \"" + parent->name() + "\"."); + "Therefore, \"" + + result->name() + "\" must be unique within " + outer_scope + + ", not just within \"" + parent->name() + "\"."); } // An enum is allowed to define two numbers that refer to the same value. @@ -5230,67 +5932,57 @@ void DescriptorBuilder::BuildEnumValue(const EnumValueDescriptorProto& proto, void DescriptorBuilder::BuildService(const ServiceDescriptorProto& proto, const void* /* dummy */, ServiceDescriptor* result) { - string* full_name = tables_->AllocateString(file_->package()); - if (!full_name->empty()) full_name->append(1, '.'); - full_name->append(proto.name()); - - ValidateSymbolName(proto.name(), *full_name, proto); - - result->name_ = tables_->AllocateString(proto.name()); - result->full_name_ = full_name; - result->file_ = file_; + result->all_names_ = AllocateNameStrings(file_->package(), proto.name()); + result->file_ = file_; + ValidateSymbolName(proto.name(), result->full_name(), proto); BUILD_ARRAY(proto, result, method, BuildMethod, result); // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { - AllocateOptions(proto.options(), result); + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { + AllocateOptions(proto.options(), result, + ServiceDescriptorProto::kOptionsFieldNumber, + "google.protobuf.ServiceOptions"); } - AddSymbol(result->full_name(), NULL, result->name(), - proto, Symbol(result)); + AddSymbol(result->full_name(), nullptr, result->name(), proto, + Symbol(result)); } void DescriptorBuilder::BuildMethod(const MethodDescriptorProto& proto, const ServiceDescriptor* parent, MethodDescriptor* result) { - result->name_ = tables_->AllocateString(proto.name()); result->service_ = parent; + result->all_names_ = AllocateNameStrings(parent->full_name(), proto.name()); - string* full_name = tables_->AllocateString(parent->full_name()); - full_name->append(1, '.'); - full_name->append(*result->name_); - result->full_name_ = full_name; - - ValidateSymbolName(proto.name(), *full_name, proto); + ValidateSymbolName(proto.name(), result->full_name(), proto); // These will be filled in when cross-linking. result->input_type_.Init(); result->output_type_.Init(); // Copy options. - if (!proto.has_options()) { - result->options_ = NULL; // Will set to default_instance later. - } else { - AllocateOptions(proto.options(), result); + result->options_ = nullptr; // Set to default_instance later if necessary. + if (proto.has_options()) { + AllocateOptions(proto.options(), result, + MethodDescriptorProto::kOptionsFieldNumber, + "google.protobuf.MethodOptions"); } result->client_streaming_ = proto.client_streaming(); result->server_streaming_ = proto.server_streaming(); - AddSymbol(result->full_name(), parent, result->name(), - proto, Symbol(result)); + AddSymbol(result->full_name(), parent, result->name(), proto, Symbol(result)); } #undef BUILD_ARRAY // ------------------------------------------------------------------- -void DescriptorBuilder::CrossLinkFile( - FileDescriptor* file, const FileDescriptorProto& proto) { - if (file->options_ == NULL) { +void DescriptorBuilder::CrossLinkFile(FileDescriptor* file, + const FileDescriptorProto& proto) { + if (file->options_ == nullptr) { file->options_ = &FileOptions::default_instance(); } @@ -5311,9 +6003,9 @@ void DescriptorBuilder::CrossLinkFile( } } -void DescriptorBuilder::CrossLinkMessage( - Descriptor* message, const DescriptorProto& proto) { - if (message->options_ == NULL) { +void DescriptorBuilder::CrossLinkMessage(Descriptor* message, + const DescriptorProto& proto) { + if (message->options_ == nullptr) { message->options_ = &MessageOptions::default_instance(); } @@ -5343,7 +6035,7 @@ void DescriptorBuilder::CrossLinkMessage( // First count the number of fields per oneof. for (int i = 0; i < message->field_count(); i++) { const OneofDescriptor* oneof_decl = message->field(i)->containing_oneof(); - if (oneof_decl != NULL) { + if (oneof_decl != nullptr) { // Make sure fields belonging to the same oneof are defined consecutively. // This enables optimizations in codegens and reflection libraries to // skip fields in the oneof group, as only one of the field can be set. @@ -5352,65 +6044,95 @@ void DescriptorBuilder::CrossLinkMessage( // safe. if (oneof_decl->field_count() > 0 && message->field(i - 1)->containing_oneof() != oneof_decl) { - AddError( - message->full_name() + "." + message->field(i - 1)->name(), - proto.field(i - 1), DescriptorPool::ErrorCollector::OTHER, - strings::Substitute( - "Fields in the same oneof must be defined consecutively. " - "\"$0\" cannot be defined before the completion of the " - "\"$1\" oneof definition.", - message->field(i - 1)->name(), oneof_decl->name())); + AddError(message->full_name() + "." + message->field(i - 1)->name(), + proto.field(i - 1), DescriptorPool::ErrorCollector::TYPE, + strings::Substitute( + "Fields in the same oneof must be defined consecutively. " + "\"$0\" cannot be defined before the completion of the " + "\"$1\" oneof definition.", + message->field(i - 1)->name(), oneof_decl->name())); } // Must go through oneof_decls_ array to get a non-const version of the // OneofDescriptor. - ++message->oneof_decls_[oneof_decl->index()].field_count_; + auto& out_oneof_decl = message->oneof_decls_[oneof_decl->index()]; + if (out_oneof_decl.field_count_ == 0) { + out_oneof_decl.fields_ = message->field(i); + } + + if (!had_errors_) { + // Verify that they are contiguous. + // This is assumed by OneofDescriptor::field(i). + // But only if there are no errors. + GOOGLE_CHECK_EQ(out_oneof_decl.fields_ + out_oneof_decl.field_count_, + message->field(i)); + } + ++out_oneof_decl.field_count_; } } - // Then allocate the arrays. + // Then verify the sizes. for (int i = 0; i < message->oneof_decl_count(); i++) { OneofDescriptor* oneof_decl = &message->oneof_decls_[i]; if (oneof_decl->field_count() == 0) { AddError(message->full_name() + "." + oneof_decl->name(), - proto.oneof_decl(i), - DescriptorPool::ErrorCollector::NAME, + proto.oneof_decl(i), DescriptorPool::ErrorCollector::NAME, "Oneof must have at least one field."); } - oneof_decl->fields_ = - tables_->AllocateArray(oneof_decl->field_count_); - oneof_decl->field_count_ = 0; - - if (oneof_decl->options_ == NULL) { + if (oneof_decl->options_ == nullptr) { oneof_decl->options_ = &OneofOptions::default_instance(); } } - // Then fill them in. for (int i = 0; i < message->field_count(); i++) { - const OneofDescriptor* oneof_decl = message->field(i)->containing_oneof(); - if (oneof_decl != NULL) { - OneofDescriptor* mutable_oneof_decl = - &message->oneof_decls_[oneof_decl->index()]; - message->fields_[i].index_in_oneof_ = mutable_oneof_decl->field_count_; - mutable_oneof_decl->fields_[mutable_oneof_decl->field_count_++] = - message->field(i); + const FieldDescriptor* field = message->field(i); + if (field->proto3_optional_) { + if (!field->containing_oneof() || + !field->containing_oneof()->is_synthetic()) { + AddError(message->full_name(), proto.field(i), + DescriptorPool::ErrorCollector::OTHER, + "Fields with proto3_optional set must be " + "a member of a one-field oneof"); + } + } + } + + // Synthetic oneofs must be last. + int first_synthetic = -1; + for (int i = 0; i < message->oneof_decl_count(); i++) { + const OneofDescriptor* oneof = message->oneof_decl(i); + if (oneof->is_synthetic()) { + if (first_synthetic == -1) { + first_synthetic = i; + } + } else { + if (first_synthetic != -1) { + AddError(message->full_name(), proto.oneof_decl(i), + DescriptorPool::ErrorCollector::OTHER, + "Synthetic oneofs must be after all other oneofs"); + } } } + + if (first_synthetic == -1) { + message->real_oneof_decl_count_ = message->oneof_decl_count_; + } else { + message->real_oneof_decl_count_ = first_synthetic; + } } void DescriptorBuilder::CrossLinkExtensionRange( Descriptor::ExtensionRange* range, - const DescriptorProto::ExtensionRange& proto) { - if (range->options_ == NULL) { + const DescriptorProto::ExtensionRange& /*proto*/) { + if (range->options_ == nullptr) { range->options_ = &ExtensionRangeOptions::default_instance(); } } -void DescriptorBuilder::CrossLinkField( - FieldDescriptor* field, const FieldDescriptorProto& proto) { - if (field->options_ == NULL) { +void DescriptorBuilder::CrossLinkField(FieldDescriptor* field, + const FieldDescriptorProto& proto) { + if (field->options_ == nullptr) { field->options_ = &FieldOptions::default_instance(); } @@ -5426,34 +6148,41 @@ void DescriptorBuilder::CrossLinkField( DescriptorPool::ErrorCollector::EXTENDEE, proto.extendee()); return; - } else if (extendee.type != Symbol::MESSAGE) { + } else if (extendee.type() != Symbol::MESSAGE) { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::EXTENDEE, "\"" + proto.extendee() + "\" is not a message type."); return; } - field->containing_type_ = extendee.descriptor; + field->containing_type_ = extendee.descriptor(); - const Descriptor::ExtensionRange* extension_range = field->containing_type() - ->FindExtensionRangeContainingNumber(field->number()); + const Descriptor::ExtensionRange* extension_range = + field->containing_type()->FindExtensionRangeContainingNumber( + field->number()); - if (extension_range == NULL) { - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::NUMBER, - strings::Substitute("\"$0\" does not declare $1 as an " - "extension number.", - field->containing_type()->full_name(), - field->number())); + if (extension_range == nullptr) { + // Set of valid extension numbers for MessageSet is different (< 2^32) + // from other extendees (< 2^29). If unknown deps are allowed, we may not + // have that information, and wrongly deem the extension as invalid. + auto skip_check = get_allow_unknown(pool_) && + proto.extendee() == "google.protobuf.bridge.MessageSet"; + if (!skip_check) { + AddError(field->full_name(), proto, + DescriptorPool::ErrorCollector::NUMBER, + strings::Substitute("\"$0\" does not declare $1 as an " + "extension number.", + field->containing_type()->full_name(), + field->number())); + } } } - if (field->containing_oneof() != NULL) { + if (field->containing_oneof() != nullptr) { if (field->label() != FieldDescriptor::LABEL_OPTIONAL) { // Note that this error will never happen when parsing .proto files. // It can only happen if you manually construct a FileDescriptorProto // that is incorrect. - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::NAME, + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Fields of oneofs must themselves have label LABEL_OPTIONAL."); } } @@ -5466,7 +6195,7 @@ void DescriptorBuilder::CrossLinkField( proto.has_default_value(); // In case of weak fields we force building the dependency. We need to know - // if the type exist or not. If it doesnt exist we substitute Empty which + // if the type exist or not. If it doesn't exist we substitute Empty which // should only be done if the type can't be found in the generated pool. // TODO(gerbens) Ideally we should query the database directly to check // if weak fields exist or not so that we don't need to force building @@ -5485,13 +6214,13 @@ void DescriptorBuilder::CrossLinkField( if (is_lazy) { // Save the symbol names for later for lookup, and allocate the once // object needed for the accessors. - string name = proto.type_name(); - field->type_once_ = tables_->AllocateOnceDynamic(); - field->type_name_ = tables_->AllocateString(name); - if (proto.has_default_value()) { - field->default_value_enum_name_ = - tables_->AllocateString(proto.default_value()); - } + std::string name = proto.type_name(); + field->type_once_ = tables_->Create(); + field->type_descriptor_.lazy_type_name = tables_->Strdup(name); + field->lazy_default_value_enum_name_ = + proto.has_default_value() ? tables_->Strdup(proto.default_value()) + : nullptr; + // AddFieldByNumber and AddExtension are done later in this function, // and can/must be done if the field type was not found. The related // error checking is not necessary when in lazily_build_dependencies_ @@ -5519,9 +6248,9 @@ void DescriptorBuilder::CrossLinkField( if (!proto.has_type()) { // Choose field type based on symbol. - if (type.type == Symbol::MESSAGE) { + if (type.type() == Symbol::MESSAGE) { field->type_ = FieldDescriptor::TYPE_MESSAGE; - } else if (type.type == Symbol::ENUM) { + } else if (type.type() == Symbol::ENUM) { field->type_ = FieldDescriptor::TYPE_ENUM; } else { AddError(field->full_name(), proto, @@ -5532,13 +6261,13 @@ void DescriptorBuilder::CrossLinkField( } if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { - if (type.type != Symbol::MESSAGE) { + field->type_descriptor_.message_type = type.descriptor(); + if (field->type_descriptor_.message_type == nullptr) { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "\"" + proto.type_name() + "\" is not a message type."); return; } - field->message_type_ = type.descriptor; if (field->has_default_value()) { AddError(field->full_name(), proto, @@ -5546,13 +6275,13 @@ void DescriptorBuilder::CrossLinkField( "Messages can't have default values."); } } else if (field->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) { - if (type.type != Symbol::ENUM) { + field->type_descriptor_.enum_type = type.enum_descriptor(); + if (field->type_descriptor_.enum_type == nullptr) { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "\"" + proto.type_name() + "\" is not an enum type."); return; } - field->enum_type_ = type.enum_descriptor; if (field->enum_type()->is_placeholder_) { // We can't look up default values for placeholder types. We'll have @@ -5574,20 +6303,20 @@ void DescriptorBuilder::CrossLinkField( // We can't just use field->enum_type()->FindValueByName() here // because that locks the pool's mutex, which we have already locked // at this point. - Symbol default_value = - LookupSymbolNoPlaceholder(proto.default_value(), - field->enum_type()->full_name()); - - if (default_value.type == Symbol::ENUM_VALUE && - default_value.enum_value_descriptor->type() == - field->enum_type()) { - field->default_value_enum_ = default_value.enum_value_descriptor; + const EnumValueDescriptor* default_value = + LookupSymbolNoPlaceholder(proto.default_value(), + field->enum_type()->full_name()) + .enum_value_descriptor(); + + if (default_value != nullptr && + default_value->type() == field->enum_type()) { + field->default_value_enum_ = default_value; } else { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::DEFAULT_VALUE, "Enum type \"" + field->enum_type()->full_name() + - "\" has no value named \"" + proto.default_value() + - "\"."); + "\" has no value named \"" + proto.default_value() + + "\"."); } } } else if (field->enum_type()->value_count() > 0) { @@ -5615,41 +6344,41 @@ void DescriptorBuilder::CrossLinkField( // risk to calling containing_type() or other accessors that will build // dependencies. if (!file_tables_->AddFieldByNumber(field)) { - const FieldDescriptor* conflicting_field = - file_tables_->FindFieldByNumber(field->containing_type(), - field->number()); - string containing_type_name = field->containing_type() == NULL - ? "unknown" - : field->containing_type()->full_name(); + const FieldDescriptor* conflicting_field = file_tables_->FindFieldByNumber( + field->containing_type(), field->number()); + std::string containing_type_name = + field->containing_type() == nullptr + ? "unknown" + : field->containing_type()->full_name(); if (field->is_extension()) { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Extension number $0 has already been used " - "in \"$1\" by extension \"$2\".", - field->number(), - containing_type_name, - conflicting_field->full_name())); + "in \"$1\" by extension \"$2\".", + field->number(), containing_type_name, + conflicting_field->full_name())); } else { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::NUMBER, strings::Substitute("Field number $0 has already been used in " - "\"$1\" by field \"$2\".", - field->number(), - containing_type_name, - conflicting_field->name())); + "\"$1\" by field \"$2\".", + field->number(), containing_type_name, + conflicting_field->name())); } } else { if (field->is_extension()) { if (!tables_->AddExtension(field)) { const FieldDescriptor* conflicting_field = tables_->FindExtension(field->containing_type(), field->number()); - string error_msg = strings::Substitute( + std::string containing_type_name = + field->containing_type() == nullptr + ? "unknown" + : field->containing_type()->full_name(); + std::string error_msg = strings::Substitute( "Extension number $0 has already been used in \"$1\" by extension " "\"$2\" defined in $3.", - field->number(), - field->containing_type()->full_name(), - conflicting_field->full_name(), - conflicting_field->file()->name()); + field->number(), containing_type_name, + conflicting_field->full_name(), conflicting_field->file()->name()); // Conflicting extension numbers should be an error. However, before // turning this into an error we need to fix all existing broken // protos first. @@ -5661,9 +6390,9 @@ void DescriptorBuilder::CrossLinkField( } } -void DescriptorBuilder::CrossLinkEnum( - EnumDescriptor* enum_type, const EnumDescriptorProto& proto) { - if (enum_type->options_ == NULL) { +void DescriptorBuilder::CrossLinkEnum(EnumDescriptor* enum_type, + const EnumDescriptorProto& proto) { + if (enum_type->options_ == nullptr) { enum_type->options_ = &EnumOptions::default_instance(); } @@ -5675,14 +6404,14 @@ void DescriptorBuilder::CrossLinkEnum( void DescriptorBuilder::CrossLinkEnumValue( EnumValueDescriptor* enum_value, const EnumValueDescriptorProto& /* proto */) { - if (enum_value->options_ == NULL) { + if (enum_value->options_ == nullptr) { enum_value->options_ = &EnumValueOptions::default_instance(); } } -void DescriptorBuilder::CrossLinkService( - ServiceDescriptor* service, const ServiceDescriptorProto& proto) { - if (service->options_ == NULL) { +void DescriptorBuilder::CrossLinkService(ServiceDescriptor* service, + const ServiceDescriptorProto& proto) { + if (service->options_ == nullptr) { service->options_ = &ServiceOptions::default_instance(); } @@ -5691,9 +6420,9 @@ void DescriptorBuilder::CrossLinkService( } } -void DescriptorBuilder::CrossLinkMethod( - MethodDescriptor* method, const MethodDescriptorProto& proto) { - if (method->options_ == NULL) { +void DescriptorBuilder::CrossLinkMethod(MethodDescriptor* method, + const MethodDescriptorProto& proto) { + if (method->options_ == nullptr) { method->options_ = &MethodOptions::default_instance(); } @@ -5709,12 +6438,12 @@ void DescriptorBuilder::CrossLinkMethod( } else { method->input_type_.SetLazy(proto.input_type(), file_); } - } else if (input_type.type != Symbol::MESSAGE) { + } else if (input_type.type() != Symbol::MESSAGE) { AddError(method->full_name(), proto, DescriptorPool::ErrorCollector::INPUT_TYPE, "\"" + proto.input_type() + "\" is not a message type."); } else { - method->input_type_.Set(input_type.descriptor); + method->input_type_.Set(input_type.descriptor()); } Symbol output_type = @@ -5729,21 +6458,21 @@ void DescriptorBuilder::CrossLinkMethod( } else { method->output_type_.SetLazy(proto.output_type(), file_); } - } else if (output_type.type != Symbol::MESSAGE) { + } else if (output_type.type() != Symbol::MESSAGE) { AddError(method->full_name(), proto, DescriptorPool::ErrorCollector::OUTPUT_TYPE, "\"" + proto.output_type() + "\" is not a message type."); } else { - method->output_type_.Set(output_type.descriptor); + method->output_type_.Set(output_type.descriptor()); } } // ------------------------------------------------------------------- -#define VALIDATE_OPTIONS_FROM_ARRAY(descriptor, array_name, type) \ - for (int i = 0; i < descriptor->array_name##_count(); ++i) { \ - Validate##type##Options(descriptor->array_name##s_ + i, \ - proto.array_name(i)); \ +#define VALIDATE_OPTIONS_FROM_ARRAY(descriptor, array_name, type) \ + for (int i = 0; i < descriptor->array_name##_count(); ++i) { \ + Validate##type##Options(descriptor->array_name##s_ + i, \ + proto.array_name(i)); \ } // Determine if the file uses optimize_for = LITE_RUNTIME, being careful to @@ -5751,7 +6480,7 @@ void DescriptorBuilder::CrossLinkMethod( static bool IsLite(const FileDescriptor* file) { // TODO(kenton): I don't even remember how many of these conditions are // actually possible. I'm just being super-safe. - return file != NULL && + return file != nullptr && &file->options() != &FileOptions::default_instance() && file->options().optimize_for() == FileOptions::LITE_RUNTIME; } @@ -5768,11 +6497,12 @@ void DescriptorBuilder::ValidateFileOptions(FileDescriptor* file, for (int i = 0; i < file->dependency_count(); i++) { if (IsLite(file->dependency(i))) { AddError( - file->name(), proto, - DescriptorPool::ErrorCollector::OTHER, - "Files that do not use optimize_for = LITE_RUNTIME cannot import " - "files which do use this option. This file is not lite, but it " - "imports \"" + file->dependency(i)->name() + "\" which is."); + file->dependency(i)->name(), proto, + DescriptorPool::ErrorCollector::IMPORT, + "Files that do not use optimize_for = LITE_RUNTIME cannot import " + "files which do use this option. This file is not lite, but it " + "imports \"" + + file->dependency(i)->name() + "\" which is."); break; } } @@ -5782,8 +6512,8 @@ void DescriptorBuilder::ValidateFileOptions(FileDescriptor* file, } } -void DescriptorBuilder::ValidateProto3( - FileDescriptor* file, const FileDescriptorProto& proto) { +void DescriptorBuilder::ValidateProto3(FileDescriptor* file, + const FileDescriptorProto& proto) { for (int i = 0; i < file->extension_count(); ++i) { ValidateProto3Field(file->extensions_ + i, proto.extension(i)); } @@ -5795,111 +6525,106 @@ void DescriptorBuilder::ValidateProto3( } } -static string ToLowercaseWithoutUnderscores(const string& name) { - string result; - for (int i = 0; i < name.size(); ++i) { - if (name[i] != '_') { - if (name[i] >= 'A' && name[i] <= 'Z') { - result.push_back(name[i] - 'A' + 'a'); +static std::string ToLowercaseWithoutUnderscores(const std::string& name) { + std::string result; + for (char character : name) { + if (character != '_') { + if (character >= 'A' && character <= 'Z') { + result.push_back(character - 'A' + 'a'); } else { - result.push_back(name[i]); + result.push_back(character); } } } return result; } -void DescriptorBuilder::ValidateProto3Message( - Descriptor* message, const DescriptorProto& proto) { +void DescriptorBuilder::ValidateProto3Message(Descriptor* message, + const DescriptorProto& proto) { for (int i = 0; i < message->nested_type_count(); ++i) { - ValidateProto3Message(message->nested_types_ + i, - proto.nested_type(i)); + ValidateProto3Message(message->nested_types_ + i, proto.nested_type(i)); } for (int i = 0; i < message->enum_type_count(); ++i) { - ValidateProto3Enum(message->enum_types_ + i, - proto.enum_type(i)); + ValidateProto3Enum(message->enum_types_ + i, proto.enum_type(i)); } for (int i = 0; i < message->field_count(); ++i) { ValidateProto3Field(message->fields_ + i, proto.field(i)); } for (int i = 0; i < message->extension_count(); ++i) { - ValidateProto3Field(message->extensions_ +i, proto.extension(i)); + ValidateProto3Field(message->extensions_ + i, proto.extension(i)); } if (message->extension_range_count() > 0) { - AddError(message->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(message->full_name(), proto.extension_range(0), + DescriptorPool::ErrorCollector::NUMBER, "Extension ranges are not allowed in proto3."); } if (message->options().message_set_wire_format()) { // Using MessageSet doesn't make sense since we disallow extensions. - AddError(message->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(message->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "MessageSet is not supported in proto3."); } // In proto3, we reject field names if they conflict in camelCase. // Note that we currently enforce a stricter rule: Field names must be // unique after being converted to lowercase with underscores removed. - std::map name_to_field; + std::map name_to_field; for (int i = 0; i < message->field_count(); ++i) { - string lowercase_name = ToLowercaseWithoutUnderscores( - message->field(i)->name()); + std::string lowercase_name = + ToLowercaseWithoutUnderscores(message->field(i)->name()); if (name_to_field.find(lowercase_name) != name_to_field.end()) { - AddError(message->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(message->full_name(), proto.field(i), + DescriptorPool::ErrorCollector::NAME, "The JSON camel-case name of field \"" + - message->field(i)->name() + "\" conflicts with field \"" + - name_to_field[lowercase_name]->name() + "\". This is not " + - "allowed in proto3."); + message->field(i)->name() + "\" conflicts with field \"" + + name_to_field[lowercase_name]->name() + "\". This is not " + + "allowed in proto3."); } else { name_to_field[lowercase_name] = message->field(i); } } } -void DescriptorBuilder::ValidateProto3Field( - FieldDescriptor* field, const FieldDescriptorProto& proto) { +void DescriptorBuilder::ValidateProto3Field(FieldDescriptor* field, + const FieldDescriptorProto& proto) { if (field->is_extension() && !AllowedExtendeeInProto3(field->containing_type()->full_name())) { AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + DescriptorPool::ErrorCollector::EXTENDEE, "Extensions in proto3 are only allowed for defining options."); } if (field->is_required()) { - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "Required fields are not allowed in proto3."); } if (field->has_default_value()) { - AddError( - field->full_name(), proto, DescriptorPool::ErrorCollector::OTHER, - "Explicit default values are not allowed in proto3."); + AddError(field->full_name(), proto, + DescriptorPool::ErrorCollector::DEFAULT_VALUE, + "Explicit default values are not allowed in proto3."); } if (field->cpp_type() == FieldDescriptor::CPPTYPE_ENUM && field->enum_type() && - field->enum_type()->file()->syntax() != FileDescriptor::SYNTAX_PROTO3) { + field->enum_type()->file()->syntax() != FileDescriptor::SYNTAX_PROTO3 && + field->enum_type()->file()->syntax() != FileDescriptor::SYNTAX_UNKNOWN) { // Proto3 messages can only use Proto3 enum types; otherwise we can't // guarantee that the default value is zero. - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::TYPE, + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "Enum type \"" + field->enum_type()->full_name() + - "\" is not a proto3 enum, but is used in \"" + - field->containing_type()->full_name() + - "\" which is a proto3 message type."); + "\" is not a proto3 enum, but is used in \"" + + field->containing_type()->full_name() + + "\" which is a proto3 message type."); } if (field->type() == FieldDescriptor::TYPE_GROUP) { - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::TYPE, + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "Groups are not supported in proto3 syntax."); } } -void DescriptorBuilder::ValidateProto3Enum( - EnumDescriptor* enm, const EnumDescriptorProto& proto) { +void DescriptorBuilder::ValidateProto3Enum(EnumDescriptor* enm, + const EnumDescriptorProto& proto) { if (enm->value_count() > 0 && enm->value(0)->number() != 0) { - AddError( - enm->full_name(), proto, DescriptorPool::ErrorCollector::OTHER, - "The first enum value must be zero in proto3."); + AddError(enm->full_name(), proto.value(0), + DescriptorPool::ErrorCollector::NUMBER, + "The first enum value must be zero in proto3."); } } @@ -5910,32 +6635,34 @@ void DescriptorBuilder::ValidateMessageOptions(Descriptor* message, VALIDATE_OPTIONS_FROM_ARRAY(message, enum_type, Enum); VALIDATE_OPTIONS_FROM_ARRAY(message, extension, Field); - const int64 max_extension_range = - static_cast(message->options().message_set_wire_format() ? - kint32max : - FieldDescriptor::kMaxNumber); + const int64_t max_extension_range = + static_cast(message->options().message_set_wire_format() + ? std::numeric_limits::max() + : FieldDescriptor::kMaxNumber); for (int i = 0; i < message->extension_range_count(); ++i) { if (message->extension_range(i)->end > max_extension_range + 1) { - AddError( - message->full_name(), proto.extension_range(i), - DescriptorPool::ErrorCollector::NUMBER, - strings::Substitute("Extension numbers cannot be greater than $0.", - max_extension_range)); + AddError(message->full_name(), proto.extension_range(i), + DescriptorPool::ErrorCollector::NUMBER, + strings::Substitute("Extension numbers cannot be greater than $0.", + max_extension_range)); } + + ValidateExtensionRangeOptions(message->full_name(), + message->extension_ranges_ + i, + proto.extension_range(i)); } } -void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field, - const FieldDescriptorProto& proto) { +void DescriptorBuilder::ValidateFieldOptions( + FieldDescriptor* field, const FieldDescriptorProto& proto) { if (pool_->lazily_build_dependencies_ && (!field || !field->message_type())) { return; } // Only message type fields may be lazy. if (field->options().lazy()) { if (field->type() != FieldDescriptor::TYPE_MESSAGE) { - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::TYPE, + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "[lazy = true] can only be specified for submessage fields."); } } @@ -5943,16 +6670,15 @@ void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field, // Only repeated primitive fields may be packed. if (field->options().packed() && !field->is_packable()) { AddError( - field->full_name(), proto, - DescriptorPool::ErrorCollector::TYPE, - "[packed = true] can only be specified for repeated primitive fields."); + field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, + "[packed = true] can only be specified for repeated primitive fields."); } // Note: Default instance may not yet be initialized here, so we have to // avoid reading from it. - if (field->containing_type_ != NULL && + if (field->containing_type_ != nullptr && &field->containing_type()->options() != - &MessageOptions::default_instance() && + &MessageOptions::default_instance() && field->containing_type()->options().message_set_wire_format()) { if (field->is_extension()) { if (!field->is_optional() || @@ -5962,15 +6688,13 @@ void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field, "Extensions of MessageSets must be optional messages."); } } else { - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::NAME, + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "MessageSets cannot have fields, only extensions."); } } // Lite extensions can only be of Lite types. - if (IsLite(field->file()) && - field->containing_type_ != NULL && + if (IsLite(field->file()) && field->containing_type_ != nullptr && !IsLite(field->containing_type()->file())) { AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::EXTENDEE, @@ -5982,8 +6706,7 @@ void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field, // Validate map types. if (field->is_map()) { if (!ValidateMapEntry(field, proto)) { - AddError(field->full_name(), proto, - DescriptorPool::ErrorCollector::OTHER, + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "map_entry should not be set explicitly. Use map instead."); } @@ -5991,30 +6714,43 @@ void DescriptorBuilder::ValidateFieldOptions(FieldDescriptor* field, ValidateJSType(field, proto); + // json_name option is not allowed on extension fields. Note that the + // json_name field in FieldDescriptorProto is always populated by protoc + // when it sends descriptor data to plugins (calculated from field name if + // the option is not explicitly set) so we can't rely on its presence to + // determine whether the json_name option is set on the field. Here we + // compare it against the default calculated json_name value and consider + // the option set if they are different. This won't catch the case when + // an user explicitly sets json_name to the default value, but should be + // good enough to catch common misuses. + if (field->is_extension() && + (field->has_json_name() && + field->json_name() != ToJsonName(field->name()))) { + AddError(field->full_name(), proto, + DescriptorPool::ErrorCollector::OPTION_NAME, + "option json_name is not allowed on extension fields."); + } + } void DescriptorBuilder::ValidateEnumOptions(EnumDescriptor* enm, const EnumDescriptorProto& proto) { VALIDATE_OPTIONS_FROM_ARRAY(enm, value, EnumValue); if (!enm->options().has_allow_alias() || !enm->options().allow_alias()) { - std::map used_values; + std::map used_values; for (int i = 0; i < enm->value_count(); ++i) { const EnumValueDescriptor* enum_value = enm->value(i); if (used_values.find(enum_value->number()) != used_values.end()) { - string error = + std::string error = "\"" + enum_value->full_name() + "\" uses the same enum value as \"" + - used_values[enum_value->number()] + "\". If this is intended, set " + used_values[enum_value->number()] + + "\". If this is intended, set " "'option allow_alias = true;' to the enum definition."; if (!enm->options().allow_alias()) { // Generate error if duplicated enum values are explicitly disallowed. - AddError(enm->full_name(), proto, - DescriptorPool::ErrorCollector::NUMBER, - error); - } else { - // Generate warning if duplicated values are found but the option - // isn't set. - GOOGLE_LOG(ERROR) << error; + AddError(enm->full_name(), proto.value(i), + DescriptorPool::ErrorCollector::NUMBER, error); } } else { used_values[enum_value->number()] = enum_value->full_name(); @@ -6028,31 +6764,38 @@ void DescriptorBuilder::ValidateEnumValueOptions( const EnumValueDescriptorProto& /* proto */) { // Nothing to do so far. } -void DescriptorBuilder::ValidateServiceOptions(ServiceDescriptor* service, - const ServiceDescriptorProto& proto) { + +void DescriptorBuilder::ValidateExtensionRangeOptions( + const std::string& full_name, Descriptor::ExtensionRange* extension_range, + const DescriptorProto_ExtensionRange& proto) { + (void)full_name; // Parameter is used by Google-internal code. + (void)extension_range; // Parameter is used by Google-internal code. +} + +void DescriptorBuilder::ValidateServiceOptions( + ServiceDescriptor* service, const ServiceDescriptorProto& proto) { if (IsLite(service->file()) && (service->file()->options().cc_generic_services() || service->file()->options().java_generic_services())) { - AddError(service->full_name(), proto, - DescriptorPool::ErrorCollector::NAME, + AddError(service->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Files with optimize_for = LITE_RUNTIME cannot define services " "unless you set both options cc_generic_services and " - "java_generic_sevices to false."); + "java_generic_services to false."); } VALIDATE_OPTIONS_FROM_ARRAY(service, method, Method); } -void DescriptorBuilder::ValidateMethodOptions(MethodDescriptor* /* method */, - const MethodDescriptorProto& /* proto */) { +void DescriptorBuilder::ValidateMethodOptions( + MethodDescriptor* /* method */, const MethodDescriptorProto& /* proto */) { // Nothing to do so far. } bool DescriptorBuilder::ValidateMapEntry(FieldDescriptor* field, const FieldDescriptorProto& proto) { const Descriptor* message = field->message_type(); - if (// Must not contain extensions, extension range or nested message or - // enums + if ( // Must not contain extensions, extension range or nested message or + // enums message->extension_count() != 0 || field->label() != FieldDescriptor::LABEL_REPEATED || message->extension_range_count() != 0 || @@ -6066,8 +6809,8 @@ bool DescriptorBuilder::ValidateMapEntry(FieldDescriptor* field, return false; } - const FieldDescriptor* key = message->field(0); - const FieldDescriptor* value = message->field(1); + const FieldDescriptor* key = message->map_key(); + const FieldDescriptor* value = message->map_value(); if (key->label() != FieldDescriptor::LABEL_OPTIONAL || key->number() != 1 || key->name() != "key") { return false; @@ -6080,9 +6823,8 @@ bool DescriptorBuilder::ValidateMapEntry(FieldDescriptor* field, // Check key types are legal. switch (key->type()) { case FieldDescriptor::TYPE_ENUM: - AddError( - field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, - "Key in map fields cannot be enum types."); + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, + "Key in map fields cannot be enum types."); break; case FieldDescriptor::TYPE_FLOAT: case FieldDescriptor::TYPE_DOUBLE: @@ -6107,15 +6849,14 @@ bool DescriptorBuilder::ValidateMapEntry(FieldDescriptor* field, case FieldDescriptor::TYPE_SFIXED64: // Legal cases break; - // Do not add a default, so that the compiler will complain when new types - // are added. + // Do not add a default, so that the compiler will complain when new types + // are added. } if (value->type() == FieldDescriptor::TYPE_ENUM) { if (value->enum_type()->value(0)->number() != 0) { - AddError( - field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, - "Enum value in map must define 0 as the first value."); + AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, + "Enum value in map must define 0 as the first value."); } } @@ -6124,10 +6865,10 @@ bool DescriptorBuilder::ValidateMapEntry(FieldDescriptor* field, void DescriptorBuilder::DetectMapConflicts(const Descriptor* message, const DescriptorProto& proto) { - std::map seen_types; + std::map seen_types; for (int i = 0; i < message->nested_type_count(); ++i) { const Descriptor* nested = message->nested_type(i); - std::pair::iterator, bool> result = + std::pair::iterator, bool> result = seen_types.insert(std::make_pair(nested->name(), nested)); if (!result.second) { if (result.first->second->options().map_entry() || @@ -6135,7 +6876,7 @@ void DescriptorBuilder::DetectMapConflicts(const Descriptor* message, AddError(message->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Expanded map entry type " + nested->name() + - " conflicts with an existing nested message type."); + " conflicts with an existing nested message type."); } } // Recursively test on the nested types. @@ -6144,37 +6885,37 @@ void DescriptorBuilder::DetectMapConflicts(const Descriptor* message, // Check for conflicted field names. for (int i = 0; i < message->field_count(); ++i) { const FieldDescriptor* field = message->field(i); - std::map::iterator iter = + std::map::iterator iter = seen_types.find(field->name()); if (iter != seen_types.end() && iter->second->options().map_entry()) { AddError(message->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Expanded map entry type " + iter->second->name() + - " conflicts with an existing field."); + " conflicts with an existing field."); } } // Check for conflicted enum names. for (int i = 0; i < message->enum_type_count(); ++i) { const EnumDescriptor* enum_desc = message->enum_type(i); - std::map::iterator iter = + std::map::iterator iter = seen_types.find(enum_desc->name()); if (iter != seen_types.end() && iter->second->options().map_entry()) { AddError(message->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Expanded map entry type " + iter->second->name() + - " conflicts with an existing enum type."); + " conflicts with an existing enum type."); } } // Check for conflicted oneof names. for (int i = 0; i < message->oneof_decl_count(); ++i) { const OneofDescriptor* oneof_desc = message->oneof_decl(i); - std::map::iterator iter = + std::map::iterator iter = seen_types.find(oneof_desc->name()); if (iter != seen_types.end() && iter->second->options().map_entry()) { AddError(message->full_name(), proto, DescriptorPool::ErrorCollector::NAME, "Expanded map entry type " + iter->second->name() + - " conflicts with an existing oneof type."); + " conflicts with an existing oneof type."); } } } @@ -6202,7 +6943,7 @@ void DescriptorBuilder::ValidateJSType(FieldDescriptor* field, AddError(field->full_name(), proto, DescriptorPool::ErrorCollector::TYPE, "Illegal jstype for int64, uint64, sint64, fixed64 " "or sfixed64 field: " + - FieldOptions_JSType_descriptor()->value(jstype)->name()); + FieldOptions_JSType_descriptor()->value(jstype)->name()); break; // No other types permit a jstype option. @@ -6219,12 +6960,12 @@ void DescriptorBuilder::ValidateJSType(FieldDescriptor* field, // ------------------------------------------------------------------- DescriptorBuilder::OptionInterpreter::OptionInterpreter( - DescriptorBuilder* builder) : builder_(builder) { + DescriptorBuilder* builder) + : builder_(builder) { GOOGLE_CHECK(builder_); } -DescriptorBuilder::OptionInterpreter::~OptionInterpreter() { -} +DescriptorBuilder::OptionInterpreter::~OptionInterpreter() {} bool DescriptorBuilder::OptionInterpreter::InterpretOptions( OptionsToInterpret* options_to_interpret) { @@ -6240,32 +6981,39 @@ bool DescriptorBuilder::OptionInterpreter::InterpretOptions( // and clear them, since we're about to interpret them. const FieldDescriptor* uninterpreted_options_field = options->GetDescriptor()->FindFieldByName("uninterpreted_option"); - GOOGLE_CHECK(uninterpreted_options_field != NULL) + GOOGLE_CHECK(uninterpreted_options_field != nullptr) << "No field named \"uninterpreted_option\" in the Options proto."; options->GetReflection()->ClearField(options, uninterpreted_options_field); + std::vector src_path = options_to_interpret->element_path; + src_path.push_back(uninterpreted_options_field->number()); + // Find the uninterpreted_option field in the original options. const FieldDescriptor* original_uninterpreted_options_field = - original_options->GetDescriptor()-> - FindFieldByName("uninterpreted_option"); - GOOGLE_CHECK(original_uninterpreted_options_field != NULL) + original_options->GetDescriptor()->FindFieldByName( + "uninterpreted_option"); + GOOGLE_CHECK(original_uninterpreted_options_field != nullptr) << "No field named \"uninterpreted_option\" in the Options proto."; - const int num_uninterpreted_options = original_options->GetReflection()-> - FieldSize(*original_options, original_uninterpreted_options_field); + const int num_uninterpreted_options = + original_options->GetReflection()->FieldSize( + *original_options, original_uninterpreted_options_field); for (int i = 0; i < num_uninterpreted_options; ++i) { + src_path.push_back(i); uninterpreted_option_ = down_cast( &original_options->GetReflection()->GetRepeatedMessage( *original_options, original_uninterpreted_options_field, i)); - if (!InterpretSingleOption(options)) { + if (!InterpretSingleOption(options, src_path, + options_to_interpret->element_path)) { // Error already added by InterpretSingleOption(). failed = true; break; } + src_path.pop_back(); } // Reset these, so we don't have any dangling pointers. - uninterpreted_option_ = NULL; - options_to_interpret_ = NULL; + uninterpreted_option_ = nullptr; + options_to_interpret_ = nullptr; if (!failed) { // InterpretSingleOption() added the interpreted options in the @@ -6276,25 +7024,35 @@ bool DescriptorBuilder::OptionInterpreter::InterpretOptions( // If they are not known, that's OK too. They will get reparsed into the // UnknownFieldSet and wait there until the message is parsed by something // that does know about the options. - string buf; - GOOGLE_CHECK(options->AppendPartialToString(&buf)) - << "Protocol message could not be serialized."; - GOOGLE_CHECK(options->ParsePartialFromString(buf)) - << "Protocol message serialized itself in invalid fashion."; - if (!options->IsInitialized()) { - builder_->AddWarning( + + // Keep the unparsed options around in case the reparsing fails. + std::unique_ptr unparsed_options(options->New()); + options->GetReflection()->Swap(unparsed_options.get(), options); + + std::string buf; + if (!unparsed_options->AppendToString(&buf) || + !options->ParseFromString(buf)) { + builder_->AddError( options_to_interpret->element_name, *original_options, DescriptorPool::ErrorCollector::OTHER, - "Options could not be fully parsed using the proto descriptors " - "compiled into this binary. Missing required fields: " + - options->InitializationErrorString()); + "Some options could not be correctly parsed using the proto " + "descriptors compiled into this binary.\n" + "Unparsed options: " + + unparsed_options->ShortDebugString() + + "\n" + "Parsing attempt: " + + options->ShortDebugString()); + // Restore the unparsed options. + options->GetReflection()->Swap(unparsed_options.get(), options); } } + return !failed; } bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( - Message* options) { + Message* options, const std::vector& src_path, + const std::vector& options_path) { // First do some basic validation. if (uninterpreted_option_->name_size() == 0) { // This should never happen unless the parser has gone seriously awry or @@ -6302,15 +7060,16 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( return AddNameError("Option must have a name."); } if (uninterpreted_option_->name(0).name_part() == "uninterpreted_option") { - return AddNameError("Option must not use reserved name " - "\"uninterpreted_option\"."); + return AddNameError( + "Option must not use reserved name " + "\"uninterpreted_option\"."); } - const Descriptor* options_descriptor = NULL; + const Descriptor* options_descriptor = nullptr; // Get the options message's descriptor from the builder's pool, so that we - // get the version that knows about any extension options declared in the - // file we're currently building. The descriptor should be there as long as - // the file we're building imported "google/protobuf/descriptors.proto". + // get the version that knows about any extension options declared in the file + // we're currently building. The descriptor should be there as long as the + // file we're building imported descriptor.proto. // Note that we use DescriptorBuilder::FindSymbolNotEnforcingDeps(), not // DescriptorPool::FindMessageTypeByName() because we're already holding the @@ -6318,10 +7077,9 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( // FindSymbol() because files that use custom options only need to depend on // the file that defines the option, not descriptor.proto itself. Symbol symbol = builder_->FindSymbolNotEnforcingDeps( - options->GetDescriptor()->full_name()); - if (!symbol.IsNull() && symbol.type == Symbol::MESSAGE) { - options_descriptor = symbol.descriptor; - } else { + options->GetDescriptor()->full_name()); + options_descriptor = symbol.descriptor(); + if (options_descriptor == nullptr) { // The options message's descriptor was not in the builder's pool, so use // the standard version from the generated pool. We're not holding the // generated pool's mutex, so we can search it the straightforward way. @@ -6336,12 +7094,15 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( // through in |intermediate_fields|. As we go, we reconstruct the full option // name in |debug_msg_name|, for use in error messages. const Descriptor* descriptor = options_descriptor; - const FieldDescriptor* field = NULL; + const FieldDescriptor* field = nullptr; std::vector intermediate_fields; - string debug_msg_name = ""; + std::string debug_msg_name = ""; + + std::vector dest_path = options_path; for (int i = 0; i < uninterpreted_option_->name_size(); ++i) { - const string& name_part = uninterpreted_option_->name(i).name_part(); + builder_->undefine_resolved_name_.clear(); + const std::string& name_part = uninterpreted_option_->name(i).name_part(); if (debug_msg_name.size() > 0) { debug_msg_name += "."; } @@ -6352,11 +7113,9 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( // DescriptorPool::FindExtensionByName(), for two reasons: 1) It allows // relative lookups, and 2) because we're already holding the pool's // mutex, and the latter method locks it again. - symbol = builder_->LookupSymbol(name_part, - options_to_interpret_->name_scope); - if (!symbol.IsNull() && symbol.type == Symbol::FIELD) { - field = symbol.field_descriptor; - } + symbol = + builder_->LookupSymbol(name_part, options_to_interpret_->name_scope); + field = symbol.field_descriptor(); // If we don't find the field then the field's descriptor was not in the // builder's pool, but there's no point in looking in the generated // pool. We require that you import the file that defines any extensions @@ -6367,7 +7126,7 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( field = descriptor->FindFieldByName(name_part); } - if (field == NULL) { + if (field == nullptr) { if (get_allow_unknown(builder_->pool_)) { // We can't find the option, but AllowUnknownDependencies() is enabled, // so we will just leave it as uninterpreted. @@ -6383,7 +7142,10 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( debug_msg_name.substr(1) + "\") to start from the outermost scope."); } else { - return AddNameError("Option \"" + debug_msg_name + "\" unknown."); + return AddNameError( + "Option \"" + debug_msg_name + + "\" unknown. Ensure that your proto" + + " definition file imports the proto which defines the option."); } } else if (field->containing_type() != descriptor) { if (get_is_placeholder(field->containing_type())) { @@ -6402,19 +7164,24 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( "\" is not a field or extension of message \"" + descriptor->name() + "\"."); } - } else if (i < uninterpreted_option_->name_size() - 1) { - if (field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { - return AddNameError("Option \"" + debug_msg_name + - "\" is an atomic type, not a message."); - } else if (field->is_repeated()) { - return AddNameError("Option field \"" + debug_msg_name + - "\" is a repeated message. Repeated message " - "options must be initialized using an " - "aggregate value."); - } else { - // Drill down into the submessage. - intermediate_fields.push_back(field); - descriptor = field->message_type(); + } else { + // accumulate field numbers to form path to interpreted option + dest_path.push_back(field->number()); + + if (i < uninterpreted_option_->name_size() - 1) { + if (field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { + return AddNameError("Option \"" + debug_msg_name + + "\" is an atomic type, not a message."); + } else if (field->is_repeated()) { + return AddNameError("Option field \"" + debug_msg_name + + "\" is a repeated message. Repeated message " + "options must be initialized using an " + "aggregate value."); + } else { + // Drill down into the submessage. + intermediate_fields.push_back(field); + descriptor = field->message_type(); + } } } } @@ -6427,18 +7194,17 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( // known will populate them correctly. // First see if the option is already set. - if (!field->is_repeated() && !ExamineIfOptionIsSet( - intermediate_fields.begin(), - intermediate_fields.end(), - field, debug_msg_name, + if (!field->is_repeated() && + !ExamineIfOptionIsSet( + intermediate_fields.begin(), intermediate_fields.end(), field, + debug_msg_name, options->GetReflection()->GetUnknownFields(*options))) { return false; // ExamineIfOptionIsSet() already added the error. } - // First set the value on the UnknownFieldSet corresponding to the // innermost message. - google::protobuf::scoped_ptr unknown_fields(new UnknownFieldSet()); + std::unique_ptr unknown_fields(new UnknownFieldSet()); if (!SetOptionValue(field, unknown_fields.get())) { return false; // SetOptionValue() already added the error. } @@ -6448,7 +7214,7 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( for (std::vector::reverse_iterator iter = intermediate_fields.rbegin(); iter != intermediate_fields.rend(); ++iter) { - google::protobuf::scoped_ptr parent_unknown_fields( + std::unique_ptr parent_unknown_fields( new UnknownFieldSet()); switch ((*iter)->type()) { case FieldDescriptor::TYPE_MESSAGE: { @@ -6463,8 +7229,8 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( } case FieldDescriptor::TYPE_GROUP: { - parent_unknown_fields->AddGroup((*iter)->number()) - ->MergeFrom(*unknown_fields); + parent_unknown_fields->AddGroup((*iter)->number()) + ->MergeFrom(*unknown_fields); break; } @@ -6481,24 +7247,125 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( options->GetReflection()->MutableUnknownFields(options)->MergeFrom( *unknown_fields); + // record the element path of the interpreted option + if (field->is_repeated()) { + int index = repeated_option_counts_[dest_path]++; + dest_path.push_back(index); + } + interpreted_paths_[src_path] = dest_path; + return true; } +void DescriptorBuilder::OptionInterpreter::UpdateSourceCodeInfo( + SourceCodeInfo* info) { + if (interpreted_paths_.empty()) { + // nothing to do! + return; + } + + // We find locations that match keys in interpreted_paths_ and + // 1) replace the path with the corresponding value in interpreted_paths_ + // 2) remove any subsequent sub-locations (sub-location is one whose path + // has the parent path as a prefix) + // + // To avoid quadratic behavior of removing interior rows as we go, + // we keep a copy. But we don't actually copy anything until we've + // found the first match (so if the source code info has no locations + // that need to be changed, there is zero copy overhead). + + RepeatedPtrField* locs = info->mutable_location(); + RepeatedPtrField new_locs; + bool copying = false; + + std::vector pathv; + bool matched = false; + + for (RepeatedPtrField::iterator loc = locs->begin(); + loc != locs->end(); loc++) { + if (matched) { + // see if this location is in the range to remove + bool loc_matches = true; + if (loc->path_size() < static_cast(pathv.size())) { + loc_matches = false; + } else { + for (size_t j = 0; j < pathv.size(); j++) { + if (loc->path(j) != pathv[j]) { + loc_matches = false; + break; + } + } + } + + if (loc_matches) { + // don't copy this row since it is a sub-location that we're removing + continue; + } + + matched = false; + } + + pathv.clear(); + for (int j = 0; j < loc->path_size(); j++) { + pathv.push_back(loc->path(j)); + } + + std::map, std::vector>::iterator entry = + interpreted_paths_.find(pathv); + + if (entry == interpreted_paths_.end()) { + // not a match + if (copying) { + *new_locs.Add() = *loc; + } + continue; + } + + matched = true; + + if (!copying) { + // initialize the copy we are building + copying = true; + new_locs.Reserve(locs->size()); + for (RepeatedPtrField::iterator it = + locs->begin(); + it != loc; it++) { + *new_locs.Add() = *it; + } + } + + // add replacement and update its path + SourceCodeInfo_Location* replacement = new_locs.Add(); + *replacement = *loc; + replacement->clear_path(); + for (std::vector::iterator rit = entry->second.begin(); + rit != entry->second.end(); rit++) { + replacement->add_path(*rit); + } + } + + // if we made a changed copy, put it in place + if (copying) { + *locs = new_locs; + } +} + void DescriptorBuilder::OptionInterpreter::AddWithoutInterpreting( const UninterpretedOption& uninterpreted_option, Message* options) { const FieldDescriptor* field = - options->GetDescriptor()->FindFieldByName("uninterpreted_option"); - GOOGLE_CHECK(field != NULL); + options->GetDescriptor()->FindFieldByName("uninterpreted_option"); + GOOGLE_CHECK(field != nullptr); - options->GetReflection()->AddMessage(options, field) - ->CopyFrom(uninterpreted_option); + options->GetReflection() + ->AddMessage(options, field) + ->CopyFrom(uninterpreted_option); } bool DescriptorBuilder::OptionInterpreter::ExamineIfOptionIsSet( std::vector::const_iterator intermediate_fields_iter, std::vector::const_iterator intermediate_fields_end, - const FieldDescriptor* innermost_field, const string& debug_msg_name, + const FieldDescriptor* innermost_field, const std::string& debug_msg_name, const UnknownFieldSet& unknown_fields) { // We do linear searches of the UnknownFieldSet and its sub-groups. This // should be fine since it's unlikely that any one options structure will @@ -6528,8 +7395,8 @@ bool DescriptorBuilder::OptionInterpreter::ExamineIfOptionIsSet( if (intermediate_unknown_fields.ParseFromString( unknown_field->length_delimited()) && !ExamineIfOptionIsSet(intermediate_fields_iter + 1, - intermediate_fields_end, - innermost_field, debug_msg_name, + intermediate_fields_end, innermost_field, + debug_msg_name, intermediate_unknown_fields)) { return false; // Error already added. } @@ -6539,9 +7406,8 @@ bool DescriptorBuilder::OptionInterpreter::ExamineIfOptionIsSet( case FieldDescriptor::TYPE_GROUP: if (unknown_field->type() == UnknownField::TYPE_GROUP) { if (!ExamineIfOptionIsSet(intermediate_fields_iter + 1, - intermediate_fields_end, - innermost_field, debug_msg_name, - unknown_field->group())) { + intermediate_fields_end, innermost_field, + debug_msg_name, unknown_field->group())) { return false; // Error already added. } } @@ -6557,15 +7423,13 @@ bool DescriptorBuilder::OptionInterpreter::ExamineIfOptionIsSet( } bool DescriptorBuilder::OptionInterpreter::SetOptionValue( - const FieldDescriptor* option_field, - UnknownFieldSet* unknown_fields) { + const FieldDescriptor* option_field, UnknownFieldSet* unknown_fields) { // We switch on the CppType to validate. switch (option_field->cpp_type()) { - case FieldDescriptor::CPPTYPE_INT32: if (uninterpreted_option_->has_positive_int_value()) { if (uninterpreted_option_->positive_int_value() > - static_cast(kint32max)) { + static_cast(std::numeric_limits::max())) { return AddValueError("Value out of range for int32 option \"" + option_field->full_name() + "\"."); } else { @@ -6575,7 +7439,7 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( } } else if (uninterpreted_option_->has_negative_int_value()) { if (uninterpreted_option_->negative_int_value() < - static_cast(kint32min)) { + static_cast(std::numeric_limits::min())) { return AddValueError("Value out of range for int32 option \"" + option_field->full_name() + "\"."); } else { @@ -6592,7 +7456,7 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( case FieldDescriptor::CPPTYPE_INT64: if (uninterpreted_option_->has_positive_int_value()) { if (uninterpreted_option_->positive_int_value() > - static_cast(kint64max)) { + static_cast(std::numeric_limits::max())) { return AddValueError("Value out of range for int64 option \"" + option_field->full_name() + "\"."); } else { @@ -6612,7 +7476,8 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( case FieldDescriptor::CPPTYPE_UINT32: if (uninterpreted_option_->has_positive_int_value()) { - if (uninterpreted_option_->positive_int_value() > kuint32max) { + if (uninterpreted_option_->positive_int_value() > + std::numeric_limits::max()) { return AddValueError("Value out of range for uint32 option \"" + option_field->name() + "\"."); } else { @@ -6621,8 +7486,10 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( option_field->type(), unknown_fields); } } else { - return AddValueError("Value must be non-negative integer for uint32 " - "option \"" + option_field->full_name() + "\"."); + return AddValueError( + "Value must be non-negative integer for uint32 " + "option \"" + + option_field->full_name() + "\"."); } break; @@ -6632,8 +7499,10 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( uninterpreted_option_->positive_int_value(), option_field->type(), unknown_fields); } else { - return AddValueError("Value must be non-negative integer for uint64 " - "option \"" + option_field->full_name() + "\"."); + return AddValueError( + "Value must be non-negative integer for uint64 " + "option \"" + + option_field->full_name() + "\"."); } break; @@ -6650,7 +7519,7 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( option_field->full_name() + "\"."); } unknown_fields->AddFixed32(option_field->number(), - google::protobuf::internal::WireFormatLite::EncodeFloat(value)); + internal::WireFormatLite::EncodeFloat(value)); break; } @@ -6667,40 +7536,46 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( option_field->full_name() + "\"."); } unknown_fields->AddFixed64(option_field->number(), - google::protobuf::internal::WireFormatLite::EncodeDouble(value)); + internal::WireFormatLite::EncodeDouble(value)); break; } case FieldDescriptor::CPPTYPE_BOOL: - uint64 value; + uint64_t value; if (!uninterpreted_option_->has_identifier_value()) { - return AddValueError("Value must be identifier for boolean option " - "\"" + option_field->full_name() + "\"."); + return AddValueError( + "Value must be identifier for boolean option " + "\"" + + option_field->full_name() + "\"."); } if (uninterpreted_option_->identifier_value() == "true") { value = 1; } else if (uninterpreted_option_->identifier_value() == "false") { value = 0; } else { - return AddValueError("Value must be \"true\" or \"false\" for boolean " - "option \"" + option_field->full_name() + "\"."); + return AddValueError( + "Value must be \"true\" or \"false\" for boolean " + "option \"" + + option_field->full_name() + "\"."); } unknown_fields->AddVarint(option_field->number(), value); break; case FieldDescriptor::CPPTYPE_ENUM: { if (!uninterpreted_option_->has_identifier_value()) { - return AddValueError("Value must be identifier for enum-valued option " - "\"" + option_field->full_name() + "\"."); + return AddValueError( + "Value must be identifier for enum-valued option " + "\"" + + option_field->full_name() + "\"."); } const EnumDescriptor* enum_type = option_field->enum_type(); - const string& value_name = uninterpreted_option_->identifier_value(); - const EnumValueDescriptor* enum_value = NULL; + const std::string& value_name = uninterpreted_option_->identifier_value(); + const EnumValueDescriptor* enum_value = nullptr; if (enum_type->file()->pool() != DescriptorPool::generated_pool()) { // Note that the enum value's fully-qualified name is a sibling of the // enum's name, not a child of it. - string fully_qualified_name = enum_type->full_name(); + std::string fully_qualified_name = enum_type->full_name(); fully_qualified_name.resize(fully_qualified_name.size() - enum_type->name().size()); fully_qualified_name += value_name; @@ -6710,15 +7585,16 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( // DescriptorPool::FindEnumValueByName() because we're already holding // the pool's mutex, and the latter method locks it again. Symbol symbol = - builder_->FindSymbolNotEnforcingDeps(fully_qualified_name); - if (!symbol.IsNull() && symbol.type == Symbol::ENUM_VALUE) { - if (symbol.enum_value_descriptor->type() != enum_type) { - return AddValueError("Enum type \"" + enum_type->full_name() + + builder_->FindSymbolNotEnforcingDeps(fully_qualified_name); + if (auto* candicate_descriptor = symbol.enum_value_descriptor()) { + if (candicate_descriptor->type() != enum_type) { + return AddValueError( + "Enum type \"" + enum_type->full_name() + "\" has no value named \"" + value_name + "\" for option \"" + option_field->full_name() + "\". This appears to be a value from a sibling type."); } else { - enum_value = symbol.enum_value_descriptor; + enum_value = candicate_descriptor; } } } else { @@ -6727,28 +7603,33 @@ bool DescriptorBuilder::OptionInterpreter::SetOptionValue( enum_value = enum_type->FindValueByName(value_name); } - if (enum_value == NULL) { + if (enum_value == nullptr) { return AddValueError("Enum type \"" + option_field->enum_type()->full_name() + - "\" has no value named \"" + value_name + "\" for " - "option \"" + option_field->full_name() + "\"."); + "\" has no value named \"" + value_name + + "\" for " + "option \"" + + option_field->full_name() + "\"."); } else { - // Sign-extension is not a problem, since we cast directly from int32 to - // uint64, without first going through uint32. - unknown_fields->AddVarint(option_field->number(), - static_cast(static_cast(enum_value->number()))); + // Sign-extension is not a problem, since we cast directly from int32_t + // to uint64_t, without first going through uint32_t. + unknown_fields->AddVarint( + option_field->number(), + static_cast(static_cast(enum_value->number()))); } break; } case FieldDescriptor::CPPTYPE_STRING: if (!uninterpreted_option_->has_string_value()) { - return AddValueError("Value must be quoted string for string option " - "\"" + option_field->full_name() + "\"."); + return AddValueError( + "Value must be quoted string for string option " + "\"" + + option_field->full_name() + "\"."); } // The string has already been unquoted and unescaped by the parser. unknown_fields->AddLengthDelimited(option_field->number(), - uninterpreted_option_->string_value()); + uninterpreted_option_->string_value()); break; case FieldDescriptor::CPPTYPE_MESSAGE: @@ -6766,18 +7647,28 @@ class DescriptorBuilder::OptionInterpreter::AggregateOptionFinder public: DescriptorBuilder* builder_; - virtual const FieldDescriptor* FindExtension( - Message* message, const string& name) const { + const Descriptor* FindAnyType(const Message& /*message*/, + const std::string& prefix, + const std::string& name) const override { + if (prefix != internal::kTypeGoogleApisComPrefix && + prefix != internal::kTypeGoogleProdComPrefix) { + return nullptr; + } + assert_mutex_held(builder_->pool_); + return builder_->FindSymbol(name).descriptor(); + } + + const FieldDescriptor* FindExtension(Message* message, + const std::string& name) const override { assert_mutex_held(builder_->pool_); const Descriptor* descriptor = message->GetDescriptor(); - Symbol result = builder_->LookupSymbolNoPlaceholder( - name, descriptor->full_name()); - if (result.type == Symbol::FIELD && - result.field_descriptor->is_extension()) { - return result.field_descriptor; - } else if (result.type == Symbol::MESSAGE && + Symbol result = + builder_->LookupSymbolNoPlaceholder(name, descriptor->full_name()); + if (auto* field = result.field_descriptor()) { + return field; + } else if (result.type() == Symbol::MESSAGE && descriptor->options().message_set_wire_format()) { - const Descriptor* foreign_type = result.descriptor; + const Descriptor* foreign_type = result.descriptor(); // The text format allows MessageSet items to be specified using // the type name, rather than the extension identifier. If the symbol // lookup returned a Message, and the enclosing Message has @@ -6794,7 +7685,7 @@ class DescriptorBuilder::OptionInterpreter::AggregateOptionFinder } } } - return NULL; + return nullptr; } }; @@ -6802,42 +7693,42 @@ class DescriptorBuilder::OptionInterpreter::AggregateOptionFinder namespace { class AggregateErrorCollector : public io::ErrorCollector { public: - string error_; + std::string error_; - virtual void AddError(int /* line */, int /* column */, - const string& message) { + void AddError(int /* line */, int /* column */, + const std::string& message) override { if (!error_.empty()) { error_ += "; "; } error_ += message; } - virtual void AddWarning(int /* line */, int /* column */, - const string& /* message */) { + void AddWarning(int /* line */, int /* column */, + const std::string& /* message */) override { // Ignore warnings } }; -} +} // namespace // We construct a dynamic message of the type corresponding to // option_field, parse the supplied text-format string into this // message, and serialize the resulting message to produce the value. bool DescriptorBuilder::OptionInterpreter::SetAggregateOption( - const FieldDescriptor* option_field, - UnknownFieldSet* unknown_fields) { + const FieldDescriptor* option_field, UnknownFieldSet* unknown_fields) { if (!uninterpreted_option_->has_aggregate_value()) { return AddValueError("Option \"" + option_field->full_name() + "\" is a message. To set the entire message, use " - "syntax like \"" + option_field->name() + + "syntax like \"" + + option_field->name() + " = { }\". " "To set fields within it, use " - "syntax like \"" + option_field->name() + - ".foo = value\"."); + "syntax like \"" + + option_field->name() + ".foo = value\"."); } const Descriptor* type = option_field->message_type(); - google::protobuf::scoped_ptr dynamic(dynamic_factory_.GetPrototype(type)->New()); - GOOGLE_CHECK(dynamic.get() != NULL) + std::unique_ptr dynamic(dynamic_factory_.GetPrototype(type)->New()); + GOOGLE_CHECK(dynamic.get() != nullptr) << "Could not create an instance of " << option_field->DebugString(); AggregateErrorCollector collector; @@ -6852,12 +7743,12 @@ bool DescriptorBuilder::OptionInterpreter::SetAggregateOption( option_field->name() + "\": " + collector.error_); return false; } else { - string serial; + std::string serial; dynamic->SerializeToString(&serial); // Never fails if (option_field->type() == FieldDescriptor::TYPE_MESSAGE) { unknown_fields->AddLengthDelimited(option_field->number(), serial); } else { - GOOGLE_CHECK_EQ(option_field->type(), FieldDescriptor::TYPE_GROUP); + GOOGLE_CHECK_EQ(option_field->type(), FieldDescriptor::TYPE_GROUP); UnknownFieldSet* group = unknown_fields->AddGroup(option_field->number()); group->ParseFromString(serial); } @@ -6865,21 +7756,22 @@ bool DescriptorBuilder::OptionInterpreter::SetAggregateOption( } } -void DescriptorBuilder::OptionInterpreter::SetInt32(int number, int32 value, - FieldDescriptor::Type type, UnknownFieldSet* unknown_fields) { +void DescriptorBuilder::OptionInterpreter::SetInt32( + int number, int32_t value, FieldDescriptor::Type type, + UnknownFieldSet* unknown_fields) { switch (type) { case FieldDescriptor::TYPE_INT32: - unknown_fields->AddVarint(number, - static_cast(static_cast(value))); + unknown_fields->AddVarint( + number, static_cast(static_cast(value))); break; case FieldDescriptor::TYPE_SFIXED32: - unknown_fields->AddFixed32(number, static_cast(value)); + unknown_fields->AddFixed32(number, static_cast(value)); break; case FieldDescriptor::TYPE_SINT32: - unknown_fields->AddVarint(number, - google::protobuf::internal::WireFormatLite::ZigZagEncode32(value)); + unknown_fields->AddVarint( + number, internal::WireFormatLite::ZigZagEncode32(value)); break; default: @@ -6888,20 +7780,21 @@ void DescriptorBuilder::OptionInterpreter::SetInt32(int number, int32 value, } } -void DescriptorBuilder::OptionInterpreter::SetInt64(int number, int64 value, - FieldDescriptor::Type type, UnknownFieldSet* unknown_fields) { +void DescriptorBuilder::OptionInterpreter::SetInt64( + int number, int64_t value, FieldDescriptor::Type type, + UnknownFieldSet* unknown_fields) { switch (type) { case FieldDescriptor::TYPE_INT64: - unknown_fields->AddVarint(number, static_cast(value)); + unknown_fields->AddVarint(number, static_cast(value)); break; case FieldDescriptor::TYPE_SFIXED64: - unknown_fields->AddFixed64(number, static_cast(value)); + unknown_fields->AddFixed64(number, static_cast(value)); break; case FieldDescriptor::TYPE_SINT64: - unknown_fields->AddVarint(number, - google::protobuf::internal::WireFormatLite::ZigZagEncode64(value)); + unknown_fields->AddVarint( + number, internal::WireFormatLite::ZigZagEncode64(value)); break; default: @@ -6910,15 +7803,16 @@ void DescriptorBuilder::OptionInterpreter::SetInt64(int number, int64 value, } } -void DescriptorBuilder::OptionInterpreter::SetUInt32(int number, uint32 value, - FieldDescriptor::Type type, UnknownFieldSet* unknown_fields) { +void DescriptorBuilder::OptionInterpreter::SetUInt32( + int number, uint32_t value, FieldDescriptor::Type type, + UnknownFieldSet* unknown_fields) { switch (type) { case FieldDescriptor::TYPE_UINT32: - unknown_fields->AddVarint(number, static_cast(value)); + unknown_fields->AddVarint(number, static_cast(value)); break; case FieldDescriptor::TYPE_FIXED32: - unknown_fields->AddFixed32(number, static_cast(value)); + unknown_fields->AddFixed32(number, static_cast(value)); break; default: @@ -6927,8 +7821,9 @@ void DescriptorBuilder::OptionInterpreter::SetUInt32(int number, uint32 value, } } -void DescriptorBuilder::OptionInterpreter::SetUInt64(int number, uint64 value, - FieldDescriptor::Type type, UnknownFieldSet* unknown_fields) { +void DescriptorBuilder::OptionInterpreter::SetUInt64( + int number, uint64_t value, FieldDescriptor::Type type, + UnknownFieldSet* unknown_fields) { switch (type) { case FieldDescriptor::TYPE_UINT64: unknown_fields->AddVarint(number, value); @@ -6946,43 +7841,31 @@ void DescriptorBuilder::OptionInterpreter::SetUInt64(int number, uint64 value, void DescriptorBuilder::LogUnusedDependency(const FileDescriptorProto& proto, const FileDescriptor* result) { + (void)result; // Parameter is used by Google-internal code. if (!unused_dependency_.empty()) { - std::set annotation_extensions; - annotation_extensions.insert("google.protobuf.MessageOptions"); - annotation_extensions.insert("google.protobuf.FileOptions"); - annotation_extensions.insert("google.protobuf.FieldOptions"); - annotation_extensions.insert("google.protobuf.EnumOptions"); - annotation_extensions.insert("google.protobuf.EnumValueOptions"); - annotation_extensions.insert("google.protobuf.EnumValueOptions"); - annotation_extensions.insert("google.protobuf.ServiceOptions"); - annotation_extensions.insert("google.protobuf.MethodOptions"); - annotation_extensions.insert("google.protobuf.StreamOptions"); - for (std::set::const_iterator - it = unused_dependency_.begin(); + auto itr = pool_->unused_import_track_files_.find(proto.name()); + bool is_error = + itr != pool_->unused_import_track_files_.end() && itr->second; + for (std::set::const_iterator it = + unused_dependency_.begin(); it != unused_dependency_.end(); ++it) { - // Do not log warnings for proto files which extend annotations. - int i; - for (i = 0 ; i < (*it)->extension_count(); ++i) { - if (annotation_extensions.find( - (*it)->extension(i)->containing_type()->full_name()) - != annotation_extensions.end()) { - break; - } - } - // Log warnings for unused imported files. - if (i == (*it)->extension_count()) { - string error_message = "Import " + (*it)->name() + " but not used."; - AddWarning((*it)->name(), proto, DescriptorPool::ErrorCollector::OTHER, + std::string error_message = "Import " + (*it)->name() + " is unused."; + if (is_error) { + AddError((*it)->name(), proto, DescriptorPool::ErrorCollector::IMPORT, + error_message); + } else { + AddWarning((*it)->name(), proto, DescriptorPool::ErrorCollector::IMPORT, error_message); } } } } -Symbol DescriptorPool::CrossLinkOnDemandHelper(const string& name, +Symbol DescriptorPool::CrossLinkOnDemandHelper(StringPiece name, bool expecting_enum) const { - string lookup_name = name; + (void)expecting_enum; // Parameter is used by Google-internal code. + auto lookup_name = std::string(name); if (!lookup_name.empty() && lookup_name[0] == '.') { lookup_name = lookup_name.substr(1); } @@ -6996,39 +7879,39 @@ Symbol DescriptorPool::CrossLinkOnDemandHelper(const string& name, // enum_type_, message_type_, and default_value_enum_ appropriately. void FieldDescriptor::InternalTypeOnceInit() const { GOOGLE_CHECK(file()->finished_building_ == true); - if (type_name_) { - Symbol result = file()->pool()->CrossLinkOnDemandHelper( - *type_name_, type_ == FieldDescriptor::TYPE_ENUM); - if (result.type == Symbol::MESSAGE) { - type_ = FieldDescriptor::TYPE_MESSAGE; - message_type_ = result.descriptor; - } else if (result.type == Symbol::ENUM) { - type_ = FieldDescriptor::TYPE_ENUM; - enum_type_ = result.enum_descriptor; - } - } - if (enum_type_ && !default_value_enum_) { - if (default_value_enum_name_) { + const EnumDescriptor* enum_type = nullptr; + Symbol result = file()->pool()->CrossLinkOnDemandHelper( + type_descriptor_.lazy_type_name, type_ == FieldDescriptor::TYPE_ENUM); + if (result.type() == Symbol::MESSAGE) { + type_ = FieldDescriptor::TYPE_MESSAGE; + type_descriptor_.message_type = result.descriptor(); + } else if (result.type() == Symbol::ENUM) { + type_ = FieldDescriptor::TYPE_ENUM; + enum_type = type_descriptor_.enum_type = result.enum_descriptor(); + } + + if (enum_type) { + if (lazy_default_value_enum_name_) { // Have to build the full name now instead of at CrossLink time, - // because enum_type_ may not be known at the time. - string name = enum_type_->full_name(); + // because enum_type may not be known at the time. + std::string name = enum_type->full_name(); // Enum values reside in the same scope as the enum type. - string::size_type last_dot = name.find_last_of('.'); - if (last_dot != string::npos) { - name = name.substr(0, last_dot) + "." + *default_value_enum_name_; + std::string::size_type last_dot = name.find_last_of('.'); + if (last_dot != std::string::npos) { + name = name.substr(0, last_dot) + "." + lazy_default_value_enum_name_; } else { - name = *default_value_enum_name_; + name = lazy_default_value_enum_name_; } Symbol result = file()->pool()->CrossLinkOnDemandHelper(name, true); - if (result.type == Symbol::ENUM_VALUE) { - default_value_enum_ = result.enum_value_descriptor; - } + default_value_enum_ = result.enum_value_descriptor(); + } else { + default_value_enum_ = nullptr; } if (!default_value_enum_) { // We use the first defined value as the default // if a default is not explicitly defined. - GOOGLE_CHECK(enum_type_->value_count()); - default_value_enum_ = enum_type_->value(0); + GOOGLE_CHECK(enum_type->value_count()); + default_value_enum_ = enum_type->value(0); } } } @@ -7038,34 +7921,46 @@ void FieldDescriptor::TypeOnceInit(const FieldDescriptor* to_init) { } // message_type(), enum_type(), default_value_enum(), and type() -// all share the same GoogleOnceDynamic init path to do lazy +// all share the same internal::call_once init path to do lazy // import building and cross linking of a field of a message. const Descriptor* FieldDescriptor::message_type() const { if (type_once_) { - type_once_->Init(&FieldDescriptor::TypeOnceInit, this); + internal::call_once(*type_once_, FieldDescriptor::TypeOnceInit, this); } - return message_type_; + return type_ == TYPE_MESSAGE || type_ == TYPE_GROUP + ? type_descriptor_.message_type + : nullptr; } const EnumDescriptor* FieldDescriptor::enum_type() const { if (type_once_) { - type_once_->Init(&FieldDescriptor::TypeOnceInit, this); + internal::call_once(*type_once_, FieldDescriptor::TypeOnceInit, this); } - return enum_type_; + return type_ == TYPE_ENUM ? type_descriptor_.enum_type : nullptr; } const EnumValueDescriptor* FieldDescriptor::default_value_enum() const { if (type_once_) { - type_once_->Init(&FieldDescriptor::TypeOnceInit, this); + internal::call_once(*type_once_, FieldDescriptor::TypeOnceInit, this); } return default_value_enum_; } +const std::string& FieldDescriptor::PrintableNameForExtension() const { + const bool is_message_set_extension = + is_extension() && + containing_type()->options().message_set_wire_format() && + type() == FieldDescriptor::TYPE_MESSAGE && is_optional() && + extension_scope() == message_type(); + return is_message_set_extension ? message_type()->full_name() : full_name(); +} + void FileDescriptor::InternalDependenciesOnceInit() const { GOOGLE_CHECK(finished_building_ == true); + auto* names = dependencies_once_->dependencies_names; for (int i = 0; i < dependency_count(); i++) { - if (dependencies_names_[i]) { - dependencies_[i] = pool_->FindFileByName(*dependencies_names_[i]); + if (names[i]) { + dependencies_[i] = pool_->FindFileByName(names[i]); } } } @@ -7076,62 +7971,55 @@ void FileDescriptor::DependenciesOnceInit(const FileDescriptor* to_init) { const FileDescriptor* FileDescriptor::dependency(int index) const { if (dependencies_once_) { - // Do once init for all indicies, as it's unlikely only a single index would - // be called, and saves on GoogleOnceDynamic allocations. - dependencies_once_->Init(&FileDescriptor::DependenciesOnceInit, this); + // Do once init for all indices, as it's unlikely only a single index would + // be called, and saves on internal::call_once allocations. + internal::call_once(dependencies_once_->once, + FileDescriptor::DependenciesOnceInit, this); } return dependencies_[index]; } const Descriptor* MethodDescriptor::input_type() const { - return input_type_.Get(); + return input_type_.Get(service()); } const Descriptor* MethodDescriptor::output_type() const { - return output_type_.Get(); + return output_type_.Get(service()); } namespace internal { void LazyDescriptor::Set(const Descriptor* descriptor) { - GOOGLE_CHECK(!name_); GOOGLE_CHECK(!once_); - GOOGLE_CHECK(!file_); descriptor_ = descriptor; } -void LazyDescriptor::SetLazy(const string& name, const FileDescriptor* file) { +void LazyDescriptor::SetLazy(StringPiece name, + const FileDescriptor* file) { // verify Init() has been called and Set hasn't been called yet. GOOGLE_CHECK(!descriptor_); - GOOGLE_CHECK(!file_); - GOOGLE_CHECK(!name_); GOOGLE_CHECK(!once_); GOOGLE_CHECK(file && file->pool_); GOOGLE_CHECK(file->pool_->lazily_build_dependencies_); GOOGLE_CHECK(!file->finished_building_); - file_ = file; - name_ = file->pool_->tables_->AllocateString(name); - once_ = file->pool_->tables_->AllocateOnceDynamic(); + once_ = file->pool_->tables_->Create(); + lazy_name_ = file->pool_->tables_->Strdup(name); } -void LazyDescriptor::Once() { +void LazyDescriptor::Once(const ServiceDescriptor* service) { if (once_) { - once_->Init(&LazyDescriptor::OnceStatic, this); + internal::call_once(*once_, [&] { + auto* file = service->file(); + GOOGLE_CHECK(file->finished_building_); + descriptor_ = + file->pool_->CrossLinkOnDemandHelper(lazy_name_, false).descriptor(); + }); } } -void LazyDescriptor::OnceStatic(LazyDescriptor* lazy) { lazy->OnceInternal(); } - -void LazyDescriptor::OnceInternal() { - GOOGLE_CHECK(file_->finished_building_); - if (!descriptor_ && name_) { - Symbol result = file_->pool_->CrossLinkOnDemandHelper(*name_, false); - if (!result.IsNull() && result.type == Symbol::MESSAGE) { - descriptor_ = result.descriptor; - } - } -} } // namespace internal } // namespace protobuf } // namespace google + +#include diff --git a/3rdparty/protobuf/src/google/protobuf/descriptor.h b/3rdparty/protobuf/src/google/protobuf/descriptor.h index 5f5159a8ad9e..e74e355b5a38 100644 --- a/3rdparty/protobuf/src/google/protobuf/descriptor.h +++ b/3rdparty/protobuf/src/google/protobuf/descriptor.h @@ -54,22 +54,30 @@ #ifndef GOOGLE_PROTOBUF_DESCRIPTOR_H__ #define GOOGLE_PROTOBUF_DESCRIPTOR_H__ +#include +#include #include -#ifndef _SHARED_PTR_H -#include -#endif #include #include #include + #include +#include #include #include +#include +#include // TYPE_BOOL is defined in the MacOS's ConditionalMacros.h. #ifdef TYPE_BOOL #undef TYPE_BOOL #endif // TYPE_BOOL +#ifdef SWIG +#define PROTOBUF_EXPORT +#endif + + namespace google { namespace protobuf { @@ -87,6 +95,7 @@ class DescriptorPool; // Defined in descriptor.proto class DescriptorProto; +class DescriptorProto_ExtensionRange; class FieldDescriptorProto; class OneofDescriptorProto; class EnumDescriptorProto; @@ -108,23 +117,23 @@ class SourceCodeInfo; // Defined in message.h class Message; +class Reflection; // Defined in descriptor.cc class DescriptorBuilder; class FileDescriptorTables; -struct Symbol; +class Symbol; // Defined in unknown_field_set.h. class UnknownField; -// Defined in generated_message_reflection.h. -namespace internal { -class GeneratedMessageReflection; -} // namespace internal - // Defined in command_line_interface.cc namespace compiler { class CommandLineInterface; +namespace cpp { +// Defined in helpers.h +class Formatter; +} // namespace cpp } // namespace compiler namespace descriptor_unittest { @@ -145,9 +154,9 @@ struct SourceLocation { // Doc comments found at the source location. // See the comments in SourceCodeInfo.Location (descriptor.proto) for details. - string leading_comments; - string trailing_comments; - std::vector leading_detached_comments; + std::string leading_comments; + std::string trailing_comments; + std::vector leading_detached_comments; }; // Options when generating machine-parsable output from a descriptor with @@ -165,7 +174,8 @@ struct DebugStringOptions { DebugStringOptions() : include_comments(false), elide_group_body(false), - elide_oneof_body(false) {} + elide_oneof_body(false) { + } }; // A class to handle the simplest cases of a lazily linked descriptor @@ -173,15 +183,14 @@ struct DebugStringOptions { // which is needed when a pool has lazily_build_dependencies_ set. // Must be instantiated as mutable in a descriptor. namespace internal { -class LIBPROTOBUF_EXPORT LazyDescriptor { + +class PROTOBUF_EXPORT LazyDescriptor { public: // Init function to be called at init time of a descriptor containing // a LazyDescriptor. void Init() { - descriptor_ = NULL; - name_ = NULL; - once_ = NULL; - file_ = NULL; + descriptor_ = nullptr; + once_ = nullptr; } // Sets the value of the descriptor if it is known during the descriptor @@ -195,26 +204,39 @@ class LIBPROTOBUF_EXPORT LazyDescriptor { // build time if the symbol wasn't found and building of the file containing // that type is delayed because lazily_build_dependencies_ is set on the pool. // Should not be called after Set() has been called. - void SetLazy(const string& name, const FileDescriptor* file); + void SetLazy(StringPiece name, const FileDescriptor* file); // Returns the current value of the descriptor, thread-safe. If SetLazy(...) // has been called, will do a one-time cross link of the type specified, // building the descriptor file that contains the type if necessary. - inline const Descriptor* Get() { - Once(); + inline const Descriptor* Get(const ServiceDescriptor* service) { + Once(service); return descriptor_; } private: - static void OnceStatic(LazyDescriptor* lazy); - void OnceInternal(); - void Once(); + void Once(const ServiceDescriptor* service); - const Descriptor* descriptor_; - const string* name_; - GoogleOnceDynamic* once_; - const FileDescriptor* file_; + union { + const Descriptor* descriptor_; + const char* lazy_name_; + }; + internal::once_flag* once_; }; + +class PROTOBUF_EXPORT SymbolBase { + private: + friend class google::protobuf::Symbol; + uint8_t symbol_type_; +}; + +// Some types have more than one SymbolBase because they have multiple +// identities in the table. We can't have duplicate direct bases, so we use this +// intermediate base to do so. +// See BuildEnumValue for details. +template +class PROTOBUF_EXPORT SymbolBaseN : public SymbolBase {}; + } // namespace internal // Describes a type of protocol message, or a particular group within a @@ -222,34 +244,35 @@ class LIBPROTOBUF_EXPORT LazyDescriptor { // Message::GetDescriptor(). Generated message classes also have a // static method called descriptor() which returns the type's descriptor. // Use DescriptorPool to construct your own descriptors. -class LIBPROTOBUF_EXPORT Descriptor { +class PROTOBUF_EXPORT Descriptor : private internal::SymbolBase { public: + typedef DescriptorProto Proto; + // The name of the message type, not including its scope. - const string& name() const; + const std::string& name() const; // The fully-qualified name of the message type, scope delimited by // periods. For example, message type "Foo" which is declared in package // "bar" has full name "bar.Foo". If a type "Baz" is nested within // Foo, Baz's full_name is "bar.Foo.Baz". To get only the part that // comes after the last '.', use name(). - const string& full_name() const; + const std::string& full_name() const; // Index of this descriptor within the file or containing type's message // type array. int index() const; - // The .proto file in which this message type was defined. Never NULL. + // The .proto file in which this message type was defined. Never nullptr. const FileDescriptor* file() const; // If this Descriptor describes a nested type, this returns the type - // in which it is nested. Otherwise, returns NULL. + // in which it is nested. Otherwise, returns nullptr. const Descriptor* containing_type() const; // Get options for this message type. These are specified in the .proto file // by placing lines like "option foo = 1234;" in the message definition. - // Allowed options are defined by MessageOptions in - // google/protobuf/descriptor.proto, and any available extensions of that - // message. + // Allowed options are defined by MessageOptions in descriptor.proto, and any + // available extensions of that message. const MessageOptions& options() const; // Write the contents of this Descriptor into the given DescriptorProto. @@ -257,19 +280,49 @@ class LIBPROTOBUF_EXPORT Descriptor { // isn't, the result may be garbage. void CopyTo(DescriptorProto* proto) const; - // Write the contents of this decriptor in a human-readable form. Output + // Write the contents of this descriptor in a human-readable form. Output // will be suitable for re-parsing. - string DebugString() const; + std::string DebugString() const; // Similar to DebugString(), but additionally takes options (e.g., // include original user comments in output). - string DebugStringWithOptions(const DebugStringOptions& options) const; + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Returns true if this is a placeholder for an unknown type. This will // only be the case if this descriptor comes from a DescriptorPool // with AllowUnknownDependencies() set. bool is_placeholder() const; + enum WellKnownType { + WELLKNOWNTYPE_UNSPECIFIED, // Not a well-known type. + + // Wrapper types. + WELLKNOWNTYPE_DOUBLEVALUE, // google.protobuf.DoubleValue + WELLKNOWNTYPE_FLOATVALUE, // google.protobuf.FloatValue + WELLKNOWNTYPE_INT64VALUE, // google.protobuf.Int64Value + WELLKNOWNTYPE_UINT64VALUE, // google.protobuf.UInt64Value + WELLKNOWNTYPE_INT32VALUE, // google.protobuf.Int32Value + WELLKNOWNTYPE_UINT32VALUE, // google.protobuf.UInt32Value + WELLKNOWNTYPE_STRINGVALUE, // google.protobuf.StringValue + WELLKNOWNTYPE_BYTESVALUE, // google.protobuf.BytesValue + WELLKNOWNTYPE_BOOLVALUE, // google.protobuf.BoolValue + + // Other well known types. + WELLKNOWNTYPE_ANY, // google.protobuf.Any + WELLKNOWNTYPE_FIELDMASK, // google.protobuf.FieldMask + WELLKNOWNTYPE_DURATION, // google.protobuf.Duration + WELLKNOWNTYPE_TIMESTAMP, // google.protobuf.Timestamp + WELLKNOWNTYPE_VALUE, // google.protobuf.Value + WELLKNOWNTYPE_LISTVALUE, // google.protobuf.ListValue + WELLKNOWNTYPE_STRUCT, // google.protobuf.Struct + + // New well-known types may be added in the future. + // Please make sure any switch() statements have a 'default' case. + __WELLKNOWNTYPE__DO_NOT_USE__ADD_DEFAULT_INSTEAD__, + }; + + WellKnownType well_known_type() const; + // Field stuff ----------------------------------------------------- // The number of fields in this message type. @@ -278,33 +331,37 @@ class LIBPROTOBUF_EXPORT Descriptor { // These are returned in the order they were defined in the .proto file. const FieldDescriptor* field(int index) const; - // Looks up a field by declared tag number. Returns NULL if no such field + // Looks up a field by declared tag number. Returns nullptr if no such field // exists. const FieldDescriptor* FindFieldByNumber(int number) const; - // Looks up a field by name. Returns NULL if no such field exists. - const FieldDescriptor* FindFieldByName(const string& name) const; + // Looks up a field by name. Returns nullptr if no such field exists. + const FieldDescriptor* FindFieldByName(ConstStringParam name) const; // Looks up a field by lowercased name (as returned by lowercase_name()). // This lookup may be ambiguous if multiple field names differ only by case, // in which case the field returned is chosen arbitrarily from the matches. const FieldDescriptor* FindFieldByLowercaseName( - const string& lowercase_name) const; + ConstStringParam lowercase_name) const; // Looks up a field by camel-case name (as returned by camelcase_name()). // This lookup may be ambiguous if multiple field names differ in a way that // leads them to have identical camel-case names, in which case the field // returned is chosen arbitrarily from the matches. const FieldDescriptor* FindFieldByCamelcaseName( - const string& camelcase_name) const; + ConstStringParam camelcase_name) const; // The number of oneofs in this message type. int oneof_decl_count() const; + // The number of oneofs in this message type, excluding synthetic oneofs. + // Real oneofs always come first, so iterating up to real_oneof_decl_cout() + // will yield all real oneofs. + int real_oneof_decl_count() const; // Get a oneof by index, where 0 <= index < oneof_decl_count(). // These are returned in the order they were defined in the .proto file. const OneofDescriptor* oneof_decl(int index) const; - // Looks up a oneof by name. Returns NULL if no such oneof exists. - const OneofDescriptor* FindOneofByName(const string& name) const; + // Looks up a oneof by name. Returns nullptr if no such oneof exists. + const OneofDescriptor* FindOneofByName(ConstStringParam name) const; // Nested type stuff ----------------------------------------------- @@ -314,9 +371,9 @@ class LIBPROTOBUF_EXPORT Descriptor { // These are returned in the order they were defined in the .proto file. const Descriptor* nested_type(int index) const; - // Looks up a nested type by name. Returns NULL if no such nested type + // Looks up a nested type by name. Returns nullptr if no such nested type // exists. - const Descriptor* FindNestedTypeByName(const string& name) const; + const Descriptor* FindNestedTypeByName(ConstStringParam name) const; // Enum stuff ------------------------------------------------------ @@ -326,20 +383,26 @@ class LIBPROTOBUF_EXPORT Descriptor { // These are returned in the order they were defined in the .proto file. const EnumDescriptor* enum_type(int index) const; - // Looks up an enum type by name. Returns NULL if no such enum type exists. - const EnumDescriptor* FindEnumTypeByName(const string& name) const; + // Looks up an enum type by name. Returns nullptr if no such enum type + // exists. + const EnumDescriptor* FindEnumTypeByName(ConstStringParam name) const; // Looks up an enum value by name, among all enum types in this message. - // Returns NULL if no such value exists. - const EnumValueDescriptor* FindEnumValueByName(const string& name) const; + // Returns nullptr if no such value exists. + const EnumValueDescriptor* FindEnumValueByName(ConstStringParam name) const; // Extensions ------------------------------------------------------ // A range of field numbers which are designated for third-party // extensions. struct ExtensionRange { + typedef DescriptorProto_ExtensionRange Proto; + typedef ExtensionRangeOptions OptionsType; + // See Descriptor::CopyTo(). + void CopyTo(DescriptorProto_ExtensionRange* proto) const; + int start; // inclusive int end; // exclusive @@ -356,11 +419,33 @@ class LIBPROTOBUF_EXPORT Descriptor { // Returns true if the number is in one of the extension ranges. bool IsExtensionNumber(int number) const; - // Returns NULL if no extension range contains the given number. + // Returns nullptr if no extension range contains the given number. const ExtensionRange* FindExtensionRangeContainingNumber(int number) const; - // The number of extensions -- extending *other* messages -- that were - // defined nested within this message type's scope. + // The number of extensions defined nested within this message type's scope. + // See doc: + // https://developers.google.com/protocol-buffers/docs/proto#nested-extensions + // + // Note that the extensions may be extending *other* messages. + // + // For example: + // message M1 { + // extensions 1 to max; + // } + // + // message M2 { + // extend M1 { + // optional int32 foo = 1; + // } + // } + // + // In this case, + // DescriptorPool::generated_pool() + // ->FindMessageTypeByName("M2") + // ->extension(0) + // will return "foo", even though "foo" is an extension of M1. + // To find all known extensions of a given message, instead use + // DescriptorPool::FindAllExtensions. int extension_count() const; // Get an extension by index, where 0 <= index < extension_count(). // These are returned in the order they were defined in the .proto file. @@ -368,15 +453,17 @@ class LIBPROTOBUF_EXPORT Descriptor { // Looks up a named extension (which extends some *other* message type) // defined within this message type's scope. - const FieldDescriptor* FindExtensionByName(const string& name) const; + const FieldDescriptor* FindExtensionByName(ConstStringParam name) const; // Similar to FindFieldByLowercaseName(), but finds extensions defined within // this message type's scope. - const FieldDescriptor* FindExtensionByLowercaseName(const string& name) const; + const FieldDescriptor* FindExtensionByLowercaseName( + ConstStringParam name) const; // Similar to FindFieldByCamelcaseName(), but finds extensions defined within // this message type's scope. - const FieldDescriptor* FindExtensionByCamelcaseName(const string& name) const; + const FieldDescriptor* FindExtensionByCamelcaseName( + ConstStringParam name) const; // Reserved fields ------------------------------------------------- @@ -396,17 +483,17 @@ class LIBPROTOBUF_EXPORT Descriptor { // Returns true if the number is in one of the reserved ranges. bool IsReservedNumber(int number) const; - // Returns NULL if no reserved range contains the given number. + // Returns nullptr if no reserved range contains the given number. const ReservedRange* FindReservedRangeContainingNumber(int number) const; // The number of reserved field names in this message type. int reserved_name_count() const; // Gets a reserved name by index, where 0 <= index < reserved_name_count(). - const string& reserved_name(int index) const; + const std::string& reserved_name(int index) const; // Returns true if the field name is reserved. - bool IsReservedName(const string& name) const; + bool IsReservedName(ConstStringParam name) const; // Source Location --------------------------------------------------- @@ -415,14 +502,26 @@ class LIBPROTOBUF_EXPORT Descriptor { // |*out_location| unchanged iff location information was not available. bool GetSourceLocation(SourceLocation* out_location) const; + // Maps -------------------------------------------------------------- + + // Returns the FieldDescriptor for the "key" field. If this isn't a map entry + // field, returns nullptr. + const FieldDescriptor* map_key() const; + + // Returns the FieldDescriptor for the "value" field. If this isn't a map + // entry field, returns nullptr. + const FieldDescriptor* map_value() const; + private: + friend class Symbol; typedef MessageOptions OptionsType; // Allows tests to test CopyTo(proto, true). - friend class ::google::protobuf::descriptor_unittest::DescriptorTest; + friend class descriptor_unittest::DescriptorTest; // Allows access to GetLocationPath for annotations. - friend class ::google::protobuf::io::Printer; + friend class io::Printer; + friend class compiler::cpp::Formatter; // Fill the json_name field of FieldDescriptorProto. void CopyJsonNameTo(DescriptorProto* proto) const; @@ -431,7 +530,7 @@ class LIBPROTOBUF_EXPORT Descriptor { // correct depth. Takes |options| to control debug-string options, and // |include_opening_clause| to indicate whether the "message ... " part of the // clause has already been generated (this varies depending on context). - void DebugString(int depth, string *contents, + void DebugString(int depth, std::string* contents, const DebugStringOptions& options, bool include_opening_clause) const; @@ -439,8 +538,26 @@ class LIBPROTOBUF_EXPORT Descriptor { // to this descriptor from the file root. void GetLocationPath(std::vector* output) const; - const string* name_; - const string* full_name_; + // True if this is a placeholder for an unknown type. + bool is_placeholder_ : 1; + // True if this is a placeholder and the type name wasn't fully-qualified. + bool is_unqualified_placeholder_ : 1; + // Well known type. Stored like this to conserve space. + uint8_t well_known_type_ : 5; + + // This points to the last field _number_ that is part of the sequence + // starting at 1, where + // `desc->field(i)->number() == i + 1` + // A value of `0` means no field matches. That is, there are no fields or the + // first field is not field `1`. + // Uses 16-bit to avoid extra padding. Unlikely to have more than 2^16 + // sequentially numbered fields in a message. + uint16_t sequential_field_limit_; + + int field_count_; + + // all_names_ = [name, full_name] + const std::string* all_names_; const FileDescriptor* file_; const Descriptor* containing_type_; const MessageOptions* options_; @@ -453,10 +570,10 @@ class LIBPROTOBUF_EXPORT Descriptor { ExtensionRange* extension_ranges_; FieldDescriptor* extensions_; ReservedRange* reserved_ranges_; - const string** reserved_names_; + const std::string** reserved_names_; - int field_count_; int oneof_decl_count_; + int real_oneof_decl_count_; int nested_type_count_; int enum_type_count_; int extension_range_count_; @@ -464,11 +581,6 @@ class LIBPROTOBUF_EXPORT Descriptor { int reserved_range_count_; int reserved_name_count_; - // True if this is a placeholder for an unknown type. - bool is_placeholder_; - // True if this is a placeholder and the type name wasn't fully-qualified. - bool is_unqualified_placeholder_; - // IMPORTANT: If you add a new field, make sure to search for all instances // of Allocate() and AllocateArray() in descriptor.cc // and update them to initialize the field. @@ -479,6 +591,7 @@ class LIBPROTOBUF_EXPORT Descriptor { friend class DescriptorPool; friend class EnumDescriptor; friend class FieldDescriptor; + friend class FileDescriptorTables; friend class OneofDescriptor; friend class MethodDescriptor; friend class FileDescriptor; @@ -493,72 +606,72 @@ class LIBPROTOBUF_EXPORT Descriptor { // - Get the Descriptor or FileDescriptor for its containing scope, then // call Descriptor::FindExtensionByName() or // FileDescriptor::FindExtensionByName(). -// - Given a DescriptorPool, call DescriptorPool::FindExtensionByNumber(). -// - Given a Reflection for a message object, call -// Reflection::FindKnownExtensionByName() or -// Reflection::FindKnownExtensionByNumber(). +// - Given a DescriptorPool, call DescriptorPool::FindExtensionByNumber() or +// DescriptorPool::FindExtensionByPrintableName(). // Use DescriptorPool to construct your own descriptors. -class LIBPROTOBUF_EXPORT FieldDescriptor { +class PROTOBUF_EXPORT FieldDescriptor : private internal::SymbolBase { public: + typedef FieldDescriptorProto Proto; + // Identifies a field type. 0 is reserved for errors. The order is weird // for historical reasons. Types 12 and up are new in proto2. enum Type { - TYPE_DOUBLE = 1, // double, exactly eight bytes on the wire. - TYPE_FLOAT = 2, // float, exactly four bytes on the wire. - TYPE_INT64 = 3, // int64, varint on the wire. Negative numbers - // take 10 bytes. Use TYPE_SINT64 if negative - // values are likely. - TYPE_UINT64 = 4, // uint64, varint on the wire. - TYPE_INT32 = 5, // int32, varint on the wire. Negative numbers - // take 10 bytes. Use TYPE_SINT32 if negative - // values are likely. - TYPE_FIXED64 = 6, // uint64, exactly eight bytes on the wire. - TYPE_FIXED32 = 7, // uint32, exactly four bytes on the wire. - TYPE_BOOL = 8, // bool, varint on the wire. - TYPE_STRING = 9, // UTF-8 text. - TYPE_GROUP = 10, // Tag-delimited message. Deprecated. - TYPE_MESSAGE = 11, // Length-delimited message. - - TYPE_BYTES = 12, // Arbitrary byte array. - TYPE_UINT32 = 13, // uint32, varint on the wire - TYPE_ENUM = 14, // Enum, varint on the wire - TYPE_SFIXED32 = 15, // int32, exactly four bytes on the wire - TYPE_SFIXED64 = 16, // int64, exactly eight bytes on the wire - TYPE_SINT32 = 17, // int32, ZigZag-encoded varint on the wire - TYPE_SINT64 = 18, // int64, ZigZag-encoded varint on the wire - - MAX_TYPE = 18, // Constant useful for defining lookup tables - // indexed by Type. + TYPE_DOUBLE = 1, // double, exactly eight bytes on the wire. + TYPE_FLOAT = 2, // float, exactly four bytes on the wire. + TYPE_INT64 = 3, // int64, varint on the wire. Negative numbers + // take 10 bytes. Use TYPE_SINT64 if negative + // values are likely. + TYPE_UINT64 = 4, // uint64, varint on the wire. + TYPE_INT32 = 5, // int32, varint on the wire. Negative numbers + // take 10 bytes. Use TYPE_SINT32 if negative + // values are likely. + TYPE_FIXED64 = 6, // uint64, exactly eight bytes on the wire. + TYPE_FIXED32 = 7, // uint32, exactly four bytes on the wire. + TYPE_BOOL = 8, // bool, varint on the wire. + TYPE_STRING = 9, // UTF-8 text. + TYPE_GROUP = 10, // Tag-delimited message. Deprecated. + TYPE_MESSAGE = 11, // Length-delimited message. + + TYPE_BYTES = 12, // Arbitrary byte array. + TYPE_UINT32 = 13, // uint32, varint on the wire + TYPE_ENUM = 14, // Enum, varint on the wire + TYPE_SFIXED32 = 15, // int32, exactly four bytes on the wire + TYPE_SFIXED64 = 16, // int64, exactly eight bytes on the wire + TYPE_SINT32 = 17, // int32, ZigZag-encoded varint on the wire + TYPE_SINT64 = 18, // int64, ZigZag-encoded varint on the wire + + MAX_TYPE = 18, // Constant useful for defining lookup tables + // indexed by Type. }; // Specifies the C++ data type used to represent the field. There is a // fixed mapping from Type to CppType where each Type maps to exactly one // CppType. 0 is reserved for errors. enum CppType { - CPPTYPE_INT32 = 1, // TYPE_INT32, TYPE_SINT32, TYPE_SFIXED32 - CPPTYPE_INT64 = 2, // TYPE_INT64, TYPE_SINT64, TYPE_SFIXED64 - CPPTYPE_UINT32 = 3, // TYPE_UINT32, TYPE_FIXED32 - CPPTYPE_UINT64 = 4, // TYPE_UINT64, TYPE_FIXED64 - CPPTYPE_DOUBLE = 5, // TYPE_DOUBLE - CPPTYPE_FLOAT = 6, // TYPE_FLOAT - CPPTYPE_BOOL = 7, // TYPE_BOOL - CPPTYPE_ENUM = 8, // TYPE_ENUM - CPPTYPE_STRING = 9, // TYPE_STRING, TYPE_BYTES - CPPTYPE_MESSAGE = 10, // TYPE_MESSAGE, TYPE_GROUP - - MAX_CPPTYPE = 10, // Constant useful for defining lookup tables - // indexed by CppType. + CPPTYPE_INT32 = 1, // TYPE_INT32, TYPE_SINT32, TYPE_SFIXED32 + CPPTYPE_INT64 = 2, // TYPE_INT64, TYPE_SINT64, TYPE_SFIXED64 + CPPTYPE_UINT32 = 3, // TYPE_UINT32, TYPE_FIXED32 + CPPTYPE_UINT64 = 4, // TYPE_UINT64, TYPE_FIXED64 + CPPTYPE_DOUBLE = 5, // TYPE_DOUBLE + CPPTYPE_FLOAT = 6, // TYPE_FLOAT + CPPTYPE_BOOL = 7, // TYPE_BOOL + CPPTYPE_ENUM = 8, // TYPE_ENUM + CPPTYPE_STRING = 9, // TYPE_STRING, TYPE_BYTES + CPPTYPE_MESSAGE = 10, // TYPE_MESSAGE, TYPE_GROUP + + MAX_CPPTYPE = 10, // Constant useful for defining lookup tables + // indexed by CppType. }; // Identifies whether the field is optional, required, or repeated. 0 is // reserved for errors. enum Label { - LABEL_OPTIONAL = 1, // optional - LABEL_REQUIRED = 2, // required - LABEL_REPEATED = 3, // repeated + LABEL_OPTIONAL = 1, // optional + LABEL_REQUIRED = 2, // required + LABEL_REPEATED = 3, // repeated - MAX_LABEL = 3, // Constant useful for defining lookup tables - // indexed by Label. + MAX_LABEL = 3, // Constant useful for defining lookup tables + // indexed by Label. }; // Valid field numbers are positive integers up to kMaxNumber. @@ -569,14 +682,14 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { static const int kFirstReservedNumber = 19000; // Last field number reserved for the protocol buffer library implementation. // Users may not declare fields that use reserved numbers. - static const int kLastReservedNumber = 19999; + static const int kLastReservedNumber = 19999; - const string& name() const; // Name of this field within the message. - const string& full_name() const; // Fully-qualified name of the field. - const string& json_name() const; // JSON name of this field. - const FileDescriptor* file() const;// File in which this field was defined. - bool is_extension() const; // Is this an extension field? - int number() const; // Declared tag number. + const std::string& name() const; // Name of this field within the message. + const std::string& full_name() const; // Fully-qualified name of the field. + const std::string& json_name() const; // JSON name of this field. + const FileDescriptor* file() const; // File in which this field was defined. + bool is_extension() const; // Is this an extension field? + int number() const; // Declared tag number. // Same as name() except converted to lower-case. This (and especially the // FindFieldByLowercaseName() method) can be useful when parsing formats @@ -584,7 +697,7 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // field names should be lowercased anyway according to the protobuf style // guide, so this only makes a difference when dealing with old .proto files // which do not follow the guide.) - const string& lowercase_name() const; + const std::string& lowercase_name() const; // Same as name() except converted to camel-case. In this conversion, any // time an underscore appears in the name, it is removed and the next @@ -595,7 +708,7 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // fooBar -> fooBar // This (and especially the FindFieldByCamelcaseName() method) can be useful // when parsing formats which prefer to use camel-case naming style. - const string& camelcase_name() const; + const std::string& camelcase_name() const; Type type() const; // Declared type of this field. const char* type_name() const; // Name of the declared type. @@ -603,15 +716,28 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { const char* cpp_type_name() const; // Name of the C++ type. Label label() const; // optional/required/repeated - bool is_required() const; // shorthand for label() == LABEL_REQUIRED - bool is_optional() const; // shorthand for label() == LABEL_OPTIONAL - bool is_repeated() const; // shorthand for label() == LABEL_REPEATED - bool is_packable() const; // shorthand for is_repeated() && - // IsTypePackable(type()) - bool is_packed() const; // shorthand for is_packable() && - // options().packed() - bool is_map() const; // shorthand for type() == TYPE_MESSAGE && - // message_type()->options().map_entry() + bool is_required() const; // shorthand for label() == LABEL_REQUIRED + bool is_optional() const; // shorthand for label() == LABEL_OPTIONAL + bool is_repeated() const; // shorthand for label() == LABEL_REPEATED + bool is_packable() const; // shorthand for is_repeated() && + // IsTypePackable(type()) + bool is_packed() const; // shorthand for is_packable() && + // options().packed() + bool is_map() const; // shorthand for type() == TYPE_MESSAGE && + // message_type()->options().map_entry() + + // Returns true if this field was syntactically written with "optional" in the + // .proto file. Excludes singular proto3 fields that do not have a label. + bool has_optional_keyword() const; + + // Returns true if this field tracks presence, ie. does the field + // distinguish between "unset" and "present with default value." + // This includes required, optional, and oneof fields. It excludes maps, + // repeated fields, and singular proto3 fields without "optional". + // + // For fields where has_presence() == true, the return value of + // Reflection::HasField() is semantically meaningful. + bool has_presence() const; // Index of this field within the message's field array, or the file or // extension scope's extensions array. @@ -626,16 +752,20 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // Get the field default value if cpp_type() == CPPTYPE_INT32. If no // explicit default was defined, the default is 0. - int32 default_value_int32() const; + int32_t default_value_int32_t() const; + int32_t default_value_int32() const { return default_value_int32_t(); } // Get the field default value if cpp_type() == CPPTYPE_INT64. If no // explicit default was defined, the default is 0. - int64 default_value_int64() const; + int64_t default_value_int64_t() const; + int64_t default_value_int64() const { return default_value_int64_t(); } // Get the field default value if cpp_type() == CPPTYPE_UINT32. If no // explicit default was defined, the default is 0. - uint32 default_value_uint32() const; + uint32_t default_value_uint32_t() const; + uint32_t default_value_uint32() const { return default_value_uint32_t(); } // Get the field default value if cpp_type() == CPPTYPE_UINT64. If no // explicit default was defined, the default is 0. - uint64 default_value_uint64() const; + uint64_t default_value_uint64_t() const; + uint64_t default_value_uint64() const { return default_value_uint64_t(); } // Get the field default value if cpp_type() == CPPTYPE_FLOAT. If no // explicit default was defined, the default is 0.0. float default_value_float() const; @@ -648,26 +778,30 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // Get the field default value if cpp_type() == CPPTYPE_ENUM. If no // explicit default was defined, the default is the first value defined // in the enum type (all enum types are required to have at least one value). - // This never returns NULL. + // This never returns nullptr. const EnumValueDescriptor* default_value_enum() const; // Get the field default value if cpp_type() == CPPTYPE_STRING. If no // explicit default was defined, the default is the empty string. - const string& default_value_string() const; + const std::string& default_value_string() const; // The Descriptor for the message of which this is a field. For extensions, - // this is the extended type. Never NULL. + // this is the extended type. Never nullptr. const Descriptor* containing_type() const; // If the field is a member of a oneof, this is the one, otherwise this is - // NULL. + // nullptr. const OneofDescriptor* containing_oneof() const; + // If the field is a member of a non-synthetic oneof, returns the descriptor + // for the oneof, otherwise returns nullptr. + const OneofDescriptor* real_containing_oneof() const; + // If the field is a member of a oneof, returns the index in that oneof. int index_in_oneof() const; // An extension may be declared within the scope of another message. If this // field is an extension (is_extension() is true), then extension_scope() - // returns that message, or NULL if the extension was declared at global + // returns that message, or nullptr if the extension was declared at global // scope. If this is not an extension, extension_scope() is undefined (may // assert-fail). const Descriptor* extension_scope() const; @@ -682,19 +816,18 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // Get the FieldOptions for this field. This includes things listed in // square brackets after the field definition. E.g., the field: // optional string text = 1 [ctype=CORD]; - // has the "ctype" option set. Allowed options are defined by FieldOptions - // in google/protobuf/descriptor.proto, and any available extensions of that - // message. + // has the "ctype" option set. Allowed options are defined by FieldOptions in + // descriptor.proto, and any available extensions of that message. const FieldOptions& options() const; // See Descriptor::CopyTo(). void CopyTo(FieldDescriptorProto* proto) const; // See Descriptor::DebugString(). - string DebugString() const; + std::string DebugString() const; // See Descriptor::DebugStringWithOptions(). - string DebugStringWithOptions(const DebugStringOptions& options) const; + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Helper method to get the CppType for a particular Type. static CppType TypeToCppType(Type type); @@ -708,6 +841,21 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // Return true iff [packed = true] is valid for fields of this type. static inline bool IsTypePackable(Type field_type); + // Returns full_name() except if the field is a MessageSet extension, + // in which case it returns the full_name() of the containing message type + // for backwards compatibility with proto1. + // + // A MessageSet extension is defined as an optional message extension + // whose containing type has the message_set_wire_format option set. + // This should be true of extensions of google.protobuf.bridge.MessageSet; + // by convention, such extensions are named "message_set_extension". + // + // The opposite operation (looking up an extension's FieldDescriptor given + // its printable name) can be accomplished with + // message->file()->pool()->FindExtensionByPrintableName(message, name) + // where the extension extends "message". + const std::string& PrintableNameForExtension() const; + // Source Location --------------------------------------------------- // Updates |*out_location| to the source location of the complete @@ -716,26 +864,28 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { bool GetSourceLocation(SourceLocation* out_location) const; private: + friend class Symbol; typedef FieldOptions OptionsType; // Allows access to GetLocationPath for annotations. - friend class ::google::protobuf::io::Printer; + friend class io::Printer; + friend class compiler::cpp::Formatter; + friend class Reflection; // Fill the json_name field of FieldDescriptorProto. void CopyJsonNameTo(FieldDescriptorProto* proto) const; // See Descriptor::DebugString(). - enum PrintLabelFlag { PRINT_LABEL, OMIT_LABEL }; - void DebugString(int depth, PrintLabelFlag print_label_flag, - string* contents, const DebugStringOptions& options) const; + void DebugString(int depth, std::string* contents, + const DebugStringOptions& options) const; // formats the default value appropriately and returns it as a string. // Must have a default value to call this. If quote_string_type is true, then // types of CPPTYPE_STRING whill be surrounded by quotes and CEscaped. - string DefaultValueAsString(bool quote_string_type) const; + std::string DefaultValueAsString(bool quote_string_type) const; // Helper function that returns the field type name for DebugString. - string FieldTypeNameDebugString() const; + std::string FieldTypeNameDebugString() const; // Walks up the descriptor tree to generate the source location path // to this descriptor from the file root. @@ -744,58 +894,76 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // Returns true if this is a map message type. bool is_map_message_type() const; - const string* name_; - const string* full_name_; - const string* lowercase_name_; - const string* camelcase_name_; - // If has_json_name_ is true, it's the value specified by the user. - // Otherwise, it has the same value as camelcase_name_. - const string* json_name_; - const FileDescriptor* file_; - GoogleOnceDynamic* type_once_; - static void TypeOnceInit(const FieldDescriptor* to_init); - void InternalTypeOnceInit() const; - mutable Type type_; - Label label_; - bool has_default_value_; + bool has_default_value_ : 1; + bool proto3_optional_ : 1; // Whether the user has specified the json_name field option in the .proto // file. - bool has_json_name_; - bool is_extension_; + bool has_json_name_ : 1; + bool is_extension_ : 1; + bool is_oneof_ : 1; + + // Actually a `Label` but stored as uint8_t to save space. + uint8_t label_ : 2; + + // Actually a `Type`, but stored as uint8_t to save space. + mutable uint8_t type_; + + // Logically: + // all_names_ = [name, full_name, lower, camel, json] + // However: + // duplicates will be omitted, so lower/camel/json might be in the same + // position. + // We store the true offset for each name here, and the bit width must be + // large enough to account for the worst case where all names are present. + uint8_t lowercase_name_index_ : 2; + uint8_t camelcase_name_index_ : 2; + uint8_t json_name_index_ : 3; + // Sadly, `number_` located here to reduce padding. Unrelated to all_names_ + // and its indices above. int number_; - int index_in_oneof_; + const std::string* all_names_; + const FileDescriptor* file_; + + internal::once_flag* type_once_; + static void TypeOnceInit(const FieldDescriptor* to_init); + void InternalTypeOnceInit() const; const Descriptor* containing_type_; - const OneofDescriptor* containing_oneof_; - const Descriptor* extension_scope_; - mutable const Descriptor* message_type_; - mutable const EnumDescriptor* enum_type_; + union { + const OneofDescriptor* containing_oneof; + const Descriptor* extension_scope; + } scope_; + union { + mutable const Descriptor* message_type; + mutable const EnumDescriptor* enum_type; + const char* lazy_type_name; + } type_descriptor_; const FieldOptions* options_; - const string* type_name_; - const string* default_value_enum_name_; // IMPORTANT: If you add a new field, make sure to search for all instances // of Allocate() and AllocateArray() in // descriptor.cc and update them to initialize the field. union { - int32 default_value_int32_; - int64 default_value_int64_; - uint32 default_value_uint32_; - uint64 default_value_uint64_; - float default_value_float_; + int32_t default_value_int32_t_; + int64_t default_value_int64_t_; + uint32_t default_value_uint32_t_; + uint64_t default_value_uint64_t_; + float default_value_float_; double default_value_double_; - bool default_value_bool_; + bool default_value_bool_; mutable const EnumValueDescriptor* default_value_enum_; - const string* default_value_string_; + const char* lazy_default_value_enum_name_; + const std::string* default_value_string_; + mutable std::atomic default_generated_instance_; }; static const CppType kTypeToCppTypeMap[MAX_TYPE + 1]; - static const char * const kTypeToName[MAX_TYPE + 1]; + static const char* const kTypeToName[MAX_TYPE + 1]; - static const char * const kCppTypeToName[MAX_CPPTYPE + 1]; + static const char* const kCppTypeToName[MAX_CPPTYPE + 1]; - static const char * const kLabelToName[MAX_LABEL + 1]; + static const char* const kLabelToName[MAX_LABEL + 1]; // Must be constructed using DescriptorPool. FieldDescriptor() {} @@ -808,15 +976,21 @@ class LIBPROTOBUF_EXPORT FieldDescriptor { // Describes a oneof defined in a message type. -class LIBPROTOBUF_EXPORT OneofDescriptor { +class PROTOBUF_EXPORT OneofDescriptor : private internal::SymbolBase { public: - const string& name() const; // Name of this oneof. - const string& full_name() const; // Fully-qualified name of the oneof. + typedef OneofDescriptorProto Proto; + + const std::string& name() const; // Name of this oneof. + const std::string& full_name() const; // Fully-qualified name of the oneof. // Index of this oneof within the message's oneof array. int index() const; - // The .proto file in which this oneof was defined. Never NULL. + // Returns whether this oneof was inserted by the compiler to wrap a proto3 + // optional field. If this returns true, code generators should *not* emit it. + bool is_synthetic() const; + + // The .proto file in which this oneof was defined. Never nullptr. const FileDescriptor* file() const; // The Descriptor for the message containing this oneof. const Descriptor* containing_type() const; @@ -833,10 +1007,10 @@ class LIBPROTOBUF_EXPORT OneofDescriptor { void CopyTo(OneofDescriptorProto* proto) const; // See Descriptor::DebugString(). - string DebugString() const; + std::string DebugString() const; // See Descriptor::DebugStringWithOptions(). - string DebugStringWithOptions(const DebugStringOptions& options) const; + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Source Location --------------------------------------------------- @@ -846,26 +1020,28 @@ class LIBPROTOBUF_EXPORT OneofDescriptor { bool GetSourceLocation(SourceLocation* out_location) const; private: + friend class Symbol; typedef OneofOptions OptionsType; // Allows access to GetLocationPath for annotations. - friend class ::google::protobuf::io::Printer; + friend class io::Printer; + friend class compiler::cpp::Formatter; // See Descriptor::DebugString(). - void DebugString(int depth, string* contents, + void DebugString(int depth, std::string* contents, const DebugStringOptions& options) const; // Walks up the descriptor tree to generate the source location path // to this descriptor from the file root. void GetLocationPath(std::vector* output) const; - const string* name_; - const string* full_name_; - const Descriptor* containing_type_; - bool is_extendable_; int field_count_; - const FieldDescriptor** fields_; + + // all_names_ = [name, full_name] + const std::string* all_names_; + const Descriptor* containing_type_; const OneofOptions* options_; + const FieldDescriptor* fields_; // IMPORTANT: If you add a new field, make sure to search for all instances // of Allocate() and AllocateArray() @@ -881,18 +1057,20 @@ class LIBPROTOBUF_EXPORT OneofDescriptor { // Describes an enum type defined in a .proto file. To get the EnumDescriptor // for a generated enum type, call TypeName_descriptor(). Use DescriptorPool // to construct your own descriptors. -class LIBPROTOBUF_EXPORT EnumDescriptor { +class PROTOBUF_EXPORT EnumDescriptor : private internal::SymbolBase { public: + typedef EnumDescriptorProto Proto; + // The name of this enum type in the containing scope. - const string& name() const; + const std::string& name() const; // The fully-qualified name of the enum type, scope delimited by periods. - const string& full_name() const; + const std::string& full_name() const; // Index of this enum within the file or containing message's enum array. int index() const; - // The .proto file in which this enum type was defined. Never NULL. + // The .proto file in which this enum type was defined. Never nullptr. const FileDescriptor* file() const; // The number of values for this EnumDescriptor. Guaranteed to be greater @@ -902,30 +1080,30 @@ class LIBPROTOBUF_EXPORT EnumDescriptor { // These are returned in the order they were defined in the .proto file. const EnumValueDescriptor* value(int index) const; - // Looks up a value by name. Returns NULL if no such value exists. - const EnumValueDescriptor* FindValueByName(const string& name) const; - // Looks up a value by number. Returns NULL if no such value exists. If + // Looks up a value by name. Returns nullptr if no such value exists. + const EnumValueDescriptor* FindValueByName(ConstStringParam name) const; + // Looks up a value by number. Returns nullptr if no such value exists. If // multiple values have this number, the first one defined is returned. const EnumValueDescriptor* FindValueByNumber(int number) const; // If this enum type is nested in a message type, this is that message type. - // Otherwise, NULL. + // Otherwise, nullptr. const Descriptor* containing_type() const; // Get options for this enum type. These are specified in the .proto file by // placing lines like "option foo = 1234;" in the enum definition. Allowed - // options are defined by EnumOptions in google/protobuf/descriptor.proto, - // and any available extensions of that message. + // options are defined by EnumOptions in descriptor.proto, and any available + // extensions of that message. const EnumOptions& options() const; // See Descriptor::CopyTo(). void CopyTo(EnumDescriptorProto* proto) const; // See Descriptor::DebugString(). - string DebugString() const; + std::string DebugString() const; // See Descriptor::DebugStringWithOptions(). - string DebugStringWithOptions(const DebugStringOptions& options) const; + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Returns true if this is a placeholder for an unknown enum. This will // only be the case if this descriptor comes from a DescriptorPool @@ -950,18 +1128,18 @@ class LIBPROTOBUF_EXPORT EnumDescriptor { // Returns true if the number is in one of the reserved ranges. bool IsReservedNumber(int number) const; - // Returns NULL if no reserved range contains the given number. - const EnumDescriptor::ReservedRange* - FindReservedRangeContainingNumber(int number) const; + // Returns nullptr if no reserved range contains the given number. + const EnumDescriptor::ReservedRange* FindReservedRangeContainingNumber( + int number) const; // The number of reserved field names in this message type. int reserved_name_count() const; // Gets a reserved name by index, where 0 <= index < reserved_name_count(). - const string& reserved_name(int index) const; + const std::string& reserved_name(int index) const; // Returns true if the field name is reserved. - bool IsReservedName(const string& name) const; + bool IsReservedName(ConstStringParam name) const; // Source Location --------------------------------------------------- @@ -971,49 +1149,62 @@ class LIBPROTOBUF_EXPORT EnumDescriptor { bool GetSourceLocation(SourceLocation* out_location) const; private: + friend class Symbol; typedef EnumOptions OptionsType; // Allows access to GetLocationPath for annotations. - friend class ::google::protobuf::io::Printer; + friend class io::Printer; + friend class compiler::cpp::Formatter; + + // Allow access to FindValueByNumberCreatingIfUnknown. + friend class descriptor_unittest::DescriptorTest; // Looks up a value by number. If the value does not exist, dynamically // creates a new EnumValueDescriptor for that value, assuming that it was // unknown. If a new descriptor is created, this is done in a thread-safe way, // and future calls will return the same value descriptor pointer. // - // This is private but is used by GeneratedMessageReflection (which is - // friended below) to return a valid EnumValueDescriptor from GetEnum() when - // this feature is enabled. - const EnumValueDescriptor* - FindValueByNumberCreatingIfUnknown(int number) const; - + // This is private but is used by Reflection (which is friended below) to + // return a valid EnumValueDescriptor from GetEnum() when this feature is + // enabled. + const EnumValueDescriptor* FindValueByNumberCreatingIfUnknown( + int number) const; // See Descriptor::DebugString(). - void DebugString(int depth, string *contents, + void DebugString(int depth, std::string* contents, const DebugStringOptions& options) const; // Walks up the descriptor tree to generate the source location path // to this descriptor from the file root. void GetLocationPath(std::vector* output) const; - const string* name_; - const string* full_name_; - const FileDescriptor* file_; - const Descriptor* containing_type_; - const EnumOptions* options_; - // True if this is a placeholder for an unknown type. - bool is_placeholder_; + bool is_placeholder_ : 1; // True if this is a placeholder and the type name wasn't fully-qualified. - bool is_unqualified_placeholder_; + bool is_unqualified_placeholder_ : 1; + + // This points to the last value _index_ that is part of the sequence starting + // with the first label, where + // `enum->value(i)->number() == enum->value(0)->number() + i` + // We measure relative to the first label to adapt to enum labels starting at + // 0 or 1. + // Uses 16-bit to avoid extra padding. Unlikely to have more than 2^15 + // sequentially numbered labels in an enum. + int16_t sequential_value_limit_; int value_count_; + + // all_names_ = [name, full_name] + const std::string* all_names_; + const FileDescriptor* file_; + const Descriptor* containing_type_; + const EnumOptions* options_; EnumValueDescriptor* values_; int reserved_range_count_; int reserved_name_count_; EnumDescriptor::ReservedRange* reserved_ranges_; - const string** reserved_names_; + const std::string** reserved_names_; // IMPORTANT: If you add a new field, make sure to search for all instances // of Allocate() and AllocateArray() in @@ -1024,10 +1215,11 @@ class LIBPROTOBUF_EXPORT EnumDescriptor { friend class DescriptorBuilder; friend class Descriptor; friend class FieldDescriptor; + friend class FileDescriptorTables; friend class EnumValueDescriptor; friend class FileDescriptor; friend class DescriptorPool; - friend class internal::GeneratedMessageReflection; + friend class Reflection; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(EnumDescriptor); }; @@ -1036,40 +1228,41 @@ class LIBPROTOBUF_EXPORT EnumDescriptor { // for its type, then use EnumDescriptor::FindValueByName() or // EnumDescriptor::FindValueByNumber(). Use DescriptorPool to construct // your own descriptors. -class LIBPROTOBUF_EXPORT EnumValueDescriptor { +class PROTOBUF_EXPORT EnumValueDescriptor : private internal::SymbolBaseN<0>, + private internal::SymbolBaseN<1> { public: - const string& name() const; // Name of this enum constant. - int index() const; // Index within the enums's Descriptor. - int number() const; // Numeric value of this enum constant. + typedef EnumValueDescriptorProto Proto; + + const std::string& name() const; // Name of this enum constant. + int index() const; // Index within the enums's Descriptor. + int number() const; // Numeric value of this enum constant. // The full_name of an enum value is a sibling symbol of the enum type. // e.g. the full name of FieldDescriptorProto::TYPE_INT32 is actually // "google.protobuf.FieldDescriptorProto.TYPE_INT32", NOT // "google.protobuf.FieldDescriptorProto.Type.TYPE_INT32". This is to conform // with C++ scoping rules for enums. - const string& full_name() const; + const std::string& full_name() const; - // The .proto file in which this value was defined. Never NULL. + // The .proto file in which this value was defined. Never nullptr. const FileDescriptor* file() const; - // The type of this value. Never NULL. + // The type of this value. Never nullptr. const EnumDescriptor* type() const; - // Get options for this enum value. These are specified in the .proto file - // by adding text like "[foo = 1234]" after an enum value definition. - // Allowed options are defined by EnumValueOptions in - // google/protobuf/descriptor.proto, and any available extensions of that - // message. + // Get options for this enum value. These are specified in the .proto file by + // adding text like "[foo = 1234]" after an enum value definition. Allowed + // options are defined by EnumValueOptions in descriptor.proto, and any + // available extensions of that message. const EnumValueOptions& options() const; // See Descriptor::CopyTo(). void CopyTo(EnumValueDescriptorProto* proto) const; // See Descriptor::DebugString(). - string DebugString() const; + std::string DebugString() const; // See Descriptor::DebugStringWithOptions(). - string DebugStringWithOptions(const DebugStringOptions& options) const; - + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Source Location --------------------------------------------------- @@ -1079,22 +1272,24 @@ class LIBPROTOBUF_EXPORT EnumValueDescriptor { bool GetSourceLocation(SourceLocation* out_location) const; private: + friend class Symbol; typedef EnumValueOptions OptionsType; // Allows access to GetLocationPath for annotations. - friend class ::google::protobuf::io::Printer; + friend class io::Printer; + friend class compiler::cpp::Formatter; // See Descriptor::DebugString(). - void DebugString(int depth, string *contents, + void DebugString(int depth, std::string* contents, const DebugStringOptions& options) const; // Walks up the descriptor tree to generate the source location path // to this descriptor from the file root. void GetLocationPath(std::vector* output) const; - const string* name_; - const string* full_name_; int number_; + // all_names_ = [name, full_name] + const std::string* all_names_; const EnumDescriptor* type_; const EnumValueOptions* options_; // IMPORTANT: If you add a new field, make sure to search for all instances @@ -1107,30 +1302,30 @@ class LIBPROTOBUF_EXPORT EnumValueDescriptor { friend class EnumDescriptor; friend class DescriptorPool; friend class FileDescriptorTables; + friend class Reflection; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(EnumValueDescriptor); }; -// Describes an RPC service. To get the ServiceDescriptor for a service, -// call Service::GetDescriptor(). Generated service classes also have a -// static method called descriptor() which returns the type's -// ServiceDescriptor. Use DescriptorPool to construct your own descriptors. -class LIBPROTOBUF_EXPORT ServiceDescriptor { +// Describes an RPC service. Use DescriptorPool to construct your own +// descriptors. +class PROTOBUF_EXPORT ServiceDescriptor : private internal::SymbolBase { public: + typedef ServiceDescriptorProto Proto; + // The name of the service, not including its containing scope. - const string& name() const; + const std::string& name() const; // The fully-qualified name of the service, scope delimited by periods. - const string& full_name() const; + const std::string& full_name() const; // Index of this service within the file's services array. int index() const; - // The .proto file in which this service was defined. Never NULL. + // The .proto file in which this service was defined. Never nullptr. const FileDescriptor* file() const; // Get options for this service type. These are specified in the .proto file // by placing lines like "option foo = 1234;" in the service definition. - // Allowed options are defined by ServiceOptions in - // google/protobuf/descriptor.proto, and any available extensions of that - // message. + // Allowed options are defined by ServiceOptions in descriptor.proto, and any + // available extensions of that message. const ServiceOptions& options() const; // The number of methods this service defines. @@ -1140,16 +1335,15 @@ class LIBPROTOBUF_EXPORT ServiceDescriptor { const MethodDescriptor* method(int index) const; // Look up a MethodDescriptor by name. - const MethodDescriptor* FindMethodByName(const string& name) const; + const MethodDescriptor* FindMethodByName(ConstStringParam name) const; // See Descriptor::CopyTo(). void CopyTo(ServiceDescriptorProto* proto) const; // See Descriptor::DebugString(). - string DebugString() const; + std::string DebugString() const; // See Descriptor::DebugStringWithOptions(). - string DebugStringWithOptions(const DebugStringOptions& options) const; - + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Source Location --------------------------------------------------- @@ -1159,20 +1353,23 @@ class LIBPROTOBUF_EXPORT ServiceDescriptor { bool GetSourceLocation(SourceLocation* out_location) const; private: + friend class Symbol; typedef ServiceOptions OptionsType; // Allows access to GetLocationPath for annotations. - friend class ::google::protobuf::io::Printer; + friend class io::Printer; + friend class compiler::cpp::Formatter; // See Descriptor::DebugString(). - void DebugString(string *contents, const DebugStringOptions& options) const; + void DebugString(std::string* contents, + const DebugStringOptions& options) const; // Walks up the descriptor tree to generate the source location path // to this descriptor from the file root. void GetLocationPath(std::vector* output) const; - const string* name_; - const string* full_name_; + // all_names_ = [name, full_name] + const std::string* all_names_; const FileDescriptor* file_; const ServiceOptions* options_; MethodDescriptor* methods_; @@ -1194,18 +1391,20 @@ class LIBPROTOBUF_EXPORT ServiceDescriptor { // a service, first get its ServiceDescriptor, then call // ServiceDescriptor::FindMethodByName(). Use DescriptorPool to construct your // own descriptors. -class LIBPROTOBUF_EXPORT MethodDescriptor { +class PROTOBUF_EXPORT MethodDescriptor : private internal::SymbolBase { public: + typedef MethodDescriptorProto Proto; + // Name of this method, not including containing scope. - const string& name() const; + const std::string& name() const; // The fully-qualified name of the method, scope delimited by periods. - const string& full_name() const; + const std::string& full_name() const; // Index within the service's Descriptor. int index() const; - // The .proto file in which this method was defined. Never NULL. + // The .proto file in which this method was defined. Never nullptr. const FileDescriptor* file() const; - // Gets the service to which this method belongs. Never NULL. + // Gets the service to which this method belongs. Never nullptr. const ServiceDescriptor* service() const; // Gets the type of protocol message which this method accepts as input. @@ -1221,19 +1420,17 @@ class LIBPROTOBUF_EXPORT MethodDescriptor { // Get options for this method. These are specified in the .proto file by // placing lines like "option foo = 1234;" in curly-braces after a method // declaration. Allowed options are defined by MethodOptions in - // google/protobuf/descriptor.proto, and any available extensions of that - // message. + // descriptor.proto, and any available extensions of that message. const MethodOptions& options() const; // See Descriptor::CopyTo(). void CopyTo(MethodDescriptorProto* proto) const; // See Descriptor::DebugString(). - string DebugString() const; + std::string DebugString() const; // See Descriptor::DebugStringWithOptions(). - string DebugStringWithOptions(const DebugStringOptions& options) const; - + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Source Location --------------------------------------------------- @@ -1243,27 +1440,29 @@ class LIBPROTOBUF_EXPORT MethodDescriptor { bool GetSourceLocation(SourceLocation* out_location) const; private: + friend class Symbol; typedef MethodOptions OptionsType; // Allows access to GetLocationPath for annotations. - friend class ::google::protobuf::io::Printer; + friend class io::Printer; + friend class compiler::cpp::Formatter; // See Descriptor::DebugString(). - void DebugString(int depth, string *contents, + void DebugString(int depth, std::string* contents, const DebugStringOptions& options) const; // Walks up the descriptor tree to generate the source location path // to this descriptor from the file root. void GetLocationPath(std::vector* output) const; - const string* name_; - const string* full_name_; + bool client_streaming_; + bool server_streaming_; + // all_names_ = [name, full_name] + const std::string* all_names_; const ServiceDescriptor* service_; mutable internal::LazyDescriptor input_type_; mutable internal::LazyDescriptor output_type_; const MethodOptions* options_; - bool client_streaming_; - bool server_streaming_; // IMPORTANT: If you add a new field, make sure to search for all instances // of Allocate() and AllocateArray() in // descriptor.cc and update them to initialize the field. @@ -1279,17 +1478,19 @@ class LIBPROTOBUF_EXPORT MethodDescriptor { // Describes a whole .proto file. To get the FileDescriptor for a compiled-in // file, get the descriptor for something defined in that file and call // descriptor->file(). Use DescriptorPool to construct your own descriptors. -class LIBPROTOBUF_EXPORT FileDescriptor { +class PROTOBUF_EXPORT FileDescriptor { public: + typedef FileDescriptorProto Proto; + // The filename, relative to the source tree. - // e.g. "google/protobuf/descriptor.proto" - const string& name() const; + // e.g. "foo/bar/baz.proto" + const std::string& name() const; // The package, e.g. "google.protobuf.compiler". - const string& package() const; + const std::string& package() const; // The DescriptorPool in which this FileDescriptor and all its contents were - // allocated. Never NULL. + // allocated. Never nullptr. const DescriptorPool* pool() const; // The number of files imported by this one. @@ -1344,36 +1545,39 @@ class LIBPROTOBUF_EXPORT FileDescriptor { // Get options for this file. These are specified in the .proto file by // placing lines like "option foo = 1234;" at the top level, outside of any // other definitions. Allowed options are defined by FileOptions in - // google/protobuf/descriptor.proto, and any available extensions of that - // message. + // descriptor.proto, and any available extensions of that message. const FileOptions& options() const; // Syntax of this file. enum Syntax { SYNTAX_UNKNOWN = 0, - SYNTAX_PROTO2 = 2, - SYNTAX_PROTO3 = 3, + SYNTAX_PROTO2 = 2, + SYNTAX_PROTO3 = 3, }; Syntax syntax() const; static const char* SyntaxName(Syntax syntax); - // Find a top-level message type by name. Returns NULL if not found. - const Descriptor* FindMessageTypeByName(const string& name) const; - // Find a top-level enum type by name. Returns NULL if not found. - const EnumDescriptor* FindEnumTypeByName(const string& name) const; - // Find an enum value defined in any top-level enum by name. Returns NULL if + // Find a top-level message type by name (not full_name). Returns nullptr if // not found. - const EnumValueDescriptor* FindEnumValueByName(const string& name) const; - // Find a service definition by name. Returns NULL if not found. - const ServiceDescriptor* FindServiceByName(const string& name) const; - // Find a top-level extension definition by name. Returns NULL if not found. - const FieldDescriptor* FindExtensionByName(const string& name) const; + const Descriptor* FindMessageTypeByName(ConstStringParam name) const; + // Find a top-level enum type by name. Returns nullptr if not found. + const EnumDescriptor* FindEnumTypeByName(ConstStringParam name) const; + // Find an enum value defined in any top-level enum by name. Returns nullptr + // if not found. + const EnumValueDescriptor* FindEnumValueByName(ConstStringParam name) const; + // Find a service definition by name. Returns nullptr if not found. + const ServiceDescriptor* FindServiceByName(ConstStringParam name) const; + // Find a top-level extension definition by name. Returns nullptr if not + // found. + const FieldDescriptor* FindExtensionByName(ConstStringParam name) const; // Similar to FindExtensionByName(), but searches by lowercased-name. See // Descriptor::FindFieldByLowercaseName(). - const FieldDescriptor* FindExtensionByLowercaseName(const string& name) const; + const FieldDescriptor* FindExtensionByLowercaseName( + ConstStringParam name) const; // Similar to FindExtensionByName(), but searches by camelcased-name. See // Descriptor::FindFieldByCamelcaseName(). - const FieldDescriptor* FindExtensionByCamelcaseName(const string& name) const; + const FieldDescriptor* FindExtensionByCamelcaseName( + ConstStringParam name) const; // See Descriptor::CopyTo(). // Notes: @@ -1388,10 +1592,10 @@ class LIBPROTOBUF_EXPORT FileDescriptor { void CopyJsonNameTo(FileDescriptorProto* proto) const; // See Descriptor::DebugString(). - string DebugString() const; + std::string DebugString() const; // See Descriptor::DebugStringWithOptions(). - string DebugStringWithOptions(const DebugStringOptions& options) const; + std::string DebugStringWithOptions(const DebugStringOptions& options) const; // Returns true if this is a placeholder for an unknown file. This will // only be the case if this descriptor comes from a DescriptorPool @@ -1413,31 +1617,41 @@ class LIBPROTOBUF_EXPORT FileDescriptor { private: typedef FileOptions OptionsType; - const string* name_; - const string* package_; + const std::string* name_; + const std::string* package_; const DescriptorPool* pool_; - GoogleOnceDynamic* dependencies_once_; + + // Data required to do lazy initialization. + struct PROTOBUF_EXPORT LazyInitData { +#ifndef SWIG + internal::once_flag once; +#endif + const char** dependencies_names; + }; + + LazyInitData* dependencies_once_; static void DependenciesOnceInit(const FileDescriptor* to_init); void InternalDependenciesOnceInit() const; - // These are arranged to minimze padding on 64-bit. + // These are arranged to minimize padding on 64-bit. int dependency_count_; int public_dependency_count_; int weak_dependency_count_; int message_type_count_; int enum_type_count_; int service_count_; - int extension_count_; - Syntax syntax_; - bool is_placeholder_; + bool is_placeholder_; // Indicates the FileDescriptor is completed building. Used to verify // that type accessor functions that can possibly build a dependent file // aren't called during the process of building the file. bool finished_building_; + // Actually a `Syntax` but stored as uint8_t to save space. + uint8_t syntax_; + // This one is here to fill the padding. + int extension_count_; mutable const FileDescriptor** dependencies_; - const string** dependencies_names_; int* public_dependencies_; int* weak_dependencies_; Descriptor* message_types_; @@ -1494,7 +1708,7 @@ class LIBPROTOBUF_EXPORT FileDescriptor { // // You can also search for descriptors within a DescriptorPool by name, and // extensions by number. -class LIBPROTOBUF_EXPORT DescriptorPool { +class PROTOBUF_EXPORT DescriptorPool { public: // Create a normal, empty DescriptorPool. DescriptorPool(); @@ -1524,7 +1738,7 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // in subsequent lookups in the DescriptorPool. class ErrorCollector; explicit DescriptorPool(DescriptorDatabase* fallback_database, - ErrorCollector* error_collector = NULL); + ErrorCollector* error_collector = nullptr); ~DescriptorPool(); @@ -1534,36 +1748,44 @@ class LIBPROTOBUF_EXPORT DescriptorPool { static const DescriptorPool* generated_pool(); - // Find a FileDescriptor in the pool by file name. Returns NULL if not + // Find a FileDescriptor in the pool by file name. Returns nullptr if not // found. - const FileDescriptor* FindFileByName(const string& name) const; + const FileDescriptor* FindFileByName(ConstStringParam name) const; // Find the FileDescriptor in the pool which defines the given symbol. // If any of the Find*ByName() methods below would succeed, then this is // equivalent to calling that method and calling the result's file() method. - // Otherwise this returns NULL. + // Otherwise this returns nullptr. const FileDescriptor* FindFileContainingSymbol( - const string& symbol_name) const; + ConstStringParam symbol_name) const; // Looking up descriptors ------------------------------------------ // These find descriptors by fully-qualified name. These will find both - // top-level descriptors and nested descriptors. They return NULL if not + // top-level descriptors and nested descriptors. They return nullptr if not // found. - const Descriptor* FindMessageTypeByName(const string& name) const; - const FieldDescriptor* FindFieldByName(const string& name) const; - const FieldDescriptor* FindExtensionByName(const string& name) const; - const OneofDescriptor* FindOneofByName(const string& name) const; - const EnumDescriptor* FindEnumTypeByName(const string& name) const; - const EnumValueDescriptor* FindEnumValueByName(const string& name) const; - const ServiceDescriptor* FindServiceByName(const string& name) const; - const MethodDescriptor* FindMethodByName(const string& name) const; + const Descriptor* FindMessageTypeByName(ConstStringParam name) const; + const FieldDescriptor* FindFieldByName(ConstStringParam name) const; + const FieldDescriptor* FindExtensionByName(ConstStringParam name) const; + const OneofDescriptor* FindOneofByName(ConstStringParam name) const; + const EnumDescriptor* FindEnumTypeByName(ConstStringParam name) const; + const EnumValueDescriptor* FindEnumValueByName(ConstStringParam name) const; + const ServiceDescriptor* FindServiceByName(ConstStringParam name) const; + const MethodDescriptor* FindMethodByName(ConstStringParam name) const; // Finds an extension of the given type by number. The extendee must be // a member of this DescriptorPool or one of its underlays. const FieldDescriptor* FindExtensionByNumber(const Descriptor* extendee, int number) const; + // Finds an extension of the given type by its printable name. + // See comments above PrintableNameForExtension() for the definition of + // "printable name". The extendee must be a member of this DescriptorPool + // or one of its underlays. Returns nullptr if there is no known message + // extension with the given printable name. + const FieldDescriptor* FindExtensionByPrintableName( + const Descriptor* extendee, ConstStringParam printable_name) const; + // Finds extensions of extendee. The extensions will be appended to // out in an undefined order. Only extensions defined directly in // this DescriptorPool or one of its underlays are guaranteed to be @@ -1577,7 +1799,7 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // When converting a FileDescriptorProto to a FileDescriptor, various // errors might be detected in the input. The caller may handle these // programmatically by implementing an ErrorCollector. - class LIBPROTOBUF_EXPORT ErrorCollector { + class PROTOBUF_EXPORT ErrorCollector { public: inline ErrorCollector() {} virtual ~ErrorCollector(); @@ -1586,37 +1808,40 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // This is useful e.g. for mapping the error back to an exact location // in a .proto file. enum ErrorLocation { - NAME, // the symbol name, or the package name for files - NUMBER, // field or extension range number - TYPE, // field type - EXTENDEE, // field extendee - DEFAULT_VALUE, // field default value - INPUT_TYPE, // method input type - OUTPUT_TYPE, // method output type - OPTION_NAME, // name in assignment - OPTION_VALUE, // value in option assignment - OTHER // some other problem + NAME, // the symbol name, or the package name for files + NUMBER, // field or extension range number + TYPE, // field type + EXTENDEE, // field extendee + DEFAULT_VALUE, // field default value + INPUT_TYPE, // method input type + OUTPUT_TYPE, // method output type + OPTION_NAME, // name in assignment + OPTION_VALUE, // value in option assignment + IMPORT, // import error + OTHER // some other problem }; // Reports an error in the FileDescriptorProto. Use this function if the // problem occurred should interrupt building the FileDescriptorProto. virtual void AddError( - const string& filename, // File name in which the error occurred. - const string& element_name, // Full name of the erroneous element. - const Message* descriptor, // Descriptor of the erroneous element. - ErrorLocation location, // One of the location constants, above. - const string& message // Human-readable error message. - ) = 0; + const std::string& filename, // File name in which the error occurred. + const std::string& element_name, // Full name of the erroneous element. + const Message* descriptor, // Descriptor of the erroneous element. + ErrorLocation location, // One of the location constants, above. + const std::string& message // Human-readable error message. + ) = 0; // Reports a warning in the FileDescriptorProto. Use this function if the // problem occurred should NOT interrupt building the FileDescriptorProto. virtual void AddWarning( - const string& /*filename*/, // File name in which the error occurred. - const string& /*element_name*/, // Full name of the erroneous element. - const Message* /*descriptor*/, // Descriptor of the erroneous element. - ErrorLocation /*location*/, // One of the location constants, above. - const string& /*message*/ // Human-readable error message. - ) {} + const std::string& /*filename*/, // File name in which the error + // occurred. + const std::string& /*element_name*/, // Full name of the erroneous + // element. + const Message* /*descriptor*/, // Descriptor of the erroneous element. + ErrorLocation /*location*/, // One of the location constants, above. + const std::string& /*message*/ // Human-readable error message. + ) {} private: GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ErrorCollector); @@ -1624,15 +1849,14 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // Convert the FileDescriptorProto to real descriptors and place them in // this DescriptorPool. All dependencies of the file must already be in - // the pool. Returns the resulting FileDescriptor, or NULL if there were + // the pool. Returns the resulting FileDescriptor, or nullptr if there were // problems with the input (e.g. the message was invalid, or dependencies // were missing). Details about the errors are written to GOOGLE_LOG(ERROR). const FileDescriptor* BuildFile(const FileDescriptorProto& proto); // Same as BuildFile() except errors are sent to the given ErrorCollector. const FileDescriptor* BuildFileCollectingErrors( - const FileDescriptorProto& proto, - ErrorCollector* error_collector); + const FileDescriptorProto& proto, ErrorCollector* error_collector); // By default, it is an error if a FileDescriptorProto contains references // to types or other files that are not found in the DescriptorPool (or its @@ -1687,8 +1911,8 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // Called by generated classes at init time to add their descriptors to // generated_pool. Do NOT call this in your own code! filename must be a // permanent string (e.g. a string literal). - static void InternalAddGeneratedFile( - const void* encoded_file_descriptor, int size); + static void InternalAddGeneratedFile(const void* encoded_file_descriptor, + int size); // Disallow [enforce_utf8 = false] in .proto files. void DisallowEnforceUtf8() { disallow_enforce_utf8_ = true; } @@ -1700,6 +1924,11 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // the underlay takes precedence. static DescriptorPool* internal_generated_pool(); + // For internal use only: Gets a non-const pointer to the generated + // descriptor database. + // Only used for testing. + static DescriptorDatabase* internal_generated_database(); + // For internal use only: Changes the behavior of BuildFile() such that it // allows the file to make reference to message types declared in other files // which it did not officially declare as dependencies. @@ -1709,7 +1938,7 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // Delay the building of dependencies of a file descriptor until absolutely // necessary, like when message_type() is called on a field that is defined // in that dependency's file. This will cause functional issues if a proto - // or one of it's dependencies has errors. Should only be enabled for the + // or one of its dependencies has errors. Should only be enabled for the // generated_pool_ (because no descriptor build errors are guaranteed by // the compilation generation process), testing, or if a lack of descriptor // build errors can be guaranteed for a pool. @@ -1728,12 +1957,12 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // For internal (unit test) use only: Returns true if a FileDescriptor has // been constructed for the given file, false otherwise. Useful for testing // lazy descriptor initialization behavior. - bool InternalIsFileLoaded(const string& filename) const; - + bool InternalIsFileLoaded(ConstStringParam filename) const; // Add a file to unused_import_track_files_. DescriptorBuilder will log - // warnings for those files if there is any unused import. - void AddUnusedImportTrackFile(const string& file_name); + // warnings or errors for those files if there is any unused import. + void AddUnusedImportTrackFile(ConstStringParam file_name, + bool is_error = false); void ClearUnusedImportTrackFiles(); private: @@ -1751,33 +1980,40 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // Return true if the given name is a sub-symbol of any non-package // descriptor that already exists in the descriptor pool. (The full // definition of such types is already known.) - bool IsSubSymbolOfBuiltType(const string& name) const; + bool IsSubSymbolOfBuiltType(StringPiece name) const; // Tries to find something in the fallback database and link in the // corresponding proto file. Returns true if successful, in which case // the caller should search for the thing again. These are declared // const because they are called by (semantically) const methods. - bool TryFindFileInFallbackDatabase(const string& name) const; - bool TryFindSymbolInFallbackDatabase(const string& name) const; + bool TryFindFileInFallbackDatabase(StringPiece name) const; + bool TryFindSymbolInFallbackDatabase(StringPiece name) const; bool TryFindExtensionInFallbackDatabase(const Descriptor* containing_type, int field_number) const; + // This internal find extension method only check with its table and underlay + // descriptor_pool's table. It does not check with fallback DB and no + // additional proto file will be build in this method. + const FieldDescriptor* InternalFindExtensionByNumberNoLock( + const Descriptor* extendee, int number) const; + // Like BuildFile() but called internally when the file has been loaded from // fallback_database_. Declared const because it is called by (semantically) // const methods. const FileDescriptor* BuildFileFromDatabase( - const FileDescriptorProto& proto) const; + const FileDescriptorProto& proto) const; // Helper for when lazily_build_dependencies_ is set, can look up a symbol // after the file's descriptor is built, and can build the file where that // symbol is defined if necessary. Will create a placeholder if the type // doesn't exist in the fallback database, or the file doesn't build // successfully. - Symbol CrossLinkOnDemandHelper(const string& name, bool expecting_enum) const; + Symbol CrossLinkOnDemandHelper(StringPiece name, + bool expecting_enum) const; // Create a placeholder FileDescriptor of the specified name - FileDescriptor* NewPlaceholderFile(const string& name) const; - FileDescriptor* NewPlaceholderFileWithMutexHeld(const string& name) const; + FileDescriptor* NewPlaceholderFile(StringPiece name) const; + FileDescriptor* NewPlaceholderFileWithMutexHeld(StringPiece name) const; enum PlaceholderType { PLACEHOLDER_MESSAGE, @@ -1785,14 +2021,14 @@ class LIBPROTOBUF_EXPORT DescriptorPool { PLACEHOLDER_EXTENDABLE_MESSAGE }; // Create a placeholder Descriptor of the specified name - Symbol NewPlaceholder(const string& name, + Symbol NewPlaceholder(StringPiece name, PlaceholderType placeholder_type) const; - Symbol NewPlaceholderWithMutexHeld(const string& name, + Symbol NewPlaceholderWithMutexHeld(StringPiece name, PlaceholderType placeholder_type) const; - // If fallback_database_ is NULL, this is NULL. Otherwise, this is a mutex - // which must be locked while accessing tables_. - Mutex* mutex_; + // If fallback_database_ is nullptr, this is nullptr. Otherwise, this is a + // mutex which must be locked while accessing tables_. + internal::WrappedMutex* mutex_; // See constructor. DescriptorDatabase* fallback_database_; @@ -1802,14 +2038,17 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // This class contains a lot of hash maps with complicated types that // we'd like to keep out of the header. class Tables; - google::protobuf::scoped_ptr tables_; + std::unique_ptr tables_; bool enforce_dependencies_; bool lazily_build_dependencies_; bool allow_unknown_; bool enforce_weak_; bool disallow_enforce_utf8_; - std::set unused_import_track_files_; + + // Set of files to track for unused imports. The bool value when true means + // unused imports are treated as errors (and as warnings when false). + std::map unused_import_track_files_; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(DescriptorPool); }; @@ -1823,7 +2062,12 @@ class LIBPROTOBUF_EXPORT DescriptorPool { // Strings fields are stored as pointers but returned as const references. #define PROTOBUF_DEFINE_STRING_ACCESSOR(CLASS, FIELD) \ - inline const string& CLASS::FIELD() const { return *FIELD##_; } + inline const std::string& CLASS::FIELD() const { return *FIELD##_; } + +// Name and full name are stored in a single array to save space. +#define PROTOBUF_DEFINE_NAME_ACCESSOR(CLASS) \ + inline const std::string& CLASS::name() const { return all_names_[0]; } \ + inline const std::string& CLASS::full_name() const { return all_names_[1]; } // Arrays take an index parameter, obviously. #define PROTOBUF_DEFINE_ARRAY_ACCESSOR(CLASS, FIELD, TYPE) \ @@ -1832,13 +2076,13 @@ class LIBPROTOBUF_EXPORT DescriptorPool { #define PROTOBUF_DEFINE_OPTIONS_ACCESSOR(CLASS, TYPE) \ inline const TYPE& CLASS::options() const { return *options_; } -PROTOBUF_DEFINE_STRING_ACCESSOR(Descriptor, name) -PROTOBUF_DEFINE_STRING_ACCESSOR(Descriptor, full_name) +PROTOBUF_DEFINE_NAME_ACCESSOR(Descriptor) PROTOBUF_DEFINE_ACCESSOR(Descriptor, file, const FileDescriptor*) PROTOBUF_DEFINE_ACCESSOR(Descriptor, containing_type, const Descriptor*) PROTOBUF_DEFINE_ACCESSOR(Descriptor, field_count, int) PROTOBUF_DEFINE_ACCESSOR(Descriptor, oneof_decl_count, int) +PROTOBUF_DEFINE_ACCESSOR(Descriptor, real_oneof_decl_count, int) PROTOBUF_DEFINE_ACCESSOR(Descriptor, nested_type_count, int) PROTOBUF_DEFINE_ACCESSOR(Descriptor, enum_type_count, int) @@ -1851,8 +2095,7 @@ PROTOBUF_DEFINE_ACCESSOR(Descriptor, extension_range_count, int) PROTOBUF_DEFINE_ACCESSOR(Descriptor, extension_count, int) PROTOBUF_DEFINE_ARRAY_ACCESSOR(Descriptor, extension_range, const Descriptor::ExtensionRange*) -PROTOBUF_DEFINE_ARRAY_ACCESSOR(Descriptor, extension, - const FieldDescriptor*) +PROTOBUF_DEFINE_ARRAY_ACCESSOR(Descriptor, extension, const FieldDescriptor*) PROTOBUF_DEFINE_ACCESSOR(Descriptor, reserved_range_count, int) PROTOBUF_DEFINE_ARRAY_ACCESSOR(Descriptor, reserved_range, @@ -1862,40 +2105,30 @@ PROTOBUF_DEFINE_ACCESSOR(Descriptor, reserved_name_count, int) PROTOBUF_DEFINE_OPTIONS_ACCESSOR(Descriptor, MessageOptions) PROTOBUF_DEFINE_ACCESSOR(Descriptor, is_placeholder, bool) -PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, name) -PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, full_name) -PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, json_name) -PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, lowercase_name) -PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, camelcase_name) +PROTOBUF_DEFINE_NAME_ACCESSOR(FieldDescriptor) PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, file, const FileDescriptor*) PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, number, int) PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, is_extension, bool) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, label, FieldDescriptor::Label) PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, containing_type, const Descriptor*) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, containing_oneof, - const OneofDescriptor*) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, index_in_oneof, int) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, extension_scope, const Descriptor*) PROTOBUF_DEFINE_OPTIONS_ACCESSOR(FieldDescriptor, FieldOptions) PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, has_default_value, bool) PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, has_json_name, bool) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_int32 , int32 ) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_int64 , int64 ) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_uint32, uint32) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_uint64, uint64) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_float , float ) +PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_int32_t, int32_t) +PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_int64_t, int64_t) +PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_uint32_t, uint32_t) +PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_uint64_t, uint64_t) +PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_float, float) PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_double, double) -PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_bool , bool ) +PROTOBUF_DEFINE_ACCESSOR(FieldDescriptor, default_value_bool, bool) PROTOBUF_DEFINE_STRING_ACCESSOR(FieldDescriptor, default_value_string) -PROTOBUF_DEFINE_STRING_ACCESSOR(OneofDescriptor, name) -PROTOBUF_DEFINE_STRING_ACCESSOR(OneofDescriptor, full_name) +PROTOBUF_DEFINE_NAME_ACCESSOR(OneofDescriptor) PROTOBUF_DEFINE_ACCESSOR(OneofDescriptor, containing_type, const Descriptor*) PROTOBUF_DEFINE_ACCESSOR(OneofDescriptor, field_count, int) +PROTOBUF_DEFINE_ARRAY_ACCESSOR(OneofDescriptor, field, const FieldDescriptor*) PROTOBUF_DEFINE_OPTIONS_ACCESSOR(OneofDescriptor, OneofOptions) -PROTOBUF_DEFINE_STRING_ACCESSOR(EnumDescriptor, name) -PROTOBUF_DEFINE_STRING_ACCESSOR(EnumDescriptor, full_name) +PROTOBUF_DEFINE_NAME_ACCESSOR(EnumDescriptor) PROTOBUF_DEFINE_ACCESSOR(EnumDescriptor, file, const FileDescriptor*) PROTOBUF_DEFINE_ACCESSOR(EnumDescriptor, containing_type, const Descriptor*) PROTOBUF_DEFINE_ACCESSOR(EnumDescriptor, value_count, int) @@ -1908,22 +2141,19 @@ PROTOBUF_DEFINE_ARRAY_ACCESSOR(EnumDescriptor, reserved_range, const EnumDescriptor::ReservedRange*) PROTOBUF_DEFINE_ACCESSOR(EnumDescriptor, reserved_name_count, int) -PROTOBUF_DEFINE_STRING_ACCESSOR(EnumValueDescriptor, name) -PROTOBUF_DEFINE_STRING_ACCESSOR(EnumValueDescriptor, full_name) +PROTOBUF_DEFINE_NAME_ACCESSOR(EnumValueDescriptor) PROTOBUF_DEFINE_ACCESSOR(EnumValueDescriptor, number, int) PROTOBUF_DEFINE_ACCESSOR(EnumValueDescriptor, type, const EnumDescriptor*) PROTOBUF_DEFINE_OPTIONS_ACCESSOR(EnumValueDescriptor, EnumValueOptions) -PROTOBUF_DEFINE_STRING_ACCESSOR(ServiceDescriptor, name) -PROTOBUF_DEFINE_STRING_ACCESSOR(ServiceDescriptor, full_name) +PROTOBUF_DEFINE_NAME_ACCESSOR(ServiceDescriptor) PROTOBUF_DEFINE_ACCESSOR(ServiceDescriptor, file, const FileDescriptor*) PROTOBUF_DEFINE_ACCESSOR(ServiceDescriptor, method_count, int) PROTOBUF_DEFINE_ARRAY_ACCESSOR(ServiceDescriptor, method, const MethodDescriptor*) PROTOBUF_DEFINE_OPTIONS_ACCESSOR(ServiceDescriptor, ServiceOptions) -PROTOBUF_DEFINE_STRING_ACCESSOR(MethodDescriptor, name) -PROTOBUF_DEFINE_STRING_ACCESSOR(MethodDescriptor, full_name) +PROTOBUF_DEFINE_NAME_ACCESSOR(MethodDescriptor) PROTOBUF_DEFINE_ACCESSOR(MethodDescriptor, service, const ServiceDescriptor*) PROTOBUF_DEFINE_OPTIONS_ACCESSOR(MethodDescriptor, MethodOptions) PROTOBUF_DEFINE_ACCESSOR(MethodDescriptor, client_streaming, bool) @@ -1955,17 +2185,21 @@ PROTOBUF_DEFINE_ARRAY_ACCESSOR(FileDescriptor, extension, // A few accessors differ from the macros... +inline Descriptor::WellKnownType Descriptor::well_known_type() const { + return static_cast(well_known_type_); +} + inline bool Descriptor::IsExtensionNumber(int number) const { - return FindExtensionRangeContainingNumber(number) != NULL; + return FindExtensionRangeContainingNumber(number) != nullptr; } inline bool Descriptor::IsReservedNumber(int number) const { - return FindReservedRangeContainingNumber(number) != NULL; + return FindReservedRangeContainingNumber(number) != nullptr; } -inline bool Descriptor::IsReservedName(const string& name) const { +inline bool Descriptor::IsReservedName(ConstStringParam name) const { for (int i = 0; i < reserved_name_count(); i++) { - if (name == reserved_name(i)) { + if (name == static_cast(reserved_name(i))) { return true; } } @@ -1974,17 +2208,17 @@ inline bool Descriptor::IsReservedName(const string& name) const { // Can't use PROTOBUF_DEFINE_ARRAY_ACCESSOR because reserved_names_ is actually // an array of pointers rather than the usual array of objects. -inline const string& Descriptor::reserved_name(int index) const { +inline const std::string& Descriptor::reserved_name(int index) const { return *reserved_names_[index]; } inline bool EnumDescriptor::IsReservedNumber(int number) const { - return FindReservedRangeContainingNumber(number) != NULL; + return FindReservedRangeContainingNumber(number) != nullptr; } -inline bool EnumDescriptor::IsReservedName(const string& name) const { +inline bool EnumDescriptor::IsReservedName(ConstStringParam name) const { for (int i = 0; i < reserved_name_count(); i++) { - if (name == reserved_name(i)) { + if (name == static_cast(reserved_name(i))) { return true; } } @@ -1993,15 +2227,45 @@ inline bool EnumDescriptor::IsReservedName(const string& name) const { // Can't use PROTOBUF_DEFINE_ARRAY_ACCESSOR because reserved_names_ is actually // an array of pointers rather than the usual array of objects. -inline const string& EnumDescriptor::reserved_name(int index) const { +inline const std::string& EnumDescriptor::reserved_name(int index) const { return *reserved_names_[index]; } +inline const std::string& FieldDescriptor::lowercase_name() const { + return all_names_[lowercase_name_index_]; +} + +inline const std::string& FieldDescriptor::camelcase_name() const { + return all_names_[camelcase_name_index_]; +} + +inline const std::string& FieldDescriptor::json_name() const { + return all_names_[json_name_index_]; +} + +inline const OneofDescriptor* FieldDescriptor::containing_oneof() const { + return is_oneof_ ? scope_.containing_oneof : nullptr; +} + +inline int FieldDescriptor::index_in_oneof() const { + GOOGLE_DCHECK(is_oneof_); + return static_cast(this - scope_.containing_oneof->field(0)); +} + +inline const Descriptor* FieldDescriptor::extension_scope() const { + GOOGLE_CHECK(is_extension_); + return scope_.extension_scope; +} + +inline FieldDescriptor::Label FieldDescriptor::label() const { + return static_cast