Merge pull request intel-retail#386 from antoniomtz/barcode-integration

feat: Barcode detection and decode

benchmark-scripts/download_sample_videos.sh

@@ -10,3 +10,4 @@
 ./format_avc_mp4.sh vehicle-bike.mp4 https://www.pexels.com/video/853908/download/ "$1" "$2" "$3"
 #./format_avc_mp4.sh grocery-items-on-the-kitchen-shelf-4983686.mp4 https://www.pexels.com/video/4983686/download/ $1 $2 $3
 ./format_avc_mp4.sh video_of_people_walking_855564.mp4 https://www.pexels.com/video/855564/download/ "$1" "$2" "$3"
+./format_avc_mp4.sh barcode.mp4 https://github.com/antoniomtz/sample-clips/raw/main/barcode.mp4 "$1" "$2" "$3"

configs/opencv-ovms/envs/capi_yolov5_ensemble.env

@@ -9,4 +9,5 @@ RENDER_MODE=1
 cl_cache_dir=/home/intel/gst-ovms/.cl-cache
 WINDOW_WIDTH=1280
 WINDOW_HEIGHT=720
-DETECTION_THRESHOLD=0.7
+DETECTION_THRESHOLD=0.7
+BARCODE=1

configs/opencv-ovms/gst_capi/launch-pipeline.sh

@@ -16,9 +16,10 @@ RENDER_PORTRAIT_MODE=$5
 WINDOW_WIDTH=$6
 WINDOW_HEIGHT=$7
 DETECTION_THRESHOLD=$8
+BARCODE=$9
 
 # Obtaining codec_type (avc or hevc)
 is_avc=`gst-discoverer-1.0 "$INPUTSRC" | grep -i h.264 | wc -l`
 
-echo "./$PIPELINE_EXEC_PATH $INPUTSRC $USE_ONEVPL $RENDER_MODE $RENDER_PORTRAIT_MODE $is_avc $WINDOW_WIDTH $WINDOW_HEIGHT $DETECTION_THRESHOLD"
-./"$PIPELINE_EXEC_PATH" "$INPUTSRC" "$USE_ONEVPL" "$RENDER_MODE" "$RENDER_PORTRAIT_MODE" "$is_avc" "$WINDOW_WIDTH" "$WINDOW_HEIGHT" "$DETECTION_THRESHOLD"
+echo "./$PIPELINE_EXEC_PATH $INPUTSRC $USE_ONEVPL $RENDER_MODE $RENDER_PORTRAIT_MODE $is_avc $WINDOW_WIDTH $WINDOW_HEIGHT $DETECTION_THRESHOLD" "$BARCODE"
+./"$PIPELINE_EXEC_PATH" "$INPUTSRC" "$USE_ONEVPL" "$RENDER_MODE" "$RENDER_PORTRAIT_MODE" "$is_avc" "$WINDOW_WIDTH" "$WINDOW_HEIGHT" "$DETECTION_THRESHOLD" "$BARCODE"

configs/opencv-ovms/gst_capi/pipelines/capi_yolov5_ensemble/Makefile

@@ -24,5 +24,5 @@ build:
 	g++ -shared ${CUSTOM_NODE_FLAGS} -o libcustom_node_efficientnetb0-yolov5.so efficientnetb0_node.o ${CV_LIBS} ${CV_INCLUDES} -lopencv_core -lopencv_imgproc -lopencv_imgcodecs
 	cp libcustom_node_efficientnetb0-yolov5.so /ovms/lib/
 
-	g++ main.cpp -I/usr/include/gstreamer-1.0/usr/lib/x86_64-linux-gnu/ -I/usr/local/include/gstreamer-1.0 -I/usr/include/glib-2.0 -I/usr/lib/x86_64-linux-gnu/glib-2.0/include -I/usr/include/gstreamer-1.0/ -I/ovms/include $(CV_INCLUDES) $(CV_LIBS) -L/usr/lib/x86_64-linux-gnu/gstreamer-1.0 -lgstbase-1.0 -lgobject-2.0 -lglib-2.0 -lgstreamer-1.0 -lgstapp-1.0 -lopencv_imgcodecs -lopencv_imgproc -lopencv_videoio -lopencv_core -lopencv_videoio_gstreamer -lovms_shared -lopencv_highgui -lpthread -fPIC --std=c++17 -o capi_yolov5_ensemble
+	g++ barcode.cpp main.cpp -I/usr/include/gstreamer-1.0/usr/lib/x86_64-linux-gnu/ -I/usr/local/include/gstreamer-1.0 -I/usr/include/glib-2.0 -I/usr/lib/x86_64-linux-gnu/glib-2.0/include -I/usr/include/gstreamer-1.0/ -I/ovms/include $(CV_INCLUDES) $(CV_LIBS) -L/usr/lib/x86_64-linux-gnu/gstreamer-1.0 -lgstbase-1.0 -lgobject-2.0 -lglib-2.0 -lgstreamer-1.0 -lgstapp-1.0 -lopencv_imgcodecs -lopencv_imgproc -lopencv_videoio -lopencv_core -lopencv_videoio_gstreamer -lovms_shared -lopencv_highgui -lpthread -lopencv_barcode -fPIC --std=c++17 -o capi_yolov5_ensemble
 

configs/opencv-ovms/gst_capi/pipelines/capi_yolov5_ensemble/barcode.cpp

@@ -0,0 +1,70 @@
+//*****************************************************************************
+// Copyright 2023 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#include <iostream>
+#include <opencv2/highgui.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/barcode.hpp>
+#include <iomanip>
+
+class BarcodeProcessor
+{
+
+private:
+    cv::barcode::BarcodeDetector barcodeDetector;
+    cv::VideoCapture cap;    
+    std::vector<cv::String> decoded_info;
+    std::vector<cv::barcode::BarcodeType> decoded_type;
+    std::vector<cv::Point> corners;
+
+public:
+    void decode(cv::Mat frame)
+    {
+        barcodeDetector.detectAndDecode(frame, decoded_info, decoded_type, corners);
+
+        if (!corners.empty())
+        {
+            for (int i = 0; i < static_cast<int>(corners.size()); i += 4)
+            {                
+                if (decoded_info[i / 4].empty())
+                {
+                    this->drawBoundingBox(frame,corners[i], corners[i + 1], corners[i + 2], corners[i + 3], cv::Scalar(255, 0, 0));
+                } // Barcode decoded
+                else
+                {
+                    this->drawBoundingBox(frame,corners[i], corners[i + 1], corners[i + 2], corners[i + 3], cv::Scalar(0, 255, 255));
+                    this->displayText(frame,decoded_type[i / 4] + ":  " + decoded_info[i / 4], corners[i]);
+                }
+            }
+        }
+
+    }
+
+private:
+    void drawBoundingBox(cv::Mat frame,const cv::Point &p1, const cv::Point &p2, const cv::Point &p3, const cv::Point &p4, const cv::Scalar &color)
+    {
+        cv::line(frame, p1, p2, color, 2);
+        cv::line(frame, p2, p3, color, 2);
+        cv::line(frame, p3, p4, color, 2);
+        cv::line(frame, p4, p1, color, 2);
+    }
+
+    void displayText(cv::Mat frame,const std::string &text, const cv::Point &position)
+    {
+        cv::putText(frame, text, position, cv::FONT_HERSHEY_COMPLEX, 0.8, cv::Scalar(0, 0, 255));
+        std::cout << text << std::endl;
+    }
+};

configs/opencv-ovms/gst_capi/pipelines/capi_yolov5_ensemble/main.cpp

@@ -47,6 +47,8 @@
 
 #include "ovms.h"  // NOLINT
 
+#include "barcode.cpp"
+
 using namespace std;
 using namespace cv;
 
@@ -110,6 +112,7 @@ std::vector<cv::VideoCapture> _vidcaps;
 int _window_width = 1280;
 int _window_height = 720;
 float _detection_threshold = 0.5;
+bool _barcode = 1;
 
 class GStreamerMediaPipelineService : public MediaPipelineServiceInterface {
 public:
@@ -1721,6 +1724,8 @@ void run_stream(std::string mediaPath, GstElement* pipeline, GstElement* appsink
     long long numberOfSkipFrames = 0;
     OVMS_Status* res = NULL;
 
+    BarcodeProcessor barcode;
+
     while (!shutdown_request) {
         auto startTime = std::chrono::high_resolution_clock::now();
 
@@ -1920,6 +1925,11 @@ void run_stream(std::string mediaPath, GstElement* pipeline, GstElement* appsink
             }
         } // end lock on inference request to server        
 
+        // Barcode detect and decode
+        if(_barcode == 1){
+            barcode.decode(img);
+        }        
+
         metricStartTime = std::chrono::high_resolution_clock::now();
         OVMS_InferenceResponseOutputCount(response, &outputCount);
         outputId = 0;
@@ -2107,7 +2117,9 @@ int main(int argc, char** argv) {
         if (_detection_threshold > 1.0 || _detection_threshold < 0.0) {
             std::cout << "detection_threshold: " << _detection_threshold << ", is confidence threshold value in floating point that needs to be between 0.0 to 1.0.\n" << endl;
             return 1;
-        } 
+        }
+         _barcode = std::stoi(argv[9]);
+        std::cout << "_barcode: " << _barcode << std::endl;
 
         if (_renderPortrait) {
             int tmp = _window_width;

configs/opencv-ovms/gst_capi/run_gst_capi.sh

@@ -16,6 +16,7 @@ TARGET_GPU_DEVICE="${TARGET_GPU_DEVICE:=--privileged}"
 WINDOW_WIDTH="${WINDOW_WIDTH:=1920}"
 WINDOW_HEIGHT="${WINDOW_HEIGHT:=1080}"
 DETECTION_THRESHOLD="${DETECTION_THRESHOLD:=0.5}"
+BARCODE="${BARCODE:=1}"
 
 update_media_device_engine() {
 	# Use discrete GPU if it exists, otherwise use iGPU or CPU
@@ -33,7 +34,7 @@ update_media_device_engine() {
 update_media_device_engine
 
 chmod +x $PIPELINE_EXEC_PATH
-bash_cmd="./launch-pipeline.sh $PIPELINE_EXEC_PATH $INPUTSRC $USE_ONEVPL $RENDER_MODE $RENDER_PORTRAIT_MODE $WINDOW_WIDTH $WINDOW_HEIGHT $DETECTION_THRESHOLD"
+bash_cmd="./launch-pipeline.sh $PIPELINE_EXEC_PATH $INPUTSRC $USE_ONEVPL $RENDER_MODE $RENDER_PORTRAIT_MODE $WINDOW_WIDTH $WINDOW_HEIGHT $DETECTION_THRESHOLD $BARCODE"
 
 echo "BashCmd: $bash_cmd with media on $GST_VAAPI_DRM_DEVICE with USE_ONEVPL=$USE_ONEVPL"
 
@@ -52,5 +53,6 @@ PIPELINE_EXEC_PATH="$PIPELINE_EXEC_PATH" \
 RENDER_PORTRAIT_MODE="$RENDER_PORTRAIT_MODE" \
 USE_ONEVPL="$USE_ONEVPL" \
 DETECTION_THRESHOLD="$DETECTION_THRESHOLD" \
+BARCODE="$BARCODE" \
 $bash_cmd \
 2>&1 | tee >/tmp/results/gst-capi_$cid_count.log >(stdbuf -oL sed -n -e 's/^.*FPS: //p' | stdbuf -oL cut -d , -f 1 > /tmp/results/pipeline$cid_count.log)