diff --git a/components/vision_extra/CMakeLists.txt b/components/vision_extra/CMakeLists.txt new file mode 100644 index 00000000..a38011da --- /dev/null +++ b/components/vision_extra/CMakeLists.txt @@ -0,0 +1,78 @@ +# Config enable component2 or not in Kconfig +################# Add include ################# +list(APPEND ADD_INCLUDE "include" +) +if(PLATFORM_LINUX) + list(APPEND ADD_PRIVATE_INCLUDE "port/linux") +elseif(PLATFORM_MAIXCAM) + list(APPEND ADD_PRIVATE_INCLUDE "port/maixcam") +endif() +# list(APPEND ADD_PRIVATE_INCLUDE "include_private") +############################################### + +############## Add source files ############### +# list(APPEND ADD_SRCS "src/lib2.c" +# ) +# FILE(GLOB_RECURSE EXTRA_SRC "src/*.c") +# FILE(GLOB EXTRA_SRC "src/*.c") +# list(APPEND ADD_SRCS ${EXTRA_SRC}) +# aux_source_directory(src ADD_SRCS) # collect all source file in src dir, will set var ADD_SRCS +append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS +if(PLATFORM_LINUX) + append_srcs_dir(ADD_SRCS "port/linux") +elseif(PLATFORM_MAIXCAM) + append_srcs_dir(ADD_SRCS "port/maixcam") +endif() +# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c") +# set(ADD_ASM_SRCS "src/asm.S") +# list(APPEND ADD_SRCS ${ADD_ASM_SRCS}) +# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S ASM file as C language +# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB") +############################################### + + +###### Add required/dependent components ###### +if(PLATFORM_LINUX) + list(APPEND ADD_REQUIREMENTS nn) +elseif(PLATFORM_MAIXCAM) + list(APPEND ADD_REQUIREMENTS nn) +endif() +############################################### + +###### Add link search path for requirements/libs ###### +# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib") +# list(APPEND ADD_REQUIREMENTS pthread m) # add system libs, pthread and math lib for example here +# set (OpenCV_DIR opencv/lib/cmake/opencv4) +# find_package(OpenCV REQUIRED) +############################################### + +############ Add static libs ################## +# list(APPEND ADD_STATIC_LIB "lib/libtest.a") +############################################### + +############ Add dynamic libs ################## +# list(APPEND ADD_DYNAMIC_LIB "lib/arch/v831/libmaix_nn.so" +# "lib/arch/v831/libmaix_cam.so" +# ) +############################################### + +#### Add compile option for this component #### +#### Just for this component, won't affect other +#### modules, including component that depend +#### on this component +# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1) + +#### Add compile option for this component +#### and components denpend on this component +# list(APPEND ADD_DEFINITIONS -DAAAAA222=1 +# -DAAAAA333=1) +############################################### + +############ Add static libs ################## +#### Update parent's variables like CMAKE_C_LINK_FLAGS +# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE) +############################################### + +# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib +register_component() + diff --git a/components/vision_extra/Kconfig b/components/vision_extra/Kconfig new file mode 100644 index 00000000..e69de29b diff --git a/components/vision_extra/include/maix_image_extra.hpp b/components/vision_extra/include/maix_image_extra.hpp new file mode 100644 index 00000000..b0ee0c36 --- /dev/null +++ b/components/vision_extra/include/maix_image_extra.hpp @@ -0,0 +1,52 @@ +/** + * @author neucrack@sipeed, lxowalle@sipeed + * @copyright Sipeed Ltd 2023- + * @license Apache 2.0 + * @update 2023.9.8: Add framework, create this file. + */ + +#pragma once + +#include "maix_tensor.hpp" +#include "maix_log.hpp" +#include "maix_err.hpp" +#include "maix_fs.hpp" +#include "maix_image_def.hpp" +#include "maix_image_color.hpp" +#include "maix_image_obj.hpp" +#include "maix_type.hpp" +#include + +namespace maix::image +{ + /** + * QRCodeDetector class + * @maixpy maix.image.QRCodeDetector + */ + class QRCodeDetector { + private: + void *_param; + public: + /** + * QRCodeDetector constructor. + * Use npu to accelerate the speed of QR code scanning, note that this object will occupy npu resources + * @maixpy maix.image.QRCodeDetector.__init__ + */ + QRCodeDetector(); + + ~QRCodeDetector(); + + /** + * @brief Finds all qrcodes in the image. + * @param img The image to find qrcodes. + * @param roi The region of interest, input in the format of (x, y, w, h), x and y are the coordinates of the upper left corner, w and h are the width and height of roi. + * default is None, means whole image. + * @param decoder_type Select the QR code decoding method. Choosing QRCODE_DECODER_TYPE_QUIRC allows for retrieving QR code version, ECC level, mask, data type, and other details, + * though it may decode slower at lower resolutions. Opting for QRCODE_DECODER_TYPE_ZBAR enables faster decoding at lower resolutions but may slow down at higher resolutions, + * providing only the QR code content and position information. default is QRCODE_DECODER_TYPE_ZBAR. + * @return Returns the qrcodes of the image + * @maixpy maix.image.QRCodeDetector.detect + */ + std::vector detect(image::Image *img, std::vector roi = std::vector(), image::QRCodeDecoderType decoder_type = image::QRCodeDecoderType::QRCODE_DECODER_TYPE_ZBAR); + }; +} // namespace maix::image diff --git a/components/vision_extra/port/linux/maix_image_qrcode_detector.cpp b/components/vision_extra/port/linux/maix_image_qrcode_detector.cpp new file mode 100644 index 00000000..81136c22 --- /dev/null +++ b/components/vision_extra/port/linux/maix_image_qrcode_detector.cpp @@ -0,0 +1,31 @@ +/** + * @author lxowalle@sipeed + * @copyright Sipeed Ltd 2023- + * @license Apache 2.0 + * @update 2023.11.27: Add framework, create this file. + */ + +#include "maix_image.hpp" +#include "maix_image_obj.hpp" +#include "maix_image_extra.hpp" + +using namespace maix; + +namespace maix::image +{ + QRCodeDetector::QRCodeDetector() { + + } + + QRCodeDetector::~QRCodeDetector() { + + } + + std::vector QRCodeDetector::detect(image::Image *img, std::vector roi, image::QRCodeDecoderType decoder_type) + { + (void)img; + (void)roi; + (void)decoder_type; + return std::vector(); + } +} \ No newline at end of file diff --git a/components/vision_extra/port/maixcam/maix_image_qrcode_detector.cpp b/components/vision_extra/port/maixcam/maix_image_qrcode_detector.cpp new file mode 100644 index 00000000..753b0bbe --- /dev/null +++ b/components/vision_extra/port/maixcam/maix_image_qrcode_detector.cpp @@ -0,0 +1,75 @@ +/** + * @author lxowalle@sipeed + * @copyright Sipeed Ltd 2023- + * @license Apache 2.0 + * @update 2023.11.27: Add framework, create this file. + */ + +#include "maix_image.hpp" +#include "maix_image_obj.hpp" +#include "maix_nn_yolov8.hpp" +#include "maix_image_extra.hpp" + +using namespace maix; + +namespace maix::image +{ + typedef struct { + std::string path = ""; + nn::YOLOv8 *detector = nullptr; + bool dual_buffer = true; + } param_t; + + QRCodeDetector::QRCodeDetector() { + _param = new param_t(); + param_t *param = (param_t *)_param; + param->path = "/root/models/yolov8_qr_det_seg_320.mud"; + param->dual_buffer = true; + param->detector = new nn::YOLOv8("", param->dual_buffer); + err::check_raise(param->detector->load(param->path), "load model failed"); + } + + QRCodeDetector::~QRCodeDetector() { + if (_param) { + param_t *param = (param_t *)_param; + if (param->detector) { + delete param->detector; + param->detector = nullptr; + } + + delete param; + _param = nullptr; + } + } + + std::vector QRCodeDetector::detect(image::Image *img, std::vector roi, image::QRCodeDecoderType decoder_type) + { + param_t *param = (param_t *)_param; + auto detector = param->detector; + auto out = std::vector(); + + auto result = detector->detect(*img); + if (result) { + for (auto &r : *result) { + float scale = 0.1; + std::vector new_roi = {r->x, r->y, r->w, r->h}; + new_roi[0] = new_roi[0] - r->w * scale; + new_roi[0] = new_roi[0] > 0 ? new_roi[0] : 0; + new_roi[1] = new_roi[1] - r->h * scale; + new_roi[1] = new_roi[1] > 0 ? new_roi[1] : 0; + new_roi[2] = r->w * (1 + scale * 2); + new_roi[2] = new_roi[2] >= img->width() ? img->width() - r->x : new_roi[2]; + new_roi[3] = r->h * (1 + scale * 2); + new_roi[3] = new_roi[3] >= img->height() ? img->height() - r->y : new_roi[3]; + + auto qrcode_result = img->find_qrcodes(new_roi, decoder_type); + for (auto &qrcode : qrcode_result) { + out.push_back(qrcode); + } + } + delete result; + } + + return out; + } +} \ No newline at end of file diff --git a/examples/image_method/main/CMakeLists.txt b/examples/image_method/main/CMakeLists.txt index 97550a8f..ae62f9b8 100644 --- a/examples/image_method/main/CMakeLists.txt +++ b/examples/image_method/main/CMakeLists.txt @@ -23,7 +23,7 @@ append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD ############################################### ###### Add required/dependent components ###### -list(APPEND ADD_REQUIREMENTS vision quirc) +list(APPEND ADD_REQUIREMENTS vision quirc vision_extra) ############################################### ###### Add link search path for requirements/libs ###### diff --git a/examples/image_method/main/src/main.cpp b/examples/image_method/main/src/main.cpp index ee9a8b1d..74de1b04 100644 --- a/examples/image_method/main/src/main.cpp +++ b/examples/image_method/main/src/main.cpp @@ -33,6 +33,7 @@ typedef struct { } priv_t; static priv_t priv; +param_t g_param; static int cmd_init(int argc, char* argv[]); int _main(int argc, char* argv[]) @@ -43,6 +44,7 @@ int _main(int argc, char* argv[]) if (!priv.use_picture) { cam = new camera::Camera(priv.cam_w ,priv.cam_h, priv.cam_fmt, nullptr, priv.cam_fps, priv.cam_buffnum); log::info("camera size: %dx%d\n", cam->width(), cam->height()); + g_param.cam = cam; } else { picture = image::load(priv.picture_path.c_str(), priv.cam_fmt); image::Image *new_picture = picture->resize(priv.cam_w, priv.cam_h); @@ -129,7 +131,8 @@ static int cmd_init(int argc, char* argv[]) priv.method_list.push_back(image_method_t{"no method(default)", NULL}); priv.method_list.push_back(image_method_t{"gaussian", test_gaussion}); priv.method_list.push_back(image_method_t{"find_blobs", test_find_blobs}); - priv.method_list.push_back(image_method_t{"zbar", test_find_qrcode}); + priv.method_list.push_back(image_method_t{"find_qrcode", test_find_qrcode}); + priv.method_list.push_back(image_method_t{"qrcode_detector", test_qrcode_detector}); // Get init param if (argc > 1) { diff --git a/examples/image_method/main/src/test_image.hpp b/examples/image_method/main/src/test_image.hpp index 666cab18..538626a5 100644 --- a/examples/image_method/main/src/test_image.hpp +++ b/examples/image_method/main/src/test_image.hpp @@ -5,8 +5,13 @@ using namespace maix; +typedef struct { + camera::Camera *cam; +} param_t; + int test_find_blobs(image::Image *img); int test_gaussion(image::Image *img); int test_find_qrcode(image::Image *img); +int test_qrcode_detector(image::Image *img); #endif \ No newline at end of file diff --git a/examples/image_method/main/src/test_qrcode_detector.cpp b/examples/image_method/main/src/test_qrcode_detector.cpp new file mode 100644 index 00000000..25c2be98 --- /dev/null +++ b/examples/image_method/main/src/test_qrcode_detector.cpp @@ -0,0 +1,99 @@ +#include "test_image.hpp" +#include "maix_image.hpp" +#include "maix_image_obj.hpp" +#include "maix_image_extra.hpp" + +extern param_t g_param; + +int test_qrcode_detector(image::Image *img) { + static image::QRCodeDetector *qrcode_detector = nullptr; + if (qrcode_detector == nullptr) { + qrcode_detector = new image::QRCodeDetector(); + err::check_null_raise(qrcode_detector, "create qrcode detecetor failed!"); + } + + uint64_t t = time::ticks_ms(), t2 = 0; + auto result = qrcode_detector->detect(img); + t2 = time::ticks_ms(), log::info("test_qrcode_detector use %lld ms, fps:%f", t2 - t, 1000.0 / (t2 - t)); + for (auto &i : result) + { + log::info("result: %s", i.payload().c_str()); + std::vector> corners = i.corners(); + for (int i = 0; i < 4; i ++) { + img->draw_line(corners[i][0], corners[i][1], corners[(i + 1) % 4][0], corners[(i + 1) % 4][1], maix::image::Color::from_rgb(0, 255, 0), 2); + } + } + + // if (result.size() > 0) + // { + // auto r = result[0]; + // auto x = r.x(); + // auto y = r.y(); + // auto w = r.w(); + // auto h = r.h(); + // auto sensor_size = g_param.cam->get_sensor_size(); + // float scale_x = (float)sensor_size[0] / img->width(); + // float scale_y = (float)sensor_size[1] / img->height(); + + // float scale_valid = scale_x > scale_y ? scale_y : scale_x; + // int sensor_valid_w = img->width() * scale_valid; + // int sensor_valid_h = img->height() * scale_valid; + // log::info("scale valid:%f sensor valid solution:%dx%d", scale_valid, (int)sensor_valid_w, (int)sensor_valid_h); + + // int sensor_oft_x = (sensor_size[0] - sensor_valid_w) / 2; + // int sensor_oft_y = (sensor_size[1] - sensor_valid_h) / 2; + // log::info("sensor oft_x:%d, oft_y:%d", sensor_oft_x, sensor_oft_y); + + // auto dst_x = x - w; + // auto dst_y = y - h; + // auto dst_w = w * 2; + // auto dst_h = h * 2; + + // auto dst_sensor_x = sensor_oft_x + dst_x * scale_x; + // auto dst_sensor_y = sensor_oft_y + dst_y * scale_y; + // auto dst_sensor_w = dst_w * scale_x; + // auto dst_sensor_h = dst_h * scale_y; + // log::info("sensor dst x:%d, y:%d, w:%d, h:%d", (int)dst_sensor_x, (int)dst_sensor_y, (int)dst_sensor_w, (int)dst_sensor_h); + // } + // static bool config_to_default = true; + // static int qrcode_not_found_count = 0; + // if (result.size() > 0) { + // auto r = result[0]; + // auto x = r.x(); + // auto y = r.y(); + // auto w = r.w(); + // auto h = r.h(); + // auto sensor_size = g_param.cam->get_sensor_size(); + // float scale_x = (float)sensor_size[0] / img->width(); + // float scale_y = (float)sensor_size[1] / img->height(); + // int sensor_x = (int)(x * scale_x + 63) & (~63); + // int sensor_y = (int)(y * scale_y + 63) & (~63); + // int sensor_w = (int)(w * scale_x + 63) & (~63); + // int sensor_h = (int)(h * scale_y + 63) & (~63); + + // float scale_valid = scale_x > scale_y ? scale_y : scale_x; + // float sensor_valid_w = img->width() * scale_valid; + // float sensor_valid_h = img->height() * scale_valid; + + // // sensor_x = sensor_x < 0 ? 0 : sensor_x; + // // sensor_y = sensor_y < 0 ? 0 : sensor_y; + // // sensor_w = sensor_w + sensor_x > input_size[0] ? input_size[0] - sensor_x : sensor_w; + // // sensor_h = sensor_h + sensor_y > input_size[1] ? input_size[1] - sensor_y : sensor_h; + // // log::info("sensor_size:%dx%d sensor windowing: %d, %d, %d, %d", sensor_size[0], sensor_size[1], sensor_x, sensor_y, sensor_w, sensor_h); + // // g_param.cam->set_windowing({sensor_x, sensor_y, sensor_w, sensor_h}); + + // // config_to_default = false; + // // qrcode_not_found_count = 0; + // } else { + // // if (!config_to_default) { + // // qrcode_not_found_count ++; + // // if (qrcode_not_found_count > 5) { + // // auto sensor_size = g_param.cam->get_sensor_size(); + // // g_param.cam->set_windowing({0, 0, sensor_size[0], sensor_size[1]}); + // // config_to_default = true; + // // } + // // } + // } + + return 0; +} \ No newline at end of file