From 2c694d5dd0daf7efb36d0c611d716ed91bd257ac Mon Sep 17 00:00:00 2001 From: lxowalle Date: Sat, 11 May 2024 18:32:21 +0800 Subject: [PATCH] * optimize video --- components/vision/include/maix_image.hpp | 8 +++ components/vision/include/maix_video.hpp | 26 ++++++-- .../vision/port/linux/maix_video_linux.cpp | 5 +- .../vision/port/maixcam/maix_video_mmf.cpp | 62 +++++++++++++++++-- components/vision/src/maix_image.cpp | 2 + examples/video_demo/main/src/main.cpp | 54 +++++++++++++--- 6 files changed, 137 insertions(+), 20 deletions(-) diff --git a/components/vision/include/maix_image.hpp b/components/vision/include/maix_image.hpp index 0d68cc70..ee17e215 100644 --- a/components/vision/include/maix_image.hpp +++ b/components/vision/include/maix_image.hpp @@ -57,6 +57,14 @@ namespace maix::image */ Image(int width, int height, image::Format format, uint8_t *data, int data_size, bool copy); + Image() { + _width = 0; + _height = 0; + _format = image::Format::FMT_INVALID; + _data = nullptr; + _data_size = 0; + _is_malloc = false; + } ~Image(); void operator=(image::Image &img); diff --git a/components/vision/include/maix_video.hpp b/components/vision/include/maix_video.hpp index 017305dd..a1663d81 100644 --- a/components/vision/include/maix_video.hpp +++ b/components/vision/include/maix_video.hpp @@ -18,6 +18,8 @@ */ namespace maix::video { + extern maix::image::Image NoneImage; + /** * Video type * @maixpy maix.video.VideoType @@ -174,11 +176,12 @@ namespace maix::video * @param time_base frame time base. time_base default is 30, means 1/30 ms * @param framerate frame rate. framerate default is 30, means 30 frames per second * for video. 1/time_base is not the average frame rate if the frame rate is not constant. + * @param capture enable capture, if true, you can use capture() function to get an image object * @param open If true, video will automatically call open() after creation. default is true. * @maixpy maix.video.Video.__init__ * @maixcdk maix.video.Video.Video */ - Video(std::string path = std::string(), int width = 2560, int height = 1440, image::Format format = image::Format::FMT_YVU420SP, int time_base = 30, int framerate = 30, bool open = true); + Video(std::string path = std::string(), int width = 2560, int height = 1440, image::Format format = image::Format::FMT_YVU420SP, int time_base = 30, int framerate = 30, bool capture = false, bool open = true); ~Video(); /** @@ -208,11 +211,11 @@ namespace maix::video /** * Encode image. * @param img the image will be encode. - * if the img is NULL, this function will try to get image from camera, you must use bind_camera to bind camera. + * if the img is NULL, this function will try to get image from camera, you must use bind_camera() function to bind the camera. * @return encode result * @maixpy maix.video.Video.encode */ - video::Packet encode(image::Image *img = NULL); + video::Packet *encode(image::Image *img = &maix::video::NoneImage); /** * Decode frame @@ -220,7 +223,7 @@ namespace maix::video * @return decode result * @maixpy maix.video.Video.decode */ - image::Image *decode(video::Frame *frame = NULL); + image::Image *decode(video::Frame *frame = nullptr); /** * Encode or decode finish @@ -229,6 +232,19 @@ namespace maix::video */ err::Err finish(); + /** + * Capture image + * @attention Each time encode is called, the last captured image will be released. + * @return error code + * @maixpy maix.video.Video.capture + */ + image::Image *capture() { + err::check_null_raise(_capture_image, "Can't capture image, please make sure the capture flag is set, and run this api after encode()."); + image::Image *new_image = new image::Image(_capture_image->width(), _capture_image->height(), _capture_image->format(), + (uint8_t *)_capture_image->data(), _capture_image->data_size(), false); + return new_image; + } + /** * Check if video is recording * @return true if video is recording, false if not @@ -304,6 +320,8 @@ namespace maix::video uint64_t _record_ms; uint64_t _record_start_ms; int _fd; + bool _need_capture; + image::Image *_capture_image; bool _need_auto_config; int _time_base; diff --git a/components/vision/port/linux/maix_video_linux.cpp b/components/vision/port/linux/maix_video_linux.cpp index 251b8aab..766f1db5 100644 --- a/components/vision/port/linux/maix_video_linux.cpp +++ b/components/vision/port/linux/maix_video_linux.cpp @@ -42,9 +42,8 @@ namespace maix::video return err; } - video::Packet Video::encode(image::Image *img) { - video::Packet packet(NULL, 0); - return packet; + video::Packet *Video::encode(image::Image *img) { + return nullptr; } image::Image *Video::decode(video::Frame *frame) { diff --git a/components/vision/port/maixcam/maix_video_mmf.cpp b/components/vision/port/maixcam/maix_video_mmf.cpp index b717b2e9..c6bb3d29 100644 --- a/components/vision/port/maixcam/maix_video_mmf.cpp +++ b/components/vision/port/maixcam/maix_video_mmf.cpp @@ -21,7 +21,9 @@ #define MMF_VENC_CHN (1) namespace maix::video { - Video::Video(std::string path, int width, int height, image::Format format, int time_base, int framerate, bool open) + maix::image::Image NoneImage = maix::image::Image(); + + Video::Video(std::string path, int width, int height, image::Format format, int time_base, int framerate, bool capture, bool open) { this->_pre_path = path; this->_video_type = VIDEO_NONE; @@ -35,6 +37,8 @@ namespace maix::video this->_pre_width = width; this->_pre_height = height; this->_last_pts = 0; + this->_capture_image = nullptr; + this->_need_capture = capture; this->_is_opened = false; if (open) { @@ -80,6 +84,10 @@ namespace maix::video mmf_enc_h265_deinit(MMF_VENC_CHN); } + if (_capture_image && _capture_image->data()) { + delete _capture_image; + _capture_image = nullptr; + } this->_is_opened = false; } @@ -125,7 +133,7 @@ namespace maix::video return video_type; } - video::Packet Video::encode(image::Image *img) { + video::Packet *Video::encode(image::Image *img) { uint8_t *stream_buffer = NULL; int stream_size = 0; @@ -139,7 +147,7 @@ namespace maix::video _need_auto_config = false; } - if (img) { // encode from image + if (img && img->data() != NULL) { // encode from image if (img->data_size() > 2560 * 1440 * 3 / 2) { log::error("image is too large!\r\n"); goto _exit; @@ -286,7 +294,7 @@ namespace maix::video } } else { // encode from camera if (!this->_bind_camera) { - log::warn("You need use bind_camera() to bind camera!\r\n"); + log::warn("You need use bind_camera() function to bind the camera!\r\n"); goto _exit; } @@ -347,6 +355,50 @@ namespace maix::video goto _exit; } + if (_need_capture) { + if (_capture_image && _capture_image->data()) { + delete _capture_image; + _capture_image = NULL; + } + + image::Format capture_format = (image::Format)mmf_invert_format_to_maix(format); + bool need_align = (width % mmf_vi_aligned_width(vi_ch) == 0) ? false : true; // Width need align only + switch (capture_format) { + case image::Format::FMT_BGR888: // fall through + case image::Format::FMT_RGB888: + { + _capture_image = new image::Image(width, height, capture_format); + uint8_t * image_data = (uint8_t *)_capture_image->data(); + if (need_align) { + for (int h = 0; h < height; h++) { + memcpy((uint8_t *)image_data + h * width * 3, (uint8_t *)data + h * width * 3, width * 3); + } + } else { + memcpy(image_data, data, width * height * 3); + } + } + break; + case image::Format::FMT_YVU420SP: + { + _capture_image = new image::Image(width, height, capture_format); + uint8_t * image_data = (uint8_t *)_capture_image->data(); + if (need_align) { + for (int h = 0; h < height * 3 / 2; h ++) { + memcpy((uint8_t *)image_data + h * width, (uint8_t *)data + h * width, width); + } + } else { + memcpy(image_data, data, width * height * 3 / 2); + } + break; + } + default: + { + _capture_image = NULL; + break; + } + } + } + if (mmf_enc_h265_push(MMF_VENC_CHN, (uint8_t *)data, width, height, format)) { log::warn("mmf_enc_h265_push failed\n"); mmf_enc_h265_deinit(MMF_VENC_CHN); @@ -474,7 +526,7 @@ namespace maix::video } } _exit: - video::Packet packet(stream_buffer, stream_size); + video::Packet *packet = new video::Packet(stream_buffer, stream_size); return packet; } diff --git a/components/vision/src/maix_image.cpp b/components/vision/src/maix_image.cpp index 81efdd03..48cbef6e 100644 --- a/components/vision/src/maix_image.cpp +++ b/components/vision/src/maix_image.cpp @@ -328,6 +328,8 @@ namespace maix::image { // log::debug("free image data\n"); free(_actual_data); + _actual_data = NULL; + _data = NULL; } } diff --git a/examples/video_demo/main/src/main.cpp b/examples/video_demo/main/src/main.cpp index bad20bb5..0a4395e9 100644 --- a/examples/video_demo/main/src/main.cpp +++ b/examples/video_demo/main/src/main.cpp @@ -18,6 +18,7 @@ static void helper(void) "1 [record_time] [output_path]: record from camera and save to mp4\r\n" "2 [record_time] [output_path]: encode image and save to h265\r\n" "3 [record_time] [output_path]: record from camera and save to h265\r\n" + "4 [record_time] [output_path]: record from camera and save to h265, then display\r\n" "\r\n" "Example: ./video_demo 0 5 output.mp4 # means record 5s from camera, and save to output.mp4\r\n" "==================================\r\n"); @@ -50,7 +51,7 @@ int _main(int argc, char* argv[]) int count = 0; while(!app::need_exit()) { image::Image *img = cam.read(); - video::Packet packet = v.encode(img); + video::Packet *packet = v.encode(img); if (time::time_ms() - start_ms > record_s * 1000) { log::info("finish\r\n"); v.finish(); @@ -58,7 +59,8 @@ int _main(int argc, char* argv[]) } delete img; - printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet.data(), packet.data_size(), time::time_ms() - last_loop); + printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet->data(), packet->data_size(), time::time_ms() - last_loop); + delete packet; last_loop = time::time_ms(); } break; @@ -79,14 +81,15 @@ int _main(int argc, char* argv[]) uint64_t last_loop = start_ms; int count = 0; while(!app::need_exit()) { - video::Packet packet = v.encode(); + video::Packet *packet = v.encode(); if (time::time_ms() - start_ms > record_s * 1000) { log::info("finish\r\n"); v.finish(); app::set_exit_flag(true); } - printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet.data(), packet.data_size(), time::time_ms() - last_loop); + printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet->data(), packet->data_size(), time::time_ms() - last_loop); + delete packet; last_loop = time::time_ms(); } break; @@ -107,7 +110,7 @@ int _main(int argc, char* argv[]) int count = 0; while(!app::need_exit()) { image::Image *img = cam.read(); - video::Packet packet = v.encode(img); + video::Packet *packet = v.encode(img); if (time::time_ms() - start_ms > record_s * 1000) { log::info("finish\r\n"); v.finish(); @@ -115,7 +118,8 @@ int _main(int argc, char* argv[]) } delete img; - printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet.data(), packet.data_size(), time::time_ms() - last_loop); + printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet->data(), packet->data_size(), time::time_ms() - last_loop); + delete packet; last_loop = time::time_ms(); } break; @@ -136,14 +140,48 @@ int _main(int argc, char* argv[]) uint64_t last_loop = start_ms; int count = 0; while(!app::need_exit()) { - video::Packet packet = v.encode(); + video::Packet *packet = v.encode(); if (time::time_ms() - start_ms > record_s * 1000) { log::info("finish\r\n"); v.finish(); app::set_exit_flag(true); } - printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet.data(), packet.data_size(), time::time_ms() - last_loop); + printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet->data(), packet->data_size(), time::time_ms() - last_loop); + delete packet; + last_loop = time::time_ms(); + } + break; + } + case 4: + { + uint64_t record_s = 5; + std::string path = "output.h265"; + if (argc > 2) record_s = atoi(argv[2]); + if (argc > 3) path = argv[3]; + log::info("Ready to record %ld s, and save to %s\r\n", record_s, path.c_str()); + + camera::Camera cam = camera::Camera(640, 480, image::Format::FMT_YVU420SP); + display::Display disp = display::Display(); + video::Video v = video::Video(path, 640, 480, image::Format::FMT_YVU420SP, 30, 30, true); + v.bind_camera(&cam); + + uint64_t start_ms = time::time_ms(); + uint64_t last_loop = start_ms; + int count = 0; + while(!app::need_exit()) { + video::Packet *packet = v.encode(); + if (time::time_ms() - start_ms > record_s * 1000) { + log::info("finish\r\n"); + v.finish(); + app::set_exit_flag(true); + } + + image::Image *img = v.capture(); + disp.show(*img); + + printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet->data(), packet->data_size(), time::time_ms() - last_loop); + delete packet; last_loop = time::time_ms(); } break;