Skip to content

Commit

Permalink
* optimize video
Browse files Browse the repository at this point in the history
  • Loading branch information
@lxowalle
lxowalle committed May 11, 2024
1 parent de53cb4 commit 2c694d5
Show file tree
Hide file tree
Showing 6 changed files with 137 additions and 20 deletions.
8 changes: 8 additions & 0 deletions components/vision/include/maix_image.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,14 @@ namespace maix::image
*/
Image(int width, int height, image::Format format, uint8_t *data, int data_size, bool copy);

Image() {
_width = 0;
_height = 0;
_format = image::Format::FMT_INVALID;
_data = nullptr;
_data_size = 0;
_is_malloc = false;
}
~Image();

void operator=(image::Image &img);
Expand Down
26 changes: 22 additions & 4 deletions components/vision/include/maix_video.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
*/
namespace maix::video
{
extern maix::image::Image NoneImage;

/**
* Video type
* @maixpy maix.video.VideoType
Expand Down Expand Up @@ -174,11 +176,12 @@ namespace maix::video
* @param time_base frame time base. time_base default is 30, means 1/30 ms
* @param framerate frame rate. framerate default is 30, means 30 frames per second
* for video. 1/time_base is not the average frame rate if the frame rate is not constant.
* @param capture enable capture, if true, you can use capture() function to get an image object
* @param open If true, video will automatically call open() after creation. default is true.
* @maixpy maix.video.Video.__init__
* @maixcdk maix.video.Video.Video
*/
Video(std::string path = std::string(), int width = 2560, int height = 1440, image::Format format = image::Format::FMT_YVU420SP, int time_base = 30, int framerate = 30, bool open = true);
Video(std::string path = std::string(), int width = 2560, int height = 1440, image::Format format = image::Format::FMT_YVU420SP, int time_base = 30, int framerate = 30, bool capture = false, bool open = true);
~Video();

/**
Expand Down Expand Up @@ -208,19 +211,19 @@ namespace maix::video
/**
* Encode image.
* @param img the image will be encode.
* if the img is NULL, this function will try to get image from camera, you must use bind_camera to bind camera.
* if the img is NULL, this function will try to get image from camera, you must use bind_camera() function to bind the camera.
* @return encode result
* @maixpy maix.video.Video.encode
*/
video::Packet encode(image::Image *img = NULL);
video::Packet *encode(image::Image *img = &maix::video::NoneImage);

/**
* Decode frame
* @param frame the frame will be decode
* @return decode result
* @maixpy maix.video.Video.decode
*/
image::Image *decode(video::Frame *frame = NULL);
image::Image *decode(video::Frame *frame = nullptr);

/**
* Encode or decode finish
Expand All @@ -229,6 +232,19 @@ namespace maix::video
*/
err::Err finish();

/**
* Capture image
* @attention Each time encode is called, the last captured image will be released.
* @return error code
* @maixpy maix.video.Video.capture
*/
image::Image *capture() {
err::check_null_raise(_capture_image, "Can't capture image, please make sure the capture flag is set, and run this api after encode().");
image::Image *new_image = new image::Image(_capture_image->width(), _capture_image->height(), _capture_image->format(),
(uint8_t *)_capture_image->data(), _capture_image->data_size(), false);
return new_image;
}

/**
* Check if video is recording
* @return true if video is recording, false if not
Expand Down Expand Up @@ -304,6 +320,8 @@ namespace maix::video
uint64_t _record_ms;
uint64_t _record_start_ms;
int _fd;
bool _need_capture;
image::Image *_capture_image;

bool _need_auto_config;
int _time_base;
Expand Down
5 changes: 2 additions & 3 deletions components/vision/port/linux/maix_video_linux.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,8 @@ namespace maix::video
return err;
}

video::Packet Video::encode(image::Image *img) {
video::Packet packet(NULL, 0);
return packet;
video::Packet *Video::encode(image::Image *img) {
return nullptr;
}

image::Image *Video::decode(video::Frame *frame) {
Expand Down
62 changes: 57 additions & 5 deletions components/vision/port/maixcam/maix_video_mmf.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@
#define MMF_VENC_CHN (1)
namespace maix::video
{
Video::Video(std::string path, int width, int height, image::Format format, int time_base, int framerate, bool open)
maix::image::Image NoneImage = maix::image::Image();

Video::Video(std::string path, int width, int height, image::Format format, int time_base, int framerate, bool capture, bool open)
{
this->_pre_path = path;
this->_video_type = VIDEO_NONE;
Expand All @@ -35,6 +37,8 @@ namespace maix::video
this->_pre_width = width;
this->_pre_height = height;
this->_last_pts = 0;
this->_capture_image = nullptr;
this->_need_capture = capture;
this->_is_opened = false;

if (open) {
Expand Down Expand Up @@ -80,6 +84,10 @@ namespace maix::video
mmf_enc_h265_deinit(MMF_VENC_CHN);
}

if (_capture_image && _capture_image->data()) {
delete _capture_image;
_capture_image = nullptr;
}
this->_is_opened = false;
}

Expand Down Expand Up @@ -125,7 +133,7 @@ namespace maix::video
return video_type;
}

video::Packet Video::encode(image::Image *img) {
video::Packet *Video::encode(image::Image *img) {
uint8_t *stream_buffer = NULL;
int stream_size = 0;

Expand All @@ -139,7 +147,7 @@ namespace maix::video
_need_auto_config = false;
}

if (img) { // encode from image
if (img && img->data() != NULL) { // encode from image
if (img->data_size() > 2560 * 1440 * 3 / 2) {
log::error("image is too large!\r\n");
goto _exit;
Expand Down Expand Up @@ -286,7 +294,7 @@ namespace maix::video
}
} else { // encode from camera
if (!this->_bind_camera) {
log::warn("You need use bind_camera() to bind camera!\r\n");
log::warn("You need use bind_camera() function to bind the camera!\r\n");
goto _exit;
}

Expand Down Expand Up @@ -347,6 +355,50 @@ namespace maix::video
goto _exit;
}

if (_need_capture) {
if (_capture_image && _capture_image->data()) {
delete _capture_image;
_capture_image = NULL;
}

image::Format capture_format = (image::Format)mmf_invert_format_to_maix(format);
bool need_align = (width % mmf_vi_aligned_width(vi_ch) == 0) ? false : true; // Width need align only
switch (capture_format) {
case image::Format::FMT_BGR888: // fall through
case image::Format::FMT_RGB888:
{
_capture_image = new image::Image(width, height, capture_format);
uint8_t * image_data = (uint8_t *)_capture_image->data();
if (need_align) {
for (int h = 0; h < height; h++) {
memcpy((uint8_t *)image_data + h * width * 3, (uint8_t *)data + h * width * 3, width * 3);
}
} else {
memcpy(image_data, data, width * height * 3);
}
}
break;
case image::Format::FMT_YVU420SP:
{
_capture_image = new image::Image(width, height, capture_format);
uint8_t * image_data = (uint8_t *)_capture_image->data();
if (need_align) {
for (int h = 0; h < height * 3 / 2; h ++) {
memcpy((uint8_t *)image_data + h * width, (uint8_t *)data + h * width, width);
}
} else {
memcpy(image_data, data, width * height * 3 / 2);
}
break;
}
default:
{
_capture_image = NULL;
break;
}
}
}

if (mmf_enc_h265_push(MMF_VENC_CHN, (uint8_t *)data, width, height, format)) {
log::warn("mmf_enc_h265_push failed\n");
mmf_enc_h265_deinit(MMF_VENC_CHN);
Expand Down Expand Up @@ -474,7 +526,7 @@ namespace maix::video
}
}
_exit:
video::Packet packet(stream_buffer, stream_size);
video::Packet *packet = new video::Packet(stream_buffer, stream_size);
return packet;
}

Expand Down
2 changes: 2 additions & 0 deletions components/vision/src/maix_image.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,8 @@ namespace maix::image
{
// log::debug("free image data\n");
free(_actual_data);
_actual_data = NULL;
_data = NULL;
}
}

Expand Down
54 changes: 46 additions & 8 deletions examples/video_demo/main/src/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ static void helper(void)
"1 [record_time] [output_path]: record from camera and save to mp4\r\n"
"2 [record_time] [output_path]: encode image and save to h265\r\n"
"3 [record_time] [output_path]: record from camera and save to h265\r\n"
"4 [record_time] [output_path]: record from camera and save to h265, then display\r\n"
"\r\n"
"Example: ./video_demo 0 5 output.mp4 # means record 5s from camera, and save to output.mp4\r\n"
"==================================\r\n");
Expand Down Expand Up @@ -50,15 +51,16 @@ int _main(int argc, char* argv[])
int count = 0;
while(!app::need_exit()) {
image::Image *img = cam.read();
video::Packet packet = v.encode(img);
video::Packet *packet = v.encode(img);
if (time::time_ms() - start_ms > record_s * 1000) {
log::info("finish\r\n");
v.finish();
app::set_exit_flag(true);
}
delete img;

printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet.data(), packet.data_size(), time::time_ms() - last_loop);
printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet->data(), packet->data_size(), time::time_ms() - last_loop);
delete packet;
last_loop = time::time_ms();
}
break;
Expand All @@ -79,14 +81,15 @@ int _main(int argc, char* argv[])
uint64_t last_loop = start_ms;
int count = 0;
while(!app::need_exit()) {
video::Packet packet = v.encode();
video::Packet *packet = v.encode();
if (time::time_ms() - start_ms > record_s * 1000) {
log::info("finish\r\n");
v.finish();
app::set_exit_flag(true);
}

printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet.data(), packet.data_size(), time::time_ms() - last_loop);
printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet->data(), packet->data_size(), time::time_ms() - last_loop);
delete packet;
last_loop = time::time_ms();
}
break;
Expand All @@ -107,15 +110,16 @@ int _main(int argc, char* argv[])
int count = 0;
while(!app::need_exit()) {
image::Image *img = cam.read();
video::Packet packet = v.encode(img);
video::Packet *packet = v.encode(img);
if (time::time_ms() - start_ms > record_s * 1000) {
log::info("finish\r\n");
v.finish();
app::set_exit_flag(true);
}
delete img;

printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet.data(), packet.data_size(), time::time_ms() - last_loop);
printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet->data(), packet->data_size(), time::time_ms() - last_loop);
delete packet;
last_loop = time::time_ms();
}
break;
Expand All @@ -136,14 +140,48 @@ int _main(int argc, char* argv[])
uint64_t last_loop = start_ms;
int count = 0;
while(!app::need_exit()) {
video::Packet packet = v.encode();
video::Packet *packet = v.encode();
if (time::time_ms() - start_ms > record_s * 1000) {
log::info("finish\r\n");
v.finish();
app::set_exit_flag(true);
}

printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet.data(), packet.data_size(), time::time_ms() - last_loop);
printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet->data(), packet->data_size(), time::time_ms() - last_loop);
delete packet;
last_loop = time::time_ms();
}
break;
}
case 4:
{
uint64_t record_s = 5;
std::string path = "output.h265";
if (argc > 2) record_s = atoi(argv[2]);
if (argc > 3) path = argv[3];
log::info("Ready to record %ld s, and save to %s\r\n", record_s, path.c_str());

camera::Camera cam = camera::Camera(640, 480, image::Format::FMT_YVU420SP);
display::Display disp = display::Display();
video::Video v = video::Video(path, 640, 480, image::Format::FMT_YVU420SP, 30, 30, true);
v.bind_camera(&cam);

uint64_t start_ms = time::time_ms();
uint64_t last_loop = start_ms;
int count = 0;
while(!app::need_exit()) {
video::Packet *packet = v.encode();
if (time::time_ms() - start_ms > record_s * 1000) {
log::info("finish\r\n");
v.finish();
app::set_exit_flag(true);
}

image::Image *img = v.capture();
disp.show(*img);

printf("Packet[%d] data:%p size:%ld use %ld ms\r\n", count ++, packet->data(), packet->data_size(), time::time_ms() - last_loop);
delete packet;
last_loop = time::time_ms();
}
break;
Expand Down

0 comments on commit 2c694d5

Please sign in to comment.