From cc03f852d6ea262be10ac79132b4c76ee606fb1c Mon Sep 17 00:00:00 2001 From: lxowalle Date: Wed, 27 Nov 2024 16:01:53 +0800 Subject: [PATCH] * rtsp supports audio * fix camera bug --- components/3rd_party/FFmpeg/maix_ffmpeg.hpp | 251 ++++++++- .../3rd_party/RtspServer/CMakeLists.txt | 77 +++ components/3rd_party/RtspServer/Kconfig | 15 + components/3rd_party/RtspServer/component.py | 36 ++ .../3rd_party/RtspServer/maix_rtsp_server.hpp | 159 ++++++ components/vision/CMakeLists.txt | 2 +- components/vision/include/maix_rtsp.hpp | 36 +- .../vision/port/linux/maix_rtsp_linux.cpp | 14 +- .../vision/port/maixcam/maix_display_mmf.hpp | 2 +- .../vision/port/maixcam/maix_rtmp_maixcam.cpp | 8 - .../vision/port/maixcam/maix_rtsp_maixcam.cpp | 510 ++++++++++++++---- .../vision/port/maixcam/maix_video_mmf.cpp | 1 - examples/rtsp_demo/main/src/main.cpp | 15 +- projects/app_camera/main/app/app.cpp | 7 + 14 files changed, 989 insertions(+), 144 deletions(-) create mode 100644 components/3rd_party/RtspServer/CMakeLists.txt create mode 100644 components/3rd_party/RtspServer/Kconfig create mode 100644 components/3rd_party/RtspServer/component.py create mode 100644 components/3rd_party/RtspServer/maix_rtsp_server.hpp diff --git a/components/3rd_party/FFmpeg/maix_ffmpeg.hpp b/components/3rd_party/FFmpeg/maix_ffmpeg.hpp index 7efef138..ea86b4ef 100644 --- a/components/3rd_party/FFmpeg/maix_ffmpeg.hpp +++ b/components/3rd_party/FFmpeg/maix_ffmpeg.hpp @@ -310,6 +310,7 @@ class FFmpegPacker { audio_codec_ctx->sample_fmt = AV_SAMPLE_FMT_FLTP; audio_codec_ctx->time_base = (AVRational){1, sample_rate}; audio_codec_ctx->bit_rate = bitrate; + audio_codec_ctx->profile = FF_PROFILE_AAC_LOW; audio_stream->time_base = audio_codec_ctx->time_base; if (0 > avcodec_open2(audio_codec_ctx, audio_codec, NULL)) { @@ -360,14 +361,16 @@ class FFmpegPacker { _audio_format = format; } - if (avio_open(&format_context->pb, _path.c_str(), AVIO_FLAG_WRITE) < 0) { - printf("rtmp connect failed!\r\n"); - goto _free_audio_frame; - } + if (_path.length() > 0) { + if (avio_open(&format_context->pb, _path.c_str(), AVIO_FLAG_WRITE) < 0) { + printf("avio open failed!\r\n"); + goto _free_audio_frame; + } - if (avformat_write_header(format_context, NULL) < 0) { - printf("rtmp write header failed!\r\n"); - goto _close_io; + if (avformat_write_header(format_context, NULL) < 0) { + printf("avformat write header failed!\r\n"); + goto _close_io; + } } _format_context = format_context; @@ -403,6 +406,17 @@ class FFmpegPacker { return -1; } + void reset() { + if (_pcm_list) { + for (auto it = _pcm_list->begin(); it != _pcm_list->end(); ++it) { + auto &item = *it; + Bytes *pcm = item.second; + delete pcm; + it = _pcm_list->erase(it); + } + } + } + void close() { if (!_open) return; @@ -433,7 +447,9 @@ class FFmpegPacker { } if (_format_context) { - av_write_trailer(_format_context); + if (_path.length() > 0) { + av_write_trailer(_format_context); + } // av_write_trailer(_format_context); if (_format_context && _format_context->pb) { @@ -447,6 +463,49 @@ class FFmpegPacker { _video_last_pts = 0; } + std::vector pack_pcm(uint8_t *frame, size_t frame_size) { + std::vector output_data; + if (!_open) { + return output_data; + } + + if (_has_audio) { + AVPacket *pkt = av_packet_alloc(); + if (!pkt) { + fprintf(stderr, "Can't malloc avpacket\r\n"); + return output_data; + } + + if (frame && frame_size > 0) { + AVFrame *audio_frame = _audio_frame; + AVCodecContext *audio_codec_ctx = _audio_codec_ctx; + SwrContext *swr_ctx = _audio_swr_ctx; + AVPacket *audio_packet = pkt; + + const uint8_t *in[] = {frame}; + uint8_t *out[] = {audio_frame->data[0]}; + swr_convert(swr_ctx, out, audio_codec_ctx->frame_size, in, audio_codec_ctx->frame_size); + if (avcodec_send_frame(audio_codec_ctx, audio_frame) < 0) { + printf("Error send audio_frame to encoder.\n"); + return output_data; + } + + if (avcodec_receive_packet(audio_codec_ctx, audio_packet) != 0) { + printf("Error receive audio_frame to encoder.\n"); + return output_data; + } + + output_data.resize(audio_packet->size); + memcpy(output_data.data(), audio_packet->data, audio_packet->size); + av_packet_unref(audio_packet); + } + + av_packet_unref(pkt); + av_packet_free(&pkt); + } + + return output_data; + } int push(uint8_t *frame, size_t frame_size, uint64_t pts, bool is_audio = false) { @@ -595,6 +654,176 @@ class FFmpegPacker { return 0; } + void add_adts_header(uint8_t *adts_header, int aac_frame_length, int profile, int sample_rate_index, int channel_config) { + // ADTS 固定头部的前 5 字节 + adts_header[0] = 0xFF; // Syncword 0xFFF, 高8位 + adts_header[1] = 0xF1; // Syncword低4位+ID=0+Layer=0+protection_absent=1 + adts_header[2] = ((profile - 1) << 6) // Profile（2位）：AAC-LC 为 1（profile - 1 后为 0） + | (sample_rate_index << 2) // Sampling frequency index（4位） + | (channel_config >> 2); // Channel configuration（高2位） + adts_header[3] = ((channel_config & 0x3) << 6) // Channel configuration（低2位） + | ((aac_frame_length + 7) >> 11); // Frame length（高3位） + adts_header[4] = ((aac_frame_length + 7) >> 3) & 0xFF; // Frame length（中间 8 位） + adts_header[5] = (((aac_frame_length + 7) & 0x7) << 5) // Frame length（低 3 位） + | 0x1F; // Buffer fullness（高5位，固定值0x7FF） + adts_header[6] = 0xFC; // Buffer fullness（低2位，固定值0x7FF）+ number_of_raw_data_blocks（2位，固定0） + } + + static int get_sample_rate_index(int sample_rate) + { + switch (sample_rate) { + case 96000: return 0; + case 88200: return 1; + case 64000: return 2; + case 48000: return 3; + case 44100: return 4; + case 32000: return 5; + case 24000: return 6; + case 22050: return 7; + case 16000: return 8; + case 12000: return 9; + case 11025: return 10; + case 8000: return 11; + case 7350: return 12; + default: return 3; + } + } + + int push2(uint8_t *frame, size_t frame_size, uint64_t pts, bool is_audio, void (*callback)(void*, size_t, size_t, void *args), void *args) + { + if (!_open) { + return -1; + } + + if (!is_audio) { + // log::info("[VIDEO] frame:%p frame_size:%d pts:%ld(%f s)", frame, frame_size, pts, this->video_pts_to_us(pts) / 1000); + if (callback) { + callback(frame, frame_size, pts, args); + } + } else { + if (_has_audio) { + AVPacket *pkt = av_packet_alloc(); + if (!pkt) { + fprintf(stderr, "Can't malloc avpacket\r\n"); + return -1; + } + + if (frame && frame_size > 0) { + auto pcm_list = _pcm_list; + AVFrame *audio_frame = _audio_frame; + AVStream *audio_stream = _audio_stream; + AVCodecContext *audio_codec_ctx = _audio_codec_ctx; + SwrContext *swr_ctx = _audio_swr_ctx; + AVPacket *audio_packet = pkt; + size_t buffer_size = av_samples_get_buffer_size(NULL, _audio_channels, audio_frame->nb_samples, _audio_format, 1); + size_t pcm_remain_len = frame_size; + + // fill last pcm to buffer_size + size_t next_pts = pts; + if (!pcm_list->empty()) { + auto last_item = pcm_list->back(); + Bytes *last_pcm = last_item.second; + if (last_pcm && last_pcm->data_len < buffer_size) { + int temp_size = pcm_remain_len + last_pcm->data_len >= buffer_size ? buffer_size : pcm_remain_len + last_pcm->data_len; + uint8_t *temp = (uint8_t *)malloc(temp_size); + if (!temp) { + fprintf(stderr, "malloc failed!\r\n"); + return -1; + } + memcpy(temp, last_pcm->data, last_pcm->data_len); + if (pcm_remain_len + last_pcm->data_len < buffer_size) { + memcpy(temp + last_pcm->data_len, frame, pcm_remain_len); + pcm_remain_len = 0; + } else { + memcpy(temp + last_pcm->data_len, frame, buffer_size - last_pcm->data_len); + pcm_remain_len -= (buffer_size - last_pcm->data_len); + } + + Bytes *new_pcm = new Bytes(temp, temp_size, true, false); + pcm_list->pop_back(); + delete last_pcm; + + size_t new_pts = last_item.first; + next_pts = new_pts + get_audio_pts_from_pcm_size(new_pcm->data_len); + pcm_list->push_back(std::make_pair(new_pts, new_pcm)); + } + } + + // fill other pcm + while (pcm_remain_len > 0) { + int temp_size = pcm_remain_len >= buffer_size ? buffer_size : pcm_remain_len; + uint8_t *temp = (uint8_t *)malloc(temp_size); + if (!temp) { + fprintf(stderr, "malloc failed!\r\n"); + return -1; + } + memcpy(temp, frame + frame_size - pcm_remain_len, temp_size); + pcm_remain_len -= temp_size; + + Bytes *new_pcm = new Bytes(temp, temp_size, true, false); + pcm_list->push_back(std::make_pair(next_pts, new_pcm)); + next_pts += get_audio_pts_from_pcm_size(temp_size); + } + + // for (auto it = _pcm_list->begin(); it != _pcm_list->end(); ++it) { + // auto &item = *it; + // log::info("PTS:%d PCM:%p PCM_SIZE:%d", item.first, item.second->data, item.second->data_len); + // } + + // audio process + while (pcm_list->size() > 0) { + auto item = pcm_list->front(); + auto next_pts = item.first; + Bytes *pcm = item.second; + if (pcm) { + if (pcm->data_len == buffer_size) { + const uint8_t *in[] = {pcm->data}; + uint8_t *out[] = {audio_frame->data[0]}; + swr_convert(swr_ctx, out, audio_codec_ctx->frame_size, in, audio_codec_ctx->frame_size); + audio_frame->pts = next_pts; + if (avcodec_send_frame(audio_codec_ctx, audio_frame) < 0) { + printf("Error sending audio_frame to encoder.\n"); + break; + } + + while (avcodec_receive_packet(audio_codec_ctx, audio_packet) == 0) { + audio_packet->stream_index = audio_stream->index; + audio_packet->pts = audio_packet->dts = next_pts; + audio_packet->duration = get_audio_pts_from_pcm_size(pcm->data_len); + + // log::info("[AUIDIO] frame:%p frame_size:%d pts:%ld(%f s)", pcm->data, pcm->data_len, pkt->pts, this->audio_pts_to_us(pkt->pts) / 1000); + if (callback) { + int new_size = audio_packet->size + 7; + uint8_t *new_data = (uint8_t *)malloc(new_size); + if (new_data) { + int profile = 2; + int sample_rate_index = get_sample_rate_index(_audio_sample_rate); + add_adts_header(new_data, audio_frame->nb_samples, profile, sample_rate_index, _audio_channels); + memcpy(new_data + 7, audio_packet->data, audio_packet->size); + callback(new_data, new_size, pts, args); + free(new_data); + } + } + av_packet_unref(audio_packet); + } + pcm_list->pop_front(); + delete pcm; + } else { + break; + } + } else { + fprintf(stderr, "pcm data is nullptr..\r\n"); + } + } + } + + av_packet_unref(pkt); + av_packet_free(&pkt); + } + } + + return 0; + } uint64_t get_audio_pts_from_pcm_size(size_t pcm_length) { if (!_open || !_has_audio) return 0; @@ -623,7 +852,7 @@ class FFmpegPacker { } uint64_t audio_us_to_pts(uint64_t us) { - if (!_open || !_has_video) { + if (!_open || !_has_audio) { return 0; } @@ -631,8 +860,10 @@ class FFmpegPacker { } int get_audio_frame_size_per_second() { - if (!_open || !_has_video) + if (!_open || !_has_audio) { return 0; + } + return _audio_frame->sample_rate * _audio_frame->channels * av_get_bytes_per_sample(_audio_format); } }; diff --git a/components/3rd_party/RtspServer/CMakeLists.txt b/components/3rd_party/RtspServer/CMakeLists.txt new file mode 100644 index 00000000..9bf16405 --- /dev/null +++ b/components/3rd_party/RtspServer/CMakeLists.txt @@ -0,0 +1,77 @@ +set(RtspServer_version_str "${CONFIG_RTSP_SERVER_VERSION_MAJOR}.${CONFIG_RTSP_SERVER_VERSION_MINOR}.${CONFIG_RTSP_SERVER_VERSION_PATCH}") +set(RtspServer_unzip_path "${DL_EXTRACTED_PATH}/RtspServer_srcs") +set(src_path "${RtspServer_unzip_path}/RtspServer") +############### Add include ################### +list(APPEND ADD_INCLUDE ".") +set(RtspServer_include_dir "${src_path}/src" + "${src_path}/src/3rdpart") +list(APPEND ADD_INCLUDE ${RtspServer_include_dir}) +set_property(SOURCE ${RtspServer_include_dir} PROPERTY GENERATED 1) +############################################### + +############ Add source files ################# +append_srcs_dir(ADD_SRCS "${src_path}/src/net" + "${src_path}/src/xop") +# aux_source_directory("src" ADD_SRCS) +# set_property(SOURCE ${ADD_SRCS} PROPERTY GENERATED 1) + +# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c") +# FILE(GLOB_RECURSE EXTRA_SRC "src/*.c") +# FILE(GLOB EXTRA_SRC "src/*.c") +# list(APPEND ADD_SRCS ${EXTRA_SRC}) +# aux_source_directory(src ADD_SRCS) # collect all source file in src dir, will set var ADD_SRCS +# append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS +# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c") +# set(ADD_ASM_SRCS "src/asm.S") +# list(APPEND ADD_SRCS ${ADD_ASM_SRCS}) +# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S ASM file as C language +# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB") +############################################### + +###### Add required/dependent components ###### +# list(APPEND ADD_REQUIREMENTS) +# list(APPEND ADD_FILE_DEPENDS include/axx.h) +# set_property(SOURCE ${python_h_path} PROPERTY GENERATED 1) +# add_custom_command(OUTPUT include/axx.h +# COMMAND echo "" > include/axx.h +# COMMENT "Generating axx.h ..." +# ) +############################################### + +###### Add link search path for requirements/libs ###### +# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib") +# list(APPEND ADD_REQUIREMENTS pthread m) # add system libs, pthread and math lib for example here +# set (RtspServer_DIR RtspServer/lib/cmake/RtspServer4) +# find_package(RtspServer REQUIRED) +############################################### + +############ Add static libs ################## +# set(RtspServer_static_lib_file +# ${src_path}/lib/libavutil.a +# ${src_path}/lib/libavformat.a +# ${src_path}/lib/libavcodec.a +# ${src_path}/lib/libavfilter.a +# ) +# list(APPEND ADD_STATIC_LIB ${RtspServer_static_lib_file}) +# set_property(SOURCE ${RtspServer_static_lib_file} PROPERTY GENERATED 1) +# message(STATUS "===========================RtspServer_static_lib_file: ${RtspServer_static_lib_file}") +############################################### + +############ Add dynamic libs ################## + +############################################### + +#### Add compile option for this component #### +#### Just for this component, won't affect other +#### modules, including component that depend +#### on this component +# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1) + +#### Add compile option for this component +#### and components denpend on this component +# list(APPEND ADD_DEFINITIONS -D__ERROR__=00*10000000+__LINE__*1000 -DOS_LINUX) + +############################################### + +# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib +register_component() diff --git a/components/3rd_party/RtspServer/Kconfig b/components/3rd_party/RtspServer/Kconfig new file mode 100644 index 00000000..ebee9b4f --- /dev/null +++ b/components/3rd_party/RtspServer/Kconfig @@ -0,0 +1,15 @@ + + +menu "RtspServer component configuration" + menu "RtspServer version" + config RTSP_SERVER_VERSION_MAJOR + int "RtspServer version major" + default 1 + config RTSP_SERVER_VERSION_MINOR + int "RtspServer version minor" + default 0 + config RTSP_SERVER_VERSION_PATCH + int "RtspServer version patch" + default 0 + endmenu +endmenu diff --git a/components/3rd_party/RtspServer/component.py b/components/3rd_party/RtspServer/component.py new file mode 100644 index 00000000..623f5e62 --- /dev/null +++ b/components/3rd_party/RtspServer/component.py @@ -0,0 +1,36 @@ + +def add_file_downloads(confs : dict) -> list: + ''' + @param confs kconfig vars, dict type + @return list type, items is dict type + ''' + version = f"{confs['CONFIG_RTSP_SERVER_VERSION_MAJOR']}.{confs['CONFIG_RTSP_SERVER_VERSION_MINOR']}.{confs['CONFIG_RTSP_SERVER_VERSION_PATCH']}" + url = f"https://github.com/PHZ76/RtspServer/archive/refs/tags/{version}.tar.gz" + if version == "1.0.0": + sha256sum = "7d21dd494380dc24006ca83b01aad05ca97734b052e3075299ebe222370531d3" + else: + raise Exception(f"version {version} not support") + sites = ["https://github.com/sipeed/MaixCDK/releases/tag/v0.0.0"] + filename = f"RtspServer-{version}.tar.gz" + path = f"RtspServer_srcs" + check_file = 'RtspServer' + rename = { + f'RtspServer-{version}': 'RtspServer' + } + + return [ + { + 'url': f'{url}', + 'urls': [], + 'sites': sites, + 'sha256sum': sha256sum, + 'filename': filename, + 'path': path, + 'check_files': [ + check_file + ], + 'rename': rename + } + ] + + diff --git a/components/3rd_party/RtspServer/maix_rtsp_server.hpp b/components/3rd_party/RtspServer/maix_rtsp_server.hpp new file mode 100644 index 00000000..437a3cee --- /dev/null +++ b/components/3rd_party/RtspServer/maix_rtsp_server.hpp @@ -0,0 +1,159 @@ +#ifndef __MAIX_RTSP_SERVER_HPP__ +#define __MAIX_RTSP_SERVER_HPP__ + +#include "xop/RtspServer.h" +#include "net/Timer.h" +#include +#include +#include + +class MaixRtspServer +{ + int *clients; +public: + xop::MediaSessionId session_id; + std::shared_ptr server; + std::shared_ptr event_loop; + + MaixRtspServer(int *clients, std::shared_ptr server_ptr, xop::MediaSessionId session_id, std::shared_ptr event_loop_ptr) + { + this->clients = clients; + this->server = std::move(server_ptr); + this->session_id = session_id; + this->event_loop = std::move(event_loop_ptr); + } + + ~MaixRtspServer() { + + if (clients != nullptr) { + free(clients); + clients = nullptr; + } + } + + int get_clients() { + int value = 0; + if (clients != nullptr) { + value = *clients; + } + return value; + } + + int video_frame_push(uint8_t *frame, size_t frame_size, size_t timestamp, bool i_or_b = true) { + xop::AVFrame videoFrame = {0}; + videoFrame.type = 0; + videoFrame.size = frame_size; + // videoFrame.timestamp = timestamp; + videoFrame.timestamp = xop::H264Source::GetTimestamp(); + videoFrame.buffer.reset(new uint8_t[videoFrame.size]); + memcpy(videoFrame.buffer.get(), frame, videoFrame.size); + + xop::RtspServer *rtsp_server = this->server.get(); + rtsp_server->PushFrame(this->session_id, xop::channel_0, videoFrame); //送到服务器进行转发, 接口线程安全 + return 0; + } + + int audio_frame_push(uint8_t *frame, size_t frame_size, size_t timestamp) { + xop::AVFrame audioFrame = {0}; + audioFrame.type = xop::AUDIO_FRAME; + audioFrame.size = frame_size; + // audioFrame.timestamp = timestamp; + audioFrame.timestamp = xop::AACSource::GetTimestamp(48000); + audioFrame.buffer.reset(new uint8_t[audioFrame.size]); + memcpy(audioFrame.buffer.get(), frame, audioFrame.size); + + xop::RtspServer *rtsp_server = this->server.get(); + rtsp_server->PushFrame(this->session_id, xop::channel_1, audioFrame); + return 0; + } +}; + +class MaixRtspServerBuilder { + std::string _ip = "0.0.0.0"; + int _port = 8554; + std::string _session_name = "live"; + bool _has_audio = false; + int _audio_sample_rate = 48000; + int _audio_channels = 1; +public: + MaixRtspServerBuilder() { + + } + + ~MaixRtspServerBuilder() { + + } + + MaixRtspServerBuilder &set_ip(std::string ip) { + _ip = ip; + return *this; + } + + MaixRtspServerBuilder &set_port(int port) { + _port = port; + return *this; + } + + MaixRtspServerBuilder &set_session_name(std::string session_name) { + _session_name = session_name; + return *this; + } + + MaixRtspServerBuilder &set_audio(bool en) { + _has_audio = en; + return *this; + } + + MaixRtspServerBuilder &set_audio_sample_rate(int sample_rate) { + _audio_sample_rate = sample_rate; + return *this; + } + + MaixRtspServerBuilder &set_audio_channels(int channels) { + _audio_channels = channels; + return *this; + } + + MaixRtspServer *build() { + std::string ip = _ip; + int port = _port; + std::string session_name = _session_name; + int audio_sample_rate = _audio_sample_rate; + int audio_channels = _audio_channels; + // printf("ip=%s, port=%d, session_name=%s, audio_sample_rate=%d, audio_channels=%d\r\n", ip.c_str(), port, session_name.c_str(), audio_sample_rate, audio_channels); + + std::shared_ptr event_loop(new xop::EventLoop()); + std::shared_ptr server = xop::RtspServer::Create(event_loop.get()); + if (!server->Start(ip, port)) { + throw "rtsp server start failed"; + } + + xop::MediaSession *session = xop::MediaSession::CreateNew(session_name); + session->AddSource(xop::channel_0, xop::H264Source::CreateNew()); + if (_has_audio) { + session->AddSource(xop::channel_1, xop::AACSource::CreateNew(audio_sample_rate, audio_channels)); + } + int *clients = (int *)malloc(sizeof(int)); + if (clients == nullptr) { + throw "alloc memory failed"; + } + *clients = 0; + session->AddNotifyConnectedCallback([clients] (xop::MediaSessionId sessionId, std::string peer_ip, uint16_t peer_port){ + printf("RTSP client connect, ip=%s, port=%hu \n", peer_ip.c_str(), peer_port); + (*clients) ++; + }); + session->AddNotifyDisconnectedCallback([clients](xop::MediaSessionId sessionId, std::string peer_ip, uint16_t peer_port) { + printf("RTSP client disconnect, ip=%s, port=%hu \n", peer_ip.c_str(), peer_port); + if (*clients > 0) { + (*clients) --; + } + }); + xop::MediaSessionId session_id = server->AddSession(session); + + MaixRtspServer *new_server(new MaixRtspServer(clients, server, session_id, event_loop)); + + return new_server; + } +}; + +#endif // __MAIX_RTSP_SERVER_HPP__ \ No newline at end of file diff --git a/components/vision/CMakeLists.txt b/components/vision/CMakeLists.txt index f8bcf706..b75fe6d1 100644 --- a/components/vision/CMakeLists.txt +++ b/components/vision/CMakeLists.txt @@ -43,7 +43,7 @@ list(APPEND ADD_REQUIREMENTS zbar omv) if(PLATFORM_LINUX) list(APPEND ADD_REQUIREMENTS sdl) elseif(PLATFORM_MAIXCAM) - list(APPEND ADD_REQUIREMENTS FFmpeg maixcam_lib media_server) + list(APPEND ADD_REQUIREMENTS FFmpeg maixcam_lib RtspServer) if(NOT CONFIG_MAIXCAM_LIB_COMPILE_FROM_SOURCE) set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group vision/libvision.a -lmaixcam_lib -Wl,--end-group" PARENT_SCOPE) set(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -Wl,--start-group vision/libvision.a -lmaixcam_lib -Wl,--end-group" PARENT_SCOPE) diff --git a/components/vision/include/maix_rtsp.hpp b/components/vision/include/maix_rtsp.hpp index c1a4f3d4..29e52406 100644 --- a/components/vision/include/maix_rtsp.hpp +++ b/components/vision/include/maix_rtsp.hpp @@ -24,6 +24,7 @@ namespace maix::rtsp enum RtspStreamType { RTSP_STREAM_NONE = 0, // format invalid + RTSP_STREAM_H264, RTSP_STREAM_H265, }; @@ -87,10 +88,11 @@ namespace maix::rtsp * @param port rtsp port * @param fps rtsp fps * @param stream_type rtsp stream type + * @param bitrate rtsp bitrate * @maixpy maix.rtsp.Rtsp.__init__ * @maixcdk maix.rtsp.Rtsp.Rtsp */ - Rtsp(std::string ip = std::string(), int port = 8554, int fps = 30, rtsp::RtspStreamType stream_type = rtsp::RtspStreamType::RTSP_STREAM_H265); + Rtsp(std::string ip = std::string(), int port = 8554, int fps = 30, rtsp::RtspStreamType stream_type = rtsp::RtspStreamType::RTSP_STREAM_H264, int bitrate = 3000 * 1000); ~Rtsp(); /** @@ -116,7 +118,16 @@ namespace maix::rtsp err::Err bind_camera(camera::Camera *camera); /** - * @brief Write data to rtsp + * @brief Bind audio recorder + * @note If the audio_recorder object is bound, the audio_recorder object cannot be used elsewhere. + * @param recorder audio_recorder object + * @return error code, err::ERR_NONE means success, others means failed + * @maixpy maix.rtsp.Rtsp.bind_audio_recorder + */ + err::Err bind_audio_recorder(audio::Recorder *recorder); + + /** + * @brief Write data to rtsp(This function will be removed in the future) * @param frame video frame data * @return error code, err::ERR_NONE means success, others means failed * @maixpy maix.rtsp.Rtsp.write @@ -142,14 +153,12 @@ namespace maix::rtsp * @return camera object * @maixpy maix.rtsp.Rtsp.to_camera */ - camera::Camera *to_camera() { - return this->_camera; - } + camera::Camera *to_camera(); /** * @brief return rtsp start status * @return true means rtsp is start, false means rtsp is stop. - * @maixpy maix.rtsp.Rtsp.rtsp_is_start + * @maixcdk maix.rtsp.Rtsp.rtsp_is_start */ bool rtsp_is_start() { @@ -157,7 +166,7 @@ namespace maix::rtsp } /** - * @brief return a region object, you can draw image on the region. + * @brief return a region object, you can draw image on the region.(This function will be removed in the future) * @param x region coordinate x * @param y region coordinate y * @param width region width @@ -169,21 +178,21 @@ namespace maix::rtsp rtsp::Region *add_region(int x, int y, int width, int height, image::Format format = image::Format::FMT_BGRA8888); /** - * @brief update and show region + * @brief update and show region(This function will be removed in the future) * @return error code * @maixpy maix.rtsp.Rtsp.update_region */ err::Err update_region(rtsp::Region ®ion); /** - * @brief del region + * @brief del region(This function will be removed in the future) * @return error code * @maixpy maix.rtsp.Rtsp.del_region */ err::Err del_region(rtsp::Region *region); /** - * @brief return region list + * @brief return region list(This function will be removed in the future) * @attention DO NOT ADD THIS FUNC TO MAIXPY * @return return a list of region */ @@ -192,7 +201,7 @@ namespace maix::rtsp } /** - * @brief Draw a rectangle on the canvas + * @brief Draw a rectangle on the canvas(This function will be removed in the future) * @param id region id * @param x rectangle coordinate x * @param y rectangle coordinate y @@ -206,7 +215,7 @@ namespace maix::rtsp err::Err draw_rect(int id, int x, int y, int width, int height, image::Color color, int thickness = 1); /** - * @brief Draw a string on the canvas + * @brief Draw a string on the canvas(This function will be removed in the future) * @param id region id * @param x string coordinate x * @param y string coordinate y @@ -242,8 +251,6 @@ namespace maix::rtsp int _fps; rtsp::RtspStreamType _stream_type; bool _is_start; - bool _bind_camera; - camera::Camera *_camera; thread::Thread *_thread; std::vector _region_list; std::vector _region_used_list; @@ -252,6 +259,7 @@ namespace maix::rtsp uint64_t _timestamp; uint64_t _last_ms; int _region_max_number; + void *_param; }; } // namespace maix::rtsp diff --git a/components/vision/port/linux/maix_rtsp_linux.cpp b/components/vision/port/linux/maix_rtsp_linux.cpp index f5af2ccb..b7b209d5 100644 --- a/components/vision/port/linux/maix_rtsp_linux.cpp +++ b/components/vision/port/linux/maix_rtsp_linux.cpp @@ -35,11 +35,12 @@ namespace maix::rtsp return err::ERR_NOT_IMPL; } - Rtsp::Rtsp(std::string ip, int port, int fps, rtsp::RtspStreamType stream_type) { + Rtsp::Rtsp(std::string ip, int port, int fps, rtsp::RtspStreamType stream_type, int bitrate) { (void)ip; (void)port; (void)fps; (void)stream_type; + (void)bitrate; } Rtsp::~Rtsp() { @@ -61,12 +62,23 @@ namespace maix::rtsp return err; } + err::Err Rtsp::bind_audio_recorder(audio::Recorder *recorder) { + err::Err err = err::ERR_NOT_IMPL; + (void)recorder; + return err; + } + err::Err Rtsp::write(video::Frame &stream) { err::Err err = err::ERR_NOT_IMPL; (void)stream; return err; } + camera::Camera *Rtsp::to_camera() { + err::check_null_raise(NULL, "The camera object is NULL"); + return NULL; + } + std::string Rtsp::get_url() { return "not supported"; } diff --git a/components/vision/port/maixcam/maix_display_mmf.hpp b/components/vision/port/maixcam/maix_display_mmf.hpp index d5d31d7a..2593516c 100644 --- a/components/vision/port/maixcam/maix_display_mmf.hpp +++ b/components/vision/port/maixcam/maix_display_mmf.hpp @@ -148,7 +148,7 @@ namespace maix::display } } - log::info("disp config flip: %d, mirror: %d, max_backlight: %.1f", flip, mirror, max_backlight); + // log::info("disp config flip: %d, mirror: %d, max_backlight: %.1f", flip, mirror, max_backlight); } diff --git a/components/vision/port/maixcam/maix_rtmp_maixcam.cpp b/components/vision/port/maixcam/maix_rtmp_maixcam.cpp index a5326f17..43f1d224 100644 --- a/components/vision/port/maixcam/maix_rtmp_maixcam.cpp +++ b/components/vision/port/maixcam/maix_rtmp_maixcam.cpp @@ -4,14 +4,6 @@ #include "maix_basic.hpp" #include "sophgo_middleware.hpp" #include -#include "sockutil.h" -#include "sys/system.h" -#include "rtmp-client.h" -#include "flv-reader.h" -#include "flv-writer.h" -#include "flv-header.h" -#include "flv-proto.h" -#include "maix_avc2flv.h" #include #include #include diff --git a/components/vision/port/maixcam/maix_rtsp_maixcam.cpp b/components/vision/port/maixcam/maix_rtsp_maixcam.cpp index e766bbde..32e4986c 100644 --- a/components/vision/port/maixcam/maix_rtsp_maixcam.cpp +++ b/components/vision/port/maixcam/maix_rtsp_maixcam.cpp @@ -7,10 +7,15 @@ #include "maix_rtsp.hpp" #include "maix_err.hpp" -#include "rtsp_server.h" #include "maix_basic.hpp" +#include "maix_audio.hpp" +#include "maix_video.hpp" +#include "maix_ffmpeg.hpp" #include #include "sophgo_middleware.hpp" +#include "maix_rtsp_server.hpp" +#include +#include namespace maix::rtsp { @@ -31,6 +36,20 @@ namespace maix::rtsp int flip = true; int mirror = true; + auto configs = sys::device_configs(true); + for (auto &item : configs) { + log::info("device:%s value:%s", item.first.c_str(), item.second.c_str()); + } + auto mirror_string = configs.find("cam_flip"); + auto flip_string = configs.find("cam_mirror"); + if (mirror_string != configs.end()) { + mirror = !atoi(mirror_string->second.c_str()); + } + + if (flip_string != configs.end()) { + flip = !atoi(flip_string->second.c_str()); + } + int x2 = flip ? camera->width() - width - x : x; int y2 = mirror ? camera->height() - height - y : y; @@ -122,16 +141,39 @@ namespace maix::rtsp return err::ERR_NONE; } - Rtsp::Rtsp(std::string ip, int port, int fps, rtsp::RtspStreamType stream_type) { - err::check_bool_raise(stream_type == rtsp::RtspStreamType::RTSP_STREAM_H265, - "support RTSP_STREAM_H265 only!"); + enum RtspStatus{ + RTSP_IDLE = 0, + RTSP_RUNNING, + RTSP_STOP, + }; + + typedef struct { + MaixRtspServer *rtsp_server; + enum RtspStatus status; + int *clients; + camera::Camera *camera; + video::Encoder *encoder; + audio::Recorder *audio_recorder; + ffmpeg::FFmpegPacker *ffmpeg_packer; + bool bind_camera; + bool bind_audio_recorder; + int encoder_bitrate; + int fps; + int audio_bitrate; + } rtsp_param_t; + + Rtsp::Rtsp(std::string ip, int port, int fps, rtsp::RtspStreamType stream_type, int bitrate) { + rtsp_param_t *param = (rtsp_param_t *)malloc(sizeof(rtsp_param_t)); + err::check_null_raise(param, "malloc failed!"); + memset(param, 0, sizeof(rtsp_param_t)); + this->_ip = ip; this->_port = port; this->_fps = fps; this->_stream_type = stream_type; - this->_bind_camera = false; this->_is_start = false; this->_thread = NULL; + this->_param = param; this->_region_max_number = 16; for (int i = 0; i < this->_region_max_number; i ++) { this->_region_list.push_back(NULL); @@ -139,123 +181,312 @@ namespace maix::rtsp this->_region_used_list.push_back(false); } - char *new_ip = NULL; - if (this->_ip.size() != 0) { - new_ip = (char *)this->_ip.c_str(); + if (_ip.size() == 0) { + _ip = "0.0.0.0"; } + this->_timestamp = 0; this->_last_ms = 0; - err::check_bool_raise(!rtsp_server_init(new_ip, this->_port), "Rtsp init failed!"); + + param->status = RTSP_IDLE; + param->camera = nullptr; + param->audio_recorder = nullptr; + param->bind_camera = false; + param->bind_audio_recorder = false; + param->encoder_bitrate = bitrate; + param->fps = fps; + param->audio_bitrate = 128000; } Rtsp::~Rtsp() { - if (this->_is_start) { - this->stop(); - } + rtsp_param_t *param = (rtsp_param_t *)_param; + if (param) { + if (param->status != RTSP_IDLE) { + this->stop(); + } + + if (param->encoder) { + delete param->encoder; + param->encoder = nullptr; + } - if (0 != rtsp_server_deinit()) { - log::warn("rtsp deinit failed!\r\n"); + if (param->rtsp_server) { + delete param->rtsp_server; + param->rtsp_server = nullptr; + } } for (auto ®ion : this->_region_list) { delete region; } } +#if 0 + static void dump_type(uint8_t *m_buf, int bytes_read) + { + int i = 0, start_code = 0; + for (i=0; ito_camera()->fps(); - uint64_t wait_us = 1000000 / fps; - uint64_t last_us = time::time_us(); - while (rtsp->rtsp_is_start()) { - rtsp->update_timestamp(); - uint64_t timestamp = rtsp->get_timestamp(); - - mmf_h265_stream_t stream; - if (!mmf_enc_h265_pop(enc_ch, &stream)) { - int stream_size = 0; - for (int i = 0; i < stream.count; i ++) { - // log::info("[%d] stream.data:%p stream.len:%d\n", i, stream.data[i], stream.data_size[i]); - stream_size += stream.data_size[i]; + if (start_code > 0) { + printf("[%d] nalu type:%d\r\n", i, m_buf[i+start_code]&0x1F); } - if (stream.count > 1) { - uint8_t *stream_buffer = (uint8_t *)malloc(stream_size); - if (stream_buffer) { - int copy_length = 0; - for (int i = 0; i < stream.count; i ++) { - memcpy(stream_buffer + copy_length, stream.data[i], stream.data_size[i]); - copy_length += stream.data_size[i]; + i += start_code; + continue; + } + else if(m_buf[i] == 0 && m_buf[i+1] == 0 && m_buf[i+2] == 0 && m_buf[i+3] == 1) { + start_code = 4; + + if (start_code > 0) { + printf("[%d] nalu type:%d\r\n", i, m_buf[i+start_code]&0x1F); + } + + i += start_code; + continue; + } + else { + continue; + } + } + } +#endif + + static uint64_t get_video_timestamp() { + return time::ticks_ms() * 90; + } + + static uint64_t get_audio_timestamp() { + return get_video_timestamp(); + } + + static void _camera_push_thread(void *args) { + rtsp_param_t *param = (rtsp_param_t *)args; + MaixRtspServer *rtsp_server = param->rtsp_server; + int vi_ch = param->camera->get_channel(); + int enc_ch = 1; + size_t video_pts = 0, audio_pts = 0; + uint64_t last_ms = time::ticks_ms(); + bool found_first_pps_sps = false; + uint64_t last_read_pcm_ms = time::ticks_ms(); + + typedef struct { + MaixRtspServer *rtsp_server; + bool is_i_or_b; + } temp_param_t; + + while (param->status == RTSP_RUNNING) { + void *frame = NULL; + bool found_camera_frame = false; + bool found_venc_stream = false; + bool found_audio_data = false; + mmf_frame_info_t f_info; + mmf_stream_t venc_stream = {0}; + + while (((time::ticks_ms() - last_ms) * 1000) < ((uint64_t)(1000000 / param->fps) - 1000)) { + time::sleep_ms(1); + } + + // log::info(" loop use %lld ms", time::ticks_ms() - last_ms); + last_ms = time::ticks_ms(); + + Bytes *pcm = nullptr; + if (param->bind_audio_recorder) { + int frame_size_per_second = param->ffmpeg_packer->get_audio_frame_size_per_second(); + uint64_t loop_ms = time::ticks_ms() - last_read_pcm_ms; + last_read_pcm_ms = time::ticks_ms(); + int read_pcm_size = frame_size_per_second * loop_ms * 1.5 / 1000; + pcm = param->audio_recorder->record_bytes(read_pcm_size); + } + + if (pcm) { + found_audio_data = true; + } + + if (0 == mmf_vi_frame_pop2(vi_ch, &frame, &f_info) && frame) { + found_camera_frame = true; + } + + if (0 == mmf_venc_pop(enc_ch, &venc_stream) && venc_stream.count > 0) { + found_venc_stream = true; + } + + if (rtsp_server->get_clients() > 0) { + if (found_venc_stream) { + if (!found_first_pps_sps) { + if (venc_stream.count > 1) { + found_first_pps_sps = true; + video_pts = 0; + } + } + + if (found_first_pps_sps) { + int venc_output_data_size = 0; + uint8_t *venc_output_data = nullptr; + for (int i = 0; i < venc_stream.count; i ++) { + // printf("[%d] stream.data:%p stream.len:%d\n", i, venc_stream.data[i], venc_stream.data_size[i]); + venc_output_data_size += venc_stream.data_size[i]; + } + + venc_output_data = (uint8_t *)malloc(venc_output_data_size); + if (venc_output_data) { + int curr_data_size = 0; + for (int i = 0; i < venc_stream.count; i ++) { + memcpy(venc_output_data + curr_data_size, venc_stream.data[i], venc_stream.data_size[i]); + curr_data_size += venc_stream.data_size[i]; + } + + + auto _callback = [](void *data, size_t data_size, size_t pts, void *args) { + temp_param_t *param = (temp_param_t *)args; + param->rtsp_server->video_frame_push((uint8_t *)data, data_size, pts, param->is_i_or_b); + }; + + temp_param_t temp_param; + temp_param.rtsp_server = rtsp_server; + temp_param.is_i_or_b = venc_stream.count > 1 ? true : false; + + video_pts = get_video_timestamp(); + param->ffmpeg_packer->push2(venc_output_data, venc_output_data_size, video_pts, false, _callback, &temp_param); + free(venc_output_data); } - rtsp_send_h265_data(timestamp, stream_buffer, copy_length); - free(stream_buffer); - } else { - log::warn("malloc failed!\r\n"); } - } else if (stream.count == 1) { - rtsp_send_h265_data(timestamp, (uint8_t *)stream.data[0], stream.data_size[0]); } - if (mmf_enc_h265_free(enc_ch)) { - log::warn("mmf_enc_h265_free failed\n"); - continue; + if (found_audio_data) { + if (pcm && pcm->data_len > 0) { + auto _callback = [](void *data, size_t data_size, size_t pts, void *args) { + temp_param_t *param = (temp_param_t *)args; + param->rtsp_server->audio_frame_push((uint8_t *)data, data_size, pts); + }; + temp_param_t temp_param; + temp_param.rtsp_server = rtsp_server; + + audio_pts = get_audio_timestamp(); + param->ffmpeg_packer->push2(pcm->data, pcm->data_len, audio_pts, true, _callback, &temp_param); + } } + } else { + found_first_pps_sps = false; } - if (mmf_vi_frame_pop(vi_ch, &data, &data_size, &width, &height, &format)) { - continue; - } - while (time::ticks_us() - last_us < wait_us) { - time::sleep_us(50); + if (found_venc_stream) { + if (0 != mmf_venc_free(enc_ch)) { + log::error("mmf_venc_free failed!\r\n"); + } } - last_us = time::ticks_us(); - if (mmf_enc_h265_push(enc_ch, (uint8_t *)data, width, height, format)) { - log::warn("mmf_enc_h265_push failed\n"); - continue; + if (found_camera_frame) { + if (0 != mmf_venc_push2(enc_ch, frame)) { + log::error("mmf_venc_push2 failed!\r\n"); + } + + mmf_vi_frame_free(vi_ch); } - mmf_vi_frame_free(vi_ch); + if (found_audio_data) { + delete pcm; + } } + + param->status = RTSP_IDLE; } err::Err Rtsp::start() { err::Err err = err::ERR_NONE; + rtsp_param_t *param = (rtsp_param_t *)_param; + if (!param) { + return err::ERR_RUNTIME; + } - if (0 != rtsp_server_start()) { - log::error("rtsp start failed!\r\n"); + if (param->status != RTSP_IDLE) { + return err::ERR_BUSY; + } + + // check camera + if (!param->bind_camera || !param->camera) { + log::error("You need bind a camera!"); + return err::ERR_RUNTIME; + } + + if (param->camera->width() % 32 != 0) { + log::error("camera width must be multiple of 32!\r\n"); return err::ERR_RUNTIME; } - if (this->_bind_camera) { - this->_thread = new thread::Thread(_camera_push_thread, this); - if (this->_thread == NULL) { - log::error("create camera thread failed!\r\n"); + // create rtsp server + MaixRtspServerBuilder rtsp_builder = MaixRtspServerBuilder() + .set_ip(_ip) + .set_port(this->_port) + .set_session_name("live"); + if (param->bind_audio_recorder && param->audio_recorder) { + rtsp_builder = rtsp_builder.set_audio(true) + .set_audio_channels(param->audio_recorder->channel()) + .set_audio_sample_rate(param->audio_recorder->sample_rate()); + } + param->rtsp_server = rtsp_builder.build(); + + // create encoder + if (param->encoder) { + delete param->encoder; + param->encoder = nullptr; + } + param->encoder = new video::Encoder("", param->camera->width(), param->camera->height(), image::Format::FMT_YVU420SP, video::VIDEO_H264, param->fps, 50, param->encoder_bitrate); + err::check_null_raise(param->encoder, "Create video encoder failed!"); + + // create frame package + param->ffmpeg_packer = new ffmpeg::FFmpegPacker(); + err::check_null_raise(param->ffmpeg_packer, "ffmpeg packer init failed"); + err::check_bool_raise(!param->ffmpeg_packer->config2("context_format_name", "flv"), "rtmp config failed!"); + if (param->bind_audio_recorder && param->audio_recorder) { + if (param->audio_recorder->format() != audio::FMT_S16_LE) { + log::error("Only support audio::FMT_S16_LE format!"); return err::ERR_RUNTIME; } + err::check_bool_raise(!param->ffmpeg_packer->config("has_audio", true), "rtmp config failed!"); + err::check_bool_raise(!param->ffmpeg_packer->config("audio_sample_rate", param->audio_recorder->sample_rate()), "rtmp config failed!"); + err::check_bool_raise(!param->ffmpeg_packer->config("audio_channels", param->audio_recorder->channel()), "rtmp config failed!"); + err::check_bool_raise(!param->ffmpeg_packer->config("audio_bitrate", param->audio_bitrate), "rtmp config failed!"); + err::check_bool_raise(!param->ffmpeg_packer->config("audio_format", AV_SAMPLE_FMT_S16), "rtmp config failed!"); + } + param->ffmpeg_packer->open(); + + // create runtime thread + param->status = RTSP_RUNNING; + _thread = new thread::Thread(_camera_push_thread, param); + if (_thread == NULL) { + log::error("create camera thread failed!\r\n"); + return err::ERR_RUNTIME; } - - this->_is_start = true; - return err; } err::Err Rtsp::stop() { err::Err err = err::ERR_NONE; + rtsp_param_t *param = (rtsp_param_t *)_param; + if (param->status != RTSP_RUNNING) { + return err::ERR_NONE; + } - this->_is_start = false; + param->status = RTSP_STOP; + if (_thread) { + _thread->join(); + _thread = nullptr; + } - if (this->_bind_camera) { - this->_thread->join(); + if (param->ffmpeg_packer) { + delete param->ffmpeg_packer; + param->ffmpeg_packer = nullptr; } - if (0 != rtsp_server_stop()) { - log::error("rtsp stop failed!\r\n"); - this->_is_start = true; - return err::ERR_RUNTIME; + if (param->encoder) { + delete param->encoder; + param->encoder = nullptr; + } + + if (param->rtsp_server) { + delete param->rtsp_server; + param->rtsp_server = nullptr; } return err; @@ -263,56 +494,128 @@ namespace maix::rtsp err::Err Rtsp::bind_camera(camera::Camera *camera) { err::Err err = err::ERR_NONE; + rtsp_param_t *param = (rtsp_param_t *)_param; + if (!param) { + return err::ERR_RUNTIME; + } if (camera->format() != image::Format::FMT_YVU420SP) { err::check_raise(err::ERR_RUNTIME, "bind camera failed! support FMT_YVU420SP only!\r\n"); return err::ERR_RUNTIME; } - this->_camera = camera; - this->_bind_camera = true; + param->camera = camera; + param->bind_camera = true; return err; } - err::Err Rtsp::write(video::Frame &frame) { + err::Err Rtsp::bind_audio_recorder(audio::Recorder *recorder) { err::Err err = err::ERR_NONE; - - if (frame.type() != video::VideoType::VIDEO_ENC_H265_CBR) { - log::warn("You passed in an unsupported type!\r\n"); + rtsp_param_t *param = (rtsp_param_t *)_param; + if (!param) { return err::ERR_RUNTIME; } - void *data; - int data_len = 0; - if (err::ERR_NONE != frame.get(&data, &data_len) || data_len == 0) { - return err::ERR_NONE; - } - - this->update_timestamp(); - uint64_t timestamp = this->get_timestamp(); - rtsp_send_h265_data(timestamp, (uint8_t *)data, data_len); + param->audio_recorder = recorder; + param->bind_audio_recorder = true; + return err; + } + err::Err Rtsp::write(video::Frame &frame) { + err::Err err = err::ERR_NONE; + err::check_raise(err::ERR_NOT_IMPL, "write frame not impl!"); return err; } + camera::Camera *Rtsp::to_camera() { + rtsp_param_t *param = (rtsp_param_t *)_param; + err::check_null_raise(param->camera, "camera is null!"); + return param->camera; + } + std::string Rtsp::get_url() { - std::string real_ip = std::string(rtsp_get_server_ip()); - std::string real_port = std::to_string(rtsp_get_server_port()); - return "rtsp://" + real_ip + ":" + real_port + "/live"; + return "rtsp://" + _ip + ":" + std::to_string(_port) + "/live"; + } + + + static int get_ip(char *hw, char ip[16]) + { + struct ifaddrs *ifaddr, *ifa; + int family, s; + char host[NI_MAXHOST]; + + if (getifaddrs(&ifaddr) == -1) { + perror("getifaddrs"); + return -1; + } + + for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) { + if (ifa->ifa_addr == NULL) { + continue; + } + + family = ifa->ifa_addr->sa_family; + + if (family == AF_INET) { + s = getnameinfo(ifa->ifa_addr, (family == AF_INET) ? sizeof(struct sockaddr_in) : + sizeof(struct sockaddr_in6), host, NI_MAXHOST, NULL, 0, NI_NUMERICHOST); + if (s != 0) { + printf("getnameinfo() failed: %s\n", gai_strerror(s)); + return -1; + } + + if (!strcmp(ifa->ifa_name, hw)) { + strncpy(ip, host, 16); + freeifaddrs(ifaddr); + return 0; + } + } + } + + freeifaddrs(ifaddr); + return -1; + } + + static std::vector rtsp_get_server_urls(std::string ip, int port) + { + char new_ip[16] = {0}; + + std::vector ip_list; + + if (!strcmp("0.0.0.0", ip.c_str())) { + if (!get_ip((char *)"eth0", new_ip)) { + ip_list.push_back("rtsp://" + std::string(new_ip) + ":" + std::to_string(port) + "/live"); + } + if (!get_ip((char *)"usb0", new_ip)) { + ip_list.push_back("rtsp://" + std::string(new_ip) + ":" + std::to_string(port) + "/live"); + } + if (!get_ip((char *)"wlan0", new_ip)) { + ip_list.push_back("rtsp://" + std::string(new_ip) + ":" + std::to_string(port) + "/live"); + } + } else { + ip_list.push_back("rtsp://" + ip + ":" + std::to_string(port) + "/live"); + } + + return ip_list; } std::vector Rtsp::get_urls() { - return rtsp_get_server_urls(); + return rtsp_get_server_urls(_ip, _port); } rtsp::Region *Rtsp::add_region(int x, int y, int width, int height, image::Format format) { + rtsp_param_t *param = (rtsp_param_t *)_param; + if (!param) { + return nullptr; + } + if (format != image::Format::FMT_BGRA8888) { log::error("region support FMT_BGRA8888 only!\r\n"); return NULL; } - if (!this->_bind_camera) { + if (!param->bind_camera) { log::error("You must use bind camera firstly!\r\n"); return NULL; } @@ -328,7 +631,7 @@ namespace maix::rtsp err::check_bool_raise(unused_idx != -1, "Unused region not found"); // Create region - rtsp::Region *region = new rtsp::Region(x, y, width, height, format, this->_camera); + rtsp::Region *region = new rtsp::Region(x, y, width, height, format, param->camera); err::check_null_raise(region, "Create region failed!"); this->_region_list[unused_idx] = region; this->_region_used_list[unused_idx] = true; @@ -359,6 +662,11 @@ namespace maix::rtsp err::Err Rtsp::draw_rect(int id, int x, int y, int width, int height, image::Color color, int thickness) { + rtsp_param_t *param = (rtsp_param_t *)_param; + if (!param) { + return err::ERR_RUNTIME; + } + // Check id if (id < 0 || id > 3) { log::error("region id is invalid! range is [0, 3"); @@ -375,22 +683,22 @@ namespace maix::rtsp y = 0; } - if (x > this->_camera->width()) { + if (x > param->camera->width()) { x = 0; width = 0; } - if (y > this->_camera->height()) { + if (y > param->camera->height()) { y = 0; height = 0; } - if (x + width > this->_camera->width()) { - width = this->_camera->width() - x; + if (x + width > param->camera->width()) { + width = param->camera->width() - x; } - if (y + height > this->_camera->height()) { - height = this->_camera->height() - y; + if (y + height > param->camera->height()) { + height = param->camera->height() - y; } // Check if the region [id, id + 4) is used for other functions diff --git a/components/vision/port/maixcam/maix_video_mmf.cpp b/components/vision/port/maixcam/maix_video_mmf.cpp index 4484506f..7f0fc60f 100644 --- a/components/vision/port/maixcam/maix_video_mmf.cpp +++ b/components/vision/port/maixcam/maix_video_mmf.cpp @@ -2565,7 +2565,6 @@ namespace maix::video int resample_sample_rate = param->resample_sample_rate; enum AVSampleFormat resample_format = param->resample_sample_format; SwrContext *swr_ctx = param->swr_ctx; - image::Image *img = NULL; video::Context *context = NULL; uint64_t last_pts = 0; bool is_video = false; diff --git a/examples/rtsp_demo/main/src/main.cpp b/examples/rtsp_demo/main/src/main.cpp index 7ff8dd68..613b0211 100644 --- a/examples/rtsp_demo/main/src/main.cpp +++ b/examples/rtsp_demo/main/src/main.cpp @@ -1,4 +1,3 @@ - #include "stdio.h" #include "main.h" #include "maix_util.hpp" @@ -18,7 +17,6 @@ using namespace maix; int _main(int argc, char* argv[]) { - int cnt = 0; int cam_w = -1; int cam_h = -1; image::Format cam_fmt = image::Format::FMT_YVU420SP; @@ -42,8 +40,10 @@ int _main(int argc, char* argv[]) camera::Camera cam = camera::Camera(cam_w, cam_h, cam_fmt, "", cam_fps, cam_buffer_num); camera::Camera *cam2 = cam.add_channel(640, 480); display::Display disp = display::Display(); + auto audio_recorder = audio::Recorder(); rtsp::Rtsp rtsp = rtsp::Rtsp(); rtsp.bind_camera(&cam); + rtsp.bind_audio_recorder(&audio_recorder); rtsp::Region *region = rtsp.add_region(0, 0, 200, 100); rtsp::Region *region3 = rtsp.add_region(400, 200, 200, 100); @@ -52,14 +52,15 @@ int _main(int argc, char* argv[]) rgn_img->draw_string(0, 0, "hello"); region3->update_canvas(); + log::info("url:%s", rtsp.get_url().c_str()); std::vector url = rtsp.get_urls(); for (size_t i = 0; i < url.size(); i ++) { - log::info("%s\r\n", url[i].c_str()); + log::info("url[%d]:%s", i, url[i].c_str()); } - - rtsp.start(); + err::check_raise(rtsp.start()); uint64_t last_ms = time::ticks_ms(); + int cnt = 0; while(!app::need_exit()) { cnt ++; image::Color color = image::COLOR_BLACK; @@ -80,12 +81,12 @@ int _main(int argc, char* argv[]) maix::image::Image *img = cam2->read(); disp.show(*img); delete img; - uint64_t curr_ms = time::ticks_ms(); log::info("loop use %lld ms\r\n", curr_ms - last_ms); last_ms = curr_ms; } + rtsp.stop(); delete cam2; return 0; @@ -100,4 +101,4 @@ int main(int argc, char* argv[]) // if we don't catch exception, when program throw exception, the objects will not be destructed. // So we catch exception here to let resources be released(call objects' destructor) before exit. CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv); -} +} \ No newline at end of file diff --git a/projects/app_camera/main/app/app.cpp b/projects/app_camera/main/app/app.cpp index 399d3c85..f425b7f3 100644 --- a/projects/app_camera/main/app/app.cpp +++ b/projects/app_camera/main/app/app.cpp @@ -787,6 +787,13 @@ static void _capture_image(maix::camera::Camera &camera, maix::image::Image *img fs::mkdir(picture_path); } std::vector *file_list = fs::listdir(picture_path); + if (file_list) { + auto it = std::find(file_list->begin(), file_list->end(), ".thumbnail"); + if (it != file_list->end()) { + file_list->erase(it); + } + } + printf("file_list_cnt:%ld\n", file_list->size()); string picture_save_path = picture_path + "/" + std::to_string(file_list->size()) +".jpg"; std::string thumbnail_path = picture_path + "/.thumbnail/" + std::to_string(file_list->size()) +".jpg";