From 1e4ff84e44c54d8f3ad5cb2c7ccef185619ddbf7 Mon Sep 17 00:00:00 2001 From: xucz Date: Fri, 16 Jun 2023 11:22:35 +0800 Subject: [PATCH 01/13] adapt to 421-hotfix sdk. --- Android/APIExample-Audio/app/build.gradle | 2 +- .../src/main/cpp/AgoraRtcKit/AgoraBase.h | 640 +++++++++++------- .../src/main/cpp/AgoraRtcKit/AgoraMediaBase.h | 236 ++++--- .../cpp/AgoraRtcKit/AgoraMediaPlayerTypes.h | 5 + .../cpp/AgoraRtcKit/IAgoraMediaPlayerSource.h | 10 +- .../AgoraRtcKit/IAgoraRtmpStreamingService.h | 6 +- .../src/main/cpp/AgoraRtcKit/IAgoraService.h | 43 +- .../AgoraRtcKit/NGIAgoraAudioDeviceManager.h | 9 +- .../main/cpp/AgoraRtcKit/NGIAgoraAudioTrack.h | 52 +- .../cpp/AgoraRtcKit/NGIAgoraCameraCapturer.h | 4 +- .../cpp/AgoraRtcKit/NGIAgoraDataChannel.h | 159 +++++ .../main/cpp/AgoraRtcKit/NGIAgoraLocalUser.h | 112 ++- .../cpp/AgoraRtcKit/NGIAgoraRtcConnection.h | 28 - .../cpp/AgoraRtcKit/NGIAgoraRtmpConnection.h | 14 +- .../cpp/AgoraRtcKit/NGIAgoraRtmpLocalUser.h | 90 ++- .../main/cpp/AgoraRtcKit/NGIAgoraVideoFrame.h | 2 + .../main/cpp/AgoraRtcKit/NGIAgoraVideoTrack.h | 14 +- Android/APIExample/app/build.gradle | 3 +- windows/APIExample/install.ps1 | 2 +- 19 files changed, 974 insertions(+), 457 deletions(-) create mode 100644 Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraDataChannel.h diff --git a/Android/APIExample-Audio/app/build.gradle b/Android/APIExample-Audio/app/build.gradle index 6f958c176..e14e9946b 100644 --- a/Android/APIExample-Audio/app/build.gradle +++ b/Android/APIExample-Audio/app/build.gradle @@ -48,7 +48,7 @@ dependencies { implementation fileTree(dir: "${localSdkPath}", include: ['*.jar', '*.aar']) } else{ - def agora_sdk_version = "4.2.0" + def agora_sdk_version = "4.2.1" // case 1: full single lib with voice only implementation "io.agora.rtc:voice-sdk:${agora_sdk_version}" // case 2: partial libs with voice only diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/AgoraBase.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/AgoraBase.h index b9dfd935d..b5c2c4112 100644 --- a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/AgoraBase.h +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/AgoraBase.h @@ -83,25 +83,6 @@ #define INVALID_DISPLAY_ID 0xffff namespace agora { -namespace commons { -namespace cjson { -class JsonWrapper; -} // namespace cjson -} // namespace commons - -typedef commons::cjson::JsonWrapper any_document_t; - -namespace base { -class IEngineBase; - -class IParameterEngine { - public: - virtual int setParameters(const char* parameters) = 0; - virtual int getParameters(const char* key, any_document_t& result) = 0; - virtual ~IParameterEngine() {} -}; -} // namespace base - namespace util { template @@ -166,7 +147,7 @@ class CopyableAutoPtr : public AutoPtr { public: explicit CopyableAutoPtr(pointer_type p = 0) : AutoPtr(p) {} - explicit CopyableAutoPtr(const CopyableAutoPtr& rhs) { this->reset(rhs.clone()); } + CopyableAutoPtr(const CopyableAutoPtr& rhs) { this->reset(rhs.clone()); } CopyableAutoPtr& operator=(const CopyableAutoPtr& rhs) { if (this != &rhs) this->reset(rhs.clone()); return *this; @@ -871,10 +852,10 @@ enum INTERFACE_ID_TYPE { AGORA_IID_MEDIA_ENGINE_REGULATOR = 9, AGORA_IID_CLOUD_SPATIAL_AUDIO = 10, AGORA_IID_LOCAL_SPATIAL_AUDIO = 11, - AGORA_IID_MEDIA_RECORDER = 12, AGORA_IID_STATE_SYNC = 13, AGORA_IID_METACHAT_SERVICE = 14, AGORA_IID_MUSIC_CONTENT_CENTER = 15, + AGORA_IID_H265_TRANSCODER = 16, }; /** @@ -996,11 +977,11 @@ enum FRAME_RATE { }; enum FRAME_WIDTH { - FRAME_WIDTH_640 = 640, + FRAME_WIDTH_960 = 960, }; enum FRAME_HEIGHT { - FRAME_HEIGHT_360 = 360, + FRAME_HEIGHT_540 = 540, }; @@ -1029,7 +1010,7 @@ enum ORIENTATION_MODE { /** * 0: The output video always follows the orientation of the captured video. The receiver takes * the rotational information passed on from the video encoder. This mode applies to scenarios - * where video orientation can be adjusted on the receiver： + * where video orientation can be adjusted on the receiver: * - If the captured video is in landscape mode, the output video is in landscape mode. * - If the captured video is in portrait mode, the output video is in portrait mode. */ @@ -1130,6 +1111,15 @@ const int DEFAULT_MIN_BITRATE = -1; */ const int DEFAULT_MIN_BITRATE_EQUAL_TO_TARGET_BITRATE = -2; +/** + * screen sharing supported capability level. + */ +enum SCREEN_CAPTURE_FRAMERATE_CAPABILITY { + SCREEN_CAPTURE_FRAMERATE_CAPABILITY_15_FPS = 0, + SCREEN_CAPTURE_FRAMERATE_CAPABILITY_30_FPS = 1, + SCREEN_CAPTURE_FRAMERATE_CAPABILITY_60_FPS = 2, +}; + /** * The video codec types. */ @@ -1627,7 +1617,7 @@ struct EncodedVideoFrameInfo { }; /** -* Video Compression Preference. +* Video compression preference. */ enum COMPRESSION_PREFERENCE { /** @@ -1662,15 +1652,30 @@ enum ENCODING_PREFERENCE { * The definition of the AdvanceOptions struct. */ struct AdvanceOptions { + /** * The video encoder type preference.. */ ENCODING_PREFERENCE encodingPreference; - AdvanceOptions() : encodingPreference(PREFER_AUTO) {} - AdvanceOptions(ENCODING_PREFERENCE encoding_preference) : encodingPreference(encoding_preference) {} + + /** + * Video compression preference. + */ + COMPRESSION_PREFERENCE compressionPreference; + + AdvanceOptions() : encodingPreference(PREFER_AUTO), + compressionPreference(PREFER_LOW_LATENCY) {} + + AdvanceOptions(ENCODING_PREFERENCE encoding_preference, + COMPRESSION_PREFERENCE compression_preference) : + encodingPreference(encoding_preference), + compressionPreference(compression_preference) {} + bool operator==(const AdvanceOptions& rhs) const { - return encodingPreference == rhs.encodingPreference; + return encodingPreference == rhs.encodingPreference && + compressionPreference == rhs.compressionPreference; } + }; /** @@ -1678,7 +1683,7 @@ struct AdvanceOptions { */ enum VIDEO_MIRROR_MODE_TYPE { /** - * (Default) 0: The mirror mode determined by the SDK. + * 0: The mirror mode determined by the SDK. */ VIDEO_MIRROR_MODE_AUTO = 0, /** @@ -1691,6 +1696,33 @@ enum VIDEO_MIRROR_MODE_TYPE { VIDEO_MIRROR_MODE_DISABLED = 2, }; + +/** Supported codec type bit mask. */ +enum CODEC_CAP_MASK { + /** 0: No codec support. */ + CODEC_CAP_MASK_NONE = 0, + + /** bit 1: Hardware decoder support flag. */ + CODEC_CAP_MASK_HW_DEC = 1 << 0, + + /** bit 2: Hardware encoder support flag. */ + CODEC_CAP_MASK_HW_ENC = 1 << 1, + + /** bit 3: Software decoder support flag. */ + CODEC_CAP_MASK_SW_DEC = 1 << 2, + + /** bit 4: Software encoder support flag. */ + CODEC_CAP_MASK_SW_ENC = 1 << 3, +}; + +/** The codec support information. */ +struct CodecCapInfo { + /** The codec type: #VIDEO_CODEC_TYPE. */ + VIDEO_CODEC_TYPE codecType; + /** The codec support flag. */ + int codecCapMask; +}; + /** * The definition of the VideoEncoderConfiguration struct. */ @@ -1728,38 +1760,40 @@ struct VideoEncoderConfiguration { * * | Resolution | Frame Rate (fps) | Base Bitrate (Kbps) | Live Bitrate (Kbps)| * |------------------------|------------------|---------------------|--------------------| - * | 160 * 120 | 15 | 65 | 130 | - * | 120 * 120 | 15 | 50 | 100 | - * | 320 * 180 | 15 | 140 | 280 | - * | 180 * 180 | 15 | 100 | 200 | - * | 240 * 180 | 15 | 120 | 240 | - * | 320 * 240 | 15 | 200 | 400 | - * | 240 * 240 | 15 | 140 | 280 | - * | 424 * 240 | 15 | 220 | 440 | - * | 640 * 360 | 15 | 400 | 800 | - * | 360 * 360 | 15 | 260 | 520 | - * | 640 * 360 | 30 | 600 | 1200 | - * | 360 * 360 | 30 | 400 | 800 | - * | 480 * 360 | 15 | 320 | 640 | - * | 480 * 360 | 30 | 490 | 980 | - * | 640 * 480 | 15 | 500 | 1000 | - * | 480 * 480 | 15 | 400 | 800 | - * | 640 * 480 | 30 | 750 | 1500 | - * | 480 * 480 | 30 | 600 | 1200 | - * | 848 * 480 | 15 | 610 | 1220 | - * | 848 * 480 | 30 | 930 | 1860 | - * | 640 * 480 | 10 | 400 | 800 | - * | 1280 * 720 | 15 | 1130 | 2260 | - * | 1280 * 720 | 30 | 1710 | 3420 | - * | 960 * 720 | 15 | 910 | 1820 | - * | 960 * 720 | 30 | 1380 | 2760 | - * | 1920 * 1080 | 15 | 2080 | 4160 | - * | 1920 * 1080 | 30 | 3150 | 6300 | - * | 1920 * 1080 | 60 | 4780 | 6500 | - * | 2560 * 1440 | 30 | 4850 | 6500 | - * | 2560 * 1440 | 60 | 6500 | 6500 | - * | 3840 * 2160 | 30 | 6500 | 6500 | - * | 3840 * 2160 | 60 | 6500 | 6500 | + * | 160 * 120 | 15 | 65 | 110 | + * | 120 * 120 | 15 | 50 | 90 | + * | 320 * 180 | 15 | 140 | 240 | + * | 180 * 180 | 15 | 100 | 160 | + * | 240 * 180 | 15 | 120 | 200 | + * | 320 * 240 | 15 | 200 | 300 | + * | 240 * 240 | 15 | 140 | 240 | + * | 424 * 240 | 15 | 220 | 370 | + * | 640 * 360 | 15 | 400 | 680 | + * | 360 * 360 | 15 | 260 | 440 | + * | 640 * 360 | 30 | 600 | 1030 | + * | 360 * 360 | 30 | 400 | 670 | + * | 480 * 360 | 15 | 320 | 550 | + * | 480 * 360 | 30 | 490 | 830 | + * | 640 * 480 | 15 | 500 | 750 | + * | 480 * 480 | 15 | 400 | 680 | + * | 640 * 480 | 30 | 750 | 1130 | + * | 480 * 480 | 30 | 600 | 1030 | + * | 848 * 480 | 15 | 610 | 920 | + * | 848 * 480 | 30 | 930 | 1400 | + * | 640 * 480 | 10 | 400 | 600 | + * | 960 * 540 | 15 | 750 | 1100 | + * | 960 * 540 | 30 | 1110 | 1670 | + * | 1280 * 720 | 15 | 1130 | 1600 | + * | 1280 * 720 | 30 | 1710 | 2400 | + * | 960 * 720 | 15 | 910 | 1280 | + * | 960 * 720 | 30 | 1380 | 2000 | + * | 1920 * 1080 | 15 | 2080 | 2500 | + * | 1920 * 1080 | 30 | 3150 | 3780 | + * | 1920 * 1080 | 60 | 4780 | 5730 | + * | 2560 * 1440 | 30 | 4850 | 4850 | + * | 2560 * 1440 | 60 | 7350 | 7350 | + * | 3840 * 2160 | 30 | 8910 | 8910 | + * | 3840 * 2160 | 60 | 13500 | 13500 | */ int bitrate; @@ -1789,20 +1823,17 @@ struct VideoEncoderConfiguration { DEGRADATION_PREFERENCE degradationPreference; /** + * The mirror mode is disabled by default * If mirror_type is set to VIDEO_MIRROR_MODE_ENABLED, then the video frame would be mirrored before encoding. */ VIDEO_MIRROR_MODE_TYPE mirrorMode; - /** - * The video compressionPreference: #compressionPreference. - */ - COMPRESSION_PREFERENCE compressionPreference; /** - * The video encoder hw: #.hardwareEncoding + * The advanced options for the video encoder configuration. See AdvanceOptions. */ AdvanceOptions advanceOptions; - VideoEncoderConfiguration(const VideoDimensions& d, int f, int b, ORIENTATION_MODE m, VIDEO_MIRROR_MODE_TYPE mirror = VIDEO_MIRROR_MODE_DISABLED, COMPRESSION_PREFERENCE compressionPreference = PREFER_LOW_LATENCY) + VideoEncoderConfiguration(const VideoDimensions& d, int f, int b, ORIENTATION_MODE m, VIDEO_MIRROR_MODE_TYPE mirror = VIDEO_MIRROR_MODE_DISABLED) : codecType(VIDEO_CODEC_H264), dimensions(d), frameRate(f), @@ -1811,9 +1842,8 @@ struct VideoEncoderConfiguration { orientationMode(m), degradationPreference(MAINTAIN_QUALITY), mirrorMode(mirror), - compressionPreference(compressionPreference), - advanceOptions(PREFER_AUTO) {} - VideoEncoderConfiguration(int width, int height, int f, int b, ORIENTATION_MODE m, VIDEO_MIRROR_MODE_TYPE mirror = VIDEO_MIRROR_MODE_DISABLED, COMPRESSION_PREFERENCE compressionPreference = PREFER_LOW_LATENCY) + advanceOptions(PREFER_AUTO, PREFER_LOW_LATENCY) {} + VideoEncoderConfiguration(int width, int height, int f, int b, ORIENTATION_MODE m, VIDEO_MIRROR_MODE_TYPE mirror = VIDEO_MIRROR_MODE_DISABLED) : codecType(VIDEO_CODEC_H264), dimensions(width, height), frameRate(f), @@ -1822,8 +1852,7 @@ struct VideoEncoderConfiguration { orientationMode(m), degradationPreference(MAINTAIN_QUALITY), mirrorMode(mirror), - compressionPreference(compressionPreference), - advanceOptions(PREFER_AUTO) {} + advanceOptions(PREFER_AUTO, PREFER_LOW_LATENCY) {} VideoEncoderConfiguration(const VideoEncoderConfiguration& config) : codecType(config.codecType), dimensions(config.dimensions), @@ -1833,19 +1862,17 @@ struct VideoEncoderConfiguration { orientationMode(config.orientationMode), degradationPreference(config.degradationPreference), mirrorMode(config.mirrorMode), - compressionPreference(config.compressionPreference), advanceOptions(config.advanceOptions) {} VideoEncoderConfiguration() : codecType(VIDEO_CODEC_H264), - dimensions(FRAME_WIDTH_640, FRAME_HEIGHT_360), + dimensions(FRAME_WIDTH_960, FRAME_HEIGHT_540), frameRate(FRAME_RATE_FPS_15), bitrate(STANDARD_BITRATE), minBitrate(DEFAULT_MIN_BITRATE), orientationMode(ORIENTATION_MODE_ADAPTIVE), degradationPreference(MAINTAIN_QUALITY), mirrorMode(VIDEO_MIRROR_MODE_DISABLED), - compressionPreference(PREFER_LOW_LATENCY), - advanceOptions(PREFER_AUTO) {} + advanceOptions(PREFER_AUTO, PREFER_LOW_LATENCY) {} VideoEncoderConfiguration& operator=(const VideoEncoderConfiguration& rhs) { if (this == &rhs) return *this; @@ -1857,7 +1884,6 @@ struct VideoEncoderConfiguration { orientationMode = rhs.orientationMode; degradationPreference = rhs.degradationPreference; mirrorMode = rhs.mirrorMode; - compressionPreference = rhs.compressionPreference; advanceOptions = rhs.advanceOptions; return *this; } @@ -2220,68 +2246,6 @@ struct RtcStats { rxPacketLossRate(0) {} }; -/** - * The capture type of the custom video source. - */ -enum VIDEO_SOURCE_TYPE { - /** - * 0: The primary camera. - */ - VIDEO_SOURCE_CAMERA_PRIMARY = 0, - /** - * The camera. - */ - VIDEO_SOURCE_CAMERA = VIDEO_SOURCE_CAMERA_PRIMARY, - /** - * 1: The secondary camera. - */ - VIDEO_SOURCE_CAMERA_SECONDARY = 1, - /** - * 2: The primary screen. - */ - VIDEO_SOURCE_SCREEN_PRIMARY = 2, - /** - * The screen. - */ - VIDEO_SOURCE_SCREEN = VIDEO_SOURCE_SCREEN_PRIMARY, - /** - * 3: The secondary screen. - */ - VIDEO_SOURCE_SCREEN_SECONDARY = 3, - /** - * 4: The custom video source. - */ - VIDEO_SOURCE_CUSTOM = 4, - /** - * 5: The video source from the media player. - */ - VIDEO_SOURCE_MEDIA_PLAYER = 5, - /** - * 6: The video source is a PNG image. - */ - VIDEO_SOURCE_RTC_IMAGE_PNG = 6, - /** - * 7: The video source is a JPEG image. - */ - VIDEO_SOURCE_RTC_IMAGE_JPEG = 7, - /** - * 8: The video source is a GIF image. - */ - VIDEO_SOURCE_RTC_IMAGE_GIF = 8, - /** - * 9: The video source is remote video acquired by the network. - */ - VIDEO_SOURCE_REMOTE = 9, - /** - * 10: A transcoded video source. - */ - VIDEO_SOURCE_TRANSCODED = 10, - /** - * 100: An unknown video source. - */ - VIDEO_SOURCE_UNKNOWN = 100 -}; - /** * User role types. */ @@ -2382,108 +2346,21 @@ enum EXPERIENCE_POOR_REASON { }; /** - * Audio statistics of the remote user. + * Audio AINS mode */ -struct RemoteAudioStats -{ - /** - * User ID of the remote user sending the audio stream. - */ - uid_t uid; - /** - * The quality of the remote audio: #QUALITY_TYPE. - */ - int quality; - /** - * The network delay (ms) from the sender to the receiver. - */ - int networkTransportDelay; - /** - * The network delay (ms) from the receiver to the jitter buffer. - * @note When the receiving end is an audience member and `audienceLatencyLevel` of `ClientRoleOptions` - * is 1, this parameter does not take effect. - */ - int jitterBufferDelay; - /** - * The audio frame loss rate in the reported interval. - */ - int audioLossRate; - /** - * The number of channels. - */ - int numChannels; - /** - * The sample rate (Hz) of the remote audio stream in the reported interval. - */ - int receivedSampleRate; - /** - * The average bitrate (Kbps) of the remote audio stream in the reported - * interval. - */ - int receivedBitrate; - /** - * The total freeze time (ms) of the remote audio stream after the remote - * user joins the channel. - * - * In a session, audio freeze occurs when the audio frame loss rate reaches 4%. - */ - int totalFrozenTime; - /** - * The total audio freeze time as a percentage (%) of the total time when the - * audio is available. - */ - int frozenRate; - /** - * The quality of the remote audio stream as determined by the Agora - * real-time audio MOS (Mean Opinion Score) measurement method in the - * reported interval. The return value ranges from 0 to 500. Dividing the - * return value by 100 gets the MOS score, which ranges from 0 to 5. The - * higher the score, the better the audio quality. - * - * | MOS score | Perception of audio quality | - * |-----------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| - * | Greater than 4 | Excellent. The audio sounds clear and smooth. | - * | From 3.5 to 4 | Good. The audio has some perceptible impairment, but still sounds clear. | - * | From 3 to 3.5 | Fair. The audio freezes occasionally and requires attentive listening. | - * | From 2.5 to 3 | Poor. The audio sounds choppy and requires considerable effort to understand. | - * | From 2 to 2.5 | Bad. The audio has occasional noise. Consecutive audio dropouts occur, resulting in some information loss. The users can communicate only with difficulty. | - * | Less than 2 | Very bad. The audio has persistent noise. Consecutive audio dropouts are frequent, resulting in severe information loss. Communication is nearly impossible. | - */ - int mosValue; - /** - * The total time (ms) when the remote user neither stops sending the audio - * stream nor disables the audio module after joining the channel. - */ - int totalActiveTime; - /** - * The total publish duration (ms) of the remote audio stream. - */ - int publishDuration; - /** - * Quality of experience (QoE) of the local user when receiving a remote audio stream. See #EXPERIENCE_QUALITY_TYPE. - */ - int qoeQuality; - /** - * The reason for poor QoE of the local user when receiving a remote audio stream. See #EXPERIENCE_POOR_REASON. - */ - int qualityChangedReason; - - RemoteAudioStats() : - uid(0), - quality(0), - networkTransportDelay(0), - jitterBufferDelay(0), - audioLossRate(0), - numChannels(0), - receivedSampleRate(0), - receivedBitrate(0), - totalFrozenTime(0), - frozenRate(0), - mosValue(0), - totalActiveTime(0), - publishDuration(0), - qoeQuality(0), - qualityChangedReason(0) {} +enum AUDIO_AINS_MODE { + /** + * AINS mode with soft suppression level. + */ + AINS_MODE_BALANCED = 0, + /** + * AINS mode with high suppression level. + */ + AINS_MODE_AGGRESSIVE = 1, + /** + * AINS mode with high suppression level and ultra-low-latency + */ + AINS_MODE_ULTRALOWLATENCY = 2 }; /** @@ -2592,7 +2469,7 @@ struct VideoFormat { * The video frame rate (fps). */ int fps; - VideoFormat() : width(FRAME_WIDTH_640), height(FRAME_HEIGHT_360), fps(FRAME_RATE_FPS_15) {} + VideoFormat() : width(FRAME_WIDTH_960), height(FRAME_HEIGHT_540), fps(FRAME_RATE_FPS_15) {} VideoFormat(int w, int h, int f) : width(w), height(h), fps(f) {} bool operator<(const VideoFormat& fmt) const { @@ -2662,6 +2539,21 @@ enum SCREEN_SCENARIO_TYPE { SCREEN_SCENARIO_RDC = 4, }; + +/** + * The video application scenario type. + */ +enum VIDEO_APPLICATION_SCENARIO_TYPE { + /** + * 0: Default Scenario. + */ + APPLICATION_SCENARIO_GENERAL = 0, + /** + * 1: Meeting Scenario. This scenario is the best QoE practice of meeting application. + */ + APPLICATION_SCENARIO_MEETING = 1, +}; + /** * The brightness level of the video image captured by the local camera. */ @@ -2803,7 +2695,7 @@ enum LOCAL_VIDEO_STREAM_ERROR { */ LOCAL_VIDEO_STREAM_ERROR_CAPTURE_FAILURE = 4, /** - * 5: The local video encoding fails. + * 5: The local video encoder is not supported. */ LOCAL_VIDEO_STREAM_ERROR_ENCODE_FAILURE = 5, /** @@ -3028,6 +2920,10 @@ enum REMOTE_VIDEO_STATE_REASON { */ REMOTE_VIDEO_STATE_REASON_SDK_IN_BACKGROUND = 12, + /** 13: The remote video stream is not supported by the decoder + */ + REMOTE_VIDEO_STATE_REASON_CODEC_NOT_SUPPORT = 13, + }; /** @@ -3761,7 +3657,7 @@ struct TranscodingVideoStream { /** * The source type of video for the video mixing on the local client. See #VIDEO_SOURCE_TYPE. */ - agora::media::MEDIA_SOURCE_TYPE sourceType; + VIDEO_SOURCE_TYPE sourceType; /** * The ID of the remote user. * @note Use this parameter only when the source type of the video for the video mixing on the local client is `VIDEO_SOURCE_REMOTE`. @@ -3772,6 +3668,10 @@ struct TranscodingVideoStream { * @note Use this parameter only when the source type of the video for the video mixing on the local client is `RTC_IMAGE`. */ const char* imageUrl; + /** + * MediaPlayer id if sourceType is MEDIA_PLAYER_SOURCE. + */ + int mediaPlayerId; /** * The horizontal displacement of the top-left corner of the video for the video mixing on the client relative to the top-left corner (origin) of the canvas for this video mixing. */ @@ -3807,7 +3707,7 @@ struct TranscodingVideoStream { bool mirror; TranscodingVideoStream() - : sourceType(agora::media::PRIMARY_CAMERA_SOURCE), + : sourceType(VIDEO_SOURCE_CAMERA_PRIMARY), remoteUserUid(0), imageUrl(NULL), x(0), @@ -3831,7 +3731,7 @@ struct LocalTranscoderConfiguration { /** * The video streams for the video mixing on the local client. See TranscodingVideoStream. */ - TranscodingVideoStream* VideoInputStreams; + TranscodingVideoStream* videoInputStreams; /** * The encoding configuration of the mixed video stream after the video mixing on the local client. See VideoEncoderConfiguration. */ @@ -3845,11 +3745,42 @@ struct LocalTranscoderConfiguration { LocalTranscoderConfiguration() : streamCount(0), - VideoInputStreams(NULL), + videoInputStreams(NULL), videoOutputConfiguration(), syncWithPrimaryCamera(true) {} }; +enum VIDEO_TRANSCODER_ERROR { + /** + * No error + */ + VT_ERR_OK = 0, + /** + * The video track of the video source is not started. + */ + VT_ERR_VIDEO_SOURCE_NOT_READY = 1, + /** + * The video source type is not supported. + */ + VT_ERR_INVALID_VIDEO_SOURCE_TYPE = 2, + /** + * The image url is not correctly of image source. + */ + VT_ERR_INVALID_IMAGE_PATH = 3, + /** + * The image format not the type png/jpeg/gif of image source. + */ + VT_ERR_UNSUPPORT_IMAGE_FORMAT = 4, + /** + * The layout is invalid such as width is zero. + */ + VT_ERR_INVALID_LAYOUT = 5, + /** + * Internal error. + */ + VT_ERR_INTERNAL = 20 +}; + /** * Configurations of the last-mile network test. */ @@ -4039,9 +3970,9 @@ enum CONNECTION_CHANGED_REASON_TYPE CONNECTION_CHANGED_TOO_MANY_BROADCASTERS = 20, /** - * 21: The connection is failed due to license verify failed. + * 21: The connection is failed due to license validation failure. */ - CONNECTION_CHANGED_LICENSE_VERIFY_FAILED = 21, + CONNECTION_CHANGED_LICENSE_VALIDATION_FAILURE = 21, }; /** @@ -4222,17 +4153,24 @@ struct VideoCanvas { */ Rectangle cropArea; + /** + * Whether to apply alpha mask to the video frame if exsit: + * true: Apply alpha mask to video frame. + * false: (Default) Do not apply alpha mask to video frame. + */ + bool enableAlphaMask; + VideoCanvas() : view(NULL), uid(0), renderMode(media::base::RENDER_MODE_HIDDEN), mirrorMode(VIDEO_MIRROR_MODE_AUTO), - setupMode(VIDEO_VIEW_SETUP_REPLACE), sourceType(VIDEO_SOURCE_CAMERA_PRIMARY), mediaPlayerId(-ERR_NOT_READY), cropArea(0, 0, 0, 0) {} + setupMode(VIDEO_VIEW_SETUP_REPLACE), sourceType(VIDEO_SOURCE_CAMERA_PRIMARY), mediaPlayerId(-ERR_NOT_READY), cropArea(0, 0, 0, 0), enableAlphaMask(false) {} VideoCanvas(view_t v, media::base::RENDER_MODE_TYPE m, VIDEO_MIRROR_MODE_TYPE mt, uid_t u) : view(v), uid(u), renderMode(m), mirrorMode(mt), setupMode(VIDEO_VIEW_SETUP_REPLACE), - sourceType(VIDEO_SOURCE_CAMERA_PRIMARY), mediaPlayerId(-ERR_NOT_READY), cropArea(0, 0, 0, 0) {} + sourceType(VIDEO_SOURCE_CAMERA_PRIMARY), mediaPlayerId(-ERR_NOT_READY), cropArea(0, 0, 0, 0), enableAlphaMask(false) {} VideoCanvas(view_t v, media::base::RENDER_MODE_TYPE m, VIDEO_MIRROR_MODE_TYPE mt, user_id_t) : view(v), uid(0), renderMode(m), mirrorMode(mt), setupMode(VIDEO_VIEW_SETUP_REPLACE), - sourceType(VIDEO_SOURCE_CAMERA_PRIMARY), mediaPlayerId(-ERR_NOT_READY), cropArea(0, 0, 0, 0) {} + sourceType(VIDEO_SOURCE_CAMERA_PRIMARY), mediaPlayerId(-ERR_NOT_READY), cropArea(0, 0, 0, 0), enableAlphaMask(false) {} }; /** Image enhancement options. @@ -4381,22 +4319,32 @@ struct ColorEnhanceOptions { * The custom background image. */ struct VirtualBackgroundSource { - /** The type of the custom background image. + /** The type of the custom background source. */ enum BACKGROUND_SOURCE_TYPE { /** - * 1: (Default) The background image is a solid color. + * 0: Enable segementation with the captured video frame without replacing the background. + */ + BACKGROUND_NONE = 0, + /** + * 1: (Default) The background source is a solid color. */ BACKGROUND_COLOR = 1, /** - * The background image is a file in PNG or JPG format. + * The background source is a file in PNG or JPG format. */ BACKGROUND_IMG = 2, - /** The background image is the blurred background. */ + /** + * The background source is the blurred original video frame. + * */ BACKGROUND_BLUR = 3, + /** + * The background source is a file in MP4, AVI, MKV, FLV format. + * */ + BACKGROUND_VIDEO = 4, }; - /** The degree of blurring applied to the custom background image.. + /** The degree of blurring applied to the background source. */ enum BACKGROUND_BLUR_DEGREE { /** 1: The degree of blurring applied to the custom background image is low. The user can almost see the background clearly. */ @@ -4453,6 +4401,42 @@ struct SegmentationProperty { SegmentationProperty() : modelType(SEG_MODEL_AI), greenCapacity(0.5){} }; +/** The type of custom audio track +*/ +enum AUDIO_TRACK_TYPE { + /** + * -1: Invalid audio track + */ + AUDIO_TRACK_INVALID = -1, + /** + * 0: Mixable audio track + * You can push more than one mixable Audio tracks into one RTC connection(channel id + uid), + * and SDK will mix these tracks into one audio track automatically. + * However, compare to direct audio track, mixable track might cause extra 30ms+ delay. + */ + AUDIO_TRACK_MIXABLE = 0, + /** + * 1: Direct audio track + * You can only push one direct (non-mixable) audio track into one RTC connection(channel id + uid). + * Compare to mixable stream, you can have lower lantency using direct audio track. + */ + AUDIO_TRACK_DIRECT = 1, +}; + +/** The configuration of custom audio track +*/ +struct AudioTrackConfig { + /** + * Enable local playback, enabled by default + * true: (Default) Enable local playback + * false: Do not enable local playback + */ + bool enableLocalPlayback; + + AudioTrackConfig() + : enableLocalPlayback(true) {} +}; + /** * Preset local voice reverberation options. * bitmap allocation: @@ -4692,7 +4676,41 @@ enum VOICE_CONVERSION_PRESET { VOICE_CHANGER_SOLID = 0x03010300, /** A deep voice. To avoid audio distortion, ensure that you use this enumerator to process a male-sounding voice. */ - VOICE_CHANGER_BASS = 0x03010400 + VOICE_CHANGER_BASS = 0x03010400, + /** A voice like a cartoon character. + */ + VOICE_CHANGER_CARTOON = 0x03010500, + /** A voice like a child. + */ + VOICE_CHANGER_CHILDLIKE = 0x03010600, + /** A voice like a phone operator. + */ + VOICE_CHANGER_PHONE_OPERATOR = 0x03010700, + /** A monster voice. + */ + VOICE_CHANGER_MONSTER = 0x03010800, + /** A voice like Transformers. + */ + VOICE_CHANGER_TRANSFORMERS = 0x03010900, + /** A voice like Groot. + */ + VOICE_CHANGER_GROOT = 0x03010A00, + /** A voice like Darth Vader. + */ + VOICE_CHANGER_DARTH_VADER = 0x03010B00, + /** A rough female voice. + */ + VOICE_CHANGER_IRON_LADY = 0x03010C00, + /** A voice like Crayon Shin-chan. + */ + VOICE_CHANGER_SHIN_CHAN = 0x03010D00, + /** A voice like a castrato. + */ + VOICE_CHANGER_GIRLISH_MAN = 0x03010E00, + /** A voice like chipmunk. + */ + VOICE_CHANGER_CHIPMUNK = 0x03010F00, + }; /** The options for SDK preset headphone equalizer. @@ -4959,7 +4977,7 @@ class IAudioEncodedFrameObserver { * @param length The data length (byte) of the audio frame. * @param audioEncodedFrameInfo Audio information after encoding. For details, see `EncodedAudioFrameInfo`. */ -virtual void OnRecordAudioEncodedFrame(const uint8_t* frameBuffer, int length, const EncodedAudioFrameInfo& audioEncodedFrameInfo) = 0; +virtual void onRecordAudioEncodedFrame(const uint8_t* frameBuffer, int length, const EncodedAudioFrameInfo& audioEncodedFrameInfo) = 0; /** * Gets the encoded audio data of all remote users. @@ -4971,7 +4989,7 @@ virtual void OnRecordAudioEncodedFrame(const uint8_t* frameBuffer, int length, * @param length The data length (byte) of the audio frame. * @param audioEncodedFrameInfo Audio information after encoding. For details, see `EncodedAudioFrameInfo`. */ -virtual void OnPlaybackAudioEncodedFrame(const uint8_t* frameBuffer, int length, const EncodedAudioFrameInfo& audioEncodedFrameInfo) = 0; +virtual void onPlaybackAudioEncodedFrame(const uint8_t* frameBuffer, int length, const EncodedAudioFrameInfo& audioEncodedFrameInfo) = 0; /** * Gets the mixed and encoded audio data of the local and all remote users. @@ -4983,7 +5001,7 @@ virtual void OnPlaybackAudioEncodedFrame(const uint8_t* frameBuffer, int length * @param length The data length (byte) of the audio frame. * @param audioEncodedFrameInfo Audio information after encoding. For details, see `EncodedAudioFrameInfo`. */ -virtual void OnMixedAudioEncodedFrame(const uint8_t* frameBuffer, int length, const EncodedAudioFrameInfo& audioEncodedFrameInfo) = 0; +virtual void onMixedAudioEncodedFrame(const uint8_t* frameBuffer, int length, const EncodedAudioFrameInfo& audioEncodedFrameInfo) = 0; virtual ~IAudioEncodedFrameObserver () {} }; @@ -5559,12 +5577,13 @@ struct EchoTestConfiguration { bool enableVideo; const char* token; const char* channelId; + int intervalInSeconds; - EchoTestConfiguration(view_t v, bool ea, bool ev, const char* t, const char* c) - : view(v), enableAudio(ea), enableVideo(ev), token(t), channelId(c) {} + EchoTestConfiguration(view_t v, bool ea, bool ev, const char* t, const char* c, const int is) + : view(v), enableAudio(ea), enableVideo(ev), token(t), channelId(c), intervalInSeconds(is) {} EchoTestConfiguration() - : view(OPTIONAL_NULLPTR), enableAudio(true), enableVideo(true), token(OPTIONAL_NULLPTR), channelId(OPTIONAL_NULLPTR) {} + : view(OPTIONAL_NULLPTR), enableAudio(true), enableVideo(true), token(OPTIONAL_NULLPTR), channelId(OPTIONAL_NULLPTR), intervalInSeconds(2) {} }; /** @@ -5740,6 +5759,101 @@ struct ScreenCaptureParameters2 { }; #endif +/** + * The tracing event of media rendering. + */ +enum MEDIA_TRACE_EVENT { + /** + * 0: The media frame has been rendered. + */ + MEDIA_TRACE_EVENT_VIDEO_RENDERED = 0, + /** + * 1: The media frame has been decoded. + */ + MEDIA_TRACE_EVENT_VIDEO_DECODED, +}; + +/** + * The video rendering tracing result + */ +struct VideoRenderingTracingInfo { + /** + * Elapsed time from the start tracing time to the time when the tracing event occurred. + */ + int elapsedTime; + /** + * Elapsed time from the start tracing time to the time when join channel. + * + * **Note** + * If the start tracing time is behind the time when join channel, this value will be negative. + */ + int start2JoinChannel; + /** + * Elapsed time from joining channel to finishing joining channel. + */ + int join2JoinSuccess; + /** + * Elapsed time from finishing joining channel to remote user joined. + * + * **Note** + * If the start tracing time is after the time finishing join channel, this value will be + * the elapsed time from the start tracing time to remote user joined. The minimum value is 0. + */ + int joinSuccess2RemoteJoined; + /** + * Elapsed time from remote user joined to set the view. + * + * **Note** + * If the start tracing time is after the time when remote user joined, this value will be + * the elapsed time from the start tracing time to set the view. The minimum value is 0. + */ + int remoteJoined2SetView; + /** + * Elapsed time from remote user joined to the time subscribing remote video stream. + * + * **Note** + * If the start tracing time is after the time when remote user joined, this value will be + * the elapsed time from the start tracing time to the time subscribing remote video stream. + * The minimum value is 0. + */ + int remoteJoined2UnmuteVideo; + /** + * Elapsed time from remote user joined to the remote video packet received. + * + * **Note** + * If the start tracing time is after the time when remote user joined, this value will be + * the elapsed time from the start tracing time to the time subscribing remote video stream. + * The minimum value is 0. + */ + int remoteJoined2PacketReceived; +}; + +enum CONFIG_FETCH_TYPE { + /** + * 1: Fetch config when initializing RtcEngine, without channel info. + */ + CONFIG_FETCH_TYPE_INITIALIZE = 1, + /** + * 2: Fetch config when joining channel with channel info, such as channel name and uid. + */ + CONFIG_FETCH_TYPE_JOIN_CHANNEL = 2, +}; + +/** + * media recorder source stream information + */ +struct RecorderStreamInfo { + /** + * The channel ID of the video track. + */ + const char* channelId; + /** + * The user ID. + */ + uid_t uid; + RecorderStreamInfo() : channelId(NULL), uid(0) {} +}; + } // namespace rtc namespace base { diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/AgoraMediaBase.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/AgoraMediaBase.h index ce4931599..074d02efc 100644 --- a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/AgoraMediaBase.h +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/AgoraMediaBase.h @@ -31,12 +31,70 @@ typedef unsigned int track_id_t; typedef unsigned int conn_id_t; typedef unsigned int video_track_id_t; +static const unsigned int INVALID_TRACK_ID = 0xffffffff; static const unsigned int DEFAULT_CONNECTION_ID = 0; static const unsigned int DUMMY_CONNECTION_ID = (std::numeric_limits::max)(); struct EncodedVideoFrameInfo; + +/** +* Video source types definition. +**/ +enum VIDEO_SOURCE_TYPE { + /** Video captured by the camera. + */ + VIDEO_SOURCE_CAMERA_PRIMARY = 0, + VIDEO_SOURCE_CAMERA = VIDEO_SOURCE_CAMERA_PRIMARY, + /** Video captured by the secondary camera. + */ + VIDEO_SOURCE_CAMERA_SECONDARY = 1, + /** Video for screen sharing. + */ + VIDEO_SOURCE_SCREEN_PRIMARY = 2, + VIDEO_SOURCE_SCREEN = VIDEO_SOURCE_SCREEN_PRIMARY, + /** Video for secondary screen sharing. + */ + VIDEO_SOURCE_SCREEN_SECONDARY = 3, + /** Not define. + */ + VIDEO_SOURCE_CUSTOM = 4, + /** Video for media player sharing. + */ + VIDEO_SOURCE_MEDIA_PLAYER = 5, + /** Video for png image. + */ + VIDEO_SOURCE_RTC_IMAGE_PNG = 6, + /** Video for png image. + */ + VIDEO_SOURCE_RTC_IMAGE_JPEG = 7, + /** Video for png image. + */ + VIDEO_SOURCE_RTC_IMAGE_GIF = 8, + /** Remote video received from network. + */ + VIDEO_SOURCE_REMOTE = 9, + /** Video for transcoded. + */ + VIDEO_SOURCE_TRANSCODED = 10, + + /** Video captured by the third camera. + */ + VIDEO_SOURCE_CAMERA_THIRD = 11, + /** Video captured by the fourth camera. + */ + VIDEO_SOURCE_CAMERA_FOURTH = 12, + /** Video for third screen sharing. + */ + VIDEO_SOURCE_SCREEN_THIRD = 13, + /** Video for fourth screen sharing. + */ + VIDEO_SOURCE_SCREEN_FOURTH = 14, + + VIDEO_SOURCE_UNKNOWN = 100 +}; + /** * Audio routes. */ @@ -385,22 +443,6 @@ enum AUDIO_DUAL_MONO_MODE { /**< ChanLout=ChanRout=(ChanLin+ChanRin)/2 */ AUDIO_DUAL_MONO_MIX = 3 }; -/** - * The audio frame observer. - */ -class IAudioFrameObserver { - public: - /** - * Occurs when each time the player receives an audio frame. - * - * After registering the audio frame observer, - * the callback occurs when each time the player receives an audio frame, - * reporting the detailed information of the audio frame. - * @param frame The detailed information of the audio frame. See {@link AudioPcmFrame}. - */ - virtual void onFrame(AudioPcmFrame* frame) = 0; - virtual ~IAudioFrameObserver() {} -}; /** * Video pixel formats. @@ -479,9 +521,9 @@ enum RENDER_MODE_TYPE { }; /** - * The video source type + * The camera video source type */ -enum VIDEO_SOURCE_TYPE { +enum CAMERA_VIDEO_SOURCE_TYPE { /** * 0: the video frame comes from the front camera */ @@ -516,7 +558,8 @@ struct ExternalVideoFrame { eglType(EGL_CONTEXT10), textureId(0), metadata_buffer(NULL), - metadata_size(0){} + metadata_size(0), + alphaBuffer(NULL){} /** * The EGL context type. @@ -632,6 +675,12 @@ struct ExternalVideoFrame { * The default value is 0 */ int metadata_size; + /** + * Indicates the output data of the portrait segmentation algorithm, which is consistent with the size of the video frame. + * The value range of each pixel is [0,255], where 0 represents the background; 255 represents the foreground (portrait). + * The default value is NULL + */ + uint8_t* alphaBuffer; }; /** @@ -655,8 +704,10 @@ struct VideoFrame { metadata_size(0), sharedContext(0), textureId(0), - alphaBuffer(NULL){} - + alphaBuffer(NULL), + pixelBuffer(NULL){ + memset(matrix, 0, sizeof(matrix)); + } /** * The video pixel format: #VIDEO_PIXEL_FORMAT. */ @@ -731,11 +782,15 @@ struct VideoFrame { */ float matrix[16]; /** - * Portrait Segmentation meta buffer, dimension of which is the same as VideoFrame. - * Pixl value is between 0-255, 0 represents totally background, 255 represents totally foreground. + * Indicates the output data of the portrait segmentation algorithm, which is consistent with the size of the video frame. + * The value range of each pixel is [0,255], where 0 represents the background; 255 represents the foreground (portrait). * The default value is NULL */ uint8_t* alphaBuffer; + /** + *The type of CVPixelBufferRef, for iOS and macOS only. + */ + void* pixelBuffer; }; /** @@ -784,6 +839,23 @@ enum VIDEO_MODULE_POSITION { } // namespace base +/** + * The audio frame observer. + */ +class IAudioPcmFrameSink { + public: + /** + * Occurs when each time the player receives an audio frame. + * + * After registering the audio frame observer, + * the callback occurs when each time the player receives an audio frame, + * reporting the detailed information of the audio frame. + * @param frame The detailed information of the audio frame. See {@link AudioPcmFrame}. + */ + virtual void onFrame(agora::media::base::AudioPcmFrame* frame) = 0; + virtual ~IAudioPcmFrameSink() {} +}; + /** * The IAudioFrameObserverBase class. */ @@ -1178,11 +1250,12 @@ class IVideoFrameObserver { * - The video data that this callback gets has not been pre-processed, such as watermarking, cropping content, rotating, or image enhancement. * * @param videoFrame A pointer to the video frame: VideoFrame + * @param sourceType source type of video frame. See #VIDEO_SOURCE_TYPE. * @return Determines whether to ignore the current video frame if the pre-processing fails: * - true: Do not ignore. * - false: Ignore, in which case this method does not sent the current video frame to the SDK. */ - virtual bool onCaptureVideoFrame(VideoFrame& videoFrame) = 0; + virtual bool onCaptureVideoFrame(agora::rtc::VIDEO_SOURCE_TYPE sourceType, VideoFrame& videoFrame) = 0; /** * Occurs each time the SDK receives a video frame before encoding. @@ -1200,78 +1273,13 @@ class IVideoFrameObserver { * - This callback does not support sending processed RGBA video data back to the SDK. * * @param videoFrame A pointer to the video frame: VideoFrame + * @param sourceType source type of video frame. See #VIDEO_SOURCE_TYPE. * @return Determines whether to ignore the current video frame if the pre-processing fails: * - true: Do not ignore. * - false: Ignore, in which case this method does not sent the current video frame to the SDK. */ - virtual bool onPreEncodeVideoFrame(VideoFrame& videoFrame) = 0; - - virtual bool onSecondaryCameraCaptureVideoFrame(VideoFrame& videoFrame) = 0; - - /** - * Gets the video data captured from the second camera before encoding. - * - * After you successfully register the video frame observer, the SDK triggers this callback each time - * when it receives a video frame. In this callback, you can get the video data before encoding. You can then - * process the data according to your particular scenarios. - * - * After processing, you can send the processed video data back to the SDK by setting the - * `videoFrame` parameter in this callback. - * - * @note - * - This callback is for Windows. - * - You need to set (1 << 2) as a frame position by `getObservedFramePosition` before you can use this callback to get the video data captured from the second screen and before encoding. - * - The video data that this callback gets has been pre-processed, such as watermarking, cropping content, rotating, or image enhancement. - * - This callback does not support sending processed RGBA video data back to the SDK. - * - * @param videoFrame A pointer to the video frame: VideoFrame - * @return Determines whether to ignore the current video frame if the pre-processing fails: - * - true: Do not ignore. - * - false: Ignore, in which case this method does not sent the current video frame to the SDK. - */ - virtual bool onSecondaryPreEncodeCameraVideoFrame(VideoFrame& videoFrame) = 0; + virtual bool onPreEncodeVideoFrame(agora::rtc::VIDEO_SOURCE_TYPE sourceType, VideoFrame& videoFrame) = 0; - /** - * Occurs each time the SDK receives a video frame captured by the screen. - * - * After you successfully register the video frame observer, the SDK triggers this callback each time - * a video frame is received. In this callback, you can get the video data captured by the screen. - * You can then pre-process the data according to your scenarios. - * - * After pre-processing, you can send the processed video data back to the SDK by setting the - * `videoFrame` parameter in this callback. - * - * @note - * - If you get the video data in RGBA color encoding format, Agora does not support using this callback to send the processed data in RGBA color encoding format back to the SDK. - * - The video data obtained through this callback has not undergone preprocessing, such as watermarking, cropping content, rotating, or image enhancement. - * - * @param videoFrame A pointer to the video frame: VideoFrame - * @return Determines whether to ignore the current video frame if the pre-processing fails: - * - true: Do not ignore. - * - false: Ignore, in which case this method does not sent the current video frame to the SDK. - */ - virtual bool onScreenCaptureVideoFrame(VideoFrame& videoFrame) = 0; - /** - * Gets the video data captured from the screen before encoding. - * - * After you successfully register the video frame observer, the SDK triggers this callback each - * time it receives a video frame. In this callback, you can get the video data captured from the - * screen before encoding and then process the data according to your particular scenarios. - * - * After processing, you can send the processed video data back to the SDK in this callback. - * - * @note - * - To get the video data captured from the second screen before encoding, you need to set - * (1 << 2) as a frame position through `getObservedFramePosition`. - * - The video data that this callback gets has been preprocessed, such as watermarking, cropping content, rotating, or image enhancement. - * - This callback does not support sending processed RGBA video data back to the SDK. - * - * @param videoFrame A pointer to the video frame: VideoFrame - * @return Determines whether to ignore the current video frame if the pre-processing fails: - * - true: Do not ignore. - * - false: Ignore, in which case this method does not sent the current video frame to the SDK. - */ - virtual bool onPreEncodeScreenVideoFrame(VideoFrame& videoFrame) = 0; /** * Occurs each time the SDK receives a video frame decoded by the MediaPlayer. * @@ -1281,38 +1289,18 @@ class IVideoFrameObserver { * * After pre-processing, you can send the processed video data back to the SDK by setting the * `videoFrame` parameter in this callback. - * - * @param videoFrame A pointer to the video frame: VideoFrame - * @param mediaPlayerId ID of the mediaPlayer. - * @return Determines whether to ignore the current video frame if the pre-processing fails: - * - true: Do not ignore. - * - false: Ignore, in which case this method does not sent the current video frame to the SDK. - */ - virtual bool onMediaPlayerVideoFrame(VideoFrame& videoFrame, int mediaPlayerId) = 0; - - virtual bool onSecondaryScreenCaptureVideoFrame(VideoFrame& videoFrame) = 0; - /** - * Gets the video data captured from the second camera before encoding. - * - * After you successfully register the video frame observer, the SDK triggers this callback each - * time it receives a video frame. In this callback, you can get the video data captured from the - * second camera before encoding and then process the data according to your particular scenarios. - * - * After processing, you can send the processed video data back to the SDK in this callback. * * @note - * - This callback is for Windows. - * - You need to set (1 << 2) as a frame position by `getObservedFramePosition` before you can - * use this callback to get the video data captured from the second screen and before encoding. - * - The video data that this callback gets has been preprocessed, such as watermarking, cropping content, rotating, or image enhancement. - * - This callback does not support sending processed RGBA video data back to the SDK. + * - This callback will not be affected by the return values of \ref getVideoFrameProcessMode "getVideoFrameProcessMode", \ref getRotationApplied "getRotationApplied", \ref getMirrorApplied "getMirrorApplied", \ref getObservedFramePosition "getObservedFramePosition". + * - On Android, this callback is not affected by the return value of \ref getVideoFormatPreference "getVideoFormatPreference" * * @param videoFrame A pointer to the video frame: VideoFrame + * @param mediaPlayerId ID of the mediaPlayer. * @return Determines whether to ignore the current video frame if the pre-processing fails: * - true: Do not ignore. * - false: Ignore, in which case this method does not sent the current video frame to the SDK. */ - virtual bool onSecondaryPreEncodeScreenVideoFrame(VideoFrame& videoFrame) = 0; + virtual bool onMediaPlayerVideoFrame(VideoFrame& videoFrame, int mediaPlayerId) = 0; /** * Occurs each time the SDK receives a video frame sent by the remote user. @@ -1584,28 +1572,32 @@ class IMediaRecorderObserver { /** * Occurs when the recording state changes. * - * @since v3.5.2 + * @since v4.0.0 * * When the local audio and video recording state changes, the SDK triggers this callback to report the current * recording state and the reason for the change. * - * @param state The current recording state. See \ref agora::rtc::RecorderState "RecorderState". - * @param error The reason for the state change. See \ref agora::rtc::RecorderErrorCode "RecorderErrorCode". + * @param channelId The channel name. + * @param uid ID of the user. + * @param state The current recording state. See \ref agora::media::RecorderState "RecorderState". + * @param error The reason for the state change. See \ref agora::media::RecorderErrorCode "RecorderErrorCode". */ - virtual void onRecorderStateChanged(RecorderState state, RecorderErrorCode error) = 0; + virtual void onRecorderStateChanged(const char* channelId, rtc::uid_t uid, RecorderState state, RecorderErrorCode error) = 0; /** * Occurs when the recording information is updated. * - * @since v3.5.2 + * @since v4.0.0 * * After you successfully register this callback and enable the local audio and video recording, the SDK periodically triggers * the `onRecorderInfoUpdated` callback based on the set value of `recorderInfoUpdateInterval`. This callback reports the * filename, duration, and size of the current recording file. * - * @param info Information for the recording file. See RecorderInfo. + * @param channelId The channel name. + * @param uid ID of the user. + * @param info Information about the recording file. See \ref agora::media::RecorderInfo "RecorderInfo". * */ - virtual void onRecorderInfoUpdated(const RecorderInfo& info) = 0; + virtual void onRecorderInfoUpdated(const char* channelId, rtc::uid_t uid, const RecorderInfo& info) = 0; virtual ~IMediaRecorderObserver() {} }; } // namespace media diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/AgoraMediaPlayerTypes.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/AgoraMediaPlayerTypes.h index 73a196959..d1bb17bb3 100644 --- a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/AgoraMediaPlayerTypes.h +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/AgoraMediaPlayerTypes.h @@ -427,6 +427,11 @@ struct MediaSource { /** * Determines whether to enable cache streaming to local files. If enable cached, the media player will * use the url or uri as the cache index. + * + * @note + * The local cache function only supports on-demand video/audio streams and does not support live streams. + * Caching video and audio files based on the HLS protocol (m3u8) to your local device is not supported. + * * - true: Enable cache. * - false: (Default) Disable cache. */ diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/IAgoraMediaPlayerSource.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/IAgoraMediaPlayerSource.h index 17a176d63..8d1a95be0 100644 --- a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/IAgoraMediaPlayerSource.h +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/IAgoraMediaPlayerSource.h @@ -52,7 +52,7 @@ class IMediaPlayerSource : public RefCountInterface { * - 0: Success. * - < 0: Failure. */ - virtual int openWithCustomSource(int64_t startPos, media::base::IMediaPlayerCustomDataProvider* provider) = 0; + virtual int openWithCustomSource(int64_t startPos, media::base::IMediaPlayerCustomDataProvider* provider) __deprecated = 0; /** * Opens a media file with a media file source. @@ -247,21 +247,21 @@ class IMediaPlayerSource : public RefCountInterface { /** * Registers the audio frame observer. * - * @param observer The pointer to the {@link media::base::IAudioFrameObserver IAudioFrameObserver} object. + * @param observer The pointer to the {@link media::IAudioPcmFrameSink observer} object. * @return * - 0: Success. * - < 0: Failure. See {@link media::base::MEDIA_PLAYER_ERROR MEDIA_PLAYER_ERROR}. */ - virtual int registerAudioFrameObserver(media::base::IAudioFrameObserver* observer) = 0; + virtual int registerAudioFrameObserver(media::IAudioPcmFrameSink* observer) = 0; /** * Releases the audio frame observer. - * @param observer The pointer to the {@link media::base::IAudioFrameObserver IAudioFrameObserver} object. + * @param observer The pointer to the {@link media::IAudioPcmFrameSink observer} object. * @return * - 0: Success. * - < 0: Failure. See {@link media::base::MEDIA_PLAYER_ERROR MEDIA_PLAYER_ERROR}. */ - virtual int unregisterAudioFrameObserver(media::base::IAudioFrameObserver* observer) = 0; + virtual int unregisterAudioFrameObserver(media::IAudioPcmFrameSink* observer) = 0; /** * Open the Agora CDN media source. diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/IAgoraRtmpStreamingService.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/IAgoraRtmpStreamingService.h index beb635ea1..59736d4ab 100644 --- a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/IAgoraRtmpStreamingService.h +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/IAgoraRtmpStreamingService.h @@ -105,8 +105,9 @@ class IRtmpStreamingService : public RefCountInterface { * @return * - 0: Success. * - < 0: Failure. - * - #ERR_INVALID_ARGUMENT (2): The RTMP URL address is NULL or has a string length of 0. - * - #ERR_NOT_INITIALIZED (7): You have not initialized the RTC engine when publishing the stream. + * - #ERR_INVALID_ARGUMENT (2): The RTMP URL address is NULL or has a string length of 0. + * - #ERR_NOT_INITIALIZED (7): You have not initialized the RTC engine when publishing the stream. + * - #ERR_ALREADY_IN_USE (19): This streaming URL is already in use. Use a new streaming URL for CDN streaming. */ virtual int startRtmpStreamWithoutTranscoding(const char* url) = 0; @@ -129,6 +130,7 @@ class IRtmpStreamingService : public RefCountInterface { * - < 0: Failure. * - #ERR_INVALID_ARGUMENT (2): The RTMP URL address is NULL or has a string length of 0. * - #ERR_NOT_INITIALIZED (7): You have not initialized the RTC engine when publishing the stream. + * - #ERR_ALREADY_IN_USE (19): This streaming URL is already in use. Use a new streaming URL for CDN streaming. */ virtual int startRtmpStreamWithTranscoding(const char* url, const LiveTranscoding& transcoding) = 0; diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/IAgoraService.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/IAgoraService.h index 9e1c0b443..c9f9dcf70 100644 --- a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/IAgoraService.h +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/IAgoraService.h @@ -10,6 +10,10 @@ #include "AgoraOptional.h" namespace agora { +class ILocalDataChannel; +class IRemoteDataChannel; +struct DataChannelConfig; + namespace rtc { class IRtcConnection; class IRtmpConnection; @@ -345,6 +349,18 @@ class IServiceObserver { * @param error {@link ERROR_CODE_TYPE} */ virtual void onAudioDeviceError(ERROR_CODE_TYPE error, const char* description) {} + /** + * Reports the config fetch result. + * + * @param code The error code of fetching config. + * - 0(ERR_OK): Success. + * - 10(ERR_TIMEDOUT): Fetching config is timed out. + * @param configType The type of fetching config. + * - 1(CONFIG_FETCH_TYPE_INITIALIZE): Fetch config when initializing RtcEngine without channel info. + * - 2(CONFIG_FETCH_TYPE_JOIN_CHANNEL): Fetch config when joining channel with channel info, such as channel name and uid. + * @param configContent The config fetched from server. + */ + virtual void onFetchConfigResult(int code, rtc::CONFIG_FETCH_TYPE configType, const char* configContent) {} }; /** @@ -528,23 +544,6 @@ class IAgoraService { virtual agora_refptr createCustomAudioTrack( agora_refptr audioSource, bool enableAec) = 0; - /** - * Creates a local audio track object with two PCM data sender and returns the pointer. - * - * Once created, this track can be used to send PCM audio data with echo cancellation. - * - * @param audioSource The pointer to the recording PCM audio data sender: \ref agora::rtc::IAudioPcmDataSender "IAudioPcmDataSender". - * @param audioReverseSource The pointer to the playout PCM audio data sender: \ref agora::rtc::IAudioPcmDataSender "IAudioPcmDataSender". - * @param audioSourceDelay The delay of recording pcm data when do echo cancellation. - * @return - * - The pointer to \ref rtc::ILocalAudioTrack "ILocalAudioTrack": Success. - * - A null pointer: Failure. - * - `INVALID_STATE`, if `enableAudioProcessor` in \ref agora::base::AgoraServiceConfiguration "AgoraServiceConfiguration" is set as `false`. - */ - virtual agora_refptr createCustomAudioTrack( - agora_refptr audioSource, - agora_refptr audioReverseSource, int audioSourceDelay) = 0; - /** * Creates a local audio track object with a audio mixer source and returns the pointer. * @@ -880,6 +879,16 @@ class IAgoraService { */ virtual agora_refptr createAudioDeviceManagerComponent( rtc::IAudioDeviceManagerObserver *observer) = 0; + + /** + * Creates an data channel and returns the pointer. + * + * @return + * - The pointer to \ref rtc::ILocalDataChannel "ILocalDataChannel": Success. + * - A null pointer: Failure. + */ + virtual agora_refptr createLocalDataChannel(const DataChannelConfig& config) = 0; + /** * @brief Get the ID of the registered extension * diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraAudioDeviceManager.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraAudioDeviceManager.h index ac9709627..243cd3f03 100644 --- a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraAudioDeviceManager.h +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraAudioDeviceManager.h @@ -105,11 +105,12 @@ class IAudioDeviceManagerObserver */ virtual void onAudioDeviceStateChanged(const char *deviceId, int deviceType, int deviceState) = 0; - /** Indicates incoming volume. This can be used to test microphone. + /** Indicates incoming volume. This can be used to test microphone or speaker. * + * @param deviceType Device type: #MEDIA_DEVICE_TYPE. * @param volume volume between 0 (lowest volume) to 255 (highest volume). */ - virtual void onVolumeIndication(int volume) = 0; + virtual void onVolumeIndication(int deviceType, int volume) = 0; /** * Occurs when the audio route changes. @@ -164,7 +165,7 @@ class IRecordingDeviceSource : public RefCountInterface { * - 0: Success. * - < 0: Failure. */ - virtual int registerAudioFrameObserver(media::base::IAudioFrameObserver* observer) = 0; + virtual int registerAudioFrameObserver(media::IAudioPcmFrameSink* observer) = 0; /** * Releases the registered IAudioFrameObserver object. @@ -174,7 +175,7 @@ class IRecordingDeviceSource : public RefCountInterface { * - 0: Success. * - < 0: Failure. */ - virtual int unregisterAudioFrameObserver(media::base::IAudioFrameObserver* observer) = 0; + virtual int unregisterAudioFrameObserver(media::IAudioPcmFrameSink* observer) = 0; /** * Set parameter to object loopback device; diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraAudioTrack.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraAudioTrack.h index 77127a797..e6adb3e75 100644 --- a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraAudioTrack.h +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraAudioTrack.h @@ -359,11 +359,15 @@ class ILocalAudioTrack : public IAudioTrack { * @param enable Whether to enable local playback: * - `true`: Enable local playback. * - `false`: Disable local playback. + * @param sync Whether to destroy local playback synchronously: + * - `true`: Destroy local playback synchronously. + * - `false`: Destroy local playback asynchronously. * @return * - 0: Success. * - < 0: Failure. */ - virtual int enableLocalPlayback(bool enable) = 0; + virtual int enableLocalPlayback(bool enable, bool sync = true) = 0; + /** * Enables in-ear monitoring (for Android and iOS only). * @@ -537,6 +541,19 @@ struct RemoteAudioTrackStats { * The MOS value */ uint32_t mos_value; + /** + * If the packet loss concealment (PLC) occurs for N consecutive times, freeze is considered as PLC occurring for M consecutive times. + * freeze cnt = (n_plc - n) / m + */ + uint32_t frozen_rate_by_custom_plc_count; + /** + * The number of audio packet loss concealment + */ + uint32_t plc_count; + /** + * Duration of inbandfec + */ + int32_t fec_decode_ms; /** * The total time (ms) when the remote user neither stops sending the audio * stream nor disables the audio module after joining the channel. @@ -596,6 +613,9 @@ struct RemoteAudioTrackStats { frozen_time_200_ms(0), delay_estimate_ms(0), mos_value(0), + frozen_rate_by_custom_plc_count(0), + plc_count(0), + fec_decode_ms(-1), total_active_time(0), publish_duration(0), e2e_delay_ms(0), @@ -684,6 +704,36 @@ class IRemoteAudioTrack : public IAudioTrack { */ virtual int setRemoteVoicePosition(float pan, float gain) = 0; + /** set percentage of audio acceleration during poor network + + @note + - The relationship between this percentage and the degree of audio acceleration is non-linear and varies with different audio material. + + @param percentage The percentage of audio acceleration. The value ranges from 0 to 100. The higher the + * percentage, the faster the acceleration. The default value is 100 (no change to the acceleration): + - 0: disable audio acceleration. + - > 0: enable audio acceleration. + @return + - 0: Success. + - < 0: Failure. + */ + virtual int adjustAudioAcceleration(int percentage) = 0; + + /** set percentage of audio deceleration during poor network + + @note + - The relationship between this percentage and the degree of audio deceleration is non-linear and varies with different audio material. + + @param percentage The percentage of audio deceleration. The value ranges from 0 to 100. The higher the + * percentage, the faster the deceleration. The default value is 100 (no change to the deceleration): + - 0: disable audio deceleration. + - > 0: enable audio deceleration. + @return + - 0: Success. + - < 0: Failure. + */ + virtual int adjustAudioDeceleration(int percentage) = 0; + /** enable spatial audio @param enabled enable/disable spatial audio: diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraCameraCapturer.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraCameraCapturer.h index 552ee5aa9..a5076aa0f 100644 --- a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraCameraCapturer.h +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraCameraCapturer.h @@ -457,14 +457,14 @@ class ICameraCaptureObserver { * * @param imageWidth The width (px) of the local video. * @param imageHeight The height (px) of the local video. - * @param vecRectangle The position and size of the human face on the local video: + * @param vecRectangle A Rectangle array of length 'numFaces', which represents the position and size of the human face on the local video： * - `x`: The x coordinate (px) of the human face in the local video. Taking the top left corner of the captured video as the origin, * the x coordinate represents the relative lateral displacement of the top left corner of the human face to the origin. * - `y`: The y coordinate (px) of the human face in the local video. Taking the top left corner of the captured video as the origin, * the y coordinate represents the relative longitudinal displacement of the top left corner of the human face to the origin. * - `width`: The width (px) of the human face in the captured video. * - `height`: The height (px) of the human face in the captured video. - * @param vecDistance The distance (cm) between the human face and the screen. + * @param vecDistance An int array of length 'numFaces', which represents distance (cm) between the human face and the screen. * @param numFaces The number of faces detected. If the value is 0, it means that no human face is detected. */ virtual void onFacePositionChanged( diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraDataChannel.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraDataChannel.h new file mode 100644 index 000000000..02b4983f8 --- /dev/null +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraDataChannel.h @@ -0,0 +1,159 @@ + +// Copyright (c) 2022 Agora.io. All rights reserved + +// This program is confidential and proprietary to Agora.io. +// And may not be copied, reproduced, modified, disclosed to others, published +// or used, in whole or in part, without the express prior written permission +// of Agora.io. +#pragma once // NOLINT(build/header_guard) + +#include "AgoraRefPtr.h" +#include "AgoraBase.h" +namespace agora { + +/** + * The definition of the DataChannelConfig struct. + */ +struct DataChannelConfig { + /** + * Whether this channel should sync with media stream: + * - true: sync . + * - false: not sync. + */ + bool syncWithMedia; + /** + * Whether this channel should ensure oredered message: + * - true: orderd . + * - false: not orderd. + */ + bool ordered; + + /** + * Whether this channel should compress the data packet: + * - <= 0: We dont compress the data packet + * - > 0: Once the packet length exceeds the compressionLength, we compress it. + */ + int compressionLength; + // optional + Optional channelId; // 0~7 + DataChannelConfig() : + syncWithMedia(false), + ordered(false), + compressionLength(0) {} +}; + +/** + * The definition of the DataChannelInfo struct. + */ +struct DataChannelInfo { + /** + * The Id of the data channel + */ + int dataChannelId; + /** + * The metaData of the data channel + */ + util::AString metadata; +}; + +/** + * The definition of the DataChannelInfo struct. + */ +struct UserDataChannelInfo { + /** + * The user Id of the data channel + */ + util::AString userId; + /** + * The data channel infos + */ + const DataChannelInfo* infos; + /** + * The info size + */ + size_t info_size; +}; + +class ILocalDataChannel : public RefCountInterface { + public: + /** + * Send data packet to this data channel after publishing. + * + * @param [in] packet packet buffer pointer. + * @param [in] length packet buffer length. + * @return + * - 0: Success. + * - < 0: Failure. + */ + virtual int sendDataPacket(const char* packet, size_t length) = 0; + /** + * Send meta data to this data channel before publishing. + * + * @param [in] metaData meta data pointer. + * @param [in] length meta data length. + * @return···· + * - 0: Success. + * - < 0: Failure. + */ + virtual int setMetaData(const char* metaData, size_t length) = 0; + + /** + * return configured channel id + * + * @return + */ + virtual Optional configuredChannelId() const = 0; + + protected: + virtual ~ILocalDataChannel() {} +}; + +class IRemoteDataChannel: public RefCountInterface { + public: + virtual util::AString UserId() const = 0; + virtual int ChannelId() const = 0; + virtual util::AString Meta() = 0; + + protected: + virtual ~IRemoteDataChannel() {} +}; + +class IDataChannelObserver { + public: + /** + * Occurs when the channe is ready to send the data packet. + * @param channel the published channel. + */ + virtual void onLocalDataChannelPublished(agora_refptr channel) {} + + /** + * Occurs when the the channe is added and ready to receive data packet. + * @param channel the remote channel pointer. + */ + virtual void onRemoteDataChannelSubscribed(agora_refptr channel) {} + + /** + * Occurs when the the channe is removed. + * @param channel the remote channel pointer. + */ + virtual void onRemoteDataChannelUnsubscribed(agora_refptr channel) {} + /** + * Occurs when the packet is received. + * @param info the channel Info. + * @param packet the received packet. + */ + virtual void onRemoteDataPacketReceived(const UserDataChannelInfo& info, util::AString packet) {} + + /** + * Occurs when the remote data channel info updated. + * @param modified_infos the modifed channel Infos, add or update. + * @param modified_infos_size the size of modifed channel Infos . + * @param deleted_infos the deleted channel Infos. + * @param deleted_infos_size the size of deleted channel Infos . + */ + virtual void onRemoteDataChannelInfoUpdated(const UserDataChannelInfo* modified_infos, size_t modified_infos_size, + const UserDataChannelInfo* deleted_infos, size_t deleted_infos_size) {} + virtual ~IDataChannelObserver() {} +}; + +} // namespace agora diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraLocalUser.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraLocalUser.h index d0346e6e9..3f6e1d2cc 100644 --- a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraLocalUser.h +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraLocalUser.h @@ -8,7 +8,7 @@ #pragma once // NOLINT(build/header_guard) #include - +#include #include "AgoraBase.h" #include "AgoraOptional.h" @@ -17,6 +17,9 @@ namespace media { class IAudioFrameObserver; } +class ILocalDataChannel; +class IDataChannelObserver; + namespace rtc { class IAudioEngineWrapper; @@ -164,7 +167,7 @@ class ILocalUser { */ agora::Optional delay_ms; }; - + /** * The detailed statistics of the local audio. */ @@ -241,6 +244,21 @@ class ILocalUser { memset(codec_name, 0, sizeof(codec_name)); } }; + + enum NS_MODE { + ElderNsStatistical = 0, /* Elder Statistical Noise Suppression.*/ + NsNGStatistical = 1, /* Next Generation Statistical Noise Suppression.*/ + NsNG = 2 /* Next Generation Noise Suppression.*/ + }; + enum NS_LEVEL { + Soft = 0,/* Soft Noise Suppression.*/ + Aggressive = 1 /* Aggressiveness Noise Suppression.*/ + }; + enum NS_DELAY { + HighQuality = 0,/* High Audio Quality with High Delay.*/ + Balance = 1,/* Balanced Audio Quality and Delay.*/ + LowDelay = 2/* Slight Low Audio Quality with Low Delay.*/ + }; public: virtual ~ILocalUser() {} @@ -298,6 +316,18 @@ class ILocalUser { */ virtual int setAudioScenario(AUDIO_SCENARIO_TYPE scenario) = 0; + /** + * You can call this method to set the expected video scenario. + * The SDK will optimize the video experience for each scenario you set. + * + * @param scenarioType The video application scenario. + * + * @return + * - 0: Success. + * - < 0: Failure. + */ + virtual int setVideoScenario(VIDEO_APPLICATION_SCENARIO_TYPE scenarioType) = 0; + /** * Gets the detailed statistics of the local audio. * @@ -754,6 +784,10 @@ class ILocalUser { virtual int setVideoSubscriptionOptions(user_id_t userId, const VideoSubscriptionOptions& options) = 0; + + virtual int setHighPriorityUserList(uid_t* vipList, int uidNum, int option) = 0; + + virtual int getHighPriorityUserList(std::vector& vipList, int& option) = 0; /** * Sets the blocklist of subscribe remote stream audio. @@ -1011,6 +1045,78 @@ class ILocalUser { * - <0: failure */ virtual int getRemoteAudioTrackFilterProperty(user_id_t userId, const char* id, const char* key, char* jsonValue, size_t bufSize) = 0; + /** + * Publishes a local data channel to the channel. + * + * @param channel The data stream to be published. + * @return + * - 0: Success. + * - < 0: Failure. + */ + virtual int publishDataChannel(agora_refptr channel) = 0; + /** + * Stops publishing the data channel to the channel. + * + * @param channel The data channel that you want to stop publishing. + * @return + * - 0: Success. + * - < 0: Failure. + */ + virtual int unpublishDataChannel(agora_refptr channel) = 0; + /** + * Subscribes to a specified data channel of a specified remote user in channel. + * + * @param userId The ID of the remote user whose data channel you want to subscribe to. + * @param channelId The ID of the data channel that you want to subscribe to. + * @return + * - 0: Success. + * - < 0: Failure. + */ + virtual int subscribeDataChannel(user_id_t userId, int channelId) = 0; + /** + * Stops subscribing to the data channel of a specified remote user in the channel. + * + * @param userId The ID of the remote user whose data channel you want to stop subscribing to. + * @param channelId The ID of the data channel that you want to stop subscribing to. + * @return + * - 0: Success. + * - < 0: Failure. + * - -2(ERR_INVALID_ARGUMENT), if no such user exists or `userId` is invalid. + */ + + virtual int unsubscribeDataChannel(user_id_t userId, int channelId) = 0; + /** + * Registers an data channel observer. + * + * You need to implement the `IDataChannelObserver` class in this method + * + * @param observer A pointer to the data channel observer: + * @return + * - 0: Success. + * - < 0: Failure. + */ + virtual int registerDataChannelObserver(IDataChannelObserver * observer) = 0; + /** + * Releases the data channel observer. + * + * @param observer The pointer to the data channel observer + * @return + * - 0: Success. + * - < 0: Failure. + */ + virtual int unregisterDataChannelObserver(IDataChannelObserver * observer) = 0; + /** + * set the profile of audio noise suppression module + * + * @param NsEnable enable ns or not + * @param NsMode type of ns + * @param NsLevel level of the suppression + * @param NsDelay algorithm delay + * @return + * - 0: success + * - <0: failure + */ + virtual int SetAudioNsMode(bool NsEnable, NS_MODE NsMode, NS_LEVEL NsLevel, NS_DELAY NsDelay) = 0; }; /** @@ -1402,6 +1508,8 @@ class ILocalUserObserver { * @param state The remote user state.Just & #REMOTE_USER_STATE */ virtual void onUserStateChanged(user_id_t userId, uint32_t state){} + + virtual void onVideoRenderingTracingResult(user_id_t user_id, MEDIA_TRACE_EVENT currentState, VideoRenderingTracingInfo tracingInfo) {} }; class IVideoFrameObserver2 { diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraRtcConnection.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraRtcConnection.h index db16e8e69..886b2a257 100644 --- a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraRtcConnection.h +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraRtcConnection.h @@ -490,19 +490,6 @@ class IRtcConnection : public RefCountInterface { * - < 0: Failure. */ virtual int getUserInfoByUid(uid_t uid, rtc::UserInfo* userInfo) = 0; - /** Gets the NTP time. - * - * @note - * - Returns the wallclock time which is represented using the timestamp format of the Network Time Protocol (NTP), which is in milliseconds relative to 0h UTC on 1 January 1900. - * - * - The returned value may not be accurate, depending on whether the connection is normal to the NTP server. - * - * - The returned value can be validated by base::NtpTime::Valid(). - * - * @return - * - A NtpTime object. - */ - virtual base::NtpTime getNtpTime() = 0; }; /** @@ -684,21 +671,6 @@ class IRtcConnectionObserver { (void)type; } - /** - * Occurs when an API method is executed. - * @deprecated This callback is deprecated. Use other specific event callbacks instead. - * - * @param err The error code that the SDK reports when the method call fails. If the SDK reports 0, - * the method call succeeds. - * @param api The API method that is executed. - * @param result The result of the method call. - */ - virtual void onApiCallExecuted(int err, const char* api, const char* result) __deprecated { - (void)err; - (void)api; - (void)result; - } - /** Reports result of Content Inspect*/ virtual void onContentInspectResult(media::CONTENT_INSPECT_RESULT result) { (void)result; } /** Occurs when takeSnapshot API result is obtained diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraRtmpConnection.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraRtmpConnection.h index 136fb6ddf..98bdfad9c 100644 --- a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraRtmpConnection.h +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraRtmpConnection.h @@ -107,6 +107,14 @@ struct RtmpStreamingVideoConfiguration { */ int encoderHwSwMode; + /** + * Whether the encoder enables CBR coding or VBR coding. + * The default value is 0. + * 0: CBR + * 1: VBR + */ + int encoderBitrateControlMode; + /** * The orientation mode. * See {@link ORIENTATION_MODE ORIENTATION_MODE}. @@ -114,7 +122,7 @@ struct RtmpStreamingVideoConfiguration { ORIENTATION_MODE orientationMode; RtmpStreamingVideoConfiguration(): width(640), height(360), framerate(15), - bitrate(800), maxBitrate(960), minBitrate(600), gopInMs(2000), encoderHwSwMode(0), + bitrate(800), maxBitrate(960), minBitrate(600), gopInMs(2000), encoderHwSwMode(0),encoderBitrateControlMode(0), orientationMode(ORIENTATION_MODE_ADAPTIVE) {} }; @@ -223,7 +231,9 @@ struct RtmpConnectionConfiguration { RtmpStreamingAudioConfiguration audioConfig; RtmpStreamingVideoConfiguration videoConfig; bool enableWriteFlvFile; - RtmpConnectionConfiguration() : enableWriteFlvFile(false) {} + bool audioOnly; + RtmpConnectionConfiguration() : enableWriteFlvFile(false), + audioOnly(false) {} }; /** diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraRtmpLocalUser.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraRtmpLocalUser.h index aedc927f0..22172b769 100644 --- a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraRtmpLocalUser.h +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraRtmpLocalUser.h @@ -60,7 +60,15 @@ class IRtmpLocalUserObserver { * @param audioTrack The pointer to ILocalAudioTrack. */ virtual void onAudioTrackPublishSuccess(agora_refptr audioTrack) = 0; + + /** + * @deprecated This method will not be called back + */ virtual void onAudioTrackPublishStart(agora_refptr audioTrack) = 0; + + /** + * @deprecated This method will not be called back + */ virtual void onAudioTrackUnpublished(agora_refptr audioTrack) = 0; /** @@ -78,7 +86,15 @@ class IRtmpLocalUserObserver { * @param videoTrack The pointer to ILocalVideoTrack. */ virtual void onVideoTrackPublishSuccess(agora_refptr videoTrack) = 0; + + /** + * @deprecated This method will not be called back + */ virtual void onVideoTrackPublishStart(agora_refptr videoTrack) = 0; + + /** + * @deprecated This method will not be called back + */ virtual void onVideoTrackUnpublished(agora_refptr videoTrack) = 0; /** @@ -109,18 +125,88 @@ class IRtmpLocalUser { virtual ~IRtmpLocalUser() {} + /** + * Set the parameters of the audio encoder when pushing the stream + * + * @param config Audio encoder parameters + * + * @return + * - 0: Success. + * - < 0: Failure. + */ virtual int setAudioStreamConfiguration(const RtmpStreamingAudioConfiguration& config) = 0; + /** + * Set the parameters of the video encoder when pushing the stream + * + * @param config Video encoder parameters + * + * @return + * - 0: Success. + * - < 0: Failure. + */ virtual int setVideoStreamConfiguration(const RtmpStreamingVideoConfiguration& config) = 0; + /** + * Adjusts the audio volume for publishing. + * + * @param volume The volume for publishing. The value ranges between 0 and 100 (default). + * + * @return + * - 0: Success. + * - < 0: Failure. + */ virtual int adjustRecordingSignalVolume(int volume) = 0; + /** + * Gets the current volume for publishing. + * @param volume A pointer to the publishing volume. + * + * @return + * - 0: Success. + * - < 0: Failure. + */ virtual int getRecordingSignalVolume(int32_t* volume) = 0; + /** + * Set whether to enable local audio + * @param enabled Whether to enable local audio: + * - `true`: Enable local audio. + * - `false`: Disable local audio. + * + * @return + * - 0: Success. + * - < 0: Failure. + */ virtual int setAudioEnabled(bool enabled) = 0; + /** + * Dynamically adjust the bit rate parameters of the video encoder in the push stream + * + * @note: When increasing the bit rate, each call increases by 50kbps; + * When you lower the bit rate, you reduce it by 100kbps per call + * + * @param type The type of adjustment mode for the bit-rate parameter of the video encoder in the push stream: + * - `Increasing`: Increase the video encoding bitrate. + * - `Decreasing`: Reduce video encoding bitrate + * + * @return + * - 0: Success. + * - < 0: Failure. + */ virtual void adjustVideoBitrate(VideoBitrateAdjustType type) = 0; + /** + * Set whether to enable local video + * + * @param enabled Whether to enable local video: + * - `true`: Enable local video. + * - `false`: Disable local video. + * + * @return + * - 0: Success. + * - < 0: Failure. + */ virtual int setVideoEnabled(bool enabled) = 0; /** @@ -215,14 +301,14 @@ class IRtmpLocalUser { * - 0: Success. * - < 0: Failure. */ - virtual int registerAudioFrameObserver(media::base::IAudioFrameObserver* observer) = 0; + virtual int registerAudioFrameObserver(media::IAudioPcmFrameSink* observer) = 0; /** * Unregisters an audio frame observer object. * * @param observer A pointer to the audio frame observer object: IAudioFrameObserver. */ - virtual void unregisterAudioFrameObserver(media::base::IAudioFrameObserver* observer) = 0; + virtual void unregisterAudioFrameObserver(media::IAudioPcmFrameSink* observer) = 0; /** * Registers a video frame observer object. diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraVideoFrame.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraVideoFrame.h index 0477a061f..d2f6945c6 100644 --- a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraVideoFrame.h +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraVideoFrame.h @@ -54,6 +54,7 @@ struct RawPixelBuffer { kI422, kNV21, kNV12, + kI010, kRGBA, kARGB, kBGRA @@ -184,6 +185,7 @@ OPTIONAL_ENUM_CLASS VideoFrameMetaDataType { kAlphaChannel, kScreenMetaInfo, kVideoSourceType, + kFaceInfo, // Add other types afterwards }; diff --git a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraVideoTrack.h b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraVideoTrack.h index 7e0ce48f6..cb927ae9f 100644 --- a/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraVideoTrack.h +++ b/Android/APIExample/agora-simple-filter/src/main/cpp/AgoraRtcKit/NGIAgoraVideoTrack.h @@ -365,6 +365,10 @@ struct RemoteVideoTrackStats { * The overall delay (ms) of the video frames. */ int delay; + /** + * End-to-end delay from video capturer to video renderer. Hardware capture or render delay is excluded. + */ + int e2eDelay; /** * The width (pixel) of the remote video track. */ @@ -403,6 +407,10 @@ struct RemoteVideoTrackStats { The total video freeze time as a percentage (%) of the total time when the video is available. */ int frozenRate; + /** + * The number of video bytes received. + */ + uint32_t received_bytes; /** The total number of decoded video frames. */ @@ -430,8 +438,6 @@ struct RemoteVideoTrackStats { The total publish duration (ms) of the remote video stream. */ uint64_t publishDuration; - int superResolutionType; - /** decoded frame vqa mos value after all filter. */ @@ -444,9 +450,9 @@ struct RemoteVideoTrackStats { RemoteVideoTrackStats() : uid(0), delay(0), width(0), height(0), receivedBitrate(0), decoderOutputFrameRate(0), rendererOutputFrameRate(0), frameLossRate(0), packetLossRate(0), rxStreamType(VIDEO_STREAM_HIGH), - totalFrozenTime(0), frozenRate(0), totalDecodedFrames(0), avSyncTimeMs(0), + totalFrozenTime(0), frozenRate(0), received_bytes(0), totalDecodedFrames(0), avSyncTimeMs(0), downlink_process_time_ms(0), frame_render_delay_ms(0), totalActiveTime(0), - publishDuration(0), superResolutionType(0), vqa_mos(0), vqa_avg_cost_ms(0) {} + publishDuration(0), vqa_mos(0), vqa_avg_cost_ms(0) {} }; /** diff --git a/Android/APIExample/app/build.gradle b/Android/APIExample/app/build.gradle index 43c2ba5fc..5a6774f26 100644 --- a/Android/APIExample/app/build.gradle +++ b/Android/APIExample/app/build.gradle @@ -17,6 +17,7 @@ android { manifestPlaceholders = [ AppId: "${applicationId}" ] + ndk.abiFilters 'armeabi-v7a'// 'arm64-v8a', 'x86', 'x86-64' } buildTypes { @@ -61,7 +62,7 @@ dependencies { implementation fileTree(dir: "${localSdkPath}", include: ['*.jar', '*.aar']) } else{ - def agora_sdk_version = "4.2.0" + def agora_sdk_version = "4.2.1" // case 1: full libs implementation "io.agora.rtc:full-sdk:${agora_sdk_version}" implementation "io.agora.rtc:full-screen-sharing:${agora_sdk_version}" diff --git a/windows/APIExample/install.ps1 b/windows/APIExample/install.ps1 index d7f189223..ebe3db8dd 100644 --- a/windows/APIExample/install.ps1 +++ b/windows/APIExample/install.ps1 @@ -1,6 +1,6 @@ $ThirdPartysrc = 'https://agora-adc-artifacts.oss-cn-beijing.aliyuncs.com/libs/ThirdParty.zip' $ThirdPartydes = 'ThirdParty.zip' -$agora_sdk = 'https://download.agora.io/sdk/release/Agora_Native_SDK_for_Windows_v4.2.0_FULL.zip' +$agora_sdk = 'https://download.agora.io/sdk/release/Agora_Native_SDK_for_Windows_v4.2.1_FULL.zip' $agora_des = 'AgoraSdk.zip' $agora_local_sdk = '../../sdk' From 6e02134ec6868fbf1eea9f80f30476a9b5fa1d51 Mon Sep 17 00:00:00 2001 From: xucz Date: Tue, 20 Jun 2023 11:19:34 +0800 Subject: [PATCH 02/13] [Windows]Try to fix sei bug. --- .../APIExample/Advanced/VideoMetadata/CAgoraMetaDataDlg.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows/APIExample/APIExample/Advanced/VideoMetadata/CAgoraMetaDataDlg.cpp b/windows/APIExample/APIExample/Advanced/VideoMetadata/CAgoraMetaDataDlg.cpp index fdd9dad4e..9931ecfde 100755 --- a/windows/APIExample/APIExample/Advanced/VideoMetadata/CAgoraMetaDataDlg.cpp +++ b/windows/APIExample/APIExample/Advanced/VideoMetadata/CAgoraMetaDataDlg.cpp @@ -457,7 +457,7 @@ LRESULT CAgoraMetaDataDlg::OnEIDMetadataReceived(WPARAM wParam, LPARAM lParam) { IMetadataObserver::Metadata* metaData = (IMetadataObserver::Metadata*)wParam; CString strInfo; - strInfo.Format(_T("onMetadataReceived:uid:%u, ts=%d, size:%d."), metaData->uid, metaData->timeStampMs, metaData->size, metaData->buffer); + strInfo.Format(_T("onMetadataReceived:uid:%u, ts=%d, size:%d."), metaData->uid, metaData->timeStampMs, metaData->size); if (metaData->size > 0) { CString str; From 6643baaa91d466965ae70cdf4ef4e7a2a538683c Mon Sep 17 00:00:00 2001 From: xucz Date: Tue, 20 Jun 2023 11:28:52 +0800 Subject: [PATCH 03/13] [Windows]fix sei crash. --- .../APIExample/Advanced/VideoMetadata/CAgoraMetaDataDlg.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/windows/APIExample/APIExample/Advanced/VideoMetadata/CAgoraMetaDataDlg.cpp b/windows/APIExample/APIExample/Advanced/VideoMetadata/CAgoraMetaDataDlg.cpp index 9931ecfde..94b80c292 100755 --- a/windows/APIExample/APIExample/Advanced/VideoMetadata/CAgoraMetaDataDlg.cpp +++ b/windows/APIExample/APIExample/Advanced/VideoMetadata/CAgoraMetaDataDlg.cpp @@ -461,7 +461,9 @@ LRESULT CAgoraMetaDataDlg::OnEIDMetadataReceived(WPARAM wParam, LPARAM lParam) if (metaData->size > 0) { CString str; - str.Format(_T("Info: %s"), utf82cs(std::string((char*)metaData->buffer))); + TCHAR szBuf[2 * MAX_PATH] = { 0 }; + MultiByteToWideChar(CP_UTF8, MB_USEGLYPHCHARS, std::string((char*)metaData->buffer).c_str(), 2 * MAX_PATH, szBuf, 2 * MAX_PATH); + str.Format(_T("Info: %s"), CString(szBuf)); strInfo += str; } m_edtRecvSEI.SetWindowText(strInfo); From 72d7714a93c3c3e09b75fedefe221cf9f139a4af Mon Sep 17 00:00:00 2001 From: xucz Date: Tue, 20 Jun 2023 12:34:45 +0800 Subject: [PATCH 04/13] [Windows]fix sei crash final. --- .../VideoMetadata/CAgoraMetaDataDlg.cpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/windows/APIExample/APIExample/Advanced/VideoMetadata/CAgoraMetaDataDlg.cpp b/windows/APIExample/APIExample/Advanced/VideoMetadata/CAgoraMetaDataDlg.cpp index 94b80c292..3c1f81ec0 100755 --- a/windows/APIExample/APIExample/Advanced/VideoMetadata/CAgoraMetaDataDlg.cpp +++ b/windows/APIExample/APIExample/Advanced/VideoMetadata/CAgoraMetaDataDlg.cpp @@ -57,7 +57,7 @@ void CAgoraMetaDataObserver::onMetadataReceived(const Metadata &metadata) recvMetaData->uid = metadata.uid; recvMetaData->timeStampMs = metadata.timeStampMs; if (metadata.size > 0) { - recvMetaData->buffer = new unsigned char[metadata.size]; + recvMetaData->buffer = new unsigned char[metadata.size + 1]; memcpy_s(recvMetaData->buffer, metadata.size, metadata.buffer, metadata.size); recvMetaData->buffer[metadata.size] = 0; } @@ -456,17 +456,17 @@ LRESULT CAgoraMetaDataDlg::OnEIDRemoteVideoStateChanged(WPARAM wParam, LPARAM lP LRESULT CAgoraMetaDataDlg::OnEIDMetadataReceived(WPARAM wParam, LPARAM lParam) { IMetadataObserver::Metadata* metaData = (IMetadataObserver::Metadata*)wParam; - CString strInfo; - strInfo.Format(_T("onMetadataReceived:uid:%u, ts=%d, size:%d."), metaData->uid, metaData->timeStampMs, metaData->size); + CString strInfo; + strInfo.Format(_T("onMetadataReceived:uid:%u, ts=%d, size:%d."), metaData->uid, metaData->timeStampMs, metaData->size); - if (metaData->size > 0) { - CString str; - TCHAR szBuf[2 * MAX_PATH] = { 0 }; - MultiByteToWideChar(CP_UTF8, MB_USEGLYPHCHARS, std::string((char*)metaData->buffer).c_str(), 2 * MAX_PATH, szBuf, 2 * MAX_PATH); - str.Format(_T("Info: %s"), CString(szBuf)); - strInfo += str; - } + if (metaData->size > 0) { + CString str; + str.Format(_T("Info: %s"), utf82cs((char *)metaData->buffer)); + strInfo += str; + } m_edtRecvSEI.SetWindowText(strInfo); + delete metaData->buffer; + delete metaData; return 0; } From 5cb27149dbbfb2ea5b7c09ac0a1e7a2c1a93dcd3 Mon Sep 17 00:00:00 2001 From: zhaoyongqiang Date: Tue, 20 Jun 2023 16:58:06 +0800 Subject: [PATCH 05/13] fix timer crash bug --- macOS/APIExample/Common/ExternalVideo/AgoraYUVImageSourcePush.m | 1 + 1 file changed, 1 insertion(+) diff --git a/macOS/APIExample/Common/ExternalVideo/AgoraYUVImageSourcePush.m b/macOS/APIExample/Common/ExternalVideo/AgoraYUVImageSourcePush.m index 5a49969dd..1a4ca8aaf 100644 --- a/macOS/APIExample/Common/ExternalVideo/AgoraYUVImageSourcePush.m +++ b/macOS/APIExample/Common/ExternalVideo/AgoraYUVImageSourcePush.m @@ -58,6 +58,7 @@ - (void) initTimer { -(void)startSource { + [self stopSource]; if(self.timer == nil) { [self initTimer]; } From b9b2d5bb5bb023f6d19ef9685fad6c165ac674d8 Mon Sep 17 00:00:00 2001 From: zhaoyongqiang Date: Tue, 20 Jun 2023 17:16:06 +0800 Subject: [PATCH 06/13] fix recorder video crash Bug --- .../APIExample.xcodeproj/project.pbxproj | 18 ++++++++++++++++++ .../JoinChannelVideoRecorder.swift | 6 ++++++ 2 files changed, 24 insertions(+) diff --git a/iOS/APIExample/APIExample.xcodeproj/project.pbxproj b/iOS/APIExample/APIExample.xcodeproj/project.pbxproj index 519d9dd84..026252fd2 100644 --- a/iOS/APIExample/APIExample.xcodeproj/project.pbxproj +++ b/iOS/APIExample/APIExample.xcodeproj/project.pbxproj @@ -1396,6 +1396,7 @@ 03D13BCA2448758900B599B3 /* Resources */, 1B6F6CF9B678035E221EAFDE /* [CP] Embed Pods Frameworks */, 0339BEBA25205B80007D4FDD /* Embed App Extensions */, + 923E6839F72D5F4A404092CE /* [CP] Copy Pods Resources */, ); buildRules = ( ); @@ -1651,6 +1652,23 @@ shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; showEnvVarsInLog = 0; }; + 923E6839F72D5F4A404092CE /* [CP] Copy Pods Resources */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-APIExample/Pods-APIExample-resources-${CONFIGURATION}-input-files.xcfilelist", + ); + name = "[CP] Copy Pods Resources"; + outputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-APIExample/Pods-APIExample-resources-${CONFIGURATION}-output-files.xcfilelist", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-APIExample/Pods-APIExample-resources.sh\"\n"; + showEnvVarsInLog = 0; + }; /* End PBXShellScriptBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ diff --git a/iOS/APIExample/APIExample/Examples/Basic/JoinChannelVideo(Recorder)/JoinChannelVideoRecorder.swift b/iOS/APIExample/APIExample/Examples/Basic/JoinChannelVideo(Recorder)/JoinChannelVideoRecorder.swift index 1fddef9ff..aa98f21e3 100644 --- a/iOS/APIExample/APIExample/Examples/Basic/JoinChannelVideo(Recorder)/JoinChannelVideoRecorder.swift +++ b/iOS/APIExample/APIExample/Examples/Basic/JoinChannelVideo(Recorder)/JoinChannelVideoRecorder.swift @@ -246,6 +246,8 @@ class JoinChannelVideoRecorder: BaseViewController { agoraKit.disableAudio() agoraKit.disableVideo() if isJoined { + localRecord.stopRecording() + remoteRecord.stopRecording() agoraKit.destroy(localRecord) agoraKit.destroy(remoteRecord) agoraKit.stopPreview() @@ -285,6 +287,10 @@ class JoinChannelVideoRecorder: BaseViewController { ToastView.show(text: path) } } + + deinit { + AgoraRtcEngineKit.destroy() + } } extension JoinChannelVideoRecorder: AgoraMediaRecorderDelegate { From 1fe66a2d923083af3d8ec6d76d163c32ece2707f Mon Sep 17 00:00:00 2001 From: zhaoyongqiang Date: Tue, 20 Jun 2023 17:43:05 +0800 Subject: [PATCH 07/13] Fixed an issue where the wheat does not flow --- .../Examples/Advanced/LiveStreaming/LiveStreaming.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/iOS/APIExample/APIExample/Examples/Advanced/LiveStreaming/LiveStreaming.swift b/iOS/APIExample/APIExample/Examples/Advanced/LiveStreaming/LiveStreaming.swift index f1b668294..e2bcaa8e5 100644 --- a/iOS/APIExample/APIExample/Examples/Advanced/LiveStreaming/LiveStreaming.swift +++ b/iOS/APIExample/APIExample/Examples/Advanced/LiveStreaming/LiveStreaming.swift @@ -317,6 +317,10 @@ class LiveStreamingMain: BaseViewController { } else { becomeAudience() } + let option = AgoraRtcChannelMediaOptions() + option.publishCameraTrack = role == .broadcaster + option.publishMicrophoneTrack = role == .broadcaster + agoraKit.updateChannel(with: option) } @IBAction func onToggleUltraLowLatency(_ sender:UISwitch) { From 91f380de91d40b5c76650fcbecc9e022ce8a148e Mon Sep 17 00:00:00 2001 From: zhaoyongqiang Date: Wed, 21 Jun 2023 15:17:22 +0800 Subject: [PATCH 08/13] fix publish stream bug --- .../Advanced/LiveStreaming/LiveStreaming.swift | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/macOS/APIExample/Examples/Advanced/LiveStreaming/LiveStreaming.swift b/macOS/APIExample/Examples/Advanced/LiveStreaming/LiveStreaming.swift index d8d82639c..739b14e71 100644 --- a/macOS/APIExample/Examples/Advanced/LiveStreaming/LiveStreaming.swift +++ b/macOS/APIExample/Examples/Advanced/LiveStreaming/LiveStreaming.swift @@ -220,7 +220,13 @@ class LiveStreamingMain: BaseViewController { selectRolePicker.onSelectChanged { guard let selected = self.selectedRole else { return } if self.isJoined { + let mediaOption = AgoraRtcChannelMediaOptions() + mediaOption.publishCameraTrack = selected == .broadcaster + mediaOption.publishMicrophoneTrack = selected == .broadcaster + mediaOption.clientRoleType = selected + self.agoraKit.updateChannel(with: mediaOption) self.agoraKit.setClientRole(selected) + _ = selected == .broadcaster ? self.agoraKit.startPreview() : self.agoraKit.stopPreview() } self.waterMarkContainer.isHidden = selected == .audience self.bFrameContainer.isHidden = selected == .audience @@ -427,7 +433,9 @@ class LiveStreamingMain: BaseViewController { videoCanvas.renderMode = .hidden agoraKit.setupLocalVideo(videoCanvas) // you have to call startPreview to see local video - agoraKit.startPreview() + if role == .broadcaster { + agoraKit.startPreview() + } // start joining channel // 1. Users can only see each other after they join the @@ -438,6 +446,7 @@ class LiveStreamingMain: BaseViewController { isProcessing = true let option = AgoraRtcChannelMediaOptions() option.publishCameraTrack = role == .broadcaster + option.publishMicrophoneTrack = role == .broadcaster option.clientRoleType = role NetworkManager.shared.generateToken(channelName: channel, success: { token in let result = self.agoraKit.joinChannel(byToken: token, channelId: channel, uid: 0, mediaOptions: option) From 3419323b92516760ff576cf777c963da453bbe36 Mon Sep 17 00:00:00 2001 From: xucz Date: Wed, 21 Jun 2023 20:19:17 +0800 Subject: [PATCH 09/13] update pod rtc sdk to 4.2.1. --- iOS/APIExample-Audio/Podfile | 2 +- iOS/APIExample/Podfile | 2 +- macOS/Podfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/iOS/APIExample-Audio/Podfile b/iOS/APIExample-Audio/Podfile index d8a06674f..b2f6c1b26 100644 --- a/iOS/APIExample-Audio/Podfile +++ b/iOS/APIExample-Audio/Podfile @@ -7,7 +7,7 @@ target 'APIExample-Audio' do pod 'Floaty', '~> 4.2.0' pod 'AGEVideoLayout', '~> 1.0.2' - pod 'AgoraAudio_iOS', '4.2.0' + pod 'AgoraAudio_iOS', '4.2.1' # pod 'sdk', :path => 'sdk.podspec' end diff --git a/iOS/APIExample/Podfile b/iOS/APIExample/Podfile index c2cd9b7f6..e4e76b300 100644 --- a/iOS/APIExample/Podfile +++ b/iOS/APIExample/Podfile @@ -18,7 +18,7 @@ end target 'Agora-ScreenShare-Extension' do use_frameworks! # pod 'sdk', :path => 'sdk.podspec' - pod 'AgoraRtcEngine_iOS', '4.2.0' + pod 'AgoraRtcEngine_iOS', '4.2.1' end target 'SimpleFilter' do diff --git a/macOS/Podfile b/macOS/Podfile index 278c8d3e4..c09a6c0c8 100644 --- a/macOS/Podfile +++ b/macOS/Podfile @@ -11,5 +11,5 @@ end target 'SimpleFilter' do use_frameworks! # pod 'sdk', :path => 'sdk.podspec' - pod 'AgoraRtcEngine_macOS', '4.2.0' + pod 'AgoraRtcEngine_macOS', '4.2.1' end From 53f93332ccee4f75dde4d333d9b6ccecba3673af Mon Sep 17 00:00:00 2001 From: xucz Date: Wed, 21 Jun 2023 20:32:49 +0800 Subject: [PATCH 10/13] update pod rtc sdk to 4.2.1. --- iOS/APIExample/Podfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iOS/APIExample/Podfile b/iOS/APIExample/Podfile index e4e76b300..ccf675ad1 100644 --- a/iOS/APIExample/Podfile +++ b/iOS/APIExample/Podfile @@ -24,5 +24,5 @@ end target 'SimpleFilter' do use_frameworks! # pod 'sdk', :path => 'sdk.podspec' - pod 'AgoraRtcEngine_iOS', '4.2.0' + pod 'AgoraRtcEngine_iOS', '4.2.1' end From 7eb30fd7f289759ef28abf1eb38ca07ae6d6d49d Mon Sep 17 00:00:00 2001 From: xucz Date: Wed, 21 Jun 2023 20:50:59 +0800 Subject: [PATCH 11/13] update pod rtc sdk to 4.2.1. --- iOS/APIExample/Podfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iOS/APIExample/Podfile b/iOS/APIExample/Podfile index ccf675ad1..dcfb746a4 100644 --- a/iOS/APIExample/Podfile +++ b/iOS/APIExample/Podfile @@ -8,7 +8,7 @@ target 'APIExample' do pod 'Floaty', '~> 4.2.0' pod 'AGEVideoLayout', '~> 1.0.2' pod 'CocoaAsyncSocket', '7.6.5' - pod 'AgoraRtcEngine_iOS', '4.2.0' + pod 'AgoraRtcEngine_iOS', '4.2.1' # pod 'sdk', :path => 'sdk.podspec' # pod 'senseLib', :path => 'sense.podspec' # pod 'bytedEffect', :path => 'bytedEffect.podspec' From 0a68e0cef6d9513603a3c2ccfdee3df155a0b21e Mon Sep 17 00:00:00 2001 From: zhaoyongqiang Date: Sun, 25 Jun 2023 11:43:26 +0800 Subject: [PATCH 12/13] update sdk version to 4.2.1 --- iOS/APIExample-Audio/Podfile | 2 +- iOS/APIExample/Podfile | 14 +++++++------- macOS/Podfile | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/iOS/APIExample-Audio/Podfile b/iOS/APIExample-Audio/Podfile index d8a06674f..b2f6c1b26 100644 --- a/iOS/APIExample-Audio/Podfile +++ b/iOS/APIExample-Audio/Podfile @@ -7,7 +7,7 @@ target 'APIExample-Audio' do pod 'Floaty', '~> 4.2.0' pod 'AGEVideoLayout', '~> 1.0.2' - pod 'AgoraAudio_iOS', '4.2.0' + pod 'AgoraAudio_iOS', '4.2.1' # pod 'sdk', :path => 'sdk.podspec' end diff --git a/iOS/APIExample/Podfile b/iOS/APIExample/Podfile index c2cd9b7f6..86e01fad6 100644 --- a/iOS/APIExample/Podfile +++ b/iOS/APIExample/Podfile @@ -8,21 +8,21 @@ target 'APIExample' do pod 'Floaty', '~> 4.2.0' pod 'AGEVideoLayout', '~> 1.0.2' pod 'CocoaAsyncSocket', '7.6.5' - pod 'AgoraRtcEngine_iOS', '4.2.0' -# pod 'sdk', :path => 'sdk.podspec' + pod 'AgoraRtcEngine_iOS', '4.2.1' + # pod 'sdk', :path => 'sdk.podspec' # pod 'senseLib', :path => 'sense.podspec' # pod 'bytedEffect', :path => 'bytedEffect.podspec' -# pod 'fuLib', :path => 'fu.podspec' + # pod 'fuLib', :path => 'fu.podspec' end target 'Agora-ScreenShare-Extension' do use_frameworks! -# pod 'sdk', :path => 'sdk.podspec' - pod 'AgoraRtcEngine_iOS', '4.2.0' + # pod 'sdk', :path => 'sdk.podspec' + pod 'AgoraRtcEngine_iOS', '4.2.1' end target 'SimpleFilter' do use_frameworks! -# pod 'sdk', :path => 'sdk.podspec' - pod 'AgoraRtcEngine_iOS', '4.2.0' + # pod 'sdk', :path => 'sdk.podspec' + pod 'AgoraRtcEngine_iOS', '4.2.1' end diff --git a/macOS/Podfile b/macOS/Podfile index 278c8d3e4..af151eaf4 100644 --- a/macOS/Podfile +++ b/macOS/Podfile @@ -4,12 +4,12 @@ target 'APIExample' do use_frameworks! pod 'AGEVideoLayout', '1.0.2' - pod 'AgoraRtcEngine_macOS', '4.2.0' -# pod 'sdk', :path => 'sdk.podspec' + pod 'AgoraRtcEngine_macOS', '4.2.1' + # pod 'sdk', :path => 'sdk.podspec' end target 'SimpleFilter' do use_frameworks! -# pod 'sdk', :path => 'sdk.podspec' - pod 'AgoraRtcEngine_macOS', '4.2.0' + # pod 'sdk', :path => 'sdk.podspec' + pod 'AgoraRtcEngine_macOS', '4.2.1' end From 458d6f8e2e6fe9369afe30f1436d711e8b931682 Mon Sep 17 00:00:00 2001 From: xucz Date: Sun, 25 Jun 2023 12:12:59 +0800 Subject: [PATCH 13/13] [Android]fix gradle url. --- .../APIExample-Audio/gradle/wrapper/gradle-wrapper.properties | 4 ++-- Android/APIExample/gradle/wrapper/gradle-wrapper.properties | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Android/APIExample-Audio/gradle/wrapper/gradle-wrapper.properties b/Android/APIExample-Audio/gradle/wrapper/gradle-wrapper.properties index 4b5a2fecd..04180faf8 100644 --- a/Android/APIExample-Audio/gradle/wrapper/gradle-wrapper.properties +++ b/Android/APIExample-Audio/gradle/wrapper/gradle-wrapper.properties @@ -3,5 +3,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -# distributionUrl=https://services.gradle.org/distributions/gradle-7.3.3-bin.zip -distributionUrl=https://mirrors.cloud.tencent.com/gradle/gradle-7.3.3-bin.zip +distributionUrl=https://services.gradle.org/distributions/gradle-7.3.3-bin.zip +# distributionUrl=https://mirrors.cloud.tencent.com/gradle/gradle-7.3.3-bin.zip diff --git a/Android/APIExample/gradle/wrapper/gradle-wrapper.properties b/Android/APIExample/gradle/wrapper/gradle-wrapper.properties index 4b5a2fecd..04180faf8 100644 --- a/Android/APIExample/gradle/wrapper/gradle-wrapper.properties +++ b/Android/APIExample/gradle/wrapper/gradle-wrapper.properties @@ -3,5 +3,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -# distributionUrl=https://services.gradle.org/distributions/gradle-7.3.3-bin.zip -distributionUrl=https://mirrors.cloud.tencent.com/gradle/gradle-7.3.3-bin.zip +distributionUrl=https://services.gradle.org/distributions/gradle-7.3.3-bin.zip +# distributionUrl=https://mirrors.cloud.tencent.com/gradle/gradle-7.3.3-bin.zip