[Android]Update beauty api and fix bug(NMS-20393)

AgoraIO · Apr 8, 2024 · 13641d7 · 13641d7
1 parent 48b0f29
commit 13641d7
Show file tree

Hide file tree

Showing 9 changed files with 83 additions and 31 deletions.
diff --git a/...ple/app/src/main/java/io/agora/api/example/examples/advanced/beauty/FaceUnityBeautySDK.kt b/...ple/app/src/main/java/io/agora/api/example/examples/advanced/beauty/FaceUnityBeautySDK.kt
@@ -60,7 +60,6 @@ object FaceUnityBeautySDK {
                 Log.e(TAG, "FURenderManager onFail -- code=$errCode, msg=$errMsg")
             }
         })
-        beautyConfig.resume()
         return true
     }
 
@@ -92,8 +91,17 @@ object FaceUnityBeautySDK {
         private val fuRenderKit = FURenderKit.getInstance()
 
         // 美颜配置
-        private val faceBeauty =
-            FaceBeauty(FUBundleData("graphics" + File.separator + "face_beautification.bundle"))
+        private val faceBeauty: FaceBeauty
+            get() {
+                var faceBeauty = fuRenderKit.faceBeauty
+                if (faceBeauty == null) {
+                    faceBeauty =
+                        FaceBeauty(FUBundleData("graphics" + File.separator + "face_beautification.bundle"))
+                    fuRenderKit.faceBeauty = faceBeauty
+                }
+                return faceBeauty
+            }
+
 
         // 资源基础路径
         private val resourceBase = "beauty_faceunity"
@@ -285,7 +293,7 @@ object FaceUnityBeautySDK {
             }
 
 
-        internal fun reset() {
+        fun reset() {
             smooth = 0.65f
             whiten = 0.65f
             thinFace = 0.3f
@@ -307,11 +315,6 @@ object FaceUnityBeautySDK {
             sticker = null
         }
 
-        internal fun resume() {
-            runOnBeautyThread {
-                fuRenderKit.faceBeauty = faceBeauty
-            }
-        }
     }
 
     data class MakeUpItem(

diff --git a/Android/APIExample/app/src/main/java/io/agora/beautyapi/bytedance/ByteDanceBeautyAPI.kt b/Android/APIExample/app/src/main/java/io/agora/beautyapi/bytedance/ByteDanceBeautyAPI.kt
@@ -31,7 +31,7 @@ import io.agora.base.VideoFrame
 import io.agora.rtc2.Constants
 import io.agora.rtc2.RtcEngine
 
-const val VERSION = "1.0.5"
+const val VERSION = "1.0.6"
 
 enum class CaptureMode{
     Agora, // 使用声网内部的祼数据接口进行处理
@@ -95,9 +95,8 @@ enum class ErrorCode(val value: Int) {
     ERROR_HAS_INITIALIZED(102), // 已经Initialize成功后再次调用报错
     ERROR_HAS_RELEASED(103), // 已经调用release销毁后还调用其他API
     ERROR_PROCESS_NOT_CUSTOM(104), // 非Custom处理模式下调用onFrame接口从外部传入视频帧
-    ERROR_PROCESS_DISABLE(105), // 当调用enable(false)禁用美颜后调用onFrame接口返回
-    ERROR_VIEW_TYPE_ERROR(106), // 当调用setupLocalVideo时view类型错误时返回
-    ERROR_FRAME_SKIPPED(107), // 当处理帧忽略时在onFrame返回
+    ERROR_VIEW_TYPE_ERROR(105), // 当调用setupLocalVideo时view类型错误时返回
+    ERROR_FRAME_SKIPPED(106), // 当处理帧忽略时在onFrame返回
 }
 
 enum class BeautyPreset {

diff --git a/Android/APIExample/app/src/main/java/io/agora/beautyapi/bytedance/ByteDanceBeautyAPIImpl.kt b/Android/APIExample/app/src/main/java/io/agora/beautyapi/bytedance/ByteDanceBeautyAPIImpl.kt
@@ -73,6 +73,8 @@ class ByteDanceBeautyAPIImpl : ByteDanceBeautyAPI, IVideoFrameObserver {
     private var cameraConfig = CameraConfig()
     private var localVideoRenderMode = Constants.RENDER_MODE_HIDDEN
     private val pendingProcessRunList = Collections.synchronizedList(mutableListOf<()->Unit>())
+    private var frameWidth = 0
+    private var frameHeight = 0
 
     private enum class BeautyProcessType{
         UNKNOWN, TEXTURE_OES, TEXTURE_2D, I420
@@ -93,10 +95,7 @@ class ByteDanceBeautyAPIImpl : ByteDanceBeautyAPI, IVideoFrameObserver {
         }
         LogUtils.i(TAG, "initialize >> config = $config")
         LogUtils.i(TAG, "initialize >> beauty api version=$VERSION, beauty sdk version=${RenderManager.getSDKVersion()}")
-        config.rtcEngine.sendCustomReportMessage(reportId,
-            reportCategory,
-            "initialize",
-            "$config", 0)
+        config.rtcEngine.sendCustomReportMessage(reportId, reportCategory, "initialize", "$config", 0)
         return ErrorCode.ERROR_OK.value
     }
 
@@ -376,6 +375,17 @@ class ByteDanceBeautyAPIImpl : ByteDanceBeautyAPI, IVideoFrameObserver {
             return false
         }
 
+        val oldFrameWidth = frameWidth
+        val oldFrameHeight = frameHeight
+        frameWidth = videoFrame.rotatedWidth
+        frameHeight = videoFrame.rotatedHeight
+        if (oldFrameWidth > 0 || oldFrameHeight > 0) {
+            if(oldFrameWidth != frameWidth || oldFrameHeight != frameHeight){
+                skipFrame = 2
+                return false
+            }
+        }
+
         if(!enable){
             return true
         }

diff --git a/Android/APIExample/app/src/main/java/io/agora/beautyapi/faceunity/FaceUnityBeautyAPI.kt b/Android/APIExample/app/src/main/java/io/agora/beautyapi/faceunity/FaceUnityBeautyAPI.kt
@@ -86,9 +86,8 @@ enum class ErrorCode(val value: Int) {
     ERROR_HAS_INITIALIZED(102), // 已经Initialize成功后再次调用报错
     ERROR_HAS_RELEASED(103), // 已经调用release销毁后还调用其他API
     ERROR_PROCESS_NOT_CUSTOM(104), // 非Custom处理模式下调用onFrame接口从外部传入视频帧
-    ERROR_PROCESS_DISABLE(105), // 当调用enable(false)禁用美颜后调用onFrame接口返回
-    ERROR_VIEW_TYPE_ERROR(106), // 当调用setupLocalVideo时view类型错误时返回
-    ERROR_FRAME_SKIPPED(107), // 当处理帧忽略时在onFrame返回
+    ERROR_VIEW_TYPE_ERROR(105), // 当调用setupLocalVideo时view类型错误时返回
+    ERROR_FRAME_SKIPPED(106), // 当处理帧忽略时在onFrame返回
 }
 
 enum class BeautyPreset {

diff --git a/Android/APIExample/app/src/main/java/io/agora/beautyapi/faceunity/FaceUnityBeautyAPIImpl.kt b/Android/APIExample/app/src/main/java/io/agora/beautyapi/faceunity/FaceUnityBeautyAPIImpl.kt
@@ -66,7 +66,7 @@ class FaceUnityBeautyAPIImpl : FaceUnityBeautyAPI, IVideoFrameObserver {
     private val reportId = "scenarioAPI"
     private val reportCategory = "beauty_android_$VERSION"
     private var beautyMode = 0 // 0: 自动根据buffer类型切换，1：固定使用OES纹理，2：固定使用i420，3: 单纹理模式
-    private var enableTextureAsync = true // 是否开启纹理+异步缓存处理，不支持在预览中实时切换。对于GPU性能好的手机可以减小美颜处理耗时，对于中端机开启后效果也不明显。
+    private var enableTextureAsync = true // 是否开启纹理+异步缓存处理。对于GPU性能好的手机可以减小美颜处理耗时，对于中端机开启后效果也不明显。
 
     private var textureBufferHelper: TextureBufferHelper? = null
     private var wrapTextureBufferHelper: TextureBufferHelper? = null
@@ -239,8 +239,7 @@ class FaceUnityBeautyAPIImpl : FaceUnityBeautyAPI, IVideoFrameObserver {
         LogUtils.i(TAG, "setBeautyPreset >> preset = $preset")
         config?.rtcEngine?.sendCustomReportMessage(reportId, reportCategory, "enable", "preset=$preset", 0)
 
-        val recommendFaceBeauty = conf.fuRenderKit.faceBeauty ?:
-            FaceBeauty(FUBundleData("graphics" + File.separator + "face_beautification.bundle"))
+        val recommendFaceBeauty = FaceBeauty(FUBundleData("graphics" + File.separator + "face_beautification.bundle"))
         if (preset == BeautyPreset.DEFAULT) {
             recommendFaceBeauty.filterName = FaceBeautyFilterEnum.FENNEN_1
             recommendFaceBeauty.filterIntensity = 0.7
@@ -569,7 +568,7 @@ class FaceUnityBeautyAPIImpl : FaceUnityBeautyAPI, IVideoFrameObserver {
     }
 
     private fun processBeautySingleTexture(videoFrame: VideoFrame): Int {
-        val texBufferHelper = wrapTextureBufferHelper ?: return -1
+        val texBufferHelper = textureBufferHelper ?: return -1
         val textureBuffer = videoFrame.buffer as? TextureBuffer ?: return -1
 
         when(textureBuffer.type){

diff --git a/...IExample/app/src/main/java/io/agora/beautyapi/faceunity/utils/egl/TextureProcessHelper.kt b/...IExample/app/src/main/java/io/agora/beautyapi/faceunity/utils/egl/TextureProcessHelper.kt
@@ -173,7 +173,7 @@ class TextureProcessHelper(
             future = futureQueue.poll()
         }
         glTextureBufferQueueIn.reset()
-        glFrameBuffer.release()
+//        glFrameBuffer.release()
         executeSync {
             glTextureBufferQueueOut.reset()
         }

diff --git a/Android/APIExample/app/src/main/java/io/agora/beautyapi/sensetime/SenseTimeBeautyAPI.kt b/Android/APIExample/app/src/main/java/io/agora/beautyapi/sensetime/SenseTimeBeautyAPI.kt
@@ -92,9 +92,8 @@ enum class ErrorCode(val value: Int) {
     ERROR_HAS_INITIALIZED(102), // 已经Initialize成功后再次调用报错
     ERROR_HAS_RELEASED(103), // 已经调用release销毁后还调用其他API
     ERROR_PROCESS_NOT_CUSTOM(104), // 非Custom处理模式下调用onFrame接口从外部传入视频帧
-    ERROR_PROCESS_DISABLE(105), // 当调用enable(false)禁用美颜后调用onFrame接口返回
-    ERROR_VIEW_TYPE_ERROR(106), // 当调用setupLocalVideo时view类型错误时返回
-    ERROR_FRAME_SKIPPED(107), // 当处理帧忽略时在onFrame返回
+    ERROR_VIEW_TYPE_ERROR(105), // 当调用setupLocalVideo时view类型错误时返回
+    ERROR_FRAME_SKIPPED(106), // 当处理帧忽略时在onFrame返回
 }
 
 enum class BeautyPreset {

diff --git a/Android/APIExample/app/src/main/java/io/agora/beautyapi/sensetime/SenseTimeBeautyAPIImpl.kt b/Android/APIExample/app/src/main/java/io/agora/beautyapi/sensetime/SenseTimeBeautyAPIImpl.kt
@@ -308,7 +308,7 @@ class SenseTimeBeautyAPIImpl : SenseTimeBeautyAPI, IVideoFrameObserver {
         if (textureBufferHelper?.handler?.looper?.thread == Thread.currentThread()) {
             run.invoke()
         } else if (textureBufferHelper != null) {
-            textureBufferHelper?.handler?.post(run)
+            textureBufferHelper?.invoke(run)
         } else {
             pendingProcessRunList.add(run)
         }
@@ -350,7 +350,7 @@ class SenseTimeBeautyAPIImpl : SenseTimeBeautyAPI, IVideoFrameObserver {
         workerThreadExecutor.shutdown()
         textureBufferHelper?.let {
             textureBufferHelper = null
-            it.handler.removeCallbacksAndMessages(null)
+            // it.handler.removeCallbacksAndMessages(null)
             it.invoke {
                 beautyProcessor?.release()
                 null

diff --git a/...Example/app/src/main/java/io/agora/beautyapi/sensetime/utils/processor/BeautyProcessor.kt b/...Example/app/src/main/java/io/agora/beautyapi/sensetime/utils/processor/BeautyProcessor.kt
@@ -45,10 +45,18 @@ class BeautyProcessor : IBeautyProcessor {
     private var mInputHeight = 0
     private var mInputOrientation = 0
     private var isLastFrontCamera = false
+    private var skipFrame = 0
+    private var processMode = ProcessMode.DOUBLE_INPUT
 
     @Volatile
     private var isReleased = false
 
+    enum class ProcessMode {
+        DOUBLE_INPUT,
+        SINGLE_BYTES_INPUT,
+        SINGLE_TEXTURE_INPUT
+    }
+
     override fun initialize(
         effectNative: STMobileEffectNative,
         humanActionNative: STMobileHumanActionNative
@@ -106,13 +114,31 @@ class BeautyProcessor : IBeautyProcessor {
             return null
         }
         return if (input.bytes != null && input.textureId != null) {
+            if(processMode != ProcessMode.DOUBLE_INPUT){
+                processMode = ProcessMode.DOUBLE_INPUT
+                if (mInputWidth > 0 || mInputHeight > 0) {
+                    skipFrame = 3
+                }
+            }
             processDoubleInput(input)
         } else if (input.bytes != null) {
+            if(processMode != ProcessMode.SINGLE_BYTES_INPUT){
+                processMode = ProcessMode.SINGLE_BYTES_INPUT
+                if (mInputWidth > 0 || mInputHeight > 0) {
+                    skipFrame = 3
+                }
+            }
             processSingleBytesInput(input)
         } else if (input.textureId != null && Build.VERSION.SDK_INT >= 26) {
+            if(processMode != ProcessMode.SINGLE_TEXTURE_INPUT){
+                processMode = ProcessMode.SINGLE_TEXTURE_INPUT
+                if (mInputWidth > 0 || mInputHeight > 0) {
+                    skipFrame = 3
+                }
+            }
             processSingleTextureInput(input)
         } else {
-            null
+            throw RuntimeException("Single texture input is not supported when SDK_INT < 26!");
         }
     }
 
@@ -252,6 +278,9 @@ class BeautyProcessor : IBeautyProcessor {
             return null
         }
         if (mInputWidth != input.width || mInputHeight != input.height || mInputOrientation != input.cameraOrientation || isLastFrontCamera != input.isFrontCamera) {
+            if(mInputWidth > 0 || mInputHeight > 0){
+                skipFrame = 3
+            }
             mInputWidth = input.width
             mInputHeight = input.height
             mInputOrientation = input.cameraOrientation
@@ -260,6 +289,8 @@ class BeautyProcessor : IBeautyProcessor {
             return null
         }
 
+
+
         val diff = glTextureBufferQueue.size() - mFaceDetector.size()
         if(diff < input.diffBetweenBytesAndTexture){
             glTextureBufferQueue.enqueue(
@@ -330,6 +361,11 @@ class BeautyProcessor : IBeautyProcessor {
             )
         )
 
+        if(skipFrame > 0){
+            skipFrame --
+            return null
+        }
+
         return out
     }
 
@@ -404,16 +440,23 @@ class BeautyProcessor : IBeautyProcessor {
             STEffectParam.EFFECT_PARAM_USE_INPUT_TIMESTAMP,
             1.0f
         )
+        if (isReleased) {
+            return -1
+        }
         mSTMobileEffectNative.render(
             sTEffectRenderInParam,
             stEffectRenderOutParam,
             false
         )
 
+
         if (event == mCustomEvent) {
             mCustomEvent = 0
         }
 
+        if (isReleased) {
+            return -1
+        }
         glFrameBuffer.setSize(width, height)
         glFrameBuffer.resetTransform()
         glFrameBuffer.setFlipV(true)