HarmonyOS support for VAD. (#1561)

k2-fsa · Nov 24, 2024 · 31d6206 · 31d6206
1 parent e424cc9
commit 31d6206
Show file tree

Hide file tree

Showing 15 changed files with 229 additions and 69 deletions.
diff --git a/build-ohos-arm64-v8a.sh b/build-ohos-arm64-v8a.sh
@@ -19,7 +19,6 @@ cd $dir
 # rm commandline-tools-linux-x64-5.0.5.200.zip
 if [ -z $OHOS_SDK_NATIVE_DIR ]; then
   OHOS_SDK_NATIVE_DIR=/star-fj/fangjun/software/huawei/command-line-tools/sdk/default/openharmony/native/
-  export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH
   # You can find the following content inside OHOS_SDK_NATIVE_DIR
   # ls -lh /star-fj/fangjun/software/huawei/command-line-tools/sdk/default/openharmony/native/
   # total 524K
@@ -35,32 +34,39 @@ if [ -z $OHOS_SDK_NATIVE_DIR ]; then
   # drwxr-xr-x  3 kuangfangjun root    0 Nov  6 22:36 sysroot
 fi
 
-# If you don't want to install commandline tools, you can install the SDK
-# using DevEco Studio. The following uses API version 10 as an example and
-# it has installed the SDK to
-# /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native
-#
-# Remember to select ``native`` when you install the SDK
 if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then
-  OHOS_SDK_NATIVE_DIR=/Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native
-  # export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH
-  # ls -lh /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native/
-  # total 1560
-  # -rw-r--r--   1 fangjun  staff   764K Jan  1  2001 NOTICE.txt
-  # drwxr-xr-x   3 fangjun  staff    96B Nov 19 22:42 build
-  # drwxr-xr-x   3 fangjun  staff    96B Nov 19 22:42 build-tools
-  # drwxr-xr-x  10 fangjun  staff   320B Nov 19 22:42 llvm
-  # -rw-r--r--   1 fangjun  staff   4.0K Jan  1  2001 nativeapi_syscap_config.json
-  # -rw-r--r--   1 fangjun  staff   1.9K Jan  1  2001 ndk_system_capability.json
-  # -rw-r--r--   1 fangjun  staff   169B Jan  1  2001 oh-uni-package.json
-  # drwxr-xr-x   3 fangjun  staff    96B Nov 19 22:42 sysroot
+  OHOS_SDK_NATIVE_DIR=/Users/fangjun/software/command-line-tools/sdk/default/openharmony/native
+  # (py38) fangjuns-MacBook-Pro:software fangjun$ ls -lh command-line-tools/sdk/default/openharmony/native/
+  # total 752
+  # -rw-r--r--   1 fangjun  staff   341K Jan  1  2001 NOTICE.txt
+  # drwxr-xr-x   3 fangjun  staff    96B Nov  6 21:17 build
+  # drwxr-xr-x   3 fangjun  staff    96B Nov  6 21:18 build-tools
+  # -rw-r--r--   1 fangjun  staff   371B Jan  1  2001 compatible_config.json
+  # drwxr-xr-x  10 fangjun  staff   320B Nov  6 21:18 llvm
+  # -rw-r--r--   1 fangjun  staff    16K Jan  1  2001 nativeapi_syscap_config.json
+  # -rw-r--r--   1 fangjun  staff   5.9K Jan  1  2001 ndk_system_capability.json
+  # -rw-r--r--   1 fangjun  staff   167B Jan  1  2001 oh-uni-package.json
+  # drwxr-xr-x   3 fangjun  staff    96B Nov  6 21:17 sysroot
 fi
 
 if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then
   echo "Please first download Command Line Tools for HarmonyOS"
+  echo "See https://developer.huawei.com/consumer/cn/download/"
+  echo "or"
+  echo "https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main"
+  exit 1
+fi
+
+if [ ! -f $OHOS_SDK_NATIVE_DIR/llvm/bin/aarch64-unknown-linux-ohos-clang ]; then
+  echo "$OHOS_SDK_NATIVE_DIR/llvm/bin/aarch64-unknown-linux-ohos-clang does not exist"
+  echo "Please first download Command Line Tools for HarmonyOS"
+  echo "See https://developer.huawei.com/consumer/cn/download/"
+  echo "or"
+  echo "https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main"
   exit 1
 fi
 
+export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH
 export PATH=$OHOS_SDK_NATIVE_DIR/llvm/bin:$PATH
 
 OHOS_TOOLCHAIN_FILE=$OHOS_SDK_NATIVE_DIR/build/cmake/ohos.toolchain.cmake
@@ -71,7 +77,6 @@ if [ ! -f $OHOS_TOOLCHAIN_FILE ]; then
   exit 1
 fi
 
-
 sleep 1
 onnxruntime_version=1.16.3
 onnxruntime_dir=onnxruntime-ohos-arm64-v8a-$onnxruntime_version
@@ -126,7 +131,6 @@ cmake \
 make -j2
 make install/strip
 cp -fv $onnxruntime_dir/lib/libonnxruntime.so install/lib
-cp -fv $OHOS_SDK_NATIVE_DIR/llvm/lib/aarch64-linux-ohos/libc++_shared.so install/lib
 
 rm -rf install/share
 rm -rf install/lib/pkgconfig
diff --git a/build-ohos-armeabi-v7a.sh b/build-ohos-armeabi-v7a.sh
@@ -35,32 +35,24 @@ if [ -z $OHOS_SDK_NATIVE_DIR ]; then
   # drwxr-xr-x  3 kuangfangjun root    0 Nov  6 22:36 sysroot
 fi
 
-# If you don't want to install commandline tools, you can install the SDK
-# using DevEco Studio. The following uses API version 10 as an example and
-# it has installed the SDK to
-# /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native
-#
-# Remember to select ``native`` when you install the SDK
 if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then
-  OHOS_SDK_NATIVE_DIR=/Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native
-  # export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH
-  # ls -lh /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native/
-  # total 1560
-  # -rw-r--r--   1 fangjun  staff   764K Jan  1  2001 NOTICE.txt
-  # drwxr-xr-x   3 fangjun  staff    96B Nov 19 22:42 build
-  # drwxr-xr-x   3 fangjun  staff    96B Nov 19 22:42 build-tools
-  # drwxr-xr-x  10 fangjun  staff   320B Nov 19 22:42 llvm
-  # -rw-r--r--   1 fangjun  staff   4.0K Jan  1  2001 nativeapi_syscap_config.json
-  # -rw-r--r--   1 fangjun  staff   1.9K Jan  1  2001 ndk_system_capability.json
-  # -rw-r--r--   1 fangjun  staff   169B Jan  1  2001 oh-uni-package.json
-  # drwxr-xr-x   3 fangjun  staff    96B Nov 19 22:42 sysroot
+  echo "Please first download Command Line Tools for HarmonyOS"
+  echo "See https://developer.huawei.com/consumer/cn/download/"
+  echo "or"
+  echo "https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main"
+  exit 1
 fi
 
-if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then
+if [ ! -f $OHOS_SDK_NATIVE_DIR/llvm/bin/armv7-unknown-linux-ohos-clang ]; then
+  echo "$OHOS_SDK_NATIVE_DIR/llvm/bin/armv7-unknown-linux-ohos-clang does not exist"
   echo "Please first download Command Line Tools for HarmonyOS"
+  echo "See https://developer.huawei.com/consumer/cn/download/"
+  echo "or"
+  echo "https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main"
   exit 1
 fi
 
+export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH
 export PATH=$OHOS_SDK_NATIVE_DIR/llvm/bin:$PATH
 
 OHOS_TOOLCHAIN_FILE=$OHOS_SDK_NATIVE_DIR/build/cmake/ohos.toolchain.cmake
@@ -71,7 +63,6 @@ if [ ! -f $OHOS_TOOLCHAIN_FILE ]; then
   exit 1
 fi
 
-
 sleep 1
 onnxruntime_version=1.16.3
 onnxruntime_dir=onnxruntime-ohos-armeabi-v7a-$onnxruntime_version
@@ -130,7 +121,6 @@ cmake \
 make -j2
 make install/strip
 cp -fv $onnxruntime_dir/lib/libonnxruntime.so install/lib
-cp -fv $OHOS_SDK_NATIVE_DIR/llvm/lib/arm-linux-ohos/libc++_shared.so install/lib
 
 rm -rf install/share
 rm -rf install/lib/pkgconfig
diff --git a/build-ohos-x86-64.sh b/build-ohos-x86-64.sh
@@ -19,7 +19,6 @@ cd $dir
 # rm commandline-tools-linux-x64-5.0.5.200.zip
 if [ -z $OHOS_SDK_NATIVE_DIR ]; then
   OHOS_SDK_NATIVE_DIR=/star-fj/fangjun/software/huawei/command-line-tools/sdk/default/openharmony/native/
-  export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH
   # You can find the following content inside OHOS_SDK_NATIVE_DIR
   # ls -lh /star-fj/fangjun/software/huawei/command-line-tools/sdk/default/openharmony/native/
   # total 524K
@@ -35,32 +34,39 @@ if [ -z $OHOS_SDK_NATIVE_DIR ]; then
   # drwxr-xr-x  3 kuangfangjun root    0 Nov  6 22:36 sysroot
 fi
 
-# If you don't want to install commandline tools, you can install the SDK
-# using DevEco Studio. The following uses API version 10 as an example and
-# it has installed the SDK to
-# /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native
-#
-# Remember to select ``native`` when you install the SDK
 if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then
-  OHOS_SDK_NATIVE_DIR=/Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native
-  # export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH
-  # ls -lh /Users/fangjun/software/huawei/OpenHarmony/Sdk/10/native/
-  # total 1560
-  # -rw-r--r--   1 fangjun  staff   764K Jan  1  2001 NOTICE.txt
-  # drwxr-xr-x   3 fangjun  staff    96B Nov 19 22:42 build
-  # drwxr-xr-x   3 fangjun  staff    96B Nov 19 22:42 build-tools
-  # drwxr-xr-x  10 fangjun  staff   320B Nov 19 22:42 llvm
-  # -rw-r--r--   1 fangjun  staff   4.0K Jan  1  2001 nativeapi_syscap_config.json
-  # -rw-r--r--   1 fangjun  staff   1.9K Jan  1  2001 ndk_system_capability.json
-  # -rw-r--r--   1 fangjun  staff   169B Jan  1  2001 oh-uni-package.json
-  # drwxr-xr-x   3 fangjun  staff    96B Nov 19 22:42 sysroot
+  OHOS_SDK_NATIVE_DIR=/Users/fangjun/software/command-line-tools/sdk/default/openharmony/native
+  # (py38) fangjuns-MacBook-Pro:software fangjun$ ls -lh command-line-tools/sdk/default/openharmony/native/
+  # total 752
+  # -rw-r--r--   1 fangjun  staff   341K Jan  1  2001 NOTICE.txt
+  # drwxr-xr-x   3 fangjun  staff    96B Nov  6 21:17 build
+  # drwxr-xr-x   3 fangjun  staff    96B Nov  6 21:18 build-tools
+  # -rw-r--r--   1 fangjun  staff   371B Jan  1  2001 compatible_config.json
+  # drwxr-xr-x  10 fangjun  staff   320B Nov  6 21:18 llvm
+  # -rw-r--r--   1 fangjun  staff    16K Jan  1  2001 nativeapi_syscap_config.json
+  # -rw-r--r--   1 fangjun  staff   5.9K Jan  1  2001 ndk_system_capability.json
+  # -rw-r--r--   1 fangjun  staff   167B Jan  1  2001 oh-uni-package.json
+  # drwxr-xr-x   3 fangjun  staff    96B Nov  6 21:17 sysroot
 fi
 
 if [ ! -d $OHOS_SDK_NATIVE_DIR ]; then
   echo "Please first download Command Line Tools for HarmonyOS"
+  echo "See https://developer.huawei.com/consumer/cn/download/"
+  echo "or"
+  echo "https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main"
+  exit 1
+fi
+
+if [ ! -f $OHOS_SDK_NATIVE_DIR/llvm/bin/x86_64-unknown-linux-ohos-clang ]; then
+  echo "$OHOS_SDK_NATIVE_DIR/llvm/bin/x86_64-unknown-linux-ohos-clang does not exist"
+  echo "Please first download Command Line Tools for HarmonyOS"
+  echo "See https://developer.huawei.com/consumer/cn/download/"
+  echo "or"
+  echo "https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main"
   exit 1
 fi
 
+export PATH=$OHOS_SDK_NATIVE_DIR/build-tools/cmake/bin:$PATH
 export PATH=$OHOS_SDK_NATIVE_DIR/llvm/bin:$PATH
 
 OHOS_TOOLCHAIN_FILE=$OHOS_SDK_NATIVE_DIR/build/cmake/ohos.toolchain.cmake
@@ -71,7 +77,6 @@ if [ ! -f $OHOS_TOOLCHAIN_FILE ]; then
   exit 1
 fi
 
-
 sleep 1
 onnxruntime_version=1.16.3
 onnxruntime_dir=onnxruntime-ohos-x86_64-$onnxruntime_version
@@ -126,7 +131,6 @@ cmake \
 make -j2
 make install/strip
 cp -fv $onnxruntime_dir/lib/libonnxruntime.so install/lib
-cp -fv $OHOS_SDK_NATIVE_DIR/llvm/lib/x86_64-linux-ohos/libc++_shared.so install/lib
 
 rm -rf install/share
 rm -rf install/lib/pkgconfig
diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc
@@ -11,6 +11,10 @@
 #include <utility>
 #include <vector>
 
+#if __OHOS__
+#include "rawfile/raw_file_manager.h"
+#endif
+
 #include "sherpa-onnx/csrc/audio-tagging.h"
 #include "sherpa-onnx/csrc/circular-buffer.h"
 #include "sherpa-onnx/csrc/display.h"
@@ -917,8 +921,8 @@ struct SherpaOnnxVoiceActivityDetector {
   std::unique_ptr<sherpa_onnx::VoiceActivityDetector> impl;
 };
 
-SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetector(
-    const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds) {
+sherpa_onnx::VadModelConfig GetVadModelConfig(
+    const SherpaOnnxVadModelConfig *config) {
   sherpa_onnx::VadModelConfig vad_config;
 
   vad_config.silero_vad.model = SHERPA_ONNX_OR(config->silero_vad.model, "");
@@ -947,9 +951,20 @@ SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetector(
   vad_config.debug = SHERPA_ONNX_OR(config->debug, false);
 
   if (vad_config.debug) {
+#if __OHOS__
+    SHERPA_ONNX_LOGE("%{public}s", vad_config.ToString().c_str());
+#else
     SHERPA_ONNX_LOGE("%s", vad_config.ToString().c_str());
+#endif
   }
 
+  return vad_config;
+}
+
+SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetector(
+    const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds) {
+  auto vad_config = GetVadModelConfig(config);
+
   if (!vad_config.Validate()) {
     SHERPA_ONNX_LOGE("Errors in config");
     return nullptr;
@@ -962,6 +977,25 @@ SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetector(
   return p;
 }
 
+#ifdef __OHOS__
+SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetectorOHOS(
+    const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds,
+    NativeResourceManager *mgr) {
+  if (mgr == nullptr) {
+    return SherpaOnnxCreateVoiceActivityDetector(config,
+                                                 buffer_size_in_seconds);
+  }
+
+  auto vad_config = GetVadModelConfig(config);
+
+  SherpaOnnxVoiceActivityDetector *p = new SherpaOnnxVoiceActivityDetector;
+  p->impl = std::make_unique<sherpa_onnx::VoiceActivityDetector>(
+      mgr, vad_config, buffer_size_in_seconds);
+
+  return p;
+}
+#endif
+
 void SherpaOnnxDestroyVoiceActivityDetector(
     SherpaOnnxVoiceActivityDetector *p) {
   delete p;

diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h
@@ -841,6 +841,21 @@ SHERPA_ONNX_API SherpaOnnxVoiceActivityDetector *
 SherpaOnnxCreateVoiceActivityDetector(const SherpaOnnxVadModelConfig *config,
                                       float buffer_size_in_seconds);
 
+#ifdef __OHOS__
+
+// Return an instance of VoiceActivityDetector.
+// The user has to use SherpaOnnxDestroyVoiceActivityDetector() to free
+// the returned pointer to avoid memory leak.
+//
+// It is for HarmonyOS
+typedef struct NativeResourceManager NativeResourceManager;
+
+SHERPA_ONNX_API SherpaOnnxVoiceActivityDetector *
+SherpaOnnxCreateVoiceActivityDetectorOHOS(
+    const SherpaOnnxVadModelConfig *config, float buffer_size_in_seconds,
+    NativeResourceManager *mgr);
+#endif
+
 SHERPA_ONNX_API void SherpaOnnxDestroyVoiceActivityDetector(
     SherpaOnnxVoiceActivityDetector *p);
 

diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt
@@ -207,6 +207,12 @@ target_link_libraries(sherpa-onnx-core
   kaldi-decoder-core
   ssentencepiece_core
 )
+if(DEFINED OHOS AND x${OHOS} STREQUAL xOHOS)
+  target_link_libraries(sherpa-onnx-core
+    hilog_ndk.z
+    rawfile.z
+  )
+endif()
 
 if(SHERPA_ONNX_ENABLE_GPU)
   target_link_libraries(sherpa-onnx-core

diff --git a/sherpa-onnx/csrc/macros.h b/sherpa-onnx/csrc/macros.h
@@ -8,6 +8,16 @@
 #include <stdlib.h>
 
 #include <utility>
+#if __OHOS__
+#include "hilog/log.h"
+
+#undef LOG_DOMAIN
+#undef LOG_TAG
+
+// https://gitee.com/openharmony/docs/blob/145a084f0b742e4325915e32f8184817927d1251/en/contribute/OpenHarmony-Log-guide.md#hilog-api-usage-specifications
+#define LOG_DOMAIN 0x6666
+#define LOG_TAG "sherpa_onnx"
+#endif
 
 #if __ANDROID_API__ >= 8
 #include "android/log.h"
@@ -19,6 +29,8 @@
     fprintf(stderr, "\n");                                               \
     __android_log_print(ANDROID_LOG_WARN, "sherpa-onnx", ##__VA_ARGS__); \
   } while (0)
+#elif defined(__OHOS__)
+#define SHERPA_ONNX_LOGE(...) OH_LOG_INFO(LOG_APP, ##__VA_ARGS__)
 #elif SHERPA_ONNX_ENABLE_WASM
 #define SHERPA_ONNX_LOGE(...)                        \
   do {                                               \