diff --git a/hybridse/src/udf/dynamic_lib_manager.cc b/hybridse/src/udf/dynamic_lib_manager.cc
index c6a034247cd..37074e44f80 100644
--- a/hybridse/src/udf/dynamic_lib_manager.cc
+++ b/hybridse/src/udf/dynamic_lib_manager.cc
@@ -25,8 +25,8 @@ namespace udf {
 DynamicLibManager::~DynamicLibManager() {
     for (const auto& kv : handle_map_) {
         auto so_handle = kv.second;
-        if (so_handle) {
-            dlclose(so_handle->handle);
+        if (so_handle != nullptr) {
+            dlclose(so_handle);
         }
     }
     handle_map_.clear();
@@ -35,63 +35,53 @@ DynamicLibManager::~DynamicLibManager() {
 base::Status DynamicLibManager::ExtractFunction(const std::string& name, bool is_aggregate, const std::string& file,
                                                 std::vector<void*>* funs) {
     CHECK_TRUE(funs != nullptr, common::kExternalUDFError, "funs is nullptr")
-    std::shared_ptr<DynamicLibHandle> so_handle;
-    {
-        std::lock_guard<std::mutex> lock(mu_);
-        auto iter = handle_map_.find(file);
-        if (iter != handle_map_.end()) {
-            so_handle = iter->second;
-            so_handle->ref_cnt++;
+    void* handle = dlopen(file.c_str(), RTLD_LAZY);
+    if (handle == nullptr) {
+        std::string err_msg;
+        err_msg = "can not open the dynamic library: " + file + ", error: " + dlerror() + ", try to use abs path";
+        LOG(WARNING) << err_msg;
+        // try to use abs path to avoid dlopen failed but it only works in the same path(e.g. in yarn mode)
+        char abs_path_buff[PATH_MAX];
+        if (realpath(file.c_str(), abs_path_buff) == NULL) {
+            err_msg.append(", can not get real path, error: ").append(strerror(errno));
+            return {common::kExternalUDFError, err_msg};
         }
-    }
-    if (!so_handle) {
-        void* handle = dlopen(file.c_str(), RTLD_LAZY);
-        if (handle == nullptr) {
-            std::string err_msg;
-            err_msg = "can not open the dynamic library: " + file + ", error: " + dlerror() + ", try to use abs path";
-            LOG(WARNING) << err_msg;
-            // try to use abs path to avoid dlopen failed but it only works in the same path(e.g. in yarn mode)
-            char abs_path_buff[PATH_MAX];
-            if (realpath(file.c_str(), abs_path_buff) == NULL) {
-                err_msg.append(", can not get real path, error: ").append(strerror(errno));
-                return {common::kExternalUDFError, err_msg};
-            }
 
-            handle = dlopen(abs_path_buff, RTLD_LAZY);
-            if (handle == nullptr) {
-                err_msg.append("dlopen abs path failed, error: ").append(dlerror());
-                return {common::kExternalUDFError, err_msg};
-            }
+        handle = dlopen(abs_path_buff, RTLD_LAZY);
+        if (handle == nullptr) {
+            err_msg.append("dlopen abs path failed, error: ").append(dlerror());
+            return {common::kExternalUDFError, err_msg};
         }
+    }
 
-        so_handle = std::make_shared<DynamicLibHandle>(handle);
+    {
         std::lock_guard<std::mutex> lock(mu_);
-        handle_map_.emplace(file, so_handle);
+        handle_map_[file] = handle;
     }
     if (is_aggregate) {
         std::string init_fun_name = name + "_init";
-        auto init_fun = dlsym(so_handle->handle, init_fun_name.c_str());
+        auto init_fun = dlsym(handle, init_fun_name.c_str());
         if (init_fun == nullptr) {
             RemoveHandler(file);
             return {common::kExternalUDFError, "can not find the init function: " + init_fun_name};
         }
         funs->emplace_back(init_fun);
         std::string update_fun_name = name + "_update";
-        auto update_fun = dlsym(so_handle->handle, update_fun_name.c_str());
+        auto update_fun = dlsym(handle, update_fun_name.c_str());
         if (update_fun == nullptr) {
             RemoveHandler(file);
             return {common::kExternalUDFError, "can not find the update function: " + update_fun_name};
         }
         funs->emplace_back(update_fun);
         std::string output_fun_name = name + "_output";
-        auto output_fun = dlsym(so_handle->handle, output_fun_name.c_str());
+        auto output_fun = dlsym(handle, output_fun_name.c_str());
         if (output_fun == nullptr) {
             RemoveHandler(file);
             return {common::kExternalUDFError, "can not find the output function: " + output_fun_name};
         }
         funs->emplace_back(output_fun);
     } else {
-        auto fun = dlsym(so_handle->handle, name.c_str());
+        auto fun = dlsym(handle, name.c_str());
         if (fun == nullptr) {
             RemoveHandler(file);
             return {common::kExternalUDFError, "can not find the function: " + name};
@@ -102,20 +92,18 @@ base::Status DynamicLibManager::ExtractFunction(const std::string& name, bool is
 }
 
 base::Status DynamicLibManager::RemoveHandler(const std::string& file) {
-    std::shared_ptr<DynamicLibHandle> so_handle;
+    void* handle = nullptr;
     {
         std::lock_guard<std::mutex> lock(mu_);
         if (auto iter = handle_map_.find(file); iter != handle_map_.end()) {
-            iter->second->ref_cnt--;
-            if (iter->second->ref_cnt == 0) {
-                so_handle = iter->second;
-                handle_map_.erase(iter);
+            if (iter->second != nullptr) {
+                handle = iter->second;
             }
         }
     }
-    if (so_handle) {
-        CHECK_TRUE(dlclose(so_handle->handle) == 0, common::kExternalUDFError,
-                   "dlclose run error. file is " + file + ", error: " + dlerror())
+    if (handle != nullptr) {
+        CHECK_TRUE(dlclose(handle) == 0, common::kExternalUDFError,
+                   "can not close the dynamic library: " + file + ", error: " + dlerror())
     }
     return {};
 }
diff --git a/hybridse/src/udf/dynamic_lib_manager.h b/hybridse/src/udf/dynamic_lib_manager.h
index e5c8e4e1271..02dbd0a9a04 100644
--- a/hybridse/src/udf/dynamic_lib_manager.h
+++ b/hybridse/src/udf/dynamic_lib_manager.h
@@ -27,15 +27,6 @@
 namespace hybridse {
 namespace udf {
 
-struct DynamicLibHandle {
-    explicit DynamicLibHandle(void* ptr) {
-        handle = ptr;
-        ref_cnt = 1;
-    }
-    void* handle = nullptr;
-    uint32_t ref_cnt = 0;
-};
-
 class DynamicLibManager {
  public:
     DynamicLibManager() = default;
@@ -50,7 +41,7 @@ class DynamicLibManager {
 
  private:
     std::mutex mu_;
-    std::map<std::string, std::shared_ptr<DynamicLibHandle>> handle_map_;
+    std::map<std::string, void*> handle_map_;
 };
 
 }  // namespace udf