diff --git a/src/audio/module_adapter/module/modules.c b/src/audio/module_adapter/module/modules.c
index 53ee61647829..a91cffeb8026 100644
--- a/src/audio/module_adapter/module/modules.c
+++ b/src/audio/module_adapter/module/modules.c
@@ -13,6 +13,7 @@
 #include <sof/lib_manager.h>
 #include <sof/audio/module_adapter/module/module_interface.h>
 #include <module_api_ver.h>
+#include <zephyr/llext/llext.h>
 
 /* Intel module adapter is an extension to SOF module adapter component that allows to integrate
  * modules developed under IADK (Intel Audio Development Kit) Framework. IADK modules uses uniform
@@ -60,20 +61,23 @@ static int modules_init(struct processing_module *mod)
 	struct module_data *md = &mod->priv;
 	struct comp_dev *dev = mod->dev;
 	const struct ipc4_base_module_cfg *src_cfg = &md->cfg.base_cfg;
+	struct comp_ipc_config *config = &mod->dev->ipc_config;
+	const void *buildinfo = NULL;
+	/*
+	 * Allocate module resources and move it to L2 memory, or just increment
+	 * the use-count.
+	 */
+	uintptr_t module_entry_point = lib_manager_allocate_module(mod, config,
+								   src_cfg, &buildinfo);
 	int ret;
 
-	if (!md->module_adapter) {
-		struct comp_ipc_config *config = &(mod->dev->ipc_config);
-		/* At this point module resources are allocated and it is moved to L2 memory. */
-		const void *buildinfo = NULL;
-		uintptr_t module_entry_point = lib_manager_allocate_module(mod, config,
-									   src_cfg, &buildinfo);
-
-		if (module_entry_point == 0) {
-			comp_err(dev, "modules_init(), lib_manager_allocate_module() failed!");
-			return -EINVAL;
-		}
+	if (module_entry_point == 0) {
+		comp_err(dev, "modules_init(), lib_manager_allocate_module() failed!");
+		return -EINVAL;
+	}
 
+	if (!md->module_adapter) {
+		/* First load */
 		uint32_t module_id = IPC4_MOD_ID(config->id);
 		uint32_t instance_id = IPC4_INST_ID(config->id);
 		/* Connect loadable module interfaces with module adapter entity. */
@@ -277,6 +281,11 @@ static int modules_process_raw(struct processing_module *mod,
 	return ret;
 }
 
+static void lib_manager_module_free_cb(struct llext *, const void *ctx)
+{
+	lib_manager_free_module(NULL, ctx);
+}
+
 /**
  * \brief modules_free.
  * \param[in] mod - processing module pointer.
@@ -302,6 +311,10 @@ static int modules_free(struct processing_module *mod)
 	rfree(md->mpd.in_buff);
 	rfree(md->mpd.out_buff);
 
+	if (md->llext)
+		/* This uses a reference counter to decide, when to unload */
+		llext_unload(&md->llext, lib_manager_module_free_cb, &dev->ipc_config);
+
 #if 0
 	/*
 	 * Module unloading isn't implemented yet, it has to be done from a
diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h
index c5e9321dff29..2f51ccfa83df 100644
--- a/src/include/sof/lib_manager.h
+++ b/src/include/sof/lib_manager.h
@@ -151,7 +151,7 @@ uintptr_t lib_manager_allocate_module(struct processing_module *proc,
  * Function is responsible to free module resources in HP memory.
  */
 int lib_manager_free_module(struct processing_module *proc,
-			    struct comp_ipc_config *ipc_config);
+			    const struct comp_ipc_config *ipc_config);
 /*
  * \brief Load library
  *
diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c
index fa1f68669af8..8048e7d96d55 100644
--- a/src/library_manager/lib_manager.c
+++ b/src/library_manager/lib_manager.c
@@ -275,8 +275,16 @@ uintptr_t lib_manager_allocate_module(struct processing_module *proc,
 						       SOF_MAN_ELF_TEXT_OFFSET + 0x8000,
 						       mod_size);
 	ret = llext_load(&ebl.loader, mod->name, &proc->priv.llext, false);
-	if (ret < 0)
-		return ret;
+	if (ret > 0)
+		/* Already loaded */
+		return proc->priv.module_entry_point;
+	if (ret < 0) {
+		tr_err(&lib_manager_tr,
+		       "lib_manager_allocate_module(): llext loading failed: %d", ret);
+		return 0;
+	}
+
+	assert(proc->priv.llext);
 
 	mod->segment[SOF_MAN_SEGMENT_TEXT].v_base_addr = ebl.loader.sects[LLEXT_SECT_TEXT].sh_addr;
 	mod->segment[SOF_MAN_SEGMENT_TEXT].flags.r.length = ebl.loader.sects[LLEXT_SECT_TEXT].sh_size;
@@ -301,29 +309,38 @@ uintptr_t lib_manager_allocate_module(struct processing_module *proc,
 
 	/* Map .text and the rest as .data */
 	ret = lib_manager_load_module(module_id, mod, desc);
-	if (ret < 0)
-		return 0;
+	if (ret < 0) {
+		tr_err(&lib_manager_tr,
+		       "lib_manager_allocate_module(): module loading failed: %d", ret);
+		goto e_llext;
+	}
 
+	/* Map .bss */
 	ret = lib_manager_allocate_module_instance(module_id, IPC4_INST_ID(ipc_config->id), mod);
 	if (ret < 0) {
 		tr_err(&lib_manager_tr,
 		       "lib_manager_allocate_module(): module allocation failed: %d", ret);
-		return 0;
+		goto e_bss;
 	}
 
 	ssize_t mod_o = llext_find_section(&ebl.loader, ".module");
-	const struct sof_man_module_manifest *mod_manifest = llext_peek(&ebl.loader,
-									 mod_o);
+	const struct sof_man_module_manifest *mod_manifest = llext_peek(&ebl.loader, mod_o);
 
 	ssize_t binfo_o = llext_find_section(&ebl.loader, ".mod_buildinfo");
 	if (binfo_o)
 		*buildinfo = llext_peek(&ebl.loader, binfo_o);
 
 	return mod_manifest->module.entry_point;
+
+e_bss:
+	lib_manager_unload_module(module_id, mod, desc);
+e_llext:
+	llext_unload(&proc->priv.llext, NULL, NULL);
+	return 0;
 }
 
 int lib_manager_free_module(struct processing_module *proc,
-			    struct comp_ipc_config *ipc_config)
+			    const struct comp_ipc_config *ipc_config)
 {
 	struct sof_man_fw_desc *desc;
 	struct sof_man_module *mod;