diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h
index 5e0e47fdacb7..e8dfef8d7048 100644
--- a/src/include/sof/lib_manager.h
+++ b/src/include/sof/lib_manager.h
@@ -78,9 +78,11 @@ struct ipc_lib_msg {
 	struct list_item list;
 };
 
+struct lib_manager_mod_ctx;
+
 struct ext_library {
 	struct k_spinlock lock;	/* last locking CPU record */
-	struct sof_man_fw_desc *desc[LIB_MANAGER_MAX_LIBS];
+	struct lib_manager_mod_ctx *desc[LIB_MANAGER_MAX_LIBS];
 	uint32_t mods_exec_load_cnt;
 	struct ipc_lib_msg *lib_notif_pool;
 	uint32_t lib_notif_count;
diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c
index 7d458b48982b..c7e4a855fd47 100644
--- a/src/library_manager/lib_manager.c
+++ b/src/library_manager/lib_manager.c
@@ -47,6 +47,42 @@ struct lib_manager_dma_ext {
 
 static struct ext_library loader_ext_lib;
 
+struct lib_manager_mod_ctx {
+	struct sof_man_fw_desc *desc;
+	size_t segment_size[3];
+};
+
+static struct lib_manager_mod_ctx *lib_manager_get_mod_ctx(int module_id)
+{
+	uint32_t lib_id = LIB_MANAGER_GET_LIB_ID(module_id);
+	struct ext_library *_ext_lib = ext_lib_get();
+
+	return _ext_lib->desc[lib_id];
+}
+
+struct sof_man_fw_desc *lib_manager_get_library_module_desc(int module_id)
+{
+	struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id);
+	uint8_t *buffptr = (uint8_t *)(ctx ? ctx->desc : NULL);
+
+	if (!buffptr)
+		return NULL;
+	return (struct sof_man_fw_desc *)(buffptr + SOF_MAN_ELF_TEXT_OFFSET);
+}
+
+static void lib_manager_update_sof_ctx(struct sof_man_fw_desc *desc, uint32_t lib_id)
+{
+	struct ext_library *_ext_lib = ext_lib_get();
+	/* Never freed, will panic if fails */
+	struct lib_manager_mod_ctx *ctx = rmalloc(SOF_MEM_ZONE_SYS, 0, SOF_MEM_CAPS_RAM,
+						  sizeof(*ctx));
+
+	ctx->desc = desc;
+
+	_ext_lib->desc[lib_id] = ctx;
+	/* TODO: maybe need to call dcache_writeback here? */
+}
+
 #if IS_ENABLED(CONFIG_MM_DRV)
 
 #define PAGE_SZ		CONFIG_MM_DRV_PAGE_SIZE
@@ -158,20 +194,18 @@ static int lib_manager_load_module(uint32_t module_id, struct sof_man_module *mo
 	struct ext_library *ext_lib = ext_lib_get();
 	uint32_t lib_id = LIB_MANAGER_GET_LIB_ID(module_id);
 	size_t load_offset = (size_t)((void *)ext_lib->desc[lib_id]);
+	struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id);
 	void __sparse_cache *va_base_text = (void __sparse_cache *)
 		mod->segment[SOF_MAN_SEGMENT_TEXT].v_base_addr;
 	void *src_txt = (void *)(mod->segment[SOF_MAN_SEGMENT_TEXT].file_offset + load_offset);
-	size_t st_text_size = mod->segment[SOF_MAN_SEGMENT_TEXT].flags.r.length;
+	size_t st_text_size = ctx->segment_size[SOF_MAN_SEGMENT_TEXT];
 	void __sparse_cache *va_base_rodata = (void __sparse_cache *)
 		mod->segment[SOF_MAN_SEGMENT_RODATA].v_base_addr;
 	void *src_rodata =
 		(void *)(mod->segment[SOF_MAN_SEGMENT_RODATA].file_offset + load_offset);
-	size_t st_rodata_size = mod->segment[SOF_MAN_SEGMENT_RODATA].flags.r.length;
+	size_t st_rodata_size = ctx->segment_size[SOF_MAN_SEGMENT_RODATA];
 	int ret;
 
-	st_text_size = st_text_size * PAGE_SZ;
-	st_rodata_size = st_rodata_size * PAGE_SZ;
-
 	/* Copy Code */
 	ret = lib_manager_load_data_from_storage(va_base_text, src_txt, st_text_size,
 						 SYS_MM_MEM_PERM_RW | SYS_MM_MEM_PERM_EXEC);
@@ -203,18 +237,16 @@ static int lib_manager_load_module(uint32_t module_id, struct sof_man_module *mo
 static int lib_manager_unload_module(uint32_t module_id, struct sof_man_module *mod,
 				     struct sof_man_fw_desc *desc)
 {
+	struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id);
 	uint32_t lib_id = LIB_MANAGER_GET_LIB_ID(module_id);
 	void __sparse_cache *va_base_text = (void __sparse_cache *)
 		mod->segment[SOF_MAN_SEGMENT_TEXT].v_base_addr;
-	size_t st_text_size = mod->segment[SOF_MAN_SEGMENT_TEXT].flags.r.length;
+	size_t st_text_size = ctx->segment_size[SOF_MAN_SEGMENT_TEXT];
 	void __sparse_cache *va_base_rodata = (void __sparse_cache *)
 		mod->segment[SOF_MAN_SEGMENT_RODATA].v_base_addr;
-	size_t st_rodata_size = mod->segment[SOF_MAN_SEGMENT_RODATA].flags.r.length;
+	size_t st_rodata_size = ctx->segment_size[SOF_MAN_SEGMENT_RODATA];
 	int ret;
 
-	st_text_size = st_text_size * PAGE_SZ;
-	st_rodata_size = st_rodata_size * PAGE_SZ;
-
 	ret = lib_manager_align_unmap(va_base_text, st_text_size);
 	if (ret < 0)
 		return ret;
@@ -253,15 +285,10 @@ static void __sparse_cache *lib_manager_get_instance_bss_address(uint32_t module
 static int lib_manager_allocate_module_instance(uint32_t module_id, uint32_t instance_id,
 						uint32_t is_pages, struct sof_man_module *mod)
 {
+	struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id);
+	size_t bss_size = ctx->segment_size[SOF_MAN_SEGMENT_BSS];
 	void __sparse_cache *va_base = lib_manager_get_instance_bss_address(module_id,
 									    instance_id, mod);
-	size_t bss_size;
-
-	if (mod->type.load_type == SOF_MAN_MOD_TYPE_LLEXT)
-		bss_size = mod->segment[SOF_MAN_SEGMENT_BSS].flags.r.length * PAGE_SZ;
-	else
-		bss_size = (mod->segment[SOF_MAN_SEGMENT_BSS].flags.r.length /
-			    mod->instance_max_count) * PAGE_SZ;
 
 	if ((is_pages * PAGE_SZ) > bss_size) {
 		tr_err(&lib_manager_tr,
@@ -282,16 +309,11 @@ static int lib_manager_allocate_module_instance(uint32_t module_id, uint32_t ins
 static int lib_manager_free_module_instance(uint32_t module_id, uint32_t instance_id,
 					    struct sof_man_module *mod)
 {
-	size_t bss_size;
+	struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id);
+	size_t bss_size = ctx->segment_size[SOF_MAN_SEGMENT_BSS];
 	void __sparse_cache *va_base = lib_manager_get_instance_bss_address(module_id,
 									    instance_id, mod);
 
-	if (mod->type.load_type == SOF_MAN_MOD_TYPE_LLEXT)
-		bss_size = mod->segment[SOF_MAN_SEGMENT_BSS].flags.r.length * PAGE_SZ;
-	else
-		bss_size = (mod->segment[SOF_MAN_SEGMENT_BSS].flags.r.length /
-			    mod->instance_max_count) * PAGE_SZ;
-
 	/* Unmap bss memory. */
 	return lib_manager_align_unmap(va_base, bss_size);
 }
@@ -306,12 +328,12 @@ uint32_t lib_manager_allocate_module(const struct comp_driver *drv,
 	int ret;
 	uint32_t module_id = IPC4_MOD_ID(ipc_config->id);
 	uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id);
+	struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id);
 
-	tr_dbg(&lib_manager_tr, "lib_manager_allocate_module(): mod_id: %#x",
-	       ipc_config->id);
+	tr_dbg(&lib_manager_tr, "lib_manager_allocate_module(): mod_id: %#x", module_id);
 
 	desc = lib_manager_get_library_module_desc(module_id);
-	if (!desc) {
+	if (!ctx || !desc) {
 		tr_err(&lib_manager_tr,
 		       "lib_manager_allocate_module(): failed to get module descriptor");
 		return 0;
@@ -319,6 +341,13 @@ uint32_t lib_manager_allocate_module(const struct comp_driver *drv,
 
 	mod = (struct sof_man_module *)((char *)desc + SOF_MAN_MODULE_OFFSET(entry_index));
 
+	for (unsigned int i = 0; i < ARRAY_SIZE(ctx->segment_size); i++) {
+		ctx->segment_size[i] = mod->segment[i].flags.r.length * PAGE_SZ;
+
+		if (i == SOF_MAN_SEGMENT_BSS && mod->type.load_type != SOF_MAN_MOD_TYPE_LLEXT)
+			ctx->segment_size[i] /= mod->instance_max_count;
+	}
+
 	ret = lib_manager_load_module(module_id, mod, desc);
 	if (ret < 0)
 		return 0;
@@ -391,25 +420,6 @@ void lib_manager_init(void)
 		sof->ext_library = &loader_ext_lib;
 }
 
-struct sof_man_fw_desc *lib_manager_get_library_module_desc(int module_id)
-{
-	uint32_t lib_id = LIB_MANAGER_GET_LIB_ID(module_id);
-	struct ext_library *_ext_lib = ext_lib_get();
-	uint8_t *buffptr = (uint8_t *)_ext_lib->desc[lib_id];
-
-	if (!buffptr)
-		return NULL;
-	return (struct sof_man_fw_desc *)(buffptr + SOF_MAN_ELF_TEXT_OFFSET);
-}
-
-static void lib_manager_update_sof_ctx(struct sof_man_fw_desc *desc, uint32_t lib_id)
-{
-	struct ext_library *_ext_lib = ext_lib_get();
-
-	_ext_lib->desc[lib_id] = desc;
-	/* TODO: maybe need to call here dcache_writeback here? */
-}
-
 #if CONFIG_INTEL_MODULES
 int lib_manager_register_module(struct sof_man_fw_desc *desc, int module_id)
 {