[Userspace LL] scattered commits from #10945 without dependencies

src/audio/buffers/comp_buffer.c

@@ -11,6 +11,7 @@
 #include <sof/audio/sink_api.h>
 #include <sof/audio/source_api.h>
 #include <sof/audio/sink_source_utils.h>
+#include <sof/audio/module_adapter/module/generic.h>
 #include <rtos/userspace_helper.h>
 #include <sof/common.h>
 #include <rtos/interrupt.h>
@@ -165,8 +166,7 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer)
 
 	if (alloc && alloc->vreg) {
 		vregion_free(alloc->vreg, buffer);
-		if (!vregion_put(alloc->vreg))
-			rfree(alloc);
+		module_adapter_vreg_free(alloc);
 	} else {
 		sof_heap_free(alloc ? alloc->heap : NULL, buffer);
 	}

src/audio/module_adapter/module_adapter.c

@@ -58,14 +58,84 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv,
 #define PAGE_SZ HOST_PAGE_SIZE
 #endif
 
-static struct vregion *module_adapter_dp_heap_new(const struct comp_ipc_config *config,
-						  size_t *heap_size)
+struct vregion *z_impl_module_adapter_vreg_new(const struct comp_ipc_config *config,
+					       uintptr_t *vreg_start, size_t *vreg_size)
 {
 	/* src-lite with 8 channels has been seen allocating 14k in one go */
 	/* FIXME: the size will be derived from configuration */
 	const size_t buf_size = 28 * 1024;
+	struct vregion *vr = vregion_create(buf_size);
 
-	return vregion_create(buf_size);
+	if (!vr)
+		return NULL;
+
+#ifdef CONFIG_SOF_USERSPACE_LL
+	vregion_mem_info(vr, vreg_size, vreg_start);
+
+	/*
+	 * In the userspace LL case allocations are also performed by the
+	 * userspace IPC thread, which is also the one, executing this syscall
+	 */
+	struct k_mem_partition cached_part = {
+		.start = *vreg_start,
+		.size = *vreg_size,
+		.attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB,
+	};
+	int ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cached_part);
+
+	if (ret < 0) {
+		vregion_put(vr);
+		return NULL;
+	}
+
+	struct k_mem_partition uncached_part = {
+		.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)cached_part.start),
+		.size = cached_part.size,
+		.attr = K_MEM_PARTITION_P_RW_U_RW,
+	};
+
+	ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &uncached_part);
+	if (ret < 0) {
+		k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &cached_part);
+		vregion_put(vr);
+		return NULL;
+	}
+#else
+	ARG_UNUSED(vreg_start);
+	ARG_UNUSED(vreg_size);
+#endif
+
+	return vr;
+}
+
+void z_impl_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc)
+{
+#ifdef CONFIG_SOF_USERSPACE_LL
+	struct k_mem_partition part = {
+		.start = alloc->vreg_start,
+		.size = alloc->vreg_size,
+		.attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB,
+	};
+
+	k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &part);
+
+	part.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)part.start);
+	part.attr = K_MEM_PARTITION_P_RW_U_RW;
+
+	k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &part);
+#else
+	ARG_UNUSED(alloc);
+#endif
+}
+
+void module_adapter_vreg_free(struct mod_alloc_ctx *alloc)
+{
+	if (vregion_put(alloc->vreg))
+		return;
+
+	module_adapter_vreg_unmap(alloc);
+
+	sof_heap_free(alloc->heap, alloc);
 }
 
 static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv,
@@ -84,24 +154,28 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 	 */
 	uint32_t flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ?
 		SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER;
-	size_t heap_size;
+	size_t vreg_size = 0;
+	uintptr_t vreg_start = 0;
 
 	if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_SOF_VREGIONS) &&
 	    IS_ENABLED(CONFIG_USERSPACE) && !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) {
-		mod_vreg = module_adapter_dp_heap_new(config, &heap_size);
+		mod_vreg = module_adapter_vreg_new(config, &vreg_start, &vreg_size);
 		if (!mod_vreg) {
 			comp_cl_err(drv, "Failed to allocate DP module heap / vregion");
 			return NULL;
 		}
+#ifdef CONFIG_SOF_USERSPACE_LL
+		mod_heap = sof_sys_user_heap_get();
+#else
 		mod_heap = NULL;
+#endif
 	} else {
 #ifdef CONFIG_SOF_USERSPACE_LL
 		mod_heap = sof_sys_user_heap_get();
 		comp_cl_dbg(drv, "using ll user heap for module");
 #else
 		mod_heap = drv->user_heap;
 #endif
-		heap_size = 0;
 		mod_vreg = NULL;
 	}
 
@@ -125,6 +199,8 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 	memset(mod, 0, sizeof(*mod));
 	alloc->heap = mod_heap;
 	alloc->vreg = mod_vreg;
+	alloc->vreg_start = vreg_start;
+	alloc->vreg_size = vreg_size;
 	mod->priv.resources.alloc = alloc;
 	mod_resource_init(mod);
 
@@ -165,6 +241,25 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 	return NULL;
 }
 
+#ifdef CONFIG_USERSPACE
+#include <zephyr/internal/syscall_handler.h>
+struct vregion *z_vrfy_module_adapter_vreg_new(const struct comp_ipc_config *config,
+					       uintptr_t *vreg_start, size_t *vreg_size)
+{
+	K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_start, sizeof(*vreg_start)));
+	K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_size, sizeof(*vreg_size)));
+	K_OOPS(K_SYSCALL_MEMORY_READ(config, sizeof(*config)));
+	return z_impl_module_adapter_vreg_new(config, vreg_start, vreg_size);
+}
+#include <zephyr/syscalls/module_adapter_vreg_new_mrsh.c>
+void z_vrfy_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc)
+{
+	K_OOPS(K_SYSCALL_MEMORY_READ(alloc, sizeof(*alloc)));
+	z_impl_module_adapter_vreg_unmap(alloc);
+}
+#include <zephyr/syscalls/module_adapter_vreg_unmap_mrsh.c>
+#endif
+
 static void module_adapter_mem_free(struct processing_module *mod)
 {
 	struct mod_alloc_ctx *alloc = mod->priv.resources.alloc;
@@ -182,8 +277,7 @@ static void module_adapter_mem_free(struct processing_module *mod)
 
 		vregion_free(mod_vreg, mod->dev);
 		vregion_free(mod_vreg, mod);
-		if (!vregion_put(mod_vreg))
-			sof_heap_free(alloc->heap, alloc);
+		module_adapter_vreg_free(alloc);
 	} else {
 		sof_heap_free(mod_heap, mod->dev);
 		sof_heap_free(mod_heap, mod);

src/include/sof/audio/component.h

@@ -593,6 +593,7 @@ struct comp_driver {
 							  * Currently used by module_adapter.
 							  */
 	struct k_heap *user_heap;			/**< Userspace heap */
+	struct sof_uuid uid_cp;				/**< UUID copy for LLEXT modules */
 };
 
 /** \brief Holds constant pointer to component driver */

src/include/sof/audio/module_adapter/module/generic.h

@@ -193,11 +193,18 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t
 #endif
 void mod_resource_init(struct processing_module *mod);
 void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start);
+void module_adapter_vreg_free(struct mod_alloc_ctx *alloc);
 #if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION)
+__syscall struct vregion *module_adapter_vreg_new(const struct comp_ipc_config *config,
+						  uintptr_t *vreg_start, size_t *vreg_size);
+__syscall void module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc);
 __syscall void *mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size,
 			      size_t alignment);
 __syscall int mod_free(struct processing_module *mod, const void *ptr);
 #else
+struct vregion *z_impl_module_adapter_vreg_new(const struct comp_ipc_config *config,
+					       uintptr_t *vreg_start, size_t *vreg_size);
+void z_impl_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc);
 void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size,
 			   size_t alignment);
 int z_impl_mod_free(struct processing_module *mod, const void *ptr);

src/include/sof/lib_manager.h

@@ -114,7 +114,9 @@ struct lib_manager_module {
 	struct llext *llext; /* Zephyr loadable extension context */
 	struct llext_buf_loader *ebl; /* Zephyr loadable extension buffer loader */
 	unsigned int n_dependent; /* For auxiliary modules: number of dependents */
+	unsigned int n_mod;
 	bool mapped;
+	bool domain_dp;
 	struct lib_manager_segment_desc segment[LIB_MANAGER_N_SEGMENTS];
 };
 
@@ -217,6 +219,25 @@ void lib_manager_get_instance_bss_address(uint32_t instance_id,
  */
 int lib_manager_load_library(uint32_t dma_id, uint32_t lib_id, uint32_t type);
 
+struct userspace_context;
+/*
+ * \brief Allocate the module and start the agent if needed
+ */
+int lib_manager_mod_create_priv(const struct comp_driver *drv,
+				const struct comp_ipc_config *config,
+				const void *spec, void **adapter_priv,
+				struct userspace_context **userspace,
+				const struct module_interface **ops);
+
+#if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION)
+__syscall int lib_manager_free_module(const uint32_t component_id);
+
+#include <zephyr/syscalls/lib_manager.h>
+#else
+int z_impl_lib_manager_free_module(const uint32_t component_id);
+#define lib_manager_free_module z_impl_lib_manager_free_module
+#endif
+
 /*
  * \brief Initialize message
  *

src/ipc/ipc-common.c

@@ -318,7 +318,6 @@ void ipc_schedule_process(struct ipc *ipc)
 #define IPC_USER_EVENT_CMD      BIT(0)
 #define IPC_USER_EVENT_STOP     BIT(1)
 
-static struct k_thread ipc_user_thread;
 static K_THREAD_STACK_DEFINE(ipc_user_stack, CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE);
 
 /**
@@ -419,10 +418,59 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3)
 	}
 }
 
+__cold static int ipc_user_init_thread(struct ipc_user *ipc_user)
+{
+	char thread_name[] = "ll_user0";
+	int ret;
+
+	assert_can_be_cold();
+
+	/* Allocate kernel objects for the user-space thread */
+	ipc_user->event = k_object_alloc(K_OBJ_EVENT);
+	if (!ipc_user->event) {
+		LOG_ERR("user IPC event alloc failed");
+		return -ENOMEM;
+	}
+	k_event_init(ipc_user->event);
+
+	ipc_user->thread = k_object_alloc(K_OBJ_THREAD);
+	if (!ipc_user->thread) {
+		LOG_ERR("user IPC thread alloc failed");
+		ret = -ENOMEM;
+		goto e_event;
+	}
+
+	k_thread_create(ipc_user->thread, ipc_user_stack,
+			CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE,
+			ipc_user_thread_fn, ipc_user, NULL, NULL,
+			-1, K_USER, K_FOREVER);
+
+	k_thread_cpu_pin(ipc_user->thread, PLATFORM_PRIMARY_CORE_ID);
+	k_thread_name_set(ipc_user->thread, thread_name);
+
+	/*
+	 * Each userspace IPC thread must be able to wait on its private event
+	 * and signal completion on the primary core semaphore
+	 */
+	k_thread_access_grant(ipc_user->thread, ipc_user->sem, ipc_user->event);
+	user_grant_dai_access_all(ipc_user->thread);
+	user_grant_dma_access_all(ipc_user->thread);
+	k_mem_domain_add_thread(zephyr_ll_mem_domain(), ipc_user->thread);
+	user_ll_grant_access(ipc_user->thread, PLATFORM_PRIMARY_CORE_ID);
+
+	return 0;
+
+e_event:
+	k_object_free(ipc_user->event);
+
+	return ret;
+}
+
 __cold static void ipc_user_init(void)
 {
 	struct ipc *ipc = ipc_get();
-	struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), SOF_MEM_FLAG_USER,
+	struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(),
+						   SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,
 						   sizeof(*ipc_user), 0);
 	int ret;
 
@@ -443,38 +491,22 @@ __cold static void ipc_user_init(void)
 
 	k_sem_init(ipc_user->sem, 0, 1);
 
-	/* Allocate kernel objects for the user-space thread */
-	ipc_user->event = k_object_alloc(K_OBJ_EVENT);
-	if (!ipc_user->event) {
-		LOG_ERR("user IPC event alloc failed");
+	ret = ipc_user_init_thread(ipc_user);
+	if (ret < 0) {
+		LOG_ERR("user IPC thread initialization failed");
 		sof_panic(SOF_IPC_PANIC_IPC);
 	}
-	k_event_init(ipc_user->event);
 
-	k_thread_create(&ipc_user_thread, ipc_user_stack,
-			CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE,
-			ipc_user_thread_fn, ipc_user, NULL, NULL,
-			-1, K_USER, K_FOREVER);
-
-	ipc_user->thread = &ipc_user_thread;
-	k_thread_access_grant(&ipc_user_thread, ipc_user->sem, ipc_user->event);
-	user_grant_dai_access_all(&ipc_user_thread);
-	user_grant_dma_access_all(&ipc_user_thread);
-	ret = user_access_to_mailbox(zephyr_ll_mem_domain(), &ipc_user_thread);
+	ret = user_access_to_mailbox(zephyr_ll_mem_domain(), ipc_user->thread);
 	if (ret < 0) {
 		LOG_ERR("ipc user: mailbox access grant failed: %d", ret);
 		sof_panic(SOF_IPC_PANIC_IPC);
 	}
-	user_ll_grant_access(&ipc_user_thread, PLATFORM_PRIMARY_CORE_ID);
-	k_mem_domain_add_thread(zephyr_ll_mem_domain(), &ipc_user_thread);
-
-	k_thread_cpu_pin(&ipc_user_thread, PLATFORM_PRIMARY_CORE_ID);
-	k_thread_name_set(&ipc_user_thread, "ipc_user");
 
 	/* Store references in ipc struct so kernel handler can forward commands */
 	ipc->ipc_user_pdata = ipc_user;
 
-	k_thread_start(&ipc_user_thread);
+	k_thread_start(ipc_user->thread);
 
 	struct task *task = zephyr_ll_task_alloc();
 
@@ -486,7 +518,7 @@ __cold static void ipc_user_init(void)
 	 * Needed so user-space dai_common_new() can call
 	 * k_thread_access_grant(audio_thread, dai_mutex) from user context.
 	 */
-	k_thread_access_grant(&ipc_user_thread, ipc_user->audio_thread);
+	k_thread_access_grant(ipc_user->thread, ipc_user->audio_thread);
 
 	/* Wait for user thread startup — consumes the initial k_sem_give from thread */
 	k_sem_take(ipc->ipc_user_pdata->sem, K_FOREVER);

src/ipc/ipc-helper.c

@@ -297,7 +297,9 @@ __cold int ipc_comp_free(struct ipc *ipc, uint32_t comp_id)
 	struct comp_buffer *buffer;
 	struct comp_buffer *safe;
 	struct list_item *clist;
+#ifndef CONFIG_SOF_USERSPACE_LL
 	uint32_t flags;
+#endif
 
 	assert_can_be_cold();