diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index 8a3d44133d4b..f64b74add169 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -165,8 +166,7 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer) if (alloc && alloc->vreg) { vregion_free(alloc->vreg, buffer); - if (!vregion_put(alloc->vreg)) - rfree(alloc); + module_adapter_vreg_free(alloc); } else { sof_heap_free(alloc ? alloc->heap : NULL, buffer); } diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index 53e8c75c42e2..0e28447c922c 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -58,14 +58,84 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv, #define PAGE_SZ HOST_PAGE_SIZE #endif -static struct vregion *module_adapter_dp_heap_new(const struct comp_ipc_config *config, - size_t *heap_size) +struct vregion *z_impl_module_adapter_vreg_new(const struct comp_ipc_config *config, + uintptr_t *vreg_start, size_t *vreg_size) { /* src-lite with 8 channels has been seen allocating 14k in one go */ /* FIXME: the size will be derived from configuration */ const size_t buf_size = 28 * 1024; + struct vregion *vr = vregion_create(buf_size); - return vregion_create(buf_size); + if (!vr) + return NULL; + +#ifdef CONFIG_SOF_USERSPACE_LL + vregion_mem_info(vr, vreg_size, vreg_start); + + /* + * In the userspace LL case allocations are also performed by the + * userspace IPC thread, which is also the one, executing this syscall + */ + struct k_mem_partition cached_part = { + .start = *vreg_start, + .size = *vreg_size, + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + }; + int ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cached_part); + + if (ret < 0) { + vregion_put(vr); + return NULL; + } + + struct k_mem_partition uncached_part = { + .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)cached_part.start), + .size = cached_part.size, + .attr = K_MEM_PARTITION_P_RW_U_RW, + }; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &uncached_part); + if (ret < 0) { + k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &cached_part); + vregion_put(vr); + return NULL; + } +#else + ARG_UNUSED(vreg_start); + ARG_UNUSED(vreg_size); +#endif + + return vr; +} + +void z_impl_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc) +{ +#ifdef CONFIG_SOF_USERSPACE_LL + struct k_mem_partition part = { + .start = alloc->vreg_start, + .size = alloc->vreg_size, + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + }; + + k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &part); + + part.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)part.start); + part.attr = K_MEM_PARTITION_P_RW_U_RW; + + k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &part); +#else + ARG_UNUSED(alloc); +#endif +} + +void module_adapter_vreg_free(struct mod_alloc_ctx *alloc) +{ + if (vregion_put(alloc->vreg)) + return; + + module_adapter_vreg_unmap(alloc); + + sof_heap_free(alloc->heap, alloc); } static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv, @@ -84,16 +154,21 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv */ uint32_t flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ? SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER; - size_t heap_size; + size_t vreg_size = 0; + uintptr_t vreg_start = 0; if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_SOF_VREGIONS) && IS_ENABLED(CONFIG_USERSPACE) && !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) { - mod_vreg = module_adapter_dp_heap_new(config, &heap_size); + mod_vreg = module_adapter_vreg_new(config, &vreg_start, &vreg_size); if (!mod_vreg) { comp_cl_err(drv, "Failed to allocate DP module heap / vregion"); return NULL; } +#ifdef CONFIG_SOF_USERSPACE_LL + mod_heap = sof_sys_user_heap_get(); +#else mod_heap = NULL; +#endif } else { #ifdef CONFIG_SOF_USERSPACE_LL mod_heap = sof_sys_user_heap_get(); @@ -101,7 +176,6 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv #else mod_heap = drv->user_heap; #endif - heap_size = 0; mod_vreg = NULL; } @@ -125,6 +199,8 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv memset(mod, 0, sizeof(*mod)); alloc->heap = mod_heap; alloc->vreg = mod_vreg; + alloc->vreg_start = vreg_start; + alloc->vreg_size = vreg_size; mod->priv.resources.alloc = alloc; mod_resource_init(mod); @@ -165,6 +241,25 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv return NULL; } +#ifdef CONFIG_USERSPACE +#include +struct vregion *z_vrfy_module_adapter_vreg_new(const struct comp_ipc_config *config, + uintptr_t *vreg_start, size_t *vreg_size) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_start, sizeof(*vreg_start))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_size, sizeof(*vreg_size))); + K_OOPS(K_SYSCALL_MEMORY_READ(config, sizeof(*config))); + return z_impl_module_adapter_vreg_new(config, vreg_start, vreg_size); +} +#include +void z_vrfy_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc) +{ + K_OOPS(K_SYSCALL_MEMORY_READ(alloc, sizeof(*alloc))); + z_impl_module_adapter_vreg_unmap(alloc); +} +#include +#endif + static void module_adapter_mem_free(struct processing_module *mod) { struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; @@ -182,8 +277,7 @@ static void module_adapter_mem_free(struct processing_module *mod) vregion_free(mod_vreg, mod->dev); vregion_free(mod_vreg, mod); - if (!vregion_put(mod_vreg)) - sof_heap_free(alloc->heap, alloc); + module_adapter_vreg_free(alloc); } else { sof_heap_free(mod_heap, mod->dev); sof_heap_free(mod_heap, mod); diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index db8a69e63646..928590f43e2a 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -593,6 +593,7 @@ struct comp_driver { * Currently used by module_adapter. */ struct k_heap *user_heap; /**< Userspace heap */ + struct sof_uuid uid_cp; /**< UUID copy for LLEXT modules */ }; /** \brief Holds constant pointer to component driver */ diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index 6740593a7cf1..122409884fa6 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -193,11 +193,18 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t #endif void mod_resource_init(struct processing_module *mod); void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start); +void module_adapter_vreg_free(struct mod_alloc_ctx *alloc); #if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +__syscall struct vregion *module_adapter_vreg_new(const struct comp_ipc_config *config, + uintptr_t *vreg_start, size_t *vreg_size); +__syscall void module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc); __syscall void *mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, size_t alignment); __syscall int mod_free(struct processing_module *mod, const void *ptr); #else +struct vregion *z_impl_module_adapter_vreg_new(const struct comp_ipc_config *config, + uintptr_t *vreg_start, size_t *vreg_size); +void z_impl_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc); void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size, size_t alignment); int z_impl_mod_free(struct processing_module *mod, const void *ptr); diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index 29c226eb61a7..d52f8e047b79 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -114,7 +114,9 @@ struct lib_manager_module { struct llext *llext; /* Zephyr loadable extension context */ struct llext_buf_loader *ebl; /* Zephyr loadable extension buffer loader */ unsigned int n_dependent; /* For auxiliary modules: number of dependents */ + unsigned int n_mod; bool mapped; + bool domain_dp; struct lib_manager_segment_desc segment[LIB_MANAGER_N_SEGMENTS]; }; @@ -217,6 +219,25 @@ void lib_manager_get_instance_bss_address(uint32_t instance_id, */ int lib_manager_load_library(uint32_t dma_id, uint32_t lib_id, uint32_t type); +struct userspace_context; +/* + * \brief Allocate the module and start the agent if needed + */ +int lib_manager_mod_create_priv(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const void *spec, void **adapter_priv, + struct userspace_context **userspace, + const struct module_interface **ops); + +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +__syscall int lib_manager_free_module(const uint32_t component_id); + +#include +#else +int z_impl_lib_manager_free_module(const uint32_t component_id); +#define lib_manager_free_module z_impl_lib_manager_free_module +#endif + /* * \brief Initialize message * diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index afc8fe45de05..96b957f9fabb 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -318,7 +318,6 @@ void ipc_schedule_process(struct ipc *ipc) #define IPC_USER_EVENT_CMD BIT(0) #define IPC_USER_EVENT_STOP BIT(1) -static struct k_thread ipc_user_thread; static K_THREAD_STACK_DEFINE(ipc_user_stack, CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); /** @@ -419,10 +418,59 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) } } +__cold static int ipc_user_init_thread(struct ipc_user *ipc_user) +{ + char thread_name[] = "ll_user0"; + int ret; + + assert_can_be_cold(); + + /* Allocate kernel objects for the user-space thread */ + ipc_user->event = k_object_alloc(K_OBJ_EVENT); + if (!ipc_user->event) { + LOG_ERR("user IPC event alloc failed"); + return -ENOMEM; + } + k_event_init(ipc_user->event); + + ipc_user->thread = k_object_alloc(K_OBJ_THREAD); + if (!ipc_user->thread) { + LOG_ERR("user IPC thread alloc failed"); + ret = -ENOMEM; + goto e_event; + } + + k_thread_create(ipc_user->thread, ipc_user_stack, + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, + ipc_user_thread_fn, ipc_user, NULL, NULL, + -1, K_USER, K_FOREVER); + + k_thread_cpu_pin(ipc_user->thread, PLATFORM_PRIMARY_CORE_ID); + k_thread_name_set(ipc_user->thread, thread_name); + + /* + * Each userspace IPC thread must be able to wait on its private event + * and signal completion on the primary core semaphore + */ + k_thread_access_grant(ipc_user->thread, ipc_user->sem, ipc_user->event); + user_grant_dai_access_all(ipc_user->thread); + user_grant_dma_access_all(ipc_user->thread); + k_mem_domain_add_thread(zephyr_ll_mem_domain(), ipc_user->thread); + user_ll_grant_access(ipc_user->thread, PLATFORM_PRIMARY_CORE_ID); + + return 0; + +e_event: + k_object_free(ipc_user->event); + + return ret; +} + __cold static void ipc_user_init(void) { struct ipc *ipc = ipc_get(); - struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), SOF_MEM_FLAG_USER, + struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*ipc_user), 0); int ret; @@ -443,38 +491,22 @@ __cold static void ipc_user_init(void) k_sem_init(ipc_user->sem, 0, 1); - /* Allocate kernel objects for the user-space thread */ - ipc_user->event = k_object_alloc(K_OBJ_EVENT); - if (!ipc_user->event) { - LOG_ERR("user IPC event alloc failed"); + ret = ipc_user_init_thread(ipc_user); + if (ret < 0) { + LOG_ERR("user IPC thread initialization failed"); sof_panic(SOF_IPC_PANIC_IPC); } - k_event_init(ipc_user->event); - k_thread_create(&ipc_user_thread, ipc_user_stack, - CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, - ipc_user_thread_fn, ipc_user, NULL, NULL, - -1, K_USER, K_FOREVER); - - ipc_user->thread = &ipc_user_thread; - k_thread_access_grant(&ipc_user_thread, ipc_user->sem, ipc_user->event); - user_grant_dai_access_all(&ipc_user_thread); - user_grant_dma_access_all(&ipc_user_thread); - ret = user_access_to_mailbox(zephyr_ll_mem_domain(), &ipc_user_thread); + ret = user_access_to_mailbox(zephyr_ll_mem_domain(), ipc_user->thread); if (ret < 0) { LOG_ERR("ipc user: mailbox access grant failed: %d", ret); sof_panic(SOF_IPC_PANIC_IPC); } - user_ll_grant_access(&ipc_user_thread, PLATFORM_PRIMARY_CORE_ID); - k_mem_domain_add_thread(zephyr_ll_mem_domain(), &ipc_user_thread); - - k_thread_cpu_pin(&ipc_user_thread, PLATFORM_PRIMARY_CORE_ID); - k_thread_name_set(&ipc_user_thread, "ipc_user"); /* Store references in ipc struct so kernel handler can forward commands */ ipc->ipc_user_pdata = ipc_user; - k_thread_start(&ipc_user_thread); + k_thread_start(ipc_user->thread); struct task *task = zephyr_ll_task_alloc(); @@ -486,7 +518,7 @@ __cold static void ipc_user_init(void) * Needed so user-space dai_common_new() can call * k_thread_access_grant(audio_thread, dai_mutex) from user context. */ - k_thread_access_grant(&ipc_user_thread, ipc_user->audio_thread); + k_thread_access_grant(ipc_user->thread, ipc_user->audio_thread); /* Wait for user thread startup — consumes the initial k_sem_give from thread */ k_sem_take(ipc->ipc_user_pdata->sem, K_FOREVER); diff --git a/src/ipc/ipc-helper.c b/src/ipc/ipc-helper.c index cbd17d00da4c..567255a9e22a 100644 --- a/src/ipc/ipc-helper.c +++ b/src/ipc/ipc-helper.c @@ -297,7 +297,9 @@ __cold int ipc_comp_free(struct ipc *ipc, uint32_t comp_id) struct comp_buffer *buffer; struct comp_buffer *safe; struct list_item *clist; +#ifndef CONFIG_SOF_USERSPACE_LL uint32_t flags; +#endif assert_can_be_cold(); diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index bc1712b2762a..00b890749c88 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -428,6 +428,44 @@ __cold const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data_wrapper return ipc4_get_pipeline_data(); } +static int ipc4_pipeline_id_get(struct ipc4_message_request *ipc4, + struct ipc4_pipeline_set_state *state, + const uint32_t **ppl_id, unsigned int *ppl_count) +{ + if (!state->extension.r.multi_ppl) { + if (ppl_count) + *ppl_count = 1; + if (ppl_id) + *ppl_id = NULL; + return state->primary.r.ppl_id; + } + + const struct ipc4_pipeline_set_state_data *ppl_data = ipc4_get_pipeline_data(); + unsigned int cnt = ppl_data->pipelines_count; + + /* + * pipelines_count is read straight from the host-provided + * mailbox payload, so cap it at what the mailbox can + * physically hold. Anything larger means the host promised + * more ppl_id[] entries than fit in MAILBOX_HOSTBOX, and + * dereferencing the flex array would read out of bounds. + */ + if (cnt > (MAILBOX_HOSTBOX_SIZE - sizeof(struct ipc4_pipeline_set_state_data)) / + sizeof(uint32_t)) { + ipc_cmd_err(&ipc_tr, "ipc: pipelines_count %u exceeds mailbox bound", + cnt); + return -EINVAL; + } + dcache_invalidate_region((__sparse_force void __sparse_cache *)ppl_data->ppl_id, + sizeof(int) * cnt); + if (ppl_count) + *ppl_count = cnt; + if (ppl_id) + *ppl_id = ppl_data->ppl_id; + + return ppl_data->ppl_id[0]; +} + /** * \brief Process SET_PIPELINE_STATE IPC4 message (prepare + trigger phases). * @param[in] ipc4 IPC4 message request. @@ -440,7 +478,7 @@ int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4) struct ipc_comp_dev *ppl_icd; struct ipc *ipc = ipc_get(); uint32_t cmd, ppl_count; - uint32_t id = 0; + int id; const uint32_t *ppl_id; bool use_idc = false; uint32_t idx; @@ -452,31 +490,12 @@ int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4) cmd = state.primary.r.ppl_state; ppl_data = ipc4_get_pipeline_data(); - if (state.extension.r.multi_ppl) { - ppl_count = ppl_data->pipelines_count; - /* - * pipelines_count is read straight from the host-provided - * mailbox payload, so cap it at what the mailbox can - * physically hold. Anything larger means the host promised - * more ppl_id[] entries than fit in MAILBOX_HOSTBOX, and - * dereferencing the flex array would read out of bounds. - */ - if (ppl_count > (MAILBOX_HOSTBOX_SIZE - - sizeof(struct ipc4_pipeline_set_state_data)) / - sizeof(uint32_t)) { - ipc_cmd_err(&ipc_tr, - "ipc: pipelines_count %u exceeds mailbox bound", - ppl_count); - return IPC4_ERROR_INVALID_PARAM; - } - ppl_id = ppl_data->ppl_id; - dcache_invalidate_region((__sparse_force void __sparse_cache *)ppl_id, - sizeof(int) * ppl_count); - } else { - ppl_count = 1; - id = state.primary.r.ppl_id; + id = ipc4_pipeline_id_get(ipc4, &state, &ppl_id, &ppl_count); + if (id < 0) + return IPC4_ERROR_INVALID_PARAM; + + if (ppl_count == 1) ppl_id = &id; - } for (i = 0; i < ppl_count; i++) { ppl_icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index fd8bd8bf3dbc..fa19a0fe6079 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -88,24 +88,25 @@ void ipc_build_trace_posn(struct sof_ipc_dma_trace_posn *posn) } #if CONFIG_LIBRARY -static inline char *ipc4_get_comp_new_data(void) +static inline unsigned char *ipc4_get_comp_new_data(void) { struct ipc *ipc = ipc_get(); - char *data = (char *)ipc->comp_data + sizeof(struct ipc4_module_init_instance); + unsigned char *data = (unsigned char *)ipc->comp_data + + sizeof(struct ipc4_module_init_instance); return data; } -static const struct comp_driver *ipc4_library_get_comp_drv(char *data) +static const struct comp_driver *ipc4_library_get_comp_drv(unsigned char *data) { return ipc4_get_drv(data); } #else -__cold static inline char *ipc4_get_comp_new_data(void) +__cold static inline unsigned char *ipc4_get_comp_new_data(void) { assert_can_be_cold(); - return (char *)MAILBOX_HOSTBOX_BASE; + return (unsigned char *)MAILBOX_HOSTBOX_BASE; } #endif @@ -116,7 +117,7 @@ __cold struct comp_dev *comp_new_ipc4(struct ipc4_module_init_instance *module_i const struct comp_driver *drv; struct comp_dev *dev; uint32_t comp_id; - char *data; + unsigned char *data; assert_can_be_cold(); @@ -190,13 +191,13 @@ __cold struct comp_dev *comp_new_ipc4(struct ipc4_module_init_instance *module_i if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { - .data = (const unsigned char *)data, + .data = data, .size = ipc_config.ipc_config_size, }; - dev = drv->ops.create(drv, &ipc_config, (const void *)&spec); + dev = drv->ops.create(drv, &ipc_config, &spec); } else { - dev = drv->ops.create(drv, &ipc_config, (const void *)data); + dev = drv->ops.create(drv, &ipc_config, data); } if (!dev) return NULL; @@ -241,7 +242,7 @@ __cold struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, struct comp_ipc_config ipc_config; struct comp_dev *dev; uint32_t comp_id; - char *data; + unsigned char *data; int ret; assert_can_be_cold(); @@ -303,13 +304,13 @@ __cold struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { - .data = (const unsigned char *)data, + .data = data, .size = ipc_config.ipc_config_size, }; - dev = drv->ops.create(drv, &ipc_config, (const void *)&spec); + dev = drv->ops.create(drv, &ipc_config, &spec); } else { - dev = drv->ops.create(drv, &ipc_config, (const void *)data); + dev = drv->ops.create(drv, &ipc_config, data); } if (!dev) return NULL; diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index 19c317ed1089..7492a225656d 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -415,7 +415,7 @@ static uintptr_t lib_manager_allocate_module(const struct sof_man_fw_desc *const * * Function is responsible to free module resources in HP memory. */ -static int lib_manager_free_module(const uint32_t component_id) +int z_impl_lib_manager_free_module(const uint32_t component_id) { const struct sof_man_module *mod; const uint32_t module_id = IPC4_MOD_ID(component_id); @@ -462,7 +462,7 @@ static uintptr_t lib_manager_allocate_module(const struct sof_man_fw_desc *const return 0; } -static int lib_manager_free_module(const uint32_t component_id) +static int z_impl_lib_manager_free_module(const uint32_t component_id) { /* Since we cannot allocate the freeing is not considered to be an error */ tr_warn(&lib_manager_tr, "Dynamic module freeing is not supported"); @@ -642,34 +642,35 @@ static enum buildinfo_mod_type lib_manager_get_module_type(const struct sof_man_ } } -/* - * \brief Load module code, allocate its instance and create a module adapter component. - * \param[in] drv - component driver pointer. - * \param[in] config - component ipc descriptor pointer. - * \param[in] spec - passdowned data from driver. - * - * \return: a pointer to newly created module adapter component on success. NULL on error. - */ -static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, - const struct comp_ipc_config *config, - const void *spec) +static void lib_manager_mod_free_priv(const struct comp_driver *drv, + const struct comp_ipc_config *config, + struct userspace_context *userspace) +{ +#if CONFIG_SOF_USERSPACE_PROXY + if (userspace) + userspace_proxy_destroy(drv, userspace); +#endif /* CONFIG_SOF_USERSPACE_PROXY */ + lib_manager_free_module(config->id); +} + +int lib_manager_mod_create_priv(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const void *spec, void **adapter_priv, + struct userspace_context **userspace, + const struct module_interface **ops) { const struct sof_man_fw_desc *const desc = lib_manager_get_library_manifest(config->id); const struct ipc_config_process *args = (const struct ipc_config_process *)spec; const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(config->id); - struct userspace_context *userspace = NULL; - const struct module_interface *ops; const struct sof_man_module *mod; system_agent_start_fn agent; - void *adapter_priv = NULL; const void **agent_iface; - struct comp_dev *dev; int ret; #ifdef CONFIG_SOF_USERSPACE_PROXY if (drv->user_heap && config->proc_domain != COMP_PROCESSING_DOMAIN_DP) { tr_err(&lib_manager_tr, "Userspace supports only DP modules."); - return NULL; + return -EOPNOTSUPP; } #endif @@ -677,12 +678,12 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, if (!desc) { tr_err(&lib_manager_tr, "Error: Couldn't find loadable module with id %u.", config->id); - return NULL; + return -ENOENT; } if (entry_index >= desc->header.num_module_entries) { tr_err(&lib_manager_tr, "Entry index %u out of bounds.", entry_index); - return NULL; + return -EINVAL; } mod = (const struct sof_man_module *) @@ -693,53 +694,91 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, if (!module_entry_point) { tr_err(&lib_manager_tr, "lib_manager_allocate_module() failed!"); - return NULL; + return -ENOENT; } switch (lib_manager_get_module_type(desc, mod)) { case MOD_TYPE_LLEXT: agent = NULL; - ops = (const struct module_interface *)module_entry_point; + *ops = (const struct module_interface *)module_entry_point; agent_iface = NULL; break; case MOD_TYPE_LMDK: agent = &native_system_agent_start; - agent_iface = (const void **)&ops; + agent_iface = (const void **)ops; break; #if CONFIG_INTEL_MODULES case MOD_TYPE_IADK: agent = &system_agent_start; - ops = &processing_module_adapter_interface; - agent_iface = (const void **)&adapter_priv; + *ops = &processing_module_adapter_interface; + agent_iface = (const void **)adapter_priv; break; #endif case MOD_TYPE_INVALID: + default: + ret = -EINVAL; goto err; } if (agent || IS_ENABLED(CONFIG_SOF_USERSPACE_PROXY)) { /* At this point module resources are allocated and it is moved to L2 memory. */ ret = lib_manager_start_agent(drv, config, mod, args, module_entry_point, agent, - agent_iface, &userspace, &ops); + agent_iface, userspace, ops); if (ret) goto err; } - if (comp_set_adapter_ops(drv, ops) < 0) + ret = comp_set_adapter_ops(drv, *ops); + if (ret < 0) goto err; - dev = module_adapter_new_ext(drv, config, spec, adapter_priv, userspace, NULL); + return 0; + +err: + lib_manager_mod_free_priv(drv, config, *userspace); + return ret; +} + +#ifdef CONFIG_USERSPACE +#include + +static int z_vrfy_lib_manager_free_module(const uint32_t component_id) +{ + return z_impl_lib_manager_free_module(component_id); +} +#include + +#endif /* CONFIG_USERSPACE */ + +/* + * \brief Load module code, allocate its instance and create a module adapter component. + * \param[in] drv - component driver pointer. + * \param[in] config - component ipc descriptor pointer. + * \param[in] spec - passdowned data from driver. + * + * \return: a pointer to newly created module adapter component on success. NULL on error. + */ +static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const void *spec) +{ + struct userspace_context *userspace = NULL; + const struct module_interface *ops; + void *adapter_priv = NULL; + struct comp_dev *dev; + int ret = lib_manager_mod_create_priv(drv, config, spec, &adapter_priv, &userspace, &ops); + + if (ret < 0) + return NULL; + + dev = module_adapter_new_ext(drv, config, spec, adapter_priv, userspace, ops); if (!dev) goto err; return dev; err: -#if CONFIG_SOF_USERSPACE_PROXY - if (userspace) - userspace_proxy_destroy(drv, userspace); -#endif /* CONFIG_SOF_USERSPACE_PROXY */ - lib_manager_free_module(config->id); + lib_manager_mod_free_priv(drv, config, userspace); return NULL; } @@ -760,7 +799,8 @@ static void lib_manager_module_free(struct comp_dev *dev) static void lib_manager_prepare_module_adapter(struct comp_driver *drv, const struct sof_uuid *uuid) { drv->type = SOF_COMP_MODULE_ADAPTER; - drv->uid = uuid; + drv->uid_cp = *uuid; + drv->uid = &drv->uid_cp; drv->tctx = &lib_manager_tr; drv->ops.create = lib_manager_module_create; drv->ops.prepare = module_adapter_prepare; @@ -819,6 +859,8 @@ int lib_manager_register_module(const uint32_t component_id) goto cleanup; } } +#else + drv_heap = sof_sys_user_heap_get(); #endif /* CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP */ drv = sof_heap_alloc(drv_heap, SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index 4c1e4f02d5b5..94dd3944645f 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include @@ -199,6 +201,11 @@ static void llext_manager_unmap_detached_sections(const struct llext_loader *ldr #endif } +static int llext_manager_add_mod_domain(struct lib_manager_module *mctx, + struct k_mem_domain *domain); +static int llext_manager_rm_mod_domain(struct lib_manager_module *mctx, + struct k_mem_domain *domain); + static int llext_manager_load_module(struct lib_manager_module *mctx) { /* Executable code (.text) */ @@ -292,8 +299,22 @@ static int llext_manager_load_module(struct lib_manager_module *mctx) memset((__sparse_force void *)bss_addr, 0, bss_size); mctx->mapped = true; - return 0; +#ifdef CONFIG_SOF_USERSPACE_LL + if (!mctx->domain_dp) { + ret = llext_manager_add_mod_domain(mctx, zephyr_ll_mem_domain()); + if (ret < 0) { + tr_err(&lib_manager_tr, "failed to add domain: %d", ret); + goto e_data; + } + } +#endif + return 0; +#ifdef CONFIG_SOF_USERSPACE_LL +e_data: + if (data_size) + llext_manager_align_unmap(va_base_data, data_size); +#endif e_rodata: if (rodata_size) llext_manager_align_unmap(va_base_rodata, rodata_size); @@ -353,6 +374,11 @@ static int llext_manager_unload_module(struct lib_manager_module *mctx) mctx->mapped = false; +#ifdef CONFIG_SOF_USERSPACE_LL + if (!mctx->domain_dp) + llext_manager_rm_mod_domain(mctx, zephyr_ll_mem_domain()); +#endif + return err; } @@ -447,6 +473,7 @@ static int llext_manager_link(const char *name, if (ret >= 0) { llext_get_section_info(ldr, *llext, ret, &hdr, NULL, NULL); *mod_manifest = llext_peek(ldr, hdr->sh_offset); + mctx->n_mod = hdr->sh_size / sizeof(struct sof_man_module_manifest); } return *buildinfo && *mod_manifest ? 0 : -EPROTO; @@ -496,6 +523,7 @@ static int llext_manager_mod_init(struct lib_manager_mod_ctx *ctx, if (mod_array[i].segment[LIB_MANAGER_TEXT].file_offset != offs) { offs = mod_array[i].segment[LIB_MANAGER_TEXT].file_offset; ctx->mod[n_mod].mapped = false; + ctx->mod[n_mod].domain_dp = false; ctx->mod[n_mod].llext = NULL; ctx->mod[n_mod].ebl = NULL; ctx->mod[n_mod].n_dependent = 0; @@ -506,7 +534,7 @@ static int llext_manager_mod_init(struct lib_manager_mod_ctx *ctx, } /* Find a module context, containing the driver with the supplied index */ -static unsigned int llext_manager_mod_find(const struct lib_manager_mod_ctx *ctx, unsigned int idx) +static int llext_manager_mod_find(const struct lib_manager_mod_ctx *ctx, unsigned int idx) { unsigned int i; @@ -514,6 +542,9 @@ static unsigned int llext_manager_mod_find(const struct lib_manager_mod_ctx *ctx if (ctx->mod[i].start_idx > idx) break; + if (i == ctx->n_mod && ctx->mod[i - 1].start_idx + ctx->mod[i - 1].n_mod <= idx) + return -ENOENT; + return i - 1; } @@ -535,7 +566,11 @@ static int llext_manager_link_single(uint32_t module_id, const struct sof_man_fw return -EINVAL; } - unsigned int mod_ctx_idx = llext_manager_mod_find(ctx, entry_index); + int mod_ctx_idx = llext_manager_mod_find(ctx, entry_index); + + if (mod_ctx_idx < 0) + return mod_ctx_idx; + struct lib_manager_module *mctx = ctx->mod + mod_ctx_idx; size_t mod_size; int i, inst_idx; @@ -738,6 +773,8 @@ uintptr_t llext_manager_allocate_module(const struct comp_ipc_config *ipc_config dep_ctx[i] = dep; } + /* Avoid mapping DP modules to the LL domain */ + mctx->domain_dp = ipc_config->proc_domain == COMP_PROCESSING_DOMAIN_DP; /* Map executable code and data */ ret = llext_manager_load_module(mctx); if (ret < 0) @@ -778,13 +815,8 @@ static int llext_manager_rm_partition(struct k_mem_domain *domain, return k_mem_domain_remove_partition(domain, &part); } -int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *domain) +static int llext_manager_add_mod_domain(struct lib_manager_module *mctx, struct k_mem_domain *domain) { - const uint32_t module_id = IPC4_MOD_ID(component_id); - struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id); - const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id); - const unsigned int mod_idx = llext_manager_mod_find(ctx, entry_index); - struct lib_manager_module *mctx = ctx->mod + mod_idx; const struct llext *ext = mctx->llext; const struct llext_loader *ldr = &mctx->ebl->loader; @@ -800,6 +832,13 @@ int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *d uintptr_t va_base_data = mctx->segment[LIB_MANAGER_DATA].addr; size_t data_size = mctx->segment[LIB_MANAGER_DATA].size; + /* + * Add to domain on first load: for "normal" modules use_count == 1, + * for dependencies use_count == 2 and n_dependent == 1 + */ + if (ext->use_count > 1 && mctx->n_dependent != 1) + return 0; + int ret = llext_manager_add_partition(domain, va_base_text, text_size, K_MEM_PARTITION_P_RX_U_RX | XTENSA_MMU_CACHED_WB); @@ -914,14 +953,24 @@ int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *d return ret; } -int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *domain) +int llext_manager_add_domain(const uint32_t component_id, struct k_mem_domain *domain) { const uint32_t module_id = IPC4_MOD_ID(component_id); struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id); const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id); - const unsigned int mod_idx = llext_manager_mod_find(ctx, entry_index); + const int mod_idx = llext_manager_mod_find(ctx, entry_index); + + if (mod_idx < 0) + return mod_idx; + struct lib_manager_module *mctx = ctx->mod + mod_idx; + /* FIXME: handle dependencies */ + return llext_manager_add_mod_domain(mctx, domain); +} + +static int llext_manager_rm_mod_domain(struct lib_manager_module *mctx, struct k_mem_domain *domain) +{ /* Executable code (.text) */ uintptr_t va_base_text = mctx->segment[LIB_MANAGER_TEXT].addr; size_t text_size = mctx->segment[LIB_MANAGER_TEXT].size; @@ -988,6 +1037,21 @@ int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *do return ret; } + +int llext_manager_rm_domain(const uint32_t component_id, struct k_mem_domain *domain) +{ + const uint32_t module_id = IPC4_MOD_ID(component_id); + struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(module_id); + const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(module_id); + const int mod_idx = llext_manager_mod_find(ctx, entry_index); + + if (mod_idx < 0) + return mod_idx; + + struct lib_manager_module *mctx = ctx->mod + mod_idx; + + return llext_manager_rm_mod_domain(mctx, domain); +} #endif int llext_manager_free_module(const uint32_t component_id) @@ -1008,7 +1072,11 @@ int llext_manager_free_module(const uint32_t component_id) return -ENOENT; } - unsigned int mod_idx = llext_manager_mod_find(ctx, entry_index); + int mod_idx = llext_manager_mod_find(ctx, entry_index); + + if (mod_idx < 0) + return mod_idx; + struct lib_manager_module *mctx = ctx->mod + mod_idx; /* Protected by IPC serialization */ diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index 25fa8b319457..b3a5530b27d4 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -29,6 +28,9 @@ #include +#include +#include + LOG_MODULE_REGISTER(dp_schedule, CONFIG_SOF_LOG_LEVEL); SOF_DEFINE_REG_UUID(dp_sched); @@ -337,7 +339,7 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta return 0; } -static struct scheduler_ops schedule_dp_ops = { +APP_SYSUSER_DATA static struct scheduler_ops schedule_dp_ops = { .schedule_task = scheduler_dp_task_shedule, #if CONFIG_SOF_USERSPACE_APPLICATION .schedule_task_cancel = scheduler_dp_task_cancel, @@ -353,12 +355,12 @@ __cold int scheduler_dp_init(void) assert_can_be_cold(); - struct scheduler_dp_data *dp_sch = rzalloc(SOF_MEM_FLAG_KERNEL, - sizeof(struct scheduler_dp_data)); + struct scheduler_dp_data *dp_sch = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_KERNEL, sizeof(*dp_sch), 0); if (!dp_sch) return -ENOMEM; - dp_sch->ll_tick_src.priv_data = NULL; + memset(dp_sch, 0, sizeof(*dp_sch)); list_init(&dp_sch->tasks); scheduler_init(SOF_SCHEDULE_DP, &schedule_dp_ops, dp_sch); diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 4b61a9517d46..04bb3649dc2f 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -627,6 +627,7 @@ zephyr_syscall_header(${SOF_SRC_PATH}/include/ipc4/handler.h) zephyr_syscall_header(include/rtos/alloc.h) zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_ALLOC syscall/alloc.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/dai-zephyr.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib_manager.h) zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/dai.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/user/debug_stream_slot.h) diff --git a/zephyr/include/rtos/alloc.h b/zephyr/include/rtos/alloc.h index b727d1e700fb..d512ccdff2ef 100644 --- a/zephyr/include/rtos/alloc.h +++ b/zephyr/include/rtos/alloc.h @@ -167,6 +167,8 @@ size_t get_shared_buffer_heap_size(void); struct mod_alloc_ctx { struct k_heap *heap; struct vregion *vreg; + uintptr_t vreg_start; + size_t vreg_size; }; /**