From 611db7c753388bc823974955bea060a62fbc738c Mon Sep 17 00:00:00 2001 From: Adrian Warecki Date: Mon, 28 Jul 2025 18:26:56 +0200 Subject: [PATCH 1/4] ptl: userspace: Add overlay config for userspace New overlay config created to build SOF with userspace mode enabled. Signed-off-by: Jaroslaw Stelter Signed-off-by: Adrian Warecki --- app/overlays/ptl/userspace_overlay.conf | 13 +++++++++++++ zephyr/Kconfig | 2 ++ 2 files changed, 15 insertions(+) create mode 100644 app/overlays/ptl/userspace_overlay.conf diff --git a/app/overlays/ptl/userspace_overlay.conf b/app/overlays/ptl/userspace_overlay.conf new file mode 100644 index 000000000000..331e2b4280d6 --- /dev/null +++ b/app/overlays/ptl/userspace_overlay.conf @@ -0,0 +1,13 @@ +CONFIG_USERSPACE=y +CONFIG_XTENSA_MMU_NUM_L2_TABLES=86 +CONFIG_MAX_THREAD_BYTES=4 + +CONFIG_INIT_STACKS=n +CONFIG_THREAD_STACK_INFO=n +CONFIG_DYNAMIC_THREAD_PREFER_ALLOC=y +CONFIG_DYNAMIC_THREAD=y +CONFIG_DYNAMIC_THREAD_POOL_SIZE=4 +CONFIG_DYNAMIC_THREAD_ALLOC=n +# IADK module adapter store some buffers on stack, so we need to increase stack size +CONFIG_DYNAMIC_THREAD_STACK_SIZE=8192 +CONFIG_SOF_STACK_SIZE=8192 diff --git a/zephyr/Kconfig b/zephyr/Kconfig index 0bb95b759c96..e291259f4605 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -46,6 +46,7 @@ config SOF_ZEPHYR_SHARED_BUFFER_HEAP_SIZE default 0x1E000 if SOC_INTEL_ACE15_MTPM || SOC_INTEL_ACE20_LNL default 0x1A000 if SOC_INTEL_ACE30 default 0x0 + depends on USERSPACE help The size of the zephyr heap portion designated as the shared buffer heap. This heap is shared between the sof and userspace modules. It is used exclusively for @@ -72,6 +73,7 @@ config SOF_ZEPHYR_VIRTUAL_HEAP_REGION_SIZE config SOF_ZEPHYR_USERSPACE_MODULE_HEAP_SIZE hex "Size of the private heap created for each userspace module" default 0x1000 + depends on USERSPACE help The size of the private heap created for each userspace module. Each userspace module has its own independent heap to which only it has access. This heap is From dbe716823ad9c0f074cdafbf0c9b16ecfa981a76 Mon Sep 17 00:00:00 2001 From: Adrian Warecki Date: Thu, 28 Aug 2025 15:12:13 +0200 Subject: [PATCH 2/4] component: dp: Allocate task structure from module driver heap Allocate task structures from the module's driver heap to enable userspace access. This change allows the DP task to operate in userspace context. Also update LL task allocation to use the same heap for consistency, as the memory release logic is shared and does not differentiate between heaps. Signed-off-by: Adrian Warecki --- src/idc/idc.c | 6 +++--- src/include/sof/audio/component_ext.h | 2 +- src/schedule/zephyr_dp_schedule.c | 7 ++++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/idc/idc.c b/src/idc/idc.c index 1b6a5e39c24a..466f84eee63f 100644 --- a/src/idc/idc.c +++ b/src/idc/idc.c @@ -184,8 +184,8 @@ static int idc_prepare(uint32_t comp_id) /* we're running LL on different core, so allocate our own task */ if (!dev->task && dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_LL) { /* allocate task for shared component */ - dev->task = rzalloc(SOF_MEM_FLAG_USER, - sizeof(*dev->task)); + dev->task = module_driver_heap_rzalloc(dev->drv->user_heap, SOF_MEM_FLAG_USER, + sizeof(*dev->task)); if (!dev->task) { ret = -ENOMEM; goto out; @@ -197,7 +197,7 @@ static int idc_prepare(uint32_t comp_id) dev->priority, comp_task, dev, dev->ipc_config.core, 0); if (ret < 0) { - rfree(dev->task); + module_driver_heap_free(dev->drv->user_heap, dev->task); goto out; } } diff --git a/src/include/sof/audio/component_ext.h b/src/include/sof/audio/component_ext.h index 3370cc110d2e..3762fcde7791 100644 --- a/src/include/sof/audio/component_ext.h +++ b/src/include/sof/audio/component_ext.h @@ -50,7 +50,7 @@ static inline void comp_free(struct comp_dev *dev) if ((dev->is_shared || dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) && dev->task) { schedule_task_free(dev->task); - rfree(dev->task); + module_driver_heap_free(dev->drv->user_heap, dev->task); dev->task = NULL; } diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index cbb36391b310..10b39c77f5ed 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -477,6 +477,7 @@ int scheduler_dp_task_init(struct task **task, size_t stack_size) { void __sparse_cache *p_stack = NULL; + struct sys_heap *const user_heap = mod->dev->drv->user_heap; /* memory allocation helper structure */ struct { @@ -496,8 +497,8 @@ int scheduler_dp_task_init(struct task **task, * As the structure contains zephyr kernel specific data, it must be located in * shared, non cached memory */ - task_memory = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, - sizeof(*task_memory)); + task_memory = module_driver_heap_rzalloc(user_heap, SOF_MEM_FLAG_USER | + SOF_MEM_FLAG_COHERENT, sizeof(*task_memory)); if (!task_memory) { tr_err(&dp_tr, "memory alloc failed"); return -ENOMEM; @@ -542,7 +543,7 @@ int scheduler_dp_task_init(struct task **task, err: /* cleanup - free all allocated resources */ rfree((__sparse_force void *)p_stack); - rfree(task_memory); + module_driver_heap_free(user_heap, task_memory); return ret; } From 2b71b0cc3b0338ce6247700fd546b602cf5e8d53 Mon Sep 17 00:00:00 2001 From: Adrian Warecki Date: Thu, 4 Sep 2025 19:20:39 +0200 Subject: [PATCH 3/4] userspace: Add userspace_context and common_partition memory domain Introduce the userspace_context structure to store data required by modules operating in userspace. Add the common_partition memory domain to share SOF components that need to be accessible from userspace. Signed-off-by: Adrian Warecki --- posix/include/rtos/userspace_helper.h | 3 +++ src/audio/buffers/comp_buffer.c | 5 ++-- src/include/module/module/base.h | 4 +++ .../module_adapter/library/userspace_proxy.h | 25 +++++++++++++++++++ zephyr/include/rtos/userspace_helper.h | 25 ++++++++++++++++++- zephyr/lib/userspace_helper.c | 21 ++++++++++++++++ 6 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 src/include/sof/audio/module_adapter/library/userspace_proxy.h diff --git a/posix/include/rtos/userspace_helper.h b/posix/include/rtos/userspace_helper.h index e535576efac7..dda440a33ac4 100644 --- a/posix/include/rtos/userspace_helper.h +++ b/posix/include/rtos/userspace_helper.h @@ -17,6 +17,9 @@ #include +#define APP_TASK_BSS +#define APP_TASK_DATA + struct sys_heap; #ifdef CONFIG_USERSPACE diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index 8b21b06c454d..b4f53e7cf7f1 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -161,7 +162,7 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer) rfree(buffer); } -static const struct source_ops comp_buffer_source_ops = { +APP_TASK_DATA static const struct source_ops comp_buffer_source_ops = { .get_data_available = comp_buffer_get_data_available, .get_data = comp_buffer_get_data, .release_data = comp_buffer_release_data, @@ -170,7 +171,7 @@ static const struct source_ops comp_buffer_source_ops = { .set_alignment_constants = audio_buffer_source_set_alignment_constants, }; -static const struct sink_ops comp_buffer_sink_ops = { +APP_TASK_DATA static const struct sink_ops comp_buffer_sink_ops = { .get_free_size = comp_buffer_get_free_size, .get_buffer = comp_buffer_get_buffer, .commit_buffer = comp_buffer_commit_buffer, diff --git a/src/include/module/module/base.h b/src/include/module/module/base.h index 881e74a2d108..bee718d5771a 100644 --- a/src/include/module/module/base.h +++ b/src/include/module/module/base.h @@ -15,6 +15,7 @@ #include "interface.h" #include "../ipc4/base-config.h" +#include #define module_get_private_data(mod) ((mod)->priv.private) #define module_set_private_data(mod, data) ((mod)->priv.private = data) @@ -180,6 +181,9 @@ struct processing_module { uint32_t max_sinks; enum module_processing_type proc_type; +#if CONFIG_USERSPACE + struct userspace_context *user_ctx; +#endif /* CONFIG_USERSPACE */ #endif /* SOF_MODULE_PRIVATE */ }; diff --git a/src/include/sof/audio/module_adapter/library/userspace_proxy.h b/src/include/sof/audio/module_adapter/library/userspace_proxy.h new file mode 100644 index 000000000000..52381ac7c175 --- /dev/null +++ b/src/include/sof/audio/module_adapter/library/userspace_proxy.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2025 Intel Corporation. All rights reserved. + * + * Author: Jaroslaw Stelter + * Author: Adrian Warecki + */ + +#ifndef __SOF_AUDIO_USERSPACE_PROXY_H__ +#define __SOF_AUDIO_USERSPACE_PROXY_H__ + +#if CONFIG_USERSPACE +#include +#include + +#include + +/* Processing module structure fields needed for user mode */ +struct userspace_context { + struct k_mem_domain *comp_dom; /* Module specific memory domain */ +}; + +#endif /* CONFIG_USERSPACE */ + +#endif /* __SOF_AUDIO_USERSPACE_PROXY_H__ */ diff --git a/zephyr/include/rtos/userspace_helper.h b/zephyr/include/rtos/userspace_helper.h index 1e7af9ad3773..d7a4afa0a914 100644 --- a/zephyr/include/rtos/userspace_helper.h +++ b/zephyr/include/rtos/userspace_helper.h @@ -12,9 +12,20 @@ #ifndef __ZEPHYR_LIB_USERSPACE_HELPER_H__ #define __ZEPHYR_LIB_USERSPACE_HELPER_H__ -#ifdef CONFIG_USERSPACE +#ifndef CONFIG_USERSPACE +#define APP_TASK_BSS +#define APP_TASK_DATA +#else + +#include + #define DRV_HEAP_SIZE ALIGN_UP(CONFIG_SOF_ZEPHYR_USERSPACE_MODULE_HEAP_SIZE, \ CONFIG_MM_DRV_PAGE_SIZE) +#define APP_TASK_BSS K_APP_BMEM(common_partition) +#define APP_TASK_DATA K_APP_DMEM(common_partition) + +struct processing_module; +struct userspace_context; /** * Initialize private processing module heap. @@ -29,6 +40,18 @@ */ struct sys_heap *module_driver_heap_init(void); +/** + * Add DP scheduler created thread to module memory domain. + * @param thread_id - id of thread to be added to memory domain. + * @param module - processing module structure + * + * @return 0 for success, error otherwise. + * + * @note + * Function used only when CONFIG_USERSPACE is set. + */ +int user_memory_init_shared(k_tid_t thread_id, struct processing_module *mod); + #endif /** diff --git a/zephyr/lib/userspace_helper.c b/zephyr/lib/userspace_helper.c index 74698d7712c1..dc7f117abbd6 100644 --- a/zephyr/lib/userspace_helper.c +++ b/zephyr/lib/userspace_helper.c @@ -16,10 +16,19 @@ #include #include +#include +#include #define MODULE_DRIVER_HEAP_CACHED CONFIG_SOF_ZEPHYR_HEAP_CACHED +/* Zephyr includes */ +#include +#include + #if CONFIG_USERSPACE + +K_APPMEM_PARTITION_DEFINE(common_partition); + struct sys_heap *module_driver_heap_init(void) { struct sys_heap *mod_drv_heap = rballoc(SOF_MEM_FLAG_USER, sizeof(struct sys_heap)); @@ -122,6 +131,18 @@ void module_driver_heap_remove(struct sys_heap *mod_drv_heap) } } +int user_memory_init_shared(k_tid_t thread_id, struct processing_module *mod) +{ + struct k_mem_domain *comp_dom = mod->user_ctx->comp_dom; + int ret; + + ret = k_mem_domain_add_partition(comp_dom, &common_partition); + if (ret < 0) + return ret; + + return k_mem_domain_add_thread(comp_dom, thread_id); +} + #else /* CONFIG_USERSPACE */ void *module_driver_heap_rmalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes) From 73abfd89c2e7c1b7a856e317ac399d4418d49194 Mon Sep 17 00:00:00 2001 From: Jaroslaw Stelter Date: Thu, 24 Aug 2023 15:33:28 +0200 Subject: [PATCH 4/4] dp: userspace: Modify dp scheduler to work with USERSPACE K_USER option flag is ignored until CONFIG_USERSPACE is not set. When set, it creates thread in non_priviledged mode. Without proper support such thread will fail. This change is required to run DP scheduler with CONFIG_USERSPACE flag set. When DP scheduler supports non-privileged modules use of irq_lock is not allowed. Therefore for this configuration it uses static semaphores one per core. These could be accesses from user space thread. New created thread with K_USER option (non-privileged) must be added to module memory domain to get access to its private memory space. Additionally allocate scheduler data and task_memory from shared pool. Signed-off-by: Jaroslaw Stelter Signed-off-by: Adrian Warecki --- src/audio/pipeline/pipeline-schedule.c | 9 +- src/include/module/module/base.h | 3 +- src/include/sof/schedule/dp_schedule.h | 4 +- src/schedule/zephyr_dp_schedule.c | 145 ++++++++++++++++++++----- zephyr/include/rtos/userspace_helper.h | 27 +++++ zephyr/lib/userspace_helper.c | 29 +++++ 6 files changed, 186 insertions(+), 31 deletions(-) diff --git a/src/audio/pipeline/pipeline-schedule.c b/src/audio/pipeline/pipeline-schedule.c index 20afcc6fb552..a13e95982ccc 100644 --- a/src/audio/pipeline/pipeline-schedule.c +++ b/src/audio/pipeline/pipeline-schedule.c @@ -26,6 +26,7 @@ #include #include #include +#include LOG_MODULE_DECLARE(pipe, CONFIG_SOF_LOG_LEVEL); @@ -39,7 +40,7 @@ SOF_DEFINE_REG_UUID(dp_task); * current static stack size for each DP component * TODO: to be taken from module manifest */ -#define TASK_DP_STACK_SIZE 8192 +#define TASK_DP_STACK_SIZE CONFIG_SOF_STACK_SIZE #endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ @@ -397,7 +398,11 @@ int pipeline_comp_dp_task_init(struct comp_dev *comp) &ops, mod, comp->ipc_config.core, - TASK_DP_STACK_SIZE); + TASK_DP_STACK_SIZE, +#if CONFIG_USERSPACE + mod->user_ctx ? K_USER : +#endif /* CONFIG_USERSPACE */ + 0); if (ret < 0) return ret; } diff --git a/src/include/module/module/base.h b/src/include/module/module/base.h index bee718d5771a..ed116681afd2 100644 --- a/src/include/module/module/base.h +++ b/src/include/module/module/base.h @@ -15,7 +15,6 @@ #include "interface.h" #include "../ipc4/base-config.h" -#include #define module_get_private_data(mod) ((mod)->priv.private) #define module_set_private_data(mod, data) ((mod)->priv.private = data) @@ -71,6 +70,8 @@ enum module_processing_type { MODULE_PROCESS_TYPE_RAW, }; +struct userspace_context; + /* * A pointer to this structure is passed to module API functions (from struct module_interface). * This structure should contain only fields that should be available to a module. diff --git a/src/include/sof/schedule/dp_schedule.h b/src/include/sof/schedule/dp_schedule.h index 0a064204e8d1..360d7f47c1de 100644 --- a/src/include/sof/schedule/dp_schedule.h +++ b/src/include/sof/schedule/dp_schedule.h @@ -67,13 +67,15 @@ int scheduler_dp_init(void); * \param[in] mod pointer to the module to be run * \param[in] core CPU the thread should run on * \param[in] stack_size size of stack for a zephyr task + * \param[in] options task options used for creation */ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, const struct task_ops *ops, struct processing_module *mod, uint16_t core, - size_t stack_size); + size_t stack_size, + uint32_t options); /** * \brief Extract information about scheduler's tasks diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index 10b39c77f5ed..c4f3aaedeb37 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,8 @@ #include #include #include +#include +#include #include #include @@ -34,20 +37,59 @@ struct scheduler_dp_data { struct task_dp_pdata { k_tid_t thread_id; /* zephyr thread ID */ - struct k_thread thread; /* memory space for a thread */ + struct k_thread *thread; /* pointer to the kernels' thread object */ + struct k_thread thread_struct; /* thread object for kernel threads */ uint32_t deadline_clock_ticks; /* dp module deadline in Zephyr ticks */ k_thread_stack_t __sparse_cache *p_stack; /* pointer to thread stack */ size_t stack_size; /* size of the stack in bytes */ - struct k_sem sem; /* semaphore for task scheduling */ + struct k_sem *sem; /* pointer to semaphore for task scheduling */ + struct k_sem sem_struct; /* semaphore for task scheduling for kernel threads */ struct processing_module *mod; /* the module to be scheduled */ uint32_t ll_cycles_to_start; /* current number of LL cycles till delayed start */ }; +#ifdef CONFIG_USERSPACE +/* Single CPU-wide lock + * The irq_lock is not available for USERSPACE (non-privileged) threads. + * Therefore semaphore is used to control critical section. + */ +#define DP_LOCK_INIT(i, _) Z_SEM_INITIALIZER(dp_lock[i], 1, 1) +#define DP_LOCK_INIT_LIST LISTIFY(CONFIG_MP_MAX_NUM_CPUS, DP_LOCK_INIT, (,)) + +/* User threads don't need access to this array. Access is performed from + * the kernel space via a syscall. Array must be placed in special section + * to be qualified as initialized by the gen_kobject_list.py script. + */ +static +STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_lock, CONFIG_MP_MAX_NUM_CPUS) = { DP_LOCK_INIT_LIST }; + +static inline unsigned int scheduler_dp_lock(uint16_t core) +{ + k_sem_take(&dp_lock[core], K_FOREVER); + return core; +} + +static inline void scheduler_dp_unlock(unsigned int key) +{ + k_sem_give(&dp_lock[key]); +} + +static inline void scheduler_dp_grant(k_tid_t thread_id, uint16_t core) +{ + k_thread_access_grant(thread_id, &dp_lock[core]); +} + +#else /* CONFIG_USERSPACE */ + +static inline void scheduler_dp_grant(k_tid_t thread_id, uint16_t core) +{ +} + /* Single CPU-wide lock * as each per-core instance if dp-scheduler has separate structures, it is enough to * use irq_lock instead of cross-core spinlocks */ -static inline unsigned int scheduler_dp_lock(void) +static inline unsigned int scheduler_dp_lock(uint16_t core) { return irq_lock(); } @@ -56,6 +98,7 @@ static inline void scheduler_dp_unlock(unsigned int key) { irq_unlock(key); } +#endif /* dummy LL task - to start LL on secondary cores */ static enum task_state scheduler_dp_ll_tick_dummy(void *data) @@ -226,7 +269,7 @@ void scheduler_dp_ll_tick(void *receiver_data, enum notify_id event_type, void * unsigned int lock_key; struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); - lock_key = scheduler_dp_lock(); + lock_key = scheduler_dp_lock(cpu_get_id()); list_for_item(tlist, &dp_sch->tasks) { curr_task = container_of(tlist, struct task, list); pdata = curr_task->priv_data; @@ -256,7 +299,7 @@ void scheduler_dp_ll_tick(void *receiver_data, enum notify_id event_type, void * /* trigger the task */ curr_task->state = SOF_TASK_STATE_RUNNING; - k_sem_give(&pdata->sem); + k_sem_give(pdata->sem); } } } @@ -271,7 +314,7 @@ static int scheduler_dp_task_cancel(void *data, struct task *task) /* this is asyn cancel - mark the task as canceled and remove it from scheduling */ - lock_key = scheduler_dp_lock(); + lock_key = scheduler_dp_lock(cpu_get_id()); task->state = SOF_TASK_STATE_CANCEL; list_item_del(&task->list); @@ -281,7 +324,7 @@ static int scheduler_dp_task_cancel(void *data, struct task *task) schedule_task_cancel(&dp_sch->ll_tick_src); /* if the task is waiting on a semaphore - let it run and self-terminate */ - k_sem_give(&pdata->sem); + k_sem_give(pdata->sem); scheduler_dp_unlock(lock_key); /* wait till the task has finished, if there was any task created */ @@ -294,6 +337,7 @@ static int scheduler_dp_task_cancel(void *data, struct task *task) static int scheduler_dp_task_free(void *data, struct task *task) { struct task_dp_pdata *pdata = task->priv_data; + int ret; scheduler_dp_task_cancel(data, task); @@ -305,12 +349,19 @@ static int scheduler_dp_task_free(void *data, struct task *task) pdata->thread_id = NULL; } +#ifdef CONFIG_USERSPACE + if (pdata->sem != &pdata->sem_struct) + k_object_free(pdata->sem); + if (pdata->thread != &pdata->thread_struct) + k_object_free(pdata->thread); +#endif + /* free task stack */ - rfree((__sparse_force void *)pdata->p_stack); + ret = user_stack_free((__sparse_force void *)pdata->p_stack); pdata->p_stack = NULL; /* all other memory has been allocated as a single malloc, will be freed later by caller */ - return 0; + return ret; } /* Thread function called in component context, on target core */ @@ -329,14 +380,14 @@ static void dp_thread_fn(void *p1, void *p2, void *p3) * the thread is started immediately after creation, it will stop on semaphore * Semaphore will be released once the task is ready to process */ - k_sem_take(&task_pdata->sem, K_FOREVER); + k_sem_take(task_pdata->sem, K_FOREVER); if (task->state == SOF_TASK_STATE_RUNNING) state = task_run(task); else state = task->state; /* to avoid undefined variable warning */ - lock_key = scheduler_dp_lock(); + lock_key = scheduler_dp_lock(task->core); /* * check if task is still running, may have been canceled by external call * if not, set the state returned by run procedure @@ -382,7 +433,7 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta uint64_t deadline_clock_ticks; int ret; - lock_key = scheduler_dp_lock(); + lock_key = scheduler_dp_lock(cpu_get_id()); if (task->state != SOF_TASK_STATE_INIT && task->state != SOF_TASK_STATE_CANCEL && @@ -392,10 +443,17 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta } /* create a zephyr thread for the task */ - pdata->thread_id = k_thread_create(&pdata->thread, (__sparse_force void *)pdata->p_stack, + pdata->thread_id = k_thread_create(pdata->thread, (__sparse_force void *)pdata->p_stack, pdata->stack_size, dp_thread_fn, task, NULL, NULL, - CONFIG_DP_THREAD_PRIORITY, K_USER, K_FOREVER); + CONFIG_DP_THREAD_PRIORITY, task->flags, K_FOREVER); + if (!pdata->thread_id) { + tr_err(&dp_tr, "DP thread creation failed"); + scheduler_dp_unlock(lock_key); + return -ECHILD; + } + k_thread_access_grant(pdata->thread_id, pdata->sem); + scheduler_dp_grant(pdata->thread_id, cpu_get_id()); /* pin the thread to specific core */ ret = k_thread_cpu_pin(pdata->thread_id, task->core); if (ret < 0) { @@ -403,8 +461,18 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta goto err; } - /* start the thread, it should immediately stop at a semaphore, so clean it */ - k_sem_reset(&pdata->sem); +#ifdef CONFIG_USERSPACE + if (task->flags & K_USER) { + ret = user_memory_init_shared(pdata->thread_id, pdata->mod); + if (ret < 0) { + tr_err(&dp_tr, "user_memory_init_shared() failed"); + goto err; + } + } +#endif /* CONFIG_USERSPACE */ + + /* start the thread, it should immediately stop at a semaphore, so clean it */ + k_sem_init(pdata->sem, 0, 1); k_thread_start(pdata->thread_id); /* if there's no DP tasks scheduled yet, run ll tick source task */ @@ -474,7 +542,8 @@ int scheduler_dp_task_init(struct task **task, const struct task_ops *ops, struct processing_module *mod, uint16_t core, - size_t stack_size) + size_t stack_size, + uint32_t options) { void __sparse_cache *p_stack = NULL; struct sys_heap *const user_heap = mod->dev->drv->user_heap; @@ -505,9 +574,7 @@ int scheduler_dp_task_init(struct task **task, } /* allocate stack - must be aligned and cached so a separate alloc */ - stack_size = Z_KERNEL_STACK_SIZE_ADJUST(stack_size); - p_stack = (__sparse_force void __sparse_cache *) - rballoc_align(SOF_MEM_FLAG_KERNEL, stack_size, Z_KERNEL_STACK_OBJ_ALIGN); + p_stack = user_stack_allocate(stack_size, options); if (!p_stack) { tr_err(&dp_tr, "stack alloc failed"); ret = -ENOMEM; @@ -516,21 +583,41 @@ int scheduler_dp_task_init(struct task **task, /* internal SOF task init */ ret = schedule_task_init(&task_memory->task, uid, SOF_SCHEDULE_DP, 0, ops->run, - mod, core, 0); + mod, core, options); if (ret < 0) { tr_err(&dp_tr, "schedule_task_init failed"); goto err; } + /* Point to ksem semaphore for kernel threads synchronization */ + /* It will be overwritten for K_USER threads to dynamic ones. */ + task_memory->pdata.sem = &task_memory->pdata.sem_struct; + task_memory->pdata.thread = &task_memory->pdata.thread_struct; + +#ifdef CONFIG_USERSPACE + if (options & K_USER) { + task_memory->pdata.sem = k_object_alloc(K_OBJ_SEM); + if (!task_memory->pdata.sem) { + tr_err(&dp_tr, "Semaphore object allocation failed"); + ret = -ENOMEM; + goto err; + } + + task_memory->pdata.thread = k_object_alloc(K_OBJ_THREAD); + if (!task_memory->pdata.thread) { + tr_err(&dp_tr, "Thread object allocation failed"); + ret = -ENOMEM; + goto err; + } + } +#endif /* CONFIG_USERSPACE */ + /* initialize other task structures */ task_memory->task.ops.complete = ops->complete; task_memory->task.ops.get_deadline = ops->get_deadline; task_memory->task.state = SOF_TASK_STATE_INIT; task_memory->task.core = core; - /* initialize semaprhore */ - k_sem_init(&task_memory->pdata.sem, 0, 1); - /* success, fill the structures */ task_memory->task.priv_data = &task_memory->pdata; task_memory->pdata.p_stack = p_stack; @@ -538,11 +625,15 @@ int scheduler_dp_task_init(struct task **task, task_memory->pdata.mod = mod; *task = &task_memory->task; - return 0; err: /* cleanup - free all allocated resources */ - rfree((__sparse_force void *)p_stack); + if (user_stack_free((__sparse_force void *)p_stack)) + tr_err(&dp_tr, "user_stack_free failed!"); + + /* k_object_free looks for a pointer in the list, any invalid value can be passed */ + k_object_free(task_memory->pdata.sem); + k_object_free(task_memory->pdata.thread); module_driver_heap_free(user_heap, task_memory); return ret; } @@ -555,7 +646,7 @@ void scheduler_get_task_info_dp(struct scheduler_props *scheduler_props, uint32_ struct scheduler_dp_data *dp_sch = (struct scheduler_dp_data *)scheduler_get_data(SOF_SCHEDULE_DP); - lock_key = scheduler_dp_lock(); + lock_key = scheduler_dp_lock(cpu_get_id()); scheduler_get_task_info(scheduler_props, data_off_size, &dp_sch->tasks); scheduler_dp_unlock(lock_key); } diff --git a/zephyr/include/rtos/userspace_helper.h b/zephyr/include/rtos/userspace_helper.h index d7a4afa0a914..254c8a2f96f1 100644 --- a/zephyr/include/rtos/userspace_helper.h +++ b/zephyr/include/rtos/userspace_helper.h @@ -54,6 +54,33 @@ int user_memory_init_shared(k_tid_t thread_id, struct processing_module *mod); #endif +/** + * Allocates thread stack memory. + * @param stack_size Required stack size. + * @param options Stack configuration options + * K_USER - when creating user thread + * 0 - when creating kernel thread + * @return pointer to the stack or NULL if not created. + * + * When CONFIG_USERSPACE not set function calls rballoc_align(), + * otherwise it uses k_thread_stack_alloc() routine. + * + */ +void *user_stack_allocate(size_t stack_size, uint32_t options); + +/** + * Free thread stack memory. + * @param p_stack Pointer to the stack. + * + * @return 0 for success, error otherwise. + * + * @note + * When CONFIG_USERSPACE not set function calls rfree(), + * otherwise it uses k_thread_stack_free() routine. + * + */ +int user_stack_free(void *p_stack); + /** * Allocates memory block from private module sys_heap if exists, otherwise call rballoc_align(). * @param sys_heap - pointer to the sys_heap structure diff --git a/zephyr/lib/userspace_helper.c b/zephyr/lib/userspace_helper.c index dc7f117abbd6..b26370b12d25 100644 --- a/zephyr/lib/userspace_helper.c +++ b/zephyr/lib/userspace_helper.c @@ -131,6 +131,19 @@ void module_driver_heap_remove(struct sys_heap *mod_drv_heap) } } +void *user_stack_allocate(size_t stack_size, uint32_t options) +{ + return (__sparse_force void __sparse_cache *) + k_thread_stack_alloc(stack_size, options & K_USER); +} + +int user_stack_free(void *p_stack) +{ + if (!p_stack) + return 0; + return k_thread_stack_free((__sparse_force void *)p_stack); +} + int user_memory_init_shared(k_tid_t thread_id, struct processing_module *mod) { struct k_mem_domain *comp_dom = mod->user_ctx->comp_dom; @@ -145,6 +158,22 @@ int user_memory_init_shared(k_tid_t thread_id, struct processing_module *mod) #else /* CONFIG_USERSPACE */ +void *user_stack_allocate(size_t stack_size, uint32_t options) +{ + /* allocate stack - must be aligned and cached so a separate alloc */ + stack_size = K_KERNEL_STACK_LEN(stack_size); + void *p_stack = (__sparse_force void __sparse_cache *) + rballoc_align(SOF_MEM_FLAG_USER, stack_size, Z_KERNEL_STACK_OBJ_ALIGN); + + return p_stack; +} + +int user_stack_free(void *p_stack) +{ + rfree((__sparse_force void *)p_stack); + return 0; +} + void *module_driver_heap_rmalloc(struct sys_heap *mod_drv_heap, uint32_t flags, size_t bytes) { return rmalloc(flags, bytes);