[DNM][WiP][PoC] userspace LL scheduling: LLEXT & multicore

app/overlays/ptl/ll_userspace_overlay.conf

@@ -24,15 +24,12 @@ CONFIG_COLD_STORE_EXECUTE_DEBUG=n
 CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=n
 CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n
 
-# disable loadable modules (hits privilege issues in user-space now)
-CONFIG_LLEXT_STORAGE_WRITABLE=n
-CONFIG_LLEXT_EXPERIMENTAL=n
-CONFIG_MODULES=n
-
 # some of current boot tests interfere with user-space setup
 CONFIG_SOF_BOOT_TEST_ALLOWED=n
 
 # misc features not yet compatible with user-space
 CONFIG_CROSS_CORE_STREAM=n
 CONFIG_INTEL_ADSP_MIC_PRIVACY=n
 CONFIG_XRUN_NOTIFICATIONS_ENABLE=n
+
+CONFIG_MAX_THREAD_BYTES=4

posix/include/rtos/mutex.h

@@ -62,4 +62,25 @@ static inline int sys_mutex_unlock(struct sys_mutex *mutex)
 	return 0;
 }
 
+/* provide a no-op implementation for zephyr/sys/sem.h */
+
+struct sys_sem {
+};
+
+static inline int sys_sem_init(struct sys_sem *sem, unsigned int initial_count,
+			       unsigned int limit)
+{
+	return 0;
+}
+
+static inline int sys_sem_give(struct sys_sem *sem)
+{
+	return 0;
+}
+
+static inline int sys_sem_take(struct sys_sem *sem, k_timeout_t timeout)
+{
+	return 0;
+}
+
 #endif

src/audio/buffers/comp_buffer.c

@@ -11,6 +11,7 @@
 #include <sof/audio/sink_api.h>
 #include <sof/audio/source_api.h>
 #include <sof/audio/sink_source_utils.h>
+#include <sof/audio/module_adapter/module/generic.h>
 #include <rtos/userspace_helper.h>
 #include <sof/common.h>
 #include <rtos/interrupt.h>
@@ -165,8 +166,7 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer)
 
 	if (alloc && alloc->vreg) {
 		vregion_free(alloc->vreg, buffer);
-		if (!vregion_put(alloc->vreg))
-			rfree(alloc);
+		module_adapter_vreg_free(alloc);
 	} else {
 		sof_heap_free(alloc ? alloc->heap : NULL, buffer);
 	}

src/audio/chain_dma.c

@@ -15,7 +15,9 @@
 #include <ipc/dai.h>
 #include <ipc4/gateway.h>
 #include <sof/schedule/ll_schedule.h>
+#include <sof/schedule/ll_schedule_domain.h>
 #include <sof/schedule/schedule.h>
+#include <rtos/alloc.h>
 #include <rtos/task.h>
 #include <sof/lib/dma.h>
 #include <sof/lib/memory.h>
@@ -61,13 +63,13 @@ struct chain_dma_data {
 
 	/* local host DMA config */
 	struct sof_dma *dma_host;
-	struct dma_chan_data *chan_host;
+	int chan_host_index;
 	struct dma_config z_config_host;
 	struct dma_block_config dma_block_cfg_host;
 
 	/* local link DMA config */
 	struct sof_dma *dma_link;
-	struct dma_chan_data *chan_link;
+	int chan_link_index;
 	struct dma_config z_config_link;
 	struct dma_block_config dma_block_cfg_link;
 
@@ -79,12 +81,12 @@ static int chain_host_start(struct comp_dev *dev)
 	struct chain_dma_data *cd = comp_get_drvdata(dev);
 	int err;
 
-	err = dma_start(cd->chan_host->dma->z_dev, cd->chan_host->index);
+	err = sof_dma_start(cd->dma_host, cd->chan_host_index);
 	if (err < 0)
 		return err;
 
 	comp_info(dev, "dma_start() host chan_index = %u",
-		  cd->chan_host->index);
+		  cd->chan_host_index);
 	return 0;
 }
 
@@ -93,12 +95,12 @@ static int chain_link_start(struct comp_dev *dev)
 	struct chain_dma_data *cd = comp_get_drvdata(dev);
 	int err;
 
-	err = dma_start(cd->chan_link->dma->z_dev, cd->chan_link->index);
+	err = sof_dma_start(cd->dma_link, cd->chan_link_index);
 	if (err < 0)
 		return err;
 
 	comp_info(dev, "dma_start() link chan_index = %u",
-		  cd->chan_link->index);
+		  cd->chan_link_index);
 	return 0;
 }
 
@@ -107,12 +109,12 @@ static int chain_link_stop(struct comp_dev *dev)
 	struct chain_dma_data *cd = comp_get_drvdata(dev);
 	int err;
 
-	err = dma_stop(cd->chan_link->dma->z_dev, cd->chan_link->index);
+	err = sof_dma_stop(cd->dma_link, cd->chan_link_index);
 	if (err < 0)
 		return err;
 
 	comp_info(dev, "dma_stop() link chan_index = %u",
-		  cd->chan_link->index);
+		  cd->chan_link_index);
 
 	return 0;
 }
@@ -122,12 +124,12 @@ static int chain_host_stop(struct comp_dev *dev)
 	struct chain_dma_data *cd = comp_get_drvdata(dev);
 	int err;
 
-	err = dma_stop(cd->chan_host->dma->z_dev, cd->chan_host->index);
+	err = sof_dma_stop(cd->dma_host, cd->chan_host_index);
 	if (err < 0)
 		return err;
 
 	comp_info(dev, "dma_stop() host chan_index = %u",
-		  cd->chan_host->index);
+		  cd->chan_host_index);
 
 	return 0;
 }
@@ -165,7 +167,7 @@ static enum task_state chain_task_run(void *data)
 	/* Link DMA can return -EPIPE and current status if xrun occurs, then it is not critical
 	 * and flow shall continue. Other error values will be treated as critical.
 	 */
-	ret = dma_get_status(cd->chan_link->dma->z_dev, cd->chan_link->index, &stat);
+	ret = sof_dma_get_status(cd->dma_link, cd->chan_link_index, &stat);
 	switch (ret) {
 	case 0:
 #if CONFIG_XRUN_NOTIFICATIONS_ENABLE
@@ -189,7 +191,7 @@ static enum task_state chain_task_run(void *data)
 	link_read_pos = stat.read_position;
 
 	/* Host DMA does not report xruns. All error values will be treated as critical. */
-	ret = dma_get_status(cd->chan_host->dma->z_dev, cd->chan_host->index, &stat);
+	ret = sof_dma_get_status(cd->dma_host, cd->chan_host_index, &stat);
 	if (ret < 0) {
 		tr_err(&chain_dma_tr, "dma_get_status() error, ret = %d", ret);
 		return SOF_TASK_STATE_COMPLETED;
@@ -207,14 +209,14 @@ static enum task_state chain_task_run(void *data)
 		 */
 		const size_t increment = MIN(host_free_bytes, link_avail_bytes);
 
-		ret = dma_reload(cd->chan_host->dma->z_dev, cd->chan_host->index, 0, 0, increment);
+		ret = sof_dma_reload(cd->dma_host, cd->chan_host_index, increment);
 		if (ret < 0) {
 			tr_err(&chain_dma_tr,
 			       "dma_reload() host error, ret = %d", ret);
 			return SOF_TASK_STATE_COMPLETED;
 		}
 
-		ret = dma_reload(cd->chan_link->dma->z_dev, cd->chan_link->index, 0, 0, increment);
+		ret = sof_dma_reload(cd->dma_link, cd->chan_link_index, increment);
 		if (ret < 0) {
 			tr_err(&chain_dma_tr,
 			       "dma_reload() link error, ret = %d", ret);
@@ -230,9 +232,8 @@ static enum task_state chain_task_run(void *data)
 		const size_t half_buff_size = buff_size / 2;
 
 		if (!cd->first_data_received && host_avail_bytes > half_buff_size) {
-			ret = dma_reload(cd->chan_link->dma->z_dev,
-					 cd->chan_link->index, 0, 0,
-					 MIN(host_avail_bytes, link_free_bytes));
+			ret = sof_dma_reload(cd->dma_link, cd->chan_link_index,
+					     MIN(host_avail_bytes, link_free_bytes));
 			if (ret < 0) {
 				tr_err(&chain_dma_tr,
 				       "dma_reload() link error, ret = %d", ret);
@@ -246,8 +247,8 @@ static enum task_state chain_task_run(void *data)
 								host_read_pos,
 								buff_size);
 
-			ret = dma_reload(cd->chan_host->dma->z_dev, cd->chan_host->index,
-					 0, 0, transferred);
+			ret = sof_dma_reload(cd->dma_host, cd->chan_host_index,
+					     transferred);
 			if (ret < 0) {
 				tr_err(&chain_dma_tr,
 				       "dma_reload() host error, ret = %d", ret);
@@ -256,8 +257,8 @@ static enum task_state chain_task_run(void *data)
 
 			if (host_avail_bytes >= half_buff_size &&
 			    link_free_bytes >= half_buff_size) {
-				ret = dma_reload(cd->chan_link->dma->z_dev, cd->chan_link->index,
-						 0, 0, half_buff_size);
+				ret = sof_dma_reload(cd->dma_link, cd->chan_link_index,
+						     half_buff_size);
 				if (ret < 0) {
 					tr_err(&chain_dma_tr,
 					       "dma_reload() link error, ret = %d", ret);
@@ -345,6 +346,7 @@ static int chain_task_pause(struct comp_dev *dev)
 		return 0;
 
 	cd->first_data_received = false;
+
 	if (cd->stream_direction == SOF_IPC_STREAM_PLAYBACK) {
 		ret = chain_host_stop(dev);
 		ret2 = chain_link_stop(dev);
@@ -367,9 +369,9 @@ __cold static void chain_release(struct comp_dev *dev)
 
 	assert_can_be_cold();
 
-	dma_release_channel(cd->chan_host->dma->z_dev, cd->chan_host->index);
+	sof_dma_release_channel(cd->dma_host, cd->chan_host_index);
 	sof_dma_put(cd->dma_host);
-	dma_release_channel(cd->chan_link->dma->z_dev, cd->chan_link->index);
+	sof_dma_release_channel(cd->dma_link, cd->chan_link_index);
 	sof_dma_put(cd->dma_link);
 
 	if (cd->dma_buffer) {
@@ -457,53 +459,52 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length)
 
 	/* get host DMA channel */
 	channel = cd->host_connector_node_id.f.v_index;
-	channel = dma_request_channel(cd->dma_host->z_dev, &channel);
+	channel = sof_dma_request_channel(cd->dma_host, channel);
 	if (channel < 0) {
 		comp_err(dev, "host dma_request_channel() failed for %u",
 			 cd->host_connector_node_id.f.v_index);
 		return channel;
 	}
 
-	cd->chan_host = &cd->dma_host->chan[channel];
+	cd->chan_host_index = channel;
 
-	err = dma_config(cd->dma_host->z_dev, cd->chan_host->index, dma_cfg_host);
+	err = sof_dma_config(cd->dma_host, cd->chan_host_index, dma_cfg_host);
 	if (err < 0) {
 		comp_err(dev, "host dma_config() failed for %d", channel);
 		goto error_host;
 	}
 
 	/* get link DMA channel */
 	channel = cd->link_connector_node_id.f.v_index;
-	channel = dma_request_channel(cd->dma_link->z_dev, &channel);
+	channel = sof_dma_request_channel(cd->dma_link, channel);
 	if (channel < 0) {
 		comp_err(dev, "link dma_request_channel() failed for %u",
 			 cd->link_connector_node_id.f.v_index);
 		err = channel;
 		goto error_host;
 	}
 
-	cd->chan_link = &cd->dma_link->chan[channel];
+	cd->chan_link_index = channel;
 
-	err = dma_config(cd->dma_link->z_dev, cd->chan_link->index, dma_cfg_link);
+	err = sof_dma_config(cd->dma_link, cd->chan_link_index, dma_cfg_link);
 	if (err < 0) {
 		comp_err(dev, "link dma_config() failed for %d", channel);
 		goto error_link;
 	}
 	return 0;
 
 error_link:
-	dma_release_channel(cd->dma_link->z_dev, cd->chan_link->index);
-	cd->chan_link = NULL;
+	sof_dma_release_channel(cd->dma_link, cd->chan_link_index);
 error_host:
-	dma_release_channel(cd->dma_host->z_dev, cd->chan_host->index);
-	cd->chan_host = NULL;
+	sof_dma_release_channel(cd->dma_host, cd->chan_host_index);
 	return err;
 }
 
 __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uint8_t link_dma_id,
 				  uint32_t fifo_size)
 {
 	struct chain_dma_data *cd = comp_get_drvdata(dev);
+	struct mod_alloc_ctx *alloc_ctx = NULL;
 	uint32_t addr_align;
 	size_t buff_size;
 	void *buff_addr;
@@ -553,8 +554,8 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin
 	}
 
 	/* retrieve DMA buffer address alignment */
-	ret = dma_get_attribute(cd->dma_host->z_dev, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT,
-				&addr_align);
+	ret = sof_dma_get_attribute(cd->dma_host, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT,
+				    &addr_align);
 	if (ret < 0) {
 		comp_err(dev,
 			 "could not get dma buffer address alignment, err = %d", ret);
@@ -582,8 +583,14 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin
 	}
 
 	fifo_size = ALIGN_UP_INTERNAL(fifo_size, addr_align);
+
+#ifdef CONFIG_SOF_USERSPACE_LL
+	alloc_ctx = ipc_get()->ll_alloc;
+#endif
+
 	/* allocate not shared buffer */
-	cd->dma_buffer = buffer_alloc(NULL, fifo_size, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA,
+	cd->dma_buffer = buffer_alloc(alloc_ctx, fifo_size,
+				      SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA,
 				      addr_align, BUFFER_USAGE_NOT_SHARED);
 
 	if (!cd->dma_buffer) {
@@ -643,14 +650,31 @@ __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv,
 	if (host_dma_id >= max_chain_number)
 		return NULL;
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+	dev = sof_heap_alloc(sof_sys_user_heap_get(),
+			     SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,
+			     sizeof(*dev), 0);
+	if (!dev)
+		return NULL;
+
+	memset(dev, 0, sizeof(*dev));
+	comp_init(drv, dev, sizeof(*dev));
+#else
 	dev = comp_alloc(drv, sizeof(*dev));
 	if (!dev)
 		return NULL;
+#endif
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+	cd = sof_heap_alloc(sof_sys_user_heap_get(), SOF_MEM_FLAG_USER, sizeof(*cd), 0);
+#else
 	cd = rzalloc(SOF_MEM_FLAG_USER, sizeof(*cd));
+#endif
 	if (!cd)
 		goto error;
 
+	memset(cd, 0, sizeof(*cd));
+
 	cd->first_data_received = false;
 	cd->cs = scs ? 2 : 4;
 	cd->chain_task.state = SOF_TASK_STATE_INIT;
@@ -661,9 +685,17 @@ __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv,
 	if (!ret)
 		return dev;
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+	sof_heap_free(sof_sys_user_heap_get(), cd);
+#else
 	rfree(cd);
+#endif
 error:
+#ifdef CONFIG_SOF_USERSPACE_LL
+	sof_heap_free(sof_sys_user_heap_get(), dev);
+#else
 	comp_free_device(dev);
+#endif
 	return NULL;
 }
 
@@ -674,8 +706,16 @@ __cold static void chain_task_free(struct comp_dev *dev)
 	assert_can_be_cold();
 
 	chain_release(dev);
+#ifdef CONFIG_SOF_USERSPACE_LL
+	sof_heap_free(sof_sys_user_heap_get(), cd);
+#else
 	rfree(cd);
+#endif
+#ifdef CONFIG_SOF_USERSPACE_LL
+	sof_heap_free(sof_sys_user_heap_get(), dev);
+#else
 	comp_free_device(dev);
+#endif
 }
 
 static const struct comp_driver comp_chain_dma = {