[DM/FEATURE] Support DMA management (#9682)

1. DMA pool management for platform.
2. DMA engine driver API.

Signed-off-by: GuEe-GUI <2991707448@qq.com>

components/drivers/Kconfig

@@ -23,6 +23,7 @@ rsource "hwcrypto/Kconfig"
 rsource "wlan/Kconfig"
 rsource "block/Kconfig"
 rsource "virtio/Kconfig"
+rsource "dma/Kconfig"
 rsource "mfd/Kconfig"
 rsource "ofw/Kconfig"
 rsource "pci/Kconfig"

components/drivers/dma/Kconfig

@@ -0,0 +1,10 @@
+menuconfig RT_USING_DMA
+    bool "Using Direct Memory Access (DMA)"
+    depends on RT_USING_DM
+    select RT_USING_ADT
+    select RT_USING_ADT_BITMAP
+    default n
+
+if RT_USING_DMA
+    osource "$(SOC_DM_DMA_DIR)/Kconfig"
+endif

components/drivers/dma/SConscript

@@ -0,0 +1,15 @@
+from building import *
+
+group = []
+
+if not GetDepend(['RT_USING_DMA']):
+    Return('group')
+
+cwd = GetCurrentDir()
+CPPPATH = [cwd + '/../include']
+
+src = ['dma.c', 'dma_pool.c']
+
+group = DefineGroup('DeviceDrivers', src, depend = [''], CPPPATH = CPPPATH)
+
+Return('group')

components/drivers/dma/dma.c

@@ -0,0 +1,596 @@
+/*
+ * Copyright (c) 2006-2023, RT-Thread Development Team
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Change Logs:
+ * Date           Author       Notes
+ * 2023-02-25     GuEe-GUI     the first version
+ */
+
+#include <rthw.h>
+#include <rtthread.h>
+#include <rtdevice.h>
+
+#define DBG_TAG "rtdm.dma"
+#define DBG_LVL DBG_INFO
+#include <rtdbg.h>
+
+static rt_list_t dmac_nodes = RT_LIST_OBJECT_INIT(dmac_nodes);
+static struct rt_spinlock dmac_nodes_lock = {};
+
+rt_err_t rt_dma_controller_register(struct rt_dma_controller *ctrl)
+{
+    const char *dev_name;
+    char dma_name[RT_NAME_MAX];
+
+    if (!ctrl || !ctrl->dev || !ctrl->ops)
+    {
+        return -RT_EINVAL;
+    }
+
+    dev_name = rt_dm_dev_get_name(ctrl->dev);
+
+    if (rt_bitmap_next_set_bit(ctrl->dir_cap, 0, RT_DMA_DIR_MAX) == RT_DMA_DIR_MAX)
+    {
+        LOG_E("%s: Not direction capability", dev_name);
+
+        return -RT_EINVAL;
+    }
+
+    rt_snprintf(dma_name, sizeof(dma_name), "%s-dmac", dev_name);
+
+    rt_list_init(&ctrl->list);
+
+    rt_spin_lock(&dmac_nodes_lock);
+    rt_list_insert_before(&dmac_nodes, &ctrl->list);
+    rt_spin_unlock(&dmac_nodes_lock);
+
+    rt_list_init(&ctrl->channels_nodes);
+    rt_mutex_init(&ctrl->mutex, dma_name, RT_IPC_FLAG_PRIO);
+
+    if (ctrl->dev->ofw_node)
+    {
+        rt_dm_dev_bind_fwdata(ctrl->dev, RT_NULL, ctrl);
+    }
+
+    return RT_EOK;
+}
+
+rt_err_t rt_dma_controller_unregister(struct rt_dma_controller *ctrl)
+{
+    if (!ctrl)
+    {
+        return -RT_EINVAL;
+    }
+
+    rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER);
+
+    if (!rt_list_isempty(&ctrl->channels_nodes))
+    {
+        rt_mutex_release(&ctrl->mutex);
+        return -RT_EBUSY;
+    }
+
+    if (ctrl->dev->ofw_node)
+    {
+        rt_dm_dev_unbind_fwdata(ctrl->dev, RT_NULL);
+    }
+
+    rt_mutex_release(&ctrl->mutex);
+    rt_mutex_detach(&ctrl->mutex);
+
+    rt_spin_lock(&dmac_nodes_lock);
+    rt_list_remove(&ctrl->list);
+    rt_spin_unlock(&dmac_nodes_lock);
+
+    return RT_EOK;
+}
+
+rt_err_t rt_dma_chan_start(struct rt_dma_chan *chan)
+{
+    rt_err_t err;
+    struct rt_dma_controller *ctrl;
+
+    if (!chan)
+    {
+        return -RT_EINVAL;
+    }
+
+    if (chan->prep_err)
+    {
+        LOG_D("%s: Not config done", rt_dm_dev_get_name(chan->slave));
+
+        return chan->prep_err;
+    }
+
+    ctrl = chan->ctrl;
+
+    rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER);
+
+    err = ctrl->ops->start(chan);
+
+    rt_mutex_release(&ctrl->mutex);
+
+    return err;
+}
+
+rt_err_t rt_dma_chan_stop(struct rt_dma_chan *chan)
+{
+    rt_err_t err;
+    struct rt_dma_controller *ctrl;
+
+    if (!chan)
+    {
+        return -RT_EINVAL;
+    }
+
+    if (chan->prep_err)
+    {
+        LOG_D("%s: Not prepare done", rt_dm_dev_get_name(chan->slave));
+
+        return chan->prep_err;
+    }
+
+    ctrl = chan->ctrl;
+
+    rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER);
+
+    err = ctrl->ops->stop(chan);
+
+    rt_mutex_release(&ctrl->mutex);
+
+    return err;
+}
+
+rt_err_t rt_dma_chan_config(struct rt_dma_chan *chan,
+        struct rt_dma_slave_config *conf)
+{
+    rt_err_t err;
+    struct rt_dma_controller *ctrl;
+    enum rt_dma_transfer_direction dir;
+
+    if (!chan || !conf)
+    {
+        err = -RT_EINVAL;
+        goto _end;
+    }
+
+    dir = conf->direction;
+
+    if (dir >= RT_DMA_DIR_MAX)
+    {
+        err = -RT_EINVAL;
+        goto _end;
+    }
+
+    if (conf->src_addr_width >= RT_DMA_SLAVE_BUSWIDTH_BYTES_MAX ||
+        conf->dst_addr_width >= RT_DMA_SLAVE_BUSWIDTH_BYTES_MAX)
+    {
+        err = -RT_EINVAL;
+        goto _end;
+    }
+
+    ctrl = chan->ctrl;
+
+    if (!rt_bitmap_test_bit(ctrl->dir_cap, dir))
+    {
+        err = -RT_ENOSYS;
+        goto _end;
+    }
+
+    if (!chan->name && dir != RT_DMA_MEM_TO_MEM)
+    {
+        LOG_E("%s: illegal config for uname channels",
+                rt_dm_dev_get_name(ctrl->dev));
+
+        err = -RT_EINVAL;
+        goto _end;
+    }
+
+    rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER);
+
+    err = ctrl->ops->config(chan, conf);
+
+    rt_mutex_release(&ctrl->mutex);
+
+    if (!err)
+    {
+        rt_memcpy(&chan->conf, conf, sizeof(*conf));
+    }
+
+_end:
+    chan->conf_err = err;
+
+    return err;
+}
+
+rt_err_t rt_dma_chan_done(struct rt_dma_chan *chan, rt_size_t size)
+{
+    if (!chan)
+    {
+        return -RT_EINVAL;
+    }
+
+    if (chan->callback)
+    {
+        chan->callback(chan, size);
+    }
+
+    return RT_EOK;
+}
+
+static rt_bool_t range_is_illegal(const char *name, const char *desc,
+        rt_ubase_t addr0, rt_ubase_t addr1)
+{
+    rt_bool_t illegal = addr0 < addr1;
+
+    if (illegal)
+    {
+        LOG_E("%s: %s %p is out of config %p", name, desc, addr0, addr1);
+    }
+
+    return illegal;
+}
+
+rt_err_t rt_dma_prep_memcpy(struct rt_dma_chan *chan,
+        struct rt_dma_slave_transfer *transfer)
+{
+    rt_err_t err;
+    rt_size_t len;
+    rt_ubase_t dma_addr_src, dma_addr_dst;
+    struct rt_dma_controller *ctrl;
+    struct rt_dma_slave_config *conf;
+
+    if (!chan || !transfer)
+    {
+        return -RT_EINVAL;
+    }
+
+    ctrl = chan->ctrl;
+    conf = &chan->conf;
+
+    if (chan->conf_err)
+    {
+        LOG_D("%s: Not config done", rt_dm_dev_get_name(chan->slave));
+
+        return chan->conf_err;
+    }
+
+    RT_ASSERT(chan->conf.direction == RT_DMA_MEM_TO_MEM);
+    dma_addr_src = transfer->src_addr;
+    dma_addr_dst = transfer->dst_addr;
+    len = transfer->buffer_len;
+
+    if (range_is_illegal(rt_dm_dev_get_name(ctrl->dev), "source",
+        dma_addr_src, conf->src_addr))
+    {
+        return -RT_EINVAL;
+    }
+
+    if (range_is_illegal(rt_dm_dev_get_name(ctrl->dev), "dest",
+        dma_addr_dst, conf->dst_addr))
+    {
+        return -RT_EINVAL;
+    }
+
+    if (ctrl->ops->prep_memcpy)
+    {
+        rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER);
+
+        err = ctrl->ops->prep_memcpy(chan, dma_addr_dst, dma_addr_src, len);
+
+        rt_mutex_release(&ctrl->mutex);
+    }
+    else
+    {
+        err = -RT_ENOSYS;
+    }
+
+    if (!err)
+    {
+        rt_memcpy(&chan->transfer, transfer, sizeof(*transfer));
+    }
+
+    chan->prep_err = err;
+
+    return err;
+}
+
+rt_err_t rt_dma_prep_cyclic(struct rt_dma_chan *chan,
+        struct rt_dma_slave_transfer *transfer)
+{
+    rt_err_t err;
+    rt_ubase_t dma_buf_addr;
+    struct rt_dma_controller *ctrl;
+    struct rt_dma_slave_config *conf;
+    enum rt_dma_transfer_direction dir;
+
+    if (!chan || !transfer)
+    {
+        return -RT_EINVAL;
+    }
+
+    ctrl = chan->ctrl;
+    conf = &chan->conf;
+
+    if (chan->conf_err)
+    {
+        LOG_D("%s: Not config done", rt_dm_dev_get_name(chan->slave));
+
+        return chan->conf_err;
+    }
+
+    dir = chan->conf.direction;
+
+    if (dir == RT_DMA_MEM_TO_DEV || dir == RT_DMA_MEM_TO_MEM)
+    {
+        dma_buf_addr = transfer->src_addr;
+
+        if (range_is_illegal(rt_dm_dev_get_name(ctrl->dev), "source",
+            dma_buf_addr, conf->src_addr))
+        {
+            return -RT_EINVAL;
+        }
+    }
+    else if (dir == RT_DMA_DEV_TO_MEM)
+    {
+        dma_buf_addr = transfer->dst_addr;
+
+        if (range_is_illegal(rt_dm_dev_get_name(ctrl->dev), "dest",
+            dma_buf_addr, conf->dst_addr))
+        {
+            return -RT_EINVAL;
+        }
+    }
+    else
+    {
+        dma_buf_addr = ~0UL;
+    }
+
+    if (ctrl->ops->prep_cyclic)
+    {
+        rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER);
+
+        err = ctrl->ops->prep_cyclic(chan, dma_buf_addr,
+                transfer->buffer_len, transfer->period_len, dir);
+
+        rt_mutex_release(&ctrl->mutex);
+    }
+    else
+    {
+        err = -RT_ENOSYS;
+    }
+
+    if (!err)
+    {
+        rt_memcpy(&chan->transfer, transfer, sizeof(*transfer));
+    }
+
+    chan->prep_err = err;
+
+    return err;
+}
+
+rt_err_t rt_dma_prep_single(struct rt_dma_chan *chan,
+        struct rt_dma_slave_transfer *transfer)
+{
+    rt_err_t err;
+    rt_ubase_t dma_buf_addr;
+    struct rt_dma_controller *ctrl;
+    struct rt_dma_slave_config *conf;
+    enum rt_dma_transfer_direction dir;
+
+    if (!chan || !transfer)
+    {
+        return -RT_EINVAL;
+    }
+
+    ctrl = chan->ctrl;
+    conf = &chan->conf;
+
+    if (chan->conf_err)
+    {
+        LOG_D("%s: Not config done", rt_dm_dev_get_name(chan->slave));
+
+        return chan->conf_err;
+    }
+
+    dir = chan->conf.direction;
+
+    if (dir == RT_DMA_MEM_TO_DEV || dir == RT_DMA_MEM_TO_MEM)
+    {
+        dma_buf_addr = transfer->src_addr;
+
+        if (range_is_illegal(rt_dm_dev_get_name(ctrl->dev), "source",
+            dma_buf_addr, conf->src_addr))
+        {
+            return -RT_EINVAL;
+        }
+    }
+    else if (dir == RT_DMA_DEV_TO_MEM)
+    {
+        dma_buf_addr = transfer->dst_addr;
+
+        if (range_is_illegal(rt_dm_dev_get_name(ctrl->dev), "dest",
+            dma_buf_addr, conf->dst_addr))
+        {
+            return -RT_EINVAL;
+        }
+    }
+    else
+    {
+        dma_buf_addr = ~0UL;
+    }
+
+    if (ctrl->ops->prep_single)
+    {
+        rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER);
+
+        err = ctrl->ops->prep_single(chan, dma_buf_addr,
+                transfer->buffer_len, dir);
+
+        rt_mutex_release(&ctrl->mutex);
+    }
+    else
+    {
+        err = -RT_ENOSYS;
+    }
+
+    if (!err)
+    {
+        rt_memcpy(&chan->transfer, transfer, sizeof(*transfer));
+    }
+
+    chan->prep_err = err;
+
+    return err;
+}
+
+static struct rt_dma_controller *ofw_find_dma_controller(struct rt_device *dev,
+        const char *name)
+{
+    struct rt_dma_controller *ctrl = RT_NULL;
+#ifdef RT_USING_OFW
+    int index;
+    rt_err_t err;
+    struct rt_ofw_cell_args dma_args = {};
+    struct rt_ofw_node *np = dev->ofw_node, *ctrl_np;
+
+    if (!np)
+    {
+        return RT_NULL;
+    }
+
+    index = rt_ofw_prop_index_of_string(np, "dma-names", name);
+
+    if (index < 0)
+    {
+        return RT_NULL;
+    }
+
+    if (!rt_ofw_parse_phandle_cells(np, "dmas", "#dma-cells", index, &dma_args))
+    {
+        ctrl_np = dma_args.data;
+
+        if (!rt_ofw_data(ctrl_np))
+        {
+            rt_platform_ofw_request(ctrl_np);
+        }
+
+        ctrl = rt_ofw_data(ctrl_np);
+        rt_ofw_node_put(ctrl_np);
+
+        if (ctrl && ctrl->ops->ofw_parse)
+        {
+            if ((err = ctrl->ops->ofw_parse(ctrl, &dma_args)))
+            {
+                ctrl = rt_err_ptr(err);
+            }
+        }
+    }
+#endif /* RT_USING_OFW */
+    return ctrl;
+}
+
+struct rt_dma_chan *rt_dma_chan_request(struct rt_device *dev, const char *name)
+{
+    struct rt_dma_chan *chan;
+    struct rt_dma_controller *ctrl = RT_NULL;
+
+    if (!dev)
+    {
+        return rt_err_ptr(-RT_EINVAL);
+    }
+
+    if (name)
+    {
+        ctrl = ofw_find_dma_controller(dev, name);
+    }
+    else
+    {
+        struct rt_dma_controller *ctrl_tmp;
+
+        rt_spin_lock(&dmac_nodes_lock);
+        rt_list_for_each_entry(ctrl_tmp, &dmac_nodes, list)
+        {
+            /* Only memory to memory for uname request */
+            if (rt_bitmap_test_bit(ctrl_tmp->dir_cap, RT_DMA_MEM_TO_MEM))
+            {
+                ctrl = ctrl_tmp;
+                break;
+            }
+        }
+        rt_spin_unlock(&dmac_nodes_lock);
+    }
+
+    if (rt_is_err_or_null(ctrl))
+    {
+        return ctrl ? ctrl : rt_err_ptr(-RT_ENOSYS);
+    }
+
+    if (ctrl->ops->request_chan)
+    {
+        chan = ctrl->ops->request_chan(ctrl, dev);
+    }
+    else
+    {
+        chan = rt_calloc(1, sizeof(*chan));
+
+        if (!chan)
+        {
+            chan = rt_err_ptr(-RT_ENOMEM);
+        }
+    }
+
+    if (rt_is_err(chan))
+    {
+        return chan;
+    }
+
+    if (!chan)
+    {
+        LOG_E("%s: unset request channels error", rt_dm_dev_get_name(ctrl->dev));
+
+        return rt_err_ptr(-RT_ERROR);
+    }
+
+    chan->name = name;
+    chan->ctrl = ctrl;
+    chan->slave = dev;
+
+    rt_list_init(&chan->list);
+    chan->conf_err = -RT_ERROR;
+    chan->prep_err = -RT_ERROR;
+
+    rt_mutex_take(&ctrl->mutex, RT_WAITING_FOREVER);
+    rt_list_insert_before(&ctrl->channels_nodes, &chan->list);
+    rt_mutex_release(&ctrl->mutex);
+
+    return chan;
+}
+
+rt_err_t rt_dma_chan_release(struct rt_dma_chan *chan)
+{
+    rt_err_t err = RT_EOK;
+
+    if (!chan)
+    {
+        return -RT_EINVAL;
+    }
+
+    rt_mutex_take(&chan->ctrl->mutex, RT_WAITING_FOREVER);
+    rt_list_remove(&chan->list);
+    rt_mutex_release(&chan->ctrl->mutex);
+
+    if (chan->ctrl->ops->release_chan)
+    {
+        err = chan->ctrl->ops->release_chan(chan);
+    }
+    else
+    {
+        rt_free(chan);
+    }
+
+    return err;
+}

components/drivers/dma/dma_pool.c

@@ -0,0 +1,685 @@
+/*
+ * Copyright (c) 2006-2023, RT-Thread Development Team
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Change Logs:
+ * Date           Author       Notes
+ * 2023-02-25     GuEe-GUI     the first version
+ */
+
+#include <rthw.h>
+#include <rtthread.h>
+#include <rtdevice.h>
+
+#define DBG_TAG "dma.pool"
+#define DBG_LVL DBG_INFO
+#include <rtdbg.h>
+
+#include <mm_aspace.h>
+#include <dt-bindings/size.h>
+
+static struct rt_spinlock dma_pools_lock = {};
+static rt_list_t dma_pool_nodes = RT_LIST_OBJECT_INIT(dma_pool_nodes);
+
+static struct rt_dma_pool *dma_pool_install(rt_region_t *region);
+
+static void *dma_alloc(struct rt_device *dev, rt_size_t size,
+        rt_ubase_t *dma_handle, rt_ubase_t flags);
+static void dma_free(struct rt_device *dev, rt_size_t size,
+        void *cpu_addr, rt_ubase_t dma_handle, rt_ubase_t flags);
+
+rt_inline void region_pool_lock(void)
+{
+    rt_hw_spin_lock(&dma_pools_lock.lock);
+}
+
+rt_inline void region_pool_unlock(void)
+{
+    rt_hw_spin_unlock(&dma_pools_lock.lock);
+}
+
+static rt_err_t dma_map_coherent_sync_out_data(struct rt_device *dev,
+        void *data, rt_size_t size, rt_ubase_t *dma_handle, rt_ubase_t flags)
+{
+    if (dma_handle)
+    {
+        *dma_handle = (rt_ubase_t)rt_kmem_v2p(data);
+    }
+    rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, data, size);
+
+    return RT_EOK;
+}
+
+static rt_err_t dma_map_coherent_sync_in_data(struct rt_device *dev,
+        void *out_data, rt_size_t size, rt_ubase_t dma_handle, rt_ubase_t flags)
+{
+    rt_hw_cpu_dcache_ops(RT_HW_CACHE_INVALIDATE, out_data, size);
+
+    return RT_EOK;
+}
+
+static const struct rt_dma_map_ops dma_map_coherent_ops =
+{
+    .sync_out_data = dma_map_coherent_sync_out_data,
+    .sync_in_data = dma_map_coherent_sync_in_data,
+};
+
+static rt_err_t dma_map_nocoherent_sync_out_data(struct rt_device *dev,
+        void *data, rt_size_t size, rt_ubase_t *dma_handle, rt_ubase_t flags)
+{
+    if (dma_handle)
+    {
+        *dma_handle = (rt_ubase_t)rt_kmem_v2p(data);
+    }
+
+    return RT_EOK;
+}
+
+static rt_err_t dma_map_nocoherent_sync_in_data(struct rt_device *dev,
+        void *out_data, rt_size_t size, rt_ubase_t dma_handle, rt_ubase_t flags)
+{
+    return RT_EOK;
+}
+
+static const struct rt_dma_map_ops dma_map_nocoherent_ops =
+{
+    .sync_out_data = dma_map_nocoherent_sync_out_data,
+    .sync_in_data = dma_map_nocoherent_sync_in_data,
+};
+
+#ifdef RT_USING_OFW
+rt_inline rt_ubase_t ofw_addr_cpu2dma(struct rt_device *dev, rt_ubase_t addr)
+{
+    return (rt_ubase_t)rt_ofw_translate_cpu2dma(dev->ofw_node, addr);
+}
+
+rt_inline rt_ubase_t ofw_addr_dma2cpu(struct rt_device *dev, rt_ubase_t addr)
+{
+    return (rt_ubase_t)rt_ofw_translate_dma2cpu(dev->ofw_node, addr);
+}
+
+static void *ofw_dma_map_alloc(struct rt_device *dev, rt_size_t size,
+        rt_ubase_t *dma_handle, rt_ubase_t flags)
+{
+    void *cpu_addr = dma_alloc(dev, size, dma_handle, flags);
+
+    if (cpu_addr && dma_handle)
+    {
+        *dma_handle = ofw_addr_cpu2dma(dev, *dma_handle);
+    }
+
+    return cpu_addr;
+}
+
+static void ofw_dma_map_free(struct rt_device *dev, rt_size_t size,
+        void *cpu_addr, rt_ubase_t dma_handle, rt_ubase_t flags)
+{
+    dma_handle = ofw_addr_dma2cpu(dev, dma_handle);
+
+    dma_free(dev, size, cpu_addr, dma_handle, flags);
+}
+
+static rt_err_t ofw_dma_map_sync_out_data(struct rt_device *dev,
+        void *data, rt_size_t size,
+        rt_ubase_t *dma_handle, rt_ubase_t flags)
+{
+    rt_err_t err;
+
+    if (flags & RT_DMA_F_NOCACHE)
+    {
+        err = dma_map_nocoherent_sync_out_data(dev, data, size, dma_handle, flags);
+    }
+    else
+    {
+        err = dma_map_coherent_sync_out_data(dev, data, size, dma_handle, flags);
+    }
+
+    if (!err && dma_handle)
+    {
+        *dma_handle = ofw_addr_cpu2dma(dev, *dma_handle);
+    }
+
+    return err;
+}
+
+static rt_err_t ofw_dma_map_sync_in_data(struct rt_device *dev,
+        void *out_data, rt_size_t size,
+        rt_ubase_t dma_handle, rt_ubase_t flags)
+{
+    dma_handle = ofw_addr_dma2cpu(dev, dma_handle);
+
+    if (flags & RT_DMA_F_NOCACHE)
+    {
+        return dma_map_nocoherent_sync_in_data(dev, out_data, size, dma_handle, flags);
+    }
+
+    return dma_map_coherent_sync_in_data(dev, out_data, size, dma_handle, flags);
+}
+
+static const struct rt_dma_map_ops ofw_dma_map_ops =
+{
+    .alloc = ofw_dma_map_alloc,
+    .free = ofw_dma_map_free,
+    .sync_out_data = ofw_dma_map_sync_out_data,
+    .sync_in_data = ofw_dma_map_sync_in_data,
+};
+
+static const struct rt_dma_map_ops *ofw_device_dma_ops(struct rt_device *dev)
+{
+    rt_err_t err;
+    int region_nr = 0;
+    const fdt32_t *cell;
+    rt_phandle phandle;
+    rt_region_t region;
+    struct rt_ofw_prop *prop;
+    struct rt_dma_pool *dma_pool;
+    const struct rt_dma_map_ops *ops = RT_NULL;
+    struct rt_ofw_node *mem_np, *np = dev->ofw_node;
+
+    rt_ofw_foreach_prop_u32(np, "memory-region", prop, cell, phandle)
+    {
+        rt_uint64_t addr, size;
+
+        if (!(mem_np = rt_ofw_find_node_by_phandle(phandle)))
+        {
+            if (region_nr == 0)
+            {
+                return RT_NULL;
+            }
+
+            break;
+        }
+
+        if ((err = rt_ofw_get_address(mem_np, 0, &addr, &size)))
+        {
+            LOG_E("%s: Read '%s' error = %s", rt_ofw_node_full_name(mem_np),
+                    "memory-region", rt_strerror(err));
+
+            rt_ofw_node_put(mem_np);
+            continue;
+        }
+
+        region.start = addr;
+        region.end = addr + size;
+        region.name = rt_dm_dev_get_name(dev);
+
+        rt_ofw_node_put(mem_np);
+
+        if (!(dma_pool = dma_pool_install(&region)))
+        {
+            return RT_NULL;
+        }
+
+        if (rt_ofw_prop_read_bool(mem_np, "no-map"))
+        {
+            dma_pool->flags |= RT_DMA_F_NOMAP;
+        }
+
+        if (!rt_dma_device_is_coherent(dev))
+        {
+            dma_pool->flags |= RT_DMA_F_NOCACHE;
+        }
+
+        dma_pool->dev = dev;
+        ++region_nr;
+    }
+
+    if (region_nr)
+    {
+        ops = &ofw_dma_map_ops;
+    }
+
+    return ops;
+}
+#endif /* RT_USING_OFW */
+
+static const struct rt_dma_map_ops *device_dma_ops(struct rt_device *dev)
+{
+    const struct rt_dma_map_ops *ops = dev->dma_ops;
+
+    if (ops)
+    {
+        return ops;
+    }
+
+#ifdef RT_USING_OFW
+    if (dev->ofw_node && (ops = ofw_device_dma_ops(dev)))
+    {
+        return ops;
+    }
+#endif
+
+    if (rt_dma_device_is_coherent(dev))
+    {
+        ops = &dma_map_coherent_ops;
+    }
+    else
+    {
+        ops = &dma_map_nocoherent_ops;
+    }
+
+    dev->dma_ops = ops;
+
+    return ops;
+}
+
+static rt_ubase_t dma_pool_alloc(struct rt_dma_pool *pool, rt_size_t size)
+{
+    rt_size_t bit, next_bit, end_bit, max_bits;
+
+    size = RT_DIV_ROUND_UP(size, ARCH_PAGE_SIZE);
+    max_bits = pool->bits - size;
+
+    rt_bitmap_for_each_clear_bit(pool->map, bit, max_bits)
+    {
+        end_bit = bit + size;
+
+        for (next_bit = bit + 1; next_bit < end_bit; ++next_bit)
+        {
+            if (rt_bitmap_test_bit(pool->map, next_bit))
+            {
+                bit = next_bit;
+                goto _next;
+            }
+        }
+
+        if (next_bit == end_bit)
+        {
+            while (next_bit --> bit)
+            {
+                rt_bitmap_set_bit(pool->map, next_bit);
+            }
+
+            return pool->start + bit * ARCH_PAGE_SIZE;
+        }
+    _next:
+    }
+
+    return RT_NULL;
+}
+
+static void dma_pool_free(struct rt_dma_pool *pool, rt_ubase_t offset, rt_size_t size)
+{
+    rt_size_t bit = (offset - pool->start) / ARCH_PAGE_SIZE, end_bit;
+
+    size = RT_DIV_ROUND_UP(size, ARCH_PAGE_SIZE);
+    end_bit = bit + size;
+
+    for (; bit < end_bit; ++bit)
+    {
+        rt_bitmap_clear_bit(pool->map, bit);
+    }
+}
+
+static void *dma_alloc(struct rt_device *dev, rt_size_t size,
+        rt_ubase_t *dma_handle, rt_ubase_t flags)
+{
+    void *dma_buffer = RT_NULL;
+    struct rt_dma_pool *pool;
+
+    region_pool_lock();
+
+    rt_list_for_each_entry(pool, &dma_pool_nodes, list)
+    {
+        if ((flags & RT_DMA_F_DEVICE) &&
+            (!(pool->flags & RT_DMA_F_DEVICE) || pool->dev != dev))
+        {
+            continue;
+        }
+
+        if ((flags & RT_DMA_F_NOMAP) && !((pool->flags & RT_DMA_F_NOMAP)))
+        {
+            continue;
+        }
+
+        if ((flags & RT_DMA_F_32BITS) && !((pool->flags & RT_DMA_F_32BITS)))
+        {
+            continue;
+        }
+
+        if ((flags & RT_DMA_F_LINEAR) && !((pool->flags & RT_DMA_F_LINEAR)))
+        {
+            continue;
+        }
+
+        *dma_handle = dma_pool_alloc(pool, size);
+
+        if (*dma_handle && !(flags & RT_DMA_F_NOMAP))
+        {
+            if (flags & RT_DMA_F_NOCACHE)
+            {
+                dma_buffer = rt_ioremap_nocache((void *)*dma_handle, size);
+            }
+            else
+            {
+                dma_buffer = rt_ioremap_cached((void *)*dma_handle, size);
+            }
+
+            if (!dma_buffer)
+            {
+                dma_pool_free(pool, *dma_handle, size);
+
+                continue;
+            }
+
+            break;
+        }
+        else if (*dma_handle)
+        {
+            dma_buffer = (void *)*dma_handle;
+
+            break;
+        }
+    }
+
+    region_pool_unlock();
+
+    return dma_buffer;
+}
+
+static void dma_free(struct rt_device *dev, rt_size_t size,
+        void *cpu_addr, rt_ubase_t dma_handle, rt_ubase_t flags)
+{
+    struct rt_dma_pool *pool;
+
+    region_pool_lock();
+
+    rt_list_for_each_entry(pool, &dma_pool_nodes, list)
+    {
+        if (dma_handle >= pool->region.start &&
+            dma_handle <= pool->region.end)
+        {
+            rt_iounmap(cpu_addr);
+
+            dma_pool_free(pool, dma_handle, size);
+
+            break;
+        }
+    }
+
+    region_pool_unlock();
+}
+
+void *rt_dma_alloc(struct rt_device *dev, rt_size_t size,
+        rt_ubase_t *dma_handle, rt_ubase_t flags)
+{
+    void *dma_buffer = RT_NULL;
+    rt_ubase_t dma_handle_s = 0;
+    const struct rt_dma_map_ops *ops;
+
+    if (!dev || !size)
+    {
+        return RT_NULL;
+    }
+
+    ops = device_dma_ops(dev);
+
+    if (ops->alloc)
+    {
+        dma_buffer = ops->alloc(dev, size, &dma_handle_s, flags);
+    }
+    else
+    {
+        dma_buffer = dma_alloc(dev, size, &dma_handle_s, flags);
+    }
+
+    if (!dma_buffer)
+    {
+        return dma_buffer;
+    }
+
+    if (dma_handle)
+    {
+        *dma_handle = dma_handle_s;
+    }
+
+    return dma_buffer;
+}
+
+void rt_dma_free(struct rt_device *dev, rt_size_t size,
+        void *cpu_addr, rt_ubase_t dma_handle, rt_ubase_t flags)
+{
+    const struct rt_dma_map_ops *ops;
+
+    if (!dev || !size || !cpu_addr)
+    {
+        return;
+    }
+
+    ops = device_dma_ops(dev);
+
+    if (ops->free)
+    {
+        ops->free(dev, size, cpu_addr, dma_handle, flags);
+    }
+    else
+    {
+        dma_free(dev, size, cpu_addr, dma_handle, flags);
+    }
+}
+
+rt_err_t rt_dma_sync_out_data(struct rt_device *dev, void *data, rt_size_t size,
+        rt_ubase_t *dma_handle, rt_ubase_t flags)
+{
+    rt_err_t err;
+    rt_ubase_t dma_handle_s = 0;
+    const struct rt_dma_map_ops *ops;
+
+    if (!data || !size)
+    {
+        return -RT_EINVAL;
+    }
+
+    ops = device_dma_ops(dev);
+    err = ops->sync_out_data(dev, data, size, &dma_handle_s, flags);
+
+    if (dma_handle)
+    {
+        *dma_handle = dma_handle_s;
+    }
+
+    return err;
+}
+
+rt_err_t rt_dma_sync_in_data(struct rt_device *dev, void *out_data, rt_size_t size,
+        rt_ubase_t dma_handle, rt_ubase_t flags)
+{
+    rt_err_t err;
+    const struct rt_dma_map_ops *ops;
+
+    if (!out_data || !size)
+    {
+        return -RT_EINVAL;
+    }
+
+    ops = device_dma_ops(dev);
+    err = ops->sync_in_data(dev, out_data, size, dma_handle, flags);
+
+    return err;
+}
+
+static struct rt_dma_pool *dma_pool_install(rt_region_t *region)
+{
+    rt_err_t err;
+    struct rt_dma_pool *pool;
+
+    if (!(pool = rt_calloc(1, sizeof(*pool))))
+    {
+        LOG_E("Install pool[%p, %p] error = %s",
+                region->start, region->end, rt_strerror(-RT_ENOMEM));
+
+        return RT_NULL;
+    }
+
+    rt_memcpy(&pool->region, region, sizeof(*region));
+
+    pool->flags |= RT_DMA_F_LINEAR;
+
+    if (region->end < 4UL * SIZE_GB)
+    {
+        pool->flags |= RT_DMA_F_32BITS;
+    }
+
+    pool->start = RT_ALIGN(pool->region.start, ARCH_PAGE_SIZE);
+    pool->bits = (pool->region.end - pool->start) / ARCH_PAGE_SIZE;
+
+    if (!pool->bits)
+    {
+        err = -RT_EINVAL;
+        goto _fail;
+    }
+
+    pool->map = rt_calloc(RT_BITMAP_LEN(pool->bits), sizeof(*pool->map));
+
+    if (!pool->map)
+    {
+        err = -RT_ENOMEM;
+        goto _fail;
+    }
+
+    rt_list_init(&pool->list);
+
+    region_pool_lock();
+    rt_list_insert_before(&dma_pool_nodes, &pool->list);
+    region_pool_unlock();
+
+    return pool;
+
+_fail:
+    rt_free(pool);
+
+    LOG_E("Install pool[%p, %p] error = %s",
+            region->start, region->end, rt_strerror(err));
+
+    return RT_NULL;
+}
+
+struct rt_dma_pool *rt_dma_pool_install(rt_region_t *region)
+{
+    struct rt_dma_pool *pool;
+
+    if (!region)
+    {
+        return RT_NULL;
+    }
+
+    if ((pool = dma_pool_install(region)))
+    {
+        region = &pool->region;
+
+        LOG_I("%s: Reserved %u.%u MiB at %p",
+                region->name,
+                (region->end - region->start) / SIZE_MB,
+                (region->end - region->start) / SIZE_KB & (SIZE_KB - 1),
+                region->start);
+    }
+
+    return pool;
+}
+
+rt_err_t rt_dma_pool_extract(rt_region_t *region_list, rt_size_t list_len,
+        rt_size_t cma_size, rt_size_t coherent_pool_size)
+{
+    struct rt_dma_pool *pool;
+    rt_region_t *region = region_list, *region_high = RT_NULL, cma, coherent_pool;
+
+    if (!region_list || !list_len || cma_size < coherent_pool_size)
+    {
+        return -RT_EINVAL;
+    }
+
+    for (rt_size_t i = 0; i < list_len; ++i, ++region)
+    {
+        if (!region->name)
+        {
+            continue;
+        }
+
+        /* Always use low address in 4G */
+        if (region->end - region->start >= cma_size)
+        {
+            if ((rt_ssize_t)((4UL * SIZE_GB) - region->start) < cma_size)
+            {
+                region_high = region;
+                continue;
+            }
+
+            goto _found;
+        }
+    }
+
+    if (region_high)
+    {
+        region = region_high;
+        LOG_W("No available DMA zone in 4G");
+
+        goto _found;
+    }
+
+    return -RT_EEMPTY;
+
+_found:
+    if (region->end - region->start != cma_size)
+    {
+        cma.start = region->start;
+        cma.end = cma.start + cma_size;
+
+        /* Update input region */
+        region->start += cma_size;
+    }
+    else
+    {
+        rt_memcpy(&cma, region, sizeof(cma));
+    }
+
+    coherent_pool.name = "coherent-pool";
+    coherent_pool.start = cma.start;
+    coherent_pool.end = coherent_pool.start + coherent_pool_size;
+
+    cma.name = "cma";
+    cma.start += coherent_pool_size;
+
+    if (!(pool = rt_dma_pool_install(&coherent_pool)))
+    {
+        return -RT_ENOMEM;
+    }
+
+    /* Use: CMA > coherent-pool */
+    if (!(pool = rt_dma_pool_install(&cma)))
+    {
+        return -RT_ENOMEM;
+    }
+
+    return RT_EOK;
+}
+
+#if defined(RT_USING_CONSOLE) && defined(RT_USING_MSH)
+static int list_dma_pool(int argc, char**argv)
+{
+    int count = 0;
+    rt_region_t *region;
+    struct rt_dma_pool *pool;
+
+    rt_kprintf("%-*.s Region\n", RT_NAME_MAX, "Name");
+
+    region_pool_lock();
+
+    rt_list_for_each_entry(pool, &dma_pool_nodes, list)
+    {
+        region = &pool->region;
+
+        rt_kprintf("%-*.s [%p, %p]\n", RT_NAME_MAX, region->name,
+                region->start, region->end);
+
+        ++count;
+    }
+
+    rt_kprintf("%d DMA memory found\n", count);
+
+    region_pool_unlock();
+
+    return 0;
+}
+MSH_CMD_EXPORT(list_dma_pool, dump all dma memory pool);
+#endif /* RT_USING_CONSOLE && RT_USING_MSH */

components/drivers/include/drivers/dma.h

@@ -0,0 +1,235 @@
+/*
+ * Copyright (c) 2006-2023, RT-Thread Development Team
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Change Logs:
+ * Date           Author       Notes
+ * 2023-02-25     GuEe-GUI     the first version
+ */
+
+#ifndef __DMA_H__
+#define __DMA_H__
+
+#include <rtthread.h>
+#include <drivers/ofw.h>
+#include <drivers/misc.h>
+#include <drivers/core/dm.h>
+
+#include <mmu.h>
+#include <mm_page.h>
+#include <bitmap.h>
+
+struct rt_dma_chan;
+struct rt_dma_controller_ops;
+
+enum rt_dma_transfer_direction
+{
+    RT_DMA_MEM_TO_MEM,
+    RT_DMA_MEM_TO_DEV,
+    RT_DMA_DEV_TO_MEM,
+    RT_DMA_DEV_TO_DEV,
+
+    RT_DMA_DIR_MAX,
+};
+
+enum rt_dma_slave_buswidth
+{
+    RT_DMA_SLAVE_BUSWIDTH_UNDEFINED     = 0,
+    RT_DMA_SLAVE_BUSWIDTH_1_BYTE        = 1,
+    RT_DMA_SLAVE_BUSWIDTH_2_BYTES       = 2,
+    RT_DMA_SLAVE_BUSWIDTH_3_BYTES       = 3,
+    RT_DMA_SLAVE_BUSWIDTH_4_BYTES       = 4,
+    RT_DMA_SLAVE_BUSWIDTH_8_BYTES       = 8,
+    RT_DMA_SLAVE_BUSWIDTH_16_BYTES      = 16,
+    RT_DMA_SLAVE_BUSWIDTH_32_BYTES      = 32,
+    RT_DMA_SLAVE_BUSWIDTH_64_BYTES      = 64,
+    RT_DMA_SLAVE_BUSWIDTH_128_BYTES     = 128,
+
+    RT_DMA_SLAVE_BUSWIDTH_BYTES_MAX,
+};
+
+struct rt_dma_slave_config
+{
+    enum rt_dma_transfer_direction direction;
+    enum rt_dma_slave_buswidth src_addr_width;
+    enum rt_dma_slave_buswidth dst_addr_width;
+
+    rt_ubase_t src_addr;
+    rt_ubase_t dst_addr;
+
+    rt_uint32_t src_maxburst;
+    rt_uint32_t dst_maxburst;
+    rt_uint32_t src_port_window_size;
+    rt_uint32_t dst_port_window_size;
+};
+
+struct rt_dma_slave_transfer
+{
+    rt_ubase_t src_addr;
+    rt_ubase_t dst_addr;
+
+    void *buffer;
+    rt_ubase_t dma_handle;
+    rt_size_t buffer_len;
+    rt_size_t period_len;
+};
+
+struct rt_dma_controller
+{
+    rt_list_t list;
+
+    struct rt_device *dev;
+
+    RT_BITMAP_DECLARE(dir_cap, RT_DMA_DIR_MAX);
+    const struct rt_dma_controller_ops *ops;
+
+    rt_list_t channels_nodes;
+    struct rt_mutex mutex;
+};
+
+struct rt_dma_controller_ops
+{
+    struct rt_dma_chan *(*request_chan)(struct rt_dma_controller *ctrl, struct rt_device *slave);
+    rt_err_t (*release_chan)(struct rt_dma_chan *chan);
+
+    rt_err_t (*start)(struct rt_dma_chan *chan);
+    rt_err_t (*stop)(struct rt_dma_chan *chan);
+    rt_err_t (*config)(struct rt_dma_chan *chan, struct rt_dma_slave_config *conf);
+
+    rt_err_t (*prep_memcpy)(struct rt_dma_chan *chan,
+            rt_ubase_t dma_addr_src, rt_ubase_t dma_addr_dst, rt_size_t len);
+
+    rt_err_t (*prep_cyclic)(struct rt_dma_chan *chan,
+            rt_ubase_t dma_buf_addr, rt_size_t buf_len, rt_size_t period_len,
+            enum rt_dma_transfer_direction dir);
+
+    rt_err_t (*prep_single)(struct rt_dma_chan *chan,
+            rt_ubase_t dma_buf_addr, rt_size_t buf_len,
+            enum rt_dma_transfer_direction dir);
+
+    rt_err_t (*ofw_parse)(struct rt_dma_controller *ctrl, struct rt_ofw_cell_args *dma_args);
+};
+
+struct rt_dma_chan
+{
+    const char *name;
+
+    struct rt_dma_controller *ctrl;
+    struct rt_device *slave;
+
+    rt_list_t list;
+    rt_err_t conf_err;
+    rt_err_t prep_err;
+    struct rt_dma_slave_config conf;
+    struct rt_dma_slave_transfer transfer;
+
+    void (*callback)(struct rt_dma_chan *chan, rt_size_t size);
+
+    void *priv;
+};
+
+struct rt_dma_pool
+{
+    rt_region_t region;
+
+    rt_list_t list;
+
+    rt_ubase_t flags;
+
+    rt_bitmap_t *map;
+    rt_size_t bits;
+    rt_ubase_t start;
+
+    struct rt_device *dev;
+};
+
+struct rt_dma_map_ops
+{
+    void *(*alloc)(struct rt_device *dev, rt_size_t size,
+            rt_ubase_t *dma_handle, rt_ubase_t flags);
+    void (*free)(struct rt_device *dev, rt_size_t size,
+            void *cpu_addr, rt_ubase_t dma_handle, rt_ubase_t flags);
+    rt_err_t (*sync_out_data)(struct rt_device *dev, void *data, rt_size_t size,
+            rt_ubase_t *dma_handle, rt_ubase_t flags);
+    rt_err_t (*sync_in_data)(struct rt_device *dev, void *out_data, rt_size_t size,
+            rt_ubase_t dma_handle, rt_ubase_t flags);
+};
+
+rt_inline void rt_dma_controller_add_direction(struct rt_dma_controller *ctrl,
+        enum rt_dma_transfer_direction dir)
+{
+    RT_ASSERT(ctrl != RT_NULL);
+    RT_ASSERT(dir < RT_DMA_DIR_MAX);
+
+    rt_bitmap_set_bit(ctrl->dir_cap, dir);
+}
+
+rt_err_t rt_dma_controller_register(struct rt_dma_controller *ctrl);
+rt_err_t rt_dma_controller_unregister(struct rt_dma_controller *ctrl);
+
+rt_err_t rt_dma_chan_start(struct rt_dma_chan *chan);
+rt_err_t rt_dma_chan_stop(struct rt_dma_chan *chan);
+rt_err_t rt_dma_chan_config(struct rt_dma_chan *chan,
+        struct rt_dma_slave_config *conf);
+rt_err_t rt_dma_chan_done(struct rt_dma_chan *chan, rt_size_t size);
+
+rt_err_t rt_dma_prep_memcpy(struct rt_dma_chan *chan,
+        struct rt_dma_slave_transfer *transfer);
+rt_err_t rt_dma_prep_cyclic(struct rt_dma_chan *chan,
+        struct rt_dma_slave_transfer *transfer);
+rt_err_t rt_dma_prep_single(struct rt_dma_chan *chan,
+        struct rt_dma_slave_transfer *transfer);
+
+struct rt_dma_chan *rt_dma_chan_request(struct rt_device *dev, const char *name);
+rt_err_t rt_dma_chan_release(struct rt_dma_chan *chan);
+
+#define RT_DMA_F_LINEAR     RT_BIT(0)
+#define RT_DMA_F_32BITS     RT_BIT(1)
+#define RT_DMA_F_NOCACHE    RT_BIT(2)
+#define RT_DMA_F_DEVICE     RT_BIT(3)
+#define RT_DMA_F_NOMAP      RT_BIT(4)
+
+#define RT_DMA_PAGE_SIZE    ARCH_PAGE_SIZE
+
+void *rt_dma_alloc(struct rt_device *dev, rt_size_t size,
+        rt_ubase_t *dma_handle, rt_ubase_t flags);
+
+void rt_dma_free(struct rt_device *dev, rt_size_t size,
+        void *cpu_addr, rt_ubase_t dma_handle, rt_ubase_t flags);
+
+rt_inline void *rt_dma_alloc_coherent(struct rt_device *dev, rt_size_t size,
+        rt_ubase_t *dma_handle)
+{
+    return rt_dma_alloc(dev, size, dma_handle,
+            RT_DMA_F_NOCACHE | RT_DMA_F_LINEAR);
+}
+
+rt_inline void rt_dma_free_coherent(struct rt_device *dev, rt_size_t size,
+        void *cpu_addr, rt_ubase_t dma_handle)
+{
+    rt_dma_free(dev, size, cpu_addr, dma_handle,
+            RT_DMA_F_NOCACHE | RT_DMA_F_LINEAR);
+}
+
+rt_err_t rt_dma_sync_out_data(struct rt_device *dev, void *data, rt_size_t size,
+        rt_ubase_t *dma_handle, rt_ubase_t flags);
+rt_err_t rt_dma_sync_in_data(struct rt_device *dev, void *out_data, rt_size_t size,
+        rt_ubase_t dma_handle, rt_ubase_t flags);
+
+rt_inline rt_bool_t rt_dma_device_is_coherent(struct rt_device *dev)
+{
+    return rt_dm_dev_prop_read_bool(dev, "dma-coherent");
+}
+
+rt_inline void rt_dma_device_set_ops(struct rt_device *dev,
+        const struct rt_dma_map_ops *ops)
+{
+    dev->dma_ops = ops;
+}
+
+struct rt_dma_pool *rt_dma_pool_install(rt_region_t *region);
+rt_err_t rt_dma_pool_extract(rt_region_t *region_list, rt_size_t list_len,
+        rt_size_t cma_size, rt_size_t coherent_pool_size);
+
+#endif /* __DMA_H__ */

components/drivers/include/rtdevice.h

@@ -49,6 +49,10 @@ extern "C" {
 #include "drivers/blk.h"
 #endif
 
+#ifdef RT_USING_DMA
+#include "drivers/dma.h"
+#endif
+
 #include "drivers/iio.h"
 
 #ifdef RT_USING_OFW