DMA transaction requires copying into buffer every time? - linux-kernel

Question

It might be a silly question, but I haven't got it so far about DMA.
When doing memory to memory DMAing, it requires to allocate DMA buffer
(for example with dma_alloc_coherent()), then for each transfer we need to copy the buffer to the allocated memory (source buffer) and then trigger DMA transaction.
So, if it requires additional memcpy() for each transaction, what's the
benefit of using DMA?
Steps for copying source to destination - without DMA:
copy buffer (memcpy()) from source to destination
Steps for copying source to destination - with DMA:
copy buffer (memcpy()) from source to DMA buffer
trigger DMA transaction (which shall copy the buffer eventually to
destination buffer)
An example of this problem is with Ethernet driver, which need to copy from the recieved sk_buf into physical address of FPGA. In this case it shall requires copying the sk_buf into the DMA source buffer (from dma_alloc_coherent()) first.

Answer 1

If you can use dma_map_single() with the sk_buf pointer, then you do not have to copy it into a buffer allocated with dma_alloc_coherent(). There are many examples of this in network device drivers.
int dma_len = skb->len;
dma_addr_t dma_addr = dma_map_single(dev, skb->data, skb->len, DMA_TO_DEVICE);
// error checking code here
// then send the dma_addr to the drvice
// when it is done, unmap it
dma_unmap_single(dev, dma_addr, dma_len, DMA_TO_DEVICE);
See the DMA Mapping API documentation for more details.

Answer 2

I guess my answer is no longer relevant to the post owner, but maybe it would help some other programmers in the future.
As mentioned in one of the comments here, if your FPGA have a DMA controller (which allows the FPGA to read the memory that has been mapped to the DMA), then you should be able to do DMA without memcpy() operations.
I will try to give here a short (as possible...) example of how it can be implemented in an Ethernet driver in the Rx flow (but there are more than one method of how to implement it and this is only a general example to understand the basic steps and concept). Please note that I've tried to simplify it, so do not try to compile it (this is not the full code - for a full Ethernet driver review, you can try start looking on this driver).
Now, let's focus on these basic steps:
Initialize the Rx Buffers
Rx Frame Reception
Prepare the Rx buffers for the next DMA transactions
Free the Rx Buffers
Initialize the Rx Buffers
static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
{
struct my_private *tp = (struct my_private *)dev->priv;
int ret = -ENOMEM;
int i;
tp->dma_buf_sz = BUF_SIZE_16KiB;
for (i = 0; i < DMA_RX_SIZE; i++) {
ret = init_rx_buffers(dev, i, flags);
if (ret)
goto err_init_rx_buffers;
}
return 0;
err_init_rx_buffers:
for (i = 0; i < DMA_RX_SIZE; i++) {
ret = free_rx_buffer(dev, i);
if (ret)
goto err_init_rx_buffers;
}
return ret;
}
static int init_rx_buffers(struct net_device *dev, int i, gfp_t flags)
{
struct my_private *tp = (struct my_private *)dev->priv;
struct sk_buff *skb = __netdev_alloc_skb_ip_align(dev, tp->dma_buf_sz, flags);
if (!skb) {
printk("Rx init fails; skb is NULL\n");
return -ENOMEM;
}
tp->rx_skbuff[i] = skb;
tp->rx_skbuff_dma[i] = dma_map_single(tp->device, skb->data,
tp->dma_buf_sz, DMA_FROM_DEVICE);
if (dma_mapping_error(tp->device, tp->rx_skbuff_dma[i])) {
printk("DMA mapping error\n");
dev_kfree_skb_any(skb);
return -EINVAL;
}
return 0;
}
Rx Frame Reception
/* should be called by the interrupt handler or NAPI poll method*/
static void receive_packets(struct net_device *dev)
{
struct my_private *tp = (struct my_private *)dev->priv;
unsigned int count = 0;
int rx_work_limit = tp->dirty_rx + RX_RING_SIZE - tp->cur_rx;
unsigned int next_entry = tp->cur_rx;
while (count < rx_work_limit) {
int entry = next_entry;
/* read the status of the incoming frame */
int status = get_rx_status(tp);
/* check if managed by the DMA otherwise go ahead */
if (unlikely(status & dma_own))
break;
count++;
tp->cur_rx = get_rx_entry(tp->cur_rx, DMA_RX_SIZE);
next_entry = tp->cur_rx;
/* If frame length is greater than skb buffer size
(preallocated during init) then the packet is ignored */
int frame_len = get_rx_frame_len(tp);
if (frame_len > tp->dma_buf_sz) {
printk("len %d larger than size (%d)\n", frame_len, tp->dma_buf_sz);
continue;
}
struct sk_buff *skb = tp->rx_skbuff[entry];
if (unlikely(!skb)) {
printk("Inconsistent Rx chain\n");
continue;
}
prefetch(skb->data - NET_IP_ALIGN);
tp->rx_skbuff[entry] = NULL;
skb_put(skb, frame_len);
dma_unmap_single(tp->device, tp->rx_skbuff_dma[entry],
tp->dma_buf_sz, DMA_FROM_DEVICE);
/* from this point it is safe to access the data of the rx skb.
the DMA transaction is already complete
and the rx buffer is unmapped from the DMA */
netif_receive_skb(skb);
}
rx_refill(dev);
return count;
}
Prepare the Rx buffers for the next DMA transactions
static inline void rx_refill(struct net_device *dev)
{
struct my_private *tp = (struct my_private *)dev->priv;
int dirty = get_num_of_rx_dirty(tp);
unsigned int entry = tp->dirty_rx;
while (dirty-- > 0) {
if (likely(!tp->rx_skbuff[entry])) {
struct sk_buff *skb;
skb = netdev_alloc_skb_ip_align(dev, tp->dma_buf_sz);
if (unlikely(!skb)) {
printk("fail to alloc skb entry %d\n", entry);
break;
}
rx_q->rx_skbuff[entry] = skb;
rx_q->rx_skbuff_dma[entry] = dma_map_single(tp->device, skb->data,
tp->dma_buf_sz, DMA_FROM_DEVICE);
if (dma_mapping_error(tp->device, tp->rx_skbuff_dma[entry])) {
printk("Rx DMA map failed\n");
dev_kfree_skb(skb);
break;
}
}
entry = get_rx_entry(entry, DMA_RX_SIZE);
}
tp->dirty_rx = entry;
}
Free the Rx Buffers
static void free_rx_buffer(struct net_device *dev, int i)
{
struct my_private *tp = (struct my_private *)dev->priv;
if (tp->rx_skbuff[i]) {
dma_unmap_single(tp->device, tp->rx_skbuff_dma[i],
tp->dma_buf_sz, DMA_FROM_DEVICE);
dev_kfree_skb_any(tp->rx_skbuff[i]);
}
tp->rx_skbuff[i] = NULL;
}