Thomas Gleixner | d2912cb | 2019-06-04 10:11:33 +0200 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 2 | /* |
| 3 | * Virtual DMA channel support for DMAengine |
| 4 | * |
| 5 | * Copyright (C) 2012 Russell King |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 6 | */ |
| 7 | #ifndef VIRT_DMA_H |
| 8 | #define VIRT_DMA_H |
| 9 | |
| 10 | #include <linux/dmaengine.h> |
| 11 | #include <linux/interrupt.h> |
| 12 | |
| 13 | #include "dmaengine.h" |
| 14 | |
| 15 | struct virt_dma_desc { |
| 16 | struct dma_async_tx_descriptor tx; |
Alexandru Ardelean | 09d5b70 | 2019-06-06 13:45:47 +0300 | [diff] [blame] | 17 | struct dmaengine_result tx_result; |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 18 | /* protected by vc.lock */ |
| 19 | struct list_head node; |
| 20 | }; |
| 21 | |
| 22 | struct virt_dma_chan { |
| 23 | struct dma_chan chan; |
| 24 | struct tasklet_struct task; |
| 25 | void (*desc_free)(struct virt_dma_desc *); |
| 26 | |
| 27 | spinlock_t lock; |
| 28 | |
| 29 | /* protected by vc.lock */ |
Robert Jarzmik | 13bb26a | 2015-10-13 21:54:28 +0200 | [diff] [blame] | 30 | struct list_head desc_allocated; |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 31 | struct list_head desc_submitted; |
| 32 | struct list_head desc_issued; |
| 33 | struct list_head desc_completed; |
Sascha Hauer | f8821011 | 2019-12-16 11:53:23 +0100 | [diff] [blame] | 34 | struct list_head desc_terminated; |
Russell King | 571fa74 | 2012-05-14 15:17:20 +0100 | [diff] [blame] | 35 | |
| 36 | struct virt_dma_desc *cyclic; |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 37 | }; |
| 38 | |
| 39 | static inline struct virt_dma_chan *to_virt_chan(struct dma_chan *chan) |
| 40 | { |
| 41 | return container_of(chan, struct virt_dma_chan, chan); |
| 42 | } |
| 43 | |
| 44 | void vchan_dma_desc_free_list(struct virt_dma_chan *vc, struct list_head *head); |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 45 | void vchan_init(struct virt_dma_chan *vc, struct dma_device *dmadev); |
Russell King | fe04587 | 2012-05-10 23:39:27 +0100 | [diff] [blame] | 46 | struct virt_dma_desc *vchan_find_desc(struct virt_dma_chan *, dma_cookie_t); |
Baoyou Xie | 02aa848 | 2016-09-24 12:37:05 +0800 | [diff] [blame] | 47 | extern dma_cookie_t vchan_tx_submit(struct dma_async_tx_descriptor *); |
| 48 | extern int vchan_tx_desc_free(struct dma_async_tx_descriptor *); |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 49 | |
| 50 | /** |
| 51 | * vchan_tx_prep - prepare a descriptor |
Lars-Peter Clausen | 28ca3e8 | 2015-10-20 13:14:45 +0200 | [diff] [blame] | 52 | * @vc: virtual channel allocating this descriptor |
| 53 | * @vd: virtual descriptor to prepare |
| 54 | * @tx_flags: flags argument passed in to prepare function |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 55 | */ |
| 56 | static inline struct dma_async_tx_descriptor *vchan_tx_prep(struct virt_dma_chan *vc, |
| 57 | struct virt_dma_desc *vd, unsigned long tx_flags) |
| 58 | { |
Robert Jarzmik | 13bb26a | 2015-10-13 21:54:28 +0200 | [diff] [blame] | 59 | unsigned long flags; |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 60 | |
| 61 | dma_async_tx_descriptor_init(&vd->tx, &vc->chan); |
| 62 | vd->tx.flags = tx_flags; |
| 63 | vd->tx.tx_submit = vchan_tx_submit; |
Robert Jarzmik | 13bb26a | 2015-10-13 21:54:28 +0200 | [diff] [blame] | 64 | vd->tx.desc_free = vchan_tx_desc_free; |
| 65 | |
Alexandru Ardelean | 09d5b70 | 2019-06-06 13:45:47 +0300 | [diff] [blame] | 66 | vd->tx_result.result = DMA_TRANS_NOERROR; |
| 67 | vd->tx_result.residue = 0; |
| 68 | |
Robert Jarzmik | 13bb26a | 2015-10-13 21:54:28 +0200 | [diff] [blame] | 69 | spin_lock_irqsave(&vc->lock, flags); |
| 70 | list_add_tail(&vd->node, &vc->desc_allocated); |
| 71 | spin_unlock_irqrestore(&vc->lock, flags); |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 72 | |
| 73 | return &vd->tx; |
| 74 | } |
| 75 | |
| 76 | /** |
| 77 | * vchan_issue_pending - move submitted descriptors to issued list |
Lars-Peter Clausen | 28ca3e8 | 2015-10-20 13:14:45 +0200 | [diff] [blame] | 78 | * @vc: virtual channel to update |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 79 | * |
| 80 | * vc.lock must be held by caller |
| 81 | */ |
| 82 | static inline bool vchan_issue_pending(struct virt_dma_chan *vc) |
| 83 | { |
| 84 | list_splice_tail_init(&vc->desc_submitted, &vc->desc_issued); |
| 85 | return !list_empty(&vc->desc_issued); |
| 86 | } |
| 87 | |
| 88 | /** |
| 89 | * vchan_cookie_complete - report completion of a descriptor |
Lars-Peter Clausen | 28ca3e8 | 2015-10-20 13:14:45 +0200 | [diff] [blame] | 90 | * @vd: virtual descriptor to update |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 91 | * |
| 92 | * vc.lock must be held by caller |
| 93 | */ |
| 94 | static inline void vchan_cookie_complete(struct virt_dma_desc *vd) |
| 95 | { |
| 96 | struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan); |
Jonas Jensen | af58652 | 2013-12-06 16:42:09 +0100 | [diff] [blame] | 97 | dma_cookie_t cookie; |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 98 | |
Jonas Jensen | af58652 | 2013-12-06 16:42:09 +0100 | [diff] [blame] | 99 | cookie = vd->tx.cookie; |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 100 | dma_cookie_complete(&vd->tx); |
| 101 | dev_vdbg(vc->chan.device->dev, "txd %p[%x]: marked complete\n", |
Jonas Jensen | af58652 | 2013-12-06 16:42:09 +0100 | [diff] [blame] | 102 | vd, cookie); |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 103 | list_add_tail(&vd->node, &vc->desc_completed); |
| 104 | |
| 105 | tasklet_schedule(&vc->task); |
| 106 | } |
| 107 | |
| 108 | /** |
Peter Ujfalusi | 6af149d | 2017-11-14 16:32:03 +0200 | [diff] [blame] | 109 | * vchan_vdesc_fini - Free or reuse a descriptor |
| 110 | * @vd: virtual descriptor to free/reuse |
| 111 | */ |
| 112 | static inline void vchan_vdesc_fini(struct virt_dma_desc *vd) |
| 113 | { |
| 114 | struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan); |
| 115 | |
Sascha Hauer | 9f91e6b | 2019-12-16 11:53:24 +0100 | [diff] [blame] | 116 | if (dmaengine_desc_test_reuse(&vd->tx)) { |
| 117 | unsigned long flags; |
| 118 | |
| 119 | spin_lock_irqsave(&vc->lock, flags); |
Peter Ujfalusi | 6af149d | 2017-11-14 16:32:03 +0200 | [diff] [blame] | 120 | list_add(&vd->node, &vc->desc_allocated); |
Sascha Hauer | 9f91e6b | 2019-12-16 11:53:24 +0100 | [diff] [blame] | 121 | spin_unlock_irqrestore(&vc->lock, flags); |
| 122 | } else { |
Peter Ujfalusi | 6af149d | 2017-11-14 16:32:03 +0200 | [diff] [blame] | 123 | vc->desc_free(vd); |
Sascha Hauer | 9f91e6b | 2019-12-16 11:53:24 +0100 | [diff] [blame] | 124 | } |
Peter Ujfalusi | 6af149d | 2017-11-14 16:32:03 +0200 | [diff] [blame] | 125 | } |
| 126 | |
| 127 | /** |
Russell King | 571fa74 | 2012-05-14 15:17:20 +0100 | [diff] [blame] | 128 | * vchan_cyclic_callback - report the completion of a period |
Lars-Peter Clausen | 28ca3e8 | 2015-10-20 13:14:45 +0200 | [diff] [blame] | 129 | * @vd: virtual descriptor |
Russell King | 571fa74 | 2012-05-14 15:17:20 +0100 | [diff] [blame] | 130 | */ |
| 131 | static inline void vchan_cyclic_callback(struct virt_dma_desc *vd) |
| 132 | { |
| 133 | struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan); |
| 134 | |
| 135 | vc->cyclic = vd; |
| 136 | tasklet_schedule(&vc->task); |
| 137 | } |
| 138 | |
| 139 | /** |
Peter Ujfalusi | 1c7f072 | 2017-11-14 16:32:04 +0200 | [diff] [blame] | 140 | * vchan_terminate_vdesc - Disable pending cyclic callback |
| 141 | * @vd: virtual descriptor to be terminated |
| 142 | * |
| 143 | * vc.lock must be held by caller |
| 144 | */ |
| 145 | static inline void vchan_terminate_vdesc(struct virt_dma_desc *vd) |
| 146 | { |
| 147 | struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan); |
| 148 | |
Sascha Hauer | f8821011 | 2019-12-16 11:53:23 +0100 | [diff] [blame] | 149 | list_add_tail(&vd->node, &vc->desc_terminated); |
Peter Ujfalusi | 1c7f072 | 2017-11-14 16:32:04 +0200 | [diff] [blame] | 150 | |
Peter Ujfalusi | 1c7f072 | 2017-11-14 16:32:04 +0200 | [diff] [blame] | 151 | if (vc->cyclic == vd) |
| 152 | vc->cyclic = NULL; |
| 153 | } |
| 154 | |
| 155 | /** |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 156 | * vchan_next_desc - peek at the next descriptor to be processed |
Lars-Peter Clausen | 28ca3e8 | 2015-10-20 13:14:45 +0200 | [diff] [blame] | 157 | * @vc: virtual channel to obtain descriptor from |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 158 | * |
| 159 | * vc.lock must be held by caller |
| 160 | */ |
| 161 | static inline struct virt_dma_desc *vchan_next_desc(struct virt_dma_chan *vc) |
| 162 | { |
Masahiro Yamada | 360af35 | 2016-09-13 03:08:17 +0900 | [diff] [blame] | 163 | return list_first_entry_or_null(&vc->desc_issued, |
| 164 | struct virt_dma_desc, node); |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 165 | } |
| 166 | |
| 167 | /** |
Jun Nie | 8c8fe97 | 2015-07-10 20:02:49 +0800 | [diff] [blame] | 168 | * vchan_get_all_descriptors - obtain all submitted and issued descriptors |
Lars-Peter Clausen | 28ca3e8 | 2015-10-20 13:14:45 +0200 | [diff] [blame] | 169 | * @vc: virtual channel to get descriptors from |
| 170 | * @head: list of descriptors found |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 171 | * |
| 172 | * vc.lock must be held by caller |
| 173 | * |
| 174 | * Removes all submitted and issued descriptors from internal lists, and |
| 175 | * provides a list of all descriptors found |
| 176 | */ |
| 177 | static inline void vchan_get_all_descriptors(struct virt_dma_chan *vc, |
| 178 | struct list_head *head) |
| 179 | { |
Robert Jarzmik | 13bb26a | 2015-10-13 21:54:28 +0200 | [diff] [blame] | 180 | list_splice_tail_init(&vc->desc_allocated, head); |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 181 | list_splice_tail_init(&vc->desc_submitted, head); |
| 182 | list_splice_tail_init(&vc->desc_issued, head); |
| 183 | list_splice_tail_init(&vc->desc_completed, head); |
Sascha Hauer | f8821011 | 2019-12-16 11:53:23 +0100 | [diff] [blame] | 184 | list_splice_tail_init(&vc->desc_terminated, head); |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 185 | } |
| 186 | |
| 187 | static inline void vchan_free_chan_resources(struct virt_dma_chan *vc) |
| 188 | { |
Robert Jarzmik | 13bb26a | 2015-10-13 21:54:28 +0200 | [diff] [blame] | 189 | struct virt_dma_desc *vd; |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 190 | unsigned long flags; |
| 191 | LIST_HEAD(head); |
| 192 | |
| 193 | spin_lock_irqsave(&vc->lock, flags); |
| 194 | vchan_get_all_descriptors(vc, &head); |
Robert Jarzmik | 13bb26a | 2015-10-13 21:54:28 +0200 | [diff] [blame] | 195 | list_for_each_entry(vd, &head, node) |
| 196 | dmaengine_desc_clear_reuse(&vd->tx); |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 197 | spin_unlock_irqrestore(&vc->lock, flags); |
| 198 | |
| 199 | vchan_dma_desc_free_list(vc, &head); |
| 200 | } |
| 201 | |
Lars-Peter Clausen | 2ed0862 | 2015-10-20 11:46:29 +0200 | [diff] [blame] | 202 | /** |
| 203 | * vchan_synchronize() - synchronize callback execution to the current context |
| 204 | * @vc: virtual channel to synchronize |
| 205 | * |
| 206 | * Makes sure that all scheduled or active callbacks have finished running. For |
| 207 | * proper operation the caller has to ensure that no new callbacks are scheduled |
| 208 | * after the invocation of this function started. |
Peter Ujfalusi | 1c7f072 | 2017-11-14 16:32:04 +0200 | [diff] [blame] | 209 | * Free up the terminated cyclic descriptor to prevent memory leakage. |
Lars-Peter Clausen | 2ed0862 | 2015-10-20 11:46:29 +0200 | [diff] [blame] | 210 | */ |
| 211 | static inline void vchan_synchronize(struct virt_dma_chan *vc) |
| 212 | { |
Sascha Hauer | f8821011 | 2019-12-16 11:53:23 +0100 | [diff] [blame] | 213 | LIST_HEAD(head); |
Peter Ujfalusi | 1c7f072 | 2017-11-14 16:32:04 +0200 | [diff] [blame] | 214 | unsigned long flags; |
| 215 | |
Lars-Peter Clausen | 2ed0862 | 2015-10-20 11:46:29 +0200 | [diff] [blame] | 216 | tasklet_kill(&vc->task); |
Peter Ujfalusi | 1c7f072 | 2017-11-14 16:32:04 +0200 | [diff] [blame] | 217 | |
| 218 | spin_lock_irqsave(&vc->lock, flags); |
Sascha Hauer | f8821011 | 2019-12-16 11:53:23 +0100 | [diff] [blame] | 219 | |
| 220 | list_splice_tail_init(&vc->desc_terminated, &head); |
| 221 | |
Peter Ujfalusi | 1c7f072 | 2017-11-14 16:32:04 +0200 | [diff] [blame] | 222 | spin_unlock_irqrestore(&vc->lock, flags); |
Sascha Hauer | f8821011 | 2019-12-16 11:53:23 +0100 | [diff] [blame] | 223 | |
| 224 | vchan_dma_desc_free_list(vc, &head); |
Lars-Peter Clausen | 2ed0862 | 2015-10-20 11:46:29 +0200 | [diff] [blame] | 225 | } |
| 226 | |
Russell King | 50437bf | 2012-04-13 12:07:23 +0100 | [diff] [blame] | 227 | #endif |