Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 2 | * Copyright (C) 2006 Benjamin Herrenschmidt, IBM Corporation |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3 | * |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 4 | * Provide default implementations of the DMA mapping callbacks for |
| 5 | * directly mapped busses and busses using the iommu infrastructure |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6 | */ |
| 7 | |
| 8 | #include <linux/device.h> |
| 9 | #include <linux/dma-mapping.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10 | #include <asm/bug.h> |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 11 | #include <asm/iommu.h> |
| 12 | #include <asm/abs_addr.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 13 | |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 14 | /* |
| 15 | * Generic iommu implementation |
| 16 | */ |
| 17 | |
| 18 | static inline unsigned long device_to_mask(struct device *dev) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 19 | { |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 20 | if (dev->dma_mask && *dev->dma_mask) |
| 21 | return *dev->dma_mask; |
| 22 | /* Assume devices without mask can take 32 bit addresses */ |
| 23 | return 0xfffffffful; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 24 | } |
| 25 | |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 26 | |
| 27 | /* Allocates a contiguous real buffer and creates mappings over it. |
| 28 | * Returns the virtual address of the buffer and sets dma_handle |
| 29 | * to the dma address (mapping) of the first page. |
| 30 | */ |
| 31 | static void *dma_iommu_alloc_coherent(struct device *dev, size_t size, |
| 32 | dma_addr_t *dma_handle, gfp_t flag) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 33 | { |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 34 | return iommu_alloc_coherent(dev->archdata.dma_data, size, dma_handle, |
| 35 | device_to_mask(dev), flag, |
| 36 | dev->archdata.numa_node); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 37 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 38 | |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 39 | static void dma_iommu_free_coherent(struct device *dev, size_t size, |
| 40 | void *vaddr, dma_addr_t dma_handle) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 41 | { |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 42 | iommu_free_coherent(dev->archdata.dma_data, size, vaddr, dma_handle); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 43 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 44 | |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 45 | /* Creates TCEs for a user provided buffer. The user buffer must be |
| 46 | * contiguous real kernel storage (not vmalloc). The address of the buffer |
| 47 | * passed here is the kernel (virtual) address of the buffer. The buffer |
| 48 | * need not be page aligned, the dma_addr_t returned will point to the same |
| 49 | * byte within the page as vaddr. |
| 50 | */ |
| 51 | static dma_addr_t dma_iommu_map_single(struct device *dev, void *vaddr, |
| 52 | size_t size, |
| 53 | enum dma_data_direction direction) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 54 | { |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 55 | return iommu_map_single(dev->archdata.dma_data, vaddr, size, |
| 56 | device_to_mask(dev), direction); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 57 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 58 | |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 59 | |
| 60 | static void dma_iommu_unmap_single(struct device *dev, dma_addr_t dma_handle, |
| 61 | size_t size, |
| 62 | enum dma_data_direction direction) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 63 | { |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 64 | iommu_unmap_single(dev->archdata.dma_data, dma_handle, size, direction); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 65 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 66 | |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 67 | |
| 68 | static int dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist, |
| 69 | int nelems, enum dma_data_direction direction) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 70 | { |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 71 | return iommu_map_sg(dev->archdata.dma_data, sglist, nelems, |
| 72 | device_to_mask(dev), direction); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 73 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 74 | |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 75 | static void dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist, |
| 76 | int nelems, enum dma_data_direction direction) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 77 | { |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 78 | iommu_unmap_sg(dev->archdata.dma_data, sglist, nelems, direction); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 79 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 80 | |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 81 | /* We support DMA to/from any memory page via the iommu */ |
| 82 | static int dma_iommu_dma_supported(struct device *dev, u64 mask) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 83 | { |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 84 | struct iommu_table *tbl = dev->archdata.dma_data; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 85 | |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 86 | if (!tbl || tbl->it_offset > mask) { |
| 87 | printk(KERN_INFO |
| 88 | "Warning: IOMMU offset too big for device mask\n"); |
| 89 | if (tbl) |
| 90 | printk(KERN_INFO |
| 91 | "mask: 0x%08lx, table offset: 0x%08lx\n", |
| 92 | mask, tbl->it_offset); |
| 93 | else |
| 94 | printk(KERN_INFO "mask: 0x%08lx, table unavailable\n", |
| 95 | mask); |
| 96 | return 0; |
| 97 | } else |
| 98 | return 1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 99 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 100 | |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 101 | struct dma_mapping_ops dma_iommu_ops = { |
| 102 | .alloc_coherent = dma_iommu_alloc_coherent, |
| 103 | .free_coherent = dma_iommu_free_coherent, |
| 104 | .map_single = dma_iommu_map_single, |
| 105 | .unmap_single = dma_iommu_unmap_single, |
| 106 | .map_sg = dma_iommu_map_sg, |
| 107 | .unmap_sg = dma_iommu_unmap_sg, |
| 108 | .dma_supported = dma_iommu_dma_supported, |
| 109 | }; |
| 110 | EXPORT_SYMBOL(dma_iommu_ops); |
| 111 | |
| 112 | /* |
| 113 | * Generic direct DMA implementation |
Benjamin Herrenschmidt | 92b20c4 | 2006-11-11 17:25:14 +1100 | [diff] [blame] | 114 | * |
| 115 | * This implementation supports a global offset that can be applied if |
| 116 | * the address at which memory is visible to devices is not 0. |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 117 | */ |
Benjamin Herrenschmidt | 92b20c4 | 2006-11-11 17:25:14 +1100 | [diff] [blame] | 118 | unsigned long dma_direct_offset; |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 119 | |
| 120 | static void *dma_direct_alloc_coherent(struct device *dev, size_t size, |
| 121 | dma_addr_t *dma_handle, gfp_t flag) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 122 | { |
Benjamin Herrenschmidt | c80d913 | 2006-11-11 17:25:16 +1100 | [diff] [blame] | 123 | struct page *page; |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 124 | void *ret; |
Benjamin Herrenschmidt | c80d913 | 2006-11-11 17:25:16 +1100 | [diff] [blame] | 125 | int node = dev->archdata.numa_node; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 126 | |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 127 | /* TODO: Maybe use the numa node here too ? */ |
Benjamin Herrenschmidt | c80d913 | 2006-11-11 17:25:16 +1100 | [diff] [blame] | 128 | page = alloc_pages_node(node, flag, get_order(size)); |
| 129 | if (page == NULL) |
| 130 | return NULL; |
| 131 | ret = page_address(page); |
| 132 | memset(ret, 0, size); |
| 133 | *dma_handle = virt_to_abs(ret) | dma_direct_offset; |
| 134 | |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 135 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 136 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 137 | |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 138 | static void dma_direct_free_coherent(struct device *dev, size_t size, |
| 139 | void *vaddr, dma_addr_t dma_handle) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 140 | { |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 141 | free_pages((unsigned long)vaddr, get_order(size)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 142 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 143 | |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 144 | static dma_addr_t dma_direct_map_single(struct device *dev, void *ptr, |
| 145 | size_t size, |
| 146 | enum dma_data_direction direction) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 147 | { |
Benjamin Herrenschmidt | 92b20c4 | 2006-11-11 17:25:14 +1100 | [diff] [blame] | 148 | return virt_to_abs(ptr) | dma_direct_offset; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 149 | } |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 150 | |
| 151 | static void dma_direct_unmap_single(struct device *dev, dma_addr_t dma_addr, |
| 152 | size_t size, |
| 153 | enum dma_data_direction direction) |
| 154 | { |
| 155 | } |
| 156 | |
Jens Axboe | 78bdc31 | 2007-10-12 13:44:12 +0200 | [diff] [blame] | 157 | static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 158 | int nents, enum dma_data_direction direction) |
| 159 | { |
Jens Axboe | 78bdc31 | 2007-10-12 13:44:12 +0200 | [diff] [blame] | 160 | struct scatterlist *sg; |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 161 | int i; |
| 162 | |
Jens Axboe | 78bdc31 | 2007-10-12 13:44:12 +0200 | [diff] [blame] | 163 | for_each_sg(sgl, sg, nents, i) { |
Jens Axboe | 58b053e | 2007-10-22 20:02:46 +0200 | [diff] [blame] | 164 | sg->dma_address = sg_phys(sg) | dma_direct_offset; |
Benjamin Herrenschmidt | 12d04ee | 2006-11-11 17:25:02 +1100 | [diff] [blame] | 165 | sg->dma_length = sg->length; |
| 166 | } |
| 167 | |
| 168 | return nents; |
| 169 | } |
| 170 | |
| 171 | static void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sg, |
| 172 | int nents, enum dma_data_direction direction) |
| 173 | { |
| 174 | } |
| 175 | |
| 176 | static int dma_direct_dma_supported(struct device *dev, u64 mask) |
| 177 | { |
| 178 | /* Could be improved to check for memory though it better be |
| 179 | * done via some global so platforms can set the limit in case |
| 180 | * they have limited DMA windows |
| 181 | */ |
| 182 | return mask >= DMA_32BIT_MASK; |
| 183 | } |
| 184 | |
| 185 | struct dma_mapping_ops dma_direct_ops = { |
| 186 | .alloc_coherent = dma_direct_alloc_coherent, |
| 187 | .free_coherent = dma_direct_free_coherent, |
| 188 | .map_single = dma_direct_map_single, |
| 189 | .unmap_single = dma_direct_unmap_single, |
| 190 | .map_sg = dma_direct_map_sg, |
| 191 | .unmap_sg = dma_direct_unmap_sg, |
| 192 | .dma_supported = dma_direct_dma_supported, |
| 193 | }; |
| 194 | EXPORT_SYMBOL(dma_direct_ops); |