Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Dynamic DMA mapping support for AMD Hammer. |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 3 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4 | * Use the integrated AGP GART in the Hammer northbridge as an IOMMU for PCI. |
| 5 | * This allows to use PCI devices that only support 32bit addresses on systems |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 6 | * with more than 4GB. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7 | * |
Paul Bolle | 395cf96 | 2011-08-15 02:02:26 +0200 | [diff] [blame] | 8 | * See Documentation/DMA-API-HOWTO.txt for the interface specification. |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 9 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10 | * Copyright 2002 Andi Kleen, SuSE Labs. |
Andi Kleen | ff7f364 | 2007-10-17 18:04:37 +0200 | [diff] [blame] | 11 | * Subject to the GNU General Public License v2 only. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 12 | */ |
| 13 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 14 | #include <linux/types.h> |
| 15 | #include <linux/ctype.h> |
| 16 | #include <linux/agp_backend.h> |
| 17 | #include <linux/init.h> |
| 18 | #include <linux/mm.h> |
Alexey Dobriyan | d43c36d | 2009-10-07 17:09:06 +0400 | [diff] [blame] | 19 | #include <linux/sched.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | #include <linux/string.h> |
| 21 | #include <linux/spinlock.h> |
| 22 | #include <linux/pci.h> |
| 23 | #include <linux/module.h> |
| 24 | #include <linux/topology.h> |
| 25 | #include <linux/interrupt.h> |
Akinobu Mita | a66022c | 2009-12-15 16:48:28 -0800 | [diff] [blame] | 26 | #include <linux/bitmap.h> |
Christoph Hellwig | 1eeb66a | 2007-05-08 00:27:03 -0700 | [diff] [blame] | 27 | #include <linux/kdebug.h> |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 28 | #include <linux/scatterlist.h> |
FUJITA Tomonori | fde9a10 | 2008-02-04 22:28:11 -0800 | [diff] [blame] | 29 | #include <linux/iommu-helper.h> |
Rafael J. Wysocki | f3c6ea1 | 2011-03-23 22:15:54 +0100 | [diff] [blame] | 30 | #include <linux/syscore_ops.h> |
Joerg Roedel | 237a622 | 2008-09-25 12:13:53 +0200 | [diff] [blame] | 31 | #include <linux/io.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 32 | #include <linux/gfp.h> |
Arun Sharma | 60063497 | 2011-07-26 16:09:06 -0700 | [diff] [blame] | 33 | #include <linux/atomic.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 34 | #include <asm/mtrr.h> |
| 35 | #include <asm/pgtable.h> |
| 36 | #include <asm/proto.h> |
FUJITA Tomonori | 46a7fa2 | 2008-07-11 10:23:42 +0900 | [diff] [blame] | 37 | #include <asm/iommu.h> |
Joerg Roedel | 395624f | 2007-10-24 12:49:47 +0200 | [diff] [blame] | 38 | #include <asm/gart.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 39 | #include <asm/cacheflush.h> |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 40 | #include <asm/swiotlb.h> |
| 41 | #include <asm/dma.h> |
Andreas Herrmann | 23ac4ae | 2010-09-17 18:03:43 +0200 | [diff] [blame] | 42 | #include <asm/amd_nb.h> |
FUJITA Tomonori | 338bac5 | 2009-10-27 16:34:44 +0900 | [diff] [blame] | 43 | #include <asm/x86_init.h> |
Konrad Rzeszutek Wilk | 22e6daf | 2010-08-26 13:58:03 -0400 | [diff] [blame] | 44 | #include <asm/iommu_table.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 45 | |
Joerg Roedel | 79da087 | 2007-10-24 12:49:49 +0200 | [diff] [blame] | 46 | static unsigned long iommu_bus_base; /* GART remapping area (physical) */ |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 47 | static unsigned long iommu_size; /* size of remapping area bytes */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 48 | static unsigned long iommu_pages; /* .. and in pages */ |
| 49 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 50 | static u32 *iommu_gatt_base; /* Remapping table */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 51 | |
FUJITA Tomonori | 4210919 | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 52 | static dma_addr_t bad_dma_addr; |
| 53 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 54 | /* |
| 55 | * If this is disabled the IOMMU will use an optimized flushing strategy |
| 56 | * of only flushing when an mapping is reused. With it true the GART is |
| 57 | * flushed for every mapping. Problem is that doing the lazy flush seems |
| 58 | * to trigger bugs with some popular PCI cards, in particular 3ware (but |
| 59 | * has been also also seen with Qlogic at least). |
| 60 | */ |
Jaswinder Singh Rajput | c854c91 | 2008-12-29 20:38:09 +0530 | [diff] [blame] | 61 | static int iommu_fullflush = 1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 62 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 63 | /* Allocation bitmap for the remapping area: */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 64 | static DEFINE_SPINLOCK(iommu_bitmap_lock); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 65 | /* Guarded by iommu_bitmap_lock: */ |
| 66 | static unsigned long *iommu_gart_bitmap; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 67 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 68 | static u32 gart_unmapped_entry; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 69 | |
| 70 | #define GPTE_VALID 1 |
| 71 | #define GPTE_COHERENT 2 |
| 72 | #define GPTE_ENCODE(x) \ |
| 73 | (((x) & 0xfffff000) | (((x) >> 32) << 4) | GPTE_VALID | GPTE_COHERENT) |
| 74 | #define GPTE_DECODE(x) (((x) & 0xfffff000) | (((u64)(x) & 0xff0) << 28)) |
| 75 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 76 | #define EMERGENCY_PAGES 32 /* = 128KB */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 77 | |
| 78 | #ifdef CONFIG_AGP |
| 79 | #define AGPEXTERN extern |
| 80 | #else |
| 81 | #define AGPEXTERN |
| 82 | #endif |
| 83 | |
Joerg Roedel | 665d3e2 | 2011-04-18 15:45:46 +0200 | [diff] [blame] | 84 | /* GART can only remap to physical addresses < 1TB */ |
| 85 | #define GART_MAX_PHYS_ADDR (1ULL << 40) |
| 86 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 87 | /* backdoor interface to AGP driver */ |
| 88 | AGPEXTERN int agp_memory_reserved; |
| 89 | AGPEXTERN __u32 *agp_gatt_table; |
| 90 | |
| 91 | static unsigned long next_bit; /* protected by iommu_bitmap_lock */ |
Joerg Roedel | 3610f21 | 2008-09-25 12:13:54 +0200 | [diff] [blame] | 92 | static bool need_flush; /* global flush state. set for each gart wrap */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 93 | |
FUJITA Tomonori | 7b22ff5 | 2008-08-18 00:36:18 +0900 | [diff] [blame] | 94 | static unsigned long alloc_iommu(struct device *dev, int size, |
| 95 | unsigned long align_mask) |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 96 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 97 | unsigned long offset, flags; |
FUJITA Tomonori | fde9a10 | 2008-02-04 22:28:11 -0800 | [diff] [blame] | 98 | unsigned long boundary_size; |
| 99 | unsigned long base_index; |
| 100 | |
| 101 | base_index = ALIGN(iommu_bus_base & dma_get_seg_boundary(dev), |
| 102 | PAGE_SIZE) >> PAGE_SHIFT; |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 103 | boundary_size = ALIGN((u64)dma_get_seg_boundary(dev) + 1, |
FUJITA Tomonori | fde9a10 | 2008-02-04 22:28:11 -0800 | [diff] [blame] | 104 | PAGE_SIZE) >> PAGE_SHIFT; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 105 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 106 | spin_lock_irqsave(&iommu_bitmap_lock, flags); |
FUJITA Tomonori | fde9a10 | 2008-02-04 22:28:11 -0800 | [diff] [blame] | 107 | offset = iommu_area_alloc(iommu_gart_bitmap, iommu_pages, next_bit, |
FUJITA Tomonori | 7b22ff5 | 2008-08-18 00:36:18 +0900 | [diff] [blame] | 108 | size, base_index, boundary_size, align_mask); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 109 | if (offset == -1) { |
Joerg Roedel | 3610f21 | 2008-09-25 12:13:54 +0200 | [diff] [blame] | 110 | need_flush = true; |
FUJITA Tomonori | fde9a10 | 2008-02-04 22:28:11 -0800 | [diff] [blame] | 111 | offset = iommu_area_alloc(iommu_gart_bitmap, iommu_pages, 0, |
FUJITA Tomonori | 7b22ff5 | 2008-08-18 00:36:18 +0900 | [diff] [blame] | 112 | size, base_index, boundary_size, |
| 113 | align_mask); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 114 | } |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 115 | if (offset != -1) { |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 116 | next_bit = offset+size; |
| 117 | if (next_bit >= iommu_pages) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 118 | next_bit = 0; |
Joerg Roedel | 3610f21 | 2008-09-25 12:13:54 +0200 | [diff] [blame] | 119 | need_flush = true; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 120 | } |
| 121 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 122 | if (iommu_fullflush) |
Joerg Roedel | 3610f21 | 2008-09-25 12:13:54 +0200 | [diff] [blame] | 123 | need_flush = true; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 124 | spin_unlock_irqrestore(&iommu_bitmap_lock, flags); |
| 125 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 126 | return offset; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 127 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 128 | |
| 129 | static void free_iommu(unsigned long offset, int size) |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 130 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 131 | unsigned long flags; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 132 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 133 | spin_lock_irqsave(&iommu_bitmap_lock, flags); |
Akinobu Mita | a66022c | 2009-12-15 16:48:28 -0800 | [diff] [blame] | 134 | bitmap_clear(iommu_gart_bitmap, offset, size); |
Joerg Roedel | 70d7d35 | 2008-12-02 20:16:03 +0100 | [diff] [blame] | 135 | if (offset >= next_bit) |
| 136 | next_bit = offset + size; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 137 | spin_unlock_irqrestore(&iommu_bitmap_lock, flags); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 138 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 139 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 140 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 141 | * Use global flush state to avoid races with multiple flushers. |
| 142 | */ |
Andi Kleen | a32073b | 2006-06-26 13:56:40 +0200 | [diff] [blame] | 143 | static void flush_gart(void) |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 144 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 145 | unsigned long flags; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 146 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 147 | spin_lock_irqsave(&iommu_bitmap_lock, flags); |
Andi Kleen | a32073b | 2006-06-26 13:56:40 +0200 | [diff] [blame] | 148 | if (need_flush) { |
Hans Rosenfeld | eec1d4f | 2010-10-29 17:14:30 +0200 | [diff] [blame] | 149 | amd_flush_garts(); |
Joerg Roedel | 3610f21 | 2008-09-25 12:13:54 +0200 | [diff] [blame] | 150 | need_flush = false; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 151 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 152 | spin_unlock_irqrestore(&iommu_bitmap_lock, flags); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 153 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 154 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 155 | #ifdef CONFIG_IOMMU_LEAK |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 156 | /* Debugging aid for drivers that don't free their IOMMU tables */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 157 | static int leak_trace; |
Joerg Roedel | 79da087 | 2007-10-24 12:49:49 +0200 | [diff] [blame] | 158 | static int iommu_leak_pages = 20; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 159 | |
Joerg Roedel | 79da087 | 2007-10-24 12:49:49 +0200 | [diff] [blame] | 160 | static void dump_leak(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 161 | { |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 162 | static int dump; |
| 163 | |
FUJITA Tomonori | 19c1a6f | 2009-04-14 09:43:19 +0900 | [diff] [blame] | 164 | if (dump) |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 165 | return; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 166 | dump = 1; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 167 | |
FUJITA Tomonori | 19c1a6f | 2009-04-14 09:43:19 +0900 | [diff] [blame] | 168 | show_stack(NULL, NULL); |
| 169 | debug_dma_dump_mappings(NULL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 170 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 171 | #endif |
| 172 | |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 173 | static void iommu_full(struct device *dev, size_t size, int dir) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 174 | { |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 175 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 176 | * Ran out of IOMMU space for this operation. This is very bad. |
| 177 | * Unfortunately the drivers cannot handle this operation properly. |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 178 | * Return some non mapped prereserved space in the aperture and |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 179 | * let the Northbridge deal with it. This will result in garbage |
| 180 | * in the IO operation. When the size exceeds the prereserved space |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 181 | * memory corruption will occur or random memory will be DMAed |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 182 | * out. Hopefully no network devices use single mappings that big. |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 183 | */ |
| 184 | |
Greg Kroah-Hartman | fc3a882 | 2008-05-02 06:02:41 +0200 | [diff] [blame] | 185 | dev_err(dev, "PCI-DMA: Out of IOMMU space for %lu bytes\n", size); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 186 | |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 187 | if (size > PAGE_SIZE*EMERGENCY_PAGES) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 188 | if (dir == PCI_DMA_FROMDEVICE || dir == PCI_DMA_BIDIRECTIONAL) |
| 189 | panic("PCI-DMA: Memory would be corrupted\n"); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 190 | if (dir == PCI_DMA_TODEVICE || dir == PCI_DMA_BIDIRECTIONAL) |
| 191 | panic(KERN_ERR |
| 192 | "PCI-DMA: Random memory would be DMAed\n"); |
| 193 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 194 | #ifdef CONFIG_IOMMU_LEAK |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 195 | dump_leak(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 196 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 197 | } |
| 198 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 199 | static inline int |
| 200 | need_iommu(struct device *dev, unsigned long addr, size_t size) |
| 201 | { |
FUJITA Tomonori | a4c2baa | 2009-07-10 10:04:55 +0900 | [diff] [blame] | 202 | return force_iommu || !dma_capable(dev, addr, size); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 203 | } |
| 204 | |
| 205 | static inline int |
| 206 | nonforced_iommu(struct device *dev, unsigned long addr, size_t size) |
| 207 | { |
FUJITA Tomonori | a4c2baa | 2009-07-10 10:04:55 +0900 | [diff] [blame] | 208 | return !dma_capable(dev, addr, size); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 209 | } |
| 210 | |
| 211 | /* Map a single continuous physical area into the IOMMU. |
| 212 | * Caller needs to check if the iommu is needed and flush. |
| 213 | */ |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 214 | static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem, |
FUJITA Tomonori | 7b22ff5 | 2008-08-18 00:36:18 +0900 | [diff] [blame] | 215 | size_t size, int dir, unsigned long align_mask) |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 216 | { |
Joerg Roedel | 1477b8e | 2008-10-15 22:02:11 -0700 | [diff] [blame] | 217 | unsigned long npages = iommu_num_pages(phys_mem, size, PAGE_SIZE); |
Joerg Roedel | 665d3e2 | 2011-04-18 15:45:46 +0200 | [diff] [blame] | 218 | unsigned long iommu_page; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 219 | int i; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 220 | |
Joerg Roedel | 665d3e2 | 2011-04-18 15:45:46 +0200 | [diff] [blame] | 221 | if (unlikely(phys_mem + size > GART_MAX_PHYS_ADDR)) |
| 222 | return bad_dma_addr; |
| 223 | |
| 224 | iommu_page = alloc_iommu(dev, npages, align_mask); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 225 | if (iommu_page == -1) { |
| 226 | if (!nonforced_iommu(dev, phys_mem, size)) |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 227 | return phys_mem; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 228 | if (panic_on_overflow) |
| 229 | panic("dma_map_area overflow %lu bytes\n", size); |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 230 | iommu_full(dev, size, dir); |
FUJITA Tomonori | 4210919 | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 231 | return bad_dma_addr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 232 | } |
| 233 | |
| 234 | for (i = 0; i < npages; i++) { |
| 235 | iommu_gatt_base[iommu_page + i] = GPTE_ENCODE(phys_mem); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 236 | phys_mem += PAGE_SIZE; |
| 237 | } |
| 238 | return iommu_bus_base + iommu_page*PAGE_SIZE + (phys_mem & ~PAGE_MASK); |
| 239 | } |
| 240 | |
| 241 | /* Map a single area into the IOMMU */ |
FUJITA Tomonori | 052aedb | 2009-01-05 23:47:23 +0900 | [diff] [blame] | 242 | static dma_addr_t gart_map_page(struct device *dev, struct page *page, |
| 243 | unsigned long offset, size_t size, |
| 244 | enum dma_data_direction dir, |
| 245 | struct dma_attrs *attrs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 246 | { |
Ingo Molnar | 2be6214 | 2008-04-19 19:19:56 +0200 | [diff] [blame] | 247 | unsigned long bus; |
FUJITA Tomonori | 052aedb | 2009-01-05 23:47:23 +0900 | [diff] [blame] | 248 | phys_addr_t paddr = page_to_phys(page) + offset; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 249 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 250 | if (!dev) |
Joerg Roedel | 6c505ce | 2008-08-19 16:32:45 +0200 | [diff] [blame] | 251 | dev = &x86_dma_fallback_dev; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 252 | |
Ingo Molnar | 2be6214 | 2008-04-19 19:19:56 +0200 | [diff] [blame] | 253 | if (!need_iommu(dev, paddr, size)) |
| 254 | return paddr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 255 | |
FUJITA Tomonori | 7b22ff5 | 2008-08-18 00:36:18 +0900 | [diff] [blame] | 256 | bus = dma_map_area(dev, paddr, size, dir, 0); |
| 257 | flush_gart(); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 258 | |
| 259 | return bus; |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 260 | } |
| 261 | |
| 262 | /* |
Jon Mason | 7c2d9cd | 2006-06-26 13:56:37 +0200 | [diff] [blame] | 263 | * Free a DMA mapping. |
| 264 | */ |
FUJITA Tomonori | 052aedb | 2009-01-05 23:47:23 +0900 | [diff] [blame] | 265 | static void gart_unmap_page(struct device *dev, dma_addr_t dma_addr, |
| 266 | size_t size, enum dma_data_direction dir, |
| 267 | struct dma_attrs *attrs) |
Jon Mason | 7c2d9cd | 2006-06-26 13:56:37 +0200 | [diff] [blame] | 268 | { |
| 269 | unsigned long iommu_page; |
| 270 | int npages; |
| 271 | int i; |
| 272 | |
| 273 | if (dma_addr < iommu_bus_base + EMERGENCY_PAGES*PAGE_SIZE || |
| 274 | dma_addr >= iommu_bus_base + iommu_size) |
| 275 | return; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 276 | |
Jon Mason | 7c2d9cd | 2006-06-26 13:56:37 +0200 | [diff] [blame] | 277 | iommu_page = (dma_addr - iommu_bus_base)>>PAGE_SHIFT; |
Joerg Roedel | 1477b8e | 2008-10-15 22:02:11 -0700 | [diff] [blame] | 278 | npages = iommu_num_pages(dma_addr, size, PAGE_SIZE); |
Jon Mason | 7c2d9cd | 2006-06-26 13:56:37 +0200 | [diff] [blame] | 279 | for (i = 0; i < npages; i++) { |
| 280 | iommu_gatt_base[iommu_page + i] = gart_unmapped_entry; |
Jon Mason | 7c2d9cd | 2006-06-26 13:56:37 +0200 | [diff] [blame] | 281 | } |
| 282 | free_iommu(iommu_page, npages); |
| 283 | } |
| 284 | |
| 285 | /* |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 286 | * Wrapper for pci_unmap_single working with scatterlists. |
| 287 | */ |
FUJITA Tomonori | 160c1d8 | 2009-01-05 23:59:02 +0900 | [diff] [blame] | 288 | static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, |
| 289 | enum dma_data_direction dir, struct dma_attrs *attrs) |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 290 | { |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 291 | struct scatterlist *s; |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 292 | int i; |
| 293 | |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 294 | for_each_sg(sg, s, nents, i) { |
Jon Mason | 60b08c6 | 2006-02-26 04:18:22 +0100 | [diff] [blame] | 295 | if (!s->dma_length || !s->length) |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 296 | break; |
FUJITA Tomonori | d7dff84 | 2009-01-05 23:47:28 +0900 | [diff] [blame] | 297 | gart_unmap_page(dev, s->dma_address, s->dma_length, dir, NULL); |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 298 | } |
| 299 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 300 | |
| 301 | /* Fallback for dma_map_sg in case of overflow */ |
| 302 | static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg, |
| 303 | int nents, int dir) |
| 304 | { |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 305 | struct scatterlist *s; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 306 | int i; |
| 307 | |
| 308 | #ifdef CONFIG_IOMMU_DEBUG |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 309 | pr_debug("dma_map_sg overflow\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 310 | #endif |
| 311 | |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 312 | for_each_sg(sg, s, nents, i) { |
Jens Axboe | 58b053e | 2007-10-22 20:02:46 +0200 | [diff] [blame] | 313 | unsigned long addr = sg_phys(s); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 314 | |
| 315 | if (nonforced_iommu(dev, addr, s->length)) { |
FUJITA Tomonori | 7b22ff5 | 2008-08-18 00:36:18 +0900 | [diff] [blame] | 316 | addr = dma_map_area(dev, addr, s->length, dir, 0); |
FUJITA Tomonori | 4210919 | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 317 | if (addr == bad_dma_addr) { |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 318 | if (i > 0) |
FUJITA Tomonori | 160c1d8 | 2009-01-05 23:59:02 +0900 | [diff] [blame] | 319 | gart_unmap_sg(dev, sg, i, dir, NULL); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 320 | nents = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 321 | sg[0].dma_length = 0; |
| 322 | break; |
| 323 | } |
| 324 | } |
| 325 | s->dma_address = addr; |
| 326 | s->dma_length = s->length; |
| 327 | } |
Andi Kleen | a32073b | 2006-06-26 13:56:40 +0200 | [diff] [blame] | 328 | flush_gart(); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 329 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 330 | return nents; |
| 331 | } |
| 332 | |
| 333 | /* Map multiple scatterlist entries continuous into the first. */ |
FUJITA Tomonori | fde9a10 | 2008-02-04 22:28:11 -0800 | [diff] [blame] | 334 | static int __dma_map_cont(struct device *dev, struct scatterlist *start, |
| 335 | int nelems, struct scatterlist *sout, |
| 336 | unsigned long pages) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 337 | { |
FUJITA Tomonori | 7b22ff5 | 2008-08-18 00:36:18 +0900 | [diff] [blame] | 338 | unsigned long iommu_start = alloc_iommu(dev, pages, 0); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 339 | unsigned long iommu_page = iommu_start; |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 340 | struct scatterlist *s; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 341 | int i; |
| 342 | |
| 343 | if (iommu_start == -1) |
| 344 | return -1; |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 345 | |
| 346 | for_each_sg(start, s, nelems, i) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 347 | unsigned long pages, addr; |
| 348 | unsigned long phys_addr = s->dma_address; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 349 | |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 350 | BUG_ON(s != start && s->offset); |
| 351 | if (s == start) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 352 | sout->dma_address = iommu_bus_base; |
| 353 | sout->dma_address += iommu_page*PAGE_SIZE + s->offset; |
| 354 | sout->dma_length = s->length; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 355 | } else { |
| 356 | sout->dma_length += s->length; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 357 | } |
| 358 | |
| 359 | addr = phys_addr; |
Joerg Roedel | 1477b8e | 2008-10-15 22:02:11 -0700 | [diff] [blame] | 360 | pages = iommu_num_pages(s->offset, s->length, PAGE_SIZE); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 361 | while (pages--) { |
| 362 | iommu_gatt_base[iommu_page] = GPTE_ENCODE(addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 363 | addr += PAGE_SIZE; |
| 364 | iommu_page++; |
Andi Kleen | 0d541064 | 2006-02-12 14:34:59 -0800 | [diff] [blame] | 365 | } |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 366 | } |
| 367 | BUG_ON(iommu_page - iommu_start != pages); |
| 368 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 369 | return 0; |
| 370 | } |
| 371 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 372 | static inline int |
FUJITA Tomonori | fde9a10 | 2008-02-04 22:28:11 -0800 | [diff] [blame] | 373 | dma_map_cont(struct device *dev, struct scatterlist *start, int nelems, |
| 374 | struct scatterlist *sout, unsigned long pages, int need) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 375 | { |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 376 | if (!need) { |
| 377 | BUG_ON(nelems != 1); |
FUJITA Tomonori | e88a39d | 2007-10-25 09:13:32 +0200 | [diff] [blame] | 378 | sout->dma_address = start->dma_address; |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 379 | sout->dma_length = start->length; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 380 | return 0; |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 381 | } |
FUJITA Tomonori | fde9a10 | 2008-02-04 22:28:11 -0800 | [diff] [blame] | 382 | return __dma_map_cont(dev, start, nelems, sout, pages); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 383 | } |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 384 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 385 | /* |
| 386 | * DMA map all entries in a scatterlist. |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 387 | * Merge chunks that have page aligned sizes into a continuous mapping. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 388 | */ |
FUJITA Tomonori | 160c1d8 | 2009-01-05 23:59:02 +0900 | [diff] [blame] | 389 | static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, |
| 390 | enum dma_data_direction dir, struct dma_attrs *attrs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 391 | { |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 392 | struct scatterlist *s, *ps, *start_sg, *sgmap; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 393 | int need = 0, nextneed, i, out, start; |
| 394 | unsigned long pages = 0; |
FUJITA Tomonori | 42d0028 | 2008-02-04 22:27:56 -0800 | [diff] [blame] | 395 | unsigned int seg_size; |
| 396 | unsigned int max_seg_size; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 397 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 398 | if (nents == 0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 399 | return 0; |
| 400 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 401 | if (!dev) |
Joerg Roedel | 6c505ce | 2008-08-19 16:32:45 +0200 | [diff] [blame] | 402 | dev = &x86_dma_fallback_dev; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 403 | |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 404 | out = 0; |
| 405 | start = 0; |
| 406 | start_sg = sg; |
| 407 | sgmap = sg; |
| 408 | seg_size = 0; |
| 409 | max_seg_size = dma_get_max_seg_size(dev); |
| 410 | ps = NULL; /* shut up gcc */ |
| 411 | |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 412 | for_each_sg(sg, s, nents, i) { |
Jens Axboe | 58b053e | 2007-10-22 20:02:46 +0200 | [diff] [blame] | 413 | dma_addr_t addr = sg_phys(s); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 414 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 415 | s->dma_address = addr; |
| 416 | BUG_ON(s->length == 0); |
| 417 | |
| 418 | nextneed = need_iommu(dev, addr, s->length); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 419 | |
| 420 | /* Handle the previous not yet processed entries */ |
| 421 | if (i > start) { |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 422 | /* |
| 423 | * Can only merge when the last chunk ends on a |
| 424 | * page boundary and the new one doesn't have an |
| 425 | * offset. |
| 426 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 427 | if (!iommu_merge || !nextneed || !need || s->offset || |
FUJITA Tomonori | 42d0028 | 2008-02-04 22:27:56 -0800 | [diff] [blame] | 428 | (s->length + seg_size > max_seg_size) || |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 429 | (ps->offset + ps->length) % PAGE_SIZE) { |
FUJITA Tomonori | fde9a10 | 2008-02-04 22:28:11 -0800 | [diff] [blame] | 430 | if (dma_map_cont(dev, start_sg, i - start, |
| 431 | sgmap, pages, need) < 0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 432 | goto error; |
| 433 | out++; |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 434 | |
| 435 | seg_size = 0; |
| 436 | sgmap = sg_next(sgmap); |
| 437 | pages = 0; |
| 438 | start = i; |
| 439 | start_sg = s; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 440 | } |
| 441 | } |
| 442 | |
FUJITA Tomonori | 42d0028 | 2008-02-04 22:27:56 -0800 | [diff] [blame] | 443 | seg_size += s->length; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 444 | need = nextneed; |
Joerg Roedel | 1477b8e | 2008-10-15 22:02:11 -0700 | [diff] [blame] | 445 | pages += iommu_num_pages(s->offset, s->length, PAGE_SIZE); |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 446 | ps = s; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 447 | } |
FUJITA Tomonori | fde9a10 | 2008-02-04 22:28:11 -0800 | [diff] [blame] | 448 | if (dma_map_cont(dev, start_sg, i - start, sgmap, pages, need) < 0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 449 | goto error; |
| 450 | out++; |
Andi Kleen | a32073b | 2006-06-26 13:56:40 +0200 | [diff] [blame] | 451 | flush_gart(); |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 452 | if (out < nents) { |
| 453 | sgmap = sg_next(sgmap); |
| 454 | sgmap->dma_length = 0; |
| 455 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 456 | return out; |
| 457 | |
| 458 | error: |
Andi Kleen | a32073b | 2006-06-26 13:56:40 +0200 | [diff] [blame] | 459 | flush_gart(); |
FUJITA Tomonori | 160c1d8 | 2009-01-05 23:59:02 +0900 | [diff] [blame] | 460 | gart_unmap_sg(dev, sg, out, dir, NULL); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 461 | |
Kevin VanMaren | a1002a48 | 2006-02-03 21:51:32 +0100 | [diff] [blame] | 462 | /* When it was forced or merged try again in a dumb way */ |
| 463 | if (force_iommu || iommu_merge) { |
| 464 | out = dma_map_sg_nonforce(dev, sg, nents, dir); |
| 465 | if (out > 0) |
| 466 | return out; |
| 467 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 468 | if (panic_on_overflow) |
| 469 | panic("dma_map_sg: overflow on %lu pages\n", pages); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 470 | |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 471 | iommu_full(dev, pages << PAGE_SHIFT, dir); |
Jens Axboe | 9ee1bea | 2007-10-04 09:35:37 +0200 | [diff] [blame] | 472 | for_each_sg(sg, s, nents, i) |
FUJITA Tomonori | 4210919 | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 473 | s->dma_address = bad_dma_addr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 474 | return 0; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 475 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 476 | |
Joerg Roedel | 9458109 | 2008-08-19 16:32:39 +0200 | [diff] [blame] | 477 | /* allocate and map a coherent mapping */ |
| 478 | static void * |
| 479 | gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, |
Andrzej Pietrasiewicz | baa676f | 2012-03-27 14:28:18 +0200 | [diff] [blame] | 480 | gfp_t flag, struct dma_attrs *attrs) |
Joerg Roedel | 9458109 | 2008-08-19 16:32:39 +0200 | [diff] [blame] | 481 | { |
FUJITA Tomonori | f6a32a3 | 2008-09-11 23:08:48 +0900 | [diff] [blame] | 482 | dma_addr_t paddr; |
FUJITA Tomonori | 421076e | 2008-08-22 16:29:10 +0900 | [diff] [blame] | 483 | unsigned long align_mask; |
FUJITA Tomonori | 1d99088 | 2008-09-24 20:48:37 +0900 | [diff] [blame] | 484 | struct page *page; |
Joerg Roedel | 9458109 | 2008-08-19 16:32:39 +0200 | [diff] [blame] | 485 | |
FUJITA Tomonori | 1d99088 | 2008-09-24 20:48:37 +0900 | [diff] [blame] | 486 | if (force_iommu && !(flag & GFP_DMA)) { |
| 487 | flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); |
| 488 | page = alloc_pages(flag | __GFP_ZERO, get_order(size)); |
| 489 | if (!page) |
| 490 | return NULL; |
Joerg Roedel | 9458109 | 2008-08-19 16:32:39 +0200 | [diff] [blame] | 491 | |
FUJITA Tomonori | 1d99088 | 2008-09-24 20:48:37 +0900 | [diff] [blame] | 492 | align_mask = (1UL << get_order(size)) - 1; |
| 493 | paddr = dma_map_area(dev, page_to_phys(page), size, |
| 494 | DMA_BIDIRECTIONAL, align_mask); |
FUJITA Tomonori | f6a32a3 | 2008-09-11 23:08:48 +0900 | [diff] [blame] | 495 | |
FUJITA Tomonori | 1d99088 | 2008-09-24 20:48:37 +0900 | [diff] [blame] | 496 | flush_gart(); |
FUJITA Tomonori | 4210919 | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 497 | if (paddr != bad_dma_addr) { |
FUJITA Tomonori | 1d99088 | 2008-09-24 20:48:37 +0900 | [diff] [blame] | 498 | *dma_addr = paddr; |
| 499 | return page_address(page); |
| 500 | } |
| 501 | __free_pages(page, get_order(size)); |
| 502 | } else |
Andrzej Pietrasiewicz | baa676f | 2012-03-27 14:28:18 +0200 | [diff] [blame] | 503 | return dma_generic_alloc_coherent(dev, size, dma_addr, flag, |
| 504 | attrs); |
Joerg Roedel | 9458109 | 2008-08-19 16:32:39 +0200 | [diff] [blame] | 505 | |
| 506 | return NULL; |
| 507 | } |
| 508 | |
Joerg Roedel | 43a5a5a | 2008-08-19 16:32:40 +0200 | [diff] [blame] | 509 | /* free a coherent mapping */ |
| 510 | static void |
| 511 | gart_free_coherent(struct device *dev, size_t size, void *vaddr, |
Andrzej Pietrasiewicz | baa676f | 2012-03-27 14:28:18 +0200 | [diff] [blame] | 512 | dma_addr_t dma_addr, struct dma_attrs *attrs) |
Joerg Roedel | 43a5a5a | 2008-08-19 16:32:40 +0200 | [diff] [blame] | 513 | { |
FUJITA Tomonori | d7dff84 | 2009-01-05 23:47:28 +0900 | [diff] [blame] | 514 | gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, NULL); |
Joerg Roedel | 43a5a5a | 2008-08-19 16:32:40 +0200 | [diff] [blame] | 515 | free_pages((unsigned long)vaddr, get_order(size)); |
| 516 | } |
| 517 | |
FUJITA Tomonori | 4210919 | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 518 | static int gart_mapping_error(struct device *dev, dma_addr_t dma_addr) |
| 519 | { |
| 520 | return (dma_addr == bad_dma_addr); |
| 521 | } |
| 522 | |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 523 | static int no_agp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 524 | |
| 525 | static __init unsigned long check_iommu_size(unsigned long aper, u64 aper_size) |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 526 | { |
| 527 | unsigned long a; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 528 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 529 | if (!iommu_size) { |
| 530 | iommu_size = aper_size; |
| 531 | if (!no_agp) |
| 532 | iommu_size /= 2; |
| 533 | } |
| 534 | |
| 535 | a = aper + iommu_size; |
Andi Kleen | 31422c5 | 2008-02-04 16:48:08 +0100 | [diff] [blame] | 536 | iommu_size -= round_up(a, PMD_PAGE_SIZE) - a; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 537 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 538 | if (iommu_size < 64*1024*1024) { |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 539 | pr_warning( |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 540 | "PCI-DMA: Warning: Small IOMMU %luMB." |
| 541 | " Consider increasing the AGP aperture in BIOS\n", |
| 542 | iommu_size >> 20); |
| 543 | } |
| 544 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 545 | return iommu_size; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 546 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 547 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 548 | static __init unsigned read_aperture(struct pci_dev *dev, u32 *size) |
| 549 | { |
| 550 | unsigned aper_size = 0, aper_base_32, aper_order; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 551 | u64 aper_base; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 552 | |
Pavel Machek | 3bb6fbf | 2008-04-15 12:43:57 +0200 | [diff] [blame] | 553 | pci_read_config_dword(dev, AMD64_GARTAPERTUREBASE, &aper_base_32); |
| 554 | pci_read_config_dword(dev, AMD64_GARTAPERTURECTL, &aper_order); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 555 | aper_order = (aper_order >> 1) & 7; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 556 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 557 | aper_base = aper_base_32 & 0x7fff; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 558 | aper_base <<= 25; |
| 559 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 560 | aper_size = (32 * 1024 * 1024) << aper_order; |
| 561 | if (aper_base + aper_size > 0x100000000UL || !aper_size) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 562 | aper_base = 0; |
| 563 | |
| 564 | *size = aper_size; |
| 565 | return aper_base; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 566 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 567 | |
Rafael J. Wysocki | 6703f6d | 2008-06-10 00:10:48 +0200 | [diff] [blame] | 568 | static void enable_gart_translations(void) |
| 569 | { |
| 570 | int i; |
| 571 | |
Hans Rosenfeld | 9653a5c | 2010-10-29 17:14:31 +0200 | [diff] [blame] | 572 | if (!amd_nb_has_feature(AMD_NB_GART)) |
Andreas Herrmann | 900f9ac | 2010-09-17 18:02:54 +0200 | [diff] [blame] | 573 | return; |
| 574 | |
Hans Rosenfeld | 9653a5c | 2010-10-29 17:14:31 +0200 | [diff] [blame] | 575 | for (i = 0; i < amd_nb_num(); i++) { |
| 576 | struct pci_dev *dev = node_to_amd_nb(i)->misc; |
Rafael J. Wysocki | 6703f6d | 2008-06-10 00:10:48 +0200 | [diff] [blame] | 577 | |
| 578 | enable_gart_translation(dev, __pa(agp_gatt_table)); |
| 579 | } |
Joerg Roedel | 4b83873 | 2010-04-07 12:57:35 +0200 | [diff] [blame] | 580 | |
| 581 | /* Flush the GART-TLB to remove stale entries */ |
Hans Rosenfeld | eec1d4f | 2010-10-29 17:14:30 +0200 | [diff] [blame] | 582 | amd_flush_garts(); |
Rafael J. Wysocki | 6703f6d | 2008-06-10 00:10:48 +0200 | [diff] [blame] | 583 | } |
| 584 | |
| 585 | /* |
| 586 | * If fix_up_north_bridges is set, the north bridges have to be fixed up on |
| 587 | * resume in the same way as they are handled in gart_iommu_hole_init(). |
| 588 | */ |
| 589 | static bool fix_up_north_bridges; |
| 590 | static u32 aperture_order; |
| 591 | static u32 aperture_alloc; |
| 592 | |
| 593 | void set_up_gart_resume(u32 aper_order, u32 aper_alloc) |
| 594 | { |
| 595 | fix_up_north_bridges = true; |
| 596 | aperture_order = aper_order; |
| 597 | aperture_alloc = aper_alloc; |
| 598 | } |
| 599 | |
Rafael J. Wysocki | f3c6ea1 | 2011-03-23 22:15:54 +0100 | [diff] [blame] | 600 | static void gart_fixup_northbridges(void) |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 601 | { |
| 602 | int i; |
| 603 | |
| 604 | if (!fix_up_north_bridges) |
| 605 | return; |
| 606 | |
Hans Rosenfeld | 9653a5c | 2010-10-29 17:14:31 +0200 | [diff] [blame] | 607 | if (!amd_nb_has_feature(AMD_NB_GART)) |
Andreas Herrmann | 900f9ac | 2010-09-17 18:02:54 +0200 | [diff] [blame] | 608 | return; |
| 609 | |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 610 | pr_info("PCI-DMA: Restoring GART aperture settings\n"); |
| 611 | |
Hans Rosenfeld | 9653a5c | 2010-10-29 17:14:31 +0200 | [diff] [blame] | 612 | for (i = 0; i < amd_nb_num(); i++) { |
| 613 | struct pci_dev *dev = node_to_amd_nb(i)->misc; |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 614 | |
| 615 | /* |
| 616 | * Don't enable translations just yet. That is the next |
| 617 | * step. Restore the pre-suspend aperture settings. |
| 618 | */ |
Borislav Petkov | 260133a | 2010-09-03 18:39:40 +0200 | [diff] [blame] | 619 | gart_set_size_and_enable(dev, aperture_order); |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 620 | pci_write_config_dword(dev, AMD64_GARTAPERTUREBASE, aperture_alloc >> 25); |
| 621 | } |
| 622 | } |
| 623 | |
Rafael J. Wysocki | f3c6ea1 | 2011-03-23 22:15:54 +0100 | [diff] [blame] | 624 | static void gart_resume(void) |
Pavel Machek | cd76374 | 2008-05-29 00:30:21 -0700 | [diff] [blame] | 625 | { |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 626 | pr_info("PCI-DMA: Resuming GART IOMMU\n"); |
Rafael J. Wysocki | 6703f6d | 2008-06-10 00:10:48 +0200 | [diff] [blame] | 627 | |
Rafael J. Wysocki | f3c6ea1 | 2011-03-23 22:15:54 +0100 | [diff] [blame] | 628 | gart_fixup_northbridges(); |
Rafael J. Wysocki | 6703f6d | 2008-06-10 00:10:48 +0200 | [diff] [blame] | 629 | |
| 630 | enable_gart_translations(); |
Pavel Machek | cd76374 | 2008-05-29 00:30:21 -0700 | [diff] [blame] | 631 | } |
| 632 | |
Rafael J. Wysocki | f3c6ea1 | 2011-03-23 22:15:54 +0100 | [diff] [blame] | 633 | static struct syscore_ops gart_syscore_ops = { |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 634 | .resume = gart_resume, |
Pavel Machek | cd76374 | 2008-05-29 00:30:21 -0700 | [diff] [blame] | 635 | |
| 636 | }; |
| 637 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 638 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 639 | * Private Northbridge GATT initialization in case we cannot use the |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 640 | * AGP driver for some reason. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 641 | */ |
Hans Rosenfeld | eec1d4f | 2010-10-29 17:14:30 +0200 | [diff] [blame] | 642 | static __init int init_amd_gatt(struct agp_kern_info *info) |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 643 | { |
| 644 | unsigned aper_size, gatt_size, new_aper_size; |
| 645 | unsigned aper_base, new_aper_base; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 646 | struct pci_dev *dev; |
| 647 | void *gatt; |
Rafael J. Wysocki | f3c6ea1 | 2011-03-23 22:15:54 +0100 | [diff] [blame] | 648 | int i; |
Andi Kleen | a32073b | 2006-06-26 13:56:40 +0200 | [diff] [blame] | 649 | |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 650 | pr_info("PCI-DMA: Disabling AGP.\n"); |
| 651 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 652 | aper_size = aper_base = info->aper_size = 0; |
Andi Kleen | a32073b | 2006-06-26 13:56:40 +0200 | [diff] [blame] | 653 | dev = NULL; |
Hans Rosenfeld | 9653a5c | 2010-10-29 17:14:31 +0200 | [diff] [blame] | 654 | for (i = 0; i < amd_nb_num(); i++) { |
| 655 | dev = node_to_amd_nb(i)->misc; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 656 | new_aper_base = read_aperture(dev, &new_aper_size); |
| 657 | if (!new_aper_base) |
| 658 | goto nommu; |
| 659 | |
| 660 | if (!aper_base) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 661 | aper_size = new_aper_size; |
| 662 | aper_base = new_aper_base; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 663 | } |
| 664 | if (aper_size != new_aper_size || aper_base != new_aper_base) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 665 | goto nommu; |
| 666 | } |
| 667 | if (!aper_base) |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 668 | goto nommu; |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 669 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 670 | info->aper_base = aper_base; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 671 | info->aper_size = aper_size >> 20; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 672 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 673 | gatt_size = (aper_size >> PAGE_SHIFT) * sizeof(u32); |
Joerg Roedel | 0114267 | 2008-09-25 12:42:12 +0200 | [diff] [blame] | 674 | gatt = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, |
| 675 | get_order(gatt_size)); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 676 | if (!gatt) |
Joachim Deguara | cf6387d | 2007-04-24 13:05:36 +0200 | [diff] [blame] | 677 | panic("Cannot allocate GATT table"); |
Arjan van de Ven | 6d238cc | 2008-01-30 13:34:06 +0100 | [diff] [blame] | 678 | if (set_memory_uc((unsigned long)gatt, gatt_size >> PAGE_SHIFT)) |
Joachim Deguara | cf6387d | 2007-04-24 13:05:36 +0200 | [diff] [blame] | 679 | panic("Could not set GART PTEs to uncacheable pages"); |
Joachim Deguara | cf6387d | 2007-04-24 13:05:36 +0200 | [diff] [blame] | 680 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 681 | agp_gatt_table = gatt; |
Andi Kleen | a32073b | 2006-06-26 13:56:40 +0200 | [diff] [blame] | 682 | |
Rafael J. Wysocki | f3c6ea1 | 2011-03-23 22:15:54 +0100 | [diff] [blame] | 683 | register_syscore_ops(&gart_syscore_ops); |
Rafael J. Wysocki | 6703f6d | 2008-06-10 00:10:48 +0200 | [diff] [blame] | 684 | |
Andi Kleen | a32073b | 2006-06-26 13:56:40 +0200 | [diff] [blame] | 685 | flush_gart(); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 686 | |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 687 | pr_info("PCI-DMA: aperture base @ %x size %u KB\n", |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 688 | aper_base, aper_size>>10); |
Yinghai Lu | 7ab073b | 2008-07-12 14:30:35 -0700 | [diff] [blame] | 689 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 690 | return 0; |
| 691 | |
| 692 | nommu: |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 693 | /* Should not happen anymore */ |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 694 | pr_warning("PCI-DMA: More than 4GB of RAM and no IOMMU\n" |
Joe Perches | ad361c9 | 2009-07-06 13:05:40 -0700 | [diff] [blame] | 695 | "falling back to iommu=soft.\n"); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 696 | return -1; |
| 697 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 698 | |
FUJITA Tomonori | 160c1d8 | 2009-01-05 23:59:02 +0900 | [diff] [blame] | 699 | static struct dma_map_ops gart_dma_ops = { |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 700 | .map_sg = gart_map_sg, |
| 701 | .unmap_sg = gart_unmap_sg, |
FUJITA Tomonori | 052aedb | 2009-01-05 23:47:23 +0900 | [diff] [blame] | 702 | .map_page = gart_map_page, |
| 703 | .unmap_page = gart_unmap_page, |
Andrzej Pietrasiewicz | baa676f | 2012-03-27 14:28:18 +0200 | [diff] [blame] | 704 | .alloc = gart_alloc_coherent, |
| 705 | .free = gart_free_coherent, |
FUJITA Tomonori | 4210919 | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 706 | .mapping_error = gart_mapping_error, |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 707 | }; |
| 708 | |
FUJITA Tomonori | 338bac5 | 2009-10-27 16:34:44 +0900 | [diff] [blame] | 709 | static void gart_iommu_shutdown(void) |
Yinghai Lu | bc2cea6 | 2007-07-21 17:11:28 +0200 | [diff] [blame] | 710 | { |
| 711 | struct pci_dev *dev; |
| 712 | int i; |
| 713 | |
Yinghai Lu | f3eee54 | 2009-12-14 11:52:15 +0900 | [diff] [blame] | 714 | /* don't shutdown it if there is AGP installed */ |
| 715 | if (!no_agp) |
Yinghai Lu | bc2cea6 | 2007-07-21 17:11:28 +0200 | [diff] [blame] | 716 | return; |
| 717 | |
Hans Rosenfeld | 9653a5c | 2010-10-29 17:14:31 +0200 | [diff] [blame] | 718 | if (!amd_nb_has_feature(AMD_NB_GART)) |
Andreas Herrmann | 900f9ac | 2010-09-17 18:02:54 +0200 | [diff] [blame] | 719 | return; |
| 720 | |
Hans Rosenfeld | 9653a5c | 2010-10-29 17:14:31 +0200 | [diff] [blame] | 721 | for (i = 0; i < amd_nb_num(); i++) { |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 722 | u32 ctl; |
Yinghai Lu | bc2cea6 | 2007-07-21 17:11:28 +0200 | [diff] [blame] | 723 | |
Hans Rosenfeld | 9653a5c | 2010-10-29 17:14:31 +0200 | [diff] [blame] | 724 | dev = node_to_amd_nb(i)->misc; |
Pavel Machek | 3bb6fbf | 2008-04-15 12:43:57 +0200 | [diff] [blame] | 725 | pci_read_config_dword(dev, AMD64_GARTAPERTURECTL, &ctl); |
Yinghai Lu | bc2cea6 | 2007-07-21 17:11:28 +0200 | [diff] [blame] | 726 | |
Pavel Machek | 3bb6fbf | 2008-04-15 12:43:57 +0200 | [diff] [blame] | 727 | ctl &= ~GARTEN; |
Yinghai Lu | bc2cea6 | 2007-07-21 17:11:28 +0200 | [diff] [blame] | 728 | |
Pavel Machek | 3bb6fbf | 2008-04-15 12:43:57 +0200 | [diff] [blame] | 729 | pci_write_config_dword(dev, AMD64_GARTAPERTURECTL, ctl); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 730 | } |
Yinghai Lu | bc2cea6 | 2007-07-21 17:11:28 +0200 | [diff] [blame] | 731 | } |
| 732 | |
FUJITA Tomonori | de95762 | 2009-11-10 19:46:14 +0900 | [diff] [blame] | 733 | int __init gart_iommu_init(void) |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 734 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 735 | struct agp_kern_info info; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 736 | unsigned long iommu_start; |
Yinghai Lu | d99e901 | 2008-10-04 15:55:12 -0700 | [diff] [blame] | 737 | unsigned long aper_base, aper_size; |
| 738 | unsigned long start_pfn, end_pfn; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 739 | unsigned long scratch; |
| 740 | long i; |
| 741 | |
Hans Rosenfeld | 9653a5c | 2010-10-29 17:14:31 +0200 | [diff] [blame] | 742 | if (!amd_nb_has_feature(AMD_NB_GART)) |
FUJITA Tomonori | de95762 | 2009-11-10 19:46:14 +0900 | [diff] [blame] | 743 | return 0; |
Andi Kleen | a32073b | 2006-06-26 13:56:40 +0200 | [diff] [blame] | 744 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 745 | #ifndef CONFIG_AGP_AMD64 |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 746 | no_agp = 1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 747 | #else |
| 748 | /* Makefile puts PCI initialization via subsys_initcall first. */ |
Hans Rosenfeld | eec1d4f | 2010-10-29 17:14:30 +0200 | [diff] [blame] | 749 | /* Add other AMD AGP bridge drivers here */ |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 750 | no_agp = no_agp || |
| 751 | (agp_amd64_init() < 0) || |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 752 | (agp_copy_info(agp_bridge, &info) < 0); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 753 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 754 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 755 | if (no_iommu || |
Yinghai Lu | c987d12 | 2008-06-24 22:14:09 -0700 | [diff] [blame] | 756 | (!force_iommu && max_pfn <= MAX_DMA32_PFN) || |
Joerg Roedel | 0440d4c | 2007-10-24 12:49:50 +0200 | [diff] [blame] | 757 | !gart_iommu_aperture || |
Hans Rosenfeld | eec1d4f | 2010-10-29 17:14:30 +0200 | [diff] [blame] | 758 | (no_agp && init_amd_gatt(&info) < 0)) { |
Yinghai Lu | c987d12 | 2008-06-24 22:14:09 -0700 | [diff] [blame] | 759 | if (max_pfn > MAX_DMA32_PFN) { |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 760 | pr_warning("More than 4GB of memory but GART IOMMU not available.\n"); |
| 761 | pr_warning("falling back to iommu=soft.\n"); |
Jon Mason | 5b7b644 | 2006-02-03 21:51:59 +0100 | [diff] [blame] | 762 | } |
FUJITA Tomonori | de95762 | 2009-11-10 19:46:14 +0900 | [diff] [blame] | 763 | return 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 764 | } |
| 765 | |
Yinghai Lu | d99e901 | 2008-10-04 15:55:12 -0700 | [diff] [blame] | 766 | /* need to map that range */ |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 767 | aper_size = info.aper_size << 20; |
| 768 | aper_base = info.aper_base; |
| 769 | end_pfn = (aper_base>>PAGE_SHIFT) + (aper_size>>PAGE_SHIFT); |
| 770 | |
Yinghai Lu | d99e901 | 2008-10-04 15:55:12 -0700 | [diff] [blame] | 771 | if (end_pfn > max_low_pfn_mapped) { |
| 772 | start_pfn = (aper_base>>PAGE_SHIFT); |
| 773 | init_memory_mapping(start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT); |
| 774 | } |
| 775 | |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 776 | pr_info("PCI-DMA: using GART IOMMU.\n"); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 777 | iommu_size = check_iommu_size(info.aper_base, aper_size); |
| 778 | iommu_pages = iommu_size >> PAGE_SHIFT; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 779 | |
Joerg Roedel | 0114267 | 2008-09-25 12:42:12 +0200 | [diff] [blame] | 780 | iommu_gart_bitmap = (void *) __get_free_pages(GFP_KERNEL | __GFP_ZERO, |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 781 | get_order(iommu_pages/8)); |
| 782 | if (!iommu_gart_bitmap) |
| 783 | panic("Cannot allocate iommu bitmap\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 784 | |
| 785 | #ifdef CONFIG_IOMMU_LEAK |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 786 | if (leak_trace) { |
FUJITA Tomonori | 19c1a6f | 2009-04-14 09:43:19 +0900 | [diff] [blame] | 787 | int ret; |
| 788 | |
| 789 | ret = dma_debug_resize_entries(iommu_pages); |
| 790 | if (ret) |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 791 | pr_debug("PCI-DMA: Cannot trace all the entries\n"); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 792 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 793 | #endif |
| 794 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 795 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 796 | * Out of IOMMU space handling. |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 797 | * Reserve some invalid pages at the beginning of the GART. |
| 798 | */ |
Akinobu Mita | a66022c | 2009-12-15 16:48:28 -0800 | [diff] [blame] | 799 | bitmap_set(iommu_gart_bitmap, 0, EMERGENCY_PAGES); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 800 | |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 801 | pr_info("PCI-DMA: Reserving %luMB of IOMMU area in the AGP aperture\n", |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 802 | iommu_size >> 20); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 803 | |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 804 | agp_memory_reserved = iommu_size; |
| 805 | iommu_start = aper_size - iommu_size; |
| 806 | iommu_bus_base = info.aper_base + iommu_start; |
| 807 | bad_dma_addr = iommu_bus_base; |
| 808 | iommu_gatt_base = agp_gatt_table + (iommu_start>>PAGE_SHIFT); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 809 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 810 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 811 | * Unmap the IOMMU part of the GART. The alias of the page is |
| 812 | * always mapped with cache enabled and there is no full cache |
| 813 | * coherency across the GART remapping. The unmapping avoids |
| 814 | * automatic prefetches from the CPU allocating cache lines in |
| 815 | * there. All CPU accesses are done via the direct mapping to |
| 816 | * the backing memory. The GART address is only used by PCI |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 817 | * devices. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 818 | */ |
Andi Kleen | 28d6ee4 | 2008-02-04 16:48:08 +0100 | [diff] [blame] | 819 | set_memory_np((unsigned long)__va(iommu_bus_base), |
| 820 | iommu_size >> PAGE_SHIFT); |
Ingo Molnar | 184652e | 2008-02-14 23:30:20 +0100 | [diff] [blame] | 821 | /* |
| 822 | * Tricky. The GART table remaps the physical memory range, |
| 823 | * so the CPU wont notice potential aliases and if the memory |
| 824 | * is remapped to UC later on, we might surprise the PCI devices |
| 825 | * with a stray writeout of a cacheline. So play it sure and |
| 826 | * do an explicit, full-scale wbinvd() _after_ having marked all |
| 827 | * the pages as Not-Present: |
| 828 | */ |
| 829 | wbinvd(); |
Ingo Molnar | 123bf0e | 2009-11-15 21:19:52 +0900 | [diff] [blame] | 830 | |
Mark Langsdorf | fe2245c | 2009-07-05 15:50:52 -0500 | [diff] [blame] | 831 | /* |
| 832 | * Now all caches are flushed and we can safely enable |
| 833 | * GART hardware. Doing it early leaves the possibility |
| 834 | * of stale cache entries that can lead to GART PTE |
| 835 | * errors. |
| 836 | */ |
| 837 | enable_gart_translations(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 838 | |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 839 | /* |
Pavel Machek | fa3d319 | 2008-06-26 00:25:43 +0200 | [diff] [blame] | 840 | * Try to workaround a bug (thanks to BenH): |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 841 | * Set unmapped entries to a scratch page instead of 0. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 842 | * Any prefetches that hit unmapped entries won't get an bus abort |
Pavel Machek | fa3d319 | 2008-06-26 00:25:43 +0200 | [diff] [blame] | 843 | * then. (P2P bridge may be prefetching on DMA reads). |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 844 | */ |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 845 | scratch = get_zeroed_page(GFP_KERNEL); |
| 846 | if (!scratch) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 847 | panic("Cannot allocate iommu scratch page"); |
| 848 | gart_unmapped_entry = GPTE_ENCODE(__pa(scratch)); |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 849 | for (i = EMERGENCY_PAGES; i < iommu_pages; i++) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 850 | iommu_gatt_base[i] = gart_unmapped_entry; |
| 851 | |
Andi Kleen | a32073b | 2006-06-26 13:56:40 +0200 | [diff] [blame] | 852 | flush_gart(); |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 853 | dma_ops = &gart_dma_ops; |
FUJITA Tomonori | 338bac5 | 2009-10-27 16:34:44 +0900 | [diff] [blame] | 854 | x86_platform.iommu_shutdown = gart_iommu_shutdown; |
FUJITA Tomonori | 75f1cdf | 2009-11-10 19:46:20 +0900 | [diff] [blame] | 855 | swiotlb = 0; |
FUJITA Tomonori | de95762 | 2009-11-10 19:46:14 +0900 | [diff] [blame] | 856 | |
| 857 | return 0; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 858 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 859 | |
Sam Ravnborg | 43999d9 | 2007-03-16 21:07:36 +0100 | [diff] [blame] | 860 | void __init gart_parse_options(char *p) |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 861 | { |
| 862 | int arg; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 863 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 864 | #ifdef CONFIG_IOMMU_LEAK |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 865 | if (!strncmp(p, "leak", 4)) { |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 866 | leak_trace = 1; |
| 867 | p += 4; |
Joerg Roedel | 237a622 | 2008-09-25 12:13:53 +0200 | [diff] [blame] | 868 | if (*p == '=') |
| 869 | ++p; |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 870 | if (isdigit(*p) && get_option(&p, &arg)) |
| 871 | iommu_leak_pages = arg; |
| 872 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 873 | #endif |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 874 | if (isdigit(*p) && get_option(&p, &arg)) |
| 875 | iommu_size = arg; |
Joe Perches | 41855b7 | 2009-11-09 17:58:50 -0800 | [diff] [blame] | 876 | if (!strncmp(p, "fullflush", 9)) |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 877 | iommu_fullflush = 1; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 878 | if (!strncmp(p, "nofullflush", 11)) |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 879 | iommu_fullflush = 0; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 880 | if (!strncmp(p, "noagp", 5)) |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 881 | no_agp = 1; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 882 | if (!strncmp(p, "noaperture", 10)) |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 883 | fix_aperture = 0; |
| 884 | /* duplicated from pci-dma.c */ |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 885 | if (!strncmp(p, "force", 5)) |
Joerg Roedel | 0440d4c | 2007-10-24 12:49:50 +0200 | [diff] [blame] | 886 | gart_iommu_aperture_allowed = 1; |
Ingo Molnar | 05fccb0 | 2008-01-30 13:30:12 +0100 | [diff] [blame] | 887 | if (!strncmp(p, "allowed", 7)) |
Joerg Roedel | 0440d4c | 2007-10-24 12:49:50 +0200 | [diff] [blame] | 888 | gart_iommu_aperture_allowed = 1; |
Muli Ben-Yehuda | 17a941d | 2006-01-11 22:44:42 +0100 | [diff] [blame] | 889 | if (!strncmp(p, "memaper", 7)) { |
| 890 | fallback_aper_force = 1; |
| 891 | p += 7; |
| 892 | if (*p == '=') { |
| 893 | ++p; |
| 894 | if (get_option(&p, &arg)) |
| 895 | fallback_aper_order = arg; |
| 896 | } |
| 897 | } |
| 898 | } |
Konrad Rzeszutek Wilk | 22e6daf | 2010-08-26 13:58:03 -0400 | [diff] [blame] | 899 | IOMMU_INIT_POST(gart_iommu_hole_init); |