Thomas Gleixner | c942fdd | 2019-05-27 08:55:06 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
Jérôme Glisse | 133ff0e | 2017-09-08 16:11:23 -0700 | [diff] [blame] | 2 | /* |
| 3 | * Copyright 2013 Red Hat Inc. |
| 4 | * |
Jérôme Glisse | f813f21 | 2018-10-30 15:04:06 -0700 | [diff] [blame] | 5 | * Authors: Jérôme Glisse <jglisse@redhat.com> |
Jérôme Glisse | 133ff0e | 2017-09-08 16:11:23 -0700 | [diff] [blame] | 6 | */ |
| 7 | /* |
| 8 | * Refer to include/linux/hmm.h for information about heterogeneous memory |
| 9 | * management or HMM for short. |
| 10 | */ |
| 11 | #include <linux/mm.h> |
| 12 | #include <linux/hmm.h> |
Jérôme Glisse | 858b54d | 2017-09-08 16:12:02 -0700 | [diff] [blame] | 13 | #include <linux/init.h> |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 14 | #include <linux/rmap.h> |
| 15 | #include <linux/swap.h> |
Jérôme Glisse | 133ff0e | 2017-09-08 16:11:23 -0700 | [diff] [blame] | 16 | #include <linux/slab.h> |
| 17 | #include <linux/sched.h> |
Jérôme Glisse | 4ef589d | 2017-09-08 16:11:58 -0700 | [diff] [blame] | 18 | #include <linux/mmzone.h> |
| 19 | #include <linux/pagemap.h> |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 20 | #include <linux/swapops.h> |
| 21 | #include <linux/hugetlb.h> |
Jérôme Glisse | 4ef589d | 2017-09-08 16:11:58 -0700 | [diff] [blame] | 22 | #include <linux/memremap.h> |
Jason Gunthorpe | c8a53b2 | 2019-05-23 10:36:46 -0300 | [diff] [blame] | 23 | #include <linux/sched/mm.h> |
Jérôme Glisse | 7b2d55d2 | 2017-09-08 16:11:46 -0700 | [diff] [blame] | 24 | #include <linux/jump_label.h> |
Jérôme Glisse | 55c0ece | 2019-05-13 17:20:28 -0700 | [diff] [blame] | 25 | #include <linux/dma-mapping.h> |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 26 | #include <linux/mmu_notifier.h> |
Jérôme Glisse | 4ef589d | 2017-09-08 16:11:58 -0700 | [diff] [blame] | 27 | #include <linux/memory_hotplug.h> |
| 28 | |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 29 | static const struct mmu_notifier_ops hmm_mmu_notifier_ops; |
| 30 | |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 31 | /** |
| 32 | * hmm_get_or_create - register HMM against an mm (HMM internal) |
| 33 | * |
| 34 | * @mm: mm struct to attach to |
| 35 | * Returns: returns an HMM object, either by referencing the existing |
| 36 | * (per-process) object, or by creating a new one. |
| 37 | * |
| 38 | * This is not intended to be used directly by device drivers. If mm already |
| 39 | * has an HMM struct then it get a reference on it and returns it. Otherwise |
| 40 | * it allocates an HMM struct, initializes it, associate it with the mm and |
| 41 | * returns it. |
| 42 | */ |
| 43 | static struct hmm *hmm_get_or_create(struct mm_struct *mm) |
| 44 | { |
Jason Gunthorpe | 8a9320b | 2019-05-23 10:24:13 -0300 | [diff] [blame] | 45 | struct hmm *hmm; |
Jérôme Glisse | 133ff0e | 2017-09-08 16:11:23 -0700 | [diff] [blame] | 46 | |
Linus Torvalds | fec88ab | 2019-07-14 19:42:11 -0700 | [diff] [blame] | 47 | lockdep_assert_held_write(&mm->mmap_sem); |
Jason Gunthorpe | 8a9320b | 2019-05-23 10:24:13 -0300 | [diff] [blame] | 48 | |
| 49 | /* Abuse the page_table_lock to also protect mm->hmm. */ |
| 50 | spin_lock(&mm->page_table_lock); |
| 51 | hmm = mm->hmm; |
| 52 | if (mm->hmm && kref_get_unless_zero(&mm->hmm->kref)) |
| 53 | goto out_unlock; |
| 54 | spin_unlock(&mm->page_table_lock); |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 55 | |
| 56 | hmm = kmalloc(sizeof(*hmm), GFP_KERNEL); |
| 57 | if (!hmm) |
| 58 | return NULL; |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 59 | init_waitqueue_head(&hmm->wq); |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 60 | INIT_LIST_HEAD(&hmm->mirrors); |
| 61 | init_rwsem(&hmm->mirrors_sem); |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 62 | hmm->mmu_notifier.ops = NULL; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 63 | INIT_LIST_HEAD(&hmm->ranges); |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 64 | spin_lock_init(&hmm->ranges_lock); |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 65 | kref_init(&hmm->kref); |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 66 | hmm->notifiers = 0; |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 67 | hmm->mm = mm; |
| 68 | |
Jason Gunthorpe | 8a9320b | 2019-05-23 10:24:13 -0300 | [diff] [blame] | 69 | hmm->mmu_notifier.ops = &hmm_mmu_notifier_ops; |
| 70 | if (__mmu_notifier_register(&hmm->mmu_notifier, mm)) { |
| 71 | kfree(hmm); |
| 72 | return NULL; |
| 73 | } |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 74 | |
Jason Gunthorpe | c8a53b2 | 2019-05-23 10:36:46 -0300 | [diff] [blame] | 75 | mmgrab(hmm->mm); |
Ralph Campbell | 86a2d59 | 2018-10-30 15:04:14 -0700 | [diff] [blame] | 76 | |
| 77 | /* |
Jason Gunthorpe | 8a9320b | 2019-05-23 10:24:13 -0300 | [diff] [blame] | 78 | * We hold the exclusive mmap_sem here so we know that mm->hmm is |
| 79 | * still NULL or 0 kref, and is safe to update. |
Ralph Campbell | 86a2d59 | 2018-10-30 15:04:14 -0700 | [diff] [blame] | 80 | */ |
Ralph Campbell | 86a2d59 | 2018-10-30 15:04:14 -0700 | [diff] [blame] | 81 | spin_lock(&mm->page_table_lock); |
Jason Gunthorpe | 8a9320b | 2019-05-23 10:24:13 -0300 | [diff] [blame] | 82 | mm->hmm = hmm; |
| 83 | |
| 84 | out_unlock: |
Ralph Campbell | 86a2d59 | 2018-10-30 15:04:14 -0700 | [diff] [blame] | 85 | spin_unlock(&mm->page_table_lock); |
Jason Gunthorpe | 8a9320b | 2019-05-23 10:24:13 -0300 | [diff] [blame] | 86 | return hmm; |
Jérôme Glisse | 133ff0e | 2017-09-08 16:11:23 -0700 | [diff] [blame] | 87 | } |
| 88 | |
Jason Gunthorpe | 6d7c3cd | 2019-05-22 16:52:52 -0300 | [diff] [blame] | 89 | static void hmm_free_rcu(struct rcu_head *rcu) |
| 90 | { |
Jason Gunthorpe | 8a9320b | 2019-05-23 10:24:13 -0300 | [diff] [blame] | 91 | struct hmm *hmm = container_of(rcu, struct hmm, rcu); |
| 92 | |
| 93 | mmdrop(hmm->mm); |
Jérôme Glisse | 133ff0e | 2017-09-08 16:11:23 -0700 | [diff] [blame] | 94 | kfree(hmm); |
Jérôme Glisse | 133ff0e | 2017-09-08 16:11:23 -0700 | [diff] [blame] | 95 | } |
| 96 | |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 97 | static void hmm_free(struct kref *kref) |
| 98 | { |
| 99 | struct hmm *hmm = container_of(kref, struct hmm, kref); |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 100 | |
Jason Gunthorpe | 8a9320b | 2019-05-23 10:24:13 -0300 | [diff] [blame] | 101 | spin_lock(&hmm->mm->page_table_lock); |
| 102 | if (hmm->mm->hmm == hmm) |
| 103 | hmm->mm->hmm = NULL; |
| 104 | spin_unlock(&hmm->mm->page_table_lock); |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 105 | |
Jason Gunthorpe | 8a9320b | 2019-05-23 10:24:13 -0300 | [diff] [blame] | 106 | mmu_notifier_unregister_no_release(&hmm->mmu_notifier, hmm->mm); |
Jason Gunthorpe | 6d7c3cd | 2019-05-22 16:52:52 -0300 | [diff] [blame] | 107 | mmu_notifier_call_srcu(&hmm->rcu, hmm_free_rcu); |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 108 | } |
| 109 | |
| 110 | static inline void hmm_put(struct hmm *hmm) |
| 111 | { |
| 112 | kref_put(&hmm->kref, hmm_free); |
| 113 | } |
| 114 | |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 115 | static void hmm_release(struct mmu_notifier *mn, struct mm_struct *mm) |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 116 | { |
Jason Gunthorpe | 6d7c3cd | 2019-05-22 16:52:52 -0300 | [diff] [blame] | 117 | struct hmm *hmm = container_of(mn, struct hmm, mmu_notifier); |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 118 | struct hmm_mirror *mirror; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 119 | |
Jason Gunthorpe | 6d7c3cd | 2019-05-22 16:52:52 -0300 | [diff] [blame] | 120 | /* Bail out if hmm is in the process of being freed */ |
| 121 | if (!kref_get_unless_zero(&hmm->kref)) |
| 122 | return; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 123 | |
Jason Gunthorpe | 47f2459 | 2019-05-23 11:08:28 -0300 | [diff] [blame] | 124 | /* |
| 125 | * Since hmm_range_register() holds the mmget() lock hmm_release() is |
| 126 | * prevented as long as a range exists. |
| 127 | */ |
| 128 | WARN_ON(!list_empty_careful(&hmm->ranges)); |
Ralph Campbell | e140151 | 2018-04-10 16:28:19 -0700 | [diff] [blame] | 129 | |
Jason Gunthorpe | 1433172 | 2019-05-24 12:14:08 -0300 | [diff] [blame] | 130 | down_read(&hmm->mirrors_sem); |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 131 | list_for_each_entry(mirror, &hmm->mirrors, list) { |
Jason Gunthorpe | 1433172 | 2019-05-24 12:14:08 -0300 | [diff] [blame] | 132 | /* |
| 133 | * Note: The driver is not allowed to trigger |
| 134 | * hmm_mirror_unregister() from this thread. |
| 135 | */ |
| 136 | if (mirror->ops->release) |
Ralph Campbell | e140151 | 2018-04-10 16:28:19 -0700 | [diff] [blame] | 137 | mirror->ops->release(mirror); |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 138 | } |
| 139 | up_read(&hmm->mirrors_sem); |
| 140 | |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 141 | hmm_put(hmm); |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 142 | } |
| 143 | |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 144 | static void notifiers_decrement(struct hmm *hmm) |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 145 | { |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 146 | unsigned long flags; |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 147 | |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 148 | spin_lock_irqsave(&hmm->ranges_lock, flags); |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 149 | hmm->notifiers--; |
| 150 | if (!hmm->notifiers) { |
| 151 | struct hmm_range *range; |
| 152 | |
| 153 | list_for_each_entry(range, &hmm->ranges, list) { |
| 154 | if (range->valid) |
| 155 | continue; |
| 156 | range->valid = true; |
| 157 | } |
| 158 | wake_up_all(&hmm->wq); |
| 159 | } |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 160 | spin_unlock_irqrestore(&hmm->ranges_lock, flags); |
| 161 | } |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 162 | |
Jérôme Glisse | 133ff0e | 2017-09-08 16:11:23 -0700 | [diff] [blame] | 163 | static int hmm_invalidate_range_start(struct mmu_notifier *mn, |
| 164 | const struct mmu_notifier_range *nrange) |
| 165 | { |
Jason Gunthorpe | 6d7c3cd | 2019-05-22 16:52:52 -0300 | [diff] [blame] | 166 | struct hmm *hmm = container_of(mn, struct hmm, mmu_notifier); |
Jérôme Glisse | 133ff0e | 2017-09-08 16:11:23 -0700 | [diff] [blame] | 167 | struct hmm_mirror *mirror; |
| 168 | struct hmm_update update; |
| 169 | struct hmm_range *range; |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 170 | unsigned long flags; |
Jérôme Glisse | 133ff0e | 2017-09-08 16:11:23 -0700 | [diff] [blame] | 171 | int ret = 0; |
| 172 | |
Jason Gunthorpe | 6d7c3cd | 2019-05-22 16:52:52 -0300 | [diff] [blame] | 173 | if (!kref_get_unless_zero(&hmm->kref)) |
| 174 | return 0; |
Jérôme Glisse | 133ff0e | 2017-09-08 16:11:23 -0700 | [diff] [blame] | 175 | |
| 176 | update.start = nrange->start; |
| 177 | update.end = nrange->end; |
| 178 | update.event = HMM_UPDATE_INVALIDATE; |
| 179 | update.blockable = mmu_notifier_range_blockable(nrange); |
| 180 | |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 181 | spin_lock_irqsave(&hmm->ranges_lock, flags); |
Jérôme Glisse | 133ff0e | 2017-09-08 16:11:23 -0700 | [diff] [blame] | 182 | hmm->notifiers++; |
| 183 | list_for_each_entry(range, &hmm->ranges, list) { |
| 184 | if (update.end < range->start || update.start >= range->end) |
| 185 | continue; |
| 186 | |
| 187 | range->valid = false; |
| 188 | } |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 189 | spin_unlock_irqrestore(&hmm->ranges_lock, flags); |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 190 | |
| 191 | if (mmu_notifier_range_blockable(nrange)) |
| 192 | down_read(&hmm->mirrors_sem); |
| 193 | else if (!down_read_trylock(&hmm->mirrors_sem)) { |
| 194 | ret = -EAGAIN; |
| 195 | goto out; |
| 196 | } |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 197 | |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 198 | list_for_each_entry(mirror, &hmm->mirrors, list) { |
| 199 | int rc; |
| 200 | |
| 201 | rc = mirror->ops->sync_cpu_device_pagetables(mirror, &update); |
| 202 | if (rc) { |
| 203 | if (WARN_ON(update.blockable || rc != -EAGAIN)) |
| 204 | continue; |
| 205 | ret = -EAGAIN; |
Ralph Campbell | 085ea25 | 2019-05-06 16:29:39 -0700 | [diff] [blame] | 206 | break; |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 207 | } |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 208 | } |
| 209 | up_read(&hmm->mirrors_sem); |
| 210 | |
| 211 | out: |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 212 | if (ret) |
| 213 | notifiers_decrement(hmm); |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 214 | hmm_put(hmm); |
| 215 | return ret; |
| 216 | } |
| 217 | |
| 218 | static void hmm_invalidate_range_end(struct mmu_notifier *mn, |
| 219 | const struct mmu_notifier_range *nrange) |
| 220 | { |
Jason Gunthorpe | 6d7c3cd | 2019-05-22 16:52:52 -0300 | [diff] [blame] | 221 | struct hmm *hmm = container_of(mn, struct hmm, mmu_notifier); |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 222 | |
Jason Gunthorpe | 6d7c3cd | 2019-05-22 16:52:52 -0300 | [diff] [blame] | 223 | if (!kref_get_unless_zero(&hmm->kref)) |
| 224 | return; |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 225 | |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 226 | notifiers_decrement(hmm); |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 227 | hmm_put(hmm); |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 228 | } |
| 229 | |
| 230 | static const struct mmu_notifier_ops hmm_mmu_notifier_ops = { |
Ralph Campbell | e140151 | 2018-04-10 16:28:19 -0700 | [diff] [blame] | 231 | .release = hmm_release, |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 232 | .invalidate_range_start = hmm_invalidate_range_start, |
| 233 | .invalidate_range_end = hmm_invalidate_range_end, |
| 234 | }; |
| 235 | |
| 236 | /* |
| 237 | * hmm_mirror_register() - register a mirror against an mm |
| 238 | * |
| 239 | * @mirror: new mirror struct to register |
| 240 | * @mm: mm to register against |
Ralph Campbell | 085ea25 | 2019-05-06 16:29:39 -0700 | [diff] [blame] | 241 | * Return: 0 on success, -ENOMEM if no memory, -EINVAL if invalid arguments |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 242 | * |
| 243 | * To start mirroring a process address space, the device driver must register |
| 244 | * an HMM mirror struct. |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 245 | */ |
| 246 | int hmm_mirror_register(struct hmm_mirror *mirror, struct mm_struct *mm) |
| 247 | { |
Linus Torvalds | fec88ab | 2019-07-14 19:42:11 -0700 | [diff] [blame] | 248 | lockdep_assert_held_write(&mm->mmap_sem); |
Jason Gunthorpe | 8a1a0cd | 2019-05-23 11:23:30 -0300 | [diff] [blame] | 249 | |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 250 | /* Sanity check */ |
| 251 | if (!mm || !mirror || !mirror->ops) |
| 252 | return -EINVAL; |
| 253 | |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 254 | mirror->hmm = hmm_get_or_create(mm); |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 255 | if (!mirror->hmm) |
| 256 | return -ENOMEM; |
| 257 | |
| 258 | down_write(&mirror->hmm->mirrors_sem); |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 259 | list_add(&mirror->list, &mirror->hmm->mirrors); |
| 260 | up_write(&mirror->hmm->mirrors_sem); |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 261 | |
| 262 | return 0; |
| 263 | } |
| 264 | EXPORT_SYMBOL(hmm_mirror_register); |
| 265 | |
| 266 | /* |
| 267 | * hmm_mirror_unregister() - unregister a mirror |
| 268 | * |
Ralph Campbell | 085ea25 | 2019-05-06 16:29:39 -0700 | [diff] [blame] | 269 | * @mirror: mirror struct to unregister |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 270 | * |
| 271 | * Stop mirroring a process address space, and cleanup. |
| 272 | */ |
| 273 | void hmm_mirror_unregister(struct hmm_mirror *mirror) |
| 274 | { |
Jason Gunthorpe | 187229c | 2019-05-23 11:31:45 -0300 | [diff] [blame] | 275 | struct hmm *hmm = mirror->hmm; |
Jérôme Glisse | c01cbba | 2018-04-10 16:28:23 -0700 | [diff] [blame] | 276 | |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 277 | down_write(&hmm->mirrors_sem); |
Jason Gunthorpe | 1433172 | 2019-05-24 12:14:08 -0300 | [diff] [blame] | 278 | list_del(&mirror->list); |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 279 | up_write(&hmm->mirrors_sem); |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 280 | hmm_put(hmm); |
Jérôme Glisse | c0b1240 | 2017-09-08 16:11:27 -0700 | [diff] [blame] | 281 | } |
| 282 | EXPORT_SYMBOL(hmm_mirror_unregister); |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 283 | |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 284 | struct hmm_vma_walk { |
| 285 | struct hmm_range *range; |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 286 | struct dev_pagemap *pgmap; |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 287 | unsigned long last; |
| 288 | bool fault; |
| 289 | bool block; |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 290 | }; |
| 291 | |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 292 | static int hmm_vma_do_fault(struct mm_walk *walk, unsigned long addr, |
| 293 | bool write_fault, uint64_t *pfn) |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 294 | { |
Kuehling, Felix | 9b1ae60 | 2019-05-10 19:53:24 +0000 | [diff] [blame] | 295 | unsigned int flags = FAULT_FLAG_REMOTE; |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 296 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 297 | struct hmm_range *range = hmm_vma_walk->range; |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 298 | struct vm_area_struct *vma = walk->vma; |
Souptick Joarder | 50a7ca3 | 2018-08-17 15:44:47 -0700 | [diff] [blame] | 299 | vm_fault_t ret; |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 300 | |
| 301 | flags |= hmm_vma_walk->block ? 0 : FAULT_FLAG_ALLOW_RETRY; |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 302 | flags |= write_fault ? FAULT_FLAG_WRITE : 0; |
Souptick Joarder | 50a7ca3 | 2018-08-17 15:44:47 -0700 | [diff] [blame] | 303 | ret = handle_mm_fault(vma, addr, flags); |
| 304 | if (ret & VM_FAULT_RETRY) |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 305 | return -EAGAIN; |
Souptick Joarder | 50a7ca3 | 2018-08-17 15:44:47 -0700 | [diff] [blame] | 306 | if (ret & VM_FAULT_ERROR) { |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 307 | *pfn = range->values[HMM_PFN_ERROR]; |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 308 | return -EFAULT; |
| 309 | } |
| 310 | |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 311 | return -EBUSY; |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 312 | } |
| 313 | |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 314 | static int hmm_pfns_bad(unsigned long addr, |
| 315 | unsigned long end, |
| 316 | struct mm_walk *walk) |
| 317 | { |
Jérôme Glisse | c719547 | 2018-04-10 16:28:27 -0700 | [diff] [blame] | 318 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
| 319 | struct hmm_range *range = hmm_vma_walk->range; |
Jérôme Glisse | ff05c0c | 2018-04-10 16:28:38 -0700 | [diff] [blame] | 320 | uint64_t *pfns = range->pfns; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 321 | unsigned long i; |
| 322 | |
| 323 | i = (addr - range->start) >> PAGE_SHIFT; |
| 324 | for (; addr < end; addr += PAGE_SIZE, i++) |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 325 | pfns[i] = range->values[HMM_PFN_ERROR]; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 326 | |
| 327 | return 0; |
| 328 | } |
| 329 | |
Jérôme Glisse | 5504ed2 | 2018-04-10 16:28:46 -0700 | [diff] [blame] | 330 | /* |
| 331 | * hmm_vma_walk_hole() - handle a range lacking valid pmd or pte(s) |
| 332 | * @start: range virtual start address (inclusive) |
| 333 | * @end: range virtual end address (exclusive) |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 334 | * @fault: should we fault or not ? |
| 335 | * @write_fault: write fault ? |
Jérôme Glisse | 5504ed2 | 2018-04-10 16:28:46 -0700 | [diff] [blame] | 336 | * @walk: mm_walk structure |
Ralph Campbell | 085ea25 | 2019-05-06 16:29:39 -0700 | [diff] [blame] | 337 | * Return: 0 on success, -EBUSY after page fault, or page fault error |
Jérôme Glisse | 5504ed2 | 2018-04-10 16:28:46 -0700 | [diff] [blame] | 338 | * |
| 339 | * This function will be called whenever pmd_none() or pte_none() returns true, |
| 340 | * or whenever there is no page directory covering the virtual address range. |
| 341 | */ |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 342 | static int hmm_vma_walk_hole_(unsigned long addr, unsigned long end, |
| 343 | bool fault, bool write_fault, |
| 344 | struct mm_walk *walk) |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 345 | { |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 346 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
| 347 | struct hmm_range *range = hmm_vma_walk->range; |
Jérôme Glisse | ff05c0c | 2018-04-10 16:28:38 -0700 | [diff] [blame] | 348 | uint64_t *pfns = range->pfns; |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 349 | unsigned long i, page_size; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 350 | |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 351 | hmm_vma_walk->last = addr; |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 352 | page_size = hmm_range_page_size(range); |
| 353 | i = (addr - range->start) >> range->page_shift; |
| 354 | |
| 355 | for (; addr < end; addr += page_size, i++) { |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 356 | pfns[i] = range->values[HMM_PFN_NONE]; |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 357 | if (fault || write_fault) { |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 358 | int ret; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 359 | |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 360 | ret = hmm_vma_do_fault(walk, addr, write_fault, |
| 361 | &pfns[i]); |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 362 | if (ret != -EBUSY) |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 363 | return ret; |
| 364 | } |
| 365 | } |
| 366 | |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 367 | return (fault || write_fault) ? -EBUSY : 0; |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 368 | } |
| 369 | |
| 370 | static inline void hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk, |
| 371 | uint64_t pfns, uint64_t cpu_flags, |
| 372 | bool *fault, bool *write_fault) |
| 373 | { |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 374 | struct hmm_range *range = hmm_vma_walk->range; |
| 375 | |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 376 | if (!hmm_vma_walk->fault) |
| 377 | return; |
| 378 | |
Jérôme Glisse | 023a019 | 2019-05-13 17:20:05 -0700 | [diff] [blame] | 379 | /* |
| 380 | * So we not only consider the individual per page request we also |
| 381 | * consider the default flags requested for the range. The API can |
| 382 | * be use in 2 fashions. The first one where the HMM user coalesce |
| 383 | * multiple page fault into one request and set flags per pfns for |
| 384 | * of those faults. The second one where the HMM user want to pre- |
| 385 | * fault a range with specific flags. For the latter one it is a |
| 386 | * waste to have the user pre-fill the pfn arrays with a default |
| 387 | * flags value. |
| 388 | */ |
| 389 | pfns = (pfns & range->pfn_flags_mask) | range->default_flags; |
| 390 | |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 391 | /* We aren't ask to do anything ... */ |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 392 | if (!(pfns & range->flags[HMM_PFN_VALID])) |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 393 | return; |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 394 | /* If this is device memory than only fault if explicitly requested */ |
| 395 | if ((cpu_flags & range->flags[HMM_PFN_DEVICE_PRIVATE])) { |
| 396 | /* Do we fault on device memory ? */ |
| 397 | if (pfns & range->flags[HMM_PFN_DEVICE_PRIVATE]) { |
| 398 | *write_fault = pfns & range->flags[HMM_PFN_WRITE]; |
| 399 | *fault = true; |
| 400 | } |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 401 | return; |
| 402 | } |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 403 | |
| 404 | /* If CPU page table is not valid then we need to fault */ |
| 405 | *fault = !(cpu_flags & range->flags[HMM_PFN_VALID]); |
| 406 | /* Need to write fault ? */ |
| 407 | if ((pfns & range->flags[HMM_PFN_WRITE]) && |
| 408 | !(cpu_flags & range->flags[HMM_PFN_WRITE])) { |
| 409 | *write_fault = true; |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 410 | *fault = true; |
| 411 | } |
| 412 | } |
| 413 | |
| 414 | static void hmm_range_need_fault(const struct hmm_vma_walk *hmm_vma_walk, |
| 415 | const uint64_t *pfns, unsigned long npages, |
| 416 | uint64_t cpu_flags, bool *fault, |
| 417 | bool *write_fault) |
| 418 | { |
| 419 | unsigned long i; |
| 420 | |
| 421 | if (!hmm_vma_walk->fault) { |
| 422 | *fault = *write_fault = false; |
| 423 | return; |
| 424 | } |
| 425 | |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 426 | *fault = *write_fault = false; |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 427 | for (i = 0; i < npages; ++i) { |
| 428 | hmm_pte_need_fault(hmm_vma_walk, pfns[i], cpu_flags, |
| 429 | fault, write_fault); |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 430 | if ((*write_fault)) |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 431 | return; |
| 432 | } |
| 433 | } |
| 434 | |
| 435 | static int hmm_vma_walk_hole(unsigned long addr, unsigned long end, |
| 436 | struct mm_walk *walk) |
| 437 | { |
| 438 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
| 439 | struct hmm_range *range = hmm_vma_walk->range; |
| 440 | bool fault, write_fault; |
| 441 | unsigned long i, npages; |
| 442 | uint64_t *pfns; |
| 443 | |
| 444 | i = (addr - range->start) >> PAGE_SHIFT; |
| 445 | npages = (end - addr) >> PAGE_SHIFT; |
| 446 | pfns = &range->pfns[i]; |
| 447 | hmm_range_need_fault(hmm_vma_walk, pfns, npages, |
| 448 | 0, &fault, &write_fault); |
| 449 | return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk); |
| 450 | } |
| 451 | |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 452 | static inline uint64_t pmd_to_hmm_pfn_flags(struct hmm_range *range, pmd_t pmd) |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 453 | { |
| 454 | if (pmd_protnone(pmd)) |
| 455 | return 0; |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 456 | return pmd_write(pmd) ? range->flags[HMM_PFN_VALID] | |
| 457 | range->flags[HMM_PFN_WRITE] : |
| 458 | range->flags[HMM_PFN_VALID]; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 459 | } |
| 460 | |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 461 | static inline uint64_t pud_to_hmm_pfn_flags(struct hmm_range *range, pud_t pud) |
| 462 | { |
| 463 | if (!pud_present(pud)) |
| 464 | return 0; |
| 465 | return pud_write(pud) ? range->flags[HMM_PFN_VALID] | |
| 466 | range->flags[HMM_PFN_WRITE] : |
| 467 | range->flags[HMM_PFN_VALID]; |
| 468 | } |
| 469 | |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 470 | static int hmm_vma_handle_pmd(struct mm_walk *walk, |
| 471 | unsigned long addr, |
| 472 | unsigned long end, |
| 473 | uint64_t *pfns, |
| 474 | pmd_t pmd) |
| 475 | { |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 476 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 477 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 478 | struct hmm_range *range = hmm_vma_walk->range; |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 479 | unsigned long pfn, npages, i; |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 480 | bool fault, write_fault; |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 481 | uint64_t cpu_flags; |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 482 | |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 483 | npages = (end - addr) >> PAGE_SHIFT; |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 484 | cpu_flags = pmd_to_hmm_pfn_flags(range, pmd); |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 485 | hmm_range_need_fault(hmm_vma_walk, pfns, npages, cpu_flags, |
| 486 | &fault, &write_fault); |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 487 | |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 488 | if (pmd_protnone(pmd) || fault || write_fault) |
| 489 | return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk); |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 490 | |
| 491 | pfn = pmd_pfn(pmd) + pte_index(addr); |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 492 | for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) { |
| 493 | if (pmd_devmap(pmd)) { |
| 494 | hmm_vma_walk->pgmap = get_dev_pagemap(pfn, |
| 495 | hmm_vma_walk->pgmap); |
| 496 | if (unlikely(!hmm_vma_walk->pgmap)) |
| 497 | return -EBUSY; |
| 498 | } |
Jérôme Glisse | 391aab1 | 2019-05-13 17:20:31 -0700 | [diff] [blame] | 499 | pfns[i] = hmm_device_entry_from_pfn(range, pfn) | cpu_flags; |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 500 | } |
| 501 | if (hmm_vma_walk->pgmap) { |
| 502 | put_dev_pagemap(hmm_vma_walk->pgmap); |
| 503 | hmm_vma_walk->pgmap = NULL; |
| 504 | } |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 505 | hmm_vma_walk->last = end; |
| 506 | return 0; |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 507 | #else |
| 508 | /* If THP is not enabled then we should never reach that code ! */ |
| 509 | return -EINVAL; |
| 510 | #endif |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 511 | } |
| 512 | |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 513 | static inline uint64_t pte_to_hmm_pfn_flags(struct hmm_range *range, pte_t pte) |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 514 | { |
Philip Yang | 789c2af | 2019-05-23 16:32:31 -0400 | [diff] [blame] | 515 | if (pte_none(pte) || !pte_present(pte) || pte_protnone(pte)) |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 516 | return 0; |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 517 | return pte_write(pte) ? range->flags[HMM_PFN_VALID] | |
| 518 | range->flags[HMM_PFN_WRITE] : |
| 519 | range->flags[HMM_PFN_VALID]; |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 520 | } |
| 521 | |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 522 | static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr, |
| 523 | unsigned long end, pmd_t *pmdp, pte_t *ptep, |
| 524 | uint64_t *pfn) |
| 525 | { |
| 526 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 527 | struct hmm_range *range = hmm_vma_walk->range; |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 528 | struct vm_area_struct *vma = walk->vma; |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 529 | bool fault, write_fault; |
| 530 | uint64_t cpu_flags; |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 531 | pte_t pte = *ptep; |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 532 | uint64_t orig_pfn = *pfn; |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 533 | |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 534 | *pfn = range->values[HMM_PFN_NONE]; |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 535 | fault = write_fault = false; |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 536 | |
| 537 | if (pte_none(pte)) { |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 538 | hmm_pte_need_fault(hmm_vma_walk, orig_pfn, 0, |
| 539 | &fault, &write_fault); |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 540 | if (fault || write_fault) |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 541 | goto fault; |
| 542 | return 0; |
| 543 | } |
| 544 | |
| 545 | if (!pte_present(pte)) { |
| 546 | swp_entry_t entry = pte_to_swp_entry(pte); |
| 547 | |
| 548 | if (!non_swap_entry(entry)) { |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 549 | if (fault || write_fault) |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 550 | goto fault; |
| 551 | return 0; |
| 552 | } |
| 553 | |
| 554 | /* |
| 555 | * This is a special swap entry, ignore migration, use |
| 556 | * device and report anything else as error. |
| 557 | */ |
| 558 | if (is_device_private_entry(entry)) { |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 559 | cpu_flags = range->flags[HMM_PFN_VALID] | |
| 560 | range->flags[HMM_PFN_DEVICE_PRIVATE]; |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 561 | cpu_flags |= is_write_device_private_entry(entry) ? |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 562 | range->flags[HMM_PFN_WRITE] : 0; |
| 563 | hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags, |
| 564 | &fault, &write_fault); |
| 565 | if (fault || write_fault) |
| 566 | goto fault; |
Jérôme Glisse | 391aab1 | 2019-05-13 17:20:31 -0700 | [diff] [blame] | 567 | *pfn = hmm_device_entry_from_pfn(range, |
| 568 | swp_offset(entry)); |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 569 | *pfn |= cpu_flags; |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 570 | return 0; |
| 571 | } |
| 572 | |
| 573 | if (is_migration_entry(entry)) { |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 574 | if (fault || write_fault) { |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 575 | pte_unmap(ptep); |
| 576 | hmm_vma_walk->last = addr; |
| 577 | migration_entry_wait(vma->vm_mm, |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 578 | pmdp, addr); |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 579 | return -EBUSY; |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 580 | } |
| 581 | return 0; |
| 582 | } |
| 583 | |
| 584 | /* Report error for everything else */ |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 585 | *pfn = range->values[HMM_PFN_ERROR]; |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 586 | return -EFAULT; |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 587 | } else { |
| 588 | cpu_flags = pte_to_hmm_pfn_flags(range, pte); |
| 589 | hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags, |
| 590 | &fault, &write_fault); |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 591 | } |
| 592 | |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 593 | if (fault || write_fault) |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 594 | goto fault; |
| 595 | |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 596 | if (pte_devmap(pte)) { |
| 597 | hmm_vma_walk->pgmap = get_dev_pagemap(pte_pfn(pte), |
| 598 | hmm_vma_walk->pgmap); |
| 599 | if (unlikely(!hmm_vma_walk->pgmap)) |
| 600 | return -EBUSY; |
| 601 | } else if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) && pte_special(pte)) { |
| 602 | *pfn = range->values[HMM_PFN_SPECIAL]; |
| 603 | return -EFAULT; |
| 604 | } |
| 605 | |
Jérôme Glisse | 391aab1 | 2019-05-13 17:20:31 -0700 | [diff] [blame] | 606 | *pfn = hmm_device_entry_from_pfn(range, pte_pfn(pte)) | cpu_flags; |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 607 | return 0; |
| 608 | |
| 609 | fault: |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 610 | if (hmm_vma_walk->pgmap) { |
| 611 | put_dev_pagemap(hmm_vma_walk->pgmap); |
| 612 | hmm_vma_walk->pgmap = NULL; |
| 613 | } |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 614 | pte_unmap(ptep); |
| 615 | /* Fault any virtual address we were asked to fault */ |
Jérôme Glisse | 2aee09d | 2018-04-10 16:29:02 -0700 | [diff] [blame] | 616 | return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk); |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 617 | } |
| 618 | |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 619 | static int hmm_vma_walk_pmd(pmd_t *pmdp, |
| 620 | unsigned long start, |
| 621 | unsigned long end, |
| 622 | struct mm_walk *walk) |
| 623 | { |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 624 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
| 625 | struct hmm_range *range = hmm_vma_walk->range; |
Jérôme Glisse | d08faca | 2018-10-30 15:04:20 -0700 | [diff] [blame] | 626 | struct vm_area_struct *vma = walk->vma; |
Jérôme Glisse | ff05c0c | 2018-04-10 16:28:38 -0700 | [diff] [blame] | 627 | uint64_t *pfns = range->pfns; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 628 | unsigned long addr = start, i; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 629 | pte_t *ptep; |
Jérôme Glisse | d08faca | 2018-10-30 15:04:20 -0700 | [diff] [blame] | 630 | pmd_t pmd; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 631 | |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 632 | |
| 633 | again: |
Jérôme Glisse | d08faca | 2018-10-30 15:04:20 -0700 | [diff] [blame] | 634 | pmd = READ_ONCE(*pmdp); |
| 635 | if (pmd_none(pmd)) |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 636 | return hmm_vma_walk_hole(start, end, walk); |
| 637 | |
Jérôme Glisse | d08faca | 2018-10-30 15:04:20 -0700 | [diff] [blame] | 638 | if (pmd_huge(pmd) && (range->vma->vm_flags & VM_HUGETLB)) |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 639 | return hmm_pfns_bad(start, end, walk); |
| 640 | |
Jérôme Glisse | d08faca | 2018-10-30 15:04:20 -0700 | [diff] [blame] | 641 | if (thp_migration_supported() && is_pmd_migration_entry(pmd)) { |
| 642 | bool fault, write_fault; |
| 643 | unsigned long npages; |
| 644 | uint64_t *pfns; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 645 | |
Jérôme Glisse | d08faca | 2018-10-30 15:04:20 -0700 | [diff] [blame] | 646 | i = (addr - range->start) >> PAGE_SHIFT; |
| 647 | npages = (end - addr) >> PAGE_SHIFT; |
| 648 | pfns = &range->pfns[i]; |
| 649 | |
| 650 | hmm_range_need_fault(hmm_vma_walk, pfns, npages, |
| 651 | 0, &fault, &write_fault); |
| 652 | if (fault || write_fault) { |
| 653 | hmm_vma_walk->last = addr; |
| 654 | pmd_migration_entry_wait(vma->vm_mm, pmdp); |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 655 | return -EBUSY; |
Jérôme Glisse | d08faca | 2018-10-30 15:04:20 -0700 | [diff] [blame] | 656 | } |
| 657 | return 0; |
| 658 | } else if (!pmd_present(pmd)) |
| 659 | return hmm_pfns_bad(start, end, walk); |
| 660 | |
| 661 | if (pmd_devmap(pmd) || pmd_trans_huge(pmd)) { |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 662 | /* |
| 663 | * No need to take pmd_lock here, even if some other threads |
| 664 | * is splitting the huge pmd we will get that event through |
| 665 | * mmu_notifier callback. |
| 666 | * |
| 667 | * So just read pmd value and check again its a transparent |
| 668 | * huge or device mapping one and compute corresponding pfn |
| 669 | * values. |
| 670 | */ |
| 671 | pmd = pmd_read_atomic(pmdp); |
| 672 | barrier(); |
| 673 | if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd)) |
| 674 | goto again; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 675 | |
Jérôme Glisse | d08faca | 2018-10-30 15:04:20 -0700 | [diff] [blame] | 676 | i = (addr - range->start) >> PAGE_SHIFT; |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 677 | return hmm_vma_handle_pmd(walk, addr, end, &pfns[i], pmd); |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 678 | } |
| 679 | |
Jérôme Glisse | d08faca | 2018-10-30 15:04:20 -0700 | [diff] [blame] | 680 | /* |
| 681 | * We have handled all the valid case above ie either none, migration, |
| 682 | * huge or transparent huge. At this point either it is a valid pmd |
| 683 | * entry pointing to pte directory or it is a bad pmd that will not |
| 684 | * recover. |
| 685 | */ |
| 686 | if (pmd_bad(pmd)) |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 687 | return hmm_pfns_bad(start, end, walk); |
| 688 | |
| 689 | ptep = pte_offset_map(pmdp, addr); |
Jérôme Glisse | d08faca | 2018-10-30 15:04:20 -0700 | [diff] [blame] | 690 | i = (addr - range->start) >> PAGE_SHIFT; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 691 | for (; addr < end; addr += PAGE_SIZE, ptep++, i++) { |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 692 | int r; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 693 | |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 694 | r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, &pfns[i]); |
| 695 | if (r) { |
| 696 | /* hmm_vma_handle_pte() did unmap pte directory */ |
| 697 | hmm_vma_walk->last = addr; |
| 698 | return r; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 699 | } |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 700 | } |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 701 | if (hmm_vma_walk->pgmap) { |
| 702 | /* |
| 703 | * We do put_dev_pagemap() here and not in hmm_vma_handle_pte() |
| 704 | * so that we can leverage get_dev_pagemap() optimization which |
| 705 | * will not re-take a reference on a pgmap if we already have |
| 706 | * one. |
| 707 | */ |
| 708 | put_dev_pagemap(hmm_vma_walk->pgmap); |
| 709 | hmm_vma_walk->pgmap = NULL; |
| 710 | } |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 711 | pte_unmap(ptep - 1); |
| 712 | |
Jérôme Glisse | 53f5c3f | 2018-04-10 16:28:59 -0700 | [diff] [blame] | 713 | hmm_vma_walk->last = addr; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 714 | return 0; |
| 715 | } |
| 716 | |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 717 | static int hmm_vma_walk_pud(pud_t *pudp, |
| 718 | unsigned long start, |
| 719 | unsigned long end, |
| 720 | struct mm_walk *walk) |
| 721 | { |
| 722 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
| 723 | struct hmm_range *range = hmm_vma_walk->range; |
| 724 | unsigned long addr = start, next; |
| 725 | pmd_t *pmdp; |
| 726 | pud_t pud; |
| 727 | int ret; |
| 728 | |
| 729 | again: |
| 730 | pud = READ_ONCE(*pudp); |
| 731 | if (pud_none(pud)) |
| 732 | return hmm_vma_walk_hole(start, end, walk); |
| 733 | |
| 734 | if (pud_huge(pud) && pud_devmap(pud)) { |
| 735 | unsigned long i, npages, pfn; |
| 736 | uint64_t *pfns, cpu_flags; |
| 737 | bool fault, write_fault; |
| 738 | |
| 739 | if (!pud_present(pud)) |
| 740 | return hmm_vma_walk_hole(start, end, walk); |
| 741 | |
| 742 | i = (addr - range->start) >> PAGE_SHIFT; |
| 743 | npages = (end - addr) >> PAGE_SHIFT; |
| 744 | pfns = &range->pfns[i]; |
| 745 | |
| 746 | cpu_flags = pud_to_hmm_pfn_flags(range, pud); |
| 747 | hmm_range_need_fault(hmm_vma_walk, pfns, npages, |
| 748 | cpu_flags, &fault, &write_fault); |
| 749 | if (fault || write_fault) |
| 750 | return hmm_vma_walk_hole_(addr, end, fault, |
| 751 | write_fault, walk); |
| 752 | |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 753 | pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); |
| 754 | for (i = 0; i < npages; ++i, ++pfn) { |
| 755 | hmm_vma_walk->pgmap = get_dev_pagemap(pfn, |
| 756 | hmm_vma_walk->pgmap); |
| 757 | if (unlikely(!hmm_vma_walk->pgmap)) |
| 758 | return -EBUSY; |
Jérôme Glisse | 391aab1 | 2019-05-13 17:20:31 -0700 | [diff] [blame] | 759 | pfns[i] = hmm_device_entry_from_pfn(range, pfn) | |
| 760 | cpu_flags; |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 761 | } |
| 762 | if (hmm_vma_walk->pgmap) { |
| 763 | put_dev_pagemap(hmm_vma_walk->pgmap); |
| 764 | hmm_vma_walk->pgmap = NULL; |
| 765 | } |
| 766 | hmm_vma_walk->last = end; |
| 767 | return 0; |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 768 | } |
| 769 | |
| 770 | split_huge_pud(walk->vma, pudp, addr); |
| 771 | if (pud_none(*pudp)) |
| 772 | goto again; |
| 773 | |
| 774 | pmdp = pmd_offset(pudp, addr); |
| 775 | do { |
| 776 | next = pmd_addr_end(addr, end); |
| 777 | ret = hmm_vma_walk_pmd(pmdp, addr, next, walk); |
| 778 | if (ret) |
| 779 | return ret; |
| 780 | } while (pmdp++, addr = next, addr != end); |
| 781 | |
| 782 | return 0; |
| 783 | } |
| 784 | |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 785 | static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask, |
| 786 | unsigned long start, unsigned long end, |
| 787 | struct mm_walk *walk) |
| 788 | { |
| 789 | #ifdef CONFIG_HUGETLB_PAGE |
| 790 | unsigned long addr = start, i, pfn, mask, size, pfn_inc; |
| 791 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
| 792 | struct hmm_range *range = hmm_vma_walk->range; |
| 793 | struct vm_area_struct *vma = walk->vma; |
| 794 | struct hstate *h = hstate_vma(vma); |
| 795 | uint64_t orig_pfn, cpu_flags; |
| 796 | bool fault, write_fault; |
| 797 | spinlock_t *ptl; |
| 798 | pte_t entry; |
| 799 | int ret = 0; |
| 800 | |
| 801 | size = 1UL << huge_page_shift(h); |
| 802 | mask = size - 1; |
| 803 | if (range->page_shift != PAGE_SHIFT) { |
| 804 | /* Make sure we are looking at full page. */ |
| 805 | if (start & mask) |
| 806 | return -EINVAL; |
| 807 | if (end < (start + size)) |
| 808 | return -EINVAL; |
| 809 | pfn_inc = size >> PAGE_SHIFT; |
| 810 | } else { |
| 811 | pfn_inc = 1; |
| 812 | size = PAGE_SIZE; |
| 813 | } |
| 814 | |
| 815 | |
| 816 | ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte); |
| 817 | entry = huge_ptep_get(pte); |
| 818 | |
| 819 | i = (start - range->start) >> range->page_shift; |
| 820 | orig_pfn = range->pfns[i]; |
| 821 | range->pfns[i] = range->values[HMM_PFN_NONE]; |
| 822 | cpu_flags = pte_to_hmm_pfn_flags(range, entry); |
| 823 | fault = write_fault = false; |
| 824 | hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags, |
| 825 | &fault, &write_fault); |
| 826 | if (fault || write_fault) { |
| 827 | ret = -ENOENT; |
| 828 | goto unlock; |
| 829 | } |
| 830 | |
| 831 | pfn = pte_pfn(entry) + ((start & mask) >> range->page_shift); |
| 832 | for (; addr < end; addr += size, i++, pfn += pfn_inc) |
Jérôme Glisse | 391aab1 | 2019-05-13 17:20:31 -0700 | [diff] [blame] | 833 | range->pfns[i] = hmm_device_entry_from_pfn(range, pfn) | |
| 834 | cpu_flags; |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 835 | hmm_vma_walk->last = end; |
| 836 | |
| 837 | unlock: |
| 838 | spin_unlock(ptl); |
| 839 | |
| 840 | if (ret == -ENOENT) |
| 841 | return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk); |
| 842 | |
| 843 | return ret; |
| 844 | #else /* CONFIG_HUGETLB_PAGE */ |
| 845 | return -EINVAL; |
| 846 | #endif |
| 847 | } |
| 848 | |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 849 | static void hmm_pfns_clear(struct hmm_range *range, |
| 850 | uint64_t *pfns, |
Jérôme Glisse | 33cd47d | 2018-04-10 16:28:54 -0700 | [diff] [blame] | 851 | unsigned long addr, |
| 852 | unsigned long end) |
| 853 | { |
| 854 | for (; addr < end; addr += PAGE_SIZE, pfns++) |
Jérôme Glisse | f88a1e9 | 2018-04-10 16:29:06 -0700 | [diff] [blame] | 855 | *pfns = range->values[HMM_PFN_NONE]; |
Jérôme Glisse | 33cd47d | 2018-04-10 16:28:54 -0700 | [diff] [blame] | 856 | } |
| 857 | |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 858 | /* |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 859 | * hmm_range_register() - start tracking change to CPU page table over a range |
Jérôme Glisse | 25f23a0 | 2019-05-13 17:19:55 -0700 | [diff] [blame] | 860 | * @range: range |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 861 | * @mm: the mm struct for the range of virtual address |
| 862 | * @start: start virtual address (inclusive) |
| 863 | * @end: end virtual address (exclusive) |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 864 | * @page_shift: expect page shift for the range |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 865 | * Returns 0 on success, -EFAULT if the address space is no longer valid |
Jérôme Glisse | 25f23a0 | 2019-05-13 17:19:55 -0700 | [diff] [blame] | 866 | * |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 867 | * Track updates to the CPU page table see include/linux/hmm.h |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 868 | */ |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 869 | int hmm_range_register(struct hmm_range *range, |
Jason Gunthorpe | e36acfe | 2019-05-23 09:41:19 -0300 | [diff] [blame] | 870 | struct hmm_mirror *mirror, |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 871 | unsigned long start, |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 872 | unsigned long end, |
| 873 | unsigned page_shift) |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 874 | { |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 875 | unsigned long mask = ((1UL << page_shift) - 1UL); |
Jason Gunthorpe | e36acfe | 2019-05-23 09:41:19 -0300 | [diff] [blame] | 876 | struct hmm *hmm = mirror->hmm; |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 877 | unsigned long flags; |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 878 | |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 879 | range->valid = false; |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 880 | range->hmm = NULL; |
| 881 | |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 882 | if ((start & mask) || (end & mask)) |
| 883 | return -EINVAL; |
| 884 | if (start >= end) |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 885 | return -EINVAL; |
| 886 | |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 887 | range->page_shift = page_shift; |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 888 | range->start = start; |
| 889 | range->end = end; |
| 890 | |
Jason Gunthorpe | 47f2459 | 2019-05-23 11:08:28 -0300 | [diff] [blame] | 891 | /* Prevent hmm_release() from running while the range is valid */ |
| 892 | if (!mmget_not_zero(hmm->mm)) |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 893 | return -EFAULT; |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 894 | |
Ralph Campbell | 085ea25 | 2019-05-06 16:29:39 -0700 | [diff] [blame] | 895 | /* Initialize range to track CPU page table updates. */ |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 896 | spin_lock_irqsave(&hmm->ranges_lock, flags); |
Jérôme Glisse | 86586a4 | 2018-04-10 16:28:34 -0700 | [diff] [blame] | 897 | |
Ralph Campbell | 085ea25 | 2019-05-06 16:29:39 -0700 | [diff] [blame] | 898 | range->hmm = hmm; |
Jason Gunthorpe | e36acfe | 2019-05-23 09:41:19 -0300 | [diff] [blame] | 899 | kref_get(&hmm->kref); |
Jason Gunthorpe | 157816f | 2019-05-23 11:43:43 -0300 | [diff] [blame] | 900 | list_add(&range->list, &hmm->ranges); |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 901 | |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 902 | /* |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 903 | * If there are any concurrent notifiers we have to wait for them for |
| 904 | * the range to be valid (see hmm_range_wait_until_valid()). |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 905 | */ |
Ralph Campbell | 085ea25 | 2019-05-06 16:29:39 -0700 | [diff] [blame] | 906 | if (!hmm->notifiers) |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 907 | range->valid = true; |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 908 | spin_unlock_irqrestore(&hmm->ranges_lock, flags); |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 909 | |
| 910 | return 0; |
| 911 | } |
| 912 | EXPORT_SYMBOL(hmm_range_register); |
| 913 | |
| 914 | /* |
| 915 | * hmm_range_unregister() - stop tracking change to CPU page table over a range |
| 916 | * @range: range |
| 917 | * |
| 918 | * Range struct is used to track updates to the CPU page table after a call to |
| 919 | * hmm_range_register(). See include/linux/hmm.h for how to use it. |
| 920 | */ |
| 921 | void hmm_range_unregister(struct hmm_range *range) |
| 922 | { |
Ralph Campbell | 085ea25 | 2019-05-06 16:29:39 -0700 | [diff] [blame] | 923 | struct hmm *hmm = range->hmm; |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 924 | unsigned long flags; |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 925 | |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 926 | spin_lock_irqsave(&hmm->ranges_lock, flags); |
Jason Gunthorpe | 47f2459 | 2019-05-23 11:08:28 -0300 | [diff] [blame] | 927 | list_del_init(&range->list); |
Jason Gunthorpe | 5a136b4 | 2019-06-07 12:10:33 -0300 | [diff] [blame] | 928 | spin_unlock_irqrestore(&hmm->ranges_lock, flags); |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 929 | |
| 930 | /* Drop reference taken by hmm_range_register() */ |
Jason Gunthorpe | 47f2459 | 2019-05-23 11:08:28 -0300 | [diff] [blame] | 931 | mmput(hmm->mm); |
Ralph Campbell | 085ea25 | 2019-05-06 16:29:39 -0700 | [diff] [blame] | 932 | hmm_put(hmm); |
Jason Gunthorpe | 2dcc3eb | 2019-05-23 11:40:24 -0300 | [diff] [blame] | 933 | |
| 934 | /* |
| 935 | * The range is now invalid and the ref on the hmm is dropped, so |
| 936 | * poison the pointer. Leave other fields in place, for the caller's |
| 937 | * use. |
| 938 | */ |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 939 | range->valid = false; |
Jason Gunthorpe | 2dcc3eb | 2019-05-23 11:40:24 -0300 | [diff] [blame] | 940 | memset(&range->hmm, POISON_INUSE, sizeof(range->hmm)); |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 941 | } |
| 942 | EXPORT_SYMBOL(hmm_range_unregister); |
| 943 | |
| 944 | /* |
| 945 | * hmm_range_snapshot() - snapshot CPU page table for a range |
| 946 | * @range: range |
Ralph Campbell | 085ea25 | 2019-05-06 16:29:39 -0700 | [diff] [blame] | 947 | * Return: -EINVAL if invalid argument, -ENOMEM out of memory, -EPERM invalid |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 948 | * permission (for instance asking for write and range is read only), |
| 949 | * -EAGAIN if you need to retry, -EFAULT invalid (ie either no valid |
| 950 | * vma or it is illegal to access that range), number of valid pages |
| 951 | * in range->pfns[] (from range start address). |
| 952 | * |
| 953 | * This snapshots the CPU page table for a range of virtual addresses. Snapshot |
| 954 | * validity is tracked by range struct. See in include/linux/hmm.h for example |
| 955 | * on how to use. |
| 956 | */ |
| 957 | long hmm_range_snapshot(struct hmm_range *range) |
| 958 | { |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 959 | const unsigned long device_vma = VM_IO | VM_PFNMAP | VM_MIXEDMAP; |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 960 | unsigned long start = range->start, end; |
| 961 | struct hmm_vma_walk hmm_vma_walk; |
| 962 | struct hmm *hmm = range->hmm; |
| 963 | struct vm_area_struct *vma; |
| 964 | struct mm_walk mm_walk; |
| 965 | |
Jason Gunthorpe | 47f2459 | 2019-05-23 11:08:28 -0300 | [diff] [blame] | 966 | lockdep_assert_held(&hmm->mm->mmap_sem); |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 967 | do { |
| 968 | /* If range is no longer valid force retry. */ |
| 969 | if (!range->valid) |
| 970 | return -EAGAIN; |
| 971 | |
| 972 | vma = find_vma(hmm->mm, start); |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 973 | if (vma == NULL || (vma->vm_flags & device_vma)) |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 974 | return -EFAULT; |
| 975 | |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 976 | if (is_vm_hugetlb_page(vma)) { |
Jason Gunthorpe | 1c2308f0 | 2019-05-27 17:02:21 -0300 | [diff] [blame] | 977 | if (huge_page_shift(hstate_vma(vma)) != |
| 978 | range->page_shift && |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 979 | range->page_shift != PAGE_SHIFT) |
| 980 | return -EINVAL; |
| 981 | } else { |
| 982 | if (range->page_shift != PAGE_SHIFT) |
| 983 | return -EINVAL; |
| 984 | } |
| 985 | |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 986 | if (!(vma->vm_flags & VM_READ)) { |
| 987 | /* |
| 988 | * If vma do not allow read access, then assume that it |
| 989 | * does not allow write access, either. HMM does not |
| 990 | * support architecture that allow write without read. |
| 991 | */ |
| 992 | hmm_pfns_clear(range, range->pfns, |
| 993 | range->start, range->end); |
| 994 | return -EPERM; |
| 995 | } |
| 996 | |
| 997 | range->vma = vma; |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 998 | hmm_vma_walk.pgmap = NULL; |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 999 | hmm_vma_walk.last = start; |
| 1000 | hmm_vma_walk.fault = false; |
| 1001 | hmm_vma_walk.range = range; |
| 1002 | mm_walk.private = &hmm_vma_walk; |
| 1003 | end = min(range->end, vma->vm_end); |
| 1004 | |
| 1005 | mm_walk.vma = vma; |
| 1006 | mm_walk.mm = vma->vm_mm; |
| 1007 | mm_walk.pte_entry = NULL; |
| 1008 | mm_walk.test_walk = NULL; |
| 1009 | mm_walk.hugetlb_entry = NULL; |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 1010 | mm_walk.pud_entry = hmm_vma_walk_pud; |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 1011 | mm_walk.pmd_entry = hmm_vma_walk_pmd; |
| 1012 | mm_walk.pte_hole = hmm_vma_walk_hole; |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 1013 | mm_walk.hugetlb_entry = hmm_vma_walk_hugetlb_entry; |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 1014 | |
| 1015 | walk_page_range(start, end, &mm_walk); |
| 1016 | start = end; |
| 1017 | } while (start < range->end); |
| 1018 | |
Jérôme Glisse | 25f23a0 | 2019-05-13 17:19:55 -0700 | [diff] [blame] | 1019 | return (hmm_vma_walk.last - range->start) >> PAGE_SHIFT; |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 1020 | } |
Jérôme Glisse | 25f23a0 | 2019-05-13 17:19:55 -0700 | [diff] [blame] | 1021 | EXPORT_SYMBOL(hmm_range_snapshot); |
Jérôme Glisse | da4c3c7 | 2017-09-08 16:11:31 -0700 | [diff] [blame] | 1022 | |
| 1023 | /* |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 1024 | * hmm_range_fault() - try to fault some address in a virtual address range |
Jérôme Glisse | 08232a4 | 2018-04-10 16:28:30 -0700 | [diff] [blame] | 1025 | * @range: range being faulted |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 1026 | * @block: allow blocking on fault (if true it sleeps and do not drop mmap_sem) |
Ralph Campbell | 085ea25 | 2019-05-06 16:29:39 -0700 | [diff] [blame] | 1027 | * Return: number of valid pages in range->pfns[] (from range start |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 1028 | * address). This may be zero. If the return value is negative, |
| 1029 | * then one of the following values may be returned: |
| 1030 | * |
| 1031 | * -EINVAL invalid arguments or mm or virtual address are in an |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 1032 | * invalid vma (for instance device file vma). |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 1033 | * -ENOMEM: Out of memory. |
| 1034 | * -EPERM: Invalid permission (for instance asking for write and |
| 1035 | * range is read only). |
| 1036 | * -EAGAIN: If you need to retry and mmap_sem was drop. This can only |
| 1037 | * happens if block argument is false. |
| 1038 | * -EBUSY: If the the range is being invalidated and you should wait |
| 1039 | * for invalidation to finish. |
| 1040 | * -EFAULT: Invalid (ie either no valid vma or it is illegal to access |
| 1041 | * that range), number of valid pages in range->pfns[] (from |
| 1042 | * range start address). |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 1043 | * |
| 1044 | * This is similar to a regular CPU page fault except that it will not trigger |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 1045 | * any memory migration if the memory being faulted is not accessible by CPUs |
| 1046 | * and caller does not ask for migration. |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 1047 | * |
Jérôme Glisse | ff05c0c | 2018-04-10 16:28:38 -0700 | [diff] [blame] | 1048 | * On error, for one virtual address in the range, the function will mark the |
| 1049 | * corresponding HMM pfn entry with an error flag. |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 1050 | */ |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 1051 | long hmm_range_fault(struct hmm_range *range, bool block) |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 1052 | { |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 1053 | const unsigned long device_vma = VM_IO | VM_PFNMAP | VM_MIXEDMAP; |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 1054 | unsigned long start = range->start, end; |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 1055 | struct hmm_vma_walk hmm_vma_walk; |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 1056 | struct hmm *hmm = range->hmm; |
| 1057 | struct vm_area_struct *vma; |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 1058 | struct mm_walk mm_walk; |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 1059 | int ret; |
| 1060 | |
Jason Gunthorpe | 47f2459 | 2019-05-23 11:08:28 -0300 | [diff] [blame] | 1061 | lockdep_assert_held(&hmm->mm->mmap_sem); |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 1062 | |
| 1063 | do { |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 1064 | /* If range is no longer valid force retry. */ |
| 1065 | if (!range->valid) { |
| 1066 | up_read(&hmm->mm->mmap_sem); |
| 1067 | return -EAGAIN; |
| 1068 | } |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 1069 | |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 1070 | vma = find_vma(hmm->mm, start); |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 1071 | if (vma == NULL || (vma->vm_flags & device_vma)) |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 1072 | return -EFAULT; |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 1073 | |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 1074 | if (is_vm_hugetlb_page(vma)) { |
| 1075 | if (huge_page_shift(hstate_vma(vma)) != |
| 1076 | range->page_shift && |
| 1077 | range->page_shift != PAGE_SHIFT) |
| 1078 | return -EINVAL; |
| 1079 | } else { |
| 1080 | if (range->page_shift != PAGE_SHIFT) |
| 1081 | return -EINVAL; |
| 1082 | } |
| 1083 | |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 1084 | if (!(vma->vm_flags & VM_READ)) { |
| 1085 | /* |
| 1086 | * If vma do not allow read access, then assume that it |
| 1087 | * does not allow write access, either. HMM does not |
| 1088 | * support architecture that allow write without read. |
| 1089 | */ |
| 1090 | hmm_pfns_clear(range, range->pfns, |
| 1091 | range->start, range->end); |
| 1092 | return -EPERM; |
| 1093 | } |
| 1094 | |
| 1095 | range->vma = vma; |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 1096 | hmm_vma_walk.pgmap = NULL; |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 1097 | hmm_vma_walk.last = start; |
| 1098 | hmm_vma_walk.fault = true; |
| 1099 | hmm_vma_walk.block = block; |
| 1100 | hmm_vma_walk.range = range; |
| 1101 | mm_walk.private = &hmm_vma_walk; |
| 1102 | end = min(range->end, vma->vm_end); |
| 1103 | |
| 1104 | mm_walk.vma = vma; |
| 1105 | mm_walk.mm = vma->vm_mm; |
| 1106 | mm_walk.pte_entry = NULL; |
| 1107 | mm_walk.test_walk = NULL; |
| 1108 | mm_walk.hugetlb_entry = NULL; |
Jérôme Glisse | 992de9a | 2019-05-13 17:20:21 -0700 | [diff] [blame] | 1109 | mm_walk.pud_entry = hmm_vma_walk_pud; |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 1110 | mm_walk.pmd_entry = hmm_vma_walk_pmd; |
| 1111 | mm_walk.pte_hole = hmm_vma_walk_hole; |
Jérôme Glisse | 63d5066 | 2019-05-13 17:20:18 -0700 | [diff] [blame] | 1112 | mm_walk.hugetlb_entry = hmm_vma_walk_hugetlb_entry; |
Jérôme Glisse | a3e0d41 | 2019-05-13 17:20:01 -0700 | [diff] [blame] | 1113 | |
| 1114 | do { |
| 1115 | ret = walk_page_range(start, end, &mm_walk); |
| 1116 | start = hmm_vma_walk.last; |
| 1117 | |
| 1118 | /* Keep trying while the range is valid. */ |
| 1119 | } while (ret == -EBUSY && range->valid); |
| 1120 | |
| 1121 | if (ret) { |
| 1122 | unsigned long i; |
| 1123 | |
| 1124 | i = (hmm_vma_walk.last - range->start) >> PAGE_SHIFT; |
| 1125 | hmm_pfns_clear(range, &range->pfns[i], |
| 1126 | hmm_vma_walk.last, range->end); |
| 1127 | return ret; |
| 1128 | } |
| 1129 | start = end; |
| 1130 | |
| 1131 | } while (start < range->end); |
Jérôme Glisse | 704f3f2 | 2019-05-13 17:19:48 -0700 | [diff] [blame] | 1132 | |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 1133 | return (hmm_vma_walk.last - range->start) >> PAGE_SHIFT; |
Jérôme Glisse | 74eee18 | 2017-09-08 16:11:35 -0700 | [diff] [blame] | 1134 | } |
Jérôme Glisse | 7323161 | 2019-05-13 17:19:58 -0700 | [diff] [blame] | 1135 | EXPORT_SYMBOL(hmm_range_fault); |
Jérôme Glisse | 55c0ece | 2019-05-13 17:20:28 -0700 | [diff] [blame] | 1136 | |
| 1137 | /** |
| 1138 | * hmm_range_dma_map() - hmm_range_fault() and dma map page all in one. |
| 1139 | * @range: range being faulted |
| 1140 | * @device: device against to dma map page to |
| 1141 | * @daddrs: dma address of mapped pages |
| 1142 | * @block: allow blocking on fault (if true it sleeps and do not drop mmap_sem) |
Ralph Campbell | 085ea25 | 2019-05-06 16:29:39 -0700 | [diff] [blame] | 1143 | * Return: number of pages mapped on success, -EAGAIN if mmap_sem have been |
Jérôme Glisse | 55c0ece | 2019-05-13 17:20:28 -0700 | [diff] [blame] | 1144 | * drop and you need to try again, some other error value otherwise |
| 1145 | * |
| 1146 | * Note same usage pattern as hmm_range_fault(). |
| 1147 | */ |
| 1148 | long hmm_range_dma_map(struct hmm_range *range, |
| 1149 | struct device *device, |
| 1150 | dma_addr_t *daddrs, |
| 1151 | bool block) |
| 1152 | { |
| 1153 | unsigned long i, npages, mapped; |
| 1154 | long ret; |
| 1155 | |
| 1156 | ret = hmm_range_fault(range, block); |
| 1157 | if (ret <= 0) |
| 1158 | return ret ? ret : -EBUSY; |
| 1159 | |
| 1160 | npages = (range->end - range->start) >> PAGE_SHIFT; |
| 1161 | for (i = 0, mapped = 0; i < npages; ++i) { |
| 1162 | enum dma_data_direction dir = DMA_TO_DEVICE; |
| 1163 | struct page *page; |
| 1164 | |
| 1165 | /* |
| 1166 | * FIXME need to update DMA API to provide invalid DMA address |
| 1167 | * value instead of a function to test dma address value. This |
| 1168 | * would remove lot of dumb code duplicated accross many arch. |
| 1169 | * |
| 1170 | * For now setting it to 0 here is good enough as the pfns[] |
| 1171 | * value is what is use to check what is valid and what isn't. |
| 1172 | */ |
| 1173 | daddrs[i] = 0; |
| 1174 | |
Jérôme Glisse | 391aab1 | 2019-05-13 17:20:31 -0700 | [diff] [blame] | 1175 | page = hmm_device_entry_to_page(range, range->pfns[i]); |
Jérôme Glisse | 55c0ece | 2019-05-13 17:20:28 -0700 | [diff] [blame] | 1176 | if (page == NULL) |
| 1177 | continue; |
| 1178 | |
| 1179 | /* Check if range is being invalidated */ |
| 1180 | if (!range->valid) { |
| 1181 | ret = -EBUSY; |
| 1182 | goto unmap; |
| 1183 | } |
| 1184 | |
| 1185 | /* If it is read and write than map bi-directional. */ |
| 1186 | if (range->pfns[i] & range->flags[HMM_PFN_WRITE]) |
| 1187 | dir = DMA_BIDIRECTIONAL; |
| 1188 | |
| 1189 | daddrs[i] = dma_map_page(device, page, 0, PAGE_SIZE, dir); |
| 1190 | if (dma_mapping_error(device, daddrs[i])) { |
| 1191 | ret = -EFAULT; |
| 1192 | goto unmap; |
| 1193 | } |
| 1194 | |
| 1195 | mapped++; |
| 1196 | } |
| 1197 | |
| 1198 | return mapped; |
| 1199 | |
| 1200 | unmap: |
| 1201 | for (npages = i, i = 0; (i < npages) && mapped; ++i) { |
| 1202 | enum dma_data_direction dir = DMA_TO_DEVICE; |
| 1203 | struct page *page; |
| 1204 | |
Jérôme Glisse | 391aab1 | 2019-05-13 17:20:31 -0700 | [diff] [blame] | 1205 | page = hmm_device_entry_to_page(range, range->pfns[i]); |
Jérôme Glisse | 55c0ece | 2019-05-13 17:20:28 -0700 | [diff] [blame] | 1206 | if (page == NULL) |
| 1207 | continue; |
| 1208 | |
| 1209 | if (dma_mapping_error(device, daddrs[i])) |
| 1210 | continue; |
| 1211 | |
| 1212 | /* If it is read and write than map bi-directional. */ |
| 1213 | if (range->pfns[i] & range->flags[HMM_PFN_WRITE]) |
| 1214 | dir = DMA_BIDIRECTIONAL; |
| 1215 | |
| 1216 | dma_unmap_page(device, daddrs[i], PAGE_SIZE, dir); |
| 1217 | mapped--; |
| 1218 | } |
| 1219 | |
| 1220 | return ret; |
| 1221 | } |
| 1222 | EXPORT_SYMBOL(hmm_range_dma_map); |
| 1223 | |
| 1224 | /** |
| 1225 | * hmm_range_dma_unmap() - unmap range of that was map with hmm_range_dma_map() |
| 1226 | * @range: range being unmapped |
| 1227 | * @vma: the vma against which the range (optional) |
| 1228 | * @device: device against which dma map was done |
| 1229 | * @daddrs: dma address of mapped pages |
| 1230 | * @dirty: dirty page if it had the write flag set |
Ralph Campbell | 085ea25 | 2019-05-06 16:29:39 -0700 | [diff] [blame] | 1231 | * Return: number of page unmapped on success, -EINVAL otherwise |
Jérôme Glisse | 55c0ece | 2019-05-13 17:20:28 -0700 | [diff] [blame] | 1232 | * |
| 1233 | * Note that caller MUST abide by mmu notifier or use HMM mirror and abide |
| 1234 | * to the sync_cpu_device_pagetables() callback so that it is safe here to |
| 1235 | * call set_page_dirty(). Caller must also take appropriate locks to avoid |
| 1236 | * concurrent mmu notifier or sync_cpu_device_pagetables() to make progress. |
| 1237 | */ |
| 1238 | long hmm_range_dma_unmap(struct hmm_range *range, |
| 1239 | struct vm_area_struct *vma, |
| 1240 | struct device *device, |
| 1241 | dma_addr_t *daddrs, |
| 1242 | bool dirty) |
| 1243 | { |
| 1244 | unsigned long i, npages; |
| 1245 | long cpages = 0; |
| 1246 | |
| 1247 | /* Sanity check. */ |
| 1248 | if (range->end <= range->start) |
| 1249 | return -EINVAL; |
| 1250 | if (!daddrs) |
| 1251 | return -EINVAL; |
| 1252 | if (!range->pfns) |
| 1253 | return -EINVAL; |
| 1254 | |
| 1255 | npages = (range->end - range->start) >> PAGE_SHIFT; |
| 1256 | for (i = 0; i < npages; ++i) { |
| 1257 | enum dma_data_direction dir = DMA_TO_DEVICE; |
| 1258 | struct page *page; |
| 1259 | |
Jérôme Glisse | 391aab1 | 2019-05-13 17:20:31 -0700 | [diff] [blame] | 1260 | page = hmm_device_entry_to_page(range, range->pfns[i]); |
Jérôme Glisse | 55c0ece | 2019-05-13 17:20:28 -0700 | [diff] [blame] | 1261 | if (page == NULL) |
| 1262 | continue; |
| 1263 | |
| 1264 | /* If it is read and write than map bi-directional. */ |
| 1265 | if (range->pfns[i] & range->flags[HMM_PFN_WRITE]) { |
| 1266 | dir = DMA_BIDIRECTIONAL; |
| 1267 | |
| 1268 | /* |
| 1269 | * See comments in function description on why it is |
| 1270 | * safe here to call set_page_dirty() |
| 1271 | */ |
| 1272 | if (dirty) |
| 1273 | set_page_dirty(page); |
| 1274 | } |
| 1275 | |
| 1276 | /* Unmap and clear pfns/dma address */ |
| 1277 | dma_unmap_page(device, daddrs[i], PAGE_SIZE, dir); |
| 1278 | range->pfns[i] = range->values[HMM_PFN_NONE]; |
| 1279 | /* FIXME see comments in hmm_vma_dma_map() */ |
| 1280 | daddrs[i] = 0; |
| 1281 | cpages++; |
| 1282 | } |
| 1283 | |
| 1284 | return cpages; |
| 1285 | } |
| 1286 | EXPORT_SYMBOL(hmm_range_dma_unmap); |