Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 1 | #define pr_fmt(fmt) "Hyper-V: " fmt |
| 2 | |
| 3 | #include <linux/hyperv.h> |
| 4 | #include <linux/log2.h> |
| 5 | #include <linux/slab.h> |
| 6 | #include <linux/types.h> |
| 7 | |
| 8 | #include <asm/fpu/api.h> |
| 9 | #include <asm/mshyperv.h> |
| 10 | #include <asm/msr.h> |
| 11 | #include <asm/tlbflush.h> |
Peter Zijlstra | 48a8b97 | 2018-08-22 17:30:16 +0200 | [diff] [blame] | 12 | #include <asm/tlb.h> |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 13 | |
Vitaly Kuznetsov | 773b79f | 2017-08-02 18:09:21 +0200 | [diff] [blame] | 14 | #define CREATE_TRACE_POINTS |
| 15 | #include <asm/trace/hyperv.h> |
| 16 | |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 17 | /* Each gva in gva_list encodes up to 4096 pages to flush */ |
| 18 | #define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE) |
| 19 | |
Vitaly Kuznetsov | 0e4c88f | 2018-06-21 15:32:38 +0200 | [diff] [blame] | 20 | static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus, |
| 21 | const struct flush_tlb_info *info); |
Vitaly Kuznetsov | 628f54c | 2017-08-02 18:09:20 +0200 | [diff] [blame] | 22 | |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 23 | /* |
| 24 | * Fills in gva_list starting from offset. Returns the number of items added. |
| 25 | */ |
| 26 | static inline int fill_gva_list(u64 gva_list[], int offset, |
| 27 | unsigned long start, unsigned long end) |
| 28 | { |
| 29 | int gva_n = offset; |
| 30 | unsigned long cur = start, diff; |
| 31 | |
| 32 | do { |
| 33 | diff = end > cur ? end - cur : 0; |
| 34 | |
| 35 | gva_list[gva_n] = cur & PAGE_MASK; |
| 36 | /* |
| 37 | * Lower 12 bits encode the number of additional |
| 38 | * pages to flush (in addition to the 'cur' page). |
| 39 | */ |
Tianyu Lan | 4030b4c | 2019-09-02 20:41:43 +0800 | [diff] [blame] | 40 | if (diff >= HV_TLB_FLUSH_UNIT) { |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 41 | gva_list[gva_n] |= ~PAGE_MASK; |
Tianyu Lan | 4030b4c | 2019-09-02 20:41:43 +0800 | [diff] [blame] | 42 | cur += HV_TLB_FLUSH_UNIT; |
| 43 | } else if (diff) { |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 44 | gva_list[gva_n] |= (diff - 1) >> PAGE_SHIFT; |
Tianyu Lan | 4030b4c | 2019-09-02 20:41:43 +0800 | [diff] [blame] | 45 | cur = end; |
| 46 | } |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 47 | |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 48 | gva_n++; |
| 49 | |
| 50 | } while (cur < end); |
| 51 | |
| 52 | return gva_n - offset; |
| 53 | } |
| 54 | |
| 55 | static void hyperv_flush_tlb_others(const struct cpumask *cpus, |
| 56 | const struct flush_tlb_info *info) |
| 57 | { |
| 58 | int cpu, vcpu, gva_n, max_gvas; |
Vitaly Kuznetsov | c9c92be | 2018-05-16 17:21:24 +0200 | [diff] [blame] | 59 | struct hv_tlb_flush **flush_pcpu; |
| 60 | struct hv_tlb_flush *flush; |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 61 | u64 status = U64_MAX; |
| 62 | unsigned long flags; |
| 63 | |
Vitaly Kuznetsov | 773b79f | 2017-08-02 18:09:21 +0200 | [diff] [blame] | 64 | trace_hyperv_mmu_flush_tlb_others(cpus, info); |
| 65 | |
K. Y. Srinivasan | 9a2d78e | 2018-05-16 14:53:34 -0700 | [diff] [blame] | 66 | if (!hv_hypercall_pg) |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 67 | goto do_native; |
| 68 | |
| 69 | if (cpumask_empty(cpus)) |
| 70 | return; |
| 71 | |
| 72 | local_irq_save(flags); |
| 73 | |
Vitaly Kuznetsov | c9c92be | 2018-05-16 17:21:24 +0200 | [diff] [blame] | 74 | flush_pcpu = (struct hv_tlb_flush **) |
K. Y. Srinivasan | 9a2d78e | 2018-05-16 14:53:34 -0700 | [diff] [blame] | 75 | this_cpu_ptr(hyperv_pcpu_input_arg); |
Vitaly Kuznetsov | 60d73a7 | 2017-10-05 13:39:24 +0200 | [diff] [blame] | 76 | |
| 77 | flush = *flush_pcpu; |
| 78 | |
| 79 | if (unlikely(!flush)) { |
| 80 | local_irq_restore(flags); |
| 81 | goto do_native; |
| 82 | } |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 83 | |
| 84 | if (info->mm) { |
Vitaly Kuznetsov | 617ab45 | 2018-01-24 11:36:29 +0100 | [diff] [blame] | 85 | /* |
| 86 | * AddressSpace argument must match the CR3 with PCID bits |
| 87 | * stripped out. |
| 88 | */ |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 89 | flush->address_space = virt_to_phys(info->mm->pgd); |
Vitaly Kuznetsov | 617ab45 | 2018-01-24 11:36:29 +0100 | [diff] [blame] | 90 | flush->address_space &= CR3_ADDR_MASK; |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 91 | flush->flags = 0; |
| 92 | } else { |
| 93 | flush->address_space = 0; |
| 94 | flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES; |
| 95 | } |
| 96 | |
| 97 | flush->processor_mask = 0; |
| 98 | if (cpumask_equal(cpus, cpu_present_mask)) { |
| 99 | flush->flags |= HV_FLUSH_ALL_PROCESSORS; |
| 100 | } else { |
Vitaly Kuznetsov | 0e4c88f | 2018-06-21 15:32:38 +0200 | [diff] [blame] | 101 | /* |
| 102 | * From the supplied CPU set we need to figure out if we can get |
| 103 | * away with cheaper HVCALL_FLUSH_VIRTUAL_ADDRESS_{LIST,SPACE} |
| 104 | * hypercalls. This is possible when the highest VP number in |
| 105 | * the set is < 64. As VP numbers are usually in ascending order |
| 106 | * and match Linux CPU ids, here is an optimization: we check |
| 107 | * the VP number for the highest bit in the supplied set first |
| 108 | * so we can quickly find out if using *_EX hypercalls is a |
| 109 | * must. We will also check all VP numbers when walking the |
| 110 | * supplied CPU set to remain correct in all cases. |
| 111 | */ |
| 112 | if (hv_cpu_number_to_vp_number(cpumask_last(cpus)) >= 64) |
| 113 | goto do_ex_hypercall; |
| 114 | |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 115 | for_each_cpu(cpu, cpus) { |
| 116 | vcpu = hv_cpu_number_to_vp_number(cpu); |
Vitaly Kuznetsov | 110d2a7 | 2018-07-09 19:40:12 +0200 | [diff] [blame] | 117 | if (vcpu == VP_INVAL) { |
| 118 | local_irq_restore(flags); |
| 119 | goto do_native; |
| 120 | } |
| 121 | |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 122 | if (vcpu >= 64) |
Vitaly Kuznetsov | 0e4c88f | 2018-06-21 15:32:38 +0200 | [diff] [blame] | 123 | goto do_ex_hypercall; |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 124 | |
| 125 | __set_bit(vcpu, (unsigned long *) |
| 126 | &flush->processor_mask); |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | /* |
| 131 | * We can flush not more than max_gvas with one hypercall. Flush the |
| 132 | * whole address space if we were asked to do more. |
| 133 | */ |
| 134 | max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]); |
| 135 | |
| 136 | if (info->end == TLB_FLUSH_ALL) { |
| 137 | flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY; |
| 138 | status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE, |
| 139 | flush, NULL); |
| 140 | } else if (info->end && |
| 141 | ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) { |
| 142 | status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE, |
| 143 | flush, NULL); |
| 144 | } else { |
| 145 | gva_n = fill_gva_list(flush->gva_list, 0, |
| 146 | info->start, info->end); |
| 147 | status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST, |
| 148 | gva_n, 0, flush, NULL); |
| 149 | } |
Vitaly Kuznetsov | 0e4c88f | 2018-06-21 15:32:38 +0200 | [diff] [blame] | 150 | goto check_status; |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 151 | |
Vitaly Kuznetsov | 0e4c88f | 2018-06-21 15:32:38 +0200 | [diff] [blame] | 152 | do_ex_hypercall: |
| 153 | status = hyperv_flush_tlb_others_ex(cpus, info); |
| 154 | |
| 155 | check_status: |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 156 | local_irq_restore(flags); |
| 157 | |
| 158 | if (!(status & HV_HYPERCALL_RESULT_MASK)) |
| 159 | return; |
| 160 | do_native: |
| 161 | native_flush_tlb_others(cpus, info); |
| 162 | } |
| 163 | |
Vitaly Kuznetsov | 0e4c88f | 2018-06-21 15:32:38 +0200 | [diff] [blame] | 164 | static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus, |
| 165 | const struct flush_tlb_info *info) |
Vitaly Kuznetsov | 628f54c | 2017-08-02 18:09:20 +0200 | [diff] [blame] | 166 | { |
| 167 | int nr_bank = 0, max_gvas, gva_n; |
Vitaly Kuznetsov | c9c92be | 2018-05-16 17:21:24 +0200 | [diff] [blame] | 168 | struct hv_tlb_flush_ex **flush_pcpu; |
| 169 | struct hv_tlb_flush_ex *flush; |
Vitaly Kuznetsov | 0e4c88f | 2018-06-21 15:32:38 +0200 | [diff] [blame] | 170 | u64 status; |
Vitaly Kuznetsov | 628f54c | 2017-08-02 18:09:20 +0200 | [diff] [blame] | 171 | |
Vitaly Kuznetsov | 0e4c88f | 2018-06-21 15:32:38 +0200 | [diff] [blame] | 172 | if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) |
| 173 | return U64_MAX; |
Vitaly Kuznetsov | 628f54c | 2017-08-02 18:09:20 +0200 | [diff] [blame] | 174 | |
Vitaly Kuznetsov | c9c92be | 2018-05-16 17:21:24 +0200 | [diff] [blame] | 175 | flush_pcpu = (struct hv_tlb_flush_ex **) |
K. Y. Srinivasan | 9a2d78e | 2018-05-16 14:53:34 -0700 | [diff] [blame] | 176 | this_cpu_ptr(hyperv_pcpu_input_arg); |
Vitaly Kuznetsov | 60d73a7 | 2017-10-05 13:39:24 +0200 | [diff] [blame] | 177 | |
| 178 | flush = *flush_pcpu; |
| 179 | |
Vitaly Kuznetsov | 628f54c | 2017-08-02 18:09:20 +0200 | [diff] [blame] | 180 | if (info->mm) { |
Vitaly Kuznetsov | 617ab45 | 2018-01-24 11:36:29 +0100 | [diff] [blame] | 181 | /* |
| 182 | * AddressSpace argument must match the CR3 with PCID bits |
| 183 | * stripped out. |
| 184 | */ |
Vitaly Kuznetsov | 628f54c | 2017-08-02 18:09:20 +0200 | [diff] [blame] | 185 | flush->address_space = virt_to_phys(info->mm->pgd); |
Vitaly Kuznetsov | 617ab45 | 2018-01-24 11:36:29 +0100 | [diff] [blame] | 186 | flush->address_space &= CR3_ADDR_MASK; |
Vitaly Kuznetsov | 628f54c | 2017-08-02 18:09:20 +0200 | [diff] [blame] | 187 | flush->flags = 0; |
| 188 | } else { |
| 189 | flush->address_space = 0; |
| 190 | flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES; |
| 191 | } |
| 192 | |
| 193 | flush->hv_vp_set.valid_bank_mask = 0; |
| 194 | |
Vitaly Kuznetsov | 0e4c88f | 2018-06-21 15:32:38 +0200 | [diff] [blame] | 195 | flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K; |
| 196 | nr_bank = cpumask_to_vpset(&(flush->hv_vp_set), cpus); |
Vitaly Kuznetsov | 0f0caa5 | 2018-07-09 19:40:11 +0200 | [diff] [blame] | 197 | if (nr_bank < 0) |
| 198 | return U64_MAX; |
Vitaly Kuznetsov | 628f54c | 2017-08-02 18:09:20 +0200 | [diff] [blame] | 199 | |
| 200 | /* |
| 201 | * We can flush not more than max_gvas with one hypercall. Flush the |
| 202 | * whole address space if we were asked to do more. |
| 203 | */ |
| 204 | max_gvas = |
| 205 | (PAGE_SIZE - sizeof(*flush) - nr_bank * |
| 206 | sizeof(flush->hv_vp_set.bank_contents[0])) / |
| 207 | sizeof(flush->gva_list[0]); |
| 208 | |
| 209 | if (info->end == TLB_FLUSH_ALL) { |
| 210 | flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY; |
| 211 | status = hv_do_rep_hypercall( |
| 212 | HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX, |
Marcelo Henrique Cerri | ab7ff47 | 2017-10-05 10:34:29 -0300 | [diff] [blame] | 213 | 0, nr_bank, flush, NULL); |
Vitaly Kuznetsov | 628f54c | 2017-08-02 18:09:20 +0200 | [diff] [blame] | 214 | } else if (info->end && |
| 215 | ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) { |
| 216 | status = hv_do_rep_hypercall( |
| 217 | HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX, |
Marcelo Henrique Cerri | ab7ff47 | 2017-10-05 10:34:29 -0300 | [diff] [blame] | 218 | 0, nr_bank, flush, NULL); |
Vitaly Kuznetsov | 628f54c | 2017-08-02 18:09:20 +0200 | [diff] [blame] | 219 | } else { |
| 220 | gva_n = fill_gva_list(flush->gva_list, nr_bank, |
| 221 | info->start, info->end); |
| 222 | status = hv_do_rep_hypercall( |
| 223 | HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX, |
Marcelo Henrique Cerri | ab7ff47 | 2017-10-05 10:34:29 -0300 | [diff] [blame] | 224 | gva_n, nr_bank, flush, NULL); |
Vitaly Kuznetsov | 628f54c | 2017-08-02 18:09:20 +0200 | [diff] [blame] | 225 | } |
| 226 | |
Vitaly Kuznetsov | 0e4c88f | 2018-06-21 15:32:38 +0200 | [diff] [blame] | 227 | return status; |
Vitaly Kuznetsov | 628f54c | 2017-08-02 18:09:20 +0200 | [diff] [blame] | 228 | } |
| 229 | |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 230 | void hyperv_setup_mmu_ops(void) |
| 231 | { |
Vitaly Kuznetsov | 628f54c | 2017-08-02 18:09:20 +0200 | [diff] [blame] | 232 | if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED)) |
| 233 | return; |
| 234 | |
Vitaly Kuznetsov | 0e4c88f | 2018-06-21 15:32:38 +0200 | [diff] [blame] | 235 | pr_info("Using hypercall for remote TLB flush\n"); |
Juergen Gross | 5c83511 | 2018-08-28 09:40:19 +0200 | [diff] [blame] | 236 | pv_ops.mmu.flush_tlb_others = hyperv_flush_tlb_others; |
| 237 | pv_ops.mmu.tlb_remove_table = tlb_remove_table; |
Vitaly Kuznetsov | 2ffd9e3 | 2017-08-02 18:09:19 +0200 | [diff] [blame] | 238 | } |