blob: 5208ba49c89a96144ba52d82cabc21ad39a743e8 [file] [log] [blame]
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +02001#define pr_fmt(fmt) "Hyper-V: " fmt
2
3#include <linux/hyperv.h>
4#include <linux/log2.h>
5#include <linux/slab.h>
6#include <linux/types.h>
7
8#include <asm/fpu/api.h>
9#include <asm/mshyperv.h>
10#include <asm/msr.h>
11#include <asm/tlbflush.h>
Peter Zijlstra48a8b972018-08-22 17:30:16 +020012#include <asm/tlb.h>
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +020013
Vitaly Kuznetsov773b79f2017-08-02 18:09:21 +020014#define CREATE_TRACE_POINTS
15#include <asm/trace/hyperv.h>
16
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +020017/* Each gva in gva_list encodes up to 4096 pages to flush */
18#define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE)
19
Vitaly Kuznetsov0e4c88f2018-06-21 15:32:38 +020020static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
21 const struct flush_tlb_info *info);
Vitaly Kuznetsov628f54c2017-08-02 18:09:20 +020022
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +020023/*
24 * Fills in gva_list starting from offset. Returns the number of items added.
25 */
26static inline int fill_gva_list(u64 gva_list[], int offset,
27 unsigned long start, unsigned long end)
28{
29 int gva_n = offset;
30 unsigned long cur = start, diff;
31
32 do {
33 diff = end > cur ? end - cur : 0;
34
35 gva_list[gva_n] = cur & PAGE_MASK;
36 /*
37 * Lower 12 bits encode the number of additional
38 * pages to flush (in addition to the 'cur' page).
39 */
Tianyu Lan4030b4c2019-09-02 20:41:43 +080040 if (diff >= HV_TLB_FLUSH_UNIT) {
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +020041 gva_list[gva_n] |= ~PAGE_MASK;
Tianyu Lan4030b4c2019-09-02 20:41:43 +080042 cur += HV_TLB_FLUSH_UNIT;
43 } else if (diff) {
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +020044 gva_list[gva_n] |= (diff - 1) >> PAGE_SHIFT;
Tianyu Lan4030b4c2019-09-02 20:41:43 +080045 cur = end;
46 }
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +020047
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +020048 gva_n++;
49
50 } while (cur < end);
51
52 return gva_n - offset;
53}
54
55static void hyperv_flush_tlb_others(const struct cpumask *cpus,
56 const struct flush_tlb_info *info)
57{
58 int cpu, vcpu, gva_n, max_gvas;
Vitaly Kuznetsovc9c92be2018-05-16 17:21:24 +020059 struct hv_tlb_flush **flush_pcpu;
60 struct hv_tlb_flush *flush;
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +020061 u64 status = U64_MAX;
62 unsigned long flags;
63
Vitaly Kuznetsov773b79f2017-08-02 18:09:21 +020064 trace_hyperv_mmu_flush_tlb_others(cpus, info);
65
K. Y. Srinivasan9a2d78e2018-05-16 14:53:34 -070066 if (!hv_hypercall_pg)
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +020067 goto do_native;
68
69 if (cpumask_empty(cpus))
70 return;
71
72 local_irq_save(flags);
73
Vitaly Kuznetsovc9c92be2018-05-16 17:21:24 +020074 flush_pcpu = (struct hv_tlb_flush **)
K. Y. Srinivasan9a2d78e2018-05-16 14:53:34 -070075 this_cpu_ptr(hyperv_pcpu_input_arg);
Vitaly Kuznetsov60d73a72017-10-05 13:39:24 +020076
77 flush = *flush_pcpu;
78
79 if (unlikely(!flush)) {
80 local_irq_restore(flags);
81 goto do_native;
82 }
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +020083
84 if (info->mm) {
Vitaly Kuznetsov617ab452018-01-24 11:36:29 +010085 /*
86 * AddressSpace argument must match the CR3 with PCID bits
87 * stripped out.
88 */
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +020089 flush->address_space = virt_to_phys(info->mm->pgd);
Vitaly Kuznetsov617ab452018-01-24 11:36:29 +010090 flush->address_space &= CR3_ADDR_MASK;
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +020091 flush->flags = 0;
92 } else {
93 flush->address_space = 0;
94 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
95 }
96
97 flush->processor_mask = 0;
98 if (cpumask_equal(cpus, cpu_present_mask)) {
99 flush->flags |= HV_FLUSH_ALL_PROCESSORS;
100 } else {
Vitaly Kuznetsov0e4c88f2018-06-21 15:32:38 +0200101 /*
102 * From the supplied CPU set we need to figure out if we can get
103 * away with cheaper HVCALL_FLUSH_VIRTUAL_ADDRESS_{LIST,SPACE}
104 * hypercalls. This is possible when the highest VP number in
105 * the set is < 64. As VP numbers are usually in ascending order
106 * and match Linux CPU ids, here is an optimization: we check
107 * the VP number for the highest bit in the supplied set first
108 * so we can quickly find out if using *_EX hypercalls is a
109 * must. We will also check all VP numbers when walking the
110 * supplied CPU set to remain correct in all cases.
111 */
112 if (hv_cpu_number_to_vp_number(cpumask_last(cpus)) >= 64)
113 goto do_ex_hypercall;
114
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +0200115 for_each_cpu(cpu, cpus) {
116 vcpu = hv_cpu_number_to_vp_number(cpu);
Vitaly Kuznetsov110d2a72018-07-09 19:40:12 +0200117 if (vcpu == VP_INVAL) {
118 local_irq_restore(flags);
119 goto do_native;
120 }
121
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +0200122 if (vcpu >= 64)
Vitaly Kuznetsov0e4c88f2018-06-21 15:32:38 +0200123 goto do_ex_hypercall;
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +0200124
125 __set_bit(vcpu, (unsigned long *)
126 &flush->processor_mask);
127 }
128 }
129
130 /*
131 * We can flush not more than max_gvas with one hypercall. Flush the
132 * whole address space if we were asked to do more.
133 */
134 max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]);
135
136 if (info->end == TLB_FLUSH_ALL) {
137 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
138 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
139 flush, NULL);
140 } else if (info->end &&
141 ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
142 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
143 flush, NULL);
144 } else {
145 gva_n = fill_gva_list(flush->gva_list, 0,
146 info->start, info->end);
147 status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST,
148 gva_n, 0, flush, NULL);
149 }
Vitaly Kuznetsov0e4c88f2018-06-21 15:32:38 +0200150 goto check_status;
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +0200151
Vitaly Kuznetsov0e4c88f2018-06-21 15:32:38 +0200152do_ex_hypercall:
153 status = hyperv_flush_tlb_others_ex(cpus, info);
154
155check_status:
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +0200156 local_irq_restore(flags);
157
158 if (!(status & HV_HYPERCALL_RESULT_MASK))
159 return;
160do_native:
161 native_flush_tlb_others(cpus, info);
162}
163
Vitaly Kuznetsov0e4c88f2018-06-21 15:32:38 +0200164static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
165 const struct flush_tlb_info *info)
Vitaly Kuznetsov628f54c2017-08-02 18:09:20 +0200166{
167 int nr_bank = 0, max_gvas, gva_n;
Vitaly Kuznetsovc9c92be2018-05-16 17:21:24 +0200168 struct hv_tlb_flush_ex **flush_pcpu;
169 struct hv_tlb_flush_ex *flush;
Vitaly Kuznetsov0e4c88f2018-06-21 15:32:38 +0200170 u64 status;
Vitaly Kuznetsov628f54c2017-08-02 18:09:20 +0200171
Vitaly Kuznetsov0e4c88f2018-06-21 15:32:38 +0200172 if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
173 return U64_MAX;
Vitaly Kuznetsov628f54c2017-08-02 18:09:20 +0200174
Vitaly Kuznetsovc9c92be2018-05-16 17:21:24 +0200175 flush_pcpu = (struct hv_tlb_flush_ex **)
K. Y. Srinivasan9a2d78e2018-05-16 14:53:34 -0700176 this_cpu_ptr(hyperv_pcpu_input_arg);
Vitaly Kuznetsov60d73a72017-10-05 13:39:24 +0200177
178 flush = *flush_pcpu;
179
Vitaly Kuznetsov628f54c2017-08-02 18:09:20 +0200180 if (info->mm) {
Vitaly Kuznetsov617ab452018-01-24 11:36:29 +0100181 /*
182 * AddressSpace argument must match the CR3 with PCID bits
183 * stripped out.
184 */
Vitaly Kuznetsov628f54c2017-08-02 18:09:20 +0200185 flush->address_space = virt_to_phys(info->mm->pgd);
Vitaly Kuznetsov617ab452018-01-24 11:36:29 +0100186 flush->address_space &= CR3_ADDR_MASK;
Vitaly Kuznetsov628f54c2017-08-02 18:09:20 +0200187 flush->flags = 0;
188 } else {
189 flush->address_space = 0;
190 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
191 }
192
193 flush->hv_vp_set.valid_bank_mask = 0;
194
Vitaly Kuznetsov0e4c88f2018-06-21 15:32:38 +0200195 flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K;
196 nr_bank = cpumask_to_vpset(&(flush->hv_vp_set), cpus);
Vitaly Kuznetsov0f0caa52018-07-09 19:40:11 +0200197 if (nr_bank < 0)
198 return U64_MAX;
Vitaly Kuznetsov628f54c2017-08-02 18:09:20 +0200199
200 /*
201 * We can flush not more than max_gvas with one hypercall. Flush the
202 * whole address space if we were asked to do more.
203 */
204 max_gvas =
205 (PAGE_SIZE - sizeof(*flush) - nr_bank *
206 sizeof(flush->hv_vp_set.bank_contents[0])) /
207 sizeof(flush->gva_list[0]);
208
209 if (info->end == TLB_FLUSH_ALL) {
210 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
211 status = hv_do_rep_hypercall(
212 HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
Marcelo Henrique Cerriab7ff472017-10-05 10:34:29 -0300213 0, nr_bank, flush, NULL);
Vitaly Kuznetsov628f54c2017-08-02 18:09:20 +0200214 } else if (info->end &&
215 ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
216 status = hv_do_rep_hypercall(
217 HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
Marcelo Henrique Cerriab7ff472017-10-05 10:34:29 -0300218 0, nr_bank, flush, NULL);
Vitaly Kuznetsov628f54c2017-08-02 18:09:20 +0200219 } else {
220 gva_n = fill_gva_list(flush->gva_list, nr_bank,
221 info->start, info->end);
222 status = hv_do_rep_hypercall(
223 HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX,
Marcelo Henrique Cerriab7ff472017-10-05 10:34:29 -0300224 gva_n, nr_bank, flush, NULL);
Vitaly Kuznetsov628f54c2017-08-02 18:09:20 +0200225 }
226
Vitaly Kuznetsov0e4c88f2018-06-21 15:32:38 +0200227 return status;
Vitaly Kuznetsov628f54c2017-08-02 18:09:20 +0200228}
229
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +0200230void hyperv_setup_mmu_ops(void)
231{
Vitaly Kuznetsov628f54c2017-08-02 18:09:20 +0200232 if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED))
233 return;
234
Vitaly Kuznetsov0e4c88f2018-06-21 15:32:38 +0200235 pr_info("Using hypercall for remote TLB flush\n");
Juergen Gross5c835112018-08-28 09:40:19 +0200236 pv_ops.mmu.flush_tlb_others = hyperv_flush_tlb_others;
237 pv_ops.mmu.tlb_remove_table = tlb_remove_table;
Vitaly Kuznetsov2ffd9e32017-08-02 18:09:19 +0200238}