Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 1 | /* |
Scott Wood | 49ea069 | 2011-03-28 15:01:24 -0500 | [diff] [blame] | 2 | * Copyright (C) 2008-2011 Freescale Semiconductor, Inc. All rights reserved. |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 3 | * |
| 4 | * Author: Yu Liu, yu.liu@freescale.com |
| 5 | * |
| 6 | * Description: |
| 7 | * This file is based on arch/powerpc/kvm/44x_tlb.c, |
| 8 | * by Hollis Blanchard <hollisb@us.ibm.com>. |
| 9 | * |
| 10 | * This program is free software; you can redistribute it and/or modify |
| 11 | * it under the terms of the GNU General Public License, version 2, as |
| 12 | * published by the Free Software Foundation. |
| 13 | */ |
| 14 | |
| 15 | #include <linux/types.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 16 | #include <linux/slab.h> |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 17 | #include <linux/string.h> |
| 18 | #include <linux/kvm.h> |
| 19 | #include <linux/kvm_host.h> |
| 20 | #include <linux/highmem.h> |
| 21 | #include <asm/kvm_ppc.h> |
| 22 | #include <asm/kvm_e500.h> |
| 23 | |
Liu Yu | 9aa4dd5 | 2009-01-14 10:47:38 -0600 | [diff] [blame] | 24 | #include "../mm/mmu_decl.h" |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 25 | #include "e500_tlb.h" |
Marcelo Tosatti | 46f43c6 | 2009-06-18 11:47:27 -0300 | [diff] [blame] | 26 | #include "trace.h" |
Scott Wood | 49ea069 | 2011-03-28 15:01:24 -0500 | [diff] [blame] | 27 | #include "timing.h" |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 28 | |
| 29 | #define to_htlb1_esel(esel) (tlb1_entry_num - (esel) - 1) |
| 30 | |
| 31 | static unsigned int tlb1_entry_num; |
| 32 | |
| 33 | void kvmppc_dump_tlbs(struct kvm_vcpu *vcpu) |
| 34 | { |
| 35 | struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); |
| 36 | struct tlbe *tlbe; |
| 37 | int i, tlbsel; |
| 38 | |
| 39 | printk("| %8s | %8s | %8s | %8s | %8s |\n", |
| 40 | "nr", "mas1", "mas2", "mas3", "mas7"); |
| 41 | |
| 42 | for (tlbsel = 0; tlbsel < 2; tlbsel++) { |
| 43 | printk("Guest TLB%d:\n", tlbsel); |
| 44 | for (i = 0; i < vcpu_e500->guest_tlb_size[tlbsel]; i++) { |
| 45 | tlbe = &vcpu_e500->guest_tlb[tlbsel][i]; |
| 46 | if (tlbe->mas1 & MAS1_VALID) |
| 47 | printk(" G[%d][%3d] | %08X | %08X | %08X | %08X |\n", |
| 48 | tlbsel, i, tlbe->mas1, tlbe->mas2, |
| 49 | tlbe->mas3, tlbe->mas7); |
| 50 | } |
| 51 | } |
| 52 | |
| 53 | for (tlbsel = 0; tlbsel < 2; tlbsel++) { |
| 54 | printk("Shadow TLB%d:\n", tlbsel); |
| 55 | for (i = 0; i < vcpu_e500->shadow_tlb_size[tlbsel]; i++) { |
| 56 | tlbe = &vcpu_e500->shadow_tlb[tlbsel][i]; |
| 57 | if (tlbe->mas1 & MAS1_VALID) |
| 58 | printk(" S[%d][%3d] | %08X | %08X | %08X | %08X |\n", |
| 59 | tlbsel, i, tlbe->mas1, tlbe->mas2, |
| 60 | tlbe->mas3, tlbe->mas7); |
| 61 | } |
| 62 | } |
| 63 | } |
| 64 | |
| 65 | static inline unsigned int tlb0_get_next_victim( |
| 66 | struct kvmppc_vcpu_e500 *vcpu_e500) |
| 67 | { |
| 68 | unsigned int victim; |
| 69 | |
| 70 | victim = vcpu_e500->guest_tlb_nv[0]++; |
| 71 | if (unlikely(vcpu_e500->guest_tlb_nv[0] >= KVM_E500_TLB0_WAY_NUM)) |
| 72 | vcpu_e500->guest_tlb_nv[0] = 0; |
| 73 | |
| 74 | return victim; |
| 75 | } |
| 76 | |
| 77 | static inline unsigned int tlb1_max_shadow_size(void) |
| 78 | { |
| 79 | return tlb1_entry_num - tlbcam_index; |
| 80 | } |
| 81 | |
| 82 | static inline int tlbe_is_writable(struct tlbe *tlbe) |
| 83 | { |
| 84 | return tlbe->mas3 & (MAS3_SW|MAS3_UW); |
| 85 | } |
| 86 | |
| 87 | static inline u32 e500_shadow_mas3_attrib(u32 mas3, int usermode) |
| 88 | { |
| 89 | /* Mask off reserved bits. */ |
| 90 | mas3 &= MAS3_ATTRIB_MASK; |
| 91 | |
| 92 | if (!usermode) { |
| 93 | /* Guest is in supervisor mode, |
| 94 | * so we need to translate guest |
| 95 | * supervisor permissions into user permissions. */ |
| 96 | mas3 &= ~E500_TLB_USER_PERM_MASK; |
| 97 | mas3 |= (mas3 & E500_TLB_SUPER_PERM_MASK) << 1; |
| 98 | } |
| 99 | |
| 100 | return mas3 | E500_TLB_SUPER_PERM_MASK; |
| 101 | } |
| 102 | |
| 103 | static inline u32 e500_shadow_mas2_attrib(u32 mas2, int usermode) |
| 104 | { |
Liu Yu | 046a48b | 2009-03-17 16:57:46 +0800 | [diff] [blame] | 105 | #ifdef CONFIG_SMP |
| 106 | return (mas2 & MAS2_ATTRIB_MASK) | MAS2_M; |
| 107 | #else |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 108 | return mas2 & MAS2_ATTRIB_MASK; |
Liu Yu | 046a48b | 2009-03-17 16:57:46 +0800 | [diff] [blame] | 109 | #endif |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 110 | } |
| 111 | |
| 112 | /* |
| 113 | * writing shadow tlb entry to host TLB |
| 114 | */ |
| 115 | static inline void __write_host_tlbe(struct tlbe *stlbe) |
| 116 | { |
| 117 | mtspr(SPRN_MAS1, stlbe->mas1); |
| 118 | mtspr(SPRN_MAS2, stlbe->mas2); |
| 119 | mtspr(SPRN_MAS3, stlbe->mas3); |
| 120 | mtspr(SPRN_MAS7, stlbe->mas7); |
| 121 | __asm__ __volatile__ ("tlbwe\n" : : ); |
| 122 | } |
| 123 | |
| 124 | static inline void write_host_tlbe(struct kvmppc_vcpu_e500 *vcpu_e500, |
| 125 | int tlbsel, int esel) |
| 126 | { |
| 127 | struct tlbe *stlbe = &vcpu_e500->shadow_tlb[tlbsel][esel]; |
| 128 | |
| 129 | local_irq_disable(); |
| 130 | if (tlbsel == 0) { |
| 131 | __write_host_tlbe(stlbe); |
| 132 | } else { |
| 133 | unsigned register mas0; |
| 134 | |
| 135 | mas0 = mfspr(SPRN_MAS0); |
| 136 | |
| 137 | mtspr(SPRN_MAS0, MAS0_TLBSEL(1) | MAS0_ESEL(to_htlb1_esel(esel))); |
| 138 | __write_host_tlbe(stlbe); |
| 139 | |
| 140 | mtspr(SPRN_MAS0, mas0); |
| 141 | } |
| 142 | local_irq_enable(); |
| 143 | } |
| 144 | |
| 145 | void kvmppc_e500_tlb_load(struct kvm_vcpu *vcpu, int cpu) |
| 146 | { |
| 147 | struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); |
| 148 | int i; |
| 149 | unsigned register mas0; |
| 150 | |
| 151 | /* Load all valid TLB1 entries to reduce guest tlb miss fault */ |
| 152 | local_irq_disable(); |
| 153 | mas0 = mfspr(SPRN_MAS0); |
| 154 | for (i = 0; i < tlb1_max_shadow_size(); i++) { |
| 155 | struct tlbe *stlbe = &vcpu_e500->shadow_tlb[1][i]; |
| 156 | |
| 157 | if (get_tlb_v(stlbe)) { |
| 158 | mtspr(SPRN_MAS0, MAS0_TLBSEL(1) |
| 159 | | MAS0_ESEL(to_htlb1_esel(i))); |
| 160 | __write_host_tlbe(stlbe); |
| 161 | } |
| 162 | } |
| 163 | mtspr(SPRN_MAS0, mas0); |
| 164 | local_irq_enable(); |
| 165 | } |
| 166 | |
| 167 | void kvmppc_e500_tlb_put(struct kvm_vcpu *vcpu) |
| 168 | { |
Liu Yu | 9aa4dd5 | 2009-01-14 10:47:38 -0600 | [diff] [blame] | 169 | _tlbil_all(); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 170 | } |
| 171 | |
| 172 | /* Search the guest TLB for a matching entry. */ |
| 173 | static int kvmppc_e500_tlb_index(struct kvmppc_vcpu_e500 *vcpu_e500, |
| 174 | gva_t eaddr, int tlbsel, unsigned int pid, int as) |
| 175 | { |
| 176 | int i; |
| 177 | |
| 178 | /* XXX Replace loop with fancy data structures. */ |
| 179 | for (i = 0; i < vcpu_e500->guest_tlb_size[tlbsel]; i++) { |
| 180 | struct tlbe *tlbe = &vcpu_e500->guest_tlb[tlbsel][i]; |
| 181 | unsigned int tid; |
| 182 | |
| 183 | if (eaddr < get_tlb_eaddr(tlbe)) |
| 184 | continue; |
| 185 | |
| 186 | if (eaddr > get_tlb_end(tlbe)) |
| 187 | continue; |
| 188 | |
| 189 | tid = get_tlb_tid(tlbe); |
| 190 | if (tid && (tid != pid)) |
| 191 | continue; |
| 192 | |
| 193 | if (!get_tlb_v(tlbe)) |
| 194 | continue; |
| 195 | |
| 196 | if (get_tlb_ts(tlbe) != as && as != -1) |
| 197 | continue; |
| 198 | |
| 199 | return i; |
| 200 | } |
| 201 | |
| 202 | return -1; |
| 203 | } |
| 204 | |
| 205 | static void kvmppc_e500_shadow_release(struct kvmppc_vcpu_e500 *vcpu_e500, |
| 206 | int tlbsel, int esel) |
| 207 | { |
| 208 | struct tlbe *stlbe = &vcpu_e500->shadow_tlb[tlbsel][esel]; |
| 209 | struct page *page = vcpu_e500->shadow_pages[tlbsel][esel]; |
| 210 | |
| 211 | if (page) { |
| 212 | vcpu_e500->shadow_pages[tlbsel][esel] = NULL; |
| 213 | |
| 214 | if (get_tlb_v(stlbe)) { |
| 215 | if (tlbe_is_writable(stlbe)) |
| 216 | kvm_release_page_dirty(page); |
| 217 | else |
| 218 | kvm_release_page_clean(page); |
| 219 | } |
| 220 | } |
| 221 | } |
| 222 | |
| 223 | static void kvmppc_e500_stlbe_invalidate(struct kvmppc_vcpu_e500 *vcpu_e500, |
| 224 | int tlbsel, int esel) |
| 225 | { |
| 226 | struct tlbe *stlbe = &vcpu_e500->shadow_tlb[tlbsel][esel]; |
| 227 | |
| 228 | kvmppc_e500_shadow_release(vcpu_e500, tlbsel, esel); |
| 229 | stlbe->mas1 = 0; |
Kyle Moffett | 21e537b | 2010-08-30 11:38:39 -0400 | [diff] [blame] | 230 | trace_kvm_stlb_inval(index_of(tlbsel, esel)); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 231 | } |
| 232 | |
| 233 | static void kvmppc_e500_tlb1_invalidate(struct kvmppc_vcpu_e500 *vcpu_e500, |
| 234 | gva_t eaddr, gva_t eend, u32 tid) |
| 235 | { |
| 236 | unsigned int pid = tid & 0xff; |
| 237 | unsigned int i; |
| 238 | |
| 239 | /* XXX Replace loop with fancy data structures. */ |
| 240 | for (i = 0; i < vcpu_e500->guest_tlb_size[1]; i++) { |
| 241 | struct tlbe *stlbe = &vcpu_e500->shadow_tlb[1][i]; |
| 242 | unsigned int tid; |
| 243 | |
| 244 | if (!get_tlb_v(stlbe)) |
| 245 | continue; |
| 246 | |
| 247 | if (eend < get_tlb_eaddr(stlbe)) |
| 248 | continue; |
| 249 | |
| 250 | if (eaddr > get_tlb_end(stlbe)) |
| 251 | continue; |
| 252 | |
| 253 | tid = get_tlb_tid(stlbe); |
| 254 | if (tid && (tid != pid)) |
| 255 | continue; |
| 256 | |
| 257 | kvmppc_e500_stlbe_invalidate(vcpu_e500, 1, i); |
| 258 | write_host_tlbe(vcpu_e500, 1, i); |
| 259 | } |
| 260 | } |
| 261 | |
| 262 | static inline void kvmppc_e500_deliver_tlb_miss(struct kvm_vcpu *vcpu, |
| 263 | unsigned int eaddr, int as) |
| 264 | { |
| 265 | struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); |
| 266 | unsigned int victim, pidsel, tsized; |
| 267 | int tlbsel; |
| 268 | |
Liu Yu | fb2838d | 2009-01-14 10:47:37 -0600 | [diff] [blame] | 269 | /* since we only have two TLBs, only lower bit is used. */ |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 270 | tlbsel = (vcpu_e500->mas4 >> 28) & 0x1; |
| 271 | victim = (tlbsel == 0) ? tlb0_get_next_victim(vcpu_e500) : 0; |
| 272 | pidsel = (vcpu_e500->mas4 >> 16) & 0xf; |
Liu Yu | 0cfb50e | 2009-06-05 14:54:29 +0800 | [diff] [blame] | 273 | tsized = (vcpu_e500->mas4 >> 7) & 0x1f; |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 274 | |
| 275 | vcpu_e500->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(victim) |
| 276 | | MAS0_NV(vcpu_e500->guest_tlb_nv[tlbsel]); |
| 277 | vcpu_e500->mas1 = MAS1_VALID | (as ? MAS1_TS : 0) |
| 278 | | MAS1_TID(vcpu_e500->pid[pidsel]) |
| 279 | | MAS1_TSIZE(tsized); |
| 280 | vcpu_e500->mas2 = (eaddr & MAS2_EPN) |
| 281 | | (vcpu_e500->mas4 & MAS2_ATTRIB_MASK); |
| 282 | vcpu_e500->mas3 &= MAS3_U0 | MAS3_U1 | MAS3_U2 | MAS3_U3; |
| 283 | vcpu_e500->mas6 = (vcpu_e500->mas6 & MAS6_SPID1) |
| 284 | | (get_cur_pid(vcpu) << 16) |
| 285 | | (as ? MAS6_SAS : 0); |
| 286 | vcpu_e500->mas7 = 0; |
| 287 | } |
| 288 | |
| 289 | static inline void kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500, |
| 290 | u64 gvaddr, gfn_t gfn, struct tlbe *gtlbe, int tlbsel, int esel) |
| 291 | { |
| 292 | struct page *new_page; |
| 293 | struct tlbe *stlbe; |
| 294 | hpa_t hpaddr; |
| 295 | |
| 296 | stlbe = &vcpu_e500->shadow_tlb[tlbsel][esel]; |
| 297 | |
| 298 | /* Get reference to new page. */ |
| 299 | new_page = gfn_to_page(vcpu_e500->vcpu.kvm, gfn); |
| 300 | if (is_error_page(new_page)) { |
Alexander Graf | 344941b | 2010-08-31 03:45:39 +0200 | [diff] [blame] | 301 | printk(KERN_ERR "Couldn't get guest page for gfn %lx!\n", |
| 302 | (long)gfn); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 303 | kvm_release_page_clean(new_page); |
| 304 | return; |
| 305 | } |
| 306 | hpaddr = page_to_phys(new_page); |
| 307 | |
| 308 | /* Drop reference to old page. */ |
| 309 | kvmppc_e500_shadow_release(vcpu_e500, tlbsel, esel); |
| 310 | |
| 311 | vcpu_e500->shadow_pages[tlbsel][esel] = new_page; |
| 312 | |
| 313 | /* Force TS=1 IPROT=0 TSIZE=4KB for all guest mappings. */ |
Liu Yu | 0cfb50e | 2009-06-05 14:54:29 +0800 | [diff] [blame] | 314 | stlbe->mas1 = MAS1_TSIZE(BOOK3E_PAGESZ_4K) |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 315 | | MAS1_TID(get_tlb_tid(gtlbe)) | MAS1_TS | MAS1_VALID; |
| 316 | stlbe->mas2 = (gvaddr & MAS2_EPN) |
| 317 | | e500_shadow_mas2_attrib(gtlbe->mas2, |
Alexander Graf | 666e725 | 2010-07-29 14:47:43 +0200 | [diff] [blame] | 318 | vcpu_e500->vcpu.arch.shared->msr & MSR_PR); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 319 | stlbe->mas3 = (hpaddr & MAS3_RPN) |
| 320 | | e500_shadow_mas3_attrib(gtlbe->mas3, |
Alexander Graf | 666e725 | 2010-07-29 14:47:43 +0200 | [diff] [blame] | 321 | vcpu_e500->vcpu.arch.shared->msr & MSR_PR); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 322 | stlbe->mas7 = (hpaddr >> 32) & MAS7_RPN; |
| 323 | |
Marcelo Tosatti | 46f43c6 | 2009-06-18 11:47:27 -0300 | [diff] [blame] | 324 | trace_kvm_stlb_write(index_of(tlbsel, esel), stlbe->mas1, stlbe->mas2, |
| 325 | stlbe->mas3, stlbe->mas7); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 326 | } |
| 327 | |
| 328 | /* XXX only map the one-one case, for now use TLB0 */ |
| 329 | static int kvmppc_e500_stlbe_map(struct kvmppc_vcpu_e500 *vcpu_e500, |
| 330 | int tlbsel, int esel) |
| 331 | { |
| 332 | struct tlbe *gtlbe; |
| 333 | |
| 334 | gtlbe = &vcpu_e500->guest_tlb[tlbsel][esel]; |
| 335 | |
| 336 | kvmppc_e500_shadow_map(vcpu_e500, get_tlb_eaddr(gtlbe), |
| 337 | get_tlb_raddr(gtlbe) >> PAGE_SHIFT, |
| 338 | gtlbe, tlbsel, esel); |
| 339 | |
| 340 | return esel; |
| 341 | } |
| 342 | |
| 343 | /* Caller must ensure that the specified guest TLB entry is safe to insert into |
| 344 | * the shadow TLB. */ |
| 345 | /* XXX for both one-one and one-to-many , for now use TLB1 */ |
| 346 | static int kvmppc_e500_tlb1_map(struct kvmppc_vcpu_e500 *vcpu_e500, |
| 347 | u64 gvaddr, gfn_t gfn, struct tlbe *gtlbe) |
| 348 | { |
| 349 | unsigned int victim; |
| 350 | |
| 351 | victim = vcpu_e500->guest_tlb_nv[1]++; |
| 352 | |
| 353 | if (unlikely(vcpu_e500->guest_tlb_nv[1] >= tlb1_max_shadow_size())) |
| 354 | vcpu_e500->guest_tlb_nv[1] = 0; |
| 355 | |
| 356 | kvmppc_e500_shadow_map(vcpu_e500, gvaddr, gfn, gtlbe, 1, victim); |
| 357 | |
| 358 | return victim; |
| 359 | } |
| 360 | |
| 361 | /* Invalidate all guest kernel mappings when enter usermode, |
| 362 | * so that when they fault back in they will get the |
| 363 | * proper permission bits. */ |
| 364 | void kvmppc_mmu_priv_switch(struct kvm_vcpu *vcpu, int usermode) |
| 365 | { |
| 366 | if (usermode) { |
| 367 | struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); |
| 368 | int i; |
| 369 | |
| 370 | /* XXX Replace loop with fancy data structures. */ |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 371 | for (i = 0; i < tlb1_max_shadow_size(); i++) |
| 372 | kvmppc_e500_stlbe_invalidate(vcpu_e500, 1, i); |
| 373 | |
Liu Yu | 9aa4dd5 | 2009-01-14 10:47:38 -0600 | [diff] [blame] | 374 | _tlbil_all(); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 375 | } |
| 376 | } |
| 377 | |
| 378 | static int kvmppc_e500_gtlbe_invalidate(struct kvmppc_vcpu_e500 *vcpu_e500, |
| 379 | int tlbsel, int esel) |
| 380 | { |
| 381 | struct tlbe *gtlbe = &vcpu_e500->guest_tlb[tlbsel][esel]; |
| 382 | |
| 383 | if (unlikely(get_tlb_iprot(gtlbe))) |
| 384 | return -1; |
| 385 | |
| 386 | if (tlbsel == 1) { |
| 387 | kvmppc_e500_tlb1_invalidate(vcpu_e500, get_tlb_eaddr(gtlbe), |
| 388 | get_tlb_end(gtlbe), |
| 389 | get_tlb_tid(gtlbe)); |
| 390 | } else { |
| 391 | kvmppc_e500_stlbe_invalidate(vcpu_e500, tlbsel, esel); |
| 392 | } |
| 393 | |
| 394 | gtlbe->mas1 = 0; |
| 395 | |
| 396 | return 0; |
| 397 | } |
| 398 | |
Liu Yu | b0a1835 | 2009-02-17 16:52:08 +0800 | [diff] [blame] | 399 | int kvmppc_e500_emul_mt_mmucsr0(struct kvmppc_vcpu_e500 *vcpu_e500, ulong value) |
| 400 | { |
| 401 | int esel; |
| 402 | |
| 403 | if (value & MMUCSR0_TLB0FI) |
| 404 | for (esel = 0; esel < vcpu_e500->guest_tlb_size[0]; esel++) |
| 405 | kvmppc_e500_gtlbe_invalidate(vcpu_e500, 0, esel); |
| 406 | if (value & MMUCSR0_TLB1FI) |
| 407 | for (esel = 0; esel < vcpu_e500->guest_tlb_size[1]; esel++) |
| 408 | kvmppc_e500_gtlbe_invalidate(vcpu_e500, 1, esel); |
| 409 | |
| 410 | _tlbil_all(); |
| 411 | |
| 412 | return EMULATE_DONE; |
| 413 | } |
| 414 | |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 415 | int kvmppc_e500_emul_tlbivax(struct kvm_vcpu *vcpu, int ra, int rb) |
| 416 | { |
| 417 | struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); |
| 418 | unsigned int ia; |
| 419 | int esel, tlbsel; |
| 420 | gva_t ea; |
| 421 | |
Alexander Graf | 8e5b26b | 2010-01-08 02:58:01 +0100 | [diff] [blame] | 422 | ea = ((ra) ? kvmppc_get_gpr(vcpu, ra) : 0) + kvmppc_get_gpr(vcpu, rb); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 423 | |
| 424 | ia = (ea >> 2) & 0x1; |
| 425 | |
Liu Yu | fb2838d | 2009-01-14 10:47:37 -0600 | [diff] [blame] | 426 | /* since we only have two TLBs, only lower bit is used. */ |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 427 | tlbsel = (ea >> 3) & 0x1; |
| 428 | |
| 429 | if (ia) { |
| 430 | /* invalidate all entries */ |
| 431 | for (esel = 0; esel < vcpu_e500->guest_tlb_size[tlbsel]; esel++) |
| 432 | kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel); |
| 433 | } else { |
| 434 | ea &= 0xfffff000; |
| 435 | esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, |
| 436 | get_cur_pid(vcpu), -1); |
| 437 | if (esel >= 0) |
| 438 | kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel); |
| 439 | } |
| 440 | |
Liu Yu | 9aa4dd5 | 2009-01-14 10:47:38 -0600 | [diff] [blame] | 441 | _tlbil_all(); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 442 | |
| 443 | return EMULATE_DONE; |
| 444 | } |
| 445 | |
| 446 | int kvmppc_e500_emul_tlbre(struct kvm_vcpu *vcpu) |
| 447 | { |
| 448 | struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); |
| 449 | int tlbsel, esel; |
| 450 | struct tlbe *gtlbe; |
| 451 | |
| 452 | tlbsel = get_tlb_tlbsel(vcpu_e500); |
| 453 | esel = get_tlb_esel(vcpu_e500, tlbsel); |
| 454 | |
| 455 | gtlbe = &vcpu_e500->guest_tlb[tlbsel][esel]; |
Liu Yu | bc35cbc | 2009-03-17 16:57:45 +0800 | [diff] [blame] | 456 | vcpu_e500->mas0 &= ~MAS0_NV(~0); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 457 | vcpu_e500->mas0 |= MAS0_NV(vcpu_e500->guest_tlb_nv[tlbsel]); |
| 458 | vcpu_e500->mas1 = gtlbe->mas1; |
| 459 | vcpu_e500->mas2 = gtlbe->mas2; |
| 460 | vcpu_e500->mas3 = gtlbe->mas3; |
| 461 | vcpu_e500->mas7 = gtlbe->mas7; |
| 462 | |
| 463 | return EMULATE_DONE; |
| 464 | } |
| 465 | |
| 466 | int kvmppc_e500_emul_tlbsx(struct kvm_vcpu *vcpu, int rb) |
| 467 | { |
| 468 | struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); |
| 469 | int as = !!get_cur_sas(vcpu_e500); |
| 470 | unsigned int pid = get_cur_spid(vcpu_e500); |
| 471 | int esel, tlbsel; |
| 472 | struct tlbe *gtlbe = NULL; |
| 473 | gva_t ea; |
| 474 | |
Alexander Graf | 8e5b26b | 2010-01-08 02:58:01 +0100 | [diff] [blame] | 475 | ea = kvmppc_get_gpr(vcpu, rb); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 476 | |
| 477 | for (tlbsel = 0; tlbsel < 2; tlbsel++) { |
| 478 | esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, pid, as); |
| 479 | if (esel >= 0) { |
| 480 | gtlbe = &vcpu_e500->guest_tlb[tlbsel][esel]; |
| 481 | break; |
| 482 | } |
| 483 | } |
| 484 | |
| 485 | if (gtlbe) { |
| 486 | vcpu_e500->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(esel) |
| 487 | | MAS0_NV(vcpu_e500->guest_tlb_nv[tlbsel]); |
| 488 | vcpu_e500->mas1 = gtlbe->mas1; |
| 489 | vcpu_e500->mas2 = gtlbe->mas2; |
| 490 | vcpu_e500->mas3 = gtlbe->mas3; |
| 491 | vcpu_e500->mas7 = gtlbe->mas7; |
| 492 | } else { |
| 493 | int victim; |
| 494 | |
Liu Yu | fb2838d | 2009-01-14 10:47:37 -0600 | [diff] [blame] | 495 | /* since we only have two TLBs, only lower bit is used. */ |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 496 | tlbsel = vcpu_e500->mas4 >> 28 & 0x1; |
| 497 | victim = (tlbsel == 0) ? tlb0_get_next_victim(vcpu_e500) : 0; |
| 498 | |
| 499 | vcpu_e500->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(victim) |
| 500 | | MAS0_NV(vcpu_e500->guest_tlb_nv[tlbsel]); |
| 501 | vcpu_e500->mas1 = (vcpu_e500->mas6 & MAS6_SPID0) |
| 502 | | (vcpu_e500->mas6 & (MAS6_SAS ? MAS1_TS : 0)) |
| 503 | | (vcpu_e500->mas4 & MAS4_TSIZED(~0)); |
| 504 | vcpu_e500->mas2 &= MAS2_EPN; |
| 505 | vcpu_e500->mas2 |= vcpu_e500->mas4 & MAS2_ATTRIB_MASK; |
| 506 | vcpu_e500->mas3 &= MAS3_U0 | MAS3_U1 | MAS3_U2 | MAS3_U3; |
| 507 | vcpu_e500->mas7 = 0; |
| 508 | } |
| 509 | |
Scott Wood | 49ea069 | 2011-03-28 15:01:24 -0500 | [diff] [blame] | 510 | kvmppc_set_exit_type(vcpu, EMULATED_TLBSX_EXITS); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 511 | return EMULATE_DONE; |
| 512 | } |
| 513 | |
| 514 | int kvmppc_e500_emul_tlbwe(struct kvm_vcpu *vcpu) |
| 515 | { |
| 516 | struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); |
| 517 | u64 eaddr; |
| 518 | u64 raddr; |
| 519 | u32 tid; |
| 520 | struct tlbe *gtlbe; |
| 521 | int tlbsel, esel, stlbsel, sesel; |
| 522 | |
| 523 | tlbsel = get_tlb_tlbsel(vcpu_e500); |
| 524 | esel = get_tlb_esel(vcpu_e500, tlbsel); |
| 525 | |
| 526 | gtlbe = &vcpu_e500->guest_tlb[tlbsel][esel]; |
| 527 | |
| 528 | if (get_tlb_v(gtlbe) && tlbsel == 1) { |
| 529 | eaddr = get_tlb_eaddr(gtlbe); |
| 530 | tid = get_tlb_tid(gtlbe); |
| 531 | kvmppc_e500_tlb1_invalidate(vcpu_e500, eaddr, |
| 532 | get_tlb_end(gtlbe), tid); |
| 533 | } |
| 534 | |
| 535 | gtlbe->mas1 = vcpu_e500->mas1; |
| 536 | gtlbe->mas2 = vcpu_e500->mas2; |
| 537 | gtlbe->mas3 = vcpu_e500->mas3; |
| 538 | gtlbe->mas7 = vcpu_e500->mas7; |
| 539 | |
Marcelo Tosatti | 46f43c6 | 2009-06-18 11:47:27 -0300 | [diff] [blame] | 540 | trace_kvm_gtlb_write(vcpu_e500->mas0, gtlbe->mas1, gtlbe->mas2, |
| 541 | gtlbe->mas3, gtlbe->mas7); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 542 | |
| 543 | /* Invalidate shadow mappings for the about-to-be-clobbered TLBE. */ |
| 544 | if (tlbe_is_host_safe(vcpu, gtlbe)) { |
| 545 | switch (tlbsel) { |
| 546 | case 0: |
| 547 | /* TLB0 */ |
| 548 | gtlbe->mas1 &= ~MAS1_TSIZE(~0); |
Liu Yu | 0cfb50e | 2009-06-05 14:54:29 +0800 | [diff] [blame] | 549 | gtlbe->mas1 |= MAS1_TSIZE(BOOK3E_PAGESZ_4K); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 550 | |
| 551 | stlbsel = 0; |
| 552 | sesel = kvmppc_e500_stlbe_map(vcpu_e500, 0, esel); |
| 553 | |
| 554 | break; |
| 555 | |
| 556 | case 1: |
| 557 | /* TLB1 */ |
| 558 | eaddr = get_tlb_eaddr(gtlbe); |
| 559 | raddr = get_tlb_raddr(gtlbe); |
| 560 | |
| 561 | /* Create a 4KB mapping on the host. |
| 562 | * If the guest wanted a large page, |
| 563 | * only the first 4KB is mapped here and the rest |
| 564 | * are mapped on the fly. */ |
| 565 | stlbsel = 1; |
| 566 | sesel = kvmppc_e500_tlb1_map(vcpu_e500, eaddr, |
| 567 | raddr >> PAGE_SHIFT, gtlbe); |
| 568 | break; |
| 569 | |
| 570 | default: |
| 571 | BUG(); |
| 572 | } |
| 573 | write_host_tlbe(vcpu_e500, stlbsel, sesel); |
| 574 | } |
| 575 | |
Scott Wood | 49ea069 | 2011-03-28 15:01:24 -0500 | [diff] [blame] | 576 | kvmppc_set_exit_type(vcpu, EMULATED_TLBWE_EXITS); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 577 | return EMULATE_DONE; |
| 578 | } |
| 579 | |
| 580 | int kvmppc_mmu_itlb_index(struct kvm_vcpu *vcpu, gva_t eaddr) |
| 581 | { |
Alexander Graf | 666e725 | 2010-07-29 14:47:43 +0200 | [diff] [blame] | 582 | unsigned int as = !!(vcpu->arch.shared->msr & MSR_IS); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 583 | |
| 584 | return kvmppc_e500_tlb_search(vcpu, eaddr, get_cur_pid(vcpu), as); |
| 585 | } |
| 586 | |
| 587 | int kvmppc_mmu_dtlb_index(struct kvm_vcpu *vcpu, gva_t eaddr) |
| 588 | { |
Alexander Graf | 666e725 | 2010-07-29 14:47:43 +0200 | [diff] [blame] | 589 | unsigned int as = !!(vcpu->arch.shared->msr & MSR_DS); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 590 | |
| 591 | return kvmppc_e500_tlb_search(vcpu, eaddr, get_cur_pid(vcpu), as); |
| 592 | } |
| 593 | |
| 594 | void kvmppc_mmu_itlb_miss(struct kvm_vcpu *vcpu) |
| 595 | { |
Alexander Graf | 666e725 | 2010-07-29 14:47:43 +0200 | [diff] [blame] | 596 | unsigned int as = !!(vcpu->arch.shared->msr & MSR_IS); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 597 | |
| 598 | kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.pc, as); |
| 599 | } |
| 600 | |
| 601 | void kvmppc_mmu_dtlb_miss(struct kvm_vcpu *vcpu) |
| 602 | { |
Alexander Graf | 666e725 | 2010-07-29 14:47:43 +0200 | [diff] [blame] | 603 | unsigned int as = !!(vcpu->arch.shared->msr & MSR_DS); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 604 | |
| 605 | kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.fault_dear, as); |
| 606 | } |
| 607 | |
| 608 | gpa_t kvmppc_mmu_xlate(struct kvm_vcpu *vcpu, unsigned int index, |
| 609 | gva_t eaddr) |
| 610 | { |
| 611 | struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); |
| 612 | struct tlbe *gtlbe = |
| 613 | &vcpu_e500->guest_tlb[tlbsel_of(index)][esel_of(index)]; |
| 614 | u64 pgmask = get_tlb_bytes(gtlbe) - 1; |
| 615 | |
| 616 | return get_tlb_raddr(gtlbe) | (eaddr & pgmask); |
| 617 | } |
| 618 | |
| 619 | void kvmppc_mmu_destroy(struct kvm_vcpu *vcpu) |
| 620 | { |
| 621 | struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); |
| 622 | int tlbsel, i; |
| 623 | |
| 624 | for (tlbsel = 0; tlbsel < 2; tlbsel++) |
| 625 | for (i = 0; i < vcpu_e500->guest_tlb_size[tlbsel]; i++) |
| 626 | kvmppc_e500_shadow_release(vcpu_e500, tlbsel, i); |
| 627 | |
| 628 | /* discard all guest mapping */ |
Liu Yu | 9aa4dd5 | 2009-01-14 10:47:38 -0600 | [diff] [blame] | 629 | _tlbil_all(); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 630 | } |
| 631 | |
| 632 | void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 eaddr, gpa_t gpaddr, |
| 633 | unsigned int index) |
| 634 | { |
| 635 | struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); |
| 636 | int tlbsel = tlbsel_of(index); |
| 637 | int esel = esel_of(index); |
| 638 | int stlbsel, sesel; |
| 639 | |
| 640 | switch (tlbsel) { |
| 641 | case 0: |
| 642 | stlbsel = 0; |
| 643 | sesel = esel; |
| 644 | break; |
| 645 | |
| 646 | case 1: { |
| 647 | gfn_t gfn = gpaddr >> PAGE_SHIFT; |
| 648 | struct tlbe *gtlbe |
| 649 | = &vcpu_e500->guest_tlb[tlbsel][esel]; |
| 650 | |
| 651 | stlbsel = 1; |
| 652 | sesel = kvmppc_e500_tlb1_map(vcpu_e500, eaddr, gfn, gtlbe); |
| 653 | break; |
| 654 | } |
| 655 | |
| 656 | default: |
| 657 | BUG(); |
| 658 | break; |
| 659 | } |
| 660 | write_host_tlbe(vcpu_e500, stlbsel, sesel); |
| 661 | } |
| 662 | |
| 663 | int kvmppc_e500_tlb_search(struct kvm_vcpu *vcpu, |
| 664 | gva_t eaddr, unsigned int pid, int as) |
| 665 | { |
| 666 | struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); |
| 667 | int esel, tlbsel; |
| 668 | |
| 669 | for (tlbsel = 0; tlbsel < 2; tlbsel++) { |
| 670 | esel = kvmppc_e500_tlb_index(vcpu_e500, eaddr, tlbsel, pid, as); |
| 671 | if (esel >= 0) |
| 672 | return index_of(tlbsel, esel); |
| 673 | } |
| 674 | |
| 675 | return -1; |
| 676 | } |
| 677 | |
Scott Wood | 5ce941e | 2011-04-27 17:24:21 -0500 | [diff] [blame] | 678 | void kvmppc_set_pid(struct kvm_vcpu *vcpu, u32 pid) |
| 679 | { |
| 680 | struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu); |
| 681 | |
| 682 | vcpu_e500->pid[0] = vcpu->arch.shadow_pid = |
| 683 | vcpu->arch.pid = pid; |
| 684 | } |
| 685 | |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 686 | void kvmppc_e500_tlb_setup(struct kvmppc_vcpu_e500 *vcpu_e500) |
| 687 | { |
| 688 | struct tlbe *tlbe; |
| 689 | |
| 690 | /* Insert large initial mapping for guest. */ |
| 691 | tlbe = &vcpu_e500->guest_tlb[1][0]; |
Liu Yu | 0cfb50e | 2009-06-05 14:54:29 +0800 | [diff] [blame] | 692 | tlbe->mas1 = MAS1_VALID | MAS1_TSIZE(BOOK3E_PAGESZ_256M); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 693 | tlbe->mas2 = 0; |
| 694 | tlbe->mas3 = E500_TLB_SUPER_PERM_MASK; |
| 695 | tlbe->mas7 = 0; |
| 696 | |
| 697 | /* 4K map for serial output. Used by kernel wrapper. */ |
| 698 | tlbe = &vcpu_e500->guest_tlb[1][1]; |
Liu Yu | 0cfb50e | 2009-06-05 14:54:29 +0800 | [diff] [blame] | 699 | tlbe->mas1 = MAS1_VALID | MAS1_TSIZE(BOOK3E_PAGESZ_4K); |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 700 | tlbe->mas2 = (0xe0004500 & 0xFFFFF000) | MAS2_I | MAS2_G; |
| 701 | tlbe->mas3 = (0xe0004500 & 0xFFFFF000) | E500_TLB_SUPER_PERM_MASK; |
| 702 | tlbe->mas7 = 0; |
| 703 | } |
| 704 | |
| 705 | int kvmppc_e500_tlb_init(struct kvmppc_vcpu_e500 *vcpu_e500) |
| 706 | { |
| 707 | tlb1_entry_num = mfspr(SPRN_TLB1CFG) & 0xFFF; |
| 708 | |
| 709 | vcpu_e500->guest_tlb_size[0] = KVM_E500_TLB0_SIZE; |
| 710 | vcpu_e500->guest_tlb[0] = |
| 711 | kzalloc(sizeof(struct tlbe) * KVM_E500_TLB0_SIZE, GFP_KERNEL); |
| 712 | if (vcpu_e500->guest_tlb[0] == NULL) |
| 713 | goto err_out; |
| 714 | |
| 715 | vcpu_e500->shadow_tlb_size[0] = KVM_E500_TLB0_SIZE; |
| 716 | vcpu_e500->shadow_tlb[0] = |
| 717 | kzalloc(sizeof(struct tlbe) * KVM_E500_TLB0_SIZE, GFP_KERNEL); |
| 718 | if (vcpu_e500->shadow_tlb[0] == NULL) |
| 719 | goto err_out_guest0; |
| 720 | |
| 721 | vcpu_e500->guest_tlb_size[1] = KVM_E500_TLB1_SIZE; |
| 722 | vcpu_e500->guest_tlb[1] = |
| 723 | kzalloc(sizeof(struct tlbe) * KVM_E500_TLB1_SIZE, GFP_KERNEL); |
| 724 | if (vcpu_e500->guest_tlb[1] == NULL) |
| 725 | goto err_out_shadow0; |
| 726 | |
| 727 | vcpu_e500->shadow_tlb_size[1] = tlb1_entry_num; |
| 728 | vcpu_e500->shadow_tlb[1] = |
| 729 | kzalloc(sizeof(struct tlbe) * tlb1_entry_num, GFP_KERNEL); |
| 730 | if (vcpu_e500->shadow_tlb[1] == NULL) |
| 731 | goto err_out_guest1; |
| 732 | |
| 733 | vcpu_e500->shadow_pages[0] = (struct page **) |
| 734 | kzalloc(sizeof(struct page *) * KVM_E500_TLB0_SIZE, GFP_KERNEL); |
| 735 | if (vcpu_e500->shadow_pages[0] == NULL) |
| 736 | goto err_out_shadow1; |
| 737 | |
| 738 | vcpu_e500->shadow_pages[1] = (struct page **) |
| 739 | kzalloc(sizeof(struct page *) * tlb1_entry_num, GFP_KERNEL); |
| 740 | if (vcpu_e500->shadow_pages[1] == NULL) |
| 741 | goto err_out_page0; |
| 742 | |
Liu Yu | da15bf4 | 2010-01-22 19:36:53 +0800 | [diff] [blame] | 743 | /* Init TLB configuration register */ |
| 744 | vcpu_e500->tlb0cfg = mfspr(SPRN_TLB0CFG) & ~0xfffUL; |
| 745 | vcpu_e500->tlb0cfg |= vcpu_e500->guest_tlb_size[0]; |
| 746 | vcpu_e500->tlb1cfg = mfspr(SPRN_TLB1CFG) & ~0xfffUL; |
| 747 | vcpu_e500->tlb1cfg |= vcpu_e500->guest_tlb_size[1]; |
| 748 | |
Hollis Blanchard | bc8080c | 2009-01-03 16:23:10 -0600 | [diff] [blame] | 749 | return 0; |
| 750 | |
| 751 | err_out_page0: |
| 752 | kfree(vcpu_e500->shadow_pages[0]); |
| 753 | err_out_shadow1: |
| 754 | kfree(vcpu_e500->shadow_tlb[1]); |
| 755 | err_out_guest1: |
| 756 | kfree(vcpu_e500->guest_tlb[1]); |
| 757 | err_out_shadow0: |
| 758 | kfree(vcpu_e500->shadow_tlb[0]); |
| 759 | err_out_guest0: |
| 760 | kfree(vcpu_e500->guest_tlb[0]); |
| 761 | err_out: |
| 762 | return -1; |
| 763 | } |
| 764 | |
| 765 | void kvmppc_e500_tlb_uninit(struct kvmppc_vcpu_e500 *vcpu_e500) |
| 766 | { |
| 767 | kfree(vcpu_e500->shadow_pages[1]); |
| 768 | kfree(vcpu_e500->shadow_pages[0]); |
| 769 | kfree(vcpu_e500->shadow_tlb[1]); |
| 770 | kfree(vcpu_e500->guest_tlb[1]); |
| 771 | kfree(vcpu_e500->shadow_tlb[0]); |
| 772 | kfree(vcpu_e500->guest_tlb[0]); |
| 773 | } |