blob: 8047379e575ad39cb47cdbb055131e9bb094bb4d [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
Dave Jones835c34a2007-10-12 21:10:53 -04003 * prepare to run common code
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 */
7
Andrey Ryabininbe3606f2017-03-13 19:33:37 +03008#define DISABLE_BRANCH_PROFILING
Kirill A. Shutemovad3fe522018-05-18 13:35:23 +03009
10/* cpu_feature_enabled() cannot be used this early */
11#define USE_EARLY_PGTABLE_L5
12
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/init.h>
14#include <linux/linkage.h>
15#include <linux/types.h>
16#include <linux/kernel.h>
17#include <linux/string.h>
18#include <linux/percpu.h>
Thomas Gleixnereaf76e82008-01-30 13:30:19 +010019#include <linux/start_kernel.h>
Huang, Ying8b664aa2008-03-28 10:49:44 +080020#include <linux/io.h>
Yinghai Lu72d7c3b2010-08-25 13:39:17 -070021#include <linux/memblock.h>
Tom Lendacky5868f362017-07-17 16:10:05 -050022#include <linux/mem_encrypt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023
24#include <asm/processor.h>
25#include <asm/proto.h>
26#include <asm/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#include <asm/setup.h>
28#include <asm/desc.h>
Siddha, Suresh Bf6c2e332005-11-05 17:25:53 +010029#include <asm/pgtable.h>
Vivek Goyalcfd243d2007-05-02 19:27:07 +020030#include <asm/tlbflush.h>
Andi Kleen2bc04142005-11-05 17:25:53 +010031#include <asm/sections.h>
Thomas Gleixner718fc132008-01-30 13:30:17 +010032#include <asm/kdebug.h>
Ingo Molnar66441bd2017-01-27 10:27:10 +010033#include <asm/e820/api.h>
Thomas Gleixner47a3d5d2009-08-29 15:03:59 +020034#include <asm/bios_ebda.h>
H. Peter Anvin5dcd14e2013-01-29 01:05:24 -080035#include <asm/bootparam_utils.h>
Fenghua Yufeddc9d2012-12-20 23:44:30 -080036#include <asm/microcode.h>
Andrey Ryabininef7f0d62015-02-13 14:39:25 -080037#include <asm/kasan.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070038
H. Peter Anvin8170e6b2013-01-24 12:19:52 -080039/*
40 * Manage page tables very early on.
41 */
H. Peter Anvin8170e6b2013-01-24 12:19:52 -080042extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +030043static unsigned int __initdata next_early_pgt;
Linus Torvalds5e427ec2013-05-20 11:36:03 -070044pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
H. Peter Anvin8170e6b2013-01-24 12:19:52 -080045
Kirill A. Shutemove626e6b2018-02-14 14:16:51 +030046#ifdef CONFIG_X86_5LEVEL
Kirill A. Shutemov51be1332018-06-23 01:08:40 +030047unsigned int __pgtable_l5_enabled __ro_after_init;
Kirill A. Shutemovb16e7702018-02-14 21:25:35 +030048unsigned int pgdir_shift __ro_after_init = 39;
Kirill A. Shutemovc65e7742018-02-14 14:16:53 +030049EXPORT_SYMBOL(pgdir_shift);
Kirill A. Shutemovb16e7702018-02-14 21:25:35 +030050unsigned int ptrs_per_p4d __ro_after_init = 1;
Kirill A. Shutemovc65e7742018-02-14 14:16:53 +030051EXPORT_SYMBOL(ptrs_per_p4d);
Kirill A. Shutemove626e6b2018-02-14 14:16:51 +030052#endif
53
Kirill A. Shutemoveedb92a2018-02-14 14:16:50 +030054#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT
Kirill A. Shutemov4fa56622018-02-14 21:25:36 +030055unsigned long page_offset_base __ro_after_init = __PAGE_OFFSET_BASE_L4;
Kirill A. Shutemoveedb92a2018-02-14 14:16:50 +030056EXPORT_SYMBOL(page_offset_base);
Kirill A. Shutemova7412542018-02-14 21:25:37 +030057unsigned long vmalloc_base __ro_after_init = __VMALLOC_BASE_L4;
Kirill A. Shutemoveedb92a2018-02-14 14:16:50 +030058EXPORT_SYMBOL(vmalloc_base);
Kirill A. Shutemov9b46a052018-02-14 21:25:38 +030059unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4;
Kirill A. Shutemoveedb92a2018-02-14 14:16:50 +030060EXPORT_SYMBOL(vmemmap_base);
61#endif
62
Kirill A. Shutemov26179672017-06-16 14:30:24 +030063#define __head __section(.head.text)
64
65static void __head *fixup_pointer(void *ptr, unsigned long physaddr)
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +030066{
67 return ptr - (void *)_text + (void *)physaddr;
68}
69
Kirill A. Shutemov4fa56622018-02-14 21:25:36 +030070static unsigned long __head *fixup_long(void *ptr, unsigned long physaddr)
71{
72 return fixup_pointer(ptr, physaddr);
73}
74
Kirill A. Shutemov4c2b4052018-02-14 21:25:34 +030075#ifdef CONFIG_X86_5LEVEL
76static unsigned int __head *fixup_int(void *ptr, unsigned long physaddr)
77{
78 return fixup_pointer(ptr, physaddr);
79}
80
Kirill A. Shutemov6f9dd322018-02-14 21:25:39 +030081static bool __head check_la57_support(unsigned long physaddr)
Kirill A. Shutemov4c2b4052018-02-14 21:25:34 +030082{
Kirill A. Shutemov372fddf2018-05-18 13:35:25 +030083 /*
84 * 5-level paging is detected and enabled at kernel decomression
85 * stage. Only check if it has been enabled there.
86 */
87 if (!(native_read_cr4() & X86_CR4_LA57))
Kirill A. Shutemov6f9dd322018-02-14 21:25:39 +030088 return false;
Kirill A. Shutemov4c2b4052018-02-14 21:25:34 +030089
Kirill A. Shutemovad3fe522018-05-18 13:35:23 +030090 *fixup_int(&__pgtable_l5_enabled, physaddr) = 1;
Kirill A. Shutemovb16e7702018-02-14 21:25:35 +030091 *fixup_int(&pgdir_shift, physaddr) = 48;
92 *fixup_int(&ptrs_per_p4d, physaddr) = 512;
Kirill A. Shutemov4fa56622018-02-14 21:25:36 +030093 *fixup_long(&page_offset_base, physaddr) = __PAGE_OFFSET_BASE_L5;
Kirill A. Shutemova7412542018-02-14 21:25:37 +030094 *fixup_long(&vmalloc_base, physaddr) = __VMALLOC_BASE_L5;
Kirill A. Shutemov9b46a052018-02-14 21:25:38 +030095 *fixup_long(&vmemmap_base, physaddr) = __VMEMMAP_BASE_L5;
Kirill A. Shutemov6f9dd322018-02-14 21:25:39 +030096
97 return true;
Kirill A. Shutemov4c2b4052018-02-14 21:25:34 +030098}
99#else
Kirill A. Shutemov6f9dd322018-02-14 21:25:39 +0300100static bool __head check_la57_support(unsigned long physaddr)
101{
102 return false;
103}
Kirill A. Shutemov4c2b4052018-02-14 21:25:34 +0300104#endif
105
Alexander Potapenko4a09f022018-05-09 11:18:22 +0200106/* Code in __startup_64() can be relocated during execution, but the compiler
107 * doesn't have to generate PC-relative relocations when accessing globals from
108 * that function. Clang actually does not generate them, which leads to
109 * boot-time crashes. To work around this problem, every global pointer must
110 * be adjusted using fixup_pointer().
111 */
Tom Lendackyaca20d52017-07-17 16:10:35 -0500112unsigned long __head __startup_64(unsigned long physaddr,
113 struct boot_params *bp)
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +0300114{
Kirill A. Shutemov6f9dd322018-02-14 21:25:39 +0300115 unsigned long load_delta, *p;
Tom Lendacky5868f362017-07-17 16:10:05 -0500116 unsigned long pgtable_flags;
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +0300117 pgdval_t *pgd;
Kirill A. Shutemov032370b2017-06-06 14:31:28 +0300118 p4dval_t *p4d;
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +0300119 pudval_t *pud;
120 pmdval_t *pmd, pmd_entry;
Alexander Potapenko4a09f022018-05-09 11:18:22 +0200121 pteval_t *mask_ptr;
Kirill A. Shutemov6f9dd322018-02-14 21:25:39 +0300122 bool la57;
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +0300123 int i;
Alexander Potapenko187e91f2017-08-16 21:08:08 +0200124 unsigned int *next_pgt_ptr;
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +0300125
Kirill A. Shutemov6f9dd322018-02-14 21:25:39 +0300126 la57 = check_la57_support(physaddr);
Kirill A. Shutemov4c2b4052018-02-14 21:25:34 +0300127
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +0300128 /* Is the address too large? */
129 if (physaddr >> MAX_PHYSMEM_BITS)
130 for (;;);
131
132 /*
133 * Compute the delta between the address I am compiled to run at
134 * and the address I am actually running at.
135 */
136 load_delta = physaddr - (unsigned long)(_text - __START_KERNEL_map);
137
138 /* Is the address not 2M aligned? */
139 if (load_delta & ~PMD_PAGE_MASK)
140 for (;;);
141
Tom Lendacky5868f362017-07-17 16:10:05 -0500142 /* Activate Secure Memory Encryption (SME) if supported and enabled */
Tom Lendackyaca20d52017-07-17 16:10:35 -0500143 sme_enable(bp);
Tom Lendacky5868f362017-07-17 16:10:05 -0500144
145 /* Include the SME encryption mask in the fixup value */
146 load_delta += sme_get_me_mask();
147
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +0300148 /* Fixup the physical addresses in the page table */
149
Kirill A. Shutemov65ade2f2017-06-06 14:31:27 +0300150 pgd = fixup_pointer(&early_top_pgt, physaddr);
Kirill A. Shutemov6f9dd322018-02-14 21:25:39 +0300151 p = pgd + pgd_index(__START_KERNEL_map);
152 if (la57)
153 *p = (unsigned long)level4_kernel_pgt;
154 else
155 *p = (unsigned long)level3_kernel_pgt;
156 *p += _PAGE_TABLE_NOENC - __START_KERNEL_map + load_delta;
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +0300157
Kirill A. Shutemov6f9dd322018-02-14 21:25:39 +0300158 if (la57) {
Kirill A. Shutemov032370b2017-06-06 14:31:28 +0300159 p4d = fixup_pointer(&level4_kernel_pgt, physaddr);
160 p4d[511] += load_delta;
161 }
162
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +0300163 pud = fixup_pointer(&level3_kernel_pgt, physaddr);
164 pud[510] += load_delta;
165 pud[511] += load_delta;
166
167 pmd = fixup_pointer(level2_fixmap_pgt, physaddr);
168 pmd[506] += load_delta;
169
170 /*
171 * Set up the identity mapping for the switchover. These
172 * entries should *NOT* have the global bit set! This also
173 * creates a bunch of nonsense entries but that is fine --
174 * it avoids problems around wraparound.
175 */
176
Alexander Potapenko187e91f2017-08-16 21:08:08 +0200177 next_pgt_ptr = fixup_pointer(&next_early_pgt, physaddr);
178 pud = fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], physaddr);
179 pmd = fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], physaddr);
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +0300180
Tom Lendacky21729f82017-07-17 16:10:07 -0500181 pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +0300182
Kirill A. Shutemov6f9dd322018-02-14 21:25:39 +0300183 if (la57) {
Kirill A. Shutemov032370b2017-06-06 14:31:28 +0300184 p4d = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);
185
186 i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
Tom Lendacky5868f362017-07-17 16:10:05 -0500187 pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
188 pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;
Kirill A. Shutemov032370b2017-06-06 14:31:28 +0300189
190 i = (physaddr >> P4D_SHIFT) % PTRS_PER_P4D;
Tom Lendacky5868f362017-07-17 16:10:05 -0500191 p4d[i + 0] = (pgdval_t)pud + pgtable_flags;
192 p4d[i + 1] = (pgdval_t)pud + pgtable_flags;
Kirill A. Shutemov032370b2017-06-06 14:31:28 +0300193 } else {
194 i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
Tom Lendacky5868f362017-07-17 16:10:05 -0500195 pgd[i + 0] = (pgdval_t)pud + pgtable_flags;
196 pgd[i + 1] = (pgdval_t)pud + pgtable_flags;
Kirill A. Shutemov032370b2017-06-06 14:31:28 +0300197 }
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +0300198
199 i = (physaddr >> PUD_SHIFT) % PTRS_PER_PUD;
Tom Lendacky5868f362017-07-17 16:10:05 -0500200 pud[i + 0] = (pudval_t)pmd + pgtable_flags;
201 pud[i + 1] = (pudval_t)pmd + pgtable_flags;
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +0300202
203 pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
Dave Hansenfb43d6c2018-04-06 13:55:09 -0700204 /* Filter out unsupported __PAGE_KERNEL_* bits: */
Alexander Potapenko4a09f022018-05-09 11:18:22 +0200205 mask_ptr = fixup_pointer(&__supported_pte_mask, physaddr);
206 pmd_entry &= *mask_ptr;
Tom Lendacky5868f362017-07-17 16:10:05 -0500207 pmd_entry += sme_get_me_mask();
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +0300208 pmd_entry += physaddr;
209
210 for (i = 0; i < DIV_ROUND_UP(_end - _text, PMD_SIZE); i++) {
211 int idx = i + (physaddr >> PMD_SHIFT) % PTRS_PER_PMD;
212 pmd[idx] = pmd_entry + i * PMD_SIZE;
213 }
214
215 /*
216 * Fixup the kernel text+data virtual addresses. Note that
217 * we might write invalid pmds, when the kernel is relocated
218 * cleanup_highmap() fixes this up along with the mappings
219 * beyond _end.
220 */
221
222 pmd = fixup_pointer(level2_kernel_pgt, physaddr);
223 for (i = 0; i < PTRS_PER_PMD; i++) {
224 if (pmd[i] & _PAGE_PRESENT)
225 pmd[i] += load_delta;
226 }
227
Tom Lendacky5868f362017-07-17 16:10:05 -0500228 /*
229 * Fixup phys_base - remove the memory encryption mask to obtain
230 * the true physical address.
231 */
Kirill A. Shutemov4fa56622018-02-14 21:25:36 +0300232 *fixup_long(&phys_base, physaddr) += load_delta - sme_get_me_mask();
Tom Lendacky5868f362017-07-17 16:10:05 -0500233
Tom Lendacky107cd252018-01-10 13:26:34 -0600234 /* Encrypt the kernel and related (if SME is active) */
235 sme_encrypt_kernel(bp);
Tom Lendacky5868f362017-07-17 16:10:05 -0500236
237 /*
238 * Return the SME encryption mask (if SME is active) to be used as a
239 * modifier for the initial pgdir entry programmed into CR3.
240 */
241 return sme_get_me_mask();
242}
243
244unsigned long __startup_secondary_64(void)
245{
246 /*
247 * Return the SME encryption mask (if SME is active) to be used as a
248 * modifier for the initial pgdir entry programmed into CR3.
249 */
250 return sme_get_me_mask();
Kirill A. Shutemovc88d71502017-06-06 14:31:26 +0300251}
252
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800253/* Wipe all early page tables except for the kernel symbol map */
254static void __init reset_early_page_tables(void)
Vivek Goyalcfd243d2007-05-02 19:27:07 +0200255{
Kirill A. Shutemov65ade2f2017-06-06 14:31:27 +0300256 memset(early_top_pgt, 0, sizeof(pgd_t)*(PTRS_PER_PGD-1));
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800257 next_early_pgt = 0;
Tom Lendacky21729f82017-07-17 16:10:07 -0500258 write_cr3(__sme_pa_nodebug(early_top_pgt));
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800259}
260
261/* Create a new PMD entry */
Tom Lendackyb9d05202017-07-17 16:10:11 -0500262int __init __early_make_pgtable(unsigned long address, pmdval_t pmd)
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800263{
264 unsigned long physaddr = address - __PAGE_OFFSET;
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800265 pgdval_t pgd, *pgd_p;
Kirill A. Shutemov032370b2017-06-06 14:31:28 +0300266 p4dval_t p4d, *p4d_p;
Yinghai Lu6b9c75a2013-01-24 12:19:53 -0800267 pudval_t pud, *pud_p;
Tom Lendackyb9d05202017-07-17 16:10:11 -0500268 pmdval_t *pmd_p;
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800269
270 /* Invalid address or early pgt is done ? */
Kirill A. Shutemov65ade2f2017-06-06 14:31:27 +0300271 if (physaddr >= MAXMEM || read_cr3_pa() != __pa_nodebug(early_top_pgt))
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800272 return -1;
273
Yinghai Lu6b9c75a2013-01-24 12:19:53 -0800274again:
Kirill A. Shutemov65ade2f2017-06-06 14:31:27 +0300275 pgd_p = &early_top_pgt[pgd_index(address)].pgd;
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800276 pgd = *pgd_p;
277
278 /*
279 * The use of __START_KERNEL_map rather than __PAGE_OFFSET here is
280 * critical -- __PAGE_OFFSET would point us back into the dynamic
281 * range and we might end up looping forever...
282 */
Kirill A. Shutemoved7588d2018-05-18 13:35:24 +0300283 if (!pgtable_l5_enabled())
Kirill A. Shutemov032370b2017-06-06 14:31:28 +0300284 p4d_p = pgd_p;
285 else if (pgd)
286 p4d_p = (p4dval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
287 else {
288 if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
289 reset_early_page_tables();
290 goto again;
291 }
292
293 p4d_p = (p4dval_t *)early_dynamic_pgts[next_early_pgt++];
294 memset(p4d_p, 0, sizeof(*p4d_p) * PTRS_PER_P4D);
295 *pgd_p = (pgdval_t)p4d_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
296 }
297 p4d_p += p4d_index(address);
298 p4d = *p4d_p;
299
300 if (p4d)
301 pud_p = (pudval_t *)((p4d & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
Yinghai Lu6b9c75a2013-01-24 12:19:53 -0800302 else {
303 if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800304 reset_early_page_tables();
Yinghai Lu6b9c75a2013-01-24 12:19:53 -0800305 goto again;
306 }
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800307
308 pud_p = (pudval_t *)early_dynamic_pgts[next_early_pgt++];
Alexander Kuleshova91bbe02016-02-09 19:44:54 +0600309 memset(pud_p, 0, sizeof(*pud_p) * PTRS_PER_PUD);
Kirill A. Shutemov032370b2017-06-06 14:31:28 +0300310 *p4d_p = (p4dval_t)pud_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800311 }
Yinghai Lu6b9c75a2013-01-24 12:19:53 -0800312 pud_p += pud_index(address);
313 pud = *pud_p;
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800314
Yinghai Lu6b9c75a2013-01-24 12:19:53 -0800315 if (pud)
316 pmd_p = (pmdval_t *)((pud & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
317 else {
318 if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
319 reset_early_page_tables();
320 goto again;
321 }
322
323 pmd_p = (pmdval_t *)early_dynamic_pgts[next_early_pgt++];
Alexander Kuleshova91bbe02016-02-09 19:44:54 +0600324 memset(pmd_p, 0, sizeof(*pmd_p) * PTRS_PER_PMD);
Yinghai Lu6b9c75a2013-01-24 12:19:53 -0800325 *pud_p = (pudval_t)pmd_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800326 }
Yinghai Lu6b9c75a2013-01-24 12:19:53 -0800327 pmd_p[pmd_index(address)] = pmd;
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800328
329 return 0;
Vivek Goyalcfd243d2007-05-02 19:27:07 +0200330}
331
Tom Lendackyb9d05202017-07-17 16:10:11 -0500332int __init early_make_pgtable(unsigned long address)
333{
334 unsigned long physaddr = address - __PAGE_OFFSET;
335 pmdval_t pmd;
336
337 pmd = (physaddr & PMD_MASK) + early_pmd_flags;
338
339 return __early_make_pgtable(address, pmd);
340}
341
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342/* Don't add a printk in there. printk relies on the PDA which is not initialized
343 yet. */
344static void __init clear_bss(void)
345{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346 memset(__bss_start, 0,
Andi Kleen2bc04142005-11-05 17:25:53 +0100347 (unsigned long) __bss_stop - (unsigned long) __bss_start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348}
349
Yinghai Luf1da8342013-01-24 12:19:57 -0800350static unsigned long get_cmd_line_ptr(void)
351{
352 unsigned long cmd_line_ptr = boot_params.hdr.cmd_line_ptr;
353
Yinghai Luee92d812013-01-28 20:16:44 -0800354 cmd_line_ptr |= (u64)boot_params.ext_cmd_line_ptr << 32;
355
Yinghai Luf1da8342013-01-24 12:19:57 -0800356 return cmd_line_ptr;
357}
358
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359static void __init copy_bootdata(char *real_mode_data)
360{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 char * command_line;
Yinghai Luf1da8342013-01-24 12:19:57 -0800362 unsigned long cmd_line_ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363
Tom Lendackyb9d05202017-07-17 16:10:11 -0500364 /*
365 * If SME is active, this will create decrypted mappings of the
366 * boot data in advance of the copy operations.
367 */
368 sme_map_bootdata(real_mode_data);
369
H. Peter Anvin30c82642007-10-15 17:13:22 -0700370 memcpy(&boot_params, real_mode_data, sizeof boot_params);
H. Peter Anvin5dcd14e2013-01-29 01:05:24 -0800371 sanitize_boot_params(&boot_params);
Yinghai Luf1da8342013-01-24 12:19:57 -0800372 cmd_line_ptr = get_cmd_line_ptr();
373 if (cmd_line_ptr) {
374 command_line = __va(cmd_line_ptr);
H. Peter Anvin30c82642007-10-15 17:13:22 -0700375 memcpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376 }
Tom Lendackyb9d05202017-07-17 16:10:11 -0500377
378 /*
379 * The old boot data is no longer needed and won't be reserved,
380 * freeing up that memory for use by the system. If SME is active,
381 * we need to remove the mappings that were created so that the
382 * memory doesn't remain mapped as decrypted.
383 */
384 sme_unmap_bootdata(real_mode_data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385}
386
Andi Kleen2605fc22014-05-02 00:44:37 +0200387asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388{
Ingo Molnarb4e04092008-02-21 13:45:16 +0100389 /*
390 * Build-time sanity checks on the kernel image and module
391 * area mappings. (these are purely build-time and produce no code)
392 */
Borislav Petkov8e3c2a82013-03-04 21:16:17 +0100393 BUILD_BUG_ON(MODULES_VADDR < __START_KERNEL_map);
394 BUILD_BUG_ON(MODULES_VADDR - __START_KERNEL_map < KERNEL_IMAGE_SIZE);
Ingo Molnarb4e04092008-02-21 13:45:16 +0100395 BUILD_BUG_ON(MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
Borislav Petkov8e3c2a82013-03-04 21:16:17 +0100396 BUILD_BUG_ON((__START_KERNEL_map & ~PMD_MASK) != 0);
Ingo Molnarb4e04092008-02-21 13:45:16 +0100397 BUILD_BUG_ON((MODULES_VADDR & ~PMD_MASK) != 0);
398 BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
Kirill A. Shutemovc65e7742018-02-14 14:16:53 +0300399 MAYBE_BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
Ingo Molnarb4e04092008-02-21 13:45:16 +0100400 (__START_KERNEL & PGDIR_MASK)));
Jan Beulich66d4bdf2008-07-31 16:48:31 +0100401 BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);
Ingo Molnarb4e04092008-02-21 13:45:16 +0100402
Andy Lutomirski1e02ce42014-10-24 15:58:08 -0700403 cr4_init_shadow();
404
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800405 /* Kill off the identity-map trampoline */
406 reset_early_page_tables();
407
Yinghai Lu3df0af02006-12-07 02:14:12 +0100408 clear_bss();
409
Kirill A. Shutemov65ade2f2017-06-06 14:31:27 +0300410 clear_page(init_top_pgt);
Andrey Ryabinind0f77d42015-07-02 12:09:33 +0300411
Tom Lendacky21729f82017-07-17 16:10:07 -0500412 /*
413 * SME support may update early_pmd_flags to include the memory
414 * encryption mask, so it needs to be called before anything
415 * that may generate a page fault.
416 */
417 sme_early_init();
418
Alexander Popov5d5aa3c2015-07-02 12:09:34 +0300419 kasan_early_init();
420
Thomas Gleixner588787f2017-08-28 08:47:47 +0200421 idt_setup_early_handler();
Siddha, Suresh Bf6c2e332005-11-05 17:25:53 +0100422
Yinghai Lufa2bbce2013-01-24 12:19:49 -0800423 copy_bootdata(__va(real_mode_data));
424
Fenghua Yufeddc9d2012-12-20 23:44:30 -0800425 /*
426 * Load microcode early on BSP.
427 */
428 load_ucode_bsp();
429
Kirill A. Shutemov65ade2f2017-06-06 14:31:27 +0300430 /* set init_top_pgt kernel high mapping*/
431 init_top_pgt[511] = early_top_pgt[511];
H. Peter Anvin8170e6b2013-01-24 12:19:52 -0800432
Jeremy Fitzhardingef97013f2008-06-25 00:19:18 -0400433 x86_64_start_reservations(real_mode_data);
434}
435
436void __init x86_64_start_reservations(char *real_mode_data)
437{
Yinghai Lufa2bbce2013-01-24 12:19:49 -0800438 /* version is always not zero if it is copied */
439 if (!boot_params.hdr.version)
440 copy_bootdata(__va(real_mode_data));
Yinghai Lu9de819f2008-01-30 13:30:46 +0100441
Luis R. Rodriguez8d152e72016-04-13 17:04:34 -0700442 x86_early_init_platform_quirks();
Andi Kleen75175272008-01-30 13:33:17 +0100443
Andy Shevchenko3fda5bb2016-01-15 22:11:07 +0200444 switch (boot_params.hdr.hardware_subarch) {
445 case X86_SUBARCH_INTEL_MID:
446 x86_intel_mid_early_setup();
447 break;
448 default:
449 break;
450 }
451
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452 start_kernel();
453}