blob: 86ef06d3dbe3a067ee00fe31c924b11be7340d31 [file] [log] [blame]
Dave Young2965faa2015-09-09 15:38:55 -07001/*
2 * kexec.c - kexec system call core code.
3 * Copyright (C) 2002-2004 Eric Biederman <ebiederm@xmission.com>
4 *
5 * This source code is licensed under the GNU General Public License,
6 * Version 2. See the file COPYING for more details.
7 */
8
Minfei Huangde90a6b2015-11-06 16:32:45 -08009#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
Dave Young2965faa2015-09-09 15:38:55 -070010
11#include <linux/capability.h>
12#include <linux/mm.h>
13#include <linux/file.h>
14#include <linux/slab.h>
15#include <linux/fs.h>
16#include <linux/kexec.h>
17#include <linux/mutex.h>
18#include <linux/list.h>
19#include <linux/highmem.h>
20#include <linux/syscalls.h>
21#include <linux/reboot.h>
22#include <linux/ioport.h>
23#include <linux/hardirq.h>
24#include <linux/elf.h>
25#include <linux/elfcore.h>
26#include <linux/utsname.h>
27#include <linux/numa.h>
28#include <linux/suspend.h>
29#include <linux/device.h>
30#include <linux/freezer.h>
31#include <linux/pm.h>
32#include <linux/cpu.h>
33#include <linux/uaccess.h>
34#include <linux/io.h>
35#include <linux/console.h>
36#include <linux/vmalloc.h>
37#include <linux/swap.h>
38#include <linux/syscore_ops.h>
39#include <linux/compiler.h>
40#include <linux/hugetlb.h>
Josh Poimboeufc207aee2017-06-28 10:11:06 -050041#include <linux/frame.h>
Dave Young2965faa2015-09-09 15:38:55 -070042
43#include <asm/page.h>
44#include <asm/sections.h>
45
46#include <crypto/hash.h>
47#include <crypto/sha.h>
48#include "kexec_internal.h"
49
50DEFINE_MUTEX(kexec_mutex);
51
52/* Per cpu memory for storing cpu states in case of system crash. */
53note_buf_t __percpu *crash_notes;
54
Dave Young2965faa2015-09-09 15:38:55 -070055/* Flag to indicate we are going to kexec a new kernel */
56bool kexec_in_progress = false;
57
58
59/* Location of the reserved area for the crash kernel */
60struct resource crashk_res = {
61 .name = "Crash kernel",
62 .start = 0,
63 .end = 0,
Toshi Kani1a085d02016-01-26 21:57:23 +010064 .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM,
65 .desc = IORES_DESC_CRASH_KERNEL
Dave Young2965faa2015-09-09 15:38:55 -070066};
67struct resource crashk_low_res = {
68 .name = "Crash kernel",
69 .start = 0,
70 .end = 0,
Toshi Kani1a085d02016-01-26 21:57:23 +010071 .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM,
72 .desc = IORES_DESC_CRASH_KERNEL
Dave Young2965faa2015-09-09 15:38:55 -070073};
74
75int kexec_should_crash(struct task_struct *p)
76{
77 /*
78 * If crash_kexec_post_notifiers is enabled, don't run
79 * crash_kexec() here yet, which must be run after panic
80 * notifiers in panic().
81 */
82 if (crash_kexec_post_notifiers)
83 return 0;
84 /*
85 * There are 4 panic() calls in do_exit() path, each of which
86 * corresponds to each of these 4 conditions.
87 */
88 if (in_interrupt() || !p->pid || is_global_init(p) || panic_on_oops)
89 return 1;
90 return 0;
91}
92
Petr Tesarik21db79e2016-08-02 14:06:16 -070093int kexec_crash_loaded(void)
94{
95 return !!kexec_crash_image;
96}
97EXPORT_SYMBOL_GPL(kexec_crash_loaded);
98
Dave Young2965faa2015-09-09 15:38:55 -070099/*
100 * When kexec transitions to the new kernel there is a one-to-one
101 * mapping between physical and virtual addresses. On processors
102 * where you can disable the MMU this is trivial, and easy. For
103 * others it is still a simple predictable page table to setup.
104 *
105 * In that environment kexec copies the new kernel to its final
106 * resting place. This means I can only support memory whose
107 * physical address can fit in an unsigned long. In particular
108 * addresses where (pfn << PAGE_SHIFT) > ULONG_MAX cannot be handled.
109 * If the assembly stub has more restrictive requirements
110 * KEXEC_SOURCE_MEMORY_LIMIT and KEXEC_DEST_MEMORY_LIMIT can be
111 * defined more restrictively in <asm/kexec.h>.
112 *
113 * The code for the transition from the current kernel to the
114 * the new kernel is placed in the control_code_buffer, whose size
115 * is given by KEXEC_CONTROL_PAGE_SIZE. In the best case only a single
116 * page of memory is necessary, but some architectures require more.
117 * Because this memory must be identity mapped in the transition from
118 * virtual to physical addresses it must live in the range
119 * 0 - TASK_SIZE, as only the user space mappings are arbitrarily
120 * modifiable.
121 *
122 * The assembly stub in the control code buffer is passed a linked list
123 * of descriptor pages detailing the source pages of the new kernel,
124 * and the destination addresses of those source pages. As this data
125 * structure is not used in the context of the current OS, it must
126 * be self-contained.
127 *
128 * The code has been made to work with highmem pages and will use a
129 * destination page in its final resting place (if it happens
130 * to allocate it). The end product of this is that most of the
131 * physical address space, and most of RAM can be used.
132 *
133 * Future directions include:
134 * - allocating a page table with the control code buffer identity
135 * mapped, to simplify machine_kexec and make kexec_on_panic more
136 * reliable.
137 */
138
139/*
140 * KIMAGE_NO_DEST is an impossible destination address..., for
141 * allocating pages whose destination address we do not care about.
142 */
143#define KIMAGE_NO_DEST (-1UL)
zhong jiang1730f142016-08-02 14:06:22 -0700144#define PAGE_COUNT(x) (((x) + PAGE_SIZE - 1) >> PAGE_SHIFT)
Dave Young2965faa2015-09-09 15:38:55 -0700145
146static struct page *kimage_alloc_page(struct kimage *image,
147 gfp_t gfp_mask,
148 unsigned long dest);
149
150int sanity_check_segment_list(struct kimage *image)
151{
Minfei Huang4caf9612016-08-02 14:05:45 -0700152 int i;
Dave Young2965faa2015-09-09 15:38:55 -0700153 unsigned long nr_segments = image->nr_segments;
zhong jiang1730f142016-08-02 14:06:22 -0700154 unsigned long total_pages = 0;
Dave Young2965faa2015-09-09 15:38:55 -0700155
156 /*
157 * Verify we have good destination addresses. The caller is
158 * responsible for making certain we don't attempt to load
159 * the new image into invalid or reserved areas of RAM. This
160 * just verifies it is an address we can use.
161 *
162 * Since the kernel does everything in page size chunks ensure
163 * the destination addresses are page aligned. Too many
164 * special cases crop of when we don't do this. The most
165 * insidious is getting overlapping destination addresses
166 * simply because addresses are changed to page size
167 * granularity.
168 */
Dave Young2965faa2015-09-09 15:38:55 -0700169 for (i = 0; i < nr_segments; i++) {
170 unsigned long mstart, mend;
171
172 mstart = image->segment[i].mem;
173 mend = mstart + image->segment[i].memsz;
Russell King465d3772016-08-02 14:05:57 -0700174 if (mstart > mend)
175 return -EADDRNOTAVAIL;
Dave Young2965faa2015-09-09 15:38:55 -0700176 if ((mstart & ~PAGE_MASK) || (mend & ~PAGE_MASK))
Minfei Huang4caf9612016-08-02 14:05:45 -0700177 return -EADDRNOTAVAIL;
Dave Young2965faa2015-09-09 15:38:55 -0700178 if (mend >= KEXEC_DESTINATION_MEMORY_LIMIT)
Minfei Huang4caf9612016-08-02 14:05:45 -0700179 return -EADDRNOTAVAIL;
Dave Young2965faa2015-09-09 15:38:55 -0700180 }
181
182 /* Verify our destination addresses do not overlap.
183 * If we alloed overlapping destination addresses
184 * through very weird things can happen with no
185 * easy explanation as one segment stops on another.
186 */
Dave Young2965faa2015-09-09 15:38:55 -0700187 for (i = 0; i < nr_segments; i++) {
188 unsigned long mstart, mend;
189 unsigned long j;
190
191 mstart = image->segment[i].mem;
192 mend = mstart + image->segment[i].memsz;
193 for (j = 0; j < i; j++) {
194 unsigned long pstart, pend;
195
196 pstart = image->segment[j].mem;
197 pend = pstart + image->segment[j].memsz;
198 /* Do the segments overlap ? */
199 if ((mend > pstart) && (mstart < pend))
Minfei Huang4caf9612016-08-02 14:05:45 -0700200 return -EINVAL;
Dave Young2965faa2015-09-09 15:38:55 -0700201 }
202 }
203
204 /* Ensure our buffer sizes are strictly less than
205 * our memory sizes. This should always be the case,
206 * and it is easier to check up front than to be surprised
207 * later on.
208 */
Dave Young2965faa2015-09-09 15:38:55 -0700209 for (i = 0; i < nr_segments; i++) {
210 if (image->segment[i].bufsz > image->segment[i].memsz)
Minfei Huang4caf9612016-08-02 14:05:45 -0700211 return -EINVAL;
Dave Young2965faa2015-09-09 15:38:55 -0700212 }
213
214 /*
zhong jiang1730f142016-08-02 14:06:22 -0700215 * Verify that no more than half of memory will be consumed. If the
216 * request from userspace is too large, a large amount of time will be
217 * wasted allocating pages, which can cause a soft lockup.
218 */
219 for (i = 0; i < nr_segments; i++) {
220 if (PAGE_COUNT(image->segment[i].memsz) > totalram_pages / 2)
221 return -EINVAL;
222
223 total_pages += PAGE_COUNT(image->segment[i].memsz);
224 }
225
226 if (total_pages > totalram_pages / 2)
227 return -EINVAL;
228
229 /*
Dave Young2965faa2015-09-09 15:38:55 -0700230 * Verify we have good destination addresses. Normally
231 * the caller is responsible for making certain we don't
232 * attempt to load the new image into invalid or reserved
233 * areas of RAM. But crash kernels are preloaded into a
234 * reserved area of ram. We must ensure the addresses
235 * are in the reserved area otherwise preloading the
236 * kernel could corrupt things.
237 */
238
239 if (image->type == KEXEC_TYPE_CRASH) {
Dave Young2965faa2015-09-09 15:38:55 -0700240 for (i = 0; i < nr_segments; i++) {
241 unsigned long mstart, mend;
242
243 mstart = image->segment[i].mem;
244 mend = mstart + image->segment[i].memsz - 1;
245 /* Ensure we are within the crash kernel limits */
Russell King43546d82016-08-02 14:06:04 -0700246 if ((mstart < phys_to_boot_phys(crashk_res.start)) ||
247 (mend > phys_to_boot_phys(crashk_res.end)))
Minfei Huang4caf9612016-08-02 14:05:45 -0700248 return -EADDRNOTAVAIL;
Dave Young2965faa2015-09-09 15:38:55 -0700249 }
250 }
251
252 return 0;
253}
254
255struct kimage *do_kimage_alloc_init(void)
256{
257 struct kimage *image;
258
259 /* Allocate a controlling structure */
260 image = kzalloc(sizeof(*image), GFP_KERNEL);
261 if (!image)
262 return NULL;
263
264 image->head = 0;
265 image->entry = &image->head;
266 image->last_entry = &image->head;
267 image->control_page = ~0; /* By default this does not apply */
268 image->type = KEXEC_TYPE_DEFAULT;
269
270 /* Initialize the list of control pages */
271 INIT_LIST_HEAD(&image->control_pages);
272
273 /* Initialize the list of destination pages */
274 INIT_LIST_HEAD(&image->dest_pages);
275
276 /* Initialize the list of unusable pages */
277 INIT_LIST_HEAD(&image->unusable_pages);
278
279 return image;
280}
281
282int kimage_is_destination_range(struct kimage *image,
283 unsigned long start,
284 unsigned long end)
285{
286 unsigned long i;
287
288 for (i = 0; i < image->nr_segments; i++) {
289 unsigned long mstart, mend;
290
291 mstart = image->segment[i].mem;
292 mend = mstart + image->segment[i].memsz;
293 if ((end > mstart) && (start < mend))
294 return 1;
295 }
296
297 return 0;
298}
299
300static struct page *kimage_alloc_pages(gfp_t gfp_mask, unsigned int order)
301{
302 struct page *pages;
303
Tom Lendackybba4ed02017-07-17 16:10:28 -0500304 pages = alloc_pages(gfp_mask & ~__GFP_ZERO, order);
Dave Young2965faa2015-09-09 15:38:55 -0700305 if (pages) {
306 unsigned int count, i;
307
308 pages->mapping = NULL;
309 set_page_private(pages, order);
310 count = 1 << order;
311 for (i = 0; i < count; i++)
312 SetPageReserved(pages + i);
Tom Lendackybba4ed02017-07-17 16:10:28 -0500313
314 arch_kexec_post_alloc_pages(page_address(pages), count,
315 gfp_mask);
316
317 if (gfp_mask & __GFP_ZERO)
318 for (i = 0; i < count; i++)
319 clear_highpage(pages + i);
Dave Young2965faa2015-09-09 15:38:55 -0700320 }
321
322 return pages;
323}
324
325static void kimage_free_pages(struct page *page)
326{
327 unsigned int order, count, i;
328
329 order = page_private(page);
330 count = 1 << order;
Tom Lendackybba4ed02017-07-17 16:10:28 -0500331
332 arch_kexec_pre_free_pages(page_address(page), count);
333
Dave Young2965faa2015-09-09 15:38:55 -0700334 for (i = 0; i < count; i++)
335 ClearPageReserved(page + i);
336 __free_pages(page, order);
337}
338
339void kimage_free_page_list(struct list_head *list)
340{
Geliang Tang2b246922016-01-20 15:00:34 -0800341 struct page *page, *next;
Dave Young2965faa2015-09-09 15:38:55 -0700342
Geliang Tang2b246922016-01-20 15:00:34 -0800343 list_for_each_entry_safe(page, next, list, lru) {
Dave Young2965faa2015-09-09 15:38:55 -0700344 list_del(&page->lru);
345 kimage_free_pages(page);
346 }
347}
348
349static struct page *kimage_alloc_normal_control_pages(struct kimage *image,
350 unsigned int order)
351{
352 /* Control pages are special, they are the intermediaries
353 * that are needed while we copy the rest of the pages
354 * to their final resting place. As such they must
355 * not conflict with either the destination addresses
356 * or memory the kernel is already using.
357 *
358 * The only case where we really need more than one of
359 * these are for architectures where we cannot disable
360 * the MMU and must instead generate an identity mapped
361 * page table for all of the memory.
362 *
363 * At worst this runs in O(N) of the image size.
364 */
365 struct list_head extra_pages;
366 struct page *pages;
367 unsigned int count;
368
369 count = 1 << order;
370 INIT_LIST_HEAD(&extra_pages);
371
372 /* Loop while I can allocate a page and the page allocated
373 * is a destination page.
374 */
375 do {
376 unsigned long pfn, epfn, addr, eaddr;
377
378 pages = kimage_alloc_pages(KEXEC_CONTROL_MEMORY_GFP, order);
379 if (!pages)
380 break;
Russell King43546d82016-08-02 14:06:04 -0700381 pfn = page_to_boot_pfn(pages);
Dave Young2965faa2015-09-09 15:38:55 -0700382 epfn = pfn + count;
383 addr = pfn << PAGE_SHIFT;
384 eaddr = epfn << PAGE_SHIFT;
385 if ((epfn >= (KEXEC_CONTROL_MEMORY_LIMIT >> PAGE_SHIFT)) ||
386 kimage_is_destination_range(image, addr, eaddr)) {
387 list_add(&pages->lru, &extra_pages);
388 pages = NULL;
389 }
390 } while (!pages);
391
392 if (pages) {
393 /* Remember the allocated page... */
394 list_add(&pages->lru, &image->control_pages);
395
396 /* Because the page is already in it's destination
397 * location we will never allocate another page at
398 * that address. Therefore kimage_alloc_pages
399 * will not return it (again) and we don't need
400 * to give it an entry in image->segment[].
401 */
402 }
403 /* Deal with the destination pages I have inadvertently allocated.
404 *
405 * Ideally I would convert multi-page allocations into single
406 * page allocations, and add everything to image->dest_pages.
407 *
408 * For now it is simpler to just free the pages.
409 */
410 kimage_free_page_list(&extra_pages);
411
412 return pages;
413}
414
415static struct page *kimage_alloc_crash_control_pages(struct kimage *image,
416 unsigned int order)
417{
418 /* Control pages are special, they are the intermediaries
419 * that are needed while we copy the rest of the pages
420 * to their final resting place. As such they must
421 * not conflict with either the destination addresses
422 * or memory the kernel is already using.
423 *
424 * Control pages are also the only pags we must allocate
425 * when loading a crash kernel. All of the other pages
426 * are specified by the segments and we just memcpy
427 * into them directly.
428 *
429 * The only case where we really need more than one of
430 * these are for architectures where we cannot disable
431 * the MMU and must instead generate an identity mapped
432 * page table for all of the memory.
433 *
434 * Given the low demand this implements a very simple
435 * allocator that finds the first hole of the appropriate
436 * size in the reserved memory region, and allocates all
437 * of the memory up to and including the hole.
438 */
439 unsigned long hole_start, hole_end, size;
440 struct page *pages;
441
442 pages = NULL;
443 size = (1 << order) << PAGE_SHIFT;
444 hole_start = (image->control_page + (size - 1)) & ~(size - 1);
445 hole_end = hole_start + size - 1;
446 while (hole_end <= crashk_res.end) {
447 unsigned long i;
448
zhong jiang8e53c072016-12-14 15:04:23 -0800449 cond_resched();
450
Dave Young2965faa2015-09-09 15:38:55 -0700451 if (hole_end > KEXEC_CRASH_CONTROL_MEMORY_LIMIT)
452 break;
453 /* See if I overlap any of the segments */
454 for (i = 0; i < image->nr_segments; i++) {
455 unsigned long mstart, mend;
456
457 mstart = image->segment[i].mem;
458 mend = mstart + image->segment[i].memsz - 1;
459 if ((hole_end >= mstart) && (hole_start <= mend)) {
460 /* Advance the hole to the end of the segment */
461 hole_start = (mend + (size - 1)) & ~(size - 1);
462 hole_end = hole_start + size - 1;
463 break;
464 }
465 }
466 /* If I don't overlap any segments I have found my hole! */
467 if (i == image->nr_segments) {
468 pages = pfn_to_page(hole_start >> PAGE_SHIFT);
Minfei Huang04e99492015-09-09 15:38:58 -0700469 image->control_page = hole_end;
Dave Young2965faa2015-09-09 15:38:55 -0700470 break;
471 }
472 }
Dave Young2965faa2015-09-09 15:38:55 -0700473
Lianbo Jiang9cf38d52018-09-30 11:10:31 +0800474 /* Ensure that these pages are decrypted if SME is enabled. */
475 if (pages)
476 arch_kexec_post_alloc_pages(page_address(pages), 1 << order, 0);
477
Dave Young2965faa2015-09-09 15:38:55 -0700478 return pages;
479}
480
481
482struct page *kimage_alloc_control_pages(struct kimage *image,
483 unsigned int order)
484{
485 struct page *pages = NULL;
486
487 switch (image->type) {
488 case KEXEC_TYPE_DEFAULT:
489 pages = kimage_alloc_normal_control_pages(image, order);
490 break;
491 case KEXEC_TYPE_CRASH:
492 pages = kimage_alloc_crash_control_pages(image, order);
493 break;
494 }
495
496 return pages;
497}
498
Xunlei Pang12293842017-07-12 14:33:21 -0700499int kimage_crash_copy_vmcoreinfo(struct kimage *image)
500{
501 struct page *vmcoreinfo_page;
502 void *safecopy;
503
504 if (image->type != KEXEC_TYPE_CRASH)
505 return 0;
506
507 /*
508 * For kdump, allocate one vmcoreinfo safe copy from the
509 * crash memory. as we have arch_kexec_protect_crashkres()
510 * after kexec syscall, we naturally protect it from write
511 * (even read) access under kernel direct mapping. But on
512 * the other hand, we still need to operate it when crash
513 * happens to generate vmcoreinfo note, hereby we rely on
514 * vmap for this purpose.
515 */
516 vmcoreinfo_page = kimage_alloc_control_pages(image, 0);
517 if (!vmcoreinfo_page) {
518 pr_warn("Could not allocate vmcoreinfo buffer\n");
519 return -ENOMEM;
520 }
521 safecopy = vmap(&vmcoreinfo_page, 1, VM_MAP, PAGE_KERNEL);
522 if (!safecopy) {
523 pr_warn("Could not vmap vmcoreinfo buffer\n");
524 return -ENOMEM;
525 }
526
527 image->vmcoreinfo_data_copy = safecopy;
528 crash_update_vmcoreinfo_safecopy(safecopy);
529
530 return 0;
531}
532
Dave Young2965faa2015-09-09 15:38:55 -0700533static int kimage_add_entry(struct kimage *image, kimage_entry_t entry)
534{
535 if (*image->entry != 0)
536 image->entry++;
537
538 if (image->entry == image->last_entry) {
539 kimage_entry_t *ind_page;
540 struct page *page;
541
542 page = kimage_alloc_page(image, GFP_KERNEL, KIMAGE_NO_DEST);
543 if (!page)
544 return -ENOMEM;
545
546 ind_page = page_address(page);
Russell King43546d82016-08-02 14:06:04 -0700547 *image->entry = virt_to_boot_phys(ind_page) | IND_INDIRECTION;
Dave Young2965faa2015-09-09 15:38:55 -0700548 image->entry = ind_page;
549 image->last_entry = ind_page +
550 ((PAGE_SIZE/sizeof(kimage_entry_t)) - 1);
551 }
552 *image->entry = entry;
553 image->entry++;
554 *image->entry = 0;
555
556 return 0;
557}
558
559static int kimage_set_destination(struct kimage *image,
560 unsigned long destination)
561{
562 int result;
563
564 destination &= PAGE_MASK;
565 result = kimage_add_entry(image, destination | IND_DESTINATION);
566
567 return result;
568}
569
570
571static int kimage_add_page(struct kimage *image, unsigned long page)
572{
573 int result;
574
575 page &= PAGE_MASK;
576 result = kimage_add_entry(image, page | IND_SOURCE);
577
578 return result;
579}
580
581
582static void kimage_free_extra_pages(struct kimage *image)
583{
584 /* Walk through and free any extra destination pages I may have */
585 kimage_free_page_list(&image->dest_pages);
586
587 /* Walk through and free any unusable pages I have cached */
588 kimage_free_page_list(&image->unusable_pages);
589
590}
591void kimage_terminate(struct kimage *image)
592{
593 if (*image->entry != 0)
594 image->entry++;
595
596 *image->entry = IND_DONE;
597}
598
599#define for_each_kimage_entry(image, ptr, entry) \
600 for (ptr = &image->head; (entry = *ptr) && !(entry & IND_DONE); \
601 ptr = (entry & IND_INDIRECTION) ? \
Russell King43546d82016-08-02 14:06:04 -0700602 boot_phys_to_virt((entry & PAGE_MASK)) : ptr + 1)
Dave Young2965faa2015-09-09 15:38:55 -0700603
604static void kimage_free_entry(kimage_entry_t entry)
605{
606 struct page *page;
607
Russell King43546d82016-08-02 14:06:04 -0700608 page = boot_pfn_to_page(entry >> PAGE_SHIFT);
Dave Young2965faa2015-09-09 15:38:55 -0700609 kimage_free_pages(page);
610}
611
612void kimage_free(struct kimage *image)
613{
614 kimage_entry_t *ptr, entry;
615 kimage_entry_t ind = 0;
616
617 if (!image)
618 return;
619
Xunlei Pang12293842017-07-12 14:33:21 -0700620 if (image->vmcoreinfo_data_copy) {
621 crash_update_vmcoreinfo_safecopy(NULL);
622 vunmap(image->vmcoreinfo_data_copy);
623 }
624
Dave Young2965faa2015-09-09 15:38:55 -0700625 kimage_free_extra_pages(image);
626 for_each_kimage_entry(image, ptr, entry) {
627 if (entry & IND_INDIRECTION) {
628 /* Free the previous indirection page */
629 if (ind & IND_INDIRECTION)
630 kimage_free_entry(ind);
631 /* Save this indirection page until we are
632 * done with it.
633 */
634 ind = entry;
635 } else if (entry & IND_SOURCE)
636 kimage_free_entry(entry);
637 }
638 /* Free the final indirection page */
639 if (ind & IND_INDIRECTION)
640 kimage_free_entry(ind);
641
642 /* Handle any machine specific cleanup */
643 machine_kexec_cleanup(image);
644
645 /* Free the kexec control pages... */
646 kimage_free_page_list(&image->control_pages);
647
648 /*
649 * Free up any temporary buffers allocated. This might hit if
650 * error occurred much later after buffer allocation.
651 */
652 if (image->file_mode)
653 kimage_file_post_load_cleanup(image);
654
655 kfree(image);
656}
657
658static kimage_entry_t *kimage_dst_used(struct kimage *image,
659 unsigned long page)
660{
661 kimage_entry_t *ptr, entry;
662 unsigned long destination = 0;
663
664 for_each_kimage_entry(image, ptr, entry) {
665 if (entry & IND_DESTINATION)
666 destination = entry & PAGE_MASK;
667 else if (entry & IND_SOURCE) {
668 if (page == destination)
669 return ptr;
670 destination += PAGE_SIZE;
671 }
672 }
673
674 return NULL;
675}
676
677static struct page *kimage_alloc_page(struct kimage *image,
678 gfp_t gfp_mask,
679 unsigned long destination)
680{
681 /*
682 * Here we implement safeguards to ensure that a source page
683 * is not copied to its destination page before the data on
684 * the destination page is no longer useful.
685 *
686 * To do this we maintain the invariant that a source page is
687 * either its own destination page, or it is not a
688 * destination page at all.
689 *
690 * That is slightly stronger than required, but the proof
691 * that no problems will not occur is trivial, and the
692 * implementation is simply to verify.
693 *
694 * When allocating all pages normally this algorithm will run
695 * in O(N) time, but in the worst case it will run in O(N^2)
696 * time. If the runtime is a problem the data structures can
697 * be fixed.
698 */
699 struct page *page;
700 unsigned long addr;
701
702 /*
703 * Walk through the list of destination pages, and see if I
704 * have a match.
705 */
706 list_for_each_entry(page, &image->dest_pages, lru) {
Russell King43546d82016-08-02 14:06:04 -0700707 addr = page_to_boot_pfn(page) << PAGE_SHIFT;
Dave Young2965faa2015-09-09 15:38:55 -0700708 if (addr == destination) {
709 list_del(&page->lru);
710 return page;
711 }
712 }
713 page = NULL;
714 while (1) {
715 kimage_entry_t *old;
716
717 /* Allocate a page, if we run out of memory give up */
718 page = kimage_alloc_pages(gfp_mask, 0);
719 if (!page)
720 return NULL;
721 /* If the page cannot be used file it away */
Russell King43546d82016-08-02 14:06:04 -0700722 if (page_to_boot_pfn(page) >
Dave Young2965faa2015-09-09 15:38:55 -0700723 (KEXEC_SOURCE_MEMORY_LIMIT >> PAGE_SHIFT)) {
724 list_add(&page->lru, &image->unusable_pages);
725 continue;
726 }
Russell King43546d82016-08-02 14:06:04 -0700727 addr = page_to_boot_pfn(page) << PAGE_SHIFT;
Dave Young2965faa2015-09-09 15:38:55 -0700728
729 /* If it is the destination page we want use it */
730 if (addr == destination)
731 break;
732
733 /* If the page is not a destination page use it */
734 if (!kimage_is_destination_range(image, addr,
735 addr + PAGE_SIZE))
736 break;
737
738 /*
739 * I know that the page is someones destination page.
740 * See if there is already a source page for this
741 * destination page. And if so swap the source pages.
742 */
743 old = kimage_dst_used(image, addr);
744 if (old) {
745 /* If so move it */
746 unsigned long old_addr;
747 struct page *old_page;
748
749 old_addr = *old & PAGE_MASK;
Russell King43546d82016-08-02 14:06:04 -0700750 old_page = boot_pfn_to_page(old_addr >> PAGE_SHIFT);
Dave Young2965faa2015-09-09 15:38:55 -0700751 copy_highpage(page, old_page);
752 *old = addr | (*old & ~PAGE_MASK);
753
754 /* The old page I have found cannot be a
755 * destination page, so return it if it's
756 * gfp_flags honor the ones passed in.
757 */
758 if (!(gfp_mask & __GFP_HIGHMEM) &&
759 PageHighMem(old_page)) {
760 kimage_free_pages(old_page);
761 continue;
762 }
763 addr = old_addr;
764 page = old_page;
765 break;
766 }
767 /* Place the page on the destination list, to be used later */
768 list_add(&page->lru, &image->dest_pages);
769 }
770
771 return page;
772}
773
774static int kimage_load_normal_segment(struct kimage *image,
775 struct kexec_segment *segment)
776{
777 unsigned long maddr;
778 size_t ubytes, mbytes;
779 int result;
780 unsigned char __user *buf = NULL;
781 unsigned char *kbuf = NULL;
782
783 result = 0;
784 if (image->file_mode)
785 kbuf = segment->kbuf;
786 else
787 buf = segment->buf;
788 ubytes = segment->bufsz;
789 mbytes = segment->memsz;
790 maddr = segment->mem;
791
792 result = kimage_set_destination(image, maddr);
793 if (result < 0)
794 goto out;
795
796 while (mbytes) {
797 struct page *page;
798 char *ptr;
799 size_t uchunk, mchunk;
800
801 page = kimage_alloc_page(image, GFP_HIGHUSER, maddr);
802 if (!page) {
803 result = -ENOMEM;
804 goto out;
805 }
Russell King43546d82016-08-02 14:06:04 -0700806 result = kimage_add_page(image, page_to_boot_pfn(page)
Dave Young2965faa2015-09-09 15:38:55 -0700807 << PAGE_SHIFT);
808 if (result < 0)
809 goto out;
810
811 ptr = kmap(page);
812 /* Start with a clear page */
813 clear_page(ptr);
814 ptr += maddr & ~PAGE_MASK;
815 mchunk = min_t(size_t, mbytes,
816 PAGE_SIZE - (maddr & ~PAGE_MASK));
817 uchunk = min(ubytes, mchunk);
818
819 /* For file based kexec, source pages are in kernel memory */
820 if (image->file_mode)
821 memcpy(ptr, kbuf, uchunk);
822 else
823 result = copy_from_user(ptr, buf, uchunk);
824 kunmap(page);
825 if (result) {
826 result = -EFAULT;
827 goto out;
828 }
829 ubytes -= uchunk;
830 maddr += mchunk;
831 if (image->file_mode)
832 kbuf += mchunk;
833 else
834 buf += mchunk;
835 mbytes -= mchunk;
Jarrett Farnitanoa8311f62018-06-14 15:26:31 -0700836
837 cond_resched();
Dave Young2965faa2015-09-09 15:38:55 -0700838 }
839out:
840 return result;
841}
842
843static int kimage_load_crash_segment(struct kimage *image,
844 struct kexec_segment *segment)
845{
846 /* For crash dumps kernels we simply copy the data from
847 * user space to it's destination.
848 * We do things a page at a time for the sake of kmap.
849 */
850 unsigned long maddr;
851 size_t ubytes, mbytes;
852 int result;
853 unsigned char __user *buf = NULL;
854 unsigned char *kbuf = NULL;
855
856 result = 0;
857 if (image->file_mode)
858 kbuf = segment->kbuf;
859 else
860 buf = segment->buf;
861 ubytes = segment->bufsz;
862 mbytes = segment->memsz;
863 maddr = segment->mem;
864 while (mbytes) {
865 struct page *page;
866 char *ptr;
867 size_t uchunk, mchunk;
868
Russell King43546d82016-08-02 14:06:04 -0700869 page = boot_pfn_to_page(maddr >> PAGE_SHIFT);
Dave Young2965faa2015-09-09 15:38:55 -0700870 if (!page) {
871 result = -ENOMEM;
872 goto out;
873 }
Lianbo Jiang9cf38d52018-09-30 11:10:31 +0800874 arch_kexec_post_alloc_pages(page_address(page), 1, 0);
Dave Young2965faa2015-09-09 15:38:55 -0700875 ptr = kmap(page);
876 ptr += maddr & ~PAGE_MASK;
877 mchunk = min_t(size_t, mbytes,
878 PAGE_SIZE - (maddr & ~PAGE_MASK));
879 uchunk = min(ubytes, mchunk);
880 if (mchunk > uchunk) {
881 /* Zero the trailing part of the page */
882 memset(ptr + uchunk, 0, mchunk - uchunk);
883 }
884
885 /* For file based kexec, source pages are in kernel memory */
886 if (image->file_mode)
887 memcpy(ptr, kbuf, uchunk);
888 else
889 result = copy_from_user(ptr, buf, uchunk);
890 kexec_flush_icache_page(page);
891 kunmap(page);
Lianbo Jiang9cf38d52018-09-30 11:10:31 +0800892 arch_kexec_pre_free_pages(page_address(page), 1);
Dave Young2965faa2015-09-09 15:38:55 -0700893 if (result) {
894 result = -EFAULT;
895 goto out;
896 }
897 ubytes -= uchunk;
898 maddr += mchunk;
899 if (image->file_mode)
900 kbuf += mchunk;
901 else
902 buf += mchunk;
903 mbytes -= mchunk;
Jarrett Farnitanoa8311f62018-06-14 15:26:31 -0700904
905 cond_resched();
Dave Young2965faa2015-09-09 15:38:55 -0700906 }
907out:
908 return result;
909}
910
911int kimage_load_segment(struct kimage *image,
912 struct kexec_segment *segment)
913{
914 int result = -ENOMEM;
915
916 switch (image->type) {
917 case KEXEC_TYPE_DEFAULT:
918 result = kimage_load_normal_segment(image, segment);
919 break;
920 case KEXEC_TYPE_CRASH:
921 result = kimage_load_crash_segment(image, segment);
922 break;
923 }
924
925 return result;
926}
927
928struct kimage *kexec_image;
929struct kimage *kexec_crash_image;
930int kexec_load_disabled;
931
Hidehiro Kawai7bbee5c2015-12-14 11:19:11 +0100932/*
933 * No panic_cpu check version of crash_kexec(). This function is called
934 * only when panic_cpu holds the current CPU number; this is the only CPU
935 * which processes crash_kexec routines.
936 */
Josh Poimboeufc207aee2017-06-28 10:11:06 -0500937void __noclone __crash_kexec(struct pt_regs *regs)
Dave Young2965faa2015-09-09 15:38:55 -0700938{
939 /* Take the kexec_mutex here to prevent sys_kexec_load
940 * running on one cpu from replacing the crash kernel
941 * we are using after a panic on a different cpu.
942 *
943 * If the crash kernel was not located in a fixed area
944 * of memory the xchg(&kexec_crash_image) would be
945 * sufficient. But since I reuse the memory...
946 */
947 if (mutex_trylock(&kexec_mutex)) {
948 if (kexec_crash_image) {
949 struct pt_regs fixed_regs;
950
951 crash_setup_regs(&fixed_regs, regs);
952 crash_save_vmcoreinfo();
953 machine_crash_shutdown(&fixed_regs);
954 machine_kexec(kexec_crash_image);
955 }
956 mutex_unlock(&kexec_mutex);
957 }
958}
Josh Poimboeufc207aee2017-06-28 10:11:06 -0500959STACK_FRAME_NON_STANDARD(__crash_kexec);
Dave Young2965faa2015-09-09 15:38:55 -0700960
Hidehiro Kawai7bbee5c2015-12-14 11:19:11 +0100961void crash_kexec(struct pt_regs *regs)
962{
963 int old_cpu, this_cpu;
964
965 /*
966 * Only one CPU is allowed to execute the crash_kexec() code as with
967 * panic(). Otherwise parallel calls of panic() and crash_kexec()
968 * may stop each other. To exclude them, we use panic_cpu here too.
969 */
970 this_cpu = raw_smp_processor_id();
971 old_cpu = atomic_cmpxchg(&panic_cpu, PANIC_CPU_INVALID, this_cpu);
972 if (old_cpu == PANIC_CPU_INVALID) {
973 /* This is the 1st CPU which comes here, so go ahead. */
Sergey Senozhatskyf92bac32016-12-27 23:16:05 +0900974 printk_safe_flush_on_panic();
Hidehiro Kawai7bbee5c2015-12-14 11:19:11 +0100975 __crash_kexec(regs);
976
977 /*
978 * Reset panic_cpu to allow another panic()/crash_kexec()
979 * call.
980 */
981 atomic_set(&panic_cpu, PANIC_CPU_INVALID);
982 }
983}
984
Dave Young2965faa2015-09-09 15:38:55 -0700985size_t crash_get_memory_size(void)
986{
987 size_t size = 0;
988
989 mutex_lock(&kexec_mutex);
990 if (crashk_res.end != crashk_res.start)
991 size = resource_size(&crashk_res);
992 mutex_unlock(&kexec_mutex);
993 return size;
994}
995
996void __weak crash_free_reserved_phys_range(unsigned long begin,
997 unsigned long end)
998{
999 unsigned long addr;
1000
1001 for (addr = begin; addr < end; addr += PAGE_SIZE)
Russell King43546d82016-08-02 14:06:04 -07001002 free_reserved_page(boot_pfn_to_page(addr >> PAGE_SHIFT));
Dave Young2965faa2015-09-09 15:38:55 -07001003}
1004
1005int crash_shrink_memory(unsigned long new_size)
1006{
1007 int ret = 0;
1008 unsigned long start, end;
1009 unsigned long old_size;
1010 struct resource *ram_res;
1011
1012 mutex_lock(&kexec_mutex);
1013
1014 if (kexec_crash_image) {
1015 ret = -ENOENT;
1016 goto unlock;
1017 }
1018 start = crashk_res.start;
1019 end = crashk_res.end;
1020 old_size = (end == 0) ? 0 : end - start + 1;
1021 if (new_size >= old_size) {
1022 ret = (new_size == old_size) ? 0 : -EINVAL;
1023 goto unlock;
1024 }
1025
1026 ram_res = kzalloc(sizeof(*ram_res), GFP_KERNEL);
1027 if (!ram_res) {
1028 ret = -ENOMEM;
1029 goto unlock;
1030 }
1031
1032 start = roundup(start, KEXEC_CRASH_MEM_ALIGN);
1033 end = roundup(start + new_size, KEXEC_CRASH_MEM_ALIGN);
1034
Dave Young2965faa2015-09-09 15:38:55 -07001035 crash_free_reserved_phys_range(end, crashk_res.end);
1036
1037 if ((start == end) && (crashk_res.parent != NULL))
1038 release_resource(&crashk_res);
1039
1040 ram_res->start = end;
1041 ram_res->end = crashk_res.end;
Toshi Kani1a085d02016-01-26 21:57:23 +01001042 ram_res->flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM;
Dave Young2965faa2015-09-09 15:38:55 -07001043 ram_res->name = "System RAM";
1044
1045 crashk_res.end = end - 1;
1046
1047 insert_resource(&iomem_resource, ram_res);
Dave Young2965faa2015-09-09 15:38:55 -07001048
1049unlock:
1050 mutex_unlock(&kexec_mutex);
1051 return ret;
1052}
1053
Dave Young2965faa2015-09-09 15:38:55 -07001054void crash_save_cpu(struct pt_regs *regs, int cpu)
1055{
1056 struct elf_prstatus prstatus;
1057 u32 *buf;
1058
1059 if ((cpu < 0) || (cpu >= nr_cpu_ids))
1060 return;
1061
1062 /* Using ELF notes here is opportunistic.
1063 * I need a well defined structure format
1064 * for the data I pass, and I need tags
1065 * on the data to indicate what information I have
1066 * squirrelled away. ELF notes happen to provide
1067 * all of that, so there is no need to invent something new.
1068 */
1069 buf = (u32 *)per_cpu_ptr(crash_notes, cpu);
1070 if (!buf)
1071 return;
1072 memset(&prstatus, 0, sizeof(prstatus));
1073 prstatus.pr_pid = current->pid;
1074 elf_core_copy_kernel_regs(&prstatus.pr_reg, regs);
1075 buf = append_elf_note(buf, KEXEC_CORE_NOTE_NAME, NT_PRSTATUS,
1076 &prstatus, sizeof(prstatus));
1077 final_note(buf);
1078}
1079
1080static int __init crash_notes_memory_init(void)
1081{
1082 /* Allocate memory for saving cpu registers. */
Baoquan Hebbb78b82015-09-09 15:39:00 -07001083 size_t size, align;
1084
1085 /*
1086 * crash_notes could be allocated across 2 vmalloc pages when percpu
1087 * is vmalloc based . vmalloc doesn't guarantee 2 continuous vmalloc
1088 * pages are also on 2 continuous physical pages. In this case the
1089 * 2nd part of crash_notes in 2nd page could be lost since only the
1090 * starting address and size of crash_notes are exported through sysfs.
1091 * Here round up the size of crash_notes to the nearest power of two
1092 * and pass it to __alloc_percpu as align value. This can make sure
1093 * crash_notes is allocated inside one physical page.
1094 */
1095 size = sizeof(note_buf_t);
1096 align = min(roundup_pow_of_two(sizeof(note_buf_t)), PAGE_SIZE);
1097
1098 /*
1099 * Break compile if size is bigger than PAGE_SIZE since crash_notes
1100 * definitely will be in 2 pages with that.
1101 */
1102 BUILD_BUG_ON(size > PAGE_SIZE);
1103
1104 crash_notes = __alloc_percpu(size, align);
Dave Young2965faa2015-09-09 15:38:55 -07001105 if (!crash_notes) {
Minfei Huangde90a6b2015-11-06 16:32:45 -08001106 pr_warn("Memory allocation for saving cpu register states failed\n");
Dave Young2965faa2015-09-09 15:38:55 -07001107 return -ENOMEM;
1108 }
1109 return 0;
1110}
1111subsys_initcall(crash_notes_memory_init);
1112
1113
1114/*
Dave Young2965faa2015-09-09 15:38:55 -07001115 * Move into place and start executing a preloaded standalone
1116 * executable. If nothing was preloaded return an error.
1117 */
1118int kernel_kexec(void)
1119{
1120 int error = 0;
1121
1122 if (!mutex_trylock(&kexec_mutex))
1123 return -EBUSY;
1124 if (!kexec_image) {
1125 error = -EINVAL;
1126 goto Unlock;
1127 }
1128
1129#ifdef CONFIG_KEXEC_JUMP
1130 if (kexec_image->preserve_context) {
1131 lock_system_sleep();
1132 pm_prepare_console();
1133 error = freeze_processes();
1134 if (error) {
1135 error = -EBUSY;
1136 goto Restore_console;
1137 }
1138 suspend_console();
1139 error = dpm_suspend_start(PMSG_FREEZE);
1140 if (error)
1141 goto Resume_console;
1142 /* At this point, dpm_suspend_start() has been called,
1143 * but *not* dpm_suspend_end(). We *must* call
1144 * dpm_suspend_end() now. Otherwise, drivers for
1145 * some devices (e.g. interrupt controllers) become
1146 * desynchronized with the actual state of the
1147 * hardware at resume time, and evil weirdness ensues.
1148 */
1149 error = dpm_suspend_end(PMSG_FREEZE);
1150 if (error)
1151 goto Resume_devices;
1152 error = disable_nonboot_cpus();
1153 if (error)
1154 goto Enable_cpus;
1155 local_irq_disable();
1156 error = syscore_suspend();
1157 if (error)
1158 goto Enable_irqs;
1159 } else
1160#endif
1161 {
1162 kexec_in_progress = true;
1163 kernel_restart_prepare(NULL);
1164 migrate_to_reboot_cpu();
1165
1166 /*
1167 * migrate_to_reboot_cpu() disables CPU hotplug assuming that
1168 * no further code needs to use CPU hotplug (which is true in
1169 * the reboot case). However, the kexec path depends on using
1170 * CPU hotplug again; so re-enable it here.
1171 */
1172 cpu_hotplug_enable();
1173 pr_emerg("Starting new kernel\n");
1174 machine_shutdown();
1175 }
1176
1177 machine_kexec(kexec_image);
1178
1179#ifdef CONFIG_KEXEC_JUMP
1180 if (kexec_image->preserve_context) {
1181 syscore_resume();
1182 Enable_irqs:
1183 local_irq_enable();
1184 Enable_cpus:
1185 enable_nonboot_cpus();
1186 dpm_resume_start(PMSG_RESTORE);
1187 Resume_devices:
1188 dpm_resume_end(PMSG_RESTORE);
1189 Resume_console:
1190 resume_console();
1191 thaw_processes();
1192 Restore_console:
1193 pm_restore_console();
1194 unlock_system_sleep();
1195 }
1196#endif
1197
1198 Unlock:
1199 mutex_unlock(&kexec_mutex);
1200 return error;
1201}
1202
1203/*
Xunlei Pang7a0058e2016-05-23 16:24:22 -07001204 * Protection mechanism for crashkernel reserved memory after
1205 * the kdump kernel is loaded.
Dave Young2965faa2015-09-09 15:38:55 -07001206 *
1207 * Provide an empty default implementation here -- architecture
1208 * code may override this
1209 */
Xunlei Pang9b492cf2016-05-23 16:24:10 -07001210void __weak arch_kexec_protect_crashkres(void)
1211{}
1212
1213void __weak arch_kexec_unprotect_crashkres(void)
1214{}