Thomas Gleixner | caab277 | 2019-06-03 07:44:50 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 2 | /* |
| 3 | * NUMA support, based on the x86 implementation. |
| 4 | * |
| 5 | * Copyright (C) 2015 Cavium Inc. |
| 6 | * Author: Ganapatrao Kulkarni <gkulkarni@cavium.com> |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 7 | */ |
| 8 | |
Kefeng Wang | f11c7ba | 2016-09-01 14:54:59 +0800 | [diff] [blame] | 9 | #define pr_fmt(fmt) "NUMA: " fmt |
| 10 | |
Hanjun Guo | d8b47fc | 2016-05-24 15:35:44 -0700 | [diff] [blame] | 11 | #include <linux/acpi.h> |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 12 | #include <linux/memblock.h> |
| 13 | #include <linux/module.h> |
| 14 | #include <linux/of.h> |
| 15 | |
Zhen Lei | 7af3a0a | 2016-09-01 14:55:00 +0800 | [diff] [blame] | 16 | #include <asm/sections.h> |
Kefeng Wang | 09cea61 | 2021-11-05 13:39:44 -0700 | [diff] [blame] | 17 | #include <asm/pgalloc.h> |
Catalin Marinas | bfe6c8a | 2016-08-15 16:33:10 +0100 | [diff] [blame] | 18 | |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 19 | struct pglist_data *node_data[MAX_NUMNODES] __read_mostly; |
| 20 | EXPORT_SYMBOL(node_data); |
| 21 | nodemask_t numa_nodes_parsed __initdata; |
| 22 | static int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE }; |
| 23 | |
| 24 | static int numa_distance_cnt; |
| 25 | static u8 *numa_distance; |
Boris Ostrovsky | aec03f8 | 2016-12-12 23:18:29 -0500 | [diff] [blame] | 26 | bool numa_off; |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 27 | |
| 28 | static __init int numa_parse_early_param(char *opt) |
| 29 | { |
| 30 | if (!opt) |
| 31 | return -EINVAL; |
Chuhong Yuan | b3e089c | 2019-07-30 10:44:15 +0800 | [diff] [blame] | 32 | if (str_has_prefix(opt, "off")) |
David Daney | 34c3337 | 2016-05-24 15:35:37 -0700 | [diff] [blame] | 33 | numa_off = true; |
Kefeng Wang | f11c7ba | 2016-09-01 14:54:59 +0800 | [diff] [blame] | 34 | |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 35 | return 0; |
| 36 | } |
| 37 | early_param("numa", numa_parse_early_param); |
| 38 | |
| 39 | cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; |
| 40 | EXPORT_SYMBOL(node_to_cpumask_map); |
| 41 | |
| 42 | #ifdef CONFIG_DEBUG_PER_CPU_MAPS |
| 43 | |
| 44 | /* |
| 45 | * Returns a pointer to the bitmask of CPUs on Node 'node'. |
| 46 | */ |
| 47 | const struct cpumask *cpumask_of_node(int node) |
| 48 | { |
Zhengyuan Liu | a194c5f | 2020-09-21 10:39:36 +0800 | [diff] [blame] | 49 | |
| 50 | if (node == NUMA_NO_NODE) |
| 51 | return cpu_all_mask; |
| 52 | |
| 53 | if (WARN_ON(node < 0 || node >= nr_node_ids)) |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 54 | return cpu_none_mask; |
| 55 | |
| 56 | if (WARN_ON(node_to_cpumask_map[node] == NULL)) |
| 57 | return cpu_online_mask; |
| 58 | |
| 59 | return node_to_cpumask_map[node]; |
| 60 | } |
| 61 | EXPORT_SYMBOL(cpumask_of_node); |
| 62 | |
| 63 | #endif |
| 64 | |
Sudeep Holla | 97fd601 | 2018-07-06 12:02:43 +0100 | [diff] [blame] | 65 | static void numa_update_cpu(unsigned int cpu, bool remove) |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 66 | { |
Sudeep Holla | 97fd601 | 2018-07-06 12:02:43 +0100 | [diff] [blame] | 67 | int nid = cpu_to_node(cpu); |
| 68 | |
| 69 | if (nid == NUMA_NO_NODE) |
| 70 | return; |
| 71 | |
| 72 | if (remove) |
| 73 | cpumask_clear_cpu(cpu, node_to_cpumask_map[nid]); |
| 74 | else |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 75 | cpumask_set_cpu(cpu, node_to_cpumask_map[nid]); |
| 76 | } |
| 77 | |
Sudeep Holla | 97fd601 | 2018-07-06 12:02:43 +0100 | [diff] [blame] | 78 | void numa_add_cpu(unsigned int cpu) |
| 79 | { |
| 80 | numa_update_cpu(cpu, false); |
| 81 | } |
| 82 | |
| 83 | void numa_remove_cpu(unsigned int cpu) |
| 84 | { |
| 85 | numa_update_cpu(cpu, true); |
| 86 | } |
| 87 | |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 88 | void numa_clear_node(unsigned int cpu) |
| 89 | { |
Sudeep Holla | 97fd601 | 2018-07-06 12:02:43 +0100 | [diff] [blame] | 90 | numa_remove_cpu(cpu); |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 91 | set_cpu_numa_node(cpu, NUMA_NO_NODE); |
| 92 | } |
| 93 | |
| 94 | /* |
| 95 | * Allocate node_to_cpumask_map based on number of available nodes |
| 96 | * Requires node_possible_map to be valid. |
| 97 | * |
| 98 | * Note: cpumask_of_node() is not valid until after this is done. |
| 99 | * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.) |
| 100 | */ |
| 101 | static void __init setup_node_to_cpumask_map(void) |
| 102 | { |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 103 | int node; |
| 104 | |
| 105 | /* setup nr_node_ids if not done yet */ |
| 106 | if (nr_node_ids == MAX_NUMNODES) |
| 107 | setup_nr_node_ids(); |
| 108 | |
| 109 | /* allocate and clear the mapping */ |
| 110 | for (node = 0; node < nr_node_ids; node++) { |
| 111 | alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); |
| 112 | cpumask_clear(node_to_cpumask_map[node]); |
| 113 | } |
| 114 | |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 115 | /* cpumask_of_node() will now work */ |
Alexey Dobriyan | b9726c2 | 2019-03-05 15:48:26 -0800 | [diff] [blame] | 116 | pr_debug("Node to cpumask map for %u nodes\n", nr_node_ids); |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 117 | } |
| 118 | |
| 119 | /* |
Qian Cai | b1ce45e | 2019-03-14 12:16:19 -0400 | [diff] [blame] | 120 | * Set the cpu to node and mem mapping |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 121 | */ |
| 122 | void numa_store_cpu_info(unsigned int cpu) |
| 123 | { |
Sudeep Holla | 97fd601 | 2018-07-06 12:02:43 +0100 | [diff] [blame] | 124 | set_cpu_numa_node(cpu, cpu_to_node_map[cpu]); |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 125 | } |
| 126 | |
| 127 | void __init early_map_cpu_to_node(unsigned int cpu, int nid) |
| 128 | { |
| 129 | /* fallback to node 0 */ |
Zhen Lei | 7ba5f60 | 2016-09-01 14:55:04 +0800 | [diff] [blame] | 130 | if (nid < 0 || nid >= MAX_NUMNODES || numa_off) |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 131 | nid = 0; |
| 132 | |
| 133 | cpu_to_node_map[cpu] = nid; |
Zhen Lei | 7ba5f60 | 2016-09-01 14:55:04 +0800 | [diff] [blame] | 134 | |
| 135 | /* |
| 136 | * We should set the numa node of cpu0 as soon as possible, because it |
| 137 | * has already been set up online before. cpu_to_node(0) will soon be |
| 138 | * called. |
| 139 | */ |
| 140 | if (!cpu) |
| 141 | set_cpu_numa_node(cpu, nid); |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 142 | } |
| 143 | |
Zhen Lei | 7af3a0a | 2016-09-01 14:55:00 +0800 | [diff] [blame] | 144 | #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA |
| 145 | unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; |
| 146 | EXPORT_SYMBOL(__per_cpu_offset); |
| 147 | |
| 148 | static int __init early_cpu_to_node(int cpu) |
| 149 | { |
| 150 | return cpu_to_node_map[cpu]; |
| 151 | } |
| 152 | |
| 153 | static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) |
| 154 | { |
Yisheng Xie | 26984c3 | 2016-10-21 16:13:55 +0800 | [diff] [blame] | 155 | return node_distance(early_cpu_to_node(from), early_cpu_to_node(to)); |
Zhen Lei | 7af3a0a | 2016-09-01 14:55:00 +0800 | [diff] [blame] | 156 | } |
| 157 | |
| 158 | static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, |
| 159 | size_t align) |
| 160 | { |
| 161 | int nid = early_cpu_to_node(cpu); |
| 162 | |
Mike Rapoport | eb31d55 | 2018-10-30 15:08:04 -0700 | [diff] [blame] | 163 | return memblock_alloc_try_nid(size, align, |
Zhen Lei | 7af3a0a | 2016-09-01 14:55:00 +0800 | [diff] [blame] | 164 | __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid); |
| 165 | } |
| 166 | |
| 167 | static void __init pcpu_fc_free(void *ptr, size_t size) |
| 168 | { |
Mike Rapoport | 4421cca0 | 2021-11-05 13:43:22 -0700 | [diff] [blame] | 169 | memblock_free(ptr, size); |
Zhen Lei | 7af3a0a | 2016-09-01 14:55:00 +0800 | [diff] [blame] | 170 | } |
| 171 | |
Kefeng Wang | 09cea61 | 2021-11-05 13:39:44 -0700 | [diff] [blame] | 172 | #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK |
| 173 | static void __init pcpu_populate_pte(unsigned long addr) |
| 174 | { |
| 175 | pgd_t *pgd = pgd_offset_k(addr); |
| 176 | p4d_t *p4d; |
| 177 | pud_t *pud; |
| 178 | pmd_t *pmd; |
| 179 | |
| 180 | p4d = p4d_offset(pgd, addr); |
| 181 | if (p4d_none(*p4d)) { |
| 182 | pud_t *new; |
| 183 | |
| 184 | new = memblock_alloc(PAGE_SIZE, PAGE_SIZE); |
| 185 | if (!new) |
| 186 | goto err_alloc; |
| 187 | p4d_populate(&init_mm, p4d, new); |
| 188 | } |
| 189 | |
| 190 | pud = pud_offset(p4d, addr); |
| 191 | if (pud_none(*pud)) { |
| 192 | pmd_t *new; |
| 193 | |
| 194 | new = memblock_alloc(PAGE_SIZE, PAGE_SIZE); |
| 195 | if (!new) |
| 196 | goto err_alloc; |
| 197 | pud_populate(&init_mm, pud, new); |
| 198 | } |
| 199 | |
| 200 | pmd = pmd_offset(pud, addr); |
| 201 | if (!pmd_present(*pmd)) { |
| 202 | pte_t *new; |
| 203 | |
| 204 | new = memblock_alloc(PAGE_SIZE, PAGE_SIZE); |
| 205 | if (!new) |
| 206 | goto err_alloc; |
| 207 | pmd_populate_kernel(&init_mm, pmd, new); |
| 208 | } |
| 209 | |
| 210 | return; |
| 211 | |
| 212 | err_alloc: |
| 213 | panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n", |
| 214 | __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); |
| 215 | } |
| 216 | #endif |
| 217 | |
Zhen Lei | 7af3a0a | 2016-09-01 14:55:00 +0800 | [diff] [blame] | 218 | void __init setup_per_cpu_areas(void) |
| 219 | { |
| 220 | unsigned long delta; |
| 221 | unsigned int cpu; |
Kefeng Wang | 09cea61 | 2021-11-05 13:39:44 -0700 | [diff] [blame] | 222 | int rc = -EINVAL; |
Zhen Lei | 7af3a0a | 2016-09-01 14:55:00 +0800 | [diff] [blame] | 223 | |
Kefeng Wang | 09cea61 | 2021-11-05 13:39:44 -0700 | [diff] [blame] | 224 | if (pcpu_chosen_fc != PCPU_FC_PAGE) { |
| 225 | /* |
| 226 | * Always reserve area for module percpu variables. That's |
| 227 | * what the legacy allocator did. |
| 228 | */ |
| 229 | rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, |
| 230 | PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, |
| 231 | pcpu_cpu_distance, |
| 232 | pcpu_fc_alloc, pcpu_fc_free); |
| 233 | #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK |
| 234 | if (rc < 0) |
| 235 | pr_warn("PERCPU: %s allocator failed (%d), falling back to page size\n", |
| 236 | pcpu_fc_names[pcpu_chosen_fc], rc); |
| 237 | #endif |
| 238 | } |
| 239 | |
| 240 | #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK |
Zhen Lei | 7af3a0a | 2016-09-01 14:55:00 +0800 | [diff] [blame] | 241 | if (rc < 0) |
Kefeng Wang | 09cea61 | 2021-11-05 13:39:44 -0700 | [diff] [blame] | 242 | rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, |
| 243 | pcpu_fc_alloc, |
| 244 | pcpu_fc_free, |
| 245 | pcpu_populate_pte); |
| 246 | #endif |
| 247 | if (rc < 0) |
| 248 | panic("Failed to initialize percpu areas (err=%d).", rc); |
Zhen Lei | 7af3a0a | 2016-09-01 14:55:00 +0800 | [diff] [blame] | 249 | |
| 250 | delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; |
| 251 | for_each_possible_cpu(cpu) |
| 252 | __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu]; |
| 253 | } |
| 254 | #endif |
| 255 | |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 256 | /** |
Qian Cai | b1ce45e | 2019-03-14 12:16:19 -0400 | [diff] [blame] | 257 | * numa_add_memblk() - Set node id to memblk |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 258 | * @nid: NUMA node ID of the new memblk |
| 259 | * @start: Start address of the new memblk |
Hanjun Guo | 8ccbbda | 2016-05-24 15:35:36 -0700 | [diff] [blame] | 260 | * @end: End address of the new memblk |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 261 | * |
| 262 | * RETURNS: |
| 263 | * 0 on success, -errno on failure. |
| 264 | */ |
Hanjun Guo | 8ccbbda | 2016-05-24 15:35:36 -0700 | [diff] [blame] | 265 | int __init numa_add_memblk(int nid, u64 start, u64 end) |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 266 | { |
| 267 | int ret; |
| 268 | |
Hanjun Guo | 8ccbbda | 2016-05-24 15:35:36 -0700 | [diff] [blame] | 269 | ret = memblock_set_node(start, (end - start), &memblock.memory, nid); |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 270 | if (ret < 0) { |
Kefeng Wang | f11c7ba | 2016-09-01 14:54:59 +0800 | [diff] [blame] | 271 | pr_err("memblock [0x%llx - 0x%llx] failed to add on node %d\n", |
Hanjun Guo | 8ccbbda | 2016-05-24 15:35:36 -0700 | [diff] [blame] | 272 | start, (end - 1), nid); |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 273 | return ret; |
| 274 | } |
| 275 | |
| 276 | node_set(nid, numa_nodes_parsed); |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 277 | return ret; |
| 278 | } |
| 279 | |
Qian Cai | b1ce45e | 2019-03-14 12:16:19 -0400 | [diff] [blame] | 280 | /* |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 281 | * Initialize NODE_DATA for a node on the local memory |
| 282 | */ |
| 283 | static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn) |
| 284 | { |
| 285 | const size_t nd_size = roundup(sizeof(pg_data_t), SMP_CACHE_BYTES); |
| 286 | u64 nd_pa; |
| 287 | void *nd; |
| 288 | int tnid; |
| 289 | |
Punit Agrawal | ece4b20 | 2017-07-20 12:03:59 +0100 | [diff] [blame] | 290 | if (start_pfn >= end_pfn) |
Hanjun Guo | 3f7a09f | 2016-10-21 16:13:56 +0800 | [diff] [blame] | 291 | pr_info("Initmem setup node %d [<memory-less node>]\n", nid); |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 292 | |
Mike Rapoport | 9a8dd70 | 2018-10-30 15:07:59 -0700 | [diff] [blame] | 293 | nd_pa = memblock_phys_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid); |
Mike Rapoport | 3375557 | 2019-03-11 23:29:21 -0700 | [diff] [blame] | 294 | if (!nd_pa) |
| 295 | panic("Cannot allocate %zu bytes for node %d data\n", |
| 296 | nd_size, nid); |
| 297 | |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 298 | nd = __va(nd_pa); |
| 299 | |
| 300 | /* report and initialize */ |
Kefeng Wang | f11c7ba | 2016-09-01 14:54:59 +0800 | [diff] [blame] | 301 | pr_info("NODE_DATA [mem %#010Lx-%#010Lx]\n", |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 302 | nd_pa, nd_pa + nd_size - 1); |
| 303 | tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT); |
| 304 | if (tnid != nid) |
Kefeng Wang | f11c7ba | 2016-09-01 14:54:59 +0800 | [diff] [blame] | 305 | pr_info("NODE_DATA(%d) on node %d\n", nid, tnid); |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 306 | |
| 307 | node_data[nid] = nd; |
| 308 | memset(NODE_DATA(nid), 0, sizeof(pg_data_t)); |
| 309 | NODE_DATA(nid)->node_id = nid; |
| 310 | NODE_DATA(nid)->node_start_pfn = start_pfn; |
| 311 | NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn; |
| 312 | } |
| 313 | |
Qian Cai | b1ce45e | 2019-03-14 12:16:19 -0400 | [diff] [blame] | 314 | /* |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 315 | * numa_free_distance |
| 316 | * |
| 317 | * The current table is freed. |
| 318 | */ |
| 319 | void __init numa_free_distance(void) |
| 320 | { |
| 321 | size_t size; |
| 322 | |
| 323 | if (!numa_distance) |
| 324 | return; |
| 325 | |
| 326 | size = numa_distance_cnt * numa_distance_cnt * |
| 327 | sizeof(numa_distance[0]); |
| 328 | |
Mike Rapoport | 4421cca0 | 2021-11-05 13:43:22 -0700 | [diff] [blame] | 329 | memblock_free(numa_distance, size); |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 330 | numa_distance_cnt = 0; |
| 331 | numa_distance = NULL; |
| 332 | } |
| 333 | |
Qian Cai | b1ce45e | 2019-03-14 12:16:19 -0400 | [diff] [blame] | 334 | /* |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 335 | * Create a new NUMA distance table. |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 336 | */ |
| 337 | static int __init numa_alloc_distance(void) |
| 338 | { |
| 339 | size_t size; |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 340 | int i, j; |
| 341 | |
| 342 | size = nr_node_ids * nr_node_ids * sizeof(numa_distance[0]); |
Mike Rapoport | 5787ea5 | 2021-11-05 13:43:07 -0700 | [diff] [blame] | 343 | numa_distance = memblock_alloc(size, PAGE_SIZE); |
| 344 | if (WARN_ON(!numa_distance)) |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 345 | return -ENOMEM; |
| 346 | |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 347 | numa_distance_cnt = nr_node_ids; |
| 348 | |
| 349 | /* fill with the default distances */ |
| 350 | for (i = 0; i < numa_distance_cnt; i++) |
| 351 | for (j = 0; j < numa_distance_cnt; j++) |
| 352 | numa_distance[i * numa_distance_cnt + j] = i == j ? |
| 353 | LOCAL_DISTANCE : REMOTE_DISTANCE; |
| 354 | |
Kefeng Wang | f11c7ba | 2016-09-01 14:54:59 +0800 | [diff] [blame] | 355 | pr_debug("Initialized distance table, cnt=%d\n", numa_distance_cnt); |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 356 | |
| 357 | return 0; |
| 358 | } |
| 359 | |
| 360 | /** |
Qian Cai | b1ce45e | 2019-03-14 12:16:19 -0400 | [diff] [blame] | 361 | * numa_set_distance() - Set inter node NUMA distance from node to node. |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 362 | * @from: the 'from' node to set distance |
| 363 | * @to: the 'to' node to set distance |
| 364 | * @distance: NUMA distance |
| 365 | * |
| 366 | * Set the distance from node @from to @to to @distance. |
| 367 | * If distance table doesn't exist, a warning is printed. |
| 368 | * |
| 369 | * If @from or @to is higher than the highest known node or lower than zero |
| 370 | * or @distance doesn't make sense, the call is ignored. |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 371 | */ |
| 372 | void __init numa_set_distance(int from, int to, int distance) |
| 373 | { |
| 374 | if (!numa_distance) { |
Kefeng Wang | f11c7ba | 2016-09-01 14:54:59 +0800 | [diff] [blame] | 375 | pr_warn_once("Warning: distance table not allocated yet\n"); |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 376 | return; |
| 377 | } |
| 378 | |
| 379 | if (from >= numa_distance_cnt || to >= numa_distance_cnt || |
| 380 | from < 0 || to < 0) { |
Kefeng Wang | f11c7ba | 2016-09-01 14:54:59 +0800 | [diff] [blame] | 381 | pr_warn_once("Warning: node ids are out of bound, from=%d to=%d distance=%d\n", |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 382 | from, to, distance); |
| 383 | return; |
| 384 | } |
| 385 | |
| 386 | if ((u8)distance != distance || |
| 387 | (from == to && distance != LOCAL_DISTANCE)) { |
Kefeng Wang | f11c7ba | 2016-09-01 14:54:59 +0800 | [diff] [blame] | 388 | pr_warn_once("Warning: invalid distance parameter, from=%d to=%d distance=%d\n", |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 389 | from, to, distance); |
| 390 | return; |
| 391 | } |
| 392 | |
| 393 | numa_distance[from * numa_distance_cnt + to] = distance; |
| 394 | } |
| 395 | |
Qian Cai | b1ce45e | 2019-03-14 12:16:19 -0400 | [diff] [blame] | 396 | /* |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 397 | * Return NUMA distance @from to @to |
| 398 | */ |
| 399 | int __node_distance(int from, int to) |
| 400 | { |
| 401 | if (from >= numa_distance_cnt || to >= numa_distance_cnt) |
| 402 | return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE; |
| 403 | return numa_distance[from * numa_distance_cnt + to]; |
| 404 | } |
| 405 | EXPORT_SYMBOL(__node_distance); |
| 406 | |
| 407 | static int __init numa_register_nodes(void) |
| 408 | { |
| 409 | int nid; |
| 410 | struct memblock_region *mblk; |
| 411 | |
| 412 | /* Check that valid nid is set to memblks */ |
Mike Rapoport | cc6de168 | 2020-10-13 16:58:30 -0700 | [diff] [blame] | 413 | for_each_mem_region(mblk) { |
Mike Rapoport | d622abf | 2020-06-03 15:56:53 -0700 | [diff] [blame] | 414 | int mblk_nid = memblock_get_region_node(mblk); |
Randy Dunlap | 4cd48bb | 2021-01-27 19:55:33 -0800 | [diff] [blame] | 415 | phys_addr_t start = mblk->base; |
| 416 | phys_addr_t end = mblk->base + mblk->size - 1; |
Mike Rapoport | d622abf | 2020-06-03 15:56:53 -0700 | [diff] [blame] | 417 | |
| 418 | if (mblk_nid == NUMA_NO_NODE || mblk_nid >= MAX_NUMNODES) { |
Randy Dunlap | 4cd48bb | 2021-01-27 19:55:33 -0800 | [diff] [blame] | 419 | pr_warn("Warning: invalid memblk node %d [mem %pap-%pap]\n", |
| 420 | mblk_nid, &start, &end); |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 421 | return -EINVAL; |
| 422 | } |
Mike Rapoport | d622abf | 2020-06-03 15:56:53 -0700 | [diff] [blame] | 423 | } |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 424 | |
| 425 | /* Finally register nodes. */ |
| 426 | for_each_node_mask(nid, numa_nodes_parsed) { |
| 427 | unsigned long start_pfn, end_pfn; |
| 428 | |
| 429 | get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); |
| 430 | setup_node_data(nid, start_pfn, end_pfn); |
| 431 | node_set_online(nid); |
| 432 | } |
| 433 | |
| 434 | /* Setup online nodes to actual nodes*/ |
| 435 | node_possible_map = numa_nodes_parsed; |
| 436 | |
| 437 | return 0; |
| 438 | } |
| 439 | |
| 440 | static int __init numa_init(int (*init_func)(void)) |
| 441 | { |
| 442 | int ret; |
| 443 | |
| 444 | nodes_clear(numa_nodes_parsed); |
| 445 | nodes_clear(node_possible_map); |
| 446 | nodes_clear(node_online_map); |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 447 | |
| 448 | ret = numa_alloc_distance(); |
| 449 | if (ret < 0) |
| 450 | return ret; |
| 451 | |
| 452 | ret = init_func(); |
| 453 | if (ret < 0) |
Anshuman Khandual | 5233808 | 2018-09-22 21:09:56 +0530 | [diff] [blame] | 454 | goto out_free_distance; |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 455 | |
Zhen Lei | 794224ea | 2016-09-01 14:54:56 +0800 | [diff] [blame] | 456 | if (nodes_empty(numa_nodes_parsed)) { |
| 457 | pr_info("No NUMA configuration found\n"); |
Anshuman Khandual | 5233808 | 2018-09-22 21:09:56 +0530 | [diff] [blame] | 458 | ret = -EINVAL; |
| 459 | goto out_free_distance; |
Zhen Lei | 794224ea | 2016-09-01 14:54:56 +0800 | [diff] [blame] | 460 | } |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 461 | |
| 462 | ret = numa_register_nodes(); |
| 463 | if (ret < 0) |
Anshuman Khandual | 5233808 | 2018-09-22 21:09:56 +0530 | [diff] [blame] | 464 | goto out_free_distance; |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 465 | |
| 466 | setup_node_to_cpumask_map(); |
| 467 | |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 468 | return 0; |
Anshuman Khandual | 5233808 | 2018-09-22 21:09:56 +0530 | [diff] [blame] | 469 | out_free_distance: |
| 470 | numa_free_distance(); |
| 471 | return ret; |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 472 | } |
| 473 | |
| 474 | /** |
Qian Cai | b1ce45e | 2019-03-14 12:16:19 -0400 | [diff] [blame] | 475 | * dummy_numa_init() - Fallback dummy NUMA init |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 476 | * |
| 477 | * Used if there's no underlying NUMA architecture, NUMA initialization |
| 478 | * fails, or NUMA is disabled on the command line. |
| 479 | * |
| 480 | * Must online at least one node (node 0) and add memory blocks that cover all |
| 481 | * allowed memory. It is unlikely that this function fails. |
Qian Cai | b1ce45e | 2019-03-14 12:16:19 -0400 | [diff] [blame] | 482 | * |
| 483 | * Return: 0 on success, -errno on failure. |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 484 | */ |
| 485 | static int __init dummy_numa_init(void) |
| 486 | { |
Mike Rapoport | ab8f21a | 2020-10-13 16:57:31 -0700 | [diff] [blame] | 487 | phys_addr_t start = memblock_start_of_DRAM(); |
Randy Dunlap | 4cd48bb | 2021-01-27 19:55:33 -0800 | [diff] [blame] | 488 | phys_addr_t end = memblock_end_of_DRAM() - 1; |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 489 | int ret; |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 490 | |
David Daney | 34c3337 | 2016-05-24 15:35:37 -0700 | [diff] [blame] | 491 | if (numa_off) |
| 492 | pr_info("NUMA disabled\n"); /* Forced off on command line. */ |
Randy Dunlap | 4cd48bb | 2021-01-27 19:55:33 -0800 | [diff] [blame] | 493 | pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end); |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 494 | |
Randy Dunlap | 4cd48bb | 2021-01-27 19:55:33 -0800 | [diff] [blame] | 495 | ret = numa_add_memblk(0, start, end + 1); |
Mike Rapoport | ab8f21a | 2020-10-13 16:57:31 -0700 | [diff] [blame] | 496 | if (ret) { |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 497 | pr_err("NUMA init failed\n"); |
| 498 | return ret; |
| 499 | } |
| 500 | |
David Daney | 34c3337 | 2016-05-24 15:35:37 -0700 | [diff] [blame] | 501 | numa_off = true; |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 502 | return 0; |
| 503 | } |
| 504 | |
Atish Patra | eb75541 | 2020-11-18 16:38:25 -0800 | [diff] [blame] | 505 | #ifdef CONFIG_ACPI_NUMA |
| 506 | static int __init arch_acpi_numa_init(void) |
| 507 | { |
| 508 | int ret; |
| 509 | |
| 510 | ret = acpi_numa_init(); |
| 511 | if (ret) { |
| 512 | pr_info("Failed to initialise from firmware\n"); |
| 513 | return ret; |
| 514 | } |
| 515 | |
| 516 | return srat_disabled() ? -EINVAL : 0; |
| 517 | } |
| 518 | #else |
| 519 | static int __init arch_acpi_numa_init(void) |
| 520 | { |
| 521 | return -EOPNOTSUPP; |
| 522 | } |
| 523 | #endif |
| 524 | |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 525 | /** |
Atish Patra | eb75541 | 2020-11-18 16:38:25 -0800 | [diff] [blame] | 526 | * arch_numa_init() - Initialize NUMA |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 527 | * |
Qian Cai | b1ce45e | 2019-03-14 12:16:19 -0400 | [diff] [blame] | 528 | * Try each configured NUMA initialization method until one succeeds. The |
Yanfei Xu | 9a747c9 | 2020-09-01 17:11:54 +0800 | [diff] [blame] | 529 | * last fallback is dummy single node config encompassing whole memory. |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 530 | */ |
Atish Patra | eb75541 | 2020-11-18 16:38:25 -0800 | [diff] [blame] | 531 | void __init arch_numa_init(void) |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 532 | { |
| 533 | if (!numa_off) { |
Atish Patra | eb75541 | 2020-11-18 16:38:25 -0800 | [diff] [blame] | 534 | if (!acpi_disabled && !numa_init(arch_acpi_numa_init)) |
Hanjun Guo | d8b47fc | 2016-05-24 15:35:44 -0700 | [diff] [blame] | 535 | return; |
| 536 | if (acpi_disabled && !numa_init(of_numa_init)) |
Ganapatrao Kulkarni | 1a2db30 | 2016-04-08 15:50:27 -0700 | [diff] [blame] | 537 | return; |
| 538 | } |
| 539 | |
| 540 | numa_init(dummy_numa_init); |
| 541 | } |