Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 1 | /* |
| 2 | * linux/mm/allocpercpu.c |
| 3 | * |
Christoph Lameter | cde5353 | 2008-07-04 09:59:22 -0700 | [diff] [blame] | 4 | * Separated from slab.c August 11, 2006 Christoph Lameter |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 5 | */ |
| 6 | #include <linux/mm.h> |
| 7 | #include <linux/module.h> |
| 8 | |
Eric Dumazet | be85279 | 2008-03-04 14:28:35 -0800 | [diff] [blame] | 9 | #ifndef cache_line_size |
| 10 | #define cache_line_size() L1_CACHE_BYTES |
| 11 | #endif |
| 12 | |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 13 | /** |
| 14 | * percpu_depopulate - depopulate per-cpu data for given cpu |
| 15 | * @__pdata: per-cpu data to depopulate |
| 16 | * @cpu: depopulate per-cpu data for this cpu |
| 17 | * |
| 18 | * Depopulating per-cpu data for a cpu going offline would be a typical |
| 19 | * use case. You need to register a cpu hotplug handler for that purpose. |
| 20 | */ |
Adrian Bunk | 9d8fddfb | 2008-07-25 19:46:23 -0700 | [diff] [blame] | 21 | static void percpu_depopulate(void *__pdata, int cpu) |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 22 | { |
| 23 | struct percpu_data *pdata = __percpu_disguise(__pdata); |
Alan Stern | a120586 | 2006-12-06 20:32:37 -0800 | [diff] [blame] | 24 | |
| 25 | kfree(pdata->ptrs[cpu]); |
| 26 | pdata->ptrs[cpu] = NULL; |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 27 | } |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 28 | |
| 29 | /** |
| 30 | * percpu_depopulate_mask - depopulate per-cpu data for some cpu's |
| 31 | * @__pdata: per-cpu data to depopulate |
| 32 | * @mask: depopulate per-cpu data for cpu's selected through mask bits |
| 33 | */ |
Adrian Bunk | 9d8fddfb | 2008-07-25 19:46:23 -0700 | [diff] [blame] | 34 | static void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask) |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 35 | { |
| 36 | int cpu; |
Mike Travis | 6d6a436 | 2008-05-12 21:21:13 +0200 | [diff] [blame] | 37 | for_each_cpu_mask_nr(cpu, *mask) |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 38 | percpu_depopulate(__pdata, cpu); |
| 39 | } |
Adrian Bunk | 9d8fddfb | 2008-07-25 19:46:23 -0700 | [diff] [blame] | 40 | |
| 41 | #define percpu_depopulate_mask(__pdata, mask) \ |
| 42 | __percpu_depopulate_mask((__pdata), &(mask)) |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 43 | |
| 44 | /** |
| 45 | * percpu_populate - populate per-cpu data for given cpu |
| 46 | * @__pdata: per-cpu data to populate further |
| 47 | * @size: size of per-cpu object |
| 48 | * @gfp: may sleep or not etc. |
| 49 | * @cpu: populate per-data for this cpu |
| 50 | * |
| 51 | * Populating per-cpu data for a cpu coming online would be a typical |
| 52 | * use case. You need to register a cpu hotplug handler for that purpose. |
| 53 | * Per-cpu object is populated with zeroed buffer. |
| 54 | */ |
Adrian Bunk | 9d8fddfb | 2008-07-25 19:46:23 -0700 | [diff] [blame] | 55 | static void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu) |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 56 | { |
| 57 | struct percpu_data *pdata = __percpu_disguise(__pdata); |
| 58 | int node = cpu_to_node(cpu); |
| 59 | |
Eric Dumazet | be85279 | 2008-03-04 14:28:35 -0800 | [diff] [blame] | 60 | /* |
| 61 | * We should make sure each CPU gets private memory. |
| 62 | */ |
| 63 | size = roundup(size, cache_line_size()); |
| 64 | |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 65 | BUG_ON(pdata->ptrs[cpu]); |
Christoph Lameter | 94f6030 | 2007-07-17 04:03:29 -0700 | [diff] [blame] | 66 | if (node_online(node)) |
| 67 | pdata->ptrs[cpu] = kmalloc_node(size, gfp|__GFP_ZERO, node); |
| 68 | else |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 69 | pdata->ptrs[cpu] = kzalloc(size, gfp); |
| 70 | return pdata->ptrs[cpu]; |
| 71 | } |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 72 | |
| 73 | /** |
| 74 | * percpu_populate_mask - populate per-cpu data for more cpu's |
| 75 | * @__pdata: per-cpu data to populate further |
| 76 | * @size: size of per-cpu object |
| 77 | * @gfp: may sleep or not etc. |
| 78 | * @mask: populate per-cpu data for cpu's selected through mask bits |
| 79 | * |
| 80 | * Per-cpu objects are populated with zeroed buffers. |
| 81 | */ |
Adrian Bunk | 9d8fddfb | 2008-07-25 19:46:23 -0700 | [diff] [blame] | 82 | static int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp, |
| 83 | cpumask_t *mask) |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 84 | { |
Mike Travis | d366f8c | 2008-04-04 18:11:12 -0700 | [diff] [blame] | 85 | cpumask_t populated; |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 86 | int cpu; |
| 87 | |
Mike Travis | d366f8c | 2008-04-04 18:11:12 -0700 | [diff] [blame] | 88 | cpus_clear(populated); |
Mike Travis | 6d6a436 | 2008-05-12 21:21:13 +0200 | [diff] [blame] | 89 | for_each_cpu_mask_nr(cpu, *mask) |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 90 | if (unlikely(!percpu_populate(__pdata, size, gfp, cpu))) { |
| 91 | __percpu_depopulate_mask(__pdata, &populated); |
| 92 | return -ENOMEM; |
| 93 | } else |
| 94 | cpu_set(cpu, populated); |
| 95 | return 0; |
| 96 | } |
Adrian Bunk | 9d8fddfb | 2008-07-25 19:46:23 -0700 | [diff] [blame] | 97 | |
| 98 | #define percpu_populate_mask(__pdata, size, gfp, mask) \ |
| 99 | __percpu_populate_mask((__pdata), (size), (gfp), &(mask)) |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 100 | |
| 101 | /** |
| 102 | * percpu_alloc_mask - initial setup of per-cpu data |
| 103 | * @size: size of per-cpu object |
| 104 | * @gfp: may sleep or not etc. |
| 105 | * @mask: populate per-data for cpu's selected through mask bits |
| 106 | * |
| 107 | * Populating per-cpu data for all online cpu's would be a typical use case, |
| 108 | * which is simplified by the percpu_alloc() wrapper. |
| 109 | * Per-cpu objects are populated with zeroed buffers. |
| 110 | */ |
| 111 | void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask) |
| 112 | { |
Eric Dumazet | be85279 | 2008-03-04 14:28:35 -0800 | [diff] [blame] | 113 | /* |
| 114 | * We allocate whole cache lines to avoid false sharing |
| 115 | */ |
| 116 | size_t sz = roundup(nr_cpu_ids * sizeof(void *), cache_line_size()); |
| 117 | void *pdata = kzalloc(sz, gfp); |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 118 | void *__pdata = __percpu_disguise(pdata); |
| 119 | |
| 120 | if (unlikely(!pdata)) |
| 121 | return NULL; |
| 122 | if (likely(!__percpu_populate_mask(__pdata, size, gfp, mask))) |
| 123 | return __pdata; |
| 124 | kfree(pdata); |
| 125 | return NULL; |
| 126 | } |
| 127 | EXPORT_SYMBOL_GPL(__percpu_alloc_mask); |
| 128 | |
| 129 | /** |
| 130 | * percpu_free - final cleanup of per-cpu data |
| 131 | * @__pdata: object to clean up |
| 132 | * |
| 133 | * We simply clean up any per-cpu object left. No need for the client to |
| 134 | * track and specify through a bis mask which per-cpu objects are to free. |
| 135 | */ |
| 136 | void percpu_free(void *__pdata) |
| 137 | { |
Alan Stern | a120586 | 2006-12-06 20:32:37 -0800 | [diff] [blame] | 138 | if (unlikely(!__pdata)) |
| 139 | return; |
Christoph Lameter | d00bcc9 | 2006-09-25 23:31:50 -0700 | [diff] [blame] | 140 | __percpu_depopulate_mask(__pdata, &cpu_possible_map); |
| 141 | kfree(__percpu_disguise(__pdata)); |
| 142 | } |
| 143 | EXPORT_SYMBOL_GPL(percpu_free); |