KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | /* |
| 3 | * Copyright (c) 2020 Facebook |
| 4 | * Copyright 2020 Google LLC. |
| 5 | */ |
| 6 | |
| 7 | #include <linux/pid.h> |
| 8 | #include <linux/sched.h> |
| 9 | #include <linux/rculist.h> |
| 10 | #include <linux/list.h> |
| 11 | #include <linux/hash.h> |
| 12 | #include <linux/types.h> |
| 13 | #include <linux/spinlock.h> |
| 14 | #include <linux/bpf.h> |
| 15 | #include <linux/bpf_local_storage.h> |
| 16 | #include <linux/filter.h> |
| 17 | #include <uapi/linux/btf.h> |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 18 | #include <linux/btf_ids.h> |
| 19 | #include <linux/fdtable.h> |
KP Singh | 0fe4b38 | 2021-12-24 15:29:15 +0000 | [diff] [blame] | 20 | #include <linux/rcupdate_trace.h> |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 21 | |
| 22 | DEFINE_BPF_STORAGE_CACHE(task_cache); |
| 23 | |
Wei Yongjun | 4d0b938 | 2021-03-11 13:15:05 +0000 | [diff] [blame] | 24 | static DEFINE_PER_CPU(int, bpf_task_storage_busy); |
Song Liu | bc235cd | 2021-02-25 15:43:15 -0800 | [diff] [blame] | 25 | |
| 26 | static void bpf_task_storage_lock(void) |
| 27 | { |
| 28 | migrate_disable(); |
Hou Tao | 197827a0 | 2022-09-01 14:19:35 +0800 | [diff] [blame] | 29 | this_cpu_inc(bpf_task_storage_busy); |
Song Liu | bc235cd | 2021-02-25 15:43:15 -0800 | [diff] [blame] | 30 | } |
| 31 | |
| 32 | static void bpf_task_storage_unlock(void) |
| 33 | { |
Hou Tao | 197827a0 | 2022-09-01 14:19:35 +0800 | [diff] [blame] | 34 | this_cpu_dec(bpf_task_storage_busy); |
Song Liu | bc235cd | 2021-02-25 15:43:15 -0800 | [diff] [blame] | 35 | migrate_enable(); |
| 36 | } |
| 37 | |
| 38 | static bool bpf_task_storage_trylock(void) |
| 39 | { |
| 40 | migrate_disable(); |
Hou Tao | 197827a0 | 2022-09-01 14:19:35 +0800 | [diff] [blame] | 41 | if (unlikely(this_cpu_inc_return(bpf_task_storage_busy) != 1)) { |
| 42 | this_cpu_dec(bpf_task_storage_busy); |
Song Liu | bc235cd | 2021-02-25 15:43:15 -0800 | [diff] [blame] | 43 | migrate_enable(); |
| 44 | return false; |
| 45 | } |
| 46 | return true; |
| 47 | } |
| 48 | |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 49 | static struct bpf_local_storage __rcu **task_storage_ptr(void *owner) |
| 50 | { |
| 51 | struct task_struct *task = owner; |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 52 | |
Song Liu | a10787e | 2021-02-25 15:43:14 -0800 | [diff] [blame] | 53 | return &task->bpf_storage; |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 54 | } |
| 55 | |
| 56 | static struct bpf_local_storage_data * |
| 57 | task_storage_lookup(struct task_struct *task, struct bpf_map *map, |
| 58 | bool cacheit_lockit) |
| 59 | { |
| 60 | struct bpf_local_storage *task_storage; |
| 61 | struct bpf_local_storage_map *smap; |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 62 | |
KP Singh | 0fe4b38 | 2021-12-24 15:29:15 +0000 | [diff] [blame] | 63 | task_storage = |
| 64 | rcu_dereference_check(task->bpf_storage, bpf_rcu_lock_held()); |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 65 | if (!task_storage) |
| 66 | return NULL; |
| 67 | |
| 68 | smap = (struct bpf_local_storage_map *)map; |
| 69 | return bpf_local_storage_lookup(task_storage, smap, cacheit_lockit); |
| 70 | } |
| 71 | |
| 72 | void bpf_task_storage_free(struct task_struct *task) |
| 73 | { |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 74 | struct bpf_local_storage *local_storage; |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 75 | |
| 76 | rcu_read_lock(); |
| 77 | |
Song Liu | a10787e | 2021-02-25 15:43:14 -0800 | [diff] [blame] | 78 | local_storage = rcu_dereference(task->bpf_storage); |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 79 | if (!local_storage) { |
| 80 | rcu_read_unlock(); |
| 81 | return; |
| 82 | } |
| 83 | |
Song Liu | bc235cd | 2021-02-25 15:43:15 -0800 | [diff] [blame] | 84 | bpf_task_storage_lock(); |
Martin KaFai Lau | 2ffcb6f | 2023-03-07 22:59:21 -0800 | [diff] [blame] | 85 | bpf_local_storage_destroy(local_storage); |
Song Liu | bc235cd | 2021-02-25 15:43:15 -0800 | [diff] [blame] | 86 | bpf_task_storage_unlock(); |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 87 | rcu_read_unlock(); |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 88 | } |
| 89 | |
| 90 | static void *bpf_pid_task_storage_lookup_elem(struct bpf_map *map, void *key) |
| 91 | { |
| 92 | struct bpf_local_storage_data *sdata; |
| 93 | struct task_struct *task; |
| 94 | unsigned int f_flags; |
| 95 | struct pid *pid; |
| 96 | int fd, err; |
| 97 | |
| 98 | fd = *(int *)key; |
| 99 | pid = pidfd_get_pid(fd, &f_flags); |
| 100 | if (IS_ERR(pid)) |
| 101 | return ERR_CAST(pid); |
| 102 | |
| 103 | /* We should be in an RCU read side critical section, it should be safe |
| 104 | * to call pid_task. |
| 105 | */ |
| 106 | WARN_ON_ONCE(!rcu_read_lock_held()); |
| 107 | task = pid_task(pid, PIDTYPE_PID); |
| 108 | if (!task) { |
| 109 | err = -ENOENT; |
| 110 | goto out; |
| 111 | } |
| 112 | |
Song Liu | bc235cd | 2021-02-25 15:43:15 -0800 | [diff] [blame] | 113 | bpf_task_storage_lock(); |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 114 | sdata = task_storage_lookup(task, map, true); |
Song Liu | bc235cd | 2021-02-25 15:43:15 -0800 | [diff] [blame] | 115 | bpf_task_storage_unlock(); |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 116 | put_pid(pid); |
| 117 | return sdata ? sdata->data : NULL; |
| 118 | out: |
| 119 | put_pid(pid); |
| 120 | return ERR_PTR(err); |
| 121 | } |
| 122 | |
JP Kobryn | d7ba4cc | 2023-03-22 12:47:54 -0700 | [diff] [blame] | 123 | static long bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key, |
| 124 | void *value, u64 map_flags) |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 125 | { |
| 126 | struct bpf_local_storage_data *sdata; |
| 127 | struct task_struct *task; |
| 128 | unsigned int f_flags; |
| 129 | struct pid *pid; |
| 130 | int fd, err; |
| 131 | |
| 132 | fd = *(int *)key; |
| 133 | pid = pidfd_get_pid(fd, &f_flags); |
| 134 | if (IS_ERR(pid)) |
| 135 | return PTR_ERR(pid); |
| 136 | |
| 137 | /* We should be in an RCU read side critical section, it should be safe |
| 138 | * to call pid_task. |
| 139 | */ |
| 140 | WARN_ON_ONCE(!rcu_read_lock_held()); |
| 141 | task = pid_task(pid, PIDTYPE_PID); |
Song Liu | a10787e | 2021-02-25 15:43:14 -0800 | [diff] [blame] | 142 | if (!task) { |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 143 | err = -ENOENT; |
| 144 | goto out; |
| 145 | } |
| 146 | |
Song Liu | bc235cd | 2021-02-25 15:43:15 -0800 | [diff] [blame] | 147 | bpf_task_storage_lock(); |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 148 | sdata = bpf_local_storage_update( |
Joanne Koong | b00fa38 | 2022-03-17 21:55:52 -0700 | [diff] [blame] | 149 | task, (struct bpf_local_storage_map *)map, value, map_flags, |
| 150 | GFP_ATOMIC); |
Song Liu | bc235cd | 2021-02-25 15:43:15 -0800 | [diff] [blame] | 151 | bpf_task_storage_unlock(); |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 152 | |
| 153 | err = PTR_ERR_OR_ZERO(sdata); |
| 154 | out: |
| 155 | put_pid(pid); |
| 156 | return err; |
| 157 | } |
| 158 | |
Martin KaFai Lau | fda64ae | 2022-10-25 11:45:21 -0700 | [diff] [blame] | 159 | static int task_storage_delete(struct task_struct *task, struct bpf_map *map, |
| 160 | bool nobusy) |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 161 | { |
| 162 | struct bpf_local_storage_data *sdata; |
| 163 | |
| 164 | sdata = task_storage_lookup(task, map, false); |
| 165 | if (!sdata) |
| 166 | return -ENOENT; |
| 167 | |
Martin KaFai Lau | fda64ae | 2022-10-25 11:45:21 -0700 | [diff] [blame] | 168 | if (!nobusy) |
| 169 | return -EBUSY; |
| 170 | |
Martin KaFai Lau | a47eabf | 2023-03-07 22:59:25 -0800 | [diff] [blame] | 171 | bpf_selem_unlink(SELEM(sdata), false); |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 172 | |
| 173 | return 0; |
| 174 | } |
| 175 | |
JP Kobryn | d7ba4cc | 2023-03-22 12:47:54 -0700 | [diff] [blame] | 176 | static long bpf_pid_task_storage_delete_elem(struct bpf_map *map, void *key) |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 177 | { |
| 178 | struct task_struct *task; |
| 179 | unsigned int f_flags; |
| 180 | struct pid *pid; |
| 181 | int fd, err; |
| 182 | |
| 183 | fd = *(int *)key; |
| 184 | pid = pidfd_get_pid(fd, &f_flags); |
| 185 | if (IS_ERR(pid)) |
| 186 | return PTR_ERR(pid); |
| 187 | |
| 188 | /* We should be in an RCU read side critical section, it should be safe |
| 189 | * to call pid_task. |
| 190 | */ |
| 191 | WARN_ON_ONCE(!rcu_read_lock_held()); |
| 192 | task = pid_task(pid, PIDTYPE_PID); |
| 193 | if (!task) { |
| 194 | err = -ENOENT; |
| 195 | goto out; |
| 196 | } |
| 197 | |
Song Liu | bc235cd | 2021-02-25 15:43:15 -0800 | [diff] [blame] | 198 | bpf_task_storage_lock(); |
Martin KaFai Lau | fda64ae | 2022-10-25 11:45:21 -0700 | [diff] [blame] | 199 | err = task_storage_delete(task, map, true); |
Song Liu | bc235cd | 2021-02-25 15:43:15 -0800 | [diff] [blame] | 200 | bpf_task_storage_unlock(); |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 201 | out: |
| 202 | put_pid(pid); |
| 203 | return err; |
| 204 | } |
| 205 | |
Martin KaFai Lau | 6d65500 | 2022-10-25 11:45:18 -0700 | [diff] [blame] | 206 | /* Called by bpf_task_storage_get*() helpers */ |
| 207 | static void *__bpf_task_storage_get(struct bpf_map *map, |
| 208 | struct task_struct *task, void *value, |
Martin KaFai Lau | e8b0229 | 2022-10-25 11:45:19 -0700 | [diff] [blame] | 209 | u64 flags, gfp_t gfp_flags, bool nobusy) |
Martin KaFai Lau | 6d65500 | 2022-10-25 11:45:18 -0700 | [diff] [blame] | 210 | { |
| 211 | struct bpf_local_storage_data *sdata; |
| 212 | |
Martin KaFai Lau | e8b0229 | 2022-10-25 11:45:19 -0700 | [diff] [blame] | 213 | sdata = task_storage_lookup(task, map, nobusy); |
Martin KaFai Lau | 6d65500 | 2022-10-25 11:45:18 -0700 | [diff] [blame] | 214 | if (sdata) |
| 215 | return sdata->data; |
| 216 | |
| 217 | /* only allocate new storage, when the task is refcounted */ |
| 218 | if (refcount_read(&task->usage) && |
Martin KaFai Lau | e8b0229 | 2022-10-25 11:45:19 -0700 | [diff] [blame] | 219 | (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy) { |
Martin KaFai Lau | 6d65500 | 2022-10-25 11:45:18 -0700 | [diff] [blame] | 220 | sdata = bpf_local_storage_update( |
| 221 | task, (struct bpf_local_storage_map *)map, value, |
| 222 | BPF_NOEXIST, gfp_flags); |
| 223 | return IS_ERR(sdata) ? NULL : sdata->data; |
| 224 | } |
| 225 | |
| 226 | return NULL; |
| 227 | } |
| 228 | |
Joanne Koong | b00fa38 | 2022-03-17 21:55:52 -0700 | [diff] [blame] | 229 | /* *gfp_flags* is a hidden argument provided by the verifier */ |
Martin KaFai Lau | 0593dd3 | 2022-10-25 11:45:17 -0700 | [diff] [blame] | 230 | BPF_CALL_5(bpf_task_storage_get_recur, struct bpf_map *, map, struct task_struct *, |
Joanne Koong | b00fa38 | 2022-03-17 21:55:52 -0700 | [diff] [blame] | 231 | task, void *, value, u64, flags, gfp_t, gfp_flags) |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 232 | { |
Martin KaFai Lau | e8b0229 | 2022-10-25 11:45:19 -0700 | [diff] [blame] | 233 | bool nobusy; |
Martin KaFai Lau | 6d65500 | 2022-10-25 11:45:18 -0700 | [diff] [blame] | 234 | void *data; |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 235 | |
KP Singh | 0fe4b38 | 2021-12-24 15:29:15 +0000 | [diff] [blame] | 236 | WARN_ON_ONCE(!bpf_rcu_lock_held()); |
Martin KaFai Lau | 6d65500 | 2022-10-25 11:45:18 -0700 | [diff] [blame] | 237 | if (flags & ~BPF_LOCAL_STORAGE_GET_F_CREATE || !task) |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 238 | return (unsigned long)NULL; |
| 239 | |
Martin KaFai Lau | e8b0229 | 2022-10-25 11:45:19 -0700 | [diff] [blame] | 240 | nobusy = bpf_task_storage_trylock(); |
Martin KaFai Lau | 6d65500 | 2022-10-25 11:45:18 -0700 | [diff] [blame] | 241 | data = __bpf_task_storage_get(map, task, value, flags, |
Martin KaFai Lau | e8b0229 | 2022-10-25 11:45:19 -0700 | [diff] [blame] | 242 | gfp_flags, nobusy); |
| 243 | if (nobusy) |
| 244 | bpf_task_storage_unlock(); |
Martin KaFai Lau | 6d65500 | 2022-10-25 11:45:18 -0700 | [diff] [blame] | 245 | return (unsigned long)data; |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 246 | } |
| 247 | |
Martin KaFai Lau | 4279adb | 2022-10-25 11:45:20 -0700 | [diff] [blame] | 248 | /* *gfp_flags* is a hidden argument provided by the verifier */ |
| 249 | BPF_CALL_5(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *, |
| 250 | task, void *, value, u64, flags, gfp_t, gfp_flags) |
| 251 | { |
| 252 | void *data; |
| 253 | |
| 254 | WARN_ON_ONCE(!bpf_rcu_lock_held()); |
| 255 | if (flags & ~BPF_LOCAL_STORAGE_GET_F_CREATE || !task) |
| 256 | return (unsigned long)NULL; |
| 257 | |
| 258 | bpf_task_storage_lock(); |
| 259 | data = __bpf_task_storage_get(map, task, value, flags, |
| 260 | gfp_flags, true); |
| 261 | bpf_task_storage_unlock(); |
| 262 | return (unsigned long)data; |
| 263 | } |
| 264 | |
Martin KaFai Lau | 0593dd3 | 2022-10-25 11:45:17 -0700 | [diff] [blame] | 265 | BPF_CALL_2(bpf_task_storage_delete_recur, struct bpf_map *, map, struct task_struct *, |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 266 | task) |
| 267 | { |
Martin KaFai Lau | fda64ae | 2022-10-25 11:45:21 -0700 | [diff] [blame] | 268 | bool nobusy; |
Song Liu | bc235cd | 2021-02-25 15:43:15 -0800 | [diff] [blame] | 269 | int ret; |
| 270 | |
KP Singh | 0fe4b38 | 2021-12-24 15:29:15 +0000 | [diff] [blame] | 271 | WARN_ON_ONCE(!bpf_rcu_lock_held()); |
KP Singh | 1a9c72a | 2021-01-12 07:55:24 +0000 | [diff] [blame] | 272 | if (!task) |
| 273 | return -EINVAL; |
| 274 | |
Martin KaFai Lau | fda64ae | 2022-10-25 11:45:21 -0700 | [diff] [blame] | 275 | nobusy = bpf_task_storage_trylock(); |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 276 | /* This helper must only be called from places where the lifetime of the task |
| 277 | * is guaranteed. Either by being refcounted or by being protected |
| 278 | * by an RCU read-side critical section. |
| 279 | */ |
Martin KaFai Lau | fda64ae | 2022-10-25 11:45:21 -0700 | [diff] [blame] | 280 | ret = task_storage_delete(task, map, nobusy); |
| 281 | if (nobusy) |
| 282 | bpf_task_storage_unlock(); |
Song Liu | bc235cd | 2021-02-25 15:43:15 -0800 | [diff] [blame] | 283 | return ret; |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 284 | } |
| 285 | |
Martin KaFai Lau | 8a7dac37 | 2022-10-25 11:45:22 -0700 | [diff] [blame] | 286 | BPF_CALL_2(bpf_task_storage_delete, struct bpf_map *, map, struct task_struct *, |
| 287 | task) |
| 288 | { |
| 289 | int ret; |
| 290 | |
| 291 | WARN_ON_ONCE(!bpf_rcu_lock_held()); |
| 292 | if (!task) |
| 293 | return -EINVAL; |
| 294 | |
| 295 | bpf_task_storage_lock(); |
| 296 | /* This helper must only be called from places where the lifetime of the task |
| 297 | * is guaranteed. Either by being refcounted or by being protected |
| 298 | * by an RCU read-side critical section. |
| 299 | */ |
| 300 | ret = task_storage_delete(task, map, true); |
| 301 | bpf_task_storage_unlock(); |
| 302 | return ret; |
| 303 | } |
| 304 | |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 305 | static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key) |
| 306 | { |
| 307 | return -ENOTSUPP; |
| 308 | } |
| 309 | |
| 310 | static struct bpf_map *task_storage_map_alloc(union bpf_attr *attr) |
| 311 | { |
Martin KaFai Lau | 08a7ce3 | 2023-03-22 14:52:43 -0700 | [diff] [blame] | 312 | return bpf_local_storage_map_alloc(attr, &task_cache, true); |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 313 | } |
| 314 | |
| 315 | static void task_storage_map_free(struct bpf_map *map) |
| 316 | { |
Yonghong Song | c83597f | 2022-10-25 21:28:45 -0700 | [diff] [blame] | 317 | bpf_local_storage_map_free(map, &task_cache, &bpf_task_storage_busy); |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 318 | } |
| 319 | |
Yonghong Song | 3144bfa | 2022-11-29 21:21:47 -0800 | [diff] [blame] | 320 | BTF_ID_LIST_GLOBAL_SINGLE(bpf_local_storage_map_btf_id, struct, bpf_local_storage_map) |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 321 | const struct bpf_map_ops task_storage_map_ops = { |
| 322 | .map_meta_equal = bpf_map_meta_equal, |
| 323 | .map_alloc_check = bpf_local_storage_map_alloc_check, |
| 324 | .map_alloc = task_storage_map_alloc, |
| 325 | .map_free = task_storage_map_free, |
| 326 | .map_get_next_key = notsupp_get_next_key, |
| 327 | .map_lookup_elem = bpf_pid_task_storage_lookup_elem, |
| 328 | .map_update_elem = bpf_pid_task_storage_update_elem, |
| 329 | .map_delete_elem = bpf_pid_task_storage_delete_elem, |
| 330 | .map_check_btf = bpf_local_storage_map_check_btf, |
Yafang Shao | 7490b7f | 2023-03-05 12:46:11 +0000 | [diff] [blame] | 331 | .map_mem_usage = bpf_local_storage_map_mem_usage, |
Yonghong Song | 3144bfa | 2022-11-29 21:21:47 -0800 | [diff] [blame] | 332 | .map_btf_id = &bpf_local_storage_map_btf_id[0], |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 333 | .map_owner_storage_ptr = task_storage_ptr, |
| 334 | }; |
| 335 | |
Martin KaFai Lau | 0593dd3 | 2022-10-25 11:45:17 -0700 | [diff] [blame] | 336 | const struct bpf_func_proto bpf_task_storage_get_recur_proto = { |
| 337 | .func = bpf_task_storage_get_recur, |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 338 | .gpl_only = false, |
| 339 | .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, |
| 340 | .arg1_type = ARG_CONST_MAP_PTR, |
Alexei Starovoitov | 91571a5 | 2023-04-03 21:50:25 -0700 | [diff] [blame] | 341 | .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, |
Song Liu | d19ddb4 | 2021-11-12 07:02:43 -0800 | [diff] [blame] | 342 | .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 343 | .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, |
| 344 | .arg4_type = ARG_ANYTHING, |
| 345 | }; |
| 346 | |
Martin KaFai Lau | 4279adb | 2022-10-25 11:45:20 -0700 | [diff] [blame] | 347 | const struct bpf_func_proto bpf_task_storage_get_proto = { |
| 348 | .func = bpf_task_storage_get, |
| 349 | .gpl_only = false, |
| 350 | .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, |
| 351 | .arg1_type = ARG_CONST_MAP_PTR, |
Alexei Starovoitov | 91571a5 | 2023-04-03 21:50:25 -0700 | [diff] [blame] | 352 | .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, |
Martin KaFai Lau | 4279adb | 2022-10-25 11:45:20 -0700 | [diff] [blame] | 353 | .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], |
| 354 | .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, |
| 355 | .arg4_type = ARG_ANYTHING, |
| 356 | }; |
| 357 | |
Martin KaFai Lau | 0593dd3 | 2022-10-25 11:45:17 -0700 | [diff] [blame] | 358 | const struct bpf_func_proto bpf_task_storage_delete_recur_proto = { |
| 359 | .func = bpf_task_storage_delete_recur, |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 360 | .gpl_only = false, |
| 361 | .ret_type = RET_INTEGER, |
| 362 | .arg1_type = ARG_CONST_MAP_PTR, |
Alexei Starovoitov | 91571a5 | 2023-04-03 21:50:25 -0700 | [diff] [blame] | 363 | .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, |
Song Liu | d19ddb4 | 2021-11-12 07:02:43 -0800 | [diff] [blame] | 364 | .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], |
KP Singh | 4cf1bc1 | 2020-11-06 10:37:40 +0000 | [diff] [blame] | 365 | }; |
Martin KaFai Lau | 8a7dac37 | 2022-10-25 11:45:22 -0700 | [diff] [blame] | 366 | |
| 367 | const struct bpf_func_proto bpf_task_storage_delete_proto = { |
| 368 | .func = bpf_task_storage_delete, |
| 369 | .gpl_only = false, |
| 370 | .ret_type = RET_INTEGER, |
| 371 | .arg1_type = ARG_CONST_MAP_PTR, |
Alexei Starovoitov | 91571a5 | 2023-04-03 21:50:25 -0700 | [diff] [blame] | 372 | .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, |
Martin KaFai Lau | 8a7dac37 | 2022-10-25 11:45:22 -0700 | [diff] [blame] | 373 | .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], |
| 374 | }; |