| // SPDX-License-Identifier: GPL-2.0-or-later |
| /* |
| * Linux Socket Filter - Kernel level socket filtering |
| * |
| * Based on the design of the Berkeley Packet Filter. The new |
| * internal format has been designed by PLUMgrid: |
| * |
| * Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com |
| * |
| * Authors: |
| * |
| * Jay Schulist <jschlst@samba.org> |
| * Alexei Starovoitov <ast@plumgrid.com> |
| * Daniel Borkmann <dborkman@redhat.com> |
| * |
| * Andi Kleen - Fix a few bad bugs and races. |
| * Kris Katterjohn - Added many additional checks in bpf_check_classic() |
| */ |
| |
| #include <linux/atomic.h> |
| #include <linux/bpf_verifier.h> |
| #include <linux/module.h> |
| #include <linux/types.h> |
| #include <linux/mm.h> |
| #include <linux/fcntl.h> |
| #include <linux/socket.h> |
| #include <linux/sock_diag.h> |
| #include <linux/in.h> |
| #include <linux/inet.h> |
| #include <linux/netdevice.h> |
| #include <linux/if_packet.h> |
| #include <linux/if_arp.h> |
| #include <linux/gfp.h> |
| #include <net/inet_common.h> |
| #include <net/ip.h> |
| #include <net/protocol.h> |
| #include <net/netlink.h> |
| #include <linux/skbuff.h> |
| #include <linux/skmsg.h> |
| #include <net/sock.h> |
| #include <net/flow_dissector.h> |
| #include <linux/errno.h> |
| #include <linux/timer.h> |
| #include <linux/uaccess.h> |
| #include <asm/unaligned.h> |
| #include <linux/filter.h> |
| #include <linux/ratelimit.h> |
| #include <linux/seccomp.h> |
| #include <linux/if_vlan.h> |
| #include <linux/bpf.h> |
| #include <linux/btf.h> |
| #include <net/sch_generic.h> |
| #include <net/cls_cgroup.h> |
| #include <net/dst_metadata.h> |
| #include <net/dst.h> |
| #include <net/sock_reuseport.h> |
| #include <net/busy_poll.h> |
| #include <net/tcp.h> |
| #include <net/xfrm.h> |
| #include <net/udp.h> |
| #include <linux/bpf_trace.h> |
| #include <net/xdp_sock.h> |
| #include <linux/inetdevice.h> |
| #include <net/inet_hashtables.h> |
| #include <net/inet6_hashtables.h> |
| #include <net/ip_fib.h> |
| #include <net/nexthop.h> |
| #include <net/flow.h> |
| #include <net/arp.h> |
| #include <net/ipv6.h> |
| #include <net/net_namespace.h> |
| #include <linux/seg6_local.h> |
| #include <net/seg6.h> |
| #include <net/seg6_local.h> |
| #include <net/lwtunnel.h> |
| #include <net/ipv6_stubs.h> |
| #include <net/bpf_sk_storage.h> |
| #include <net/transp_v6.h> |
| #include <linux/btf_ids.h> |
| #include <net/tls.h> |
| #include <net/xdp.h> |
| #include <net/mptcp.h> |
| #include <net/netfilter/nf_conntrack_bpf.h> |
| #include <net/netkit.h> |
| #include <linux/un.h> |
| #include <net/xdp_sock_drv.h> |
| |
| #include "dev.h" |
| |
| /* Keep the struct bpf_fib_lookup small so that it fits into a cacheline */ |
| static_assert(sizeof(struct bpf_fib_lookup) == 64, "struct bpf_fib_lookup size check"); |
| |
| static const struct bpf_func_proto * |
| bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); |
| |
| int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len) |
| { |
| if (in_compat_syscall()) { |
| struct compat_sock_fprog f32; |
| |
| if (len != sizeof(f32)) |
| return -EINVAL; |
| if (copy_from_sockptr(&f32, src, sizeof(f32))) |
| return -EFAULT; |
| memset(dst, 0, sizeof(*dst)); |
| dst->len = f32.len; |
| dst->filter = compat_ptr(f32.filter); |
| } else { |
| if (len != sizeof(*dst)) |
| return -EINVAL; |
| if (copy_from_sockptr(dst, src, sizeof(*dst))) |
| return -EFAULT; |
| } |
| |
| return 0; |
| } |
| EXPORT_SYMBOL_GPL(copy_bpf_fprog_from_user); |
| |
| /** |
| * sk_filter_trim_cap - run a packet through a socket filter |
| * @sk: sock associated with &sk_buff |
| * @skb: buffer to filter |
| * @cap: limit on how short the eBPF program may trim the packet |
| * |
| * Run the eBPF program and then cut skb->data to correct size returned by |
| * the program. If pkt_len is 0 we toss packet. If skb->len is smaller |
| * than pkt_len we keep whole skb->data. This is the socket level |
| * wrapper to bpf_prog_run. It returns 0 if the packet should |
| * be accepted or -EPERM if the packet should be tossed. |
| * |
| */ |
| int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap) |
| { |
| int err; |
| struct sk_filter *filter; |
| |
| /* |
| * If the skb was allocated from pfmemalloc reserves, only |
| * allow SOCK_MEMALLOC sockets to use it as this socket is |
| * helping free memory |
| */ |
| if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC)) { |
| NET_INC_STATS(sock_net(sk), LINUX_MIB_PFMEMALLOCDROP); |
| return -ENOMEM; |
| } |
| err = BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb); |
| if (err) |
| return err; |
| |
| err = security_sock_rcv_skb(sk, skb); |
| if (err) |
| return err; |
| |
| rcu_read_lock(); |
| filter = rcu_dereference(sk->sk_filter); |
| if (filter) { |
| struct sock *save_sk = skb->sk; |
| unsigned int pkt_len; |
| |
| skb->sk = sk; |
| pkt_len = bpf_prog_run_save_cb(filter->prog, skb); |
| skb->sk = save_sk; |
| err = pkt_len ? pskb_trim(skb, max(cap, pkt_len)) : -EPERM; |
| } |
| rcu_read_unlock(); |
| |
| return err; |
| } |
| EXPORT_SYMBOL(sk_filter_trim_cap); |
| |
| BPF_CALL_1(bpf_skb_get_pay_offset, struct sk_buff *, skb) |
| { |
| return skb_get_poff(skb); |
| } |
| |
| BPF_CALL_3(bpf_skb_get_nlattr, struct sk_buff *, skb, u32, a, u32, x) |
| { |
| struct nlattr *nla; |
| |
| if (skb_is_nonlinear(skb)) |
| return 0; |
| |
| if (skb->len < sizeof(struct nlattr)) |
| return 0; |
| |
| if (a > skb->len - sizeof(struct nlattr)) |
| return 0; |
| |
| nla = nla_find((struct nlattr *) &skb->data[a], skb->len - a, x); |
| if (nla) |
| return (void *) nla - (void *) skb->data; |
| |
| return 0; |
| } |
| |
| BPF_CALL_3(bpf_skb_get_nlattr_nest, struct sk_buff *, skb, u32, a, u32, x) |
| { |
| struct nlattr *nla; |
| |
| if (skb_is_nonlinear(skb)) |
| return 0; |
| |
| if (skb->len < sizeof(struct nlattr)) |
| return 0; |
| |
| if (a > skb->len - sizeof(struct nlattr)) |
| return 0; |
| |
| nla = (struct nlattr *) &skb->data[a]; |
| if (!nla_ok(nla, skb->len - a)) |
| return 0; |
| |
| nla = nla_find_nested(nla, x); |
| if (nla) |
| return (void *) nla - (void *) skb->data; |
| |
| return 0; |
| } |
| |
| BPF_CALL_4(bpf_skb_load_helper_8, const struct sk_buff *, skb, const void *, |
| data, int, headlen, int, offset) |
| { |
| u8 tmp, *ptr; |
| const int len = sizeof(tmp); |
| |
| if (offset >= 0) { |
| if (headlen - offset >= len) |
| return *(u8 *)(data + offset); |
| if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp))) |
| return tmp; |
| } else { |
| ptr = bpf_internal_load_pointer_neg_helper(skb, offset, len); |
| if (likely(ptr)) |
| return *(u8 *)ptr; |
| } |
| |
| return -EFAULT; |
| } |
| |
| BPF_CALL_2(bpf_skb_load_helper_8_no_cache, const struct sk_buff *, skb, |
| int, offset) |
| { |
| return ____bpf_skb_load_helper_8(skb, skb->data, skb->len - skb->data_len, |
| offset); |
| } |
| |
| BPF_CALL_4(bpf_skb_load_helper_16, const struct sk_buff *, skb, const void *, |
| data, int, headlen, int, offset) |
| { |
| __be16 tmp, *ptr; |
| const int len = sizeof(tmp); |
| |
| if (offset >= 0) { |
| if (headlen - offset >= len) |
| return get_unaligned_be16(data + offset); |
| if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp))) |
| return be16_to_cpu(tmp); |
| } else { |
| ptr = bpf_internal_load_pointer_neg_helper(skb, offset, len); |
| if (likely(ptr)) |
| return get_unaligned_be16(ptr); |
| } |
| |
| return -EFAULT; |
| } |
| |
| BPF_CALL_2(bpf_skb_load_helper_16_no_cache, const struct sk_buff *, skb, |
| int, offset) |
| { |
| return ____bpf_skb_load_helper_16(skb, skb->data, skb->len - skb->data_len, |
| offset); |
| } |
| |
| BPF_CALL_4(bpf_skb_load_helper_32, const struct sk_buff *, skb, const void *, |
| data, int, headlen, int, offset) |
| { |
| __be32 tmp, *ptr; |
| const int len = sizeof(tmp); |
| |
| if (likely(offset >= 0)) { |
| if (headlen - offset >= len) |
| return get_unaligned_be32(data + offset); |
| if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp))) |
| return be32_to_cpu(tmp); |
| } else { |
| ptr = bpf_internal_load_pointer_neg_helper(skb, offset, len); |
| if (likely(ptr)) |
| return get_unaligned_be32(ptr); |
| } |
| |
| return -EFAULT; |
| } |
| |
| BPF_CALL_2(bpf_skb_load_helper_32_no_cache, const struct sk_buff *, skb, |
| int, offset) |
| { |
| return ____bpf_skb_load_helper_32(skb, skb->data, skb->len - skb->data_len, |
| offset); |
| } |
| |
| static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg, |
| struct bpf_insn *insn_buf) |
| { |
| struct bpf_insn *insn = insn_buf; |
| |
| switch (skb_field) { |
| case SKF_AD_MARK: |
| BUILD_BUG_ON(sizeof_field(struct sk_buff, mark) != 4); |
| |
| *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, |
| offsetof(struct sk_buff, mark)); |
| break; |
| |
| case SKF_AD_PKTTYPE: |
| *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_TYPE_OFFSET); |
| *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, PKT_TYPE_MAX); |
| #ifdef __BIG_ENDIAN_BITFIELD |
| *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, 5); |
| #endif |
| break; |
| |
| case SKF_AD_QUEUE: |
| BUILD_BUG_ON(sizeof_field(struct sk_buff, queue_mapping) != 2); |
| |
| *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, |
| offsetof(struct sk_buff, queue_mapping)); |
| break; |
| |
| case SKF_AD_VLAN_TAG: |
| BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_tci) != 2); |
| |
| /* dst_reg = *(u16 *) (src_reg + offsetof(vlan_tci)) */ |
| *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, |
| offsetof(struct sk_buff, vlan_tci)); |
| break; |
| case SKF_AD_VLAN_TAG_PRESENT: |
| BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_all) != 4); |
| *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, |
| offsetof(struct sk_buff, vlan_all)); |
| *insn++ = BPF_JMP_IMM(BPF_JEQ, dst_reg, 0, 1); |
| *insn++ = BPF_ALU32_IMM(BPF_MOV, dst_reg, 1); |
| break; |
| } |
| |
| return insn - insn_buf; |
| } |
| |
| static bool convert_bpf_extensions(struct sock_filter *fp, |
| struct bpf_insn **insnp) |
| { |
| struct bpf_insn *insn = *insnp; |
| u32 cnt; |
| |
| switch (fp->k) { |
| case SKF_AD_OFF + SKF_AD_PROTOCOL: |
| BUILD_BUG_ON(sizeof_field(struct sk_buff, protocol) != 2); |
| |
| /* A = *(u16 *) (CTX + offsetof(protocol)) */ |
| *insn++ = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX, |
| offsetof(struct sk_buff, protocol)); |
| /* A = ntohs(A) [emitting a nop or swap16] */ |
| *insn = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, 16); |
| break; |
| |
| case SKF_AD_OFF + SKF_AD_PKTTYPE: |
| cnt = convert_skb_access(SKF_AD_PKTTYPE, BPF_REG_A, BPF_REG_CTX, insn); |
| insn += cnt - 1; |
| break; |
| |
| case SKF_AD_OFF + SKF_AD_IFINDEX: |
| case SKF_AD_OFF + SKF_AD_HATYPE: |
| BUILD_BUG_ON(sizeof_field(struct net_device, ifindex) != 4); |
| BUILD_BUG_ON(sizeof_field(struct net_device, type) != 2); |
| |
| *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev), |
| BPF_REG_TMP, BPF_REG_CTX, |
| offsetof(struct sk_buff, dev)); |
| /* if (tmp != 0) goto pc + 1 */ |
| *insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_TMP, 0, 1); |
| *insn++ = BPF_EXIT_INSN(); |
| if (fp->k == SKF_AD_OFF + SKF_AD_IFINDEX) |
| *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_TMP, |
| offsetof(struct net_device, ifindex)); |
| else |
| *insn = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_TMP, |
| offsetof(struct net_device, type)); |
| break; |
| |
| case SKF_AD_OFF + SKF_AD_MARK: |
| cnt = convert_skb_access(SKF_AD_MARK, BPF_REG_A, BPF_REG_CTX, insn); |
| insn += cnt - 1; |
| break; |
| |
| case SKF_AD_OFF + SKF_AD_RXHASH: |
| BUILD_BUG_ON(sizeof_field(struct sk_buff, hash) != 4); |
| |
| *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX, |
| offsetof(struct sk_buff, hash)); |
| break; |
| |
| case SKF_AD_OFF + SKF_AD_QUEUE: |
| cnt = convert_skb_access(SKF_AD_QUEUE, BPF_REG_A, BPF_REG_CTX, insn); |
| insn += cnt - 1; |
| break; |
| |
| case SKF_AD_OFF + SKF_AD_VLAN_TAG: |
| cnt = convert_skb_access(SKF_AD_VLAN_TAG, |
| BPF_REG_A, BPF_REG_CTX, insn); |
| insn += cnt - 1; |
| break; |
| |
| case SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT: |
| cnt = convert_skb_access(SKF_AD_VLAN_TAG_PRESENT, |
| BPF_REG_A, BPF_REG_CTX, insn); |
| insn += cnt - 1; |
| break; |
| |
| case SKF_AD_OFF + SKF_AD_VLAN_TPID: |
| BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_proto) != 2); |
| |
| /* A = *(u16 *) (CTX + offsetof(vlan_proto)) */ |
| *insn++ = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX, |
| offsetof(struct sk_buff, vlan_proto)); |
| /* A = ntohs(A) [emitting a nop or swap16] */ |
| *insn = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, 16); |
| break; |
| |
| case SKF_AD_OFF + SKF_AD_PAY_OFFSET: |
| case SKF_AD_OFF + SKF_AD_NLATTR: |
| case SKF_AD_OFF + SKF_AD_NLATTR_NEST: |
| case SKF_AD_OFF + SKF_AD_CPU: |
| case SKF_AD_OFF + SKF_AD_RANDOM: |
| /* arg1 = CTX */ |
| *insn++ = BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX); |
| /* arg2 = A */ |
| *insn++ = BPF_MOV64_REG(BPF_REG_ARG2, BPF_REG_A); |
| /* arg3 = X */ |
| *insn++ = BPF_MOV64_REG(BPF_REG_ARG3, BPF_REG_X); |
| /* Emit call(arg1=CTX, arg2=A, arg3=X) */ |
| switch (fp->k) { |
| case SKF_AD_OFF + SKF_AD_PAY_OFFSET: |
| *insn = BPF_EMIT_CALL(bpf_skb_get_pay_offset); |
| break; |
| case SKF_AD_OFF + SKF_AD_NLATTR: |
| *insn = BPF_EMIT_CALL(bpf_skb_get_nlattr); |
| break; |
| case SKF_AD_OFF + SKF_AD_NLATTR_NEST: |
| *insn = BPF_EMIT_CALL(bpf_skb_get_nlattr_nest); |
| break; |
| case SKF_AD_OFF + SKF_AD_CPU: |
| *insn = BPF_EMIT_CALL(bpf_get_raw_cpu_id); |
| break; |
| case SKF_AD_OFF + SKF_AD_RANDOM: |
| *insn = BPF_EMIT_CALL(bpf_user_rnd_u32); |
| bpf_user_rnd_init_once(); |
| break; |
| } |
| break; |
| |
| case SKF_AD_OFF + SKF_AD_ALU_XOR_X: |
| /* A ^= X */ |
| *insn = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_X); |
| break; |
| |
| default: |
| /* This is just a dummy call to avoid letting the compiler |
| * evict __bpf_call_base() as an optimization. Placed here |
| * where no-one bothers. |
| */ |
| BUG_ON(__bpf_call_base(0, 0, 0, 0, 0) != 0); |
| return false; |
| } |
| |
| *insnp = insn; |
| return true; |
| } |
| |
| static bool convert_bpf_ld_abs(struct sock_filter *fp, struct bpf_insn **insnp) |
| { |
| const bool unaligned_ok = IS_BUILTIN(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS); |
| int size = bpf_size_to_bytes(BPF_SIZE(fp->code)); |
| bool endian = BPF_SIZE(fp->code) == BPF_H || |
| BPF_SIZE(fp->code) == BPF_W; |
| bool indirect = BPF_MODE(fp->code) == BPF_IND; |
| const int ip_align = NET_IP_ALIGN; |
| struct bpf_insn *insn = *insnp; |
| int offset = fp->k; |
| |
| if (!indirect && |
| ((unaligned_ok && offset >= 0) || |
| (!unaligned_ok && offset >= 0 && |
| offset + ip_align >= 0 && |
| offset + ip_align % size == 0))) { |
| bool ldx_off_ok = offset <= S16_MAX; |
| |
| *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H); |
| if (offset) |
| *insn++ = BPF_ALU64_IMM(BPF_SUB, BPF_REG_TMP, offset); |
| *insn++ = BPF_JMP_IMM(BPF_JSLT, BPF_REG_TMP, |
| size, 2 + endian + (!ldx_off_ok * 2)); |
| if (ldx_off_ok) { |
| *insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A, |
| BPF_REG_D, offset); |
| } else { |
| *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_D); |
| *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_TMP, offset); |
| *insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A, |
| BPF_REG_TMP, 0); |
| } |
| if (endian) |
| *insn++ = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, size * 8); |
| *insn++ = BPF_JMP_A(8); |
| } |
| |
| *insn++ = BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX); |
| *insn++ = BPF_MOV64_REG(BPF_REG_ARG2, BPF_REG_D); |
| *insn++ = BPF_MOV64_REG(BPF_REG_ARG3, BPF_REG_H); |
| if (!indirect) { |
| *insn++ = BPF_MOV64_IMM(BPF_REG_ARG4, offset); |
| } else { |
| *insn++ = BPF_MOV64_REG(BPF_REG_ARG4, BPF_REG_X); |
| if (fp->k) |
| *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_ARG4, offset); |
| } |
| |
| switch (BPF_SIZE(fp->code)) { |
| case BPF_B: |
| *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_8); |
| break; |
| case BPF_H: |
| *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_16); |
| break; |
| case BPF_W: |
| *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_32); |
| break; |
| default: |
| return false; |
| } |
| |
| *insn++ = BPF_JMP_IMM(BPF_JSGE, BPF_REG_A, 0, 2); |
| *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A); |
| *insn = BPF_EXIT_INSN(); |
| |
| *insnp = insn; |
| return true; |
| } |
| |
| /** |
| * bpf_convert_filter - convert filter program |
| * @prog: the user passed filter program |
| * @len: the length of the user passed filter program |
| * @new_prog: allocated 'struct bpf_prog' or NULL |
| * @new_len: pointer to store length of converted program |
| * @seen_ld_abs: bool whether we've seen ld_abs/ind |
| * |
| * Remap 'sock_filter' style classic BPF (cBPF) instruction set to 'bpf_insn' |
| * style extended BPF (eBPF). |
| * Conversion workflow: |
| * |
| * 1) First pass for calculating the new program length: |
| * bpf_convert_filter(old_prog, old_len, NULL, &new_len, &seen_ld_abs) |
| * |
| * 2) 2nd pass to remap in two passes: 1st pass finds new |
| * jump offsets, 2nd pass remapping: |
| * bpf_convert_filter(old_prog, old_len, new_prog, &new_len, &seen_ld_abs) |
| */ |
| static int bpf_convert_filter(struct sock_filter *prog, int len, |
| struct bpf_prog *new_prog, int *new_len, |
| bool *seen_ld_abs) |
| { |
| int new_flen = 0, pass = 0, target, i, stack_off; |
| struct bpf_insn *new_insn, *first_insn = NULL; |
| struct sock_filter *fp; |
| int *addrs = NULL; |
| u8 bpf_src; |
| |
| BUILD_BUG_ON(BPF_MEMWORDS * sizeof(u32) > MAX_BPF_STACK); |
| BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); |
| |
| if (len <= 0 || len > BPF_MAXINSNS) |
| return -EINVAL; |
| |
| if (new_prog) { |
| first_insn = new_prog->insnsi; |
| addrs = kcalloc(len, sizeof(*addrs), |
| GFP_KERNEL | __GFP_NOWARN); |
| if (!addrs) |
| return -ENOMEM; |
| } |
| |
| do_pass: |
| new_insn = first_insn; |
| fp = prog; |
| |
| /* Classic BPF related prologue emission. */ |
| if (new_prog) { |
| /* Classic BPF expects A and X to be reset first. These need |
| * to be guaranteed to be the first two instructions. |
| */ |
| *new_insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A); |
| *new_insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_X, BPF_REG_X); |
| |
| /* All programs must keep CTX in callee saved BPF_REG_CTX. |
| * In eBPF case it's done by the compiler, here we need to |
| * do this ourself. Initial CTX is present in BPF_REG_ARG1. |
| */ |
| *new_insn++ = BPF_MOV64_REG(BPF_REG_CTX, BPF_REG_ARG1); |
| if (*seen_ld_abs) { |
| /* For packet access in classic BPF, cache skb->data |
| * in callee-saved BPF R8 and skb->len - skb->data_len |
| * (headlen) in BPF R9. Since classic BPF is read-only |
| * on CTX, we only need to cache it once. |
| */ |
| *new_insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data), |
| BPF_REG_D, BPF_REG_CTX, |
| offsetof(struct sk_buff, data)); |
| *new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_H, BPF_REG_CTX, |
| offsetof(struct sk_buff, len)); |
| *new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_TMP, BPF_REG_CTX, |
| offsetof(struct sk_buff, data_len)); |
| *new_insn++ = BPF_ALU32_REG(BPF_SUB, BPF_REG_H, BPF_REG_TMP); |
| } |
| } else { |
| new_insn += 3; |
| } |
| |
| for (i = 0; i < len; fp++, i++) { |
| struct bpf_insn tmp_insns[32] = { }; |
| struct bpf_insn *insn = tmp_insns; |
| |
| if (addrs) |
| addrs[i] = new_insn - first_insn; |
| |
| switch (fp->code) { |
| /* All arithmetic insns and skb loads map as-is. */ |
| case BPF_ALU | BPF_ADD | BPF_X: |
| case BPF_ALU | BPF_ADD | BPF_K: |
| case BPF_ALU | BPF_SUB | BPF_X: |
| case BPF_ALU | BPF_SUB | BPF_K: |
| case BPF_ALU | BPF_AND | BPF_X: |
| case BPF_ALU | BPF_AND | BPF_K: |
| case BPF_ALU | BPF_OR | BPF_X: |
| case BPF_ALU | BPF_OR | BPF_K: |
| case BPF_ALU | BPF_LSH | BPF_X: |
| case BPF_ALU | BPF_LSH | BPF_K: |
| case BPF_ALU | BPF_RSH | BPF_X: |
| case BPF_ALU | BPF_RSH | BPF_K: |
| case BPF_ALU | BPF_XOR | BPF_X: |
| case BPF_ALU | BPF_XOR | BPF_K: |
| case BPF_ALU | BPF_MUL | BPF_X: |
| case BPF_ALU | BPF_MUL | BPF_K: |
| case BPF_ALU | BPF_DIV | BPF_X: |
| case BPF_ALU | BPF_DIV | BPF_K: |
| case BPF_ALU | BPF_MOD | BPF_X: |
| case BPF_ALU | BPF_MOD | BPF_K: |
| case BPF_ALU | BPF_NEG: |
| case BPF_LD | BPF_ABS | BPF_W: |
| case BPF_LD | BPF_ABS | BPF_H: |
| case BPF_LD | BPF_ABS | BPF_B: |
| case BPF_LD | BPF_IND | BPF_W: |
| case BPF_LD | BPF_IND | BPF_H: |
| case BPF_LD | BPF_IND | BPF_B: |
| /* Check for overloaded BPF extension and |
| * directly convert it if found, otherwise |
| * just move on with mapping. |
| */ |
| if (BPF_CLASS(fp->code) == BPF_LD && |
| BPF_MODE(fp->code) == BPF_ABS && |
| convert_bpf_extensions(fp, &insn)) |
| break; |
| if (BPF_CLASS(fp->code) == BPF_LD && |
| convert_bpf_ld_abs(fp, &insn)) { |
| *seen_ld_abs = true; |
| break; |
| } |
| |
| if (fp->code == (BPF_ALU | BPF_DIV | BPF_X) || |
| fp->code == (BPF_ALU | BPF_MOD | BPF_X)) { |
| *insn++ = BPF_MOV32_REG(BPF_REG_X, BPF_REG_X); |
| /* Error with exception code on div/mod by 0. |
| * For cBPF programs, this was always return 0. |
| */ |
| *insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_X, 0, 2); |
| *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A); |
| *insn++ = BPF_EXIT_INSN(); |
| } |
| |
| *insn = BPF_RAW_INSN(fp->code, BPF_REG_A, BPF_REG_X, 0, fp->k); |
| break; |
| |
| /* Jump transformation cannot use BPF block macros |
| * everywhere as offset calculation and target updates |
| * require a bit more work than the rest, i.e. jump |
| * opcodes map as-is, but offsets need adjustment. |
| */ |
| |
| #define BPF_EMIT_JMP \ |
| do { \ |
| const s32 off_min = S16_MIN, off_max = S16_MAX; \ |
| s32 off; \ |
| \ |
| if (target >= len || target < 0) \ |
| goto err; \ |
| off = addrs ? addrs[target] - addrs[i] - 1 : 0; \ |
| /* Adjust pc relative offset for 2nd or 3rd insn. */ \ |
| off -= insn - tmp_insns; \ |
| /* Reject anything not fitting into insn->off. */ \ |
| if (off < off_min || off > off_max) \ |
| goto err; \ |
| insn->off = off; \ |
| } while (0) |
| |
| case BPF_JMP | BPF_JA: |
| target = i + fp->k + 1; |
| insn->code = fp->code; |
| BPF_EMIT_JMP; |
| break; |
| |
| case BPF_JMP | BPF_JEQ | BPF_K: |
| case BPF_JMP | BPF_JEQ | BPF_X: |
| case BPF_JMP | BPF_JSET | BPF_K: |
| case BPF_JMP | BPF_JSET | BPF_X: |
| case BPF_JMP | BPF_JGT | BPF_K: |
| case BPF_JMP | BPF_JGT | BPF_X: |
| case BPF_JMP | BPF_JGE | BPF_K: |
| case BPF_JMP | BPF_JGE | BPF_X: |
| if (BPF_SRC(fp->code) == BPF_K && (int) fp->k < 0) { |
| /* BPF immediates are signed, zero extend |
| * immediate into tmp register and use it |
| * in compare insn. |
| */ |
| *insn++ = BPF_MOV32_IMM(BPF_REG_TMP, fp->k); |
| |
| insn->dst_reg = BPF_REG_A; |
| insn->src_reg = BPF_REG_TMP; |
| bpf_src = BPF_X; |
| } else { |
| insn->dst_reg = BPF_REG_A; |
| insn->imm = fp->k; |
| bpf_src = BPF_SRC(fp->code); |
| insn->src_reg = bpf_src == BPF_X ? BPF_REG_X : 0; |
| } |
| |
| /* Common case where 'jump_false' is next insn. */ |
| if (fp->jf == 0) { |
| insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src; |
| target = i + fp->jt + 1; |
| BPF_EMIT_JMP; |
| break; |
| } |
| |
| /* Convert some jumps when 'jump_true' is next insn. */ |
| if (fp->jt == 0) { |
| switch (BPF_OP(fp->code)) { |
| case BPF_JEQ: |
| insn->code = BPF_JMP | BPF_JNE | bpf_src; |
| break; |
| case BPF_JGT: |
| insn->code = BPF_JMP | BPF_JLE | bpf_src; |
| break; |
| case BPF_JGE: |
| insn->code = BPF_JMP | BPF_JLT | bpf_src; |
| break; |
| default: |
| goto jmp_rest; |
| } |
| |
| target = i + fp->jf + 1; |
| BPF_EMIT_JMP; |
| break; |
| } |
| jmp_rest: |
| /* Other jumps are mapped into two insns: Jxx and JA. */ |
| target = i + fp->jt + 1; |
| insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src; |
| BPF_EMIT_JMP; |
| insn++; |
| |
| insn->code = BPF_JMP | BPF_JA; |
| target = i + fp->jf + 1; |
| BPF_EMIT_JMP; |
| break; |
| |
| /* ldxb 4 * ([14] & 0xf) is remapped into 6 insns. */ |
| case BPF_LDX | BPF_MSH | BPF_B: { |
| struct sock_filter tmp = { |
| .code = BPF_LD | BPF_ABS | BPF_B, |
| .k = fp->k, |
| }; |
| |
| *seen_ld_abs = true; |
| |
| /* X = A */ |
| *insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A); |
| /* A = BPF_R0 = *(u8 *) (skb->data + K) */ |
| convert_bpf_ld_abs(&tmp, &insn); |
| insn++; |
| /* A &= 0xf */ |
| *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_A, 0xf); |
| /* A <<= 2 */ |
| *insn++ = BPF_ALU32_IMM(BPF_LSH, BPF_REG_A, 2); |
| /* tmp = X */ |
| *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_X); |
| /* X = A */ |
| *insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A); |
| /* A = tmp */ |
| *insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_TMP); |
| break; |
| } |
| /* RET_K is remapped into 2 insns. RET_A case doesn't need an |
| * extra mov as BPF_REG_0 is already mapped into BPF_REG_A. |
| */ |
| case BPF_RET | BPF_A: |
| case BPF_RET | BPF_K: |
| if (BPF_RVAL(fp->code) == BPF_K) |
| *insn++ = BPF_MOV32_RAW(BPF_K, BPF_REG_0, |
| 0, fp->k); |
| *insn = BPF_EXIT_INSN(); |
| break; |
| |
| /* Store to stack. */ |
| case BPF_ST: |
| case BPF_STX: |
| stack_off = fp->k * 4 + 4; |
| *insn = BPF_STX_MEM(BPF_W, BPF_REG_FP, BPF_CLASS(fp->code) == |
| BPF_ST ? BPF_REG_A : BPF_REG_X, |
| -stack_off); |
| /* check_load_and_stores() verifies that classic BPF can |
| * load from stack only after write, so tracking |
| * stack_depth for ST|STX insns is enough |
| */ |
| if (new_prog && new_prog->aux->stack_depth < stack_off) |
| new_prog->aux->stack_depth = stack_off; |
| break; |
| |
| /* Load from stack. */ |
| case BPF_LD | BPF_MEM: |
| case BPF_LDX | BPF_MEM: |
| stack_off = fp->k * 4 + 4; |
| *insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD ? |
| BPF_REG_A : BPF_REG_X, BPF_REG_FP, |
| -stack_off); |
| break; |
| |
| /* A = K or X = K */ |
| case BPF_LD | BPF_IMM: |
| case BPF_LDX | BPF_IMM: |
| *insn = BPF_MOV32_IMM(BPF_CLASS(fp->code) == BPF_LD ? |
| BPF_REG_A : BPF_REG_X, fp->k); |
| break; |
| |
| /* X = A */ |
| case BPF_MISC | BPF_TAX: |
| *insn = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A); |
| break; |
| |
| /* A = X */ |
| case BPF_MISC | BPF_TXA: |
| *insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_X); |
| break; |
| |
| /* A = skb->len or X = skb->len */ |
| case BPF_LD | BPF_W | BPF_LEN: |
| case BPF_LDX | BPF_W | BPF_LEN: |
| *insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD ? |
| BPF_REG_A : BPF_REG_X, BPF_REG_CTX, |
| offsetof(struct sk_buff, len)); |
| break; |
| |
| /* Access seccomp_data fields. */ |
| case BPF_LDX | BPF_ABS | BPF_W: |
| /* A = *(u32 *) (ctx + K) */ |
| *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX, fp->k); |
| break; |
| |
| /* Unknown instruction. */ |
| default: |
| goto err; |
| } |
| |
| insn++; |
| if (new_prog) |
| memcpy(new_insn, tmp_insns, |
| sizeof(*insn) * (insn - tmp_insns)); |
| new_insn += insn - tmp_insns; |
| } |
| |
| if (!new_prog) { |
| /* Only calculating new length. */ |
| *new_len = new_insn - first_insn; |
| if (*seen_ld_abs) |
| *new_len += 4; /* Prologue bits. */ |
| return 0; |
| } |
| |
| pass++; |
| if (new_flen != new_insn - first_insn) { |
| new_flen = new_insn - first_insn; |
| if (pass > 2) |
| goto err; |
| goto do_pass; |
| } |
| |
| kfree(addrs); |
| BUG_ON(*new_len != new_flen); |
| return 0; |
| err: |
| kfree(addrs); |
| return -EINVAL; |
| } |
| |
| /* Security: |
| * |
| * As we dont want to clear mem[] array for each packet going through |
| * __bpf_prog_run(), we check that filter loaded by user never try to read |
| * a cell if not previously written, and we check all branches to be sure |
| * a malicious user doesn't try to abuse us. |
| */ |
| static int check_load_and_stores(const struct sock_filter *filter, int flen) |
| { |
| u16 *masks, memvalid = 0; /* One bit per cell, 16 cells */ |
| int pc, ret = 0; |
| |
| BUILD_BUG_ON(BPF_MEMWORDS > 16); |
| |
| masks = kmalloc_array(flen, sizeof(*masks), GFP_KERNEL); |
| if (!masks) |
| return -ENOMEM; |
| |
| memset(masks, 0xff, flen * sizeof(*masks)); |
| |
| for (pc = 0; pc < flen; pc++) { |
| memvalid &= masks[pc]; |
| |
| switch (filter[pc].code) { |
| case BPF_ST: |
| case BPF_STX: |
| memvalid |= (1 << filter[pc].k); |
| break; |
| case BPF_LD | BPF_MEM: |
| case BPF_LDX | BPF_MEM: |
| if (!(memvalid & (1 << filter[pc].k))) { |
| ret = -EINVAL; |
| goto error; |
| } |
| break; |
| case BPF_JMP | BPF_JA: |
| /* A jump must set masks on target */ |
| masks[pc + 1 + filter[pc].k] &= memvalid; |
| memvalid = ~0; |
| break; |
| case BPF_JMP | BPF_JEQ | BPF_K: |
| case BPF_JMP | BPF_JEQ | BPF_X: |
| case BPF_JMP | BPF_JGE | BPF_K: |
| case BPF_JMP | BPF_JGE | BPF_X: |
| case BPF_JMP | BPF_JGT | BPF_K: |
| case BPF_JMP | BPF_JGT | BPF_X: |
| case BPF_JMP | BPF_JSET | BPF_K: |
| case BPF_JMP | BPF_JSET | BPF_X: |
| /* A jump must set masks on targets */ |
| masks[pc + 1 + filter[pc].jt] &= memvalid; |
| masks[pc + 1 + filter[pc].jf] &= memvalid; |
| memvalid = ~0; |
| break; |
| } |
| } |
| error: |
| kfree(masks); |
| return ret; |
| } |
| |
| static bool chk_code_allowed(u16 code_to_probe) |
| { |
| static const bool codes[] = { |
| /* 32 bit ALU operations */ |
| [BPF_ALU | BPF_ADD | BPF_K] = true, |
| [BPF_ALU | BPF_ADD | BPF_X] = true, |
| [BPF_ALU | BPF_SUB | BPF_K] = true, |
| [BPF_ALU | BPF_SUB | BPF_X] = true, |
| [BPF_ALU | BPF_MUL | BPF_K] = true, |
| [BPF_ALU | BPF_MUL | BPF_X] = true, |
| [BPF_ALU | BPF_DIV | BPF_K] = true, |
| [BPF_ALU | BPF_DIV | BPF_X] = true, |
| [BPF_ALU | BPF_MOD | BPF_K] = true, |
| [BPF_ALU | BPF_MOD | BPF_X] = true, |
| [BPF_ALU | BPF_AND | BPF_K] = true, |
| [BPF_ALU | BPF_AND | BPF_X] = true, |
| [BPF_ALU | BPF_OR | BPF_K] = true, |
| [BPF_ALU | BPF_OR | BPF_X] = true, |
| [BPF_ALU | BPF_XOR | BPF_K] = true, |
| [BPF_ALU | BPF_XOR | BPF_X] = true, |
| [BPF_ALU | BPF_LSH | BPF_K] = true, |
| [BPF_ALU | BPF_LSH | BPF_X] = true, |
| [BPF_ALU | BPF_RSH | BPF_K] = true, |
| [BPF_ALU | BPF_RSH | BPF_X] = true, |
| [BPF_ALU | BPF_NEG] = true, |
| /* Load instructions */ |
| [BPF_LD | BPF_W | BPF_ABS] = true, |
| [BPF_LD | BPF_H | BPF_ABS] = true, |
| [BPF_LD | BPF_B | BPF_ABS] = true, |
| [BPF_LD | BPF_W | BPF_LEN] = true, |
| [BPF_LD | BPF_W | BPF_IND] = true, |
| [BPF_LD | BPF_H | BPF_IND] = true, |
| [BPF_LD | BPF_B | BPF_IND] = true, |
| [BPF_LD | BPF_IMM] = true, |
| [BPF_LD | BPF_MEM] = true, |
| [BPF_LDX | BPF_W | BPF_LEN] = true, |
| [BPF_LDX | BPF_B | BPF_MSH] = true, |
| [BPF_LDX | BPF_IMM] = true, |
| [BPF_LDX | BPF_MEM] = true, |
| /* Store instructions */ |
| [BPF_ST] = true, |
| [BPF_STX] = true, |
| /* Misc instructions */ |
| [BPF_MISC | BPF_TAX] = true, |
| [BPF_MISC | BPF_TXA] = true, |
| /* Return instructions */ |
| [BPF_RET | BPF_K] = true, |
| [BPF_RET | BPF_A] = true, |
| /* Jump instructions */ |
| [BPF_JMP | BPF_JA] = true, |
| [BPF_JMP | BPF_JEQ | BPF_K] = true, |
| [BPF_JMP | BPF_JEQ | BPF_X] = true, |
| [BPF_JMP | BPF_JGE | BPF_K] = true, |
| [BPF_JMP | BPF_JGE | BPF_X] = true, |
| [BPF_JMP | BPF_JGT | BPF_K] = true, |
| [BPF_JMP | BPF_JGT | BPF_X] = true, |
| [BPF_JMP | BPF_JSET | BPF_K] = true, |
| [BPF_JMP | BPF_JSET | BPF_X] = true, |
| }; |
| |
| if (code_to_probe >= ARRAY_SIZE(codes)) |
| return false; |
| |
| return codes[code_to_probe]; |
| } |
| |
| static bool bpf_check_basics_ok(const struct sock_filter *filter, |
| unsigned int flen) |
| { |
| if (filter == NULL) |
| return false; |
| if (flen == 0 || flen > BPF_MAXINSNS) |
| return false; |
| |
| return true; |
| } |
| |
| /** |
| * bpf_check_classic - verify socket filter code |
| * @filter: filter to verify |
| * @flen: length of filter |
| * |
| * Check the user's filter code. If we let some ugly |
| * filter code slip through kaboom! The filter must contain |
| * no references or jumps that are out of range, no illegal |
| * instructions, and must end with a RET instruction. |
| * |
| * All jumps are forward as they are not signed. |
| * |
| * Returns 0 if the rule set is legal or -EINVAL if not. |
| */ |
| static int bpf_check_classic(const struct sock_filter *filter, |
| unsigned int flen) |
| { |
| bool anc_found; |
| int pc; |
| |
| /* Check the filter code now */ |
| for (pc = 0; pc < flen; pc++) { |
| const struct sock_filter *ftest = &filter[pc]; |
| |
| /* May we actually operate on this code? */ |
| if (!chk_code_allowed(ftest->code)) |
| return -EINVAL; |
| |
| /* Some instructions need special checks */ |
| switch (ftest->code) { |
| case BPF_ALU | BPF_DIV | BPF_K: |
| case BPF_ALU | BPF_MOD | BPF_K: |
| /* Check for division by zero */ |
| if (ftest->k == 0) |
| return -EINVAL; |
| break; |
| case BPF_ALU | BPF_LSH | BPF_K: |
| case BPF_ALU | BPF_RSH | BPF_K: |
| if (ftest->k >= 32) |
| return -EINVAL; |
| break; |
| case BPF_LD | BPF_MEM: |
| case BPF_LDX | BPF_MEM: |
| case BPF_ST: |
| case BPF_STX: |
| /* Check for invalid memory addresses */ |
| if (ftest->k >= BPF_MEMWORDS) |
| return -EINVAL; |
| break; |
| case BPF_JMP | BPF_JA: |
| /* Note, the large ftest->k might cause loops. |
| * Compare this with conditional jumps below, |
| * where offsets are limited. --ANK (981016) |
| */ |
| if (ftest->k >= (unsigned int)(flen - pc - 1)) |
| return -EINVAL; |
| break; |
| case BPF_JMP | BPF_JEQ | BPF_K: |
| case BPF_JMP | BPF_JEQ | BPF_X: |
| case BPF_JMP | BPF_JGE | BPF_K: |
| case BPF_JMP | BPF_JGE | BPF_X: |
| case BPF_JMP | BPF_JGT | BPF_K: |
| case BPF_JMP | BPF_JGT | BPF_X: |
| case BPF_JMP | BPF_JSET | BPF_K: |
| case BPF_JMP | BPF_JSET | BPF_X: |
| /* Both conditionals must be safe */ |
| if (pc + ftest->jt + 1 >= flen || |
| pc + ftest->jf + 1 >= flen) |
| return -EINVAL; |
| break; |
| case BPF_LD | BPF_W | BPF_ABS: |
| case BPF_LD | BPF_H | BPF_ABS: |
| case BPF_LD | BPF_B | BPF_ABS: |
| anc_found = false; |
| if (bpf_anc_helper(ftest) & BPF_ANC) |
| anc_found = true; |
| /* Ancillary operation unknown or unsupported */ |
| if (anc_found == false && ftest->k >= SKF_AD_OFF) |
| return -EINVAL; |
| } |
| } |
| |
| /* Last instruction must be a RET code */ |
| switch (filter[flen - 1].code) { |
| case BPF_RET | BPF_K: |
| case BPF_RET | BPF_A: |
| return check_load_and_stores(filter, flen); |
| } |
| |
| return -EINVAL; |
| } |
| |
| static int bpf_prog_store_orig_filter(struct bpf_prog *fp, |
| const struct sock_fprog *fprog) |
| { |
| unsigned int fsize = bpf_classic_proglen(fprog); |
| struct sock_fprog_kern *fkprog; |
| |
| fp->orig_prog = kmalloc(sizeof(*fkprog), GFP_KERNEL); |
| if (!fp->orig_prog) |
| return -ENOMEM; |
| |
| fkprog = fp->orig_prog; |
| fkprog->len = fprog->len; |
| |
| fkprog->filter = kmemdup(fp->insns, fsize, |
| GFP_KERNEL | __GFP_NOWARN); |
| if (!fkprog->filter) { |
| kfree(fp->orig_prog); |
| return -ENOMEM; |
| } |
| |
| return 0; |
| } |
| |
| static void bpf_release_orig_filter(struct bpf_prog *fp) |
| { |
| struct sock_fprog_kern *fprog = fp->orig_prog; |
| |
| if (fprog) { |
| kfree(fprog->filter); |
| kfree(fprog); |
| } |
| } |
| |
| static void __bpf_prog_release(struct bpf_prog *prog) |
| { |
| if (prog->type == BPF_PROG_TYPE_SOCKET_FILTER) { |
| bpf_prog_put(prog); |
| } else { |
| bpf_release_orig_filter(prog); |
| bpf_prog_free(prog); |
| } |
| } |
| |
| static void __sk_filter_release(struct sk_filter *fp) |
| { |
| __bpf_prog_release(fp->prog); |
| kfree(fp); |
| } |
| |
| /** |
| * sk_filter_release_rcu - Release a socket filter by rcu_head |
| * @rcu: rcu_head that contains the sk_filter to free |
| */ |
| static void sk_filter_release_rcu(struct rcu_head *rcu) |
| { |
| struct sk_filter *fp = container_of(rcu, struct sk_filter, rcu); |
| |
| __sk_filter_release(fp); |
| } |
| |
| /** |
| * sk_filter_release - release a socket filter |
| * @fp: filter to remove |
| * |
| * Remove a filter from a socket and release its resources. |
| */ |
| static void sk_filter_release(struct sk_filter *fp) |
| { |
| if (refcount_dec_and_test(&fp->refcnt)) |
| call_rcu(&fp->rcu, sk_filter_release_rcu); |
| } |
| |
| void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp) |
| { |
| u32 filter_size = bpf_prog_size(fp->prog->len); |
| |
| atomic_sub(filter_size, &sk->sk_omem_alloc); |
| sk_filter_release(fp); |
| } |
| |
| /* try to charge the socket memory if there is space available |
| * return true on success |
| */ |
| static bool __sk_filter_charge(struct sock *sk, struct sk_filter *fp) |
| { |
| int optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max); |
| u32 filter_size = bpf_prog_size(fp->prog->len); |
| |
| /* same check as in sock_kmalloc() */ |
| if (filter_size <= optmem_max && |
| atomic_read(&sk->sk_omem_alloc) + filter_size < optmem_max) { |
| atomic_add(filter_size, &sk->sk_omem_alloc); |
| return true; |
| } |
| return false; |
| } |
| |
| bool sk_filter_charge(struct sock *sk, struct sk_filter *fp) |
| { |
| if (!refcount_inc_not_zero(&fp->refcnt)) |
| return false; |
| |
| if (!__sk_filter_charge(sk, fp)) { |
| sk_filter_release(fp); |
| return false; |
| } |
| return true; |
| } |
| |
| static struct bpf_prog *bpf_migrate_filter(struct bpf_prog *fp) |
| { |
| struct sock_filter *old_prog; |
| struct bpf_prog *old_fp; |
| int err, new_len, old_len = fp->len; |
| bool seen_ld_abs = false; |
| |
| /* We are free to overwrite insns et al right here as it won't be used at |
| * this point in time anymore internally after the migration to the eBPF |
| * instruction representation. |
| */ |
| BUILD_BUG_ON(sizeof(struct sock_filter) != |
| sizeof(struct bpf_insn)); |
| |
| /* Conversion cannot happen on overlapping memory areas, |
| * so we need to keep the user BPF around until the 2nd |
| * pass. At this time, the user BPF is stored in fp->insns. |
| */ |
| old_prog = kmemdup(fp->insns, old_len * sizeof(struct sock_filter), |
| GFP_KERNEL | __GFP_NOWARN); |
| if (!old_prog) { |
| err = -ENOMEM; |
| goto out_err; |
| } |
| |
| /* 1st pass: calculate the new program length. */ |
| err = bpf_convert_filter(old_prog, old_len, NULL, &new_len, |
| &seen_ld_abs); |
| if (err) |
| goto out_err_free; |
| |
| /* Expand fp for appending the new filter representation. */ |
| old_fp = fp; |
| fp = bpf_prog_realloc(old_fp, bpf_prog_size(new_len), 0); |
| if (!fp) { |
| /* The old_fp is still around in case we couldn't |
| * allocate new memory, so uncharge on that one. |
| */ |
| fp = old_fp; |
| err = -ENOMEM; |
| goto out_err_free; |
| } |
| |
| fp->len = new_len; |
| |
| /* 2nd pass: remap sock_filter insns into bpf_insn insns. */ |
| err = bpf_convert_filter(old_prog, old_len, fp, &new_len, |
| &seen_ld_abs); |
| if (err) |
| /* 2nd bpf_convert_filter() can fail only if it fails |
| * to allocate memory, remapping must succeed. Note, |
| * that at this time old_fp has already been released |
| * by krealloc(). |
| */ |
| goto out_err_free; |
| |
| fp = bpf_prog_select_runtime(fp, &err); |
| if (err) |
| goto out_err_free; |
| |
| kfree(old_prog); |
| return fp; |
| |
| out_err_free: |
| kfree(old_prog); |
| out_err: |
| __bpf_prog_release(fp); |
| return ERR_PTR(err); |
| } |
| |
| static struct bpf_prog *bpf_prepare_filter(struct bpf_prog *fp, |
| bpf_aux_classic_check_t trans) |
| { |
| int err; |
| |
| fp->bpf_func = NULL; |
| fp->jited = 0; |
| |
| err = bpf_check_classic(fp->insns, fp->len); |
| if (err) { |
| __bpf_prog_release(fp); |
| return ERR_PTR(err); |
| } |
| |
| /* There might be additional checks and transformations |
| * needed on classic filters, f.e. in case of seccomp. |
| */ |
| if (trans) { |
| err = trans(fp->insns, fp->len); |
| if (err) { |
| __bpf_prog_release(fp); |
| return ERR_PTR(err); |
| } |
| } |
| |
| /* Probe if we can JIT compile the filter and if so, do |
| * the compilation of the filter. |
| */ |
| bpf_jit_compile(fp); |
| |
| /* JIT compiler couldn't process this filter, so do the eBPF translation |
| * for the optimized interpreter. |
| */ |
| if (!fp->jited) |
| fp = bpf_migrate_filter(fp); |
| |
| return fp; |
| } |
| |
| /** |
| * bpf_prog_create - create an unattached filter |
| * @pfp: the unattached filter that is created |
| * @fprog: the filter program |
| * |
| * Create a filter independent of any socket. We first run some |
| * sanity checks on it to make sure it does not explode on us later. |
| * If an error occurs or there is insufficient memory for the filter |
| * a negative errno code is returned. On success the return is zero. |
| */ |
| int bpf_prog_create(struct bpf_prog **pfp, struct sock_fprog_kern *fprog) |
| { |
| unsigned int fsize = bpf_classic_proglen(fprog); |
| struct bpf_prog *fp; |
| |
| /* Make sure new filter is there and in the right amounts. */ |
| if (!bpf_check_basics_ok(fprog->filter, fprog->len)) |
| return -EINVAL; |
| |
| fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0); |
| if (!fp) |
| return -ENOMEM; |
| |
| memcpy(fp->insns, fprog->filter, fsize); |
| |
| fp->len = fprog->len; |
| /* Since unattached filters are not copied back to user |
| * space through sk_get_filter(), we do not need to hold |
| * a copy here, and can spare us the work. |
| */ |
| fp->orig_prog = NULL; |
| |
| /* bpf_prepare_filter() already takes care of freeing |
| * memory in case something goes wrong. |
| */ |
| fp = bpf_prepare_filter(fp, NULL); |
| if (IS_ERR(fp)) |
| return PTR_ERR(fp); |
| |
| *pfp = fp; |
| return 0; |
| } |
| EXPORT_SYMBOL_GPL(bpf_prog_create); |
| |
| /** |
| * bpf_prog_create_from_user - create an unattached filter from user buffer |
| * @pfp: the unattached filter that is created |
| * @fprog: the filter program |
| * @trans: post-classic verifier transformation handler |
| * @save_orig: save classic BPF program |
| * |
| * This function effectively does the same as bpf_prog_create(), only |
| * that it builds up its insns buffer from user space provided buffer. |
| * It also allows for passing a bpf_aux_classic_check_t handler. |
| */ |
| int bpf_prog_create_from_user(struct bpf_prog **pfp, struct sock_fprog *fprog, |
| bpf_aux_classic_check_t trans, bool save_orig) |
| { |
| unsigned int fsize = bpf_classic_proglen(fprog); |
| struct bpf_prog *fp; |
| int err; |
| |
| /* Make sure new filter is there and in the right amounts. */ |
| if (!bpf_check_basics_ok(fprog->filter, fprog->len)) |
| return -EINVAL; |
| |
| fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0); |
| if (!fp) |
| return -ENOMEM; |
| |
| if (copy_from_user(fp->insns, fprog->filter, fsize)) { |
| __bpf_prog_free(fp); |
| return -EFAULT; |
| } |
| |
| fp->len = fprog->len; |
| fp->orig_prog = NULL; |
| |
| if (save_orig) { |
| err = bpf_prog_store_orig_filter(fp, fprog); |
| if (err) { |
| __bpf_prog_free(fp); |
| return -ENOMEM; |
| } |
| } |
| |
| /* bpf_prepare_filter() already takes care of freeing |
| * memory in case something goes wrong. |
| */ |
| fp = bpf_prepare_filter(fp, trans); |
| if (IS_ERR(fp)) |
| return PTR_ERR(fp); |
| |
| *pfp = fp; |
| return 0; |
| } |
| EXPORT_SYMBOL_GPL(bpf_prog_create_from_user); |
| |
| void bpf_prog_destroy(struct bpf_prog *fp) |
| { |
| __bpf_prog_release(fp); |
| } |
| EXPORT_SYMBOL_GPL(bpf_prog_destroy); |
| |
| static int __sk_attach_prog(struct bpf_prog *prog, struct sock *sk) |
| { |
| struct sk_filter *fp, *old_fp; |
| |
| fp = kmalloc(sizeof(*fp), GFP_KERNEL); |
| if (!fp) |
| return -ENOMEM; |
| |
| fp->prog = prog; |
| |
| if (!__sk_filter_charge(sk, fp)) { |
| kfree(fp); |
| return -ENOMEM; |
| } |
| refcount_set(&fp->refcnt, 1); |
| |
| old_fp = rcu_dereference_protected(sk->sk_filter, |
| lockdep_sock_is_held(sk)); |
| rcu_assign_pointer(sk->sk_filter, fp); |
| |
| if (old_fp) |
| sk_filter_uncharge(sk, old_fp); |
| |
| return 0; |
| } |
| |
| static |
| struct bpf_prog *__get_filter(struct sock_fprog *fprog, struct sock *sk) |
| { |
| unsigned int fsize = bpf_classic_proglen(fprog); |
| struct bpf_prog *prog; |
| int err; |
| |
| if (sock_flag(sk, SOCK_FILTER_LOCKED)) |
| return ERR_PTR(-EPERM); |
| |
| /* Make sure new filter is there and in the right amounts. */ |
| if (!bpf_check_basics_ok(fprog->filter, fprog->len)) |
| return ERR_PTR(-EINVAL); |
| |
| prog = bpf_prog_alloc(bpf_prog_size(fprog->len), 0); |
| if (!prog) |
| return ERR_PTR(-ENOMEM); |
| |
| if (copy_from_user(prog->insns, fprog->filter, fsize)) { |
| __bpf_prog_free(prog); |
| return ERR_PTR(-EFAULT); |
| } |
| |
| prog->len = fprog->len; |
| |
| err = bpf_prog_store_orig_filter(prog, fprog); |
| if (err) { |
| __bpf_prog_free(prog); |
| return ERR_PTR(-ENOMEM); |
| } |
| |
| /* bpf_prepare_filter() already takes care of freeing |
| * memory in case something goes wrong. |
| */ |
| return bpf_prepare_filter(prog, NULL); |
| } |
| |
| /** |
| * sk_attach_filter - attach a socket filter |
| * @fprog: the filter program |
| * @sk: the socket to use |
| * |
| * Attach the user's filter code. We first run some sanity checks on |
| * it to make sure it does not explode on us later. If an error |
| * occurs or there is insufficient memory for the filter a negative |
| * errno code is returned. On success the return is zero. |
| */ |
| int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk) |
| { |
| struct bpf_prog *prog = __get_filter(fprog, sk); |
| int err; |
| |
| if (IS_ERR(prog)) |
| return PTR_ERR(prog); |
| |
| err = __sk_attach_prog(prog, sk); |
| if (err < 0) { |
| __bpf_prog_release(prog); |
| return err; |
| } |
| |
| return 0; |
| } |
| EXPORT_SYMBOL_GPL(sk_attach_filter); |
| |
| int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk) |
| { |
| struct bpf_prog *prog = __get_filter(fprog, sk); |
| int err, optmem_max; |
| |
| if (IS_ERR(prog)) |
| return PTR_ERR(prog); |
| |
| optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max); |
| if (bpf_prog_size(prog->len) > optmem_max) |
| err = -ENOMEM; |
| else |
| err = reuseport_attach_prog(sk, prog); |
| |
| if (err) |
| __bpf_prog_release(prog); |
| |
| return err; |
| } |
| |
| static struct bpf_prog *__get_bpf(u32 ufd, struct sock *sk) |
| { |
| if (sock_flag(sk, SOCK_FILTER_LOCKED)) |
| return ERR_PTR(-EPERM); |
| |
| return bpf_prog_get_type(ufd, BPF_PROG_TYPE_SOCKET_FILTER); |
| } |
| |
| int sk_attach_bpf(u32 ufd, struct sock *sk) |
| { |
| struct bpf_prog *prog = __get_bpf(ufd, sk); |
| int err; |
| |
| if (IS_ERR(prog)) |
| return PTR_ERR(prog); |
| |
| err = __sk_attach_prog(prog, sk); |
| if (err < 0) { |
| bpf_prog_put(prog); |
| return err; |
| } |
| |
| return 0; |
| } |
| |
| int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk) |
| { |
| struct bpf_prog *prog; |
| int err, optmem_max; |
| |
| if (sock_flag(sk, SOCK_FILTER_LOCKED)) |
| return -EPERM; |
| |
| prog = bpf_prog_get_type(ufd, BPF_PROG_TYPE_SOCKET_FILTER); |
| if (PTR_ERR(prog) == -EINVAL) |
| prog = bpf_prog_get_type(ufd, BPF_PROG_TYPE_SK_REUSEPORT); |
| if (IS_ERR(prog)) |
| return PTR_ERR(prog); |
| |
| if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT) { |
| /* Like other non BPF_PROG_TYPE_SOCKET_FILTER |
| * bpf prog (e.g. sockmap). It depends on the |
| * limitation imposed by bpf_prog_load(). |
| * Hence, sysctl_optmem_max is not checked. |
| */ |
| if ((sk->sk_type != SOCK_STREAM && |
| sk->sk_type != SOCK_DGRAM) || |
| (sk->sk_protocol != IPPROTO_UDP && |
| sk->sk_protocol != IPPROTO_TCP) || |
| (sk->sk_family != AF_INET && |
| sk->sk_family != AF_INET6)) { |
| err = -ENOTSUPP; |
| goto err_prog_put; |
| } |
| } else { |
| /* BPF_PROG_TYPE_SOCKET_FILTER */ |
| optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max); |
| if (bpf_prog_size(prog->len) > optmem_max) { |
| err = -ENOMEM; |
| goto err_prog_put; |
| } |
| } |
| |
| err = reuseport_attach_prog(sk, prog); |
| err_prog_put: |
| if (err) |
| bpf_prog_put(prog); |
| |
| return err; |
| } |
| |
| void sk_reuseport_prog_free(struct bpf_prog *prog) |
| { |
| if (!prog) |
| return; |
| |
| if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT) |
| bpf_prog_put(prog); |
| else |
| bpf_prog_destroy(prog); |
| } |
| |
| struct bpf_scratchpad { |
| union { |
| __be32 diff[MAX_BPF_STACK / sizeof(__be32)]; |
| u8 buff[MAX_BPF_STACK]; |
| }; |
| local_lock_t bh_lock; |
| }; |
| |
| static DEFINE_PER_CPU(struct bpf_scratchpad, bpf_sp) = { |
| .bh_lock = INIT_LOCAL_LOCK(bh_lock), |
| }; |
| |
| static inline int __bpf_try_make_writable(struct sk_buff *skb, |
| unsigned int write_len) |
| { |
| #ifdef CONFIG_DEBUG_NET |
| /* Avoid a splat in pskb_may_pull_reason() */ |
| if (write_len > INT_MAX) |
| return -EINVAL; |
| #endif |
| return skb_ensure_writable(skb, write_len); |
| } |
| |
| static inline int bpf_try_make_writable(struct sk_buff *skb, |
| unsigned int write_len) |
| { |
| int err = __bpf_try_make_writable(skb, write_len); |
| |
| bpf_compute_data_pointers(skb); |
| return err; |
| } |
| |
| static int bpf_try_make_head_writable(struct sk_buff *skb) |
| { |
| return bpf_try_make_writable(skb, skb_headlen(skb)); |
| } |
| |
| static inline void bpf_push_mac_rcsum(struct sk_buff *skb) |
| { |
| if (skb_at_tc_ingress(skb)) |
| skb_postpush_rcsum(skb, skb_mac_header(skb), skb->mac_len); |
| } |
| |
| static inline void bpf_pull_mac_rcsum(struct sk_buff *skb) |
| { |
| if (skb_at_tc_ingress(skb)) |
| skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len); |
| } |
| |
| BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset, |
| const void *, from, u32, len, u64, flags) |
| { |
| void *ptr; |
| |
| if (unlikely(flags & ~(BPF_F_RECOMPUTE_CSUM | BPF_F_INVALIDATE_HASH))) |
| return -EINVAL; |
| if (unlikely(offset > INT_MAX)) |
| return -EFAULT; |
| if (unlikely(bpf_try_make_writable(skb, offset + len))) |
| return -EFAULT; |
| |
| ptr = skb->data + offset; |
| if (flags & BPF_F_RECOMPUTE_CSUM) |
| __skb_postpull_rcsum(skb, ptr, len, offset); |
| |
| memcpy(ptr, from, len); |
| |
| if (flags & BPF_F_RECOMPUTE_CSUM) |
| __skb_postpush_rcsum(skb, ptr, len, offset); |
| if (flags & BPF_F_INVALIDATE_HASH) |
| skb_clear_hash(skb); |
| |
| return 0; |
| } |
| |
| static const struct bpf_func_proto bpf_skb_store_bytes_proto = { |
| .func = bpf_skb_store_bytes, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_CTX, |
| .arg2_type = ARG_ANYTHING, |
| .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, |
| .arg4_type = ARG_CONST_SIZE, |
| .arg5_type = ARG_ANYTHING, |
| }; |
| |
| int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, |
| u32 len, u64 flags) |
| { |
| return ____bpf_skb_store_bytes(skb, offset, from, len, flags); |
| } |
| |
| BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset, |
| void *, to, u32, len) |
| { |
| void *ptr; |
| |
| if (unlikely(offset > INT_MAX)) |
| goto err_clear; |
| |
| ptr = skb_header_pointer(skb, offset, len, to); |
| if (unlikely(!ptr)) |
| goto err_clear; |
| if (ptr != to) |
| memcpy(to, ptr, len); |
| |
| return 0; |
| err_clear: |
| memset(to, 0, len); |
| return -EFAULT; |
| } |
| |
| static const struct bpf_func_proto bpf_skb_load_bytes_proto = { |
| .func = bpf_skb_load_bytes, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_CTX, |
| .arg2_type = ARG_ANYTHING, |
| .arg3_type = ARG_PTR_TO_UNINIT_MEM, |
| .arg4_type = ARG_CONST_SIZE, |
| }; |
| |
| int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len) |
| { |
| return ____bpf_skb_load_bytes(skb, offset, to, len); |
| } |
| |
| BPF_CALL_4(bpf_flow_dissector_load_bytes, |
| const struct bpf_flow_dissector *, ctx, u32, offset, |
| void *, to, u32, len) |
| { |
| void *ptr; |
| |
| if (unlikely(offset > 0xffff)) |
| goto err_clear; |
| |
| if (unlikely(!ctx->skb)) |
| goto err_clear; |
| |
| ptr = skb_header_pointer(ctx->skb, offset, len, to); |
| if (unlikely(!ptr)) |
| goto err_clear; |
| if (ptr != to) |
| memcpy(to, ptr, len); |
| |
| return 0; |
| err_clear: |
| memset(to, 0, len); |
| return -EFAULT; |
| } |
| |
| static const struct bpf_func_proto bpf_flow_dissector_load_bytes_proto = { |
| .func = bpf_flow_dissector_load_bytes, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_CTX, |
| .arg2_type = ARG_ANYTHING, |
| .arg3_type = ARG_PTR_TO_UNINIT_MEM, |
| .arg4_type = ARG_CONST_SIZE, |
| }; |
| |
| BPF_CALL_5(bpf_skb_load_bytes_relative, const struct sk_buff *, skb, |
| u32, offset, void *, to, u32, len, u32, start_header) |
| { |
| u8 *end = skb_tail_pointer(skb); |
| u8 *start, *ptr; |
| |
| if (unlikely(offset > 0xffff)) |
| goto err_clear; |
| |
| switch (start_header) { |
| case BPF_HDR_START_MAC: |
| if (unlikely(!skb_mac_header_was_set(skb))) |
| goto err_clear; |
| start = skb_mac_header(skb); |
| break; |
| case BPF_HDR_START_NET: |
| start = skb_network_header(skb); |
| break; |
| default: |
| goto err_clear; |
| } |
| |
| ptr = start + offset; |
| |
| if (likely(ptr + len <= end)) { |
| memcpy(to, ptr, len); |
| return 0; |
| } |
| |
| err_clear: |
| memset(to, 0, len); |
| return -EFAULT; |
| } |
| |
| static const struct bpf_func_proto bpf_skb_load_bytes_relative_proto = { |
| .func = bpf_skb_load_bytes_relative, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_CTX, |
| .arg2_type = ARG_ANYTHING, |
| .arg3_type = ARG_PTR_TO_UNINIT_MEM, |
| .arg4_type = ARG_CONST_SIZE, |
| .arg5_type = ARG_ANYTHING, |
| }; |
| |
| BPF_CALL_2(bpf_skb_pull_data, struct sk_buff *, skb, u32, len) |
| { |
| /* Idea is the following: should the needed direct read/write |
| * test fail during runtime, we can pull in more data and redo |
| * again, since implicitly, we invalidate previous checks here. |
| * |
| * Or, since we know how much we need to make read/writeable, |
| * this can be done once at the program beginning for direct |
| * access case. By this we overcome limitations of only current |
| * headroom being accessible. |
| */ |
| return bpf_try_make_writable(skb, len ? : skb_headlen(skb)); |
| } |
| |
| static const struct bpf_func_proto bpf_skb_pull_data_proto = { |
| .func = bpf_skb_pull_data, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_CTX, |
| .arg2_type = ARG_ANYTHING, |
| }; |
| |
| BPF_CALL_1(bpf_sk_fullsock, struct sock *, sk) |
| { |
| return sk_fullsock(sk) ? (unsigned long)sk : (unsigned long)NULL; |
| } |
| |
| static const struct bpf_func_proto bpf_sk_fullsock_proto = { |
| .func = bpf_sk_fullsock, |
| .gpl_only = false, |
| .ret_type = RET_PTR_TO_SOCKET_OR_NULL, |
| .arg1_type = ARG_PTR_TO_SOCK_COMMON, |
| }; |
| |
| static inline int sk_skb_try_make_writable(struct sk_buff *skb, |
| unsigned int write_len) |
| { |
| return __bpf_try_make_writable(skb, write_len); |
| } |
| |
| BPF_CALL_2(sk_skb_pull_data, struct sk_buff *, skb, u32, len) |
| { |
| /* Idea is the following: should the needed direct read/write |
| * test fail during runtime, we can pull in more data and redo |
| * again, since implicitly, we invalidate previous checks here. |
| * |
| * Or, since we know how much we need to make read/writeable, |
| * this can be done once at the program beginning for direct |
| * access case. By this we overcome limitations of only current |
| * headroom being accessible. |
| */ |
| return sk_skb_try_make_writable(skb, len ? : skb_headlen(skb)); |
| } |
| |
| static const struct bpf_func_proto sk_skb_pull_data_proto = { |
| .func = sk_skb_pull_data, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_CTX, |
| .arg2_type = ARG_ANYTHING, |
| }; |
| |
| BPF_CALL_5(bpf_l3_csum_replace, struct sk_buff *, skb, u32, offset, |
| u64, from, u64, to, u64, flags) |
| { |
| __sum16 *ptr; |
| |
| if (unlikely(flags & ~(BPF_F_HDR_FIELD_MASK))) |
| return -EINVAL; |
| if (unlikely(offset > 0xffff || offset & 1)) |
| return -EFAULT; |
| if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr)))) |
| return -EFAULT; |
| |
| ptr = (__sum16 *)(skb->data + offset); |
| switch (flags & BPF_F_HDR_FIELD_MASK) { |
| case 0: |
| if (unlikely(from != 0)) |
| return -EINVAL; |
| |
| csum_replace_by_diff(ptr, to); |
| break; |
| case 2: |
| csum_replace2(ptr, from, to); |
| break; |
| case 4: |
| csum_replace4(ptr, from, to); |
| break; |
| default: |
| return -EINVAL; |
| } |
| |
| return 0; |
| } |
| |
| static const struct bpf_func_proto bpf_l3_csum_replace_proto = { |
| .func = bpf_l3_csum_replace, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_CTX, |
| .arg2_type = ARG_ANYTHING, |
| .arg3_type = ARG_ANYTHING, |
| .arg4_type = ARG_ANYTHING, |
| .arg5_type = ARG_ANYTHING, |
| }; |
| |
| BPF_CALL_5(bpf_l4_csum_replace, struct sk_buff *, skb, u32, offset, |
| u64, from, u64, to, u64, flags) |
| { |
| bool is_pseudo = flags & BPF_F_PSEUDO_HDR; |
| bool is_mmzero = flags & BPF_F_MARK_MANGLED_0; |
| bool do_mforce = flags & BPF_F_MARK_ENFORCE; |
| __sum16 *ptr; |
| |
| if (unlikely(flags & ~(BPF_F_MARK_MANGLED_0 | BPF_F_MARK_ENFORCE | |
| BPF_F_PSEUDO_HDR | BPF_F_HDR_FIELD_MASK))) |
| return -EINVAL; |
| if (unlikely(offset > 0xffff || offset & 1)) |
| return -EFAULT; |
| if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr)))) |
| return -EFAULT; |
| |
| ptr = (__sum16 *)(skb->data + offset); |
| if (is_mmzero && !do_mforce && !*ptr) |
| return 0; |
| |
| switch (flags & BPF_F_HDR_FIELD_MASK) { |
| case 0: |
| if (unlikely(from != 0)) |
| return -EINVAL; |
| |
| inet_proto_csum_replace_by_diff(ptr, skb, to, is_pseudo); |
| break; |
| case 2: |
| inet_proto_csum_replace2(ptr, skb, from, to, is_pseudo); |
| break; |
| case 4: |
| inet_proto_csum_replace4(ptr, skb, from, to, is_pseudo); |
| break; |
| default: |
| return -EINVAL; |
| } |
| |
| if (is_mmzero && !*ptr) |
| *ptr = CSUM_MANGLED_0; |
| return 0; |
| } |
| |
| static const struct bpf_func_proto bpf_l4_csum_replace_proto = { |
| .func = bpf_l4_csum_replace, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_CTX, |
| .arg2_type = ARG_ANYTHING, |
| .arg3_type = ARG_ANYTHING, |
| .arg4_type = ARG_ANYTHING, |
| .arg5_type = ARG_ANYTHING, |
| }; |
| |
| BPF_CALL_5(bpf_csum_diff, __be32 *, from, u32, from_size, |
| __be32 *, to, u32, to_size, __wsum, seed) |
| { |
| struct bpf_scratchpad *sp = this_cpu_ptr(&bpf_sp); |
| u32 diff_size = from_size + to_size; |
| int i, j = 0; |
| __wsum ret; |
| |
| /* This is quite flexible, some examples: |
| * |
| * from_size == 0, to_size > 0, seed := csum --> pushing data |
| * from_size > 0, to_size == 0, seed := csum --> pulling data |
| * from_size > 0, to_size > 0, seed := 0 --> diffing data |
| * |
| * Even for diffing, from_size and to_size don't need to be equal. |
| */ |
| if (unlikely(((from_size | to_size) & (sizeof(__be32) - 1)) || |
| diff_size > sizeof(sp->diff))) |
| return -EINVAL; |
| |
| local_lock_nested_bh(&bpf_sp.bh_lock); |
| for (i = 0; i < from_size / sizeof(__be32); i++, j++) |
| sp->diff[j] = ~from[i]; |
| for (i = 0; i < to_size / sizeof(__be32); i++, j++) |
| sp->diff[j] = to[i]; |
| |
| ret = csum_partial(sp->diff, diff_size, seed); |
| local_unlock_nested_bh(&bpf_sp.bh_lock); |
| return ret; |
| } |
| |
| static const struct bpf_func_proto bpf_csum_diff_proto = { |
| .func = bpf_csum_diff, |
| .gpl_only = false, |
| .pkt_access = true, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, |
| .arg2_type = ARG_CONST_SIZE_OR_ZERO, |
| .arg3_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, |
| .arg4_type = ARG_CONST_SIZE_OR_ZERO, |
| .arg5_type = ARG_ANYTHING, |
| }; |
| |
| BPF_CALL_2(bpf_csum_update, struct sk_buff *, skb, __wsum, csum) |
| { |
| /* The interface is to be used in combination with bpf_csum_diff() |
| * for direct packet writes. csum rotation for alignment as well |
| * as emulating csum_sub() can be done from the eBPF program. |
| */ |
| if (skb->ip_summed == CHECKSUM_COMPLETE) |
| return (skb->csum = csum_add(skb->csum, csum)); |
| |
| return -ENOTSUPP; |
| } |
| |
| static const struct bpf_func_proto bpf_csum_update_proto = { |
| .func = bpf_csum_update, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_CTX, |
| .arg2_type = ARG_ANYTHING, |
| }; |
| |
| BPF_CALL_2(bpf_csum_level, struct sk_buff *, skb, u64, level) |
| { |
| /* The interface is to be used in combination with bpf_skb_adjust_room() |
| * for encap/decap of packet headers when BPF_F_ADJ_ROOM_NO_CSUM_RESET |
| * is passed as flags, for example. |
| */ |
| switch (level) { |
| case BPF_CSUM_LEVEL_INC: |
| __skb_incr_checksum_unnecessary(skb); |
| break; |
| case BPF_CSUM_LEVEL_DEC: |
| __skb_decr_checksum_unnecessary(skb); |
| break; |
| case BPF_CSUM_LEVEL_RESET: |
| __skb_reset_checksum_unnecessary(skb); |
| break; |
| case BPF_CSUM_LEVEL_QUERY: |
| return skb->ip_summed == CHECKSUM_UNNECESSARY ? |
| skb->csum_level : -EACCES; |
| default: |
| return -EINVAL; |
| } |
| |
| return 0; |
| } |
| |
| static const struct bpf_func_proto bpf_csum_level_proto = { |
| .func = bpf_csum_level, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_CTX, |
| .arg2_type = ARG_ANYTHING, |
| }; |
| |
| static inline int __bpf_rx_skb(struct net_device *dev, struct sk_buff *skb) |
| { |
| return dev_forward_skb_nomtu(dev, skb); |
| } |
| |
| static inline int __bpf_rx_skb_no_mac(struct net_device *dev, |
| struct sk_buff *skb) |
| { |
| int ret = ____dev_forward_skb(dev, skb, false); |
| |
| if (likely(!ret)) { |
| skb->dev = dev; |
| ret = netif_rx(skb); |
| } |
| |
| return ret; |
| } |
| |
| static inline int __bpf_tx_skb(struct net_device *dev, struct sk_buff *skb) |
| { |
| int ret; |
| |
| if (dev_xmit_recursion()) { |
| net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n"); |
| kfree_skb(skb); |
| return -ENETDOWN; |
| } |
| |
| skb->dev = dev; |
| skb_set_redirected_noclear(skb, skb_at_tc_ingress(skb)); |
| skb_clear_tstamp(skb); |
| |
| dev_xmit_recursion_inc(); |
| ret = dev_queue_xmit(skb); |
| dev_xmit_recursion_dec(); |
| |
| return ret; |
| } |
| |
| static int __bpf_redirect_no_mac(struct sk_buff *skb, struct net_device *dev, |
| u32 flags) |
| { |
| unsigned int mlen = skb_network_offset(skb); |
| |
| if (unlikely(skb->len <= mlen)) { |
| kfree_skb(skb); |
| return -ERANGE; |
| } |
| |
| if (mlen) { |
| __skb_pull(skb, mlen); |
| |
| /* At ingress, the mac header has already been pulled once. |
| * At egress, skb_pospull_rcsum has to be done in case that |
| * the skb is originated from ingress (i.e. a forwarded skb) |
| * to ensure that rcsum starts at net header. |
| */ |
| if (!skb_at_tc_ingress(skb)) |
| skb_postpull_rcsum(skb, skb_mac_header(skb), mlen); |
| } |
| skb_pop_mac_header(skb); |
| skb_reset_mac_len(skb); |
| return flags & BPF_F_INGRESS ? |
| __bpf_rx_skb_no_mac(dev, skb) : __bpf_tx_skb(dev, skb); |
| } |
| |
| static int __bpf_redirect_common(struct sk_buff *skb, struct net_device *dev, |
| u32 flags) |
| { |
| /* Verify that a link layer header is carried */ |
| if (unlikely(skb->mac_header >= skb->network_header || skb->len == 0)) { |
| kfree_skb(skb); |
| return -ERANGE; |
| } |
| |
| bpf_push_mac_rcsum(skb); |
| return flags & BPF_F_INGRESS ? |
| __bpf_rx_skb(dev, skb) : __bpf_tx_skb(dev, skb); |
| } |
| |
| static int __bpf_redirect(struct sk_buff *skb, struct net_device *dev, |
| u32 flags) |
| { |
| if (dev_is_mac_header_xmit(dev)) |
| return __bpf_redirect_common(skb, dev, flags); |
| else |
| return __bpf_redirect_no_mac(skb, dev, flags); |
| } |
| |
| #if IS_ENABLED(CONFIG_IPV6) |
| static int bpf_out_neigh_v6(struct net *net, struct sk_buff *skb, |
| struct net_device *dev, struct bpf_nh_params *nh) |
| { |
| u32 hh_len = LL_RESERVED_SPACE(dev); |
| const struct in6_addr *nexthop; |
| struct dst_entry *dst = NULL; |
| struct neighbour *neigh; |
| |
| if (dev_xmit_recursion()) { |
| net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n"); |
| goto out_drop; |
| } |
| |
| skb->dev = dev; |
| skb_clear_tstamp(skb); |
| |
| if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) { |
| skb = skb_expand_head(skb, hh_len); |
| if (!skb) |
| return -ENOMEM; |
| } |
| |
| rcu_read_lock(); |
| if (!nh) { |
| dst = skb_dst(skb); |
| nexthop = rt6_nexthop(dst_rt6_info(dst), |
| &ipv6_hdr(skb)->daddr); |
| } else { |
| nexthop = &nh->ipv6_nh; |
| } |
| neigh = ip_neigh_gw6(dev, nexthop); |
| if (likely(!IS_ERR(neigh))) { |
| int ret; |
| |
| sock_confirm_neigh(skb, neigh); |
| local_bh_disable(); |
| dev_xmit_recursion_inc(); |
| ret = neigh_output(neigh, skb, false); |
| dev_xmit_recursion_dec(); |
| local_bh_enable(); |
| rcu_read_unlock(); |
| return ret; |
| } |
| rcu_read_unlock_bh(); |
| if (dst) |
| IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES); |
| out_drop: |
| kfree_skb(skb); |
| return -ENETDOWN; |
| } |
| |
| static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev, |
| struct bpf_nh_params *nh) |
| { |
| const struct ipv6hdr *ip6h = ipv6_hdr(skb); |
| struct net *net = dev_net(dev); |
| int err, ret = NET_XMIT_DROP; |
| |
| if (!nh) { |
| struct dst_entry *dst; |
| struct flowi6 fl6 = { |
| .flowi6_flags = FLOWI_FLAG_ANYSRC, |
| .flowi6_mark = skb->mark, |
| .flowlabel = ip6_flowinfo(ip6h), |
| .flowi6_oif = dev->ifindex, |
| .flowi6_proto = ip6h->nexthdr, |
| .daddr = ip6h->daddr, |
| .saddr = ip6h->saddr, |
| }; |
| |
| dst = ipv6_stub->ipv6_dst_lookup_flow(net, NULL, &fl6, NULL); |
| if (IS_ERR(dst)) |
| goto out_drop; |
| |
| skb_dst_set(skb, dst); |
| } else if (nh->nh_family != AF_INET6) { |
| goto out_drop; |
| } |
| |
| err = bpf_out_neigh_v6(net, skb, dev, nh); |
| if (unlikely(net_xmit_eval(err))) |
| DEV_STATS_INC(dev, tx_errors); |
| else |
| ret = NET_XMIT_SUCCESS; |
| goto out_xmit; |
| out_drop: |
| DEV_STATS_INC(dev, tx_errors); |
| kfree_skb(skb); |
| out_xmit: |
| return ret; |
| } |
| #else |
| static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev, |
| struct bpf_nh_params *nh) |
| { |
| kfree_skb(skb); |
| return NET_XMIT_DROP; |
| } |
| #endif /* CONFIG_IPV6 */ |
| |
| #if IS_ENABLED(CONFIG_INET) |
| static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb, |
| struct net_device *dev, struct bpf_nh_params *nh) |
| { |
| u32 hh_len = LL_RESERVED_SPACE(dev); |
| struct neighbour *neigh; |
| bool is_v6gw = false; |
| |
| if (dev_xmit_recursion()) { |
| net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n"); |
| goto out_drop; |
| } |
| |
| skb->dev = dev; |
| skb_clear_tstamp(skb); |
| |
| if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) { |
| skb = skb_expand_head(skb, hh_len); |
| if (!skb) |
| return -ENOMEM; |
| } |
| |
| rcu_read_lock(); |
| if (!nh) { |
| struct rtable *rt = skb_rtable(skb); |
| |
| neigh = ip_neigh_for_gw(rt, skb, &is_v6gw); |
| } else if (nh->nh_family == AF_INET6) { |
| neigh = ip_neigh_gw6(dev, &nh->ipv6_nh); |
| is_v6gw = true; |
| } else if (nh->nh_family == AF_INET) { |
| neigh = ip_neigh_gw4(dev, nh->ipv4_nh); |
| } else { |
| rcu_read_unlock(); |
| goto out_drop; |
| } |
| |
| if (likely(!IS_ERR(neigh))) { |
| int ret; |
| |
| sock_confirm_neigh(skb, neigh); |
| local_bh_disable(); |
| dev_xmit_recursion_inc(); |
| ret = neigh_output(neigh, skb, is_v6gw); |
| dev_xmit_recursion_dec(); |
| local_bh_enable(); |
| rcu_read_unlock(); |
| return ret; |
| } |
| rcu_read_unlock(); |
| out_drop: |
| kfree_skb(skb); |
| return -ENETDOWN; |
| } |
| |
| static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev, |
| struct bpf_nh_params *nh) |
| { |
| const struct iphdr *ip4h = ip_hdr(skb); |
| struct net *net = dev_net(dev); |
| int err, ret = NET_XMIT_DROP; |
| |
| if (!nh) { |
| struct flowi4 fl4 = { |
| .flowi4_flags = FLOWI_FLAG_ANYSRC, |
| .flowi4_mark = skb->mark, |
| .flowi4_tos = RT_TOS(ip4h->tos), |
| .flowi4_oif = dev->ifindex, |
| .flowi4_proto = ip4h->protocol, |
| .daddr = ip4h->daddr, |
| .saddr = ip4h->saddr, |
| }; |
| struct rtable *rt; |
| |
| rt = ip_route_output_flow(net, &fl4, NULL); |
| if (IS_ERR(rt)) |
| goto out_drop; |
| if (rt->rt_type != RTN_UNICAST && rt->rt_type != RTN_LOCAL) { |
| ip_rt_put(rt); |
| goto out_drop; |
| } |
| |
| skb_dst_set(skb, &rt->dst); |
| } |
| |
| err = bpf_out_neigh_v4(net, skb, dev, nh); |
| if (unlikely(net_xmit_eval(err))) |
| DEV_STATS_INC(dev, tx_errors); |
| else |
| ret = NET_XMIT_SUCCESS; |
| goto out_xmit; |
| out_drop: |
| DEV_STATS_INC(dev, tx_errors); |
| kfree_skb(skb); |
| out_xmit: |
| return ret; |
| } |
| #else |
| static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev, |
| struct bpf_nh_params *nh) |
| { |
| kfree_skb(skb); |
| return NET_XMIT_DROP; |
| } |
| #endif /* CONFIG_INET */ |
| |
| static int __bpf_redirect_neigh(struct sk_buff *skb, struct net_device *dev, |
| struct bpf_nh_params *nh) |
| { |
| struct ethhdr *ethh = eth_hdr(skb); |
| |
| if (unlikely(skb->mac_header >= skb->network_header)) |
| goto out; |
| bpf_push_mac_rcsum(skb); |
| if (is_multicast_ether_addr(ethh->h_dest)) |
| goto out; |
| |
| skb_pull(skb, sizeof(*ethh)); |
| skb_unset_mac_header(skb); |
| skb_reset_network_header(skb); |
| |
| if (skb->protocol == htons(ETH_P_IP)) |
| return __bpf_redirect_neigh_v4(skb, dev, nh); |
| else if (skb->protocol == htons(ETH_P_IPV6)) |
| return __bpf_redirect_neigh_v6(skb, dev, nh); |
| out: |
| kfree_skb(skb); |
| return -ENOTSUPP; |
| } |
| |
| /* Internal, non-exposed redirect flags. */ |
| enum { |
| BPF_F_NEIGH = (1ULL << 1), |
| BPF_F_PEER = (1ULL << 2), |
| BPF_F_NEXTHOP = (1ULL << 3), |
| #define BPF_F_REDIRECT_INTERNAL (BPF_F_NEIGH | BPF_F_PEER | BPF_F_NEXTHOP) |
| }; |
| |
| BPF_CALL_3(bpf_clone_redirect, struct sk_buff *, skb, u32, ifindex, u64, flags) |
| { |
| struct net_device *dev; |
| struct sk_buff *clone; |
| int ret; |
| |
| if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL))) |
| return -EINVAL; |
| |
| dev = dev_get_by_index_rcu(dev_net(skb->dev), ifindex); |
| if (unlikely(!dev)) |
| return -EINVAL; |
| |
| clone = skb_clone(skb, GFP_ATOMIC); |
| if (unlikely(!clone)) |
| return -ENOMEM; |
| |
| /* For direct write, we need to keep the invariant that the skbs |
| * we're dealing with need to be uncloned. Should uncloning fail |
| * here, we need to free the just generated clone to unclone once |
| * again. |
| */ |
| ret = bpf_try_make_head_writable(skb); |
| if (unlikely(ret)) { |
| kfree_skb(clone); |
| return -ENOMEM; |
| } |
| |
| return __bpf_redirect(clone, dev, flags); |
| } |
| |
| static const struct bpf_func_proto bpf_clone_redirect_proto = { |
| .func = bpf_clone_redirect, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_CTX, |
| .arg2_type = ARG_ANYTHING, |
| .arg3_type = ARG_ANYTHING, |
| }; |
| |
| static struct net_device *skb_get_peer_dev(struct net_device *dev) |
| { |
| const struct net_device_ops *ops = dev->netdev_ops; |
| |
| if (likely(ops->ndo_get_peer_dev)) |
| return INDIRECT_CALL_1(ops->ndo_get_peer_dev, |
| netkit_peer_dev, dev); |
| return NULL; |
| } |
| |
| int skb_do_redirect(struct sk_buff *skb) |
| { |
| struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); |
| struct net *net = dev_net(skb->dev); |
| struct net_device *dev; |
| u32 flags = ri->flags; |
| |
| dev = dev_get_by_index_rcu(net, ri->tgt_index); |
| ri->tgt_index = 0; |
| ri->flags = 0; |
| if (unlikely(!dev)) |
| goto out_drop; |
| if (flags & BPF_F_PEER) { |
| if (unlikely(!skb_at_tc_ingress(skb))) |
| goto out_drop; |
| dev = skb_get_peer_dev(dev); |
| if (unlikely(!dev || |
| !(dev->flags & IFF_UP) || |
| net_eq(net, dev_net(dev)))) |
| goto out_drop; |
| skb->dev = dev; |
| dev_sw_netstats_rx_add(dev, skb->len); |
| return -EAGAIN; |
| } |
| return flags & BPF_F_NEIGH ? |
| __bpf_redirect_neigh(skb, dev, flags & BPF_F_NEXTHOP ? |
| &ri->nh : NULL) : |
| __bpf_redirect(skb, dev, flags); |
| out_drop: |
| kfree_skb(skb); |
| return -EINVAL; |
| } |
| |
| BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags) |
| { |
| struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); |
| |
| if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL))) |
| return TC_ACT_SHOT; |
| |
| ri->flags = flags; |
| ri->tgt_index = ifindex; |
| |
| return TC_ACT_REDIRECT; |
| } |
| |
| static const struct bpf_func_proto bpf_redirect_proto = { |
| .func = bpf_redirect, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_ANYTHING, |
| .arg2_type = ARG_ANYTHING, |
| }; |
| |
| BPF_CALL_2(bpf_redirect_peer, u32, ifindex, u64, flags) |
| { |
| struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); |
| |
| if (unlikely(flags)) |
| return TC_ACT_SHOT; |
| |
| ri->flags = BPF_F_PEER; |
| ri->tgt_index = ifindex; |
| |
| return TC_ACT_REDIRECT; |
| } |
| |
| static const struct bpf_func_proto bpf_redirect_peer_proto = { |
| .func = bpf_redirect_peer, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_ANYTHING, |
| .arg2_type = ARG_ANYTHING, |
| }; |
| |
| BPF_CALL_4(bpf_redirect_neigh, u32, ifindex, struct bpf_redir_neigh *, params, |
| int, plen, u64, flags) |
| { |
| struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); |
| |
| if (unlikely((plen && plen < sizeof(*params)) || flags)) |
| return TC_ACT_SHOT; |
| |
| ri->flags = BPF_F_NEIGH | (plen ? BPF_F_NEXTHOP : 0); |
| ri->tgt_index = ifindex; |
| |
| BUILD_BUG_ON(sizeof(struct bpf_redir_neigh) != sizeof(struct bpf_nh_params)); |
| if (plen) |
| memcpy(&ri->nh, params, sizeof(ri->nh)); |
| |
| return TC_ACT_REDIRECT; |
| } |
| |
| static const struct bpf_func_proto bpf_redirect_neigh_proto = { |
| .func = bpf_redirect_neigh, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_ANYTHING, |
| .arg2_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, |
| .arg3_type = ARG_CONST_SIZE_OR_ZERO, |
| .arg4_type = ARG_ANYTHING, |
| }; |
| |
| BPF_CALL_2(bpf_msg_apply_bytes, struct sk_msg *, msg, u32, bytes) |
| { |
| msg->apply_bytes = bytes; |
| return 0; |
| } |
| |
| static const struct bpf_func_proto bpf_msg_apply_bytes_proto = { |
| .func = bpf_msg_apply_bytes, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_CTX, |
| .arg2_type = ARG_ANYTHING, |
| }; |
| |
| BPF_CALL_2(bpf_msg_cork_bytes, struct sk_msg *, msg, u32, bytes) |
| { |
| msg->cork_bytes = bytes; |
| return 0; |
| } |
| |
| static void sk_msg_reset_curr(struct sk_msg *msg) |
| { |
| u32 i = msg->sg.start; |
| u32 len = 0; |
| |
| do { |
| len += sk_msg_elem(msg, i)->length; |
| sk_msg_iter_var_next(i); |
| if (len >= msg->sg.size) |
| break; |
| } while (i != msg->sg.end); |
| |
| msg->sg.curr = i; |
| msg->sg.copybreak = 0; |
| } |
| |
| static const struct bpf_func_proto bpf_msg_cork_bytes_proto = { |
| .func = bpf_msg_cork_bytes, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_CTX, |
| .arg2_type = ARG_ANYTHING, |
| }; |
| |
| BPF_CALL_4(bpf_msg_pull_data, struct sk_msg *, msg, u32, start, |
| u32, end, u64, flags) |
| { |
| u32 len = 0, offset = 0, copy = 0, poffset = 0, bytes = end - start; |
| u32 first_sge, last_sge, i, shift, bytes_sg_total; |
| struct scatterlist *sge; |
| u8 *raw, *to, *from; |
| struct page *page; |
| |
| if (unlikely(flags || end <= start)) |
| return -EINVAL; |
| |
| /* First find the starting scatterlist element */ |
| i = msg->sg.start; |
| do { |
| offset += len; |
| len = sk_msg_elem(msg, i)->length; |
| if (start < offset + len) |
| break; |
| sk_msg_iter_var_next(i); |
| } while (i != msg->sg.end); |
| |
| if (unlikely(start >= offset + len)) |
| return -EINVAL; |
| |
| first_sge = i; |
| /* The start may point into the sg element so we need to also |
| * account for the headroom. |
| */ |
| bytes_sg_total = start - offset + bytes; |
| if (!test_bit(i, msg->sg.copy) && bytes_sg_total <= len) |
| goto out; |
| |
| /* At this point we need to linearize multiple scatterlist |
| * elements or a single shared page. Either way we need to |
| * copy into a linear buffer exclusively owned by BPF. Then |
| * place the buffer in the scatterlist and fixup the original |
| * entries by removing the entries now in the linear buffer |
| * and shifting the remaining entries. For now we do not try |
| * to copy partial entries to avoid complexity of running out |
| * of sg_entry slots. The downside is reading a single byte |
| * will copy the entire sg entry. |
| */ |
| do { |
| copy += sk_msg_elem(msg, i)->length; |
| sk_msg_iter_var_next(i); |
| if (bytes_sg_total <= copy) |
| break; |
| } while (i != msg->sg.end); |
| last_sge = i; |
| |
| if (unlikely(bytes_sg_total > copy)) |
| return -EINVAL; |
| |
| page = alloc_pages(__GFP_NOWARN | GFP_ATOMIC | __GFP_COMP, |
| get_order(copy)); |
| if (unlikely(!page)) |
| return -ENOMEM; |
| |
| raw = page_address(page); |
| i = first_sge; |
| do { |
| sge = sk_msg_elem(msg, i); |
| from = sg_virt(sge); |
| len = sge->length; |
| to = raw + poffset; |
| |
| memcpy(to, from, len); |
| poffset += len; |
| sge->length = 0; |
| put_page(sg_page(sge)); |
| |
| sk_msg_iter_var_next(i); |
| } while (i != last_sge); |
| |
| sg_set_page(&msg->sg.data[first_sge], page, copy, 0); |
| |
| /* To repair sg ring we need to shift entries. If we only |
| * had a single entry though we can just replace it and |
| * be done. Otherwise walk the ring and shift the entries. |
| */ |
| WARN_ON_ONCE(last_sge == first_sge); |
| shift = last_sge > first_sge ? |
| last_sge - first_sge - 1 : |
| NR_MSG_FRAG_IDS - first_sge + last_sge - 1; |
| if (!shift) |
| goto out; |
| |
| i = first_sge; |
| sk_msg_iter_var_next(i); |
| do { |
| u32 move_from; |
| |
| if (i + shift >= NR_MSG_FRAG_IDS) |
| move_from = i + shift - NR_MSG_FRAG_IDS; |
| else |
| move_from = i + shift; |
| if (move_from == msg->sg.end) |
| break; |
| |
| msg->sg.data[i] = msg->sg.data[move_from]; |
| msg->sg.data[move_from].length = 0; |
| msg->sg.data[move_from].page_link = 0; |
| msg->sg.data[move_from].offset = 0; |
| sk_msg_iter_var_next(i); |
| } while (1); |
| |
| msg->sg.end = msg->sg.end - shift > msg->sg.end ? |
| msg->sg.end - shift + NR_MSG_FRAG_IDS : |
| msg->sg.end - shift; |
| out: |
| sk_msg_reset_curr(msg); |
| msg->data = sg_virt(&msg->sg.data[first_sge]) + start - offset; |
| msg->data_end = msg->data + bytes; |
| return 0; |
| } |
| |
| static const struct bpf_func_proto bpf_msg_pull_data_proto = { |
| .func = bpf_msg_pull_data, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_CTX, |
| .arg2_type = ARG_ANYTHING, |
| .arg3_type = ARG_ANYTHING, |
| .arg4_type = ARG_ANYTHING, |
| }; |
| |
| BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start, |
| u32, len, u64, flags) |
| { |
| struct scatterlist sge, nsge, nnsge, rsge = {0}, *psge; |
| u32 new, i = 0, l = 0, space, copy = 0, offset = 0; |
| u8 *raw, *to, *from; |
| struct page *page; |
| |
| if (unlikely(flags)) |
| return -EINVAL; |
| |
| if (unlikely(len == 0)) |
| return 0; |
| |
| /* First find the starting scatterlist element */ |
| i = msg->sg.start; |
| do { |
| offset += l; |
| l = sk_msg_elem(msg, i)->length; |
| |
| if (start < offset + l) |
| break; |
| sk_msg_iter_var_next(i); |
| } while (i != msg->sg.end); |
| |
| if (start >= offset + l) |
| return -EINVAL; |
| |
| space = MAX_MSG_FRAGS - sk_msg_elem_used(msg); |
| |
| /* If no space available will fallback to copy, we need at |
| * least one scatterlist elem available to push data into |
| * when start aligns to the beginning of an element or two |
| * when it falls inside an element. We handle the start equals |
| * offset case because its the common case for inserting a |
| * header. |
| */ |
| if (!space || (space == 1 && start != offset)) |
| copy = msg->sg.data[i].length; |
| |
| page = alloc_pages(__GFP_NOWARN | GFP_ATOMIC | __GFP_COMP, |
| get_order(copy + len)); |
| if (unlikely(!page)) |
| return -ENOMEM; |
| |
| if (copy) { |
| int front, back; |
| |
| raw = page_address(page); |
| |
| psge = sk_msg_elem(msg, i); |
| front = start - offset; |
| back = psge->length - front; |
| from = sg_virt(psge); |
| |
| if (front) |
| memcpy(raw, from, front); |
| |
| if (back) { |
| from += front; |
| to = raw + front + len; |
| |
| memcpy(to, from, back); |
| } |
| |
| put_page(sg_page(psge)); |
| } else if (start - offset) { |
| psge = sk_msg_elem(msg, i); |
| rsge = sk_msg_elem_cpy(msg, i); |
| |
| psge->length = start - offset; |
| rsge.length -= psge->length; |
| rsge.offset += start; |
| |
| sk_msg_iter_var_next(i); |
| sg_unmark_end(psge); |
| sg_unmark_end(&rsge); |
| sk_msg_iter_next(msg, end); |
| } |
| |
| /* Slot(s) to place newly allocated data */ |
| new = i; |
| |
| /* Shift one or two slots as needed */ |
| if (!copy) { |
| sge = sk_msg_elem_cpy(msg, i); |
| |
| sk_msg_iter_var_next(i); |
| sg_unmark_end(&sge); |
| sk_msg_iter_next(msg, end); |
| |
| nsge = sk_msg_elem_cpy(msg, i); |
| if (rsge.length) { |
| sk_msg_iter_var_next(i); |
| nnsge = sk_msg_elem_cpy(msg, i); |
| } |
| |
| while (i != msg->sg.end) { |
| msg->sg.data[i] = sge; |
| sge = nsge; |
| sk_msg_iter_var_next(i); |
| if (rsge.length) { |
| nsge = nnsge; |
| nnsge = sk_msg_elem_cpy(msg, i); |
| } else { |
| nsge = sk_msg_elem_cpy(msg, i); |
| } |
| } |
| } |
| |
| /* Place newly allocated data buffer */ |
| sk_mem_charge(msg->sk, len); |
| msg->sg.size += len; |
| __clear_bit(new, msg->sg.copy); |
| sg_set_page(&msg->sg.data[new], page, len + copy, 0); |
| if (rsge.length) { |
| get_page(sg_page(&rsge)); |
| sk_msg_iter_var_next(new); |
| msg->sg.data[new] = rsge; |
| } |
| |
| sk_msg_reset_curr(msg); |
| sk_msg_compute_data_pointers(msg); |
| return 0; |
| } |
| |
| static const struct bpf_func_proto bpf_msg_push_data_proto = { |
| .func = bpf_msg_push_data, |
| .gpl_only = false, |
| .ret_type = RET_INTEGER, |
| .arg1_type = ARG_PTR_TO_CTX, |
| .arg2_type = ARG_ANYTHING, |
| .arg3_type = ARG_ANYTHING, |
| .arg4_type = ARG_ANYTHING, |
| }; |
| |
| static void sk_msg_shift_left(struct sk_msg *msg, int i) |
| { |
| int prev; |
| |
| do { |
| prev = i; |
| sk_msg_iter_var_next(i); |
| msg->sg.data[prev] = msg->sg.data[i]; |
| } while (i != msg->sg.end); |
| |
| sk_msg_iter_prev(msg, end); |
| } |
| |
| static void sk_msg_shift_right(struct sk_msg *msg, int i) |
| { |
| struct scatterlist tmp, sge; |
| |
| sk_msg_iter_next(msg, end); |
| sge = sk_msg_elem_cpy(msg, i); |
| sk_msg_iter_var_next(i); |
| tmp = sk_msg_elem_cpy(msg, i); |
| |
| while (i != msg->sg.end) { |
| msg->sg.data[i] = sge; |
| sk_msg_iter_var_next(i); |
| sge = tmp; |
| tmp = sk_msg_elem_cpy(msg, i); |
| } |
| } |
| |
| BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start, |
| u32, len, u64, flags) |
| { |
| u32 i = 0, l = 0, space, offset = 0; |
| u64 last = start + len; |
| int pop; |
| |
| if (unlikely(flags)) |
| return -EINVAL; |
| |
| /* First find the starting scatterlist element */ |
| i = msg->sg.start; |
| do { |
| offset += l; |
| l = sk_msg_elem(msg, i)->length; |
| |
| if (start < offset + l) |
| break; |
| sk_msg_iter_var_next(i); |
| } while (i != msg->sg.end); |
| |
| /* Bounds checks: start and pop must be inside message */ |
| if (start >= offset + l || last >= msg->sg.size) |
| return -EINVAL; |
| |
| space = MAX_MSG_FRAGS - sk_msg_elem_used(msg); |
| |
| pop = len; |
| /* --------------| offset |
| * -| start |-------- len -------| |
| * |
| * |----- a ----|-------- pop -------|----- b ----| |
| * |______________________________________________| length |
| * |
| * |
| * a: region at front of scatter element to save |
| * b: region at back of scatter element to save when length > A + pop |
| * pop: region to pop from element, same as input 'pop' here will be |
| * decremented below per iteration. |
| * |
| * Two top-level cases to handle when start != offset, first B is non |
| * zero and second B is zero corresponding to when a pop includes more |
| * than one element. |
| * |
| * Then if B is non-zero AND there is no space allocate space and |
| * compact A, B regions into page. If there is space shift ring to |
| * the right free'ing the next element in ring to place B, leaving |
| * A untouched except to reduce length. |
| */ |
| if (start != offset) { |
| struct scatterlist *nsge, *sge = sk_msg_elem(msg, i); |
| int a = start; |
| int b = sge->length - pop - a; |
| |
| sk_msg_iter_var_next(i); |
| |
| if (pop < sge->length - a) { |
| if (space) { |
| sge->length = a; |
| sk_msg_shift_right(msg, i); |
| nsge = sk_msg_elem(msg, i); |
| get_page(sg_page(sge)); |
| sg_set_page(nsge, |
| sg_page(sge), |
| b, sge->offset + pop + a); |
| } else { |
| struct page *page, *orig; |
| u8 *to, *from; |
| |
| page = alloc_pages(__GFP_NOWARN | |
| __GFP_COMP | GFP_ATOMIC, |
| get_order(a + b)); |
| if (unlikely(!page)) |
| return -ENOMEM; |
| |
| sge->length = a; |
| orig = sg_page(sge); |
| from = sg_virt(sge); |
| to = page_address(page); |
| memcpy(to, from, a); |
| memcpy(to + a, from + a |