blob: 50429cbd42da42928f585daddec4ffa5016588ad [file] [log] [blame]
Thomas Gleixnerd2912cb2019-06-04 10:11:33 +02001// SPDX-License-Identifier: GPL-2.0-only
Patrick McHardy96518512013-10-14 11:00:02 +02002/*
Patrick McHardyef1f7df2013-10-10 11:41:20 +02003 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
Pablo Neira Ayuso18140962016-11-24 12:04:55 +01004 * Copyright (c) 2016 Pablo Neira Ayuso <pablo@netfilter.org>
Patrick McHardy96518512013-10-14 11:00:02 +02005 *
Patrick McHardy96518512013-10-14 11:00:02 +02006 * Development of this code funded by Astaro AG (http://www.astaro.com/)
7 */
8
9#include <linux/kernel.h>
Florian Westphal8cfd23e2015-08-17 18:09:55 +020010#include <linux/if_vlan.h>
Patrick McHardy96518512013-10-14 11:00:02 +020011#include <linux/init.h>
12#include <linux/module.h>
13#include <linux/netlink.h>
14#include <linux/netfilter.h>
15#include <linux/netfilter/nf_tables.h>
16#include <net/netfilter/nf_tables_core.h>
17#include <net/netfilter/nf_tables.h>
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +020018#include <net/netfilter/nf_tables_offload.h>
Pablo Neira Ayuso18140962016-11-24 12:04:55 +010019/* For layer 4 checksum field offset. */
20#include <linux/tcp.h>
21#include <linux/udp.h>
Pablo Neira Ayusoc2478972022-10-17 13:03:29 +020022#include <net/gre.h>
Pablo Neira Ayuso18140962016-11-24 12:04:55 +010023#include <linux/icmpv6.h>
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +020024#include <linux/ip.h>
25#include <linux/ipv6.h>
Davide Caratti346e3202020-10-15 18:39:27 +020026#include <net/sctp/checksum.h>
Patrick McHardy96518512013-10-14 11:00:02 +020027
Pablo Neira Ayuso8dfd8b02019-10-31 15:51:22 +010028static bool nft_payload_rebuild_vlan_hdr(const struct sk_buff *skb, int mac_off,
29 struct vlan_ethhdr *veth)
30{
31 if (skb_copy_bits(skb, mac_off, veth, ETH_HLEN))
32 return false;
33
34 veth->h_vlan_proto = skb->vlan_proto;
35 veth->h_vlan_TCI = htons(skb_vlan_tag_get(skb));
36 veth->h_vlan_encapsulated_proto = skb->protocol;
37
38 return true;
39}
40
Florian Westphal8cfd23e2015-08-17 18:09:55 +020041/* add vlan header into the user buffer for if tag was removed by offloads */
42static bool
43nft_payload_copy_vlan(u32 *d, const struct sk_buff *skb, u8 offset, u8 len)
44{
45 int mac_off = skb_mac_header(skb) - skb->data;
Pablo Neira Ayusof41f72d2019-10-31 15:51:21 +010046 u8 *vlanh, *dst_u8 = (u8 *) d;
Florian Westphal8cfd23e2015-08-17 18:09:55 +020047 struct vlan_ethhdr veth;
48
49 vlanh = (u8 *) &veth;
Pablo Neira Ayusoaff5c012024-05-09 23:02:24 +020050 if (offset < VLAN_ETH_HLEN) {
Pablo Neira Ayusof41f72d2019-10-31 15:51:21 +010051 u8 ethlen = len;
Florian Westphal8cfd23e2015-08-17 18:09:55 +020052
Pablo Neira Ayusoaff5c012024-05-09 23:02:24 +020053 if (!nft_payload_rebuild_vlan_hdr(skb, mac_off, &veth))
Florian Westphal8cfd23e2015-08-17 18:09:55 +020054 return false;
55
Pablo Neira Ayusoaff5c012024-05-09 23:02:24 +020056 if (offset + len > VLAN_ETH_HLEN)
57 ethlen -= offset + len - VLAN_ETH_HLEN;
Florian Westphal8cfd23e2015-08-17 18:09:55 +020058
Pablo Neira Ayusoaff5c012024-05-09 23:02:24 +020059 memcpy(dst_u8, vlanh + offset, ethlen);
Florian Westphal8cfd23e2015-08-17 18:09:55 +020060
61 len -= ethlen;
62 if (len == 0)
63 return true;
64
65 dst_u8 += ethlen;
Pablo Neira Ayusoaff5c012024-05-09 23:02:24 +020066 offset = ETH_HLEN;
Pablo Neira Ayusof41f72d2019-10-31 15:51:21 +010067 } else {
Pablo Neira Ayusoaff5c012024-05-09 23:02:24 +020068 offset -= VLAN_HLEN;
Florian Westphal8cfd23e2015-08-17 18:09:55 +020069 }
70
Florian Westphal8cfd23e2015-08-17 18:09:55 +020071 return skb_copy_bits(skb, offset + mac_off, dst_u8, len) == 0;
72}
73
Pablo Neira Ayusoc46b38d2021-10-28 22:15:00 +020074static int __nft_payload_inner_offset(struct nft_pktinfo *pkt)
75{
76 unsigned int thoff = nft_thoff(pkt);
77
Florian Westphala9e85032022-01-29 17:13:23 +010078 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
Pablo Neira Ayusoc46b38d2021-10-28 22:15:00 +020079 return -1;
80
81 switch (pkt->tprot) {
82 case IPPROTO_UDP:
83 pkt->inneroff = thoff + sizeof(struct udphdr);
84 break;
85 case IPPROTO_TCP: {
86 struct tcphdr *th, _tcph;
87
88 th = skb_header_pointer(pkt->skb, thoff, sizeof(_tcph), &_tcph);
89 if (!th)
90 return -1;
91
92 pkt->inneroff = thoff + __tcp_hdrlen(th);
93 }
94 break;
Pablo Neira Ayusoc2478972022-10-17 13:03:29 +020095 case IPPROTO_GRE: {
Pablo Neira Ayuso66394122022-10-28 10:05:57 +020096 u32 offset = sizeof(struct gre_base_hdr);
Pablo Neira Ayusoc2478972022-10-17 13:03:29 +020097 struct gre_base_hdr *gre, _gre;
Pablo Neira Ayuso66394122022-10-28 10:05:57 +020098 __be16 version;
Pablo Neira Ayusoc2478972022-10-17 13:03:29 +020099
100 gre = skb_header_pointer(pkt->skb, thoff, sizeof(_gre), &_gre);
101 if (!gre)
102 return -1;
103
104 version = gre->flags & GRE_VERSION;
105 switch (version) {
106 case GRE_VERSION_0:
107 if (gre->flags & GRE_ROUTING)
108 return -1;
109
110 if (gre->flags & GRE_CSUM) {
111 offset += sizeof_field(struct gre_full_hdr, csum) +
112 sizeof_field(struct gre_full_hdr, reserved1);
113 }
114 if (gre->flags & GRE_KEY)
115 offset += sizeof_field(struct gre_full_hdr, key);
116
117 if (gre->flags & GRE_SEQ)
118 offset += sizeof_field(struct gre_full_hdr, seq);
119 break;
120 default:
121 return -1;
122 }
123
124 pkt->inneroff = thoff + offset;
125 }
126 break;
Pablo Neira Ayuso3927ce82022-10-17 13:03:30 +0200127 case IPPROTO_IPIP:
128 pkt->inneroff = thoff;
129 break;
Pablo Neira Ayusoc46b38d2021-10-28 22:15:00 +0200130 default:
131 return -1;
132 }
133
134 pkt->flags |= NFT_PKTINFO_INNER;
135
136 return 0;
137}
138
Pablo Neira Ayuso3a073272022-10-25 13:48:15 +0200139int nft_payload_inner_offset(const struct nft_pktinfo *pkt)
Pablo Neira Ayusoc46b38d2021-10-28 22:15:00 +0200140{
141 if (!(pkt->flags & NFT_PKTINFO_INNER) &&
142 __nft_payload_inner_offset((struct nft_pktinfo *)pkt) < 0)
143 return -1;
144
145 return pkt->inneroff;
146}
147
Pablo Neira Ayuso33c563e2024-05-08 22:50:34 +0200148static bool nft_payload_need_vlan_adjust(u32 offset, u32 len)
Florian Westphalaf84f9e2023-09-29 10:42:10 +0200149{
Pablo Neira Ayuso33c563e2024-05-08 22:50:34 +0200150 unsigned int boundary = offset + len;
Florian Westphalaf84f9e2023-09-29 10:42:10 +0200151
152 /* data past ether src/dst requested, copy needed */
Pablo Neira Ayuso33c563e2024-05-08 22:50:34 +0200153 if (boundary > offsetof(struct ethhdr, h_proto))
Florian Westphalaf84f9e2023-09-29 10:42:10 +0200154 return true;
155
156 return false;
157}
158
Florian Westphal10870dd2019-01-08 17:35:34 +0100159void nft_payload_eval(const struct nft_expr *expr,
160 struct nft_regs *regs,
161 const struct nft_pktinfo *pkt)
Patrick McHardy96518512013-10-14 11:00:02 +0200162{
163 const struct nft_payload *priv = nft_expr_priv(expr);
164 const struct sk_buff *skb = pkt->skb;
Patrick McHardy49499c32015-04-11 02:27:37 +0100165 u32 *dest = &regs->data[priv->dreg];
Patrick McHardy96518512013-10-14 11:00:02 +0200166 int offset;
167
Florian Westphal1e105e62020-08-20 21:05:50 +0200168 if (priv->len % NFT_REG32_SIZE)
169 dest[priv->len / NFT_REG32_SIZE] = 0;
170
Patrick McHardy96518512013-10-14 11:00:02 +0200171 switch (priv->base) {
172 case NFT_PAYLOAD_LL_HEADER:
Florian Westphald351c1e2023-10-08 19:36:53 +0200173 if (!skb_mac_header_was_set(skb) || skb_mac_header_len(skb) == 0)
Patrick McHardy96518512013-10-14 11:00:02 +0200174 goto err;
Florian Westphal8cfd23e2015-08-17 18:09:55 +0200175
Pablo Neira Ayusode6843b2023-06-06 09:38:42 +0200176 if (skb_vlan_tag_present(skb) &&
Pablo Neira Ayuso33c563e2024-05-08 22:50:34 +0200177 nft_payload_need_vlan_adjust(priv->offset, priv->len)) {
Florian Westphal8cfd23e2015-08-17 18:09:55 +0200178 if (!nft_payload_copy_vlan(dest, skb,
179 priv->offset, priv->len))
180 goto err;
181 return;
182 }
Patrick McHardy96518512013-10-14 11:00:02 +0200183 offset = skb_mac_header(skb) - skb->data;
184 break;
185 case NFT_PAYLOAD_NETWORK_HEADER:
186 offset = skb_network_offset(skb);
187 break;
188 case NFT_PAYLOAD_TRANSPORT_HEADER:
Florian Westphala9e85032022-01-29 17:13:23 +0100189 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
Liping Zhanga20877b2016-09-17 14:31:20 +0800190 goto err;
Florian Westphal2d7b4ac2021-05-28 12:30:06 +0200191 offset = nft_thoff(pkt);
Patrick McHardy96518512013-10-14 11:00:02 +0200192 break;
Pablo Neira Ayusoc46b38d2021-10-28 22:15:00 +0200193 case NFT_PAYLOAD_INNER_HEADER:
194 offset = nft_payload_inner_offset(pkt);
195 if (offset < 0)
196 goto err;
197 break;
Patrick McHardy96518512013-10-14 11:00:02 +0200198 default:
Pablo Neira Ayuso8801d792021-12-10 00:09:49 +0100199 WARN_ON_ONCE(1);
200 goto err;
Patrick McHardy96518512013-10-14 11:00:02 +0200201 }
202 offset += priv->offset;
203
Patrick McHardyfad136e2015-04-11 02:27:33 +0100204 if (skb_copy_bits(skb, offset, dest, priv->len) < 0)
Patrick McHardy96518512013-10-14 11:00:02 +0200205 goto err;
206 return;
207err:
Patrick McHardya55e22e2015-04-11 02:27:31 +0100208 regs->verdict.code = NFT_BREAK;
Patrick McHardy96518512013-10-14 11:00:02 +0200209}
210
211static const struct nla_policy nft_payload_policy[NFTA_PAYLOAD_MAX + 1] = {
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000212 [NFTA_PAYLOAD_SREG] = { .type = NLA_U32 },
213 [NFTA_PAYLOAD_DREG] = { .type = NLA_U32 },
214 [NFTA_PAYLOAD_BASE] = { .type = NLA_U32 },
Florian Westphalecaf75f2022-10-31 13:34:07 +0100215 [NFTA_PAYLOAD_OFFSET] = NLA_POLICY_MAX(NLA_BE32, 255),
216 [NFTA_PAYLOAD_LEN] = NLA_POLICY_MAX(NLA_BE32, 255),
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000217 [NFTA_PAYLOAD_CSUM_TYPE] = { .type = NLA_U32 },
Florian Westphalecaf75f2022-10-31 13:34:07 +0100218 [NFTA_PAYLOAD_CSUM_OFFSET] = NLA_POLICY_MAX(NLA_BE32, 255),
Jakub Kicinski9d6effb2020-03-02 21:08:32 -0800219 [NFTA_PAYLOAD_CSUM_FLAGS] = { .type = NLA_U32 },
Patrick McHardy96518512013-10-14 11:00:02 +0200220};
221
222static int nft_payload_init(const struct nft_ctx *ctx,
223 const struct nft_expr *expr,
224 const struct nlattr * const tb[])
225{
226 struct nft_payload *priv = nft_expr_priv(expr);
Patrick McHardy96518512013-10-14 11:00:02 +0200227
Patrick McHardyc29b72e2013-10-10 11:06:41 +0200228 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
Patrick McHardy96518512013-10-14 11:00:02 +0200229 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
230 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
Patrick McHardy96518512013-10-14 11:00:02 +0200231
Pablo Neira Ayuso345023b2021-01-25 18:27:22 +0100232 return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG],
233 &priv->dreg, NULL, NFT_DATA_VALUE,
234 priv->len);
Patrick McHardy96518512013-10-14 11:00:02 +0200235}
236
Phil Sutter7d34aa32022-10-14 23:45:58 +0200237static int nft_payload_dump(struct sk_buff *skb,
238 const struct nft_expr *expr, bool reset)
Patrick McHardy96518512013-10-14 11:00:02 +0200239{
240 const struct nft_payload *priv = nft_expr_priv(expr);
241
Patrick McHardyb1c96ed2015-04-11 02:27:36 +0100242 if (nft_dump_register(skb, NFTA_PAYLOAD_DREG, priv->dreg) ||
Patrick McHardy96518512013-10-14 11:00:02 +0200243 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) ||
244 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) ||
245 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)))
246 goto nla_put_failure;
247 return 0;
248
249nla_put_failure:
250 return -1;
251}
252
Pablo Neira Ayusoa7c176b2022-01-09 17:11:22 +0100253static bool nft_payload_reduce(struct nft_regs_track *track,
254 const struct nft_expr *expr)
255{
256 const struct nft_payload *priv = nft_expr_priv(expr);
257 const struct nft_payload *payload;
258
Pablo Neira Ayuso34cc9e52022-03-14 18:23:01 +0100259 if (!nft_reg_track_cmp(track, expr, priv->dreg)) {
260 nft_reg_track_update(track, expr, priv->dreg, priv->len);
Pablo Neira Ayusoa7c176b2022-01-09 17:11:22 +0100261 return false;
262 }
263
264 payload = nft_expr_priv(track->regs[priv->dreg].selector);
265 if (priv->base != payload->base ||
266 priv->offset != payload->offset ||
267 priv->len != payload->len) {
Pablo Neira Ayuso34cc9e52022-03-14 18:23:01 +0100268 nft_reg_track_update(track, expr, priv->dreg, priv->len);
Pablo Neira Ayusoa7c176b2022-01-09 17:11:22 +0100269 return false;
270 }
271
272 if (!track->regs[priv->dreg].bitwise)
273 return true;
274
Pablo Neira Ayusobe5650f2022-01-09 17:11:24 +0100275 return nft_expr_reduce_bitwise(track, expr);
Pablo Neira Ayusoa7c176b2022-01-09 17:11:22 +0100276}
277
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100278static bool nft_payload_offload_mask(struct nft_offload_reg *reg,
279 u32 priv_len, u32 field_len)
280{
281 unsigned int remainder, delta, k;
282 struct nft_data mask = {};
283 __be32 remainder_mask;
284
285 if (priv_len == field_len) {
286 memset(&reg->mask, 0xff, priv_len);
287 return true;
288 } else if (priv_len > field_len) {
289 return false;
290 }
291
292 memset(&mask, 0xff, field_len);
293 remainder = priv_len % sizeof(u32);
294 if (remainder) {
295 k = priv_len / sizeof(u32);
296 delta = field_len - priv_len;
297 remainder_mask = htonl(~((1 << (delta * BITS_PER_BYTE)) - 1));
298 mask.data[k] = (__force u32)remainder_mask;
299 }
300
301 memcpy(&reg->mask, &mask, field_len);
302
303 return true;
304}
305
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200306static int nft_payload_offload_ll(struct nft_offload_ctx *ctx,
307 struct nft_flow_rule *flow,
308 const struct nft_payload *priv)
309{
310 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
311
312 switch (priv->offset) {
313 case offsetof(struct ethhdr, h_source):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100314 if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN))
wenxua69a85da2019-10-24 15:52:45 +0800315 return -EOPNOTSUPP;
316
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200317 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
318 src, ETH_ALEN, reg);
319 break;
320 case offsetof(struct ethhdr, h_dest):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100321 if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN))
wenxua69a85da2019-10-24 15:52:45 +0800322 return -EOPNOTSUPP;
323
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200324 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
325 dst, ETH_ALEN, reg);
326 break;
Pablo Neira Ayusoa82055a2019-11-19 23:05:54 +0100327 case offsetof(struct ethhdr, h_proto):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100328 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
Pablo Neira Ayusoa82055a2019-11-19 23:05:54 +0100329 return -EOPNOTSUPP;
330
331 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic,
332 n_proto, sizeof(__be16), reg);
333 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
334 break;
335 case offsetof(struct vlan_ethhdr, h_vlan_TCI):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100336 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
Pablo Neira Ayusoa82055a2019-11-19 23:05:54 +0100337 return -EOPNOTSUPP;
338
Pablo Neira Ayusoff4d90a2021-04-12 14:20:15 +0200339 NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_VLAN, vlan,
340 vlan_tci, sizeof(__be16), reg,
341 NFT_OFFLOAD_F_NETWORK2HOST);
Pablo Neira Ayusoa82055a2019-11-19 23:05:54 +0100342 break;
343 case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100344 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
Pablo Neira Ayusoa82055a2019-11-19 23:05:54 +0100345 return -EOPNOTSUPP;
346
347 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_VLAN, vlan,
348 vlan_tpid, sizeof(__be16), reg);
Pablo Neira Ayuso89d8fd42019-11-19 23:05:55 +0100349 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
350 break;
351 case offsetof(struct vlan_ethhdr, h_vlan_TCI) + sizeof(struct vlan_hdr):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100352 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
Pablo Neira Ayuso89d8fd42019-11-19 23:05:55 +0100353 return -EOPNOTSUPP;
354
Pablo Neira Ayusoff4d90a2021-04-12 14:20:15 +0200355 NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_CVLAN, cvlan,
356 vlan_tci, sizeof(__be16), reg,
357 NFT_OFFLOAD_F_NETWORK2HOST);
Pablo Neira Ayuso89d8fd42019-11-19 23:05:55 +0100358 break;
359 case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto) +
360 sizeof(struct vlan_hdr):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100361 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
Pablo Neira Ayuso89d8fd42019-11-19 23:05:55 +0100362 return -EOPNOTSUPP;
363
Pablo Neira Ayuso14c20642021-04-12 14:11:39 +0200364 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_CVLAN, cvlan,
Pablo Neira Ayuso89d8fd42019-11-19 23:05:55 +0100365 vlan_tpid, sizeof(__be16), reg);
Pablo Neira Ayuso14c20642021-04-12 14:11:39 +0200366 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
Pablo Neira Ayusoa82055a2019-11-19 23:05:54 +0100367 break;
wenxua69a85da2019-10-24 15:52:45 +0800368 default:
369 return -EOPNOTSUPP;
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200370 }
371
372 return 0;
373}
374
375static int nft_payload_offload_ip(struct nft_offload_ctx *ctx,
376 struct nft_flow_rule *flow,
377 const struct nft_payload *priv)
378{
379 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
380
381 switch (priv->offset) {
382 case offsetof(struct iphdr, saddr):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100383 if (!nft_payload_offload_mask(reg, priv->len,
384 sizeof(struct in_addr)))
wenxua69a85da2019-10-24 15:52:45 +0800385 return -EOPNOTSUPP;
386
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200387 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, src,
388 sizeof(struct in_addr), reg);
Pablo Neira Ayuso3c78e9e2020-11-25 23:50:07 +0100389 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200390 break;
391 case offsetof(struct iphdr, daddr):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100392 if (!nft_payload_offload_mask(reg, priv->len,
393 sizeof(struct in_addr)))
wenxua69a85da2019-10-24 15:52:45 +0800394 return -EOPNOTSUPP;
395
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200396 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, dst,
397 sizeof(struct in_addr), reg);
Pablo Neira Ayuso3c78e9e2020-11-25 23:50:07 +0100398 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200399 break;
400 case offsetof(struct iphdr, protocol):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100401 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8)))
wenxua69a85da2019-10-24 15:52:45 +0800402 return -EOPNOTSUPP;
403
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200404 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
405 sizeof(__u8), reg);
406 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
407 break;
408 default:
409 return -EOPNOTSUPP;
410 }
411
412 return 0;
413}
414
415static int nft_payload_offload_ip6(struct nft_offload_ctx *ctx,
416 struct nft_flow_rule *flow,
417 const struct nft_payload *priv)
418{
419 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
420
421 switch (priv->offset) {
422 case offsetof(struct ipv6hdr, saddr):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100423 if (!nft_payload_offload_mask(reg, priv->len,
424 sizeof(struct in6_addr)))
wenxua69a85da2019-10-24 15:52:45 +0800425 return -EOPNOTSUPP;
426
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200427 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, src,
428 sizeof(struct in6_addr), reg);
Pablo Neira Ayuso3c78e9e2020-11-25 23:50:07 +0100429 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200430 break;
431 case offsetof(struct ipv6hdr, daddr):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100432 if (!nft_payload_offload_mask(reg, priv->len,
433 sizeof(struct in6_addr)))
wenxua69a85da2019-10-24 15:52:45 +0800434 return -EOPNOTSUPP;
435
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200436 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, dst,
437 sizeof(struct in6_addr), reg);
Pablo Neira Ayuso3c78e9e2020-11-25 23:50:07 +0100438 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200439 break;
440 case offsetof(struct ipv6hdr, nexthdr):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100441 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8)))
wenxua69a85da2019-10-24 15:52:45 +0800442 return -EOPNOTSUPP;
443
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200444 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
445 sizeof(__u8), reg);
446 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
447 break;
448 default:
449 return -EOPNOTSUPP;
450 }
451
452 return 0;
453}
454
455static int nft_payload_offload_nh(struct nft_offload_ctx *ctx,
456 struct nft_flow_rule *flow,
457 const struct nft_payload *priv)
458{
459 int err;
460
461 switch (ctx->dep.l3num) {
462 case htons(ETH_P_IP):
463 err = nft_payload_offload_ip(ctx, flow, priv);
464 break;
465 case htons(ETH_P_IPV6):
466 err = nft_payload_offload_ip6(ctx, flow, priv);
467 break;
468 default:
469 return -EOPNOTSUPP;
470 }
471
472 return err;
473}
474
475static int nft_payload_offload_tcp(struct nft_offload_ctx *ctx,
476 struct nft_flow_rule *flow,
477 const struct nft_payload *priv)
478{
479 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
480
481 switch (priv->offset) {
482 case offsetof(struct tcphdr, source):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100483 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
wenxua69a85da2019-10-24 15:52:45 +0800484 return -EOPNOTSUPP;
485
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200486 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
487 sizeof(__be16), reg);
488 break;
489 case offsetof(struct tcphdr, dest):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100490 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
wenxua69a85da2019-10-24 15:52:45 +0800491 return -EOPNOTSUPP;
492
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200493 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
494 sizeof(__be16), reg);
495 break;
496 default:
497 return -EOPNOTSUPP;
498 }
499
500 return 0;
501}
502
503static int nft_payload_offload_udp(struct nft_offload_ctx *ctx,
504 struct nft_flow_rule *flow,
505 const struct nft_payload *priv)
506{
507 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
508
509 switch (priv->offset) {
510 case offsetof(struct udphdr, source):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100511 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
wenxua69a85da2019-10-24 15:52:45 +0800512 return -EOPNOTSUPP;
513
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200514 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
515 sizeof(__be16), reg);
516 break;
517 case offsetof(struct udphdr, dest):
Pablo Neira Ayusoa5d45bc2020-11-25 23:50:17 +0100518 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
wenxua69a85da2019-10-24 15:52:45 +0800519 return -EOPNOTSUPP;
520
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200521 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
522 sizeof(__be16), reg);
523 break;
524 default:
525 return -EOPNOTSUPP;
526 }
527
528 return 0;
529}
530
531static int nft_payload_offload_th(struct nft_offload_ctx *ctx,
532 struct nft_flow_rule *flow,
533 const struct nft_payload *priv)
534{
535 int err;
536
537 switch (ctx->dep.protonum) {
538 case IPPROTO_TCP:
539 err = nft_payload_offload_tcp(ctx, flow, priv);
540 break;
541 case IPPROTO_UDP:
542 err = nft_payload_offload_udp(ctx, flow, priv);
543 break;
544 default:
545 return -EOPNOTSUPP;
546 }
547
548 return err;
549}
550
551static int nft_payload_offload(struct nft_offload_ctx *ctx,
552 struct nft_flow_rule *flow,
553 const struct nft_expr *expr)
554{
555 const struct nft_payload *priv = nft_expr_priv(expr);
556 int err;
557
558 switch (priv->base) {
559 case NFT_PAYLOAD_LL_HEADER:
560 err = nft_payload_offload_ll(ctx, flow, priv);
561 break;
562 case NFT_PAYLOAD_NETWORK_HEADER:
563 err = nft_payload_offload_nh(ctx, flow, priv);
564 break;
565 case NFT_PAYLOAD_TRANSPORT_HEADER:
566 err = nft_payload_offload_th(ctx, flow, priv);
567 break;
568 default:
569 err = -EOPNOTSUPP;
570 break;
571 }
572 return err;
573}
574
Patrick McHardyef1f7df2013-10-10 11:41:20 +0200575static const struct nft_expr_ops nft_payload_ops = {
576 .type = &nft_payload_type,
Patrick McHardy96518512013-10-14 11:00:02 +0200577 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
Patrick McHardy96518512013-10-14 11:00:02 +0200578 .eval = nft_payload_eval,
579 .init = nft_payload_init,
580 .dump = nft_payload_dump,
Pablo Neira Ayusoa7c176b2022-01-09 17:11:22 +0100581 .reduce = nft_payload_reduce,
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200582 .offload = nft_payload_offload,
Patrick McHardyef1f7df2013-10-10 11:41:20 +0200583};
584
Patrick McHardyc29b72e2013-10-10 11:06:41 +0200585const struct nft_expr_ops nft_payload_fast_ops = {
586 .type = &nft_payload_type,
587 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
588 .eval = nft_payload_eval,
589 .init = nft_payload_init,
590 .dump = nft_payload_dump,
Pablo Neira Ayusoa7c176b2022-01-09 17:11:22 +0100591 .reduce = nft_payload_reduce,
Pablo Neira Ayusoc9626a22019-07-09 23:00:43 +0200592 .offload = nft_payload_offload,
Patrick McHardyc29b72e2013-10-10 11:06:41 +0200593};
594
Pablo Neira Ayuso3a073272022-10-25 13:48:15 +0200595void nft_payload_inner_eval(const struct nft_expr *expr, struct nft_regs *regs,
596 const struct nft_pktinfo *pkt,
597 struct nft_inner_tun_ctx *tun_ctx)
598{
599 const struct nft_payload *priv = nft_expr_priv(expr);
600 const struct sk_buff *skb = pkt->skb;
601 u32 *dest = &regs->data[priv->dreg];
602 int offset;
603
604 if (priv->len % NFT_REG32_SIZE)
605 dest[priv->len / NFT_REG32_SIZE] = 0;
606
607 switch (priv->base) {
608 case NFT_PAYLOAD_TUN_HEADER:
609 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_TUN))
610 goto err;
611
612 offset = tun_ctx->inner_tunoff;
613 break;
614 case NFT_PAYLOAD_LL_HEADER:
615 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_LL))
616 goto err;
617
618 offset = tun_ctx->inner_lloff;
619 break;
620 case NFT_PAYLOAD_NETWORK_HEADER:
621 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_NH))
622 goto err;
623
624 offset = tun_ctx->inner_nhoff;
625 break;
626 case NFT_PAYLOAD_TRANSPORT_HEADER:
627 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_TH))
628 goto err;
629
630 offset = tun_ctx->inner_thoff;
631 break;
632 default:
633 WARN_ON_ONCE(1);
634 goto err;
635 }
636 offset += priv->offset;
637
638 if (skb_copy_bits(skb, offset, dest, priv->len) < 0)
639 goto err;
640
641 return;
642err:
643 regs->verdict.code = NFT_BREAK;
644}
645
646static int nft_payload_inner_init(const struct nft_ctx *ctx,
647 const struct nft_expr *expr,
648 const struct nlattr * const tb[])
649{
650 struct nft_payload *priv = nft_expr_priv(expr);
651 u32 base;
652
Davide Ornaghic4ab9da2024-06-05 13:03:45 +0200653 if (!tb[NFTA_PAYLOAD_BASE] || !tb[NFTA_PAYLOAD_OFFSET] ||
654 !tb[NFTA_PAYLOAD_LEN] || !tb[NFTA_PAYLOAD_DREG])
655 return -EINVAL;
656
Pablo Neira Ayuso3a073272022-10-25 13:48:15 +0200657 base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
658 switch (base) {
659 case NFT_PAYLOAD_TUN_HEADER:
660 case NFT_PAYLOAD_LL_HEADER:
661 case NFT_PAYLOAD_NETWORK_HEADER:
662 case NFT_PAYLOAD_TRANSPORT_HEADER:
663 break;
664 default:
665 return -EOPNOTSUPP;
666 }
667
668 priv->base = base;
669 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
670 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
671
672 return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG],
673 &priv->dreg, NULL, NFT_DATA_VALUE,
674 priv->len);
675}
676
677static const struct nft_expr_ops nft_payload_inner_ops = {
678 .type = &nft_payload_type,
679 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
680 .init = nft_payload_inner_init,
681 .dump = nft_payload_dump,
682 /* direct call to nft_payload_inner_eval(). */
683};
684
Pablo Neira Ayuso18140962016-11-24 12:04:55 +0100685static inline void nft_csum_replace(__sum16 *sum, __wsum fsum, __wsum tsum)
686{
687 *sum = csum_fold(csum_add(csum_sub(~csum_unfold(*sum), fsum), tsum));
688 if (*sum == 0)
689 *sum = CSUM_MANGLED_0;
690}
691
692static bool nft_payload_udp_checksum(struct sk_buff *skb, unsigned int thoff)
693{
694 struct udphdr *uh, _uh;
695
696 uh = skb_header_pointer(skb, thoff, sizeof(_uh), &_uh);
697 if (!uh)
698 return false;
699
Florian Westphal5fd02eb2017-08-21 16:19:26 +0200700 return (__force bool)uh->check;
Pablo Neira Ayuso18140962016-11-24 12:04:55 +0100701}
702
703static int nft_payload_l4csum_offset(const struct nft_pktinfo *pkt,
704 struct sk_buff *skb,
705 unsigned int *l4csum_offset)
706{
Pablo Neira Ayuso4e1860a2022-01-05 16:09:57 +0100707 if (pkt->fragoff)
708 return -1;
709
Pablo Neira Ayuso18140962016-11-24 12:04:55 +0100710 switch (pkt->tprot) {
711 case IPPROTO_TCP:
712 *l4csum_offset = offsetof(struct tcphdr, check);
713 break;
714 case IPPROTO_UDP:
Florian Westphal2d7b4ac2021-05-28 12:30:06 +0200715 if (!nft_payload_udp_checksum(skb, nft_thoff(pkt)))
Pablo Neira Ayuso18140962016-11-24 12:04:55 +0100716 return -1;
Gustavo A. R. Silva954d8292020-07-08 15:09:39 -0500717 fallthrough;
Pablo Neira Ayuso18140962016-11-24 12:04:55 +0100718 case IPPROTO_UDPLITE:
719 *l4csum_offset = offsetof(struct udphdr, check);
720 break;
721 case IPPROTO_ICMPV6:
722 *l4csum_offset = offsetof(struct icmp6hdr, icmp6_cksum);
723 break;
724 default:
725 return -1;
726 }
727
Florian Westphal2d7b4ac2021-05-28 12:30:06 +0200728 *l4csum_offset += nft_thoff(pkt);
Pablo Neira Ayuso18140962016-11-24 12:04:55 +0100729 return 0;
730}
731
Davide Caratti346e3202020-10-15 18:39:27 +0200732static int nft_payload_csum_sctp(struct sk_buff *skb, int offset)
733{
734 struct sctphdr *sh;
735
736 if (skb_ensure_writable(skb, offset + sizeof(*sh)))
737 return -1;
738
739 sh = (struct sctphdr *)(skb->data + offset);
740 sh->checksum = sctp_compute_cksum(skb, offset);
741 skb->ip_summed = CHECKSUM_UNNECESSARY;
742 return 0;
743}
744
Pablo Neira Ayuso18140962016-11-24 12:04:55 +0100745static int nft_payload_l4csum_update(const struct nft_pktinfo *pkt,
746 struct sk_buff *skb,
747 __wsum fsum, __wsum tsum)
748{
749 int l4csum_offset;
750 __sum16 sum;
751
752 /* If we cannot determine layer 4 checksum offset or this packet doesn't
753 * require layer 4 checksum recalculation, skip this packet.
754 */
755 if (nft_payload_l4csum_offset(pkt, skb, &l4csum_offset) < 0)
756 return 0;
757
758 if (skb_copy_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
759 return -1;
760
761 /* Checksum mangling for an arbitrary amount of bytes, based on
762 * inet_proto_csum_replace*() functions.
763 */
764 if (skb->ip_summed != CHECKSUM_PARTIAL) {
765 nft_csum_replace(&sum, fsum, tsum);
766 if (skb->ip_summed == CHECKSUM_COMPLETE) {
767 skb->csum = ~csum_add(csum_sub(~(skb->csum), fsum),
768 tsum);
769 }
770 } else {
771 sum = ~csum_fold(csum_add(csum_sub(csum_unfold(sum), fsum),
772 tsum));
773 }
774
Florian Westphal7418ee42019-05-23 15:44:09 +0200775 if (skb_ensure_writable(skb, l4csum_offset + sizeof(sum)) ||
Pablo Neira Ayuso18140962016-11-24 12:04:55 +0100776 skb_store_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
777 return -1;
778
779 return 0;
780}
781
Pablo Neira Ayuso053d20f2016-12-14 12:15:49 +0100782static int nft_payload_csum_inet(struct sk_buff *skb, const u32 *src,
783 __wsum fsum, __wsum tsum, int csum_offset)
784{
785 __sum16 sum;
786
787 if (skb_copy_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
788 return -1;
789
790 nft_csum_replace(&sum, fsum, tsum);
Florian Westphal7418ee42019-05-23 15:44:09 +0200791 if (skb_ensure_writable(skb, csum_offset + sizeof(sum)) ||
Pablo Neira Ayuso053d20f2016-12-14 12:15:49 +0100792 skb_store_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
793 return -1;
794
795 return 0;
796}
797
Pablo Neira Ayusoac1f8c02022-09-28 23:55:06 +0200798struct nft_payload_set {
799 enum nft_payload_bases base:8;
800 u8 offset;
801 u8 len;
802 u8 sreg;
803 u8 csum_type;
804 u8 csum_offset;
805 u8 csum_flags;
806};
807
Pablo Neira Ayuso33c563e2024-05-08 22:50:34 +0200808/* This is not struct vlan_hdr. */
809struct nft_payload_vlan_hdr {
810 __be16 h_vlan_proto;
811 __be16 h_vlan_TCI;
812};
813
814static bool
815nft_payload_set_vlan(const u32 *src, struct sk_buff *skb, u8 offset, u8 len,
816 int *vlan_hlen)
817{
818 struct nft_payload_vlan_hdr *vlanh;
819 __be16 vlan_proto;
820 u16 vlan_tci;
821
822 if (offset >= offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto)) {
823 *vlan_hlen = VLAN_HLEN;
824 return true;
825 }
826
827 switch (offset) {
828 case offsetof(struct vlan_ethhdr, h_vlan_proto):
829 if (len == 2) {
830 vlan_proto = nft_reg_load_be16(src);
831 skb->vlan_proto = vlan_proto;
832 } else if (len == 4) {
833 vlanh = (struct nft_payload_vlan_hdr *)src;
834 __vlan_hwaccel_put_tag(skb, vlanh->h_vlan_proto,
835 ntohs(vlanh->h_vlan_TCI));
836 } else {
837 return false;
838 }
839 break;
840 case offsetof(struct vlan_ethhdr, h_vlan_TCI):
841 if (len != 2)
842 return false;
843
844 vlan_tci = ntohs(nft_reg_load_be16(src));
845 skb->vlan_tci = vlan_tci;
846 break;
847 default:
848 return false;
849 }
850
851 return true;
852}
853
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000854static void nft_payload_set_eval(const struct nft_expr *expr,
855 struct nft_regs *regs,
856 const struct nft_pktinfo *pkt)
857{
858 const struct nft_payload_set *priv = nft_expr_priv(expr);
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000859 const u32 *src = &regs->data[priv->sreg];
Pablo Neira Ayuso33c563e2024-05-08 22:50:34 +0200860 int offset, csum_offset, vlan_hlen = 0;
861 struct sk_buff *skb = pkt->skb;
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000862 __wsum fsum, tsum;
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000863
864 switch (priv->base) {
865 case NFT_PAYLOAD_LL_HEADER:
866 if (!skb_mac_header_was_set(skb))
867 goto err;
Pablo Neira Ayuso33c563e2024-05-08 22:50:34 +0200868
869 if (skb_vlan_tag_present(skb) &&
870 nft_payload_need_vlan_adjust(priv->offset, priv->len)) {
871 if (!nft_payload_set_vlan(src, skb,
872 priv->offset, priv->len,
873 &vlan_hlen))
874 goto err;
875
876 if (!vlan_hlen)
877 return;
878 }
879
880 offset = skb_mac_header(skb) - skb->data - vlan_hlen;
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000881 break;
882 case NFT_PAYLOAD_NETWORK_HEADER:
883 offset = skb_network_offset(skb);
884 break;
885 case NFT_PAYLOAD_TRANSPORT_HEADER:
Florian Westphala9e85032022-01-29 17:13:23 +0100886 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
Liping Zhanga20877b2016-09-17 14:31:20 +0800887 goto err;
Florian Westphal2d7b4ac2021-05-28 12:30:06 +0200888 offset = nft_thoff(pkt);
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000889 break;
Pablo Neira Ayusoc46b38d2021-10-28 22:15:00 +0200890 case NFT_PAYLOAD_INNER_HEADER:
891 offset = nft_payload_inner_offset(pkt);
892 if (offset < 0)
893 goto err;
894 break;
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000895 default:
Pablo Neira Ayuso8801d792021-12-10 00:09:49 +0100896 WARN_ON_ONCE(1);
897 goto err;
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000898 }
899
900 csum_offset = offset + priv->csum_offset;
901 offset += priv->offset;
902
Pablo Neira Ayuso053d20f2016-12-14 12:15:49 +0100903 if ((priv->csum_type == NFT_PAYLOAD_CSUM_INET || priv->csum_flags) &&
Pablo Neira Ayusoc46b38d2021-10-28 22:15:00 +0200904 ((priv->base != NFT_PAYLOAD_TRANSPORT_HEADER &&
905 priv->base != NFT_PAYLOAD_INNER_HEADER) ||
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000906 skb->ip_summed != CHECKSUM_PARTIAL)) {
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000907 fsum = skb_checksum(skb, offset, priv->len, 0);
908 tsum = csum_partial(src, priv->len, 0);
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000909
Pablo Neira Ayuso053d20f2016-12-14 12:15:49 +0100910 if (priv->csum_type == NFT_PAYLOAD_CSUM_INET &&
911 nft_payload_csum_inet(skb, src, fsum, tsum, csum_offset))
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000912 goto err;
Pablo Neira Ayuso18140962016-11-24 12:04:55 +0100913
914 if (priv->csum_flags &&
915 nft_payload_l4csum_update(pkt, skb, fsum, tsum) < 0)
916 goto err;
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000917 }
918
Florian Westphal7418ee42019-05-23 15:44:09 +0200919 if (skb_ensure_writable(skb, max(offset + priv->len, 0)) ||
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000920 skb_store_bits(skb, offset, src, priv->len) < 0)
921 goto err;
922
Davide Caratti346e3202020-10-15 18:39:27 +0200923 if (priv->csum_type == NFT_PAYLOAD_CSUM_SCTP &&
924 pkt->tprot == IPPROTO_SCTP &&
925 skb->ip_summed != CHECKSUM_PARTIAL) {
Florian Westphala9e85032022-01-29 17:13:23 +0100926 if (pkt->fragoff == 0 &&
927 nft_payload_csum_sctp(skb, nft_thoff(pkt)))
Davide Caratti346e3202020-10-15 18:39:27 +0200928 goto err;
929 }
930
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000931 return;
932err:
933 regs->verdict.code = NFT_BREAK;
934}
935
936static int nft_payload_set_init(const struct nft_ctx *ctx,
937 const struct nft_expr *expr,
938 const struct nlattr * const tb[])
939{
940 struct nft_payload_set *priv = nft_expr_priv(expr);
Pablo Neira Ayuso7044ab22022-08-21 11:55:19 +0200941 u32 csum_offset, csum_type = NFT_PAYLOAD_CSUM_NONE;
942 int err;
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000943
944 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
945 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
946 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000947
948 if (tb[NFTA_PAYLOAD_CSUM_TYPE])
Pablo Neira Ayuso7044ab22022-08-21 11:55:19 +0200949 csum_type = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_TYPE]));
950 if (tb[NFTA_PAYLOAD_CSUM_OFFSET]) {
951 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_CSUM_OFFSET], U8_MAX,
952 &csum_offset);
953 if (err < 0)
954 return err;
955
956 priv->csum_offset = csum_offset;
957 }
Pablo Neira Ayuso18140962016-11-24 12:04:55 +0100958 if (tb[NFTA_PAYLOAD_CSUM_FLAGS]) {
959 u32 flags;
960
961 flags = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_FLAGS]));
962 if (flags & ~NFT_PAYLOAD_L4CSUM_PSEUDOHDR)
963 return -EINVAL;
964
965 priv->csum_flags = flags;
966 }
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000967
Pablo Neira Ayuso7044ab22022-08-21 11:55:19 +0200968 switch (csum_type) {
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000969 case NFT_PAYLOAD_CSUM_NONE:
970 case NFT_PAYLOAD_CSUM_INET:
971 break;
Davide Caratti346e3202020-10-15 18:39:27 +0200972 case NFT_PAYLOAD_CSUM_SCTP:
973 if (priv->base != NFT_PAYLOAD_TRANSPORT_HEADER)
974 return -EINVAL;
975
976 if (priv->csum_offset != offsetof(struct sctphdr, checksum))
977 return -EINVAL;
978 break;
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000979 default:
980 return -EOPNOTSUPP;
981 }
Pablo Neira Ayuso7044ab22022-08-21 11:55:19 +0200982 priv->csum_type = csum_type;
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000983
Pablo Neira Ayuso4f16d252021-01-25 17:28:18 +0100984 return nft_parse_register_load(tb[NFTA_PAYLOAD_SREG], &priv->sreg,
985 priv->len);
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000986}
987
Phil Sutter7d34aa32022-10-14 23:45:58 +0200988static int nft_payload_set_dump(struct sk_buff *skb,
989 const struct nft_expr *expr, bool reset)
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +0000990{
991 const struct nft_payload_set *priv = nft_expr_priv(expr);
992
993 if (nft_dump_register(skb, NFTA_PAYLOAD_SREG, priv->sreg) ||
994 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) ||
995 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) ||
996 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)) ||
997 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_TYPE, htonl(priv->csum_type)) ||
998 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_OFFSET,
Pablo Neira Ayuso18140962016-11-24 12:04:55 +0100999 htonl(priv->csum_offset)) ||
1000 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_FLAGS, htonl(priv->csum_flags)))
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +00001001 goto nla_put_failure;
1002 return 0;
1003
1004nla_put_failure:
1005 return -1;
1006}
1007
Pablo Neira Ayusocc003c72022-01-09 17:11:25 +01001008static bool nft_payload_set_reduce(struct nft_regs_track *track,
1009 const struct nft_expr *expr)
1010{
1011 int i;
1012
1013 for (i = 0; i < NFT_REG32_NUM; i++) {
1014 if (!track->regs[i].selector)
1015 continue;
1016
1017 if (track->regs[i].selector->ops != &nft_payload_ops &&
1018 track->regs[i].selector->ops != &nft_payload_fast_ops)
1019 continue;
1020
Pablo Neira Ayuso34cc9e52022-03-14 18:23:01 +01001021 __nft_reg_track_cancel(track, i);
Pablo Neira Ayusocc003c72022-01-09 17:11:25 +01001022 }
1023
1024 return false;
1025}
1026
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +00001027static const struct nft_expr_ops nft_payload_set_ops = {
1028 .type = &nft_payload_type,
1029 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload_set)),
1030 .eval = nft_payload_set_eval,
1031 .init = nft_payload_set_init,
1032 .dump = nft_payload_set_dump,
Pablo Neira Ayusocc003c72022-01-09 17:11:25 +01001033 .reduce = nft_payload_set_reduce,
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +00001034};
1035
Pablo Neira Ayuso0ca743a2013-10-14 00:06:06 +02001036static const struct nft_expr_ops *
1037nft_payload_select_ops(const struct nft_ctx *ctx,
1038 const struct nlattr * const tb[])
Patrick McHardyc29b72e2013-10-10 11:06:41 +02001039{
1040 enum nft_payload_bases base;
1041 unsigned int offset, len;
Pablo Neira Ayuso94254f92022-08-21 11:47:04 +02001042 int err;
Patrick McHardyc29b72e2013-10-10 11:06:41 +02001043
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +00001044 if (tb[NFTA_PAYLOAD_BASE] == NULL ||
Patrick McHardyc29b72e2013-10-10 11:06:41 +02001045 tb[NFTA_PAYLOAD_OFFSET] == NULL ||
1046 tb[NFTA_PAYLOAD_LEN] == NULL)
1047 return ERR_PTR(-EINVAL);
1048
1049 base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
1050 switch (base) {
1051 case NFT_PAYLOAD_LL_HEADER:
1052 case NFT_PAYLOAD_NETWORK_HEADER:
1053 case NFT_PAYLOAD_TRANSPORT_HEADER:
Pablo Neira Ayusoc46b38d2021-10-28 22:15:00 +02001054 case NFT_PAYLOAD_INNER_HEADER:
Patrick McHardyc29b72e2013-10-10 11:06:41 +02001055 break;
1056 default:
1057 return ERR_PTR(-EOPNOTSUPP);
1058 }
1059
Patrick McHardy7ec3f7b2015-11-24 10:00:22 +00001060 if (tb[NFTA_PAYLOAD_SREG] != NULL) {
1061 if (tb[NFTA_PAYLOAD_DREG] != NULL)
1062 return ERR_PTR(-EINVAL);
1063 return &nft_payload_set_ops;
1064 }
1065
1066 if (tb[NFTA_PAYLOAD_DREG] == NULL)
1067 return ERR_PTR(-EINVAL);
1068
Pablo Neira Ayuso94254f92022-08-21 11:47:04 +02001069 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U8_MAX, &offset);
1070 if (err < 0)
1071 return ERR_PTR(err);
1072
1073 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_LEN], U8_MAX, &len);
1074 if (err < 0)
1075 return ERR_PTR(err);
Patrick McHardyc29b72e2013-10-10 11:06:41 +02001076
Nikolay Aleksandrovf627ed92014-02-16 14:01:58 +01001077 if (len <= 4 && is_power_of_2(len) && IS_ALIGNED(offset, len) &&
Pablo Neira Ayusoc46b38d2021-10-28 22:15:00 +02001078 base != NFT_PAYLOAD_LL_HEADER && base != NFT_PAYLOAD_INNER_HEADER)
Patrick McHardyc29b72e2013-10-10 11:06:41 +02001079 return &nft_payload_fast_ops;
1080 else
1081 return &nft_payload_ops;
1082}
1083
Liping Zhang4e248772016-11-06 21:15:51 +08001084struct nft_expr_type nft_payload_type __read_mostly = {
Patrick McHardyef1f7df2013-10-10 11:41:20 +02001085 .name = "payload",
Patrick McHardyc29b72e2013-10-10 11:06:41 +02001086 .select_ops = nft_payload_select_ops,
Pablo Neira Ayuso3a073272022-10-25 13:48:15 +02001087 .inner_ops = &nft_payload_inner_ops,
Patrick McHardy96518512013-10-14 11:00:02 +02001088 .policy = nft_payload_policy,
1089 .maxattr = NFTA_PAYLOAD_MAX,
Patrick McHardyef1f7df2013-10-10 11:41:20 +02001090 .owner = THIS_MODULE,
Patrick McHardy96518512013-10-14 11:00:02 +02001091};