blob: 0ec501845cb3bb51082f8091b4e0ebb32f83bf33 [file] [log] [blame]
Thomas Gleixner457c8992019-05-19 13:08:55 +01001// SPDX-License-Identifier: GPL-2.0-only
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -07002/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Generic TIME_WAIT sockets functions
8 *
9 * From code orinally in TCP
10 */
11
Ilpo Järvinen172589c2007-08-28 15:50:33 -070012#include <linux/kernel.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090013#include <linux/slab.h>
Paul Gortmaker3a9a2312011-05-27 09:12:25 -040014#include <linux/module.h>
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070015#include <net/inet_hashtables.h>
16#include <net/inet_timewait_sock.h>
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -070017#include <net/ip.h>
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070018
Eric Dumazet13475a32009-12-02 22:31:19 +000019
Eric Dumazet2a8875e2009-12-08 20:19:53 -080020/**
Eric Dumazet2a8875e2009-12-08 20:19:53 -080021 * inet_twsk_bind_unhash - unhash a timewait socket from bind hash
22 * @tw: timewait socket
23 * @hashinfo: hashinfo pointer
24 *
25 * unhash a timewait socket from bind hash, if hashed.
26 * bind hash lock must be held by caller.
27 * Returns 1 if caller should call inet_twsk_put() after lock release.
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000028 */
Eric Dumazetfc01538f2015-07-08 14:28:29 -070029void inet_twsk_bind_unhash(struct inet_timewait_sock *tw,
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000030 struct inet_hashinfo *hashinfo)
31{
32 struct inet_bind_bucket *tb = tw->tw_tb;
33
34 if (!tb)
Eric Dumazetfc01538f2015-07-08 14:28:29 -070035 return;
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000036
37 __hlist_del(&tw->tw_bind_node);
38 tw->tw_tb = NULL;
39 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
Eric Dumazetfc01538f2015-07-08 14:28:29 -070040 __sock_put((struct sock *)tw);
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000041}
42
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070043/* Must be called with locally disabled BHs. */
Eric Dumazet789f5582015-04-12 18:51:09 -070044static void inet_twsk_kill(struct inet_timewait_sock *tw)
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070045{
Eric Dumazet789f5582015-04-12 18:51:09 -070046 struct inet_hashinfo *hashinfo = tw->tw_dr->hashinfo;
Eric Dumazet9db66bd2008-11-20 20:39:09 -080047 spinlock_t *lock = inet_ehash_lockp(hashinfo, tw->tw_hash);
Eric Dumazetfc01538f2015-07-08 14:28:29 -070048 struct inet_bind_hashbucket *bhead;
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070049
Eric Dumazet9db66bd2008-11-20 20:39:09 -080050 spin_lock(lock);
Eric Dumazetfc01538f2015-07-08 14:28:29 -070051 sk_nulls_del_node_init_rcu((struct sock *)tw);
Eric Dumazet9db66bd2008-11-20 20:39:09 -080052 spin_unlock(lock);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070053
54 /* Disassociate with bind bucket. */
Eric Dumazet04c494e2022-05-12 14:14:56 -070055 bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), tw->tw_num,
56 hashinfo->bhash_size)];
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000057
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070058 spin_lock(&bhead->lock);
Eric Dumazetfc01538f2015-07-08 14:28:29 -070059 inet_twsk_bind_unhash(tw, hashinfo);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070060 spin_unlock(&bhead->lock);
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000061
Eric Dumazetfbb82952022-01-26 10:07:14 -080062 if (refcount_dec_and_test(&tw->tw_dr->tw_refcount))
63 kfree(tw->tw_dr);
64
Eric Dumazet789f5582015-04-12 18:51:09 -070065 inet_twsk_put(tw);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070066}
67
Eric Dumazet05dbc7b2013-10-03 00:22:02 -070068void inet_twsk_free(struct inet_timewait_sock *tw)
Arnaldo Carvalho de Melo4dbc8ef2009-05-06 16:50:52 -070069{
70 struct module *owner = tw->tw_prot->owner;
71 twsk_destructor((struct sock *)tw);
72#ifdef SOCK_REFCNT_DEBUG
73 pr_debug("%s timewait_sock %p released\n", tw->tw_prot->name, tw);
74#endif
Arnaldo Carvalho de Melo4dbc8ef2009-05-06 16:50:52 -070075 kmem_cache_free(tw->tw_prot->twsk_prot->twsk_slab, tw);
76 module_put(owner);
77}
78
Pavel Emelyanov7054fb92007-12-20 15:32:54 -080079void inet_twsk_put(struct inet_timewait_sock *tw)
80{
Reshetova, Elena41c6d652017-06-30 13:08:01 +030081 if (refcount_dec_and_test(&tw->tw_refcnt))
Arnaldo Carvalho de Melo4dbc8ef2009-05-06 16:50:52 -070082 inet_twsk_free(tw);
Pavel Emelyanov7054fb92007-12-20 15:32:54 -080083}
84EXPORT_SYMBOL_GPL(inet_twsk_put);
85
Eric Dumazet05dbc7b2013-10-03 00:22:02 -070086static void inet_twsk_add_node_rcu(struct inet_timewait_sock *tw,
87 struct hlist_nulls_head *list)
88{
89 hlist_nulls_add_head_rcu(&tw->tw_node, list);
90}
91
92static void inet_twsk_add_bind_node(struct inet_timewait_sock *tw,
93 struct hlist_head *list)
94{
95 hlist_add_head(&tw->tw_bind_node, list);
96}
97
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070098/*
Eric Dumazete599ea12017-12-01 15:28:44 -080099 * Enter the time wait state. This is called with locally disabled BH.
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700100 * Essentially we whip up a timewait bucket, copy the relevant info into it
101 * from the SK, and mess with hash chains and list linkage.
102 */
Eric Dumazetec94c262017-12-11 21:25:12 -0800103void inet_twsk_hashdance(struct inet_timewait_sock *tw, struct sock *sk,
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700104 struct inet_hashinfo *hashinfo)
105{
106 const struct inet_sock *inet = inet_sk(sk);
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700107 const struct inet_connection_sock *icsk = inet_csk(sk);
Eric Dumazet81c3d542005-10-03 14:13:38 -0700108 struct inet_ehash_bucket *ehead = inet_ehash_bucket(hashinfo, sk->sk_hash);
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800109 spinlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700110 struct inet_bind_hashbucket *bhead;
111 /* Step 1: Put TW into bind hash. Original socket stays there too.
112 Note, that any socket with inet->num != 0 MUST be bound in
113 binding cache, even if it is closed.
114 */
Eric Dumazet04c494e2022-05-12 14:14:56 -0700115 bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), inet->inet_num,
116 hashinfo->bhash_size)];
Eric Dumazete599ea12017-12-01 15:28:44 -0800117 spin_lock(&bhead->lock);
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700118 tw->tw_tb = icsk->icsk_bind_hash;
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700119 WARN_ON(!icsk->icsk_bind_hash);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700120 inet_twsk_add_bind_node(tw, &tw->tw_tb->owners);
121 spin_unlock(&bhead->lock);
122
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800123 spin_lock(lock);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700124
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700125 inet_twsk_add_node_rcu(tw, &ehead->chain);
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800126
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700127 /* Step 3: Remove SK from hash chain */
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800128 if (__sk_nulls_del_node_init_rcu(sk))
129 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700130
Eric Dumazete599ea12017-12-01 15:28:44 -0800131 spin_unlock(lock);
Eric Dumazetec94c262017-12-11 21:25:12 -0800132
133 /* tw_refcnt is set to 3 because we have :
134 * - one reference for bhash chain.
135 * - one reference for ehash chain.
136 * - one reference for timer.
137 * We can use atomic_set() because prior spin_lock()/spin_unlock()
138 * committed into memory all tw fields.
139 * Also note that after this point, we lost our implicit reference
140 * so we are not allowed to use tw anymore.
141 */
142 refcount_set(&tw->tw_refcnt, 3);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700143}
Eric Dumazetec94c262017-12-11 21:25:12 -0800144EXPORT_SYMBOL_GPL(inet_twsk_hashdance);
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700145
Kees Cook1ab791d2017-10-16 17:29:41 -0700146static void tw_timer_handler(struct timer_list *t)
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700147{
Kees Cook1ab791d2017-10-16 17:29:41 -0700148 struct inet_timewait_sock *tw = from_timer(tw, t, tw_timer);
Eric Dumazet789f5582015-04-12 18:51:09 -0700149
Eric Dumazet789f5582015-04-12 18:51:09 -0700150 inet_twsk_kill(tw);
151}
152
153struct inet_timewait_sock *inet_twsk_alloc(const struct sock *sk,
154 struct inet_timewait_death_row *dr,
155 const int state)
156{
157 struct inet_timewait_sock *tw;
158
Eric Dumazetfbb82952022-01-26 10:07:14 -0800159 if (refcount_read(&dr->tw_refcount) - 1 >= dr->sysctl_max_tw_buckets)
Eric Dumazet789f5582015-04-12 18:51:09 -0700160 return NULL;
161
162 tw = kmem_cache_alloc(sk->sk_prot_creator->twsk_prot->twsk_slab,
163 GFP_ATOMIC);
Ian Morris00db4122015-04-03 09:17:27 +0100164 if (tw) {
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700165 const struct inet_sock *inet = inet_sk(sk);
166
Eric Dumazet789f5582015-04-12 18:51:09 -0700167 tw->tw_dr = dr;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700168 /* Give us an identity. */
Eric Dumazetc720c7e82009-10-15 06:30:45 +0000169 tw->tw_daddr = inet->inet_daddr;
170 tw->tw_rcv_saddr = inet->inet_rcv_saddr;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700171 tw->tw_bound_dev_if = sk->sk_bound_dev_if;
Eric Dumazet66b13d92011-10-24 03:06:21 -0400172 tw->tw_tos = inet->tos;
Eric Dumazetc720c7e82009-10-15 06:30:45 +0000173 tw->tw_num = inet->inet_num;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700174 tw->tw_state = TCP_TIME_WAIT;
175 tw->tw_substate = state;
Eric Dumazetc720c7e82009-10-15 06:30:45 +0000176 tw->tw_sport = inet->inet_sport;
177 tw->tw_dport = inet->inet_dport;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700178 tw->tw_family = sk->sk_family;
179 tw->tw_reuse = sk->sk_reuse;
Eric Dumazet3099a522018-04-07 13:42:43 -0700180 tw->tw_reuseport = sk->sk_reuseport;
Eric Dumazet81c3d542005-10-03 14:13:38 -0700181 tw->tw_hash = sk->sk_hash;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700182 tw->tw_ipv6only = 0;
KOVACS Krisztianf5715ae2008-10-01 07:30:02 -0700183 tw->tw_transparent = inet->transparent;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700184 tw->tw_prot = sk->sk_prot_creator;
Eric Dumazet33cf7c92015-03-11 18:53:14 -0700185 atomic64_set(&tw->tw_cookie, atomic64_read(&sk->sk_cookie));
Eric W. Biedermanefd7ef12015-03-11 23:04:08 -0500186 twsk_net_set(tw, sock_net(sk));
Kees Cook1ab791d2017-10-16 17:29:41 -0700187 timer_setup(&tw->tw_timer, tw_timer_handler, TIMER_PINNED);
Eric Dumazet47e1c322009-12-03 00:49:01 +0000188 /*
189 * Because we use RCU lookups, we should not set tw_refcnt
190 * to a non null value before everything is setup for this
191 * timewait socket.
192 */
Reshetova, Elena41c6d652017-06-30 13:08:01 +0300193 refcount_set(&tw->tw_refcnt, 0);
Eric Dumazet789f5582015-04-12 18:51:09 -0700194
Arnaldo Carvalho de Meloeeb2b852005-10-10 21:25:23 -0700195 __module_get(tw->tw_prot->owner);
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700196 }
197
198 return tw;
199}
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700200EXPORT_SYMBOL_GPL(inet_twsk_alloc);
201
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700202/* These are always called from BH context. See callers in
203 * tcp_input.c to verify this.
204 */
205
Eric Dumazetdbe7faa2015-07-08 14:28:30 -0700206/* This is for handling early-kills of TIME_WAIT sockets.
207 * Warning : consume reference.
208 * Caller should not access tw anymore.
209 */
210void inet_twsk_deschedule_put(struct inet_timewait_sock *tw)
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700211{
Eric Dumazet789f5582015-04-12 18:51:09 -0700212 if (del_timer_sync(&tw->tw_timer))
213 inet_twsk_kill(tw);
Eric Dumazetdbe7faa2015-07-08 14:28:30 -0700214 inet_twsk_put(tw);
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700215}
Eric Dumazetdbe7faa2015-07-08 14:28:30 -0700216EXPORT_SYMBOL(inet_twsk_deschedule_put);
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700217
Eric Dumazeted2e9232015-09-19 09:08:34 -0700218void __inet_twsk_schedule(struct inet_timewait_sock *tw, int timeo, bool rearm)
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700219{
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700220 /* timeout := RTO * 3.5
221 *
222 * 3.5 = 1+2+0.5 to wait for two retransmits.
223 *
224 * RATIONALE: if FIN arrived and we entered TIME-WAIT state,
225 * our ACK acking that FIN can be lost. If N subsequent retransmitted
226 * FINs (or previous seqments) are lost (probability of such event
227 * is p^(N+1), where p is probability to lose single packet and
228 * time to detect the loss is about RTO*(2^N - 1) with exponential
229 * backoff). Normal timewait length is calculated so, that we
230 * waited at least for one retransmitted FIN (maximal RTO is 120sec).
231 * [ BTW Linux. following BSD, violates this requirement waiting
232 * only for 60sec, we should wait at least for 240 secs.
233 * Well, 240 consumes too much of resources 8)
234 * ]
235 * This interval is not reduced to catch old duplicate and
236 * responces to our wandering segments living for two MSLs.
237 * However, if we use PAWS to detect
238 * old duplicates, we can reduce the interval to bounds required
239 * by RTO, rather than MSL. So, if peer understands PAWS, we
240 * kill tw bucket after 3.5*RTO (it is important that this number
241 * is greater than TS tick!) and detect old duplicates with help
242 * of PAWS.
243 */
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700244
Eric Dumazeted2e9232015-09-19 09:08:34 -0700245 if (!rearm) {
Eric Dumazet27dd35e2022-01-24 12:24:53 -0800246 bool kill = timeo <= 4*HZ;
247
248 __NET_INC_STATS(twsk_net(tw), kill ? LINUX_MIB_TIMEWAITKILLED :
249 LINUX_MIB_TIMEWAITED);
Thomas Gleixnerf3438bc2016-07-04 09:50:23 +0000250 BUG_ON(mod_timer(&tw->tw_timer, jiffies + timeo));
Eric Dumazetfbb82952022-01-26 10:07:14 -0800251 refcount_inc(&tw->tw_dr->tw_refcount);
Eric Dumazeted2e9232015-09-19 09:08:34 -0700252 } else {
253 mod_timer_pending(&tw->tw_timer, jiffies + timeo);
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700254 }
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700255}
Eric Dumazeted2e9232015-09-19 09:08:34 -0700256EXPORT_SYMBOL_GPL(__inet_twsk_schedule);
Eric Dumazet04c494e2022-05-12 14:14:56 -0700257
258void inet_twsk_purge(struct inet_hashinfo *hashinfo, int family)
259{
260 struct inet_timewait_sock *tw;
261 struct sock *sk;
262 struct hlist_nulls_node *node;
263 unsigned int slot;
264
265 for (slot = 0; slot <= hashinfo->ehash_mask; slot++) {
266 struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
267restart_rcu:
268 cond_resched();
269 rcu_read_lock();
270restart:
271 sk_nulls_for_each_rcu(sk, node, &head->chain) {
272 if (sk->sk_state != TCP_TIME_WAIT)
273 continue;
274 tw = inet_twsk(sk);
275 if ((tw->tw_family != family) ||
276 refcount_read(&twsk_net(tw)->ns.count))
277 continue;
278
279 if (unlikely(!refcount_inc_not_zero(&tw->tw_refcnt)))
280 continue;
281
282 if (unlikely((tw->tw_family != family) ||
283 refcount_read(&twsk_net(tw)->ns.count))) {
284 inet_twsk_put(tw);
285 goto restart;
286 }
287
288 rcu_read_unlock();
289 local_bh_disable();
290 inet_twsk_deschedule_put(tw);
291 local_bh_enable();
292 goto restart_rcu;
293 }
294 /* If the nulls value we got at the end of this lookup is
295 * not the expected one, we must restart lookup.
296 * We probably met an item that was moved to another chain.
297 */
298 if (get_nulls_value(node) != slot)
299 goto restart;
300 rcu_read_unlock();
301 }
302}
303EXPORT_SYMBOL_GPL(inet_twsk_purge);