blob: 47ccc343c9fb0a995b817d938f25a779dd8221c0 [file] [log] [blame]
Thomas Gleixner457c8992019-05-19 13:08:55 +01001// SPDX-License-Identifier: GPL-2.0-only
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -07002/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Generic TIME_WAIT sockets functions
8 *
9 * From code orinally in TCP
10 */
11
Ilpo Järvinen172589c2007-08-28 15:50:33 -070012#include <linux/kernel.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090013#include <linux/slab.h>
Paul Gortmaker3a9a2312011-05-27 09:12:25 -040014#include <linux/module.h>
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070015#include <net/inet_hashtables.h>
16#include <net/inet_timewait_sock.h>
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -070017#include <net/ip.h>
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070018
Eric Dumazet13475a32009-12-02 22:31:19 +000019
Eric Dumazet2a8875e2009-12-08 20:19:53 -080020/**
Eric Dumazet2a8875e2009-12-08 20:19:53 -080021 * inet_twsk_bind_unhash - unhash a timewait socket from bind hash
22 * @tw: timewait socket
23 * @hashinfo: hashinfo pointer
24 *
25 * unhash a timewait socket from bind hash, if hashed.
26 * bind hash lock must be held by caller.
27 * Returns 1 if caller should call inet_twsk_put() after lock release.
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000028 */
Eric Dumazetfc01538f2015-07-08 14:28:29 -070029void inet_twsk_bind_unhash(struct inet_timewait_sock *tw,
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000030 struct inet_hashinfo *hashinfo)
31{
32 struct inet_bind_bucket *tb = tw->tw_tb;
33
34 if (!tb)
Eric Dumazetfc01538f2015-07-08 14:28:29 -070035 return;
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000036
37 __hlist_del(&tw->tw_bind_node);
38 tw->tw_tb = NULL;
39 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
Eric Dumazetfc01538f2015-07-08 14:28:29 -070040 __sock_put((struct sock *)tw);
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000041}
42
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070043/* Must be called with locally disabled BHs. */
Eric Dumazet789f5582015-04-12 18:51:09 -070044static void inet_twsk_kill(struct inet_timewait_sock *tw)
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070045{
Eric Dumazet789f5582015-04-12 18:51:09 -070046 struct inet_hashinfo *hashinfo = tw->tw_dr->hashinfo;
Eric Dumazet9db66bd2008-11-20 20:39:09 -080047 spinlock_t *lock = inet_ehash_lockp(hashinfo, tw->tw_hash);
Eric Dumazetfc01538f2015-07-08 14:28:29 -070048 struct inet_bind_hashbucket *bhead;
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070049
Eric Dumazet9db66bd2008-11-20 20:39:09 -080050 spin_lock(lock);
Eric Dumazetfc01538f2015-07-08 14:28:29 -070051 sk_nulls_del_node_init_rcu((struct sock *)tw);
Eric Dumazet9db66bd2008-11-20 20:39:09 -080052 spin_unlock(lock);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070053
54 /* Disassociate with bind bucket. */
Eric Dumazet04c494e2022-05-12 14:14:56 -070055 bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), tw->tw_num,
56 hashinfo->bhash_size)];
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000057
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070058 spin_lock(&bhead->lock);
Eric Dumazetfc01538f2015-07-08 14:28:29 -070059 inet_twsk_bind_unhash(tw, hashinfo);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070060 spin_unlock(&bhead->lock);
Eric Dumazet3cdaeda2009-12-04 03:47:42 +000061
Eric Dumazetfbb82952022-01-26 10:07:14 -080062 if (refcount_dec_and_test(&tw->tw_dr->tw_refcount))
63 kfree(tw->tw_dr);
64
Eric Dumazet789f5582015-04-12 18:51:09 -070065 inet_twsk_put(tw);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070066}
67
Eric Dumazet05dbc7b2013-10-03 00:22:02 -070068void inet_twsk_free(struct inet_timewait_sock *tw)
Arnaldo Carvalho de Melo4dbc8ef2009-05-06 16:50:52 -070069{
70 struct module *owner = tw->tw_prot->owner;
71 twsk_destructor((struct sock *)tw);
72#ifdef SOCK_REFCNT_DEBUG
73 pr_debug("%s timewait_sock %p released\n", tw->tw_prot->name, tw);
74#endif
Arnaldo Carvalho de Melo4dbc8ef2009-05-06 16:50:52 -070075 kmem_cache_free(tw->tw_prot->twsk_prot->twsk_slab, tw);
76 module_put(owner);
77}
78
Pavel Emelyanov7054fb92007-12-20 15:32:54 -080079void inet_twsk_put(struct inet_timewait_sock *tw)
80{
Reshetova, Elena41c6d652017-06-30 13:08:01 +030081 if (refcount_dec_and_test(&tw->tw_refcnt))
Arnaldo Carvalho de Melo4dbc8ef2009-05-06 16:50:52 -070082 inet_twsk_free(tw);
Pavel Emelyanov7054fb92007-12-20 15:32:54 -080083}
84EXPORT_SYMBOL_GPL(inet_twsk_put);
85
Eric Dumazet05dbc7b2013-10-03 00:22:02 -070086static void inet_twsk_add_node_rcu(struct inet_timewait_sock *tw,
87 struct hlist_nulls_head *list)
88{
89 hlist_nulls_add_head_rcu(&tw->tw_node, list);
90}
91
92static void inet_twsk_add_bind_node(struct inet_timewait_sock *tw,
93 struct hlist_head *list)
94{
95 hlist_add_head(&tw->tw_bind_node, list);
96}
97
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -070098/*
Eric Dumazete599ea12017-12-01 15:28:44 -080099 * Enter the time wait state. This is called with locally disabled BH.
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700100 * Essentially we whip up a timewait bucket, copy the relevant info into it
101 * from the SK, and mess with hash chains and list linkage.
102 */
Eric Dumazetec94c262017-12-11 21:25:12 -0800103void inet_twsk_hashdance(struct inet_timewait_sock *tw, struct sock *sk,
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700104 struct inet_hashinfo *hashinfo)
105{
106 const struct inet_sock *inet = inet_sk(sk);
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700107 const struct inet_connection_sock *icsk = inet_csk(sk);
Eric Dumazet81c3d542005-10-03 14:13:38 -0700108 struct inet_ehash_bucket *ehead = inet_ehash_bucket(hashinfo, sk->sk_hash);
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800109 spinlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700110 struct inet_bind_hashbucket *bhead;
111 /* Step 1: Put TW into bind hash. Original socket stays there too.
112 Note, that any socket with inet->num != 0 MUST be bound in
113 binding cache, even if it is closed.
114 */
Eric Dumazet04c494e2022-05-12 14:14:56 -0700115 bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), inet->inet_num,
116 hashinfo->bhash_size)];
Eric Dumazete599ea12017-12-01 15:28:44 -0800117 spin_lock(&bhead->lock);
Arnaldo Carvalho de Melo463c84b2005-08-09 20:10:42 -0700118 tw->tw_tb = icsk->icsk_bind_hash;
Ilpo Järvinen547b7922008-07-25 21:43:18 -0700119 WARN_ON(!icsk->icsk_bind_hash);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700120 inet_twsk_add_bind_node(tw, &tw->tw_tb->owners);
121 spin_unlock(&bhead->lock);
122
Eric Dumazet9db66bd2008-11-20 20:39:09 -0800123 spin_lock(lock);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700124
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700125 inet_twsk_add_node_rcu(tw, &ehead->chain);
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800126
Eric Dumazet05dbc7b2013-10-03 00:22:02 -0700127 /* Step 3: Remove SK from hash chain */
Eric Dumazet3ab5aee2008-11-16 19:40:17 -0800128 if (__sk_nulls_del_node_init_rcu(sk))
129 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700130
Eric Dumazete599ea12017-12-01 15:28:44 -0800131 spin_unlock(lock);
Eric Dumazetec94c262017-12-11 21:25:12 -0800132
133 /* tw_refcnt is set to 3 because we have :
134 * - one reference for bhash chain.
135 * - one reference for ehash chain.
136 * - one reference for timer.
137 * We can use atomic_set() because prior spin_lock()/spin_unlock()
138 * committed into memory all tw fields.
139 * Also note that after this point, we lost our implicit reference
140 * so we are not allowed to use tw anymore.
141 */
142 refcount_set(&tw->tw_refcnt, 3);
Arnaldo Carvalho de Meloe48c4142005-08-09 20:09:46 -0700143}
Eric Dumazetec94c262017-12-11 21:25:12 -0800144EXPORT_SYMBOL_GPL(inet_twsk_hashdance);
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700145
Kees Cook1ab791d2017-10-16 17:29:41 -0700146static void tw_timer_handler(struct timer_list *t)
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700147{
Kees Cook1ab791d2017-10-16 17:29:41 -0700148 struct inet_timewait_sock *tw = from_timer(tw, t, tw_timer);
Eric Dumazet789f5582015-04-12 18:51:09 -0700149
Eric Dumazet789f5582015-04-12 18:51:09 -0700150 inet_twsk_kill(tw);
151}
152
153struct inet_timewait_sock *inet_twsk_alloc(const struct sock *sk,
154 struct inet_timewait_death_row *dr,
155 const int state)
156{
157 struct inet_timewait_sock *tw;
158
Kuniyuki Iwashima6f605b52022-07-11 17:15:21 -0700159 if (refcount_read(&dr->tw_refcount) - 1 >=
160 READ_ONCE(dr->sysctl_max_tw_buckets))
Eric Dumazet789f5582015-04-12 18:51:09 -0700161 return NULL;
162
163 tw = kmem_cache_alloc(sk->sk_prot_creator->twsk_prot->twsk_slab,
164 GFP_ATOMIC);
Ian Morris00db4122015-04-03 09:17:27 +0100165 if (tw) {
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700166 const struct inet_sock *inet = inet_sk(sk);
167
Eric Dumazet789f5582015-04-12 18:51:09 -0700168 tw->tw_dr = dr;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700169 /* Give us an identity. */
Eric Dumazetc720c7e82009-10-15 06:30:45 +0000170 tw->tw_daddr = inet->inet_daddr;
171 tw->tw_rcv_saddr = inet->inet_rcv_saddr;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700172 tw->tw_bound_dev_if = sk->sk_bound_dev_if;
Eric Dumazet66b13d92011-10-24 03:06:21 -0400173 tw->tw_tos = inet->tos;
Eric Dumazetc720c7e82009-10-15 06:30:45 +0000174 tw->tw_num = inet->inet_num;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700175 tw->tw_state = TCP_TIME_WAIT;
176 tw->tw_substate = state;
Eric Dumazetc720c7e82009-10-15 06:30:45 +0000177 tw->tw_sport = inet->inet_sport;
178 tw->tw_dport = inet->inet_dport;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700179 tw->tw_family = sk->sk_family;
180 tw->tw_reuse = sk->sk_reuse;
Eric Dumazet3099a522018-04-07 13:42:43 -0700181 tw->tw_reuseport = sk->sk_reuseport;
Eric Dumazet81c3d542005-10-03 14:13:38 -0700182 tw->tw_hash = sk->sk_hash;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700183 tw->tw_ipv6only = 0;
KOVACS Krisztianf5715ae2008-10-01 07:30:02 -0700184 tw->tw_transparent = inet->transparent;
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700185 tw->tw_prot = sk->sk_prot_creator;
Eric Dumazet33cf7c92015-03-11 18:53:14 -0700186 atomic64_set(&tw->tw_cookie, atomic64_read(&sk->sk_cookie));
Eric W. Biedermanefd7ef12015-03-11 23:04:08 -0500187 twsk_net_set(tw, sock_net(sk));
Kees Cook1ab791d2017-10-16 17:29:41 -0700188 timer_setup(&tw->tw_timer, tw_timer_handler, TIMER_PINNED);
Eric Dumazet47e1c322009-12-03 00:49:01 +0000189 /*
190 * Because we use RCU lookups, we should not set tw_refcnt
191 * to a non null value before everything is setup for this
192 * timewait socket.
193 */
Reshetova, Elena41c6d652017-06-30 13:08:01 +0300194 refcount_set(&tw->tw_refcnt, 0);
Eric Dumazet789f5582015-04-12 18:51:09 -0700195
Arnaldo Carvalho de Meloeeb2b852005-10-10 21:25:23 -0700196 __module_get(tw->tw_prot->owner);
Arnaldo Carvalho de Meloc6762702005-08-09 20:09:59 -0700197 }
198
199 return tw;
200}
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700201EXPORT_SYMBOL_GPL(inet_twsk_alloc);
202
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700203/* These are always called from BH context. See callers in
204 * tcp_input.c to verify this.
205 */
206
Eric Dumazetdbe7faa2015-07-08 14:28:30 -0700207/* This is for handling early-kills of TIME_WAIT sockets.
208 * Warning : consume reference.
209 * Caller should not access tw anymore.
210 */
211void inet_twsk_deschedule_put(struct inet_timewait_sock *tw)
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700212{
Eric Dumazet789f5582015-04-12 18:51:09 -0700213 if (del_timer_sync(&tw->tw_timer))
214 inet_twsk_kill(tw);
Eric Dumazetdbe7faa2015-07-08 14:28:30 -0700215 inet_twsk_put(tw);
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700216}
Eric Dumazetdbe7faa2015-07-08 14:28:30 -0700217EXPORT_SYMBOL(inet_twsk_deschedule_put);
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700218
Eric Dumazeted2e9232015-09-19 09:08:34 -0700219void __inet_twsk_schedule(struct inet_timewait_sock *tw, int timeo, bool rearm)
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700220{
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700221 /* timeout := RTO * 3.5
222 *
223 * 3.5 = 1+2+0.5 to wait for two retransmits.
224 *
225 * RATIONALE: if FIN arrived and we entered TIME-WAIT state,
226 * our ACK acking that FIN can be lost. If N subsequent retransmitted
227 * FINs (or previous seqments) are lost (probability of such event
228 * is p^(N+1), where p is probability to lose single packet and
229 * time to detect the loss is about RTO*(2^N - 1) with exponential
230 * backoff). Normal timewait length is calculated so, that we
231 * waited at least for one retransmitted FIN (maximal RTO is 120sec).
232 * [ BTW Linux. following BSD, violates this requirement waiting
233 * only for 60sec, we should wait at least for 240 secs.
234 * Well, 240 consumes too much of resources 8)
235 * ]
236 * This interval is not reduced to catch old duplicate and
237 * responces to our wandering segments living for two MSLs.
238 * However, if we use PAWS to detect
239 * old duplicates, we can reduce the interval to bounds required
240 * by RTO, rather than MSL. So, if peer understands PAWS, we
241 * kill tw bucket after 3.5*RTO (it is important that this number
242 * is greater than TS tick!) and detect old duplicates with help
243 * of PAWS.
244 */
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700245
Eric Dumazeted2e9232015-09-19 09:08:34 -0700246 if (!rearm) {
Eric Dumazet27dd35e2022-01-24 12:24:53 -0800247 bool kill = timeo <= 4*HZ;
248
249 __NET_INC_STATS(twsk_net(tw), kill ? LINUX_MIB_TIMEWAITKILLED :
250 LINUX_MIB_TIMEWAITED);
Thomas Gleixnerf3438bc2016-07-04 09:50:23 +0000251 BUG_ON(mod_timer(&tw->tw_timer, jiffies + timeo));
Eric Dumazetfbb82952022-01-26 10:07:14 -0800252 refcount_inc(&tw->tw_dr->tw_refcount);
Eric Dumazeted2e9232015-09-19 09:08:34 -0700253 } else {
254 mod_timer_pending(&tw->tw_timer, jiffies + timeo);
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700255 }
Arnaldo Carvalho de Melo696ab2d2005-08-09 20:45:03 -0700256}
Eric Dumazeted2e9232015-09-19 09:08:34 -0700257EXPORT_SYMBOL_GPL(__inet_twsk_schedule);
Eric Dumazet04c494e2022-05-12 14:14:56 -0700258
259void inet_twsk_purge(struct inet_hashinfo *hashinfo, int family)
260{
261 struct inet_timewait_sock *tw;
262 struct sock *sk;
263 struct hlist_nulls_node *node;
264 unsigned int slot;
265
266 for (slot = 0; slot <= hashinfo->ehash_mask; slot++) {
267 struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
268restart_rcu:
269 cond_resched();
270 rcu_read_lock();
271restart:
272 sk_nulls_for_each_rcu(sk, node, &head->chain) {
273 if (sk->sk_state != TCP_TIME_WAIT)
274 continue;
275 tw = inet_twsk(sk);
276 if ((tw->tw_family != family) ||
277 refcount_read(&twsk_net(tw)->ns.count))
278 continue;
279
280 if (unlikely(!refcount_inc_not_zero(&tw->tw_refcnt)))
281 continue;
282
283 if (unlikely((tw->tw_family != family) ||
284 refcount_read(&twsk_net(tw)->ns.count))) {
285 inet_twsk_put(tw);
286 goto restart;
287 }
288
289 rcu_read_unlock();
290 local_bh_disable();
291 inet_twsk_deschedule_put(tw);
292 local_bh_enable();
293 goto restart_rcu;
294 }
295 /* If the nulls value we got at the end of this lookup is
296 * not the expected one, we must restart lookup.
297 * We probably met an item that was moved to another chain.
298 */
299 if (get_nulls_value(node) != slot)
300 goto restart;
301 rcu_read_unlock();
302 }
303}
304EXPORT_SYMBOL_GPL(inet_twsk_purge);