blob: 0a3f88f08727f1f1217560407ff539c8a8c17496 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * net/sched/gen_estimator.c Simple rate estimator.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080010 * Eric Dumazet <edumazet@google.com>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 *
12 * Changes:
13 * Jamal Hadi Salim - moved it to net/core and reshulfed
14 * names to make it usable in general net subsystem.
15 */
16
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080017#include <linux/uaccess.h>
Jiri Slaby1977f032007-10-18 23:40:25 -070018#include <linux/bitops.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/module.h>
20#include <linux/types.h>
21#include <linux/kernel.h>
22#include <linux/jiffies.h>
23#include <linux/string.h>
24#include <linux/mm.h>
25#include <linux/socket.h>
26#include <linux/sockios.h>
27#include <linux/in.h>
28#include <linux/errno.h>
29#include <linux/interrupt.h>
30#include <linux/netdevice.h>
31#include <linux/skbuff.h>
32#include <linux/rtnetlink.h>
33#include <linux/init.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090034#include <linux/slab.h>
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080035#include <linux/seqlock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036#include <net/sock.h>
37#include <net/gen_stats.h>
38
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080039/* This code is NOT intended to be used for statistics collection,
40 * its purpose is to provide a base for statistical multiplexing
41 * for controlled load service.
42 * If you need only statistics, run a user level daemon which
43 * periodically reads byte counters.
Linus Torvalds1da177e2005-04-16 15:20:36 -070044 */
45
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080046struct net_rate_estimator {
Eric Dumazetc1a8f1f2009-08-16 09:36:49 +000047 struct gnet_stats_basic_packed *bstats;
Linus Torvalds1da177e2005-04-16 15:20:36 -070048 spinlock_t *stats_lock;
Eric Dumazetedb09eb2016-06-06 09:37:16 -070049 seqcount_t *running;
John Fastabend22e0f8b2014-09-28 11:52:56 -070050 struct gnet_stats_basic_cpu __percpu *cpu_bstats;
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080051 u8 ewma_log;
52 u8 intvl_log; /* period : (250ms << intvl_log) */
53
54 seqcount_t seq;
55 u32 last_packets;
56 u64 last_bytes;
57
58 u64 avpps;
59 u64 avbps;
60
61 unsigned long next_jiffies;
62 struct timer_list timer;
63 struct rcu_head rcu;
Linus Torvalds1da177e2005-04-16 15:20:36 -070064};
65
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080066static void est_fetch_counters(struct net_rate_estimator *e,
67 struct gnet_stats_basic_packed *b)
68{
69 if (e->stats_lock)
70 spin_lock(e->stats_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080072 __gnet_stats_copy_basic(e->running, b, e->cpu_bstats, e->bstats);
Linus Torvalds1da177e2005-04-16 15:20:36 -070073
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080074 if (e->stats_lock)
75 spin_unlock(e->stats_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070076
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080077}
Jarek Poplawski4db0acf32008-11-24 15:48:05 -080078
Kees Cooke99e88a2017-10-16 14:43:17 -070079static void est_timer(struct timer_list *t)
Linus Torvalds1da177e2005-04-16 15:20:36 -070080{
Kees Cooke99e88a2017-10-16 14:43:17 -070081 struct net_rate_estimator *est = from_timer(est, t, timer);
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080082 struct gnet_stats_basic_packed b;
83 u64 rate, brate;
Linus Torvalds1da177e2005-04-16 15:20:36 -070084
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080085 est_fetch_counters(est, &b);
Eric Dumazetca558e12017-09-13 11:16:45 -070086 brate = (b.bytes - est->last_bytes) << (10 - est->ewma_log - est->intvl_log);
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080087 brate -= (est->avbps >> est->ewma_log);
Linus Torvalds1da177e2005-04-16 15:20:36 -070088
Eric Dumazetca558e12017-09-13 11:16:45 -070089 rate = (u64)(b.packets - est->last_packets) << (10 - est->ewma_log - est->intvl_log);
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080090 rate -= (est->avpps >> est->ewma_log);
Ranko Zivojnovic0929c2d2007-07-16 18:28:32 -070091
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080092 write_seqcount_begin(&est->seq);
93 est->avbps += brate;
94 est->avpps += rate;
95 write_seqcount_end(&est->seq);
John Fastabend22e0f8b2014-09-28 11:52:56 -070096
Eric Dumazet1c0d32f2016-12-04 09:48:16 -080097 est->last_bytes = b.bytes;
98 est->last_packets = b.packets;
Linus Torvalds1da177e2005-04-16 15:20:36 -070099
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800100 est->next_jiffies += ((HZ/4) << est->intvl_log);
101
102 if (unlikely(time_after_eq(jiffies, est->next_jiffies))) {
103 /* Ouch... timer was delayed. */
104 est->next_jiffies = jiffies + 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105 }
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800106 mod_timer(&est->timer, est->next_jiffies);
Jarek Poplawski4db0acf32008-11-24 15:48:05 -0800107}
108
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109/**
110 * gen_new_estimator - create a new rate estimator
111 * @bstats: basic statistics
Luis de Bethencourte9fc2f02016-03-19 21:31:38 +0000112 * @cpu_bstats: bstats per cpu
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 * @rate_est: rate estimator statistics
114 * @stats_lock: statistics lock
Eric Dumazetedb09eb2016-06-06 09:37:16 -0700115 * @running: qdisc running seqcount
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116 * @opt: rate estimator configuration TLV
117 *
118 * Creates a new rate estimator with &bstats as source and &rate_est
119 * as destination. A new timer with the interval specified in the
120 * configuration TLV is created. Upon each interval, the latest statistics
121 * will be read from &bstats and the estimated rate will be stored in
Masanari Iidae793c0f2014-09-04 23:44:36 +0900122 * &rate_est with the statistics lock grabbed during this period.
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900123 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124 * Returns 0 on success or a negative error code.
Ranko Zivojnovic0929c2d2007-07-16 18:28:32 -0700125 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126 */
Eric Dumazetc1a8f1f2009-08-16 09:36:49 +0000127int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
John Fastabend22e0f8b2014-09-28 11:52:56 -0700128 struct gnet_stats_basic_cpu __percpu *cpu_bstats,
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800129 struct net_rate_estimator __rcu **rate_est,
Ranko Zivojnovic0929c2d2007-07-16 18:28:32 -0700130 spinlock_t *stats_lock,
Eric Dumazetedb09eb2016-06-06 09:37:16 -0700131 seqcount_t *running,
Patrick McHardy1e904742008-01-22 22:11:17 -0800132 struct nlattr *opt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133{
Patrick McHardy1e904742008-01-22 22:11:17 -0800134 struct gnet_estimator *parm = nla_data(opt);
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800135 struct net_rate_estimator *old, *est;
136 struct gnet_stats_basic_packed b;
137 int intvl_log;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138
Patrick McHardy1e904742008-01-22 22:11:17 -0800139 if (nla_len(opt) < sizeof(*parm))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140 return -EINVAL;
141
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800142 /* allowed timer periods are :
143 * -2 : 250ms, -1 : 500ms, 0 : 1 sec
144 * 1 : 2 sec, 2 : 4 sec, 3 : 8 sec
145 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 if (parm->interval < -2 || parm->interval > 3)
147 return -EINVAL;
148
Andrew Morton77d04bd2006-04-07 14:52:59 -0700149 est = kzalloc(sizeof(*est), GFP_KERNEL);
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800150 if (!est)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151 return -ENOBUFS;
152
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800153 seqcount_init(&est->seq);
154 intvl_log = parm->interval + 2;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155 est->bstats = bstats;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 est->stats_lock = stats_lock;
Eric Dumazetedb09eb2016-06-06 09:37:16 -0700157 est->running = running;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158 est->ewma_log = parm->ewma_log;
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800159 est->intvl_log = intvl_log;
John Fastabend22e0f8b2014-09-28 11:52:56 -0700160 est->cpu_bstats = cpu_bstats;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161
Eric Dumazet40ca54e2018-01-27 10:58:43 -0800162 if (stats_lock)
163 local_bh_disable();
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800164 est_fetch_counters(est, &b);
Eric Dumazet40ca54e2018-01-27 10:58:43 -0800165 if (stats_lock)
166 local_bh_enable();
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800167 est->last_bytes = b.bytes;
168 est->last_packets = b.packets;
169 old = rcu_dereference_protected(*rate_est, 1);
170 if (old) {
171 del_timer_sync(&old->timer);
172 est->avbps = old->avbps;
173 est->avpps = old->avpps;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174 }
Ranko Zivojnovic0929c2d2007-07-16 18:28:32 -0700175
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800176 est->next_jiffies = jiffies + ((HZ/4) << intvl_log);
Kees Cooke99e88a2017-10-16 14:43:17 -0700177 timer_setup(&est->timer, est_timer, 0);
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800178 mod_timer(&est->timer, est->next_jiffies);
Jarek Poplawski4db0acf32008-11-24 15:48:05 -0800179
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800180 rcu_assign_pointer(*rate_est, est);
181 if (old)
182 kfree_rcu(old, rcu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183 return 0;
184}
Stephen Hemmingerc1b56872008-11-25 21:14:06 -0800185EXPORT_SYMBOL(gen_new_estimator);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186
187/**
188 * gen_kill_estimator - remove a rate estimator
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800189 * @rate_est: rate estimator
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 *
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800191 * Removes the rate estimator.
Ranko Zivojnovic0929c2d2007-07-16 18:28:32 -0700192 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193 */
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800194void gen_kill_estimator(struct net_rate_estimator __rcu **rate_est)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195{
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800196 struct net_rate_estimator *est;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800198 est = xchg((__force struct net_rate_estimator **)rate_est, NULL);
199 if (est) {
200 del_timer_sync(&est->timer);
201 kfree_rcu(est, rcu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 }
203}
Stephen Hemmingerc1b56872008-11-25 21:14:06 -0800204EXPORT_SYMBOL(gen_kill_estimator);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205
206/**
Jarek Poplawski96750162008-01-21 02:36:02 -0800207 * gen_replace_estimator - replace rate estimator configuration
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208 * @bstats: basic statistics
Luis de Bethencourte9fc2f02016-03-19 21:31:38 +0000209 * @cpu_bstats: bstats per cpu
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210 * @rate_est: rate estimator statistics
211 * @stats_lock: statistics lock
Eric Dumazetedb09eb2016-06-06 09:37:16 -0700212 * @running: qdisc running seqcount (might be NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213 * @opt: rate estimator configuration TLV
214 *
215 * Replaces the configuration of a rate estimator by calling
216 * gen_kill_estimator() and gen_new_estimator().
YOSHIFUJI Hideaki4ec93ed2007-02-09 23:24:36 +0900217 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218 * Returns 0 on success or a negative error code.
219 */
Eric Dumazetc1a8f1f2009-08-16 09:36:49 +0000220int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
John Fastabend22e0f8b2014-09-28 11:52:56 -0700221 struct gnet_stats_basic_cpu __percpu *cpu_bstats,
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800222 struct net_rate_estimator __rcu **rate_est,
Eric Dumazetedb09eb2016-06-06 09:37:16 -0700223 spinlock_t *stats_lock,
224 seqcount_t *running, struct nlattr *opt)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225{
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800226 return gen_new_estimator(bstats, cpu_bstats, rate_est,
227 stats_lock, running, opt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229EXPORT_SYMBOL(gen_replace_estimator);
Stephen Hemmingerc1b56872008-11-25 21:14:06 -0800230
231/**
232 * gen_estimator_active - test if estimator is currently in use
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800233 * @rate_est: rate estimator
Stephen Hemmingerc1b56872008-11-25 21:14:06 -0800234 *
Jarek Poplawski244e6c22008-11-26 15:24:32 -0800235 * Returns true if estimator is active, and false if not.
Stephen Hemmingerc1b56872008-11-25 21:14:06 -0800236 */
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800237bool gen_estimator_active(struct net_rate_estimator __rcu **rate_est)
Stephen Hemmingerc1b56872008-11-25 21:14:06 -0800238{
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800239 return !!rcu_access_pointer(*rate_est);
Stephen Hemmingerc1b56872008-11-25 21:14:06 -0800240}
241EXPORT_SYMBOL(gen_estimator_active);
Eric Dumazet1c0d32f2016-12-04 09:48:16 -0800242
243bool gen_estimator_read(struct net_rate_estimator __rcu **rate_est,
244 struct gnet_stats_rate_est64 *sample)
245{
246 struct net_rate_estimator *est;
247 unsigned seq;
248
249 rcu_read_lock();
250 est = rcu_dereference(*rate_est);
251 if (!est) {
252 rcu_read_unlock();
253 return false;
254 }
255
256 do {
257 seq = read_seqcount_begin(&est->seq);
258 sample->bps = est->avbps >> 8;
259 sample->pps = est->avpps >> 8;
260 } while (read_seqcount_retry(&est->seq, seq));
261
262 rcu_read_unlock();
263 return true;
264}
265EXPORT_SYMBOL(gen_estimator_read);