Thomas Gleixner | 2874c5f | 2019-05-27 08:55:01 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | /* |
| 3 | * net/sched/sch_tbf.c Token Bucket Filter queue. |
| 4 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5 | * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> |
| 6 | * Dmitry Torokhov <dtor@mail.ru> - allow attaching inner qdiscs - |
| 7 | * original idea by Martin Devera |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8 | */ |
| 9 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10 | #include <linux/module.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 11 | #include <linux/types.h> |
| 12 | #include <linux/kernel.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 13 | #include <linux/string.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 14 | #include <linux/errno.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 15 | #include <linux/skbuff.h> |
Patrick McHardy | 0ba4805 | 2007-07-02 22:49:07 -0700 | [diff] [blame] | 16 | #include <net/netlink.h> |
Jiri Pirko | b757c93 | 2013-02-12 00:12:05 +0000 | [diff] [blame] | 17 | #include <net/sch_generic.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 18 | #include <net/pkt_sched.h> |
| 19 | |
| 20 | |
| 21 | /* Simple Token Bucket Filter. |
| 22 | ======================================= |
| 23 | |
| 24 | SOURCE. |
| 25 | ------- |
| 26 | |
| 27 | None. |
| 28 | |
| 29 | Description. |
| 30 | ------------ |
| 31 | |
| 32 | A data flow obeys TBF with rate R and depth B, if for any |
| 33 | time interval t_i...t_f the number of transmitted bits |
| 34 | does not exceed B + R*(t_f-t_i). |
| 35 | |
| 36 | Packetized version of this definition: |
| 37 | The sequence of packets of sizes s_i served at moments t_i |
| 38 | obeys TBF, if for any i<=k: |
| 39 | |
| 40 | s_i+....+s_k <= B + R*(t_k - t_i) |
| 41 | |
| 42 | Algorithm. |
| 43 | ---------- |
| 44 | |
| 45 | Let N(t_i) be B/R initially and N(t) grow continuously with time as: |
| 46 | |
| 47 | N(t+delta) = min{B/R, N(t) + delta} |
| 48 | |
| 49 | If the first packet in queue has length S, it may be |
| 50 | transmitted only at the time t_* when S/R <= N(t_*), |
| 51 | and in this case N(t) jumps: |
| 52 | |
| 53 | N(t_* + 0) = N(t_* - 0) - S/R. |
| 54 | |
| 55 | |
| 56 | |
| 57 | Actually, QoS requires two TBF to be applied to a data stream. |
| 58 | One of them controls steady state burst size, another |
| 59 | one with rate P (peak rate) and depth M (equal to link MTU) |
| 60 | limits bursts at a smaller time scale. |
| 61 | |
| 62 | It is easy to see that P>R, and B>M. If P is infinity, this double |
| 63 | TBF is equivalent to a single one. |
| 64 | |
| 65 | When TBF works in reshaping mode, latency is estimated as: |
| 66 | |
| 67 | lat = max ((L-B)/R, (L-M)/P) |
| 68 | |
| 69 | |
| 70 | NOTES. |
| 71 | ------ |
| 72 | |
| 73 | If TBF throttles, it starts a watchdog timer, which will wake it up |
| 74 | when it is ready to transmit. |
| 75 | Note that the minimal timer resolution is 1/HZ. |
| 76 | If no new packets arrive during this period, |
| 77 | or if the device is not awaken by EOI for some previous packet, |
| 78 | TBF can stop its activity for 1/HZ. |
| 79 | |
| 80 | |
| 81 | This means, that with depth B, the maximal rate is |
| 82 | |
| 83 | R_crit = B*HZ |
| 84 | |
| 85 | F.e. for 10Mbit ethernet and HZ=100 the minimal allowed B is ~10Kbytes. |
| 86 | |
| 87 | Note that the peak rate TBF is much more tough: with MTU 1500 |
| 88 | P_crit = 150Kbytes/sec. So, if you need greater peak |
| 89 | rates, use alpha with HZ=1000 :-) |
| 90 | |
| 91 | With classful TBF, limit is just kept for backwards compatibility. |
| 92 | It is passed to the default bfifo qdisc - if the inner qdisc is |
| 93 | changed the limit is not effective anymore. |
| 94 | */ |
| 95 | |
Eric Dumazet | cc7ec45 | 2011-01-19 19:26:56 +0000 | [diff] [blame] | 96 | struct tbf_sched_data { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 97 | /* Parameters */ |
| 98 | u32 limit; /* Maximal length of backlog: bytes */ |
Hiroaki SHIMODA | a135e59 | 2014-03-02 17:30:26 +0900 | [diff] [blame] | 99 | u32 max_size; |
Jiri Pirko | b757c93 | 2013-02-12 00:12:05 +0000 | [diff] [blame] | 100 | s64 buffer; /* Token bucket depth/rate: MUST BE >= MTU/B */ |
| 101 | s64 mtu; |
Jiri Pirko | b757c93 | 2013-02-12 00:12:05 +0000 | [diff] [blame] | 102 | struct psched_ratecfg rate; |
| 103 | struct psched_ratecfg peak; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 104 | |
| 105 | /* Variables */ |
Jiri Pirko | b757c93 | 2013-02-12 00:12:05 +0000 | [diff] [blame] | 106 | s64 tokens; /* Current number of B tokens */ |
| 107 | s64 ptokens; /* Current number of P tokens */ |
| 108 | s64 t_c; /* Time check-point */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 109 | struct Qdisc *qdisc; /* Inner qdisc, default - bfifo queue */ |
Patrick McHardy | f7f593e | 2007-03-16 01:20:07 -0700 | [diff] [blame] | 110 | struct qdisc_watchdog watchdog; /* Watchdog timer */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 111 | }; |
| 112 | |
Eric Dumazet | e43ac79 | 2013-05-21 08:16:46 +0000 | [diff] [blame] | 113 | |
Yang Yingliang | cc106e4 | 2013-12-10 14:59:27 +0800 | [diff] [blame] | 114 | /* Time to Length, convert time in ns to length in bytes |
| 115 | * to determinate how many bytes can be sent in given time. |
| 116 | */ |
| 117 | static u64 psched_ns_t2l(const struct psched_ratecfg *r, |
| 118 | u64 time_in_ns) |
| 119 | { |
| 120 | /* The formula is : |
| 121 | * len = (time_in_ns * r->rate_bytes_ps) / NSEC_PER_SEC |
| 122 | */ |
| 123 | u64 len = time_in_ns * r->rate_bytes_ps; |
| 124 | |
| 125 | do_div(len, NSEC_PER_SEC); |
| 126 | |
Yang Yingliang | d55d282 | 2013-12-12 10:57:22 +0800 | [diff] [blame] | 127 | if (unlikely(r->linklayer == TC_LINKLAYER_ATM)) { |
| 128 | do_div(len, 53); |
| 129 | len = len * 48; |
| 130 | } |
Yang Yingliang | cc106e4 | 2013-12-10 14:59:27 +0800 | [diff] [blame] | 131 | |
| 132 | if (len > r->overhead) |
| 133 | len -= r->overhead; |
| 134 | else |
| 135 | len = 0; |
| 136 | |
| 137 | return len; |
| 138 | } |
| 139 | |
Eric Dumazet | e43ac79 | 2013-05-21 08:16:46 +0000 | [diff] [blame] | 140 | /* GSO packet is too big, segment it so that tbf can transmit |
| 141 | * each segment in time |
| 142 | */ |
Eric Dumazet | 520ac30 | 2016-06-21 23:16:49 -0700 | [diff] [blame] | 143 | static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch, |
| 144 | struct sk_buff **to_free) |
Eric Dumazet | e43ac79 | 2013-05-21 08:16:46 +0000 | [diff] [blame] | 145 | { |
| 146 | struct tbf_sched_data *q = qdisc_priv(sch); |
| 147 | struct sk_buff *segs, *nskb; |
| 148 | netdev_features_t features = netif_skb_features(skb); |
WANG Cong | 2ccccf5 | 2016-02-25 14:55:01 -0800 | [diff] [blame] | 149 | unsigned int len = 0, prev_len = qdisc_pkt_len(skb); |
Eric Dumazet | e43ac79 | 2013-05-21 08:16:46 +0000 | [diff] [blame] | 150 | int ret, nb; |
| 151 | |
| 152 | segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK); |
| 153 | |
| 154 | if (IS_ERR_OR_NULL(segs)) |
Eric Dumazet | 520ac30 | 2016-06-21 23:16:49 -0700 | [diff] [blame] | 155 | return qdisc_drop(skb, sch, to_free); |
Eric Dumazet | e43ac79 | 2013-05-21 08:16:46 +0000 | [diff] [blame] | 156 | |
| 157 | nb = 0; |
| 158 | while (segs) { |
| 159 | nskb = segs->next; |
David S. Miller | a8305bf | 2018-07-29 20:42:53 -0700 | [diff] [blame] | 160 | skb_mark_not_on_list(segs); |
Eric Dumazet | 4d0820c | 2013-11-23 12:59:20 -0800 | [diff] [blame] | 161 | qdisc_skb_cb(segs)->pkt_len = segs->len; |
WANG Cong | 2ccccf5 | 2016-02-25 14:55:01 -0800 | [diff] [blame] | 162 | len += segs->len; |
Eric Dumazet | 520ac30 | 2016-06-21 23:16:49 -0700 | [diff] [blame] | 163 | ret = qdisc_enqueue(segs, q->qdisc, to_free); |
Eric Dumazet | e43ac79 | 2013-05-21 08:16:46 +0000 | [diff] [blame] | 164 | if (ret != NET_XMIT_SUCCESS) { |
| 165 | if (net_xmit_drop_count(ret)) |
John Fastabend | 25331d6 | 2014-09-28 11:53:29 -0700 | [diff] [blame] | 166 | qdisc_qstats_drop(sch); |
Eric Dumazet | e43ac79 | 2013-05-21 08:16:46 +0000 | [diff] [blame] | 167 | } else { |
| 168 | nb++; |
| 169 | } |
| 170 | segs = nskb; |
| 171 | } |
| 172 | sch->q.qlen += nb; |
| 173 | if (nb > 1) |
WANG Cong | 2ccccf5 | 2016-02-25 14:55:01 -0800 | [diff] [blame] | 174 | qdisc_tree_reduce_backlog(sch, 1 - nb, prev_len - len); |
Eric Dumazet | e43ac79 | 2013-05-21 08:16:46 +0000 | [diff] [blame] | 175 | consume_skb(skb); |
| 176 | return nb > 0 ? NET_XMIT_SUCCESS : NET_XMIT_DROP; |
| 177 | } |
| 178 | |
Eric Dumazet | 520ac30 | 2016-06-21 23:16:49 -0700 | [diff] [blame] | 179 | static int tbf_enqueue(struct sk_buff *skb, struct Qdisc *sch, |
| 180 | struct sk_buff **to_free) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 181 | { |
| 182 | struct tbf_sched_data *q = qdisc_priv(sch); |
Toke Høiland-Jørgensen | f6bab19 | 2019-01-09 17:09:42 +0100 | [diff] [blame] | 183 | unsigned int len = qdisc_pkt_len(skb); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 184 | int ret; |
| 185 | |
Eric Dumazet | e43ac79 | 2013-05-21 08:16:46 +0000 | [diff] [blame] | 186 | if (qdisc_pkt_len(skb) > q->max_size) { |
Daniel Axtens | ee78bbe | 2018-03-01 17:13:38 +1100 | [diff] [blame] | 187 | if (skb_is_gso(skb) && |
| 188 | skb_gso_validate_mac_len(skb, q->max_size)) |
Eric Dumazet | 520ac30 | 2016-06-21 23:16:49 -0700 | [diff] [blame] | 189 | return tbf_segment(skb, sch, to_free); |
| 190 | return qdisc_drop(skb, sch, to_free); |
Eric Dumazet | e43ac79 | 2013-05-21 08:16:46 +0000 | [diff] [blame] | 191 | } |
Eric Dumazet | 520ac30 | 2016-06-21 23:16:49 -0700 | [diff] [blame] | 192 | ret = qdisc_enqueue(skb, q->qdisc, to_free); |
Ben Greear | 9871e50 | 2010-08-10 01:45:40 -0700 | [diff] [blame] | 193 | if (ret != NET_XMIT_SUCCESS) { |
Jarek Poplawski | 378a2f0 | 2008-08-04 22:31:03 -0700 | [diff] [blame] | 194 | if (net_xmit_drop_count(ret)) |
John Fastabend | 25331d6 | 2014-09-28 11:53:29 -0700 | [diff] [blame] | 195 | qdisc_qstats_drop(sch); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 196 | return ret; |
| 197 | } |
| 198 | |
Toke Høiland-Jørgensen | f6bab19 | 2019-01-09 17:09:42 +0100 | [diff] [blame] | 199 | sch->qstats.backlog += len; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 200 | sch->q.qlen++; |
Ben Greear | 9871e50 | 2010-08-10 01:45:40 -0700 | [diff] [blame] | 201 | return NET_XMIT_SUCCESS; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 202 | } |
| 203 | |
Hiroaki SHIMODA | a135e59 | 2014-03-02 17:30:26 +0900 | [diff] [blame] | 204 | static bool tbf_peak_present(const struct tbf_sched_data *q) |
| 205 | { |
| 206 | return q->peak.rate_bytes_ps; |
| 207 | } |
| 208 | |
Eric Dumazet | cc7ec45 | 2011-01-19 19:26:56 +0000 | [diff] [blame] | 209 | static struct sk_buff *tbf_dequeue(struct Qdisc *sch) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 210 | { |
| 211 | struct tbf_sched_data *q = qdisc_priv(sch); |
| 212 | struct sk_buff *skb; |
| 213 | |
Jarek Poplawski | 03c05f0 | 2008-10-31 00:46:19 -0700 | [diff] [blame] | 214 | skb = q->qdisc->ops->peek(q->qdisc); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 215 | |
| 216 | if (skb) { |
Jiri Pirko | b757c93 | 2013-02-12 00:12:05 +0000 | [diff] [blame] | 217 | s64 now; |
| 218 | s64 toks; |
| 219 | s64 ptoks = 0; |
Jussi Kivilinna | 0abf77e | 2008-07-20 00:08:27 -0700 | [diff] [blame] | 220 | unsigned int len = qdisc_pkt_len(skb); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 221 | |
Eric Dumazet | d2de875 | 2014-08-22 18:32:09 -0700 | [diff] [blame] | 222 | now = ktime_get_ns(); |
Jiri Pirko | b757c93 | 2013-02-12 00:12:05 +0000 | [diff] [blame] | 223 | toks = min_t(s64, now - q->t_c, q->buffer); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 224 | |
Hiroaki SHIMODA | a135e59 | 2014-03-02 17:30:26 +0900 | [diff] [blame] | 225 | if (tbf_peak_present(q)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 226 | ptoks = toks + q->ptokens; |
Jiri Pirko | b757c93 | 2013-02-12 00:12:05 +0000 | [diff] [blame] | 227 | if (ptoks > q->mtu) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 228 | ptoks = q->mtu; |
Jiri Pirko | b757c93 | 2013-02-12 00:12:05 +0000 | [diff] [blame] | 229 | ptoks -= (s64) psched_l2t_ns(&q->peak, len); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 230 | } |
| 231 | toks += q->tokens; |
Jiri Pirko | b757c93 | 2013-02-12 00:12:05 +0000 | [diff] [blame] | 232 | if (toks > q->buffer) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 233 | toks = q->buffer; |
Jiri Pirko | b757c93 | 2013-02-12 00:12:05 +0000 | [diff] [blame] | 234 | toks -= (s64) psched_l2t_ns(&q->rate, len); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 235 | |
| 236 | if ((toks|ptoks) >= 0) { |
Jarek Poplawski | 77be155 | 2008-10-31 00:47:01 -0700 | [diff] [blame] | 237 | skb = qdisc_dequeue_peeked(q->qdisc); |
Jarek Poplawski | 03c05f0 | 2008-10-31 00:46:19 -0700 | [diff] [blame] | 238 | if (unlikely(!skb)) |
| 239 | return NULL; |
| 240 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 241 | q->t_c = now; |
| 242 | q->tokens = toks; |
| 243 | q->ptokens = ptoks; |
WANG Cong | 8d5958f | 2016-06-01 16:15:19 -0700 | [diff] [blame] | 244 | qdisc_qstats_backlog_dec(sch, skb); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 245 | sch->q.qlen--; |
Eric Dumazet | 9190b3b | 2011-01-20 23:31:33 -0800 | [diff] [blame] | 246 | qdisc_bstats_update(sch, skb); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 247 | return skb; |
| 248 | } |
| 249 | |
Jiri Pirko | b757c93 | 2013-02-12 00:12:05 +0000 | [diff] [blame] | 250 | qdisc_watchdog_schedule_ns(&q->watchdog, |
Eric Dumazet | 45f50be | 2016-06-10 16:41:39 -0700 | [diff] [blame] | 251 | now + max_t(long, -toks, -ptoks)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 252 | |
| 253 | /* Maybe we have a shorter packet in the queue, |
| 254 | which can be sent now. It sounds cool, |
| 255 | but, however, this is wrong in principle. |
| 256 | We MUST NOT reorder packets under these circumstances. |
| 257 | |
| 258 | Really, if we split the flow into independent |
| 259 | subflows, it would be a very good solution. |
| 260 | This is the main idea of all FQ algorithms |
| 261 | (cf. CSZ, HPFQ, HFSC) |
| 262 | */ |
| 263 | |
John Fastabend | 25331d6 | 2014-09-28 11:53:29 -0700 | [diff] [blame] | 264 | qdisc_qstats_overlimit(sch); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 265 | } |
| 266 | return NULL; |
| 267 | } |
| 268 | |
Eric Dumazet | cc7ec45 | 2011-01-19 19:26:56 +0000 | [diff] [blame] | 269 | static void tbf_reset(struct Qdisc *sch) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 270 | { |
| 271 | struct tbf_sched_data *q = qdisc_priv(sch); |
| 272 | |
| 273 | qdisc_reset(q->qdisc); |
WANG Cong | 8d5958f | 2016-06-01 16:15:19 -0700 | [diff] [blame] | 274 | sch->qstats.backlog = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 275 | sch->q.qlen = 0; |
Eric Dumazet | d2de875 | 2014-08-22 18:32:09 -0700 | [diff] [blame] | 276 | q->t_c = ktime_get_ns(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 277 | q->tokens = q->buffer; |
| 278 | q->ptokens = q->mtu; |
Patrick McHardy | f7f593e | 2007-03-16 01:20:07 -0700 | [diff] [blame] | 279 | qdisc_watchdog_cancel(&q->watchdog); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 280 | } |
| 281 | |
Patrick McHardy | 27a3421 | 2008-01-23 20:35:39 -0800 | [diff] [blame] | 282 | static const struct nla_policy tbf_policy[TCA_TBF_MAX + 1] = { |
| 283 | [TCA_TBF_PARMS] = { .len = sizeof(struct tc_tbf_qopt) }, |
| 284 | [TCA_TBF_RTAB] = { .type = NLA_BINARY, .len = TC_RTAB_SIZE }, |
| 285 | [TCA_TBF_PTAB] = { .type = NLA_BINARY, .len = TC_RTAB_SIZE }, |
Yang Yingliang | a33c4a2 | 2013-11-08 10:23:34 +0800 | [diff] [blame] | 286 | [TCA_TBF_RATE64] = { .type = NLA_U64 }, |
| 287 | [TCA_TBF_PRATE64] = { .type = NLA_U64 }, |
Yang Yingliang | 2e04ad4 | 2013-12-20 09:24:47 +0800 | [diff] [blame] | 288 | [TCA_TBF_BURST] = { .type = NLA_U32 }, |
| 289 | [TCA_TBF_PBURST] = { .type = NLA_U32 }, |
Patrick McHardy | 27a3421 | 2008-01-23 20:35:39 -0800 | [diff] [blame] | 290 | }; |
| 291 | |
Alexander Aring | 2030721 | 2017-12-20 12:35:14 -0500 | [diff] [blame] | 292 | static int tbf_change(struct Qdisc *sch, struct nlattr *opt, |
| 293 | struct netlink_ext_ack *extack) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 294 | { |
Patrick McHardy | cee6372 | 2008-01-23 20:33:32 -0800 | [diff] [blame] | 295 | int err; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 296 | struct tbf_sched_data *q = qdisc_priv(sch); |
Yang Yingliang | a33c4a2 | 2013-11-08 10:23:34 +0800 | [diff] [blame] | 297 | struct nlattr *tb[TCA_TBF_MAX + 1]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 298 | struct tc_tbf_qopt *qopt; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 299 | struct Qdisc *child = NULL; |
Yang Yingliang | cc106e4 | 2013-12-10 14:59:27 +0800 | [diff] [blame] | 300 | struct psched_ratecfg rate; |
| 301 | struct psched_ratecfg peak; |
| 302 | u64 max_size; |
| 303 | s64 buffer, mtu; |
Yang Yingliang | a33c4a2 | 2013-11-08 10:23:34 +0800 | [diff] [blame] | 304 | u64 rate64 = 0, prate64 = 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 305 | |
Johannes Berg | 8cb0817 | 2019-04-26 14:07:28 +0200 | [diff] [blame] | 306 | err = nla_parse_nested_deprecated(tb, TCA_TBF_MAX, opt, tbf_policy, |
| 307 | NULL); |
Patrick McHardy | cee6372 | 2008-01-23 20:33:32 -0800 | [diff] [blame] | 308 | if (err < 0) |
| 309 | return err; |
| 310 | |
| 311 | err = -EINVAL; |
Patrick McHardy | 27a3421 | 2008-01-23 20:35:39 -0800 | [diff] [blame] | 312 | if (tb[TCA_TBF_PARMS] == NULL) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 313 | goto done; |
| 314 | |
Patrick McHardy | 1e90474 | 2008-01-22 22:11:17 -0800 | [diff] [blame] | 315 | qopt = nla_data(tb[TCA_TBF_PARMS]); |
Yang Yingliang | cc106e4 | 2013-12-10 14:59:27 +0800 | [diff] [blame] | 316 | if (qopt->rate.linklayer == TC_LINKLAYER_UNAWARE) |
| 317 | qdisc_put_rtab(qdisc_get_rtab(&qopt->rate, |
Alexander Aring | e9bc3fa | 2017-12-20 12:35:18 -0500 | [diff] [blame] | 318 | tb[TCA_TBF_RTAB], |
| 319 | NULL)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 320 | |
Yang Yingliang | cc106e4 | 2013-12-10 14:59:27 +0800 | [diff] [blame] | 321 | if (qopt->peakrate.linklayer == TC_LINKLAYER_UNAWARE) |
| 322 | qdisc_put_rtab(qdisc_get_rtab(&qopt->peakrate, |
Alexander Aring | e9bc3fa | 2017-12-20 12:35:18 -0500 | [diff] [blame] | 323 | tb[TCA_TBF_PTAB], |
| 324 | NULL)); |
Eric Dumazet | 4d0820c | 2013-11-23 12:59:20 -0800 | [diff] [blame] | 325 | |
Yang Yingliang | cc106e4 | 2013-12-10 14:59:27 +0800 | [diff] [blame] | 326 | buffer = min_t(u64, PSCHED_TICKS2NS(qopt->buffer), ~0U); |
| 327 | mtu = min_t(u64, PSCHED_TICKS2NS(qopt->mtu), ~0U); |
| 328 | |
| 329 | if (tb[TCA_TBF_RATE64]) |
| 330 | rate64 = nla_get_u64(tb[TCA_TBF_RATE64]); |
| 331 | psched_ratecfg_precompute(&rate, &qopt->rate, rate64); |
| 332 | |
Yang Yingliang | 2e04ad4 | 2013-12-20 09:24:47 +0800 | [diff] [blame] | 333 | if (tb[TCA_TBF_BURST]) { |
| 334 | max_size = nla_get_u32(tb[TCA_TBF_BURST]); |
| 335 | buffer = psched_l2t_ns(&rate, max_size); |
| 336 | } else { |
| 337 | max_size = min_t(u64, psched_ns_t2l(&rate, buffer), ~0U); |
| 338 | } |
Yang Yingliang | cc106e4 | 2013-12-10 14:59:27 +0800 | [diff] [blame] | 339 | |
| 340 | if (qopt->peakrate.rate) { |
| 341 | if (tb[TCA_TBF_PRATE64]) |
| 342 | prate64 = nla_get_u64(tb[TCA_TBF_PRATE64]); |
| 343 | psched_ratecfg_precompute(&peak, &qopt->peakrate, prate64); |
| 344 | if (peak.rate_bytes_ps <= rate.rate_bytes_ps) { |
| 345 | pr_warn_ratelimited("sch_tbf: peakrate %llu is lower than or equals to rate %llu !\n", |
Yang Yingliang | 2e04ad4 | 2013-12-20 09:24:47 +0800 | [diff] [blame] | 346 | peak.rate_bytes_ps, rate.rate_bytes_ps); |
Yang Yingliang | cc106e4 | 2013-12-10 14:59:27 +0800 | [diff] [blame] | 347 | err = -EINVAL; |
| 348 | goto done; |
| 349 | } |
| 350 | |
Yang Yingliang | 2e04ad4 | 2013-12-20 09:24:47 +0800 | [diff] [blame] | 351 | if (tb[TCA_TBF_PBURST]) { |
| 352 | u32 pburst = nla_get_u32(tb[TCA_TBF_PBURST]); |
| 353 | max_size = min_t(u32, max_size, pburst); |
| 354 | mtu = psched_l2t_ns(&peak, pburst); |
| 355 | } else { |
| 356 | max_size = min_t(u64, max_size, psched_ns_t2l(&peak, mtu)); |
| 357 | } |
Hiroaki SHIMODA | a135e59 | 2014-03-02 17:30:26 +0900 | [diff] [blame] | 358 | } else { |
| 359 | memset(&peak, 0, sizeof(peak)); |
Yang Yingliang | cc106e4 | 2013-12-10 14:59:27 +0800 | [diff] [blame] | 360 | } |
| 361 | |
| 362 | if (max_size < psched_mtu(qdisc_dev(sch))) |
| 363 | pr_warn_ratelimited("sch_tbf: burst %llu is lower than device %s mtu (%u) !\n", |
| 364 | max_size, qdisc_dev(sch)->name, |
| 365 | psched_mtu(qdisc_dev(sch))); |
| 366 | |
| 367 | if (!max_size) { |
| 368 | err = -EINVAL; |
| 369 | goto done; |
| 370 | } |
| 371 | |
Hiroaki SHIMODA | 724b9e1 | 2014-02-26 21:43:42 +0900 | [diff] [blame] | 372 | if (q->qdisc != &noop_qdisc) { |
| 373 | err = fifo_set_limit(q->qdisc, qopt->limit); |
| 374 | if (err) |
| 375 | goto done; |
| 376 | } else if (qopt->limit > 0) { |
Alexander Aring | a38a9882 | 2017-12-20 12:35:21 -0500 | [diff] [blame] | 377 | child = fifo_create_dflt(sch, &bfifo_qdisc_ops, qopt->limit, |
| 378 | extack); |
Hiroaki SHIMODA | 724b9e1 | 2014-02-26 21:43:42 +0900 | [diff] [blame] | 379 | if (IS_ERR(child)) { |
| 380 | err = PTR_ERR(child); |
| 381 | goto done; |
| 382 | } |
Paolo Abeni | 44a63b13 | 2018-05-18 14:51:44 +0200 | [diff] [blame] | 383 | |
| 384 | /* child is fifo, no need to check for noop_qdisc */ |
| 385 | qdisc_hash_add(child, true); |
Hiroaki SHIMODA | 724b9e1 | 2014-02-26 21:43:42 +0900 | [diff] [blame] | 386 | } |
| 387 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 388 | sch_tree_lock(sch); |
Patrick McHardy | 5e50da0 | 2006-11-29 17:36:20 -0800 | [diff] [blame] | 389 | if (child) { |
Paolo Abeni | e5f0e8f | 2019-03-28 16:53:13 +0100 | [diff] [blame] | 390 | qdisc_tree_flush_backlog(q->qdisc); |
Vlad Buslov | 86bd446 | 2018-09-24 19:22:50 +0300 | [diff] [blame] | 391 | qdisc_put(q->qdisc); |
Patrick McHardy | b94c8af | 2008-11-20 04:11:36 -0800 | [diff] [blame] | 392 | q->qdisc = child; |
Patrick McHardy | 5e50da0 | 2006-11-29 17:36:20 -0800 | [diff] [blame] | 393 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 394 | q->limit = qopt->limit; |
Yang Yingliang | 2e04ad4 | 2013-12-20 09:24:47 +0800 | [diff] [blame] | 395 | if (tb[TCA_TBF_PBURST]) |
| 396 | q->mtu = mtu; |
| 397 | else |
| 398 | q->mtu = PSCHED_TICKS2NS(qopt->mtu); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 399 | q->max_size = max_size; |
Yang Yingliang | 2e04ad4 | 2013-12-20 09:24:47 +0800 | [diff] [blame] | 400 | if (tb[TCA_TBF_BURST]) |
| 401 | q->buffer = buffer; |
| 402 | else |
| 403 | q->buffer = PSCHED_TICKS2NS(qopt->buffer); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 404 | q->tokens = q->buffer; |
| 405 | q->ptokens = q->mtu; |
Patrick McHardy | b94c8af | 2008-11-20 04:11:36 -0800 | [diff] [blame] | 406 | |
Yang Yingliang | cc106e4 | 2013-12-10 14:59:27 +0800 | [diff] [blame] | 407 | memcpy(&q->rate, &rate, sizeof(struct psched_ratecfg)); |
Hiroaki SHIMODA | a135e59 | 2014-03-02 17:30:26 +0900 | [diff] [blame] | 408 | memcpy(&q->peak, &peak, sizeof(struct psched_ratecfg)); |
Patrick McHardy | b94c8af | 2008-11-20 04:11:36 -0800 | [diff] [blame] | 409 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 410 | sch_tree_unlock(sch); |
| 411 | err = 0; |
| 412 | done: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 413 | return err; |
| 414 | } |
| 415 | |
Alexander Aring | e63d7dfd | 2017-12-20 12:35:13 -0500 | [diff] [blame] | 416 | static int tbf_init(struct Qdisc *sch, struct nlattr *opt, |
| 417 | struct netlink_ext_ack *extack) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 418 | { |
| 419 | struct tbf_sched_data *q = qdisc_priv(sch); |
| 420 | |
Nikolay Aleksandrov | c2d6511 | 2017-08-30 12:49:05 +0300 | [diff] [blame] | 421 | qdisc_watchdog_init(&q->watchdog, sch); |
| 422 | q->qdisc = &noop_qdisc; |
| 423 | |
Alexander Aring | ac8ef4a | 2017-12-20 12:35:11 -0500 | [diff] [blame] | 424 | if (!opt) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 425 | return -EINVAL; |
| 426 | |
Eric Dumazet | d2de875 | 2014-08-22 18:32:09 -0700 | [diff] [blame] | 427 | q->t_c = ktime_get_ns(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 428 | |
Alexander Aring | 2030721 | 2017-12-20 12:35:14 -0500 | [diff] [blame] | 429 | return tbf_change(sch, opt, extack); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 430 | } |
| 431 | |
| 432 | static void tbf_destroy(struct Qdisc *sch) |
| 433 | { |
| 434 | struct tbf_sched_data *q = qdisc_priv(sch); |
| 435 | |
Patrick McHardy | f7f593e | 2007-03-16 01:20:07 -0700 | [diff] [blame] | 436 | qdisc_watchdog_cancel(&q->watchdog); |
Vlad Buslov | 86bd446 | 2018-09-24 19:22:50 +0300 | [diff] [blame] | 437 | qdisc_put(q->qdisc); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 438 | } |
| 439 | |
| 440 | static int tbf_dump(struct Qdisc *sch, struct sk_buff *skb) |
| 441 | { |
| 442 | struct tbf_sched_data *q = qdisc_priv(sch); |
Patrick McHardy | 4b3550ef | 2008-01-23 20:34:11 -0800 | [diff] [blame] | 443 | struct nlattr *nest; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 444 | struct tc_tbf_qopt opt; |
| 445 | |
Eric Dumazet | b0460e4 | 2011-12-28 23:27:44 +0000 | [diff] [blame] | 446 | sch->qstats.backlog = q->qdisc->qstats.backlog; |
Michal Kubecek | ae0be8d | 2019-04-26 11:13:06 +0200 | [diff] [blame] | 447 | nest = nla_nest_start_noflag(skb, TCA_OPTIONS); |
Patrick McHardy | 4b3550ef | 2008-01-23 20:34:11 -0800 | [diff] [blame] | 448 | if (nest == NULL) |
| 449 | goto nla_put_failure; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 450 | |
| 451 | opt.limit = q->limit; |
Eric Dumazet | 01cb71d | 2013-06-02 13:55:05 +0000 | [diff] [blame] | 452 | psched_ratecfg_getrate(&opt.rate, &q->rate); |
Hiroaki SHIMODA | a135e59 | 2014-03-02 17:30:26 +0900 | [diff] [blame] | 453 | if (tbf_peak_present(q)) |
Eric Dumazet | 01cb71d | 2013-06-02 13:55:05 +0000 | [diff] [blame] | 454 | psched_ratecfg_getrate(&opt.peakrate, &q->peak); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 455 | else |
| 456 | memset(&opt.peakrate, 0, sizeof(opt.peakrate)); |
Jiri Pirko | b757c93 | 2013-02-12 00:12:05 +0000 | [diff] [blame] | 457 | opt.mtu = PSCHED_NS2TICKS(q->mtu); |
| 458 | opt.buffer = PSCHED_NS2TICKS(q->buffer); |
David S. Miller | 1b34ec4 | 2012-03-29 05:11:39 -0400 | [diff] [blame] | 459 | if (nla_put(skb, TCA_TBF_PARMS, sizeof(opt), &opt)) |
| 460 | goto nla_put_failure; |
Yang Yingliang | a33c4a2 | 2013-11-08 10:23:34 +0800 | [diff] [blame] | 461 | if (q->rate.rate_bytes_ps >= (1ULL << 32) && |
Nicolas Dichtel | 2a51c1e | 2016-04-25 10:25:15 +0200 | [diff] [blame] | 462 | nla_put_u64_64bit(skb, TCA_TBF_RATE64, q->rate.rate_bytes_ps, |
| 463 | TCA_TBF_PAD)) |
Yang Yingliang | a33c4a2 | 2013-11-08 10:23:34 +0800 | [diff] [blame] | 464 | goto nla_put_failure; |
Hiroaki SHIMODA | a135e59 | 2014-03-02 17:30:26 +0900 | [diff] [blame] | 465 | if (tbf_peak_present(q) && |
Yang Yingliang | a33c4a2 | 2013-11-08 10:23:34 +0800 | [diff] [blame] | 466 | q->peak.rate_bytes_ps >= (1ULL << 32) && |
Nicolas Dichtel | 2a51c1e | 2016-04-25 10:25:15 +0200 | [diff] [blame] | 467 | nla_put_u64_64bit(skb, TCA_TBF_PRATE64, q->peak.rate_bytes_ps, |
| 468 | TCA_TBF_PAD)) |
Yang Yingliang | a33c4a2 | 2013-11-08 10:23:34 +0800 | [diff] [blame] | 469 | goto nla_put_failure; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 470 | |
Yang Yingliang | d59b7d8 | 2014-03-12 10:20:32 +0800 | [diff] [blame] | 471 | return nla_nest_end(skb, nest); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 472 | |
Patrick McHardy | 1e90474 | 2008-01-22 22:11:17 -0800 | [diff] [blame] | 473 | nla_put_failure: |
Patrick McHardy | 4b3550ef | 2008-01-23 20:34:11 -0800 | [diff] [blame] | 474 | nla_nest_cancel(skb, nest); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 475 | return -1; |
| 476 | } |
| 477 | |
| 478 | static int tbf_dump_class(struct Qdisc *sch, unsigned long cl, |
| 479 | struct sk_buff *skb, struct tcmsg *tcm) |
| 480 | { |
| 481 | struct tbf_sched_data *q = qdisc_priv(sch); |
| 482 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 483 | tcm->tcm_handle |= TC_H_MIN(1); |
| 484 | tcm->tcm_info = q->qdisc->handle; |
| 485 | |
| 486 | return 0; |
| 487 | } |
| 488 | |
| 489 | static int tbf_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new, |
Alexander Aring | 653d6fd | 2017-12-20 12:35:17 -0500 | [diff] [blame] | 490 | struct Qdisc **old, struct netlink_ext_ack *extack) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 491 | { |
| 492 | struct tbf_sched_data *q = qdisc_priv(sch); |
| 493 | |
| 494 | if (new == NULL) |
| 495 | new = &noop_qdisc; |
| 496 | |
WANG Cong | 86a7996 | 2016-02-25 14:55:00 -0800 | [diff] [blame] | 497 | *old = qdisc_replace(sch, new, &q->qdisc); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 498 | return 0; |
| 499 | } |
| 500 | |
| 501 | static struct Qdisc *tbf_leaf(struct Qdisc *sch, unsigned long arg) |
| 502 | { |
| 503 | struct tbf_sched_data *q = qdisc_priv(sch); |
| 504 | return q->qdisc; |
| 505 | } |
| 506 | |
WANG Cong | 143976c | 2017-08-24 16:51:29 -0700 | [diff] [blame] | 507 | static unsigned long tbf_find(struct Qdisc *sch, u32 classid) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 508 | { |
| 509 | return 1; |
| 510 | } |
| 511 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 512 | static void tbf_walk(struct Qdisc *sch, struct qdisc_walker *walker) |
| 513 | { |
| 514 | if (!walker->stop) { |
| 515 | if (walker->count >= walker->skip) |
| 516 | if (walker->fn(sch, 1, walker) < 0) { |
| 517 | walker->stop = 1; |
| 518 | return; |
| 519 | } |
| 520 | walker->count++; |
| 521 | } |
| 522 | } |
| 523 | |
Eric Dumazet | cc7ec45 | 2011-01-19 19:26:56 +0000 | [diff] [blame] | 524 | static const struct Qdisc_class_ops tbf_class_ops = { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 525 | .graft = tbf_graft, |
| 526 | .leaf = tbf_leaf, |
WANG Cong | 143976c | 2017-08-24 16:51:29 -0700 | [diff] [blame] | 527 | .find = tbf_find, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 528 | .walk = tbf_walk, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 529 | .dump = tbf_dump_class, |
| 530 | }; |
| 531 | |
Eric Dumazet | 20fea08 | 2007-11-14 01:44:41 -0800 | [diff] [blame] | 532 | static struct Qdisc_ops tbf_qdisc_ops __read_mostly = { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 533 | .next = NULL, |
| 534 | .cl_ops = &tbf_class_ops, |
| 535 | .id = "tbf", |
| 536 | .priv_size = sizeof(struct tbf_sched_data), |
| 537 | .enqueue = tbf_enqueue, |
| 538 | .dequeue = tbf_dequeue, |
Jarek Poplawski | 77be155 | 2008-10-31 00:47:01 -0700 | [diff] [blame] | 539 | .peek = qdisc_peek_dequeued, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 540 | .init = tbf_init, |
| 541 | .reset = tbf_reset, |
| 542 | .destroy = tbf_destroy, |
| 543 | .change = tbf_change, |
| 544 | .dump = tbf_dump, |
| 545 | .owner = THIS_MODULE, |
| 546 | }; |
| 547 | |
| 548 | static int __init tbf_module_init(void) |
| 549 | { |
| 550 | return register_qdisc(&tbf_qdisc_ops); |
| 551 | } |
| 552 | |
| 553 | static void __exit tbf_module_exit(void) |
| 554 | { |
| 555 | unregister_qdisc(&tbf_qdisc_ops); |
| 556 | } |
| 557 | module_init(tbf_module_init) |
| 558 | module_exit(tbf_module_exit) |
| 559 | MODULE_LICENSE("GPL"); |