| #!/bin/bash |
| # SPDX-License-Identifier: GPL-2.0 |
| # |
| # A test for switch behavior under MC overload. An issue in Spectrum chips |
| # causes throughput of UC traffic to drop severely when a switch is under heavy |
| # MC load. This issue can be overcome by putting the switch to MC-aware mode. |
| # This test verifies that UC performance stays intact even as the switch is |
| # under MC flood, and therefore that the MC-aware mode is enabled and correctly |
| # configured. |
| # |
| # Because mlxsw throttles CPU port, the traffic can't actually reach userspace |
| # at full speed. That makes it impossible to use iperf3 to simply measure the |
| # throughput, because many packets (that reach $h3) don't get to the kernel at |
| # all even in UDP mode (the situation is even worse in TCP mode, where one can't |
| # hope to see more than a couple Mbps). |
| # |
| # So instead we send traffic with mausezahn and use RX ethtool counters at $h3. |
| # Multicast traffic is untagged, unicast traffic is tagged with PCP 1. Therefore |
| # each gets a different priority and we can use per-prio ethtool counters to |
| # measure the throughput. In order to avoid prioritizing unicast traffic, prio |
| # qdisc is installed on $swp3 and maps all priorities to the same band #7 (and |
| # thus TC 0). |
| # |
| # Mausezahn can't actually saturate the links unless it's using large frames. |
| # Thus we set MTU to 10K on all involved interfaces. Then both unicast and |
| # multicast traffic uses 8K frames. |
| # |
| # +---------------------------+ +----------------------------------+ |
| # | H1 | | H2 | |
| # | | | unicast --> + $h2.111 | |
| # | multicast | | traffic | 192.0.2.129/28 | |
| # | traffic | | | e-qos-map 0:1 | |
| # | $h1 + <----- | | | | |
| # | 192.0.2.65/28 | | | + $h2 | |
| # +---------------|-----------+ +--------------|-------------------+ |
| # | | |
| # +---------------|---------------------------------------|-------------------+ |
| # | $swp1 + + $swp2 | |
| # | >1Gbps | | >1Gbps | |
| # | +-------------|------+ +----------|----------------+ | |
| # | | $swp1.1 + | | + $swp2.111 | | |
| # | | BR1 | SW | BR111 | | |
| # | | $swp3.1 + | | + $swp3.111 | | |
| # | +-------------|------+ +----------|----------------+ | |
| # | \_______________________________________/ | |
| # | | | |
| # | + $swp3 | |
| # | | 1Gbps bottleneck | |
| # | | prio qdisc: {0..7} -> 7 | |
| # +------------------------------------|--------------------------------------+ |
| # | |
| # +--|-----------------+ |
| # | + $h3 H3 | |
| # | | 192.0.2.66/28 | |
| # | | | |
| # | + $h3.111 | |
| # | 192.0.2.130/28 | |
| # +--------------------+ |
| |
| ALL_TESTS=" |
| ping_ipv4 |
| test_mc_aware |
| test_uc_aware |
| " |
| |
| lib_dir=$(dirname $0)/../../../net/forwarding |
| |
| NUM_NETIFS=6 |
| source $lib_dir/lib.sh |
| source $lib_dir/devlink_lib.sh |
| source qos_lib.sh |
| |
| h1_create() |
| { |
| simple_if_init $h1 192.0.2.65/28 |
| mtu_set $h1 10000 |
| } |
| |
| h1_destroy() |
| { |
| mtu_restore $h1 |
| simple_if_fini $h1 192.0.2.65/28 |
| } |
| |
| h2_create() |
| { |
| simple_if_init $h2 |
| mtu_set $h2 10000 |
| |
| vlan_create $h2 111 v$h2 192.0.2.129/28 |
| ip link set dev $h2.111 type vlan egress-qos-map 0:1 |
| } |
| |
| h2_destroy() |
| { |
| vlan_destroy $h2 111 |
| |
| mtu_restore $h2 |
| simple_if_fini $h2 |
| } |
| |
| h3_create() |
| { |
| simple_if_init $h3 192.0.2.66/28 |
| mtu_set $h3 10000 |
| |
| vlan_create $h3 111 v$h3 192.0.2.130/28 |
| } |
| |
| h3_destroy() |
| { |
| vlan_destroy $h3 111 |
| |
| mtu_restore $h3 |
| simple_if_fini $h3 192.0.2.66/28 |
| } |
| |
| switch_create() |
| { |
| ip link set dev $swp1 up |
| mtu_set $swp1 10000 |
| |
| ip link set dev $swp2 up |
| mtu_set $swp2 10000 |
| |
| ip link set dev $swp3 up |
| mtu_set $swp3 10000 |
| |
| vlan_create $swp2 111 |
| vlan_create $swp3 111 |
| |
| tc qdisc replace dev $swp3 root handle 3: tbf rate 1gbit \ |
| burst 128K limit 1G |
| tc qdisc replace dev $swp3 parent 3:3 handle 33: \ |
| prio bands 8 priomap 7 7 7 7 7 7 7 7 |
| |
| ip link add name br1 type bridge vlan_filtering 0 |
| ip link set dev br1 up |
| ip link set dev $swp1 master br1 |
| ip link set dev $swp3 master br1 |
| |
| ip link add name br111 type bridge vlan_filtering 0 |
| ip link set dev br111 up |
| ip link set dev $swp2.111 master br111 |
| ip link set dev $swp3.111 master br111 |
| |
| # Make sure that ingress quotas are smaller than egress so that there is |
| # room for both streams of traffic to be admitted to shared buffer. |
| devlink_port_pool_th_save $swp1 0 |
| devlink_port_pool_th_set $swp1 0 5 |
| devlink_tc_bind_pool_th_save $swp1 0 ingress |
| devlink_tc_bind_pool_th_set $swp1 0 ingress 0 5 |
| |
| devlink_port_pool_th_save $swp2 0 |
| devlink_port_pool_th_set $swp2 0 5 |
| devlink_tc_bind_pool_th_save $swp2 1 ingress |
| devlink_tc_bind_pool_th_set $swp2 1 ingress 0 5 |
| |
| devlink_port_pool_th_save $swp3 4 |
| devlink_port_pool_th_set $swp3 4 12 |
| } |
| |
| switch_destroy() |
| { |
| devlink_port_pool_th_restore $swp3 4 |
| |
| devlink_tc_bind_pool_th_restore $swp2 1 ingress |
| devlink_port_pool_th_restore $swp2 0 |
| |
| devlink_tc_bind_pool_th_restore $swp1 0 ingress |
| devlink_port_pool_th_restore $swp1 0 |
| |
| ip link del dev br111 |
| ip link del dev br1 |
| |
| tc qdisc del dev $swp3 parent 3:3 handle 33: |
| tc qdisc del dev $swp3 root handle 3: |
| |
| vlan_destroy $swp3 111 |
| vlan_destroy $swp2 111 |
| |
| mtu_restore $swp3 |
| ip link set dev $swp3 down |
| |
| mtu_restore $swp2 |
| ip link set dev $swp2 down |
| |
| mtu_restore $swp1 |
| ip link set dev $swp1 down |
| } |
| |
| setup_prepare() |
| { |
| h1=${NETIFS[p1]} |
| swp1=${NETIFS[p2]} |
| |
| swp2=${NETIFS[p3]} |
| h2=${NETIFS[p4]} |
| |
| swp3=${NETIFS[p5]} |
| h3=${NETIFS[p6]} |
| |
| h3mac=$(mac_get $h3) |
| |
| vrf_prepare |
| |
| h1_create |
| h2_create |
| h3_create |
| switch_create |
| } |
| |
| cleanup() |
| { |
| pre_cleanup |
| |
| switch_destroy |
| h3_destroy |
| h2_destroy |
| h1_destroy |
| |
| vrf_cleanup |
| } |
| |
| ping_ipv4() |
| { |
| ping_test $h2 192.0.2.130 |
| } |
| |
| test_mc_aware() |
| { |
| RET=0 |
| |
| local -a uc_rate |
| start_traffic $h2.111 192.0.2.129 192.0.2.130 $h3mac |
| uc_rate=($(measure_rate $swp2 $h3 rx_octets_prio_1 "UC-only")) |
| check_err $? "Could not get high enough UC-only ingress rate" |
| stop_traffic |
| local ucth1=${uc_rate[1]} |
| |
| start_traffic $h1 192.0.2.65 bc bc |
| |
| local d0=$(date +%s) |
| local t0=$(ethtool_stats_get $h3 rx_octets_prio_0) |
| local u0=$(ethtool_stats_get $swp1 rx_octets_prio_0) |
| |
| local -a uc_rate_2 |
| start_traffic $h2.111 192.0.2.129 192.0.2.130 $h3mac |
| uc_rate_2=($(measure_rate $swp2 $h3 rx_octets_prio_1 "UC+MC")) |
| check_err $? "Could not get high enough UC+MC ingress rate" |
| stop_traffic |
| local ucth2=${uc_rate_2[1]} |
| |
| local d1=$(date +%s) |
| local t1=$(ethtool_stats_get $h3 rx_octets_prio_0) |
| local u1=$(ethtool_stats_get $swp1 rx_octets_prio_0) |
| |
| local deg=$(bc <<< " |
| scale=2 |
| ret = 100 * ($ucth1 - $ucth2) / $ucth1 |
| if (ret > 0) { ret } else { 0 } |
| ") |
| |
| # Minimum shaper of 200Mbps on MC TCs should cause about 20% of |
| # degradation on 1Gbps link. |
| check_err $(bc <<< "$deg < 15") "Minimum shaper not in effect" |
| check_err $(bc <<< "$deg > 25") "MC traffic degrades UC performance too much" |
| |
| local interval=$((d1 - d0)) |
| local mc_ir=$(rate $u0 $u1 $interval) |
| local mc_er=$(rate $t0 $t1 $interval) |
| |
| stop_traffic |
| |
| log_test "UC performance under MC overload" |
| |
| echo "UC-only throughput $(humanize $ucth1)" |
| echo "UC+MC throughput $(humanize $ucth2)" |
| echo "Degradation $deg %" |
| echo |
| echo "Full report:" |
| echo " UC only:" |
| echo " ingress UC throughput $(humanize ${uc_rate[0]})" |
| echo " egress UC throughput $(humanize ${uc_rate[1]})" |
| echo " UC+MC:" |
| echo " ingress UC throughput $(humanize ${uc_rate_2[0]})" |
| echo " egress UC throughput $(humanize ${uc_rate_2[1]})" |
| echo " ingress MC throughput $(humanize $mc_ir)" |
| echo " egress MC throughput $(humanize $mc_er)" |
| echo |
| } |
| |
| test_uc_aware() |
| { |
| RET=0 |
| |
| start_traffic $h2.111 192.0.2.129 192.0.2.130 $h3mac |
| |
| local d0=$(date +%s) |
| local t0=$(ethtool_stats_get $h3 rx_octets_prio_1) |
| local u0=$(ethtool_stats_get $swp2 rx_octets_prio_1) |
| sleep 1 |
| |
| local attempts=50 |
| local passes=0 |
| local i |
| |
| for ((i = 0; i < attempts; ++i)); do |
| if $ARPING -c 1 -I $h1 -b 192.0.2.66 -q -w 1; then |
| ((passes++)) |
| fi |
| |
| sleep 0.1 |
| done |
| |
| local d1=$(date +%s) |
| local t1=$(ethtool_stats_get $h3 rx_octets_prio_1) |
| local u1=$(ethtool_stats_get $swp2 rx_octets_prio_1) |
| |
| local interval=$((d1 - d0)) |
| local uc_ir=$(rate $u0 $u1 $interval) |
| local uc_er=$(rate $t0 $t1 $interval) |
| |
| ((attempts == passes)) |
| check_err $? |
| |
| stop_traffic |
| |
| log_test "MC performance under UC overload" |
| echo " ingress UC throughput $(humanize ${uc_ir})" |
| echo " egress UC throughput $(humanize ${uc_er})" |
| echo " sent $attempts BC ARPs, got $passes responses" |
| } |
| |
| trap cleanup EXIT |
| |
| setup_prepare |
| setup_wait |
| |
| tests_run |
| |
| exit $EXIT_STATUS |
