Cai Huoqing | d164bf64 | 2021-08-23 10:35:30 +0800 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause |
Brian Welty | 696513e | 2017-02-08 05:27:07 -0800 | [diff] [blame] | 2 | /* |
| 3 | * Copyright(c) 2016 Intel Corporation. |
Brian Welty | 696513e | 2017-02-08 05:27:07 -0800 | [diff] [blame] | 4 | */ |
| 5 | |
Mike Marciniszyn | 715ab1a | 2019-04-11 07:16:11 -0700 | [diff] [blame] | 6 | #include <rdma/rdmavt_qp.h> |
Don Hiatt | 832666c | 2017-02-08 05:28:25 -0800 | [diff] [blame] | 7 | #include <rdma/ib_hdrs.h> |
Brian Welty | 696513e | 2017-02-08 05:27:07 -0800 | [diff] [blame] | 8 | |
| 9 | /* |
| 10 | * Convert the AETH credit code into the number of credits. |
| 11 | */ |
| 12 | static const u16 credit_table[31] = { |
| 13 | 0, /* 0 */ |
| 14 | 1, /* 1 */ |
| 15 | 2, /* 2 */ |
| 16 | 3, /* 3 */ |
| 17 | 4, /* 4 */ |
| 18 | 6, /* 5 */ |
| 19 | 8, /* 6 */ |
| 20 | 12, /* 7 */ |
| 21 | 16, /* 8 */ |
| 22 | 24, /* 9 */ |
| 23 | 32, /* A */ |
| 24 | 48, /* B */ |
| 25 | 64, /* C */ |
| 26 | 96, /* D */ |
| 27 | 128, /* E */ |
| 28 | 192, /* F */ |
| 29 | 256, /* 10 */ |
| 30 | 384, /* 11 */ |
| 31 | 512, /* 12 */ |
| 32 | 768, /* 13 */ |
| 33 | 1024, /* 14 */ |
| 34 | 1536, /* 15 */ |
| 35 | 2048, /* 16 */ |
| 36 | 3072, /* 17 */ |
| 37 | 4096, /* 18 */ |
| 38 | 6144, /* 19 */ |
| 39 | 8192, /* 1A */ |
| 40 | 12288, /* 1B */ |
| 41 | 16384, /* 1C */ |
| 42 | 24576, /* 1D */ |
| 43 | 32768 /* 1E */ |
| 44 | }; |
| 45 | |
| 46 | /** |
| 47 | * rvt_compute_aeth - compute the AETH (syndrome + MSN) |
| 48 | * @qp: the queue pair to compute the AETH for |
| 49 | * |
| 50 | * Returns the AETH. |
| 51 | */ |
| 52 | __be32 rvt_compute_aeth(struct rvt_qp *qp) |
| 53 | { |
Don Hiatt | 832666c | 2017-02-08 05:28:25 -0800 | [diff] [blame] | 54 | u32 aeth = qp->r_msn & IB_MSN_MASK; |
Brian Welty | 696513e | 2017-02-08 05:27:07 -0800 | [diff] [blame] | 55 | |
| 56 | if (qp->ibqp.srq) { |
| 57 | /* |
| 58 | * Shared receive queues don't generate credits. |
| 59 | * Set the credit field to the invalid value. |
| 60 | */ |
Don Hiatt | 832666c | 2017-02-08 05:28:25 -0800 | [diff] [blame] | 61 | aeth |= IB_AETH_CREDIT_INVAL << IB_AETH_CREDIT_SHIFT; |
Brian Welty | 696513e | 2017-02-08 05:27:07 -0800 | [diff] [blame] | 62 | } else { |
| 63 | u32 min, max, x; |
| 64 | u32 credits; |
Brian Welty | 696513e | 2017-02-08 05:27:07 -0800 | [diff] [blame] | 65 | u32 head; |
| 66 | u32 tail; |
| 67 | |
Kamenee Arumugam | f592ae3 | 2019-06-28 14:04:30 -0400 | [diff] [blame] | 68 | credits = READ_ONCE(qp->r_rq.kwq->count); |
| 69 | if (credits == 0) { |
| 70 | /* sanity check pointers before trusting them */ |
| 71 | if (qp->ip) { |
| 72 | head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head); |
| 73 | tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail); |
| 74 | } else { |
| 75 | head = READ_ONCE(qp->r_rq.kwq->head); |
| 76 | tail = READ_ONCE(qp->r_rq.kwq->tail); |
| 77 | } |
| 78 | if (head >= qp->r_rq.size) |
| 79 | head = 0; |
| 80 | if (tail >= qp->r_rq.size) |
| 81 | tail = 0; |
| 82 | /* |
| 83 | * Compute the number of credits available (RWQEs). |
| 84 | * There is a small chance that the pair of reads are |
| 85 | * not atomic, which is OK, since the fuzziness is |
| 86 | * resolved as further ACKs go out. |
| 87 | */ |
Mike Marciniszyn | 54a485e | 2020-07-28 14:38:48 -0400 | [diff] [blame] | 88 | credits = rvt_get_rq_count(&qp->r_rq, head, tail); |
Kamenee Arumugam | dabac6e | 2019-06-28 14:04:24 -0400 | [diff] [blame] | 89 | } |
Brian Welty | 696513e | 2017-02-08 05:27:07 -0800 | [diff] [blame] | 90 | /* |
| 91 | * Binary search the credit table to find the code to |
| 92 | * use. |
| 93 | */ |
| 94 | min = 0; |
| 95 | max = 31; |
| 96 | for (;;) { |
| 97 | x = (min + max) / 2; |
| 98 | if (credit_table[x] == credits) |
| 99 | break; |
| 100 | if (credit_table[x] > credits) { |
| 101 | max = x; |
| 102 | } else { |
| 103 | if (min == x) |
| 104 | break; |
| 105 | min = x; |
| 106 | } |
| 107 | } |
Don Hiatt | 832666c | 2017-02-08 05:28:25 -0800 | [diff] [blame] | 108 | aeth |= x << IB_AETH_CREDIT_SHIFT; |
Brian Welty | 696513e | 2017-02-08 05:27:07 -0800 | [diff] [blame] | 109 | } |
| 110 | return cpu_to_be32(aeth); |
| 111 | } |
| 112 | EXPORT_SYMBOL(rvt_compute_aeth); |
| 113 | |
| 114 | /** |
| 115 | * rvt_get_credit - flush the send work queue of a QP |
| 116 | * @qp: the qp who's send work queue to flush |
| 117 | * @aeth: the Acknowledge Extended Transport Header |
| 118 | * |
| 119 | * The QP s_lock should be held. |
| 120 | */ |
| 121 | void rvt_get_credit(struct rvt_qp *qp, u32 aeth) |
| 122 | { |
| 123 | struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); |
Don Hiatt | 832666c | 2017-02-08 05:28:25 -0800 | [diff] [blame] | 124 | u32 credit = (aeth >> IB_AETH_CREDIT_SHIFT) & IB_AETH_CREDIT_MASK; |
Brian Welty | 696513e | 2017-02-08 05:27:07 -0800 | [diff] [blame] | 125 | |
| 126 | lockdep_assert_held(&qp->s_lock); |
| 127 | /* |
| 128 | * If the credit is invalid, we can send |
| 129 | * as many packets as we like. Otherwise, we have to |
| 130 | * honor the credit field. |
| 131 | */ |
Don Hiatt | 832666c | 2017-02-08 05:28:25 -0800 | [diff] [blame] | 132 | if (credit == IB_AETH_CREDIT_INVAL) { |
Brian Welty | 696513e | 2017-02-08 05:27:07 -0800 | [diff] [blame] | 133 | if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) { |
| 134 | qp->s_flags |= RVT_S_UNLIMITED_CREDIT; |
| 135 | if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) { |
| 136 | qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT; |
| 137 | rdi->driver_f.schedule_send(qp); |
| 138 | } |
| 139 | } |
| 140 | } else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) { |
| 141 | /* Compute new LSN (i.e., MSN + credit) */ |
Don Hiatt | 832666c | 2017-02-08 05:28:25 -0800 | [diff] [blame] | 142 | credit = (aeth + credit_table[credit]) & IB_MSN_MASK; |
Brian Welty | 696513e | 2017-02-08 05:27:07 -0800 | [diff] [blame] | 143 | if (rvt_cmp_msn(credit, qp->s_lsn) > 0) { |
| 144 | qp->s_lsn = credit; |
| 145 | if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) { |
| 146 | qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT; |
| 147 | rdi->driver_f.schedule_send(qp); |
| 148 | } |
| 149 | } |
| 150 | } |
| 151 | } |
| 152 | EXPORT_SYMBOL(rvt_get_credit); |
Kaike Wan | 385156c | 2019-01-23 19:29:44 -0800 | [diff] [blame] | 153 | |
Mike Marciniszyn | 4ad6429 | 2019-12-19 16:19:34 -0500 | [diff] [blame] | 154 | /** |
| 155 | * rvt_restart_sge - rewind the sge state for a wqe |
| 156 | * @ss: the sge state pointer |
| 157 | * @wqe: the wqe to rewind |
| 158 | * @len: the data length from the start of the wqe in bytes |
| 159 | * |
| 160 | * Returns the remaining data length. |
| 161 | */ |
Kaike Wan | 385156c | 2019-01-23 19:29:44 -0800 | [diff] [blame] | 162 | u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len) |
| 163 | { |
| 164 | ss->sge = wqe->sg_list[0]; |
| 165 | ss->sg_list = wqe->sg_list + 1; |
| 166 | ss->num_sge = wqe->wr.num_sge; |
| 167 | ss->total_len = wqe->length; |
| 168 | rvt_skip_sge(ss, len, false); |
| 169 | return wqe->length - len; |
| 170 | } |
| 171 | EXPORT_SYMBOL(rvt_restart_sge); |
| 172 | |