| // SPDX-License-Identifier: GPL-2.0 | |
| /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ | |
| #include <linux/dma-mapping.h> | |
| #include "ionic_fw.h" | |
| #include "ionic_ibdev.h" | |
| static int ionic_v1_stat_normalize(struct ionic_v1_stat *hw_stats, | |
| int hw_stats_count) | |
| { | |
| int hw_stat_i; | |
| for (hw_stat_i = 0; hw_stat_i < hw_stats_count; ++hw_stat_i) { | |
| struct ionic_v1_stat *stat = &hw_stats[hw_stat_i]; | |
| stat->type_off = be32_to_cpu(stat->be_type_off); | |
| stat->name[sizeof(stat->name) - 1] = 0; | |
| if (ionic_v1_stat_type(stat) == IONIC_V1_STAT_TYPE_NONE) | |
| break; | |
| } | |
| return hw_stat_i; | |
| } | |
| static void ionic_fill_stats_desc(struct rdma_stat_desc *hw_stats_hdrs, | |
| struct ionic_v1_stat *hw_stats, | |
| int hw_stats_count) | |
| { | |
| int hw_stat_i; | |
| for (hw_stat_i = 0; hw_stat_i < hw_stats_count; ++hw_stat_i) { | |
| struct ionic_v1_stat *stat = &hw_stats[hw_stat_i]; | |
| hw_stats_hdrs[hw_stat_i].name = stat->name; | |
| } | |
| } | |
| static u64 ionic_v1_stat_val(struct ionic_v1_stat *stat, | |
| void *vals_buf, size_t vals_len) | |
| { | |
| unsigned int off = ionic_v1_stat_off(stat); | |
| int type = ionic_v1_stat_type(stat); | |
| #define __ionic_v1_stat_validate(__type) \ | |
| ((off + sizeof(__type) <= vals_len) && \ | |
| (IS_ALIGNED(off, sizeof(__type)))) | |
| switch (type) { | |
| case IONIC_V1_STAT_TYPE_8: | |
| if (__ionic_v1_stat_validate(u8)) | |
| return *(u8 *)(vals_buf + off); | |
| break; | |
| case IONIC_V1_STAT_TYPE_LE16: | |
| if (__ionic_v1_stat_validate(__le16)) | |
| return le16_to_cpu(*(__le16 *)(vals_buf + off)); | |
| break; | |
| case IONIC_V1_STAT_TYPE_LE32: | |
| if (__ionic_v1_stat_validate(__le32)) | |
| return le32_to_cpu(*(__le32 *)(vals_buf + off)); | |
| break; | |
| case IONIC_V1_STAT_TYPE_LE64: | |
| if (__ionic_v1_stat_validate(__le64)) | |
| return le64_to_cpu(*(__le64 *)(vals_buf + off)); | |
| break; | |
| case IONIC_V1_STAT_TYPE_BE16: | |
| if (__ionic_v1_stat_validate(__be16)) | |
| return be16_to_cpu(*(__be16 *)(vals_buf + off)); | |
| break; | |
| case IONIC_V1_STAT_TYPE_BE32: | |
| if (__ionic_v1_stat_validate(__be32)) | |
| return be32_to_cpu(*(__be32 *)(vals_buf + off)); | |
| break; | |
| case IONIC_V1_STAT_TYPE_BE64: | |
| if (__ionic_v1_stat_validate(__be64)) | |
| return be64_to_cpu(*(__be64 *)(vals_buf + off)); | |
| break; | |
| } | |
| return ~0ull; | |
| #undef __ionic_v1_stat_validate | |
| } | |
| static int ionic_hw_stats_cmd(struct ionic_ibdev *dev, | |
| dma_addr_t dma, size_t len, int qid, int op) | |
| { | |
| struct ionic_admin_wr wr = { | |
| .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), | |
| .wqe = { | |
| .op = op, | |
| .len = cpu_to_le16(IONIC_ADMIN_STATS_HDRS_IN_V1_LEN), | |
| .cmd.stats = { | |
| .dma_addr = cpu_to_le64(dma), | |
| .length = cpu_to_le32(len), | |
| .id_ver = cpu_to_le32(qid), | |
| }, | |
| } | |
| }; | |
| if (dev->lif_cfg.admin_opcodes <= op) | |
| return -EBADRQC; | |
| ionic_admin_post(dev, &wr); | |
| return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_INTERRUPT); | |
| } | |
| static int ionic_init_hw_stats(struct ionic_ibdev *dev) | |
| { | |
| dma_addr_t hw_stats_dma; | |
| int rc, hw_stats_count; | |
| if (dev->hw_stats_hdrs) | |
| return 0; | |
| dev->hw_stats_count = 0; | |
| /* buffer for current values from the device */ | |
| dev->hw_stats_buf = kzalloc(PAGE_SIZE, GFP_KERNEL); | |
| if (!dev->hw_stats_buf) { | |
| rc = -ENOMEM; | |
| goto err_buf; | |
| } | |
| /* buffer for names, sizes, offsets of values */ | |
| dev->hw_stats = kzalloc(PAGE_SIZE, GFP_KERNEL); | |
| if (!dev->hw_stats) { | |
| rc = -ENOMEM; | |
| goto err_hw_stats; | |
| } | |
| /* request the names, sizes, offsets */ | |
| hw_stats_dma = dma_map_single(dev->lif_cfg.hwdev, dev->hw_stats, | |
| PAGE_SIZE, DMA_FROM_DEVICE); | |
| rc = dma_mapping_error(dev->lif_cfg.hwdev, hw_stats_dma); | |
| if (rc) | |
| goto err_dma; | |
| rc = ionic_hw_stats_cmd(dev, hw_stats_dma, PAGE_SIZE, 0, | |
| IONIC_V1_ADMIN_STATS_HDRS); | |
| if (rc) | |
| goto err_cmd; | |
| dma_unmap_single(dev->lif_cfg.hwdev, hw_stats_dma, PAGE_SIZE, DMA_FROM_DEVICE); | |
| /* normalize and count the number of hw_stats */ | |
| hw_stats_count = | |
| ionic_v1_stat_normalize(dev->hw_stats, | |
| PAGE_SIZE / sizeof(*dev->hw_stats)); | |
| if (!hw_stats_count) { | |
| rc = -ENODATA; | |
| goto err_dma; | |
| } | |
| dev->hw_stats_count = hw_stats_count; | |
| /* alloc and init array of names, for alloc_hw_stats */ | |
| dev->hw_stats_hdrs = kzalloc_objs(*dev->hw_stats_hdrs, hw_stats_count); | |
| if (!dev->hw_stats_hdrs) { | |
| rc = -ENOMEM; | |
| goto err_dma; | |
| } | |
| ionic_fill_stats_desc(dev->hw_stats_hdrs, dev->hw_stats, | |
| hw_stats_count); | |
| return 0; | |
| err_cmd: | |
| dma_unmap_single(dev->lif_cfg.hwdev, hw_stats_dma, PAGE_SIZE, DMA_FROM_DEVICE); | |
| err_dma: | |
| kfree(dev->hw_stats); | |
| err_hw_stats: | |
| kfree(dev->hw_stats_buf); | |
| err_buf: | |
| dev->hw_stats_count = 0; | |
| dev->hw_stats = NULL; | |
| dev->hw_stats_buf = NULL; | |
| dev->hw_stats_hdrs = NULL; | |
| return rc; | |
| } | |
| static struct rdma_hw_stats *ionic_alloc_hw_stats(struct ib_device *ibdev, | |
| u32 port) | |
| { | |
| struct ionic_ibdev *dev = to_ionic_ibdev(ibdev); | |
| if (port != 1) | |
| return NULL; | |
| return rdma_alloc_hw_stats_struct(dev->hw_stats_hdrs, | |
| dev->hw_stats_count, | |
| RDMA_HW_STATS_DEFAULT_LIFESPAN); | |
| } | |
| static int ionic_get_hw_stats(struct ib_device *ibdev, | |
| struct rdma_hw_stats *hw_stats, | |
| u32 port, int index) | |
| { | |
| struct ionic_ibdev *dev = to_ionic_ibdev(ibdev); | |
| dma_addr_t hw_stats_dma; | |
| int rc, hw_stat_i; | |
| if (port != 1) | |
| return -EINVAL; | |
| hw_stats_dma = dma_map_single(dev->lif_cfg.hwdev, dev->hw_stats_buf, | |
| PAGE_SIZE, DMA_FROM_DEVICE); | |
| rc = dma_mapping_error(dev->lif_cfg.hwdev, hw_stats_dma); | |
| if (rc) | |
| goto err_dma; | |
| rc = ionic_hw_stats_cmd(dev, hw_stats_dma, PAGE_SIZE, | |
| 0, IONIC_V1_ADMIN_STATS_VALS); | |
| if (rc) | |
| goto err_cmd; | |
| dma_unmap_single(dev->lif_cfg.hwdev, hw_stats_dma, | |
| PAGE_SIZE, DMA_FROM_DEVICE); | |
| for (hw_stat_i = 0; hw_stat_i < dev->hw_stats_count; ++hw_stat_i) | |
| hw_stats->value[hw_stat_i] = | |
| ionic_v1_stat_val(&dev->hw_stats[hw_stat_i], | |
| dev->hw_stats_buf, PAGE_SIZE); | |
| return hw_stat_i; | |
| err_cmd: | |
| dma_unmap_single(dev->lif_cfg.hwdev, hw_stats_dma, | |
| PAGE_SIZE, DMA_FROM_DEVICE); | |
| err_dma: | |
| return rc; | |
| } | |
| static struct rdma_hw_stats * | |
| ionic_counter_alloc_stats(struct rdma_counter *counter) | |
| { | |
| struct ionic_rdma_counter *cntr = to_ionic_rdma_counter(counter); | |
| struct ionic_ibdev *dev = to_ionic_ibdev(counter->device); | |
| struct rdma_hw_stats *stats; | |
| int id; | |
| cntr->vals = kzalloc(PAGE_SIZE, GFP_KERNEL); | |
| if (!cntr->vals) | |
| return NULL; | |
| id = ida_alloc_max(&dev->counter_stats->counter_ida, | |
| IONIC_MAX_QPID, GFP_KERNEL); | |
| if (id < 0) | |
| goto err_ida; | |
| counter->id = id; | |
| stats = rdma_alloc_hw_stats_struct(dev->counter_stats->stats_hdrs, | |
| dev->counter_stats->queue_stats_count, | |
| RDMA_HW_STATS_DEFAULT_LIFESPAN); | |
| if (!stats) | |
| goto err_hw_stats; | |
| return stats; | |
| err_hw_stats: | |
| ida_free(&dev->counter_stats->counter_ida, id); | |
| err_ida: | |
| kfree(cntr->vals); | |
| return NULL; | |
| } | |
| static int ionic_counter_dealloc(struct rdma_counter *counter) | |
| { | |
| struct ionic_ibdev *dev = to_ionic_ibdev(counter->device); | |
| struct ionic_rdma_counter *cntr = to_ionic_rdma_counter(counter); | |
| ida_free(&dev->counter_stats->counter_ida, counter->id); | |
| kfree(cntr->vals); | |
| return 0; | |
| } | |
| static int ionic_counter_bind_qp(struct rdma_counter *counter, | |
| struct ib_qp *ibqp, | |
| u32 port) | |
| { | |
| struct ionic_rdma_counter *cntr = to_ionic_rdma_counter(counter); | |
| struct ionic_qp *qp = to_ionic_qp(ibqp); | |
| list_add_tail(&qp->qp_list_counter, &cntr->qp_list); | |
| ibqp->counter = counter; | |
| return 0; | |
| } | |
| static int ionic_counter_unbind_qp(struct ib_qp *ibqp, u32 port) | |
| { | |
| struct ionic_qp *qp = to_ionic_qp(ibqp); | |
| if (ibqp->counter) { | |
| list_del(&qp->qp_list_counter); | |
| ibqp->counter = NULL; | |
| } | |
| return 0; | |
| } | |
| static int ionic_counter_update_stats(struct rdma_counter *counter) | |
| { | |
| struct ionic_rdma_counter *cntr = to_ionic_rdma_counter(counter); | |
| struct ionic_ibdev *dev = to_ionic_ibdev(counter->device); | |
| struct ionic_counter_stats *cs = dev->counter_stats; | |
| dma_addr_t hw_stats_dma; | |
| struct ionic_qp *qp; | |
| int rc, stat_i = 0; | |
| hw_stats_dma = dma_map_single(dev->lif_cfg.hwdev, cntr->vals, | |
| PAGE_SIZE, DMA_FROM_DEVICE); | |
| rc = dma_mapping_error(dev->lif_cfg.hwdev, hw_stats_dma); | |
| if (rc) | |
| return rc; | |
| memset(counter->stats->value, 0, | |
| sizeof(u64) * counter->stats->num_counters); | |
| list_for_each_entry(qp, &cntr->qp_list, qp_list_counter) { | |
| rc = ionic_hw_stats_cmd(dev, hw_stats_dma, PAGE_SIZE, | |
| qp->qpid, | |
| IONIC_V1_ADMIN_QP_STATS_VALS); | |
| if (rc) | |
| goto err_cmd; | |
| for (stat_i = 0; stat_i < cs->queue_stats_count; ++stat_i) | |
| counter->stats->value[stat_i] += | |
| ionic_v1_stat_val(&cs->hdr[stat_i], | |
| cntr->vals, | |
| PAGE_SIZE); | |
| } | |
| dma_unmap_single(dev->lif_cfg.hwdev, hw_stats_dma, PAGE_SIZE, DMA_FROM_DEVICE); | |
| return stat_i; | |
| err_cmd: | |
| dma_unmap_single(dev->lif_cfg.hwdev, hw_stats_dma, PAGE_SIZE, DMA_FROM_DEVICE); | |
| return rc; | |
| } | |
| static int ionic_alloc_counters(struct ionic_ibdev *dev) | |
| { | |
| struct ionic_counter_stats *cs = dev->counter_stats; | |
| int rc, hw_stats_count; | |
| dma_addr_t hdr_dma; | |
| /* buffer for names, sizes, offsets of values */ | |
| cs->hdr = kzalloc(PAGE_SIZE, GFP_KERNEL); | |
| if (!cs->hdr) | |
| return -ENOMEM; | |
| hdr_dma = dma_map_single(dev->lif_cfg.hwdev, cs->hdr, | |
| PAGE_SIZE, DMA_FROM_DEVICE); | |
| rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma); | |
| if (rc) | |
| goto err_dma; | |
| rc = ionic_hw_stats_cmd(dev, hdr_dma, PAGE_SIZE, 0, | |
| IONIC_V1_ADMIN_QP_STATS_HDRS); | |
| if (rc) | |
| goto err_cmd; | |
| dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, PAGE_SIZE, DMA_FROM_DEVICE); | |
| /* normalize and count the number of hw_stats */ | |
| hw_stats_count = ionic_v1_stat_normalize(cs->hdr, | |
| PAGE_SIZE / sizeof(*cs->hdr)); | |
| if (!hw_stats_count) { | |
| rc = -ENODATA; | |
| goto err_dma; | |
| } | |
| cs->queue_stats_count = hw_stats_count; | |
| /* alloc and init array of names */ | |
| cs->stats_hdrs = kzalloc_objs(*cs->stats_hdrs, hw_stats_count); | |
| if (!cs->stats_hdrs) { | |
| rc = -ENOMEM; | |
| goto err_dma; | |
| } | |
| ionic_fill_stats_desc(cs->stats_hdrs, cs->hdr, hw_stats_count); | |
| return 0; | |
| err_cmd: | |
| dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, PAGE_SIZE, DMA_FROM_DEVICE); | |
| err_dma: | |
| kfree(cs->hdr); | |
| return rc; | |
| } | |
| static const struct ib_device_ops ionic_hw_stats_ops = { | |
| .driver_id = RDMA_DRIVER_IONIC, | |
| .alloc_hw_port_stats = ionic_alloc_hw_stats, | |
| .get_hw_stats = ionic_get_hw_stats, | |
| }; | |
| static void ionic_counter_init(struct rdma_counter *counter) | |
| { | |
| struct ionic_rdma_counter *cntr = to_ionic_rdma_counter(counter); | |
| INIT_LIST_HEAD(&cntr->qp_list); | |
| } | |
| static const struct ib_device_ops ionic_counter_stats_ops = { | |
| .counter_alloc_stats = ionic_counter_alloc_stats, | |
| .counter_dealloc = ionic_counter_dealloc, | |
| .counter_bind_qp = ionic_counter_bind_qp, | |
| .counter_unbind_qp = ionic_counter_unbind_qp, | |
| .counter_update_stats = ionic_counter_update_stats, | |
| .counter_init = ionic_counter_init, | |
| INIT_RDMA_OBJ_SIZE(rdma_counter, ionic_rdma_counter, rdma_counter), | |
| }; | |
| void ionic_stats_init(struct ionic_ibdev *dev) | |
| { | |
| u16 stats_type = dev->lif_cfg.stats_type; | |
| int rc; | |
| if (stats_type & IONIC_LIF_RDMA_STAT_GLOBAL) { | |
| rc = ionic_init_hw_stats(dev); | |
| if (rc) | |
| ibdev_dbg(&dev->ibdev, "Failed to init hw stats\n"); | |
| else | |
| ib_set_device_ops(&dev->ibdev, &ionic_hw_stats_ops); | |
| } | |
| if (stats_type & IONIC_LIF_RDMA_STAT_QP) { | |
| dev->counter_stats = kzalloc_obj(*dev->counter_stats); | |
| if (!dev->counter_stats) | |
| return; | |
| rc = ionic_alloc_counters(dev); | |
| if (rc) { | |
| ibdev_dbg(&dev->ibdev, "Failed to init counter stats\n"); | |
| kfree(dev->counter_stats); | |
| dev->counter_stats = NULL; | |
| return; | |
| } | |
| ida_init(&dev->counter_stats->counter_ida); | |
| ib_set_device_ops(&dev->ibdev, &ionic_counter_stats_ops); | |
| } | |
| } | |
| void ionic_stats_cleanup(struct ionic_ibdev *dev) | |
| { | |
| if (dev->counter_stats) { | |
| ida_destroy(&dev->counter_stats->counter_ida); | |
| kfree(dev->counter_stats->hdr); | |
| kfree(dev->counter_stats->stats_hdrs); | |
| kfree(dev->counter_stats); | |
| dev->counter_stats = NULL; | |
| } | |
| kfree(dev->hw_stats); | |
| kfree(dev->hw_stats_buf); | |
| kfree(dev->hw_stats_hdrs); | |
| } |