drivers/vdpa/octeon_ep/octep_vdpa_hw.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2024 Marvell. */

#include <linux/iopoll.h>

#include "octep_vdpa.h"

enum octep_mbox_ids {
	OCTEP_MBOX_MSG_SET_VQ_STATE = 1,
	OCTEP_MBOX_MSG_GET_VQ_STATE,
};

#define OCTEP_HW_TIMEOUT       10000000

#define MBOX_OFFSET            64
#define MBOX_RSP_MASK          0x00000001
#define MBOX_RC_MASK           0x0000FFFE

#define MBOX_RSP_TO_ERR(val)   (-(((val) & MBOX_RC_MASK) >> 2))
#define MBOX_AVAIL(val)        (((val) & MBOX_RSP_MASK))
#define MBOX_RSP(val)          ((val) & (MBOX_RC_MASK | MBOX_RSP_MASK))

#define DEV_RST_ACK_BIT        7
#define FEATURE_SEL_ACK_BIT    15
#define QUEUE_SEL_ACK_BIT      15

struct octep_mbox_hdr {
	u8 ver;
	u8 rsvd1;
	u16 id;
	u16 rsvd2;
#define MBOX_REQ_SIG (0xdead)
#define MBOX_RSP_SIG (0xbeef)
	u16 sig;
};

struct octep_mbox_sts {
	u16 rsp:1;
	u16 rc:15;
	u16 rsvd;
};

struct octep_mbox {
	struct octep_mbox_hdr hdr;
	struct octep_mbox_sts sts;
	u64 rsvd;
	u32 data[];
};

static inline struct octep_mbox __iomem *octep_get_mbox(struct octep_hw *oct_hw)
{
	return (struct octep_mbox __iomem *)(oct_hw->dev_cfg + MBOX_OFFSET);
}

static inline int octep_wait_for_mbox_avail(struct octep_mbox __iomem *mbox)
{
	u32 val;

	return readx_poll_timeout(ioread32, &mbox->sts, val, MBOX_AVAIL(val), 10,
				  OCTEP_HW_TIMEOUT);
}

static inline int octep_wait_for_mbox_rsp(struct octep_mbox __iomem *mbox)
{
	u32 val;

	return readx_poll_timeout(ioread32, &mbox->sts, val, MBOX_RSP(val), 10,
				  OCTEP_HW_TIMEOUT);
}

static inline void octep_write_hdr(struct octep_mbox __iomem *mbox, u16 id, u16 sig)
{
	iowrite16(id, &mbox->hdr.id);
	iowrite16(sig, &mbox->hdr.sig);
}

static inline u32 octep_read_sig(struct octep_mbox __iomem *mbox)
{
	return ioread16(&mbox->hdr.sig);
}

static inline void octep_write_sts(struct octep_mbox __iomem *mbox, u32 sts)
{
	iowrite32(sts, &mbox->sts);
}

static inline u32 octep_read_sts(struct octep_mbox __iomem *mbox)
{
	return ioread32(&mbox->sts);
}

static inline u32 octep_read32_word(struct octep_mbox __iomem *mbox, u16 word_idx)
{
	return ioread32(&mbox->data[word_idx]);
}

static inline void octep_write32_word(struct octep_mbox __iomem *mbox, u16 word_idx, u32 word)
{
	return iowrite32(word, &mbox->data[word_idx]);
}

static int octep_process_mbox(struct octep_hw *oct_hw, u16 id, u16 qid, void *buffer,
			      u32 buf_size, bool write)
{
	struct octep_mbox __iomem *mbox = octep_get_mbox(oct_hw);
	struct pci_dev *pdev = oct_hw->pdev;
	u32 *p = (u32 *)buffer;
	u16 data_wds;
	int ret, i;
	u32 val;

	if (!IS_ALIGNED(buf_size, 4))
		return -EINVAL;

	/* Make sure mbox space is available */
	ret = octep_wait_for_mbox_avail(mbox);
	if (ret) {
		dev_warn(&pdev->dev, "Timeout waiting for previous mbox data to be consumed\n");
		return ret;
	}
	data_wds = buf_size / 4;

	if (write) {
		for (i = 1; i <= data_wds; i++) {
			octep_write32_word(mbox, i, *p);
			p++;
		}
	}
	octep_write32_word(mbox, 0, (u32)qid);
	octep_write_sts(mbox, 0);

	octep_write_hdr(mbox, id, MBOX_REQ_SIG);

	ret = octep_wait_for_mbox_rsp(mbox);
	if (ret) {
		dev_warn(&pdev->dev, "Timeout waiting for mbox : %d response\n", id);
		return ret;
	}

	val = octep_read_sig(mbox);
	if ((val & 0xFFFF) != MBOX_RSP_SIG) {
		dev_warn(&pdev->dev, "Invalid Signature from mbox : %d response\n", id);
		return -EINVAL;
	}

	val = octep_read_sts(mbox);
	if (val & MBOX_RC_MASK) {
		ret = MBOX_RSP_TO_ERR(val);
		dev_warn(&pdev->dev, "Error while processing mbox : %d, err %d\n", id, ret);
		return ret;
	}

	if (!write)
		for (i = 1; i <= data_wds; i++)
			*p++ = octep_read32_word(mbox, i);

	return 0;
}

static void octep_mbox_init(struct octep_mbox __iomem *mbox)
{
	iowrite32(1, &mbox->sts);
}

int octep_verify_features(u64 features)
{
	/* Minimum features to expect */
	if (!(features & BIT_ULL(VIRTIO_F_VERSION_1)))
		return -EOPNOTSUPP;

	if (!(features & BIT_ULL(VIRTIO_F_NOTIFICATION_DATA)))
		return -EOPNOTSUPP;

	if (!(features & BIT_ULL(VIRTIO_F_RING_PACKED)))
		return -EOPNOTSUPP;

	return 0;
}

u8 octep_hw_get_status(struct octep_hw *oct_hw)
{
	return ioread8(&oct_hw->common_cfg->device_status);
}

void octep_hw_set_status(struct octep_hw *oct_hw, u8 status)
{
	iowrite8(status, &oct_hw->common_cfg->device_status);
}

void octep_hw_reset(struct octep_hw *oct_hw)
{
	u8 val;

	octep_hw_set_status(oct_hw, 0 | BIT(DEV_RST_ACK_BIT));
	if (readx_poll_timeout(ioread8, &oct_hw->common_cfg->device_status, val, !val, 10,
			       OCTEP_HW_TIMEOUT)) {
		dev_warn(&oct_hw->pdev->dev, "Octeon device reset timeout\n");
		return;
	}
}

static int feature_sel_write_with_timeout(struct octep_hw *oct_hw, u32 select, void __iomem *addr)
{
	u32 val;

	iowrite32(select | BIT(FEATURE_SEL_ACK_BIT), addr);

	if (readx_poll_timeout(ioread32, addr, val, val == select, 10, OCTEP_HW_TIMEOUT)) {
		dev_warn(&oct_hw->pdev->dev, "Feature select%d write timeout\n", select);
		return -1;
	}
	return 0;
}

u64 octep_hw_get_dev_features(struct octep_hw *oct_hw)
{
	u32 features_lo, features_hi;

	if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->device_feature_select))
		return 0;

	features_lo = ioread32(&oct_hw->common_cfg->device_feature);

	if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->device_feature_select))
		return 0;

	features_hi = ioread32(&oct_hw->common_cfg->device_feature);

	return ((u64)features_hi << 32) | features_lo;
}

u64 octep_hw_get_drv_features(struct octep_hw *oct_hw)
{
	u32 features_lo, features_hi;

	if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->guest_feature_select))
		return 0;

	features_lo = ioread32(&oct_hw->common_cfg->guest_feature);

	if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->guest_feature_select))
		return 0;

	features_hi = ioread32(&oct_hw->common_cfg->guest_feature);

	return ((u64)features_hi << 32) | features_lo;
}

void octep_hw_set_drv_features(struct octep_hw *oct_hw, u64 features)
{
	if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->guest_feature_select))
		return;

	iowrite32(features & (BIT_ULL(32) - 1), &oct_hw->common_cfg->guest_feature);

	if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->guest_feature_select))
		return;

	iowrite32(features >> 32, &oct_hw->common_cfg->guest_feature);
}

void octep_write_queue_select(struct octep_hw *oct_hw, u16 queue_id)
{
	u16 val;

	iowrite16(queue_id | BIT(QUEUE_SEL_ACK_BIT), &oct_hw->common_cfg->queue_select);

	if (readx_poll_timeout(ioread16, &oct_hw->common_cfg->queue_select, val, val == queue_id,
			       10, OCTEP_HW_TIMEOUT)) {
		dev_warn(&oct_hw->pdev->dev, "Queue select write timeout\n");
		return;
	}
}

void octep_notify_queue(struct octep_hw *oct_hw, u16 qid)
{
	iowrite16(qid, oct_hw->vqs[qid].notify_addr);
}

void octep_read_dev_config(struct octep_hw *oct_hw, u64 offset, void *dst, int length)
{
	u8 old_gen, new_gen, *p;
	int i;

	if (WARN_ON(offset + length > oct_hw->config_size))
		return;

	do {
		old_gen = ioread8(&oct_hw->common_cfg->config_generation);
		p = dst;
		for (i = 0; i < length; i++)
			*p++ = ioread8(oct_hw->dev_cfg + offset + i);

		new_gen = ioread8(&oct_hw->common_cfg->config_generation);
	} while (old_gen != new_gen);
}

int octep_set_vq_address(struct octep_hw *oct_hw, u16 qid, u64 desc_area, u64 driver_area,
			 u64 device_area)
{
	struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;

	octep_write_queue_select(oct_hw, qid);
	vp_iowrite64_twopart(desc_area, &cfg->queue_desc_lo,
			     &cfg->queue_desc_hi);
	vp_iowrite64_twopart(driver_area, &cfg->queue_avail_lo,
			     &cfg->queue_avail_hi);
	vp_iowrite64_twopart(device_area, &cfg->queue_used_lo,
			     &cfg->queue_used_hi);

	return 0;
}

int octep_get_vq_state(struct octep_hw *oct_hw, u16 qid, struct vdpa_vq_state *state)
{
	return octep_process_mbox(oct_hw, OCTEP_MBOX_MSG_GET_VQ_STATE, qid, state,
				  sizeof(*state), 0);
}

int octep_set_vq_state(struct octep_hw *oct_hw, u16 qid, const struct vdpa_vq_state *state)
{
	struct vdpa_vq_state q_state;

	memcpy(&q_state, state, sizeof(struct vdpa_vq_state));
	return octep_process_mbox(oct_hw, OCTEP_MBOX_MSG_SET_VQ_STATE, qid, &q_state,
				  sizeof(*state), 1);
}

void octep_set_vq_num(struct octep_hw *oct_hw, u16 qid, u32 num)
{
	struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;

	octep_write_queue_select(oct_hw, qid);
	iowrite16(num, &cfg->queue_size);
}

void octep_set_vq_ready(struct octep_hw *oct_hw, u16 qid, bool ready)
{
	struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;

	octep_write_queue_select(oct_hw, qid);
	iowrite16(ready, &cfg->queue_enable);
}

bool octep_get_vq_ready(struct octep_hw *oct_hw, u16 qid)
{
	struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;

	octep_write_queue_select(oct_hw, qid);
	return ioread16(&cfg->queue_enable);
}

u16 octep_get_vq_size(struct octep_hw *oct_hw)
{
	octep_write_queue_select(oct_hw, 0);
	return ioread16(&oct_hw->common_cfg->queue_size);
}

static u32 octep_get_config_size(struct octep_hw *oct_hw)
{
	return sizeof(struct virtio_net_config);
}

static void __iomem *octep_get_cap_addr(struct octep_hw *oct_hw, struct virtio_pci_cap *cap)
{
	struct device *dev = &oct_hw->pdev->dev;
	u32 length = le32_to_cpu(cap->length);
	u32 offset = le32_to_cpu(cap->offset);
	u8  bar    = cap->bar;
	u32 len;

	if (bar != OCTEP_HW_CAPS_BAR) {
		dev_err(dev, "Invalid bar: %u\n", bar);
		return NULL;
	}
	if (offset + length < offset) {
		dev_err(dev, "offset(%u) + length(%u) overflows\n",
			offset, length);
		return NULL;
	}
	len = pci_resource_len(oct_hw->pdev, bar);
	if (offset + length > len) {
		dev_err(dev, "invalid cap: overflows bar space: %u > %u\n",
			offset + length, len);
		return NULL;
	}
	return oct_hw->base[bar] + offset;
}

/* In Octeon DPU device, the virtio config space is completely
 * emulated by the device's firmware. So, the standard pci config
 * read apis can't be used for reading the virtio capability.
 */
static void octep_pci_caps_read(struct octep_hw *oct_hw, void *buf, size_t len, off_t offset)
{
	u8 __iomem *bar = oct_hw->base[OCTEP_HW_CAPS_BAR];
	u8 *p = buf;
	size_t i;

	for (i = 0; i < len; i++)
		*p++ = ioread8(bar + offset + i);
}

static int octep_pci_signature_verify(struct octep_hw *oct_hw)
{
	u32 signature[2];

	octep_pci_caps_read(oct_hw, &signature, sizeof(signature), 0);

	if (signature[0] != OCTEP_FW_READY_SIGNATURE0)
		return -1;

	if (signature[1] != OCTEP_FW_READY_SIGNATURE1)
		return -1;

	return 0;
}

int octep_hw_caps_read(struct octep_hw *oct_hw, struct pci_dev *pdev)
{
	struct octep_mbox __iomem *mbox;
	struct device *dev = &pdev->dev;
	struct virtio_pci_cap cap;
	u16 notify_off;
	int i, ret;
	u8 pos;

	oct_hw->pdev = pdev;
	ret = octep_pci_signature_verify(oct_hw);
	if (ret) {
		dev_err(dev, "Octeon Virtio FW is not initialized\n");
		return -EIO;
	}

	octep_pci_caps_read(oct_hw, &pos, 1, PCI_CAPABILITY_LIST);

	while (pos) {
		octep_pci_caps_read(oct_hw, &cap, 2, pos);

		if (cap.cap_vndr != PCI_CAP_ID_VNDR) {
			dev_err(dev, "Found invalid capability vndr id: %d\n", cap.cap_vndr);
			break;
		}

		octep_pci_caps_read(oct_hw, &cap, sizeof(cap), pos);

		dev_info(dev, "[%2x] cfg type: %u, bar: %u, offset: %04x, len: %u\n",
			 pos, cap.cfg_type, cap.bar, cap.offset, cap.length);

		switch (cap.cfg_type) {
		case VIRTIO_PCI_CAP_COMMON_CFG:
			oct_hw->common_cfg = octep_get_cap_addr(oct_hw, &cap);
			break;
		case VIRTIO_PCI_CAP_NOTIFY_CFG:
			octep_pci_caps_read(oct_hw, &oct_hw->notify_off_multiplier,
					    4, pos + sizeof(cap));

			oct_hw->notify_base = octep_get_cap_addr(oct_hw, &cap);
			oct_hw->notify_bar = cap.bar;
			oct_hw->notify_base_pa = pci_resource_start(pdev, cap.bar) +
						 le32_to_cpu(cap.offset);
			break;
		case VIRTIO_PCI_CAP_DEVICE_CFG:
			oct_hw->dev_cfg = octep_get_cap_addr(oct_hw, &cap);
			break;
		case VIRTIO_PCI_CAP_ISR_CFG:
			oct_hw->isr = octep_get_cap_addr(oct_hw, &cap);
			break;
		}

		pos = cap.cap_next;
	}
	if (!oct_hw->common_cfg || !oct_hw->notify_base ||
	    !oct_hw->dev_cfg    || !oct_hw->isr) {
		dev_err(dev, "Incomplete PCI capabilities");
		return -EIO;
	}
	dev_info(dev, "common cfg mapped at: 0x%016llx\n", (u64)(uintptr_t)oct_hw->common_cfg);
	dev_info(dev, "device cfg mapped at: 0x%016llx\n", (u64)(uintptr_t)oct_hw->dev_cfg);
	dev_info(dev, "isr cfg mapped at: 0x%016llx\n", (u64)(uintptr_t)oct_hw->isr);
	dev_info(dev, "notify base: 0x%016llx, notify off multiplier: %u\n",
		 (u64)(uintptr_t)oct_hw->notify_base, oct_hw->notify_off_multiplier);

	oct_hw->config_size = octep_get_config_size(oct_hw);
	oct_hw->features = octep_hw_get_dev_features(oct_hw);

	ret = octep_verify_features(oct_hw->features);
	if (ret) {
		dev_err(&pdev->dev, "Couldn't read features from the device FW\n");
		return ret;
	}
	oct_hw->nr_vring = vp_ioread16(&oct_hw->common_cfg->num_queues);

	oct_hw->vqs = devm_kcalloc(&pdev->dev, oct_hw->nr_vring, sizeof(*oct_hw->vqs), GFP_KERNEL);
	if (!oct_hw->vqs)
		return -ENOMEM;

	oct_hw->irq = -1;

	dev_info(&pdev->dev, "Device features : %llx\n", oct_hw->features);
	dev_info(&pdev->dev, "Maximum queues : %u\n", oct_hw->nr_vring);

	for (i = 0; i < oct_hw->nr_vring; i++) {
		octep_write_queue_select(oct_hw, i);
		notify_off = vp_ioread16(&oct_hw->common_cfg->queue_notify_off);
		oct_hw->vqs[i].notify_addr = oct_hw->notify_base +
			notify_off * oct_hw->notify_off_multiplier;
		oct_hw->vqs[i].cb_notify_addr = (u32 __iomem *)oct_hw->vqs[i].notify_addr + 1;
		oct_hw->vqs[i].notify_pa = oct_hw->notify_base_pa +
			notify_off * oct_hw->notify_off_multiplier;
	}
	mbox = octep_get_mbox(oct_hw);
	octep_mbox_init(mbox);
	dev_info(dev, "mbox mapped at: 0x%016llx\n", (u64)(uintptr_t)mbox);

	return 0;
}