drivers/net/ethernet/pensando/ionic/ionic_dev.c

// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */

#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/io.h>
#include <linux/slab.h>
#include <linux/etherdevice.h>
#include "ionic.h"
#include "ionic_dev.h"
#include "ionic_lif.h"

static void ionic_watchdog_cb(struct timer_list *t)
{
	struct ionic *ionic = timer_container_of(ionic, t, watchdog_timer);
	struct ionic_lif *lif = ionic->lif;
	struct ionic_deferred_work *work;
	int hb;

	mod_timer(&ionic->watchdog_timer,
		  round_jiffies(jiffies + ionic->watchdog_period));

	if (!lif)
		return;

	hb = ionic_heartbeat_check(ionic);
	dev_dbg(ionic->dev, "%s: hb %d running %d UP %d\n",
		__func__, hb, netif_running(lif->netdev),
		test_bit(IONIC_LIF_F_UP, lif->state));

	if (hb >= 0 &&
	    !test_bit(IONIC_LIF_F_FW_RESET, lif->state))
		ionic_link_status_check_request(lif, CAN_NOT_SLEEP);

	if (test_bit(IONIC_LIF_F_FILTER_SYNC_NEEDED, lif->state) &&
	    !test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
		work = kzalloc(sizeof(*work), GFP_ATOMIC);
		if (!work) {
			netdev_err(lif->netdev, "rxmode change dropped\n");
			return;
		}

		work->type = IONIC_DW_TYPE_RX_MODE;
		netdev_dbg(lif->netdev, "deferred: rx_mode\n");
		ionic_lif_deferred_enqueue(lif, work);
	}
}

static void ionic_napi_schedule_do_softirq(struct napi_struct *napi)
{
	local_bh_disable();
	napi_schedule(napi);
	local_bh_enable();
}

void ionic_doorbell_napi_work(struct work_struct *work)
{
	struct ionic_qcq *qcq = container_of(work, struct ionic_qcq,
					     doorbell_napi_work);
	unsigned long now, then, dif;

	now = READ_ONCE(jiffies);
	then = qcq->q.dbell_jiffies;
	dif = now - then;

	if (dif > qcq->q.dbell_deadline)
		ionic_napi_schedule_do_softirq(&qcq->napi);
}

static int ionic_get_preferred_cpu(struct ionic *ionic,
				   struct ionic_intr_info *intr)
{
	int cpu;

	cpu = cpumask_first_and(*intr->affinity_mask, cpu_online_mask);
	if (cpu >= nr_cpu_ids)
		cpu = cpumask_local_spread(0, dev_to_node(ionic->dev));

	return cpu;
}

static void ionic_queue_dbell_napi_work(struct ionic *ionic,
					struct ionic_qcq *qcq)
{
	int cpu;

	if (!(qcq->flags & IONIC_QCQ_F_INTR))
		return;

	cpu = ionic_get_preferred_cpu(ionic, &qcq->intr);
	queue_work_on(cpu, ionic->wq, &qcq->doorbell_napi_work);
}

static void ionic_doorbell_check_dwork(struct work_struct *work)
{
	struct ionic *ionic = container_of(work, struct ionic,
					   doorbell_check_dwork.work);
	struct ionic_lif *lif = ionic->lif;

	mutex_lock(&lif->queue_lock);

	if (test_bit(IONIC_LIF_F_FW_STOPPING, lif->state) ||
	    test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
		mutex_unlock(&lif->queue_lock);
		return;
	}

	ionic_napi_schedule_do_softirq(&lif->adminqcq->napi);

	if (test_bit(IONIC_LIF_F_UP, lif->state)) {
		int i;

		for (i = 0; i < lif->nxqs; i++) {
			ionic_queue_dbell_napi_work(ionic, lif->txqcqs[i]);
			ionic_queue_dbell_napi_work(ionic, lif->rxqcqs[i]);
		}

		if (lif->hwstamp_txq &&
		    lif->hwstamp_txq->flags & IONIC_QCQ_F_INTR)
			ionic_napi_schedule_do_softirq(&lif->hwstamp_txq->napi);
		if (lif->hwstamp_rxq &&
		    lif->hwstamp_rxq->flags & IONIC_QCQ_F_INTR)
			ionic_napi_schedule_do_softirq(&lif->hwstamp_rxq->napi);
	}
	mutex_unlock(&lif->queue_lock);

	ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE);
}

bool ionic_doorbell_wa(struct ionic *ionic)
{
	u8 asic_type = ionic->idev.dev_info.asic_type;

	return !asic_type || asic_type == IONIC_ASIC_TYPE_ELBA;
}

static int ionic_watchdog_init(struct ionic *ionic)
{
	struct ionic_dev *idev = &ionic->idev;

	timer_setup(&ionic->watchdog_timer, ionic_watchdog_cb, 0);
	ionic->watchdog_period = IONIC_WATCHDOG_SECS * HZ;

	/* set times to ensure the first check will proceed */
	atomic_long_set(&idev->last_check_time, jiffies - 2 * HZ);
	idev->last_hb_time = jiffies - 2 * ionic->watchdog_period;
	/* init as ready, so no transition if the first check succeeds */
	idev->last_fw_hb = 0;
	idev->fw_hb_ready = true;
	idev->fw_status_ready = true;
	idev->fw_generation = IONIC_FW_STS_F_GENERATION &
			      ioread8(&idev->dev_info_regs->fw_status);

	ionic->wq = alloc_workqueue("%s-wq", WQ_UNBOUND, 0,
				    dev_name(ionic->dev));
	if (!ionic->wq) {
		dev_err(ionic->dev, "alloc_workqueue failed");
		return -ENOMEM;
	}

	if (ionic_doorbell_wa(ionic))
		INIT_DELAYED_WORK(&ionic->doorbell_check_dwork,
				  ionic_doorbell_check_dwork);

	return 0;
}

void ionic_queue_doorbell_check(struct ionic *ionic, int delay)
{
	int cpu;

	if (!ionic->lif->doorbell_wa)
		return;

	cpu = ionic_get_preferred_cpu(ionic, &ionic->lif->adminqcq->intr);
	queue_delayed_work_on(cpu, ionic->wq, &ionic->doorbell_check_dwork,
			      delay);
}

void ionic_init_devinfo(struct ionic *ionic)
{
	struct ionic_dev *idev = &ionic->idev;

	idev->dev_info.asic_type = ioread8(&idev->dev_info_regs->asic_type);
	idev->dev_info.asic_rev = ioread8(&idev->dev_info_regs->asic_rev);

	memcpy_fromio(idev->dev_info.fw_version,
		      idev->dev_info_regs->fw_version,
		      IONIC_DEVINFO_FWVERS_BUFLEN);

	memcpy_fromio(idev->dev_info.serial_num,
		      idev->dev_info_regs->serial_num,
		      IONIC_DEVINFO_SERIAL_BUFLEN);

	idev->dev_info.fw_version[IONIC_DEVINFO_FWVERS_BUFLEN] = 0;
	idev->dev_info.serial_num[IONIC_DEVINFO_SERIAL_BUFLEN] = 0;

	dev_dbg(ionic->dev, "fw_version %s\n", idev->dev_info.fw_version);
}

static void ionic_map_disc_cmb(struct ionic *ionic)
{
	struct ionic_identity *ident = &ionic->ident;
	u32 length_reg0, length, offset, num_regions;
	struct ionic_dev_bar *bar = ionic->bars;
	struct ionic_dev *idev = &ionic->idev;
	struct device *dev = ionic->dev;
	int err, sz, i;
	u64 end;

	mutex_lock(&ionic->dev_cmd_lock);

	ionic_dev_cmd_discover_cmb(idev);
	err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
	if (!err) {
		sz = min(sizeof(ident->cmb_layout),
			 sizeof(idev->dev_cmd_regs->data));
		memcpy_fromio(&ident->cmb_layout,
			      &idev->dev_cmd_regs->data, sz);
	}
	mutex_unlock(&ionic->dev_cmd_lock);

	if (err) {
		dev_warn(dev, "Cannot discover CMB layout, disabling CMB\n");
		return;
	}

	bar += 2;

	num_regions = le32_to_cpu(ident->cmb_layout.num_regions);
	if (!num_regions || num_regions > IONIC_MAX_CMB_REGIONS) {
		dev_warn(dev, "Invalid number of CMB entries (%d)\n",
			 num_regions);
		return;
	}

	dev_dbg(dev, "ionic_cmb_layout_identity num_regions %d flags %x:\n",
		num_regions, ident->cmb_layout.flags);

	for (i = 0; i < num_regions; i++) {
		offset = le32_to_cpu(ident->cmb_layout.region[i].offset);
		length = le32_to_cpu(ident->cmb_layout.region[i].length);
		end = offset + length;

		dev_dbg(dev, "CMB entry %d: bar_num %u cmb_type %u offset %x length %u\n",
			i, ident->cmb_layout.region[i].bar_num,
			ident->cmb_layout.region[i].cmb_type,
			offset, length);

		if (end > (bar->len >> IONIC_CMB_SHIFT_64K)) {
			dev_warn(dev, "Out of bounds CMB region %d offset %x length %u\n",
				 i, offset, length);
			return;
		}
	}

	/* if first entry matches PCI config, expdb is not supported */
	if (ident->cmb_layout.region[0].bar_num == bar->res_index &&
	    le32_to_cpu(ident->cmb_layout.region[0].length) == bar->len &&
	    !ident->cmb_layout.region[0].offset) {
		dev_warn(dev, "No CMB mapping discovered\n");
		return;
	}

	/* process first entry for regular mapping */
	length_reg0 = le32_to_cpu(ident->cmb_layout.region[0].length);
	if (!length_reg0) {
		dev_warn(dev, "region len = 0. No CMB mapping discovered\n");
		return;
	}

	/* Verify first entry size matches expected 8MB size (in 64KB pages) */
	if (length_reg0 != IONIC_BAR2_CMB_ENTRY_SIZE >> IONIC_CMB_SHIFT_64K) {
		dev_warn(dev, "Unexpected CMB size in entry 0: %u pages\n",
			 length_reg0);
		return;
	}

	sz = BITS_TO_LONGS((length_reg0 << IONIC_CMB_SHIFT_64K) /
			    PAGE_SIZE) * sizeof(long);
	idev->cmb_inuse = kzalloc(sz, GFP_KERNEL);
	if (!idev->cmb_inuse) {
		dev_warn(dev, "No memory for CMB, disabling\n");
		idev->phy_cmb_pages = 0;
		idev->phy_cmb_expdb64_pages = 0;
		idev->phy_cmb_expdb128_pages = 0;
		idev->phy_cmb_expdb256_pages = 0;
		idev->phy_cmb_expdb512_pages = 0;
		idev->cmb_npages = 0;
		return;
	}

	for (i = 0; i < num_regions; i++) {
		/* check this region matches first region length as to
		 * ease implementation
		 */
		if (le32_to_cpu(ident->cmb_layout.region[i].length) !=
		    length_reg0)
			continue;

		offset = le32_to_cpu(ident->cmb_layout.region[i].offset);

		switch (ident->cmb_layout.region[i].cmb_type) {
		case IONIC_CMB_TYPE_DEVMEM:
			idev->phy_cmb_pages = bar->bus_addr + offset;
			idev->cmb_npages =
			    (length_reg0 << IONIC_CMB_SHIFT_64K) / PAGE_SIZE;
			dev_dbg(dev, "regular cmb mapping: bar->bus_addr %pa region[%d].length %u\n",
				&bar->bus_addr, i, length);
			dev_dbg(dev, "idev->phy_cmb_pages %pad, idev->cmb_npages %u\n",
				&idev->phy_cmb_pages, idev->cmb_npages);
			break;

		case IONIC_CMB_TYPE_EXPDB64:
			idev->phy_cmb_expdb64_pages =
				bar->bus_addr + (offset << IONIC_CMB_SHIFT_64K);
			dev_dbg(dev, "idev->phy_cmb_expdb64_pages %pad\n",
				&idev->phy_cmb_expdb64_pages);
			break;

		case IONIC_CMB_TYPE_EXPDB128:
			idev->phy_cmb_expdb128_pages =
				bar->bus_addr + (offset << IONIC_CMB_SHIFT_64K);
			dev_dbg(dev, "idev->phy_cmb_expdb128_pages %pad\n",
				&idev->phy_cmb_expdb128_pages);
			break;

		case IONIC_CMB_TYPE_EXPDB256:
			idev->phy_cmb_expdb256_pages =
				bar->bus_addr + (offset << IONIC_CMB_SHIFT_64K);
			dev_dbg(dev, "idev->phy_cmb_expdb256_pages %pad\n",
				&idev->phy_cmb_expdb256_pages);
			break;

		case IONIC_CMB_TYPE_EXPDB512:
			idev->phy_cmb_expdb512_pages =
				bar->bus_addr + (offset << IONIC_CMB_SHIFT_64K);
			dev_dbg(dev, "idev->phy_cmb_expdb512_pages %pad\n",
				&idev->phy_cmb_expdb512_pages);
			break;

		default:
			dev_warn(dev, "[%d] Invalid cmb_type (%d)\n",
				 i, ident->cmb_layout.region[i].cmb_type);
			break;
		}
	}
}

static void ionic_map_classic_cmb(struct ionic *ionic)
{
	struct ionic_dev_bar *bar = ionic->bars;
	struct ionic_dev *idev = &ionic->idev;
	struct device *dev = ionic->dev;
	int sz;

	bar += 2;
	/* classic CMB mapping */
	idev->phy_cmb_pages = bar->bus_addr;
	idev->cmb_npages = bar->len / PAGE_SIZE;
	dev_dbg(dev, "classic cmb mapping: bar->bus_addr %pa bar->len %lu\n",
		&bar->bus_addr, bar->len);
	dev_dbg(dev, "idev->phy_cmb_pages %pad, idev->cmb_npages %u\n",
		&idev->phy_cmb_pages, idev->cmb_npages);

	sz = BITS_TO_LONGS(idev->cmb_npages) * sizeof(long);
	idev->cmb_inuse = kzalloc(sz, GFP_KERNEL);
	if (!idev->cmb_inuse) {
		idev->phy_cmb_pages = 0;
		idev->cmb_npages = 0;
	}
}

void ionic_map_cmb(struct ionic *ionic)
{
	struct pci_dev *pdev = ionic->pdev;
	struct device *dev = ionic->dev;

	if (!(pci_resource_flags(pdev, 4) & IORESOURCE_MEM)) {
		dev_dbg(dev, "No CMB, disabling\n");
		return;
	}

	if (ionic->ident.dev.capabilities & cpu_to_le64(IONIC_DEV_CAP_DISC_CMB))
		ionic_map_disc_cmb(ionic);
	else
		ionic_map_classic_cmb(ionic);
}

int ionic_dev_setup(struct ionic *ionic)
{
	struct ionic_dev_bar *bar = ionic->bars;
	unsigned int num_bars = ionic->num_bars;
	struct ionic_dev *idev = &ionic->idev;
	struct device *dev = ionic->dev;
	u32 sig;
	int err;

	/* BAR0: dev_cmd and interrupts */
	if (num_bars < 1) {
		dev_err(dev, "No bars found, aborting\n");
		return -EFAULT;
	}

	if (bar->len < IONIC_BAR0_SIZE) {
		dev_err(dev, "Resource bar size %lu too small, aborting\n",
			bar->len);
		return -EFAULT;
	}

	idev->dev_info_regs = bar->vaddr + IONIC_BAR0_DEV_INFO_REGS_OFFSET;
	idev->dev_cmd_regs = bar->vaddr + IONIC_BAR0_DEV_CMD_REGS_OFFSET;
	idev->intr_status = bar->vaddr + IONIC_BAR0_INTR_STATUS_OFFSET;
	idev->intr_ctrl = bar->vaddr + IONIC_BAR0_INTR_CTRL_OFFSET;

	idev->hwstamp_regs = &idev->dev_info_regs->hwstamp;

	sig = ioread32(&idev->dev_info_regs->signature);
	if (sig != IONIC_DEV_INFO_SIGNATURE) {
		dev_err(dev, "Incompatible firmware signature %x", sig);
		return -EFAULT;
	}

	ionic_init_devinfo(ionic);

	/* BAR1: doorbells */
	bar++;
	if (num_bars < 2) {
		dev_err(dev, "Doorbell bar missing, aborting\n");
		return -EFAULT;
	}

	err = ionic_watchdog_init(ionic);
	if (err)
		return err;

	idev->db_pages = bar->vaddr;
	idev->phy_db_pages = bar->bus_addr;

	/* BAR2: optional controller memory mapping */
	bar++;
	mutex_init(&idev->cmb_inuse_lock);
	if (num_bars < 3 || !ionic->bars[IONIC_PCI_BAR_CMB].len) {
		idev->cmb_inuse = NULL;
		idev->phy_cmb_pages = 0;
		idev->cmb_npages = 0;
		return 0;
	}

	return 0;
}

void ionic_dev_teardown(struct ionic *ionic)
{
	struct ionic_dev *idev = &ionic->idev;

	kfree(idev->cmb_inuse);
	idev->cmb_inuse = NULL;
	idev->phy_cmb_pages = 0;
	idev->cmb_npages = 0;

	idev->phy_cmb_expdb64_pages = 0;
	idev->phy_cmb_expdb128_pages = 0;
	idev->phy_cmb_expdb256_pages = 0;
	idev->phy_cmb_expdb512_pages = 0;

	if (ionic->wq) {
		destroy_workqueue(ionic->wq);
		ionic->wq = NULL;
	}
	mutex_destroy(&idev->cmb_inuse_lock);
}

/* Devcmd Interface */
static bool __ionic_is_fw_running(struct ionic_dev *idev, u8 *status_ptr)
{
	u8 fw_status;

	if (!idev->dev_info_regs) {
		if (status_ptr)
			*status_ptr = 0xff;
		return false;
	}

	fw_status = ioread8(&idev->dev_info_regs->fw_status);
	if (status_ptr)
		*status_ptr = fw_status;

	/* firmware is useful only if the running bit is set and
	 * fw_status != 0xff (bad PCI read)
	 */
	return (fw_status != 0xff) && (fw_status & IONIC_FW_STS_F_RUNNING);
}

bool ionic_is_fw_running(struct ionic_dev *idev)
{
	return __ionic_is_fw_running(idev, NULL);
}

int ionic_heartbeat_check(struct ionic *ionic)
{
	unsigned long check_time, last_check_time;
	struct ionic_dev *idev = &ionic->idev;
	struct ionic_lif *lif = ionic->lif;
	bool fw_status_ready = true;
	bool fw_hb_ready;
	u8 fw_generation;
	u8 fw_status;
	u32 fw_hb;

	/* wait a least one second before testing again */
	check_time = jiffies;
	last_check_time = atomic_long_read(&idev->last_check_time);
do_check_time:
	if (time_before(check_time, last_check_time + HZ))
		return 0;
	if (!atomic_long_try_cmpxchg_relaxed(&idev->last_check_time,
					     &last_check_time, check_time)) {
		/* if called concurrently, only the first should proceed. */
		dev_dbg(ionic->dev, "%s: do_check_time again\n", __func__);
		goto do_check_time;
	}

	/* If fw_status is not ready don't bother with the generation */
	if (!__ionic_is_fw_running(idev, &fw_status)) {
		fw_status_ready = false;
	} else {
		fw_generation = fw_status & IONIC_FW_STS_F_GENERATION;
		if (idev->fw_generation != fw_generation) {
			dev_info(ionic->dev, "FW generation 0x%02x -> 0x%02x\n",
				 idev->fw_generation, fw_generation);

			idev->fw_generation = fw_generation;

			/* If the generation changed, the fw status is not
			 * ready so we need to trigger a fw-down cycle.  After
			 * the down, the next watchdog will see the fw is up
			 * and the generation value stable, so will trigger
			 * the fw-up activity.
			 *
			 * If we had already moved to FW_RESET from a RESET event,
			 * it is possible that we never saw the fw_status go to 0,
			 * so we fake the current idev->fw_status_ready here to
			 * force the transition and get FW up again.
			 */
			if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
				idev->fw_status_ready = false;	/* go to running */
			else
				fw_status_ready = false;	/* go to down */
		}
	}

	dev_dbg(ionic->dev, "fw_status 0x%02x ready %d idev->ready %d last_hb 0x%x state 0x%02lx\n",
		fw_status, fw_status_ready, idev->fw_status_ready,
		idev->last_fw_hb, lif->state[0]);

	/* is this a transition? */
	if (fw_status_ready != idev->fw_status_ready &&
	    !test_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
		bool trigger = false;

		idev->fw_status_ready = fw_status_ready;

		if (!fw_status_ready &&
		    !test_bit(IONIC_LIF_F_FW_RESET, lif->state) &&
		    !test_and_set_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
			dev_info(ionic->dev, "FW stopped 0x%02x\n", fw_status);
			trigger = true;

		} else if (fw_status_ready &&
			   test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
			dev_info(ionic->dev, "FW running 0x%02x\n", fw_status);
			trigger = true;
		}

		if (trigger) {
			struct ionic_deferred_work *work;

			work = kzalloc(sizeof(*work), GFP_ATOMIC);
			if (work) {
				work->type = IONIC_DW_TYPE_LIF_RESET;
				work->fw_status = fw_status_ready;
				ionic_lif_deferred_enqueue(lif, work);
			}
		}
	}

	if (!idev->fw_status_ready)
		return -ENXIO;

	/* Because of some variability in the actual FW heartbeat, we
	 * wait longer than the DEVCMD_TIMEOUT before checking again.
	 */
	last_check_time = idev->last_hb_time;
	if (time_before(check_time, last_check_time + DEVCMD_TIMEOUT * 2 * HZ))
		return 0;

	fw_hb = ioread32(&idev->dev_info_regs->fw_heartbeat);
	fw_hb_ready = fw_hb != idev->last_fw_hb;

	/* early FW version had no heartbeat, so fake it */
	if (!fw_hb_ready && !fw_hb)
		fw_hb_ready = true;

	dev_dbg(ionic->dev, "%s: fw_hb %u last_fw_hb %u ready %u\n",
		__func__, fw_hb, idev->last_fw_hb, fw_hb_ready);

	idev->last_fw_hb = fw_hb;

	/* log a transition */
	if (fw_hb_ready != idev->fw_hb_ready) {
		idev->fw_hb_ready = fw_hb_ready;
		if (!fw_hb_ready)
			dev_info(ionic->dev, "FW heartbeat stalled at %u\n", fw_hb);
		else
			dev_info(ionic->dev, "FW heartbeat restored at %u\n", fw_hb);
	}

	if (!fw_hb_ready)
		return -ENXIO;

	idev->last_hb_time = check_time;

	return 0;
}

u8 ionic_dev_cmd_status(struct ionic_dev *idev)
{
	if (!idev->dev_cmd_regs)
		return (u8)PCI_ERROR_RESPONSE;
	return ioread8(&idev->dev_cmd_regs->comp.comp.status);
}

bool ionic_dev_cmd_done(struct ionic_dev *idev)
{
	if (!idev->dev_cmd_regs)
		return false;
	return ioread32(&idev->dev_cmd_regs->done) & IONIC_DEV_CMD_DONE;
}

void ionic_dev_cmd_comp(struct ionic_dev *idev, union ionic_dev_cmd_comp *comp)
{
	if (!idev->dev_cmd_regs)
		return;
	memcpy_fromio(comp, &idev->dev_cmd_regs->comp, sizeof(*comp));
}

void ionic_dev_cmd_go(struct ionic_dev *idev, union ionic_dev_cmd *cmd)
{
	idev->opcode = cmd->cmd.opcode;

	if (!idev->dev_cmd_regs)
		return;

	memcpy_toio(&idev->dev_cmd_regs->cmd, cmd, sizeof(*cmd));
	iowrite32(0, &idev->dev_cmd_regs->done);
	iowrite32(1, &idev->dev_cmd_regs->doorbell);
}

/* Device commands */
void ionic_dev_cmd_identify(struct ionic_dev *idev, u8 ver)
{
	union ionic_dev_cmd cmd = {
		.identify.opcode = IONIC_CMD_IDENTIFY,
		.identify.ver = ver,
	};

	ionic_dev_cmd_go(idev, &cmd);
}

void ionic_dev_cmd_init(struct ionic_dev *idev)
{
	union ionic_dev_cmd cmd = {
		.init.opcode = IONIC_CMD_INIT,
		.init.type = 0,
	};

	ionic_dev_cmd_go(idev, &cmd);
}

void ionic_dev_cmd_reset(struct ionic_dev *idev)
{
	union ionic_dev_cmd cmd = {
		.reset.opcode = IONIC_CMD_RESET,
	};

	ionic_dev_cmd_go(idev, &cmd);
}

/* Port commands */
void ionic_dev_cmd_port_identify(struct ionic_dev *idev)
{
	union ionic_dev_cmd cmd = {
		.port_init.opcode = IONIC_CMD_PORT_IDENTIFY,
		.port_init.index = 0,
	};

	ionic_dev_cmd_go(idev, &cmd);
}

void ionic_dev_cmd_port_init(struct ionic_dev *idev)
{
	union ionic_dev_cmd cmd = {
		.port_init.opcode = IONIC_CMD_PORT_INIT,
		.port_init.index = 0,
		.port_init.info_pa = cpu_to_le64(idev->port_info_pa),
	};

	ionic_dev_cmd_go(idev, &cmd);
}

void ionic_dev_cmd_port_reset(struct ionic_dev *idev)
{
	union ionic_dev_cmd cmd = {
		.port_reset.opcode = IONIC_CMD_PORT_RESET,
		.port_reset.index = 0,
	};

	ionic_dev_cmd_go(idev, &cmd);
}

void ionic_dev_cmd_port_state(struct ionic_dev *idev, u8 state)
{
	union ionic_dev_cmd cmd = {
		.port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
		.port_setattr.index = 0,
		.port_setattr.attr = IONIC_PORT_ATTR_STATE,
		.port_setattr.state = state,
	};

	ionic_dev_cmd_go(idev, &cmd);
}

void ionic_dev_cmd_port_speed(struct ionic_dev *idev, u32 speed)
{
	union ionic_dev_cmd cmd = {
		.port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
		.port_setattr.index = 0,
		.port_setattr.attr = IONIC_PORT_ATTR_SPEED,
		.port_setattr.speed = cpu_to_le32(speed),
	};

	ionic_dev_cmd_go(idev, &cmd);
}

void ionic_dev_cmd_port_autoneg(struct ionic_dev *idev, u8 an_enable)
{
	union ionic_dev_cmd cmd = {
		.port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
		.port_setattr.index = 0,
		.port_setattr.attr = IONIC_PORT_ATTR_AUTONEG,
		.port_setattr.an_enable = an_enable,
	};

	ionic_dev_cmd_go(idev, &cmd);
}

void ionic_dev_cmd_port_fec(struct ionic_dev *idev, u8 fec_type)
{
	union ionic_dev_cmd cmd = {
		.port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
		.port_setattr.index = 0,
		.port_setattr.attr = IONIC_PORT_ATTR_FEC,
		.port_setattr.fec_type = fec_type,
	};

	ionic_dev_cmd_go(idev, &cmd);
}

void ionic_dev_cmd_port_pause(struct ionic_dev *idev, u8 pause_type)
{
	union ionic_dev_cmd cmd = {
		.port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
		.port_setattr.index = 0,
		.port_setattr.attr = IONIC_PORT_ATTR_PAUSE,
		.port_setattr.pause_type = pause_type,
	};

	ionic_dev_cmd_go(idev, &cmd);
}

/* VF commands */
int ionic_set_vf_config(struct ionic *ionic, int vf,
			struct ionic_vf_setattr_cmd *vfc)
{
	union ionic_dev_cmd cmd = {
		.vf_setattr.opcode = IONIC_CMD_VF_SETATTR,
		.vf_setattr.attr = vfc->attr,
		.vf_setattr.vf_index = cpu_to_le16(vf),
	};
	int err;

	memcpy(cmd.vf_setattr.pad, vfc->pad, sizeof(vfc->pad));

	mutex_lock(&ionic->dev_cmd_lock);
	ionic_dev_cmd_go(&ionic->idev, &cmd);
	err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
	mutex_unlock(&ionic->dev_cmd_lock);

	return err;
}

void ionic_vf_start(struct ionic *ionic)
{
	union ionic_dev_cmd cmd = {
		.vf_ctrl.opcode = IONIC_CMD_VF_CTRL,
		.vf_ctrl.ctrl_opcode = IONIC_VF_CTRL_START_ALL,
	};

	if (!(ionic->ident.dev.capabilities & cpu_to_le64(IONIC_DEV_CAP_VF_CTRL)))
		return;

	ionic_dev_cmd_go(&ionic->idev, &cmd);
	ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
}

/* LIF commands */
void ionic_dev_cmd_queue_identify(struct ionic_dev *idev,
				  u16 lif_type, u8 qtype, u8 qver)
{
	union ionic_dev_cmd cmd = {
		.q_identify.opcode = IONIC_CMD_Q_IDENTIFY,
		.q_identify.lif_type = cpu_to_le16(lif_type),
		.q_identify.type = qtype,
		.q_identify.ver = qver,
	};

	ionic_dev_cmd_go(idev, &cmd);
}

void ionic_dev_cmd_lif_identify(struct ionic_dev *idev, u8 type, u8 ver)
{
	union ionic_dev_cmd cmd = {
		.lif_identify.opcode = IONIC_CMD_LIF_IDENTIFY,
		.lif_identify.type = type,
		.lif_identify.ver = ver,
	};

	ionic_dev_cmd_go(idev, &cmd);
}

void ionic_dev_cmd_lif_init(struct ionic_dev *idev, u16 lif_index,
			    dma_addr_t info_pa)
{
	union ionic_dev_cmd cmd = {
		.lif_init.opcode = IONIC_CMD_LIF_INIT,
		.lif_init.index = cpu_to_le16(lif_index),
		.lif_init.info_pa = cpu_to_le64(info_pa),
	};

	ionic_dev_cmd_go(idev, &cmd);
}

void ionic_dev_cmd_lif_reset(struct ionic_dev *idev, u16 lif_index)
{
	union ionic_dev_cmd cmd = {
		.lif_init.opcode = IONIC_CMD_LIF_RESET,
		.lif_init.index = cpu_to_le16(lif_index),
	};

	ionic_dev_cmd_go(idev, &cmd);
}

void ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq,
			       u16 lif_index, u16 intr_index)
{
	struct ionic_queue *q = &qcq->q;
	struct ionic_cq *cq = &qcq->cq;

	union ionic_dev_cmd cmd = {
		.q_init.opcode = IONIC_CMD_Q_INIT,
		.q_init.lif_index = cpu_to_le16(lif_index),
		.q_init.type = q->type,
		.q_init.ver = qcq->q.lif->qtype_info[q->type].version,
		.q_init.index = cpu_to_le32(q->index),
		.q_init.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
					    IONIC_QINIT_F_ENA),
		.q_init.pid = cpu_to_le16(q->pid),
		.q_init.intr_index = cpu_to_le16(intr_index),
		.q_init.ring_size = ilog2(q->num_descs),
		.q_init.ring_base = cpu_to_le64(q->base_pa),
		.q_init.cq_ring_base = cpu_to_le64(cq->base_pa),
	};

	ionic_dev_cmd_go(idev, &cmd);
}

void ionic_dev_cmd_discover_cmb(struct ionic_dev *idev)
{
	union ionic_dev_cmd cmd = {
		.discover_cmb.opcode = IONIC_CMD_DISCOVER_CMB,
	};

	ionic_dev_cmd_go(idev, &cmd);
}

int ionic_db_page_num(struct ionic_lif *lif, int pid)
{
	return (lif->hw_index * lif->dbid_count) + pid;
}

int ionic_get_cmb(struct ionic_lif *lif, u32 *pgid, phys_addr_t *pgaddr,
		  int order, u8 stride_log2, bool *expdb)
{
	struct ionic_dev *idev = &lif->ionic->idev;
	void __iomem *nonexpdb_pgptr;
	phys_addr_t nonexpdb_pgaddr;
	int i, idx;

	mutex_lock(&idev->cmb_inuse_lock);
	idx = bitmap_find_free_region(idev->cmb_inuse, idev->cmb_npages, order);
	mutex_unlock(&idev->cmb_inuse_lock);

	if (idx < 0)
		return idx;

	*pgid = (u32)idx;

	if (idev->phy_cmb_expdb64_pages &&
	    stride_log2 == IONIC_EXPDB_64B_WQE_LG2) {
		*pgaddr = idev->phy_cmb_expdb64_pages + idx * PAGE_SIZE;
		if (expdb)
			*expdb = true;
	} else if (idev->phy_cmb_expdb128_pages &&
		  stride_log2 == IONIC_EXPDB_128B_WQE_LG2) {
		*pgaddr = idev->phy_cmb_expdb128_pages + idx * PAGE_SIZE;
		if (expdb)
			*expdb = true;
	} else if (idev->phy_cmb_expdb256_pages &&
		  stride_log2 == IONIC_EXPDB_256B_WQE_LG2) {
		*pgaddr = idev->phy_cmb_expdb256_pages + idx * PAGE_SIZE;
		if (expdb)
			*expdb = true;
	} else if (idev->phy_cmb_expdb512_pages &&
		  stride_log2 == IONIC_EXPDB_512B_WQE_LG2) {
		*pgaddr = idev->phy_cmb_expdb512_pages + idx * PAGE_SIZE;
		if (expdb)
			*expdb = true;
	} else {
		*pgaddr = idev->phy_cmb_pages + idx * PAGE_SIZE;
		if (expdb)
			*expdb = false;
	}

	/* clear the requested CMB region, 1 PAGE_SIZE ioremap at a time */
	nonexpdb_pgaddr = idev->phy_cmb_pages + idx * PAGE_SIZE;
	for (i = 0; i < (1 << order); i++) {
		nonexpdb_pgptr =
			ioremap_wc(nonexpdb_pgaddr + i * PAGE_SIZE, PAGE_SIZE);
		if (!nonexpdb_pgptr) {
			ionic_put_cmb(lif, *pgid, order);
			return -ENOMEM;
		}
		memset_io(nonexpdb_pgptr, 0, PAGE_SIZE);
		iounmap(nonexpdb_pgptr);
	}

	return 0;
}
EXPORT_SYMBOL_NS(ionic_get_cmb, "NET_IONIC");

void ionic_put_cmb(struct ionic_lif *lif, u32 pgid, int order)
{
	struct ionic_dev *idev = &lif->ionic->idev;

	mutex_lock(&idev->cmb_inuse_lock);
	bitmap_release_region(idev->cmb_inuse, pgid, order);
	mutex_unlock(&idev->cmb_inuse_lock);
}
EXPORT_SYMBOL_NS(ionic_put_cmb, "NET_IONIC");

int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq,
		  struct ionic_intr_info *intr,
		  unsigned int num_descs, size_t desc_size)
{
	unsigned int ring_size;

	if (desc_size == 0 || !is_power_of_2(num_descs))
		return -EINVAL;

	ring_size = ilog2(num_descs);
	if (ring_size < 2 || ring_size > 16)
		return -EINVAL;

	cq->lif = lif;
	cq->bound_intr = intr;
	cq->num_descs = num_descs;
	cq->desc_size = desc_size;
	cq->tail_idx = 0;
	cq->done_color = 1;
	cq->idev = &lif->ionic->idev;

	return 0;
}

unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do,
			      ionic_cq_cb cb, ionic_cq_done_cb done_cb,
			      void *done_arg)
{
	unsigned int work_done = 0;

	if (work_to_do == 0)
		return 0;

	while (cb(cq)) {
		if (cq->tail_idx == cq->num_descs - 1)
			cq->done_color = !cq->done_color;

		cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);

		if (++work_done >= work_to_do)
			break;
	}

	if (work_done && done_cb)
		done_cb(done_arg);

	return work_done;
}

int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev,
		 struct ionic_queue *q, unsigned int index, const char *name,
		 unsigned int num_descs, size_t desc_size,
		 size_t sg_desc_size, unsigned int pid)
{
	unsigned int ring_size;

	if (desc_size == 0 || !is_power_of_2(num_descs))
		return -EINVAL;

	ring_size = ilog2(num_descs);
	if (ring_size < 2 || ring_size > 16)
		return -EINVAL;

	q->lif = lif;
	q->index = index;
	q->num_descs = num_descs;
	q->desc_size = desc_size;
	q->sg_desc_size = sg_desc_size;
	q->tail_idx = 0;
	q->head_idx = 0;
	q->pid = pid;

	snprintf(q->name, sizeof(q->name), "L%d-%s%u", lif->index, name, index);

	return 0;
}

void ionic_q_post(struct ionic_queue *q, bool ring_doorbell)
{
	struct ionic_lif *lif = q->lif;
	struct device *dev = q->dev;

	q->head_idx = (q->head_idx + 1) & (q->num_descs - 1);

	dev_dbg(dev, "lif=%d qname=%s qid=%d qtype=%d p_index=%d ringdb=%d\n",
		q->lif->index, q->name, q->hw_type, q->hw_index,
		q->head_idx, ring_doorbell);

	if (ring_doorbell) {
		ionic_dbell_ring(lif->kern_dbpage, q->hw_type,
				 q->dbval | q->head_idx);

		q->dbell_jiffies = jiffies;
	}
}

bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos)
{
	unsigned int mask, tail, head;

	mask = q->num_descs - 1;
	tail = q->tail_idx;
	head = q->head_idx;

	return ((pos - tail) & mask) < ((head - tail) & mask);
}