drivers/virt/acrn/ioreq.c

// SPDX-License-Identifier: GPL-2.0
/*
 * ACRN_HSM: Handle I/O requests
 *
 * Copyright (C) 2020 Intel Corporation. All rights reserved.
 *
 * Authors:
 *	Jason Chen CJ <jason.cj.chen@intel.com>
 *	Fengwei Yin <fengwei.yin@intel.com>
 */

#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/kthread.h>
#include <linux/mm.h>
#include <linux/slab.h>

#include <asm/acrn.h>

#include "acrn_drv.h"

static void ioreq_pause(void);
static void ioreq_resume(void);

static void ioreq_dispatcher(struct work_struct *work);
static struct workqueue_struct *ioreq_wq;
static DECLARE_WORK(ioreq_work, ioreq_dispatcher);

static inline bool has_pending_request(struct acrn_ioreq_client *client)
{
	return !bitmap_empty(client->ioreqs_map, ACRN_IO_REQUEST_MAX);
}

static inline bool is_destroying(struct acrn_ioreq_client *client)
{
	return test_bit(ACRN_IOREQ_CLIENT_DESTROYING, &client->flags);
}

static int ioreq_complete_request(struct acrn_vm *vm, u16 vcpu,
				  struct acrn_io_request *acrn_req)
{
	bool polling_mode;
	int ret = 0;

	polling_mode = acrn_req->completion_polling;
	/* Add barrier() to make sure the writes are done before completion */
	smp_store_release(&acrn_req->processed, ACRN_IOREQ_STATE_COMPLETE);

	/*
	 * To fulfill the requirement of real-time in several industry
	 * scenarios, like automotive, ACRN can run under the partition mode,
	 * in which User VMs and Service VM are bound to dedicated CPU cores.
	 * Polling mode of handling the I/O request is introduced to achieve a
	 * faster I/O request handling. In polling mode, the hypervisor polls
	 * I/O request's completion. Once an I/O request is marked as
	 * ACRN_IOREQ_STATE_COMPLETE, hypervisor resumes from the polling point
	 * to continue the I/O request flow. Thus, the completion notification
	 * from HSM of I/O request is not needed.  Please note,
	 * completion_polling needs to be read before the I/O request being
	 * marked as ACRN_IOREQ_STATE_COMPLETE to avoid racing with the
	 * hypervisor.
	 */
	if (!polling_mode) {
		ret = hcall_notify_req_finish(vm->vmid, vcpu);
		if (ret < 0)
			dev_err(acrn_dev.this_device,
				"Notify I/O request finished failed!\n");
	}

	return ret;
}

static int acrn_ioreq_complete_request(struct acrn_ioreq_client *client,
				       u16 vcpu,
				       struct acrn_io_request *acrn_req)
{
	int ret;

	if (vcpu >= client->vm->vcpu_num)
		return -EINVAL;

	clear_bit(vcpu, client->ioreqs_map);
	if (!acrn_req) {
		acrn_req = (struct acrn_io_request *)client->vm->ioreq_buf;
		acrn_req += vcpu;
	}

	ret = ioreq_complete_request(client->vm, vcpu, acrn_req);

	return ret;
}

int acrn_ioreq_request_default_complete(struct acrn_vm *vm, u16 vcpu)
{
	int ret = 0;

	spin_lock_bh(&vm->ioreq_clients_lock);
	if (vm->default_client)
		ret = acrn_ioreq_complete_request(vm->default_client,
						  vcpu, NULL);
	spin_unlock_bh(&vm->ioreq_clients_lock);

	return ret;
}

/**
 * acrn_ioreq_range_add() - Add an iorange monitored by an ioreq client
 * @client:	The ioreq client
 * @type:	Type (ACRN_IOREQ_TYPE_MMIO or ACRN_IOREQ_TYPE_PORTIO)
 * @start:	Start address of iorange
 * @end:	End address of iorange
 *
 * Return: 0 on success, <0 on error
 */
int acrn_ioreq_range_add(struct acrn_ioreq_client *client,
			 u32 type, u64 start, u64 end)
{
	struct acrn_ioreq_range *range;

	if (end < start) {
		dev_err(acrn_dev.this_device,
			"Invalid IO range [0x%llx,0x%llx]\n", start, end);
		return -EINVAL;
	}

	range = kzalloc(sizeof(*range), GFP_KERNEL);
	if (!range)
		return -ENOMEM;

	range->type = type;
	range->start = start;
	range->end = end;

	write_lock_bh(&client->range_lock);
	list_add(&range->list, &client->range_list);
	write_unlock_bh(&client->range_lock);

	return 0;
}

/**
 * acrn_ioreq_range_del() - Del an iorange monitored by an ioreq client
 * @client:	The ioreq client
 * @type:	Type (ACRN_IOREQ_TYPE_MMIO or ACRN_IOREQ_TYPE_PORTIO)
 * @start:	Start address of iorange
 * @end:	End address of iorange
 */
void acrn_ioreq_range_del(struct acrn_ioreq_client *client,
			  u32 type, u64 start, u64 end)
{
	struct acrn_ioreq_range *range;

	write_lock_bh(&client->range_lock);
	list_for_each_entry(range, &client->range_list, list) {
		if (type == range->type &&
		    start == range->start &&
		    end == range->end) {
			list_del(&range->list);
			kfree(range);
			break;
		}
	}
	write_unlock_bh(&client->range_lock);
}

/*
 * ioreq_task() is the execution entity of handler thread of an I/O client.
 * The handler callback of the I/O client is called within the handler thread.
 */
static int ioreq_task(void *data)
{
	struct acrn_ioreq_client *client = data;
	struct acrn_io_request *req;
	unsigned long *ioreqs_map;
	int vcpu, ret;

	/*
	 * Lockless access to ioreqs_map is safe, because
	 * 1) set_bit() and clear_bit() are atomic operations.
	 * 2) I/O requests arrives serialized. The access flow of ioreqs_map is:
	 *	set_bit() - in ioreq_work handler
	 *	Handler callback handles corresponding I/O request
	 *	clear_bit() - in handler thread (include ACRN userspace)
	 *	Mark corresponding I/O request completed
	 *	Loop again if a new I/O request occurs
	 */
	ioreqs_map = client->ioreqs_map;
	while (!kthread_should_stop()) {
		acrn_ioreq_client_wait(client);
		while (has_pending_request(client)) {
			vcpu = find_first_bit(ioreqs_map, client->vm->vcpu_num);
			req = client->vm->ioreq_buf->req_slot + vcpu;
			ret = client->handler(client, req);
			if (ret < 0) {
				dev_err(acrn_dev.this_device,
					"IO handle failure: %d\n", ret);
				break;
			}
			acrn_ioreq_complete_request(client, vcpu, req);
		}
	}

	return 0;
}

/*
 * For the non-default I/O clients, give them chance to complete the current
 * I/O requests if there are any. For the default I/O client, it is safe to
 * clear all pending I/O requests because the clearing request is from ACRN
 * userspace.
 */
void acrn_ioreq_request_clear(struct acrn_vm *vm)
{
	struct acrn_ioreq_client *client;
	bool has_pending = false;
	unsigned long vcpu;
	int retry = 10;

	/*
	 * IO requests of this VM will be completed directly in
	 * acrn_ioreq_dispatch if ACRN_VM_FLAG_CLEARING_IOREQ flag is set.
	 */
	set_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags);

	/*
	 * acrn_ioreq_request_clear is only called in VM reset case. Simply
	 * wait 100ms in total for the IO requests' completion.
	 */
	do {
		spin_lock_bh(&vm->ioreq_clients_lock);
		list_for_each_entry(client, &vm->ioreq_clients, list) {
			has_pending = has_pending_request(client);
			if (has_pending)
				break;
		}
		spin_unlock_bh(&vm->ioreq_clients_lock);

		if (has_pending)
			schedule_timeout_interruptible(HZ / 100);
	} while (has_pending && --retry > 0);
	if (retry == 0)
		dev_warn(acrn_dev.this_device,
			 "%s cannot flush pending request!\n", client->name);

	/* Clear all ioreqs belonging to the default client */
	spin_lock_bh(&vm->ioreq_clients_lock);
	client = vm->default_client;
	if (client) {
		vcpu = find_next_bit(client->ioreqs_map,
				     ACRN_IO_REQUEST_MAX, 0);
		while (vcpu < ACRN_IO_REQUEST_MAX) {
			acrn_ioreq_complete_request(client, vcpu, NULL);
			vcpu = find_next_bit(client->ioreqs_map,
					     ACRN_IO_REQUEST_MAX, vcpu + 1);
		}
	}
	spin_unlock_bh(&vm->ioreq_clients_lock);

	/* Clear ACRN_VM_FLAG_CLEARING_IOREQ flag after the clearing */
	clear_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags);
}

int acrn_ioreq_client_wait(struct acrn_ioreq_client *client)
{
	if (client->is_default) {
		/*
		 * In the default client, a user space thread waits on the
		 * waitqueue. The is_destroying() check is used to notify user
		 * space the client is going to be destroyed.
		 */
		wait_event_interruptible(client->wq,
					 has_pending_request(client) ||
					 is_destroying(client));
		if (is_destroying(client))
			return -ENODEV;
	} else {
		wait_event_interruptible(client->wq,
					 has_pending_request(client) ||
					 kthread_should_stop());
	}

	return 0;
}

static bool is_cfg_addr(struct acrn_io_request *req)
{
	return ((req->type == ACRN_IOREQ_TYPE_PORTIO) &&
		(req->reqs.pio_request.address == 0xcf8));
}

static bool is_cfg_data(struct acrn_io_request *req)
{
	return ((req->type == ACRN_IOREQ_TYPE_PORTIO) &&
		((req->reqs.pio_request.address >= 0xcfc) &&
		 (req->reqs.pio_request.address < (0xcfc + 4))));
}

/* The low 8-bit of supported pci_reg addr.*/
#define PCI_LOWREG_MASK  0xFC
/* The high 4-bit of supported pci_reg addr */
#define PCI_HIGHREG_MASK 0xF00
/* Max number of supported functions */
#define PCI_FUNCMAX	7
/* Max number of supported slots */
#define PCI_SLOTMAX	31
/* Max number of supported buses */
#define PCI_BUSMAX	255
#define CONF1_ENABLE	0x80000000UL
/*
 * A PCI configuration space access via PIO 0xCF8 and 0xCFC normally has two
 * following steps:
 *   1) writes address into 0xCF8 port
 *   2) accesses data in/from 0xCFC
 * This function combines such paired PCI configuration space I/O requests into
 * one ACRN_IOREQ_TYPE_PCICFG type I/O request and continues the processing.
 */
static bool handle_cf8cfc(struct acrn_vm *vm,
			  struct acrn_io_request *req, u16 vcpu)
{
	int offset, pci_cfg_addr, pci_reg;
	bool is_handled = false;

	if (is_cfg_addr(req)) {
		WARN_ON(req->reqs.pio_request.size != 4);
		if (req->reqs.pio_request.direction == ACRN_IOREQ_DIR_WRITE)
			vm->pci_conf_addr = req->reqs.pio_request.value;
		else
			req->reqs.pio_request.value = vm->pci_conf_addr;
		is_handled = true;
	} else if (is_cfg_data(req)) {
		if (!(vm->pci_conf_addr & CONF1_ENABLE)) {
			if (req->reqs.pio_request.direction ==
					ACRN_IOREQ_DIR_READ)
				req->reqs.pio_request.value = 0xffffffff;
			is_handled = true;
		} else {
			offset = req->reqs.pio_request.address - 0xcfc;

			req->type = ACRN_IOREQ_TYPE_PCICFG;
			pci_cfg_addr = vm->pci_conf_addr;
			req->reqs.pci_request.bus =
					(pci_cfg_addr >> 16) & PCI_BUSMAX;
			req->reqs.pci_request.dev =
					(pci_cfg_addr >> 11) & PCI_SLOTMAX;
			req->reqs.pci_request.func =
					(pci_cfg_addr >> 8) & PCI_FUNCMAX;
			pci_reg = (pci_cfg_addr & PCI_LOWREG_MASK) +
				   ((pci_cfg_addr >> 16) & PCI_HIGHREG_MASK);
			req->reqs.pci_request.reg = pci_reg + offset;
		}
	}

	if (is_handled)
		ioreq_complete_request(vm, vcpu, req);

	return is_handled;
}

static bool in_range(struct acrn_ioreq_range *range,
		     struct acrn_io_request *req)
{
	bool ret = false;

	if (range->type == req->type) {
		switch (req->type) {
		case ACRN_IOREQ_TYPE_MMIO:
			if (req->reqs.mmio_request.address >= range->start &&
			    (req->reqs.mmio_request.address +
			     req->reqs.mmio_request.size - 1) <= range->end)
				ret = true;
			break;
		case ACRN_IOREQ_TYPE_PORTIO:
			if (req->reqs.pio_request.address >= range->start &&
			    (req->reqs.pio_request.address +
			     req->reqs.pio_request.size - 1) <= range->end)
				ret = true;
			break;
		default:
			break;
		}
	}

	return ret;
}

static struct acrn_ioreq_client *find_ioreq_client(struct acrn_vm *vm,
						   struct acrn_io_request *req)
{
	struct acrn_ioreq_client *client, *found = NULL;
	struct acrn_ioreq_range *range;

	lockdep_assert_held(&vm->ioreq_clients_lock);

	list_for_each_entry(client, &vm->ioreq_clients, list) {
		read_lock_bh(&client->range_lock);
		list_for_each_entry(range, &client->range_list, list) {
			if (in_range(range, req)) {
				found = client;
				break;
			}
		}
		read_unlock_bh(&client->range_lock);
		if (found)
			break;
	}
	return found ? found : vm->default_client;
}

/**
 * acrn_ioreq_client_create() - Create an ioreq client
 * @vm:		The VM that this client belongs to
 * @handler:	The ioreq_handler of ioreq client acrn_hsm will create a kernel
 *		thread and call the handler to handle I/O requests.
 * @priv:	Private data for the handler
 * @is_default:	If it is the default client
 * @name:	The name of ioreq client
 *
 * Return: acrn_ioreq_client pointer on success, NULL on error
 */
struct acrn_ioreq_client *acrn_ioreq_client_create(struct acrn_vm *vm,
						   ioreq_handler_t handler,
						   void *priv, bool is_default,
						   const char *name)
{
	struct acrn_ioreq_client *client;

	if (!handler && !is_default) {
		dev_dbg(acrn_dev.this_device,
			"Cannot create non-default client w/o handler!\n");
		return NULL;
	}
	client = kzalloc(sizeof(*client), GFP_KERNEL);
	if (!client)
		return NULL;

	client->handler = handler;
	client->vm = vm;
	client->priv = priv;
	client->is_default = is_default;
	if (name)
		strncpy(client->name, name, sizeof(client->name) - 1);
	rwlock_init(&client->range_lock);
	INIT_LIST_HEAD(&client->range_list);
	init_waitqueue_head(&client->wq);

	if (client->handler) {
		client->thread = kthread_run(ioreq_task, client, "VM%u-%s",
					     client->vm->vmid, client->name);
		if (IS_ERR(client->thread)) {
			kfree(client);
			return NULL;
		}
	}

	spin_lock_bh(&vm->ioreq_clients_lock);
	if (is_default)
		vm->default_client = client;
	else
		list_add(&client->list, &vm->ioreq_clients);
	spin_unlock_bh(&vm->ioreq_clients_lock);

	dev_dbg(acrn_dev.this_device, "Created ioreq client %s.\n", name);
	return client;
}

/**
 * acrn_ioreq_client_destroy() - Destroy an ioreq client
 * @client:	The ioreq client
 */
void acrn_ioreq_client_destroy(struct acrn_ioreq_client *client)
{
	struct acrn_ioreq_range *range, *next;
	struct acrn_vm *vm = client->vm;

	dev_dbg(acrn_dev.this_device,
		"Destroy ioreq client %s.\n", client->name);
	ioreq_pause();
	set_bit(ACRN_IOREQ_CLIENT_DESTROYING, &client->flags);
	if (client->is_default)
		wake_up_interruptible(&client->wq);
	else
		kthread_stop(client->thread);

	spin_lock_bh(&vm->ioreq_clients_lock);
	if (client->is_default)
		vm->default_client = NULL;
	else
		list_del(&client->list);
	spin_unlock_bh(&vm->ioreq_clients_lock);

	write_lock_bh(&client->range_lock);
	list_for_each_entry_safe(range, next, &client->range_list, list) {
		list_del(&range->list);
		kfree(range);
	}
	write_unlock_bh(&client->range_lock);
	kfree(client);

	ioreq_resume();
}

static int acrn_ioreq_dispatch(struct acrn_vm *vm)
{
	struct acrn_ioreq_client *client;
	struct acrn_io_request *req;
	int i;

	for (i = 0; i < vm->vcpu_num; i++) {
		req = vm->ioreq_buf->req_slot + i;

		/* barrier the read of processed of acrn_io_request */
		if (smp_load_acquire(&req->processed) ==
				     ACRN_IOREQ_STATE_PENDING) {
			/* Complete the IO request directly in clearing stage */
			if (test_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags)) {
				ioreq_complete_request(vm, i, req);
				continue;
			}
			if (handle_cf8cfc(vm, req, i))
				continue;

			spin_lock_bh(&vm->ioreq_clients_lock);
			client = find_ioreq_client(vm, req);
			if (!client) {
				dev_err(acrn_dev.this_device,
					"Failed to find ioreq client!\n");
				spin_unlock_bh(&vm->ioreq_clients_lock);
				return -EINVAL;
			}
			if (!client->is_default)
				req->kernel_handled = 1;
			else
				req->kernel_handled = 0;
			/*
			 * Add barrier() to make sure the writes are done
			 * before setting ACRN_IOREQ_STATE_PROCESSING
			 */
			smp_store_release(&req->processed,
					  ACRN_IOREQ_STATE_PROCESSING);
			set_bit(i, client->ioreqs_map);
			wake_up_interruptible(&client->wq);
			spin_unlock_bh(&vm->ioreq_clients_lock);
		}
	}

	return 0;
}

static void ioreq_dispatcher(struct work_struct *work)
{
	struct acrn_vm *vm;

	read_lock(&acrn_vm_list_lock);
	list_for_each_entry(vm, &acrn_vm_list, list) {
		if (!vm->ioreq_buf)
			break;
		acrn_ioreq_dispatch(vm);
	}
	read_unlock(&acrn_vm_list_lock);
}

static void ioreq_intr_handler(void)
{
	queue_work(ioreq_wq, &ioreq_work);
}

static void ioreq_pause(void)
{
	/* Flush and unarm the handler to ensure no I/O requests pending */
	acrn_remove_intr_handler();
	drain_workqueue(ioreq_wq);
}

static void ioreq_resume(void)
{
	/* Schedule after enabling in case other clients miss interrupt */
	acrn_setup_intr_handler(ioreq_intr_handler);
	queue_work(ioreq_wq, &ioreq_work);
}

int acrn_ioreq_intr_setup(void)
{
	acrn_setup_intr_handler(ioreq_intr_handler);
	ioreq_wq = alloc_workqueue("ioreq_wq",
				   WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_UNBOUND, 1);
	if (!ioreq_wq) {
		dev_err(acrn_dev.this_device, "Failed to alloc workqueue!\n");
		acrn_remove_intr_handler();
		return -ENOMEM;
	}
	return 0;
}

void acrn_ioreq_intr_remove(void)
{
	if (ioreq_wq)
		destroy_workqueue(ioreq_wq);
	acrn_remove_intr_handler();
}

int acrn_ioreq_init(struct acrn_vm *vm, u64 buf_vma)
{
	struct acrn_ioreq_buffer *set_buffer;
	struct page *page;
	int ret;

	if (vm->ioreq_buf)
		return -EEXIST;

	set_buffer = kzalloc(sizeof(*set_buffer), GFP_KERNEL);
	if (!set_buffer)
		return -ENOMEM;

	ret = pin_user_pages_fast(buf_vma, 1,
				  FOLL_WRITE | FOLL_LONGTERM, &page);
	if (unlikely(ret != 1) || !page) {
		dev_err(acrn_dev.this_device, "Failed to pin ioreq page!\n");
		ret = -EFAULT;
		goto free_buf;
	}

	vm->ioreq_buf = page_address(page);
	vm->ioreq_page = page;
	set_buffer->ioreq_buf = page_to_phys(page);
	ret = hcall_set_ioreq_buffer(vm->vmid, virt_to_phys(set_buffer));
	if (ret < 0) {
		dev_err(acrn_dev.this_device, "Failed to init ioreq buffer!\n");
		unpin_user_page(page);
		vm->ioreq_buf = NULL;
		goto free_buf;
	}

	dev_dbg(acrn_dev.this_device,
		"Init ioreq buffer %pK!\n", vm->ioreq_buf);
	ret = 0;
free_buf:
	kfree(set_buffer);
	return ret;
}

void acrn_ioreq_deinit(struct acrn_vm *vm)
{
	struct acrn_ioreq_client *client, *next;

	dev_dbg(acrn_dev.this_device,
		"Deinit ioreq buffer %pK!\n", vm->ioreq_buf);
	/* Destroy all clients belonging to this VM */
	list_for_each_entry_safe(client, next, &vm->ioreq_clients, list)
		acrn_ioreq_client_destroy(client);
	if (vm->default_client)
		acrn_ioreq_client_destroy(vm->default_client);

	if (vm->ioreq_buf && vm->ioreq_page) {
		unpin_user_page(vm->ioreq_page);
		vm->ioreq_buf = NULL;
	}
}