arch/riscv/kvm/aia_aplic.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2021 Western Digital Corporation or its affiliates.
  * Copyright (C) 2022 Ventana Micro Systems Inc.
  *
  * Authors:
  *	Anup Patel <apatel@ventanamicro.com>
  */

 #include <linux/kvm_host.h>
 #include <linux/math.h>
 #include <linux/spinlock.h>
 #include <linux/swab.h>
 #include <kvm/iodev.h>
 #include <asm/kvm_aia_aplic.h>

 struct aplic_irq {
 	raw_spinlock_t lock;
 	u32 sourcecfg;
 	u32 state;
 #define APLIC_IRQ_STATE_PENDING		BIT(0)
 #define APLIC_IRQ_STATE_ENABLED		BIT(1)
 #define APLIC_IRQ_STATE_ENPEND		(APLIC_IRQ_STATE_PENDING | \
 					 APLIC_IRQ_STATE_ENABLED)
 #define APLIC_IRQ_STATE_INPUT		BIT(8)
 	u32 target;
 };

 struct aplic {
 	struct kvm_io_device iodev;

 	u32 domaincfg;
 	u32 genmsi;

 	u32 nr_irqs;
 	u32 nr_words;
 	struct aplic_irq *irqs;
 };

 static u32 aplic_read_sourcecfg(struct aplic *aplic, u32 irq)
 {
 	u32 ret;
 	unsigned long flags;
 	struct aplic_irq *irqd;

 	if (!irq || aplic->nr_irqs <= irq)
 		return 0;
 	irqd = &aplic->irqs[irq];

 	raw_spin_lock_irqsave(&irqd->lock, flags);
 	ret = irqd->sourcecfg;
 	raw_spin_unlock_irqrestore(&irqd->lock, flags);

 	return ret;
 }

 static void aplic_write_sourcecfg(struct aplic *aplic, u32 irq, u32 val)
 {
 	unsigned long flags;
 	struct aplic_irq *irqd;

 	if (!irq || aplic->nr_irqs <= irq)
 		return;
 	irqd = &aplic->irqs[irq];

 	if (val & APLIC_SOURCECFG_D)
 		val = 0;
 	else
 		val &= APLIC_SOURCECFG_SM_MASK;

 	raw_spin_lock_irqsave(&irqd->lock, flags);
 	irqd->sourcecfg = val;
 	raw_spin_unlock_irqrestore(&irqd->lock, flags);
 }

 static u32 aplic_read_target(struct aplic *aplic, u32 irq)
 {
 	u32 ret;
 	unsigned long flags;
 	struct aplic_irq *irqd;

 	if (!irq || aplic->nr_irqs <= irq)
 		return 0;
 	irqd = &aplic->irqs[irq];

 	raw_spin_lock_irqsave(&irqd->lock, flags);
 	ret = irqd->target;
 	raw_spin_unlock_irqrestore(&irqd->lock, flags);

 	return ret;
 }

 static void aplic_write_target(struct aplic *aplic, u32 irq, u32 val)
 {
 	unsigned long flags;
 	struct aplic_irq *irqd;

 	if (!irq || aplic->nr_irqs <= irq)
 		return;
 	irqd = &aplic->irqs[irq];

 	val &= APLIC_TARGET_EIID_MASK |
 	       (APLIC_TARGET_HART_IDX_MASK << APLIC_TARGET_HART_IDX_SHIFT) |
 	       (APLIC_TARGET_GUEST_IDX_MASK << APLIC_TARGET_GUEST_IDX_SHIFT);

 	raw_spin_lock_irqsave(&irqd->lock, flags);
 	irqd->target = val;
 	raw_spin_unlock_irqrestore(&irqd->lock, flags);
 }

 static bool aplic_read_pending(struct aplic *aplic, u32 irq)
 {
 	bool ret;
 	unsigned long flags;
 	struct aplic_irq *irqd;

 	if (!irq || aplic->nr_irqs <= irq)
 		return false;
 	irqd = &aplic->irqs[irq];

 	raw_spin_lock_irqsave(&irqd->lock, flags);
 	ret = (irqd->state & APLIC_IRQ_STATE_PENDING) ? true : false;
 	raw_spin_unlock_irqrestore(&irqd->lock, flags);

 	return ret;
 }

 static void aplic_write_pending(struct aplic *aplic, u32 irq, bool pending)
 {
 	unsigned long flags, sm;
 	struct aplic_irq *irqd;

 	if (!irq || aplic->nr_irqs <= irq)
 		return;
 	irqd = &aplic->irqs[irq];

 	raw_spin_lock_irqsave(&irqd->lock, flags);

 	sm = irqd->sourcecfg & APLIC_SOURCECFG_SM_MASK;
 	if (!pending &&
 	    ((sm == APLIC_SOURCECFG_SM_LEVEL_HIGH) ||
 	     (sm == APLIC_SOURCECFG_SM_LEVEL_LOW)))
 		goto skip_write_pending;

 	if (pending)
 		irqd->state |= APLIC_IRQ_STATE_PENDING;
 	else
 		irqd->state &= ~APLIC_IRQ_STATE_PENDING;

 skip_write_pending:
 	raw_spin_unlock_irqrestore(&irqd->lock, flags);
 }

 static bool aplic_read_enabled(struct aplic *aplic, u32 irq)
 {
 	bool ret;
 	unsigned long flags;
 	struct aplic_irq *irqd;

 	if (!irq || aplic->nr_irqs <= irq)
 		return false;
 	irqd = &aplic->irqs[irq];

 	raw_spin_lock_irqsave(&irqd->lock, flags);
 	ret = (irqd->state & APLIC_IRQ_STATE_ENABLED) ? true : false;
 	raw_spin_unlock_irqrestore(&irqd->lock, flags);

 	return ret;
 }

 static void aplic_write_enabled(struct aplic *aplic, u32 irq, bool enabled)
 {
 	unsigned long flags;
 	struct aplic_irq *irqd;

 	if (!irq || aplic->nr_irqs <= irq)
 		return;
 	irqd = &aplic->irqs[irq];

 	raw_spin_lock_irqsave(&irqd->lock, flags);
 	if (enabled)
 		irqd->state |= APLIC_IRQ_STATE_ENABLED;
 	else
 		irqd->state &= ~APLIC_IRQ_STATE_ENABLED;
 	raw_spin_unlock_irqrestore(&irqd->lock, flags);
 }

 static bool aplic_read_input(struct aplic *aplic, u32 irq)
 {
 	bool ret;
 	unsigned long flags;
 	struct aplic_irq *irqd;

 	if (!irq || aplic->nr_irqs <= irq)
 		return false;
 	irqd = &aplic->irqs[irq];

 	raw_spin_lock_irqsave(&irqd->lock, flags);
 	ret = (irqd->state & APLIC_IRQ_STATE_INPUT) ? true : false;
 	raw_spin_unlock_irqrestore(&irqd->lock, flags);

 	return ret;
 }

 static void aplic_inject_msi(struct kvm *kvm, u32 irq, u32 target)
 {
 	u32 hart_idx, guest_idx, eiid;

 	hart_idx = target >> APLIC_TARGET_HART_IDX_SHIFT;
 	hart_idx &= APLIC_TARGET_HART_IDX_MASK;
 	guest_idx = target >> APLIC_TARGET_GUEST_IDX_SHIFT;
 	guest_idx &= APLIC_TARGET_GUEST_IDX_MASK;
 	eiid = target & APLIC_TARGET_EIID_MASK;
 	kvm_riscv_aia_inject_msi_by_id(kvm, hart_idx, guest_idx, eiid);
 }

 static void aplic_update_irq_range(struct kvm *kvm, u32 first, u32 last)
 {
 	bool inject;
 	u32 irq, target;
 	unsigned long flags;
 	struct aplic_irq *irqd;
 	struct aplic *aplic = kvm->arch.aia.aplic_state;

 	if (!(aplic->domaincfg & APLIC_DOMAINCFG_IE))
 		return;

 	for (irq = first; irq <= last; irq++) {
 		if (!irq || aplic->nr_irqs <= irq)
 			continue;
 		irqd = &aplic->irqs[irq];

 		raw_spin_lock_irqsave(&irqd->lock, flags);

 		inject = false;
 		target = irqd->target;
 		if ((irqd->state & APLIC_IRQ_STATE_ENPEND) ==
 		    APLIC_IRQ_STATE_ENPEND) {
 			irqd->state &= ~APLIC_IRQ_STATE_PENDING;
 			inject = true;
 		}

 		raw_spin_unlock_irqrestore(&irqd->lock, flags);

 		if (inject)
 			aplic_inject_msi(kvm, irq, target);
 	}
 }

 int kvm_riscv_aia_aplic_inject(struct kvm *kvm, u32 source, bool level)
 {
 	u32 target;
 	bool inject = false, ie;
 	unsigned long flags;
 	struct aplic_irq *irqd;
 	struct aplic *aplic = kvm->arch.aia.aplic_state;

 	if (!aplic || !source || (aplic->nr_irqs <= source))
 		return -ENODEV;
 	irqd = &aplic->irqs[source];
 	ie = (aplic->domaincfg & APLIC_DOMAINCFG_IE) ? true : false;

 	raw_spin_lock_irqsave(&irqd->lock, flags);

 	if (irqd->sourcecfg & APLIC_SOURCECFG_D)
 		goto skip_unlock;

 	switch (irqd->sourcecfg & APLIC_SOURCECFG_SM_MASK) {
 	case APLIC_SOURCECFG_SM_EDGE_RISE:
 		if (level && !(irqd->state & APLIC_IRQ_STATE_INPUT) &&
 		    !(irqd->state & APLIC_IRQ_STATE_PENDING))
 			irqd->state |= APLIC_IRQ_STATE_PENDING;
 		break;
 	case APLIC_SOURCECFG_SM_EDGE_FALL:
 		if (!level && (irqd->state & APLIC_IRQ_STATE_INPUT) &&
 		    !(irqd->state & APLIC_IRQ_STATE_PENDING))
 			irqd->state |= APLIC_IRQ_STATE_PENDING;
 		break;
 	case APLIC_SOURCECFG_SM_LEVEL_HIGH:
 		if (level && !(irqd->state & APLIC_IRQ_STATE_PENDING))
 			irqd->state |= APLIC_IRQ_STATE_PENDING;
 		break;
 	case APLIC_SOURCECFG_SM_LEVEL_LOW:
 		if (!level && !(irqd->state & APLIC_IRQ_STATE_PENDING))
 			irqd->state |= APLIC_IRQ_STATE_PENDING;
 		break;
 	}

 	if (level)
 		irqd->state |= APLIC_IRQ_STATE_INPUT;
 	else
 		irqd->state &= ~APLIC_IRQ_STATE_INPUT;

 	target = irqd->target;
 	if (ie && ((irqd->state & APLIC_IRQ_STATE_ENPEND) ==
 		   APLIC_IRQ_STATE_ENPEND)) {
 		irqd->state &= ~APLIC_IRQ_STATE_PENDING;
 		inject = true;
 	}

 skip_unlock:
 	raw_spin_unlock_irqrestore(&irqd->lock, flags);

 	if (inject)
 		aplic_inject_msi(kvm, source, target);

 	return 0;
 }

 static u32 aplic_read_input_word(struct aplic *aplic, u32 word)
 {
 	u32 i, ret = 0;

 	for (i = 0; i < 32; i++)
 		ret |= aplic_read_input(aplic, word * 32 + i) ? BIT(i) : 0;

 	return ret;
 }

 static u32 aplic_read_pending_word(struct aplic *aplic, u32 word)
 {
 	u32 i, ret = 0;

 	for (i = 0; i < 32; i++)
 		ret |= aplic_read_pending(aplic, word * 32 + i) ? BIT(i) : 0;

 	return ret;
 }

 static void aplic_write_pending_word(struct aplic *aplic, u32 word,
 				     u32 val, bool pending)
 {
 	u32 i;

 	for (i = 0; i < 32; i++) {
 		if (val & BIT(i))
 			aplic_write_pending(aplic, word * 32 + i, pending);
 	}
 }

 static u32 aplic_read_enabled_word(struct aplic *aplic, u32 word)
 {
 	u32 i, ret = 0;

 	for (i = 0; i < 32; i++)
 		ret |= aplic_read_enabled(aplic, word * 32 + i) ? BIT(i) : 0;

 	return ret;
 }

 static void aplic_write_enabled_word(struct aplic *aplic, u32 word,
 				     u32 val, bool enabled)
 {
 	u32 i;

 	for (i = 0; i < 32; i++) {
 		if (val & BIT(i))
 			aplic_write_enabled(aplic, word * 32 + i, enabled);
 	}
 }

 static int aplic_mmio_read_offset(struct kvm *kvm, gpa_t off, u32 *val32)
 {
 	u32 i;
 	struct aplic *aplic = kvm->arch.aia.aplic_state;

 	if ((off & 0x3) != 0)
 		return -EOPNOTSUPP;

 	if (off == APLIC_DOMAINCFG) {
 		*val32 = APLIC_DOMAINCFG_RDONLY |
 			 aplic->domaincfg | APLIC_DOMAINCFG_DM;
 	} else if ((off >= APLIC_SOURCECFG_BASE) &&
 		 (off < (APLIC_SOURCECFG_BASE + (aplic->nr_irqs - 1) * 4))) {
 		i = ((off - APLIC_SOURCECFG_BASE) >> 2) + 1;
 		*val32 = aplic_read_sourcecfg(aplic, i);
 	} else if ((off >= APLIC_SETIP_BASE) &&
 		   (off < (APLIC_SETIP_BASE + aplic->nr_words * 4))) {
 		i = (off - APLIC_SETIP_BASE) >> 2;
 		*val32 = aplic_read_pending_word(aplic, i);
 	} else if (off == APLIC_SETIPNUM) {
 		*val32 = 0;
 	} else if ((off >= APLIC_CLRIP_BASE) &&
 		   (off < (APLIC_CLRIP_BASE + aplic->nr_words * 4))) {
 		i = (off - APLIC_CLRIP_BASE) >> 2;
 		*val32 = aplic_read_input_word(aplic, i);
 	} else if (off == APLIC_CLRIPNUM) {
 		*val32 = 0;
 	} else if ((off >= APLIC_SETIE_BASE) &&
 		   (off < (APLIC_SETIE_BASE + aplic->nr_words * 4))) {
 		i = (off - APLIC_SETIE_BASE) >> 2;
 		*val32 = aplic_read_enabled_word(aplic, i);
 	} else if (off == APLIC_SETIENUM) {
 		*val32 = 0;
 	} else if ((off >= APLIC_CLRIE_BASE) &&
 		   (off < (APLIC_CLRIE_BASE + aplic->nr_words * 4))) {
 		*val32 = 0;
 	} else if (off == APLIC_CLRIENUM) {
 		*val32 = 0;
 	} else if (off == APLIC_SETIPNUM_LE) {
 		*val32 = 0;
 	} else if (off == APLIC_SETIPNUM_BE) {
 		*val32 = 0;
 	} else if (off == APLIC_GENMSI) {
 		*val32 = aplic->genmsi;
 	} else if ((off >= APLIC_TARGET_BASE) &&
 		   (off < (APLIC_TARGET_BASE + (aplic->nr_irqs - 1) * 4))) {
 		i = ((off - APLIC_TARGET_BASE) >> 2) + 1;
 		*val32 = aplic_read_target(aplic, i);
 	} else
 		return -ENODEV;

 	return 0;
 }

 static int aplic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
 			   gpa_t addr, int len, void *val)
 {
 	if (len != 4)
 		return -EOPNOTSUPP;

 	return aplic_mmio_read_offset(vcpu->kvm,
 				      addr - vcpu->kvm->arch.aia.aplic_addr,
 				      val);
 }

 static int aplic_mmio_write_offset(struct kvm *kvm, gpa_t off, u32 val32)
 {
 	u32 i;
 	struct aplic *aplic = kvm->arch.aia.aplic_state;

 	if ((off & 0x3) != 0)
 		return -EOPNOTSUPP;

 	if (off == APLIC_DOMAINCFG) {
 		/* Only IE bit writeable */
 		aplic->domaincfg = val32 & APLIC_DOMAINCFG_IE;
 	} else if ((off >= APLIC_SOURCECFG_BASE) &&
 		 (off < (APLIC_SOURCECFG_BASE + (aplic->nr_irqs - 1) * 4))) {
 		i = ((off - APLIC_SOURCECFG_BASE) >> 2) + 1;
 		aplic_write_sourcecfg(aplic, i, val32);
 	} else if ((off >= APLIC_SETIP_BASE) &&
 		   (off < (APLIC_SETIP_BASE + aplic->nr_words * 4))) {
 		i = (off - APLIC_SETIP_BASE) >> 2;
 		aplic_write_pending_word(aplic, i, val32, true);
 	} else if (off == APLIC_SETIPNUM) {
 		aplic_write_pending(aplic, val32, true);
 	} else if ((off >= APLIC_CLRIP_BASE) &&
 		   (off < (APLIC_CLRIP_BASE + aplic->nr_words * 4))) {
 		i = (off - APLIC_CLRIP_BASE) >> 2;
 		aplic_write_pending_word(aplic, i, val32, false);
 	} else if (off == APLIC_CLRIPNUM) {
 		aplic_write_pending(aplic, val32, false);
 	} else if ((off >= APLIC_SETIE_BASE) &&
 		   (off < (APLIC_SETIE_BASE + aplic->nr_words * 4))) {
 		i = (off - APLIC_SETIE_BASE) >> 2;
 		aplic_write_enabled_word(aplic, i, val32, true);
 	} else if (off == APLIC_SETIENUM) {
 		aplic_write_enabled(aplic, val32, true);
 	} else if ((off >= APLIC_CLRIE_BASE) &&
 		   (off < (APLIC_CLRIE_BASE + aplic->nr_words * 4))) {
 		i = (off - APLIC_CLRIE_BASE) >> 2;
 		aplic_write_enabled_word(aplic, i, val32, false);
 	} else if (off == APLIC_CLRIENUM) {
 		aplic_write_enabled(aplic, val32, false);
 	} else if (off == APLIC_SETIPNUM_LE) {
 		aplic_write_pending(aplic, val32, true);
 	} else if (off == APLIC_SETIPNUM_BE) {
 		aplic_write_pending(aplic, __swab32(val32), true);
 	} else if (off == APLIC_GENMSI) {
 		aplic->genmsi = val32 & ~(APLIC_TARGET_GUEST_IDX_MASK <<
 					  APLIC_TARGET_GUEST_IDX_SHIFT);
 		kvm_riscv_aia_inject_msi_by_id(kvm,
 				val32 >> APLIC_TARGET_HART_IDX_SHIFT, 0,
 				val32 & APLIC_TARGET_EIID_MASK);
 	} else if ((off >= APLIC_TARGET_BASE) &&
 		   (off < (APLIC_TARGET_BASE + (aplic->nr_irqs - 1) * 4))) {
 		i = ((off - APLIC_TARGET_BASE) >> 2) + 1;
 		aplic_write_target(aplic, i, val32);
 	} else
 		return -ENODEV;

 	aplic_update_irq_range(kvm, 1, aplic->nr_irqs - 1);

 	return 0;
 }

 static int aplic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
 			    gpa_t addr, int len, const void *val)
 {
 	if (len != 4)
 		return -EOPNOTSUPP;

 	return aplic_mmio_write_offset(vcpu->kvm,
 				       addr - vcpu->kvm->arch.aia.aplic_addr,
 				       *((const u32 *)val));
 }

 static struct kvm_io_device_ops aplic_iodoev_ops = {
 	.read = aplic_mmio_read,
 	.write = aplic_mmio_write,
 };

 int kvm_riscv_aia_aplic_set_attr(struct kvm *kvm, unsigned long type, u32 v)
 {
 	int rc;

 	if (!kvm->arch.aia.aplic_state)
 		return -ENODEV;

 	rc = aplic_mmio_write_offset(kvm, type, v);
 	if (rc)
 		return rc;

 	return 0;
 }

 int kvm_riscv_aia_aplic_get_attr(struct kvm *kvm, unsigned long type, u32 *v)
 {
 	int rc;

 	if (!kvm->arch.aia.aplic_state)
 		return -ENODEV;

 	rc = aplic_mmio_read_offset(kvm, type, v);
 	if (rc)
 		return rc;

 	return 0;
 }

 int kvm_riscv_aia_aplic_has_attr(struct kvm *kvm, unsigned long type)
 {
 	int rc;
 	u32 val;

 	if (!kvm->arch.aia.aplic_state)
 		return -ENODEV;

 	rc = aplic_mmio_read_offset(kvm, type, &val);
 	if (rc)
 		return rc;

 	return 0;
 }

 int kvm_riscv_aia_aplic_init(struct kvm *kvm)
 {
 	int i, ret = 0;
 	struct aplic *aplic;

 	/* Do nothing if we have zero sources */
 	if (!kvm->arch.aia.nr_sources)
 		return 0;

 	/* Allocate APLIC global state */
 	aplic = kzalloc(sizeof(*aplic), GFP_KERNEL);
 	if (!aplic)
 		return -ENOMEM;
 	kvm->arch.aia.aplic_state = aplic;

 	/* Setup APLIC IRQs */
 	aplic->nr_irqs = kvm->arch.aia.nr_sources + 1;
 	aplic->nr_words = DIV_ROUND_UP(aplic->nr_irqs, 32);
 	aplic->irqs = kcalloc(aplic->nr_irqs,
 			      sizeof(*aplic->irqs), GFP_KERNEL);
 	if (!aplic->irqs) {
 		ret = -ENOMEM;
 		goto fail_free_aplic;
 	}
 	for (i = 0; i < aplic->nr_irqs; i++)
 		raw_spin_lock_init(&aplic->irqs[i].lock);

 	/* Setup IO device */
 	kvm_iodevice_init(&aplic->iodev, &aplic_iodoev_ops);
 	mutex_lock(&kvm->slots_lock);
 	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS,
 				      kvm->arch.aia.aplic_addr,
 				      KVM_DEV_RISCV_APLIC_SIZE,
 				      &aplic->iodev);
 	mutex_unlock(&kvm->slots_lock);
 	if (ret)
 		goto fail_free_aplic_irqs;

 	/* Setup default IRQ routing */
 	ret = kvm_riscv_setup_default_irq_routing(kvm, aplic->nr_irqs);
 	if (ret)
 		goto fail_unreg_iodev;

 	return 0;

 fail_unreg_iodev:
 	mutex_lock(&kvm->slots_lock);
 	kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev);
 	mutex_unlock(&kvm->slots_lock);
 fail_free_aplic_irqs:
 	kfree(aplic->irqs);
 fail_free_aplic:
 	kvm->arch.aia.aplic_state = NULL;
 	kfree(aplic);
 	return ret;
 }

 void kvm_riscv_aia_aplic_cleanup(struct kvm *kvm)
 {
 	struct aplic *aplic = kvm->arch.aia.aplic_state;

 	if (!aplic)
 		return;

 	mutex_lock(&kvm->slots_lock);
 	kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev);
 	mutex_unlock(&kvm->slots_lock);

 	kfree(aplic->irqs);

 	kvm->arch.aia.aplic_state = NULL;
 	kfree(aplic);
 }
	// SPDX-License-Identifier: GPL-2.0
	/*
	* Copyright (C) 2021 Western Digital Corporation or its affiliates.
	* Copyright (C) 2022 Ventana Micro Systems Inc.
	*
	* Authors:
	* Anup Patel <apatel@ventanamicro.com>
	*/

	#include <linux/kvm_host.h>
	#include <linux/math.h>
	#include <linux/spinlock.h>
	#include <linux/swab.h>
	#include <kvm/iodev.h>
	#include <asm/kvm_aia_aplic.h>

	struct aplic_irq {
	raw_spinlock_t lock;
	u32 sourcecfg;
	u32 state;
	#define APLIC_IRQ_STATE_PENDING BIT(0)
	#define APLIC_IRQ_STATE_ENABLED BIT(1)
	#define APLIC_IRQ_STATE_ENPEND (APLIC_IRQ_STATE_PENDING \| \
	APLIC_IRQ_STATE_ENABLED)
	#define APLIC_IRQ_STATE_INPUT BIT(8)
	u32 target;
	};

	struct aplic {
	struct kvm_io_device iodev;

	u32 domaincfg;
	u32 genmsi;

	u32 nr_irqs;
	u32 nr_words;
	struct aplic_irq *irqs;
	};

	static u32 aplic_read_sourcecfg(struct aplic *aplic, u32 irq)
	{
	u32 ret;
	unsigned long flags;
	struct aplic_irq *irqd;

	if (!irq \|\| aplic->nr_irqs <= irq)
	return 0;
	irqd = &aplic->irqs[irq];

	raw_spin_lock_irqsave(&irqd->lock, flags);
	ret = irqd->sourcecfg;
	raw_spin_unlock_irqrestore(&irqd->lock, flags);

	return ret;
	}

	static void aplic_write_sourcecfg(struct aplic *aplic, u32 irq, u32 val)
	{
	unsigned long flags;
	struct aplic_irq *irqd;

	if (!irq \|\| aplic->nr_irqs <= irq)
	return;
	irqd = &aplic->irqs[irq];

	if (val & APLIC_SOURCECFG_D)
	val = 0;
	else
	val &= APLIC_SOURCECFG_SM_MASK;

	raw_spin_lock_irqsave(&irqd->lock, flags);
	irqd->sourcecfg = val;
	raw_spin_unlock_irqrestore(&irqd->lock, flags);
	}

	static u32 aplic_read_target(struct aplic *aplic, u32 irq)
	{
	u32 ret;
	unsigned long flags;
	struct aplic_irq *irqd;

	if (!irq \|\| aplic->nr_irqs <= irq)
	return 0;
	irqd = &aplic->irqs[irq];

	raw_spin_lock_irqsave(&irqd->lock, flags);
	ret = irqd->target;
	raw_spin_unlock_irqrestore(&irqd->lock, flags);

	return ret;
	}

	static void aplic_write_target(struct aplic *aplic, u32 irq, u32 val)
	{
	unsigned long flags;
	struct aplic_irq *irqd;

	if (!irq \|\| aplic->nr_irqs <= irq)
	return;
	irqd = &aplic->irqs[irq];

	val &= APLIC_TARGET_EIID_MASK \|
	(APLIC_TARGET_HART_IDX_MASK << APLIC_TARGET_HART_IDX_SHIFT) \|
	(APLIC_TARGET_GUEST_IDX_MASK << APLIC_TARGET_GUEST_IDX_SHIFT);

	raw_spin_lock_irqsave(&irqd->lock, flags);
	irqd->target = val;
	raw_spin_unlock_irqrestore(&irqd->lock, flags);
	}

	static bool aplic_read_pending(struct aplic *aplic, u32 irq)
	{
	bool ret;
	unsigned long flags;
	struct aplic_irq *irqd;

	if (!irq \|\| aplic->nr_irqs <= irq)
	return false;
	irqd = &aplic->irqs[irq];

	raw_spin_lock_irqsave(&irqd->lock, flags);
	ret = (irqd->state & APLIC_IRQ_STATE_PENDING) ? true : false;
	raw_spin_unlock_irqrestore(&irqd->lock, flags);

	return ret;
	}

	static void aplic_write_pending(struct aplic *aplic, u32 irq, bool pending)
	{
	unsigned long flags, sm;
	struct aplic_irq *irqd;

	if (!irq \|\| aplic->nr_irqs <= irq)
	return;
	irqd = &aplic->irqs[irq];

	raw_spin_lock_irqsave(&irqd->lock, flags);

	sm = irqd->sourcecfg & APLIC_SOURCECFG_SM_MASK;
	if (!pending &&
	((sm == APLIC_SOURCECFG_SM_LEVEL_HIGH) \|\|
	(sm == APLIC_SOURCECFG_SM_LEVEL_LOW)))
	goto skip_write_pending;

	if (pending)
	irqd->state \|= APLIC_IRQ_STATE_PENDING;
	else
	irqd->state &= ~APLIC_IRQ_STATE_PENDING;

	skip_write_pending:
	raw_spin_unlock_irqrestore(&irqd->lock, flags);
	}

	static bool aplic_read_enabled(struct aplic *aplic, u32 irq)
	{
	bool ret;
	unsigned long flags;
	struct aplic_irq *irqd;

	if (!irq \|\| aplic->nr_irqs <= irq)
	return false;
	irqd = &aplic->irqs[irq];

	raw_spin_lock_irqsave(&irqd->lock, flags);
	ret = (irqd->state & APLIC_IRQ_STATE_ENABLED) ? true : false;
	raw_spin_unlock_irqrestore(&irqd->lock, flags);

	return ret;
	}

	static void aplic_write_enabled(struct aplic *aplic, u32 irq, bool enabled)
	{
	unsigned long flags;
	struct aplic_irq *irqd;

	if (!irq \|\| aplic->nr_irqs <= irq)
	return;
	irqd = &aplic->irqs[irq];

	raw_spin_lock_irqsave(&irqd->lock, flags);
	if (enabled)
	irqd->state \|= APLIC_IRQ_STATE_ENABLED;
	else
	irqd->state &= ~APLIC_IRQ_STATE_ENABLED;
	raw_spin_unlock_irqrestore(&irqd->lock, flags);
	}

	static bool aplic_read_input(struct aplic *aplic, u32 irq)
	{
	bool ret;
	unsigned long flags;
	struct aplic_irq *irqd;

	if (!irq \|\| aplic->nr_irqs <= irq)
	return false;
	irqd = &aplic->irqs[irq];

	raw_spin_lock_irqsave(&irqd->lock, flags);
	ret = (irqd->state & APLIC_IRQ_STATE_INPUT) ? true : false;
	raw_spin_unlock_irqrestore(&irqd->lock, flags);

	return ret;
	}

	static void aplic_inject_msi(struct kvm *kvm, u32 irq, u32 target)
	{
	u32 hart_idx, guest_idx, eiid;

	hart_idx = target >> APLIC_TARGET_HART_IDX_SHIFT;
	hart_idx &= APLIC_TARGET_HART_IDX_MASK;
	guest_idx = target >> APLIC_TARGET_GUEST_IDX_SHIFT;
	guest_idx &= APLIC_TARGET_GUEST_IDX_MASK;
	eiid = target & APLIC_TARGET_EIID_MASK;
	kvm_riscv_aia_inject_msi_by_id(kvm, hart_idx, guest_idx, eiid);
	}

	static void aplic_update_irq_range(struct kvm *kvm, u32 first, u32 last)
	{
	bool inject;
	u32 irq, target;
	unsigned long flags;
	struct aplic_irq *irqd;
	struct aplic *aplic = kvm->arch.aia.aplic_state;

	if (!(aplic->domaincfg & APLIC_DOMAINCFG_IE))
	return;

	for (irq = first; irq <= last; irq++) {
	if (!irq \|\| aplic->nr_irqs <= irq)
	continue;
	irqd = &aplic->irqs[irq];

	raw_spin_lock_irqsave(&irqd->lock, flags);

	inject = false;
	target = irqd->target;
	if ((irqd->state & APLIC_IRQ_STATE_ENPEND) ==
	APLIC_IRQ_STATE_ENPEND) {
	irqd->state &= ~APLIC_IRQ_STATE_PENDING;
	inject = true;
	}

	raw_spin_unlock_irqrestore(&irqd->lock, flags);

	if (inject)
	aplic_inject_msi(kvm, irq, target);
	}
	}

	int kvm_riscv_aia_aplic_inject(struct kvm *kvm, u32 source, bool level)
	{
	u32 target;
	bool inject = false, ie;
	unsigned long flags;
	struct aplic_irq *irqd;
	struct aplic *aplic = kvm->arch.aia.aplic_state;

	if (!aplic \|\| !source \|\| (aplic->nr_irqs <= source))
	return -ENODEV;
	irqd = &aplic->irqs[source];
	ie = (aplic->domaincfg & APLIC_DOMAINCFG_IE) ? true : false;

	raw_spin_lock_irqsave(&irqd->lock, flags);

	if (irqd->sourcecfg & APLIC_SOURCECFG_D)
	goto skip_unlock;

	switch (irqd->sourcecfg & APLIC_SOURCECFG_SM_MASK) {
	case APLIC_SOURCECFG_SM_EDGE_RISE:
	if (level && !(irqd->state & APLIC_IRQ_STATE_INPUT) &&
	!(irqd->state & APLIC_IRQ_STATE_PENDING))
	irqd->state \|= APLIC_IRQ_STATE_PENDING;
	break;
	case APLIC_SOURCECFG_SM_EDGE_FALL:
	if (!level && (irqd->state & APLIC_IRQ_STATE_INPUT) &&
	!(irqd->state & APLIC_IRQ_STATE_PENDING))
	irqd->state \|= APLIC_IRQ_STATE_PENDING;
	break;
	case APLIC_SOURCECFG_SM_LEVEL_HIGH:
	if (level && !(irqd->state & APLIC_IRQ_STATE_PENDING))
	irqd->state \|= APLIC_IRQ_STATE_PENDING;
	break;
	case APLIC_SOURCECFG_SM_LEVEL_LOW:
	if (!level && !(irqd->state & APLIC_IRQ_STATE_PENDING))
	irqd->state \|= APLIC_IRQ_STATE_PENDING;
	break;
	}

	if (level)
	irqd->state \|= APLIC_IRQ_STATE_INPUT;
	else
	irqd->state &= ~APLIC_IRQ_STATE_INPUT;

	target = irqd->target;
	if (ie && ((irqd->state & APLIC_IRQ_STATE_ENPEND) ==
	APLIC_IRQ_STATE_ENPEND)) {
	irqd->state &= ~APLIC_IRQ_STATE_PENDING;
	inject = true;
	}

	skip_unlock:
	raw_spin_unlock_irqrestore(&irqd->lock, flags);

	if (inject)
	aplic_inject_msi(kvm, source, target);

	return 0;
	}

	static u32 aplic_read_input_word(struct aplic *aplic, u32 word)
	{
	u32 i, ret = 0;

	for (i = 0; i < 32; i++)
	ret \|= aplic_read_input(aplic, word * 32 + i) ? BIT(i) : 0;

	return ret;
	}

	static u32 aplic_read_pending_word(struct aplic *aplic, u32 word)
	{
	u32 i, ret = 0;

	for (i = 0; i < 32; i++)
	ret \|= aplic_read_pending(aplic, word * 32 + i) ? BIT(i) : 0;

	return ret;
	}

	static void aplic_write_pending_word(struct aplic *aplic, u32 word,
	u32 val, bool pending)
	{
	u32 i;

	for (i = 0; i < 32; i++) {
	if (val & BIT(i))
	aplic_write_pending(aplic, word * 32 + i, pending);
	}
	}

	static u32 aplic_read_enabled_word(struct aplic *aplic, u32 word)
	{
	u32 i, ret = 0;

	for (i = 0; i < 32; i++)
	ret \|= aplic_read_enabled(aplic, word * 32 + i) ? BIT(i) : 0;

	return ret;
	}

	static void aplic_write_enabled_word(struct aplic *aplic, u32 word,
	u32 val, bool enabled)
	{
	u32 i;

	for (i = 0; i < 32; i++) {
	if (val & BIT(i))
	aplic_write_enabled(aplic, word * 32 + i, enabled);
	}
	}

	static int aplic_mmio_read_offset(struct kvm kvm, gpa_t off, u32 val32)
	{
	u32 i;
	struct aplic *aplic = kvm->arch.aia.aplic_state;

	if ((off & 0x3) != 0)
	return -EOPNOTSUPP;

	if (off == APLIC_DOMAINCFG) {
	*val32 = APLIC_DOMAINCFG_RDONLY \|
	aplic->domaincfg \| APLIC_DOMAINCFG_DM;
	} else if ((off >= APLIC_SOURCECFG_BASE) &&
	(off < (APLIC_SOURCECFG_BASE + (aplic->nr_irqs - 1) * 4))) {
	i = ((off - APLIC_SOURCECFG_BASE) >> 2) + 1;
	*val32 = aplic_read_sourcecfg(aplic, i);
	} else if ((off >= APLIC_SETIP_BASE) &&
	(off < (APLIC_SETIP_BASE + aplic->nr_words * 4))) {
	i = (off - APLIC_SETIP_BASE) >> 2;
	*val32 = aplic_read_pending_word(aplic, i);
	} else if (off == APLIC_SETIPNUM) {
	*val32 = 0;
	} else if ((off >= APLIC_CLRIP_BASE) &&
	(off < (APLIC_CLRIP_BASE + aplic->nr_words * 4))) {
	i = (off - APLIC_CLRIP_BASE) >> 2;
	*val32 = aplic_read_input_word(aplic, i);
	} else if (off == APLIC_CLRIPNUM) {
	*val32 = 0;
	} else if ((off >= APLIC_SETIE_BASE) &&
	(off < (APLIC_SETIE_BASE + aplic->nr_words * 4))) {
	i = (off - APLIC_SETIE_BASE) >> 2;
	*val32 = aplic_read_enabled_word(aplic, i);
	} else if (off == APLIC_SETIENUM) {
	*val32 = 0;
	} else if ((off >= APLIC_CLRIE_BASE) &&
	(off < (APLIC_CLRIE_BASE + aplic->nr_words * 4))) {
	*val32 = 0;
	} else if (off == APLIC_CLRIENUM) {
	*val32 = 0;
	} else if (off == APLIC_SETIPNUM_LE) {
	*val32 = 0;
	} else if (off == APLIC_SETIPNUM_BE) {
	*val32 = 0;
	} else if (off == APLIC_GENMSI) {
	*val32 = aplic->genmsi;
	} else if ((off >= APLIC_TARGET_BASE) &&
	(off < (APLIC_TARGET_BASE + (aplic->nr_irqs - 1) * 4))) {
	i = ((off - APLIC_TARGET_BASE) >> 2) + 1;
	*val32 = aplic_read_target(aplic, i);
	} else
	return -ENODEV;

	return 0;
	}

	static int aplic_mmio_read(struct kvm_vcpu vcpu, struct kvm_io_device dev,
	gpa_t addr, int len, void *val)
	{
	if (len != 4)
	return -EOPNOTSUPP;

	return aplic_mmio_read_offset(vcpu->kvm,
	addr - vcpu->kvm->arch.aia.aplic_addr,
	val);
	}

	static int aplic_mmio_write_offset(struct kvm *kvm, gpa_t off, u32 val32)
	{
	u32 i;
	struct aplic *aplic = kvm->arch.aia.aplic_state;

	if ((off & 0x3) != 0)
	return -EOPNOTSUPP;

	if (off == APLIC_DOMAINCFG) {
	/* Only IE bit writeable */
	aplic->domaincfg = val32 & APLIC_DOMAINCFG_IE;
	} else if ((off >= APLIC_SOURCECFG_BASE) &&
	(off < (APLIC_SOURCECFG_BASE + (aplic->nr_irqs - 1) * 4))) {
	i = ((off - APLIC_SOURCECFG_BASE) >> 2) + 1;
	aplic_write_sourcecfg(aplic, i, val32);
	} else if ((off >= APLIC_SETIP_BASE) &&
	(off < (APLIC_SETIP_BASE + aplic->nr_words * 4))) {
	i = (off - APLIC_SETIP_BASE) >> 2;
	aplic_write_pending_word(aplic, i, val32, true);
	} else if (off == APLIC_SETIPNUM) {
	aplic_write_pending(aplic, val32, true);
	} else if ((off >= APLIC_CLRIP_BASE) &&
	(off < (APLIC_CLRIP_BASE + aplic->nr_words * 4))) {
	i = (off - APLIC_CLRIP_BASE) >> 2;
	aplic_write_pending_word(aplic, i, val32, false);
	} else if (off == APLIC_CLRIPNUM) {
	aplic_write_pending(aplic, val32, false);
	} else if ((off >= APLIC_SETIE_BASE) &&
	(off < (APLIC_SETIE_BASE + aplic->nr_words * 4))) {
	i = (off - APLIC_SETIE_BASE) >> 2;
	aplic_write_enabled_word(aplic, i, val32, true);
	} else if (off == APLIC_SETIENUM) {
	aplic_write_enabled(aplic, val32, true);
	} else if ((off >= APLIC_CLRIE_BASE) &&
	(off < (APLIC_CLRIE_BASE + aplic->nr_words * 4))) {
	i = (off - APLIC_CLRIE_BASE) >> 2;
	aplic_write_enabled_word(aplic, i, val32, false);
	} else if (off == APLIC_CLRIENUM) {
	aplic_write_enabled(aplic, val32, false);
	} else if (off == APLIC_SETIPNUM_LE) {
	aplic_write_pending(aplic, val32, true);
	} else if (off == APLIC_SETIPNUM_BE) {
	aplic_write_pending(aplic, __swab32(val32), true);
	} else if (off == APLIC_GENMSI) {
	aplic->genmsi = val32 & ~(APLIC_TARGET_GUEST_IDX_MASK <<
	APLIC_TARGET_GUEST_IDX_SHIFT);
	kvm_riscv_aia_inject_msi_by_id(kvm,
	val32 >> APLIC_TARGET_HART_IDX_SHIFT, 0,
	val32 & APLIC_TARGET_EIID_MASK);
	} else if ((off >= APLIC_TARGET_BASE) &&
	(off < (APLIC_TARGET_BASE + (aplic->nr_irqs - 1) * 4))) {
	i = ((off - APLIC_TARGET_BASE) >> 2) + 1;
	aplic_write_target(aplic, i, val32);
	} else
	return -ENODEV;

	aplic_update_irq_range(kvm, 1, aplic->nr_irqs - 1);

	return 0;
	}

	static int aplic_mmio_write(struct kvm_vcpu vcpu, struct kvm_io_device dev,
	gpa_t addr, int len, const void *val)
	{
	if (len != 4)
	return -EOPNOTSUPP;

	return aplic_mmio_write_offset(vcpu->kvm,
	addr - vcpu->kvm->arch.aia.aplic_addr,
	((const u32 )val));
	}

	static struct kvm_io_device_ops aplic_iodoev_ops = {
	.read = aplic_mmio_read,
	.write = aplic_mmio_write,
	};

	int kvm_riscv_aia_aplic_set_attr(struct kvm *kvm, unsigned long type, u32 v)
	{
	int rc;

	if (!kvm->arch.aia.aplic_state)
	return -ENODEV;

	rc = aplic_mmio_write_offset(kvm, type, v);
	if (rc)
	return rc;

	return 0;
	}

	int kvm_riscv_aia_aplic_get_attr(struct kvm kvm, unsigned long type, u32 v)
	{
	int rc;

	if (!kvm->arch.aia.aplic_state)
	return -ENODEV;

	rc = aplic_mmio_read_offset(kvm, type, v);
	if (rc)
	return rc;

	return 0;
	}

	int kvm_riscv_aia_aplic_has_attr(struct kvm *kvm, unsigned long type)
	{
	int rc;
	u32 val;

	if (!kvm->arch.aia.aplic_state)
	return -ENODEV;

	rc = aplic_mmio_read_offset(kvm, type, &val);
	if (rc)
	return rc;

	return 0;
	}

	int kvm_riscv_aia_aplic_init(struct kvm *kvm)
	{
	int i, ret = 0;
	struct aplic *aplic;

	/* Do nothing if we have zero sources */
	if (!kvm->arch.aia.nr_sources)
	return 0;

	/* Allocate APLIC global state */
	aplic = kzalloc(sizeof(*aplic), GFP_KERNEL);
	if (!aplic)
	return -ENOMEM;
	kvm->arch.aia.aplic_state = aplic;

	/* Setup APLIC IRQs */
	aplic->nr_irqs = kvm->arch.aia.nr_sources + 1;
	aplic->nr_words = DIV_ROUND_UP(aplic->nr_irqs, 32);
	aplic->irqs = kcalloc(aplic->nr_irqs,
	sizeof(*aplic->irqs), GFP_KERNEL);
	if (!aplic->irqs) {
	ret = -ENOMEM;
	goto fail_free_aplic;
	}
	for (i = 0; i < aplic->nr_irqs; i++)
	raw_spin_lock_init(&aplic->irqs[i].lock);

	/* Setup IO device */
	kvm_iodevice_init(&aplic->iodev, &aplic_iodoev_ops);
	mutex_lock(&kvm->slots_lock);
	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS,
	kvm->arch.aia.aplic_addr,
	KVM_DEV_RISCV_APLIC_SIZE,
	&aplic->iodev);
	mutex_unlock(&kvm->slots_lock);
	if (ret)
	goto fail_free_aplic_irqs;

	/* Setup default IRQ routing */
	ret = kvm_riscv_setup_default_irq_routing(kvm, aplic->nr_irqs);
	if (ret)
	goto fail_unreg_iodev;

	return 0;

	fail_unreg_iodev:
	mutex_lock(&kvm->slots_lock);
	kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev);
	mutex_unlock(&kvm->slots_lock);
	fail_free_aplic_irqs:
	kfree(aplic->irqs);
	fail_free_aplic:
	kvm->arch.aia.aplic_state = NULL;
	kfree(aplic);
	return ret;
	}

	void kvm_riscv_aia_aplic_cleanup(struct kvm *kvm)
	{
	struct aplic *aplic = kvm->arch.aia.aplic_state;

	if (!aplic)
	return;

	mutex_lock(&kvm->slots_lock);
	kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev);
	mutex_unlock(&kvm->slots_lock);

	kfree(aplic->irqs);

	kvm->arch.aia.aplic_state = NULL;
	kfree(aplic);
	}