drivers/misc/habanalabs/common/irq.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0

 /*
  * Copyright 2016-2019 HabanaLabs, Ltd.
  * All Rights Reserved.
  */

 #include "habanalabs.h"

 #include <linux/slab.h>

 /**
  * struct hl_eqe_work - This structure is used to schedule work of EQ
  *                      entry and cpucp_reset event
  *
  * @eq_work:          workqueue object to run when EQ entry is received
  * @hdev:             pointer to device structure
  * @eq_entry:         copy of the EQ entry
  */
 struct hl_eqe_work {
 	struct work_struct	eq_work;
 	struct hl_device	*hdev;
 	struct hl_eq_entry	eq_entry;
 };

 /**
  * hl_cq_inc_ptr - increment ci or pi of cq
  *
  * @ptr: the current ci or pi value of the completion queue
  *
  * Increment ptr by 1. If it reaches the number of completion queue
  * entries, set it to 0
  */
 inline u32 hl_cq_inc_ptr(u32 ptr)
 {
 	ptr++;
 	if (unlikely(ptr == HL_CQ_LENGTH))
 		ptr = 0;
 	return ptr;
 }

 /**
  * hl_eq_inc_ptr - increment ci of eq
  *
  * @ptr: the current ci value of the event queue
  *
  * Increment ptr by 1. If it reaches the number of event queue
  * entries, set it to 0
  */
 static inline u32 hl_eq_inc_ptr(u32 ptr)
 {
 	ptr++;
 	if (unlikely(ptr == HL_EQ_LENGTH))
 		ptr = 0;
 	return ptr;
 }

 static void irq_handle_eqe(struct work_struct *work)
 {
 	struct hl_eqe_work *eqe_work = container_of(work, struct hl_eqe_work,
 							eq_work);
 	struct hl_device *hdev = eqe_work->hdev;

 	hdev->asic_funcs->handle_eqe(hdev, &eqe_work->eq_entry);

 	kfree(eqe_work);
 }

 /**
  * hl_irq_handler_cq - irq handler for completion queue
  *
  * @irq: irq number
  * @arg: pointer to completion queue structure
  *
  */
 irqreturn_t hl_irq_handler_cq(int irq, void *arg)
 {
 	struct hl_cq *cq = arg;
 	struct hl_device *hdev = cq->hdev;
 	struct hl_hw_queue *queue;
 	struct hl_cs_job *job;
 	bool shadow_index_valid;
 	u16 shadow_index;
 	struct hl_cq_entry *cq_entry, *cq_base;

 	if (hdev->disabled) {
 		dev_dbg(hdev->dev,
 			"Device disabled but received IRQ %d for CQ %d\n",
 			irq, cq->hw_queue_id);
 		return IRQ_HANDLED;
 	}

 	cq_base = cq->kernel_address;

 	while (1) {
 		bool entry_ready = ((le32_to_cpu(cq_base[cq->ci].data) &
 					CQ_ENTRY_READY_MASK)
 						>> CQ_ENTRY_READY_SHIFT);

 		if (!entry_ready)
 			break;

 		cq_entry = (struct hl_cq_entry *) &cq_base[cq->ci];

 		/* Make sure we read CQ entry contents after we've
 		 * checked the ownership bit.
 		 */
 		dma_rmb();

 		shadow_index_valid = ((le32_to_cpu(cq_entry->data) &
 					CQ_ENTRY_SHADOW_INDEX_VALID_MASK)
 					>> CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT);

 		shadow_index = (u16) ((le32_to_cpu(cq_entry->data) &
 					CQ_ENTRY_SHADOW_INDEX_MASK)
 					>> CQ_ENTRY_SHADOW_INDEX_SHIFT);

 		queue = &hdev->kernel_queues[cq->hw_queue_id];

 		if ((shadow_index_valid) && (!hdev->disabled)) {
 			job = queue->shadow_queue[hl_pi_2_offset(shadow_index)];
 			queue_work(hdev->cq_wq[cq->cq_idx], &job->finish_work);
 		}

 		atomic_inc(&queue->ci);

 		/* Clear CQ entry ready bit */
 		cq_entry->data = cpu_to_le32(le32_to_cpu(cq_entry->data) &
 						~CQ_ENTRY_READY_MASK);

 		cq->ci = hl_cq_inc_ptr(cq->ci);

 		/* Increment free slots */
 		atomic_inc(&cq->free_slots_cnt);
 	}

 	return IRQ_HANDLED;
 }

 static void handle_user_cq(struct hl_device *hdev,
 			struct hl_user_interrupt *user_cq)
 {
 	struct hl_user_pending_interrupt *pend;
 	ktime_t now = ktime_get();

 	spin_lock(&user_cq->wait_list_lock);
 	list_for_each_entry(pend, &user_cq->wait_list_head, wait_list_node) {
 		pend->fence.timestamp = now;
 		complete_all(&pend->fence.completion);
 	}
 	spin_unlock(&user_cq->wait_list_lock);
 }

 /**
  * hl_irq_handler_user_cq - irq handler for user completion queues
  *
  * @irq: irq number
  * @arg: pointer to user interrupt structure
  *
  */
 irqreturn_t hl_irq_handler_user_cq(int irq, void *arg)
 {
 	struct hl_user_interrupt *user_cq = arg;
 	struct hl_device *hdev = user_cq->hdev;

 	dev_dbg(hdev->dev,
 		"got user completion interrupt id %u",
 		user_cq->interrupt_id);

 	/* Handle user cq interrupts registered on all interrupts */
 	handle_user_cq(hdev, &hdev->common_user_interrupt);

 	/* Handle user cq interrupts registered on this specific interrupt */
 	handle_user_cq(hdev, user_cq);

 	return IRQ_HANDLED;
 }

 /**
  * hl_irq_handler_default - default irq handler
  *
  * @irq: irq number
  * @arg: pointer to user interrupt structure
  *
  */
 irqreturn_t hl_irq_handler_default(int irq, void *arg)
 {
 	struct hl_user_interrupt *user_interrupt = arg;
 	struct hl_device *hdev = user_interrupt->hdev;
 	u32 interrupt_id = user_interrupt->interrupt_id;

 	dev_err(hdev->dev,
 		"got invalid user interrupt %u",
 		interrupt_id);

 	return IRQ_HANDLED;
 }

 /**
  * hl_irq_handler_eq - irq handler for event queue
  *
  * @irq: irq number
  * @arg: pointer to event queue structure
  *
  */
 irqreturn_t hl_irq_handler_eq(int irq, void *arg)
 {
 	struct hl_eq *eq = arg;
 	struct hl_device *hdev = eq->hdev;
 	struct hl_eq_entry *eq_entry;
 	struct hl_eq_entry *eq_base;
 	struct hl_eqe_work *handle_eqe_work;
 	bool entry_ready;
 	u32 cur_eqe;
 	u16 cur_eqe_index;

 	eq_base = eq->kernel_address;

 	while (1) {
 		cur_eqe = le32_to_cpu(eq_base[eq->ci].hdr.ctl);
 		entry_ready = !!FIELD_GET(EQ_CTL_READY_MASK, cur_eqe);

 		if (!entry_ready)
 			break;

 		cur_eqe_index = FIELD_GET(EQ_CTL_INDEX_MASK, cur_eqe);
 		if ((hdev->event_queue.check_eqe_index) &&
 				(((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK)
 							!= cur_eqe_index)) {
 			dev_dbg(hdev->dev,
 				"EQE 0x%x in queue is ready but index does not match %d!=%d",
 				eq_base[eq->ci].hdr.ctl,
 				((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK),
 				cur_eqe_index);
 			break;
 		}

 		eq->prev_eqe_index++;

 		eq_entry = &eq_base[eq->ci];

 		/*
 		 * Make sure we read EQ entry contents after we've
 		 * checked the ownership bit.
 		 */
 		dma_rmb();

 		if (hdev->disabled) {
 			dev_warn(hdev->dev,
 				"Device disabled but received IRQ %d for EQ\n",
 					irq);
 			goto skip_irq;
 		}

 		handle_eqe_work = kmalloc(sizeof(*handle_eqe_work), GFP_ATOMIC);
 		if (handle_eqe_work) {
 			INIT_WORK(&handle_eqe_work->eq_work, irq_handle_eqe);
 			handle_eqe_work->hdev = hdev;

 			memcpy(&handle_eqe_work->eq_entry, eq_entry,
 					sizeof(*eq_entry));

 			queue_work(hdev->eq_wq, &handle_eqe_work->eq_work);
 		}
 skip_irq:
 		/* Clear EQ entry ready bit */
 		eq_entry->hdr.ctl =
 			cpu_to_le32(le32_to_cpu(eq_entry->hdr.ctl) &
 							~EQ_CTL_READY_MASK);

 		eq->ci = hl_eq_inc_ptr(eq->ci);

 		hdev->asic_funcs->update_eq_ci(hdev, eq->ci);
 	}

 	return IRQ_HANDLED;
 }

 /**
  * hl_cq_init - main initialization function for an cq object
  *
  * @hdev: pointer to device structure
  * @q: pointer to cq structure
  * @hw_queue_id: The H/W queue ID this completion queue belongs to
  *
  * Allocate dma-able memory for the completion queue and initialize fields
  * Returns 0 on success
  */
 int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id)
 {
 	void *p;

 	p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
 				&q->bus_address, GFP_KERNEL | __GFP_ZERO);
 	if (!p)
 		return -ENOMEM;

 	q->hdev = hdev;
 	q->kernel_address = p;
 	q->hw_queue_id = hw_queue_id;
 	q->ci = 0;
 	q->pi = 0;

 	atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);

 	return 0;
 }

 /**
  * hl_cq_fini - destroy completion queue
  *
  * @hdev: pointer to device structure
  * @q: pointer to cq structure
  *
  * Free the completion queue memory
  */
 void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q)
 {
 	hdev->asic_funcs->asic_dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
 						 q->kernel_address,
 						 q->bus_address);
 }

 void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q)
 {
 	q->ci = 0;
 	q->pi = 0;

 	atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);

 	/*
 	 * It's not enough to just reset the PI/CI because the H/W may have
 	 * written valid completion entries before it was halted and therefore
 	 * we need to clean the actual queues so we won't process old entries
 	 * when the device is operational again
 	 */

 	memset(q->kernel_address, 0, HL_CQ_SIZE_IN_BYTES);
 }

 /**
  * hl_eq_init - main initialization function for an event queue object
  *
  * @hdev: pointer to device structure
  * @q: pointer to eq structure
  *
  * Allocate dma-able memory for the event queue and initialize fields
  * Returns 0 on success
  */
 int hl_eq_init(struct hl_device *hdev, struct hl_eq *q)
 {
 	void *p;

 	p = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
 							HL_EQ_SIZE_IN_BYTES,
 							&q->bus_address);
 	if (!p)
 		return -ENOMEM;

 	q->hdev = hdev;
 	q->kernel_address = p;
 	q->ci = 0;
 	q->prev_eqe_index = 0;

 	return 0;
 }

 /**
  * hl_eq_fini - destroy event queue
  *
  * @hdev: pointer to device structure
  * @q: pointer to eq structure
  *
  * Free the event queue memory
  */
 void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q)
 {
 	flush_workqueue(hdev->eq_wq);

 	hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
 					HL_EQ_SIZE_IN_BYTES,
 					q->kernel_address);
 }

 void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q)
 {
 	q->ci = 0;
 	q->prev_eqe_index = 0;

 	/*
 	 * It's not enough to just reset the PI/CI because the H/W may have
 	 * written valid completion entries before it was halted and therefore
 	 * we need to clean the actual queues so we won't process old entries
 	 * when the device is operational again
 	 */

 	memset(q->kernel_address, 0, HL_EQ_SIZE_IN_BYTES);
 }
	// SPDX-License-Identifier: GPL-2.0

	/*
	* Copyright 2016-2019 HabanaLabs, Ltd.
	* All Rights Reserved.
	*/

	#include "habanalabs.h"

	#include <linux/slab.h>

	/**
	* struct hl_eqe_work - This structure is used to schedule work of EQ
	* entry and cpucp_reset event
	*
	* @eq_work: workqueue object to run when EQ entry is received
	* @hdev: pointer to device structure
	* @eq_entry: copy of the EQ entry
	*/
	struct hl_eqe_work {
	struct work_struct eq_work;
	struct hl_device *hdev;
	struct hl_eq_entry eq_entry;
	};

	/**
	* hl_cq_inc_ptr - increment ci or pi of cq
	*
	* @ptr: the current ci or pi value of the completion queue
	*
	* Increment ptr by 1. If it reaches the number of completion queue
	* entries, set it to 0
	*/
	inline u32 hl_cq_inc_ptr(u32 ptr)
	{
	ptr++;
	if (unlikely(ptr == HL_CQ_LENGTH))
	ptr = 0;
	return ptr;
	}

	/**
	* hl_eq_inc_ptr - increment ci of eq
	*
	* @ptr: the current ci value of the event queue
	*
	* Increment ptr by 1. If it reaches the number of event queue
	* entries, set it to 0
	*/
	static inline u32 hl_eq_inc_ptr(u32 ptr)
	{
	ptr++;
	if (unlikely(ptr == HL_EQ_LENGTH))
	ptr = 0;
	return ptr;
	}

	static void irq_handle_eqe(struct work_struct *work)
	{
	struct hl_eqe_work *eqe_work = container_of(work, struct hl_eqe_work,
	eq_work);
	struct hl_device *hdev = eqe_work->hdev;

	hdev->asic_funcs->handle_eqe(hdev, &eqe_work->eq_entry);

	kfree(eqe_work);
	}

	/**
	* hl_irq_handler_cq - irq handler for completion queue
	*
	* @irq: irq number
	* @arg: pointer to completion queue structure
	*
	*/
	irqreturn_t hl_irq_handler_cq(int irq, void *arg)
	{
	struct hl_cq *cq = arg;
	struct hl_device *hdev = cq->hdev;
	struct hl_hw_queue *queue;
	struct hl_cs_job *job;
	bool shadow_index_valid;
	u16 shadow_index;
	struct hl_cq_entry cq_entry, cq_base;

	if (hdev->disabled) {
	dev_dbg(hdev->dev,
	"Device disabled but received IRQ %d for CQ %d\n",
	irq, cq->hw_queue_id);
	return IRQ_HANDLED;
	}

	cq_base = cq->kernel_address;

	while (1) {
	bool entry_ready = ((le32_to_cpu(cq_base[cq->ci].data) &
	CQ_ENTRY_READY_MASK)
	>> CQ_ENTRY_READY_SHIFT);

	if (!entry_ready)
	break;

	cq_entry = (struct hl_cq_entry *) &cq_base[cq->ci];

	/* Make sure we read CQ entry contents after we've
	* checked the ownership bit.
	*/
	dma_rmb();

	shadow_index_valid = ((le32_to_cpu(cq_entry->data) &
	CQ_ENTRY_SHADOW_INDEX_VALID_MASK)
	>> CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT);

	shadow_index = (u16) ((le32_to_cpu(cq_entry->data) &
	CQ_ENTRY_SHADOW_INDEX_MASK)
	>> CQ_ENTRY_SHADOW_INDEX_SHIFT);

	queue = &hdev->kernel_queues[cq->hw_queue_id];

	if ((shadow_index_valid) && (!hdev->disabled)) {
	job = queue->shadow_queue[hl_pi_2_offset(shadow_index)];
	queue_work(hdev->cq_wq[cq->cq_idx], &job->finish_work);
	}

	atomic_inc(&queue->ci);

	/* Clear CQ entry ready bit */
	cq_entry->data = cpu_to_le32(le32_to_cpu(cq_entry->data) &
	~CQ_ENTRY_READY_MASK);

	cq->ci = hl_cq_inc_ptr(cq->ci);

	/* Increment free slots */
	atomic_inc(&cq->free_slots_cnt);
	}

	return IRQ_HANDLED;
	}

	static void handle_user_cq(struct hl_device *hdev,
	struct hl_user_interrupt *user_cq)
	{
	struct hl_user_pending_interrupt *pend;
	ktime_t now = ktime_get();

	spin_lock(&user_cq->wait_list_lock);
	list_for_each_entry(pend, &user_cq->wait_list_head, wait_list_node) {
	pend->fence.timestamp = now;
	complete_all(&pend->fence.completion);
	}
	spin_unlock(&user_cq->wait_list_lock);
	}

	/**
	* hl_irq_handler_user_cq - irq handler for user completion queues
	*
	* @irq: irq number
	* @arg: pointer to user interrupt structure
	*
	*/
	irqreturn_t hl_irq_handler_user_cq(int irq, void *arg)
	{
	struct hl_user_interrupt *user_cq = arg;
	struct hl_device *hdev = user_cq->hdev;

	dev_dbg(hdev->dev,
	"got user completion interrupt id %u",
	user_cq->interrupt_id);

	/* Handle user cq interrupts registered on all interrupts */
	handle_user_cq(hdev, &hdev->common_user_interrupt);

	/* Handle user cq interrupts registered on this specific interrupt */
	handle_user_cq(hdev, user_cq);

	return IRQ_HANDLED;
	}

	/**
	* hl_irq_handler_default - default irq handler
	*
	* @irq: irq number
	* @arg: pointer to user interrupt structure
	*
	*/
	irqreturn_t hl_irq_handler_default(int irq, void *arg)
	{
	struct hl_user_interrupt *user_interrupt = arg;
	struct hl_device *hdev = user_interrupt->hdev;
	u32 interrupt_id = user_interrupt->interrupt_id;

	dev_err(hdev->dev,
	"got invalid user interrupt %u",
	interrupt_id);

	return IRQ_HANDLED;
	}

	/**
	* hl_irq_handler_eq - irq handler for event queue
	*
	* @irq: irq number
	* @arg: pointer to event queue structure
	*
	*/
	irqreturn_t hl_irq_handler_eq(int irq, void *arg)
	{
	struct hl_eq *eq = arg;
	struct hl_device *hdev = eq->hdev;
	struct hl_eq_entry *eq_entry;
	struct hl_eq_entry *eq_base;
	struct hl_eqe_work *handle_eqe_work;
	bool entry_ready;
	u32 cur_eqe;
	u16 cur_eqe_index;

	eq_base = eq->kernel_address;

	while (1) {
	cur_eqe = le32_to_cpu(eq_base[eq->ci].hdr.ctl);
	entry_ready = !!FIELD_GET(EQ_CTL_READY_MASK, cur_eqe);

	if (!entry_ready)
	break;

	cur_eqe_index = FIELD_GET(EQ_CTL_INDEX_MASK, cur_eqe);
	if ((hdev->event_queue.check_eqe_index) &&
	(((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK)
	!= cur_eqe_index)) {
	dev_dbg(hdev->dev,
	"EQE 0x%x in queue is ready but index does not match %d!=%d",
	eq_base[eq->ci].hdr.ctl,
	((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK),
	cur_eqe_index);
	break;
	}

	eq->prev_eqe_index++;

	eq_entry = &eq_base[eq->ci];

	/*
	* Make sure we read EQ entry contents after we've
	* checked the ownership bit.
	*/
	dma_rmb();

	if (hdev->disabled) {
	dev_warn(hdev->dev,
	"Device disabled but received IRQ %d for EQ\n",
	irq);
	goto skip_irq;
	}

	handle_eqe_work = kmalloc(sizeof(*handle_eqe_work), GFP_ATOMIC);
	if (handle_eqe_work) {
	INIT_WORK(&handle_eqe_work->eq_work, irq_handle_eqe);
	handle_eqe_work->hdev = hdev;

	memcpy(&handle_eqe_work->eq_entry, eq_entry,
	sizeof(*eq_entry));

	queue_work(hdev->eq_wq, &handle_eqe_work->eq_work);
	}
	skip_irq:
	/* Clear EQ entry ready bit */
	eq_entry->hdr.ctl =
	cpu_to_le32(le32_to_cpu(eq_entry->hdr.ctl) &
	~EQ_CTL_READY_MASK);

	eq->ci = hl_eq_inc_ptr(eq->ci);

	hdev->asic_funcs->update_eq_ci(hdev, eq->ci);
	}

	return IRQ_HANDLED;
	}

	/**
	* hl_cq_init - main initialization function for an cq object
	*
	* @hdev: pointer to device structure
	* @q: pointer to cq structure
	* @hw_queue_id: The H/W queue ID this completion queue belongs to
	*
	* Allocate dma-able memory for the completion queue and initialize fields
	* Returns 0 on success
	*/
	int hl_cq_init(struct hl_device hdev, struct hl_cq q, u32 hw_queue_id)
	{
	void *p;

	p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
	&q->bus_address, GFP_KERNEL \| __GFP_ZERO);
	if (!p)
	return -ENOMEM;

	q->hdev = hdev;
	q->kernel_address = p;
	q->hw_queue_id = hw_queue_id;
	q->ci = 0;
	q->pi = 0;

	atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);

	return 0;
	}

	/**
	* hl_cq_fini - destroy completion queue
	*
	* @hdev: pointer to device structure
	* @q: pointer to cq structure
	*
	* Free the completion queue memory
	*/
	void hl_cq_fini(struct hl_device hdev, struct hl_cq q)
	{
	hdev->asic_funcs->asic_dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES,
	q->kernel_address,
	q->bus_address);
	}

	void hl_cq_reset(struct hl_device hdev, struct hl_cq q)
	{
	q->ci = 0;
	q->pi = 0;

	atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);

	/*
	* It's not enough to just reset the PI/CI because the H/W may have
	* written valid completion entries before it was halted and therefore
	* we need to clean the actual queues so we won't process old entries
	* when the device is operational again
	*/

	memset(q->kernel_address, 0, HL_CQ_SIZE_IN_BYTES);
	}

	/**
	* hl_eq_init - main initialization function for an event queue object
	*
	* @hdev: pointer to device structure
	* @q: pointer to eq structure
	*
	* Allocate dma-able memory for the event queue and initialize fields
	* Returns 0 on success
	*/
	int hl_eq_init(struct hl_device hdev, struct hl_eq q)
	{
	void *p;

	p = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev,
	HL_EQ_SIZE_IN_BYTES,
	&q->bus_address);
	if (!p)
	return -ENOMEM;

	q->hdev = hdev;
	q->kernel_address = p;
	q->ci = 0;
	q->prev_eqe_index = 0;

	return 0;
	}

	/**
	* hl_eq_fini - destroy event queue
	*
	* @hdev: pointer to device structure
	* @q: pointer to eq structure
	*
	* Free the event queue memory
	*/
	void hl_eq_fini(struct hl_device hdev, struct hl_eq q)
	{
	flush_workqueue(hdev->eq_wq);

	hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
	HL_EQ_SIZE_IN_BYTES,
	q->kernel_address);
	}

	void hl_eq_reset(struct hl_device hdev, struct hl_eq q)
	{
	q->ci = 0;
	q->prev_eqe_index = 0;

	/*
	* It's not enough to just reset the PI/CI because the H/W may have
	* written valid completion entries before it was halted and therefore
	* we need to clean the actual queues so we won't process old entries
	* when the device is operational again
	*/

	memset(q->kernel_address, 0, HL_EQ_SIZE_IN_BYTES);
	}