core/drivers/crypto/caam/caam_jr.c

// SPDX-License-Identifier: BSD-2-Clause
/*
 * Copyright 2018-2019 NXP
 *
 * Brief   CAAM Job Rings manager.
 *         Implementation of functions to enqueue/dequeue CAAM Job Descriptor
 */
#include <caam_common.h>
#include <caam_desc_helper.h>
#include <caam_hal_jr.h>
#include <caam_io.h>
#include <caam_jr.h>
#include <caam_rng.h>
#include <caam_utils_delay.h>
#include <caam_utils_mem.h>
#include <kernel/interrupt.h>
#include <kernel/panic.h>
#include <kernel/pm.h>
#include <kernel/spinlock.h>
#include <mm/core_memprot.h>
#include <tee/cache.h>

/*
 * The CAAM physical address is decorrelated from the CPU addressing mode.
 * CAAM can manage 32 or 64 bits address depending on its version and the
 * device.
 */
/*
 * Definition of input and output ring object
 */
#ifdef CFG_CAAM_64BIT
struct inring_entry {
	uint64_t desc;   /* Physical address of the descriptor */
};

struct __packed outring_entry {
	uint64_t desc;   /* Physical address of the descriptor */
	uint32_t status; /* Status of the executed job */
};
#else
struct inring_entry {
	uint32_t desc;   /* Physical address of the descriptor */
};

struct __packed outring_entry {
	uint32_t desc;   /* Physical address of the descriptor */
	uint32_t status; /* Status of the executed job */
};
#endif /* CFG_CAAM_64BIT */

/*
 * Job Free define
 */
#define JR_JOB_FREE	0

/*
 * Caller information context object
 */
struct caller_info {
	struct caam_jobctx *jobctx; /* Caller job context object */
	uint32_t job_id;            /* Current Job ID */
	paddr_t pdesc;              /* Physical address of the descriptor */
};

/*
 * Job Ring module private data
 */
struct jr_privdata {
	vaddr_t baseaddr;        /* Job Ring base address */

	vaddr_t ctrladdr;        /* CAAM virtual base address */
	paddr_t jroffset;        /* Job Ring address offset */
	uint64_t paddr_inrings;  /* CAAM physical addr of input queue */
	uint64_t paddr_outrings; /* CAAM physical addr of output queue */

	uint8_t nb_jobs;         /* Number of Job ring entries managed */

	/* Input Job Ring Variables */
	struct inring_entry *inrings; /* Input JR HW queue */
	unsigned int inlock;          /* Input JR spin lock */
	uint16_t inwrite_index;       /* SW Index - next JR entry free */

	/* Output Job Ring Variables */
	struct outring_entry *outrings; /* Output JR HW queue */
	unsigned int outlock;           /* Output JR spin lock */
	uint16_t outread_index;         /* SW Index - next JR output done */

	/* Caller Information Variables */
	struct caller_info *callers;    /* Job Ring Caller information */
	unsigned int callers_lock;      /* Job Ring Caller spin lock */

	struct itr_handler it_handler;  /* Interrupt handler */
};

/*
 * Job Ring module private data reference
 */
static struct jr_privdata *jr_privdata;

/*
 * Free module resources
 *
 * @jr_priv   Reference to the module private data
 */
static void do_jr_free(struct jr_privdata *jr_priv)
{
	if (jr_priv) {
		caam_free(jr_priv->inrings);
		caam_free(jr_priv->outrings);
		caam_free(jr_priv->callers);
		caam_free(jr_priv);
	}
}

/*
 * Allocate module resources
 *
 * @privdata  [out] Allocated Job Ring private data
 * @nb_jobs   Number of jobs to manage in the queue
 */
static enum caam_status do_jr_alloc(struct jr_privdata **privdata,
				    uint8_t nb_jobs)
{
	enum caam_status retstatus = CAAM_OUT_MEMORY;
	struct jr_privdata *jr_priv = NULL;

	/* Allocate the Job Ring private data */
	jr_priv = caam_calloc(sizeof(*jr_priv));

	if (!jr_priv) {
		JR_TRACE("Private Data allocation error");
		goto end_alloc;
	}

	/* Setup the number of jobs */
	jr_priv->nb_jobs = nb_jobs;

	/* Allocate the input and output job ring queues */
	jr_priv->inrings =
		caam_calloc_align(nb_jobs * sizeof(struct inring_entry));
	jr_priv->outrings =
		caam_calloc_align(nb_jobs * sizeof(struct outring_entry));

	/* Allocate the callers information */
	jr_priv->callers = caam_calloc(nb_jobs * sizeof(struct caller_info));

	if (!jr_priv->inrings || !jr_priv->outrings || !jr_priv->callers) {
		JR_TRACE("JR resources allocation error");
		goto end_alloc;
	}

	/* Initialize the spin locks */
	jr_priv->inlock = SPINLOCK_UNLOCK;
	jr_priv->outlock = SPINLOCK_UNLOCK;
	jr_priv->callers_lock = SPINLOCK_UNLOCK;

	/* Initialize the queue counter */
	jr_priv->inwrite_index = 0;
	jr_priv->outread_index = 0;

	/*
	 * Ensure that allocated queue initialization is pushed to the physical
	 * memory
	 */
	cache_operation(TEE_CACHEFLUSH, jr_priv->inrings,
			nb_jobs * sizeof(struct inring_entry));
	cache_operation(TEE_CACHEFLUSH, jr_priv->outrings,
			nb_jobs * sizeof(struct outring_entry));

	retstatus = CAAM_NO_ERROR;
end_alloc:
	if (retstatus != CAAM_NO_ERROR)
		do_jr_free(jr_priv);
	else
		*privdata = jr_priv;

	return retstatus;
}

/*
 * Job Ring Interrupt handler
 *
 * @handler  Interrupt Handler structure
 */
static enum itr_return caam_jr_irqhandler(struct itr_handler *handler)
{
	JR_TRACE("Disable the interrupt");
	itr_disable(handler->it);

	/* Send a signal to exit WFE loop */
	sev();

	return ITRR_HANDLED;
}

/*
 * Returns all jobs completed depending on the input @wait_job_ids mask.
 *
 * Dequeues all Jobs completed. Call the job context callback
 * function. Function returns the bit mask of the expected completed job
 * (@wait_job_ids parameter)
 *
 * @wait_job_ids  Expected Jobs to be complete
 */
static uint32_t do_jr_dequeue(uint32_t wait_job_ids)
{
	uint32_t ret_job_id = 0;
	struct caller_info *caller = NULL;
	struct outring_entry *jr_out = NULL;
	struct caam_jobctx *jobctx = NULL;
	uint32_t exceptions = 0;
	bool found = false;
	uint16_t idx_jr = 0;
	uint32_t nb_jobs_done = 0;
	size_t nb_jobs_inv = 0;

	exceptions = cpu_spin_lock_xsave(&jr_privdata->outlock);

	nb_jobs_done = caam_hal_jr_get_nbjob_done(jr_privdata->baseaddr);

	if (nb_jobs_done == 0) {
		cpu_spin_unlock_xrestore(&jr_privdata->outlock, exceptions);
		return ret_job_id;
	}

	/* Ensure that output ring descriptor entries are not in cache */
	if ((jr_privdata->outread_index + nb_jobs_done) >
	    jr_privdata->nb_jobs) {
		/*
		 * Invalidate the whole circular job buffer because some
		 * completed job rings are at the beginning of the buffer
		 */
		jr_out = jr_privdata->outrings;
		nb_jobs_inv = jr_privdata->nb_jobs;
	} else {
		/* Invalidate only the completed job */
		jr_out = &jr_privdata->outrings[jr_privdata->outread_index];
		nb_jobs_inv = nb_jobs_done;
	}

	cache_operation(TEE_CACHEINVALIDATE, jr_out,
			sizeof(struct outring_entry) * nb_jobs_inv);

	for (; nb_jobs_done; nb_jobs_done--) {
		jr_out = &jr_privdata->outrings[jr_privdata->outread_index];

		/*
		 * Lock the caller information array because enqueue is
		 * also touching it
		 */
		cpu_spin_lock(&jr_privdata->callers_lock);
		for (idx_jr = 0, found = false; idx_jr < jr_privdata->nb_jobs;
		     idx_jr++) {
			/*
			 * Search for the caller information corresponding to
			 * the completed JR.
			 * Don't use the outread_index or inwrite_index because
			 * completion can be out of order compared to input
			 * buffer
			 */
			caller = &jr_privdata->callers[idx_jr];
			if (caam_desc_pop(&jr_out->desc) == caller->pdesc) {
				jobctx = caller->jobctx;
				jobctx->status =
					caam_read_jobstatus(&jr_out->status);

				/* Update return Job IDs mask */
				if (caller->job_id & wait_job_ids)
					ret_job_id |= caller->job_id;

				JR_TRACE("JR id=%" PRId32
					 ", context @0x%08" PRIxVA,
					 caller->job_id, (vaddr_t)jobctx);
				/* Clear the Entry Descriptor DMA */
				caller->pdesc = 0;
				caller->job_id = JR_JOB_FREE;
				found = true;
				JR_TRACE("Free space #%" PRId16
					 " in the callers array",
					 idx_jr);
				break;
			}
		}
		cpu_spin_unlock(&jr_privdata->callers_lock);

		/*
		 * Remove the JR from the output list even if no
		 * JR caller found
		 */
		caam_hal_jr_del_job(jr_privdata->baseaddr);

		/*
		 * Increment index to next JR output entry taking care that
		 * it is a circular buffer of nb_jobs size.
		 */
		jr_privdata->outread_index++;
		jr_privdata->outread_index %= jr_privdata->nb_jobs;

		if (found && jobctx->callback) {
			/* Finally, execute user's callback */
			jobctx->callback(jobctx);
		}
	}

	cpu_spin_unlock_xrestore(&jr_privdata->outlock, exceptions);

	return ret_job_id;
}

/*
 * Enqueues a new job in the Job Ring input queue. Keep the caller's
 * job context in private array.
 *
 * @jobctx   Caller's job context
 * @job_id   [out] Job ID enqueued
 */
static enum caam_status do_jr_enqueue(struct caam_jobctx *jobctx,
				      uint32_t *job_id)
{
	enum caam_status retstatus = CAAM_BUSY;
	struct inring_entry *cur_inrings = NULL;
	struct caller_info *caller = NULL;
	uint32_t exceptions = 0;
	uint32_t job_mask = 0;
	uint8_t idx_jr = 0;
	bool found = false;

	exceptions = cpu_spin_lock_xsave(&jr_privdata->inlock);

	/*
	 * Stay locked until a job is available
	 * Check if there is an available JR index in the HW
	 */
	while (caam_hal_jr_read_nbslot_available(jr_privdata->baseaddr) == 0) {
		/*
		 * WFE will return thanks to a SEV generated by the
		 * interrupt handler or by a spin_unlock
		 */
		wfe();
	};

	/*
	 * There is a space free in the input ring but it doesn't mean
	 * that the job pushed is completed.
	 * Completion is out of order. Look for a free space in the
	 * caller data to push them and get a job ID for the completion
	 *
	 * Lock the caller information array because dequeue is
	 * also touching it
	 */
	cpu_spin_lock(&jr_privdata->callers_lock);
	for (idx_jr = 0; idx_jr < jr_privdata->nb_jobs; idx_jr++) {
		if (jr_privdata->callers[idx_jr].job_id == JR_JOB_FREE) {
			JR_TRACE("Found a space #%" PRId8
				 " free in the callers array",
				 idx_jr);
			job_mask = 1 << idx_jr;

			/* Store the caller information for the JR completion */
			caller = &jr_privdata->callers[idx_jr];
			caller->job_id = job_mask;
			caller->jobctx = jobctx;
			caller->pdesc = virt_to_phys((void *)jobctx->desc);

			found = true;
			break;
		}
	}
	cpu_spin_unlock(&jr_privdata->callers_lock);

	if (!found) {
		JR_TRACE("Error didn't find a free space in the callers array");
		goto end_enqueue;
	}

	JR_TRACE("Push id=%" PRId16 ", job (0x%08" PRIx32
		 ") context @0x%08" PRIxVA,
		 jr_privdata->inwrite_index, job_mask, (vaddr_t)jobctx);

	cur_inrings = &jr_privdata->inrings[jr_privdata->inwrite_index];

	/* Push the descriptor into the JR HW list */
	caam_desc_push(&cur_inrings->desc, caller->pdesc);

	/* Ensure that physical memory is up to date */
	cache_operation(TEE_CACHECLEAN, cur_inrings,
			sizeof(struct inring_entry));

	/*
	 * Increment index to next JR input entry taking care that
	 * it is a circular buffer of nb_jobs size.
	 */
	jr_privdata->inwrite_index++;
	jr_privdata->inwrite_index %= jr_privdata->nb_jobs;

	/* Ensure that input descriptor is pushed in physical memory */
	cache_operation(TEE_CACHECLEAN, jobctx->desc,
			DESC_SZBYTES(caam_desc_get_len(jobctx->desc)));

	/* Inform HW that a new JR is available */
	caam_hal_jr_add_newjob(jr_privdata->baseaddr);

	*job_id = job_mask;
	retstatus = CAAM_NO_ERROR;

end_enqueue:
	cpu_spin_unlock_xrestore(&jr_privdata->inlock, exceptions);

	return retstatus;
}

/*
 * Synchronous job completion callback
 *
 * @jobctx   Job context
 */
static void job_done(struct caam_jobctx *jobctx)
{
	jobctx->completion = true;
}

void caam_jr_cancel(uint32_t job_id)
{
	unsigned int idx = 0;

	JR_TRACE("Job cancel 0x%" PRIx32, job_id);
	for (idx = 0; idx < jr_privdata->nb_jobs; idx++) {
		/*
		 * Search for the caller information corresponding to
		 * the job_id mask.
		 */
		if (jr_privdata->callers[idx].job_id == job_id) {
			/* Clear the Entry Descriptor */
			jr_privdata->callers[idx].pdesc = 0;
			jr_privdata->callers[idx].job_id = JR_JOB_FREE;
			return;
		}
	}
}

enum caam_status caam_jr_dequeue(uint32_t job_ids, unsigned int timeout_ms)
{
	uint32_t job_complete = 0;
	uint32_t nb_loop = 0;
	bool infinite = false;

	if (timeout_ms == UINT_MAX)
		infinite = true;
	else
		nb_loop = timeout_ms * 100;

	do {
		/* Call the do_jr_dequeue function to dequeue the jobs */
		job_complete = do_jr_dequeue(job_ids);

		if (job_complete & job_ids)
			return CAAM_NO_ERROR;

		/* Check if JR interrupt otherwise wait a bit */
		if (!caam_hal_jr_check_ack_itr(jr_privdata->baseaddr))
			caam_udelay(10);
	} while (infinite || (nb_loop--));

	return CAAM_TIMEOUT;
}

enum caam_status caam_jr_enqueue(struct caam_jobctx *jobctx, uint32_t *job_id)
{
	enum caam_status retstatus = CAAM_FAILURE;
	__maybe_unused int timeout  = 10; /* Nb loops to pool job completion */

	if (!jobctx)
		return CAAM_BAD_PARAM;

	JR_DUMPDESC(jobctx->desc);

	if (!jobctx->callback && job_id) {
		JR_TRACE("Job Callback not defined whereas asynchronous");
		return CAAM_BAD_PARAM;
	}

	if (jobctx->callback && !job_id) {
		JR_TRACE("Job Id not defined whereas asynchronous");
		return CAAM_BAD_PARAM;
	}

	jobctx->completion = false;
	jobctx->status = 0;

	/*
	 * If parameter job_id is NULL, the job is synchronous, hence use
	 * the local job_done callback function
	 */
	if (!jobctx->callback && !job_id) {
		jobctx->callback = job_done;
		jobctx->context = jobctx;
	}

	retstatus = do_jr_enqueue(jobctx, &jobctx->id);

	if (retstatus != CAAM_NO_ERROR) {
		JR_TRACE("enqueue job error 0x%08x", retstatus);
		return retstatus;
	}

	/*
	 * If parameter job_id is defined, the job is asynchronous, so
	 * returns with setting the job_id value
	 */
	if (job_id) {
		*job_id = jobctx->id;
		return CAAM_PENDING;
	}

#ifdef TIMEOUT_COMPLETION
	/*
	 * Job is synchronous wait until job completion or timeout
	 */
	while (!jobctx->completion && timeout--)
		caam_jr_dequeue(jobctx->id, 100);

	if (timeout <= 0) {
		/* Job timeout, cancel it and return in error */
		caam_jr_cancel(jobctx->id);
		retstatus = CAAM_TIMEOUT;
	} else {
		if (JRSTA_SRC_GET(jobctx->status) != JRSTA_SRC(NONE))
			retstatus = CAAM_JOB_STATUS;
		else
			retstatus = CAAM_NO_ERROR;
	}
#else
	/*
	 * Job is synchronous wait until job complete
	 * Don't use a timeout because there is no HW timer and
	 * so the timeout is not precise
	 */
	while (!jobctx->completion)
		caam_jr_dequeue(jobctx->id, 100);

	if (JRSTA_SRC_GET(jobctx->status) != JRSTA_SRC(NONE))
		retstatus = CAAM_JOB_STATUS;
	else
		retstatus = CAAM_NO_ERROR;
#endif

	/* Erase local callback function */
	jobctx->callback = NULL;

	return retstatus;
}

enum caam_status caam_jr_init(struct caam_jrcfg *jrcfg)
{
	enum caam_status retstatus = CAAM_FAILURE;

	JR_TRACE("Initialization");

	/* Allocate the Job Ring resources */
	retstatus = do_jr_alloc(&jr_privdata, jrcfg->nb_jobs);
	if (retstatus != CAAM_NO_ERROR)
		goto end_init;

	jr_privdata->ctrladdr = jrcfg->base;
	jr_privdata->jroffset = jrcfg->offset;

	retstatus =
		caam_hal_jr_setowner(jrcfg->base, jrcfg->offset, JROWN_ARM_S);
	JR_TRACE("JR setowner returned 0x%x", retstatus);

	if (retstatus != CAAM_NO_ERROR)
		goto end_init;

	jr_privdata->baseaddr = jrcfg->base + jrcfg->offset;
	retstatus = caam_hal_jr_reset(jr_privdata->baseaddr);
	if (retstatus != CAAM_NO_ERROR)
		goto end_init;

	/*
	 * Get the physical address of the Input/Output queue
	 * The HW configuration is 64 bits registers regardless
	 * the CAAM or CPU addressing mode.
	 */
	jr_privdata->paddr_inrings = virt_to_phys(jr_privdata->inrings);
	jr_privdata->paddr_outrings = virt_to_phys(jr_privdata->outrings);
	if (!jr_privdata->paddr_inrings || !jr_privdata->paddr_outrings) {
		JR_TRACE("JR bad queue pointers");
		retstatus = CAAM_FAILURE;
		goto end_init;
	}

	caam_hal_jr_config(jr_privdata->baseaddr, jr_privdata->nb_jobs,
			   jr_privdata->paddr_inrings,
			   jr_privdata->paddr_outrings);

	/*
	 * Prepare the interrupt handler to secure the interrupt even
	 * if the interrupt is not used
	 */
	jr_privdata->it_handler.it = jrcfg->it_num;
	jr_privdata->it_handler.flags = ITRF_TRIGGER_LEVEL;
	jr_privdata->it_handler.handler = caam_jr_irqhandler;
	jr_privdata->it_handler.data = jr_privdata;

#ifdef CFG_NXP_CAAM_RUNTIME_JR
	itr_add(&jr_privdata->it_handler);
#endif
	caam_hal_jr_enable_itr(jr_privdata->baseaddr);

	retstatus = CAAM_NO_ERROR;

end_init:
	if (retstatus != CAAM_NO_ERROR)
		do_jr_free(jr_privdata);

	return retstatus;
}

enum caam_status caam_jr_halt(void)
{
	return caam_hal_jr_halt(jr_privdata->baseaddr);
}

enum caam_status caam_jr_flush(void)
{
	return caam_hal_jr_flush(jr_privdata->baseaddr);
}

void caam_jr_resume(uint32_t pm_hint)
{
	if (pm_hint == PM_HINT_CONTEXT_STATE) {
#if !(defined(CFG_MX6DL) || defined(CFG_MX6D) || defined(CFG_MX6Q) ||          \
	defined(CFG_MX6QP))

#ifndef CFG_NXP_CAAM_RUNTIME_JR
		/*
		 * In case the CAAM is not used the JR used to
		 * instantiate the RNG has been released to Non-Secure
		 * hence, need reconfigure the Secure JR and release
		 * it after RNG instantiation
		 */
		caam_hal_jr_setowner(jr_privdata->ctrladdr,
				     jr_privdata->jroffset, JROWN_ARM_S);

		caam_hal_jr_config(jr_privdata->baseaddr, jr_privdata->nb_jobs,
				   jr_privdata->paddr_inrings,
				   jr_privdata->paddr_outrings);
#endif /* CFG_NXP_CAAM_RUNTIME_JR */

		/* Read the current job ring index */
		jr_privdata->inwrite_index =
			caam_hal_jr_input_index(jr_privdata->baseaddr);
		/* Read the current output ring index */
		jr_privdata->outread_index =
			caam_hal_jr_output_index(jr_privdata->baseaddr);

		if (caam_rng_instantiation() != CAAM_NO_ERROR)
			panic();

#ifndef CFG_NXP_CAAM_RUNTIME_JR
		caam_hal_jr_setowner(jr_privdata->ctrladdr,
				     jr_privdata->jroffset, JROWN_ARM_NS);
#endif /* CFG_NXP_CAAM_RUNTIME_JR */
#endif /* !(CFG_MX6DL || CFG_MX6D || CFG_MX6Q || CFG_MX6QP) */
	} else {
		caam_hal_jr_resume(jr_privdata->baseaddr);
	}
}

enum caam_status caam_jr_complete(void)
{
	enum caam_status ret = CAAM_BUSY;

	ret = caam_hal_jr_flush(jr_privdata->baseaddr);
	if (ret == CAAM_NO_ERROR)
		caam_hal_jr_resume(jr_privdata->baseaddr);

	return ret;
}