drivers/misc/mediatek/gpu/gpu_rgx/m1.11_5516664/pvr_sync_file.c

/* -*- mode: c; indent-tabs-mode: t; c-basic-offset: 8; tab-width: 8 -*- */
/* vi: set ts=8 sw=8 sts=8: */
/*************************************************************************/ /*!
@File           pvr_sync_file.c
@Title          Kernel driver for Android's sync mechanism
@Codingstyle    LinuxKernel
@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
@License        Dual MIT/GPLv2

The contents of this file are subject to the MIT license as set out below.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

Alternatively, the contents of this file may be used under the terms of
the GNU General Public License Version 2 ("GPL") in which case the provisions
of GPL are applicable instead of those above.

If you wish to allow use of your version of this file only under the terms of
GPL, and not to allow others to use your version of this file under the terms
of the MIT license, indicate your decision by deleting the provisions above
and replace them with the notice and other provisions required by GPL as set
out in the file called "GPL-COPYING" included in this distribution. If you do
not delete the provisions above, a recipient may use your version of this file
under the terms of either the MIT license or GPL.

This License is also included in this distribution in the file called
"MIT-COPYING".

EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/ /**************************************************************************/

#include "services_kernel_client.h"
#include "pvr_drv.h"
#include "pvr_sync.h"
#include "pvr_fence.h"
#include "pvr_counting_timeline.h"

#include "linux_sw_sync.h"

#include <linux/version.h>
#include <linux/slab.h>
#include <linux/fs.h>
#include <linux/sync_file.h>
#include <linux/file.h>
#include <linux/miscdevice.h>
#include <linux/uaccess.h>

/* This header must always be included last */
#include "kernel_compatibility.h"

#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0)) && !defined(CHROMIUMOS_KERNEL)
#define sync_file_user_name(s)	((s)->name)
#else
#define sync_file_user_name(s)	((s)->user_name)
#endif

#define PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, fmt, ...) \
	do {                                                             \
		if (pfnDumpDebugPrintf)                                  \
			pfnDumpDebugPrintf(pvDumpDebugFile, fmt,         \
					   ## __VA_ARGS__);              \
		else                                                     \
			pr_err(fmt "\n", ## __VA_ARGS__);                \
	} while (0)

#define	FILE_NAME "pvr_sync_file"

struct sw_sync_create_fence_data {
	__u32 value;
	char name[32];
	__s32 fence;
};
#define SW_SYNC_IOC_MAGIC 'W'
#define SW_SYNC_IOC_CREATE_FENCE \
	(_IOWR(SW_SYNC_IOC_MAGIC, 0, struct sw_sync_create_fence_data))
#define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32)

/* Global data for the sync driver */
static struct {
	void *dev_cookie;
	void *dbg_request_handle;
	struct workqueue_struct *fence_status_wq;
	struct pvr_fence_context *foreign_fence_context;
#if defined(NO_HARDWARE)
	spinlock_t pvr_timeline_active_list_lock;
	struct list_head pvr_timeline_active_list;
#endif
	PFN_SYNC_CHECKPOINT_STRUCT sync_checkpoint_ops;
} pvr_sync_data;

static const struct file_operations pvr_sync_fops;

/* This is the actual timeline metadata. We might keep this around after the
 * base sync driver has destroyed the pvr_sync_timeline_wrapper object.
 */
struct pvr_sync_timeline {
	char name[32];
	struct file *file;
	bool is_sw;
	/* Fence context used for hw fences */
	struct pvr_fence_context *hw_fence_context;
	/* Timeline and context for sw fences */
	struct pvr_counting_fence_timeline *sw_fence_timeline;
#if defined(NO_HARDWARE)
	/* List of all timelines (used to advance all timelines in nohw builds) */
	struct list_head list;
#endif
};

static
void pvr_sync_free_checkpoint_list_mem(void *mem_ptr)
{
	kfree(mem_ptr);
}

#if defined(NO_HARDWARE)
/* function used to signal pvr fence in nohw builds */
static
void pvr_sync_nohw_signal_fence(void *fence_data_to_signal)
{
	struct pvr_sync_timeline *this_timeline;
	unsigned long flags;

	spin_lock_irqsave(&pvr_sync_data.pvr_timeline_active_list_lock, flags);
	list_for_each_entry(this_timeline, &pvr_sync_data.pvr_timeline_active_list, list) {
		pvr_fence_context_signal_fences_nohw(this_timeline->hw_fence_context);
	}
	spin_unlock_irqrestore(&pvr_sync_data.pvr_timeline_active_list_lock, flags);
}
#endif

static bool is_pvr_timeline(struct file *file)
{
	return file->f_op == &pvr_sync_fops;
}

static struct pvr_sync_timeline *pvr_sync_timeline_fget(int fd)
{
	struct file *file = fget(fd);

	if (!file)
		return NULL;

	if (!is_pvr_timeline(file)) {
		fput(file);
		return NULL;
	}

	return file->private_data;
}

static void pvr_sync_timeline_fput(struct pvr_sync_timeline *timeline)
{
	fput(timeline->file);
}

/* ioctl and fops handling */

static int pvr_sync_open(struct inode *inode, struct file *file)
{
	struct pvr_sync_timeline *timeline;
	char task_comm[TASK_COMM_LEN];
	int err = -ENOMEM;

	get_task_comm(task_comm, current);

	timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
	if (!timeline)
		goto err_out;

	strlcpy(timeline->name, task_comm, sizeof(timeline->name));
	timeline->file = file;
	timeline->is_sw = false;

	file->private_data = timeline;
	err = 0;
err_out:
	return err;
}

static int pvr_sync_close(struct inode *inode, struct file *file)
{
	struct pvr_sync_timeline *timeline = file->private_data;

	if (timeline->sw_fence_timeline) {
		/* This makes sure any outstanding SW syncs are marked as
		 * complete at timeline close time. Otherwise it'll leak the
		 * timeline (as outstanding fences hold a ref) and possibly
		 * wedge the system if something is waiting on one of those
		 * fences
		 */
		pvr_counting_fence_timeline_force_complete(
			timeline->sw_fence_timeline);
		pvr_counting_fence_timeline_put(timeline->sw_fence_timeline);
	}

	if (timeline->hw_fence_context) {
#if defined(NO_HARDWARE)
		list_del(&timeline->list);
#endif
		pvr_fence_context_destroy(timeline->hw_fence_context);
	}

	kfree(timeline);

	return 0;
}

/*
 * This is the function that kick code will call in order to 'finalise' a
 * created output fence just prior to returning from the kick function.
 * The OS native sync code needs to implement a function meeting this
 * specification - the implementation may be a nop if the OS does not need
 * to perform any actions at this point.
 *
 * Input: fence_fd            The PVRSRV_FENCE to be 'finalised'. This value
 *                            will have been returned by an earlier call to
 *                            pvr_sync_create_fence().
 * Input: finalise_data       The finalise data returned by an earlier call
 *                            to pvr_sync_create_fence().
 */
static enum PVRSRV_ERROR
pvr_sync_finalise_fence(PVRSRV_FENCE fence_fd, void *finalise_data)
{
	struct sync_file *sync_file = finalise_data;
	struct pvr_fence *pvr_fence;

	if (!sync_file || (fence_fd < 0)) {
		pr_err(FILE_NAME ": %s: Invalid input fence\n", __func__);
		return PVRSRV_ERROR_INVALID_PARAMS;
	}

	pvr_fence = to_pvr_fence(sync_file->fence);

	/* pvr fences can be signalled any time after creation */
	dma_fence_enable_sw_signaling(&pvr_fence->base);

	fd_install(fence_fd, sync_file->file);

	return PVRSRV_OK;
}

/*
 * This is the function that kick code will call in order to obtain a new
 * PVRSRV_FENCE from the OS native sync code and the PSYNC_CHECKPOINT used
 * in that fence. The OS native sync code needs to implement a function
 * meeting this specification.
 *
 * Input: fence_name               A string to annotate the fence with (for
 *                                 debug).
 * Input: timeline                 The timeline on which the new fence is to be
 *                                 created.
 * Output: new_fence               The new PVRSRV_FENCE to be returned by the
 *                                 kick call.
 * Output: fence_uid               Unique ID of the update fence.
 * Output: fence_finalise_data     Pointer to data needed to finalise the fence.
 * Output: new_checkpoint_handle   The PSYNC_CHECKPOINT used by the new fence.
 */
static enum PVRSRV_ERROR
pvr_sync_create_fence(const char *fence_name,
		      PVRSRV_TIMELINE new_fence_timeline,
		      PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
		      PVRSRV_FENCE *new_fence, u64 *fence_uid,
		      void **fence_finalise_data,
		      PSYNC_CHECKPOINT *new_checkpoint_handle,
		      void **timeline_update_sync,
		      __u32 *timeline_update_value)
{
	PVRSRV_ERROR err = PVRSRV_OK;
	PVRSRV_FENCE new_fence_fd = -1;
	struct pvr_sync_timeline *timeline;
	struct pvr_fence *pvr_fence;
	PSYNC_CHECKPOINT checkpoint;
	struct sync_file *sync_file;

	if (new_fence_timeline < 0 || !new_fence || !new_checkpoint_handle
		|| !fence_finalise_data) {
		pr_err(FILE_NAME ": %s: Invalid input params\n", __func__);
		err =  PVRSRV_ERROR_INVALID_PARAMS;
		goto err_out;
	}

	/* We reserve the new fence FD before taking any operations
	 * as we do not want to fail (e.g. run out of FDs)
	 */
	new_fence_fd = get_unused_fd_flags(O_CLOEXEC);
	if (new_fence_fd < 0) {
		pr_err(FILE_NAME ": %s: Failed to get fd\n", __func__);
		err = PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
		goto err_out;
	}

	timeline = pvr_sync_timeline_fget(new_fence_timeline);
	if (!timeline) {
		pr_err(FILE_NAME ": %s: Failed to open supplied timeline fd (%d)\n",
			__func__, new_fence_timeline);
		err = PVRSRV_ERROR_INVALID_PARAMS;
		goto err_put_fd;
	}

	if (timeline->is_sw) {
		/* This should never happen! */
		pr_err(FILE_NAME ": %s: Request to create a pvr fence on sw timeline (%d)\n",
			__func__, new_fence_timeline);
		err = PVRSRV_ERROR_INVALID_PARAMS;
		goto err_put_timeline;
	}

	if (!timeline->hw_fence_context) {
#if defined(NO_HARDWARE)
		unsigned long flags;
#endif
		/* First time we use this timeline, so create a context. */
		timeline->hw_fence_context =
			pvr_fence_context_create(pvr_sync_data.dev_cookie,
						 pvr_sync_data.fence_status_wq,
						 timeline->name);
		if (!timeline->hw_fence_context) {
			pr_err(FILE_NAME ": %s: Failed to create fence context (%d)\n",
			       __func__, new_fence_timeline);
			err = PVRSRV_ERROR_OUT_OF_MEMORY;
			goto err_put_timeline;
		}
#if defined(NO_HARDWARE)
		/* Add timeline to active list */
		INIT_LIST_HEAD(&timeline->list);
		spin_lock_irqsave(&pvr_sync_data.pvr_timeline_active_list_lock, flags);
		list_add_tail(&timeline->list, &pvr_sync_data.pvr_timeline_active_list);
		spin_unlock_irqrestore(&pvr_sync_data.pvr_timeline_active_list_lock, flags);
#endif
	}

	pvr_fence = pvr_fence_create(timeline->hw_fence_context,
								 psSyncCheckpointContext,
								 new_fence_timeline,
								 fence_name);
	if (!pvr_fence) {
		pr_err(FILE_NAME ": %s: Failed to create new pvr_fence\n",
			__func__);
		err = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto err_put_timeline;
	}

	checkpoint = pvr_fence_get_checkpoint(pvr_fence);
	if (!checkpoint) {
		pr_err(FILE_NAME ": %s: Failed to get fence checkpoint\n",
			__func__);
		err = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto err_destroy_fence;
	}

	sync_file = sync_file_create(&pvr_fence->base);
	if (!sync_file) {
		pr_err(FILE_NAME ": %s: Failed to create sync_file\n",
			__func__);
		err = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto err_destroy_fence;
	}
	strlcpy(sync_file_user_name(sync_file),
		pvr_fence->name,
		sizeof(sync_file_user_name(sync_file)));
	dma_fence_put(&pvr_fence->base);

	*new_fence = new_fence_fd;
	*fence_finalise_data = sync_file;
	*new_checkpoint_handle = checkpoint;
	*fence_uid = OSGetCurrentClientProcessIDKM();
	*fence_uid = (*fence_uid << 32) | (new_fence_fd & U32_MAX);
	/* not used but don't want to return dangling pointers */
	*timeline_update_sync = NULL;
	*timeline_update_value = 0;

	pvr_sync_timeline_fput(timeline);
err_out:
	return err;

err_destroy_fence:
	pvr_fence_destroy(pvr_fence);
err_put_timeline:
	pvr_sync_timeline_fput(timeline);
err_put_fd:
	put_unused_fd(new_fence_fd);
	*fence_uid = PVRSRV_NO_FENCE;
	goto err_out;
}

/*
 * This is the function that kick code will call in order to 'rollback' a
 * created output fence should an error occur when submitting the kick.
 * The OS native sync code needs to implement a function meeting this
 * specification.
 *
 * Input: fence_to_rollback The PVRSRV_FENCE to be 'rolled back'. The fence
 *                          should be destroyed and any actions taken due to
 *                          its creation that need to be undone should be
 *                          reverted.
 * Input: finalise_data     The finalise data for the fence to be 'rolled back'.
 */
static enum PVRSRV_ERROR
pvr_sync_rollback_fence_data(PVRSRV_FENCE fence_to_rollback,
			     void *fence_data_to_rollback)
{
	struct sync_file *sync_file = fence_data_to_rollback;
	struct pvr_fence *pvr_fence;

	if (!sync_file || fence_to_rollback < 0) {
		pr_err(FILE_NAME ": %s: Invalid fence (%d)\n", __func__,
			fence_to_rollback);
		return PVRSRV_ERROR_INVALID_PARAMS;
	}

	pvr_fence = to_pvr_fence(sync_file->fence);
	if (!pvr_fence) {
		pr_err(FILE_NAME
			": %s: Non-PVR fence (%p)\n",
			__func__, sync_file->fence);
		return PVRSRV_ERROR_INVALID_PARAMS;
	}

	fput(sync_file->file);

	put_unused_fd(fence_to_rollback);

	pvr_fence_destroy(pvr_fence);

	return PVRSRV_OK;
}

/*
 * This is the function that kick code will call in order to obtain a list of
 * the PSYNC_CHECKPOINTs for a given PVRSRV_FENCE passed to a kick function.
 * The OS native sync code will allocate the memory to hold the returned list
 * of PSYNC_CHECKPOINT ptrs. The caller will free this memory once it has
 * finished referencing it.
 *
 * Input: fence                     The input (check) fence
 * Output: nr_checkpoints           The number of PVRSRV_SYNC_CHECKPOINT ptrs
 *                                  returned in the checkpoint_handles
 *                                  parameter.
 * Output: fence_uid                Unique ID of the check fence
 * Input/Output: checkpoint_handles The returned list of PVRSRV_SYNC_CHECKPOINTs.
 */
static enum PVRSRV_ERROR
pvr_sync_resolve_fence(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
		       PVRSRV_FENCE fence_to_resolve, u32 *nr_checkpoints,
		       PSYNC_CHECKPOINT **checkpoint_handles, u64 *fence_uid)
{
	PSYNC_CHECKPOINT *checkpoints = NULL;
	unsigned int i, num_fences, num_used_fences = 0;
	struct dma_fence **fences = NULL;
	struct dma_fence *fence;
	PVRSRV_ERROR err = PVRSRV_OK;

	if (!nr_checkpoints || !checkpoint_handles || !fence_uid) {
		pr_err(FILE_NAME ": %s: Invalid input checkpoint pointer\n",
			__func__);
		err =  PVRSRV_ERROR_INVALID_PARAMS;
		goto err_out;
	}

	*nr_checkpoints = 0;
	*checkpoint_handles = NULL;
	*fence_uid = 0;

	if (fence_to_resolve < 0)
		goto err_out;

	fence = sync_file_get_fence(fence_to_resolve);
	if (!fence) {
		pr_err(FILE_NAME ": %s: Failed to read sync private data for fd %d\n",
			__func__, fence_to_resolve);
		err = PVRSRV_ERROR_HANDLE_NOT_FOUND;
		goto err_out;
	}

	if (dma_fence_is_array(fence)) {
		struct dma_fence_array *array = to_dma_fence_array(fence);

		fences = array->fences;
		num_fences = array->num_fences;
	} else {
		fences = &fence;
		num_fences = 1;
	}

	checkpoints = kmalloc_array(num_fences, sizeof(PSYNC_CHECKPOINT),
			      GFP_KERNEL);
	if (!checkpoints) {
		err = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto err_put_fence;
	}
	for (i = 0; i < num_fences; i++) {
		/* Only return the checkpoint if the fence is still active. */
		if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
			      &fences[i]->flags)) {
			struct pvr_fence *pvr_fence =
				pvr_fence_create_from_fence(
					pvr_sync_data.foreign_fence_context,
					psSyncCheckpointContext,
					fences[i],
					fence_to_resolve,
					"foreign");
			if (!pvr_fence) {
				pr_err(FILE_NAME ": %s: Failed to create fence\n",
				       __func__);
				err = PVRSRV_ERROR_OUT_OF_MEMORY;
				goto err_free_checkpoints;
			}
			checkpoints[num_used_fences] =
				pvr_fence_get_checkpoint(pvr_fence);
			SyncCheckpointTakeRef(checkpoints[num_used_fences]);
			++num_used_fences;
			dma_fence_put(&pvr_fence->base);
		}
	}
	/* If we don't return any checkpoints, delete the array because
	 * the caller will not.
	 */
	if (num_used_fences == 0) {
		kfree(checkpoints);
		checkpoints = NULL;
	}

	*checkpoint_handles = checkpoints;
	*nr_checkpoints = num_used_fences;
	*fence_uid = OSGetCurrentClientProcessIDKM();
	*fence_uid = (*fence_uid << 32) | (fence_to_resolve & U32_MAX);

err_put_fence:
	dma_fence_put(fence);
err_out:
	return err;

err_free_checkpoints:
	for (i = 0; i < num_used_fences; i++) {
		if (checkpoints[i])
			SyncCheckpointDropRef(checkpoints[i]);
	}
	kfree(checkpoints);
	goto err_put_fence;
}

/*
 * This is the function that driver code will call in order to request the
 * sync implementation to output debug information relating to any sync
 * checkpoints it may have created which appear in the provided array of
 * FW addresses of Unified Fence Objects (UFOs).
 *
 * Input: nr_ufos             The number of FW addresses provided in the
 *                            vaddrs parameter.
 * Input: vaddrs              The array of FW addresses of UFOs. The sync
 *                            implementation should check each of these to
 *                            see if any relate to sync checkpoints it has
 *                            created and where they do output debug information
 *                            pertaining to the native/fallback sync with
 *                            which it is associated.
 */
static u32
pvr_sync_dump_info_on_stalled_ufos(u32 nr_ufos, u32 *vaddrs)
{
	return pvr_fence_dump_info_on_stalled_ufos(pvr_sync_data.foreign_fence_context,
						   nr_ufos,
						   vaddrs);
}

#if defined(PVRSRV_SYNC_CHECKPOINT_CCB)
static enum tag_img_bool pvr_sync_checkpoint_ufo_has_signalled(u32 fwaddr,
	u32 value)
{
	return pvr_fence_checkpoint_ufo_has_signalled(fwaddr, value);
}

static void pvr_sync_check_state(void)
{
	pvr_fence_check_state();
}
#endif /* defined(PVRSRV_SYNC_CHECKPOINT_CCB) */

static long pvr_sync_ioctl_rename(struct pvr_sync_timeline *timeline,
	void __user *user_data)
{
	int err = 0;
	struct pvr_sync_rename_ioctl_data data;

	if (!access_ok(user_data, sizeof(data))) {
		err = -EFAULT;
		goto err;
	}

	if (copy_from_user(&data, user_data, sizeof(data))) {
		err = -EFAULT;
		goto err;
	}

	data.szName[sizeof(data.szName) - 1] = '\0';
	strlcpy(timeline->name, data.szName, sizeof(timeline->name));
	if (timeline->hw_fence_context)
		strlcpy(timeline->hw_fence_context->name, data.szName,
			sizeof(timeline->hw_fence_context->name));

err:
	return err;
}

static long pvr_sync_ioctl_force_sw_only(struct pvr_sync_timeline *timeline,
	void **private_data)
{
	/* Already in SW mode? */
	if (timeline->sw_fence_timeline)
		return 0;

	/* Create a sw_sync timeline with the old GPU timeline's name */
	timeline->sw_fence_timeline = pvr_counting_fence_timeline_create(
		pvr_sync_data.dev_cookie,
		timeline->name);
	if (!timeline->sw_fence_timeline)
		return -ENOMEM;

	timeline->is_sw = true;

	return 0;
}

static long pvr_sync_ioctl_sw_create_fence(struct pvr_sync_timeline *timeline,
	void __user *user_data)
{
	struct pvr_sw_sync_create_fence_data data;
	struct sync_file *sync_file;
	int fd = get_unused_fd_flags(O_CLOEXEC);
	struct dma_fence *fence;
	int err = -EFAULT;

	if (fd < 0) {
		pr_err(FILE_NAME ": %s: Failed to find unused fd (%d)\n",
		       __func__, fd);
		err = -EMFILE;
		goto err_out;
	}

	if (copy_from_user(&data, user_data, sizeof(data))) {
		pr_err(FILE_NAME ": %s: Failed copy from user\n", __func__);
		goto err_put_fd;
	}

	fence = pvr_counting_fence_create(timeline->sw_fence_timeline, &data.sync_pt_idx);
	if (!fence) {
		pr_err(FILE_NAME ": %s: Failed to create a sync point (%d)\n",
		       __func__, fd);
		err = -ENOMEM;
		goto err_put_fd;
	}

	sync_file = sync_file_create(fence);
	if (!sync_file) {
		pr_err(FILE_NAME ": %s: Failed to create a sync point (%d)\n",
			__func__, fd);
		 err = -ENOMEM;
		goto err_put_fence;
	}

	data.fence = fd;

	if (copy_to_user(user_data, &data, sizeof(data))) {
		pr_err(FILE_NAME ": %s: Failed copy to user\n", __func__);
		goto err_put_fence;
	}

	fd_install(fd, sync_file->file);
	err = 0;

	dma_fence_put(fence);
err_out:
	return err;

err_put_fence:
	dma_fence_put(fence);
err_put_fd:
	put_unused_fd(fd);
	goto err_out;
}

static long pvr_sync_ioctl_sw_inc(struct pvr_sync_timeline *timeline,
                                  void __user *user_data)
{
	bool res;
	struct pvr_sw_timeline_advance_data data;

	res = pvr_counting_fence_timeline_inc(timeline->sw_fence_timeline, &data.sync_pt_idx);

	/* pvr_counting_fence_timeline_inc won't allow sw timeline to be
	 * advanced beyond the last defined point
	 */
	if (!res) {
		pr_err("pvr_sync_file: attempt to advance SW timeline beyond last defined point\n");
		return -EPERM;
	}

	if (copy_to_user(user_data, &data, sizeof(data))) {
		pr_err(FILE_NAME ": %s: Failed copy to user\n", __func__);
		return -EFAULT;
	}

	return 0;
}

static long
pvr_sync_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
	void __user *user_data = (void __user *)arg;
	long err = -ENOTTY;
	struct pvr_sync_timeline *timeline = file->private_data;

	if (!timeline->is_sw) {

		switch (cmd) {
		case PVR_SYNC_IOC_RENAME:
			err = pvr_sync_ioctl_rename(timeline, user_data);
			break;
		case PVR_SYNC_IOC_FORCE_SW_ONLY:
			err = pvr_sync_ioctl_force_sw_only(timeline,
				&file->private_data);
			break;
		default:
			break;
		}
	} else {

		switch (cmd) {
		case PVR_SW_SYNC_IOC_CREATE_FENCE:
			err = pvr_sync_ioctl_sw_create_fence(timeline,
							     user_data);
			break;
		case PVR_SW_SYNC_IOC_INC:
			err = pvr_sync_ioctl_sw_inc(timeline, user_data);
			break;
		default:
			break;
		}
	}

	return err;
}

static const struct file_operations pvr_sync_fops = {
	.owner          = THIS_MODULE,
	.open           = pvr_sync_open,
	.release        = pvr_sync_close,
	.unlocked_ioctl = pvr_sync_ioctl,
	.compat_ioctl   = pvr_sync_ioctl,
};

static struct miscdevice pvr_sync_device = {
	.minor          = MISC_DYNAMIC_MINOR,
	.name           = PVRSYNC_MODNAME,
	.fops           = &pvr_sync_fops,
};

static void
pvr_sync_debug_request_heading(void *data, u32 verbosity,
				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
				void *pvDumpDebugFile)
{
	if (DD_VERB_LVL_ENABLED(verbosity, DEBUG_REQUEST_VERBOSITY_MEDIUM))
		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, "------[ Native Fence Sync: timelines ]------");
}

enum PVRSRV_ERROR pvr_sync_init(struct device *dev)
{
	struct drm_device *ddev = dev_get_drvdata(dev);
	struct pvr_drm_private *priv = ddev->dev_private;
	enum PVRSRV_ERROR error;
	int err;

	error = PVRSRVRegisterDbgRequestNotify(&pvr_sync_data.dbg_request_handle,
				priv->dev_node,
				pvr_sync_debug_request_heading,
				DEBUG_REQUEST_LINUXFENCE,
				NULL);
	if (error != PVRSRV_OK) {
		pr_err("%s: failed to register debug request callback (%s)\n",
		       __func__, PVRSRVGetErrorString(error));
		goto err_out;
	}

	pvr_sync_data.dev_cookie = priv->dev_node;
	pvr_sync_data.fence_status_wq = priv->fence_status_wq;

	pvr_sync_data.foreign_fence_context =
		pvr_fence_context_create(pvr_sync_data.dev_cookie,
					 pvr_sync_data.fence_status_wq,
					 "foreign_sync");
	if (!pvr_sync_data.foreign_fence_context) {
		pr_err(FILE_NAME ": %s: Failed to create foreign sync context\n",
			__func__);
		error = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto err_out;
	}

#if defined(NO_HARDWARE)
	INIT_LIST_HEAD(&pvr_sync_data.pvr_timeline_active_list);
#endif

	/* Register the resolve fence and create fence functions with
	 * sync_checkpoint.c
	 * The pvr_fence context registers its own EventObject callback to
	 * update sync status
	 */
	/* Initialise struct and register with sync_checkpoint.c */
	pvr_sync_data.sync_checkpoint_ops.pfnFenceResolve = pvr_sync_resolve_fence;
	pvr_sync_data.sync_checkpoint_ops.pfnFenceCreate = pvr_sync_create_fence;
	pvr_sync_data.sync_checkpoint_ops.pfnFenceDataRollback = pvr_sync_rollback_fence_data;
	pvr_sync_data.sync_checkpoint_ops.pfnFenceFinalise = pvr_sync_finalise_fence;
#if defined(NO_HARDWARE)
	pvr_sync_data.sync_checkpoint_ops.pfnNoHWUpdateTimelines = pvr_sync_nohw_signal_fence;
#else
	pvr_sync_data.sync_checkpoint_ops.pfnNoHWUpdateTimelines = NULL;
#endif
	pvr_sync_data.sync_checkpoint_ops.pfnFreeCheckpointListMem = pvr_sync_free_checkpoint_list_mem;
	pvr_sync_data.sync_checkpoint_ops.pfnDumpInfoOnStalledUFOs = pvr_sync_dump_info_on_stalled_ufos;
#if defined(PVRSRV_SYNC_CHECKPOINT_CCB)
	pvr_sync_data.sync_checkpoint_ops.pfnCheckpointHasSignalled = pvr_sync_checkpoint_ufo_has_signalled;
	pvr_sync_data.sync_checkpoint_ops.pfnCheckState = pvr_sync_check_state;
	pvr_sync_data.sync_checkpoint_ops.pfnSignalWaiters = NULL;
#endif /* defined(PVRSRV_SYNC_CHECKPOINT_CCB) */
	strlcpy(pvr_sync_data.sync_checkpoint_ops.pszImplName, "pvr_sync_file", SYNC_CHECKPOINT_IMPL_MAX_STRLEN);

	SyncCheckpointRegisterFunctions(&pvr_sync_data.sync_checkpoint_ops);

	err = misc_register(&pvr_sync_device);
	if (err) {
		pr_err(FILE_NAME ": %s: Failed to register pvr_sync device (%d)\n",
		       __func__, err);
		error = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
		goto err_unregister_checkpoint_funcs;
	}
	error = PVRSRV_OK;

err_out:
	return error;

err_unregister_checkpoint_funcs:
	SyncCheckpointRegisterFunctions(NULL);
	pvr_fence_context_destroy(pvr_sync_data.foreign_fence_context);
	goto err_out;
}

void pvr_sync_deinit(void)
{
	SyncCheckpointRegisterFunctions(NULL);
	misc_deregister(&pvr_sync_device);
	pvr_fence_context_destroy(pvr_sync_data.foreign_fence_context);
	PVRSRVUnregisterDbgRequestNotify(pvr_sync_data.dbg_request_handle);
}

enum PVRSRV_ERROR pvr_sync_fence_wait(void *fence, u32 timeout_in_ms)
{
	long timeout = msecs_to_jiffies(timeout_in_ms);
	int err;

	err = dma_fence_wait_timeout(fence, true, timeout);
	/*
	 * dma_fence_wait_timeout returns:
	 * - the remaining timeout on success
	 * - 0 on timeout
	 * - -ERESTARTSYS if interrupted
	 */
	if (err > 0)
		return PVRSRV_OK;
	else if (err == 0)
		return PVRSRV_ERROR_TIMEOUT;

	return PVRSRV_ERROR_FAILED_DEPENDENCIES;
}

enum PVRSRV_ERROR pvr_sync_fence_release(void *fence)
{
	dma_fence_put(fence);

	return PVRSRV_OK;
}

enum PVRSRV_ERROR pvr_sync_fence_get(int fence_fd, void **fence_out)
{
	struct dma_fence *fence;

	fence = sync_file_get_fence(fence_fd);
	if (fence == NULL)
		return PVRSRV_ERROR_INVALID_PARAMS;

	*fence_out = fence;

	return PVRSRV_OK;
}

enum PVRSRV_ERROR pvr_sync_sw_timeline_fence_create(int timeline_fd,
						    const char *fence_name,
						    int *fence_fd_out,
						    u64 *sync_pt_idx)
{
	enum PVRSRV_ERROR srv_err;
	struct pvr_sync_timeline *timeline;
	struct dma_fence *fence = NULL;
	struct sync_file *sync_file = NULL;
	int fd;

	fd = get_unused_fd_flags(O_CLOEXEC);
	if (fd < 0)
		return PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;

	timeline = pvr_sync_timeline_fget(timeline_fd);
	if (!timeline) {
		/* unrecognised timeline */
		srv_err = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
		goto err_put_fd;
	}

	fence = pvr_counting_fence_create(timeline->sw_fence_timeline, sync_pt_idx);
	pvr_sync_timeline_fput(timeline);
	if (!fence) {
		srv_err = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto err_put_fd;
	}

	sync_file = sync_file_create(fence);
	if (!sync_file) {
		srv_err = PVRSRV_ERROR_OUT_OF_MEMORY;
		goto err_put_fence;
	}

	fd_install(fd, sync_file->file);

	*fence_fd_out = fd;

	return PVRSRV_OK;

err_put_fence:
	dma_fence_put(fence);
err_put_fd:
	put_unused_fd(fd);
	return srv_err;
}

enum PVRSRV_ERROR pvr_sync_sw_timeline_advance(void *timeline, u64 *sync_pt_idx)
{
	if (timeline == NULL)
		return PVRSRV_ERROR_INVALID_PARAMS;

	pvr_counting_fence_timeline_inc(timeline, sync_pt_idx);

	return PVRSRV_OK;
}

enum PVRSRV_ERROR pvr_sync_sw_timeline_release(void *timeline)
{
	if (timeline == NULL)
		return PVRSRV_ERROR_INVALID_PARAMS;

	pvr_counting_fence_timeline_put(timeline);

	return PVRSRV_OK;
}

enum PVRSRV_ERROR pvr_sync_sw_timeline_get(int timeline_fd,
					   void **timeline_out)
{
	struct pvr_counting_fence_timeline *sw_timeline;
	struct pvr_sync_timeline *timeline;

	timeline = pvr_sync_timeline_fget(timeline_fd);
	if (!timeline)
		return PVRSRV_ERROR_INVALID_PARAMS;

	sw_timeline =
		pvr_counting_fence_timeline_get(timeline->sw_fence_timeline);
	pvr_sync_timeline_fput(timeline);
	if (!sw_timeline)
		return PVRSRV_ERROR_INVALID_PARAMS;

	*timeline_out = sw_timeline;

	return PVRSRV_OK;
}
static void _dump_sync_point(struct dma_fence *fence,
							  DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
							  void *dump_debug_file)
{
	const struct dma_fence_ops *fence_ops = fence->ops;
	bool signaled = dma_fence_is_signaled(fence);
	char time[16] = { '\0' };

	fence_ops->timeline_value_str(fence, time, sizeof(time));

	PVR_DUMPDEBUG_LOG(dump_debug_printf,
					  dump_debug_file,
					  "<%p> Seq#=%u TS=%s State=%s TLN=%s",
					  fence,
					  fence->seqno,
					  time,
					  (signaled) ? "Signalled" : "Active",
					  fence_ops->get_timeline_name(fence));
}

static void _dump_fence(struct dma_fence *fence,
			DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
			void *dump_debug_file)
{
	if (dma_fence_is_array(fence)) {
		struct dma_fence_array *fence_array = to_dma_fence_array(fence);
		int i;

		PVR_DUMPDEBUG_LOG(dump_debug_printf,
				  dump_debug_file,
				  "Fence: [%p] Sync Points:\n",
				  fence_array);

		for (i = 0; i < fence_array->num_fences; i++)
			_dump_sync_point(fence_array->fences[i],
					 dump_debug_printf,
					 dump_debug_file);

	} else {
		_dump_sync_point(fence, dump_debug_printf, dump_debug_file);
	}
}

enum PVRSRV_ERROR
sync_dump_fence(void *sw_fence_obj,
		DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
		void *dump_debug_file)
{
	struct dma_fence *fence = (struct dma_fence *) sw_fence_obj;

	_dump_fence(fence, dump_debug_printf, dump_debug_file);

	return PVRSRV_OK;
}

enum PVRSRV_ERROR
sync_sw_dump_timeline(void *sw_timeline_obj,
		      DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
		      void *dump_debug_file)
{
	pvr_counting_fence_timeline_dump_timeline(sw_timeline_obj,
						  dump_debug_printf,
						  dump_debug_file);

	return PVRSRV_OK;
}