/*************************************************************************/ /*!
@File           cache_km.c
@Title          CPU d-cache maintenance operations framework
@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
@Description    Implements server side code for CPU d-cache maintenance taking
                into account the idiosyncrasies of the various types of CPU
                d-cache instruction-set architecture (ISA) maintenance
                mechanisms.
@License        Dual MIT/GPLv2

The contents of this file are subject to the MIT license as set out below.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

Alternatively, the contents of this file may be used under the terms of
the GNU General Public License Version 2 ("GPL") in which case the provisions
of GPL are applicable instead of those above.

If you wish to allow use of your version of this file only under the terms of
GPL, and not to allow others to use your version of this file under the terms
of the MIT license, indicate your decision by deleting the provisions above
and replace them with the notice and other provisions required by GPL as set
out in the file called "GPL-COPYING" included in this distribution. If you do
not delete the provisions above, a recipient may use your version of this file
under the terms of either the MIT license or GPL.

This License is also included in this distribution in the file called
"MIT-COPYING".

EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/ /**************************************************************************/
#if defined(LINUX)
#include <linux/version.h>
#include <linux/uaccess.h>
#include <asm/current.h>
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#endif

#include "pmr.h"
#include "log2.h"
#include "device.h"
#include "pvrsrv.h"
#include "osfunc.h"
#include "cache_km.h"
#include "pvr_debug.h"
#include "lock_types.h"
#include "allocmem.h"
#include "process_stats.h"
#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
#include "ri_server.h"
#endif
#include "devicemem.h"
#include "pvrsrv_apphint.h"
#include "pvrsrv_sync_server.h"
#include "km_apphint_defs.h"

/* This header must always be included last */
#if defined(LINUX)
#include "kernel_compatibility.h"
#endif

/* Top-level file-local build definitions */
#if defined(PVRSRV_ENABLE_CACHEOP_STATS) && defined(LINUX)
#define CACHEOP_DEBUG
#define CACHEOP_STATS_ITEMS_MAX 			32
#define INCR_WRAP(x)						((x+1) >= CACHEOP_STATS_ITEMS_MAX ? 0 : (x+1))
#define DECR_WRAP(x)						((x-1) < 0 ? (CACHEOP_STATS_ITEMS_MAX-1) : (x-1))
#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
/* Refer to CacheOpStatsExecLogHeader() for header item names */
#define CACHEOP_RI_PRINTF_HEADER			"%-8s %-10s %-10s %-5s %-16s %-16s %-10s %-10s %-18s %-18s %-12s"
#define CACHEOP_RI_PRINTF					"%-8d %-10s %-10s %-5s 0x%-14llx 0x%-14llx 0x%-8llx 0x%-8llx %-18llu %-18llu 0x%-10x\n"
#else
#define CACHEOP_PRINTF_HEADER				"%-8s %-10s %-10s %-5s %-10s %-10s %-18s %-18s %-12s"
#define CACHEOP_PRINTF						"%-8d %-10s %-10s %-5s 0x%-8llx 0x%-8llx %-18llu %-18llu 0x%-10x\n"
#endif
#endif

//#define CACHEOP_NO_CACHE_LINE_ALIGNED_ROUNDING		/* Force OS page (not cache line) flush granularity */
#define CACHEOP_PVR_ASSERT(x)							/* Define as PVR_ASSERT(x), enable for swdev & testing */
#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
#define CACHEOP_THREAD_WAIT_TIMEOUT			0ULL		/* Wait indefinitely */
#else
#define CACHEOP_THREAD_WAIT_TIMEOUT			500000ULL	/* Wait 500ms between wait unless woken-up on demand */
#endif
#define CACHEOP_FENCE_WAIT_TIMEOUT			1000ULL		/* Wait 1ms between wait events unless woken-up */
#define CACHEOP_FENCE_RETRY_ABORT			1000ULL		/* Fence retries that aborts fence operation */
#define CACHEOP_SEQ_MIDPOINT (IMG_UINT32)	0x7FFFFFFF	/* Where seqNum(s) are rebase, compared at */
#define CACHEOP_ABORT_FENCE_ERROR_STRING	"detected stalled client, retrying cacheop fence"
#define CACHEOP_NO_GFLUSH_ERROR_STRING		"global flush requested on CPU without support"
#define CACHEOP_DEVMEM_OOR_ERROR_STRING		"cacheop device memory request is out of range"
#define CACHEOP_MAX_DEBUG_MESSAGE_LEN		160

typedef struct _CACHEOP_WORK_ITEM_
{
	PMR *psPMR;
	IMG_UINT32 ui32GFSeqNum;
	IMG_UINT32 ui32OpSeqNum;
	IMG_DEVMEM_SIZE_T uiSize;
	PVRSRV_CACHE_OP uiCacheOp;
	IMG_DEVMEM_OFFSET_T uiOffset;
	PVRSRV_TIMELINE iTimeline;
	SYNC_TIMELINE_OBJ sSWTimelineObj;
	PVRSRV_DEVICE_NODE *psDevNode;
#if defined(CACHEOP_DEBUG)
	IMG_UINT64 ui64EnqueuedTime;
	IMG_UINT64 ui64DequeuedTime;
	IMG_UINT64 ui64ExecuteTime;
	IMG_BOOL bDeferred;
	IMG_BOOL bKMReq;
	IMG_BOOL bRBF;
	IMG_BOOL bUMF;
	IMG_PID pid;
#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
	RGXFWIF_DM eFenceOpType;
#endif
#endif
} CACHEOP_WORK_ITEM;

typedef struct _CACHEOP_STATS_EXEC_ITEM_
{
	IMG_PID pid;
	IMG_UINT32 ui32OpSeqNum;
	PVRSRV_CACHE_OP uiCacheOp;
	IMG_DEVMEM_SIZE_T uiOffset;
	IMG_DEVMEM_SIZE_T uiSize;
	IMG_UINT64 ui64EnqueuedTime;
	IMG_UINT64 ui64DequeuedTime;
	IMG_UINT64 ui64ExecuteTime;
	IMG_BOOL bIsFence;
	IMG_BOOL bKMReq;
	IMG_BOOL bRBF;
	IMG_BOOL bUMF;
	IMG_BOOL bDeferred;
#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
	IMG_DEV_VIRTADDR sDevVAddr;
	IMG_DEV_PHYADDR sDevPAddr;
	RGXFWIF_DM eFenceOpType;
#endif
} CACHEOP_STATS_EXEC_ITEM;

typedef enum _CACHEOP_CONFIG_
{
	CACHEOP_CONFIG_DEFAULT = 0,
	/* cache flush mechanism types */
	CACHEOP_CONFIG_KRBF    = 1,
	CACHEOP_CONFIG_KGF     = 2,
	CACHEOP_CONFIG_URBF    = 4,
	/* sw-emulated deferred flush mechanism */
	CACHEOP_CONFIG_KDF     = 8,
	/* pseudo configuration items */
	CACHEOP_CONFIG_LAST    = 16,
	CACHEOP_CONFIG_KLOG    = 16,
	CACHEOP_CONFIG_ALL     = 31
} CACHEOP_CONFIG;

typedef struct _CACHEOP_WORK_QUEUE_
{
/*
 * Init. state & primary device node framework
 * is anchored on.
 */
	IMG_BOOL bInit;
/*
  MMU page size/shift & d-cache line size
 */
	size_t uiPageSize;
	IMG_UINT32 uiLineSize;
	IMG_UINT32 uiLineShift;
	IMG_UINT32 uiPageShift;
	PVRSRV_CACHE_OP_ADDR_TYPE uiCacheOpAddrType;
/*
  CacheOp deferred queueing protocol
  + Implementation geared for performance, atomic counter based
	- Value Space is 0 -> 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7 -> 8 -> n.
	- Index Space is 0 -> 1 -> 2 -> 3 -> 0 -> 1 -> 2 -> 3 -> 0 -> m.
		- Index = Value modulo CACHEOP_INDICES_LOG2_SIZE.
  + Write counter never collides with read counter in index space
	- Unless at start of day when both are initialised to zero.
	- This means we sacrifice one entry when the queue is full.
	- Incremented by producer
		- Value space tracks total number of CacheOps queued.
		- Index space identifies CacheOp CCB queue index.
  + Read counter increments towards write counter in value space
	- Empty queue occurs when read equals write counter.
	- Wrap-round logic handled by consumer as/when needed.
	- Incremented by consumer
		- Value space tracks total # of CacheOps executed.
		- Index space identifies CacheOp CCB queue index.
  + Total queued size adjusted up/down during write/read activity
	- Counter might overflow but does not compromise framework.
 */
	ATOMIC_T hReadCounter;
	ATOMIC_T hWriteCounter;
/*
  CacheOp sequence numbers
  + hCommonSeqNum:
	- Common sequence, numbers every CacheOp operation in both UM/KM.
	- In KM
		- Every deferred CacheOp (on behalf of UM) gets a unique seqNum.
		- Last executed deferred CacheOp updates gsCwq.hCompletedSeqNum.
		- Every GF operation (if supported) also gets a unique seqNum.
		- Last executed GF operation updates CACHEOP_INFO_GFSEQNUM0.
		- Under debug, all CacheOp gets a unique seqNum for tracking.
		- This includes all UM/KM synchronous non-deferred CacheOp(s)
	- In UM
		- If the processor architecture supports GF maintenance (in KM)
		- All UM CacheOp samples CACHEOP_INFO_GFSEQNUM0 via info. page.
		- CacheOp(s) discarded if another GF occurs before execution.
		- CacheOp(s) discarding happens in both UM and KM space.
  + hCompletedSeqNum:
	- Tracks last executed KM/deferred RBF/Global<timeline> CacheOp(s)
  + hDeferredSize:
	- Running total of size of currently deferred CacheOp in queue.
 */
	ATOMIC_T hDeferredSize;
	ATOMIC_T hCommonSeqNum;
	ATOMIC_T hCompletedSeqNum;
/*
  CacheOp information page
  + psInfoPagePMR:
	- Single system-wide OS page that is multi-mapped in UM/KM.
	- Mapped into clients using read-only memory protection.
	- Mapped into server using read/write memory protection.
	- Contains information pertaining to cache framework.
  + pui32InfoPage:
	- Server linear address pointer to said information page.
	- Each info-page entry currently of sizeof(IMG_UINT32).
 */
	PMR *psInfoPagePMR;
	IMG_UINT32 *pui32InfoPage;
/*
  CacheOp deferred work-item queue
  + CACHEOP_INDICES_LOG2_SIZE
	- Sized using GF/RBF ratio
 */
#define CACHEOP_INDICES_LOG2_SIZE	(4)
#define CACHEOP_INDICES_MAX			(1 << CACHEOP_INDICES_LOG2_SIZE)
#define CACHEOP_INDICES_MASK		(CACHEOP_INDICES_MAX-1)
	CACHEOP_WORK_ITEM asWorkItems[CACHEOP_INDICES_MAX];
#if defined(CACHEOP_DEBUG)
/*
  CacheOp statistics
 */
	void *pvStatsEntry;
	IMG_HANDLE hStatsExecLock;
	IMG_UINT32 ui32ServerASync;
	IMG_UINT32 ui32ServerSyncVA;
	IMG_UINT32 ui32ServerSync;
	IMG_UINT32 ui32ServerRBF;
	IMG_UINT32 ui32ServerGF;
	IMG_UINT32 ui32ServerDGF;
	IMG_UINT32 ui32ServerDTL;
	IMG_UINT32 ui32ClientSync;
	IMG_UINT32 ui32ClientRBF;
	IMG_UINT32 ui32KMDiscards;
	IMG_UINT32 ui32UMDiscards;
	IMG_UINT32 ui32TotalFenceOps;
	IMG_UINT32 ui32TotalExecOps;
	IMG_UINT32 ui32AvgExecTime;
	IMG_UINT32 ui32AvgFenceTime;
	IMG_INT32 i32StatsExecWriteIdx;
	CACHEOP_STATS_EXEC_ITEM asStatsExecuted[CACHEOP_STATS_ITEMS_MAX];
#endif
/*
  CacheOp (re)configuration
 */
	void *pvConfigTune;
	IMG_HANDLE hConfigLock;
/*
  CacheOp deferred worker thread
  + eConfig
	- Runtime configuration
  + hWorkerThread
	- CacheOp thread handler
  + hThreadWakeUpEvtObj
	- Event object to drive CacheOp worker thread sleep/wake-ups.
  + hClientWakeUpEvtObj
	- Event object to unblock stalled clients waiting on queue.
 */
	CACHEOP_CONFIG	eConfig;
	IMG_UINT32		ui32Config;
	IMG_HANDLE		hWorkerThread;
	IMG_HANDLE 		hDeferredLock;
	IMG_HANDLE 		hGlobalFlushLock;
	IMG_HANDLE		hThreadWakeUpEvtObj;
	IMG_HANDLE		hClientWakeUpEvtObj;
	IMG_UINT32		ui32FenceWaitTimeUs;
	IMG_UINT32		ui32FenceRetryAbort;
	IMG_BOOL		bNoGlobalFlushImpl;
	IMG_BOOL		bSupportsUMFlush;
} CACHEOP_WORK_QUEUE;

/* Top-level CacheOp framework object */
static CACHEOP_WORK_QUEUE gsCwq;

#define CacheOpConfigSupports(e) ((gsCwq.eConfig & (e)) ? IMG_TRUE : IMG_FALSE)

static INLINE IMG_UINT32 CacheOpIdxRead(ATOMIC_T *phCounter)
{
	IMG_UINT32 ui32Idx = OSAtomicRead(phCounter);
	return ui32Idx & CACHEOP_INDICES_MASK;
}

static INLINE IMG_UINT32 CacheOpIdxIncrement(ATOMIC_T *phCounter)
{
	IMG_UINT32 ui32Idx = OSAtomicIncrement(phCounter);
	return ui32Idx & CACHEOP_INDICES_MASK;
}

static INLINE IMG_UINT32 CacheOpIdxNext(ATOMIC_T *phCounter)
{
	IMG_UINT32 ui32Idx = OSAtomicRead(phCounter);
	return ++ui32Idx & CACHEOP_INDICES_MASK;
}

static INLINE IMG_UINT32 CacheOpIdxSpan(ATOMIC_T *phLhs, ATOMIC_T *phRhs)
{
	return OSAtomicRead(phLhs) - OSAtomicRead(phRhs);
}

static INLINE IMG_UINT64 DivBy10(IMG_UINT64 uiNum)
{
	IMG_UINT64 uiQuot;
	IMG_UINT64 uiRem;

	uiQuot = (uiNum >> 1) + (uiNum >> 2);
	uiQuot = uiQuot + (uiQuot >> 4);
	uiQuot = uiQuot + (uiQuot >> 8);
	uiQuot = uiQuot + (uiQuot >> 16);
	uiQuot = uiQuot >> 3;
	uiRem  = uiNum - (((uiQuot << 2) + uiQuot) << 1);

	return uiQuot + (uiRem > 9);
}

/* Callback to dump info of cacheop thread in debug_dump */
static void CacheOpThreadDumpInfo(DUMPDEBUG_PRINTF_FUNC* pfnDumpDebugPrintf,
				  				void *pvDumpDebugFile)
{
	PVR_DUMPDEBUG_LOG("    Configuration: QSZ: %d, UKT: %d, KDFT: %d, "
			  "KGFT: %d, LINESIZE: %d, PGSIZE: %d, KDF: %s, "
			  "URBF: %s, KGF: %s, KRBF: %s",
			  CACHEOP_INDICES_MAX,
			  gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD],
			  gsCwq.pui32InfoPage[CACHEOP_INFO_KMDFTHRESHLD],
			  gsCwq.pui32InfoPage[CACHEOP_INFO_KMGFTHRESHLD],
			  gsCwq.pui32InfoPage[CACHEOP_INFO_LINESIZE],
			  gsCwq.pui32InfoPage[CACHEOP_INFO_PGSIZE],
			  gsCwq.eConfig & CACHEOP_CONFIG_KDF  ? "Yes" : "No",
			  gsCwq.eConfig & CACHEOP_CONFIG_URBF ? "Yes" : "No",
			  gsCwq.eConfig & CACHEOP_CONFIG_KGF  ? "Yes" : "No",
			  gsCwq.eConfig & CACHEOP_CONFIG_KRBF ? "Yes" : "No"
			  );
	PVR_DUMPDEBUG_LOG("    Pending deferred CacheOp entries : %u",
			  CacheOpIdxSpan(&gsCwq.hWriteCounter, &gsCwq.hReadCounter));
}

#if defined(CACHEOP_DEBUG)
static INLINE void CacheOpStatsExecLogHeader(IMG_CHAR szBuffer[CACHEOP_MAX_DEBUG_MESSAGE_LEN])
{
	OSSNPrintf(szBuffer, CACHEOP_MAX_DEBUG_MESSAGE_LEN,
#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
				CACHEOP_RI_PRINTF_HEADER,
#else
				CACHEOP_PRINTF_HEADER,
#endif
				"Pid",
				"CacheOp",
				"  Type",
				"Mode",
#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
				"DevVAddr",
				"DevPAddr",
#endif
				"Offset",
				"Size",
				"xTime (us)",
				"qTime (us)",
				"SeqNum");
}

static void CacheOpStatsExecLogWrite(CACHEOP_WORK_ITEM *psCacheOpWorkItem)
{
	IMG_UINT64 ui64ExecuteTime;
	IMG_UINT64 ui64EnqueuedTime;
	IMG_INT32 i32WriteOffset;

	if (!psCacheOpWorkItem->ui32OpSeqNum && !psCacheOpWorkItem->uiCacheOp)
	{
		/* This breaks the logic of read-out, so we do not queue items
		   with zero sequence number and no CacheOp */
		return;
	}
	else if (psCacheOpWorkItem->bKMReq && !CacheOpConfigSupports(CACHEOP_CONFIG_KLOG))
	{
		/* KM logs spams the history due to frequency, this remove its completely */
		return;
	}

	OSLockAcquire(gsCwq.hStatsExecLock);

	i32WriteOffset = gsCwq.i32StatsExecWriteIdx;
	gsCwq.asStatsExecuted[i32WriteOffset].pid = psCacheOpWorkItem->pid;
	gsCwq.i32StatsExecWriteIdx = INCR_WRAP(gsCwq.i32StatsExecWriteIdx);
	gsCwq.asStatsExecuted[i32WriteOffset].bRBF = psCacheOpWorkItem->bRBF;
	gsCwq.asStatsExecuted[i32WriteOffset].bUMF = psCacheOpWorkItem->bUMF;
	gsCwq.asStatsExecuted[i32WriteOffset].uiSize = psCacheOpWorkItem->uiSize;
	gsCwq.asStatsExecuted[i32WriteOffset].bKMReq = psCacheOpWorkItem->bKMReq;
	gsCwq.asStatsExecuted[i32WriteOffset].uiOffset	= psCacheOpWorkItem->uiOffset;
	gsCwq.asStatsExecuted[i32WriteOffset].uiCacheOp = psCacheOpWorkItem->uiCacheOp;
	gsCwq.asStatsExecuted[i32WriteOffset].bDeferred = psCacheOpWorkItem->bDeferred;
	gsCwq.asStatsExecuted[i32WriteOffset].ui32OpSeqNum	= psCacheOpWorkItem->ui32OpSeqNum;
	gsCwq.asStatsExecuted[i32WriteOffset].ui64ExecuteTime = psCacheOpWorkItem->ui64ExecuteTime;
	gsCwq.asStatsExecuted[i32WriteOffset].ui64EnqueuedTime = psCacheOpWorkItem->ui64EnqueuedTime;
	gsCwq.asStatsExecuted[i32WriteOffset].ui64DequeuedTime = psCacheOpWorkItem->ui64DequeuedTime;
	/* During early system initialisation, only non-fence & non-PMR CacheOps are processed */
	gsCwq.asStatsExecuted[i32WriteOffset].bIsFence = gsCwq.bInit && !psCacheOpWorkItem->psPMR;
	CACHEOP_PVR_ASSERT(gsCwq.asStatsExecuted[i32WriteOffset].pid);
#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
	if (gsCwq.bInit && psCacheOpWorkItem->psPMR)
	{
		IMG_CPU_PHYADDR sDevPAddr;
		PVRSRV_ERROR eError;
		IMG_BOOL bValid;

		/* Get more detailed information regarding the sub allocations that
		   PMR has from RI manager for process that requested the CacheOp */
		eError = RIDumpProcessListKM(psCacheOpWorkItem->psPMR,
									 gsCwq.asStatsExecuted[i32WriteOffset].pid,
									 gsCwq.asStatsExecuted[i32WriteOffset].uiOffset,
									 &gsCwq.asStatsExecuted[i32WriteOffset].sDevVAddr);
		if (eError != PVRSRV_OK)
		{
			goto e0;
		}

		/* (Re)lock here as some PMR might have not been locked */
		eError = PMRLockSysPhysAddresses(psCacheOpWorkItem->psPMR);
		if (eError != PVRSRV_OK)
		{
			goto e0;
		}

		eError = PMR_CpuPhysAddr(psCacheOpWorkItem->psPMR,
								 gsCwq.uiPageShift,
								 1,
								 gsCwq.asStatsExecuted[i32WriteOffset].uiOffset,
								 &sDevPAddr,
								 &bValid);
		if (eError != PVRSRV_OK)
		{
			eError = PMRUnlockSysPhysAddresses(psCacheOpWorkItem->psPMR);
			PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
			goto e0;
		}

		eError = PMRUnlockSysPhysAddresses(psCacheOpWorkItem->psPMR);
		PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");

		gsCwq.asStatsExecuted[i32WriteOffset].sDevPAddr.uiAddr = sDevPAddr.uiAddr;
	}

	if (gsCwq.asStatsExecuted[i32WriteOffset].bIsFence)
	{
		gsCwq.asStatsExecuted[i32WriteOffset].eFenceOpType = psCacheOpWorkItem->eFenceOpType;
	}
#endif

	/* Convert timing from nano-seconds to micro-seconds */
	ui64ExecuteTime = gsCwq.asStatsExecuted[i32WriteOffset].ui64ExecuteTime;
	ui64EnqueuedTime = gsCwq.asStatsExecuted[i32WriteOffset].ui64EnqueuedTime;
	ui64ExecuteTime = DivBy10(DivBy10(DivBy10(ui64ExecuteTime)));
	ui64EnqueuedTime = DivBy10(DivBy10(DivBy10(ui64EnqueuedTime)));

	/* Coalesced (to global) deferred CacheOps do not contribute to statistics,
	   as both enqueue/execute time is identical for these CacheOps */
	if (!gsCwq.asStatsExecuted[i32WriteOffset].bIsFence)
	{
		/* Calculate the rolling approximate average execution time */
		IMG_UINT32 ui32Time = ui64EnqueuedTime < ui64ExecuteTime ?
									ui64ExecuteTime - ui64EnqueuedTime :
									ui64EnqueuedTime - ui64ExecuteTime;
		if (gsCwq.ui32TotalExecOps > 2 && ui32Time)
		{
			gsCwq.ui32AvgExecTime -= (gsCwq.ui32AvgExecTime / gsCwq.ui32TotalExecOps);
			gsCwq.ui32AvgExecTime += (ui32Time / gsCwq.ui32TotalExecOps);
		}
		else if (ui32Time)
		{
			gsCwq.ui32AvgExecTime = (IMG_UINT32)ui32Time;
		}
	}

	if (! gsCwq.asStatsExecuted[i32WriteOffset].bKMReq)
	{
		/* This operation queues only UM CacheOp in per-PID process statistics database */
		PVRSRVStatsUpdateCacheOpStats(gsCwq.asStatsExecuted[i32WriteOffset].uiCacheOp,
						gsCwq.asStatsExecuted[i32WriteOffset].ui32OpSeqNum,
#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
						gsCwq.asStatsExecuted[i32WriteOffset].sDevVAddr,
						gsCwq.asStatsExecuted[i32WriteOffset].sDevPAddr,
						gsCwq.asStatsExecuted[i32WriteOffset].eFenceOpType,
#endif
						gsCwq.asStatsExecuted[i32WriteOffset].uiOffset,
						gsCwq.asStatsExecuted[i32WriteOffset].uiSize,
						ui64EnqueuedTime < ui64ExecuteTime ?
							ui64ExecuteTime - ui64EnqueuedTime:
							ui64EnqueuedTime - ui64ExecuteTime,
						gsCwq.asStatsExecuted[i32WriteOffset].bRBF,
						gsCwq.asStatsExecuted[i32WriteOffset].bUMF,
						gsCwq.asStatsExecuted[i32WriteOffset].bIsFence,
						psCacheOpWorkItem->pid);
	}

#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
e0:
#endif
	OSLockRelease(gsCwq.hStatsExecLock);
}

static void CacheOpStatsExecLogRead(void *pvFilePtr, void *pvData,
								OS_STATS_PRINTF_FUNC* pfnOSStatsPrintf)
{
	IMG_CHAR *pszFlushype;
	IMG_CHAR *pszCacheOpType;
	IMG_CHAR *pszFlushSource;
	IMG_INT32 i32ReadOffset;
	IMG_INT32 i32WriteOffset;
	IMG_UINT64 ui64EnqueuedTime;
	IMG_UINT64 ui64DequeuedTime;
	IMG_UINT64 ui64ExecuteTime;
	IMG_CHAR szBuffer[CACHEOP_MAX_DEBUG_MESSAGE_LEN] = {0};
	PVR_UNREFERENCED_PARAMETER(pvData);

	OSLockAcquire(gsCwq.hStatsExecLock);

	pfnOSStatsPrintf(pvFilePtr,
			"Primary CPU d-cache architecture: LSZ: 0x%d, URBF: %s, KGF: %s, KRBF: %s\n",
			gsCwq.uiLineSize,
			gsCwq.bSupportsUMFlush ? "Yes" : "No",
			!gsCwq.bNoGlobalFlushImpl ? "Yes" : "No",
			"Yes" /* KRBF mechanism always available */
		);

	pfnOSStatsPrintf(pvFilePtr,
			"Configuration: QSZ: %d, UKT: %d, KDFT: %d, KGFT: %d, KDF: %s, URBF: %s, KGF: %s, KRBF: %s\n",
			CACHEOP_INDICES_MAX,
			gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD],
			gsCwq.pui32InfoPage[CACHEOP_INFO_KMDFTHRESHLD],
			gsCwq.pui32InfoPage[CACHEOP_INFO_KMGFTHRESHLD],
			gsCwq.eConfig & CACHEOP_CONFIG_KDF  ? "Yes" : "No",
			gsCwq.eConfig & CACHEOP_CONFIG_URBF ? "Yes" : "No",
			gsCwq.eConfig & CACHEOP_CONFIG_KGF  ? "Yes" : "No",
			gsCwq.eConfig & CACHEOP_CONFIG_KRBF ? "Yes" : "No"
		);

	pfnOSStatsPrintf(pvFilePtr,
			"Summary: OP[F][TL] (tot.avg): %d.%d/%d.%d/%d, [KM][UM][A]SYNC: %d.%d/%d/%d, RBF (um/km): %d/%d, [D]GF (km): %d/%d, DSC (um/km): %d/%d\n",
			gsCwq.ui32TotalExecOps, gsCwq.ui32AvgExecTime, gsCwq.ui32TotalFenceOps, gsCwq.ui32AvgFenceTime, gsCwq.ui32ServerDTL,
			gsCwq.ui32ServerSync, gsCwq.ui32ServerSyncVA, gsCwq.ui32ClientSync,	gsCwq.ui32ServerASync,
			gsCwq.ui32ClientRBF,   gsCwq.ui32ServerRBF,
			gsCwq.ui32ServerDGF,   gsCwq.ui32ServerGF,
			gsCwq.ui32UMDiscards,  gsCwq.ui32KMDiscards
		);

	CacheOpStatsExecLogHeader(szBuffer);
	pfnOSStatsPrintf(pvFilePtr, "%s\n", szBuffer);

	i32WriteOffset = gsCwq.i32StatsExecWriteIdx;
	for (i32ReadOffset = DECR_WRAP(i32WriteOffset);
		 i32ReadOffset != i32WriteOffset;
		 i32ReadOffset = DECR_WRAP(i32ReadOffset))
	{
		if (!gsCwq.asStatsExecuted[i32ReadOffset].ui32OpSeqNum &&
			!gsCwq.asStatsExecuted[i32ReadOffset].uiCacheOp)
		{
			break;
		}

		/* Convert from nano-seconds to micro-seconds */
		ui64ExecuteTime = gsCwq.asStatsExecuted[i32ReadOffset].ui64ExecuteTime;
		ui64EnqueuedTime = gsCwq.asStatsExecuted[i32ReadOffset].ui64EnqueuedTime;
		ui64DequeuedTime = gsCwq.asStatsExecuted[i32ReadOffset].ui64DequeuedTime;
		ui64ExecuteTime = DivBy10(DivBy10(DivBy10(ui64ExecuteTime)));
		ui64EnqueuedTime = DivBy10(DivBy10(DivBy10(ui64EnqueuedTime)));
		ui64DequeuedTime = ui64DequeuedTime ? DivBy10(DivBy10(DivBy10(ui64DequeuedTime))) : 0;

		if (gsCwq.asStatsExecuted[i32ReadOffset].bIsFence)
		{
			IMG_CHAR *pszMode = "";
			IMG_CHAR *pszFenceType = "";
			pszCacheOpType = "Fence";

#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
			pszMode = gsCwq.asStatsExecuted[i32ReadOffset].uiCacheOp != PVRSRV_CACHE_OP_GLOBAL ? "" : "  GF  ";
			switch (gsCwq.asStatsExecuted[i32ReadOffset].eFenceOpType)
			{
				case RGXFWIF_DM_GP:
					pszFenceType = " GP/GF";
					break;

				case RGXFWIF_DM_TDM:
					pszFenceType = "  TDM ";
					break;

				case RGXFWIF_DM_TA:
					pszFenceType = "  TA ";
					break;

				case RGXFWIF_DM_3D:
					pszFenceType = "  PDM ";
					break;

				case RGXFWIF_DM_CDM:
					pszFenceType = "  CDM ";
					break;

				default:
					CACHEOP_PVR_ASSERT(0);
					break;
			}
#else
			/* The CacheOp fence operation also triggered a global cache flush operation */
			pszFenceType =
				gsCwq.asStatsExecuted[i32ReadOffset].uiCacheOp != PVRSRV_CACHE_OP_GLOBAL ? "" : "   GF ";
#endif
			pfnOSStatsPrintf(pvFilePtr,
#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
							CACHEOP_RI_PRINTF,
#else
							CACHEOP_PRINTF,
#endif
							gsCwq.asStatsExecuted[i32ReadOffset].pid,
							pszCacheOpType,
							pszFenceType,
							pszMode,
#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
							"",
							"",
#endif
							gsCwq.asStatsExecuted[i32ReadOffset].uiOffset,
							gsCwq.asStatsExecuted[i32ReadOffset].uiSize,
							ui64EnqueuedTime < ui64ExecuteTime ?
									ui64ExecuteTime - ui64EnqueuedTime
										:
									ui64EnqueuedTime - ui64ExecuteTime,
							ui64EnqueuedTime < ui64DequeuedTime ?
									ui64DequeuedTime - ui64EnqueuedTime
										:
									!ui64DequeuedTime ? 0 : ui64EnqueuedTime - ui64DequeuedTime,
							gsCwq.asStatsExecuted[i32ReadOffset].ui32OpSeqNum);
		}
		else
		{
			if (gsCwq.asStatsExecuted[i32ReadOffset].bRBF)
			{
				IMG_DEVMEM_SIZE_T ui64NumOfPages;

				ui64NumOfPages = gsCwq.asStatsExecuted[i32ReadOffset].uiSize >> gsCwq.uiPageShift;
				if (ui64NumOfPages <= PMR_MAX_TRANSLATION_STACK_ALLOC)
				{
					pszFlushype = "RBF.Fast";
				}
				else
				{
					pszFlushype = "RBF.Slow";
				}
			}
			else
			{
				pszFlushype = "   GF ";
			}

			if (gsCwq.asStatsExecuted[i32ReadOffset].bUMF)
			{
				pszFlushSource = " UM";
			}
			else
			{
				/*
				   - Request originates directly from a KM thread or in KM (KM<), or
				   - Request originates from a UM thread and is KM deferred (KM+), or
				   - Request is/was discarded due to an 'else-[when,where]' GFlush
				     - i.e. GF occurs either (a)sync to current UM/KM thread
				*/
				pszFlushSource =
					gsCwq.asStatsExecuted[i32ReadOffset].bKMReq ? " KM<" :
					gsCwq.asStatsExecuted[i32ReadOffset].bDeferred && gsCwq.asStatsExecuted[i32ReadOffset].ui64ExecuteTime ? " KM+" :
					!gsCwq.asStatsExecuted[i32ReadOffset].ui64ExecuteTime ? " KM-" : " KM";
			}

			switch (gsCwq.asStatsExecuted[i32ReadOffset].uiCacheOp)
			{
				case PVRSRV_CACHE_OP_NONE:
					pszCacheOpType = "None";
					break;
				case PVRSRV_CACHE_OP_CLEAN:
					pszCacheOpType = "Clean";
					break;
				case PVRSRV_CACHE_OP_INVALIDATE:
					pszCacheOpType = "Invalidate";
					break;
				case PVRSRV_CACHE_OP_FLUSH:
					pszCacheOpType = "Flush";
					break;
				case PVRSRV_CACHE_OP_GLOBAL:
					pszCacheOpType = "GFlush";
					break;
				case PVRSRV_CACHE_OP_TIMELINE:
					pszCacheOpType = "Timeline";
					pszFlushype = "      ";
					break;
				default:
					if ((IMG_UINT32)gsCwq.asStatsExecuted[i32ReadOffset].uiCacheOp == (IMG_UINT32)(PVRSRV_CACHE_OP_GLOBAL|PVRSRV_CACHE_OP_TIMELINE))
					{
						pszCacheOpType = "Timeline";
					}
					else
					{
						pszCacheOpType = "Unknown";
						gsCwq.asStatsExecuted[i32ReadOffset].ui32OpSeqNum =
								(IMG_UINT32) gsCwq.asStatsExecuted[i32ReadOffset].uiCacheOp;
					}
					break;
			}

			pfnOSStatsPrintf(pvFilePtr,
#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
							CACHEOP_RI_PRINTF,
#else
							CACHEOP_PRINTF,
#endif
							gsCwq.asStatsExecuted[i32ReadOffset].pid,
							pszCacheOpType,
							pszFlushype,
							pszFlushSource,
#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
							gsCwq.asStatsExecuted[i32ReadOffset].sDevVAddr.uiAddr,
							gsCwq.asStatsExecuted[i32ReadOffset].sDevPAddr.uiAddr,
#endif
							gsCwq.asStatsExecuted[i32ReadOffset].uiOffset,
							gsCwq.asStatsExecuted[i32ReadOffset].uiSize,
							ui64EnqueuedTime < ui64ExecuteTime ?
										ui64ExecuteTime - ui64EnqueuedTime
											:
										ui64EnqueuedTime - ui64ExecuteTime,
							ui64EnqueuedTime < ui64DequeuedTime ?
									ui64DequeuedTime - ui64EnqueuedTime
										:
									!ui64DequeuedTime ? 0 : ui64EnqueuedTime - ui64DequeuedTime,
							gsCwq.asStatsExecuted[i32ReadOffset].ui32OpSeqNum);
		}
	}

	OSLockRelease(gsCwq.hStatsExecLock);
}
#endif /* defined(CACHEOP_DEBUG) */

static INLINE void CacheOpStatsReset(void)
{
#if defined(CACHEOP_DEBUG)
	gsCwq.ui32KMDiscards    = 0;
	gsCwq.ui32UMDiscards    = 0;
	gsCwq.ui32TotalExecOps  = 0;
	gsCwq.ui32TotalFenceOps = 0;
	gsCwq.ui32AvgExecTime   = 0;
	gsCwq.ui32AvgFenceTime  = 0;
	gsCwq.ui32ClientRBF     = 0;
	gsCwq.ui32ClientSync    = 0;
	gsCwq.ui32ServerRBF     = 0;
	gsCwq.ui32ServerASync   = 0;
	gsCwq.ui32ServerSyncVA  = 0;
	gsCwq.ui32ServerSync    = 0;
	gsCwq.ui32ServerGF      = 0;
	gsCwq.ui32ServerDGF     = 0;
	gsCwq.ui32ServerDTL     = 0;
	gsCwq.i32StatsExecWriteIdx = 0;
	OSCachedMemSet(gsCwq.asStatsExecuted, 0, sizeof(gsCwq.asStatsExecuted));
#endif
}

static void CacheOpConfigUpdate(IMG_UINT32 ui32Config)
{
	OSLockAcquire(gsCwq.hConfigLock);

	/* Step 0, set the gsCwq.eConfig bits */
	if (!(ui32Config & (CACHEOP_CONFIG_LAST - 1)))
	{
		gsCwq.eConfig = CACHEOP_CONFIG_KRBF | CACHEOP_CONFIG_KDF;
		if (! gsCwq.bNoGlobalFlushImpl)
		{
			gsCwq.eConfig |= CACHEOP_CONFIG_KGF;
		}
		if (gsCwq.bSupportsUMFlush)
		{
			gsCwq.eConfig |= CACHEOP_CONFIG_URBF;
		}
	}
	else
	{
		if (ui32Config & CACHEOP_CONFIG_KRBF)
		{
			gsCwq.eConfig |= CACHEOP_CONFIG_KRBF;
		}
		else
		{
			gsCwq.eConfig &= ~CACHEOP_CONFIG_KRBF;
		}

		if (ui32Config & CACHEOP_CONFIG_KDF)
		{
			gsCwq.eConfig |= CACHEOP_CONFIG_KDF;
		}
		else
		{
			gsCwq.eConfig &= ~CACHEOP_CONFIG_KDF;
		}

		if (!gsCwq.bNoGlobalFlushImpl && (ui32Config & CACHEOP_CONFIG_KGF))
		{
			gsCwq.eConfig |= CACHEOP_CONFIG_KGF;
		}
		else
		{
			gsCwq.eConfig &= ~CACHEOP_CONFIG_KGF;
		}

		if (gsCwq.bSupportsUMFlush && (ui32Config & CACHEOP_CONFIG_URBF))
		{
			gsCwq.eConfig |= CACHEOP_CONFIG_URBF;
		}
		else
		{
			gsCwq.eConfig &= ~CACHEOP_CONFIG_URBF;
		}
	}

	if (ui32Config & CACHEOP_CONFIG_KLOG)
	{
		/* Suppress logs from KM caller */
		gsCwq.eConfig |= CACHEOP_CONFIG_KLOG;
	}
	else
	{
		gsCwq.eConfig &= ~CACHEOP_CONFIG_KLOG;
	}

	/* Step 1, set gsCwq.ui32Config based on gsCwq.eConfig */
	ui32Config = 0;
	if (gsCwq.eConfig & CACHEOP_CONFIG_KRBF)
	{
		ui32Config |= CACHEOP_CONFIG_KRBF;
	}
	if (gsCwq.eConfig & CACHEOP_CONFIG_KDF)
	{
		ui32Config |= CACHEOP_CONFIG_KDF;
	}
	if (gsCwq.eConfig & CACHEOP_CONFIG_KGF)
	{
		ui32Config |= CACHEOP_CONFIG_KGF;
	}
	if (gsCwq.eConfig & CACHEOP_CONFIG_URBF)
	{
		ui32Config |= CACHEOP_CONFIG_URBF;
	}
	if (gsCwq.eConfig & CACHEOP_CONFIG_KLOG)
	{
		ui32Config |= CACHEOP_CONFIG_KLOG;
	}
	gsCwq.ui32Config = ui32Config;

	/* Step 2, Bar RBF promotion to GF, unless a GF is implemented */
	gsCwq.pui32InfoPage[CACHEOP_INFO_KMGFTHRESHLD] = (IMG_UINT32)~0;
	if (! gsCwq.bNoGlobalFlushImpl)
	{
		gsCwq.pui32InfoPage[CACHEOP_INFO_KMGFTHRESHLD] = (IMG_UINT32)PVR_DIRTY_BYTES_FLUSH_THRESHOLD;
	}

	/* Step 3, in certain cases where a CacheOp/VA is provided, this threshold determines at what point
	   the optimisation due to the presence of said VA (i.e. us not having to remap the PMR pages in KM)
	   is clawed-back because of the overhead of maintaining such large request which might stalls the
	   user thread; so to hide this latency have these CacheOps executed on deferred CacheOp thread */
	gsCwq.pui32InfoPage[CACHEOP_INFO_KMDFTHRESHLD] = (IMG_UINT32)(PVR_DIRTY_BYTES_FLUSH_THRESHOLD >> 2);

	/* Step 4, if no UM support, all requests are done in KM so zero these forcing all client requests
	   to come down into the KM for maintenance */
	gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD] = 0;
	gsCwq.pui32InfoPage[CACHEOP_INFO_UMRBFONLY] = 0;
	if (gsCwq.bSupportsUMFlush)
	{
		/* If URBF has been selected exclusively OR selected but there is no GF implementation */
		if ((gsCwq.eConfig & CACHEOP_CONFIG_URBF) &&
			(gsCwq.bNoGlobalFlushImpl || !((gsCwq.ui32Config & (CACHEOP_CONFIG_LAST-1)) & ~CACHEOP_CONFIG_URBF)))
		{
			/* If only URBF has been selected, simulate without GF support OR no GF support means all client
			   requests should be done in UM. In both cases, set this threshold to the highest value to
			   prevent any client requests coming down to the server for maintenance. */
			gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD] = (IMG_UINT32)~0;
			gsCwq.pui32InfoPage[CACHEOP_INFO_UMRBFONLY] = 1;
		}
		/* This is the default entry for setting the UM info. page entries */
		else if ((gsCwq.eConfig & CACHEOP_CONFIG_URBF) && !gsCwq.bNoGlobalFlushImpl)
		{
			/* Set UM/KM threshold, all request sizes above this goes to server for GF maintenance _only_
			   because UM flushes already have VA acquired, no cost is incurred in per-page (re)mapping
			   of the to-be maintained PMR/page(s) as it the case with KM flushing so disallow KDF */

			/* Assume an average UM flush performance, anything above should be promoted to GF.
			   For x86 UMA/LMA, we avoid KDF because remapping PMR/pages in KM might fail due to exhausted
			   or fragmented VMALLOC kernel VA space */
			gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD] = gsCwq.pui32InfoPage[CACHEOP_INFO_KMGFTHRESHLD];
		}
	}

	/* Step 5, reset stats. */
	CacheOpStatsReset();

	OSLockRelease(gsCwq.hConfigLock);
}

static INLINE void CacheOpConfigRead(void *pvFilePtr,
									 void *pvData,
									 OS_STATS_PRINTF_FUNC* pfnOSStatsPrintf)
{
	PVR_UNREFERENCED_PARAMETER(pvData);
	pfnOSStatsPrintf(pvFilePtr,
			"KDF: %s, URBF: %s, KGF: %s, KRBF: %s\n",
			gsCwq.eConfig & CACHEOP_CONFIG_KDF  ? "Yes" : "No",
			gsCwq.eConfig & CACHEOP_CONFIG_URBF ? "Yes" : "No",
			gsCwq.eConfig & CACHEOP_CONFIG_KGF  ? "Yes" : "No",
			gsCwq.eConfig & CACHEOP_CONFIG_KRBF ? "Yes" : "No"
		);
}

static INLINE PVRSRV_ERROR CacheOpConfigQuery(const PVRSRV_DEVICE_NODE *psDevNode,
											const void *psPrivate,
											IMG_UINT32 *pui32Value)
{
	IMG_UINT32 ui32ID = (IMG_UINT32)(uintptr_t) psPrivate;
	PVR_UNREFERENCED_PARAMETER(psDevNode);

	switch (ui32ID)
	{
		case APPHINT_ID_CacheOpConfig:
			*pui32Value = gsCwq.ui32Config;
			break;

		case APPHINT_ID_CacheOpGFThresholdSize:
			*pui32Value = gsCwq.pui32InfoPage[CACHEOP_INFO_KMGFTHRESHLD];
			break;

		case APPHINT_ID_CacheOpUMKMThresholdSize:
			*pui32Value = gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD];
			break;

		default:
			break;
	}

	return PVRSRV_OK;
}

static INLINE PVRSRV_ERROR CacheOpConfigSet(const PVRSRV_DEVICE_NODE *psDevNode,
											const void *psPrivate,
											IMG_UINT32 ui32Value)
{
	IMG_UINT32 ui32ID = (IMG_UINT32)(uintptr_t) psPrivate;
	PVR_UNREFERENCED_PARAMETER(psDevNode);

	switch (ui32ID)
	{
		case APPHINT_ID_CacheOpConfig:
			CacheOpConfigUpdate(ui32Value & CACHEOP_CONFIG_ALL);
			break;

		case APPHINT_ID_CacheOpGFThresholdSize:
		{
			if (!ui32Value || gsCwq.bNoGlobalFlushImpl)
			{
				/* CPU ISA does not support GF, silently ignore request to adjust threshold */
				PVR_ASSERT(gsCwq.pui32InfoPage[CACHEOP_INFO_KMGFTHRESHLD] == (IMG_UINT32)~0);
				break;
			}
			else if (ui32Value < gsCwq.uiPageSize)
			{
				/* Silently round-up to OS page size */
				ui32Value = gsCwq.uiPageSize;
			}

			/* Align to OS page size */
			ui32Value &= ~(gsCwq.uiPageSize - 1);

			/* Adjust KM deferred threshold given this updated KM global threshold */
			if (ui32Value == gsCwq.uiPageSize || ui32Value < gsCwq.pui32InfoPage[CACHEOP_INFO_KMDFTHRESHLD])
			{
				gsCwq.pui32InfoPage[CACHEOP_INFO_KMDFTHRESHLD] = ui32Value >> 2;
			}

			gsCwq.pui32InfoPage[CACHEOP_INFO_KMGFTHRESHLD] = ui32Value;

			break;
		}

		case APPHINT_ID_CacheOpUMKMThresholdSize:
		{
			if (!ui32Value || !gsCwq.bSupportsUMFlush)
			{
				/* CPU ISA does not support UM flush, therefore every request goes down into
				   the KM, silently ignore request to adjust threshold */
				PVR_ASSERT(! gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD]);
				break;
			}
			else if (ui32Value < gsCwq.uiPageSize)
			{
				/* Silently round-up to OS page size */
				ui32Value = gsCwq.uiPageSize;
			}

			/* Align to OS page size */
			ui32Value &= ~(gsCwq.uiPageSize - 1);

			if (gsCwq.bNoGlobalFlushImpl || ui32Value < gsCwq.pui32InfoPage[CACHEOP_INFO_KMDFTHRESHLD])
			{
				/* CPU ISA does not support GF also, therefore it seems there is no benefit to
				   re-routing this to KM as request won't be promoted to GF but request can
				   benefit from KM async. execution so ensure KM deferred threshold applies */
				PVR_ASSERT(gsCwq.pui32InfoPage[CACHEOP_INFO_KMGFTHRESHLD] == (IMG_UINT32)~0);
				gsCwq.pui32InfoPage[CACHEOP_INFO_KMDFTHRESHLD] = ui32Value >> 1;
				break;
			}

			gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD] = ui32Value;

			break;
		}

		default:
			break;
	}

	return PVRSRV_OK;
}

static INLINE void CacheOpQItemRecycle(CACHEOP_WORK_ITEM *psCacheOpWorkItem)
{
	PVRSRV_ERROR eError;
	eError = PMRUnlockSysPhysAddresses(psCacheOpWorkItem->psPMR);
	PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
	/* Set to max as precaution should recycling this CacheOp index fail
	   to reset it, this is purely to safe-guard having to discard such
	   subsequent deferred CacheOps or signal the sw sync timeline */
	psCacheOpWorkItem->iTimeline = PVRSRV_NO_TIMELINE;
	psCacheOpWorkItem->ui32GFSeqNum = (IMG_UINT32)~0;
	psCacheOpWorkItem->ui32OpSeqNum = (IMG_UINT32)~0;
#if defined(CACHEOP_DEBUG)
	psCacheOpWorkItem->psPMR = (void *)(uintptr_t)~0;
#endif
}

static INLINE void CacheOpQItemReadCheck(CACHEOP_WORK_ITEM *psCacheOpWorkItem)
{
#if defined(CACHEOP_DEBUG)
	CACHEOP_PVR_ASSERT(psCacheOpWorkItem->psPMR);
	CACHEOP_PVR_ASSERT(psCacheOpWorkItem->psPMR != (void *)(uintptr_t)~0);
	CACHEOP_PVR_ASSERT(psCacheOpWorkItem->ui32OpSeqNum != (IMG_UINT32)~0);
	if (CacheOpConfigSupports(CACHEOP_CONFIG_KGF))
	{
		CACHEOP_PVR_ASSERT(psCacheOpWorkItem->ui32GFSeqNum != (IMG_UINT32)~0);
	}
#else
	PVR_UNREFERENCED_PARAMETER(psCacheOpWorkItem);
#endif
}

static INLINE void CacheOpQItemWriteCheck(CACHEOP_WORK_ITEM *psCacheOpWorkItem)
{
#if defined(CACHEOP_DEBUG)
	CACHEOP_PVR_ASSERT(psCacheOpWorkItem->psPMR == (void *)(uintptr_t)~0);
	CACHEOP_PVR_ASSERT(psCacheOpWorkItem->ui32OpSeqNum == (IMG_UINT32)~0);
	CACHEOP_PVR_ASSERT(psCacheOpWorkItem->ui32GFSeqNum == (IMG_UINT32)~0);
	CACHEOP_PVR_ASSERT(psCacheOpWorkItem->iTimeline == PVRSRV_NO_TIMELINE);
#else
	PVR_UNREFERENCED_PARAMETER(psCacheOpWorkItem);
#endif
}

static INLINE IMG_UINT32 CacheOpGetNextCommonSeqNum(void)
{
	IMG_UINT32 ui32SeqNum = OSAtomicIncrement(&gsCwq.hCommonSeqNum);
	if (! ui32SeqNum)
	{
		/* Zero is _not_ a valid sequence value, doing so simplifies _all_
		   subsequent fence checking when no cache maintenance operation
		   is outstanding as in this case a fence value of zero is supplied. */
		if (CacheOpConfigSupports(CACHEOP_CONFIG_KGF))
		{
			/* Also when seqNum wraps around/crosses zero, this requires us to
			   ensure that GFSEQNUM is not erroneously higher than any/all client
			   seqNum(s) in the system during this wrap-around transition so we
			   disable both momentarily until the next GF comes along. This has
			   the effect that all subsequent in-flight discards using ">" is
			   never true seeing zero is _not_ greater than anything and all
			   "<=" comparison are always true seeing zero is always less than
			   all non-zero integers. The additional GF here is done mostly to
			   account for race condition(s) during this transition for all
			   pending seqNum(s) that are still behind zero. */
			gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0] = 0;
			gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM1] = 0;
			ui32SeqNum = OSAtomicIncrement(&gsCwq.hCommonSeqNum);
			(void) OSCPUOperation(PVRSRV_CACHE_OP_FLUSH);
		}
		else
		{
			ui32SeqNum = OSAtomicIncrement(&gsCwq.hCommonSeqNum);
		}
	}
	return ui32SeqNum;
}

static INLINE IMG_BOOL CacheOpFenceCheck(IMG_UINT32 ui32CompletedSeqNum,
										 IMG_UINT32 ui32FenceSeqNum)
{
	IMG_UINT32 ui32RebasedCompletedNum;
	IMG_UINT32 ui32RebasedFenceNum;
	IMG_UINT32 ui32Rebase;

	if (ui32FenceSeqNum == 0)
	{
		return IMG_TRUE;
	}

	/*
	   The problem statement is how to compare two values on
	   a numerical sequentially incrementing timeline in the
	   presence of wrap around arithmetic semantics using a
	   single ui32 counter & atomic (increment) operations.

	   The rationale for the solution here is to rebase the
	   incoming values to the sequence midpoint and perform
	   comparisons there; this allows us to handle overflow
	   or underflow wrap-round using only a single integer.

	   NOTE: Here we assume that the absolute value of the
	   difference between the two incoming values in _not_
	   greater than CACHEOP_SEQ_MIDPOINT. This assumption
	   holds as it implies that it is very _unlikely_ that 2
	   billion CacheOp requests could have been made between
	   a single client's CacheOp request & the corresponding
	   fence check. This code sequence is hopefully a _more_
	   hand optimised (branchless) version of this:

		   x = ui32CompletedOpSeqNum
		   y = ui32FenceOpSeqNum

		   if (|x - y| < CACHEOP_SEQ_MIDPOINT)
			   return (x - y) >= 0 ? true : false
		   else
			   return (y - x) >= 0 ? true : false
	 */
	ui32Rebase = CACHEOP_SEQ_MIDPOINT - ui32CompletedSeqNum;

	/* ui32Rebase could be either positive/negative, in
	   any case we still perform operation using unsigned
	   semantics as 2's complement notation always means
	   we end up with the correct result */
	ui32RebasedCompletedNum = ui32Rebase + ui32CompletedSeqNum;
	ui32RebasedFenceNum = ui32Rebase + ui32FenceSeqNum;

	return (ui32RebasedCompletedNum >= ui32RebasedFenceNum);
}

static INLINE PVRSRV_ERROR CacheOpTimelineBind(PVRSRV_DEVICE_NODE *psDevNode,
											   CACHEOP_WORK_ITEM *psCacheOpWorkItem,
											   PVRSRV_TIMELINE iTimeline)
{
	PVRSRV_ERROR eError;

	/* Always default the incoming CacheOp work-item to safe values */
	SyncClearTimelineObj(&psCacheOpWorkItem->sSWTimelineObj);
	psCacheOpWorkItem->iTimeline = PVRSRV_NO_TIMELINE;
	psCacheOpWorkItem->psDevNode = psDevNode;
	if (iTimeline == PVRSRV_NO_TIMELINE)
	{
		return PVRSRV_OK;
	}

#if defined(PVR_USE_FENCE_SYNC_MODEL)
	psCacheOpWorkItem->iTimeline = iTimeline;
	eError = SyncSWGetTimelineObj(iTimeline, &psCacheOpWorkItem->sSWTimelineObj);
	PVR_LOG_IF_ERROR(eError, "SyncSWGetTimelineObj");
#else
	eError = PVRSRV_ERROR_NOT_IMPLEMENTED;
#endif

	return eError;
}

static INLINE PVRSRV_ERROR CacheOpTimelineExec(CACHEOP_WORK_ITEM *psCacheOpWorkItem)
{
	PVRSRV_ERROR eError;

	if (psCacheOpWorkItem->iTimeline == PVRSRV_NO_TIMELINE)
	{
		return PVRSRV_OK;
	}
	CACHEOP_PVR_ASSERT(psCacheOpWorkItem->sSWTimelineObj.pvTlObj);

#if defined(PVR_USE_FENCE_SYNC_MODEL)
	eError = SyncSWTimelineAdvanceKM(psCacheOpWorkItem->psDevNode,
	                                 &psCacheOpWorkItem->sSWTimelineObj);
	(void) SyncSWTimelineReleaseKM(&psCacheOpWorkItem->sSWTimelineObj);
#else
	eError = PVRSRV_ERROR_NOT_IMPLEMENTED;
#endif

	return eError;
}

static INLINE PVRSRV_ERROR CacheOpGlobalFlush(void)
{
#if !defined(CACHEFLUSH_ISA_SUPPORTS_GLOBAL_FLUSH)
	return PVRSRV_ERROR_NOT_IMPLEMENTED;
#else
	PVRSRV_ERROR eError;
	IMG_UINT32 ui32OpSeqNum = gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0];

	if (! CacheOpConfigSupports(CACHEOP_CONFIG_KGF))
	{
		return PVRSRV_ERROR_NOT_SUPPORTED;
	}

	OSLockAcquire(gsCwq.hGlobalFlushLock);
	if (ui32OpSeqNum < gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0])
	{
#if defined(CACHEOP_DEBUG)
		gsCwq.ui32KMDiscards += 1;
#endif
		eError = PVRSRV_OK;
		goto exit;
	}

	/* User space sampling the information-page seqNumbers after this point
	   and before the corresponding GFSEQNUM0 update leads to an invalid
	   sampling which must be discarded by UM. This implements a lockless
	   critical region for a single KM(writer) & multiple UM/KM(readers) */
	ui32OpSeqNum = CacheOpGetNextCommonSeqNum();
	gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM1] = ui32OpSeqNum;

	eError = OSCPUOperation(PVRSRV_CACHE_OP_FLUSH);
	PVR_LOGG_IF_ERROR(eError, "OSCPUOperation(PVRSRV_CACHE_OP_FLUSH)", exit);

	gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0] = ui32OpSeqNum;
	OSAtomicWrite(&gsCwq.hDeferredSize, 0);
#if defined(CACHEOP_DEBUG)
	gsCwq.ui32ServerGF += 1;
#endif

exit:
	OSLockRelease(gsCwq.hGlobalFlushLock);
	return eError;
#endif
}

static INLINE void CacheOpExecRangeBased(PVRSRV_DEVICE_NODE *psDevNode,
										PVRSRV_CACHE_OP uiCacheOp,
										IMG_BYTE *pbCpuVirtAddr,
										IMG_CPU_PHYADDR sCpuPhyAddr,
										IMG_DEVMEM_OFFSET_T uiPgAlignedOffset,
										IMG_DEVMEM_OFFSET_T uiCLAlignedStartOffset,
										IMG_DEVMEM_OFFSET_T uiCLAlignedEndOffset)
{
	IMG_BYTE *pbCpuVirtAddrEnd;
	IMG_BYTE *pbCpuVirtAddrStart;
	IMG_CPU_PHYADDR sCpuPhyAddrEnd;
	IMG_CPU_PHYADDR sCpuPhyAddrStart;
	IMG_DEVMEM_SIZE_T uiRelFlushSize;
	IMG_DEVMEM_OFFSET_T uiRelFlushOffset;
	IMG_DEVMEM_SIZE_T uiNextPgAlignedOffset;

	/* These quantities allows us to perform cache operations
	   at cache-line granularity thereby ensuring we do not
	   perform more than is necessary */
	CACHEOP_PVR_ASSERT(uiPgAlignedOffset < uiCLAlignedEndOffset);
	uiRelFlushSize = (IMG_DEVMEM_SIZE_T)gsCwq.uiPageSize;
	uiRelFlushOffset = 0;

	if (uiCLAlignedStartOffset > uiPgAlignedOffset)
	{
		/* Zero unless initially starting at an in-page offset */
		uiRelFlushOffset = uiCLAlignedStartOffset - uiPgAlignedOffset;
		uiRelFlushSize -= uiRelFlushOffset;
	}

	/* uiRelFlushSize is gsCwq.uiPageSize unless current outstanding CacheOp
	   size is smaller. The 1st case handles in-page CacheOp range and
	   the 2nd case handles multiple-page CacheOp range with a last
	   CacheOp size that is less than gsCwq.uiPageSize */
	uiNextPgAlignedOffset = uiPgAlignedOffset + (IMG_DEVMEM_SIZE_T)gsCwq.uiPageSize;
	if (uiNextPgAlignedOffset < uiPgAlignedOffset)
	{
		/* uiNextPgAlignedOffset is greater than uiCLAlignedEndOffset
		   by implication of this wrap-round; this only happens when
		   uiPgAlignedOffset is the last page aligned offset */
		uiRelFlushSize = uiRelFlushOffset ?
				uiCLAlignedEndOffset - uiCLAlignedStartOffset :
				uiCLAlignedEndOffset - uiPgAlignedOffset;
	}
	else
	{
		if (uiNextPgAlignedOffset > uiCLAlignedEndOffset)
		{
			uiRelFlushSize = uiRelFlushOffset ?
					uiCLAlignedEndOffset - uiCLAlignedStartOffset :
					uiCLAlignedEndOffset - uiPgAlignedOffset;
		}
	}

	/* More efficient to request cache maintenance operation for full
	   relative range as opposed to multiple cache-aligned ranges */
	sCpuPhyAddrStart.uiAddr = sCpuPhyAddr.uiAddr + uiRelFlushOffset;
	sCpuPhyAddrEnd.uiAddr = sCpuPhyAddrStart.uiAddr + uiRelFlushSize;
	if (pbCpuVirtAddr)
	{
		pbCpuVirtAddrStart = pbCpuVirtAddr + uiRelFlushOffset;
		pbCpuVirtAddrEnd = pbCpuVirtAddrStart + uiRelFlushSize;
	}
	else
	{
		/* Some OS/Env layer support functions expect NULL(s) */
		pbCpuVirtAddrStart = NULL;
		pbCpuVirtAddrEnd = NULL;
	}

	/* Perform requested CacheOp on the CPU data cache for successive cache
	   line worth of bytes up to page or in-page cache-line boundary */
	switch (uiCacheOp)
	{
		case PVRSRV_CACHE_OP_CLEAN:
			OSCPUCacheCleanRangeKM(psDevNode, pbCpuVirtAddrStart, pbCpuVirtAddrEnd,
									sCpuPhyAddrStart, sCpuPhyAddrEnd);
			break;
		case PVRSRV_CACHE_OP_INVALIDATE:
			OSCPUCacheInvalidateRangeKM(psDevNode, pbCpuVirtAddrStart, pbCpuVirtAddrEnd,
									sCpuPhyAddrStart, sCpuPhyAddrEnd);
			break;
		case PVRSRV_CACHE_OP_FLUSH:
			OSCPUCacheFlushRangeKM(psDevNode, pbCpuVirtAddrStart, pbCpuVirtAddrEnd,
									sCpuPhyAddrStart, sCpuPhyAddrEnd);
			break;
		default:
			PVR_DPF((PVR_DBG_ERROR,	"%s: Invalid cache operation type %d",
					__func__, uiCacheOp));
			break;
	}

#if defined(CACHEOP_DEBUG)
	/* Tracks the number of kernel-mode cacheline maintenance instructions */
	gsCwq.ui32ServerRBF += (uiRelFlushSize & ((IMG_DEVMEM_SIZE_T)~(gsCwq.uiLineSize - 1))) >> gsCwq.uiLineShift;
#endif
}

static INLINE void CacheOpExecRangeBasedVA(PVRSRV_DEVICE_NODE *psDevNode,
										 IMG_CPU_VIRTADDR pvAddress,
										 IMG_DEVMEM_SIZE_T uiSize,
										 PVRSRV_CACHE_OP uiCacheOp)
{
	IMG_CPU_PHYADDR sCpuPhyAddrUnused =
		{ IMG_CAST_TO_CPUPHYADDR_UINT(0xCAFEF00DDEADBEEFULL) };
	IMG_BYTE *pbEnd = (IMG_BYTE*)((uintptr_t)pvAddress + (uintptr_t)uiSize);
	IMG_BYTE *pbStart = (IMG_BYTE*)((uintptr_t)pvAddress & ~((uintptr_t)gsCwq.uiLineSize-1));

	/*
	  If the start/end address isn't aligned to cache line size, round it up to the
	  nearest multiple; this ensures that we flush all the cache lines affected by
	  unaligned start/end addresses.
	 */
	pbEnd = (IMG_BYTE *) PVR_ALIGN((uintptr_t)pbEnd, (uintptr_t)gsCwq.uiLineSize);
	switch (uiCacheOp)
	{
		case PVRSRV_CACHE_OP_CLEAN:
			OSCPUCacheCleanRangeKM(psDevNode, pbStart, pbEnd, sCpuPhyAddrUnused, sCpuPhyAddrUnused);
			break;
		case PVRSRV_CACHE_OP_INVALIDATE:
			OSCPUCacheInvalidateRangeKM(psDevNode, pbStart, pbEnd, sCpuPhyAddrUnused, sCpuPhyAddrUnused);
			break;
		case PVRSRV_CACHE_OP_FLUSH:
			OSCPUCacheFlushRangeKM(psDevNode, pbStart, pbEnd, sCpuPhyAddrUnused, sCpuPhyAddrUnused);
			break;
		default:
			PVR_DPF((PVR_DBG_ERROR,	"%s: Invalid cache operation type %d",
					 __func__, uiCacheOp));
			break;
	}

#if defined(CACHEOP_DEBUG)
	/* Tracks the number of kernel-mode cacheline maintenance instructions */
	gsCwq.ui32ServerRBF += (uiSize & ((IMG_DEVMEM_SIZE_T)~(gsCwq.uiLineSize - 1))) >> gsCwq.uiLineShift;
#endif
}

static INLINE PVRSRV_ERROR CacheOpValidateVAOffset(PMR *psPMR,
												IMG_CPU_VIRTADDR pvAddress,
												IMG_DEVMEM_OFFSET_T uiOffset,
												IMG_DEVMEM_SIZE_T uiSize,
												void **ppvOutAddress)
{
	PVRSRV_ERROR eError = PVRSRV_OK;
#if defined(LINUX) && !defined(CACHEFLUSH_NO_KMRBF_USING_UMVA)
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;
#endif
	void __user *pvAddr;

	if (! pvAddress)
	{
		/* As pvAddress is optional, NULL is expected from UM/KM requests */
		pvAddr = NULL;
		goto e0;
	}

#if !defined(LINUX) || defined(CACHEFLUSH_NO_KMRBF_USING_UMVA)
	pvAddr = NULL;
#else
	/* Validate VA, assume most basic address limit access_ok() check */
	pvAddr = (void __user *)(uintptr_t)((uintptr_t)pvAddress + uiOffset);
	if (!access_ok(pvAddr, uiSize))
	{
		pvAddr = NULL;
		if (! mm)
		{
			/* Bad KM request, don't silently ignore */
			eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
			goto e0;
		}
	}
	else if (mm)
	{
		down_read(&mm->mmap_sem);

		vma = find_vma(mm, (unsigned long)(uintptr_t)pvAddr);
		if (!vma ||
			vma->vm_start > (unsigned long)(uintptr_t)pvAddr ||
			vma->vm_end - vma->vm_start > (unsigned long)(uintptr_t)uiSize)
		{
			/* Out of range mm_struct->vm_area VA */
			pvAddr = NULL;
		}
		else if (vma->vm_private_data != psPMR)
		{
			/*
			   Unknown mm_struct->vm_area VA, can't risk dcache maintenance using
			   this VA as the client user space mapping could be removed without
			   us knowing which might induce CPU fault during cache maintenance.
			*/
			pvAddr = NULL;
		}
		else if ((uintptr_t)pvAddress < (uintptr_t)gsCwq.uiPageSize)
		{
			/* Silently suppress UM NULL page pointers */
			pvAddr = NULL;
		}

		up_read(&mm->mmap_sem);
	}
	else
	{
		pgd_t *pgd;
		p4d_t *p4d;
		pud_t *pud;
		pmd_t *pmd;
		pte_t *ptep;
		mm = current->active_mm;

		/*
			For KM requests perform additional VA validation, so we walk the
			kernel page-table structures to be sure VA is safe to use.
		*/
		pgd = pgd_offset(mm, (uintptr_t)pvAddr);
		if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
		{
			eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
			pvAddr = NULL;
			goto e0;
		}

		p4d = p4d_offset(pgd, (uintptr_t)pvAddr);
		if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
		{
			eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
			pvAddr = NULL;
			goto e0;
		}

		pud = pud_offset(p4d, (uintptr_t)pvAddr);
		if (pud_none(*pud) || unlikely(pud_bad(*pud)))
		{
			eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
			pvAddr = NULL;
			goto e0;
		}

		pmd = pmd_offset(pud, (uintptr_t)pvAddr);
		if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
		{
			eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
			pvAddr = NULL;
			goto e0;
		}

		ptep = pte_offset_map(pmd, (uintptr_t)pvAddr);
		if (!ptep || !pte_present(*ptep))
		{
			eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
			pvAddr = NULL;
			goto e0;
		}
	}
#endif

e0:
	*ppvOutAddress = (IMG_CPU_VIRTADDR __force) pvAddr;
	return eError;
}

static PVRSRV_ERROR CacheOpPMRExec (PMR *psPMR,
									IMG_CPU_VIRTADDR pvAddress,
									IMG_DEVMEM_OFFSET_T uiOffset,
									IMG_DEVMEM_SIZE_T uiSize,
									PVRSRV_CACHE_OP uiCacheOp,
									IMG_UINT32 ui32GFlushSeqNum,
									IMG_BOOL bIsRequestValidated,
									IMG_BOOL *pbUsedGlobalFlush)
{
	IMG_HANDLE hPrivOut;
	IMG_BOOL bPMRIsSparse;
	IMG_UINT32 ui32PageIndex;
	IMG_UINT32 ui32NumOfPages;
	IMG_DEVMEM_SIZE_T uiOutSize;
	PVRSRV_DEVICE_NODE *psDevNode;
	IMG_DEVMEM_SIZE_T uiPgAlignedSize;
	IMG_DEVMEM_OFFSET_T uiPgAlignedOffset;
	IMG_DEVMEM_OFFSET_T uiCLAlignedEndOffset;
	IMG_DEVMEM_OFFSET_T uiPgAlignedEndOffset;
	IMG_DEVMEM_OFFSET_T uiCLAlignedStartOffset;
	IMG_DEVMEM_OFFSET_T uiPgAlignedStartOffset;
	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
	IMG_CPU_PHYADDR asCpuPhyAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
	IMG_CPU_PHYADDR *psCpuPhyAddr = asCpuPhyAddr;
	IMG_BOOL bIsPMRInfoValid = IMG_FALSE;
	PVRSRV_ERROR eError = PVRSRV_OK;
	IMG_BYTE *pbCpuVirtAddr = NULL;
	IMG_BOOL *pbValid = abValid;
	*pbUsedGlobalFlush = IMG_FALSE;

	if (uiCacheOp == PVRSRV_CACHE_OP_NONE || uiCacheOp == PVRSRV_CACHE_OP_TIMELINE)
	{
		return PVRSRV_OK;
	}

	/* Check for explicitly requested-for KGF or KRBF promoted to KGF requests */
	if (uiCacheOp == PVRSRV_CACHE_OP_GLOBAL || uiSize == 0 ||
	    (IMG_UINT32)uiSize > gsCwq.pui32InfoPage[CACHEOP_INFO_KMGFTHRESHLD])
	{
		/* Discard if an else-when KGF has occurred in the interim time */
		if (gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0] > ui32GFlushSeqNum)
		{
#if defined(CACHEOP_DEBUG)
			gsCwq.ui32KMDiscards += 1;
#endif
			return PVRSRV_OK;
		}
		/* Some CPU ISA support KGF, if it fails fall-back to KRBF */
		else if ((eError = CacheOpGlobalFlush()) == PVRSRV_OK)
		{
			*pbUsedGlobalFlush = IMG_TRUE;
			return PVRSRV_OK;
		}
		/* Request with uiSize=0 is treated as a KGF request as well */
		else if (uiCacheOp == PVRSRV_CACHE_OP_GLOBAL || uiSize == 0)
		{
			/* Cannot fall-back to KRBF as an explicit KGF was erroneously requested for */
			PVR_LOGR_IF_ERROR(eError, CACHEOP_NO_GFLUSH_ERROR_STRING);
			CACHEOP_PVR_ASSERT(0);
		}
	}

	if (! bIsRequestValidated)
	{
		IMG_DEVMEM_SIZE_T uiLPhysicalSize;

		/* Need to validate parameters before proceeding */
		eError = PMR_PhysicalSize(psPMR, &uiLPhysicalSize);
		PVR_LOGR_IF_ERROR(eError, "uiLPhysicalSize");

		PVR_LOGR_IF_FALSE(((uiOffset+uiSize) <= uiLPhysicalSize), CACHEOP_DEVMEM_OOR_ERROR_STRING, PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE);

		eError = PMRLockSysPhysAddresses(psPMR);
		PVR_LOGR_IF_ERROR(eError, "PMRLockSysPhysAddresses");
	}

	/* Fast track the request if a CPU VA is provided and CPU ISA supports VA only maintenance */
	eError = CacheOpValidateVAOffset(psPMR, pvAddress, uiOffset, uiSize, (void**)&pbCpuVirtAddr);
	if (eError == PVRSRV_OK)
	{
		pvAddress = pbCpuVirtAddr;

		if (pvAddress && gsCwq.uiCacheOpAddrType == PVRSRV_CACHE_OP_ADDR_TYPE_VIRTUAL)
		{
			CacheOpExecRangeBasedVA(PMR_DeviceNode(psPMR), pvAddress, uiSize, uiCacheOp);
			if (! bIsRequestValidated)
			{
				eError = PMRUnlockSysPhysAddresses(psPMR);
				PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
			}
#if defined(CACHEOP_DEBUG)
			gsCwq.ui32ServerSyncVA += 1;
#endif
			return PVRSRV_OK;
		}
		else if (pvAddress)
		{
			/* Round down the incoming VA (if any) down to the nearest page aligned VA */
			 pvAddress = (void*)((uintptr_t)pvAddress & ~((uintptr_t)gsCwq.uiPageSize-1));
#if defined(CACHEOP_DEBUG)
			gsCwq.ui32ServerSyncVA += 1;
#endif
		}
	}
	else
	{
		/*
		 * This validation pathway has been added to accommodate any/all requests that might
		 * cause the kernel to Oops; essentially, KM requests should prevalidate cache maint.
		 * parameters but if this fails then we would rather fail gracefully than cause the
		 * kernel to Oops so instead we log the fact that an invalid KM virtual address was
		 * supplied and what action was taken to mitigate against kernel Oops(ing) if any.
		 */
		CACHEOP_PVR_ASSERT(pbCpuVirtAddr == NULL);

		if (gsCwq.uiCacheOpAddrType == PVRSRV_CACHE_OP_ADDR_TYPE_PHYSICAL)
		{
			PVR_DPF((PVR_DBG_WARNING,
					"%s: Invalid vaddress 0x%p in CPU d-cache maint. op, using paddress",
					__func__,
					pvAddress));

			/* We can still proceed as kernel/cpu uses CPU PA for d-cache maintenance */
			pvAddress = NULL;
		}
		else if (CacheOpGlobalFlush() == PVRSRV_OK)
		{
			PVR_DPF((PVR_DBG_WARNING,
					"%s: Invalid vaddress 0x%p in CPU d-cache maint. op, used global flush",
					__func__,
					pvAddress));

			/* Saved by global flush impl. */
			*pbUsedGlobalFlush = IMG_TRUE;
			eError = PVRSRV_OK;
			goto e0;
		}
		else
		{
			/*
			 * The approach here is to attempt a reacquisition of the PMR kernel VA and see if
			 * said VA corresponds to the parameter VA, if so fail requested cache maint. op.
			 * cause this indicates some kind of internal, memory and/or meta-data corruption
			 * else we reissue the request using this (re)acquired alias PMR kernel VA.
			 */
			if (PMR_IsSparse(psPMR))
			{
				eError = PMRAcquireSparseKernelMappingData(psPMR,
														   0,
														   gsCwq.uiPageSize,
														   (void **)&pbCpuVirtAddr,
														   (size_t*)&uiOutSize,
														   &hPrivOut);
				PVR_LOGG_IF_ERROR(eError, "PMRAcquireSparseKernelMappingData", e0);
			}
			else
			{
				eError = PMRAcquireKernelMappingData(psPMR,
													 0,
													 gsCwq.uiPageSize,
													 (void **)&pbCpuVirtAddr,
													 (size_t*)&uiOutSize,
													 &hPrivOut);
				PVR_LOGG_IF_ERROR(eError, "PMRAcquireKernelMappingData", e0);
			}

			/* Here, we only compare these CPU virtual addresses at granularity of the OS page size */
			if ((uintptr_t)pbCpuVirtAddr == ((uintptr_t)pvAddress & ~((uintptr_t)gsCwq.uiPageSize-1)))
			{
				PVR_DPF((PVR_DBG_ERROR,
						"%s: Invalid vaddress 0x%p in CPU d-cache maint. op, no alt. so failing request",
						__func__,
						pvAddress));

				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
				PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");

				eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
				goto e0;
			}
			else if (gsCwq.uiCacheOpAddrType == PVRSRV_CACHE_OP_ADDR_TYPE_VIRTUAL)
			{
				PVR_DPF((PVR_DBG_WARNING,
						"%s: Bad vaddress 0x%p in CPU d-cache maint. op, using reacquired vaddress 0x%p",
						__func__,
						pvAddress,
						pbCpuVirtAddr));

				/* Note that this might still fail if there is kernel memory/meta-data corruption;
				   there is not much we can do here but at the least we will be informed of this
				   before the kernel Oops(ing) */
				CacheOpExecRangeBasedVA(PMR_DeviceNode(psPMR), pbCpuVirtAddr, uiSize, uiCacheOp);

				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
				PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");

				eError = PVRSRV_OK;
				goto e0;
			}
			else
			{
				/* At this junction, we have exhausted every possible work-around possible but we do
				   know that VA reacquisition returned another/alias page-aligned VA; so with this
				   future expectation of PMRAcquireKernelMappingData(), we proceed */
				PVR_DPF((PVR_DBG_WARNING,
						"%s: Bad vaddress %p in CPU d-cache maint. op, will use reacquired vaddress",
						__func__,
						pvAddress));

				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
				PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");

				/* NULL this to force per-page reacquisition down-stream */
				pvAddress = NULL;
			}
		}
	}

	/* NULL clobbered var., OK to proceed */
	pbCpuVirtAddr = NULL;
	eError = PVRSRV_OK;

	/* Need this for kernel mapping */
	bPMRIsSparse = PMR_IsSparse(psPMR);
	psDevNode = PMR_DeviceNode(psPMR);

	/* Round the incoming offset down to the nearest cache-line / page aligned-address */
	uiCLAlignedEndOffset = uiOffset + uiSize;
	uiCLAlignedEndOffset = PVR_ALIGN(uiCLAlignedEndOffset, (IMG_DEVMEM_SIZE_T)gsCwq.uiLineSize);
	uiCLAlignedStartOffset = (uiOffset & ~((IMG_DEVMEM_OFFSET_T)gsCwq.uiLineSize-1));

	uiPgAlignedEndOffset = uiCLAlignedEndOffset;
	uiPgAlignedEndOffset = PVR_ALIGN(uiPgAlignedEndOffset, (IMG_DEVMEM_SIZE_T)gsCwq.uiPageSize);
	uiPgAlignedStartOffset = (uiOffset & ~((IMG_DEVMEM_OFFSET_T)gsCwq.uiPageSize-1));
	uiPgAlignedSize = uiPgAlignedEndOffset - uiPgAlignedStartOffset;

#if defined(CACHEOP_NO_CACHE_LINE_ALIGNED_ROUNDING)
	/* For internal debug if cache-line optimised
	   flushing is suspected of causing data corruption */
	uiCLAlignedStartOffset = uiPgAlignedStartOffset;
	uiCLAlignedEndOffset = uiPgAlignedEndOffset;
#endif

	/* Type of allocation backing the PMR data */
	ui32NumOfPages = uiPgAlignedSize >> gsCwq.uiPageShift;
	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
	{
		/* The pbValid array is allocated first as it is needed in
		   both physical/virtual cache maintenance methods */
		pbValid = OSAllocZMem(ui32NumOfPages * sizeof(IMG_BOOL));
		if (! pbValid)
		{
			pbValid = abValid;
		}
		else if (gsCwq.uiCacheOpAddrType != PVRSRV_CACHE_OP_ADDR_TYPE_VIRTUAL)
		{
			psCpuPhyAddr = OSAllocZMem(ui32NumOfPages * sizeof(IMG_CPU_PHYADDR));
			if (! psCpuPhyAddr)
			{
				psCpuPhyAddr = asCpuPhyAddr;
				OSFreeMem(pbValid);
				pbValid = abValid;
			}
		}
	}

	/* We always retrieve PMR data in bulk, up-front if number of pages is within
	   PMR_MAX_TRANSLATION_STACK_ALLOC limits else we check to ensure that a
	   dynamic buffer has been allocated to satisfy requests outside limits */
	if (ui32NumOfPages <= PMR_MAX_TRANSLATION_STACK_ALLOC || pbValid != abValid)
	{
		if (gsCwq.uiCacheOpAddrType != PVRSRV_CACHE_OP_ADDR_TYPE_VIRTUAL)
		{
			/* Look-up PMR CpuPhyAddr once, if possible */
			eError = PMR_CpuPhysAddr(psPMR,
									 gsCwq.uiPageShift,
									 ui32NumOfPages,
									 uiPgAlignedStartOffset,
									 psCpuPhyAddr,
									 pbValid);
			if (eError == PVRSRV_OK)
			{
				bIsPMRInfoValid = IMG_TRUE;
			}
		}
		else
		{
			/* Look-up PMR per-page validity once, if possible */
			eError = PMR_IsOffsetValid(psPMR,
									   gsCwq.uiPageShift,
									   ui32NumOfPages,
									   uiPgAlignedStartOffset,
									   pbValid);
			bIsPMRInfoValid = (eError == PVRSRV_OK) ? IMG_TRUE : IMG_FALSE;
		}
	}

	/* For each (possibly non-contiguous) PMR page(s), carry out the requested cache maint. op. */
	for (uiPgAlignedOffset = uiPgAlignedStartOffset, ui32PageIndex = 0;
		 uiPgAlignedOffset < uiPgAlignedEndOffset;
		 uiPgAlignedOffset += (IMG_DEVMEM_OFFSET_T) gsCwq.uiPageSize, ui32PageIndex += 1)
	{
		/* Just before issuing the CacheOp RBF, check if it can be discarded */
		if (gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0] > ui32GFlushSeqNum)
		{
#if defined(CACHEOP_DEBUG)
			gsCwq.ui32KMDiscards += 1;
#endif
			break;
		}

		if (! bIsPMRInfoValid)
		{
			/* Never cross page boundary without looking up corresponding PMR page physical
			   address and/or page validity if these were not looked-up, in bulk, up-front */
			ui32PageIndex = 0;
			if (gsCwq.uiCacheOpAddrType != PVRSRV_CACHE_OP_ADDR_TYPE_VIRTUAL)
			{
				eError = PMR_CpuPhysAddr(psPMR,
										 gsCwq.uiPageShift,
										 1,
										 uiPgAlignedOffset,
										 psCpuPhyAddr,
										 pbValid);
				PVR_LOGG_IF_ERROR(eError, "PMR_CpuPhysAddr", e0);
			}
			else
			{
				eError = PMR_IsOffsetValid(psPMR,
										  gsCwq.uiPageShift,
										  1,
										  uiPgAlignedOffset,
										  pbValid);
				PVR_LOGG_IF_ERROR(eError, "PMR_IsOffsetValid", e0);
			}
		}

		/* Skip invalid PMR pages (i.e. sparse) */
		if (pbValid[ui32PageIndex] == IMG_FALSE)
		{
			CACHEOP_PVR_ASSERT(bPMRIsSparse);
			continue;
		}

		if (pvAddress)
		{
			/* The caller has supplied either a KM/UM CpuVA, so use it unconditionally */
			pbCpuVirtAddr =
				(void *)(uintptr_t)((uintptr_t)pvAddress + (uintptr_t)(uiPgAlignedOffset-uiPgAlignedStartOffset));
		}
		/* Skip CpuVA acquire if CacheOp can be maintained entirely using CpuPA */
		else if (gsCwq.uiCacheOpAddrType != PVRSRV_CACHE_OP_ADDR_TYPE_PHYSICAL)
		{
			if (bPMRIsSparse)
			{
				eError =
					PMRAcquireSparseKernelMappingData(psPMR,
													  uiPgAlignedOffset,
													  gsCwq.uiPageSize,
													  (void **)&pbCpuVirtAddr,
													  (size_t*)&uiOutSize,
													  &hPrivOut);
				PVR_LOGG_IF_ERROR(eError, "PMRAcquireSparseKernelMappingData", e0);
			}
			else
			{
				eError =
					PMRAcquireKernelMappingData(psPMR,
												uiPgAlignedOffset,
												gsCwq.uiPageSize,
												(void **)&pbCpuVirtAddr,
												(size_t*)&uiOutSize,
												&hPrivOut);
				PVR_LOGG_IF_ERROR(eError, "PMRAcquireKernelMappingData", e0);
			}
		}

		/* Issue actual cache maintenance for PMR */
		CacheOpExecRangeBased(psDevNode,
							uiCacheOp,
							pbCpuVirtAddr,
							(gsCwq.uiCacheOpAddrType != PVRSRV_CACHE_OP_ADDR_TYPE_VIRTUAL) ?
								psCpuPhyAddr[ui32PageIndex] : psCpuPhyAddr[0],
							uiPgAlignedOffset,
							uiCLAlignedStartOffset,
							uiCLAlignedEndOffset);

		if (! pvAddress)
		{
			/* The caller has not supplied either a KM/UM CpuVA, release mapping */
			if (gsCwq.uiCacheOpAddrType != PVRSRV_CACHE_OP_ADDR_TYPE_PHYSICAL)
			{
				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
				PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
			}
		}
	}

e0:
	if (psCpuPhyAddr != asCpuPhyAddr)
	{
		OSFreeMem(psCpuPhyAddr);
	}

	if (pbValid != abValid)
	{
		OSFreeMem(pbValid);
	}

	if (! bIsRequestValidated)
	{
		eError = PMRUnlockSysPhysAddresses(psPMR);
		PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
	}

	return eError;
}

static PVRSRV_ERROR CacheOpQListExecGlobal(void)
{
	PVRSRV_ERROR eError = PVRSRV_OK;
	IMG_UINT32 ui32NumOfEntries;
	CACHEOP_WORK_ITEM *psCacheOpWorkItem;
#if defined(CACHEOP_DEBUG)
	IMG_UINT64 uiTimeNow = 0;
	IMG_UINT64 ui64DequeuedTime;
	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
#endif
	CACHEOP_PVR_ASSERT(!gsCwq.bNoGlobalFlushImpl);

	/* Take the current snapshot of queued CacheOps before we issue a global cache
	   flush operation so that we retire the right amount of CacheOps that has
	   been affected by the to-be executed global CacheOp */
	ui32NumOfEntries = CacheOpIdxSpan(&gsCwq.hWriteCounter, &gsCwq.hReadCounter);
	if (OSAtomicRead(&gsCwq.hWriteCounter) < OSAtomicRead(&gsCwq.hReadCounter))
	{
		/* Branch handles when the write-counter has wrapped-around in value space.
		   The logic here works seeing the read-counter does not change value for
		   the duration of this function so we don't run the risk of it too wrapping
		   round whilst the number of entries is being determined here, that is to
		   say, the consumer in this framework is single threaded and this function
		   is that consumer thread */
		ui32NumOfEntries = CacheOpIdxSpan(&gsCwq.hReadCounter, &gsCwq.hWriteCounter);

		/* Two's complement arithmetic gives the number of entries */
		ui32NumOfEntries = CACHEOP_INDICES_MAX - ui32NumOfEntries;
	}
	if (! ui32NumOfEntries)
	{
		return PVRSRV_OK;
	}
#if defined(CACHEOP_DEBUG)
	CACHEOP_PVR_ASSERT(ui32NumOfEntries < CACHEOP_INDICES_MAX);
#endif

	/* Use the current/latest queue-tail work-item's GF/SeqNum to predicate GF */
	psCacheOpWorkItem = &gsCwq.asWorkItems[CacheOpIdxRead(&gsCwq.hWriteCounter)];
	CacheOpQItemReadCheck(psCacheOpWorkItem);
#if defined(CACHEOP_DEBUG)
	/* The time waiting in the queue to be serviced */
	ui64DequeuedTime = OSClockns64();
#endif

	/* Check if items between [hRead/hWrite]Counter can be discarded before issuing GF */
	if (gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0] > psCacheOpWorkItem->ui32GFSeqNum)
	{
		/* The currently discarded CacheOp item updates gsCwq.hCompletedSeqNum */
		OSAtomicWrite(&gsCwq.hCompletedSeqNum, psCacheOpWorkItem->ui32OpSeqNum);
#if defined(CACHEOP_DEBUG)
		gsCwq.ui32KMDiscards += ui32NumOfEntries;
#endif
	}
	else
	{
		eError = CacheOpGlobalFlush();
		PVR_LOGR_IF_ERROR(eError, "CacheOpGlobalFlush");
#if defined(CACHEOP_DEBUG)
		uiTimeNow = OSClockns64();
		sCacheOpWorkItem.bDeferred = IMG_TRUE;
		sCacheOpWorkItem.ui64ExecuteTime = uiTimeNow;
		sCacheOpWorkItem.psPMR = gsCwq.psInfoPagePMR;
		sCacheOpWorkItem.pid = OSGetCurrentProcessID();
		sCacheOpWorkItem.uiCacheOp = PVRSRV_CACHE_OP_GLOBAL;
		sCacheOpWorkItem.ui64DequeuedTime = ui64DequeuedTime;
		sCacheOpWorkItem.ui64EnqueuedTime = psCacheOpWorkItem->ui64EnqueuedTime;
		sCacheOpWorkItem.ui32OpSeqNum = gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0];
#endif
	}

	while (ui32NumOfEntries)
	{
		psCacheOpWorkItem = &gsCwq.asWorkItems[CacheOpIdxNext(&gsCwq.hReadCounter)];
		CacheOpQItemReadCheck(psCacheOpWorkItem);

#if defined(CACHEOP_DEBUG)
		if (psCacheOpWorkItem->uiCacheOp != PVRSRV_CACHE_OP_GLOBAL)
		{
			psCacheOpWorkItem->bRBF = IMG_FALSE;
			if (! uiTimeNow)
			{
				/* Measure deferred queueing overhead only */
				uiTimeNow = OSClockns64();
				psCacheOpWorkItem->ui64ExecuteTime = uiTimeNow;
			}
			else
			{
				psCacheOpWorkItem->ui64ExecuteTime = uiTimeNow;
			}
			psCacheOpWorkItem->ui64DequeuedTime = ui64DequeuedTime;
			CacheOpStatsExecLogWrite(psCacheOpWorkItem);
		}
		/* Something's gone horribly wrong if these 2 counters are identical at this point */
		CACHEOP_PVR_ASSERT(OSAtomicRead(&gsCwq.hWriteCounter) != OSAtomicRead(&gsCwq.hReadCounter));
#endif

		/* If CacheOp is timeline(d), notify timeline waiters */
		eError = CacheOpTimelineExec(psCacheOpWorkItem);
		PVR_LOG_IF_ERROR(eError, "CacheOpTimelineExec");

		/* Mark index as ready for recycling for next CacheOp */
		CacheOpQItemRecycle(psCacheOpWorkItem);
		(void) CacheOpIdxIncrement(&gsCwq.hReadCounter);
		ui32NumOfEntries = ui32NumOfEntries - 1;
	}

#if defined(CACHEOP_DEBUG)
	if (uiTimeNow)
	{
		/* Only log GF that was actually executed */
		CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
	}
#endif

	return eError;
}

static PVRSRV_ERROR CacheOpQListExecRangeBased(void)
{
	IMG_UINT32 ui32NumOfEntries;
	PVRSRV_ERROR eError = PVRSRV_OK;
	IMG_UINT32 ui32WriteCounter = ~0;
	IMG_BOOL bUsedGlobalFlush = IMG_FALSE;
	CACHEOP_WORK_ITEM *psCacheOpWorkItem = NULL;
#if defined(CACHEOP_DEBUG)
	IMG_UINT64 uiTimeNow = 0;
#endif

	/* Take a snapshot of the current count of deferred entries at this junction */
	ui32NumOfEntries = CacheOpIdxSpan(&gsCwq.hWriteCounter, &gsCwq.hReadCounter);
	if (! ui32NumOfEntries)
	{
		return PVRSRV_OK;
	}
#if defined(CACHEOP_DEBUG)
	CACHEOP_PVR_ASSERT(ui32NumOfEntries < CACHEOP_INDICES_MAX);
#endif

	while (ui32NumOfEntries)
	{
		if (! OSAtomicRead(&gsCwq.hReadCounter))
		{
			/* Normally, the read-counter will trail the write counter until the write
			   counter wraps-round to zero. Under this condition we (re)calculate as the
			   read-counter too is wrapping around at this point */
			ui32NumOfEntries = CacheOpIdxSpan(&gsCwq.hWriteCounter, &gsCwq.hReadCounter);
		}
#if defined(CACHEOP_DEBUG)
		/* Something's gone horribly wrong if these 2 counters are identical at this point */
		CACHEOP_PVR_ASSERT(OSAtomicRead(&gsCwq.hWriteCounter) != OSAtomicRead(&gsCwq.hReadCounter));
#endif

		/* Select the next pending deferred work-item for RBF cache maintenance */
		psCacheOpWorkItem = &gsCwq.asWorkItems[CacheOpIdxNext(&gsCwq.hReadCounter)];
		CacheOpQItemReadCheck(psCacheOpWorkItem);
#if defined(CACHEOP_DEBUG)
		/* The time waiting in the queue to be serviced */
		psCacheOpWorkItem->ui64DequeuedTime = OSClockns64();
#endif

		/* The following CacheOpPMRExec() could trigger a GF, so we (re)read this
		   counter just in case so that we know all such pending CacheOp(s) that will
		   benefit from this soon-to-be-executed GF */
		ui32WriteCounter = CacheOpConfigSupports(CACHEOP_CONFIG_KGF) ?
								OSAtomicRead(&gsCwq.hWriteCounter) : ui32WriteCounter;

		eError = CacheOpPMRExec(psCacheOpWorkItem->psPMR,
								NULL, /* No UM virtual address */
								psCacheOpWorkItem->uiOffset,
								psCacheOpWorkItem->uiSize,
								psCacheOpWorkItem->uiCacheOp,
								psCacheOpWorkItem->ui32GFSeqNum,
								IMG_TRUE, /* PMR is pre-validated */
								&bUsedGlobalFlush);
		if (eError != PVRSRV_OK)
		{
#if defined(CACHEOP_DEBUG)
#define PID_FMTSPEC " PID:%u"
#define CACHE_OP_WORK_PID psCacheOpWorkItem->pid
#else
#define PID_FMTSPEC "%s"
#define CACHE_OP_WORK_PID ""
#endif

			PVR_LOG(("Deferred CacheOpPMRExec failed:"
					 PID_FMTSPEC
					 " PMR:%p"
					 " Offset:%" IMG_UINT64_FMTSPECX
					 " Size:%" IMG_UINT64_FMTSPECX
					 " CacheOp:%d,"
					 " error: %d",
					CACHE_OP_WORK_PID,
					psCacheOpWorkItem->psPMR,
					psCacheOpWorkItem->uiOffset,
					psCacheOpWorkItem->uiSize,
					psCacheOpWorkItem->uiCacheOp,
					eError));

#undef PID_FMTSPEC
#undef CACHE_OP_WORK_PID
		}
		else if (bUsedGlobalFlush)
		{
#if defined(CACHEOP_DEBUG)
			psCacheOpWorkItem->ui32OpSeqNum = gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0];
#endif
			break;
		}

#if defined(CACHEOP_DEBUG)
		if (psCacheOpWorkItem->uiCacheOp != PVRSRV_CACHE_OP_GLOBAL)
		{
			psCacheOpWorkItem->bRBF = IMG_TRUE;
			psCacheOpWorkItem->ui64ExecuteTime = OSClockns64();
			CacheOpStatsExecLogWrite(psCacheOpWorkItem);
		}
		else
		{
			CACHEOP_PVR_ASSERT(!gsCwq.bNoGlobalFlushImpl);
		}
#endif

		/* The currently executed CacheOp item updates gsCwq.hCompletedSeqNum.
		   NOTE: This CacheOp item might be a discard item, if so its seqNum
		   still updates the gsCwq.hCompletedSeqNum */
		OSAtomicWrite(&gsCwq.hCompletedSeqNum, psCacheOpWorkItem->ui32OpSeqNum);
		OSAtomicSubtract(&gsCwq.hDeferredSize, psCacheOpWorkItem->uiSize);

		/* If CacheOp is timeline(d), notify timeline waiters */
		eError = CacheOpTimelineExec(psCacheOpWorkItem);
		PVR_LOG_IF_ERROR(eError, "CacheOpTimelineExec");

		/* Indicate that this CCB work-item slot is now free for (re)use */
		CacheOpQItemRecycle(psCacheOpWorkItem);
		(void) CacheOpIdxIncrement(&gsCwq.hReadCounter);
		ui32NumOfEntries = ui32NumOfEntries - 1;
	}

	if (bUsedGlobalFlush)
	{
#if defined(CACHEOP_DEBUG)
		uiTimeNow = OSClockns64();
		CACHEOP_PVR_ASSERT(OSAtomicRead(&gsCwq.hWriteCounter) != OSAtomicRead(&gsCwq.hReadCounter));
#endif

		/* Snapshot of queued CacheOps before the global cache flush was issued */
		ui32NumOfEntries = ui32WriteCounter - OSAtomicRead(&gsCwq.hReadCounter);
		if (ui32WriteCounter < OSAtomicRead(&gsCwq.hReadCounter))
		{
			/* Branch handles when the write-counter has wrapped-around in value space */
			ui32NumOfEntries = OSAtomicRead(&gsCwq.hReadCounter) - ui32WriteCounter;
			ui32NumOfEntries = CACHEOP_INDICES_MAX - ui32NumOfEntries;
		}

		while (ui32NumOfEntries)
		{
			CacheOpQItemReadCheck(psCacheOpWorkItem);

#if defined(CACHEOP_DEBUG)
			psCacheOpWorkItem->bRBF = IMG_FALSE;
			psCacheOpWorkItem->ui64ExecuteTime = uiTimeNow;
			if (psCacheOpWorkItem->uiCacheOp == PVRSRV_CACHE_OP_GLOBAL)
			{
				CACHEOP_PVR_ASSERT(!gsCwq.bNoGlobalFlushImpl);
				psCacheOpWorkItem->pid = OSGetCurrentProcessID();
			}
			CacheOpStatsExecLogWrite(psCacheOpWorkItem);
#endif

			eError = CacheOpTimelineExec(psCacheOpWorkItem);
			PVR_LOG_IF_ERROR(eError, "CacheOpTimelineExec");

			/* Mark index as ready for recycling for next CacheOp */
			CacheOpQItemRecycle(psCacheOpWorkItem);
			(void) CacheOpIdxIncrement(&gsCwq.hReadCounter);
			ui32NumOfEntries = ui32NumOfEntries - 1;
			psCacheOpWorkItem = &gsCwq.asWorkItems[CacheOpIdxNext(&gsCwq.hReadCounter)];
		}
	}

	return eError;
}

static INLINE PVRSRV_ERROR CacheOpQListExec(void)
{
	PVRSRV_ERROR eError;

	if (CacheOpConfigSupports(CACHEOP_CONFIG_KGF) &&
		(!CacheOpConfigSupports(CACHEOP_CONFIG_KRBF)
		 || OSAtomicRead(&gsCwq.hDeferredSize) > gsCwq.pui32InfoPage[CACHEOP_INFO_KMGFTHRESHLD]))
	{
		eError = CacheOpQListExecGlobal();
		PVR_LOG_IF_ERROR(eError, "CacheOpQListExecGlobal");
	}
	else
	{
		eError = CacheOpQListExecRangeBased();
		PVR_LOG_IF_ERROR(eError, "CacheOpQListExecRangeBased");
	}

	/* Signal any waiting threads blocked on CacheOp fence checks update
	   completed sequence number to last queue work item */
	eError = OSEventObjectSignal(gsCwq.hClientWakeUpEvtObj);
	PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");

	return eError;
}

static void CacheOpThread(void *pvData)
{
	PVRSRV_DATA *psPVRSRVData = pvData;
	IMG_HANDLE hOSEvent;
	PVRSRV_ERROR eError;

	/* Open CacheOp thread event object, abort driver if event object open fails */
	eError = OSEventObjectOpen(gsCwq.hThreadWakeUpEvtObj, &hOSEvent);
	PVR_LOG_IF_ERROR(eError, "OSEventObjectOpen");

	/* While driver is in good state & loaded, perform pending cache maintenance */
	while ((psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK) && gsCwq.bInit)
	{
		/* Sleep-wait here until when signalled for new queued CacheOp work items;
		   when woken-up, drain deferred queue completely before next event-wait */
		(void) OSEventObjectWaitKernel(hOSEvent, CACHEOP_THREAD_WAIT_TIMEOUT);
		while (CacheOpIdxSpan(&gsCwq.hWriteCounter, &gsCwq.hReadCounter))
		{
			eError = CacheOpQListExec();
			PVR_LOG_IF_ERROR(eError, "CacheOpQListExec");
		}
	}

	eError = CacheOpQListExec();
	PVR_LOG_IF_ERROR(eError, "CacheOpQListExec");

	eError = OSEventObjectClose(hOSEvent);
	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
}

static PVRSRV_ERROR CacheOpBatchExecTimeline(PVRSRV_DEVICE_NODE *psDevNode,
											 PVRSRV_TIMELINE iTimeline,
											 IMG_BOOL bUsedGlobalFlush,
											 IMG_UINT32 ui32CurrentFenceSeqNum,
											 IMG_UINT32 *pui32NextFenceSeqNum)
{
	PVRSRV_ERROR eError;
	IMG_UINT32 ui32NextIdx;
	CACHEOP_WORK_ITEM sCacheOpWorkItem = { };
	CACHEOP_WORK_ITEM *psCacheOpWorkItem = NULL;

	eError = CacheOpTimelineBind(psDevNode, &sCacheOpWorkItem, iTimeline);
	PVR_LOGR_IF_ERROR(eError, "CacheOpTimelineBind");

	OSLockAcquire(gsCwq.hDeferredLock);

	/*
	   Check if there is any deferred queueing space available and that nothing is
	   currently queued. This second check is required as Android where timelines
	   are used sets a timeline signalling deadline of 1000ms to signal timelines
	   else complains. So seeing we cannot be sure how long the CacheOp presently
	   in the queue would take we should not send this timeline down the queue as
	   well.
	 */
	ui32NextIdx = CacheOpIdxNext(&gsCwq.hWriteCounter);
	if (!CacheOpIdxSpan(&gsCwq.hWriteCounter, &gsCwq.hReadCounter) &&
		CacheOpIdxRead(&gsCwq.hReadCounter) != ui32NextIdx)
	{
		psCacheOpWorkItem = &gsCwq.asWorkItems[ui32NextIdx];
		CacheOpQItemWriteCheck(psCacheOpWorkItem);

		psCacheOpWorkItem->sSWTimelineObj = sCacheOpWorkItem.sSWTimelineObj;
		psCacheOpWorkItem->iTimeline = sCacheOpWorkItem.iTimeline;
		psCacheOpWorkItem->ui32OpSeqNum = CacheOpGetNextCommonSeqNum();
		psCacheOpWorkItem->uiCacheOp = PVRSRV_CACHE_OP_TIMELINE;
		psCacheOpWorkItem->uiOffset = (IMG_DEVMEM_OFFSET_T)0;
		psCacheOpWorkItem->uiSize = (IMG_DEVMEM_SIZE_T)0;
		psCacheOpWorkItem->ui32GFSeqNum = 0;
		psCacheOpWorkItem->psDevNode = psDevNode;
		/* Defer timeline using information page PMR */
		psCacheOpWorkItem->psPMR = gsCwq.psInfoPagePMR;
		eError = PMRLockSysPhysAddresses(psCacheOpWorkItem->psPMR);
		PVR_LOGG_IF_ERROR(eError, "PMRLockSysPhysAddresses", e0);
#if defined(CACHEOP_DEBUG)
		psCacheOpWorkItem->pid = OSGetCurrentClientProcessIDKM();
		psCacheOpWorkItem->ui64EnqueuedTime = OSClockns64();
		gsCwq.ui32ServerASync += 1;
		gsCwq.ui32ServerDTL += 1;
#endif

		/* Mark index ready for cache maintenance */
		(void) CacheOpIdxIncrement(&gsCwq.hWriteCounter);

		OSLockRelease(gsCwq.hDeferredLock);

		/* Signal the CacheOp thread to ensure this GF get processed */
		eError = OSEventObjectSignal(gsCwq.hThreadWakeUpEvtObj);
		PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
	}
	else
	{
		IMG_BOOL bExecTimeline = IMG_TRUE;
		IMG_UINT32 ui32CompletedOpSeqNum = OSAtomicRead(&gsCwq.hCompletedSeqNum);

		OSLockRelease(gsCwq.hDeferredLock);

		/*
		   This pathway requires careful handling here as the client CacheOp(s) predicated on this
		   timeline might have been broken-up (i.e. batched) into several server requests by client:
		   1 - In the first case, a CacheOp from an earlier batch is still in-flight, so we check if
		   this is the case because even though we might have executed all the CacheOps in this batch
		   synchronously, we cannot be sure that any in-flight CacheOp pending on this client is not
		   predicated on this timeline hence we need to synchronise here for safety by fencing until
		   all in-flight CacheOps are completed. NOTE: On Android, this might cause issues due to
		   timelines notification deadlines so we do not fence (i.e. cannot sleep or wait) here to
		   synchronise, instead nudge services client to retry the request if there is no GF support.
		   2 - In the second case, there is no in-flight CacheOp for this client in which case just
		   continue processing as normal.
		 */
		if (!bUsedGlobalFlush && !CacheOpFenceCheck(ui32CompletedOpSeqNum, ui32CurrentFenceSeqNum))
		{
#if defined(ANDROID)
			bExecTimeline = IMG_TRUE;
			if (CacheOpGlobalFlush() != PVRSRV_OK)
			{
				bExecTimeline = IMG_FALSE;
				eError = PVRSRV_ERROR_RETRY;
			}
#else
			eError = CacheOpFence ((RGXFWIF_DM)0, ui32CurrentFenceSeqNum);
			PVR_LOG_IF_ERROR(eError, "CacheOpFence");

			/* CacheOpFence() might have triggered a GF so we take advantage of it */
			if (gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0] > ui32CurrentFenceSeqNum)
			{
				*pui32NextFenceSeqNum = 0;
			}
#endif
		}

		if (bExecTimeline)
		{
			/* CacheOp fence requirement met, signal timeline */
			eError = CacheOpTimelineExec(&sCacheOpWorkItem);
			PVR_LOG_IF_ERROR(eError, "CacheOpTimelineExec");
		}
	}

	return eError;
e0:
	if (psCacheOpWorkItem)
	{
		/* Need to ensure we leave this CacheOp QItem in the proper recycled state */
		CacheOpQItemRecycle(psCacheOpWorkItem);
		OSLockRelease(gsCwq.hDeferredLock);
	}

	return eError;
}

static PVRSRV_ERROR CacheOpBatchExecRangeBased(PVRSRV_DEVICE_NODE *psDevNode,
											PMR **ppsPMR,
											IMG_CPU_VIRTADDR *pvAddress,
											IMG_DEVMEM_OFFSET_T *puiOffset,
											IMG_DEVMEM_SIZE_T *puiSize,
											PVRSRV_CACHE_OP *puiCacheOp,
											IMG_UINT32 ui32NumCacheOps,
											PVRSRV_TIMELINE uiTimeline,
											IMG_UINT32 ui32GlobalFlushSeqNum,
											IMG_UINT32 uiCurrentFenceSeqNum,
											IMG_UINT32 *pui32NextFenceSeqNum)
{
	IMG_UINT32 ui32Idx;
	IMG_UINT32 ui32NextIdx;
	IMG_BOOL bBatchHasTimeline;
	IMG_BOOL bCacheOpConfigKDF;
	IMG_BOOL bCacheOpConfigKRBF;
	IMG_DEVMEM_SIZE_T uiLogicalSize;
	PVRSRV_ERROR eError = PVRSRV_OK;
	IMG_BOOL bUseGlobalFlush = IMG_FALSE;
	CACHEOP_WORK_ITEM *psCacheOpWorkItem = NULL;
#if defined(CACHEOP_DEBUG)
	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
	IMG_UINT32 ui32OpSeqNum = CacheOpGetNextCommonSeqNum();
	sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
#endif

	/* Check if batch has an associated timeline update */
	bBatchHasTimeline = puiCacheOp[ui32NumCacheOps-1] & PVRSRV_CACHE_OP_TIMELINE;
	puiCacheOp[ui32NumCacheOps-1] &= ~(PVRSRV_CACHE_OP_GLOBAL | PVRSRV_CACHE_OP_TIMELINE);

	/* Check if config. supports kernel deferring of cacheops */
	bCacheOpConfigKDF = CacheOpConfigSupports(CACHEOP_CONFIG_KDF);
	bCacheOpConfigKRBF = CacheOpConfigSupports(CACHEOP_CONFIG_KRBF);

	/*
	   Client expects the next fence seqNum to be zero unless the server has deferred
	   at least one CacheOp in the submitted queue in which case the server informs
	   the client of the last CacheOp seqNum deferred in this batch.
	*/
	for (*pui32NextFenceSeqNum = 0, ui32Idx = 0; ui32Idx < ui32NumCacheOps; ui32Idx++)
	{
		if (! puiSize[ui32Idx])
		{
			/* Fail UM request, don't silently ignore */
			eError = PVRSRV_ERROR_INVALID_PARAMS;
			goto e0;
		}
		else if (bCacheOpConfigKDF)
		{
			/* Check if there is deferred queueing space available */
			ui32NextIdx = CacheOpIdxNext(&gsCwq.hWriteCounter);
			if (ui32NextIdx != CacheOpIdxRead(&gsCwq.hReadCounter))
			{
				psCacheOpWorkItem = &gsCwq.asWorkItems[ui32NextIdx];
			}
		}

		/*
		   Normally, we would like to defer client CacheOp(s) but we may not always be in a
		   position or is necessary to do so based on the following reasons:
		   0 - There is currently no queueing space left to enqueue this CacheOp, this might
		       imply the system is queueing more requests than can be consumed by the CacheOp
		       thread in time.
		   1 - Batch has timeline, action this now due to Android timeline signaling deadlines.
		   2 - Configuration does not support deferring of cache maintenance operations so we
		       execute the batch synchronously/immediately.
		   3 - CacheOp has an INVALIDATE, as this is used to transfer device memory buffer
		       ownership back to the processor, we cannot defer it so action it immediately.
		   4 - CacheOp size too small (single OS page size) to warrant overhead of deferment,
		       this will not be considered if KRBF is not present, as it implies defer all.
		   5 - CacheOp size OK for deferment, but a client virtual address is supplied so we
		       might has well just take advantage of said VA & flush immediately in UM context.
		   6 - Prevent DoS attack if a malicious client queues something very large, say 1GiB
		       and the processor cache ISA does not have a global flush implementation. Here
			   we upper bound this threshold to PVR_DIRTY_BYTES_FLUSH_THRESHOLD.
		   7 - Ensure QoS (load balancing) by not over-loading queue with too much requests,
		       here the (pseudo) alternate queue is the user context so we execute directly
		       on it if the processor cache ISA does not have a global flush implementation.
		*/
		if (!psCacheOpWorkItem  ||
			bBatchHasTimeline   ||
			!bCacheOpConfigKDF  ||
			puiCacheOp[ui32Idx] & PVRSRV_CACHE_OP_INVALIDATE ||
			(bCacheOpConfigKRBF && puiSize[ui32Idx] <= (IMG_DEVMEM_SIZE_T)gsCwq.uiPageSize) ||
			(pvAddress[ui32Idx] && puiSize[ui32Idx] < (IMG_DEVMEM_SIZE_T)gsCwq.pui32InfoPage[CACHEOP_INFO_KMDFTHRESHLD]) ||
			(gsCwq.bNoGlobalFlushImpl && puiSize[ui32Idx] >= (IMG_DEVMEM_SIZE_T)(gsCwq.pui32InfoPage[CACHEOP_INFO_KMDFTHRESHLD] << 2)) ||
			(gsCwq.bNoGlobalFlushImpl && OSAtomicRead(&gsCwq.hDeferredSize) >= gsCwq.pui32InfoPage[CACHEOP_INFO_KMDFTHRESHLD] << CACHEOP_INDICES_LOG2_SIZE))
		{
			/* When the CacheOp thread not keeping up, trash d-cache */
			bUseGlobalFlush = !psCacheOpWorkItem && bCacheOpConfigKDF ? IMG_TRUE : IMG_FALSE;
#if defined(CACHEOP_DEBUG)
			sCacheOpWorkItem.ui64EnqueuedTime = OSClockns64();
			gsCwq.ui32ServerSync += 1;
#endif
			psCacheOpWorkItem = NULL;

			eError = CacheOpPMRExec(ppsPMR[ui32Idx],
									pvAddress[ui32Idx],
									puiOffset[ui32Idx],
									puiSize[ui32Idx],
									puiCacheOp[ui32Idx],
									ui32GlobalFlushSeqNum,
									IMG_FALSE,
									&bUseGlobalFlush);
			PVR_LOGG_IF_ERROR(eError, "CacheOpExecPMR", e0);

#if defined(CACHEOP_DEBUG)
			sCacheOpWorkItem.ui64ExecuteTime = OSClockns64();
			sCacheOpWorkItem.bRBF = !bUseGlobalFlush;
			sCacheOpWorkItem.ui32OpSeqNum = bUseGlobalFlush ?
				gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0] : ui32OpSeqNum;
			sCacheOpWorkItem.psPMR = ppsPMR[ui32Idx];
			sCacheOpWorkItem.uiSize = puiSize[ui32Idx];
			sCacheOpWorkItem.uiOffset = puiOffset[ui32Idx];
			sCacheOpWorkItem.uiCacheOp = puiCacheOp[ui32Idx];
			CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
#endif

			if (bUseGlobalFlush) break;
			continue;
		}

		/* Need to validate request parameters here before enqueing */
		eError = PMR_LogicalSize(ppsPMR[ui32Idx], &uiLogicalSize);
		PVR_LOGG_IF_ERROR(eError, "PMR_LogicalSize", e0);
		eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
		PVR_LOGG_IF_FALSE(((puiOffset[ui32Idx]+puiSize[ui32Idx]) <= uiLogicalSize), CACHEOP_DEVMEM_OOR_ERROR_STRING, e0);
		eError = PVRSRV_OK;

		/* For safety, take reference here in user context */
		eError = PMRLockSysPhysAddresses(ppsPMR[ui32Idx]);
		PVR_LOGG_IF_ERROR(eError, "PMRLockSysPhysAddresses", e0);

		OSLockAcquire(gsCwq.hDeferredLock);

		/* Select next item off the queue to defer with */
		ui32NextIdx = CacheOpIdxNext(&gsCwq.hWriteCounter);
		if (ui32NextIdx != CacheOpIdxRead(&gsCwq.hReadCounter))
		{
			psCacheOpWorkItem = &gsCwq.asWorkItems[ui32NextIdx];
			CacheOpQItemWriteCheck(psCacheOpWorkItem);
		}
		else
		{
			/* Retry, disable KDF for this batch */
			OSLockRelease(gsCwq.hDeferredLock);
			bCacheOpConfigKDF = IMG_FALSE;
			psCacheOpWorkItem = NULL;
			ui32Idx = ui32Idx - 1;
			continue;
		}

		/* Timeline need to be looked-up (i.e. bind) in the user context
		   before deferring into the CacheOp thread kernel context */
		eError = CacheOpTimelineBind(psDevNode, psCacheOpWorkItem, PVRSRV_NO_TIMELINE);
		PVR_LOGG_IF_ERROR(eError, "CacheOpTimelineBind", e1);

		/* Prepare & enqueue next deferred work item for CacheOp thread */
		psCacheOpWorkItem->ui32OpSeqNum = CacheOpGetNextCommonSeqNum();
		*pui32NextFenceSeqNum = psCacheOpWorkItem->ui32OpSeqNum;
		psCacheOpWorkItem->ui32GFSeqNum = ui32GlobalFlushSeqNum;
		psCacheOpWorkItem->uiCacheOp = puiCacheOp[ui32Idx];
		psCacheOpWorkItem->uiOffset = puiOffset[ui32Idx];
		psCacheOpWorkItem->uiSize = puiSize[ui32Idx];
		psCacheOpWorkItem->psPMR = ppsPMR[ui32Idx];
		psCacheOpWorkItem->psDevNode = psDevNode;
#if defined(CACHEOP_DEBUG)
		psCacheOpWorkItem->ui64EnqueuedTime = OSClockns64();
		psCacheOpWorkItem->pid = sCacheOpWorkItem.pid;
		psCacheOpWorkItem->bDeferred = IMG_TRUE;
		psCacheOpWorkItem->bKMReq = IMG_FALSE;
		psCacheOpWorkItem->bUMF = IMG_FALSE;
		gsCwq.ui32ServerASync += 1;
#endif

		/* Increment deferred size & mark index ready for cache maintenance */
		OSAtomicAdd(&gsCwq.hDeferredSize, (IMG_UINT32)puiSize[ui32Idx]);
		(void) CacheOpIdxIncrement(&gsCwq.hWriteCounter);

		OSLockRelease(gsCwq.hDeferredLock);
		psCacheOpWorkItem = NULL;
	}

	/* Signal the CacheOp thread to ensure these items get processed */
	eError = OSEventObjectSignal(gsCwq.hThreadWakeUpEvtObj);
	PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");

	if (bUseGlobalFlush)
	{
#if defined(CACHEOP_DEBUG)
		/* GF was logged already in the loop above, so rest if any are discards */
		sCacheOpWorkItem.ui64ExecuteTime = sCacheOpWorkItem.ui64EnqueuedTime;
		sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
		while (++ui32Idx < ui32NumCacheOps)
		{
			sCacheOpWorkItem.psPMR = ppsPMR[ui32Idx];
			sCacheOpWorkItem.uiSize = puiSize[ui32Idx];
			sCacheOpWorkItem.uiOffset = puiOffset[ui32Idx];
			sCacheOpWorkItem.uiCacheOp = puiCacheOp[ui32Idx];
			CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
			gsCwq.ui32KMDiscards += 1;
		}
#endif

		/* No next UM fence seqNum */
		*pui32NextFenceSeqNum = 0;
	}

e1:
	if (psCacheOpWorkItem)
	{
		/* Need to ensure we leave this CacheOp QItem in the proper recycled state */
		CacheOpQItemRecycle(psCacheOpWorkItem);
		OSLockRelease(gsCwq.hDeferredLock);
	}
e0:
	if (bBatchHasTimeline)
	{
		PVRSRV_ERROR eError2;
		eError2 = CacheOpBatchExecTimeline(psDevNode, uiTimeline, bUseGlobalFlush,
										   uiCurrentFenceSeqNum, pui32NextFenceSeqNum);
		eError = (eError2 == PVRSRV_ERROR_RETRY) ? eError2 : eError;
	}

	return eError;
}

static PVRSRV_ERROR CacheOpBatchExecGlobal(PVRSRV_DEVICE_NODE *psDevNode,
									PMR **ppsPMR,
									IMG_CPU_VIRTADDR *pvAddress,
									IMG_DEVMEM_OFFSET_T *puiOffset,
									IMG_DEVMEM_SIZE_T *puiSize,
									PVRSRV_CACHE_OP *puiCacheOp,
									IMG_UINT32 ui32NumCacheOps,
									PVRSRV_TIMELINE uiTimeline,
									IMG_UINT32 ui32GlobalFlushSeqNum,
									IMG_UINT32 uiCurrentFenceSeqNum,
									IMG_UINT32 *pui32NextFenceSeqNum)
{
	IMG_UINT32 ui32Idx;
	IMG_BOOL bBatchHasTimeline;
	PVRSRV_ERROR eError = PVRSRV_OK;
	IMG_BOOL bUseGlobalFlush = IMG_FALSE;
	CACHEOP_WORK_ITEM *psCacheOpWorkItem = NULL;
#if	defined(CACHEOP_DEBUG)
	IMG_DEVMEM_SIZE_T uiTotalSize = 0;
	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
	sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
#endif
#if !defined(CACHEFLUSH_ISA_SUPPORTS_GLOBAL_FLUSH)
	PVR_LOGR_IF_ERROR(PVRSRV_ERROR_NOT_SUPPORTED, CACHEOP_NO_GFLUSH_ERROR_STRING);
#endif
	PVR_UNREFERENCED_PARAMETER(pvAddress);

	/* Check if batch has an associated timeline update request */
	bBatchHasTimeline = puiCacheOp[ui32NumCacheOps-1] & PVRSRV_CACHE_OP_TIMELINE;
	puiCacheOp[ui32NumCacheOps-1] &= ~(PVRSRV_CACHE_OP_GLOBAL | PVRSRV_CACHE_OP_TIMELINE);

	/* Skip operation if an else-when GF has occurred in the interim time */
	if (gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0] > ui32GlobalFlushSeqNum)
	{
#if	defined(CACHEOP_DEBUG)
		sCacheOpWorkItem.ui32OpSeqNum = ui32GlobalFlushSeqNum;
#endif
		bUseGlobalFlush = IMG_TRUE;
		*pui32NextFenceSeqNum = 0;
		goto exec_timeline;
	}

	/* Here we need to check that client batch does not contain an INVALIDATE CacheOp */
	for (*pui32NextFenceSeqNum = 0, ui32Idx = 0; ui32Idx < ui32NumCacheOps; ui32Idx++)
	{
#if	defined(CACHEOP_DEBUG)
		IMG_DEVMEM_SIZE_T uiLogicalSize;
		uiTotalSize += puiSize[ui32Idx];
		/* There is no need to validate request parameters as we are about
		   to issue a GF but this might lead to issues being reproducible
		   in one config but not the other, so valid under debug */
		eError = PMR_LogicalSize(ppsPMR[ui32Idx], &uiLogicalSize);
		PVR_LOGG_IF_ERROR(eError, "PMR_LogicalSize", e0);
		eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
		PVR_LOGG_IF_FALSE(((puiOffset[ui32Idx]+puiSize[ui32Idx]) <= uiLogicalSize), CACHEOP_DEVMEM_OOR_ERROR_STRING, e0);
		eError = PVRSRV_OK;
#endif
		if (! puiSize[ui32Idx])
		{
			/* Fail UM request, don't silently ignore */
			eError = PVRSRV_ERROR_INVALID_PARAMS;
			goto e0;
		}
		else if (puiCacheOp[ui32Idx] & PVRSRV_CACHE_OP_INVALIDATE)
		{
			/* Invalidates cannot be deferred */
			bUseGlobalFlush = IMG_TRUE;
		}
	}

	OSLockAcquire(gsCwq.hDeferredLock);

	/*
	   Normally, we would like to defer client CacheOp(s) but we may not always be in a
	   position to do so based on the following reasons:
	   0 - Batch has an INVALIDATE, as this is used to transfer device memory buffer
	       ownership back to the processor, we cannot defer it so action it immediately.
	   1 - Configuration does not support deferring of cache maintenance operations so
		   we execute synchronously/immediately.
	   2 - There is currently no queueing space left to enqueue this CacheOp, this might
	       imply the system is queueing more requests that can be consumed by the CacheOp
	       thread in time.
	   3 - Batch has a timeline and there is currently something queued, we cannot defer
	       because currently queued operation(s) might take quite a while to action which
	       might cause a timeline deadline timeout.
	*/
	if (bUseGlobalFlush ||
		!CacheOpConfigSupports(CACHEOP_CONFIG_KDF) ||
		CacheOpIdxNext(&gsCwq.hWriteCounter) == CacheOpIdxRead(&gsCwq.hReadCounter) ||
		(bBatchHasTimeline && CacheOpIdxSpan(&gsCwq.hWriteCounter, &gsCwq.hReadCounter)))

	{
		OSLockRelease(gsCwq.hDeferredLock);
#if	defined(CACHEOP_DEBUG)
		sCacheOpWorkItem.ui32OpSeqNum =	CacheOpGetNextCommonSeqNum();
		sCacheOpWorkItem.ui64EnqueuedTime = OSClockns64();
#endif
		eError = CacheOpGlobalFlush();
		PVR_LOGG_IF_ERROR(eError, "CacheOpGlobalFlush", e0);
		bUseGlobalFlush = IMG_TRUE;
#if	defined(CACHEOP_DEBUG)
		sCacheOpWorkItem.ui64ExecuteTime = OSClockns64();
		gsCwq.ui32ServerSync += 1;
#endif
		goto exec_timeline;
	}

	/* Select next item off queue to defer this GF and possibly timeline with */
	psCacheOpWorkItem = &gsCwq.asWorkItems[CacheOpIdxNext(&gsCwq.hWriteCounter)];
	CacheOpQItemWriteCheck(psCacheOpWorkItem);

	/* Defer the GF using information page PMR */
	psCacheOpWorkItem->psPMR = gsCwq.psInfoPagePMR;
	eError = PMRLockSysPhysAddresses(psCacheOpWorkItem->psPMR);
	PVR_LOGG_IF_ERROR(eError, "PMRLockSysPhysAddresses", e0);

	/* Timeline object has to be looked-up here in user context */
	eError = CacheOpTimelineBind(psDevNode, psCacheOpWorkItem, uiTimeline);
	PVR_LOGG_IF_ERROR(eError, "CacheOpTimelineBind", e0);

	/* Prepare & enqueue next deferred work item for CacheOp thread */
	*pui32NextFenceSeqNum = CacheOpGetNextCommonSeqNum();
	psCacheOpWorkItem->ui32OpSeqNum = *pui32NextFenceSeqNum;
	psCacheOpWorkItem->ui32GFSeqNum = ui32GlobalFlushSeqNum;
	psCacheOpWorkItem->uiCacheOp = PVRSRV_CACHE_OP_GLOBAL;
	psCacheOpWorkItem->uiOffset = (IMG_DEVMEM_OFFSET_T)0;
	psCacheOpWorkItem->uiSize = (IMG_DEVMEM_SIZE_T)0;
#if defined(CACHEOP_DEBUG)
	/* Note client pid & queueing time of deferred GF CacheOp */
	psCacheOpWorkItem->ui64EnqueuedTime = OSClockns64();
	psCacheOpWorkItem->pid = sCacheOpWorkItem.pid;
	OSAtomicAdd(&gsCwq.hDeferredSize, uiTotalSize);
	psCacheOpWorkItem->uiSize = uiTotalSize;
	psCacheOpWorkItem->bDeferred = IMG_TRUE;
	psCacheOpWorkItem->bKMReq = IMG_FALSE;
	psCacheOpWorkItem->bUMF = IMG_FALSE;
	/* Client CacheOp is logged using the deferred seqNum */
	sCacheOpWorkItem.ui32OpSeqNum =	*pui32NextFenceSeqNum;
	sCacheOpWorkItem.ui64EnqueuedTime = psCacheOpWorkItem->ui64EnqueuedTime;
	sCacheOpWorkItem.ui64ExecuteTime = psCacheOpWorkItem->ui64EnqueuedTime;
	/* Update the CacheOp statistics */
	gsCwq.ui32ServerASync += 1;
	gsCwq.ui32ServerDGF += 1;
#endif

	/* Mark index ready for cache maintenance */
	(void) CacheOpIdxIncrement(&gsCwq.hWriteCounter);

	OSLockRelease(gsCwq.hDeferredLock);

	/* Signal CacheOp thread to ensure this GF get processed */
	eError = OSEventObjectSignal(gsCwq.hThreadWakeUpEvtObj);
	PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");

exec_timeline:
	if (bUseGlobalFlush && bBatchHasTimeline)
	{
		eError = CacheOpBatchExecTimeline(psDevNode, uiTimeline, bUseGlobalFlush,
										  uiCurrentFenceSeqNum, pui32NextFenceSeqNum);
	}

#if	defined(CACHEOP_DEBUG)
	for (ui32Idx = 0; ui32Idx < ui32NumCacheOps; ui32Idx++)
	{
		sCacheOpWorkItem.psPMR = ppsPMR[ui32Idx];
		sCacheOpWorkItem.uiSize = puiSize[ui32Idx];
		sCacheOpWorkItem.uiOffset = puiOffset[ui32Idx];
		sCacheOpWorkItem.uiCacheOp = puiCacheOp[ui32Idx];
		if (bUseGlobalFlush)
		{
			if (sCacheOpWorkItem.ui64ExecuteTime && ui32Idx)
			{
				/* Only first item carries the real execution time, rest are discards */
				sCacheOpWorkItem.ui64EnqueuedTime = sCacheOpWorkItem.ui64ExecuteTime;
			}
			gsCwq.ui32KMDiscards += !sCacheOpWorkItem.ui64ExecuteTime ? 1 : ui32Idx ? 1 : 0;
		}
		CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
	}
#endif

	return eError;
e0:
	if (psCacheOpWorkItem)
	{
		/* Need to ensure we leave this CacheOp QItem in the proper recycled state */
		CacheOpQItemRecycle(psCacheOpWorkItem);
		OSLockRelease(gsCwq.hDeferredLock);
	}

	if (bBatchHasTimeline)
	{
		PVRSRV_ERROR eError2;
		eError2 = CacheOpBatchExecTimeline(psDevNode, uiTimeline, IMG_FALSE,
										   uiCurrentFenceSeqNum, pui32NextFenceSeqNum);
		eError = (eError2 == PVRSRV_ERROR_RETRY) ? eError2 : eError;
	}

	return eError;
}

PVRSRV_ERROR CacheOpExec (PPVRSRV_DEVICE_NODE psDevNode,
						  void *pvVirtStart,
						  void *pvVirtEnd,
						  IMG_CPU_PHYADDR sCPUPhysStart,
						  IMG_CPU_PHYADDR sCPUPhysEnd,
						  PVRSRV_CACHE_OP uiCacheOp)
{
	PVRSRV_ERROR eError = PVRSRV_ERROR_RETRY;
#if	defined(CACHEOP_DEBUG)
	IMG_BOOL bUsedGlobalFlush = IMG_FALSE;
	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
	sCacheOpWorkItem.ui64EnqueuedTime = OSClockns64();
#endif

	if (gsCwq.bInit)
	{
		IMG_DEVMEM_SIZE_T uiSize = sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr;
		if ((IMG_UINT32)uiSize > gsCwq.pui32InfoPage[CACHEOP_INFO_KMGFTHRESHLD])
		{
			eError = CacheOpGlobalFlush();
		}
	}

	if (eError == PVRSRV_OK)
	{
#if	defined(CACHEOP_DEBUG)
		bUsedGlobalFlush = IMG_TRUE;
#endif
	}
	else
	{
		switch (uiCacheOp)
		{
			case PVRSRV_CACHE_OP_CLEAN:
				OSCPUCacheCleanRangeKM(psDevNode, pvVirtStart, pvVirtEnd, sCPUPhysStart, sCPUPhysEnd);
				break;
			case PVRSRV_CACHE_OP_INVALIDATE:
				OSCPUCacheInvalidateRangeKM(psDevNode, pvVirtStart, pvVirtEnd, sCPUPhysStart, sCPUPhysEnd);
				break;
			case PVRSRV_CACHE_OP_FLUSH:
				OSCPUCacheFlushRangeKM(psDevNode, pvVirtStart, pvVirtEnd, sCPUPhysStart, sCPUPhysEnd);
				break;
			default:
				PVR_DPF((PVR_DBG_ERROR,	"%s: Invalid cache operation type %d",
						 __func__, uiCacheOp));
				break;
		}
		eError = PVRSRV_OK;
	}

#if	defined(CACHEOP_DEBUG)
	if (! CacheOpConfigSupports(CACHEOP_CONFIG_KLOG))
	{
		if (bUsedGlobalFlush)
		{
			/* Undo the accounting for server GF done in CacheOpGlobalFlush() */
			gsCwq.ui32ServerGF -= 1;
		}
	}
	else
	{
		gsCwq.ui32TotalExecOps += 1;
		if (! bUsedGlobalFlush)
		{
			gsCwq.ui32ServerSync += 1;
			gsCwq.ui32ServerRBF +=
				((sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr) & ((IMG_DEVMEM_SIZE_T)~(gsCwq.uiLineSize - 1))) >> gsCwq.uiLineShift;
		}
		sCacheOpWorkItem.uiOffset = 0;
		sCacheOpWorkItem.bKMReq = IMG_TRUE;
		sCacheOpWorkItem.uiCacheOp = uiCacheOp;
		sCacheOpWorkItem.bRBF = !bUsedGlobalFlush;
		/* Use information page PMR for logging KM request */
		sCacheOpWorkItem.psPMR = gsCwq.psInfoPagePMR;
		sCacheOpWorkItem.ui64ExecuteTime = OSClockns64();
		sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
		sCacheOpWorkItem.ui32OpSeqNum = CacheOpGetNextCommonSeqNum();
		sCacheOpWorkItem.uiSize = (sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr);
		CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
	}
#endif

	return eError;
}

PVRSRV_ERROR CacheOpValExec(PMR *psPMR,
						    IMG_UINT64 uiAddress,
						    IMG_DEVMEM_OFFSET_T uiOffset,
						    IMG_DEVMEM_SIZE_T uiSize,
						    PVRSRV_CACHE_OP uiCacheOp)
{
	PVRSRV_ERROR eError;
	IMG_CPU_VIRTADDR pvAddress = (IMG_CPU_VIRTADDR)(uintptr_t)uiAddress;
	IMG_BOOL bUseGlobalFlush = (IMG_UINT32)uiSize > gsCwq.pui32InfoPage[CACHEOP_INFO_KMGFTHRESHLD];
#if	defined(CACHEOP_DEBUG)
	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
	gsCwq.ui32TotalExecOps += 1;
	gsCwq.ui32ServerSync += 1;
	sCacheOpWorkItem.psPMR = psPMR;
	sCacheOpWorkItem.uiSize = uiSize;
	sCacheOpWorkItem.uiOffset = uiOffset;
	sCacheOpWorkItem.uiCacheOp = uiCacheOp;
	sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
	sCacheOpWorkItem.ui32OpSeqNum = CacheOpGetNextCommonSeqNum();
	sCacheOpWorkItem.ui64EnqueuedTime = OSClockns64();
#endif

	eError = CacheOpPMRExec(psPMR,
							pvAddress,
							uiOffset,
							uiSize,
							uiCacheOp,
							gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0],
							IMG_FALSE,
							&bUseGlobalFlush);
	PVR_LOGG_IF_ERROR(eError, "CacheOpPMRExec", e0);

#if	defined(CACHEOP_DEBUG)
	sCacheOpWorkItem.bRBF = !bUseGlobalFlush;
	sCacheOpWorkItem.ui64ExecuteTime = OSClockns64();
	CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
#endif

e0:
	return eError;
}

PVRSRV_ERROR CacheOpQueue (CONNECTION_DATA *psConnection,
						   PVRSRV_DEVICE_NODE *psDevNode,
						   IMG_UINT32 ui32NumCacheOps,
						   PMR **ppsPMR,
						   IMG_UINT64 *puiAddress,
						   IMG_DEVMEM_OFFSET_T *puiOffset,
						   IMG_DEVMEM_SIZE_T *puiSize,
						   PVRSRV_CACHE_OP *puiCacheOp,
						   IMG_UINT32 ui32OpTimeline,
						   IMG_UINT32 ui32ClientGFSeqNum,
						   IMG_UINT32 uiCurrentFenceSeqNum,
						   IMG_UINT32 *pui32NextFenceSeqNum)
{
	PVRSRV_ERROR eError;
	PVRSRV_TIMELINE uiTimeline = (PVRSRV_TIMELINE)ui32OpTimeline;
	IMG_CPU_VIRTADDR *pvAddress = (IMG_CPU_VIRTADDR*)(uintptr_t)puiAddress;

	PVR_UNREFERENCED_PARAMETER(psConnection);

#if !defined(CACHEFLUSH_ISA_SUPPORTS_GLOBAL_FLUSH)
	PVR_LOGR_IF_FALSE((ui32ClientGFSeqNum == 0),
					  "CacheOpQueue(ui32ClientGFSeqNum > 0)",
	                  PVRSRV_ERROR_INVALID_PARAMS);
#endif
#if defined(CACHEOP_DEBUG)
	gsCwq.ui32TotalExecOps += ui32NumCacheOps;
#endif

	if (! gsCwq.bInit)
	{
		PVR_LOG(("CacheOp framework not initialised, failing request"));
		return PVRSRV_ERROR_NOT_INITIALISED;
	}
	else if (! ui32NumCacheOps)
	{
		return PVRSRV_ERROR_INVALID_PARAMS;
	}
	/* Ensure any single timeline CacheOp request is processed immediately */
	else if (ui32NumCacheOps == 1 && puiCacheOp[0] == PVRSRV_CACHE_OP_TIMELINE)
	{
		eError = CacheOpBatchExecTimeline(psDevNode, uiTimeline, IMG_TRUE, uiCurrentFenceSeqNum, pui32NextFenceSeqNum);
	}
	/* Services client explicitly requested a GF or config is GF only (i.e. no KRBF support), this takes priority */
	else if (CacheOpConfigSupports(CACHEOP_CONFIG_KGF) &&
			 ((puiCacheOp[ui32NumCacheOps-1] & PVRSRV_CACHE_OP_GLOBAL) || !CacheOpConfigSupports(CACHEOP_CONFIG_KRBF)))
	{
		eError =
			CacheOpBatchExecGlobal(psDevNode,
								   ppsPMR,
								   pvAddress,
								   puiOffset,
								   puiSize,
								   puiCacheOp,
								   ui32NumCacheOps,
								   uiTimeline,
								   ui32ClientGFSeqNum,
								   uiCurrentFenceSeqNum,
								   pui32NextFenceSeqNum);
	}
	/* This is the default entry for all client requests */
	else
	{
		if (!(gsCwq.eConfig & (CACHEOP_CONFIG_LAST-1)))
		{
			/* default the configuration before execution */
			CacheOpConfigUpdate(CACHEOP_CONFIG_DEFAULT);
		}

		eError =
			CacheOpBatchExecRangeBased(psDevNode,
									   ppsPMR,
									   pvAddress,
									   puiOffset,
									   puiSize,
									   puiCacheOp,
									   ui32NumCacheOps,
									   uiTimeline,
									   ui32ClientGFSeqNum,
									   uiCurrentFenceSeqNum,
									   pui32NextFenceSeqNum);
	}

	return eError;
}

PVRSRV_ERROR CacheOpFence (RGXFWIF_DM eFenceOpType, IMG_UINT32 ui32FenceOpSeqNum)
{
	IMG_HANDLE hOSEvent;
	PVRSRV_ERROR eError2;
	IMG_UINT32 ui32RetryAbort;
	IMG_UINT32 ui32CompletedOpSeqNum;
	PVRSRV_ERROR eError = PVRSRV_OK;
#if defined(CACHEOP_DEBUG)
	IMG_UINT64 uiTimeNow;
	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
	sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
	sCacheOpWorkItem.ui32OpSeqNum = ui32FenceOpSeqNum;
	sCacheOpWorkItem.ui64EnqueuedTime = OSClockns64();
	uiTimeNow = sCacheOpWorkItem.ui64EnqueuedTime;
#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
	sCacheOpWorkItem.eFenceOpType = eFenceOpType;
#endif
	sCacheOpWorkItem.uiSize = (uintptr_t) OSAtomicRead(&gsCwq.hCompletedSeqNum);
	sCacheOpWorkItem.uiOffset = (uintptr_t) gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0];
#endif
	PVR_UNREFERENCED_PARAMETER(eFenceOpType);

	/* CacheOp(s) this thread is fencing for has already been satisfied by an
	   else-when GF. Another way of looking at this, if last else-when GF is
	   logically behind or momentarily disabled (zero) then we have to flush
	   the cache */
	if (gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0] > ui32FenceOpSeqNum)
	{
#if defined(CACHEOP_DEBUG)
		sCacheOpWorkItem.uiOffset = (uintptr_t) gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0];
#endif
		goto e0;
	}

	/* If initial fence check fails, then wait-and-retry in loop */
	ui32CompletedOpSeqNum = OSAtomicRead(&gsCwq.hCompletedSeqNum);
	if (CacheOpFenceCheck(ui32CompletedOpSeqNum, ui32FenceOpSeqNum))
	{
#if defined(CACHEOP_DEBUG)
		sCacheOpWorkItem.uiSize = (uintptr_t) ui32CompletedOpSeqNum;
#endif
		goto e0;
	}

	/* Open CacheOp update event object, if event open fails return error */
	eError2 = OSEventObjectOpen(gsCwq.hClientWakeUpEvtObj, &hOSEvent);
	PVR_LOGG_IF_ERROR(eError2, "OSEventObjectOpen", e0);

	/* Linear (i.e. use exponential?) back-off, upper bounds user wait */
	for (ui32RetryAbort = gsCwq.ui32FenceRetryAbort; ;--ui32RetryAbort)
	{
		/* (Re)read completed CacheOp sequence number before waiting */
		ui32CompletedOpSeqNum = OSAtomicRead(&gsCwq.hCompletedSeqNum);
		if (CacheOpFenceCheck(ui32CompletedOpSeqNum, ui32FenceOpSeqNum))
		{
#if defined(CACHEOP_DEBUG)
			sCacheOpWorkItem.uiSize = (uintptr_t) ui32CompletedOpSeqNum;
#endif
			break;
		}

		/*
		   For cache ISA with GF support, the wait(ms) must be set to be around
		   25% GF overhead and as such there is no point waiting longer, we just
		   perform a GF as it means the CacheOp thread is really lagging behind.
		   Lastly, we cannot (or should not) hang the client thread indefinitely
		   so after a certain duration, we just give up. What this duration is,
		   is hard to state but for now we set it to be 1 seconds, which is the
		   product of CACHEOP_FENCE_[WAIT_TIMEOUT * RETRY_ABORT]. We ask the
		   client to retry the operation by exiting with PVRSRV_ERROR_RETRY.
		*/
		(void) OSEventObjectWaitTimeout(hOSEvent, gsCwq.ui32FenceWaitTimeUs);
		if (gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0] > ui32FenceOpSeqNum)
		{
#if defined(CACHEOP_DEBUG)
			sCacheOpWorkItem.uiOffset = (uintptr_t) gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0];
			uiTimeNow = OSClockns64();
#endif
			break;
		}
		else if (CacheOpConfigSupports(CACHEOP_CONFIG_KGF))
		{
			eError2 = CacheOpGlobalFlush();
			PVR_LOG_IF_ERROR(eError2, "CacheOpGlobalFlush");
#if defined(CACHEOP_DEBUG)
			sCacheOpWorkItem.uiCacheOp = PVRSRV_CACHE_OP_GLOBAL;
			sCacheOpWorkItem.uiOffset = (uintptr_t) gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0];
			uiTimeNow = OSClockns64();
#endif
			break;
		}
		else if (! ui32RetryAbort)
		{
#if defined(CACHEOP_DEBUG)
			sCacheOpWorkItem.uiSize = (uintptr_t) OSAtomicRead(&gsCwq.hCompletedSeqNum);
			sCacheOpWorkItem.uiOffset = (uintptr_t) gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0];
			uiTimeNow = OSClockns64();
#endif
			PVR_LOG(("CacheOpFence() event: "CACHEOP_ABORT_FENCE_ERROR_STRING));
			eError = PVRSRV_ERROR_RETRY;
			break;
		}
		else
		{
#if defined(CACHEOP_DEBUG)
			uiTimeNow = OSClockns64();
#endif
		}
	}

	eError2 = OSEventObjectClose(hOSEvent);
	PVR_LOG_IF_ERROR(eError2, "OSEventObjectOpen");

e0:
#if defined(CACHEOP_DEBUG)
	sCacheOpWorkItem.ui64ExecuteTime = uiTimeNow;
	if (ui32FenceOpSeqNum)
	{
		/* Only fence(s) pending on CacheOp(s) contribute towards statistics,
		   here we calculate the rolling approximate average waiting time
		   for these fence(s) */
		IMG_UINT32 ui64EnqueuedTime = sCacheOpWorkItem.ui64EnqueuedTime;
		IMG_UINT32 ui64ExecuteTime = sCacheOpWorkItem.ui64ExecuteTime;
		IMG_UINT32 ui32Time = ui64EnqueuedTime < ui64ExecuteTime ?
									ui64ExecuteTime - ui64EnqueuedTime :
									ui64EnqueuedTime - ui64ExecuteTime;
		ui32Time = DivBy10(DivBy10(DivBy10(ui32Time)));
		gsCwq.ui32TotalFenceOps += 1;
		if (gsCwq.ui32TotalFenceOps > 2)
		{
			gsCwq.ui32AvgFenceTime -= (gsCwq.ui32AvgFenceTime / gsCwq.ui32TotalFenceOps);
			gsCwq.ui32AvgFenceTime += (ui32Time / gsCwq.ui32TotalFenceOps);
		}
		else if (ui32Time)
		{
			gsCwq.ui32AvgFenceTime = (IMG_UINT32)ui32Time;
		}
	}
	CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
#endif

	return eError;
}

PVRSRV_ERROR CacheOpLog (PMR *psPMR,
						 IMG_UINT64 puiAddress,
						 IMG_DEVMEM_OFFSET_T uiOffset,
						 IMG_DEVMEM_SIZE_T uiSize,
						 IMG_UINT64 ui64EnqueuedTimeUs,
						 IMG_UINT64 ui64ExecuteTimeUs,
						 IMG_UINT32 ui32NumRBF,
						 IMG_BOOL bIsDiscard,
						 PVRSRV_CACHE_OP uiCacheOp)
{
#if defined(CACHEOP_DEBUG)
	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
	PVR_UNREFERENCED_PARAMETER(puiAddress);

	sCacheOpWorkItem.psPMR = psPMR;
	sCacheOpWorkItem.uiSize = uiSize;
	sCacheOpWorkItem.uiOffset = uiOffset;
	sCacheOpWorkItem.uiCacheOp = uiCacheOp;
	sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
	sCacheOpWorkItem.ui32OpSeqNum = CacheOpGetNextCommonSeqNum();

	sCacheOpWorkItem.ui64EnqueuedTime = ui64EnqueuedTimeUs;
	sCacheOpWorkItem.ui64ExecuteTime = ui64ExecuteTimeUs;
	sCacheOpWorkItem.bUMF = IMG_TRUE;
	sCacheOpWorkItem.bRBF = bIsDiscard ? IMG_FALSE : IMG_TRUE;
	gsCwq.ui32UMDiscards += bIsDiscard ? 1 : 0;
	gsCwq.ui32ClientRBF += bIsDiscard ? 0 : ui32NumRBF;
	gsCwq.ui32ClientSync += 1;
	gsCwq.ui32TotalExecOps += 1;

	CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
#else
	PVR_UNREFERENCED_PARAMETER(psPMR);
	PVR_UNREFERENCED_PARAMETER(uiSize);
	PVR_UNREFERENCED_PARAMETER(uiOffset);
	PVR_UNREFERENCED_PARAMETER(uiCacheOp);
	PVR_UNREFERENCED_PARAMETER(ui32NumRBF);
	PVR_UNREFERENCED_PARAMETER(puiAddress);
	PVR_UNREFERENCED_PARAMETER(ui64ExecuteTimeUs);
	PVR_UNREFERENCED_PARAMETER(ui64EnqueuedTimeUs);
#endif
	return PVRSRV_OK;
}

PVRSRV_ERROR CacheOpInit2 (void)
{
	PVRSRV_ERROR eError;
	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();

	/* Create an event object for pending CacheOp work items */
	eError = OSEventObjectCreate("PVRSRV_CACHEOP_EVENTOBJECT", &gsCwq.hThreadWakeUpEvtObj);
	PVR_LOGG_IF_ERROR(eError, "OSEventObjectCreate", e0);

	/* Create an event object for updating pending fence checks on CacheOp */
	eError = OSEventObjectCreate("PVRSRV_CACHEOP_EVENTOBJECT", &gsCwq.hClientWakeUpEvtObj);
	PVR_LOGG_IF_ERROR(eError, "OSEventObjectCreate", e0);

	/* Appending work-items is not concurrent, lock protects against this */
	eError = OSLockCreate((POS_LOCK*)&gsCwq.hDeferredLock);
	PVR_LOGG_IF_ERROR(eError, "OSLockCreate", e0);

	/* Apphint read/write is not concurrent, so lock protects against this */
	eError = OSLockCreate((POS_LOCK*)&gsCwq.hConfigLock);
	PVR_LOGG_IF_ERROR(eError, "OSLockCreate", e0);

	/* Determine CPU cache ISA maintenance mechanism available, GF and UMF */
#if defined(__arm__) || defined(__arm64__) || defined(__aarch64__)
	gsCwq.bNoGlobalFlushImpl = IMG_TRUE;
#else
	gsCwq.bNoGlobalFlushImpl = (OSCPUOperation(PVRSRV_CACHE_OP_FLUSH) != PVRSRV_OK) ? IMG_TRUE : IMG_FALSE;
#endif
	if (! gsCwq.bNoGlobalFlushImpl)
	{
		IMG_UINT64 uiIdx;
		IMG_UINT64 uiTime = 0;
		IMG_UINT64 uiTimeAfter;
		IMG_UINT64 uiTimeBefore;

		for (uiIdx = 0; uiIdx < 4; uiIdx++)
		{
			/* Take average of four GF */
			uiTimeBefore = OSClockns64();
			(void) OSCPUOperation(PVRSRV_CACHE_OP_FLUSH);
			uiTimeAfter = OSClockns64();

			uiTimeBefore = DivBy10(DivBy10(DivBy10(uiTimeBefore)));
			uiTimeAfter = DivBy10(DivBy10(DivBy10(uiTimeAfter)));
			uiTime += uiTimeBefore < uiTimeAfter ?
								uiTimeAfter  - uiTimeBefore :
								uiTimeBefore - uiTimeAfter;
		}

		gsCwq.ui32FenceWaitTimeUs = (IMG_UINT32)(uiTime >> 2);
		gsCwq.ui32FenceRetryAbort = ~0;
	}
	else
	{
		gsCwq.ui32FenceWaitTimeUs = CACHEOP_FENCE_WAIT_TIMEOUT;
		gsCwq.ui32FenceRetryAbort = CACHEOP_FENCE_RETRY_ABORT;
	}
#if defined(CACHEFLUSH_ISA_SUPPORTS_UM_FLUSH)
	gsCwq.bSupportsUMFlush = IMG_TRUE;
#else
	gsCwq.bSupportsUMFlush = IMG_FALSE;
#endif

	gsCwq.pui32InfoPage = psPVRSRVData->pui32InfoPage;
	gsCwq.psInfoPagePMR = psPVRSRVData->psInfoPagePMR;

	/* Normally, platforms should use their default configurations, put exceptions here */
#if defined(__i386__) || defined(__x86_64__)
#if !defined(TC_MEMORY_CONFIG)
	CacheOpConfigUpdate(CACHEOP_CONFIG_URBF | CACHEOP_CONFIG_KGF | CACHEOP_CONFIG_KDF);
#else
	CacheOpConfigUpdate(CACHEOP_CONFIG_KGF | CACHEOP_CONFIG_KDF);
#endif
#else /* defined(__x86__) */
	CacheOpConfigUpdate(CACHEOP_CONFIG_DEFAULT);
#endif

	/* Initialise the remaining occupants of the CacheOp information page */
	gsCwq.pui32InfoPage[CACHEOP_INFO_PGSIZE]    = (IMG_UINT32)gsCwq.uiPageSize;
	gsCwq.pui32InfoPage[CACHEOP_INFO_LINESIZE]  = (IMG_UINT32)gsCwq.uiLineSize;
	gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM0] = (IMG_UINT32)0;
	gsCwq.pui32InfoPage[CACHEOP_INFO_GFSEQNUM1] = (IMG_UINT32)0;

	/* Set before spawning thread */
	gsCwq.bInit = IMG_TRUE;

	/* Create a thread which is used to execute the deferred CacheOp(s),
	   these are CacheOp(s) executed by the server on behalf of clients
	   asynchronously. All clients synchronise with the server before
	   submitting any HW operation (i.e. device kicks) to ensure that
	   client device work-load memory is coherent */
	eError = OSThreadCreatePriority(&gsCwq.hWorkerThread,
									"pvr_cacheop",
									CacheOpThread,
									CacheOpThreadDumpInfo,
									IMG_TRUE,
									psPVRSRVData,
									OS_THREAD_HIGHEST_PRIORITY);
	PVR_LOGG_IF_ERROR(eError, "OSThreadCreatePriority", e0);

	/* Writing the unsigned integer binary encoding of CACHEOP_CONFIG
	   into this file cycles through avail. configuration(s) */
	gsCwq.pvConfigTune = OSCreateStatisticEntry("cacheop_config",
											NULL,
											CacheOpConfigRead,
											NULL);
	PVR_LOGG_IF_FALSE(gsCwq.pvConfigTune, "OSCreateStatisticEntry", e0);

	/* Register the CacheOp framework (re)configuration handlers */
	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_CacheOpConfig,
										CacheOpConfigQuery,
										CacheOpConfigSet,
										APPHINT_OF_DRIVER_NO_DEVICE,
										(void *) APPHINT_ID_CacheOpConfig);

	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_CacheOpGFThresholdSize,
										CacheOpConfigQuery,
										CacheOpConfigSet,
										APPHINT_OF_DRIVER_NO_DEVICE,
										(void *) APPHINT_ID_CacheOpGFThresholdSize);

	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_CacheOpUMKMThresholdSize,
										CacheOpConfigQuery,
										CacheOpConfigSet,
										APPHINT_OF_DRIVER_NO_DEVICE,
										(void *) APPHINT_ID_CacheOpUMKMThresholdSize);

	return PVRSRV_OK;
e0:
	CacheOpDeInit2();
	return eError;
}

void CacheOpDeInit2 (void)
{
	PVRSRV_ERROR eError = PVRSRV_OK;

	gsCwq.bInit = IMG_FALSE;

	if (gsCwq.hThreadWakeUpEvtObj)
	{
		eError = OSEventObjectSignal(gsCwq.hThreadWakeUpEvtObj);
		PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
	}

	if (gsCwq.hClientWakeUpEvtObj)
	{
		eError = OSEventObjectSignal(gsCwq.hClientWakeUpEvtObj);
		PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
	}

	if (gsCwq.hWorkerThread)
	{
		LOOP_UNTIL_TIMEOUT(OS_THREAD_DESTROY_TIMEOUT_US)
		{
			eError = OSThreadDestroy(gsCwq.hWorkerThread);
			if (PVRSRV_OK == eError)
			{
				gsCwq.hWorkerThread = NULL;
				break;
			}
			OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US/OS_THREAD_DESTROY_RETRY_COUNT);
		} END_LOOP_UNTIL_TIMEOUT();
		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
		gsCwq.hWorkerThread = NULL;
	}

	if (gsCwq.hClientWakeUpEvtObj)
	{
		eError = OSEventObjectDestroy(gsCwq.hClientWakeUpEvtObj);
		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
		gsCwq.hClientWakeUpEvtObj = NULL;
	}

	if (gsCwq.hThreadWakeUpEvtObj)
	{
		eError = OSEventObjectDestroy(gsCwq.hThreadWakeUpEvtObj);
		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
		gsCwq.hThreadWakeUpEvtObj = NULL;
	}

	if (gsCwq.hConfigLock)
	{
		eError = OSLockDestroy(gsCwq.hConfigLock);
		PVR_LOG_IF_ERROR(eError, "OSLockDestroy");
		gsCwq.hConfigLock = NULL;
	}

	if (gsCwq.hDeferredLock)
	{
		eError = OSLockDestroy(gsCwq.hDeferredLock);
		PVR_LOG_IF_ERROR(eError, "OSLockDestroy");
		gsCwq.hDeferredLock = NULL;
	}

	if (gsCwq.pvConfigTune)
	{
		OSRemoveStatisticEntry(&gsCwq.pvConfigTune);
	}

	gsCwq.pui32InfoPage = NULL;
	gsCwq.psInfoPagePMR = NULL;
}

PVRSRV_ERROR CacheOpInit (void)
{
	IMG_UINT32 idx;
	PVRSRV_ERROR eError = PVRSRV_OK;

	/* DDK initialisation is anticipated to be performed on the boot
	   processor (little core in big/little systems) though this may
	   not always be the case. If so, the value cached here is the
	   system wide safe (i.e. smallest) L1 d-cache line size value
	   on any/such platforms with mismatched d-cache line sizes */
	gsCwq.uiPageSize = OSGetPageSize();
	gsCwq.uiPageShift = OSGetPageShift();
	gsCwq.uiLineSize = OSCPUCacheAttributeSize(PVR_DCACHE_LINE_SIZE);
	gsCwq.uiLineShift = ExactLog2(gsCwq.uiLineSize);
	PVR_LOGR_IF_FALSE((gsCwq.uiLineSize && gsCwq.uiPageSize && gsCwq.uiPageShift), "", PVRSRV_ERROR_INIT_FAILURE);
	gsCwq.uiCacheOpAddrType = OSCPUCacheOpAddressType();

	/* More information regarding these atomic counters can be found
	   in the CACHEOP_WORK_QUEUE type definition at top of file */
	OSAtomicWrite(&gsCwq.hCompletedSeqNum, 0);
	OSAtomicWrite(&gsCwq.hCommonSeqNum, 0);
	OSAtomicWrite(&gsCwq.hDeferredSize, 0);
	OSAtomicWrite(&gsCwq.hWriteCounter, 0);
	OSAtomicWrite(&gsCwq.hReadCounter, 0);

	for (idx = 0; idx < CACHEOP_INDICES_MAX; idx++)
	{
		gsCwq.asWorkItems[idx].iTimeline = PVRSRV_NO_TIMELINE;
		gsCwq.asWorkItems[idx].psPMR = (void *)(uintptr_t)~0;
		gsCwq.asWorkItems[idx].ui32OpSeqNum = (IMG_UINT32)~0;
		gsCwq.asWorkItems[idx].ui32GFSeqNum = (IMG_UINT32)~0;
	}

	/* Lock prevents multiple threads from issuing surplus to requirement GF */
	eError = OSLockCreate((POS_LOCK*)&gsCwq.hGlobalFlushLock);
	PVR_LOGG_IF_ERROR(eError, "OSLockCreate", e0);

#if defined(CACHEOP_DEBUG)
	/* debugfs file read-out is not concurrent, so lock protects against this */
	eError = OSLockCreate((POS_LOCK*)&gsCwq.hStatsExecLock);
	PVR_LOGG_IF_ERROR(eError, "OSLockCreate", e0);

	gsCwq.i32StatsExecWriteIdx = 0;
	OSCachedMemSet(gsCwq.asStatsExecuted, 0, sizeof(gsCwq.asStatsExecuted));

	/* File captures the most recent subset of CacheOp(s) executed */
	gsCwq.pvStatsEntry = OSCreateStatisticEntry("cacheop_history",
												NULL,
												CacheOpStatsExecLogRead,
												NULL);
	PVR_LOGG_IF_ERROR(eError, "OSCreateStatisticEntry", e0);
#endif

e0:
	return eError;
}

void CacheOpDeInit (void)
{
#if defined(CACHEOP_DEBUG)
	if (gsCwq.hStatsExecLock)
	{
		(void) OSLockDestroy(gsCwq.hStatsExecLock);
		gsCwq.hStatsExecLock = NULL;
	}

	if (gsCwq.pvStatsEntry)
	{
		OSRemoveStatisticEntry(&gsCwq.pvStatsEntry);
	}
#endif
	if (gsCwq.hGlobalFlushLock)
	{
		(void) OSLockDestroy(gsCwq.hGlobalFlushLock);
		gsCwq.hGlobalFlushLock = NULL;
	}
}